tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(plugins): 方舟 Agent Plan - 合并 PR #527 图片视频插件并阻止图片误用 asset://

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ddCat
2026-09-18 10:41:19 +08:00
parent f3875357a8 60bad8cad4
commit da1560aa56
49 changed files with 3263 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
web/src/services/api/generation-task.ts
View File

@@ -95,7 +95,7 @@ export async function runBackendGenerationTask(
throwIfAborted(signal);
assertClientPromptLimit(mode, prompt, config, metadata);
assertBackendRuntimeConfigured(config, mode);
const prepared = await prepareGenerationReferences({ config, referenceImages, referenceVideos, referenceAudios, mask });
const prepared = await prepareGenerationReferences({ config, mode, referenceImages, referenceVideos, referenceAudios, mask });
throwIfAborted(signal);
return createAndWaitGenerationTask({ projectId, mode, prompt, config, referenceImages, referenceVideos, referenceAudios, textHistory, signal, metadata, onTaskUpdate, onTextDelta, streamText, enableThinking, clientOperationId, retryOf, attemptGroupId }, prepared, dependencies);
}
@@ -233,12 +233,14 @@ function assertClientPromptLimit(mode: BackendGenerationMode, prompt: string, co
async function prepareGenerationReferences({
config,
mode,
referenceImages = [],
referenceVideos = [],
referenceAudios = [],
mask,
}: Pick<BackendGenerationTaskOptions, "config" | "referenceImages" | "referenceVideos" | "referenceAudios" | "mask">): Promise<PreparedGenerationReferences> {
const preferArkAssetUrl = usesArkVideoAssetReference(config);
}: Pick<BackendGenerationTaskOptions, "config" | "mode" | "referenceImages" | "referenceVideos" | "referenceAudios" | "mask">): Promise<PreparedGenerationReferences> {
// asset:// 仅视频生成可用；Agent Plan Seedream 与 Seedance 共用 /api/plan/v3，不能按 BaseURL 误判。
const preferArkAssetUrl = mode === "video" && usesArkVideoAssetReference(config);
const preparedImages = await Promise.all(referenceImages.map((image) => prepareBackendImageReference(image, preferArkAssetUrl)));
const preparedVideos = await Promise.all(referenceVideos.map(prepareBackendMediaReference));
const preparedAudios = await Promise.all(referenceAudios.map(prepareBackendMediaReference));
@@ -249,7 +251,10 @@ async function prepareGenerationReferences({
// 与后端 isArkPrivateAssetVideoConfig 对齐：方舟视频渠道允许参考图直接 asset:// 引用，
// 跳过可信素材上传同步（适合已录入方舟素材 ID 的素材，例如被授权的真人像素材）。
function usesArkVideoAssetReference(config: AiConfig) {
return resolveModelRequestConfig(config, config.model).interfaceType === "volcengine-ark-video" || isArkPlanBaseUrl(config.baseUrl || "");
const interfaceType = resolveModelRequestConfig(config, config.model).interfaceType;
if (interfaceType === "volcengine-ark-video" || interfaceType === "volcengine-ark-agent-plan-video") return true;
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-agent-plan-image") return false;
return isArkPlanBaseUrl(config.baseUrl || "");
}
async function createAndWaitGenerationTask(options: BackendGenerationTaskOptions, prepared: PreparedGenerationReferences, dependencies: GenerationTaskDependencies) {

5
web/src/services/api/image.ts
View File

@@ -6,6 +6,7 @@ import type { ReferenceImage } from "@/types/image";
import { withOpenAIPromptCacheKey } from "@/lib/openai-prompt-cache";
import { modelCapabilityConfigFor, normalizeImageValue } from "@/lib/model-capabilities";
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl, type GeminiImageGenerationConfig } from "@/lib/gemini-image";
import { isVolcengineArkImageProtocol } from "@/lib/model-protocols";
import { aiApiUrl, aiHeaders, imageChannelTransport, postChannelJSON, postGeminiJSON, postVolcengineArkImage } from "@/services/api/image-transport";
const IMAGE_OUTPUT_FORMAT = "png";
@@ -94,7 +95,7 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
}
const quality = imageProfile.quality.supported && normalizedImage.quality !== "auto" ? normalizeQuality(normalizedImage.quality) || normalizedImage.quality : undefined;
const requestSize = resolveImageRequestSize(imageProfile, quality, normalizedImage.size);
const isVolcengineArk = requestConfig.interfaceType === "volcengine-ark-image";
const isVolcengineArk = isVolcengineArkImageProtocol(requestConfig.interfaceType);
const normalizedRequestSize = requestSize?.parameter === "size" && isVolcengineArk ? { ...requestSize, value: normalizeVolcengineArkImageSize(requestSize.value)! } : requestSize;
try {
const payload = isVolcengineArk
@@ -174,7 +175,7 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
throw new Error(readAxiosError(error, "Grok 图片编辑失败"));
}
}
if (requestConfig.interfaceType === "volcengine-ark-image") {
if (isVolcengineArkImageProtocol(requestConfig.interfaceType)) {
if (mask) throw new Error("火山方舟图片协议不支持蒙版编辑，请移除蒙版后重试");
const quality = imageProfile.quality.supported && normalizedImage.quality !== "auto" ? normalizeQuality(normalizedImage.quality) || normalizedImage.quality : undefined;
const sizeRequest = resolveImageRequestSize(imageProfile, quality, normalizedImage.size);

11
web/src/services/api/video-provider-seedance.ts
View File

@@ -1,4 +1,5 @@
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
import { isVolcengineArkVideoProtocol } from "@/lib/model-protocols";
import { boolConfig, buildSeedancePromptText, isArkPlanBaseUrl, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { getResourceOSSUrl } from "@/services/api/resources";
import { getMediaBlob } from "@/services/file-storage";
@@ -13,13 +14,13 @@ import type { VideoProviderDeps } from "./video-provider-deps";
import { hasExplicitVideoFrames, resolveVideoImageReferences } from "./video-reference-roles";
export function isSeedanceConfig(config: ResolvedAiConfig) {
return config.interfaceType === "volcengine-ark-video" || isSeedanceVideoConfig(config);
return isVolcengineArkVideoProtocol(config.interfaceType) || isSeedanceVideoConfig(config);
}
export async function createSeedanceTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
assertSeedanceVideoReferences(videoReferences);
assertSeedanceAudioReferences(audioReferences);
const isVolcengineArk = config.interfaceType === "volcengine-ark-video";
const isVolcengineArk = isVolcengineArkVideoProtocol(config.interfaceType);
const payload = isVolcengineArk || isArkPlanBaseUrl(config.baseUrl)
? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options)
: await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options);
@@ -77,15 +78,15 @@ function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) {
}
function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) {
if (config.interfaceType === "volcengine-ark-video" || isArkPlanBaseUrl(config.baseUrl)) return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
if (isVolcengineArkVideoProtocol(config.interfaceType) || isArkPlanBaseUrl(config.baseUrl)) return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
return buildApiUrl(config.baseUrl, `/videos${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
}
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) {
if (audioReferences.length && !references.length && !videoReferences.length) {
throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
throw new Error(isVolcengineArkVideoProtocol(config.interfaceType) ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
}
const content = config.interfaceType === "volcengine-ark-video"
const content = isVolcengineArkVideoProtocol(config.interfaceType)
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences, options)
: await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences, deps, options);
if (!content.length) throw new Error("请输入视频提示词，或连接参考图片/视频/音频");