tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(plugins): 方舟 Agent Plan - 合并 PR #527 图片视频插件并阻止图片误用 asset://

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ddCat
2026-09-18 10:41:19 +08:00
parent f3875357a8 60bad8cad4
commit da1560aa56
49 changed files with 3263 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
web/src/components/video-settings-panel.tsx
View File

@@ -3,6 +3,7 @@ import { Switch } from "@/components/ui/base/switch";
import { ImageSettingsTheme } from "@/components/image-settings-panel";
import { boolConfig, isSeedanceFastModel, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceRatioOptions } from "@/lib/seedance-video";
import { isVolcengineArkVideoProtocol } from "@/lib/model-protocols";
import { type CanvasTheme } from "@/lib/canvas-theme";
import { formatVideoResolutionLabel, isVideoResolutionMatch, normalizeVideoDuration, videoDimensionsForRatioAndResolution, videoResolutionComparisonKey, VIDEO_DURATION_MIN } from "@/lib/video-generation-options";
import { modelCapabilityConfigFor, resolveVideoRatioValue, resolveVideoResolutionValue, videoDurationOptions, type VideoCapabilityConfig } from "@/lib/model-capabilities";
@@ -115,7 +116,7 @@ function SeedanceVideoSettingsPanel({ config, profile, priceTiers, onConfigChang
const generateAudio = boolConfig(config.videoGenerateAudio, profile.generateAudio.default);
const watermark = boolConfig(config.videoWatermark, profile.watermark.default);
const useArkPrivateAssets = boolConfig(config.videoArkPrivateAssetUpload, true);
const isArkSeedance = resolveModelRequestConfig(config, config.model).interfaceType === "volcengine-ark-video";
const isArkSeedance = isVolcengineArkVideoProtocol(resolveModelRequestConfig(config, config.model).interfaceType);
return (
<ImageSettingsTheme theme={theme}>

8
web/src/lib/model-capabilities.ts
View File

@@ -216,7 +216,7 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
image.responseFormat = { supported: true };
image.outputFormat = { supported: false };
image.maxOutputs = 1;
} else if (protocol === "volcengine-ark-image") {
} else if (protocol === "volcengine-ark-image" || protocol === "volcengine-ark-agent-plan-image") {
image.references.maskSupported = false;
image.quality.supported = false;
image.transparentBackground.supported = false;
@@ -320,7 +320,7 @@ export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = "
video.duration = { selection: "enum", values: [4, 6, 8], default: 6 };
video.resolutions = ["720p", "1080p"];
}
if (protocol === "volcengine-ark-video" || protocol === "newapi-channel-1" || protocol === "newapi-channel-2") {
if (protocol === "volcengine-ark-video" || protocol === "volcengine-ark-agent-plan-video" || protocol === "newapi-channel-1" || protocol === "newapi-channel-2") {
video.references.maxVideos = 3;
video.references.maxAudios = 3;
video.references.maxVideoBytes = 200 * 1024 * 1024;
@@ -329,8 +329,8 @@ export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = "
video.references.maxAudioDurationSeconds = 15;
video.generateAudio = { supported: true, default: true };
}
if (protocol === "volcengine-ark-video" || protocol === "newapi-channel-1") video.resolutions = ["480p", "720p", "1080p"];
if (protocol === "volcengine-ark-video") {
if (protocol === "volcengine-ark-video" || protocol === "volcengine-ark-agent-plan-video" || protocol === "newapi-channel-1") video.resolutions = ["480p", "720p", "1080p"];
if (protocol === "volcengine-ark-video" || protocol === "volcengine-ark-agent-plan-video") {
video.watermark = { supported: true, default: false };
video.operations.push("reference_to_video", "audio_to_video");
}

10
web/src/lib/model-protocols.ts
View File

@@ -9,8 +9,16 @@ export function protocolGroups(protocols: ModelProtocolDefinition[]) {
export function modelProtocolDefinition(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { return definitions.find((item) => item.value === value); }
export function modelProtocolLabel(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { return modelProtocolDefinition(value, definitions)?.label || (value ? value : "未安装协议"); }
export function modelProtocolCapability(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { return modelProtocolDefinition(value, definitions)?.capability; }
export function isVolcengineArkImageProtocol(protocol?: string) {
return protocol === "volcengine-ark-image" || protocol === "volcengine-ark-agent-plan-image";
}
export function isVolcengineArkVideoProtocol(protocol?: string) {
return protocol === "volcengine-ark-video" || protocol === "volcengine-ark-agent-plan-video";
}
export function modelProtocolSupportsTokenBilling(capability?: string, protocol?: string) {
return capability === "text" || (capability === "video" && protocol === "volcengine-ark-video");
return capability === "text" || (capability === "video" && isVolcengineArkVideoProtocol(protocol));
}
export function protocolForModelCatalog(_endpointTypes: string[] = []): ModelProtocol | undefined {

4
web/src/lib/plugins/builtin/prompt-optimizer.ts
View File

@@ -136,7 +136,7 @@ function resolveModelAdaptationProfile(input: PromptOptimizationInput): ModelAda
avoid: ["不要添加 SD 权重、Midjourney 参数或模型未确认支持的控制语法。"],
};
}
if (containsAny(model, ["seedream", "jimeng", "doubao", "volcengine-ark-image", "volcengine-jimeng-image"])) {
if (containsAny(model, ["seedream", "jimeng", "doubao", "volcengine-ark-image", "volcengine-ark-agent-plan-image", "volcengine-jimeng-image"])) {
return {
id: "seedream-image",
label: "Seedream / 即梦 / 火山图片模型",
@@ -184,7 +184,7 @@ function resolveModelAdaptationProfile(input: PromptOptimizationInput): ModelAda
return genericImageProfile;
}
if (containsAny(model, ["seedance", "doubao-seedance", "volcengine-ark-video", "volcengine-jimeng-video", "jimeng-video"])) {
if (containsAny(model, ["seedance", "doubao-seedance", "volcengine-ark-video", "volcengine-ark-agent-plan-video", "volcengine-jimeng-video", "jimeng-video"])) {
return {
id: "seedance-video",
label: "Seedance / 即梦视频模型",

7
web/src/lib/seedance-video.ts
View File

@@ -59,7 +59,12 @@ const seedancePixels = {
export function isSeedanceVideoConfig(config: AiConfig | Pick<AiConfig, "model" | "videoModel" | "baseUrl">) {
const requestConfig = "channels" in config ? resolveModelRequestConfig(config, config.model || config.videoModel) : config;
return isSeedanceVideoModel(modelOptionName(requestConfig.model || requestConfig.videoModel)) || isArkPlanBaseUrl(requestConfig.baseUrl);
// Agent Plan 图片与视频共用 /api/plan/v3；按协议排除图片，避免 Seedream 误判为 Seedance。
if ("interfaceType" in requestConfig) {
const interfaceType = requestConfig.interfaceType;
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-agent-plan-image") return false;
}
return isSeedanceVideoModel(modelOptionName(requestConfig.model || requestConfig.videoModel)) || isArkPlanBaseUrl(requestConfig.baseUrl || "");
}
export function isSeedanceVideoModel(model: string) {

13
web/src/services/api/generation-task.ts
View File

@@ -95,7 +95,7 @@ export async function runBackendGenerationTask(
throwIfAborted(signal);
assertClientPromptLimit(mode, prompt, config, metadata);
assertBackendRuntimeConfigured(config, mode);
const prepared = await prepareGenerationReferences({ config, referenceImages, referenceVideos, referenceAudios, mask });
const prepared = await prepareGenerationReferences({ config, mode, referenceImages, referenceVideos, referenceAudios, mask });
throwIfAborted(signal);
return createAndWaitGenerationTask({ projectId, mode, prompt, config, referenceImages, referenceVideos, referenceAudios, textHistory, signal, metadata, onTaskUpdate, onTextDelta, streamText, enableThinking, clientOperationId, retryOf, attemptGroupId }, prepared, dependencies);
}
@@ -233,12 +233,14 @@ function assertClientPromptLimit(mode: BackendGenerationMode, prompt: string, co
async function prepareGenerationReferences({
config,
mode,
referenceImages = [],
referenceVideos = [],
referenceAudios = [],
mask,
}: Pick<BackendGenerationTaskOptions, "config" | "referenceImages" | "referenceVideos" | "referenceAudios" | "mask">): Promise<PreparedGenerationReferences> {
const preferArkAssetUrl = usesArkVideoAssetReference(config);
}: Pick<BackendGenerationTaskOptions, "config" | "mode" | "referenceImages" | "referenceVideos" | "referenceAudios" | "mask">): Promise<PreparedGenerationReferences> {
// asset:// 仅视频生成可用；Agent Plan Seedream 与 Seedance 共用 /api/plan/v3，不能按 BaseURL 误判。
const preferArkAssetUrl = mode === "video" && usesArkVideoAssetReference(config);
const preparedImages = await Promise.all(referenceImages.map((image) => prepareBackendImageReference(image, preferArkAssetUrl)));
const preparedVideos = await Promise.all(referenceVideos.map(prepareBackendMediaReference));
const preparedAudios = await Promise.all(referenceAudios.map(prepareBackendMediaReference));
@@ -249,7 +251,10 @@ async function prepareGenerationReferences({
// 与后端 isArkPrivateAssetVideoConfig 对齐：方舟视频渠道允许参考图直接 asset:// 引用，
// 跳过可信素材上传同步（适合已录入方舟素材 ID 的素材，例如被授权的真人像素材）。
function usesArkVideoAssetReference(config: AiConfig) {
return resolveModelRequestConfig(config, config.model).interfaceType === "volcengine-ark-video" || isArkPlanBaseUrl(config.baseUrl || "");
const interfaceType = resolveModelRequestConfig(config, config.model).interfaceType;
if (interfaceType === "volcengine-ark-video" || interfaceType === "volcengine-ark-agent-plan-video") return true;
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-agent-plan-image") return false;
return isArkPlanBaseUrl(config.baseUrl || "");
}
async function createAndWaitGenerationTask(options: BackendGenerationTaskOptions, prepared: PreparedGenerationReferences, dependencies: GenerationTaskDependencies) {

5
web/src/services/api/image.ts
View File

@@ -6,6 +6,7 @@ import type { ReferenceImage } from "@/types/image";
import { withOpenAIPromptCacheKey } from "@/lib/openai-prompt-cache";
import { modelCapabilityConfigFor, normalizeImageValue } from "@/lib/model-capabilities";
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl, type GeminiImageGenerationConfig } from "@/lib/gemini-image";
import { isVolcengineArkImageProtocol } from "@/lib/model-protocols";
import { aiApiUrl, aiHeaders, imageChannelTransport, postChannelJSON, postGeminiJSON, postVolcengineArkImage } from "@/services/api/image-transport";
const IMAGE_OUTPUT_FORMAT = "png";
@@ -94,7 +95,7 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
}
const quality = imageProfile.quality.supported && normalizedImage.quality !== "auto" ? normalizeQuality(normalizedImage.quality) || normalizedImage.quality : undefined;
const requestSize = resolveImageRequestSize(imageProfile, quality, normalizedImage.size);
const isVolcengineArk = requestConfig.interfaceType === "volcengine-ark-image";
const isVolcengineArk = isVolcengineArkImageProtocol(requestConfig.interfaceType);
const normalizedRequestSize = requestSize?.parameter === "size" && isVolcengineArk ? { ...requestSize, value: normalizeVolcengineArkImageSize(requestSize.value)! } : requestSize;
try {
const payload = isVolcengineArk
@@ -174,7 +175,7 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
throw new Error(readAxiosError(error, "Grok 图片编辑失败"));
}
}
if (requestConfig.interfaceType === "volcengine-ark-image") {
if (isVolcengineArkImageProtocol(requestConfig.interfaceType)) {
if (mask) throw new Error("火山方舟图片协议不支持蒙版编辑，请移除蒙版后重试");
const quality = imageProfile.quality.supported && normalizedImage.quality !== "auto" ? normalizeQuality(normalizedImage.quality) || normalizedImage.quality : undefined;
const sizeRequest = resolveImageRequestSize(imageProfile, quality, normalizedImage.size);

11
web/src/services/api/video-provider-seedance.ts
View File

@@ -1,4 +1,5 @@
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
import { isVolcengineArkVideoProtocol } from "@/lib/model-protocols";
import { boolConfig, buildSeedancePromptText, isArkPlanBaseUrl, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { getResourceOSSUrl } from "@/services/api/resources";
import { getMediaBlob } from "@/services/file-storage";
@@ -13,13 +14,13 @@ import type { VideoProviderDeps } from "./video-provider-deps";
import { hasExplicitVideoFrames, resolveVideoImageReferences } from "./video-reference-roles";
export function isSeedanceConfig(config: ResolvedAiConfig) {
return config.interfaceType === "volcengine-ark-video" || isSeedanceVideoConfig(config);
return isVolcengineArkVideoProtocol(config.interfaceType) || isSeedanceVideoConfig(config);
}
export async function createSeedanceTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
assertSeedanceVideoReferences(videoReferences);
assertSeedanceAudioReferences(audioReferences);
const isVolcengineArk = config.interfaceType === "volcengine-ark-video";
const isVolcengineArk = isVolcengineArkVideoProtocol(config.interfaceType);
const payload = isVolcengineArk || isArkPlanBaseUrl(config.baseUrl)
? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options)
: await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options);
@@ -77,15 +78,15 @@ function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) {
}
function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) {
if (config.interfaceType === "volcengine-ark-video" || isArkPlanBaseUrl(config.baseUrl)) return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
if (isVolcengineArkVideoProtocol(config.interfaceType) || isArkPlanBaseUrl(config.baseUrl)) return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
return buildApiUrl(config.baseUrl, `/videos${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
}
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) {
if (audioReferences.length && !references.length && !videoReferences.length) {
throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
throw new Error(isVolcengineArkVideoProtocol(config.interfaceType) ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
}
const content = config.interfaceType === "volcengine-ark-video"
const content = isVolcengineArkVideoProtocol(config.interfaceType)
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences, options)
: await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences, deps, options);
if (!content.length) throw new Error("请输入视频提示词，或连接参考图片/视频/音频");

1
web/src/stores/use-config-store.ts
View File

@@ -912,6 +912,7 @@ export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) {
export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfaceType) {
if (interfaceType === "gemini-veo" || interfaceType === "gemini-image") return GEMINI_BASE_URL;
if (interfaceType === "novita-video") return "https://api.novita.ai/v3";
if (interfaceType === "volcengine-ark-agent-plan-image" || interfaceType === "volcengine-ark-agent-plan-video") return "https://ark.cn-beijing.volces.com/api/plan/v3";
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-video") return "https://ark.cn-beijing.volces.com/api/v3";
if (interfaceType === "volcengine-jimeng-image" || interfaceType === "volcengine-jimeng-video") return "https://visual.volcengineapi.com";
if (interfaceType === "minimax-video") return "https://api.minimaxi.com";