diff --git a/backend/internal/protocol/autodl_indextts2_test.go b/backend/internal/protocol/autodl_indextts2_test.go new file mode 100644 index 000000000..8a92092c6 --- /dev/null +++ b/backend/internal/protocol/autodl_indextts2_test.go @@ -0,0 +1,95 @@ +package protocol + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// IndexTTS2 的音色完全来自 prompt_simple 参考音频,缺失时上游返回“缺少必填参数”, +// 因此插件必须把画布连线的参考音频映射过去,并在本地拦截未连接音频的请求。 +func TestAutoDLIndexTTS2Workflow(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "..", "plugin-packages", "autodl-comfyui.yingce-plugin")) + if err != nil { + t.Fatal(err) + } + pkg, err := ParsePluginPackage(data) + if err != nil { + t.Fatal(err) + } + adapters, err := LoadInstalledProviders(pkg.ManifestRaw, nil) + if err != nil { + t.Fatal(err) + } + var adapter Adapter + for _, a := range adapters { + if a.Metadata().ID == "autodl-comfyui-audio" { + adapter = a + } + } + if adapter == nil { + t.Fatal("autodl-comfyui-audio provider adapter not found") + } + if !adapter.Metadata().RequiresPublicMediaURLs { + t.Fatal("IndexTTS2 reference audio must be sent as public URLs") + } + + if _, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ + Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "你好,这是一段测试文本", + }}); err == nil || !strings.Contains(err.Error(), "音色参考音频") { + t.Fatalf("missing prompt_simple must be rejected locally, err = %v", err) + } + + create, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ + Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "你好,这是一段测试文本", + Audios: []MediaReference{{URL: "https://cdn.example/voice.mp3", Order: 0}}, + Extra: map[string]any{"audioVoice": "alloy"}, + }}) + if err != nil { + t.Fatal(err) + } + if create.Method != "POST" || create.Path != "/api/v1/comfyui/comfyui_workflow/indextts2-v1" { + t.Fatalf("create spec = %#v", create) + } + body := create.Body.(map[string]any) + if body["prompt_text"] != "你好,这是一段测试文本" || body["prompt_simple"] != "https://cdn.example/voice.mp3" || body["emo_control_method"] != "与音色参考音频相同" || body["emo_random"] != false { + t.Fatalf("single reference body = %#v", body) + } + for _, name := range []string{"emo_happy", "emo_angry", "emo_sad", "emo_afraid", "emo_disgusted", "emo_melancholic", "emo_surprised", "emo_calm"} { + if value, ok := body[name]; !ok || manifestFloat(value) != 0 { + t.Fatalf("%s = %#v, body = %#v", name, value, body) + } + } + for _, name := range []string{"emo_ref_audio", "voice", "prompt"} { + if _, exists := body[name]; exists { + t.Fatalf("single reference body must not send %s: %#v", name, body) + } + } + + withEmotion, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ + Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "测试", + Audios: []MediaReference{{URL: "https://cdn.example/emotion.mp3", Order: 2}, {URL: "https://cdn.example/voice.mp3", Order: 1}}, + ProviderOptions: map[string]map[string]any{"autodl-comfyui-audio": {"emo_happy": 0.5, "emo_calm": 0.3}}, + }}) + if err != nil { + t.Fatal(err) + } + emotionBody := withEmotion.Body.(map[string]any) + if emotionBody["prompt_simple"] != "https://cdn.example/voice.mp3" || emotionBody["emo_ref_audio"] != "https://cdn.example/emotion.mp3" || emotionBody["emo_control_method"] != "使用情感参考音频" || manifestFloat(emotionBody["emo_happy"]) != 0.5 || manifestFloat(emotionBody["emo_calm"]) != 0.3 { + t.Fatalf("emotion reference body = %#v", emotionBody) + } + + if _, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ + Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "测试", + Audios: []MediaReference{{URL: "https://cdn.example/a.mp3"}, {URL: "https://cdn.example/b.mp3", Order: 1}, {URL: "https://cdn.example/c.mp3", Order: 2}}, + }}); err == nil { + t.Fatal("more than two reference audios must be rejected") + } + + completed, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: "671ce5ca"}, []byte(`{"msg":"","code":"Success","data":{"status":"completed","results":[{"url":"https://cos.example/ComfyUI_00010_.wav","type":"audio","node_id":"9","file_type":"wav","output_type":"output"}],"task_id":"671ce5ca"}}`)) + if err != nil || completed.Status != StatusSucceeded || completed.Result == nil || len(completed.Result.Audios) != 1 || completed.Result.Audios[0].URL != "https://cos.example/ComfyUI_00010_.wav" { + t.Fatalf("completed = %#v, err = %v", completed, err) + } +} diff --git a/plugin-packages/autodl-comfyui.yingce-plugin b/plugin-packages/autodl-comfyui.yingce-plugin index 8af8ce60d..a163beb68 100644 Binary files a/plugin-packages/autodl-comfyui.yingce-plugin and b/plugin-packages/autodl-comfyui.yingce-plugin differ diff --git a/web/src/components/audio-settings-panel.tsx b/web/src/components/audio-settings-panel.tsx index 22ee09864..7262614c7 100644 --- a/web/src/components/audio-settings-panel.tsx +++ b/web/src/components/audio-settings-panel.tsx @@ -7,7 +7,7 @@ import type { AiConfig } from "@/stores/use-config-store"; const speedOptions = ["0.75", "1", "1.25", "1.5"]; -type AudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions"; +type AudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions" | "audioEmotionControlMethod" | "audioEmotionRandom" | "audioEmotionHappy" | "audioEmotionAngry" | "audioEmotionSad" | "audioEmotionAfraid" | "audioEmotionDisgusted" | "audioEmotionMelancholic" | "audioEmotionSurprised" | "audioEmotionCalm"; type AudioSettingsPanelProps = { config: AiConfig; @@ -75,11 +75,46 @@ export function AudioSettingsPanel({ config, onConfigChange, theme, showTitle = onMouseDown={(event) => event.stopPropagation()} /> + + onConfigChange("audioEmotionControlMethod", event.target.value)} + onMouseDown={(event) => event.stopPropagation()} + > + 与音色参考音频相同 + 使用情感参考音频 + + + 随机情感 + onConfigChange("audioEmotionRandom", String(event.target.checked))} /> + + + {emotionFields.map(([key, label]) => ( + + {label} + onConfigChange(key, event.target.value)} onBlur={(event) => onConfigChange(key, normalizeEmotionWeight(event.target.value))} onMouseDown={(event) => event.stopPropagation()} /> + + ))} + + ); } +const emotionFields: Array<[Extract, string]> = [ + ["audioEmotionHappy", "快乐"], ["audioEmotionAngry", "愤怒"], ["audioEmotionSad", "悲伤"], ["audioEmotionAfraid", "害怕"], + ["audioEmotionDisgusted", "厌恶"], ["audioEmotionMelancholic", "忧郁"], ["audioEmotionSurprised", "惊讶"], ["audioEmotionCalm", "平静"], +]; + +function normalizeEmotionWeight(value: string) { + const number = Number(value); + if (!Number.isFinite(number)) return "0"; + return String(Math.max(0, Math.min(1, number))); +} + function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) { return ( ; context: Pick; @@ -45,6 +55,16 @@ export function canvasGenerationRequestFingerprint(input: CanvasGenerationReques workflow: input.workflow, operation: input.operation, audioInstructions: input.audioInstructions, + audioEmotionControlMethod: input.audioEmotionControlMethod, + audioEmotionRandom: input.audioEmotionRandom, + audioEmotionHappy: input.audioEmotionHappy, + audioEmotionAngry: input.audioEmotionAngry, + audioEmotionSad: input.audioEmotionSad, + audioEmotionAfraid: input.audioEmotionAfraid, + audioEmotionDisgusted: input.audioEmotionDisgusted, + audioEmotionMelancholic: input.audioEmotionMelancholic, + audioEmotionSurprised: input.audioEmotionSurprised, + audioEmotionCalm: input.audioEmotionCalm, promptTemplateOperation: input.promptTemplateOperation, promptTemplateVariables: input.promptTemplateVariables, references: { diff --git a/web/src/lib/canvas/canvas-project-generation.ts b/web/src/lib/canvas/canvas-project-generation.ts index 79f01b3fb..4dd5cd338 100644 --- a/web/src/lib/canvas/canvas-project-generation.ts +++ b/web/src/lib/canvas/canvas-project-generation.ts @@ -242,6 +242,16 @@ export function buildAudioGenerationMetadata(config: AiConfig): CanvasNodeMetada audioFormat: config.audioFormat, audioSpeed: config.audioSpeed, audioInstructions: config.audioInstructions, + audioEmotionControlMethod: config.audioEmotionControlMethod, + audioEmotionRandom: config.audioEmotionRandom, + audioEmotionHappy: config.audioEmotionHappy, + audioEmotionAngry: config.audioEmotionAngry, + audioEmotionSad: config.audioEmotionSad, + audioEmotionAfraid: config.audioEmotionAfraid, + audioEmotionDisgusted: config.audioEmotionDisgusted, + audioEmotionMelancholic: config.audioEmotionMelancholic, + audioEmotionSurprised: config.audioEmotionSurprised, + audioEmotionCalm: config.audioEmotionCalm, }; } diff --git a/web/src/lib/model-selection.ts b/web/src/lib/model-selection.ts index 4fa9b4edf..0aebd4b09 100644 --- a/web/src/lib/model-selection.ts +++ b/web/src/lib/model-selection.ts @@ -134,7 +134,7 @@ export function modelRequestOptions(config: AiConfig, capability: ModelCapabilit case "video": return { size: config.size, videoSeconds: Number(config.videoSeconds), vquality: config.vquality, videoGenerateAudio: config.videoGenerateAudio === "true", videoWatermark: config.videoWatermark === "true" }; case "audio": - return { audioVoice: config.audioVoice, audioFormat: config.audioFormat, audioSpeed: Number(config.audioSpeed) }; + return { audioVoice: config.audioVoice, audioFormat: config.audioFormat, audioSpeed: Number(config.audioSpeed), audioEmotionControlMethod: config.audioEmotionControlMethod, audioEmotionRandom: config.audioEmotionRandom === "true", audioEmotionHappy: Number(config.audioEmotionHappy), audioEmotionAngry: Number(config.audioEmotionAngry), audioEmotionSad: Number(config.audioEmotionSad), audioEmotionAfraid: Number(config.audioEmotionAfraid), audioEmotionDisgusted: Number(config.audioEmotionDisgusted), audioEmotionMelancholic: Number(config.audioEmotionMelancholic), audioEmotionSurprised: Number(config.audioEmotionSurprised), audioEmotionCalm: Number(config.audioEmotionCalm) }; default: return {}; } diff --git a/web/src/pages/canvas/use-canvas-generation-executor.ts b/web/src/pages/canvas/use-canvas-generation-executor.ts index aad16fb1c..774b4e895 100644 --- a/web/src/pages/canvas/use-canvas-generation-executor.ts +++ b/web/src/pages/canvas/use-canvas-generation-executor.ts @@ -236,6 +236,16 @@ export function useCanvasGenerationExecutor({ : undefined, operation: sourceNode?.metadata?.videoEditOperation, audioInstructions: generationConfig.audioInstructions, + audioEmotionControlMethod: generationConfig.audioEmotionControlMethod, + audioEmotionRandom: generationConfig.audioEmotionRandom, + audioEmotionHappy: generationConfig.audioEmotionHappy, + audioEmotionAngry: generationConfig.audioEmotionAngry, + audioEmotionSad: generationConfig.audioEmotionSad, + audioEmotionAfraid: generationConfig.audioEmotionAfraid, + audioEmotionDisgusted: generationConfig.audioEmotionDisgusted, + audioEmotionMelancholic: generationConfig.audioEmotionMelancholic, + audioEmotionSurprised: generationConfig.audioEmotionSurprised, + audioEmotionCalm: generationConfig.audioEmotionCalm, promptTemplateOperation: sourceNode?.metadata?.promptTemplateOperation, promptTemplateVariables: sourceNode?.metadata?.promptTemplateVariables, context: generationContext, diff --git a/web/src/services/api/generation-task.ts b/web/src/services/api/generation-task.ts index b1962c7be..d714a755b 100644 --- a/web/src/services/api/generation-task.ts +++ b/web/src/services/api/generation-task.ts @@ -22,6 +22,7 @@ export type BackendGenerationResult = { images?: Array<{ dataUrl: string; storageKey?: string; width?: number; height?: number; bytes?: number; mimeType?: string }>; video?: { dataUrl: string; storageKey?: string; width?: number; height?: number; durationMs?: number; bytes?: number; mimeType?: string }; audio?: { dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string }; + audios?: Array<{ dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string }>; text?: string; toolCalls?: Array<{ id: string; type: "function"; function: { name: string; arguments: string }; thoughtSignature?: string }>; reasoning?: string; @@ -305,7 +306,7 @@ function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepa referenceVideos: prepared.referenceVideos, referenceAudios: prepared.referenceAudios, mask: prepared.mask, - metadata: generationMetadata(config, { + metadata: generationMetadata(config, mode, { ...metadata, ...(options.clientOperationId ? { clientOperationId: options.clientOperationId } : {}), ...(options.retryOf ? { retryOf: options.retryOf } : {}), @@ -315,20 +316,33 @@ function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepa }; } -function generationMetadata(config: AiConfig, metadata?: Record) { +function generationMetadata(config: AiConfig, mode: BackendGenerationMode, metadata?: Record) { const channel = resolveModelChannel(config, config.model); const model = modelOptionName(config.model); const modelCost = channel.modelCosts?.find((item) => item.model === model); const protocol = modelCost?.protocol || channel.interfaceType; - const defaults = modelCost?.defaultOptions; - if (!protocol || !defaults || !Object.keys(defaults).length) return metadata; + if (!protocol) return metadata; const existing = metadata?.providerOptions && typeof metadata.providerOptions === "object" && !Array.isArray(metadata.providerOptions) ? metadata.providerOptions as Record : {}; const namespace = existing[protocol] && typeof existing[protocol] === "object" && !Array.isArray(existing[protocol]) ? existing[protocol] as Record : {}; - return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...namespace } } }; + const defaults = modelCost?.defaultOptions && typeof modelCost.defaultOptions === "object" ? modelCost.defaultOptions : {}; + const audioOptions = mode === "audio" ? { + emo_control_method: config.audioEmotionControlMethod || "与音色参考音频相同", + emo_random: config.audioEmotionRandom === "true", + emo_happy: Number(config.audioEmotionHappy || 0), + emo_angry: Number(config.audioEmotionAngry || 0), + emo_sad: Number(config.audioEmotionSad || 0), + emo_afraid: Number(config.audioEmotionAfraid || 0), + emo_disgusted: Number(config.audioEmotionDisgusted || 0), + emo_melancholic: Number(config.audioEmotionMelancholic || 0), + emo_surprised: Number(config.audioEmotionSurprised || 0), + emo_calm: Number(config.audioEmotionCalm || 0), + } : {}; + if (!Object.keys(defaults).length && !Object.keys(audioOptions).length && !Object.keys(namespace).length) return metadata; + return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...audioOptions, ...namespace } } }; } async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAudio) { diff --git a/web/src/stores/use-config-store.ts b/web/src/stores/use-config-store.ts index 754c314ef..bde99e61a 100644 --- a/web/src/stores/use-config-store.ts +++ b/web/src/stores/use-config-store.ts @@ -130,6 +130,16 @@ export type AiConfig = { audioFormat: string; audioSpeed: string; audioInstructions: string; + audioEmotionControlMethod: string; + audioEmotionRandom: string; + audioEmotionHappy: string; + audioEmotionAngry: string; + audioEmotionSad: string; + audioEmotionAfraid: string; + audioEmotionDisgusted: string; + audioEmotionMelancholic: string; + audioEmotionSurprised: string; + audioEmotionCalm: string; videoSeconds: string; vquality: string; videoGenerateAudio: string; @@ -172,6 +182,16 @@ export const defaultConfig: AiConfig = { audioFormat: "mp3", audioSpeed: "1", audioInstructions: "", + audioEmotionControlMethod: "与音色参考音频相同", + audioEmotionRandom: "false", + audioEmotionHappy: "0", + audioEmotionAngry: "0", + audioEmotionSad: "0", + audioEmotionAfraid: "0", + audioEmotionDisgusted: "0", + audioEmotionMelancholic: "0", + audioEmotionSurprised: "0", + audioEmotionCalm: "0", videoSeconds: "6", vquality: "720", videoGenerateAudio: "true", @@ -419,6 +439,16 @@ export function normalizeConfigSnapshot(snapshot: ConfigStoreSnapshot | undefine audioFormat: config.audioFormat || defaultConfig.audioFormat, audioSpeed: config.audioSpeed || defaultConfig.audioSpeed, audioInstructions: config.audioInstructions || "", + audioEmotionControlMethod: config.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod, + audioEmotionRandom: config.audioEmotionRandom || defaultConfig.audioEmotionRandom, + audioEmotionHappy: config.audioEmotionHappy || defaultConfig.audioEmotionHappy, + audioEmotionAngry: config.audioEmotionAngry || defaultConfig.audioEmotionAngry, + audioEmotionSad: config.audioEmotionSad || defaultConfig.audioEmotionSad, + audioEmotionAfraid: config.audioEmotionAfraid || defaultConfig.audioEmotionAfraid, + audioEmotionDisgusted: config.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted, + audioEmotionMelancholic: config.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic, + audioEmotionSurprised: config.audioEmotionSurprised || defaultConfig.audioEmotionSurprised, + audioEmotionCalm: config.audioEmotionCalm || defaultConfig.audioEmotionCalm, // 旧版全局 systemPrompt 会跨任务污染请求;提示词定制现已按 operation 由服务端编译。 systemPrompt: "", videoSeconds: normalizeVideoDuration(config.videoSeconds), diff --git a/web/src/types/canvas.ts b/web/src/types/canvas.ts index afbe8c48d..a184e164f 100644 --- a/web/src/types/canvas.ts +++ b/web/src/types/canvas.ts @@ -296,6 +296,16 @@ export type CanvasNodeMetadata = { audioFormat?: string; audioSpeed?: string; audioInstructions?: string; + audioEmotionControlMethod?: string; + audioEmotionRandom?: string; + audioEmotionHappy?: string; + audioEmotionAngry?: string; + audioEmotionSad?: string; + audioEmotionAfraid?: string; + audioEmotionDisgusted?: string; + audioEmotionMelancholic?: string; + audioEmotionSurprised?: string; + audioEmotionCalm?: string; references?: string[]; naturalWidth?: number; naturalHeight?: number;