tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(media): 音频生成 - 支持 IndexTTS2 情感控制参数

Browse Source
This commit is contained in:
ddcat666
2026-09-30 14:38:09 +08:00
parent 123c7cd285
commit b080674524
13 changed files with 266 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
backend/internal/protocol/autodl_indextts2_test.go Normal file
View File

@@ -0,0 +1,95 @@
package protocol
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
)
// IndexTTS2 的音色完全来自 prompt_simple 参考音频，缺失时上游返回“缺少必填参数”，
// 因此插件必须把画布连线的参考音频映射过去，并在本地拦截未连接音频的请求。
func TestAutoDLIndexTTS2Workflow(t *testing.T) {
data, err := os.ReadFile(filepath.Join("..", "..", "..", "plugin-packages", "autodl-comfyui.yingce-plugin"))
if err != nil {
t.Fatal(err)
}
pkg, err := ParsePluginPackage(data)
if err != nil {
t.Fatal(err)
}
adapters, err := LoadInstalledProviders(pkg.ManifestRaw, nil)
if err != nil {
t.Fatal(err)
}
var adapter Adapter
for _, a := range adapters {
if a.Metadata().ID == "autodl-comfyui-audio" {
adapter = a
}
}
if adapter == nil {
t.Fatal("autodl-comfyui-audio provider adapter not found")
}
if !adapter.Metadata().RequiresPublicMediaURLs {
t.Fatal("IndexTTS2 reference audio must be sent as public URLs")
}
if _, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{
Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "你好，这是一段测试文本",
}}); err == nil || !strings.Contains(err.Error(), "音色参考音频") {
t.Fatalf("missing prompt_simple must be rejected locally, err = %v", err)
}
create, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{
Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "你好，这是一段测试文本",
Audios: []MediaReference{{URL: "https://cdn.example/voice.mp3", Order: 0}},
Extra: map[string]any{"audioVoice": "alloy"},
}})
if err != nil {
t.Fatal(err)
}
if create.Method != "POST" || create.Path != "/api/v1/comfyui/comfyui_workflow/indextts2-v1" {
t.Fatalf("create spec = %#v", create)
}
body := create.Body.(map[string]any)
if body["prompt_text"] != "你好，这是一段测试文本" || body["prompt_simple"] != "https://cdn.example/voice.mp3" || body["emo_control_method"] != "与音色参考音频相同" || body["emo_random"] != false {
t.Fatalf("single reference body = %#v", body)
}
for _, name := range []string{"emo_happy", "emo_angry", "emo_sad", "emo_afraid", "emo_disgusted", "emo_melancholic", "emo_surprised", "emo_calm"} {
if value, ok := body[name]; !ok || manifestFloat(value) != 0 {
t.Fatalf("%s = %#v, body = %#v", name, value, body)
}
}
for _, name := range []string{"emo_ref_audio", "voice", "prompt"} {
if _, exists := body[name]; exists {
t.Fatalf("single reference body must not send %s: %#v", name, body)
}
}
withEmotion, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{
Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "测试",
Audios: []MediaReference{{URL: "https://cdn.example/emotion.mp3", Order: 2}, {URL: "https://cdn.example/voice.mp3", Order: 1}},
ProviderOptions: map[string]map[string]any{"autodl-comfyui-audio": {"emo_happy": 0.5, "emo_calm": 0.3}},
}})
if err != nil {
t.Fatal(err)
}
emotionBody := withEmotion.Body.(map[string]any)
if emotionBody["prompt_simple"] != "https://cdn.example/voice.mp3" || emotionBody["emo_ref_audio"] != "https://cdn.example/emotion.mp3" || emotionBody["emo_control_method"] != "使用情感参考音频" || manifestFloat(emotionBody["emo_happy"]) != 0.5 || manifestFloat(emotionBody["emo_calm"]) != 0.3 {
t.Fatalf("emotion reference body = %#v", emotionBody)
}
if _, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{
Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "测试",
Audios: []MediaReference{{URL: "https://cdn.example/a.mp3"}, {URL: "https://cdn.example/b.mp3", Order: 1}, {URL: "https://cdn.example/c.mp3", Order: 2}},
}}); err == nil {
t.Fatal("more than two reference audios must be rejected")
}
completed, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: "671ce5ca"}, []byte(`{"msg":"","code":"Success","data":{"status":"completed","results":[{"url":"https://cos.example/ComfyUI_00010_.wav","type":"audio","node_id":"9","file_type":"wav","output_type":"output"}],"task_id":"671ce5ca"}}`))
if err != nil || completed.Status != StatusSucceeded || completed.Result == nil || len(completed.Result.Audios) != 1 || completed.Result.Audios[0].URL != "https://cos.example/ComfyUI_00010_.wav" {
t.Fatalf("completed = %#v, err = %v", completed, err)
}
}

BIN
plugin-packages/autodl-comfyui.yingce-plugin
View File

Binary file not shown.

37
web/src/components/audio-settings-panel.tsx
View File

@@ -7,7 +7,7 @@ import type { AiConfig } from "@/stores/use-config-store";
const speedOptions = ["0.75", "1", "1.25", "1.5"];
type AudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions";
type AudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions" | "audioEmotionControlMethod" | "audioEmotionRandom" | "audioEmotionHappy" | "audioEmotionAngry" | "audioEmotionSad" | "audioEmotionAfraid" | "audioEmotionDisgusted" | "audioEmotionMelancholic" | "audioEmotionSurprised" | "audioEmotionCalm";
type AudioSettingsPanelProps = {
config: AiConfig;
@@ -75,11 +75,46 @@ export function AudioSettingsPanel({ config, onConfigChange, theme, showTitle =
onMouseDown={(event) => event.stopPropagation()}
/>
</SettingGroup>
<SettingGroup title="情感控制（IndexTTS2）" color={theme.node.muted}>
<select
className="h-9 w-full rounded-xl border bg-transparent px-3 text-sm outline-none"
style={{ borderColor: theme.node.stroke, color: theme.node.text, background: theme.spatial.elevated }}
value={config.audioEmotionControlMethod || "与音色参考音频相同"}
onChange={(event) => onConfigChange("audioEmotionControlMethod", event.target.value)}
onMouseDown={(event) => event.stopPropagation()}
>
<option value="与音色参考音频相同">与音色参考音频相同</option>
<option value="使用情感参考音频">使用情感参考音频</option>
</select>
<label className="flex items-center justify-between gap-3 text-sm">
<span>随机情感</span>
<input type="checkbox" checked={config.audioEmotionRandom === "true"} onChange={(event) => onConfigChange("audioEmotionRandom", String(event.target.checked))} />
</label>
<div className="grid grid-cols-2 gap-2">
{emotionFields.map(([key, label]) => (
<label key={key} className="flex items-center gap-2 text-xs">
<span className="min-w-0 flex-1">{label}</span>
<input type="number" min={0} max={1} step={0.01} className="h-8 w-20 rounded-lg border bg-transparent px-2 text-right outline-none" style={{ borderColor: theme.node.stroke, color: theme.node.text }} value={config[key] || "0"} onChange={(event) => onConfigChange(key, event.target.value)} onBlur={(event) => onConfigChange(key, normalizeEmotionWeight(event.target.value))} onMouseDown={(event) => event.stopPropagation()} />
</label>
))}
</div>
</SettingGroup>
</div>
</ImageSettingsTheme>
);
}
const emotionFields: Array<[Extract<AudioSettingKey, `audioEmotion${string}`>, string]> = [
["audioEmotionHappy", "快乐"], ["audioEmotionAngry", "愤怒"], ["audioEmotionSad", "悲伤"], ["audioEmotionAfraid", "害怕"],
["audioEmotionDisgusted", "厌恶"], ["audioEmotionMelancholic", "忧郁"], ["audioEmotionSurprised", "惊讶"], ["audioEmotionCalm", "平静"],
];
function normalizeEmotionWeight(value: string) {
const number = Number(value);
if (!Number.isFinite(number)) return "0";
return String(Math.max(0, Math.min(1, number)));
}
function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) {
return (
<button

2
web/src/components/canvas/canvas-audio-settings-popover.tsx
View File

@@ -9,7 +9,7 @@ import { canvasThemes } from "@/lib/canvas-theme";
import { useActiveTheme } from "@/stores/canvas/use-canvas-theme-store";
import type { AiConfig } from "@/stores/use-config-store";
export type CanvasAudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions";
export type CanvasAudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions" | "audioEmotionControlMethod" | "audioEmotionRandom" | "audioEmotionHappy" | "audioEmotionAngry" | "audioEmotionSad" | "audioEmotionAfraid" | "audioEmotionDisgusted" | "audioEmotionMelancholic" | "audioEmotionSurprised" | "audioEmotionCalm";
type CanvasAudioSettingsPopoverProps = {
config: AiConfig;

23
web/src/components/canvas/canvas-config-node-panel.tsx
View File

@@ -501,6 +501,16 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can
audioFormat: node.metadata?.audioFormat || globalConfig.audioFormat || defaultConfig.audioFormat,
audioSpeed: node.metadata?.audioSpeed || globalConfig.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node.metadata?.audioInstructions || globalConfig.audioInstructions || defaultConfig.audioInstructions,
audioEmotionControlMethod: node.metadata?.audioEmotionControlMethod || globalConfig.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: node.metadata?.audioEmotionRandom || globalConfig.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: node.metadata?.audioEmotionHappy || globalConfig.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: node.metadata?.audioEmotionAngry || globalConfig.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: node.metadata?.audioEmotionSad || globalConfig.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: node.metadata?.audioEmotionAfraid || globalConfig.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: node.metadata?.audioEmotionDisgusted || globalConfig.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: node.metadata?.audioEmotionMelancholic || globalConfig.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: node.metadata?.audioEmotionSurprised || globalConfig.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: node.metadata?.audioEmotionCalm || globalConfig.audioEmotionCalm || defaultConfig.audioEmotionCalm,
count: normalizedImage?.count || String(node.metadata?.count || (mode === "image" ? globalConfig.canvasImageCount || globalConfig.count : globalConfig.count) || defaultConfig.count),
};
}
@@ -557,6 +567,16 @@ function buildModelNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode
audioFormat: node.metadata?.audioFormat || globalConfig.audioFormat || defaultConfig.audioFormat,
audioSpeed: node.metadata?.audioSpeed || globalConfig.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node.metadata?.audioInstructions || globalConfig.audioInstructions || defaultConfig.audioInstructions,
audioEmotionControlMethod: node.metadata?.audioEmotionControlMethod || globalConfig.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: node.metadata?.audioEmotionRandom || globalConfig.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: node.metadata?.audioEmotionHappy || globalConfig.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: node.metadata?.audioEmotionAngry || globalConfig.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: node.metadata?.audioEmotionSad || globalConfig.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: node.metadata?.audioEmotionAfraid || globalConfig.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: node.metadata?.audioEmotionDisgusted || globalConfig.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: node.metadata?.audioEmotionMelancholic || globalConfig.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: node.metadata?.audioEmotionSurprised || globalConfig.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: node.metadata?.audioEmotionCalm || globalConfig.audioEmotionCalm || defaultConfig.audioEmotionCalm,
count: generationDefaults.count || String(node.metadata?.count || (mode === "image" ? globalConfig.canvasImageCount || globalConfig.count : globalConfig.count) || defaultConfig.count),
};
}
@@ -573,5 +593,6 @@ function audioConfigPatch(key: CanvasAudioSettingKey, value: string) {
if (key === "audioVoice") return { audioVoice: value };
if (key === "audioFormat") return { audioFormat: value };
if (key === "audioSpeed") return { audioSpeed: value };
return { audioInstructions: value };
if (key === "audioInstructions") return { audioInstructions: value };
return { [key]: value };
}

13
web/src/components/canvas/canvas-node-prompt-config.ts
View File

@@ -71,6 +71,16 @@ export function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mo
audioFormat: node.metadata?.audioFormat || globalConfig.audioFormat || defaultConfig.audioFormat,
audioSpeed: node.metadata?.audioSpeed || globalConfig.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node.metadata?.audioInstructions || globalConfig.audioInstructions || defaultConfig.audioInstructions,
audioEmotionControlMethod: node.metadata?.audioEmotionControlMethod || globalConfig.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: node.metadata?.audioEmotionRandom || globalConfig.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: node.metadata?.audioEmotionHappy || globalConfig.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: node.metadata?.audioEmotionAngry || globalConfig.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: node.metadata?.audioEmotionSad || globalConfig.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: node.metadata?.audioEmotionAfraid || globalConfig.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: node.metadata?.audioEmotionDisgusted || globalConfig.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: node.metadata?.audioEmotionMelancholic || globalConfig.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: node.metadata?.audioEmotionSurprised || globalConfig.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: node.metadata?.audioEmotionCalm || globalConfig.audioEmotionCalm || defaultConfig.audioEmotionCalm,
count: defaults.count ?? String(node.metadata?.count || (mode === "image" ? globalConfig.canvasImageCount || globalConfig.count : globalConfig.count) || defaultConfig.count),
};
}
@@ -94,5 +104,6 @@ export function audioConfigPatch(key: CanvasAudioSettingKey, value: string) {
if (key === "audioVoice") return { audioVoice: value };
if (key === "audioFormat") return { audioFormat: value };
if (key === "audioSpeed") return { audioSpeed: value };
return { audioInstructions: value };
if (key === "audioInstructions") return { audioInstructions: value };
return { [key]: value };
}

20
web/src/lib/canvas/canvas-generation-submission.ts
View File

@@ -20,6 +20,16 @@ export type CanvasGenerationRequestFingerprintInput = {
};
operation?: string;
audioInstructions?: string;
audioEmotionControlMethod?: string;
audioEmotionRandom?: string;
audioEmotionHappy?: string;
audioEmotionAngry?: string;
audioEmotionSad?: string;
audioEmotionAfraid?: string;
audioEmotionDisgusted?: string;
audioEmotionMelancholic?: string;
audioEmotionSurprised?: string;
audioEmotionCalm?: string;
promptTemplateOperation?: string;
promptTemplateVariables?: Record<string, string>;
context: Pick<NodeGenerationContext, "referenceImages" | "referenceVideos" | "referenceAudios" | "characterReferences" | "resolvedCharacterVersions" | "resolvedCharacterVoices">;
@@ -45,6 +55,16 @@ export function canvasGenerationRequestFingerprint(input: CanvasGenerationReques
workflow: input.workflow,
operation: input.operation,
audioInstructions: input.audioInstructions,
audioEmotionControlMethod: input.audioEmotionControlMethod,
audioEmotionRandom: input.audioEmotionRandom,
audioEmotionHappy: input.audioEmotionHappy,
audioEmotionAngry: input.audioEmotionAngry,
audioEmotionSad: input.audioEmotionSad,
audioEmotionAfraid: input.audioEmotionAfraid,
audioEmotionDisgusted: input.audioEmotionDisgusted,
audioEmotionMelancholic: input.audioEmotionMelancholic,
audioEmotionSurprised: input.audioEmotionSurprised,
audioEmotionCalm: input.audioEmotionCalm,
promptTemplateOperation: input.promptTemplateOperation,
promptTemplateVariables: input.promptTemplateVariables,
references: {

10
web/src/lib/canvas/canvas-project-generation.ts
View File

@@ -242,6 +242,16 @@ export function buildAudioGenerationMetadata(config: AiConfig): CanvasNodeMetada
audioFormat: config.audioFormat,
audioSpeed: config.audioSpeed,
audioInstructions: config.audioInstructions,
audioEmotionControlMethod: config.audioEmotionControlMethod,
audioEmotionRandom: config.audioEmotionRandom,
audioEmotionHappy: config.audioEmotionHappy,
audioEmotionAngry: config.audioEmotionAngry,
audioEmotionSad: config.audioEmotionSad,
audioEmotionAfraid: config.audioEmotionAfraid,
audioEmotionDisgusted: config.audioEmotionDisgusted,
audioEmotionMelancholic: config.audioEmotionMelancholic,
audioEmotionSurprised: config.audioEmotionSurprised,
audioEmotionCalm: config.audioEmotionCalm,
};
}

2
web/src/lib/model-selection.ts
View File

@@ -134,7 +134,7 @@ export function modelRequestOptions(config: AiConfig, capability: ModelCapabilit
case "video":
return { size: config.size, videoSeconds: Number(config.videoSeconds), vquality: config.vquality, videoGenerateAudio: config.videoGenerateAudio === "true", videoWatermark: config.videoWatermark === "true" };
case "audio":
return { audioVoice: config.audioVoice, audioFormat: config.audioFormat, audioSpeed: Number(config.audioSpeed) };
return { audioVoice: config.audioVoice, audioFormat: config.audioFormat, audioSpeed: Number(config.audioSpeed), audioEmotionControlMethod: config.audioEmotionControlMethod, audioEmotionRandom: config.audioEmotionRandom === "true", audioEmotionHappy: Number(config.audioEmotionHappy), audioEmotionAngry: Number(config.audioEmotionAngry), audioEmotionSad: Number(config.audioEmotionSad), audioEmotionAfraid: Number(config.audioEmotionAfraid), audioEmotionDisgusted: Number(config.audioEmotionDisgusted), audioEmotionMelancholic: Number(config.audioEmotionMelancholic), audioEmotionSurprised: Number(config.audioEmotionSurprised), audioEmotionCalm: Number(config.audioEmotionCalm) };
default:
return {};
}

10
web/src/pages/canvas/use-canvas-generation-executor.ts
View File

@@ -236,6 +236,16 @@ export function useCanvasGenerationExecutor({
: undefined,
operation: sourceNode?.metadata?.videoEditOperation,
audioInstructions: generationConfig.audioInstructions,
audioEmotionControlMethod: generationConfig.audioEmotionControlMethod,
audioEmotionRandom: generationConfig.audioEmotionRandom,
audioEmotionHappy: generationConfig.audioEmotionHappy,
audioEmotionAngry: generationConfig.audioEmotionAngry,
audioEmotionSad: generationConfig.audioEmotionSad,
audioEmotionAfraid: generationConfig.audioEmotionAfraid,
audioEmotionDisgusted: generationConfig.audioEmotionDisgusted,
audioEmotionMelancholic: generationConfig.audioEmotionMelancholic,
audioEmotionSurprised: generationConfig.audioEmotionSurprised,
audioEmotionCalm: generationConfig.audioEmotionCalm,
promptTemplateOperation: sourceNode?.metadata?.promptTemplateOperation,
promptTemplateVariables: sourceNode?.metadata?.promptTemplateVariables,
context: generationContext,

24
web/src/services/api/generation-task.ts
View File

@@ -22,6 +22,7 @@ export type BackendGenerationResult = {
images?: Array<{ dataUrl: string; storageKey?: string; width?: number; height?: number; bytes?: number; mimeType?: string }>;
video?: { dataUrl: string; storageKey?: string; width?: number; height?: number; durationMs?: number; bytes?: number; mimeType?: string };
audio?: { dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string };
audios?: Array<{ dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string }>;
text?: string;
toolCalls?: Array<{ id: string; type: "function"; function: { name: string; arguments: string }; thoughtSignature?: string }>;
reasoning?: string;
@@ -305,7 +306,7 @@ function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepa
referenceVideos: prepared.referenceVideos,
referenceAudios: prepared.referenceAudios,
mask: prepared.mask,
metadata: generationMetadata(config, {
metadata: generationMetadata(config, mode, {
...metadata,
...(options.clientOperationId ? { clientOperationId: options.clientOperationId } : {}),
...(options.retryOf ? { retryOf: options.retryOf } : {}),
@@ -315,20 +316,33 @@ function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepa
};
}
function generationMetadata(config: AiConfig, metadata?: Record<string, unknown>) {
function generationMetadata(config: AiConfig, mode: BackendGenerationMode, metadata?: Record<string, unknown>) {
const channel = resolveModelChannel(config, config.model);
const model = modelOptionName(config.model);
const modelCost = channel.modelCosts?.find((item) => item.model === model);
const protocol = modelCost?.protocol || channel.interfaceType;
const defaults = modelCost?.defaultOptions;
if (!protocol || !defaults || !Object.keys(defaults).length) return metadata;
if (!protocol) return metadata;
const existing = metadata?.providerOptions && typeof metadata.providerOptions === "object" && !Array.isArray(metadata.providerOptions)
? metadata.providerOptions as Record<string, unknown>
: {};
const namespace = existing[protocol] && typeof existing[protocol] === "object" && !Array.isArray(existing[protocol])
? existing[protocol] as Record<string, unknown>
: {};
return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...namespace } } };
const defaults = modelCost?.defaultOptions && typeof modelCost.defaultOptions === "object" ? modelCost.defaultOptions : {};
const audioOptions = mode === "audio" ? {
emo_control_method: config.audioEmotionControlMethod || "与音色参考音频相同",
emo_random: config.audioEmotionRandom === "true",
emo_happy: Number(config.audioEmotionHappy || 0),
emo_angry: Number(config.audioEmotionAngry || 0),
emo_sad: Number(config.audioEmotionSad || 0),
emo_afraid: Number(config.audioEmotionAfraid || 0),
emo_disgusted: Number(config.audioEmotionDisgusted || 0),
emo_melancholic: Number(config.audioEmotionMelancholic || 0),
emo_surprised: Number(config.audioEmotionSurprised || 0),
emo_calm: Number(config.audioEmotionCalm || 0),
} : {};
if (!Object.keys(defaults).length && !Object.keys(audioOptions).length && !Object.keys(namespace).length) return metadata;
return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...audioOptions, ...namespace } } };
}
async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAudio) {

30
web/src/stores/use-config-store.ts
View File

@@ -130,6 +130,16 @@ export type AiConfig = {
audioFormat: string;
audioSpeed: string;
audioInstructions: string;
audioEmotionControlMethod: string;
audioEmotionRandom: string;
audioEmotionHappy: string;
audioEmotionAngry: string;
audioEmotionSad: string;
audioEmotionAfraid: string;
audioEmotionDisgusted: string;
audioEmotionMelancholic: string;
audioEmotionSurprised: string;
audioEmotionCalm: string;
videoSeconds: string;
vquality: string;
videoGenerateAudio: string;
@@ -172,6 +182,16 @@ export const defaultConfig: AiConfig = {
audioFormat: "mp3",
audioSpeed: "1",
audioInstructions: "",
audioEmotionControlMethod: "与音色参考音频相同",
audioEmotionRandom: "false",
audioEmotionHappy: "0",
audioEmotionAngry: "0",
audioEmotionSad: "0",
audioEmotionAfraid: "0",
audioEmotionDisgusted: "0",
audioEmotionMelancholic: "0",
audioEmotionSurprised: "0",
audioEmotionCalm: "0",
videoSeconds: "6",
vquality: "720",
videoGenerateAudio: "true",
@@ -419,6 +439,16 @@ export function normalizeConfigSnapshot(snapshot: ConfigStoreSnapshot | undefine
audioFormat: config.audioFormat || defaultConfig.audioFormat,
audioSpeed: config.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: config.audioInstructions || "",
audioEmotionControlMethod: config.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: config.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: config.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: config.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: config.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: config.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: config.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: config.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: config.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: config.audioEmotionCalm || defaultConfig.audioEmotionCalm,
// 旧版全局 systemPrompt 会跨任务污染请求；提示词定制现已按 operation 由服务端编译。
systemPrompt: "",
videoSeconds: normalizeVideoDuration(config.videoSeconds),

10
web/src/types/canvas.ts
View File

@@ -296,6 +296,16 @@ export type CanvasNodeMetadata = {
audioFormat?: string;
audioSpeed?: string;
audioInstructions?: string;
audioEmotionControlMethod?: string;
audioEmotionRandom?: string;
audioEmotionHappy?: string;
audioEmotionAngry?: string;
audioEmotionSad?: string;
audioEmotionAfraid?: string;
audioEmotionDisgusted?: string;
audioEmotionMelancholic?: string;
audioEmotionSurprised?: string;
audioEmotionCalm?: string;
references?: string[];
naturalWidth?: number;
naturalHeight?: number;