tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(media): 音频生成 - 支持 IndexTTS2 情感控制参数

Browse Source
This commit is contained in:
ddcat666
2026-09-30 14:38:09 +08:00
parent 123c7cd285
commit b080674524
13 changed files with 266 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
web/src/components/audio-settings-panel.tsx
View File

@@ -7,7 +7,7 @@ import type { AiConfig } from "@/stores/use-config-store";
const speedOptions = ["0.75", "1", "1.25", "1.5"];
type AudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions";
type AudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions" | "audioEmotionControlMethod" | "audioEmotionRandom" | "audioEmotionHappy" | "audioEmotionAngry" | "audioEmotionSad" | "audioEmotionAfraid" | "audioEmotionDisgusted" | "audioEmotionMelancholic" | "audioEmotionSurprised" | "audioEmotionCalm";
type AudioSettingsPanelProps = {
config: AiConfig;
@@ -75,11 +75,46 @@ export function AudioSettingsPanel({ config, onConfigChange, theme, showTitle =
onMouseDown={(event) => event.stopPropagation()}
/>
</SettingGroup>
<SettingGroup title="情感控制（IndexTTS2）" color={theme.node.muted}>
<select
className="h-9 w-full rounded-xl border bg-transparent px-3 text-sm outline-none"
style={{ borderColor: theme.node.stroke, color: theme.node.text, background: theme.spatial.elevated }}
value={config.audioEmotionControlMethod || "与音色参考音频相同"}
onChange={(event) => onConfigChange("audioEmotionControlMethod", event.target.value)}
onMouseDown={(event) => event.stopPropagation()}
>
<option value="与音色参考音频相同">与音色参考音频相同</option>
<option value="使用情感参考音频">使用情感参考音频</option>
</select>
<label className="flex items-center justify-between gap-3 text-sm">
<span>随机情感</span>
<input type="checkbox" checked={config.audioEmotionRandom === "true"} onChange={(event) => onConfigChange("audioEmotionRandom", String(event.target.checked))} />
</label>
<div className="grid grid-cols-2 gap-2">
{emotionFields.map(([key, label]) => (
<label key={key} className="flex items-center gap-2 text-xs">
<span className="min-w-0 flex-1">{label}</span>
<input type="number" min={0} max={1} step={0.01} className="h-8 w-20 rounded-lg border bg-transparent px-2 text-right outline-none" style={{ borderColor: theme.node.stroke, color: theme.node.text }} value={config[key] || "0"} onChange={(event) => onConfigChange(key, event.target.value)} onBlur={(event) => onConfigChange(key, normalizeEmotionWeight(event.target.value))} onMouseDown={(event) => event.stopPropagation()} />
</label>
))}
</div>
</SettingGroup>
</div>
</ImageSettingsTheme>
);
}
const emotionFields: Array<[Extract<AudioSettingKey, `audioEmotion${string}`>, string]> = [
["audioEmotionHappy", "快乐"], ["audioEmotionAngry", "愤怒"], ["audioEmotionSad", "悲伤"], ["audioEmotionAfraid", "害怕"],
["audioEmotionDisgusted", "厌恶"], ["audioEmotionMelancholic", "忧郁"], ["audioEmotionSurprised", "惊讶"], ["audioEmotionCalm", "平静"],
];
function normalizeEmotionWeight(value: string) {
const number = Number(value);
if (!Number.isFinite(number)) return "0";
return String(Math.max(0, Math.min(1, number)));
}
function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) {
return (
<button

2
web/src/components/canvas/canvas-audio-settings-popover.tsx
View File

@@ -9,7 +9,7 @@ import { canvasThemes } from "@/lib/canvas-theme";
import { useActiveTheme } from "@/stores/canvas/use-canvas-theme-store";
import type { AiConfig } from "@/stores/use-config-store";
export type CanvasAudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions";
export type CanvasAudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions" | "audioEmotionControlMethod" | "audioEmotionRandom" | "audioEmotionHappy" | "audioEmotionAngry" | "audioEmotionSad" | "audioEmotionAfraid" | "audioEmotionDisgusted" | "audioEmotionMelancholic" | "audioEmotionSurprised" | "audioEmotionCalm";
type CanvasAudioSettingsPopoverProps = {
config: AiConfig;

23
web/src/components/canvas/canvas-config-node-panel.tsx
View File

@@ -501,6 +501,16 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can
audioFormat: node.metadata?.audioFormat || globalConfig.audioFormat || defaultConfig.audioFormat,
audioSpeed: node.metadata?.audioSpeed || globalConfig.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node.metadata?.audioInstructions || globalConfig.audioInstructions || defaultConfig.audioInstructions,
audioEmotionControlMethod: node.metadata?.audioEmotionControlMethod || globalConfig.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: node.metadata?.audioEmotionRandom || globalConfig.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: node.metadata?.audioEmotionHappy || globalConfig.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: node.metadata?.audioEmotionAngry || globalConfig.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: node.metadata?.audioEmotionSad || globalConfig.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: node.metadata?.audioEmotionAfraid || globalConfig.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: node.metadata?.audioEmotionDisgusted || globalConfig.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: node.metadata?.audioEmotionMelancholic || globalConfig.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: node.metadata?.audioEmotionSurprised || globalConfig.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: node.metadata?.audioEmotionCalm || globalConfig.audioEmotionCalm || defaultConfig.audioEmotionCalm,
count: normalizedImage?.count || String(node.metadata?.count || (mode === "image" ? globalConfig.canvasImageCount || globalConfig.count : globalConfig.count) || defaultConfig.count),
};
}
@@ -557,6 +567,16 @@ function buildModelNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode
audioFormat: node.metadata?.audioFormat || globalConfig.audioFormat || defaultConfig.audioFormat,
audioSpeed: node.metadata?.audioSpeed || globalConfig.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node.metadata?.audioInstructions || globalConfig.audioInstructions || defaultConfig.audioInstructions,
audioEmotionControlMethod: node.metadata?.audioEmotionControlMethod || globalConfig.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: node.metadata?.audioEmotionRandom || globalConfig.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: node.metadata?.audioEmotionHappy || globalConfig.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: node.metadata?.audioEmotionAngry || globalConfig.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: node.metadata?.audioEmotionSad || globalConfig.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: node.metadata?.audioEmotionAfraid || globalConfig.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: node.metadata?.audioEmotionDisgusted || globalConfig.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: node.metadata?.audioEmotionMelancholic || globalConfig.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: node.metadata?.audioEmotionSurprised || globalConfig.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: node.metadata?.audioEmotionCalm || globalConfig.audioEmotionCalm || defaultConfig.audioEmotionCalm,
count: generationDefaults.count || String(node.metadata?.count || (mode === "image" ? globalConfig.canvasImageCount || globalConfig.count : globalConfig.count) || defaultConfig.count),
};
}
@@ -573,5 +593,6 @@ function audioConfigPatch(key: CanvasAudioSettingKey, value: string) {
if (key === "audioVoice") return { audioVoice: value };
if (key === "audioFormat") return { audioFormat: value };
if (key === "audioSpeed") return { audioSpeed: value };
return { audioInstructions: value };
if (key === "audioInstructions") return { audioInstructions: value };
return { [key]: value };
}

13
web/src/components/canvas/canvas-node-prompt-config.ts
View File

@@ -71,6 +71,16 @@ export function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mo
audioFormat: node.metadata?.audioFormat || globalConfig.audioFormat || defaultConfig.audioFormat,
audioSpeed: node.metadata?.audioSpeed || globalConfig.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node.metadata?.audioInstructions || globalConfig.audioInstructions || defaultConfig.audioInstructions,
audioEmotionControlMethod: node.metadata?.audioEmotionControlMethod || globalConfig.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: node.metadata?.audioEmotionRandom || globalConfig.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: node.metadata?.audioEmotionHappy || globalConfig.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: node.metadata?.audioEmotionAngry || globalConfig.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: node.metadata?.audioEmotionSad || globalConfig.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: node.metadata?.audioEmotionAfraid || globalConfig.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: node.metadata?.audioEmotionDisgusted || globalConfig.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: node.metadata?.audioEmotionMelancholic || globalConfig.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: node.metadata?.audioEmotionSurprised || globalConfig.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: node.metadata?.audioEmotionCalm || globalConfig.audioEmotionCalm || defaultConfig.audioEmotionCalm,
count: defaults.count ?? String(node.metadata?.count || (mode === "image" ? globalConfig.canvasImageCount || globalConfig.count : globalConfig.count) || defaultConfig.count),
};
}
@@ -94,5 +104,6 @@ export function audioConfigPatch(key: CanvasAudioSettingKey, value: string) {
if (key === "audioVoice") return { audioVoice: value };
if (key === "audioFormat") return { audioFormat: value };
if (key === "audioSpeed") return { audioSpeed: value };
return { audioInstructions: value };
if (key === "audioInstructions") return { audioInstructions: value };
return { [key]: value };
}

20
web/src/lib/canvas/canvas-generation-submission.ts
View File

@@ -20,6 +20,16 @@ export type CanvasGenerationRequestFingerprintInput = {
};
operation?: string;
audioInstructions?: string;
audioEmotionControlMethod?: string;
audioEmotionRandom?: string;
audioEmotionHappy?: string;
audioEmotionAngry?: string;
audioEmotionSad?: string;
audioEmotionAfraid?: string;
audioEmotionDisgusted?: string;
audioEmotionMelancholic?: string;
audioEmotionSurprised?: string;
audioEmotionCalm?: string;
promptTemplateOperation?: string;
promptTemplateVariables?: Record<string, string>;
context: Pick<NodeGenerationContext, "referenceImages" | "referenceVideos" | "referenceAudios" | "characterReferences" | "resolvedCharacterVersions" | "resolvedCharacterVoices">;
@@ -45,6 +55,16 @@ export function canvasGenerationRequestFingerprint(input: CanvasGenerationReques
workflow: input.workflow,
operation: input.operation,
audioInstructions: input.audioInstructions,
audioEmotionControlMethod: input.audioEmotionControlMethod,
audioEmotionRandom: input.audioEmotionRandom,
audioEmotionHappy: input.audioEmotionHappy,
audioEmotionAngry: input.audioEmotionAngry,
audioEmotionSad: input.audioEmotionSad,
audioEmotionAfraid: input.audioEmotionAfraid,
audioEmotionDisgusted: input.audioEmotionDisgusted,
audioEmotionMelancholic: input.audioEmotionMelancholic,
audioEmotionSurprised: input.audioEmotionSurprised,
audioEmotionCalm: input.audioEmotionCalm,
promptTemplateOperation: input.promptTemplateOperation,
promptTemplateVariables: input.promptTemplateVariables,
references: {

10
web/src/lib/canvas/canvas-project-generation.ts
View File

@@ -242,6 +242,16 @@ export function buildAudioGenerationMetadata(config: AiConfig): CanvasNodeMetada
audioFormat: config.audioFormat,
audioSpeed: config.audioSpeed,
audioInstructions: config.audioInstructions,
audioEmotionControlMethod: config.audioEmotionControlMethod,
audioEmotionRandom: config.audioEmotionRandom,
audioEmotionHappy: config.audioEmotionHappy,
audioEmotionAngry: config.audioEmotionAngry,
audioEmotionSad: config.audioEmotionSad,
audioEmotionAfraid: config.audioEmotionAfraid,
audioEmotionDisgusted: config.audioEmotionDisgusted,
audioEmotionMelancholic: config.audioEmotionMelancholic,
audioEmotionSurprised: config.audioEmotionSurprised,
audioEmotionCalm: config.audioEmotionCalm,
};
}

2
web/src/lib/model-selection.ts
View File

@@ -134,7 +134,7 @@ export function modelRequestOptions(config: AiConfig, capability: ModelCapabilit
case "video":
return { size: config.size, videoSeconds: Number(config.videoSeconds), vquality: config.vquality, videoGenerateAudio: config.videoGenerateAudio === "true", videoWatermark: config.videoWatermark === "true" };
case "audio":
return { audioVoice: config.audioVoice, audioFormat: config.audioFormat, audioSpeed: Number(config.audioSpeed) };
return { audioVoice: config.audioVoice, audioFormat: config.audioFormat, audioSpeed: Number(config.audioSpeed), audioEmotionControlMethod: config.audioEmotionControlMethod, audioEmotionRandom: config.audioEmotionRandom === "true", audioEmotionHappy: Number(config.audioEmotionHappy), audioEmotionAngry: Number(config.audioEmotionAngry), audioEmotionSad: Number(config.audioEmotionSad), audioEmotionAfraid: Number(config.audioEmotionAfraid), audioEmotionDisgusted: Number(config.audioEmotionDisgusted), audioEmotionMelancholic: Number(config.audioEmotionMelancholic), audioEmotionSurprised: Number(config.audioEmotionSurprised), audioEmotionCalm: Number(config.audioEmotionCalm) };
default:
return {};
}

10
web/src/pages/canvas/use-canvas-generation-executor.ts
View File

@@ -236,6 +236,16 @@ export function useCanvasGenerationExecutor({
: undefined,
operation: sourceNode?.metadata?.videoEditOperation,
audioInstructions: generationConfig.audioInstructions,
audioEmotionControlMethod: generationConfig.audioEmotionControlMethod,
audioEmotionRandom: generationConfig.audioEmotionRandom,
audioEmotionHappy: generationConfig.audioEmotionHappy,
audioEmotionAngry: generationConfig.audioEmotionAngry,
audioEmotionSad: generationConfig.audioEmotionSad,
audioEmotionAfraid: generationConfig.audioEmotionAfraid,
audioEmotionDisgusted: generationConfig.audioEmotionDisgusted,
audioEmotionMelancholic: generationConfig.audioEmotionMelancholic,
audioEmotionSurprised: generationConfig.audioEmotionSurprised,
audioEmotionCalm: generationConfig.audioEmotionCalm,
promptTemplateOperation: sourceNode?.metadata?.promptTemplateOperation,
promptTemplateVariables: sourceNode?.metadata?.promptTemplateVariables,
context: generationContext,

24
web/src/services/api/generation-task.ts
View File

@@ -22,6 +22,7 @@ export type BackendGenerationResult = {
images?: Array<{ dataUrl: string; storageKey?: string; width?: number; height?: number; bytes?: number; mimeType?: string }>;
video?: { dataUrl: string; storageKey?: string; width?: number; height?: number; durationMs?: number; bytes?: number; mimeType?: string };
audio?: { dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string };
audios?: Array<{ dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string }>;
text?: string;
toolCalls?: Array<{ id: string; type: "function"; function: { name: string; arguments: string }; thoughtSignature?: string }>;
reasoning?: string;
@@ -305,7 +306,7 @@ function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepa
referenceVideos: prepared.referenceVideos,
referenceAudios: prepared.referenceAudios,
mask: prepared.mask,
metadata: generationMetadata(config, {
metadata: generationMetadata(config, mode, {
...metadata,
...(options.clientOperationId ? { clientOperationId: options.clientOperationId } : {}),
...(options.retryOf ? { retryOf: options.retryOf } : {}),
@@ -315,20 +316,33 @@ function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepa
};
}
function generationMetadata(config: AiConfig, metadata?: Record<string, unknown>) {
function generationMetadata(config: AiConfig, mode: BackendGenerationMode, metadata?: Record<string, unknown>) {
const channel = resolveModelChannel(config, config.model);
const model = modelOptionName(config.model);
const modelCost = channel.modelCosts?.find((item) => item.model === model);
const protocol = modelCost?.protocol || channel.interfaceType;
const defaults = modelCost?.defaultOptions;
if (!protocol || !defaults || !Object.keys(defaults).length) return metadata;
if (!protocol) return metadata;
const existing = metadata?.providerOptions && typeof metadata.providerOptions === "object" && !Array.isArray(metadata.providerOptions)
? metadata.providerOptions as Record<string, unknown>
: {};
const namespace = existing[protocol] && typeof existing[protocol] === "object" && !Array.isArray(existing[protocol])
? existing[protocol] as Record<string, unknown>
: {};
return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...namespace } } };
const defaults = modelCost?.defaultOptions && typeof modelCost.defaultOptions === "object" ? modelCost.defaultOptions : {};
const audioOptions = mode === "audio" ? {
emo_control_method: config.audioEmotionControlMethod || "与音色参考音频相同",
emo_random: config.audioEmotionRandom === "true",
emo_happy: Number(config.audioEmotionHappy || 0),
emo_angry: Number(config.audioEmotionAngry || 0),
emo_sad: Number(config.audioEmotionSad || 0),
emo_afraid: Number(config.audioEmotionAfraid || 0),
emo_disgusted: Number(config.audioEmotionDisgusted || 0),
emo_melancholic: Number(config.audioEmotionMelancholic || 0),
emo_surprised: Number(config.audioEmotionSurprised || 0),
emo_calm: Number(config.audioEmotionCalm || 0),
} : {};
if (!Object.keys(defaults).length && !Object.keys(audioOptions).length && !Object.keys(namespace).length) return metadata;
return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...audioOptions, ...namespace } } };
}
async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAudio) {

30
web/src/stores/use-config-store.ts
View File

@@ -130,6 +130,16 @@ export type AiConfig = {
audioFormat: string;
audioSpeed: string;
audioInstructions: string;
audioEmotionControlMethod: string;
audioEmotionRandom: string;
audioEmotionHappy: string;
audioEmotionAngry: string;
audioEmotionSad: string;
audioEmotionAfraid: string;
audioEmotionDisgusted: string;
audioEmotionMelancholic: string;
audioEmotionSurprised: string;
audioEmotionCalm: string;
videoSeconds: string;
vquality: string;
videoGenerateAudio: string;
@@ -172,6 +182,16 @@ export const defaultConfig: AiConfig = {
audioFormat: "mp3",
audioSpeed: "1",
audioInstructions: "",
audioEmotionControlMethod: "与音色参考音频相同",
audioEmotionRandom: "false",
audioEmotionHappy: "0",
audioEmotionAngry: "0",
audioEmotionSad: "0",
audioEmotionAfraid: "0",
audioEmotionDisgusted: "0",
audioEmotionMelancholic: "0",
audioEmotionSurprised: "0",
audioEmotionCalm: "0",
videoSeconds: "6",
vquality: "720",
videoGenerateAudio: "true",
@@ -419,6 +439,16 @@ export function normalizeConfigSnapshot(snapshot: ConfigStoreSnapshot | undefine
audioFormat: config.audioFormat || defaultConfig.audioFormat,
audioSpeed: config.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: config.audioInstructions || "",
audioEmotionControlMethod: config.audioEmotionControlMethod || defaultConfig.audioEmotionControlMethod,
audioEmotionRandom: config.audioEmotionRandom || defaultConfig.audioEmotionRandom,
audioEmotionHappy: config.audioEmotionHappy || defaultConfig.audioEmotionHappy,
audioEmotionAngry: config.audioEmotionAngry || defaultConfig.audioEmotionAngry,
audioEmotionSad: config.audioEmotionSad || defaultConfig.audioEmotionSad,
audioEmotionAfraid: config.audioEmotionAfraid || defaultConfig.audioEmotionAfraid,
audioEmotionDisgusted: config.audioEmotionDisgusted || defaultConfig.audioEmotionDisgusted,
audioEmotionMelancholic: config.audioEmotionMelancholic || defaultConfig.audioEmotionMelancholic,
audioEmotionSurprised: config.audioEmotionSurprised || defaultConfig.audioEmotionSurprised,
audioEmotionCalm: config.audioEmotionCalm || defaultConfig.audioEmotionCalm,
// 旧版全局 systemPrompt 会跨任务污染请求；提示词定制现已按 operation 由服务端编译。
systemPrompt: "",
videoSeconds: normalizeVideoDuration(config.videoSeconds),

10
web/src/types/canvas.ts
View File

@@ -296,6 +296,16 @@ export type CanvasNodeMetadata = {
audioFormat?: string;
audioSpeed?: string;
audioInstructions?: string;
audioEmotionControlMethod?: string;
audioEmotionRandom?: string;
audioEmotionHappy?: string;
audioEmotionAngry?: string;
audioEmotionSad?: string;
audioEmotionAfraid?: string;
audioEmotionDisgusted?: string;
audioEmotionMelancholic?: string;
audioEmotionSurprised?: string;
audioEmotionCalm?: string;
references?: string[];
naturalWidth?: number;
naturalHeight?: number;