tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(web): 角色与音频工作台 - 完善素材选择、生成参数与同步交互

Browse Source
This commit is contained in:
ddcat666
2026-10-01 09:43:02 +08:00
parent 5650da6d31
commit 22e7331d04
77 changed files with 3641 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
web/src/lib/asset-record.ts
View File

@@ -101,7 +101,13 @@ export function parseAssetRecord(value: unknown): Asset {
return { ...base, kind, data: parsed } satisfies ModelAsset;
}
if (kind === "entity") {
return { ...base, kind, data: { definition: requireRecord(data.definition, "definition") } } satisfies EntityAsset;
const entity: EntityAsset["data"] = { definition: requireRecord(data.definition, "definition") };
// 以下是列表接口补的只读展示字段，缺失或类型不对时直接忽略，不影响角色合同。
if (typeof data.version === "number") entity.version = data.version;
for (const key of ["coverStorageKey", "voiceName", "voiceSampleStorageKey", "visualStatus", "voiceStatus"] as const) {
if (typeof data[key] === "string" && data[key]) entity[key] = data[key];
}
return { ...base, kind, data: entity } satisfies EntityAsset;
}
throw new Error(`不支持的素材类型 ${kind}`);
}

113
web/src/lib/audio-generation.ts
View File

@@ -1,4 +1,10 @@
export const audioVoiceOptions = [
import { doubaoAudioVoiceOptions, doubaoDefaultAudioVoice } from "@/lib/doubao-audio-voices";
import { resolveModelRequestConfig } from "@/stores/use-config-store";
import type { AiConfig } from "@/stores/use-config-store";
export { doubaoAudioVoiceOptions, doubaoDefaultAudioVoice };
export const openAIAudioVoiceOptions = [
{ value: "alloy", label: "Alloy" },
{ value: "ash", label: "Ash" },
{ value: "ballad", label: "Ballad" },
@@ -14,7 +20,54 @@ export const audioVoiceOptions = [
{ value: "cedar", label: "Cedar" },
];
export const audioFormatOptions = [
export const doubaoAudioLanguageOptions = [
{ value: "", label: "自动识别" },
{ value: "zh-cn", label: "中文（普通话）" },
{ value: "en", label: "英语" },
{ value: "ja", label: "日语" },
{ value: "es-mx", label: "西班牙语（墨西哥）" },
{ value: "id", label: "印尼语" },
{ value: "pt-br", label: "葡萄牙语（巴西）" },
{ value: "pt", label: "葡萄牙语" },
{ value: "ko", label: "韩语" },
{ value: "it", label: "意大利语" },
{ value: "de", label: "德语" },
{ value: "fr", label: "法语" },
{ value: "th", label: "泰语" },
{ value: "vi", label: "越南语" },
{ value: "ru", label: "俄语" },
{ value: "fil", label: "菲律宾语" },
{ value: "ms", label: "马来语" },
{ value: "ar", label: "阿拉伯语" },
{ value: "pl", label: "波兰语" },
{ value: "tr", label: "土耳其语" },
{ value: "sv", label: "瑞典语" },
{ value: "nl", label: "荷兰语" },
{ value: "no", label: "挪威语" },
{ value: "uk", label: "乌克兰语" },
{ value: "fi", label: "芬兰语" },
{ value: "da", label: "丹麦语" },
{ value: "cs", label: "捷克语" },
{ value: "hu", label: "匈牙利语" },
{ value: "el", label: "希腊语" },
{ value: "ro", label: "罗马尼亚语" },
{ value: "hi", label: "印地语" },
{ value: "es-es", label: "西班牙语（西班牙）" },
];
export const doubaoAudioDialectOptions = [
{ value: "", label: "自动" },
{ value: "beijing", label: "北京话" },
{ value: "dongbei", label: "东北话" },
{ value: "henan", label: "河南话" },
{ value: "shaanxi", label: "陕西话" },
{ value: "shanghai", label: "上海话" },
{ value: "sichuan", label: "四川话" },
{ value: "tianjin", label: "天津话" },
{ value: "yue", label: "粤语" },
];
export const openAIAudioFormatOptions = [
{ value: "mp3", label: "MP3" },
{ value: "wav", label: "WAV" },
{ value: "opus", label: "Opus" },
@@ -23,12 +76,60 @@ export const audioFormatOptions = [
{ value: "pcm", label: "PCM" },
];
export const doubaoAudioFormatOptions = [
{ value: "mp3", label: "MP3" },
{ value: "wav", label: "WAV" },
{ value: "ogg_opus", label: "OGG Opus" },
{ value: "pcm", label: "PCM" },
];
export function isDoubaoAudioConfig(config: AiConfig) {
return resolveModelRequestConfig(config, config.model || config.audioModel).interfaceType === "doubao-streaming-tts";
}
export function audioVoiceOptionsForConfig(config: AiConfig) {
return isDoubaoAudioConfig(config) ? doubaoAudioVoiceOptions : openAIAudioVoiceOptions;
}
export function audioFormatOptionsForConfig(config: AiConfig) {
return isDoubaoAudioConfig(config) ? doubaoAudioFormatOptions : openAIAudioFormatOptions;
}
export function normalizeAudioVoiceForConfig(config: AiConfig, value: string) {
const options = audioVoiceOptionsForConfig(config);
const normalized = String(value || "").trim();
if (isDoubaoAudioConfig(config)) {
if (!normalized || normalized === "alloy") return "";
return normalized;
}
if (options.some((item) => item.value === normalized)) return normalized;
return normalized || "alloy";
}
export function normalizeAudioFormatForConfig(config: AiConfig, value: string) {
const options = audioFormatOptionsForConfig(config);
return options.some((item) => item.value === value) ? value : "mp3";
}
export function audioVoiceLabelForConfig(config: AiConfig, value: string) {
const normalized = normalizeAudioVoiceForConfig(config, value);
const options = audioVoiceOptionsForConfig(config);
return options.find((item) => item.value === normalized)?.label || normalized;
}
export function audioFormatLabelForConfig(config: AiConfig, value: string) {
const normalized = normalizeAudioFormatForConfig(config, value);
const options = audioFormatOptionsForConfig(config);
return options.find((item) => item.value === normalized)?.label || normalized;
}
export function normalizeAudioVoiceValue(value: string) {
return audioVoiceOptions.some((item) => item.value === value) ? value : "alloy";
const normalized = String(value || "").trim();
return normalized || "alloy";
}
export function normalizeAudioFormatValue(value: string) {
return audioFormatOptions.some((item) => item.value === value) ? value : "mp3";
return openAIAudioFormatOptions.some((item) => item.value === value) ? value : "mp3";
}
export function normalizeAudioSpeedValue(value: string) {
@@ -39,12 +140,12 @@ export function normalizeAudioSpeedValue(value: string) {
export function audioVoiceLabel(value: string) {
const voice = normalizeAudioVoiceValue(value);
return audioVoiceOptions.find((item) => item.value === voice)?.label || voice;
return openAIAudioVoiceOptions.find((item) => item.value === voice)?.label || voice;
}
export function audioFormatLabel(value: string) {
const format = normalizeAudioFormatValue(value);
return audioFormatOptions.find((item) => item.value === format)?.label || format;
return openAIAudioFormatOptions.find((item) => item.value === format)?.label || format;
}
export function audioSpeedLabel(value: string) {

4
web/src/lib/canvas/agent-approval-presentation.ts
View File

@@ -31,7 +31,7 @@ function shortReference(value: unknown): string {
function nodeTypeLabel(value: unknown): string {
const type = text(value, 32).toLowerCase();
return ({ text: "文本", markdown: "Markdown", image: "图片", video: "视频", audio: "音频", frame: "画框", script: "脚本" } as Record<string, string>)[type] || (type ? "节点" : "");
return ({ text: "文本", markdown: "Markdown", image: "图片", video: "视频", audio: "音频", frame: "画框", script: "脚本", character: "角色卡" } as Record<string, string>)[type] || (type ? "节点" : "");
}
function fieldLabel(key: string, nodeType: string): string {
@@ -41,7 +41,7 @@ function fieldLabel(key: string, nodeType: string): string {
}
function operation(value: unknown): AgentApprovalPreviewOperation | null {
return value === "add_node" || value === "update_node" || value === "connect_nodes" || value === "arrange_nodes" || value === "generate_media" || value === "create_storyboard" || value === "edit_storyboard" || value === "plan_step" ? value : null;
return value === "add_node" || value === "update_node" || value === "connect_nodes" || value === "arrange_nodes" || value === "generate_media" || value === "create_storyboard" || value === "edit_storyboard" || value === "create_character" || value === "plan_step" ? value : null;
}
function normalizeServerPreview(value: unknown): AgentApprovalPresentation | null {

4
web/src/lib/canvas/agent-canvas-actions.ts
View File

@@ -67,9 +67,9 @@ export function agentCanvasActions(toolName: string, detail: unknown, references
}
export function agentCanvasActionLabel(action: AgentCanvasAction) {
const kind = ({ image: "图片", video: "视频", audio: "音频", text: "文本", markdown: "Markdown", script: "分镜" } as Record<string, string>)[action.nodeType] || "";
const kind = ({ image: "图片", video: "视频", audio: "音频", text: "文本", markdown: "Markdown", script: "分镜", character: "角色卡" } as Record<string, string>)[action.nodeType] || "";
if (action.action === "referenced" && action.targetTitle) {
const targetKind = ({ image: "图片", video: "视频", audio: "音频", text: "文本", markdown: "Markdown", script: "分镜" } as Record<string, string>)[action.targetNodeType || ""] || "";
const targetKind = ({ image: "图片", video: "视频", audio: "音频", text: "文本", markdown: "Markdown", script: "分镜", character: "角色卡" } as Record<string, string>)[action.targetNodeType || ""] || "";
return `建立引用：${kind}节点《${action.title}》 → ${targetKind}节点《${action.targetTitle}》`;
}
// read 的动词必须是"存在式"：canvas_get_state 只读到清单，说成"读取了图片节点"会被读成看过画面。

9
web/src/lib/canvas/agent-canvas-patch.ts
View File

@@ -83,14 +83,15 @@ export function applyAgentCanvasPatch(project: CanvasProject, patch: AgentCanvas
return { ...project, nodes, connections, updatedAt: patch.updatedAt || project.updatedAt };
}
export function mergeAgentCanvasEditor(previous: CanvasProject, incoming: CanvasProject, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
export function mergeAgentCanvasEditor(previous: CanvasProject, incoming: CanvasProject, nodes: CanvasNodeData[], connections: CanvasConnection[], editorProject?: CanvasProject) {
const changes = <T extends { id: string }>(before: T[], after: T[]): Change<T>[] => {
const byId = new Map(before.map((item) => [item.id, item]));
const afterIds = new Set(after.map((item) => item.id));
return [...after.filter((item) => !equal(item, byId.get(item.id))).map((item) => ({ before: byId.get(item.id) ?? null, after: item })), ...before.filter((item) => !afterIds.has(item.id)).map((item) => ({ before: item, after: null }))];
};
const editorState = editorProject || { ...previous, nodes, connections };
const projected = applyAgentCanvasPatch(
{ ...previous, nodes, connections },
{ ...previous, nodes: editorState.nodes, connections: editorState.connections },
{
canvasId: previous.id,
updatedAt: incoming.updatedAt,
@@ -101,12 +102,12 @@ export function mergeAgentCanvasEditor(previous: CanvasProject, incoming: Canvas
const merged = { ...projected } as CanvasProject & Record<string, unknown>;
const before = previous as CanvasProject & Record<string, unknown>;
const after = incoming as CanvasProject & Record<string, unknown>;
const editor = { ...previous, nodes, connections } as CanvasProject & Record<string, unknown>;
const editor = editorState as CanvasProject & Record<string, unknown>;
for (const key of new Set([...Object.keys(before), ...Object.keys(after), ...Object.keys(editor)])) {
if (["id", "revision", "updatedAt", "remoteContentHash", "viewport", "nodes", "connections"].includes(key)) continue;
const value = mergeValue(editor[key], before[key], after[key]);
if (value === undefined) delete merged[key];
else merged[key] = value;
}
return { ...merged, revision: incoming.revision, updatedAt: incoming.updatedAt, viewport: editor.viewport } as CanvasProject;
return { ...merged, revision: incoming.revision, updatedAt: incoming.updatedAt, viewport: editorState.viewport } as CanvasProject;
}

4
web/src/lib/canvas/agent-tool-presentation.ts
View File

@@ -13,6 +13,7 @@ export const AGENT_TOOL_NAMES = [
"ask_user",
"canvas_apply_ops",
"canvas_arrange_nodes",
"canvas_create_character",
"canvas_create_storyboard",
"canvas_edit_batch_table",
"canvas_edit_storyboard",
@@ -50,6 +51,7 @@ export const AGENT_TOOL_METADATA: Record<string, AgentToolMetadataEntry> = {
task_get: { summary: "已查询任务状态", failureMessage: "查询任务状态失败" },
canvas_apply_ops: { summary: ({ pending }) => (pending ? "准备更新画布内容" : "画布内容已保存至服务端"), failureMessage: "更新画布内容失败" },
model_list: { summary: "已获取可用模型", failureMessage: "获取可用模型失败" },
canvas_create_character: { summary: ({ pending }) => (pending ? "准备创建角色卡" : "角色卡已创建并放到画布"), failureMessage: "创建角色卡失败" },
generate_media: { summary: ({ pending, detail }) => (pending ? "准备创建媒体节点并生成" : field(detail, "eventType") === "tool_completed" ? "生成结果已回写画布节点" : "媒体节点已创建，生成任务已提交"), failureMessage: "媒体生成未完成" },
// 看图：只有这个工具会把图片字节真的交给模型，所以只有它可以说"画面/看图"。
// 回执字段来源：`backend/internal/app/cloud_agent_runtime.go`（payload["result"] = 回执）。
@@ -77,7 +79,7 @@ const AGENT_CANVAS_READ_TOOLS = new Set(["canvas_get_state", "canvas_list_node_t
/** 非画布信息的只读工具（模型、任务、技能、偏好）。`skills_load` 是历史事件名，保留兼容。 */
const AGENT_INFO_READ_TOOLS = new Set(["model_list", "task_get", "skill_search", "skill_read_file", "skills_load", "agent_profile_read"]);
const AGENT_VISION_TOOLS = new Set(["canvas_inspect_image"]);
const AGENT_CREATE_TOOLS = new Set(["generate_media", "canvas_create_storyboard", "image_annotation_render"]);
const AGENT_CREATE_TOOLS = new Set(["generate_media", "canvas_create_storyboard", "canvas_create_character", "image_annotation_render"]);
const AGENT_OPERATE_TOOLS = new Set(["canvas_apply_ops", "canvas_arrange_nodes", "canvas_edit_storyboard", "canvas_edit_batch_table", "image_layer_split", "image_text_detect"]);
export type AgentToolCategory = "read" | "vision" | "create" | "operate" | "other";

24
web/src/lib/canvas/canvas-connection-policy.ts
View File

@@ -1,5 +1,6 @@
import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection";
import { audioModelForConnection, audioReferenceCapacity, doubaoAudioInputError, isDoubaoAudioModel, maxModelInputCapacity, resolveCompatibleModel, type ModelInputSummary } from "@/lib/model-selection";
import { getNodeAcceptedInputKinds, getNodeGenerationMode, getNodeInputKind, getNodeMaxInputCount } from "@/lib/canvas/node-registry";
import { readNodeGenerationSpec, resolveGenerationSelection } from "@/lib/canvas/generation-contract";
import type { AiConfig } from "@/stores/use-config-store";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
@@ -49,11 +50,28 @@ export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[],
return options.ignoreCapacity ? "" : capacityError(config, mode, "image", visualInputCount, "参考图") || capacityError(config, mode, "video", input.videoCount, "参考视频") || capacityError(config, mode, "audio", input.audioCount, "参考音频");
}
if (mode === "text" && input.audioCount > 0) return "文本生成节点不能连接参考音频";
if (mode === "audio" && input.characterCount > 1) return "角色配音一次只能连接一个角色卡";
if (mode === "audio" && (input.imageCount > 0 || input.videoCount > 0 || input.audioCount > 0)) return "音频生成节点只接受文本或单个角色卡输入";
if (mode === "audio") {
const model = routedAudioModel(config, target, input);
if (isDoubaoAudioModel(config, model)) return doubaoAudioInputError(input);
if (input.imageCount > 0 || input.videoCount > 0) return "音频生成节点不能连接参考图片或参考视频";
if (input.characterCount > 1) return "角色配音一次只能连接一个角色卡";
const maxAudios = audioReferenceCapacity(config, model);
if (input.audioCount > maxAudios) {
return maxAudios > 0 ? `音频生成节点最多连接 ${maxAudios} 个参考音频` : "当前音频模型不支持参考音频";
}
}
return "";
}
function routedAudioModel(config: AiConfig, target: CanvasNodeData, input: ModelInputSummary) {
const generationSpec = readNodeGenerationSpec(target);
const selectedModel = generationSpec?.mode === "audio" ? resolveGenerationSelection(config, generationSpec.modelSelection) : "";
const explicit = audioModelForConnection(config, selectedModel || target.metadata?.model || "");
// 同名模型组会在生成时改选到能接受当前参考的成员。连线必须按这个结果判断，
// 不能拿组里一个不支持图片的模型把豆包音频挡掉。
return resolveCompatibleModel(config, explicit, { capability: "audio", input }) || explicit;
}
function acceptedInputKindLabel(kind: "image" | "video" | "audio" | "text" | "table_data") {
if (kind === "image") return "图片";
if (kind === "video") return "视频";

4
web/src/lib/canvas/canvas-generation-submission.ts
View File

@@ -19,6 +19,8 @@ export type CanvasGenerationRequestFingerprintInput = {
parameters?: Record<string, unknown>;
};
operation?: string;
audioLanguage?: string;
audioDialect?: string;
audioInstructions?: string;
audioEmotionControlMethod?: string;
audioEmotionRandom?: string;
@@ -54,6 +56,8 @@ export function canvasGenerationRequestFingerprint(input: CanvasGenerationReques
options: input.options,
workflow: input.workflow,
operation: input.operation,
audioLanguage: input.audioLanguage,
audioDialect: input.audioDialect,
audioInstructions: input.audioInstructions,
audioEmotionControlMethod: input.audioEmotionControlMethod,
audioEmotionRandom: input.audioEmotionRandom,

2
web/src/lib/canvas/canvas-project-domain.ts
View File

@@ -241,6 +241,8 @@ const NODE_MODEL_GENERATION_PARAMS: ReadonlyArray<keyof CanvasNodeMetadata> = [
"audioVoice",
"audioFormat",
"audioSpeed",
"audioLanguage",
"audioDialect",
"audioInstructions",
];

4
web/src/lib/canvas/canvas-project-generation.ts
View File

@@ -241,6 +241,8 @@ export function buildAudioGenerationMetadata(config: AiConfig): CanvasNodeMetada
audioVoice: config.audioVoice,
audioFormat: config.audioFormat,
audioSpeed: config.audioSpeed,
audioLanguage: config.audioLanguage,
audioDialect: config.audioDialect,
audioInstructions: config.audioInstructions,
audioEmotionControlMethod: config.audioEmotionControlMethod,
audioEmotionRandom: config.audioEmotionRandom,
@@ -447,6 +449,8 @@ export function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | u
audioVoice: node?.metadata?.audioVoice ?? config.audioVoice ?? defaultConfig.audioVoice,
audioFormat: node?.metadata?.audioFormat ?? config.audioFormat ?? defaultConfig.audioFormat,
audioSpeed: node?.metadata?.audioSpeed ?? config.audioSpeed ?? defaultConfig.audioSpeed,
audioLanguage: node?.metadata?.audioLanguage ?? config.audioLanguage ?? defaultConfig.audioLanguage,
audioDialect: node?.metadata?.audioDialect ?? config.audioDialect ?? defaultConfig.audioDialect,
audioInstructions: node?.metadata?.audioInstructions ?? config.audioInstructions ?? defaultConfig.audioInstructions,
count: String(node?.metadata?.count ?? (mode === "image" ? config.canvasImageCount || config.count || defaultConfig.count : config.count || defaultConfig.count)),
};

20
web/src/lib/canvas/generation-contract.generated.ts
View File

@@ -16,6 +16,8 @@ export type Options = {
audioVoice?: string;
audioFormat?: string;
audioSpeed?: string;
audioLanguage?: string;
audioDialect?: string;
audioInstructions?: string;
};
@@ -141,6 +143,24 @@ export const GENERATION_OPTION_FIELDS = [
"node": "audioSpeed",
"task": "audioSpeed"
},
{
"kind": "string",
"modes": [
"audio"
],
"name": "audioLanguage",
"node": "audioLanguage",
"task": "audioLanguage"
},
{
"kind": "string",
"modes": [
"audio"
],
"name": "audioDialect",
"node": "audioDialect",
"task": "audioDialect"
},
{
"kind": "string",
"modes": [

2
web/src/lib/canvas/node-registry/definitions/builtin-nodes.tsx
View File

@@ -91,7 +91,7 @@ const BUILTIN_NODE_TRAITS = {
icon: <Music2 />,
minSize: DEFAULT_MIN_SIZE,
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "audio" : null),
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "audio" : null),
generationMode: () => "audio",
inputKind: "audio",
},

2
web/src/lib/canvas/tool-registry/definitions/add-node-menu-tools.tsx
View File

@@ -19,6 +19,7 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
nodeCommand(CanvasNodeType.Frame, { defaultOrder: 40, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddFrame() }),
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
{ id: "project-character", label: "角色卡", icon: <UserRound />, section: "node", defaultOrder: 55, run: (ctx) => ctx.handlers.onOpenProjectCharacters() },
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
nodeCommand(CanvasNodeType.BatchTable, { defaultOrder: 66, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.BatchTable) }),
nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }),
@@ -29,7 +30,6 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
{ id: "workflow", label: "工作流", icon: <Workflow />, section: "workflow", defaultOrder: 10, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddWorkflow() },
// 导入资源
{ id: "upload", label: "上传文件", icon: <UploadCloud />, section: "resource", defaultOrder: 10, run: (ctx) => ctx.handlers.onUpload() },
{ id: "project-character", label: "添加角色卡", icon: <UserRound />, section: "resource", defaultOrder: 20, applicable: (ctx) => ctx.isProjectLinked, run: (ctx) => ctx.handlers.onOpenProjectCharacters() },
{ id: "assets", label: "素材库", icon: <FolderOpen />, section: "resource", defaultOrder: 30, applicable: (ctx) => !ctx.isProjectLinked, run: (ctx) => ctx.handlers.onOpenMyAssets() },
];

118
web/src/lib/doubao-audio-voices.ts Normal file
View File

@@ -0,0 +1,118 @@
// 来源：火山引擎《音色列表》中的「豆包语音合成模型1.0」。
// seed-audio-1.0 对应资源 seed-tts-1.0，只能使用这一代音色，不能混入 2.0。
// 文档里同一个音色编号会按语种重复出现，下拉框按音色编号去重。
export type DoubaoAudioVoiceOption = {
scene: string;
value: string;
label: string;
language: string;
};
export const doubaoDefaultAudioVoice = "zh_female_shuangkuaisisi_moon_bigtts";
export const doubaoAudioVoiceOptions: DoubaoAudioVoiceOption[] = [
{ scene: "多情感", value: "zh_male_lengkugege_emo_v2_mars_bigtts", label: "冷酷哥哥", language: "中文" },
{ scene: "多情感", value: "zh_female_tianxinxiaomei_emo_v2_mars_bigtts", label: "甜心小美", language: "中文" },
{ scene: "多情感", value: "zh_male_guangzhoudege_emo_mars_bigtts", label: "广州德哥", language: "中文" },
{ scene: "多情感", value: "zh_male_jingqiangkanye_emo_mars_bigtts", label: "京腔侃爷", language: "中文" },
{ scene: "多情感", value: "zh_female_linjuayi_emo_v2_mars_bigtts", label: "邻居阿姨", language: "中文" },
{ scene: "多情感", value: "zh_male_yourougongzi_emo_v2_mars_bigtts", label: "优柔公子", language: "中文" },
{ scene: "多情感", value: "zh_male_ruyayichen_emo_v2_mars_bigtts", label: "儒雅男友", language: "中文" },
{ scene: "多情感", value: "zh_male_junlangnanyou_emo_v2_mars_bigtts", label: "俊朗男友", language: "中文" },
{ scene: "多情感", value: "zh_male_beijingxiaoye_emo_v2_mars_bigtts", label: "北京小爷", language: "中文" },
{ scene: "多情感", value: "zh_female_roumeinvyou_emo_v2_mars_bigtts", label: "柔美女友", language: "中文" },
{ scene: "多情感", value: "zh_male_yangguangqingnian_emo_v2_mars_bigtts", label: "阳光青年", language: "中文" },
{ scene: "多情感", value: "zh_female_meilinvyou_emo_v2_mars_bigtts", label: "魅力女友", language: "中文" },
{ scene: "多情感", value: "zh_female_shuangkuaisisi_emo_v2_mars_bigtts", label: "爽快思思", language: "中文、英式英语" },
{ scene: "多情感", value: "zh_male_shenyeboke_emo_v2_mars_bigtts", label: "深夜播客", language: "中文" },
{ scene: "多情感", value: "en_female_candice_emo_v2_mars_bigtts", label: "美式英语 · 坎迪丝", language: "美式英语" },
{ scene: "教育场景", value: "zh_female_yingyujiaoyu_mars_bigtts", label: "蒂娜老师", language: "中文、英式英语" },
{ scene: "通用场景", value: "zh_female_qinqienvsheng_moon_bigtts", label: "亲切女声", language: "中文" },
{ scene: "通用场景", value: "zh_male_qingyiyuxuan_mars_bigtts", label: "阳光阿辰", language: "中文" },
{ scene: "通用场景", value: "zh_male_xudong_conversation_wvae_bigtts", label: "快乐小东", language: "中文、美式英语" },
{ scene: "通用场景", value: "en_male_jason_conversation_wvae_bigtts", label: "开朗学长", language: "中文" },
{ scene: "通用场景", value: "zh_female_tianmeitaozi_mars_bigtts", label: "甜美桃子", language: "中文" },
{ scene: "通用场景", value: "zh_female_qingxinnvsheng_mars_bigtts", label: "清新女声", language: "中文" },
{ scene: "通用场景", value: "zh_female_zhixingnvsheng_mars_bigtts", label: "知性女声", language: "中文" },
{ scene: "通用场景", value: "zh_male_qingshuangnanda_mars_bigtts", label: "清爽男大", language: "中文" },
{ scene: "通用场景", value: "zh_female_linjianvhai_moon_bigtts", label: "邻家女孩", language: "中文" },
{ scene: "通用场景", value: "zh_male_yuanboxiaoshu_moon_bigtts", label: "渊博小叔", language: "中文" },
{ scene: "通用场景", value: "zh_male_yangguangqingnian_moon_bigtts", label: "阳光青年", language: "中文" },
{ scene: "通用场景", value: "zh_female_tianmeixiaoyuan_moon_bigtts", label: "甜美小源", language: "中文" },
{ scene: "通用场景", value: "zh_female_qingchezizi_moon_bigtts", label: "清澈梓梓", language: "中文" },
{ scene: "通用场景", value: "zh_male_jieshuoxiaoming_moon_bigtts", label: "解说小明", language: "中文" },
{ scene: "通用场景", value: "zh_female_kailangjiejie_moon_bigtts", label: "开朗姐姐", language: "中文" },
{ scene: "通用场景", value: "zh_male_linjiananhai_moon_bigtts", label: "邻家男孩", language: "中文" },
{ scene: "通用场景", value: "zh_female_tianmeiyueyue_moon_bigtts", label: "甜美悦悦", language: "中文" },
{ scene: "通用场景", value: "zh_female_xinlingjitang_moon_bigtts", label: "心灵鸡汤", language: "中文" },
{ scene: "通用场景", value: "zh_male_wenrouxiaoge_mars_bigtts", label: "温柔小哥", language: "中文" },
{ scene: "通用场景", value: "zh_female_cancan_mars_bigtts", label: "灿灿", language: "中文、美式英语" },
{ scene: "通用场景", value: "zh_female_shuangkuaisisi_moon_bigtts", label: "爽快思思", language: "中文、美式英语" },
{ scene: "通用场景", value: "zh_male_wennuanahu_moon_bigtts", label: "温暖阿虎", language: "中文、美式英语" },
{ scene: "通用场景", value: "zh_male_shaonianzixin_moon_bigtts", label: "少年梓辛", language: "中文、美式英语" },
{ scene: "趣味口音", value: "zh_female_yueyunv_mars_bigtts", label: "粤语小溏", language: "中文" },
{ scene: "趣味口音", value: "zh_male_yuzhouzixuan_moon_bigtts", label: "豫州子轩", language: "河南口音" },
{ scene: "趣味口音", value: "zh_female_daimengchuanmei_moon_bigtts", label: "呆萌川妹", language: "四川口音" },
{ scene: "趣味口音", value: "zh_male_guangxiyuanzhou_moon_bigtts", label: "广西远舟", language: "广西口音" },
{ scene: "趣味口音", value: "zh_male_zhoujielun_emo_v2_mars_bigtts", label: "双节棍小哥", language: "台湾口音" },
{ scene: "趣味口音", value: "zh_female_wanwanxiaohe_moon_bigtts", label: "湾湾小何", language: "台湾口音" },
{ scene: "趣味口音", value: "zh_female_wanqudashu_moon_bigtts", label: "湾区大叔", language: "广东口音" },
{ scene: "趣味口音", value: "zh_male_guozhoudege_moon_bigtts", label: "广州德哥", language: "广东口音" },
{ scene: "趣味口音", value: "zh_male_haoyuxiaoge_moon_bigtts", label: "浩宇小哥", language: "青岛口音" },
{ scene: "趣味口音", value: "zh_male_beijingxiaoye_moon_bigtts", label: "北京小爷", language: "北京口音" },
{ scene: "趣味口音", value: "zh_male_jingqiangkanye_moon_bigtts", label: "京腔侃爷", language: "北京口音、美式英语" },
{ scene: "趣味口音", value: "zh_female_meituojieer_moon_bigtts", label: "妹坨洁儿", language: "长沙口音" },
{ scene: "角色扮演", value: "zh_male_naiqimengwa_mars_bigtts", label: "奶气萌娃", language: "中文" },
{ scene: "角色扮演", value: "zh_female_popo_mars_bigtts", label: "婆婆", language: "中文" },
{ scene: "角色扮演", value: "zh_female_gaolengyujie_moon_bigtts", label: "高冷御姐", language: "中文" },
{ scene: "角色扮演", value: "zh_male_aojiaobazong_moon_bigtts", label: "傲娇霸总", language: "中文" },
{ scene: "角色扮演", value: "zh_female_meilinvyou_moon_bigtts", label: "魅力女友", language: "中文" },
{ scene: "角色扮演", value: "zh_male_shenyeboke_moon_bigtts", label: "深夜播客", language: "中文" },
{ scene: "角色扮演", value: "zh_female_sajiaonvyou_moon_bigtts", label: "柔美女友", language: "中文" },
{ scene: "角色扮演", value: "zh_female_yuanqinvyou_moon_bigtts", label: "撒娇学妹", language: "中文" },
{ scene: "角色扮演", value: "zh_male_dongfanghaoran_moon_bigtts", label: "东方浩然", language: "中文" },
{ scene: "外语音色", value: "en_male_adam_mars_bigtts", label: "美式英语 · 亚当", language: "美式英语" },
{ scene: "外语音色", value: "en_female_amanda_mars_bigtts", label: "美式英语 · 阿曼达", language: "美式英语" },
{ scene: "外语音色", value: "en_male_jackson_mars_bigtts", label: "美式英语 · 杰克逊", language: "美式英语" },
{ scene: "外语音色", value: "en_male_smith_mars_bigtts", label: "英式英语 · 史密斯", language: "英式英语" },
{ scene: "外语音色", value: "en_female_anna_mars_bigtts", label: "英式英语 · 安娜", language: "英式英语" },
{ scene: "外语音色", value: "en_female_sarah_mars_bigtts", label: "澳洲英语 · 萨拉", language: "澳洲英语" },
{ scene: "外语音色", value: "en_male_dryw_mars_bigtts", label: "澳洲英语 · 德鲁", language: "澳洲英语" },
{ scene: "外语音色", value: "multi_male_M100_conversation_wvae_bigtts", label: "西语 · 卢西亚", language: "西语" },
{ scene: "外语音色", value: "multi_zh_male_youyoujunzi_moon_bigtts", label: "日语 · 光", language: "日语" },
{ scene: "外语音色", value: "multi_female_sophie_conversation_wvae_bigtts", label: "日语 · 智美", language: "日语、西语" },
{ scene: "外语音色", value: "multi_male_xudong_conversation_wvae_bigtts", label: "日语 · 正男", language: "日语、西语" },
{ scene: "外语音色", value: "multi_female_maomao_conversation_wvae_bigtts", label: "日语 · 月", language: "日语、西语" },
{ scene: "外语音色", value: "multi_female_gaolengyujie_moon_bigtts", label: "日语 · 朱美", language: "日语" },
{ scene: "外语音色", value: "multi_male_jingqiangkanye_moon_bigtts", label: "日语 · 和音", language: "日语、西语" },
{ scene: "外语音色", value: "multi_female_shuangkuaisisi_moon_bigtts", label: "日语 · 晴子", language: "日语、西语" },
{ scene: "外语音色", value: "multi_male_wanqudashu_moon_bigtts", label: "日语 · 广志", language: "日语、西语" },
{ scene: "视频配音", value: "zh_male_M100_conversation_wvae_bigtts", label: "悠悠君子", language: "中文、美式英语" },
{ scene: "视频配音", value: "zh_female_maomao_conversation_wvae_bigtts", label: "文静毛毛", language: "中文" },
{ scene: "视频配音", value: "zh_female_wenrouxiaoya_moon_bigtts", label: "温柔小雅", language: "中文" },
{ scene: "视频配音", value: "zh_male_tiancaitongsheng_mars_bigtts", label: "天才童声", language: "中文" },
{ scene: "视频配音", value: "zh_male_sunwukong_mars_bigtts", label: "猴哥", language: "中文" },
{ scene: "视频配音", value: "zh_male_xionger_mars_bigtts", label: "熊二", language: "中文" },
{ scene: "视频配音", value: "zh_female_peiqi_mars_bigtts", label: "佩奇猪", language: "中文" },
{ scene: "视频配音", value: "zh_female_wuzetian_mars_bigtts", label: "武则天", language: "中文" },
{ scene: "视频配音", value: "zh_female_gujie_mars_bigtts", label: "顾姐", language: "中文" },
{ scene: "视频配音", value: "zh_female_yingtaowanzi_mars_bigtts", label: "樱桃丸子", language: "中文" },
{ scene: "视频配音", value: "zh_male_chunhui_mars_bigtts", label: "广告解说", language: "中文" },
{ scene: "视频配音", value: "zh_female_shaoergushi_mars_bigtts", label: "少儿故事", language: "中文" },
{ scene: "视频配音", value: "zh_male_silang_mars_bigtts", label: "四郎", language: "中文" },
{ scene: "视频配音", value: "zh_female_qiaopinvsheng_mars_bigtts", label: "俏皮女声", language: "中文" },
{ scene: "视频配音", value: "zh_male_lanxiaoyang_mars_bigtts", label: "懒音绵宝", language: "中文" },
{ scene: "视频配音", value: "zh_male_dongmanhaimian_mars_bigtts", label: "亮嗓萌仔", language: "中文" },
{ scene: "视频配音", value: "zh_female_jitangmeimei_mars_bigtts", label: "鸡汤妹妹", language: "中文、美式英语" },
{ scene: "视频配音", value: "zh_female_tiexinnvsheng_mars_bigtts", label: "贴心女声", language: "中文、美式英语" },
{ scene: "视频配音", value: "zh_female_mengyatou_mars_bigtts", label: "萌丫头", language: "中文、美式英语" },
{ scene: "有声阅读", value: "zh_male_changtianyi_mars_bigtts", label: "悬疑解说", language: "中文" },
{ scene: "有声阅读", value: "zh_male_ruyaqingnian_mars_bigtts", label: "儒雅青年", language: "中文" },
{ scene: "有声阅读", value: "zh_male_baqiqingshu_mars_bigtts", label: "霸气青叔", language: "中文" },
{ scene: "有声阅读", value: "zh_male_qingcang_mars_bigtts", label: "擎苍", language: "中文" },
{ scene: "有声阅读", value: "zh_male_yangguangqingnian_mars_bigtts", label: "活力小哥", language: "中文" },
{ scene: "有声阅读", value: "zh_female_gufengshaoyu_mars_bigtts", label: "古风少御", language: "中文" },
{ scene: "有声阅读", value: "zh_female_wenroushunv_mars_bigtts", label: "温柔淑女", language: "中文" },
{ scene: "有声阅读", value: "zh_male_fanjuanqingnian_mars_bigtts", label: "反卷青年", language: "中文" },
];

56
web/src/lib/model-selection.ts
View File

@@ -32,6 +32,54 @@ export type ModelReferenceLimits = {
maxAudios: number;
};
/**
* 普通音频模型只接受文本或音色 ID。豆包音频生成可以连接最多 3 段参考音频，
* 或 1 张参考图片，两者不能同时使用。
*/
export function audioReferenceCapacity(config: AiConfig, model: string) {
return audioProtocol(config, model) === "doubao-streaming-tts" ? 3 : 0;
}
export function isDoubaoAudioModel(config: AiConfig, model: string) {
if (!String(model || "").trim()) return false;
if (audioProtocol(config, model) === "doubao-streaming-tts") return true;
// seed-audio-1.0 与 seed-audio-1.0-multilingual 共用同一套 references 合同。
return modelOptionName(model).toLowerCase().startsWith("seed-audio");
}
export function audioModelForConnection(config: AiConfig, explicit = "") {
const selected = String(explicit || "").trim();
if (selected) return selected;
if (String(config.audioModel || "").trim()) return config.audioModel;
return (config.audioModels || []).find((item) => isDoubaoAudioModel(config, item)) || "";
}
/**
* 从连线新建音频节点时选模型：默认音频模型接得住当前输入就用它，
* 否则改用第一个接得住的音频模型（例如图片连过来时改用豆包音频）。
*/
export function audioModelForConnectionInput(config: AiConfig, input: ModelInputSummary) {
const preferred = audioModelForConnection(config);
const requirements: ModelRequirements = { capability: "audio", input };
if (preferred && !modelCompatibilityError(config, preferred, requirements)) return preferred;
const candidates = config.audioModels?.length ? config.audioModels : selectableModelsByCapability(config, "audio");
return candidates.find((model) => !modelCompatibilityError(config, model, requirements)) || preferred;
}
export function doubaoAudioInputError(input: ModelInputSummary) {
if (input.videoCount > 0) return "音频生成不能连接参考视频";
if (input.imageCount > 0 && input.audioCount > 0) return "参考图片和参考音频不能同时连接";
if (input.imageCount > 1) return "最多连接 1 张参考图片";
if (input.audioCount > 3) return "最多连接 3 段参考音频";
return "";
}
function audioProtocol(config: AiConfig, model: string) {
const channel = resolveModelChannel(config, model);
const cost = channel.modelCosts?.find((item) => item.model === modelOptionName(model));
return cost?.protocol || channel.interfaceType;
}
export function isDirectSystemModel(config: AiConfig, value: string) {
if (!value) return false;
const channel = resolveModelChannel(config, value);
@@ -110,8 +158,14 @@ export function modelCompatibilityError(config: AiConfig, model: string, require
return "";
}
if (isDoubaoAudioModel(config, model)) return doubaoAudioInputError(input);
if (input.imageCount > 0 || input.videoCount > 0) return "音频模型不能连接参考图片或参考视频";
if (input.characterCount > 1) return "角色配音一次只能引用一个角色卡";
return input.imageCount > 0 || input.videoCount > 0 || input.audioCount > 0 ? "音频模型只接受文本或单个角色卡输入" : "";
const maxAudios = audioReferenceCapacity(config, model);
if (input.audioCount > maxAudios) {
return maxAudios > 0 ? `当前音频模型最多支持 ${maxAudios} 个参考音频` : "当前音频模型不支持参考音频";
}
return "";
}
export function modelPromptLengthError(config: AiConfig, model: string, capability: ModelCapability, prompt: string) {