tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(canvas): 画布引用 - 统一角色与图片编号并保持生成资源顺序

Browse Source
This commit is contained in:
ddcat666
2026-10-08 14:20:45 +08:00
parent ef82ca1b62
commit 9ef10b7b2d
12 changed files with 368 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
web/src/components/canvas/canvas-node-generation.ts
View File

@@ -12,6 +12,7 @@ import { mediaConversionSourceFingerprint } from "@/lib/media-conversion/contrac
import { audioFileExtension } from "@/lib/character-voice-formats";
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
import { compileCharacterReferencePrompt, normalizeCharacterImageMentions } from "@/lib/canvas/canvas-character-reference";
import { createCanvasReferenceLabeler } from "@/lib/canvas/canvas-reference-slots";
import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation";
import { isCanvasWorkflowProvider } from "@/lib/canvas/canvas-workflow";
import { nodeGenerationPrompt } from "@/lib/canvas/generation-contract";
@@ -42,6 +43,8 @@ export type NodeGenerationContext = {
referenceImages: ReferenceImage[];
referenceVideos: ReferenceVideo[];
referenceAudios: ReferenceAudio[];
/** 图片和角色卡主图在最终 image URL 列表中的槽位顺序。 */
referenceImageSlots: Array<{ nodeId: string; kind: "image" | "character"; imageId?: string }>;
characterReferences: CharacterGenerationReference[];
resolvedCharacterVersions: Array<{ assetId: string; versionId: string }>;
resolvedCharacterVoices: ResolvedCharacterVoice[];
@@ -97,7 +100,6 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
prompt,
// 工作流节点由字段映射接收全部连线媒体；视频节点的历史首尾帧字段不能再额外追加参考图。
autoIncludeWorkflowMedia || (promptOnly && hasConnectedMedia) ? [] : [sourceNode?.metadata?.videoStartFrameNodeId, sourceNode?.metadata?.videoEndFrameNodeId].filter((id): id is string => Boolean(id)),
promptOnly,
autoIncludeWorkflowMedia || (promptOnly && hasConnectedMedia),
connectedInputs,
);
@@ -114,12 +116,14 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
const referenceImages = connectedInputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
const referenceVideos = connectedInputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video));
const referenceAudios = connectedInputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio));
const referenceImageSlots = generationImageSlots(connectedInputs);
return {
prompt: promptOnly ? prompt : upstreamText ? `${basePrompt}\n\n${upstreamText}` : basePrompt,
referenceImages,
referenceVideos,
referenceAudios,
referenceImageSlots,
characterReferences,
resolvedCharacterVersions: [],
resolvedCharacterVoices: [],
@@ -186,7 +190,6 @@ function buildComposerGenerationContext(
inputs: NodeGenerationInput[],
prompt: string,
videoFrameNodeIds: string[] = [],
promptOnly = false,
autoIncludeWorkflowMedia = false,
workflowMediaInputs: NodeGenerationInput[] = [],
): NodeGenerationContext {
@@ -194,102 +197,50 @@ function buildComposerGenerationContext(
const slotInputByToken = new Map(generationSlotEntries(inputs).map(({ token, input }) => [token, input]));
const assetInputById = new Map(inputs.filter((input) => input.nodeId.startsWith("asset:")).map((input) => [input.nodeId.slice("asset:".length), input]));
const nodeInputById = new Map(inputs.filter((input) => !input.nodeId.startsWith("asset:")).map((input) => [input.nodeId, input]));
const selectedInputs: NodeGenerationInput[] = [];
const labelByNodeId = new Map<string, string>();
const textBlocks: string[] = [];
const counts = { image: 0, drawing: 0, video: 0, audio: 0, text: 0, character: 0 };
let hasToken = false;
const mentions = [...normalizedPrompt.matchAll(GENERATION_MENTION_PATTERN)].map((match) => ({
match,
input: resolveGenerationMention(normalizedPrompt, match, slotInputByToken, nodeInputById, assetInputById),
}));
const selectedIds = new Set(mentions.flatMap(({ input }) => input ? [input.nodeId] : []));
if (autoIncludeWorkflowMedia) workflowMediaInputs.forEach((input) => {
if (input.type !== "text") selectedIds.add(input.nodeId);
});
videoFrameNodeIds.forEach((id) => {
if (nodeInputById.get(id)?.image) selectedIds.add(id);
});
// 始终先按槽位选定资源，再一次性编译提示词；提及先后不能改变 URL 顺序。
const selected = inputs.filter((input) => selectedIds.has(input.nodeId));
const selectedInputs = selected.filter((input) => input.type !== "text");
const nextLabel = createCanvasReferenceLabeler();
const providerLabels = new Map(selected.map((input) => [input.nodeId, nextLabel(input.type === "character" ? "image" : input.type)]));
const textBlocks = selected.filter((input) => input.type === "text").map((input) => `【${providerLabels.get(input.nodeId)}】\n${input.text || ""}`);
let lastIndex = 0;
let nextPrompt = "";
if (autoIncludeWorkflowMedia) {
// 先固定“图片1/视频1”等提示词标签的顺序，和工作流槽位保持一致；
// 用户先 @ 第二张图时，提示词也不会把它误标成第一张。
workflowMediaInputs.forEach((input) => {
if (input.type === "text" || labelByNodeId.has(input.nodeId)) return;
const labelKind = input.sourceKind === "drawing" ? "drawing" : input.type;
labelByNodeId.set(input.nodeId, generationLabel(labelKind, counts[labelKind]++));
});
for (const { match, input } of mentions) {
const index = match.index!;
nextPrompt += normalizedPrompt.slice(lastIndex, index);
const label = input ? providerLabels.get(input.nodeId) : undefined;
nextPrompt += label ? input?.type === "text" ? `【${label}】` : `@${label}` : match[0];
lastIndex = index + match[0].length;
}
for (const match of normalizedPrompt.matchAll(GENERATION_MENTION_PATTERN)) {
if (match.index === undefined) continue;
nextPrompt += normalizedPrompt.slice(lastIndex, match.index);
const input = resolveGenerationMention(normalizedPrompt, match, slotInputByToken, nodeInputById, assetInputById);
if (input) {
hasToken = true;
let label = labelByNodeId.get(input.nodeId);
if (!label) {
const labelKind = input.sourceKind === "drawing" ? "drawing" : input.type;
label = generationLabel(labelKind, counts[labelKind]++);
labelByNodeId.set(input.nodeId, label);
if (input.type === "text") textBlocks.push(`【${label}】\n${input.text || ""}`);
else selectedInputs.push(input);
}
nextPrompt += input.type === "text" ? `【${label}】` : `@${label}`;
} else nextPrompt += match[0];
lastIndex = match.index + match[0].length;
}
nextPrompt += normalizedPrompt.slice(lastIndex);
// 显式 @文本 引用是用户写进输入框的内容，必须内联真实文本；promptOnly 只拦自动上游文本。
if (textBlocks.length) nextPrompt = `${nextPrompt.trim()}\n\n${textBlocks.join("\n\n")}`;
if (autoIncludeWorkflowMedia) {
// RunningHub/ComfyUI 工作流按保存的字段槽位接收图片、视频和音频；
// 配置节点不能因为提示词里没有逐个 @ 就丢失已连接媒体。
// 先按连线顺序放入媒体，避免用户在提示词里 @图片2 后改变工作流槽位的索引；
// 素材库中的显式 @ 引用仍保留在后面，不会被自动模式吞掉。
const explicitInputs = selectedInputs.splice(0);
const selectedNodeIds = new Set<string>();
workflowMediaInputs.forEach((input) => {
if (input.type === "text" || selectedNodeIds.has(input.nodeId)) return;
selectedInputs.push(input);
selectedNodeIds.add(input.nodeId);
});
explicitInputs.forEach((input) => {
if (selectedNodeIds.has(input.nodeId)) return;
selectedInputs.push(input);
selectedNodeIds.add(input.nodeId);
});
}
// 首尾帧是结构化生成参数，不受提示词中的 @ 引用筛选影响。
const selectedNodeIds = new Set(selectedInputs.map((input) => input.nodeId));
videoFrameNodeIds.forEach((nodeId) => {
const input = nodeInputById.get(nodeId);
if (!input?.image || selectedNodeIds.has(nodeId)) return;
selectedInputs.push(input);
selectedNodeIds.add(nodeId);
});
const referenceImages = selectedInputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
const referenceVideos = selectedInputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video));
const referenceAudios = selectedInputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio));
const characterReferences = selectedInputs.map((input) => input.character).filter((item): item is CharacterGenerationReference => Boolean(item));
if (!hasToken && !textBlocks.length && !selectedInputs.length) {
return {
prompt,
referenceImages: [],
referenceVideos: [],
referenceAudios: [],
characterReferences: [],
resolvedCharacterVersions: [],
resolvedCharacterVoices: [],
textCount: 0,
imageCount: 0,
videoCount: 0,
audioCount: 0,
};
}
const referenceImageSlots = generationImageSlots(selectedInputs);
return {
prompt: nextPrompt,
referenceImages,
referenceVideos,
referenceAudios,
referenceImageSlots,
characterReferences,
resolvedCharacterVersions: [],
resolvedCharacterVoices: [],
textCount: counts.text,
textCount: textBlocks.length,
imageCount: referenceImages.length,
videoCount: referenceVideos.length,
audioCount: referenceAudios.length,
@@ -310,6 +261,7 @@ export function normalizeGenerationNodeMentionTokens(prompt: string, inputs: Nod
const label = labelByNodeId.get(nodeId);
return label ? `@${label}` : token;
});
// 仅保留无普通图片时的既有别名；混合引用不能猜测旧的独立编号。
return normalizeCharacterImageMentions(normalized, slots.filter(({ input }) => input.type === "image").length, slots.filter(({ input }) => input.type === "character").map(({ label }) => label));
}
@@ -342,15 +294,20 @@ function inspectGenerationMentions(prompt: string, inputs: NodeGenerationInput[]
}
function generationSlotEntries(inputs: NodeGenerationInput[]) {
const counts = { image: 0, drawing: 0, video: 0, audio: 0, text: 0, character: 0 };
const nextLabel = createCanvasReferenceLabeler();
return inputs.flatMap((input) => {
if (input.nodeId.startsWith("asset:")) return [];
const kind = input.sourceKind === "drawing" ? "drawing" : input.type;
const label = generationLabel(kind, counts[kind]++);
const label = nextLabel(input.sourceKind === "drawing" ? "drawing" : input.type);
return [{ input, label, token: `@${label}` }];
});
}
function generationImageSlots(inputs: NodeGenerationInput[]): NodeGenerationContext["referenceImageSlots"] {
return inputs.flatMap((input): NodeGenerationContext["referenceImageSlots"] => input.character
? [{ nodeId: input.nodeId, kind: "character" }]
: input.image ? [{ nodeId: input.nodeId, kind: "image", imageId: input.image.id }] : []);
}
function resolveGenerationMention(
prompt: string,
match: RegExpMatchArray,
@@ -369,7 +326,8 @@ function resolveGenerationMention(
function hasMentionBoundary(value: string, index: number) {
const char = value[index];
return !char || /\s|[,.!?;:，。！？；：、)\]}】）]/.test(char);
// 与编辑器一致：编号可紧邻中文或下一个引用，不能截断更长的数字。
return !char || !/[0-9]/.test(char);
}
export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
@@ -500,27 +458,71 @@ export async function hydrateNodeGenerationContext(context: NodeGenerationContex
const { getResource, resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey } = await import("@/services/api/resources");
const details = await Promise.all(context.characterReferences.map((reference) => getCharacter(reference.assetId)));
const remainingBudget = Math.max(0, (referenceLimits?.maxImages ?? 9) - referenceImages.length);
const selected = details.flatMap((detail) => {
const selectedEntries = details.flatMap((detail, index) => {
const reference = context.characterReferences[index];
if (reference.requestedVersionId && reference.requestedVersionId !== detail.character.versionId) throw new Error(`角色「${detail.asset.title}」的固定版本已变化，请重新选择角色后生成`);
const representation = preferredCharacterRepresentation(detail.character.representations);
return representation ? [representation] : [];
if (!representation && mode !== "audio" && mode !== "text") throw new Error(`角色「${detail.asset.title}」缺少可用的主参考图，请补充形象后再生成`);
return representation ? [{ nodeId: reference.nodeId, representation }] : [];
});
if (selected.length > remainingBudget) throw new Error(`当前模型参考图容量不足：角色至少需要 ${selected.length} 张主参考图`);
const usedResourceIds = new Set(selected.map((item) => item.resourceId));
const supplements = details.flatMap((detail) => detail.character.representations.filter((item) => {
if (!["front", "side", "back", "turnaround_sheet"].includes(item.role) || usedResourceIds.has(item.resourceId)) return false;
usedResourceIds.add(item.resourceId);
return true;
}));
const characterImages = [...selected, ...supplements].slice(0, Math.max(0, remainingBudget)).map((representation, index) => ({
if (selectedEntries.length > remainingBudget) throw new Error(`当前模型参考图容量不足：角色至少需要 ${selectedEntries.length} 张主参考图`);
const usedResourceIds = new Set(selectedEntries.map((item) => item.representation.resourceId));
const supplementEntries = details.flatMap((detail, index) => {
const reference = context.characterReferences[index];
if (!reference) return [];
return detail.character.representations.flatMap((representation) => {
if (!["front", "side", "back", "turnaround_sheet"].includes(representation.role) || !representation.resourceId || usedResourceIds.has(representation.resourceId)) return [];
usedResourceIds.add(representation.resourceId);
return [{ nodeId: reference.nodeId, representation }];
});
});
const imageEntries = [...selectedEntries, ...supplementEntries].slice(0, Math.max(0, remainingBudget));
const characterImages = imageEntries.map((entry, index) => ({
id: `character-reference-${index + 1}`,
name: `character-reference-${index + 1}.png`,
type: "image/png",
dataUrl: "",
storageKey: resourceStorageKey(representation.resourceId),
storageKey: resourceStorageKey(entry.representation.resourceId),
} satisfies ReferenceImage));
const hydratedCharacterImages = await Promise.all(characterImages.map(async (image) => ({ ...image, dataUrl: await imageToDataUrl(image) })));
referenceImages = [...referenceImages, ...hydratedCharacterImages];
const characterBlocks = details.map((detail) => compileCharacterReferencePrompt(detail.asset.title, detail.character.definition));
const primaryByNodeId = new Map<string, ReferenceImage>();
selectedEntries.forEach((entry, index) => {
const image = hydratedCharacterImages[index];
if (image) primaryByNodeId.set(entry.nodeId, image);
});
const regularById = new Map(referenceImages.map((image) => [image.id, image]));
const orderedIndexByNodeId = new Map<string, number>();
const providerMentionBySlot = new Map<string, string>();
const orderedImages: ReferenceImage[] = [];
context.referenceImageSlots.forEach((slot, index) => {
const image = slot.kind === "character" ? primaryByNodeId.get(slot.nodeId) : regularById.get(slot.imageId || slot.nodeId);
if (!image) {
const characterIndex = context.characterReferences.findIndex((reference) => reference.nodeId === slot.nodeId);
const detail = details[characterIndex];
if (slot.kind !== "character" || !detail || (mode !== "text" && mode !== "audio")) throw new Error("参考图片槽位无法解析，请重新选择引用后生成");
providerMentionBySlot.set(`@图片${index + 1}`, detail.asset.title.replaceAll("@", "＠"));
return;
}
orderedImages.push(image);
orderedIndexByNodeId.set(slot.nodeId, orderedImages.length);
providerMentionBySlot.set(`@图片${index + 1}`, `@图片${orderedImages.length}`);
});
const compiledPrompt = context.prompt.replace(/@图片[1-9]\d*/g, (token, offset: number) => hasMentionBoundary(context.prompt, offset + token.length) ? providerMentionBySlot.get(token) || token : token);
const supplementLabelsByNodeId = new Map<string, string[]>();
supplementEntries.slice(0, Math.max(0, remainingBudget - selectedEntries.length)).forEach((entry, index) => {
const label = imageReferenceLabel(orderedImages.length + index);
supplementLabelsByNodeId.set(entry.nodeId, [...(supplementLabelsByNodeId.get(entry.nodeId) || []), `@${label}`]);
});
const supplementImages = hydratedCharacterImages.slice(selectedEntries.length);
referenceImages = [...orderedImages, ...supplementImages];
const characterBlocks = details.map((detail, index) => {
const nodeId = context.characterReferences[index]?.nodeId;
const primaryLabel = nodeId ? orderedIndexByNodeId.get(nodeId) : undefined;
const supplementLabels = nodeId ? supplementLabelsByNodeId.get(nodeId) || [] : [];
const referenceLine = primaryLabel ? `参考图片：@${imageReferenceLabel(primaryLabel - 1)}${supplementLabels.length ? `；补充视角：${supplementLabels.join("、")}` : ""}` : "";
const heading = includeCharacterPrompt ? compileCharacterReferencePrompt(detail.asset.title, detail.character.definition) : `【角色卡：${detail.asset.title}】`;
return [heading, referenceLine].filter(Boolean).join("\n");
});
const resolvedCharacterVersions = details.map((detail) => ({ assetId: detail.asset.id, versionId: detail.character.versionId }));
const resolvedCharacterVoices = details.flatMap((detail): ResolvedCharacterVoice[] => {
const voice = detail.character.voice;
@@ -567,10 +569,17 @@ export async function hydrateNodeGenerationContext(context: NodeGenerationContex
} satisfies ReferenceAudio;
}));
const referenceAudios = [...context.referenceAudios, ...characterVoiceAudios];
const voiceBlocks = mode === "video" ? resolvedCharacterVoices.map(compileResolvedVoicePrompt) : [];
const voiceBlocks = mode === "video" ? resolvedCharacterVoices.flatMap((voice) => {
const sampleIndex = voice.sampleResourceId ? referenceAudios.findIndex((audio) => resourceIdFromStorageKey(audio.storageKey) === voice.sampleResourceId) : -1;
if (sampleIndex < 0 && !includeCharacterPrompt) return [];
return [[
includeCharacterPrompt ? compileResolvedVoicePrompt(voice) : `【角色声音：${voice.characterName}】`,
sampleIndex >= 0 ? `声音参考：@音频${sampleIndex + 1}` : "",
].filter(Boolean).join("\n")];
}) : [];
return {
...context,
prompt: includeCharacterPrompt ? [context.prompt.trim(), ...characterBlocks, ...voiceBlocks].filter(Boolean).join("\n\n") : context.prompt,
prompt: [compiledPrompt.trim(), ...characterBlocks, ...voiceBlocks].filter(Boolean).join("\n\n"),
referenceImages,
referenceAudios,
resolvedCharacterVersions,
@@ -592,7 +601,7 @@ function readCharacterReference(node: CanvasNodeData): CharacterGenerationRefere
}
function preferredCharacterRepresentation(representations: Array<{ id: string; resourceId: string; role: string }>) {
return ["turnaround_sheet", "primary", "front", "side", "back"].map((role) => representations.find((item) => item.role === role)).find(Boolean);
return ["turnaround_sheet", "primary", "front", "side", "back"].map((role) => representations.find((item) => item.role === role && item.resourceId.trim())).find(Boolean);
}
function compileResolvedVoicePrompt(voice: ResolvedCharacterVoice) {

2
web/src/lib/canvas/canvas-character-reference.ts
View File

@@ -20,7 +20,7 @@ export type CharacterBreakdown = {
export function normalizeCharacterImageMentions(prompt: string, imageCount: number, characterLabels: string[]) {
if (imageCount > 0 || !characterLabels.length) return prompt;
return prompt.replace(/@图片([1-9]\d*)(?=$|\s|[,.!?;:，。！？；：、)\]}】）])/g, (token, index: string) => {
return prompt.replace(/@图片([1-9]\d*)(?![0-9])/g, (token, index: string) => {
const label = characterLabels[Number(index) - 1];
return label ? "@" + label : token;
});

11
web/src/lib/canvas/canvas-reference-slots.ts Normal file
View File

@@ -0,0 +1,11 @@
export type CanvasReferenceLabelKind = "image" | "character" | "drawing" | "video" | "audio" | "text" | "skill" | "tool";
/** 图片、角色主图和绘图共享图片数组的位置；保留类型前缀供编辑器识别。 */
export function createCanvasReferenceLabeler() {
const counts: Record<CanvasReferenceLabelKind, number> = { image: 0, character: 0, drawing: 0, video: 0, audio: 0, text: 0, skill: 0, tool: 0 };
const prefixes: Record<CanvasReferenceLabelKind, string> = { image: "图片", character: "角色", drawing: "绘图", video: "视频", audio: "音频", text: "文本", skill: "技能", tool: "工具" };
return (kind: CanvasReferenceLabelKind) => {
const counter = kind === "character" || kind === "drawing" ? "image" : kind;
return `${prefixes[kind]}${++counts[counter]}`;
};
}

21
web/src/lib/canvas/canvas-resource-references.ts
View File

@@ -1,10 +1,9 @@
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
import { createCanvasReferenceLabeler } from "@/lib/canvas/canvas-reference-slots";
import { normalizeCharacterImageMentions } from "@/lib/canvas/canvas-character-reference";
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import { nodeGenerationPrompt } from "@/lib/canvas/generation-contract";
import { writeCanvasNodePrompt } from "@/lib/canvas/canvas-node-prompt";
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
import { seedanceReferenceLabel } from "@/lib/seedance-video";
import type { Skill } from "@/services/api/skills";
import type { Asset, AssetCategory } from "@/stores/use-asset-store";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData, type CanvasNodeTypeId } from "@/types/canvas";
@@ -292,7 +291,7 @@ export function replaceCanvasMentionToken(value: string, token: string, replacem
if (!token) return value;
const escapedToken = token.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
// Numbered media mentions can touch Chinese prose or another mention, but not a longer number.
const boundary = /^@(图片|视频|音频|文本)\d+$/.test(token)
const boundary = /^@(图片|角色|绘图|视频|音频|文本)\d+$/.test(token)
? "(?![0-9])"
: token.startsWith("@[node:") ? "" : `(?=${CANVAS_RESOURCE_MENTION_BOUNDARY.source})`;
return value.replace(new RegExp(`${escapedToken}${boundary}`, "gu"), replacement);
@@ -525,13 +524,11 @@ export function reorderCanvasResourceConnections(targetNodeId: string, orderedNo
}
function labelResourceNodes(nodes: CanvasNodeData[], active: boolean) {
const counts: Record<CanvasResourceKind, number> = { image: 0, video: 0, audio: 0, text: 0, skill: 0, character: 0, tool: 0 };
let drawingCount = 0;
const nextLabel = createCanvasReferenceLabeler();
return nodes.flatMap((node): CanvasResourceReference[] => {
const kind = resourceKind(node);
if (!kind) return [];
const index = node.type === CanvasNodeType.Drawing ? drawingCount++ : counts[kind]++;
const label = node.type === CanvasNodeType.Drawing ? `绘图${index + 1}` : labelForKind(kind, index);
const label = nextLabel(node.type === CanvasNodeType.Drawing ? "drawing" : kind);
return [
{
id: node.id,
@@ -558,16 +555,6 @@ function labelResourceNodes(nodes: CanvasNodeData[], active: boolean) {
});
}
function labelForKind(kind: CanvasResourceKind, index: number) {
if (kind === "character") return `角色${index + 1}`;
if (kind === "image") return imageReferenceLabel(index);
if (kind === "video") return seedanceReferenceLabel("video", index);
if (kind === "audio") return seedanceReferenceLabel("audio", index);
if (kind === "skill") return `技能${index + 1}`;
if (kind === "tool") return `工具${index + 1}`;
return `文本${index + 1}`;
}
function isResourceNode(node: CanvasNodeData) {
return Boolean(resourceKind(node));
}

185
web/test/canvas-character-generation.test.ts Normal file
View File

@@ -0,0 +1,185 @@
import { afterEach, describe, expect, spyOn, test } from "bun:test";
import { buildNodeGenerationContext, hydrateNodeGenerationContext } from "../src/components/canvas/canvas-node-generation";
import { prepareBackendGenerationTask } from "../src/services/api/generation-task";
import { http } from "../src/services/api/request";
import type { ProjectCharacterDetail } from "../src/services/api/projects";
import * as imageStorage from "../src/services/image-storage";
import { createModelChannel, defaultConfig, encodeChannelModel } from "../src/stores/use-config-store";
import type { Asset } from "../src/stores/use-asset-store";
import { CanvasNodeType, type CanvasNodeData } from "../src/types/canvas";
const restores: Array<() => void> = [];
afterEach(() =>
restores
.splice(0)
.reverse()
.forEach((restore) => restore()),
);
function node(id: string, type: CanvasNodeType, metadata: CanvasNodeData["metadata"] = {}): CanvasNodeData {
return { id, type, title: id, metadata, position: { x: 0, y: 0 }, width: 100, height: 100 };
}
function characterNode(id: string) {
return node(id, CanvasNodeType.Text, { workflowKind: "character", characterAssetId: id });
}
function detail(id: string, representations: Array<[string, string]> = [["primary", `${id}-main`]]): ProjectCharacterDetail {
return {
asset: { id, title: id, category: "character", mediaType: "entity", status: "ready", versionCount: 1, usages: [], position: 0, updatedAt: "" },
character: {
versionId: `${id}-version`,
version: 1,
definition: { appearance: "角色外观设定" },
visualStatus: "ready",
voiceStatus: "missing",
representations: representations.map(([role, resourceId], index) => ({ id: `${id}-${index}`, role, resourceId, mediaType: "image/png" })),
},
};
}
function mockResources(details: ProjectCharacterDetail[]) {
const get = spyOn(http, "get").mockImplementation(async (url) => {
const character = details.find((item) => url === `/characters/${item.asset.id}`);
if (character) return character as never;
if (url === "/resources/shared-voice") return { resource: { id: "shared-voice", mimeType: "audio/wav", objectKey: "voice.wav" } } as never;
throw new Error(`Unexpected request: ${url}`);
});
const images = spyOn(imageStorage, "imageToDataUrl").mockImplementation(async (image) => {
// Resource I/O is stubbed; ordering, character resolution and task serialization run unchanged.
return `data:image/png;base64,${btoa(image.storageKey || image.dataUrl || "image")}`;
});
restores.push(
() => get.mockRestore(),
() => images.mockRestore(),
);
return { get, images };
}
function contextFor(sources: CanvasNodeData[], prompt: string, assets: Asset[] = [], mode = CanvasNodeType.Video) {
const target = node("target", mode);
const connections = sources.map((source) => ({ id: `edge-${source.id}`, fromNodeId: source.id, toNodeId: target.id }));
return buildNodeGenerationContext(target.id, [...sources, target], connections, prompt, assets, true);
}
function videoConfig() {
const channel = createModelChannel({
id: "video-test",
name: "Video",
baseUrl: "https://example.com",
apiKey: "test-key",
interfaceType: "volcengine-ark-video",
models: ["video-model"],
modelCosts: [{ model: "video-model", capability: "video", protocol: "volcengine-ark-video", billingMode: "fixed_request", unitPriceMicrocredits: 1 }],
});
const model = encodeChannelModel(channel.id, "video-model");
return { ...defaultConfig, channels: [channel], model, videoModel: model };
}
describe("canvas character reference delivery", () => {
test("mixed references preserve primary slots through hydration and backend task preparation", async () => {
const a = detail("a", [
["primary", "a-main"],
["front", "a-front"],
["side", "a-front"],
["back", ""],
]);
const b = detail("b", [
["primary", "b-main"],
["front", "b-front"],
]);
const voice = {
profile: { id: "voice", name: "声音", provider: "sample", voiceKey: "voice-key", language: "中文", timbre: "低沉", sampleResourceId: "shared-voice", compatibleModels: [], status: "ready" },
instructions: "轻声朗读",
};
a.character.voice = voice;
b.character.voice = voice;
mockResources([a, b]);
const asset: Asset = {
id: "library-image",
kind: "image",
title: "素材图",
tags: [],
createdAt: "",
updatedAt: "",
data: { dataUrl: "", storageKey: "resource:library", mimeType: "image/png", bytes: 1, width: 1, height: 1 },
};
const context = contextFor(
[characterNode("a"), node("scene", CanvasNodeType.Image, { storageKey: "resource:scene" }), characterNode("b"), node("audio", CanvasNodeType.Audio, { storageKey: "resource:music" })],
"@图片2场景里@角色3看向@角色1，参考@[asset:library-image]，配合@音频1",
[asset],
);
const hydrated = await hydrateNodeGenerationContext(context, "canvas", undefined, "video", true, false, { maxImages: 6, maxVideos: 1, maxAudios: 2 });
const expectedKeys = ["resource:a-main", "resource:scene", "resource:b-main", "resource:library", "resource:a-front", "resource:b-front"];
expect(hydrated.referenceImages.map((image) => image.storageKey)).toEqual(expectedKeys);
expect(hydrated.prompt).toStartWith("@图片2场景里@图片3看向@图片1，参考@图片4，配合@音频1");
expect(hydrated.prompt).toContain("【角色卡：a】\n参考图片：@图片1；补充视角：@图片5");
expect(hydrated.prompt).toContain("【角色卡：b】\n参考图片：@图片3；补充视角：@图片6");
expect(hydrated.prompt).toContain("【角色声音：a】\n声音参考：@音频2");
expect(hydrated.prompt).toContain("【角色声音：b】\n声音参考：@音频2");
expect(hydrated.prompt).not.toContain("角色外观设定");
expect(hydrated.prompt).not.toContain("轻声朗读");
expect(hydrated.referenceAudios.map((audio) => audio.storageKey)).toEqual(["resource:music", "resource:shared-voice"]);
expect(hydrated.resolvedCharacterVersions).toEqual([
{ assetId: "a", versionId: "a-version" },
{ assetId: "b", versionId: "b-version" },
]);
const task = await prepareBackendGenerationTask({ mode: "video", config: videoConfig(), prompt: hydrated.prompt, referenceImages: hydrated.referenceImages, referenceAudios: hydrated.referenceAudios });
const input = task.input as { prompt: string; referenceImages: Array<{ storageKey: string }>; referenceAudios: Array<{ storageKey: string }> };
expect(input.prompt).toBe(hydrated.prompt);
expect(input.referenceImages.map((image) => image.storageKey)).toEqual(expectedKeys);
expect(input.referenceAudios.map((audio) => audio.storageKey)).toEqual(["resource:music", "resource:shared-voice"]);
});
test("reserved primary images cannot be displaced by supplements", async () => {
mockResources([
detail("a", [
["primary", "a-main"],
["front", "a-front"],
]),
detail("b"),
]);
const context = contextFor([characterNode("a"), node("scene", CanvasNodeType.Image, { storageKey: "resource:scene" }), characterNode("b")], "@角色3与@角色1");
const hydrated = await hydrateNodeGenerationContext(context, "canvas", undefined, "video", false, true, { maxImages: 3, maxVideos: 0, maxAudios: 0 });
expect(hydrated.referenceImages.map((image) => image.storageKey)).toEqual(["resource:a-main", "resource:scene", "resource:b-main"]);
expect(hydrated.prompt).not.toContain("补充视角");
await expect(hydrateNodeGenerationContext(context, "canvas", undefined, "video", false, true, { maxImages: 2, maxVideos: 0, maxAudios: 0 })).rejects.toThrow("参考图容量不足");
});
test("pinned version changes and missing image identities stop submission", async () => {
mockResources([detail("a")]);
const source = characterNode("a");
source.metadata = { ...source.metadata, characterVersionPolicy: "pinned", characterVersionId: "old-version" };
await expect(hydrateNodeGenerationContext(contextFor([source], "@角色1"), "canvas", undefined, "video")).rejects.toThrow("固定版本已变化");
});
test("image and video generation reject character cards without a usable main image", async () => {
mockResources([detail("a", [["primary", ""]])]);
const context = contextFor([characterNode("a")], "@角色1");
for (const mode of ["image", "video"] as const) {
await expect(hydrateNodeGenerationContext(context, "canvas", undefined, mode)).rejects.toThrow("缺少可用的主参考图");
}
});
test("text and audio may use an imageless character without shifting another image to the wrong mention", async () => {
mockResources([detail("a", []), detail("b")]);
const context = contextFor([characterNode("a"), node("scene", CanvasNodeType.Image, { storageKey: "resource:scene" }), characterNode("b")], "@角色1对@角色3说话，位于@图片2");
for (const mode of ["text", "audio"] as const) {
const hydrated = await hydrateNodeGenerationContext(context, "canvas", undefined, mode);
expect(hydrated.prompt).toStartWith("a对@图片2说话，位于@图片1");
expect(hydrated.referenceImages.map((image) => image.storageKey)).toEqual(["resource:scene", "resource:b-main"]);
expect(hydrated.prompt).toContain("【角色卡：b】\n角色外观设定\n参考图片：@图片2");
}
});
test("character authorization and image delivery failures remain errors", async () => {
const { get, images } = mockResources([detail("a")]);
const context = contextFor([characterNode("a")], "@角色1");
get.mockRejectedValueOnce(new Error("无权读取角色"));
await expect(hydrateNodeGenerationContext(context, "canvas", undefined, "video")).rejects.toThrow("无权读取角色");
images.mockRejectedValueOnce(new Error("图片下载失败"));
await expect(hydrateNodeGenerationContext(context, "canvas", undefined, "video")).rejects.toThrow("图片下载失败");
});
});

26
web/test/canvas-node-generation-mentions.test.ts
View File

@@ -43,24 +43,44 @@ describe("canvas node generation position mentions", () => {
const prompt = "严格参考 @图片1，保持服装一致。";
expect(normalizeCanvasNodeMentionTokens(prompt, references)).toBe("严格参考 @角色1，保持服装一致。");
const context = buildNodeGenerationContext(target.id, nodes, connections, prompt, []);
expect(context.prompt).toBe("严格参考 @角色1，保持服装一致。");
expect(context.prompt).toBe("严格参考 @图片1，保持服装一致。");
expect(context.characterReferences).toEqual([{ nodeId: character.id, assetId: "character-asset", requestedVersionId: undefined }]);
expect(() => buildNodeGenerationContext(target.id, nodes, connections, "参考 @图片2", [])).toThrow("@图片2");
});
test("角色卡与普通图片混合时，图片编号不指向角色卡", () => {
test("角色卡与普通图片混合时共享图片序号，旧独立编号拒绝猜测迁移", () => {
const target = targetNode();
const character = node("character", CanvasNodeType.Text, "");
character.metadata = { workflowKind: "character", characterAssetId: "character-asset" };
const image = node("scene", CanvasNodeType.Image, "data:image/png;base64,AA==");
const nodes = [target, character, image];
const connections = [connection(character.id), connection(image.id)];
const prompt = "@角色1 站在 @图片1 场景内。";
const prompt = "@角色1站在@图片2场景内。";
const references = buildCanvasResourceReferences(nodes, connections, target.id);
expect(normalizeCanvasNodeMentionTokens(prompt, references)).toBe(prompt);
const context = buildNodeGenerationContext(target.id, nodes, connections, prompt, []);
expect(context.prompt).toBe("@图片1站在@图片2场景内。");
expect(context.referenceImages.map((item) => item.id)).toEqual([image.id]);
expect(context.characterReferences.map((item) => item.nodeId)).toEqual([character.id]);
expect(() => buildNodeGenerationContext(target.id, nodes, connections, "@角色1站在@图片1场景内", [])).toThrow("@图片1");
});
test("混合角色、绘图和图片按连接位置编译，逆序提及不会重排资源", () => {
const character = node("character", CanvasNodeType.Text, "");
character.metadata = { workflowKind: "character", characterAssetId: "character-asset" };
const drawing = node("drawing", CanvasNodeType.Drawing, "");
drawing.metadata = { drawingId: "drawing-document" };
const image = node("scene", CanvasNodeType.Image, "data:image/png;base64,AA==");
const target = targetNode();
const nodes = [target, image, drawing, character];
const connections = [character, drawing, image].map((source) => connection(source.id));
const references = buildCanvasResourceReferences(nodes, connections, target.id).filter((reference) => reference.active);
expect(Object.fromEntries(references.map((reference) => [reference.nodeId, reference.label]))).toEqual({ character: "角色1", drawing: "绘图2", scene: "图片3" });
const context = buildNodeGenerationContext(target.id, nodes, connections, "@图片3背景，@绘图2@角色1保持一致", [], true);
expect(context.prompt).toBe("@图片3背景，@图片2@图片1保持一致");
expect(context.referenceImageSlots.map((slot) => slot.nodeId)).toEqual([character.id, drawing.id, image.id]);
expect(context.referenceImages.map((item) => item.id)).toEqual([drawing.id, image.id]);
expect(() => buildNodeGenerationContext(target.id, nodes, connections, "@角色10保持一致", [], true)).toThrow("@角色10");
});
test("父图无预览时仍可继承输入，显式子图输入优先且去重", () => {

14
web/test/canvas-resource-references.test.ts
View File

@@ -356,6 +356,20 @@ describe("image generation reference connections", () => {
});
describe("reorder canvas resource connections", () => {
test("混合角色、绘图和图片拖动后按节点身份同步所有提及", () => {
const character = { ...textNode("character"), metadata: { workflowKind: "character" as const, characterAssetId: "character-asset" } };
const drawing: CanvasNodeData = { ...imageNode("drawing"), type: CanvasNodeType.Drawing, metadata: { drawingId: "drawing-document" } };
const image = imageNode("scene");
const target = { ...videoNode("target"), metadata: { composerContent: "@角色1参考@绘图2，背景@图片3，@角色1@绘图2" } };
const nodes = [character, drawing, image, target];
const before = [character, drawing, image].map((source) => connection(source.id, target.id));
const after = reorderCanvasResourceConnections(target.id, [image.id, character.id, drawing.id], nodes, before);
const synced = applyCanvasConnectionPromptSync(nodes, before, nodes, after);
expect(synced.find((item) => item.id === target.id)?.metadata?.composerContent).toBe("@角色2参考@绘图3，背景@图片1，@角色2@绘图3");
expect(replaceCanvasMentionToken("@角色1@角色10@绘图2@绘图20", "@角色1", "@角色2")).toBe("@角色2@角色10@绘图2@绘图20");
expect(replaceCanvasMentionToken("@绘图2@绘图20", "@绘图2", "@绘图3")).toBe("@绘图3@绘图20");
});
test("直接引用按指定顺序换位并同步编号提示词", () => {
const imageA = imageNode("image-a");
const imageB = imageNode("image-b");

2
web/test/canvas-storyboard-assets.test.ts
View File

@@ -70,7 +70,7 @@ describe("storyboard target materializer", () => {
it("combines stable bindings and produces position mention tokens", () => {
const references = storyboardRowReferenceNodeIds(script, row, nodes, connections, true);
expect(references).toEqual(["project-style", "prop", "character", "manual", "first-frame"]);
expect(storyboardComposerContent("快速跟拍", references, nodes)).toBe("参考资产：@图片1 @图片2 @角色1 @视频1 @图片3\n快速跟拍");
expect(storyboardComposerContent("快速跟拍", references, nodes)).toBe("参考资产：@图片1 @图片2 @角色3 @视频1 @图片4\n快速跟拍");
const withDirectManualInput = storyboardRowReferenceNodeIds(script, row, nodes, [...connections, { id: "direct", fromNodeId: "direct-manual", toNodeId: "target" }], false, "target");
expect(withDirectManualInput).toContain("direct-manual");