tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(canvas): 媒体生成 - 修复已有图片引用和图片音频视频模型匹配

Browse Source
This commit is contained in:
ddCat
2026-08-29 14:49:59 +08:00
parent d66c7adc2c
commit d84b2b69cc
8 changed files with 159 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
web/package.json
View File

@@ -8,7 +8,7 @@
"dev": "vite --host 0.0.0.0 --port 3000",
"build": "bun run build:bridge && tsc --noEmit && vite build",
"typecheck": "tsc --noEmit",
"test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
"test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
"test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs",
"start": "vite preview --host 0.0.0.0 --port 3000",
"format": "prettier --write .",

52
web/src/components/canvas/canvas-node-generation.ts
View File

@@ -4,7 +4,7 @@ import { seedanceReferenceLabel } from "@/lib/seedance-video";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
import { CanvasNodeType, type CanvasConnection, type CanvasGenerationMode, type CanvasNodeData } from "@/types/canvas";
import { getGenerationResourceNodes, getContextResourceNodes } from "@/lib/canvas/canvas-resource-references";
import { getGenerationResourceNodes, getContextResourceNodes, getMentionResourceNodes } from "@/lib/canvas/canvas-resource-references";
import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade";
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
@@ -69,14 +69,17 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
return image ? [{ nodeId: sourceNode.id, type: "image" as const, title: sourceNode.title, image }] : [];
})()
: [];
const inputs = [...connectedInputs, ...portraitTextureInput, ...buildAssetGenerationInputs(assets)];
// 显式 @ 引用必须与提示词面板展示的资源集合一致；默认自动输入仍只取入边，
// 避免已有图片在没有 @图片N 时被悄悄当作自身参考图。
const mentionInputs = mergeGenerationInputs(buildNodeMentionGenerationInputs(nodeId, nodes, connections), portraitTextureInput, buildAssetGenerationInputs(assets));
const storyboardInputs = getConnectedStoryboardRows(nodeId, nodes, connections);
const hasExplicitResourceMention = hasResolvableGenerationMention(prompt, inputs);
assertResolvableGenerationMentions(prompt, mentionInputs);
const hasExplicitResourceMention = hasResolvableGenerationMention(prompt, mentionInputs);
const isWorkflowSource = sourceNode?.type === CanvasNodeType.Config && isCanvasWorkflowProvider(sourceNode.metadata);
if ((Boolean(sourceNode?.metadata?.composerContent?.trim()) && (sourceNode?.type === CanvasNodeType.Config || isWorkflowSource)) || hasExplicitResourceMention) {
const autoIncludeWorkflowMedia = isWorkflowSource;
return buildComposerGenerationContext(
inputs,
mentionInputs,
prompt,
// 工作流节点由字段映射接收全部连线媒体；视频节点的历史首尾帧字段不能再额外追加参考图。
autoIncludeWorkflowMedia ? [] : [sourceNode?.metadata?.videoStartFrameNodeId, sourceNode?.metadata?.videoEndFrameNodeId].filter((id): id is string => Boolean(id)),
@@ -260,14 +263,31 @@ export function normalizeGenerationNodeMentionTokens(prompt: string, inputs: Nod
}
function hasResolvableGenerationMention(prompt: string, inputs: NodeGenerationInput[]) {
return inspectGenerationMentions(prompt, inputs).hasResolved;
}
function assertResolvableGenerationMentions(prompt: string, inputs: NodeGenerationInput[]) {
const { unresolved } = inspectGenerationMentions(prompt, inputs);
if (!unresolved.length) return;
throw new Error(`提示词中的 ${unresolved.join("、")} 没有对应的画布资源，请重新选择引用后再生成`);
}
function inspectGenerationMentions(prompt: string, inputs: NodeGenerationInput[]) {
const normalizedPrompt = normalizeGenerationNodeMentionTokens(prompt, inputs);
const slotInputByToken = new Map(generationSlotEntries(inputs).map(({ token, input }) => [token, input]));
const nodeInputById = new Map(inputs.filter((input) => !input.nodeId.startsWith("asset:")).map((input) => [input.nodeId, input]));
const assetInputById = new Map(inputs.filter((input) => input.nodeId.startsWith("asset:")).map((input) => [input.nodeId.slice("asset:".length), input]));
const unresolved = new Set<string>();
let hasResolved = false;
for (const match of normalizedPrompt.matchAll(GENERATION_MENTION_PATTERN)) {
if (resolveGenerationMention(normalizedPrompt, match, slotInputByToken, nodeInputById, assetInputById)) return true;
if (match[3]) {
const end = (match.index || 0) + match[0].length;
if (!hasMentionBoundary(normalizedPrompt, end)) continue;
}
if (resolveGenerationMention(normalizedPrompt, match, slotInputByToken, nodeInputById, assetInputById)) hasResolved = true;
else unresolved.add(match[0]);
}
return false;
return { hasResolved, unresolved: [...unresolved] };
}
function generationSlotEntries(inputs: NodeGenerationInput[]) {
@@ -302,7 +322,14 @@ function hasMentionBoundary(value: string, index: number) {
}
export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
const resourceNodes = getGenerationResourceNodes(nodeId, nodes, connections);
return buildGenerationInputs(getGenerationResourceNodes(nodeId, nodes, connections), nodes, connections);
}
function buildNodeMentionGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
return buildGenerationInputs(getMentionResourceNodes(nodeId, nodes, connections), nodes, connections);
}
function buildGenerationInputs(resourceNodes: CanvasNodeData[], nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
return resourceNodes.flatMap((node): NodeGenerationInput[] => {
const character = readCharacterReference(node);
if (character) return [{ nodeId: node.id, type: "character" as const, title: node.title, character }];
@@ -320,6 +347,17 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[
});
}
function mergeGenerationInputs(...groups: NodeGenerationInput[][]) {
const seen = new Set<string>();
return groups.flatMap((group) =>
group.filter((input) => {
if (seen.has(input.nodeId)) return false;
seen.add(input.nodeId);
return true;
}),
);
}
function buildAssetGenerationInputs(assets: Asset[]): NodeGenerationInput[] {
return assets.flatMap((asset): NodeGenerationInput[] => {
const nodeId = `asset:${asset.id}`;

6
web/src/lib/model-selection.ts
View File

@@ -319,11 +319,11 @@ export function modelGroupReferenceLimits(config: AiConfig, selected: string, ca
export function inferVideoOperation(input: ModelInputSummary) {
const visualInputCount = input.imageCount + input.characterCount;
// 纯音频参考使用独立能力；音频与图片、角色或视频组合时属于全模态参考，
// 不能把组合请求误路由到只支持 audio_to_video 的细分模型。
if (input.audioCount > 0) return visualInputCount > 0 || input.videoCount > 0 ? "reference_to_video" : "audio_to_video";
// 图片或角色决定图生视频主模式，音频只作为附加参考，不应把组合请求
// 提升为全模态参考；纯音频输入才使用独立的 audio_to_video 能力。
if (input.videoCount > 0 || visualInputCount > 2) return "reference_to_video";
if (visualInputCount > 0) return "image_to_video";
if (input.audioCount > 0) return "audio_to_video";
return "text_to_video";
}

18
web/test/canvas-model-policy.test.ts
View File

@@ -216,10 +216,26 @@ describe("逻辑模型选择", () => {
test("音频仅在单独参考时归类为音频生视频", () => {
expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("audio_to_video");
expect(inferVideoOperation({ textCount: 1, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
expect(inferVideoOperation({ textCount: 1, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("image_to_video");
expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 1, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
});
test("图片加音频可匹配支持图生视频和参考音频的模型", () => {
const config = policyConfig();
const imageModel = "relay::cinema-image";
const imageCost = config.channels[0]?.modelCosts?.find((item) => item.model === "cinema-image");
if (!imageCost?.capabilityConfig?.video) throw new Error("缺少图生视频能力配置");
imageCost.capabilityConfig.video.references.maxAudios = 1;
const requirements = {
capability: "video" as const,
input: { textCount: 0, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 },
videoSeconds: "6",
};
expect(modelCompatibilityError(config, imageModel, requirements)).toBe("");
expect(resolveCompatibleModel(config, "relay::cinema-text", requirements)).toBe(imageModel);
});
test("逻辑视频模型将 720 与 720p 视为同一分辨率", () => {
const model = "cinema-720p";
const channel: ModelChannel = {

25
web/test/canvas-node-generation-mentions.test.ts
View File

@@ -32,6 +32,31 @@ function connection(fromNodeId: string): CanvasConnection {
}
describe("canvas node generation position mentions", () => {
test("已有图片节点显式引用自身时作为图生图参考图提交", () => {
const source = node("image-self", CanvasNodeType.Image, "data:image/png;base64,a");
source.metadata.composerContent = "将 @图片1 图片变清晰";
const context = buildNodeGenerationContext(source.id, [source], [], source.metadata.composerContent, []);
expect(context.referenceImages.map((image) => image.id)).toEqual([source.id]);
expect(context.imageCount).toBe(1);
expect(context.prompt).toBe("将 图片1 图片变清晰");
});
test("已有图片节点未显式引用自身时不自动退化为图生图", () => {
const source = node("image-self", CanvasNodeType.Image, "data:image/png;base64,a");
const context = buildNodeGenerationContext(source.id, [source], [], "生成一个新的构图", []);
expect(context.referenceImages).toEqual([]);
expect(context.imageCount).toBe(0);
expect(context.prompt).toBe("生成一个新的构图");
});
test("无法解析的画布引用会阻止静默降级为文生图", () => {
const target = targetNode();
expect(() => buildNodeGenerationContext(target.id, [target], [], "将 @图片1 图片变清晰", [])).toThrow("@图片1 没有对应的画布资源");
});
test("同一个 @图片1 在换线后自动指向新的第一张图片", () => {
const target = targetNode();
const imageA = node("image-a", CanvasNodeType.Image, "data:image/png;base64,a");

2
web/test/generation-task-local.test.ts
View File

@@ -1325,7 +1325,7 @@ test("remote image video and audio references keep Backend parity without Dreami
referenceAudios: [{ id: "remote-image-audio-audio-0001", name: "reference.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-image-audio-audio-0001" }],
},
result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-image-audio-output" } },
operation: "reference_to_video",
operation: "image_to_video",
},
{
mode: "audio" as const,