tt/ttstd-infinite-canvas
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): 分镜脚本 - 关联画布资产并自动物化视频引用

Browse Source
This commit is contained in:
ddCat committed 2026-08-25 11:24:47 +08:00
1 parent 03597c6d51
commit 625a97560b
21 files changed
+684 -249
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

web/src/pages/canvas/project.tsx
+2
View File
@@ -1601,6 +1601,7 @@ function InfiniteCanvasPage() {
return (
<CanvasScriptNodeContent
node={contentNode}
nodes={nodesRef.current}
batch={visibleGenerationBatch(contentNode)}
pipeline={pipeline}
scale={viewport.k}
@@ -2385,6 +2386,7 @@ function InfiniteCanvasPage() {
<CanvasScriptEditor
node={activeScriptNode}
nodes={nodes}
open={Boolean(activeScriptNode)}
onClose={() => setScriptEditorNodeId(null)}
onUpdateRows={(rows) => activeScriptNode && replaceScriptRows(activeScriptNode.id, rows)}
web/src/pages/canvas/use-canvas-storyboard.ts
+32 -61
View File
@@ -8,7 +8,7 @@ import {
buildGenerationConfig,
generationTaskMetadata,
resetGenerationTaskMetadata,
logicalModelIDForConfig,
logicalModelIDForConfig,
} from "@/lib/canvas/canvas-project-generation";
import {
cinematicStoryboardColumns,
@@ -19,7 +19,9 @@ import {
storyboardPromptTemplateMetadata,
} from "@/lib/canvas/canvas-project-domain";
import { buildNodeMentionReferences } from "@/lib/canvas/canvas-resource-references";
import { buildStoryboardAssetCatalog } from "@/lib/canvas/canvas-storyboard-assets";
import { resolveStoryboardGenerationContext } from "@/lib/canvas/canvas-storyboard-context";
import { reconcileStoryboardTargetConnections, storyboardComposerContent, storyboardRowReferenceNodeIds } from "@/lib/canvas/canvas-storyboard-materializer";
import { generationErrorMessage } from "@/lib/generation-error";
import { navigateToSettings } from "@/lib/settings-navigation";
import { createGenerationTask, waitForGenerationTask } from "@/services/api/task-center";
@@ -80,7 +82,7 @@ export function useCanvasStoryboard({
...node.metadata,
storyboard: {
rows: updater(node.metadata?.storyboard?.rows || []),
visibleColumns: node.metadata?.storyboard?.visibleColumns || ["shotNumber", "plotDescription", "videoMotionPrompt", "dialogue"],
visibleColumns: node.metadata?.storyboard?.visibleColumns || ["shotNumber", "durationSeconds", "videoMotionPrompt", "dialogue", "assets"],
referenceNodeIds: node.metadata?.storyboard?.referenceNodeIds || [],
},
},
@@ -89,9 +91,11 @@ export function useCanvasStoryboard({
const replaceScriptRows = useCallback((nodeId: string, rows: StoryboardRow[]) => {
const rowIds = new Set(rows.map((row) => `row:${row.id}`));
const storyboardRowIds = new Set(rows.map((row) => row.id));
const previousRows = new Map((nodesRef.current.find((node) => node.id === nodeId)?.metadata?.storyboard?.rows || []).map((row) => [row.id, row]));
const nextRows = rows.map((row) => invalidateEditedPromptVariables(previousRows.get(row.id), row));
setConnections((current) => current
.filter((connection) => !connection.storyboardRowId || storyboardRowIds.has(connection.storyboardRowId))
.filter((connection) => connection.fromNodeId !== nodeId || !connection.fromHandleId || rowIds.has(connection.fromHandleId))
.filter((connection) => connection.toNodeId !== nodeId || !connection.toHandleId || rowIds.has(connection.toHandleId)));
updateScriptRows(nodeId, () => nextRows);
@@ -139,9 +143,9 @@ export function useCanvasStoryboard({
operation: "storyboard_rows",
prompt: expandedPrompt,
model: generationConfig.model,
...(logicalModelIDForConfig(generationConfig) ? { logicalModelId: logicalModelIDForConfig(generationConfig) } : {}),
...(logicalModelIDForConfig(generationConfig) ? { logicalModelId: logicalModelIDForConfig(generationConfig) } : {}),
input: {
canvasSnapshot: { nodes: nodesRef.current, connections: connectionsRef.current },
canvasAssets: buildStoryboardAssetCatalog(nodesRef.current),
requirements: "输出可直接编辑并用于批量生成图片和视频的分镜表。",
projectStyle: storyboardContext.projectStyle,
characters: storyboardContext.characters,
@@ -189,32 +193,25 @@ export function useCanvasStoryboard({
const imageSpec = NODE_DEFAULT_SIZE[CanvasNodeType.Image];
const startX = scriptNode.position.x + scriptNode.width + 120;
const nextNodes = [...nodesRef.current];
const nextConnections = [...connectionsRef.current];
let nextConnections = [...connectionsRef.current];
const targets: Array<{ row: StoryboardRow; node: CanvasNodeData; prompt: string }> = [];
rows.forEach((row, index) => {
const prompt = (row.imageGenerationPrompt || row.plotDescription).trim();
const existing = row.imageNodeId ? nextNodes.find((node) => node.id === row.imageNodeId && node.type === CanvasNodeType.Image) : undefined;
const existingMetadata = existing?.metadata?.content ? existing.metadata : resetGenerationTaskMetadata(existing?.metadata);
const referenceIds = storyboardRowReferenceNodeIds(scriptNode, row, nextNodes, nextConnections, false, existing?.id);
const composerContent = storyboardComposerContent(prompt, referenceIds);
const imageNode = existing
? { ...existing, metadata: { ...existingMetadata, prompt, ...storyboardPromptTemplateMetadata(row, "image"), workflowKind: "shot" as const, workflowTitle: `镜头 ${row.shotNumber} 分镜图`, shotIndex: row.shotNumber } }
: createCanvasNode(CanvasNodeType.Image, { x: startX + imageSpec.width / 2, y: scriptNode.position.y + index * (imageSpec.height + 36) + imageSpec.height / 2 }, { prompt, ...storyboardPromptTemplateMetadata(row, "image"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 分镜图`, shotIndex: row.shotNumber, status: NODE_STATUS_IDLE });
? { ...existing, metadata: { ...existingMetadata, prompt, composerContent, ...storyboardPromptTemplateMetadata(row, "image"), workflowKind: "shot" as const, workflowTitle: `镜头 ${row.shotNumber} 分镜图`, shotIndex: row.shotNumber } }
: createCanvasNode(CanvasNodeType.Image, { x: startX + imageSpec.width / 2, y: scriptNode.position.y + index * (imageSpec.height + 36) + imageSpec.height / 2 }, { prompt, composerContent, ...storyboardPromptTemplateMetadata(row, "image"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 分镜图`, shotIndex: row.shotNumber, status: NODE_STATUS_IDLE });
if (!existing) {
imageNode.title = `镜头 ${row.shotNumber} · 分镜图`;
nextNodes.push(imageNode);
nextConnections.push({ id: nanoid(), fromNodeId: scriptNode.id, toNodeId: imageNode.id, fromHandleId: `row:${row.id}` });
} else {
const existingIndex = nextNodes.findIndex((node) => node.id === existing.id);
nextNodes[existingIndex] = imageNode;
}
const referenceIds = new Set([
...(scriptNode.metadata?.storyboard?.referenceNodeIds || []),
...(row.referenceNodeIds || []),
...characterReferenceNodeIds(row, nextNodes),
...nextConnections.filter((connection) => connection.toNodeId === scriptNode.id && connection.toHandleId === `row:${row.id}`).map((connection) => connection.fromNodeId),
]);
referenceIds.forEach((referenceId) => {
if (referenceId !== imageNode.id && !nextConnections.some((connection) => connection.fromNodeId === referenceId && connection.toNodeId === imageNode.id)) nextConnections.push({ id: nanoid(), fromNodeId: referenceId, toNodeId: imageNode.id });
});
nextConnections = reconcileStoryboardTargetConnections(nextConnections, scriptNode, row, imageNode.id, referenceIds);
targets.push({ row, node: imageNode, prompt });
});
const imageNodeByRowId = new Map(targets.map((target) => [target.row.id, target.node.id]));
@@ -225,7 +222,7 @@ export function useCanvasStoryboard({
...scriptNode.metadata,
storyboard: {
rows: (scriptNode.metadata?.storyboard?.rows || []).map((row) => ({ ...row, imageNodeId: imageNodeByRowId.get(row.id) || row.imageNodeId })),
visibleColumns: scriptNode.metadata?.storyboard?.visibleColumns || ["shotNumber", "plotDescription", "videoMotionPrompt", "dialogue"],
visibleColumns: scriptNode.metadata?.storyboard?.visibleColumns || ["shotNumber", "durationSeconds", "videoMotionPrompt", "dialogue", "assets"],
referenceNodeIds: scriptNode.metadata?.storyboard?.referenceNodeIds || [],
},
},
@@ -280,29 +277,27 @@ export function useCanvasStoryboard({
const startLeft = scriptNode.position.x + scriptNode.width + 120;
const videoModel = buildGenerationConfig(effectiveConfig, undefined, "video").model;
const nextNodes = [...nodesRef.current];
const nextConnections = [...connectionsRef.current];
let nextConnections = [...connectionsRef.current];
const videoNodeByRowId = new Map<string, string>();
let createdCount = 0;
rows.forEach((row, index) => {
const prompt = (row.videoMotionPrompt || row.plotDescription).trim();
const existingIndex = row.videoNodeId ? nextNodes.findIndex((node) => node.id === row.videoNodeId && node.type === CanvasNodeType.Video) : -1;
const existing = existingIndex >= 0 ? nextNodes[existingIndex] : undefined;
const referenceIds = storyboardRowReferenceNodeIds(scriptNode, row, nextNodes, nextConnections, false, existing?.id);
const composerContent = storyboardComposerContent(prompt, referenceIds);
if (existingIndex >= 0) {
const existing = nextNodes[existingIndex];
const existingMetadata = existing.metadata?.content ? existing.metadata : resetGenerationTaskMetadata(existing.metadata);
nextNodes[existingIndex] = { ...existing, metadata: { ...existingMetadata, prompt, composerContent: prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), seconds: String(row.durationSeconds), shotIndex: row.shotNumber, workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, generationMode: "video", videoEditOperation: existing.metadata?.videoEditOperation || "text_to_video" } };
storyboardRowReferenceNodeIds(scriptNode, row, nextNodes, nextConnections, false).forEach((referenceId) => {
if (!nextConnections.some((connection) => connection.fromNodeId === referenceId && connection.toNodeId === existing.id)) nextConnections.push({ id: nanoid(), fromNodeId: referenceId, toNodeId: existing.id });
});
nextNodes[existingIndex] = { ...existing, metadata: { ...existingMetadata, prompt, composerContent, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), seconds: String(row.durationSeconds), shotIndex: row.shotNumber, workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, generationMode: "video", videoEditOperation: existing.metadata?.videoEditOperation || "text_to_video" } };
nextConnections = reconcileStoryboardTargetConnections(nextConnections, scriptNode, row, existing.id, referenceIds);
videoNodeByRowId.set(row.id, existing.id);
return;
}
const videoNode = createCanvasNode(CanvasNodeType.Video, { x: startLeft + videoSpec.width / 2, y: scriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, composerContent: prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "text_to_video", status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
const videoNode = createCanvasNode(CanvasNodeType.Video, { x: startLeft + videoSpec.width / 2, y: scriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, composerContent, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "text_to_video", status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
videoNode.title = `镜头 ${row.shotNumber} · 视频`;
nextNodes.push(videoNode);
nextConnections.push({ id: nanoid(), fromNodeId: scriptNode.id, toNodeId: videoNode.id, fromHandleId: `row:${row.id}` });
storyboardRowReferenceNodeIds(scriptNode, row, nextNodes, nextConnections, false).forEach((referenceId) => {
if (!nextConnections.some((connection) => connection.fromNodeId === referenceId && connection.toNodeId === videoNode.id)) nextConnections.push({ id: nanoid(), fromNodeId: referenceId, toNodeId: videoNode.id });
});
nextConnections = reconcileStoryboardTargetConnections(nextConnections, scriptNode, row, videoNode.id, referenceIds);
videoNodeByRowId.set(row.id, videoNode.id);
createdCount += 1;
});
@@ -313,7 +308,7 @@ export function useCanvasStoryboard({
...scriptNode.metadata,
storyboard: {
rows: allRows.map((row) => ({ ...row, videoNodeId: videoNodeByRowId.get(row.id) || row.videoNodeId })),
visibleColumns: scriptNode.metadata?.storyboard?.visibleColumns || ["shotNumber", "plotDescription", "videoMotionPrompt", "dialogue"],
visibleColumns: scriptNode.metadata?.storyboard?.visibleColumns || ["shotNumber", "durationSeconds", "videoMotionPrompt", "dialogue", "assets"],
referenceNodeIds: scriptNode.metadata?.storyboard?.referenceNodeIds || [],
},
},
@@ -361,12 +356,9 @@ export function useCanvasStoryboard({
const targetById = new Map(targets.map((target) => [target.videoNode.id, target]));
const nextNodes = nodesRef.current.map((node) => {
const target = targetById.get(node.id);
return target ? { ...node, metadata: { ...node.metadata, prompt: target.prompt, composerContent: target.prompt, model: videoModel, ...storyboardPromptTemplateMetadata(target.row, "video"), generationMode: "video" as const, videoEditOperation: "text_to_video" as const, videoStartFrameNodeId: undefined } } : node;
return target ? { ...node, metadata: { ...node.metadata, prompt: target.prompt, composerContent: target.videoNode.metadata?.composerContent || target.prompt, model: videoModel, ...storyboardPromptTemplateMetadata(target.row, "video"), generationMode: "video" as const, videoEditOperation: "text_to_video" as const, videoStartFrameNodeId: undefined } } : node;
});
const dedicatedFirstFrameConnections = new Set(targets
.filter((target) => Boolean(target.row.imageNodeId))
.map((target) => `${target.row.imageNodeId}:${target.videoNode.id}`));
const nextConnections = connectionsRef.current.filter((connection) => !dedicatedFirstFrameConnections.has(`${connection.fromNodeId}:${connection.toNodeId}`));
const nextConnections = connectionsRef.current;
nodesRef.current = nextNodes;
connectionsRef.current = nextConnections;
setNodes(nextNodes);
@@ -449,26 +441,25 @@ export function useCanvasStoryboard({
const currentRows = targetRows.map((row) => currentScriptNode.metadata?.storyboard?.rows.find((item) => item.id === row.id) || row);
const startX = Math.max(...currentRows.map((row) => nodesRef.current.find((node) => node.id === row.imageNodeId)?.position.x || currentScriptNode.position.x + currentScriptNode.width)) + videoSpec.width + 120;
const nextNodes = [...nodesRef.current];
const nextConnections = [...connectionsRef.current];
let nextConnections = [...connectionsRef.current];
const targets: Array<{ row: StoryboardRow; node: CanvasNodeData; prompt: string }> = [];
currentRows.forEach((row, index) => {
const prompt = (row.videoMotionPrompt || row.plotDescription).trim();
const existing = row.videoNodeId ? nextNodes.find((node) => node.id === row.videoNodeId && node.type === CanvasNodeType.Video) : undefined;
const referenceIds = storyboardRowReferenceNodeIds(currentScriptNode, row, nextNodes, nextConnections, true, existing?.id);
const composerContent = storyboardComposerContent(prompt, referenceIds);
const existingMetadata = existing?.metadata?.content ? existing.metadata : resetGenerationTaskMetadata(existing?.metadata);
const videoNode = existing
? { ...existing, metadata: { ...existingMetadata, prompt, composerContent: prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot" as const, workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video" as const, videoEditOperation: "image_to_video" as const, videoStartFrameNodeId: row.imageNodeId, seconds: String(row.durationSeconds) } }
: createCanvasNode(CanvasNodeType.Video, { x: startX, y: currentScriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "image_to_video", videoStartFrameNodeId: row.imageNodeId, status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
? { ...existing, metadata: { ...existingMetadata, prompt, composerContent, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot" as const, workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video" as const, videoEditOperation: "image_to_video" as const, videoStartFrameNodeId: row.imageNodeId, seconds: String(row.durationSeconds) } }
: createCanvasNode(CanvasNodeType.Video, { x: startX, y: currentScriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, composerContent, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "image_to_video", videoStartFrameNodeId: row.imageNodeId, status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
if (!existing) {
videoNode.title = `镜头 ${row.shotNumber} · 视频`;
nextNodes.push(videoNode);
nextConnections.push({ id: nanoid(), fromNodeId: currentScriptNode.id, toNodeId: videoNode.id, fromHandleId: `row:${row.id}` });
} else {
const existingIndex = nextNodes.findIndex((node) => node.id === existing.id);
nextNodes[existingIndex] = videoNode;
}
storyboardRowReferenceNodeIds(currentScriptNode, row, nextNodes, nextConnections, true).forEach((referenceId) => {
if (!nextConnections.some((connection) => connection.fromNodeId === referenceId && connection.toNodeId === videoNode.id)) nextConnections.push({ id: nanoid(), fromNodeId: referenceId, toNodeId: videoNode.id });
});
nextConnections = reconcileStoryboardTargetConnections(nextConnections, currentScriptNode, row, videoNode.id, referenceIds);
targets.push({ row, node: videoNode, prompt });
});
nodesRef.current = nextNodes;
@@ -494,13 +485,6 @@ export function useCanvasStoryboard({
};
}
function characterReferenceNodeIds(row: StoryboardRow, nodes: CanvasNodeData[]) {
const assetIds = new Set(row.characters.map((character) => character.characterAssetId).filter((assetId): assetId is string => Boolean(assetId)));
return nodes
.filter((node) => node.metadata?.workflowKind === "character" && Boolean(node.metadata.characterAssetId) && assetIds.has(node.metadata.characterAssetId!))
.map((node) => node.id);
}
function invalidateEditedPromptVariables(previous: StoryboardRow | undefined, next: StoryboardRow) {
if (!previous) return next;
return {
@@ -510,19 +494,6 @@ function invalidateEditedPromptVariables(previous: StoryboardRow | undefined, ne
};
}
function storyboardRowReferenceNodeIds(scriptNode: CanvasNodeData, row: StoryboardRow, nodes: CanvasNodeData[], connections: CanvasConnection[], includeFirstFrame: boolean) {
const referenceIds = new Set([
...(scriptNode.metadata?.storyboard?.referenceNodeIds || []),
...(row.referenceNodeIds || []),
...characterReferenceNodeIds(row, nodes),
...connections.filter((connection) => connection.toNodeId === scriptNode.id && connection.toHandleId === `row:${row.id}`).map((connection) => connection.fromNodeId),
...(includeFirstFrame && row.imageNodeId ? [row.imageNodeId] : []),
]);
if (!includeFirstFrame && row.imageNodeId) referenceIds.delete(row.imageNodeId);
referenceIds.delete(scriptNode.id);
return Array.from(referenceIds);
}
function activeGenerationBatchNodeIds(node: CanvasNodeData, mode: CanvasGenerationBatchMode) {
return new Set((node.metadata?.generationBatches || [])
.filter((batch) => batch.mode === mode)