tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): AI 拆图 - 完善规划、图层编辑与素材复用

Browse Source
This commit is contained in:
Codex
2026-10-09 16:42:56 +08:00
parent 85c2aa178b
commit e9227e7941
56 changed files with 3541 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
web/src/lib/canvas/canvas-generation-task-sync.ts
View File

@@ -1,6 +1,6 @@
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { commitCanvasGenerationResult } from "@/lib/canvas/canvas-generation-result";
import { imageGenerationChildPosition } from "@/lib/canvas/canvas-generation-layout";
import { findAvailableGenerationGroupPosition, imageGenerationChildPosition } from "@/lib/canvas/canvas-generation-layout";
import { fitNodeSize, nodeSizeFromRatio, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { compositeEmotionImage } from "@/lib/canvas/canvas-emotion";
import { storeGeneratedAudio } from "@/services/api/audio";
@@ -14,12 +14,17 @@ import { useCanvasStore } from "@/stores/canvas/use-canvas-store";
import { useAssetStore } from "@/stores/use-asset-store";
import { applyGenerationConsumerEffect, generationEffectApplied } from "@/services/generation-consumer-dedupe";
import { commitProducedModel } from "@/lib/canvas/produced-model";
import { experimentalLayerSignature, hasUsableLayerTransparency, imageLayerCompositeSignature, validateImageLayerOutput } from "@/lib/canvas/canvas-image-layers";
import { composeCanvasImageLayerGroup, decodeAndComposeImageLayers, inspectImageLayer, normalizeImageLayerCanvas } from "@/services/canvas-image-layer-compositor";
import { patchImageLayerBackground } from "@/services/canvas-image-layer-source";
import { imageLayerTargetName } from "@/lib/canvas/canvas-image-layer-plan";
import { resetGenerationTaskMetadata } from "@/lib/canvas/canvas-task-state";
import { CanvasNodeType, type CanvasGenerationMode, type CanvasNodeData, type CanvasNodeMetadata } from "@/types/canvas";
export function generationTaskInput(task: GenerationTask) {
if (!task.inputJson) return null;
try {
return JSON.parse(task.inputJson) as { mode?: CanvasGenerationMode; metadata?: { nodeId?: string; sourceNodeId?: string; domainProjectId?: string }; prompt?: string };
return JSON.parse(task.inputJson) as { mode?: CanvasGenerationMode; metadata?: { nodeId?: string; sourceNodeId?: string; domainProjectId?: string; layerDecomposition?: boolean }; prompt?: string };
} catch {
return null;
}
@@ -128,12 +133,45 @@ export function applyGeneratedMediaResultMetadata(node: CanvasNodeData, media: C
);
}
export async function buildSourceImageLayerNodeResult(node: CanvasNodeData, image: { dataUrl: string }, signal: AbortSignal): Promise<CanvasNodeData> {
const extraction = node.metadata?.layerExtraction;
if (!extraction?.strategy || extraction.strategy.method === "generate") throw new Error("本层不是原图提取结果");
if ((extraction.index === 0) !== (extraction.strategy.method === "source")) throw new Error("原图提取方式与本层角色不一致");
if (signal.aborted) throw new DOMException("拆层已停止", "AbortError");
const uploaded = await uploadImage(image.dataUrl);
if (signal.aborted) throw new DOMException("拆层已停止", "AbortError");
return {
...node,
metadata: applyGeneratedMediaResultMetadata({ ...node, metadata: resetGenerationTaskMetadata(node.metadata) }, imageMetadata(uploaded), {
model: undefined,
producedModelCandidate: undefined,
producedModel: undefined,
layerExtraction: { ...extraction, phase: "complete", rejectedTaskId: undefined, extractionTaskId: undefined },
}),
};
}
export async function buildGenerationTaskNodeResult(node: CanvasNodeData, task: GenerationTask, nodes: CanvasNodeData[] = [node], outputIndex = 0): Promise<CanvasNodeData> {
const mode = generationTaskMode(task, node.type === CanvasNodeType.Text ? "text" : node.type === CanvasNodeType.Video ? "video" : node.type === CanvasNodeType.Audio ? "audio" : "image");
const prompt = node.metadata?.prompt || task.prompt;
const result = parseBackendGenerationResult(task);
if (mode === "image") {
const sourceResultStorageKey = result.images?.[outputIndex]?.storageKey;
let extraction = node.metadata?.layerExtraction;
let needsRemoval = false;
if (node.metadata?.layerExtraction) {
if (result.images?.length !== 1) throw new Error("逐层提取必须返回一张独立图片，不能返回拼版或多张候选图");
const normalized = await normalizeImageLayerCanvas(result.images[0], extraction?.canvas);
const info = normalized.info;
result.images[0] = normalized.image;
if (!info.nonempty) throw new Error("提取结果是完全透明的空图层");
if (extraction?.canvas && (info.width !== extraction.canvas.width || info.height !== extraction.canvas.height)) throw new Error("图层尺寸与底图不一致，已停止后续调用");
needsRemoval = Boolean(extraction?.index && !hasUsableLayerTransparency(info) && extraction.allowBackgroundRemoval && extraction.phase === "extract");
if (!needsRemoval) validateImageLayerOutput(info);
if (extraction?.index === 0 && extraction.backgroundPatch) result.images[0] = await patchImageLayerBackground(normalized.image, extraction.backgroundPatch);
extraction = { ...extraction!, phase: needsRemoval ? "background-removal-required" : "complete", extractionTaskId: extraction?.phase === "extract" ? task.id : extraction?.extractionTaskId, rejectedTaskId: undefined };
}
const image = result.images?.[outputIndex];
if (!image?.dataUrl) throw new Error("后端任务没有返回图片");
let resultDataUrl = image.dataUrl;
@@ -151,6 +189,7 @@ export async function buildGenerationTaskNodeResult(node: CanvasNodeData, task:
? { url: await resolveImageUrl(image.storageKey, image.dataUrl), storageKey: image.storageKey, width: image.width || 1024, height: image.height || 1024, bytes: image.bytes || 0, mimeType: image.mimeType || "image/png" }
: await uploadImage(resultDataUrl);
const imageConfig = NODE_DEFAULT_SIZE[CanvasNodeType.Image];
if (extraction) extraction = { ...extraction, sourceResultStorageKey: uploaded.storageKey !== sourceResultStorageKey ? sourceResultStorageKey : undefined };
const requestedImageSize = nodeSizeFromRatio(node.metadata?.size || "auto", imageConfig.width, imageConfig.height);
const imageSizeBounds = requestedImageSize || { width: node.width || imageConfig.width, height: node.height || imageConfig.height };
const hasReportedImageSize = Boolean(image.width && image.width > 0 && image.height && image.height > 0);
@@ -165,7 +204,12 @@ export async function buildGenerationTaskNodeResult(node: CanvasNodeData, task:
width: imageSize.width,
height: imageSize.height,
position: { x: node.position.x + node.width / 2 - imageSize.width / 2, y: node.position.y + node.height / 2 - imageSize.height / 2 },
metadata: applyGeneratedMediaResultMetadata(node, imageMetadata(normalizedImage), { prompt, ...completedTaskMetadata(task), generationOutputCount: 1 }, task.model),
metadata: applyGeneratedMediaResultMetadata(
node,
imageMetadata(normalizedImage),
{ prompt, ...completedTaskMetadata(task), generationOutputCount: 1, ...(extraction ? { layerExtraction: extraction, status: needsRemoval ? "idle" : "success" } : {}) },
task.model,
),
};
}
@@ -292,7 +336,13 @@ export async function applyMaterializedGenerationTaskResultToNodes(nodes: Canvas
}
const { node: updatedNode, additionalNodes } = await buildGenerationTaskNodeResults(node, { ...task, resultJson: JSON.stringify(result) }, nodes);
const stamp = (item: CanvasNodeData) => ({ ...item, metadata: applyGenerationConsumerEffect(item.metadata || {}, effectKey, (metadata) => metadata).value });
const durableNode = stamp({ ...updatedNode, metadata: { ...updatedNode.metadata, assetId: updatedNode.metadata?.assetId || asset.id } });
const durableNode = stamp({
...updatedNode,
metadata: {
...updatedNode.metadata,
assetId: updatedNode.metadata?.assetId || (updatedNode.metadata?.imageLayerGroup || (updatedNode.metadata?.layerExtraction && updatedNode.metadata.storageKey !== asset.data.storageKey) ? undefined : asset.id),
},
});
const durableChildren = additionalNodes.map(stamp);
return {
nodes: commitCanvasGenerationResult(nodes, node, durableNode, task.id, durableChildren),
@@ -304,6 +354,7 @@ export async function applyMaterializedGenerationTaskResultToNodes(nodes: Canvas
}
export function generationTaskOutputsApplied(node: CanvasNodeData, task: GenerationTask) {
if (node.metadata?.layerExtraction?.extractionTaskId === task.id && node.metadata.content) return true;
if (node.metadata?.taskId !== task.id || node.metadata.status !== "success" || !node.metadata.content) return false;
if (generationTaskMode(task) !== "image") return true;
const count = parseBackendGenerationResult(task).images?.length || 1;
@@ -323,6 +374,7 @@ export function shouldRecoverCanvasImageOutputs(node: CanvasNodeData) {
}
async function buildGenerationTaskNodeResults(node: CanvasNodeData, task: GenerationTask, nodes: CanvasNodeData[]) {
if (generationTaskInput(task)?.metadata?.layerDecomposition) return buildImageLayerTaskResult(node, task, nodes);
const resultNode = await buildGenerationTaskNodeResult(node, task, nodes);
const images = generationTaskMode(task) === "image" ? parseBackendGenerationResult(task).images || [] : [];
if (images.length <= 1) return { node: resultNode, additionalNodes: [] as CanvasNodeData[] };
@@ -379,6 +431,189 @@ async function buildGenerationTaskNodeResults(node: CanvasNodeData, task: Genera
};
}
export async function buildImageLayerTaskResult(node: CanvasNodeData, task: GenerationTask, nodes: CanvasNodeData[]) {
const images = parseBackendGenerationResult(task).images || [];
if (node.metadata?.taskId === task.id && node.metadata.content && node.metadata.imageLayerGroup && node.metadata.generationOutputCount === images.length)
return {
node: node.metadata.status === "success" ? node : { ...node, metadata: { ...node.metadata, ...completedTaskMetadata(task), status: "success" as const } },
additionalNodes: [] as CanvasNodeData[],
};
const layerImages = [];
for (const image of images) layerImages.push((await normalizeImageLayerCanvas(image, node.metadata?.layerDecomposition?.canvas)).image);
const decoded = await decodeAndComposeImageLayers(layerImages);
const size = fitNodeSize(decoded.width, decoded.height, node.width, node.height);
const firstPosition = imageGenerationChildPosition(node.position, size.width, size, 0);
const lastPosition = imageGenerationChildPosition(node.position, size.width, size, images.length - 1);
const secondPosition = imageGenerationChildPosition(node.position, size.width, size, 1);
const layoutPosition = findAvailableGenerationGroupPosition(nodes, firstPosition, {
width: secondPosition.x - firstPosition.x + size.width,
height: lastPosition.y - firstPosition.y + size.height,
});
const children: CanvasNodeData[] = [];
for (let index = 0; index < images.length; index += 1) {
const id = `${node.id}:task:${task.id}:layer:${index}`;
const existing = nodes.find((item) => item.id === id);
if (existing) {
children.push(existing);
continue;
}
const image = images[index];
const stored: UploadedImage = image.storageKey
? {
url: await resolveImageUrl(image.storageKey, image.dataUrl),
storageKey: image.storageKey,
width: decoded.width,
height: decoded.height,
bytes: image.bytes || decoded.layers[index].blob.size,
mimeType: decoded.layers[index].blob.type || "image/png",
}
: await uploadImage(decoded.layers[index].blob);
if (stored.pendingRemoteUpload) throw new Error("图层仅缓存在本机，服务端保存未完成；请重试保存结果");
children.push({
id,
type: CanvasNodeType.Image,
title: decoded.layers[index].transparent ? `图层 ${index + 1}` : "底图",
...size,
position: (() => {
const position = imageGenerationChildPosition(node.position, size.width, size, index);
return { x: position.x + layoutPosition.x - firstPosition.x, y: position.y + layoutPosition.y - firstPosition.y };
})(),
metadata: commitProducedModel(
{
...imageMetadata(stored),
assetId: task.outputs?.find((output) => output.outputIndex === index)?.materializedAssetId,
batchRootId: node.id,
imageLayer: { groupId: node.id, outputIndex: index, kind: decoded.layers[index].transparent ? "transparent" : "base" },
model: node.metadata?.model || task.model,
producedModel: task.model,
},
task.model,
),
});
}
const composite = await uploadImage(decoded.composite);
if (composite.pendingRemoteUpload) throw new Error("合成图仅缓存在本机，服务端保存未完成；请重试保存结果");
const group: NonNullable<CanvasNodeMetadata["imageLayerGroup"]> = {
width: decoded.width,
height: decoded.height,
compositeStatus: "ready",
layers: decoded.order.map((index) => ({ nodeId: children[index].id, x: 0, y: 0, visible: true })),
};
group.compositeSignature = imageLayerCompositeSignature(group, children);
return {
node: {
...node,
...size,
metadata: applyGeneratedMediaResultMetadata(
node,
imageMetadata(composite),
{
...completedTaskMetadata(task),
imageLayerGroup: group,
isBatchRoot: true,
batchChildIds: group.layers.map((layer) => layer.nodeId),
primaryImageId: undefined,
imageBatchExpanded: node.metadata?.imageBatchExpanded ?? false,
generationOutputCount: images.length,
},
task.model,
),
},
additionalNodes: children,
};
}
/** 仅合成已验收的层，保留失败项身份；重试新增结果不会覆盖已有排序与可见性。 */
export async function buildExperimentalImageLayerResult(root: CanvasNodeData, nodes: CanvasNodeData[]) {
const plan = root.metadata?.experimentalLayerPlan;
if (!plan || !plan.requests.length || plan.requests.length > 8) throw new Error("实验拆层计划无效");
const candidates = plan.requests.flatMap((request, index) => {
const node = nodes.find((node) => node.id === request.nodeId);
return node?.metadata?.content && node.metadata.status === "success" && (!node.metadata.layerExtraction?.phase || node.metadata.layerExtraction.phase === "complete") ? [{ node, index: node.metadata.layerExtraction?.index ?? index }] : [];
});
const ready: typeof candidates = [];
const rejected: CanvasNodeData[] = [];
for (const item of candidates) {
try {
validateImageLayerOutput(await inspectImageLayer({ dataUrl: item.node.metadata!.content!, storageKey: item.node.metadata?.storageKey }));
ready.push(item);
} catch (error) {
rejected.push({
...item.node,
metadata: {
...item.node.metadata,
status: "error",
errorDetails: error instanceof Error ? error.message : "图层资源验收失败",
...(item.node.metadata?.layerExtraction ? { layerExtraction: { ...item.node.metadata.layerExtraction, rejectedTaskId: item.node.metadata.taskId } } : {}),
},
});
}
}
if (!ready.length) throw new Error("没有已验收的图层可生成合成图");
const previous = root.metadata?.imageLayerGroup;
const order = previous?.layers.flatMap((layer) => ready.filter((item) => item.node.id === layer.nodeId)) || [];
for (const item of ready) {
if (order.some((existing) => existing.node.id === item.node.id)) continue;
const before = order.findIndex((existing) => existing.index > item.index);
order.splice(before < 0 ? order.length : before, 0, item);
}
const sources = order.map((item) => item.node);
const decoded = await decodeAndComposeImageLayers(
sources.map((node) => ({ dataUrl: node.metadata!.content!, storageKey: node.metadata?.storageKey })),
order.map((item) => item.index),
);
const updatedChildren = sources.map((node, index) => ({
...node,
title: node.metadata?.imageLayer ? node.title : imageLayerTargetName(plan.requests.find((request) => request.nodeId === node.id)!.target),
metadata: { ...node.metadata, batchRootId: root.id, imageLayer: { groupId: root.id, outputIndex: order[index].index, kind: order[index].index === 0 ? ("base" as const) : ("transparent" as const) } },
}));
const completed = sources.length;
const group: NonNullable<CanvasNodeMetadata["imageLayerGroup"]> = {
width: decoded.width,
height: decoded.height,
compositeStatus: "ready",
layers: decoded.order.map(
(index) =>
previous?.layers.find((layer) => layer.nodeId === sources[index].id) || {
nodeId: sources[index].id,
x: 0,
y: 0,
visible: !order.some((item) => item.index === 0) || order[index].index === 0 || plan.requests.find((request) => request.nodeId === sources[index].id)?.removeFromBackground !== false,
},
),
...(completed < plan.requests.length ? { incomplete: { completed, total: plan.requests.length, failed: plan.requests.length - completed, missingBackground: !order.some((item) => item.index === 0) } } : {}),
};
const composite = await uploadImage(previous || group.layers.some((layer) => !layer.visible) ? await composeCanvasImageLayerGroup(group, updatedChildren) : decoded.composite);
if (composite.pendingRemoteUpload) throw new Error("合成图尚未保存到服务端，请重试本地合成");
group.compositeSignature = imageLayerCompositeSignature(group, updatedChildren);
return {
node: {
...root,
...fitNodeSize(decoded.width, decoded.height, root.width, root.height),
metadata: applyGeneratedMediaResultMetadata(root, imageMetadata(composite), {
imageLayerGroup: group,
isBatchRoot: true,
batchChildIds: plan.requests.map((request) => request.nodeId),
imageBatchExpanded: root.metadata?.imageBatchExpanded ?? false,
primaryImageId: undefined,
experimentalLayerPlan: {
...plan,
composedSignature: experimentalLayerSignature(
root,
nodes.map((node) => rejected.find((item) => item.id === node.id) || node),
),
errorSignature: undefined,
},
status: "success",
errorDetails: undefined,
batchFailedCount: plan.requests.length - completed,
generationOutputCount: sources.length,
}),
},
additionalNodes: [...updatedChildren, ...rejected],
};
}
export async function syncGenerationTaskToCanvasStore(task: GenerationTask) {
if (task.status !== "succeeded" || !task.projectId) return false;
// 短剧任务使用业务项目 ID，不能拿它请求同名的画布项目。