tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(*): 画布创作 - 支持文本多模态生成并优化媒体节点、连接线与视频预览

Browse Source 
Fixes #59
This commit is contained in:
ddCat
2026-07-31 19:44:02 +08:00
parent 8ce97f267b
commit 14da5e8a19
29 changed files with 231 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
web/src/pages/canvas/canvas-media-generation-executors.ts
View File

@@ -2,7 +2,7 @@ import { nanoid } from "nanoid";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { audioMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { fitNodeSize, nodeSizeFromRatio } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, nodeSizeFromRatio, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { nextCanvasVersionLabel } from "@/lib/canvas/canvas-layout";
import { buildAudioGenerationMetadata, buildVideoGenerationMetadata, generationReferenceUrls, runBackendCanvasGenerationTask } from "@/lib/canvas/canvas-project-generation";
import { storeGeneratedAudio } from "@/services/api/audio";
@@ -11,8 +11,6 @@ import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
import type { CanvasGenerationExecution } from "./canvas-generation-executor-types";
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
@@ -82,7 +80,7 @@ export async function executeVideoGeneration({
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: videoId, mode: "video", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, referenceVideos: generationContext.referenceVideos, referenceAudios: generationContext.referenceAudios, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions, resolvedCharacterVoices: generationContext.resolvedCharacterVoices, ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(videoId, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
setNodes((current) => current.map((node) => {
if (node.id !== videoId) return node;
const geometry = node.metadata?.locked ? {} : { width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 } };

2
web/src/pages/canvas/canvas-text-generation-executor.ts
View File

@@ -58,7 +58,7 @@ export async function executeTextGeneration({
const answers = await Promise.all(
textTargetIds.map((targetNodeId) => {
let localStreamed = "";
return runBackendCanvasGenerationTask({ projectId, nodeId: targetNodeId, mode: "text", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(targetNodeId, task) })
return runBackendCanvasGenerationTask({ projectId, nodeId: targetNodeId, mode: "text", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, referenceVideos: generationContext.referenceVideos, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(targetNodeId, task) })
.then((result) => {
localStreamed = result.text || "";
streamed = localStreamed;

2
web/src/pages/canvas/project.tsx
View File

@@ -202,7 +202,7 @@ function InfiniteCanvasPage() {
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
const [contextMenu, setContextMenu] = useState<ContextMenuState | null>(null);
const [isMiniMapOpen, setIsMiniMapOpen] = useState(false);
const [backgroundMode, setBackgroundMode] = useState<CanvasBackgroundMode>("lines");
const [backgroundMode, setBackgroundMode] = useState<CanvasBackgroundMode>("dots");
const [showImageInfo, setShowImageInfo] = useState(false);
const [mediaPerformanceMode, setMediaPerformanceMode] = useState<CanvasMediaPerformanceMode>(readCanvasMediaPerformanceMode);
const [projectLoaded, setProjectLoaded] = useState(false);

7
web/src/pages/canvas/shared.tsx
View File

@@ -15,6 +15,7 @@ import { WorkspaceState } from "@/components/layout/workspace-state";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { canvasThemes } from "@/lib/canvas-theme";
import { isFrameNode, isNodeHiddenByCollapsedFrame, resolveFrameConnection } from "@/lib/canvas/canvas-frame";
import { ensureMediaNodeMinimumSize } from "@/lib/canvas/canvas-node-size";
import { getPublicCanvasShare } from "@/services/api/canvas-share";
import { useThemeStore } from "@/stores/use-theme-store";
import { CanvasNodeType, type CanvasNodeData, type Position, type ViewportTransform } from "@/types/canvas";
@@ -33,7 +34,7 @@ export default function SharedCanvasPage() {
const [title, setTitle] = useState("共享画布");
const [nodes, setNodes] = useState<CanvasNodeData[]>([]);
const [connections, setConnections] = useState<Awaited<ReturnType<typeof getPublicCanvasShare>>["project"]["connections"]>([]);
const [backgroundMode, setBackgroundMode] = useState<"lines" | "dots" | "blank">("lines");
const [backgroundMode, setBackgroundMode] = useState<"lines" | "dots" | "blank">("dots");
const [viewport, setViewport] = useState<ViewportTransform>({ x: 0, y: 0, k: 1 });
const [selectedNodeId, setSelectedNodeId] = useState<string | null>(null);
const [infoNodeId, setInfoNodeId] = useState<string | null>(null);
@@ -77,9 +78,9 @@ export default function SharedCanvasPage() {
getPublicCanvasShare(token).then(({ project }) => {
if (!active) return;
setTitle(project.title || "共享画布");
setNodes(project.nodes || []);
setNodes((project.nodes || []).map(ensureMediaNodeMinimumSize));
setConnections(project.connections || []);
setBackgroundMode(project.backgroundMode || "lines");
setBackgroundMode(project.backgroundMode || "dots");
const initial = project.viewport || { x: 0, y: 0, k: 1 };
viewportRef.current = initial;
setViewport(initial);

8
web/src/pages/canvas/use-canvas-generation-retry.ts
View File

@@ -5,7 +5,7 @@ import { buildNodeGenerationContext, hydrateNodeGenerationContext } from "@/comp
import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { audioMetadata, imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { buildEmotionImageArtifacts, compositeEmotionImage, emotionGenerationSize } from "@/lib/canvas/canvas-emotion";
import {
buildAudioGenerationMetadata,
@@ -46,8 +46,6 @@ type UseCanvasGenerationRetryOptions = {
bindGenerationTask: (targetNodeId: string, task: GenerationTask) => void;
};
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
const NODE_STATUS_ERROR = "error" as const;
@@ -143,7 +141,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
try {
if (node.type === CanvasNodeType.Text) {
if (!context) return;
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "text", prompt, config: generationConfig, referenceImages: context.referenceImages, signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "text", prompt, config: generationConfig, referenceImages: context.referenceImages, referenceVideos: context.referenceVideos, signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.text) throw new Error("后端任务没有返回文本");
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, type: CanvasNodeType.Text, metadata: { ...item.metadata, content: result.text, richText: undefined, prompt, status: NODE_STATUS_SUCCESS, errorDetails: undefined, generationErrorCode: undefined, failedPromptFingerprint: undefined } } : item)));
return;
@@ -153,7 +151,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "video", prompt, config: generationConfig, referenceImages: videoContext?.referenceImages || [], referenceVideos: videoContext?.referenceVideos || [], referenceAudios: videoContext?.referenceAudios || [], signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], resolvedCharacterVoices: context?.resolvedCharacterVoices || [], ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, ...videoGenerationMetadata, references: videoContext ? generationReferenceUrls(videoContext) : item.metadata?.references } } : item)));
return;
}

10
web/src/pages/canvas/use-canvas-media-tools.ts
View File

@@ -19,7 +19,7 @@ import {
isGenerationCanceled,
runBackendCanvasGenerationTask,
} from "@/lib/canvas/canvas-project-generation";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { compositeEmotionImage, emotionGenerationSize } from "@/lib/canvas/canvas-emotion";
import { DEFAULT_PORTRAIT_TEXTURE_SETTINGS } from "@/lib/canvas/canvas-portrait-texture";
import { captureVideoLastFrame } from "@/lib/canvas/canvas-video-frame";
@@ -56,8 +56,6 @@ type UseCanvasMediaToolsOptions = {
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
const NODE_STATUS_ERROR = "error" as const;
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const IMAGE_PROMPT_REVERSE_PRESET = `请根据参考图片反推一段适合用于 AI 生图的提示词。
要求：
@@ -170,9 +168,9 @@ export function useCanvasMediaTools({
if (!node.metadata?.content) return;
const cropped = await cropDataUrl(node.metadata.content, crop);
const image = await uploadImage(cropped);
const width = Math.min(node.width, Math.max(220, image.width));
const size = fitNodeSize(image.width, image.height, node.width, node.height);
const childId = nanoid();
const child: CanvasNodeData = { id: childId, type: CanvasNodeType.Image, title: "Cropped Image", position: { x: node.position.x + node.width + 96, y: node.position.y }, width, height: width * (image.height / image.width), metadata: { ...imageMetadata(image), prompt: node.metadata?.prompt } };
const child: CanvasNodeData = { id: childId, type: CanvasNodeType.Image, title: "Cropped Image", position: { x: node.position.x + node.width + 96, y: node.position.y }, width: size.width, height: size.height, metadata: { ...imageMetadata(image), prompt: node.metadata?.prompt } };
setNodes((current) => [...current, child]);
setConnections((current) => [...current, { id: nanoid(), fromNodeId: node.id, toNodeId: childId }]);
setSelectedNodeIds(new Set([childId]));
@@ -254,7 +252,7 @@ export function useCanvasMediaTools({
const blob = await mergeVideos(videos.map((node) => ({ id: node.id, url: node.metadata?.content, storageKey: node.metadata?.storageKey })), setMergeVideoProgress);
setMergeVideoProgress({ phase: "encoding", progress: 98 });
const uploaded = await storeGeneratedVideo({ blob });
const size = fitNodeSize(uploaded.width || 1280, uploaded.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const size = fitNodeSize(uploaded.width || 1280, uploaded.height || 720, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
const left = Math.max(...videos.map((node) => node.position.x + node.width)) + 120;
const top = Math.min(...videos.map((node) => node.position.y));
const mergedNode = createCanvasNode(CanvasNodeType.Video, { x: left + size.width / 2, y: top + size.height / 2 }, {

2
web/src/pages/canvas/use-canvas-project-lifecycle.ts
View File

@@ -92,7 +92,7 @@ export function useCanvasProjectLifecycle({
connections: project.connections,
chatSessions: restoredSessions,
activeChatId: project.activeChatId || null,
backgroundMode: project.backgroundMode,
backgroundMode: project.backgroundMode || "dots",
showImageInfo: project.showImageInfo || false,
};
nodesRef.current = snapshot.nodes;

8
web/src/pages/canvas/use-canvas-upload.ts
View File

@@ -10,7 +10,7 @@ import { getDataUrlByteSize, readImageMeta } from "@/lib/image-utils";
import { audioMetadata, imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { createCanvasNode } from "@/lib/canvas/canvas-project-domain";
import { isAudioFile } from "@/lib/canvas/canvas-project-generation";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { uploadMediaFile } from "@/services/file-storage";
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
import { getProjectUnit } from "@/services/api/projects";
@@ -39,8 +39,6 @@ export type StartCanvasUploadStatus = (title: string, detail: string, total?: nu
fail: (detail?: string) => void;
};
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const NODE_STATUS_SUCCESS = "success" as const;
export function useCanvasUpload({
@@ -187,7 +185,7 @@ export function useCanvasUpload({
progress.update("上传到服务器并同步资源", 2);
const video = await uploadMediaFile(file, "video");
progress.update("更新画布节点", 3);
const size = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const size = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
const node = { id, type: CanvasNodeType.Video, title: file.name, position: { x: position.x - size.width / 2, y: position.y - size.height / 2 }, width: size.width, height: size.height, metadata: videoMetadata(video) } satisfies CanvasNodeData;
setNodes((current) => [...current, node]);
@@ -570,7 +568,7 @@ export function useCanvasUpload({
}
if (payload.kind === "video") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const size = fitNodeSize(payload.width || spec.width, payload.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const size = fitNodeSize(payload.width || spec.width, payload.height || spec.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
return { id, type: CanvasNodeType.Video, title: payload.title, position: { x: center.x - size.width / 2, y: center.y - size.height / 2 }, width: size.width, height: size.height, metadata: { content: payload.url, storageKey: payload.storageKey, status: NODE_STATUS_SUCCESS, naturalWidth: payload.width, naturalHeight: payload.height, durationMs: payload.durationMs, bytes: payload.bytes, mimeType: payload.mimeType || "video/mp4", assetId: payload.assetId } } satisfies CanvasNodeData;
}