tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(*): 画布创作 - 支持文本多模态生成并优化媒体节点、连接线与视频预览

Browse Source 
Fixes #59
This commit is contained in:
ddCat
2026-07-31 19:44:02 +08:00
parent 5db39ccad4
commit 8de9b42c00
29 changed files with 231 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
web/src/components/asset-media-preview.tsx Normal file
View File

@@ -0,0 +1,38 @@
import type { ReactNode } from "react";
import type { Asset } from "@/stores/use-asset-store";
type AssetMediaPreviewProps = {
asset?: Asset | null;
alt: string;
className?: string;
fallback?: ReactNode;
};
export function AssetMediaPreview({ asset, alt, className = "", fallback = null }: AssetMediaPreviewProps) {
if (!asset) return fallback;
if (asset.kind === "video" && asset.data.url) {
const poster = asset.coverUrl && asset.coverUrl !== asset.data.url ? asset.coverUrl : undefined;
return (
<video
src={asset.data.url}
poster={poster}
aria-label={alt}
muted
playsInline
preload="metadata"
className={className}
onLoadedMetadata={(event) => {
// 主动触发首帧附近的解码，避免只有 metadata 时长期停留在空白画面。
const video = event.currentTarget;
if (!poster && video.currentTime === 0 && video.duration > 0) video.currentTime = Math.min(0.001, video.duration);
}}
/>
);
}
const imageUrl = asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : "");
if (!imageUrl) return fallback;
return <img src={imageUrl} alt={alt} loading="lazy" decoding="async" className={className} />;
}

12
web/src/components/canvas/asset-picker-modal.tsx
View File

@@ -3,6 +3,7 @@ import { Input, Modal, Pagination, Tag } from "antd";
import { Search } from "lucide-react";
import { WorkspaceState } from "@/components/layout/workspace-state";
import { AssetMediaPreview } from "@/components/asset-media-preview";
import { cn } from "@/lib/utils";
import { useAssetStore, type Asset } from "@/stores/use-asset-store";
@@ -40,18 +41,15 @@ const kindOptions = [
{ label: "音频", value: "audio" },
];
function PickerCard({ title, kind, cover, onClick }: { title: string; kind: string; cover: string; onClick: () => void }) {
function PickerCard({ asset, onClick }: { asset: InsertableAsset; onClick: () => void }) {
const { title, kind } = asset;
return (
<button
type="button"
className="group relative cursor-pointer overflow-hidden rounded-lg border border-stone-200 bg-white text-left transition hover:border-stone-400 hover:shadow-md dark:border-stone-700 dark:bg-stone-900 dark:hover:border-stone-500"
onClick={onClick}
>
{cover ? (
<img src={cover} alt={title} loading="lazy" decoding="async" className="aspect-[4/3] w-full object-cover" />
) : (
<div className="flex aspect-[4/3] items-center justify-center bg-stone-100 p-3 text-center text-xs leading-5 text-stone-500 dark:bg-stone-800 dark:text-stone-400">{title}</div>
)}
<AssetMediaPreview asset={asset} alt={title} className="aspect-[4/3] w-full bg-black object-cover" fallback={<div className="flex aspect-[4/3] items-center justify-center bg-stone-100 p-3 text-center text-xs leading-5 text-stone-500 dark:bg-stone-800 dark:text-stone-400">{title}</div>} />
<div className="p-2.5">
<div className="flex items-center justify-between gap-2">
<span className="line-clamp-1 text-xs font-medium text-stone-800 dark:text-stone-200">{title}</span>
@@ -131,7 +129,7 @@ function MyAssetsTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) =>
{visible.length ? (
<div className="grid grid-cols-4 gap-3">
{visible.map((asset) => (
<PickerCard key={asset.id} title={asset.title} kind={asset.kind} cover={asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : "")} onClick={() => handleInsert(asset)} />
<PickerCard key={asset.id} asset={asset} onClick={() => handleInsert(asset)} />
))}
</div>
) : (

14
web/src/components/canvas/canvas-leafer-graphics-layer.tsx
View File

@@ -2,7 +2,7 @@ import { useLayoutEffect, useRef, type RefObject } from "react";
import { Group, Leafer, Path, Rect } from "leafer-ui";
import { activeConnectionPath, canvasConnectionPath } from "@/components/canvas/canvas-connections";
import { subscribeCanvasSelectionPreview, subscribeCanvasViewportPreview } from "@/lib/canvas/canvas-live-viewport";
import { subscribeCanvasGraphicsViewportPreview, subscribeCanvasSelectionPreview } from "@/lib/canvas/canvas-live-viewport";
import { calculateCanvasPreviewTransform, sameCanvasViewport, shouldRebaseCanvasRaster } from "@/lib/canvas/canvas-leafer-viewport";
import type { CanvasTheme } from "@/lib/canvas-theme";
import type { CanvasDisplayConnection, CanvasNodeData, ConnectionHandle, Position, SelectionBox, ViewportTransform } from "@/types/canvas";
@@ -72,17 +72,17 @@ export function CanvasLeaferGraphicsLayer(props: CanvasLeaferGraphicsLayerProps)
underlay.leafer.resize(size);
overlay.leafer.resize(size);
syncViewport(rasterViewportRef.current, size.width, size.height, underlay, overlay, propsRef.current);
if (isZoomPreview(container, viewportRef.current, rasterViewportRef.current)) {
if (isViewportPreview(container, viewportRef.current, rasterViewportRef.current)) {
applyScenePreview(viewportRef.current, rasterViewportRef.current, underlay, overlay);
}
};
const resizeObserver = new ResizeObserver(resize);
resizeObserver.observe(container);
window.addEventListener("resize", resize);
const unsubscribe = subscribeCanvasViewportPreview(container, (next) => {
const unsubscribe = subscribeCanvasGraphicsViewportPreview(container, (next) => {
viewportRef.current = next;
const rect = container.getBoundingClientRect();
if (isZoomPreview(container, next, rasterViewportRef.current)) {
if (isViewportPreview(container, next, rasterViewportRef.current)) {
if (shouldRebaseCanvasRaster(next, rasterViewportRef.current)) {
syncViewport(next, rect.width, rect.height, underlay, overlay, propsRef.current);
rasterViewportRef.current = next;
@@ -151,7 +151,7 @@ export function CanvasLeaferGraphicsLayer(props: CanvasLeaferGraphicsLayerProps)
if (!underlay || !overlay || !container) return;
const rect = container.getBoundingClientRect();
syncViewport(rasterViewportRef.current, rect.width, rect.height, underlay, overlay, props);
if (isZoomPreview(container, viewportRef.current, rasterViewportRef.current)) {
if (isViewportPreview(container, viewportRef.current, rasterViewportRef.current)) {
applyScenePreview(viewportRef.current, rasterViewportRef.current, underlay, overlay);
}
}, [props.alignmentGuides, props.containerRef, props.theme]);
@@ -268,8 +268,8 @@ function syncViewport(viewport: ViewportTransform, width: number, height: number
});
}
function isZoomPreview(container: HTMLDivElement, viewport: ViewportTransform, rasterViewport: ViewportTransform) {
return container.dataset.canvasViewportInteracting === "true" && Math.abs(viewport.k - rasterViewport.k) > 0.0001;
function isViewportPreview(container: HTMLDivElement, viewport: ViewportTransform, rasterViewport: ViewportTransform) {
return container.dataset.canvasViewportInteracting === "true" && !sameCanvasViewport(viewport, rasterViewport);
}
function applyScenePreview(viewport: ViewportTransform, rasterViewport: ViewportTransform, ...scenes: LeaferScene[]) {

2
web/src/components/canvas/canvas-node-hover-toolbar.tsx
View File

@@ -233,7 +233,7 @@ export function CanvasNodeHoverToolbar({
...(hasVideo && !simpleMode ? [{ id: "extractLastFrame", title: extractingVideoFrame ? "正在截取尾帧" : "截取尾帧", label: extractingVideoFrame ? "截取中" : "尾帧", icon: extractingVideoFrame ? <LoaderCircle className="size-3.5 animate-spin" /> : <GalleryHorizontalEnd className="size-3.5" />, onClick: () => onExtractVideoLastFrame(node), disabled: extractingVideoFrame }] : []),
...(hasImage || hasVideo || isEditableText ? [{ id: "saveAsset", title: "加入我的素材", label: "存素材", icon: <FolderPlus className="size-3.5" />, onClick: () => onSaveAsset(node) }] : []),
...(hasImage || hasVideo || hasAudio ? [{ id: "download", title: hasAudio ? "下载音频" : hasVideo ? "下载视频" : "下载图片", label: "下载", icon: <Download className="size-3.5" />, onClick: () => onDownload(node) }] : []),
...(canOpenDialog ? [{ id: "edit", title: isEditableText ? "生成提示与参数" : "编辑", label: isEditableText ? "生成设置" : "编辑", icon: <MessageSquare className="size-3.5" />, onClick: () => onToggleDialog(node) }] : []),
...(canOpenDialog ? [{ id: "edit", title: isEditableText ? "调用文本模型生成内容" : "编辑", label: isEditableText ? "文本生成" : "编辑", icon: <MessageSquare className="size-3.5" />, onClick: () => onToggleDialog(node) }] : []),
...(isEditableText ? [{ id: "editText", title: "放大编辑文本", label: "放大编辑", icon: <Maximize2 className="size-3.5" />, onClick: () => onEditText(node) }] : []),
...(isEditableText ? [{ id: "generateImage", title: "用文本生图", label: "生图", icon: <ImageIcon className="size-3.5" />, onClick: () => onGenerateImage(node) }] : []),
...(isConfig && !simpleMode ? [{ id: "config", title: "生成配置", label: "生成配置", icon: <Settings2 className="size-3.5" />, onClick: () => onToggleDialog(node) }] : []),

10
web/src/components/canvas/canvas-node.tsx
View File

@@ -16,6 +16,7 @@ import { storyboardMinNodeHeight } from "./canvas-script-node";
import { CanvasNodeType, type CanvasNodeData, type Position } from "@/types/canvas";
import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { loadCanvasDrawingPreview } from "@/lib/canvas/canvas-drawing-storage";
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
type ResizeCorner = "top-left" | "top-right" | "bottom-left" | "bottom-right";
type CanvasTheme = (typeof canvasThemes)[keyof typeof canvasThemes];
@@ -203,8 +204,9 @@ export const CanvasNode = React.memo(function CanvasNode({
const dx = (event.clientX - resizeRef.current.startX) / scale;
const dy = (event.clientY - resizeRef.current.startY) / scale;
const minWidth = data.type === CanvasNodeType.Script ? 800 : 220;
const minHeight = scriptMinHeight || 160;
const isMediaNode = data.type === CanvasNodeType.Image || data.type === CanvasNodeType.Video;
const minWidth = data.type === CanvasNodeType.Script ? 800 : isMediaNode ? MEDIA_NODE_MIN_SIZE.width : 220;
const minHeight = scriptMinHeight || (isMediaNode ? MEDIA_NODE_MIN_SIZE.height : 160);
const startRight = resizeRef.current.startLeft + resizeRef.current.startWidth;
const startBottom = resizeRef.current.startTop + resizeRef.current.startHeight;
const fromLeft = resizeRef.current.corner.includes("left");
@@ -1119,6 +1121,8 @@ function ResizeHandle({ corner, onMouseDown }: { corner: ResizeCorner; onMouseDo
return <div className={`absolute z-50 size-7 ${positionClass}`} onMouseDown={(event) => onMouseDown(event, corner)} />;
}
const NODE_EXTERNAL_HEADER_MIN_SCALE = 0.35;
function NodeExternalHeader({ node, scale, active, editable, editing, draft, theme, onDraftChange, onEdit, onCommit, onCancel }: {
node: CanvasNodeData;
scale: number;
@@ -1132,6 +1136,8 @@ function NodeExternalHeader({ node, scale, active, editable, editing, draft, the
onCommit: () => void;
onCancel: () => void;
}) {
// 标题保持屏幕尺寸只适用于近景；远景继续反向缩放会遮住节点和连线。
if (scale < NODE_EXTERNAL_HEADER_MIN_SCALE && !editing) return null;
const inverseScale = 1 / Math.max(scale, 0.05);
const Icon = nodeTypeIcon(node.type);

5
web/src/components/canvas/canvas-project-asset-modal.tsx
View File

@@ -3,6 +3,7 @@ import { Button, Modal } from "antd";
import { Check, FileText, Image as ImageIcon, Music2, UserRound, Video } from "lucide-react";
import type { InsertAssetPayload } from "@/components/canvas/asset-picker-modal";
import { AssetMediaPreview } from "@/components/asset-media-preview";
import { WorkspaceState } from "@/components/layout/workspace-state";
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
import { resourceFileUrl } from "@/services/api/resources";
@@ -57,11 +58,11 @@ function ProjectAssetCard({ item, selected, onToggle }: { item: ProjectPickerIte
const character = item.character;
const media = item.media;
const coverRepresentation = character?.character?.representations.find((representation) => representation.role === "turnaround_sheet") || character?.character?.representations.find((representation) => representation.role === "primary") || character?.character?.representations.find((representation) => representation.role === "front");
const cover = coverRepresentation ? resourceFileUrl(coverRepresentation.resourceId) : media?.coverUrl || (media?.kind === "image" ? media.data.dataUrl : "");
const cover = coverRepresentation ? resourceFileUrl(coverRepresentation.resourceId) : "";
const Icon = character ? UserRound : media?.kind === "video" ? Video : media?.kind === "audio" ? Music2 : media?.kind === "text" ? FileText : ImageIcon;
const label = character ? "角色卡" : media?.kind === "video" ? "视频" : media?.kind === "audio" ? "音频" : media?.kind === "text" ? "文本" : "图片";
const title = character?.title || media?.title || "未命名资产";
return <button type="button" onClick={onToggle} className={`relative min-w-0 overflow-hidden rounded-md border text-left transition-colors ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent-soft)]" : "border-border/80 hover:border-foreground/30"}`}><div className="relative aspect-[4/3] overflow-hidden bg-foreground/[.04]">{cover ? <img src={cover} alt={title} loading="lazy" decoding="async" className={`h-full w-full ${character ? "object-contain p-1" : "object-cover"}`} /> : media?.kind === "video" ? <video src={media.data.url} muted preload="metadata" className="h-full w-full bg-black object-cover" /> : <div className="grid h-full place-items-center text-foreground/25"><Icon className="size-7" /></div>}<span className={`absolute right-1.5 top-1.5 grid size-5 place-items-center rounded border ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent)] text-white" : "border-white/60 bg-black/25 text-transparent backdrop-blur"}`}><Check className="size-3" /></span><span className="absolute bottom-1.5 left-1.5 rounded bg-black/55 px-1.5 py-0.5 text-[9px] text-white">{label}</span></div><div className="px-2 py-1.5"><div className="truncate text-[11px] font-medium">{title}</div>{character ? <div className="mt-0.5 truncate text-[9px] text-foreground/42">{character.character?.visualStatus === "ready" ? "形象就绪" : "形象待完善"} · {character.character?.voiceStatus === "ready" ? "声音已绑定" : "声音未绑定"}</div> : null}</div></button>;
return <button type="button" onClick={onToggle} className={`relative min-w-0 overflow-hidden rounded-md border text-left transition-colors ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent-soft)]" : "border-border/80 hover:border-foreground/30"}`}><div className="relative aspect-[4/3] overflow-hidden bg-foreground/[.04]">{cover ? <img src={cover} alt={title} loading="lazy" decoding="async" className="h-full w-full object-contain p-1" /> : <AssetMediaPreview asset={media} alt={title} className="h-full w-full bg-black object-cover" fallback={<div className="grid h-full place-items-center text-foreground/25"><Icon className="size-7" /></div>} />}<span className={`absolute right-1.5 top-1.5 grid size-5 place-items-center rounded border ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent)] text-white" : "border-white/60 bg-black/25 text-transparent backdrop-blur"}`}><Check className="size-3" /></span><span className="absolute bottom-1.5 left-1.5 rounded bg-black/55 px-1.5 py-0.5 text-[9px] text-white">{label}</span></div><div className="px-2 py-1.5"><div className="truncate text-[11px] font-medium">{title}</div>{character ? <div className="mt-0.5 truncate text-[9px] text-foreground/42">{character.character?.visualStatus === "ready" ? "形象就绪" : "形象待完善"} · {character.character?.voiceStatus === "ready" ? "声音已绑定" : "声音未绑定"}</div> : null}</div></button>;
}
function toInsertPayload(item: ProjectPickerItem): InsertAssetPayload {

6
web/src/components/canvas/canvas-script-node.tsx
View File

@@ -465,17 +465,17 @@ function editorRow(shotNumber: number): StoryboardRow {
function RowHandle({ side, top, scale, tone, theme, title, onPointerDown }: { side: "left" | "right"; top: number; scale: number; tone?: StoryboardRow["status"]; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; title?: string; onPointerDown: (event: ReactPointerEvent) => void }) {
const color = tone === "loading" ? theme.accent.primary : tone === "error" ? theme.accent.danger : tone === "success" ? theme.node.activeStroke : theme.node.muted;
const inverseScale = 1 / Math.max(scale, 0.05);
const inverseHitScale = 1 / Math.max(scale, 0.05);
return (
<button
type="button"
aria-label={title || `${side === "left" ? "输入" : "输出"}连接点`}
title={title || `${side === "left" ? "引入参考" : "连接到图片、视频或生成节点"}`}
className={`canvas-connection-handle absolute z-50 flex -translate-y-1/2 cursor-pointer items-center justify-center rounded-full outline-none focus-visible:ring-2 ${side === "left" ? "left-0 -translate-x-1/2" : "right-0 translate-x-1/2"}`}
style={{ top, width: 32 * inverseScale, height: 32 * inverseScale, "--tw-ring-color": theme.accent.primary } as CSSProperties}
style={{ top, width: 32 * inverseHitScale, height: 32 * inverseHitScale, "--tw-ring-color": theme.accent.primary } as CSSProperties}
onPointerDown={onPointerDown}
>
<span className="block rounded-full shadow-sm transition-transform hover:scale-110" style={{ width: 14 * inverseScale, height: 14 * inverseScale, border: `${2 * inverseScale}px solid ${theme.node.panel}`, background: color }} />
<span className="block size-2.5 rounded-full border-2 shadow-sm transition-transform hover:scale-110" style={{ boxSizing: "border-box", borderColor: theme.node.panel, background: color }} />
</button>
);
}

5
web/src/components/canvas/infinite-canvas.tsx
View File

@@ -38,7 +38,7 @@ type PinchState = {
initialScale: number;
};
export function InfiniteCanvas({ containerRef, viewport, backgroundMode = "lines", onViewportChange, onViewportPreviewChange, onCanvasMouseDown, onCanvasDoubleClick, onCanvasDeselect, onContextMenu, onDrop, onFileDragEnter, onFileDragLeave, onFileDragOver, graphicsLayer, children }: InfiniteCanvasProps) {
export function InfiniteCanvas({ containerRef, viewport, backgroundMode = "dots", onViewportChange, onViewportPreviewChange, onCanvasMouseDown, onCanvasDoubleClick, onCanvasDeselect, onContextMenu, onDrop, onFileDragEnter, onFileDragLeave, onFileDragOver, graphicsLayer, children }: InfiniteCanvasProps) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
const panState = useRef({
isPanning: false,
@@ -418,12 +418,13 @@ function CanvasGrid({ mode }: { mode: CanvasBackgroundMode }) {
return (
<div
data-canvas-grid-layer
className="pointer-events-none absolute opacity-40"
className="pointer-events-none absolute"
style={{
inset: "calc(-1 * var(--canvas-grid-size))",
backgroundImage,
backgroundSize: "var(--canvas-grid-size) var(--canvas-grid-size)",
transform: "translate3d(var(--canvas-grid-x), var(--canvas-grid-y), 0)",
opacity: mode === "dots" ? 0.52 : 0.36,
willChange: "transform",
}}
/>