tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(*): 画布创作 - 支持文本多模态生成并优化媒体节点、连接线与视频预览

Browse Source 
Fixes #59
This commit is contained in:
ddCat
2026-07-31 19:44:02 +08:00
parent 8ce97f267b
commit 14da5e8a19
29 changed files with 231 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
backend/internal/service/provider.go
View File

@@ -593,7 +593,7 @@ func runChatCompletionsTextTask(ctx context.Context, input canvasGenerationInput
func textResponseInput(input canvasGenerationInput) (interface{}, error) {
systemPrompt := strings.TrimSpace(input.Config.SystemPrompt)
if len(input.ReferenceImages) == 0 {
if len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
return withSystemPrompt(input.Config, input.Prompt), nil
}
messages := make([]map[string]interface{}, 0, 2)
@@ -617,11 +617,18 @@ func textResponseContent(input canvasGenerationInput) ([]map[string]interface{},
}
content = append(content, map[string]interface{}{"type": "input_image", "image_url": url})
}
for _, video := range input.ReferenceVideos {
url, err := openAIVideoInputURL(video)
if err != nil {
return nil, err
}
content = append(content, map[string]interface{}{"type": "input_video", "video_url": url})
}
return content, nil
}
func textChatContent(input canvasGenerationInput) (interface{}, error) {
if len(input.ReferenceImages) == 0 {
if len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
return input.Prompt, nil
}
content := []map[string]interface{}{{"type": "text", "text": input.Prompt}}
@@ -632,6 +639,13 @@ func textChatContent(input canvasGenerationInput) (interface{}, error) {
}
content = append(content, map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": url}})
}
for _, video := range input.ReferenceVideos {
url, err := openAIVideoInputURL(video)
if err != nil {
return nil, err
}
content = append(content, map[string]interface{}{"type": "video_url", "video_url": map[string]interface{}{"url": url}})
}
return content, nil
}
@@ -653,6 +667,24 @@ func openAIImageInputURL(media providerMedia) (string, error) {
return "", errors.New("OpenAI 文本多模态参考图片需要公网 URL 或 base64 data URL")
}
func openAIVideoInputURL(media providerMedia) (string, error) {
value := strings.TrimSpace(media.DataURL)
if strings.HasPrefix(value, "data:video/") {
return value, nil
}
if strings.HasPrefix(value, "data:") {
return "", errors.New("参考视频 MIME 类型无效，请重新读取或上传视频")
}
value = strings.TrimSpace(media.URL)
if strings.HasPrefix(value, "data:video/") || isPublicMediaURL(value) {
return value, nil
}
if strings.HasPrefix(value, "data:") {
return "", errors.New("参考视频 MIME 类型无效，请重新读取或上传视频")
}
return "", errors.New("文本多模态参考视频需要公网 URL 或 base64 data URL")
}
func shouldFallbackTextToChat(err error) bool {
var httpErr providerHTTPError
if !errors.As(err, &httpErr) {

28
backend/internal/service/provider_test.go
View File

@@ -117,13 +117,16 @@ func TestDoBinaryRejectsOversizedProviderResponse(t *testing.T) {
}
}
func TestTextResponseInputIncludesReferenceImages(t *testing.T) {
func TestTextResponseInputIncludesReferenceMedia(t *testing.T) {
input := canvasGenerationInput{
Prompt: "describe this image",
Config: providerConfig{SystemPrompt: "answer in Chinese"},
ReferenceImages: []providerMedia{
{ID: "image-1", Name: "image.png", Type: "image/png", DataURL: testReferenceImageDataURL},
},
ReferenceVideos: []providerMedia{
{ID: "video-1", Name: "video.mp4", Type: "video/mp4", URL: "https://example.com/reference.mp4"},
},
}
value, err := textResponseInput(input)
@@ -144,8 +147,8 @@ func TestTextResponseInputIncludesReferenceImages(t *testing.T) {
if !ok {
t.Fatalf("user content = %T, want []map[string]interface{}", messages[1]["content"])
}
if len(content) != 2 {
t.Fatalf("len(content) = %d, want 2", len(content))
if len(content) != 3 {
t.Fatalf("len(content) = %d, want 3", len(content))
}
if content[0]["type"] != "input_text" || content[0]["text"] != "describe this image" {
t.Fatalf("text content = %#v", content[0])
@@ -153,14 +156,20 @@ func TestTextResponseInputIncludesReferenceImages(t *testing.T) {
if content[1]["type"] != "input_image" || content[1]["image_url"] != testReferenceImageDataURL {
t.Fatalf("image content = %#v", content[1])
}
if content[2]["type"] != "input_video" || content[2]["video_url"] != "https://example.com/reference.mp4" {
t.Fatalf("video content = %#v", content[2])
}
}
func TestTextChatContentIncludesReferenceImages(t *testing.T) {
func TestTextChatContentIncludesReferenceMedia(t *testing.T) {
input := canvasGenerationInput{
Prompt: "describe this image",
ReferenceImages: []providerMedia{
{ID: "image-1", Name: "image.png", Type: "image/png", DataURL: testReferenceImageDataURL},
},
ReferenceVideos: []providerMedia{
{ID: "video-1", Name: "video.mp4", Type: "video/mp4", URL: "https://example.com/reference.mp4"},
},
}
value, err := textChatContent(input)
@@ -171,8 +180,8 @@ func TestTextChatContentIncludesReferenceImages(t *testing.T) {
if !ok {
t.Fatalf("textChatContent() = %T, want []map[string]interface{}", value)
}
if len(content) != 2 {
t.Fatalf("len(content) = %d, want 2", len(content))
if len(content) != 3 {
t.Fatalf("len(content) = %d, want 3", len(content))
}
if content[0]["type"] != "text" || content[0]["text"] != "describe this image" {
t.Fatalf("text content = %#v", content[0])
@@ -184,6 +193,13 @@ func TestTextChatContentIncludesReferenceImages(t *testing.T) {
if content[1]["type"] != "image_url" || imageURL["url"] != testReferenceImageDataURL {
t.Fatalf("image content = %#v", content[1])
}
videoURL, ok := content[2]["video_url"].(map[string]interface{})
if !ok {
t.Fatalf("video_url = %T, want map[string]interface{}", content[2]["video_url"])
}
if content[2]["type"] != "video_url" || videoURL["url"] != "https://example.com/reference.mp4" {
t.Fatalf("video content = %#v", content[2])
}
}
func TestTextReferenceImageRejectsInternalAssetURL(t *testing.T) {

38
web/src/components/asset-media-preview.tsx Normal file
View File

@@ -0,0 +1,38 @@
import type { ReactNode } from "react";
import type { Asset } from "@/stores/use-asset-store";
type AssetMediaPreviewProps = {
asset?: Asset | null;
alt: string;
className?: string;
fallback?: ReactNode;
};
export function AssetMediaPreview({ asset, alt, className = "", fallback = null }: AssetMediaPreviewProps) {
if (!asset) return fallback;
if (asset.kind === "video" && asset.data.url) {
const poster = asset.coverUrl && asset.coverUrl !== asset.data.url ? asset.coverUrl : undefined;
return (
<video
src={asset.data.url}
poster={poster}
aria-label={alt}
muted
playsInline
preload="metadata"
className={className}
onLoadedMetadata={(event) => {
// 主动触发首帧附近的解码，避免只有 metadata 时长期停留在空白画面。
const video = event.currentTarget;
if (!poster && video.currentTime === 0 && video.duration > 0) video.currentTime = Math.min(0.001, video.duration);
}}
/>
);
}
const imageUrl = asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : "");
if (!imageUrl) return fallback;
return <img src={imageUrl} alt={alt} loading="lazy" decoding="async" className={className} />;
}

12
web/src/components/canvas/asset-picker-modal.tsx
View File

@@ -3,6 +3,7 @@ import { Input, Modal, Pagination, Tag } from "antd";
import { Search } from "lucide-react";
import { WorkspaceState } from "@/components/layout/workspace-state";
import { AssetMediaPreview } from "@/components/asset-media-preview";
import { cn } from "@/lib/utils";
import { useAssetStore, type Asset } from "@/stores/use-asset-store";
@@ -40,18 +41,15 @@ const kindOptions = [
{ label: "音频", value: "audio" },
];
function PickerCard({ title, kind, cover, onClick }: { title: string; kind: string; cover: string; onClick: () => void }) {
function PickerCard({ asset, onClick }: { asset: InsertableAsset; onClick: () => void }) {
const { title, kind } = asset;
return (
<button
type="button"
className="group relative cursor-pointer overflow-hidden rounded-lg border border-stone-200 bg-white text-left transition hover:border-stone-400 hover:shadow-md dark:border-stone-700 dark:bg-stone-900 dark:hover:border-stone-500"
onClick={onClick}
>
{cover ? (
<img src={cover} alt={title} loading="lazy" decoding="async" className="aspect-[4/3] w-full object-cover" />
) : (
<div className="flex aspect-[4/3] items-center justify-center bg-stone-100 p-3 text-center text-xs leading-5 text-stone-500 dark:bg-stone-800 dark:text-stone-400">{title}</div>
)}
<AssetMediaPreview asset={asset} alt={title} className="aspect-[4/3] w-full bg-black object-cover" fallback={<div className="flex aspect-[4/3] items-center justify-center bg-stone-100 p-3 text-center text-xs leading-5 text-stone-500 dark:bg-stone-800 dark:text-stone-400">{title}</div>} />
<div className="p-2.5">
<div className="flex items-center justify-between gap-2">
<span className="line-clamp-1 text-xs font-medium text-stone-800 dark:text-stone-200">{title}</span>
@@ -131,7 +129,7 @@ function MyAssetsTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) =>
{visible.length ? (
<div className="grid grid-cols-4 gap-3">
{visible.map((asset) => (
<PickerCard key={asset.id} title={asset.title} kind={asset.kind} cover={asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : "")} onClick={() => handleInsert(asset)} />
<PickerCard key={asset.id} asset={asset} onClick={() => handleInsert(asset)} />
))}
</div>
) : (

14
web/src/components/canvas/canvas-leafer-graphics-layer.tsx
View File

@@ -2,7 +2,7 @@ import { useLayoutEffect, useRef, type RefObject } from "react";
import { Group, Leafer, Path, Rect } from "leafer-ui";
import { activeConnectionPath, canvasConnectionPath } from "@/components/canvas/canvas-connections";
import { subscribeCanvasSelectionPreview, subscribeCanvasViewportPreview } from "@/lib/canvas/canvas-live-viewport";
import { subscribeCanvasGraphicsViewportPreview, subscribeCanvasSelectionPreview } from "@/lib/canvas/canvas-live-viewport";
import { calculateCanvasPreviewTransform, sameCanvasViewport, shouldRebaseCanvasRaster } from "@/lib/canvas/canvas-leafer-viewport";
import type { CanvasTheme } from "@/lib/canvas-theme";
import type { CanvasDisplayConnection, CanvasNodeData, ConnectionHandle, Position, SelectionBox, ViewportTransform } from "@/types/canvas";
@@ -72,17 +72,17 @@ export function CanvasLeaferGraphicsLayer(props: CanvasLeaferGraphicsLayerProps)
underlay.leafer.resize(size);
overlay.leafer.resize(size);
syncViewport(rasterViewportRef.current, size.width, size.height, underlay, overlay, propsRef.current);
if (isZoomPreview(container, viewportRef.current, rasterViewportRef.current)) {
if (isViewportPreview(container, viewportRef.current, rasterViewportRef.current)) {
applyScenePreview(viewportRef.current, rasterViewportRef.current, underlay, overlay);
}
};
const resizeObserver = new ResizeObserver(resize);
resizeObserver.observe(container);
window.addEventListener("resize", resize);
const unsubscribe = subscribeCanvasViewportPreview(container, (next) => {
const unsubscribe = subscribeCanvasGraphicsViewportPreview(container, (next) => {
viewportRef.current = next;
const rect = container.getBoundingClientRect();
if (isZoomPreview(container, next, rasterViewportRef.current)) {
if (isViewportPreview(container, next, rasterViewportRef.current)) {
if (shouldRebaseCanvasRaster(next, rasterViewportRef.current)) {
syncViewport(next, rect.width, rect.height, underlay, overlay, propsRef.current);
rasterViewportRef.current = next;
@@ -151,7 +151,7 @@ export function CanvasLeaferGraphicsLayer(props: CanvasLeaferGraphicsLayerProps)
if (!underlay || !overlay || !container) return;
const rect = container.getBoundingClientRect();
syncViewport(rasterViewportRef.current, rect.width, rect.height, underlay, overlay, props);
if (isZoomPreview(container, viewportRef.current, rasterViewportRef.current)) {
if (isViewportPreview(container, viewportRef.current, rasterViewportRef.current)) {
applyScenePreview(viewportRef.current, rasterViewportRef.current, underlay, overlay);
}
}, [props.alignmentGuides, props.containerRef, props.theme]);
@@ -268,8 +268,8 @@ function syncViewport(viewport: ViewportTransform, width: number, height: number
});
}
function isZoomPreview(container: HTMLDivElement, viewport: ViewportTransform, rasterViewport: ViewportTransform) {
return container.dataset.canvasViewportInteracting === "true" && Math.abs(viewport.k - rasterViewport.k) > 0.0001;
function isViewportPreview(container: HTMLDivElement, viewport: ViewportTransform, rasterViewport: ViewportTransform) {
return container.dataset.canvasViewportInteracting === "true" && !sameCanvasViewport(viewport, rasterViewport);
}
function applyScenePreview(viewport: ViewportTransform, rasterViewport: ViewportTransform, ...scenes: LeaferScene[]) {

2
web/src/components/canvas/canvas-node-hover-toolbar.tsx
View File

@@ -233,7 +233,7 @@ export function CanvasNodeHoverToolbar({
...(hasVideo && !simpleMode ? [{ id: "extractLastFrame", title: extractingVideoFrame ? "正在截取尾帧" : "截取尾帧", label: extractingVideoFrame ? "截取中" : "尾帧", icon: extractingVideoFrame ? <LoaderCircle className="size-3.5 animate-spin" /> : <GalleryHorizontalEnd className="size-3.5" />, onClick: () => onExtractVideoLastFrame(node), disabled: extractingVideoFrame }] : []),
...(hasImage || hasVideo || isEditableText ? [{ id: "saveAsset", title: "加入我的素材", label: "存素材", icon: <FolderPlus className="size-3.5" />, onClick: () => onSaveAsset(node) }] : []),
...(hasImage || hasVideo || hasAudio ? [{ id: "download", title: hasAudio ? "下载音频" : hasVideo ? "下载视频" : "下载图片", label: "下载", icon: <Download className="size-3.5" />, onClick: () => onDownload(node) }] : []),
...(canOpenDialog ? [{ id: "edit", title: isEditableText ? "生成提示与参数" : "编辑", label: isEditableText ? "生成设置" : "编辑", icon: <MessageSquare className="size-3.5" />, onClick: () => onToggleDialog(node) }] : []),
...(canOpenDialog ? [{ id: "edit", title: isEditableText ? "调用文本模型生成内容" : "编辑", label: isEditableText ? "文本生成" : "编辑", icon: <MessageSquare className="size-3.5" />, onClick: () => onToggleDialog(node) }] : []),
...(isEditableText ? [{ id: "editText", title: "放大编辑文本", label: "放大编辑", icon: <Maximize2 className="size-3.5" />, onClick: () => onEditText(node) }] : []),
...(isEditableText ? [{ id: "generateImage", title: "用文本生图", label: "生图", icon: <ImageIcon className="size-3.5" />, onClick: () => onGenerateImage(node) }] : []),
...(isConfig && !simpleMode ? [{ id: "config", title: "生成配置", label: "生成配置", icon: <Settings2 className="size-3.5" />, onClick: () => onToggleDialog(node) }] : []),

10
web/src/components/canvas/canvas-node.tsx
View File

@@ -16,6 +16,7 @@ import { storyboardMinNodeHeight } from "./canvas-script-node";
import { CanvasNodeType, type CanvasNodeData, type Position } from "@/types/canvas";
import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { loadCanvasDrawingPreview } from "@/lib/canvas/canvas-drawing-storage";
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
type ResizeCorner = "top-left" | "top-right" | "bottom-left" | "bottom-right";
type CanvasTheme = (typeof canvasThemes)[keyof typeof canvasThemes];
@@ -203,8 +204,9 @@ export const CanvasNode = React.memo(function CanvasNode({
const dx = (event.clientX - resizeRef.current.startX) / scale;
const dy = (event.clientY - resizeRef.current.startY) / scale;
const minWidth = data.type === CanvasNodeType.Script ? 800 : 220;
const minHeight = scriptMinHeight || 160;
const isMediaNode = data.type === CanvasNodeType.Image || data.type === CanvasNodeType.Video;
const minWidth = data.type === CanvasNodeType.Script ? 800 : isMediaNode ? MEDIA_NODE_MIN_SIZE.width : 220;
const minHeight = scriptMinHeight || (isMediaNode ? MEDIA_NODE_MIN_SIZE.height : 160);
const startRight = resizeRef.current.startLeft + resizeRef.current.startWidth;
const startBottom = resizeRef.current.startTop + resizeRef.current.startHeight;
const fromLeft = resizeRef.current.corner.includes("left");
@@ -1119,6 +1121,8 @@ function ResizeHandle({ corner, onMouseDown }: { corner: ResizeCorner; onMouseDo
return <div className={`absolute z-50 size-7 ${positionClass}`} onMouseDown={(event) => onMouseDown(event, corner)} />;
}
const NODE_EXTERNAL_HEADER_MIN_SCALE = 0.35;
function NodeExternalHeader({ node, scale, active, editable, editing, draft, theme, onDraftChange, onEdit, onCommit, onCancel }: {
node: CanvasNodeData;
scale: number;
@@ -1132,6 +1136,8 @@ function NodeExternalHeader({ node, scale, active, editable, editing, draft, the
onCommit: () => void;
onCancel: () => void;
}) {
// 标题保持屏幕尺寸只适用于近景；远景继续反向缩放会遮住节点和连线。
if (scale < NODE_EXTERNAL_HEADER_MIN_SCALE && !editing) return null;
const inverseScale = 1 / Math.max(scale, 0.05);
const Icon = nodeTypeIcon(node.type);

5
web/src/components/canvas/canvas-project-asset-modal.tsx
View File

@@ -3,6 +3,7 @@ import { Button, Modal } from "antd";
import { Check, FileText, Image as ImageIcon, Music2, UserRound, Video } from "lucide-react";
import type { InsertAssetPayload } from "@/components/canvas/asset-picker-modal";
import { AssetMediaPreview } from "@/components/asset-media-preview";
import { WorkspaceState } from "@/components/layout/workspace-state";
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
import { resourceFileUrl } from "@/services/api/resources";
@@ -57,11 +58,11 @@ function ProjectAssetCard({ item, selected, onToggle }: { item: ProjectPickerIte
const character = item.character;
const media = item.media;
const coverRepresentation = character?.character?.representations.find((representation) => representation.role === "turnaround_sheet") || character?.character?.representations.find((representation) => representation.role === "primary") || character?.character?.representations.find((representation) => representation.role === "front");
const cover = coverRepresentation ? resourceFileUrl(coverRepresentation.resourceId) : media?.coverUrl || (media?.kind === "image" ? media.data.dataUrl : "");
const cover = coverRepresentation ? resourceFileUrl(coverRepresentation.resourceId) : "";
const Icon = character ? UserRound : media?.kind === "video" ? Video : media?.kind === "audio" ? Music2 : media?.kind === "text" ? FileText : ImageIcon;
const label = character ? "角色卡" : media?.kind === "video" ? "视频" : media?.kind === "audio" ? "音频" : media?.kind === "text" ? "文本" : "图片";
const title = character?.title || media?.title || "未命名资产";
return <button type="button" onClick={onToggle} className={`relative min-w-0 overflow-hidden rounded-md border text-left transition-colors ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent-soft)]" : "border-border/80 hover:border-foreground/30"}`}><div className="relative aspect-[4/3] overflow-hidden bg-foreground/[.04]">{cover ? <img src={cover} alt={title} loading="lazy" decoding="async" className={`h-full w-full ${character ? "object-contain p-1" : "object-cover"}`} /> : media?.kind === "video" ? <video src={media.data.url} muted preload="metadata" className="h-full w-full bg-black object-cover" /> : <div className="grid h-full place-items-center text-foreground/25"><Icon className="size-7" /></div>}<span className={`absolute right-1.5 top-1.5 grid size-5 place-items-center rounded border ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent)] text-white" : "border-white/60 bg-black/25 text-transparent backdrop-blur"}`}><Check className="size-3" /></span><span className="absolute bottom-1.5 left-1.5 rounded bg-black/55 px-1.5 py-0.5 text-[9px] text-white">{label}</span></div><div className="px-2 py-1.5"><div className="truncate text-[11px] font-medium">{title}</div>{character ? <div className="mt-0.5 truncate text-[9px] text-foreground/42">{character.character?.visualStatus === "ready" ? "形象就绪" : "形象待完善"} · {character.character?.voiceStatus === "ready" ? "声音已绑定" : "声音未绑定"}</div> : null}</div></button>;
return <button type="button" onClick={onToggle} className={`relative min-w-0 overflow-hidden rounded-md border text-left transition-colors ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent-soft)]" : "border-border/80 hover:border-foreground/30"}`}><div className="relative aspect-[4/3] overflow-hidden bg-foreground/[.04]">{cover ? <img src={cover} alt={title} loading="lazy" decoding="async" className="h-full w-full object-contain p-1" /> : <AssetMediaPreview asset={media} alt={title} className="h-full w-full bg-black object-cover" fallback={<div className="grid h-full place-items-center text-foreground/25"><Icon className="size-7" /></div>} />}<span className={`absolute right-1.5 top-1.5 grid size-5 place-items-center rounded border ${selected ? "border-[var(--workspace-accent)] bg-[var(--workspace-accent)] text-white" : "border-white/60 bg-black/25 text-transparent backdrop-blur"}`}><Check className="size-3" /></span><span className="absolute bottom-1.5 left-1.5 rounded bg-black/55 px-1.5 py-0.5 text-[9px] text-white">{label}</span></div><div className="px-2 py-1.5"><div className="truncate text-[11px] font-medium">{title}</div>{character ? <div className="mt-0.5 truncate text-[9px] text-foreground/42">{character.character?.visualStatus === "ready" ? "形象就绪" : "形象待完善"} · {character.character?.voiceStatus === "ready" ? "声音已绑定" : "声音未绑定"}</div> : null}</div></button>;
}
function toInsertPayload(item: ProjectPickerItem): InsertAssetPayload {

6
web/src/components/canvas/canvas-script-node.tsx
View File

@@ -465,17 +465,17 @@ function editorRow(shotNumber: number): StoryboardRow {
function RowHandle({ side, top, scale, tone, theme, title, onPointerDown }: { side: "left" | "right"; top: number; scale: number; tone?: StoryboardRow["status"]; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; title?: string; onPointerDown: (event: ReactPointerEvent) => void }) {
const color = tone === "loading" ? theme.accent.primary : tone === "error" ? theme.accent.danger : tone === "success" ? theme.node.activeStroke : theme.node.muted;
const inverseScale = 1 / Math.max(scale, 0.05);
const inverseHitScale = 1 / Math.max(scale, 0.05);
return (
<button
type="button"
aria-label={title || `${side === "left" ? "输入" : "输出"}连接点`}
title={title || `${side === "left" ? "引入参考" : "连接到图片、视频或生成节点"}`}
className={`canvas-connection-handle absolute z-50 flex -translate-y-1/2 cursor-pointer items-center justify-center rounded-full outline-none focus-visible:ring-2 ${side === "left" ? "left-0 -translate-x-1/2" : "right-0 translate-x-1/2"}`}
style={{ top, width: 32 * inverseScale, height: 32 * inverseScale, "--tw-ring-color": theme.accent.primary } as CSSProperties}
style={{ top, width: 32 * inverseHitScale, height: 32 * inverseHitScale, "--tw-ring-color": theme.accent.primary } as CSSProperties}
onPointerDown={onPointerDown}
>
<span className="block rounded-full shadow-sm transition-transform hover:scale-110" style={{ width: 14 * inverseScale, height: 14 * inverseScale, border: `${2 * inverseScale}px solid ${theme.node.panel}`, background: color }} />
<span className="block size-2.5 rounded-full border-2 shadow-sm transition-transform hover:scale-110" style={{ boxSizing: "border-box", borderColor: theme.node.panel, background: color }} />
</button>
);
}

5
web/src/components/canvas/infinite-canvas.tsx
View File

@@ -38,7 +38,7 @@ type PinchState = {
initialScale: number;
};
export function InfiniteCanvas({ containerRef, viewport, backgroundMode = "lines", onViewportChange, onViewportPreviewChange, onCanvasMouseDown, onCanvasDoubleClick, onCanvasDeselect, onContextMenu, onDrop, onFileDragEnter, onFileDragLeave, onFileDragOver, graphicsLayer, children }: InfiniteCanvasProps) {
export function InfiniteCanvas({ containerRef, viewport, backgroundMode = "dots", onViewportChange, onViewportPreviewChange, onCanvasMouseDown, onCanvasDoubleClick, onCanvasDeselect, onContextMenu, onDrop, onFileDragEnter, onFileDragLeave, onFileDragOver, graphicsLayer, children }: InfiniteCanvasProps) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
const panState = useRef({
isPanning: false,
@@ -418,12 +418,13 @@ function CanvasGrid({ mode }: { mode: CanvasBackgroundMode }) {
return (
<div
data-canvas-grid-layer
className="pointer-events-none absolute opacity-40"
className="pointer-events-none absolute"
style={{
inset: "calc(-1 * var(--canvas-grid-size))",
backgroundImage,
backgroundSize: "var(--canvas-grid-size) var(--canvas-grid-size)",
transform: "translate3d(var(--canvas-grid-x), var(--canvas-grid-y), 0)",
opacity: mode === "dots" ? 0.52 : 0.36,
willChange: "transform",
}}
/>

4
web/src/constant/canvas.ts
View File

@@ -9,13 +9,13 @@ type CanvasNodeSpec = {
};
export const NODE_DEFAULT_SIZE = {
[CanvasNodeType.Image]: { width: 340, height: 240, title: "New Generation" },
[CanvasNodeType.Image]: { width: 420, height: 300, title: "New Generation" },
[CanvasNodeType.Text]: { width: 340, height: 240, title: "Note" },
[CanvasNodeType.Drawing]: { width: 440, height: 300, title: "绘图" },
[CanvasNodeType.Script]: { width: 920, height: 360, title: "分镜脚本" },
[CanvasNodeType.Skill]: { width: 360, height: 220, title: "技能" },
[CanvasNodeType.Config]: { width: 340, height: 300, title: "生成配置" },
[CanvasNodeType.Video]: { width: 420, height: 236, title: "Video" },
[CanvasNodeType.Video]: { width: 480, height: 270, title: "Video" },
[CanvasNodeType.Audio]: { width: 340, height: 120, title: "Audio" },
[CanvasNodeType.Frame]: { width: 760, height: 520, title: "未命名背板" },
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;

54
web/src/lib/canvas-theme.ts
View File

@@ -51,48 +51,48 @@ export const canvasThemes = {
},
dark: {
canvas: {
background: "#111111",
dot: "rgba(245,245,245,.16)",
line: "rgba(245,245,245,.065)",
selectionFill: "rgba(91,110,225,.16)",
background: "#090a0c",
dot: "rgba(174,184,199,.18)",
line: "rgba(174,184,199,.06)",
selectionFill: "rgba(96,126,234,.18)",
},
node: {
label: "#a3a3a3",
fill: "#242424",
panel: "#202020",
stroke: "rgba(255,255,255,.13)",
activeStroke: "#f5f5f5",
placeholder: "#777777",
text: "#f5f5f5",
fill: "#181818",
panel: "#141414",
stroke: "rgba(255,255,255,.12)",
activeStroke: "#f1f1f1",
placeholder: "#737373",
text: "#ededed",
muted: "#a3a3a3",
faint: "#666666",
},
frame: {
fill: "rgba(255,255,255,.025)",
stroke: "rgba(255,255,255,.18)",
activeFill: "rgba(91,110,225,.08)",
activeStroke: "#8290f0",
preview: "rgba(24,24,24,.86)",
stroke: "rgba(190,198,210,.15)",
activeFill: "rgba(96,126,234,.1)",
activeStroke: "#8f9bd6",
preview: "rgba(20,20,20,.94)",
},
toolbar: {
panel: "rgba(36,36,36,.94)",
border: "rgba(255,255,255,.12)",
panel: "rgba(20,20,20,.97)",
border: "rgba(255,255,255,.1)",
item: "#d4d4d4",
itemHover: "rgba(255,255,255,.08)",
activeBg: "rgba(255,255,255,.13)",
activeText: "#ffffff",
itemHover: "rgba(255,255,255,.07)",
activeBg: "rgba(143,155,214,.14)",
activeText: "#f5f6f8",
},
spatial: {
surface: "rgba(30,30,32,.72)",
elevated: "rgba(22,22,24,.94)",
dropzone: "rgba(10,10,12,.78)",
glow: "rgba(130,144,240,.2)",
glowStrong: "rgba(130,144,240,.58)",
shadow: "rgba(0,0,0,.46)",
surface: "rgba(22,22,22,.82)",
elevated: "rgba(15,15,15,.97)",
dropzone: "rgba(8,8,8,.9)",
glow: "rgba(107,120,190,.18)",
glowStrong: "rgba(143,155,214,.48)",
shadow: "rgba(0,0,0,.6)",
},
accent: {
primary: "#8290f0",
primarySoft: "rgba(91,110,225,.2)",
primary: "#8f9bd6",
primarySoft: "rgba(107,120,190,.2)",
danger: "#fb7185",
},
},

6
web/src/lib/canvas/canvas-generation-task-sync.ts
View File

@@ -1,5 +1,5 @@
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { compositeEmotionImage } from "@/lib/canvas/canvas-emotion";
import { storeGeneratedAudio } from "@/services/api/audio";
import { storeGeneratedVideo } from "@/services/api/video";
@@ -17,8 +17,6 @@ type BackendGenerationResult = {
text?: string;
};
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
export function parseBackendGenerationResult(task: GenerationTask): BackendGenerationResult {
if (!task.resultJson) throw new Error("后端任务没有返回结果");
@@ -102,7 +100,7 @@ export async function buildGenerationTaskNodeResult(node: CanvasNodeData, task:
const video = result.video.storageKey
? { url: await resolveMediaUrl(result.video.storageKey, result.video.dataUrl), storageKey: result.video.storageKey, width: result.video.width, height: result.video.height, durationMs: result.video.durationMs, bytes: result.video.bytes || 0, mimeType: result.video.mimeType || "video/mp4" }
: await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || node.width || VIDEO_NODE_MAX_WIDTH, video.height || node.height || VIDEO_NODE_MAX_HEIGHT, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const videoSize = fitNodeSize(video.width || node.width || VIDEO_NODE_MAX_SIZE.width, video.height || node.height || VIDEO_NODE_MAX_SIZE.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
return {
...node,
type: CanvasNodeType.Video,

9
web/src/lib/canvas/canvas-live-viewport.ts
View File

@@ -1,6 +1,7 @@
import type { SelectionBox, ViewportTransform } from "@/types/canvas";
export const CANVAS_VIEWPORT_PREVIEW_EVENT = "canvas:viewport-preview";
export const CANVAS_GRAPHICS_VIEWPORT_PREVIEW_EVENT = "canvas:graphics-viewport-preview";
export const CANVAS_SELECTION_PREVIEW_EVENT = "canvas:selection-preview";
export function applyCanvasLiveViewport(container: HTMLDivElement | null, viewport: ViewportTransform, notify = true) {
@@ -15,6 +16,8 @@ export function applyCanvasLiveViewport(container: HTMLDivElement | null, viewpo
container.style.setProperty("--canvas-grid-x", `${viewport.x % gridSize}px`);
container.style.setProperty("--canvas-grid-y", `${viewport.y % gridSize}px`);
container.style.setProperty("--canvas-dot-size", viewport.k < 0.12 ? "0.8px" : "1.15px");
// 图形层必须逐帧跟随 DOM 世界层；浮层和滚动通知仍可按原频率节流。
container.dispatchEvent(new CustomEvent<ViewportTransform>(CANVAS_GRAPHICS_VIEWPORT_PREVIEW_EVENT, { detail: viewport }));
if (notify) {
container.dispatchEvent(new CustomEvent<ViewportTransform>(CANVAS_VIEWPORT_PREVIEW_EVENT, { detail: viewport }));
// Ant Design overlays watch scrollable ancestors, but CSS transforms do not emit layout events.
@@ -22,6 +25,12 @@ export function applyCanvasLiveViewport(container: HTMLDivElement | null, viewpo
}
}
export function subscribeCanvasGraphicsViewportPreview(container: HTMLDivElement, listener: (viewport: ViewportTransform) => void) {
const handlePreview = (event: Event) => listener((event as CustomEvent<ViewportTransform>).detail);
container.addEventListener(CANVAS_GRAPHICS_VIEWPORT_PREVIEW_EVENT, handlePreview);
return () => container.removeEventListener(CANVAS_GRAPHICS_VIEWPORT_PREVIEW_EVENT, handlePreview);
}
export function subscribeCanvasViewportPreview(container: HTMLDivElement, listener: (viewport: ViewportTransform) => void) {
const handlePreview = (event: Event) => listener((event as CustomEvent<ViewportTransform>).detail);
container.addEventListener(CANVAS_VIEWPORT_PREVIEW_EVENT, handlePreview);

32
web/src/lib/canvas/canvas-node-size.ts
View File

@@ -1,7 +1,15 @@
export function fitNodeSize(width: number, height: number, maxWidth = 640, maxHeight = 640) {
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
export const MEDIA_NODE_MIN_SIZE = { width: 300, height: 220 } as const;
export const VIDEO_NODE_MAX_SIZE = { width: 480, height: 480 } as const;
export function fitNodeSize(width: number, height: number, maxWidth = 640, maxHeight = 640, minWidth = MEDIA_NODE_MIN_SIZE.width, minHeight = MEDIA_NODE_MIN_SIZE.height) {
const w = Math.max(1, width);
const h = Math.max(1, height);
const scale = Math.min(1, maxWidth / w, maxHeight / h);
// 媒体节点既要保留原始比例，也要给生成状态、操作按钮留下稳定的可读空间。
const preferredScale = Math.min(1, maxWidth / w, maxHeight / h);
const minimumScale = Math.max(minWidth / w, minHeight / h);
const scale = Math.max(preferredScale, minimumScale);
return { width: w * scale, height: h * scale };
}
@@ -16,5 +24,23 @@ export function nodeSizeFromRatio(size: string, baseWidth: number, baseHeight: n
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return null;
const ratio = width / Math.max(1, height);
if (ratio < 0.25 || ratio > 4) return { width: baseWidth, height: baseHeight };
return ratio >= baseWidth / baseHeight ? { width: baseWidth, height: baseWidth / ratio } : { width: baseHeight * ratio, height: baseHeight };
const candidateSize = ratio >= baseWidth / baseHeight ? { width: baseWidth, height: baseWidth / ratio } : { width: baseHeight * ratio, height: baseHeight };
return fitNodeSize(candidateSize.width, candidateSize.height, baseWidth, baseHeight);
}
export function ensureMediaNodeMinimumSize(node: CanvasNodeData) {
if (node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video) return node;
if (node.width >= MEDIA_NODE_MIN_SIZE.width && node.height >= MEDIA_NODE_MIN_SIZE.height) return node;
const scale = Math.max(1, MEDIA_NODE_MIN_SIZE.width / Math.max(1, node.width), MEDIA_NODE_MIN_SIZE.height / Math.max(1, node.height));
const width = node.width * scale;
const height = node.height * scale;
return {
...node,
position: {
x: node.position.x + node.width / 2 - width / 2,
y: node.position.y + node.height / 2 - height / 2,
},
width,
height,
};
}

4
web/src/lib/canvas/canvas-project-generation.ts
View File

@@ -7,6 +7,7 @@ import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { normalizeVideoDuration, normalizeVideoResolution } from "@/lib/video-generation-options";
import { isSeedanceVideoConfig } from "@/lib/seedance-video";
import { imageMetadata, parseBackendGenerationResult } from "@/lib/canvas/canvas-generation-task-sync";
import { ensureMediaNodeMinimumSize } from "@/lib/canvas/canvas-node-size";
import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel";
import { CanvasNodeType, type CanvasAssistantSession, type CanvasConnection, type CanvasImageGenerationType, type CanvasNodeData, type CanvasNodeMetadata, type CanvasVideoEditOperation } from "@/types/canvas";
import type { ReferenceImage } from "@/types/image";
@@ -363,7 +364,8 @@ export function supportsVideoReferenceAudio(config: AiConfig) {
export function resetInterruptedGeneration(nodes: CanvasNodeData[]) {
const configHeight = NODE_DEFAULT_SIZE[CanvasNodeType.Config].height;
return nodes.map((node) => {
const resizedNode = node.type === CanvasNodeType.Config && node.height < configHeight ? { ...node, height: configHeight } : node.type === CanvasNodeType.Script && node.height < NODE_DEFAULT_SIZE[CanvasNodeType.Script].height ? { ...node, height: NODE_DEFAULT_SIZE[CanvasNodeType.Script].height } : node;
const mediaNode = ensureMediaNodeMinimumSize(node);
const resizedNode = mediaNode.type === CanvasNodeType.Config && mediaNode.height < configHeight ? { ...mediaNode, height: configHeight } : mediaNode.type === CanvasNodeType.Script && mediaNode.height < NODE_DEFAULT_SIZE[CanvasNodeType.Script].height ? { ...mediaNode, height: NODE_DEFAULT_SIZE[CanvasNodeType.Script].height } : mediaNode;
return resizedNode.metadata?.status === "loading" ? { ...resizedNode, metadata: { ...resizedNode.metadata, errorDetails: "正在从任务中心恢复生成状态..." } } : resizedNode;
});
}

13
web/src/pages/assets/index.tsx
View File

@@ -1,9 +1,10 @@
import { Copy, Download, FileUp, MoreHorizontal, PencilLine, Plus, Search, Trash2, Upload } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { App, Button, Drawer, Dropdown, Form, Image, Input, Modal, Select, Space, Tag, Typography } from "antd";
import { App, Button, Drawer, Dropdown, Form, Input, Modal, Select, Space, Tag, Typography } from "antd";
import { CollectionGrid, ListToolbar, PageHeader, PaginationBar, WorkspacePage } from "@/components/layout/workspace-page";
import { WorkspaceState } from "@/components/layout/workspace-state";
import { AssetMediaPreview } from "@/components/asset-media-preview";
import { saveAs } from "file-saver";
import { useCopyText } from "@/hooks/use-copy-text";
@@ -424,13 +425,12 @@ export default function AssetsPage() {
}
function AssetCard({ asset, selected, onSelect, onOpen, onEdit, onCopy, onDownload, onDelete }: { asset: LibraryAsset; selected: boolean; onSelect: (selected: boolean) => void; onOpen: () => void; onEdit: () => void; onCopy: (asset: LibraryAsset) => void; onDownload: (asset: LibraryAsset) => void; onDelete: () => void }) {
const cover = asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : "");
const summary = assetSummary(asset);
return (
<article className={`app-collection-card group h-full overflow-hidden ${selected ? "border-foreground/45" : ""}`}>
<div className="relative">
<button type="button" className="block w-full text-left" onClick={onOpen}>
{cover ? <img src={cover} alt={asset.title} loading="lazy" decoding="async" className="aspect-[16/10] w-full object-cover" /> : <div className="flex aspect-[16/10] items-center justify-center bg-stone-100 p-4 text-center text-xs leading-5 text-stone-600 dark:bg-stone-900 dark:text-stone-300">{asset.kind === "text" ? asset.data.content : "暂无封面"}</div>}
<AssetMediaPreview asset={asset} alt={asset.title} className="aspect-[16/10] w-full bg-black object-cover" fallback={<div className="flex aspect-[16/10] items-center justify-center bg-stone-100 p-4 text-center text-xs leading-5 text-stone-600 dark:bg-stone-900 dark:text-stone-300">{asset.kind === "text" ? asset.data.content : "暂无封面"}</div>} />
</button>
<input type="checkbox" checked={selected} onClick={(event) => event.stopPropagation()} onChange={(event) => onSelect(event.target.checked)} className={`absolute left-2 top-2 size-4 accent-[var(--workspace-accent)] ${selected ? "opacity-100" : "opacity-100 sm:opacity-0 sm:group-hover:opacity-100 sm:group-focus-within:opacity-100"}`} aria-label={`选择 ${asset.title}`} />
</div>
@@ -497,16 +497,11 @@ function AssetFilterGroup({ title, options, value, counts, onChange, className =
}
function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: LibraryAsset | null; onClose: () => void; onCopy: (asset: LibraryAsset) => void; onDownload: (asset: LibraryAsset) => void }) {
const cover = asset ? asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : "") : "";
return (
<Drawer title="素材详情" open={Boolean(asset)} size="large" onClose={onClose}>
{asset ? (
<div className="space-y-5">
{cover ? (
<Image src={cover} alt={asset.title} className="rounded-lg" />
) : (
<div className="rounded-lg border border-stone-200 bg-stone-50 p-5 text-sm leading-6 text-stone-600 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-300">{asset.kind === "text" ? asset.data.content : "暂无封面"}</div>
)}
<AssetMediaPreview asset={asset} alt={asset.title} className="max-h-[420px] w-full rounded-lg bg-black object-contain" fallback={<div className="rounded-lg border border-stone-200 bg-stone-50 p-5 text-sm leading-6 text-stone-600 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-300">{asset.kind === "text" ? asset.data.content : "暂无封面"}</div>} />
<div>
<Typography.Title level={4} className="!mb-2">
{asset.title}

6
web/src/pages/canvas/canvas-media-generation-executors.ts
View File

@@ -2,7 +2,7 @@ import { nanoid } from "nanoid";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { audioMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { fitNodeSize, nodeSizeFromRatio } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, nodeSizeFromRatio, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { nextCanvasVersionLabel } from "@/lib/canvas/canvas-layout";
import { buildAudioGenerationMetadata, buildVideoGenerationMetadata, generationReferenceUrls, runBackendCanvasGenerationTask } from "@/lib/canvas/canvas-project-generation";
import { storeGeneratedAudio } from "@/services/api/audio";
@@ -11,8 +11,6 @@ import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
import type { CanvasGenerationExecution } from "./canvas-generation-executor-types";
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
@@ -82,7 +80,7 @@ export async function executeVideoGeneration({
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: videoId, mode: "video", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, referenceVideos: generationContext.referenceVideos, referenceAudios: generationContext.referenceAudios, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions, resolvedCharacterVoices: generationContext.resolvedCharacterVoices, ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(videoId, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
setNodes((current) => current.map((node) => {
if (node.id !== videoId) return node;
const geometry = node.metadata?.locked ? {} : { width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 } };

2
web/src/pages/canvas/canvas-text-generation-executor.ts
View File

@@ -58,7 +58,7 @@ export async function executeTextGeneration({
const answers = await Promise.all(
textTargetIds.map((targetNodeId) => {
let localStreamed = "";
return runBackendCanvasGenerationTask({ projectId, nodeId: targetNodeId, mode: "text", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(targetNodeId, task) })
return runBackendCanvasGenerationTask({ projectId, nodeId: targetNodeId, mode: "text", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, referenceVideos: generationContext.referenceVideos, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(targetNodeId, task) })
.then((result) => {
localStreamed = result.text || "";
streamed = localStreamed;

2
web/src/pages/canvas/project.tsx
View File

@@ -202,7 +202,7 @@ function InfiniteCanvasPage() {
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
const [contextMenu, setContextMenu] = useState<ContextMenuState | null>(null);
const [isMiniMapOpen, setIsMiniMapOpen] = useState(false);
const [backgroundMode, setBackgroundMode] = useState<CanvasBackgroundMode>("lines");
const [backgroundMode, setBackgroundMode] = useState<CanvasBackgroundMode>("dots");
const [showImageInfo, setShowImageInfo] = useState(false);
const [mediaPerformanceMode, setMediaPerformanceMode] = useState<CanvasMediaPerformanceMode>(readCanvasMediaPerformanceMode);
const [projectLoaded, setProjectLoaded] = useState(false);

7
web/src/pages/canvas/shared.tsx
View File

@@ -15,6 +15,7 @@ import { WorkspaceState } from "@/components/layout/workspace-state";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { canvasThemes } from "@/lib/canvas-theme";
import { isFrameNode, isNodeHiddenByCollapsedFrame, resolveFrameConnection } from "@/lib/canvas/canvas-frame";
import { ensureMediaNodeMinimumSize } from "@/lib/canvas/canvas-node-size";
import { getPublicCanvasShare } from "@/services/api/canvas-share";
import { useThemeStore } from "@/stores/use-theme-store";
import { CanvasNodeType, type CanvasNodeData, type Position, type ViewportTransform } from "@/types/canvas";
@@ -33,7 +34,7 @@ export default function SharedCanvasPage() {
const [title, setTitle] = useState("共享画布");
const [nodes, setNodes] = useState<CanvasNodeData[]>([]);
const [connections, setConnections] = useState<Awaited<ReturnType<typeof getPublicCanvasShare>>["project"]["connections"]>([]);
const [backgroundMode, setBackgroundMode] = useState<"lines" | "dots" | "blank">("lines");
const [backgroundMode, setBackgroundMode] = useState<"lines" | "dots" | "blank">("dots");
const [viewport, setViewport] = useState<ViewportTransform>({ x: 0, y: 0, k: 1 });
const [selectedNodeId, setSelectedNodeId] = useState<string | null>(null);
const [infoNodeId, setInfoNodeId] = useState<string | null>(null);
@@ -77,9 +78,9 @@ export default function SharedCanvasPage() {
getPublicCanvasShare(token).then(({ project }) => {
if (!active) return;
setTitle(project.title || "共享画布");
setNodes(project.nodes || []);
setNodes((project.nodes || []).map(ensureMediaNodeMinimumSize));
setConnections(project.connections || []);
setBackgroundMode(project.backgroundMode || "lines");
setBackgroundMode(project.backgroundMode || "dots");
const initial = project.viewport || { x: 0, y: 0, k: 1 };
viewportRef.current = initial;
setViewport(initial);

8
web/src/pages/canvas/use-canvas-generation-retry.ts
View File

@@ -5,7 +5,7 @@ import { buildNodeGenerationContext, hydrateNodeGenerationContext } from "@/comp
import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { audioMetadata, imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { buildEmotionImageArtifacts, compositeEmotionImage, emotionGenerationSize } from "@/lib/canvas/canvas-emotion";
import {
buildAudioGenerationMetadata,
@@ -46,8 +46,6 @@ type UseCanvasGenerationRetryOptions = {
bindGenerationTask: (targetNodeId: string, task: GenerationTask) => void;
};
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
const NODE_STATUS_ERROR = "error" as const;
@@ -143,7 +141,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
try {
if (node.type === CanvasNodeType.Text) {
if (!context) return;
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "text", prompt, config: generationConfig, referenceImages: context.referenceImages, signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "text", prompt, config: generationConfig, referenceImages: context.referenceImages, referenceVideos: context.referenceVideos, signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.text) throw new Error("后端任务没有返回文本");
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, type: CanvasNodeType.Text, metadata: { ...item.metadata, content: result.text, richText: undefined, prompt, status: NODE_STATUS_SUCCESS, errorDetails: undefined, generationErrorCode: undefined, failedPromptFingerprint: undefined } } : item)));
return;
@@ -153,7 +151,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "video", prompt, config: generationConfig, referenceImages: videoContext?.referenceImages || [], referenceVideos: videoContext?.referenceVideos || [], referenceAudios: videoContext?.referenceAudios || [], signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], resolvedCharacterVoices: context?.resolvedCharacterVoices || [], ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, ...videoGenerationMetadata, references: videoContext ? generationReferenceUrls(videoContext) : item.metadata?.references } } : item)));
return;
}

10
web/src/pages/canvas/use-canvas-media-tools.ts
View File

@@ -19,7 +19,7 @@ import {
isGenerationCanceled,
runBackendCanvasGenerationTask,
} from "@/lib/canvas/canvas-project-generation";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { compositeEmotionImage, emotionGenerationSize } from "@/lib/canvas/canvas-emotion";
import { DEFAULT_PORTRAIT_TEXTURE_SETTINGS } from "@/lib/canvas/canvas-portrait-texture";
import { captureVideoLastFrame } from "@/lib/canvas/canvas-video-frame";
@@ -56,8 +56,6 @@ type UseCanvasMediaToolsOptions = {
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
const NODE_STATUS_ERROR = "error" as const;
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const IMAGE_PROMPT_REVERSE_PRESET = `请根据参考图片反推一段适合用于 AI 生图的提示词。
要求：
@@ -170,9 +168,9 @@ export function useCanvasMediaTools({
if (!node.metadata?.content) return;
const cropped = await cropDataUrl(node.metadata.content, crop);
const image = await uploadImage(cropped);
const width = Math.min(node.width, Math.max(220, image.width));
const size = fitNodeSize(image.width, image.height, node.width, node.height);
const childId = nanoid();
const child: CanvasNodeData = { id: childId, type: CanvasNodeType.Image, title: "Cropped Image", position: { x: node.position.x + node.width + 96, y: node.position.y }, width, height: width * (image.height / image.width), metadata: { ...imageMetadata(image), prompt: node.metadata?.prompt } };
const child: CanvasNodeData = { id: childId, type: CanvasNodeType.Image, title: "Cropped Image", position: { x: node.position.x + node.width + 96, y: node.position.y }, width: size.width, height: size.height, metadata: { ...imageMetadata(image), prompt: node.metadata?.prompt } };
setNodes((current) => [...current, child]);
setConnections((current) => [...current, { id: nanoid(), fromNodeId: node.id, toNodeId: childId }]);
setSelectedNodeIds(new Set([childId]));
@@ -254,7 +252,7 @@ export function useCanvasMediaTools({
const blob = await mergeVideos(videos.map((node) => ({ id: node.id, url: node.metadata?.content, storageKey: node.metadata?.storageKey })), setMergeVideoProgress);
setMergeVideoProgress({ phase: "encoding", progress: 98 });
const uploaded = await storeGeneratedVideo({ blob });
const size = fitNodeSize(uploaded.width || 1280, uploaded.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const size = fitNodeSize(uploaded.width || 1280, uploaded.height || 720, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
const left = Math.max(...videos.map((node) => node.position.x + node.width)) + 120;
const top = Math.min(...videos.map((node) => node.position.y));
const mergedNode = createCanvasNode(CanvasNodeType.Video, { x: left + size.width / 2, y: top + size.height / 2 }, {

2
web/src/pages/canvas/use-canvas-project-lifecycle.ts
View File

@@ -92,7 +92,7 @@ export function useCanvasProjectLifecycle({
connections: project.connections,
chatSessions: restoredSessions,
activeChatId: project.activeChatId || null,
backgroundMode: project.backgroundMode,
backgroundMode: project.backgroundMode || "dots",
showImageInfo: project.showImageInfo || false,
};
nodesRef.current = snapshot.nodes;

8
web/src/pages/canvas/use-canvas-upload.ts
View File

@@ -10,7 +10,7 @@ import { getDataUrlByteSize, readImageMeta } from "@/lib/image-utils";
import { audioMetadata, imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { createCanvasNode } from "@/lib/canvas/canvas-project-domain";
import { isAudioFile } from "@/lib/canvas/canvas-project-generation";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { uploadMediaFile } from "@/services/file-storage";
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
import { getProjectUnit } from "@/services/api/projects";
@@ -39,8 +39,6 @@ export type StartCanvasUploadStatus = (title: string, detail: string, total?: nu
fail: (detail?: string) => void;
};
const VIDEO_NODE_MAX_WIDTH = 420;
const VIDEO_NODE_MAX_HEIGHT = 420;
const NODE_STATUS_SUCCESS = "success" as const;
export function useCanvasUpload({
@@ -187,7 +185,7 @@ export function useCanvasUpload({
progress.update("上传到服务器并同步资源", 2);
const video = await uploadMediaFile(file, "video");
progress.update("更新画布节点", 3);
const size = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const size = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
const node = { id, type: CanvasNodeType.Video, title: file.name, position: { x: position.x - size.width / 2, y: position.y - size.height / 2 }, width: size.width, height: size.height, metadata: videoMetadata(video) } satisfies CanvasNodeData;
setNodes((current) => [...current, node]);
@@ -570,7 +568,7 @@ export function useCanvasUpload({
}
if (payload.kind === "video") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const size = fitNodeSize(payload.width || spec.width, payload.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
const size = fitNodeSize(payload.width || spec.width, payload.height || spec.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
return { id, type: CanvasNodeType.Video, title: payload.title, position: { x: center.x - size.width / 2, y: center.y - size.height / 2 }, width: size.width, height: size.height, metadata: { content: payload.url, storageKey: payload.storageKey, status: NODE_STATUS_SUCCESS, naturalWidth: payload.width, naturalHeight: payload.height, durationMs: payload.durationMs, bytes: payload.bytes, mimeType: payload.mimeType || "video/mp4", assetId: payload.assetId } } satisfies CanvasNodeData;
}

5
web/src/pages/projects/detail/assets.tsx
View File

@@ -4,6 +4,7 @@ import { App, Button, Dropdown, Form, Input, Modal, Popconfirm, Select, Tabs, ty
import { Box, Check, ChevronDown, FileText, Image as ImageIcon, Link2, Music2, Plus, RefreshCw, Sparkles, Trash2, UserRound, Video, VolumeX } from "lucide-react";
import { WorkspaceState } from "@/components/layout/workspace-state";
import { AssetMediaPreview } from "@/components/asset-media-preview";
import { WorkspaceSignalIcon } from "@/components/ui/aceternity/workspace-signal-icon";
import { canvasStylePresets } from "@/components/canvas/canvas-style-picker-modal";
import { resourceFileUrl, resourceIdFromStorageKey } from "@/services/api/resources";
@@ -225,8 +226,7 @@ function CharacterEditorModal({ open, editing, form, loading, onClose, onSave }:
type CharacterFormKey = (typeof characterFields)[number][0];
function MediaAssetCard({ asset, personalAsset, onCategoryChange, onVersion, onRemove, loading }: { asset: ProjectAsset; personalAsset?: Asset; onCategoryChange: (category: string) => void; onVersion: () => void; onRemove: () => void; loading: boolean }) {
const cover = personalAsset ? assetCover(personalAsset) : "";
return <article className="overflow-hidden rounded-lg border border-border/80 bg-background"><div className="relative aspect-[4/3] overflow-hidden bg-foreground/[.05]">{cover ? <img src={cover} alt={asset.title} loading="lazy" decoding="async" className="h-full w-full object-cover" /> : <div className="grid h-full place-items-center text-foreground/25"><MediaIcon kind={asset.mediaType} /></div>}<div className="absolute inset-x-2 top-2 flex items-center justify-between"><StatusPill status={asset.status} /><span className="rounded bg-black/50 px-1.5 py-0.5 text-[9px] text-white">{mediaLabel(asset.mediaType)}</span></div></div><div className="p-2.5"><div className="flex items-start justify-between gap-2"><div className="min-w-0"><h3 className="truncate text-xs font-medium">{asset.title}</h3><div className="mt-1 flex items-center gap-1.5 text-[10px] text-foreground/42"><Dropdown trigger={["click"]} menu={{ selectedKeys: [asset.category], items: Object.entries(categoryLabels).filter(([value]) => value !== "character").map(([value, label]) => ({ key: value, label })), onClick: ({ key }) => onCategoryChange(key) }}><button type="button" disabled={loading} className="inline-flex h-6 items-center gap-1 rounded px-1.5 text-[10px] text-foreground/50 hover:bg-foreground/[.055]"><span>{categoryLabel(asset.category)}</span><ChevronDown className="size-3" /></button></Dropdown><span>·</span><span>v{Math.max(1, asset.versionCount)}</span></div></div><Link2 className="mt-0.5 size-3.5 shrink-0 text-[var(--workspace-accent)]" /></div><div className="mt-2 flex items-center justify-between gap-2 border-t border-border/60 pt-1.5"><span className="text-[9px] text-foreground/38">{formatTime(asset.updatedAt)}</span><div className="flex items-center"><Button type="text" size="small" icon={<RefreshCw className="size-3.5" />} loading={loading} onClick={onVersion} /><Popconfirm title="移出项目资产？" okText="移出" cancelText="取消" onConfirm={onRemove}><Button type="text" danger size="small" icon={<Trash2 className="size-3.5" />} loading={loading} /></Popconfirm></div></div></div></article>;
return <article className="overflow-hidden rounded-lg border border-border/80 bg-background"><div className="relative aspect-[4/3] overflow-hidden bg-foreground/[.05]"><AssetMediaPreview asset={personalAsset} alt={asset.title} className="h-full w-full bg-black object-cover" fallback={<div className="grid h-full place-items-center text-foreground/25"><MediaIcon kind={asset.mediaType} /></div>} /><div className="absolute inset-x-2 top-2 flex items-center justify-between"><StatusPill status={asset.status} /><span className="rounded bg-black/50 px-1.5 py-0.5 text-[9px] text-white">{mediaLabel(asset.mediaType)}</span></div></div><div className="p-2.5"><div className="flex items-start justify-between gap-2"><div className="min-w-0"><h3 className="truncate text-xs font-medium">{asset.title}</h3><div className="mt-1 flex items-center gap-1.5 text-[10px] text-foreground/42"><Dropdown trigger={["click"]} menu={{ selectedKeys: [asset.category], items: Object.entries(categoryLabels).filter(([value]) => value !== "character").map(([value, label]) => ({ key: value, label })), onClick: ({ key }) => onCategoryChange(key) }}><button type="button" disabled={loading} className="inline-flex h-6 items-center gap-1 rounded px-1.5 text-[10px] text-foreground/50 hover:bg-foreground/[.055]"><span>{categoryLabel(asset.category)}</span><ChevronDown className="size-3" /></button></Dropdown><span>·</span><span>v{Math.max(1, asset.versionCount)}</span></div></div><Link2 className="mt-0.5 size-3.5 shrink-0 text-[var(--workspace-accent)]" /></div><div className="mt-2 flex items-center justify-between gap-2 border-t border-border/60 pt-1.5"><span className="text-[9px] text-foreground/38">{formatTime(asset.updatedAt)}</span><div className="flex items-center"><Button type="text" size="small" icon={<RefreshCw className="size-3.5" />} loading={loading} onClick={onVersion} /><Popconfirm title="移出项目资产？" okText="移出" cancelText="取消" onConfirm={onRemove}><Button type="text" danger size="small" icon={<Trash2 className="size-3.5" />} loading={loading} /></Popconfirm></div></div></div></article>;
}
function characterDefinition(values: CharacterForm) {
@@ -257,5 +257,4 @@ function syncPersonalCharacterProjection(asset: ProjectAsset) {
};
useAssetStore.getState().replaceAssets([projected, ...current.filter((item) => item.id !== asset.id)]);
}
function assetCover(asset: Asset) { if (asset.coverUrl) return asset.coverUrl; if (asset.kind === "image") return asset.data.dataUrl; if (asset.kind === "video") return asset.data.url; return ""; }
function MediaIcon({ kind }: { kind: string }) { if (kind === "image") return <ImageIcon className="size-10" />; if (kind === "video") return <Video className="size-10" />; if (kind === "audio") return <Music2 className="size-10" />; if (kind === "model") return <Box className="size-10" />; return <FileText className="size-10" />; }

4
web/src/services/user-data-sync.ts
View File

@@ -171,11 +171,11 @@ async function hydrateAssets(assets: Asset[]): Promise<Asset[]> {
}
if (asset.kind === "video" && asset.data.storageKey) {
const url = await resolveResourceOrMediaUrl(asset.data.storageKey, asset.data.url);
return { ...asset, coverUrl: shouldReplaceEphemeralUrl(asset.coverUrl) ? url : asset.coverUrl, data: { ...asset.data, url } };
return { ...asset, data: { ...asset.data, url } };
}
if (asset.kind === "audio" && asset.data.storageKey) {
const url = await resolveResourceOrMediaUrl(asset.data.storageKey, asset.data.url);
return { ...asset, coverUrl: shouldReplaceEphemeralUrl(asset.coverUrl) ? url : asset.coverUrl, data: { ...asset.data, url } };
return { ...asset, data: { ...asset.data, url } };
}
if (asset.kind === "model" && asset.data.storageKey) {
const url = await resolveResourceOrMediaUrl(asset.data.storageKey, asset.data.url);

4
web/src/stores/canvas/use-canvas-store.ts
View File

@@ -96,7 +96,7 @@ export const useCanvasStore = create<CanvasStore>()(
connections: [],
chatSessions: [],
activeChatId: null,
backgroundMode: "lines",
backgroundMode: "dots",
showImageInfo: false,
viewport: initialViewport,
directorScenes: [],
@@ -116,7 +116,7 @@ export const useCanvasStore = create<CanvasStore>()(
connections: source.connections || [],
chatSessions: source.chatSessions || [],
activeChatId: source.activeChatId || null,
backgroundMode: source.backgroundMode || "lines",
backgroundMode: source.backgroundMode || "dots",
showImageInfo: source.showImageInfo || false,
viewport: source.viewport || initialViewport,
directorScenes: source.directorScenes || [],

6
web/src/styles/globals.css
View File

@@ -1528,6 +1528,12 @@
box-shadow: none !important;
}
.dark .canvas-composer-model-picker.ant-select,
.dark .canvas-composer-model-picker.ant-select .ant-select-content,
.dark .canvas-model-picker-popup.ant-select-dropdown {
background: #000 !important;
}
.canvas-composer-model-picker.ant-select-focused .ant-select-content {
border-color: color-mix(in srgb, var(--foreground) 8%, transparent) !important;
box-shadow: none !important;