tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(canvas): 画布媒体 - 优化媒体加载、播放器与节点交互 (#357)

Browse Source 
Merge origin/main at fb089b2 and retain canvas media performance, player layout, and interaction updates.

Co-authored-by: ddCat <2695092045@qq.com>
This commit is contained in:
wurendao007
2026-08-31 09:00:37 +08:00
committed by GitHub
parent 21facff923
commit 8ebceecbb6
58 changed files with 2662 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
web/src/lib/canvas/canvas-asset-handoff.ts
View File

@@ -43,6 +43,7 @@ export function canvasAssetHandoffPayloads(assets: Asset[], assetIds: Iterable<s
width: asset.data.width,
height: asset.data.height,
durationMs: asset.data.durationMs,
hasAudio: asset.data.hasAudio,
bytes: asset.data.bytes,
mimeType: asset.data.mimeType,
assetId: asset.id,

9
web/src/lib/canvas/canvas-generation-task-sync.ts
View File

@@ -66,6 +66,15 @@ export function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
bytes: video.bytes,
mimeType: video.mimeType || "video/mp4",
durationMs: video.durationMs,
hasAudio: video.hasAudio,
videoPreview: video.preview ? {
content: video.preview.url,
storageKey: video.preview.storageKey,
width: video.preview.width,
height: video.preview.height,
bytes: video.preview.bytes,
mimeType: video.preview.mimeType,
} : undefined,
errorDetails: undefined,
generationErrorCode: undefined,
resourceReloadAvailable: undefined,

23
web/src/lib/canvas/canvas-media-preview.ts Normal file
View File

@@ -0,0 +1,23 @@
import { buildLibTVVideoPreviewUrl } from "@/lib/canvas/libtv-import";
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
/**
* Returns an image source that is safe to mount for a passive video preview.
* The original video URL is deliberately never returned: callers must fall
* back to a video icon instead of creating a decoder outside the active node.
*/
export function canvasNodeVideoPreviewUrl(node: CanvasNodeData) {
if (node.type !== CanvasNodeType.Video) return "";
const content = node.metadata?.content || "";
const generatedPreview = node.metadata?.videoPreview?.content || "";
if (generatedPreview && generatedPreview !== content) return generatedPreview;
const explicitPreview = node.metadata?.previewContent || "";
if (explicitPreview && explicitPreview !== content) return explicitPreview;
return buildLibTVVideoPreviewUrl(content);
}
export function canvasVideoAssetPreviewUrl(videoUrl: string, coverUrl?: string) {
const explicitCover = coverUrl?.trim() || "";
if (explicitCover && explicitCover !== videoUrl) return explicitCover;
return buildLibTVVideoPreviewUrl(videoUrl);
}

4
web/src/lib/canvas/canvas-node-asset.ts
View File

@@ -1,4 +1,5 @@
import { getDataUrlByteSize } from "@/lib/image-utils";
import { canvasNodeVideoPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import type { Asset, AssetCategory, NewAsset } from "@/stores/use-asset-store";
import { CanvasNodeType, type CanvasNodeData, type CanvasNodeTypeId } from "@/types/canvas";
@@ -24,7 +25,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
};
const base = {
title,
coverUrl: node.type === CanvasNodeType.Image ? content : "",
coverUrl: node.type === CanvasNodeType.Image ? content : node.type === CanvasNodeType.Video ? canvasNodeVideoPreviewUrl(node) : "",
tags: node.metadata?.assetTags || [],
category: canvasNodeAssetCategory(node),
status: "confirmed" as const,
@@ -60,6 +61,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
width: node.metadata?.naturalWidth || node.width,
height: node.metadata?.naturalHeight || node.height,
durationMs: node.metadata?.durationMs,
hasAudio: node.metadata?.hasAudio,
bytes: node.metadata?.bytes || 0,
mimeType: node.metadata?.mimeType || "video/mp4",
},

25
web/src/lib/canvas/canvas-node-stack-order.ts Normal file
View File

@@ -0,0 +1,25 @@
export type CanvasNodeStackOrder = string[];
/**
* Keeps the most recently interacted node at the end of the paint order.
* This is session-local UI state and is intentionally not persisted with the
* canvas document.
*/
export function bringCanvasNodeToFront(order: CanvasNodeStackOrder, nodeId: string): CanvasNodeStackOrder {
if (!nodeId) return order;
if (order.at(-1) === nodeId) return order;
return [...order.filter((id) => id !== nodeId), nodeId];
}
/**
* Applies the interaction order while preserving the relative order of nodes
* that have not been interacted with yet.
*/
export function sortCanvasNodesByStackOrder<T extends { id: string }>(nodes: T[], order: CanvasNodeStackOrder): T[] {
if (nodes.length < 2 || order.length === 0) return nodes;
const rankById = new Map(order.map((id, index) => [id, index]));
return nodes
.map((node, index) => ({ node, index, rank: rankById.get(node.id) ?? -1 }))
.sort((left, right) => left.rank - right.rank || left.index - right.index)
.map(({ node }) => node);
}

2
web/src/lib/canvas/canvas-node-timestamps.ts
View File

@@ -13,6 +13,8 @@ const HYDRATED_MEDIA_KEYS = new Set<keyof CanvasNodeMetadata>([
"bytes",
"mimeType",
"durationMs",
"hasAudio",
"videoPreview",
"drawingPreviewUrl",
]);

8
web/src/lib/canvas/canvas-performance-mode.ts
View File

@@ -25,3 +25,11 @@ export function shouldReduceCanvasMediaEffects(mode: CanvasMediaPerformanceMode,
const mediaCount = nodes.filter((node) => node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio).length;
return nodes.length >= 80 || mediaCount >= 32;
}
export function resolveActiveCanvasMediaNodeId(selectedNodeIds: ReadonlySet<string>, nodeById: ReadonlyMap<string, CanvasNodeData>) {
if (selectedNodeIds.size !== 1) return null;
const nodeId = selectedNodeIds.values().next().value;
if (!nodeId) return null;
const type = nodeById.get(nodeId)?.type;
return type === CanvasNodeType.Video || type === CanvasNodeType.Audio ? nodeId : null;
}

6
web/src/lib/canvas/canvas-project-generation.ts
View File

@@ -354,7 +354,11 @@ export async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
return Promise.all(
nodes.map(async (node) => {
const content = node.metadata?.content;
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
const videoPreview = node.type === CanvasNodeType.Video && node.metadata?.videoPreview?.storageKey
? { ...node.metadata.videoPreview, content: await resolveImageUrl(node.metadata.videoPreview.storageKey, node.metadata.videoPreview.content, { cacheMiss: true }) }
: node.metadata?.videoPreview;
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content), videoPreview } };
if (videoPreview !== node.metadata?.videoPreview) return { ...node, metadata: { ...node.metadata, videoPreview } };
if (node.type === CanvasNodeType.Image && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content, { cacheMiss: true }) } };
if (node.type !== CanvasNodeType.Image || !content) return node;
if (!content.startsWith("data:image/")) return node;

17
web/src/lib/canvas/canvas-resource-references.ts
View File

@@ -1,4 +1,5 @@
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
import { seedanceReferenceLabel } from "@/lib/seedance-video";
import type { Skill } from "@/services/api/skills";
@@ -14,7 +15,11 @@ export type CanvasResourceReference = {
label: string;
title: string;
previewUrl?: string;
/** 仅素材库视频在没有静态封面时使用的首帧回退源。 */
mediaUrl?: string;
storageKey?: string;
/** 视频首帧是独立的图片资源，不能用视频 storageKey 解析。 */
previewStorageKey?: string;
text?: string;
active: boolean;
sourceType?: CanvasNodeTypeId;
@@ -50,7 +55,7 @@ export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceRefe
return assets.flatMap((asset): CanvasResourceReference[] => {
if (asset.kind === "model") return [];
const kind: CanvasResourceKind = asset.kind === "entity" ? "character" : asset.kind;
const previewUrl = asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? asset.data.url : asset.coverUrl;
const previewUrl = asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? canvasVideoAssetPreviewUrl(asset.data.url, asset.coverUrl) : asset.coverUrl;
const text = asset.kind === "text" ? asset.data.content : undefined;
return [{
id: `asset:${asset.id}`,
@@ -60,6 +65,7 @@ export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceRefe
label: asset.title,
title: asset.title,
previewUrl,
mediaUrl: asset.kind === "video" && !previewUrl ? asset.data.url : undefined,
storageKey: "storageKey" in asset.data ? asset.data.storageKey : undefined,
text,
active: false,
@@ -148,8 +154,15 @@ function labelResourceNodes(nodes: CanvasNodeData[], active: boolean) {
kind,
label,
title: node.title || label,
previewUrl: node.metadata?.workflowKind === "character" ? node.metadata.characterCoverUrl : node.type === CanvasNodeType.Drawing ? node.metadata?.drawingPreviewUrl : node.metadata?.previewContent || node.metadata?.content,
previewUrl: node.metadata?.workflowKind === "character"
? node.metadata.characterCoverUrl
: node.type === CanvasNodeType.Drawing
? node.metadata?.drawingPreviewUrl
: node.type === CanvasNodeType.Video
? canvasNodeVideoPreviewUrl(node)
: node.metadata?.previewContent || node.metadata?.content,
storageKey: node.metadata?.storageKey,
previewStorageKey: node.type === CanvasNodeType.Video ? node.metadata?.videoPreview?.storageKey : undefined,
text: node.metadata?.workflowKind === "character" ? node.metadata.characterPrompt : node.type === CanvasNodeType.Text ? node.metadata?.content || node.metadata?.prompt : node.type === CanvasNodeType.Skill ? skillResourceText(node) : undefined,
active,
sourceType: node.type,

30
web/src/lib/canvas/libtv-import.ts
View File

@@ -1,6 +1,7 @@
const LIBTV_PROJECT_UUID_PATTERN = /^[a-f0-9]{32}$/i;
const LIBTV_RESOURCE_HOST = "libtv-res.liblib.art";
const LIBTV_PREVIEW_WIDTH = 960;
const LIBTV_VIDEO_PREVIEW_WIDTH = 400;
export function parseLibTVProjectUUID(value: string) {
const trimmed = value.trim();
@@ -34,3 +35,32 @@ export function buildLibTVImagePreviewUrl(rawUrl: string) {
return rawUrl;
}
}
export function buildLibTVVideoPreviewUrl(rawUrl: string) {
try {
const url = new URL(rawUrl);
if (url.protocol !== "https:" || url.hostname !== LIBTV_RESOURCE_HOST) return "";
if (!url.searchParams.has("x-oss-process")) {
url.searchParams.set("x-oss-process", `video/snapshot,t_0,f_jpg,w_${LIBTV_VIDEO_PREVIEW_WIDTH},m_fast,ar_auto`);
}
return url.toString();
} catch {
return "";
}
}
/**
* Restores the playable LibTV asset URL when a snapshot transform was
* persisted as the node's content by an older importer. The OSS transform
* belongs only on previewContent; VideoPlayer must receive the original MP4.
*/
export function buildLibTVVideoSourceUrl(rawUrl: string) {
try {
const url = new URL(rawUrl);
if (url.protocol !== "https:" || url.hostname !== LIBTV_RESOURCE_HOST) return rawUrl;
url.searchParams.delete("x-oss-process");
return url.toString();
} catch {
return rawUrl;
}
}