perf(canvas): 画布媒体 - 优化媒体加载、播放器与节点交互 (#357)

Merge origin/main at fb089b2 and retain canvas media performance, player layout, and interaction updates.

Co-authored-by: ddCat <2695092045@qq.com>
This commit is contained in:
wurendao007
2026-08-31 09:00:37 +08:00
committed by GitHub
parent 21facff923
commit 8ebceecbb6
58 changed files with 2662 additions and 207 deletions

View File

@@ -43,6 +43,7 @@ export function canvasAssetHandoffPayloads(assets: Asset[], assetIds: Iterable<s
width: asset.data.width,
height: asset.data.height,
durationMs: asset.data.durationMs,
hasAudio: asset.data.hasAudio,
bytes: asset.data.bytes,
mimeType: asset.data.mimeType,
assetId: asset.id,

View File

@@ -66,6 +66,15 @@ export function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
bytes: video.bytes,
mimeType: video.mimeType || "video/mp4",
durationMs: video.durationMs,
hasAudio: video.hasAudio,
videoPreview: video.preview ? {
content: video.preview.url,
storageKey: video.preview.storageKey,
width: video.preview.width,
height: video.preview.height,
bytes: video.preview.bytes,
mimeType: video.preview.mimeType,
} : undefined,
errorDetails: undefined,
generationErrorCode: undefined,
resourceReloadAvailable: undefined,

View File

@@ -0,0 +1,23 @@
import { buildLibTVVideoPreviewUrl } from "@/lib/canvas/libtv-import";
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
/**
* Returns an image source that is safe to mount for a passive video preview.
* The original video URL is deliberately never returned: callers must fall
* back to a video icon instead of creating a decoder outside the active node.
*/
export function canvasNodeVideoPreviewUrl(node: CanvasNodeData) {
if (node.type !== CanvasNodeType.Video) return "";
const content = node.metadata?.content || "";
const generatedPreview = node.metadata?.videoPreview?.content || "";
if (generatedPreview && generatedPreview !== content) return generatedPreview;
const explicitPreview = node.metadata?.previewContent || "";
if (explicitPreview && explicitPreview !== content) return explicitPreview;
return buildLibTVVideoPreviewUrl(content);
}
export function canvasVideoAssetPreviewUrl(videoUrl: string, coverUrl?: string) {
const explicitCover = coverUrl?.trim() || "";
if (explicitCover && explicitCover !== videoUrl) return explicitCover;
return buildLibTVVideoPreviewUrl(videoUrl);
}

View File

@@ -1,4 +1,5 @@
import { getDataUrlByteSize } from "@/lib/image-utils";
import { canvasNodeVideoPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import type { Asset, AssetCategory, NewAsset } from "@/stores/use-asset-store";
import { CanvasNodeType, type CanvasNodeData, type CanvasNodeTypeId } from "@/types/canvas";
@@ -24,7 +25,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
};
const base = {
title,
coverUrl: node.type === CanvasNodeType.Image ? content : "",
coverUrl: node.type === CanvasNodeType.Image ? content : node.type === CanvasNodeType.Video ? canvasNodeVideoPreviewUrl(node) : "",
tags: node.metadata?.assetTags || [],
category: canvasNodeAssetCategory(node),
status: "confirmed" as const,
@@ -60,6 +61,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
width: node.metadata?.naturalWidth || node.width,
height: node.metadata?.naturalHeight || node.height,
durationMs: node.metadata?.durationMs,
hasAudio: node.metadata?.hasAudio,
bytes: node.metadata?.bytes || 0,
mimeType: node.metadata?.mimeType || "video/mp4",
},

View File

@@ -0,0 +1,25 @@
export type CanvasNodeStackOrder = string[];
/**
* Keeps the most recently interacted node at the end of the paint order.
* This is session-local UI state and is intentionally not persisted with the
* canvas document.
*/
export function bringCanvasNodeToFront(order: CanvasNodeStackOrder, nodeId: string): CanvasNodeStackOrder {
if (!nodeId) return order;
if (order.at(-1) === nodeId) return order;
return [...order.filter((id) => id !== nodeId), nodeId];
}
/**
* Applies the interaction order while preserving the relative order of nodes
* that have not been interacted with yet.
*/
export function sortCanvasNodesByStackOrder<T extends { id: string }>(nodes: T[], order: CanvasNodeStackOrder): T[] {
if (nodes.length < 2 || order.length === 0) return nodes;
const rankById = new Map(order.map((id, index) => [id, index]));
return nodes
.map((node, index) => ({ node, index, rank: rankById.get(node.id) ?? -1 }))
.sort((left, right) => left.rank - right.rank || left.index - right.index)
.map(({ node }) => node);
}

View File

@@ -13,6 +13,8 @@ const HYDRATED_MEDIA_KEYS = new Set<keyof CanvasNodeMetadata>([
"bytes",
"mimeType",
"durationMs",
"hasAudio",
"videoPreview",
"drawingPreviewUrl",
]);

View File

@@ -25,3 +25,11 @@ export function shouldReduceCanvasMediaEffects(mode: CanvasMediaPerformanceMode,
const mediaCount = nodes.filter((node) => node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio).length;
return nodes.length >= 80 || mediaCount >= 32;
}
export function resolveActiveCanvasMediaNodeId(selectedNodeIds: ReadonlySet<string>, nodeById: ReadonlyMap<string, CanvasNodeData>) {
if (selectedNodeIds.size !== 1) return null;
const nodeId = selectedNodeIds.values().next().value;
if (!nodeId) return null;
const type = nodeById.get(nodeId)?.type;
return type === CanvasNodeType.Video || type === CanvasNodeType.Audio ? nodeId : null;
}

View File

@@ -354,7 +354,11 @@ export async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
return Promise.all(
nodes.map(async (node) => {
const content = node.metadata?.content;
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
const videoPreview = node.type === CanvasNodeType.Video && node.metadata?.videoPreview?.storageKey
? { ...node.metadata.videoPreview, content: await resolveImageUrl(node.metadata.videoPreview.storageKey, node.metadata.videoPreview.content, { cacheMiss: true }) }
: node.metadata?.videoPreview;
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content), videoPreview } };
if (videoPreview !== node.metadata?.videoPreview) return { ...node, metadata: { ...node.metadata, videoPreview } };
if (node.type === CanvasNodeType.Image && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content, { cacheMiss: true }) } };
if (node.type !== CanvasNodeType.Image || !content) return node;
if (!content.startsWith("data:image/")) return node;

View File

@@ -1,4 +1,5 @@
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
import { seedanceReferenceLabel } from "@/lib/seedance-video";
import type { Skill } from "@/services/api/skills";
@@ -14,7 +15,11 @@ export type CanvasResourceReference = {
label: string;
title: string;
previewUrl?: string;
/** 仅素材库视频在没有静态封面时使用的首帧回退源。 */
mediaUrl?: string;
storageKey?: string;
/** 视频首帧是独立的图片资源,不能用视频 storageKey 解析。 */
previewStorageKey?: string;
text?: string;
active: boolean;
sourceType?: CanvasNodeTypeId;
@@ -50,7 +55,7 @@ export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceRefe
return assets.flatMap((asset): CanvasResourceReference[] => {
if (asset.kind === "model") return [];
const kind: CanvasResourceKind = asset.kind === "entity" ? "character" : asset.kind;
const previewUrl = asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? asset.data.url : asset.coverUrl;
const previewUrl = asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? canvasVideoAssetPreviewUrl(asset.data.url, asset.coverUrl) : asset.coverUrl;
const text = asset.kind === "text" ? asset.data.content : undefined;
return [{
id: `asset:${asset.id}`,
@@ -60,6 +65,7 @@ export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceRefe
label: asset.title,
title: asset.title,
previewUrl,
mediaUrl: asset.kind === "video" && !previewUrl ? asset.data.url : undefined,
storageKey: "storageKey" in asset.data ? asset.data.storageKey : undefined,
text,
active: false,
@@ -148,8 +154,15 @@ function labelResourceNodes(nodes: CanvasNodeData[], active: boolean) {
kind,
label,
title: node.title || label,
previewUrl: node.metadata?.workflowKind === "character" ? node.metadata.characterCoverUrl : node.type === CanvasNodeType.Drawing ? node.metadata?.drawingPreviewUrl : node.metadata?.previewContent || node.metadata?.content,
previewUrl: node.metadata?.workflowKind === "character"
? node.metadata.characterCoverUrl
: node.type === CanvasNodeType.Drawing
? node.metadata?.drawingPreviewUrl
: node.type === CanvasNodeType.Video
? canvasNodeVideoPreviewUrl(node)
: node.metadata?.previewContent || node.metadata?.content,
storageKey: node.metadata?.storageKey,
previewStorageKey: node.type === CanvasNodeType.Video ? node.metadata?.videoPreview?.storageKey : undefined,
text: node.metadata?.workflowKind === "character" ? node.metadata.characterPrompt : node.type === CanvasNodeType.Text ? node.metadata?.content || node.metadata?.prompt : node.type === CanvasNodeType.Skill ? skillResourceText(node) : undefined,
active,
sourceType: node.type,

View File

@@ -1,6 +1,7 @@
const LIBTV_PROJECT_UUID_PATTERN = /^[a-f0-9]{32}$/i;
const LIBTV_RESOURCE_HOST = "libtv-res.liblib.art";
const LIBTV_PREVIEW_WIDTH = 960;
const LIBTV_VIDEO_PREVIEW_WIDTH = 400;
export function parseLibTVProjectUUID(value: string) {
const trimmed = value.trim();
@@ -34,3 +35,32 @@ export function buildLibTVImagePreviewUrl(rawUrl: string) {
return rawUrl;
}
}
export function buildLibTVVideoPreviewUrl(rawUrl: string) {
try {
const url = new URL(rawUrl);
if (url.protocol !== "https:" || url.hostname !== LIBTV_RESOURCE_HOST) return "";
if (!url.searchParams.has("x-oss-process")) {
url.searchParams.set("x-oss-process", `video/snapshot,t_0,f_jpg,w_${LIBTV_VIDEO_PREVIEW_WIDTH},m_fast,ar_auto`);
}
return url.toString();
} catch {
return "";
}
}
/**
* Restores the playable LibTV asset URL when a snapshot transform was
* persisted as the node's content by an older importer. The OSS transform
* belongs only on previewContent; VideoPlayer must receive the original MP4.
*/
export function buildLibTVVideoSourceUrl(rawUrl: string) {
try {
const url = new URL(rawUrl);
if (url.protocol !== "https:" || url.hostname !== LIBTV_RESOURCE_HOST) return rawUrl;
url.searchParams.delete("x-oss-process");
return url.toString();
} catch {
return rawUrl;
}
}

308
web/src/lib/video-poster.ts Normal file
View File

@@ -0,0 +1,308 @@
export type CapturedVideoPoster = {
width: number;
height: number;
durationMs?: number;
hasAudio?: boolean;
poster?: Blob;
};
type CaptureVideoPosterOptions = {
signal?: AbortSignal;
maxWidth?: number;
timeoutMs?: number;
};
const MAX_MP4_AUDIO_PROBE_BYTES = 32 * 1024 * 1024;
const MP4_AUDIO_PROBE_RANGE_BYTES = 1024 * 1024;
const videoAudioProbeCache = new Map<string, Promise<boolean | undefined>>();
let captureQueue: Promise<void> = Promise.resolve();
/** Serializes browser video decoding so batch uploads and poster hydration never fan out decoders. */
export function captureVideoPoster(source: string, options: CaptureVideoPosterOptions = {}) {
const capture = captureQueue.then(() => captureVideoPosterNow(source, options));
captureQueue = capture.then(() => undefined, () => undefined);
return capture;
}
function captureVideoPosterNow(source: string, options: CaptureVideoPosterOptions) {
return new Promise<CapturedVideoPoster>((resolve, reject) => {
const signal = options.signal;
if (signal?.aborted) {
reject(abortError());
return;
}
const video = document.createElement("video");
const fallback = { width: 1280, height: 720 };
let settled = false;
let metadata: Omit<CapturedVideoPoster, "poster"> = fallback;
const cleanup = () => {
window.clearTimeout(timeoutId);
signal?.removeEventListener("abort", handleAbort);
video.onloadedmetadata = null;
video.onloadeddata = null;
video.onerror = null;
video.pause();
video.removeAttribute("src");
video.load();
};
const finish = (result: CapturedVideoPoster) => {
if (settled) return;
settled = true;
cleanup();
resolve(result);
};
const fail = (error: Error) => {
if (settled) return;
settled = true;
cleanup();
reject(error);
};
const readMetadata = () => {
metadata = {
width: video.videoWidth || fallback.width,
height: video.videoHeight || fallback.height,
durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined,
hasAudio: detectVideoAudioTrack(video),
};
};
const handleAbort = () => fail(abortError());
const timeoutId = window.setTimeout(() => finish(metadata), options.timeoutMs ?? 15_000);
signal?.addEventListener("abort", handleAbort, { once: true });
video.preload = "auto";
video.muted = true;
video.playsInline = true;
if (isCrossOriginHttpUrl(source)) video.crossOrigin = "anonymous";
video.onloadedmetadata = readMetadata;
video.onerror = () => finish(metadata);
video.onloadeddata = () => {
readMetadata();
if (signal?.aborted) {
fail(abortError());
return;
}
const maxWidth = Math.max(1, options.maxWidth ?? 400);
const width = Math.max(1, Math.min(metadata.width, maxWidth));
const height = Math.max(1, Math.round(width * metadata.height / Math.max(1, metadata.width)));
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d");
if (!context) {
finish(metadata);
return;
}
try {
context.fillStyle = "#000";
context.fillRect(0, 0, width, height);
context.drawImage(video, 0, 0, width, height);
canvas.toBlob((poster) => finish({ ...metadata, poster: poster || undefined }), "image/jpeg", 0.82);
} catch {
// Cross-origin videos without CORS can still expose metadata but cannot be drawn safely.
finish(metadata);
}
};
video.src = source;
video.load();
});
}
function isCrossOriginHttpUrl(source: string) {
try {
const url = new URL(source, window.location.href);
return /^https?:$/.test(url.protocol) && url.origin !== window.location.origin;
} catch {
return false;
}
}
/** Audio track inspection is not standardized across browsers. */
export function detectVideoAudioTrack(video: HTMLMediaElement): boolean | undefined {
const tracks = (video as HTMLVideoElement & { audioTracks?: { length: number } }).audioTracks;
if (tracks && typeof tracks.length === "number") {
// A zero-length list before media data is loaded can mean that the
// browser has not populated its native track list yet.
if (tracks.length > 0) return true;
// Do not treat an empty list as proof of silence. Chromium-based
// browsers can expose AudioTrackList while leaving it empty even for
// a video that has an audio stream; continue with the other probes.
}
const mozHasAudio = (video as HTMLVideoElement & { mozHasAudio?: boolean }).mozHasAudio;
if (typeof mozHasAudio === "boolean") return mozHasAudio;
// Chromium and WebKit expose this legacy counter on media elements. It is
// meaningful once media data is available; before that point zero only
// means that decoding has not started yet.
const webkitAudioDecodedByteCount = (video as HTMLVideoElement & { webkitAudioDecodedByteCount?: number }).webkitAudioDecodedByteCount;
if (typeof webkitAudioDecodedByteCount === "number" && video.readyState >= 3 && webkitAudioDecodedByteCount > 0) return true;
const captureStream = (video as HTMLVideoElement & { captureStream?: () => MediaStream }).captureStream;
if (typeof captureStream === "function") {
try {
const audioTracks = captureStream.call(video).getAudioTracks();
if (audioTracks.length > 0) return true;
} catch {
// Some browsers expose captureStream but reject it for remote media.
}
}
return undefined;
}
/**
* Inspects the ISO-BMFF track headers without decoding the media. This is a
* bounded fallback for local MP4/MOV uploads where browsers expose no audio
* track API. Remote sources are intentionally not fetched by this helper.
*/
export async function detectVideoAudioTrackFromBlob(blob: Blob): Promise<boolean | undefined> {
if (!isIsoBmffVideoType(blob.type) || blob.size <= 0 || blob.size > MAX_MP4_AUDIO_PROBE_BYTES) return undefined;
try {
const bytes = new Uint8Array(await blob.arrayBuffer());
return parseIsoBmffAudioTrack(bytes);
} catch {
return undefined;
}
}
/**
* Probes a remote MP4/MOV without downloading the whole file when possible.
* The browser media element does not expose a portable audio-track API, while
* LibTV assets are CORS-enabled and expose ISO-BMFF headers through Range.
*/
export function detectVideoAudioTrackFromUrl(source: string, signal?: AbortSignal) {
const normalized = source.trim();
if (!normalized || videoAudioProbeCache.has(normalized)) return videoAudioProbeCache.get(normalized) || Promise.resolve(undefined);
const probe = probeVideoAudioUrl(normalized, signal)
.then((result) => {
if (result === undefined) videoAudioProbeCache.delete(normalized);
return result;
})
.catch(() => {
videoAudioProbeCache.delete(normalized);
return undefined;
});
videoAudioProbeCache.set(normalized, probe);
return probe;
}
async function probeVideoAudioUrl(source: string, signal?: AbortSignal): Promise<boolean | undefined> {
if (!isLikelyIsoBmffVideoUrl(source)) return undefined;
const first = await fetchVideoRange(source, 0, MP4_AUDIO_PROBE_RANGE_BYTES - 1, signal);
if (!first) return undefined;
const firstResult = parseIsoBmffAudioTrack(first.bytes);
if (firstResult !== undefined) return firstResult;
const totalBytes = first.totalBytes;
if (totalBytes && totalBytes > MP4_AUDIO_PROBE_RANGE_BYTES) {
const tailStart = Math.max(0, totalBytes - MP4_AUDIO_PROBE_RANGE_BYTES);
const tail = await fetchVideoRange(source, tailStart, totalBytes - 1, signal);
const tailResult = tail ? parseIsoBmffAudioTrack(tail.bytes) : undefined;
if (tailResult !== undefined) return tailResult;
}
// Some origins ignore Range and return the complete response. Only accept
// a bounded full response so a missing Content-Range cannot trigger a
// large duplicate download for an active video.
if (first.status === 200 && first.bytes.byteLength <= MAX_MP4_AUDIO_PROBE_BYTES) return parseIsoBmffAudioTrack(first.bytes);
return undefined;
}
async function fetchVideoRange(source: string, start: number, end: number, signal?: AbortSignal) {
const response = await fetch(source, { headers: { Range: `bytes=${start}-${end}` }, signal, cache: "force-cache" });
if (!response.ok) return undefined;
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && (response.status === 200 && declaredLength > MAX_MP4_AUDIO_PROBE_BYTES || response.status === 206 && declaredLength > MP4_AUDIO_PROBE_RANGE_BYTES)) return undefined;
const bytes = new Uint8Array(await response.arrayBuffer());
const contentRange = response.headers.get("content-range") || "";
const rangeMatch = /bytes\s+\d+-\d+\/(\d+)/i.exec(contentRange);
const totalBytes = rangeMatch ? Number(rangeMatch[1]) : Number.isFinite(declaredLength) ? declaredLength : undefined;
return { bytes, status: response.status, totalBytes };
}
function isIsoBmffVideoType(mimeType: string) {
const normalized = mimeType.toLowerCase().split(";", 1)[0].trim();
return normalized === "video/mp4" || normalized === "video/quicktime" || normalized === "application/mp4";
}
function isLikelyIsoBmffVideoUrl(source: string) {
try {
const baseUrl = typeof window === "undefined" ? "http://localhost" : window.location?.href || "http://localhost";
const url = new URL(source, baseUrl);
return /\.(mp4|m4v|mov)(?:$|[?#])/i.test(url.pathname + url.search);
} catch {
return false;
}
}
function parseIsoBmffAudioTrack(bytes: Uint8Array): boolean | undefined {
const direct = parseIsoBmffAudioTrackFromOffset(bytes, 0);
if (direct !== undefined) return direct;
// A suffix Range can begin in the middle of an earlier box. Locate a
// complete `moov` header inside that suffix before giving up.
for (let offset = 0; offset + 8 <= bytes.byteLength; offset += 1) {
if (readFourcc(bytes, offset + 4) !== "moov") continue;
const result = parseIsoBmffAudioTrackFromOffset(bytes, offset);
if (result !== undefined) return result;
}
return undefined;
}
function parseIsoBmffAudioTrackFromOffset(bytes: Uint8Array, startOffset: number): boolean | undefined {
let foundMoov = false;
let foundAudio = false;
const visit = (start: number, end: number, insideMoov: boolean) => {
let offset = start;
while (offset + 8 <= end) {
const size32 = readUint32(bytes, offset);
const type = readFourcc(bytes, offset + 4);
let headerSize = 8;
let boxEnd: number;
if (size32 === 1) {
if (offset + 16 > end) return;
const high = readUint32(bytes, offset + 8);
const low = readUint32(bytes, offset + 12);
const size = high * 0x100000000 + low;
headerSize = 16;
boxEnd = offset + size;
} else if (size32 === 0) {
boxEnd = end;
} else {
boxEnd = offset + size32;
}
// Empty leaf boxes (size == header size) are valid ISO-BMFF boxes
// such as `free`; only reject truncated or backwards boxes.
if (!type || boxEnd < offset + headerSize || boxEnd > end) return;
const isMoov = type === "moov";
if (isMoov) foundMoov = true;
if (insideMoov && type === "hdlr" && offset + 16 <= boxEnd && readFourcc(bytes, offset + 16) === "soun") foundAudio = true;
if (isMoov || insideMoov && (type === "trak" || type === "mdia")) {
visit(offset + headerSize, boxEnd, insideMoov || isMoov);
}
if (foundAudio) return;
offset = boxEnd;
}
};
visit(startOffset, bytes.byteLength, false);
return foundMoov ? foundAudio : undefined;
}
function readUint32(bytes: Uint8Array, offset: number) {
return ((bytes[offset] || 0) * 0x1000000) + ((bytes[offset + 1] || 0) << 16) + ((bytes[offset + 2] || 0) << 8) + (bytes[offset + 3] || 0);
}
function readFourcc(bytes: Uint8Array, offset: number) {
if (offset + 4 > bytes.byteLength) return "";
return String.fromCharCode(bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]);
}
function abortError() {
return new DOMException("Video poster capture aborted", "AbortError");
}