perf(canvas): 画布媒体 - 优化媒体加载、播放器与节点交互 (#357)

Merge origin/main at fb089b2 and retain canvas media performance, player layout, and interaction updates.

Co-authored-by: ddCat <2695092045@qq.com>
This commit is contained in:
wurendao007
2026-08-31 09:00:37 +08:00
committed by GitHub
parent 21facff923
commit 8ebceecbb6
58 changed files with 2662 additions and 207 deletions

View File

@@ -0,0 +1,281 @@
import { cacheResourceObjectUrl } from "@/services/resource-blob-cache";
import { resourceIdFromStorageKey } from "@/services/api/resources";
export type CanvasAudioSource = {
nodeId: string;
content: string;
storageKey?: string;
mimeType?: string;
durationMs?: number;
};
export type CanvasAudioPlaybackPhase = "idle" | "loading" | "playing" | "paused" | "ended" | "error";
export type CanvasAudioPlaybackSnapshot = {
nodeId: string | null;
phase: CanvasAudioPlaybackPhase;
currentTimeMs: number;
durationMs: number;
error?: string;
};
type Listener = () => void;
const EMPTY_SNAPSHOT = (nodeId: string): CanvasAudioPlaybackSnapshot => ({
nodeId,
phase: "idle",
currentTimeMs: 0,
durationMs: 0,
});
class CanvasAudioPlaybackController {
private audio: HTMLAudioElement | null = null;
private activeSource: CanvasAudioSource | null = null;
private generation = 0;
private pendingSeekMs: number | null = null;
private autoplayToken: number | null = null;
private readonly snapshots = new Map<string, CanvasAudioPlaybackSnapshot>();
private readonly listeners = new Map<string, Set<Listener>>();
getSnapshot(nodeId: string) {
const existing = this.snapshots.get(nodeId);
if (existing) return existing;
const snapshot = EMPTY_SNAPSHOT(nodeId);
this.snapshots.set(nodeId, snapshot);
return snapshot;
}
subscribeNode(nodeId: string, listener: Listener) {
const nodeListeners = this.listeners.get(nodeId) || new Set<Listener>();
nodeListeners.add(listener);
this.listeners.set(nodeId, nodeListeners);
return () => {
nodeListeners.delete(listener);
if (!nodeListeners.size) this.listeners.delete(nodeId);
};
}
async toggle(source: CanvasAudioSource) {
const active = this.activeSource;
const snapshot = this.getSnapshot(source.nodeId);
if (active && active.nodeId === source.nodeId && sameSource(active, source)) {
if (snapshot.phase === "playing") {
this.audio?.pause();
return;
}
if (snapshot.phase === "loading") {
if (this.autoplayToken === null) {
this.autoplayToken = this.generation;
await this.playActive(source.nodeId);
} else {
this.stop(source.nodeId);
}
return;
}
if (snapshot.phase === "paused" && this.audio?.src) {
await this.resume(source.nodeId);
return;
}
}
if (this.activeSource) this.stop(this.activeSource.nodeId);
return this.begin(source);
}
pause(nodeId?: string) {
if (!this.activeSource || (nodeId && this.activeSource.nodeId !== nodeId)) return;
this.audio?.pause();
}
stop(nodeId?: string) {
if (!this.activeSource || (nodeId && this.activeSource.nodeId !== nodeId)) return;
const previous = this.activeSource;
this.generation += 1;
this.activeSource = null;
this.pendingSeekMs = null;
this.autoplayToken = null;
if (this.audio) {
this.audio.pause();
this.audio.removeAttribute("src");
this.audio.load();
}
const previousSnapshot = this.getSnapshot(previous.nodeId);
this.update(previous.nodeId, {
phase: "idle",
currentTimeMs: 0,
durationMs: previousSnapshot.durationMs || previous.durationMs || 0,
error: undefined,
});
}
async seek(nodeId: string, timeMs: number, source?: CanvasAudioSource) {
if (!this.activeSource || this.activeSource.nodeId !== nodeId) {
if (source) {
// A seek on another node changes the single shared audio instance.
// Reset the previous node first so its control does not remain in
// the playing state after ownership moves to the target node.
if (this.activeSource) this.stop(this.activeSource.nodeId);
await this.begin(source, false, timeMs);
}
return;
}
const durationMs = Number.isFinite(this.audio?.duration) && (this.audio?.duration || 0) > 0 ? (this.audio?.duration || 0) * 1000 : this.getSnapshot(nodeId).durationMs;
const nextTimeMs = Math.max(0, Math.min(Math.max(0, durationMs), Number.isFinite(timeMs) ? timeMs : 0));
if (!this.audio?.src || this.getSnapshot(nodeId).phase === "loading") {
this.pendingSeekMs = nextTimeMs;
this.update(nodeId, { currentTimeMs: nextTimeMs });
return;
}
try {
this.audio.currentTime = nextTimeMs / 1000;
} catch {
return;
}
const wasEnded = this.getSnapshot(nodeId).phase === "ended";
this.update(nodeId, wasEnded ? { currentTimeMs: nextTimeMs, phase: "paused" } : { currentTimeMs: nextTimeMs });
}
private async begin(source: CanvasAudioSource, autoplay = true, seekTimeMs?: number) {
const token = ++this.generation;
this.activeSource = source;
this.pendingSeekMs = seekTimeMs == null ? null : Math.max(0, seekTimeMs);
this.autoplayToken = autoplay ? token : null;
this.update(source.nodeId, {
phase: "loading",
currentTimeMs: this.pendingSeekMs || 0,
durationMs: source.durationMs || 0,
error: undefined,
});
const url = await resolveAudioSource(source);
if (!this.isCurrent(source, token)) return;
if (!url) {
this.update(source.nodeId, { phase: "error", error: "音频资源不可用" });
return;
}
const audio = this.ensureAudio();
audio.preload = "metadata";
audio.src = url;
audio.load();
if (!autoplay && this.autoplayToken !== token) return;
return this.playActive(source.nodeId, token);
}
private async playActive(nodeId: string, token = this.generation) {
if (!this.audio || !this.activeSource || this.activeSource.nodeId !== nodeId || token !== this.generation) return;
try {
await this.audio.play();
if (this.activeSource?.nodeId === nodeId && token === this.generation) this.update(nodeId, { phase: "playing", error: undefined });
} catch (error) {
if (this.activeSource?.nodeId === nodeId && token === this.generation) this.update(nodeId, { phase: "error", error: mediaErrorMessage(error) });
}
}
private async resume(nodeId: string) {
if (!this.audio || !this.activeSource || this.activeSource.nodeId !== nodeId) return;
try {
await this.audio.play();
if (this.activeSource?.nodeId === nodeId) this.update(nodeId, { phase: "playing", error: undefined });
} catch (error) {
this.update(nodeId, { phase: "error", error: mediaErrorMessage(error) });
}
}
private ensureAudio() {
if (this.audio) return this.audio;
if (typeof Audio === "undefined") throw new Error("当前环境不支持音频播放");
const audio = new Audio();
audio.preload = "metadata";
audio.addEventListener("loadedmetadata", () => {
const active = this.activeSource;
if (!active) return;
const durationMs = Number.isFinite(audio.duration) && audio.duration > 0 ? Math.round(audio.duration * 1000) : active.durationMs || 0;
const pendingSeekMs = this.pendingSeekMs;
this.pendingSeekMs = null;
if (pendingSeekMs != null) {
const nextTimeMs = Math.max(0, Math.min(Math.max(0, durationMs), pendingSeekMs));
try {
audio.currentTime = nextTimeMs / 1000;
} catch {
// Some browsers defer currentTime assignment until metadata is ready.
}
this.update(active.nodeId, this.autoplayToken === null ? { currentTimeMs: nextTimeMs, durationMs, phase: "paused" } : { currentTimeMs: nextTimeMs, durationMs });
return;
}
this.update(active.nodeId, { durationMs });
});
audio.addEventListener("durationchange", () => {
const active = this.activeSource;
if (!active || !Number.isFinite(audio.duration) || audio.duration <= 0) return;
this.update(active.nodeId, { durationMs: Math.round(audio.duration * 1000) });
});
audio.addEventListener("timeupdate", () => {
const active = this.activeSource;
if (!active) return;
this.update(active.nodeId, { currentTimeMs: Math.round(Math.max(0, audio.currentTime) * 1000) });
});
audio.addEventListener("playing", () => {
const active = this.activeSource;
if (active) this.update(active.nodeId, { phase: "playing", error: undefined });
});
audio.addEventListener("pause", () => {
const active = this.activeSource;
if (active && this.getSnapshot(active.nodeId).phase !== "ended") this.update(active.nodeId, { phase: "paused" });
});
audio.addEventListener("ended", () => {
const active = this.activeSource;
if (!active) return;
const durationMs = Number.isFinite(audio.duration) && audio.duration > 0 ? Math.round(audio.duration * 1000) : this.getSnapshot(active.nodeId).durationMs;
this.update(active.nodeId, { phase: "ended", currentTimeMs: durationMs, durationMs });
});
audio.addEventListener("error", () => {
const active = this.activeSource;
if (active) this.update(active.nodeId, { phase: "error", error: "音频加载失败" });
});
this.audio = audio;
return audio;
}
private isCurrent(source: CanvasAudioSource, token: number) {
return token === this.generation && this.activeSource?.nodeId === source.nodeId && sameSource(this.activeSource, source);
}
private update(nodeId: string, patch: Partial<CanvasAudioPlaybackSnapshot>) {
const previous = this.getSnapshot(nodeId);
const next: CanvasAudioPlaybackSnapshot = { ...previous, ...patch, nodeId };
if (snapshotEqual(previous, next)) return;
this.snapshots.set(nodeId, next);
this.listeners.get(nodeId)?.forEach((listener) => listener());
}
}
function sameSource(left: CanvasAudioSource, right: CanvasAudioSource) {
return left.content === right.content && left.storageKey === right.storageKey && left.mimeType === right.mimeType;
}
async function resolveAudioSource(source: CanvasAudioSource) {
if (source.storageKey && resourceIdFromStorageKey(source.storageKey)) {
const cached = await cacheResourceObjectUrl(source.storageKey).catch(() => "");
return cached || source.content;
}
return source.content;
}
function snapshotEqual(left: CanvasAudioPlaybackSnapshot, right: CanvasAudioPlaybackSnapshot) {
return left.phase === right.phase && left.currentTimeMs === right.currentTimeMs && left.durationMs === right.durationMs && left.error === right.error;
}
function mediaErrorMessage(error: unknown) {
if (error instanceof Error && error.message.trim()) return error.message;
return "音频播放失败";
}
export const canvasAudioPlayback = new CanvasAudioPlaybackController();
export const getCanvasAudioPlaybackSnapshot = (nodeId: string) => canvasAudioPlayback.getSnapshot(nodeId);
export const subscribeCanvasAudioNode = (nodeId: string, listener: Listener) => canvasAudioPlayback.subscribeNode(nodeId, listener);
export const toggleCanvasAudio = (source: CanvasAudioSource) => canvasAudioPlayback.toggle(source);
export const pauseCanvasAudio = (nodeId?: string) => canvasAudioPlayback.pause(nodeId);
export const stopCanvasAudio = (nodeId?: string) => canvasAudioPlayback.stop(nodeId);
export const seekCanvasAudio = (nodeId: string, timeMs: number, source?: CanvasAudioSource) => canvasAudioPlayback.seek(nodeId, timeMs, source);

View File

@@ -0,0 +1,85 @@
import { captureVideoPoster } from "@/lib/video-poster";
import { resolveMediaUrl } from "@/services/file-storage";
import { uploadImage } from "@/services/image-storage";
import type { CanvasNodeData, CanvasNodeMetadata } from "@/types/canvas";
type VideoPreview = NonNullable<CanvasNodeMetadata["videoPreview"]>;
const previewRequests = new Map<string, Promise<VideoPreview | null>>();
export function hydrateCanvasVideoPreview(node: CanvasNodeData, signal?: AbortSignal) {
const sourceKey = node.metadata?.storageKey || node.metadata?.content || "";
if (!sourceKey) return Promise.resolve(null);
const requestKey = `${node.id}:${sourceKey}`;
const existing = previewRequests.get(requestKey);
if (existing) return existing;
const request = generateCanvasVideoPreview(node, signal).catch((error: unknown) => {
if (error instanceof DOMException && error.name === "AbortError") {
previewRequests.delete(requestKey);
throw error;
}
return null;
});
previewRequests.set(requestKey, request);
return request;
}
async function generateCanvasVideoPreview(node: CanvasNodeData, signal?: AbortSignal): Promise<VideoPreview | null> {
await waitForBrowserIdle(signal);
throwIfAborted(signal);
const source = await resolveMediaUrl(node.metadata?.storageKey, node.metadata?.content || "");
if (!source) return null;
const captured = await captureVideoPoster(source, { signal, maxWidth: 400 });
throwIfAborted(signal);
if (!captured.poster) return null;
const preview = await uploadImage(captured.poster);
throwIfAborted(signal);
return {
content: preview.url,
storageKey: preview.storageKey,
width: preview.width,
height: preview.height,
bytes: preview.bytes,
mimeType: preview.mimeType,
};
}
function waitForBrowserIdle(signal?: AbortSignal) {
return new Promise<void>((resolve, reject) => {
if (signal?.aborted) {
reject(abortError());
return;
}
let idleId: number | undefined;
let timerId: ReturnType<typeof globalThis.setTimeout> | undefined;
const idleWindow = window as unknown as {
requestIdleCallback?: Window["requestIdleCallback"];
cancelIdleCallback?: Window["cancelIdleCallback"];
};
const cleanup = () => {
signal?.removeEventListener("abort", handleAbort);
if (idleId !== undefined) idleWindow.cancelIdleCallback?.(idleId);
if (timerId !== undefined) globalThis.clearTimeout(timerId);
};
const finish = () => {
cleanup();
resolve();
};
const handleAbort = () => {
cleanup();
reject(abortError());
};
signal?.addEventListener("abort", handleAbort, { once: true });
if (idleWindow.requestIdleCallback) idleId = idleWindow.requestIdleCallback(finish, { timeout: 1_000 });
else timerId = globalThis.setTimeout(finish, 250);
});
}
function throwIfAborted(signal?: AbortSignal) {
if (signal?.aborted) throw abortError();
}
function abortError() {
return new DOMException("Canvas video preview hydration aborted", "AbortError");
}

View File

@@ -2,10 +2,12 @@ import localforage from "localforage";
import { nanoid } from "nanoid";
import { getActiveUserScope } from "@/lib/user-scope";
import { captureVideoPoster, detectVideoAudioTrackFromBlob } from "@/lib/video-poster";
import { resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey, uploadResourceFile } from "@/services/api/resources";
import { uploadImage, type UploadedImage } from "@/services/image-storage";
import { getCachedResourceBlob, getCachedResourceObjectUrl, primeResourceBlobCache } from "@/services/resource-blob-cache";
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number; durationMs?: number };
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number; durationMs?: number; hasAudio?: boolean; preview?: UploadedImage };
const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" });
const objectUrls = new Map<string, string>();
@@ -13,13 +15,24 @@ const objectUrls = new Map<string, string>();
export async function uploadMediaFile(input: string | Blob, prefix = "file"): Promise<UploadedFile> {
const blob = typeof input === "string" ? await (await fetch(input)).blob() : input;
const previewUrl = URL.createObjectURL(blob);
const meta: { width?: number; height?: number; durationMs?: number } = blob.type.startsWith("video/") ? await readVideoMeta(previewUrl) : blob.type.startsWith("audio/") ? await readAudioMeta(previewUrl) : {};
const captured = blob.type.startsWith("video/") ? await captureVideoPoster(previewUrl).catch(() => undefined) : undefined;
// Browser track probes can report a false negative for MP4/MOV files.
// Re-check only when capture did not positively confirm an audio track so
// normal uploads avoid an extra full-blob parse.
const parsedHasAudio = blob.type.startsWith("video/") && captured?.hasAudio !== true ? await detectVideoAudioTrackFromBlob(blob) : undefined;
const resolvedHasAudio = parsedHasAudio ?? (captured?.hasAudio === false ? undefined : captured?.hasAudio);
const meta: { width?: number; height?: number; durationMs?: number; hasAudio?: boolean } = captured
? { width: captured.width, height: captured.height, durationMs: captured.durationMs, hasAudio: resolvedHasAudio }
: blob.type.startsWith("audio/")
? await readAudioMeta(previewUrl)
: { hasAudio: resolvedHasAudio };
const poster = captured?.poster ? await uploadImage(captured.poster).catch(() => undefined) : undefined;
try {
const kind = blob.type.startsWith("video/") ? "video" : blob.type.startsWith("audio/") ? "audio" : "file";
const resource = await uploadResourceFile(blob, kind, { ...meta, fileName: input instanceof File ? input.name : undefined });
await primeResourceBlobCache(resourceStorageKey(resource.id), blob).catch(() => "");
URL.revokeObjectURL(previewUrl);
return { url: resource.publicUrl || resourceFileUrl(resource.id), storageKey: resourceStorageKey(resource.id), bytes: resource.size || blob.size, mimeType: resource.mimeType || blob.type || "application/octet-stream", width: resource.width || meta.width, height: resource.height || meta.height, durationMs: resource.durationMs || meta.durationMs };
return { url: resource.publicUrl || resourceFileUrl(resource.id), storageKey: resourceStorageKey(resource.id), bytes: resource.size || blob.size, mimeType: resource.mimeType || blob.type || "application/octet-stream", width: resource.width || meta.width, height: resource.height || meta.height, durationMs: resource.durationMs || meta.durationMs, hasAudio: meta.hasAudio, preview: poster };
} catch {
// OSS is optional during local/self-hosted setup. Keep the existing local fallback.
}
@@ -27,7 +40,7 @@ export async function uploadMediaFile(input: string | Blob, prefix = "file"): Pr
await store.setItem(storageKey, blob);
const url = previewUrl;
objectUrls.set(storageKey, url);
return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta };
return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta, preview: poster };
}
export async function resolveMediaUrl(storageKey?: string, fallback = "") {
@@ -89,16 +102,6 @@ export function collectMediaStorageKeys(value: unknown, keys = new Set<string>()
return keys;
}
function readVideoMeta(url: string) {
return new Promise<{ width: number; height: number; durationMs?: number }>((resolve) => {
const video = document.createElement("video");
const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720, durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined });
video.onloadedmetadata = done;
video.onerror = done;
video.src = url;
});
}
function readAudioMeta(url: string) {
return new Promise<{ durationMs?: number }>((resolve) => {
const audio = document.createElement("audio");

View File

@@ -1,4 +1,5 @@
import { canvasNodeToAsset, declaredCanvasNodeAssetCategory, findCanvasNodeAsset, type CanvasAssetSource } from "@/lib/canvas/canvas-node-asset";
import { canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import { readImageMeta } from "@/lib/image-utils";
import { parseBackendGenerationResult, type BackendGenerationResult } from "@/services/api/generation-task";
import { linkProjectAsset, moveProjectAsset, updateProjectAssetCategory } from "@/services/api/projects";
@@ -284,7 +285,7 @@ async function generationOutputAsset(input: Parameters<MaterializeGenerationTask
return {
kind: "video",
title: "生成视频",
coverUrl: stored.url,
coverUrl: canvasVideoAssetPreviewUrl(stored.url),
tags: ["生成"],
status: "confirmed",
source: "生成任务",