perf(canvas): 画布媒体 - 优化媒体加载、播放器与节点交互 (#357)
Merge origin/main at fb089b2 and retain canvas media performance, player layout, and interaction updates.
Co-authored-by: ddCat <2695092045@qq.com>
This commit is contained in:
@@ -43,6 +43,7 @@ export function canvasAssetHandoffPayloads(assets: Asset[], assetIds: Iterable<s
|
||||
width: asset.data.width,
|
||||
height: asset.data.height,
|
||||
durationMs: asset.data.durationMs,
|
||||
hasAudio: asset.data.hasAudio,
|
||||
bytes: asset.data.bytes,
|
||||
mimeType: asset.data.mimeType,
|
||||
assetId: asset.id,
|
||||
|
||||
@@ -66,6 +66,15 @@ export function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
|
||||
bytes: video.bytes,
|
||||
mimeType: video.mimeType || "video/mp4",
|
||||
durationMs: video.durationMs,
|
||||
hasAudio: video.hasAudio,
|
||||
videoPreview: video.preview ? {
|
||||
content: video.preview.url,
|
||||
storageKey: video.preview.storageKey,
|
||||
width: video.preview.width,
|
||||
height: video.preview.height,
|
||||
bytes: video.preview.bytes,
|
||||
mimeType: video.preview.mimeType,
|
||||
} : undefined,
|
||||
errorDetails: undefined,
|
||||
generationErrorCode: undefined,
|
||||
resourceReloadAvailable: undefined,
|
||||
|
||||
23
web/src/lib/canvas/canvas-media-preview.ts
Normal file
23
web/src/lib/canvas/canvas-media-preview.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { buildLibTVVideoPreviewUrl } from "@/lib/canvas/libtv-import";
|
||||
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
|
||||
|
||||
/**
|
||||
* Returns an image source that is safe to mount for a passive video preview.
|
||||
* The original video URL is deliberately never returned: callers must fall
|
||||
* back to a video icon instead of creating a decoder outside the active node.
|
||||
*/
|
||||
export function canvasNodeVideoPreviewUrl(node: CanvasNodeData) {
|
||||
if (node.type !== CanvasNodeType.Video) return "";
|
||||
const content = node.metadata?.content || "";
|
||||
const generatedPreview = node.metadata?.videoPreview?.content || "";
|
||||
if (generatedPreview && generatedPreview !== content) return generatedPreview;
|
||||
const explicitPreview = node.metadata?.previewContent || "";
|
||||
if (explicitPreview && explicitPreview !== content) return explicitPreview;
|
||||
return buildLibTVVideoPreviewUrl(content);
|
||||
}
|
||||
|
||||
export function canvasVideoAssetPreviewUrl(videoUrl: string, coverUrl?: string) {
|
||||
const explicitCover = coverUrl?.trim() || "";
|
||||
if (explicitCover && explicitCover !== videoUrl) return explicitCover;
|
||||
return buildLibTVVideoPreviewUrl(videoUrl);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { getDataUrlByteSize } from "@/lib/image-utils";
|
||||
import { canvasNodeVideoPreviewUrl } from "@/lib/canvas/canvas-media-preview";
|
||||
import type { Asset, AssetCategory, NewAsset } from "@/stores/use-asset-store";
|
||||
import { CanvasNodeType, type CanvasNodeData, type CanvasNodeTypeId } from "@/types/canvas";
|
||||
|
||||
@@ -24,7 +25,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
|
||||
};
|
||||
const base = {
|
||||
title,
|
||||
coverUrl: node.type === CanvasNodeType.Image ? content : "",
|
||||
coverUrl: node.type === CanvasNodeType.Image ? content : node.type === CanvasNodeType.Video ? canvasNodeVideoPreviewUrl(node) : "",
|
||||
tags: node.metadata?.assetTags || [],
|
||||
category: canvasNodeAssetCategory(node),
|
||||
status: "confirmed" as const,
|
||||
@@ -60,6 +61,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
|
||||
width: node.metadata?.naturalWidth || node.width,
|
||||
height: node.metadata?.naturalHeight || node.height,
|
||||
durationMs: node.metadata?.durationMs,
|
||||
hasAudio: node.metadata?.hasAudio,
|
||||
bytes: node.metadata?.bytes || 0,
|
||||
mimeType: node.metadata?.mimeType || "video/mp4",
|
||||
},
|
||||
|
||||
25
web/src/lib/canvas/canvas-node-stack-order.ts
Normal file
25
web/src/lib/canvas/canvas-node-stack-order.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
export type CanvasNodeStackOrder = string[];
|
||||
|
||||
/**
|
||||
* Keeps the most recently interacted node at the end of the paint order.
|
||||
* This is session-local UI state and is intentionally not persisted with the
|
||||
* canvas document.
|
||||
*/
|
||||
export function bringCanvasNodeToFront(order: CanvasNodeStackOrder, nodeId: string): CanvasNodeStackOrder {
|
||||
if (!nodeId) return order;
|
||||
if (order.at(-1) === nodeId) return order;
|
||||
return [...order.filter((id) => id !== nodeId), nodeId];
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the interaction order while preserving the relative order of nodes
|
||||
* that have not been interacted with yet.
|
||||
*/
|
||||
export function sortCanvasNodesByStackOrder<T extends { id: string }>(nodes: T[], order: CanvasNodeStackOrder): T[] {
|
||||
if (nodes.length < 2 || order.length === 0) return nodes;
|
||||
const rankById = new Map(order.map((id, index) => [id, index]));
|
||||
return nodes
|
||||
.map((node, index) => ({ node, index, rank: rankById.get(node.id) ?? -1 }))
|
||||
.sort((left, right) => left.rank - right.rank || left.index - right.index)
|
||||
.map(({ node }) => node);
|
||||
}
|
||||
@@ -13,6 +13,8 @@ const HYDRATED_MEDIA_KEYS = new Set<keyof CanvasNodeMetadata>([
|
||||
"bytes",
|
||||
"mimeType",
|
||||
"durationMs",
|
||||
"hasAudio",
|
||||
"videoPreview",
|
||||
"drawingPreviewUrl",
|
||||
]);
|
||||
|
||||
|
||||
@@ -25,3 +25,11 @@ export function shouldReduceCanvasMediaEffects(mode: CanvasMediaPerformanceMode,
|
||||
const mediaCount = nodes.filter((node) => node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio).length;
|
||||
return nodes.length >= 80 || mediaCount >= 32;
|
||||
}
|
||||
|
||||
export function resolveActiveCanvasMediaNodeId(selectedNodeIds: ReadonlySet<string>, nodeById: ReadonlyMap<string, CanvasNodeData>) {
|
||||
if (selectedNodeIds.size !== 1) return null;
|
||||
const nodeId = selectedNodeIds.values().next().value;
|
||||
if (!nodeId) return null;
|
||||
const type = nodeById.get(nodeId)?.type;
|
||||
return type === CanvasNodeType.Video || type === CanvasNodeType.Audio ? nodeId : null;
|
||||
}
|
||||
|
||||
@@ -354,7 +354,11 @@ export async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
|
||||
return Promise.all(
|
||||
nodes.map(async (node) => {
|
||||
const content = node.metadata?.content;
|
||||
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
|
||||
const videoPreview = node.type === CanvasNodeType.Video && node.metadata?.videoPreview?.storageKey
|
||||
? { ...node.metadata.videoPreview, content: await resolveImageUrl(node.metadata.videoPreview.storageKey, node.metadata.videoPreview.content, { cacheMiss: true }) }
|
||||
: node.metadata?.videoPreview;
|
||||
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content), videoPreview } };
|
||||
if (videoPreview !== node.metadata?.videoPreview) return { ...node, metadata: { ...node.metadata, videoPreview } };
|
||||
if (node.type === CanvasNodeType.Image && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content, { cacheMiss: true }) } };
|
||||
if (node.type !== CanvasNodeType.Image || !content) return node;
|
||||
if (!content.startsWith("data:image/")) return node;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
|
||||
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
|
||||
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
|
||||
import { seedanceReferenceLabel } from "@/lib/seedance-video";
|
||||
import type { Skill } from "@/services/api/skills";
|
||||
@@ -14,7 +15,11 @@ export type CanvasResourceReference = {
|
||||
label: string;
|
||||
title: string;
|
||||
previewUrl?: string;
|
||||
/** 仅素材库视频在没有静态封面时使用的首帧回退源。 */
|
||||
mediaUrl?: string;
|
||||
storageKey?: string;
|
||||
/** 视频首帧是独立的图片资源,不能用视频 storageKey 解析。 */
|
||||
previewStorageKey?: string;
|
||||
text?: string;
|
||||
active: boolean;
|
||||
sourceType?: CanvasNodeTypeId;
|
||||
@@ -50,7 +55,7 @@ export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceRefe
|
||||
return assets.flatMap((asset): CanvasResourceReference[] => {
|
||||
if (asset.kind === "model") return [];
|
||||
const kind: CanvasResourceKind = asset.kind === "entity" ? "character" : asset.kind;
|
||||
const previewUrl = asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? asset.data.url : asset.coverUrl;
|
||||
const previewUrl = asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? canvasVideoAssetPreviewUrl(asset.data.url, asset.coverUrl) : asset.coverUrl;
|
||||
const text = asset.kind === "text" ? asset.data.content : undefined;
|
||||
return [{
|
||||
id: `asset:${asset.id}`,
|
||||
@@ -60,6 +65,7 @@ export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceRefe
|
||||
label: asset.title,
|
||||
title: asset.title,
|
||||
previewUrl,
|
||||
mediaUrl: asset.kind === "video" && !previewUrl ? asset.data.url : undefined,
|
||||
storageKey: "storageKey" in asset.data ? asset.data.storageKey : undefined,
|
||||
text,
|
||||
active: false,
|
||||
@@ -148,8 +154,15 @@ function labelResourceNodes(nodes: CanvasNodeData[], active: boolean) {
|
||||
kind,
|
||||
label,
|
||||
title: node.title || label,
|
||||
previewUrl: node.metadata?.workflowKind === "character" ? node.metadata.characterCoverUrl : node.type === CanvasNodeType.Drawing ? node.metadata?.drawingPreviewUrl : node.metadata?.previewContent || node.metadata?.content,
|
||||
previewUrl: node.metadata?.workflowKind === "character"
|
||||
? node.metadata.characterCoverUrl
|
||||
: node.type === CanvasNodeType.Drawing
|
||||
? node.metadata?.drawingPreviewUrl
|
||||
: node.type === CanvasNodeType.Video
|
||||
? canvasNodeVideoPreviewUrl(node)
|
||||
: node.metadata?.previewContent || node.metadata?.content,
|
||||
storageKey: node.metadata?.storageKey,
|
||||
previewStorageKey: node.type === CanvasNodeType.Video ? node.metadata?.videoPreview?.storageKey : undefined,
|
||||
text: node.metadata?.workflowKind === "character" ? node.metadata.characterPrompt : node.type === CanvasNodeType.Text ? node.metadata?.content || node.metadata?.prompt : node.type === CanvasNodeType.Skill ? skillResourceText(node) : undefined,
|
||||
active,
|
||||
sourceType: node.type,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
const LIBTV_PROJECT_UUID_PATTERN = /^[a-f0-9]{32}$/i;
|
||||
const LIBTV_RESOURCE_HOST = "libtv-res.liblib.art";
|
||||
const LIBTV_PREVIEW_WIDTH = 960;
|
||||
const LIBTV_VIDEO_PREVIEW_WIDTH = 400;
|
||||
|
||||
export function parseLibTVProjectUUID(value: string) {
|
||||
const trimmed = value.trim();
|
||||
@@ -34,3 +35,32 @@ export function buildLibTVImagePreviewUrl(rawUrl: string) {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildLibTVVideoPreviewUrl(rawUrl: string) {
|
||||
try {
|
||||
const url = new URL(rawUrl);
|
||||
if (url.protocol !== "https:" || url.hostname !== LIBTV_RESOURCE_HOST) return "";
|
||||
if (!url.searchParams.has("x-oss-process")) {
|
||||
url.searchParams.set("x-oss-process", `video/snapshot,t_0,f_jpg,w_${LIBTV_VIDEO_PREVIEW_WIDTH},m_fast,ar_auto`);
|
||||
}
|
||||
return url.toString();
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Restores the playable LibTV asset URL when a snapshot transform was
|
||||
* persisted as the node's content by an older importer. The OSS transform
|
||||
* belongs only on previewContent; VideoPlayer must receive the original MP4.
|
||||
*/
|
||||
export function buildLibTVVideoSourceUrl(rawUrl: string) {
|
||||
try {
|
||||
const url = new URL(rawUrl);
|
||||
if (url.protocol !== "https:" || url.hostname !== LIBTV_RESOURCE_HOST) return rawUrl;
|
||||
url.searchParams.delete("x-oss-process");
|
||||
return url.toString();
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
308
web/src/lib/video-poster.ts
Normal file
308
web/src/lib/video-poster.ts
Normal file
@@ -0,0 +1,308 @@
|
||||
export type CapturedVideoPoster = {
|
||||
width: number;
|
||||
height: number;
|
||||
durationMs?: number;
|
||||
hasAudio?: boolean;
|
||||
poster?: Blob;
|
||||
};
|
||||
|
||||
type CaptureVideoPosterOptions = {
|
||||
signal?: AbortSignal;
|
||||
maxWidth?: number;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
const MAX_MP4_AUDIO_PROBE_BYTES = 32 * 1024 * 1024;
|
||||
const MP4_AUDIO_PROBE_RANGE_BYTES = 1024 * 1024;
|
||||
|
||||
const videoAudioProbeCache = new Map<string, Promise<boolean | undefined>>();
|
||||
|
||||
let captureQueue: Promise<void> = Promise.resolve();
|
||||
|
||||
/** Serializes browser video decoding so batch uploads and poster hydration never fan out decoders. */
|
||||
export function captureVideoPoster(source: string, options: CaptureVideoPosterOptions = {}) {
|
||||
const capture = captureQueue.then(() => captureVideoPosterNow(source, options));
|
||||
captureQueue = capture.then(() => undefined, () => undefined);
|
||||
return capture;
|
||||
}
|
||||
|
||||
function captureVideoPosterNow(source: string, options: CaptureVideoPosterOptions) {
|
||||
return new Promise<CapturedVideoPoster>((resolve, reject) => {
|
||||
const signal = options.signal;
|
||||
if (signal?.aborted) {
|
||||
reject(abortError());
|
||||
return;
|
||||
}
|
||||
|
||||
const video = document.createElement("video");
|
||||
const fallback = { width: 1280, height: 720 };
|
||||
let settled = false;
|
||||
let metadata: Omit<CapturedVideoPoster, "poster"> = fallback;
|
||||
|
||||
const cleanup = () => {
|
||||
window.clearTimeout(timeoutId);
|
||||
signal?.removeEventListener("abort", handleAbort);
|
||||
video.onloadedmetadata = null;
|
||||
video.onloadeddata = null;
|
||||
video.onerror = null;
|
||||
video.pause();
|
||||
video.removeAttribute("src");
|
||||
video.load();
|
||||
};
|
||||
const finish = (result: CapturedVideoPoster) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve(result);
|
||||
};
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const readMetadata = () => {
|
||||
metadata = {
|
||||
width: video.videoWidth || fallback.width,
|
||||
height: video.videoHeight || fallback.height,
|
||||
durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined,
|
||||
hasAudio: detectVideoAudioTrack(video),
|
||||
};
|
||||
};
|
||||
const handleAbort = () => fail(abortError());
|
||||
const timeoutId = window.setTimeout(() => finish(metadata), options.timeoutMs ?? 15_000);
|
||||
|
||||
signal?.addEventListener("abort", handleAbort, { once: true });
|
||||
video.preload = "auto";
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
if (isCrossOriginHttpUrl(source)) video.crossOrigin = "anonymous";
|
||||
video.onloadedmetadata = readMetadata;
|
||||
video.onerror = () => finish(metadata);
|
||||
video.onloadeddata = () => {
|
||||
readMetadata();
|
||||
if (signal?.aborted) {
|
||||
fail(abortError());
|
||||
return;
|
||||
}
|
||||
const maxWidth = Math.max(1, options.maxWidth ?? 400);
|
||||
const width = Math.max(1, Math.min(metadata.width, maxWidth));
|
||||
const height = Math.max(1, Math.round(width * metadata.height / Math.max(1, metadata.width)));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) {
|
||||
finish(metadata);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
context.fillStyle = "#000";
|
||||
context.fillRect(0, 0, width, height);
|
||||
context.drawImage(video, 0, 0, width, height);
|
||||
canvas.toBlob((poster) => finish({ ...metadata, poster: poster || undefined }), "image/jpeg", 0.82);
|
||||
} catch {
|
||||
// Cross-origin videos without CORS can still expose metadata but cannot be drawn safely.
|
||||
finish(metadata);
|
||||
}
|
||||
};
|
||||
video.src = source;
|
||||
video.load();
|
||||
});
|
||||
}
|
||||
|
||||
function isCrossOriginHttpUrl(source: string) {
|
||||
try {
|
||||
const url = new URL(source, window.location.href);
|
||||
return /^https?:$/.test(url.protocol) && url.origin !== window.location.origin;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Audio track inspection is not standardized across browsers. */
|
||||
export function detectVideoAudioTrack(video: HTMLMediaElement): boolean | undefined {
|
||||
const tracks = (video as HTMLVideoElement & { audioTracks?: { length: number } }).audioTracks;
|
||||
if (tracks && typeof tracks.length === "number") {
|
||||
// A zero-length list before media data is loaded can mean that the
|
||||
// browser has not populated its native track list yet.
|
||||
if (tracks.length > 0) return true;
|
||||
// Do not treat an empty list as proof of silence. Chromium-based
|
||||
// browsers can expose AudioTrackList while leaving it empty even for
|
||||
// a video that has an audio stream; continue with the other probes.
|
||||
}
|
||||
|
||||
const mozHasAudio = (video as HTMLVideoElement & { mozHasAudio?: boolean }).mozHasAudio;
|
||||
if (typeof mozHasAudio === "boolean") return mozHasAudio;
|
||||
|
||||
// Chromium and WebKit expose this legacy counter on media elements. It is
|
||||
// meaningful once media data is available; before that point zero only
|
||||
// means that decoding has not started yet.
|
||||
const webkitAudioDecodedByteCount = (video as HTMLVideoElement & { webkitAudioDecodedByteCount?: number }).webkitAudioDecodedByteCount;
|
||||
if (typeof webkitAudioDecodedByteCount === "number" && video.readyState >= 3 && webkitAudioDecodedByteCount > 0) return true;
|
||||
|
||||
const captureStream = (video as HTMLVideoElement & { captureStream?: () => MediaStream }).captureStream;
|
||||
if (typeof captureStream === "function") {
|
||||
try {
|
||||
const audioTracks = captureStream.call(video).getAudioTracks();
|
||||
if (audioTracks.length > 0) return true;
|
||||
} catch {
|
||||
// Some browsers expose captureStream but reject it for remote media.
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspects the ISO-BMFF track headers without decoding the media. This is a
|
||||
* bounded fallback for local MP4/MOV uploads where browsers expose no audio
|
||||
* track API. Remote sources are intentionally not fetched by this helper.
|
||||
*/
|
||||
export async function detectVideoAudioTrackFromBlob(blob: Blob): Promise<boolean | undefined> {
|
||||
if (!isIsoBmffVideoType(blob.type) || blob.size <= 0 || blob.size > MAX_MP4_AUDIO_PROBE_BYTES) return undefined;
|
||||
try {
|
||||
const bytes = new Uint8Array(await blob.arrayBuffer());
|
||||
return parseIsoBmffAudioTrack(bytes);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probes a remote MP4/MOV without downloading the whole file when possible.
|
||||
* The browser media element does not expose a portable audio-track API, while
|
||||
* LibTV assets are CORS-enabled and expose ISO-BMFF headers through Range.
|
||||
*/
|
||||
export function detectVideoAudioTrackFromUrl(source: string, signal?: AbortSignal) {
|
||||
const normalized = source.trim();
|
||||
if (!normalized || videoAudioProbeCache.has(normalized)) return videoAudioProbeCache.get(normalized) || Promise.resolve(undefined);
|
||||
const probe = probeVideoAudioUrl(normalized, signal)
|
||||
.then((result) => {
|
||||
if (result === undefined) videoAudioProbeCache.delete(normalized);
|
||||
return result;
|
||||
})
|
||||
.catch(() => {
|
||||
videoAudioProbeCache.delete(normalized);
|
||||
return undefined;
|
||||
});
|
||||
videoAudioProbeCache.set(normalized, probe);
|
||||
return probe;
|
||||
}
|
||||
|
||||
async function probeVideoAudioUrl(source: string, signal?: AbortSignal): Promise<boolean | undefined> {
|
||||
if (!isLikelyIsoBmffVideoUrl(source)) return undefined;
|
||||
const first = await fetchVideoRange(source, 0, MP4_AUDIO_PROBE_RANGE_BYTES - 1, signal);
|
||||
if (!first) return undefined;
|
||||
const firstResult = parseIsoBmffAudioTrack(first.bytes);
|
||||
if (firstResult !== undefined) return firstResult;
|
||||
|
||||
const totalBytes = first.totalBytes;
|
||||
if (totalBytes && totalBytes > MP4_AUDIO_PROBE_RANGE_BYTES) {
|
||||
const tailStart = Math.max(0, totalBytes - MP4_AUDIO_PROBE_RANGE_BYTES);
|
||||
const tail = await fetchVideoRange(source, tailStart, totalBytes - 1, signal);
|
||||
const tailResult = tail ? parseIsoBmffAudioTrack(tail.bytes) : undefined;
|
||||
if (tailResult !== undefined) return tailResult;
|
||||
}
|
||||
|
||||
// Some origins ignore Range and return the complete response. Only accept
|
||||
// a bounded full response so a missing Content-Range cannot trigger a
|
||||
// large duplicate download for an active video.
|
||||
if (first.status === 200 && first.bytes.byteLength <= MAX_MP4_AUDIO_PROBE_BYTES) return parseIsoBmffAudioTrack(first.bytes);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function fetchVideoRange(source: string, start: number, end: number, signal?: AbortSignal) {
|
||||
const response = await fetch(source, { headers: { Range: `bytes=${start}-${end}` }, signal, cache: "force-cache" });
|
||||
if (!response.ok) return undefined;
|
||||
const declaredLength = Number(response.headers.get("content-length"));
|
||||
if (Number.isFinite(declaredLength) && (response.status === 200 && declaredLength > MAX_MP4_AUDIO_PROBE_BYTES || response.status === 206 && declaredLength > MP4_AUDIO_PROBE_RANGE_BYTES)) return undefined;
|
||||
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||
const contentRange = response.headers.get("content-range") || "";
|
||||
const rangeMatch = /bytes\s+\d+-\d+\/(\d+)/i.exec(contentRange);
|
||||
const totalBytes = rangeMatch ? Number(rangeMatch[1]) : Number.isFinite(declaredLength) ? declaredLength : undefined;
|
||||
return { bytes, status: response.status, totalBytes };
|
||||
}
|
||||
|
||||
function isIsoBmffVideoType(mimeType: string) {
|
||||
const normalized = mimeType.toLowerCase().split(";", 1)[0].trim();
|
||||
return normalized === "video/mp4" || normalized === "video/quicktime" || normalized === "application/mp4";
|
||||
}
|
||||
|
||||
function isLikelyIsoBmffVideoUrl(source: string) {
|
||||
try {
|
||||
const baseUrl = typeof window === "undefined" ? "http://localhost" : window.location?.href || "http://localhost";
|
||||
const url = new URL(source, baseUrl);
|
||||
return /\.(mp4|m4v|mov)(?:$|[?#])/i.test(url.pathname + url.search);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function parseIsoBmffAudioTrack(bytes: Uint8Array): boolean | undefined {
|
||||
const direct = parseIsoBmffAudioTrackFromOffset(bytes, 0);
|
||||
if (direct !== undefined) return direct;
|
||||
// A suffix Range can begin in the middle of an earlier box. Locate a
|
||||
// complete `moov` header inside that suffix before giving up.
|
||||
for (let offset = 0; offset + 8 <= bytes.byteLength; offset += 1) {
|
||||
if (readFourcc(bytes, offset + 4) !== "moov") continue;
|
||||
const result = parseIsoBmffAudioTrackFromOffset(bytes, offset);
|
||||
if (result !== undefined) return result;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseIsoBmffAudioTrackFromOffset(bytes: Uint8Array, startOffset: number): boolean | undefined {
|
||||
let foundMoov = false;
|
||||
let foundAudio = false;
|
||||
|
||||
const visit = (start: number, end: number, insideMoov: boolean) => {
|
||||
let offset = start;
|
||||
while (offset + 8 <= end) {
|
||||
const size32 = readUint32(bytes, offset);
|
||||
const type = readFourcc(bytes, offset + 4);
|
||||
let headerSize = 8;
|
||||
let boxEnd: number;
|
||||
if (size32 === 1) {
|
||||
if (offset + 16 > end) return;
|
||||
const high = readUint32(bytes, offset + 8);
|
||||
const low = readUint32(bytes, offset + 12);
|
||||
const size = high * 0x100000000 + low;
|
||||
headerSize = 16;
|
||||
boxEnd = offset + size;
|
||||
} else if (size32 === 0) {
|
||||
boxEnd = end;
|
||||
} else {
|
||||
boxEnd = offset + size32;
|
||||
}
|
||||
// Empty leaf boxes (size == header size) are valid ISO-BMFF boxes
|
||||
// such as `free`; only reject truncated or backwards boxes.
|
||||
if (!type || boxEnd < offset + headerSize || boxEnd > end) return;
|
||||
|
||||
const isMoov = type === "moov";
|
||||
if (isMoov) foundMoov = true;
|
||||
if (insideMoov && type === "hdlr" && offset + 16 <= boxEnd && readFourcc(bytes, offset + 16) === "soun") foundAudio = true;
|
||||
|
||||
if (isMoov || insideMoov && (type === "trak" || type === "mdia")) {
|
||||
visit(offset + headerSize, boxEnd, insideMoov || isMoov);
|
||||
}
|
||||
if (foundAudio) return;
|
||||
offset = boxEnd;
|
||||
}
|
||||
};
|
||||
|
||||
visit(startOffset, bytes.byteLength, false);
|
||||
return foundMoov ? foundAudio : undefined;
|
||||
}
|
||||
|
||||
function readUint32(bytes: Uint8Array, offset: number) {
|
||||
return ((bytes[offset] || 0) * 0x1000000) + ((bytes[offset + 1] || 0) << 16) + ((bytes[offset + 2] || 0) << 8) + (bytes[offset + 3] || 0);
|
||||
}
|
||||
|
||||
function readFourcc(bytes: Uint8Array, offset: number) {
|
||||
if (offset + 4 > bytes.byteLength) return "";
|
||||
return String.fromCharCode(bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]);
|
||||
}
|
||||
|
||||
function abortError() {
|
||||
return new DOMException("Video poster capture aborted", "AbortError");
|
||||
}
|
||||
Reference in New Issue
Block a user