tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): 视频画面提取 - 支持批量取帧并将片段截取与生成解耦

Browse Source
This commit is contained in:
ddCat
2026-08-29 08:48:38 +08:00
parent 6202b6e762
commit 86019cccea
18 changed files with 586 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
web/src/lib/canvas/canvas-video-frame-nodes.ts Normal file
View File

@@ -0,0 +1,51 @@
import { nanoid } from "nanoid";
import { imageMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
import { formatVideoFrameTime } from "@/lib/canvas/canvas-video-frame";
import type { UploadedImage } from "@/services/image-storage";
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
type UploadedVideoFrame = {
timeMs: number;
image: UploadedImage;
};
const FRAME_NODE_GAP = 24;
const FRAME_NODE_OFFSET = 96;
const FRAME_GRID_MAX_COLUMNS = 3;
export function buildVideoFrameNodes(source: CanvasNodeData, frames: UploadedVideoFrame[]): CanvasNodeData[] {
if (!frames.length) return [];
const sizes = frames.map(({ image }) => fitNodeSize(image.width, image.height, source.width, source.height));
const cellWidth = Math.max(...sizes.map((size) => size.width));
const cellHeight = Math.max(...sizes.map((size) => size.height));
const columns = Math.min(FRAME_GRID_MAX_COLUMNS, Math.max(1, Math.ceil(Math.sqrt(frames.length))));
const startX = source.position.x + source.width + FRAME_NODE_OFFSET;
return frames.map(({ timeMs, image }, index) => {
const size = sizes[index];
const column = index % columns;
const row = Math.floor(index / columns);
return {
id: nanoid(),
type: CanvasNodeType.Image,
title: `画面 ${formatVideoFrameTime(timeMs)} · ${source.title || "视频"}`,
position: {
x: startX + column * (cellWidth + FRAME_NODE_GAP) + (cellWidth - size.width) / 2,
y: source.position.y + row * (cellHeight + FRAME_NODE_GAP) + (cellHeight - size.height) / 2,
},
width: size.width,
height: size.height,
metadata: {
...imageMetadata(image),
prompt: source.metadata?.prompt,
workflowKind: source.metadata?.workflowKind,
workflowTitle: source.metadata?.workflowTitle,
shotIndex: source.metadata?.shotIndex,
videoFrameSourceNodeId: source.id,
videoFrameTimeMs: timeMs,
},
} satisfies CanvasNodeData;
});
}

70
web/src/lib/canvas/canvas-video-frame.ts
View File

@@ -1,7 +1,30 @@
const VIDEO_FRAME_TIMEOUT_MS = 20_000;
const LAST_FRAME_EPSILON_SECONDS = 0.001;
const LAST_FRAME_EPSILON_MS = 1;
export async function captureVideoLastFrame(source: Blob | string) {
export type CapturedVideoFrame = {
timeMs: number;
blob: Blob;
width: number;
height: number;
};
export type VideoFrameCaptureFailure = {
timeMs: number;
error: string;
};
export type VideoFrameCaptureResult = {
frames: CapturedVideoFrame[];
failures: VideoFrameCaptureFailure[];
};
export function normalizeVideoFrameTimes(timesMs: number[], durationMs: number) {
if (!Number.isFinite(durationMs) || durationMs <= 0) return [];
const lastFrameMs = Math.max(0, Math.round(durationMs) - LAST_FRAME_EPSILON_MS);
return Array.from(new Set(timesMs.filter(Number.isFinite).map((timeMs) => Math.min(lastFrameMs, Math.max(0, Math.round(timeMs)))))).sort((left, right) => left - right);
}
export async function captureVideoFrames(source: Blob | string, timesMs: number[]): Promise<VideoFrameCaptureResult> {
const blob = await readVideoBlob(source);
const objectUrl = URL.createObjectURL(blob);
const video = document.createElement("video");
@@ -16,21 +39,32 @@ export async function captureVideoLastFrame(source: Blob | string) {
await loaded;
if (!Number.isFinite(video.duration) || video.duration <= 0) throw new Error("无法确定视频时长");
const targetTime = Math.max(0, video.duration - LAST_FRAME_EPSILON_SECONDS);
if (targetTime > 0) {
const seeked = waitForVideoEvent(video, "seeked", "无法定位到视频最后一帧");
video.currentTime = targetTime;
await seeked;
}
if (!video.videoWidth || !video.videoHeight) throw new Error("无法读取视频画面尺寸");
const normalizedTimes = normalizeVideoFrameTimes(timesMs, video.duration * 1000);
if (!normalizedTimes.length) throw new Error("请至少选择一个有效时间点");
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const context = canvas.getContext("2d");
if (!context) throw new Error("浏览器无法创建图片画布");
context.drawImage(video, 0, 0, canvas.width, canvas.height);
return canvasToPngBlob(canvas);
const frames: CapturedVideoFrame[] = [];
const failures: VideoFrameCaptureFailure[] = [];
for (const timeMs of normalizedTimes) {
try {
const targetSeconds = timeMs / 1000;
if (Math.abs(video.currentTime - targetSeconds) > 0.0005) {
const seeked = waitForVideoEvent(video, "seeked", `无法定位到 ${formatVideoFrameTime(timeMs)}`);
video.currentTime = targetSeconds;
await seeked;
}
context.drawImage(video, 0, 0, canvas.width, canvas.height);
frames.push({ timeMs, blob: await canvasToPngBlob(canvas), width: canvas.width, height: canvas.height });
} catch (error) {
failures.push({ timeMs, error: error instanceof Error ? error.message : "画面提取失败" });
}
}
if (!frames.length) throw new Error(failures[0]?.error || "画面提取失败");
return { frames, failures };
} finally {
video.pause();
video.removeAttribute("src");
@@ -42,14 +76,22 @@ export async function captureVideoLastFrame(source: Blob | string) {
async function readVideoBlob(source: Blob | string) {
if (source instanceof Blob) return source;
try {
const response = await fetch(source);
const response = await fetch(source, { credentials: "same-origin" });
if (!response.ok) throw new Error(String(response.status));
return await response.blob();
} catch {
throw new Error("无法读取视频文件，请重新上传视频后再截取尾帧");
throw new Error("无法读取视频文件，请重新上传视频后再提取画面");
}
}
export function formatVideoFrameTime(timeMs: number) {
const totalMs = Math.max(0, Math.round(timeMs));
const minutes = Math.floor(totalMs / 60_000);
const seconds = Math.floor((totalMs % 60_000) / 1000);
const milliseconds = totalMs % 1000;
return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`;
}
function waitForVideoEvent(video: HTMLVideoElement, eventName: "loadeddata" | "seeked", errorMessage: string) {
return new Promise<void>((resolve, reject) => {
let timer = 0;
@@ -73,5 +115,5 @@ function waitForVideoEvent(video: HTMLVideoElement, eventName: "loadeddata" | "s
}
function canvasToPngBlob(canvas: HTMLCanvasElement) {
return new Promise<Blob>((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("尾帧图片编码失败"))), "image/png"));
return new Promise<Blob>((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("画面图片编码失败"))), "image/png"));
}

2
web/src/lib/canvas/canvas-video-regeneration.ts
View File

@@ -15,7 +15,7 @@ export function listVideoReferenceModels(config: AiConfig): string[] {
export function videoReferenceRegenerationError(config: AiConfig): string {
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
if (!videoProfile || videoProfile.references.maxVideos < 1) {
return "当前所选视频模型不支持参考视频，无法使用截取重生成。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。";
return "当前所选视频模型不支持参考视频，无法为片段创建待生成节点。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。";
}
if (!videoProfile.operations.length) return "当前视频模型没有可用的视频生成模式";
return "";

20
web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx
View File

@@ -1,4 +1,4 @@
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
@@ -63,17 +63,17 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
run: (ctx) => ctx.handlers.onNodeRetry(ctx.node!),
},
{
id: "extractLastFrame",
id: "extractFrames",
toolbar: "node-hover",
category: "node-state",
label: (ctx) => ctx.extractingVideoFrame ? "正在截取尾帧" : "截取尾帧",
displayLabel: (ctx) => ctx.extractingVideoFrame ? "截取中" : "尾帧",
icon: (ctx) => ctx.extractingVideoFrame ? <LoaderCircle className="size-3.5 animate-spin" /> : <GalleryHorizontalEnd className="size-3.5" />,
label: (ctx) => ctx.extractingVideoFrames ? "正在提取视频画面" : "提取一个或多个视频画面",
displayLabel: (ctx) => ctx.extractingVideoFrames ? "提取中" : "提取画面",
icon: (ctx) => ctx.extractingVideoFrames ? <LoaderCircle className="size-3.5 animate-spin" /> : <Images className="size-3.5" />,
defaultVisible: true,
defaultOrder: 40,
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
disabled: (ctx) => ctx.extractingVideoFrame,
run: (ctx) => ctx.handlers.onNodeExtractVideoLastFrame(ctx.node!),
disabled: (ctx) => ctx.extractingVideoFrames,
run: (ctx) => ctx.handlers.onNodeExtractVideoFrames(ctx.node!),
},
{
id: "extractAudio",
@@ -92,14 +92,14 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
id: "trimRegenerate",
toolbar: "node-hover",
category: "node-state",
label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "按段截取并重生成",
displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取重生成",
label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "截取一个或多个视频片段",
displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取片段",
icon: (ctx) => ctx.trimmingVideo ? <LoaderCircle className="size-3.5 animate-spin" /> : <Scissors className="size-3.5" />,
defaultVisible: true,
defaultOrder: 44,
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo,
run: (ctx) => ctx.handlers.onNodeTrimVideoRegenerate(ctx.node!),
run: (ctx) => ctx.handlers.onNodeTrimVideoSegments(ctx.node!),
},
{
id: "saveAsset",

8
web/src/lib/canvas/tool-registry/tool-definition.ts
View File

@@ -89,9 +89,9 @@ export type ToolbarHandlers = {
onNodeSuperResolve: (node: CanvasNodeData) => void;
onNodeAngle: (node: CanvasNodeData) => void;
onNodeViewImage: (node: CanvasNodeData) => void;
onNodeExtractVideoLastFrame: (node: CanvasNodeData) => void;
onNodeExtractVideoFrames: (node: CanvasNodeData) => void;
onNodeExtractAudioFromVideo: (node: CanvasNodeData) => void;
onNodeTrimVideoRegenerate: (node: CanvasNodeData) => void;
onNodeTrimVideoSegments: (node: CanvasNodeData) => void;
onNodeSubtitles: (node: CanvasNodeData) => void;
onNodeTimeline: (node: CanvasNodeData) => void;
onNodeReversePrompt: (node: CanvasNodeData) => void;
@@ -114,8 +114,8 @@ export type ToolContext = {
node?: CanvasNodeData;
/** 便捷访问 node.metadata（node 为空时为 undefined） */
nodeMetadata?: CanvasNodeMetadata;
/** 视频尾帧提取中（节点悬停工具栏用） */
extractingVideoFrame: boolean;
/** 视频画面提取中（节点悬停工具栏用） */
extractingVideoFrames: boolean;
/** 视频音频提取/片段截取进行中（节点悬停工具栏用） */
extractingAudio: boolean;
trimmingVideo: boolean;