feat(canvas): 视频画面提取 - 支持批量取帧并将片段截取与生成解耦

This commit is contained in:
ddCat
2026-08-29 08:48:38 +08:00
parent 6202b6e762
commit 86019cccea
18 changed files with 586 additions and 155 deletions

View File

@@ -40,12 +40,12 @@ type CanvasNodeToolbarProps = {
onSuperResolve: (node: CanvasNodeData) => void;
onAngle: (node: CanvasNodeData) => void;
onViewImage: (node: CanvasNodeData) => void;
onExtractVideoLastFrame: (node: CanvasNodeData) => void;
onExtractVideoFrames: (node: CanvasNodeData) => void;
onExtractAudioFromVideo: (node: CanvasNodeData) => void;
onTrimVideoRegenerate: (node: CanvasNodeData) => void;
onTrimVideoSegments: (node: CanvasNodeData) => void;
onSubtitles: (node: CanvasNodeData) => void;
onTimeline: (node: CanvasNodeData) => void;
extractingVideoFrame: boolean;
extractingVideoFrames: boolean;
extractingAudio: boolean;
trimmingVideo: boolean;
onReversePrompt: (node: CanvasNodeData) => void;
@@ -103,12 +103,12 @@ export function CanvasNodeToolbar({
onSuperResolve,
onAngle,
onViewImage,
onExtractVideoLastFrame,
onExtractVideoFrames,
onExtractAudioFromVideo,
onTrimVideoRegenerate,
onTrimVideoSegments,
onSubtitles,
onTimeline,
extractingVideoFrame,
extractingVideoFrames,
extractingAudio,
trimmingVideo,
onReversePrompt,
@@ -201,7 +201,7 @@ export function CanvasNodeToolbar({
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoRegenerate: onTrimVideoRegenerate, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
} as Partial<ToolbarHandlers> as ToolbarHandlers;
@@ -216,7 +216,7 @@ export function CanvasNodeToolbar({
canRedo: false,
node,
nodeMetadata: node.metadata,
extractingVideoFrame,
extractingVideoFrames,
extractingAudio,
trimmingVideo,
mergingVideos: false,
@@ -249,9 +249,9 @@ export function CanvasNodeToolbar({
const imageEditTools = takeTools(["maskEdit", "crop", "split"]);
const imagePortraitTools = takeTools(["emotion", "portraitTexture"]).map((tool) => tool.id === "emotion" ? { ...tool, label: "人物情绪" } : tool);
const imageAngleTool = toolById.get("angle");
const videoTools = takeTools(["delete", "download", "subtitles", "timeline", "extractLastFrame", "extractAudio", "trimRegenerate", "uploadVideo"]).map((tool) => {
if (tool.id === "extractLastFrame") return { ...tool, label: "尾帧截取" };
if (tool.id === "trimRegenerate") return { ...tool, label: "截取重生成" };
const videoTools = takeTools(["delete", "download", "subtitles", "timeline", "extractFrames", "extractAudio", "trimRegenerate", "uploadVideo"]).map((tool) => {
if (tool.id === "extractFrames") return { ...tool, label: "提取画面" };
if (tool.id === "trimRegenerate") return { ...tool, label: "截取片段" };
return tool;
});
const genericTools = takeTools(isAudio ? ["delete", "download", "timeline", "uploadAudio"] : isEditableText ? ["delete", "edit", "editText", "generateImage", "saveAsset"] : ["delete", "info", "config"]);

View File

@@ -151,7 +151,7 @@ export function CanvasToolbar({
onNodeInfo: () => {}, onNodeDelete: () => {}, onNodeRetry: () => {}, onNodeEditText: () => {}, onNodeDecreaseFont: () => {}, onNodeIncreaseFont: () => {},
onNodeToggleDialog: () => {}, onNodeAnnotate: () => {}, onNodeGenerateImage: () => {}, onNodeUpload: () => {}, onNodeDownload: () => {}, onNodeSaveAsset: () => {},
onNodeMaskEdit: () => {}, onNodeEmotion: () => {}, onNodePortraitTexture: () => {}, onNodeCrop: () => {}, onNodeSplit: () => {}, onNodeUpscale: () => {},
onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoRegenerate: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {},
onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoFrames: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoSegments: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {},
onNodeToggleFreeResize: () => {}, onNodeToggleLocked: () => {}, onNodeCopyPrompt: () => {},
} as ToolbarHandlers;
@@ -164,7 +164,7 @@ export function CanvasToolbar({
isProjectLinked,
canUndo,
canRedo,
extractingVideoFrame: false,
extractingVideoFrames: false,
extractingAudio: false,
trimmingVideo: false,
mergingVideos: false,

View File

@@ -0,0 +1,220 @@
import { useEffect, useRef, useState, type CSSProperties } from "react";
import { App, Button, InputNumber, Modal } from "antd";
import { Check, Image as ImageIcon, SkipBack, SkipForward, Trash2 } from "lucide-react";
import { nanoid } from "nanoid";
import { canvasThemes } from "@/lib/canvas-theme";
import { formatVideoFrameTime, normalizeVideoFrameTimes } from "@/lib/canvas/canvas-video-frame";
import { resourceIdFromStorageKey } from "@/services/api/resources";
import { resolveMediaUrl } from "@/services/file-storage";
import { cacheResourceObjectUrl } from "@/services/resource-blob-cache";
import { useThemeStore } from "@/stores/use-theme-store";
import type { CanvasNodeData } from "@/types/canvas";
type SelectedVideoFrame = {
id: string;
timeMs: number;
};
export type CanvasVideoFrameParams = {
timesMs: number[];
};
type CanvasVideoFrameDialogProps = {
node: CanvasNodeData;
open: boolean;
onClose: () => void;
onConfirm: (params: CanvasVideoFrameParams) => void;
};
const MAX_SELECTED_FRAMES = 30;
export function CanvasVideoFrameDialog({ node, open, onClose, onConfirm }: CanvasVideoFrameDialogProps) {
const { message } = App.useApp();
const theme = canvasThemes[useThemeStore((state) => state.theme)];
const videoRef = useRef<HTMLVideoElement>(null);
const [videoUrl, setVideoUrl] = useState("");
const [videoError, setVideoError] = useState(false);
const [durationMs, setDurationMs] = useState(0);
const [currentTimeMs, setCurrentTimeMs] = useState(0);
const [frames, setFrames] = useState<SelectedVideoFrame[]>([]);
useEffect(() => {
if (!open) return;
let cancelled = false;
setVideoUrl("");
setVideoError(false);
setDurationMs(0);
setCurrentTimeMs(0);
setFrames([]);
const storageKey = node.metadata?.storageKey || "";
const fallback = node.metadata?.content || "";
const applyUrl = (url: string) => {
if (!cancelled) setVideoUrl(url);
};
if (resourceIdFromStorageKey(storageKey)) {
void cacheResourceObjectUrl(storageKey)
.then((cached) => {
if (cancelled) return;
if (cached) setVideoUrl(cached);
else void resolveMediaUrl(storageKey, fallback).then(applyUrl);
})
.catch(() => {
if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl);
});
} else {
void resolveMediaUrl(storageKey, fallback).then(applyUrl);
}
return () => {
cancelled = true;
};
}, [node.id, node.metadata?.content, node.metadata?.storageKey, open]);
const seekTo = (timeMs: number) => {
const normalized = normalizeVideoFrameTimes([timeMs], durationMs)[0];
if (normalized === undefined) return;
setCurrentTimeMs(normalized);
if (videoRef.current) videoRef.current.currentTime = normalized / 1000;
};
const addFrame = (timeMs: number) => {
const normalized = normalizeVideoFrameTimes([timeMs], durationMs)[0];
if (normalized === undefined) {
message.warning("视频时长未就绪,请稍候再试");
return;
}
if (frames.some((frame) => frame.timeMs === normalized)) {
message.info("该时间点已经添加");
return;
}
if (frames.length >= MAX_SELECTED_FRAMES) {
message.warning(`单次最多提取 ${MAX_SELECTED_FRAMES} 帧,请先完成当前批次`);
return;
}
setFrames((current) => [...current, { id: nanoid(), timeMs: normalized }].sort((left, right) => left.timeMs - right.timeMs));
};
const removeFrame = (id: string) => setFrames((current) => current.filter((frame) => frame.id !== id));
const lastFrameMs = Math.max(0, durationMs - 1);
const title = (
<div className="flex min-w-0 items-center gap-2.5">
<span className="grid size-8 shrink-0 place-items-center rounded-lg" style={{ background: theme.accent.primarySoft, color: theme.accent.primary }}>
<ImageIcon className="size-4" />
</span>
<div className="min-w-0">
<div className="truncate font-semibold leading-6">提取视频画面</div>
<div className="truncate text-xs opacity-45">{node.title || "视频节点"}</div>
</div>
</div>
);
return (
<Modal title={title} open={open} onCancel={onClose} footer={null} width={760} centered destroyOnHidden>
<div className="space-y-4">
<div className="flex min-h-0 items-center justify-center overflow-hidden rounded-xl bg-black">
{videoUrl ? (
<video
ref={videoRef}
src={videoUrl}
controls
playsInline
preload="metadata"
className="block max-h-[var(--video-segment-preview-max-height)] w-full"
onLoadedMetadata={(event) => {
const video = event.currentTarget;
if (!Number.isFinite(video.duration) || video.duration <= 0) return;
setDurationMs(Math.round(video.duration * 1000));
setCurrentTimeMs(Math.round(video.currentTime * 1000));
}}
onTimeUpdate={(event) => setCurrentTimeMs(Math.round(event.currentTarget.currentTime * 1000))}
onSeeked={(event) => setCurrentTimeMs(Math.round(event.currentTarget.currentTime * 1000))}
onError={() => setVideoError(true)}
/>
) : (
<div className="grid h-44 w-full place-items-center text-xs opacity-60">{videoError ? "视频预览加载失败,请检查素材是否仍然可用" : "正在加载视频…"}</div>
)}
</div>
<div className="flex flex-wrap items-center gap-2 rounded-xl px-3 py-2.5" style={{ background: theme.toolbar.itemHover }}>
<span className="text-xs font-medium opacity-60">当前时间</span>
<InputNumber
size="small"
min={0}
max={Math.max(0, lastFrameMs / 1000)}
step={0.001}
precision={3}
value={currentTimeMs / 1000}
className="w-28"
aria-label="当前取帧时间(秒)"
onChange={(value) => seekTo(Math.round((value || 0) * 1000))}
/>
<span className="mr-auto text-xs opacity-45">/ {durationMs ? formatVideoFrameTime(durationMs) : "--:--.---"}</span>
<Button
size="small"
icon={<SkipBack className="size-3.5" />}
disabled={!durationMs}
onClick={() => {
seekTo(0);
addFrame(0);
}}
>
首帧
</Button>
<Button size="small" type="primary" icon={<ImageIcon className="size-3.5" />} disabled={!durationMs} onClick={() => addFrame(currentTimeMs)}>
添加当前画面
</Button>
<Button
size="small"
icon={<SkipForward className="size-3.5" />}
disabled={!durationMs}
onClick={() => {
seekTo(lastFrameMs);
addFrame(lastFrameMs);
}}
>
尾帧
</Button>
</div>
<section aria-labelledby="selected-video-frames-title">
<div className="mb-2 flex items-center justify-between gap-3">
<div id="selected-video-frames-title" className="text-sm font-medium">
已选画面
</div>
<div className="text-xs opacity-45">
{frames.length}/{MAX_SELECTED_FRAMES}
</div>
</div>
{frames.length ? (
<div className="thin-scrollbar grid max-h-52 grid-cols-2 gap-2 overflow-y-auto pr-1 sm:grid-cols-3">
{frames.map((frame, index) => (
<div key={frame.id} className="flex min-w-0 items-center gap-2 rounded-lg px-2.5 py-2" style={{ background: theme.toolbar.itemHover }}>
<button type="button" className="min-w-0 flex-1 text-left outline-none focus-visible:ring-2" style={{ "--tw-ring-color": theme.node.activeStroke } as CSSProperties} onClick={() => seekTo(frame.timeMs)}>
<span className="block text-[var(--fs-micro)] opacity-45">画面 {index + 1}</span>
<span className="block truncate font-mono text-xs font-medium">{formatVideoFrameTime(frame.timeMs)}</span>
</button>
<Button type="text" size="small" danger icon={<Trash2 className="size-3.5" />} aria-label={`删除画面 ${index + 1}`} onClick={() => removeFrame(frame.id)} />
</div>
))}
</div>
) : (
<div className="rounded-lg px-3 py-5 text-center text-xs opacity-50" style={{ background: theme.toolbar.itemHover }}>
播放或拖动到需要的位置,然后添加当前画面。可以连续添加多个时间点。
</div>
)}
</section>
<div className="flex items-center justify-between gap-3">
<div className="text-xs opacity-45">提取后只创建图片节点,不会自动发起生成任务。</div>
<div className="flex shrink-0 items-center gap-2">
<Button onClick={onClose}>取消</Button>
<Button type="primary" icon={<Check className="size-4" />} disabled={!frames.length} onClick={() => onConfirm({ timesMs: frames.map((frame) => frame.timeMs) })}>
{frames.length ? `提取 ${frames.length} 帧` : "提取画面"}
</Button>
</div>
</div>
</div>
</Modal>
);
}

View File

@@ -1,5 +1,5 @@
import { useEffect, useMemo, useRef, useState } from "react";
import { App, Button, Input, InputNumber, Modal, Select } from "antd";
import { App, Button, Input, InputNumber, Modal, Segmented, Select } from "antd";
import { AudioLines, Check, ListVideo, Plus, Scissors, SkipBack, SkipForward, Trash2 } from "lucide-react";
import { nanoid } from "nanoid";
@@ -22,6 +22,7 @@ export type CanvasVideoSegmentItem = CanvasTimelineSegmentItem;
export type CanvasVideoSegmentParams = {
mode: "audio" | "video";
action?: "extract" | "create-generation-nodes";
startMs: number;
endMs: number;
prompt?: string;
@@ -58,6 +59,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
const [segments, setSegments] = useState<CanvasVideoSegmentItem[]>([]);
const [model, setModel] = useState("");
const [operation, setOperation] = useState<CanvasVideoEditOperation>("extend");
const [videoAction, setVideoAction] = useState<"extract" | "create-generation-nodes">("extract");
const eligibleModels = useMemo(() => listVideoReferenceModels(config), [config]);
// 打开弹窗时解析视频地址,读取时长并初始化起止时间。
@@ -71,6 +73,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
setEndSec(0);
setPrompt("");
setSegments([]);
setVideoAction("extract");
segmentsSeededRef.current = false;
const defaultModel = config.videoModel || config.model || "";
const initialModel = eligibleModels.includes(defaultModel) ? defaultModel : eligibleModels[0] || defaultModel;
@@ -108,18 +111,19 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
}, [durationMs, mode, open]);
const isVideoMode = mode === "video";
const createsGenerationNodes = videoAction === "create-generation-nodes";
const durationSec = durationMs > 0 ? durationMs / 1000 : 0;
const hasTimelineVideoClips = Boolean(timeline?.clips.some((clip) => clip.kind === "video"));
const hasPrompt = Boolean(prompt.trim());
const modelRequirements = useMemo<ModelRequirements>(
() => ({
capability: "video",
input: { textCount: hasPrompt ? 1 : 0, imageCount: 0, videoCount: 1, audioCount: 0, characterCount: 0 },
input: { textCount: createsGenerationNodes && hasPrompt ? 1 : 0, imageCount: 0, videoCount: createsGenerationNodes ? 1 : 0, audioCount: 0, characterCount: 0 },
videoOperation: operation,
videoSeconds: config.videoSeconds,
options: modelRequestOptions(config, "video"),
}),
[config.videoSeconds, hasPrompt, operation],
[config.videoSeconds, createsGenerationNodes, hasPrompt, operation],
);
const resolvedModel = resolveCompatibleModel(config, model, modelRequirements) || model;
const videoProfile = useMemo(() => (resolvedModel ? modelCapabilityConfigFor(config, resolvedModel).video : undefined), [config, resolvedModel]);
@@ -179,7 +183,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
const handleConfirm = () => {
if (isVideoMode) {
if (!model) {
if (createsGenerationNodes && !model) {
message.warning("请选择视频模型");
return;
}
@@ -195,12 +199,13 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
}
onConfirm({
mode: "video",
action: videoAction,
startMs: 0,
endMs: 0,
segments: segments.map(({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl }) => ({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl })),
model: resolvedModel,
operation,
prompt: prompt.trim(),
model: createsGenerationNodes ? resolvedModel : undefined,
operation: createsGenerationNodes ? operation : undefined,
prompt: createsGenerationNodes ? prompt.trim() : undefined,
});
return;
}
@@ -221,7 +226,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
{isVideoMode ? <Scissors className="size-4" /> : <AudioLines className="size-4" />}
</span>
<div className="min-w-0">
<div className="truncate font-semibold leading-6">{isVideoMode ? "截取片段并批量重生成" : "从视频提取声音"}</div>
<div className="truncate font-semibold leading-6">{isVideoMode ? "截取视频片段" : "从视频提取声音"}</div>
<div className="truncate text-xs opacity-45">{node.title || "视频节点"}</div>
</div>
</div>
@@ -256,7 +261,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
{isVideoMode ? (
<div className="space-y-2.5">
{!defaultModelSupported ? (
{createsGenerationNodes && !defaultModelSupported ? (
<div className="flex flex-wrap items-center justify-between gap-2 rounded-lg border px-3 py-2 text-xs" style={{ background: theme.accent.primarySoft + "1a", borderColor: theme.accent.primarySoft, color: theme.node.muted }}>
<span className="min-w-0 flex-1">
{hasEligibleModels
@@ -331,22 +336,38 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
</div>
)}
<div className="grid gap-3 md:grid-cols-2">
<label className="block min-w-0">
<div className="mb-1.5 text-sm font-medium">重生成模型</div>
<ModelPicker config={config} value={resolvedModel} onChange={setModel} capability="video" requirements={modelRequirements} fullWidth onMissingConfig={() => message.warning("请先配置支持参考视频的视频模型")} />
</label>
<label className="block min-w-0">
<div className="mb-1.5 text-sm font-medium">生成模式</div>
<Select className="w-full" size="small" value={operation} options={operationOptions} placeholder="选择生成模式" onChange={(value) => setOperation(value as CanvasVideoEditOperation)} />
</label>
<div className="rounded-xl px-3 py-2.5" style={{ background: theme.toolbar.itemHover }}>
<div className="mb-2 flex items-center justify-between gap-3">
<div>
<div className="text-sm font-medium">截取后</div>
<div className="mt-0.5 text-xs opacity-45">默认只创建片段,生成由你稍后决定。</div>
</div>
<Segmented size="small" value={videoAction} options={[{ label: "仅创建片段", value: "extract" }, { label: "创建生成节点", value: "create-generation-nodes" }]} onChange={(value) => setVideoAction(value as "extract" | "create-generation-nodes")} />
</div>
</div>
<label className="block">
<div className="mb-1.5 text-sm font-medium">生成提示词</div>
<Input.TextArea autoSize={{ minRows: 2, maxRows: 4 }} value={prompt} placeholder="描述要生成的视频内容,例如:保持画面主体与镜头,重新生成这一段视频" onChange={(event) => setPrompt(event.target.value)} />
<div className="mt-1 text-xs opacity-45">每个片段都会生成独立片段节点和新生成结果节点;片段将作为参考视频送入所选模型,时长需符合模型参考视频限制(Seedance 单段 2-15 秒)。</div>
</label>
{createsGenerationNodes ? (
<>
<div className="grid gap-3 md:grid-cols-2">
<label className="block min-w-0">
<div className="mb-1.5 text-sm font-medium">生成节点模型</div>
<ModelPicker config={config} value={resolvedModel} onChange={setModel} capability="video" requirements={modelRequirements} fullWidth onMissingConfig={() => message.warning("请先配置支持参考视频的视频模型")} />
</label>
<label className="block min-w-0">
<div className="mb-1.5 text-sm font-medium">生成模式</div>
<Select className="w-full" size="small" value={operation} options={operationOptions} placeholder="选择生成模式" onChange={(value) => setOperation(value as CanvasVideoEditOperation)} />
</label>
</div>
<label className="block">
<div className="mb-1.5 text-sm font-medium">生成提示词</div>
<Input.TextArea autoSize={{ minRows: 2, maxRows: 4 }} value={prompt} placeholder="描述后续要生成的视频内容…" onChange={(event) => setPrompt(event.target.value)} />
<div className="mt-1 text-xs opacity-45">每个片段会连接一个待生成视频节点,但不会自动提交任务。你可以继续调整参考素材、模型和提示词。</div>
</label>
</>
) : (
<div className="text-xs opacity-45">片段将作为独立视频节点创建并保持选中,不需要配置视频模型。</div>
)}
</div>
) : (
<div className="flex flex-wrap items-center gap-x-6 gap-y-2 rounded-lg border px-3 py-2 text-sm" style={{ borderColor: theme.toolbar.border }}>
@@ -401,7 +422,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
<div className="flex items-center justify-end gap-2">
<Button onClick={onClose}>取消</Button>
<Button type="primary" icon={<Check className="size-4" />} disabled={isVideoMode ? !segments.length : !durationSec} onClick={handleConfirm}>
{isVideoMode ? (segments.length > 1 ? `截取 ${segments.length} 段并生成` : "截取并生成") : "提取音频"}
{isVideoMode ? (createsGenerationNodes ? `截取 ${segments.length} 段并创建生成节点` : `截取 ${segments.length} 段`) : "提取音频"}
</Button>
</div>
</div>

View File

@@ -23,7 +23,7 @@ const settingsMockContext: ToolContext = {
isProjectLinked: false,
canUndo: false,
canRedo: false,
extractingVideoFrame: false,
extractingVideoFrames: false,
extractingAudio: false,
trimmingVideo: false,
mergingVideos: false,