feat(canvas): 视频画面提取 - 支持批量取帧并将片段截取与生成解耦
This commit is contained in:
ddCat
@@ -5,6 +5,16 @@ description: 影策已实现功能的手工验证清单与未确认边界
|
||||
|
||||
# 待测试
|
||||
|
||||
## 视频画面与片段提取
|
||||
|
||||
- [x] 视频节点“尾帧截取”已合并为“提取画面”，支持首帧、当前帧、尾帧和最多 30 个自选时间点；批量结果保留来源节点与时间点元数据。
|
||||
- [x] “截取重生成”已调整为“截取片段”，默认只创建片段素材；可选同时创建待生成节点，但不会自动提交生成任务。
|
||||
- [x] 多张提取图片会保持全部选中，可继续使用画布现有批量连接能力自行创建并连接生成节点。
|
||||
- [ ] 在真实视频节点提取首帧、多个中间帧和尾帧，确认画面、时间标记、网格布局、来源连线和多选状态正确。
|
||||
- [ ] 选择接近重复的时间点、超过 30 个时间点及部分帧解码失败场景，确认去重、上限提示和部分成功反馈正确。
|
||||
- [ ] 分别验证“只创建视频片段”和“同时创建待生成节点”，确认前者不要求模型，后者创建后不会自动扣费或启动任务。
|
||||
- [ ] 选中多张提取图片后新建视频生成节点，确认批量连线和手动生成流程可用。
|
||||
|
||||
## 画布节点搜索
|
||||
|
||||
- [x] 搜索结果改为紧凑媒体行，图片、视频、绘图优先显示真实缩略图，文本节点显示内容片段；素材类型、尺寸、时长、文件大小和模型按可用信息摘要展示。
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"dev": "vite --host 0.0.0.0 --port 3000",
|
||||
"build": "bun run build:bridge && tsc --noEmit && vite build",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
|
||||
"test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
|
||||
"test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs",
|
||||
"start": "vite preview --host 0.0.0.0 --port 3000",
|
||||
"format": "prettier --write .",
|
||||
|
||||
@@ -40,12 +40,12 @@ type CanvasNodeToolbarProps = {
|
||||
onSuperResolve: (node: CanvasNodeData) => void;
|
||||
onAngle: (node: CanvasNodeData) => void;
|
||||
onViewImage: (node: CanvasNodeData) => void;
|
||||
onExtractVideoLastFrame: (node: CanvasNodeData) => void;
|
||||
onExtractVideoFrames: (node: CanvasNodeData) => void;
|
||||
onExtractAudioFromVideo: (node: CanvasNodeData) => void;
|
||||
onTrimVideoRegenerate: (node: CanvasNodeData) => void;
|
||||
onTrimVideoSegments: (node: CanvasNodeData) => void;
|
||||
onSubtitles: (node: CanvasNodeData) => void;
|
||||
onTimeline: (node: CanvasNodeData) => void;
|
||||
extractingVideoFrame: boolean;
|
||||
extractingVideoFrames: boolean;
|
||||
extractingAudio: boolean;
|
||||
trimmingVideo: boolean;
|
||||
onReversePrompt: (node: CanvasNodeData) => void;
|
||||
@@ -103,12 +103,12 @@ export function CanvasNodeToolbar({
|
||||
onSuperResolve,
|
||||
onAngle,
|
||||
onViewImage,
|
||||
onExtractVideoLastFrame,
|
||||
onExtractVideoFrames,
|
||||
onExtractAudioFromVideo,
|
||||
onTrimVideoRegenerate,
|
||||
onTrimVideoSegments,
|
||||
onSubtitles,
|
||||
onTimeline,
|
||||
extractingVideoFrame,
|
||||
extractingVideoFrames,
|
||||
extractingAudio,
|
||||
trimmingVideo,
|
||||
onReversePrompt,
|
||||
@@ -201,7 +201,7 @@ export function CanvasNodeToolbar({
|
||||
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
|
||||
onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
|
||||
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
|
||||
onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoRegenerate: onTrimVideoRegenerate, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
|
||||
onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
|
||||
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
|
||||
} as Partial<ToolbarHandlers> as ToolbarHandlers;
|
||||
|
||||
@@ -216,7 +216,7 @@ export function CanvasNodeToolbar({
|
||||
canRedo: false,
|
||||
node,
|
||||
nodeMetadata: node.metadata,
|
||||
extractingVideoFrame,
|
||||
extractingVideoFrames,
|
||||
extractingAudio,
|
||||
trimmingVideo,
|
||||
mergingVideos: false,
|
||||
@@ -249,9 +249,9 @@ export function CanvasNodeToolbar({
|
||||
const imageEditTools = takeTools(["maskEdit", "crop", "split"]);
|
||||
const imagePortraitTools = takeTools(["emotion", "portraitTexture"]).map((tool) => tool.id === "emotion" ? { ...tool, label: "人物情绪" } : tool);
|
||||
const imageAngleTool = toolById.get("angle");
|
||||
const videoTools = takeTools(["delete", "download", "subtitles", "timeline", "extractLastFrame", "extractAudio", "trimRegenerate", "uploadVideo"]).map((tool) => {
|
||||
if (tool.id === "extractLastFrame") return { ...tool, label: "尾帧截取" };
|
||||
if (tool.id === "trimRegenerate") return { ...tool, label: "截取重生成" };
|
||||
const videoTools = takeTools(["delete", "download", "subtitles", "timeline", "extractFrames", "extractAudio", "trimRegenerate", "uploadVideo"]).map((tool) => {
|
||||
if (tool.id === "extractFrames") return { ...tool, label: "提取画面" };
|
||||
if (tool.id === "trimRegenerate") return { ...tool, label: "截取片段" };
|
||||
return tool;
|
||||
});
|
||||
const genericTools = takeTools(isAudio ? ["delete", "download", "timeline", "uploadAudio"] : isEditableText ? ["delete", "edit", "editText", "generateImage", "saveAsset"] : ["delete", "info", "config"]);
|
||||
|
||||
@@ -151,7 +151,7 @@ export function CanvasToolbar({
|
||||
onNodeInfo: () => {}, onNodeDelete: () => {}, onNodeRetry: () => {}, onNodeEditText: () => {}, onNodeDecreaseFont: () => {}, onNodeIncreaseFont: () => {},
|
||||
onNodeToggleDialog: () => {}, onNodeAnnotate: () => {}, onNodeGenerateImage: () => {}, onNodeUpload: () => {}, onNodeDownload: () => {}, onNodeSaveAsset: () => {},
|
||||
onNodeMaskEdit: () => {}, onNodeEmotion: () => {}, onNodePortraitTexture: () => {}, onNodeCrop: () => {}, onNodeSplit: () => {}, onNodeUpscale: () => {},
|
||||
onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoRegenerate: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {},
|
||||
onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoFrames: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoSegments: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {},
|
||||
onNodeToggleFreeResize: () => {}, onNodeToggleLocked: () => {}, onNodeCopyPrompt: () => {},
|
||||
} as ToolbarHandlers;
|
||||
|
||||
@@ -164,7 +164,7 @@ export function CanvasToolbar({
|
||||
isProjectLinked,
|
||||
canUndo,
|
||||
canRedo,
|
||||
extractingVideoFrame: false,
|
||||
extractingVideoFrames: false,
|
||||
extractingAudio: false,
|
||||
trimmingVideo: false,
|
||||
mergingVideos: false,
|
||||
|
||||
220
web/src/components/canvas/canvas-video-frame-dialog.tsx
Normal file
220
web/src/components/canvas/canvas-video-frame-dialog.tsx
Normal file
@@ -0,0 +1,220 @@
|
||||
import { useEffect, useRef, useState, type CSSProperties } from "react";
|
||||
import { App, Button, InputNumber, Modal } from "antd";
|
||||
import { Check, Image as ImageIcon, SkipBack, SkipForward, Trash2 } from "lucide-react";
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import { canvasThemes } from "@/lib/canvas-theme";
|
||||
import { formatVideoFrameTime, normalizeVideoFrameTimes } from "@/lib/canvas/canvas-video-frame";
|
||||
import { resourceIdFromStorageKey } from "@/services/api/resources";
|
||||
import { resolveMediaUrl } from "@/services/file-storage";
|
||||
import { cacheResourceObjectUrl } from "@/services/resource-blob-cache";
|
||||
import { useThemeStore } from "@/stores/use-theme-store";
|
||||
import type { CanvasNodeData } from "@/types/canvas";
|
||||
|
||||
type SelectedVideoFrame = {
|
||||
id: string;
|
||||
timeMs: number;
|
||||
};
|
||||
|
||||
export type CanvasVideoFrameParams = {
|
||||
timesMs: number[];
|
||||
};
|
||||
|
||||
type CanvasVideoFrameDialogProps = {
|
||||
node: CanvasNodeData;
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
onConfirm: (params: CanvasVideoFrameParams) => void;
|
||||
};
|
||||
|
||||
const MAX_SELECTED_FRAMES = 30;
|
||||
|
||||
export function CanvasVideoFrameDialog({ node, open, onClose, onConfirm }: CanvasVideoFrameDialogProps) {
|
||||
const { message } = App.useApp();
|
||||
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const [videoUrl, setVideoUrl] = useState("");
|
||||
const [videoError, setVideoError] = useState(false);
|
||||
const [durationMs, setDurationMs] = useState(0);
|
||||
const [currentTimeMs, setCurrentTimeMs] = useState(0);
|
||||
const [frames, setFrames] = useState<SelectedVideoFrame[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
let cancelled = false;
|
||||
setVideoUrl("");
|
||||
setVideoError(false);
|
||||
setDurationMs(0);
|
||||
setCurrentTimeMs(0);
|
||||
setFrames([]);
|
||||
const storageKey = node.metadata?.storageKey || "";
|
||||
const fallback = node.metadata?.content || "";
|
||||
const applyUrl = (url: string) => {
|
||||
if (!cancelled) setVideoUrl(url);
|
||||
};
|
||||
if (resourceIdFromStorageKey(storageKey)) {
|
||||
void cacheResourceObjectUrl(storageKey)
|
||||
.then((cached) => {
|
||||
if (cancelled) return;
|
||||
if (cached) setVideoUrl(cached);
|
||||
else void resolveMediaUrl(storageKey, fallback).then(applyUrl);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl);
|
||||
});
|
||||
} else {
|
||||
void resolveMediaUrl(storageKey, fallback).then(applyUrl);
|
||||
}
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [node.id, node.metadata?.content, node.metadata?.storageKey, open]);
|
||||
|
||||
const seekTo = (timeMs: number) => {
|
||||
const normalized = normalizeVideoFrameTimes([timeMs], durationMs)[0];
|
||||
if (normalized === undefined) return;
|
||||
setCurrentTimeMs(normalized);
|
||||
if (videoRef.current) videoRef.current.currentTime = normalized / 1000;
|
||||
};
|
||||
|
||||
const addFrame = (timeMs: number) => {
|
||||
const normalized = normalizeVideoFrameTimes([timeMs], durationMs)[0];
|
||||
if (normalized === undefined) {
|
||||
message.warning("视频时长未就绪，请稍候再试");
|
||||
return;
|
||||
}
|
||||
if (frames.some((frame) => frame.timeMs === normalized)) {
|
||||
message.info("该时间点已经添加");
|
||||
return;
|
||||
}
|
||||
if (frames.length >= MAX_SELECTED_FRAMES) {
|
||||
message.warning(`单次最多提取 ${MAX_SELECTED_FRAMES} 帧，请先完成当前批次`);
|
||||
return;
|
||||
}
|
||||
setFrames((current) => [...current, { id: nanoid(), timeMs: normalized }].sort((left, right) => left.timeMs - right.timeMs));
|
||||
};
|
||||
|
||||
const removeFrame = (id: string) => setFrames((current) => current.filter((frame) => frame.id !== id));
|
||||
const lastFrameMs = Math.max(0, durationMs - 1);
|
||||
|
||||
const title = (
|
||||
<div className="flex min-w-0 items-center gap-2.5">
|
||||
<span className="grid size-8 shrink-0 place-items-center rounded-lg" style={{ background: theme.accent.primarySoft, color: theme.accent.primary }}>
|
||||
<ImageIcon className="size-4" />
|
||||
</span>
|
||||
<div className="min-w-0">
|
||||
<div className="truncate font-semibold leading-6">提取视频画面</div>
|
||||
<div className="truncate text-xs opacity-45">{node.title || "视频节点"}</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<Modal title={title} open={open} onCancel={onClose} footer={null} width={760} centered destroyOnHidden>
|
||||
<div className="space-y-4">
|
||||
<div className="flex min-h-0 items-center justify-center overflow-hidden rounded-xl bg-black">
|
||||
{videoUrl ? (
|
||||
<video
|
||||
ref={videoRef}
|
||||
src={videoUrl}
|
||||
controls
|
||||
playsInline
|
||||
preload="metadata"
|
||||
className="block max-h-[var(--video-segment-preview-max-height)] w-full"
|
||||
onLoadedMetadata={(event) => {
|
||||
const video = event.currentTarget;
|
||||
if (!Number.isFinite(video.duration) || video.duration <= 0) return;
|
||||
setDurationMs(Math.round(video.duration * 1000));
|
||||
setCurrentTimeMs(Math.round(video.currentTime * 1000));
|
||||
}}
|
||||
onTimeUpdate={(event) => setCurrentTimeMs(Math.round(event.currentTarget.currentTime * 1000))}
|
||||
onSeeked={(event) => setCurrentTimeMs(Math.round(event.currentTarget.currentTime * 1000))}
|
||||
onError={() => setVideoError(true)}
|
||||
/>
|
||||
) : (
|
||||
<div className="grid h-44 w-full place-items-center text-xs opacity-60">{videoError ? "视频预览加载失败，请检查素材是否仍然可用" : "正在加载视频…"}</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2 rounded-xl px-3 py-2.5" style={{ background: theme.toolbar.itemHover }}>
|
||||
<span className="text-xs font-medium opacity-60">当前时间</span>
|
||||
<InputNumber
|
||||
size="small"
|
||||
min={0}
|
||||
max={Math.max(0, lastFrameMs / 1000)}
|
||||
step={0.001}
|
||||
precision={3}
|
||||
value={currentTimeMs / 1000}
|
||||
className="w-28"
|
||||
aria-label="当前取帧时间（秒）"
|
||||
onChange={(value) => seekTo(Math.round((value || 0) * 1000))}
|
||||
/>
|
||||
<span className="mr-auto text-xs opacity-45">/ {durationMs ? formatVideoFrameTime(durationMs) : "--:--.---"}</span>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<SkipBack className="size-3.5" />}
|
||||
disabled={!durationMs}
|
||||
onClick={() => {
|
||||
seekTo(0);
|
||||
addFrame(0);
|
||||
}}
|
||||
>
|
||||
首帧
|
||||
</Button>
|
||||
<Button size="small" type="primary" icon={<ImageIcon className="size-3.5" />} disabled={!durationMs} onClick={() => addFrame(currentTimeMs)}>
|
||||
添加当前画面
|
||||
</Button>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<SkipForward className="size-3.5" />}
|
||||
disabled={!durationMs}
|
||||
onClick={() => {
|
||||
seekTo(lastFrameMs);
|
||||
addFrame(lastFrameMs);
|
||||
}}
|
||||
>
|
||||
尾帧
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<section aria-labelledby="selected-video-frames-title">
|
||||
<div className="mb-2 flex items-center justify-between gap-3">
|
||||
<div id="selected-video-frames-title" className="text-sm font-medium">
|
||||
已选画面
|
||||
</div>
|
||||
<div className="text-xs opacity-45">
|
||||
{frames.length}/{MAX_SELECTED_FRAMES}
|
||||
</div>
|
||||
</div>
|
||||
{frames.length ? (
|
||||
<div className="thin-scrollbar grid max-h-52 grid-cols-2 gap-2 overflow-y-auto pr-1 sm:grid-cols-3">
|
||||
{frames.map((frame, index) => (
|
||||
<div key={frame.id} className="flex min-w-0 items-center gap-2 rounded-lg px-2.5 py-2" style={{ background: theme.toolbar.itemHover }}>
|
||||
<button type="button" className="min-w-0 flex-1 text-left outline-none focus-visible:ring-2" style={{ "--tw-ring-color": theme.node.activeStroke } as CSSProperties} onClick={() => seekTo(frame.timeMs)}>
|
||||
<span className="block text-[var(--fs-micro)] opacity-45">画面 {index + 1}</span>
|
||||
<span className="block truncate font-mono text-xs font-medium">{formatVideoFrameTime(frame.timeMs)}</span>
|
||||
</button>
|
||||
<Button type="text" size="small" danger icon={<Trash2 className="size-3.5" />} aria-label={`删除画面 ${index + 1}`} onClick={() => removeFrame(frame.id)} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-lg px-3 py-5 text-center text-xs opacity-50" style={{ background: theme.toolbar.itemHover }}>
|
||||
播放或拖动到需要的位置，然后添加当前画面。可以连续添加多个时间点。
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="text-xs opacity-45">提取后只创建图片节点，不会自动发起生成任务。</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<Button onClick={onClose}>取消</Button>
|
||||
<Button type="primary" icon={<Check className="size-4" />} disabled={!frames.length} onClick={() => onConfirm({ timesMs: frames.map((frame) => frame.timeMs) })}>
|
||||
{frames.length ? `提取 ${frames.length} 帧` : "提取画面"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { App, Button, Input, InputNumber, Modal, Select } from "antd";
|
||||
import { App, Button, Input, InputNumber, Modal, Segmented, Select } from "antd";
|
||||
import { AudioLines, Check, ListVideo, Plus, Scissors, SkipBack, SkipForward, Trash2 } from "lucide-react";
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
@@ -22,6 +22,7 @@ export type CanvasVideoSegmentItem = CanvasTimelineSegmentItem;
|
||||
|
||||
export type CanvasVideoSegmentParams = {
|
||||
mode: "audio" | "video";
|
||||
action?: "extract" | "create-generation-nodes";
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
prompt?: string;
|
||||
@@ -58,6 +59,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
const [segments, setSegments] = useState<CanvasVideoSegmentItem[]>([]);
|
||||
const [model, setModel] = useState("");
|
||||
const [operation, setOperation] = useState<CanvasVideoEditOperation>("extend");
|
||||
const [videoAction, setVideoAction] = useState<"extract" | "create-generation-nodes">("extract");
|
||||
const eligibleModels = useMemo(() => listVideoReferenceModels(config), [config]);
|
||||
|
||||
// 打开弹窗时解析视频地址，读取时长并初始化起止时间。
|
||||
@@ -71,6 +73,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
setEndSec(0);
|
||||
setPrompt("");
|
||||
setSegments([]);
|
||||
setVideoAction("extract");
|
||||
segmentsSeededRef.current = false;
|
||||
const defaultModel = config.videoModel || config.model || "";
|
||||
const initialModel = eligibleModels.includes(defaultModel) ? defaultModel : eligibleModels[0] || defaultModel;
|
||||
@@ -108,18 +111,19 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
}, [durationMs, mode, open]);
|
||||
|
||||
const isVideoMode = mode === "video";
|
||||
const createsGenerationNodes = videoAction === "create-generation-nodes";
|
||||
const durationSec = durationMs > 0 ? durationMs / 1000 : 0;
|
||||
const hasTimelineVideoClips = Boolean(timeline?.clips.some((clip) => clip.kind === "video"));
|
||||
const hasPrompt = Boolean(prompt.trim());
|
||||
const modelRequirements = useMemo<ModelRequirements>(
|
||||
() => ({
|
||||
capability: "video",
|
||||
input: { textCount: hasPrompt ? 1 : 0, imageCount: 0, videoCount: 1, audioCount: 0, characterCount: 0 },
|
||||
input: { textCount: createsGenerationNodes && hasPrompt ? 1 : 0, imageCount: 0, videoCount: createsGenerationNodes ? 1 : 0, audioCount: 0, characterCount: 0 },
|
||||
videoOperation: operation,
|
||||
videoSeconds: config.videoSeconds,
|
||||
options: modelRequestOptions(config, "video"),
|
||||
}),
|
||||
[config.videoSeconds, hasPrompt, operation],
|
||||
[config.videoSeconds, createsGenerationNodes, hasPrompt, operation],
|
||||
);
|
||||
const resolvedModel = resolveCompatibleModel(config, model, modelRequirements) || model;
|
||||
const videoProfile = useMemo(() => (resolvedModel ? modelCapabilityConfigFor(config, resolvedModel).video : undefined), [config, resolvedModel]);
|
||||
@@ -179,7 +183,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
|
||||
const handleConfirm = () => {
|
||||
if (isVideoMode) {
|
||||
if (!model) {
|
||||
if (createsGenerationNodes && !model) {
|
||||
message.warning("请选择视频模型");
|
||||
return;
|
||||
}
|
||||
@@ -195,12 +199,13 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
}
|
||||
onConfirm({
|
||||
mode: "video",
|
||||
action: videoAction,
|
||||
startMs: 0,
|
||||
endMs: 0,
|
||||
segments: segments.map(({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl }) => ({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl })),
|
||||
model: resolvedModel,
|
||||
operation,
|
||||
prompt: prompt.trim(),
|
||||
model: createsGenerationNodes ? resolvedModel : undefined,
|
||||
operation: createsGenerationNodes ? operation : undefined,
|
||||
prompt: createsGenerationNodes ? prompt.trim() : undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -221,7 +226,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
{isVideoMode ? <Scissors className="size-4" /> : <AudioLines className="size-4" />}
|
||||
</span>
|
||||
<div className="min-w-0">
|
||||
<div className="truncate font-semibold leading-6">{isVideoMode ? "截取片段并批量重生成" : "从视频提取声音"}</div>
|
||||
<div className="truncate font-semibold leading-6">{isVideoMode ? "截取视频片段" : "从视频提取声音"}</div>
|
||||
<div className="truncate text-xs opacity-45">{node.title || "视频节点"}</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -256,7 +261,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
|
||||
{isVideoMode ? (
|
||||
<div className="space-y-2.5">
|
||||
{!defaultModelSupported ? (
|
||||
{createsGenerationNodes && !defaultModelSupported ? (
|
||||
<div className="flex flex-wrap items-center justify-between gap-2 rounded-lg border px-3 py-2 text-xs" style={{ background: theme.accent.primarySoft + "1a", borderColor: theme.accent.primarySoft, color: theme.node.muted }}>
|
||||
<span className="min-w-0 flex-1">
|
||||
{hasEligibleModels
|
||||
@@ -331,22 +336,38 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid gap-3 md:grid-cols-2">
|
||||
<label className="block min-w-0">
|
||||
<div className="mb-1.5 text-sm font-medium">重生成模型</div>
|
||||
<ModelPicker config={config} value={resolvedModel} onChange={setModel} capability="video" requirements={modelRequirements} fullWidth onMissingConfig={() => message.warning("请先配置支持参考视频的视频模型")} />
|
||||
</label>
|
||||
<label className="block min-w-0">
|
||||
<div className="mb-1.5 text-sm font-medium">生成模式</div>
|
||||
<Select className="w-full" size="small" value={operation} options={operationOptions} placeholder="选择生成模式" onChange={(value) => setOperation(value as CanvasVideoEditOperation)} />
|
||||
</label>
|
||||
<div className="rounded-xl px-3 py-2.5" style={{ background: theme.toolbar.itemHover }}>
|
||||
<div className="mb-2 flex items-center justify-between gap-3">
|
||||
<div>
|
||||
<div className="text-sm font-medium">截取后</div>
|
||||
<div className="mt-0.5 text-xs opacity-45">默认只创建片段，生成由你稍后决定。</div>
|
||||
</div>
|
||||
<Segmented size="small" value={videoAction} options={[{ label: "仅创建片段", value: "extract" }, { label: "创建生成节点", value: "create-generation-nodes" }]} onChange={(value) => setVideoAction(value as "extract" | "create-generation-nodes")} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label className="block">
|
||||
<div className="mb-1.5 text-sm font-medium">生成提示词</div>
|
||||
<Input.TextArea autoSize={{ minRows: 2, maxRows: 4 }} value={prompt} placeholder="描述要生成的视频内容，例如：保持画面主体与镜头，重新生成这一段视频" onChange={(event) => setPrompt(event.target.value)} />
|
||||
<div className="mt-1 text-xs opacity-45">每个片段都会生成独立片段节点和新生成结果节点；片段将作为参考视频送入所选模型，时长需符合模型参考视频限制（Seedance 单段 2-15 秒）。</div>
|
||||
</label>
|
||||
{createsGenerationNodes ? (
|
||||
<>
|
||||
<div className="grid gap-3 md:grid-cols-2">
|
||||
<label className="block min-w-0">
|
||||
<div className="mb-1.5 text-sm font-medium">生成节点模型</div>
|
||||
<ModelPicker config={config} value={resolvedModel} onChange={setModel} capability="video" requirements={modelRequirements} fullWidth onMissingConfig={() => message.warning("请先配置支持参考视频的视频模型")} />
|
||||
</label>
|
||||
<label className="block min-w-0">
|
||||
<div className="mb-1.5 text-sm font-medium">生成模式</div>
|
||||
<Select className="w-full" size="small" value={operation} options={operationOptions} placeholder="选择生成模式" onChange={(value) => setOperation(value as CanvasVideoEditOperation)} />
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<label className="block">
|
||||
<div className="mb-1.5 text-sm font-medium">生成提示词</div>
|
||||
<Input.TextArea autoSize={{ minRows: 2, maxRows: 4 }} value={prompt} placeholder="描述后续要生成的视频内容…" onChange={(event) => setPrompt(event.target.value)} />
|
||||
<div className="mt-1 text-xs opacity-45">每个片段会连接一个待生成视频节点，但不会自动提交任务。你可以继续调整参考素材、模型和提示词。</div>
|
||||
</label>
|
||||
</>
|
||||
) : (
|
||||
<div className="text-xs opacity-45">片段将作为独立视频节点创建并保持选中，不需要配置视频模型。</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-wrap items-center gap-x-6 gap-y-2 rounded-lg border px-3 py-2 text-sm" style={{ borderColor: theme.toolbar.border }}>
|
||||
@@ -401,7 +422,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
|
||||
<div className="flex items-center justify-end gap-2">
|
||||
<Button onClick={onClose}>取消</Button>
|
||||
<Button type="primary" icon={<Check className="size-4" />} disabled={isVideoMode ? !segments.length : !durationSec} onClick={handleConfirm}>
|
||||
{isVideoMode ? (segments.length > 1 ? `截取 ${segments.length} 段并生成` : "截取并生成") : "提取音频"}
|
||||
{isVideoMode ? (createsGenerationNodes ? `截取 ${segments.length} 段并创建生成节点` : `截取 ${segments.length} 段`) : "提取音频"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -23,7 +23,7 @@ const settingsMockContext: ToolContext = {
|
||||
isProjectLinked: false,
|
||||
canUndo: false,
|
||||
canRedo: false,
|
||||
extractingVideoFrame: false,
|
||||
extractingVideoFrames: false,
|
||||
extractingAudio: false,
|
||||
trimmingVideo: false,
|
||||
mergingVideos: false,
|
||||
|
||||
51
web/src/lib/canvas/canvas-video-frame-nodes.ts
Normal file
51
web/src/lib/canvas/canvas-video-frame-nodes.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import { imageMetadata } from "@/lib/canvas/canvas-generation-task-sync";
|
||||
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
|
||||
import { formatVideoFrameTime } from "@/lib/canvas/canvas-video-frame";
|
||||
import type { UploadedImage } from "@/services/image-storage";
|
||||
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
|
||||
|
||||
type UploadedVideoFrame = {
|
||||
timeMs: number;
|
||||
image: UploadedImage;
|
||||
};
|
||||
|
||||
const FRAME_NODE_GAP = 24;
|
||||
const FRAME_NODE_OFFSET = 96;
|
||||
const FRAME_GRID_MAX_COLUMNS = 3;
|
||||
|
||||
export function buildVideoFrameNodes(source: CanvasNodeData, frames: UploadedVideoFrame[]): CanvasNodeData[] {
|
||||
if (!frames.length) return [];
|
||||
const sizes = frames.map(({ image }) => fitNodeSize(image.width, image.height, source.width, source.height));
|
||||
const cellWidth = Math.max(...sizes.map((size) => size.width));
|
||||
const cellHeight = Math.max(...sizes.map((size) => size.height));
|
||||
const columns = Math.min(FRAME_GRID_MAX_COLUMNS, Math.max(1, Math.ceil(Math.sqrt(frames.length))));
|
||||
const startX = source.position.x + source.width + FRAME_NODE_OFFSET;
|
||||
|
||||
return frames.map(({ timeMs, image }, index) => {
|
||||
const size = sizes[index];
|
||||
const column = index % columns;
|
||||
const row = Math.floor(index / columns);
|
||||
return {
|
||||
id: nanoid(),
|
||||
type: CanvasNodeType.Image,
|
||||
title: `画面 ${formatVideoFrameTime(timeMs)} · ${source.title || "视频"}`,
|
||||
position: {
|
||||
x: startX + column * (cellWidth + FRAME_NODE_GAP) + (cellWidth - size.width) / 2,
|
||||
y: source.position.y + row * (cellHeight + FRAME_NODE_GAP) + (cellHeight - size.height) / 2,
|
||||
},
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
metadata: {
|
||||
...imageMetadata(image),
|
||||
prompt: source.metadata?.prompt,
|
||||
workflowKind: source.metadata?.workflowKind,
|
||||
workflowTitle: source.metadata?.workflowTitle,
|
||||
shotIndex: source.metadata?.shotIndex,
|
||||
videoFrameSourceNodeId: source.id,
|
||||
videoFrameTimeMs: timeMs,
|
||||
},
|
||||
} satisfies CanvasNodeData;
|
||||
});
|
||||
}
|
||||
@@ -1,7 +1,30 @@
|
||||
const VIDEO_FRAME_TIMEOUT_MS = 20_000;
|
||||
const LAST_FRAME_EPSILON_SECONDS = 0.001;
|
||||
const LAST_FRAME_EPSILON_MS = 1;
|
||||
|
||||
export async function captureVideoLastFrame(source: Blob | string) {
|
||||
export type CapturedVideoFrame = {
|
||||
timeMs: number;
|
||||
blob: Blob;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
export type VideoFrameCaptureFailure = {
|
||||
timeMs: number;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type VideoFrameCaptureResult = {
|
||||
frames: CapturedVideoFrame[];
|
||||
failures: VideoFrameCaptureFailure[];
|
||||
};
|
||||
|
||||
export function normalizeVideoFrameTimes(timesMs: number[], durationMs: number) {
|
||||
if (!Number.isFinite(durationMs) || durationMs <= 0) return [];
|
||||
const lastFrameMs = Math.max(0, Math.round(durationMs) - LAST_FRAME_EPSILON_MS);
|
||||
return Array.from(new Set(timesMs.filter(Number.isFinite).map((timeMs) => Math.min(lastFrameMs, Math.max(0, Math.round(timeMs)))))).sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
export async function captureVideoFrames(source: Blob | string, timesMs: number[]): Promise<VideoFrameCaptureResult> {
|
||||
const blob = await readVideoBlob(source);
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const video = document.createElement("video");
|
||||
@@ -16,21 +39,32 @@ export async function captureVideoLastFrame(source: Blob | string) {
|
||||
await loaded;
|
||||
|
||||
if (!Number.isFinite(video.duration) || video.duration <= 0) throw new Error("无法确定视频时长");
|
||||
const targetTime = Math.max(0, video.duration - LAST_FRAME_EPSILON_SECONDS);
|
||||
if (targetTime > 0) {
|
||||
const seeked = waitForVideoEvent(video, "seeked", "无法定位到视频最后一帧");
|
||||
video.currentTime = targetTime;
|
||||
await seeked;
|
||||
}
|
||||
|
||||
if (!video.videoWidth || !video.videoHeight) throw new Error("无法读取视频画面尺寸");
|
||||
const normalizedTimes = normalizeVideoFrameTimes(timesMs, video.duration * 1000);
|
||||
if (!normalizedTimes.length) throw new Error("请至少选择一个有效时间点");
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("浏览器无法创建图片画布");
|
||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
return canvasToPngBlob(canvas);
|
||||
const frames: CapturedVideoFrame[] = [];
|
||||
const failures: VideoFrameCaptureFailure[] = [];
|
||||
for (const timeMs of normalizedTimes) {
|
||||
try {
|
||||
const targetSeconds = timeMs / 1000;
|
||||
if (Math.abs(video.currentTime - targetSeconds) > 0.0005) {
|
||||
const seeked = waitForVideoEvent(video, "seeked", `无法定位到 ${formatVideoFrameTime(timeMs)}`);
|
||||
video.currentTime = targetSeconds;
|
||||
await seeked;
|
||||
}
|
||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
frames.push({ timeMs, blob: await canvasToPngBlob(canvas), width: canvas.width, height: canvas.height });
|
||||
} catch (error) {
|
||||
failures.push({ timeMs, error: error instanceof Error ? error.message : "画面提取失败" });
|
||||
}
|
||||
}
|
||||
if (!frames.length) throw new Error(failures[0]?.error || "画面提取失败");
|
||||
return { frames, failures };
|
||||
} finally {
|
||||
video.pause();
|
||||
video.removeAttribute("src");
|
||||
@@ -42,14 +76,22 @@ export async function captureVideoLastFrame(source: Blob | string) {
|
||||
async function readVideoBlob(source: Blob | string) {
|
||||
if (source instanceof Blob) return source;
|
||||
try {
|
||||
const response = await fetch(source);
|
||||
const response = await fetch(source, { credentials: "same-origin" });
|
||||
if (!response.ok) throw new Error(String(response.status));
|
||||
return await response.blob();
|
||||
} catch {
|
||||
throw new Error("无法读取视频文件，请重新上传视频后再截取尾帧");
|
||||
throw new Error("无法读取视频文件，请重新上传视频后再提取画面");
|
||||
}
|
||||
}
|
||||
|
||||
export function formatVideoFrameTime(timeMs: number) {
|
||||
const totalMs = Math.max(0, Math.round(timeMs));
|
||||
const minutes = Math.floor(totalMs / 60_000);
|
||||
const seconds = Math.floor((totalMs % 60_000) / 1000);
|
||||
const milliseconds = totalMs % 1000;
|
||||
return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`;
|
||||
}
|
||||
|
||||
function waitForVideoEvent(video: HTMLVideoElement, eventName: "loadeddata" | "seeked", errorMessage: string) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let timer = 0;
|
||||
@@ -73,5 +115,5 @@ function waitForVideoEvent(video: HTMLVideoElement, eventName: "loadeddata" | "s
|
||||
}
|
||||
|
||||
function canvasToPngBlob(canvas: HTMLCanvasElement) {
|
||||
return new Promise<Blob>((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("尾帧图片编码失败"))), "image/png"));
|
||||
return new Promise<Blob>((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("画面图片编码失败"))), "image/png"));
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ export function listVideoReferenceModels(config: AiConfig): string[] {
|
||||
export function videoReferenceRegenerationError(config: AiConfig): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
if (!videoProfile || videoProfile.references.maxVideos < 1) {
|
||||
return "当前所选视频模型不支持参考视频，无法使用截取重生成。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。";
|
||||
return "当前所选视频模型不支持参考视频，无法为片段创建待生成节点。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。";
|
||||
}
|
||||
if (!videoProfile.operations.length) return "当前视频模型没有可用的视频生成模式";
|
||||
return "";
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
||||
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
||||
|
||||
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
|
||||
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
|
||||
@@ -63,17 +63,17 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
|
||||
run: (ctx) => ctx.handlers.onNodeRetry(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "extractLastFrame",
|
||||
id: "extractFrames",
|
||||
toolbar: "node-hover",
|
||||
category: "node-state",
|
||||
label: (ctx) => ctx.extractingVideoFrame ? "正在截取尾帧" : "截取尾帧",
|
||||
displayLabel: (ctx) => ctx.extractingVideoFrame ? "截取中" : "尾帧",
|
||||
icon: (ctx) => ctx.extractingVideoFrame ? <LoaderCircle className="size-3.5 animate-spin" /> : <GalleryHorizontalEnd className="size-3.5" />,
|
||||
label: (ctx) => ctx.extractingVideoFrames ? "正在提取视频画面" : "提取一个或多个视频画面",
|
||||
displayLabel: (ctx) => ctx.extractingVideoFrames ? "提取中" : "提取画面",
|
||||
icon: (ctx) => ctx.extractingVideoFrames ? <LoaderCircle className="size-3.5 animate-spin" /> : <Images className="size-3.5" />,
|
||||
defaultVisible: true,
|
||||
defaultOrder: 40,
|
||||
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
|
||||
disabled: (ctx) => ctx.extractingVideoFrame,
|
||||
run: (ctx) => ctx.handlers.onNodeExtractVideoLastFrame(ctx.node!),
|
||||
disabled: (ctx) => ctx.extractingVideoFrames,
|
||||
run: (ctx) => ctx.handlers.onNodeExtractVideoFrames(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "extractAudio",
|
||||
@@ -92,14 +92,14 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
|
||||
id: "trimRegenerate",
|
||||
toolbar: "node-hover",
|
||||
category: "node-state",
|
||||
label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "按段截取并重生成",
|
||||
displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取重生成",
|
||||
label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "截取一个或多个视频片段",
|
||||
displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取片段",
|
||||
icon: (ctx) => ctx.trimmingVideo ? <LoaderCircle className="size-3.5 animate-spin" /> : <Scissors className="size-3.5" />,
|
||||
defaultVisible: true,
|
||||
defaultOrder: 44,
|
||||
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
|
||||
disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo,
|
||||
run: (ctx) => ctx.handlers.onNodeTrimVideoRegenerate(ctx.node!),
|
||||
run: (ctx) => ctx.handlers.onNodeTrimVideoSegments(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "saveAsset",
|
||||
|
||||
@@ -89,9 +89,9 @@ export type ToolbarHandlers = {
|
||||
onNodeSuperResolve: (node: CanvasNodeData) => void;
|
||||
onNodeAngle: (node: CanvasNodeData) => void;
|
||||
onNodeViewImage: (node: CanvasNodeData) => void;
|
||||
onNodeExtractVideoLastFrame: (node: CanvasNodeData) => void;
|
||||
onNodeExtractVideoFrames: (node: CanvasNodeData) => void;
|
||||
onNodeExtractAudioFromVideo: (node: CanvasNodeData) => void;
|
||||
onNodeTrimVideoRegenerate: (node: CanvasNodeData) => void;
|
||||
onNodeTrimVideoSegments: (node: CanvasNodeData) => void;
|
||||
onNodeSubtitles: (node: CanvasNodeData) => void;
|
||||
onNodeTimeline: (node: CanvasNodeData) => void;
|
||||
onNodeReversePrompt: (node: CanvasNodeData) => void;
|
||||
@@ -114,8 +114,8 @@ export type ToolContext = {
|
||||
node?: CanvasNodeData;
|
||||
/** 便捷访问 node.metadata（node 为空时为 undefined） */
|
||||
nodeMetadata?: CanvasNodeMetadata;
|
||||
/** 视频尾帧提取中（节点悬停工具栏用） */
|
||||
extractingVideoFrame: boolean;
|
||||
/** 视频画面提取中（节点悬停工具栏用） */
|
||||
extractingVideoFrames: boolean;
|
||||
/** 视频音频提取/片段截取进行中（节点悬停工具栏用） */
|
||||
extractingAudio: boolean;
|
||||
trimmingVideo: boolean;
|
||||
|
||||
@@ -38,7 +38,7 @@ export function CanvasProjectSelectionToolbar({ anchorRef, containerRef, count,
|
||||
isProjectLinked: false,
|
||||
canUndo: false,
|
||||
canRedo: false,
|
||||
extractingVideoFrame: false,
|
||||
extractingVideoFrames: false,
|
||||
extractingAudio: false,
|
||||
trimmingVideo: false,
|
||||
mergingVideos,
|
||||
|
||||
@@ -39,6 +39,7 @@ import { resolveProjectCanvasStyle } from "@/components/canvas/canvas-style-pick
|
||||
import { createStyleProfileSnapshot, resolveStyleProfile, serializeStyleProfile } from "@/lib/canvas/style-profile";
|
||||
import { CanvasNodeToolbar, CanvasNodeInfoModal } from "@/components/canvas/canvas-node-toolbar";
|
||||
import { CanvasSubtitleDialog } from "@/components/canvas/canvas-subtitle-dialog";
|
||||
import { CanvasVideoFrameDialog } from "@/components/canvas/canvas-video-frame-dialog";
|
||||
import { CanvasVideoSegmentDialog } from "@/components/canvas/canvas-video-segment-dialog";
|
||||
import { CanvasTimelineDialog } from "@/components/canvas/canvas-timeline-dialog";
|
||||
import { syncNodeSubtitlesToTimeline } from "@/lib/timeline/timeline-build";
|
||||
@@ -801,10 +802,12 @@ function InfiniteCanvasPage() {
|
||||
openPortraitTextureEditor,
|
||||
cropImageNode,
|
||||
cropNodeId,
|
||||
closeFrameDialog,
|
||||
closeSegmentDialog,
|
||||
extractAudioFromVideo,
|
||||
extractVideoLastFrame,
|
||||
extractingVideoFrameNodeId,
|
||||
extractVideoFrames,
|
||||
extractingVideoFramesNodeId,
|
||||
frameDialogNodeId,
|
||||
generateAngleNode,
|
||||
generateEmotionNode,
|
||||
handleSegmentConfirm,
|
||||
@@ -817,6 +820,7 @@ function InfiniteCanvasPage() {
|
||||
segmentDialogMode,
|
||||
segmentDialogNodeId,
|
||||
segmentRunningMode,
|
||||
setFrameDialogNodeId,
|
||||
setSegmentDialogNodeId,
|
||||
setAngleNodeId,
|
||||
setEmotionNodeId,
|
||||
@@ -827,7 +831,8 @@ function InfiniteCanvasPage() {
|
||||
setUpscaleNodeId,
|
||||
splitImageNode,
|
||||
splitNodeId,
|
||||
trimVideoAndRegenerate,
|
||||
openVideoFrameExtractor,
|
||||
openVideoSegmentExtractor,
|
||||
upscaleImageNode,
|
||||
upscaleNodeId,
|
||||
} = useCanvasMediaTools({
|
||||
@@ -849,7 +854,6 @@ function InfiniteCanvasPage() {
|
||||
startGenerationRequest,
|
||||
finishGenerationRequest,
|
||||
bindGenerationTask,
|
||||
onGenerateVideoNode: (nodeId, mode, prompt) => generateNodeRef.current?.(nodeId, mode, prompt),
|
||||
});
|
||||
|
||||
const handleNodesDeleted = useCallback(
|
||||
@@ -863,6 +867,7 @@ function InfiniteCanvasPage() {
|
||||
setDrawingNodeId(clearDeletedId);
|
||||
setInfoNodeId(clearDeletedId);
|
||||
setSubtitleNodeId(clearDeletedId);
|
||||
setFrameDialogNodeId(clearDeletedId);
|
||||
setSegmentDialogNodeId(clearDeletedId);
|
||||
setCropNodeId(clearDeletedId);
|
||||
setMaskEditNodeId(clearDeletedId);
|
||||
@@ -886,7 +891,7 @@ function InfiniteCanvasPage() {
|
||||
}
|
||||
cleanupCanvasFiles({ projectId, nodes: nextNodes, chatSessions });
|
||||
},
|
||||
[chatSessions, cleanupCanvasFiles, message, projectId, setAngleNodeId, setAnnotationNodeId, setCropNodeId, setEmotionNodeId, setMaskEditNodeId, setSegmentDialogNodeId, setSplitNodeId, setUpscaleNodeId, setRunningNodeId],
|
||||
[chatSessions, cleanupCanvasFiles, message, projectId, setAngleNodeId, setAnnotationNodeId, setCropNodeId, setEmotionNodeId, setFrameDialogNodeId, setMaskEditNodeId, setSegmentDialogNodeId, setSplitNodeId, setUpscaleNodeId, setRunningNodeId],
|
||||
);
|
||||
|
||||
const {
|
||||
@@ -1254,6 +1259,7 @@ function InfiniteCanvasPage() {
|
||||
const dialogNode = dialogNodeId ? nodeById.get(dialogNodeId) || null : null;
|
||||
const subtitleNode = subtitleNodeId ? nodeById.get(subtitleNodeId) || null : null;
|
||||
const timelineNode = timelineNodeId ? nodeById.get(timelineNodeId) || null : null;
|
||||
const frameNode = frameDialogNodeId ? nodeById.get(frameDialogNodeId) || null : null;
|
||||
const segmentNode = segmentDialogNodeId ? nodeById.get(segmentDialogNodeId) || null : null;
|
||||
const textEditorNode = textEditorNodeId ? nodeById.get(textEditorNodeId) || null : null;
|
||||
const characterReferenceNode = characterReferenceNodeId ? nodeById.get(characterReferenceNodeId) || null : null;
|
||||
@@ -2392,12 +2398,12 @@ function InfiniteCanvasPage() {
|
||||
setAngleNodeId((current) => (current === node.id ? null : node.id));
|
||||
}}
|
||||
onViewImage={(node) => setPreviewNodeId(node.id)}
|
||||
onExtractVideoLastFrame={(node) => void extractVideoLastFrame(node)}
|
||||
onExtractVideoFrames={openVideoFrameExtractor}
|
||||
onExtractAudioFromVideo={(node) => void extractAudioFromVideo(node)}
|
||||
onTrimVideoRegenerate={(node) => void trimVideoAndRegenerate(node)}
|
||||
onTrimVideoSegments={openVideoSegmentExtractor}
|
||||
onSubtitles={(node) => setSubtitleNodeId(node.id)}
|
||||
onTimeline={(node) => setTimelineNodeId(node.id)}
|
||||
extractingVideoFrame={toolbarNode?.id === extractingVideoFrameNodeId}
|
||||
extractingVideoFrames={toolbarNode?.id === extractingVideoFramesNodeId}
|
||||
extractingAudio={segmentRunningMode === "audio"}
|
||||
trimmingVideo={segmentRunningMode === "video"}
|
||||
onReversePrompt={createImageReversePromptNodes}
|
||||
@@ -2505,6 +2511,15 @@ function InfiniteCanvasPage() {
|
||||
/>
|
||||
) : null}
|
||||
|
||||
{frameNode ? (
|
||||
<CanvasVideoFrameDialog
|
||||
node={frameNode}
|
||||
open={Boolean(frameNode)}
|
||||
onClose={closeFrameDialog}
|
||||
onConfirm={(params) => void extractVideoFrames(frameNode, params)}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
{segmentNode && segmentDialogMode ? (
|
||||
<CanvasVideoSegmentDialog
|
||||
node={segmentNode}
|
||||
|
||||
@@ -256,7 +256,7 @@ export default function SharedCanvasPage() {
|
||||
}} onHoverStart={keepToolbar} onHoverEnd={hideToolbar} onConnectStart={unauthorized} onResize={() => undefined} onContentChange={unauthorized} onRetry={unauthorized} onOpenTaskDetails={unauthorized} onViewImage={(target) => setInfoNodeId(target.id)} onContextMenu={(event, nodeId) => openContextMenu(event, nodeId)} />)}
|
||||
</InfiniteCanvas>
|
||||
|
||||
<CanvasNodeToolbar node={dragRef.current ? null : toolbarNode} viewport={viewport} containerRef={containerRef} onKeep={keepToolbar} onLeave={hideToolbar} onInfo={(node) => setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoLastFrame={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoRegenerate={unauthorized} extractingVideoFrame={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} />
|
||||
<CanvasNodeToolbar node={dragRef.current ? null : toolbarNode} viewport={viewport} containerRef={containerRef} onKeep={keepToolbar} onLeave={hideToolbar} onInfo={(node) => setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoFrames={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoSegments={unauthorized} extractingVideoFrames={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} />
|
||||
|
||||
<div className="absolute bottom-5 left-5 z-[var(--z-panel-floating)]"><CanvasZoomControls scale={viewport.k} containerRef={containerRef} onScaleChange={setZoom} onFitContent={resetViewport} isMiniMapOpen={false} onToggleMiniMap={unauthorized} onOpenShortcuts={unauthorized} /></div>
|
||||
<div className="pointer-events-none absolute bottom-5 right-5 z-[var(--z-panel-floating)] max-w-[340px] text-right text-xs leading-5" style={{ color: theme.node.muted }}>访客操作仅在当前页面临时生效</div>
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { CanvasImageSplitParams } from "@/components/canvas/canvas-node-spl
|
||||
import type { CanvasImageUpscaleParams } from "@/components/canvas/canvas-node-upscale-dialog";
|
||||
import type { CanvasImageAngleParams } from "@/components/canvas/canvas-node-angle-dialog";
|
||||
import type { CanvasImageEmotionPayload } from "@/components/canvas/canvas-node-emotion-panel";
|
||||
import type { CanvasVideoFrameParams } from "@/components/canvas/canvas-video-frame-dialog";
|
||||
import type { CanvasVideoSegmentParams } from "@/components/canvas/canvas-video-segment-dialog";
|
||||
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
|
||||
import { cropDataUrl, splitDataUrl, upscaleDataUrl } from "@/lib/canvas/canvas-image-data";
|
||||
@@ -25,7 +26,8 @@ import {
|
||||
import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
|
||||
import { compositeEmotionImage, emotionGenerationSize, emotionProviderMask, normalizeEmotionPromptForProvider, resolveEmotionEditPlan } from "@/lib/canvas/canvas-emotion";
|
||||
import { DEFAULT_PORTRAIT_TEXTURE_SETTINGS } from "@/lib/canvas/canvas-portrait-texture";
|
||||
import { captureVideoLastFrame } from "@/lib/canvas/canvas-video-frame";
|
||||
import { captureVideoFrames } from "@/lib/canvas/canvas-video-frame";
|
||||
import { buildVideoFrameNodes } from "@/lib/canvas/canvas-video-frame-nodes";
|
||||
import { mergeVideos, type MergeVideoProgress } from "@/lib/canvas/canvas-video-merge";
|
||||
import { extractVideoAudio, trimVideoSegment } from "@/lib/canvas/canvas-video-segment";
|
||||
import { generationErrorMessage } from "@/lib/generation-error";
|
||||
@@ -59,8 +61,6 @@ type UseCanvasMediaToolsOptions = {
|
||||
startGenerationRequest: (targetNodeId: string, originNodeId: string, runningId?: string, controller?: AbortController) => AbortController;
|
||||
finishGenerationRequest: (targetNodeId: string, controller: AbortController) => void;
|
||||
bindGenerationTask: (targetNodeId: string, task: GenerationTask) => void;
|
||||
/** 创建空视频节点后触发既有生成执行器（由页面层接线），用于“截取片段后调用视频模型重生成” */
|
||||
onGenerateVideoNode?: (nodeId: string, mode: "video", prompt: string) => Promise<void> | void;
|
||||
};
|
||||
|
||||
const NODE_STATUS_LOADING = "loading" as const;
|
||||
@@ -92,12 +92,11 @@ export function useCanvasMediaTools({
|
||||
startGenerationRequest,
|
||||
finishGenerationRequest,
|
||||
bindGenerationTask,
|
||||
onGenerateVideoNode,
|
||||
}: UseCanvasMediaToolsOptions) {
|
||||
const { message } = App.useApp();
|
||||
const effectiveConfig = useEffectiveConfig();
|
||||
const isAiConfigReady = useConfigStore((state) => state.isAiConfigReady);
|
||||
const extractingVideoFrameNodeIdRef = useRef<string | null>(null);
|
||||
const extractingVideoFramesNodeIdRef = useRef<string | null>(null);
|
||||
const mergeVideoRunningRef = useRef(false);
|
||||
const [cropNodeId, setCropNodeId] = useState<string | null>(null);
|
||||
const [annotationNodeId, setAnnotationNodeId] = useState<string | null>(null);
|
||||
@@ -106,7 +105,8 @@ export function useCanvasMediaTools({
|
||||
const [upscaleNodeId, setUpscaleNodeId] = useState<string | null>(null);
|
||||
const [angleNodeId, setAngleNodeId] = useState<string | null>(null);
|
||||
const [emotionNodeId, setEmotionNodeId] = useState<string | null>(null);
|
||||
const [extractingVideoFrameNodeId, setExtractingVideoFrameNodeId] = useState<string | null>(null);
|
||||
const [frameDialogNodeId, setFrameDialogNodeId] = useState<string | null>(null);
|
||||
const [extractingVideoFramesNodeId, setExtractingVideoFramesNodeId] = useState<string | null>(null);
|
||||
const [mergeVideoProgress, setMergeVideoProgress] = useState<MergeVideoProgress | null>(null);
|
||||
const [segmentDialogNodeId, setSegmentDialogNodeId] = useState<string | null>(null);
|
||||
const [segmentDialogMode, setSegmentDialogMode] = useState<"audio" | "video" | null>(null);
|
||||
@@ -200,45 +200,69 @@ export function useCanvasMediaTools({
|
||||
message.success("标注图片已保存为新节点");
|
||||
}, [message, setConnections, setDialogNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds]);
|
||||
|
||||
const extractVideoLastFrame = useCallback(async (node: CanvasNodeData) => {
|
||||
const openVideoFrameExtractor = useCallback((node: CanvasNodeData) => {
|
||||
if (!node.metadata?.content) {
|
||||
message.warning("视频节点为空，无法提取画面");
|
||||
return;
|
||||
}
|
||||
if (extractingVideoFramesNodeIdRef.current) return;
|
||||
setHoveredNodeId(null);
|
||||
setToolbarNodeId(null);
|
||||
setFrameDialogNodeId(node.id);
|
||||
}, [message, setHoveredNodeId, setToolbarNodeId]);
|
||||
|
||||
const closeFrameDialog = useCallback(() => {
|
||||
if (extractingVideoFramesNodeIdRef.current) return;
|
||||
setFrameDialogNodeId(null);
|
||||
}, []);
|
||||
|
||||
const extractVideoFrames = useCallback(async (node: CanvasNodeData, params: CanvasVideoFrameParams) => {
|
||||
const content = node.metadata?.content;
|
||||
if (!content || extractingVideoFrameNodeIdRef.current) return;
|
||||
const progress = startUploadStatus("截取视频尾帧", "读取视频资源");
|
||||
extractingVideoFrameNodeIdRef.current = node.id;
|
||||
setExtractingVideoFrameNodeId(node.id);
|
||||
if (!content || extractingVideoFramesNodeIdRef.current || !params.timesMs.length) return;
|
||||
const progress = startUploadStatus("提取视频画面", "读取视频资源", params.timesMs.length + 2);
|
||||
extractingVideoFramesNodeIdRef.current = node.id;
|
||||
setExtractingVideoFramesNodeId(node.id);
|
||||
setFrameDialogNodeId(null);
|
||||
try {
|
||||
const storedBlob = node.metadata?.storageKey ? await getMediaBlob(node.metadata.storageKey).catch(() => null) : null;
|
||||
progress.update("定位并绘制最后一帧", 2);
|
||||
const frameBlob = await captureVideoLastFrame(storedBlob || content);
|
||||
progress.update("保存尾帧图片并创建节点", 3);
|
||||
const image = await uploadImage(frameBlob);
|
||||
const size = fitNodeSize(image.width, image.height, node.width, node.height);
|
||||
const childId = nanoid();
|
||||
const child: CanvasNodeData = {
|
||||
id: childId,
|
||||
type: CanvasNodeType.Image,
|
||||
title: `尾帧 · ${node.title || "视频"}`,
|
||||
position: { x: node.position.x + node.width + 96, y: node.position.y },
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
metadata: { ...imageMetadata(image), prompt: node.metadata?.prompt, workflowKind: node.metadata?.workflowKind, workflowTitle: node.metadata?.workflowTitle, shotIndex: node.metadata?.shotIndex },
|
||||
};
|
||||
setNodes((current) => [...current, child]);
|
||||
setConnections((current) => [...current, { id: nanoid(), fromNodeId: node.id, toNodeId: childId }]);
|
||||
setSelectedNodeIds(new Set([childId]));
|
||||
progress.update("定位并绘制所选画面", 2);
|
||||
const captured = await captureVideoFrames(storedBlob || content, params.timesMs);
|
||||
const uploadedFrames = [];
|
||||
const uploadFailures: string[] = [];
|
||||
for (let index = 0; index < captured.frames.length; index += 1) {
|
||||
const frame = captured.frames[index];
|
||||
try {
|
||||
progress.update(`保存画面（${index + 1}/${captured.frames.length}）`, index + 3);
|
||||
uploadedFrames.push({ timeMs: frame.timeMs, image: await uploadImage(frame.blob) });
|
||||
} catch (error) {
|
||||
uploadFailures.push(error instanceof Error ? error.message : "画面图片上传失败");
|
||||
}
|
||||
}
|
||||
const frameNodes = buildVideoFrameNodes(node, uploadedFrames);
|
||||
if (!frameNodes.length) throw new Error(uploadFailures[0] || "画面图片保存失败");
|
||||
const links = frameNodes.map((frameNode) => ({ id: nanoid(), fromNodeId: node.id, toNodeId: frameNode.id }));
|
||||
const nextNodes = [...nodesRef.current, ...frameNodes];
|
||||
const nextConnections = [...connectionsRef.current, ...links];
|
||||
const selection = new Set(frameNodes.map((frameNode) => frameNode.id));
|
||||
nodesRef.current = nextNodes;
|
||||
connectionsRef.current = nextConnections;
|
||||
selectedNodeIdsRef.current = selection;
|
||||
setNodes(nextNodes);
|
||||
setConnections(nextConnections);
|
||||
setSelectedNodeIds(selection);
|
||||
setSelectedConnectionId(null);
|
||||
setHoveredNodeId(null);
|
||||
setToolbarNodeId(null);
|
||||
progress.done("尾帧图片已创建");
|
||||
const failedCount = captured.failures.length + uploadFailures.length;
|
||||
progress.done(failedCount ? `已提取 ${frameNodes.length} 帧，${failedCount} 帧失败` : `已提取 ${frameNodes.length} 帧并创建图片节点`);
|
||||
if (failedCount) message.warning(`${failedCount} 个时间点提取失败，其余画面已创建`);
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : "尾帧截取失败";
|
||||
const details = error instanceof Error ? error.message : "视频画面提取失败";
|
||||
progress.fail(details);
|
||||
message.error(details);
|
||||
} finally {
|
||||
extractingVideoFrameNodeIdRef.current = null;
|
||||
setExtractingVideoFrameNodeId(null);
|
||||
extractingVideoFramesNodeIdRef.current = null;
|
||||
setExtractingVideoFramesNodeId(null);
|
||||
}
|
||||
}, [message, setConnections, setHoveredNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, setToolbarNodeId, startUploadStatus]);
|
||||
}, [connectionsRef, message, nodesRef, selectedNodeIdsRef, setConnections, setNodes, setSelectedConnectionId, setSelectedNodeIds, startUploadStatus]);
|
||||
|
||||
const extractAudioFromVideo = useCallback((node: CanvasNodeData) => {
|
||||
if (!node.metadata?.content) {
|
||||
@@ -252,7 +276,7 @@ export function useCanvasMediaTools({
|
||||
setSegmentDialogMode("audio");
|
||||
}, [message, setHoveredNodeId, setToolbarNodeId]);
|
||||
|
||||
const trimVideoAndRegenerate = useCallback((node: CanvasNodeData) => {
|
||||
const openVideoSegmentExtractor = useCallback((node: CanvasNodeData) => {
|
||||
if (!node.metadata?.content) {
|
||||
message.warning("视频节点为空，无法截取片段");
|
||||
return;
|
||||
@@ -305,23 +329,26 @@ export function useCanvasMediaTools({
|
||||
}
|
||||
}, [domainProjectId, message, projectId, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]);
|
||||
|
||||
// 按段截取视频：FFmpeg 批量截取 → 每段创建片段节点与空结果节点 → 调用视频模型逐段重生成。
|
||||
const runTrimVideoAndRegenerate = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => {
|
||||
// 按段截取视频：默认只创建片段节点；用户明确选择时再附带创建待生成节点，生成任务仍由用户手动发起。
|
||||
const runTrimVideoSegments = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => {
|
||||
const segments = params.segments || [];
|
||||
if (!segments.length) {
|
||||
message.warning("请至少添加一个截取片段");
|
||||
return;
|
||||
}
|
||||
const generationConfig = buildGenerationConfig(effectiveConfig, node, "video");
|
||||
const selectedConfig = { ...generationConfig, model: params.model || generationConfig.model };
|
||||
const batchError = validateVideoSegmentBatch(selectedConfig, segments, params.operation);
|
||||
if (batchError) {
|
||||
message.warning(batchError);
|
||||
return;
|
||||
const createsGenerationNodes = params.action === "create-generation-nodes";
|
||||
const generationConfig = createsGenerationNodes ? buildGenerationConfig(effectiveConfig, node, "video") : null;
|
||||
const selectedConfig = generationConfig ? { ...generationConfig, model: params.model || generationConfig.model } : null;
|
||||
if (selectedConfig) {
|
||||
const batchError = validateVideoSegmentBatch(selectedConfig, segments, params.operation);
|
||||
if (batchError) {
|
||||
message.warning(batchError);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const progress = startUploadStatus("截取视频片段", "加载 FFmpeg", segments.length * 4);
|
||||
try {
|
||||
const prepared: Array<{ segmentNode: CanvasNodeData; targetNode: CanvasNodeData }> = [];
|
||||
const prepared: Array<{ segmentNode: CanvasNodeData; targetNode?: CanvasNodeData }> = [];
|
||||
const failedSegments: string[] = [];
|
||||
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
|
||||
const baseX = node.position.x + node.width + 96;
|
||||
@@ -354,16 +381,17 @@ export function useCanvasMediaTools({
|
||||
height: size.height,
|
||||
metadata: { ...videoMetadata(uploaded), prompt: `从「${sourceNode?.title || node.title || "视频"}」截取的片段 ${index + 1}`, status: NODE_STATUS_SUCCESS },
|
||||
};
|
||||
const targetId = nanoid();
|
||||
const targetNode: CanvasNodeData = {
|
||||
id: targetId,
|
||||
type: CanvasNodeType.Video,
|
||||
title: `重生成 ${index + 1} · ${sourceNode?.title || node.title || "视频"}`,
|
||||
position: { x: segmentNode.position.x + size.width + 96, y: segmentNode.position.y + (size.height - spec.height) / 2 },
|
||||
width: spec.width,
|
||||
height: spec.height,
|
||||
metadata: { prompt: effectivePrompt, status: "idle", generationMode: "video", model: selectedConfig.model, videoEditOperation: params.operation, seconds: generationConfig.videoSeconds, size: generationConfig.size },
|
||||
};
|
||||
const targetNode: CanvasNodeData | undefined = selectedConfig && generationConfig
|
||||
? {
|
||||
id: nanoid(),
|
||||
type: CanvasNodeType.Video,
|
||||
title: `待生成 ${index + 1} · ${sourceNode?.title || node.title || "视频"}`,
|
||||
position: { x: segmentNode.position.x + size.width + 96, y: segmentNode.position.y + (size.height - spec.height) / 2 },
|
||||
width: spec.width,
|
||||
height: spec.height,
|
||||
metadata: { prompt: effectivePrompt, status: "idle", generationMode: "video", model: selectedConfig.model, videoEditOperation: params.operation, seconds: generationConfig.videoSeconds, size: generationConfig.size },
|
||||
}
|
||||
: undefined;
|
||||
prepared.push({ segmentNode, targetNode });
|
||||
} catch (segmentError) {
|
||||
failedSegments.push(segmentError instanceof Error ? segmentError.message : "视频截取失败");
|
||||
@@ -371,46 +399,41 @@ export function useCanvasMediaTools({
|
||||
}
|
||||
if (!prepared.length) throw new Error(failedSegments[0] || "视频截取失败");
|
||||
const segmentNodes = prepared.map((item) => item.segmentNode);
|
||||
const targetNodes = prepared.map((item) => item.targetNode);
|
||||
// 先同步 ref 再 setState，保证生成执行器能立即读到新节点与连接。
|
||||
const targetNodes = prepared.flatMap((item) => item.targetNode ? [item.targetNode] : []);
|
||||
const nextNodes = [...nodesRef.current, ...segmentNodes, ...targetNodes];
|
||||
const nextConnections = [
|
||||
...connectionsRef.current,
|
||||
...prepared.flatMap((item) => [
|
||||
{ id: nanoid(), fromNodeId: node.id, toNodeId: item.segmentNode.id },
|
||||
{ id: nanoid(), fromNodeId: item.segmentNode.id, toNodeId: item.targetNode.id },
|
||||
...(item.targetNode ? [{ id: nanoid(), fromNodeId: item.segmentNode.id, toNodeId: item.targetNode.id }] : []),
|
||||
]),
|
||||
];
|
||||
nodesRef.current = nextNodes;
|
||||
connectionsRef.current = nextConnections;
|
||||
setNodes(nextNodes);
|
||||
setConnections(nextConnections);
|
||||
setSelectedNodeIds(new Set(targetNodes.map((item) => item.id)));
|
||||
const selectedNodes = targetNodes.length ? targetNodes : segmentNodes;
|
||||
const selection = new Set(selectedNodes.map((item) => item.id));
|
||||
selectedNodeIdsRef.current = selection;
|
||||
setSelectedNodeIds(selection);
|
||||
setSelectedConnectionId(null);
|
||||
progress.update("创建生成任务", segments.length * 4);
|
||||
progress.done(`已截取 ${prepared.length}/${segments.length} 段，正在调用视频模型重新生成`);
|
||||
progress.done(targetNodes.length ? `已截取 ${prepared.length}/${segments.length} 段并创建待生成节点` : `已截取 ${prepared.length}/${segments.length} 段视频`);
|
||||
segmentNodes.forEach((segmentNode) => {
|
||||
void ensureCanvasNodeAsset({ canvasId: projectId, domainProjectId, node: segmentNode, source: "canvas-manual" })
|
||||
.then((result) => setNodes((current) => current.map((item) => (item.id === segmentNode.id ? { ...item, metadata: { ...item.metadata, assetId: result.assetId } } : item))))
|
||||
.catch((assetError) => message.warning(`片段已截取并开始重生成，但素材库写入失败：${assetError instanceof Error ? assetError.message : "未知错误"}`));
|
||||
.catch((assetError) => message.warning(`片段已截取，但素材库写入失败：${assetError instanceof Error ? assetError.message : "未知错误"}`));
|
||||
});
|
||||
// 任务提交瞬时打满后端会触发限流；先错峰提交，后续可替换为批次调度的容量控制。
|
||||
for (let index = 0; index < targetNodes.length; index += 1) {
|
||||
const targetNode = targetNodes[index];
|
||||
void onGenerateVideoNode?.(targetNode.id, "video", effectivePrompt);
|
||||
if (index + 1 < targetNodes.length) await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
}
|
||||
if (failedSegments.length) message.warning(`${failedSegments.length} 段截取失败，已为其余 ${prepared.length} 段创建生成任务`);
|
||||
if (failedSegments.length) message.warning(`${failedSegments.length} 段截取失败，其余 ${prepared.length} 段已创建`);
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : "视频截取失败";
|
||||
progress.fail(details);
|
||||
message.error(details);
|
||||
}
|
||||
}, [connectionsRef, domainProjectId, effectiveConfig, message, nodesRef, onGenerateVideoNode, projectId, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]);
|
||||
}, [connectionsRef, domainProjectId, effectiveConfig, message, nodesRef, projectId, selectedNodeIdsRef, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]);
|
||||
|
||||
const handleSegmentConfirm = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => {
|
||||
if (segmentRunningRef.current || !node.metadata?.content) return;
|
||||
if (params.mode === "video") {
|
||||
if (params.mode === "video" && params.action === "create-generation-nodes") {
|
||||
const generationConfig = buildGenerationConfig(effectiveConfig, node, "video");
|
||||
const selectedConfig = { ...generationConfig, model: params.model || generationConfig.model };
|
||||
const batchError = validateVideoSegmentBatch(selectedConfig, params.segments || [], params.operation);
|
||||
@@ -424,13 +447,13 @@ export function useCanvasMediaTools({
|
||||
setSegmentDialogNodeId(null);
|
||||
setSegmentDialogMode(null);
|
||||
try {
|
||||
if (params.mode === "video") await runTrimVideoAndRegenerate(node, params);
|
||||
if (params.mode === "video") await runTrimVideoSegments(node, params);
|
||||
else await runExtractVideoAudio(node, params);
|
||||
} finally {
|
||||
segmentRunningRef.current = false;
|
||||
setSegmentRunningMode(null);
|
||||
}
|
||||
}, [effectiveConfig, message, runExtractVideoAudio, runTrimVideoAndRegenerate]);
|
||||
}, [effectiveConfig, message, runExtractVideoAudio, runTrimVideoSegments]);
|
||||
|
||||
const mergeVideosByIds = useCallback(async (videoNodeIds: string[]) => {
|
||||
if (mergeVideoRunningRef.current) return;
|
||||
@@ -683,10 +706,12 @@ export function useCanvasMediaTools({
|
||||
openPortraitTextureEditor,
|
||||
cropImageNode,
|
||||
cropNodeId,
|
||||
closeFrameDialog,
|
||||
closeSegmentDialog,
|
||||
extractAudioFromVideo,
|
||||
extractVideoLastFrame,
|
||||
extractingVideoFrameNodeId,
|
||||
extractVideoFrames,
|
||||
extractingVideoFramesNodeId,
|
||||
frameDialogNodeId,
|
||||
handleSegmentConfirm,
|
||||
generateAngleNode,
|
||||
maskEditImageNode,
|
||||
@@ -698,6 +723,7 @@ export function useCanvasMediaTools({
|
||||
segmentDialogMode,
|
||||
segmentDialogNodeId,
|
||||
segmentRunningMode,
|
||||
setFrameDialogNodeId,
|
||||
setSegmentDialogNodeId,
|
||||
setAngleNodeId,
|
||||
generateEmotionNode,
|
||||
@@ -709,7 +735,8 @@ export function useCanvasMediaTools({
|
||||
setUpscaleNodeId,
|
||||
splitImageNode,
|
||||
splitNodeId,
|
||||
trimVideoAndRegenerate,
|
||||
openVideoFrameExtractor,
|
||||
openVideoSegmentExtractor,
|
||||
upscaleImageNode,
|
||||
upscaleNodeId,
|
||||
};
|
||||
|
||||
@@ -320,6 +320,8 @@ export type CanvasNodeMetadata = {
|
||||
videoCameraMovePrompt?: string;
|
||||
videoStartFrameNodeId?: string;
|
||||
videoEndFrameNodeId?: string;
|
||||
videoFrameSourceNodeId?: string;
|
||||
videoFrameTimeMs?: number;
|
||||
versionOfNodeId?: string;
|
||||
versionLabel?: string;
|
||||
versionPrimary?: boolean;
|
||||
|
||||
43
web/test/canvas-video-frame.test.ts
Normal file
43
web/test/canvas-video-frame.test.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { buildVideoFrameNodes } from "../src/lib/canvas/canvas-video-frame-nodes";
|
||||
import { normalizeVideoFrameTimes } from "../src/lib/canvas/canvas-video-frame";
|
||||
import { CanvasNodeType, type CanvasNodeData } from "../src/types/canvas";
|
||||
|
||||
const sourceNode: CanvasNodeData = {
|
||||
id: "video-source",
|
||||
type: CanvasNodeType.Video,
|
||||
title: "测试视频",
|
||||
position: { x: 100, y: 200 },
|
||||
width: 640,
|
||||
height: 360,
|
||||
metadata: { workflowKind: "shot", workflowTitle: "镜头 1", shotIndex: 1 },
|
||||
};
|
||||
|
||||
describe("normalizeVideoFrameTimes", () => {
|
||||
test("按时间排序、去重并限制在可解码范围内", () => {
|
||||
expect(normalizeVideoFrameTimes([1500, 0, 1500.4, -20, 16000, Number.NaN], 15000)).toEqual([0, 1500, 14999]);
|
||||
});
|
||||
|
||||
test("视频时长无效时不返回时间点", () => {
|
||||
expect(normalizeVideoFrameTimes([0, 1000], 0)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildVideoFrameNodes", () => {
|
||||
test("为每个时间点创建可追溯的图片节点并保持网格布局", () => {
|
||||
const nodes = buildVideoFrameNodes(sourceNode, [
|
||||
{ timeMs: 0, image: { url: "blob:first", storageKey: "image:first", width: 1280, height: 720, bytes: 10, mimeType: "image/png" } },
|
||||
{ timeMs: 1500, image: { url: "blob:middle", storageKey: "image:middle", width: 1280, height: 720, bytes: 20, mimeType: "image/png" } },
|
||||
{ timeMs: 14999, image: { url: "blob:last", storageKey: "image:last", width: 1280, height: 720, bytes: 30, mimeType: "image/png" } },
|
||||
]);
|
||||
|
||||
expect(nodes).toHaveLength(3);
|
||||
expect(nodes.map((node) => node.metadata?.videoFrameTimeMs)).toEqual([0, 1500, 14999]);
|
||||
expect(nodes.every((node) => node.metadata?.videoFrameSourceNodeId === sourceNode.id)).toBe(true);
|
||||
expect(nodes.every((node) => node.metadata?.workflowKind === "shot" && node.metadata?.shotIndex === 1)).toBe(true);
|
||||
expect(nodes[0].position.x).toBe(sourceNode.position.x + sourceNode.width + 96);
|
||||
expect(nodes[1].position.x).toBeGreaterThan(nodes[0].position.x);
|
||||
expect(nodes[2].position.y).toBeGreaterThan(nodes[0].position.y);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user