diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index e01dac2e9..a490cc2e6 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -5,6 +5,16 @@ description: 影策已实现功能的手工验证清单与未确认边界 # 待测试 +## 视频画面与片段提取 + +- [x] 视频节点“尾帧截取”已合并为“提取画面”，支持首帧、当前帧、尾帧和最多 30 个自选时间点；批量结果保留来源节点与时间点元数据。 +- [x] “截取重生成”已调整为“截取片段”，默认只创建片段素材；可选同时创建待生成节点，但不会自动提交生成任务。 +- [x] 多张提取图片会保持全部选中，可继续使用画布现有批量连接能力自行创建并连接生成节点。 +- [ ] 在真实视频节点提取首帧、多个中间帧和尾帧，确认画面、时间标记、网格布局、来源连线和多选状态正确。 +- [ ] 选择接近重复的时间点、超过 30 个时间点及部分帧解码失败场景，确认去重、上限提示和部分成功反馈正确。 +- [ ] 分别验证“只创建视频片段”和“同时创建待生成节点”，确认前者不要求模型，后者创建后不会自动扣费或启动任务。 +- [ ] 选中多张提取图片后新建视频生成节点，确认批量连线和手动生成流程可用。 + ## 画布节点搜索 - [x] 搜索结果改为紧凑媒体行，图片、视频、绘图优先显示真实缩略图，文本节点显示内容片段；素材类型、尺寸、时长、文件大小和模型按可用信息摘要展示。 diff --git a/web/package.json b/web/package.json index f8dd53359..1197d551d 100644 --- a/web/package.json +++ b/web/package.json @@ -8,7 +8,7 @@ "dev": "vite --host 0.0.0.0 --port 3000", "build": "bun run build:bridge && tsc --noEmit && vite build", "typecheck": "tsc --noEmit", - "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", + "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", "test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", diff --git a/web/src/components/canvas/canvas-node-toolbar.tsx b/web/src/components/canvas/canvas-node-toolbar.tsx index cc0aed9b5..87ab4e1d4 100644 --- a/web/src/components/canvas/canvas-node-toolbar.tsx +++ b/web/src/components/canvas/canvas-node-toolbar.tsx @@ -40,12 +40,12 @@ type CanvasNodeToolbarProps = { onSuperResolve: (node: CanvasNodeData) => void; onAngle: (node: CanvasNodeData) => void; onViewImage: (node: CanvasNodeData) => void; - onExtractVideoLastFrame: (node: CanvasNodeData) => void; + onExtractVideoFrames: (node: CanvasNodeData) => void; onExtractAudioFromVideo: (node: CanvasNodeData) => void; - onTrimVideoRegenerate: (node: CanvasNodeData) => void; + onTrimVideoSegments: (node: CanvasNodeData) => void; onSubtitles: (node: CanvasNodeData) => void; onTimeline: (node: CanvasNodeData) => void; - extractingVideoFrame: boolean; + extractingVideoFrames: boolean; extractingAudio: boolean; trimmingVideo: boolean; onReversePrompt: (node: CanvasNodeData) => void; @@ -103,12 +103,12 @@ export function CanvasNodeToolbar({ onSuperResolve, onAngle, onViewImage, - onExtractVideoLastFrame, + onExtractVideoFrames, onExtractAudioFromVideo, - onTrimVideoRegenerate, + onTrimVideoSegments, onSubtitles, onTimeline, - extractingVideoFrame, + extractingVideoFrames, extractingAudio, trimmingVideo, onReversePrompt, @@ -201,7 +201,7 @@ export function CanvasNodeToolbar({ onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload, onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop, onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage, - onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoRegenerate: onTrimVideoRegenerate, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, + onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt, } as Partial as ToolbarHandlers; @@ -216,7 +216,7 @@ export function CanvasNodeToolbar({ canRedo: false, node, nodeMetadata: node.metadata, - extractingVideoFrame, + extractingVideoFrames, extractingAudio, trimmingVideo, mergingVideos: false, @@ -249,9 +249,9 @@ export function CanvasNodeToolbar({ const imageEditTools = takeTools(["maskEdit", "crop", "split"]); const imagePortraitTools = takeTools(["emotion", "portraitTexture"]).map((tool) => tool.id === "emotion" ? { ...tool, label: "人物情绪" } : tool); const imageAngleTool = toolById.get("angle"); - const videoTools = takeTools(["delete", "download", "subtitles", "timeline", "extractLastFrame", "extractAudio", "trimRegenerate", "uploadVideo"]).map((tool) => { - if (tool.id === "extractLastFrame") return { ...tool, label: "尾帧截取" }; - if (tool.id === "trimRegenerate") return { ...tool, label: "截取重生成" }; + const videoTools = takeTools(["delete", "download", "subtitles", "timeline", "extractFrames", "extractAudio", "trimRegenerate", "uploadVideo"]).map((tool) => { + if (tool.id === "extractFrames") return { ...tool, label: "提取画面" }; + if (tool.id === "trimRegenerate") return { ...tool, label: "截取片段" }; return tool; }); const genericTools = takeTools(isAudio ? ["delete", "download", "timeline", "uploadAudio"] : isEditableText ? ["delete", "edit", "editText", "generateImage", "saveAsset"] : ["delete", "info", "config"]); diff --git a/web/src/components/canvas/canvas-toolbar.tsx b/web/src/components/canvas/canvas-toolbar.tsx index 9ea3be52a..749d217f6 100644 --- a/web/src/components/canvas/canvas-toolbar.tsx +++ b/web/src/components/canvas/canvas-toolbar.tsx @@ -151,7 +151,7 @@ export function CanvasToolbar({ onNodeInfo: () => {}, onNodeDelete: () => {}, onNodeRetry: () => {}, onNodeEditText: () => {}, onNodeDecreaseFont: () => {}, onNodeIncreaseFont: () => {}, onNodeToggleDialog: () => {}, onNodeAnnotate: () => {}, onNodeGenerateImage: () => {}, onNodeUpload: () => {}, onNodeDownload: () => {}, onNodeSaveAsset: () => {}, onNodeMaskEdit: () => {}, onNodeEmotion: () => {}, onNodePortraitTexture: () => {}, onNodeCrop: () => {}, onNodeSplit: () => {}, onNodeUpscale: () => {}, - onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoRegenerate: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {}, + onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoFrames: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoSegments: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {}, onNodeToggleFreeResize: () => {}, onNodeToggleLocked: () => {}, onNodeCopyPrompt: () => {}, } as ToolbarHandlers; @@ -164,7 +164,7 @@ export function CanvasToolbar({ isProjectLinked, canUndo, canRedo, - extractingVideoFrame: false, + extractingVideoFrames: false, extractingAudio: false, trimmingVideo: false, mergingVideos: false, diff --git a/web/src/components/canvas/canvas-video-frame-dialog.tsx b/web/src/components/canvas/canvas-video-frame-dialog.tsx new file mode 100644 index 000000000..2f9021268 --- /dev/null +++ b/web/src/components/canvas/canvas-video-frame-dialog.tsx @@ -0,0 +1,220 @@ +import { useEffect, useRef, useState, type CSSProperties } from "react"; +import { App, Button, InputNumber, Modal } from "antd"; +import { Check, Image as ImageIcon, SkipBack, SkipForward, Trash2 } from "lucide-react"; +import { nanoid } from "nanoid"; + +import { canvasThemes } from "@/lib/canvas-theme"; +import { formatVideoFrameTime, normalizeVideoFrameTimes } from "@/lib/canvas/canvas-video-frame"; +import { resourceIdFromStorageKey } from "@/services/api/resources"; +import { resolveMediaUrl } from "@/services/file-storage"; +import { cacheResourceObjectUrl } from "@/services/resource-blob-cache"; +import { useThemeStore } from "@/stores/use-theme-store"; +import type { CanvasNodeData } from "@/types/canvas"; + +type SelectedVideoFrame = { + id: string; + timeMs: number; +}; + +export type CanvasVideoFrameParams = { + timesMs: number[]; +}; + +type CanvasVideoFrameDialogProps = { + node: CanvasNodeData; + open: boolean; + onClose: () => void; + onConfirm: (params: CanvasVideoFrameParams) => void; +}; + +const MAX_SELECTED_FRAMES = 30; + +export function CanvasVideoFrameDialog({ node, open, onClose, onConfirm }: CanvasVideoFrameDialogProps) { + const { message } = App.useApp(); + const theme = canvasThemes[useThemeStore((state) => state.theme)]; + const videoRef = useRef(null); + const [videoUrl, setVideoUrl] = useState(""); + const [videoError, setVideoError] = useState(false); + const [durationMs, setDurationMs] = useState(0); + const [currentTimeMs, setCurrentTimeMs] = useState(0); + const [frames, setFrames] = useState([]); + + useEffect(() => { + if (!open) return; + let cancelled = false; + setVideoUrl(""); + setVideoError(false); + setDurationMs(0); + setCurrentTimeMs(0); + setFrames([]); + const storageKey = node.metadata?.storageKey || ""; + const fallback = node.metadata?.content || ""; + const applyUrl = (url: string) => { + if (!cancelled) setVideoUrl(url); + }; + if (resourceIdFromStorageKey(storageKey)) { + void cacheResourceObjectUrl(storageKey) + .then((cached) => { + if (cancelled) return; + if (cached) setVideoUrl(cached); + else void resolveMediaUrl(storageKey, fallback).then(applyUrl); + }) + .catch(() => { + if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl); + }); + } else { + void resolveMediaUrl(storageKey, fallback).then(applyUrl); + } + return () => { + cancelled = true; + }; + }, [node.id, node.metadata?.content, node.metadata?.storageKey, open]); + + const seekTo = (timeMs: number) => { + const normalized = normalizeVideoFrameTimes([timeMs], durationMs)[0]; + if (normalized === undefined) return; + setCurrentTimeMs(normalized); + if (videoRef.current) videoRef.current.currentTime = normalized / 1000; + }; + + const addFrame = (timeMs: number) => { + const normalized = normalizeVideoFrameTimes([timeMs], durationMs)[0]; + if (normalized === undefined) { + message.warning("视频时长未就绪，请稍候再试"); + return; + } + if (frames.some((frame) => frame.timeMs === normalized)) { + message.info("该时间点已经添加"); + return; + } + if (frames.length >= MAX_SELECTED_FRAMES) { + message.warning(`单次最多提取 ${MAX_SELECTED_FRAMES} 帧，请先完成当前批次`); + return; + } + setFrames((current) => [...current, { id: nanoid(), timeMs: normalized }].sort((left, right) => left.timeMs - right.timeMs)); + }; + + const removeFrame = (id: string) => setFrames((current) => current.filter((frame) => frame.id !== id)); + const lastFrameMs = Math.max(0, durationMs - 1); + + const title = ( +
+ + + +
+
提取视频画面
+
{node.title || "视频节点"}
+
+
+ ); + + return ( + +
+
+ {videoUrl ? ( +
+ +
+ 当前时间 + seekTo(Math.round((value || 0) * 1000))} + /> + / {durationMs ? formatVideoFrameTime(durationMs) : "--:--.---"} + + + +
+ +
+
+
+ 已选画面 +
+
+ {frames.length}/{MAX_SELECTED_FRAMES} +
+
+ {frames.length ? ( +
+ {frames.map((frame, index) => ( +
+ +
+ ))} +
+ ) : ( +
+ 播放或拖动到需要的位置，然后添加当前画面。可以连续添加多个时间点。 +
+ )} +
+ +
+
提取后只创建图片节点，不会自动发起生成任务。
+
+ + +
+
+
+ + ); +} diff --git a/web/src/components/canvas/canvas-video-segment-dialog.tsx b/web/src/components/canvas/canvas-video-segment-dialog.tsx index dc3a4896f..ad5b9b73e 100644 --- a/web/src/components/canvas/canvas-video-segment-dialog.tsx +++ b/web/src/components/canvas/canvas-video-segment-dialog.tsx @@ -1,5 +1,5 @@ import { useEffect, useMemo, useRef, useState } from "react"; -import { App, Button, Input, InputNumber, Modal, Select } from "antd"; +import { App, Button, Input, InputNumber, Modal, Segmented, Select } from "antd"; import { AudioLines, Check, ListVideo, Plus, Scissors, SkipBack, SkipForward, Trash2 } from "lucide-react"; import { nanoid } from "nanoid"; @@ -22,6 +22,7 @@ export type CanvasVideoSegmentItem = CanvasTimelineSegmentItem; export type CanvasVideoSegmentParams = { mode: "audio" | "video"; + action?: "extract" | "create-generation-nodes"; startMs: number; endMs: number; prompt?: string; @@ -58,6 +59,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, const [segments, setSegments] = useState([]); const [model, setModel] = useState(""); const [operation, setOperation] = useState("extend"); + const [videoAction, setVideoAction] = useState<"extract" | "create-generation-nodes">("extract"); const eligibleModels = useMemo(() => listVideoReferenceModels(config), [config]); // 打开弹窗时解析视频地址，读取时长并初始化起止时间。 @@ -71,6 +73,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, setEndSec(0); setPrompt(""); setSegments([]); + setVideoAction("extract"); segmentsSeededRef.current = false; const defaultModel = config.videoModel || config.model || ""; const initialModel = eligibleModels.includes(defaultModel) ? defaultModel : eligibleModels[0] || defaultModel; @@ -108,18 +111,19 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, }, [durationMs, mode, open]); const isVideoMode = mode === "video"; + const createsGenerationNodes = videoAction === "create-generation-nodes"; const durationSec = durationMs > 0 ? durationMs / 1000 : 0; const hasTimelineVideoClips = Boolean(timeline?.clips.some((clip) => clip.kind === "video")); const hasPrompt = Boolean(prompt.trim()); const modelRequirements = useMemo( () => ({ capability: "video", - input: { textCount: hasPrompt ? 1 : 0, imageCount: 0, videoCount: 1, audioCount: 0, characterCount: 0 }, + input: { textCount: createsGenerationNodes && hasPrompt ? 1 : 0, imageCount: 0, videoCount: createsGenerationNodes ? 1 : 0, audioCount: 0, characterCount: 0 }, videoOperation: operation, videoSeconds: config.videoSeconds, options: modelRequestOptions(config, "video"), }), - [config.videoSeconds, hasPrompt, operation], + [config.videoSeconds, createsGenerationNodes, hasPrompt, operation], ); const resolvedModel = resolveCompatibleModel(config, model, modelRequirements) || model; const videoProfile = useMemo(() => (resolvedModel ? modelCapabilityConfigFor(config, resolvedModel).video : undefined), [config, resolvedModel]); @@ -179,7 +183,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, const handleConfirm = () => { if (isVideoMode) { - if (!model) { + if (createsGenerationNodes && !model) { message.warning("请选择视频模型"); return; } @@ -195,12 +199,13 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, } onConfirm({ mode: "video", + action: videoAction, startMs: 0, endMs: 0, segments: segments.map(({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl }) => ({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl })), - model: resolvedModel, - operation, - prompt: prompt.trim(), + model: createsGenerationNodes ? resolvedModel : undefined, + operation: createsGenerationNodes ? operation : undefined, + prompt: createsGenerationNodes ? prompt.trim() : undefined, }); return; } @@ -221,7 +226,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, {isVideoMode ? : }
-
{isVideoMode ? "截取片段并批量重生成" : "从视频提取声音"}
+
{isVideoMode ? "截取视频片段" : "从视频提取声音"}
{node.title || "视频节点"}
@@ -256,7 +261,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, {isVideoMode ? (
- {!defaultModelSupported ? ( + {createsGenerationNodes && !defaultModelSupported ? (
{hasEligibleModels @@ -331,22 +336,38 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
)} -
- - +
+ + + + ) : ( +
片段将作为独立视频节点创建并保持选中，不需要配置视频模型。
+ )}
) : (
@@ -401,7 +422,7 @@ export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode,
diff --git a/web/src/components/canvas/toolbars/toolbar-settings-modal.tsx b/web/src/components/canvas/toolbars/toolbar-settings-modal.tsx index d7487b29f..8a0e542ab 100644 --- a/web/src/components/canvas/toolbars/toolbar-settings-modal.tsx +++ b/web/src/components/canvas/toolbars/toolbar-settings-modal.tsx @@ -23,7 +23,7 @@ const settingsMockContext: ToolContext = { isProjectLinked: false, canUndo: false, canRedo: false, - extractingVideoFrame: false, + extractingVideoFrames: false, extractingAudio: false, trimmingVideo: false, mergingVideos: false, diff --git a/web/src/lib/canvas/canvas-video-frame-nodes.ts b/web/src/lib/canvas/canvas-video-frame-nodes.ts new file mode 100644 index 000000000..bc0c77d80 --- /dev/null +++ b/web/src/lib/canvas/canvas-video-frame-nodes.ts @@ -0,0 +1,51 @@ +import { nanoid } from "nanoid"; + +import { imageMetadata } from "@/lib/canvas/canvas-generation-task-sync"; +import { fitNodeSize } from "@/lib/canvas/canvas-node-size"; +import { formatVideoFrameTime } from "@/lib/canvas/canvas-video-frame"; +import type { UploadedImage } from "@/services/image-storage"; +import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas"; + +type UploadedVideoFrame = { + timeMs: number; + image: UploadedImage; +}; + +const FRAME_NODE_GAP = 24; +const FRAME_NODE_OFFSET = 96; +const FRAME_GRID_MAX_COLUMNS = 3; + +export function buildVideoFrameNodes(source: CanvasNodeData, frames: UploadedVideoFrame[]): CanvasNodeData[] { + if (!frames.length) return []; + const sizes = frames.map(({ image }) => fitNodeSize(image.width, image.height, source.width, source.height)); + const cellWidth = Math.max(...sizes.map((size) => size.width)); + const cellHeight = Math.max(...sizes.map((size) => size.height)); + const columns = Math.min(FRAME_GRID_MAX_COLUMNS, Math.max(1, Math.ceil(Math.sqrt(frames.length)))); + const startX = source.position.x + source.width + FRAME_NODE_OFFSET; + + return frames.map(({ timeMs, image }, index) => { + const size = sizes[index]; + const column = index % columns; + const row = Math.floor(index / columns); + return { + id: nanoid(), + type: CanvasNodeType.Image, + title: `画面 ${formatVideoFrameTime(timeMs)} · ${source.title || "视频"}`, + position: { + x: startX + column * (cellWidth + FRAME_NODE_GAP) + (cellWidth - size.width) / 2, + y: source.position.y + row * (cellHeight + FRAME_NODE_GAP) + (cellHeight - size.height) / 2, + }, + width: size.width, + height: size.height, + metadata: { + ...imageMetadata(image), + prompt: source.metadata?.prompt, + workflowKind: source.metadata?.workflowKind, + workflowTitle: source.metadata?.workflowTitle, + shotIndex: source.metadata?.shotIndex, + videoFrameSourceNodeId: source.id, + videoFrameTimeMs: timeMs, + }, + } satisfies CanvasNodeData; + }); +} diff --git a/web/src/lib/canvas/canvas-video-frame.ts b/web/src/lib/canvas/canvas-video-frame.ts index d01c151bb..1a2999afd 100644 --- a/web/src/lib/canvas/canvas-video-frame.ts +++ b/web/src/lib/canvas/canvas-video-frame.ts @@ -1,7 +1,30 @@ const VIDEO_FRAME_TIMEOUT_MS = 20_000; -const LAST_FRAME_EPSILON_SECONDS = 0.001; +const LAST_FRAME_EPSILON_MS = 1; -export async function captureVideoLastFrame(source: Blob | string) { +export type CapturedVideoFrame = { + timeMs: number; + blob: Blob; + width: number; + height: number; +}; + +export type VideoFrameCaptureFailure = { + timeMs: number; + error: string; +}; + +export type VideoFrameCaptureResult = { + frames: CapturedVideoFrame[]; + failures: VideoFrameCaptureFailure[]; +}; + +export function normalizeVideoFrameTimes(timesMs: number[], durationMs: number) { + if (!Number.isFinite(durationMs) || durationMs <= 0) return []; + const lastFrameMs = Math.max(0, Math.round(durationMs) - LAST_FRAME_EPSILON_MS); + return Array.from(new Set(timesMs.filter(Number.isFinite).map((timeMs) => Math.min(lastFrameMs, Math.max(0, Math.round(timeMs)))))).sort((left, right) => left - right); +} + +export async function captureVideoFrames(source: Blob | string, timesMs: number[]): Promise { const blob = await readVideoBlob(source); const objectUrl = URL.createObjectURL(blob); const video = document.createElement("video"); @@ -16,21 +39,32 @@ export async function captureVideoLastFrame(source: Blob | string) { await loaded; if (!Number.isFinite(video.duration) || video.duration <= 0) throw new Error("无法确定视频时长"); - const targetTime = Math.max(0, video.duration - LAST_FRAME_EPSILON_SECONDS); - if (targetTime > 0) { - const seeked = waitForVideoEvent(video, "seeked", "无法定位到视频最后一帧"); - video.currentTime = targetTime; - await seeked; - } - if (!video.videoWidth || !video.videoHeight) throw new Error("无法读取视频画面尺寸"); + const normalizedTimes = normalizeVideoFrameTimes(timesMs, video.duration * 1000); + if (!normalizedTimes.length) throw new Error("请至少选择一个有效时间点"); const canvas = document.createElement("canvas"); canvas.width = video.videoWidth; canvas.height = video.videoHeight; const context = canvas.getContext("2d"); if (!context) throw new Error("浏览器无法创建图片画布"); - context.drawImage(video, 0, 0, canvas.width, canvas.height); - return canvasToPngBlob(canvas); + const frames: CapturedVideoFrame[] = []; + const failures: VideoFrameCaptureFailure[] = []; + for (const timeMs of normalizedTimes) { + try { + const targetSeconds = timeMs / 1000; + if (Math.abs(video.currentTime - targetSeconds) > 0.0005) { + const seeked = waitForVideoEvent(video, "seeked", `无法定位到 ${formatVideoFrameTime(timeMs)}`); + video.currentTime = targetSeconds; + await seeked; + } + context.drawImage(video, 0, 0, canvas.width, canvas.height); + frames.push({ timeMs, blob: await canvasToPngBlob(canvas), width: canvas.width, height: canvas.height }); + } catch (error) { + failures.push({ timeMs, error: error instanceof Error ? error.message : "画面提取失败" }); + } + } + if (!frames.length) throw new Error(failures[0]?.error || "画面提取失败"); + return { frames, failures }; } finally { video.pause(); video.removeAttribute("src"); @@ -42,14 +76,22 @@ export async function captureVideoLastFrame(source: Blob | string) { async function readVideoBlob(source: Blob | string) { if (source instanceof Blob) return source; try { - const response = await fetch(source); + const response = await fetch(source, { credentials: "same-origin" }); if (!response.ok) throw new Error(String(response.status)); return await response.blob(); } catch { - throw new Error("无法读取视频文件，请重新上传视频后再截取尾帧"); + throw new Error("无法读取视频文件，请重新上传视频后再提取画面"); } } +export function formatVideoFrameTime(timeMs: number) { + const totalMs = Math.max(0, Math.round(timeMs)); + const minutes = Math.floor(totalMs / 60_000); + const seconds = Math.floor((totalMs % 60_000) / 1000); + const milliseconds = totalMs % 1000; + return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`; +} + function waitForVideoEvent(video: HTMLVideoElement, eventName: "loadeddata" | "seeked", errorMessage: string) { return new Promise((resolve, reject) => { let timer = 0; @@ -73,5 +115,5 @@ function waitForVideoEvent(video: HTMLVideoElement, eventName: "loadeddata" | "s } function canvasToPngBlob(canvas: HTMLCanvasElement) { - return new Promise((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("尾帧图片编码失败"))), "image/png")); + return new Promise((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("画面图片编码失败"))), "image/png")); } diff --git a/web/src/lib/canvas/canvas-video-regeneration.ts b/web/src/lib/canvas/canvas-video-regeneration.ts index 112a30d01..6c4e6c768 100644 --- a/web/src/lib/canvas/canvas-video-regeneration.ts +++ b/web/src/lib/canvas/canvas-video-regeneration.ts @@ -15,7 +15,7 @@ export function listVideoReferenceModels(config: AiConfig): string[] { export function videoReferenceRegenerationError(config: AiConfig): string { const videoProfile = modelCapabilityConfigFor(config, config.model).video; if (!videoProfile || videoProfile.references.maxVideos < 1) { - return "当前所选视频模型不支持参考视频，无法使用截取重生成。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。"; + return "当前所选视频模型不支持参考视频，无法为片段创建待生成节点。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。"; } if (!videoProfile.operations.length) return "当前视频模型没有可用的视频生成模式"; return ""; diff --git a/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx b/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx index e98fe808f..15178a83b 100644 --- a/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx +++ b/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx @@ -1,4 +1,4 @@ -import { AudioLines, Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react"; +import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react"; import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error"; import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry"; @@ -63,17 +63,17 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [ run: (ctx) => ctx.handlers.onNodeRetry(ctx.node!), }, { - id: "extractLastFrame", + id: "extractFrames", toolbar: "node-hover", category: "node-state", - label: (ctx) => ctx.extractingVideoFrame ? "正在截取尾帧" : "截取尾帧", - displayLabel: (ctx) => ctx.extractingVideoFrame ? "截取中" : "尾帧", - icon: (ctx) => ctx.extractingVideoFrame ? : , + label: (ctx) => ctx.extractingVideoFrames ? "正在提取视频画面" : "提取一个或多个视频画面", + displayLabel: (ctx) => ctx.extractingVideoFrames ? "提取中" : "提取画面", + icon: (ctx) => ctx.extractingVideoFrames ? : , defaultVisible: true, defaultOrder: 40, applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx), - disabled: (ctx) => ctx.extractingVideoFrame, - run: (ctx) => ctx.handlers.onNodeExtractVideoLastFrame(ctx.node!), + disabled: (ctx) => ctx.extractingVideoFrames, + run: (ctx) => ctx.handlers.onNodeExtractVideoFrames(ctx.node!), }, { id: "extractAudio", @@ -92,14 +92,14 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [ id: "trimRegenerate", toolbar: "node-hover", category: "node-state", - label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "按段截取并重生成", - displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取重生成", + label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "截取一个或多个视频片段", + displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取片段", icon: (ctx) => ctx.trimmingVideo ? : , defaultVisible: true, defaultOrder: 44, applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx), disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo, - run: (ctx) => ctx.handlers.onNodeTrimVideoRegenerate(ctx.node!), + run: (ctx) => ctx.handlers.onNodeTrimVideoSegments(ctx.node!), }, { id: "saveAsset", diff --git a/web/src/lib/canvas/tool-registry/tool-definition.ts b/web/src/lib/canvas/tool-registry/tool-definition.ts index ea9a0dab5..3292a5cd9 100644 --- a/web/src/lib/canvas/tool-registry/tool-definition.ts +++ b/web/src/lib/canvas/tool-registry/tool-definition.ts @@ -89,9 +89,9 @@ export type ToolbarHandlers = { onNodeSuperResolve: (node: CanvasNodeData) => void; onNodeAngle: (node: CanvasNodeData) => void; onNodeViewImage: (node: CanvasNodeData) => void; - onNodeExtractVideoLastFrame: (node: CanvasNodeData) => void; + onNodeExtractVideoFrames: (node: CanvasNodeData) => void; onNodeExtractAudioFromVideo: (node: CanvasNodeData) => void; - onNodeTrimVideoRegenerate: (node: CanvasNodeData) => void; + onNodeTrimVideoSegments: (node: CanvasNodeData) => void; onNodeSubtitles: (node: CanvasNodeData) => void; onNodeTimeline: (node: CanvasNodeData) => void; onNodeReversePrompt: (node: CanvasNodeData) => void; @@ -114,8 +114,8 @@ export type ToolContext = { node?: CanvasNodeData; /** 便捷访问 node.metadata（node 为空时为 undefined） */ nodeMetadata?: CanvasNodeMetadata; - /** 视频尾帧提取中（节点悬停工具栏用） */ - extractingVideoFrame: boolean; + /** 视频画面提取中（节点悬停工具栏用） */ + extractingVideoFrames: boolean; /** 视频音频提取/片段截取进行中（节点悬停工具栏用） */ extractingAudio: boolean; trimmingVideo: boolean; diff --git a/web/src/pages/canvas/canvas-project-selection-toolbar.tsx b/web/src/pages/canvas/canvas-project-selection-toolbar.tsx index 241aee1fe..2dcde3080 100644 --- a/web/src/pages/canvas/canvas-project-selection-toolbar.tsx +++ b/web/src/pages/canvas/canvas-project-selection-toolbar.tsx @@ -38,7 +38,7 @@ export function CanvasProjectSelectionToolbar({ anchorRef, containerRef, count, isProjectLinked: false, canUndo: false, canRedo: false, - extractingVideoFrame: false, + extractingVideoFrames: false, extractingAudio: false, trimmingVideo: false, mergingVideos, diff --git a/web/src/pages/canvas/project.tsx b/web/src/pages/canvas/project.tsx index ec2a46889..7af0f70cf 100644 --- a/web/src/pages/canvas/project.tsx +++ b/web/src/pages/canvas/project.tsx @@ -39,6 +39,7 @@ import { resolveProjectCanvasStyle } from "@/components/canvas/canvas-style-pick import { createStyleProfileSnapshot, resolveStyleProfile, serializeStyleProfile } from "@/lib/canvas/style-profile"; import { CanvasNodeToolbar, CanvasNodeInfoModal } from "@/components/canvas/canvas-node-toolbar"; import { CanvasSubtitleDialog } from "@/components/canvas/canvas-subtitle-dialog"; +import { CanvasVideoFrameDialog } from "@/components/canvas/canvas-video-frame-dialog"; import { CanvasVideoSegmentDialog } from "@/components/canvas/canvas-video-segment-dialog"; import { CanvasTimelineDialog } from "@/components/canvas/canvas-timeline-dialog"; import { syncNodeSubtitlesToTimeline } from "@/lib/timeline/timeline-build"; @@ -801,10 +802,12 @@ function InfiniteCanvasPage() { openPortraitTextureEditor, cropImageNode, cropNodeId, + closeFrameDialog, closeSegmentDialog, extractAudioFromVideo, - extractVideoLastFrame, - extractingVideoFrameNodeId, + extractVideoFrames, + extractingVideoFramesNodeId, + frameDialogNodeId, generateAngleNode, generateEmotionNode, handleSegmentConfirm, @@ -817,6 +820,7 @@ function InfiniteCanvasPage() { segmentDialogMode, segmentDialogNodeId, segmentRunningMode, + setFrameDialogNodeId, setSegmentDialogNodeId, setAngleNodeId, setEmotionNodeId, @@ -827,7 +831,8 @@ function InfiniteCanvasPage() { setUpscaleNodeId, splitImageNode, splitNodeId, - trimVideoAndRegenerate, + openVideoFrameExtractor, + openVideoSegmentExtractor, upscaleImageNode, upscaleNodeId, } = useCanvasMediaTools({ @@ -849,7 +854,6 @@ function InfiniteCanvasPage() { startGenerationRequest, finishGenerationRequest, bindGenerationTask, - onGenerateVideoNode: (nodeId, mode, prompt) => generateNodeRef.current?.(nodeId, mode, prompt), }); const handleNodesDeleted = useCallback( @@ -863,6 +867,7 @@ function InfiniteCanvasPage() { setDrawingNodeId(clearDeletedId); setInfoNodeId(clearDeletedId); setSubtitleNodeId(clearDeletedId); + setFrameDialogNodeId(clearDeletedId); setSegmentDialogNodeId(clearDeletedId); setCropNodeId(clearDeletedId); setMaskEditNodeId(clearDeletedId); @@ -886,7 +891,7 @@ function InfiniteCanvasPage() { } cleanupCanvasFiles({ projectId, nodes: nextNodes, chatSessions }); }, - [chatSessions, cleanupCanvasFiles, message, projectId, setAngleNodeId, setAnnotationNodeId, setCropNodeId, setEmotionNodeId, setMaskEditNodeId, setSegmentDialogNodeId, setSplitNodeId, setUpscaleNodeId, setRunningNodeId], + [chatSessions, cleanupCanvasFiles, message, projectId, setAngleNodeId, setAnnotationNodeId, setCropNodeId, setEmotionNodeId, setFrameDialogNodeId, setMaskEditNodeId, setSegmentDialogNodeId, setSplitNodeId, setUpscaleNodeId, setRunningNodeId], ); const { @@ -1254,6 +1259,7 @@ function InfiniteCanvasPage() { const dialogNode = dialogNodeId ? nodeById.get(dialogNodeId) || null : null; const subtitleNode = subtitleNodeId ? nodeById.get(subtitleNodeId) || null : null; const timelineNode = timelineNodeId ? nodeById.get(timelineNodeId) || null : null; + const frameNode = frameDialogNodeId ? nodeById.get(frameDialogNodeId) || null : null; const segmentNode = segmentDialogNodeId ? nodeById.get(segmentDialogNodeId) || null : null; const textEditorNode = textEditorNodeId ? nodeById.get(textEditorNodeId) || null : null; const characterReferenceNode = characterReferenceNodeId ? nodeById.get(characterReferenceNodeId) || null : null; @@ -2392,12 +2398,12 @@ function InfiniteCanvasPage() { setAngleNodeId((current) => (current === node.id ? null : node.id)); }} onViewImage={(node) => setPreviewNodeId(node.id)} - onExtractVideoLastFrame={(node) => void extractVideoLastFrame(node)} + onExtractVideoFrames={openVideoFrameExtractor} onExtractAudioFromVideo={(node) => void extractAudioFromVideo(node)} - onTrimVideoRegenerate={(node) => void trimVideoAndRegenerate(node)} + onTrimVideoSegments={openVideoSegmentExtractor} onSubtitles={(node) => setSubtitleNodeId(node.id)} onTimeline={(node) => setTimelineNodeId(node.id)} - extractingVideoFrame={toolbarNode?.id === extractingVideoFrameNodeId} + extractingVideoFrames={toolbarNode?.id === extractingVideoFramesNodeId} extractingAudio={segmentRunningMode === "audio"} trimmingVideo={segmentRunningMode === "video"} onReversePrompt={createImageReversePromptNodes} @@ -2505,6 +2511,15 @@ function InfiniteCanvasPage() { /> ) : null} + {frameNode ? ( + void extractVideoFrames(frameNode, params)} + /> + ) : null} + {segmentNode && segmentDialogMode ? ( undefined} onContentChange={unauthorized} onRetry={unauthorized} onOpenTaskDetails={unauthorized} onViewImage={(target) => setInfoNodeId(target.id)} onContextMenu={(event, nodeId) => openContextMenu(event, nodeId)} />)} - setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoLastFrame={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoRegenerate={unauthorized} extractingVideoFrame={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} /> + setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoFrames={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoSegments={unauthorized} extractingVideoFrames={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} />
访客操作仅在当前页面临时生效
diff --git a/web/src/pages/canvas/use-canvas-media-tools.ts b/web/src/pages/canvas/use-canvas-media-tools.ts index 1ceedc365..6f7da9ae1 100644 --- a/web/src/pages/canvas/use-canvas-media-tools.ts +++ b/web/src/pages/canvas/use-canvas-media-tools.ts @@ -8,6 +8,7 @@ import type { CanvasImageSplitParams } from "@/components/canvas/canvas-node-spl import type { CanvasImageUpscaleParams } from "@/components/canvas/canvas-node-upscale-dialog"; import type { CanvasImageAngleParams } from "@/components/canvas/canvas-node-angle-dialog"; import type { CanvasImageEmotionPayload } from "@/components/canvas/canvas-node-emotion-panel"; +import type { CanvasVideoFrameParams } from "@/components/canvas/canvas-video-frame-dialog"; import type { CanvasVideoSegmentParams } from "@/components/canvas/canvas-video-segment-dialog"; import { NODE_DEFAULT_SIZE } from "@/constant/canvas"; import { cropDataUrl, splitDataUrl, upscaleDataUrl } from "@/lib/canvas/canvas-image-data"; @@ -25,7 +26,8 @@ import { import { fitNodeSize, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size"; import { compositeEmotionImage, emotionGenerationSize, emotionProviderMask, normalizeEmotionPromptForProvider, resolveEmotionEditPlan } from "@/lib/canvas/canvas-emotion"; import { DEFAULT_PORTRAIT_TEXTURE_SETTINGS } from "@/lib/canvas/canvas-portrait-texture"; -import { captureVideoLastFrame } from "@/lib/canvas/canvas-video-frame"; +import { captureVideoFrames } from "@/lib/canvas/canvas-video-frame"; +import { buildVideoFrameNodes } from "@/lib/canvas/canvas-video-frame-nodes"; import { mergeVideos, type MergeVideoProgress } from "@/lib/canvas/canvas-video-merge"; import { extractVideoAudio, trimVideoSegment } from "@/lib/canvas/canvas-video-segment"; import { generationErrorMessage } from "@/lib/generation-error"; @@ -59,8 +61,6 @@ type UseCanvasMediaToolsOptions = { startGenerationRequest: (targetNodeId: string, originNodeId: string, runningId?: string, controller?: AbortController) => AbortController; finishGenerationRequest: (targetNodeId: string, controller: AbortController) => void; bindGenerationTask: (targetNodeId: string, task: GenerationTask) => void; - /** 创建空视频节点后触发既有生成执行器（由页面层接线），用于“截取片段后调用视频模型重生成” */ - onGenerateVideoNode?: (nodeId: string, mode: "video", prompt: string) => Promise | void; }; const NODE_STATUS_LOADING = "loading" as const; @@ -92,12 +92,11 @@ export function useCanvasMediaTools({ startGenerationRequest, finishGenerationRequest, bindGenerationTask, - onGenerateVideoNode, }: UseCanvasMediaToolsOptions) { const { message } = App.useApp(); const effectiveConfig = useEffectiveConfig(); const isAiConfigReady = useConfigStore((state) => state.isAiConfigReady); - const extractingVideoFrameNodeIdRef = useRef(null); + const extractingVideoFramesNodeIdRef = useRef(null); const mergeVideoRunningRef = useRef(false); const [cropNodeId, setCropNodeId] = useState(null); const [annotationNodeId, setAnnotationNodeId] = useState(null); @@ -106,7 +105,8 @@ export function useCanvasMediaTools({ const [upscaleNodeId, setUpscaleNodeId] = useState(null); const [angleNodeId, setAngleNodeId] = useState(null); const [emotionNodeId, setEmotionNodeId] = useState(null); - const [extractingVideoFrameNodeId, setExtractingVideoFrameNodeId] = useState(null); + const [frameDialogNodeId, setFrameDialogNodeId] = useState(null); + const [extractingVideoFramesNodeId, setExtractingVideoFramesNodeId] = useState(null); const [mergeVideoProgress, setMergeVideoProgress] = useState(null); const [segmentDialogNodeId, setSegmentDialogNodeId] = useState(null); const [segmentDialogMode, setSegmentDialogMode] = useState<"audio" | "video" | null>(null); @@ -200,45 +200,69 @@ export function useCanvasMediaTools({ message.success("标注图片已保存为新节点"); }, [message, setConnections, setDialogNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds]); - const extractVideoLastFrame = useCallback(async (node: CanvasNodeData) => { + const openVideoFrameExtractor = useCallback((node: CanvasNodeData) => { + if (!node.metadata?.content) { + message.warning("视频节点为空，无法提取画面"); + return; + } + if (extractingVideoFramesNodeIdRef.current) return; + setHoveredNodeId(null); + setToolbarNodeId(null); + setFrameDialogNodeId(node.id); + }, [message, setHoveredNodeId, setToolbarNodeId]); + + const closeFrameDialog = useCallback(() => { + if (extractingVideoFramesNodeIdRef.current) return; + setFrameDialogNodeId(null); + }, []); + + const extractVideoFrames = useCallback(async (node: CanvasNodeData, params: CanvasVideoFrameParams) => { const content = node.metadata?.content; - if (!content || extractingVideoFrameNodeIdRef.current) return; - const progress = startUploadStatus("截取视频尾帧", "读取视频资源"); - extractingVideoFrameNodeIdRef.current = node.id; - setExtractingVideoFrameNodeId(node.id); + if (!content || extractingVideoFramesNodeIdRef.current || !params.timesMs.length) return; + const progress = startUploadStatus("提取视频画面", "读取视频资源", params.timesMs.length + 2); + extractingVideoFramesNodeIdRef.current = node.id; + setExtractingVideoFramesNodeId(node.id); + setFrameDialogNodeId(null); try { const storedBlob = node.metadata?.storageKey ? await getMediaBlob(node.metadata.storageKey).catch(() => null) : null; - progress.update("定位并绘制最后一帧", 2); - const frameBlob = await captureVideoLastFrame(storedBlob || content); - progress.update("保存尾帧图片并创建节点", 3); - const image = await uploadImage(frameBlob); - const size = fitNodeSize(image.width, image.height, node.width, node.height); - const childId = nanoid(); - const child: CanvasNodeData = { - id: childId, - type: CanvasNodeType.Image, - title: `尾帧 · ${node.title || "视频"}`, - position: { x: node.position.x + node.width + 96, y: node.position.y }, - width: size.width, - height: size.height, - metadata: { ...imageMetadata(image), prompt: node.metadata?.prompt, workflowKind: node.metadata?.workflowKind, workflowTitle: node.metadata?.workflowTitle, shotIndex: node.metadata?.shotIndex }, - }; - setNodes((current) => [...current, child]); - setConnections((current) => [...current, { id: nanoid(), fromNodeId: node.id, toNodeId: childId }]); - setSelectedNodeIds(new Set([childId])); + progress.update("定位并绘制所选画面", 2); + const captured = await captureVideoFrames(storedBlob || content, params.timesMs); + const uploadedFrames = []; + const uploadFailures: string[] = []; + for (let index = 0; index < captured.frames.length; index += 1) { + const frame = captured.frames[index]; + try { + progress.update(`保存画面（${index + 1}/${captured.frames.length}）`, index + 3); + uploadedFrames.push({ timeMs: frame.timeMs, image: await uploadImage(frame.blob) }); + } catch (error) { + uploadFailures.push(error instanceof Error ? error.message : "画面图片上传失败"); + } + } + const frameNodes = buildVideoFrameNodes(node, uploadedFrames); + if (!frameNodes.length) throw new Error(uploadFailures[0] || "画面图片保存失败"); + const links = frameNodes.map((frameNode) => ({ id: nanoid(), fromNodeId: node.id, toNodeId: frameNode.id })); + const nextNodes = [...nodesRef.current, ...frameNodes]; + const nextConnections = [...connectionsRef.current, ...links]; + const selection = new Set(frameNodes.map((frameNode) => frameNode.id)); + nodesRef.current = nextNodes; + connectionsRef.current = nextConnections; + selectedNodeIdsRef.current = selection; + setNodes(nextNodes); + setConnections(nextConnections); + setSelectedNodeIds(selection); setSelectedConnectionId(null); - setHoveredNodeId(null); - setToolbarNodeId(null); - progress.done("尾帧图片已创建"); + const failedCount = captured.failures.length + uploadFailures.length; + progress.done(failedCount ? `已提取 ${frameNodes.length} 帧，${failedCount} 帧失败` : `已提取 ${frameNodes.length} 帧并创建图片节点`); + if (failedCount) message.warning(`${failedCount} 个时间点提取失败，其余画面已创建`); } catch (error) { - const details = error instanceof Error ? error.message : "尾帧截取失败"; + const details = error instanceof Error ? error.message : "视频画面提取失败"; progress.fail(details); message.error(details); } finally { - extractingVideoFrameNodeIdRef.current = null; - setExtractingVideoFrameNodeId(null); + extractingVideoFramesNodeIdRef.current = null; + setExtractingVideoFramesNodeId(null); } - }, [message, setConnections, setHoveredNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, setToolbarNodeId, startUploadStatus]); + }, [connectionsRef, message, nodesRef, selectedNodeIdsRef, setConnections, setNodes, setSelectedConnectionId, setSelectedNodeIds, startUploadStatus]); const extractAudioFromVideo = useCallback((node: CanvasNodeData) => { if (!node.metadata?.content) { @@ -252,7 +276,7 @@ export function useCanvasMediaTools({ setSegmentDialogMode("audio"); }, [message, setHoveredNodeId, setToolbarNodeId]); - const trimVideoAndRegenerate = useCallback((node: CanvasNodeData) => { + const openVideoSegmentExtractor = useCallback((node: CanvasNodeData) => { if (!node.metadata?.content) { message.warning("视频节点为空，无法截取片段"); return; @@ -305,23 +329,26 @@ export function useCanvasMediaTools({ } }, [domainProjectId, message, projectId, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]); - // 按段截取视频：FFmpeg 批量截取 → 每段创建片段节点与空结果节点 → 调用视频模型逐段重生成。 - const runTrimVideoAndRegenerate = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => { + // 按段截取视频：默认只创建片段节点；用户明确选择时再附带创建待生成节点，生成任务仍由用户手动发起。 + const runTrimVideoSegments = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => { const segments = params.segments || []; if (!segments.length) { message.warning("请至少添加一个截取片段"); return; } - const generationConfig = buildGenerationConfig(effectiveConfig, node, "video"); - const selectedConfig = { ...generationConfig, model: params.model || generationConfig.model }; - const batchError = validateVideoSegmentBatch(selectedConfig, segments, params.operation); - if (batchError) { - message.warning(batchError); - return; + const createsGenerationNodes = params.action === "create-generation-nodes"; + const generationConfig = createsGenerationNodes ? buildGenerationConfig(effectiveConfig, node, "video") : null; + const selectedConfig = generationConfig ? { ...generationConfig, model: params.model || generationConfig.model } : null; + if (selectedConfig) { + const batchError = validateVideoSegmentBatch(selectedConfig, segments, params.operation); + if (batchError) { + message.warning(batchError); + return; + } } const progress = startUploadStatus("截取视频片段", "加载 FFmpeg", segments.length * 4); try { - const prepared: Array<{ segmentNode: CanvasNodeData; targetNode: CanvasNodeData }> = []; + const prepared: Array<{ segmentNode: CanvasNodeData; targetNode?: CanvasNodeData }> = []; const failedSegments: string[] = []; const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video]; const baseX = node.position.x + node.width + 96; @@ -354,16 +381,17 @@ export function useCanvasMediaTools({ height: size.height, metadata: { ...videoMetadata(uploaded), prompt: `从「${sourceNode?.title || node.title || "视频"}」截取的片段 ${index + 1}`, status: NODE_STATUS_SUCCESS }, }; - const targetId = nanoid(); - const targetNode: CanvasNodeData = { - id: targetId, - type: CanvasNodeType.Video, - title: `重生成 ${index + 1} · ${sourceNode?.title || node.title || "视频"}`, - position: { x: segmentNode.position.x + size.width + 96, y: segmentNode.position.y + (size.height - spec.height) / 2 }, - width: spec.width, - height: spec.height, - metadata: { prompt: effectivePrompt, status: "idle", generationMode: "video", model: selectedConfig.model, videoEditOperation: params.operation, seconds: generationConfig.videoSeconds, size: generationConfig.size }, - }; + const targetNode: CanvasNodeData | undefined = selectedConfig && generationConfig + ? { + id: nanoid(), + type: CanvasNodeType.Video, + title: `待生成 ${index + 1} · ${sourceNode?.title || node.title || "视频"}`, + position: { x: segmentNode.position.x + size.width + 96, y: segmentNode.position.y + (size.height - spec.height) / 2 }, + width: spec.width, + height: spec.height, + metadata: { prompt: effectivePrompt, status: "idle", generationMode: "video", model: selectedConfig.model, videoEditOperation: params.operation, seconds: generationConfig.videoSeconds, size: generationConfig.size }, + } + : undefined; prepared.push({ segmentNode, targetNode }); } catch (segmentError) { failedSegments.push(segmentError instanceof Error ? segmentError.message : "视频截取失败"); @@ -371,46 +399,41 @@ export function useCanvasMediaTools({ } if (!prepared.length) throw new Error(failedSegments[0] || "视频截取失败"); const segmentNodes = prepared.map((item) => item.segmentNode); - const targetNodes = prepared.map((item) => item.targetNode); - // 先同步 ref 再 setState，保证生成执行器能立即读到新节点与连接。 + const targetNodes = prepared.flatMap((item) => item.targetNode ? [item.targetNode] : []); const nextNodes = [...nodesRef.current, ...segmentNodes, ...targetNodes]; const nextConnections = [ ...connectionsRef.current, ...prepared.flatMap((item) => [ { id: nanoid(), fromNodeId: node.id, toNodeId: item.segmentNode.id }, - { id: nanoid(), fromNodeId: item.segmentNode.id, toNodeId: item.targetNode.id }, + ...(item.targetNode ? [{ id: nanoid(), fromNodeId: item.segmentNode.id, toNodeId: item.targetNode.id }] : []), ]), ]; nodesRef.current = nextNodes; connectionsRef.current = nextConnections; setNodes(nextNodes); setConnections(nextConnections); - setSelectedNodeIds(new Set(targetNodes.map((item) => item.id))); + const selectedNodes = targetNodes.length ? targetNodes : segmentNodes; + const selection = new Set(selectedNodes.map((item) => item.id)); + selectedNodeIdsRef.current = selection; + setSelectedNodeIds(selection); setSelectedConnectionId(null); - progress.update("创建生成任务", segments.length * 4); - progress.done(`已截取 ${prepared.length}/${segments.length} 段，正在调用视频模型重新生成`); + progress.done(targetNodes.length ? `已截取 ${prepared.length}/${segments.length} 段并创建待生成节点` : `已截取 ${prepared.length}/${segments.length} 段视频`); segmentNodes.forEach((segmentNode) => { void ensureCanvasNodeAsset({ canvasId: projectId, domainProjectId, node: segmentNode, source: "canvas-manual" }) .then((result) => setNodes((current) => current.map((item) => (item.id === segmentNode.id ? { ...item, metadata: { ...item.metadata, assetId: result.assetId } } : item)))) - .catch((assetError) => message.warning(`片段已截取并开始重生成，但素材库写入失败：${assetError instanceof Error ? assetError.message : "未知错误"}`)); + .catch((assetError) => message.warning(`片段已截取，但素材库写入失败：${assetError instanceof Error ? assetError.message : "未知错误"}`)); }); - // 任务提交瞬时打满后端会触发限流；先错峰提交，后续可替换为批次调度的容量控制。 - for (let index = 0; index < targetNodes.length; index += 1) { - const targetNode = targetNodes[index]; - void onGenerateVideoNode?.(targetNode.id, "video", effectivePrompt); - if (index + 1 < targetNodes.length) await new Promise((resolve) => setTimeout(resolve, 300)); - } - if (failedSegments.length) message.warning(`${failedSegments.length} 段截取失败，已为其余 ${prepared.length} 段创建生成任务`); + if (failedSegments.length) message.warning(`${failedSegments.length} 段截取失败，其余 ${prepared.length} 段已创建`); } catch (error) { const details = error instanceof Error ? error.message : "视频截取失败"; progress.fail(details); message.error(details); } - }, [connectionsRef, domainProjectId, effectiveConfig, message, nodesRef, onGenerateVideoNode, projectId, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]); + }, [connectionsRef, domainProjectId, effectiveConfig, message, nodesRef, projectId, selectedNodeIdsRef, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]); const handleSegmentConfirm = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => { if (segmentRunningRef.current || !node.metadata?.content) return; - if (params.mode === "video") { + if (params.mode === "video" && params.action === "create-generation-nodes") { const generationConfig = buildGenerationConfig(effectiveConfig, node, "video"); const selectedConfig = { ...generationConfig, model: params.model || generationConfig.model }; const batchError = validateVideoSegmentBatch(selectedConfig, params.segments || [], params.operation); @@ -424,13 +447,13 @@ export function useCanvasMediaTools({ setSegmentDialogNodeId(null); setSegmentDialogMode(null); try { - if (params.mode === "video") await runTrimVideoAndRegenerate(node, params); + if (params.mode === "video") await runTrimVideoSegments(node, params); else await runExtractVideoAudio(node, params); } finally { segmentRunningRef.current = false; setSegmentRunningMode(null); } - }, [effectiveConfig, message, runExtractVideoAudio, runTrimVideoAndRegenerate]); + }, [effectiveConfig, message, runExtractVideoAudio, runTrimVideoSegments]); const mergeVideosByIds = useCallback(async (videoNodeIds: string[]) => { if (mergeVideoRunningRef.current) return; @@ -683,10 +706,12 @@ export function useCanvasMediaTools({ openPortraitTextureEditor, cropImageNode, cropNodeId, + closeFrameDialog, closeSegmentDialog, extractAudioFromVideo, - extractVideoLastFrame, - extractingVideoFrameNodeId, + extractVideoFrames, + extractingVideoFramesNodeId, + frameDialogNodeId, handleSegmentConfirm, generateAngleNode, maskEditImageNode, @@ -698,6 +723,7 @@ export function useCanvasMediaTools({ segmentDialogMode, segmentDialogNodeId, segmentRunningMode, + setFrameDialogNodeId, setSegmentDialogNodeId, setAngleNodeId, generateEmotionNode, @@ -709,7 +735,8 @@ export function useCanvasMediaTools({ setUpscaleNodeId, splitImageNode, splitNodeId, - trimVideoAndRegenerate, + openVideoFrameExtractor, + openVideoSegmentExtractor, upscaleImageNode, upscaleNodeId, }; diff --git a/web/src/types/canvas.ts b/web/src/types/canvas.ts index a0d6ae70e..96c57a371 100644 --- a/web/src/types/canvas.ts +++ b/web/src/types/canvas.ts @@ -320,6 +320,8 @@ export type CanvasNodeMetadata = { videoCameraMovePrompt?: string; videoStartFrameNodeId?: string; videoEndFrameNodeId?: string; + videoFrameSourceNodeId?: string; + videoFrameTimeMs?: number; versionOfNodeId?: string; versionLabel?: string; versionPrimary?: boolean; diff --git a/web/test/canvas-video-frame.test.ts b/web/test/canvas-video-frame.test.ts new file mode 100644 index 000000000..ce9206333 --- /dev/null +++ b/web/test/canvas-video-frame.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; + +import { buildVideoFrameNodes } from "../src/lib/canvas/canvas-video-frame-nodes"; +import { normalizeVideoFrameTimes } from "../src/lib/canvas/canvas-video-frame"; +import { CanvasNodeType, type CanvasNodeData } from "../src/types/canvas"; + +const sourceNode: CanvasNodeData = { + id: "video-source", + type: CanvasNodeType.Video, + title: "测试视频", + position: { x: 100, y: 200 }, + width: 640, + height: 360, + metadata: { workflowKind: "shot", workflowTitle: "镜头 1", shotIndex: 1 }, +}; + +describe("normalizeVideoFrameTimes", () => { + test("按时间排序、去重并限制在可解码范围内", () => { + expect(normalizeVideoFrameTimes([1500, 0, 1500.4, -20, 16000, Number.NaN], 15000)).toEqual([0, 1500, 14999]); + }); + + test("视频时长无效时不返回时间点", () => { + expect(normalizeVideoFrameTimes([0, 1000], 0)).toEqual([]); + }); +}); + +describe("buildVideoFrameNodes", () => { + test("为每个时间点创建可追溯的图片节点并保持网格布局", () => { + const nodes = buildVideoFrameNodes(sourceNode, [ + { timeMs: 0, image: { url: "blob:first", storageKey: "image:first", width: 1280, height: 720, bytes: 10, mimeType: "image/png" } }, + { timeMs: 1500, image: { url: "blob:middle", storageKey: "image:middle", width: 1280, height: 720, bytes: 20, mimeType: "image/png" } }, + { timeMs: 14999, image: { url: "blob:last", storageKey: "image:last", width: 1280, height: 720, bytes: 30, mimeType: "image/png" } }, + ]); + + expect(nodes).toHaveLength(3); + expect(nodes.map((node) => node.metadata?.videoFrameTimeMs)).toEqual([0, 1500, 14999]); + expect(nodes.every((node) => node.metadata?.videoFrameSourceNodeId === sourceNode.id)).toBe(true); + expect(nodes.every((node) => node.metadata?.workflowKind === "shot" && node.metadata?.shotIndex === 1)).toBe(true); + expect(nodes[0].position.x).toBe(sourceNode.position.x + sourceNode.width + 96); + expect(nodes[1].position.x).toBeGreaterThan(nodes[0].position.x); + expect(nodes[2].position.y).toBeGreaterThan(nodes[0].position.y); + }); +});