From d33c6389d4df7ba473371f0e9191289113aa6fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=88=B1=E7=AC=91=E7=9A=84=E6=AF=9B=E6=AF=9B=E8=99=AB?= <315515767@qq.com> Date: Mon, 10 Aug 2026 00:13:02 +0800 Subject: [PATCH] =?UTF-8?q?feat(canvas):=20=E8=A7=86=E9=A2=91=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E6=88=AA=E5=8F=96=E9=87=8D=E7=94=9F=E6=88=90=20-=20?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=9A=E6=AE=B5/=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=BA=BF=E5=AF=BC=E5=85=A5=E5=B9=B6=E8=B0=83=E7=94=A8=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E6=A8=A1=E5=9E=8B=E6=89=B9=E9=87=8F=E9=87=8D=E7=94=9F?= =?UTF-8?q?=E6=88=90=20(#197)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(canvas): 视频节点截取重生成 - 支持多段/时间线导入并调用视频模型批量重生成 * test(canvas): 视频节点时间线导入 - 修正旧数据标题回退测试 * style(canvas): 视频节点截取重生成 - 格式化新增文件并修正测试 --------- Co-authored-by: sikongyue <3155157671@qq.com> --- web/package.json | 2 +- .../components/canvas/canvas-node-toolbar.tsx | 12 +- web/src/components/canvas/canvas-toolbar.tsx | 4 +- .../canvas/canvas-video-segment-dialog.tsx | 422 ++++++++++++++++++ .../toolbars/toolbar-settings-modal.tsx | 2 + .../lib/canvas/canvas-resource-references.ts | 16 + .../lib/canvas/canvas-video-regeneration.ts | 54 +++ web/src/lib/canvas/canvas-video-segment.ts | 104 +++++ .../canvas/canvas-video-timeline-segments.ts | 54 +++ .../definitions/node-hover-tools.tsx | 28 +- .../canvas/tool-registry/tool-definition.ts | 5 + .../canvas-project-selection-toolbar.tsx | 2 + web/src/pages/canvas/project.tsx | 33 +- web/src/pages/canvas/shared.tsx | 2 +- .../pages/canvas/use-canvas-media-tools.ts | 217 ++++++++- web/src/styles/globals.css | 1 + web/test/canvas-resource-references.test.ts | 52 +++ web/test/canvas-video-regeneration.test.ts | 128 ++++++ .../canvas-video-timeline-segments.test.ts | 76 ++++ 19 files changed, 1206 insertions(+), 8 deletions(-) create mode 100644 web/src/components/canvas/canvas-video-segment-dialog.tsx create mode 100644 web/src/lib/canvas/canvas-video-regeneration.ts create mode 100644 web/src/lib/canvas/canvas-video-segment.ts create mode 100644 web/src/lib/canvas/canvas-video-timeline-segments.ts create mode 100644 web/test/canvas-resource-references.test.ts create mode 100644 web/test/canvas-video-regeneration.test.ts create mode 100644 web/test/canvas-video-timeline-segments.test.ts diff --git a/web/package.json b/web/package.json index a530a1691..3e8ce83c9 100644 --- a/web/package.json +++ b/web/package.json @@ -7,7 +7,7 @@ "dev": "vite --host 0.0.0.0 --port 3000", "build": "tsc --noEmit && vite build", "typecheck": "tsc --noEmit", - "test": "bun test test/agent-prompt-cache.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-generation-layout.test.ts test/canvas-leafer-viewport.test.ts test/client-id.test.ts", + "test": "bun test test/agent-prompt-cache.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-generation-layout.test.ts test/canvas-leafer-viewport.test.ts test/canvas-resource-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/client-id.test.ts", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", "format:check": "prettier --check ." diff --git a/web/src/components/canvas/canvas-node-toolbar.tsx b/web/src/components/canvas/canvas-node-toolbar.tsx index dffd7da6d..2bc6f98de 100644 --- a/web/src/components/canvas/canvas-node-toolbar.tsx +++ b/web/src/components/canvas/canvas-node-toolbar.tsx @@ -42,9 +42,13 @@ type CanvasNodeToolbarProps = { onAngle: (node: CanvasNodeData) => void; onViewImage: (node: CanvasNodeData) => void; onExtractVideoLastFrame: (node: CanvasNodeData) => void; + onExtractAudioFromVideo: (node: CanvasNodeData) => void; + onTrimVideoRegenerate: (node: CanvasNodeData) => void; onSubtitles: (node: CanvasNodeData) => void; onTimeline: (node: CanvasNodeData) => void; extractingVideoFrame: boolean; + extractingAudio: boolean; + trimmingVideo: boolean; onReversePrompt: (node: CanvasNodeData) => void; onRetry: (node: CanvasNodeData) => void; onToggleFreeResize: (node: CanvasNodeData) => void; @@ -105,9 +109,13 @@ export function CanvasNodeToolbar({ onAngle, onViewImage, onExtractVideoLastFrame, + onExtractAudioFromVideo, + onTrimVideoRegenerate, onSubtitles, onTimeline, extractingVideoFrame, + extractingAudio, + trimmingVideo, onReversePrompt, onRetry, onToggleFreeResize, @@ -243,7 +251,7 @@ export function CanvasNodeToolbar({ onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload, onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop, onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage, - onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, + onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoRegenerate: onTrimVideoRegenerate, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt, } as Partial as ToolbarHandlers; @@ -259,6 +267,8 @@ export function CanvasNodeToolbar({ node, nodeMetadata: node.metadata, extractingVideoFrame, + extractingAudio, + trimmingVideo, mergingVideos: false, addPanelOpen: false, appearancePanelOpen: false, diff --git a/web/src/components/canvas/canvas-toolbar.tsx b/web/src/components/canvas/canvas-toolbar.tsx index f9e2ad7ee..0805bb523 100644 --- a/web/src/components/canvas/canvas-toolbar.tsx +++ b/web/src/components/canvas/canvas-toolbar.tsx @@ -139,7 +139,7 @@ export function CanvasToolbar({ onNodeInfo: () => {}, onNodeDelete: () => {}, onNodeRetry: () => {}, onNodeEditText: () => {}, onNodeDecreaseFont: () => {}, onNodeIncreaseFont: () => {}, onNodeToggleDialog: () => {}, onNodeAnnotate: () => {}, onNodeGenerateImage: () => {}, onNodeUpload: () => {}, onNodeDownload: () => {}, onNodeSaveAsset: () => {}, onNodeMaskEdit: () => {}, onNodeEmotion: () => {}, onNodePortraitTexture: () => {}, onNodeCrop: () => {}, onNodeSplit: () => {}, onNodeUpscale: () => {}, - onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {}, + onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoRegenerate: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {}, onNodeToggleFreeResize: () => {}, onNodeToggleLocked: () => {}, onNodeCopyPrompt: () => {}, } as ToolbarHandlers; @@ -153,6 +153,8 @@ export function CanvasToolbar({ canUndo, canRedo, extractingVideoFrame: false, + extractingAudio: false, + trimmingVideo: false, mergingVideos: false, addPanelOpen: addOpen, appearancePanelOpen: appearanceOpen, diff --git a/web/src/components/canvas/canvas-video-segment-dialog.tsx b/web/src/components/canvas/canvas-video-segment-dialog.tsx new file mode 100644 index 000000000..13f2d91aa --- /dev/null +++ b/web/src/components/canvas/canvas-video-segment-dialog.tsx @@ -0,0 +1,422 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { App, Button, Input, InputNumber, Modal, Select } from "antd"; +import { AudioLines, Check, ListVideo, Plus, Scissors, SkipBack, SkipForward, Trash2 } from "lucide-react"; +import { nanoid } from "nanoid"; + +import { ModelPicker } from "@/components/model-picker"; +import { canvasThemes } from "@/lib/canvas-theme"; +import { buildTimelineImportSegments, type CanvasTimelineSegmentItem } from "@/lib/canvas/canvas-video-timeline-segments"; +import { listVideoReferenceModels } from "@/lib/canvas/canvas-video-regeneration"; +import { modelCapabilityConfigFor } from "@/lib/model-capabilities"; +import { navigateToSettings } from "@/lib/settings-navigation"; +import { useThemeStore } from "@/stores/use-theme-store"; +import { resolveMediaUrl } from "@/services/file-storage"; +import { cacheResourceObjectUrl } from "@/services/resource-blob-cache"; +import { resourceIdFromStorageKey } from "@/services/api/resources"; +import { modelDisplayName, type AiConfig } from "@/stores/use-config-store"; +import { type CanvasConnection, type CanvasNodeData, type CanvasVideoEditOperation } from "@/types/canvas"; +import type { TimelineProject } from "@/types/timeline"; + +export type CanvasVideoSegmentItem = CanvasTimelineSegmentItem; + +export type CanvasVideoSegmentParams = { + mode: "audio" | "video"; + startMs: number; + endMs: number; + prompt?: string; + segments?: CanvasVideoSegmentItem[]; + model?: string; + operation?: CanvasVideoEditOperation; +}; + +type CanvasVideoSegmentDialogProps = { + node: CanvasNodeData; + nodes: CanvasNodeData[]; + connections: CanvasConnection[]; + open: boolean; + mode: "audio" | "video"; + config: AiConfig; + timeline?: TimelineProject | null; + onClose: () => void; + onConfirm: (params: CanvasVideoSegmentParams) => void; +}; + +const MIN_SEGMENT_MS = 100; + +export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, config, timeline, onClose, onConfirm }: CanvasVideoSegmentDialogProps) { + const { message } = App.useApp(); + const theme = canvasThemes[useThemeStore((state) => state.theme)]; + const videoRef = useRef(null); + const segmentsSeededRef = useRef(false); + const [videoUrl, setVideoUrl] = useState(""); + const [videoError, setVideoError] = useState(false); + const [durationMs, setDurationMs] = useState(0); + const [startSec, setStartSec] = useState(0); + const [endSec, setEndSec] = useState(0); + const [prompt, setPrompt] = useState(""); + const [segments, setSegments] = useState([]); + const [model, setModel] = useState(""); + const [operation, setOperation] = useState("extend"); + const eligibleModels = useMemo(() => listVideoReferenceModels(config), [config]); + + // 打开弹窗时解析视频地址,读取时长并初始化起止时间。 + useEffect(() => { + if (!open) return; + let cancelled = false; + setVideoUrl(""); + setVideoError(false); + setDurationMs(0); + setStartSec(0); + setEndSec(0); + setPrompt(""); + setSegments([]); + segmentsSeededRef.current = false; + const defaultModel = config.videoModel || config.model || ""; + const initialModel = eligibleModels.includes(defaultModel) ? defaultModel : eligibleModels[0] || defaultModel; + setModel(initialModel); + const profile = initialModel ? modelCapabilityConfigFor(config, initialModel).video : undefined; + setOperation(profile?.operations.includes("extend") ? "extend" : (profile?.operations[0] as CanvasVideoEditOperation | undefined) || "extend"); + const storageKey = node.metadata?.storageKey || ""; + const fallback = node.metadata?.content || ""; + const applyUrl = (url: string) => { + if (!cancelled) setVideoUrl(url); + }; + if (resourceIdFromStorageKey(storageKey)) { + void cacheResourceObjectUrl(storageKey) + .then((cached) => { + if (cancelled) return; + if (cached) setVideoUrl(cached); + else void resolveMediaUrl(storageKey, fallback).then(applyUrl); + }) + .catch(() => { + if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl); + }); + } else { + void resolveMediaUrl(storageKey, fallback).then(applyUrl); + } + return () => { + cancelled = true; + }; + }, [config, eligibleModels, node, open]); + + // 视频模式等待时长元数据就绪后,默认放入一个完整时长的片段。 + useEffect(() => { + if (!open || mode !== "video" || segmentsSeededRef.current || !durationMs) return; + segmentsSeededRef.current = true; + setSegments([{ id: nanoid(), startMs: 0, endMs: durationMs, sourceNodeId: node.id }]); + }, [durationMs, mode, open]); + + const isVideoMode = mode === "video"; + const durationSec = durationMs > 0 ? durationMs / 1000 : 0; + const hasTimelineVideoClips = Boolean(timeline?.clips.some((clip) => clip.kind === "video")); + const videoProfile = useMemo(() => (model ? modelCapabilityConfigFor(config, model).video : undefined), [config, model]); + const defaultModel = config.videoModel || config.model || ""; + const defaultModelSupported = eligibleModels.includes(defaultModel); + const hasEligibleModels = eligibleModels.length > 0; + const firstEligibleModel = eligibleModels[0] || ""; + + // 切换模型后保持所选模式仍然有效,优先回退到“视频续写”。 + useEffect(() => { + if (!videoProfile?.operations.length || videoProfile.operations.includes(operation)) return; + setOperation(videoProfile.operations.includes("extend") ? "extend" : (videoProfile.operations[0] as CanvasVideoEditOperation)); + }, [operation, videoProfile]); + + const operationOptions = (videoProfile?.operations || []).map((value) => { + const operation = value as CanvasVideoEditOperation; + return { value: operation, label: videoOperationLabel(operation) }; + }); + + const rangeMs = useMemo(() => { + const start = Math.max(0, Math.round((startSec || 0) * 1000)); + const end = durationMs > 0 ? Math.min(durationMs, Math.round((endSec || 0) * 1000)) : Math.round((endSec || 0) * 1000); + return { startMs: start, endMs: Math.max(start, end) }; + }, [durationMs, endSec, startSec]); + + const addManualSegment = () => { + const last = segments[segments.length - 1]; + const maxEndMs = durationMs || last?.endMs || 0; + const startMs = last ? Math.min(maxEndMs, last.endMs) : 0; + const endMs = Math.max(startMs + MIN_SEGMENT_MS, maxEndMs); + setSegments((current) => [...current, { id: nanoid(), startMs, endMs, sourceNodeId: node.id }]); + }; + + const importTimelineSegments = () => { + try { + const result = buildTimelineImportSegments(node, nodes, connections, timeline, durationMs); + if (!result.ok) { + message.warning(result.error); + return; + } + setSegments(result.segments); + setPrompt((current) => current || `按时间线截取的 ${result.segments.length} 段视频,保持画面主体与镜头,重新生成每一段`); + message.success(`已从时间线导入 ${result.segments.length} 个片段`); + } catch (error) { + console.warn("时间线片段导入失败", error); + message.warning("读取时间线片段失败,请刷新后重试"); + } + }; + + const updateSegment = (id: string, patch: Partial>) => { + setSegments((current) => current.map((item) => (item.id === id ? { ...item, ...patch } : item))); + }; + + const removeSegment = (id: string) => { + setSegments((current) => current.filter((item) => item.id !== id)); + }; + + const handleConfirm = () => { + if (isVideoMode) { + if (!model) { + message.warning("请选择视频模型"); + return; + } + if (!segments.length) { + message.warning("请至少添加一个截取片段"); + return; + } + for (const segment of segments) { + if (segment.endMs - segment.startMs < MIN_SEGMENT_MS) { + message.warning("片段时长至少 0.1 秒"); + return; + } + } + onConfirm({ + mode: "video", + startMs: 0, + endMs: 0, + segments: segments.map(({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl }) => ({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl })), + model, + operation, + prompt: prompt.trim(), + }); + return; + } + if (!durationMs) { + message.warning("视频时长未就绪,请稍候再试"); + return; + } + if (rangeMs.endMs - rangeMs.startMs < MIN_SEGMENT_MS) { + message.warning("片段时长至少 0.1 秒"); + return; + } + onConfirm({ mode: "audio", startMs: rangeMs.startMs, endMs: rangeMs.endMs }); + }; + + const title = ( +
+ + {isVideoMode ? : } + +
+
{isVideoMode ? "截取片段并批量重生成" : "从视频提取声音"}
+
{node.title || "视频节点"}
+
+
+ ); + + return ( + +
+
+ {videoUrl ? ( +
+ + {isVideoMode ? ( +
+ {!defaultModelSupported ? ( +
+ + {hasEligibleModels + ? `当前默认模型${defaultModel ? `「${modelDisplayName(config, defaultModel)}」` : ""}不支持参考视频,已自动切换到「${modelDisplayName(config, firstEligibleModel)}」。` + : "当前配置中没有支持参考视频的视频模型,请先到设置里配置 Seedance / Agent Plan / NewAPI 渠道。"} + + {!hasEligibleModels ? ( + + ) : null} +
+ ) : null} + +
+
+ 时长 + {durationSec ? formatSegmentTime(durationSec) : "未知"} + 已选 + {segments.length} 段 +
+
+ + +
+
+ + {segments.length ? ( +
+ {segments.map((segment, index) => ( +
+
+ 片段 {index + 1} +
+ 起点 + updateSegment(segment.id, { startMs: Math.round((value ?? 0) * 1000) })} + className="w-full" + aria-label={`片段 ${index + 1} 起点(秒)`} + /> +
+
+ 终点 + updateSegment(segment.id, { endMs: Math.round((value ?? 0) * 1000) })} + className="w-full" + aria-label={`片段 ${index + 1} 终点(秒)`} + /> +
+
+
+ ))} +
+ ) : ( +
+ 还没有片段:手动添加起点/终点,或从时间线导入该节点的已有片段。 +
+ )} + +
+ +