feat(canvas): 视频节点截取重生成 - 支持多段/时间线导入并调用视频模型批量重生成 (#197)
* feat(canvas): 视频节点截取重生成 - 支持多段/时间线导入并调用视频模型批量重生成 * test(canvas): 视频节点时间线导入 - 修正旧数据标题回退测试 * style(canvas): 视频节点截取重生成 - 格式化新增文件并修正测试 --------- Co-authored-by: sikongyue <3155157671@qq.com>
This commit is contained in:
爱笑的毛毛虫
@@ -7,7 +7,7 @@
|
||||
"dev": "vite --host 0.0.0.0 --port 3000",
|
||||
"build": "tsc --noEmit && vite build",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "bun test test/agent-prompt-cache.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-generation-layout.test.ts test/canvas-leafer-viewport.test.ts test/client-id.test.ts",
|
||||
"test": "bun test test/agent-prompt-cache.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-generation-layout.test.ts test/canvas-leafer-viewport.test.ts test/canvas-resource-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/client-id.test.ts",
|
||||
"start": "vite preview --host 0.0.0.0 --port 3000",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
|
||||
@@ -42,9 +42,13 @@ type CanvasNodeToolbarProps = {
|
||||
onAngle: (node: CanvasNodeData) => void;
|
||||
onViewImage: (node: CanvasNodeData) => void;
|
||||
onExtractVideoLastFrame: (node: CanvasNodeData) => void;
|
||||
onExtractAudioFromVideo: (node: CanvasNodeData) => void;
|
||||
onTrimVideoRegenerate: (node: CanvasNodeData) => void;
|
||||
onSubtitles: (node: CanvasNodeData) => void;
|
||||
onTimeline: (node: CanvasNodeData) => void;
|
||||
extractingVideoFrame: boolean;
|
||||
extractingAudio: boolean;
|
||||
trimmingVideo: boolean;
|
||||
onReversePrompt: (node: CanvasNodeData) => void;
|
||||
onRetry: (node: CanvasNodeData) => void;
|
||||
onToggleFreeResize: (node: CanvasNodeData) => void;
|
||||
@@ -105,9 +109,13 @@ export function CanvasNodeToolbar({
|
||||
onAngle,
|
||||
onViewImage,
|
||||
onExtractVideoLastFrame,
|
||||
onExtractAudioFromVideo,
|
||||
onTrimVideoRegenerate,
|
||||
onSubtitles,
|
||||
onTimeline,
|
||||
extractingVideoFrame,
|
||||
extractingAudio,
|
||||
trimmingVideo,
|
||||
onReversePrompt,
|
||||
onRetry,
|
||||
onToggleFreeResize,
|
||||
@@ -243,7 +251,7 @@ export function CanvasNodeToolbar({
|
||||
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
|
||||
onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
|
||||
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
|
||||
onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
|
||||
onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoRegenerate: onTrimVideoRegenerate, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
|
||||
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
|
||||
} as Partial<ToolbarHandlers> as ToolbarHandlers;
|
||||
|
||||
@@ -259,6 +267,8 @@ export function CanvasNodeToolbar({
|
||||
node,
|
||||
nodeMetadata: node.metadata,
|
||||
extractingVideoFrame,
|
||||
extractingAudio,
|
||||
trimmingVideo,
|
||||
mergingVideos: false,
|
||||
addPanelOpen: false,
|
||||
appearancePanelOpen: false,
|
||||
|
||||
@@ -139,7 +139,7 @@ export function CanvasToolbar({
|
||||
onNodeInfo: () => {}, onNodeDelete: () => {}, onNodeRetry: () => {}, onNodeEditText: () => {}, onNodeDecreaseFont: () => {}, onNodeIncreaseFont: () => {},
|
||||
onNodeToggleDialog: () => {}, onNodeAnnotate: () => {}, onNodeGenerateImage: () => {}, onNodeUpload: () => {}, onNodeDownload: () => {}, onNodeSaveAsset: () => {},
|
||||
onNodeMaskEdit: () => {}, onNodeEmotion: () => {}, onNodePortraitTexture: () => {}, onNodeCrop: () => {}, onNodeSplit: () => {}, onNodeUpscale: () => {},
|
||||
onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {},
|
||||
onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeExtractAudioFromVideo: () => {}, onNodeTrimVideoRegenerate: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {},
|
||||
onNodeToggleFreeResize: () => {}, onNodeToggleLocked: () => {}, onNodeCopyPrompt: () => {},
|
||||
} as ToolbarHandlers;
|
||||
|
||||
@@ -153,6 +153,8 @@ export function CanvasToolbar({
|
||||
canUndo,
|
||||
canRedo,
|
||||
extractingVideoFrame: false,
|
||||
extractingAudio: false,
|
||||
trimmingVideo: false,
|
||||
mergingVideos: false,
|
||||
addPanelOpen: addOpen,
|
||||
appearancePanelOpen: appearanceOpen,
|
||||
|
||||
422
web/src/components/canvas/canvas-video-segment-dialog.tsx
Normal file
422
web/src/components/canvas/canvas-video-segment-dialog.tsx
Normal file
@@ -0,0 +1,422 @@
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { App, Button, Input, InputNumber, Modal, Select } from "antd";
|
||||
import { AudioLines, Check, ListVideo, Plus, Scissors, SkipBack, SkipForward, Trash2 } from "lucide-react";
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import { ModelPicker } from "@/components/model-picker";
|
||||
import { canvasThemes } from "@/lib/canvas-theme";
|
||||
import { buildTimelineImportSegments, type CanvasTimelineSegmentItem } from "@/lib/canvas/canvas-video-timeline-segments";
|
||||
import { listVideoReferenceModels } from "@/lib/canvas/canvas-video-regeneration";
|
||||
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
|
||||
import { navigateToSettings } from "@/lib/settings-navigation";
|
||||
import { useThemeStore } from "@/stores/use-theme-store";
|
||||
import { resolveMediaUrl } from "@/services/file-storage";
|
||||
import { cacheResourceObjectUrl } from "@/services/resource-blob-cache";
|
||||
import { resourceIdFromStorageKey } from "@/services/api/resources";
|
||||
import { modelDisplayName, type AiConfig } from "@/stores/use-config-store";
|
||||
import { type CanvasConnection, type CanvasNodeData, type CanvasVideoEditOperation } from "@/types/canvas";
|
||||
import type { TimelineProject } from "@/types/timeline";
|
||||
|
||||
export type CanvasVideoSegmentItem = CanvasTimelineSegmentItem;
|
||||
|
||||
export type CanvasVideoSegmentParams = {
|
||||
mode: "audio" | "video";
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
prompt?: string;
|
||||
segments?: CanvasVideoSegmentItem[];
|
||||
model?: string;
|
||||
operation?: CanvasVideoEditOperation;
|
||||
};
|
||||
|
||||
type CanvasVideoSegmentDialogProps = {
|
||||
node: CanvasNodeData;
|
||||
nodes: CanvasNodeData[];
|
||||
connections: CanvasConnection[];
|
||||
open: boolean;
|
||||
mode: "audio" | "video";
|
||||
config: AiConfig;
|
||||
timeline?: TimelineProject | null;
|
||||
onClose: () => void;
|
||||
onConfirm: (params: CanvasVideoSegmentParams) => void;
|
||||
};
|
||||
|
||||
const MIN_SEGMENT_MS = 100;
|
||||
|
||||
export function CanvasVideoSegmentDialog({ node, nodes, connections, open, mode, config, timeline, onClose, onConfirm }: CanvasVideoSegmentDialogProps) {
|
||||
const { message } = App.useApp();
|
||||
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const segmentsSeededRef = useRef(false);
|
||||
const [videoUrl, setVideoUrl] = useState("");
|
||||
const [videoError, setVideoError] = useState(false);
|
||||
const [durationMs, setDurationMs] = useState(0);
|
||||
const [startSec, setStartSec] = useState(0);
|
||||
const [endSec, setEndSec] = useState(0);
|
||||
const [prompt, setPrompt] = useState("");
|
||||
const [segments, setSegments] = useState<CanvasVideoSegmentItem[]>([]);
|
||||
const [model, setModel] = useState("");
|
||||
const [operation, setOperation] = useState<CanvasVideoEditOperation>("extend");
|
||||
const eligibleModels = useMemo(() => listVideoReferenceModels(config), [config]);
|
||||
|
||||
// 打开弹窗时解析视频地址，读取时长并初始化起止时间。
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
let cancelled = false;
|
||||
setVideoUrl("");
|
||||
setVideoError(false);
|
||||
setDurationMs(0);
|
||||
setStartSec(0);
|
||||
setEndSec(0);
|
||||
setPrompt("");
|
||||
setSegments([]);
|
||||
segmentsSeededRef.current = false;
|
||||
const defaultModel = config.videoModel || config.model || "";
|
||||
const initialModel = eligibleModels.includes(defaultModel) ? defaultModel : eligibleModels[0] || defaultModel;
|
||||
setModel(initialModel);
|
||||
const profile = initialModel ? modelCapabilityConfigFor(config, initialModel).video : undefined;
|
||||
setOperation(profile?.operations.includes("extend") ? "extend" : (profile?.operations[0] as CanvasVideoEditOperation | undefined) || "extend");
|
||||
const storageKey = node.metadata?.storageKey || "";
|
||||
const fallback = node.metadata?.content || "";
|
||||
const applyUrl = (url: string) => {
|
||||
if (!cancelled) setVideoUrl(url);
|
||||
};
|
||||
if (resourceIdFromStorageKey(storageKey)) {
|
||||
void cacheResourceObjectUrl(storageKey)
|
||||
.then((cached) => {
|
||||
if (cancelled) return;
|
||||
if (cached) setVideoUrl(cached);
|
||||
else void resolveMediaUrl(storageKey, fallback).then(applyUrl);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl);
|
||||
});
|
||||
} else {
|
||||
void resolveMediaUrl(storageKey, fallback).then(applyUrl);
|
||||
}
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [config, eligibleModels, node, open]);
|
||||
|
||||
// 视频模式等待时长元数据就绪后，默认放入一个完整时长的片段。
|
||||
useEffect(() => {
|
||||
if (!open || mode !== "video" || segmentsSeededRef.current || !durationMs) return;
|
||||
segmentsSeededRef.current = true;
|
||||
setSegments([{ id: nanoid(), startMs: 0, endMs: durationMs, sourceNodeId: node.id }]);
|
||||
}, [durationMs, mode, open]);
|
||||
|
||||
const isVideoMode = mode === "video";
|
||||
const durationSec = durationMs > 0 ? durationMs / 1000 : 0;
|
||||
const hasTimelineVideoClips = Boolean(timeline?.clips.some((clip) => clip.kind === "video"));
|
||||
const videoProfile = useMemo(() => (model ? modelCapabilityConfigFor(config, model).video : undefined), [config, model]);
|
||||
const defaultModel = config.videoModel || config.model || "";
|
||||
const defaultModelSupported = eligibleModels.includes(defaultModel);
|
||||
const hasEligibleModels = eligibleModels.length > 0;
|
||||
const firstEligibleModel = eligibleModels[0] || "";
|
||||
|
||||
// 切换模型后保持所选模式仍然有效，优先回退到“视频续写”。
|
||||
useEffect(() => {
|
||||
if (!videoProfile?.operations.length || videoProfile.operations.includes(operation)) return;
|
||||
setOperation(videoProfile.operations.includes("extend") ? "extend" : (videoProfile.operations[0] as CanvasVideoEditOperation));
|
||||
}, [operation, videoProfile]);
|
||||
|
||||
const operationOptions = (videoProfile?.operations || []).map((value) => {
|
||||
const operation = value as CanvasVideoEditOperation;
|
||||
return { value: operation, label: videoOperationLabel(operation) };
|
||||
});
|
||||
|
||||
const rangeMs = useMemo(() => {
|
||||
const start = Math.max(0, Math.round((startSec || 0) * 1000));
|
||||
const end = durationMs > 0 ? Math.min(durationMs, Math.round((endSec || 0) * 1000)) : Math.round((endSec || 0) * 1000);
|
||||
return { startMs: start, endMs: Math.max(start, end) };
|
||||
}, [durationMs, endSec, startSec]);
|
||||
|
||||
const addManualSegment = () => {
|
||||
const last = segments[segments.length - 1];
|
||||
const maxEndMs = durationMs || last?.endMs || 0;
|
||||
const startMs = last ? Math.min(maxEndMs, last.endMs) : 0;
|
||||
const endMs = Math.max(startMs + MIN_SEGMENT_MS, maxEndMs);
|
||||
setSegments((current) => [...current, { id: nanoid(), startMs, endMs, sourceNodeId: node.id }]);
|
||||
};
|
||||
|
||||
const importTimelineSegments = () => {
|
||||
try {
|
||||
const result = buildTimelineImportSegments(node, nodes, connections, timeline, durationMs);
|
||||
if (!result.ok) {
|
||||
message.warning(result.error);
|
||||
return;
|
||||
}
|
||||
setSegments(result.segments);
|
||||
setPrompt((current) => current || `按时间线截取的 ${result.segments.length} 段视频，保持画面主体与镜头，重新生成每一段`);
|
||||
message.success(`已从时间线导入 ${result.segments.length} 个片段`);
|
||||
} catch (error) {
|
||||
console.warn("时间线片段导入失败", error);
|
||||
message.warning("读取时间线片段失败，请刷新后重试");
|
||||
}
|
||||
};
|
||||
|
||||
const updateSegment = (id: string, patch: Partial<Pick<CanvasVideoSegmentItem, "startMs" | "endMs">>) => {
|
||||
setSegments((current) => current.map((item) => (item.id === id ? { ...item, ...patch } : item)));
|
||||
};
|
||||
|
||||
const removeSegment = (id: string) => {
|
||||
setSegments((current) => current.filter((item) => item.id !== id));
|
||||
};
|
||||
|
||||
const handleConfirm = () => {
|
||||
if (isVideoMode) {
|
||||
if (!model) {
|
||||
message.warning("请选择视频模型");
|
||||
return;
|
||||
}
|
||||
if (!segments.length) {
|
||||
message.warning("请至少添加一个截取片段");
|
||||
return;
|
||||
}
|
||||
for (const segment of segments) {
|
||||
if (segment.endMs - segment.startMs < MIN_SEGMENT_MS) {
|
||||
message.warning("片段时长至少 0.1 秒");
|
||||
return;
|
||||
}
|
||||
}
|
||||
onConfirm({
|
||||
mode: "video",
|
||||
startMs: 0,
|
||||
endMs: 0,
|
||||
segments: segments.map(({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl }) => ({ id, startMs, endMs, sourceNodeId, sourceStorageKey, sourceUrl })),
|
||||
model,
|
||||
operation,
|
||||
prompt: prompt.trim(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!durationMs) {
|
||||
message.warning("视频时长未就绪，请稍候再试");
|
||||
return;
|
||||
}
|
||||
if (rangeMs.endMs - rangeMs.startMs < MIN_SEGMENT_MS) {
|
||||
message.warning("片段时长至少 0.1 秒");
|
||||
return;
|
||||
}
|
||||
onConfirm({ mode: "audio", startMs: rangeMs.startMs, endMs: rangeMs.endMs });
|
||||
};
|
||||
|
||||
const title = (
|
||||
<div className="flex min-w-0 items-center gap-2.5">
|
||||
<span className="grid size-8 shrink-0 place-items-center rounded-lg" style={{ background: theme.accent.primarySoft, color: theme.accent.primary }}>
|
||||
{isVideoMode ? <Scissors className="size-4" /> : <AudioLines className="size-4" />}
|
||||
</span>
|
||||
<div className="min-w-0">
|
||||
<div className="truncate font-semibold leading-6">{isVideoMode ? "截取片段并批量重生成" : "从视频提取声音"}</div>
|
||||
<div className="truncate text-xs opacity-45">{node.title || "视频节点"}</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<Modal title={title} open={open} onCancel={onClose} footer={null} width={680} centered destroyOnHidden>
|
||||
<div className="space-y-4">
|
||||
<div className="flex min-h-0 items-center justify-center overflow-hidden rounded-xl border bg-black" style={{ borderColor: theme.toolbar.border }}>
|
||||
{videoUrl ? (
|
||||
<video
|
||||
ref={videoRef}
|
||||
src={videoUrl}
|
||||
controls
|
||||
playsInline
|
||||
preload="metadata"
|
||||
className="block max-h-[var(--video-segment-preview-max-height)] w-full"
|
||||
onLoadedMetadata={(event) => {
|
||||
const video = event.currentTarget;
|
||||
if (Number.isFinite(video.duration) && video.duration > 0) {
|
||||
const totalMs = Math.round(video.duration * 1000);
|
||||
setDurationMs(totalMs);
|
||||
setEndSec(totalMs / 1000);
|
||||
}
|
||||
}}
|
||||
onError={() => setVideoError(true)}
|
||||
/>
|
||||
) : (
|
||||
<div className="grid h-40 w-full place-items-center text-xs opacity-60">{videoError ? "视频预览加载失败，请检查素材是否仍然可用" : "正在加载视频…"}</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isVideoMode ? (
|
||||
<div className="space-y-2.5">
|
||||
{!defaultModelSupported ? (
|
||||
<div className="flex flex-wrap items-center justify-between gap-2 rounded-lg border px-3 py-2 text-xs" style={{ background: theme.accent.primarySoft + "1a", borderColor: theme.accent.primarySoft, color: theme.node.muted }}>
|
||||
<span className="min-w-0 flex-1">
|
||||
{hasEligibleModels
|
||||
? `当前默认模型${defaultModel ? `「${modelDisplayName(config, defaultModel)}」` : ""}不支持参考视频，已自动切换到「${modelDisplayName(config, firstEligibleModel)}」。`
|
||||
: "当前配置中没有支持参考视频的视频模型，请先到设置里配置 Seedance / Agent Plan / NewAPI 渠道。"}
|
||||
</span>
|
||||
{!hasEligibleModels ? (
|
||||
<Button size="small" type="primary" onClick={() => navigateToSettings({ section: "channels", continueCreation: true })}>
|
||||
去设置配置渠道
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="flex items-center gap-2 text-sm">
|
||||
<span className="opacity-60">时长</span>
|
||||
<span>{durationSec ? formatSegmentTime(durationSec) : "未知"}</span>
|
||||
<span className="opacity-60">已选</span>
|
||||
<span>{segments.length} 段</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button size="small" icon={<Plus className="size-3.5" />} disabled={!durationSec} onClick={addManualSegment}>
|
||||
添加片段
|
||||
</Button>
|
||||
<Button size="small" icon={<ListVideo className="size-3.5" />} disabled={!hasTimelineVideoClips} onClick={importTimelineSegments}>
|
||||
从时间线导入
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{segments.length ? (
|
||||
<div className="thin-scrollbar max-h-56 space-y-2 overflow-y-auto pr-1">
|
||||
{segments.map((segment, index) => (
|
||||
<div key={segment.id} className="rounded-lg border px-2.5 py-2" style={{ borderColor: theme.toolbar.border }}>
|
||||
<div className="grid grid-cols-[auto_1fr_1fr_auto] items-center gap-2">
|
||||
<span className="w-14 shrink-0 text-xs font-medium">片段 {index + 1}</span>
|
||||
<div className="flex min-w-0 items-center gap-1">
|
||||
<span className="shrink-0 text-xs opacity-60">起点</span>
|
||||
<InputNumber
|
||||
size="small"
|
||||
min={0}
|
||||
max={Math.max(0, durationSec - 0.1)}
|
||||
step={0.1}
|
||||
value={segment.startMs / 1000}
|
||||
onChange={(value) => updateSegment(segment.id, { startMs: Math.round((value ?? 0) * 1000) })}
|
||||
className="w-full"
|
||||
aria-label={`片段 ${index + 1} 起点（秒）`}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center gap-1">
|
||||
<span className="shrink-0 text-xs opacity-60">终点</span>
|
||||
<InputNumber
|
||||
size="small"
|
||||
min={0}
|
||||
max={Math.max(0, durationSec)}
|
||||
step={0.1}
|
||||
value={segment.endMs / 1000}
|
||||
onChange={(value) => updateSegment(segment.id, { endMs: Math.round((value ?? 0) * 1000) })}
|
||||
className="w-full"
|
||||
aria-label={`片段 ${index + 1} 终点（秒）`}
|
||||
/>
|
||||
</div>
|
||||
<Button size="small" type="text" danger icon={<Trash2 className="size-3.5" />} aria-label={`删除片段 ${index + 1}`} onClick={() => removeSegment(segment.id)} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-lg border border-dashed px-3 py-3 text-xs opacity-55" style={{ borderColor: theme.toolbar.border }}>
|
||||
还没有片段：手动添加起点/终点，或从时间线导入该节点的已有片段。
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid gap-3 md:grid-cols-2">
|
||||
<label className="block min-w-0">
|
||||
<div className="mb-1.5 text-sm font-medium">重生成模型</div>
|
||||
<ModelPicker config={config} value={model} onChange={setModel} capability="video" fullWidth onMissingConfig={() => message.warning("请先配置支持参考视频的视频模型")} />
|
||||
</label>
|
||||
<label className="block min-w-0">
|
||||
<div className="mb-1.5 text-sm font-medium">生成模式</div>
|
||||
<Select className="w-full" size="small" value={operation} options={operationOptions} placeholder="选择生成模式" onChange={(value) => setOperation(value as CanvasVideoEditOperation)} />
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<label className="block">
|
||||
<div className="mb-1.5 text-sm font-medium">生成提示词</div>
|
||||
<Input.TextArea autoSize={{ minRows: 2, maxRows: 4 }} value={prompt} placeholder="描述要生成的视频内容，例如：保持画面主体与镜头，重新生成这一段视频" onChange={(event) => setPrompt(event.target.value)} />
|
||||
<div className="mt-1 text-xs opacity-45">每个片段都会生成独立片段节点和新生成结果节点；片段将作为参考视频送入所选模型，时长需符合模型参考视频限制（Seedance 单段 2-15 秒）。</div>
|
||||
</label>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-wrap items-center gap-x-6 gap-y-2 rounded-lg border px-3 py-2 text-sm" style={{ borderColor: theme.toolbar.border }}>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="opacity-60">时长</span>
|
||||
<span>{durationSec ? formatSegmentTime(durationSec) : "未知"}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="opacity-60">起点</span>
|
||||
<InputNumber size="small" min={0} max={Math.max(0, durationSec - 0.1)} step={0.1} value={startSec} onChange={(value) => setStartSec(value ?? 0)} className="w-28" aria-label="片段起点（秒）" />
|
||||
<Button
|
||||
size="small"
|
||||
type="text"
|
||||
icon={<SkipBack className="size-3.5" />}
|
||||
aria-label="跳转到起点"
|
||||
onClick={() => {
|
||||
if (videoRef.current) videoRef.current.currentTime = startSec;
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="opacity-60">终点</span>
|
||||
<InputNumber size="small" min={0} max={Math.max(0, durationSec)} step={0.1} value={endSec} onChange={(value) => setEndSec(value ?? 0)} className="w-28" aria-label="片段终点（秒）" />
|
||||
<Button
|
||||
size="small"
|
||||
type="text"
|
||||
icon={<SkipForward className="size-3.5" />}
|
||||
aria-label="跳转到终点"
|
||||
onClick={() => {
|
||||
if (videoRef.current) videoRef.current.currentTime = endSec;
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="opacity-60">片段</span>
|
||||
<span>{formatSegmentTime((rangeMs.endMs - rangeMs.startMs) / 1000)}</span>
|
||||
</div>
|
||||
<Button
|
||||
size="small"
|
||||
disabled={!durationSec}
|
||||
onClick={() => {
|
||||
setStartSec(0);
|
||||
setEndSec(durationSec);
|
||||
}}
|
||||
>
|
||||
使用全部
|
||||
</Button>
|
||||
<div className="w-full text-xs opacity-45">提取的 MP3 会保存到素材库，并生成一个音频节点放在当前视频节点下游。</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center justify-end gap-2">
|
||||
<Button onClick={onClose}>取消</Button>
|
||||
<Button type="primary" icon={<Check className="size-4" />} disabled={isVideoMode ? !segments.length : !durationSec} onClick={handleConfirm}>
|
||||
{isVideoMode ? (segments.length > 1 ? `截取 ${segments.length} 段并生成` : "截取并生成") : "提取音频"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
function videoOperationLabel(operation: CanvasVideoEditOperation) {
|
||||
const labels: Record<CanvasVideoEditOperation, string> = {
|
||||
text_to_video: "文生视频",
|
||||
image_to_video: "图生视频",
|
||||
extend: "视频续写",
|
||||
inpaint: "局部修改",
|
||||
replace_element: "元素替换",
|
||||
camera_motion: "运镜调整",
|
||||
style_transfer: "风格迁移",
|
||||
audio_to_video: "音频生视频",
|
||||
compare_versions: "版本对比",
|
||||
concat: "拼接成片",
|
||||
};
|
||||
return labels[operation] || operation;
|
||||
}
|
||||
|
||||
function formatSegmentTime(seconds: number) {
|
||||
const total = Math.max(0, seconds);
|
||||
const hours = Math.floor(total / 3600);
|
||||
const minutes = Math.floor((total % 3600) / 60);
|
||||
const rawSeconds = total % 60;
|
||||
const secs = rawSeconds % 1 === 0 ? String(Math.floor(rawSeconds)).padStart(2, "0") : rawSeconds.toFixed(1).padStart(4, "0");
|
||||
return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${secs}` : `${minutes}:${secs}`;
|
||||
}
|
||||
@@ -24,6 +24,8 @@ const settingsMockContext: ToolContext = {
|
||||
canUndo: false,
|
||||
canRedo: false,
|
||||
extractingVideoFrame: false,
|
||||
extractingAudio: false,
|
||||
trimmingVideo: false,
|
||||
mergingVideos: false,
|
||||
addPanelOpen: false,
|
||||
appearancePanelOpen: false,
|
||||
|
||||
@@ -52,6 +52,22 @@ export function getGenerationResourceNodes(nodeId: string, nodes: CanvasNodeData
|
||||
return [];
|
||||
}
|
||||
|
||||
/** 收集节点自身及其上游链路中的视频节点，用于时间线片段导入定位真正的视频源。 */
|
||||
export function collectUpstreamVideoNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): CanvasNodeData[] {
|
||||
const queue = [nodeId];
|
||||
const visited = new Set<string>();
|
||||
const result: CanvasNodeData[] = [];
|
||||
while (queue.length) {
|
||||
const currentId = queue.shift()!;
|
||||
if (visited.has(currentId)) continue;
|
||||
visited.add(currentId);
|
||||
const node = nodes.find((item) => item.id === currentId);
|
||||
if (node?.type === CanvasNodeType.Video && Boolean(node.metadata?.content || node.metadata?.storageKey)) result.push(node);
|
||||
connections.filter((connection) => connection.toNodeId === currentId).forEach((connection) => queue.push(connection.fromNodeId));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function getContextResourceNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
|
||||
return connections
|
||||
.filter((connection) => connection.toNodeId === nodeId)
|
||||
|
||||
54
web/src/lib/canvas/canvas-video-regeneration.ts
Normal file
54
web/src/lib/canvas/canvas-video-regeneration.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
|
||||
import { isSeedanceVideoConfig } from "@/lib/seedance-video";
|
||||
import type { CanvasVideoEditOperation } from "@/types/canvas";
|
||||
import { resolveModelChannel, selectableModelsByCapability, type AiConfig } from "@/stores/use-config-store";
|
||||
|
||||
export function listVideoReferenceModels(config: AiConfig): string[] {
|
||||
return selectableModelsByCapability(config, "video").filter((model) => {
|
||||
const profile = modelCapabilityConfigFor(config, model).video;
|
||||
if (!profile || profile.references.maxVideos < 1 || !profile.operations.length) return false;
|
||||
const channel = resolveModelChannel(config, model);
|
||||
return Boolean(channel.baseUrl.trim() && channel.apiKey.trim());
|
||||
});
|
||||
}
|
||||
|
||||
export function videoReferenceRegenerationError(config: AiConfig): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
if (!videoProfile || videoProfile.references.maxVideos < 1) {
|
||||
return "当前所选视频模型不支持参考视频，无法使用截取重生成。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。";
|
||||
}
|
||||
if (!videoProfile.operations.length) return "当前视频模型没有可用的视频生成模式";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function videoReferenceOperationError(config: AiConfig, operation: CanvasVideoEditOperation): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
if (!videoProfile?.operations.includes(operation)) return "当前视频模型不支持所选生成模式";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function validateVideoSegmentBatch(config: AiConfig, segments: Array<{ startMs: number; endMs: number }>, operation?: CanvasVideoEditOperation): string {
|
||||
const referenceError = videoReferenceRegenerationError(config);
|
||||
if (referenceError) return referenceError;
|
||||
if (operation) {
|
||||
const operationError = videoReferenceOperationError(config, operation);
|
||||
if (operationError) return operationError;
|
||||
}
|
||||
for (const segment of segments) {
|
||||
const segmentError = videoReferenceSegmentError(config, segment.endMs - segment.startMs);
|
||||
if (segmentError) return segmentError;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function videoReferenceSegmentError(config: AiConfig, durationMs: number): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
const maxSeconds = videoProfile?.references.maxVideoDurationSeconds || 0;
|
||||
if (maxSeconds > 0 && durationMs > maxSeconds * 1000) {
|
||||
return `截取片段不能超过当前模型参考视频上限（${maxSeconds} 秒）`;
|
||||
}
|
||||
if (isSeedanceVideoConfig(config) && durationMs < 2000) {
|
||||
return "Seedance 参考视频单段至少 2 秒，请扩大截取范围";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
104
web/src/lib/canvas/canvas-video-segment.ts
Normal file
104
web/src/lib/canvas/canvas-video-segment.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { fetchFile } from "@ffmpeg/util";
|
||||
|
||||
import { getMediaBlob } from "@/services/file-storage";
|
||||
import { loadFFmpeg } from "./canvas-video-merge";
|
||||
|
||||
export type VideoSegmentRange = {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
};
|
||||
|
||||
export type VideoSegmentSource = {
|
||||
url?: string;
|
||||
storageKey?: string;
|
||||
};
|
||||
|
||||
export type VideoSegmentProgress = {
|
||||
phase: "loading" | "reading" | "encoding";
|
||||
progress: number;
|
||||
};
|
||||
|
||||
const INPUT_NAME = "segment-input.mp4";
|
||||
const OUTPUT_NAME = "segment-output.mp4";
|
||||
|
||||
function assertValidRange(range: VideoSegmentRange, durationMs?: number) {
|
||||
const startMs = Math.max(0, Math.round(range.startMs));
|
||||
const endMs = Math.round(range.endMs);
|
||||
if (endMs <= startMs) throw new Error("片段结束时间必须晚于开始时间");
|
||||
if (durationMs !== undefined && endMs > Math.round(durationMs)) throw new Error("片段结束时间超过视频时长");
|
||||
}
|
||||
|
||||
async function readVideoSourceBlob(source: VideoSegmentSource) {
|
||||
if (source.storageKey) {
|
||||
const stored = await getMediaBlob(source.storageKey);
|
||||
if (stored) return stored;
|
||||
}
|
||||
if (source.url) {
|
||||
const response = await fetch(source.url);
|
||||
if (!response.ok) throw new Error(`视频资源请求失败（${response.status}）`);
|
||||
return response.blob();
|
||||
}
|
||||
throw new Error("找不到视频素材，请重新上传后再操作");
|
||||
}
|
||||
|
||||
async function runSegmentJob(
|
||||
source: VideoSegmentSource,
|
||||
range: VideoSegmentRange,
|
||||
durationMs: number | undefined,
|
||||
buildArgs: (startSec: string, durationSec: string) => string[],
|
||||
onProgress?: (progress: VideoSegmentProgress) => void,
|
||||
outputType = "video/mp4",
|
||||
) {
|
||||
assertValidRange(range, durationMs);
|
||||
const ffmpeg = await loadFFmpeg(({ phase, progress }) => onProgress?.({ phase: phase === "loading" ? "loading" : "reading", progress }));
|
||||
const blob = await readVideoSourceBlob(source);
|
||||
onProgress?.({ phase: "reading", progress: 45 });
|
||||
await ffmpeg.writeFile(INPUT_NAME, await fetchFile(blob));
|
||||
const startSec = String(range.startMs / 1000);
|
||||
const durationSec = String((range.endMs - range.startMs) / 1000);
|
||||
onProgress?.({ phase: "encoding", progress: 55 });
|
||||
try {
|
||||
const exitCode = await ffmpeg.exec(["-y", ...buildArgs(startSec, durationSec)]);
|
||||
if (exitCode !== 0) throw new Error("媒体处理失败，请确认视频编码格式兼容");
|
||||
const output = await ffmpeg.readFile(OUTPUT_NAME);
|
||||
onProgress?.({ phase: "encoding", progress: 100 });
|
||||
return new Blob([output as BlobPart], { type: outputType });
|
||||
} finally {
|
||||
await Promise.all([INPUT_NAME, OUTPUT_NAME].map((file) => ffmpeg.deleteFile(file).catch(() => undefined)));
|
||||
}
|
||||
}
|
||||
|
||||
/** 按片段范围截取视频，输出统一编码 MP4（复用时间线 trim 的参数模板）。 */
|
||||
export async function trimVideoSegment(source: VideoSegmentSource, range: VideoSegmentRange, durationMs?: number, onProgress?: (progress: VideoSegmentProgress) => void) {
|
||||
return runSegmentJob(
|
||||
source,
|
||||
range,
|
||||
durationMs,
|
||||
(startSec, durationSec) => ["-ss", startSec, "-i", INPUT_NAME, "-t", durationSec, "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-movflags", "+faststart", OUTPUT_NAME],
|
||||
onProgress,
|
||||
"video/mp4",
|
||||
);
|
||||
}
|
||||
|
||||
/** 从视频片段提取声音为 MP3；优先 libmp3lame，内核不支持时回退默认 mp3 编码器。 */
|
||||
export async function extractVideoAudio(source: VideoSegmentSource, range: VideoSegmentRange, durationMs?: number, onProgress?: (progress: VideoSegmentProgress) => void) {
|
||||
const ffmpeg = await loadFFmpeg(({ phase, progress }) => onProgress?.({ phase: phase === "loading" ? "loading" : "reading", progress }));
|
||||
const blob = await readVideoSourceBlob(source);
|
||||
onProgress?.({ phase: "reading", progress: 45 });
|
||||
await ffmpeg.writeFile(INPUT_NAME, await fetchFile(blob));
|
||||
assertValidRange(range, durationMs);
|
||||
const startSec = String(range.startMs / 1000);
|
||||
const durationSec = String((range.endMs - range.startMs) / 1000);
|
||||
onProgress?.({ phase: "encoding", progress: 55 });
|
||||
try {
|
||||
const args = (audioCodec: string) => ["-ss", startSec, "-i", INPUT_NAME, "-t", durationSec, "-vn", "-c:a", audioCodec, "-q:a", "2", OUTPUT_NAME];
|
||||
let exitCode = await ffmpeg.exec(["-y", ...args("libmp3lame")]);
|
||||
if (exitCode !== 0) exitCode = await ffmpeg.exec(["-y", ...args("mp3")]);
|
||||
if (exitCode !== 0) throw new Error("音频提取失败：当前 FFmpeg 内核不支持 MP3 编码");
|
||||
const output = await ffmpeg.readFile(OUTPUT_NAME);
|
||||
onProgress?.({ phase: "encoding", progress: 100 });
|
||||
return new Blob([output as BlobPart], { type: "audio/mpeg" });
|
||||
} finally {
|
||||
await Promise.all([INPUT_NAME, OUTPUT_NAME].map((file) => ffmpeg.deleteFile(file).catch(() => undefined)));
|
||||
}
|
||||
}
|
||||
54
web/src/lib/canvas/canvas-video-timeline-segments.ts
Normal file
54
web/src/lib/canvas/canvas-video-timeline-segments.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { collectUpstreamVideoNodes } from "@/lib/canvas/canvas-resource-references";
|
||||
import type { CanvasConnection, CanvasNodeData } from "@/types/canvas";
|
||||
import type { TimelineClip, TimelineProject } from "@/types/timeline";
|
||||
|
||||
export type CanvasTimelineSegmentItem = {
|
||||
id: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
sourceNodeId?: string;
|
||||
sourceStorageKey?: string;
|
||||
sourceUrl?: string;
|
||||
};
|
||||
|
||||
export type TimelineImportResult = { ok: true; segments: CanvasTimelineSegmentItem[] } | { ok: false; error: string };
|
||||
|
||||
const MIN_TIMELINE_SEGMENT_MS = 100;
|
||||
|
||||
export function buildTimelineImportSegments(node: CanvasNodeData, nodes: CanvasNodeData[], connections: CanvasConnection[], timeline: TimelineProject | null | undefined, durationMs: number): TimelineImportResult {
|
||||
const candidateNodes = collectUpstreamVideoNodes(node.id, nodes, connections);
|
||||
const matchesCandidate = (clip: TimelineClip, candidate: CanvasNodeData) => {
|
||||
if (candidate.id === clip.nodeId) return true;
|
||||
if (candidate.metadata?.assetId && clip.nodeId === candidate.metadata.assetId) return true;
|
||||
if (candidate.metadata?.storageKey && clip.directMedia?.storageKey === candidate.metadata.storageKey) return true;
|
||||
// 旧数据里 nodeId 可能被重新生成过；标题一致且唯一时才回退，避免同名视频误导入。
|
||||
if (candidate.title && clip.title === candidate.title && !clip.directMedia) {
|
||||
return candidateNodes.filter((item) => item.title === candidate.title).length === 1;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const clips = (timeline?.clips || []).filter((clip) => clip.kind === "video" && candidateNodes.some((candidate) => matchesCandidate(clip, candidate)));
|
||||
if (!clips.length) {
|
||||
return { ok: false, error: `未找到「${node.title || "当前视频节点"}」或其上游视频节点在时间线中的片段，请先在时间线中加入对应视频节点` };
|
||||
}
|
||||
const segments = clips.flatMap((clip) => {
|
||||
const matchedNode = candidateNodes.find((candidate) => matchesCandidate(clip, candidate));
|
||||
const startMs = Math.max(0, Math.round(clip.sourceStartMs ?? 0));
|
||||
const rawEndMs = startMs + Math.max(MIN_TIMELINE_SEGMENT_MS, Math.round(clip.sourceDurationMs || clip.durationMs));
|
||||
const sourceDurationMs = matchedNode?.metadata?.durationMs || durationMs;
|
||||
const endMs = sourceDurationMs > 0 ? Math.min(sourceDurationMs, rawEndMs) : rawEndMs;
|
||||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs) return [];
|
||||
return [
|
||||
{
|
||||
id: `timeline-${clip.id}`,
|
||||
startMs,
|
||||
endMs,
|
||||
sourceNodeId: matchedNode?.id,
|
||||
sourceStorageKey: clip.directMedia?.storageKey || matchedNode?.metadata?.storageKey,
|
||||
sourceUrl: clip.directMedia?.url || matchedNode?.metadata?.content,
|
||||
},
|
||||
];
|
||||
});
|
||||
if (!segments.length) return { ok: false, error: "时间线片段数据不完整，请先在时间线中重新分割或保存" };
|
||||
return { ok: true, segments };
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
||||
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
||||
|
||||
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
|
||||
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
|
||||
@@ -73,6 +73,32 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
|
||||
disabled: (ctx) => ctx.extractingVideoFrame,
|
||||
run: (ctx) => ctx.handlers.onNodeExtractVideoLastFrame(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "extractAudio",
|
||||
toolbar: "node-hover",
|
||||
category: "node-state",
|
||||
label: (ctx) => ctx.extractingAudio ? "正在提取声音为 MP3" : "提取声音为 MP3",
|
||||
displayLabel: (ctx) => ctx.extractingAudio ? "提取中" : "提取音频",
|
||||
icon: (ctx) => ctx.extractingAudio ? <LoaderCircle className="size-3.5 animate-spin" /> : <AudioLines className="size-3.5" />,
|
||||
defaultVisible: true,
|
||||
defaultOrder: 42,
|
||||
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
|
||||
disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo,
|
||||
run: (ctx) => ctx.handlers.onNodeExtractAudioFromVideo(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "trimRegenerate",
|
||||
toolbar: "node-hover",
|
||||
category: "node-state",
|
||||
label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "按段截取并重生成",
|
||||
displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取重生成",
|
||||
icon: (ctx) => ctx.trimmingVideo ? <LoaderCircle className="size-3.5 animate-spin" /> : <Scissors className="size-3.5" />,
|
||||
defaultVisible: true,
|
||||
defaultOrder: 44,
|
||||
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
|
||||
disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo,
|
||||
run: (ctx) => ctx.handlers.onNodeTrimVideoRegenerate(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "saveAsset",
|
||||
toolbar: "node-hover",
|
||||
|
||||
@@ -81,6 +81,8 @@ export type ToolbarHandlers = {
|
||||
onNodeAngle: (node: CanvasNodeData) => void;
|
||||
onNodeViewImage: (node: CanvasNodeData) => void;
|
||||
onNodeExtractVideoLastFrame: (node: CanvasNodeData) => void;
|
||||
onNodeExtractAudioFromVideo: (node: CanvasNodeData) => void;
|
||||
onNodeTrimVideoRegenerate: (node: CanvasNodeData) => void;
|
||||
onNodeSubtitles: (node: CanvasNodeData) => void;
|
||||
onNodeTimeline: (node: CanvasNodeData) => void;
|
||||
onNodeReversePrompt: (node: CanvasNodeData) => void;
|
||||
@@ -105,6 +107,9 @@ export type ToolContext = {
|
||||
nodeMetadata?: CanvasNodeMetadata;
|
||||
/** 视频尾帧提取中（节点悬停工具栏用） */
|
||||
extractingVideoFrame: boolean;
|
||||
/** 视频音频提取/片段截取进行中（节点悬停工具栏用） */
|
||||
extractingAudio: boolean;
|
||||
trimmingVideo: boolean;
|
||||
/** 合并视频中（多选工具栏用） */
|
||||
mergingVideos: boolean;
|
||||
/** 主工具栏面板开关状态（仅主工具栏使用） */
|
||||
|
||||
@@ -38,6 +38,8 @@ export function CanvasProjectSelectionToolbar({ anchorRef, containerRef, count,
|
||||
canUndo: false,
|
||||
canRedo: false,
|
||||
extractingVideoFrame: false,
|
||||
extractingAudio: false,
|
||||
trimmingVideo: false,
|
||||
mergingVideos,
|
||||
addPanelOpen: false,
|
||||
appearancePanelOpen: false,
|
||||
|
||||
@@ -31,6 +31,7 @@ import { resolveProjectCanvasStyle } from "@/components/canvas/canvas-style-pick
|
||||
import { createStyleProfileSnapshot, resolveStyleProfile, serializeStyleProfile } from "@/lib/canvas/style-profile";
|
||||
import { CanvasNodeToolbar, CanvasNodeInfoModal } from "@/components/canvas/canvas-node-toolbar";
|
||||
import { CanvasSubtitleDialog } from "@/components/canvas/canvas-subtitle-dialog";
|
||||
import { CanvasVideoSegmentDialog } from "@/components/canvas/canvas-video-segment-dialog";
|
||||
import { CanvasTimelineDialog } from "@/components/canvas/canvas-timeline-dialog";
|
||||
import { syncNodeSubtitlesToTimeline } from "@/lib/timeline/timeline-build";
|
||||
import type { TimelineDirectMedia } from "@/types/timeline";
|
||||
@@ -583,16 +584,23 @@ function InfiniteCanvasPage() {
|
||||
generatePortraitTextureNode,
|
||||
cropImageNode,
|
||||
cropNodeId,
|
||||
closeSegmentDialog,
|
||||
extractAudioFromVideo,
|
||||
extractVideoLastFrame,
|
||||
extractingVideoFrameNodeId,
|
||||
generateAngleNode,
|
||||
generateEmotionNode,
|
||||
handleSegmentConfirm,
|
||||
maskEditImageNode,
|
||||
maskEditNodeId,
|
||||
mergeSelectedVideos,
|
||||
mergeVideosByIds,
|
||||
mergeVideoProgress,
|
||||
saveAnnotatedImageNode,
|
||||
segmentDialogMode,
|
||||
segmentDialogNodeId,
|
||||
segmentRunningMode,
|
||||
setSegmentDialogNodeId,
|
||||
setAngleNodeId,
|
||||
setEmotionNodeId,
|
||||
setAnnotationNodeId,
|
||||
@@ -602,10 +610,12 @@ function InfiniteCanvasPage() {
|
||||
setUpscaleNodeId,
|
||||
splitImageNode,
|
||||
splitNodeId,
|
||||
trimVideoAndRegenerate,
|
||||
upscaleImageNode,
|
||||
upscaleNodeId,
|
||||
} = useCanvasMediaTools({
|
||||
projectId,
|
||||
domainProjectId: linkedProjectId,
|
||||
nodesRef,
|
||||
connectionsRef,
|
||||
selectedNodeIdsRef,
|
||||
@@ -622,6 +632,7 @@ function InfiniteCanvasPage() {
|
||||
startGenerationRequest,
|
||||
finishGenerationRequest,
|
||||
bindGenerationTask,
|
||||
onGenerateVideoNode: (nodeId, mode, prompt) => generateNodeRef.current?.(nodeId, mode, prompt),
|
||||
});
|
||||
|
||||
const handleNodesDeleted = useCallback(
|
||||
@@ -635,6 +646,7 @@ function InfiniteCanvasPage() {
|
||||
setDrawingNodeId(clearDeletedId);
|
||||
setInfoNodeId(clearDeletedId);
|
||||
setSubtitleNodeId(clearDeletedId);
|
||||
setSegmentDialogNodeId(clearDeletedId);
|
||||
setCropNodeId(clearDeletedId);
|
||||
setMaskEditNodeId(clearDeletedId);
|
||||
setAnnotationNodeId(clearDeletedId);
|
||||
@@ -656,7 +668,7 @@ function InfiniteCanvasPage() {
|
||||
}
|
||||
cleanupCanvasFiles({ projectId, nodes: nextNodes, chatSessions });
|
||||
},
|
||||
[chatSessions, cleanupCanvasFiles, message, projectId, setAngleNodeId, setAnnotationNodeId, setCropNodeId, setEmotionNodeId, setMaskEditNodeId, setSplitNodeId, setUpscaleNodeId, setRunningNodeId],
|
||||
[chatSessions, cleanupCanvasFiles, message, projectId, setAngleNodeId, setAnnotationNodeId, setCropNodeId, setEmotionNodeId, setMaskEditNodeId, setSegmentDialogNodeId, setSplitNodeId, setUpscaleNodeId, setRunningNodeId],
|
||||
);
|
||||
|
||||
const {
|
||||
@@ -875,6 +887,7 @@ function InfiniteCanvasPage() {
|
||||
const dialogNode = dialogNodeId ? nodeById.get(dialogNodeId) || null : null;
|
||||
const subtitleNode = subtitleNodeId ? nodeById.get(subtitleNodeId) || null : null;
|
||||
const timelineNode = timelineNodeId ? nodeById.get(timelineNodeId) || null : null;
|
||||
const segmentNode = segmentDialogNodeId ? nodeById.get(segmentDialogNodeId) || null : null;
|
||||
const textEditorNode = textEditorNodeId ? nodeById.get(textEditorNodeId) || null : null;
|
||||
const characterReferenceNode = characterReferenceNodeId ? nodeById.get(characterReferenceNodeId) || null : null;
|
||||
const drawingNode = drawingNodeId ? nodeById.get(drawingNodeId) || null : null;
|
||||
@@ -1840,9 +1853,13 @@ function InfiniteCanvasPage() {
|
||||
}}
|
||||
onViewImage={(node) => setPreviewNodeId(node.id)}
|
||||
onExtractVideoLastFrame={(node) => void extractVideoLastFrame(node)}
|
||||
onExtractAudioFromVideo={(node) => void extractAudioFromVideo(node)}
|
||||
onTrimVideoRegenerate={(node) => void trimVideoAndRegenerate(node)}
|
||||
onSubtitles={(node) => setSubtitleNodeId(node.id)}
|
||||
onTimeline={(node) => setTimelineNodeId(node.id)}
|
||||
extractingVideoFrame={toolbarNode?.id === extractingVideoFrameNodeId}
|
||||
extractingAudio={segmentRunningMode === "audio"}
|
||||
trimmingVideo={segmentRunningMode === "video"}
|
||||
onReversePrompt={createImageReversePromptNodes}
|
||||
onRetry={(node) => void handleRetryNode(node)}
|
||||
onToggleFreeResize={(node) => toggleNodeFreeResize(node.id)}
|
||||
@@ -1944,6 +1961,20 @@ function InfiniteCanvasPage() {
|
||||
/>
|
||||
) : null}
|
||||
|
||||
{segmentNode && segmentDialogMode ? (
|
||||
<CanvasVideoSegmentDialog
|
||||
node={segmentNode}
|
||||
nodes={nodes}
|
||||
connections={connections}
|
||||
open={Boolean(segmentNode && segmentDialogMode)}
|
||||
mode={segmentDialogMode}
|
||||
config={effectiveConfig}
|
||||
timeline={currentProject?.timeline || null}
|
||||
onClose={closeSegmentDialog}
|
||||
onConfirm={(params) => void handleSegmentConfirm(segmentNode, params)}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
{timelineNode ? (
|
||||
<CanvasTimelineDialog
|
||||
node={timelineNode}
|
||||
|
||||
@@ -256,7 +256,7 @@ export default function SharedCanvasPage() {
|
||||
}} onHoverStart={keepToolbar} onHoverEnd={hideToolbar} onConnectStart={unauthorized} onResize={() => undefined} onContentChange={unauthorized} onRetry={unauthorized} onCancelTask={unauthorized} onOpenTaskDetails={unauthorized} onViewImage={(target) => setInfoNodeId(target.id)} onContextMenu={(event, nodeId) => openContextMenu(event, nodeId)} />)}
|
||||
</InfiniteCanvas>
|
||||
|
||||
<CanvasNodeToolbar node={dragRef.current ? null : toolbarNode} viewport={viewport} containerRef={containerRef} onKeep={keepToolbar} onLeave={hideToolbar} onInfo={(node) => setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoLastFrame={unauthorized} extractingVideoFrame={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} />
|
||||
<CanvasNodeToolbar node={dragRef.current ? null : toolbarNode} viewport={viewport} containerRef={containerRef} onKeep={keepToolbar} onLeave={hideToolbar} onInfo={(node) => setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoLastFrame={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoRegenerate={unauthorized} extractingVideoFrame={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} />
|
||||
|
||||
<div className="absolute bottom-5 left-5 z-[var(--z-panel-floating)]"><CanvasZoomControls scale={viewport.k} containerRef={containerRef} onScaleChange={setZoom} onReset={resetViewport} isMiniMapOpen={false} onToggleMiniMap={unauthorized} onOpenShortcuts={unauthorized} /></div>
|
||||
<div className="pointer-events-none absolute bottom-5 right-5 z-[var(--z-panel-floating)] max-w-[340px] text-right text-xs leading-5" style={{ color: theme.node.muted }}>访客操作仅在当前页面临时生效</div>
|
||||
|
||||
@@ -8,10 +8,12 @@ import type { CanvasImageSplitParams } from "@/components/canvas/canvas-node-spl
|
||||
import type { CanvasImageUpscaleParams } from "@/components/canvas/canvas-node-upscale-dialog";
|
||||
import type { CanvasImageAngleParams } from "@/components/canvas/canvas-node-angle-dialog";
|
||||
import type { CanvasImageEmotionPayload } from "@/components/canvas/canvas-node-emotion-panel";
|
||||
import type { CanvasVideoSegmentParams } from "@/components/canvas/canvas-video-segment-dialog";
|
||||
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
|
||||
import { cropDataUrl, splitDataUrl, upscaleDataUrl } from "@/lib/canvas/canvas-image-data";
|
||||
import { imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
|
||||
import { audioMetadata, imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
|
||||
import { buildAngleLabel, buildAnglePrompt, createCanvasNode } from "@/lib/canvas/canvas-project-domain";
|
||||
import { validateVideoSegmentBatch } from "@/lib/canvas/canvas-video-regeneration";
|
||||
import { resolveCanvasStyleExecution } from "@/lib/canvas/canvas-style-execution";
|
||||
import {
|
||||
buildGenerationConfig,
|
||||
@@ -25,11 +27,13 @@ import { compositeEmotionImage, emotionGenerationSize } from "@/lib/canvas/canva
|
||||
import { DEFAULT_PORTRAIT_TEXTURE_SETTINGS, buildPortraitTexturePrompt } from "@/lib/canvas/canvas-portrait-texture";
|
||||
import { captureVideoLastFrame } from "@/lib/canvas/canvas-video-frame";
|
||||
import { mergeVideos, type MergeVideoProgress } from "@/lib/canvas/canvas-video-merge";
|
||||
import { extractVideoAudio, trimVideoSegment } from "@/lib/canvas/canvas-video-segment";
|
||||
import { generationErrorMessage } from "@/lib/generation-error";
|
||||
import { navigateToSettings } from "@/lib/settings-navigation";
|
||||
import { storeGeneratedVideo } from "@/services/api/video";
|
||||
import { getMediaBlob } from "@/services/file-storage";
|
||||
import { getMediaBlob, uploadMediaFile } from "@/services/file-storage";
|
||||
import { uploadImage } from "@/services/image-storage";
|
||||
import { ensureCanvasNodeAsset } from "@/services/project-asset-sync";
|
||||
import type { GenerationTask } from "@/services/api/task-center";
|
||||
import { defaultConfig, resolveModelRequestConfig, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
||||
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData, type ContextMenuState } from "@/types/canvas";
|
||||
@@ -37,6 +41,7 @@ import type { StartCanvasUploadStatus } from "./use-canvas-upload";
|
||||
|
||||
type UseCanvasMediaToolsOptions = {
|
||||
projectId: string;
|
||||
domainProjectId?: string;
|
||||
nodesRef: { current: CanvasNodeData[] };
|
||||
connectionsRef: { current: CanvasConnection[] };
|
||||
selectedNodeIdsRef: { current: Set<string> };
|
||||
@@ -53,6 +58,8 @@ type UseCanvasMediaToolsOptions = {
|
||||
startGenerationRequest: (targetNodeId: string, originNodeId: string, runningId?: string, controller?: AbortController) => AbortController;
|
||||
finishGenerationRequest: (targetNodeId: string, controller: AbortController) => void;
|
||||
bindGenerationTask: (targetNodeId: string, task: GenerationTask) => void;
|
||||
/** 创建空视频节点后触发既有生成执行器（由页面层接线），用于“截取片段后调用视频模型重生成” */
|
||||
onGenerateVideoNode?: (nodeId: string, mode: "video", prompt: string) => Promise<void> | void;
|
||||
};
|
||||
|
||||
const NODE_STATUS_LOADING = "loading" as const;
|
||||
@@ -67,6 +74,7 @@ const IMAGE_PROMPT_REVERSE_PRESET = `请根据参考图片反推一段适合用
|
||||
|
||||
export function useCanvasMediaTools({
|
||||
projectId,
|
||||
domainProjectId,
|
||||
nodesRef,
|
||||
connectionsRef,
|
||||
selectedNodeIdsRef,
|
||||
@@ -83,6 +91,7 @@ export function useCanvasMediaTools({
|
||||
startGenerationRequest,
|
||||
finishGenerationRequest,
|
||||
bindGenerationTask,
|
||||
onGenerateVideoNode,
|
||||
}: UseCanvasMediaToolsOptions) {
|
||||
const { message } = App.useApp();
|
||||
const effectiveConfig = useEffectiveConfig();
|
||||
@@ -98,6 +107,10 @@ export function useCanvasMediaTools({
|
||||
const [emotionNodeId, setEmotionNodeId] = useState<string | null>(null);
|
||||
const [extractingVideoFrameNodeId, setExtractingVideoFrameNodeId] = useState<string | null>(null);
|
||||
const [mergeVideoProgress, setMergeVideoProgress] = useState<MergeVideoProgress | null>(null);
|
||||
const [segmentDialogNodeId, setSegmentDialogNodeId] = useState<string | null>(null);
|
||||
const [segmentDialogMode, setSegmentDialogMode] = useState<"audio" | "video" | null>(null);
|
||||
const [segmentRunningMode, setSegmentRunningMode] = useState<"audio" | "video" | null>(null);
|
||||
const segmentRunningRef = useRef(false);
|
||||
|
||||
const resolveImageEditStyle = useCallback((node: CanvasNodeData, prompt: string, config: AiConfig) => {
|
||||
try {
|
||||
@@ -259,6 +272,198 @@ export function useCanvasMediaTools({
|
||||
}
|
||||
}, [message, setConnections, setHoveredNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, setToolbarNodeId, startUploadStatus]);
|
||||
|
||||
const extractAudioFromVideo = useCallback((node: CanvasNodeData) => {
|
||||
if (!node.metadata?.content) {
|
||||
message.warning("视频节点为空，无法提取声音");
|
||||
return;
|
||||
}
|
||||
if (segmentRunningRef.current) return;
|
||||
setHoveredNodeId(null);
|
||||
setToolbarNodeId(null);
|
||||
setSegmentDialogNodeId(node.id);
|
||||
setSegmentDialogMode("audio");
|
||||
}, [message, setHoveredNodeId, setToolbarNodeId]);
|
||||
|
||||
const trimVideoAndRegenerate = useCallback((node: CanvasNodeData) => {
|
||||
if (!node.metadata?.content) {
|
||||
message.warning("视频节点为空，无法截取片段");
|
||||
return;
|
||||
}
|
||||
if (segmentRunningRef.current) return;
|
||||
setHoveredNodeId(null);
|
||||
setToolbarNodeId(null);
|
||||
setSegmentDialogNodeId(node.id);
|
||||
setSegmentDialogMode("video");
|
||||
}, [message, setHoveredNodeId, setToolbarNodeId]);
|
||||
|
||||
const closeSegmentDialog = useCallback(() => {
|
||||
if (segmentRunningRef.current) return;
|
||||
setSegmentDialogNodeId(null);
|
||||
setSegmentDialogMode(null);
|
||||
}, []);
|
||||
|
||||
// 从视频片段提取声音：FFmpeg 提取 MP3 → 上传为音频资源 → 创建音频节点 → 写入素材库/项目资产。
|
||||
const runExtractVideoAudio = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => {
|
||||
const progress = startUploadStatus("提取音频", "加载 FFmpeg", 4);
|
||||
try {
|
||||
const mp3 = await extractVideoAudio({ url: node.metadata?.content, storageKey: node.metadata?.storageKey }, { startMs: params.startMs, endMs: params.endMs }, node.metadata?.durationMs, (status) => {
|
||||
progress.update(status.phase === "loading" ? "加载 FFmpeg" : status.phase === "reading" ? "读取视频资源" : "正在提取音频", status.phase === "encoding" ? 3 : 2);
|
||||
});
|
||||
progress.update("上传音频到服务器", 4);
|
||||
const uploaded = await uploadMediaFile(mp3, "audio");
|
||||
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio];
|
||||
const audioNode = createCanvasNode(
|
||||
CanvasNodeType.Audio,
|
||||
{ x: node.position.x + node.width + 96 + spec.width / 2, y: node.position.y + node.height / 2 },
|
||||
{ ...audioMetadata(uploaded), prompt: `从「${node.title || "视频"}」提取的声音`, status: NODE_STATUS_SUCCESS },
|
||||
);
|
||||
audioNode.title = `声音 · ${node.title || "视频"}`;
|
||||
const audioNodeId = audioNode.id;
|
||||
setNodes((current) => [...current, audioNode]);
|
||||
setConnections((current) => [...current, { id: nanoid(), fromNodeId: node.id, toNodeId: audioNodeId }]);
|
||||
setSelectedNodeIds(new Set([audioNodeId]));
|
||||
setSelectedConnectionId(null);
|
||||
try {
|
||||
const result = await ensureCanvasNodeAsset({ canvasId: projectId, domainProjectId, node: audioNode, source: "canvas-manual" });
|
||||
setNodes((current) => current.map((item) => (item.id === audioNodeId ? { ...item, metadata: { ...item.metadata, assetId: result.assetId } } : item)));
|
||||
progress.done(result.linkedToProject ? "声音已提取并加入素材库与项目资产" : "声音已提取并加入素材库");
|
||||
} catch (assetError) {
|
||||
progress.done(`声音已提取并生成音频节点，素材库写入失败：${assetError instanceof Error ? assetError.message : "未知错误"}`);
|
||||
}
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : "音频提取失败";
|
||||
progress.fail(details);
|
||||
message.error(details);
|
||||
}
|
||||
}, [domainProjectId, message, projectId, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]);
|
||||
|
||||
// 按段截取视频：FFmpeg 批量截取 → 每段创建片段节点与空结果节点 → 调用视频模型逐段重生成。
|
||||
const runTrimVideoAndRegenerate = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => {
|
||||
const segments = params.segments || [];
|
||||
if (!segments.length) {
|
||||
message.warning("请至少添加一个截取片段");
|
||||
return;
|
||||
}
|
||||
const generationConfig = buildGenerationConfig(effectiveConfig, node, "video");
|
||||
const selectedConfig = { ...generationConfig, model: params.model || generationConfig.model };
|
||||
const batchError = validateVideoSegmentBatch(selectedConfig, segments, params.operation);
|
||||
if (batchError) {
|
||||
message.warning(batchError);
|
||||
return;
|
||||
}
|
||||
const progress = startUploadStatus("截取视频片段", "加载 FFmpeg", segments.length * 4);
|
||||
try {
|
||||
const prepared: Array<{ segmentNode: CanvasNodeData; targetNode: CanvasNodeData }> = [];
|
||||
const failedSegments: string[] = [];
|
||||
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
|
||||
const baseX = node.position.x + node.width + 96;
|
||||
const baseY = node.position.y;
|
||||
const effectivePrompt = (params.prompt || "保持画面主体与镜头，重新生成这一段视频").trim();
|
||||
for (let index = 0; index < segments.length; index += 1) {
|
||||
const segment = segments[index];
|
||||
try {
|
||||
const sourceNode = segment.sourceNodeId ? nodesRef.current.find((item) => item.id === segment.sourceNodeId) : undefined;
|
||||
const trimSource = sourceNode
|
||||
? { url: sourceNode.metadata?.content, storageKey: sourceNode.metadata?.storageKey }
|
||||
: segment.sourceStorageKey || segment.sourceUrl
|
||||
? { url: segment.sourceUrl, storageKey: segment.sourceStorageKey }
|
||||
: { url: node.metadata?.content, storageKey: node.metadata?.storageKey };
|
||||
const trimDurationMs = sourceNode?.metadata?.durationMs || node.metadata?.durationMs;
|
||||
progress.update(`加载 FFmpeg（${index + 1}/${segments.length}）`, index * 4 + 1);
|
||||
const mp4 = await trimVideoSegment(trimSource, { startMs: segment.startMs, endMs: segment.endMs }, trimDurationMs, (status) => {
|
||||
progress.update(status.phase === "loading" ? `加载 FFmpeg（${index + 1}/${segments.length}）` : status.phase === "reading" ? `读取视频资源（${index + 1}/${segments.length}）` : `正在截取片段（${index + 1}/${segments.length}）`, status.phase === "encoding" ? index * 4 + 3 : index * 4 + 2);
|
||||
});
|
||||
progress.update(`上传片段到服务器（${index + 1}/${segments.length}）`, index * 4 + 3);
|
||||
const uploaded = await uploadMediaFile(mp4, "video");
|
||||
const size = fitNodeSize(uploaded.width || 1280, uploaded.height || 720, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
|
||||
const segmentId = nanoid();
|
||||
const segmentNode: CanvasNodeData = {
|
||||
id: segmentId,
|
||||
type: CanvasNodeType.Video,
|
||||
title: `片段 ${index + 1} · ${sourceNode?.title || node.title || "视频"}`,
|
||||
position: { x: baseX, y: baseY + index * (Math.max(size.height, spec.height) + 24) },
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
metadata: { ...videoMetadata(uploaded), prompt: `从「${sourceNode?.title || node.title || "视频"}」截取的片段 ${index + 1}`, status: NODE_STATUS_SUCCESS },
|
||||
};
|
||||
const targetId = nanoid();
|
||||
const targetNode: CanvasNodeData = {
|
||||
id: targetId,
|
||||
type: CanvasNodeType.Video,
|
||||
title: `重生成 ${index + 1} · ${sourceNode?.title || node.title || "视频"}`,
|
||||
position: { x: segmentNode.position.x + size.width + 96, y: segmentNode.position.y + (size.height - spec.height) / 2 },
|
||||
width: spec.width,
|
||||
height: spec.height,
|
||||
metadata: { prompt: effectivePrompt, status: "idle", generationMode: "video", model: selectedConfig.model, videoEditOperation: params.operation, seconds: generationConfig.videoSeconds, size: generationConfig.size },
|
||||
};
|
||||
prepared.push({ segmentNode, targetNode });
|
||||
} catch (segmentError) {
|
||||
failedSegments.push(segmentError instanceof Error ? segmentError.message : "视频截取失败");
|
||||
}
|
||||
}
|
||||
if (!prepared.length) throw new Error(failedSegments[0] || "视频截取失败");
|
||||
const segmentNodes = prepared.map((item) => item.segmentNode);
|
||||
const targetNodes = prepared.map((item) => item.targetNode);
|
||||
// 先同步 ref 再 setState，保证生成执行器能立即读到新节点与连接。
|
||||
const nextNodes = [...nodesRef.current, ...segmentNodes, ...targetNodes];
|
||||
const nextConnections = [
|
||||
...connectionsRef.current,
|
||||
...prepared.flatMap((item) => [
|
||||
{ id: nanoid(), fromNodeId: node.id, toNodeId: item.segmentNode.id },
|
||||
{ id: nanoid(), fromNodeId: item.segmentNode.id, toNodeId: item.targetNode.id },
|
||||
]),
|
||||
];
|
||||
nodesRef.current = nextNodes;
|
||||
connectionsRef.current = nextConnections;
|
||||
setNodes(nextNodes);
|
||||
setConnections(nextConnections);
|
||||
setSelectedNodeIds(new Set(targetNodes.map((item) => item.id)));
|
||||
setSelectedConnectionId(null);
|
||||
progress.update("创建生成任务", segments.length * 4);
|
||||
progress.done(`已截取 ${prepared.length}/${segments.length} 段，正在调用视频模型重新生成`);
|
||||
segmentNodes.forEach((segmentNode) => {
|
||||
void ensureCanvasNodeAsset({ canvasId: projectId, domainProjectId, node: segmentNode, source: "canvas-manual" })
|
||||
.then((result) => setNodes((current) => current.map((item) => (item.id === segmentNode.id ? { ...item, metadata: { ...item.metadata, assetId: result.assetId } } : item))))
|
||||
.catch((assetError) => message.warning(`片段已截取并开始重生成，但素材库写入失败：${assetError instanceof Error ? assetError.message : "未知错误"}`));
|
||||
});
|
||||
// 任务提交瞬时打满后端会触发限流；先错峰提交，后续可替换为批次调度的容量控制。
|
||||
for (let index = 0; index < targetNodes.length; index += 1) {
|
||||
const targetNode = targetNodes[index];
|
||||
void onGenerateVideoNode?.(targetNode.id, "video", effectivePrompt);
|
||||
if (index + 1 < targetNodes.length) await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
}
|
||||
if (failedSegments.length) message.warning(`${failedSegments.length} 段截取失败，已为其余 ${prepared.length} 段创建生成任务`);
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : "视频截取失败";
|
||||
progress.fail(details);
|
||||
message.error(details);
|
||||
}
|
||||
}, [connectionsRef, domainProjectId, effectiveConfig, message, nodesRef, onGenerateVideoNode, projectId, setConnections, setSelectedConnectionId, setSelectedNodeIds, setNodes, startUploadStatus]);
|
||||
|
||||
const handleSegmentConfirm = useCallback(async (node: CanvasNodeData, params: CanvasVideoSegmentParams) => {
|
||||
if (segmentRunningRef.current || !node.metadata?.content) return;
|
||||
if (params.mode === "video") {
|
||||
const generationConfig = buildGenerationConfig(effectiveConfig, node, "video");
|
||||
const selectedConfig = { ...generationConfig, model: params.model || generationConfig.model };
|
||||
const batchError = validateVideoSegmentBatch(selectedConfig, params.segments || [], params.operation);
|
||||
if (batchError) {
|
||||
message.warning(batchError);
|
||||
return;
|
||||
}
|
||||
}
|
||||
segmentRunningRef.current = true;
|
||||
setSegmentRunningMode(params.mode);
|
||||
setSegmentDialogNodeId(null);
|
||||
setSegmentDialogMode(null);
|
||||
try {
|
||||
if (params.mode === "video") await runTrimVideoAndRegenerate(node, params);
|
||||
else await runExtractVideoAudio(node, params);
|
||||
} finally {
|
||||
segmentRunningRef.current = false;
|
||||
setSegmentRunningMode(null);
|
||||
}
|
||||
}, [effectiveConfig, message, runExtractVideoAudio, runTrimVideoAndRegenerate]);
|
||||
|
||||
const mergeVideosByIds = useCallback(async (videoNodeIds: string[]) => {
|
||||
if (mergeVideoRunningRef.current) return;
|
||||
const requestedIds = new Set(videoNodeIds);
|
||||
@@ -505,8 +710,11 @@ export function useCanvasMediaTools({
|
||||
generatePortraitTextureNode,
|
||||
cropImageNode,
|
||||
cropNodeId,
|
||||
closeSegmentDialog,
|
||||
extractAudioFromVideo,
|
||||
extractVideoLastFrame,
|
||||
extractingVideoFrameNodeId,
|
||||
handleSegmentConfirm,
|
||||
generateAngleNode,
|
||||
maskEditImageNode,
|
||||
maskEditNodeId,
|
||||
@@ -514,6 +722,10 @@ export function useCanvasMediaTools({
|
||||
mergeVideosByIds,
|
||||
mergeVideoProgress,
|
||||
saveAnnotatedImageNode,
|
||||
segmentDialogMode,
|
||||
segmentDialogNodeId,
|
||||
segmentRunningMode,
|
||||
setSegmentDialogNodeId,
|
||||
setAngleNodeId,
|
||||
generateEmotionNode,
|
||||
setEmotionNodeId,
|
||||
@@ -524,6 +736,7 @@ export function useCanvasMediaTools({
|
||||
setUpscaleNodeId,
|
||||
splitImageNode,
|
||||
splitNodeId,
|
||||
trimVideoAndRegenerate,
|
||||
upscaleImageNode,
|
||||
upscaleNodeId,
|
||||
};
|
||||
|
||||
@@ -696,6 +696,7 @@
|
||||
/* Canvas 布局（画布页浮层定位，对应审计 P0-8.1/P1-2.1）*/
|
||||
--canvas-sidebar-width: 232px; /* 项目侧栏宽度（短剧模式）*/
|
||||
--canvas-panel-width: min(332px, calc(100vw - 24px)); /* 浮动面板默认宽度（活动任务等）*/
|
||||
--video-segment-preview-max-height: min(46vh, 480px);
|
||||
|
||||
/* Prompt 面板（对应 #98 决策2 渐进式三区）*/
|
||||
--prompt-panel-max-height: min(70vh, 640px);
|
||||
|
||||
52
web/test/canvas-resource-references.test.ts
Normal file
52
web/test/canvas-resource-references.test.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { collectUpstreamVideoNodes } from "../src/lib/canvas/canvas-resource-references";
|
||||
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
|
||||
|
||||
function videoNode(id: string): CanvasNodeData {
|
||||
return {
|
||||
id,
|
||||
type: CanvasNodeType.Video,
|
||||
title: id,
|
||||
position: { x: 0, y: 0 },
|
||||
width: 100,
|
||||
height: 100,
|
||||
metadata: { content: `data:video/mp4;base64,${id}` },
|
||||
};
|
||||
}
|
||||
|
||||
function textNode(id: string): CanvasNodeData {
|
||||
return {
|
||||
id,
|
||||
type: CanvasNodeType.Text,
|
||||
title: id,
|
||||
position: { x: 0, y: 0 },
|
||||
width: 100,
|
||||
height: 60,
|
||||
metadata: { content: id },
|
||||
};
|
||||
}
|
||||
|
||||
function connection(fromNodeId: string, toNodeId: string): CanvasConnection {
|
||||
return { id: `conn-${fromNodeId}-${toNodeId}`, fromNodeId, toNodeId };
|
||||
}
|
||||
|
||||
describe("collectUpstreamVideoNodes", () => {
|
||||
test("下游视频节点能回溯到上游视频源", () => {
|
||||
const source = videoNode("source-video");
|
||||
const segment = videoNode("segment-video");
|
||||
const target = videoNode("target-video");
|
||||
const text = textNode("script");
|
||||
const nodes = [target, segment, source, text];
|
||||
const connections = [connection("source-video", "segment-video"), connection("segment-video", "target-video"), connection("script", "segment-video")];
|
||||
expect(collectUpstreamVideoNodes("target-video", nodes, connections).map((node) => node.id)).toEqual(["target-video", "segment-video", "source-video"]);
|
||||
});
|
||||
|
||||
test("存在环时不会死循环", () => {
|
||||
const a = videoNode("a");
|
||||
const b = videoNode("b");
|
||||
const nodes = [a, b];
|
||||
const connections = [connection("a", "b"), connection("b", "a")];
|
||||
expect(collectUpstreamVideoNodes("a", nodes, connections).length).toBe(2);
|
||||
});
|
||||
});
|
||||
128
web/test/canvas-video-regeneration.test.ts
Normal file
128
web/test/canvas-video-regeneration.test.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { listVideoReferenceModels, validateVideoSegmentBatch, videoReferenceOperationError, videoReferenceRegenerationError, videoReferenceSegmentError } from "../src/lib/canvas/canvas-video-regeneration";
|
||||
import { defaultModelCapabilityConfig } from "../src/lib/model-capabilities";
|
||||
import { defaultConfig, type AiConfig, type ModelChannel } from "../src/stores/use-config-store";
|
||||
|
||||
function supportedVideoConfig(overrides: { model?: string; maxVideos?: number; maxVideoDurationSeconds?: number; operations?: string[]; baseUrl?: string } = {}): AiConfig {
|
||||
const model = overrides.model || "seedance-1.0-pro";
|
||||
const capability = defaultModelCapabilityConfig("newapi-channel-2", model);
|
||||
if (capability.video) {
|
||||
capability.video.references.maxVideos = overrides.maxVideos ?? 3;
|
||||
capability.video.references.maxVideoDurationSeconds = overrides.maxVideoDurationSeconds ?? 15;
|
||||
capability.video.operations = overrides.operations ?? ["text_to_video", "image_to_video", "extend"];
|
||||
}
|
||||
const channel: ModelChannel = {
|
||||
id: "test",
|
||||
name: "测试渠道",
|
||||
baseUrl: overrides.baseUrl || "https://api.example.com",
|
||||
apiKey: "test-key",
|
||||
apiFormat: "openai",
|
||||
interfaceType: "newapi-channel-2",
|
||||
models: [model],
|
||||
scope: "user",
|
||||
modelCosts: [
|
||||
{
|
||||
model,
|
||||
capability: "video",
|
||||
protocol: "newapi-channel-2",
|
||||
billingMode: "per_second",
|
||||
unitPriceMicrocredits: 1,
|
||||
capabilityConfig: capability,
|
||||
},
|
||||
],
|
||||
};
|
||||
return {
|
||||
...defaultConfig,
|
||||
model: `test::${model}`,
|
||||
videoModel: `test::${model}`,
|
||||
models: [`test::${model}`],
|
||||
videoModels: [`test::${model}`],
|
||||
channels: [channel],
|
||||
baseUrl: channel.baseUrl,
|
||||
};
|
||||
}
|
||||
|
||||
describe("videoReferenceRegenerationError", () => {
|
||||
test("模型不支持参考视频时返回提示", () => {
|
||||
const config: AiConfig = { ...defaultConfig, model: "default::grok-imagine-video", videoModel: "default::grok-imagine-video" };
|
||||
expect(videoReferenceRegenerationError(config)).toContain("不支持参考视频");
|
||||
});
|
||||
|
||||
test("模型没有可用生成模式时返回提示", () => {
|
||||
const config = supportedVideoConfig({ operations: [] });
|
||||
expect(videoReferenceRegenerationError(config)).toContain("没有可用的视频生成模式");
|
||||
});
|
||||
|
||||
test("模型支持参考视频且有可用模式时无错误", () => {
|
||||
expect(videoReferenceRegenerationError(supportedVideoConfig())).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("videoReferenceOperationError", () => {
|
||||
test("所选模式不在模型能力中时返回提示", () => {
|
||||
const config = supportedVideoConfig({ operations: ["text_to_video", "image_to_video", "extend"] });
|
||||
expect(videoReferenceOperationError(config, "inpaint")).toContain("不支持所选生成模式");
|
||||
});
|
||||
|
||||
test("所选模式在模型能力中时无错误", () => {
|
||||
expect(videoReferenceOperationError(supportedVideoConfig(), "extend")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("listVideoReferenceModels", () => {
|
||||
test("默认配置没有可用的参考视频模型", () => {
|
||||
const config: AiConfig = { ...defaultConfig, model: "default::grok-imagine-video", videoModel: "default::grok-imagine-video" };
|
||||
expect(listVideoReferenceModels(config)).toEqual([]);
|
||||
});
|
||||
|
||||
test("只返回支持参考视频且有可用模式的模型", () => {
|
||||
expect(listVideoReferenceModels(supportedVideoConfig())).toEqual(["test::seedance-1.0-pro"]);
|
||||
});
|
||||
|
||||
test("过滤掉没有可用生成模式的模型", () => {
|
||||
const config = supportedVideoConfig({ operations: [] });
|
||||
expect(listVideoReferenceModels(config)).toEqual([]);
|
||||
});
|
||||
|
||||
test("过滤掉未配置 API Key 的渠道模型", () => {
|
||||
const config = supportedVideoConfig();
|
||||
config.channels[0].apiKey = "";
|
||||
expect(listVideoReferenceModels(config)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("videoReferenceSegmentError", () => {
|
||||
test("片段超过模型参考视频上限时返回提示", () => {
|
||||
const config = supportedVideoConfig({ maxVideoDurationSeconds: 15 });
|
||||
expect(videoReferenceSegmentError(config, 16000)).toContain("不能超过当前模型参考视频上限（15 秒）");
|
||||
});
|
||||
|
||||
test("Seedance 单段少于 2 秒时返回提示", () => {
|
||||
const config = supportedVideoConfig({ model: "seedance-1.0-pro" });
|
||||
expect(videoReferenceSegmentError(config, 1500)).toContain("至少 2 秒");
|
||||
});
|
||||
|
||||
test("合法片段时长无错误", () => {
|
||||
expect(videoReferenceSegmentError(supportedVideoConfig(), 5000)).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateVideoSegmentBatch", () => {
|
||||
test("模型、模式与片段时长都合法时无错误", () => {
|
||||
expect(validateVideoSegmentBatch(supportedVideoConfig(), [{ startMs: 2000, endMs: 5000 }], "extend")).toBe("");
|
||||
});
|
||||
|
||||
test("模型不支持参考视频时返回首个错误", () => {
|
||||
const config: AiConfig = { ...defaultConfig, model: "default::grok-imagine-video", videoModel: "default::grok-imagine-video" };
|
||||
expect(validateVideoSegmentBatch(config, [{ startMs: 2000, endMs: 5000 }], "extend")).toContain("不支持参考视频");
|
||||
});
|
||||
|
||||
test("模式不在模型能力中时返回操作错误", () => {
|
||||
expect(validateVideoSegmentBatch(supportedVideoConfig(), [{ startMs: 2000, endMs: 5000 }], "inpaint")).toContain("不支持所选生成模式");
|
||||
});
|
||||
|
||||
test("片段超过模型时长上限时返回片段错误", () => {
|
||||
expect(validateVideoSegmentBatch(supportedVideoConfig({ maxVideoDurationSeconds: 15 }), [{ startMs: 0, endMs: 16000 }], "extend")).toContain("不能超过当前模型参考视频上限");
|
||||
});
|
||||
});
|
||||
76
web/test/canvas-video-timeline-segments.test.ts
Normal file
76
web/test/canvas-video-timeline-segments.test.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { buildTimelineImportSegments } from "../src/lib/canvas/canvas-video-timeline-segments";
|
||||
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
|
||||
import type { TimelineProject } from "../src/types/timeline";
|
||||
|
||||
function videoNode(id: string, title: string): CanvasNodeData {
|
||||
return {
|
||||
id,
|
||||
type: CanvasNodeType.Video,
|
||||
title,
|
||||
position: { x: 0, y: 0 },
|
||||
width: 100,
|
||||
height: 100,
|
||||
metadata: { content: `data:video/mp4;base64,${id}`, durationMs: 10_000 },
|
||||
};
|
||||
}
|
||||
|
||||
function connection(fromNodeId: string, toNodeId: string): CanvasConnection {
|
||||
return { id: `conn-${fromNodeId}-${toNodeId}`, fromNodeId, toNodeId };
|
||||
}
|
||||
|
||||
const timeline: TimelineProject = {
|
||||
version: 2,
|
||||
tracks: [],
|
||||
clips: [
|
||||
{
|
||||
id: "clip-1",
|
||||
kind: "video",
|
||||
nodeId: "source-video",
|
||||
trackId: "video",
|
||||
startMs: 0,
|
||||
durationMs: 3_000,
|
||||
title: "源视频",
|
||||
sourceStartMs: 2_000,
|
||||
sourceDurationMs: 3_000,
|
||||
},
|
||||
],
|
||||
durationMs: 3_000,
|
||||
};
|
||||
|
||||
describe("buildTimelineImportSegments", () => {
|
||||
test("下游节点能从上游视频节点导入时间线片段", () => {
|
||||
const source = videoNode("source-video", "源视频");
|
||||
const segment = videoNode("segment-video", "片段");
|
||||
const target = videoNode("target-video", "结果");
|
||||
const nodes = [target, segment, source];
|
||||
const connections = [connection("source-video", "segment-video"), connection("segment-video", "target-video")];
|
||||
const result = buildTimelineImportSegments(target, nodes, connections, timeline, 10_000);
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
expect(result.segments).toHaveLength(1);
|
||||
expect(result.segments[0]).toMatchObject({ startMs: 2_000, endMs: 5_000, sourceNodeId: "source-video" });
|
||||
});
|
||||
|
||||
test("旧数据 nodeId 不一致时按唯一标题回退匹配", () => {
|
||||
const source = videoNode("source-video", "源视频");
|
||||
const target = videoNode("target-video", "结果");
|
||||
const nodes = [target, source];
|
||||
const connections = [connection("source-video", "target-video")];
|
||||
const oldTimeline: TimelineProject = {
|
||||
...timeline,
|
||||
clips: [{ ...timeline.clips[0], nodeId: "legacy-source-id" }],
|
||||
};
|
||||
const result = buildTimelineImportSegments(target, nodes, connections, oldTimeline, 10_000);
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
expect(result.segments[0].sourceNodeId).toBe("source-video");
|
||||
});
|
||||
|
||||
test("没有匹配片段时返回错误", () => {
|
||||
const target = videoNode("target-video", "结果");
|
||||
const result = buildTimelineImportSegments(target, [target], [], timeline, 10_000);
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user