feat(canvas): 视频节点截取重生成 - 支持多段/时间线导入并调用视频模型批量重生成 (#197)
* feat(canvas): 视频节点截取重生成 - 支持多段/时间线导入并调用视频模型批量重生成 * test(canvas): 视频节点时间线导入 - 修正旧数据标题回退测试 * style(canvas): 视频节点截取重生成 - 格式化新增文件并修正测试 --------- Co-authored-by: sikongyue <3155157671@qq.com>
This commit is contained in:
爱笑的毛毛虫
@@ -52,6 +52,22 @@ export function getGenerationResourceNodes(nodeId: string, nodes: CanvasNodeData
|
||||
return [];
|
||||
}
|
||||
|
||||
/** 收集节点自身及其上游链路中的视频节点，用于时间线片段导入定位真正的视频源。 */
|
||||
export function collectUpstreamVideoNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): CanvasNodeData[] {
|
||||
const queue = [nodeId];
|
||||
const visited = new Set<string>();
|
||||
const result: CanvasNodeData[] = [];
|
||||
while (queue.length) {
|
||||
const currentId = queue.shift()!;
|
||||
if (visited.has(currentId)) continue;
|
||||
visited.add(currentId);
|
||||
const node = nodes.find((item) => item.id === currentId);
|
||||
if (node?.type === CanvasNodeType.Video && Boolean(node.metadata?.content || node.metadata?.storageKey)) result.push(node);
|
||||
connections.filter((connection) => connection.toNodeId === currentId).forEach((connection) => queue.push(connection.fromNodeId));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function getContextResourceNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
|
||||
return connections
|
||||
.filter((connection) => connection.toNodeId === nodeId)
|
||||
|
||||
54
web/src/lib/canvas/canvas-video-regeneration.ts
Normal file
54
web/src/lib/canvas/canvas-video-regeneration.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
|
||||
import { isSeedanceVideoConfig } from "@/lib/seedance-video";
|
||||
import type { CanvasVideoEditOperation } from "@/types/canvas";
|
||||
import { resolveModelChannel, selectableModelsByCapability, type AiConfig } from "@/stores/use-config-store";
|
||||
|
||||
export function listVideoReferenceModels(config: AiConfig): string[] {
|
||||
return selectableModelsByCapability(config, "video").filter((model) => {
|
||||
const profile = modelCapabilityConfigFor(config, model).video;
|
||||
if (!profile || profile.references.maxVideos < 1 || !profile.operations.length) return false;
|
||||
const channel = resolveModelChannel(config, model);
|
||||
return Boolean(channel.baseUrl.trim() && channel.apiKey.trim());
|
||||
});
|
||||
}
|
||||
|
||||
export function videoReferenceRegenerationError(config: AiConfig): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
if (!videoProfile || videoProfile.references.maxVideos < 1) {
|
||||
return "当前所选视频模型不支持参考视频，无法使用截取重生成。请选择支持参考视频的模型，或在设置中配置 Seedance / Agent Plan / NewAPI 渠道。";
|
||||
}
|
||||
if (!videoProfile.operations.length) return "当前视频模型没有可用的视频生成模式";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function videoReferenceOperationError(config: AiConfig, operation: CanvasVideoEditOperation): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
if (!videoProfile?.operations.includes(operation)) return "当前视频模型不支持所选生成模式";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function validateVideoSegmentBatch(config: AiConfig, segments: Array<{ startMs: number; endMs: number }>, operation?: CanvasVideoEditOperation): string {
|
||||
const referenceError = videoReferenceRegenerationError(config);
|
||||
if (referenceError) return referenceError;
|
||||
if (operation) {
|
||||
const operationError = videoReferenceOperationError(config, operation);
|
||||
if (operationError) return operationError;
|
||||
}
|
||||
for (const segment of segments) {
|
||||
const segmentError = videoReferenceSegmentError(config, segment.endMs - segment.startMs);
|
||||
if (segmentError) return segmentError;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function videoReferenceSegmentError(config: AiConfig, durationMs: number): string {
|
||||
const videoProfile = modelCapabilityConfigFor(config, config.model).video;
|
||||
const maxSeconds = videoProfile?.references.maxVideoDurationSeconds || 0;
|
||||
if (maxSeconds > 0 && durationMs > maxSeconds * 1000) {
|
||||
return `截取片段不能超过当前模型参考视频上限（${maxSeconds} 秒）`;
|
||||
}
|
||||
if (isSeedanceVideoConfig(config) && durationMs < 2000) {
|
||||
return "Seedance 参考视频单段至少 2 秒，请扩大截取范围";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
104
web/src/lib/canvas/canvas-video-segment.ts
Normal file
104
web/src/lib/canvas/canvas-video-segment.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { fetchFile } from "@ffmpeg/util";
|
||||
|
||||
import { getMediaBlob } from "@/services/file-storage";
|
||||
import { loadFFmpeg } from "./canvas-video-merge";
|
||||
|
||||
export type VideoSegmentRange = {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
};
|
||||
|
||||
export type VideoSegmentSource = {
|
||||
url?: string;
|
||||
storageKey?: string;
|
||||
};
|
||||
|
||||
export type VideoSegmentProgress = {
|
||||
phase: "loading" | "reading" | "encoding";
|
||||
progress: number;
|
||||
};
|
||||
|
||||
const INPUT_NAME = "segment-input.mp4";
|
||||
const OUTPUT_NAME = "segment-output.mp4";
|
||||
|
||||
function assertValidRange(range: VideoSegmentRange, durationMs?: number) {
|
||||
const startMs = Math.max(0, Math.round(range.startMs));
|
||||
const endMs = Math.round(range.endMs);
|
||||
if (endMs <= startMs) throw new Error("片段结束时间必须晚于开始时间");
|
||||
if (durationMs !== undefined && endMs > Math.round(durationMs)) throw new Error("片段结束时间超过视频时长");
|
||||
}
|
||||
|
||||
async function readVideoSourceBlob(source: VideoSegmentSource) {
|
||||
if (source.storageKey) {
|
||||
const stored = await getMediaBlob(source.storageKey);
|
||||
if (stored) return stored;
|
||||
}
|
||||
if (source.url) {
|
||||
const response = await fetch(source.url);
|
||||
if (!response.ok) throw new Error(`视频资源请求失败（${response.status}）`);
|
||||
return response.blob();
|
||||
}
|
||||
throw new Error("找不到视频素材，请重新上传后再操作");
|
||||
}
|
||||
|
||||
async function runSegmentJob(
|
||||
source: VideoSegmentSource,
|
||||
range: VideoSegmentRange,
|
||||
durationMs: number | undefined,
|
||||
buildArgs: (startSec: string, durationSec: string) => string[],
|
||||
onProgress?: (progress: VideoSegmentProgress) => void,
|
||||
outputType = "video/mp4",
|
||||
) {
|
||||
assertValidRange(range, durationMs);
|
||||
const ffmpeg = await loadFFmpeg(({ phase, progress }) => onProgress?.({ phase: phase === "loading" ? "loading" : "reading", progress }));
|
||||
const blob = await readVideoSourceBlob(source);
|
||||
onProgress?.({ phase: "reading", progress: 45 });
|
||||
await ffmpeg.writeFile(INPUT_NAME, await fetchFile(blob));
|
||||
const startSec = String(range.startMs / 1000);
|
||||
const durationSec = String((range.endMs - range.startMs) / 1000);
|
||||
onProgress?.({ phase: "encoding", progress: 55 });
|
||||
try {
|
||||
const exitCode = await ffmpeg.exec(["-y", ...buildArgs(startSec, durationSec)]);
|
||||
if (exitCode !== 0) throw new Error("媒体处理失败，请确认视频编码格式兼容");
|
||||
const output = await ffmpeg.readFile(OUTPUT_NAME);
|
||||
onProgress?.({ phase: "encoding", progress: 100 });
|
||||
return new Blob([output as BlobPart], { type: outputType });
|
||||
} finally {
|
||||
await Promise.all([INPUT_NAME, OUTPUT_NAME].map((file) => ffmpeg.deleteFile(file).catch(() => undefined)));
|
||||
}
|
||||
}
|
||||
|
||||
/** 按片段范围截取视频，输出统一编码 MP4（复用时间线 trim 的参数模板）。 */
|
||||
export async function trimVideoSegment(source: VideoSegmentSource, range: VideoSegmentRange, durationMs?: number, onProgress?: (progress: VideoSegmentProgress) => void) {
|
||||
return runSegmentJob(
|
||||
source,
|
||||
range,
|
||||
durationMs,
|
||||
(startSec, durationSec) => ["-ss", startSec, "-i", INPUT_NAME, "-t", durationSec, "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-movflags", "+faststart", OUTPUT_NAME],
|
||||
onProgress,
|
||||
"video/mp4",
|
||||
);
|
||||
}
|
||||
|
||||
/** 从视频片段提取声音为 MP3；优先 libmp3lame，内核不支持时回退默认 mp3 编码器。 */
|
||||
export async function extractVideoAudio(source: VideoSegmentSource, range: VideoSegmentRange, durationMs?: number, onProgress?: (progress: VideoSegmentProgress) => void) {
|
||||
const ffmpeg = await loadFFmpeg(({ phase, progress }) => onProgress?.({ phase: phase === "loading" ? "loading" : "reading", progress }));
|
||||
const blob = await readVideoSourceBlob(source);
|
||||
onProgress?.({ phase: "reading", progress: 45 });
|
||||
await ffmpeg.writeFile(INPUT_NAME, await fetchFile(blob));
|
||||
assertValidRange(range, durationMs);
|
||||
const startSec = String(range.startMs / 1000);
|
||||
const durationSec = String((range.endMs - range.startMs) / 1000);
|
||||
onProgress?.({ phase: "encoding", progress: 55 });
|
||||
try {
|
||||
const args = (audioCodec: string) => ["-ss", startSec, "-i", INPUT_NAME, "-t", durationSec, "-vn", "-c:a", audioCodec, "-q:a", "2", OUTPUT_NAME];
|
||||
let exitCode = await ffmpeg.exec(["-y", ...args("libmp3lame")]);
|
||||
if (exitCode !== 0) exitCode = await ffmpeg.exec(["-y", ...args("mp3")]);
|
||||
if (exitCode !== 0) throw new Error("音频提取失败：当前 FFmpeg 内核不支持 MP3 编码");
|
||||
const output = await ffmpeg.readFile(OUTPUT_NAME);
|
||||
onProgress?.({ phase: "encoding", progress: 100 });
|
||||
return new Blob([output as BlobPart], { type: "audio/mpeg" });
|
||||
} finally {
|
||||
await Promise.all([INPUT_NAME, OUTPUT_NAME].map((file) => ffmpeg.deleteFile(file).catch(() => undefined)));
|
||||
}
|
||||
}
|
||||
54
web/src/lib/canvas/canvas-video-timeline-segments.ts
Normal file
54
web/src/lib/canvas/canvas-video-timeline-segments.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { collectUpstreamVideoNodes } from "@/lib/canvas/canvas-resource-references";
|
||||
import type { CanvasConnection, CanvasNodeData } from "@/types/canvas";
|
||||
import type { TimelineClip, TimelineProject } from "@/types/timeline";
|
||||
|
||||
export type CanvasTimelineSegmentItem = {
|
||||
id: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
sourceNodeId?: string;
|
||||
sourceStorageKey?: string;
|
||||
sourceUrl?: string;
|
||||
};
|
||||
|
||||
export type TimelineImportResult = { ok: true; segments: CanvasTimelineSegmentItem[] } | { ok: false; error: string };
|
||||
|
||||
const MIN_TIMELINE_SEGMENT_MS = 100;
|
||||
|
||||
export function buildTimelineImportSegments(node: CanvasNodeData, nodes: CanvasNodeData[], connections: CanvasConnection[], timeline: TimelineProject | null | undefined, durationMs: number): TimelineImportResult {
|
||||
const candidateNodes = collectUpstreamVideoNodes(node.id, nodes, connections);
|
||||
const matchesCandidate = (clip: TimelineClip, candidate: CanvasNodeData) => {
|
||||
if (candidate.id === clip.nodeId) return true;
|
||||
if (candidate.metadata?.assetId && clip.nodeId === candidate.metadata.assetId) return true;
|
||||
if (candidate.metadata?.storageKey && clip.directMedia?.storageKey === candidate.metadata.storageKey) return true;
|
||||
// 旧数据里 nodeId 可能被重新生成过；标题一致且唯一时才回退，避免同名视频误导入。
|
||||
if (candidate.title && clip.title === candidate.title && !clip.directMedia) {
|
||||
return candidateNodes.filter((item) => item.title === candidate.title).length === 1;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const clips = (timeline?.clips || []).filter((clip) => clip.kind === "video" && candidateNodes.some((candidate) => matchesCandidate(clip, candidate)));
|
||||
if (!clips.length) {
|
||||
return { ok: false, error: `未找到「${node.title || "当前视频节点"}」或其上游视频节点在时间线中的片段，请先在时间线中加入对应视频节点` };
|
||||
}
|
||||
const segments = clips.flatMap((clip) => {
|
||||
const matchedNode = candidateNodes.find((candidate) => matchesCandidate(clip, candidate));
|
||||
const startMs = Math.max(0, Math.round(clip.sourceStartMs ?? 0));
|
||||
const rawEndMs = startMs + Math.max(MIN_TIMELINE_SEGMENT_MS, Math.round(clip.sourceDurationMs || clip.durationMs));
|
||||
const sourceDurationMs = matchedNode?.metadata?.durationMs || durationMs;
|
||||
const endMs = sourceDurationMs > 0 ? Math.min(sourceDurationMs, rawEndMs) : rawEndMs;
|
||||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs) return [];
|
||||
return [
|
||||
{
|
||||
id: `timeline-${clip.id}`,
|
||||
startMs,
|
||||
endMs,
|
||||
sourceNodeId: matchedNode?.id,
|
||||
sourceStorageKey: clip.directMedia?.storageKey || matchedNode?.metadata?.storageKey,
|
||||
sourceUrl: clip.directMedia?.url || matchedNode?.metadata?.content,
|
||||
},
|
||||
];
|
||||
});
|
||||
if (!segments.length) return { ok: false, error: "时间线片段数据不完整，请先在时间线中重新分割或保存" };
|
||||
return { ok: true, segments };
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
||||
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
||||
|
||||
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
|
||||
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
|
||||
@@ -73,6 +73,32 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
|
||||
disabled: (ctx) => ctx.extractingVideoFrame,
|
||||
run: (ctx) => ctx.handlers.onNodeExtractVideoLastFrame(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "extractAudio",
|
||||
toolbar: "node-hover",
|
||||
category: "node-state",
|
||||
label: (ctx) => ctx.extractingAudio ? "正在提取声音为 MP3" : "提取声音为 MP3",
|
||||
displayLabel: (ctx) => ctx.extractingAudio ? "提取中" : "提取音频",
|
||||
icon: (ctx) => ctx.extractingAudio ? <LoaderCircle className="size-3.5 animate-spin" /> : <AudioLines className="size-3.5" />,
|
||||
defaultVisible: true,
|
||||
defaultOrder: 42,
|
||||
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
|
||||
disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo,
|
||||
run: (ctx) => ctx.handlers.onNodeExtractAudioFromVideo(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "trimRegenerate",
|
||||
toolbar: "node-hover",
|
||||
category: "node-state",
|
||||
label: (ctx) => ctx.trimmingVideo ? "正在截取片段" : "按段截取并重生成",
|
||||
displayLabel: (ctx) => ctx.trimmingVideo ? "截取中" : "截取重生成",
|
||||
icon: (ctx) => ctx.trimmingVideo ? <LoaderCircle className="size-3.5 animate-spin" /> : <Scissors className="size-3.5" />,
|
||||
defaultVisible: true,
|
||||
defaultOrder: 44,
|
||||
applicable: (ctx) => hasVideo(ctx) && !simpleMode(ctx),
|
||||
disabled: (ctx) => ctx.extractingAudio || ctx.trimmingVideo,
|
||||
run: (ctx) => ctx.handlers.onNodeTrimVideoRegenerate(ctx.node!),
|
||||
},
|
||||
{
|
||||
id: "saveAsset",
|
||||
toolbar: "node-hover",
|
||||
|
||||
@@ -81,6 +81,8 @@ export type ToolbarHandlers = {
|
||||
onNodeAngle: (node: CanvasNodeData) => void;
|
||||
onNodeViewImage: (node: CanvasNodeData) => void;
|
||||
onNodeExtractVideoLastFrame: (node: CanvasNodeData) => void;
|
||||
onNodeExtractAudioFromVideo: (node: CanvasNodeData) => void;
|
||||
onNodeTrimVideoRegenerate: (node: CanvasNodeData) => void;
|
||||
onNodeSubtitles: (node: CanvasNodeData) => void;
|
||||
onNodeTimeline: (node: CanvasNodeData) => void;
|
||||
onNodeReversePrompt: (node: CanvasNodeData) => void;
|
||||
@@ -105,6 +107,9 @@ export type ToolContext = {
|
||||
nodeMetadata?: CanvasNodeMetadata;
|
||||
/** 视频尾帧提取中（节点悬停工具栏用） */
|
||||
extractingVideoFrame: boolean;
|
||||
/** 视频音频提取/片段截取进行中（节点悬停工具栏用） */
|
||||
extractingAudio: boolean;
|
||||
trimmingVideo: boolean;
|
||||
/** 合并视频中（多选工具栏用） */
|
||||
mergingVideos: boolean;
|
||||
/** 主工具栏面板开关状态（仅主工具栏使用） */
|
||||
|
||||
Reference in New Issue
Block a user