From ff7a27494af7af6350dc3052ab828fb643a224d9 Mon Sep 17 00:00:00 2001 From: dlh <2773843782@qq.com> Date: Fri, 14 Aug 2026 20:44:50 +0800 Subject: [PATCH] =?UTF-8?q?fix(canvas):=20=E6=97=B6=E9=97=B4=E7=BA=BF?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=20-=20=E4=BF=AE=E5=A4=8D=E9=BB=91=E5=9C=BA?= =?UTF-8?q?=E9=A1=BA=E5=BA=8F=E3=80=81=E6=97=A0=E6=BA=90=E7=89=87=E6=AE=B5?= =?UTF-8?q?=E5=AF=B9=E9=BD=90=E4=B8=8E=E8=A3=81=E5=88=87=E5=81=8F=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #198 Co-authored-by: dlh <2773843782@qq.com> Original-PR: https://github.com/ddcat-ai/open-ai-canvas/pull/222 --- web/package.json | 2 +- web/src/lib/timeline/timeline-to-ffmpeg.ts | 40 ++--- web/test/timeline-to-ffmpeg.test.ts | 162 +++++++++++++++++++++ 3 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 web/test/timeline-to-ffmpeg.test.ts diff --git a/web/package.json b/web/package.json index 61733c36b..4c75edacb 100644 --- a/web/package.json +++ b/web/package.json @@ -7,7 +7,7 @@ "dev": "vite --host 0.0.0.0 --port 3000", "build": "tsc --noEmit && vite build", "typecheck": "tsc --noEmit", - "test": "bun test test/agent-prompt-cache.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-generation-layout.test.ts test/canvas-leafer-viewport.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/client-id.test.ts", + "test": "bun test test/agent-prompt-cache.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-generation-layout.test.ts test/canvas-leafer-viewport.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/client-id.test.ts test/timeline-to-ffmpeg.test.ts", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", "format:check": "prettier --check ." diff --git a/web/src/lib/timeline/timeline-to-ffmpeg.ts b/web/src/lib/timeline/timeline-to-ffmpeg.ts index 7c7625905..debfbcee7 100644 --- a/web/src/lib/timeline/timeline-to-ffmpeg.ts +++ b/web/src/lib/timeline/timeline-to-ffmpeg.ts @@ -97,27 +97,16 @@ export function buildTimelineRenderPlan(timeline: TimelineProject, sources: Time const concatEntries: string[] = []; const videoClips = getOrderedVideoClips(timeline); - // 1) 逐片段裁切：按源素材内部时间定位，输出统一编码便于 concat。 + // 1)+2) 逐片段裁切，并按时间线顺序在片段前补黑场（含静音音轨），输出统一编码便于 concat。 + // 黑场必须插入对应片段之前的 concat 位置：concat 按列表顺序拼接，若先收完所有 trim 再把 gap 追加到 + // 结尾，任何存在空隙的时间线（如 A(0-15s) 与 B(25-40s) 之间的 10s）都会把黑场拼到片尾、字幕整体错位。 + // 无源片段（如节点已被删除）既不产 trim 也不补自己的黑场，直接跳过：cursor 不推进，其跨度由下一个 + // 有源片段前的单个 gap 统一覆盖。若先为无源片段补 gap 再等 cursor，同一段空隙会被重复计长、成片超长 + // （线上实锤：缺源片段前有空隙时黑场翻倍、字幕继续漂移）。 + let cursorMs = 0; videoClips.forEach((clip, index) => { const source = sourceByNode.get(clip.nodeId); if (!source) return; - const output = `trim-${index}.mp4`; - // 裁切时长取时间线片段时长（clip.durationMs），而不是源素材剩余时长： - // 左缘裁剪后 sourceStartMs 前移但 sourceDurationMs 仍为源全长，若按源时长 -t 会把旧片段尾部多裁出来， - // 表现为「裁剪后播放仍从最原始视频开始/出现旧片段」；-ss 已定位源内起点，-t 必须等于片段展示时长。 - const durationSec = Math.max(0.1, clip.durationMs / 1000); - steps.push({ - kind: "trim", - output, - args: ["-ss", String((clip.sourceStartMs || 0) / 1000), "-i", source.fileName, "-t", String(durationSec), "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "128k", output], - description: `裁切片段 ${index + 1}（${clip.title || clip.nodeId}）`, - }); - concatEntries.push(output); - }); - - // 2) 空隙补黑场（时间线与成片对齐），含静音音轨。 - let cursorMs = 0; - videoClips.forEach((clip, index) => { const gapMs = clip.startMs - cursorMs; if (gapMs > 100) { const output = `gap-${index}.mp4`; @@ -151,6 +140,21 @@ export function buildTimelineRenderPlan(timeline: TimelineProject, sources: Time }); concatEntries.push(output); } + const output = `trim-${index}.mp4`; + // 裁切时长取时间线片段时长（clip.durationMs），而不是源素材剩余时长： + // 左缘裁剪后 sourceStartMs 前移但 sourceDurationMs 仍为源全长，若按源时长 -t 会把旧片段尾部多裁出来， + // 表现为「裁剪后播放仍从最原始视频开始/出现旧片段」；-ss 已定位源内起点，-t 必须等于片段展示时长。 + // -ss 必须放在 -i 之后（输出 seek）：放在 -i 之前是输入 seek，MP4/H.264 只会定位到目标时间戳 + // 之前最近的关键帧，切点会偏移最多一个 GOP（常见 0.5-2s）、片尾被 -t 截掉、音视频在切点处错位。 + // 本步骤已 -c:v libx264 重编码，输出 seek 帧精确，代价只是多解码。 + const durationSec = Math.max(0.1, clip.durationMs / 1000); + steps.push({ + kind: "trim", + output, + args: ["-i", source.fileName, "-ss", String((clip.sourceStartMs || 0) / 1000), "-t", String(durationSec), "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "128k", output], + description: `裁切片段 ${index + 1}（${clip.title || clip.nodeId}）`, + }); + concatEntries.push(output); cursorMs = clip.startMs + clip.durationMs; }); diff --git a/web/test/timeline-to-ffmpeg.test.ts b/web/test/timeline-to-ffmpeg.test.ts new file mode 100644 index 000000000..d8bc8a278 --- /dev/null +++ b/web/test/timeline-to-ffmpeg.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, test } from "bun:test"; + +import { buildTimelineRenderPlan, formatSrtTimestamp, type TimelineRenderSource } from "../src/lib/timeline/timeline-to-ffmpeg"; +import type { TimelineClip, TimelineProject } from "../src/types/timeline"; + +function videoClip(id: string, nodeId: string, startMs: number, durationMs: number): TimelineClip { + return { id, kind: "video", nodeId, trackId: "video", startMs, durationMs, title: id, sourceStartMs: 0, sourceDurationMs: durationMs }; +} + +function timeline(clips: TimelineClip[]): TimelineProject { + return { version: 2, tracks: [], clips, durationMs: clips.reduce((max, clip) => Math.max(max, clip.startMs + clip.durationMs), 0) }; +} + +function source(nodeId: string): TimelineRenderSource { + return { nodeId, fileName: `input-${nodeId}.mp4`, durationMs: 15_000, url: `file:///${nodeId}.mp4` }; +} + +/** 从导出计划推导 concat 后的总时长：trim 用 -t，gap 用 lavfi 黑场 d=。 */ +function concatTotalSeconds(plan: ReturnType): number { + let total = 0; + for (const entry of plan.concatEntries) { + const step = plan.steps.find((item) => item.output === entry); + if (step?.kind === "trim") total += Number(step.args[step.args.indexOf("-t") + 1]); + if (step?.kind === "gap") { + const lavfi = step.args.find((arg) => arg.startsWith("color=c=black")) || ""; + total += Number(lavfi.split("d=")[1]); + } + } + return total; +} + +/** 按 concat 顺序逐段累加，校验每个片段/黑场在成片中的实际起点与时间线期望一致。 */ +function concatStartOffsetsMs(plan: ReturnType): number[] { + const offsets: number[] = []; + let cursor = 0; + for (const entry of plan.concatEntries) { + const step = plan.steps.find((item) => item.output === entry); + if (!step) continue; + offsets.push(cursor); + if (step.kind === "trim") cursor += Number(step.args[step.args.indexOf("-t") + 1]) * 1000; + if (step.kind === "gap") { + const lavfi = step.args.find((arg) => arg.startsWith("color=c=black")) || ""; + cursor += Number(lavfi.split("d=")[1]) * 1000; + } + } + return offsets; +} + +describe("buildTimelineRenderPlan 片段与黑场对齐", () => { + test("全部片段有源素材且首尾相接：无黑场、concat 顺序=片段顺序、总长等于时间线", () => { + const project = timeline([videoClip("a", "node-a", 0, 15_000), videoClip("b", "node-b", 15_000, 4_000), videoClip("c", "node-c", 19_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a"), source("node-b"), source("node-c")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + expect(plan.steps.filter((step) => step.kind === "gap")).toHaveLength(0); + expect(concatTotalSeconds(plan)).toBe(34); + expect(plan.concatEntries).toEqual(["trim-0.mp4", "trim-1.mp4", "trim-2.mp4"]); + }); + + test("全部有源但中间有空隙：黑场必须插在片段之间，而不是追加到片尾", () => { + // A(0-15s) 与 B(25-40s) 之间有 10s 空隙：修复前 concat=[trim-0,trim-1,gap-1]（黑场在片尾，字幕错位）。 + const project = timeline([videoClip("a", "node-a", 0, 15_000), videoClip("b", "node-b", 25_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a"), source("node-b")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + expect(plan.concatEntries).toEqual(["trim-0.mp4", "gap-1.mp4", "trim-1.mp4"]); + const gaps = plan.steps.filter((step) => step.kind === "gap"); + expect(gaps).toHaveLength(1); + expect(gaps[0].args.join(" ")).toContain("d=10"); + expect(concatTotalSeconds(plan)).toBe(40); + // B 在成片中的起点 = 15s + 10s 黑场 = 25s，与时间线一致。 + expect(concatStartOffsetsMs(plan)).toEqual([0, 15_000, 25_000]); + }); + + test("中间片段无源素材（节点已删除）：该片段跨度补黑场，顺序与总长保持时间线语义", () => { + const project = timeline([videoClip("a", "node-a", 0, 15_000), videoClip("b", "node-b", 15_000, 4_000), videoClip("c", "node-c", 19_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a"), source("node-c")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + // 黑场 4s 顶替无源的 B，插在 A 与 C 之间：修复前为 [trim-0,trim-2,gap-2]（黑场跑到了片尾）。 + expect(plan.concatEntries).toEqual(["trim-0.mp4", "gap-2.mp4", "trim-2.mp4"]); + const gaps = plan.steps.filter((step) => step.kind === "gap"); + expect(gaps).toHaveLength(1); + expect(gaps[0].args.join(" ")).toContain("d=4"); + expect(concatTotalSeconds(plan)).toBe(34); + // C 的起点 = 15s + 4s 黑场 = 19s，与时间线一致，后续字幕不漂移。 + expect(concatStartOffsetsMs(plan)).toEqual([0, 15_000, 19_000]); + }); + + test("无源片段前有空隙：只补一个 gap，黑场不重复、成片不超长", () => { + // a(0-15s 有源)、b(20-24s 无源)、c(24-39s 有源)：b 前有 5s 空隙。 + // 修复前 b 会先产出 gap(d=5)，c 又按全跨度产出 gap(d=9)，黑场重复计长、成片 44s≠39s。 + const project = timeline([videoClip("a", "node-a", 0, 15_000), videoClip("b", "node-b", 20_000, 4_000), videoClip("c", "node-c", 24_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a"), source("node-c")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + const gaps = plan.steps.filter((step) => step.kind === "gap"); + expect(gaps).toHaveLength(1); + // 单个 gap 覆盖 b 的跨度 + 空隙 = 24s - 15s = 9s。 + expect(gaps[0].args.join(" ")).toContain("d=9"); + expect(plan.concatEntries).toEqual(["trim-0.mp4", "gap-2.mp4", "trim-2.mp4"]); + expect(concatTotalSeconds(plan)).toBe(39); + expect(concatStartOffsetsMs(plan)).toEqual([0, 15_000, 24_000]); + }); + + test("连续两个无源片段：合并为一个黑场，跨度等于两段之和", () => { + // a(0-15s 有源)、b(15-19s 无源)、c(19-27s 无源)、d(27-42s 有源)。 + const project = timeline([videoClip("a", "node-a", 0, 15_000), videoClip("b", "node-b", 15_000, 4_000), videoClip("c", "node-c", 19_000, 8_000), videoClip("d", "node-d", 27_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a"), source("node-d")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + const gaps = plan.steps.filter((step) => step.kind === "gap"); + expect(gaps).toHaveLength(1); + // 15s→27s 之间（b+c）只补一个黑场 d=12。 + expect(gaps[0].args.join(" ")).toContain("d=12"); + expect(plan.concatEntries).toEqual(["trim-0.mp4", "gap-3.mp4", "trim-3.mp4"]); + expect(concatTotalSeconds(plan)).toBe(42); + expect(concatStartOffsetsMs(plan)).toEqual([0, 15_000, 27_000]); + }); + + test("开头无源且起点非零：单个 gap 从 0 补到首个有源片段起点", () => { + // x(5-9s 无源)、a(9-24s 有源)：开头到 a 之间应只有一个 d=9 的黑场。 + const project = timeline([videoClip("x", "node-x", 5_000, 4_000), videoClip("a", "node-a", 9_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + const gaps = plan.steps.filter((step) => step.kind === "gap"); + expect(gaps).toHaveLength(1); + expect(gaps[0].args.join(" ")).toContain("d=9"); + expect(plan.concatEntries).toEqual(["gap-1.mp4", "trim-1.mp4"]); + expect(concatTotalSeconds(plan)).toBe(24); + expect(concatStartOffsetsMs(plan)).toEqual([0, 9_000]); + }); + + test("首个片段无源素材：开头补黑场，后续片段位置保持", () => { + const project = timeline([videoClip("b", "node-b", 0, 4_000), videoClip("c", "node-c", 4_000, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-c")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + expect(plan.concatEntries).toEqual(["gap-1.mp4", "trim-1.mp4"]); + const gaps = plan.steps.filter((step) => step.kind === "gap"); + expect(gaps).toHaveLength(1); + expect(gaps[0].args.join(" ")).toContain("d=4"); + expect(concatTotalSeconds(plan)).toBe(19); + }); + + test("无任何源素材：不产出 concat 与最终输出步骤", () => { + const project = timeline([videoClip("a", "node-a", 0, 15_000)]); + const plan = buildTimelineRenderPlan(project, [], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + expect(plan.concatEntries).toEqual([]); + expect(plan.steps.some((step) => step.output === "out.mp4")).toBe(false); + }); +}); + +describe("trim 步骤输出 seek（-ss 在 -i 之后）", () => { + test("裁切参数必须把 -ss 放在 -i 之后：输入 seek 会按关键帧对齐导致切点偏移", () => { + const project = timeline([videoClip("a", "node-a", 0, 15_000)]); + const plan = buildTimelineRenderPlan(project, [source("node-a")], { width: 1280, height: 720, fps: 30, outputName: "out.mp4" }); + const trims = plan.steps.filter((step) => step.kind === "trim"); + expect(trims).toHaveLength(1); + const args = trims[0].args; + const inputIndex = args.indexOf("-i"); + const ssIndex = args.indexOf("-ss"); + expect(inputIndex).toBeGreaterThan(-1); + expect(ssIndex).toBeGreaterThan(-1); + // -ss 必须在 -i 之后（输出 seek，帧精确）；在 -i 之前是输入 seek，MP4/H.264 只对齐关键帧。 + expect(ssIndex).toBeGreaterThan(inputIndex); + }); +}); + +describe("formatSrtTimestamp", () => { + test("SRT 时间码毫秒对齐三位", () => { + expect(formatSrtTimestamp(3_600_000 + 60_000 + 1_234)).toBe("01:01:01,234"); + expect(formatSrtTimestamp(0)).toBe("00:00:00,000"); + }); +});