From 62b9c420dde11b2792cdcea0f0c00741f04beb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=88=B1=E7=AC=91=E7=9A=84=E6=AF=9B=E6=AF=9B=E8=99=AB?= <315515767@qq.com> Date: Sat, 8 Aug 2026 18:28:28 +0800 Subject: [PATCH] =?UTF-8?q?feat(canvas):=20=E6=96=B0=E5=A2=9E=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E8=8A=82=E7=82=B9=E5=AD=97=E5=B9=95=E4=BD=93=E7=B3=BB?= =?UTF-8?q?=E4=B8=8E=E5=A4=9A=E8=BD=A8=E6=97=B6=E9=97=B4=E7=BA=BF=E7=BC=96?= =?UTF-8?q?=E8=BE=91=EF=BC=88=E5=85=B3=E8=81=94=20#187=EF=BC=89=20(#191)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): 后端测试 - 修复 responses 流式测试数据与 Grok 生图测试 SSRF 放行 * feat(canvas): 新增视频节点字幕体系与多轨时间线编辑（关联 #187） --------- Co-authored-by: zhaoyue --- .../components/canvas/canvas-node-toolbar.tsx | 6 +- web/src/components/canvas/canvas-node.tsx | 59 +- .../canvas/canvas-project-asset-modal.tsx | 9 +- .../canvas/canvas-subtitle-dialog.tsx | 616 ++++++++++++ .../canvas/canvas-subtitle-overlay.tsx | 38 + .../canvas/canvas-subtitle-text.tsx | 31 + .../canvas/canvas-timeline-dialog.tsx | 922 ++++++++++++++++++ .../canvas/canvas-timeline-preview.tsx | 185 ++++ .../canvas/canvas-timeline-ruler.tsx | 73 ++ web/src/components/canvas/canvas-toolbar.tsx | 2 +- web/src/lib/canvas-theme.ts | 29 + web/src/lib/canvas/canvas-video-merge.ts | 2 +- .../definitions/node-hover-tools.tsx | 26 +- .../canvas/tool-registry/tool-definition.ts | 2 + web/src/lib/timeline/srt-parser.ts | 67 ++ web/src/lib/timeline/srt-resegment.ts | 131 +++ web/src/lib/timeline/subtitle-highlight-ai.ts | 53 + .../lib/timeline/subtitle-highlight-runner.ts | 113 +++ .../timeline/subtitle-highlight-service.ts | 44 + web/src/lib/timeline/subtitle-highlights.ts | 91 ++ web/src/lib/timeline/timeline-build.ts | 184 ++++ web/src/lib/timeline/timeline-export.ts | 93 ++ web/src/lib/timeline/timeline-placement.ts | 196 ++++ web/src/lib/timeline/timeline-snap.ts | 61 ++ web/src/lib/timeline/timeline-to-ffmpeg.ts | 204 ++++ web/src/lib/timeline/timeline-tracks.ts | 54 + web/src/lib/timeline/timeline-view.ts | 67 ++ web/src/pages/canvas/project.tsx | 176 +++- web/src/pages/canvas/shared.tsx | 2 +- web/src/pages/canvas/use-canvas-upload.ts | 95 +- web/src/stores/canvas/use-canvas-store.ts | 4 +- web/src/types/canvas.ts | 5 + web/src/types/timeline.ts | 114 +++ 33 files changed, 3724 insertions(+), 30 deletions(-) create mode 100644 web/src/components/canvas/canvas-subtitle-dialog.tsx create mode 100644 web/src/components/canvas/canvas-subtitle-overlay.tsx create mode 100644 web/src/components/canvas/canvas-subtitle-text.tsx create mode 100644 web/src/components/canvas/canvas-timeline-dialog.tsx create mode 100644 web/src/components/canvas/canvas-timeline-preview.tsx create mode 100644 web/src/components/canvas/canvas-timeline-ruler.tsx create mode 100644 web/src/lib/timeline/srt-parser.ts create mode 100644 web/src/lib/timeline/srt-resegment.ts create mode 100644 web/src/lib/timeline/subtitle-highlight-ai.ts create mode 100644 web/src/lib/timeline/subtitle-highlight-runner.ts create mode 100644 web/src/lib/timeline/subtitle-highlight-service.ts create mode 100644 web/src/lib/timeline/subtitle-highlights.ts create mode 100644 web/src/lib/timeline/timeline-build.ts create mode 100644 web/src/lib/timeline/timeline-export.ts create mode 100644 web/src/lib/timeline/timeline-placement.ts create mode 100644 web/src/lib/timeline/timeline-snap.ts create mode 100644 web/src/lib/timeline/timeline-to-ffmpeg.ts create mode 100644 web/src/lib/timeline/timeline-tracks.ts create mode 100644 web/src/lib/timeline/timeline-view.ts create mode 100644 web/src/types/timeline.ts diff --git a/web/src/components/canvas/canvas-node-toolbar.tsx b/web/src/components/canvas/canvas-node-toolbar.tsx index 1a0990b47..dffd7da6d 100644 --- a/web/src/components/canvas/canvas-node-toolbar.tsx +++ b/web/src/components/canvas/canvas-node-toolbar.tsx @@ -42,6 +42,8 @@ type CanvasNodeToolbarProps = { onAngle: (node: CanvasNodeData) => void; onViewImage: (node: CanvasNodeData) => void; onExtractVideoLastFrame: (node: CanvasNodeData) => void; + onSubtitles: (node: CanvasNodeData) => void; + onTimeline: (node: CanvasNodeData) => void; extractingVideoFrame: boolean; onReversePrompt: (node: CanvasNodeData) => void; onRetry: (node: CanvasNodeData) => void; @@ -103,6 +105,8 @@ export function CanvasNodeToolbar({ onAngle, onViewImage, onExtractVideoLastFrame, + onSubtitles, + onTimeline, extractingVideoFrame, onReversePrompt, onRetry, @@ -240,7 +244,7 @@ export function CanvasNodeToolbar({ onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop, onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage, onNodeExtractVideoLastFrame: onExtractVideoLastFrame, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, - onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt, + onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt, } as Partial as ToolbarHandlers; const nodeHoverCtx: ToolContext = { diff --git a/web/src/components/canvas/canvas-node.tsx b/web/src/components/canvas/canvas-node.tsx index 4e24453dc..a0b764b80 100644 --- a/web/src/components/canvas/canvas-node.tsx +++ b/web/src/components/canvas/canvas-node.tsx @@ -18,6 +18,8 @@ import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-refer import { loadCanvasDrawingPreview } from "@/lib/canvas/canvas-drawing-storage"; import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size"; import { VideoPlayer } from "@/components/video-player"; +import { createDefaultSubtitleStyle } from "@/types/timeline"; +import { CanvasSubtitleOverlay } from "./canvas-subtitle-overlay"; type ResizeCorner = "top-left" | "top-right" | "bottom-left" | "bottom-right"; type CanvasTheme = (typeof canvasThemes)[keyof typeof canvasThemes]; @@ -901,7 +903,34 @@ function EmptyImageContent({ node, theme, isBatchRoot, batchCount, batchExpanded function VideoNodeContent({ node, theme, reduceMediaEffects }: NodeContentRendererProps) { const playWhenReadyRef = useRef(false); + const playerBoxRef = useRef(null); const { url, loading, load } = useNodeResourceUrl(node, false); + const subtitleEntries = node.metadata?.subtitleEntries || []; + const subtitleStyle = node.metadata?.subtitleStyle || createDefaultSubtitleStyle(); + const [currentTimeMs, setCurrentTimeMs] = useState(0); + const [videoSize, setVideoSize] = useState<{ width: number; height: number } | null>(null); + + // 视频元素由 vidstack 内部创建，直接监听原生事件取分辨率；只有存在字幕时才跟踪播放时间，避免无字幕节点频繁重渲染。 + useEffect(() => { + const box = playerBoxRef.current; + const video = box?.querySelector("video"); + if (!video) return; + const handleLoadedMetadata = () => { + if (video.videoWidth > 0 && video.videoHeight > 0) setVideoSize({ width: video.videoWidth, height: video.videoHeight }); + }; + video.addEventListener("loadedmetadata", handleLoadedMetadata); + handleLoadedMetadata(); + if (!subtitleEntries.length) { + return () => video.removeEventListener("loadedmetadata", handleLoadedMetadata); + } + const handleTimeUpdate = () => setCurrentTimeMs(Math.round(video.currentTime * 1000)); + video.addEventListener("timeupdate", handleTimeUpdate); + return () => { + video.removeEventListener("timeupdate", handleTimeUpdate); + video.removeEventListener("loadedmetadata", handleLoadedMetadata); + }; + }, [subtitleEntries.length, url]); + if (!node.metadata?.content) return (
@@ -912,7 +941,35 @@ function VideoNodeContent({ node, theme, reduceMediaEffects }: NodeContentRender if (!url) { return : } label={loading ? "正在缓存视频" : "加载并缓存视频"} disabled={loading} onClick={() => { playWhenReadyRef.current = true; void load(); }} />; } - return { playWhenReadyRef.current = false; }} brandColor={theme.accent.primary} className="h-full w-full rounded-[var(--node-radius)] bg-black" dataCanvasNoZoom compactControls />; + + // 视频画面按实际分辨率等比适配节点盒子，字幕叠加层与画面同框，不在黑边上错位。 + const sourceRatio = (videoSize?.width || node.metadata?.naturalWidth || node.width) / Math.max(1, videoSize?.height || node.metadata?.naturalHeight || node.height); + const fitHeight = Math.min(node.height, node.width / Math.max(0.01, sourceRatio)); + const fitWidth = Math.round(fitHeight * sourceRatio); + const activeEntry = subtitleEntries.find((entry) => currentTimeMs >= entry.startMs && currentTimeMs < entry.endMs); + const activeHighlight = activeEntry ? (node.metadata?.subtitleHighlights || []).find((item) => item.entryIndex === activeEntry.index) : undefined; + + return ( +
+
+ { playWhenReadyRef.current = false; }} + brandColor={theme.accent.primary} + className="h-full w-full rounded-[var(--node-radius)] bg-black" + dataCanvasNoZoom + compactControls + /> + {activeEntry && activeEntry.text.trim() ? ( + + ) : null} +
+
+ ); } function AudioNodeContent({ node, theme }: NodeContentRendererProps) { diff --git a/web/src/components/canvas/canvas-project-asset-modal.tsx b/web/src/components/canvas/canvas-project-asset-modal.tsx index 37b265f3a..e3e38b8ae 100644 --- a/web/src/components/canvas/canvas-project-asset-modal.tsx +++ b/web/src/components/canvas/canvas-project-asset-modal.tsx @@ -20,11 +20,16 @@ export function CanvasProjectAssetModal({ open, detail, initialCategory = "all", const [inserting, setInserting] = useState(false); const items = useMemo(() => { const mediaById = new Map(mediaAssets.map((asset) => [asset.id, asset])); - return (detail?.assets || []).flatMap((asset): ProjectPickerItem[] => { + const projectItems = (detail?.assets || []).flatMap((asset): ProjectPickerItem[] => { if (asset.category === "character" && asset.character) return [{ id: asset.id, category: "character", character: asset }]; const media = mediaById.get(asset.id); return media && media.kind !== "model" && media.kind !== "entity" ? [{ id: asset.id, category: asset.category || media.category || "other", media }] : []; }); + if (projectItems.length) return projectItems; + // 自由画布（未关联短剧项目）或项目资产为空/后端未返回时，回退到本地素材库，避免弹窗空白 + return mediaAssets + .filter((asset) => asset.kind !== "model" && asset.kind !== "entity") + .map((media) => ({ id: media.id, category: media.category || "other", media })); }, [detail?.assets, mediaAssets]); const categories = useMemo(() => ["all", ...Array.from(new Set(items.map((item) => item.category)))], [items]); const visible = category === "all" ? items : items.filter((item) => item.category === category); @@ -47,7 +52,7 @@ export function CanvasProjectAssetModal({ open, detail, initialCategory = "all",

引用项目角色与资产

已选 {selectedIds.size} 项
-
{visible.length ?
{visible.map((item) => toggle(item.id)} />)}
: }
+
{visible.length ?
{visible.map((item) => toggle(item.id)} />)}
: }
角色引用会在生成时解析当前角色版本
diff --git a/web/src/components/canvas/canvas-subtitle-dialog.tsx b/web/src/components/canvas/canvas-subtitle-dialog.tsx new file mode 100644 index 000000000..d270bde75 --- /dev/null +++ b/web/src/components/canvas/canvas-subtitle-dialog.tsx @@ -0,0 +1,616 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { App, Button, ColorPicker, Input, InputNumber, Modal, Progress, Segmented, Switch } from "antd"; +import { Captions, FileDown, FileUp, ListPlus, LoaderCircle, Plus, Scissors, Sparkles, Trash2 } from "lucide-react"; +import { saveAs } from "file-saver"; + +import { canvasThemes } from "@/lib/canvas-theme"; +import { useThemeStore } from "@/stores/use-theme-store"; +import { useConfigStore, type AiConfig } from "@/stores/use-config-store"; +import { resolveMediaUrl } from "@/services/file-storage"; +import { cacheResourceObjectUrl } from "@/services/resource-blob-cache"; +import { resourceIdFromStorageKey } from "@/services/api/resources"; +import { parseSrt, serializeSrtEntries } from "@/lib/timeline/srt-parser"; +import { DEFAULT_MAX_CHARS_PER_ENTRY, MAX_CHARS_PER_ENTRY_LIMIT, MIN_CHARS_PER_ENTRY, resegmentSrtEntries, splitLongEntry } from "@/lib/timeline/srt-resegment"; +import { buildFallbackHighlights, remapHighlightsAfterResegment } from "@/lib/timeline/subtitle-highlights"; +import { generateSubtitleHighlights, type SubtitleHighlightProgress } from "@/lib/timeline/subtitle-highlight-runner"; +import { createDefaultSubtitleStyle, type SrtEntry, type SubtitleHighlight, type SubtitlePosition, type SubtitleStyle } from "@/types/timeline"; +import type { CanvasNodeData, CanvasNodeMetadata } from "@/types/canvas"; +import { SubtitleHighlightedText } from "./canvas-subtitle-text"; + +type CanvasSubtitleDialogProps = { + node: CanvasNodeData; + open: boolean; + projectId: string; + config: AiConfig; + onClose: () => void; + onSave: (nodeId: string, patch: Partial) => void; +}; + +export function CanvasSubtitleDialog({ node, open, projectId, config, onClose, onSave }: CanvasSubtitleDialogProps) { + const { message, modal } = App.useApp(); + const theme = canvasThemes[useThemeStore((state) => state.theme)]; + const isAiConfigReady = useConfigStore((state) => state.isAiConfigReady); + const [entries, setEntries] = useState([]); + const [highlights, setHighlights] = useState([]); + const [style, setStyle] = useState(createDefaultSubtitleStyle); + const [running, setRunning] = useState(false); + const [progress, setProgress] = useState(null); + const [videoUrl, setVideoUrl] = useState(""); + const [currentTimeMs, setCurrentTimeMs] = useState(0); + const [videoSize, setVideoSize] = useState<{ width: number; height: number } | null>(null); + const [previewBoxWidth, setPreviewBoxWidth] = useState(0); + const [viewportHeight, setViewportHeight] = useState(0); + const [videoError, setVideoError] = useState(false); + const abortRef = useRef(null); + const fileInputRef = useRef(null); + const textInputRef = useRef(null); + const previewBoxRef = useRef(null); + const videoRef = useRef(null); + // 打开弹窗时从节点 metadata 同步字幕数据；关闭后卸载，重开重新读取。 + useEffect(() => { + if (!open) return; + setEntries(node.metadata?.subtitleEntries || []); + setHighlights(node.metadata?.subtitleHighlights || []); + setStyle(node.metadata?.subtitleStyle || createDefaultSubtitleStyle()); + setRunning(false); + setProgress(null); + }, [open, node]); + + useEffect(() => { + return () => abortRef.current?.abort(); + }, []); + + // 打开弹窗时解析视频地址，用于字幕叠加预览。 + // 远端资源优先走节点同款缓存下载（对象 URL），失败再退回资源代理地址。 + useEffect(() => { + if (!open) return; + let cancelled = false; + setCurrentTimeMs(0); + setVideoSize(null); + setVideoError(false); + const storageKey = node.metadata?.storageKey || ""; + const fallback = node.metadata?.content || ""; + const applyUrl = (url: string) => { + if (!cancelled) setVideoUrl(url); + }; + if (resourceIdFromStorageKey(storageKey)) { + void cacheResourceObjectUrl(storageKey) + .then((cached) => { + if (cancelled) return; + if (cached) { + setVideoUrl(cached); + } else { + void resolveMediaUrl(storageKey, fallback).then(applyUrl); + } + }) + .catch(() => { + if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl); + }); + } else { + void resolveMediaUrl(storageKey, fallback).then(applyUrl); + } + return () => { + cancelled = true; + }; + }, [open, node]); + + // 监听预览容器与视口尺寸，视频按分辨率等比缩放，不撑满也不变形。 + useEffect(() => { + if (!open || !videoUrl) return; + const measure = () => { + setPreviewBoxWidth(previewBoxRef.current?.clientWidth || 0); + setViewportHeight(window.innerHeight); + }; + measure(); + const observer = new ResizeObserver(measure); + if (previewBoxRef.current) observer.observe(previewBoxRef.current); + window.addEventListener("resize", measure); + return () => { + observer.disconnect(); + window.removeEventListener("resize", measure); + }; + }, [open, videoUrl]); + + const previewDisplay = useMemo(() => { + if (!videoSize || videoSize.height <= 0) return null; + const maxHeight = Math.round((viewportHeight || window.innerHeight) * 0.45); + const maxWidth = Math.max(1, previewBoxWidth || 800); + const ratio = videoSize.width / videoSize.height; + let width = Math.min(maxWidth, videoSize.width); + let height = width / ratio; + if (height > maxHeight) { + height = maxHeight; + width = Math.round(height * ratio); + } + return { width: Math.round(width), height: Math.round(height) }; + }, [previewBoxWidth, videoSize, viewportHeight]); + + const highlightByEntry = new Map(highlights.map((item) => [item.entryIndex, item])); + const activeEntry = entries.find((entry) => currentTimeMs >= entry.startMs && currentTimeMs < entry.endMs); + const activeHighlight = activeEntry ? highlightByEntry.get(activeEntry.index) : undefined; + const activePositionClass = style.position === "top" ? "top-3" : style.position === "center" ? "top-1/2 -translate-y-1/2" : "bottom-4"; + + const updateEntry = (position: number, patch: Partial) => { + setEntries((current) => current.map((entry, idx) => (idx === position ? { ...entry, ...patch } : entry))); + }; + + const renumber = (list: SrtEntry[]) => list.map((entry, idx) => ({ ...entry, index: idx + 1 })); + + const normalizeEntries = (list: SrtEntry[]) => + renumber( + list + .filter((entry) => entry.text.trim()) + .map((entry) => ({ ...entry, startMs: Math.max(0, entry.startMs), endMs: Math.max(0, entry.endMs) })) + .sort((a, b) => a.startMs - b.startMs || a.index - b.index), + ); + + const addEntry = () => { + setEntries((current) => { + const last = current[current.length - 1]; + const startMs = last ? last.endMs : 0; + return [...current, { index: current.length + 1, startMs, endMs: startMs + 2_000, text: "" }]; + }); + }; + + const deleteEntry = (position: number) => { + const removed = entries[position]; + if (!removed) return; + const next = normalizeEntries(entries.filter((_, idx) => idx !== position)); + const { remapped } = remapHighlightsAfterResegment(highlights, next); + // 互通加固：逐条删除立即持久化到节点并同步时间线，与「清空全部」行为一致， + // 避免删除后直接关闭弹窗（未点保存）导致重开视频节点旧字幕复活。 + setEntries(next); + setHighlights(next.length ? remapped : []); + onSave(node.id, { + subtitleEntries: next, + subtitleHighlights: next.length ? remapped : [], + subtitleStyle: style, + subtitleUpdatedAt: new Date().toISOString(), + }); + message.success(next.length ? "已删除该条字幕并同步到视频节点与时间线" : "字幕已清空并同步到视频节点与时间线"); + }; + + const splitEntry = (position: number) => { + const entry = entries[position]; + if (!entry || !entry.text.trim()) { + message.warning("该条字幕没有可切分的内容"); + return; + } + const segments = splitLongEntry(entry, Math.max(2, Math.ceil(entry.text.length / 2))); + const next = renumber([...entries.slice(0, position), ...segments, ...entries.slice(position + 1)]); + setEntries(next); + const { remapped } = remapHighlightsAfterResegment(highlights, next); + setHighlights(remapped); + }; + + const seekToEntry = (entry: SrtEntry) => { + const video = videoRef.current; + if (video) { + video.currentTime = Math.max(0, entry.startMs / 1000); + setCurrentTimeMs(entry.startMs); + } + }; + + const resegmentAll = () => { + if (!entries.length) return; + const maxChars = style.maxCharsPerEntry || DEFAULT_MAX_CHARS_PER_ENTRY; + const next = resegmentSrtEntries(entries, maxChars); + if (next.length === entries.length) { + message.info(`所有字幕均未超过单条上限 ${maxChars} 字，无需切分；可在“字幕样式”中调低上限后重试`); + return; + } + setEntries(next); + const { remapped, dropped } = remapHighlightsAfterResegment(highlights, next); + setHighlights(remapped); + message.success(`自动切分完成：${entries.length} 条 → ${next.length} 条`); + if (dropped.length) message.info(`已移除 ${dropped.length} 条未能匹配的旧高亮`); + }; + + const importSrt = (file: File) => { + const reader = new FileReader(); + reader.onload = () => { + const parsed = parseSrt(String(reader.result || "")); + if (!parsed.length) { + message.warning("未解析到有效字幕：文件需为 SRT 格式（序号 + 时间码 + 正文），纯文本请用“导入文本”"); + return; + } + const autoSegmented = style.autoResegment ? resegmentSrtEntries(parsed, style.maxCharsPerEntry || DEFAULT_MAX_CHARS_PER_ENTRY) : parsed; + setEntries(renumber(autoSegmented)); + setHighlights([]); + message.success(autoSegmented.length > parsed.length ? `已导入 ${parsed.length} 条字幕，按单条上限自动切分为 ${autoSegmented.length} 条` : `已导入 ${parsed.length} 条字幕`); + }; + reader.readAsText(file); + }; + + // 纯文本导入：一行一条字幕，按视频时长平均分配起止时间；视频时长未知时按每条 4 秒估算。 + const importPlainText = (file: File) => { + const reader = new FileReader(); + reader.onload = () => { + const lines = String(reader.result || "") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (!lines.length) { + message.warning("文本中没有可导入的内容"); + return; + } + const durationMs = node.metadata?.durationMs; + const stepMs = durationMs && durationMs > 0 ? durationMs / lines.length : 4_000; + const entries: SrtEntry[] = lines.map((text, idx) => ({ + index: idx + 1, + startMs: Math.round(idx * stepMs), + endMs: Math.round((idx + 1) * stepMs), + text, + })); + setEntries(entries); + setHighlights([]); + message.success(durationMs && durationMs > 0 ? `已导入 ${entries.length} 条字幕，按视频时长 ${formatDurationMs(durationMs)} 平均分配` : `已导入 ${entries.length} 条字幕（视频时长未知，按每条 4 秒估算）`); + }; + reader.readAsText(file); + }; + + const exportSrt = () => { + const content = serializeSrtEntries(entries); + if (!content) { + message.warning("暂无字幕可导出"); + return; + } + saveAs(new Blob([content], { type: "text/plain;charset=utf-8" }), `${node.title || "subtitle"}.srt`); + }; + + const runAiHighlight = async () => { + if (!entries.length) { + message.warning("请先添加字幕内容"); + return; + } + if (!isAiConfigReady(config, config.textModel)) { + // 未配置文本模型时本地回退：按终止标点取首段作为高亮，保证功能不中断。 + setHighlights(buildFallbackHighlights(entries)); + message.info("未配置可用的文本模型，已使用本地标点高亮"); + return; + } + const controller = new AbortController(); + abortRef.current = controller; + setRunning(true); + setProgress({ batchIndex: 0, batchTotal: 1, processedEntries: 0, totalEntries: entries.length, percent: 0 }); + try { + const nextHighlights = await generateSubtitleHighlights(entries, { + projectId, + nodeId: node.id, + config, + signal: controller.signal, + onProgress: setProgress, + }); + setHighlights(nextHighlights); + message.success(`已生成 ${nextHighlights.length} 条关键词高亮`); + } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") return; + message.error(error instanceof Error ? error.message : "关键词高亮生成失败"); + } finally { + setRunning(false); + setProgress(null); + abortRef.current = null; + } + }; + + const cancelAiHighlight = () => { + abortRef.current?.abort(); + }; + + const handleSave = () => { + const normalized = normalizeEntries(entries); + const { remapped } = remapHighlightsAfterResegment(highlights, normalized); + onSave(node.id, { + subtitleEntries: normalized, + subtitleHighlights: normalized.length ? remapped : [], + subtitleStyle: style, + subtitleUpdatedAt: new Date().toISOString(), + }); + message.success(normalized.length ? "字幕已保存" : "字幕已清空并保存"); + onClose(); + }; + + const formatDurationMs = (ms: number) => { + const seconds = Math.round(ms / 1000); + return seconds >= 60 ? `${Math.floor(seconds / 60)}分${seconds % 60}秒` : `${seconds}秒`; + }; + + const previewBlock = ( +
+
+ {videoUrl ? ( +
+
+ ) : ( +
{videoError ? "视频预览加载失败，请检查素材是否仍然可用" : "暂无视频素材可预览，可先在节点上传或生成视频"}
+ )} +
+
点击字幕条目可跳转预览
+
+ ); + + const styleControls = ( + <> + + +
+
位置
+ setStyle((current) => ({ ...current, position: position as SubtitlePosition }))} + /> +
+ +
+
+ 关键词高亮 + setStyle((current) => ({ ...current, highlightEnabled }))} /> +
+
+ + + + + +
+
+ + ); + + const entriesBlock = ( +
+ {entries.length ? ( + entries.map((entry, idx) => { + const highlight = highlightByEntry.get(entry.index); + return ( +
{ + const target = event.target as HTMLElement; + if (target.closest("input,button,textarea")) return; + seekToEntry(entry); + }} + > +
+ + {idx + 1} + + updateEntry(idx, { startMs: startMs ?? 0 })} className="w-32" aria-label={`第 ${idx + 1} 条开始时间（毫秒）`} /> + → + updateEntry(idx, { endMs: endMs ?? 0 })} className="w-32" aria-label={`第 ${idx + 1} 条结束时间（毫秒）`} /> + + +
+ updateEntry(idx, { text: event.target.value })} /> + {highlight ? ( +
+ 重点：{highlight.highlightText} +
+ ) : null} +
+ ); + }) + ) : ( +
+
+ + 还没有字幕：导入 SRT 或文本文件，或点击工具栏“新增字幕”。 +
+
+ )} +
+ ); + + const title = ( +
+ + + +
+
字幕编辑
+
{node.title || "视频节点"}
+
+
+ ); + + return ( + +
+
+ { + const file = event.target.files?.[0]; + if (file) importSrt(file); + event.target.value = ""; + }} + /> + { + const file = event.target.files?.[0]; + if (file) importPlainText(file); + event.target.value = ""; + }} + /> + + + + + + + +
+ + {progress && running ? ( +
+ `${progress.processedEntries}/${progress.totalEntries} 条 · 批次 ${progress.batchIndex}/${progress.batchTotal}`} /> +
+ ) : null} + +
+
+ {entriesBlock} + {previewBlock} +
+
字幕样式
+ {styleControls} +
+
+
+ +
+
+ {entries.length} 条字幕 · 单条上限 {style.maxCharsPerEntry || DEFAULT_MAX_CHARS_PER_ENTRY} 字 +
+
+ + +
+
+
+ + ); +} diff --git a/web/src/components/canvas/canvas-subtitle-overlay.tsx b/web/src/components/canvas/canvas-subtitle-overlay.tsx new file mode 100644 index 000000000..8d55235b1 --- /dev/null +++ b/web/src/components/canvas/canvas-subtitle-overlay.tsx @@ -0,0 +1,38 @@ +import type { CSSProperties } from "react"; + +import type { SubtitleHighlight, SubtitleStyle } from "@/types/timeline"; +import { SubtitleHighlightedText } from "./canvas-subtitle-text"; + +type CanvasSubtitleOverlayProps = { + text: string; + highlight?: SubtitleHighlight; + style: SubtitleStyle; +}; + +const POSITION_STYLE: Record = { + top: { top: "6%" }, + center: { top: "50%", transform: "translateY(-50%)" }, + bottom: { bottom: "8%" }, +}; + +/** + * 画布视频节点上的实时字幕叠加层：按字幕样式定位到视频画面内。 + * 纯展示层（pointer-events-none），不拦截画布拖拽与播放器控件。 + */ +export function CanvasSubtitleOverlay({ text, highlight, style }: CanvasSubtitleOverlayProps) { + return ( +
+ + + +
+ ); +} diff --git a/web/src/components/canvas/canvas-subtitle-text.tsx b/web/src/components/canvas/canvas-subtitle-text.tsx new file mode 100644 index 000000000..4c9ea0819 --- /dev/null +++ b/web/src/components/canvas/canvas-subtitle-text.tsx @@ -0,0 +1,31 @@ +import type { SubtitleHighlight, SubtitleStyle } from "@/types/timeline"; + +type SubtitleHighlightedTextProps = { + text: string; + highlight?: SubtitleHighlight; + style: SubtitleStyle; +}; + +/** + * 字幕文本的统一样式渲染：未启用高亮或没有高亮数据时原样输出，否则给重点词上底色。 + * 字幕弹窗与画布视频节点叠加层共用，保证两处高亮效果一致。 + */ +export function SubtitleHighlightedText({ text, highlight, style }: SubtitleHighlightedTextProps) { + if (!style.highlightEnabled || !highlight) return text; + return ( + <> + {text.slice(0, highlight.start)} + + {text.slice(highlight.start, highlight.end)} + + {text.slice(highlight.end)} + + ); +} diff --git a/web/src/components/canvas/canvas-timeline-dialog.tsx b/web/src/components/canvas/canvas-timeline-dialog.tsx new file mode 100644 index 000000000..bc3e64839 --- /dev/null +++ b/web/src/components/canvas/canvas-timeline-dialog.tsx @@ -0,0 +1,922 @@ +// 二期：多轨时间线编辑弹窗。 +// 数据源是项目级 TimelineProject：视频/音频节点自动入轨，字幕条目转字幕片段。 +// 交互：拖拽移动片段（吸附播放头/片段边缘）、左右边缘裁剪、删除、播放头跳转。 + +import { useCallback, useEffect, useMemo, useRef, useState, type MutableRefObject } from "react"; +import { App, Button, Dropdown, Input, InputNumber, Modal, Progress, Tooltip } from "antd"; +import { Captions, Clapperboard, FolderOpen, Library, Lock, LockOpen, Maximize2, MoreHorizontal, Music2, Plus, Scissors, Trash2, Upload, Video, Wand2, ZoomIn, ZoomOut } from "lucide-react"; +import { saveAs } from "file-saver"; + +import { CanvasTimelineRuler } from "./canvas-timeline-ruler"; +import { CanvasTimelinePreview } from "./canvas-timeline-preview"; +import { canvasThemes } from "@/lib/canvas-theme"; +import { useThemeStore } from "@/stores/use-theme-store"; +import { buildTimelineFromNodes, isNodeInTimeline, syncTimelineSubtitleClips } from "@/lib/timeline/timeline-build"; +import { canPlaceAt, clampClipDurationByNeighbors, findNearestAvailablePlacement } from "@/lib/timeline/timeline-placement"; +import { computeSnap } from "@/lib/timeline/timeline-snap"; +import { DEFAULT_AUDIO_TRACK_ID, DEFAULT_VIDEO_TRACK_ID, normalizeTimelineProject } from "@/lib/timeline/timeline-tracks"; +import { formatTimelineTime, getTimelineTrackWidth, getFitTimelineZoom, zoomIn, zoomOut } from "@/lib/timeline/timeline-view"; +import { exportTimelineToMp4 } from "@/lib/timeline/timeline-export"; +import type { TimelineRenderSource } from "@/lib/timeline/timeline-to-ffmpeg"; +import type { CanvasNodeData } from "@/types/canvas"; +import type { SrtEntry, TimelineClip, TimelineDirectMedia, TimelineProject } from "@/types/timeline"; + +const MIN_CLIP_DURATION_MS = 100; +const TRACK_ROW_HEIGHT = 52; +const SNAP_THRESHOLD_PX = 8; +const BASE_PX_PER_SECOND = 96; + +type CanvasTimelineDialogProps = { + node: CanvasNodeData; + open: boolean; + nodes: CanvasNodeData[]; + timeline: TimelineProject | null; + onClose: () => void; + onOpenSubtitleDialog?: (nodeId: string) => void; + onSave: (timeline: TimelineProject) => void; + onSaveSubtitles: (nodeId: string, entries: SrtEntry[]) => void; + /** 打开项目素材库选择器（由页面层接线） */ + onOpenAssetLibrary?: () => void; + /** 打开项目资产库选择器（由页面层接线） */ + onOpenProjectAssets?: () => void; + /** 上传本地音视频并返回直连媒体描述（仅时间线作用域，不创建画布节点） */ + onUploadLocalFiles?: (files: File[]) => Promise; + /** 页面层持有：素材库/上传创建节点后，调用该引用把节点加入时间线草稿 */ + addNodeToTimelineRef?: MutableRefObject<((node: CanvasNodeData) => void) | null>; + /** 页面层持有：素材库/项目资产直连媒体（不落画布）入轨通道 */ + addMediaToTimelineRef?: MutableRefObject<((media: TimelineDirectMedia) => void) | null>; + /** 把合成结果保存为新视频节点放回画布（页面层接线） */ + onCreateAssembledNode?: (blob: Blob, title: string) => Promise; +}; + +type ClipDragMode = "move" | "trim-start" | "trim-end"; + +type DragState = + | { mode: "move"; clipId: string; startClientX: number; originalStartMs: number } + | { mode: "trim-start"; clipId: string; startClientX: number; originalStartMs: number; originalDurationMs: number; originalSourceStartMs: number } + | { mode: "trim-end"; clipId: string; startClientX: number; originalStartMs: number; originalDurationMs: number } + | null; + +export function CanvasTimelineDialog({ + node, + open, + nodes, + timeline, + onClose, + onOpenSubtitleDialog, + onSave, + onSaveSubtitles, + onOpenAssetLibrary, + onOpenProjectAssets, + onUploadLocalFiles, + addNodeToTimelineRef, + addMediaToTimelineRef, + onCreateAssembledNode, +}: CanvasTimelineDialogProps) { + const { message } = App.useApp(); + const theme = canvasThemes[useThemeStore((state) => state.theme)]; + const [draft, setDraft] = useState(() => buildTimelineFromNodes([])); + const [playheadMs, setPlayheadMs] = useState(0); + const [zoomLevel, setZoomLevel] = useState(1); + const [selectedClipId, setSelectedClipId] = useState(null); + const [snapEnabled, setSnapEnabled] = useState(true); + const [previewPlaying, setPreviewPlaying] = useState(false); + const [exporting, setExporting] = useState(false); + const [exportPercent, setExportPercent] = useState(0); + const [exportDetail, setExportDetail] = useState(""); + const dragRef = useRef(null); + const scrollRef = useRef(null); + const trackAreaRef = useRef(null); + const uploadInputRef = useRef(null); + const toolbarRef = useRef(null); + const addGroupProbeRef = useRef(null); + const moreBtnRef = useRef(null); + + const initializedRef = useRef(false); + + // 打开弹窗：优先复用项目时间线，否则从画布节点自动构建；播放头定位到触发节点片段。 + // 仅首次打开时构建草稿：素材上传/素材库插入会触发 nodes 变化，不能因此重建草稿丢弃编辑。 + useEffect(() => { + if (!open) { + initializedRef.current = false; + return; + } + if (initializedRef.current) return; + initializedRef.current = true; + const base = timeline ? normalizeTimelineProject(timeline) : buildTimelineFromNodes(nodes); + const next = timeline ? syncTimelineSubtitleClips(base, nodes) : base; + setDraft(next); + setPlayheadMs(0); + setZoomLevel(1); + setSelectedClipId(null); + setPreviewPlaying(false); + const triggerClip = next.clips.find((clip) => clip.nodeId === node.id); + if (triggerClip) setPlayheadMs(triggerClip.startMs); + }, [open, node.id, nodes, timeline]); + + const pxPerMs = (BASE_PX_PER_SECOND / 1_000) * zoomLevel; + const durationMs = Math.max(1_000, draft.durationMs); + const [viewportWidth, setViewportWidth] = useState(900); + const trackWidth = useMemo(() => getTimelineTrackWidth(durationMs, zoomLevel, viewportWidth), [durationMs, zoomLevel, viewportWidth]); + + useEffect(() => { + if (!open) return; + const measure = () => { + setViewportWidth(trackAreaRef.current?.clientWidth || 900); + }; + measure(); + const observer = new ResizeObserver(measure); + if (trackAreaRef.current) observer.observe(trackAreaRef.current); + return () => observer.disconnect(); + }, [open]); + + // 工具栏自适应：按钮总数过多或容器变窄时，把「添加素材/上传本地/素材库/项目资产」收进「更多」菜单， + // 工具栏保持单行不换行；容器变宽后自动恢复内联展示。 + // 素材组用隐藏探针测量自然宽度，展开/收起两种形态下算出的核心区宽度一致，判定不随当前状态震荡。 + const [addGroupCollapsed, setAddGroupCollapsed] = useState(false); + const toolbarObserverRef = useRef(null); + const measureToolbar = useCallback(() => { + const el = toolbarRef.current; + const probe = addGroupProbeRef.current; + if (!el || !probe) return; + const addGroupWidth = probe.offsetWidth; + const shownAddWidth = addGroupCollapsed ? (moreBtnRef.current?.offsetWidth ?? 56) : addGroupWidth; + const expandedWidth = el.scrollWidth - shownAddWidth + addGroupWidth; + setAddGroupCollapsed(expandedWidth > el.clientWidth + 2); + }, [addGroupCollapsed]); + const ensureToolbarObserved = useCallback(() => { + const el = toolbarRef.current; + if (!el || toolbarObserverRef.current) return; + const observer = new ResizeObserver(measureToolbar); + toolbarObserverRef.current = observer; + observer.observe(el); + measureToolbar(); + }, [measureToolbar]); + useEffect(() => { + if (!open) return; + // AntD Modal 内容经门户延迟挂载，首次 effect 时 toolbarRef 可能为空； + // afterOpenChange 兜底确保动画结束后一定完成测量与观察。 + ensureToolbarObserved(); + return () => { + toolbarObserverRef.current?.disconnect(); + toolbarObserverRef.current = null; + }; + }, [open, ensureToolbarObserved]); + + const applyDraft = (updater: (current: TimelineProject) => TimelineProject) => { + setDraft((current) => { + const next = updater(current); + const endMs = next.clips.reduce((max, clip) => Math.max(max, clip.startMs + clip.durationMs), 0); + return { ...next, durationMs: Math.max(endMs, 1_000) }; + }); + }; + + const updateClip = (clipId: string, patch: Partial) => { + applyDraft((current) => ({ + ...current, + clips: current.clips.map((clip) => (clip.id === clipId ? { ...clip, ...patch } : clip)), + })); + }; + + const updateSelectedMediaNumbers = (patch: Partial) => { + if (!selectedClipId) return; + updateClip(selectedClipId, patch); + }; + + const moveClip = (clipId: string, candidateStartMs: number) => { + const clip = draft.clips.find((item) => item.id === clipId); + if (!clip) return; + const snapped = snapEnabled + ? computeSnap({ + candidateMs: candidateStartMs, + playheadMs, + clips: draft.clips, + excludeClipId: clipId, + pxPerMs, + thresholdPx: SNAP_THRESHOLD_PX, + enabled: true, + }) + : { snappedMs: candidateStartMs, targets: [] }; + const nextStartMs = Math.max(0, Math.round(snapped.snappedMs)); + const result = canPlaceAt({ + trackId: clip.trackId, + startMs: nextStartMs, + durationMs: clip.durationMs, + excludeClipId: clipId, + clips: draft.clips, + }); + if (!result.ok) return; + updateClip(clipId, { startMs: nextStartMs }); + }; + + // 裁剪统一基于拖动起点快照计算，避免“当前值 + 累计 delta”造成的漂移与越界。 + const trimClipStart = (clipId: string, deltaMs: number, original: { startMs: number; durationMs: number; sourceStartMs: number }) => { + const clip = draft.clips.find((item) => item.id === clipId); + if (!clip) return; + const rightEdgeMs = original.startMs + original.durationMs; + // 向左延长最多到源文件起点：时间轴起点不能早于 original.start - sourceStart。 + const minStartMs = clip.sourceStartMs === undefined ? 0 : Math.max(0, original.startMs - original.sourceStartMs); + const newStartMs = Math.max(minStartMs, Math.min(rightEdgeMs - MIN_CLIP_DURATION_MS, original.startMs + deltaMs)); + const newDurationMs = rightEdgeMs - newStartMs; + // 源内裁剪：sourceStart 必须落在 [0, sourceDuration - newDuration]，右缘不越源。 + const sourceDurationMs = clip.sourceDurationMs; + let sourceStartMs = original.sourceStartMs + (newStartMs - original.startMs); + if (sourceDurationMs !== undefined) { + sourceStartMs = Math.max(0, Math.min(Math.max(0, sourceDurationMs - newDurationMs), sourceStartMs)); + } + const collision = canPlaceAt({ + trackId: clip.trackId, + startMs: newStartMs, + durationMs: newDurationMs, + excludeClipId: clipId, + clips: draft.clips, + }); + if (!collision.ok) return; + updateClip(clipId, { + startMs: Math.round(newStartMs), + durationMs: Math.round(newDurationMs), + ...(clip.sourceStartMs !== undefined ? { sourceStartMs: Math.round(sourceStartMs) } : {}), + }); + }; + + const trimClipEnd = (clipId: string, deltaMs: number, original: { startMs: number; durationMs: number }) => { + const clip = draft.clips.find((item) => item.id === clipId); + if (!clip) return; + const maxDurationMs = clip.sourceDurationMs !== undefined && clip.sourceStartMs !== undefined ? Math.max(0, clip.sourceDurationMs - clip.sourceStartMs) : undefined; + const maxDuration = clampClipDurationByNeighbors({ + clipId, + startMs: original.startMs, + requestedDurationMs: original.durationMs + deltaMs, + trackId: clip.trackId, + clips: draft.clips, + maxDurationMs, + }); + const newDurationMs = Math.max(MIN_CLIP_DURATION_MS, maxDuration); + updateClip(clipId, { durationMs: Math.round(newDurationMs) }); + }; + + const handleClipPointerDown = (event: React.PointerEvent, clipId: string, mode: ClipDragMode) => { + const clip = draft.clips.find((item) => item.id === clipId); + if (!clip) return; + event.preventDefault(); + event.stopPropagation(); + (event.currentTarget as HTMLElement).setPointerCapture(event.pointerId); + setSelectedClipId(clipId); + if (mode === "move") { + dragRef.current = { mode, clipId, startClientX: event.clientX, originalStartMs: clip.startMs }; + } else if (mode === "trim-start") { + dragRef.current = { + mode, + clipId, + startClientX: event.clientX, + originalStartMs: clip.startMs, + originalDurationMs: clip.durationMs, + originalSourceStartMs: clip.sourceStartMs || 0, + }; + } else { + dragRef.current = { mode, clipId, startClientX: event.clientX, originalStartMs: clip.startMs, originalDurationMs: clip.durationMs }; + } + }; + + const handleClipPointerMove = (event: React.PointerEvent) => { + const drag = dragRef.current; + if (!drag) return; + if (!event.isPrimary) return; + const deltaMs = Math.round((event.clientX - drag.startClientX) / Math.max(pxPerMs, 1e-6)); + if (drag.mode === "move") { + moveClip(drag.clipId, drag.originalStartMs + deltaMs); + } else if (drag.mode === "trim-start") { + trimClipStart(drag.clipId, deltaMs, { + startMs: drag.originalStartMs, + durationMs: drag.originalDurationMs, + sourceStartMs: drag.originalSourceStartMs, + }); + } else { + trimClipEnd(drag.clipId, deltaMs, { + startMs: drag.originalStartMs, + durationMs: drag.originalDurationMs, + }); + } + }; + + const handleClipPointerUp = (event: React.PointerEvent) => { + dragRef.current = null; + try { + (event.currentTarget as HTMLElement).releasePointerCapture(event.pointerId); + } catch { + // 指针已释放或未捕获时忽略 + } + }; + + const deleteSelectedClip = () => { + if (!selectedClipId) { + message.info("请先点击选中一个片段"); + return; + } + applyDraft((current) => ({ + ...current, + clips: current.clips.filter((clip) => clip.id !== selectedClipId), + })); + setSelectedClipId(null); + }; + + const removeClip = (clipId: string) => { + applyDraft((current) => ({ + ...current, + clips: current.clips.filter((clip) => clip.id !== clipId), + })); + if (selectedClipId === clipId) setSelectedClipId(null); + }; + + const addNodeToTimeline = (targetNode: CanvasNodeData) => { + const kind = targetNode.type === "video" ? "video" : targetNode.type === "audio" ? "audio" : null; + if (!kind) return; + const trackId = kind === "video" ? DEFAULT_VIDEO_TRACK_ID : DEFAULT_AUDIO_TRACK_ID; + const targetTrack = draft.tracks.find((track) => track.id === trackId); + if (!targetTrack || targetTrack.locked) { + message.warning(kind === "video" ? "视频轨已锁定或不存在" : "音频轨已锁定或不存在"); + return; + } + const durationMs = targetNode.metadata?.durationMs && targetNode.metadata.durationMs > 0 ? Math.round(targetNode.metadata.durationMs) : 4_000; + const placement = findNearestAvailablePlacement({ + targetStartMs: playheadMs, + durationMs, + trackId, + clips: draft.clips, + }); + // 找不到可用空隙时（目标位置冲突且无足够间隙）回退到轨道末尾追加，避免片段重叠破坏后续放置/导出。 + const startMs = placement.fits ? placement.startMs : draft.clips.filter((item) => item.trackId === trackId).reduce((max, item) => Math.max(max, item.startMs + item.durationMs), 0); + const clip: TimelineClip = { + id: `clip-${targetNode.id}-${Date.now()}`, + kind, + nodeId: targetNode.id, + trackId, + startMs, + durationMs, + title: targetNode.title || (kind === "video" ? "视频片段" : "音频片段"), + sourceStartMs: 0, + sourceDurationMs: durationMs, + ...(kind === "audio" ? { volume: 1, fadeInMs: 0, fadeOutMs: 0 } : {}), + }; + applyDraft((current) => ({ ...current, clips: [...current.clips, clip] })); + setSelectedClipId(clip.id); + message.success(`已添加「${clip.title}」到时间线`); + }; + + // 把“添加节点”能力暴露给页面层（素材库/本地上传创建节点后回填到草稿）。 + useEffect(() => { + if (!addNodeToTimelineRef) return; + addNodeToTimelineRef.current = addNodeToTimeline; + return () => { + addNodeToTimelineRef.current = null; + }; + }, [addNodeToTimeline, addNodeToTimelineRef]); + + // 直连媒体（仅时间线作用域，不落画布）：素材库/项目资产/本地上传直接入轨。 + const addDirectMediaToTimeline = (media: TimelineDirectMedia) => { + const kind = media.kind === "video" ? "video" : media.kind === "audio" ? "audio" : null; + if (!kind) { + message.info("图片/文本素材暂不支持直接入轨，请先在画布中添加节点"); + return; + } + const trackId = kind === "video" ? DEFAULT_VIDEO_TRACK_ID : DEFAULT_AUDIO_TRACK_ID; + const targetTrack = draft.tracks.find((track) => track.id === trackId); + if (!targetTrack || targetTrack.locked) { + message.warning(kind === "video" ? "视频轨已锁定或不存在" : "音频轨已锁定或不存在"); + return; + } + const durationMs = media.durationMs && media.durationMs > 0 ? Math.round(media.durationMs) : 4_000; + const placement = findNearestAvailablePlacement({ + targetStartMs: playheadMs, + durationMs, + trackId, + clips: draft.clips, + }); + const clip: TimelineClip = { + id: `clip-media-${media.id}-${Date.now()}`, + kind, + nodeId: `timeline-media-${media.id}-${Date.now()}`, + trackId, + startMs: placement.startMs, + durationMs, + title: media.title || (kind === "video" ? "视频片段" : "音频片段"), + sourceStartMs: 0, + sourceDurationMs: durationMs, + directMedia: media, + ...(kind === "audio" ? { volume: 1, fadeInMs: 0, fadeOutMs: 0 } : {}), + }; + applyDraft((current) => ({ ...current, clips: [...current.clips, clip] })); + setSelectedClipId(clip.id); + message.success(`已添加「${clip.title}」到时间线`); + }; + + // 把“直连媒体入轨”能力暴露给页面层（时间线作用域插入不重复落画布）。 + useEffect(() => { + if (!addMediaToTimelineRef) return; + addMediaToTimelineRef.current = addDirectMediaToTimeline; + return () => { + addMediaToTimelineRef.current = null; + }; + }, [addDirectMediaToTimeline, addMediaToTimelineRef]); + + // 播放头处分割：把视频/音频片段切成两段，右段源内起点整体后移，保持源文件内容连续。 + const splitClipAtPlayhead = () => { + const clip = draft.clips.find((item) => (item.kind === "video" || item.kind === "audio") && playheadMs > item.startMs && playheadMs < item.startMs + item.durationMs); + if (!clip) { + message.info("请先将播放头移动到片段内部，再执行分割"); + return; + } + const cutMs = playheadMs - clip.startMs; + if (cutMs < MIN_CLIP_DURATION_MS || clip.durationMs - cutMs < MIN_CLIP_DURATION_MS) { + message.warning("分割点太靠近片段边缘"); + return; + } + const left: TimelineClip = { ...clip, id: `${clip.id}-left-${Date.now()}`, durationMs: Math.round(cutMs) }; + const right: TimelineClip = { + ...clip, + id: `${clip.id}-right-${Date.now()}`, + startMs: playheadMs, + durationMs: Math.round(clip.durationMs - cutMs), + sourceStartMs: (clip.sourceStartMs || 0) + Math.round(cutMs), + sourceDurationMs: clip.sourceDurationMs !== undefined ? Math.max(0, Math.round(clip.sourceDurationMs - cutMs)) : Math.round(clip.durationMs - cutMs), + }; + applyDraft((current) => ({ ...current, clips: [...current.clips.filter((item) => item.id !== clip.id), left, right] })); + setSelectedClipId(left.id); + message.success("已在播放头处分割片段"); + }; + + const handleLocalUpload = async (files: FileList | null) => { + const list = files ? Array.from(files) : []; + if (!list.length) return; + if (!onUploadLocalFiles) { + message.warning("本地上传暂未接线"); + return; + } + const medias = await onUploadLocalFiles(list); + for (const media of medias) addDirectMediaToTimeline(media); + }; + + const handleSave = () => { + // 互通修复：时间线草稿只在打开时初始化，期间节点字幕可能在字幕弹窗中被清空（或节点数据被外部更新）。 + // 保存前以节点当前字幕为准做定向校准：节点字幕已为空时，剔除草稿残留的旧字幕片段并回写空数组， + // 避免「清空后重开视频节点旧字幕复活」；节点仍有字幕时保留草稿内用户的时间线编辑（拖动/删减/文本）。 + const clearedSubtitleNodeIds = new Set(nodes.filter((item) => (item.metadata?.subtitleEntries?.length ?? 0) === 0).map((item) => item.id)); + const base = normalizeTimelineProject({ ...draft, updatedAt: new Date().toISOString() }); + const reconciledClips = clearedSubtitleNodeIds.size ? base.clips.filter((clip) => !(clip.kind === "subtitle" && clearedSubtitleNodeIds.has(clip.nodeId))) : base.clips; + const normalized = normalizeTimelineProject({ ...base, clips: reconciledClips }); + onSave(normalized); + // 整合方向：时间线字幕片段与节点字幕互通。保存时按视频节点回写 subtitleEntries（含空数组）。 + // 回写集合 = 项目时间线原有字幕节点 ∪ 校准后仍有字幕片段的节点 ∪ 节点当前仍有字幕数据的节点： + // 覆盖“首次打开时间线（项目尚无 timeline）时删除字幕片段”这类 timeline 为空导致的漏回写， + // 保证删除全部字幕片段后节点字幕被同步清空，重开视频节点不再显示旧字幕。 + const previousSubtitleNodeIds = new Set((timeline?.clips || []).filter((clip) => clip.kind === "subtitle").map((clip) => clip.nodeId)); + const nextSubtitleNodeIds = new Set(normalized.clips.filter((clip) => clip.kind === "subtitle").map((clip) => clip.nodeId)); + const nodeSubtitleNodeIds = new Set(nodes.filter((item) => (item.metadata?.subtitleEntries?.length ?? 0) > 0).map((item) => item.id)); + const subtitleNodeIds = new Set([...previousSubtitleNodeIds, ...nextSubtitleNodeIds, ...nodeSubtitleNodeIds]); + subtitleNodeIds.forEach((subNodeId) => { + onSaveSubtitles(subNodeId, buildSubtitleEntriesForNode(subNodeId, normalized)); + }); + message.success("时间线已保存"); + onClose(); + }; + + // 组装导出：把当前草稿按片段顺序合成一个 MP4 Blob（导出下载与生成新片段共用）。 + const runExport = async (): Promise => { + const videoClips = draft.clips.filter((clip) => clip.kind === "video"); + if (!videoClips.length) throw new Error("时间线没有视频片段，无法导出"); + const sources: TimelineRenderSource[] = []; + for (const clip of videoClips) { + const sourceNode = nodes.find((item) => item.id === clip.nodeId); + const media = clip.directMedia; + if (!sourceNode && !media) continue; + sources.push({ + nodeId: clip.nodeId, + fileName: "input-" + sources.length + ".mp4", + durationMs: clip.sourceDurationMs || clip.durationMs, + storageKey: sourceNode?.metadata?.storageKey || media?.storageKey, + url: sourceNode?.metadata?.content || media?.url || undefined, + }); + } + if (!sources.length) throw new Error("找不到可导出的视频素材，请确认视频节点包含媒体"); + setExporting(true); + setExportPercent(0); + setExportDetail("准备导出"); + try { + return await exportTimelineToMp4(normalizeTimelineProject(draft), sources, { + onProgress: ({ percent, detail }) => { + setExportPercent(percent); + setExportDetail(detail); + }, + }); + } finally { + setExporting(false); + setExportPercent(0); + setExportDetail(""); + } + }; + + const handleExport = async () => { + try { + const blob = await runExport(); + saveAs(blob, (node.title || "成片") + ".mp4"); + message.success("成片导出完成"); + } catch (error) { + message.error(error instanceof Error ? error.message : "导出失败"); + } + }; + + // 组装能力闭环：合成结果不落地下载，而是作为新视频节点放回画布，可继续编辑字幕与样式。 + const handleCreateAssembledNode = async () => { + if (!onCreateAssembledNode) { + message.warning("保存回画布暂未接线"); + return; + } + try { + const blob = await runExport(); + const created = await onCreateAssembledNode(blob, (node.title || "成片") + "-新片段"); + if (created) message.success("已生成新视频片段并放到画布，可继续编辑字幕与样式"); + } catch (error) { + message.error(error instanceof Error ? error.message : "生成新片段失败"); + } + }; + + const addableNodes = nodes.filter((item) => (item.type === "video" || item.type === "audio") && Boolean(item.metadata?.content || item.metadata?.storageKey) && !isNodeInTimeline(item.id, draft)); + + const selectedSubtitleClip = selectedClipId ? draft.clips.find((clip) => clip.id === selectedClipId && clip.kind === "subtitle") || null : null; + const selectedMediaClip = selectedClipId ? draft.clips.find((clip) => clip.id === selectedClipId && (clip.kind === "video" || clip.kind === "audio")) || null : null; + const selectedMediaRightEdgeMs = selectedMediaClip ? selectedMediaClip.startMs + selectedMediaClip.durationMs : 0; + const selectedMediaMinStartMs = selectedMediaClip ? (selectedMediaClip.sourceStartMs ? Math.max(0, selectedMediaClip.startMs - selectedMediaClip.sourceStartMs) : 0) : 0; + const selectedMediaMaxStartMs = selectedMediaClip ? Math.max(selectedMediaMinStartMs, selectedMediaRightEdgeMs - MIN_CLIP_DURATION_MS) : 0; + const selectedMediaSourceDurationMs = selectedMediaClip?.sourceDurationMs; + const selectedMediaMaxDurationMs = selectedMediaSourceDurationMs !== undefined ? Math.max(MIN_CLIP_DURATION_MS, selectedMediaSourceDurationMs) : 3_600_000; + const selectedMediaMaxSourceStartMs = selectedMediaSourceDurationMs !== undefined ? Math.max(0, selectedMediaSourceDurationMs - (selectedMediaClip?.durationMs || 0)) : 3_600_000; + const hasVideoClips = draft.clips.some((clip) => clip.kind === "video"); + + // 时间线全局时间 → 视频节点本地时间：以该节点视频片段起点为 0 基准重建 SrtEntry。 + // source 传保存时校准后的时间线（normalized）：节点字幕已清空时直接回写空数组， + // 避免草稿残留的旧字幕片段把节点上已清空的字幕重新写回。 + const buildSubtitleEntriesForNode = (subNodeId: string, source: TimelineProject = draft): SrtEntry[] => { + const currentNode = nodes.find((item) => item.id === subNodeId); + if ((currentNode?.metadata?.subtitleEntries?.length ?? 0) === 0) return []; + const videoClip = source.clips.find((clip) => clip.nodeId === subNodeId && clip.kind === "video"); + const baseMs = videoClip?.startMs ?? 0; + return source.clips + .filter((clip) => clip.nodeId === subNodeId && clip.kind === "subtitle" && Boolean(clip.text)) + .sort((a, b) => a.startMs - b.startMs) + .map((clip, index) => ({ + index: index + 1, + startMs: Math.max(0, Math.round(clip.startMs - baseMs)), + endMs: Math.max(0, Math.round(clip.startMs + clip.durationMs - baseMs)), + text: (clip.text || "").trim(), + })); + }; + + const trackLabel = (trackId: string) => draft.tracks.find((track) => track.id === trackId)?.label || trackId; + const renderTrackRow = (trackId: string, label: string) => { + const clips = draft.clips.filter((clip) => clip.trackId === trackId).sort((a, b) => a.startMs - b.startMs); + return ( +
+ {clips.map((clip) => { + const isSelected = clip.id === selectedClipId; + const left = Math.round(clip.startMs * pxPerMs); + const width = Math.max(8, Math.round(clip.durationMs * pxPerMs)); + const isSubtitle = clip.kind === "subtitle"; + const clipFill = isSubtitle ? theme.timeline.clipSubtitle : clip.kind === "audio" ? theme.timeline.clipAudio : theme.timeline.clipVideo; + return ( +
handleClipPointerDown(event, clip.id, "move")} + onPointerMove={handleClipPointerMove} + onPointerUp={handleClipPointerUp} + onPointerCancel={handleClipPointerUp} + onClick={(event) => event.stopPropagation()} + title={`${clip.title || label} · ${formatTimelineTime(clip.startMs)}~${formatTimelineTime(clip.startMs + clip.durationMs)}`} + > + {!isSubtitle ? ( +
handleClipPointerDown(event, clip.id, "trim-start")} + /> + ) : null} + {clip.title || trackLabel(clip.trackId)} + {!isSubtitle ? ( +
handleClipPointerDown(event, clip.id, "trim-end")} /> + ) : null} +
+ ); + })} +
+
+ ); + }; + + const title = ( +
+ + + +
+
多轨时间线
+
第二期 · 轨道编辑与素材编排
+
+
+ ); + + return ( + { + if (visible) ensureToolbarObserved(); + }} + styles={{ container: { padding: 0, overflow: "hidden" }, body: { padding: 0 } }} + > +
+
+ + {formatTimelineTime(playheadMs)} + + + + + + {addGroupCollapsed ? ( + , + children: addableNodes.map((item) => ({ + key: item.id, + label: ( + + {item.type === "video" ? + ), + onClick: () => addNodeToTimeline(item), + })), + }, + ] + : []), + { key: "upload", label: "上传本地", icon: , onClick: () => uploadInputRef.current?.click() }, + { key: "library", label: "素材库", icon: , disabled: !onOpenAssetLibrary, onClick: () => onOpenAssetLibrary?.() }, + { key: "assets", label: "项目资产", icon: , disabled: !onOpenProjectAssets, onClick: () => onOpenProjectAssets?.() }, + ], + }} + > + + + + + ) : ( + <> + ({ + key: item.id, + label: ( + + {item.type === "video" ? + ), + onClick: () => addNodeToTimeline(item), + })), + }} + > + + + + + + + )} + + { + void handleLocalUpload(event.target.files); + event.target.value = ""; + }} + /> + +
+ + + + + +
+
+ + setPreviewPlaying((value) => !value)} onPlayheadChange={setPlayheadMs} /> + +
+
+
+ {draft.tracks + .slice() + .sort((a, b) => a.order - b.order) + .map((track) => ( +
+ + {track.kind === "video" ? "V" : track.kind === "audio" ? "A" : "S"} + + {track.label} + {track.locked ? : } +
+ ))} +
+ +
+
+ { + setPreviewPlaying(false); + setPlayheadMs(ms); + }} + /> + {draft.tracks + .slice() + .sort((a, b) => a.order - b.order) + .map((track) => renderTrackRow(track.id, track.label))} +
+
+
+ + {exporting ? ( +
+ exportDetail} /> +
+ ) : null} + + {selectedSubtitleClip ? ( +
+
+ + 字幕编辑 + + 修改后保存将同步写回对应视频节点的字幕数据 + {onOpenSubtitleDialog ? ( + + ) : null} +
+
+ updateClip(selectedSubtitleClip.id, { text: event.target.value })} /> +
+ updateClip(selectedSubtitleClip.id, { startMs: startMs ?? 0 })} className="w-28" /> + → + updateClip(selectedSubtitleClip.id, { durationMs: Math.max(MIN_CLIP_DURATION_MS, (endMs ?? 0) - selectedSubtitleClip.startMs) })} + className="w-28" + /> + 毫秒 +
+
+
+ ) : null} + + {selectedMediaClip ? ( +
+
+ + 片段编辑 + + {selectedMediaClip.title || trackLabel(selectedMediaClip.trackId)} + +
+
+ 起点 + updateSelectedMediaNumbers({ startMs: Math.max(selectedMediaMinStartMs, Math.min(selectedMediaMaxStartMs, value ?? selectedMediaClip.startMs)) })} + /> + 时长 + updateSelectedMediaNumbers({ durationMs: Math.max(MIN_CLIP_DURATION_MS, Math.min(selectedMediaMaxDurationMs, value ?? selectedMediaClip.durationMs)) })} + /> + 源内起点 + updateSelectedMediaNumbers({ sourceStartMs: Math.max(0, Math.min(selectedMediaMaxSourceStartMs, value ?? (selectedMediaClip.sourceStartMs || 0))) })} + /> + 源内结束 + + {formatTimelineTime((selectedMediaClip.sourceStartMs || 0) + selectedMediaClip.durationMs)} + + 数值微调精确裁剪，播放头处可分割；拖拽手柄快速裁剪 +
+
+ ) : null} + +
+ + 总时长 {formatTimelineTime(durationMs)} · {draft.clips.length} 个片段 · {draft.tracks.length} 条轨道 + + 拖拽片段移动，左右边缘裁剪，字幕片段来自视频节点的字幕数据 +
+
+ + ); +} diff --git a/web/src/components/canvas/canvas-timeline-preview.tsx b/web/src/components/canvas/canvas-timeline-preview.tsx new file mode 100644 index 000000000..2b0f262fe --- /dev/null +++ b/web/src/components/canvas/canvas-timeline-preview.tsx @@ -0,0 +1,185 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { Pause, Play } from "lucide-react"; + +import { canvasThemes } from "@/lib/canvas-theme"; +import { resolveMediaUrl } from "@/services/file-storage"; +import { cacheResourceObjectUrl } from "@/services/resource-blob-cache"; +import { resourceIdFromStorageKey } from "@/services/api/resources"; +import { formatTimelineTime } from "@/lib/timeline/timeline-view"; +import { createDefaultSubtitleStyle } from "@/types/timeline"; +import type { TimelineClip } from "@/types/timeline"; +import type { CanvasNodeData } from "@/types/canvas"; +import { CanvasSubtitleOverlay } from "./canvas-subtitle-overlay"; + +type CanvasTheme = (typeof canvasThemes)[keyof typeof canvasThemes]; + +type CanvasTimelinePreviewProps = { + clips: TimelineClip[]; + nodes: CanvasNodeData[]; + playheadMs: number; + playing: boolean; + theme: CanvasTheme; + onTogglePlay: () => void; + onPlayheadChange: (ms: number) => void; +}; + +const PREVIEW_WIDTH = 300; +const PREVIEW_HEIGHT = 168; +const AUTO_ADVANCE_GAP_MS = 500; + +/** + * 时间线弹窗的所见即所得预览：显示播放头所在视频片段，并叠加当前字幕。 + * 播放时以视频时间为基准推进播放头；暂停时播放头（标尺跳转）驱动视频画面。 + */ +export function CanvasTimelinePreview({ clips, nodes, playheadMs, playing, theme, onTogglePlay, onPlayheadChange }: CanvasTimelinePreviewProps) { + const videoRef = useRef(null); + const [videoUrl, setVideoUrl] = useState(""); + const [videoSize, setVideoSize] = useState<{ width: number; height: number } | null>(null); + // 源内定位目标（秒）；视频换源后 metadata 尚未加载时先记录，loadedmetadata 后再应用。 + const targetSeekSecRef = useRef(null); + + const videoClips = useMemo(() => clips.filter((clip) => clip.kind === "video").sort((a, b) => a.startMs - b.startMs), [clips]); + const activeVideoClip = useMemo(() => videoClips.find((clip) => playheadMs >= clip.startMs && playheadMs < clip.startMs + clip.durationMs) || null, [playheadMs, videoClips]); + const activeSubtitleClip = useMemo(() => clips.find((clip) => clip.kind === "subtitle" && playheadMs >= clip.startMs && playheadMs < clip.startMs + clip.durationMs && Boolean(clip.text?.trim())) || null, [clips, playheadMs]); + const activeNode = useMemo(() => (activeVideoClip ? nodes.find((node) => node.id === activeVideoClip.nodeId) || null : null), [activeVideoClip, nodes]); + const subtitleStyle = activeNode?.metadata?.subtitleStyle || createDefaultSubtitleStyle(); + const activeHighlight = useMemo(() => { + if (!activeSubtitleClip || activeSubtitleClip.subtitleEntryIndex === undefined || activeSubtitleClip.subtitleEntryIndex < 0 || !activeNode) return undefined; + // subtitleEntryIndex 与字幕条目 index 同为 1 基，直接匹配，不再 +1（此前高亮永远匹配不上）。 + return (activeNode.metadata?.subtitleHighlights || []).find((item) => item.entryIndex === activeSubtitleClip.subtitleEntryIndex); + }, [activeNode, activeSubtitleClip]); + + // 解析当前片段视频地址（与字幕弹窗同一套缓存/回退策略）。 + useEffect(() => { + const node = activeNode; + const media = activeVideoClip?.directMedia; + setVideoUrl(""); + setVideoSize(null); + if (!node && !media) return; + let cancelled = false; + // 直连媒体片段（directMedia，不落画布）与画布节点走同一套缓存/回退解析策略 + const storageKey = node?.metadata?.storageKey || media?.storageKey || ""; + const fallback = node?.metadata?.content || media?.url || ""; + const applyUrl = (url: string) => { + if (!cancelled) setVideoUrl(url); + }; + if (resourceIdFromStorageKey(storageKey)) { + void cacheResourceObjectUrl(storageKey) + .then((cached) => { + if (cancelled) return; + if (cached) { + setVideoUrl(cached); + } else { + void resolveMediaUrl(storageKey, fallback).then(applyUrl); + } + }) + .catch(() => { + if (!cancelled) void resolveMediaUrl(storageKey, fallback).then(applyUrl); + }); + } else { + void resolveMediaUrl(storageKey, fallback).then(applyUrl); + } + return () => { + cancelled = true; + }; + }, [activeNode, activeVideoClip]); + + // 播放/暂停与外部跳转（标尺拖动）时同步视频位置；播放前先定位到片段源内起点， + // 避免裁剪后的片段从原始视频 0s 开始播放；播放期间视频时间反向驱动播放头，不做回跳。 + useEffect(() => { + const video = videoRef.current; + if (!video || !activeVideoClip || !videoUrl) return; + const sourceStartSec = (activeVideoClip.sourceStartMs || 0) / 1000; + const sourceDurationSec = (activeVideoClip.sourceDurationMs || activeVideoClip.durationMs) / 1000; + const targetSec = sourceStartSec + Math.max(0, playheadMs - activeVideoClip.startMs) / 1000; + const clampedTargetSec = Math.min(targetSec, sourceStartSec + Math.max(0, sourceDurationSec - 0.05)); + targetSeekSecRef.current = clampedTargetSec; + if (video.readyState >= 1 && Math.abs(video.currentTime - clampedTargetSec) > 0.05) { + video.currentTime = clampedTargetSec; + } + if (playing) { + void video.play().catch(() => undefined); + } else { + video.pause(); + } + }, [activeVideoClip, playheadMs, playing, videoUrl]); + + // 视频换源后元数据加载完成，应用之前记录的源内定位目标。 + const handleVideoLoadedMetadata = (video: HTMLVideoElement) => { + if (video.videoWidth > 0 && video.videoHeight > 0) setVideoSize({ width: video.videoWidth, height: video.videoHeight }); + if (targetSeekSecRef.current != null && video.readyState >= 1) { + const target = Math.min(targetSeekSecRef.current, Math.max(0, (video.duration || 0) - 0.05)); + if (Math.abs(video.currentTime - target) > 0.05) video.currentTime = target; + } + }; + + const handleTimeUpdate = () => { + const video = videoRef.current; + if (!video || !activeVideoClip || !playing) return; + const sourceStartSec = (activeVideoClip.sourceStartMs || 0) / 1000; + const timelineMs = activeVideoClip.startMs + Math.round((video.currentTime - sourceStartSec) * 1000); + onPlayheadChange(Math.max(0, timelineMs)); + const clipEndMs = activeVideoClip.startMs + activeVideoClip.durationMs; + if (timelineMs >= clipEndMs) { + const next = videoClips.find((clip) => clip.startMs >= clipEndMs - AUTO_ADVANCE_GAP_MS && clip.id !== activeVideoClip.id); + if (next) { + onPlayheadChange(next.startMs); + } else { + onTogglePlay(); + } + } + }; + + const previewDisplay = useMemo(() => { + if (!videoSize || videoSize.height <= 0) return null; + const ratio = videoSize.width / videoSize.height; + let width = PREVIEW_WIDTH; + let height = width / ratio; + if (height > PREVIEW_HEIGHT) { + height = PREVIEW_HEIGHT; + width = Math.round(height * ratio); + } + return { width: Math.round(width), height: Math.round(height) }; + }, [videoSize]); + + return ( +
+
+ {videoUrl && activeVideoClip ? ( + <> +
+
+ + + ) : ( +
该位置无视频片段
+ )} +
+
+
+ + {activeVideoClip?.title || "无视频片段"} + + {activeVideoClip ? `${formatTimelineTime(activeVideoClip.startMs)} ~ ${formatTimelineTime(activeVideoClip.startMs + activeVideoClip.durationMs)}` : ""} +
+
+ {activeSubtitleClip ? activeSubtitleClip.text : 此处无字幕，点选字幕片段后在下方编辑} +
+
+
+ ); +} diff --git a/web/src/components/canvas/canvas-timeline-ruler.tsx b/web/src/components/canvas/canvas-timeline-ruler.tsx new file mode 100644 index 000000000..ec40565d5 --- /dev/null +++ b/web/src/components/canvas/canvas-timeline-ruler.tsx @@ -0,0 +1,73 @@ +// 时间线标尺：按缩放渲染刻度与播放头，点击/拖拽设置播放头时间。 + +import { useMemo, useRef } from "react"; + +import { formatTimelineTime, getRulerTickStep } from "@/lib/timeline/timeline-view"; +import type { CanvasTheme } from "@/lib/canvas-theme"; + +type CanvasTimelineRulerProps = { + pxPerMs: number; + durationMs: number; + playheadMs: number; + width: number; + theme: CanvasTheme; + onSeek: (ms: number) => void; +}; + +export function CanvasTimelineRuler({ pxPerMs, durationMs, playheadMs, width, theme, onSeek }: CanvasTimelineRulerProps) { + const barRef = useRef(null); + const draggingRef = useRef(false); + + const ticks = useMemo(() => { + const stepMs = getRulerTickStep(pxPerMs); + const count = Math.max(1, Math.ceil(durationMs / stepMs)); + return Array.from({ length: count + 1 }, (_, index) => { + const ms = index * stepMs; + const x = Math.round(ms * pxPerMs); + return { ms, x, major: index % 5 === 0 }; + }); + }, [durationMs, pxPerMs]); + + const seekFromEvent = (clientX: number) => { + const rect = barRef.current?.getBoundingClientRect(); + if (!rect) return; + const ratio = Math.min(1, Math.max(0, (clientX - rect.left) / Math.max(1, rect.width))); + onSeek(Math.round(ratio * durationMs)); + }; + + return ( +
{ + draggingRef.current = true; + event.currentTarget.setPointerCapture(event.pointerId); + seekFromEvent(event.clientX); + }} + onPointerMove={(event) => { + if (draggingRef.current) seekFromEvent(event.clientX); + }} + onPointerUp={(event) => { + draggingRef.current = false; + event.currentTarget.releasePointerCapture(event.pointerId); + }} + > + {ticks.map((tick) => ( +
+
+ {tick.major ? ( + + {formatTimelineTime(tick.ms)} + + ) : null} +
+ ))} +
+ + {formatTimelineTime(playheadMs)} + +
+
+ ); +} diff --git a/web/src/components/canvas/canvas-toolbar.tsx b/web/src/components/canvas/canvas-toolbar.tsx index c3132f623..f9e2ad7ee 100644 --- a/web/src/components/canvas/canvas-toolbar.tsx +++ b/web/src/components/canvas/canvas-toolbar.tsx @@ -139,7 +139,7 @@ export function CanvasToolbar({ onNodeInfo: () => {}, onNodeDelete: () => {}, onNodeRetry: () => {}, onNodeEditText: () => {}, onNodeDecreaseFont: () => {}, onNodeIncreaseFont: () => {}, onNodeToggleDialog: () => {}, onNodeAnnotate: () => {}, onNodeGenerateImage: () => {}, onNodeUpload: () => {}, onNodeDownload: () => {}, onNodeSaveAsset: () => {}, onNodeMaskEdit: () => {}, onNodeEmotion: () => {}, onNodePortraitTexture: () => {}, onNodeCrop: () => {}, onNodeSplit: () => {}, onNodeUpscale: () => {}, - onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeReversePrompt: () => {}, + onNodeSuperResolve: () => {}, onNodeAngle: () => {}, onNodeViewImage: () => {}, onNodeExtractVideoLastFrame: () => {}, onNodeSubtitles: () => {}, onNodeTimeline: () => {}, onNodeReversePrompt: () => {}, onNodeToggleFreeResize: () => {}, onNodeToggleLocked: () => {}, onNodeCopyPrompt: () => {}, } as ToolbarHandlers; diff --git a/web/src/lib/canvas-theme.ts b/web/src/lib/canvas-theme.ts index 06abb593c..9e6fa0c32 100644 --- a/web/src/lib/canvas-theme.ts +++ b/web/src/lib/canvas-theme.ts @@ -43,6 +43,21 @@ export const canvasThemes = { glowStrong: "rgba(79,110,232,.52)", shadow: "rgba(15,23,42,.18)", }, + // 时间线/字幕组件语义色：轨道底、按类型区分的片段、标尺与播放头、字幕列表条目态。 + timeline: { + trackFill: "#f6f8fb", + trackBorder: "rgba(17,24,39,.08)", + clipVideo: "rgba(79,110,232,.16)", + clipAudio: "rgba(16,185,129,.14)", + clipSubtitle: "rgba(245,158,11,.16)", + clipSelectedBorder: "#4f6ee8", + handle: "rgba(255,255,255,.72)", + rulerTick: "rgba(17,24,39,.28)", + rulerLabel: "#6b7280", + playhead: "#4f6ee8", + entryActive: "rgba(79,110,232,.10)", + entryHover: "rgba(17,24,39,.05)", + }, accent: { primary: "#4f6ee8", primarySoft: "rgba(79,110,232,.14)", @@ -90,6 +105,20 @@ export const canvasThemes = { glowStrong: "rgba(143,155,214,.48)", shadow: "rgba(0,0,0,.6)", }, + timeline: { + trackFill: "#101114", + trackBorder: "rgba(255,255,255,.08)", + clipVideo: "rgba(96,126,234,.22)", + clipAudio: "rgba(16,185,129,.20)", + clipSubtitle: "rgba(245,158,11,.20)", + clipSelectedBorder: "#8f9bd6", + handle: "rgba(255,255,255,.5)", + rulerTick: "rgba(255,255,255,.26)", + rulerLabel: "#a3a3a3", + playhead: "#8f9bd6", + entryActive: "rgba(96,126,234,.16)", + entryHover: "rgba(255,255,255,.06)", + }, accent: { primary: "#8f9bd6", primarySoft: "rgba(107,120,190,.2)", diff --git a/web/src/lib/canvas/canvas-video-merge.ts b/web/src/lib/canvas/canvas-video-merge.ts index 6dd8d07c9..960e1a7f2 100644 --- a/web/src/lib/canvas/canvas-video-merge.ts +++ b/web/src/lib/canvas/canvas-video-merge.ts @@ -10,7 +10,7 @@ export type MergeVideoProgress = { phase: "loading" | "reading" | "encoding"; pr let ffmpegPromise: Promise | null = null; // ffmpeg 只在用户明确合并视频时加载，避免把 wasm 和 worker 放进画布首屏包体。 -async function loadFFmpeg(onProgress?: (progress: MergeVideoProgress) => void) { +export async function loadFFmpeg(onProgress?: (progress: MergeVideoProgress) => void) { if (!ffmpegPromise) { ffmpegPromise = (async () => { const ffmpeg = new FFmpeg(); diff --git a/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx b/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx index 5ed1aa84c..807d14856 100644 --- a/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx +++ b/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx @@ -1,4 +1,4 @@ -import { Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react"; +import { Captions, Clapperboard, Download, FolderPlus, GalleryHorizontalEnd, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react"; import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error"; import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry"; @@ -193,6 +193,30 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [ applicable: isVideo, run: (ctx) => ctx.handlers.onNodeUpload(ctx.node!), }, + { + id: "subtitles", + toolbar: "node-hover", + category: "node-state", + label: "编辑字幕", + displayLabel: "字幕", + icon: , + defaultVisible: true, + defaultOrder: 145, + applicable: (ctx) => hasVideo(ctx), + run: (ctx) => ctx.handlers.onNodeSubtitles(ctx.node!), + }, + { + id: "timeline", + toolbar: "node-hover", + category: "node-state", + label: "时间线编辑", + displayLabel: "时间线", + icon: , + defaultVisible: true, + defaultOrder: 148, + applicable: (ctx) => hasVideo(ctx) || hasAudio(ctx), + run: (ctx) => ctx.handlers.onNodeTimeline(ctx.node!), + }, { id: "uploadAudio", toolbar: "node-hover", diff --git a/web/src/lib/canvas/tool-registry/tool-definition.ts b/web/src/lib/canvas/tool-registry/tool-definition.ts index ea5ec23e3..72c85b309 100644 --- a/web/src/lib/canvas/tool-registry/tool-definition.ts +++ b/web/src/lib/canvas/tool-registry/tool-definition.ts @@ -81,6 +81,8 @@ export type ToolbarHandlers = { onNodeAngle: (node: CanvasNodeData) => void; onNodeViewImage: (node: CanvasNodeData) => void; onNodeExtractVideoLastFrame: (node: CanvasNodeData) => void; + onNodeSubtitles: (node: CanvasNodeData) => void; + onNodeTimeline: (node: CanvasNodeData) => void; onNodeReversePrompt: (node: CanvasNodeData) => void; onNodeToggleFreeResize: (node: CanvasNodeData) => void; onNodeToggleLocked: (node: CanvasNodeData) => void; diff --git a/web/src/lib/timeline/srt-parser.ts b/web/src/lib/timeline/srt-parser.ts new file mode 100644 index 000000000..4466ce387 --- /dev/null +++ b/web/src/lib/timeline/srt-parser.ts @@ -0,0 +1,67 @@ +import type { SrtEntry } from "@/types/timeline"; + +function timeToMs(timestamp: string): number { + const [hours, minutes, secondsAndMs] = timestamp.split(":"); + const [seconds, milliseconds] = secondsAndMs.split(","); + + return Number.parseInt(hours, 10) * 3_600_000 + Number.parseInt(minutes, 10) * 60_000 + Number.parseInt(seconds, 10) * 1_000 + Number.parseInt(milliseconds, 10); +} + +export function parseSrt(content: string): SrtEntry[] { + if (!content.trim()) { + return []; + } + + const blocks = content.trim().split(/\r?\n\s*\r?\n/); + const entries: SrtEntry[] = []; + + for (const block of blocks) { + const lines = block.trim().split(/\r?\n/); + if (lines.length < 3) { + continue; + } + + const index = Number.parseInt(lines[0], 10); + if (Number.isNaN(index)) { + continue; + } + + const match = lines[1].match(/\d{2}:\d{2}:\d{2},\d{3}\s*-->\s*\d{2}:\d{2}:\d{2},\d{3}/); + if (!match) { + continue; + } + + const [start, end] = match[0].split(/\s*-->\s*/); + entries.push({ + index, + startMs: timeToMs(start), + endMs: timeToMs(end), + text: lines.slice(2).join("\n"), + }); + } + + return entries; +} + +function msToTimestamp(ms: number): string { + const safeMs = Math.max(0, Math.round(ms)); + const hours = Math.floor(safeMs / 3_600_000); + const minutes = Math.floor((safeMs % 3_600_000) / 60_000); + const seconds = Math.floor((safeMs % 60_000) / 1_000); + const milliseconds = safeMs % 1_000; + return `${String(hours).padStart(2, "0")}:` + `${String(minutes).padStart(2, "0")}:` + `${String(seconds).padStart(2, "0")},` + `${String(milliseconds).padStart(3, "0")}`; +} + +export function serializeSrtEntries(entries: SrtEntry[]): string { + if (entries.length === 0) { + return ""; + } + return ( + entries + .map((entry, idx) => { + const index = Number.isInteger(entry.index) && entry.index > 0 ? entry.index : idx + 1; + return `${index}\n${msToTimestamp(entry.startMs)} --> ${msToTimestamp(entry.endMs)}\n${entry.text}`; + }) + .join("\n\n") + "\n" + ); +} diff --git a/web/src/lib/timeline/srt-resegment.ts b/web/src/lib/timeline/srt-resegment.ts new file mode 100644 index 000000000..7c59f9fb1 --- /dev/null +++ b/web/src/lib/timeline/srt-resegment.ts @@ -0,0 +1,131 @@ +import type { SrtEntry } from "@/types/timeline"; +import { serializeSrtEntries } from "./srt-parser"; + +export const DEFAULT_MAX_CHARS_PER_ENTRY = 35; +export const MIN_CHARS_PER_ENTRY = 20; +export const MAX_CHARS_PER_ENTRY_LIMIT = 60; +export const MIN_SEGMENT_DURATION_MS = 300; + +const CJK_PUNCTUATION = "，。；！？、："; +const LATIN_PUNCTUATION = ",.;!?:"; + +/** + * 在 text 中为 targetLen 附近找最佳断点。 + * 返回切分位置 cut（前段为 text.slice(0, cut)，后段为 text.slice(cut)）。 + * 优先级：中文标点 > 英文标点 > 空格 > 硬切 targetLen。 + * 在同一优先级内选最靠右的位置（靠近 targetLen）。 + */ +export function findBestBreakPoint(text: string, targetLen: number): number { + if (text.length <= targetLen) { + return text.length; + } + + const windowStart = Math.max(1, Math.floor(targetLen * 0.6)); + const windowEnd = targetLen; + + // 优先级 1：中文标点 — 从右向左扫描，最靠右优先 + for (let i = windowEnd - 1; i >= windowStart - 1; i -= 1) { + if (CJK_PUNCTUATION.includes(text[i])) { + return i + 1; + } + } + + // 优先级 2：英文标点 + for (let i = windowEnd - 1; i >= windowStart - 1; i -= 1) { + if (LATIN_PUNCTUATION.includes(text[i])) { + return i + 1; + } + } + + // 优先级 3：空格 + for (let i = windowEnd - 1; i >= windowStart - 1; i -= 1) { + if (text[i] === " ") { + return i + 1; + } + } + + // 优先级 4：硬切 + return targetLen; +} + +/** + * 对已切分好的段列表应用最小时长约束（MIN_SEGMENT_DURATION_MS）。 + * 逐段检查：若某段时长不足，将其 endMs 向后推，同时把下一段的 startMs 相应推移。 + * 最后一段的 endMs 始终等于原始 entry.endMs，不做调整。 + * 若总时长本身不足以满足所有段的最小时长，接受违规（不崩溃，不重排）。 + */ +function enforceMinDuration(segments: SrtEntry[], originalEndMs: number): SrtEntry[] { + if (segments.length <= 1) { + return segments; + } + const result = segments.map((s) => ({ ...s })); + for (let i = 0; i < result.length - 1; i++) { + const dur = result[i].endMs - result[i].startMs; + if (dur < MIN_SEGMENT_DURATION_MS) { + const needed = MIN_SEGMENT_DURATION_MS - dur; + // 最多能推到下一段结束（即 originalEndMs 对于最后两段），但不超过 originalEndMs + const cap = Math.min(result[i + 1].endMs, originalEndMs); + const newEnd = Math.min(result[i].endMs + needed, cap); + result[i].endMs = newEnd; + result[i + 1].startMs = newEnd; + } + } + // 确保末段 endMs 不变 + result[result.length - 1].endMs = originalEndMs; + return result; +} + +/** + * 把一个超长 entry 切成若干不超过 maxChars 的子 entry，递归切分。 + * 时间按字符数等比分配，最终应用 MIN_SEGMENT_DURATION_MS 下限。 + */ +export function splitLongEntry(entry: SrtEntry, maxChars: number): SrtEntry[] { + if (entry.text.length <= maxChars) { + return [entry]; + } + + const cut = findBestBreakPoint(entry.text, maxChars); + let frontText = entry.text.slice(0, cut).replace(/\s+$/, ""); + let backText = entry.text.slice(cut).replace(/^\s+/, ""); + + if (frontText.length === 0 || backText.length === 0) { + // 退化情况：空白裁剪后一侧为空，回退到硬切 + frontText = entry.text.slice(0, maxChars); + backText = entry.text.slice(maxChars); + } + + const totalLen = frontText.length + backText.length; + const durationMs = entry.endMs - entry.startMs; + const frontDuration = Math.round((durationMs * frontText.length) / totalLen); + const splitPointMs = entry.startMs + frontDuration; + + const frontEntry: SrtEntry = { + index: entry.index, + startMs: entry.startMs, + endMs: splitPointMs, + text: frontText, + }; + const backEntry: SrtEntry = { + index: entry.index, + startMs: splitPointMs, + endMs: entry.endMs, + text: backText, + }; + + const frontSegments = frontText.length > maxChars ? splitLongEntry(frontEntry, maxChars) : [frontEntry]; + const backSegments = backText.length > maxChars ? splitLongEntry(backEntry, maxChars) : [backEntry]; + + const segments = [...frontSegments, ...backSegments]; + return enforceMinDuration(segments, entry.endMs); +} + +/** + * 遍历 entries，对每条超长的调用 splitLongEntry，最后重新编号 index 为 1..N。 + */ +export function resegmentSrtEntries(entries: SrtEntry[], maxChars: number): SrtEntry[] { + const split: SrtEntry[] = []; + for (const entry of entries) { + split.push(...splitLongEntry(entry, maxChars)); + } + return split.map((entry, idx) => ({ ...entry, index: idx + 1 })); +} diff --git a/web/src/lib/timeline/subtitle-highlight-ai.ts b/web/src/lib/timeline/subtitle-highlight-ai.ts new file mode 100644 index 000000000..e8ce027da --- /dev/null +++ b/web/src/lib/timeline/subtitle-highlight-ai.ts @@ -0,0 +1,53 @@ +import type { SrtEntry } from "@/types/timeline"; + +export interface SubtitleHighlightLLMResult { + entryIndex: number; + shouldHighlight: boolean; + highlightText: string; + start: number; + end: number; +} + +export function buildSubtitleHighlightSystemPrompt(): string { + return `你是一个视频字幕关键词高亮助手。请输出严格 JSON。 + +任务目标： +- 为每条字幕判断是否需要高亮 +- 每条字幕最多高亮 1 段 +- 只返回最值得高亮的一个关键词或短语 +- start 和 end 使用 JavaScript 字符串下标 +- end 为 exclusive +- 没有明确重点时返回 shouldHighlight=false + +高亮标准： +- 优先高亮结论词、数字、身份词、首次/突破词、强反差词 +- 不要高亮虚词、口头语、整句长短语 + +边界约束： +- 不负责颜色、字号、位置、动效 +- 只返回 highlights 数组 +- highlightText 必须严格等于原字幕文本切片结果`; +} + +export function buildSubtitleHighlightUserMessage(entries: SrtEntry[]): string { + const lines = entries.map((entry) => { + return `entryIndex=${entry.index} | startMs=${entry.startMs} | endMs=${entry.endMs} | text=${entry.text}`; + }); + + return `请根据以下字幕和上下文，为每条字幕判断是否需要高亮，并返回 JSON： + +{ + "highlights": [ + { + "entryIndex": 1, + "shouldHighlight": true, + "highlightText": "示例", + "start": 0, + "end": 2 + } + ] +} + +字幕列表： +${lines.join("\n")}`; +} diff --git a/web/src/lib/timeline/subtitle-highlight-runner.ts b/web/src/lib/timeline/subtitle-highlight-runner.ts new file mode 100644 index 000000000..bd2ee63af --- /dev/null +++ b/web/src/lib/timeline/subtitle-highlight-runner.ts @@ -0,0 +1,113 @@ +import { runBackendCanvasGenerationTask } from "@/lib/canvas/canvas-project-generation"; +import type { AiConfig } from "@/stores/use-config-store"; +import type { SrtEntry, SubtitleHighlight } from "@/types/timeline"; +import { buildSubtitleHighlightSystemPrompt, buildSubtitleHighlightUserMessage } from "./subtitle-highlight-ai"; +import { parseSubtitleHighlightResponse } from "./subtitle-highlight-service"; + +export interface SubtitleHighlightProgress { + batchIndex: number; + batchTotal: number; + processedEntries: number; + totalEntries: number; + percent: number; +} + +export type SubtitleHighlightRunnerOptions = { + projectId: string; + /** 画布节点 id，用于任务元数据标记来源 */ + nodeId: string; + config: AiConfig; + /** 每个 batch 的字幕条数。默认 30 */ + batchSize?: number; + /** 同时进行的 batch 数。默认 3；LLM 请求是 I/O 密集型，3-4 并行通常稳定低于 provider 限速 */ + concurrency?: number; + signal?: AbortSignal; + onProgress?: (progress: SubtitleHighlightProgress) => void; +}; + +export async function generateSubtitleHighlights(entries: SrtEntry[], options: SubtitleHighlightRunnerOptions): Promise { + if (entries.length === 0) { + return []; + } + + const batchSize = Math.max(1, options.batchSize ?? 30); + const concurrency = Math.max(1, Math.floor(options.concurrency ?? 3)); + const systemPrompt = buildSubtitleHighlightSystemPrompt(); + + // 切批：保留原顺序，每个 batch 独立可并行 + const batches: { index: number; entries: SrtEntry[] }[] = []; + for (let i = 0; i < entries.length; i += batchSize) { + batches.push({ index: batches.length, entries: entries.slice(i, i + batchSize) }); + } + const batchTotal = batches.length; + const results: SubtitleHighlight[][] = new Array(batchTotal).fill(null).map(() => []); + let processedEntries = 0; + + options.onProgress?.({ + batchIndex: 0, + batchTotal, + processedEntries: 0, + totalEntries: entries.length, + percent: 0, + }); + + // worker 池模式：从共享游标里抢任务，保证 concurrency 个 batch 同时跑 + let cursor = 0; + const firstError: { current: unknown } = { current: null }; + + const runOne = async (): Promise => { + while (true) { + if (firstError.current || options.signal?.aborted) return; + const myIndex = cursor; + cursor += 1; + if (myIndex >= batches.length) return; + const batch = batches[myIndex]; + try { + const result = await runBackendCanvasGenerationTask({ + projectId: options.projectId, + nodeId: `${options.nodeId}:subtitle-highlight:${myIndex + 1}/${batchTotal}`, + mode: "text", + prompt: `${systemPrompt}\n\n${buildSubtitleHighlightUserMessage(batch.entries)}`, + config: options.config, + signal: options.signal, + metadata: { operation: "subtitle_highlight", batchIndex: myIndex, batchTotal }, + }); + results[myIndex] = parseSubtitleHighlightResponse(parseJsonObject(result.text || ""), batch.entries); + processedEntries += batch.entries.length; + options.onProgress?.({ + batchIndex: myIndex + 1, + batchTotal, + processedEntries, + totalEntries: entries.length, + percent: Math.round((processedEntries / entries.length) * 100), + }); + } catch (err) { + if (!firstError.current) firstError.current = err; + return; + } + } + }; + + await Promise.all(Array.from({ length: Math.min(concurrency, batchTotal) }, () => runOne())); + + if (options.signal?.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + if (firstError.current) { + const message = firstError.current instanceof Error ? firstError.current.message : "未知错误"; + throw new Error(`字幕关键词高亮生成失败：${message}`); + } + + return results.flat(); +} + +function parseJsonObject(text: string): unknown { + if (!text?.trim()) { + return null; + } + try { + return JSON.parse(text); + } catch { + return null; + } +} diff --git a/web/src/lib/timeline/subtitle-highlight-service.ts b/web/src/lib/timeline/subtitle-highlight-service.ts new file mode 100644 index 000000000..cfe62aef5 --- /dev/null +++ b/web/src/lib/timeline/subtitle-highlight-service.ts @@ -0,0 +1,44 @@ +import type { SrtEntry, SubtitleHighlight } from "@/types/timeline"; +import { filterValidSubtitleHighlights } from "./subtitle-highlights"; +import type { SubtitleHighlightLLMResult } from "./subtitle-highlight-ai"; + +function isSubtitleHighlightLLMResult(value: unknown): value is SubtitleHighlightLLMResult { + if (!value || typeof value !== "object") { + return false; + } + + const candidate = value as Record; + return Number.isFinite(candidate.entryIndex) && typeof candidate.shouldHighlight === "boolean" && typeof candidate.highlightText === "string" && Number.isFinite(candidate.start) && Number.isFinite(candidate.end); +} + +export function parseSubtitleHighlightResponse(payload: unknown, entries: SrtEntry[]): SubtitleHighlight[] { + if (!payload || typeof payload !== "object") { + return []; + } + + const rawHighlights = Array.isArray((payload as { highlights?: unknown[] }).highlights) ? (payload as { highlights: unknown[] }).highlights : []; + const entryMap = new Map(entries.map((entry) => [entry.index, entry])); + + const highlights = rawHighlights.flatMap((item) => { + if (!isSubtitleHighlightLLMResult(item) || !item.shouldHighlight) { + return []; + } + + const entry = entryMap.get(item.entryIndex); + if (!entry) { + return []; + } + + return [ + { + entryIndex: item.entryIndex, + start: item.start, + end: item.end, + highlightText: item.highlightText, + sourceText: entry.text, + }, + ]; + }); + + return filterValidSubtitleHighlights(entries, highlights); +} diff --git a/web/src/lib/timeline/subtitle-highlights.ts b/web/src/lib/timeline/subtitle-highlights.ts new file mode 100644 index 000000000..bf7702737 --- /dev/null +++ b/web/src/lib/timeline/subtitle-highlights.ts @@ -0,0 +1,91 @@ +import type { SrtEntry, SubtitleHighlight } from "@/types/timeline"; + +export function isValidSubtitleHighlight(entry: SrtEntry | undefined, highlight: SubtitleHighlight): boolean { + if (!entry || entry.index !== highlight.entryIndex) { + return false; + } + + if (highlight.start < 0 || highlight.end <= highlight.start) { + return false; + } + + if (highlight.end > highlight.sourceText.length) { + return false; + } + + return highlight.highlightText === highlight.sourceText.slice(highlight.start, highlight.end); +} + +export function isExpiredSubtitleHighlight(entry: SrtEntry | undefined, highlight: SubtitleHighlight): boolean { + if (!entry || entry.index !== highlight.entryIndex) { + return true; + } + + return entry.text !== highlight.sourceText; +} + +export function filterValidSubtitleHighlights(entries: SrtEntry[], highlights: SubtitleHighlight[]): SubtitleHighlight[] { + const entryMap = new Map(entries.map((entry) => [entry.index, entry])); + + return highlights.filter((highlight) => { + const entry = entryMap.get(highlight.entryIndex); + return isValidSubtitleHighlight(entry, highlight) && !isExpiredSubtitleHighlight(entry, highlight); + }); +} + +/** + * 在重分段后，把旧高亮映射到新条目上。 + * 规则：highlightText 必须作为子串出现在某个新条目的 text 中； + * 找到第一个匹配的条目，更新 entryIndex 和 start/end； + * 找不到则放入 dropped（通常是跨切分点的关键词）。 + */ +export function remapHighlightsAfterResegment(oldHighlights: SubtitleHighlight[], newEntries: SrtEntry[]): { remapped: SubtitleHighlight[]; dropped: SubtitleHighlight[] } { + const remapped: SubtitleHighlight[] = []; + const dropped: SubtitleHighlight[] = []; + + for (const highlight of oldHighlights) { + const target = newEntries.find((entry) => entry.text.includes(highlight.highlightText)); + if (!target) { + dropped.push(highlight); + continue; + } + const start = target.text.indexOf(highlight.highlightText); + remapped.push({ + entryIndex: target.index, + start, + end: start + highlight.highlightText.length, + highlightText: highlight.highlightText, + sourceText: target.text, + }); + } + + return { remapped, dropped }; +} + +// 无可用大模型配置时的本地回退：按终止标点取首段作为高亮，保证功能不中断。 +const TERMINAL_PUNCTUATION = "。！？；"; + +export function buildFallbackHighlights(entries: SrtEntry[]): SubtitleHighlight[] { + const highlights: SubtitleHighlight[] = []; + for (const entry of entries) { + let terminalIndex = -1; + for (let i = 0; i < entry.text.length; i += 1) { + if (TERMINAL_PUNCTUATION.includes(entry.text[i])) { + terminalIndex = i; + break; + } + } + if (terminalIndex <= 0) { + continue; + } + const end = terminalIndex + 1; + highlights.push({ + entryIndex: entry.index, + start: 0, + end, + highlightText: entry.text.slice(0, end), + sourceText: entry.text, + }); + } + return highlights; +} diff --git a/web/src/lib/timeline/timeline-build.ts b/web/src/lib/timeline/timeline-build.ts new file mode 100644 index 000000000..d8385fb4e --- /dev/null +++ b/web/src/lib/timeline/timeline-build.ts @@ -0,0 +1,184 @@ +// 从画布节点构建时间线数据：视频/音频节点自动入轨，字幕条目转成字幕片段。 +// 构建是单向快照：时间线保存后用户可自由拖拽，重新构建不覆盖已有时间线。 + +import type { CanvasNodeData } from "@/types/canvas"; +import type { SrtEntry, TimelineClip, TimelineProject } from "@/types/timeline"; +import { DEFAULT_AUDIO_TRACK_ID, DEFAULT_SUBTITLE_TRACK_ID, DEFAULT_VIDEO_TRACK_ID, createDefaultTracks } from "./timeline-tracks"; +import { getTimelineVisualEndMs } from "./timeline-view"; + +function nodeDurationMs(node: CanvasNodeData, fallbackMs = 4_000): number { + const durationMs = node.metadata?.durationMs; + return durationMs && durationMs > 0 ? Math.round(durationMs) : fallbackMs; +} + +/** + * 从画布节点生成时间线片段。 + * - 视频节点 → 视频轨片段（顺序排布，首尾相接）； + * - 音频节点 → 音频轨片段（顺序排布）； + * - 视频节点的字幕条目 → 字幕轨片段（相对该视频片段起点偏移）。 + */ +export function buildTimelineFromNodes(nodes: CanvasNodeData[]): TimelineProject { + const videoNodes = nodes.filter((node) => node.type === "video" && Boolean(node.metadata?.content || node.metadata?.storageKey)); + const audioNodes = nodes.filter((node) => node.type === "audio" && Boolean(node.metadata?.content || node.metadata?.storageKey)); + const clips: TimelineClip[] = []; + + let videoCursorMs = 0; + for (const node of videoNodes) { + const durationMs = nodeDurationMs(node); + clips.push({ + id: `clip-${node.id}`, + kind: "video", + nodeId: node.id, + trackId: DEFAULT_VIDEO_TRACK_ID, + startMs: videoCursorMs, + durationMs, + title: node.title || "视频片段", + sourceStartMs: 0, + sourceDurationMs: durationMs, + }); + + const entries = node.metadata?.subtitleEntries || []; + for (const entry of entries) { + const startMs = videoCursorMs + Math.max(0, entry.startMs); + const endMs = videoCursorMs + Math.max(entry.startMs, Math.min(entry.endMs, durationMs)); + clips.push({ + id: `clip-${node.id}-sub-${entry.index}`, + kind: "subtitle", + nodeId: node.id, + trackId: DEFAULT_SUBTITLE_TRACK_ID, + startMs, + durationMs: Math.max(100, endMs - startMs), + title: entry.text || "字幕", + sourceStartMs: entry.startMs, + sourceDurationMs: Math.max(0, entry.endMs - entry.startMs), + subtitleEntryIndex: entry.index, + text: entry.text, + }); + } + videoCursorMs += durationMs; + } + + let audioCursorMs = 0; + for (const node of audioNodes) { + const durationMs = nodeDurationMs(node); + clips.push({ + id: `clip-${node.id}`, + kind: "audio", + nodeId: node.id, + trackId: DEFAULT_AUDIO_TRACK_ID, + startMs: audioCursorMs, + durationMs, + title: node.title || "音频片段", + sourceStartMs: 0, + sourceDurationMs: durationMs, + volume: 1, + fadeInMs: 0, + fadeOutMs: 0, + }); + audioCursorMs += durationMs; + } + + return { + version: 2, + tracks: createDefaultTracks(), + clips, + durationMs: getTimelineVisualEndMs(clips), + updatedAt: new Date().toISOString(), + }; +} + +/** 判断某个节点是否已经出现在时间线中（按 nodeId 去重）。 */ +export function isNodeInTimeline(nodeId: string, timeline: TimelineProject | null | undefined): boolean { + return Boolean(timeline?.clips.some((clip) => clip.nodeId === nodeId)); +} + +/** + * 将节点字幕条目同步回项目时间线：按视频片段起点偏移重建该节点的字幕片段； + * 条目为空数组时移除该节点全部字幕片段，保证字幕弹窗与时间线互通。 + */ +export function syncNodeSubtitlesToTimeline(timeline: TimelineProject, nodeId: string, entries: SrtEntry[]): TimelineProject { + const videoClip = timeline.clips.find((clip) => clip.nodeId === nodeId && clip.kind === "video"); + const hasSubtitleClips = timeline.clips.some((clip) => clip.nodeId === nodeId && clip.kind === "subtitle"); + if (!videoClip && !hasSubtitleClips) return timeline; + const baseMs = videoClip?.startMs ?? 0; + const sourceDurationMs = videoClip?.sourceDurationMs || videoClip?.durationMs || 0; + const clips = timeline.clips.filter((clip) => !(clip.nodeId === nodeId && clip.kind === "subtitle")); + for (const entry of entries) { + const text = (entry.text || "").trim(); + if (!text) continue; + const startMs = baseMs + Math.max(0, entry.startMs); + const endMs = baseMs + Math.max(entry.startMs, Math.min(entry.endMs, sourceDurationMs || entry.endMs)); + clips.push({ + id: `clip-${nodeId}-sub-${entry.index}`, + kind: "subtitle", + nodeId, + trackId: DEFAULT_SUBTITLE_TRACK_ID, + startMs, + durationMs: Math.max(100, endMs - startMs), + title: text, + sourceStartMs: entry.startMs, + sourceDurationMs: Math.max(0, entry.endMs - entry.startMs), + subtitleEntryIndex: entry.index, + text, + }); + } + return { + ...timeline, + clips, + durationMs: getTimelineVisualEndMs(clips), + updatedAt: new Date().toISOString(), + }; +} + +/** + * 打开时间线弹窗时的兜底同步：以时间线内已有视频片段为锚点，按节点 + * subtitleEntries 重建该节点的字幕片段（删除旧片段、补齐新增条目）。 + * 仅处理已入轨的节点，避免把未入轨节点的字幕凭空加进时间线。 + */ +export function syncTimelineSubtitleClips(timeline: TimelineProject, nodes: CanvasNodeData[]): TimelineProject { + let changed = false; + const clips = [...timeline.clips]; + for (const videoClip of timeline.clips) { + if (videoClip.kind !== "video") continue; + const node = nodes.find((item) => item.id === videoClip.nodeId); + if (!node) continue; + const stale = clips.filter((clip) => clip.kind === "subtitle" && clip.nodeId === videoClip.nodeId); + if (stale.length) { + for (const staleClip of stale) { + const index = clips.indexOf(staleClip); + if (index >= 0) clips.splice(index, 1); + } + changed = true; + } + const entries = node.metadata?.subtitleEntries || []; + const baseMs = videoClip.startMs; + const sourceDurationMs = videoClip.sourceDurationMs || videoClip.durationMs || 0; + for (const entry of entries) { + const text = (entry.text || "").trim(); + if (!text) continue; + const startMs = baseMs + Math.max(0, entry.startMs); + const endMs = baseMs + Math.max(entry.startMs, Math.min(entry.endMs, sourceDurationMs || entry.endMs)); + clips.push({ + id: `clip-${videoClip.nodeId}-sub-${entry.index}`, + kind: "subtitle", + nodeId: videoClip.nodeId, + trackId: DEFAULT_SUBTITLE_TRACK_ID, + startMs, + durationMs: Math.max(100, endMs - startMs), + title: text, + sourceStartMs: entry.startMs, + sourceDurationMs: Math.max(0, entry.endMs - entry.startMs), + subtitleEntryIndex: entry.index, + text, + }); + changed = true; + } + } + if (!changed) return timeline; + return { + ...timeline, + clips, + durationMs: getTimelineVisualEndMs(clips), + updatedAt: new Date().toISOString(), + }; +} diff --git a/web/src/lib/timeline/timeline-export.ts b/web/src/lib/timeline/timeline-export.ts new file mode 100644 index 000000000..ae66c1558 --- /dev/null +++ b/web/src/lib/timeline/timeline-export.ts @@ -0,0 +1,93 @@ +// 第三期：时间线导出运行时。把 buildTimelineRenderPlan 产出的步骤逐步在 ffmpeg.wasm 中执行， +// 复用 canvas-video-merge.ts 的 loadFFmpeg（本地同源加载 core/wasm），媒体读写与清理逻辑保持一致。 +import { fetchFile } from "@ffmpeg/util"; + +import { loadFFmpeg } from "@/lib/canvas/canvas-video-merge"; +import { getMediaBlob } from "@/services/file-storage"; +import type { TimelineProject } from "@/types/timeline"; +import { SUBTITLE_FILE, buildSubtitleSrt, buildTimelineRenderPlan, getOrderedSubtitleClips, type TimelineRenderContext, type TimelineRenderSource } from "./timeline-to-ffmpeg"; + +export type TimelineExportProgress = { + phase: "loading" | "reading" | "encoding"; + percent: number; + detail: string; +}; + +export type TimelineExportOptions = { + onProgress?: (progress: TimelineExportProgress) => void; + context?: Partial; +}; + +async function fetchSourceBlob(source: TimelineRenderSource): Promise { + if (source.storageKey) { + const stored = await getMediaBlob(source.storageKey); + if (stored) return stored; + } + if (source.url) { + const response = await fetch(source.url); + if (!response.ok) throw new Error("视频资源请求失败（" + response.status + "）"); + return response.blob(); + } + throw new Error("找不到素材 " + source.nodeId + " 的媒体文件"); +} + +/** 导出时间线为 MP4：写媒体 → 按计划逐步执行 → 返回 Blob（下载由调用方处理） */ +export async function exportTimelineToMp4(timeline: TimelineProject, sources: TimelineRenderSource[], options: TimelineExportOptions = {}): Promise { + const { onProgress, context } = options; + onProgress?.({ phase: "loading", percent: 0, detail: "加载 FFmpeg" }); + const ffmpeg = await loadFFmpeg(({ phase, progress }) => onProgress?.({ phase: phase === "loading" ? "loading" : "reading", percent: progress, detail: phase === "loading" ? "加载 FFmpeg" : "读取素材" })); + const plan = buildTimelineRenderPlan(timeline, sources, context); + const writtenFiles = new Set(); + + try { + for (const source of sources) { + onProgress?.({ phase: "reading", percent: 5, detail: "读取素材 " + source.fileName }); + const blob = await fetchSourceBlob(source); + await ffmpeg.writeFile(source.fileName, await fetchFile(blob)); + writtenFiles.add(source.fileName); + } + + const executableSteps = plan.steps.filter((step) => step.kind !== "subtitle"); + let stepIndex = 0; + for (const step of plan.steps) { + if (step.kind === "subtitle") { + const srt = buildSubtitleSrt(getOrderedSubtitleClips(timeline)); + if (srt) { + await ffmpeg.writeFile(SUBTITLE_FILE, new TextEncoder().encode(srt)); + writtenFiles.add(SUBTITLE_FILE); + } + continue; + } + if (step.args.includes("concat.txt")) { + const content = plan.concatEntries.map((file) => "file '" + file + "'").join("\n"); + await ffmpeg.writeFile("concat.txt", new TextEncoder().encode(content)); + writtenFiles.add("concat.txt"); + } + onProgress?.({ phase: "encoding", percent: Math.round(10 + (stepIndex / Math.max(1, executableSteps.length)) * 75), detail: step.description }); + + if (step.kind === "burn") { + const concatOutput = step.args[step.args.indexOf("-i") + 1]; + try { + const exitCode = await ffmpeg.exec(["-y", ...step.args]); + if (exitCode !== 0) throw new Error("burn exit " + exitCode); + } catch { + // 当前 @ffmpeg/core 可能不含 libass（subtitles 滤镜），回退为直接封装已拼接视频。 + onProgress?.({ phase: "encoding", percent: 92, detail: "字幕烧录不可用，回退为无字幕导出" }); + const exitCode = await ffmpeg.exec(["-y", "-i", concatOutput, "-c", "copy", plan.finalOutput]); + if (exitCode !== 0) throw new Error("导出失败：字幕烧录与回退均失败"); + } + } else { + const exitCode = await ffmpeg.exec(["-y", ...step.args]); + if (exitCode !== 0) throw new Error("导出失败：" + step.description); + } + writtenFiles.add(step.output); + stepIndex += 1; + } + + const output = await ffmpeg.readFile(plan.finalOutput); + onProgress?.({ phase: "encoding", percent: 100, detail: "导出完成" }); + return new Blob([output as BlobPart], { type: "video/mp4" }); + } finally { + await Promise.all([...writtenFiles, "concat.txt", plan.finalOutput].map((file) => ffmpeg.deleteFile(file).catch(() => undefined))); + } +} diff --git a/web/src/lib/timeline/timeline-placement.ts b/web/src/lib/timeline/timeline-placement.ts new file mode 100644 index 000000000..ec033726f --- /dev/null +++ b/web/src/lib/timeline/timeline-placement.ts @@ -0,0 +1,196 @@ +// 时间线放置算法（移植自 lingji-cut 的 timeline-placement.ts）。 +// 提供同轨碰撞检测、最近可用位置查找、跨轨放置与时长钳制。 + +import type { TimelineClip, TimelineTrack } from "@/types/timeline"; + +export type PlacementResult = { + startMs: number; + fits: boolean; +}; + +export type FindNearestArgs = { + targetStartMs: number; + durationMs: number; + trackId: string; + excludeClipId?: string; + clips: TimelineClip[]; +}; + +export type PlacementTrackResult = { + trackId: string | null; + startMs: number; +}; + +export type FindAvailableTrackArgs = { + targetStartMs: number; + durationMs: number; + excludeClipId?: string; + clips: TimelineClip[]; + tracks: TimelineTrack[]; +}; + +export type ClampDurationArgs = { + clipId: string; + startMs: number; + requestedDurationMs: number; + trackId: string; + clips: TimelineClip[]; + maxDurationMs?: number; +}; + +/** 半开区间 [startMs, startMs + durationMs) 重叠判断 */ +export function clipsOverlap(left: { startMs: number; durationMs: number }, right: { startMs: number; durationMs: number }): boolean { + return left.startMs < right.startMs + right.durationMs && right.startMs < left.startMs + left.durationMs; +} + +export type CanPlaceAtArgs = { + trackId: string; + startMs: number; + durationMs: number; + excludeClipId?: string; + clips: TimelineClip[]; +}; + +export type CanPlaceAtResult = { + ok: boolean; + reason?: "overlap"; +}; + +/** + * 判断指定轨道的区间 [startMs, startMs+durationMs) 是否可放置， + * 不做任何自动 snap/偏移；遇到任何同轨片段重叠即返回 ok=false。 + */ +export function canPlaceAt(args: CanPlaceAtArgs): CanPlaceAtResult { + const { trackId, startMs, durationMs, excludeClipId, clips } = args; + const candidate = { startMs, durationMs }; + for (const other of clips) { + if (other.trackId !== trackId) continue; + if (other.id === excludeClipId) continue; + if (clipsOverlap(candidate, other)) { + return { ok: false, reason: "overlap" }; + } + } + return { ok: true }; +} + +/** 返回与候选区间碰撞的所有片段（按现有顺序，不排序）。 */ +export function findCollidingItems(args: CanPlaceAtArgs): TimelineClip[] { + const { trackId, startMs, durationMs, excludeClipId, clips } = args; + const candidate = { startMs, durationMs }; + return clips.filter((c) => c.trackId === trackId && c.id !== excludeClipId && clipsOverlap(candidate, c)); +} + +function getSortedClipsOnTrack(trackId: string, clips: TimelineClip[], excludeClipId?: string): TimelineClip[] { + return clips.filter((c) => c.trackId === trackId && c.id !== excludeClipId).sort((a, b) => a.startMs - b.startMs); +} + +/** 在指定轨道上寻找离 targetStartMs 最近的可用放置位置 */ +export function findNearestAvailablePlacement(args: FindNearestArgs): PlacementResult { + const { targetStartMs, durationMs, trackId, excludeClipId, clips } = args; + const managed = getSortedClipsOnTrack(trackId, clips, excludeClipId); + + if (managed.length === 0) { + return { startMs: targetStartMs, fits: true }; + } + + const candidate = { startMs: targetStartMs, durationMs }; + const hasConflict = managed.some((c) => clipsOverlap(candidate, c)); + if (!hasConflict) { + return { startMs: targetStartMs, fits: true }; + } + + // 目标位置有冲突，扫描所有间隙寻找最近的可用位置： + // 第一个片段之前、片段之间、最后一个片段之后（无限大）。 + type Gap = { start: number; end: number }; + const gaps: Gap[] = []; + + if (managed[0].startMs > 0) { + gaps.push({ start: 0, end: managed[0].startMs }); + } + + for (let i = 0; i < managed.length - 1; i++) { + const gapStart = managed[i].startMs + managed[i].durationMs; + const gapEnd = managed[i + 1].startMs; + if (gapEnd > gapStart) { + gaps.push({ start: gapStart, end: gapEnd }); + } + } + + const lastEnd = managed[managed.length - 1].startMs + managed[managed.length - 1].durationMs; + gaps.push({ start: lastEnd, end: Number.POSITIVE_INFINITY }); + + let bestStart: number | null = null; + let bestDistance = Number.POSITIVE_INFINITY; + + for (const gap of gaps) { + const gapSize = gap.end - gap.start; + if (gapSize < durationMs) continue; + + let candidateStart: number; + if (targetStartMs >= gap.start && targetStartMs + durationMs <= gap.end) { + candidateStart = targetStartMs; + } else if (targetStartMs < gap.start) { + candidateStart = gap.start; + } else { + candidateStart = Math.max(gap.start, gap.end - durationMs); + } + + if (candidateStart + durationMs > gap.end && gap.end !== Number.POSITIVE_INFINITY) { + continue; + } + + const distance = Math.abs(candidateStart - targetStartMs); + if (distance < bestDistance) { + bestDistance = distance; + bestStart = candidateStart; + } + } + + if (bestStart !== null) { + return { startMs: bestStart, fits: true }; + } + + return { startMs: targetStartMs, fits: false }; +} + +/** 在所有可放置的视觉/音轨中寻找可放置的轨道（按 order 升序） */ +export function findAvailableTrack(args: FindAvailableTrackArgs): PlacementTrackResult { + const { targetStartMs, durationMs, excludeClipId, clips, tracks } = args; + const candidates = tracks.filter((t) => t.kind === "video" || t.kind === "audio").sort((a, b) => a.order - b.order); + + for (const track of candidates) { + const result = findNearestAvailablePlacement({ + targetStartMs, + durationMs, + trackId: track.id, + excludeClipId, + clips, + }); + if (result.fits) { + return { trackId: track.id, startMs: result.startMs }; + } + } + + return { trackId: null, startMs: targetStartMs }; +} + +/** 根据相邻片段和最大时长限制，钳制片段时长 */ +export function clampClipDurationByNeighbors(args: ClampDurationArgs): number { + const { clipId, startMs, requestedDurationMs, trackId, clips, maxDurationMs } = args; + const managed = getSortedClipsOnTrack(trackId, clips, clipId); + + let maxGap = requestedDurationMs; + for (const c of managed) { + if (c.startMs > startMs) { + const gap = c.startMs - startMs; + maxGap = Math.min(maxGap, gap); + break; + } + } + + let result = Math.min(requestedDurationMs, maxGap); + if (maxDurationMs !== undefined) { + result = Math.min(result, maxDurationMs); + } + return Math.max(0, result); +} diff --git a/web/src/lib/timeline/timeline-snap.ts b/web/src/lib/timeline/timeline-snap.ts new file mode 100644 index 000000000..bdfa15956 --- /dev/null +++ b/web/src/lib/timeline/timeline-snap.ts @@ -0,0 +1,61 @@ +// 时间线吸附算法（移植自 lingji-cut 的 timeline-snap.ts）。 +// 候选时间点吸附到播放头或其它片段边缘（起点/终点），取阈值内最近目标。 + +import type { TimelineClip } from "@/types/timeline"; + +export type SnapTargetKind = "playhead" | "clip-edge"; + +export type SnapTarget = { + ms: number; + kind: SnapTargetKind; +}; + +export type ComputeSnapArgs = { + candidateMs: number; + playheadMs: number; + clips: TimelineClip[]; + excludeClipId?: string; + pxPerMs: number; + thresholdPx: number; + enabled: boolean; +}; + +export type ComputeSnapResult = { + snappedMs: number; + targets: SnapTarget[]; +}; + +export function computeSnap(args: ComputeSnapArgs): ComputeSnapResult { + const { candidateMs, playheadMs, clips, excludeClipId, pxPerMs, thresholdPx, enabled } = args; + if (!enabled) { + return { snappedMs: candidateMs, targets: [] }; + } + + const thresholdMs = thresholdPx / Math.max(pxPerMs, 1e-6); + const candidates: SnapTarget[] = []; + + if (Math.abs(candidateMs - playheadMs) <= thresholdMs) { + candidates.push({ ms: playheadMs, kind: "playhead" }); + } + + for (const clip of clips) { + if (clip.id === excludeClipId) continue; + const start = clip.startMs; + const end = clip.startMs + clip.durationMs; + if (Math.abs(candidateMs - start) <= thresholdMs) { + candidates.push({ ms: start, kind: "clip-edge" }); + } + if (Math.abs(candidateMs - end) <= thresholdMs) { + candidates.push({ ms: end, kind: "clip-edge" }); + } + } + + if (candidates.length === 0) { + return { snappedMs: candidateMs, targets: [] }; + } + + candidates.sort((a, b) => Math.abs(a.ms - candidateMs) - Math.abs(b.ms - candidateMs)); + const chosen = candidates[0]; + const sameMsTargets = candidates.filter((t) => t.ms === chosen.ms); + return { snappedMs: chosen.ms, targets: sameMsTargets }; +} diff --git a/web/src/lib/timeline/timeline-to-ffmpeg.ts b/web/src/lib/timeline/timeline-to-ffmpeg.ts new file mode 100644 index 000000000..7c7625905 --- /dev/null +++ b/web/src/lib/timeline/timeline-to-ffmpeg.ts @@ -0,0 +1,204 @@ +// 第三期：时间线 → FFmpeg 命令序列的纯函数规划层。 +// 不直接调用 FFmpeg，只产出可执行的参数计划，方便单测与运行时逐步执行； +// 运行时负责把媒体源写入 ffmpeg 工作区、写 SRT、执行 args 并清理文件。 +// 数据流：TimelineProject + 节点媒体 → trim（按 sourceStart/sourceDuration 裁切）→ 黑场补齐空隙 → concat → 字幕 SRT → 烧录。 + +import type { TimelineClip, TimelineProject } from "@/types/timeline"; + +export type TimelineRenderSource = { + nodeId: string; + /** 已写入 ffmpeg 工作区的文件名（如 input-0.mp4） */ + fileName: string; + durationMs: number; + /** 媒体定位（运行时用）：本地缓存 storageKey 或远程资源地址，至少提供一个 */ + storageKey?: string; + url?: string; +}; + +export type TimelineRenderStep = { + kind: "trim" | "gap" | "concat" | "subtitle" | "burn"; + /** 本步骤输出文件名 */ + output: string; + /** ffmpeg 参数数组（不含可执行文件名与 -y 覆盖参数） */ + args: string[]; + description: string; + /** 该步骤依赖 libass（subtitles 滤镜）；当前 ffmpeg.wasm 内核可能不含，运行时需探测回退 */ + requiresLibass?: boolean; +}; + +export type TimelineRenderContext = { + width: number; + height: number; + fps: number; + /** 是否烧录字幕；false 时跳过 burn 步骤 */ + burnSubtitles: boolean; + /** 最终输出文件名 */ + outputName: string; +}; + +export type TimelineRenderPlan = { + steps: TimelineRenderStep[]; + finalOutput: string; + /** concat 输入文件列表（trim/gap 输出），运行时据此写 concat.txt */ + concatEntries: string[]; +}; + +export const SUBTITLE_FILE = "timeline.srt"; + +export function getOrderedVideoClips(timeline: TimelineProject): TimelineClip[] { + return timeline.clips + .filter((clip) => clip.kind === "video") + .slice() + .sort((a, b) => a.startMs - b.startMs || a.trackId.localeCompare(b.trackId)); +} + +export function getOrderedSubtitleClips(timeline: TimelineProject): TimelineClip[] { + return timeline.clips + .filter((clip) => clip.kind === "subtitle") + .slice() + .sort((a, b) => a.startMs - b.startMs || a.trackId.localeCompare(b.trackId)); +} + +/** 毫秒 → SRT 时间码 hh:mm:ss,mmm */ +export function formatSrtTimestamp(ms: number): string { + const safe = Math.max(0, Math.round(ms)); + const pad = (value: number, length = 2) => String(value).padStart(length, "0"); + const hours = Math.floor(safe / 3_600_000); + const minutes = Math.floor((safe % 3_600_000) / 60_000); + const seconds = Math.floor((safe % 60_000) / 1_000); + const millis = safe % 1_000; + return `${pad(hours)}:${pad(minutes)}:${pad(seconds)},${pad(millis, 3)}`; +} + +/** 字幕轨片段 → SRT 文件内容（时间线全局时间，直接用于烧录） */ +export function buildSubtitleSrt(clips: TimelineClip[]): string { + return getOrderedSubtitleClips({ version: 2, tracks: [], clips, durationMs: 0 }) + .filter((clip) => clip.text && clip.durationMs > 0) + .map((clip, index) => { + const text = (clip.text || "").replace(/\r?\n/g, " ").trim(); + return [String(index + 1), `${formatSrtTimestamp(clip.startMs)} --> ${formatSrtTimestamp(clip.startMs + clip.durationMs)}`, text].join("\n"); + }) + .join("\n\n"); +} + +function defaultContext(): TimelineRenderContext { + return { width: 1920, height: 1080, fps: 30, burnSubtitles: true, outputName: "export.mp4" }; +} + +/** + * 生成导出计划。 + * 视频轨按 startMs 顺序裁切并补齐空隙（lavfi 黑场 + 静音），最后 concat； + * 字幕轨独立生成 SRT 并在末步烧录（libass 滤镜），音频轨暂按现有音频片段拼接（第三期迭代）。 + */ +export function buildTimelineRenderPlan(timeline: TimelineProject, sources: TimelineRenderSource[], context: Partial = {}): TimelineRenderPlan { + const cfg: TimelineRenderContext = { ...defaultContext(), ...context }; + const sourceByNode = new Map(sources.map((item) => [item.nodeId, item])); + const steps: TimelineRenderStep[] = []; + const concatEntries: string[] = []; + const videoClips = getOrderedVideoClips(timeline); + + // 1) 逐片段裁切：按源素材内部时间定位，输出统一编码便于 concat。 + videoClips.forEach((clip, index) => { + const source = sourceByNode.get(clip.nodeId); + if (!source) return; + const output = `trim-${index}.mp4`; + // 裁切时长取时间线片段时长（clip.durationMs），而不是源素材剩余时长： + // 左缘裁剪后 sourceStartMs 前移但 sourceDurationMs 仍为源全长，若按源时长 -t 会把旧片段尾部多裁出来， + // 表现为「裁剪后播放仍从最原始视频开始/出现旧片段」；-ss 已定位源内起点，-t 必须等于片段展示时长。 + const durationSec = Math.max(0.1, clip.durationMs / 1000); + steps.push({ + kind: "trim", + output, + args: ["-ss", String((clip.sourceStartMs || 0) / 1000), "-i", source.fileName, "-t", String(durationSec), "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "128k", output], + description: `裁切片段 ${index + 1}（${clip.title || clip.nodeId}）`, + }); + concatEntries.push(output); + }); + + // 2) 空隙补黑场（时间线与成片对齐），含静音音轨。 + let cursorMs = 0; + videoClips.forEach((clip, index) => { + const gapMs = clip.startMs - cursorMs; + if (gapMs > 100) { + const output = `gap-${index}.mp4`; + const durationSec = gapMs / 1000; + steps.push({ + kind: "gap", + output, + args: [ + "-f", + "lavfi", + "-i", + `color=c=black:s=${cfg.width}x${cfg.height}:r=${cfg.fps}:d=${durationSec}`, + "-f", + "lavfi", + "-i", + "anullsrc=r=44100:cl=stereo", + "-t", + String(durationSec), + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "20", + "-c:a", + "aac", + "-shortest", + output, + ], + description: `补黑场 ${(gapMs / 1000).toFixed(2)}s`, + }); + concatEntries.push(output); + } + cursorMs = clip.startMs + clip.durationMs; + }); + + // 3) concat 拼接视频轨。 + const concatOutput = "timeline-video.mp4"; + if (concatEntries.length) { + steps.push({ + kind: "concat", + output: concatOutput, + args: ["-f", "concat", "-safe", "0", "-i", "concat.txt", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", concatOutput], + description: "拼接视频轨", + }); + } + + // 4) 字幕轨 → SRT 内容（运行时写入 SUBTITLE_FILE）。 + const subtitleClips = getOrderedSubtitleClips(timeline); + if (cfg.burnSubtitles && subtitleClips.some((clip) => clip.text)) { + steps.push({ + kind: "subtitle", + output: SUBTITLE_FILE, + args: [], + description: "生成字幕 SRT", + }); + } + + // 5) 烧录字幕并输出最终文件（subtitles 滤镜需要 libass）。 + const finalOutput = cfg.outputName; + if (concatEntries.length && steps.some((step) => step.kind === "subtitle")) { + steps.push({ + kind: "burn", + output: finalOutput, + args: ["-i", concatOutput, "-vf", `subtitles=${SUBTITLE_FILE}`, "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "copy", finalOutput], + description: "烧录字幕并输出", + requiresLibass: true, + }); + } else if (concatEntries.length) { + steps.push({ + kind: "concat", + output: finalOutput, + args: ["-i", concatOutput, "-c", "copy", finalOutput], + description: "输出成片（无字幕）", + }); + } + + return { steps, finalOutput, concatEntries }; +} + +/** 供导出对话框/文档展示的人类可读命令预览 */ +export function describeRenderPlan(plan: TimelineRenderPlan): string { + return plan.steps.map((step) => `# ${step.description}\nffmpeg -y ${step.args.join(" ")}`).join("\n\n"); +} diff --git a/web/src/lib/timeline/timeline-tracks.ts b/web/src/lib/timeline/timeline-tracks.ts new file mode 100644 index 000000000..b8f2858e2 --- /dev/null +++ b/web/src/lib/timeline/timeline-tracks.ts @@ -0,0 +1,54 @@ +// 时间线轨道管理（移植自 lingji-cut 的 timeline-tracks.ts，按本项目类型精简）。 +// 默认三轨：视频轨、音频轨、字幕轨；提供轨道查询与规范化。 + +import type { TimelineProject, TimelineTrack } from "@/types/timeline"; + +export const DEFAULT_VIDEO_TRACK_ID = "video-1"; +export const DEFAULT_AUDIO_TRACK_ID = "audio-1"; +export const DEFAULT_SUBTITLE_TRACK_ID = "subtitle-1"; + +export function createDefaultTracks(): TimelineTrack[] { + return [ + { id: DEFAULT_VIDEO_TRACK_ID, kind: "video", label: "视频 1", order: 0 }, + { id: DEFAULT_AUDIO_TRACK_ID, kind: "audio", label: "音频 1", order: 1 }, + { id: DEFAULT_SUBTITLE_TRACK_ID, kind: "subtitle", label: "字幕 1", order: 2 }, + ]; +} + +export function getVisualTracks(tracks: TimelineTrack[]): TimelineTrack[] { + return tracks.filter((track) => track.kind === "video" || track.kind === "image" || track.kind === "text").sort((a, b) => a.order - b.order || a.id.localeCompare(b.id)); +} + +export function getAudioTracks(tracks: TimelineTrack[]): TimelineTrack[] { + return tracks.filter((track) => track.kind === "audio").sort((a, b) => a.order - b.order || a.id.localeCompare(b.id)); +} + +export function getSubtitleTracks(tracks: TimelineTrack[]): TimelineTrack[] { + return tracks.filter((track) => track.kind === "subtitle").sort((a, b) => a.order - b.order || a.id.localeCompare(b.id)); +} + +export function getTracksByKind(tracks: TimelineTrack[], kind: TimelineTrack["kind"]): TimelineTrack[] { + return tracks.filter((track) => track.kind === kind).sort((a, b) => a.order - b.order || a.id.localeCompare(b.id)); +} + +/** 规范化时间线数据：轨道去重、补默认轨、时长取片段末端最大值。 */ +export function normalizeTimelineProject(timeline: Partial | null | undefined): TimelineProject { + const rawTracks = Array.isArray(timeline?.tracks) ? timeline.tracks : []; + const seen = new Set(); + const deduped = rawTracks.filter((track) => { + if (seen.has(track.id)) return false; + seen.add(track.id); + return true; + }); + const baseTracks = createDefaultTracks(); + const tracks = [...baseTracks, ...deduped.filter((track) => !baseTracks.some((base) => base.id === track.id))]; + const clips = Array.isArray(timeline?.clips) ? timeline.clips : []; + const durationMs = clips.reduce((max, clip) => Math.max(max, clip.startMs + clip.durationMs), 0); + return { + version: 2, + tracks, + clips, + durationMs, + updatedAt: timeline?.updatedAt, + }; +} diff --git a/web/src/lib/timeline/timeline-view.ts b/web/src/lib/timeline/timeline-view.ts new file mode 100644 index 000000000..8ac99d2fe --- /dev/null +++ b/web/src/lib/timeline/timeline-view.ts @@ -0,0 +1,67 @@ +// 时间线视图数学（移植自 lingji-cut 的 timeline-view.ts，按本项目类型精简）。 +// 提供缩放钳制、像素/毫秒换算、标尺刻度与时间格式化。 + +const BASE_TIMELINE_PX_PER_SECOND = 96; +const MIN_TIMELINE_TRACK_WIDTH = 960; +const MIN_TIMELINE_ZOOM = 0.02; +const MAX_TIMELINE_ZOOM = 4; +const TIMELINE_ZOOM_STEP = 1.25; + +type ZoomDirection = "in" | "out"; + +function roundZoom(value: number): number { + return Math.round(value * 100) / 100; +} + +export function clampTimelineZoom(zoomLevel: number): number { + return roundZoom(Math.min(MAX_TIMELINE_ZOOM, Math.max(MIN_TIMELINE_ZOOM, zoomLevel))); +} + +export function getBaseTimelineWidth(durationMs: number): number { + return Math.max(MIN_TIMELINE_TRACK_WIDTH, Math.ceil(Math.max(1_000, durationMs) / 1_000) * BASE_TIMELINE_PX_PER_SECOND); +} + +export function getNextTimelineZoom(zoomLevel: number, direction: ZoomDirection): number { + const nextZoom = direction === "in" ? zoomLevel * TIMELINE_ZOOM_STEP : zoomLevel / TIMELINE_ZOOM_STEP; + return clampTimelineZoom(nextZoom); +} + +export function zoomIn(zoomLevel: number): number { + return getNextTimelineZoom(zoomLevel, "in"); +} + +export function zoomOut(zoomLevel: number): number { + return getNextTimelineZoom(zoomLevel, "out"); +} + +export function getFitTimelineZoom(durationMs: number, viewportWidth: number): number { + const safeViewportWidth = Math.max(320, viewportWidth); + return clampTimelineZoom(safeViewportWidth / getBaseTimelineWidth(durationMs)); +} + +export function getTimelineTrackWidth(durationMs: number, zoomLevel: number, viewportWidth: number): number { + const safeViewportWidth = Math.max(320, viewportWidth); + const zoomedWidth = Math.round(getBaseTimelineWidth(durationMs) * clampTimelineZoom(zoomLevel)); + return Math.max(safeViewportWidth, zoomedWidth); +} + +export function getTimelineVisualEndMs(clips: { startMs: number; durationMs: number }[]): number { + return clips.reduce((max, clip) => Math.max(max, clip.startMs + clip.durationMs), 0); +} + +/** 标尺刻度间隔（毫秒），保证刻度间隔像素不小于 64px。 */ +export function getRulerTickStep(pxPerMs: number): number { + const steps = [100, 250, 500, 1_000, 2_000, 5_000, 10_000, 15_000, 30_000, 60_000, 120_000, 300_000]; + const target = 64 / Math.max(pxPerMs, 1e-6); + return steps.find((step) => step >= target) ?? steps[steps.length - 1]; +} + +/** 毫秒 → 标尺/播放头时间文案（mm:ss.d）。 */ +export function formatTimelineTime(ms: number): string { + const safeMs = Math.max(0, Math.round(ms)); + const totalSeconds = Math.floor(safeMs / 1_000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + const tenths = Math.floor((safeMs % 1_000) / 100); + return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}.${tenths}`; +} diff --git a/web/src/pages/canvas/project.tsx b/web/src/pages/canvas/project.tsx index f674c214e..8175dac64 100644 --- a/web/src/pages/canvas/project.tsx +++ b/web/src/pages/canvas/project.tsx @@ -30,6 +30,10 @@ import { WorkspaceState } from "@/components/layout/workspace-state"; import { resolveProjectCanvasStyle } from "@/components/canvas/canvas-style-picker-modal"; import { createStyleProfileSnapshot, resolveStyleProfile, serializeStyleProfile } from "@/lib/canvas/style-profile"; import { CanvasNodeToolbar, CanvasNodeInfoModal } from "@/components/canvas/canvas-node-toolbar"; +import { CanvasSubtitleDialog } from "@/components/canvas/canvas-subtitle-dialog"; +import { CanvasTimelineDialog } from "@/components/canvas/canvas-timeline-dialog"; +import { syncNodeSubtitlesToTimeline } from "@/lib/timeline/timeline-build"; +import type { TimelineDirectMedia } from "@/types/timeline"; import { CanvasNodeAnglePanel } from "@/components/canvas/canvas-node-angle-dialog"; import { CanvasTextEditorModal } from "@/components/canvas/canvas-text-editor-modal"; import { CanvasNodeSearchModal } from "@/components/canvas/canvas-node-search-modal"; @@ -40,7 +44,7 @@ import { InfiniteCanvas } from "@/components/canvas/infinite-canvas"; import { Minimap } from "@/components/canvas/canvas-mini-map"; import { CanvasNodePromptPanel, type CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel"; import { CanvasToolbar } from "@/components/canvas/canvas-toolbar"; -import { AssetPickerModal } from "@/components/canvas/asset-picker-modal"; +import { AssetPickerModal, type InsertAssetPayload } from "@/components/canvas/asset-picker-modal"; import { getProject } from "@/services/api/projects"; import { CanvasZoomControls } from "@/components/canvas/canvas-zoom-controls"; import { CanvasShareModal } from "@/components/canvas/canvas-share-modal"; @@ -186,6 +190,8 @@ function InfiniteCanvasPage() { const [projectAssetInitialCategory, setProjectAssetInitialCategory] = useState("all"); const [projectAssetInsertPosition, setProjectAssetInsertPosition] = useState(); const [infoNodeId, setInfoNodeId] = useState(null); + const [subtitleNodeId, setSubtitleNodeId] = useState(null); + const [timelineNodeId, setTimelineNodeId] = useState(null); const [superResolveNodeId, setSuperResolveNodeId] = useState(null); const [previewNodeId, setPreviewNodeId] = useState(null); const [scriptEditorNodeId, setScriptEditorNodeId] = useState(null); @@ -304,6 +310,7 @@ function InfiniteCanvasPage() { }); const linkedProjectId = shortDramaEnabled ? currentProject?.projectId || "" : ""; const linkedProjectQuery = useQuery({ queryKey: ["project", linkedProjectId], queryFn: () => getProject(linkedProjectId), enabled: Boolean(linkedProjectId) }); + const refetchLinkedProject = linkedProjectQuery.refetch; useEffect(() => { if (!projectLoaded || !linkedProjectQuery.data) return; setNodes((current) => refreshCanvasCharacterReferenceNodes(current, linkedProjectQuery.data.assets)); @@ -434,6 +441,7 @@ function InfiniteCanvasPage() { assetPickerOpen, closeUploadModal, closeAssetPicker, + createVideoNodeFromBlob, createImageAssetNode, fileDropActive, handleAssetInsert, @@ -452,6 +460,7 @@ function InfiniteCanvasPage() { pasteSystemClipboard, startUploadStatus, uploadModalOpen, + uploadTimelineMedia, uploadStatus, } = useCanvasUpload({ canvasId: projectId, @@ -467,11 +476,100 @@ function InfiniteCanvasPage() { setDialogNodeId, }); - const openProjectAssets = useCallback((initialCategory = "all", position?: Position) => { - setProjectAssetInitialCategory(initialCategory); - setProjectAssetInsertPosition(position); - setProjectAssetOpen(true); - }, []); + // 时间线弹窗内新增素材的回填通道：素材库/上传创建节点后由弹窗通过 ref 加入草稿。 + const timelineAddNodeRef = useRef<((node: CanvasNodeData) => void) | null>(null); + // 时间线作用域直连媒体入轨通道：素材库/项目资产/本地上传不落画布，仅加入时间线草稿。 + const timelineMediaAddRef = useRef<((media: TimelineDirectMedia) => void) | null>(null); + // 素材库与项目资产弹窗的插入作用域：时间线弹窗内打开时为 timeline，其余为 canvas。 + const [assetInsertScope, setAssetInsertScope] = useState<"canvas" | "timeline">("canvas"); + const [projectAssetScope, setProjectAssetScope] = useState<"canvas" | "timeline">("canvas"); + + // InsertAssetPayload → 直连媒体：仅音视频支持直接入轨；图片/文本/角色返回 null（避免在画布重复出现）。 + const payloadToTimelineMedia = (payload: InsertAssetPayload): TimelineDirectMedia | null => { + const randomSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + if (payload.kind === "video") { + return { + id: payload.assetId || `asset-${randomSuffix}`, + kind: "video", + title: payload.title, + storageKey: payload.storageKey, + url: payload.url, + width: payload.width, + height: payload.height, + durationMs: payload.durationMs, + bytes: payload.bytes, + mimeType: payload.mimeType, + }; + } + if (payload.kind === "audio") { + return { id: payload.assetId || `asset-${randomSuffix}`, kind: "audio", title: payload.title, storageKey: payload.storageKey, url: payload.url, durationMs: payload.durationMs, bytes: payload.bytes, mimeType: payload.mimeType }; + } + return null; + }; + + const handleTimelineAssetInsert = useCallback( + async (payload: InsertAssetPayload) => { + if (assetInsertScope === "timeline") { + const media = payloadToTimelineMedia(payload); + if (media) { + timelineMediaAddRef.current?.(media); + closeAssetPicker(); + } else { + message.info("图片/文本素材暂不支持直接入轨，请先在画布中添加节点"); + } + return; + } + const node = await handleAssetInsert(payload, { openDialog: false }); + if (node) timelineAddNodeRef.current?.(node); + }, + [assetInsertScope, closeAssetPicker, handleAssetInsert], + ); + + // 项目资产库引入到时间线：复用现有引入逻辑，把创建出的节点回填到弹窗草稿。 + const handleTimelineProjectAssetsInsert = useCallback( + async (payloads: InsertAssetPayload[]) => { + if (projectAssetScope === "timeline") { + let inserted = 0; + for (const payload of payloads) { + const media = payloadToTimelineMedia(payload); + if (media) { + timelineMediaAddRef.current?.(media); + inserted += 1; + } + } + if (inserted < payloads.length) message.info("图片/文本/角色素材暂不支持直接入轨，仅音视频素材已加入时间线"); + return; + } + const created = await handleProjectAssetsInsert(payloads, projectAssetInsertPosition); + created.forEach((node) => timelineAddNodeRef.current?.(node)); + }, + [handleProjectAssetsInsert, message, projectAssetInsertPosition, projectAssetScope], + ); + + const openProjectAssets = useCallback( + (initialCategory = "all", position?: Position, scope: "canvas" | "timeline" = "canvas") => { + setProjectAssetScope(scope); + setProjectAssetInitialCategory(initialCategory); + setProjectAssetInsertPosition(position); + setProjectAssetOpen(true); + // 资产与项目实时同步：打开弹窗前刷新关联短剧项目资产，避免缓存导致资产列表空白/过期。 + if (linkedProjectId) void refetchLinkedProject(); + }, + [linkedProjectId, refetchLinkedProject], + ); + + // 素材库打开入口：画布作用域（工具栏/空态/侧栏）与时间线作用域（时间线弹窗）分别标记插入目标。 + const openCanvasAssetLibrary = useCallback( + (position?: Position) => { + setAssetInsertScope("canvas"); + openAssetsAtPosition(position); + }, + [openAssetsAtPosition], + ); + const openTimelineAssetLibrary = useCallback(() => { + setAssetInsertScope("timeline"); + openAssetsAtPosition(); + }, [openAssetsAtPosition]); const closeProjectAssets = useCallback(() => { setProjectAssetOpen(false); setProjectAssetInsertPosition(undefined); @@ -536,6 +634,7 @@ function InfiniteCanvasPage() { setCharacterReferenceNodeId(clearDeletedId); setDrawingNodeId(clearDeletedId); setInfoNodeId(clearDeletedId); + setSubtitleNodeId(clearDeletedId); setCropNodeId(clearDeletedId); setMaskEditNodeId(clearDeletedId); setAnnotationNodeId(clearDeletedId); @@ -774,6 +873,8 @@ function InfiniteCanvasPage() { dialogNodeId, }); const dialogNode = dialogNodeId ? nodeById.get(dialogNodeId) || null : null; + const subtitleNode = subtitleNodeId ? nodeById.get(subtitleNodeId) || null : null; + const timelineNode = timelineNodeId ? nodeById.get(timelineNodeId) || null : null; const textEditorNode = textEditorNodeId ? nodeById.get(textEditorNodeId) || null : null; const characterReferenceNode = characterReferenceNodeId ? nodeById.get(characterReferenceNodeId) || null : null; const drawingNode = drawingNodeId ? nodeById.get(drawingNodeId) || null : null; @@ -889,6 +990,7 @@ function InfiniteCanvasPage() { setTextEditorNodeId(null); setDrawingNodeId(null); setInfoNodeId(null); + setSubtitleNodeId(null); setCropNodeId(null); setMaskEditNodeId(null); setAnnotationNodeId(null); @@ -1596,7 +1698,7 @@ function InfiniteCanvasPage() { onBackgroundModeChange={setBackgroundMode} onShowImageInfoChange={setShowImageInfo} onOpenMyAssets={() => { - openAssetsAtPosition(); + openCanvasAssetLibrary(); }} onOpenProjectCharacters={() => openProjectAssets("character")} /> @@ -1738,6 +1840,8 @@ function InfiniteCanvasPage() { }} onViewImage={(node) => setPreviewNodeId(node.id)} onExtractVideoLastFrame={(node) => void extractVideoLastFrame(node)} + onSubtitles={(node) => setSubtitleNodeId(node.id)} + onTimeline={(node) => setTimelineNodeId(node.id)} extractingVideoFrame={toolbarNode?.id === extractingVideoFrameNodeId} onReversePrompt={createImageReversePromptNodes} onRetry={(node) => void handleRetryNode(node)} @@ -1790,7 +1894,7 @@ function InfiniteCanvasPage() { onChooseStyle={() => setStylePickerOpen(true)} onOpenDirector={createDirectorShot} onUpload={(nodeId, position) => handleUploadRequest(nodeId, position)} - onOpenAssets={openAssetsAtPosition} + onOpenAssets={openCanvasAssetLibrary} onOpenProjectCharacters={(position) => openProjectAssets("character", position)} onUndo={undoCanvas} onRedo={redoCanvas} @@ -1822,6 +1926,52 @@ function InfiniteCanvasPage() { setInfoNodeId(null)} onMetadataChange={handleConfigNodeChange} /> + {subtitleNode ? ( + setSubtitleNodeId(null)} + onSave={(nodeId, patch) => { + handleConfigNodeChange(nodeId, patch); + const currentTimeline = currentProject?.timeline; + if (currentTimeline) { + const next = syncNodeSubtitlesToTimeline(currentTimeline, nodeId, patch.subtitleEntries || []); + if (next !== currentTimeline) updateProject(projectId, { timeline: next }); + } + }} + /> + ) : null} + + {timelineNode ? ( + setTimelineNodeId(null)} + onOpenSubtitleDialog={(subNodeId) => { + setTimelineNodeId(null); + setSubtitleNodeId(subNodeId); + }} + onSave={(next) => updateProject(projectId, { timeline: next })} + onSaveSubtitles={(subNodeId, entries) => + handleConfigNodeChange(subNodeId, { + subtitleEntries: entries, + ...(entries.length ? {} : { subtitleHighlights: [] }), + subtitleUpdatedAt: new Date().toISOString(), + }) + } + onOpenAssetLibrary={openTimelineAssetLibrary} + onOpenProjectAssets={() => openProjectAssets("all", undefined, "timeline")} + onUploadLocalFiles={uploadTimelineMedia} + addNodeToTimelineRef={timelineAddNodeRef} + addMediaToTimelineRef={timelineMediaAddRef} + onCreateAssembledNode={createVideoNodeFromBlob} + /> + ) : null} + setCharacterReferenceNodeId(null)} /> - - handleProjectAssetsInsert(payloads, projectAssetInsertPosition)} - /> + + {codexCompactAgent && !assistantMounted ? ( ) : null} diff --git a/web/src/pages/canvas/shared.tsx b/web/src/pages/canvas/shared.tsx index b01389fdd..7524cd2ff 100644 --- a/web/src/pages/canvas/shared.tsx +++ b/web/src/pages/canvas/shared.tsx @@ -256,7 +256,7 @@ export default function SharedCanvasPage() { }} onHoverStart={keepToolbar} onHoverEnd={hideToolbar} onConnectStart={unauthorized} onResize={() => undefined} onContentChange={unauthorized} onRetry={unauthorized} onCancelTask={unauthorized} onOpenTaskDetails={unauthorized} onViewImage={(target) => setInfoNodeId(target.id)} onContextMenu={(event, nodeId) => openContextMenu(event, nodeId)} />)} - setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoLastFrame={unauthorized} extractingVideoFrame={false} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} /> + setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onViewImage={unauthorized} onExtractVideoLastFrame={unauthorized} extractingVideoFrame={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} />
访客操作仅在当前页面临时生效
diff --git a/web/src/pages/canvas/use-canvas-upload.ts b/web/src/pages/canvas/use-canvas-upload.ts index 72d40e11e..99cf6fda5 100644 --- a/web/src/pages/canvas/use-canvas-upload.ts +++ b/web/src/pages/canvas/use-canvas-upload.ts @@ -17,6 +17,7 @@ import { getProjectUnit } from "@/services/api/projects"; import { ensureCanvasNodeAsset } from "@/services/project-asset-sync"; import { useAssetStore, type ImageAsset } from "@/stores/use-asset-store"; import { CanvasNodeType, type CanvasNodeData, type ContextMenuState, type Position } from "@/types/canvas"; +import type { TimelineDirectMedia } from "@/types/timeline"; import type { CanvasUploadStatus } from "./canvas-project-feedback"; type UseCanvasUploadOptions = { @@ -330,6 +331,85 @@ export function useCanvasUpload({ return true; }, [createAudioFileNode, createImageFileNode, createVideoFileNode, getCanvasCenter, message, setDialogNodeId, setSelectedConnectionId, setSelectedNodeIds]); + // 时间线专用：把本地音视频文件上传为直连媒体（仅时间线作用域，不创建画布节点），返回媒体描述数组。 + const uploadTimelineMedia = useCallback(async (files: File[]): Promise => { + const supportedFiles = files.filter((file) => file.type.startsWith("video/") || isAudioFile(file)); + if (!supportedFiles.length) { + message.warning("请选择视频、MP3 或 WAV 文件"); + return []; + } + const created: TimelineDirectMedia[] = []; + for (const file of supportedFiles) { + try { + if (isAudioFile(file)) { + const audio = await uploadMediaFile(file, "audio"); + created.push({ + id: `audio-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`, + kind: "audio", + title: file.name, + storageKey: audio.storageKey, + url: audio.url, + durationMs: audio.durationMs, + bytes: audio.bytes, + mimeType: audio.mimeType, + }); + } else { + const video = await uploadMediaFile(file, "video"); + created.push({ + id: `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`, + kind: "video", + title: file.name, + storageKey: video.storageKey, + url: video.url, + width: video.width, + height: video.height, + durationMs: video.durationMs, + bytes: video.bytes, + mimeType: video.mimeType, + }); + } + } catch (error) { + message.error(error instanceof Error ? `素材上传失败：${error.message}` : "素材上传失败"); + } + } + if (created.length) message.success(`已上传 ${created.length} 个素材到时间线`); + return created; + }, [message]); + + // 组装能力闭环：把时间线合成结果（MP4 Blob）上传并创建为新的视频节点放回画布， + // 复用上传/持久化/选中逻辑，新节点可继续编辑字幕与样式。 + const createVideoNodeFromBlob = useCallback(async (blob: Blob, title: string): Promise => { + const progress = startUploadStatus("合成视频片段", "上传合成结果", domainProjectId ? 4 : 3); + try { + progress.update("上传到服务器并同步资源", 2); + const video = await uploadMediaFile(blob, "video"); + progress.update("更新画布节点", 3); + const size = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height); + const center = getCanvasCenter(); + const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const node = { + id, + type: CanvasNodeType.Video, + title, + position: { x: center.x - size.width / 2, y: center.y - size.height / 2 }, + width: size.width, + height: size.height, + metadata: { ...videoMetadata(video), status: NODE_STATUS_SUCCESS }, + } satisfies CanvasNodeData; + setNodes((current) => [...current, node]); + selectInsertedNode(id, "preserve"); + if (domainProjectId) progress.update("写入项目资产", 4); + const persisted = await persistMediaNode(node); + progress.done(persisted ? "已生成新视频片段并加入项目资产" : "已生成新视频片段，项目资产待重试"); + return node; + } catch (error) { + const details = error instanceof Error ? error.message : "合成视频片段失败"; + progress.fail(details); + message.error(details); + return null; + } + }, [domainProjectId, getCanvasCenter, message, persistMediaNode, selectInsertedNode, setNodes, startUploadStatus]); + const replaceNodeMedia = useCallback(async (nodeId: string, file: File) => { const currentNode = nodesRef.current.find((node) => node.id === nodeId); if (!currentNode) return false; @@ -632,20 +712,21 @@ export function useCanvasUpload({ return { id, type: CanvasNodeType.Image, title: payload.title.slice(0, 32) || "Generated Image", position: { x: center.x - size.width / 2, y: center.y - size.height / 2 }, width: size.width, height: size.height, metadata: { ...imageMetadata({ ...storedImage, width: meta.width, height: meta.height }), prompt: payload.title, assetId: payload.assetId } } satisfies CanvasNodeData; }, []); - const handleAssetInsert = useCallback(async (payload: InsertAssetPayload) => { + const handleAssetInsert = useCallback(async (payload: InsertAssetPayload, options: { openDialog?: boolean } = {}): Promise => { const center = assetInsertPositionRef.current || getCanvasCenter(); try { const node = await createAssetPayloadNode(payload, center); setNodes((current) => [...current, node]); - selectInsertedNode(node.id, payload.kind === "image" || payload.kind === "video" ? "open" : "preserve"); + selectInsertedNode(node.id, options.openDialog === false ? "preserve" : payload.kind === "image" || payload.kind === "video" ? "open" : "preserve"); + closeAssetPicker(); + return node; } catch (error) { message.error(error instanceof Error ? error.message : "素材插入失败"); - return; + return null; } - closeAssetPicker(); }, [closeAssetPicker, createAssetPayloadNode, getCanvasCenter, message, selectInsertedNode, setNodes]); - const handleProjectAssetsInsert = useCallback(async (payloads: InsertAssetPayload[], position?: Position) => { + const handleProjectAssetsInsert = useCallback(async (payloads: InsertAssetPayload[], position?: Position): Promise => { const origin = position || getCanvasCenter(); try { const created = await Promise.all(payloads.map((payload, index) => createAssetPayloadNode(payload, { @@ -657,6 +738,7 @@ export function useCanvasUpload({ setSelectedConnectionId(null); setDialogNodeId(null); message.success(`已引入 ${created.length} 项项目资产`); + return created; } catch (error) { message.error(error instanceof Error ? error.message : "项目资产引入失败"); throw error; @@ -666,6 +748,8 @@ export function useCanvasUpload({ return { assetPickerOpen, closeAssetPicker, + createVideoNodeFromBlob, + createAssetPayloadNode, createImageAssetNode, fileDropActive, handleAssetInsert, @@ -686,6 +770,7 @@ export function useCanvasUpload({ startUploadStatus, uploadModalOpen, uploadStatus, + uploadTimelineMedia, }; } diff --git a/web/src/stores/canvas/use-canvas-store.ts b/web/src/stores/canvas/use-canvas-store.ts index 243db325b..ffe567db0 100644 --- a/web/src/stores/canvas/use-canvas-store.ts +++ b/web/src/stores/canvas/use-canvas-store.ts @@ -6,6 +6,7 @@ import { localForageStorage } from "@/lib/localforage-storage"; import type { CanvasBackgroundMode } from "@/lib/canvas-theme"; import type { CanvasAssistantSession, CanvasConnection, CanvasNodeData, ViewportTransform } from "@/types/canvas"; import type { DirectorScene } from "@/types/director"; +import type { TimelineProject } from "@/types/timeline"; export type CanvasProject = { id: string; @@ -21,6 +22,7 @@ export type CanvasProject = { showImageInfo: boolean; viewport: ViewportTransform; directorScenes: DirectorScene[]; + timeline?: TimelineProject; }; type CanvasStore = { @@ -32,7 +34,7 @@ type CanvasStore = { renameProject: (id: string, title: string) => void; deleteProjects: (ids: string[]) => void; replaceProjects: (projects: CanvasProject[]) => void; - updateProject: (id: string, patch: Partial>) => void; + updateProject: (id: string, patch: Partial>) => void; }; const initialViewport: ViewportTransform = { x: 0, y: 0, k: 1 }; diff --git a/web/src/types/canvas.ts b/web/src/types/canvas.ts index e4f860cd0..3fefe6e7e 100644 --- a/web/src/types/canvas.ts +++ b/web/src/types/canvas.ts @@ -1,5 +1,6 @@ import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture"; import type { StyleExecutionPlan } from "@/lib/canvas/style-profile"; +import type { SrtEntry, SubtitleHighlight, SubtitleStyle } from "@/types/timeline"; export type Position = { x: number; @@ -228,6 +229,10 @@ export type CanvasNodeMetadata = { directorDepthNodeId?: string; directorNormalNodeId?: string; directorClayVideoNodeId?: string; + subtitleEntries?: SrtEntry[]; + subtitleHighlights?: SubtitleHighlight[]; + subtitleStyle?: SubtitleStyle; + subtitleUpdatedAt?: string; skillId?: string; skillVersion?: number; skillSnapshot?: CanvasSkillSnapshot; diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts new file mode 100644 index 000000000..f51b54183 --- /dev/null +++ b/web/src/types/timeline.ts @@ -0,0 +1,114 @@ +// 时间线与字幕的共享数据模型。第一期落字幕数据；第二期起时间线轨道/片段在此持久化。 + +export type SrtEntry = { + index: number; + startMs: number; + endMs: number; + text: string; +}; + +export type SubtitleHighlight = { + entryIndex: number; + start: number; + end: number; + highlightText: string; + sourceText: string; +}; + +export type SubtitlePosition = "top" | "bottom" | "center"; +export type SubtitleHighlightAnimation = "pop" | "wipe" | "none"; + +export type SubtitleStyle = { + fontSize: number; + color: string; + position: SubtitlePosition; + highlightEnabled: boolean; + highlightBackgroundColor: string; + highlightTextColor: string; + highlightPaddingX: number; + highlightPaddingY: number; + highlightRadius: number; + highlightAnimation: SubtitleHighlightAnimation; + /** 单条字幕最多字符数，超过则自动切分。默认 35，范围 20~60 */ + maxCharsPerEntry: number; + /** 是否启用自动切分。默认 true */ + autoResegment: boolean; +}; + +export function createDefaultSubtitleStyle(): SubtitleStyle { + return { + fontSize: 18, + color: "#ffffff", + position: "bottom", + highlightEnabled: true, + highlightBackgroundColor: "#111111", + highlightTextColor: "#ffd54a", + highlightPaddingX: 6, + highlightPaddingY: 2, + highlightRadius: 4, + highlightAnimation: "pop", + maxCharsPerEntry: 35, + autoResegment: true, + }; +} + +export type TimelineTrackKind = "video" | "audio" | "subtitle" | "text" | "image"; + +export type TimelineTrack = { + id: string; + kind: TimelineTrackKind; + label: string; + order: number; + locked?: boolean; +}; + +export type TimelineClipKind = "video" | "audio" | "subtitle" | "text" | "image"; + +/** 仅时间线作用域的直连媒体（不落画布）：素材库/项目资产/本地上传在时间线内直接入轨时使用。 */ +export type TimelineDirectMedia = { + /** 源素材 id（本地资产 id / 上传文件 id），用于生成时间线内唯一 nodeId 前缀 */ + id: string; + kind: "video" | "audio" | "image" | "text"; + title: string; + storageKey?: string; + url?: string; + dataUrl?: string; + content?: string; + durationMs?: number; + width?: number; + height?: number; + bytes?: number; + mimeType?: string; +}; + +export type TimelineClip = { + id: string; + kind: TimelineClipKind; + nodeId: string; + trackId: string; + startMs: number; + durationMs: number; + title?: string; + /** 源素材内裁剪起点（毫秒），用于导出阶段定位源文件片段 */ + sourceStartMs?: number; + /** 源素材总时长（毫秒），用于 UI 限制裁剪上限 */ + sourceDurationMs?: number; + volume?: number; + fadeInMs?: number; + fadeOutMs?: number; + /** 字幕片段：对应源视频节点 subtitleEntries 的下标（-1 表示未关联） */ + subtitleEntryIndex?: number; + /** 字幕片段文本快照，时间线内直接展示，不与节点双向同步 */ + text?: string; + /** 仅时间线作用域的直连媒体；存在时预览/导出优先从该字段解析，不再回画布查节点 */ + directMedia?: TimelineDirectMedia; +}; + +export type TimelineProject = { + version: 2; + tracks: TimelineTrack[]; + clips: TimelineClip[]; + /** 时间线总时长（毫秒），取所有片段末端最大值 */ + durationMs: number; + updatedAt?: string; +};