From b20ea83201897b2657cf2a5de13fabdfa07a45db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=88=B1=E7=AC=91=E7=9A=84=E6=AF=9B=E6=AF=9B=E8=99=AB?= Date: Sat, 5 Sep 2026 02:52:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(editor):=20=E5=89=AA=E8=BE=91=E6=88=90?= =?UTF-8?q?=E7=89=87=E9=AA=8C=E6=94=B6=E9=97=AD=E7=8E=AF=20-=20=E6=A6=82?= =?UTF-8?q?=E8=A7=88=E6=B8=B2=E6=9F=93=E8=AE=A1=E6=95=B0=E3=80=81=E5=AF=BC?= =?UTF-8?q?=E5=85=A5/=E5=AD=97=E5=B9=95/=E9=A2=84=E8=A7=88=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E4=B8=8E=E6=9C=AC=E5=9C=B0=20whisper=20=E8=BD=AC?= =?UTF-8?q?=E5=86=99=E8=BF=90=E8=A1=8C=E6=97=B6=20(#414)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(editor): 剪辑成片验收闭环 - 概览渲染成功计数、媒体导入/字幕/预览修复与本地 whisper 转写运行时 - 概览:渲染成功计数接入项目概览 SQL/JSON,成片状态随 timeline_render 任务联动 - 修复:本地媒体导入后列表为空(挂载时服务端重同步 + 导入后校验重取)、 字幕工具残留 M6 占位文案、预览监视器悬浮遮挡层改为顶部信息条、 生成任务轮询连续查询失败即抛错不再空转至超时 - 本地转写:新增 scripts/start-whisper-local.sh 一键启动 whisper.cpp, local-development 文档补安装/启动/后端注入指引 - 同步 docs(overview/features、pending-test、index) * fix(assets): 素材库封面未知 kind 兜底,避免 AssetCover 渲染崩溃 运行时素材记录若带未知/脏 kind(历史遗留或本地存储异常),assetKindIcons[asset.kind] 得到 undefined,AssetCover/AssetDrawer 渲染 触发 'Element type is invalid ... got: undefined' 使 /assets 整页崩溃。 新增 isKnownAssetKind 守卫:未知 kind 回退 FileText 图标与通用文案,脏数据不再崩页。 * fix(editor): mp4v 播放死循环修复——MPEG-4 Part 2 归入播放转码通道，监视器加终态护栏 - 后端把 mp4v 与 H.264/AV1/VP9 一并判为浏览器可播 → playbackStatus=none； Chromium/Firefox/Edge 实际均不支持 MPEG-4 Part 2，原件解码即失败。 现将其并入 H.265 转码通道(转 H.264 兼容副本)；损坏文件转码失败落 failed，前端据此显示终止性错误而非死循环。 - 前端监视器轮询到 none/failed 或超时后不再回退 primary 重试；副本 URL 回退原件反复触发 onError 时保持终态文案，不再覆盖成"正在生成兼容版本"。 * feat(editor): 画布产物自动同步剪辑素材库，素材面板新增「画布素材」分组 - 打开剪辑器时自动把画布工作台当前采用(storyboard/action_board/分镜/导出成片， ready 且有资源)的产物 linkProjectAsset(source=canvas) 并入项目素材库； 后端幂等、前端按 storageKey+会话 ref 去重，失败静默下次进入自动补同步。 - LinkProjectAsset 支持 source 白名单(uploaded/canvas)并写入素材 payload， 供素材面板按来源分组；面板新增「画布素材」页签与分组，与「本地上传」「项目素材」并列。 - 补 source=canvas 写入断言测试。 * docs(progress): 记录 mp4v 播放死循环修复与画布产物素材同步的待浏览器验收项 --- .../repository/project_workbench_read.go | 30 ++-- backend/internal/service/project_asset.go | 18 ++- .../service/project_asset_link_test.go | 16 +++ .../service/project_workbench_read.go | 3 +- .../service/project_workbench_read_test.go | 27 ++++ backend/internal/service/video_transcode.go | 12 +- .../docs/backend/local-development.mdx | 49 +++++++ docs/content/docs/overview/features.mdx | 36 +++++ docs/content/docs/progress/pending-test.mdx | 14 +- docs/index.md | 4 +- scripts/start-whisper-local.sh | 37 +++++ web/src/components/editor/editor-context.tsx | 2 +- .../builtin/editor/editor-asset-ingest.tsx | 32 +++-- .../builtin/editor/editor-preview-monitor.tsx | 131 ++++++++++++++---- .../builtin/editor/editor-subtitle-tools.tsx | 11 +- web/src/pages/assets/index.tsx | 12 +- web/src/pages/projects/detail/editor.tsx | 78 ++++++++++- web/src/pages/projects/detail/overview.tsx | 2 +- web/src/services/api/projects.ts | 3 +- web/src/services/api/task-center.ts | 8 ++ 20 files changed, 451 insertions(+), 74 deletions(-) create mode 100755 scripts/start-whisper-local.sh diff --git a/backend/internal/repository/project_workbench_read.go b/backend/internal/repository/project_workbench_read.go index 3fbb55b9d..f050984a9 100644 --- a/backend/internal/repository/project_workbench_read.go +++ b/backend/internal/repository/project_workbench_read.go @@ -7,19 +7,20 @@ import ( ) type ProjectOverviewMetrics struct { - UnitCount int64 `gorm:"column:unit_count"` - CompletedUnitCount int64 `gorm:"column:completed_unit_count"` - TotalWordCount int64 `gorm:"column:total_word_count"` - UnitsWithoutText int64 `gorm:"column:units_without_text"` - UnitsWithoutShots int64 `gorm:"column:units_without_shots"` - CanvasCount int64 `gorm:"column:canvas_count"` - AssetCount int64 `gorm:"column:asset_count"` - ShotCount int64 `gorm:"column:shot_count"` - PendingCandidateCount int64 `gorm:"column:pending_candidate_count"` - ReadyStoryboardCount int64 `gorm:"column:ready_storyboard_count"` - ReadyPrevizCount int64 `gorm:"column:ready_previz_count"` - ReadyVideoCount int64 `gorm:"column:ready_video_count"` - StaleArtifactCount int64 `gorm:"column:stale_artifact_count"` + UnitCount int64 `gorm:"column:unit_count"` + CompletedUnitCount int64 `gorm:"column:completed_unit_count"` + TotalWordCount int64 `gorm:"column:total_word_count"` + UnitsWithoutText int64 `gorm:"column:units_without_text"` + UnitsWithoutShots int64 `gorm:"column:units_without_shots"` + CanvasCount int64 `gorm:"column:canvas_count"` + AssetCount int64 `gorm:"column:asset_count"` + ShotCount int64 `gorm:"column:shot_count"` + PendingCandidateCount int64 `gorm:"column:pending_candidate_count"` + ReadyStoryboardCount int64 `gorm:"column:ready_storyboard_count"` + ReadyPrevizCount int64 `gorm:"column:ready_previz_count"` + ReadyVideoCount int64 `gorm:"column:ready_video_count"` + TimelineRenderSucceededCount int64 `gorm:"column:timeline_render_succeeded_count"` + StaleArtifactCount int64 `gorm:"column:stale_artifact_count"` } type ProjectOverviewUnitRow struct { @@ -62,8 +63,9 @@ func (r *Repository) ProjectOverviewMetrics(projectID string) (ProjectOverviewMe (SELECT COUNT(DISTINCT shot_id) FROM shot_artifacts WHERE project_id = ? AND type = 'storyboard' AND selected = ? AND status = 'ready') AS ready_storyboard_count, (SELECT COUNT(DISTINCT shot_id) FROM shot_artifacts WHERE project_id = ? AND type = 'action_board' AND selected = ? AND status = 'ready') AS ready_previz_count, (SELECT COUNT(DISTINCT shot_id) FROM shot_artifacts WHERE project_id = ? AND type = 'video' AND selected = ? AND status = 'ready') AS ready_video_count, + (SELECT COUNT(*) FROM tasks WHERE project_id = ? AND type = ? AND status = ?) AS timeline_render_succeeded_count, (SELECT COUNT(*) FROM shot_artifacts WHERE project_id = ? AND status = 'stale') AS stale_artifact_count - `, projectID, projectID, model.ProjectUnitStatusCompleted, projectID, projectID, projectID, model.ProjectUnitStatusDraft, projectID, projectID, projectID, projectID, projectID, true, projectID, true, projectID, true, projectID).Scan(&metrics).Error +`, projectID, projectID, model.ProjectUnitStatusCompleted, projectID, projectID, projectID, model.ProjectUnitStatusDraft, projectID, projectID, projectID, projectID, projectID, true, projectID, true, projectID, true, projectID, model.TaskTypeTimelineRender, model.TaskStatusSucceeded, projectID).Scan(&metrics).Error return metrics, err } diff --git a/backend/internal/service/project_asset.go b/backend/internal/service/project_asset.go index 267239fdc..01dafe5dc 100644 --- a/backend/internal/service/project_asset.go +++ b/backend/internal/service/project_asset.go @@ -11,12 +11,20 @@ import ( "infinite-canvas/backend/internal/model" ) +// 资产来源：媒体导入合成/画布产物自动同步。画布产物用 canvas 以便素材面板分组。 +const ( + AssetSourceUploaded = "uploaded" + AssetSourceCanvas = "canvas" +) + type LinkProjectAssetRequest struct { AssetID string `json:"assetId"` Category string `json:"category"` FolderID *string `json:"folderId"` // Title 媒体导入场景下由前端携带原始文件名；已有资产时忽略。 Title string `json:"title"` + // Source 仅媒体导入合成时生效：uploaded（默认）或 canvas（画布产物自动同步）。 + Source string `json:"source"` } type UpdateProjectAssetRequest struct { @@ -105,6 +113,10 @@ func (s *Service) LinkProjectAsset(userID string, projectID string, req LinkProj return ProjectAssetSummary{}, err } assetID := strings.TrimSpace(req.AssetID) + source := strings.TrimSpace(req.Source) + if source != AssetSourceCanvas { + source = AssetSourceUploaded + } asset, err := s.repo.AssetForUser(userID, assetID) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return ProjectAssetSummary{}, err @@ -112,7 +124,7 @@ func (s *Service) LinkProjectAsset(userID string, projectID string, req LinkProj if asset == nil { // 媒体导入只落 resources 表；首次链接时按资源元数据合成资产记录， // 避免“资源存在但无资产记录”导致导入永远失败。 - asset, err = s.assetFromUploadedResource(userID, assetID, req.Title) + asset, err = s.assetFromUploadedResource(userID, assetID, req.Title, source) if err != nil { return ProjectAssetSummary{}, err } @@ -173,7 +185,7 @@ func (s *Service) LinkProjectAsset(userID string, projectID string, req LinkProj // assetFromUploadedResource 把媒体导入上传的 Resource 合成为资产记录（内存构造，不落库）。 // 资产创建与项目链接由 LinkProjectAsset 事务原子提交，避免失败留下孤立资产。 -func (s *Service) assetFromUploadedResource(userID string, resourceID string, title string) (*model.Asset, error) { +func (s *Service) assetFromUploadedResource(userID string, resourceID string, title string, source string) (*model.Asset, error) { resource, err := s.repo.ResourceForUser(userID, resourceID) if err != nil { return nil, err @@ -192,7 +204,7 @@ func (s *Service) assetFromUploadedResource(userID string, resourceID string, ti "durationMs": resource.DurationMs, "status": resource.Status, "mediaType": resource.Kind, - "source": "uploaded", + "source": source, }, }) if err != nil { diff --git a/backend/internal/service/project_asset_link_test.go b/backend/internal/service/project_asset_link_test.go index 736f03f10..bae86d985 100644 --- a/backend/internal/service/project_asset_link_test.go +++ b/backend/internal/service/project_asset_link_test.go @@ -1,6 +1,7 @@ package service import ( + "encoding/json" "testing" "infinite-canvas/backend/internal/model" @@ -58,6 +59,7 @@ func TestLinkProjectAssetCreatesAssetFromUploadedResource(t *testing.T) { AssetID: "resource-1", Category: "", Title: "开场.mp4", + Source: AssetSourceCanvas, }) if err != nil { t.Fatalf("LinkProjectAsset: %v", err) @@ -76,6 +78,20 @@ func TestLinkProjectAssetCreatesAssetFromUploadedResource(t *testing.T) { if asset.Category != model.AssetCategoryMaterial { t.Errorf("asset.Category = %q, want material", asset.Category) } + if summary.Source != AssetSourceCanvas { + t.Errorf("summary.Source = %q, want canvas", summary.Source) + } + var payload struct { + Data struct { + Source string `json:"source"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(asset.PayloadJSON), &payload); err != nil { + t.Fatalf("payload unmarshal: %v", err) + } + if payload.Data.Source != AssetSourceCanvas { + t.Errorf("payload.source = %q, want canvas", payload.Data.Source) + } } // TestLinkProjectAssetMissingResourceKeepsFailingClosed 资源与资产都不存在时仍应报错， diff --git a/backend/internal/service/project_workbench_read.go b/backend/internal/service/project_workbench_read.go index a2c5537d2..56e093b7f 100644 --- a/backend/internal/service/project_workbench_read.go +++ b/backend/internal/service/project_workbench_read.go @@ -33,6 +33,7 @@ type ProjectOverviewMetrics struct { ReadyStoryboardCount int64 `json:"readyStoryboardCount"` ReadyPrevizCount int64 `json:"readyPrevizCount"` ReadyVideoCount int64 `json:"readyVideoCount"` + RenderSucceededCount int64 `json:"renderSucceededCount"` StaleArtifactCount int64 `json:"staleArtifactCount"` } @@ -152,7 +153,7 @@ func (s *Service) ProjectOverview(userID string, projectID string) (ProjectOverv UnitsWithoutText: row.UnitsWithoutText, UnitsWithoutShots: row.UnitsWithoutShots, CanvasCount: row.CanvasCount, AssetCount: row.AssetCount, ShotCount: row.ShotCount, PendingCandidateCount: row.PendingCandidateCount, ReadyStoryboardCount: row.ReadyStoryboardCount, ReadyPrevizCount: row.ReadyPrevizCount, ReadyVideoCount: row.ReadyVideoCount, - StaleArtifactCount: row.StaleArtifactCount, + RenderSucceededCount: row.TimelineRenderSucceededCount, StaleArtifactCount: row.StaleArtifactCount, } return nil }) diff --git a/backend/internal/service/project_workbench_read_test.go b/backend/internal/service/project_workbench_read_test.go index 66c210ed7..27acf7408 100644 --- a/backend/internal/service/project_workbench_read_test.go +++ b/backend/internal/service/project_workbench_read_test.go @@ -121,6 +121,33 @@ func TestProjectOverviewUsesAggregatesWithoutLoadingProjectCollections(t *testin } } +func TestProjectOverviewCountsSucceededTimelineRenders(t *testing.T) { + service, db := newProjectWorkbenchReadTestService(t) + project := seedWorkbenchProject(t, db) + for index := 0; index < 2; index++ { + task := model.Task{ID: fmt.Sprintf("render-%02d", index), UserID: "user-1", ProjectID: project.ID, Type: model.TaskTypeTimelineRender, Status: model.TaskStatusSucceeded} + if err := db.Create(&task).Error; err != nil { + t.Fatal(err) + } + } + distractors := []model.Task{ + {ID: "render-failed", UserID: "user-1", ProjectID: project.ID, Type: model.TaskTypeTimelineRender, Status: model.TaskStatusFailed}, + {ID: "transcribe-ok", UserID: "user-1", ProjectID: project.ID, Type: model.TaskTypeTimelineTranscription, Status: model.TaskStatusSucceeded}, + } + for _, task := range distractors { + if err := db.Create(&task).Error; err != nil { + t.Fatal(err) + } + } + overview, err := service.ProjectOverview("user-1", project.ID) + if err != nil { + t.Fatal(err) + } + if overview.Metrics.RenderSucceededCount != 2 { + t.Fatalf("render succeeded count = %d, want 2", overview.Metrics.RenderSucceededCount) + } +} + func TestProjectUnitWorkspaceIsolatesShotsAndBoundAssetsByUnit(t *testing.T) { service, db := newProjectWorkbenchReadTestService(t) project := seedWorkbenchProject(t, db) diff --git a/backend/internal/service/video_transcode.go b/backend/internal/service/video_transcode.go index 85e2cabca..989d4f8a9 100644 --- a/backend/internal/service/video_transcode.go +++ b/backend/internal/service/video_transcode.go @@ -180,7 +180,10 @@ func (s *Service) maybeStartPlaybackTranscode(resource *model.Resource) { } src := filepath.Join(s.dataDir, "resources", filepath.FromSlash(resource.ObjectKey)) switch probeVideoCodec(src) { - case videoCodecH265: + case videoCodecH265, videoCodecMPEG4: + // H.265 与 MPEG-4 Part 2（mp4v）Chromium/Firefox/Edge 均不能解码： + // H.265 需转 H.264；MPEG-4 Part 2 浏览器不支持，老式/损坏文件只能转码兜底 + // （转码失败会落 failed，前端据此显示终止性错误，不再出现死循环）。 // 原子抢占（空/none → processing）：并发上传 + 回填、多实例同抢时 // 仅一个能成功置位，其余直接返回，避免重复转码。 claimed, err := s.repo.ClaimPlaybackTranscode(resource.ID) @@ -189,9 +192,10 @@ func (s *Service) maybeStartPlaybackTranscode(resource *model.Resource) { } resource.PlaybackStatus = model.PlaybackStatusProcessing go s.runPlaybackTranscode(resource.UserID, resource.ID, src) - case videoCodecH264, videoCodecAV1, videoCodecVP9, videoCodecMPEG4, "": - // H.264 浏览器可直接解码；AV1/VP9/MPEG4 暂不转码；探针读不出编码（非 mp4 / - // moov 在尾部 / 加密容器）也无法处理 —— 均标 none，避免重复探测与前端无限轮询。 + case videoCodecH264, videoCodecAV1, videoCodecVP9, "": + // H.264 浏览器可直接解码；AV1/VP9 现代浏览器可直接解码，均不需转码； + // 探针读不出编码（非 mp4 / moov 在尾部 / 加密容器）也无法处理 —— 均标 none， + // 避免重复探测与前端无限轮询。 markPlaybackNone(s, resource) } } diff --git a/docs/content/docs/backend/local-development.mdx b/docs/content/docs/backend/local-development.mdx index 79f31d98c..163b45f10 100644 --- a/docs/content/docs/backend/local-development.mdx +++ b/docs/content/docs/backend/local-development.mdx @@ -56,6 +56,55 @@ bun run dev 前端默认监听 `0.0.0.0:3000`，Vite 将 `/api` 和 OAuth 回调代理到 `127.0.0.1:8080`。如需修改代理目标，可设置 `VITE_API_PROXY_TARGET`。 +## 时间线字幕转写（whisper.cpp） + +时间线“字幕转写”由**本地 whisper.cpp server** 执行：后端将片段音频预处理为 16k mono wav 后，POST 到 `CANVAS_WHISPER_BASE_URL/inference`（whisper server 原生接口，`response_format=verbose_json`）。未配置该地址时，转写任务会立即明确失败，提示“撰写失败未配置本地转写服务”。 + +### 安装 whisper-server + +```bash +brew install whisper-cpp +``` + +### 下载模型 + +仓库脚本将模型放在 `.local/whisper-models/`（已被 `.gitignore` 忽略）。多语言模型（含中文）用 `ggml-base.bin`（约 141MB）；网络受限时可用 aifasthub 镜像： + +```bash +mkdir -p .local/whisper-models +curl -fL -o .local/whisper-models/ggml-base.bin \ + https://aifasthub.com/ggerganov/whisper.cpp/resolve/main/ggml-base.bin +``` + +### 启动服务 + +```bash +scripts/start-whisper-local.sh +``` + +脚本默认以 `ggml-base.bin` 监听 `127.0.0.1:8082`（`-l auto` 自动识别语言）。需要换模型或端口时：`WHISPER_MODEL=ggml-small.bin WHISPER_PORT=8083 scripts/start-whisper-local.sh`。 + +### 让后端指向该服务 + +后端不读取 `.env` 文件，环境变量必须由启动 shell 注入（在手工启动命令基础上追加）： + +```bash +cd backend +CANVAS_BACKEND_DATA_DIR=../.local/project-workbench-debug \ +CANVAS_WHISPER_BASE_URL=http://127.0.0.1:8082 \ +go run ./cmd/server +``` + +### 验证 + +```bash +# 服务就绪：返回 200/404 即可 +curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8082/ +# 冒烟：20 秒语音 → verbose_json 字幕段 +curl -s -X POST http://127.0.0.1:8082/inference \ + -F 'file=@sample.wav' -F 'response_format=verbose_json' +``` + ## Docker 开发 需要容器化热更新时，在仓库根目录执行： diff --git a/docs/content/docs/overview/features.mdx b/docs/content/docs/overview/features.mdx index 5ff7cbbfe..279f39432 100644 --- a/docs/content/docs/overview/features.mdx +++ b/docs/content/docs/overview/features.mdx @@ -159,3 +159,39 @@ SEO 标题、描述、可选关键词会投影到浏览器 metadata、Open Graph - `web/src/components/canvas/canvas-zoom-controls.tsx`：底部视图控制 Dock 与“自动整理节点”入口。 - `web/src/pages/canvas/use-canvas-node-operations.ts`：候选节点筛选、流程/网格布局选择和位置提交。 - `web/src/lib/canvas/canvas-layout.ts`：媒体泳道、拓扑流程、网格、行列排列和对齐算法。 + +## 剪辑成片（时间线编辑器） + +“剪辑成片”是项目详情里的时间线编辑器入口（`/projects/:id/editor`），用于把已就绪的分镜视频编排成一段可播放、可导出的成片。它运行在浏览器本地时间线模型之上，通过后端任务队列完成语音转写与成片导出，编辑器本身不直接触碰原始媒体大文件。 + +### 使用入口 + +- 项目详情“制作流程 → 剪辑成片”步骤卡，或地址栏直接访问 `/projects/:id/editor`。 +- 编辑器按项目隔离时间线状态；离开后再次进入会恢复该项目的编辑器快照。 + +### 当前能力 + +- **轨道与命令**：时间线支持添加、删除、拆分、移动、修剪片段等操作，全部通过统一命令状态机执行并进入 200 层历史记录，可撤销与重做；非法操作整批拒绝，不产生半应用状态。 +- **预设面板**：左侧“添加”分类提供 8 个内置片段/字幕插件，按分类分组接入同一插槽注册链，第三方插件可与内置面板共存。 +- **权限与执行边界**：命令执行遵循失败关闭（fail-closed）策略，`timeline.command` 权限与全局 `plugins.run` 权限分开控制；未授权插件或命令不会被执行或回显。 +- **语音转写（字幕）**：对时间线音视频发起转写任务，转写进度沿任务 SSE 回流，完成后生成字幕轨。 +- **成片导出（渲染）**：发起渲染任务，由服务端 ffmpeg 按渲染计划输出成片；渲染成功后在项目概览的“剪辑成片”步骤卡呈现“已输出 N 个成片”，并作为该步骤完成度的判定依据。 +- **AI 编辑助手**：可以用自然语言描述剪辑意图，系统将其解析为受支持的命令序列；不超过 3 条命令时直接执行，多于 3 条先给出差异预览，确认后才批量执行。 + +### 运行环境要求 + +- 语音转写依赖本地 whisper.cpp 服务（`CANVAS_WHISPER_BASE_URL`），未配置时转写任务明确失败。 +- 成片导出依赖服务端 ffmpeg（`CANVAS_FFMPEG_PATH`，留空时在 PATH 中查找）。 +- AI 编辑助手需要可用的文本模型渠道。 + +### 相关实现 + +- `web/src/pages/projects/detail/editor.tsx`、`web/src/pages/projects/detail.tsx`：编辑器页面与路由入口。 +- `web/src/pages/projects/detail/overview.tsx`：制作流程步骤卡与成片完成度展示。 +- `web/src/lib/timeline/`：时间线命令状态机、历史记录、命令 schema 与摘要（`editor-commands.ts`、`editor-history.ts`、`ai-command-schema.ts`、`timeline-summary.ts`）。 +- `web/src/stores/editor/editor-store.ts`：编辑器状态与快照持久化。 +- `web/src/lib/plugins/editor-slot-registry.ts`、`web/src/lib/plugins/builtin/editor/`：插槽注册链与内置预设插件。 +- `web/src/lib/plugins/plugin-permission-check.ts`：命令执行权限与失败关闭策略。 +- `web/src/services/api/timeline-tasks.ts`：转写与渲染任务客户端。 +- `backend/internal/service/task_timeline.go`、`task_render.go`、`timeline_render_plan.go`、`transcription_whisper.go`：后端转写、渲染与渲染计划任务。 +- `backend/internal/repository/project_workbench_read.go`：项目概览指标聚合（含成片完成度计数）。 \ No newline at end of file diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 8dd029461..a0f910f9c 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -43,7 +43,8 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 - [x] M4.1 创建 API（bf26ebb1）：`POST /api/timeline/transcriptions`（注册于 `RegisterTaskRoutes`）→ `CreateTimelineTranscriptionTask`：draining 503 → feature 门控 → 资源归属/可转写类型校验 → 配额入队（local/whisper.cpp，无 billing order）；活跃任务上限 → 可读 400；测试 3 项 PASS。 - [x] M4.2 渲染任务（3f16d13b）：`POST /api/timeline/renders` → `CreateTimelineRenderTask`：`buildRenderPlan` 校验 ≥1 可渲染媒体 → 配额入队（local/ffmpeg）；worker 分叉 `processTimelineRender`（60 分钟超时）：ffmpeg 合成 → 产物写资源存储 → `ResultJSON{resourceId,...}`；计划/失败快路径测试 PASS。 - [x] M4.3 前端字幕回写（fd5e8139）：`timeline-tasks.ts` 客户端 + `editor-transcription.tsx` 真实任务替换 mock；资源范围放宽到音视频；`resourceId` 由 `storageKey` 推导；`waitForGenerationTask` 显式 25 分钟超时轮询；结果 segments → SrtEntry → `rebuildSubtitleClips` 落字幕轨；typecheck 通过。 -- [ ] 浏览器端到端（待用户确认，需本机 whisper.cpp server）：媒体面板选音视频素材 → 转写 → 进度可见 → 完成后字幕分段入轨可编辑；错误路径（未配置/删除素材/非音视频/活跃任务上限/功能未开放）提示可读。 +- [x] M4.1 运行时端到端（2026-09-04，真实 whisper.cpp）：`brew install whisper-cpp` + `ggml-base.bin`，whisper-server `127.0.0.1:8082`（`-l auto`）；后端注入 `CANVAS_WHISPER_BASE_URL=http://127.0.0.1:8082` 重启；真实 mp4 资源经 worker 队列（status=queued → running → succeeded 100%）跑通：ffmpeg 16k mono 提取 → `/inference` verbose_json → 中文分段含 start/end/text 入库，与 `transcription_whisper.go` 解码协议一致。失败路径（未配置）此前 5 条任务均按设计报“撰写失败未配置本地转写服务”。启动步骤已写入 `local-development.mdx`，脚本 `scripts/start-whisper-local.sh`。 +- [ ] 浏览器端到端（待用户确认；本机 whisper-server 已就绪，直接重试转写即可）：媒体面板选音视频素材 → 转写 → 进度可见 → 完成后字幕分段入轨可编辑；错误路径（未配置/删除素材/非音视频/活跃任务上限/功能未开放）提示可读。 - [x] M4.2 渲染前端接线：`timeline-tasks.ts` 增 `createTimelineRenderTask`（timeline 整传 TimelineProject v2，与后端 renderProject 契约同构）；`editor-export.tsx` 主路径改提交 `POST /api/timeline/renders` → `waitForGenerationTask` 轮询（62min）→ ResultJSON.resourceId 内嵌预览/下载；ffmpeg.wasm 降级保留；tsc 全绿、无新增测试失败。 - [ ] M4.2 渲染浏览器验证（待用户确认，需 ffmpeg 可用）：导出面板「渲染成片（服务端）」→ 轮询进度 → 完成后内嵌预览/下载渲染产物；无后端时「本地导出」ffmpeg.wasm 仍可用。 - [ ] 文档回写（已提交）：`docs/plans/m4-transcription-render-design.md` §2 决策 3/4/5 按“单一池 + 类型分叉”与旁路终态修正；端点统一为复数 `/api/timeline/transcriptions` 与 `/api/timeline/renders`（注册于 `RegisterTaskRoutes`）；`auth.go` 前缀白名单补 `timeline`。 @@ -62,6 +63,8 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 - [x] 编辑器 UI 打磨（第 36 条 5 项）：① 时间线/监视器分隔条拖拽方向反转（上拖=时间线变大）；② 导出按钮图标去蓝（灰白描边风格），媒体面板来源徽章去蓝，轨道类型徽章保留语义色；③ 时间线 pxPerMs 独立化（不再随时长漂移），末片段向右拖轨道随内容等比扩展不再被钉住（新增守护测试 3 例）；④ 监视器控制条重做为 Concat 风格：回到开头/退 1s/播放/进 1s/到结尾五键传输组 + 播放速度展开菜单（0.5x/1x/1.5x/2x）+ 进度条，全部灰白描边；⑤ 检查器重做为 Concat 风格：分组小标题（片段/基本/转场）+ 紧凑控件 + sunken 滑块（新增 .editor-slider），全部使用 director token 随主题联动。 - [ ] 用户浏览器 re-test（第 36 条）：上拖分隔条时间线变大；导出按钮与监视器控制条图标均为灰白描边风格；时间线末片段可正常向右拖拽且轨道随内容扩展；监视器速度菜单展开选择生效；检查器分组与滑块在明暗两主题下配色一致（无主题不同步）。 - [x] 媒体面板改版：资产按来源分组（「本地上传」/「项目素材」），顶部提供 全部/项目素材/本地上传 筛选 Tab；分组头可折叠、资产行可展开查看详情（类型/分类/来源/预览文本）并一键添加到时间线；每项带来源徽章与计数徽章。后端 `projectAssetSummary` 透出 `source`（上传链路写入 `uploaded`），前端 `ProjectAsset` 类型补 `source?: string`，仅展示层使用，后端读取不再查询本地存储。 +- [x] 概览成片完成度接线：项目概览指标新增 `renderSucceededCount`（repository 聚合 SQL 子查询 `tasks` 表 `type=timeline_render` 且成功的计数），概览「剪辑成片」步骤卡的 complete 态由写死的 false 改为 `renderSucceededCount > 0`，提示文案随计数/就绪情况切换；新增后端专项测试 1 个、前端类型检查通过。 +- [ ] 用户浏览器 re-test（概览完成度）：渲染成功一次后刷新概览，「剪辑成片」卡应显示已输出 N 个成片且步骤完成态点亮；删除该成片对应任务后计数回退。 - [x] 时间线横跨编辑面板全宽：时间线卡从内容区独立出来，与左栏+预览+右栏同处编辑面板，横跨从左到右全宽（含拖拽高度分隔条）；左栏收窄到 256px、右栏 320px，为全宽时间线腾出空间。 - [x] 主题明暗一致性：媒体面板分组/筛选/徽章/展开详情全部使用 director token（surface/control-hover/dock-fg/sequencer-border），无新增硬编码色值；后端 `source` 字段不参与任何配色。前端 typecheck、生产构建、后端全量测试通过。 - [x] 时间线面板补齐 Concat 风格工具栏与编辑交互（对齐第 2 条 runbook 与 `Concat/desktop/.../TimelinePanel.tsx`）：新增 选择/剃刀 工具切换（剃刀点击分割、非字幕片段、边缘留 100ms）、播放头时码只读显示与标尺拖动 scrub、分割播放头处、删除选中片段、磁吸开关（默认开）、加轨道下拉（视频/音频/字幕，`addTrack` 黄金命令确定性生成 id/order/label，undo/redo 可重放）、适应宽度（`getFitTimelineZoom`）；工具栏布局重排为 Concat 顶部条结构。`editor-commands.ts` golden ops 由 9 增到 10（`addTrack`）。 @@ -859,3 +862,12 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 - [ ] 拖入时长 >5s 的视频片段，其时间线长度等于视频真实时长（不再固定 5s 截断）；时长探测上传的新素材在资源卡片/片段均显示真实时长。 - [ ] 播放副本就绪前（processing）视频黑屏时出现"正在生成兼容版本（H.264）…"浮层 + "下载原片用本地播放器观看"链接；生成失败显示失败文案而非一直转圈。 - [ ] 时间线 scrub/播放/变速往返 10 次以上无卡死、无播放头回跳、无 setState-after-unmount 报错（Console 无告警）。 + +## MPEG-4 Part 2 播放死循环 + 画布产物素材同步（待浏览器验收） + +已实施，代码/单测/编译验证通过（后端 go vet/test、前端 tsc 全绿），需浏览器最终确认： + +- [ ] 导入 MPEG-4 Part 2(mp4v)视频(如 07ee…mp4 sprite/header damaged 样例)后拖入时间线：不再出现"正在生成兼容版本"提示闪烁死循环；损坏文件转码失败后显示终止性文案(不再反复回退重试)；正常 mp4v 文件自动生成 H.264 副本并可播放。 +- [ ] 打开剪辑器进入已含画布产物(selected+ready 的 storyboard/action_board/分镜/导出成片)的项目：素材面板自动出现"画布素材"分组与对应素材；再次进入/刷新不产生重复素材。 +- [ ] 素材面板"画布素材/本地上传/项目素材"三个页签与分组切换、来源徽章与详情来源文本正确。 +- [ ] 项目素材库页与剪辑器素材面板对本轮新同步的画布素材均可见；下载/删除画布素材不影响对应画布产物资源。 diff --git a/docs/index.md b/docs/index.md index bb627831c..e1d3664ba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,6 +45,6 @@ - [功能](content/docs/overview/features.mdx) - [本地开发](content/docs/backend/local-development.mdx) - [后端数据库](content/docs/backend/backend-database.mdx) -- 代码地图：待补充 -- 待办：待补充 +- [代码地图](content/docs/backend/code-map.mdx) +- [待办](content/docs/progress/todo.mdx) - [待测试](content/docs/progress/pending-test.mdx) diff --git a/scripts/start-whisper-local.sh b/scripts/start-whisper-local.sh new file mode 100755 index 000000000..fc15221d0 --- /dev/null +++ b/scripts/start-whisper-local.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Local whisper.cpp server for timeline subtitle transcription. +# The Go backend calls {CANVAS_WHISPER_BASE_URL}/inference with a multipart +# wav upload and expects standard whisper verbose_json back (native protocol). +# +# Usage: +# scripts/start-whisper-local.sh # base model, port 8082 +# WHISPER_MODEL=ggml-small.bin scripts/start-whisper-local.sh +set -euo pipefail + +cd "$(dirname "$0")/.." +ROOT="$(pwd)" + +MODEL_NAME="${WHISPER_MODEL:-ggml-base.bin}" +MODEL_DIR="${ROOT}/.local/whisper-models" +MODEL_PATH="${MODEL_DIR}/${MODEL_NAME}" +HOST="${WHISPER_HOST:-127.0.0.1}" +PORT="${WHISPER_PORT:-8082}" +THREADS="${WHISPER_THREADS:-4}" + +if ! command -v whisper-server >/dev/null 2>&1; then + echo "whisper-server not found. Install with: brew install whisper-cpp" >&2 + exit 1 +fi + +mkdir -p "${MODEL_DIR}" + +if [ ! -f "${MODEL_PATH}" ]; then + echo "Model ${MODEL_NAME} not found at ${MODEL_PATH}" >&2 + echo "Download it first, e.g.:" >&2 + echo " curl -fL -o ${MODEL_PATH} https://aifasthub.com/ggerganov/whisper.cpp/resolve/main/${MODEL_NAME}" >&2 + echo "(aifasthub mirrors huggingface.co/ggerganov/whisper.cpp for CN networks)" >&2 + exit 1 +fi + +echo "Starting whisper-server: model=${MODEL_NAME} host=${HOST} port=${PORT}" +exec whisper-server -m "${MODEL_PATH}" --host "${HOST}" --port "${PORT}" --threads "${THREADS}" -l auto diff --git a/web/src/components/editor/editor-context.tsx b/web/src/components/editor/editor-context.tsx index 4179a1ad7..dd18425ae 100644 --- a/web/src/components/editor/editor-context.tsx +++ b/web/src/components/editor/editor-context.tsx @@ -16,7 +16,7 @@ export type EditorHostContextValue = { /** 项目资产列表（M3.4 asset-ingest 消费；由宿主在进入编辑器时获取）。 */ assets: ProjectAsset[]; /** 重新拉取项目资产（导入媒体后刷新列表）。 */ - refreshAssets: () => Promise; + refreshAssets: () => Promise; }; const EditorStoreContext = createContext(null); diff --git a/web/src/lib/plugins/builtin/editor/editor-asset-ingest.tsx b/web/src/lib/plugins/builtin/editor/editor-asset-ingest.tsx index 51bc043c7..d845f415b 100644 --- a/web/src/lib/plugins/builtin/editor/editor-asset-ingest.tsx +++ b/web/src/lib/plugins/builtin/editor/editor-asset-ingest.tsx @@ -34,15 +34,17 @@ const IMAGE_RE = /\.(png|jpe?g|webp|gif|avif)$/i; const AUDIO_RE = /\.(mp3|wav|m4a|ogg|flac|aac)$/i; const MEDIA_RE = /\.(mp4|mov|webm|mkv|m4v|avi)$/i; -/** 本地上传标记（assetFromUploadedResource 写入 payload.data.source）。 */ +/** 本地上传标记与画布产物标记（assetFromUploadedResource 写入 payload.data.source）。 */ const SOURCE_UPLOADED = "uploaded"; +const SOURCE_CANVAS = "canvas"; -type AssetFilter = "all" | "project" | "uploaded"; +type AssetFilter = "all" | "project" | "uploaded" | "canvas"; const FILTER_TABS: { id: AssetFilter; label: string }[] = [ { id: "all", label: "全部" }, { id: "project", label: "项目素材" }, { id: "uploaded", label: "本地上传" }, + { id: "canvas", label: "画布素材" }, ]; /** 按 MIME 与扩展名推断媒体 kind；非媒体文件返回 null。 */ @@ -201,15 +203,16 @@ function MediaPreview({ asset }: { asset: ProjectAsset }) { function SourceBadge({ source }: { source: string }) { const uploaded = source === SOURCE_UPLOADED; + const canvas = source === SOURCE_CANVAS; return ( - {uploaded ? "本地上传" : "项目素材"} + {uploaded ? "本地上传" : canvas ? "画布素材" : "项目素材"} ); } @@ -230,7 +233,7 @@ export function EditorAssetIngest() { const [filter, setFilter] = useState("all"); const [expandedId, setExpandedId] = useState(null); - const [openGroups, setOpenGroups] = useState<{ uploaded: boolean; project: boolean }>({ uploaded: true, project: true }); + const [openGroups, setOpenGroups] = useState<{ uploaded: boolean; project: boolean; canvas: boolean }>({ uploaded: true, project: true, canvas: true }); const inputRef = useRef(null); const lastAddKey = useRef(null); const lastAddAt = useRef(0); @@ -264,6 +267,7 @@ export function EditorAssetIngest() { }); const skipped = media.length - toImport.length; // 逐文件导入：单文件失败不中断整批，汇总失败数提示。 + const linkedIds: string[] = []; for (const { file, kind } of toImport) { try { // 上传前探测真实时长（视频/音频），随 meta 入库供时间线片段使用。 @@ -291,6 +295,7 @@ export function EditorAssetIngest() { continue; } okCount += 1; + linkedIds.push(resource.id); } catch (err) { failedNames.push(file.name); const detail = extractApiMessage(err); @@ -298,7 +303,12 @@ export function EditorAssetIngest() { } } if (okCount > 0) { - await refreshAssets(); + // 刷新后校验本次挂载的素材是否都出现在列表里；若单次刷新因网络抖动 + // 或后端提交延迟而拿不到最新结果，立即再刷新一次，避免列表停留在旧快照。 + const firstList = await refreshAssets(); + if (firstList && linkedIds.some((id) => !firstList.some((asset) => asset.id === id))) { + await refreshAssets(); + } setImportNote(skipped > 0 ? `已导入 ${okCount} 个，跳过 ${skipped} 个重复文件` : `已导入 ${okCount} 个媒体`); } else if (skipped > 0 && failedNames.length === 0) { setImportNote(`媒体库中已有同名素材，跳过 ${skipped} 个重复文件`); @@ -336,9 +346,11 @@ export function EditorAssetIngest() { }; const uploadedAssets = assets.filter((a) => a.source === SOURCE_UPLOADED); - const projectAssets = assets.filter((a) => a.source !== SOURCE_UPLOADED); - const groups: { id: "uploaded" | "project"; label: string; icon: typeof HardDrive; items: ProjectAsset[] }[] = [ + const canvasAssets = assets.filter((a) => a.source === SOURCE_CANVAS); + const projectAssets = assets.filter((a) => a.source !== SOURCE_UPLOADED && a.source !== SOURCE_CANVAS); + const groups: { id: "uploaded" | "project" | "canvas"; label: string; icon: typeof HardDrive; items: ProjectAsset[] }[] = [ { id: "uploaded", label: "本地上传", icon: HardDrive, items: uploadedAssets }, + { id: "canvas", label: "画布素材", icon: Boxes, items: canvasAssets }, { id: "project", label: "项目素材", icon: Boxes, items: projectAssets }, ]; const visibleGroups = groups.filter((g) => filter === "all" || g.id === filter); @@ -416,7 +428,7 @@ export function EditorAssetIngest() { ) : visibleGroups.every((g) => g.items.length === 0) ? (

- {filter === "uploaded" ? "暂无本地上传的媒体" : filter === "project" ? "暂无项目素材" : "项目暂无资产"} + {filter === "uploaded" ? "暂无本地上传的媒体" : filter === "canvas" ? "暂无画布素材，请先在画布工作台生成分镜/预演" : filter === "project" ? "暂无项目素材" : "项目暂无资产"}

点击上方导入媒体，或将文件拖入此区域

@@ -487,7 +499,7 @@ export function EditorAssetIngest() {
分类
{asset.category}
来源
-
{source === SOURCE_UPLOADED ? "本地上传" : "项目素材"}
+
{source === SOURCE_UPLOADED ? "本地上传" : source === SOURCE_CANVAS ? "画布素材" : "项目素材"}
{asset.durationMs ? ( <>
时长
diff --git a/web/src/lib/plugins/builtin/editor/editor-preview-monitor.tsx b/web/src/lib/plugins/builtin/editor/editor-preview-monitor.tsx index 06a7d31c1..0d8443880 100644 --- a/web/src/lib/plugins/builtin/editor/editor-preview-monitor.tsx +++ b/web/src/lib/plugins/builtin/editor/editor-preview-monitor.tsx @@ -11,7 +11,25 @@ // - 静音跟随所在音频/视频轨道（muted），不再硬编码 muted（否则"有声音"永远不成立）。 import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; -import { ChevronDown, Film, Gauge, Pause, Play, SkipBack, SkipForward, StepBack, StepForward } from "lucide-react"; +import { + AlertTriangle, + Captions, + ChevronDown, + Download, + Film, + Gauge, + Image as ImageIcon, + LoaderCircle, + Pause, + Play, + SkipBack, + SkipForward, + StepBack, + StepForward, + Type, + Video, + Volume2, +} from "lucide-react"; import { useEditorStoreContext } from "@/components/editor/editor-context"; import { formatTimelineTime } from "@/lib/timeline/timeline-view"; @@ -67,6 +85,31 @@ function clipSourceTimeMs(clip: TimelineClip, transportMs: number): number { return Math.max(0, transportMs - clip.startMs + (clip.sourceStartMs ?? 0)); } +function clipKindLabel(kind: TimelineClip["kind"]): string { + switch (kind) { + case "video": + return "视频"; + case "image": + return "图片"; + case "audio": + return "音频"; + case "subtitle": + return "字幕"; + case "text": + return "文字"; + } +} + +function clipDisplayTitle(clip: TimelineClip): string { + return clip.directMedia?.title || clip.title || clip.text || clip.nodeId || clip.id; +} + +function ClipKindIcon({ kind, className }: { kind: TimelineClip["kind"]; className?: string }) { + const Icon = + kind === "video" ? Video : kind === "image" ? ImageIcon : kind === "audio" ? Volume2 : kind === "subtitle" ? Captions : Type; + return ; +} + export function EditorPreviewMonitor() { const { project, transportMs, setTransportMs } = useEditorStoreContext(); const durationMs = project?.durationMs ?? 0; @@ -95,11 +138,15 @@ export function EditorPreviewMonitor() { const [mediaTier, setMediaTier] = useState<"primary" | "variant">("primary"); const [variantReadyTick, setVariantReadyTick] = useState(0); const [mediaErrorHint, setMediaErrorHint] = useState(null); + // 终态护栏：none/failed/超时/不可达后禁止再回退 primary 重试或覆盖终态文案。 + // 否则原件不可解码且后端无副本可生成时会 primary→variant→primary 无限循环。 + const mediaTerminalRef = useRef(false); useEffect(() => { setMediaTier("primary"); setVariantReadyTick(0); setMediaErrorHint(null); + mediaTerminalRef.current = false; }, [activeClip?.id, storageKey, activeMediaUrl]); useEffect(() => { @@ -116,16 +163,19 @@ export function EditorPreviewMonitor() { setVariantReadyTick((n) => n + 1); window.clearInterval(timer); } else if (res.playbackStatus === "failed") { + mediaTerminalRef.current = true; setMediaErrorHint("兼容副本生成失败，可下载原片后用本地播放器观看。"); window.clearInterval(timer); } else if (res.playbackStatus === "none") { - // 后端判定无需转码（远端存储/无 ffmpeg/编码不可处理）：无副本可等， - // 停轮询回到原件原生播放，避免"正在生成"永挂。 - setMediaTier("primary"); - setMediaErrorHint(null); + // 后端判定无播放副本可生成（远端存储/无 ffmpeg/编码不可处理）。 + // 原件此刻必然已 onError 失败才进入 variant 轮询，回退 primary 只会 + // 再次失败并切回 variant，形成无限循环 —— 直接进入终态提示并停轮询。 + mediaTerminalRef.current = true; + setMediaErrorHint("视频编码此浏览器暂不支持，且无可生成的兼容副本；可下载原片转换格式后重新导入。"); window.clearInterval(timer); } else if ((polls += 1) >= 120) { // processing 上限保护（约 5 分钟）：转码异常卡死时不再无限轮询。 + mediaTerminalRef.current = true; setMediaErrorHint("兼容版本生成超时，可下载原片后用本地播放器观看。"); window.clearInterval(timer); } else { @@ -136,6 +186,7 @@ export function EditorPreviewMonitor() { if (cancelled) return; failures += 1; if (failures >= 4) { + mediaTerminalRef.current = true; window.clearInterval(timer); setMediaErrorHint("兼容版本生成服务暂不可达，请稍后重新打开预览重试。"); } @@ -148,9 +199,12 @@ export function EditorPreviewMonitor() { }, [mediaTier, mediaResourceId]); const handleMediaError = () => { + // 已进入终态（副本生成失败/无副本可生成/超时/服务不可达）：保持终态文案， + // 不再改写为"正在生成"或回退重试 —— 副本 URL 在 failed/none 时会回退原件， + // 反复 onError 会把失败提示覆盖成误导性的"正在生成兼容版本"。 + if (mediaTerminalRef.current) return; // 图片/无资源：无播放副本可切，仅提示。 if (activeClip?.kind !== "video" || !mediaResourceId) { - setMediaErrorHint("该媒体无法解码，可下载原片后用本地播放器观看。"); return; } if (mediaTier === "primary") { @@ -351,6 +405,52 @@ export function EditorPreviewMonitor() { return (
+ {/* 顶部信息条：媒体标题/元数据常驻在画面外的标题栏，不再悬浮遮挡画面内容 */} +
+
+ {activeClip ? ( + <> + + + {clipDisplayTitle(activeClip)} + + + {clipKindLabel(activeClip.kind)} + + {activeTrack && ( + + {activeTrack.label} + + )} + + ) : ( + 预览监视器 + )} +
+ {mediaErrorHint && ( + + {mediaErrorHint.includes("正在生成") ? ( + + ) : ( + + )} + {mediaErrorHint} + {mediaResourceId && ( + + + + )} + + )} +
{/* 预览画面：恒黑舞台，占满控制条以上剩余空间（Concat 监视器风格） */}
{showEmpty ? ( @@ -388,25 +488,6 @@ export function EditorPreviewMonitor() { onError={handleMediaError} /> ) : null} - {mediaErrorHint && ( -
-

{mediaErrorHint}

- {mediaResourceId && ( - - 下载原片用本地播放器观看 - - )} -
- )} - {activeClip && ( -
- {activeClip.nodeId || activeClip.id} · {activeClip.kind} -
- )}
{/* 播放控制条：画面下方的独立一行（Concat 布局），不悬浮叠加、无进度条； diff --git a/web/src/lib/plugins/builtin/editor/editor-subtitle-tools.tsx b/web/src/lib/plugins/builtin/editor/editor-subtitle-tools.tsx index 0d6df4c85..ecc3db4b4 100644 --- a/web/src/lib/plugins/builtin/editor/editor-subtitle-tools.tsx +++ b/web/src/lib/plugins/builtin/editor/editor-subtitle-tools.tsx @@ -1,7 +1,11 @@ // 字幕工具（editor-shell 预设插件贡献 subtitle-tool 插槽，M3.5）。 // SRT 导入导出（复用 srt-parser）+ 「从节点重建字幕片段」命令（§3.1 快照契约： // 节点 subtitleEntries 权威 → 重建时间线字幕快照，替换过期快照，单向显式同步）。 -// AI 高亮在 M6（editor-ai-assistant + ai.text 权限）接入，此处留占位。 +// AI 字幕高亮与关键词标注已在 M6 落地，入口在画布「字幕」弹窗 +//（canvas-subtitle-dialog.tsx，走 ai.text 权限门禁）：对权威节点字幕条目生成 +// 高亮/关键词，并随字幕条目渲染。此处（时间线字幕工具）字幕片段只是快照， +// 编辑器预览尚无字幕文本渲染层，高亮视觉无处挂载，故不在此重复接 AI； +// 待 M4 数据源接入、预览可渲染字幕文本后，再将高亮打通到时间线片段。 import { useMemo, useState } from "react"; @@ -117,10 +121,11 @@ export function EditorSubtitleTools() {
AI 高亮 - M6 + 可用

- AI 字幕高亮与关键词标注将在 M6（ai-assistant 插件 + ai.text 权限）接入。 + AI 字幕高亮与关键词标注已实现，入口在画布「字幕」弹窗（点击字幕 → AI 高亮，走 ai.text 权限）。 + 时间线字幕片段由该字幕快照重建而来；待编辑器预览支持字幕文本渲染后，高亮会随快照同步到这里。

diff --git a/web/src/pages/assets/index.tsx b/web/src/pages/assets/index.tsx index 6f43c9d12..1b8306d17 100644 --- a/web/src/pages/assets/index.tsx +++ b/web/src/pages/assets/index.tsx @@ -1097,8 +1097,13 @@ function AssetCard({ ); } +function isKnownAssetKind(kind: unknown): kind is AssetKind { + return kind === "image" || kind === "video" || kind === "audio" || kind === "model" || kind === "text"; +} + function AssetCover({ asset, selected, isTrash = false, onSelect, onOpen, menuItems }: { asset: LibraryAsset; selected: boolean; isTrash?: boolean; onSelect: (selected: boolean) => void; onOpen: () => void; menuItems: MenuProps["items"] }) { - const KindIcon = assetKindIcons[asset.kind]; + const kind = isKnownAssetKind(asset.kind) ? asset.kind : undefined; + const KindIcon = kind ? assetKindIcons[kind] : FileText; const clock = asset.kind === "video" || asset.kind === "audio" ? formatAssetClock(asset.data.durationMs) : null; const showPlay = asset.kind === "video"; const isLight = asset.kind === "audio" || asset.kind === "text" || asset.kind === "model"; @@ -1133,7 +1138,7 @@ function AssetCover({ asset, selected, isTrash = false, onSelect, onOpen, menuIt - {assetKindLabel(asset.kind)} + {kind ? assetKindLabel(kind) : "素材"} {isTrash ? 回收站 : {assetCategoryLabel(asset.category)}} @@ -1320,7 +1325,8 @@ function AssetFilterGroup({ function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: LibraryAsset | null; onClose: () => void; onCopy: (asset: LibraryAsset) => void; onDownload: (asset: LibraryAsset) => void }) { const facts = asset ? assetArchiveFacts(asset) : []; - const KindIcon = asset ? assetKindIcons[asset.kind] : Clapperboard; + const kind = asset && isKnownAssetKind(asset.kind) ? asset.kind : undefined; + const KindIcon = asset ? (kind ? assetKindIcons[kind] : FileText) : Clapperboard; return ( {asset ? ( diff --git a/web/src/pages/projects/detail/editor.tsx b/web/src/pages/projects/detail/editor.tsx index 882856e1d..1b99f68b2 100644 --- a/web/src/pages/projects/detail/editor.tsx +++ b/web/src/pages/projects/detail/editor.tsx @@ -15,8 +15,8 @@ import { X, } from "lucide-react"; -import type { ProjectAsset, ProjectDetail } from "@/services/api/projects"; -import { listProjectAssets } from "@/services/api/projects"; +import type { ProjectAsset, ProjectDetail, ShotArtifact } from "@/services/api/projects"; +import { linkProjectAsset, listProjectAssets } from "@/services/api/projects"; import { useEditorSlots, type EditorSlotRegistration } from "@/lib/plugins/editor-slot-registry"; import { pluginMayRenderEditorSlot } from "@/lib/plugins/plugin-permission-check"; import { EditorStoreProvider } from "@/components/editor/editor-context"; @@ -269,11 +269,14 @@ export default function ProjectEditorView({ detail }: { detail: ProjectDetail }) const [timelineH, setTimelineH] = useState(240); const splitterRef = useRef<{ startY: number; startH: number } | null>(null); - const refreshAssets = async () => { + const refreshAssets = async (): Promise => { try { - setAssets((await listProjectAssets(projectId)).assets); + const list = (await listProjectAssets(projectId)).assets; + setAssets(list); + return list; } catch { - // 刷新失败保留现有列表；导入面板内联提示错误 + // 刷新失败保留现有列表；调用方可据此决定是否重试 + return null; } }; @@ -294,6 +297,19 @@ export default function ProjectEditorView({ detail }: { detail: ProjectDetail }) const scope = getActiveUserScope(); const projectId = detail.project.id; + // 挂载/切换项目时以服务端为准同步一次资产：detail.assets 只是详情页快照， + // 上个会话导入但详情未携带（或导入后未刷新）的素材会在进入编辑器后补现。 + useEffect(() => { + let cancelled = false; + void (async () => { + const list = await refreshAssets(); + if (!cancelled && list) setAssets(list); + })(); + return () => { + cancelled = true; + }; + }, [projectId]); + const store = useState(() => createEditorStore({ saveTimeline: async (project: TimelineProject) => { @@ -318,6 +334,58 @@ export default function ProjectEditorView({ detail }: { detail: ProjectDetail }) }; }, [scope, projectId, store]); + // 画布产物自动同步：detail.shotArtifacts 中当前采用(storyboard/action_board/ + // 分镜/导出成片，且 ready + 有资源)尚未作为素材进本项目时，自动 + // linkProjectAsset(source=canvas) 并入素材库 —— 剪辑器“素材选项”直接可用画布产物。 + // 后端幂等(已链接直接返回)，失败静默，下次进入编辑器自动补同步； + // 已完成集合记录在 ref，避免同一会话重复请求。 + const syncedCanvasResourcesRef = useRef>(new Set()); + const canvasSyncKeys = detail.shotArtifacts + .filter( + (a): a is ShotArtifact & { resourceId: string } => + a.selected && + a.status === "ready" && + !!a.resourceId && + ["storyboard", "action_board", "start_frame", "end_frame", "video", "delivery"].includes(a.type), + ) + .map((a) => `resource:${a.resourceId}`) + .join(","); + useEffect(() => { + if (!canvasSyncKeys) return; + const keys = canvasSyncKeys.split(","); + // 已同步过的(本会话 ref)或已存在于项目素材的(storageKey 命中)均跳过 —— + // storageKey 与 assetFromUploadedResource 合成格式一致(resource:)。 + const existing = new Set(assets.map((a) => a.storageKey)); + const pending = keys.filter((key) => !syncedCanvasResourcesRef.current.has(key) && !existing.has(key)); + if (!pending.length) return; + let cancelled = false; + void (async () => { + for (const key of pending) { + if (cancelled) break; + const resourceId = key.slice("resource:".length); + try { + await linkProjectAsset(projectId, { assetId: resourceId, category: "material", source: "canvas" }); + syncedCanvasResourcesRef.current.add(key); + } catch { + // 单个同步失败不阻塞其余；留待下次进入编辑器重试 + } + } + if (!cancelled) { + // 直接拉取最新列表(setAssets 稳定)而非经 refreshAssets， + // 避免闭包依赖变化导致 effect 反复触发 + try { + const list = (await listProjectAssets(projectId)).assets; + if (!cancelled) setAssets(list); + } catch { + // 刷新失败保留现有列表，下次进入编辑器再同步 + } + } + })(); + return () => { + cancelled = true; + }; + }, [canvasSyncKeys, projectId, assets]); + return (
diff --git a/web/src/pages/projects/detail/overview.tsx b/web/src/pages/projects/detail/overview.tsx index 1de926286..e331241e6 100644 --- a/web/src/pages/projects/detail/overview.tsx +++ b/web/src/pages/projects/detail/overview.tsx @@ -32,7 +32,7 @@ export default function ProjectOverviewView({ detail, overview }: ProjectDetailV { id: "storyboard", icon: Clapperboard, label: "分镜脚本与画面", description: "拆分镜头并确认构图、对白和时长", metric: `${metrics.shotCount} 镜 · ${metrics.readyStoryboardCount} 张图`, href: workflowHref("storyboard"), complete: metrics.shotCount > 0 }, { id: "previz", icon: PlaySquare, label: "动作预演", description: "检查表演节拍、运镜和连续性", metric: `${metrics.readyPrevizCount}/${metrics.shotCount || 0} 镜`, href: workflowHref("previz"), complete: metrics.shotCount > 0 && metrics.readyPrevizCount === metrics.shotCount }, { id: "video", icon: Film, label: "镜头视频", description: "逐镜生成、筛选版本并锁定成片", metric: `${metrics.readyVideoCount}/${metrics.shotCount || 0} 镜`, href: workflowHref("video"), complete: metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount }, - { id: "editor", icon: Scissors, label: "剪辑成片", description: "在时间线中编排镜头、添加字幕并输出成片", metric: metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount ? "可以开始剪辑" : `还差 ${Math.max(0, metrics.shotCount - metrics.readyVideoCount)} 镜`, href: `/projects/${project.id}/editor`, complete: false }, + { id: "editor", icon: Scissors, label: "剪辑成片", description: "在时间线中编排镜头、添加字幕并输出成片", metric: metrics.renderSucceededCount > 0 ? `已输出 ${metrics.renderSucceededCount} 个成片` : metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount ? "可以开始剪辑" : `还差 ${Math.max(0, metrics.shotCount - metrics.readyVideoCount)} 镜`, href: `/projects/${project.id}/editor`, complete: metrics.renderSucceededCount > 0 }, { id: "delivery", icon: PackageCheck, label: "交付与打包", description: "检查缺失镜头并整理最终产物", metric: metrics.readyVideoCount && metrics.readyVideoCount === metrics.shotCount ? "可以交付" : `还差 ${Math.max(0, metrics.shotCount - metrics.readyVideoCount)} 镜`, href: workflowHref("delivery"), complete: metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount }, ]; const gaps = [ diff --git a/web/src/services/api/projects.ts b/web/src/services/api/projects.ts index 6a5ccac88..9214054ce 100644 --- a/web/src/services/api/projects.ts +++ b/web/src/services/api/projects.ts @@ -278,6 +278,7 @@ export type ProjectOverviewMetrics = { readyStoryboardCount: number; readyPrevizCount: number; readyVideoCount: number; + renderSucceededCount: number; staleArtifactCount: number; }; @@ -479,7 +480,7 @@ export function unlinkCanvasProject(projectId: string, canvasId: string) { return request<{ canvasId: string }>(api.delete(`/projects/${encodeURIComponent(projectId)}/canvases/${encodeURIComponent(canvasId)}`)); } -export function linkProjectAsset(projectId: string, input: { assetId: string; category: AssetCategory; folderId?: string; title?: string }, signal?: AbortSignal) { +export function linkProjectAsset(projectId: string, input: { assetId: string; category: AssetCategory; folderId?: string; title?: string; source?: "uploaded" | "canvas" }, signal?: AbortSignal) { return request<{ asset: ProjectAsset }>(api.post(`/projects/${encodeURIComponent(projectId)}/assets`, input, { signal })); } diff --git a/web/src/services/api/task-center.ts b/web/src/services/api/task-center.ts index 988d0d071..e1786b791 100644 --- a/web/src/services/api/task-center.ts +++ b/web/src/services/api/task-center.ts @@ -532,6 +532,7 @@ export async function waitForGenerationTask(id: string, options?: WaitForGenerat const intervalMs = options?.intervalMs || 2000; let lastTask = options?.initialTask; let lastQueryError: unknown; + let consecutiveFailures = 0; try { while (Date.now() - startedAt < (options?.timeoutMs || taskWaitTimeoutMs(lastTask))) { if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError"); @@ -540,9 +541,16 @@ export async function waitForGenerationTask(id: string, options?: WaitForGenerat task = await queryGenerationTask(id, { signal: options?.signal }); lastTask = task; lastQueryError = undefined; + consecutiveFailures = 0; options?.onTaskUpdate?.(task); } catch (error) { lastQueryError = error; + consecutiveFailures += 1; + // 连续失败说明查询通道已不可用，继续轮询只会空转到整体超时； + // 保留少量容忍度（约 10 秒）以跳过瞬时抖动后直接报错，便于用户尽早处理。 + if (consecutiveFailures >= 5) { + throw error instanceof Error ? error : new Error(String(error)); + } await delay(intervalMs, options?.signal); continue; }