feat(editor): 剪辑成片验收闭环 - 概览渲染计数、导入/字幕/预览修复与本地 whisper 转写运行时 (#414)
* feat(editor): 剪辑成片验收闭环 - 概览渲染成功计数、媒体导入/字幕/预览修复与本地 whisper 转写运行时 - 概览:渲染成功计数接入项目概览 SQL/JSON,成片状态随 timeline_render 任务联动 - 修复:本地媒体导入后列表为空(挂载时服务端重同步 + 导入后校验重取)、 字幕工具残留 M6 占位文案、预览监视器悬浮遮挡层改为顶部信息条、 生成任务轮询连续查询失败即抛错不再空转至超时 - 本地转写:新增 scripts/start-whisper-local.sh 一键启动 whisper.cpp, local-development 文档补安装/启动/后端注入指引 - 同步 docs(overview/features、pending-test、index) * fix(assets): 素材库封面未知 kind 兜底,避免 AssetCover 渲染崩溃 运行时素材记录若带未知/脏 kind(历史遗留或本地存储异常),assetKindIcons[asset.kind] 得到 undefined,AssetCover/AssetDrawer 渲染 <KindIcon/> 触发 'Element type is invalid ... got: undefined' 使 /assets 整页崩溃。 新增 isKnownAssetKind 守卫:未知 kind 回退 FileText 图标与通用文案,脏数据不再崩页。 * fix(editor): mp4v 播放死循环修复——MPEG-4 Part 2 归入播放转码通道，监视器加终态护栏 - 后端把 mp4v 与 H.264/AV1/VP9 一并判为浏览器可播 → playbackStatus=none； Chromium/Firefox/Edge 实际均不支持 MPEG-4 Part 2，原件解码即失败。 现将其并入 H.265 转码通道(转 H.264 兼容副本)；损坏文件转码失败落 failed，前端据此显示终止性错误而非死循环。 - 前端监视器轮询到 none/failed 或超时后不再回退 primary 重试；副本 URL 回退原件反复触发 onError 时保持终态文案，不再覆盖成"正在生成兼容版本"。 * feat(editor): 画布产物自动同步剪辑素材库，素材面板新增「画布素材」分组 - 打开剪辑器时自动把画布工作台当前采用(storyboard/action_board/分镜/导出成片， ready 且有资源)的产物 linkProjectAsset(source=canvas) 并入项目素材库； 后端幂等、前端按 storageKey+会话 ref 去重，失败静默下次进入自动补同步。 - LinkProjectAsset 支持 source 白名单(uploaded/canvas)并写入素材 payload， 供素材面板按来源分组；面板新增「画布素材」页签与分组，与「本地上传」「项目素材」并列。 - 补 source=canvas 写入断言测试。 * docs(progress): 记录 mp4v 播放死循环修复与画布产物素材同步的待浏览器验收项
This commit is contained in:
爱笑的毛毛虫
@@ -7,19 +7,20 @@ import (
|
||||
)
|
||||
|
||||
type ProjectOverviewMetrics struct {
|
||||
UnitCount int64 `gorm:"column:unit_count"`
|
||||
CompletedUnitCount int64 `gorm:"column:completed_unit_count"`
|
||||
TotalWordCount int64 `gorm:"column:total_word_count"`
|
||||
UnitsWithoutText int64 `gorm:"column:units_without_text"`
|
||||
UnitsWithoutShots int64 `gorm:"column:units_without_shots"`
|
||||
CanvasCount int64 `gorm:"column:canvas_count"`
|
||||
AssetCount int64 `gorm:"column:asset_count"`
|
||||
ShotCount int64 `gorm:"column:shot_count"`
|
||||
PendingCandidateCount int64 `gorm:"column:pending_candidate_count"`
|
||||
ReadyStoryboardCount int64 `gorm:"column:ready_storyboard_count"`
|
||||
ReadyPrevizCount int64 `gorm:"column:ready_previz_count"`
|
||||
ReadyVideoCount int64 `gorm:"column:ready_video_count"`
|
||||
StaleArtifactCount int64 `gorm:"column:stale_artifact_count"`
|
||||
UnitCount int64 `gorm:"column:unit_count"`
|
||||
CompletedUnitCount int64 `gorm:"column:completed_unit_count"`
|
||||
TotalWordCount int64 `gorm:"column:total_word_count"`
|
||||
UnitsWithoutText int64 `gorm:"column:units_without_text"`
|
||||
UnitsWithoutShots int64 `gorm:"column:units_without_shots"`
|
||||
CanvasCount int64 `gorm:"column:canvas_count"`
|
||||
AssetCount int64 `gorm:"column:asset_count"`
|
||||
ShotCount int64 `gorm:"column:shot_count"`
|
||||
PendingCandidateCount int64 `gorm:"column:pending_candidate_count"`
|
||||
ReadyStoryboardCount int64 `gorm:"column:ready_storyboard_count"`
|
||||
ReadyPrevizCount int64 `gorm:"column:ready_previz_count"`
|
||||
ReadyVideoCount int64 `gorm:"column:ready_video_count"`
|
||||
TimelineRenderSucceededCount int64 `gorm:"column:timeline_render_succeeded_count"`
|
||||
StaleArtifactCount int64 `gorm:"column:stale_artifact_count"`
|
||||
}
|
||||
|
||||
type ProjectOverviewUnitRow struct {
|
||||
@@ -62,8 +63,9 @@ func (r *Repository) ProjectOverviewMetrics(projectID string) (ProjectOverviewMe
|
||||
(SELECT COUNT(DISTINCT shot_id) FROM shot_artifacts WHERE project_id = ? AND type = 'storyboard' AND selected = ? AND status = 'ready') AS ready_storyboard_count,
|
||||
(SELECT COUNT(DISTINCT shot_id) FROM shot_artifacts WHERE project_id = ? AND type = 'action_board' AND selected = ? AND status = 'ready') AS ready_previz_count,
|
||||
(SELECT COUNT(DISTINCT shot_id) FROM shot_artifacts WHERE project_id = ? AND type = 'video' AND selected = ? AND status = 'ready') AS ready_video_count,
|
||||
(SELECT COUNT(*) FROM tasks WHERE project_id = ? AND type = ? AND status = ?) AS timeline_render_succeeded_count,
|
||||
(SELECT COUNT(*) FROM shot_artifacts WHERE project_id = ? AND status = 'stale') AS stale_artifact_count
|
||||
`, projectID, projectID, model.ProjectUnitStatusCompleted, projectID, projectID, projectID, model.ProjectUnitStatusDraft, projectID, projectID, projectID, projectID, projectID, true, projectID, true, projectID, true, projectID).Scan(&metrics).Error
|
||||
`, projectID, projectID, model.ProjectUnitStatusCompleted, projectID, projectID, projectID, model.ProjectUnitStatusDraft, projectID, projectID, projectID, projectID, projectID, true, projectID, true, projectID, true, projectID, model.TaskTypeTimelineRender, model.TaskStatusSucceeded, projectID).Scan(&metrics).Error
|
||||
return metrics, err
|
||||
}
|
||||
|
||||
|
||||
@@ -11,12 +11,20 @@ import (
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
// 资产来源：媒体导入合成/画布产物自动同步。画布产物用 canvas 以便素材面板分组。
|
||||
const (
|
||||
AssetSourceUploaded = "uploaded"
|
||||
AssetSourceCanvas = "canvas"
|
||||
)
|
||||
|
||||
type LinkProjectAssetRequest struct {
|
||||
AssetID string `json:"assetId"`
|
||||
Category string `json:"category"`
|
||||
FolderID *string `json:"folderId"`
|
||||
// Title 媒体导入场景下由前端携带原始文件名；已有资产时忽略。
|
||||
Title string `json:"title"`
|
||||
// Source 仅媒体导入合成时生效：uploaded（默认）或 canvas（画布产物自动同步）。
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
type UpdateProjectAssetRequest struct {
|
||||
@@ -105,6 +113,10 @@ func (s *Service) LinkProjectAsset(userID string, projectID string, req LinkProj
|
||||
return ProjectAssetSummary{}, err
|
||||
}
|
||||
assetID := strings.TrimSpace(req.AssetID)
|
||||
source := strings.TrimSpace(req.Source)
|
||||
if source != AssetSourceCanvas {
|
||||
source = AssetSourceUploaded
|
||||
}
|
||||
asset, err := s.repo.AssetForUser(userID, assetID)
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return ProjectAssetSummary{}, err
|
||||
@@ -112,7 +124,7 @@ func (s *Service) LinkProjectAsset(userID string, projectID string, req LinkProj
|
||||
if asset == nil {
|
||||
// 媒体导入只落 resources 表；首次链接时按资源元数据合成资产记录，
|
||||
// 避免“资源存在但无资产记录”导致导入永远失败。
|
||||
asset, err = s.assetFromUploadedResource(userID, assetID, req.Title)
|
||||
asset, err = s.assetFromUploadedResource(userID, assetID, req.Title, source)
|
||||
if err != nil {
|
||||
return ProjectAssetSummary{}, err
|
||||
}
|
||||
@@ -173,7 +185,7 @@ func (s *Service) LinkProjectAsset(userID string, projectID string, req LinkProj
|
||||
|
||||
// assetFromUploadedResource 把媒体导入上传的 Resource 合成为资产记录（内存构造，不落库）。
|
||||
// 资产创建与项目链接由 LinkProjectAsset 事务原子提交，避免失败留下孤立资产。
|
||||
func (s *Service) assetFromUploadedResource(userID string, resourceID string, title string) (*model.Asset, error) {
|
||||
func (s *Service) assetFromUploadedResource(userID string, resourceID string, title string, source string) (*model.Asset, error) {
|
||||
resource, err := s.repo.ResourceForUser(userID, resourceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -192,7 +204,7 @@ func (s *Service) assetFromUploadedResource(userID string, resourceID string, ti
|
||||
"durationMs": resource.DurationMs,
|
||||
"status": resource.Status,
|
||||
"mediaType": resource.Kind,
|
||||
"source": "uploaded",
|
||||
"source": source,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
@@ -58,6 +59,7 @@ func TestLinkProjectAssetCreatesAssetFromUploadedResource(t *testing.T) {
|
||||
AssetID: "resource-1",
|
||||
Category: "",
|
||||
Title: "开场.mp4",
|
||||
Source: AssetSourceCanvas,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("LinkProjectAsset: %v", err)
|
||||
@@ -76,6 +78,20 @@ func TestLinkProjectAssetCreatesAssetFromUploadedResource(t *testing.T) {
|
||||
if asset.Category != model.AssetCategoryMaterial {
|
||||
t.Errorf("asset.Category = %q, want material", asset.Category)
|
||||
}
|
||||
if summary.Source != AssetSourceCanvas {
|
||||
t.Errorf("summary.Source = %q, want canvas", summary.Source)
|
||||
}
|
||||
var payload struct {
|
||||
Data struct {
|
||||
Source string `json:"source"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(asset.PayloadJSON), &payload); err != nil {
|
||||
t.Fatalf("payload unmarshal: %v", err)
|
||||
}
|
||||
if payload.Data.Source != AssetSourceCanvas {
|
||||
t.Errorf("payload.source = %q, want canvas", payload.Data.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLinkProjectAssetMissingResourceKeepsFailingClosed 资源与资产都不存在时仍应报错，
|
||||
|
||||
@@ -33,6 +33,7 @@ type ProjectOverviewMetrics struct {
|
||||
ReadyStoryboardCount int64 `json:"readyStoryboardCount"`
|
||||
ReadyPrevizCount int64 `json:"readyPrevizCount"`
|
||||
ReadyVideoCount int64 `json:"readyVideoCount"`
|
||||
RenderSucceededCount int64 `json:"renderSucceededCount"`
|
||||
StaleArtifactCount int64 `json:"staleArtifactCount"`
|
||||
}
|
||||
|
||||
@@ -152,7 +153,7 @@ func (s *Service) ProjectOverview(userID string, projectID string) (ProjectOverv
|
||||
UnitsWithoutText: row.UnitsWithoutText, UnitsWithoutShots: row.UnitsWithoutShots, CanvasCount: row.CanvasCount,
|
||||
AssetCount: row.AssetCount, ShotCount: row.ShotCount, PendingCandidateCount: row.PendingCandidateCount,
|
||||
ReadyStoryboardCount: row.ReadyStoryboardCount, ReadyPrevizCount: row.ReadyPrevizCount, ReadyVideoCount: row.ReadyVideoCount,
|
||||
StaleArtifactCount: row.StaleArtifactCount,
|
||||
RenderSucceededCount: row.TimelineRenderSucceededCount, StaleArtifactCount: row.StaleArtifactCount,
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -121,6 +121,33 @@ func TestProjectOverviewUsesAggregatesWithoutLoadingProjectCollections(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectOverviewCountsSucceededTimelineRenders(t *testing.T) {
|
||||
service, db := newProjectWorkbenchReadTestService(t)
|
||||
project := seedWorkbenchProject(t, db)
|
||||
for index := 0; index < 2; index++ {
|
||||
task := model.Task{ID: fmt.Sprintf("render-%02d", index), UserID: "user-1", ProjectID: project.ID, Type: model.TaskTypeTimelineRender, Status: model.TaskStatusSucceeded}
|
||||
if err := db.Create(&task).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
distractors := []model.Task{
|
||||
{ID: "render-failed", UserID: "user-1", ProjectID: project.ID, Type: model.TaskTypeTimelineRender, Status: model.TaskStatusFailed},
|
||||
{ID: "transcribe-ok", UserID: "user-1", ProjectID: project.ID, Type: model.TaskTypeTimelineTranscription, Status: model.TaskStatusSucceeded},
|
||||
}
|
||||
for _, task := range distractors {
|
||||
if err := db.Create(&task).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
overview, err := service.ProjectOverview("user-1", project.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if overview.Metrics.RenderSucceededCount != 2 {
|
||||
t.Fatalf("render succeeded count = %d, want 2", overview.Metrics.RenderSucceededCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectUnitWorkspaceIsolatesShotsAndBoundAssetsByUnit(t *testing.T) {
|
||||
service, db := newProjectWorkbenchReadTestService(t)
|
||||
project := seedWorkbenchProject(t, db)
|
||||
|
||||
@@ -180,7 +180,10 @@ func (s *Service) maybeStartPlaybackTranscode(resource *model.Resource) {
|
||||
}
|
||||
src := filepath.Join(s.dataDir, "resources", filepath.FromSlash(resource.ObjectKey))
|
||||
switch probeVideoCodec(src) {
|
||||
case videoCodecH265:
|
||||
case videoCodecH265, videoCodecMPEG4:
|
||||
// H.265 与 MPEG-4 Part 2（mp4v）Chromium/Firefox/Edge 均不能解码：
|
||||
// H.265 需转 H.264；MPEG-4 Part 2 浏览器不支持，老式/损坏文件只能转码兜底
|
||||
// （转码失败会落 failed，前端据此显示终止性错误，不再出现死循环）。
|
||||
// 原子抢占（空/none → processing）：并发上传 + 回填、多实例同抢时
|
||||
// 仅一个能成功置位，其余直接返回，避免重复转码。
|
||||
claimed, err := s.repo.ClaimPlaybackTranscode(resource.ID)
|
||||
@@ -189,9 +192,10 @@ func (s *Service) maybeStartPlaybackTranscode(resource *model.Resource) {
|
||||
}
|
||||
resource.PlaybackStatus = model.PlaybackStatusProcessing
|
||||
go s.runPlaybackTranscode(resource.UserID, resource.ID, src)
|
||||
case videoCodecH264, videoCodecAV1, videoCodecVP9, videoCodecMPEG4, "":
|
||||
// H.264 浏览器可直接解码；AV1/VP9/MPEG4 暂不转码；探针读不出编码（非 mp4 /
|
||||
// moov 在尾部 / 加密容器）也无法处理 —— 均标 none，避免重复探测与前端无限轮询。
|
||||
case videoCodecH264, videoCodecAV1, videoCodecVP9, "":
|
||||
// H.264 浏览器可直接解码；AV1/VP9 现代浏览器可直接解码，均不需转码；
|
||||
// 探针读不出编码（非 mp4 / moov 在尾部 / 加密容器）也无法处理 —— 均标 none，
|
||||
// 避免重复探测与前端无限轮询。
|
||||
markPlaybackNone(s, resource)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,55 @@ bun run dev
|
||||
|
||||
前端默认监听 `0.0.0.0:3000`，Vite 将 `/api` 和 OAuth 回调代理到 `127.0.0.1:8080`。如需修改代理目标，可设置 `VITE_API_PROXY_TARGET`。
|
||||
|
||||
## 时间线字幕转写（whisper.cpp）
|
||||
|
||||
时间线“字幕转写”由**本地 whisper.cpp server** 执行：后端将片段音频预处理为 16k mono wav 后，POST 到 `CANVAS_WHISPER_BASE_URL/inference`（whisper server 原生接口，`response_format=verbose_json`）。未配置该地址时，转写任务会立即明确失败，提示“撰写失败未配置本地转写服务”。
|
||||
|
||||
### 安装 whisper-server
|
||||
|
||||
```bash
|
||||
brew install whisper-cpp
|
||||
```
|
||||
|
||||
### 下载模型
|
||||
|
||||
仓库脚本将模型放在 `.local/whisper-models/`（已被 `.gitignore` 忽略）。多语言模型（含中文）用 `ggml-base.bin`（约 141MB）；网络受限时可用 aifasthub 镜像：
|
||||
|
||||
```bash
|
||||
mkdir -p .local/whisper-models
|
||||
curl -fL -o .local/whisper-models/ggml-base.bin \
|
||||
https://aifasthub.com/ggerganov/whisper.cpp/resolve/main/ggml-base.bin
|
||||
```
|
||||
|
||||
### 启动服务
|
||||
|
||||
```bash
|
||||
scripts/start-whisper-local.sh
|
||||
```
|
||||
|
||||
脚本默认以 `ggml-base.bin` 监听 `127.0.0.1:8082`（`-l auto` 自动识别语言）。需要换模型或端口时：`WHISPER_MODEL=ggml-small.bin WHISPER_PORT=8083 scripts/start-whisper-local.sh`。
|
||||
|
||||
### 让后端指向该服务
|
||||
|
||||
后端不读取 `.env` 文件，环境变量必须由启动 shell 注入（在手工启动命令基础上追加）：
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
CANVAS_BACKEND_DATA_DIR=../.local/project-workbench-debug \
|
||||
CANVAS_WHISPER_BASE_URL=http://127.0.0.1:8082 \
|
||||
go run ./cmd/server
|
||||
```
|
||||
|
||||
### 验证
|
||||
|
||||
```bash
|
||||
# 服务就绪：返回 200/404 即可
|
||||
curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8082/
|
||||
# 冒烟：20 秒语音 → verbose_json 字幕段
|
||||
curl -s -X POST http://127.0.0.1:8082/inference \
|
||||
-F 'file=@sample.wav' -F 'response_format=verbose_json'
|
||||
```
|
||||
|
||||
## Docker 开发
|
||||
|
||||
需要容器化热更新时，在仓库根目录执行：
|
||||
|
||||
@@ -159,3 +159,39 @@ SEO 标题、描述、可选关键词会投影到浏览器 metadata、Open Graph
|
||||
- `web/src/components/canvas/canvas-zoom-controls.tsx`：底部视图控制 Dock 与“自动整理节点”入口。
|
||||
- `web/src/pages/canvas/use-canvas-node-operations.ts`：候选节点筛选、流程/网格布局选择和位置提交。
|
||||
- `web/src/lib/canvas/canvas-layout.ts`：媒体泳道、拓扑流程、网格、行列排列和对齐算法。
|
||||
|
||||
## 剪辑成片（时间线编辑器）
|
||||
|
||||
“剪辑成片”是项目详情里的时间线编辑器入口（`/projects/:id/editor`），用于把已就绪的分镜视频编排成一段可播放、可导出的成片。它运行在浏览器本地时间线模型之上，通过后端任务队列完成语音转写与成片导出，编辑器本身不直接触碰原始媒体大文件。
|
||||
|
||||
### 使用入口
|
||||
|
||||
- 项目详情“制作流程 → 剪辑成片”步骤卡，或地址栏直接访问 `/projects/:id/editor`。
|
||||
- 编辑器按项目隔离时间线状态；离开后再次进入会恢复该项目的编辑器快照。
|
||||
|
||||
### 当前能力
|
||||
|
||||
- **轨道与命令**：时间线支持添加、删除、拆分、移动、修剪片段等操作，全部通过统一命令状态机执行并进入 200 层历史记录，可撤销与重做；非法操作整批拒绝，不产生半应用状态。
|
||||
- **预设面板**：左侧“添加”分类提供 8 个内置片段/字幕插件，按分类分组接入同一插槽注册链，第三方插件可与内置面板共存。
|
||||
- **权限与执行边界**：命令执行遵循失败关闭（fail-closed）策略，`timeline.command` 权限与全局 `plugins.run` 权限分开控制；未授权插件或命令不会被执行或回显。
|
||||
- **语音转写（字幕）**：对时间线音视频发起转写任务，转写进度沿任务 SSE 回流，完成后生成字幕轨。
|
||||
- **成片导出（渲染）**：发起渲染任务，由服务端 ffmpeg 按渲染计划输出成片；渲染成功后在项目概览的“剪辑成片”步骤卡呈现“已输出 N 个成片”，并作为该步骤完成度的判定依据。
|
||||
- **AI 编辑助手**：可以用自然语言描述剪辑意图，系统将其解析为受支持的命令序列；不超过 3 条命令时直接执行，多于 3 条先给出差异预览，确认后才批量执行。
|
||||
|
||||
### 运行环境要求
|
||||
|
||||
- 语音转写依赖本地 whisper.cpp 服务（`CANVAS_WHISPER_BASE_URL`），未配置时转写任务明确失败。
|
||||
- 成片导出依赖服务端 ffmpeg（`CANVAS_FFMPEG_PATH`，留空时在 PATH 中查找）。
|
||||
- AI 编辑助手需要可用的文本模型渠道。
|
||||
|
||||
### 相关实现
|
||||
|
||||
- `web/src/pages/projects/detail/editor.tsx`、`web/src/pages/projects/detail.tsx`：编辑器页面与路由入口。
|
||||
- `web/src/pages/projects/detail/overview.tsx`：制作流程步骤卡与成片完成度展示。
|
||||
- `web/src/lib/timeline/`：时间线命令状态机、历史记录、命令 schema 与摘要（`editor-commands.ts`、`editor-history.ts`、`ai-command-schema.ts`、`timeline-summary.ts`）。
|
||||
- `web/src/stores/editor/editor-store.ts`：编辑器状态与快照持久化。
|
||||
- `web/src/lib/plugins/editor-slot-registry.ts`、`web/src/lib/plugins/builtin/editor/`：插槽注册链与内置预设插件。
|
||||
- `web/src/lib/plugins/plugin-permission-check.ts`：命令执行权限与失败关闭策略。
|
||||
- `web/src/services/api/timeline-tasks.ts`：转写与渲染任务客户端。
|
||||
- `backend/internal/service/task_timeline.go`、`task_render.go`、`timeline_render_plan.go`、`transcription_whisper.go`：后端转写、渲染与渲染计划任务。
|
||||
- `backend/internal/repository/project_workbench_read.go`：项目概览指标聚合（含成片完成度计数）。
|
||||
@@ -43,7 +43,8 @@ description: 当前项目已实现功能的手工验证清单与未确认边界
|
||||
- [x] M4.1 创建 API（bf26ebb1）：`POST /api/timeline/transcriptions`（注册于 `RegisterTaskRoutes`）→ `CreateTimelineTranscriptionTask`：draining 503 → feature 门控 → 资源归属/可转写类型校验 → 配额入队（local/whisper.cpp，无 billing order）；活跃任务上限 → 可读 400；测试 3 项 PASS。
|
||||
- [x] M4.2 渲染任务（3f16d13b）：`POST /api/timeline/renders` → `CreateTimelineRenderTask`：`buildRenderPlan` 校验 ≥1 可渲染媒体 → 配额入队（local/ffmpeg）；worker 分叉 `processTimelineRender`（60 分钟超时）：ffmpeg 合成 → 产物写资源存储 → `ResultJSON{resourceId,...}`；计划/失败快路径测试 PASS。
|
||||
- [x] M4.3 前端字幕回写（fd5e8139）：`timeline-tasks.ts` 客户端 + `editor-transcription.tsx` 真实任务替换 mock；资源范围放宽到音视频；`resourceId` 由 `storageKey` 推导；`waitForGenerationTask` 显式 25 分钟超时轮询；结果 segments → SrtEntry → `rebuildSubtitleClips` 落字幕轨；typecheck 通过。
|
||||
- [ ] 浏览器端到端（待用户确认，需本机 whisper.cpp server）：媒体面板选音视频素材 → 转写 → 进度可见 → 完成后字幕分段入轨可编辑；错误路径（未配置/删除素材/非音视频/活跃任务上限/功能未开放）提示可读。
|
||||
- [x] M4.1 运行时端到端（2026-09-04，真实 whisper.cpp）：`brew install whisper-cpp` + `ggml-base.bin`，whisper-server `127.0.0.1:8082`（`-l auto`）；后端注入 `CANVAS_WHISPER_BASE_URL=http://127.0.0.1:8082` 重启；真实 mp4 资源经 worker 队列（status=queued → running → succeeded 100%）跑通：ffmpeg 16k mono 提取 → `/inference` verbose_json → 中文分段含 start/end/text 入库，与 `transcription_whisper.go` 解码协议一致。失败路径（未配置）此前 5 条任务均按设计报“撰写失败未配置本地转写服务”。启动步骤已写入 `local-development.mdx`，脚本 `scripts/start-whisper-local.sh`。
|
||||
- [ ] 浏览器端到端（待用户确认；本机 whisper-server 已就绪，直接重试转写即可）：媒体面板选音视频素材 → 转写 → 进度可见 → 完成后字幕分段入轨可编辑；错误路径（未配置/删除素材/非音视频/活跃任务上限/功能未开放）提示可读。
|
||||
- [x] M4.2 渲染前端接线：`timeline-tasks.ts` 增 `createTimelineRenderTask`（timeline 整传 TimelineProject v2，与后端 renderProject 契约同构）；`editor-export.tsx` 主路径改提交 `POST /api/timeline/renders` → `waitForGenerationTask` 轮询（62min）→ ResultJSON.resourceId 内嵌预览/下载；ffmpeg.wasm 降级保留；tsc 全绿、无新增测试失败。
|
||||
- [ ] M4.2 渲染浏览器验证（待用户确认，需 ffmpeg 可用）：导出面板「渲染成片（服务端）」→ 轮询进度 → 完成后内嵌预览/下载渲染产物；无后端时「本地导出」ffmpeg.wasm 仍可用。
|
||||
- [ ] 文档回写（已提交）：`docs/plans/m4-transcription-render-design.md` §2 决策 3/4/5 按“单一池 + 类型分叉”与旁路终态修正；端点统一为复数 `/api/timeline/transcriptions` 与 `/api/timeline/renders`（注册于 `RegisterTaskRoutes`）；`auth.go` 前缀白名单补 `timeline`。
|
||||
@@ -62,6 +63,8 @@ description: 当前项目已实现功能的手工验证清单与未确认边界
|
||||
- [x] 编辑器 UI 打磨（第 36 条 5 项）：① 时间线/监视器分隔条拖拽方向反转（上拖=时间线变大）；② 导出按钮图标去蓝（灰白描边风格），媒体面板来源徽章去蓝，轨道类型徽章保留语义色；③ 时间线 pxPerMs 独立化（不再随时长漂移），末片段向右拖轨道随内容等比扩展不再被钉住（新增守护测试 3 例）；④ 监视器控制条重做为 Concat 风格：回到开头/退 1s/播放/进 1s/到结尾五键传输组 + 播放速度展开菜单（0.5x/1x/1.5x/2x）+ 进度条，全部灰白描边；⑤ 检查器重做为 Concat 风格：分组小标题（片段/基本/转场）+ 紧凑控件 + sunken 滑块（新增 .editor-slider），全部使用 director token 随主题联动。
|
||||
- [ ] 用户浏览器 re-test（第 36 条）：上拖分隔条时间线变大；导出按钮与监视器控制条图标均为灰白描边风格；时间线末片段可正常向右拖拽且轨道随内容扩展；监视器速度菜单展开选择生效；检查器分组与滑块在明暗两主题下配色一致（无主题不同步）。
|
||||
- [x] 媒体面板改版：资产按来源分组（「本地上传」/「项目素材」），顶部提供 全部/项目素材/本地上传 筛选 Tab；分组头可折叠、资产行可展开查看详情（类型/分类/来源/预览文本）并一键添加到时间线；每项带来源徽章与计数徽章。后端 `projectAssetSummary` 透出 `source`（上传链路写入 `uploaded`），前端 `ProjectAsset` 类型补 `source?: string`，仅展示层使用，后端读取不再查询本地存储。
|
||||
- [x] 概览成片完成度接线：项目概览指标新增 `renderSucceededCount`（repository 聚合 SQL 子查询 `tasks` 表 `type=timeline_render` 且成功的计数），概览「剪辑成片」步骤卡的 complete 态由写死的 false 改为 `renderSucceededCount > 0`，提示文案随计数/就绪情况切换；新增后端专项测试 1 个、前端类型检查通过。
|
||||
- [ ] 用户浏览器 re-test（概览完成度）：渲染成功一次后刷新概览，「剪辑成片」卡应显示已输出 N 个成片且步骤完成态点亮；删除该成片对应任务后计数回退。
|
||||
- [x] 时间线横跨编辑面板全宽：时间线卡从内容区独立出来，与左栏+预览+右栏同处编辑面板，横跨从左到右全宽（含拖拽高度分隔条）；左栏收窄到 256px、右栏 320px，为全宽时间线腾出空间。
|
||||
- [x] 主题明暗一致性：媒体面板分组/筛选/徽章/展开详情全部使用 director token（surface/control-hover/dock-fg/sequencer-border），无新增硬编码色值；后端 `source` 字段不参与任何配色。前端 typecheck、生产构建、后端全量测试通过。
|
||||
- [x] 时间线面板补齐 Concat 风格工具栏与编辑交互（对齐第 2 条 runbook 与 `Concat/desktop/.../TimelinePanel.tsx`）：新增 选择/剃刀 工具切换（剃刀点击分割、非字幕片段、边缘留 100ms）、播放头时码只读显示与标尺拖动 scrub、分割播放头处、删除选中片段、磁吸开关（默认开）、加轨道下拉（视频/音频/字幕，`addTrack` 黄金命令确定性生成 id/order/label，undo/redo 可重放）、适应宽度（`getFitTimelineZoom`）；工具栏布局重排为 Concat 顶部条结构。`editor-commands.ts` golden ops 由 9 增到 10（`addTrack`）。
|
||||
@@ -859,3 +862,12 @@ description: 当前项目已实现功能的手工验证清单与未确认边界
|
||||
- [ ] 拖入时长 >5s 的视频片段，其时间线长度等于视频真实时长（不再固定 5s 截断）；时长探测上传的新素材在资源卡片/片段均显示真实时长。
|
||||
- [ ] 播放副本就绪前（processing）视频黑屏时出现"正在生成兼容版本（H.264）…"浮层 + "下载原片用本地播放器观看"链接；生成失败显示失败文案而非一直转圈。
|
||||
- [ ] 时间线 scrub/播放/变速往返 10 次以上无卡死、无播放头回跳、无 setState-after-unmount 报错（Console 无告警）。
|
||||
|
||||
## MPEG-4 Part 2 播放死循环 + 画布产物素材同步（待浏览器验收）
|
||||
|
||||
已实施，代码/单测/编译验证通过（后端 go vet/test、前端 tsc 全绿），需浏览器最终确认：
|
||||
|
||||
- [ ] 导入 MPEG-4 Part 2(mp4v)视频(如 07ee…mp4 sprite/header damaged 样例)后拖入时间线：不再出现"正在生成兼容版本"提示闪烁死循环；损坏文件转码失败后显示终止性文案(不再反复回退重试)；正常 mp4v 文件自动生成 H.264 副本并可播放。
|
||||
- [ ] 打开剪辑器进入已含画布产物(selected+ready 的 storyboard/action_board/分镜/导出成片)的项目：素材面板自动出现"画布素材"分组与对应素材；再次进入/刷新不产生重复素材。
|
||||
- [ ] 素材面板"画布素材/本地上传/项目素材"三个页签与分组切换、来源徽章与详情来源文本正确。
|
||||
- [ ] 项目素材库页与剪辑器素材面板对本轮新同步的画布素材均可见；下载/删除画布素材不影响对应画布产物资源。
|
||||
|
||||
@@ -45,6 +45,6 @@
|
||||
- [功能](content/docs/overview/features.mdx)
|
||||
- [本地开发](content/docs/backend/local-development.mdx)
|
||||
- [后端数据库](content/docs/backend/backend-database.mdx)
|
||||
- 代码地图：待补充
|
||||
- 待办：待补充
|
||||
- [代码地图](content/docs/backend/code-map.mdx)
|
||||
- [待办](content/docs/progress/todo.mdx)
|
||||
- [待测试](content/docs/progress/pending-test.mdx)
|
||||
|
||||
37
scripts/start-whisper-local.sh
Executable file
37
scripts/start-whisper-local.sh
Executable file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Local whisper.cpp server for timeline subtitle transcription.
|
||||
# The Go backend calls {CANVAS_WHISPER_BASE_URL}/inference with a multipart
|
||||
# wav upload and expects standard whisper verbose_json back (native protocol).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/start-whisper-local.sh # base model, port 8082
|
||||
# WHISPER_MODEL=ggml-small.bin scripts/start-whisper-local.sh
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
ROOT="$(pwd)"
|
||||
|
||||
MODEL_NAME="${WHISPER_MODEL:-ggml-base.bin}"
|
||||
MODEL_DIR="${ROOT}/.local/whisper-models"
|
||||
MODEL_PATH="${MODEL_DIR}/${MODEL_NAME}"
|
||||
HOST="${WHISPER_HOST:-127.0.0.1}"
|
||||
PORT="${WHISPER_PORT:-8082}"
|
||||
THREADS="${WHISPER_THREADS:-4}"
|
||||
|
||||
if ! command -v whisper-server >/dev/null 2>&1; then
|
||||
echo "whisper-server not found. Install with: brew install whisper-cpp" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${MODEL_DIR}"
|
||||
|
||||
if [ ! -f "${MODEL_PATH}" ]; then
|
||||
echo "Model ${MODEL_NAME} not found at ${MODEL_PATH}" >&2
|
||||
echo "Download it first, e.g.:" >&2
|
||||
echo " curl -fL -o ${MODEL_PATH} https://aifasthub.com/ggerganov/whisper.cpp/resolve/main/${MODEL_NAME}" >&2
|
||||
echo "(aifasthub mirrors huggingface.co/ggerganov/whisper.cpp for CN networks)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting whisper-server: model=${MODEL_NAME} host=${HOST} port=${PORT}"
|
||||
exec whisper-server -m "${MODEL_PATH}" --host "${HOST}" --port "${PORT}" --threads "${THREADS}" -l auto
|
||||
@@ -16,7 +16,7 @@ export type EditorHostContextValue = {
|
||||
/** 项目资产列表（M3.4 asset-ingest 消费；由宿主在进入编辑器时获取）。 */
|
||||
assets: ProjectAsset[];
|
||||
/** 重新拉取项目资产（导入媒体后刷新列表）。 */
|
||||
refreshAssets: () => Promise<void>;
|
||||
refreshAssets: () => Promise<ProjectAsset[] | null>;
|
||||
};
|
||||
|
||||
const EditorStoreContext = createContext<EditorStoreHook | null>(null);
|
||||
|
||||
@@ -34,15 +34,17 @@ const IMAGE_RE = /\.(png|jpe?g|webp|gif|avif)$/i;
|
||||
const AUDIO_RE = /\.(mp3|wav|m4a|ogg|flac|aac)$/i;
|
||||
const MEDIA_RE = /\.(mp4|mov|webm|mkv|m4v|avi)$/i;
|
||||
|
||||
/** 本地上传标记（assetFromUploadedResource 写入 payload.data.source）。 */
|
||||
/** 本地上传标记与画布产物标记（assetFromUploadedResource 写入 payload.data.source）。 */
|
||||
const SOURCE_UPLOADED = "uploaded";
|
||||
const SOURCE_CANVAS = "canvas";
|
||||
|
||||
type AssetFilter = "all" | "project" | "uploaded";
|
||||
type AssetFilter = "all" | "project" | "uploaded" | "canvas";
|
||||
|
||||
const FILTER_TABS: { id: AssetFilter; label: string }[] = [
|
||||
{ id: "all", label: "全部" },
|
||||
{ id: "project", label: "项目素材" },
|
||||
{ id: "uploaded", label: "本地上传" },
|
||||
{ id: "canvas", label: "画布素材" },
|
||||
];
|
||||
|
||||
/** 按 MIME 与扩展名推断媒体 kind；非媒体文件返回 null。 */
|
||||
@@ -201,15 +203,16 @@ function MediaPreview({ asset }: { asset: ProjectAsset }) {
|
||||
|
||||
function SourceBadge({ source }: { source: string }) {
|
||||
const uploaded = source === SOURCE_UPLOADED;
|
||||
const canvas = source === SOURCE_CANVAS;
|
||||
return (
|
||||
<span
|
||||
className={`rounded px-1 py-px text-[9px] leading-none ${
|
||||
uploaded
|
||||
uploaded || canvas
|
||||
? "bg-[var(--director-control-hover)] text-[var(--director-dock-fg-strong)]"
|
||||
: "bg-[var(--director-control-hover)] text-[var(--director-dock-fg)]/70"
|
||||
}`}
|
||||
>
|
||||
{uploaded ? "本地上传" : "项目素材"}
|
||||
{uploaded ? "本地上传" : canvas ? "画布素材" : "项目素材"}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
@@ -230,7 +233,7 @@ export function EditorAssetIngest() {
|
||||
|
||||
const [filter, setFilter] = useState<AssetFilter>("all");
|
||||
const [expandedId, setExpandedId] = useState<string | null>(null);
|
||||
const [openGroups, setOpenGroups] = useState<{ uploaded: boolean; project: boolean }>({ uploaded: true, project: true });
|
||||
const [openGroups, setOpenGroups] = useState<{ uploaded: boolean; project: boolean; canvas: boolean }>({ uploaded: true, project: true, canvas: true });
|
||||
const inputRef = useRef<HTMLInputElement | null>(null);
|
||||
const lastAddKey = useRef<string | null>(null);
|
||||
const lastAddAt = useRef(0);
|
||||
@@ -264,6 +267,7 @@ export function EditorAssetIngest() {
|
||||
});
|
||||
const skipped = media.length - toImport.length;
|
||||
// 逐文件导入：单文件失败不中断整批，汇总失败数提示。
|
||||
const linkedIds: string[] = [];
|
||||
for (const { file, kind } of toImport) {
|
||||
try {
|
||||
// 上传前探测真实时长（视频/音频），随 meta 入库供时间线片段使用。
|
||||
@@ -291,6 +295,7 @@ export function EditorAssetIngest() {
|
||||
continue;
|
||||
}
|
||||
okCount += 1;
|
||||
linkedIds.push(resource.id);
|
||||
} catch (err) {
|
||||
failedNames.push(file.name);
|
||||
const detail = extractApiMessage(err);
|
||||
@@ -298,7 +303,12 @@ export function EditorAssetIngest() {
|
||||
}
|
||||
}
|
||||
if (okCount > 0) {
|
||||
await refreshAssets();
|
||||
// 刷新后校验本次挂载的素材是否都出现在列表里；若单次刷新因网络抖动
|
||||
// 或后端提交延迟而拿不到最新结果，立即再刷新一次，避免列表停留在旧快照。
|
||||
const firstList = await refreshAssets();
|
||||
if (firstList && linkedIds.some((id) => !firstList.some((asset) => asset.id === id))) {
|
||||
await refreshAssets();
|
||||
}
|
||||
setImportNote(skipped > 0 ? `已导入 ${okCount} 个，跳过 ${skipped} 个重复文件` : `已导入 ${okCount} 个媒体`);
|
||||
} else if (skipped > 0 && failedNames.length === 0) {
|
||||
setImportNote(`媒体库中已有同名素材，跳过 ${skipped} 个重复文件`);
|
||||
@@ -336,9 +346,11 @@ export function EditorAssetIngest() {
|
||||
};
|
||||
|
||||
const uploadedAssets = assets.filter((a) => a.source === SOURCE_UPLOADED);
|
||||
const projectAssets = assets.filter((a) => a.source !== SOURCE_UPLOADED);
|
||||
const groups: { id: "uploaded" | "project"; label: string; icon: typeof HardDrive; items: ProjectAsset[] }[] = [
|
||||
const canvasAssets = assets.filter((a) => a.source === SOURCE_CANVAS);
|
||||
const projectAssets = assets.filter((a) => a.source !== SOURCE_UPLOADED && a.source !== SOURCE_CANVAS);
|
||||
const groups: { id: "uploaded" | "project" | "canvas"; label: string; icon: typeof HardDrive; items: ProjectAsset[] }[] = [
|
||||
{ id: "uploaded", label: "本地上传", icon: HardDrive, items: uploadedAssets },
|
||||
{ id: "canvas", label: "画布素材", icon: Boxes, items: canvasAssets },
|
||||
{ id: "project", label: "项目素材", icon: Boxes, items: projectAssets },
|
||||
];
|
||||
const visibleGroups = groups.filter((g) => filter === "all" || g.id === filter);
|
||||
@@ -416,7 +428,7 @@ export function EditorAssetIngest() {
|
||||
) : visibleGroups.every((g) => g.items.length === 0) ? (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-2 p-4 text-center">
|
||||
<p className="text-xs text-[var(--director-dock-fg)]/60">
|
||||
{filter === "uploaded" ? "暂无本地上传的媒体" : filter === "project" ? "暂无项目素材" : "项目暂无资产"}
|
||||
{filter === "uploaded" ? "暂无本地上传的媒体" : filter === "canvas" ? "暂无画布素材，请先在画布工作台生成分镜/预演" : filter === "project" ? "暂无项目素材" : "项目暂无资产"}
|
||||
</p>
|
||||
<p className="max-w-[180px] text-[11px] leading-relaxed text-[var(--director-dock-fg)]/45">点击上方导入媒体，或将文件拖入此区域</p>
|
||||
</div>
|
||||
@@ -487,7 +499,7 @@ export function EditorAssetIngest() {
|
||||
<dt className="text-[var(--director-dock-fg)]/60">分类</dt>
|
||||
<dd className="truncate text-right text-[var(--director-dock-fg-strong)]">{asset.category}</dd>
|
||||
<dt className="text-[var(--director-dock-fg)]/60">来源</dt>
|
||||
<dd className="text-right text-[var(--director-dock-fg-strong)]">{source === SOURCE_UPLOADED ? "本地上传" : "项目素材"}</dd>
|
||||
<dd className="text-right text-[var(--director-dock-fg-strong)]">{source === SOURCE_UPLOADED ? "本地上传" : source === SOURCE_CANVAS ? "画布素材" : "项目素材"}</dd>
|
||||
{asset.durationMs ? (
|
||||
<>
|
||||
<dt className="text-[var(--director-dock-fg)]/60">时长</dt>
|
||||
|
||||
@@ -11,7 +11,25 @@
|
||||
// - 静音跟随所在音频/视频轨道（muted），不再硬编码 muted（否则"有声音"永远不成立）。
|
||||
|
||||
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react";
|
||||
import { ChevronDown, Film, Gauge, Pause, Play, SkipBack, SkipForward, StepBack, StepForward } from "lucide-react";
|
||||
import {
|
||||
AlertTriangle,
|
||||
Captions,
|
||||
ChevronDown,
|
||||
Download,
|
||||
Film,
|
||||
Gauge,
|
||||
Image as ImageIcon,
|
||||
LoaderCircle,
|
||||
Pause,
|
||||
Play,
|
||||
SkipBack,
|
||||
SkipForward,
|
||||
StepBack,
|
||||
StepForward,
|
||||
Type,
|
||||
Video,
|
||||
Volume2,
|
||||
} from "lucide-react";
|
||||
|
||||
import { useEditorStoreContext } from "@/components/editor/editor-context";
|
||||
import { formatTimelineTime } from "@/lib/timeline/timeline-view";
|
||||
@@ -67,6 +85,31 @@ function clipSourceTimeMs(clip: TimelineClip, transportMs: number): number {
|
||||
return Math.max(0, transportMs - clip.startMs + (clip.sourceStartMs ?? 0));
|
||||
}
|
||||
|
||||
function clipKindLabel(kind: TimelineClip["kind"]): string {
|
||||
switch (kind) {
|
||||
case "video":
|
||||
return "视频";
|
||||
case "image":
|
||||
return "图片";
|
||||
case "audio":
|
||||
return "音频";
|
||||
case "subtitle":
|
||||
return "字幕";
|
||||
case "text":
|
||||
return "文字";
|
||||
}
|
||||
}
|
||||
|
||||
function clipDisplayTitle(clip: TimelineClip): string {
|
||||
return clip.directMedia?.title || clip.title || clip.text || clip.nodeId || clip.id;
|
||||
}
|
||||
|
||||
function ClipKindIcon({ kind, className }: { kind: TimelineClip["kind"]; className?: string }) {
|
||||
const Icon =
|
||||
kind === "video" ? Video : kind === "image" ? ImageIcon : kind === "audio" ? Volume2 : kind === "subtitle" ? Captions : Type;
|
||||
return <Icon className={className} />;
|
||||
}
|
||||
|
||||
export function EditorPreviewMonitor() {
|
||||
const { project, transportMs, setTransportMs } = useEditorStoreContext();
|
||||
const durationMs = project?.durationMs ?? 0;
|
||||
@@ -95,11 +138,15 @@ export function EditorPreviewMonitor() {
|
||||
const [mediaTier, setMediaTier] = useState<"primary" | "variant">("primary");
|
||||
const [variantReadyTick, setVariantReadyTick] = useState(0);
|
||||
const [mediaErrorHint, setMediaErrorHint] = useState<string | null>(null);
|
||||
// 终态护栏：none/failed/超时/不可达后禁止再回退 primary 重试或覆盖终态文案。
|
||||
// 否则原件不可解码且后端无副本可生成时会 primary→variant→primary 无限循环。
|
||||
const mediaTerminalRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
setMediaTier("primary");
|
||||
setVariantReadyTick(0);
|
||||
setMediaErrorHint(null);
|
||||
mediaTerminalRef.current = false;
|
||||
}, [activeClip?.id, storageKey, activeMediaUrl]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -116,16 +163,19 @@ export function EditorPreviewMonitor() {
|
||||
setVariantReadyTick((n) => n + 1);
|
||||
window.clearInterval(timer);
|
||||
} else if (res.playbackStatus === "failed") {
|
||||
mediaTerminalRef.current = true;
|
||||
setMediaErrorHint("兼容副本生成失败，可下载原片后用本地播放器观看。");
|
||||
window.clearInterval(timer);
|
||||
} else if (res.playbackStatus === "none") {
|
||||
// 后端判定无需转码（远端存储/无 ffmpeg/编码不可处理）：无副本可等，
|
||||
// 停轮询回到原件原生播放，避免"正在生成"永挂。
|
||||
setMediaTier("primary");
|
||||
setMediaErrorHint(null);
|
||||
// 后端判定无播放副本可生成（远端存储/无 ffmpeg/编码不可处理）。
|
||||
// 原件此刻必然已 onError 失败才进入 variant 轮询，回退 primary 只会
|
||||
// 再次失败并切回 variant，形成无限循环 —— 直接进入终态提示并停轮询。
|
||||
mediaTerminalRef.current = true;
|
||||
setMediaErrorHint("视频编码此浏览器暂不支持，且无可生成的兼容副本；可下载原片转换格式后重新导入。");
|
||||
window.clearInterval(timer);
|
||||
} else if ((polls += 1) >= 120) {
|
||||
// processing 上限保护（约 5 分钟）：转码异常卡死时不再无限轮询。
|
||||
mediaTerminalRef.current = true;
|
||||
setMediaErrorHint("兼容版本生成超时，可下载原片后用本地播放器观看。");
|
||||
window.clearInterval(timer);
|
||||
} else {
|
||||
@@ -136,6 +186,7 @@ export function EditorPreviewMonitor() {
|
||||
if (cancelled) return;
|
||||
failures += 1;
|
||||
if (failures >= 4) {
|
||||
mediaTerminalRef.current = true;
|
||||
window.clearInterval(timer);
|
||||
setMediaErrorHint("兼容版本生成服务暂不可达，请稍后重新打开预览重试。");
|
||||
}
|
||||
@@ -148,9 +199,12 @@ export function EditorPreviewMonitor() {
|
||||
}, [mediaTier, mediaResourceId]);
|
||||
|
||||
const handleMediaError = () => {
|
||||
// 已进入终态（副本生成失败/无副本可生成/超时/服务不可达）：保持终态文案，
|
||||
// 不再改写为"正在生成"或回退重试 —— 副本 URL 在 failed/none 时会回退原件，
|
||||
// 反复 onError 会把失败提示覆盖成误导性的"正在生成兼容版本"。
|
||||
if (mediaTerminalRef.current) return;
|
||||
// 图片/无资源：无播放副本可切，仅提示。
|
||||
if (activeClip?.kind !== "video" || !mediaResourceId) {
|
||||
setMediaErrorHint("该媒体无法解码，可下载原片后用本地播放器观看。");
|
||||
return;
|
||||
}
|
||||
if (mediaTier === "primary") {
|
||||
@@ -351,6 +405,52 @@ export function EditorPreviewMonitor() {
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col overflow-hidden bg-[var(--director-sequencer-surface)]">
|
||||
{/* 顶部信息条：媒体标题/元数据常驻在画面外的标题栏，不再悬浮遮挡画面内容 */}
|
||||
<div className="flex h-9 shrink-0 items-center gap-2 border-b border-[var(--director-sequencer-border)] bg-[var(--director-sequencer-surface-raised)] px-3">
|
||||
<div className="flex min-w-0 flex-1 items-center gap-2">
|
||||
{activeClip ? (
|
||||
<>
|
||||
<ClipKindIcon kind={activeClip.kind} className="size-3.5 shrink-0 text-[var(--director-dock-fg)]/55" />
|
||||
<span className="truncate text-xs font-medium text-[var(--director-dock-fg-strong)]" title={clipDisplayTitle(activeClip)}>
|
||||
{clipDisplayTitle(activeClip)}
|
||||
</span>
|
||||
<span className="hidden shrink-0 rounded-full bg-[var(--director-dock-active-surface)] px-1.5 py-0.5 text-[10px] leading-none text-[var(--director-dock-fg)]/85 sm:inline-flex">
|
||||
{clipKindLabel(activeClip.kind)}
|
||||
</span>
|
||||
{activeTrack && (
|
||||
<span className="hidden shrink-0 rounded-full border border-[var(--director-sequencer-border)] px-1.5 py-0.5 text-[10px] leading-none text-[var(--director-dock-fg)]/55 md:inline-flex">
|
||||
{activeTrack.label}
|
||||
</span>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<span className="truncate text-xs text-[var(--director-dock-fg)]/40">预览监视器</span>
|
||||
)}
|
||||
</div>
|
||||
{mediaErrorHint && (
|
||||
<span
|
||||
title={mediaErrorHint}
|
||||
className="flex min-w-0 max-w-[42%] items-center gap-1.5 rounded-md bg-[var(--director-danger)]/10 px-2 py-1 text-[11px] text-[var(--director-danger)]"
|
||||
>
|
||||
{mediaErrorHint.includes("正在生成") ? (
|
||||
<LoaderCircle className="size-3 shrink-0 animate-spin" />
|
||||
) : (
|
||||
<AlertTriangle className="size-3 shrink-0" />
|
||||
)}
|
||||
<span className="truncate">{mediaErrorHint}</span>
|
||||
{mediaResourceId && (
|
||||
<a
|
||||
href={resourceFileUrl(mediaResourceId)}
|
||||
download
|
||||
title="下载原片，用本地播放器观看"
|
||||
className="grid size-4 shrink-0 place-items-center rounded hover:bg-[var(--director-danger)]/15"
|
||||
>
|
||||
<Download className="size-3" />
|
||||
</a>
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{/* 预览画面：恒黑舞台，占满控制条以上剩余空间（Concat 监视器风格） */}
|
||||
<div className="relative flex min-h-0 flex-1 items-center justify-center overflow-hidden bg-black p-6">
|
||||
{showEmpty ? (
|
||||
@@ -388,25 +488,6 @@ export function EditorPreviewMonitor() {
|
||||
onError={handleMediaError}
|
||||
/>
|
||||
) : null}
|
||||
{mediaErrorHint && (
|
||||
<div className="absolute inset-x-0 bottom-0 flex flex-col items-center gap-2 bg-black/70 px-4 py-3 backdrop-blur">
|
||||
<p className="text-xs leading-relaxed text-white/90">{mediaErrorHint}</p>
|
||||
{mediaResourceId && (
|
||||
<a
|
||||
href={resourceFileUrl(mediaResourceId)}
|
||||
download
|
||||
className="text-[11px] underline decoration-white/40 underline-offset-2 text-white/70 hover:text-white"
|
||||
>
|
||||
下载原片用本地播放器观看
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{activeClip && (
|
||||
<div className="absolute left-4 top-4 rounded-md bg-black/60 px-2 py-1 text-[11px] text-white/90 backdrop-blur">
|
||||
{activeClip.nodeId || activeClip.id} · {activeClip.kind}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 播放控制条：画面下方的独立一行（Concat 布局），不悬浮叠加、无进度条；
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
// 字幕工具（editor-shell 预设插件贡献 subtitle-tool 插槽，M3.5）。
|
||||
// SRT 导入导出（复用 srt-parser）+ 「从节点重建字幕片段」命令（§3.1 快照契约：
|
||||
// 节点 subtitleEntries 权威 → 重建时间线字幕快照，替换过期快照，单向显式同步）。
|
||||
// AI 高亮在 M6（editor-ai-assistant + ai.text 权限）接入，此处留占位。
|
||||
// AI 字幕高亮与关键词标注已在 M6 落地，入口在画布「字幕」弹窗
|
||||
//（canvas-subtitle-dialog.tsx，走 ai.text 权限门禁）：对权威节点字幕条目生成
|
||||
// 高亮/关键词，并随字幕条目渲染。此处（时间线字幕工具）字幕片段只是快照，
|
||||
// 编辑器预览尚无字幕文本渲染层，高亮视觉无处挂载，故不在此重复接 AI；
|
||||
// 待 M4 数据源接入、预览可渲染字幕文本后，再将高亮打通到时间线片段。
|
||||
|
||||
import { useMemo, useState } from "react";
|
||||
|
||||
@@ -117,10 +121,11 @@ export function EditorSubtitleTools() {
|
||||
<div className="rounded-md border border-[var(--director-sequencer-border)] bg-[var(--director-control-hover)] p-2.5">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-[var(--director-dock-fg-strong)]">AI 高亮</span>
|
||||
<span className="rounded-full bg-[var(--director-dock-active-surface)] px-1.5 py-0.5 text-[10px] text-[var(--director-dock-fg)]/80">M6</span>
|
||||
<span className="rounded-full bg-[var(--director-dock-active-surface)] px-1.5 py-0.5 text-[10px] text-[var(--director-dock-fg)]/80">可用</span>
|
||||
</div>
|
||||
<p className="mt-1.5 text-[11px] leading-relaxed text-[var(--director-dock-fg)]/55">
|
||||
AI 字幕高亮与关键词标注将在 M6（ai-assistant 插件 + ai.text 权限）接入。
|
||||
AI 字幕高亮与关键词标注已实现，入口在画布「字幕」弹窗（点击字幕 → AI 高亮，走 ai.text 权限）。
|
||||
时间线字幕片段由该字幕快照重建而来；待编辑器预览支持字幕文本渲染后，高亮会随快照同步到这里。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1097,8 +1097,13 @@ function AssetCard({
|
||||
);
|
||||
}
|
||||
|
||||
function isKnownAssetKind(kind: unknown): kind is AssetKind {
|
||||
return kind === "image" || kind === "video" || kind === "audio" || kind === "model" || kind === "text";
|
||||
}
|
||||
|
||||
function AssetCover({ asset, selected, isTrash = false, onSelect, onOpen, menuItems }: { asset: LibraryAsset; selected: boolean; isTrash?: boolean; onSelect: (selected: boolean) => void; onOpen: () => void; menuItems: MenuProps["items"] }) {
|
||||
const KindIcon = assetKindIcons[asset.kind];
|
||||
const kind = isKnownAssetKind(asset.kind) ? asset.kind : undefined;
|
||||
const KindIcon = kind ? assetKindIcons[kind] : FileText;
|
||||
const clock = asset.kind === "video" || asset.kind === "audio" ? formatAssetClock(asset.data.durationMs) : null;
|
||||
const showPlay = asset.kind === "video";
|
||||
const isLight = asset.kind === "audio" || asset.kind === "text" || asset.kind === "model";
|
||||
@@ -1133,7 +1138,7 @@ function AssetCover({ asset, selected, isTrash = false, onSelect, onOpen, menuIt
|
||||
<span className="assets-cover-badges">
|
||||
<span className="assets-cover-badge is-kind">
|
||||
<KindIcon />
|
||||
{assetKindLabel(asset.kind)}
|
||||
{kind ? assetKindLabel(kind) : "素材"}
|
||||
</span>
|
||||
{isTrash ? <span className="assets-cover-badge is-category !bg-amber-500/85 !text-white">回收站</span> : <span className="assets-cover-badge is-category">{assetCategoryLabel(asset.category)}</span>}
|
||||
</span>
|
||||
@@ -1320,7 +1325,8 @@ function AssetFilterGroup({
|
||||
|
||||
function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: LibraryAsset | null; onClose: () => void; onCopy: (asset: LibraryAsset) => void; onDownload: (asset: LibraryAsset) => void }) {
|
||||
const facts = asset ? assetArchiveFacts(asset) : [];
|
||||
const KindIcon = asset ? assetKindIcons[asset.kind] : Clapperboard;
|
||||
const kind = asset && isKnownAssetKind(asset.kind) ? asset.kind : undefined;
|
||||
const KindIcon = asset ? (kind ? assetKindIcons[kind] : FileText) : Clapperboard;
|
||||
return (
|
||||
<Drawer className="library-drawer" title="素材档案" open={Boolean(asset)} size="large" onClose={onClose}>
|
||||
{asset ? (
|
||||
|
||||
@@ -15,8 +15,8 @@ import {
|
||||
X,
|
||||
} from "lucide-react";
|
||||
|
||||
import type { ProjectAsset, ProjectDetail } from "@/services/api/projects";
|
||||
import { listProjectAssets } from "@/services/api/projects";
|
||||
import type { ProjectAsset, ProjectDetail, ShotArtifact } from "@/services/api/projects";
|
||||
import { linkProjectAsset, listProjectAssets } from "@/services/api/projects";
|
||||
import { useEditorSlots, type EditorSlotRegistration } from "@/lib/plugins/editor-slot-registry";
|
||||
import { pluginMayRenderEditorSlot } from "@/lib/plugins/plugin-permission-check";
|
||||
import { EditorStoreProvider } from "@/components/editor/editor-context";
|
||||
@@ -269,11 +269,14 @@ export default function ProjectEditorView({ detail }: { detail: ProjectDetail })
|
||||
const [timelineH, setTimelineH] = useState(240);
|
||||
const splitterRef = useRef<{ startY: number; startH: number } | null>(null);
|
||||
|
||||
const refreshAssets = async () => {
|
||||
const refreshAssets = async (): Promise<ProjectAsset[] | null> => {
|
||||
try {
|
||||
setAssets((await listProjectAssets(projectId)).assets);
|
||||
const list = (await listProjectAssets(projectId)).assets;
|
||||
setAssets(list);
|
||||
return list;
|
||||
} catch {
|
||||
// 刷新失败保留现有列表；导入面板内联提示错误
|
||||
// 刷新失败保留现有列表；调用方可据此决定是否重试
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -294,6 +297,19 @@ export default function ProjectEditorView({ detail }: { detail: ProjectDetail })
|
||||
const scope = getActiveUserScope();
|
||||
const projectId = detail.project.id;
|
||||
|
||||
// 挂载/切换项目时以服务端为准同步一次资产：detail.assets 只是详情页快照，
|
||||
// 上个会话导入但详情未携带（或导入后未刷新）的素材会在进入编辑器后补现。
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
const list = await refreshAssets();
|
||||
if (!cancelled && list) setAssets(list);
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [projectId]);
|
||||
|
||||
const store = useState(() =>
|
||||
createEditorStore({
|
||||
saveTimeline: async (project: TimelineProject) => {
|
||||
@@ -318,6 +334,58 @@ export default function ProjectEditorView({ detail }: { detail: ProjectDetail })
|
||||
};
|
||||
}, [scope, projectId, store]);
|
||||
|
||||
// 画布产物自动同步：detail.shotArtifacts 中当前采用(storyboard/action_board/
|
||||
// 分镜/导出成片，且 ready + 有资源)尚未作为素材进本项目时，自动
|
||||
// linkProjectAsset(source=canvas) 并入素材库 —— 剪辑器“素材选项”直接可用画布产物。
|
||||
// 后端幂等(已链接直接返回)，失败静默，下次进入编辑器自动补同步；
|
||||
// 已完成集合记录在 ref，避免同一会话重复请求。
|
||||
const syncedCanvasResourcesRef = useRef<Set<string>>(new Set());
|
||||
const canvasSyncKeys = detail.shotArtifacts
|
||||
.filter(
|
||||
(a): a is ShotArtifact & { resourceId: string } =>
|
||||
a.selected &&
|
||||
a.status === "ready" &&
|
||||
!!a.resourceId &&
|
||||
["storyboard", "action_board", "start_frame", "end_frame", "video", "delivery"].includes(a.type),
|
||||
)
|
||||
.map((a) => `resource:${a.resourceId}`)
|
||||
.join(",");
|
||||
useEffect(() => {
|
||||
if (!canvasSyncKeys) return;
|
||||
const keys = canvasSyncKeys.split(",");
|
||||
// 已同步过的(本会话 ref)或已存在于项目素材的(storageKey 命中)均跳过 ——
|
||||
// storageKey 与 assetFromUploadedResource 合成格式一致(resource:<id>)。
|
||||
const existing = new Set(assets.map((a) => a.storageKey));
|
||||
const pending = keys.filter((key) => !syncedCanvasResourcesRef.current.has(key) && !existing.has(key));
|
||||
if (!pending.length) return;
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
for (const key of pending) {
|
||||
if (cancelled) break;
|
||||
const resourceId = key.slice("resource:".length);
|
||||
try {
|
||||
await linkProjectAsset(projectId, { assetId: resourceId, category: "material", source: "canvas" });
|
||||
syncedCanvasResourcesRef.current.add(key);
|
||||
} catch {
|
||||
// 单个同步失败不阻塞其余；留待下次进入编辑器重试
|
||||
}
|
||||
}
|
||||
if (!cancelled) {
|
||||
// 直接拉取最新列表(setAssets 稳定)而非经 refreshAssets，
|
||||
// 避免闭包依赖变化导致 effect 反复触发
|
||||
try {
|
||||
const list = (await listProjectAssets(projectId)).assets;
|
||||
if (!cancelled) setAssets(list);
|
||||
} catch {
|
||||
// 刷新失败保留现有列表，下次进入编辑器再同步
|
||||
}
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [canvasSyncKeys, projectId, assets]);
|
||||
|
||||
return (
|
||||
<EditorStoreProvider store={store} host={{ projectId, assets, refreshAssets }}>
|
||||
<div ref={workbenchRef} className="editor-workbench relative flex h-full min-h-0 flex-col bg-[var(--director-workspace-bg)]">
|
||||
|
||||
@@ -32,7 +32,7 @@ export default function ProjectOverviewView({ detail, overview }: ProjectDetailV
|
||||
{ id: "storyboard", icon: Clapperboard, label: "分镜脚本与画面", description: "拆分镜头并确认构图、对白和时长", metric: `${metrics.shotCount} 镜 · ${metrics.readyStoryboardCount} 张图`, href: workflowHref("storyboard"), complete: metrics.shotCount > 0 },
|
||||
{ id: "previz", icon: PlaySquare, label: "动作预演", description: "检查表演节拍、运镜和连续性", metric: `${metrics.readyPrevizCount}/${metrics.shotCount || 0} 镜`, href: workflowHref("previz"), complete: metrics.shotCount > 0 && metrics.readyPrevizCount === metrics.shotCount },
|
||||
{ id: "video", icon: Film, label: "镜头视频", description: "逐镜生成、筛选版本并锁定成片", metric: `${metrics.readyVideoCount}/${metrics.shotCount || 0} 镜`, href: workflowHref("video"), complete: metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount },
|
||||
{ id: "editor", icon: Scissors, label: "剪辑成片", description: "在时间线中编排镜头、添加字幕并输出成片", metric: metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount ? "可以开始剪辑" : `还差 ${Math.max(0, metrics.shotCount - metrics.readyVideoCount)} 镜`, href: `/projects/${project.id}/editor`, complete: false },
|
||||
{ id: "editor", icon: Scissors, label: "剪辑成片", description: "在时间线中编排镜头、添加字幕并输出成片", metric: metrics.renderSucceededCount > 0 ? `已输出 ${metrics.renderSucceededCount} 个成片` : metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount ? "可以开始剪辑" : `还差 ${Math.max(0, metrics.shotCount - metrics.readyVideoCount)} 镜`, href: `/projects/${project.id}/editor`, complete: metrics.renderSucceededCount > 0 },
|
||||
{ id: "delivery", icon: PackageCheck, label: "交付与打包", description: "检查缺失镜头并整理最终产物", metric: metrics.readyVideoCount && metrics.readyVideoCount === metrics.shotCount ? "可以交付" : `还差 ${Math.max(0, metrics.shotCount - metrics.readyVideoCount)} 镜`, href: workflowHref("delivery"), complete: metrics.shotCount > 0 && metrics.readyVideoCount === metrics.shotCount },
|
||||
];
|
||||
const gaps = [
|
||||
|
||||
@@ -278,6 +278,7 @@ export type ProjectOverviewMetrics = {
|
||||
readyStoryboardCount: number;
|
||||
readyPrevizCount: number;
|
||||
readyVideoCount: number;
|
||||
renderSucceededCount: number;
|
||||
staleArtifactCount: number;
|
||||
};
|
||||
|
||||
@@ -479,7 +480,7 @@ export function unlinkCanvasProject(projectId: string, canvasId: string) {
|
||||
return request<{ canvasId: string }>(api.delete(`/projects/${encodeURIComponent(projectId)}/canvases/${encodeURIComponent(canvasId)}`));
|
||||
}
|
||||
|
||||
export function linkProjectAsset(projectId: string, input: { assetId: string; category: AssetCategory; folderId?: string; title?: string }, signal?: AbortSignal) {
|
||||
export function linkProjectAsset(projectId: string, input: { assetId: string; category: AssetCategory; folderId?: string; title?: string; source?: "uploaded" | "canvas" }, signal?: AbortSignal) {
|
||||
return request<{ asset: ProjectAsset }>(api.post(`/projects/${encodeURIComponent(projectId)}/assets`, input, { signal }));
|
||||
}
|
||||
|
||||
|
||||
@@ -532,6 +532,7 @@ export async function waitForGenerationTask(id: string, options?: WaitForGenerat
|
||||
const intervalMs = options?.intervalMs || 2000;
|
||||
let lastTask = options?.initialTask;
|
||||
let lastQueryError: unknown;
|
||||
let consecutiveFailures = 0;
|
||||
try {
|
||||
while (Date.now() - startedAt < (options?.timeoutMs || taskWaitTimeoutMs(lastTask))) {
|
||||
if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError");
|
||||
@@ -540,9 +541,16 @@ export async function waitForGenerationTask(id: string, options?: WaitForGenerat
|
||||
task = await queryGenerationTask(id, { signal: options?.signal });
|
||||
lastTask = task;
|
||||
lastQueryError = undefined;
|
||||
consecutiveFailures = 0;
|
||||
options?.onTaskUpdate?.(task);
|
||||
} catch (error) {
|
||||
lastQueryError = error;
|
||||
consecutiveFailures += 1;
|
||||
// 连续失败说明查询通道已不可用，继续轮询只会空转到整体超时；
|
||||
// 保留少量容忍度（约 10 秒）以跳过瞬时抖动后直接报错，便于用户尽早处理。
|
||||
if (consecutiveFailures >= 5) {
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
await delay(intervalMs, options?.signal);
|
||||
continue;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user