fix(canvas): 短剧分镜 - 支持多镜头并行生成并正确引用角色图声资产
This commit is contained in:
ddCat
@@ -1772,6 +1772,14 @@ func (r *Repository) ShotForProject(projectID string, shotID string) (*model.Sho
|
||||
return &shot, nil
|
||||
}
|
||||
|
||||
func (r *Repository) ShotRevisionForShot(shotID string, revisionID string) (*model.ShotRevision, error) {
|
||||
var revision model.ShotRevision
|
||||
if err := r.db.First(&revision, "id = ? AND shot_id = ?", revisionID, shotID).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &revision, nil
|
||||
}
|
||||
|
||||
// DeleteProjectShot 原子删除单个镜头的领域关联，并重新压紧同章节镜头顺序。
|
||||
func (r *Repository) DeleteProjectShot(projectID string, shotID string, updatedAt time.Time) error {
|
||||
return r.db.Transaction(func(tx *gorm.DB) error {
|
||||
|
||||
@@ -44,6 +44,7 @@ type RegisterTaskOutputRequest struct {
|
||||
CanvasID string `json:"canvasId"`
|
||||
UnitID string `json:"unitId"`
|
||||
ShotID string `json:"shotId"`
|
||||
ShotRevisionID string `json:"shotRevisionId"`
|
||||
ArtifactType string `json:"artifactType"`
|
||||
AssetVersionID string `json:"assetVersionId"`
|
||||
ResourceID string `json:"resourceId"`
|
||||
@@ -240,6 +241,16 @@ func (s *Service) RegisterTaskOutput(userID string, projectID string, stepID str
|
||||
return model.WorkflowStepInstance{}, err
|
||||
}
|
||||
}
|
||||
shotRevisionID := strings.TrimSpace(req.ShotRevisionID)
|
||||
if shot != nil {
|
||||
if shotRevisionID == "" {
|
||||
shotRevisionID = shot.CurrentRevisionID
|
||||
} else if _, revisionErr := s.repo.ShotRevisionForShot(shot.ID, shotRevisionID); revisionErr != nil {
|
||||
return model.WorkflowStepInstance{}, revisionErr
|
||||
}
|
||||
} else if shotRevisionID != "" {
|
||||
return model.WorkflowStepInstance{}, BadAuthRequest("镜头版本缺少所属镜头")
|
||||
}
|
||||
if versionID := strings.TrimSpace(req.AssetVersionID); versionID != "" {
|
||||
if _, err := s.repo.AssetVersionForProject(projectID, versionID); err != nil {
|
||||
return model.WorkflowStepInstance{}, err
|
||||
@@ -305,7 +316,7 @@ func (s *Service) RegisterTaskOutput(userID string, projectID string, stepID str
|
||||
productionLink := &model.ProductionTaskLink{ID: newID(), TaskID: task.ID, ProjectID: projectID, CanvasID: canvasID, UnitID: unitID, ShotID: shotID, WorkflowStepID: step.ID, ArtifactType: artifactType, CreatedAt: now, UpdatedAt: now}
|
||||
var artifact *model.ShotArtifact
|
||||
if shot != nil && strings.TrimSpace(req.ResourceID) != "" && artifactType != "" {
|
||||
artifact = &model.ShotArtifact{ID: newID(), ProjectID: projectID, UnitID: shot.UnitID, ShotID: shot.ID, RevisionID: shot.CurrentRevisionID, TaskID: task.ID, Type: artifactType, ResourceID: strings.TrimSpace(req.ResourceID), Status: "ready", Selected: true, MetadataJSON: metadata, CreatedAt: now, UpdatedAt: now}
|
||||
artifact = &model.ShotArtifact{ID: newID(), ProjectID: projectID, UnitID: shot.UnitID, ShotID: shot.ID, RevisionID: shotRevisionID, TaskID: task.ID, Type: artifactType, ResourceID: strings.TrimSpace(req.ResourceID), Status: "ready", Selected: true, MetadataJSON: metadata, CreatedAt: now, UpdatedAt: now}
|
||||
}
|
||||
// 单镜产物成功只代表该镜头完成，不能提前放行整个章节阶段。
|
||||
if shot != nil {
|
||||
@@ -421,11 +432,13 @@ func (s *Service) RegisterTaskOutputFromTask(task model.Task) error {
|
||||
CanvasID string `json:"canvasId"`
|
||||
UnitID string `json:"unitId"`
|
||||
ShotID string `json:"shotId"`
|
||||
ShotRevisionID string `json:"shotRevisionId"`
|
||||
ArtifactType string `json:"artifactType"`
|
||||
AssetVersionID string `json:"assetVersionId"`
|
||||
ResourceID string `json:"resourceId"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Role string `json:"role"`
|
||||
MetadataJSON string `json:"metadataJson"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(decrypted), &input); err != nil {
|
||||
@@ -450,6 +463,9 @@ func (s *Service) RegisterTaskOutputFromTask(task model.Task) error {
|
||||
if input.ShotID == "" {
|
||||
input.ShotID, _ = input.Metadata["shotId"].(string)
|
||||
}
|
||||
if input.ShotRevisionID == "" {
|
||||
input.ShotRevisionID, _ = input.Metadata["shotRevisionId"].(string)
|
||||
}
|
||||
if input.ArtifactType == "" {
|
||||
input.ArtifactType, _ = input.Metadata["artifactType"].(string)
|
||||
}
|
||||
@@ -462,6 +478,13 @@ func (s *Service) RegisterTaskOutputFromTask(task model.Task) error {
|
||||
if input.Role == "" {
|
||||
input.Role, _ = input.Metadata["role"].(string)
|
||||
}
|
||||
if input.MetadataJSON == "" {
|
||||
if artifactMetadata, ok := input.Metadata["artifactMetadata"]; ok {
|
||||
if encoded, encodeErr := json.Marshal(artifactMetadata); encodeErr == nil {
|
||||
input.MetadataJSON = string(encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(input.ResourceID) == "" {
|
||||
input.ResourceID, input.MediaType = taskOutputResource(task.ResultJSON, task.Type)
|
||||
@@ -488,7 +511,7 @@ func (s *Service) RegisterTaskOutputFromTask(task model.Task) error {
|
||||
}
|
||||
input.AssetVersionID = assetVersionID
|
||||
}
|
||||
_, err = s.RegisterTaskOutput(task.UserID, projectID, input.WorkflowStepID, RegisterTaskOutputRequest{TaskID: task.ID, CanvasID: input.CanvasID, UnitID: input.UnitID, ShotID: input.ShotID, ArtifactType: input.ArtifactType, AssetVersionID: input.AssetVersionID, ResourceID: input.ResourceID, MediaType: input.MediaType, Role: input.Role, OutputJSON: task.ResultJSON})
|
||||
_, err = s.RegisterTaskOutput(task.UserID, projectID, input.WorkflowStepID, RegisterTaskOutputRequest{TaskID: task.ID, CanvasID: input.CanvasID, UnitID: input.UnitID, ShotID: input.ShotID, ShotRevisionID: input.ShotRevisionID, ArtifactType: input.ArtifactType, AssetVersionID: input.AssetVersionID, ResourceID: input.ResourceID, MediaType: input.MediaType, Role: input.Role, MetadataJSON: input.MetadataJSON, OutputJSON: task.ResultJSON})
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -50,13 +50,17 @@ func TestRegisterTaskOutputFromTaskPersistsMediaAssetAndArtifactIdempotently(t *
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
submittedRevisionID := shot.CurrentRevisionID
|
||||
if _, _, err = service.CreateShotRevision("user-1", project.ID, shot.ID, ShotRevisionInput{PlotDescription: "任务提交后修改的镜头", DurationMs: 3000}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
resource := model.Resource{ID: "resource-video-1", UserID: "user-1", Kind: "video", Status: model.ResourceStatusReady, MimeType: "video/mp4", Size: 1024, CreatedAt: now, UpdatedAt: now}
|
||||
if err := db.Create(&resource).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := model.Task{ID: "workflow-video-task-1", UserID: "user-1", ProjectID: project.ID, Type: "canvas_video", Status: model.TaskStatusSucceeded,
|
||||
InputJSON: `{"metadata":{"workflowStepId":"` + videoStep.ID + `","domainProjectId":"` + project.ID + `","unitId":"` + unit.ID + `","shotId":"` + shot.ID + `","artifactType":"video","role":"output"}}`,
|
||||
InputJSON: `{"metadata":{"workflowStepId":"` + videoStep.ID + `","domainProjectId":"` + project.ID + `","unitId":"` + unit.ID + `","shotId":"` + shot.ID + `","shotRevisionId":"` + submittedRevisionID + `","artifactType":"video","role":"output","artifactMetadata":{"model":"MiniMax-H3"}}}`,
|
||||
ResultJSON: `{"mode":"video","video":{"resourceId":"resource-video-1","storageKey":"resource:resource-video-1","mimeType":"video/mp4"}}`, CreatedAt: now, UpdatedAt: now}
|
||||
if err := db.Create(&task).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -78,6 +82,16 @@ func TestRegisterTaskOutputFromTaskPersistsMediaAssetAndArtifactIdempotently(t *
|
||||
t.Fatalf("%s count = %d, error = %v", table, count, err)
|
||||
}
|
||||
}
|
||||
var artifact model.ShotArtifact
|
||||
if err := db.First(&artifact, "task_id = ?", task.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if artifact.RevisionID != submittedRevisionID {
|
||||
t.Fatalf("artifact revision = %q, want submitted revision %q", artifact.RevisionID, submittedRevisionID)
|
||||
}
|
||||
if artifact.MetadataJSON != `{"model":"MiniMax-H3"}` {
|
||||
t.Fatalf("artifact metadata = %q", artifact.MetadataJSON)
|
||||
}
|
||||
}
|
||||
|
||||
func seedWorkflowProject(t *testing.T, db *gorm.DB) (model.Project, model.ProjectUnit) {
|
||||
|
||||
@@ -2858,18 +2858,25 @@ func runMiniMaxVideoTask(ctx context.Context, input canvasGenerationInput) (map[
|
||||
if len(input.ReferenceImages) > 9 || len(input.ReferenceVideos) > 3 || len(input.ReferenceAudios) > 3 {
|
||||
return nil, errors.New("MiniMax 视频最多支持 9 张参考图、3 个参考视频和 3 个参考音频")
|
||||
}
|
||||
operation := metadataString(input.Metadata, "videoEditOperation")
|
||||
startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId")
|
||||
endFrameID := metadataString(input.Metadata, "videoEndFrameNodeId")
|
||||
frameMode := operation != "reference_to_video" && (startFrameID != "" || endFrameID != "")
|
||||
content := []miniMaxVideoContent{{Type: "text", Text: strings.TrimSpace(input.Prompt)}}
|
||||
for index, image := range input.ReferenceImages {
|
||||
for _, image := range input.ReferenceImages {
|
||||
url, err := miniMaxMediaURLValue(image)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MiniMax 参考图无效：%w", err)
|
||||
}
|
||||
role := "reference_image"
|
||||
if len(input.ReferenceVideos) == 0 && len(input.ReferenceAudios) == 0 && len(input.ReferenceImages) <= 2 {
|
||||
if index == 0 {
|
||||
if frameMode {
|
||||
switch {
|
||||
case startFrameID != "" && image.ID == startFrameID:
|
||||
role = "first_frame"
|
||||
} else if index == 1 {
|
||||
case endFrameID != "" && image.ID == endFrameID:
|
||||
role = "last_frame"
|
||||
default:
|
||||
return nil, errors.New("MiniMax 首尾帧模式不能混合未标记的参考图")
|
||||
}
|
||||
}
|
||||
content = append(content, miniMaxVideoContent{Type: "image_url", ImageURL: &miniMaxMediaURL{URL: url}, Role: role})
|
||||
@@ -2889,7 +2896,6 @@ func runMiniMaxVideoTask(ctx context.Context, input canvasGenerationInput) (map[
|
||||
content = append(content, miniMaxVideoContent{Type: "audio_url", AudioURL: &miniMaxMediaURL{URL: url}, Role: "reference_audio"})
|
||||
}
|
||||
watermark := parseBool(input.Config.VideoWatermark, false)
|
||||
frameMode := len(input.ReferenceImages) > 0 && len(input.ReferenceImages) <= 2 && len(input.ReferenceVideos) == 0 && len(input.ReferenceAudios) == 0
|
||||
body := miniMaxVideoRequest{
|
||||
Model: input.Config.Model,
|
||||
Content: content,
|
||||
|
||||
@@ -2510,6 +2510,49 @@ func TestRunMiniMaxVideoTaskCreatesPollsAndDownloads(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMiniMaxVideoTaskUsesExplicitReferenceRoles(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
var server *httptest.Server
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method + " " + r.URL.Path {
|
||||
case "POST /v2/video_generation":
|
||||
var body miniMaxVideoRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
if len(body.Content) != 3 || body.Content[1].Role != "reference_image" || body.Content[2].Role != "reference_audio" {
|
||||
t.Errorf("content = %#v", body.Content)
|
||||
}
|
||||
if body.Ratio != "16:9" {
|
||||
t.Errorf("ratio = %q, want 16:9 for reference mode", body.Ratio)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"task_id":"minimax-reference-task"}`))
|
||||
case "GET /v2/query/video_generation/minimax-reference-task":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"task":{"status":"succeeded","content":{"url":"` + server.URL + `/video.mp4"}}}`))
|
||||
case "GET /video.mp4":
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("video"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := runVideoTask(context.Background(), canvasGenerationInput{
|
||||
Mode: "video",
|
||||
Prompt: "保持角色一致",
|
||||
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", Model: "MiniMax-H3", InterfaceType: "minimax-video", VideoSeconds: "6", VQuality: "768P", Size: "16:9"},
|
||||
ReferenceImages: []providerMedia{{ID: "character-1", URL: server.URL + "/character.png"}},
|
||||
ReferenceAudios: []providerMedia{{ID: "voice-1", URL: server.URL + "/voice.mp3"}},
|
||||
Metadata: map[string]interface{}{"videoEditOperation": "reference_to_video"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMiniMaxVideoTaskReturnsFailureReason(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -41,6 +41,7 @@ description: 影策已实现功能的手工验证清单与未确认边界
|
||||
- [x] 项目工作台新增独立分镜工作流路由，支持初始化章节流程、新增镜头、保存分镜版本、绑定资产版本、查看预演/视频版本和交付缺失状态；前端生产构建通过。
|
||||
- [x] 镜头轨道按当前制作阶段显示待生成、生成中、已生成和生成失败状态，状态标签固定在卡片右下角；横向镜头只在轨道内部滚动，不再撑宽脚本区或挤出产物预览。制作概览六个流程节点改为独立可点击卡片；前端类型检查、生产构建与相关专项测试通过。
|
||||
- [x] 分镜生成设置接入统一模型选择、能力和计价链路，支持前台模型、系统模型与用户自定义模型；切换模型会归一画幅、分辨率/画质和时长，MiniMax-H3 的旧 `720` 会切换为 `768P`。项目设置可保存默认生图/视频模型；后端全量测试、MiniMax 回归测试和前端生产构建通过。
|
||||
- [x] 单镜生成改为任务提交后立即返回，不同镜头可并行进入排队/生成状态；角色卡主图、绑定声音样本和场景资产会按 `图片N` / `音频N` 编译进提示词。MiniMax 项目资产使用 `reference_image` / `reference_audio`，仅显式首尾帧节点使用帧角色；产物固定绑定任务提交时的镜头版本。前后端全量测试与生产构建通过。
|
||||
- [ ] 在真实项目中初始化一个已有章节，确认不会误用旧 v1 工作流，刷新和切换章节后阶段、镜头和当前版本保持一致。
|
||||
- [ ] 在同一章节准备待生成、生成中、已生成和生成失败镜头，确认灰、蓝、绿、红标签始终显示在卡片右下角；增加到几十个镜头后确认只有底部轨道横向滚动，中间脚本和右侧产物预览仍完整可见。
|
||||
- [ ] 在浅色、深色、常规桌面和窄屏下检查制作概览六张流程卡片的边界、间距、完成态、悬停和键盘焦点，确认跳转目标正确。
|
||||
@@ -48,6 +49,8 @@ description: 影策已实现功能的手工验证清单与未确认边界
|
||||
- [ ] 使用画布任务和项目任务分别登记镜头产物，确认 `production_task_links` 正确记录 Project、Canvas、Unit、Shot 和 WorkflowStep，且账号归属校验有效。
|
||||
- [ ] 在浅色、深色、窄屏和常规桌面验证阶段导航、镜头列表、表单、检查器、媒体预览和长章节内容滚动。
|
||||
- [ ] 在项目设置分别选择系统、用户自定义的默认生图/视频模型，刷新后进入分镜图、动作预演和镜头视频，确认项目默认优先、已下架模型回退全局默认；切换 Seedance、MiniMax-H3 等能力不同的模型时，画幅、时长、分辨率和本次预计积分立即更新，并与实际账单一致。
|
||||
- [ ] 连续选择两个镜头并分别提交视频生成，确认两个镜头独立显示“正在提交/生成中”，第二个任务不会被第一个全局锁阻止；刷新页面后两个任务状态都能恢复，完成产物分别绑定各自提交时的镜头版本。
|
||||
- [ ] 给镜头绑定带主图和声音样本的角色卡及场景资产，使用 MiniMax-H3 生成并检查请求日志：文本包含 `【资产参考】`、角色声音说明和台词，角色/场景图片为 `reference_image`、声音为 `reference_audio`，请求不再出现误判的 `first_frame`，画幅和 `768P` / `2K` 与界面选择一致。
|
||||
- [ ] 当前“资产拆分 AI、多宫格生成提交、批量视频任务、后端交付包”仍是后续阶段，界面不得显示为已完成或可用。
|
||||
|
||||
## 角色声音素材绑定
|
||||
|
||||
@@ -11,7 +11,7 @@ import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, vid
|
||||
import { modelQuoteRequest } from "@/lib/model-pricing";
|
||||
import { modelCompatibilityError, resolveCompatibleModel, type ModelRequirements } from "@/lib/model-selection";
|
||||
import { formatVideoResolutionLabel } from "@/lib/video-generation-options";
|
||||
import { runBackendGenerationTask } from "@/services/api/generation-task";
|
||||
import { submitBackendGenerationTask } from "@/services/api/generation-task";
|
||||
import { quoteLogicalModel } from "@/services/api/logical-models";
|
||||
import { type GenerationTask } from "@/services/api/task-center";
|
||||
import {
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
deleteProjectShot,
|
||||
linkShotAsset,
|
||||
listProjectAssetsPage,
|
||||
registerProjectTaskOutput,
|
||||
saveProjectShot,
|
||||
unlinkShotAsset,
|
||||
type ProjectAsset,
|
||||
@@ -85,7 +84,10 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
const [previewTab, setPreviewTab] = useState<"latest" | "history">("latest");
|
||||
const [previewArtifactId, setPreviewArtifactId] = useState("");
|
||||
const [editorDirty, setEditorDirty] = useState(false);
|
||||
const [submittingShotIds, setSubmittingShotIds] = useState<Set<string>>(() => new Set());
|
||||
const [taskClock, setTaskClock] = useState(() => Date.now());
|
||||
const activeShotIdRef = useRef(selectedShot?.id || "");
|
||||
activeShotIdRef.current = selectedShot?.id || "";
|
||||
const shots = useMemo(() => (detail.shots || []).filter((item) => item.unitId === unitId).slice().sort((left, right) => left.position - right.position), [detail.shots, unitId]);
|
||||
const shotIndex = selectedShot ? shots.findIndex((item) => item.id === selectedShot.id) : -1;
|
||||
const revision = currentRevision(detail, selectedShot);
|
||||
@@ -119,15 +121,19 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
const referenceByVersionId = useMemo(() => new Map((detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked").map((reference) => [reference.assetVersionId, reference])), [detail.shotReferences, selectedShot?.id]);
|
||||
const currentDurationSeconds = Number(watchedDuration || Math.max(0.5, (selectedShot?.durationMs || 3000) / 1000));
|
||||
const generationSeconds = String(Math.max(1, Math.round(currentDurationSeconds)));
|
||||
const selectedVideoProfile = generationCapability === "video" && selectedModel ? modelCapabilityConfigFor(effectiveConfig, selectedModel).video : undefined;
|
||||
const generationReferenceAudios = generationCapability === "video" && (selectedVideoProfile?.references.maxAudios || 0) > 0 ? shotAssetReferenceContext.referenceAudios : [];
|
||||
const videoEditOperation = generationCapability === "video" && shotAssetReferenceContext.referenceImages.length ? "reference_to_video" : undefined;
|
||||
const modelRequirements = useMemo<ModelRequirements>(() => ({
|
||||
capability: generationCapability,
|
||||
input: { textCount: 1, imageCount: shotAssetReferenceContext.referenceImages.length, videoCount: 0, audioCount: 0, characterCount: 0 },
|
||||
input: { textCount: 1, imageCount: shotAssetReferenceContext.referenceImages.length, videoCount: 0, audioCount: generationReferenceAudios.length, characterCount: 0 },
|
||||
videoOperation: videoEditOperation,
|
||||
videoSeconds: generationCapability === "video" ? generationSeconds : undefined,
|
||||
imageSize: generationCapability === "image" ? aspectRatio : undefined,
|
||||
options: generationCapability === "video"
|
||||
? { size: aspectRatio, vquality: resolution, videoSeconds: Number(generationSeconds) }
|
||||
: { size: aspectRatio, quality: imageQuality },
|
||||
}), [aspectRatio, generationCapability, generationSeconds, imageQuality, resolution, shotAssetReferenceContext.referenceImages.length]);
|
||||
}), [aspectRatio, generationCapability, generationReferenceAudios.length, generationSeconds, imageQuality, resolution, shotAssetReferenceContext.referenceImages.length, videoEditOperation]);
|
||||
const routedModel = resolveCompatibleModel(effectiveConfig, selectedModel, modelRequirements) || selectedModel;
|
||||
const activeProfile = useMemo(() => modelCapabilityConfigFor(effectiveConfig, routedModel), [effectiveConfig, routedModel]);
|
||||
const videoProfile = generationCapability === "video" ? activeProfile.video : undefined;
|
||||
@@ -285,9 +291,12 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
onError: (error) => message.error(error instanceof Error ? error.message : "镜头资产更新失败"),
|
||||
});
|
||||
|
||||
const generateArtifact = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!selectedShot) throw new Error("请先选择镜头");
|
||||
const generateArtifact = async () => {
|
||||
if (!selectedShot || submittingShotIds.has(selectedShot.id)) return;
|
||||
const submittingShot = selectedShot;
|
||||
setSubmittingShotIds((current) => new Set(current).add(submittingShot.id));
|
||||
try {
|
||||
const values = await form.validateFields();
|
||||
let productionStep = workflowStep;
|
||||
if (!productionStep) {
|
||||
const initialized = await createUnitWorkflow(projectId, unitId);
|
||||
@@ -299,22 +308,21 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
if (routedModel.startsWith("local:dreamina-cli")) throw new Error("本机即梦任务暂不能登记到分镜产物，请选择后端模型渠道");
|
||||
const compatibilityError = modelCompatibilityError(effectiveConfig, routedModel, modelRequirements);
|
||||
if (compatibilityError) throw new Error(`当前模型配置不可用：${compatibilityError}`);
|
||||
const values = await form.validateFields();
|
||||
const saved = await saveProjectShot(projectId, {
|
||||
id: selectedShot.id,
|
||||
id: submittingShot.id,
|
||||
unitId,
|
||||
title: values.title,
|
||||
description: values.plotDescription,
|
||||
position: selectedShot.position,
|
||||
position: submittingShot.position,
|
||||
durationMs: Math.round(values.durationSeconds * 1000),
|
||||
status: selectedShot.status,
|
||||
status: submittingShot.status,
|
||||
revision: revisionInput(values),
|
||||
});
|
||||
const mode = generationCapability;
|
||||
const config = { ...generationConfig, videoSeconds: String(Math.max(1, Math.round(values.durationSeconds))) };
|
||||
if (!isAiConfigReady(config, routedModel)) throw new Error("当前模型渠道配置不完整，请先到设置中补齐");
|
||||
const basePrompt = mode === "video"
|
||||
? [values.videoPrompt || values.plotDescription, values.action, values.continuityNotes].filter(Boolean).join("\n")
|
||||
? [values.videoPrompt || values.plotDescription, values.action, values.dialogue && `台词：${values.dialogue}`, values.continuityNotes].filter(Boolean).join("\n")
|
||||
: [values.imagePrompt || values.plotDescription, values.action, "黑白分镜草图，清晰动作节拍，电影构图"].filter(Boolean).join("\n");
|
||||
const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext);
|
||||
const skillExecution = await skillRuntime.prepare({
|
||||
@@ -323,44 +331,45 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
skills: availableSkills,
|
||||
selectedSkillIds,
|
||||
});
|
||||
let taskId = "";
|
||||
const result = await runBackendGenerationTask({
|
||||
await submitBackendGenerationTask({
|
||||
projectId,
|
||||
mode,
|
||||
prompt: skillExecution.prompt,
|
||||
config,
|
||||
referenceImages: shotAssetReferenceContext.referenceImages,
|
||||
referenceAudios: generationReferenceAudios,
|
||||
metadata: {
|
||||
...skillExecution.metadata,
|
||||
workflowStepId: productionStep.id,
|
||||
domainProjectId: projectId,
|
||||
unitId,
|
||||
shotId: saved.shot.id,
|
||||
shotRevisionId: saved.shot.currentRevisionId,
|
||||
artifactType,
|
||||
role: "output",
|
||||
source: "short-drama-workflow",
|
||||
...skillExecution.metadata,
|
||||
...(mode === "video" && shotAssetReferenceContext.referenceImages.length ? { videoEditOperation: "reference_to_video" } : {}),
|
||||
resolvedCharacterVersions: shotAssetReferenceContext.resolvedCharacterVersions,
|
||||
artifactMetadata: { model: routedModel, aspectRatio, resolution, durationSeconds: values.durationSeconds, ...skillExecution.metadata },
|
||||
},
|
||||
onTaskUpdate: (task) => { taskId = task.id; },
|
||||
});
|
||||
const storageKey = mode === "video" ? result.video?.storageKey : result.images?.[0]?.storageKey;
|
||||
const resourceId = resourceIdFromStorageKey(storageKey);
|
||||
if (!taskId || !resourceId) throw new Error("生成已结束，但产物没有可登记的资源标识");
|
||||
await registerProjectTaskOutput(projectId, productionStep.id, {
|
||||
taskId,
|
||||
unitId,
|
||||
shotId: saved.shot.id,
|
||||
artifactType,
|
||||
resourceId,
|
||||
mediaType: mode,
|
||||
role: "output",
|
||||
metadataJson: JSON.stringify({ model: selectedModel, aspectRatio, resolution, durationSeconds: values.durationSeconds, ...skillExecution.metadata }),
|
||||
if (shotAssetReferenceContext.referenceAudios.length && !generationReferenceAudios.length) message.warning("当前模型不支持参考音频，已保留角色声音文字说明，但未附加声音样本");
|
||||
if (activeShotIdRef.current === submittingShot.id) setEditorDirty(false);
|
||||
await onRefresh();
|
||||
message.success(`${productionStageCopy[activeStage as "storyboard" | "previz" | "video"].label}任务已提交`);
|
||||
} catch (error) {
|
||||
message.error(error instanceof Error ? error.message : "生成任务提交失败");
|
||||
} finally {
|
||||
setSubmittingShotIds((current) => {
|
||||
const next = new Set(current);
|
||||
next.delete(submittingShot.id);
|
||||
return next;
|
||||
});
|
||||
},
|
||||
onSuccess: async () => { setEditorDirty(false); await onRefresh(); message.success(`${productionStageCopy[activeStage as "storyboard" | "previz" | "video"].label}已生成`); },
|
||||
onError: (error) => message.error(error instanceof Error ? error.message : "生成失败"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const stageCopy = productionStageCopy[activeStage as "storyboard" | "previz" | "video"];
|
||||
const selectedShotSubmitting = submittingShotIds.has(selectedShot?.id || "");
|
||||
|
||||
if (!selectedShot) {
|
||||
return <div className="workflow-empty-shot"><Empty description="当前章节还没有分镜"><Button type="primary" icon={<Plus className="size-4" />} loading={addingShot} onClick={onAddShot}>新增第一个分镜</Button></Empty></div>;
|
||||
@@ -523,7 +532,7 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
<div className="workflow-generation-cost" aria-live="polite">
|
||||
{creditsEnabled && formattedGenerationCredits ? <><CreditSymbol /><span>本次预计 {formattedGenerationCredits} 积分</span></> : creditsEnabled && routedModel ? <span>本次费用将在提交时按实际规格计算</span> : null}
|
||||
</div>
|
||||
<div className="flex items-center gap-2"><Button danger icon={<Trash2 className="size-4" />} loading={deleteShot.isPending} disabled={saveShot.isPending || generateArtifact.isPending || changeAssetBinding.isPending} onClick={requestDeleteShot}>删除镜头</Button><Button htmlType="submit" icon={<Save className="size-4" />} loading={saveShot.isPending} disabled={!editorDirty || deleteShot.isPending}>保存脚本</Button><Button type="primary" icon={<Play className="size-4" />} loading={generateArtifact.isPending || shotTask?.status === "queued" || shotTask?.status === "running"} disabled={deleteShot.isPending} onClick={() => generateArtifact.mutate()}>{shotTask?.status === "queued" || shotTask?.status === "running" ? `${stageCopy.action}（已运行${shotTaskElapsed}）` : shotTask?.status === "failed" ? `${stageCopy.action}（上次失败，可重试）` : shotTask?.status === "succeeded" && !newestArtifact ? `${stageCopy.action}（已完成，正在同步）` : newestArtifact ? `${stageCopy.action}（已生成）` : stageCopy.action}</Button></div>
|
||||
<div className="flex items-center gap-2"><Button danger icon={<Trash2 className="size-4" />} loading={deleteShot.isPending} disabled={saveShot.isPending || selectedShotSubmitting || changeAssetBinding.isPending} onClick={requestDeleteShot}>删除镜头</Button><Button htmlType="submit" icon={<Save className="size-4" />} loading={saveShot.isPending} disabled={!editorDirty || deleteShot.isPending}>保存脚本</Button><Button type="primary" icon={<Play className="size-4" />} loading={selectedShotSubmitting || shotTask?.status === "queued" || shotTask?.status === "running"} disabled={deleteShot.isPending} onClick={() => void generateArtifact()}>{selectedShotSubmitting ? `${stageCopy.action}（正在提交）` : shotTask?.status === "queued" || shotTask?.status === "running" ? `${stageCopy.action}（已运行${shotTaskElapsed}）` : shotTask?.status === "failed" ? `${stageCopy.action}（上次失败，可重试）` : shotTask?.status === "succeeded" && !newestArtifact ? `${stageCopy.action}（已完成，正在同步）` : newestArtifact ? `${stageCopy.action}（已生成）` : stageCopy.action}</Button></div>
|
||||
</footer>
|
||||
</Form>
|
||||
</section>
|
||||
@@ -546,13 +555,13 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
||||
<div className="workflow-preview-scroll thin-scrollbar">
|
||||
{previewTab === "latest" ? <LatestPreview artifact={previewArtifact} emptyText={stageCopy.empty} /> : <ArtifactHistory artifacts={artifacts} activeId={previewArtifact?.id} onSelect={(artifact) => { setPreviewArtifactId(artifact.id); setPreviewTab("latest"); }} />}
|
||||
<div className="workflow-preview-summary"><div className="flex items-center justify-between gap-2"><span className="text-xs font-medium">当前产物</span><ArtifactStatus artifact={newestArtifact} compact /></div><div className="mt-1 text-[var(--fs-micro)] text-foreground/45">{newestArtifact ? `${formatDuration(selectedShot.durationMs)} · ${resolution}p · v${newestArtifact.version}` : "当前镜头还没有生成产物"}</div></div>
|
||||
<div className="workflow-preview-actions"><Button icon={<RefreshCcw className="size-3.5" />} loading={generateArtifact.isPending} onClick={() => generateArtifact.mutate()}>重新生成</Button><Button icon={<Download className="size-3.5" />} disabled={!previewArtifact?.resourceId} onClick={() => previewArtifact?.resourceId && void downloadArtifact(previewArtifact, selectedShot.title, message.error)}>下载{activeStage === "video" ? "视频" : "图片"}</Button></div>
|
||||
<div className="workflow-preview-actions"><Button icon={<RefreshCcw className="size-3.5" />} loading={selectedShotSubmitting || shotTask?.status === "queued" || shotTask?.status === "running"} onClick={() => void generateArtifact()}>重新生成</Button><Button icon={<Download className="size-3.5" />} disabled={!previewArtifact?.resourceId} onClick={() => previewArtifact?.resourceId && void downloadArtifact(previewArtifact, selectedShot.title, message.error)}>下载{activeStage === "video" ? "视频" : "图片"}</Button></div>
|
||||
<ArtifactHistory artifacts={artifacts.slice(0, 4)} activeId={previewArtifact?.id} onSelect={(artifact) => setPreviewArtifactId(artifact.id)} compact />
|
||||
</div>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<ShotTimeline activeStage={activeStage} detail={detail} shots={shots} selectedShotId={selectedShot.id} onSelectShot={requestShotSelection} onAddShot={requestAddShot} addingShot={addingShot} />
|
||||
<ShotTimeline activeStage={activeStage} detail={detail} shots={shots} selectedShotId={selectedShot.id} submittingShotIds={submittingShotIds} onSelectShot={requestShotSelection} onAddShot={requestAddShot} addingShot={addingShot} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -655,7 +664,7 @@ function ArtifactHistory({ artifacts, activeId, onSelect, compact = false }: { a
|
||||
return <section className={`workflow-history ${compact ? "is-compact" : ""}`}><div className="workflow-history-title">历史版本</div>{artifacts.map((artifact) => <button key={artifact.id} type="button" className={artifact.id === activeId ? "is-active" : ""} onClick={() => onSelect(artifact)}>{artifact.resourceId ? artifact.type === "video" ? <video src={resourceFileUrl(artifact.resourceId)} muted preload="metadata" /> : <img src={resourceFileUrl(artifact.resourceId)} alt="" loading="lazy" /> : <span className="workflow-history-placeholder"><Layers3 /></span>}<span className="min-w-0 flex-1"><strong>v{artifact.version}{artifact.selected ? " · 当前" : ""}</strong><small>{new Date(artifact.createdAt).toLocaleString("zh-CN", { month: "numeric", day: "numeric", hour: "2-digit", minute: "2-digit" })}</small></span><ArtifactStatus artifact={artifact} compact /></button>)}</section>;
|
||||
}
|
||||
|
||||
function ShotTimeline({ activeStage, detail, shots, selectedShotId, onSelectShot, onAddShot, addingShot }: { activeStage: ShortDramaWorkflowStage; detail: ProjectDetail; shots: ProjectShot[]; selectedShotId: string; onSelectShot: (id: string) => void; onAddShot: () => void; addingShot: boolean }) {
|
||||
function ShotTimeline({ activeStage, detail, shots, selectedShotId, submittingShotIds, onSelectShot, onAddShot, addingShot }: { activeStage: ShortDramaWorkflowStage; detail: ProjectDetail; shots: ProjectShot[]; selectedShotId: string; submittingShotIds: Set<string>; onSelectShot: (id: string) => void; onAddShot: () => void; addingShot: boolean }) {
|
||||
const artifactType = artifactTypeForStage(activeStage);
|
||||
const latestTaskByShotId = useMemo(() => {
|
||||
const tasks = new Map<string, GenerationTask>();
|
||||
@@ -667,10 +676,10 @@ function ShotTimeline({ activeStage, detail, shots, selectedShotId, onSelectShot
|
||||
}
|
||||
return tasks;
|
||||
}, [artifactType, detail.tasks]);
|
||||
return <section className="workflow-shot-timeline"><header><div><strong>{detail.units.find((item) => item.id === shots[0]?.unitId)?.title || "本集"}</strong><span>{shots.length} 镜 · 总时长 {formatDuration(shots.reduce((total, item) => total + item.durationMs, 0))}</span></div><div className="flex items-center gap-1 text-[var(--fs-micro)] text-foreground/40"><List className="size-3.5" /> 共 {shots.length} 镜</div></header><div className="workflow-shot-track thin-scrollbar">{shots.map((shot, index) => <TimelineShot key={shot.id} artifactType={artifactType} detail={detail} shot={shot} task={latestTaskByShotId.get(shot.id)} index={index} selected={shot.id === selectedShotId} onSelect={() => onSelectShot(shot.id)} />)}<button type="button" className="workflow-add-shot-card" disabled={addingShot} onClick={onAddShot}><Plus className="size-5" /><span>新增分镜</span></button></div></section>;
|
||||
return <section className="workflow-shot-timeline"><header><div><strong>{detail.units.find((item) => item.id === shots[0]?.unitId)?.title || "本集"}</strong><span>{shots.length} 镜 · 总时长 {formatDuration(shots.reduce((total, item) => total + item.durationMs, 0))}</span></div><div className="flex items-center gap-1 text-[var(--fs-micro)] text-foreground/40"><List className="size-3.5" /> 共 {shots.length} 镜</div></header><div className="workflow-shot-track thin-scrollbar">{shots.map((shot, index) => <TimelineShot key={shot.id} artifactType={artifactType} detail={detail} shot={shot} task={latestTaskByShotId.get(shot.id)} submitting={submittingShotIds.has(shot.id)} index={index} selected={shot.id === selectedShotId} onSelect={() => onSelectShot(shot.id)} />)}<button type="button" className="workflow-add-shot-card" disabled={addingShot} onClick={onAddShot}><Plus className="size-5" /><span>新增分镜</span></button></div></section>;
|
||||
}
|
||||
|
||||
function TimelineShot({ artifactType, detail, shot, task, index, selected, onSelect }: { artifactType: string; detail: ProjectDetail; shot: ProjectShot; task?: GenerationTask; index: number; selected: boolean; onSelect: () => void }) {
|
||||
function TimelineShot({ artifactType, detail, shot, task, submitting, index, selected, onSelect }: { artifactType: string; detail: ProjectDetail; shot: ProjectShot; task?: GenerationTask; submitting: boolean; index: number; selected: boolean; onSelect: () => void }) {
|
||||
const video = currentArtifact(detail, shot.id, "video");
|
||||
const previz = currentArtifact(detail, shot.id, "action_board");
|
||||
const storyboard = currentArtifact(detail, shot.id, "storyboard");
|
||||
@@ -679,7 +688,7 @@ function TimelineShot({ artifactType, detail, shot, task, index, selected, onSel
|
||||
const revision = currentRevision(detail, shot);
|
||||
const stageLabel = artifactType === "video" ? "镜头视频" : artifactType === "action_board" ? "动作预演" : "分镜画面";
|
||||
const cameraMeta = [revision?.shotSize, revision?.cameraMovement].filter(Boolean).join(" · ") || "等待补充镜头参数";
|
||||
return <button type="button" className={`workflow-timeline-shot ${selected ? "is-active" : ""}`} onClick={onSelect}><span className="workflow-timeline-media">{preview?.resourceId ? preview.type === "video" ? <video src={resourceFileUrl(preview.resourceId)} muted preload="metadata" /> : <img src={resourceFileUrl(preview.resourceId)} alt="" loading="lazy" /> : <Film />}</span><span className="workflow-timeline-copy"><span className="workflow-timeline-heading"><strong>SC.{String(index + 1).padStart(2, "0")}</strong><b>{formatDuration(shot.durationMs)}</b></span><em className="workflow-timeline-title">{shot.title}</em><small className="workflow-timeline-meta">{cameraMeta}</small><span className="workflow-timeline-status"><span>{stageLabel}{stateArtifact ? ` · v${stateArtifact.version}` : ""}</span><ArtifactStatus artifact={stateArtifact} taskStatus={task?.status} compact /></span></span></button>;
|
||||
return <button type="button" className={`workflow-timeline-shot ${selected ? "is-active" : ""}`} onClick={onSelect}><span className="workflow-timeline-media">{preview?.resourceId ? preview.type === "video" ? <video src={resourceFileUrl(preview.resourceId)} muted preload="metadata" /> : <img src={resourceFileUrl(preview.resourceId)} alt="" loading="lazy" /> : <Film />}</span><span className="workflow-timeline-copy"><span className="workflow-timeline-heading"><strong>SC.{String(index + 1).padStart(2, "0")}</strong><b>{formatDuration(shot.durationMs)}</b></span><em className="workflow-timeline-title">{shot.title}</em><small className="workflow-timeline-meta">{cameraMeta}</small><span className="workflow-timeline-status"><span>{stageLabel}{stateArtifact ? ` · v${stateArtifact.version}` : ""}</span><ArtifactStatus artifact={stateArtifact} taskStatus={submitting ? "queued" : task?.status} compact /></span></span></button>;
|
||||
}
|
||||
|
||||
function revisionInput(values: ShotEditorValues): ShotRevisionInput {
|
||||
|
||||
@@ -3,10 +3,23 @@ import { resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey } from "@
|
||||
import type { ProjectAsset, ProjectDetail } from "@/services/api/projects";
|
||||
import type { AssetCategory } from "@/stores/use-asset-store";
|
||||
import type { ReferenceImage } from "@/types/image";
|
||||
import type { ReferenceAudio } from "@/types/media";
|
||||
|
||||
export type ShotAssetReferenceContext = {
|
||||
mentionReferences: CanvasResourceReference[];
|
||||
referenceImages: ReferenceImage[];
|
||||
referenceAudios: ReferenceAudio[];
|
||||
assetReferences: ShotPromptAssetReference[];
|
||||
resolvedCharacterVersions: Array<{ assetId: string; versionId: string }>;
|
||||
};
|
||||
|
||||
export type ShotPromptAssetReference = {
|
||||
assetId: string;
|
||||
title: string;
|
||||
category: string;
|
||||
imageIndex: number;
|
||||
audioIndex?: number;
|
||||
voiceDescription?: string;
|
||||
};
|
||||
|
||||
const assetCategories = new Set<AssetCategory>(["character", "environment", "wardrobe", "prop", "weapon", "style", "other"]);
|
||||
@@ -24,6 +37,35 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st
|
||||
return [{ asset, image }];
|
||||
});
|
||||
|
||||
const referenceAudios: ReferenceAudio[] = [];
|
||||
const audioIndexByResourceId = new Map<string, number>();
|
||||
const assetReferences = entries.map(({ asset }, index) => {
|
||||
const sampleResourceId = stringValue(asset.character?.voice?.profile.sampleResourceId);
|
||||
let audioIndex: number | undefined;
|
||||
if (sampleResourceId) {
|
||||
audioIndex = audioIndexByResourceId.get(sampleResourceId);
|
||||
if (!audioIndex) {
|
||||
audioIndex = referenceAudios.length + 1;
|
||||
audioIndexByResourceId.set(sampleResourceId, audioIndex);
|
||||
referenceAudios.push({
|
||||
id: `project-character-voice:${asset.id}`,
|
||||
name: `${asset.title}-声音样本`,
|
||||
type: "audio/*",
|
||||
url: resourceFileUrl(sampleResourceId),
|
||||
storageKey: resourceStorageKey(sampleResourceId),
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
assetId: asset.id,
|
||||
title: asset.title,
|
||||
category: asset.category,
|
||||
imageIndex: index + 1,
|
||||
...(audioIndex ? { audioIndex } : {}),
|
||||
...(asset.character ? { voiceDescription: characterVoiceDescription(asset) } : {}),
|
||||
} satisfies ShotPromptAssetReference;
|
||||
});
|
||||
|
||||
return {
|
||||
mentionReferences: entries.map(({ asset, image }) => ({
|
||||
id: `project-asset:${asset.id}`,
|
||||
@@ -38,6 +80,9 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st
|
||||
category: projectAssetCategory(asset.category),
|
||||
})),
|
||||
referenceImages: entries.map(({ image }) => image),
|
||||
referenceAudios,
|
||||
assetReferences,
|
||||
resolvedCharacterVersions: entries.flatMap(({ asset }) => asset.character?.versionId ? [{ assetId: asset.id, versionId: asset.character.versionId }] : []),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -53,7 +98,21 @@ export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAsset
|
||||
return label;
|
||||
});
|
||||
if (unresolved.size) throw new Error(`提示词中的 ${Array.from(unresolved).join("、")} 未绑定到当前镜头，请重新选择资产或删除引用`);
|
||||
return resolved;
|
||||
const assetBlock = compileShotAssetReferencePrompt(context.assetReferences);
|
||||
return [resolved.trim(), assetBlock].filter(Boolean).join("\n\n");
|
||||
}
|
||||
|
||||
export function compileShotAssetReferencePrompt(references: ShotPromptAssetReference[]) {
|
||||
if (!references.length) return "";
|
||||
const lines = references.flatMap((reference) => {
|
||||
const visualLabel = reference.category === "character" ? "人物参考" : `${projectAssetCategoryLabel(reference.category)}参考`;
|
||||
const media = [`${visualLabel}：图片${reference.imageIndex}`, reference.audioIndex ? `声音参考：音频${reference.audioIndex}` : ""].filter(Boolean).join("；");
|
||||
return [
|
||||
`- ${reference.title}：${media}`,
|
||||
reference.voiceDescription ? ` 声音说明：${reference.voiceDescription}` : "",
|
||||
].filter(Boolean);
|
||||
});
|
||||
return ["【资产参考】", "", ...lines].join("\n");
|
||||
}
|
||||
|
||||
function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undefined {
|
||||
@@ -87,3 +146,22 @@ function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undef
|
||||
function projectAssetCategory(value: string): AssetCategory {
|
||||
return assetCategories.has(value as AssetCategory) ? value as AssetCategory : "other";
|
||||
}
|
||||
|
||||
function projectAssetCategoryLabel(value: string) {
|
||||
return ({ environment: "场景", wardrobe: "服装", prop: "道具", weapon: "武器", style: "风格", other: "资产" } as Record<string, string>)[value] || "资产";
|
||||
}
|
||||
|
||||
function characterVoiceDescription(asset: ProjectAsset) {
|
||||
const definition = asset.character?.definition || {};
|
||||
const voice = asset.character?.voice;
|
||||
return [
|
||||
stringValue(definition.voiceLanguage) || stringValue(voice?.profile.language),
|
||||
stringValue(definition.voiceAge),
|
||||
stringValue(definition.voiceTimbre) || stringValue(voice?.profile.timbre),
|
||||
stringValue(voice?.instructions),
|
||||
].filter(Boolean).join("；");
|
||||
}
|
||||
|
||||
function stringValue(value: unknown) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
@@ -124,6 +124,20 @@ export async function runBackendGenerationTask(
|
||||
return createAndWaitGenerationTask({ projectId, mode, prompt, config, referenceImages, referenceVideos, referenceAudios, textHistory, signal, metadata, onTaskUpdate }, prepared, dependencies);
|
||||
}
|
||||
|
||||
// 分镜等后台生产流程只需要可靠提交任务；任务状态与产物由项目工作区轮询和
|
||||
// 后端自动回填负责，不能让页面 mutation 一直等待供应商完成。
|
||||
export async function submitBackendGenerationTask(
|
||||
options: BackendGenerationTaskOptions,
|
||||
dependencies: GenerationTaskDependencies = defaultDependencies,
|
||||
): Promise<GenerationTask> {
|
||||
throwIfAborted(options.signal);
|
||||
assertClientPromptLimit(options.mode, options.prompt, options.config, options.metadata);
|
||||
if (usesLocalDreamina(options.config)) throw new Error("本机即梦任务暂不支持后台提交");
|
||||
const prepared = await prepareGenerationReferences(options);
|
||||
throwIfAborted(options.signal);
|
||||
return createBackendGenerationTask(options, prepared, dependencies);
|
||||
}
|
||||
|
||||
export async function runBackendToolGenerationTask(options: {
|
||||
prompt: string;
|
||||
config: AiConfig;
|
||||
@@ -401,7 +415,14 @@ async function prepareGenerationReferences({
|
||||
}
|
||||
|
||||
async function createAndWaitGenerationTask(options: BackendGenerationTaskOptions, prepared: PreparedGenerationReferences, dependencies: GenerationTaskDependencies) {
|
||||
const { projectId, mode, prompt, config, signal, metadata, onTaskUpdate, onTextDelta } = options;
|
||||
const task = await createBackendGenerationTask(options, prepared, dependencies);
|
||||
const { signal, onTaskUpdate, onTextDelta } = options;
|
||||
const completed = await dependencies.waitTask(task.id, { signal, initialTask: task, onTaskUpdate, onTextDelta });
|
||||
return parseBackendGenerationResult(completed);
|
||||
}
|
||||
|
||||
async function createBackendGenerationTask(options: BackendGenerationTaskOptions, prepared: PreparedGenerationReferences, dependencies: GenerationTaskDependencies) {
|
||||
const { projectId, mode, prompt, config, metadata, onTaskUpdate } = options;
|
||||
const videoOperation = generationOperation(options);
|
||||
const workflow = resolveGenerationWorkflowExecution(config, mode);
|
||||
const logicalModelId = workflow ? "" : logicalModelIDForConfig(config);
|
||||
@@ -428,8 +449,7 @@ async function createAndWaitGenerationTask(options: BackendGenerationTaskOptions
|
||||
},
|
||||
});
|
||||
onTaskUpdate?.(task);
|
||||
const completed = await dependencies.waitTask(task.id, { signal, initialTask: task, onTaskUpdate, onTextDelta });
|
||||
return parseBackendGenerationResult(completed);
|
||||
return task;
|
||||
}
|
||||
|
||||
async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAudio) {
|
||||
|
||||
@@ -38,7 +38,12 @@ export type SeedanceTask = {
|
||||
};
|
||||
|
||||
export type ApiEnvelope<T> = T | { code?: number; data?: T | null; msg?: string };
|
||||
export type RequestOptions = { signal?: AbortSignal };
|
||||
export type RequestOptions = {
|
||||
signal?: AbortSignal;
|
||||
videoEditOperation?: string;
|
||||
videoStartFrameNodeId?: string;
|
||||
videoEndFrameNodeId?: string;
|
||||
};
|
||||
|
||||
export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string };
|
||||
export type VideoGenerationTask = { id: string; provider: "openai" | "agnes" | "seedance" | "video-generations" | "gemini-veo" | "novita" | "minimax"; model: string };
|
||||
|
||||
@@ -15,14 +15,16 @@ export async function createMiniMaxVideoTask(deps: VideoProviderDeps, config: Re
|
||||
const videoUrls = await Promise.all(videoReferences.slice(0, 3).map((video) => resolveMiniMaxMediaUrl(video, "参考视频")));
|
||||
const audioUrls = await Promise.all(audioReferences.slice(0, 3).map((audio) => resolveMiniMaxMediaUrl(audio, "参考音频")));
|
||||
const content: Array<Record<string, unknown>> = [{ type: "text", text: prompt.trim() }];
|
||||
const explicitFrameMode = options?.videoEditOperation !== "reference_to_video" && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId);
|
||||
imageUrls.forEach((url, index) => {
|
||||
// 首尾帧只允许成对出现；三张及以上图片统一作为多模态参考图，避免提交非法组合。
|
||||
const role = videoUrls.length || audioUrls.length || imageUrls.length > 2 ? "reference_image" : index === 0 ? "first_frame" : "last_frame";
|
||||
const reference = references[index];
|
||||
const role = miniMaxImageRole(reference?.id, options);
|
||||
if (explicitFrameMode && role === "reference_image") throw new Error("MiniMax 首尾帧模式不能混合未标记的参考图");
|
||||
content.push({ type: "image_url", image_url: { url }, role });
|
||||
});
|
||||
videoUrls.forEach((url) => content.push({ type: "video_url", video_url: { url }, role: "reference_video" }));
|
||||
audioUrls.forEach((url) => content.push({ type: "audio_url", audio_url: { url }, role: "reference_audio" }));
|
||||
const frameMode = imageUrls.length > 0 && imageUrls.length <= 2 && videoUrls.length === 0 && audioUrls.length === 0;
|
||||
const frameMode = explicitFrameMode;
|
||||
const payload = {
|
||||
model: modelOptionName(model),
|
||||
content,
|
||||
@@ -41,6 +43,13 @@ export async function createMiniMaxVideoTask(deps: VideoProviderDeps, config: Re
|
||||
}
|
||||
}
|
||||
|
||||
function miniMaxImageRole(imageId: string | undefined, options?: RequestOptions) {
|
||||
if (options?.videoEditOperation === "reference_to_video") return "reference_image";
|
||||
if (imageId && imageId === options?.videoStartFrameNodeId) return "first_frame";
|
||||
if (imageId && imageId === options?.videoEndFrameNodeId) return "last_frame";
|
||||
return "reference_image";
|
||||
}
|
||||
|
||||
export async function pollMiniMaxVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
|
||||
try {
|
||||
const response = await deps.transport.get<{ task?: MiniMaxVideoTask }>(miniMaxVideoUrl(config, `/query/video_generation/${encodeURIComponent(task.id)}`), options);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { defaultConfig } from "../src/stores/use-config-store";
|
||||
import { createSeedanceTask } from "../src/services/api/video-provider-seedance";
|
||||
import { createMiniMaxVideoTask } from "../src/services/api/video-provider-minimax";
|
||||
import type { VideoProviderDeps } from "../src/services/api/video-provider-deps";
|
||||
import { unwrapEnvelope, videoResponseTools, videoTaskId } from "../src/services/api/video-response";
|
||||
import { isPublicMediaUrl, normalizeVideoResolution, normalizeVideoSeconds, normalizeVideoSize } from "../src/services/api/video-validation";
|
||||
@@ -100,3 +101,37 @@ describe("Volcengine Ark full-modal references", () => {
|
||||
await expect(createSeedanceTask(deps, config as never, model, "跟随节奏", [], [], [{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }])).rejects.toThrow("不支持纯音频或文本+音频");
|
||||
});
|
||||
});
|
||||
|
||||
describe("MiniMax explicit reference roles", () => {
|
||||
test("project assets use reference_image and reference_audio instead of first frame", async () => {
|
||||
let requestBody: unknown;
|
||||
const deps = {
|
||||
transport: {
|
||||
post: async (_url: string, body: unknown) => {
|
||||
requestBody = body;
|
||||
return { task_id: "minimax-task-1" };
|
||||
},
|
||||
},
|
||||
response: videoResponseTools,
|
||||
} as unknown as VideoProviderDeps;
|
||||
const config = { ...defaultConfig, baseUrl: "https://api.minimaxi.com/v1", videoSeconds: "6", vquality: "768P", size: "16:9", videoWatermark: "false" };
|
||||
|
||||
await createMiniMaxVideoTask(
|
||||
deps,
|
||||
config as never,
|
||||
"MiniMax-H3",
|
||||
"保持角色一致",
|
||||
[{ id: "character-1", name: "character.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/character.png" }],
|
||||
[],
|
||||
[{ id: "voice-1", name: "voice.mp3", type: "audio/mpeg", url: "https://cdn.example.com/voice.mp3" }],
|
||||
{ videoEditOperation: "reference_to_video" },
|
||||
);
|
||||
|
||||
expect((requestBody as { content: unknown[] }).content).toEqual([
|
||||
{ type: "text", text: "保持角色一致" },
|
||||
{ type: "image_url", image_url: { url: "https://cdn.example.com/character.png" }, role: "reference_image" },
|
||||
{ type: "audio_url", audio_url: { url: "https://cdn.example.com/voice.mp3" }, role: "reference_audio" },
|
||||
]);
|
||||
expect((requestBody as { ratio: string }).ratio).toBe("16:9");
|
||||
});
|
||||
});
|
||||
|
||||
85
web/test/workflow-shot-generation.test.ts
Normal file
85
web/test/workflow-shot-generation.test.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { expect, test } from "bun:test";
|
||||
|
||||
import { defaultConfig } from "../src/stores/use-config-store";
|
||||
import { submitBackendGenerationTask, type GenerationTaskDependencies } from "../src/services/api/generation-task";
|
||||
import type { GenerationTask } from "../src/services/api/task-center";
|
||||
import type { ProjectDetail } from "../src/services/api/projects";
|
||||
import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "../src/pages/projects/detail/workflow-shot-references";
|
||||
|
||||
test("shot asset prompt keeps character names and maps character image, voice and scene assets", () => {
|
||||
const detail = {
|
||||
assets: [
|
||||
{
|
||||
id: "character-1",
|
||||
title: "张天昊",
|
||||
category: "character",
|
||||
mediaType: "image",
|
||||
primaryVersionId: "character-version-1",
|
||||
character: {
|
||||
versionId: "character-version-1",
|
||||
definition: { voiceLanguage: "普通话", voiceAge: "青年男性", voiceTimbre: "略带疲惫和震惊" },
|
||||
representations: [{ id: "representation-1", resourceId: "character-image-1", mediaType: "image/png", role: "primary" }],
|
||||
voice: { profile: { sampleResourceId: "character-audio-1", language: "普通话", timbre: "沉稳" }, instructions: "内心独白语气" },
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "scene-1",
|
||||
title: "坑底场景",
|
||||
category: "environment",
|
||||
mediaType: "image",
|
||||
primaryVersionId: "scene-version-1",
|
||||
storageKey: "resource:scene-image-1",
|
||||
},
|
||||
],
|
||||
shotReferences: [
|
||||
{ shotId: "shot-1", assetVersionId: "character-version-1", status: "linked" },
|
||||
{ shotId: "shot-1", assetVersionId: "scene-version-1", status: "linked" },
|
||||
],
|
||||
} as ProjectDetail;
|
||||
|
||||
const context = buildShotAssetReferenceContext(detail, "shot-1");
|
||||
const prompt = resolveShotAssetMentionPrompt("张天昊在 @[asset:scene-1] 睁开眼睛", context);
|
||||
|
||||
expect(context.referenceImages).toHaveLength(2);
|
||||
expect(context.referenceAudios).toHaveLength(1);
|
||||
expect(context.referenceAudios[0]?.storageKey).toBe("resource:character-audio-1");
|
||||
expect(context.resolvedCharacterVersions).toEqual([{ assetId: "character-1", versionId: "character-version-1" }]);
|
||||
expect(prompt).toContain("张天昊在 图片2 睁开眼睛");
|
||||
expect(prompt).toContain("- 张天昊：人物参考：图片1；声音参考：音频1");
|
||||
expect(prompt).toContain("声音说明：普通话；青年男性；略带疲惫和震惊；内心独白语气");
|
||||
expect(prompt).toContain("- 坑底场景：场景参考：图片2");
|
||||
});
|
||||
|
||||
test("background generation submission returns after task creation without waiting", async () => {
|
||||
let waitCalls = 0;
|
||||
const task = {
|
||||
id: "task-1",
|
||||
type: "canvas_video",
|
||||
status: "queued",
|
||||
prompt: "角色表演",
|
||||
attempts: 0,
|
||||
createdAt: "2026-08-30T00:00:00.000Z",
|
||||
updatedAt: "2026-08-30T00:00:00.000Z",
|
||||
} satisfies GenerationTask;
|
||||
const dependencies: GenerationTaskDependencies = {
|
||||
createTask: async () => task,
|
||||
waitTask: async () => {
|
||||
waitCalls += 1;
|
||||
throw new Error("should not wait");
|
||||
},
|
||||
runLocal: async () => ({ mode: "video" }),
|
||||
createId: () => "id-1",
|
||||
now: () => "2026-08-30T00:00:00.000Z",
|
||||
};
|
||||
|
||||
const submitted = await submitBackendGenerationTask({
|
||||
projectId: "project-1",
|
||||
mode: "video",
|
||||
prompt: "角色表演",
|
||||
config: { ...defaultConfig, model: "MiniMax-H3", videoModel: "MiniMax-H3" },
|
||||
metadata: { shotId: "shot-1", videoEditOperation: "reference_to_video" },
|
||||
}, dependencies);
|
||||
|
||||
expect(submitted).toBe(task);
|
||||
expect(waitCalls).toBe(0);
|
||||
});
|
||||
Reference in New Issue
Block a user