fix(canvas): 视频首尾帧 - 保留结构化帧并按协议顺序传参

2026-08-01 10:20:47 +08:00
parent 2ceaa56cd8
commit 49133a7c2a
6 changed files with 82 additions and 17 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@

 ## Unreleased

+- 修复视频 Composer 丢失已配置首尾帧的问题，并按 `POST /v1/video/generations` 协议顺序传递首帧、尾帧和普通参考图。
+
 ## v1.0.22

 - 提升提示词卡片模型菜单的可读性：使用不透明主题表面，并以黄色粗体突出积分价格。
--- a/backend/internal/service/provider.go
+++ b/backend/internal/service/provider.go
@@ -1148,6 +1148,13 @@ func newAPIChannel2VideoBody(input canvasGenerationInput) (map[string]interface{
 			return nil, fmt.Errorf("NewAPI Video Generations 的 %s 必须且只能提供 1 张参考图（当前 %d 张）", input.Config.Model, len(images))
 		}
 	}
+	frameImages, err := videoFrameImageURLs(input, images)
+	if err != nil {
+		return nil, err
+	}
+	if len(frameImages) > 0 {
+		images = frameImages
+	}

 	seconds, secondsErr := strconv.Atoi(strings.TrimSpace(input.Config.VideoSeconds))
 	if secondsErr != nil || seconds < 1 {
@@ -2066,7 +2073,11 @@ func seedanceVideosBody(input canvasGenerationInput) (map[string]interface{}, er
 		}
 		imageURLs = append(imageURLs, url)
 	}
-	if frameImageURLs := seedanceVideosFrameImageURLs(input, imageURLs); len(frameImageURLs) > 0 {
+	frameImageURLs, err := videoFrameImageURLs(input, imageURLs)
+	if err != nil {
+		return nil, err
+	}
+	if len(frameImageURLs) > 0 {
 		body["image_urls"] = frameImageURLs
 	} else if len(imageURLs) > 0 {
 		body["image_url"] = imageURLs[0]
@@ -2117,36 +2128,41 @@ func seedanceImageRole(input canvasGenerationInput, image providerMedia) string
 	return "reference_image"
 }

-func seedanceVideosFrameImageURLs(input canvasGenerationInput, imageURLs []string) []string {
+func videoFrameImageURLs(input canvasGenerationInput, imageURLs []string) ([]string, error) {
 	startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId")
 	endFrameID := metadataString(input.Metadata, "videoEndFrameNodeId")
 	if startFrameID == "" && endFrameID == "" {
-		return nil
+		return nil, nil
 	}
-	// /v1/videos 的 image_urls 只接受字符串；首帧和尾帧通过数组顺序表达。
+	// image_urls 按首帧、尾帧、普通参考图排序，保持 JSON 视频协议的结构化帧语义。
 	ordered := make([]string, 0, len(imageURLs))
 	used := make([]bool, len(imageURLs))
-	appendFrame := func(frameID string) {
+	appendFrame := func(frameID string, label string) error {
 		if frameID == "" {
-			return
+			return nil
 		}
 		for index, image := range input.ReferenceImages {
-			if index >= len(imageURLs) || used[index] || image.ID != frameID {
+			if index >= len(imageURLs) || image.ID != frameID {
 				continue
 			}
 			ordered = append(ordered, imageURLs[index])
 			used[index] = true
-			return
+			return nil
 		}
+		return fmt.Errorf("已配置的%s参考图未包含在视频请求中", label)
+	}
+	if err := appendFrame(startFrameID, "首帧"); err != nil {
+		return nil, err
+	}
+	if err := appendFrame(endFrameID, "尾帧"); err != nil {
+		return nil, err
 	}
-	appendFrame(startFrameID)
-	appendFrame(endFrameID)
 	for index, imageURL := range imageURLs {
 		if !used[index] {
 			ordered = append(ordered, imageURL)
 		}
 	}
-	return ordered
+	return ordered, nil
 }

 func metadataString(metadata map[string]interface{}, key string) string {
--- a/backend/internal/service/provider_test.go
+++ b/backend/internal/service/provider_test.go
@@ -8,6 +8,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"strconv"
 	"strings"
 	"testing"
@@ -915,6 +916,37 @@ func TestNewAPIChannel2SingleImageModelUsesReferenceForStaleTextToVideoMetadata(
 	}
 }

+func TestNewAPIChannel2OrdersFramesBeforeReferenceImages(t *testing.T) {
+	body, err := newAPIChannel2VideoBody(canvasGenerationInput{
+		Config: providerConfig{Model: "Seedance 2 Mini", VideoSeconds: "10"},
+		ReferenceImages: []providerMedia{
+			{ID: "character", DataURL: "data:image/png;base64,Y2hhcmFjdGVy"},
+			{ID: "last-frame", DataURL: "data:image/png;base64,bGFzdA=="},
+			{ID: "first-frame", DataURL: "data:image/png;base64,Zmlyc3Q="},
+		},
+		Metadata: map[string]interface{}{"videoStartFrameNodeId": "first-frame", "videoEndFrameNodeId": "last-frame", "videoEditOperation": "image_to_video"},
+	})
+	if err != nil {
+		t.Fatalf("newAPIChannel2VideoBody() error = %v", err)
+	}
+	images, ok := body["image_urls"].([]string)
+	want := []string{"data:image/png;base64,Zmlyc3Q=", "data:image/png;base64,bGFzdA==", "data:image/png;base64,Y2hhcmFjdGVy"}
+	if !ok || !reflect.DeepEqual(images, want) {
+		t.Fatalf("image_urls = %#v, want %#v", body["image_urls"], want)
+	}
+}
+
+func TestNewAPIChannel2RejectsMissingConfiguredFrame(t *testing.T) {
+	_, err := newAPIChannel2VideoBody(canvasGenerationInput{
+		Config:          providerConfig{Model: "Seedance 2 Mini", VideoSeconds: "10"},
+		ReferenceImages: []providerMedia{{ID: "character", DataURL: testReferenceImageDataURL}},
+		Metadata:        map[string]interface{}{"videoStartFrameNodeId": "missing-frame", "videoEditOperation": "image_to_video"},
+	})
+	if err == nil || !strings.Contains(err.Error(), "首帧参考图未包含") {
+		t.Fatalf("newAPIChannel2VideoBody() error = %v", err)
+	}
+}
+
 func TestValidateGenerationInterfaceRejectsMismatchedType(t *testing.T) {
 	if err := validateGenerationInterface("video", "chat-completion"); err == nil {
 		t.Fatal("validateGenerationInterface() error = nil")
--- a/web/src/components/canvas/canvas-node-generation.ts
+++ b/web/src/components/canvas/canvas-node-generation.ts
@@ -61,7 +61,7 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
    const storyboardInputs = getConnectedStoryboardRows(nodeId, nodes, connections);
    const hasExplicitNodeMention = /@\[node:[^\]]+\]/.test(normalizeLegacyNodeMentions(prompt, inputs));
    if ((sourceNode?.type === CanvasNodeType.Config && Boolean(sourceNode.metadata?.composerContent?.trim())) || hasExplicitNodeMention) {
-        return buildComposerGenerationContext(inputs, prompt);
+        return buildComposerGenerationContext(inputs, prompt, [sourceNode?.metadata?.videoStartFrameNodeId, sourceNode?.metadata?.videoEndFrameNodeId].filter((id): id is string => Boolean(id)));
    }

    const isStoryboardMedia = sourceNode?.type === CanvasNodeType.Image || sourceNode?.type === CanvasNodeType.Video;
@@ -109,7 +109,7 @@ function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]
    return next;
 }

-function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: string): NodeGenerationContext {
+function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: string, videoFrameNodeIds: string[] = []): NodeGenerationContext {
    const normalizedPrompt = normalizeLegacyNodeMentions(prompt, inputs);
    const inputByNodeId = new Map(inputs.map((input) => [input.nodeId, input]));
    const selectedInputs: NodeGenerationInput[] = [];
@@ -141,12 +141,20 @@ function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: s

    nextPrompt += normalizedPrompt.slice(lastIndex);
    if (textBlocks.length) nextPrompt = `${nextPrompt.trim()}\n\n${textBlocks.join("\n\n")}`;
+    // 首尾帧是结构化生成参数，不受提示词中的 @ 引用筛选影响。
+    const selectedNodeIds = new Set(selectedInputs.map((input) => input.nodeId));
+    videoFrameNodeIds.forEach((nodeId) => {
+        const input = inputByNodeId.get(nodeId);
+        if (!input?.image || selectedNodeIds.has(nodeId)) return;
+        selectedInputs.push(input);
+        selectedNodeIds.add(nodeId);
+    });
    const referenceImages = selectedInputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
    const referenceVideos = selectedInputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video));
    const referenceAudios = selectedInputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio));
    const characterReferences = selectedInputs.map((input) => input.character).filter((item): item is CharacterGenerationReference => Boolean(item));

-    if (!hasToken && !textBlocks.length) {
+    if (!hasToken && !textBlocks.length && !selectedInputs.length) {
        return {
            prompt,
            referenceImages: [],
--- a/web/src/lib/canvas/canvas-project-generation.ts
+++ b/web/src/lib/canvas/canvas-project-generation.ts
@@ -276,8 +276,9 @@ export function buildVideoGenerationMetadata(
    },
 ): CanvasNodeMetadata {
    const metadata = node?.metadata;
-    const startFrame = metadata?.videoStartFrameNodeId && context?.referenceImages.some((image) => image.id === metadata.videoStartFrameNodeId) ? metadata.videoStartFrameNodeId : undefined;
-    const endFrame = metadata?.videoEndFrameNodeId && context?.referenceImages.some((image) => image.id === metadata.videoEndFrameNodeId) ? metadata.videoEndFrameNodeId : undefined;
+    const referenceImageIds = new Set((context?.referenceImages || []).map((image) => image.id));
+    const startFrame = requireConnectedVideoFrame(metadata?.videoStartFrameNodeId, "首帧", referenceImageIds);
+    const endFrame = requireConnectedVideoFrame(metadata?.videoEndFrameNodeId, "尾帧", referenceImageIds);
    return {
        videoEditOperation: resolveVideoEditOperation(node, context),
        videoCameraMoveId: metadata?.videoCameraMoveId,
@@ -287,6 +288,12 @@ export function buildVideoGenerationMetadata(
    };
 }

+function requireConnectedVideoFrame(frameNodeId: string | undefined, label: string, referenceImageIds: Set<string>) {
+    if (!frameNodeId) return undefined;
+    if (referenceImageIds.has(frameNodeId)) return frameNodeId;
+    throw new Error(`已配置的${label}参考图未连接或不可用，请重新选择后再生成`);
+}
+
 export async function resolveMetadataReferences(metadata: CanvasNodeMetadata) {
    if (metadata.generationType !== "edit") return [];
    if (!metadata.references?.length) return null;
--- a/web/src/lib/model-protocols.ts
+++ b/web/src/lib/model-protocols.ts
@@ -34,7 +34,7 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [
    { value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" },
    { value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" },
    { value: "newapi-channel-1", label: "NewAPI 媒体任务", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "application/json", media: "图片、视频、音频公网 URL" },
-    { value: "newapi-channel-2", label: "NewAPI Video Generations", capability: "video", create: "POST /v1/video/generations", poll: "GET /v1/video/generations/{task_id}", contentType: "application/json", media: "image_urls / video_urls / audio_urls" },
+    { value: "newapi-channel-2", label: "NewAPI Video Generations", capability: "video", create: "POST /v1/video/generations", poll: "GET /v1/video/generations/{task_id}", contentType: "application/json", media: "image_urls（首帧、尾帧、其他参考图） / video_urls / audio_urls" },
    { value: "xai-video", label: "xAI 官方视频", capability: "video", create: "POST /v1/videos/generations", poll: "GET /v1/videos/{request_id}", contentType: "application/json", media: "单张起始图" },
    { value: "volcengine-ark-video", label: "火山方舟视频", capability: "video", create: "POST /api/v3/contents/generations/tasks", poll: "GET /api/v3/contents/generations/tasks/{task_id}", contentType: "application/json", media: "图片、视频、音频参考素材" },
    { value: "volcengine-jimeng-video", label: "即梦官方视频", capability: "video", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "文本或一张首帧图，模型标识填写 req_key" },