fix(canvas): 媒体生成 - 修复已有图片引用和图片音频视频模型匹配

backend/internal/service/provider_test.go

@@ -950,6 +950,58 @@ func TestRunImageTaskOmitsAutomaticQualityAndSize(t *testing.T) {
 	}
 }
 
+func TestRunOpenAIImageTaskUsesMultipartEditContract(t *testing.T) {
+	t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/images/edits" {
+			t.Errorf("path = %q, want /v1/images/edits", r.URL.Path)
+		}
+		if contentType := r.Header.Get("Content-Type"); !strings.HasPrefix(contentType, "multipart/form-data;") {
+			t.Errorf("Content-Type = %q, want multipart/form-data", contentType)
+		}
+		if err := r.ParseMultipartForm(2 << 20); err != nil {
+			t.Fatalf("ParseMultipartForm() error = %v", err)
+		}
+		if r.FormValue("model") != "gpt-image-2-high" || r.FormValue("prompt") != "make the reference clearer" || r.FormValue("n") != "1" {
+			t.Fatalf("form values = model:%q prompt:%q n:%q", r.FormValue("model"), r.FormValue("prompt"), r.FormValue("n"))
+		}
+		if r.FormValue("response_format") != "b64_json" || r.FormValue("output_format") != "png" || r.FormValue("size") != "1024x1024" {
+			t.Fatalf("format values = response_format:%q output_format:%q size:%q", r.FormValue("response_format"), r.FormValue("output_format"), r.FormValue("size"))
+		}
+		file, header, err := r.FormFile("image")
+		if err != nil {
+			t.Fatalf("FormFile(image) error = %v", err)
+		}
+		defer file.Close()
+		content, err := io.ReadAll(file)
+		if err != nil {
+			t.Fatalf("ReadAll(image) error = %v", err)
+		}
+		if header.Filename != "reference-reference.png" || string(content) != "hello" {
+			t.Fatalf("image = filename:%q content:%q", header.Filename, string(content))
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"data":[{"b64_json":"aGVsbG8="}]}`))
+	}))
+	defer server.Close()
+
+	profile := DefaultImageCapabilityConfig("openai-image", "gpt-image-2-high")
+	result, err := runImageTask(context.Background(), canvasGenerationInput{
+		Mode:            "image",
+		Prompt:          "make the reference clearer",
+		Config:          providerConfig{BaseURL: server.URL, APIKey: "key", Model: "gpt-image-2-high", InterfaceType: "openai-image", Size: "1024x1024"},
+		ImageCapability: profile,
+		ReferenceImages: []providerMedia{{Name: "reference.png", Type: "image/png", DataURL: testReferenceImageDataURL}},
+	})
+	if err != nil {
+		t.Fatalf("runImageTask() error = %v", err)
+	}
+	images, _ := result["images"].([]map[string]string)
+	if len(images) != 1 || images[0]["dataUrl"] != "data:image/png;base64,aGVsbG8=" {
+		t.Fatalf("images = %#v", result["images"])
+	}
+}
+
 func TestRunGrokImageTaskUsesJSONEditContract(t *testing.T) {
 	t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

docs/content/docs/progress/pending-test.mdx

@@ -134,6 +134,21 @@ description: 影策已实现功能的手工验证清单与未确认边界
 - [ ] 替换一个已经位于参考轨道中的图片，确认附件不会重复，其他引用仍指向正确素材。
 - [ ] 确认浅色/深色主题、窄窗口和同一图片多次出现时，拖入高亮与“替换”遮罩清晰且不遮挡相邻文字。
 
+## 已有图片节点图生图引用
+
+- [x] 已有图片节点显式使用 `@图片1` 时会把节点自身作为参考图提交；未显式引用时仍保持文生图语义。
+- [x] 无法解析的 `@图片N`、`@视频N`、`@音频N` 等画布引用会在提交前报错，不再静默丢失素材。
+- [x] 后端专项测试确认 OpenAI Images 图生图使用 `POST /v1/images/edits` 和 `multipart/form-data`，并携带图片文件、模型、提示词、尺寸及响应格式。
+- [ ] 在线上画布打开一张已有图片，输入“将 `@图片1` 图片变清晰”并生成，确认任务日志包含参考图、请求路径为 `/v1/images/edits`、Content-Type 为 `multipart/form-data`。
+- [ ] 在同一图片节点移除 `@图片1` 后再次生成，确认请求恢复为 `/v1/images/generations`，不会隐式携带旧图片。
+
+## 画布图片与音频组合的视频模型选择
+
+- [x] 图片或角色与音频组合时继续按 `image_to_video` 匹配，音频仅作为附加参考；纯音频仍按 `audio_to_video`，包含参考视频时仍按 `reference_to_video`。
+- [x] 模型选择专项测试覆盖“支持图生视频且允许参考音频”的模型，不再错误提示“不支持全模态参考”。
+- [ ] 在线上画布只连接一张图片和一段音频，确认支持图生视频与参考音频的视频模型可选择，并能正常提交生成任务。
+- [ ] 分别验证纯音频、图片加音频、视频加音频三种输入，确认模型筛选和任务 operation 依次为 `audio_to_video`、`image_to_video`、`reference_to_video`。
+
 用户确认上述场景后，再将对应条目移动到功能文档的已验证范围。
 
 ## 工作区、Agent、画布复制、模型配置与创作附件修复

web/package.json

@@ -8,7 +8,7 @@
         "dev": "vite --host 0.0.0.0 --port 3000",
         "build": "bun run build:bridge && tsc --noEmit && vite build",
         "typecheck": "tsc --noEmit",
-        "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
+        "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
         "test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs",
         "start": "vite preview --host 0.0.0.0 --port 3000",
         "format": "prettier --write .",

web/src/components/canvas/canvas-node-generation.ts

@@ -4,7 +4,7 @@ import { seedanceReferenceLabel } from "@/lib/seedance-video";
 import type { ReferenceImage } from "@/types/image";
 import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
 import { CanvasNodeType, type CanvasConnection, type CanvasGenerationMode, type CanvasNodeData } from "@/types/canvas";
-import { getGenerationResourceNodes, getContextResourceNodes } from "@/lib/canvas/canvas-resource-references";
+import { getGenerationResourceNodes, getContextResourceNodes, getMentionResourceNodes } from "@/lib/canvas/canvas-resource-references";
 import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade";
 import { getNodeResourceKind } from "@/lib/canvas/node-registry";
 import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
@@ -69,14 +69,17 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
               return image ? [{ nodeId: sourceNode.id, type: "image" as const, title: sourceNode.title, image }] : [];
           })()
         : [];
-    const inputs = [...connectedInputs, ...portraitTextureInput, ...buildAssetGenerationInputs(assets)];
+    // 显式 @ 引用必须与提示词面板展示的资源集合一致；默认自动输入仍只取入边，
+    // 避免已有图片在没有 @图片N 时被悄悄当作自身参考图。
+    const mentionInputs = mergeGenerationInputs(buildNodeMentionGenerationInputs(nodeId, nodes, connections), portraitTextureInput, buildAssetGenerationInputs(assets));
     const storyboardInputs = getConnectedStoryboardRows(nodeId, nodes, connections);
-    const hasExplicitResourceMention = hasResolvableGenerationMention(prompt, inputs);
+    assertResolvableGenerationMentions(prompt, mentionInputs);
+    const hasExplicitResourceMention = hasResolvableGenerationMention(prompt, mentionInputs);
     const isWorkflowSource = sourceNode?.type === CanvasNodeType.Config && isCanvasWorkflowProvider(sourceNode.metadata);
     if ((Boolean(sourceNode?.metadata?.composerContent?.trim()) && (sourceNode?.type === CanvasNodeType.Config || isWorkflowSource)) || hasExplicitResourceMention) {
         const autoIncludeWorkflowMedia = isWorkflowSource;
         return buildComposerGenerationContext(
-            inputs,
+            mentionInputs,
             prompt,
             // 工作流节点由字段映射接收全部连线媒体；视频节点的历史首尾帧字段不能再额外追加参考图。
             autoIncludeWorkflowMedia ? [] : [sourceNode?.metadata?.videoStartFrameNodeId, sourceNode?.metadata?.videoEndFrameNodeId].filter((id): id is string => Boolean(id)),
@@ -260,14 +263,31 @@ export function normalizeGenerationNodeMentionTokens(prompt: string, inputs: Nod
 }
 
 function hasResolvableGenerationMention(prompt: string, inputs: NodeGenerationInput[]) {
+    return inspectGenerationMentions(prompt, inputs).hasResolved;
+}
+
+function assertResolvableGenerationMentions(prompt: string, inputs: NodeGenerationInput[]) {
+    const { unresolved } = inspectGenerationMentions(prompt, inputs);
+    if (!unresolved.length) return;
+    throw new Error(`提示词中的 ${unresolved.join("、")} 没有对应的画布资源，请重新选择引用后再生成`);
+}
+
+function inspectGenerationMentions(prompt: string, inputs: NodeGenerationInput[]) {
     const normalizedPrompt = normalizeGenerationNodeMentionTokens(prompt, inputs);
     const slotInputByToken = new Map(generationSlotEntries(inputs).map(({ token, input }) => [token, input]));
     const nodeInputById = new Map(inputs.filter((input) => !input.nodeId.startsWith("asset:")).map((input) => [input.nodeId, input]));
     const assetInputById = new Map(inputs.filter((input) => input.nodeId.startsWith("asset:")).map((input) => [input.nodeId.slice("asset:".length), input]));
+    const unresolved = new Set<string>();
+    let hasResolved = false;
     for (const match of normalizedPrompt.matchAll(GENERATION_MENTION_PATTERN)) {
-        if (resolveGenerationMention(normalizedPrompt, match, slotInputByToken, nodeInputById, assetInputById)) return true;
+        if (match[3]) {
+            const end = (match.index || 0) + match[0].length;
+            if (!hasMentionBoundary(normalizedPrompt, end)) continue;
+        }
+        if (resolveGenerationMention(normalizedPrompt, match, slotInputByToken, nodeInputById, assetInputById)) hasResolved = true;
+        else unresolved.add(match[0]);
     }
-    return false;
+    return { hasResolved, unresolved: [...unresolved] };
 }
 
 function generationSlotEntries(inputs: NodeGenerationInput[]) {
@@ -302,7 +322,14 @@ function hasMentionBoundary(value: string, index: number) {
 }
 
 export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
-    const resourceNodes = getGenerationResourceNodes(nodeId, nodes, connections);
+    return buildGenerationInputs(getGenerationResourceNodes(nodeId, nodes, connections), nodes, connections);
+}
+
+function buildNodeMentionGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
+    return buildGenerationInputs(getMentionResourceNodes(nodeId, nodes, connections), nodes, connections);
+}
+
+function buildGenerationInputs(resourceNodes: CanvasNodeData[], nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
     return resourceNodes.flatMap((node): NodeGenerationInput[] => {
         const character = readCharacterReference(node);
         if (character) return [{ nodeId: node.id, type: "character" as const, title: node.title, character }];
@@ -320,6 +347,17 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[
     });
 }
 
+function mergeGenerationInputs(...groups: NodeGenerationInput[][]) {
+    const seen = new Set<string>();
+    return groups.flatMap((group) =>
+        group.filter((input) => {
+            if (seen.has(input.nodeId)) return false;
+            seen.add(input.nodeId);
+            return true;
+        }),
+    );
+}
+
 function buildAssetGenerationInputs(assets: Asset[]): NodeGenerationInput[] {
     return assets.flatMap((asset): NodeGenerationInput[] => {
         const nodeId = `asset:${asset.id}`;

web/src/lib/model-selection.ts

@@ -319,11 +319,11 @@ export function modelGroupReferenceLimits(config: AiConfig, selected: string, ca
 
 export function inferVideoOperation(input: ModelInputSummary) {
     const visualInputCount = input.imageCount + input.characterCount;
-    // 纯音频参考使用独立能力；音频与图片、角色或视频组合时属于全模态参考，
+    // 图片或角色决定图生视频主模式，音频只作为附加参考，不应把组合请求
-    // 不能把组合请求误路由到只支持 audio_to_video 的细分模型。
+    // 提升为全模态参考；纯音频输入才使用独立的 audio_to_video 能力。
-    if (input.audioCount > 0) return visualInputCount > 0 || input.videoCount > 0 ? "reference_to_video" : "audio_to_video";
     if (input.videoCount > 0 || visualInputCount > 2) return "reference_to_video";
     if (visualInputCount > 0) return "image_to_video";
+    if (input.audioCount > 0) return "audio_to_video";
     return "text_to_video";
 }
 

web/test/canvas-model-policy.test.ts

@@ -216,10 +216,26 @@ describe("逻辑模型选择", () => {
 
     test("音频仅在单独参考时归类为音频生视频", () => {
         expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("audio_to_video");
-        expect(inferVideoOperation({ textCount: 1, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
+        expect(inferVideoOperation({ textCount: 1, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("image_to_video");
         expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 1, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
     });
 
+    test("图片加音频可匹配支持图生视频和参考音频的模型", () => {
+        const config = policyConfig();
+        const imageModel = "relay::cinema-image";
+        const imageCost = config.channels[0]?.modelCosts?.find((item) => item.model === "cinema-image");
+        if (!imageCost?.capabilityConfig?.video) throw new Error("缺少图生视频能力配置");
+        imageCost.capabilityConfig.video.references.maxAudios = 1;
+        const requirements = {
+            capability: "video" as const,
+            input: { textCount: 0, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 },
+            videoSeconds: "6",
+        };
+
+        expect(modelCompatibilityError(config, imageModel, requirements)).toBe("");
+        expect(resolveCompatibleModel(config, "relay::cinema-text", requirements)).toBe(imageModel);
+    });
+
     test("逻辑视频模型将 720 与 720p 视为同一分辨率", () => {
         const model = "cinema-720p";
         const channel: ModelChannel = {

web/test/canvas-node-generation-mentions.test.ts

@@ -32,6 +32,31 @@ function connection(fromNodeId: string): CanvasConnection {
 }
 
 describe("canvas node generation position mentions", () => {
+    test("已有图片节点显式引用自身时作为图生图参考图提交", () => {
+        const source = node("image-self", CanvasNodeType.Image, "data:image/png;base64,a");
+        source.metadata.composerContent = "将 @图片1 图片变清晰";
+
+        const context = buildNodeGenerationContext(source.id, [source], [], source.metadata.composerContent, []);
+
+        expect(context.referenceImages.map((image) => image.id)).toEqual([source.id]);
+        expect(context.imageCount).toBe(1);
+        expect(context.prompt).toBe("将 图片1 图片变清晰");
+    });
+
+    test("已有图片节点未显式引用自身时不自动退化为图生图", () => {
+        const source = node("image-self", CanvasNodeType.Image, "data:image/png;base64,a");
+        const context = buildNodeGenerationContext(source.id, [source], [], "生成一个新的构图", []);
+
+        expect(context.referenceImages).toEqual([]);
+        expect(context.imageCount).toBe(0);
+        expect(context.prompt).toBe("生成一个新的构图");
+    });
+
+    test("无法解析的画布引用会阻止静默降级为文生图", () => {
+        const target = targetNode();
+        expect(() => buildNodeGenerationContext(target.id, [target], [], "将 @图片1 图片变清晰", [])).toThrow("@图片1 没有对应的画布资源");
+    });
+
     test("同一个 @图片1 在换线后自动指向新的第一张图片", () => {
         const target = targetNode();
         const imageA = node("image-a", CanvasNodeType.Image, "data:image/png;base64,a");

web/test/generation-task-local.test.ts

@@ -1325,7 +1325,7 @@ test("remote image video and audio references keep Backend parity without Dreami
                 referenceAudios: [{ id: "remote-image-audio-audio-0001", name: "reference.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-image-audio-audio-0001" }],
             },
             result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-image-audio-output" } },
-            operation: "reference_to_video",
+            operation: "image_to_video",
         },
         {
             mode: "audio" as const,