From 10437bdf684c91603e29725091697abed86ef724 Mon Sep 17 00:00:00 2001 From: jiabx Date: Mon, 5 Oct 2026 13:28:24 +0800 Subject: [PATCH] =?UTF-8?q?fix(agent):=20=E7=9C=8B=E5=9B=BE=20-=20?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E8=BF=90=E8=A1=8C=E6=97=B6=E7=9A=84=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E6=AD=A5=E9=AA=A4=E5=B8=A6=E4=B8=8A=E8=8E=B7=E5=87=86?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E6=B8=85=E5=8D=95=20(#671)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复现：Agent 调用 canvas_inspect_image 看图后，下一步模型调用报「模型协议引用了未获准的图片」，整轮失败；只要 Agent 看过一次图就必然复现。 原因：看图之后服务端规范历史里会多一条带 resource: 图片占位的 user 消息；旧的 Go 执行循环在发请求前把这些占位对应的资源写进 input.referenceImages，独立 Agent 运行时（Pi）的模型步骤漏了这一步，预检发现占位不在获准清单里就拒绝了请求。 修法：Pi 路径的模型步在组装任务输入前复用同一个 cloudAgentImageReferences 校验与装配（资源归属、大小、图片数量上限），把获准图片清单随请求发出。 --- .../app/cloud_agent_pi_image_references.go | 15 +++++++ .../cloud_agent_pi_image_references_test.go | 44 +++++++++++++++++++ backend/internal/app/cloud_agent_pi_model.go | 9 ++++ 3 files changed, 68 insertions(+) create mode 100644 backend/internal/app/cloud_agent_pi_image_references.go create mode 100644 backend/internal/app/cloud_agent_pi_image_references_test.go diff --git a/backend/internal/app/cloud_agent_pi_image_references.go b/backend/internal/app/cloud_agent_pi_image_references.go new file mode 100644 index 000000000..9130dc8d2 --- /dev/null +++ b/backend/internal/app/cloud_agent_pi_image_references.go @@ -0,0 +1,15 @@ +package app + +// 独立 Agent 运行时（Pi）的模型步骤带上获准图片清单。 +// +// canvas_inspect_image 看图后，服务端的规范历史里会多一条带 resource: 图片占位的 user 消息。 +// 旧的 Go 执行循环在发请求前调用 cloudAgentImageReferences，把这些占位对应的资源写进 +// input.referenceImages，供应商层据此读取真实图片；Pi 路径（runCloudAgentModelStep）漏了这一步， +// 预检发现占位不在获准清单里，整轮以「模型协议引用了未获准的图片」失败。 +// +// 这里复用同一个函数：校验资源归属与大小、按模型的图片数量上限移出多余图片，并返回清单。 +// 它会改写传入的 canonical（只影响本次请求的副本），所以必须在组装任务输入之前调用， +// 且在 state.Canonical 已经保存完整历史之后调用。 +func (s *Service) cloudAgentPiStepImageReferences(userID string, req CloudAgentRequest, canonical *canonicalAgentRequest) ([]providerMedia, error) { + return s.cloudAgentImageReferences(userID, req, canonical) +} diff --git a/backend/internal/app/cloud_agent_pi_image_references_test.go b/backend/internal/app/cloud_agent_pi_image_references_test.go new file mode 100644 index 000000000..670fead51 --- /dev/null +++ b/backend/internal/app/cloud_agent_pi_image_references_test.go @@ -0,0 +1,44 @@ +package app + +import "testing" + +// 复现：canvas_inspect_image 看图后下一次模型调用报「模型协议引用了未获准的图片」。 +func TestCloudAgentPiStepImageReferencesSatisfyPlaceholderPreflight(t *testing.T) { + s, _, _ := cloudAgentVisionFixture(t) + state := cloudAgentRuntime{Request: agentTestRequest()} + state.Request.VisionEnabled = true + call := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-cat", map[string]any{"nodeId": "cat"}) + result, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, call) + if err != nil { + t.Fatal(err) + } + history := []map[string]any{ + {"role": "user", "content": "参考左侧素材"}, + {"role": "user", "content": cloudAgentImageContentParts(result.(cloudAgentImageInspection))}, + } + + // 不带清单时预检拒绝，复现修复前的失败。 + bare := canonicalAgentRequest{ToolChoice: "auto", Messages: history} + if err := validateAgentResourcePlaceholders(canvasGenerationInput{Mode: "text", AgentRequests: &agentToolRequests{Canonical: &bare}}); err == nil { + t.Fatal("placeholder without approved references passed preflight") + } + + canonical := canonicalAgentRequest{ToolChoice: "auto", Messages: history} + references, err := s.cloudAgentPiStepImageReferences("user", state.Request, &canonical) + if err != nil { + t.Fatal(err) + } + if len(references) != 1 { + t.Fatalf("references = %#v, want the inspected image", references) + } + input := canvasGenerationInput{Mode: "text", ReferenceImages: references, AgentRequests: &agentToolRequests{Canonical: &canonical}} + if err := validateAgentResourcePlaceholders(input); err != nil { + t.Fatalf("preflight still rejects the inspected image: %v", err) + } + + // 没有图片的历史不产生清单，也不报错。 + plain := canonicalAgentRequest{ToolChoice: "auto", Messages: history[:1]} + if refs, err := s.cloudAgentPiStepImageReferences("user", state.Request, &plain); err != nil || len(refs) != 0 { + t.Fatalf("text-only history: refs=%#v err=%v", refs, err) + } +} diff --git a/backend/internal/app/cloud_agent_pi_model.go b/backend/internal/app/cloud_agent_pi_model.go index f2e6b7048..177ba8035 100644 --- a/backend/internal/app/cloud_agent_pi_model.go +++ b/backend/internal/app/cloud_agent_pi_model.go @@ -229,6 +229,12 @@ func (s *Service) runCloudAgentModelStep(ctx context.Context, userID, runID stri // 纠偏上下文只用于这一次请求，不写回运行状态，避免下一步重复出现。 canonical.Messages = append(append([]map[string]any(nil), canonical.Messages...), correction...) } + // 看图后的历史里有 resource: 图片占位，必须随请求带上获准图片清单， + // 否则预检以「模型协议引用了未获准的图片」拒绝（见 cloud_agent_pi_image_references.go）。 + references, refErr := s.cloudAgentPiStepImageReferences(userID, state.Request, &canonical) + if refErr != nil { + return nil, false, refErr + } input := map[string]any{ "mode": "text", "prompt": state.Request.Prompt, @@ -244,6 +250,9 @@ func (s *Service) runCloudAgentModelStep(ctx context.Context, userID, runID stri "maxOutputTokens": cloudAgentStepOutputBudget(state.StepLimits, state.BoostStepOutputBudget), }, } + if len(references) > 0 { + input["referenceImages"] = references + } req := CreateTaskRequest{ ProjectID: state.Request.CanvasID, Type: "canvas_text",