fix(agent): 看图 - 独立运行时的模型步骤带上获准图片清单 (#671)
复现:Agent 调用 canvas_inspect_image 看图后,下一步模型调用报「模型协议引用了未获准的图片」,整轮失败;只要 Agent 看过一次图就必然复现。 原因:看图之后服务端规范历史里会多一条带 resource: 图片占位的 user 消息;旧的 Go 执行循环在发请求前把这些占位对应的资源写进 input.referenceImages,独立 Agent 运行时(Pi)的模型步骤漏了这一步,预检发现占位不在获准清单里就拒绝了请求。 修法:Pi 路径的模型步在组装任务输入前复用同一个 cloudAgentImageReferences 校验与装配(资源归属、大小、图片数量上限),把获准图片清单随请求发出。
This commit is contained in:
1 parent
1924729af8
commit
dd563d3373
3 files changed
+68
No files matched your search
@@ -0,0 +1,15 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
// 独立 Agent 运行时(Pi)的模型步骤带上获准图片清单。
|
||||||
|
//
|
||||||
|
// canvas_inspect_image 看图后,服务端的规范历史里会多一条带 resource: 图片占位的 user 消息。
|
||||||
|
// 旧的 Go 执行循环在发请求前调用 cloudAgentImageReferences,把这些占位对应的资源写进
|
||||||
|
// input.referenceImages,供应商层据此读取真实图片;Pi 路径(runCloudAgentModelStep)漏了这一步,
|
||||||
|
// 预检发现占位不在获准清单里,整轮以「模型协议引用了未获准的图片」失败。
|
||||||
|
//
|
||||||
|
// 这里复用同一个函数:校验资源归属与大小、按模型的图片数量上限移出多余图片,并返回清单。
|
||||||
|
// 它会改写传入的 canonical(只影响本次请求的副本),所以必须在组装任务输入之前调用,
|
||||||
|
// 且在 state.Canonical 已经保存完整历史之后调用。
|
||||||
|
func (s *Service) cloudAgentPiStepImageReferences(userID string, req CloudAgentRequest, canonical *canonicalAgentRequest) ([]providerMedia, error) {
|
||||||
|
return s.cloudAgentImageReferences(userID, req, canonical)
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// 复现:canvas_inspect_image 看图后下一次模型调用报「模型协议引用了未获准的图片」。
|
||||||
|
func TestCloudAgentPiStepImageReferencesSatisfyPlaceholderPreflight(t *testing.T) {
|
||||||
|
s, _, _ := cloudAgentVisionFixture(t)
|
||||||
|
state := cloudAgentRuntime{Request: agentTestRequest()}
|
||||||
|
state.Request.VisionEnabled = true
|
||||||
|
call := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-cat", map[string]any{"nodeId": "cat"})
|
||||||
|
result, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, call)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
history := []map[string]any{
|
||||||
|
{"role": "user", "content": "参考左侧素材"},
|
||||||
|
{"role": "user", "content": cloudAgentImageContentParts(result.(cloudAgentImageInspection))},
|
||||||
|
}
|
||||||
|
|
||||||
|
// 不带清单时预检拒绝,复现修复前的失败。
|
||||||
|
bare := canonicalAgentRequest{ToolChoice: "auto", Messages: history}
|
||||||
|
if err := validateAgentResourcePlaceholders(canvasGenerationInput{Mode: "text", AgentRequests: &agentToolRequests{Canonical: &bare}}); err == nil {
|
||||||
|
t.Fatal("placeholder without approved references passed preflight")
|
||||||
|
}
|
||||||
|
|
||||||
|
canonical := canonicalAgentRequest{ToolChoice: "auto", Messages: history}
|
||||||
|
references, err := s.cloudAgentPiStepImageReferences("user", state.Request, &canonical)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(references) != 1 {
|
||||||
|
t.Fatalf("references = %#v, want the inspected image", references)
|
||||||
|
}
|
||||||
|
input := canvasGenerationInput{Mode: "text", ReferenceImages: references, AgentRequests: &agentToolRequests{Canonical: &canonical}}
|
||||||
|
if err := validateAgentResourcePlaceholders(input); err != nil {
|
||||||
|
t.Fatalf("preflight still rejects the inspected image: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 没有图片的历史不产生清单,也不报错。
|
||||||
|
plain := canonicalAgentRequest{ToolChoice: "auto", Messages: history[:1]}
|
||||||
|
if refs, err := s.cloudAgentPiStepImageReferences("user", state.Request, &plain); err != nil || len(refs) != 0 {
|
||||||
|
t.Fatalf("text-only history: refs=%#v err=%v", refs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -229,6 +229,12 @@ func (s *Service) runCloudAgentModelStep(ctx context.Context, userID, runID stri
|
|||||||
// 纠偏上下文只用于这一次请求,不写回运行状态,避免下一步重复出现。
|
// 纠偏上下文只用于这一次请求,不写回运行状态,避免下一步重复出现。
|
||||||
canonical.Messages = append(append([]map[string]any(nil), canonical.Messages...), correction...)
|
canonical.Messages = append(append([]map[string]any(nil), canonical.Messages...), correction...)
|
||||||
}
|
}
|
||||||
|
// 看图后的历史里有 resource: 图片占位,必须随请求带上获准图片清单,
|
||||||
|
// 否则预检以「模型协议引用了未获准的图片」拒绝(见 cloud_agent_pi_image_references.go)。
|
||||||
|
references, refErr := s.cloudAgentPiStepImageReferences(userID, state.Request, &canonical)
|
||||||
|
if refErr != nil {
|
||||||
|
return nil, false, refErr
|
||||||
|
}
|
||||||
input := map[string]any{
|
input := map[string]any{
|
||||||
"mode": "text",
|
"mode": "text",
|
||||||
"prompt": state.Request.Prompt,
|
"prompt": state.Request.Prompt,
|
||||||
@@ -244,6 +250,9 @@ func (s *Service) runCloudAgentModelStep(ctx context.Context, userID, runID stri
|
|||||||
"maxOutputTokens": cloudAgentStepOutputBudget(state.StepLimits, state.BoostStepOutputBudget),
|
"maxOutputTokens": cloudAgentStepOutputBudget(state.StepLimits, state.BoostStepOutputBudget),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
if len(references) > 0 {
|
||||||
|
input["referenceImages"] = references
|
||||||
|
}
|
||||||
req := CreateTaskRequest{
|
req := CreateTaskRequest{
|
||||||
ProjectID: state.Request.CanvasID,
|
ProjectID: state.Request.CanvasID,
|
||||||
Type: "canvas_text",
|
Type: "canvas_text",
|
||||||
|
|||||||
Reference in new issue
Block a user