云端 Agent 过去只能凭标题或提示词猜画面:上游既有的 image_text_detect / image_annotation_render 是"组织 OCR 请求"和"渲染标注图当参考"两种不同能力,模型自己从不看像素。本 PR 新增只读工具 canvas_inspect_image,把画面本身交给模型。 - 交付走与生成任务参考图同一套短时签名链接(providerResourceURL),Agent 上下文只增长一个 URL,不内联 base64:一张 500 KB 的内联图片会同时顶穿请求体硬限与压缩阈值。 - 能力门控:仅当本轮渠道模型声明 text.references.maxImages > 0 时暴露该工具;VisionEnabled 在 建 run 时按渠道模型合同定格并持久化(工具授权每步从落库状态重建,客户端传入值被忽略)。 - 上下文治理:图片保留最近 3 个工具轮次,同一张图一轮内最多附图 2 次(之后只回执文字,确需 重看传 refresh=true),超出保留轮次换成逐图文字占位符并落 context_images_pruned 事件。 - 视觉锚点:模型看过后写进正文的一句观察记入 ReferenceAssets[].VisualNote 并跨轮继承,下一轮 不必重复看图,也不再对这张图声称"没有视觉识别证据"。 - 同批多图合并成一条 user 消息:图片只能挂 user 消息(tool 角色只接受字符串),逐张 append 会让 历史退化成 tool → user(图) → tool → user(图),被上游以"tool_calls 之后缺少对应 tool 消息" 拒绝。改为先缓冲到 PendingImageInspections,等整批 tool 结果入历史后合并成一条;两个 flush 点 共用同一个幂等助手,ask_user 提前结束路径单独 flush。缓冲必须进检查点:一次推进只执行一个 调用,整批跨越多次转移,每次转移都从 StateJSON 重新解码。 - 附带把上述 400 从"渠道余额不足"归类里拆出来(原文含 insufficient 会落进额度类目掩盖真因), 归为"会话里的工具调用与结果不匹配"。 - 文档补该能力口径与工具全集条目;前端零改动。
203 lines
6.8 KiB
Go
203 lines
6.8 KiB
Go
package app
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"slices"
|
||
"strings"
|
||
)
|
||
|
||
type cloudAgentPlanItem struct {
|
||
ID string `json:"id"`
|
||
Title string `json:"title"`
|
||
Status string `json:"status"`
|
||
}
|
||
|
||
func cloudAgentPendingPlanItems(plan []cloudAgentPlanItem) []string {
|
||
pending := make([]string, 0, len(plan))
|
||
for _, item := range plan {
|
||
if item.Status != "done" {
|
||
pending = append(pending, item.Title)
|
||
}
|
||
}
|
||
return pending
|
||
}
|
||
|
||
const cloudAgentPlanBlockMarker = "\n\n## 本轮待办清单\n\n"
|
||
|
||
func stripCloudAgentPlanBlock(system string) string {
|
||
if i := strings.Index(system, cloudAgentPlanBlockMarker); i >= 0 {
|
||
return system[:i]
|
||
}
|
||
return system
|
||
}
|
||
|
||
func isCloudAgentRuntimeContextMessage(message map[string]any) bool {
|
||
if stringField(message, "role") != "user" {
|
||
return false
|
||
}
|
||
return stringField(message, cloudAgentContextSourceKey) == "runtime"
|
||
}
|
||
|
||
func stripCloudAgentRuntimeContext(messages []map[string]any) []map[string]any {
|
||
if len(messages) == 0 || !isCloudAgentRuntimeContextMessage(messages[len(messages)-1]) {
|
||
return messages
|
||
}
|
||
var context cloudAgentRuntimeContext
|
||
content := strings.TrimPrefix(stringField(messages[len(messages)-1], "content"), cloudAgentRuntimeContextMarker)
|
||
if json.Unmarshal([]byte(content), &context) != nil || context.Kind != cloudAgentContextPlan {
|
||
return messages
|
||
}
|
||
return messages[:len(messages)-1]
|
||
}
|
||
|
||
func attachCloudAgentPlan(canonical *canonicalAgentRequest, plan []cloudAgentPlanItem) {
|
||
if canonical == nil {
|
||
return
|
||
}
|
||
canonical.SystemPrompt = stripCloudAgentPlanBlock(canonical.SystemPrompt)
|
||
canonical.Messages = stripCloudAgentRuntimeContext(canonical.Messages)
|
||
if len(plan) == 0 {
|
||
return
|
||
}
|
||
canonical.Messages = append(slices.Clone(canonical.Messages), cloudAgentRuntimeMessage(cloudAgentRuntimeContext{Kind: cloudAgentContextPlan, Items: plan}))
|
||
}
|
||
|
||
func cloudAgentPlanRequiresFirstApproval(state *cloudAgentRuntime, call cloudAgentCall) bool {
|
||
if state == nil || state.Approval != nil || len(state.Plan) != 0 || state.Request.PermissionMode != "request_approval" {
|
||
return false
|
||
}
|
||
_, ok := cloudAgentPlanApprovalPreview(call)
|
||
return ok
|
||
}
|
||
|
||
func cloudAgentPlanNudgeMessage(state *cloudAgentRuntime, pendingTitle string) map[string]any {
|
||
return cloudAgentRuntimeMessage(cloudAgentRuntimeContext{
|
||
Kind: cloudAgentContextPendingPlan, PendingTitle: pendingTitle,
|
||
LatestUserMessage: cloudAgentLatestUserInstruction(state.Canonical.Messages),
|
||
})
|
||
}
|
||
|
||
func cloudAgentPlanApprovalPreview(call cloudAgentCall) (cloudAgentApprovalPreview, bool) {
|
||
var args struct {
|
||
Items []cloudAgentPlanItem `json:"items"`
|
||
}
|
||
if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil {
|
||
return cloudAgentApprovalPreview{}, false
|
||
}
|
||
items := make([]cloudAgentApprovalPreviewItem, 0, len(args.Items))
|
||
for _, entry := range args.Items {
|
||
title := strings.TrimSpace(entry.Title)
|
||
if title == "" {
|
||
continue
|
||
}
|
||
items = append(items, cloudAgentApprovalPreviewItem{Operation: "plan_step", Summary: truncateRunes(title, 240)})
|
||
}
|
||
if len(items) < 2 {
|
||
return cloudAgentApprovalPreview{}, false
|
||
}
|
||
return cloudAgentApprovalPreview{
|
||
Kind: "plan",
|
||
Title: "确认执行计划",
|
||
Description: fmt.Sprintf("Agent 把这轮任务拆成 %d 步。确认后才会开始执行;暂不执行则会让它改用别的做法。", len(items)),
|
||
Items: items,
|
||
}, true
|
||
}
|
||
|
||
func cloudAgentApplyPlanUpdate(state *cloudAgentRuntime, call cloudAgentCall) (any, error) {
|
||
var args struct {
|
||
Items []cloudAgentPlanItem `json:"items"`
|
||
}
|
||
if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil {
|
||
return nil, BadAuthRequest("工具参数必须是只含支持字段的JSON对象")
|
||
}
|
||
if len(args.Items) > 20 {
|
||
return nil, BadAuthRequest("待办清单最多 20 项")
|
||
}
|
||
seen := map[string]bool{}
|
||
for i := range args.Items {
|
||
args.Items[i].ID = strings.TrimSpace(args.Items[i].ID)
|
||
args.Items[i].Title = strings.TrimSpace(args.Items[i].Title)
|
||
if args.Items[i].ID == "" || args.Items[i].Title == "" {
|
||
return nil, BadAuthRequest("待办的 id 和 title 不能为空")
|
||
}
|
||
if seen[args.Items[i].ID] {
|
||
return nil, BadAuthRequest("待办 id 重复:" + args.Items[i].ID)
|
||
}
|
||
seen[args.Items[i].ID] = true
|
||
switch args.Items[i].Status {
|
||
case "pending", "doing", "done":
|
||
default:
|
||
args.Items[i].Status = "pending"
|
||
}
|
||
}
|
||
if !slices.Equal(state.Plan, args.Items) {
|
||
state.ActionNudged = false
|
||
}
|
||
state.Plan = args.Items
|
||
return map[string]any{"items": args.Items, "pendingTitles": cloudAgentPendingPlanItems(args.Items)}, nil
|
||
}
|
||
|
||
func cloudAgentAskUser(call cloudAgentCall) (any, error) {
|
||
var args struct {
|
||
Question string `json:"question"`
|
||
Options []struct {
|
||
Label string `json:"label"`
|
||
Detail string `json:"detail"`
|
||
} `json:"options"`
|
||
AllowFreeform *bool `json:"allowFreeform"`
|
||
}
|
||
if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil {
|
||
return nil, BadAuthRequest("工具参数必须是只含支持字段的JSON对象")
|
||
}
|
||
question := strings.TrimSpace(args.Question)
|
||
if question == "" {
|
||
return nil, BadAuthRequest("ask_user 必须给出 question:把要用户拍板的那一个问题一句话写清")
|
||
}
|
||
options := make([]map[string]any, 0, len(args.Options))
|
||
for _, option := range args.Options {
|
||
label := strings.TrimSpace(option.Label)
|
||
if label == "" {
|
||
continue
|
||
}
|
||
entry := map[string]any{"label": truncateRunes(label, 120)}
|
||
if detail := strings.TrimSpace(option.Detail); detail != "" {
|
||
entry["detail"] = truncateRunes(detail, 240)
|
||
}
|
||
options = append(options, entry)
|
||
}
|
||
if len(options) < 2 {
|
||
return nil, BadAuthRequest("ask_user 至少要给 2 个候选项;若你自己能定,直接做完继续,不要问")
|
||
}
|
||
if len(options) > 6 {
|
||
options = options[:6]
|
||
}
|
||
allowFreeform := true
|
||
if args.AllowFreeform != nil {
|
||
allowFreeform = *args.AllowFreeform
|
||
}
|
||
return map[string]any{
|
||
"phase": "question",
|
||
"question": truncateRunes(question, 400),
|
||
"options": options,
|
||
"allowFreeform": allowFreeform,
|
||
}, nil
|
||
}
|
||
|
||
// skipRemainingCloudAgentCalls 结束本批剩余调用(ask_user 之后本轮不再继续执行)。
|
||
// 末尾的 flush 让"本批前面的看图结果"仍能落在全部 tool 结果之后:这批调用到这里已经
|
||
// 完整(每个声明的 tool_call_id 都有回执),但本轮就此结束、不会再走 advanceCloudAgent
|
||
// 的兜底 flush,少了这一步缓冲的图片会被丢掉。
|
||
func skipRemainingCloudAgentCalls(runID string, state *cloudAgentRuntime) {
|
||
for index := state.CallIndex + 1; index < len(state.Calls); index++ {
|
||
cloudAgentToolResult(runID, state, state.Calls[index], map[string]any{"skipped": true}, BadAuthRequest("本轮已结束(等待用户决定),该调用未执行"))
|
||
}
|
||
cloudAgentFlushPendingImages(state)
|
||
}
|
||
|
||
func cloudAgentCanonicalWithPlan(state *cloudAgentRuntime) canonicalAgentRequest {
|
||
canonical := state.Canonical
|
||
attachCloudAgentPlan(&canonical, state.Plan)
|
||
return canonical
|
||
}
|