Files
ttstd-infinite-canvas/backend/internal/app/cloud_agent_plan.go
AetherNo2332 080e4b901e feat(cloud-agent): 看图工具 canvas_inspect_image 与同批多图工具结果配对
云端 Agent 过去只能凭标题或提示词猜画面:上游既有的 image_text_detect / image_annotation_render
是"组织 OCR 请求"和"渲染标注图当参考"两种不同能力,模型自己从不看像素。本 PR 新增只读工具
canvas_inspect_image,把画面本身交给模型。

- 交付走与生成任务参考图同一套短时签名链接(providerResourceURL),Agent 上下文只增长一个
  URL,不内联 base64:一张 500 KB 的内联图片会同时顶穿请求体硬限与压缩阈值。
- 能力门控:仅当本轮渠道模型声明 text.references.maxImages > 0 时暴露该工具;VisionEnabled 在
  建 run 时按渠道模型合同定格并持久化(工具授权每步从落库状态重建,客户端传入值被忽略)。
- 上下文治理:图片保留最近 3 个工具轮次,同一张图一轮内最多附图 2 次(之后只回执文字,确需
  重看传 refresh=true),超出保留轮次换成逐图文字占位符并落 context_images_pruned 事件。
- 视觉锚点:模型看过后写进正文的一句观察记入 ReferenceAssets[].VisualNote 并跨轮继承,下一轮
  不必重复看图,也不再对这张图声称"没有视觉识别证据"。
- 同批多图合并成一条 user 消息:图片只能挂 user 消息(tool 角色只接受字符串),逐张 append 会让
  历史退化成 tool → user(图) → tool → user(图),被上游以"tool_calls 之后缺少对应 tool 消息"
  拒绝。改为先缓冲到 PendingImageInspections,等整批 tool 结果入历史后合并成一条;两个 flush 点
  共用同一个幂等助手,ask_user 提前结束路径单独 flush。缓冲必须进检查点:一次推进只执行一个
  调用,整批跨越多次转移,每次转移都从 StateJSON 重新解码。
- 附带把上述 400 从"渠道余额不足"归类里拆出来(原文含 insufficient 会落进额度类目掩盖真因),
  归为"会话里的工具调用与结果不匹配"。
- 文档补该能力口径与工具全集条目;前端零改动。
2026-09-22 01:32:07 +08:00

203 lines
6.8 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package app
import (
"encoding/json"
"fmt"
"slices"
"strings"
)
type cloudAgentPlanItem struct {
ID string `json:"id"`
Title string `json:"title"`
Status string `json:"status"`
}
func cloudAgentPendingPlanItems(plan []cloudAgentPlanItem) []string {
pending := make([]string, 0, len(plan))
for _, item := range plan {
if item.Status != "done" {
pending = append(pending, item.Title)
}
}
return pending
}
const cloudAgentPlanBlockMarker = "\n\n## 本轮待办清单\n\n"
func stripCloudAgentPlanBlock(system string) string {
if i := strings.Index(system, cloudAgentPlanBlockMarker); i >= 0 {
return system[:i]
}
return system
}
func isCloudAgentRuntimeContextMessage(message map[string]any) bool {
if stringField(message, "role") != "user" {
return false
}
return stringField(message, cloudAgentContextSourceKey) == "runtime"
}
func stripCloudAgentRuntimeContext(messages []map[string]any) []map[string]any {
if len(messages) == 0 || !isCloudAgentRuntimeContextMessage(messages[len(messages)-1]) {
return messages
}
var context cloudAgentRuntimeContext
content := strings.TrimPrefix(stringField(messages[len(messages)-1], "content"), cloudAgentRuntimeContextMarker)
if json.Unmarshal([]byte(content), &context) != nil || context.Kind != cloudAgentContextPlan {
return messages
}
return messages[:len(messages)-1]
}
func attachCloudAgentPlan(canonical *canonicalAgentRequest, plan []cloudAgentPlanItem) {
if canonical == nil {
return
}
canonical.SystemPrompt = stripCloudAgentPlanBlock(canonical.SystemPrompt)
canonical.Messages = stripCloudAgentRuntimeContext(canonical.Messages)
if len(plan) == 0 {
return
}
canonical.Messages = append(slices.Clone(canonical.Messages), cloudAgentRuntimeMessage(cloudAgentRuntimeContext{Kind: cloudAgentContextPlan, Items: plan}))
}
func cloudAgentPlanRequiresFirstApproval(state *cloudAgentRuntime, call cloudAgentCall) bool {
if state == nil || state.Approval != nil || len(state.Plan) != 0 || state.Request.PermissionMode != "request_approval" {
return false
}
_, ok := cloudAgentPlanApprovalPreview(call)
return ok
}
func cloudAgentPlanNudgeMessage(state *cloudAgentRuntime, pendingTitle string) map[string]any {
return cloudAgentRuntimeMessage(cloudAgentRuntimeContext{
Kind: cloudAgentContextPendingPlan, PendingTitle: pendingTitle,
LatestUserMessage: cloudAgentLatestUserInstruction(state.Canonical.Messages),
})
}
func cloudAgentPlanApprovalPreview(call cloudAgentCall) (cloudAgentApprovalPreview, bool) {
var args struct {
Items []cloudAgentPlanItem `json:"items"`
}
if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil {
return cloudAgentApprovalPreview{}, false
}
items := make([]cloudAgentApprovalPreviewItem, 0, len(args.Items))
for _, entry := range args.Items {
title := strings.TrimSpace(entry.Title)
if title == "" {
continue
}
items = append(items, cloudAgentApprovalPreviewItem{Operation: "plan_step", Summary: truncateRunes(title, 240)})
}
if len(items) < 2 {
return cloudAgentApprovalPreview{}, false
}
return cloudAgentApprovalPreview{
Kind: "plan",
Title: "确认执行计划",
Description: fmt.Sprintf("Agent 把这轮任务拆成 %d 步。确认后才会开始执行;暂不执行则会让它改用别的做法。", len(items)),
Items: items,
}, true
}
func cloudAgentApplyPlanUpdate(state *cloudAgentRuntime, call cloudAgentCall) (any, error) {
var args struct {
Items []cloudAgentPlanItem `json:"items"`
}
if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil {
return nil, BadAuthRequest("工具参数必须是只含支持字段的JSON对象")
}
if len(args.Items) > 20 {
return nil, BadAuthRequest("待办清单最多 20 项")
}
seen := map[string]bool{}
for i := range args.Items {
args.Items[i].ID = strings.TrimSpace(args.Items[i].ID)
args.Items[i].Title = strings.TrimSpace(args.Items[i].Title)
if args.Items[i].ID == "" || args.Items[i].Title == "" {
return nil, BadAuthRequest("待办的 id 和 title 不能为空")
}
if seen[args.Items[i].ID] {
return nil, BadAuthRequest("待办 id 重复:" + args.Items[i].ID)
}
seen[args.Items[i].ID] = true
switch args.Items[i].Status {
case "pending", "doing", "done":
default:
args.Items[i].Status = "pending"
}
}
if !slices.Equal(state.Plan, args.Items) {
state.ActionNudged = false
}
state.Plan = args.Items
return map[string]any{"items": args.Items, "pendingTitles": cloudAgentPendingPlanItems(args.Items)}, nil
}
func cloudAgentAskUser(call cloudAgentCall) (any, error) {
var args struct {
Question string `json:"question"`
Options []struct {
Label string `json:"label"`
Detail string `json:"detail"`
} `json:"options"`
AllowFreeform *bool `json:"allowFreeform"`
}
if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil {
return nil, BadAuthRequest("工具参数必须是只含支持字段的JSON对象")
}
question := strings.TrimSpace(args.Question)
if question == "" {
return nil, BadAuthRequest("ask_user 必须给出 question:把要用户拍板的那一个问题一句话写清")
}
options := make([]map[string]any, 0, len(args.Options))
for _, option := range args.Options {
label := strings.TrimSpace(option.Label)
if label == "" {
continue
}
entry := map[string]any{"label": truncateRunes(label, 120)}
if detail := strings.TrimSpace(option.Detail); detail != "" {
entry["detail"] = truncateRunes(detail, 240)
}
options = append(options, entry)
}
if len(options) < 2 {
return nil, BadAuthRequest("ask_user 至少要给 2 个候选项;若你自己能定,直接做完继续,不要问")
}
if len(options) > 6 {
options = options[:6]
}
allowFreeform := true
if args.AllowFreeform != nil {
allowFreeform = *args.AllowFreeform
}
return map[string]any{
"phase": "question",
"question": truncateRunes(question, 400),
"options": options,
"allowFreeform": allowFreeform,
}, nil
}
// skipRemainingCloudAgentCalls 结束本批剩余调用(ask_user 之后本轮不再继续执行)。
// 末尾的 flush 让"本批前面的看图结果"仍能落在全部 tool 结果之后:这批调用到这里已经
// 完整(每个声明的 tool_call_id 都有回执),但本轮就此结束、不会再走 advanceCloudAgent
// 的兜底 flush,少了这一步缓冲的图片会被丢掉。
func skipRemainingCloudAgentCalls(runID string, state *cloudAgentRuntime) {
for index := state.CallIndex + 1; index < len(state.Calls); index++ {
cloudAgentToolResult(runID, state, state.Calls[index], map[string]any{"skipped": true}, BadAuthRequest("本轮已结束(等待用户决定),该调用未执行"))
}
cloudAgentFlushPendingImages(state)
}
func cloudAgentCanonicalWithPlan(state *cloudAgentRuntime) canonicalAgentRequest {
canonical := state.Canonical
attachCloudAgentPlan(&canonical, state.Plan)
return canonical
}