553 lines
22 KiB
Go
553 lines
22 KiB
Go
package app
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"errors"
|
||
"image"
|
||
"image/color"
|
||
"image/png"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
|
||
"gorm.io/gorm"
|
||
"yingce/backend/internal/model"
|
||
"yingce/backend/internal/repository"
|
||
)
|
||
|
||
func cloudAgentVisionFixture(t *testing.T) (*Service, *gorm.DB, []byte) {
|
||
t.Helper()
|
||
s, db, _ := agentMediaFixture(t)
|
||
capability := DefaultModelCapabilityConfigForModel(string(model.ChannelInterfaceChatCompletion), "text-test")
|
||
capability.Text.References.MaxImages = 2
|
||
capability.Text.References.MaxImageBytes = 1024 * 1024
|
||
if err := db.Model(&model.ChannelModel{}).Where("id = ?", "cm").Update("capability_config_json", mustEncodeModelCapabilityConfig(t, capability)).Error; err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
|
||
img.Set(0, 0, color.RGBA{R: 255, A: 255})
|
||
var encoded bytes.Buffer
|
||
if err := png.Encode(&encoded, img); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.MkdirAll(filepath.Join(s.dataDir, "resources"), 0700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, id := range []string{"ref-one", "ref-two"} {
|
||
if err := os.WriteFile(filepath.Join(s.dataDir, "resources", id+".png"), encoded.Bytes(), 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := db.Model(&model.Resource{}).Where("id = ?", id).Updates(map[string]any{"provider": "local", "object_key": id + ".png", "size": encoded.Len()}).Error; err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
return s, db, encoded.Bytes()
|
||
}
|
||
|
||
func cloudAgentVisionInput(t *testing.T, s *Service, nodes ...string) canvasGenerationInput {
|
||
t.Helper()
|
||
state := cloudAgentRuntime{Request: agentTestRequest()}
|
||
state.Request.VisionEnabled = true
|
||
var inspections []cloudAgentImageInspection
|
||
for _, node := range nodes {
|
||
call := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-"+node, map[string]any{"nodeId": node})
|
||
result, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, call)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
inspections = append(inspections, result.(cloudAgentImageInspection))
|
||
}
|
||
canonical := canonicalAgentRequest{ToolChoice: "auto", Messages: []map[string]any{{"role": "user", "content": cloudAgentImageContentParts(inspections...)}}}
|
||
refs, err := s.cloudAgentImageReferences("user", state.Request, &canonical)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
limits, err := s.cloudAgentVisionReferences(state.Request)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return canvasGenerationInput{Mode: "text", Prompt: "描述图片", ReferenceImages: refs, AgentRequests: &agentToolRequests{Canonical: &canonical}, Config: providerConfig{Model: "text-test", InterfaceType: string(model.ChannelInterfaceChatCompletion), APIKey: "test-only", CapabilityConfig: &ModelCapabilityConfig{Text: &TextCapabilityConfig{References: limits}}}}
|
||
}
|
||
|
||
func TestCloudAgentVisionWorkerSendsPixelsWithoutPublicURL(t *testing.T) {
|
||
for _, publicURL := range []string{"", "https://bucket.example.invalid"} {
|
||
t.Run("public-url-"+publicURL, func(t *testing.T) {
|
||
t.Setenv("CANVAS_PUBLIC_BASE_URL", publicURL)
|
||
t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1")
|
||
s, _, pixels := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat")
|
||
received := make(chan string, 1)
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
var body struct {
|
||
Messages []map[string]any `json:"messages"`
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||
t.Error(err)
|
||
w.WriteHeader(400)
|
||
return
|
||
}
|
||
var imageURL string
|
||
for _, message := range body.Messages {
|
||
parts, _ := message["content"].([]any)
|
||
for _, value := range parts {
|
||
part, _ := value.(map[string]any)
|
||
if image, ok := part["image_url"].(map[string]any); ok {
|
||
imageURL = stringField(image, "url")
|
||
}
|
||
}
|
||
}
|
||
received <- imageURL
|
||
w.Header().Set("Content-Type", "application/json")
|
||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"mock accepted pixels","tool_calls":[]}}]}`))
|
||
}))
|
||
defer server.Close()
|
||
input.Config.BaseURL = server.URL
|
||
stream := false
|
||
input.TextOptions.Stream = &stream
|
||
raw, err := json.Marshal(input)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if strings.Contains(string(raw), "base64,") || strings.Contains(string(raw), "signature=") {
|
||
t.Fatal("persistable task contains pixel bytes or signed URL")
|
||
}
|
||
result, err := s.processCanvasGenerationTask(context.Background(), "user", "agent-canvas", "canvas_text", "", string(raw))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if result["text"] != "mock accepted pixels" {
|
||
t.Fatalf("unexpected model response: %+v", result)
|
||
}
|
||
select {
|
||
case imageURL := <-received:
|
||
if imageURL != "data:image/png;base64,"+base64.StdEncoding.EncodeToString(pixels) {
|
||
t.Fatal("upstream did not receive exact stored image bytes")
|
||
}
|
||
default:
|
||
t.Fatal("worker did not call mock upstream")
|
||
}
|
||
if input.ReferenceImages[0].DataURL != "" || !strings.Contains(string(raw), "resource:ref-one") {
|
||
t.Fatal("worker mutated persistable resource placeholder")
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionHydrationPreservesProtocolImages(t *testing.T) {
|
||
s, _, pixels := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat", "hero")
|
||
original, _ := json.Marshal(input.AgentRequests)
|
||
if err := s.hydrateGenerationMedia("user", &input, providerMediaHydrationPolicy{preferURL: true}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
resolved, err := resolveAgentResourcePlaceholders(input, true)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
expanded, err := expandCanonicalAgentRequest(resolved.AgentRequests.Canonical, input.Config, true)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for name, body := range map[string]any{"chat": expanded.ChatCompletion, "responses": expanded.Responses, "claude": expanded.Claude, "gemini": expanded.Gemini} {
|
||
raw, _ := json.Marshal(body)
|
||
if strings.Count(string(raw), base64.StdEncoding.EncodeToString(pixels)) != 2 || strings.Contains(string(raw), "resource:") {
|
||
t.Errorf("%s lost image data or retained unresolved resource", name)
|
||
}
|
||
}
|
||
after, _ := json.Marshal(input.AgentRequests)
|
||
if !bytes.Equal(original, after) {
|
||
t.Fatal("protocol expansion mutated persisted canonical")
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionUsesStoredPixelsInsteadOfSuppliedData(t *testing.T) {
|
||
s, _, pixels := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat")
|
||
input.ReferenceImages[0].DataURL = "data:image/png;base64,bm90IHRoZSBzdG9yZWQgaW1hZ2U="
|
||
if err := s.hydrateGenerationMedia("user", &input, providerMediaHydrationPolicy{}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if input.ReferenceImages[0].DataURL != "data:image/png;base64,"+base64.StdEncoding.EncodeToString(pixels) {
|
||
t.Fatal("supplied data replaced the authorized resource's image bytes")
|
||
}
|
||
}
|
||
|
||
// 对照本次反馈使用的 Chat Completions 渠道在真实 worker 中的两条分支，
|
||
// 而不是只验证协议转换 helper：相同资源、模型和端点必须收到相同像素。
|
||
func TestCloudAgentVisionMatchesConnectedTextNode(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
protocol, endpoint, imageType, response string
|
||
}{
|
||
{"chat-completion", "/v1/chat/completions", "image_url", `{"choices":[{"message":{"content":"pixels received"}}]}`},
|
||
} {
|
||
t.Run(tc.protocol, func(t *testing.T) {
|
||
t.Setenv("CANVAS_PUBLIC_BASE_URL", "https://bucket.example.invalid")
|
||
t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1")
|
||
s, _, pixels := cloudAgentVisionFixture(t)
|
||
type capturedRequest struct {
|
||
path string
|
||
body map[string]any
|
||
}
|
||
received := make(chan capturedRequest, 2)
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
var body map[string]any
|
||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||
t.Error(err)
|
||
w.WriteHeader(http.StatusBadRequest)
|
||
return
|
||
}
|
||
received <- capturedRequest{r.URL.Path, body}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
_, _ = w.Write([]byte(tc.response))
|
||
}))
|
||
defer server.Close()
|
||
for _, agent := range []bool{false, true} {
|
||
input := cloudAgentVisionInput(t, s, "cat")
|
||
input.Config.InterfaceType = tc.protocol
|
||
input.Config.BaseURL = server.URL
|
||
stream := false
|
||
input.TextOptions.Stream = &stream
|
||
if !agent {
|
||
input.AgentRequests = nil
|
||
}
|
||
raw, err := json.Marshal(input)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
result, err := s.processCanvasGenerationTask(context.Background(), "user", "agent-canvas", "canvas_text", "", string(raw))
|
||
if err != nil {
|
||
t.Fatalf("agent=%t: %v", agent, err)
|
||
}
|
||
if result["text"] != "pixels received" {
|
||
t.Fatalf("agent=%t: unexpected mock response", agent)
|
||
}
|
||
select {
|
||
case request := <-received:
|
||
body, err := json.Marshal(request.body)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if request.path != tc.endpoint || request.body["model"] != input.Config.Model {
|
||
t.Fatalf("agent=%t: model or endpoint mismatch", agent)
|
||
}
|
||
if strings.Count(string(body), base64.StdEncoding.EncodeToString(pixels)) != 1 ||
|
||
!strings.Contains(string(body), `"type":"`+tc.imageType+`"`) ||
|
||
strings.Contains(string(body), "resource:") || strings.Contains(string(body), "bucket.example.invalid") {
|
||
t.Fatalf("agent=%t: upstream did not receive the same image bytes and protocol image part", agent)
|
||
}
|
||
default:
|
||
t.Fatalf("agent=%t: mock upstream was not called", agent)
|
||
}
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionRejectsInvalidResources(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
name, field string
|
||
value any
|
||
}{
|
||
{"foreign", "user_id", "other"},
|
||
{"not-ready", "status", "uploading"},
|
||
{"not-image", "mime_type", "text/plain"},
|
||
{"too-large", "size", int64(2 * 1024 * 1024)},
|
||
} {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
s, db, _ := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat")
|
||
if err := db.Model(&model.Resource{}).Where("id = ?", "ref-one").Update(tc.field, tc.value).Error; err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state := cloudAgentRuntime{Request: agentTestRequest()}
|
||
call := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect", map[string]any{"nodeId": "cat"})
|
||
if _, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, call); err == nil {
|
||
t.Fatal("inspection accepted invalid resource")
|
||
}
|
||
if _, err := s.cloudAgentImageReferences("user", state.Request, input.AgentRequests.Canonical); err == nil {
|
||
t.Fatal("task assembly accepted invalid resource")
|
||
}
|
||
if err := s.hydrateGenerationMedia("user", &input, providerMediaHydrationPolicy{}); err == nil {
|
||
t.Fatal("worker accepted resource changed after admission")
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionLimitsAndUnconfirmedState(t *testing.T) {
|
||
s, db, _ := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat", "hero")
|
||
canonical := input.AgentRequests.Canonical
|
||
original, _ := json.Marshal(canonical)
|
||
capability := DefaultModelCapabilityConfigForModel(string(model.ChannelInterfaceChatCompletion), "text-test")
|
||
capability.Text.References.MaxImages = 1
|
||
if err := db.Model(&model.ChannelModel{}).Where("id = ?", "cm").Update("capability_config_json", mustEncodeModelCapabilityConfig(t, capability)).Error; err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
copy := *canonical
|
||
refs, err := s.cloudAgentImageReferences("user", agentTestRequest(), ©)
|
||
if err != nil || len(refs) != 1 || refs[0].StorageKey != "resource:ref-two" {
|
||
t.Fatalf("did not retain newest image within model limit: %+v %v", refs, err)
|
||
}
|
||
after, _ := json.Marshal(canonical)
|
||
if !bytes.Equal(original, after) {
|
||
t.Fatal("limit projection mutated durable canonical")
|
||
}
|
||
state := cloudAgentRuntime{Request: agentTestRequest(), CreativeAnchor: cloudAgentCreativeAnchor{ReferenceAssets: []cloudAgentReferenceAnchor{{NodeID: "cat", VisualIdentity: "inspected", VisualNote: "无法读取图片"}}}}
|
||
state.markCanvasImageAttached("cat")
|
||
asset := state.CreativeAnchor.ReferenceAssets[0]
|
||
if asset.VisualIdentity != "unknown" || !asset.RequiresVisualInspection || asset.VisualNote != "" {
|
||
t.Fatal("attachment was treated as recognition")
|
||
}
|
||
state.PendingImageInspections = []cloudAgentImageInspection{{}}
|
||
call := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect", map[string]any{"nodeId": "hero"})
|
||
if _, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, call); err == nil {
|
||
t.Fatal("batch exceeded model image limit")
|
||
}
|
||
inherited := cloudAgentCreativeAnchor{ReferenceAssets: []cloudAgentReferenceAnchor{{NodeID: "cat", VisualIdentity: "inspected", VisualNote: "无法读取图片"}}}
|
||
canvas, err := s.repo.CanvasProjectForUser("user", "agent-canvas")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
anchor, err := cloudAgentCreativeAnchorForCanvas(s.repo, "user", canvas, "描述实际背景", &inherited)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, asset := range anchor.ReferenceAssets {
|
||
if asset.VisualIdentity != "unknown" || asset.VisualNote != "" {
|
||
t.Fatal("inherited unconfirmed observation")
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionRefreshCannotBypassPerImageLimit(t *testing.T) {
|
||
s, _, _ := cloudAgentVisionFixture(t)
|
||
state := cloudAgentRuntime{Request: agentTestRequest()}
|
||
state.Request.VisionEnabled = true
|
||
|
||
firstCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-1", map[string]any{"nodeId": "cat"})
|
||
first, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, firstCall)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state.markCanvasImageInspection("cat", strings.TrimSpace(first.(cloudAgentImageInspection).ImageURL) != "")
|
||
|
||
secondCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-2", map[string]any{"nodeId": "cat"})
|
||
second, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, secondCall)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state.markCanvasImageInspection("cat", strings.TrimSpace(second.(cloudAgentImageInspection).ImageURL) != "")
|
||
|
||
refreshCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-refresh", map[string]any{"nodeId": "cat", "refresh": true})
|
||
refreshed, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, refreshCall)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
inspection := refreshed.(cloudAgentImageInspection)
|
||
if inspection.ImageURL != "" {
|
||
t.Fatal("refresh=true bypassed the per-image inspection limit")
|
||
}
|
||
if inspection.Receipt["refreshIgnored"] != true || inspection.Receipt["repeat"] != true {
|
||
t.Fatalf("refresh protection receipt is incomplete: %+v", inspection.Receipt)
|
||
}
|
||
state.markCanvasImageInspection("cat", false)
|
||
if got := state.cloudAgentImageInspectionCount("cat"); got != 3 {
|
||
t.Fatalf("expected all successful inspection calls to count, got %d", got)
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionBlocksSameResourceInSameCanvasRevision(t *testing.T) {
|
||
s, _, _ := cloudAgentVisionFixture(t)
|
||
state := cloudAgentRuntime{Request: agentTestRequest()}
|
||
state.Request.VisionEnabled = true
|
||
|
||
firstCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-1", map[string]any{"nodeId": "cat"})
|
||
first, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, firstCall)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
inspection := first.(cloudAgentImageInspection)
|
||
if inspection.CacheKey == "" {
|
||
t.Fatal("first inspection did not produce a stable cache key")
|
||
}
|
||
state.ImageInspectionReads = map[string]int{inspection.CacheKey: 1}
|
||
|
||
refreshCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-refresh", map[string]any{"nodeId": "cat", "refresh": true})
|
||
second, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, refreshCall)
|
||
if err != nil {
|
||
t.Fatalf("repeated visual read should converge to a receipt, got: %v", err)
|
||
}
|
||
receipt := second.(cloudAgentImageInspection)
|
||
if receipt.ImageURL != "" || receipt.Receipt["repeat"] != true || receipt.CacheKey != inspection.CacheKey {
|
||
t.Fatalf("repeated visual read did not return a stable text receipt: %+v", receipt)
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionRepeatReadSoftNudgesRuntime(t *testing.T) {
|
||
s, _, _ := cloudAgentVisionFixture(t)
|
||
req := agentTestRequest()
|
||
req.VisionEnabled = true
|
||
root, err := s.CreateCloudAgentRun("user", req, "")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
run, err := s.repo.CloudAgent("user", root.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state, err := cloudAgentDecode(run)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state.Request.VisionEnabled = true
|
||
firstCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-first", map[string]any{"nodeId": "cat"})
|
||
first, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, firstCall)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cacheKey := first.(cloudAgentImageInspection).CacheKey
|
||
state.ImageInspectionReads = map[string]int{cacheKey: 1}
|
||
state.Calls = []cloudAgentCall{cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-repeat", map[string]any{"nodeId": "cat", "refresh": true})}
|
||
state.CallIndex = 0
|
||
if err := s.repo.MutateCloudAgent("user", run.ID, run.Revision, func(current *model.CloudAgentExecution, _ *repository.Repository) error {
|
||
return cloudAgentSave(current, &state)
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
run, err = s.repo.CloudAgent("user", root.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state, err = cloudAgentDecode(run)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := s.advanceCloudAgentTool(run, &state); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
output, err := s.CloudAgentRun("user", root.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if output.Status != "running" {
|
||
t.Fatalf("repeat visual read terminated the run: %+v", output)
|
||
}
|
||
hasGuard, hasFailure := false, false
|
||
for _, event := range output.Events {
|
||
hasGuard = hasGuard || event.Type == "read_loop_guard"
|
||
hasFailure = hasFailure || event.Type == "run_failed"
|
||
}
|
||
if !hasGuard || hasFailure {
|
||
t.Fatalf("repeat visual read did not preserve a soft guard: %+v", output.Events)
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionHasPerRunInspectionBudget(t *testing.T) {
|
||
s, _, _ := cloudAgentVisionFixture(t)
|
||
state := cloudAgentRuntime{Request: agentTestRequest(), ImageInspectCalls: cloudAgentMaxImageInspectionCallsPerRun - 1}
|
||
state.Request.VisionEnabled = true
|
||
call := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-last", map[string]any{"nodeId": "cat", "refresh": true})
|
||
result, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, call)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state.markCanvasImageInspection("cat", strings.TrimSpace(result.(cloudAgentImageInspection).ImageURL) != "")
|
||
if got := state.ImageInspectCalls; got != cloudAgentMaxImageInspectionCallsPerRun {
|
||
t.Fatalf("expected budget to be consumed at the boundary, got %d", got)
|
||
}
|
||
|
||
blockedCall := cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-blocked", map[string]any{"nodeId": "cat", "refresh": true})
|
||
if _, err := s.prepareCloudAgentImageInspection("user", "agent-canvas", &state, blockedCall); !errors.Is(err, errCloudAgentImageInspectionBudget) {
|
||
t.Fatalf("expected image inspection budget error, got %v", err)
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionBudgetSoftNudgesBeforeAnotherModelStep(t *testing.T) {
|
||
s, _, _ := cloudAgentVisionFixture(t)
|
||
req := agentTestRequest()
|
||
req.VisionEnabled = true
|
||
root, err := s.CreateCloudAgentRun("user", req, "")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
run, err := s.repo.CloudAgent("user", root.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state, err := cloudAgentDecode(run)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state.ActiveTaskID = ""
|
||
state.ImageInspectCalls = cloudAgentMaxImageInspectionCallsPerRun
|
||
state.Calls = []cloudAgentCall{cloudAgentStoryboardCall(t, "canvas_inspect_image", "inspect-blocked", map[string]any{"nodeId": "cat", "refresh": true})}
|
||
state.CallIndex = 0
|
||
if err := s.repo.MutateCloudAgent("user", run.ID, run.Revision, func(current *model.CloudAgentExecution, _ *repository.Repository) error {
|
||
return cloudAgentSave(current, &state)
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
run, err = s.repo.CloudAgent("user", root.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
state, err = cloudAgentDecode(run)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := s.advanceCloudAgentTool(run, &state); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
output, err := s.CloudAgentRun("user", root.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if output.Status != "running" {
|
||
t.Fatalf("inspection budget terminated the run: %+v", output)
|
||
}
|
||
hasGuard, hasFailure := false, false
|
||
for _, event := range output.Events {
|
||
hasGuard = hasGuard || event.Type == "read_loop_guard"
|
||
hasFailure = hasFailure || event.Type == "run_failed"
|
||
}
|
||
if !hasGuard || hasFailure {
|
||
t.Fatalf("inspection budget did not soft-nudge the next model request: %+v", output.Events)
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionRejectsOversizedActualBytes(t *testing.T) {
|
||
s, db, pixels := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat")
|
||
input.Config.CapabilityConfig.Text.References.MaxImageBytes = int64(len(pixels) - 1)
|
||
if err := db.Model(&model.Resource{}).Where("id = ?", "ref-one").Update("size", 1).Error; err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := s.hydrateGenerationMedia("user", &input, providerMediaHydrationPolicy{}); err == nil {
|
||
t.Fatal("worker trusted metadata instead of bounding actual bytes")
|
||
}
|
||
}
|
||
|
||
func TestCloudAgentVisionRejectsMissingFileAndExternalURL(t *testing.T) {
|
||
s, _, _ := cloudAgentVisionFixture(t)
|
||
input := cloudAgentVisionInput(t, s, "cat")
|
||
if err := os.Remove(filepath.Join(s.dataDir, "resources", "ref-one.png")); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := s.hydrateGenerationMedia("user", &input, providerMediaHydrationPolicy{}); err == nil {
|
||
t.Fatal("missing pixels were treated as a successful inspection")
|
||
}
|
||
for _, url := range []string{"https://example.invalid/image.png", "http://127.0.0.1/private", "data:image/png;base64,YQ==", "resource:private"} {
|
||
canonical := canonicalAgentRequest{Messages: []map[string]any{{"role": "user", "content": []any{map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}}}}}}
|
||
if _, err := s.cloudAgentImageReferences("user", agentTestRequest(), &canonical); err == nil {
|
||
t.Errorf("accepted unapproved image source %q", url)
|
||
}
|
||
}
|
||
}
|