From e70430e3e2bfd993a38feca2fa558892f3f710b6 Mon Sep 17 00:00:00 2001 From: ddcat666 <26564566+ddcat666@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:21:36 +0800 Subject: [PATCH] =?UTF-8?q?feat(canvas):=20=E4=BA=91=E7=AB=AF=20Agent=20?= =?UTF-8?q?=E7=94=BB=E5=B8=83=20-=20=E6=94=AF=E6=8C=81=E5=88=86=E9=95=9C?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E3=80=81=E8=84=9A=E6=9C=AC=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E4=B8=8E=E8=B5=84=E4=BA=A7=E5=85=B3=E7=B3=BB=E6=8C=81=E4=B9=85?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/cloud_agent_approval_preview.go | 220 +++++++++++++++++- .../app/cloud_agent_approval_preview_test.go | 62 +++++ .../app/cloud_agent_canvas_events_test.go | 27 +++ .../internal/app/cloud_agent_canvas_state.go | 2 +- .../internal/app/cloud_agent_contract_test.go | 35 +++ .../internal/app/cloud_agent_storyboard.go | 11 +- .../app/cloud_agent_storyboard_test.go | 91 ++++++++ backend/internal/app/cloud_agent_tools.go | 24 +- .../internal/app/cloud_agent_tools_canvas.go | 51 +++- backend/internal/canvas/capability/builtin.go | 19 +- 10 files changed, 505 insertions(+), 37 deletions(-) diff --git a/backend/internal/app/cloud_agent_approval_preview.go b/backend/internal/app/cloud_agent_approval_preview.go index 25ae1d021..d029e2606 100644 --- a/backend/internal/app/cloud_agent_approval_preview.go +++ b/backend/internal/app/cloud_agent_approval_preview.go @@ -20,6 +20,7 @@ import ( // ownership, node registry, locks and budget after approval. type cloudAgentApprovalPreview struct { Kind string `json:"kind"` + Status string `json:"status,omitempty"` Title string `json:"title"` Description string `json:"description"` Items []cloudAgentApprovalPreviewItem `json:"items"` @@ -150,32 +151,52 @@ func applyCloudAgentCanvasPlan(doc map[string]any, ops []agentCanvasOp) ([]cloud }) case "connect_nodes": if err := validateCloudAgentID(op.FromNodeID, "来源节点 ID", 80); err != nil { - return nil, err + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d].fromNodeId", opIndex), "invalid_value", cloudAgentSafeToolError(err)) } if err := validateCloudAgentID(op.ToNodeID, "目标节点 ID", 80); err != nil { - return nil, err + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d].toNodeId", opIndex), "invalid_value", cloudAgentSafeToolError(err)) } fromIndex, toIndex := cloudAgentNodeIndex(nodes, op.FromNodeID), cloudAgentNodeIndex(nodes, op.ToNodeID) - if fromIndex < 0 || toIndex < 0 || op.FromNodeID == op.ToNodeID { - return nil, BadAuthRequest("连线端点不存在或指向自身") + switch { + case fromIndex < 0: + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d].fromNodeId", opIndex), "not_found", "连线来源节点不在当前画布；请重新读取画布并使用真实节点 ID") + case toIndex < 0: + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d].toNodeId", opIndex), "not_found", "连线目标节点不在当前画布；请重新读取画布并使用真实节点 ID") + case op.FromNodeID == op.ToNodeID: + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d].toNodeId", opIndex), "invalid_value", "连线不能指向自身；请选择另一个目标节点") } - if err := validateCloudAgentConnection(nodes, op.FromNodeID, op.ToNodeID, edges); err != nil { + if err := validateCloudAgentConnectionWithHandles(nodes, op.FromNodeID, op.ToNodeID, op.FromHandleID, op.ToHandleID, edges); err != nil { return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d]", opIndex), "invalid_connection", cloudAgentSafeToolError(err)) } for _, edge := range edges { - if stringValue(edge["id"]) == op.ID || (stringValue(edge["fromNodeId"]) == op.FromNodeID && stringValue(edge["toNodeId"]) == op.ToNodeID) { - return nil, BadAuthRequest("连线重复") + if stringValue(edge["id"]) == op.ID { + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d].id", opIndex), "duplicate", "连线 ID 已存在；请为新连线指定未使用的 ID") + } + if stringValue(edge["fromNodeId"]) == op.FromNodeID && stringValue(edge["toNodeId"]) == op.ToNodeID && stringValue(edge["fromHandleId"]) == op.FromHandleID && stringValue(edge["toHandleId"]) == op.ToHandleID { + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d]", opIndex), "duplicate", "这两个节点与 handle 之间已有连线；请移除重复操作") } } fromCapability, _ := cloudAgentNodeCapabilityForNode(nodes[fromIndex]) toCapability, _ := cloudAgentNodeCapabilityForNode(nodes[toIndex]) fromTitle := cloudAgentApprovalNodeTitle(nodes[fromIndex], fromCapability.Label) toTitle := cloudAgentApprovalNodeTitle(nodes[toIndex], toCapability.Label) - edges = append(edges, map[string]any{"id": op.ID, "fromNodeId": op.FromNodeID, "toNodeId": op.ToNodeID}) + edge := map[string]any{"id": op.ID, "fromNodeId": op.FromNodeID, "toNodeId": op.ToNodeID} + if op.FromHandleID != "" { + edge["fromHandleId"] = op.FromHandleID + } + if op.ToHandleID != "" { + edge["toHandleId"] = op.ToHandleID + } + edges = append(edges, edge) + if err := applyCloudAgentStoryboardConnection(nodes, op.FromNodeID, op.ToNodeID, op.FromHandleID, op.ToHandleID); err != nil { + return nil, cloudAgentFieldError(fmt.Sprintf("ops[%d]", opIndex), "invalid_connection", cloudAgentSafeToolError(err)) + } + details := cloudAgentStoryboardConnectionDetails(op.FromHandleID, op.ToHandleID) items = append(items, cloudAgentApprovalPreviewItem{ Operation: "connect_nodes", NodeID: op.FromNodeID, NodeTitle: fromTitle, NodeType: fromCapability.Type, NodeTypeLabel: fromCapability.Label, TargetNodeID: op.ToNodeID, TargetNodeTitle: toTitle, TargetNodeType: toCapability.Type, + Details: details, Summary: fmt.Sprintf("建立《%s》→《%s》的引用连线", fromTitle, toTitle), }) case "update_node": @@ -220,6 +241,189 @@ func applyCloudAgentCanvasPlan(doc map[string]any, ops []agentCanvasOp) ([]cloud return items, nil } +func applyCloudAgentStoryboardConnection(nodes []map[string]any, fromNodeID, toNodeID, fromHandleID, toHandleID string) error { + if fromHandleID == "" && toHandleID == "" { + return nil + } + from := cloudAgentNodeByID(nodes, fromNodeID) + to := cloudAgentNodeByID(nodes, toNodeID) + if from == nil || to == nil { + return BadAuthRequest("分镜连线端点不存在") + } + scriptNode, linkedNode, handleID := from, to, fromHandleID + scriptIsSource := true + if handleID == "" { + scriptNode, linkedNode, handleID = to, from, toHandleID + scriptIsSource = false + } + if stringValue(scriptNode["type"]) != "script" { + return BadAuthRequest("分镜 handle 必须挂在分镜脚本节点上") + } + metadata, ok := scriptNode["metadata"].(map[string]any) + if !ok { + return BadAuthRequest("分镜节点数据格式无效") + } + storyboard, ok := metadata["storyboard"].(map[string]any) + if !ok { + return BadAuthRequest("分镜节点缺少结构化表格") + } + if handleID == "storyboard:context" { + ids := stringIDs(storyboard["referenceNodeIds"]) + linkedID := stringValue(linkedNode["id"]) + if linkedID != "" && !containsStoryboardString(ids, linkedID) { + storyboard["referenceNodeIds"] = append(idsAsAny(ids), linkedID) + } + return nil + } + rowID := strings.TrimPrefix(handleID, "row:") + var row map[string]any + for _, candidate := range creationMaps(storyboard["rows"]) { + if stringValue(candidate["id"]) == rowID { + row = candidate + break + } + } + if row == nil { + return BadAuthRequest("分镜行不存在，请先读取最新分镜行 ID") + } + if scriptIsSource { + switch stringValue(linkedNode["type"]) { + case "image": + row["imageNodeId"] = linkedNode["id"] + case "video": + row["videoNodeId"] = linkedNode["id"] + } + return nil + } + binding, ok := cloudAgentStoryboardBinding(linkedNode) + if !ok { + return nil + } + bindings := creationMaps(row["assetBindings"]) + for _, existing := range bindings { + if stringValue(existing["nodeId"]) == stringValue(linkedNode["id"]) { + return nil + } + } + row["assetBindings"] = append(bindingsAsAny(bindings), binding) + return nil +} + +func cloudAgentStoryboardConnectionDetails(fromHandleID, toHandleID string) []string { + handleID := fromHandleID + if handleID == "" { + handleID = toHandleID + } + if handleID == "storyboard:context" { + return []string{"挂载到分镜全局设定"} + } + if strings.HasPrefix(handleID, "row:") { + return []string{fmt.Sprintf("挂载到镜头行 %s", strings.TrimPrefix(handleID, "row:"))} + } + return nil +} + +func cloudAgentStoryboardBinding(node map[string]any) (map[string]any, bool) { + metadata, _ := node["metadata"].(map[string]any) + role := "" + switch { + case stringValue(metadata["workflowKind"]) == "character" || stringValue(metadata["assetCategory"]) == "character": + role = "character" + case stringValue(node["type"]) == "audio": + role = "audio" + case stringValue(node["type"]) == "video": + role = "motion" + case stringValue(metadata["assetCategory"]) == "environment": + role = "environment" + case stringValue(metadata["assetCategory"]) == "prop": + role = "prop" + case stringValue(node["type"]) == "image": + role = "style" + } + if role == "" || stringValue(node["id"]) == "" { + return nil, false + } + priority := 60 + switch role { + case "character": + priority = 100 + case "environment": + priority = 90 + case "prop", "wardrobe", "weapon": + priority = 80 + case "motion", "audio": + priority = 70 + } + return map[string]any{"nodeId": node["id"], "role": role, "priority": float64(priority)}, true +} + +func cloudAgentNodeByID(nodes []map[string]any, id string) map[string]any { + for _, node := range nodes { + if stringValue(node["id"]) == id { + return node + } + } + return nil +} + +func stringIDs(value any) []string { + values, _ := value.([]any) + ids := make([]string, 0, len(values)) + for _, item := range values { + if id := stringValue(item); id != "" { + ids = append(ids, id) + } + } + return ids +} + +func idsAsAny(values []string) []any { + out := make([]any, len(values)) + for index, value := range values { + out[index] = value + } + return out +} + +func bindingsAsAny(values []map[string]any) []any { + out := make([]any, len(values)) + for index, value := range values { + out[index] = value + } + return out +} + +func containsStoryboardString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func cloudAgentCanvasAppliedPreview(preview cloudAgentApprovalPreview) cloudAgentApprovalPreview { + applied := preview + counts := map[string]int{} + for _, item := range preview.Items { + counts[item.Operation]++ + } + parts := make([]string, 0, 3) + if counts["add_node"] > 0 { + parts = append(parts, fmt.Sprintf("新增 %d 个节点", counts["add_node"])) + } + if counts["update_node"] > 0 { + parts = append(parts, fmt.Sprintf("修改 %d 个节点", counts["update_node"])) + } + if counts["connect_nodes"] > 0 { + parts = append(parts, fmt.Sprintf("建立 %d 条引用连线", counts["connect_nodes"])) + } + applied.Status = "applied" + applied.Title = "画布修改已写入" + applied.Description = fmt.Sprintf("画布修改已写入：%s。", strings.Join(parts, "，")) + return applied +} + func cloudAgentCanvasApprovalPreview(items []cloudAgentApprovalPreviewItem) cloudAgentApprovalPreview { counts := map[string]int{} for _, item := range items { diff --git a/backend/internal/app/cloud_agent_approval_preview_test.go b/backend/internal/app/cloud_agent_approval_preview_test.go index 033f9582c..e02f27d03 100644 --- a/backend/internal/app/cloud_agent_approval_preview_test.go +++ b/backend/internal/app/cloud_agent_approval_preview_test.go @@ -42,6 +42,46 @@ func TestCloudAgentCanvasApprovalPreviewDescribesUpdateTargetAndFields(t *testin } } +func TestCloudAgentStoryboardConnectionsUseRowHandlesAndPersistBindings(t *testing.T) { + doc, err := creationDocument(`{"nodes":[{"id":"script-1","type":"script","title":"分镜","metadata":{"storyboard":{"rows":[{"id":"row-1","shotNumber":1,"durationSeconds":5,"assetBindings":[],"imageNodeId":""}],"referenceNodeIds":[]}}},{"id":"image-1","type":"image","title":"环境图","metadata":{"content":"ready","assetCategory":"environment"}},{"id":"video-1","type":"video","title":"镜头视频","metadata":{}}],"connections":[]}`) + if err != nil { + t.Fatal(err) + } + items, err := applyCloudAgentCanvasPlan(doc, []agentCanvasOp{ + {Type: "update_node", ID: "script-1", Patch: map[string]any{"title": "夜雨分镜", "x": 120.0, "y": 240.0}}, + {Type: "connect_nodes", ID: "edge-asset", FromNodeID: "image-1", ToNodeID: "script-1", ToHandleID: "row:row-1"}, + {Type: "connect_nodes", ID: "edge-output", FromNodeID: "script-1", ToNodeID: "video-1", FromHandleID: "row:row-1"}, + }) + if err != nil { + t.Fatal(err) + } + if len(items) != 3 || len(items[1].Details) != 1 || !strings.Contains(items[1].Details[0], "row-1") { + t.Fatalf("row handle was not reflected in approval preview: %+v", items) + } + connections := creationMaps(doc["connections"]) + if len(connections) != 2 || stringValue(connections[0]["toHandleId"]) != "row:row-1" || stringValue(connections[1]["fromHandleId"]) != "row:row-1" { + t.Fatalf("row handles were not persisted: %+v", connections) + } + script := cloudAgentNodeByID(creationMaps(doc["nodes"]), "script-1") + if script["title"] != "夜雨分镜" { + t.Fatalf("script title update was not applied: %#v", script) + } + position := script["position"].(map[string]any) + if position["x"] != 120.0 || position["y"] != 240.0 { + t.Fatalf("script position update was not applied: %#v", position) + } + metadata := script["metadata"].(map[string]any) + storyboard := metadata["storyboard"].(map[string]any) + row := creationMaps(storyboard["rows"])[0] + if stringValue(row["imageNodeId"]) != "" { + t.Fatalf("asset connection should not be treated as storyboard output: %#v", row) + } + bindings := creationMaps(row["assetBindings"]) + if len(bindings) != 1 || stringValue(bindings[0]["nodeId"]) != "image-1" || stringValue(bindings[0]["role"]) != "environment" { + t.Fatalf("row asset binding was not persisted: %#v", row) + } +} + func TestCloudAgentCanvasApprovalPreviewDescribesConnectionsAndMixedOperations(t *testing.T) { doc, err := creationDocument(`{"nodes":[{"id":"image-1","type":"image","title":"参考图"},{"id":"video-1","type":"video","title":"视频镜头"}],"connections":[]}`) if err != nil { @@ -85,6 +125,28 @@ func TestCloudAgentCanvasApprovalPreviewRejectsUnknownTargetInsteadOfFallingBack // 漏 patch 必须是**可恢复的参数错误**：运行期会把它当工具结果回给模型重试， // 而不是把整轮判死；未知操作类型（例如删除）仍按准入失败终止。 +func TestCloudAgentCanvasApprovalPreviewTreatsInvalidConnectionEndpointsAsArgumentErrors(t *testing.T) { + for _, tc := range []struct { + name, from, to, field string + }{ + {"missing source", "missing", "image-1", "ops[0].fromNodeId"}, + {"missing target", "image-1", "missing", "ops[0].toNodeId"}, + {"self connection", "image-1", "image-1", "ops[0].toNodeId"}, + } { + t.Run(tc.name, func(t *testing.T) { + doc, err := creationDocument(`{"nodes":[{"id":"image-1","type":"image","title":"参考图"}],"connections":[]}`) + if err != nil { + t.Fatal(err) + } + _, err = applyCloudAgentCanvasPlan(doc, []agentCanvasOp{{Type: "connect_nodes", ID: "edge-1", FromNodeID: tc.from, ToNodeID: tc.to}}) + var fieldErr *cloudAgentFieldArgumentError + if !errors.As(err, &fieldErr) || fieldErr.Field != tc.field { + t.Fatalf("expected repairable field error %q, got %v", tc.field, err) + } + }) + } +} + func TestCloudAgentCanvasApprovalPreviewTreatsMissingPatchAsArgumentError(t *testing.T) { doc, err := creationDocument(`{"nodes":[{"id":"image-1","type":"image","title":"参考图"}],"connections":[]}`) if err != nil { diff --git a/backend/internal/app/cloud_agent_canvas_events_test.go b/backend/internal/app/cloud_agent_canvas_events_test.go index 29a81e715..909d0f436 100644 --- a/backend/internal/app/cloud_agent_canvas_events_test.go +++ b/backend/internal/app/cloud_agent_canvas_events_test.go @@ -193,6 +193,33 @@ func TestCloudAgentCanvasOperationTraceSharesToolCallID(t *testing.T) { if trace["callId"] != call.ID || last.Type != "tool_completed" || last.Payload["callId"] != call.ID { t.Fatalf("canvas delta and tool result cannot be deduplicated: %+v / %+v", trace, last) } + result := last.Payload["result"].(map[string]any) + if result["committed"] != true { + t.Fatalf("canvas_apply_ops result did not acknowledge persistence: %+v", result) + } + preview := result["preview"].(map[string]any) + if preview["status"] != "applied" || strings.Contains(stringValue(preview["description"]), "批准后才会写入") { + t.Fatalf("canvas_apply_ops result still looks like an approval preview: %+v", preview) + } + canvas, err := s.repo.CanvasProjectForUser("user", "agent-canvas") + if err != nil { + t.Fatal(err) + } + doc, err := creationDocument(canvas.PayloadJSON) + if err != nil { + t.Fatal(err) + } + var cat map[string]any + for _, node := range creationMaps(doc["nodes"]) { + if stringValue(node["id"]) == "cat" { + cat = node + break + } + } + if cat == nil || stringValue(cat["title"]) != "叮当猫飞行参考" || stringValue(cat["metadata"].(map[string]any)["composerContent"]) != "下一版参考图提示词" { + t.Fatalf("canvas_apply_ops did not persist the requested draft: %+v", cat) + } + actions := creationMaps(trace["actions"]) byActionAndNode := map[string]map[string]any{} for _, action := range actions { diff --git a/backend/internal/app/cloud_agent_canvas_state.go b/backend/internal/app/cloud_agent_canvas_state.go index b81b7251a..4de772cf2 100644 --- a/backend/internal/app/cloud_agent_canvas_state.go +++ b/backend/internal/app/cloud_agent_canvas_state.go @@ -208,7 +208,7 @@ func cloudAgentCanvasStateSelected(repo *repository.Repository, userID, canvasID candidateNodes = append(candidateNodes, node) } } - limit := 2000 + limit := 320 if len(ids) > 0 || len(focus) > 0 { limit = 16000 } diff --git a/backend/internal/app/cloud_agent_contract_test.go b/backend/internal/app/cloud_agent_contract_test.go index 694e437eb..76b93a999 100644 --- a/backend/internal/app/cloud_agent_contract_test.go +++ b/backend/internal/app/cloud_agent_contract_test.go @@ -169,6 +169,19 @@ func TestCloudAgentToolsFollowCanvasCapabilityRegistry(t *testing.T) { } } } + script := nodeTypeDefinitions["script"] + if script == nil || script["canSource"] != true || script["canTarget"] != true || script["canReference"] != false || script["inputKind"] != "text" { + t.Fatalf("script capability is incomplete: %#v", script) + } + updateFields, _ := script["updateFields"].(map[string]any) + for _, field := range []string{"title", "x", "y"} { + if _, ok := updateFields[field]; !ok { + t.Fatalf("script update field %s is missing: %#v", field, script) + } + } + if accepted, _ := script["acceptedInputKinds"].([]string); !reflect.DeepEqual(accepted, []string{"audio", "character", "image", "text", "video"}) { + t.Fatalf("script accepted input kinds are incomplete: %#v", script["acceptedInputKinds"]) + } if got := functions["generate_media"]["properties"].(map[string]any)["mode"].(map[string]any)["enum"]; !reflect.DeepEqual(got, cloudAgentGenerationModeNames()) { t.Fatalf("generation modes differ from implemented adapters: %v", got) } @@ -353,6 +366,28 @@ func TestCloudAgentProjectionRejectsMissingAdapterAndOpaqueMetadata(t *testing.T } } +func TestCloudAgentCanvasStateBoundsDefaultBodyProjection(t *testing.T) { + body := strings.Repeat("长正文 ", 999) + "长正文" + doc := map[string]any{"nodes": []map[string]any{{"id": "markdown", "type": "markdown", "title": "长文档", "metadata": map[string]any{"content": body}}}, "connections": []map[string]any{}} + view, err := cloudAgentCanvasState(nil, "user", "agent-canvas", doc, 0, nil, 0) + if err != nil { + t.Fatal(err) + } + item := view.(map[string]any)["nodes"].([]any)[0].(map[string]any) + content := stringValue(item["content"]) + if len([]rune(content)) > 323 || item["contentTruncated"] != true { + t.Fatalf("default canvas read was not bounded: len=%d item=%v", len([]rune(content)), item) + } + precise, err := cloudAgentCanvasState(nil, "user", "agent-canvas", doc, 0, []string{"markdown"}, 0) + if err != nil { + t.Fatal(err) + } + preciseItem := precise.(map[string]any)["nodes"].([]any)[0].(map[string]any) + if stringValue(preciseItem["content"]) != body || preciseItem["contentTruncated"] != nil { + t.Fatalf("precise canvas read lost full content: got=%d want=%d item=%v", len([]rune(stringValue(preciseItem["content"]))), len([]rune(body)), preciseItem) + } +} + func TestCloudAgentCanvasStateDoesNotForwardUnknownObjectFields(t *testing.T) { doc := map[string]any{"nodes": []map[string]any{{ "id": "safe", "type": "text", "title": "镜头", diff --git a/backend/internal/app/cloud_agent_storyboard.go b/backend/internal/app/cloud_agent_storyboard.go index 428810d8a..f15182953 100644 --- a/backend/internal/app/cloud_agent_storyboard.go +++ b/backend/internal/app/cloud_agent_storyboard.go @@ -77,7 +77,7 @@ func cloudAgentStoryboardPatchSchema() map[string]any { return schema } -func validateCloudAgentStoryboardRow(row map[string]any, requireDescription bool) error { +func validateCloudAgentStoryboardRow(row map[string]any, requireDescription bool, fieldPath string) error { if row == nil { return BadAuthRequest("分镜行不能为空") } @@ -102,8 +102,9 @@ func validateCloudAgentStoryboardRow(row map[string]any, requireDescription bool if requireDescription { plot := strings.TrimSpace(stringValue(row["plotDescription"])) motion := strings.TrimSpace(stringValue(row["videoMotionPrompt"])) - if plot == "" && motion == "" { - return BadAuthRequest("新增镜头需要画面描述或视频提示词") + imagePrompt := strings.TrimSpace(stringValue(row["imageGenerationPrompt"])) + if plot == "" && motion == "" && imagePrompt == "" { + return cloudAgentFieldError(fieldPath, "required", "新增镜头需要画面描述、首帧提示词或视频提示词") } } return nil @@ -115,7 +116,7 @@ func normalizeCloudAgentStoryboardRows(rows []map[string]any, userID, nodeID, se } out := make([]any, 0, len(rows)) for index, input := range rows { - if err := validateCloudAgentStoryboardRow(input, true); err != nil { + if err := validateCloudAgentStoryboardRow(input, true, fmt.Sprintf("rows[%d]", index)); err != nil { return nil, err } row := cloudAgentStoryboardRowDefaults() @@ -294,7 +295,7 @@ func prepareCloudAgentStoryboardEdit(repo *repository.Repository, userID, canvas if len(rows) >= maxCloudAgentStoryboardRows { return nil, BadAuthRequest("单个分镜表最多100个镜头") } - if err := validateCloudAgentStoryboardRow(patch, true); err != nil { + if err := validateCloudAgentStoryboardRow(patch, true, "patch"); err != nil { return nil, err } row := cloudAgentStoryboardRowDefaults() diff --git a/backend/internal/app/cloud_agent_storyboard_test.go b/backend/internal/app/cloud_agent_storyboard_test.go index 49400403c..95f767a39 100644 --- a/backend/internal/app/cloud_agent_storyboard_test.go +++ b/backend/internal/app/cloud_agent_storyboard_test.go @@ -71,6 +71,97 @@ func createCloudAgentStoryboardForTest(t *testing.T, s *Service, canvas *model.C return rows } +func TestCloudAgentStoryboardAcceptsImagePromptAsDescription(t *testing.T) { + if err := validateCloudAgentStoryboardRow(map[string]any{ + "durationSeconds": float64(3), + "imageGenerationPrompt": "首帧：雨夜街口，主角从巷中走出", + }, true, "rows[0]"); err != nil { + t.Fatalf("a standalone imageGenerationPrompt should describe the shot: %v", err) + } + if err := validateCloudAgentStoryboardRow(map[string]any{"durationSeconds": float64(3)}, true, "rows[0]"); err == nil || !strings.Contains(err.Error(), "首帧提示词") { + t.Fatalf("missing shot description should explain accepted fields: %v", err) + } else if fieldErr, ok := err.(*cloudAgentFieldArgumentError); !ok || fieldErr.Field != "rows[0]" || fieldErr.Issue != "required" { + t.Fatalf("missing shot description should be a repairable field error, got %T: %v", err, err) + } +} + +func TestCloudAgentStoryboardArgumentFailureContinuesRun(t *testing.T) { + s, canvas := cloudAgentStoryboardFixture(t) + req := agentTestRequest() + req.PermissionMode = "request_approval" + root, err := s.CreateCloudAgentRun("user", req, "") + if err != nil { + t.Fatal(err) + } + doc, err := creationDocument(canvas.PayloadJSON) + if err != nil { + t.Fatal(err) + } + call := cloudAgentStoryboardCall(t, "canvas_create_storyboard", "storyboard-invalid-call", map[string]any{ + "snapshotHash": cloudAgentCanvasHash(doc), + "nodeId": "storyboard-invalid", + "title": "缺少描述的分镜", + "rows": []map[string]any{{"durationSeconds": 4.0}}, + }) + run, err := s.repo.CloudAgent("user", root.ID) + if err != nil { + t.Fatal(err) + } + state, err := cloudAgentDecode(run) + if err != nil { + t.Fatal(err) + } + state.ActiveTaskID = "" + state.Calls = []cloudAgentCall{call} + state.CallIndex = 0 + if err := s.repo.MutateCloudAgent("user", run.ID, run.Revision, func(current *model.CloudAgentExecution, _ *repository.Repository) error { + return cloudAgentSave(current, &state) + }); err != nil { + t.Fatal(err) + } + run, err = s.repo.CloudAgent("user", run.ID) + if err != nil { + t.Fatal(err) + } + if err := s.advanceCloudAgentTool(run, &state); err != nil { + t.Fatal(err) + } + stored, err := s.repo.CloudAgent("user", run.ID) + if err != nil { + t.Fatal(err) + } + state, err = cloudAgentDecode(stored) + if err != nil { + t.Fatal(err) + } + if stored.Status != "running" { + t.Fatalf("repairable storyboard argument failure terminated run: status=%s message=%s", stored.Status, stored.FailureMessage) + } + var failed *CloudAgentEvent + for i := range state.Events { + if state.Events[i].Type == "tool_failed" { + failed = &state.Events[i] + } + if state.Events[i].Type == "run_failed" { + t.Fatalf("argument error emitted run_failed: %+v", state.Events[i]) + } + } + if failed == nil { + t.Fatal("missing tool_failed event") + } + result, _ := failed.Payload["result"].(map[string]any) + if result["field"] != "rows[0]" || result["issue"] != "required" || result["requiredAction"] != "fix_arguments" { + t.Fatalf("missing actionable storyboard field feedback: %+v", result) + } + storedCanvas, err := s.repo.CanvasProjectForUser("user", canvas.ID) + if err != nil { + t.Fatal(err) + } + if strings.Contains(storedCanvas.PayloadJSON, "storyboard-invalid") { + t.Fatal("invalid storyboard arguments wrote to the canvas") + } +} + func TestCloudAgentNodeCapabilityCardsExplainStoryboardTradeoffs(t *testing.T) { result := cloudAgentNodeTypes() if result["schemaVersion"] != 2 { diff --git a/backend/internal/app/cloud_agent_tools.go b/backend/internal/app/cloud_agent_tools.go index 74af44de8..2ecf8b1b5 100644 --- a/backend/internal/app/cloud_agent_tools.go +++ b/backend/internal/app/cloud_agent_tools.go @@ -232,16 +232,18 @@ func compileCloudAgentTools(req CloudAgentRequest, includeProfileTool bool) []ma "globalPrompt": str("set_global_prompt 使用；非空时覆盖各任务提示词，空字符串清除全局提示词"), }, "snapshotHash", "nodeId", "action") opProperties := map[string]any{ - "type": map[string]any{"type": "string", "enum": []string{"add_node", "update_node", "connect_nodes"}, "description": "必填的操作类型；新增节点必须传 add_node，nodeType 不能代替本字段"}, - "id": str("节点或连线唯一ID"), - "nodeType": map[string]any{"type": "string", "enum": cloudAgentNodeTypeNames()}, - "title": str("标题；更新操作可选"), - "content": str("文本正文或媒体提示词；更新操作可选"), - "patch": cloudAgentPatchSchema(), - "fromNodeId": str("连线来源节点ID"), - "toNodeId": str("连线目标节点ID"), - "x": map[string]any{"type": "number"}, - "y": map[string]any{"type": "number"}, + "type": map[string]any{"type": "string", "enum": []string{"add_node", "update_node", "connect_nodes"}, "description": "必填的操作类型；新增节点必须传 add_node，nodeType 不能代替本字段"}, + "id": str("节点或连线唯一ID"), + "nodeType": map[string]any{"type": "string", "enum": cloudAgentNodeTypeNames()}, + "title": str("标题；更新操作可选"), + "content": str("文本正文或媒体提示词；更新操作可选"), + "patch": cloudAgentPatchSchema(), + "fromNodeId": str("连线来源节点ID"), + "toNodeId": str("连线目标节点ID"), + "fromHandleId": str("分镜来源 handle：row: 或 storyboard:context"), + "toHandleId": str("分镜目标 handle：row: 或 storyboard:context"), + "x": map[string]any{"type": "number"}, + "y": map[string]any{"type": "number"}, } opItem := map[string]any{ "type": "object", @@ -254,7 +256,7 @@ func compileCloudAgentTools(req CloudAgentRequest, includeProfileTool bool) []ma {"properties": map[string]any{"type": map[string]any{"const": "connect_nodes"}}, "required": []string{"fromNodeId", "toNodeId"}}, }, } - add("canvas_apply_ops", "创建空白节点、修改提示词或建立引用连线，不提交生成任务、不产生生成费用；先读取画布并传 snapshotHash。提交媒体生成使用 generate_media。每次最多20项，禁止删除、任意 metadata 和媒体 URL。每项都需要 type 和 id：add_node 还需要 nodeType（可给 x/y 指定位置；省略坐标时服务端按画布内容自动落位，不会叠在原点），update_node 还需要按节点能力清单填写 patch（可含 x/y 移动节点），connect_nodes 还需要 fromNodeId 与 toNodeId。连线是生成输入关系，不会改变已提交任务的输入；来源须 canSource，目标须 canTarget 且接受来源 inputKind，能力以注册表为准。批量整理位置用 canvas_arrange_nodes，不要用几十项 update_node 手工算坐标。", map[string]any{"snapshotHash": str("canvas_get_state返回的snapshotHash"), "ops": map[string]any{"type": "array", "maxItems": 20, "items": opItem}}, "snapshotHash", "ops") + add("canvas_apply_ops", "创建空白节点、修改提示词或建立引用连线，不提交生成任务、不产生生成费用；先读取画布并传 snapshotHash。提交媒体生成使用 generate_media。每次最多20项，禁止删除、任意 metadata 和媒体 URL。每项都需要 type 和 id：add_node 还需要 nodeType（可给 x/y 指定位置；省略坐标时服务端按画布内容自动落位，不会叠在原点），update_node 还需要按节点能力清单填写 patch（可含 x/y 移动节点），connect_nodes 还需要 fromNodeId 与 toNodeId。分镜脚本的镜头级关联必须把 canvas_get_state 或 canvas_read_storyboard 返回的 rowId 写成 fromHandleId/toHandleId 的 row:；整表设定使用 storyboard:context。连线是生成输入关系，不会改变已提交任务的输入；来源须 canSource，目标须 canTarget 且接受来源 inputKind，能力以注册表为准。批量整理位置用 canvas_arrange_nodes，不要用几十项 update_node 手工算坐标。", map[string]any{"snapshotHash": str("canvas_get_state返回的snapshotHash"), "ops": map[string]any{"type": "array", "maxItems": 20, "items": opItem}}, "snapshotHash", "ops") add("canvas_arrange_nodes", "整理画布节点位置：只改坐标，不改内容、不建连线、不增删节点，先读画布并传 snapshotHash。mode 省略即 auto（有连线按依赖分层，否则按媒体类型分区）。groups 为横向分带（label 展示名，可覆盖整组 mode）。nodeIds 省略则整理全部可整理节点（跳过锁定节点、容器、批次子节点与已归属背板者）。align 对齐/等距，dryRun 只预演；一次最多 50 个节点，只挪单个节点用 update_node 的 x/y。", map[string]any{ "snapshotHash": str("最近一次画布读取的 snapshotHash"), "nodeIds": map[string]any{"type": "array", "maxItems": cloudAgentArrangeMaxNodes, "items": str("节点ID；省略=全部可整理")}, diff --git a/backend/internal/app/cloud_agent_tools_canvas.go b/backend/internal/app/cloud_agent_tools_canvas.go index 8608427f7..3dd6ef18b 100644 --- a/backend/internal/app/cloud_agent_tools_canvas.go +++ b/backend/internal/app/cloud_agent_tools_canvas.go @@ -52,10 +52,12 @@ type agentCanvasOp struct { Patch map[string]any `json:"patch"` // X/Y 为指针：nil 表示模型没有指定坐标，服务端按画布内容自动落位（不再落到原点重叠）。 // 指针语义与 canvas/capability/builtin.go 的 positionPatchFields 一致（坐标是可选的数字）。 - X *float64 `json:"x"` - Y *float64 `json:"y"` - FromNodeID string `json:"fromNodeId"` - ToNodeID string `json:"toNodeId"` + X *float64 `json:"x"` + Y *float64 `json:"y"` + FromNodeID string `json:"fromNodeId"` + ToNodeID string `json:"toNodeId"` + FromHandleID string `json:"fromHandleId"` + ToHandleID string `json:"toHandleId"` } // Explicit node creation and edges only; no generic metadata, media URL or deletion. @@ -67,6 +69,7 @@ func applyCloudAgentCanvas(repo *repository.Repository, userID, canvasID string, if err = saveCloudAgentDocument(repo, plan.Canvas, plan.Document, policy); err != nil { return nil, err } + appliedPreview := cloudAgentCanvasAppliedPreview(plan.Preview) if len(recorder) > 0 && recorder[0] != nil { if err := recorder[0](repo, cloudAgentMutationInput{ UserID: userID, @@ -76,15 +79,19 @@ func applyCloudAgentCanvas(repo *repository.Repository, userID, canvasID string, BeforeSnapshotHash: plan.BeforeSnapshotHash, AfterSnapshotHash: cloudAgentCanvasHash(plan.Document), BeforeJSON: plan.BeforeJSON, - Preview: &plan.Preview, + Preview: &appliedPreview, }); err != nil { return nil, err } } - return map[string]any{"canvasId": canvasID, "snapshotHash": cloudAgentCanvasHash(plan.Document), "summary": fmt.Sprintf("已完成 %d 项节点/连线操作", len(plan.Args.Ops)), "preview": plan.Preview}, nil + return map[string]any{"canvasId": canvasID, "snapshotHash": cloudAgentCanvasHash(plan.Document), "committed": true, "summary": fmt.Sprintf("已完成 %d 项节点/连线操作", len(plan.Args.Ops)), "preview": appliedPreview}, nil } func validateCloudAgentConnection(nodes []map[string]any, fromID, toID string, existingConnections ...[]map[string]any) error { + return validateCloudAgentConnectionWithHandles(nodes, fromID, toID, "", "", existingConnections...) +} + +func validateCloudAgentConnectionWithHandles(nodes []map[string]any, fromID, toID, fromHandleID, toHandleID string, existingConnections ...[]map[string]any) error { if err := validateCloudAgentID(fromID, "来源节点 ID", 80); err != nil { return err } @@ -106,6 +113,12 @@ func validateCloudAgentConnection(nodes []map[string]any, fromID, toID string, e if from == nil || to == nil { return BadAuthRequest("连线端点不存在") } + if err := validateCloudAgentStoryboardHandle(from, fromHandleID, "来源"); err != nil { + return err + } + if err := validateCloudAgentStoryboardHandle(to, toHandleID, "目标"); err != nil { + return err + } fromCapability, fromKnown := cloudAgentNodeCapabilityForNode(from) toCapability, toKnown := cloudAgentNodeCapabilityForNode(to) if !fromKnown || !toKnown { @@ -123,7 +136,7 @@ func validateCloudAgentConnection(nodes []map[string]any, fromID, toID string, e connections = existingConnections[0] } for _, edge := range connections { - if stringValue(edge["toNodeId"]) == toID && stringValue(edge["fromNodeId"]) == fromID { + if stringValue(edge["toNodeId"]) == toID && stringValue(edge["fromNodeId"]) == fromID && stringValue(edge["fromHandleId"]) == fromHandleID && stringValue(edge["toHandleId"]) == toHandleID { return BadAuthRequest("连线重复") } } @@ -145,6 +158,30 @@ func validateCloudAgentConnection(nodes []map[string]any, fromID, toID string, e return nil } +func validateCloudAgentStoryboardHandle(node map[string]any, handleID, side string) error { + if handleID == "" { + return nil + } + if stringValue(node["type"]) != "script" { + return BadAuthRequest(fmt.Sprintf("只有分镜脚本节点可以指定%s handle", side)) + } + if handleID == "storyboard:context" { + return nil + } + if !strings.HasPrefix(handleID, "row:") || strings.TrimSpace(strings.TrimPrefix(handleID, "row:")) == "" { + return BadAuthRequest(fmt.Sprintf("分镜%s handle 必须是 row: 或 storyboard:context", side)) + } + metadata, _ := node["metadata"].(map[string]any) + storyboard, _ := metadata["storyboard"].(map[string]any) + rowID := strings.TrimPrefix(handleID, "row:") + for _, row := range creationMaps(storyboard["rows"]) { + if stringValue(row["id"]) == rowID { + return nil + } + } + return BadAuthRequest(fmt.Sprintf("分镜%s handle 引用的 rowId 不存在，请先读取最新分镜", side)) +} + func cloudAgentInputKindLabel(kind string) string { switch kind { case "image": diff --git a/backend/internal/canvas/capability/builtin.go b/backend/internal/canvas/capability/builtin.go index e9b5b2bfc..5aa2dc6a7 100644 --- a/backend/internal/canvas/capability/builtin.go +++ b/backend/internal/canvas/capability/builtin.go @@ -93,8 +93,12 @@ func BuiltinRegistry() *Registry { GoodFor: []string{"多镜头规划", "镜头连续性", "逐镜审查和微调", "逐镜生成图片或视频", "需要他人接手维护的内容"}, NotIdealFor: []string{"只有一个画面的快速试验", "一次性临时提示词", "仅需要阅读排版的普通文档"}, Tradeoffs: []string{"前期录入成本高于文本节点", "但能保留镜头级结构、资产关系和后续维护能力"}, - Actions: []string{"read_rows", "append_row", "update_row", "remove_row", "generate_storyboard"}, + Actions: []string{"read_rows", "append_row", "update_row", "remove_row", "generate_storyboard", "connect_assets", "update_node"}, + InputKind: "text", + Connection: ConnectionPolicy{CanSource: true, CanTarget: true, AcceptedInputKinds: []string{"audio", "character", "image", "text", "video"}}, + CanUpdate: true, SummaryFields: []string{"storyboard"}, DetailFields: []string{"storyboard"}, ProjectionKind: "storyboard", ProjectionField: "storyboard", + PatchFields: titleAndPositionPatchFields(), CreateMetadata: func(string) map[string]any { return map[string]any{"status": "idle", "workflowKind": "script", "storyboard": map[string]any{"rows": []any{}, "visibleColumns": []any{"shotNumber", "durationSeconds", "videoMotionPrompt", "dialogue", "assets"}, "referenceNodeIds": []any{}}} }, @@ -178,14 +182,11 @@ func generatedMediaSemantics(nodeType string) generatedMediaCapabilitySemantics } } -func editableNodeFields(contentPath, contentLabel, contentDescription string) map[string]PatchField { +func titleAndPositionPatchFields() map[string]PatchField { fields := map[string]PatchField{ "title": { Path: "title", Kind: patchKindString, Label: "节点名称", Order: 10, Description: "节点标题", MaxRunes: maxAgentNodeTitleRunes, }, - "content": { - Path: contentPath, Kind: patchKindString, Label: contentLabel, Order: 20, Description: contentDescription, MaxRunes: maxAgentNodeContentRunes, - }, } for key, field := range positionPatchFields() { fields[key] = field @@ -193,6 +194,14 @@ func editableNodeFields(contentPath, contentLabel, contentDescription string) ma return fields } +func editableNodeFields(contentPath, contentLabel, contentDescription string) map[string]PatchField { + fields := titleAndPositionPatchFields() + fields["content"] = PatchField{ + Path: contentPath, Kind: patchKindString, Label: contentLabel, Order: 20, Description: contentDescription, MaxRunes: maxAgentNodeContentRunes, + } + return fields +} + // positionPatchFields 是坐标字段：模型可以直接指定节点位置（微调），批量整理走 canvas_arrange_nodes。 // 坐标上限与整理侧的收敛范围一致，避免写进离谱的几何值。 func positionPatchFields() map[string]PatchField {