From 3750fada7ab28fb711d2290e46c934cd5663df35 Mon Sep 17 00:00:00 2001 From: tanzui <305375675@qq.com> Date: Tue, 6 Oct 2026 09:28:22 +0800 Subject: [PATCH] =?UTF-8?q?fix(agent):=20=E8=A7=86=E9=A2=91=E7=94=9F?= =?UTF-8?q?=E6=88=90=20-=20=E6=94=AF=E6=8C=81=20Agent=20=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E9=A6=96=E5=B0=BE=E5=B8=A7=20(#696)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(agent): 视频生成 - 支持 Agent 设置首尾帧 * fix(agent): 工具参数 - 精简首尾帧说明并保持体积预算 --------- Co-authored-by: tanzui <305375675@.com> --- .../app/cloud_agent_approval_preview.go | 9 ++ .../app/cloud_agent_generation_contract.go | 17 ++- backend/internal/app/cloud_agent_media.go | 18 ++- backend/internal/app/cloud_agent_tools.go | 6 +- .../internal/app/cloud_agent_tools_canvas.go | 31 +++++ .../cloud_agent_video_frames_postgres_test.go | 130 ++++++++++++++++++ .../app/cloud_agent_video_frames_test.go | 108 +++++++++++++++ backend/internal/canvas/capability/builtin.go | 18 ++- .../docs/backend/agent-prompt-policy.mdx | 6 + docs/content/docs/progress/pending-test.mdx | 7 + 10 files changed, 342 insertions(+), 8 deletions(-) create mode 100644 backend/internal/app/cloud_agent_video_frames_postgres_test.go create mode 100644 backend/internal/app/cloud_agent_video_frames_test.go diff --git a/backend/internal/app/cloud_agent_approval_preview.go b/backend/internal/app/cloud_agent_approval_preview.go index d029e2606..0a0e64fb1 100644 --- a/backend/internal/app/cloud_agent_approval_preview.go +++ b/backend/internal/app/cloud_agent_approval_preview.go @@ -222,6 +222,9 @@ func applyCloudAgentCanvasPlan(doc map[string]any, ops []agentCanvasOp) ([]cloud if err := capability.ApplyPatch(nodes[index], op.Patch); err != nil { return nil, BadAuthRequest(err.Error()) } + if err := validateCloudAgentVideoFramePatch(nodes[index], nodes, edges, op.Patch); err != nil { + return nil, err + } afterTitle := cloudAgentApprovalNodeTitle(nodes[index], capability.Label) resultTitle := "" if afterTitle != beforeTitle { @@ -471,6 +474,12 @@ func cloudAgentMediaApprovalPreview(plan *cloudAgentMediaPlan, modelName string) if args.Size != "" { details = append(details, "画幅："+truncateRunes(args.Size, 40)) } + if args.VideoStartFrameNodeID != "" { + details = append(details, "首帧节点："+truncateRunes(args.VideoStartFrameNodeID, 80)) + } + if args.VideoEndFrameNodeID != "" { + details = append(details, "尾帧节点："+truncateRunes(args.VideoEndFrameNodeID, 80)) + } if args.Quality != "" { details = append(details, "质量："+truncateRunes(args.Quality, 40)) } diff --git a/backend/internal/app/cloud_agent_generation_contract.go b/backend/internal/app/cloud_agent_generation_contract.go index 492af0749..012a27195 100644 --- a/backend/internal/app/cloud_agent_generation_contract.go +++ b/backend/internal/app/cloud_agent_generation_contract.go @@ -5,8 +5,8 @@ import ( "strings" ) -// Agent arguments and the canvas editor enter the same public generation codec. -// Resource materialization and provider credentials remain outside this object. +// cloudAgentGenerationSpec 将 Agent 参数和已解析素材转换为画布生成合同。 +// refs 提供真实素材类型，resolvedConfig 提供准入后的模型默认值；返回可保存的规格或参数错误。 func cloudAgentGenerationSpec(a cloudAgentMediaArgs, refs map[string]any, resolvedConfig map[string]any) (contract.GenerationSpec, error) { selection := &contract.ModelSelection{Kind: "channel", ChannelID: a.ChannelID, ModelKey: a.ChannelModelKey} if a.LogicalModelID != "" { @@ -59,6 +59,19 @@ func cloudAgentGenerationSpec(a cloudAgentMediaArgs, refs map[string]any, resolv } } bindings := []contract.ReferenceBinding{} + // 帧选择沿用视频编辑器的 metadata 字段；这里只校验真实引用，不从提示词或图片顺序猜用途。 + for _, frame := range []struct{ field, id string }{ + {"videoStartFrameNodeId", a.VideoStartFrameNodeID}, + {"videoEndFrameNodeId", a.VideoEndFrameNodeID}, + } { + if frame.id == "" { + continue + } + binding, exists := byID[frame.id] + if a.Mode != "video" || !exists || binding.MediaType != "image" || !containsString(a.ReferenceNodeIDs, frame.id) { + return contract.GenerationSpec{}, cloudAgentFieldError(frame.field, "invalid_value", "首尾帧仅用于视频，必须选择 referenceNodeIds 中的真实图片节点") + } + } for _, id := range a.ReferenceNodeIDs { binding, exists := byID[id] if !exists { diff --git a/backend/internal/app/cloud_agent_media.go b/backend/internal/app/cloud_agent_media.go index 7b5c936f4..347e4a6ed 100644 --- a/backend/internal/app/cloud_agent_media.go +++ b/backend/internal/app/cloud_agent_media.go @@ -105,6 +105,8 @@ type cloudAgentMediaArgs struct { Size string `json:"size"` Quality string `json:"quality"` VideoGenerateAudio *bool `json:"videoGenerateAudio"` + VideoStartFrameNodeID string `json:"videoStartFrameNodeId"` + VideoEndFrameNodeID string `json:"videoEndFrameNodeId"` SnapshotHash string `json:"snapshotHash"` NodeID string `json:"nodeId"` Title string `json:"title"` @@ -642,6 +644,8 @@ func (s *Service) fillCloudAgentMediaSnapshotHash(userID, canvasID string, a *cl return nil } +// prepareCloudAgentMedia 根据当前运行、画布快照和工具调用解析生成参数及素材。 +// 返回待审批的任务请求、草稿计划或校验错误；此阶段不提交收费任务。 func (s *Service) prepareCloudAgentMedia(run *model.CloudAgentExecution, state *cloudAgentRuntime, call cloudAgentCall) (CreateTaskRequest, *cloudAgentMediaPlan, error) { var a cloudAgentMediaArgs if err := decodeCloudAgentJSONObject(call.Function.Arguments, &a); err != nil { @@ -708,6 +712,13 @@ func (s *Service) prepareCloudAgentMedia(run *model.CloudAgentExecution, state * config["channelId"], config["channelModelKey"], config["model"] = a.ChannelID, a.ChannelModelKey, a.ChannelModelKey } metadata := map[string]any{"nodeId": a.NodeID, "source": "cloud_agent"} + // 首尾帧是结构化输入，不能只写进提示词；复用画布提交任务时的字段。 + if a.VideoStartFrameNodeID != "" { + metadata["videoStartFrameNodeId"] = a.VideoStartFrameNodeID + } + if a.VideoEndFrameNodeID != "" { + metadata["videoEndFrameNodeId"] = a.VideoEndFrameNodeID + } if len(a.CharacterVersions) > 0 { resolvedVersions := make([]string, 0, len(a.CharacterVersions)) for id, versionID := range a.CharacterVersions { @@ -822,7 +833,9 @@ func saveCloudAgentDocument(repo *repository.Repository, canvas *model.CanvasPro return saveCreationCanvasWithHistory(repo, canvas, before) } -// Called inside the same transaction as the task, charge reservation and Agent checkpoint. +// createCloudAgentMediaNode 为指定用户的画布保存计划中的媒体节点及引用连线。 +// task 为空时保存待审批草稿，否则绑定已创建任务；policy 校验配额，recorder 可记录撤销事件。 +// 返回保存错误；提交生成时由调用方放在任务、计费和 Agent 检查点的同一个事务中执行。 func createCloudAgentMediaNode(repo *repository.Repository, userID, canvasID string, plan *cloudAgentMediaPlan, task *model.Task, policy RuntimePolicySetting, recorder ...cloudAgentMutationRecorder) error { a := plan.Args canvas, doc, refs, err := cloudAgentMediaDocument(repo, userID, canvasID, a, plan.TransientReferences) @@ -861,6 +874,9 @@ func createCloudAgentMediaNode(repo *repository.Repository, userID, canvasID str return err } meta["status"], meta["agentDraftRunId"], meta["referenceNodeIds"] = "idle", a.DraftRunID, a.ReferenceNodeIDs + if a.Mode == "video" { + meta["videoStartFrameNodeId"], meta["videoEndFrameNodeId"] = a.VideoStartFrameNodeID, a.VideoEndFrameNodeID + } meta["prompt"] = cloudAgentMediaComposerPrompt(a.Prompt, refs) meta["composerContent"] = meta["prompt"] if task != nil { diff --git a/backend/internal/app/cloud_agent_tools.go b/backend/internal/app/cloud_agent_tools.go index 309dae8cf..4d14c7d9c 100644 --- a/backend/internal/app/cloud_agent_tools.go +++ b/backend/internal/app/cloud_agent_tools.go @@ -261,7 +261,7 @@ func compileCloudAgentTools(req CloudAgentRequest, includeProfileTool bool) []ma {"properties": map[string]any{"type": map[string]any{"const": "connect_nodes"}}, "required": []string{"fromNodeId", "toNodeId"}}, }, } - add("canvas_apply_ops", "创建空白节点、修改提示词或建立引用连线，不提交生成任务、不产生生成费用；先读取画布并传 snapshotHash。提交媒体生成使用 generate_media。每次最多20项，禁止删除、任意 metadata 和媒体 URL。每项都需要 type 和 id：add_node 还需要 nodeType（可给 x/y 指定位置；省略坐标时服务端按画布内容自动落位，不会叠在原点），update_node 还需要按节点能力清单填写 patch（可含 x/y 移动节点），connect_nodes 还需要 fromNodeId 与 toNodeId。分镜脚本的镜头级关联必须把 canvas_get_state 或 canvas_read_storyboard 返回的 rowId 写成 fromHandleId/toHandleId 的 row:；整表设定使用 storyboard:context。连线是生成输入关系，不会改变已提交任务的输入；来源须 canSource，目标须 canTarget 且接受来源 inputKind，能力以注册表为准。批量整理位置用 canvas_arrange_nodes，不要用几十项 update_node 手工算坐标。", map[string]any{"snapshotHash": str("canvas_get_state返回的snapshotHash"), "ops": map[string]any{"type": "array", "maxItems": 20, "items": opItem}}, "snapshotHash", "ops") + add("canvas_apply_ops", "创建空白节点、编辑或连线，不生成、不收费；先读画布传 snapshotHash。生成用 generate_media。每次最多20项，禁止删除、任意 metadata 和媒体 URL。操作必填字段见 schema；add_node 可用 x/y 定位，省略时自动排位；update_node 按节点能力填写 patch，可用 x/y 移动或设置视频首尾帧。分镜脚本的镜头级关联必须把 canvas_get_state 或 canvas_read_storyboard 返回的 rowId 写成 fromHandleId/toHandleId 的 row:；整表设定使用 storyboard:context。连线是生成输入关系，不会改变已提交任务的输入；来源须 canSource，目标须 canTarget 且接受来源 inputKind，能力以注册表为准。批量排位用 canvas_arrange_nodes。", map[string]any{"snapshotHash": str("canvas_get_state返回的snapshotHash"), "ops": map[string]any{"type": "array", "maxItems": 20, "items": opItem}}, "snapshotHash", "ops") add("canvas_arrange_nodes", "整理画布节点位置：只改坐标，不改内容、不建连线、不增删节点，先读画布并传 snapshotHash。mode 省略即 auto（有连线按依赖分层，否则按媒体类型分区）。groups 为横向分带（label 展示名，可覆盖整组 mode）。nodeIds 省略则整理全部可整理节点（跳过锁定节点、容器、批次子节点与已归属背板者）。align 对齐/等距，dryRun 只预演；一次最多 50 个节点，只挪单个节点用 update_node 的 x/y。", map[string]any{ "snapshotHash": str("最近一次画布读取的 snapshotHash"), "nodeIds": map[string]any{"type": "array", "maxItems": cloudAgentArrangeMaxNodes, "items": str("节点ID；省略=全部可整理")}, @@ -283,7 +283,9 @@ func compileCloudAgentTools(req CloudAgentRequest, includeProfileTool bool) []ma "mode": map[string]any{"type": "string", "enum": cloudAgentGenerationModeNames()}, "prompt": str("完整生成提示词；引用素材时在对应描述中使用 @图片1、@视频1、@音频1，各类型按 referenceNodeIds 中出现顺序独立编号，文本来源不占媒体编号。服务端会为遗漏的已选素材补齐引用标签，不推断素材用途"), "logicalModelId": str("selection.logicalModelId；与channelId/channelModelKey互斥"), "channelId": str("selection.channelId"), "channelModelKey": str("selection.channelModelKey"), "durationSeconds": map[string]any{"type": "integer", "minimum": 0}, "size": str("模型支持的画幅，例如9:16"), "quality": str("目录支持的分辨率或质量"), "videoGenerateAudio": map[string]any{"type": "boolean", "description": "是否生成音频，仅视频可用"}, - "snapshotHash": str("可省略：省略时用当前画布内容快照"), "nodeId": str("可续用的未提交媒体草稿ID；无草稿时才使用新唯一ID"), "title": str("媒体节点名称"), "sourceNodeId": str("仅文本/镜头提示词节点ID；不要填媒体节点"), "referenceNodeIds": map[string]any{"type": "array", "maxItems": 16, "items": str("画布媒体参考节点ID，按引用顺序")}, "referenceTransientIds": map[string]any{"type": "array", "maxItems": 4, "items": str("由 image_annotation_render 返回的临时参考图ID")}, + "videoStartFrameNodeId": str("视频首帧图片ID，须在 referenceNodeIds 中"), + "videoEndFrameNodeId": str("视频尾帧图片ID，须在 referenceNodeIds 中"), + "snapshotHash": str("可省略：省略时用当前画布内容快照"), "nodeId": str("可续用的未提交媒体草稿ID；无草稿时才使用新唯一ID"), "title": str("媒体节点名称"), "sourceNodeId": str("仅文本/镜头提示词节点ID；不要填媒体节点"), "referenceNodeIds": map[string]any{"type": "array", "maxItems": 16, "items": str("画布媒体参考节点ID，按引用顺序")}, "referenceTransientIds": map[string]any{"type": "array", "maxItems": 4, "items": str("由 image_annotation_render 返回的临时参考图ID")}, }, "mode", "prompt", "nodeId", "title", "referenceNodeIds") cloudAgentRequireExplicitMediaModelSelection(tools[len(tools)-1]) } diff --git a/backend/internal/app/cloud_agent_tools_canvas.go b/backend/internal/app/cloud_agent_tools_canvas.go index 3dd6ef18b..c2aec6d4d 100644 --- a/backend/internal/app/cloud_agent_tools_canvas.go +++ b/backend/internal/app/cloud_agent_tools_canvas.go @@ -38,6 +38,37 @@ func validateCloudAgentID(value, label string, maxRunes int) error { return nil } +// validateCloudAgentVideoFramePatch 校验视频帧选择是否来自当前画布的图片输入。 +// node 是修改后的节点，nodes 和 edges 是当前操作计划；patch 未修改帧时不影响旧节点。 +// 空字符串允许取消选择；非空值必须先建立图片引用连线，实际素材可用性仍由生成准入校验。 +func validateCloudAgentVideoFramePatch(node map[string]any, nodes, edges []map[string]any, patch map[string]any) error { + for _, key := range []string{"videoStartFrameNodeId", "videoEndFrameNodeId"} { + raw, changed := patch[key] + if !changed { + continue + } + id := stringValue(raw) + if id == "" { + continue + } + if err := validateCloudAgentID(id, "首尾帧图片节点 ID", 80); err != nil { + return cloudAgentFieldError("patch."+key, "invalid_value", cloudAgentSafeToolError(err)) + } + frame := cloudAgentNodeByID(nodes, id) + connected := false + for _, edge := range edges { + if stringValue(edge["fromNodeId"]) == id && stringValue(edge["toNodeId"]) == stringValue(node["id"]) { + connected = true + break + } + } + if frame == nil || stringValue(frame["type"]) != "image" || !connected { + return cloudAgentFieldError("patch."+key, "invalid_value", "首尾帧必须选择已连接到该视频节点的图片；请先建立引用连线") + } + } + return nil +} + type agentCanvasArgs struct { SnapshotHash string `json:"snapshotHash"` Ops []agentCanvasOp `json:"ops"` diff --git a/backend/internal/app/cloud_agent_video_frames_postgres_test.go b/backend/internal/app/cloud_agent_video_frames_postgres_test.go new file mode 100644 index 000000000..2cf64cd4f --- /dev/null +++ b/backend/internal/app/cloud_agent_video_frames_postgres_test.go @@ -0,0 +1,130 @@ +package app + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "testing" + "time" + + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "infinite-canvas/backend/internal/database" + "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/repository" +) + +// TestCloudAgentVideoFramesPostgres 验证 Agent 的帧选择进入任务输入、协议角色和持久化草稿。 +// 测试使用独立 schema，结束后删除测试表；不调用外部模型、不提交收费任务。 +func TestCloudAgentVideoFramesPostgres(t *testing.T) { + dsn := os.Getenv("CANVAS_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("CANVAS_TEST_POSTGRES_DSN is not configured") + } + options := &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)} + admin, err := gorm.Open(postgres.Open(dsn), options) + if err != nil { + t.Fatal(err) + } + adminSQL, err := admin.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = adminSQL.Close() }) + schema := fmt.Sprintf("agent_video_frames_%d", time.Now().UnixNano()) + if err := admin.Exec("CREATE SCHEMA " + schema).Error; err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := admin.Exec("DROP SCHEMA " + schema + " CASCADE").Error; err != nil { + t.Error(err) + } + }) + parsed, err := url.Parse(dsn) + if err != nil { + t.Fatal(err) + } + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + db, err := gorm.Open(postgres.Open(parsed.String()), options) + if err != nil { + t.Fatal(err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + if err := db.AutoMigrate(database.Models()...); err != nil { + t.Fatal(err) + } + s := &Service{repo: repository.New(db), dataDir: t.TempDir()} + doc := map[string]any{"nodes": []map[string]any{ + {"id": "first", "type": "image", "metadata": map[string]any{"storageKey": "resource:first", "status": "success"}}, + {"id": "last", "type": "image", "metadata": map[string]any{"storageKey": "resource:last", "status": "success"}}, + }, "connections": []any{}} + raw, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + for _, row := range []any{ + &model.CanvasProject{ID: "canvas", UserID: "user", PayloadJSON: string(raw)}, + &model.Resource{ID: "first", UserID: "user", Kind: "image", Status: "ready", MimeType: "image/png"}, + &model.Resource{ID: "last", UserID: "user", Kind: "image", Status: "ready", MimeType: "image/png"}, + } { + if err := db.Create(row).Error; err != nil { + t.Fatal(err) + } + } + // 故意把尾帧放在参考数组前面，帧角色必须来自明确字段，而不是数组顺序。 + a := cloudAgentMediaArgs{Mode: "video", Prompt: "猫咪做饭", ChannelID: "channel", ChannelModelKey: "video", NodeID: "target", Title: "镜头视频", ReferenceNodeIDs: []string{"last", "first"}, VideoStartFrameNodeID: "first", VideoEndFrameNodeID: "last"} + state := &cloudAgentRuntime{Request: CloudAgentRequest{CanvasID: "canvas"}} + req, plan, err := s.prepareCloudAgentMedia(&model.CloudAgentExecution{ID: "run", UserID: "user"}, state, agentMediaCall(a)) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(req.Input) + if err != nil { + t.Fatal(err) + } + var input canvasGenerationInput + if err := json.Unmarshal(encoded, &input); err != nil { + t.Fatal(err) + } + if input.Metadata["videoStartFrameNodeId"] != "first" || input.Metadata["videoEndFrameNodeId"] != "last" { + t.Fatalf("任务输入丢失帧选择：%v", input.Metadata) + } + for i := range input.ReferenceImages { + input.ReferenceImages[i].DataURL = "data:image/png;base64,aGVsbG8=" + } + references := protocolVideoImageReferences(input) + if len(references) != 2 || references[0].ID != "last" || references[0].Role != "last_frame" || references[1].ID != "first" || references[1].Role != "first_frame" { + t.Fatalf("协议收到错误的首尾帧：%+v", references) + } + policy, err := s.RuntimePolicy() + if err != nil { + t.Fatal(err) + } + if err := createCloudAgentMediaNode(s.repo, "user", "canvas", plan, nil, policy); err != nil { + t.Fatal(err) + } + canvas, err := s.repo.CanvasProjectForUser("user", "canvas") + if err != nil { + t.Fatal(err) + } + saved, err := creationDocument(canvas.PayloadJSON) + if err != nil { + t.Fatal(err) + } + node := cloudAgentNodeByID(creationMaps(saved["nodes"]), "target") + if node == nil { + t.Fatal("草稿节点未保存") + } + metadata := node["metadata"].(map[string]any) + if metadata["videoStartFrameNodeId"] != "first" || metadata["videoEndFrameNodeId"] != "last" || len(creationMaps(saved["connections"])) != 2 { + t.Fatalf("草稿未保存帧选择和引用连线：%v", saved) + } +} diff --git a/backend/internal/app/cloud_agent_video_frames_test.go b/backend/internal/app/cloud_agent_video_frames_test.go new file mode 100644 index 000000000..02a79c3f3 --- /dev/null +++ b/backend/internal/app/cloud_agent_video_frames_test.go @@ -0,0 +1,108 @@ +package app + +import ( + "testing" + + "infinite-canvas/backend/internal/canvas/capability" +) + +func TestCloudAgentVideoFrameToolSchema(t *testing.T) { + req := CloudAgentRequest{PermissionMode: "auto", ContextScope: []string{"canvas"}} + found := false + for _, tool := range cloudAgentTools(req) { + function := tool["function"].(map[string]any) + if function["name"] != "generate_media" { + continue + } + found = true + properties := function["parameters"].(map[string]any)["properties"].(map[string]any) + for _, key := range []string{"videoStartFrameNodeId", "videoEndFrameNodeId"} { + if properties[key].(map[string]any)["type"] != "string" { + t.Fatalf("生成工具未暴露 %s", key) + } + } + } + if !found { + t.Fatal("缺少生成工具") + } +} + +func TestCloudAgentVideoFrameGenerationValidation(t *testing.T) { + refs := map[string]any{ + "referenceImages": []map[string]any{{"id": "first"}, {"id": "last"}}, + "referenceVideos": []map[string]any{{"id": "clip"}}, + } + base := cloudAgentMediaArgs{Mode: "video", ChannelID: "channel", ChannelModelKey: "video", ReferenceNodeIDs: []string{"last", "first", "clip"}, VideoStartFrameNodeID: "first", VideoEndFrameNodeID: "last"} + for _, test := range []struct { + name string + edit func(*cloudAgentMediaArgs) + fail bool + }{ + {"明确首尾帧", func(*cloudAgentMediaArgs) {}, false}, + {"仅首帧", func(a *cloudAgentMediaArgs) { a.VideoEndFrameNodeID = "" }, false}, + {"仅尾帧", func(a *cloudAgentMediaArgs) { a.VideoStartFrameNodeID = "" }, false}, + {"普通参考", func(a *cloudAgentMediaArgs) { a.VideoStartFrameNodeID, a.VideoEndFrameNodeID = "", "" }, false}, + {"缺失节点", func(a *cloudAgentMediaArgs) { a.VideoStartFrameNodeID = "missing" }, true}, + {"非图片", func(a *cloudAgentMediaArgs) { a.VideoStartFrameNodeID = "clip" }, true}, + {"未选为参考", func(a *cloudAgentMediaArgs) { a.ReferenceNodeIDs = []string{"last"} }, true}, + {"非视频模式", func(a *cloudAgentMediaArgs) { a.Mode = "image" }, true}, + {"同一节点", func(a *cloudAgentMediaArgs) { a.VideoEndFrameNodeID = "first" }, false}, + } { + t.Run(test.name, func(t *testing.T) { + a := base + test.edit(&a) + _, err := cloudAgentGenerationSpec(a, refs, nil) + if (err != nil) != test.fail { + t.Fatalf("参数校验结果不符合预期：%v", err) + } + }) + } +} + +func TestCloudAgentVideoFrameCanvasEditing(t *testing.T) { + for _, test := range []struct { + name, kind, frame string + connected, fail bool + }{ + {"设置首尾帧", "video", "first", true, false}, + {"取消选择", "video", "", false, false}, + {"未连接", "video", "first", false, true}, + {"不存在", "video", "missing", true, true}, + {"非图片", "video", "text", true, true}, + {"禁止图片节点设置帧", "image", "first", true, true}, + } { + t.Run(test.name, func(t *testing.T) { + nodes := []map[string]any{ + {"id": "first", "type": "image", "metadata": map[string]any{}}, + {"id": "last", "type": "image", "metadata": map[string]any{}}, + {"id": "text", "type": "text", "metadata": map[string]any{}}, + {"id": "target", "type": test.kind, "metadata": map[string]any{}}, + } + edges := []map[string]any{{"id": "last-edge", "fromNodeId": "last", "toNodeId": "target"}} + if test.connected { + edges = append(edges, map[string]any{"id": "first-edge", "fromNodeId": test.frame, "toNodeId": "target"}) + } + doc := map[string]any{"nodes": nodes, "connections": edges} + patch := map[string]any{"videoStartFrameNodeId": test.frame, "videoEndFrameNodeId": "last"} + _, err := applyCloudAgentCanvasPlan(doc, []agentCanvasOp{{Type: "update_node", ID: "target", Patch: patch}}) + if (err != nil) != test.fail { + t.Fatalf("编辑结果不符合预期：%v", err) + } + if err != nil { + return + } + metadata := nodes[3]["metadata"].(map[string]any) + descriptor, _ := cloudAgentNodeCapabilityForType(test.kind) + projected, err := cloudAgentProjectNodeFields(nodes[3], metadata, descriptor, descriptor.DetailFields, 1000, true, 0) + if err != nil || projected["videoStartFrameNodeId"] != test.frame || projected["videoEndFrameNodeId"] != "last" { + t.Fatalf("帧选择未保存或未返回给 Agent：%v %v", projected, err) + } + }) + } + for _, kind := range []string{"text", "image", "audio"} { + descriptor, _ := capability.BuiltinRegistry().Resolve(kind) + if err := descriptor.ValidatePatch(map[string]any{"videoEndFrameNodeId": "last"}); err == nil { + t.Fatalf("%s 不应接受视频帧参数", kind) + } + } +} diff --git a/backend/internal/canvas/capability/builtin.go b/backend/internal/canvas/capability/builtin.go index d9e679350..06f2badd2 100644 --- a/backend/internal/canvas/capability/builtin.go +++ b/backend/internal/canvas/capability/builtin.go @@ -131,16 +131,28 @@ func characterDescriptor() Descriptor { } } +// generatedMediaDescriptor 根据节点类型、显示尺寸、生成模式和连接规则创建媒体能力描述。 +// 返回读写字段及创建默认值；首尾帧只对视频开放，避免其他节点接受无效视频参数。 func generatedMediaDescriptor(nodeType, version, label string, width, height float64, generationMode string, connection ConnectionPolicy) Descriptor { semantics := generatedMediaSemantics(nodeType) + fields := editableNodeFields("metadata.generationSpec.prompt", "提示词", "生成合同中的当前提示词；这是 Agent 唯一读写的媒体提示词") + readFields := []string{"prompt", "assetTags", "referenceNodeIds"} + if generationMode == "video" { + for i, frame := range []struct{ key, label string }{ + {"videoStartFrameNodeId", "首帧"}, {"videoEndFrameNodeId", "尾帧"}, + } { + fields[frame.key] = PatchField{Path: "metadata." + frame.key, Kind: patchKindString, Label: frame.label, Order: 40 + i, MaxRunes: 80, Description: "已连接图片ID；空字符串取消"} + readFields = append(readFields, frame.key) + } + } return Descriptor{ Type: nodeType, Version: version, Label: label, DefaultWidth: width, DefaultHeight: height, Purpose: semantics.Purpose, GoodFor: semantics.GoodFor, NotIdealFor: semantics.NotIdealFor, Tradeoffs: semantics.Tradeoffs, Actions: semantics.Actions, InputKind: nodeType, GenerationMode: generationMode, Connection: connection, CanUpdate: true, - SummaryFields: []string{"prompt", "assetTags", "referenceNodeIds"}, - DetailFields: []string{"prompt", "assetTags", "referenceNodeIds"}, - PatchFields: editableNodeFields("metadata.generationSpec.prompt", "提示词", "生成合同中的当前提示词；这是 Agent 唯一读写的媒体提示词"), + SummaryFields: readFields, + DetailFields: readFields, + PatchFields: fields, CreateMetadata: func(prompt string) map[string]any { return generatedMetadata(generationMode, prompt) }, } } diff --git a/docs/content/docs/backend/agent-prompt-policy.mdx b/docs/content/docs/backend/agent-prompt-policy.mdx index 2c5d0602b..dd529e238 100644 --- a/docs/content/docs/backend/agent-prompt-policy.mdx +++ b/docs/content/docs/backend/agent-prompt-policy.mdx @@ -26,6 +26,12 @@ description: 云端画布 Agent 的行为策略、上下文来源、运行提醒 ## 用户目标与参考素材 +### 视频首尾帧 + +`generate_media` 支持可选的 `videoStartFrameNodeId` 和 `videoEndFrameNodeId`。每个非空值必须是 `referenceNodeIds` 中已解析的图片引用，仅用于视频。参考数组决定素材编号，两个帧字段决定用途；在提示词中写“图片1作为首帧”不会自动选择首帧。 + +只编辑、不生成时，先用 `canvas_apply_ops` 建立图片到视频节点的引用连线，再通过 `update_node.patch` 设置同名字段；空字符串取消选择。节点读取会返回当前帧选择，生成审批也会显示首尾帧节点。生成工具仍需明确传入本次选择，不会根据历史提示词猜测用途；原有权限、素材归属、审批和计费流程不变。 + - 新消息只覆盖与之冲突的旧要求。“继续”“可以”仍结合历史理解，不丢弃有效约束；已送达的运行中插话也进入后续可信历史。 - 当前用户请求保留在 user 消息及运行记录里，不复制进 system 提示锁定目标。系统上下文只包含可验证事实，素材标题、摘要和标签编码为 JSON 数据。 - 删除原先通过“参考图”“图生视频”等关键词推断强制引用的逻辑，否定句和概念解释不再被代码标记为引用任务。 diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 7d0217941..407810e84 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -11,6 +11,13 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 - [x] 回归测试在修改前复现副本无法保存；修改后覆盖复制、普通保存、新生成结果提交及重新读取。 - [ ] 在真实画布中通过复制粘贴和“创建副本”分别验证，生成完成并刷新后副本仍保留。已丢失的副本不会自动恢复，也不会自动重试生图。 +## Agent 视频首尾帧选择 + +- 已实现：生成工具支持明确指定首尾帧，视频节点编辑支持设置和取消已连接的图片；画布读取和生成审批展示当前选择。 +- [x] 专项测试覆盖工具字段、视频模式和图片引用校验、编辑与取消、禁止其他节点类型修改帧字段。 +- [x] PostgreSQL 独立测试区验证帧选择进入任务输入和持久化草稿，协议按明确字段识别首尾帧，不依赖参考数组顺序；未调用外部模型，测试区已删除。 +- [ ] 开启新的 Agent 对话，确认 Agent 能设置下方首尾帧选项，刷新后仍保留；用户确认后再批准真实视频生成。 + ## Agent 失败消息合并与展示 - 已修复：同一运行的 `run_failed` / `error` 与失败状态快照共用一条错误消息；以公开失败原因补全内容，重放、乱序和空原因不覆盖已有详细说明。不同运行即使错误文本相同也分别保留，工具失败与连接错误不合并。