From 3aa874a70c662a9cb7241f4c73926a71c7215194 Mon Sep 17 00:00:00 2001 From: ddcat666 <26564566+ddcat666@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:42:51 +0800 Subject: [PATCH] =?UTF-8?q?feat(platform):=20Agent=20=E4=B8=8E=E5=AA=92?= =?UTF-8?q?=E4=BD=93=E8=83=BD=E5=8A=9B=20-=20=E5=AE=8C=E5=96=84=E8=A7=92?= =?UTF-8?q?=E8=89=B2=E5=8D=A1=E3=80=81=E9=9F=B3=E9=A2=91=E5=8D=8F=E8=AE=AE?= =?UTF-8?q?=E4=B8=8E=E4=BB=BB=E5=8A=A1=E6=81=A2=E5=A4=8D=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/agent-runtime/pi/agent-runtime.mjs | 45 ++- backend/builtin/影策短剧制作工作流/SKILL.md | 17 +- .../references/artifact-schemas.md | 5 +- .../影策短剧制作工作流/references/assets.md | 6 +- .../references/canvas-execution.md | 3 +- .../references/character-cards.md | 71 +++++ .../references/state-machine.md | 2 + backend/internal/agent/runtime/runtime.go | 9 +- backend/internal/app/asset_library.go | 56 +++- .../internal/app/channel_models_pricing.go | 4 +- backend/internal/app/cloud_agent.go | 2 +- backend/internal/app/cloud_agent_anchor.go | 9 +- .../app/cloud_agent_approval_preview.go | 6 +- .../internal/app/cloud_agent_canvas_events.go | 15 +- .../internal/app/cloud_agent_canvas_state.go | 9 +- backend/internal/app/cloud_agent_character.go | 9 +- .../app/cloud_agent_character_create.go | 266 ++++++++++++++++ .../app/cloud_agent_character_create_test.go | 96 ++++++ .../app/cloud_agent_character_test.go | 47 +++ .../app/cloud_agent_event_node_type_test.go | 13 + backend/internal/app/cloud_agent_layout.go | 2 +- backend/internal/app/cloud_agent_media.go | 20 +- .../internal/app/cloud_agent_media_test.go | 2 +- backend/internal/app/cloud_agent_nodes.go | 8 + .../app/cloud_agent_pi_coordinator.go | 64 +++- .../internal/app/cloud_agent_pi_helpers.go | 13 - backend/internal/app/cloud_agent_pi_model.go | 42 +++ .../app/cloud_agent_pi_recovery_test.go | 107 +++++++ .../internal/app/cloud_agent_pi_request.go | 14 +- .../internal/app/cloud_agent_pi_runtime.go | 13 +- backend/internal/app/cloud_agent_policy.go | 3 + .../internal/app/cloud_agent_runtime_tools.go | 8 + backend/internal/app/cloud_agent_test.go | 15 +- .../cloud_agent_tool_schema_budget_test.go | 13 +- backend/internal/app/cloud_agent_tools.go | 1 + .../internal/app/cloud_agent_tools_canvas.go | 8 +- .../internal/app/cloud_agent_tools_read.go | 2 +- backend/internal/app/cloud_agent_vision.go | 3 +- backend/internal/app/finance.go | 40 ++- backend/internal/app/logical_model_quote.go | 9 +- backend/internal/app/logical_models.go | 4 +- backend/internal/app/model_router_attempts.go | 4 +- backend/internal/app/payment_products.go | 27 +- backend/internal/app/project_character.go | 290 ++++++++++++++++++ .../project_character_voice_rename_test.go | 41 +++ backend/internal/app/provider.go | 5 +- backend/internal/app/provider_error.go | 41 ++- backend/internal/app/provider_error_test.go | 20 ++ backend/internal/app/provider_errors.go | 12 + backend/internal/app/provider_http_client.go | 50 ++- backend/internal/app/provider_protocol.go | 6 +- .../internal/app/provider_protocol_request.go | 12 +- .../internal/app/provider_task_recovery.go | 275 +++++++++++++++-- .../app/provider_task_recovery_test.go | 95 ++++++ backend/internal/app/provider_test.go | 13 + backend/internal/app/task_api_call_log.go | 8 +- .../internal/app/task_api_call_log_test.go | 81 +++++ backend/internal/app/task_billing.go | 19 ++ backend/internal/app/task_media_assets.go | 21 ++ backend/internal/app/task_media_storage.go | 30 +- backend/internal/app/task_terminal.go | 43 ++- backend/internal/app/task_worker.go | 6 +- backend/internal/assets/access.go | 47 ++- backend/internal/assets/access_test.go | 70 ++++- backend/internal/canvas/capability/builtin.go | 26 +- .../internal/canvas/capability/descriptor.go | 11 + .../internal/canvas/capability/registry.go | 61 +++- .../internal/canvas/contract/generation.go | 2 + backend/internal/generation/registry.go | 2 + .../internal/handler/admin_api_log_routes.go | 52 +++- .../internal/handler/project_asset_routes.go | 131 ++++++++ backend/internal/model/models.go | 1 + backend/internal/protocol/adapter.go | 4 + .../protocol/autodl_indextts2_test.go | 95 ------ .../protocol/doubao_streaming_test.go | 157 ++++++++++ backend/internal/protocol/manifest.go | 21 +- backend/internal/protocol/manifest_adapter.go | 139 ++++++++- backend/internal/repository/analytics.go | 32 +- backend/internal/repository/analytics_test.go | 38 +++ backend/internal/repository/asset_library.go | 7 +- .../internal/repository/asset_library_test.go | 25 ++ .../internal/repository/finance_audio_test.go | 79 +++++ .../internal/repository/finance_billing.go | 97 ++++++ backend/internal/repository/payment.go | 48 ++- .../repository/payment_product_limits_test.go | 85 +++++ .../internal/repository/project_character.go | 91 +++++- .../repository/provider_task_recovery_test.go | 67 ++++ backend/internal/service/aliases_types.go | 2 + backend/internal/storage/objects.go | 130 +++++--- backend/internal/storage/s3.go | 28 +- backend/internal/storage/signing_test.go | 90 ++++++ 91 files changed, 3438 insertions(+), 380 deletions(-) create mode 100644 backend/builtin/影策短剧制作工作流/references/character-cards.md create mode 100644 backend/internal/app/cloud_agent_character_create.go create mode 100644 backend/internal/app/cloud_agent_character_create_test.go create mode 100644 backend/internal/app/cloud_agent_event_node_type_test.go create mode 100644 backend/internal/app/cloud_agent_pi_recovery_test.go create mode 100644 backend/internal/app/project_character_voice_rename_test.go delete mode 100644 backend/internal/protocol/autodl_indextts2_test.go create mode 100644 backend/internal/protocol/doubao_streaming_test.go create mode 100644 backend/internal/repository/finance_audio_test.go create mode 100644 backend/internal/repository/payment_product_limits_test.go create mode 100644 backend/internal/repository/provider_task_recovery_test.go create mode 100644 backend/internal/storage/signing_test.go diff --git a/backend/agent-runtime/pi/agent-runtime.mjs b/backend/agent-runtime/pi/agent-runtime.mjs index dda6995d3..4465f85c3 100644 --- a/backend/agent-runtime/pi/agent-runtime.mjs +++ b/backend/agent-runtime/pi/agent-runtime.mjs @@ -1,7 +1,7 @@ import { rmSync } from "node:fs"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { join } from "node:path"; import { Agent, setGlobalDispatcher } from "undici"; import { createAgentSession, DefaultResourceLoader, ModelRuntime, SessionManager, SettingsManager } from "@earendil-works/pi-coding-agent"; import { createAssistantMessageEventStream } from "@earendil-works/pi-ai"; @@ -82,13 +82,20 @@ async function run() { // 每次运行使用独立的空配置目录：SDK 的 auth.json / models.json / 锁文件都落在这里， // 不读取宿主 ~/.pi，也不会执行预置 auth.json 里的 !command。运行结束即删除。 + // 会话文件和工作目录同样放在这里：它们只是本轮的工作副本，真实来源是服务端 + // 数据库里的会话快照（请求带 sessionJSONL，运行中回传）。不使用服务端的路径： + // 独立容器里没有服务端的数据目录；空工作目录也不会被发现任何项目级配置或技能。 const isolatedDir = await mkdtemp(join(tmpdir(), "agent-runtime-")); process.env.HOME = isolatedDir; process.env.PI_CODING_AGENT_DIR = isolatedDir; process.on("exit", () => { try { rmSync(isolatedDir, { recursive: true, force: true }); } catch {} }); - request.agentDir = isolatedDir; + const agentDir = isolatedDir; + const workDir = join(isolatedDir, "work"); + const sessionDir = join(isolatedDir, "sessions"); + await mkdir(workDir, { mode: 0o700 }); + await mkdir(sessionDir, { mode: 0o700 }); const modelRuntime = await ModelRuntime.create({ authPath: join(isolatedDir, "auth.json"), modelsPath: join(isolatedDir, "models.json"), @@ -210,32 +217,36 @@ async function run() { let sessionManager; if (request.sessionJSONL) { - if (!request.sessionFile) throw new Error("Agent session restore requires a session file path"); - await mkdir(dirname(request.sessionFile), { recursive: true, mode: 0o700 }); - await writeFile(request.sessionFile, request.sessionJSONL, { mode: 0o600, flag: "w" }); - sessionManager = SessionManager.open(request.sessionFile, undefined, request.cwd); + const sessionFile = join(sessionDir, "session.jsonl"); + await writeFile(sessionFile, request.sessionJSONL, { mode: 0o600, flag: "wx" }); + sessionManager = SessionManager.open(sessionFile, sessionDir, workDir); } else { - sessionManager = SessionManager.create(request.cwd, request.sessionDir); + sessionManager = SessionManager.create(workDir, sessionDir); } - // 严格隔离：不探索文件系统的 packages/skills/extensions + // 严格隔离：不探索文件系统的 packages/skills/extensions/项目配置。 + // 技能由服务端的读取工具提供，这里不再按路径加载；projectTrusted=false 让 SDK + // 不读取 /.pi 设置，也不向上层目录收集 .agents/skills。 + // 同一个设置管理器也交给 createAgentSession，否则 SDK 会另建一个读文件的实例。 + const settingsManager = SettingsManager.inMemory({}, { projectTrusted: false }); const resourceLoader = new DefaultResourceLoader({ - cwd: request.cwd, - agentDir: request.agentDir, + cwd: workDir, + agentDir, noExtensions: true, noSkills: true, noPromptTemplates: true, noThemes: true, noContextFiles: true, - additionalSkillPaths: request.skillPaths ?? [], + additionalSkillPaths: [], systemPrompt: request.systemPrompt, - settingsManager: SettingsManager.inMemory({}), + settingsManager, }); await resourceLoader.reload(); const contextWindow = Math.max(1, Number(request.model?.contextWindow || 128000)); const sessionConfig = { - cwd: request.cwd, - agentDir: request.agentDir, + cwd: workDir, + agentDir, + settingsManager, modelRuntime, model, resourceLoader, @@ -299,6 +310,9 @@ async function run() { const eventChain = { current: Promise.resolve() }; let runtimeError = null; + // 事件链串行执行。已有一次快照在排队时，它执行时读到的已是最新文件， + // 不必为每条新条目再整份上传一次。 + let snapshotQueued = false; const enqueueEvent = (payload) => { eventChain.current = eventChain.current.then(() => bridge(request, "/event", payload)); }; @@ -404,7 +418,10 @@ async function run() { approvalId: event.approvalId, }); } else if (event.type === "entry_appended") { + if (snapshotQueued) return; + snapshotQueued = true; eventChain.current = eventChain.current.then(async () => { + snapshotQueued = false; const sessionFile = sessionManager.getSessionFile(); if (!sessionFile) return; let sessionJSONL; diff --git a/backend/builtin/影策短剧制作工作流/SKILL.md b/backend/builtin/影策短剧制作工作流/SKILL.md index e486d0a6b..2eb3f5247 100644 --- a/backend/builtin/影策短剧制作工作流/SKILL.md +++ b/backend/builtin/影策短剧制作工作流/SKILL.md @@ -2,7 +2,7 @@ name: 影策短剧制作工作流 description: 将小说章节、故事素材或定稿剧本整理为可审阅的短剧方案；先确认统一视觉风格并建立、验收、锁定角色/服装/场景/道具资产，再进入制作版分镜和视频生成，最后按真实任务状态核验交付。 metadata: - version: "1.1.0" + version: "1.2.0" author: 影策 owner: yingce-system skillId: "yingce-short-drama-workflow" @@ -27,7 +27,7 @@ metadata: 1. 输入与范围记录。 2. 剧情事实账本与当前有效剧本。 -3. 角色、场景、道具和声音资产登记。 +3. 角色、场景、道具和声音资产登记；核心角色打包为角色卡。 4. 结构化分镜脚本与时间预算。 5. 按真实模型能力生成的片段提示词。 6. 画布节点、引用关系和任务状态。 @@ -72,12 +72,12 @@ metadata: 3. **事件提取与故事圣经**：从已确认原文提取事件、人物、地点、对白、物件状态和未知信息；生成包含人物、世界观、关系、调性、禁忌、视觉规则和连续性规则的故事圣经。故事圣经需要用户确认或在明确授权下锁定。 4. **故事骨架、分集与剧本**：由故事圣经约束故事骨架和分集；每轮结构化剧本携带原文版本、事件版本、圣经版本和当前分集上下文。完成后执行人物、关系、世界观、服装、道具、伤势和时间线一致性校验；冲突先列出再重写。 5. **风格确认**：在制作级资产图和正式分镜之前，向用户展示可比较的视觉方向，至少覆盖媒介/质感、时代地域、色彩基调、光影、构图和画幅。用户已有明确风格时记录并复述确认；用户授权自主决定时给出一个明确方案及假设。风格未确认，不生成正式资产或进入制作级分镜。 -6. **资产登记、资产包与验收**：依 `references/assets.md` 为实际会出镜或影响连续性的角色、服装状态、场景、关键道具/武器和需要的声音建立登记。角色必须优先建立角色设定、三视图/转面、定妆图和音色设定；环境建立空间母版；武器/道具建立结构参考。用画布背板组织角色包、声音包、环境包和武器/道具包。资产逐项核对、标记版本和状态，并由用户确认或依明确授权采用。不能生成或读取参考图时，写明缺口和影响，不得声称资产已锁定。 -7. **正式分镜与连续性检查**：只有风格已确认且所有本阶段必需资产达到“已采用/已锁定”，才能将分镜标为制作版并绑定真实资产引用。草分镜可以先行，但必须标注“草案，不可投产”。正式分镜逐镜记录起始状态、动作、结束状态、人物/服装变体、场景状态、道具状态、时间、机位、动作、对白/声音、时长及引用资产。下一镜的 `startState` 必须继承上一镜的 `endState`。 +6. **资产登记、资产包与验收**：依 `references/assets.md` 为实际会出镜或影响连续性的角色、服装状态、场景、关键道具/武器和需要的声音建立登记。角色必须优先建立角色设定、三视图/转面、定妆图和音色设定；核心角色的三视图验收后，用 `canvas_create_character` 打包为角色卡写入角色库，角色卡可读且三视图引用就绪才算该角色已锁定（见 `references/character-cards.md`）；环境建立空间母版；武器/道具建立结构参考。用画布背板组织角色包、声音包、环境包和武器/道具包。资产逐项核对、标记版本和状态，并由用户确认或依明确授权采用。不能生成或读取参考图时，写明缺口和影响，不得声称资产已锁定。 +7. **正式分镜与连续性检查**：只有风格已确认且所有本阶段必需资产达到“已采用/已锁定”，才能将分镜标为制作版并绑定真实资产引用；核心角色一律引用角色卡节点，不引用散落的三视图图片。草分镜可以先行，但必须标注“草案，不可投产”。正式分镜逐镜记录起始状态、动作、结束状态、人物/服装变体、场景状态、道具状态、时间、机位、动作、对白/声音、时长及引用资产。下一镜的 `startState` 必须继承上一镜的 `endState`。 8. **模型与生成授权**：每个生成批次调用 `model_list` 查询当前能力、参考数、时长、画幅、声音和费用；检查实际选用的参考节点是否为锁定版本。向用户说明对象、数量、模型/规格及费用范围，取得针对该批次的授权后再调用 `generate_media`。音色资产只有用户需要且能力支持时才提交音频生成。 9. **生成、验收与交付**：通过 `task_get` 或真实事件状态核验；逐片检查角色/服装、场景、道具、动作因果、对白声音、空间连续、时长和画幅。未核验、失败或不一致的片段不得作为合格成片交付；保留候选和失败记录，修订只重跑受影响部分并重新授权。 -**硬停止条件**：原文/故事原稿未确认；故事圣经未确认；风格未定；主角/关键角色的识别参考缺失或冲突；当集主要场景母版缺失；关键道具/武器没有可复用定义；跨镜服装/伤态/空间状态未记录；正式分镜没有真实资产引用；上一镜 `endState` 与下一镜 `startState` 不一致；引用资源无法读取或未验收。遇到任一项，只能继续做文本方案、草分镜或资产补齐，不得启动视频生成。 +**硬停止条件**：原文/故事原稿未确认；故事圣经未确认；风格未定；主角/关键角色的识别参考缺失或冲突，或其角色卡未建立、不可读、三视图引用未就绪；当集主要场景母版缺失；关键道具/武器没有可复用定义；跨镜服装/伤态/空间状态未记录；正式分镜没有真实资产引用；上一镜 `endState` 与下一镜 `startState` 不一致；引用资源无法读取或未验收。遇到任一项，只能继续做文本方案、草分镜或资产补齐，不得启动视频生成。 ### 状态机 @@ -107,9 +107,9 @@ INTAKE 状态只能向前推进；用户拒绝或上游版本改变时，回退到受影响阶段，并保留旧版本及其引用关系。未达到当前阶段前，不得调用依赖后续阶段的工具。 -每个产物必须记录稳定 ID、版本、状态、来源版本和影响范围。角色、声音、环境、武器/道具使用画布背板组织为资产包；普通 Markdown 不能替代结构化分镜，单张剧照不能替代角色三视图或环境母版。 +每个产物必须记录稳定 ID、版本、状态、来源版本和影响范围。角色、声音、环境、武器/道具使用画布背板组织为资产包，核心角色的角色卡放在对应 `FRAME-CHAR` 背板内；普通 Markdown 不能替代结构化分镜，单张剧照不能替代角色三视图或环境母版。 -画布操作顺序仍遵循真实能力：复用已核实节点；使用 `canvas_list_node_types` 查合法类型；分镜通过 `canvas_create_storyboard` 创建、修改前 `canvas_read_storyboard`；普通节点/引用通过 `canvas_apply_ops`；整理使用 `canvas_arrange_nodes`。工具参数、审批和失败恢复以系统级合同及本轮真实 schema 为准。 +画布操作顺序仍遵循真实能力：复用已核实节点；使用 `canvas_list_node_types` 查合法类型；分镜通过 `canvas_create_storyboard` 创建、修改前 `canvas_read_storyboard`；角色卡通过 `canvas_create_character` 创建，不用 `canvas_apply_ops` 新建文本节点冒充；普通节点/引用通过 `canvas_apply_ops`；整理使用 `canvas_arrange_nodes`。工具参数、审批和失败恢复以系统级合同及本轮真实 schema 为准。 - `canvas_apply_ops`、分镜写入和媒体生成属于写操作，遵循当前 Agent 权限模式。 - `generate_media` 是可能收费的操作。用户未明确授权时，先输出方案和预计范围，使用 `ask_user` 获取一次具体确认。 @@ -198,10 +198,11 @@ INTAKE - 阶段状态、状态回退和前置门禁：读取 `references/state-machine.md`。 - 资产清单、角色/场景/道具和声音：读取 `references/assets.md`，按需读取 `references/methods/visual-design.md` 与 `references/methods/color-and-light.md`。 - 角色三视图、定妆图、音色、环境母版、武器和道具提示词：读取 `references/asset-prompt-templates.md`。 +- 核心角色角色卡的建卡条件、字段填写、引用方式和版本维护：读取 `references/character-cards.md`。 - 镜头起止状态、画面左右/前后景和跨镜连续性：读取 `references/continuity-rules.md`。 - 画布产物 ID、资产背板和结构化分镜字段：读取 `references/artifact-schemas.md`。 - 真实画布、分镜、模型和生成任务：读取 `references/canvas-execution.md`。 -- 版本、生成尝试、采用和交付：读取 `references/state-and-delivery.md`. +- 版本、生成尝试、采用和交付：读取 `references/state-and-delivery.md`。 方法文档只提供判断方法，不会覆盖影策工具合同，也不会把未确认提案升级为事实。 diff --git a/backend/builtin/影策短剧制作工作流/references/artifact-schemas.md b/backend/builtin/影策短剧制作工作流/references/artifact-schemas.md index 577ce4f40..cac5b5e86 100644 --- a/backend/builtin/影策短剧制作工作流/references/artifact-schemas.md +++ b/backend/builtin/影策短剧制作工作流/references/artifact-schemas.md @@ -43,6 +43,7 @@ SH-001 镜头 使用 `FRAME-CHAR-`，至少包含： ```text +CH-01-character-card（角色卡节点，核心角色必需） CH-01-character-bible.md CH-01-turnaround.png CH-01-portrait.png @@ -52,6 +53,8 @@ VO-CH-01-reference.wav（用户需要且平台支持时） 可选：表情表、姿态表、服装变体、伤势变体。 +角色卡由 `canvas_create_character` 创建，台账记录角色卡节点 ID、角色资产 ID 和版本号；分镜行的 `characterAssetIds` 写作 `CH-01@角色卡v1`，生成时引用角色卡节点。 + ## 环境资产背板 使用 `FRAME-LOC-`，至少包含： @@ -82,7 +85,7 @@ PROP-01-use-state.png（需要时） "durationSeconds": 8, "sourceVersion": "SRC-01@v1", "bibleVersion": "BIB-01@v1", - "characterAssetIds": ["CH-01@v1"], + "characterAssetIds": ["CH-01@角色卡v1"], "locationAssetId": "LOC-01@v1", "propAssetIds": ["PROP-01@v1"], "startState": {}, diff --git a/backend/builtin/影策短剧制作工作流/references/assets.md b/backend/builtin/影策短剧制作工作流/references/assets.md index eaf954520..cb49984c8 100644 --- a/backend/builtin/影策短剧制作工作流/references/assets.md +++ b/backend/builtin/影策短剧制作工作流/references/assets.md @@ -12,7 +12,7 @@ 资产登记表只负责索引；可复用资产必须按类型放入画布背板，避免把一张临时图片误认为完整资产。 -- `FRAME-CHAR-`：角色设定、三视图/转面、定妆图、服装变体、表情/姿态表。 +- `FRAME-CHAR-`：角色卡（核心角色）、角色设定、三视图/转面、定妆图、服装变体、表情/姿态表。 - `FRAME-VOICE-`：音色设定、说话人映射、参考音频和授权/能力说明。 - `FRAME-LOC-`：环境空间母版、光线/天气变体、连续性说明。 - `FRAME-PROP-`：武器/道具规格、正侧面结构图、使用状态和细节图。 @@ -53,6 +53,8 @@ 声音资产至少记录年龄感、音高、音色、气息、语速、咬字、情绪范围和禁用特征。参考音频只有在用户需要、已获授权且当前工具/模型支持时建立；没有参考音频时，音色设定仍可以作为 TTS 的稳定约束。 +核心角色（主角、反派、重要配角、跨场次或跨集复用的角色）在三视图验收后必须打包为角色卡，作为该角色唯一的正式引用；建卡、引用和版本维护按 `character-cards.md` 执行。 + 优先生成能被后续制作复用的角色参考组。若无法在一张图稳定表达多视图，拆成多张并分别登记用途。参考图只作为设计候选，必须检查同一人物各视角是否像同一人、服装结构是否一致、是否混入多余人物或文字，再确认锁定。 基础服装和剧情明确需要的服装变体分开登记。每个变体记录触发场次/事件、完整服装特征、损伤/污渍/湿润状态、适用镜头范围及继承项。换衣、受伤、淋湿或破损不得在镜头之间无因消失；未知服饰不得伪称为原文事实。 @@ -91,7 +93,7 @@ ## 验收与投产门槛 逐项验收： -- **角色**：各视角身份一致，年龄感/面部/发型/体型锚点稳定，服装与身份卡相符，无意外人物或错误附件。 +- **角色**：各视角身份一致，年龄感/面部/发型/体型锚点稳定，服装与身份卡相符，无意外人物或错误附件；核心角色已建角色卡，设定与身份卡一致，三视图引用就绪。 - **服装**：变体有明确剧情触发和适用范围；连续状态合理。 - **场景**：空间母版内部自洽，出入口、家具和光源位置可复用，参考范围与声称范围一致。 - **道具/武器**：轮廓、数量、尺寸关系、结构、持有人和状态与剧情一致。 diff --git a/backend/builtin/影策短剧制作工作流/references/canvas-execution.md b/backend/builtin/影策短剧制作工作流/references/canvas-execution.md index 558c4b0da..570485aea 100644 --- a/backend/builtin/影策短剧制作工作流/references/canvas-execution.md +++ b/backend/builtin/影策短剧制作工作流/references/canvas-execution.md @@ -17,6 +17,7 @@ ### 写入 - `canvas_apply_ops`：新增合法节点、更新允许字段、建立合法引用边；一次最多 20 项，必须带最新 `snapshotHash`。 +- `canvas_create_character`：用已验收、已就绪的三视图图片（可加参考音频）创建角色卡，写入角色库并放置角色卡节点；不用文本节点冒充角色卡。 - `canvas_create_storyboard`：创建有真实镜头行的 `script` 节点，不用 Markdown 冒充分镜。 - `canvas_edit_storyboard`：追加、修改或删除单行；`update/remove` 必须使用最近读取的真实 `rowId` 和 `snapshotHash`。 - `canvas_edit_batch_table`：只修改批量计划，不提交生成任务。 @@ -30,7 +31,7 @@ 1. `canvas_get_state`，确认当前项目、已有内容、快照、关联任务和当前采用版本。 2. `canvas_list_node_types`，确认文本、分镜及媒体节点类型和字段。 3. 写入/复用输入文稿与剧情事实账本。剧本阶段可产出草分镜，但显式标记“草案，不可投产”。 -4. 先确认共享风格基底，再建立角色、服装、场景、关键道具/武器资产台账；按 `assets.md` 生成/登记参考，回读真实资源并完成验收、确认和版本锁定。音色参考只有用户选择时才准备。 +4. 先确认共享风格基底，再建立角色、服装、场景、关键道具/武器资产台账；按 `assets.md` 生成/登记参考，回读真实资源并完成验收、确认和版本锁定。核心角色验收后用 `canvas_create_character` 建角色卡并回读引用状态（见 `character-cards.md`）。音色参考只有用户选择时才准备。 5. 检查投产门槛：必需资产均已锁定、无未解决冲突，风格基底已确认。未通过则停止于资产/草案阶段，不创建制作版分镜，不提交视频任务。 6. 通过门槛后调用 `canvas_create_storyboard` 创建制作版分镜，行内写入已确认事件、时长、场景状态、服装/道具状态、对白/声音、连续性字段和真实资产引用。 7. 创建后 `canvas_read_storyboard` 回读，核对行数、顺序、时长、资产引用及真实行 ID。修改时再次读取最新快照，再调用 `canvas_edit_storyboard`；冲突则重新读取，不覆盖用户新改内容。 diff --git a/backend/builtin/影策短剧制作工作流/references/character-cards.md b/backend/builtin/影策短剧制作工作流/references/character-cards.md new file mode 100644 index 000000000..ee441b773 --- /dev/null +++ b/backend/builtin/影策短剧制作工作流/references/character-cards.md @@ -0,0 +1,71 @@ +# 核心角色卡资产 + +角色卡是核心角色在账号角色库中的正式资产，也是画布上可被分镜和生成稳定引用的角色身份。画布里的三视图、定妆图和音色说明只是候选与证据；只有打包成角色卡并通过验收，角色才算“已锁定”。 + +## 哪些角色必须建卡 + +- 主角、反派和重要配角； +- 跨两个以上场次/片段出镜，或跨集复用的角色； +- 剧情需要观众辨认身份的角色（例如回忆、伪装、相似人物对照）。 + +一次性路人、群演和只出现声音的模糊身份不建卡，直接在提示词或文本节点中描述，并在资产台账里记录“未建卡”及理由。 + +## 建卡前置条件 + +1. 风格基底已确认，故事圣经中该角色的身份卡已确认。 +2. 三视图/转面图已生成、实际查看并验收：各视角像同一人，年龄感、脸型、发型、服装、比例与身份卡一致，无多余人物、文字或手持武器。 +3. 该图片节点已保存到账号资源库且状态就绪；外部 URL、未完成任务或仅有提示词的节点不能建卡。 +4. 用户启用音色资产时，参考音频已获授权、格式受支持且已就绪；未启用时角色卡不绑定声音，不影响建卡。 +5. 先用 `canvas_get_state` 检查画布上是否已有同名角色卡；已有则复用并核对版本，不重复建卡。 + +未满足前，角色停在“待生成/待验收/待确认”，三视图和定妆图留在 `FRAME-CHAR-` 背板中作为候选。 + +## 建卡 + +使用 `canvas_create_character`，参数以本轮真实 schema 为准： + +- `name`：与故事圣经一致的角色称呼；服装/形态变体卡用“角色名·变体名”，避免与基础卡同名。 +- `imageNodeId`：已验收的三视图节点。它会成为角色卡的三视图与主形象，后续生成默认引用它，所以不要传剧照、半身特写或未验收候选。 +- `audioNodeId`：仅在用户启用音色并验收参考音频后传入。 +- `definition`：只填故事圣经和已验收图片能支撑的字段，未知项留空，不补设定。 + +| 字段 | 来源 | +|---|---| +| `role` | 剧中身份、阵营与关系 | +| `aliases` | 原文中的别称、称号 | +| `appearance` | 年龄感、脸型、发型发色、面部识别锚点 | +| `physique` | 身高比例、体型轮廓 | +| `clothing` | 基础服装、颜色、材质、固定配饰 | +| `personality` | 影响表演的性格与神态 | +| `props` | 固定随身物（武器仍单独登记为 PROP 资产） | +| `consistencyPrompt` | 不允许漂移的锚点，写成可直接拼进提示词的短句 | +| `multiViewPrompt` | 生成该三视图所用的已锁定提示词 | +| `voiceLanguage` / `voiceAge` / `voiceTimbre` | 已确认的音色设定（启用音色时） | + +建卡属于写操作，按权限审批。批准后用 `canvas_get_state` 精读该节点，确认 `character.version`、`visualStatus`、`imageReference.ready`，启用声音时确认 `audioReference.ready`；再把角色卡节点放入 `FRAME-CHAR-` 背板，并在资产台账登记 `CH-xx`、角色卡节点 ID、角色资产 ID、版本号、状态和确认依据。 + +## 在分镜与生成中使用 + +- 正式分镜行的 `characterAssetIds` 指向角色卡并写明版本，例如 `CH-01@角色卡v2`。 +- 生成时把角色卡节点作为参考节点传入：系统自动附带三视图参考和角色设定文字，音频生成自动使用绑定声音。不要再为同一角色另传三视图图片节点，也不要在提示词里重复粘贴整段设定，只写本镜的表情、动作和状态。 +- 角色卡占用图片参考名额并计入 `@图片N` 顺序；先用 `model_list` 带上真实参考节点核对参考上限。 +- 节点正文为空不代表角色卡为空，设定以 `character.definition` 为准。 +- 角色卡不可读或 `imageReference.ready=false` 时，该角色视为未锁定，触发硬停止。 + +## 维护与版本 + +Agent 不能直接修改角色卡的设定、形象或声音；这些修改在角色库中完成，每次修改都会生成新版本。维护流程： + +1. **发现需要维护**：剧情新增可核实的外观事实、验收时发现漂移、用户要求改设定或换音色。 +2. **提出变更单**：列出角色、字段、旧值、新值、依据（事件编号或用户确认），以及受影响的分镜行和已生成片段，请用户在角色库中修改。 +3. **回读核对**：修改后精读角色卡，确认版本号已递增、内容与变更单一致，并更新台账版本。 +4. **影响处理**：Agent 放置的角色卡跟随角色库当前版本，之后的生成自动使用新版本；已提交的生成任务记录了当时使用的角色卡版本。按 `state-and-delivery.md` 判断哪些片段需要重做，未受影响的已采用结果继续有效，重做需要重新授权。 + +服装、伤态等剧情变体优先作为分镜状态和 `FRAME-CHAR` 中的变体参考图管理；只有变体长期出镜且必须单独锁定外观时，才另建“角色名·变体名”角色卡，并在台账写明适用场次范围。 + +## 禁止事项 + +- 不用 `canvas_apply_ops` 新建文本节点冒充角色卡，也不手写角色卡 metadata。 +- 不用未验收图片、剧照或网络图片建卡；不为凑齐资产伪造设定。 +- 不把“角色卡已创建”报告为“角色资产已全部锁定”：服装变体、武器和音色仍按各自台账验收。 +- 不为已有同名角色重复建卡；已有卡不符合设定时走维护流程。 diff --git a/backend/builtin/影策短剧制作工作流/references/state-machine.md b/backend/builtin/影策短剧制作工作流/references/state-machine.md index 22b707466..c280510cf 100644 --- a/backend/builtin/影策短剧制作工作流/references/state-machine.md +++ b/backend/builtin/影策短剧制作工作流/references/state-machine.md @@ -64,4 +64,6 @@ MEDIA_READY -> POST_READY DRAFT -> REVIEW -> APPROVED -> LOCKED ``` +核心角色只有角色卡已创建、可读且三视图引用就绪时才能进入 `LOCKED`；角色卡在角色库中改版后，台账版本同步更新，受影响下游回到待复核。 + 用户拒绝时回到 `DRAFT`；资产内容变化时递增版本号，保留旧版本和旧引用，不覆盖仍被使用的结果。 \ No newline at end of file diff --git a/backend/internal/agent/runtime/runtime.go b/backend/internal/agent/runtime/runtime.go index c1403c72e..136d9b0ed 100644 --- a/backend/internal/agent/runtime/runtime.go +++ b/backend/internal/agent/runtime/runtime.go @@ -33,18 +33,13 @@ type Bridge struct { type ProcessRequest struct { BridgeURL string `json:"bridgeURL"` BridgeToken string `json:"bridgeToken"` - SessionFile string `json:"sessionFile,omitempty"` SessionJSONL string `json:"sessionJSONL,omitempty"` - SessionDir string `json:"sessionDir,omitempty"` SessionID string `json:"sessionId,omitempty"` UserID string `json:"userId,omitempty"` CanvasID string `json:"canvasId,omitempty"` RunID string `json:"runId,omitempty"` - CWD string `json:"cwd"` - AgentDir string `json:"agentDir"` Prompt string `json:"prompt"` SystemPrompt string `json:"systemPrompt"` - SkillPaths []string `json:"skillPaths,omitempty"` EnabledSkills []map[string]any `json:"enabledSkills,omitempty"` Profile map[string]any `json:"profile,omitempty"` Memory map[string]any `json:"memory,omitempty"` @@ -92,10 +87,10 @@ func Run(ctx context.Context, request ProcessRequest, bridge Bridge) error { cmd := exec.CommandContext(ctx, "node", "--max-old-space-size="+nodeMemoryMB(), filepath.Join(runtimeDir, "agent-runtime.mjs")) cmd.Dir = runtimeDir cmd.Stdin = strings.NewReader(string(payload)) - // 环境变量白名单：只传 Node 运行所需的最少变量。 + // 环境变量白名单：只传 Node 运行所需的最少变量。HOME 与会话、工作目录 + // 由运行时自己建的临时隔离目录提供，不使用服务端数据目录。 cmd.Env = []string{ "PATH=" + os.Getenv("PATH"), - "HOME=" + request.AgentDir, "NODE_ENV=production", "PI_OFFLINE=1", } diff --git a/backend/internal/app/asset_library.go b/backend/internal/app/asset_library.go index c9817eb1d..26c919074 100644 --- a/backend/internal/app/asset_library.go +++ b/backend/internal/app/asset_library.go @@ -58,9 +58,14 @@ func (s *Service) UserAssetsPage(userID string, page int, pageSize int, filter U } rawAssets := make([]json.RawMessage, 0, len(assets)) for _, asset := range assets { - if payload := clientAssetListPayload(asset); len(payload) > 0 { - rawAssets = append(rawAssets, payload) + payload := clientAssetListPayload(asset) + if len(payload) == 0 { + continue } + if asset.Kind == "entity" && asset.Category == model.AssetCategoryCharacter { + payload = s.characterAssetListPayload(userID, asset, payload) + } + rawAssets = append(rawAssets, payload) } kindRows, categoryRows, folderRows, err := s.repo.UserAssetFacets(userID, filter.Status) if err != nil { @@ -72,6 +77,53 @@ func (s *Service) UserAssetsPage(userID string, page int, pageSize int, filter U }, nil } +// characterAssetListPayload 给素材库里的角色卡补上当前版本的形象与声音。 +// 角色资产的 payload 只存设定，形象/声音在版本表里；不补的话列表卡片和档案只能显示空图标。 +// 只下发 resource: 存储键，展示时由前端换取授权地址，不经后端中转字节。 +func (s *Service) characterAssetListPayload(userID string, asset model.Asset, raw json.RawMessage) json.RawMessage { + card, err := s.characterCard(userID, &asset) + if err != nil { + return raw + } + var payload map[string]any + if err := json.Unmarshal(raw, &payload); err != nil || payload == nil { + return raw + } + data, _ := payload["data"].(map[string]any) + if data == nil { + data = map[string]any{} + } + data["definition"] = card.Definition + data["version"] = card.Version + data["visualStatus"], data["voiceStatus"] = card.VisualStatus, card.VoiceStatus + if cover := characterCoverResourceID(card); cover != "" { + data["coverStorageKey"] = "resource:" + cover + } + if card.Voice != nil { + data["voiceName"] = card.Voice.Profile.Name + if card.Voice.Profile.SampleResourceID != "" { + data["voiceSampleStorageKey"] = "resource:" + card.Voice.Profile.SampleResourceID + } + } + payload["data"] = data + encoded, err := json.Marshal(payload) + if err != nil { + return raw + } + return encoded +} + +func characterCoverResourceID(card CharacterCardSummary) string { + for _, role := range []string{"turnaround_sheet", "primary", "front"} { + for _, representation := range card.Representations { + if representation.Role == role && representation.ResourceID != "" { + return representation.ResourceID + } + } + } + return "" +} + func assetFacetMap(rows []repository.UserAssetFacetRow) map[string]int64 { result := make(map[string]int64, len(rows)) for _, row := range rows { diff --git a/backend/internal/app/channel_models_pricing.go b/backend/internal/app/channel_models_pricing.go index 878d5b57b..1906e7c5a 100644 --- a/backend/internal/app/channel_models_pricing.go +++ b/backend/internal/app/channel_models_pricing.go @@ -218,8 +218,8 @@ func validateChannelModelTierPricing(capability string, protocol model.ChannelIn if billingMode != "fixed_request" && billingMode != "per_second" && billingMode != "token" { return BadAuthRequest("模型计费方式仅支持按次、按秒或 Token") } - if billingMode == "per_second" && capability != "video" { - return BadAuthRequest("只有视频模型可以按秒计费") + if billingMode == "per_second" && capability != "video" && capability != "audio" { + return BadAuthRequest("只有视频或音频模型可以按秒计费") } if billingMode == "token" && !supportsTokenBilling(capability, protocol) { return BadAuthRequest("Token 计费仅支持文本和视频模型") diff --git a/backend/internal/app/cloud_agent.go b/backend/internal/app/cloud_agent.go index 28138ed64..590fced39 100644 --- a/backend/internal/app/cloud_agent.go +++ b/backend/internal/app/cloud_agent.go @@ -696,7 +696,7 @@ func cloudAgentCanvasSummary(canvas *model.CanvasProject, focusNodeIDs ...string if node.Type == "text" && node.Metadata.WorkflowKind == "character" { item["kind"] = "character" } - if _, known := cloudAgentNodeCapabilityForType(node.Type); !known { + if _, known := canvasCapabilityRegistry.ResolveNode(node.Type, node.Metadata.WorkflowKind); !known { item["agentSupported"] = false } nodes = append(nodes, item) diff --git a/backend/internal/app/cloud_agent_anchor.go b/backend/internal/app/cloud_agent_anchor.go index aa051ac55..d580ab699 100644 --- a/backend/internal/app/cloud_agent_anchor.go +++ b/backend/internal/app/cloud_agent_anchor.go @@ -50,7 +50,7 @@ func cloudAgentCreativeAnchorForCanvas(repo *repository.Repository, userID strin ids := make([]string, 0, 16) for _, node := range nodes { - descriptor, known := cloudAgentNodeCapabilityForType(stringValue(node["type"])) + descriptor, known := cloudAgentNodeCapabilityForNode(node) if known && descriptor.Connection.CanReference { ids = append(ids, stringValue(node["id"])) } @@ -63,13 +63,13 @@ func cloudAgentCreativeAnchorForCanvas(repo *repository.Repository, userID strin continue } node := byID[id] - descriptor, known := cloudAgentNodeCapabilityForType(stringValue(node["type"])) + descriptor, known := cloudAgentNodeCapabilityForNode(node) if !known || !descriptor.Connection.CanReference { continue } meta, _ := node["metadata"].(map[string]any) item := cloudAgentReferenceAnchor{ - NodeID: stringValue(node["id"]), Type: stringValue(node["type"]), + NodeID: stringValue(node["id"]), Type: descriptor.Type, Title: truncateRunes(stringValue(node["title"]), 300), Prompt: truncateRunes(firstNonEmpty(stringValue(meta["prompt"]), stringValue(meta["composerContent"])), 1000), VisualIdentity: "unknown", RequiresVisualInspection: true, @@ -82,7 +82,8 @@ func cloudAgentCreativeAnchorForCanvas(repo *repository.Repository, userID strin } } if repo != nil { - ref, _, refErr := cloudAgentReference(repo, userID, node) + // 角色卡按三视图判定是否可作参考；普通媒体按节点资源判定。 + ref, _, refErr := cloudAgentMediaReference(repo, userID, "", node) if refErr == nil { item.ReferenceReady = true item.Width, item.Height = ref["width"], ref["height"] diff --git a/backend/internal/app/cloud_agent_approval_preview.go b/backend/internal/app/cloud_agent_approval_preview.go index 5560bbec8..25ae1d021 100644 --- a/backend/internal/app/cloud_agent_approval_preview.go +++ b/backend/internal/app/cloud_agent_approval_preview.go @@ -167,8 +167,8 @@ func applyCloudAgentCanvasPlan(doc map[string]any, ops []agentCanvasOp) ([]cloud return nil, BadAuthRequest("连线重复") } } - fromCapability, _ := cloudAgentNodeCapabilityForType(stringValue(nodes[fromIndex]["type"])) - toCapability, _ := cloudAgentNodeCapabilityForType(stringValue(nodes[toIndex]["type"])) + fromCapability, _ := cloudAgentNodeCapabilityForNode(nodes[fromIndex]) + toCapability, _ := cloudAgentNodeCapabilityForNode(nodes[toIndex]) fromTitle := cloudAgentApprovalNodeTitle(nodes[fromIndex], fromCapability.Label) toTitle := cloudAgentApprovalNodeTitle(nodes[toIndex], toCapability.Label) edges = append(edges, map[string]any{"id": op.ID, "fromNodeId": op.FromNodeID, "toNodeId": op.ToNodeID}) @@ -188,7 +188,7 @@ func applyCloudAgentCanvasPlan(doc map[string]any, ops []agentCanvasOp) ([]cloud if index < 0 { return nil, BadAuthRequest("只能更新现有且受 Agent 支持的节点") } - capability, ok := cloudAgentNodeCapabilityForType(stringValue(nodes[index]["type"])) + capability, ok := cloudAgentNodeCapabilityForNode(nodes[index]) if !ok || !capability.CanUpdate { return nil, BadAuthRequest("该节点类型不支持 Agent 更新") } diff --git a/backend/internal/app/cloud_agent_canvas_events.go b/backend/internal/app/cloud_agent_canvas_events.go index 7b5b4e267..5c1b0c460 100644 --- a/backend/internal/app/cloud_agent_canvas_events.go +++ b/backend/internal/app/cloud_agent_canvas_events.go @@ -138,7 +138,7 @@ func emitCloudAgentCanvasChange(repo *repository.Repository, runID string, state if old, _ := change["before"].(map[string]any); old == nil { action = "created" } - entry := map[string]any{"action": action, "nodeId": node["id"], "title": node["title"], "nodeType": node["type"]} + entry := map[string]any{"action": action, "nodeId": node["id"], "title": node["title"], "nodeType": cloudAgentEventNodeType(node)} previewOperation := "update_node" if action == "created" { previewOperation = "add_node" @@ -166,9 +166,9 @@ func emitCloudAgentCanvasChange(repo *repository.Repository, runID string, state for _, change := range edges { edge := change["after"].(map[string]any) if node := byID[stringValue(edge["fromNodeId"])]; node != nil { - entry := map[string]any{"action": "referenced", "nodeId": node["id"], "title": node["title"], "nodeType": node["type"], "targetNodeId": edge["toNodeId"]} + entry := map[string]any{"action": "referenced", "nodeId": node["id"], "title": node["title"], "nodeType": cloudAgentEventNodeType(node), "targetNodeId": edge["toNodeId"]} if target := byID[stringValue(edge["toNodeId"])]; target != nil { - entry["targetTitle"], entry["targetNodeType"] = target["title"], target["type"] + entry["targetTitle"], entry["targetNodeType"] = target["title"], cloudAgentEventNodeType(target) } if preview, ok := findPreview("connect_nodes", stringValue(edge["fromNodeId"])); ok && preview.TargetNodeTitle != "" { entry["targetTitle"], entry["targetNodeType"] = preview.TargetNodeTitle, preview.TargetNodeType @@ -194,6 +194,15 @@ func emitCloudAgentCanvasChange(repo *repository.Repository, runID string, state return nil } +// cloudAgentEventNodeType 给回执用的节点类型：角色卡等变体底层是 text， +// 必须按 workflowKind 解析成能力类型，否则前端会把角色卡报成"文本节点"。 +func cloudAgentEventNodeType(node map[string]any) any { + if descriptor, ok := cloudAgentNodeCapabilityForNode(node); ok && descriptor.Type != "" { + return descriptor.Type + } + return node["type"] +} + func cloudAgentPatchCoversDocument(before, after map[string]any) bool { for _, key := range []string{"nodes", "connections"} { old, next := creationMaps(before[key]), creationMaps(after[key]) diff --git a/backend/internal/app/cloud_agent_canvas_state.go b/backend/internal/app/cloud_agent_canvas_state.go index 50a777ecc..b81b7251a 100644 --- a/backend/internal/app/cloud_agent_canvas_state.go +++ b/backend/internal/app/cloud_agent_canvas_state.go @@ -248,7 +248,7 @@ func cloudAgentCanvasStateSelected(repo *repository.Repository, userID, canvasID if status, ok := meta["status"].(string); ok { item["status"] = truncateRunes(status, 40) } - capability, known := cloudAgentNodeCapabilityForType(stringValue(node["type"])) + capability, known := cloudAgentNodeCapabilityForNode(node) if !known { // Read visibility is not permission to mutate or use a node as a media reference. item["agentSupported"] = false @@ -263,6 +263,10 @@ func cloudAgentCanvasStateSelected(repo *repository.Repository, userID, canvasID included[id] = true continue } + if capability.Variant != nil { + // 变体节点（如角色卡）底层 type 仍是 text，用 kind 标明真实能力，避免当普通文本处理。 + item["kind"] = capability.Type + } fields := capability.SummaryFields if precise { fields = capability.DetailFields @@ -333,7 +337,8 @@ func cloudAgentCanvasStateSelected(repo *repository.Repository, userID, canvasID } item["generationDraft"] = draft } - if capability.Connection.CanReference { + // 角色卡的引用可用性在 character.imageReference/audioReference 中给出。 + if capability.Connection.CanReference && !cloudAgentCharacterNode(node) { ref, _, err := cloudAgentReference(repo, userID, node) outputReference := map[string]any{"ready": err == nil} item["outputReference"] = outputReference diff --git a/backend/internal/app/cloud_agent_character.go b/backend/internal/app/cloud_agent_character.go index 9c1af86c0..88025d2e1 100644 --- a/backend/internal/app/cloud_agent_character.go +++ b/backend/internal/app/cloud_agent_character.go @@ -23,12 +23,13 @@ func cloudAgentCharacterNode(node map[string]any) bool { func cloudAgentResolveCharacter(repo *repository.Repository, userID, projectID string, node map[string]any) (*cloudAgentCharacter, error) { meta, _ := node["metadata"].(map[string]any) assetID := stringValue(meta["characterAssetId"]) - if projectID == "" || assetID == "" { - return nil, BadAuthRequest("角色卡未关联当前项目的角色资产") + if assetID == "" { + return nil, BadAuthRequest("角色卡未关联角色资产") } - asset, err := repo.ProjectCharacterAsset(userID, projectID, assetID) + _ = projectID + asset, err := repo.UserCharacterAsset(userID, assetID) if err != nil { - return nil, BadAuthRequest("角色卡对应的项目角色资产不存在或不可访问") + return nil, BadAuthRequest("角色卡不存在或不可访问") } policy := stringValue(meta["characterVersionPolicy"]) versionID := asset.PrimaryVersionID diff --git a/backend/internal/app/cloud_agent_character_create.go b/backend/internal/app/cloud_agent_character_create.go new file mode 100644 index 000000000..cf45a8500 --- /dev/null +++ b/backend/internal/app/cloud_agent_character_create.go @@ -0,0 +1,266 @@ +package app + +import ( + "fmt" + "strings" + "unicode/utf8" + + "infinite-canvas/backend/internal/canvas/layout" + "infinite-canvas/backend/internal/repository" +) + +const ( + cloudAgentCharacterNodeWidth = 264.0 + cloudAgentCharacterNodeHeight = 352.0 +) + +// cloudAgentCharacterDefinitionArgs 是 Agent 可写的角色设定字段；未声明的字段会被拒绝。 +type cloudAgentCharacterDefinitionArgs struct { + Role string `json:"role"` + Aliases []string `json:"aliases"` + Appearance string `json:"appearance"` + Physique string `json:"physique"` + Clothing string `json:"clothing"` + Personality string `json:"personality"` + Props string `json:"props"` + ConsistencyPrompt string `json:"consistencyPrompt"` + MultiViewPrompt string `json:"multiViewPrompt"` + VoiceLanguage string `json:"voiceLanguage"` + VoiceAge string `json:"voiceAge"` + VoiceTimbre string `json:"voiceTimbre"` +} + +type cloudAgentCharacterCreateArgs struct { + NodeID string `json:"nodeId"` + Name string `json:"name"` + ImageNodeID string `json:"imageNodeId"` + AudioNodeID string `json:"audioNodeId"` + Definition cloudAgentCharacterDefinitionArgs `json:"definition"` + X *float64 `json:"x"` + Y *float64 `json:"y"` +} + +type cloudAgentCharacterCreatePlan struct { + Args cloudAgentCharacterCreateArgs + Definition map[string]any + ImageResourceID string + AudioResourceID string + VoiceName string + Position layout.Position + Preview cloudAgentApprovalPreview +} + +// 设定字段只在描述里列出键名以控制工具 schema 体积；解码时 DisallowUnknownFields 会拒绝未声明的键。 +func cloudAgentCharacterCreateSchema() map[string]any { + str := func(description string) map[string]any { + return map[string]any{"type": "string", "description": description} + } + return map[string]any{ + "nodeId": str("新节点ID"), "name": str("角色名"), "imageNodeId": str("就绪的形象图片节点ID"), "audioNodeId": str("可选，就绪的声音音频节点ID"), + "definition": map[string]any{"type": "object", "description": "可选设定：role/appearance/physique/clothing/personality/props/consistencyPrompt/multiViewPrompt/voiceLanguage/voiceAge/voiceTimbre 为字符串，aliases 为字符串数组"}, + "x": map[string]any{"type": "number"}, "y": map[string]any{"type": "number"}, + } +} + +func cloudAgentCharacterText(value string, limit int) (string, error) { + value = strings.TrimSpace(value) + if utf8.RuneCountInString(value) > limit { + return "", BadAuthRequest(fmt.Sprintf("角色设定单项不能超过 %d 字", limit)) + } + return value, nil +} + +func (d cloudAgentCharacterDefinitionArgs) normalized() (map[string]any, error) { + result := map[string]any{} + for key, value := range map[string]string{ + "role": d.Role, "appearance": d.Appearance, "physique": d.Physique, "clothing": d.Clothing, + "personality": d.Personality, "props": d.Props, "consistencyPrompt": d.ConsistencyPrompt, + "multiViewPrompt": d.MultiViewPrompt, "voiceLanguage": d.VoiceLanguage, "voiceAge": d.VoiceAge, "voiceTimbre": d.VoiceTimbre, + } { + text, err := cloudAgentCharacterText(value, 4000) + if err != nil { + return nil, err + } + if text != "" { + result[key] = text + } + } + if len(d.Aliases) > 16 { + return nil, BadAuthRequest("角色别名最多 16 个") + } + aliases := []string{} + for _, alias := range d.Aliases { + text, err := cloudAgentCharacterText(alias, 100) + if err != nil { + return nil, err + } + if text != "" { + aliases = append(aliases, text) + } + } + result["aliases"] = aliases + return result, nil +} + +// cloudAgentCharacterMediaResource 从画布节点取已上传且就绪的账号资源；只接受真实节点，不接受 URL。 +func cloudAgentCharacterMediaResource(repo *repository.Repository, userID string, nodes map[string]map[string]any, nodeID, kind, label string) (string, map[string]any, error) { + node := nodes[nodeID] + if node == nil || stringValue(node["type"]) != kind { + return "", nil, BadAuthRequest(label + "节点不在当前画布或类型不对") + } + meta, _ := node["metadata"].(map[string]any) + key := stringValue(meta["storageKey"]) + if !strings.HasPrefix(key, "resource:") { + return "", nil, BadAuthRequest(label + "尚未保存到账号资源库，不能用于角色卡") + } + resource, err := repo.ResourceForUser(userID, strings.TrimPrefix(key, "resource:")) + if err != nil || resource == nil || resource.Kind != kind || resource.Status != "ready" { + return "", nil, BadAuthRequest(label + "资源不存在、未就绪或类型不匹配") + } + if kind == "audio" && !isSupportedVoiceSampleMimeType(resource.MimeType) { + return "", nil, BadAuthRequest("声音音频格式不支持：请使用 MP3、WAV、M4A/AAC、FLAC、OGG/Opus 或 WebM") + } + return resource.ID, node, nil +} + +func prepareCloudAgentCharacterCreate(repo *repository.Repository, userID, canvasID string, call cloudAgentCall) (*cloudAgentCharacterCreatePlan, error) { + var args cloudAgentCharacterCreateArgs + if err := decodeCloudAgentJSONObject(call.Function.Arguments, &args); err != nil { + return nil, &cloudAgentArgumentError{BadAuthRequest("创建角色卡参数无效：仅允许 nodeId、name、imageNodeId、audioNodeId、definition、x、y")} + } + if err := validateCloudAgentID(args.NodeID, "角色卡节点ID", 80); err != nil { + return nil, &cloudAgentArgumentError{err} + } + name, err := cloudAgentCharacterText(args.Name, 80) + if err != nil || name == "" { + return nil, &cloudAgentArgumentError{BadAuthRequest("角色名不能为空且不超过 80 字")} + } + if args.ImageNodeID == "" { + return nil, &cloudAgentArgumentError{BadAuthRequest("创建角色卡需要 imageNodeId：先有一张已就绪的形象/三视图图片节点")} + } + definition, err := args.Definition.normalized() + if err != nil { + return nil, &cloudAgentArgumentError{err} + } + canvas, err := repo.CanvasProjectForUser(userID, canvasID) + if err != nil { + return nil, err + } + doc, err := creationDocument(canvas.PayloadJSON) + if err != nil { + return nil, err + } + nodes, err := creationObjects(doc["nodes"]) + if err != nil { + return nil, err + } + if nodes[args.NodeID] != nil { + return nil, &cloudAgentArgumentError{BadAuthRequest("角色卡节点ID已存在，请换一个新ID")} + } + plan := &cloudAgentCharacterCreatePlan{Args: args, Definition: definition} + plan.Args.Name = name + imageID, imageNode, err := cloudAgentCharacterMediaResource(repo, userID, nodes, args.ImageNodeID, "image", "形象图片") + if err != nil { + return nil, err + } + plan.ImageResourceID = imageID + details := []string{"形象：《" + cloudAgentApprovalNodeTitle(imageNode, "图片") + "》"} + if args.AudioNodeID != "" { + audioID, audioNode, err := cloudAgentCharacterMediaResource(repo, userID, nodes, args.AudioNodeID, "audio", "声音音频") + if err != nil { + return nil, err + } + plan.AudioResourceID = audioID + plan.VoiceName = truncateRunes(strings.TrimSpace(stringValue(audioNode["title"])), 160) + details = append(details, "声音：《"+cloudAgentApprovalNodeTitle(audioNode, "音频")+"》") + } + if len(definition) > 1 { + details = append(details, fmt.Sprintf("设定 %d 项", len(definition)-1)) + } + // 显式坐标按原样使用；否则落在形象图片右侧的空位，不叠在已有节点上。 + if args.X != nil && args.Y != nil { + plan.Position = layout.Position{X: *args.X, Y: *args.Y} + } else { + occupied := cloudAgentLayoutNodes(doc) + var preferred *layout.Position + if image, ok := findLayoutNode(occupied, args.ImageNodeID); ok { + preferred = &layout.Position{X: image.X + image.Width + layout.PlacementGap, Y: image.Y} + } + plan.Position = layout.FreeSlot(cloudAgentCharacterNodeWidth, cloudAgentCharacterNodeHeight, "text", occupied, preferred, layout.PlacementGap) + } + withAudio := "" + if plan.AudioResourceID != "" { + withAudio = "和音频" + } + plan.Preview = cloudAgentApprovalPreview{ + Kind: "canvas_mutation", Title: "确认创建角色卡", + Description: fmt.Sprintf("Agent 准备把画布上的图片%s打包为角色卡《%s》：写入账号角色库并在画布放置角色卡节点。批准后才会执行。", withAudio, name), + Items: []cloudAgentApprovalPreviewItem{{Operation: "create_character", NodeID: args.NodeID, NodeTitle: name, NodeType: "character", NodeTypeLabel: "角色卡", Details: details, Summary: fmt.Sprintf("创建角色卡《%s》", name)}}, + } + return plan, nil +} + +func cloudAgentCharacterReferencePrompt(name string, definition map[string]any) string { + parts := []string{"【角色卡：" + name + "】"} + for _, key := range []string{"role", "appearance", "physique", "clothing", "personality", "props", "consistencyPrompt"} { + if value := strings.TrimSpace(stringValue(definition[key])); value != "" { + parts = append(parts, value) + } + } + return strings.Join(parts, "\n") +} + +// applyCloudAgentCharacterCreate 在 Agent 检查点事务里建角色资产并放置角色卡节点；任一步失败整体回滚。 +func applyCloudAgentCharacterCreate(repo *repository.Repository, userID, canvasID string, call cloudAgentCall, policy RuntimePolicySetting, recorder ...cloudAgentMutationRecorder) (any, error) { + plan, err := prepareCloudAgentCharacterCreate(repo, userID, canvasID, call) + if err != nil { + return nil, err + } + created, err := (&Service{repo: repo}).CreateCharacter(userID, CreateCharacterRequest{ + Name: plan.Args.Name, Definition: plan.Definition, ImageResourceID: plan.ImageResourceID, + AudioResourceID: plan.AudioResourceID, VoiceName: plan.VoiceName, + }) + if err != nil { + return nil, err + } + canvas, err := repo.CanvasProjectForUser(userID, canvasID) + if err != nil { + return nil, err + } + doc, err := creationDocument(canvas.PayloadJSON) + if err != nil { + return nil, err + } + beforeJSON, beforeHash := canvas.PayloadJSON, cloudAgentCanvasHash(doc) + card := created.Character + meta := map[string]any{ + "workflowKind": "character", "characterAssetId": created.Asset.ID, "assetId": created.Asset.ID, + "characterVersionId": card.VersionID, "characterVersionPolicy": "current", + "characterName": plan.Args.Name, "characterPrompt": cloudAgentCharacterReferencePrompt(plan.Args.Name, plan.Definition), + "characterAliases": plan.Definition["aliases"], "characterDefinition": plan.Definition, + "characterVisualStatus": card.VisualStatus, "characterVoiceStatus": card.VoiceStatus, + "status": "success", "fontSize": float64(14), + } + if card.Voice != nil { + meta["characterVoiceName"] = card.Voice.Profile.Name + } + node := map[string]any{ + "id": plan.Args.NodeID, "type": "text", "title": plan.Args.Name, + "position": map[string]any{"x": plan.Position.X, "y": plan.Position.Y}, + "width": cloudAgentCharacterNodeWidth, "height": cloudAgentCharacterNodeHeight, "metadata": meta, + } + doc["nodes"] = append(creationMaps(doc["nodes"]), node) + if err := saveCloudAgentDocument(repo, canvas, doc, policy); err != nil { + return nil, err + } + if len(recorder) > 0 && recorder[0] != nil { + if err := recorder[0](repo, cloudAgentMutationInput{UserID: userID, CanvasID: canvasID, StepID: call.ID, Operation: call.Function.Name, BeforeSnapshotHash: beforeHash, AfterSnapshotHash: cloudAgentCanvasHash(doc), BeforeJSON: beforeJSON, Preview: &plan.Preview}); err != nil { + return nil, err + } + } + return map[string]any{ + "canvasId": canvasID, "nodeId": plan.Args.NodeID, "characterAssetId": created.Asset.ID, "versionId": card.VersionID, + "visualStatus": card.VisualStatus, "voiceStatus": card.VoiceStatus, "snapshotHash": cloudAgentCanvasHash(doc), + "summary": plan.Preview.Items[0].Summary, + }, nil +} diff --git a/backend/internal/app/cloud_agent_character_create_test.go b/backend/internal/app/cloud_agent_character_create_test.go new file mode 100644 index 000000000..e68b158b1 --- /dev/null +++ b/backend/internal/app/cloud_agent_character_create_test.go @@ -0,0 +1,96 @@ +package app + +import ( + "encoding/json" + "strings" + "testing" + + "infinite-canvas/backend/internal/model" +) + +func TestCloudAgentCreatesCharacterCardFromCanvasMedia(t *testing.T) { + s, db, _, _ := creationTestService(t) + doc := map[string]any{"nodes": []any{ + map[string]any{"id": "portrait", "type": "image", "title": "李莫愁·定妆", "position": map[string]any{"x": 0.0, "y": 0.0}, "width": 300.0, "height": 400.0, "metadata": map[string]any{"storageKey": "resource:portrait-res"}}, + map[string]any{"id": "voice", "type": "audio", "title": "李莫愁·音色", "position": map[string]any{"x": 0.0, "y": 500.0}, "width": 340.0, "height": 120.0, "metadata": map[string]any{"storageKey": "resource:voice-res"}}, + map[string]any{"id": "pending", "type": "image", "title": "未上传", "metadata": map[string]any{"content": "blob:local"}}, + }, "connections": []any{}} + raw, _ := json.Marshal(doc) + for _, item := range []any{ + &model.CanvasProject{ID: "agent-canvas", UserID: "user", Title: "赤练劫", PayloadJSON: string(raw)}, + &model.Resource{ID: "portrait-res", UserID: "user", Kind: "image", Status: model.ResourceStatusReady, MimeType: "image/png", Size: 10, Width: 720, Height: 960}, + &model.Resource{ID: "voice-res", UserID: "user", Kind: "audio", Status: model.ResourceStatusReady, MimeType: "audio/mpeg", Size: 10}, + } { + if err := db.Create(item).Error; err != nil { + t.Fatal(err) + } + } + policy, err := s.RuntimePolicy() + if err != nil { + t.Fatal(err) + } + args := map[string]any{"nodeId": "lmc-card", "name": "李莫愁", "imageNodeId": "portrait", "audioNodeId": "voice", "definition": map[string]any{"role": "赤练仙子", "appearance": "素纱襌衣", "aliases": []string{"赤练仙子"}}} + + plan, err := prepareCloudAgentCharacterCreate(s.repo, "user", "agent-canvas", cloudAgentStoryboardCall(t, "canvas_create_character", "preview", args)) + if err != nil { + t.Fatal(err) + } + if plan.Preview.Items[0].Operation != "create_character" || !strings.Contains(strings.Join(plan.Preview.Items[0].Details, "\n"), "李莫愁·音色") { + t.Fatalf("approval preview does not describe the card: %#v", plan.Preview) + } + // 审批预览不能提前写入角色库。 + var count int64 + db.Model(&model.Asset{}).Where("category = ?", model.AssetCategoryCharacter).Count(&count) + if count != 0 { + t.Fatal("preparing the approval created a character asset") + } + + result, err := applyCloudAgentCharacterCreate(s.repo, "user", "agent-canvas", cloudAgentStoryboardCall(t, "canvas_create_character", "apply", args), policy) + if err != nil { + t.Fatal(err) + } + summary := result.(map[string]any) + if summary["visualStatus"] != "ready" || summary["voiceStatus"] != "ready" { + t.Fatalf("card media not bound: %#v", summary) + } + assetID := stringValue(summary["characterAssetId"]) + detail, err := s.Character("user", assetID) + if err != nil || detail.Asset.Title != "李莫愁" || detail.Character.Voice == nil || detail.Character.Voice.Profile.Name != "李莫愁·音色" || detail.Character.Definition["role"] != "赤练仙子" { + t.Fatalf("character asset = %#v, %v", detail, err) + } + canvas, _ := s.repo.CanvasProjectForUser("user", "agent-canvas") + stored, _ := creationDocument(canvas.PayloadJSON) + nodes, _ := creationObjects(stored["nodes"]) + card := nodes["lmc-card"] + if card == nil { + t.Fatal("character card node was not placed on canvas") + } + descriptor, ok := cloudAgentNodeCapabilityForNode(card) + meta := card["metadata"].(map[string]any) + if !ok || descriptor.Type != "character" || meta["characterAssetId"] != assetID || meta["characterVersionPolicy"] != "current" { + t.Fatalf("placed node is not a usable character card: %#v", card) + } + // 未指定坐标时不能叠在形象图片上。 + if position := card["position"].(map[string]any); position["x"].(float64) < 300 { + t.Fatalf("card overlaps the portrait: %#v", position) + } + if _, err := cloudAgentResolveCharacter(s.repo, "user", "", card); err != nil { + t.Fatalf("placed card cannot be resolved for generation: %v", err) + } + + for name, bad := range map[string]map[string]any{ + "duplicate node": {"nodeId": "lmc-card", "name": "李莫愁", "imageNodeId": "portrait"}, + "missing image": {"nodeId": "x1", "name": "李莫愁"}, + "unsaved image": {"nodeId": "x2", "name": "李莫愁", "imageNodeId": "pending"}, + "audio as image": {"nodeId": "x3", "name": "李莫愁", "imageNodeId": "voice"}, + "unknown field": {"nodeId": "x4", "name": "李莫愁", "imageNodeId": "portrait", "definition": map[string]any{"secret": "x"}}, + "url instead node": {"nodeId": "x5", "name": "李莫愁", "imageNodeId": "https://example.com/a.png"}, + } { + if _, err := prepareCloudAgentCharacterCreate(s.repo, "user", "agent-canvas", cloudAgentStoryboardCall(t, "canvas_create_character", name, bad)); err == nil { + t.Fatalf("%s accepted", name) + } + } + if !cloudAgentWrite("canvas_create_character") { + t.Fatal("character creation must go through the write approval path") + } +} diff --git a/backend/internal/app/cloud_agent_character_test.go b/backend/internal/app/cloud_agent_character_test.go index b7ca98990..5d86abee7 100644 --- a/backend/internal/app/cloud_agent_character_test.go +++ b/backend/internal/app/cloud_agent_character_test.go @@ -196,3 +196,50 @@ func TestCloudAgentCharacterReferenceUsesDefinitionAndTurnaround(t *testing.T) { t.Fatalf("approved character version drift not rejected: %v", err) } } + +func TestCloudAgentCharacterCardIsRegisteredCapability(t *testing.T) { + node := map[string]any{"id": "hero", "type": "text", "title": "扶颦", "metadata": map[string]any{"workflowKind": "character", "characterAssetId": "character-asset"}} + descriptor, ok := cloudAgentNodeCapabilityForNode(node) + if !ok || descriptor.Type != "character" || descriptor.InputKind != "character" || !descriptor.Connection.CanReference { + t.Fatalf("character node did not resolve to character capability: %#v, %v", descriptor, ok) + } + if plain, _ := cloudAgentNodeCapabilityForNode(map[string]any{"id": "note", "type": "text"}); plain.Type != "text" { + t.Fatalf("plain text node resolved to %q", plain.Type) + } + if _, creatable := cloudAgentNodeCapabilityForType("character"); creatable { + t.Fatal("character variant must not be resolvable as a creatable node type") + } + if err := validateCreationOps([]CreationCanvasOp{{Type: "add_node", ID: "new", NodeType: "character"}}); err == nil { + t.Fatal("agent must not create character cards from nothing") + } + + nodes := []map[string]any{node, {"id": "video", "type": "video"}, {"id": "image", "type": "image"}, {"id": "audio", "type": "audio"}, {"id": "note", "type": "text"}} + for _, target := range []string{"video", "image", "audio"} { + if err := validateCloudAgentConnection(nodes, "hero", target); err != nil { + t.Fatalf("character card cannot feed %s: %v", target, err) + } + } + if err := validateCloudAgentConnection(nodes, "hero", "note"); err == nil { + t.Fatal("character card connected to a non-generation text node") + } + // 角色卡设定只随角色资产版本变化，Agent 不能把节点正文或标题当成角色设定改写。 + if err := descriptor.ValidatePatch(map[string]any{"content": "改写设定"}); err == nil { + t.Fatal("character card accepted content patch") + } + if err := descriptor.ValidatePatch(map[string]any{"x": 10.0, "y": 20.0}); err != nil { + t.Fatalf("character card cannot move: %v", err) + } + + guide := cloudAgentCapabilityGuide() + if !strings.Contains(guide, "角色卡（character）") || !strings.Contains(guide, "workflowKind=character") { + t.Fatalf("capability guide does not introduce character cards: %s", guide) + } + view, err := cloudAgentCanvasState(nil, "user", "agent-canvas", map[string]any{"nodes": []any{node}, "connections": []any{}}, 0, nil, 0) + if err != nil { + t.Fatal(err) + } + item := view.(map[string]any)["nodes"].([]any)[0].(map[string]any) + if item["kind"] != "character" || item["agentSupported"] == false { + t.Fatalf("canvas read does not expose character kind: %#v", item) + } +} diff --git a/backend/internal/app/cloud_agent_event_node_type_test.go b/backend/internal/app/cloud_agent_event_node_type_test.go new file mode 100644 index 000000000..43d8f6392 --- /dev/null +++ b/backend/internal/app/cloud_agent_event_node_type_test.go @@ -0,0 +1,13 @@ +package app + +import "testing" + +func TestCloudAgentEventNodeTypeResolvesCharacterVariant(t *testing.T) { + character := map[string]any{"type": "text", "metadata": map[string]any{"workflowKind": "character"}} + if got := cloudAgentEventNodeType(character); got != "character" { + t.Fatalf("character card node type = %v, want character", got) + } + if got := cloudAgentEventNodeType(map[string]any{"type": "text"}); got != "text" { + t.Fatalf("plain text node type = %v, want text", got) + } +} diff --git a/backend/internal/app/cloud_agent_layout.go b/backend/internal/app/cloud_agent_layout.go index df6acabb9..821f7318c 100644 --- a/backend/internal/app/cloud_agent_layout.go +++ b/backend/internal/app/cloud_agent_layout.go @@ -465,7 +465,7 @@ func cloudAgentLayoutTitles(doc map[string]any) map[string]string { } title := strings.TrimSpace(stringValue(node["title"])) if title == "" { - if capability, ok := cloudAgentNodeCapabilityForType(stringValue(node["type"])); ok { + if capability, ok := cloudAgentNodeCapabilityForNode(node); ok { title = capability.Label } } diff --git a/backend/internal/app/cloud_agent_media.go b/backend/internal/app/cloud_agent_media.go index 079674bbe..6c2f9cda8 100644 --- a/backend/internal/app/cloud_agent_media.go +++ b/backend/internal/app/cloud_agent_media.go @@ -194,7 +194,7 @@ func (s *Service) cloudAgentMediaModelName(a cloudAgentMediaArgs) (string, error } func cloudAgentReferenceDescriptor(node map[string]any) (cloudAgentNodeCapability, cloudAgentReferenceAdapter, error) { - descriptor, known := cloudAgentNodeCapabilityForType(stringValue(node["type"])) + descriptor, known := cloudAgentNodeCapabilityForNode(node) if !known || !descriptor.Connection.CanReference || descriptor.InputKind == "" { return cloudAgentNodeCapability{}, cloudAgentReferenceAdapter{}, BadAuthRequest("该节点不能作为媒体参考资产") } @@ -343,8 +343,9 @@ func cloudAgentMediaDocument(repo *repository.Repository, userID, canvasID strin return nil, nil, nil, BadAuthRequest("来源镜头节点不在当前画布") } if source := nodes[args.SourceNodeID]; source != nil { - descriptor, known := cloudAgentNodeCapabilityForType(stringValue(source["type"])) - if !known || !descriptor.Connection.CanSource || descriptor.InputKind != "text" { + descriptor, known := cloudAgentNodeCapabilityForNode(source) + // 角色卡作为文本来源时只取其角色设定（由 cloudAgentMediaCharacters 校验非空）。 + if !known || !descriptor.Connection.CanSource || (descriptor.InputKind != "text" && descriptor.InputKind != "character") { return nil, nil, nil, BadAuthRequest("sourceNodeId 仅接受可作为文本输入的节点；媒体资产请放入 referenceNodeIds，并将 sourceNodeId 留空，不要重复传入") } } @@ -519,8 +520,17 @@ func validateCloudAgentMediaReferences(mode string, refs map[string]any) error { return BadAuthRequest("图片生成仅支持图片参考资产") } case "audio": - if imageCount > 0 || videoCount > 0 || audioCount > 0 { - return BadAuthRequest("当前音频生成只支持文本输入，暂不支持媒体参考资产") + if videoCount > 0 { + return BadAuthRequest("音频生成不能使用参考视频") + } + if imageCount > 0 && audioCount > 0 { + return BadAuthRequest("参考图片和参考音频不能同时使用") + } + if imageCount > 1 { + return BadAuthRequest("音频生成最多使用 1 张参考图片") + } + if audioCount > 3 { + return BadAuthRequest("音频生成最多使用 3 段参考音频") } case "video": // Video reference admission is completed against the selected model's diff --git a/backend/internal/app/cloud_agent_media_test.go b/backend/internal/app/cloud_agent_media_test.go index 5a6ea5265..317dd3de4 100644 --- a/backend/internal/app/cloud_agent_media_test.go +++ b/backend/internal/app/cloud_agent_media_test.go @@ -483,7 +483,7 @@ func TestCloudAgentModelListFiltersActualReferences(t *testing.T) { t.Fatalf("intent = %+v", intent) } } - for _, args := range []string{`{"mode":"video","referenceNodeIds":["hero","hero"]}`, `{"mode":"video","referenceNodeIds":["missing"]}`, `{"referenceNodeIds":["hero"]}`, `{"mode":"audio","referenceNodeIds":["hero"]}`} { + for _, args := range []string{`{"mode":"video","referenceNodeIds":["hero","hero"]}`, `{"mode":"video","referenceNodeIds":["missing"]}`, `{"referenceNodeIds":["hero"]}`, `{"mode":"audio","referenceNodeIds":["hero","cat"]}`} { if _, err := s.cloudAgentModelIntent("user", "agent-canvas", args); err == nil { t.Fatalf("accepted %s", args) } diff --git a/backend/internal/app/cloud_agent_nodes.go b/backend/internal/app/cloud_agent_nodes.go index 325b8e283..ce610f9be 100644 --- a/backend/internal/app/cloud_agent_nodes.go +++ b/backend/internal/app/cloud_agent_nodes.go @@ -21,6 +21,14 @@ func cloudAgentNodeCapabilityForType(nodeType string) (cloudAgentNodeCapability, return canvasCapabilityRegistry.Resolve(nodeType) } +// cloudAgentNodeCapabilityForNode 按画布节点本身解析能力：角色卡等 workflowKind 变体 +// 优先于底层类型。凡是手里有真实节点（读取、连线、引用、更新）的调用方都应使用它； +// 只有 add_node 这类按类型名创建的入口才使用 cloudAgentNodeCapabilityForType。 +func cloudAgentNodeCapabilityForNode(node map[string]any) (cloudAgentNodeCapability, bool) { + meta, _ := node["metadata"].(map[string]any) + return canvasCapabilityRegistry.ResolveNode(stringValue(node["type"]), stringValue(meta["workflowKind"])) +} + func cloudAgentNodeTypeNames() []string { return canvasCapabilityRegistry.Types() } func cloudAgentGenerationModeNames() []string { diff --git a/backend/internal/app/cloud_agent_pi_coordinator.go b/backend/internal/app/cloud_agent_pi_coordinator.go index 8187ad5c6..045db6162 100644 --- a/backend/internal/app/cloud_agent_pi_coordinator.go +++ b/backend/internal/app/cloud_agent_pi_coordinator.go @@ -14,8 +14,7 @@ import ( "errors" "fmt" "log" - "os" - "path/filepath" + "strings" "sync/atomic" "time" @@ -292,17 +291,25 @@ func (s *Service) runCloudAgentPiSession(ctx context.Context, userID, runID stri // 4. 准备会话文件 sessionJSONL, _ := input["piSessionJSONL"].(string) + // ownSession 表示快照是本轮自己落库的，而不是续轮沿用的上一轮会话。 + ownSession := false if saved, sessionErr := s.repo.CloudAgentPiSession(userID, runID); sessionErr == nil && saved != nil && saved.SessionJSONL != "" { sessionJSONL = saved.SessionJSONL + ownSession = true } else if sessionErr != nil && !errors.Is(sessionErr, gorm.ErrRecordNotFound) { return fmt.Errorf("load session: %w", sessionErr) } - sessionDir := filepath.Join(s.dataDir, "pi-sessions") - if err := os.MkdirAll(sessionDir, 0o700); err != nil { - return fmt.Errorf("create session dir: %w", err) + // 崩溃恢复：最后答案已写进会话快照、运行却没来得及标记完成。这一回合已经结束， + // 直接收尾；再启动运行时会对同一提示词再调用一次模型、多扣一次费。 + if ownSession && cloudAgentPiTurnSettled(sessionJSONL, &runtimeState) { + if runtimeState.PiResumePrompt != "" { + if err := s.saveCloudAgentPiResumePrompt(userID, runID, ""); err != nil { + return err + } + } + return s.completeCloudAgentPiRun(userID, runID) } - sessionFile := filepath.Join(sessionDir, runID+".jsonl") // 5. 构建完整的 Pi 请求（核心重构） request, err := s.buildEnhancedPiRequest(ctx, EnhancedPiRequestParams{ @@ -315,9 +322,7 @@ func (s *Service) runCloudAgentPiSession(ctx context.Context, userID, runID stri ModelID: modelID, RuntimeState: &runtimeState, Canonical: &canonical, - SessionFile: sessionFile, SessionJSONL: sessionJSONL, - SessionDir: sessionDir, }) if err != nil { return fmt.Errorf("build request: %w", err) @@ -366,6 +371,49 @@ func (s *Service) runCloudAgentPiSession(ctx context.Context, userID, runID stri return s.completeCloudAgentPiRun(userID, runID) } +// cloudAgentPiTurnSettled 判断本轮是否已在会话里完整结束：本轮已有成功的助手回复， +// 没有挂着的任务或审批，且会话最后一条消息是不带工具调用的正常结束回复。 +// 只能用于本轮自己落库的快照：运行时的事件按顺序提交，助手回复计数增长之前， +// 含本轮用户提示的快照已经落库，所以最后一条完成回复一定属于本轮。 +// 续轮沿用的上一轮会话最后一条也是完成回复，不能据此收尾。 +func cloudAgentPiTurnSettled(sessionJSONL string, state *cloudAgentRuntime) bool { + if sessionJSONL == "" || state == nil || state.PiAssistantResponses == 0 { + return false + } + if state.ActiveTaskID != "" || state.MediaTaskID != "" || state.Approval != nil { + return false + } + lines := strings.Split(strings.TrimRight(sessionJSONL, "\n"), "\n") + for i := len(lines) - 1; i >= 0; i-- { + var entry struct { + Type string `json:"type"` + Message struct { + Role string `json:"role"` + StopReason string `json:"stopReason"` + Content []struct { + Type string `json:"type"` + } `json:"content"` + } `json:"message"` + } + if err := json.Unmarshal([]byte(lines[i]), &entry); err != nil { + return false + } + if entry.Type != "message" { + continue + } + if entry.Message.Role != "assistant" || entry.Message.StopReason != "stop" { + return false + } + for _, block := range entry.Message.Content { + if block.Type == "toolCall" { + return false + } + } + return true + } + return false +} + // completeCloudAgentPiRun 完成运行 func (s *Service) completeCloudAgentPiRun(userID, runID string) error { for attempt := 0; attempt < 4; attempt++ { diff --git a/backend/internal/app/cloud_agent_pi_helpers.go b/backend/internal/app/cloud_agent_pi_helpers.go index 9a9be1626..34b876796 100644 --- a/backend/internal/app/cloud_agent_pi_helpers.go +++ b/backend/internal/app/cloud_agent_pi_helpers.go @@ -11,19 +11,6 @@ import ( // Skills Configuration Builders -// buildSkillPaths 构建 Skills 路径列表 -func (s *Service) buildSkillPaths(skills []cloudAgentSkill) []string { - if len(skills) == 0 { - return []string{} - } - - paths := make([]string, 0, len(skills)) - for _, skill := range skills { - paths = append(paths, cloudAgentSkillPaths(skill)...) - } - return paths -} - // buildSkillManifests 构建 Skills 清单（传递给 Pi） func (s *Service) buildSkillManifests(skills []cloudAgentSkill) []map[string]any { if len(skills) == 0 { diff --git a/backend/internal/app/cloud_agent_pi_model.go b/backend/internal/app/cloud_agent_pi_model.go index acbcf2c39..f048e7f57 100644 --- a/backend/internal/app/cloud_agent_pi_model.go +++ b/backend/internal/app/cloud_agent_pi_model.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "gorm.io/gorm" "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" ) @@ -106,6 +107,21 @@ func (s *Service) runCloudAgentModelStep(ctx context.Context, userID, runID stri } continue } + // 重启恢复：上一个运行时进程已为这一步建好模型任务（可能已经出结果）， + // 只是没来得及把结果交回就退出了。运行时对 /model 串行调用，此时仍挂着的 + // 步骤任务只可能是这种遗留任务：接手等待它的结果，不再建新任务和订单。 + if false { + adopt, released, err := s.adoptCloudAgentPiModelStep(userID, runID, state.ActiveTaskID) + if err != nil { + return nil, false, err + } + if adopt { + break + } + if released { + continue + } + } canonical := canonicalFromRuntimeMessages(messages, state.Canonical.Tools, state.Canonical.SystemPrompt) // A continuation starts a fresh Pi process. If its session snapshot was @@ -198,6 +214,32 @@ func (s *Service) runCloudAgentModelStep(ctx context.Context, userID, runID stri return map[string]any{"text": result.Text, "reasoning": result.Reasoning, "toolCalls": runtimeToolCalls(result.ToolCalls)}, false, nil } +// adoptCloudAgentPiModelStep 判断挂着的活动任务能否作为本步结果直接接手。 +// 排队、运行中或已成功的模型步骤直接接手；失败或已取消的先释放，由调用方重新调度。 +// 其它活动任务（运行根任务、上下文压缩）保持原有流程。 +func (s *Service) adoptCloudAgentPiModelStep(userID, runID, taskID string) (adopt, released bool, err error) { + task, err := s.repo.TaskForUser(userID, taskID) + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, false, nil + } + if err != nil { + return false, false, err + } + if task.Operation != cloudAgentStepOperation { + return false, false, nil + } + switch task.Status { + case model.TaskStatusQueued, model.TaskStatusRunning, model.TaskStatusSucceeded: + return true, false, nil + case model.TaskStatusFailed, model.TaskStatusCancelled: + if err := s.finishCloudAgentPiModelStep(userID, runID, taskID, "", ""); err != nil { + return false, false, err + } + return false, true, nil + } + return false, false, nil +} + // cloudAgentPiThinkingEnabled treats an omitted thinking level the same as Pi's // explicit "off" value. Pi omits the field when thinking is disabled, and the // Go JSON decoder represents that omitted field as an empty string. diff --git a/backend/internal/app/cloud_agent_pi_recovery_test.go b/backend/internal/app/cloud_agent_pi_recovery_test.go new file mode 100644 index 000000000..8095d0bca --- /dev/null +++ b/backend/internal/app/cloud_agent_pi_recovery_test.go @@ -0,0 +1,107 @@ +package app + +import ( + "context" + "strings" + "testing" + "time" + + "infinite-canvas/backend/internal/model" +) + +// 重启恢复：上一个运行时进程已为这一步建好模型任务、任务也已成功，但结果没交回。 +// 新进程重新发起 /model 时必须直接接手这份结果，不能再建任务、再扣一次费。 +func TestCloudAgentModelStepAdoptsOrphanedStepTask(t *testing.T) { + s, db, a := agentMediaFixture(t) + s.disablePiRuntime = true + run, _ := agentMediaRun(t, s, a, "auto") + + // 第一个进程：调度这一步，但还没等到结果就"崩溃"。 + ctx, cancel := context.WithCancel(context.Background()) + messages := []map[string]any{{"role": "user", "content": "继续"}} + done := make(chan struct{}) + go func() { + _, _, _ = s.runCloudAgentModelStep(ctx, "user", run.ID, messages, "off") + close(done) + }() + var orphanID string + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && orphanID == "" { + latest, err := s.repo.CloudAgent("user", run.ID) + if err != nil { + t.Fatal(err) + } + state, err := cloudAgentDecode(latest) + if err != nil { + t.Fatal(err) + } + orphanID = state.ActiveTaskID + time.Sleep(20 * time.Millisecond) + } + cancel() + <-done + if orphanID == "" { + t.Fatal("first process never scheduled a model step") + } + if err := db.Model(&model.Task{}).Where("id = ?", orphanID).Updates(map[string]any{"status": model.TaskStatusSucceeded, "result_json": `{"text":"已完成"}`}).Error; err != nil { + t.Fatal(err) + } + var before int64 + if err := db.Model(&model.Task{}).Where("agent_run_id = ? AND operation = ?", run.ID, cloudAgentStepOperation).Count(&before).Error; err != nil { + t.Fatal(err) + } + + // 第二个进程：重新发起同一步。 + result, _, err := s.runCloudAgentModelStep(context.Background(), "user", run.ID, messages, "off") + if err != nil { + t.Fatal(err) + } + if text, _ := result.(map[string]any)["text"].(string); text != "已完成" { + t.Fatalf("adopted result = %#v", result) + } + var after int64 + if err := db.Model(&model.Task{}).Where("agent_run_id = ? AND operation = ?", run.ID, cloudAgentStepOperation).Count(&after).Error; err != nil { + t.Fatal(err) + } + if after != before { + t.Fatalf("recovery created %d new step task(s); want the orphaned task to be adopted", after-before) + } + latest, err := s.repo.CloudAgent("user", run.ID) + if err != nil { + t.Fatal(err) + } + state, err := cloudAgentDecode(latest) + if err != nil { + t.Fatal(err) + } + if state.ActiveTaskID != "" { + t.Fatalf("adopted step was not released: active=%s", state.ActiveTaskID) + } +} + +func TestCloudAgentPiTurnSettled(t *testing.T) { + header := `{"type":"session","id":"s"}` + user := `{"type":"message","message":{"role":"user","content":[{"type":"text","text":"hi"}]}}` + done := `{"type":"message","message":{"role":"assistant","stopReason":"stop","content":[{"type":"text","text":"ok"}]}}` + tool := `{"type":"message","message":{"role":"assistant","stopReason":"toolUse","content":[{"type":"toolCall","id":"c"}]}}` + meta := `{"type":"thinking_level_change","thinkingLevel":"off"}` + settled := &cloudAgentRuntime{PiAssistantResponses: 1} + cases := []struct { + name string + jsonl string + state *cloudAgentRuntime + want bool + }{ + {"final answer", strings.Join([]string{header, user, done, meta}, "\n") + "\n", settled, true}, + {"pending user prompt", strings.Join([]string{header, done, user}, "\n"), settled, false}, + {"tool call pending", strings.Join([]string{header, user, tool}, "\n"), settled, false}, + {"no response this run", strings.Join([]string{header, user, done}, "\n"), &cloudAgentRuntime{}, false}, + {"active task", strings.Join([]string{header, user, done}, "\n"), &cloudAgentRuntime{PiAssistantResponses: 1, ActiveTaskID: "t"}, false}, + {"empty", "", settled, false}, + } + for _, tc := range cases { + if got := cloudAgentPiTurnSettled(tc.jsonl, tc.state); got != tc.want { + t.Errorf("%s: settled = %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/backend/internal/app/cloud_agent_pi_request.go b/backend/internal/app/cloud_agent_pi_request.go index d4cea9341..24c0ac932 100644 --- a/backend/internal/app/cloud_agent_pi_request.go +++ b/backend/internal/app/cloud_agent_pi_request.go @@ -24,9 +24,7 @@ type EnhancedPiRequestParams struct { ModelID string RuntimeState *cloudAgentRuntime Canonical *canonicalAgentRequest - SessionFile string SessionJSONL string - SessionDir string } // buildEnhancedPiRequest 构建增强的 Pi 请求（核心方法） @@ -36,8 +34,8 @@ func (s *Service) buildEnhancedPiRequest(ctx context.Context, params EnhancedPiR // 1. 构建工具列表（包括画布工具） tools := s.buildCompletePiTools(params.Canonical.Tools) - // 2. 构建 Skills 配置 - skillPaths := s.buildSkillPaths(params.RuntimeState.Skills) + // 2. 构建 Skills 配置。技能内容只经服务端读取工具提供，不交给运行时按路径加载： + // 技能路径是相对路径，运行时按自己的工作目录解析，会读到无关文件。 enabledSkills := s.buildSkillManifests(params.RuntimeState.Skills) slog.Debug("agent enabled skills", "count", len(enabledSkills)) @@ -80,12 +78,9 @@ func (s *Service) buildEnhancedPiRequest(ctx context.Context, params EnhancedPiR // 10. 组装完整请求 request := cloudAgentPiProcessRequest{ - // 会话信息 - SessionFile: params.SessionFile, + // 会话信息：只传数据库里的快照。会话文件与工作目录由运行时在自己的 + // 临时目录里创建，独立容器里没有服务端的数据目录。 SessionJSONL: params.SessionJSONL, - SessionDir: params.SessionDir, - Cwd: s.dataDir, - AgentDir: params.SessionDir, // 标识信息 SessionId: params.RunID, @@ -98,7 +93,6 @@ func (s *Service) buildEnhancedPiRequest(ctx context.Context, params EnhancedPiR SystemPrompt: params.SystemPrompt, // 核心增强：完整的上下文传递 - SkillPaths: skillPaths, EnabledSkills: enabledSkills, Profile: profile, Memory: memory, diff --git a/backend/internal/app/cloud_agent_pi_runtime.go b/backend/internal/app/cloud_agent_pi_runtime.go index 850d1fcd8..8a2c34aeb 100644 --- a/backend/internal/app/cloud_agent_pi_runtime.go +++ b/backend/internal/app/cloud_agent_pi_runtime.go @@ -16,18 +16,13 @@ type cloudAgentPiBridge = agentruntime.Bridge type cloudAgentPiProcessRequest struct { BridgeURL string `json:"bridgeURL"` BridgeToken string `json:"bridgeToken"` - SessionFile string `json:"sessionFile,omitempty"` SessionJSONL string `json:"sessionJSONL,omitempty"` - SessionDir string `json:"sessionDir,omitempty"` SessionId string `json:"sessionId,omitempty"` UserId string `json:"userId,omitempty"` CanvasId string `json:"canvasId,omitempty"` RunId string `json:"runId,omitempty"` - Cwd string `json:"cwd"` - AgentDir string `json:"agentDir"` Prompt string `json:"prompt"` SystemPrompt string `json:"systemPrompt"` - SkillPaths []string `json:"skillPaths,omitempty"` EnabledSkills []map[string]any `json:"enabledSkills,omitempty"` Profile map[string]any `json:"profile,omitempty"` Memory map[string]any `json:"memory,omitempty"` @@ -42,10 +37,10 @@ type cloudAgentPiProcessRequest struct { func runCloudAgentPi(ctx context.Context, request cloudAgentPiProcessRequest, bridge cloudAgentPiBridge) error { runtimeRequest := agentruntime.ProcessRequest{ BridgeURL: request.BridgeURL, BridgeToken: request.BridgeToken, - SessionFile: request.SessionFile, SessionJSONL: request.SessionJSONL, SessionDir: request.SessionDir, - SessionID: request.SessionId, UserID: request.UserId, CanvasID: request.CanvasId, RunID: request.RunId, - CWD: request.Cwd, AgentDir: request.AgentDir, Prompt: request.Prompt, SystemPrompt: request.SystemPrompt, - SkillPaths: request.SkillPaths, EnabledSkills: request.EnabledSkills, Profile: request.Profile, + SessionJSONL: request.SessionJSONL, + SessionID: request.SessionId, UserID: request.UserId, CanvasID: request.CanvasId, RunID: request.RunId, + Prompt: request.Prompt, SystemPrompt: request.SystemPrompt, + EnabledSkills: request.EnabledSkills, Profile: request.Profile, Memory: request.Memory, Canvas: request.Canvas, Features: request.Features, Tools: request.Tools, Compaction: request.Compaction, Permissions: request.Permissions, Model: request.Model, } diff --git a/backend/internal/app/cloud_agent_policy.go b/backend/internal/app/cloud_agent_policy.go index efb52e75c..af36889c6 100644 --- a/backend/internal/app/cloud_agent_policy.go +++ b/backend/internal/app/cloud_agent_policy.go @@ -101,6 +101,9 @@ func cloudAgentCapabilityGuide() string { b.WriteString(descriptor.Type) b.WriteString("）：") b.WriteString(descriptor.Purpose) + if variant := descriptor.Variant; variant != nil { + fmt.Fprintf(&b, " 画布识别：type=%s 且 workflowKind=%s，读取结果 kind=%s；不能用 add_node 新建。", variant.BaseType, variant.WorkflowKind, descriptor.Type) + } if len(descriptor.GoodFor) > 0 { b.WriteString(" 适合：") b.WriteString(strings.Join(descriptor.GoodFor, "、")) diff --git a/backend/internal/app/cloud_agent_runtime_tools.go b/backend/internal/app/cloud_agent_runtime_tools.go index 438c424c5..0a70c5bb0 100644 --- a/backend/internal/app/cloud_agent_runtime_tools.go +++ b/backend/internal/app/cloud_agent_runtime_tools.go @@ -456,6 +456,12 @@ func (s *Service) advanceCloudAgentTool(run *model.CloudAgentExecution, state *c if err == nil { preview = storyboardPlan.Preview } + case "canvas_create_character": + characterPlan, err := prepareCloudAgentCharacterCreate(repo, run.UserID, state.Request.CanvasID, call) + mutationErr = err + if err == nil { + preview = characterPlan.Preview + } case "canvas_edit_batch_table": batchPlan, err := prepareCloudAgentBatchTableEdit(repo, run.UserID, state.Request.CanvasID, call) mutationErr = err @@ -561,6 +567,8 @@ func (s *Service) advanceCloudAgentTool(run *model.CloudAgentExecution, state *c result, toolErr = applyCloudAgentArrangeNodes(repo, run.UserID, state.Request.CanvasID, call, policy, cloudAgentCanvasEventRecorder(run.ID, state)) case call.Function.Name == "canvas_create_storyboard", call.Function.Name == "canvas_edit_storyboard": result, toolErr = applyCloudAgentStoryboardMutation(repo, run.UserID, state.Request.CanvasID, call, policy, cloudAgentCanvasEventRecorder(run.ID, state)) + case call.Function.Name == "canvas_create_character": + result, toolErr = applyCloudAgentCharacterCreate(repo, run.UserID, state.Request.CanvasID, call, policy, cloudAgentCanvasEventRecorder(run.ID, state)) case call.Function.Name == "canvas_edit_batch_table": result, toolErr = applyCloudAgentBatchTableMutation(repo, run.UserID, state.Request.CanvasID, call, policy, cloudAgentCanvasEventRecorder(run.ID, state)) case call.Function.Name == "canvas_inspect_image": diff --git a/backend/internal/app/cloud_agent_test.go b/backend/internal/app/cloud_agent_test.go index 44680926f..c8f7acba2 100644 --- a/backend/internal/app/cloud_agent_test.go +++ b/backend/internal/app/cloud_agent_test.go @@ -620,13 +620,26 @@ func TestCloudAgentToolLoopPersistsApprovalAndAppliesCanvasWrite(t *testing.T) { func TestCloudAgentNodeTypesExposeExecutableAllowList(t *testing.T) { result := cloudAgentNodeTypes() nodes, ok := result["nodes"].([]map[string]any) - if !ok || len(nodes) != 8 { + if !ok || len(nodes) != 9 { t.Fatalf("unexpected node registry: %#v", result) } + var character map[string]any for _, node := range nodes { if node["type"] == "panorama" { t.Fatal("UI-only node must not be exposed") } + if node["type"] == "character" { + character = node + } + } + // 角色卡必须被 Agent 发现，但只是 text 节点的变体：不能出现在 add_node 的 nodeType 枚举里。 + if character == nil || character["creatable"] != false || character["canvasNodeType"] != "text" || character["workflowKind"] != "character" || character["canReference"] != true { + t.Fatalf("character capability is not discoverable as a non-creatable variant: %#v", character) + } + for _, nodeType := range cloudAgentNodeTypeNames() { + if nodeType == "character" { + t.Fatal("character variant must not be creatable through add_node") + } } } diff --git a/backend/internal/app/cloud_agent_tool_schema_budget_test.go b/backend/internal/app/cloud_agent_tool_schema_budget_test.go index 7c4a54e0e..f1fc886e4 100644 --- a/backend/internal/app/cloud_agent_tool_schema_budget_test.go +++ b/backend/internal/app/cloud_agent_tool_schema_budget_test.go @@ -52,11 +52,12 @@ func TestCloudAgentToolSchemaStaysCompact(t *testing.T) { t.Fatal("canvas_apply_ops 未暴露") } - // 体积预算：导演台、技能检索和上下文读取工具合入后，当前平台工具为 24 个，实测 - // 27,812 字节，因此把预算显式上调到 29,000（约 4.3% 余量）。新增工具或字段时请 - // 重新测量并有意识地调整这个数字，而不是让 schema 悄悄膨胀（它每一步都要发、还在 - // 前缀最前面）。 - if len(raw) > 29000 { - t.Fatalf("平台工具 schema 体积 %d 字节超出预算 29000：请压缩描述或显式调整预算", len(raw)) + // 体积预算：导演台、技能检索和上下文读取工具合入后为 24 个工具、27,812 字节，预算 29,000。 + // 之后能力注册表加入角色卡（图片/视频/音频接受 character 输入）到 28,979 字节；再新增 + // canvas_create_character（角色卡打包创建，设定字段已压成一行描述）后为 25 个工具、实测 + // 29,938 字节，因此显式上调到 30,500（约 1.9% 余量）。新增工具或字段时请重新测量并 + // 有意识地调整这个数字，而不是让 schema 悄悄膨胀（它每一步都要发、还在前缀最前面）。 + if len(raw) > 30500 { + t.Fatalf("平台工具 schema 体积 %d 字节超出预算 30500：请压缩描述或显式调整预算", len(raw)) } } diff --git a/backend/internal/app/cloud_agent_tools.go b/backend/internal/app/cloud_agent_tools.go index f678e4d0a..74af44de8 100644 --- a/backend/internal/app/cloud_agent_tools.go +++ b/backend/internal/app/cloud_agent_tools.go @@ -220,6 +220,7 @@ func compileCloudAgentTools(req CloudAgentRequest, includeProfileTool bool) []ma "rowId": str("update/remove 使用 canvas_read_storyboard 返回的真实 rowId；append 留空"), "patch": cloudAgentStoryboardPatchSchema(), }, "snapshotHash", "nodeId", "action") + add("canvas_create_character", "把画布上就绪的形象图片（可加声音音频）打包成角色卡：写入角色库并在画布放置角色卡节点，按权限审批。已有同名角色卡先复用；definition 只填有依据的设定。", cloudAgentCharacterCreateSchema(), "nodeId", "name", "imageNodeId") add("canvas_edit_batch_table", "操作批量创作表组件：追加、修改或删除任务行，切换批量换装/创意生图，设置1/5/10并发，新增或减少参考图列，或设置覆盖各任务的全局提示词。必须先用 canvas_read_batch_table 获取最新 snapshotHash 和真实 rowId。行 patch 仅允许 enabled、inputNodeIds、prompt；prompt 可使用读取结果中的 @参考图1、@参考图2 等 mentionToken 指代本行对应位置的图片。append 未传 inputNodeIds 时会继承上一行参考图；图片ID必须来自当前画布。不能写 outputNodeId、任务状态、URL、storageKey 或任意 metadata。本工具只编辑计划，不提交收费生成。", map[string]any{ "snapshotHash": str("最近一次 canvas_read_batch_table 返回的 snapshotHash（这个表节点的版本；其它节点的改动不影响它）"), "nodeId": str("真实批量创作表节点ID"), diff --git a/backend/internal/app/cloud_agent_tools_canvas.go b/backend/internal/app/cloud_agent_tools_canvas.go index 1caa7ad62..8608427f7 100644 --- a/backend/internal/app/cloud_agent_tools_canvas.go +++ b/backend/internal/app/cloud_agent_tools_canvas.go @@ -106,8 +106,8 @@ func validateCloudAgentConnection(nodes []map[string]any, fromID, toID string, e if from == nil || to == nil { return BadAuthRequest("连线端点不存在") } - fromCapability, fromKnown := cloudAgentNodeCapabilityForType(stringValue(from["type"])) - toCapability, toKnown := cloudAgentNodeCapabilityForType(stringValue(to["type"])) + fromCapability, fromKnown := cloudAgentNodeCapabilityForNode(from) + toCapability, toKnown := cloudAgentNodeCapabilityForNode(to) if !fromKnown || !toKnown { return BadAuthRequest("连线包含当前 Agent 不支持的节点类型") } @@ -185,6 +185,10 @@ func cloudAgentNodeTypes() map[string]any { "defaultSize": map[string]any{"width": capability.DefaultWidth, "height": capability.DefaultHeight}, "canUpdate": capability.CanUpdate, } + if variant := capability.Variant; variant != nil { + // 变体不能 add_node；画布里按 type+metadata.workflowKind 识别，读取结果以 kind 标出。 + item["canvasNodeType"], item["workflowKind"], item["creatable"] = variant.BaseType, variant.WorkflowKind, false + } if len(capability.GoodFor) > 0 { item["goodFor"] = capability.GoodFor } diff --git a/backend/internal/app/cloud_agent_tools_read.go b/backend/internal/app/cloud_agent_tools_read.go index 8411d0b4d..0496129a8 100644 --- a/backend/internal/app/cloud_agent_tools_read.go +++ b/backend/internal/app/cloud_agent_tools_read.go @@ -22,7 +22,7 @@ import ( ) func cloudAgentWrite(name string) bool { - return name == "canvas_apply_ops" || name == "canvas_arrange_nodes" || name == "generate_media" || name == "image_layer_split" || name == "canvas_create_storyboard" || name == "canvas_edit_storyboard" || name == "canvas_edit_batch_table" + return name == "canvas_apply_ops" || name == "canvas_arrange_nodes" || name == "generate_media" || name == "image_layer_split" || name == "canvas_create_storyboard" || name == "canvas_edit_storyboard" || name == "canvas_edit_batch_table" || name == "canvas_create_character" } // 同参缓存只能拦住“原样重复”的读取。模型也可能不断修改 offset、nodeIds 或 diff --git a/backend/internal/app/cloud_agent_vision.go b/backend/internal/app/cloud_agent_vision.go index a6a879f2c..b27491f93 100644 --- a/backend/internal/app/cloud_agent_vision.go +++ b/backend/internal/app/cloud_agent_vision.go @@ -144,7 +144,8 @@ func (s *Service) prepareCloudAgentImageInspection(userID, canvasID string, stat if node == nil { return nil, BadAuthRequest("指定节点不在当前画布") } - reference, _, err := cloudAgentReference(s.repo, userID, node) + // 角色卡查看的是其绑定的三视图/形象图；普通节点查看自身图片资源。 + reference, _, err := cloudAgentMediaReference(s.repo, userID, canvas.ProjectID, node) if err != nil { return nil, err } diff --git a/backend/internal/app/finance.go b/backend/internal/app/finance.go index 066d9634f..87a657261 100644 --- a/backend/internal/app/finance.go +++ b/backend/internal/app/finance.go @@ -457,7 +457,7 @@ func (s *Service) taskBillingOrder(userID string, task *model.Task, input map[st scene := firstNonEmpty(strings.TrimSpace(task.Operation), task.Type) intent := ModelRequestIntentFromTaskInput(input, task.Type, task.Operation) priceTierID, _ := config["priceTierId"].(string) - return s.newBillingOrderWithPriceTier(userID, task.ID, "task:"+task.ID+":"+newID(), channelID, modelKey, capability, scene, billingQuantity(capability, config["videoSeconds"]), estimateTaskBillingTokens(input, capability), strings.TrimSpace(priceTierID), intent) + return s.newBillingOrderWithPriceTier(userID, task.ID, "task:"+task.ID+":"+newID(), channelID, modelKey, capability, scene, requestedBillingQuantity(capability, config), estimateTaskBillingTokens(input, capability), strings.TrimSpace(priceTierID), intent) } func (s *Service) newLogicalModelBillingOrder(userID string, task *model.Task, input map[string]any) (*model.BillingOrder, error) { @@ -481,7 +481,7 @@ func (s *Service) newLogicalModelBillingOrder(userID string, task *model.Task, i if logicalModel.PricePolicy == "channel" { intent := ModelRequestIntentFromTaskInput(input, task.Type, task.Operation) priceTierID, _ := config["priceTierId"].(string) - order, priceErr := s.newBillingOrderWithPriceTier(userID, task.ID, "task:"+task.ID+":"+newID(), channelModel.ChannelID, channelModel.ModelKey, capability, firstNonEmpty(strings.TrimSpace(task.Operation), task.Type), billingQuantity(capability, config["videoSeconds"]), estimateTaskBillingTokens(input, capability), strings.TrimSpace(priceTierID), intent) + order, priceErr := s.newBillingOrderWithPriceTier(userID, task.ID, "task:"+task.ID+":"+newID(), channelModel.ChannelID, channelModel.ModelKey, capability, firstNonEmpty(strings.TrimSpace(task.Operation), task.Type), requestedBillingQuantity(capability, config), estimateTaskBillingTokens(input, capability), strings.TrimSpace(priceTierID), intent) if priceErr != nil { return nil, priceErr } @@ -499,8 +499,8 @@ func (s *Service) newLogicalModelBillingOrder(userID string, task *model.Task, i case "fixed_request": amount = logicalModel.UnitPriceMicrocredits case "per_second": - quantity = billingQuantity(capability, config["videoSeconds"]) - if capability != "video" || quantity <= 0 { + quantity = requestedBillingQuantity(capability, config) + if (capability != "video" && capability != "audio") || quantity <= 0 { return nil, BadAuthRequest("当前模型按时长计费，但请求未提供有效时长") } amount, err = creditAmount(logicalModel.UnitPriceMicrocredits, quantity, 10_000) @@ -543,7 +543,7 @@ func (s *Service) newLogicalModelBillingOrder(userID string, task *model.Task, i } intent := ModelRequestIntentFromTaskInput(input, task.Type, task.Operation) priceTierID, _ := config["priceTierId"].(string) - snapshotCreditCost(order, channelModelPriceTierForBilling(*channelModel, priceTierID, capability, intent), billingQuantity(capability, config["videoSeconds"]), tokenEstimate) + snapshotCreditCost(order, channelModelPriceTierForBilling(*channelModel, priceTierID, capability, intent), requestedBillingQuantity(capability, config), tokenEstimate) return order, nil } @@ -596,11 +596,11 @@ func (s *Service) newBillingOrderWithPriceTier(userID string, taskID string, ide switch tier.BillingMode { case "fixed_request": case "per_second": - if item.Capability != "video" || capability != "video" { - return nil, BadAuthRequest("按秒计费仅适用于视频生成") + if (item.Capability != "video" && item.Capability != "audio") || capability != item.Capability { + return nil, BadAuthRequest("按秒计费仅适用于视频或音频生成") } if requestedQuantity <= 0 { - return nil, BadAuthRequest("视频生成时长无效，无法按秒计费") + return nil, BadAuthRequest("生成时长无效，无法按秒计费") } quantity = requestedQuantity case "token": @@ -741,7 +741,7 @@ func tokenEstimateAmount(item *model.ChannelModel, estimate tokenBillingEstimate } func billingQuantity(capability string, value any) int64 { - if capability != "video" { + if capability != "video" && capability != "audio" { return 1 } quantity, err := strconv.ParseInt(strings.TrimSpace(fmt.Sprint(value)), 10, 64) @@ -751,6 +751,28 @@ func billingQuantity(capability string, value any) int64 { return quantity } +// requestedBillingQuantity is the quantity reserved before a provider has +// returned the generated media. Audio output length is not known at request +// time, so an audio per-second order uses one second as the minimum +// preauthorization unless the caller explicitly supplied audioSeconds. +// Settlement must replace this estimate with the measured output duration. +func requestedBillingQuantity(capability string, config map[string]any) int64 { + if capability == "audio" { + if config == nil { + return 1 + } + value, present := config["audioSeconds"] + if !present || strings.TrimSpace(fmt.Sprint(value)) == "" { + return 1 + } + return billingQuantity(capability, value) + } + if config == nil { + return billingQuantity(capability, nil) + } + return billingQuantity(capability, config["videoSeconds"]) +} + func (s *Service) MarkBillingRunning(orderID string) error { return s.taskBilling().MarkBillingRunning(orderID) } diff --git a/backend/internal/app/logical_model_quote.go b/backend/internal/app/logical_model_quote.go index 1463f8462..80bfc40c6 100644 --- a/backend/internal/app/logical_model_quote.go +++ b/backend/internal/app/logical_model_quote.go @@ -42,7 +42,7 @@ func (s *Service) QuoteChannelModel(req ChannelModelQuoteRequest) (*LogicalModel config = input["config"].(map[string]any) estimate := estimateTaskBillingTokens(input, req.Intent.Capability) intent := ModelRequestIntentFromTaskInput(input, "canvas_"+req.Intent.Capability, req.Intent.Operation) - order, err := s.newBillingOrderWithPriceTier("", "", "quote", req.ChannelID, req.ModelKey, req.Intent.Capability, "model_quote", billingQuantity(req.Intent.Capability, config["videoSeconds"]), estimate, stringValue(config["priceTierId"]), intent) + order, err := s.newBillingOrderWithPriceTier("", "", "quote", req.ChannelID, req.ModelKey, req.Intent.Capability, "model_quote", requestedBillingQuantity(req.Intent.Capability, config), estimate, stringValue(config["priceTierId"]), intent) if err != nil { return nil, err } @@ -89,8 +89,9 @@ func (s *Service) QuoteLogicalModel(logicalModelID string, intent ModelRequestIn providerModelKey = firstNonEmpty(routed.PriceTier.ProviderModelKey, providerModelKey) } input["config"].(map[string]any)["providerModelKey"] = providerModelKey - quantity := billingQuantity(capability, inputConfigValue(input, "videoSeconds")) - if capability != "video" { + config, _ := input["config"].(map[string]any) + quantity := requestedBillingQuantity(capability, config) + if capability != "video" && capability != "audio" { quantity = 1 } tokenEstimate := estimateTaskBillingTokens(input, capability) @@ -116,7 +117,7 @@ func (s *Service) QuoteLogicalModel(logicalModelID string, intent ModelRequestIn quantity = 1 amount = routed.LogicalModel.UnitPriceMicrocredits case "per_second": - if capability != "video" || quantity <= 0 { + if (capability != "video" && capability != "audio") || quantity <= 0 { return nil, BadAuthRequest("当前模型按时长计费，但请求未提供有效时长") } amount, err = creditAmount(routed.LogicalModel.UnitPriceMicrocredits, quantity, 10_000) diff --git a/backend/internal/app/logical_models.go b/backend/internal/app/logical_models.go index fbce5c57f..2b41d7603 100644 --- a/backend/internal/app/logical_models.go +++ b/backend/internal/app/logical_models.go @@ -429,8 +429,8 @@ func (s *Service) logicalModelBundle(actor *model.User, id string, req LogicalMo } else if billingMode != "fixed_request" && billingMode != "per_second" && billingMode != "token" { return nil, nil, nil, false, BadAuthRequest("前台模型计费方式仅支持按次、按秒或 Token") } - if pricePolicy == "unified" && billingMode == "per_second" && capability != "video" { - return nil, nil, nil, false, BadAuthRequest("只有视频前台模型可以按秒计费") + if pricePolicy == "unified" && billingMode == "per_second" && capability != "video" && capability != "audio" { + return nil, nil, nil, false, BadAuthRequest("只有视频或音频前台模型可以按秒计费") } if pricePolicy == "unified" && billingMode == "token" { if err := validateTokenPrices(capability, req.InputPriceMicrocredits, req.OutputPriceMicrocredits, req.CachedPriceMicrocredits); err != nil { diff --git a/backend/internal/app/model_router_attempts.go b/backend/internal/app/model_router_attempts.go index a4f4272fc..c7c3cd11f 100644 --- a/backend/internal/app/model_router_attempts.go +++ b/backend/internal/app/model_router_attempts.go @@ -274,7 +274,7 @@ func (s *Service) switchTaskToNextRoute(task *model.Task, attempts []model.Route capability = capabilityFromTaskType(task.Type) } priceTierID, _ := config["priceTierId"].(string) - replacement, err = s.newBillingOrderWithPriceTier(task.UserID, task.ID, "route-switch:"+task.ID+":"+selected.Route.ID, selected.ChannelModel.ChannelID, selected.ChannelModel.ModelKey, capability, firstNonEmpty(strings.TrimSpace(task.Operation), task.Type), billingQuantity(capability, config["videoSeconds"]), estimateTaskBillingTokens(nextInput, capability), strings.TrimSpace(priceTierID), intent) + replacement, err = s.newBillingOrderWithPriceTier(task.UserID, task.ID, "route-switch:"+task.ID+":"+selected.Route.ID, selected.ChannelModel.ChannelID, selected.ChannelModel.ModelKey, capability, firstNonEmpty(strings.TrimSpace(task.Operation), task.Type), requestedBillingQuantity(capability, config), estimateTaskBillingTokens(nextInput, capability), strings.TrimSpace(priceTierID), intent) if err != nil { return nil, err } @@ -287,7 +287,7 @@ func (s *Service) switchTaskToNextRoute(task *model.Task, attempts []model.Route } else if task.BillingOrderID != "" { config, _ := nextInput["config"].(map[string]any) capability := selected.ChannelModel.Capability - snapshotCreditCost(&costOrder, channelModelPriceTierForIntent(selected.ChannelModel, intent), billingQuantity(capability, config["videoSeconds"]), estimateTaskBillingTokens(nextInput, capability)) + snapshotCreditCost(&costOrder, channelModelPriceTierForIntent(selected.ChannelModel, intent), requestedBillingQuantity(capability, config), estimateTaskBillingTokens(nextInput, capability)) } if err := s.repo.SwitchTaskLogicalRoute(task.ID, previousRouteID, selected.Route.ID, string(encoded), task.BillingOrderID, selected.ChannelModel.ChannelID, selected.ChannelModel.ID, replacement, costOrder.BillingCostSnapshot); err != nil { if errors.Is(err, repository.ErrInsufficientCredits) { diff --git a/backend/internal/app/payment_products.go b/backend/internal/app/payment_products.go index 6c08ec2f1..868c2de7d 100644 --- a/backend/internal/app/payment_products.go +++ b/backend/internal/app/payment_products.go @@ -83,21 +83,24 @@ func (s *Service) UpdateTopupProduct(actor *model.User, id string, request Topup if err := s.RequireAdmin(actor); err != nil { return nil, err } - existing, err := s.repo.TopupProduct(id) + id = strings.TrimSpace(id) + var product *model.TopupProduct + // 已售数量必须在锁住商品行之后计算，否则会覆盖并发下单扣减或关单退回的库存。 + err := s.repo.UpdateTopupProduct(id, func(existing *model.TopupProduct) (*model.TopupProduct, error) { + consumed := existing.StockTotal - existing.StockRemaining + if existing.SaleStrategy != model.TopupSaleStrategyInventory { + consumed = 0 + } + built, buildErr := topupProductFromRequest(id, actor.ID, request, existing.StockTotal, consumed) + if buildErr != nil { + return nil, buildErr + } + product = built + return built, nil + }) if err != nil { return nil, err } - consumed := existing.StockTotal - existing.StockRemaining - if existing.SaleStrategy != model.TopupSaleStrategyInventory { - consumed = 0 - } - product, err := topupProductFromRequest(strings.TrimSpace(id), actor.ID, request, existing.StockTotal, consumed) - if err != nil { - return nil, err - } - if err := s.repo.UpdateTopupProduct(product); err != nil { - return nil, err - } if err := s.appendAdminAudit(actor, "topup_product.update", "topup_product", product.ID, "更新积分充值商品", map[string]any{"enabled": product.Enabled, "saleStrategy": product.SaleStrategy}); err != nil { return nil, err } diff --git a/backend/internal/app/project_character.go b/backend/internal/app/project_character.go index d44519fb8..4a0ab2bb6 100644 --- a/backend/internal/app/project_character.go +++ b/backend/internal/app/project_character.go @@ -18,6 +18,20 @@ type CreateProjectCharacterRequest struct { Definition map[string]any `json:"definition"` } +type CreateCharacterRequest struct { + Name string `json:"name"` + Definition map[string]any `json:"definition"` + ImageResourceID string `json:"imageResourceId"` + AudioResourceID string `json:"audioResourceId"` + VoiceName string `json:"voiceName"` + Instructions string `json:"instructions"` +} + +type CharacterAssetSummary struct { + Asset ProjectAssetSummary `json:"asset"` + Character CharacterCardSummary `json:"character"` +} + type UpdateProjectCharacterRequest struct { Name string `json:"name"` Definition map[string]any `json:"definition"` @@ -117,6 +131,164 @@ func (s *Service) ListVoiceProfiles(userID string) ([]VoiceProfileSummary, error return result, nil } +type CharacterListPage struct { + Characters []CharacterAssetSummary `json:"characters"` + Page int `json:"page"` + PageSize int `json:"pageSize"` + Total int64 `json:"total"` + HasMore bool `json:"hasMore"` +} + +func (s *Service) ListCharacters(userID string, query string, page int, pageSize int, ids []string) (CharacterListPage, error) { + var assets []model.Asset + var total int64 + var err error + if len(ids) > 0 { + assets, err = s.repo.UserCharacterAssetsByIDs(userID, ids) + total = int64(len(assets)) + page, pageSize = 1, len(assets) + } else { + assets, total, err = s.repo.UserCharacterAssetsPage(userID, query, page, pageSize) + pageSize = normalizeCharacterPageSize(pageSize) + if page < 1 { + page = 1 + } + } + if err != nil { + return CharacterListPage{}, err + } + result := make([]CharacterAssetSummary, 0, len(assets)) + for index := range assets { + detail, detailErr := s.characterAssetDetail(userID, &assets[index]) + if detailErr != nil { + return CharacterListPage{}, detailErr + } + result = append(result, detail) + } + if pageSize < 1 { + pageSize = len(result) + } + return CharacterListPage{Characters: result, Page: page, PageSize: pageSize, Total: total, HasMore: int64(page*pageSize) < total}, nil +} + +func normalizeCharacterPageSize(pageSize int) int { + if pageSize < 1 || pageSize > 48 { + return 12 + } + return pageSize +} + +func (s *Service) Character(userID string, assetID string) (CharacterAssetSummary, error) { + asset, err := s.repo.UserCharacterAsset(userID, strings.TrimSpace(assetID)) + if err != nil { + return CharacterAssetSummary{}, err + } + return s.characterAssetDetail(userID, asset) +} + +func (s *Service) CreateCharacter(userID string, req CreateCharacterRequest) (CharacterAssetSummary, error) { + name := strings.TrimSpace(req.Name) + if name == "" { + return CharacterAssetSummary{}, BadAuthRequest("角色名称不能为空") + } + definition, err := normalizedCharacterDefinition(req.Definition) + if err != nil { + return CharacterAssetSummary{}, err + } + now := time.Now() + assetID, versionID := newID(), newID() + payload, err := characterAssetPayload(assetID, versionID, name, definition, now, now) + if err != nil { + return CharacterAssetSummary{}, err + } + asset := model.Asset{ID: assetID, UserID: userID, Kind: "entity", Category: model.AssetCategoryCharacter, Status: model.AssetVersionStatusConfirmed, PrimaryVersionID: versionID, Title: name, PayloadJSON: payload, CreatedAt: now, UpdatedAt: now} + version := model.AssetVersion{ID: versionID, AssetID: assetID, Version: 1, Status: model.AssetVersionStatusConfirmed, DefinitionJSON: string(definition), CreatedAt: now, UpdatedAt: now} + representations, err := s.characterRepresentationsForResources(userID, newID(), req.ImageResourceID, now) + if err != nil { + return CharacterAssetSummary{}, err + } + voice, err := s.characterVoiceForResource(userID, req.AudioResourceID, req.VoiceName, req.Instructions, now) + if err != nil { + return CharacterAssetSummary{}, err + } + // 形象与声音绑定挂在首个版本上；漏设版本 ID 会让新卡读回来显示"形象缺失 / 声音未绑定"。 + for index := range representations { + representations[index].AssetVersionID = versionID + } + if voice != nil { + voice.AssetVersionID = versionID + } + if err := s.repo.CreateUserCharacter(&asset, &version, representations, voice); err != nil { + return CharacterAssetSummary{}, err + } + return s.characterAssetDetail(userID, &asset) +} + +func (s *Service) UpdateCharacter(userID string, assetID string, req UpdateProjectCharacterRequest) (CharacterAssetSummary, error) { + asset, err := s.repo.UserCharacterAsset(userID, strings.TrimSpace(assetID)) + if err != nil { + return CharacterAssetSummary{}, err + } + name := strings.TrimSpace(req.Name) + if name == "" { + return CharacterAssetSummary{}, BadAuthRequest("角色名称不能为空") + } + definition, err := normalizedCharacterDefinition(req.Definition) + if err != nil { + return CharacterAssetSummary{}, err + } + if _, err := s.createNextCharacterVersion("", asset, name, string(definition), nil, nil, false); err != nil { + return CharacterAssetSummary{}, err + } + return s.Character(userID, asset.ID) +} + +func (s *Service) ReplaceCharacterRepresentations(userID string, assetID string, req ReplaceCharacterRepresentationsRequest) (CharacterAssetSummary, error) { + asset, err := s.repo.UserCharacterAsset(userID, strings.TrimSpace(assetID)) + if err != nil { + return CharacterAssetSummary{}, err + } + representations, err := s.characterRepresentationsForInputs(userID, req.Representations, newID(), time.Now()) + if err != nil { + return CharacterAssetSummary{}, err + } + if _, err := s.createNextCharacterVersion("", asset, asset.Title, "", representations, nil, false); err != nil { + return CharacterAssetSummary{}, err + } + return s.Character(userID, asset.ID) +} + +func (s *Service) BindCharacterVoice(userID string, assetID string, req BindCharacterVoiceRequest) (CharacterAssetSummary, error) { + asset, err := s.repo.UserCharacterAsset(userID, strings.TrimSpace(assetID)) + if err != nil { + return CharacterAssetSummary{}, err + } + voice, err := s.characterVoiceForResource(userID, req.SampleResourceID, req.VoiceName, req.Instructions, time.Now()) + if err != nil { + return CharacterAssetSummary{}, err + } + if voice == nil { + voice, err = s.characterVoiceForProfile(userID, req.VoiceProfileID, req.Instructions, time.Now()) + if err != nil { + return CharacterAssetSummary{}, err + } + } + if _, err := s.createNextCharacterVersion("", asset, asset.Title, "", nil, voice, false); err != nil { + return CharacterAssetSummary{}, err + } + return s.Character(userID, asset.ID) +} + +func (s *Service) UnbindCharacterVoice(userID string, assetID string) (CharacterAssetSummary, error) { + asset, err := s.repo.UserCharacterAsset(userID, strings.TrimSpace(assetID)) + if err != nil { + return CharacterAssetSummary{}, err + } + if _, err := s.createNextCharacterVersion("", asset, asset.Title, "", nil, nil, true); err != nil { + return CharacterAssetSummary{}, err + } + return s.Character(userID, asset.ID) +} func (s *Service) CreateProjectCharacter(userID string, projectID string, req CreateProjectCharacterRequest) (ProjectCharacterDetail, error) { if _, err := s.repo.ProjectForUser(userID, projectID); err != nil { return ProjectCharacterDetail{}, err @@ -350,6 +522,11 @@ func (s *Service) BindProjectCharacterVoice(userID string, projectID string, ass } var profileErr error profile, profileErr = s.repo.VoiceProfileBySampleResource(userID, sampleResourceID) + if profileErr == nil { + if err := s.renameSampleVoiceProfile(userID, profile, req.VoiceName); err != nil { + return ProjectCharacterDetail{}, err + } + } if profileErr != nil { if !errors.Is(profileErr, gorm.ErrRecordNotFound) { return ProjectCharacterDetail{}, profileErr @@ -549,6 +726,119 @@ func (s *Service) characterCardVersion(userID string, version *model.AssetVersio return CharacterCardSummary{VersionID: version.ID, Version: version.Version, Definition: definition, Representations: representations, Voice: voice, VisualStatus: visualStatus, VoiceStatus: voiceStatus}, nil } +func (s *Service) characterAssetDetail(userID string, asset *model.Asset) (CharacterAssetSummary, error) { + if asset == nil || asset.Category != model.AssetCategoryCharacter || asset.Kind != "entity" { + return CharacterAssetSummary{}, BadAuthRequest("角色资产不可用") + } + versions, err := s.repo.AssetVersions(asset.ID) + if err != nil { + return CharacterAssetSummary{}, err + } + card, err := s.characterCard(userID, asset) + if err != nil { + return CharacterAssetSummary{}, err + } + storageKey, previewText, source, durationMs := projectAssetPreview(asset.PayloadJSON) + return CharacterAssetSummary{Asset: ProjectAssetSummary{ID: asset.ID, Title: asset.Title, MediaType: asset.Kind, Category: asset.Category, Status: asset.Status, PrimaryVersionID: asset.PrimaryVersionID, VersionCount: len(versions), Usages: []string{}, StorageKey: storageKey, PreviewText: previewText, DurationMs: durationMs, UpdatedAt: asset.UpdatedAt, Source: source, Character: &card}, Character: card}, nil +} + +func (s *Service) characterRepresentationsForResources(userID, taskID, imageResourceID string, now time.Time) ([]model.AssetRepresentation, error) { + imageResourceID = strings.TrimSpace(imageResourceID) + if imageResourceID == "" { + return nil, nil + } + resource, err := s.repo.ResourceForUser(userID, imageResourceID) + if err != nil || resource == nil || resource.Kind != "image" || resource.Status != model.ResourceStatusReady { + return nil, BadAuthRequest("角色形象资源不可用") + } + metadata, _ := json.Marshal(map[string]any{"source": "character_image"}) + return []model.AssetRepresentation{ + {ID: newID(), TaskID: taskID, ResourceID: imageResourceID, MediaType: "image", Role: "turnaround_sheet", MetadataJSON: string(metadata), CreatedAt: now}, + {ID: newID(), TaskID: taskID, ResourceID: imageResourceID, MediaType: "image", Role: "primary", MetadataJSON: string(metadata), CreatedAt: now}, + }, nil +} + +func (s *Service) characterRepresentationsForInputs(userID string, inputs []CharacterRepresentationInput, taskID string, now time.Time) ([]model.AssetRepresentation, error) { + if len(inputs) == 0 || len(inputs) > 8 { + return nil, BadAuthRequest("角色形象数量必须在 1 到 8 之间") + } + roles := make(map[string]struct{}, len(inputs)) + result := make([]model.AssetRepresentation, 0, len(inputs)) + for _, input := range inputs { + role := strings.TrimSpace(input.Role) + if !validCharacterRepresentationRole(role) { + return nil, BadAuthRequest("不支持的角色形象视角") + } + if _, exists := roles[role]; exists { + return nil, BadAuthRequest("同一角色形象视角不能重复") + } + roles[role] = struct{}{} + resourceID := strings.TrimSpace(input.ResourceID) + resource, err := s.repo.ResourceForUser(userID, resourceID) + if err != nil || resource == nil || resource.Kind != "image" || resource.Status != model.ResourceStatusReady { + return nil, BadAuthRequest("角色形象资源不可用") + } + metadata, marshalErr := json.Marshal(input.Metadata) + if marshalErr != nil { + return nil, BadAuthRequest("角色形象元数据格式无效") + } + result = append(result, model.AssetRepresentation{ID: newID(), TaskID: taskID, ResourceID: resourceID, MediaType: "image", Role: role, MetadataJSON: string(metadata), CreatedAt: now}) + } + return result, nil +} + +func (s *Service) characterVoiceForResource(userID, resourceID, voiceName, instructions string, now time.Time) (*model.CharacterVoiceBinding, error) { + resourceID = strings.TrimSpace(resourceID) + if resourceID == "" { + return nil, nil + } + resource, err := s.repo.ResourceForUser(userID, resourceID) + if err != nil || resource == nil || resource.Kind != "audio" || resource.Status != model.ResourceStatusReady || !isSupportedVoiceSampleMimeType(resource.MimeType) { + return nil, BadAuthRequest("请选择已上传完成的支持格式音频：MP3、WAV、M4A/AAC、FLAC、OGG/Opus 或 WebM") + } + profile, profileErr := s.repo.VoiceProfileBySampleResource(userID, resourceID) + if profileErr == nil { + if err := s.renameSampleVoiceProfile(userID, profile, voiceName); err != nil { + return nil, err + } + } + if profileErr != nil { + if !errors.Is(profileErr, gorm.ErrRecordNotFound) { + return nil, profileErr + } + voiceName = strings.TrimSpace(voiceName) + if voiceName == "" { + voiceName = "上传声音 · " + resourceID[:min(8, len(resourceID))] + } + profile = &model.VoiceProfile{ID: newID(), UserID: userID, Name: voiceName, Provider: "user_upload", VoiceKey: "sample:" + resourceID, Language: "按样本使用", Timbre: "用户上传样本", SampleResourceID: resourceID, CompatibleModelsJSON: "[]", Status: "active", CreatedAt: now, UpdatedAt: now} + if err := s.repo.CreateVoiceProfile(profile); err != nil { + return nil, err + } + } + return &model.CharacterVoiceBinding{ID: newID(), VoiceProfileID: profile.ID, Instructions: strings.TrimSpace(instructions), CreatedAt: now, UpdatedAt: now}, nil +} + +// renameSampleVoiceProfile 让同一段样本复用的声音档案跟随用户最新给的名字 +// （例如画布上把「生成音频」改名为「杨过·声音」后重新绑定）。 +func (s *Service) renameSampleVoiceProfile(userID string, profile *model.VoiceProfile, voiceName string) error { + voiceName = truncateRunes(strings.TrimSpace(voiceName), 160) + if profile == nil || profile.Provider != "user_upload" || voiceName == "" || voiceName == profile.Name { + return nil + } + if err := s.repo.RenameVoiceProfile(userID, profile.ID, voiceName); err != nil { + return err + } + profile.Name = voiceName + return nil +} + +func (s *Service) characterVoiceForProfile(userID, profileID, instructions string, now time.Time) (*model.CharacterVoiceBinding, error) { + profile, err := s.repo.VoiceProfileForUser(userID, strings.TrimSpace(profileID)) + if err != nil || profile == nil || profile.Status != "active" { + return nil, BadAuthRequest("选择的声音素材不可用") + } + return &model.CharacterVoiceBinding{ID: newID(), VoiceProfileID: profile.ID, Instructions: strings.TrimSpace(instructions), CreatedAt: now, UpdatedAt: now}, nil +} func normalizedCharacterDefinition(value map[string]any) (json.RawMessage, error) { if value == nil { value = map[string]any{} diff --git a/backend/internal/app/project_character_voice_rename_test.go b/backend/internal/app/project_character_voice_rename_test.go new file mode 100644 index 000000000..b03929d2a --- /dev/null +++ b/backend/internal/app/project_character_voice_rename_test.go @@ -0,0 +1,41 @@ +package app + +import ( + "testing" + "time" + + "infinite-canvas/backend/internal/model" +) + +func TestCharacterVoiceRebindUsesLatestSampleName(t *testing.T) { + service, db, _, _ := creationTestService(t) + now := time.Now() + for _, item := range []any{ + &model.Resource{ID: "voice-sample", UserID: "user", Kind: "audio", Status: model.ResourceStatusReady, MimeType: "audio/mpeg", Size: 10}, + &model.VoiceProfile{ID: "sample-profile", UserID: "user", Name: "生成音频", Provider: "user_upload", VoiceKey: "sample:voice-sample", SampleResourceID: "voice-sample", CompatibleModelsJSON: "[]", Status: "active", CreatedAt: now, UpdatedAt: now}, + } { + if err := db.Create(item).Error; err != nil { + t.Fatal(err) + } + } + binding, err := service.characterVoiceForResource("user", "voice-sample", "杨过·声音", "", now) + if err != nil || binding.VoiceProfileID != "sample-profile" { + t.Fatalf("binding = %#v, %v", binding, err) + } + var renamed model.VoiceProfile + if err := db.First(&renamed, "id = ?", "sample-profile").Error; err != nil || renamed.Name != "杨过·声音" { + t.Fatalf("sample voice kept stale name: %q, %v", renamed.Name, err) + } + // 空名字不能把已有名字清掉。 + if _, err := service.characterVoiceForResource("user", "voice-sample", " ", "", now); err != nil { + t.Fatal(err) + } + if err := db.First(&renamed, "id = ?", "sample-profile").Error; err != nil || renamed.Name != "杨过·声音" { + t.Fatalf("empty name overwrote voice name: %q, %v", renamed.Name, err) + } + // 内置音色名称由系统维护，不跟随用户输入改名。 + builtin := model.VoiceProfile{ID: "builtin-profile", Provider: "openai", Name: "alloy"} + if err := service.renameSampleVoiceProfile("user", &builtin, "改名"); err != nil || builtin.Name != "alloy" { + t.Fatalf("builtin voice renamed: %q, %v", builtin.Name, err) + } +} diff --git a/backend/internal/app/provider.go b/backend/internal/app/provider.go index 1e2f48e65..f9b09691e 100644 --- a/backend/internal/app/provider.go +++ b/backend/internal/app/provider.go @@ -86,6 +86,8 @@ type providerConfig struct { AudioFormat string `json:"audioFormat"` AudioSpeed string `json:"audioSpeed"` AudioInstructions string `json:"audioInstructions"` + AudioLanguage string `json:"audioLanguage"` + AudioDialect string `json:"audioDialect"` SystemPrompt string `json:"systemPrompt"` CapabilityConfig *ModelCapabilityConfig `json:"capabilityConfig"` WorkflowID string `json:"workflowId"` @@ -308,7 +310,8 @@ func providerPrefersMediaURLs(interfaceType string, input canvasGenerationInput) string(model.ChannelInterfaceXAIVideo), string(model.ChannelInterfaceNovitaVideo), string(model.ChannelInterfaceMiniMaxVideo), string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), - string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo): + string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo), + string(model.ChannelInterfaceDoubaoStreamingTTS): return true } if isGrokVideoConfig(input.Config) || isSeedanceVideoConfig(input.Config) || isArkPlanVideoConfig(input.Config) { diff --git a/backend/internal/app/provider_error.go b/backend/internal/app/provider_error.go index 9195ba63e..ee8e84cc9 100644 --- a/backend/internal/app/provider_error.go +++ b/backend/internal/app/provider_error.go @@ -1,8 +1,11 @@ package app import ( + "bytes" "encoding/json" + "errors" "fmt" + "io" "strings" ) @@ -40,14 +43,44 @@ func providerResponseBusinessFailure(responseBody []byte) (string, string, bool) if len(responseBody) == 0 { return "", "", false } - var payload map[string]any - if json.Unmarshal(responseBody, &payload) != nil { - return "", "", false + // 火山单向语音合成把多帧 JSON 直接拼在同一个 HTTP 200 里。只解析第一帧会把后面的业务失败当成成功。 + decoder := json.NewDecoder(bytes.NewReader(responseBody)) + decoder.UseNumber() + decoded := false + for { + var payload map[string]any + err := decoder.Decode(&payload) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + if !decoded { + return "", "", false + } + break + } + decoded = true + if payload == nil { + continue + } + if code, message, failed := providerPayloadBusinessFailure(payload); failed { + return code, message, true + } } - return providerPayloadBusinessFailure(payload) + return "", "", false } func providerPayloadBusinessFailure(payload map[string]any) (string, string, bool) { + if header, ok := payload["header"].(map[string]any); ok { + // 只认火山语音合成这种明确的资源拒绝。header.code 在别的协议里可能是 HTTP 状态，不能一律当成业务失败。 + rawMessage := strings.TrimSpace(stringField(header, "message")) + if speechResourceDeniedUserMessage(rawMessage) != "" { + if code, message, failed := providerBusinessFailure(header); failed { + return code, message, true + } + return "resource_not_granted", rawMessage, true + } + } if code, message, failed := providerBusinessFailure(payload); failed { return code, message, true } diff --git a/backend/internal/app/provider_error_test.go b/backend/internal/app/provider_error_test.go index e20962eb8..536d80401 100644 --- a/backend/internal/app/provider_error_test.go +++ b/backend/internal/app/provider_error_test.go @@ -81,6 +81,26 @@ func TestProviderPayloadBusinessFailureReadsNestedOutputFailure(t *testing.T) { } } +func TestProviderResponseBusinessFailureExplainsDoubaoSpeechDenial(t *testing.T) { + body := []byte(`{"code":0,"data":"AAAA"}{"header":{"code":45000030,"message":"[resource_id=volc.seedtts.default] requested resource not granted"}}`) + code, message, failed := providerResponseBusinessFailure(body) + if !failed || !strings.Contains(message, "requested resource not granted") { + t.Fatalf("business failure = (%q, %q, %v)", code, message, failed) + } + if code != "45000030" && code != "4.500003e+07" { + t.Fatalf("unexpected code %q", code) + } +} + +func TestProviderPayloadBusinessFailureIgnoresOrdinaryHeader(t *testing.T) { + if _, _, failed := providerPayloadBusinessFailure(map[string]any{ + "header": map[string]any{"code": float64(200), "message": "ok"}, + "data": map[string]any{"audio": "AAAA"}, + }); failed { + t.Fatal("ordinary header code was treated as a business failure") + } +} + func TestProviderPayloadBusinessFailureIgnoresPendingOutput(t *testing.T) { if code, message, failed := providerPayloadBusinessFailure(map[string]any{ "output": map[string]any{"task_id": "task-1", "task_status": "PENDING"}, diff --git a/backend/internal/app/provider_errors.go b/backend/internal/app/provider_errors.go index e0d7d3ada..8a170a670 100644 --- a/backend/internal/app/provider_errors.go +++ b/backend/internal/app/provider_errors.go @@ -129,9 +129,21 @@ func (e providerHTTPError) Error() string { if e.StatusCode == http.StatusBadRequest || e.StatusCode == http.StatusUnprocessableEntity { return providerErrorWithDetail(e.summary(), e.Body) } + if message := speechResourceDeniedUserMessage(e.Body); (e.StatusCode == http.StatusUnauthorized || e.StatusCode == http.StatusForbidden) && message != "" { + return message + } return appendProviderErrorDetail(e.summary(), e.Body) } +const volcengineSpeechResourceDeniedMessage = "语音合成服务未开通，或音色与模型版本不匹配。请在火山引擎控制台开通语音合成 1.0，并确认当前音色属于这一版本" + +func speechResourceDeniedUserMessage(raw string) string { + if strings.Contains(strings.ToLower(raw), "requested resource not granted") { + return volcengineSpeechResourceDeniedMessage + } + return "" +} + func (e providerHTTPError) summary() string { switch e.StatusCode { case 524: diff --git a/backend/internal/app/provider_http_client.go b/backend/internal/app/provider_http_client.go index 580910af3..6cb076e62 100644 --- a/backend/internal/app/provider_http_client.go +++ b/backend/internal/app/provider_http_client.go @@ -263,16 +263,22 @@ func doBinaryWithConsumer(req *http.Request, onChunk func(string, []byte)) ([]by channelID = metadata.ChannelID policy, err := metadata.Service.RuntimePolicy() if err != nil { - return nil, "", fmt.Errorf("读取生成资源限制失败：%w", err) + err = fmt.Errorf("读取生成资源限制失败：%w", err) + recordProviderRequest(req, startedAt, 0, nil, err) + return nil, "", err } responseLimit = megabytes(policy.Resource.GeneratedFileMB) if !cacheManagementRequest { open, err := coordinator.CircuitOpen(req.Context(), channelID) if err != nil { - return nil, "", fmt.Errorf("读取渠道熔断状态失败：%w", err) + err = fmt.Errorf("读取渠道熔断状态失败：%w", err) + recordProviderRequest(req, startedAt, 0, nil, err) + return nil, "", err } if open { - return nil, "", providerCircuitOpenError{} + err = providerCircuitOpenError{} + recordProviderRequest(req, startedAt, 0, nil, err) + return nil, "", err } slotID := channelID if slotID == "" { @@ -445,6 +451,7 @@ func recordProviderRequest(req *http.Request, startedAt time.Time, statusCode in } } metadata.Service.EnrichAPICallLog(&callLog, responseBody) + callLog.Error = withUserVisibleLogError(requestErr, callLog.Error) if err := metadata.Service.LogAPICall(callLog); err != nil { if !channelSlotFailure && metadata.Billing != nil { if uncertainErr := metadata.Billing.MarkBillingUncertain(metadata.BillingOrderID, "上游调用日志写入失败，费用状态待核对"); uncertainErr != nil { @@ -480,6 +487,43 @@ func providerRequestErrorDetails(err error) (string, string) { return "", safeProviderLogError(err) } +// withUserVisibleLogError 把画布上实际展示的失败原因写进请求日志，同时保留原始诊断。 +// 语音合成权限/音色不匹配和网络异常都会被前端改写成固定中文；只存上游原文时，后台按用户看到的文案检索不到。 +func withUserVisibleLogError(requestErr error, stored string) string { + stored = strings.TrimSpace(stored) + visible := userVisibleFailureLabel(requestErr, stored) + if visible == "" || strings.Contains(stored, visible) { + return truncateRunes(stored, 2_000) + } + if stored == "" { + return visible + } + return truncateRunes(visible+" "+stored, 2_000) +} + +func userVisibleFailureLabel(requestErr error, stored string) string { + texts := make([]string, 0, 3) + if requestErr != nil { + texts = append(texts, taskFailureMessage(requestErr), requestErr.Error()) + } + if stored != "" { + texts = append(texts, stored) + } + network := false + for _, text := range texts { + if speechResourceDeniedUserMessage(text) != "" { + return volcengineSpeechResourceDeniedMessage + } + if userFacingTaskError(text) == taskErrorNetworkMessage { + network = true + } + } + if network { + return taskErrorNetworkMessage + } + return "" +} + func safeProviderLogError(err error) string { var httpErr providerHTTPError if errors.As(err, &httpErr) { diff --git a/backend/internal/app/provider_protocol.go b/backend/internal/app/provider_protocol.go index 50c1ce802..13cb18535 100644 --- a/backend/internal/app/provider_protocol.go +++ b/backend/internal/app/provider_protocol.go @@ -66,7 +66,11 @@ func runProtocolAdapterTaskWithPolicy(ctx context.Context, input canvasGeneratio if streamedResult != nil { created = protocol.CreateResult{Status: protocol.StatusSucceeded, Result: streamedResult} } else { - created, err = adapter.ParseCreate(ctx, body) + if parser, ok := adapter.(protocol.RequestAwareCreateParser); ok { + created, err = parser.ParseCreateWithRequest(ctx, request, body) + } else { + created, err = adapter.ParseCreate(ctx, body) + } } if err != nil { return nil, err diff --git a/backend/internal/app/provider_protocol_request.go b/backend/internal/app/provider_protocol_request.go index 9e5882d16..9307b2de4 100644 --- a/backend/internal/app/provider_protocol_request.go +++ b/backend/internal/app/provider_protocol_request.go @@ -46,10 +46,14 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe Watermark: parseBool(input.Config.VideoWatermark, false), Operation: firstNonEmpty(metadataString(input.Metadata, "videoEditOperation"), metadataString(input.Metadata, "videoOperation")), Extra: map[string]any{ - "videoSeconds": input.Config.VideoSeconds, - "audioVoice": input.Config.AudioVoice, - "audioFormat": input.Config.AudioFormat, - "count": input.Config.Count, + "videoSeconds": input.Config.VideoSeconds, + "audioVoice": input.Config.AudioVoice, + "audioFormat": input.Config.AudioFormat, + "audioSpeed": input.Config.AudioSpeed, + "audioInstructions": input.Config.AudioInstructions, + "audioLanguage": input.Config.AudioLanguage, + "audioDialect": input.Config.AudioDialect, + "count": input.Config.Count, }, } for _, message := range input.TextHistory { diff --git a/backend/internal/app/provider_task_recovery.go b/backend/internal/app/provider_task_recovery.go index 455727a8b..9d5a8a518 100644 --- a/backend/internal/app/provider_task_recovery.go +++ b/backend/internal/app/provider_task_recovery.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "net/url" "strings" + "sync" "time" "infinite-canvas/backend/internal/model" @@ -19,6 +21,21 @@ type ProviderTaskQueryResult struct { ProviderStatus string `json:"providerStatus"` Recovered bool `json:"recovered"` BillingSettled bool `json:"billingSettled"` + Error string `json:"error,omitempty"` +} + +type ProviderTaskRecoveryItem struct { + LogID string `json:"logId"` + Recovered bool `json:"recovered"` + ProviderStatus string `json:"providerStatus,omitempty"` + Error string `json:"error,omitempty"` +} + +type ProviderTaskRecoveryBatchResult struct { + Items []ProviderTaskRecoveryItem `json:"items"` + Success int `json:"success"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` } func (s *Service) QueryFailedVideoTask(ctx context.Context, userID string, taskID string) (*ProviderTaskQueryResult, error) { @@ -29,7 +46,7 @@ func (s *Service) QueryFailedVideoTask(ctx context.Context, userID string, taskI return s.queryFailedVideoTask(ctx, task, strings.TrimSpace(userID)) } -func (s *Service) AdminQueryFailedVideoTask(ctx context.Context, actor *model.User, logID string) (*ProviderTaskQueryResult, error) { +func (s *Service) AdminQueryFailedVideoTask(ctx context.Context, actor *model.User, logID string, requestedProviderID string) (result *ProviderTaskQueryResult, returnErr error) { if err := s.RequireAdmin(actor); err != nil { return nil, err } @@ -37,31 +54,262 @@ func (s *Service) AdminQueryFailedVideoTask(ctx context.Context, actor *model.Us if err != nil { return nil, err } + var task *model.Task + defer func() { + metadata := map[string]any{ + "apiCallLogId": log.ID, + "taskId": log.TaskID, + "suppliedProviderRequestId": strings.TrimSpace(requestedProviderID), + "providerRequestId": "", + "providerStatus": "", + "recovered": result != nil && result.Recovered, + "outcome": "failed", + "error": errorString(returnErr), + } + if task != nil { + metadata["providerRequestId"] = task.ProviderRequestID + } + if result != nil { + metadata["providerStatus"] = result.ProviderStatus + if result.Recovered { + metadata["outcome"] = "recovered" + } else { + metadata["outcome"] = "provider_processing" + } + } + _ = s.appendAdminAudit(actor, "api_log.query_provider_task", "api_call_log", log.ID, "人工查询失败视频任务", metadata) + }() if log.Capability != "video" || strings.TrimSpace(log.TaskID) == "" { return nil, BadAuthRequest("该请求没有可查询的视频任务") } - task, err := s.repo.Task(log.TaskID) - if err != nil { - return nil, err + task, returnErr = s.repo.Task(log.TaskID) + if returnErr != nil { + return nil, returnErr } if task.UserID != log.UserID { return nil, BadAuthRequest("请求与任务归属不一致") } + if task.Status == model.TaskStatusSucceeded { + result = &ProviderTaskQueryResult{Task: taskForOutput(*task), ProviderStatus: "succeeded", Recovered: true, BillingSettled: s.billingSettled(task)} + return result, nil + } + if strings.TrimSpace(requestedProviderID) != "" { + task.ProviderRequestID = strings.TrimSpace(requestedProviderID) + } if task.ProviderRequestID == "" { task.ProviderRequestID = strings.TrimSpace(log.ProviderRequestID) } - result, err := s.queryFailedVideoTask(ctx, task, "") + result, returnErr = s.queryFailedVideoTask(ctx, task, "") + return result, returnErr +} + +func (s *Service) AdminRecoverVideoByURL(ctx context.Context, actor *model.User, logID string, rawURL string, providerRequestID string) (result *ProviderTaskQueryResult, returnErr error) { + if err := s.RequireAdmin(actor); err != nil { + return nil, err + } + log, task, err := s.adminRecoveryTask(logID) if err != nil { return nil, err } - if err := s.appendAdminAudit(actor, "api_log.query_provider_task", "task", task.ID, "人工查询失败视频任务", map[string]any{ - "apiCallLogId": log.ID, "providerRequestId": task.ProviderRequestID, "providerStatus": result.ProviderStatus, "recovered": result.Recovered, - }); err != nil { + parsedURL := "" + defer func() { + metadata := map[string]any{ + "apiCallLogId": log.ID, + "taskId": task.ID, + "suppliedProviderRequestId": strings.TrimSpace(providerRequestID), + "providerRequestId": task.ProviderRequestID, + "url": parsedURL, + "recovered": result != nil && result.Recovered, + "outcome": "failed", + "error": errorString(returnErr), + } + if result != nil { + metadata["providerStatus"] = result.ProviderStatus + if result.Recovered { + metadata["outcome"] = "recovered" + } + } + _ = s.appendAdminAudit(actor, "api_log.recover_video_url", "api_call_log", log.ID, "人工转存上游视频并回写", metadata) + }() + if task.Status == model.TaskStatusSucceeded { + result = &ProviderTaskQueryResult{Task: taskForOutput(*task), ProviderStatus: "succeeded", Recovered: true, BillingSettled: s.billingSettled(task)} + return result, nil + } + if task.Status != model.TaskStatusFailed { + return nil, BadAuthRequest("只能恢复失败的视频任务") + } + if strings.TrimSpace(providerRequestID) != "" { + task.ProviderRequestID = strings.TrimSpace(providerRequestID) + } + if task.ProviderRequestID == "" { + task.ProviderRequestID = strings.TrimSpace(log.ProviderRequestID) + } + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Hostname() == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return nil, BadAuthRequest("视频 URL 只支持有效的 http/https 地址") + } + parsedURL = recoveryURLForAudit(parsed) + owner := "manual-recovery:" + newID() + if err := s.repo.ClaimFailedTaskProviderRecovery(task.ID, "", owner, providerTaskRecoveryLeaseDuration); err != nil { + if errors.Is(err, repository.ErrTaskProviderRecoveryConflict) { + return nil, &AuthError{Status: 409, Message: "该任务正在恢复中，请稍后再试"} + } return nil, err } + task.LeaseOwner = owner + defer func() { _ = s.repo.ReleaseTaskProviderRecovery(task.ID, owner) }() + policy, err := s.RuntimePolicy() + if err != nil { + return nil, err + } + payload, err := downloadRemoteResource(parsed.String(), megabytes(policy.Resource.GeneratedFileMB)+1) + if err != nil { + return nil, err + } + if !strings.HasPrefix(strings.ToLower(payload.mimeType), "video/") { + return nil, BadAuthRequest("上游地址返回的不是视频文件") + } + resultPayload := map[string]interface{}{"mode": "video", "video": map[string]interface{}{"dataUrl": dataURL(payload.mimeType, payload.data), "mimeType": payload.mimeType}} + resultPayload, err = s.persistGeneratedMediaResult(task.UserID, resultPayload) + if err != nil { + return nil, err + } + resultJSON, err := json.Marshal(resultPayload) + if err != nil { + return nil, err + } + task.Error = "" + task.PollStage = "succeeded" + task.NextPollAt = nil + if err := s.saveTaskCompletionWithinStorageQuota(task, resultJSON, nil, false); err != nil { + return nil, err + } + if err := s.settleRecoveredBilling(task, task.ProviderRequestID); err != nil { + uncertainErr := s.taskBilling().MarkBillingUncertain(task.BillingOrderID, "人工转存视频已成功，但积分结算失败："+err.Error()) + if uncertainErr != nil { + return nil, errors.Join(err, uncertainErr) + } + return nil, err + } + if err := s.RegisterTaskOutputFromTask(*task); err != nil { + return nil, err + } + result = &ProviderTaskQueryResult{Task: taskForOutput(*task), ProviderStatus: "succeeded", Recovered: true, BillingSettled: true} return result, nil } +func (s *Service) AdminBatchQueryFailedVideoTasks(ctx context.Context, actor *model.User, logIDs []string) (*ProviderTaskRecoveryBatchResult, error) { + if err := s.RequireAdmin(actor); err != nil { + return nil, err + } + ids := uniqueNonEmpty(logIDs) + result := &ProviderTaskRecoveryBatchResult{Items: make([]ProviderTaskRecoveryItem, len(ids))} + sem := make(chan struct{}, 4) + var wg sync.WaitGroup + for index, id := range ids { + index, id := index, id + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + item := ProviderTaskRecoveryItem{LogID: id} + log, logErr := s.repo.APICallLog(id) + if logErr != nil { + item.Error = logErr.Error() + _ = s.appendAdminAudit(actor, "api_log.query_provider_task", "api_call_log", id, "批量查询视频任务失败", map[string]any{"apiCallLogId": id, "outcome": "failed", "error": item.Error}) + result.Items[index] = item + return + } + if log.Capability != "video" { + item.Error = "跳过：不是视频请求明细" + _ = s.appendAdminAudit(actor, "api_log.query_provider_task", "api_call_log", id, "批量查询跳过非视频请求", map[string]any{"apiCallLogId": id, "outcome": "skipped", "error": item.Error}) + result.Items[index] = item + return + } + queryResult, queryErr := s.AdminQueryFailedVideoTask(ctx, actor, id, "") + if queryErr != nil { + item.Error = queryErr.Error() + } else { + item.Recovered, item.ProviderStatus = queryResult.Recovered, queryResult.ProviderStatus + } + result.Items[index] = item + }() + } + wg.Wait() + for _, item := range result.Items { + if strings.HasPrefix(item.Error, "跳过：") { + result.Skipped++ + } else if item.Error != "" || !item.Recovered { + result.Failed++ + } else { + result.Success++ + } + } + return result, nil +} + +func (s *Service) adminRecoveryTask(logID string) (*model.ApiCallLog, *model.Task, error) { + log, err := s.repo.APICallLog(strings.TrimSpace(logID)) + if err != nil { + return nil, nil, err + } + if log.Capability != "video" || strings.TrimSpace(log.TaskID) == "" { + return nil, nil, BadAuthRequest("该请求没有可恢复的视频任务") + } + task, err := s.repo.Task(log.TaskID) + if err != nil { + return nil, nil, err + } + if task.UserID != log.UserID { + return nil, nil, BadAuthRequest("请求与任务归属不一致") + } + return log, task, nil +} + +func (s *Service) billingSettled(task *model.Task) bool { + if task == nil || task.BillingOrderID == "" { + return true + } + order, err := s.repo.BillingOrder(task.BillingOrderID) + return err == nil && order.Status == model.BillingStatusSettled +} + +func (s *Service) settleRecoveredBilling(task *model.Task, providerRequestID string) error { + if task == nil || task.BillingOrderID == "" { + return nil + } + billing := s.taskBilling() + order, err := billing.BillingOrder(task.BillingOrderID) + if err != nil { + return err + } + if order == nil { + return fmt.Errorf("任务计费订单不存在：%s", task.BillingOrderID) + } + if order.Status == model.BillingStatusSettled { + return nil + } + if order.Status == model.BillingStatusRefunded { + return billing.RestoreRefundedBilling(task.BillingOrderID, providerRequestID) + } + return billing.SettleBilling(task.BillingOrderID, providerRequestID) +} + +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func recoveryURLForAudit(value *url.URL) string { + if value == nil { + return "" + } + return value.Scheme + "://" + value.Host + value.Path +} + func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, claimUserID string) (*ProviderTaskQueryResult, error) { billing := s.taskBilling() ctx = withProtocolRegistry(ctx, s.protocolRegistry()) @@ -83,7 +331,6 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl if providerRequestID == "" { return nil, BadAuthRequest("该任务没有可恢复的上游任务 ID") } - wasRefunded := false if task.BillingOrderID != "" { order, err := s.repo.BillingOrder(task.BillingOrderID) if err != nil { @@ -92,7 +339,6 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl if order.UserID != task.UserID || order.TaskID != task.ID { return nil, BadAuthRequest("任务与计费订单归属不一致") } - wasRefunded = order.Status == model.BillingStatusRefunded } decryptedInput, err := s.decryptTaskInputJSON(task.InputJSON) @@ -115,9 +361,6 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl input.Config = config task.InputJSON = decryptedInput task.ProviderRequestID = providerRequestID - if err := s.repo.UpdateTaskProviderState(task.ID, providerRequestID, task.PollStage, task.NextPollAt); err != nil { - return nil, err - } owner := "manual-recovery:" + newID() if err := s.repo.ClaimFailedTaskProviderRecovery(task.ID, claimUserID, owner, providerTaskRecoveryLeaseDuration); err != nil { @@ -173,11 +416,7 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl } billingSettled := true var billingErr error - if wasRefunded { - billingErr = billing.RestoreRefundedBilling(task.BillingOrderID, providerRequestID) - } else { - billingErr = billing.SettleBilling(task.BillingOrderID, providerRequestID) - } + billingErr = s.settleRecoveredBilling(task, providerRequestID) if billingErr != nil { billingSettled = false uncertainErr := billing.MarkBillingUncertain(task.BillingOrderID, "人工查询确认生成成功，但积分结算失败："+billingErr.Error()) diff --git a/backend/internal/app/provider_task_recovery_test.go b/backend/internal/app/provider_task_recovery_test.go index 5e5a73bd0..26c065d45 100644 --- a/backend/internal/app/provider_task_recovery_test.go +++ b/backend/internal/app/provider_task_recovery_test.go @@ -3,7 +3,12 @@ package app import ( "context" "errors" + "net/http" + "net/http/httptest" + "strings" "testing" + + "infinite-canvas/backend/internal/model" ) func TestProviderTaskRecoveryContextSurvivesClientCancellation(t *testing.T) { @@ -24,6 +29,96 @@ func TestProviderTaskRecoveryContextSurvivesClientCancellation(t *testing.T) { } } +func TestSettleRecoveredBillingIsIdempotent(t *testing.T) { + tests := []struct { + name string + status model.BillingStatus + wantSettle int + wantRestore int + }{ + {name: "settled", status: model.BillingStatusSettled}, + {name: "running", status: model.BillingStatusRunning, wantSettle: 1}, + {name: "refunded", status: model.BillingStatusRefunded, wantRestore: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := &taskBillingRepositoryStub{order: &model.BillingOrder{Status: tt.status}} + svc := &Service{taskBillingCoordinator: newTaskBillingCoordinator(repo)} + if err := svc.settleRecoveredBilling(&model.Task{BillingOrderID: "order-1"}, "provider-1"); err != nil { + t.Fatalf("settleRecoveredBilling() error = %v", err) + } + if got := len(repo.settleCalls); got != tt.wantSettle { + t.Fatalf("settle calls = %d, want %d", got, tt.wantSettle) + } + if got := len(repo.restoreCalls); got != tt.wantRestore { + t.Fatalf("restore calls = %d, want %d", got, tt.wantRestore) + } + }) + } +} + +func TestDownloadRemoteResourceEnforcesVideoImportBounds(t *testing.T) { + t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") + t.Setenv("HTTP_PROXY", "") + t.Setenv("HTTPS_PROXY", "") + t.Setenv("ALL_PROXY", "") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/large" { + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("12345")) + return + } + w.Header().Set("Content-Type", "video/mp4; charset=binary") + _, _ = w.Write([]byte("video")) + })) + defer server.Close() + + payload, err := downloadRemoteResource(server.URL+"/video.mp4", 16) + if err != nil { + t.Fatalf("downloadRemoteResource() error = %v", err) + } + if payload.mimeType != "video/mp4" || string(payload.data) != "video" { + t.Fatalf("payload = %#v, want video/mp4 body", payload) + } + if _, err := downloadRemoteResource(server.URL+"/large", 5); err == nil || !strings.Contains(err.Error(), "远程资源必须小于") { + t.Fatalf("oversized download error = %v", err) + } + if _, err := downloadRemoteResource("file:///tmp/video.mp4", 16); err == nil { + t.Fatal("downloadRemoteResource() accepted a non-http URL") + } +} + +func TestAdminVideoRecoveryRequiresAdmin(t *testing.T) { + service := &Service{} + methods := map[string]func(*model.User) error{ + "query": func(actor *model.User) error { + _, err := service.AdminQueryFailedVideoTask(context.Background(), actor, "log-1", "provider-1") + return err + }, + "url": func(actor *model.User) error { + _, err := service.AdminRecoverVideoByURL(context.Background(), actor, "log-1", "https://example.com/video.mp4", "provider-1") + return err + }, + "batch": func(actor *model.User) error { + _, err := service.AdminBatchQueryFailedVideoTasks(context.Background(), actor, []string{"log-1"}) + return err + }, + } + actors := map[string]*model.User{ + "anonymous": nil, + "ordinary user": {Role: model.UserRoleUser}, + } + for actorName, actor := range actors { + for methodName, invoke := range methods { + t.Run(actorName+"/"+methodName, func(t *testing.T) { + if err := invoke(actor); err == nil { + t.Fatalf("recovery method %q accepted %s actor", methodName, actorName) + } + }) + } + } +} + func TestRetryableProtocolMediaDownload(t *testing.T) { for _, err := range []error{ errors.New("net/http: TLS handshake timeout"), diff --git a/backend/internal/app/provider_test.go b/backend/internal/app/provider_test.go index 65b914a47..576daf0da 100644 --- a/backend/internal/app/provider_test.go +++ b/backend/internal/app/provider_test.go @@ -968,6 +968,19 @@ func TestProviderUserFacingErrorMessageClassifiesRejectedRequestBodies(t *testin } } +func TestProviderUserFacingErrorMessageExplainsDoubaoResourceDenial(t *testing.T) { + message := providerUserFacingErrorMessage(providerHTTPError{ + StatusCode: http.StatusForbidden, + Body: `{"header":{"code":45000030,"message":"[resource_id=volc.seedtts.default] requested resource not granted"}}`, + }) + if !strings.Contains(message, "语音合成服务未开通") { + t.Fatalf("providerUserFacingErrorMessage() = %q", message) + } + if strings.Contains(message, "API Key") { + t.Fatalf("resource denial was reported as an API key failure: %q", message) + } +} + func TestProviderUserFacingErrorMessageOnlyClassifiesValidationStatuses(t *testing.T) { // 鉴权失败与网关错误的正文可能是密钥诊断或代理 HTML，不参与归类。 for _, statusCode := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusBadGateway} { diff --git a/backend/internal/app/task_api_call_log.go b/backend/internal/app/task_api_call_log.go index c408400cf..7bcfd8d07 100644 --- a/backend/internal/app/task_api_call_log.go +++ b/backend/internal/app/task_api_call_log.go @@ -17,12 +17,14 @@ func (s *Service) ensureFailedProviderAttemptLogged(task model.Task, taskErr err if s == nil || s.repo == nil || strings.TrimSpace(task.ID) == "" || taskErr == nil { return } - hasLog, err := s.repo.HasAPICallLogForTask(task.ID) + hasVisibleFailure, err := s.repo.HasVisibleFailedAPICallLogForTask(task.ID) if err != nil { log.Printf("provider attempt log lookup failed: task_id=%s error=%v", task.ID, err) return } - if hasLog { + // 已有一条管理端默认可看到的失败日志时不再补写。只存在成功日志或被列表隐藏的轮询/下载日志时， + // 用户看到的失败原因仍然不在请求明细里，需要补一条失败记录。 + if hasVisibleFailure { return } @@ -83,7 +85,7 @@ func (s *Service) ensureFailedProviderAttemptLogged(task model.Task, taskErr err DurationMs: now.Sub(startedAt).Milliseconds(), ProviderRequestID: metadata.ProviderRequestID, ErrorCode: errorCode, - Error: truncateRunes(s.UserFacingErrorMessage(taskErr), 2_000), + Error: truncateRunes(withUserVisibleLogError(taskErr, s.UserFacingErrorMessage(taskErr)), 2_000), StartedAt: startedAt, CreatedAt: now, } diff --git a/backend/internal/app/task_api_call_log_test.go b/backend/internal/app/task_api_call_log_test.go index 2489e779e..2ed8b9470 100644 --- a/backend/internal/app/task_api_call_log_test.go +++ b/backend/internal/app/task_api_call_log_test.go @@ -1,7 +1,9 @@ package app import ( + "context" "errors" + "net/http" "strings" "testing" "time" @@ -132,3 +134,82 @@ func TestEnsureFailedProviderAttemptLoggedDoesNotDuplicateHTTPLog(t *testing.T) t.Fatalf("api call log count = %d, want 1", count) } } + +func TestRecordProviderRequestKeepsUserVisibleSpeechAndNetworkErrors(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:provider-visible-error-log?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.SystemSetting{}, &model.Asset{}, &model.CanvasProject{}, &model.Task{}, &model.TaskLog{}, &model.Result{}, &model.ApiCallLog{}, &model.TaskTextDelta{}, &model.ModelPricing{}); err != nil { + t.Fatal(err) + } + service := &Service{repo: repository.New(db)} + speechBody := []byte(`{"code":0,"data":"AAAA"}{"header":{"code":45000030,"message":"[resource_id=volc.seedtts.default] requested resource not granted"}}`) + recordLoggedProviderRequest(t, service, "task-speech", http.StatusOK, speechBody, nil) + recordLoggedProviderRequest(t, service, "task-network", 0, nil, errors.New(`dial tcp 10.0.0.1:443: connect: connection refused`)) + recordLoggedProviderRequest(t, service, "task-gateway", http.StatusBadGateway, []byte(`Bad Gateway`), providerHTTPError{StatusCode: http.StatusBadGateway, Status: "502 Bad Gateway", Body: "Bad Gateway"}) + + assertLoggedTaskError(t, db, "task-speech", "语音合成服务未开通") + assertLoggedTaskError(t, db, "task-network", "网络异常。", "connection refused") + assertLoggedTaskError(t, db, "task-gateway", "网络异常。", "502") +} + +func assertLoggedTaskError(t *testing.T, db *gorm.DB, taskID string, parts ...string) { + t.Helper() + var log model.ApiCallLog + if err := db.Where("task_id = ?", taskID).First(&log).Error; err != nil { + t.Fatal(err) + } + if log.Status != model.ApiCallStatusFailed { + t.Fatalf("%s status = %q, want failed; error %q", taskID, log.Status, log.Error) + } + for _, part := range parts { + if !strings.Contains(log.Error, part) { + t.Fatalf("%s error %q does not contain %q", taskID, log.Error, part) + } + } +} + +func TestEnsureFailedProviderAttemptLoggedRecordsFailureHiddenBySuccessLog(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:provider-success-hides-failure?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.SystemSetting{}, &model.Asset{}, &model.CanvasProject{}, &model.Task{}, &model.TaskLog{}, &model.Result{}, &model.ApiCallLog{}, &model.TaskTextDelta{}, &model.ModelPricing{}); err != nil { + t.Fatal(err) + } + success := model.ApiCallLog{ID: "api-log-success", TaskID: "task-1", Status: model.ApiCallStatusSucceeded, RequestKind: "create"} + hidden := model.ApiCallLog{ID: "api-log-poll", TaskID: "task-1", Status: model.ApiCallStatusFailed, RequestKind: "poll", Error: "poll failed"} + if err := db.Create(&success).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&hidden).Error; err != nil { + t.Fatal(err) + } + service := &Service{repo: repository.New(db)} + service.ensureFailedProviderAttemptLogged(model.Task{ID: "task-1", UserID: "user-1", Type: "canvas_audio", Model: "seed-tts-2.0"}, errors.New("流式音频合成失败（code 45000030）：[resource_id=volc.seedtts.default] requested resource not granted")) + service.ensureFailedProviderAttemptLogged(model.Task{ID: "task-1", UserID: "user-1", Type: "canvas_audio", Model: "seed-tts-2.0"}, errors.New("流式音频合成失败（code 45000030）：[resource_id=volc.seedtts.default] requested resource not granted")) + + var logs []model.ApiCallLog + if err := db.Where("request_kind <> ?", "poll").Order("created_at asc").Find(&logs).Error; err != nil { + t.Fatal(err) + } + if len(logs) != 2 { + t.Fatalf("visible log count = %d, want success plus one failure", len(logs)) + } + if logs[1].Status != model.ApiCallStatusFailed || !strings.Contains(logs[1].Error, "语音合成服务未开通") { + t.Fatalf("visible failure = status %q error %q", logs[1].Status, logs[1].Error) + } +} + +func recordLoggedProviderRequest(t *testing.T, service *Service, taskID string, statusCode int, body []byte, requestErr error) { + t.Helper() + request, err := http.NewRequest(http.MethodPost, "https://openspeech.bytedance.com/api/v3/tts/unidirectional", strings.NewReader(`{"text":"你好"}`)) + if err != nil { + t.Fatal(err) + } + request = request.WithContext(context.WithValue(request.Context(), providerAnalyticsKey{}, providerAnalyticsContext{ + Service: service, UserID: "user-1", TaskID: taskID, ChannelID: "channel-1", Capability: "audio", Model: "seed-tts-2.0", RequestKind: "create", + })) + recordProviderRequest(request, time.Now(), statusCode, body, requestErr) +} diff --git a/backend/internal/app/task_billing.go b/backend/internal/app/task_billing.go index d6b359041..62ce0d2d8 100644 --- a/backend/internal/app/task_billing.go +++ b/backend/internal/app/task_billing.go @@ -50,6 +50,25 @@ func (c *taskBillingCoordinator) SettleBilling(orderID string, providerRequestID return c.repo.SettleBillingOrder(orderID, providerRequestID) } +func (c *taskBillingCoordinator) BillingOrder(orderID string) (*model.BillingOrder, error) { + if orderID == "" { + return nil, nil + } + return c.repo.BillingOrder(orderID) +} + +func (c *taskBillingCoordinator) SettleBillingWithAudioDuration(orderID string, providerRequestID string, durationMs int64) error { + if orderID == "" { + return nil + } + if repo, ok := c.repo.(interface { + SettleBillingOrderWithAudioDuration(string, string, int64) error + }); ok { + return repo.SettleBillingOrderWithAudioDuration(orderID, providerRequestID, durationMs) + } + return c.repo.SettleBillingOrder(orderID, providerRequestID) +} + func (c *taskBillingCoordinator) RestoreRefundedBilling(orderID string, providerRequestID string) error { if orderID == "" { return nil diff --git a/backend/internal/app/task_media_assets.go b/backend/internal/app/task_media_assets.go index 59462e4ff..81c6173b1 100644 --- a/backend/internal/app/task_media_assets.go +++ b/backend/internal/app/task_media_assets.go @@ -40,6 +40,27 @@ func (s *Service) mediaTaskProject(task *model.Task) (string, error) { return project.ID, nil } +// AudioOutputDurationMs returns the measured duration of the first persisted +// audio output. A missing or unmeasured duration is returned as an error so +// per-second billing can enter review instead of silently undercharging. +func (s *Service) AudioOutputDurationMs(task model.Task) (int64, error) { + if capabilityFromTaskType(task.Type) != "audio" { + return 0, nil + } + resourceID, _ := taskOutputResource(task.ResultJSON, task.Type) + if resourceID == "" { + return 0, errors.New("音频结果缺少已保存的资源") + } + resource, err := s.repo.ResourceForUser(task.UserID, resourceID) + if err != nil { + return 0, err + } + if resource.DurationMs <= 0 { + return 0, errors.New("音频结果缺少有效时长") + } + return resource.DurationMs, nil +} + // Uses the same effect key and ID as browser materialization, so reconnecting // the canvas cannot create a second asset for a server-delivered output. func (s *Service) registerRecoveredMediaAssets(task model.Task) error { diff --git a/backend/internal/app/task_media_storage.go b/backend/internal/app/task_media_storage.go index 2d5f36717..df253be79 100644 --- a/backend/internal/app/task_media_storage.go +++ b/backend/internal/app/task_media_storage.go @@ -8,9 +8,12 @@ import ( "fmt" "image" "io" + "math" "net/http" "os" + "os/exec" "path/filepath" + "strconv" "strings" "sync" "time" @@ -215,6 +218,7 @@ func (s *Service) storeTaskMediaFile(task *model.Task, index int, path, mimeType return nil, errors.New("暂存作品不是普通文件") } width, height := 0, 0 + durationMs := int64(0) if kind == "image" { config, _, err := image.DecodeConfig(file) if err != nil { @@ -233,6 +237,11 @@ func (s *Service) storeTaskMediaFile(task *model.Task, index int, path, mimeType if _, err := file.Seek(0, io.SeekStart); err != nil { return nil, err } + } else if kind == "audio" { + // Audio duration is the source of truth for per-second settlement. The + // media can still be saved when ffprobe is unavailable; settlement then + // moves the order to review instead of silently charging a fake length. + durationMs = probeMediaDurationMs(path) } day, err := s.reserveGeneratedResourceQuota(task.UserID, info.Size()) if err != nil { @@ -246,13 +255,14 @@ func (s *Service) storeTaskMediaFile(task *model.Task, index int, path, mimeType }() var resource *model.Resource if existing == nil { - resource, _, err = s.storeResourceWithWriter(task.UserID, kind, "generated."+extensionFromMimeType(mimeType), mimeType, info.Size(), width, height, 0, file, mediaUploadKey(task.ID, index), false, s.storeTaskMediaObject) + resource, _, err = s.storeResourceWithWriter(task.UserID, kind, "generated."+extensionFromMimeType(mimeType), mimeType, info.Size(), width, height, durationMs, file, mediaUploadKey(task.ID, index), false, s.storeTaskMediaObject) } else { // Reuse the same object key after a failed upload; no orphan per retry. resource = existing if resource.UserID != task.UserID || resource.Size != info.Size() || resource.MimeType != mimeType { return nil, errors.New("恢复文件与原始资源不一致") } + resource.DurationMs = durationMs resource.ETag, err = s.storeTaskMediaObject(resource, "generated."+extensionFromMimeType(mimeType), file) if err == nil { resource.Status, resource.Error, resource.UpdatedAt = model.ResourceStatusReady, "", time.Now() @@ -271,6 +281,24 @@ func (s *Service) storeTaskMediaFile(task *model.Task, index int, path, mimeType return resource, nil } +func probeMediaDurationMs(path string) int64 { + if strings.TrimSpace(path) == "" { + return 0 + } + if _, err := exec.LookPath("ffprobe"); err != nil { + return 0 + } + output, err := exec.Command("ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path).Output() + if err != nil { + return 0 + } + seconds, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64) + if err != nil || seconds <= 0 || math.IsNaN(seconds) || math.IsInf(seconds, 0) { + return 0 + } + return int64(math.Ceil(seconds * 1000)) +} + // Unlike ordinary uploads, generated output preserves an OSS failure for // explicit recovery instead of silently changing the storage destination. func (s *Service) storeTaskMediaObject(resource *model.Resource, _ string, body io.Reader) (string, error) { diff --git a/backend/internal/app/task_terminal.go b/backend/internal/app/task_terminal.go index c335cb4c9..9df0400fb 100644 --- a/backend/internal/app/task_terminal.go +++ b/backend/internal/app/task_terminal.go @@ -48,6 +48,18 @@ type taskOutputLifecycle interface { RegisterTaskOutputFromTask(task model.Task) error } +type audioDurationTaskBilling interface { + SettleBillingWithAudioDuration(orderID string, providerRequestID string, durationMs int64) error +} + +type billingOrderReader interface { + BillingOrder(orderID string) (*model.BillingOrder, error) +} + +type audioDurationTaskOutput interface { + AudioOutputDurationMs(task model.Task) (int64, error) +} + func newTaskTerminalCoordinator(s *Service) *taskTerminalCoordinator { return &taskTerminalCoordinator{ repo: s.repo, @@ -225,7 +237,36 @@ func (c *taskTerminalCoordinator) handleSuccess(task *model.Task) error { completionErr = fmt.Errorf("任务成功后的项目产物登记失败：%w", registerErr) } } - if err := c.billing.SettleBilling(task.BillingOrderID, ""); err != nil { + settleBilling := func() error { + if audioBilling, ok := c.billing.(audioDurationTaskBilling); ok && capabilityFromTaskType(task.Type) == "audio" { + orderReader, readerOK := c.billing.(billingOrderReader) + if readerOK { + order, err := orderReader.BillingOrder(task.BillingOrderID) + if err != nil { + return err + } + // Only audio orders priced per second need measured output duration. + // Fixed-request audio must retain the ordinary settlement path even + // when the stored resource has no duration metadata. + if order == nil || order.Capability != "audio" || order.BillingMode != "per_second" { + return c.billing.SettleBilling(task.BillingOrderID, "") + } + } + output, outputOK := c.outputs.(audioDurationTaskOutput) + if outputOK { + if completedTask == nil { + return errors.New("任务成功后无法读取音频产物") + } + durationMs, err := output.AudioOutputDurationMs(*completedTask) + if err != nil { + return err + } + return audioBilling.SettleBillingWithAudioDuration(task.BillingOrderID, "", durationMs) + } + } + return c.billing.SettleBilling(task.BillingOrderID, "") + } + if err := settleBilling(); err != nil { uncertainErr := c.billing.MarkBillingUncertain(task.BillingOrderID, "生成成功但积分结算失败："+err.Error()) _ = c.logger.log(task.UserID, task.ID, "error", "积分结算失败，已进入待核对", err.Error()) completionErr = errors.Join(completionErr, fmt.Errorf("积分结算失败：%w", err)) diff --git a/backend/internal/app/task_worker.go b/backend/internal/app/task_worker.go index cc28ca233..5a1465bc7 100644 --- a/backend/internal/app/task_worker.go +++ b/backend/internal/app/task_worker.go @@ -317,7 +317,11 @@ func taskFailureMessage(err error) string { if err == nil { return "任务处理失败" } - return truncateRunes(err.Error(), 2_000) + message := err.Error() + if denied := speechResourceDeniedUserMessage(message); denied != "" { + return denied + } + return truncateRunes(message, 2_000) } // taskLeaseRenewContext 给续租单独一份"不继承父 context 取消/时限"的上下文（仅 5 秒上限）。 diff --git a/backend/internal/assets/access.go b/backend/internal/assets/access.go index f382a349f..bd4212413 100644 --- a/backend/internal/assets/access.go +++ b/backend/internal/assets/access.go @@ -21,10 +21,12 @@ const ( PurposeProvider AccessPurpose = "provider-input" VariantOriginal ResourceVariant = "original" VariantPlayback ResourceVariant = "playback" - DeliveryCDN DeliveryMode = "cdn" - DeliveryOrigin DeliveryMode = "origin" - DeliveryLocal DeliveryMode = "platform-local" - DeliveryProxy DeliveryMode = "platform-proxy" + // VariantThumbnail 是画布展示档位；缩略图未生成前回落原件，保证请求不会因变体失败而退回平台重定向。 + VariantThumbnail ResourceVariant = "thumbnail" + DeliveryCDN DeliveryMode = "cdn" + DeliveryOrigin DeliveryMode = "origin" + DeliveryLocal DeliveryMode = "platform-local" + DeliveryProxy DeliveryMode = "platform-proxy" ) // AccessOptions contains use intent, never a client-controlled authorization or transport override. @@ -76,7 +78,7 @@ func NormalizeAccessOptions(options AccessOptions, allowProvider bool) (AccessOp default: return options, kernel.BadAuthRequest("资源访问用途无效") } - if options.Variant != VariantOriginal && options.Variant != VariantPlayback { + if options.Variant != VariantOriginal && options.Variant != VariantPlayback && options.Variant != VariantThumbnail { return options, kernel.BadAuthRequest("资源变体无效") } if options.Purpose == PurposeCopy || options.Purpose == PurposeDownload || options.Purpose == PurposeProvider { @@ -108,7 +110,16 @@ func ResolveAccess(resource *model.Resource, setting storage.Settings, options A if options.Purpose == PurposeProvider { ttl = runtime.ProviderAccessURLTTL } + // 浏览器读取的签名地址按固定时间窗对齐：窗口内同一对象的签名地址逐字节相同， + // 刷新页面或多实例签发都能命中浏览器 HTTP 缓存，不会重复从对象存储拉取字节。 + signedAt := time.Time{} + if browserCacheable(options.Purpose) { + signedAt = alignedSigningTime(now, ttl) + } expires := now.Add(ttl) + if !signedAt.IsZero() { + expires = signedAt.Add(ttl) + } if !options.ExpiresAt.IsZero() && options.ExpiresAt.Before(expires) { expires = options.ExpiresAt } @@ -124,6 +135,9 @@ func ResolveAccess(resource *model.Resource, setting storage.Settings, options A access.FallbackReason = "playback_not_ready" } } + if options.Variant == VariantThumbnail { + access.FallbackReason = "thumbnail_not_ready" + } if resource.Provider == "local" { access.Delivery = DeliveryLocal } else if options.Purpose == PurposeDownload { @@ -145,7 +159,11 @@ func ResolveAccess(resource *model.Resource, setting storage.Settings, options A } else if setting.Delivery.RequireCDN { return nil, AccessError(503, "resource_cdn_unconfigured", "CDN 访问鉴权未配置，请检查存储分发设置") } else if storage.PublicOrigin(setting) { - access.URL, err = storage.SignedOriginObjectURL(setting, resource.ObjectKey, expires) + if signedAt.IsZero() { + access.URL, err = storage.SignedOriginObjectURL(setting, resource.ObjectKey, expires) + } else { + access.URL, err = storage.SignedOriginObjectDisplayURL(setting, resource.ObjectKey, signedAt, expires) + } access.Delivery = DeliveryOrigin if setting.CDNBaseURL != "" { access.FallbackReason = "cdn_auth_unconfigured" @@ -172,3 +190,20 @@ func ResolveAccess(resource *model.Resource, setting storage.Settings, options A } return access, nil } + +// browserCacheable 标记浏览器直接读取字节、值得复用 HTTP 缓存的用途。 +// 下载地址携带一次性的 Content-Disposition，模型输入由上游服务读取，二者仍按当前时间签发。 +func browserCacheable(purpose AccessPurpose) bool { + return purpose == PurposeDisplay || purpose == PurposeProcess || purpose == PurposeCopy +} + +// alignedSigningTime 把签名起始时间对齐到 ttl/4 的时间窗（按 Unix 纪元对齐，多实例结果一致）。 +// 地址过期时间为「窗口起点 + ttl」，任何时刻签出的地址至少还剩 3/4 有效期，且总有效期不超过后台配置的 ttl。 +func alignedSigningTime(now time.Time, ttl time.Duration) time.Time { + window := ttl / 4 + if window < time.Second { + return now.UTC() + } + unix := now.UTC().UnixNano() + return time.Unix(0, unix-unix%int64(window)).UTC() +} diff --git a/backend/internal/assets/access_test.go b/backend/internal/assets/access_test.go index 912412a1c..ba3b08711 100644 --- a/backend/internal/assets/access_test.go +++ b/backend/internal/assets/access_test.go @@ -191,13 +191,77 @@ func TestResolveAccessUsesRuntimePolicyTTL(t *testing.T) { resource := testReadyResource("local") setting := storage.Settings{Runtime: storage.RuntimePolicy{AccessURLTTL: 2 * time.Hour, ProviderAccessURLTTL: 6 * time.Hour}} var variants []ResourceVariant - for purpose, want := range map[AccessPurpose]time.Duration{PurposeDisplay: 2 * time.Hour, PurposeProvider: 6 * time.Hour} { + // 展示地址按 ttl/4（30 分钟）对齐签名窗口：03:04:05 落在 03:00 窗口，过期于 05:00；模型输入仍按当前时间签发。 + for purpose, want := range map[AccessPurpose]time.Time{PurposeDisplay: time.Date(2026, time.January, 2, 5, 0, 0, 0, time.UTC), PurposeProvider: now.Add(6 * time.Hour)} { access, err := ResolveAccess(resource, setting, AccessOptions{Purpose: purpose}, now, testPlatformURL(&variants)) if err != nil { t.Fatal(err) } - if access.ExpiresAt == nil || !access.ExpiresAt.Equal(now.Add(want)) { - t.Fatalf("%s access expiry = %v, want %v", purpose, access.ExpiresAt, now.Add(want)) + if access.ExpiresAt == nil || !access.ExpiresAt.Equal(want) { + t.Fatalf("%s access expiry = %v, want %v", purpose, access.ExpiresAt, want) } } } + +func TestResolveDisplayAccessIsStableWithinSigningWindow(t *testing.T) { + setting := storage.Settings{ + Provider: "aliyun", Endpoint: "https://1.1.1.1", Bucket: "private-bucket", + AccessKeyID: "access-id", AccessKeySecret: "secret-value", + Runtime: storage.RuntimePolicy{AccessURLTTL: 24 * time.Hour}, + } + resource := testReadyResource("aliyun") + first, err := ResolveAccess(resource, setting, AccessOptions{Purpose: PurposeDisplay}, time.Date(2026, time.September, 30, 18, 0, 1, 0, time.UTC), testPlatformURL(&[]ResourceVariant{})) + if err != nil { + t.Fatal(err) + } + // 刷新画布（同一 6 小时窗口内再次签发）必须拿到逐字节相同的地址，浏览器才能复用 HTTP 缓存。 + again, err := ResolveAccess(resource, setting, AccessOptions{Purpose: PurposeDisplay}, time.Date(2026, time.September, 30, 23, 59, 0, 0, time.UTC), testPlatformURL(&[]ResourceVariant{})) + if err != nil { + t.Fatal(err) + } + if first.Delivery != DeliveryOrigin || first.URL != again.URL { + t.Fatalf("display URL changed within one signing window:\n%s\n%s", first.URL, again.URL) + } + if cacheControl := mustQuery(t, first.URL).Get("response-cache-control"); !strings.Contains(cacheControl, "max-age=86400") || !strings.Contains(cacheControl, "immutable") { + t.Fatalf("display URL cache-control = %q", cacheControl) + } + next, err := ResolveAccess(resource, setting, AccessOptions{Purpose: PurposeDisplay}, time.Date(2026, time.October, 1, 0, 0, 1, 0, time.UTC), testPlatformURL(&[]ResourceVariant{})) + if err != nil { + t.Fatal(err) + } + if next.URL == first.URL { + t.Fatal("next signing window must rotate the display URL") + } + // 下载地址携带一次性 Content-Disposition，不做对齐也不附加长缓存。 + download, err := ResolveAccess(resource, setting, AccessOptions{Purpose: PurposeDownload}, time.Now(), testPlatformURL(&[]ResourceVariant{})) + if err != nil { + t.Fatal(err) + } + if mustQuery(t, download.URL).Get("response-cache-control") != "" { + t.Fatalf("download URL must not carry display cache override: %s", download.URL) + } +} + +func TestResolveAccessAcceptsThumbnailVariantWithOriginalFallback(t *testing.T) { + now := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + var variants []ResourceVariant + access, err := ResolveAccess(testReadyResource("local"), storage.Settings{}, AccessOptions{Purpose: PurposeDisplay, Variant: VariantThumbnail}, now, testPlatformURL(&variants)) + if err != nil { + t.Fatalf("thumbnail variant must not fail (the canvas would fall back to the redirecting platform URL): %v", err) + } + if access.RequestedVariant != VariantThumbnail || access.ActualVariant != VariantOriginal || access.FallbackReason != "thumbnail_not_ready" { + t.Fatalf("thumbnail access = %#v", access) + } + if len(variants) != 1 || variants[0] != VariantOriginal { + t.Fatalf("platform URL variants = %#v, want original", variants) + } +} + +func mustQuery(t *testing.T, raw string) url.Values { + t.Helper() + parsed, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return parsed.Query() +} diff --git a/backend/internal/canvas/capability/builtin.go b/backend/internal/canvas/capability/builtin.go index a633d6d0c..e9b5b2bfc 100644 --- a/backend/internal/canvas/capability/builtin.go +++ b/backend/internal/canvas/capability/builtin.go @@ -33,14 +33,15 @@ func BuiltinRegistry() *Registry { PatchFields: editableNodeFields("metadata.content", "Markdown 正文", "Markdown 正文"), }, generatedMediaDescriptor("image", "2", "图片", 720, 405, "image", ConnectionPolicy{ - CanSource: true, CanTarget: true, CanReference: true, AcceptedInputKinds: []string{"text", "image"}, + CanSource: true, CanTarget: true, CanReference: true, AcceptedInputKinds: []string{"text", "image", "character"}, }), generatedMediaDescriptor("video", "2", "视频", 720, 405, "video", ConnectionPolicy{ - CanSource: true, CanTarget: true, CanReference: true, AcceptedInputKinds: []string{"text", "image", "video", "audio"}, + CanSource: true, CanTarget: true, CanReference: true, AcceptedInputKinds: []string{"text", "image", "video", "audio", "character"}, }), generatedMediaDescriptor("audio", "2", "音频", 340, 120, "audio", ConnectionPolicy{ - CanSource: true, CanTarget: true, CanReference: true, MaxInputCount: 1, AcceptedInputKinds: []string{"text"}, + CanSource: true, CanTarget: true, CanReference: true, MaxInputCount: 1, AcceptedInputKinds: []string{"text", "character"}, }), + characterDescriptor(), { Type: "frame", Version: "1", Label: "背板", DefaultWidth: 760, DefaultHeight: 520, Purpose: "在画布上建立可移动、可折叠的视觉分区，用来归组相关节点；背板本身不承载创作正文或生成结果。", @@ -105,6 +106,25 @@ func BuiltinRegistry() *Registry { return registry } +// characterDescriptor 是角色卡能力：画布上是 text + metadata.workflowKind=character 的节点， +// 设定、三视图与声音都来自账号内的角色资产而非节点正文。它可以被发现、精读、连线和引用， +// 但不能用 add_node 凭空创建：必须经 canvas_create_character 用真实图片/音频建角色资产，名称和设定只随角色资产更新。 +func characterDescriptor() Descriptor { + return Descriptor{ + Type: "character", Version: "1", Label: "角色卡", DefaultWidth: 264, DefaultHeight: 352, + Purpose: "引用账号角色库中的角色资产，统一提供角色设定、三视图/形象图和绑定声音，用来保持人物在多次生成中的一致性。", + GoodFor: []string{"图片或视频生成时锁定人物外观", "多镜头保持同一角色一致", "角色配音时使用绑定声音", "只取角色文字设定作为提示词来源"}, + NotIdealFor: []string{"临时一次性的人物描述（直接写进提示词或文本节点）", "用 add_node 新建（应使用 canvas_create_character 打包形象图片与声音）", "承载普通正文（节点正文不是角色设定）"}, + Tradeoffs: []string{"设定与媒体随角色资产版本变化，提交时会校验版本", "未绑定形象或声音时对应引用不可用，需先读取 character.imageReference/audioReference"}, + Actions: []string{"read_character", "use_as_reference", "use_as_text_source"}, + InputKind: "character", + Connection: ConnectionPolicy{CanSource: true, CanReference: true}, + CanUpdate: true, + PatchFields: positionPatchFields(), + Variant: &NodeVariant{BaseType: "text", WorkflowKind: "character"}, + } +} + func generatedMediaDescriptor(nodeType, version, label string, width, height float64, generationMode string, connection ConnectionPolicy) Descriptor { semantics := generatedMediaSemantics(nodeType) return Descriptor{ diff --git a/backend/internal/canvas/capability/descriptor.go b/backend/internal/canvas/capability/descriptor.go index 727c31c7e..20dc8ab16 100644 --- a/backend/internal/canvas/capability/descriptor.go +++ b/backend/internal/canvas/capability/descriptor.go @@ -34,6 +34,14 @@ type PatchField struct { MaxRunes int } +// NodeVariant 表示同一底层节点类型按 metadata.workflowKind 标记出的独立能力， +// 例如 text + workflowKind=character 的角色卡。变体可被发现、读取、引用和连线， +// 但不是可直接创建的节点类型：add_node 的 nodeType 枚举和 Resolve 都不会返回变体。 +type NodeVariant struct { + BaseType string + WorkflowKind string +} + // Descriptor is the server-owned canvas contract. Agent tools, creation, // persistence and state projection all consume this registry; no caller owns a // second node allow-list. @@ -58,6 +66,7 @@ type Descriptor struct { ProjectionField string PatchFields map[string]PatchField CreateMetadata func(content string) map[string]any + Variant *NodeVariant } func (d Descriptor) Metadata(content string) map[string]any { @@ -170,6 +179,8 @@ func inputKindLabel(kind string) string { return "视频" case "audio": return "音频" + case "character": + return "角色卡" default: return "文本" } diff --git a/backend/internal/canvas/capability/registry.go b/backend/internal/canvas/capability/registry.go index 7591802ae..f7cf1663b 100644 --- a/backend/internal/canvas/capability/registry.go +++ b/backend/internal/canvas/capability/registry.go @@ -9,12 +9,20 @@ import ( "strings" ) -const SetVersion = "canvas-capabilities/v4" +const SetVersion = "canvas-capabilities/v5" -type Registry struct{ descriptors map[string]Descriptor } +type Registry struct { + descriptors map[string]Descriptor + // variants 按 baseType+workflowKind 索引，只能通过 ResolveNode 命中， + // 不会出现在 Resolve/Types 中，因此不会变成可 add_node 的节点类型。 + variants map[string]string +} + +func variantKey(baseType, workflowKind string) string { return baseType + "\x00" + workflowKind } func NewRegistry(descriptors []Descriptor) (*Registry, error) { items := make(map[string]Descriptor, len(descriptors)) + variants := map[string]string{} generationModes := make(map[string]string, len(descriptors)) for _, descriptor := range descriptors { descriptor.Type = normalizeType(descriptor.Type) @@ -95,9 +103,30 @@ func NewRegistry(descriptors []Descriptor) (*Registry, error) { if _, exists := items[descriptor.Type]; exists { return nil, fmt.Errorf("duplicate canvas capability %q", descriptor.Type) } + if descriptor.Variant != nil { + variant := &NodeVariant{BaseType: normalizeType(descriptor.Variant.BaseType), WorkflowKind: strings.TrimSpace(descriptor.Variant.WorkflowKind)} + descriptor.Variant = variant + if variant.BaseType == "" || variant.WorkflowKind == "" || variant.BaseType == descriptor.Type { + return nil, fmt.Errorf("capability %q has invalid node variant", descriptor.Type) + } + if descriptor.GenerationMode != "" || descriptor.CreateMetadata != nil { + return nil, fmt.Errorf("capability variant %q cannot be created or generated directly", descriptor.Type) + } + key := variantKey(variant.BaseType, variant.WorkflowKind) + if existing, exists := variants[key]; exists { + return nil, fmt.Errorf("node variant %s/%s is registered by both %q and %q", variant.BaseType, variant.WorkflowKind, existing, descriptor.Type) + } + variants[key] = descriptor.Type + } items[descriptor.Type] = cloneDescriptor(descriptor) } - return &Registry{descriptors: items}, nil + for key, variantType := range variants { + base := items[variantType].Variant.BaseType + if baseDescriptor, ok := items[base]; !ok || baseDescriptor.Variant != nil { + return nil, fmt.Errorf("capability variant %q extends unknown base type %q", variantType, strings.SplitN(key, "\x00", 2)[0]) + } + } + return &Registry{descriptors: items, variants: variants}, nil } func (r *Registry) Resolve(nodeType string) (Descriptor, bool) { @@ -105,12 +134,24 @@ func (r *Registry) Resolve(nodeType string) (Descriptor, bool) { return Descriptor{}, false } d, ok := r.descriptors[normalizeType(nodeType)] - if !ok { + if !ok || d.Variant != nil { return Descriptor{}, false } d = cloneDescriptor(d) return d, ok } + +// ResolveNode 按真实画布节点解析能力：先匹配 type+metadata.workflowKind 变体（如角色卡）， +// 否则回落到底层类型。读取、连线、引用等拿到节点本身的调用方都应走这里。 +func (r *Registry) ResolveNode(nodeType, workflowKind string) (Descriptor, bool) { + if r == nil { + return Descriptor{}, false + } + if variantType, ok := r.variants[variantKey(normalizeType(nodeType), strings.TrimSpace(workflowKind))]; ok { + return cloneDescriptor(r.descriptors[variantType]), true + } + return r.Resolve(nodeType) +} func (r *Registry) List() []Descriptor { if r == nil { return nil @@ -122,11 +163,15 @@ func (r *Registry) List() []Descriptor { sort.Slice(out, func(i, j int) bool { return out[i].Type < out[j].Type }) return out } + +// Types 只返回可直接创建的底层节点类型；变体通过 List 被发现，但不能作为 add_node 的 nodeType。 func (r *Registry) Types() []string { items := r.List() out := make([]string, 0, len(items)) for _, d := range items { - out = append(out, d.Type) + if d.Variant == nil { + out = append(out, d.Type) + } } return out } @@ -199,6 +244,10 @@ func cloneDescriptor(descriptor Descriptor) Descriptor { patchFields[key] = field } descriptor.PatchFields = patchFields + if descriptor.Variant != nil { + variant := *descriptor.Variant + descriptor.Variant = &variant + } return descriptor } @@ -222,6 +271,7 @@ type hashDescriptor struct { ProjectionKind string ProjectionField string PatchFields []hashPatchField + Variant *NodeVariant } type hashConnectionPolicy struct { @@ -265,6 +315,7 @@ func registryHashItems(descriptors []Descriptor) []hashDescriptor { ProjectionKind: descriptor.ProjectionKind, ProjectionField: descriptor.ProjectionField, Connection: hashConnectionPolicy{CanSource: descriptor.Connection.CanSource, CanTarget: descriptor.Connection.CanTarget, CanReference: descriptor.Connection.CanReference, AcceptedInputKinds: descriptor.Connection.AcceptedInputKinds, RejectedInputKinds: descriptor.Connection.RejectedInputKinds, MaxInputCount: descriptor.Connection.MaxInputCount}, CanUpdate: descriptor.CanUpdate, SummaryFields: descriptor.SummaryFields, DetailFields: descriptor.DetailFields, PatchFields: patchFields, + Variant: descriptor.Variant, }) } return items diff --git a/backend/internal/canvas/contract/generation.go b/backend/internal/canvas/contract/generation.go index 7ab0b45d1..b245f8e24 100644 --- a/backend/internal/canvas/contract/generation.go +++ b/backend/internal/canvas/contract/generation.go @@ -43,6 +43,8 @@ type Options struct { AudioVoice *string `json:"audioVoice,omitempty" node:"audioVoice" task:"audioVoice" modes:"audio"` AudioFormat *string `json:"audioFormat,omitempty" node:"audioFormat" task:"audioFormat" modes:"audio"` AudioSpeed *string `json:"audioSpeed,omitempty" node:"audioSpeed" task:"audioSpeed" modes:"audio"` + AudioLanguage *string `json:"audioLanguage,omitempty" node:"audioLanguage" task:"audioLanguage" modes:"audio"` + AudioDialect *string `json:"audioDialect,omitempty" node:"audioDialect" task:"audioDialect" modes:"audio"` AudioInstructions *string `json:"audioInstructions,omitempty" node:"audioInstructions" task:"audioInstructions" modes:"audio"` } diff --git a/backend/internal/generation/registry.go b/backend/internal/generation/registry.go index 451639aef..707bc3221 100644 --- a/backend/internal/generation/registry.go +++ b/backend/internal/generation/registry.go @@ -104,6 +104,8 @@ func OfficialDeclarativeAudioInterface(interfaceType string) (string, bool) { switch strings.TrimSpace(interfaceType) { case string(model.ChannelInterfaceOpenAIAudio): return "OpenAI Audio", true + case string(model.ChannelInterfaceDoubaoStreamingTTS): + return "豆包流式语音", true case string(model.ChannelInterfaceAsyncAudio): return "异步音频", true default: diff --git a/backend/internal/handler/admin_api_log_routes.go b/backend/internal/handler/admin_api_log_routes.go index 449cbc108..149f83869 100644 --- a/backend/internal/handler/admin_api_log_routes.go +++ b/backend/internal/handler/admin_api_log_routes.go @@ -68,7 +68,57 @@ func registerAdminAPILogRoutes(r *gin.RouterGroup, svc *service.Service) { failService(c, err) return } - result, err := svc.AdminQueryFailedVideoTask(c.Request.Context(), user, c.Param("id")) + var req struct { + ProviderRequestID string `json:"providerRequestId"` + } + if c.Request.ContentLength != 0 { + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + } + result, err := svc.AdminQueryFailedVideoTask(c.Request.Context(), user, c.Param("id"), req.ProviderRequestID) + if err != nil { + failService(c, err) + return + } + ok(c, result) + }) + r.POST("/admin/api-logs/:id/recover-url", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + var req struct { + URL string `json:"url"` + ProviderRequestID string `json:"providerRequestId"` + } + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + result, err := svc.AdminRecoverVideoByURL(c.Request.Context(), user, c.Param("id"), req.URL, req.ProviderRequestID) + if err != nil { + failService(c, err) + return + } + ok(c, result) + }) + r.POST("/admin/api-logs/recover-batch", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + var req struct { + IDs []string `json:"ids"` + } + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + result, err := svc.AdminBatchQueryFailedVideoTasks(c.Request.Context(), user, req.IDs) if err != nil { failService(c, err) return diff --git a/backend/internal/handler/project_asset_routes.go b/backend/internal/handler/project_asset_routes.go index c15c11938..52c7bea63 100644 --- a/backend/internal/handler/project_asset_routes.go +++ b/backend/internal/handler/project_asset_routes.go @@ -4,6 +4,7 @@ package handler import ( "net/http" + "strings" "infinite-canvas/backend/internal/service" @@ -128,6 +129,136 @@ func registerProjectAssetRoutes(r *gin.RouterGroup, svc *service.Service) { } ok(c, gin.H{"id": c.Param("folderId")}) }) + r.GET("/characters", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + page, pageSize, parseErr := parsePaginationQuery(c, 12) + if parseErr != nil { + fail(c, http.StatusBadRequest, parseErr) + return + } + if pageSize > 48 { + pageSize = 48 + } + ids := []string{} + for _, id := range strings.Split(c.Query("ids"), ",") { + if id = strings.TrimSpace(id); id != "" { + ids = append(ids, id) + } + } + characters, err := svc.ListCharacters(user.ID, c.Query("q"), page, pageSize, ids) + if err != nil { + failService(c, err) + return + } + ok(c, characters) + }) + r.POST("/characters", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 256<<10) + var req service.CreateCharacterRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + character, err := svc.CreateCharacter(user.ID, req) + if err != nil { + failService(c, err) + return + } + ok(c, character) + }) + r.GET("/characters/:assetId", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + character, err := svc.Character(user.ID, c.Param("assetId")) + if err != nil { + failService(c, err) + return + } + ok(c, character) + }) + r.PATCH("/characters/:assetId", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 256<<10) + var req service.UpdateProjectCharacterRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + character, err := svc.UpdateCharacter(user.ID, c.Param("assetId"), req) + if err != nil { + failService(c, err) + return + } + ok(c, character) + }) + r.PUT("/characters/:assetId/representations", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 128<<10) + var req service.ReplaceCharacterRepresentationsRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + character, err := svc.ReplaceCharacterRepresentations(user.ID, c.Param("assetId"), req) + if err != nil { + failService(c, err) + return + } + ok(c, character) + }) + r.PUT("/characters/:assetId/voice", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10) + var req service.BindCharacterVoiceRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + character, err := svc.BindCharacterVoice(user.ID, c.Param("assetId"), req) + if err != nil { + failService(c, err) + return + } + ok(c, character) + }) + r.DELETE("/characters/:assetId/voice", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + character, err := svc.UnbindCharacterVoice(user.ID, c.Param("assetId")) + if err != nil { + failService(c, err) + return + } + ok(c, character) + }) + r.POST("/projects/:id/characters", func(c *gin.Context) { user, err := currentUser(c, svc) if err != nil { diff --git a/backend/internal/model/models.go b/backend/internal/model/models.go index d48b99e73..45bf9784d 100644 --- a/backend/internal/model/models.go +++ b/backend/internal/model/models.go @@ -68,6 +68,7 @@ const ( ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image" ChannelInterfaceGeminiImage ChannelInterfaceType = "gemini-image" ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio" + ChannelInterfaceDoubaoStreamingTTS ChannelInterfaceType = "doubao-streaming-tts" ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio" ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi" ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1" diff --git a/backend/internal/protocol/adapter.go b/backend/internal/protocol/adapter.go index 37f2d0999..a8d5874ac 100644 --- a/backend/internal/protocol/adapter.go +++ b/backend/internal/protocol/adapter.go @@ -13,6 +13,10 @@ type Adapter interface { BuildCancel(context.Context, PollContext) (RequestSpec, error) } +type RequestAwareCreateParser interface { + ParseCreateWithRequest(context.Context, GenerationRequest, []byte) (CreateResult, error) +} + // AgentAdapter is the optional protocol surface for tool-capable text calls. // The host still owns credentials, outbound policy and billing; a plugin only // maps the platform's agent request into the provider payload and parses the diff --git a/backend/internal/protocol/autodl_indextts2_test.go b/backend/internal/protocol/autodl_indextts2_test.go deleted file mode 100644 index 8a92092c6..000000000 --- a/backend/internal/protocol/autodl_indextts2_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package protocol - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" -) - -// IndexTTS2 的音色完全来自 prompt_simple 参考音频，缺失时上游返回“缺少必填参数”， -// 因此插件必须把画布连线的参考音频映射过去，并在本地拦截未连接音频的请求。 -func TestAutoDLIndexTTS2Workflow(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "..", "..", "plugin-packages", "autodl-comfyui.yingce-plugin")) - if err != nil { - t.Fatal(err) - } - pkg, err := ParsePluginPackage(data) - if err != nil { - t.Fatal(err) - } - adapters, err := LoadInstalledProviders(pkg.ManifestRaw, nil) - if err != nil { - t.Fatal(err) - } - var adapter Adapter - for _, a := range adapters { - if a.Metadata().ID == "autodl-comfyui-audio" { - adapter = a - } - } - if adapter == nil { - t.Fatal("autodl-comfyui-audio provider adapter not found") - } - if !adapter.Metadata().RequiresPublicMediaURLs { - t.Fatal("IndexTTS2 reference audio must be sent as public URLs") - } - - if _, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ - Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "你好，这是一段测试文本", - }}); err == nil || !strings.Contains(err.Error(), "音色参考音频") { - t.Fatalf("missing prompt_simple must be rejected locally, err = %v", err) - } - - create, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ - Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "你好，这是一段测试文本", - Audios: []MediaReference{{URL: "https://cdn.example/voice.mp3", Order: 0}}, - Extra: map[string]any{"audioVoice": "alloy"}, - }}) - if err != nil { - t.Fatal(err) - } - if create.Method != "POST" || create.Path != "/api/v1/comfyui/comfyui_workflow/indextts2-v1" { - t.Fatalf("create spec = %#v", create) - } - body := create.Body.(map[string]any) - if body["prompt_text"] != "你好，这是一段测试文本" || body["prompt_simple"] != "https://cdn.example/voice.mp3" || body["emo_control_method"] != "与音色参考音频相同" || body["emo_random"] != false { - t.Fatalf("single reference body = %#v", body) - } - for _, name := range []string{"emo_happy", "emo_angry", "emo_sad", "emo_afraid", "emo_disgusted", "emo_melancholic", "emo_surprised", "emo_calm"} { - if value, ok := body[name]; !ok || manifestFloat(value) != 0 { - t.Fatalf("%s = %#v, body = %#v", name, value, body) - } - } - for _, name := range []string{"emo_ref_audio", "voice", "prompt"} { - if _, exists := body[name]; exists { - t.Fatalf("single reference body must not send %s: %#v", name, body) - } - } - - withEmotion, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ - Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "测试", - Audios: []MediaReference{{URL: "https://cdn.example/emotion.mp3", Order: 2}, {URL: "https://cdn.example/voice.mp3", Order: 1}}, - ProviderOptions: map[string]map[string]any{"autodl-comfyui-audio": {"emo_happy": 0.5, "emo_calm": 0.3}}, - }}) - if err != nil { - t.Fatal(err) - } - emotionBody := withEmotion.Body.(map[string]any) - if emotionBody["prompt_simple"] != "https://cdn.example/voice.mp3" || emotionBody["emo_ref_audio"] != "https://cdn.example/emotion.mp3" || emotionBody["emo_control_method"] != "使用情感参考音频" || manifestFloat(emotionBody["emo_happy"]) != 0.5 || manifestFloat(emotionBody["emo_calm"]) != 0.3 { - t.Fatalf("emotion reference body = %#v", emotionBody) - } - - if _, err := adapter.BuildCreate(context.Background(), RequestContext{BaseURL: "https://autodl.art", Request: GenerationRequest{ - Capability: CapabilityAudio, Model: "indextts2-v1", Prompt: "测试", - Audios: []MediaReference{{URL: "https://cdn.example/a.mp3"}, {URL: "https://cdn.example/b.mp3", Order: 1}, {URL: "https://cdn.example/c.mp3", Order: 2}}, - }}); err == nil { - t.Fatal("more than two reference audios must be rejected") - } - - completed, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: "671ce5ca"}, []byte(`{"msg":"","code":"Success","data":{"status":"completed","results":[{"url":"https://cos.example/ComfyUI_00010_.wav","type":"audio","node_id":"9","file_type":"wav","output_type":"output"}],"task_id":"671ce5ca"}}`)) - if err != nil || completed.Status != StatusSucceeded || completed.Result == nil || len(completed.Result.Audios) != 1 || completed.Result.Audios[0].URL != "https://cos.example/ComfyUI_00010_.wav" { - t.Fatalf("completed = %#v, err = %v", completed, err) - } -} diff --git a/backend/internal/protocol/doubao_streaming_test.go b/backend/internal/protocol/doubao_streaming_test.go new file mode 100644 index 000000000..394a1adaf --- /dev/null +++ b/backend/internal/protocol/doubao_streaming_test.go @@ -0,0 +1,157 @@ +package protocol + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "strings" + "testing" +) + +func TestDoubaoStreamingTTSPackageBuildsOfficialRequest(t *testing.T) { + manifest := []byte(`{ + "apiVersion":"yingce.plugin/v2", + "id":"doubao-streaming-tts","name":"Doubao Streaming TTS","version":"2.0.0","author":"Volcengine", + "contributes":{"providers":[{"id":"doubao-streaming-tts","label":"Doubao Streaming TTS","capabilities":["audio"],"scopes":["canvas"],"auth":{"type":"header","field":"apiKey","header":"X-Api-Key"},"create":{"method":"POST","path":"/api/v3/tts/unidirectional","contentType":"application/json","body":{"req_params":{"$json":{"text":{"$ref":"request.prompt"},"speaker":{"$ref":"request.extra.audioVoice"},"model":{"$ref":"request.model"}}}},"headers":{"X-Api-Resource-Id":{"$coalesce":[{"$ref":"request.providerOptions.doubao-streaming-tts.resourceId"},"seed-tts-2.0"]},"X-Api-Request-Id":{"$ref":"request.extra.idempotencyKey"}}},"response":{"streamedJsonAudio":true,"resultKind":"audio"}}]} + }`) + adapter, err := LoadManifest(manifest) + if err != nil { + t.Fatal(err) + } + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Capability: CapabilityAudio, + Model: "seed-tts-2.0", + Prompt: "你好", + Extra: map[string]any{"audioVoice": "speaker-1", "idempotencyKey": "request-1"}, + ProviderOptions: map[string]map[string]any{ + "doubao-streaming-tts": {"resourceId": "seed-icl-2.0"}, + }, + }}) + if err != nil { + t.Fatal(err) + } + if spec.Headers["X-Api-Resource-Id"] != "seed-icl-2.0" || spec.Headers["X-Api-Request-Id"] != "request-1" { + t.Fatalf("headers = %#v", spec.Headers) + } + body, ok := spec.Body.(map[string]any) + if !ok { + t.Fatalf("body type = %T", spec.Body) + } + params, ok := body["req_params"].(string) + if !ok { + t.Fatalf("req_params type = %T, want JSON string", body["req_params"]) + } + var decoded map[string]any + if err := json.Unmarshal([]byte(params), &decoded); err != nil { + t.Fatal(err) + } + if decoded["text"] != "你好" || decoded["speaker"] != "speaker-1" || decoded["model"] != "seed-tts-2.0" { + t.Fatalf("req_params = %#v", decoded) + } +} + +func TestStreamedJSONAudioCreateResultConcatenatesFrames(t *testing.T) { + first := base64.StdEncoding.EncodeToString([]byte("first")) + second := base64.StdEncoding.EncodeToString([]byte("second")) + body := []byte(`{"code":0,"data":"` + first + `"}{"code":0,"data":"` + second + `","usage":{"characters":6}}`) + adapter := manifestAdapter{manifest: Manifest{Response: ManifestResponse{StreamedJSONAudio: true, ResultKind: "audio"}}} + result, err := adapter.ParseCreate(context.Background(), body) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusSucceeded || result.Result == nil || len(result.Result.Audios) != 1 { + t.Fatalf("result = %#v", result) + } + if !strings.HasPrefix(result.Result.Audios[0].DataURL, "data:audio/mpeg;base64,") { + t.Fatalf("data URL = %q", result.Result.Audios[0].DataURL) + } + encoded := strings.TrimPrefix(result.Result.Audios[0].DataURL, "data:audio/mpeg;base64,") + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || string(decoded) != "firstsecond" { + t.Fatalf("audio bytes = %q, err = %v", decoded, err) + } +} + +func TestStreamedJSONAudioCreateResultRejectsBusinessError(t *testing.T) { + adapter := manifestAdapter{manifest: Manifest{Response: ManifestResponse{StreamedJSONAudio: true, ResultKind: "audio"}}} + _, err := adapter.ParseCreate(context.Background(), []byte(`{"code":30000001,"message":"invalid speaker"}`)) + if err == nil || !strings.Contains(err.Error(), "invalid speaker") { + t.Fatalf("error = %v", err) + } +} + +func TestDoubaoSeedAudioCreateUsesTextPromptWithoutResourceHeader(t *testing.T) { + manifest, err := os.ReadFile("../../../plugin-packages/doubao-streaming-tts/manifest.json") + if err != nil { + t.Fatal(err) + } + adapter, err := LoadManifest(manifest) + if err != nil { + t.Fatal(err) + } + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Capability: CapabilityAudio, Model: "seed-audio-1.0", Prompt: "你好"}}) + if err != nil { + t.Fatal(err) + } + if spec.Path != "/api/v3/tts/create" { + t.Fatalf("path = %s", spec.Path) + } + if _, ok := spec.Headers["X-Api-Resource-Id"]; ok { + t.Fatalf("seed-audio create must not send a TTS resource header: %#v", spec.Headers) + } + body := spec.Body.(map[string]any) + if body["model"] != "seed-audio-1.0" || body["text_prompt"] != "你好" || body["references"] != nil { + t.Fatalf("text-only body = %#v", body) + } +} + +func TestDoubaoSeedAudioCreateSendsReferenceAudioAndImage(t *testing.T) { + manifest, err := os.ReadFile("../../../plugin-packages/doubao-streaming-tts/manifest.json") + if err != nil { + t.Fatal(err) + } + adapter, err := LoadManifest(manifest) + if err != nil { + t.Fatal(err) + } + audioSpec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Capability: CapabilityAudio, + Model: "seed-audio-1.0", + Prompt: "@Audio1 用这个声音说你好", + Audios: []MediaReference{{URL: "https://cdn.example.com/voice.mp3", Kind: "audio"}}, + Extra: map[string]any{"audioVoice": "zh_female_qingxinnvsheng_uranus_bigtts", "audioFormat": "mp3"}, + }}) + if err != nil { + t.Fatal(err) + } + audioBody := audioSpec.Body.(map[string]any) + references, _ := audioBody["references"].([]any) + if len(references) != 2 { + t.Fatalf("audio references = %#v", audioBody["references"]) + } + reference, _ := references[0].(map[string]any) + speaker, _ := references[1].(map[string]any) + if reference["audio_url"] != "https://cdn.example.com/voice.mp3" || reference["audio_data"] != nil || speaker["speaker"] != "zh_female_qingxinnvsheng_uranus_bigtts" { + t.Fatalf("audio references = %#v", references) + } + + imageSpec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Capability: CapabilityAudio, + Model: "seed-audio-1.0", + Prompt: "根据图片内容生成音频", + Images: []MediaReference{{DataURL: "data:image/png;base64,aGVsbG8=", Kind: "image"}}, + }}) + if err != nil { + t.Fatal(err) + } + imageBody := imageSpec.Body.(map[string]any) + imageReferences, _ := imageBody["references"].([]any) + if len(imageReferences) != 1 { + t.Fatalf("image references = %#v", imageBody["references"]) + } + imageReference, _ := imageReferences[0].(map[string]any) + if imageReference["image_data"] != "aGVsbG8=" || imageReference["image_url"] != nil { + t.Fatalf("image reference = %#v", imageReference) + } +} diff --git a/backend/internal/protocol/manifest.go b/backend/internal/protocol/manifest.go index 2a09b198a..6f59fa23a 100644 --- a/backend/internal/protocol/manifest.go +++ b/backend/internal/protocol/manifest.go @@ -44,15 +44,18 @@ type ManifestResponse struct { // BinaryPayload 表示 create 同步返回二进制媒体（如 /audio/speech 的音频流）。 // 声明式解析会把整个响应体包装为对应能力的单个媒体结果，不做 JSON 路径提取。 BinaryPayload bool `json:"binaryPayload,omitempty"` - TaskID any `json:"taskId,omitempty"` - Status any `json:"status,omitempty"` - Message any `json:"message,omitempty"` - Text any `json:"text,omitempty"` - Reasoning any `json:"reasoning,omitempty"` - Images any `json:"images,omitempty"` - Videos any `json:"videos,omitempty"` - Audios any `json:"audios,omitempty"` - Usage any `json:"usage,omitempty"` + // StreamedJSONAudio 表示 create 返回由连续 JSON 对象组成的 HTTP chunked 音频流； + // 每个对象可携带 data（Base64 音频片段）、code/message 和 usage。 + StreamedJSONAudio bool `json:"streamedJsonAudio,omitempty"` + TaskID any `json:"taskId,omitempty"` + Status any `json:"status,omitempty"` + Message any `json:"message,omitempty"` + Text any `json:"text,omitempty"` + Reasoning any `json:"reasoning,omitempty"` + Images any `json:"images,omitempty"` + Videos any `json:"videos,omitempty"` + Audios any `json:"audios,omitempty"` + Usage any `json:"usage,omitempty"` } // ManifestAgentResponse describes the provider response shape for a diff --git a/backend/internal/protocol/manifest_adapter.go b/backend/internal/protocol/manifest_adapter.go index 811b15514..58402cb5c 100644 --- a/backend/internal/protocol/manifest_adapter.go +++ b/backend/internal/protocol/manifest_adapter.go @@ -6,10 +6,13 @@ package protocol import ( + "bytes" "context" "crypto/sha256" "encoding/base64" + "encoding/json" "fmt" + "io" "net/http" "regexp" "strings" @@ -82,7 +85,18 @@ func syntheticAgentToolCallID(body []byte, index int) string { return fmt.Sprintf("call_%x", sum[:8]) } -func (a manifestAdapter) ParseCreate(_ context.Context, body []byte) (CreateResult, error) { +func (a manifestAdapter) ParseCreate(ctx context.Context, body []byte) (CreateResult, error) { + return a.ParseCreateWithRequest(ctx, GenerationRequest{}, body) +} + +func (a manifestAdapter) ParseCreateWithRequest(_ context.Context, request GenerationRequest, body []byte) (CreateResult, error) { + if a.manifest.Response.StreamedJSONAudio { + format := "" + if request.Extra != nil { + format = manifestString(request.Extra["audioFormat"]) + } + return streamedJSONAudioCreateResult(body, a.manifest.Response.ResultKind, format) + } if a.manifest.Response.BinaryPayload { return binaryPayloadCreateResult(a.manifest.Response.ResultKind, body) } @@ -93,8 +107,127 @@ func (a manifestAdapter) ParseCreate(_ context.Context, body []byte) (CreateResu return a.parse(payload, PollContext{}), nil } -// binaryPayloadCreateResult 把同步二进制响应包装为单个媒体结果。MIME 以响应内容探测为准， -// 空响应必须失败，不能把空内容伪装成生成成功。 +// streamedJSONAudioCreateResult decodes the chunked JSON response used by +// Doubao's unidirectional TTS endpoint. Go exposes HTTP chunked transfer as a +// single byte stream, so the decoder accepts both concatenated JSON objects and +// newline/SSE-style data frames. +const maxStreamedJSONAudioBytes = 64 << 20 + +func streamedJSONAudioCreateResult(body []byte, resultKind string, format string) (CreateResult, error) { + if len(body) == 0 { + return CreateResult{}, fmt.Errorf("streamed JSON audio response is empty") + } + if len(body) > maxStreamedJSONAudioBytes { + return CreateResult{}, fmt.Errorf("streamed JSON audio response exceeds %d bytes", maxStreamedJSONAudioBytes) + } + frames, err := decodeJSONFrames(body) + if err != nil { + return CreateResult{}, fmt.Errorf("decode streamed JSON audio response: %w", err) + } + var audio bytes.Buffer + var usage map[string]any + for _, frame := range frames { + code := strings.ToLower(strings.TrimSpace(manifestString(firstNonNilPathValue(frame, "code", "status_code", "statusCode")))) + if code != "" && code != "0" && code != "ok" && code != "success" && code != "succeeded" { + message := strings.TrimSpace(manifestString(firstNonNilPathValue(frame, "message", "msg", "error.message"))) + if message == "" { + message = "上游返回失败状态" + } + return CreateResult{}, fmt.Errorf("流式音频合成失败（code %s）：%s", code, message) + } + if value := firstPathValue(frame, "data", "audio", "audio.data"); value != "" { + decoded, decodeErr := decodeStreamedAudioBase64(value) + if decodeErr != nil { + return CreateResult{}, fmt.Errorf("decode streamed audio data: %w", decodeErr) + } + _, _ = audio.Write(decoded) + } + if rawUsage := pathValue(frame, "usage"); rawUsage != nil { + usage = manifestObject(rawUsage) + } + } + if audio.Len() == 0 { + return CreateResult{}, fmt.Errorf("streamed JSON audio response contains no audio data") + } + mimeType := streamedAudioMIMEType(format) + reference := MediaReference{DataURL: "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(audio.Bytes()), MIMEType: mimeType} + return CreateResult{Status: StatusSucceeded, Result: &Result{Audios: []MediaReference{reference}, Usage: usage}}, nil +} + +func decodeJSONFrames(body []byte) ([]map[string]any, error) { + decoder := json.NewDecoder(bytes.NewReader(body)) + frames := make([]map[string]any, 0, 4) + for { + var frame map[string]any + err := decoder.Decode(&frame) + if err == io.EOF { + return frames, nil + } + if err != nil { + break + } + if frame != nil { + frames = append(frames, frame) + } + } + // Some gateways wrap each JSON object in an SSE-like "data:" line. + for _, line := range strings.Split(string(body), "\n") { + line = strings.TrimSpace(strings.TrimSuffix(line, "\r")) + line = strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if line == "" || line == "[DONE]" { + continue + } + var frame map[string]any + if err := json.Unmarshal([]byte(line), &frame); err != nil { + return nil, err + } + frames = append(frames, frame) + } + if len(frames) == 0 { + return nil, fmt.Errorf("no JSON frames found") + } + return frames, nil +} + +func streamedAudioMIMEType(format string) string { + switch strings.ToLower(strings.TrimSpace(format)) { + case "wav": + return "audio/wav" + case "ogg_opus", "ogg": + return "audio/ogg" + case "pcm": + return "audio/pcm" + default: + return "audio/mpeg" + } +} + +func firstNonNilPathValue(payload map[string]any, paths ...string) any { + for _, path := range paths { + if value := pathValue(payload, path); value != nil { + return value + } + } + return nil +} + +func decodeStreamedAudioBase64(value string) ([]byte, error) { + value = strings.TrimSpace(value) + if value == "" { + return nil, nil + } + decoded, err := base64.StdEncoding.DecodeString(value) + if err == nil { + return decoded, nil + } + decoded, rawErr := base64.RawStdEncoding.DecodeString(value) + if rawErr == nil { + return decoded, nil + } + return nil, err +} + +// binaryPayloadCreateResult wraps a synchronous binary response as one media result. func binaryPayloadCreateResult(resultKind string, body []byte) (CreateResult, error) { if len(body) == 0 { return CreateResult{}, fmt.Errorf("binary payload response is empty") diff --git a/backend/internal/repository/analytics.go b/backend/internal/repository/analytics.go index 370b06ef8..c6d7de176 100644 --- a/backend/internal/repository/analytics.go +++ b/backend/internal/repository/analytics.go @@ -75,7 +75,7 @@ func (r *Repository) RecordUserActivity(userID string, event string, count int, func (r *Repository) AnalyticsTasks(filter AnalyticsFilter) ([]model.Task, error) { var tasks []model.Task - query := r.db.Select("id", "user_id", "type", "status", "operation", "provider", "model", "started_at", "completed_at", "created_at").Where("created_at >= ? AND created_at < ?", filter.From, filter.To) + query := whereTimeRange(r.db.Select("id", "user_id", "type", "status", "operation", "provider", "model", "started_at", "completed_at", "created_at"), "created_at", filter.From, filter.To) if filter.UserID != "" { query = query.Where("user_id = ?", filter.UserID) } @@ -250,6 +250,19 @@ func (r *Repository) HasAPICallLogForTask(taskID string) (bool, error) { return count > 0, err } +// HasVisibleFailedAPICallLogForTask 判断管理端默认请求明细里是否已经有这条任务的失败记录。 +// 轮询、下载、上传和本地保存默认不出现在请求日志列表，不能据此认为用户可见的失败已经被记录。 +func (r *Repository) HasVisibleFailedAPICallLogForTask(taskID string) (bool, error) { + if strings.TrimSpace(taskID) == "" { + return false, nil + } + var count int64 + err := r.db.Model(&model.ApiCallLog{}). + Where("task_id = ? AND status = ? AND COALESCE(request_kind, '') NOT IN ?", taskID, model.ApiCallStatusFailed, []string{"poll", "download", "upload", "local_save", "register"}). + Count(&count).Error + return count > 0, err +} + func visibleAPICallLogQuery(query *gorm.DB) *gorm.DB { // 轮询属于一次生成调用的内部状态查询，管理端不单独展示。 return query.Where("api_call_logs.request_kind <> ?", "poll") @@ -269,8 +282,23 @@ func (r *Repository) VideoAPICallRoot(log model.ApiCallLog) (*model.ApiCallLog, return &root, nil } +// whereTimeRange compares instants. SQLite stores time.Time as offset text, so a +// lexicographic compare against a UTC bound hides records written after local +// midnight until the UTC date rolls forward. +func whereTimeRange(query *gorm.DB, column string, from, to time.Time) *gorm.DB { + switch column { + case "created_at", "api_call_logs.created_at": + default: + return query.Where("1 = 0") + } + if query.Dialector.Name() == "sqlite" { + return query.Where("unixepoch("+column+") >= ? AND unixepoch("+column+") < ?", from.Unix(), to.Unix()) + } + return query.Where(column+" >= ? AND "+column+" < ?", from, to) +} + func (r *Repository) apiCallLogQuery(filter AnalyticsFilter) *gorm.DB { - query := r.db.Where("api_call_logs.created_at >= ? AND api_call_logs.created_at < ?", filter.From, filter.To) + query := whereTimeRange(r.db, "api_call_logs.created_at", filter.From, filter.To) if filter.UserID != "" { query = query.Where("api_call_logs.user_id = ?", filter.UserID) } diff --git a/backend/internal/repository/analytics_test.go b/backend/internal/repository/analytics_test.go index f02d260c0..cb525751d 100644 --- a/backend/internal/repository/analytics_test.go +++ b/backend/internal/repository/analytics_test.go @@ -60,6 +60,44 @@ func TestAPICallLogRecordTypeFiltersListAndExport(t *testing.T) { } } +func TestSQLiteAPICallLogRangeUsesInstantNotOffsetText(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:api-log-utc-range?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.ApiCallLog{}); err != nil { + t.Fatal(err) + } + local := time.FixedZone("CST", 8*3600) + inside := time.Date(2026, 10, 1, 2, 2, 34, 0, local) // 2026-09-30 18:02 UTC + outside := time.Date(2026, 10, 1, 9, 0, 0, 0, local) // 2026-10-01 01:00 UTC + for _, item := range []model.ApiCallLog{ + {ID: "inside-local-midnight", Capability: "audio", RequestKind: "create", CreatedAt: inside}, + {ID: "outside-utc-day", Capability: "audio", RequestKind: "create", CreatedAt: outside}, + } { + if err := db.Create(&item).Error; err != nil { + t.Fatal(err) + } + } + logs, total, err := New(db).QueryAPICallLogs(APICallLogFilter{ + AnalyticsFilter: AnalyticsFilter{ + From: time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC), + To: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), + }, + RecordType: "request", + }) + if err != nil { + t.Fatal(err) + } + if total != 1 || len(logs) != 1 || logs[0].ID != "inside-local-midnight" { + ids := make([]string, 0, len(logs)) + for _, log := range logs { + ids = append(ids, log.ID) + } + t.Fatalf("total=%d ids=%v, want only the 02:02+08 record inside the UTC day", total, ids) + } +} + func (l *sqlCaptureLogger) LogMode(logger.LogLevel) logger.Interface { return l } func (*sqlCaptureLogger) Info(context.Context, string, ...any) {} func (*sqlCaptureLogger) Warn(context.Context, string, ...any) {} diff --git a/backend/internal/repository/asset_library.go b/backend/internal/repository/asset_library.go index 9058fa879..b6fc77c93 100644 --- a/backend/internal/repository/asset_library.go +++ b/backend/internal/repository/asset_library.go @@ -28,9 +28,8 @@ type UserAssetFacetRow struct { func (r *Repository) UserAssetsPage(userID string, page int, pageSize int, filter UserAssetPageFilter) ([]model.Asset, int64, error) { var assets []model.Asset var total int64 - // 素材库页面只展示媒体与文本素材；entity 角色卡由项目资产页管理。列表与 facets 必须同口径排除， - // 否则 facets 会计入 entity，前端出现“全部计数 30 但列表为空”的矛盾。 - query := userAssetFilteredQuery(r.db.Model(&model.Asset{}).Where("user_id = ? AND kind <> ?", userID, "entity"), filter, true) + // 角色卡是用户素材的一种，和图片、音频一起出现在素材库；列表与 facets 使用同一查询口径。 + query := userAssetFilteredQuery(r.db.Model(&model.Asset{}).Where("user_id = ?", userID), filter, true) if err := query.Count(&total).Error; err != nil { return nil, 0, err } @@ -40,7 +39,7 @@ func (r *Repository) UserAssetsPage(userID string, page int, pageSize int, filte func (r *Repository) UserAssetFacets(userID string, status string) ([]UserAssetFacetRow, []UserAssetFacetRow, []UserAssetFacetRow, error) { base := func() *gorm.DB { - return userAssetFilteredQuery(r.db.Model(&model.Asset{}).Where("user_id = ? AND kind <> ?", userID, "entity"), UserAssetPageFilter{Status: status}, false) + return userAssetFilteredQuery(r.db.Model(&model.Asset{}).Where("user_id = ?", userID), UserAssetPageFilter{Status: status}, false) } var kindRows []UserAssetFacetRow if err := base().Select("kind AS key, COUNT(*) AS count").Group("kind").Scan(&kindRows).Error; err != nil { diff --git a/backend/internal/repository/asset_library_test.go b/backend/internal/repository/asset_library_test.go index dc1311b6d..09b7c1505 100644 --- a/backend/internal/repository/asset_library_test.go +++ b/backend/internal/repository/asset_library_test.go @@ -25,6 +25,31 @@ func newAssetLibraryTestRepository(t *testing.T) (*Repository, *gorm.DB) { return New(db), db } +func TestUserCharacterAssetsPageReturnsStablePages(t *testing.T) { + repo, db := newAssetLibraryTestRepository(t) + now := time.Now().UTC() + for index, title := range []string{"甲", "乙", "丙"} { + asset := model.Asset{ID: fmt.Sprintf("character-%d", index), UserID: "user-1", Kind: "entity", Category: model.AssetCategoryCharacter, Status: model.AssetVersionStatusConfirmed, Title: title, PayloadJSON: "{}", CreatedAt: now, UpdatedAt: now.Add(time.Duration(index) * time.Second)} + if err := db.Create(&asset).Error; err != nil { + t.Fatal(err) + } + } + if err := db.Create(&model.Asset{ID: "image-1", UserID: "user-1", Kind: "image", Category: model.AssetCategoryMaterial, Status: model.AssetVersionStatusConfirmed, Title: "不是角色", PayloadJSON: "{}", CreatedAt: now, UpdatedAt: now}).Error; err != nil { + t.Fatal(err) + } + first, total, err := repo.UserCharacterAssetsPage("user-1", "", 1, 2) + if err != nil { + t.Fatal(err) + } + second, _, err := repo.UserCharacterAssetsPage("user-1", "乙", 1, 2) + if err != nil { + t.Fatal(err) + } + if total != 3 || len(first) != 2 || first[0].ID != "character-2" || len(second) != 1 || second[0].Title != "乙" { + t.Fatalf("page = total %d first %#v search %#v", total, first, second) + } +} + func TestUserAssetsPagePaginatesAndIsolatesUsers(t *testing.T) { repo, db := newAssetLibraryTestRepository(t) now := time.Now().UTC() diff --git a/backend/internal/repository/finance_audio_test.go b/backend/internal/repository/finance_audio_test.go new file mode 100644 index 000000000..71f579feb --- /dev/null +++ b/backend/internal/repository/finance_audio_test.go @@ -0,0 +1,79 @@ +package repository + +import ( + "testing" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "infinite-canvas/backend/internal/model" +) + +func TestAudioBillingAmountRoundsUp(t *testing.T) { + amount, err := audioBillingAmount(100, 3, 10_000) + if err != nil || amount != 300 { + t.Fatalf("audioBillingAmount() = %d, %v; want 300", amount, err) + } +} + +func TestSettleAudioPerSecondOrderUsesMeasuredDuration(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:finance-audio-settle?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.CreditAccount{}, &model.BillingOrder{}, &model.CreditLedgerEntry{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.CreditAccount{UserID: "user-audio", AvailableMicrocredits: 1_000, ReservedMicrocredits: 100}).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.BillingOrder{ + ID: "audio-order", UserID: "user-audio", IdempotencyKey: "audio-task", Capability: "audio", BillingMode: "per_second", + UnitPriceMicrocredits: 100, MultiplierBasisPoints: 10_000, Quantity: 1, AmountMicrocredits: 100, + BillingCostSnapshot: model.BillingCostSnapshot{CostBillingMode: "per_second", CostQuantity: 1}, + ReservedAmountMicrocredits: 100, Status: model.BillingStatusRunning, + }).Error; err != nil { + t.Fatal(err) + } + + repo := &Repository{db: db} + if err := repo.SettleBillingOrderWithAudioDuration("audio-order", "", 2_100); err != nil { + t.Fatalf("SettleBillingOrderWithAudioDuration() error = %v", err) + } + var order model.BillingOrder + if err := db.First(&order, "id = ?", "audio-order").Error; err != nil { + t.Fatal(err) + } + if order.Status != model.BillingStatusSettled || order.Quantity != 3 || order.ActualAmountMicrocredits != 300 || order.RefundedAmountMicrocredits != 0 || order.CostQuantity != 3 { + t.Fatalf("settled audio order = %#v", order) + } + var account model.CreditAccount + if err := db.First(&account, "user_id = ?", "user-audio").Error; err != nil { + t.Fatal(err) + } + if account.AvailableMicrocredits != 800 || account.ReservedMicrocredits != 0 { + t.Fatalf("settled audio account = %#v; want available=800 reserved=0", account) + } +} + +func TestSettleAudioPerSecondOrderRequiresMeasuredDuration(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:finance-audio-duration-required?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.CreditAccount{}, &model.BillingOrder{}, &model.CreditLedgerEntry{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.CreditAccount{UserID: "user-audio", ReservedMicrocredits: 100}).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.BillingOrder{ + ID: "audio-order", UserID: "user-audio", IdempotencyKey: "audio-task", Capability: "audio", BillingMode: "per_second", + UnitPriceMicrocredits: 100, MultiplierBasisPoints: 10_000, Quantity: 1, AmountMicrocredits: 100, + ReservedAmountMicrocredits: 100, Status: model.BillingStatusRunning, + }).Error; err != nil { + t.Fatal(err) + } + if err := (&Repository{db: db}).SettleBillingOrderWithAudioDuration("audio-order", "", 0); err == nil { + t.Fatal("expected missing duration to fail") + } +} diff --git a/backend/internal/repository/finance_billing.go b/backend/internal/repository/finance_billing.go index 9eb60971d..dcd8dc4d7 100644 --- a/backend/internal/repository/finance_billing.go +++ b/backend/internal/repository/finance_billing.go @@ -7,6 +7,7 @@ package repository import ( "errors" + "fmt" "strings" "time" @@ -240,6 +241,14 @@ func (r *Repository) MarkBillingUncertain(id string, errorText string) error { } func (r *Repository) SettleBillingOrder(id string, providerRequestID string) error { + return r.settleBillingOrder(id, providerRequestID, nil) +} + +func (r *Repository) SettleBillingOrderWithAudioDuration(id string, providerRequestID string, durationMs int64) error { + return r.settleBillingOrder(id, providerRequestID, &durationMs) +} + +func (r *Repository) settleBillingOrder(id string, providerRequestID string, audioDurationMs *int64) error { var observedUsage *BillingUsage var observedUsageSource string var observedActual int64 @@ -336,6 +345,79 @@ func (r *Repository) SettleBillingOrder(id string, providerRequestID string) err } return nil } + actual := order.AmountMicrocredits + quantity := order.Quantity + refund := int64(0) + supplement := int64(0) + if order.BillingMode == "per_second" && order.Capability == "audio" { + if audioDurationMs == nil || *audioDurationMs <= 0 { + return errors.New("音频实际时长不可用，无法按秒结算") + } + quantity = (*audioDurationMs + 999) / 1000 + var err error + actual, err = audioBillingAmount(order.UnitPriceMicrocredits, quantity, order.MultiplierBasisPoints) + if err != nil { + return err + } + if billingChargeLimitApplies(order) && actual > order.ChargeLimitMicrocredits { + return errors.New("音频实际费用超过本次授权上限，需人工核对") + } + reserved := order.ReservedAmountMicrocredits + if reserved <= 0 { + reserved = order.AmountMicrocredits + } + refund = max(reserved-actual, int64(0)) + supplement = max(actual-reserved, int64(0)) + updated := tx.Model(&model.CreditAccount{}). + Where("user_id = ? AND reserved_microcredits >= ? AND available_microcredits >= ?", order.UserID, reserved, supplement). + Updates(map[string]any{ + "available_microcredits": gorm.Expr("available_microcredits + ?", refund-supplement), + "reserved_microcredits": gorm.Expr("reserved_microcredits - ?", reserved), + "version": gorm.Expr("version + 1"), + "updated_at": time.Now(), + }) + if updated.Error != nil { + return updated.Error + } + if updated.RowsAffected != 1 { + return errors.New("预留积分不足以完成音频实际结算") + } + var account model.CreditAccount + if err := tx.First(&account, "user_id = ?", order.UserID).Error; err != nil { + return err + } + now := time.Now() + costQuantity := order.CostQuantity + if order.CostBillingMode == "per_second" { + costQuantity = quantity + } + orderUpdates := map[string]any{"status": model.BillingStatusSettled, "quantity": quantity, "actual_amount_microcredits": actual, "refunded_amount_microcredits": refund, "cost_quantity": costQuantity, "settled_at": &now, "updated_at": now} + if providerRequestID != "" { + orderUpdates["provider_request_id"] = providerRequestID + } + if err := tx.Model(&order).Updates(orderUpdates).Error; err != nil { + return err + } + note := "音频按实际输出时长结算" + if supplement > 0 { + note += "，已补扣差额" + } + if err := tx.Create(&model.CreditLedgerEntry{ID: newRepositoryID(), UserID: order.UserID, Type: model.CreditLedgerConsume, + AmountMicrocredits: -actual, AvailableDeltaMicrocredits: -supplement, ReservedDeltaMicrocredits: -reserved, + AvailableAfterMicrocredits: account.AvailableMicrocredits, ReservedAfterMicrocredits: account.ReservedMicrocredits, + BillingOrderID: order.ID, Model: order.Model, ChannelID: order.ChannelID, Scene: order.Scene, Note: note}).Error; err != nil { + return err + } + if refund > 0 { + if err := tx.Create(&model.CreditLedgerEntry{ID: newRepositoryID(), UserID: order.UserID, Type: model.CreditLedgerRefund, + AmountMicrocredits: refund, AvailableDeltaMicrocredits: refund, + AvailableAfterMicrocredits: account.AvailableMicrocredits, ReservedAfterMicrocredits: account.ReservedMicrocredits, + BillingOrderID: order.ID, Model: order.Model, ChannelID: order.ChannelID, Scene: order.Scene, Note: "音频实际时长短于预授权，差额退回"}).Error; err != nil { + return err + } + } + return nil + } updated := tx.Model(&model.CreditAccount{}). Where("user_id = ? AND reserved_microcredits >= ?", order.UserID, order.AmountMicrocredits). Updates(map[string]any{ @@ -398,6 +480,21 @@ func (r *Repository) SettleBillingOrder(id string, providerRequestID string) err return err } +func audioBillingAmount(unitPrice int64, quantity int64, multiplierBPS int64) (int64, error) { + if unitPrice < 0 || quantity <= 0 || multiplierBPS <= 0 { + return 0, errors.New("积分计费参数无效") + } + const maxInt64 = int64(^uint64(0) >> 1) + if unitPrice > maxInt64/quantity || unitPrice*quantity > (maxInt64-9_999)/multiplierBPS { + return 0, errors.New("积分计费金额溢出") + } + amount := (unitPrice*quantity*multiplierBPS + 9_999) / 10_000 + if amount < 0 { + return 0, fmt.Errorf("积分计费金额无效：%d", amount) + } + return amount, nil +} + // RestoreRefundedBillingOrder compensates a billing order that was refunded // before an operator confirmed the upstream task had actually succeeded. // diff --git a/backend/internal/repository/payment.go b/backend/internal/repository/payment.go index 36f81a149..8117e7875 100644 --- a/backend/internal/repository/payment.go +++ b/backend/internal/repository/payment.go @@ -51,16 +51,30 @@ func (r *Repository) CreateTopupProduct(product *model.TopupProduct) error { return r.db.Create(product).Error } -func (r *Repository) UpdateTopupProduct(product *model.TopupProduct) error { - return r.db.Model(&model.TopupProduct{}).Where("id = ?", product.ID).Updates(map[string]any{ - "name": product.Name, "description": product.Description, "amount_fen": product.AmountFen, - "credits_microcredits": product.CreditsMicrocredits, "enabled": product.Enabled, - "sort_order": product.SortOrder, "sale_strategy": product.SaleStrategy, - "period_days": product.PeriodDays, "period_purchase_limit": product.PeriodPurchaseLimit, - "stock_total": product.StockTotal, "stock_remaining": product.StockRemaining, - "sale_start_at": product.SaleStartAt, "sale_end_at": product.SaleEndAt, - "updated_by": product.UpdatedBy, "updated_at": time.Now(), - }).Error +// UpdateTopupProduct locks the product row and derives the new values from the +// locked state. Orders reserve and release stock with stock_remaining ± 1 on the +// same row, so computing the remaining stock outside the lock would overwrite a +// concurrent reservation or release. +func (r *Repository) UpdateTopupProduct(id string, build func(existing *model.TopupProduct) (*model.TopupProduct, error)) error { + return r.db.Transaction(func(tx *gorm.DB) error { + var existing model.TopupProduct + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&existing, "id = ?", strings.TrimSpace(id)).Error; err != nil { + return err + } + product, err := build(&existing) + if err != nil { + return err + } + return tx.Model(&model.TopupProduct{}).Where("id = ?", existing.ID).Updates(map[string]any{ + "name": product.Name, "description": product.Description, "amount_fen": product.AmountFen, + "credits_microcredits": product.CreditsMicrocredits, "enabled": product.Enabled, + "sort_order": product.SortOrder, "sale_strategy": product.SaleStrategy, + "period_days": product.PeriodDays, "period_purchase_limit": product.PeriodPurchaseLimit, + "stock_total": product.StockTotal, "stock_remaining": product.StockRemaining, + "sale_start_at": product.SaleStartAt, "sale_end_at": product.SaleEndAt, + "updated_by": product.UpdatedBy, "updated_at": time.Now(), + }).Error + }) } func (r *Repository) CreatePaymentProviderConfig(config *model.PaymentProviderConfig) error { @@ -99,6 +113,13 @@ func (r *Repository) CreatePaymentOrderWithProductReservation(order *model.Payme if strategy == "" { strategy = model.TopupSaleStrategyUnlimited } + // 同一幂等键的重试直接返回原订单：不能被下面的限购/库存检查拦下， + // 尤其是周期限购已把这笔待付款订单计入名额。 + if err := tx.First(&existing, "user_id = ? AND idempotency_key = ?", order.UserID, order.IdempotencyKey).Error; err == nil { + return nil + } else if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } now := time.Now() if !product.Enabled { return ErrTopupUnavailable @@ -107,8 +128,13 @@ func (r *Repository) CreatePaymentOrderWithProductReservation(order *model.Payme return ErrTopupUnavailable } if strategy == model.TopupSaleStrategyPeriodic && product.PeriodDays > 0 && product.PeriodPurchaseLimit > 0 { + // 未关闭的订单都占用周期名额：只数已入账订单的话，用户可以先建多笔 + // 待付款订单再逐笔付款，全部入账。订单关闭（含超时关单）后名额释放。 + // 商品行已加锁，同一商品的下单在这里串行。 var count int64 - if err := tx.Model(&model.PaymentOrder{}).Where("user_id = ? AND product_id = ? AND status = ? AND created_at >= ?", order.UserID, order.ProductID, model.PaymentOrderCredited, now.Add(-time.Duration(product.PeriodDays)*24*time.Hour)).Count(&count).Error; err != nil { + if err := tx.Model(&model.PaymentOrder{}).Where("user_id = ? AND product_id = ? AND status IN ? AND created_at >= ?", order.UserID, order.ProductID, []model.PaymentOrderStatus{ + model.PaymentOrderCreated, model.PaymentOrderPending, model.PaymentOrderClosing, model.PaymentOrderCreateFailed, model.PaymentOrderCredited, + }, now.Add(-time.Duration(product.PeriodDays)*24*time.Hour)).Count(&count).Error; err != nil { return err } if count >= int64(product.PeriodPurchaseLimit) { diff --git a/backend/internal/repository/payment_product_limits_test.go b/backend/internal/repository/payment_product_limits_test.go new file mode 100644 index 000000000..3e9cdb4d1 --- /dev/null +++ b/backend/internal/repository/payment_product_limits_test.go @@ -0,0 +1,85 @@ +package repository + +import ( + "errors" + "testing" + "time" + + "infinite-canvas/backend/internal/model" +) + +func paymentLimitTestOrder(id, productID string) *model.PaymentOrder { + return &model.PaymentOrder{ + ID: id, UserID: "user-1", IdempotencyKey: "idem-" + id, MerchantOrderNo: "merchant-" + id, + ProductID: productID, ProductName: "周卡", ProviderID: "wechat-native", PluginID: "plugin-1", + ProviderConfigID: "config-1", ProviderConfigVersion: 1, AmountFen: 100, Currency: "CNY", + CreditsMicrocredits: 100_000_000, Status: model.PaymentOrderCreated, ExpiresAt: time.Now().Add(time.Hour), + } +} + +// 周期限购必须把未关闭的待付款订单计入名额，否则先建多笔再逐笔付款可以全部入账。 +func TestPeriodicPurchaseLimitCountsUnpaidOrders(t *testing.T) { + db := openPaymentTestDB(t) + repo := New(db) + product := model.TopupProduct{ID: "weekly", Name: "周卡", AmountFen: 100, CreditsMicrocredits: 100_000_000, Enabled: true, SaleStrategy: model.TopupSaleStrategyPeriodic, PeriodDays: 7, PeriodPurchaseLimit: 1} + if err := db.Create(&product).Error; err != nil { + t.Fatal(err) + } + first := paymentLimitTestOrder("order-1", product.ID) + if _, created, err := repo.CreatePaymentOrderWithProductReservation(first); err != nil || !created { + t.Fatalf("first order created=%v err=%v", created, err) + } + if _, _, err := repo.CreatePaymentOrderWithProductReservation(paymentLimitTestOrder("order-2", product.ID)); !errors.Is(err, ErrTopupUnavailable) { + t.Fatalf("second unpaid order error = %v, want %v", err, ErrTopupUnavailable) + } + + // 同一幂等键重试拿回原订单，不被限购拦下。 + retry := paymentLimitTestOrder("order-retry", product.ID) + retry.IdempotencyKey = first.IdempotencyKey + existing, created, err := repo.CreatePaymentOrderWithProductReservation(retry) + if err != nil || created || existing.ID != first.ID { + t.Fatalf("idempotent retry = %#v created=%v err=%v", existing, created, err) + } + + // 关单后名额释放。 + if err := repo.MarkPaymentOrderClosed(first.ID, "CLOSED"); err != nil { + t.Fatal(err) + } + if _, created, err := repo.CreatePaymentOrderWithProductReservation(paymentLimitTestOrder("order-3", product.ID)); err != nil || !created { + t.Fatalf("order after close created=%v err=%v", created, err) + } +} + +// 商品更新在锁内按最新库存计算，不能把并发下单扣减的库存写回旧值。 +func TestUpdateTopupProductKeepsConcurrentStockReservation(t *testing.T) { + db := openPaymentTestDB(t) + repo := New(db) + product := model.TopupProduct{ID: "stock", Name: "限量", AmountFen: 100, CreditsMicrocredits: 100_000_000, Enabled: true, SaleStrategy: model.TopupSaleStrategyInventory, StockTotal: 5, StockRemaining: 5} + if err := db.Create(&product).Error; err != nil { + t.Fatal(err) + } + // 管理员先读到剩余 5，随后用户下单把库存扣成 4，管理员再保存（只改名称）。 + stale, err := repo.TopupProduct(product.ID) + if err != nil { + t.Fatal(err) + } + if _, created, err := repo.CreatePaymentOrderWithProductReservation(paymentLimitTestOrder("order-1", product.ID)); err != nil || !created { + t.Fatalf("reserve stock created=%v err=%v", created, err) + } + err = repo.UpdateTopupProduct(product.ID, func(existing *model.TopupProduct) (*model.TopupProduct, error) { + next := *stale + next.Name = "限量（改名）" + next.StockRemaining = existing.StockRemaining + return &next, nil + }) + if err != nil { + t.Fatal(err) + } + updated, err := repo.TopupProduct(product.ID) + if err != nil { + t.Fatal(err) + } + if updated.Name != "限量（改名）" || updated.StockRemaining != 4 { + t.Fatalf("updated product = name %q remaining %d, want renamed with remaining 4", updated.Name, updated.StockRemaining) + } +} diff --git a/backend/internal/repository/project_character.go b/backend/internal/repository/project_character.go index 020e612f1..ad6bad8d2 100644 --- a/backend/internal/repository/project_character.go +++ b/backend/internal/repository/project_character.go @@ -1,6 +1,7 @@ package repository import ( + "strings" "time" "infinite-canvas/backend/internal/model" @@ -21,6 +22,52 @@ func (r *Repository) ProjectCharacterAsset(userID string, projectID string, asse return &asset, nil } +func (r *Repository) UserCharacterAsset(userID string, assetID string) (*model.Asset, error) { + var asset model.Asset + err := r.db.Where("id = ? AND user_id = ? AND category = ? AND kind = ?", assetID, userID, model.AssetCategoryCharacter, "entity").First(&asset).Error + if err != nil { + return nil, err + } + return &asset, nil +} + +func (r *Repository) characterAssetQuery(userID string, query string) *gorm.DB { + db := r.db.Model(&model.Asset{}).Where("user_id = ? AND category = ? AND kind = ? AND status <> ?", userID, model.AssetCategoryCharacter, "entity", model.AssetVersionStatusArchived) + if query = strings.TrimSpace(query); query != "" { + pattern := "%" + strings.ToLower(query) + "%" + db = db.Where("LOWER(title) LIKE ? OR LOWER(payload_json) LIKE ?", pattern, pattern) + } + return db +} + +func (r *Repository) UserCharacterAssetsPage(userID string, query string, page int, pageSize int) ([]model.Asset, int64, error) { + if page < 1 { + page = 1 + } + if pageSize < 1 || pageSize > 48 { + pageSize = 12 + } + var total int64 + if err := r.characterAssetQuery(userID, query).Count(&total).Error; err != nil { + return nil, 0, err + } + var assets []model.Asset + err := r.characterAssetQuery(userID, query).Order("updated_at desc, id desc").Offset((page - 1) * pageSize).Limit(pageSize).Find(&assets).Error + return assets, total, err +} + +func (r *Repository) UserCharacterAssetsByIDs(userID string, ids []string) ([]model.Asset, error) { + if len(ids) == 0 { + return nil, nil + } + if len(ids) > 80 { + ids = ids[:80] + } + var assets []model.Asset + err := r.db.Where("user_id = ? AND category = ? AND kind = ? AND id IN ?", userID, model.AssetCategoryCharacter, "entity", ids).Find(&assets).Error + return assets, err +} + func (r *Repository) AssetVersion(id string) (*model.AssetVersion, error) { var version model.AssetVersion if err := r.db.First(&version, "id = ?", id).Error; err != nil { @@ -86,6 +133,13 @@ func (r *Repository) CreateVoiceProfile(profile *model.VoiceProfile) error { return r.db.Create(profile).Error } +// RenameVoiceProfile 只改用户上传样本声音的显示名；内置音色的名称由系统维护。 +func (r *Repository) RenameVoiceProfile(userID, id, name string) error { + return r.db.Model(&model.VoiceProfile{}). + Where("id = ? AND user_id = ? AND provider = ?", id, userID, "user_upload"). + Updates(map[string]any{"name": name, "updated_at": time.Now()}).Error +} + func (r *Repository) VoiceProfileBySampleResource(userID string, resourceID string) (*model.VoiceProfile, error) { var profile model.VoiceProfile if err := r.db.First(&profile, "user_id = ? AND sample_resource_id = ? AND status = ?", userID, resourceID, "active").Error; err != nil { @@ -94,6 +148,28 @@ func (r *Repository) VoiceProfileBySampleResource(userID string, resourceID stri return &profile, nil } +func (r *Repository) CreateUserCharacter(asset *model.Asset, version *model.AssetVersion, representations []model.AssetRepresentation, voice *model.CharacterVoiceBinding) error { + return r.db.Transaction(func(tx *gorm.DB) error { + if err := tx.Create(asset).Error; err != nil { + return err + } + if err := tx.Create(version).Error; err != nil { + return err + } + if len(representations) > 0 { + if err := tx.Create(&representations).Error; err != nil { + return err + } + } + if voice != nil { + if err := tx.Create(voice).Error; err != nil { + return err + } + } + return nil + }) +} + // CreateProjectCharacter 将角色身份、首版本和项目关联放入同一事务。 func (r *Repository) CreateProjectCharacter(projectID string, asset *model.Asset, version *model.AssetVersion, link *model.ProjectAssetLink) error { return r.db.Transaction(func(tx *gorm.DB) error { @@ -117,8 +193,19 @@ func (r *Repository) SaveCharacterVersion(projectID string, asset *model.Asset, if err := saveCharacterVersion(tx, asset, version, representations, voice); err != nil { return err } - return tx.Model(&model.Project{}).Where("id = ?", projectID). - Updates(map[string]any{"revision": gorm.Expr("revision + 1"), "updated_at": time.Now()}).Error + projectIDs := []string{} + if projectID != "" { + projectIDs = append(projectIDs, projectID) + } else if err := tx.Model(&model.ProjectAssetLink{}).Where("asset_id = ?", asset.ID).Distinct("project_id").Pluck("project_id", &projectIDs).Error; err != nil { + return err + } + for _, linkedProjectID := range projectIDs { + if err := tx.Model(&model.Project{}).Where("id = ?", linkedProjectID). + Updates(map[string]any{"revision": gorm.Expr("revision + 1"), "updated_at": time.Now()}).Error; err != nil { + return err + } + } + return nil }) } diff --git a/backend/internal/repository/provider_task_recovery_test.go b/backend/internal/repository/provider_task_recovery_test.go new file mode 100644 index 000000000..c9dbe5827 --- /dev/null +++ b/backend/internal/repository/provider_task_recovery_test.go @@ -0,0 +1,67 @@ +package repository + +import ( + "errors" + "testing" + "time" + + "infinite-canvas/backend/internal/model" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestClaimFailedTaskProviderRecoveryUsesConditionalLease(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:provider-recovery-lease?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Task{}); err != nil { + t.Fatal(err) + } + repo := New(db) + if err := db.Create(&model.Task{ID: "task-1", UserID: "user-1", Status: model.TaskStatusFailed, CreatedAt: time.Now()}).Error; err != nil { + t.Fatal(err) + } + + if err := repo.ClaimFailedTaskProviderRecovery("task-1", "", "manual-recovery:one", time.Minute); err != nil { + t.Fatalf("first claim: %v", err) + } + if err := repo.ClaimFailedTaskProviderRecovery("task-1", "", "manual-recovery:two", time.Minute); !errors.Is(err, ErrTaskProviderRecoveryConflict) { + t.Fatalf("second claim error = %v, want conflict", err) + } + if err := repo.ReleaseTaskProviderRecovery("task-1", "manual-recovery:two"); err != nil { + t.Fatal(err) + } + if err := repo.ReleaseTaskProviderRecovery("task-1", "manual-recovery:one"); err != nil { + t.Fatal(err) + } + if err := repo.ClaimFailedTaskProviderRecovery("task-1", "user-1", "manual-recovery:two", time.Minute); err != nil { + t.Fatalf("claim after release: %v", err) + } + + var task model.Task + if err := db.First(&task, "id = ?", "task-1").Error; err != nil { + t.Fatal(err) + } + if task.LeaseOwner != "manual-recovery:two" || task.LeaseExpiresAt == nil { + t.Fatalf("task lease = owner:%q expires:%v", task.LeaseOwner, task.LeaseExpiresAt) + } +} + +func TestClaimFailedTaskProviderRecoveryRejectsNonFailedTask(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:provider-recovery-status?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Task{}); err != nil { + t.Fatal(err) + } + repo := New(db) + if err := db.Create(&model.Task{ID: "task-1", UserID: "user-1", Status: model.TaskStatusRunning, CreatedAt: time.Now()}).Error; err != nil { + t.Fatal(err) + } + if err := repo.ClaimFailedTaskProviderRecovery("task-1", "user-1", "manual-recovery:one", time.Minute); !errors.Is(err, ErrTaskProviderRecoveryConflict) { + t.Fatalf("claim running task error = %v, want conflict", err) + } +} diff --git a/backend/internal/service/aliases_types.go b/backend/internal/service/aliases_types.go index 8828ebcfc..905a08db1 100644 --- a/backend/internal/service/aliases_types.go +++ b/backend/internal/service/aliases_types.go @@ -107,6 +107,7 @@ type ( ChannelOrderItem = app.ChannelOrderItem ChannelOrderRequest = app.ChannelOrderRequest ChannelRequest = app.ChannelRequest + CharacterAssetSummary = app.CharacterAssetSummary CharacterCardSummary = app.CharacterCardSummary CharacterRepresentationInput = app.CharacterRepresentationInput CharacterRepresentationSummary = app.CharacterRepresentationSummary @@ -120,6 +121,7 @@ type ( CreateAssetFolderRequest = app.CreateAssetFolderRequest CreateAssetVersionRequest = app.CreateAssetVersionRequest CreatePaymentOrderRequest = app.CreatePaymentOrderRequest + CreateCharacterRequest = app.CreateCharacterRequest CreateProjectAssetFolderRequest = app.CreateProjectAssetFolderRequest CreateProjectCharacterRequest = app.CreateProjectCharacterRequest CreateProjectRequest = app.CreateProjectRequest diff --git a/backend/internal/storage/objects.go b/backend/internal/storage/objects.go index 26e6eea02..7318c3014 100644 --- a/backend/internal/storage/objects.go +++ b/backend/internal/storage/objects.go @@ -191,17 +191,52 @@ func SignedOSSObjectURL(setting Settings, objectKey string, expiresAt time.Time) } func SignedOriginObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + return signedOriginObjectURL(setting, objectKey, objectURLSigning{ExpiresAt: expiresAt}) +} + +// SignedOriginObjectDisplayURL 签发浏览器展示用的源站直连地址。 +// +// 浏览器 HTTP 缓存以完整 URL 为键。调用方把签名起始时间对齐到固定时间窗后，同一对象在 +// 同一窗口内得到逐字节相同的地址：刷新画布、多标签页、多个后端实例都复用浏览器已缓存的字节， +// 不再每次都从对象存储重新下载。对象键带随机 ID、内容不可变，因此同时让对象存储按签名 +// 剩余有效期回写 Cache-Control，浏览器在有效期内连条件请求都不必发出。 +func SignedOriginObjectDisplayURL(setting Settings, objectKey string, signedAt time.Time, expiresAt time.Time) (string, error) { + signing := objectURLSigning{SignedAt: signedAt, ExpiresAt: expiresAt} + if seconds := int64(expiresAt.Sub(signingTime(signedAt)) / time.Second); seconds > 0 { + signing.CacheControl = "private, max-age=" + strconv.FormatInt(seconds, 10) + ", immutable" + } + return signedOriginObjectURL(setting, objectKey, signing) +} + +// objectURLSigning 描述一次预签名 GET 的可变部分。 +type objectURLSigning struct { + // SignedAt 为零值表示一次性地址（下载、服务端读取），按当前时间签名。 + SignedAt time.Time + ExpiresAt time.Time + ContentDisposition string + CacheControl string +} + +func signingTime(signedAt time.Time) time.Time { + if signedAt.IsZero() { + return time.Now().UTC() + } + return signedAt.UTC() +} + +func signedOriginObjectURL(setting Settings, objectKey string, signing objectURLSigning) (string, error) { setting = NormalizeSettings(setting) if setting.Provider == s3Provider { - return SignedS3ObjectURL(setting, objectKey, expiresAt) + return signedS3ObjectURL(setting, objectKey, signing) } if setting.Provider == qiniuKodoProvider { - return SignedQiniuS3ObjectURL(setting, objectKey, expiresAt) + return signedQiniuS3ObjectURL(setting, objectKey, signing) } if setting.Provider == tencentCOSProvider { - return SignedCOSObjectURL(setting, objectKey, expiresAt) + return signedCOSObjectURL(setting, objectKey, signing) } - return SignedAliyunOSSObjectURL(setting, objectKey, expiresAt) + // 阿里云 V1 签名只由绝对过期时间决定，过期时间对齐即可得到稳定地址。 + return signedAliyunOSSObjectURL(setting, objectKey, signing) } // SignedOriginObjectDownloadURL signs a browser-facing object-store URL whose @@ -210,17 +245,7 @@ func SignedOriginObjectURL(setting Settings, objectKey string, expiresAt time.Ti // overrides are forwarded and signed correctly. func SignedOriginObjectDownloadURL(setting Settings, objectKey string, expiresAt time.Time, fileName string) (string, error) { disposition := objectDownloadContentDisposition(fileName, objectKey) - setting = NormalizeSettings(setting) - if setting.Provider == s3Provider { - return SignedS3ObjectDownloadURL(setting, objectKey, expiresAt, disposition) - } - if setting.Provider == qiniuKodoProvider { - return SignedQiniuS3ObjectDownloadURL(setting, objectKey, expiresAt, disposition) - } - if setting.Provider == tencentCOSProvider { - return SignedCOSObjectDownloadURL(setting, objectKey, expiresAt, disposition) - } - return SignedAliyunOSSObjectDownloadURL(setting, objectKey, expiresAt, disposition) + return signedOriginObjectURL(setting, objectKey, objectURLSigning{ExpiresAt: expiresAt, ContentDisposition: disposition}) } func objectDownloadContentDisposition(fileName string, objectKey string) string { @@ -247,14 +272,11 @@ func objectDownloadContentDisposition(fileName string, objectKey string) string } func SignedAliyunOSSObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { - return signedAliyunOSSObjectURL(setting, objectKey, expiresAt, "") + return signedAliyunOSSObjectURL(setting, objectKey, objectURLSigning{ExpiresAt: expiresAt}) } -func SignedAliyunOSSObjectDownloadURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { - return signedAliyunOSSObjectURL(setting, objectKey, expiresAt, disposition) -} - -func signedAliyunOSSObjectURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { +func signedAliyunOSSObjectURL(setting Settings, objectKey string, signing objectURLSigning) (string, error) { + expiresAt := signing.ExpiresAt baseURL, err := OssBucketBaseURL(setting) if err != nil { return "", err @@ -270,9 +292,18 @@ func signedAliyunOSSObjectURL(setting Settings, objectKey string, expiresAt time expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10) canonicalResource := "/" + setting.Bucket + "/" + objectKey query := baseURL.Query() - if disposition != "" { - query.Set("response-content-disposition", disposition) - canonicalResource += "?response-content-disposition=" + disposition + // V1 签名要求 response-* 覆盖参数按字典序写入 CanonicalizedResource，且值不做转义。 + overrides := make([]string, 0, 2) + if signing.CacheControl != "" { + query.Set("response-cache-control", signing.CacheControl) + overrides = append(overrides, "response-cache-control="+signing.CacheControl) + } + if signing.ContentDisposition != "" { + query.Set("response-content-disposition", signing.ContentDisposition) + overrides = append(overrides, "response-content-disposition="+signing.ContentDisposition) + } + if len(overrides) > 0 { + canonicalResource += "?" + strings.Join(overrides, "&") } stringToSign := strings.Join([]string{http.MethodGet, "", "", expires, canonicalResource}, "\n") mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret)) @@ -370,14 +401,11 @@ func GetQiniuObjectRange(setting Settings, objectKey string, rangeHeader string) } func SignedCOSObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { - return signedCOSObjectURL(setting, objectKey, expiresAt, "") + return signedCOSObjectURL(setting, objectKey, objectURLSigning{ExpiresAt: expiresAt}) } -func SignedCOSObjectDownloadURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { - return signedCOSObjectURL(setting, objectKey, expiresAt, disposition) -} - -func signedCOSObjectURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { +func signedCOSObjectURL(setting Settings, objectKey string, signing objectURLSigning) (string, error) { + expiresAt := signing.ExpiresAt setting = NormalizeSettings(setting) if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" { return "", errors.New("COS 访问密钥不可用") @@ -386,19 +414,24 @@ func signedCOSObjectURL(setting Settings, objectKey string, expiresAt time.Time, if objectKey == "" { return "", errors.New("COS 对象路径为空") } - expires := time.Until(expiresAt) - if expires <= 0 { + if !expiresAt.After(time.Now()) { return "", errors.New("COS 签名有效期必须晚于当前时间") } client, err := NewCOSClient(setting, transferTimeout(setting)) if err != nil { return "", err } - var options interface{} - if disposition != "" { - options = &cos.ObjectGetOptions{ResponseContentDisposition: disposition} + signedAt := signingTime(signing.SignedAt) + query := url.Values{} + if signing.CacheControl != "" { + query.Set("response-cache-control", signing.CacheControl) } - signedURL, err := client.Object.GetPresignedURL(context.Background(), http.MethodGet, objectKey, setting.AccessKeyID, setting.AccessKeySecret, expires, options) + if signing.ContentDisposition != "" { + query.Set("response-content-disposition", signing.ContentDisposition) + } + // COS 默认用 time.Now 生成 q-sign-time；显式传入对齐后的时间窗，保证同一窗口内地址稳定。 + options := &cos.PresignedURLOptions{Query: &query, AuthTime: &cos.AuthTime{SignStartTime: signedAt, SignEndTime: expiresAt, KeyStartTime: signedAt, KeyEndTime: expiresAt}} + signedURL, err := client.Object.GetPresignedURL(context.Background(), http.MethodGet, objectKey, setting.AccessKeyID, setting.AccessKeySecret, expiresAt.Sub(signedAt), options) if err != nil { return "", err } @@ -427,14 +460,10 @@ func SignedQiniuObjectURL(setting Settings, objectKey string, expiresAt time.Tim // SignedQiniuS3ObjectURL 用七牛兼容 S3 的 AWS Signature V4 访问私有空间。 // 没有绑定域名时，浏览器不直接访问该地址，而是由后端代理读取并返回文件。 func SignedQiniuS3ObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { - return signedQiniuS3ObjectURL(setting, objectKey, expiresAt, "") + return signedQiniuS3ObjectURL(setting, objectKey, objectURLSigning{ExpiresAt: expiresAt}) } -func SignedQiniuS3ObjectDownloadURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { - return signedQiniuS3ObjectURL(setting, objectKey, expiresAt, disposition) -} - -func signedQiniuS3ObjectURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { +func signedQiniuS3ObjectURL(setting Settings, objectKey string, signing objectURLSigning) (string, error) { setting = NormalizeSettings(setting) region := QiniuS3Region(setting) if region == "" { @@ -443,18 +472,25 @@ func signedQiniuS3ObjectURL(setting Settings, objectKey string, expiresAt time.T baseURL := &url.URL{Scheme: "https", Host: setting.Bucket + ".s3." + region + ".qiniucs.com"} // 保留对象键的原始路径，让 url.URL 和 AWS signer 只做一次 RFC 3986 转义。 baseURL.Path = "/" + objectKey - if disposition != "" { - query := baseURL.Query() - query.Set("response-content-disposition", disposition) - baseURL.RawQuery = query.Encode() + if !signing.ExpiresAt.After(time.Now()) { + return "", errors.New("七牛云 Kodo S3 签名有效期必须晚于当前时间") } + query := baseURL.Query() + if signing.CacheControl != "" { + query.Set("response-cache-control", signing.CacheControl) + } + if signing.ContentDisposition != "" { + query.Set("response-content-disposition", signing.ContentDisposition) + } + baseURL.RawQuery = query.Encode() req, err := http.NewRequest(http.MethodGet, baseURL.String(), nil) if err != nil { return "", err } credentialsValue := credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, "") signer := awsv4.NewSigner(credentialsValue) - if _, err := signer.Presign(req, nil, "s3", region, time.Until(expiresAt), time.Now().UTC()); err != nil { + signedAt := signingTime(signing.SignedAt) + if _, err := signer.Presign(req, nil, "s3", region, signing.ExpiresAt.Sub(signedAt), signedAt); err != nil { return "", fmt.Errorf("七牛云 Kodo S3 签名失败：%w", err) } return req.URL.String(), nil diff --git a/backend/internal/storage/s3.go b/backend/internal/storage/s3.go index b644e07ce..b32e2489d 100644 --- a/backend/internal/storage/s3.go +++ b/backend/internal/storage/s3.go @@ -16,7 +16,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" + awsv4 "github.com/aws/aws-sdk-go/aws/signer/v4" awss3 "github.com/aws/aws-sdk-go/service/s3" ) @@ -137,29 +139,33 @@ func GetS3ObjectRange(setting Settings, objectKey string, rangeHeader string) (* } func SignedS3ObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { - return signedS3ObjectURL(setting, objectKey, expiresAt, "") + return signedS3ObjectURL(setting, objectKey, objectURLSigning{ExpiresAt: expiresAt}) } -func SignedS3ObjectDownloadURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { - return signedS3ObjectURL(setting, objectKey, expiresAt, disposition) -} - -func signedS3ObjectURL(setting Settings, objectKey string, expiresAt time.Time, disposition string) (string, error) { +func signedS3ObjectURL(setting Settings, objectKey string, signing objectURLSigning) (string, error) { setting = NormalizeSettings(setting) client, err := NewS3Client(setting, transferTimeout(setting)) if err != nil { return "", err } - duration := time.Until(expiresAt) - if duration <= 0 { + if !signing.ExpiresAt.After(time.Now()) { return "", errors.New("S3 签名有效期必须晚于当前时间") } input := &awss3.GetObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))} - if disposition != "" { - input.ResponseContentDisposition = aws.String(disposition) + if signing.CacheControl != "" { + input.ResponseCacheControl = aws.String(signing.CacheControl) + } + if signing.ContentDisposition != "" { + input.ResponseContentDisposition = aws.String(signing.ContentDisposition) } req, _ := client.GetObjectRequest(input) - value, err := req.Presign(duration) + signedAt := signingTime(signing.SignedAt) + // SDK 默认以 time.Now 作为 X-Amz-Date；替换为对齐后的签名时间，保证同一窗口内地址稳定。 + // DisableURIPathEscaping 与 SDK 为 S3 注册的默认签名器保持一致。 + req.Handlers.Sign.Swap(awsv4.SignRequestHandler.Name, request.NamedHandler{Name: awsv4.SignRequestHandler.Name, Fn: func(r *request.Request) { + awsv4.SignSDKRequestWithCurrentTime(r, func() time.Time { return signedAt }, func(s *awsv4.Signer) { s.DisableURIPathEscaping = true }) + }}) + value, err := req.Presign(signing.ExpiresAt.Sub(signedAt)) if err != nil { return "", fmt.Errorf("S3 下载地址签名失败：%w", err) } diff --git a/backend/internal/storage/signing_test.go b/backend/internal/storage/signing_test.go new file mode 100644 index 000000000..39a556d16 --- /dev/null +++ b/backend/internal/storage/signing_test.go @@ -0,0 +1,90 @@ +package storage + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// 展示地址的签名时间由调用方对齐；同一时间窗内重复签发必须得到逐字节相同的 URL， +// 否则浏览器 HTTP 缓存（以完整 URL 为键）在每次刷新画布时都会失效并重新下载对象。 +func TestDisplayURLIsDeterministicForAlignedSigningTime(t *testing.T) { + t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1") + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + signedAt := time.Now().UTC().Truncate(time.Hour) + expiresAt := signedAt.Add(24 * time.Hour) + cases := map[string]Settings{ + "s3": {Provider: s3Provider, Region: "cn-nb1", Endpoint: server.URL, Bucket: "bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value"}, + "qiniu": {Provider: qiniuKodoProvider, Endpoint: "https://up-z0.qiniup.com", Bucket: "bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value"}, + "tencent": {Provider: tencentCOSProvider, Endpoint: "https://cos.ap-guangzhou.myqcloud.com", Bucket: "bucket-1250000000", AccessKeyID: "access-id", AccessKeySecret: "secret-value"}, + "aliyun": {Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value"}, + } + for name, setting := range cases { + t.Run(name, func(t *testing.T) { + first, err := SignedOriginObjectDisplayURL(setting, "users/u-1/image/a b.png", signedAt, expiresAt) + if err != nil { + t.Fatal(err) + } + time.Sleep(1100 * time.Millisecond) // 跨过秒级时间戳，证明结果不依赖 time.Now。 + second, err := SignedOriginObjectDisplayURL(setting, "users/u-1/image/a b.png", signedAt, expiresAt) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatalf("display URL is not stable:\n%s\n%s", first, second) + } + query := mustParseQuery(t, first) + if got := query.Get("response-cache-control"); got != "private, max-age=86400, immutable" { + t.Fatalf("response-cache-control = %q in %s", got, first) + } + if strings.Contains(first, "secret-value") { + t.Fatal("signed URL leaked secret key") + } + switch name { + case "s3", "qiniu": + if query.Get("X-Amz-Date") != signedAt.Format("20060102T150405Z") || query.Get("X-Amz-Expires") != "86400" { + t.Fatalf("SigV4 window = %s/%s", query.Get("X-Amz-Date"), query.Get("X-Amz-Expires")) + } + case "tencent": + window := strings.Join([]string{itoa(signedAt.Unix()), itoa(expiresAt.Unix())}, ";") + if query.Get("q-sign-time") != window || query.Get("q-key-time") != window { + t.Fatalf("COS window = %s", first) + } + } + }) + } +} + +func TestOneShotURLsKeepCurrentTimeSigningWithoutCacheOverride(t *testing.T) { + setting := Settings{Provider: qiniuKodoProvider, Endpoint: "https://up-z0.qiniup.com", Bucket: "bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value"} + value, err := SignedOriginObjectDownloadURL(setting, "users/u-1/video/clip.mp4", time.Now().Add(time.Hour), "镜头.mp4") + if err != nil { + t.Fatal(err) + } + query := mustParseQuery(t, value) + if query.Get("response-cache-control") != "" || !strings.HasPrefix(query.Get("response-content-disposition"), "attachment") { + t.Fatalf("download URL = %s", value) + } + signedAt, err := time.Parse("20060102T150405Z", query.Get("X-Amz-Date")) + if err != nil || time.Since(signedAt) > time.Minute { + t.Fatalf("download URL must be signed at the current time, got %q", query.Get("X-Amz-Date")) + } +} + +func mustParseQuery(t *testing.T, raw string) url.Values { + t.Helper() + parsed, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return parsed.Query() +} + +func itoa(value int64) string { + return strconv.FormatInt(value, 10) +}