From c0a9250cb32435cb1a9e12ad25f642f18c6eb4c2 Mon Sep 17 00:00:00 2001 From: ddcat666 <26564566+ddcat666@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:17:28 +0800 Subject: [PATCH] =?UTF-8?q?feat(previs):=20=E5=9B=BE=E7=89=87=E7=94=9F?= =?UTF-8?q?=E6=88=90=E9=A2=84=E6=BC=94=E5=8F=B0=20-=20=E4=BB=8E=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E8=8A=82=E7=82=B9=E4=BA=A4=E7=BB=99=20Agent=20?= =?UTF-8?q?=E5=A4=8D=E7=8E=B0=E4=B8=BA=E5=8F=AF=E7=BC=96=E8=BE=91=203D=20?= =?UTF-8?q?=E9=A2=84=E6=BC=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 图片工具栏新增「生成预演台」，自动选用支持看图的文本模型并提交复现指令。对话用节点卡片展示「此图」，普通图片演员只写名称时不再要求角色卡绑定。 --- .../app/cloud_agent_previs_mutation.go | 6 +- .../app/cloud_agent_previs_mutation_test.go | 18 +++ .../canvas/canvas-cloud-agent-chat-ui.tsx | 7 ++ .../canvas/canvas-cloud-agent-panel.tsx | 53 ++++++--- .../canvas/canvas-image-toolbar-tools.tsx | 13 +- .../components/canvas/canvas-node-toolbar.tsx | 4 +- web/src/lib/canvas/image-to-previs-agent.ts | 34 ++++++ web/src/pages/canvas/project.tsx | 29 ++++- web/src/pages/canvas/shared.tsx | 2 +- web/src/services/cloud-agent-conversations.ts | 9 +- web/test/agent-chat-node-links.test.tsx | 17 +++ web/test/canvas-node-toolbar.test.ts | 2 + web/test/image-to-previs-agent.test.ts | 111 ++++++++++++++++++ 13 files changed, 282 insertions(+), 23 deletions(-) create mode 100644 web/src/lib/canvas/image-to-previs-agent.ts create mode 100644 web/test/image-to-previs-agent.test.ts diff --git a/backend/internal/app/cloud_agent_previs_mutation.go b/backend/internal/app/cloud_agent_previs_mutation.go index a082af908..c0910b63b 100644 --- a/backend/internal/app/cloud_agent_previs_mutation.go +++ b/backend/internal/app/cloud_agent_previs_mutation.go @@ -1086,7 +1086,9 @@ func cloudAgentPrevisApplyName(item map[string]any, value *string, label string) } func cloudAgentPrevisApplyCharacterBinding(item map[string]any, operation cloudAgentPrevisPatchOperation) error { - provided := operation.CharacterAssetID != nil || operation.CharacterVersionID != nil || operation.ReferenceNodeID != nil || operation.CharacterName != nil + // characterName is descriptive metadata for a real character-card binding; + // a name on an actor from an ordinary reference image must not trigger binding. + provided := operation.CharacterAssetID != nil || operation.CharacterVersionID != nil || operation.ReferenceNodeID != nil if !provided { return nil } @@ -1122,7 +1124,7 @@ func cloudAgentPrevisApplyCharacterBinding(item map[string]any, operation cloudA } func cloudAgentPrevisValidateCharacterBindingInCanvas(doc map[string]any, operation cloudAgentPrevisPatchOperation) error { - if operation.CharacterAssetID == nil && operation.CharacterVersionID == nil && operation.ReferenceNodeID == nil && operation.CharacterName == nil { + if operation.CharacterAssetID == nil && operation.CharacterVersionID == nil && operation.ReferenceNodeID == nil { return nil } if operation.CharacterAssetID == nil || operation.CharacterVersionID == nil || operation.ReferenceNodeID == nil { diff --git a/backend/internal/app/cloud_agent_previs_mutation_test.go b/backend/internal/app/cloud_agent_previs_mutation_test.go index 2b702824f..3ecb79a81 100644 --- a/backend/internal/app/cloud_agent_previs_mutation_test.go +++ b/backend/internal/app/cloud_agent_previs_mutation_test.go @@ -495,6 +495,24 @@ func TestCloudAgentPrevisPatchLimitsAndClosedArguments(t *testing.T) { } } +func TestCloudAgentPrevisOrdinaryActorNameDoesNotRequireCharacterBinding(t *testing.T) { + scene, err := cloudAgentPrevisSceneCreateTemplate(cloudAgentPrevisSceneCreateArgs{SceneID: "image-scene", Title: "图片复现", TemplateID: "empty"}) + if err != nil { + t.Fatal(err) + } + _, err = cloudAgentPrevisApplyPatchOperation(scene, cloudAgentPrevisPatchOperation{ + Type: "object_add", ID: "image-actor", Kind: "actor", Primitive: "character", + Name: previsStringPtr("道姑"), CharacterName: previsStringPtr("道姑"), Pose: "stand", + }, 0) + if err != nil { + t.Fatalf("ordinary image actor was treated as a character binding: %v", err) + } + actor := cloudAgentPrevisFindByID(creationMaps(scene["objects"]), "image-actor") + if actor == nil || actor["characterBinding"] != nil { + t.Fatalf("ordinary actor unexpectedly has character binding: %#v", actor) + } +} + func TestCloudAgentPrevisCharacterBindingAndAnimationPatch(t *testing.T) { s, canvas := previsMutationFixture(t) canvas.PayloadJSON = `{"nodes":[{"id":"hero-card","type":"text","metadata":{"workflowKind":"character","characterAssetId":"hero-asset","characterVersionId":"hero-v1"}}],"connections":[],"previsScenes":[]}` diff --git a/web/src/components/canvas/canvas-cloud-agent-chat-ui.tsx b/web/src/components/canvas/canvas-cloud-agent-chat-ui.tsx index 3ae5fc422..e69450100 100644 --- a/web/src/components/canvas/canvas-cloud-agent-chat-ui.tsx +++ b/web/src/components/canvas/canvas-cloud-agent-chat-ui.tsx @@ -84,6 +84,7 @@ export type CloudAgentChatMessage = { meta?: string; detail?: unknown; attachments?: CloudAgentChatAttachment[]; + canvasReferenceNodeId?: string; interjection?: "sent" | "undelivered"; }; @@ -381,6 +382,8 @@ export function AgentChatMessage({ const formAnswer = item.formAnswer ?? (isUser ? parseCloudAgentFormAnswer(item.text) : undefined); const markdownComponents = useMemo(() => createAgentMessageMarkdownComponents(references, onFocusNode), [onFocusNode, references]); const agentMarkdownText = rewriteAgentNodeLinks(displayedText, references); + const canvasReference = item.canvasReferenceNodeId ? references.find((reference) => reference.nodeId === item.canvasReferenceNodeId) : undefined; + const userCanvasMarkdown = canvasReference ? item.text.replace("此图", createAgentNodeLink(canvasReference.nodeId, canvasReference)) : item.text; const errorTone = item.errorSeverity === "warning" ? "warning" : "error"; const color = theme.node.text; if (item.reasoning) { @@ -484,6 +487,10 @@ export function AgentChatMessage({ {agentMarkdownText} + ) : isUser && canvasReference ? ( + + {userCanvasMarkdown} + ) : formAnswer ? ( ) : ( diff --git a/web/src/components/canvas/canvas-cloud-agent-panel.tsx b/web/src/components/canvas/canvas-cloud-agent-panel.tsx index 564cf1b29..3d33ab7db 100644 --- a/web/src/components/canvas/canvas-cloud-agent-panel.tsx +++ b/web/src/components/canvas/canvas-cloud-agent-panel.tsx @@ -42,8 +42,10 @@ import { saveCloudAgentPendingSubmission, type CloudAgentConversation, type CloudAgentPendingSubmission, + type CloudAgentMessagePresentation, } from "@/services/cloud-agent-conversations"; import { logicalModelIDForConfig, modelOptionName, resolveModelRequestConfig, selectableModelsByCapability, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store"; +import { selectImageInspectionModel } from "@/lib/canvas/image-to-previs-agent"; import { useActiveTheme } from "@/stores/canvas/use-canvas-theme-store"; import { useUserStore } from "@/stores/use-user-store"; import { getActiveUserScope } from "@/lib/user-scope"; @@ -67,8 +69,8 @@ type CloudAgentPanelProps = { selectedNodeIds: string[]; references: CanvasResourceReference[]; open: boolean; - /** 外部预填请求（如右键“发送到 Agent”）；id 变化即追加一次 text。 */ - prefillRequest?: { id: number; text: string } | null; + /** 外部预填请求（如右键“发送到 Agent”）；id 变化即追加一次 text，可选自动提交和视觉模型路由。 */ + prefillRequest?: (CloudAgentMessagePresentation & { id: number; text: string; autoSubmit?: boolean; requiresVision?: boolean }) | null; onOpen: () => void; onCollapse: () => void; onFocusNode?: (nodeId: string) => void; @@ -92,6 +94,8 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se const [messages, setMessages] = useState([]); const [prompt, setPrompt] = useState(""); const lastPrefillIdRef = useRef(0); + const [pendingAutoSubmit, setPendingAutoSubmit] = useState(null); + const prefilledSubmissionRef = useRef(null); const [reasoningMode, setReasoningMode] = useState("off"); const [profileView, setProfileView] = useState(null); const [profileLoading, setProfileLoading] = useState(false); @@ -319,7 +323,9 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se if (!prefillRequest || prefillRequest.id === lastPrefillIdRef.current) return; lastPrefillIdRef.current = prefillRequest.id; if (!prefillRequest.text.trim()) return; - setPrompt((current) => appendAgentPromptPrefill(current, prefillRequest.text)); + setPrompt((current) => appendAgentPromptPrefill(current, prefillRequest.displayText || prefillRequest.text)); + prefilledSubmissionRef.current = prefillRequest; + setPendingAutoSubmit(prefillRequest.autoSubmit ? prefillRequest : null); setView("chat"); }, [prefillRequest]); @@ -490,7 +496,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se const pending = await loadCloudAgentPendingSubmission(canvasId, current.id); if (!active) return; pendingSubmission.current = pending; - if (pending?.request) setPrompt(pending.request.prompt); + if (pending?.request) setPrompt(pending.displayText || pending.request.prompt); } else { setActiveConversationId(nanoid()); pendingSubmission.current = null; @@ -550,7 +556,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se if (event.seq > lastSeqRef.current + 1) canvasSyncRef.current?.reconcile(); lastSeqRef.current = event.seq; } - setMessages((current) => current.filter((item) => item.id !== `stream-error-${run.id}`)); + setMessages((current) => current.some((item) => item.id === `stream-error-${run.id}`) ? current.filter((item) => item.id !== `stream-error-${run.id}`) : current); setContextUsage((current) => reduceAgentContextUsage(current, event)); applyAgentEvent(event, setMessages, setRun, setApproval, setPrompt); canvasSyncRef.current?.receive(event); @@ -597,8 +603,14 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se } }; - const submit = async (override?: string) => { - const value = (override ?? prompt).trim(); + const submit = async (override?: string, options?: CloudAgentMessagePresentation & { requiresVision?: boolean }) => { + const draft = (override ?? prompt).trim(); + const pendingRequest = pendingSubmission.current; + const prefilled = prefilledSubmissionRef.current; + const matchesPending = pendingRequest?.request && pendingRequest.displayText === draft; + const matchesPrefill = prefilled?.displayText === draft; + const value = matchesPending ? pendingRequest.request!.prompt : matchesPrefill ? prefilled!.text : draft; + options = matchesPending ? { ...options, displayText: pendingRequest.displayText, canvasReferenceNodeId: pendingRequest.canvasReferenceNodeId } : matchesPrefill ? prefilled! : options; if (running) { await interject(value); return; @@ -609,6 +621,8 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se setBusy(true); let accepted = false; try { + const effectiveModel = options?.requiresVision ? selectImageInspectionModel(config, selectedModel) : selectedModel; + if (options?.requiresVision && !effectiveModel) throw new Error("当前没有配置支持图片输入的文本模型，请在模型能力设置中为一个文本模型开启图片输入后重试"); const pending = pendingSubmission.current; // An ambiguous previous POST owns its body/key until reconciled. // Editing model settings or prompt must not silently create a new charge. @@ -622,16 +636,16 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se if (selectedSkillIds.length && !capabilities.skills) throw new Error("当前后端尚未接入技能库"); await saveRemoteUserDataNow(); if (currentScope.current !== scope) return; - const agentConfig = { ...config, model: selectedModel }; - const requestConfig = resolveModelRequestConfig(agentConfig, selectedModel); + const agentConfig = { ...config, model: effectiveModel }; + const requestConfig = resolveModelRequestConfig(agentConfig, effectiveModel); const logicalModelId = logicalModelIDForConfig(agentConfig); const input = { canvasId, prompt: value, reasoningMode, profileRevision: profileView.revision, - model: modelOptionName(selectedModel) || undefined, - ...(logicalModelId ? { logicalModelId } : requestConfig.channelId ? { channelId: requestConfig.channelId, channelModelKey: modelOptionName(selectedModel) || undefined } : {}), + model: modelOptionName(effectiveModel) || undefined, + ...(logicalModelId ? { logicalModelId } : requestConfig.channelId ? { channelId: requestConfig.channelId, channelModelKey: modelOptionName(effectiveModel) || undefined } : {}), skillIds: [...new Set([...selectedSkillIds, ...resolveSkillMentions(value, installedSkills).map((skill) => skill.skillId)])], focusNodeIds: selectedNodeIds.length <= 8 ? selectedNodeIds : [], permissionMode, @@ -641,7 +655,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se const fingerprint = JSON.stringify({ scope, parent: run?.id, input }); if (pending && pending.fingerprint !== fingerprint) throw new Error("上一条请求尚未确认，请恢复原消息与设置后核对，不能覆盖原幂等记录"); const key = pending?.key || crypto.randomUUID(); - const next = { fingerprint, key, request: { ...input, idempotencyKey: key }, parentRunId: run?.id, messageId: `user-${key}` }; + const next = { fingerprint, key, request: { ...input, idempotencyKey: key }, parentRunId: run?.id, messageId: `user-${key}`, displayText: options?.displayText, canvasReferenceNodeId: options?.canvasReferenceNodeId }; // Persist before sending. A failed local save must not submit a request // whose recovery identity will disappear on reload. await saveCloudAgentPendingSubmission(canvasId, activeConversationId, next); @@ -654,8 +668,9 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se const nextMessages = appendUniqueMessage(messages, { id: submission.messageId || `user-${submission.key}`, role: "user", - text: formAnswer ? "已确认创作方向" : request.prompt, + text: formAnswer ? "已确认创作方向" : submission.displayText || request.prompt, ...(formAnswer ? { formAnswer } : {}), + ...(submission.canvasReferenceNodeId ? { canvasReferenceNodeId: submission.canvasReferenceNodeId } : {}), }); const now = new Date().toISOString(); const existing = conversations.find((item) => item.id === activeConversationId); @@ -667,7 +682,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se title: cloudAgentConversationTitle(nextMessages), messages: nextMessages, run, - model: selectedModel || undefined, + model: request.model || effectiveModel || undefined, permissionMode, skillIds: selectedSkillIds, createdAt: existing?.createdAt || now, @@ -677,6 +692,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se ]); if (currentScope.current !== scope) return; setPrompt(""); + prefilledSubmissionRef.current = null; setMessages(nextMessages); const result = submission.parentRunId ? await sendAgentMessage(submission.parentRunId, request) : await createAgentRun(request); accepted = true; @@ -708,6 +724,13 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se } }; + useEffect(() => { + const pending = pendingAutoSubmit; + if (!pending || !historyHydrated || !pendingHydrated || profileLoading || !profileView || profileError || busy || running) return; + setPendingAutoSubmit(null); + void submit(pending.text, { requiresVision: pending.requiresVision, displayText: pending.displayText, canvasReferenceNodeId: pending.canvasReferenceNodeId }); + }, [busy, historyHydrated, pendingAutoSubmit, pendingHydrated, profileError, profileLoading, profileView, running]); + const stop = async () => { const activeRun = run; if (!activeRun?.id || stopping) return; @@ -857,7 +880,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se if (currentScope.current === `${canvasId}:${conversation.id}`) { pendingSubmission.current = pending; setPendingHydrated(true); - if (pending?.request) setPrompt(pending.request.prompt); + if (pending?.request) setPrompt(pending.displayText || pending.request.prompt); } }) .catch((cause) => { diff --git a/web/src/components/canvas/canvas-image-toolbar-tools.tsx b/web/src/components/canvas/canvas-image-toolbar-tools.tsx index 9d56e9eca..e4bcaa6bf 100644 --- a/web/src/components/canvas/canvas-image-toolbar-tools.tsx +++ b/web/src/components/canvas/canvas-image-toolbar-tools.tsx @@ -4,7 +4,7 @@ import { Brush, Camera, Clapperboard, Contrast, Copy, FastForward, FileText, Glo import type { CanvasNodeData } from "@/types/canvas"; import type { NodeToolbarGroup } from "@/lib/canvas/tool-registry"; -type ImageNodeActionToolId = "copyPrompt" | "reversePrompt" | "replace" | "resize" | "annotation" | "annotationEdit" | "textEdit" | "maskEdit" | "removeBackground" | "layerDecomposition" | "emotion" | "portraitTexture" | "crop" | "split" | "upscale" | "superResolve" | "angle" | "lighting" | "panorama" | "view" | "multi_camera_nine_grid" | "story_pitch_four_grid" | "character_face_three_view" | "product_three_view" | "storyboard_25_grid" | "character_three_view_generation" | "cinematic_light_correction" | "image_projection_after_3s" | "image_projection_before_5s"; +type ImageNodeActionToolId = "copyPrompt" | "reversePrompt" | "replace" | "resize" | "annotation" | "annotationEdit" | "textEdit" | "maskEdit" | "removeBackground" | "layerDecomposition" | "emotion" | "portraitTexture" | "crop" | "split" | "upscale" | "superResolve" | "angle" | "lighting" | "panorama" | "view" | "previs" | "multi_camera_nine_grid" | "story_pitch_four_grid" | "character_face_three_view" | "product_three_view" | "storyboard_25_grid" | "character_three_view_generation" | "cinematic_light_correction" | "image_projection_after_3s" | "image_projection_before_5s"; type ImageToolHandlers = { onUpload: (node: CanvasNodeData) => void; @@ -23,6 +23,7 @@ type ImageToolHandlers = { onAngle: (node: CanvasNodeData) => void; onLighting: (node: CanvasNodeData) => void; onPanorama: (node: CanvasNodeData) => void; + onPrevis: (node: CanvasNodeData) => void; onViewImage: (node: CanvasNodeData) => void; onCopyPrompt: (node: CanvasNodeData) => void; onReversePrompt: (node: CanvasNodeData) => void; @@ -58,6 +59,16 @@ function nineGridRun(node: CanvasNodeData, handlers: ImageToolHandlers, tool: Im } const imageToolDefinitions: ImageToolDefinition[] = [ + { + id: "previs", + label: "生成预演台", + icon: () => , + group: "primary", + order: 20, + section: "3D 预演", + description: "让 Agent 识别当前图片并复现为可编辑的 3D 预演台", + run: (node, handlers) => handlers.onPrevis(node), + }, { id: "copyPrompt", section: "生成信息", diff --git a/web/src/components/canvas/canvas-node-toolbar.tsx b/web/src/components/canvas/canvas-node-toolbar.tsx index 85e2431fc..c3ee9f636 100644 --- a/web/src/components/canvas/canvas-node-toolbar.tsx +++ b/web/src/components/canvas/canvas-node-toolbar.tsx @@ -51,6 +51,7 @@ type CanvasNodeToolbarProps = { onAngle: (node: CanvasNodeData) => void; onLighting: (node: CanvasNodeData) => void; onPanorama: (node: CanvasNodeData) => void; + onPrevis: (node: CanvasNodeData) => void; onViewImage: (node: CanvasNodeData) => void; onExtractVideoFrames: (node: CanvasNodeData) => void; onExtractAudioFromVideo: (node: CanvasNodeData) => void; @@ -117,6 +118,7 @@ export function CanvasNodeToolbar({ onAngle, onLighting, onPanorama, + onPrevis, onViewImage, onExtractVideoFrames, onExtractAudioFromVideo, @@ -234,7 +236,7 @@ export function CanvasNodeToolbar({ } copyText(prompt, "提示词已复制"); }; - const imageTools = buildImageToolbarTools(node, { onUpload, onToggleFreeResize, onAnnotate, onAnnotationEdit, onTextEdit, onMaskEdit, onRemoveBackground, onLayerDecomposition, onEmotion, onPortraitTexture, onCrop, onUpscale, onSuperResolve, onAngle, onLighting, onPanorama, onViewImage, onCopyPrompt: copyImagePrompt, onReversePrompt, onNineGrid }); + const imageTools = buildImageToolbarTools(node, { onUpload, onToggleFreeResize, onAnnotate, onAnnotationEdit, onTextEdit, onMaskEdit, onRemoveBackground, onLayerDecomposition, onEmotion, onPortraitTexture, onCrop, onUpscale, onSuperResolve, onAngle, onLighting, onPanorama, onPrevis, onViewImage, onCopyPrompt: copyImagePrompt, onReversePrompt, onNineGrid }); // 构建 ToolContext——供注册表解析工具 const nodeHoverHandlers = { diff --git a/web/src/lib/canvas/image-to-previs-agent.ts b/web/src/lib/canvas/image-to-previs-agent.ts new file mode 100644 index 000000000..b5c8b9b8f --- /dev/null +++ b/web/src/lib/canvas/image-to-previs-agent.ts @@ -0,0 +1,34 @@ +import { canvasResourceMentionToken, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; +import { modelCapabilityConfigFor } from "@/lib/model-capabilities"; +import { resolveModelChannel, selectableModelsByCapability, type AiConfig } from "@/stores/use-config-store"; + +export const IMAGE_TO_PREVIS_AGENT_INSTRUCTION = [ + "请把这张画布图片复现为可编辑的 3D 预演台，而不是把图片放进预演台。", + "先调用 canvas_inspect_image 读取该图片的真实画面，不能根据节点标题、提示词或文件名猜测内容。", + "如果工具列表中没有 canvas_inspect_image，立即停止并报告当前模型不支持图片输入；不要改用 image_text_detect，因为它只准备文字识别请求，不会把画面交给模型。", + "然后调用 canvas_get_state 和 previs_scene_read 读取画布及预演台状态：如果已有对应预演场景就更新它，否则调用 previs_scene_create 创建一个空场景。", + "根据实际观察，用 previs_apply_patch 建立可移动的语义对象：逐个还原人物数量、人物/道具/环境的前后景关系、站位、朝向、相对尺度、姿态和动作；用相机、镜头构图、灯光和环境设置还原画面视角与氛围。", + "人物必须使用 actor，对象必须能在预演台中单独移动、旋转和调整姿态；不要只写一段描述，也不要把原图作为 billboard、背景板、参考平面或任何图片对象加入场景。", + "普通图片中的人物不是角色卡：使用 actor 的 name、archetype、pose 和 position 描述即可，禁止传 characterName、characterAssetId、characterVersionId 或 referenceNodeId。只有 canvas_get_state 明确返回 workflowKind=character 且三个角色卡 ID 完整匹配时，才允许角色绑定。", + "完成后重新读取预演场景确认对象数量、空间布局和镜头设置，并明确说明无法从图片确认的部分。", +].join("\n"); + +export function buildImageToPrevisAgentPrompt(reference: CanvasResourceReference) { + return `${canvasResourceMentionToken(reference)}\n${IMAGE_TO_PREVIS_AGENT_INSTRUCTION}`; +} + +export function buildImageToPrevisDisplayText() { + return "根据此图生成 3D 预演台"; +} + +export function modelSupportsImageInspection(config: AiConfig, model: string) { + return (modelCapabilityConfigFor(config, model).text?.references.maxImages || 0) > 0; +} + +/** 图片复现必须在真正支持图片输入的文本模型上运行。 */ +export function selectImageInspectionModel(config: AiConfig, preferredModel: string) { + const textModels = selectableModelsByCapability(config, "text"); + const canCloudAgentUse = (model: string) => resolveModelChannel(config, model).scope === "system" && modelSupportsImageInspection(config, model); + if (preferredModel && canCloudAgentUse(preferredModel)) return preferredModel; + return textModels.find(canCloudAgentUse) || ""; +} diff --git a/web/src/pages/canvas/project.tsx b/web/src/pages/canvas/project.tsx index dfed7cab0..6baf4b142 100644 --- a/web/src/pages/canvas/project.tsx +++ b/web/src/pages/canvas/project.tsx @@ -91,6 +91,8 @@ import { replaceCanvasReferenceMentions, type CanvasResourceReference, } from "@/lib/canvas/canvas-resource-references"; +import { buildImageToPrevisAgentPrompt, buildImageToPrevisDisplayText } from "@/lib/canvas/image-to-previs-agent"; +import type { CloudAgentMessagePresentation } from "@/services/cloud-agent-conversations"; import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays"; import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer"; import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer"; @@ -241,7 +243,7 @@ function CanvasViewportPage() { const [hoveredNodeId, setHoveredNodeId] = useState(null); const [contextMenu, setContextMenu] = useState(null); // 每次发送带递增 id：重复发送同一节点时文本相同，仍需触发一次追加。 - const [agentPrefillRequest, setAgentPrefillRequest] = useState<{ id: number; text: string } | null>(null); + const [agentPrefillRequest, setAgentPrefillRequest] = useState<(CloudAgentMessagePresentation & { id: number; text: string; autoSubmit?: boolean; requiresVision?: boolean }) | null>(null); const [isMiniMapOpen, setIsMiniMapOpen] = useState(false); const [canvasAppearance, setCanvasAppearance] = useState(() => canvasAppearanceForTheme(colorTheme)); const [backgroundMode, setBackgroundMode] = useState(DEFAULT_CANVAS_BACKGROUND_MODE); @@ -423,6 +425,30 @@ function CanvasViewportPage() { openAgent(); setContextMenu(null); }, [agentMentionReferences, openAgent]); + + const sendImageToPrevisAgent = useCallback((node: CanvasNodeData) => { + if (node.type !== CanvasNodeType.Image) return; + const reference = agentMentionReferences.find((item) => item.nodeId === node.id && item.kind === "image"); + if (!reference) { + message.warning("这张图片暂时没有可供 Agent 引用的画布资源"); + return; + } + if (!selectedNodeIdsRef.current.has(node.id)) { + const selection = new Set([node.id]); + selectedNodeIdsRef.current = selection; + setSelectedNodeIds(selection); + } + setAgentPrefillRequest((current) => ({ + id: (current?.id ?? 0) + 1, + text: buildImageToPrevisAgentPrompt(reference), + displayText: buildImageToPrevisDisplayText(), + canvasReferenceNodeId: reference.nodeId, + autoSubmit: true, + requiresVision: true, + })); + openAgent(); + setContextMenu(null); + }, [agentMentionReferences, message, openAgent]); // 修复素材关联仍遵守当前画布版本，不能替用户确认覆盖云端的新内容。 const confirmForceSaveCanvas = useCallback(() => { modal.confirm({ @@ -2978,6 +3004,7 @@ function CanvasViewportPage() { setLightingNodeId((current) => (current === node.id ? null : node.id)); }} onPanorama={openPanoramaConfig} + onPrevis={sendImageToPrevisAgent} onViewImage={(node) => setPreviewNodeId(node.id)} onExtractVideoFrames={openVideoFrameExtractor} onExtractAudioFromVideo={(node) => void extractAudioFromVideo(node)} diff --git a/web/src/pages/canvas/shared.tsx b/web/src/pages/canvas/shared.tsx index 411cbd6b3..76b42df1b 100644 --- a/web/src/pages/canvas/shared.tsx +++ b/web/src/pages/canvas/shared.tsx @@ -282,7 +282,7 @@ export default function SharedCanvasPage() { }} onHoverStart={keepToolbar} onHoverEnd={hideToolbar} onConnectStart={unauthorized} onResize={() => undefined} onContentChange={unauthorized} onRetry={unauthorized} onOpenTaskDetails={unauthorized} onViewImage={(target) => setInfoNodeId(target.id)} onContextMenu={(event, nodeId) => openContextMenu(event, nodeId)} />)} - setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onAnnotationEdit={unauthorized} onTextEdit={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onRemoveBackground={unauthorized} onLayerDecomposition={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onLighting={unauthorized} onPanorama={unauthorized} onViewImage={unauthorized} onExtractVideoFrames={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoSegments={unauthorized} extractingVideoFrames={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} onNineGrid={unauthorized} /> + setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onAnnotationEdit={unauthorized} onTextEdit={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onRemoveBackground={unauthorized} onLayerDecomposition={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onLighting={unauthorized} onPanorama={unauthorized} onPrevis={unauthorized} onViewImage={unauthorized} onExtractVideoFrames={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoSegments={unauthorized} extractingVideoFrames={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} onNineGrid={unauthorized} />
访客操作仅在当前页面临时生效
diff --git a/web/src/services/cloud-agent-conversations.ts b/web/src/services/cloud-agent-conversations.ts index 7d1f4077f..99ba9c9f2 100644 --- a/web/src/services/cloud-agent-conversations.ts +++ b/web/src/services/cloud-agent-conversations.ts @@ -3,6 +3,11 @@ import { markdownPlainText } from "@/lib/markdown-plain-text"; import { getActiveUserScope } from "@/lib/user-scope"; import type { AgentPermissionMode, AgentRun } from "@/services/api/agent"; +export type CloudAgentMessagePresentation = { + displayText?: string; + canvasReferenceNodeId?: string; +}; + export type CloudAgentConversationMessage = { id: string; role: "user" | "assistant" | "system" | "tool" | "error"; @@ -37,7 +42,7 @@ type CloudAgentConversationDocument = { conversations: CloudAgentConversation[]; }; -export type CloudAgentPendingSubmission = { +export type CloudAgentPendingSubmission = CloudAgentMessagePresentation & { fingerprint: string; key: string; request?: import("@/services/api/agent").CreateAgentRunInput; @@ -89,7 +94,7 @@ export async function loadCloudAgentPendingSubmission(canvasId: string, conversa if (parsed.request !== undefined && (!parsed.request || parsed.request.idempotencyKey !== parsed.key || parsed.request.canvasId !== canvasId || typeof parsed.request.prompt !== "string")) throw new Error("invalid request"); if (parsed.parentRunId !== undefined && typeof parsed.parentRunId !== "string") throw new Error("invalid parent"); if (parsed.messageId !== undefined && typeof parsed.messageId !== "string") throw new Error("invalid message"); - return { fingerprint: parsed.fingerprint, key: parsed.key, request: parsed.request, parentRunId: parsed.parentRunId, messageId: parsed.messageId } satisfies CloudAgentPendingSubmission; + return { fingerprint: parsed.fingerprint, key: parsed.key, request: parsed.request, parentRunId: parsed.parentRunId, messageId: parsed.messageId, displayText: parsed.displayText, canvasReferenceNodeId: parsed.canvasReferenceNodeId } satisfies CloudAgentPendingSubmission; } catch { throw new Error("待确认请求记录损坏，已暂停本对话发送；请先在任务中心核对原运行，不要直接重复生成"); } diff --git a/web/test/agent-chat-node-links.test.tsx b/web/test/agent-chat-node-links.test.tsx index a6e5ddcc7..c4aca1ad4 100644 --- a/web/test/agent-chat-node-links.test.tsx +++ b/web/test/agent-chat-node-links.test.tsx @@ -64,6 +64,23 @@ test("renders an agent node link as a titled inline node card", () => { expect(visibleText).not.toContain("video-1790385749473-q4157"); }); +test("renders the image-to-previs user request with the existing canvas node card", () => { + const imageReference = { ...videoReference, kind: "image" as const, sourceType: "image" as const, label: "道姑", title: "道姑" }; + const html = renderToStaticMarkup( + {}} + />, + ); + + expect(html).toContain('class="agent-message-node-link"'); + expect(html).toContain('class="agent-message-node-link-preview"'); + expect(html).toContain("道姑"); + expect(html).toContain("3D 预演台"); +}); + test("labels character cards by resource kind instead of the underlying text type", () => { const nodeId = "text-1790385749473-char1"; const reference = { id: nodeId, nodeId, kind: "character" as const, label: "李莫愁", title: "李莫愁", sourceType: "text" as const }; diff --git a/web/test/canvas-node-toolbar.test.ts b/web/test/canvas-node-toolbar.test.ts index 9901693f5..d964b0463 100644 --- a/web/test/canvas-node-toolbar.test.ts +++ b/web/test/canvas-node-toolbar.test.ts @@ -91,5 +91,7 @@ describe("canvas node toolbar model", () => { expect(tools.find((tool) => tool.id === "crop")?.section).toBe("构图与尺寸"); expect(tools.find((tool) => tool.id === "resize")?.active).toBe(true); expect(tools.find((tool) => tool.id === "copyPrompt")?.group).toBe("more"); + expect(tools.find((tool) => tool.id === "previs")?.group).toBe("primary"); + expect(tools.find((tool) => tool.id === "previs")?.description).toContain("Agent"); }); }); diff --git a/web/test/image-to-previs-agent.test.ts b/web/test/image-to-previs-agent.test.ts new file mode 100644 index 000000000..58817f86f --- /dev/null +++ b/web/test/image-to-previs-agent.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "bun:test"; + +import { buildImageToPrevisAgentPrompt, buildImageToPrevisDisplayText, modelSupportsImageInspection, selectImageInspectionModel } from "@/lib/canvas/image-to-previs-agent"; +import type { AiConfig } from "@/stores/use-config-store"; + +describe("image to previs Agent prompt", () => { + test("requires visual inspection and editable semantic objects", () => { + const prompt = buildImageToPrevisAgentPrompt({ + id: "image-ref", + nodeId: "image-node-1", + kind: "image", + label: "场景图", + title: "场景图", + active: true, + mentionToken: "@[node:image-node-1]", + }); + + expect(prompt).toContain("@[node:image-node-1]"); + expect(prompt).toContain("canvas_inspect_image"); + expect(prompt).toContain("不要改用 image_text_detect"); + expect(prompt).toContain("previs_scene_create"); + expect(prompt).toContain("previs_scene_read"); + expect(prompt).toContain("previs_apply_patch"); + expect(prompt).toContain("actor"); + expect(prompt).toContain("不要把原图作为 billboard"); + expect(prompt).toContain("普通图片中的人物不是角色卡"); + expect(prompt).toContain("禁止传 characterName"); + expect(prompt).not.toContain("createPrevisBillboard"); + expect(buildImageToPrevisDisplayText()).toBe("根据此图生成 3D 预演台"); + }); + + test("routes the image workflow to a configured vision-capable text model", () => { + const config = { + channels: [ + { + id: "plain", + scope: "system", + models: ["plain-text"], + modelCosts: [ + { + model: "plain-text", + capability: "text", + protocol: "chat-completion", + billingMode: "fixed_request", + unitPriceMicrocredits: 1, + capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 0, maxImageBytes: 0, maxVideos: 0, maxVideoBytes: 0 } } }, + }, + ], + }, + { + id: "vision", + scope: "system", + models: ["vision-text"], + modelCosts: [ + { + model: "vision-text", + capability: "text", + protocol: "chat-completion", + billingMode: "fixed_request", + unitPriceMicrocredits: 1, + capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 3, maxImageBytes: 30 * 1024 * 1024, maxVideos: 0, maxVideoBytes: 0 } } }, + }, + ], + }, + ], + } as AiConfig; + + expect(modelSupportsImageInspection(config, "plain::plain-text")).toBe(false); + expect(selectImageInspectionModel(config, "plain::plain-text")).toBe("vision::vision-text"); + expect(selectImageInspectionModel(config, "vision::vision-text")).toBe("vision::vision-text"); + }); + + test("does not route the cloud Agent through a browser-only custom channel", () => { + const config = { + channels: [ + { + id: "custom", + scope: "user", + models: ["custom-vision"], + modelCosts: [ + { + model: "custom-vision", + capability: "text", + protocol: "chat-completion", + billingMode: "fixed_request", + unitPriceMicrocredits: 1, + capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 3, maxImageBytes: 30 * 1024 * 1024, maxVideos: 0, maxVideoBytes: 0 } } }, + }, + ], + }, + { + id: "managed", + scope: "system", + models: ["managed-vision"], + modelCosts: [ + { + model: "managed-vision", + capability: "text", + protocol: "chat-completion", + billingMode: "fixed_request", + unitPriceMicrocredits: 1, + capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 3, maxImageBytes: 30 * 1024 * 1024, maxVideos: 0, maxVideoBytes: 0 } } }, + }, + ], + }, + ], + } as unknown as AiConfig; + + expect(selectImageInspectionModel(config, "custom::custom-vision")).toBe("managed::managed-vision"); + }); +});