tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(previs): 图片生成预演台 - 从图片节点交给 Agent 复现为可编辑 3D 预演

Browse Source 
图片工具栏新增「生成预演台」，自动选用支持看图的文本模型并提交复现指令。对话用节点卡片展示「此图」，普通图片演员只写名称时不再要求角色卡绑定。
This commit is contained in:
ddcat666
2026-10-09 17:17:28 +08:00
parent 5f322f2f5e
commit 8dcf0a252b
13 changed files with 282 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/internal/app/cloud_agent_previs_mutation.go
View File

@@ -1086,7 +1086,9 @@ func cloudAgentPrevisApplyName(item map[string]any, value *string, label string)
}
func cloudAgentPrevisApplyCharacterBinding(item map[string]any, operation cloudAgentPrevisPatchOperation) error {
provided := operation.CharacterAssetID != nil || operation.CharacterVersionID != nil || operation.ReferenceNodeID != nil || operation.CharacterName != nil
// characterName is descriptive metadata for a real character-card binding;
// a name on an actor from an ordinary reference image must not trigger binding.
provided := operation.CharacterAssetID != nil || operation.CharacterVersionID != nil || operation.ReferenceNodeID != nil
if !provided {
return nil
}
@@ -1122,7 +1124,7 @@ func cloudAgentPrevisApplyCharacterBinding(item map[string]any, operation cloudA
}
func cloudAgentPrevisValidateCharacterBindingInCanvas(doc map[string]any, operation cloudAgentPrevisPatchOperation) error {
if operation.CharacterAssetID == nil && operation.CharacterVersionID == nil && operation.ReferenceNodeID == nil && operation.CharacterName == nil {
if operation.CharacterAssetID == nil && operation.CharacterVersionID == nil && operation.ReferenceNodeID == nil {
return nil
}
if operation.CharacterAssetID == nil || operation.CharacterVersionID == nil || operation.ReferenceNodeID == nil {

18
backend/internal/app/cloud_agent_previs_mutation_test.go
View File

@@ -495,6 +495,24 @@ func TestCloudAgentPrevisPatchLimitsAndClosedArguments(t *testing.T) {
}
}
func TestCloudAgentPrevisOrdinaryActorNameDoesNotRequireCharacterBinding(t *testing.T) {
scene, err := cloudAgentPrevisSceneCreateTemplate(cloudAgentPrevisSceneCreateArgs{SceneID: "image-scene", Title: "图片复现", TemplateID: "empty"})
if err != nil {
t.Fatal(err)
}
_, err = cloudAgentPrevisApplyPatchOperation(scene, cloudAgentPrevisPatchOperation{
Type: "object_add", ID: "image-actor", Kind: "actor", Primitive: "character",
Name: previsStringPtr("道姑"), CharacterName: previsStringPtr("道姑"), Pose: "stand",
}, 0)
if err != nil {
t.Fatalf("ordinary image actor was treated as a character binding: %v", err)
}
actor := cloudAgentPrevisFindByID(creationMaps(scene["objects"]), "image-actor")
if actor == nil || actor["characterBinding"] != nil {
t.Fatalf("ordinary actor unexpectedly has character binding: %#v", actor)
}
}
func TestCloudAgentPrevisCharacterBindingAndAnimationPatch(t *testing.T) {
s, canvas := previsMutationFixture(t)
canvas.PayloadJSON = `{"nodes":[{"id":"hero-card","type":"text","metadata":{"workflowKind":"character","characterAssetId":"hero-asset","characterVersionId":"hero-v1"}}],"connections":[],"previsScenes":[]}`

7
web/src/components/canvas/canvas-cloud-agent-chat-ui.tsx
View File

@@ -84,6 +84,7 @@ export type CloudAgentChatMessage = {
meta?: string;
detail?: unknown;
attachments?: CloudAgentChatAttachment[];
canvasReferenceNodeId?: string;
interjection?: "sent" | "undelivered";
};
@@ -381,6 +382,8 @@ export function AgentChatMessage({
const formAnswer = item.formAnswer ?? (isUser ? parseCloudAgentFormAnswer(item.text) : undefined);
const markdownComponents = useMemo(() => createAgentMessageMarkdownComponents(references, onFocusNode), [onFocusNode, references]);
const agentMarkdownText = rewriteAgentNodeLinks(displayedText, references);
const canvasReference = item.canvasReferenceNodeId ? references.find((reference) => reference.nodeId === item.canvasReferenceNodeId) : undefined;
const userCanvasMarkdown = canvasReference ? item.text.replace("此图", createAgentNodeLink(canvasReference.nodeId, canvasReference)) : item.text;
const errorTone = item.errorSeverity === "warning" ? "warning" : "error";
const color = theme.node.text;
if (item.reasoning) {
@@ -484,6 +487,10 @@ export function AgentChatMessage({
<AIMessageMarkdown className="text-left" isStreaming={isStreaming} streamingAnimation="none" components={markdownComponents}>
{agentMarkdownText}
</AIMessageMarkdown>
) : isUser && canvasReference ? (
<AIMessageMarkdown className="text-left" components={markdownComponents}>
{userCanvasMarkdown}
</AIMessageMarkdown>
) : formAnswer ? (
<AgentFormAnswerCard answer={formAnswer} theme={theme} />
) : (

53
web/src/components/canvas/canvas-cloud-agent-panel.tsx
View File

@@ -42,8 +42,10 @@ import {
saveCloudAgentPendingSubmission,
type CloudAgentConversation,
type CloudAgentPendingSubmission,
type CloudAgentMessagePresentation,
} from "@/services/cloud-agent-conversations";
import { logicalModelIDForConfig, modelOptionName, resolveModelRequestConfig, selectableModelsByCapability, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
import { selectImageInspectionModel } from "@/lib/canvas/image-to-previs-agent";
import { useActiveTheme } from "@/stores/canvas/use-canvas-theme-store";
import { useUserStore } from "@/stores/use-user-store";
import { getActiveUserScope } from "@/lib/user-scope";
@@ -67,8 +69,8 @@ type CloudAgentPanelProps = {
selectedNodeIds: string[];
references: CanvasResourceReference[];
open: boolean;
/** 外部预填请求（如右键“发送到 Agent”）；id 变化即追加一次 text。 */
prefillRequest?: { id: number; text: string } | null;
/** 外部预填请求（如右键“发送到 Agent”）；id 变化即追加一次 text，可选自动提交和视觉模型路由。 */
prefillRequest?: (CloudAgentMessagePresentation & { id: number; text: string; autoSubmit?: boolean; requiresVision?: boolean }) | null;
onOpen: () => void;
onCollapse: () => void;
onFocusNode?: (nodeId: string) => void;
@@ -92,6 +94,8 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
const [messages, setMessages] = useState<CloudAgentChatMessage[]>([]);
const [prompt, setPrompt] = useState("");
const lastPrefillIdRef = useRef(0);
const [pendingAutoSubmit, setPendingAutoSubmit] = useState<CloudAgentPanelProps["prefillRequest"]>(null);
const prefilledSubmissionRef = useRef<CloudAgentPanelProps["prefillRequest"]>(null);
const [reasoningMode, setReasoningMode] = useState<AgentReasoningMode>("off");
const [profileView, setProfileView] = useState<AgentProfileView | null>(null);
const [profileLoading, setProfileLoading] = useState(false);
@@ -319,7 +323,9 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
if (!prefillRequest || prefillRequest.id === lastPrefillIdRef.current) return;
lastPrefillIdRef.current = prefillRequest.id;
if (!prefillRequest.text.trim()) return;
setPrompt((current) => appendAgentPromptPrefill(current, prefillRequest.text));
setPrompt((current) => appendAgentPromptPrefill(current, prefillRequest.displayText || prefillRequest.text));
prefilledSubmissionRef.current = prefillRequest;
setPendingAutoSubmit(prefillRequest.autoSubmit ? prefillRequest : null);
setView("chat");
}, [prefillRequest]);
@@ -490,7 +496,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
const pending = await loadCloudAgentPendingSubmission(canvasId, current.id);
if (!active) return;
pendingSubmission.current = pending;
if (pending?.request) setPrompt(pending.request.prompt);
if (pending?.request) setPrompt(pending.displayText || pending.request.prompt);
} else {
setActiveConversationId(nanoid());
pendingSubmission.current = null;
@@ -550,7 +556,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
if (event.seq > lastSeqRef.current + 1) canvasSyncRef.current?.reconcile();
lastSeqRef.current = event.seq;
}
setMessages((current) => current.filter((item) => item.id !== `stream-error-${run.id}`));
setMessages((current) => current.some((item) => item.id === `stream-error-${run.id}`) ? current.filter((item) => item.id !== `stream-error-${run.id}`) : current);
setContextUsage((current) => reduceAgentContextUsage(current, event));
applyAgentEvent(event, setMessages, setRun, setApproval, setPrompt);
canvasSyncRef.current?.receive(event);
@@ -597,8 +603,14 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
}
};
const submit = async (override?: string) => {
const value = (override ?? prompt).trim();
const submit = async (override?: string, options?: CloudAgentMessagePresentation & { requiresVision?: boolean }) => {
const draft = (override ?? prompt).trim();
const pendingRequest = pendingSubmission.current;
const prefilled = prefilledSubmissionRef.current;
const matchesPending = pendingRequest?.request && pendingRequest.displayText === draft;
const matchesPrefill = prefilled?.displayText === draft;
const value = matchesPending ? pendingRequest.request!.prompt : matchesPrefill ? prefilled!.text : draft;
options = matchesPending ? { ...options, displayText: pendingRequest.displayText, canvasReferenceNodeId: pendingRequest.canvasReferenceNodeId } : matchesPrefill ? prefilled! : options;
if (running) {
await interject(value);
return;
@@ -609,6 +621,8 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
setBusy(true);
let accepted = false;
try {
const effectiveModel = options?.requiresVision ? selectImageInspectionModel(config, selectedModel) : selectedModel;
if (options?.requiresVision && !effectiveModel) throw new Error("当前没有配置支持图片输入的文本模型，请在模型能力设置中为一个文本模型开启图片输入后重试");
const pending = pendingSubmission.current;
// An ambiguous previous POST owns its body/key until reconciled.
// Editing model settings or prompt must not silently create a new charge.
@@ -622,16 +636,16 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
if (selectedSkillIds.length && !capabilities.skills) throw new Error("当前后端尚未接入技能库");
await saveRemoteUserDataNow();
if (currentScope.current !== scope) return;
const agentConfig = { ...config, model: selectedModel };
const requestConfig = resolveModelRequestConfig(agentConfig, selectedModel);
const agentConfig = { ...config, model: effectiveModel };
const requestConfig = resolveModelRequestConfig(agentConfig, effectiveModel);
const logicalModelId = logicalModelIDForConfig(agentConfig);
const input = {
canvasId,
prompt: value,
reasoningMode,
profileRevision: profileView.revision,
model: modelOptionName(selectedModel) || undefined,
...(logicalModelId ? { logicalModelId } : requestConfig.channelId ? { channelId: requestConfig.channelId, channelModelKey: modelOptionName(selectedModel) || undefined } : {}),
model: modelOptionName(effectiveModel) || undefined,
...(logicalModelId ? { logicalModelId } : requestConfig.channelId ? { channelId: requestConfig.channelId, channelModelKey: modelOptionName(effectiveModel) || undefined } : {}),
skillIds: [...new Set([...selectedSkillIds, ...resolveSkillMentions(value, installedSkills).map((skill) => skill.skillId)])],
focusNodeIds: selectedNodeIds.length <= 8 ? selectedNodeIds : [],
permissionMode,
@@ -641,7 +655,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
const fingerprint = JSON.stringify({ scope, parent: run?.id, input });
if (pending && pending.fingerprint !== fingerprint) throw new Error("上一条请求尚未确认，请恢复原消息与设置后核对，不能覆盖原幂等记录");
const key = pending?.key || crypto.randomUUID();
const next = { fingerprint, key, request: { ...input, idempotencyKey: key }, parentRunId: run?.id, messageId: `user-${key}` };
const next = { fingerprint, key, request: { ...input, idempotencyKey: key }, parentRunId: run?.id, messageId: `user-${key}`, displayText: options?.displayText, canvasReferenceNodeId: options?.canvasReferenceNodeId };
// Persist before sending. A failed local save must not submit a request
// whose recovery identity will disappear on reload.
await saveCloudAgentPendingSubmission(canvasId, activeConversationId, next);
@@ -654,8 +668,9 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
const nextMessages = appendUniqueMessage(messages, {
id: submission.messageId || `user-${submission.key}`,
role: "user",
text: formAnswer ? "已确认创作方向" : request.prompt,
text: formAnswer ? "已确认创作方向" : submission.displayText || request.prompt,
...(formAnswer ? { formAnswer } : {}),
...(submission.canvasReferenceNodeId ? { canvasReferenceNodeId: submission.canvasReferenceNodeId } : {}),
});
const now = new Date().toISOString();
const existing = conversations.find((item) => item.id === activeConversationId);
@@ -667,7 +682,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
title: cloudAgentConversationTitle(nextMessages),
messages: nextMessages,
run,
model: selectedModel || undefined,
model: request.model || effectiveModel || undefined,
permissionMode,
skillIds: selectedSkillIds,
createdAt: existing?.createdAt || now,
@@ -677,6 +692,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
]);
if (currentScope.current !== scope) return;
setPrompt("");
prefilledSubmissionRef.current = null;
setMessages(nextMessages);
const result = submission.parentRunId ? await sendAgentMessage(submission.parentRunId, request) : await createAgentRun(request);
accepted = true;
@@ -708,6 +724,13 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
}
};
useEffect(() => {
const pending = pendingAutoSubmit;
if (!pending || !historyHydrated || !pendingHydrated || profileLoading || !profileView || profileError || busy || running) return;
setPendingAutoSubmit(null);
void submit(pending.text, { requiresVision: pending.requiresVision, displayText: pending.displayText, canvasReferenceNodeId: pending.canvasReferenceNodeId });
}, [busy, historyHydrated, pendingAutoSubmit, pendingHydrated, profileError, profileLoading, profileView, running]);
const stop = async () => {
const activeRun = run;
if (!activeRun?.id || stopping) return;
@@ -857,7 +880,7 @@ export function CanvasCloudAgentPanel({ canvasId, domainProjectId, nodeCount, se
if (currentScope.current === `${canvasId}:${conversation.id}`) {
pendingSubmission.current = pending;
setPendingHydrated(true);
if (pending?.request) setPrompt(pending.request.prompt);
if (pending?.request) setPrompt(pending.displayText || pending.request.prompt);
}
})
.catch((cause) => {

13
web/src/components/canvas/canvas-image-toolbar-tools.tsx
View File

@@ -4,7 +4,7 @@ import { Brush, Camera, Clapperboard, Contrast, Copy, FastForward, FileText, Glo
import type { CanvasNodeData } from "@/types/canvas";
import type { NodeToolbarGroup } from "@/lib/canvas/tool-registry";
type ImageNodeActionToolId = "copyPrompt" | "reversePrompt" | "replace" | "resize" | "annotation" | "annotationEdit" | "textEdit" | "maskEdit" | "removeBackground" | "layerDecomposition" | "emotion" | "portraitTexture" | "crop" | "split" | "upscale" | "superResolve" | "angle" | "lighting" | "panorama" | "view" | "multi_camera_nine_grid" | "story_pitch_four_grid" | "character_face_three_view" | "product_three_view" | "storyboard_25_grid" | "character_three_view_generation" | "cinematic_light_correction" | "image_projection_after_3s" | "image_projection_before_5s";
type ImageNodeActionToolId = "copyPrompt" | "reversePrompt" | "replace" | "resize" | "annotation" | "annotationEdit" | "textEdit" | "maskEdit" | "removeBackground" | "layerDecomposition" | "emotion" | "portraitTexture" | "crop" | "split" | "upscale" | "superResolve" | "angle" | "lighting" | "panorama" | "view" | "previs" | "multi_camera_nine_grid" | "story_pitch_four_grid" | "character_face_three_view" | "product_three_view" | "storyboard_25_grid" | "character_three_view_generation" | "cinematic_light_correction" | "image_projection_after_3s" | "image_projection_before_5s";
type ImageToolHandlers = {
onUpload: (node: CanvasNodeData) => void;
@@ -23,6 +23,7 @@ type ImageToolHandlers = {
onAngle: (node: CanvasNodeData) => void;
onLighting: (node: CanvasNodeData) => void;
onPanorama: (node: CanvasNodeData) => void;
onPrevis: (node: CanvasNodeData) => void;
onViewImage: (node: CanvasNodeData) => void;
onCopyPrompt: (node: CanvasNodeData) => void;
onReversePrompt: (node: CanvasNodeData) => void;
@@ -58,6 +59,16 @@ function nineGridRun(node: CanvasNodeData, handlers: ImageToolHandlers, tool: Im
}
const imageToolDefinitions: ImageToolDefinition[] = [
{
id: "previs",
label: "生成预演台",
icon: () => <Layers3 className="size-3.5" />,
group: "primary",
order: 20,
section: "3D 预演",
description: "让 Agent 识别当前图片并复现为可编辑的 3D 预演台",
run: (node, handlers) => handlers.onPrevis(node),
},
{
id: "copyPrompt",
section: "生成信息",

4
web/src/components/canvas/canvas-node-toolbar.tsx
View File

@@ -51,6 +51,7 @@ type CanvasNodeToolbarProps = {
onAngle: (node: CanvasNodeData) => void;
onLighting: (node: CanvasNodeData) => void;
onPanorama: (node: CanvasNodeData) => void;
onPrevis: (node: CanvasNodeData) => void;
onViewImage: (node: CanvasNodeData) => void;
onExtractVideoFrames: (node: CanvasNodeData) => void;
onExtractAudioFromVideo: (node: CanvasNodeData) => void;
@@ -117,6 +118,7 @@ export function CanvasNodeToolbar({
onAngle,
onLighting,
onPanorama,
onPrevis,
onViewImage,
onExtractVideoFrames,
onExtractAudioFromVideo,
@@ -234,7 +236,7 @@ export function CanvasNodeToolbar({
}
copyText(prompt, "提示词已复制");
};
const imageTools = buildImageToolbarTools(node, { onUpload, onToggleFreeResize, onAnnotate, onAnnotationEdit, onTextEdit, onMaskEdit, onRemoveBackground, onLayerDecomposition, onEmotion, onPortraitTexture, onCrop, onUpscale, onSuperResolve, onAngle, onLighting, onPanorama, onViewImage, onCopyPrompt: copyImagePrompt, onReversePrompt, onNineGrid });
const imageTools = buildImageToolbarTools(node, { onUpload, onToggleFreeResize, onAnnotate, onAnnotationEdit, onTextEdit, onMaskEdit, onRemoveBackground, onLayerDecomposition, onEmotion, onPortraitTexture, onCrop, onUpscale, onSuperResolve, onAngle, onLighting, onPanorama, onPrevis, onViewImage, onCopyPrompt: copyImagePrompt, onReversePrompt, onNineGrid });
// 构建 ToolContext——供注册表解析工具
const nodeHoverHandlers = {

34
web/src/lib/canvas/image-to-previs-agent.ts Normal file
View File

@@ -0,0 +1,34 @@
import { canvasResourceMentionToken, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
import { resolveModelChannel, selectableModelsByCapability, type AiConfig } from "@/stores/use-config-store";
export const IMAGE_TO_PREVIS_AGENT_INSTRUCTION = [
"请把这张画布图片复现为可编辑的 3D 预演台，而不是把图片放进预演台。",
"先调用 canvas_inspect_image 读取该图片的真实画面，不能根据节点标题、提示词或文件名猜测内容。",
"如果工具列表中没有 canvas_inspect_image，立即停止并报告当前模型不支持图片输入；不要改用 image_text_detect，因为它只准备文字识别请求，不会把画面交给模型。",
"然后调用 canvas_get_state 和 previs_scene_read 读取画布及预演台状态：如果已有对应预演场景就更新它，否则调用 previs_scene_create 创建一个空场景。",
"根据实际观察，用 previs_apply_patch 建立可移动的语义对象：逐个还原人物数量、人物/道具/环境的前后景关系、站位、朝向、相对尺度、姿态和动作；用相机、镜头构图、灯光和环境设置还原画面视角与氛围。",
"人物必须使用 actor，对象必须能在预演台中单独移动、旋转和调整姿态；不要只写一段描述，也不要把原图作为 billboard、背景板、参考平面或任何图片对象加入场景。",
"普通图片中的人物不是角色卡：使用 actor 的 name、archetype、pose 和 position 描述即可，禁止传 characterName、characterAssetId、characterVersionId 或 referenceNodeId。只有 canvas_get_state 明确返回 workflowKind=character 且三个角色卡 ID 完整匹配时，才允许角色绑定。",
"完成后重新读取预演场景确认对象数量、空间布局和镜头设置，并明确说明无法从图片确认的部分。",
].join("\n");
export function buildImageToPrevisAgentPrompt(reference: CanvasResourceReference) {
return `${canvasResourceMentionToken(reference)}\n${IMAGE_TO_PREVIS_AGENT_INSTRUCTION}`;
}
export function buildImageToPrevisDisplayText() {
return "根据此图生成 3D 预演台";
}
export function modelSupportsImageInspection(config: AiConfig, model: string) {
return (modelCapabilityConfigFor(config, model).text?.references.maxImages || 0) > 0;
}
/** 图片复现必须在真正支持图片输入的文本模型上运行。 */
export function selectImageInspectionModel(config: AiConfig, preferredModel: string) {
const textModels = selectableModelsByCapability(config, "text");
const canCloudAgentUse = (model: string) => resolveModelChannel(config, model).scope === "system" && modelSupportsImageInspection(config, model);
if (preferredModel && canCloudAgentUse(preferredModel)) return preferredModel;
return textModels.find(canCloudAgentUse) || "";
}

29
web/src/pages/canvas/project.tsx
View File

@@ -91,6 +91,8 @@ import {
replaceCanvasReferenceMentions,
type CanvasResourceReference,
} from "@/lib/canvas/canvas-resource-references";
import { buildImageToPrevisAgentPrompt, buildImageToPrevisDisplayText } from "@/lib/canvas/image-to-previs-agent";
import type { CloudAgentMessagePresentation } from "@/services/cloud-agent-conversations";
import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays";
import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer";
import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer";
@@ -241,7 +243,7 @@ function CanvasViewportPage() {
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
const [contextMenu, setContextMenu] = useState<ContextMenuState | null>(null);
// 每次发送带递增 id：重复发送同一节点时文本相同，仍需触发一次追加。
const [agentPrefillRequest, setAgentPrefillRequest] = useState<{ id: number; text: string } | null>(null);
const [agentPrefillRequest, setAgentPrefillRequest] = useState<(CloudAgentMessagePresentation & { id: number; text: string; autoSubmit?: boolean; requiresVision?: boolean }) | null>(null);
const [isMiniMapOpen, setIsMiniMapOpen] = useState(false);
const [canvasAppearance, setCanvasAppearance] = useState<CanvasAppearance>(() => canvasAppearanceForTheme(colorTheme));
const [backgroundMode, setBackgroundMode] = useState<CanvasBackgroundMode>(DEFAULT_CANVAS_BACKGROUND_MODE);
@@ -423,6 +425,30 @@ function CanvasViewportPage() {
openAgent();
setContextMenu(null);
}, [agentMentionReferences, openAgent]);
const sendImageToPrevisAgent = useCallback((node: CanvasNodeData) => {
if (node.type !== CanvasNodeType.Image) return;
const reference = agentMentionReferences.find((item) => item.nodeId === node.id && item.kind === "image");
if (!reference) {
message.warning("这张图片暂时没有可供 Agent 引用的画布资源");
return;
}
if (!selectedNodeIdsRef.current.has(node.id)) {
const selection = new Set([node.id]);
selectedNodeIdsRef.current = selection;
setSelectedNodeIds(selection);
}
setAgentPrefillRequest((current) => ({
id: (current?.id ?? 0) + 1,
text: buildImageToPrevisAgentPrompt(reference),
displayText: buildImageToPrevisDisplayText(),
canvasReferenceNodeId: reference.nodeId,
autoSubmit: true,
requiresVision: true,
}));
openAgent();
setContextMenu(null);
}, [agentMentionReferences, message, openAgent]);
// 修复素材关联仍遵守当前画布版本，不能替用户确认覆盖云端的新内容。
const confirmForceSaveCanvas = useCallback(() => {
modal.confirm({
@@ -2978,6 +3004,7 @@ function CanvasViewportPage() {
setLightingNodeId((current) => (current === node.id ? null : node.id));
}}
onPanorama={openPanoramaConfig}
onPrevis={sendImageToPrevisAgent}
onViewImage={(node) => setPreviewNodeId(node.id)}
onExtractVideoFrames={openVideoFrameExtractor}
onExtractAudioFromVideo={(node) => void extractAudioFromVideo(node)}

2
web/src/pages/canvas/shared.tsx
View File

@@ -282,7 +282,7 @@ export default function SharedCanvasPage() {
}} onHoverStart={keepToolbar} onHoverEnd={hideToolbar} onConnectStart={unauthorized} onResize={() => undefined} onContentChange={unauthorized} onRetry={unauthorized} onOpenTaskDetails={unauthorized} onViewImage={(target) => setInfoNodeId(target.id)} onContextMenu={(event, nodeId) => openContextMenu(event, nodeId)} />)}
</CanvasViewport>
<CanvasNodeToolbar node={dragRef.current ? null : toolbarNode} viewport={viewport} containerRef={containerRef} onKeep={keepToolbar} onLeave={hideToolbar} onInfo={(node) => setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onAnnotationEdit={unauthorized} onTextEdit={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onRemoveBackground={unauthorized} onLayerDecomposition={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onLighting={unauthorized} onPanorama={unauthorized} onViewImage={unauthorized} onExtractVideoFrames={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoSegments={unauthorized} extractingVideoFrames={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} onNineGrid={unauthorized} />
<CanvasNodeToolbar node={dragRef.current ? null : toolbarNode} viewport={viewport} containerRef={containerRef} onKeep={keepToolbar} onLeave={hideToolbar} onInfo={(node) => setInfoNodeId(node.id)} onEditText={unauthorized} onDecreaseFont={unauthorized} onIncreaseFont={unauthorized} onToggleDialog={unauthorized} onAnnotate={unauthorized} onAnnotationEdit={unauthorized} onTextEdit={unauthorized} onGenerateImage={unauthorized} onUpload={unauthorized} onDownload={unauthorized} onSaveAsset={unauthorized} onMaskEdit={unauthorized} onRemoveBackground={unauthorized} onLayerDecomposition={unauthorized} onEmotion={unauthorized} onPortraitTexture={unauthorized} onCrop={unauthorized} onSplit={unauthorized} onUpscale={unauthorized} onSuperResolve={unauthorized} onAngle={unauthorized} onLighting={unauthorized} onPanorama={unauthorized} onPrevis={unauthorized} onViewImage={unauthorized} onExtractVideoFrames={unauthorized} onExtractAudioFromVideo={unauthorized} onTrimVideoSegments={unauthorized} extractingVideoFrames={false} extractingAudio={false} trimmingVideo={false} onSubtitles={unauthorized} onTimeline={unauthorized} onReversePrompt={unauthorized} onRetry={unauthorized} onToggleFreeResize={unauthorized} onToggleLocked={unauthorized} onDelete={unauthorized} onNineGrid={unauthorized} />
<div className="absolute bottom-5 left-5 z-[var(--z-panel-floating)]"><CanvasZoomControls scale={viewport.k} containerRef={containerRef} onScaleChange={setZoom} onFitContent={resetViewport} isMiniMapOpen={false} onToggleMiniMap={unauthorized} onOpenShortcuts={unauthorized} /></div>
<div className="pointer-events-none absolute bottom-5 right-5 z-[var(--z-panel-floating)] max-w-[340px] text-right text-xs leading-5" style={{ color: theme.node.muted }}>访客操作仅在当前页面临时生效</div>

9
web/src/services/cloud-agent-conversations.ts
View File

@@ -3,6 +3,11 @@ import { markdownPlainText } from "@/lib/markdown-plain-text";
import { getActiveUserScope } from "@/lib/user-scope";
import type { AgentPermissionMode, AgentRun } from "@/services/api/agent";
export type CloudAgentMessagePresentation = {
displayText?: string;
canvasReferenceNodeId?: string;
};
export type CloudAgentConversationMessage = {
id: string;
role: "user" | "assistant" | "system" | "tool" | "error";
@@ -37,7 +42,7 @@ type CloudAgentConversationDocument = {
conversations: CloudAgentConversation[];
};
export type CloudAgentPendingSubmission = {
export type CloudAgentPendingSubmission = CloudAgentMessagePresentation & {
fingerprint: string;
key: string;
request?: import("@/services/api/agent").CreateAgentRunInput;
@@ -89,7 +94,7 @@ export async function loadCloudAgentPendingSubmission(canvasId: string, conversa
if (parsed.request !== undefined && (!parsed.request || parsed.request.idempotencyKey !== parsed.key || parsed.request.canvasId !== canvasId || typeof parsed.request.prompt !== "string")) throw new Error("invalid request");
if (parsed.parentRunId !== undefined && typeof parsed.parentRunId !== "string") throw new Error("invalid parent");
if (parsed.messageId !== undefined && typeof parsed.messageId !== "string") throw new Error("invalid message");
return { fingerprint: parsed.fingerprint, key: parsed.key, request: parsed.request, parentRunId: parsed.parentRunId, messageId: parsed.messageId } satisfies CloudAgentPendingSubmission;
return { fingerprint: parsed.fingerprint, key: parsed.key, request: parsed.request, parentRunId: parsed.parentRunId, messageId: parsed.messageId, displayText: parsed.displayText, canvasReferenceNodeId: parsed.canvasReferenceNodeId } satisfies CloudAgentPendingSubmission;
} catch {
throw new Error("待确认请求记录损坏，已暂停本对话发送；请先在任务中心核对原运行，不要直接重复生成");
}

17
web/test/agent-chat-node-links.test.tsx
View File

@@ -64,6 +64,23 @@ test("renders an agent node link as a titled inline node card", () => {
expect(visibleText).not.toContain("video-1790385749473-q4157");
});
test("renders the image-to-previs user request with the existing canvas node card", () => {
const imageReference = { ...videoReference, kind: "image" as const, sourceType: "image" as const, label: "道姑", title: "道姑" };
const html = renderToStaticMarkup(
<AgentChatMessage
item={{ id: "user-previs", role: "user", text: "根据此图生成 3D 预演台", canvasReferenceNodeId: imageReference.nodeId }}
theme={canvasThemes.dark}
references={[imageReference]}
onFocusNode={() => {}}
/>,
);
expect(html).toContain('class="agent-message-node-link"');
expect(html).toContain('class="agent-message-node-link-preview"');
expect(html).toContain("道姑");
expect(html).toContain("3D 预演台");
});
test("labels character cards by resource kind instead of the underlying text type", () => {
const nodeId = "text-1790385749473-char1";
const reference = { id: nodeId, nodeId, kind: "character" as const, label: "李莫愁", title: "李莫愁", sourceType: "text" as const };

2
web/test/canvas-node-toolbar.test.ts
View File

@@ -91,5 +91,7 @@ describe("canvas node toolbar model", () => {
expect(tools.find((tool) => tool.id === "crop")?.section).toBe("构图与尺寸");
expect(tools.find((tool) => tool.id === "resize")?.active).toBe(true);
expect(tools.find((tool) => tool.id === "copyPrompt")?.group).toBe("more");
expect(tools.find((tool) => tool.id === "previs")?.group).toBe("primary");
expect(tools.find((tool) => tool.id === "previs")?.description).toContain("Agent");
});
});

111
web/test/image-to-previs-agent.test.ts Normal file
View File

@@ -0,0 +1,111 @@
import { describe, expect, test } from "bun:test";
import { buildImageToPrevisAgentPrompt, buildImageToPrevisDisplayText, modelSupportsImageInspection, selectImageInspectionModel } from "@/lib/canvas/image-to-previs-agent";
import type { AiConfig } from "@/stores/use-config-store";
describe("image to previs Agent prompt", () => {
test("requires visual inspection and editable semantic objects", () => {
const prompt = buildImageToPrevisAgentPrompt({
id: "image-ref",
nodeId: "image-node-1",
kind: "image",
label: "场景图",
title: "场景图",
active: true,
mentionToken: "@[node:image-node-1]",
});
expect(prompt).toContain("@[node:image-node-1]");
expect(prompt).toContain("canvas_inspect_image");
expect(prompt).toContain("不要改用 image_text_detect");
expect(prompt).toContain("previs_scene_create");
expect(prompt).toContain("previs_scene_read");
expect(prompt).toContain("previs_apply_patch");
expect(prompt).toContain("actor");
expect(prompt).toContain("不要把原图作为 billboard");
expect(prompt).toContain("普通图片中的人物不是角色卡");
expect(prompt).toContain("禁止传 characterName");
expect(prompt).not.toContain("createPrevisBillboard");
expect(buildImageToPrevisDisplayText()).toBe("根据此图生成 3D 预演台");
});
test("routes the image workflow to a configured vision-capable text model", () => {
const config = {
channels: [
{
id: "plain",
scope: "system",
models: ["plain-text"],
modelCosts: [
{
model: "plain-text",
capability: "text",
protocol: "chat-completion",
billingMode: "fixed_request",
unitPriceMicrocredits: 1,
capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 0, maxImageBytes: 0, maxVideos: 0, maxVideoBytes: 0 } } },
},
],
},
{
id: "vision",
scope: "system",
models: ["vision-text"],
modelCosts: [
{
model: "vision-text",
capability: "text",
protocol: "chat-completion",
billingMode: "fixed_request",
unitPriceMicrocredits: 1,
capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 3, maxImageBytes: 30 * 1024 * 1024, maxVideos: 0, maxVideoBytes: 0 } } },
},
],
},
],
} as AiConfig;
expect(modelSupportsImageInspection(config, "plain::plain-text")).toBe(false);
expect(selectImageInspectionModel(config, "plain::plain-text")).toBe("vision::vision-text");
expect(selectImageInspectionModel(config, "vision::vision-text")).toBe("vision::vision-text");
});
test("does not route the cloud Agent through a browser-only custom channel", () => {
const config = {
channels: [
{
id: "custom",
scope: "user",
models: ["custom-vision"],
modelCosts: [
{
model: "custom-vision",
capability: "text",
protocol: "chat-completion",
billingMode: "fixed_request",
unitPriceMicrocredits: 1,
capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 3, maxImageBytes: 30 * 1024 * 1024, maxVideos: 0, maxVideoBytes: 0 } } },
},
],
},
{
id: "managed",
scope: "system",
models: ["managed-vision"],
modelCosts: [
{
model: "managed-vision",
capability: "text",
protocol: "chat-completion",
billingMode: "fixed_request",
unitPriceMicrocredits: 1,
capabilityConfig: { version: 1, text: { contextWindowTokens: 128000, maxOutputTokens: 16384, references: { promptMaxChars: 32000, maxImages: 3, maxImageBytes: 30 * 1024 * 1024, maxVideos: 0, maxVideoBytes: 0 } } },
},
],
},
],
} as unknown as AiConfig;
expect(selectImageInspectionModel(config, "custom::custom-vision")).toBe("managed::managed-vision");
});
});