tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): 画布 Agent - 重构工作流上下文与技能交互

Browse Source
This commit is contained in:
ddCat
2026-08-24 12:09:15 +08:00
parent 919037e5d9
commit 326572ea03
33 changed files with 2230 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
web/src/components/canvas/canvas-agent-chat-ui.tsx
View File

@@ -1,5 +1,6 @@
import { useEffect, useRef, useState, type ClipboardEvent as ReactClipboardEvent, type KeyboardEvent as ReactKeyboardEvent, type ReactNode } from "react";
import { Button, Tooltip } from "antd";
import { motion, useReducedMotion } from "motion/react";
import { ArrowUp, CheckCircle2, CircleAlert, ImagePlus, LoaderCircle, Sparkles, UserRound, Wrench, X, XCircle } from "lucide-react";
import { canvasThemes } from "@/lib/canvas-theme";
@@ -22,12 +23,36 @@ export type CanvasAgentChatMessage = {
attachments?: CanvasAgentChatAttachment[];
};
export type CanvasAgentQuickAction = { label: string; prompt: string };
/**
* Turn the short numbered choices the Agent already emits into real UI actions.
* This deliberately stays conservative: only assistant messages with 1–4
* numbered lines are eligible, and code blocks are ignored.
*/
export function extractCanvasAgentQuickActions(text: string): CanvasAgentQuickAction[] {
if (!text.trim() || text.includes("```")) return [];
const actions: CanvasAgentQuickAction[] = [];
const seen = new Set<string>();
for (const line of text.split(/\r?\n/u)) {
const match = /^\s*(?:[-*]\s*)?(\d{1,2})[.)、]\s*(.+?)\s*$/u.exec(line);
if (!match) continue;
const label = match[2].replace(/^[*_\s]+|[*_\s]+$/gu, "").trim();
if (!label || label.length > 96 || seen.has(label)) continue;
seen.add(label);
actions.push({ label, prompt: label });
if (actions.length >= 4) break;
}
return actions;
}
const WORKING_TEXT = "正在推演...";
export function AgentChatMessage({ item, theme, user, isStreaming = false, onRejectTool, onApproveTool }: { item: CanvasAgentChatMessage; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; user: LocalUser | null; isStreaming?: boolean; onRejectTool?: (id: string) => void; onApproveTool?: (id: string) => void }) {
export function AgentChatMessage({ item, theme, user, isStreaming = false, onRejectTool, onApproveTool, onQuickAction }: { item: CanvasAgentChatMessage; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; user: LocalUser | null; isStreaming?: boolean; onRejectTool?: (id: string) => void; onApproveTool?: (id: string) => void; onQuickAction?: (prompt: string) => void }) {
const isUser = item.role === "user";
const isSystem = item.role === "system";
const color = item.role === "error" ? "#dc2626" : item.role === "tool" ? "#2563eb" : theme.node.text;
const quickActions = item.role === "assistant" && !isStreaming ? extractCanvasAgentQuickActions(item.text) : [];
if (isSystem) {
return (
<div className="flex justify-center text-xs">
@@ -52,6 +77,22 @@ export function AgentChatMessage({ item, theme, user, isStreaming = false, onRej
{!isUser ? <AgentAvatar theme={theme} /> : null}
<div className={`min-w-0 max-w-[86%] text-sm leading-6 ${isUser ? "rounded-md px-3 py-2.5 text-right" : "text-left"}`} style={{ color, ...(isUser ? { background: theme.accent.primarySoft } : {}) }}>
{item.role === "assistant" ? <AIMessageMarkdown className="text-left" isStreaming={isStreaming}>{item.text}</AIMessageMarkdown> : <div className="whitespace-pre-wrap break-words text-left">{item.text}</div>}
{quickActions.length && onQuickAction ? (
<div className="mt-3 flex flex-wrap gap-1.5" aria-label="快捷选项">
{quickActions.map((action) => (
<motion.button
key={action.label}
type="button"
className="rounded-full px-3 py-1.5 text-left text-xs font-medium outline-none transition-[background-color,transform,box-shadow] duration-200 focus-visible:ring-2 focus-visible:ring-current/30 hover:-translate-y-px"
style={{ background: theme.spatial.surface, color: theme.node.text, boxShadow: `0 4px 14px ${theme.spatial.shadow}` }}
whileTap={{ scale: 0.97 }}
onClick={() => onQuickAction(action.prompt)}
>
{action.label}
</motion.button>
))}
</div>
) : null}
{item.attachments?.length ? <AgentMessageAttachments attachments={item.attachments} /> : null}
{item.meta ? <div className="mt-1 text-[var(--fs-label)] opacity-45">{item.meta}</div> : null}
</div>
@@ -209,6 +250,7 @@ export function AgentChatComposer({
const [slashIndex, setSlashIndex] = useState(0);
const availableSlashSkills = slashSkills ?? [];
const canSubmit = !disabled && !sending && Boolean(prompt.trim() || attachments.length);
const reducedMotion = useReducedMotion();
const activeSlashIndex = Math.min(Math.max(slashIndex, 0), Math.max(availableSlashSkills.length - 1, 0));
// 在输入值末尾检测「/关键词」打开技能候选；选择后替换为 @[skill:xxx] 引用 token（保持在 prompt 文本里）。
@@ -270,15 +312,22 @@ export function AgentChatComposer({
};
return (
<div className="px-3 pb-3 pt-1" onWheelCapture={(event) => event.stopPropagation()}>
<div className="rounded-lg border px-3 pb-2.5 pt-3 transition-[border-color,box-shadow] duration-150 focus-within:border-current" style={{ background: theme.node.fill, borderColor: theme.toolbar.border, color: theme.accent.primary, boxShadow: `0 10px 30px ${theme.spatial.shadow}` }}>
<div className="px-3 pb-3 pt-2" onWheelCapture={(event) => event.stopPropagation()}>
<div
className="group/composer rounded-[22px] px-3 pb-2.5 pt-3 transition-[background-color,box-shadow,transform] duration-200 focus-within:-translate-y-px"
style={{
background: theme.node.fill,
color: theme.accent.primary,
boxShadow: `0 16px 40px ${theme.spatial.shadow}, inset 0 1px 0 rgba(255,255,255,0.045)`,
}}
>
{attachments.length ? (
<div className="thin-scrollbar mb-2 flex gap-2 overflow-x-auto pb-1">
{attachments.map((item) => (
<div key={item.id} className="group relative size-14 shrink-0 overflow-hidden rounded-md" title={item.name}>
<img src={item.url} alt={item.name} className="size-full object-cover" />
{onRemoveAttachment ? (
<button type="button" className="absolute right-1 top-1 grid size-5 place-items-center rounded-full border opacity-0 shadow-sm transition group-hover:opacity-100" style={{ background: theme.toolbar.panel, borderColor: theme.node.stroke, color: theme.node.text }} onClick={() => onRemoveAttachment(item.id)} aria-label="移除图片">
<button type="button" className="absolute right-1 top-1 grid size-5 place-items-center rounded-full opacity-0 shadow-sm transition group-hover:opacity-100" style={{ background: theme.toolbar.panel, color: theme.node.text }} onClick={() => onRemoveAttachment(item.id)} aria-label="移除图片">
<X className="size-3" />
</button>
) : null}
@@ -306,8 +355,8 @@ export function AgentChatComposer({
{slash && availableSlashSkills.length ? (
<div
data-agent-slash-menu
className="absolute bottom-full left-0 z-[var(--z-toolbar)] mb-1.5 w-full max-w-xs overflow-hidden rounded-lg border p-1 shadow-lg"
style={{ background: theme.toolbar.panel, borderColor: theme.toolbar.border }}
className="absolute bottom-full left-0 z-[var(--z-toolbar)] mb-2 w-full max-w-xs overflow-hidden rounded-2xl p-1.5 shadow-2xl"
style={{ background: theme.toolbar.panel, boxShadow: `0 18px 44px ${theme.spatial.shadow}` }}
onMouseDown={(event) => event.preventDefault()}
>
{availableSlashSkills.map((skill, index) => (
@@ -336,13 +385,38 @@ export function AgentChatComposer({
event.target.value = "";
}} />
<Tooltip title="上传图片">
<Button type="text" shape="circle" className="!h-8 !w-8 !min-w-8" disabled={sending} style={{ color: theme.node.muted }} icon={<ImagePlus className="size-4" />} onClick={() => fileInputRef.current?.click()} />
<Button type="text" shape="circle" className="!h-8 !w-8 !min-w-8 !transition-transform hover:!scale-105 active:!scale-95" disabled={sending} style={{ color: theme.node.muted }} icon={<ImagePlus className="size-4" />} onClick={() => fileInputRef.current?.click()} />
</Tooltip>
</>
) : null}
{left}
</div>
<Button type="primary" className="!h-8 !w-8 !min-w-8 !rounded-md !p-0" disabled={!canSubmit} icon={sending ? <LoaderCircle className="size-4 animate-spin" /> : <ArrowUp className="size-4" />} onClick={() => void onSubmit()} aria-label="发送" />
<motion.button
type="button"
disabled={!canSubmit}
aria-label={sending ? "发送中" : "发送"}
onClick={() => void onSubmit()}
whileHover={canSubmit && !reducedMotion ? { scale: 1.06, y: -1 } : undefined}
whileTap={canSubmit && !reducedMotion ? { scale: 0.9, y: 1 } : undefined}
animate={sending && !reducedMotion ? { scale: [1, 0.94, 1], rotate: [0, -5, 5, 0] } : { scale: 1, rotate: 0 }}
transition={sending && !reducedMotion ? { duration: 0.42, ease: "easeOut" } : { type: "spring", stiffness: 420, damping: 24 }}
className="grid size-9 shrink-0 place-items-center rounded-full p-0 outline-none transition-[background-color,box-shadow,color,transform] duration-200 focus-visible:ring-2 focus-visible:ring-current/35 disabled:cursor-not-allowed"
style={{
background: canSubmit || sending ? theme.accent.primary : theme.spatial.surface,
color: canSubmit || sending ? theme.accent.onPrimary : theme.node.muted,
boxShadow: canSubmit || sending ? `0 8px 20px ${theme.accent.primary}45` : "none",
}}
>
<motion.span
key={sending ? "sending" : "ready"}
initial={reducedMotion ? false : { opacity: 0, scale: 0.65, rotate: sending ? -25 : 25 }}
animate={{ opacity: 1, scale: 1, rotate: 0 }}
transition={{ duration: reducedMotion ? 0 : 0.18, ease: "easeOut" }}
className="grid place-items-center"
>
{sending ? <LoaderCircle className="size-4 animate-spin" /> : <ArrowUp className="size-4" />}
</motion.span>
</motion.button>
</div>
</div>
</div>
@@ -351,11 +425,11 @@ export function AgentChatComposer({
export function AgentPanelTabs<T extends string>({ value, items, theme, right, onChange }: { value: T; items: { value: T; label: string; icon?: ReactNode; count?: number }[]; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; right?: ReactNode; onChange: (value: T) => void }) {
return (
<div className="shrink-0 px-3 pb-2">
<div className="flex min-h-9 items-center justify-between gap-2 rounded-md p-1" style={{ background: theme.spatial.surface }}>
<nav className="grid min-w-0 flex-1 grid-flow-col auto-cols-fr items-center gap-0.5 text-xs" role="tablist" aria-label="Agent 面板">
<div className="shrink-0 px-3 pb-1">
<div className="flex min-h-8 items-center justify-between gap-2 rounded-lg px-0.5 py-0.5" style={{ background: "transparent" }}>
<nav className="grid min-w-0 flex-1 grid-flow-col auto-cols-fr items-center gap-0.5 text-[var(--fs-label)]" role="tablist" aria-label="Agent 面板">
{items.map((item) => (
<button key={item.value} type="button" role="tab" aria-selected={value === item.value} className={`inline-flex h-8 min-w-0 items-center justify-center gap-1 rounded-[var(--r-sm)] px-1.5 transition-colors ${value === item.value ? "font-medium" : "font-normal"}`} style={{ background: value === item.value ? theme.node.fill : "transparent", color: value === item.value ? theme.node.text : theme.node.muted, boxShadow: value === item.value ? `0 1px 5px ${theme.spatial.shadow}` : "none" }} onClick={() => onChange(item.value)}>
<button key={item.value} type="button" role="tab" aria-selected={value === item.value} className={`inline-flex h-7 min-w-0 items-center justify-center gap-1 rounded-md px-1.5 transition-colors ${value === item.value ? "font-medium" : "font-normal"}`} style={{ background: value === item.value ? theme.node.fill : "transparent", color: value === item.value ? theme.node.text : theme.node.muted, boxShadow: value === item.value ? `0 2px 8px ${theme.spatial.shadow}` : "none" }} onClick={() => onChange(item.value)}>
<span className="shrink-0">{item.icon}</span>
<span className="min-w-0 truncate">{item.label}</span>
{item.count ? <span className="shrink-0 tabular-nums opacity-60">{item.count}</span> : null}

66
web/src/components/canvas/canvas-agent-panel-chrome.tsx
View File

@@ -1,5 +1,6 @@
import { Button, Switch, Tooltip } from "antd";
import { BookOpenText, Bot, Clapperboard, Focus, Globe2, LayoutTemplate, Laptop, PanelRightClose, PanelsTopLeft, RotateCcw, Workflow } from "lucide-react";
import { BookOpenCheck, BookOpenText, Bot, Clapperboard, Focus, Globe2, History, LayoutTemplate, Laptop, PanelRightClose, PanelsTopLeft, Plus, RotateCcw, Workflow } from "lucide-react";
import { useNavigate } from "react-router";
import type { CanvasContextSummary } from "@/lib/canvas/canvas-context-summary";
import type { CanvasTheme } from "@/lib/canvas-theme";
@@ -18,6 +19,11 @@ export function AgentPanelChrome({
onConfirmToolsChange,
onUndo,
onCollapse,
historyCount = 0,
historyActive = false,
onOpenHistory,
onNewChat,
newChatDisabled = false,
}: {
theme: CanvasTheme;
mode: CanvasAgentMode;
@@ -30,35 +36,61 @@ export function AgentPanelChrome({
onConfirmToolsChange: (confirm: boolean) => void;
onUndo: () => void;
onCollapse: () => void;
historyCount?: number;
historyActive?: boolean;
onOpenHistory?: () => void;
onNewChat?: () => void;
newChatDisabled?: boolean;
}) {
const navigate = useNavigate();
return (
<header className="shrink-0 px-3 pb-2 pt-3">
<div className="flex min-w-0 items-center gap-2.5">
<span className="grid size-9 shrink-0 place-items-center rounded-md" style={{ background: theme.accent.primarySoft, color: theme.accent.primary }}>
<Bot className="size-[18px]" />
<header className="shrink-0 px-3 pb-1.5 pt-2.5">
<div className="flex min-w-0 items-center gap-2">
<span className="grid size-8 shrink-0 place-items-center rounded-lg" style={{ background: theme.accent.primarySoft, color: theme.accent.primary }}>
<Bot className="size-4" />
</span>
<div className="min-w-0 flex-1">
<div className="truncate text-sm font-semibold leading-5">Agent</div>
<div className="truncate text-[var(--fs-label)] leading-4" style={{ color: theme.node.muted }}>画布协作</div>
<div className="flex min-w-0 items-baseline gap-1.5">
<div className="truncate text-sm font-semibold leading-5">Agent</div>
<span className="truncate text-[var(--fs-label)]" style={{ color: theme.node.muted }}>画布协作</span>
</div>
</div>
<div className="ml-auto flex shrink-0 items-center gap-0.5">
<Tooltip title="技能库">
<Button type="text" shape="circle" className="!h-7 !w-7 !min-w-7" style={{ color: theme.node.muted }} icon={<BookOpenCheck className="size-3.5" />} onClick={() => navigate("/skills")} aria-label="打开技能库" />
</Tooltip>
{onOpenHistory ? (
<Tooltip title={historyCount ? `历史会话 · ${historyCount}` : "历史会话"}>
<Button type="text" className={`!h-7 !min-w-7 !px-1.5 ${historyActive ? "font-medium" : ""}`} style={{ color: historyActive ? theme.node.text : theme.node.muted, background: historyActive ? theme.spatial.surface : "transparent" }} icon={<History className="size-3.5" />} onClick={onOpenHistory} aria-label="打开历史会话">
{historyCount ? <span className="text-[var(--fs-tiny)] tabular-nums">{historyCount}</span> : null}
</Button>
</Tooltip>
) : null}
{onNewChat ? (
<Tooltip title="新对话">
<Button type="text" shape="circle" className="!h-7 !w-7 !min-w-7" disabled={newChatDisabled} style={{ color: theme.node.muted }} icon={<Plus className="size-3.5" />} onClick={onNewChat} aria-label="新建对话" />
</Tooltip>
) : null}
<Tooltip title="收起 Agent">
<Button type="text" shape="circle" className="!h-7 !w-7 !min-w-7" style={{ color: theme.node.muted }} icon={<PanelRightClose className="size-3.5" />} onClick={onCollapse} aria-label="收起 Agent" />
</Tooltip>
</div>
<AgentModeSwitch value={mode} theme={theme} onChange={onModeChange} />
<Tooltip title="收起 Agent">
<Button type="text" shape="circle" className="!h-8 !w-8 !min-w-8" style={{ color: theme.node.muted }} icon={<PanelRightClose className="size-4" />} onClick={onCollapse} />
</Tooltip>
</div>
<div className="mt-2 flex min-h-8 flex-wrap items-center gap-x-2.5 gap-y-1 px-0.5 text-[var(--fs-label)]" style={{ color: theme.node.muted }}>
<div className="mt-1 flex min-h-7 flex-wrap items-center gap-x-2 gap-y-0.5 px-0.5 text-[var(--fs-label)]" style={{ color: theme.node.muted }}>
<span className="font-medium" style={{ color: theme.node.text }}>{context.nodeCount} 个节点</span>
{context.selectedCount ? <span className="inline-flex items-center gap-1"><Focus className="size-3" />选中 {context.selectedCount}</span> : <span>未选择节点</span>}
{context.chapterLabel ? <span className="inline-flex min-w-0 items-center gap-1"><BookOpenText className="size-3 shrink-0" /><span className="max-w-32 truncate">{context.chapterLabel}{context.shotLabel ? ` · ${context.shotLabel}` : ""}</span></span> : null}
{referenceCount ? <span>{referenceCount} 个参考</span> : null}
<div className="ml-auto flex shrink-0 items-center gap-1.5">
<AgentModeSwitch value={mode} theme={theme} onChange={onModeChange} />
<Tooltip title={undoCount ? `撤销最近一批 Agent 写回，可撤销 ${undoCount} 批` : "没有可撤销的 Agent 写回"}>
<Button type="text" shape="circle" className="!h-7 !w-7 !min-w-7" disabled={!canUndo} style={{ color: theme.node.muted }} icon={<RotateCcw className="size-3.5" />} onClick={onUndo} aria-label="撤销最近一批 Agent 写回" />
<Button type="text" shape="circle" className="!h-6 !w-6 !min-w-6" disabled={!canUndo} style={{ color: theme.node.muted }} icon={<RotateCcw className="size-3" />} onClick={onUndo} aria-label="撤销最近一批 Agent 写回" />
</Tooltip>
<label className="flex h-7 cursor-pointer items-center gap-1.5 rounded-md px-1.5" style={{ color: theme.node.muted }}>
<label className="flex h-6 cursor-pointer items-center gap-1 rounded-md px-1" style={{ color: theme.node.muted }}>
<Switch size="small" checked={confirmTools} onChange={onConfirmToolsChange} />
<span className="whitespace-nowrap">执行前确认</span>
<span className="whitespace-nowrap">确认</span>
</label>
</div>
</div>
@@ -68,12 +100,12 @@ export function AgentPanelChrome({
function AgentModeSwitch({ value, theme, onChange }: { value: CanvasAgentMode; theme: CanvasTheme; onChange: (value: CanvasAgentMode) => void }) {
return (
<div className="inline-flex h-8 shrink-0 items-center rounded-md p-0.5 text-[var(--fs-label)]" style={{ background: theme.spatial.surface }} role="group" aria-label="Agent 运行位置">
<div className="inline-flex h-7 shrink-0 items-center rounded-full p-0.5 text-[var(--fs-label)]" style={{ background: `${theme.node.text}08`, boxShadow: `inset 0 1px 0 ${theme.node.text}08` }} role="group" aria-label="Agent 运行位置">
{(["online", "local"] as const).map((item) => {
const active = value === item;
const Icon = item === "online" ? Globe2 : Laptop;
return (
<button key={item} type="button" className="inline-flex h-7 items-center gap-1 rounded-[var(--r-sm)] px-2 transition-colors" style={{ background: active ? theme.node.fill : "transparent", color: active ? theme.node.text : theme.node.muted, boxShadow: active ? `0 1px 5px ${theme.spatial.shadow}` : "none" }} onClick={() => onChange(item)} aria-pressed={active}>
<button key={item} type="button" className="inline-flex h-6 items-center gap-1 rounded-full px-2 transition-[background-color,color,transform] duration-200 active:scale-95" style={{ background: active ? theme.node.fill : "transparent", color: active ? theme.node.text : theme.node.muted, boxShadow: active ? `0 4px 12px ${theme.spatial.shadow}` : "none" }} onClick={() => onChange(item)} aria-pressed={active}>
<Icon className="size-3" />
{item === "online" ? "网站" : "本机"}
</button>

247
web/src/components/canvas/canvas-assistant-panel.tsx
View File

@@ -1,6 +1,6 @@
import { useEffect, useMemo, useRef, useState } from "react";
import copyToClipboard from "copy-to-clipboard";
import { Copy, Cpu, History, MessageSquareText, Plus, ScrollText, Settings2, Trash2, X } from "lucide-react";
import { Copy, Cpu, Settings2, Trash2, X } from "lucide-react";
import { Button, Modal, Segmented, Select, Tooltip } from "antd";
import { motion } from "motion/react";
@@ -21,7 +21,7 @@ import { imageReferenceLabel } from "@/lib/image-reference-prompt";
import { navigateToSettings } from "@/lib/settings-navigation";
import { cinematicAgentSessionOpsJson, createCinematicAgentSession, isAgentSessionPollingAbort, resumeCinematicAgentSession } from "@/lib/canvas/canvas-agent-session";
import { summarizeCanvasContext } from "@/lib/canvas/canvas-context-summary";
import { AgentChatComposer, AgentChatMessage, AgentPanelTabs, AgentWorkingMessage, type CanvasAgentChatMessage, type CanvasAgentMode } from "./canvas-agent-chat-ui";
import { AgentChatComposer, AgentChatMessage, AgentWorkingMessage, type CanvasAgentChatMessage, type CanvasAgentMode } from "./canvas-agent-chat-ui";
import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button";
import { ModelLogo } from "@/components/model-logo";
import { AgentChatEmptyState, AgentPanelChrome } from "./canvas-agent-panel-chrome";
@@ -29,20 +29,24 @@ import { CanvasLocalAgentPanel } from "./canvas-local-agent-panel";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantPendingBackendSession, type CanvasAssistantReference, type CanvasAssistantSession, type CanvasNodeData } from "@/types/canvas";
import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store";
import { previewCanvasAgentOps, summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentOperationImpact, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context";
import { canvasAgentPostconditionMessage, previewCanvasAgentOps, summarizeCanvasAgentOps, verifyCanvasAgentOps, type CanvasAgentOp, type CanvasAgentOperationImpact, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentConnection, getCanvasAgentGenerationTasks, getCanvasAgentNode, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context";
import { canvasAgentPromptCacheKey } from "@/lib/openai-prompt-cache";
import { resolveStoryboardGenerationContext } from "@/lib/canvas/canvas-storyboard-context";
import { buildSkillMentionReferences } from "@/lib/canvas/canvas-skill-mentions";
import { buildCanvasWorkflowOps, looksLikeWorkflowRequest, type CanvasWorkflowInput } from "@/lib/canvas/canvas-agent-workflow";
import { listAddedSkills, type Skill } from "@/services/api/skills";
export const CANVAS_AGENT_PANEL_MOTION_MS = 500;
const PANEL_MOTION_SECONDS = CANVAS_AGENT_PANEL_MOTION_MS / 1000;
const ONLINE_AGENT_MAX_STEPS = 4;
const ONLINE_AGENT_PROMPT =
"你是影策网页内置在线画布助手。首轮必须先调用 canvas_get_context；涉及已有节点时用 canvas_find_nodes 获取真实 id，涉及媒体参考时用 canvas_get_resources。复杂写操作先 canvas_validate_ops，再执行 canvas_apply_ops。不要输出 JSON ops、不要猜 id、不要把未就绪资源当作可用素材、不要编造执行结果；工具返回真实结果后再回答。";
"你是影策网页内置在线画布助手。首轮必须先调用 canvas_get_context；涉及已有节点时用 canvas_find_nodes 获取真实 id，涉及媒体参考时用 canvas_get_resources。流水线、工作流、管线、节点图或用户要求连线时，必须使用 canvas_create_workflow：把需求拆成有语义的节点类型、真实内容/提示词、边和布局，禁止把业务阶段退化成几个空文本卡片；工具会自动分配 id、布局并建立连线。复杂写操作先 canvas_validate_ops，再执行 canvas_apply_ops。任何写入后都必须检查工具返回的真实节点类型、connectionCount、overlapWarnings 和 verification；没有真实连线时绝不能说已连线，没有生成资源时绝不能说已完成。不要输出 JSON ops、不要猜 id、不要把未就绪资源当作可用素材、不要编造执行结果。需要用户选择时，给出可点击的短选项，不要只让用户输入 1、2、3。技能不是被拼进用户消息的提示词：用户提及技能时，先用 canvas_get_skill 按 id 或名称加载该技能，再按工具返回的技能契约执行。";
const JSON_RECORD_SCHEMA = { type: "object", additionalProperties: true };
const POSITION_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false };
const VIEWPORT_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, k: { type: "number" } }, required: ["x", "y", "k"], additionalProperties: false };
const NODE_TYPE_SCHEMA = { type: "string", enum: ["image", "text", "skill", "video", "audio"] };
const WORKFLOW_NODE_KIND_SCHEMA = { type: "string", enum: ["text", "script", "image", "video", "audio", "character_cards", "character_three_view", "storyboard_video"] };
const GENERATION_MODE_SCHEMA = { type: "string", enum: ["text", "image", "video", "audio"] };
const GENERATION_OPTION_PROPERTIES = {
model: { type: "string" },
@@ -86,7 +90,7 @@ const CANVAS_OP_SCHEMA = {
required: ["type"],
additionalProperties: false,
};
const ONLINE_READ_TOOLS = new Set(["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"]);
const ONLINE_READ_TOOLS = new Set(["canvas_list_skills", "canvas_get_skill", "canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_node", "canvas_get_connection", "canvas_get_generation_tasks", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"]);
function toolDefinition(name: string, description: string, properties: Record<string, unknown>, required: string[] = [], strict = false): ResponseFunctionTool {
return { type: "function", function: { name, description, parameters: { type: "object", properties, required, additionalProperties: false }, strict } };
@@ -111,9 +115,14 @@ function generationToolDefinition(name: string, description: string, mode?: "tex
}
const ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [
toolDefinition("canvas_list_skills", "列出当前用户已加入、可按需加载的画布技能；只返回元数据，不返回完整指令。", {}),
toolDefinition("canvas_get_skill", "按 skillId 或技能名称按需加载一个画布技能的完整契约。技能正文通过工具结果提供，不会自动注入每条用户消息。", { skillId: { type: "string" }, name: { type: "string" } }),
toolDefinition("canvas_get_state", "读取当前网页画布的节点、连线、选区和视口。", {}),
toolDefinition("canvas_get_context", "读取语义化画布上下文、真实节点 id、连接关系、资源就绪状态和状态哈希。", {}),
toolDefinition("canvas_find_nodes", "按标题、内容、提示词、类型、状态或资产检索真实节点。", { query: { type: "string" }, ids: { type: "array", items: { type: "string" } }, types: { type: "array", items: { type: "string" } }, statuses: { type: "array", items: { type: "string" } }, resourceOnly: { type: "boolean" }, limit: { type: "number" } }),
toolDefinition("canvas_get_node", "按真实节点 id 精确读取单个节点、资源状态和关联连线。", { id: { type: "string" } }, ["id"]),
toolDefinition("canvas_get_connection", "按真实连线 id 精确读取端点节点和 handle 信息。", { id: { type: "string" } }, ["id"]),
toolDefinition("canvas_get_generation_tasks", "读取当前画布绑定的生成任务观察状态，不主动轮询上游。", { status: { type: "string" }, nodeIds: { type: "array", items: { type: "string" } }, limit: { type: "number" } }),
toolDefinition("canvas_get_resources", "读取画布媒体资源引用、类型、尺寸、大小、时长和就绪状态，不返回媒体 URL。", { nodeIds: { type: "array", items: { type: "string" } }, status: { type: "string" }, limit: { type: "number" } }),
toolDefinition("canvas_validate_ops", "在写入前校验节点 id、连接关系和批量操作参数。", { ops: { type: "array", items: CANVAS_OP_SCHEMA } }, ["ops"]),
toolDefinition("canvas_get_selection", "读取当前网页画布选中的节点。", {}),
@@ -121,10 +130,46 @@ const ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [
toolDefinition(
"canvas_apply_ops",
"批量操作当前网页画布。复杂写操作应先 canvas_validate_ops；可传 canvas_get_context 返回的 expectedStateHash 防止基于过期状态写入。",
{ ops: { type: "array", items: CANVAS_OP_SCHEMA }, expectedStateHash: { type: "string" } },
{ ops: { type: "array", items: CANVAS_OP_SCHEMA }, expectedRevision: { type: "number" }, expectedStateHash: { type: "string" } },
["ops"],
false,
),
toolDefinition(
"canvas_create_workflow",
"创建语义化工作流/流水线：节点使用真实的文本、脚本、图片、视频或音频类型；character_cards=角色拆分图片卡片，character_three_view=角色三视图，storyboard_video=分镜剧情视频。工具会自动生成唯一 id、按节点实际尺寸布局、创建 edges/referenceRefs/referenceNodeIds 连线、选择新节点并复核重叠。媒体节点必须提供有意义的 prompt 或 content；已有素材先 canvas_find_nodes/canvas_get_resources，再把真实 node id 放入 referenceNodeIds。不要用 canvas_create_text_nodes 代替工作流。",
{
title: { type: "string" },
description: { type: "string" },
nodes: {
type: "array",
minItems: 1,
items: {
type: "object",
properties: {
ref: { type: "string" },
kind: WORKFLOW_NODE_KIND_SCHEMA,
title: { type: "string" },
content: { type: "string" },
prompt: { type: "string" },
description: { type: "string" },
referenceRefs: { type: "array", items: { type: "string" } },
referenceNodeIds: { type: "array", items: { type: "string" } },
runGeneration: { type: "boolean" },
width: { type: "number" },
height: { type: "number" },
},
required: ["ref", "kind", "title"],
additionalProperties: false,
},
},
edges: { type: "array", items: { type: "object", properties: { from: { type: "string" }, to: { type: "string" } }, required: ["from", "to"], additionalProperties: false } },
direction: { type: "string", enum: ["horizontal", "vertical"] },
start: POSITION_SCHEMA,
gap: { type: "number" },
autoRun: { type: "boolean" },
},
["nodes"],
),
toolDefinition(
"canvas_create_node",
"创建任意类型节点：text、image、video、audio。适合创建文本、媒体占位或自定义 metadata 节点。",
@@ -191,7 +236,7 @@ const ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [
toolDefinition("canvas_set_viewport", "调整画布视口。", { viewport: VIEWPORT_SCHEMA }, ["viewport"]),
toolDefinition("canvas_run_generation", "触发指定节点生成；对已有生成任务明确重试时传 retry=true。", { nodeId: { type: "string" }, mode: GENERATION_MODE_SCHEMA, prompt: { type: "string" }, retry: { type: "boolean" } }, ["nodeId"]),
];
type OnlineAgentTab = "setup" | "chat" | "history" | "log";
type OnlineAgentTab = "chat" | "history";
type OnlineAgentLog = { id: string; time: string; title: string; data?: unknown };
type OnlineAgentLogContext = { model: string; running: boolean; confirmTools: boolean; messages: number; nodes: number; connections: number };
type OnlineLoopContext = { step: number };
@@ -327,6 +372,7 @@ export function CanvasAssistantPanel({
const [isRunning, setIsRunning] = useState(false);
const [deleteChatIds, setDeleteChatIds] = useState<string[]>([]);
const [onlineLogs, setOnlineLogs] = useState<OnlineAgentLog[]>([]);
const [composerSkills, setComposerSkills] = useState<Skill[]>([]);
const [removedReferenceIds, setRemovedReferenceIds] = useState<Set<string>>(new Set());
const [localSessions, setLocalSessions] = useState<CanvasAssistantSession[]>(() => (sessions.length ? sessions : [createSession()]));
const localSessionsRef = useRef(localSessions);
@@ -343,6 +389,20 @@ export function CanvasAssistantPanel({
const cinematicSessionControllersRef = useRef(new Map<string, AbortController>());
const generationConsumerControllerRef = useRef(new AbortController());
useEffect(() => {
let cancelled = false;
listAddedSkills()
.then((result) => {
if (!cancelled) setComposerSkills(result.skills || []);
})
.catch(() => {
if (!cancelled) setComposerSkills([]);
});
return () => {
cancelled = true;
};
}, []);
useEffect(() => {
if (!sessions.length) return;
if (sessions === localSessions && activeSessionId === localActiveSessionId) return;
@@ -381,8 +441,6 @@ export function CanvasAssistantPanel({
const messages = activeSession?.messages || [];
const hasMessages = messages.length > 0;
const agentBusy = isRunning || safeSessions.some((session) => session.pendingBackendSession?.status === "pending");
const activeModel = effectiveConfig.textModel || effectiveConfig.model;
const activeModelName = activeModel ? modelDisplayName(effectiveConfig, activeModel) : "";
const selectedNodeKey = useMemo(() => Array.from(selectedNodeIds).sort().join(","), [selectedNodeIds]);
const allSelectedReferences = useMemo(() => buildAssistantReferences(nodes, selectedNodeIds), [nodes, selectedNodeIds]);
const selectedReferences = useMemo(() => allSelectedReferences.filter((item) => !removedReferenceIds.has(item.id)), [allSelectedReferences, removedReferenceIds]);
@@ -610,7 +668,7 @@ export function CanvasAssistantPanel({
const requestConfig = { ...effectiveConfig, model: effectiveConfig.textModel || effectiveConfig.model };
try {
setIsRunning(true);
const messages = await buildToolAgentMessages(snapshotRef.current, history, userMessage);
const messages = await buildToolAgentMessages(snapshotRef.current, history, userMessage, composerSkills);
addOnlineLog(`Agent Tool Loop ${loop.step} 开始`, { toolChoice: "required" });
let streamed = "";
const result = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, messages, "required", userMessage.text, (text) => {
@@ -701,23 +759,52 @@ export function CanvasAssistantPanel({
const executeOps = async (ops: CanvasAgentOp[], context?: { conversationId?: string; messageId?: string; source?: "online" | "local" }) => {
const beforeSnapshot = snapshotRef.current;
const validation = validateCanvasAgentOps(beforeSnapshot, ops);
if (!validation.ok) {
throw new Error(`画布操作校验失败：${validation.issues.filter((item) => item.severity === "error").map((item) => item.message).join("；")}`);
}
const before = snapshotSignature(beforeSnapshot);
const next = await onApplyOps(ops, context);
snapshotRef.current = next;
const ranGeneration = ops.some((op) => op.type === "run_generation" && Boolean(op.nodeId));
const changed = before !== snapshotSignature(next) || ranGeneration;
const noopReason = changed ? "" : explainNoop(ops, beforeSnapshot);
return { changed, ops, ranGeneration, noopReason, before: JSON.parse(before), after: JSON.parse(snapshotSignature(next)) };
const verification = verifyCanvasAgentOps(beforeSnapshot, next, ops);
const noopReason = verification.changed ? "" : explainNoop(ops, beforeSnapshot);
return { ...verification, verification, snapshot: next, ops, noopReason, before: JSON.parse(before), after: JSON.parse(snapshotSignature(next)) };
};
const executeOnlineTool = async (sessionId: string, name: string, args: Record<string, unknown>, messageId?: string): Promise<OnlineToolResult> => {
const current = snapshotRef.current;
try {
const current = snapshotRef.current;
try {
const expectedRevision = typeof args.expectedRevision === "number" ? args.expectedRevision : undefined;
if (expectedRevision !== undefined && expectedRevision !== (current.revision ?? 0)) return { ok: false, message: "画布 revision 已变化，请重新读取 canvas_get_context 后再执行写操作。" };
const expectedStateHash = typeof args.expectedStateHash === "string" ? args.expectedStateHash : "";
if (expectedStateHash && expectedStateHash !== buildCanvasAgentContext(current).stateHash) return { ok: false, message: "画布状态已变化，请重新读取 canvas_get_context 后再执行写操作。" };
if (name === "canvas_list_skills") {
const data = composerSkills.filter((skill) => skill.is_added).map((skill) => ({ skillId: skill.skill_id, name: skill.skill_name, description: skill.description, tag: skill.tag }));
return { ok: true, message: data.length ? "已列出当前可用技能。" : "当前没有已加入技能。", data };
}
if (name === "canvas_get_skill") {
const skillId = typeof args.skillId === "string" ? args.skillId : "";
const nameQuery = typeof args.name === "string" ? args.name.trim().toLocaleLowerCase() : "";
const skill = composerSkills.find((item) => item.is_added && (item.skill_id === skillId || item.skill_name.toLocaleLowerCase() === nameQuery));
if (!skill) return { ok: false, message: "未找到已加入的技能，请先调用 canvas_list_skills。" };
return {
ok: true,
message: `已按需加载技能「${skill.skill_name}」。`,
data: { skillId: skill.skill_id, name: skill.skill_name, description: skill.description, instruction: skill.instruction || skill.description, version: skill.update_time },
};
}
if (name === "canvas_get_state") return { ok: true, message: describeCanvasSnapshot(current), data: compactSnapshot(current) };
if (name === "canvas_get_context") return { ok: true, message: "已读取语义化画布上下文。", data: buildCanvasAgentContext(current) };
if (name === "canvas_find_nodes") return { ok: true, message: "已按条件检索真实节点。", data: findCanvasAgentNodes(current, args as Parameters<typeof findCanvasAgentNodes>[1]) };
if (name === "canvas_get_node") {
const data = getCanvasAgentNode(current, { id: requireString(args.id, "id") });
return { ok: true, message: data.found ? "已精确读取节点。" : "未找到指定节点。", data };
}
if (name === "canvas_get_connection") {
const data = getCanvasAgentConnection(current, { id: requireString(args.id, "id") });
return { ok: true, message: data.found ? "已精确读取连线。" : "未找到指定连线。", data };
}
if (name === "canvas_get_generation_tasks") return { ok: true, message: "已读取画布生成任务观察状态。", data: getCanvasAgentGenerationTasks(current, args as Parameters<typeof getCanvasAgentGenerationTasks>[1]) };
if (name === "canvas_get_resources") return { ok: true, message: "已读取画布资源清单。", data: getCanvasAgentResources(current, args as Parameters<typeof getCanvasAgentResources>[1]) };
if (name === "canvas_validate_ops") {
const result = validateCanvasAgentOps(current, requireOps(args.ops));
@@ -756,7 +843,7 @@ export function CanvasAssistantPanel({
}
const ops = onlineToolToOps(name, args, current, effectiveConfig);
const result = await executeOps(ops, { source: "online", conversationId: sessionId, messageId: messageId || sessionId });
return { ok: result.changed, message: result.changed ? summarizeCanvasAgentOps(ops) || "画布操作已执行。" : result.noopReason, data: result };
return { ok: result.ok, message: result.changed ? canvasAgentPostconditionMessage(result) : result.noopReason, data: result };
} catch (error) {
if (isAgentSessionPollingAbort(error)) throw error;
return { ok: false, message: error instanceof Error ? error.message : "工具执行失败" };
@@ -838,6 +925,11 @@ export function CanvasAssistantPanel({
await sendMessage(text, messages);
};
const submitQuickAction = (text: string) => {
if (!text.trim() || agentBusy) return;
void sendMessage(text.trim(), messages);
};
useEffect(() => {
if (!cinematicEntry) return;
setCinematicEntryActive(true);
@@ -977,75 +1069,31 @@ export function CanvasAssistantPanel({
const onlineContent = (
<>
<AgentPanelTabs
value={view}
theme={theme}
items={[
{ value: "setup", label: "配置", icon: <Settings2 className="size-3.5" /> },
{ value: "chat", label: "对话", icon: <MessageSquareText className="size-3.5" /> },
{ value: "history", label: "历史", icon: <History className="size-3.5" />, count: historySessions.length },
{ value: "log", label: "记录", icon: <ScrollText className="size-3.5" />, count: onlineLogs.length },
]}
onChange={setView}
right={
<>
{view === "history" ? (
<Tooltip title="删除全部">
<Button
type="text"
shape="circle"
className="!h-8 !w-8 !min-w-8"
style={iconButtonStyle}
icon={<X className="size-4" />}
disabled={!historySessions.length}
onClick={() => setDeleteChatIds(historySessions.map((session) => session.id))}
/>
</Tooltip>
) : null}
<Tooltip title="新对话">
<Button
type="text"
shape="circle"
className="!h-8 !w-8 !min-w-8"
style={iconButtonStyle}
icon={<Plus className="size-4" />}
disabled={!hasMessages}
onClick={() => {
startChatSession();
setView("chat");
}}
/>
{view === "history" ? (
<div className="thin-scrollbar min-h-0 flex-1 overflow-y-auto px-4 py-3">
<div className="mb-2 flex items-center justify-between gap-2 px-1">
<div className="text-xs font-medium" style={{ color: theme.node.muted }}>历史会话</div>
<Tooltip title="清空历史">
<Button type="text" shape="circle" className="!h-7 !w-7 !min-w-7" style={iconButtonStyle} icon={<X className="size-3.5" />} disabled={!historySessions.length} onClick={() => setDeleteChatIds(historySessions.map((session) => session.id))} aria-label="清空历史会话" />
</Tooltip>
</>
}
/>
{view === "setup" ? (
<OnlineAgentSetupView theme={theme} activeModel={activeModelName} onOpenConfig={() => navigateToSettings({ continueCreation: true })} />
</div>
<AssistantHistory
sessions={historySessions}
activeSession={activeSession}
onOpen={(id) => {
setLocalActiveSessionId(id);
setView("chat");
}}
onDelete={(id) => setDeleteChatIds([id])}
/>
</div>
) : (
<div ref={chatListRef} className="thin-scrollbar min-h-0 flex-1 space-y-4 overflow-y-auto px-4 py-4">
{view === "history" ? (
<AssistantHistory
sessions={historySessions}
activeSession={activeSession}
onOpen={(id) => {
setLocalActiveSessionId(id);
setView("chat");
}}
onDelete={(id) => setDeleteChatIds([id])}
/>
) : view === "log" ? (
<OnlineAgentLogView
logs={onlineLogs}
theme={theme}
context={{ model: activeModelName, running: agentBusy, confirmTools, messages: messages.length, nodes: snapshot.nodes.length, connections: snapshot.connections.length }}
onClear={() => setOnlineLogs([])}
/>
) : messages.length ? (
{messages.length ? (
<>
{messages.map((message) => (
<div key={message.id} className="space-y-2">
<AgentChatMessage item={assistantMessageToChatMessage(message)} theme={theme} user={user} isStreaming={agentBusy && message.id === messages.at(-1)?.id && message.role === "assistant"} onRejectTool={rejectOnlineTool} onApproveTool={approveOnlineTool} />
<AgentChatMessage item={assistantMessageToChatMessage(message)} theme={theme} user={user} isStreaming={agentBusy && message.id === messages.at(-1)?.id && message.role === "assistant"} onRejectTool={rejectOnlineTool} onApproveTool={approveOnlineTool} onQuickAction={submitQuickAction} />
{message.references?.length ? <MessageReferences message={message} /> : null}
</div>
))}
@@ -1086,6 +1134,8 @@ export function CanvasAssistantPanel({
sending={agentBusy}
placeholder={cinematicEntryActive ? "一句话描述题材、角色和核心冲突" : "描述你想让 Agent 如何操作画布"}
theme={theme}
references={buildSkillMentionReferences(composerSkills)}
slashSkills={composerSkills}
onPromptChange={setPrompt}
onSubmit={cinematicEntryActive ? () => submitCinematicProject(prompt) : submit}
onAddFiles={addImagesToCanvas}
@@ -1132,16 +1182,15 @@ export function CanvasAssistantPanel({
return (
<motion.aside
className="pointer-events-auto relative flex h-full w-full flex-col overflow-hidden rounded-[var(--panel-radius)] border"
className="pointer-events-auto relative flex h-full w-full flex-col overflow-hidden rounded-[var(--panel-radius)]"
initial={{ x: 48, opacity: 0 }}
animate={{ x: closing ? 28 : 0, opacity: closing ? 0 : 1 }}
exit={{ opacity: 0 }}
transition={{ duration: resizing ? 0 : PANEL_MOTION_SECONDS, ease: [0.22, 1, 0.36, 1] }}
style={{
borderColor: theme.toolbar.border,
background: theme.spatial.elevated,
color: theme.node.text,
boxShadow: `0 24px 72px ${theme.spatial.shadow}`,
boxShadow: `-18px 0 48px ${theme.spatial.shadow}, 0 24px 72px ${theme.spatial.shadow}`,
}}
>
<AgentPanelChrome
@@ -1150,12 +1199,17 @@ export function CanvasAssistantPanel({
context={contextSummary}
referenceCount={selectedReferences.length}
confirmTools={confirmTools}
canUndo={agentMode === "online" && canUndoOps}
canUndo={agentMode === "online" ? canUndoOps : false}
undoCount={agentMode === "online" ? undoOpsCount : 0}
onModeChange={onAgentModeChange}
onConfirmToolsChange={(confirmTools) => setAgentState({ confirmTools })}
onUndo={undoLastOnlineBatch}
onCollapse={collapse}
historyCount={agentMode === "online" ? historySessions.length : 0}
historyActive={agentMode === "online" && view === "history"}
onOpenHistory={agentMode === "online" ? () => setView((current) => current === "history" ? "chat" : "history") : undefined}
onNewChat={agentMode === "online" ? () => { startChatSession(); setView("chat"); } : undefined}
newChatDisabled={false}
/>
{agentMode === "local" ? <CanvasLocalAgentPanel embedded snapshot={snapshot} canUndoOps={canUndoOps} undoOpsCount={undoOpsCount} onApplyOps={onApplyOps} onUndoOps={onUndoOps} autoConnect={autoConnectLocal} /> : onlineContent}
</motion.aside>
@@ -1440,6 +1494,7 @@ function parseToolArguments(value: string) {
export function onlineToolToOps(name: string, input: Record<string, unknown>, snapshot: CanvasAgentSnapshot, config: AiConfig): CanvasAgentOp[] {
if (name === "canvas_apply_ops") return requireOps(input.ops);
if (name === "canvas_create_workflow") return buildCanvasWorkflowOps(input as unknown as CanvasWorkflowInput, snapshot, config);
if (name === "canvas_create_node") {
const nodeType = requireNodeType(input.nodeType);
const x = numberOr(input.x, nextCanvasX(snapshot));
@@ -1449,6 +1504,8 @@ export function onlineToolToOps(name: string, input: Record<string, unknown>, sn
if (name === "canvas_create_text_node") return [textNodeOp(input, numberOr(input.x, nextCanvasX(snapshot)), numberOr(input.y, 0))];
if (name === "canvas_create_text_nodes") {
const items = requireRecordArray(input.items, "items");
const textBatch = items.map((item) => `${String(item.title || "")} ${String(item.text || "")}`).join(" ");
if (looksLikeWorkflowRequest(textBatch)) throw new Error("检测到流水线/工作流意图，请使用 canvas_create_workflow 创建真实类型节点和连线。" );
const x = numberOr(input.x, nextCanvasX(snapshot));
const y = numberOr(input.y, 0);
const gap = numberOr(input.gap, 40);
@@ -1630,11 +1687,21 @@ function previewOnlineToolCalls(calls: ResponseToolCall[], snapshot: CanvasAgent
}
function toolCallLabel(name: string) {
if (name === "canvas_list_skills") return "列出技能";
if (name === "canvas_get_skill") return "加载技能";
if (name === "canvas_apply_ops") return "画布操作";
if (name === "canvas_get_state") return "读取画布";
if (name === "canvas_get_context") return "读取上下文";
if (name === "canvas_find_nodes") return "检索节点";
if (name === "canvas_get_node") return "读取节点";
if (name === "canvas_get_connection") return "读取连线";
if (name === "canvas_get_generation_tasks") return "读取生成任务";
if (name === "canvas_get_resources") return "读取资源";
if (name === "canvas_validate_ops") return "校验操作";
if (name === "canvas_get_selection") return "读取选区";
if (name === "canvas_export_snapshot") return "导出快照";
if (name === "canvas_create_cinematic_session") return "创建影视项目";
if (name === "canvas_create_workflow") return "创建工作流";
if (name === "canvas_create_node") return "创建节点";
if (name === "canvas_create_text_node") return "创建文本";
if (name === "canvas_create_text_nodes") return "批量创建文本";
@@ -1828,10 +1895,16 @@ function buildAssistantReferences(nodes: CanvasNodeData[], selectedNodeIds: Set<
.filter((item): item is CanvasAssistantReference => Boolean(item));
}
async function buildToolAgentMessages(snapshot: CanvasAgentSnapshot, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage): Promise<ResponseInputMessage[]> {
async function buildToolAgentMessages(snapshot: CanvasAgentSnapshot, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage, skills: Skill[] = []): Promise<ResponseInputMessage[]> {
const refs = userMessage.references || [];
const skillCatalog = skills
.filter((skill) => skill.is_added)
.slice(0, 40)
.map((skill) => `- ${skill.skill_name}（${skill.skill_id}）：${skill.description}`)
.join("\n");
const systemContent = [ONLINE_AGENT_PROMPT, skillCatalog ? `当前可按需加载的技能（仅元数据）：\n${skillCatalog}` : ""].filter(Boolean).join("\n\n");
return [
{ role: "system", content: ONLINE_AGENT_PROMPT },
{ role: "system", content: systemContent },
...history
.filter((message) => message.role === "user" || message.role === "assistant" || message.role === "system")
.slice(-8)

186
web/src/components/canvas/canvas-local-agent-panel.tsx
View File

@@ -22,12 +22,13 @@ import {
type AgentPendingToolCall,
type AgentThreadSummary,
} from "@/stores/canvas/use-canvas-agent-store";
import { previewCanvasAgentOps, summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context";
import { canvasAgentPostconditionMessage, hashCanvasAgentSnapshot, previewCanvasAgentOps, summarizeCanvasAgentOps, verifyCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentConnection, getCanvasAgentGenerationTasks, getCanvasAgentNode, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context";
import { buildCanvasResourceReferences } from "@/lib/canvas/canvas-resource-references";
import { resolveSkillMentions } from "@/lib/canvas/canvas-skill-mentions";
import { listAddedSkills, type Skill } from "@/services/api/skills";
import { isProjectAgentReadTool, isProjectAgentToolName, runProjectAgentTool } from "@/services/api/project-agent-tools";
import { AgentChatComposer, AgentChatMessage, AgentPanelTabs, AgentPendingToolCard, AgentWorkingMessage, type CanvasAgentChatAttachment } from "./canvas-agent-chat-ui";
import { AgentChatComposer, AgentChatMessage, AgentPendingToolCard, AgentWorkingMessage, type CanvasAgentChatAttachment } from "./canvas-agent-chat-ui";
import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button";
import { AgentChatEmptyState } from "./canvas-agent-panel-chrome";
@@ -131,19 +132,42 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
const errorLoggedRef = useRef(false);
const attachmentUrlsRef = useRef(new Set<string>());
const clientIdRef = useRef(createClientId());
const runtimeRevisionRef = useRef(0);
const runtimeStateHashRef = useRef("");
const runtimeCanonicalStateHashRef = useRef("");
const runtimeSyncQueueRef = useRef<Promise<void>>(Promise.resolve());
const connectionControllerRef = useRef<AbortController | null>(null);
const activeToolRequestIdsRef = useRef(new Set<string>());
const recoveredToolResultIdsRef = useRef(new Set<string>());
const syncState = useCallback(
(clientId: string, nextSnapshot: CanvasAgentSnapshot) => {
void postCanvasRuntimeState(getLocalRuntimeSessionClient(), clientId, nextSnapshot).catch(() => {
pushEventLog({
id: `${Date.now()}-${Math.random()}`,
time: new Date().toLocaleTimeString(),
title: "状态同步失败",
text: "本机 Runtime 暂未接收画布状态",
const stateHash = hashCanvasAgentSnapshot(nextSnapshot);
if (runtimeStateHashRef.current !== stateHash) {
runtimeRevisionRef.current = runtimeStateHashRef.current ? runtimeRevisionRef.current + 1 : nextSnapshot.revision ?? 0;
runtimeStateHashRef.current = stateHash;
}
// Canvas Agent 的 canonical hash 是服务端 SHA-256；浏览器本地校验使用轻量 FNV hash。
// 不把浏览器 hash 当作服务端 hash 上送，避免跨运行时误判；服务端会根据完整快照重新计算。
const envelope = { ...nextSnapshot, revision: runtimeRevisionRef.current };
const queued = runtimeSyncQueueRef.current
.catch(() => undefined)
.then(async () => {
const result = await postCanvasRuntimeState(getLocalRuntimeSessionClient(), clientId, envelope);
runtimeRevisionRef.current = result.revision;
runtimeCanonicalStateHashRef.current = result.stateHash;
return result;
})
.catch(() => {
pushEventLog({
id: `${Date.now()}-${Math.random()}`,
time: new Date().toLocaleTimeString(),
title: "状态同步失败",
text: "本机 Runtime 暂未接收画布状态",
});
return undefined;
});
});
runtimeSyncQueueRef.current = queued.then(() => undefined);
return queued;
},
[pushEventLog],
);
@@ -174,6 +198,11 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
useEffect(() => {
snapshotRef.current = snapshot;
}, [snapshot]);
useEffect(() => {
runtimeRevisionRef.current = 0;
runtimeStateHashRef.current = "";
runtimeCanonicalStateHashRef.current = "";
}, [snapshot.projectId]);
useEffect(() => {
if (!connected) return;
const clientId = clientIdRef.current;
@@ -283,6 +312,10 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
if (connected) void loadThreads();
}, [connected, loadThreads, snapshot.projectId]);
useEffect(() => {
if (activeTab === "history" && connected) void loadThreads();
}, [activeTab, connected, loadThreads]);
useEffect(() => {
if (!connected) return;
const timer = setTimeout(() => syncState(clientIdRef.current, snapshot), 300);
@@ -292,6 +325,7 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
const sendPrompt = async (overrideText?: string) => {
const text = (overrideText ?? prompt).trim();
const files = attachments;
const mentionedSkills = resolveSkillMentions(text, composerSkills);
const requestPrompt = promptWithAttachments(text, files);
if (!connected || !requestPrompt || sending || waiting) return;
if (attachmentPayloadBytes(files) > MAX_ATTACHMENT_PAYLOAD_BYTES) {
@@ -305,7 +339,13 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
const data = await fetchAgentJson<{ threadId?: string }>("/agent/codex/turn", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ prompt: requestPrompt, canvasId: snapshotRef.current.projectId, threadId: useCanvasAgentStore.getState().activeThreadId || undefined, attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })) }),
body: JSON.stringify({
prompt: requestPrompt,
canvasId: snapshotRef.current.projectId,
threadId: useCanvasAgentStore.getState().activeThreadId || undefined,
attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })),
skills: mentionedSkills.map((skill) => ({ skillId: skill.skill_id, name: skill.skill_name, description: skill.description, instruction: skill.instruction || skill.description })),
}),
});
if (data.threadId) setAgentState({ activeThreadId: data.threadId });
addEventLog("本地 Agent 已接收", { accepted: true });
@@ -381,28 +421,63 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
const projectToolName = isProjectAgentToolName(payload.name) ? payload.name : null;
setAgentState({ activity: payload.name === "canvas_apply_ops" ? "执行画布操作" : projectToolName ? "执行项目工具" : "读取画布", waiting: true });
addEventLog(toolName(payload.name), payload, payload);
if (payload.name === "canvas_apply_ops") {
const currentSnapshot = snapshotRef.current;
if (typeof input.expectedRevision === "number" && input.expectedRevision !== runtimeRevisionRef.current) throw new Error(`画布 revision 已从 ${input.expectedRevision} 变为 ${runtimeRevisionRef.current}，请重新读取 canvas_get_context 后再执行写操作`);
// expectedStateHash is the Canvas Agent Runtime's canonical
// SHA-256 hash. The browser intentionally uses a different,
// synchronous fingerprint for local bookkeeping, so the
// Runtime has already performed the authoritative check before
// emitting this tool call. Comparing the two algorithms here
// would reject every valid local write.
const validation = validateCanvasAgentOps(currentSnapshot, (input.ops || []) as CanvasAgentOp[]);
if (!validation.ok) throw new Error(`画布操作校验失败：${validation.issues.filter((item) => item.severity === "error").map((item) => item.message).join("；")}`);
}
const result =
payload.name === "canvas_apply_ops"
? await onApplyOpsRef.current((input.ops || []) as CanvasAgentOp[], { source: "local", conversationId: activeThreadId || clientIdRef.current, messageId: payload.requestId })
? await (async () => {
const before = snapshotRef.current;
const next = await onApplyOpsRef.current((input.ops || []) as CanvasAgentOp[], { source: "local", conversationId: activeThreadId || clientIdRef.current, messageId: payload.requestId });
const verification = verifyCanvasAgentOps(before, next, (input.ops || []) as CanvasAgentOp[]);
return {
ok: verification.ok,
message: canvasAgentPostconditionMessage(verification),
data: { verification, snapshot: next },
snapshot: next,
};
})()
: payload.name === "canvas_get_state" || payload.name === "canvas_export_snapshot"
? snapshotRef.current
: payload.name === "canvas_get_context"
? buildCanvasAgentContext(snapshotRef.current)
? await readLocalCanvasContext()
: payload.name === "canvas_find_nodes"
? findCanvasAgentNodes(snapshotRef.current, input as Parameters<typeof findCanvasAgentNodes>[1])
: payload.name === "canvas_get_node"
? getCanvasAgentNode(snapshotRef.current, { id: requireString(input.id, "id") })
: payload.name === "canvas_get_connection"
? getCanvasAgentConnection(snapshotRef.current, { id: requireString(input.id, "id") })
: payload.name === "canvas_get_generation_tasks"
? getCanvasAgentGenerationTasks(snapshotRef.current, input as Parameters<typeof getCanvasAgentGenerationTasks>[1])
: payload.name === "canvas_get_resources"
? getCanvasAgentResources(snapshotRef.current, input as Parameters<typeof getCanvasAgentResources>[1])
: payload.name === "canvas_validate_ops"
: payload.name === "canvas_validate_ops"
? validateCanvasAgentOps(snapshotRef.current, (input.ops || []) as CanvasAgentOp[])
: payload.name === "canvas_get_selection"
? (() => {
const ids = new Set(snapshotRef.current.selectedNodeIds || []);
return { nodes: snapshotRef.current.nodes.filter((node) => ids.has(node.id)) };
})()
: projectToolName
? await runProjectAgentTool(projectToolName, input, snapshotRef.current.domainProjectId)
: snapshotRef.current;
await postToolResult(clientIdRef.current, { requestId: payload.requestId, result });
if (payload.name === "canvas_apply_ops") syncState(clientIdRef.current, result as CanvasAgentSnapshot);
if (payload.name === "canvas_apply_ops") syncState(clientIdRef.current, (result as { snapshot?: CanvasAgentSnapshot }).snapshot || snapshotRef.current);
setAgentState({ activity: "工具完成", waiting: true });
addEventLog(`${toolName(payload.name)}完成`, result, result);
addMessage({
role: "tool",
title: `${toolName(payload.name)}完成`,
text: payload.name === "canvas_apply_ops" ? summarizeCanvasAgentOps((input.ops || []) as CanvasAgentOp[]) || "画布操作" : "已完成",
text: payload.name === "canvas_apply_ops" ? (result as { message?: string }).message || summarizeCanvasAgentOps((input.ops || []) as CanvasAgentOp[]) || "画布操作" : "已完成",
detail: { requestId: payload.requestId, name: payload.name, input, result },
});
} catch (error) {
@@ -415,6 +490,17 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
}
};
const readLocalCanvasContext = async () => {
// Flush the latest browser snapshot first, then expose the Runtime's
// canonical state hash to Codex. This keeps the local MCP path's
// read→write precondition compatible with the server-side path.
const synced = await syncState(clientIdRef.current, snapshotRef.current);
const stateHash = synced?.stateHash || runtimeCanonicalStateHashRef.current;
return buildCanvasAgentContext(snapshotRef.current, {
...(stateHash ? { stateHash, hashSource: "canvas-agent-server" as const } : {}),
});
};
const rejectPendingTool = async () => {
if (!pendingTool) return;
await postToolResult(clientIdRef.current, { requestId: pendingTool.requestId, error: "用户取消了画布工具调用" });
@@ -584,31 +670,26 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
const content = (
<>
<AgentPanelTabs
value={activeTab}
theme={theme}
items={[
{ value: "setup", label: "连接", icon: <PlugZap className="size-3.5" /> },
{ value: "chat", label: "对话", icon: <MessageSquareText className="size-3.5" /> },
{ value: "history", label: "历史", icon: <History className="size-3.5" />, count: threads.length },
{ value: "log", label: "日志", icon: <Terminal className="size-3.5" />, count: eventLogs.length },
]}
onChange={(activeTab) => {
setAgentState({ activeTab });
if (activeTab === "history") void loadThreads();
}}
right={
<>
<Tooltip title={undoOpsCount ? `撤销最近一批 Agent 写回，可撤销 ${undoOpsCount} 批` : "没有可撤销的 Agent 写回"}>
<Button size="small" type="text" className="!h-8 !w-8 !min-w-8" disabled={!canUndoOps} icon={<RotateCcw className="size-3.5" />} onClick={undoLastTool} aria-label="撤销最近一批 Agent 写回" />
</Tooltip>
</>
}
/>
<div className="flex min-h-8 shrink-0 items-center justify-end gap-1 px-3 pb-1">
<div className="mr-auto min-w-0 truncate px-1 text-[var(--fs-tiny)]" style={{ color: connected ? "#16a34a" : theme.node.muted }}>
{connected ? "本机 Agent 已连接" : canvasAgentConnectionStatusText({ enabled, connected, activity, connectError })}
</div>
{!connected ? (
<Button size="small" type={enabled ? "default" : "primary"} className="!h-7 !px-2.5" icon={<PlugZap className="size-3.5" />} onClick={toggleAgentConnection}>
{enabled ? "连接中" : "连接"}
</Button>
) : null}
<Tooltip title={threads.length ? `历史会话 · ${threads.length}` : "历史会话"}>
<Button type="text" className={`!h-7 !min-w-7 !px-1.5 ${activeTab === "history" ? "font-medium" : ""}`} style={{ color: activeTab === "history" ? theme.node.text : theme.node.muted, background: activeTab === "history" ? theme.spatial.surface : "transparent" }} icon={<History className="size-3.5" />} onClick={() => setAgentState({ activeTab: activeTab === "history" ? "chat" : "history" })} aria-label="打开历史会话">
{threads.length ? <span className="text-[var(--fs-tiny)] tabular-nums">{threads.length}</span> : null}
</Button>
</Tooltip>
<Tooltip title="新对话">
<Button type="text" shape="circle" className="!h-7 !w-7 !min-w-7" disabled={!connected || loadingThreads} style={{ color: theme.node.muted }} icon={<Plus className="size-3.5" />} onClick={() => void startNewThread()} aria-label="新建对话" />
</Tooltip>
</div>
{activeTab === "setup" ? (
<AgentConnectView theme={theme} enabled={enabled} connected={connected} activity={activity} connectError={connectError} onToggleEnabled={toggleAgentConnection} />
) : activeTab === "history" ? (
{activeTab === "history" ? (
<AgentHistoryView
theme={theme}
threads={threads}
@@ -621,15 +702,6 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
onResumeThread={(threadId) => void resumeThread(threadId)}
onDeleteThread={confirmDeleteThread}
/>
) : activeTab === "log" ? (
<AgentLogView
logs={eventLogs}
theme={theme}
context={{ connected, enabled, activity, waiting, sending, messages: messages.length, pendingTool: pendingTool?.name }}
onClear={clearEventLogs}
onCopied={(text) => message.success(text)}
onCopyBlocked={(text) => message.warning(text)}
/>
) : (
<>
<div ref={listRef} className="thin-scrollbar min-h-0 flex-1 space-y-4 overflow-y-auto p-4">
@@ -644,7 +716,7 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({
/>
) : null}
{messages.map((item) => (
<AgentChatMessage key={item.id} item={agentMessageToChatMessage(item)} theme={theme} user={user} isStreaming={(sending || waiting) && item.id === messages.at(-1)?.id && item.role === "assistant"} />
<AgentChatMessage key={item.id} item={agentMessageToChatMessage(item)} theme={theme} user={user} isStreaming={(sending || waiting) && item.id === messages.at(-1)?.id && item.role === "assistant"} onQuickAction={(text) => void sendPrompt(text)} />
))}
{pendingTool ? (
<AgentPendingToolCard
@@ -1044,6 +1116,13 @@ function isConnectionErrorMessage(item: AgentChatItem) {
function toolName(name: string) {
if (name === "canvas_apply_ops") return "画布操作";
if (name === "canvas_get_state") return "读取画布";
if (name === "canvas_get_context") return "读取上下文";
if (name === "canvas_find_nodes") return "检索节点";
if (name === "canvas_get_node") return "读取节点";
if (name === "canvas_get_connection") return "读取连线";
if (name === "canvas_get_generation_tasks") return "读取生成任务";
if (name === "canvas_get_resources") return "读取资源";
if (name === "canvas_validate_ops") return "校验操作";
if (name === "canvas_get_selection") return "读取选区";
if (name === "canvas_export_snapshot") return "导出快照";
if (name === "canvas_create_node") return "创建节点";
@@ -1078,7 +1157,7 @@ function toolName(name: string) {
}
function isReadTool(name: string) {
return name === "canvas_get_state" || name === "canvas_get_selection" || name === "canvas_export_snapshot" || isProjectAgentReadTool(name);
return name === "canvas_get_state" || name === "canvas_get_context" || name === "canvas_find_nodes" || name === "canvas_get_node" || name === "canvas_get_connection" || name === "canvas_get_generation_tasks" || name === "canvas_get_resources" || name === "canvas_validate_ops" || name === "canvas_get_selection" || name === "canvas_export_snapshot" || isProjectAgentReadTool(name);
}
function isMcpToolItem(item?: AgentEventItem) {
@@ -1125,6 +1204,11 @@ function stringText(value: unknown) {
return typeof value === "string" ? value : "";
}
function requireString(value: unknown, field: string) {
if (typeof value !== "string" || !value.trim()) throw new Error(`${field} 必须是非空字符串`);
return value;
}
function objectField(value: unknown, key: string) {
return value && typeof value === "object" ? (value as Record<string, unknown>)[key] : undefined;
}