feat(*): 模型协议 - 升级声明式协议、扩充官方插件并完善生成任务恢复

2026-09-01 16:47:03 +08:00
parent 3f4a9ea3d0
commit 4724f0767b
132 changed files with 4576 additions and 1420 deletions
--- a/web/src/components/canvas/canvas-assistant-panel.tsx
+++ b/web/src/components/canvas/canvas-assistant-panel.tsx
@@ -7,8 +7,8 @@ import { motion } from "motion/react";
 import { modelDisplayName, modelIcon, normalizeModelOptionValue, resolveModelChannel, resolveModelRequestConfig, selectableModelsByCapability, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
 import { canvasThemes } from "@/lib/canvas-theme";
 import { nanoid } from "nanoid";
-import { requestToolResponse, type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image";
-import { backendModelRuntimeRequired, runBackendToolGenerationTask } from "@/services/api/generation-task";
+import { type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image";
+import { runBackendToolGenerationTask } from "@/services/api/generation-task";
 import { imageToDataUrl } from "@/services/image-storage";
 import { isCanvasGenerationDurableAckError, persistCanvasCinematicSessionContinuationEffect } from "@/services/canvas-generation-consumer";
 import { consumeGenerationTaskAgent } from "@/services/project-asset-sync";
@@ -32,7 +32,6 @@ import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantPendin
 import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store";
 import { canvasAgentPostconditionMessage, previewCanvasAgentOps, summarizeCanvasAgentOps, verifyCanvasAgentOps, type CanvasAgentOp, type CanvasAgentOperationImpact, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
 import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentConnection, getCanvasAgentGenerationTasks, getCanvasAgentNode, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context";
-import { canvasAgentPromptCacheKey } from "@/lib/openai-prompt-cache";
 import { resolveStoryboardGenerationContext } from "@/lib/canvas/canvas-storyboard-context";
 import { buildCanvasWorkflowOps, looksLikeWorkflowRequest, type CanvasWorkflowInput } from "@/lib/canvas/canvas-agent-workflow";
 import { listAddedSkills, type Skill } from "@/services/api/skills";
@@ -674,7 +673,7 @@ export function CanvasAssistantPanel({
            const result = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, messages, "required", userMessage.text, (text) => {
                streamed = text;
                if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text });
-            }, canvasAgentPromptCacheKey(sessionId));
+            });
            addOnlineLog("模型工具回复", result);
            if (result.toolCalls.length) {
                const writableCalls = result.toolCalls.filter(isWritableToolCall);
@@ -733,7 +732,7 @@ export function CanvasAssistantPanel({
        const next = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, nextMessages, "auto", "继续处理画布工具结果", (text) => {
            streamed = text;
            if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text });
-        }, canvasAgentPromptCacheKey(sessionId));
+        });
        addOnlineLog(`Agent Tool Loop ${step + 1} 回复`, next);
        if (next.toolCalls.length) {
            const writableCalls = next.toolCalls.filter(isWritableToolCall);
@@ -1649,12 +1648,8 @@ function toolCallToResponseInput(call: ResponseToolCall): ResponseInputMessage {
    return { type: "function_call", call_id: call.id, name: call.function.name, arguments: call.function.arguments, ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}) };
 }

-async function requestOnlineAgentModel(config: AiConfig, messages: ResponseInputMessage[], toolChoice: "auto" | "required", prompt: string, onDelta: (text: string) => void, promptCacheKey?: string) {
-    if (backendModelRuntimeRequired(config)) {
-        const result = await runBackendToolGenerationTask({ prompt, config, messages, tools: ONLINE_AGENT_TOOLS, toolChoice, onDelta });
-        return result;
-    }
-    return requestToolResponse(config, messages, ONLINE_AGENT_TOOLS, toolChoice, onDelta, { promptCacheKey });
+async function requestOnlineAgentModel(config: AiConfig, messages: ResponseInputMessage[], toolChoice: "auto" | "required", prompt: string, onDelta: (text: string) => void) {
+    return runBackendToolGenerationTask({ prompt, config, messages, tools: ONLINE_AGENT_TOOLS, toolChoice, onDelta });
 }

 function summarizeToolCalls(calls: ResponseToolCall[]) {
--- a/web/src/components/canvas/canvas-node-content.tsx
+++ b/web/src/components/canvas/canvas-node-content.tsx
@@ -54,7 +54,6 @@ export type CanvasNodeContentProps = {
    onToggleBatch?: () => void;
    reduceMediaEffects?: boolean;
    mediaActive?: boolean;
-    hydrateMediaPreview?: boolean;
 };

 export function CanvasNodeContent(props: CanvasNodeContentProps) {
@@ -417,10 +416,11 @@ function EmptyImageContent({ node, theme, isBatchRoot, batchCount, batchExpanded
    return content;
 }

-function VideoNodeContent({ node, theme, reduceMediaEffects, mediaActive = false, hydrateMediaPreview = false }: CanvasNodeContentProps) {
+function VideoNodeContent({ node, theme, reduceMediaEffects, mediaActive = false }: CanvasNodeContentProps) {
    const playerBoxRef = useRef<HTMLDivElement>(null);
    const { updateMediaNode } = useCanvasNodeActions();
-    const { url, loading } = useNodeResourceUrl(node, mediaActive);
+    const hasPassivePreview = Boolean(canvasNodeVideoPreviewUrl(node));
+    const { url, loading } = useNodeResourceUrl(node, mediaActive || !hasPassivePreview);
    const subtitleEntries = node.metadata?.subtitleEntries || [];
    const subtitleStyle = node.metadata?.subtitleStyle || createDefaultSubtitleStyle();
    const [currentTimeMs, setCurrentTimeMs] = useState(0);
@@ -449,7 +449,7 @@ function VideoNodeContent({ node, theme, reduceMediaEffects, mediaActive = false
    }, [node.id, node.metadata?.naturalHeight, node.metadata?.naturalWidth, subtitleEntries.length, updateMediaNode, url]);

    if (!node.metadata?.content) return <EmptyMediaContent icon={<Video className="size-7 opacity-35" />} label="空视频节点" color={theme.node.placeholder} />;
-    if (!mediaActive) return <InactiveVideoPreview node={node} theme={theme} hydrateMediaPreview={hydrateMediaPreview} />;
+    if (!mediaActive) return <InactiveVideoPreview node={node} theme={theme} sourceUrl={url} sourceLoading={loading} />;
    if (!url) return <MediaLoadingState icon={<LoaderCircle className="size-5 animate-spin" />} label={loading ? "正在加载视频" : "视频资源不可用"} />;

    const sourceRatio = (videoSize?.width || node.metadata?.naturalWidth || node.width) / Math.max(1, videoSize?.height || node.metadata?.naturalHeight || node.height);
@@ -484,18 +484,18 @@ function AudioNodeContent({ node, theme }: CanvasNodeContentProps) {
    return <CanvasAudioPlayer node={node} theme={theme} />;
 }

-function InactiveVideoPreview({ node, theme, hydrateMediaPreview = false }: Pick<CanvasNodeContentProps, "node" | "theme" | "hydrateMediaPreview">) {
+function InactiveVideoPreview({ node, theme, sourceUrl, sourceLoading }: Pick<CanvasNodeContentProps, "node" | "theme"> & { sourceUrl: string; sourceLoading: boolean }) {
    const previewUrl = canvasNodeVideoPreviewUrl(node);
    const { updateMetadata } = useCanvasNodeActions();
    const updateMetadataRef = useRef(updateMetadata);
-    const [hydrating, setHydrating] = useState(false);
+    const [hydrating, setHydrating] = useState(() => !previewUrl && Boolean(node.metadata?.content));

    useEffect(() => {
        updateMetadataRef.current = updateMetadata;
    }, [updateMetadata]);

    useEffect(() => {
-        if (previewUrl || !hydrateMediaPreview || !node.metadata?.content || !updateMetadataRef.current) {
+        if (previewUrl || !node.metadata?.content || !updateMetadataRef.current) {
            setHydrating(false);
            return;
        }
@@ -510,12 +510,32 @@ function InactiveVideoPreview({ node, theme, hydrateMediaPreview = false }: Pick
                if (!controller.signal.aborted) setHydrating(false);
            });
        return () => controller.abort();
-    }, [hydrateMediaPreview, node.id, node.metadata?.content, node.metadata?.storageKey, previewUrl]);
+    }, [node.id, node.metadata?.content, node.metadata?.storageKey, previewUrl]);

    if (previewUrl) {
        return <div className="relative size-full overflow-hidden rounded-[var(--node-radius)] bg-black"><img src={previewUrl} alt={`${node.title || "视频"} 静态预览`} loading="lazy" decoding="async" draggable={false} className="pointer-events-none size-full select-none object-contain" /></div>;
    }
-    return <InactiveMediaCard icon={<Video className="size-7" />} title={node.title || "视频"} hint={hydrating ? "正在生成首帧" : "选择后加载视频"} theme={theme} />;
+    if (sourceUrl) {
+        return <div className="relative size-full overflow-hidden rounded-[var(--node-radius)] bg-black">
+            <video
+                src={sourceUrl}
+                aria-label={`${node.title || "视频"} 首帧预览`}
+                muted
+                playsInline
+                preload="auto"
+                draggable={false}
+                className="pointer-events-none size-full select-none object-contain"
+                onLoadedMetadata={(event) => primePassiveVideoFrame(event.currentTarget)}
+                onLoadedData={(event) => primePassiveVideoFrame(event.currentTarget)}
+            />
+            {hydrating ? <span className="pointer-events-none absolute bottom-3 left-1/2 -translate-x-1/2 rounded-full bg-black/55 px-2 py-1 text-[var(--fs-tiny)] text-white/70 backdrop-blur">正在保存首帧</span> : null}
+        </div>;
+    }
+    return <InactiveMediaCard icon={<Video className="size-7" />} title={node.title || "视频"} hint={sourceLoading || hydrating ? "正在加载首帧" : "视频资源暂不可用"} theme={theme} />;
+}
+
+function primePassiveVideoFrame(video: HTMLVideoElement) {
+    if (video.currentTime === 0 && Number.isFinite(video.duration) && video.duration > 0) video.currentTime = Math.min(0.001, video.duration);
 }

 function InactiveMediaCard({ icon, title, hint, theme }: { icon: ReactNode; title: string; hint: string; theme: CanvasTheme }) {
--- a/web/src/components/canvas/canvas-node-generation.ts
+++ b/web/src/components/canvas/canvas-node-generation.ts
@@ -182,7 +182,7 @@ function buildComposerGenerationContext(
                if (input.type === "text") textBlocks.push(`【${label}】\n${input.text || ""}`);
                else selectedInputs.push(input);
            }
-            nextPrompt += input.type === "text" ? `【${label}】` : label;
+            nextPrompt += input.type === "text" ? `【${label}】` : `@${label}`;
        } else nextPrompt += match[0];
        lastIndex = match.index + match[0].length;
    }
--- a/web/src/components/canvas/canvas-node.tsx
+++ b/web/src/components/canvas/canvas-node.tsx
@@ -24,7 +24,6 @@ type CanvasNodeProps = {
    scale: number;
    isSelected: boolean;
    mediaActive?: boolean;
-    hydrateMediaPreview?: boolean;
    isRelated: boolean;
    isFocusRelated: boolean;
    isConnectionTarget: boolean;
@@ -70,7 +69,6 @@ export const CanvasNode = React.memo(function CanvasNode({
    scale,
    isSelected,
    mediaActive = false,
-    hydrateMediaPreview = false,
    isRelated,
    isFocusRelated,
    isConnectionTarget,
@@ -376,7 +374,6 @@ export const CanvasNode = React.memo(function CanvasNode({
                        onToggleBatch={() => onToggleBatch?.(data.id)}
                        reduceMediaEffects={reduceMediaEffects}
                        mediaActive={mediaActive}
-                        hydrateMediaPreview={hydrateMediaPreview}
                    />
                </div>