feat(canvas): 图片转换插件 - 增加本地媒体转换节点与运行时

This commit is contained in:
QAyong
2026-09-08 22:55:44 +08:00
parent 8360b2c933
commit f755d6db97
51 changed files with 3284 additions and 44 deletions

View File

@@ -374,6 +374,7 @@ function nodeTypeLabel(node?: CanvasNodeData | null) {
if (node.type === CanvasNodeType.Video) return "视频节点";
if (node.type === CanvasNodeType.Audio) return "音频节点";
if (node.type === CanvasNodeType.Drawing) return "绘图节点";
if (node.type === CanvasNodeType.MediaConversion) return "转换节点";
if (node.type === CanvasNodeType.Frame) return isCanvasFolderNode(node) ? "文件夹" : "背板";
return "生成配置节点";
}

View File

@@ -37,6 +37,8 @@ import { PanoramaNodeContent } from "./nodes/panorama-node";
import { SvgNodeContent } from "./nodes/svg-node";
import { PortraitClearanceNodeContent } from "./nodes/portrait-clearance-node";
import { ArtCritiqueNodeContent } from "./nodes/ai-art-critique-node";
import { MediaConversionNodeContent } from "./nodes/media-conversion-node";
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
export type CanvasNodeContentProps = {
node: CanvasNodeData;
@@ -74,6 +76,7 @@ export function CanvasNodeContent(props: CanvasNodeContentProps) {
if (hasCustomContent && props.renderNodeContent) return props.renderNodeContent(props.node);
if (props.node.type === PORTRAIT_CLEARANCE_NODE_TYPE) return <PortraitClearanceNodeContent node={props.node} />;
if (props.node.type === ART_CRITIQUE_NODE_TYPE) return <ArtCritiqueNodeContent node={props.node} />;
if (props.node.type === MEDIA_CONVERSION_NODE_TYPE) return <MediaConversionNodeContent node={props.node} theme={props.theme} />;
if (props.isBatchRoot) return <ImageNodeContent {...props} />;
if (props.node.metadata?.status === "loading") return <LoadingContent node={props.node} theme={props.theme} onOpenTaskDetails={props.onOpenTaskDetails} />;
if (props.node.metadata?.status === "error") return <ErrorContent node={props.node} theme={props.theme} onRetry={props.onRetry} onReloadResource={props.onReloadResource} />;

View File

@@ -8,6 +8,7 @@ import { getGenerationResourceNodes, getContextResourceNodes, getMentionResource
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade";
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
import { mediaConversionSourceFingerprint } from "@/lib/media-conversion/contracts";
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation";
@@ -64,6 +65,12 @@ export type NodeGenerationInput = {
};
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, assets: Asset[], promptOnly = false): NodeGenerationContext {
const pendingConversion = findPendingMediaConversionInput(nodeId, nodes, connections);
if (pendingConversion) {
const status = pendingConversion.metadata?.mediaConversion?.status;
const reason = status === "stale" ? "输入或方式已变化" : status === "processing" ? "仍在处理中" : "尚未完成本地转换";
throw new Error(`转换节点「${pendingConversion.title || pendingConversion.id}」${reason},完成后才能执行下游生成`);
}
const connectedInputs = buildNodeGenerationInputs(nodeId, nodes, connections);
const sourceNode = nodes.find((node) => node.id === nodeId);
const portraitTextureInput = sourceNode?.type === CanvasNodeType.Image && sourceNode.metadata?.content && sourceNode.metadata?.portraitTexture
@@ -120,6 +127,40 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
};
}
/**
* 转换节点允许提前连到下游,但只有已物化的结果才能进入生成请求。
* 沿上游链路检查也覆盖“生成节点 -> 配置节点 -> 转换节点”的常见接法。
*/
export function findPendingMediaConversionInput(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
const nodesById = new Map(nodes.map((node) => [node.id, node]));
const queue = connections.filter((connection) => connection.toNodeId === nodeId).map((connection) => connection.fromNodeId);
const visited = new Set<string>();
while (queue.length) {
const sourceId = queue.shift()!;
if (visited.has(sourceId)) continue;
visited.add(sourceId);
const source = nodesById.get(sourceId);
if (!source) continue;
if (source.type === CanvasNodeType.MediaConversion) {
const state = source.metadata?.mediaConversion;
const sourceInputs = connections
.filter((connection) => connection.toNodeId === source.id)
.map((connection) => nodesById.get(connection.fromNodeId))
.filter((input): input is CanvasNodeData => Boolean(input && (input.type === CanvasNodeType.Image || input.type === CanvasNodeType.Video)));
const sourceInput = sourceInputs[0];
const ready = state?.status === "completed"
&& Boolean(state.resultStorageKey)
&& sourceInputs.length === 1
&& Boolean(sourceInput)
&& state.sourceNodeId === sourceInput?.id
&& state.sourceFingerprint === mediaConversionSourceFingerprint(sourceInput!);
if (!ready) return source;
}
connections.filter((connection) => connection.toNodeId === sourceId).forEach((connection) => queue.push(connection.fromNodeId));
}
return null;
}
function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]) {
let next = prompt.trim();
let removed = true;

View File

@@ -32,6 +32,7 @@ type CanvasNodeToolbarProps = {
onUpload: (node: CanvasNodeData) => void;
onDownload: (node: CanvasNodeData) => void;
onSaveAsset: (node: CanvasNodeData) => void;
onCreateConversion?: (node: CanvasNodeData) => void;
onMaskEdit: (node: CanvasNodeData) => void;
onEmotion: (node: CanvasNodeData) => void;
onPortraitTexture: (node: CanvasNodeData) => void;
@@ -91,6 +92,7 @@ export function CanvasNodeToolbar({
onUpload,
onDownload,
onSaveAsset,
onCreateConversion,
onMaskEdit,
onEmotion,
onPortraitTexture,
@@ -221,7 +223,7 @@ export function CanvasNodeToolbar({
const nodeHoverHandlers = {
onNodeInfo: onInfo, onNodeDelete: onDelete, onNodeRetry: onRetry, onNodeEditText: onEditText, onNodeDecreaseFont: onDecreaseFont, onNodeIncreaseFont: onIncreaseFont,
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
onNodeSaveAsset: onSaveAsset, onNodeCreateConversion: onCreateConversion, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
@@ -400,7 +402,7 @@ export function CanvasNodeInfoModal({ node, open, onClose, onMetadataChange, rea
const [assetCategory, setAssetCategory] = useState<CanvasAssetCategory>("other");
const imageBytes = node?.type === CanvasNodeType.Image && node.metadata?.content ? getDataUrlByteSize(node.metadata.content) : 0;
const batchCount = node?.type === CanvasNodeType.Image ? node.metadata?.batchChildIds?.length || 0 : 0;
const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : "生成配置";
const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : node?.type === CanvasNodeType.MediaConversion ? "转换" : "生成配置";
useEffect(() => {
setAssetTags(node?.metadata?.assetTags || []);
setAssetTagInput("");

View File

@@ -1,6 +1,6 @@
import React, { useCallback, useEffect, useRef, useState } from "react";
import type { ReactNode } from "react";
import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video } from "lucide-react";
import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video, WandSparkles } from "lucide-react";
import { useCanvasNodeActions } from "./canvas-node-action-context";
@@ -709,6 +709,7 @@ function nodeTypeIcon(type: CanvasNodeTypeId) {
if (type === CanvasNodeType.Drawing) return Pencil;
if (type === CanvasNodeType.Script) return Clapperboard;
if (type === CanvasNodeType.Config) return Settings2;
if (type === CanvasNodeType.MediaConversion) return WandSparkles;
if (type === CanvasNodeType.Skill) return BookOpenCheck;
if (type === PORTRAIT_CLEARANCE_NODE_TYPE) return PortraitClearanceIcon;
if (type === ART_CRITIQUE_NODE_TYPE) return ScanSearch;

View File

@@ -153,7 +153,7 @@ function resolveNodePanelWidth(node: CanvasNodeData, viewport: ViewportTransform
return clamp(Math.round(node.width * viewport.k * 1.5), 680, 920);
}
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) {
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
const reducedMotion = useReducedMotion();
const menuRef = useRef<HTMLDivElement>(null);
@@ -213,6 +213,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co
{canCreateDrawing ? <ConnectionCreateOption motionEnabled={!reducedMotion} icon={<Pencil className="size-4" />} title="绘图" disabledReason={getDisabledReason(CanvasNodeType.Drawing)} onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null}
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<Video className="size-4" />} title="视频生成" disabledReason={getDisabledReason(CanvasNodeType.Video)} onClick={() => onCreate(CanvasNodeType.Video)} />
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<Music2 className="size-4" />} title="音频参考" disabledReason={getDisabledReason(CanvasNodeType.Audio)} onClick={() => onCreate(CanvasNodeType.Audio)} />
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<WandSparkles className="size-4" />} title="转换" description="本地处理图片或视频" disabledReason={getDisabledReason(CanvasNodeType.MediaConversion)} onClick={() => onCreate(CanvasNodeType.MediaConversion)} />
</div>
</SpotlightSurface>
);

View File

@@ -0,0 +1,360 @@
import { CheckCircle2, CircleAlert, Download, Image as ImageIcon, LoaderCircle, RefreshCw, Video, WandSparkles } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { getNodeInputKind } from "@/lib/canvas/node-registry";
import type { CanvasTheme } from "@/lib/canvas-theme";
import {
createDefaultMediaConversionState,
isLocalImageOperation,
mediaConversionOperationDescription,
mediaConversionOperationLabel,
mediaConversionSourceFingerprint,
MEDIA_CONVERSION_OPERATION_LABELS,
type MediaConversionNodeState,
type MediaConversionOperation,
} from "@/lib/media-conversion/contracts";
import { convertImageLocally, LocalImageConversionError } from "@/lib/media-conversion/local-converter";
import { getActiveUserScope } from "@/lib/user-scope";
import { runLocalDepthEstimation } from "@/services/depth-runtime";
import { runLocalLineartEstimation } from "@/services/lineart-runtime";
import { runLocalPoseEstimation } from "@/services/pose-runtime";
import { resolveImageUrl, setImageBlob } from "@/services/image-storage";
import { resolveMediaUrl } from "@/services/file-storage";
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import { useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
import { useCanvasNodeActions } from "../canvas-node-action-context";
import { useUpstreamNodes } from "../canvas-node-graph-context";
type MediaConversionNodeContentProps = {
node: CanvasNodeData;
theme: CanvasTheme;
};
const OPERATIONS = Object.keys(MEDIA_CONVERSION_OPERATION_LABELS) as MediaConversionOperation[];
export function MediaConversionNodeContent({ node, theme }: MediaConversionNodeContentProps) {
const { updateMetadata } = useCanvasNodeActions();
const upstream = useUpstreamNodes(node.id);
const localRuntimeConnection = useLocalRuntimeStore((runtime) => runtime.connection);
const localRuntimeConnecting = useLocalRuntimeStore((runtime) => runtime.connecting);
const reconnectLocalRuntime = useLocalRuntimeStore((runtime) => runtime.connect);
const mediaInputs = upstream.filter((item) => item.type === CanvasNodeType.Image || item.type === CanvasNodeType.Video);
const input = mediaInputs[0];
const inputKind = input ? getNodeInputKind(input.type) : undefined;
const storedState = node.metadata?.mediaConversion;
const state = storedState || createDefaultMediaConversionState();
const currentFingerprint = input ? mediaConversionSourceFingerprint(input) : "";
const hasMultipleInputs = mediaInputs.length > 1;
const isStale = Boolean(state.status === "completed" && (!currentFingerprint || state.sourceFingerprint !== currentFingerprint));
const status = isStale ? "stale" : state.status;
const [sourceUrl, setSourceUrl] = useState("");
const [resultUrl, setResultUrl] = useState("");
const [showResult, setShowResult] = useState(status === "completed" || status === "stale");
const [notice, setNotice] = useState("");
const abortRef = useRef<AbortController | null>(null);
useEffect(() => {
let active = true;
const fallback = input?.metadata?.previewContent || input?.metadata?.content || "";
setSourceUrl(fallback);
if (!input?.metadata?.storageKey) return () => { active = false; };
const resolver = input.type === CanvasNodeType.Image ? resolveImageUrl : resolveMediaUrl;
void resolver(input.metadata.storageKey, fallback)
.then((resolved) => { if (active) setSourceUrl(resolved || fallback); })
.catch(() => { if (active) setSourceUrl(fallback); });
return () => { active = false; };
}, [input?.id, input?.metadata?.content, input?.metadata?.previewContent, input?.metadata?.storageKey, input?.type]);
useEffect(() => {
let active = true;
setResultUrl("");
if (!state.resultStorageKey) return () => { active = false; };
void resolveImageUrl(state.resultStorageKey)
.then((resolved) => { if (active) setResultUrl(resolved); })
.catch(() => { if (active) setResultUrl(""); });
return () => { active = false; };
}, [state.resultStorageKey]);
useEffect(() => {
if (!updateMetadata || state.status !== "completed" || (currentFingerprint && state.sourceFingerprint === currentFingerprint)) return;
updateMetadata(node.id, {
mediaConversion: { ...state, status: "stale", updatedAt: new Date().toISOString() },
});
}, [currentFingerprint, node.id, state, updateMetadata]);
useEffect(() => {
if (status === "stale") setShowResult(false);
}, [status]);
useEffect(() => () => abortRef.current?.abort(), []);
const updateState = (patch: Partial<MediaConversionNodeState>) => {
updateMetadata?.(node.id, { mediaConversion: { ...state, ...patch, schemaVersion: 1 } });
};
const selectOperation = (operation: MediaConversionOperation) => {
if (operation === state.operation) return;
setNotice("");
updateState({
operation,
status: state.resultStorageKey ? "stale" : "idle",
errorCode: undefined,
errorMessage: undefined,
updatedAt: new Date().toISOString(),
});
setShowResult(false);
};
const run = async () => {
setNotice("");
if (!input || !currentFingerprint) {
setNotice("请先连接一张图片或一个视频");
return;
}
if (hasMultipleInputs) {
setNotice("转换节点只能连接一个输入,请移除多余连线");
return;
}
if (inputKind === "video") {
updateState({ status: "unavailable", outputKind: "video", errorCode: "video_not_ready", errorMessage: "本地视频转换还在进行短片性能验证", updatedAt: new Date().toISOString() });
setNotice("视频转换暂未开启,先完成单图模型验证");
return;
}
if (!isLocalImageOperation(state.operation) && state.operation !== "depth" && state.operation !== "lineart" && state.operation !== "pose") {
updateState({ status: "unavailable", outputKind: "image", errorCode: "model_missing", errorMessage: "该转换需要先安装并验证本地模型", updatedAt: new Date().toISOString() });
setNotice(mediaConversionOperationDescription(state.operation));
return;
}
if (!sourceUrl) {
updateState({ status: "error", errorCode: "source_unreadable", errorMessage: "图片预览尚未准备好,请稍后重试", updatedAt: new Date().toISOString() });
setNotice("图片预览尚未准备好,请稍后重试");
return;
}
abortRef.current?.abort();
const controller = new AbortController();
abortRef.current = controller;
const startedAt = new Date().toISOString();
updateState({ status: "processing", sourceNodeId: input.id, sourceFingerprint: currentFingerprint, outputKind: "image", errorCode: undefined, errorMessage: undefined, startedAt, updatedAt: startedAt });
try {
const result = state.operation === "depth"
? await runLocalDepthEstimation(sourceUrl, controller.signal)
: state.operation === "lineart"
? await runLocalLineartEstimation(sourceUrl, controller.signal)
: state.operation === "pose"
? await runLocalPoseEstimation(sourceUrl, controller.signal)
: await convertImageLocally(sourceUrl, state.operation, { signal: controller.signal, maxDimension: 1024 });
if (controller.signal.aborted) return;
const storageKey = localConversionStorageKey(node.id, currentFingerprint, state.operation);
const url = await setImageBlob(storageKey, result.blob);
const completedAt = new Date().toISOString();
const detectedPeople = state.operation === "pose" && "personCount" in result && typeof result.personCount === "number" ? result.personCount : undefined;
updateMetadata?.(node.id, {
content: "",
previewContent: "",
storageKey,
mimeType: result.blob.type || "image/png",
bytes: result.blob.size,
naturalWidth: result.width,
naturalHeight: result.height,
mediaConversion: {
...state,
schemaVersion: 1,
status: "completed",
sourceNodeId: input.id,
sourceFingerprint: currentFingerprint,
outputKind: "image",
resultStorageKey: storageKey,
resultWidth: result.width,
resultHeight: result.height,
...(detectedPeople !== undefined ? { detectedPeople } : {}),
errorCode: undefined,
errorMessage: undefined,
updatedAt: completedAt,
},
});
setResultUrl(url);
setShowResult(true);
} catch (error) {
if (controller.signal.aborted) return;
const aborted = error instanceof LocalImageConversionError && error.code === "aborted";
if (aborted) return;
if (error instanceof LocalRuntimeClientError) {
if (error.code === "pose_no_person") {
updateState({ status: "skipped", errorCode: error.code, errorMessage: error.message, detectedPeople: 0, updatedAt: new Date().toISOString() });
setNotice("未检测到可用人物姿态,可更换含人物的图片或选择其他转换方式");
return;
}
const nextStatus = ["depth_model_missing", "depth_runtime_unavailable", "lineart_model_missing", "lineart_runtime_unavailable", "pose_model_missing", "pose_runtime_unavailable"].includes(error.code) ? "unavailable" : "error";
updateState({ status: nextStatus, errorCode: error.code, errorMessage: error.message, updatedAt: new Date().toISOString() });
setNotice(localRuntimeNotice(error, useLocalRuntimeStore.getState().connection));
return;
}
const conversionError = error instanceof LocalImageConversionError ? error : new LocalImageConversionError("source_unreadable", "本地图片转换失败");
const nextStatus = conversionError.code === "model_missing" ? "unavailable" : "error";
updateState({ status: nextStatus, errorCode: conversionError.code, errorMessage: conversionError.message, updatedAt: new Date().toISOString() });
setNotice(conversionError.message);
}
};
const downloadResult = () => {
if (!resultUrl) return;
const link = document.createElement("a");
link.href = resultUrl;
link.download = `${node.title || "转换结果"}-${state.operation}.png`;
link.click();
};
const effectiveResult = Boolean(resultUrl && (status === "completed" || status === "stale"));
const displayResult = showResult && effectiveResult;
const buttonDisabled = !input || hasMultipleInputs || status === "processing";
const statusText = mediaConversionStatusLabel(status);
const previewStatusText = state.operation === "pose" && status === "completed" && typeof state.detectedPeople === "number"
? `${statusText} · ${state.detectedPeople} 人`
: statusText;
const showRuntimeRecovery = (state.operation === "depth" || state.operation === "lineart" || state.operation === "pose") && (localRuntimeConnection === "unreachable" || state.errorCode === "depth_runtime_unavailable" || state.errorCode === "lineart_runtime_unavailable" || state.errorCode === "pose_runtime_unavailable");
const retryLocalRuntime = async () => {
setNotice("");
await reconnectLocalRuntime();
const next = useLocalRuntimeStore.getState();
if (next.connection === "connected") {
setNotice("本地模型服务已连接,可以重新开始转换");
} else {
setNotice(next.error || "本地模型服务仍未连接,请先运行 start-yingce-local.cmd");
}
};
return (
<div
className="flex h-full min-h-0 w-full flex-col gap-2.5 overflow-hidden rounded-[inherit] p-3"
style={{ background: theme.node.panel, color: theme.node.text }}
onPointerDown={(event) => event.stopPropagation()}
onWheel={(event) => event.stopPropagation()}
>
<div className="flex min-w-0 items-center gap-2">
<span className="grid size-8 shrink-0 place-items-center rounded-[var(--r-md)]" style={{ background: theme.toolbar.itemHover, color: theme.accent.primary }}>
<WandSparkles className="size-4" aria-hidden="true" />
</span>
<div className="min-w-0 flex-1">
<div className="truncate text-[var(--fs-label)] font-semibold">{node.title || "转换"}</div>
<div className="truncate text-[var(--fs-tiny)]" style={{ color: theme.node.muted }}>本地处理 · 单图优先</div>
</div>
<StatusBadge status={status} />
</div>
<div className="flex min-h-0 flex-1 flex-col gap-2">
<div className="relative min-h-0 flex-1 overflow-hidden rounded-[var(--r-md)] border" style={{ borderColor: theme.node.edge, background: theme.node.fill }}>
{displayResult ? (
<img src={resultUrl} alt={`${mediaConversionOperationLabel(state.operation)}结果`} className="absolute inset-0 size-full object-contain" draggable={false} />
) : input?.type === CanvasNodeType.Image && sourceUrl ? (
<img src={sourceUrl} alt="转换输入" className="absolute inset-0 size-full object-contain" draggable={false} />
) : input?.type === CanvasNodeType.Video && sourceUrl ? (
<video src={sourceUrl} className="absolute inset-0 size-full object-contain" controls preload="metadata" />
) : (
<div className="absolute inset-0 grid place-items-center" style={{ color: theme.node.muted }}>
{input?.type === CanvasNodeType.Video ? <Video className="size-8 opacity-35" aria-hidden="true" /> : <ImageIcon className="size-8 opacity-35" aria-hidden="true" />}
</div>
)}
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex items-end justify-between gap-2 bg-gradient-to-t from-black/75 to-transparent px-2.5 pb-2 pt-8 text-white">
<div className="min-w-0">
<div className="truncate text-[var(--fs-micro)] font-semibold">{displayResult ? (status === "stale" ? "上次结果 · 待更新" : "转换结果") : input?.title || "等待输入"}</div>
<div className="mt-0.5 truncate text-[var(--fs-micro)] opacity-75">{hasMultipleInputs ? "已连接多个输入" : input ? `${input.type === CanvasNodeType.Video ? "视频" : "图片"} · ${previewStatusText}` : "连接图片或视频节点"}</div>
</div>
{state.resultWidth && state.resultHeight && displayResult ? <span className="shrink-0 text-[var(--fs-micro)] tabular-nums">{state.resultWidth} × {state.resultHeight}</span> : null}
</div>
</div>
{effectiveResult ? (
<div className="flex gap-1" data-canvas-no-zoom>
<PreviewToggle theme={theme} pressed={!showResult} label="原素材" onClick={() => setShowResult(false)} />
<PreviewToggle theme={theme} pressed={showResult} label="结果" onClick={() => setShowResult(true)} />
<button type="button" aria-label="下载转换结果" title="下载转换结果" className="grid h-6 w-8 shrink-0 place-items-center rounded-[var(--r-sm)] outline-none transition hover:bg-black/5 focus-visible:ring-2 dark:hover:bg-white/10" style={{ color: theme.node.muted, outlineColor: theme.accent.primary }} onClick={(event) => { event.stopPropagation(); downloadResult(); }} onMouseDown={(event) => event.stopPropagation()}>
<Download className="size-3.5" aria-hidden="true" />
</button>
</div>
) : null}
</div>
<div className="flex items-center gap-2" data-canvas-no-zoom onWheel={(event) => event.stopPropagation()}>
<label className="sr-only" htmlFor={`${node.id}-operation`}>转换方式</label>
<select
id={`${node.id}-operation`}
value={state.operation}
disabled={status === "processing"}
className="h-8 min-w-0 flex-1 rounded-[var(--r-sm)] border bg-transparent px-2 text-[var(--fs-tiny)] outline-none focus-visible:ring-2"
style={{ borderColor: theme.node.edge, color: theme.node.text, outlineColor: theme.accent.primary }}
onChange={(event) => selectOperation(event.target.value as MediaConversionOperation)}
onMouseDown={(event) => event.stopPropagation()}
>
{OPERATIONS.map((operation) => <option key={operation} value={operation}>{mediaConversionOperationLabel(operation)}</option>)}
</select>
<button
type="button"
data-canvas-no-zoom
className="inline-flex h-8 shrink-0 items-center gap-1.5 rounded-[var(--r-sm)] px-3 text-[var(--fs-tiny)] font-semibold outline-none transition hover:brightness-105 focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-50"
style={{ background: theme.accent.primary, color: theme.accent.onPrimary, outlineColor: theme.accent.primary }}
disabled={buttonDisabled}
onClick={(event) => { event.stopPropagation(); void run(); }}
onMouseDown={(event) => event.stopPropagation()}
>
{status === "processing" ? <LoaderCircle className="size-3.5 animate-spin" aria-hidden="true" /> : null}
{status === "processing" ? "处理中" : status === "completed" || status === "stale" ? "重新转换" : "开始转换"}
</button>
</div>
<div className="flex min-h-4 items-center gap-2 text-[var(--fs-micro)]" style={{ color: status === "unavailable" || status === "skipped" ? "var(--status-warning)" : notice || status === "error" ? "var(--status-error)" : theme.node.muted }}>
<span className="min-w-0 flex-1 truncate" title={notice || mediaConversionOperationDescription(state.operation)}>{notice || mediaConversionOperationDescription(state.operation)}</span>
{showRuntimeRecovery ? (
<button
type="button"
className="shrink-0 rounded-[var(--r-sm)] px-1.5 py-0.5 font-semibold outline-none transition hover:bg-black/5 focus-visible:ring-2 dark:hover:bg-white/10"
style={{ color: theme.accent.primary, outlineColor: theme.accent.primary }}
disabled={localRuntimeConnecting}
onClick={(event) => { event.stopPropagation(); void retryLocalRuntime(); }}
onMouseDown={(event) => event.stopPropagation()}
>
{localRuntimeConnecting ? "连接中" : "重试连接"}
</button>
) : null}
</div>
</div>
);
}
function PreviewToggle({ theme, pressed, label, onClick }: { theme: CanvasTheme; pressed: boolean; label: string; onClick: () => void }) {
return <button type="button" aria-pressed={pressed} className="h-6 flex-1 rounded-[var(--r-sm)] px-2 text-[var(--fs-micro)] font-medium outline-none transition focus-visible:ring-2" style={{ background: pressed ? theme.accent.primary : "color-mix(in oklch, var(--foreground-muted) 10%, transparent)", color: pressed ? theme.accent.onPrimary : "var(--foreground-muted)", outlineColor: theme.accent.primary }} onClick={(event) => { event.stopPropagation(); onClick(); }} onMouseDown={(event) => event.stopPropagation()}>{label}</button>;
}
function StatusBadge({ status }: { status: MediaConversionNodeState["status"] | "stale" }) {
if (status === "processing") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-info)", background: "color-mix(in oklch, var(--status-info) 15%, transparent)" }}><LoaderCircle className="size-3 animate-spin" aria-hidden="true" />处理中</span>;
if (status === "completed") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-success)", background: "color-mix(in oklch, var(--status-success) 15%, transparent)" }}><CheckCircle2 className="size-3" aria-hidden="true" />已完成</span>;
if (status === "stale") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}><RefreshCw className="size-3" aria-hidden="true" />待更新</span>;
if (status === "error") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-error)", background: "color-mix(in oklch, var(--status-error) 15%, transparent)" }}><CircleAlert className="size-3" aria-hidden="true" />失败</span>;
if (status === "unavailable") return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}>本地不可用</span>;
if (status === "skipped") return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}>已跳过</span>;
return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--foreground-muted)", background: "color-mix(in oklch, var(--foreground-muted) 12%, transparent)" }}>未开始</span>;
}
function mediaConversionStatusLabel(status: MediaConversionNodeState["status"] | "stale") {
if (status === "processing") return "正在处理";
if (status === "completed") return "已完成";
if (status === "stale") return "输入已变化,结果待更新";
if (status === "unavailable") return "本地能力不可用";
if (status === "skipped") return "未检测到人物姿态";
if (status === "error") return "处理失败";
return "尚未转换";
}
function localRuntimeNotice(error: LocalRuntimeClientError, connection: ReturnType<typeof useLocalRuntimeStore.getState>["connection"]) {
if (connection === "unreachable") return "本地模型服务未启动,请双击 start-yingce-local.cmd 后重试";
if (error.code === "depth_model_missing") return "深度模型尚未安装,请先完成本地模型安装";
if (error.code === "lineart_model_missing") return "AI 线稿依赖或模型尚未安装,请先完成本地线稿模型安装";
if (error.code === "pose_model_missing") return "姿态模型或依赖尚未安装,请先完成本地姿态模型安装";
return error.message;
}
function localConversionStorageKey(nodeId: string, fingerprint: string, operation: MediaConversionOperation) {
return `image:${getActiveUserScope()}:media-conversion:${nodeId}:${fingerprint}:${operation}`;
}

View File

@@ -26,6 +26,7 @@ export const NODE_DEFAULT_SIZE = {
[CanvasNodeType.Compare]: { width: 520, height: 320, title: "对比" },
[CanvasNodeType.Chart]: { width: 480, height: 320, title: "图表" },
[CanvasNodeType.ColorGrade]: { width: 420, height: 360, title: "调色" },
[CanvasNodeType.MediaConversion]: { width: 380, height: 440, title: "转换" },
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
export const NODE_SPECS = {
@@ -101,6 +102,10 @@ export const NODE_SPECS = {
...NODE_DEFAULT_SIZE[CanvasNodeType.ColorGrade],
metadata: { status: "idle" },
},
[CanvasNodeType.MediaConversion]: {
...NODE_DEFAULT_SIZE[CanvasNodeType.MediaConversion],
metadata: { status: "idle" },
},
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
export function getNodeSpec(type: CanvasNodeType) {

View File

@@ -399,6 +399,7 @@ function canvasNodeTypeLabel(type?: CanvasNodeType) {
if (type === CanvasNodeType.Frame) return "背板";
if (type === CanvasNodeType.Drawing) return "绘图节点";
if (type === CanvasNodeType.Skill) return "技能节点";
if (type === CanvasNodeType.MediaConversion) return "转换节点";
return "文本节点";
}

View File

@@ -1,7 +1,7 @@
import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection";
import { getNodeAcceptedInputKind, getNodeGenerationMode, getNodeInputKind } from "@/lib/canvas/node-registry";
import { getNodeAcceptedInputKinds, getNodeGenerationMode, getNodeInputKind, getNodeMaxInputCount } from "@/lib/canvas/node-registry";
import type { AiConfig } from "@/stores/use-config-store";
import { type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
type ConnectionCandidate = Pick<CanvasConnection, "fromNodeId" | "toNodeId">;
type CanvasConnectionPolicyOptions = {
@@ -12,11 +12,27 @@ type CanvasConnectionPolicyOptions = {
export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) {
const target = nodes.find((node) => node.id === candidate.toNodeId);
if (!target) return "找不到连线目标节点";
const acceptedInputKind = getNodeAcceptedInputKind(target.type);
if (acceptedInputKind) {
const acceptedInputKinds = getNodeAcceptedInputKinds(target.type);
if (acceptedInputKinds.length) {
const source = nodes.find((node) => node.id === candidate.fromNodeId);
const sourceKind = source ? getNodeInputKind(source.type) : undefined;
if (sourceKind !== acceptedInputKind) return `${acceptedInputKindLabel(acceptedInputKind)}节点只接受${acceptedInputKindLabel(acceptedInputKind)}输入`;
const isMediaConversion = target.type === CanvasNodeType.MediaConversion;
const hasAcceptedSource = isMediaConversion
? source?.type === CanvasNodeType.Image || source?.type === CanvasNodeType.Video
: Boolean(sourceKind && acceptedInputKinds.includes(sourceKind));
if (!sourceKind || !hasAcceptedSource) {
const labels = acceptedInputKinds.map(acceptedInputKindLabel).join("或");
return `${isMediaConversion ? "转换" : labels}节点只接受${labels}输入`;
}
const maxInputCount = getNodeMaxInputCount(target.type);
if (maxInputCount) {
const inputCount = new Set(
[...connections, { id: "candidate", ...candidate }]
.filter((connection) => connection.toNodeId === target.id)
.map((connection) => connection.fromNodeId),
).size;
if (inputCount > maxInputCount) return `${isMediaConversion ? "转换" : "当前"}节点最多连接 ${maxInputCount} 个输入`;
}
}
const mode = getNodeGenerationMode(target);
if (!mode) return "";

View File

@@ -1,4 +1,4 @@
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video } from "lucide-react";
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video, WandSparkles } from "lucide-react";
import { NODE_SPECS } from "@/constant/canvas";
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
@@ -25,7 +25,7 @@ const BUILTIN_NODE_TRAITS = {
minSize: MEDIA_NODE_MIN_SIZE,
keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize,
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "image" : null),
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "image" : null),
generationMode: () => "image",
inputKind: "image",
},
@@ -82,7 +82,7 @@ const BUILTIN_NODE_TRAITS = {
minSize: MEDIA_NODE_MIN_SIZE,
keepAspectRatio: () => true,
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "video" : null),
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "video" : null),
generationMode: () => "video",
inputKind: "video",
},
@@ -165,6 +165,20 @@ const BUILTIN_NODE_TRAITS = {
resourceKind: () => "image",
inputKind: "image",
},
[CanvasNodeType.MediaConversion]: {
label: "转换",
icon: <WandSparkles />,
minSize: { width: 320, height: 360 },
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => {
const conversion = node.metadata?.mediaConversion;
if (conversion?.status !== "completed" || !conversion.resultStorageKey) return null;
return conversion.outputKind === "video" ? "video" : "image";
},
acceptsInputKind: ["image", "video"],
maxInputCount: 1,
inputKind: "image",
},
} satisfies Record<string, Omit<CanvasNodeDefinition, "type" | "defaultTitle" | "defaultSize" | "defaultMetadata">>;
export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => {

View File

@@ -1,3 +1,3 @@
export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition";
export { getNodeAcceptedInputKind, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
export { getNodeAcceptedInputKind, getNodeAcceptedInputKinds, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMaxInputCount, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
import "./definitions";

View File

@@ -43,8 +43,10 @@ export type CanvasNodeDefinition = {
generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null;
/** 是否显示右侧输出连接点;缺省为 true,消费型终点节点可关闭。 */
showOutputConnection?: boolean;
/** 若设置,作为目标时只接受这一类上游节点。 */
acceptsInputKind?: CanvasNodeInputKind;
/** 若设置,作为目标时只接受这些上游节点类型。 */
acceptsInputKind?: CanvasNodeInputKind | CanvasNodeInputKind[];
/** 限制直接输入数量;转换节点等一进一出节点通常为 1。 */
maxInputCount?: number;
/**
* 作为上游输入被参考素材容量校验计数时的类别;
* 不设表示不参与计数(生成配置、背板)。与 resourceKind 不同,计数不看内容。
@@ -69,6 +71,7 @@ export function canvasNodeDefinitionFromPlugin(pluginId: string, contribution: P
showInCreateMenu: true,
showOutputConnection: contribution.showOutputConnection,
acceptsInputKind: contribution.acceptsInputKind,
maxInputCount: contribution.maxInputCount,
plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema },
};
}

View File

@@ -100,3 +100,13 @@ export function getNodeInputKind(type: CanvasNodeTypeId) {
export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) {
return definitions.get(type)?.acceptsInputKind;
}
export function getNodeAcceptedInputKinds(type: CanvasNodeTypeId) {
const accepted = definitions.get(type)?.acceptsInputKind;
if (!accepted) return [];
return Array.isArray(accepted) ? accepted : [accepted];
}
export function getNodeMaxInputCount(type: CanvasNodeTypeId) {
return definitions.get(type)?.maxInputCount;
}

View File

@@ -20,6 +20,7 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }),
// 导演台落在节点分区,但它开的是导演工作台、不是某种画布节点,故不走注册表。
{ id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() },
nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }),

View File

@@ -1,4 +1,4 @@
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video, WandSparkles } from "lucide-react";
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
@@ -139,6 +139,19 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx),
run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!),
},
{
id: "createConversion",
toolbar: "node-hover",
category: "node-state",
label: "创建转换节点",
displayLabel: "转换",
icon: <WandSparkles className="size-3.5" />,
defaultVisible: true,
defaultOrder: 65,
nodeToolbar: { group: "primary", order: 15, description: "在右侧创建转换节点并自动连接" },
applicable: (ctx) => (hasImage(ctx) || hasVideo(ctx)) && Boolean(ctx.handlers.onNodeCreateConversion),
run: (ctx) => ctx.handlers.onNodeCreateConversion?.(ctx.node!),
},
{
id: "edit",
toolbar: "node-hover",

View File

@@ -81,6 +81,8 @@ export type ToolbarHandlers = {
onNodeUpload: (node: CanvasNodeData) => void;
onNodeDownload: (node: CanvasNodeData) => void;
onNodeSaveAsset: (node: CanvasNodeData) => void;
/** 从已有图片或视频创建并自动连接一个转换节点。 */
onNodeCreateConversion?: (node: CanvasNodeData) => void;
onNodeMaskEdit: (node: CanvasNodeData) => void;
onNodeEmotion: (node: CanvasNodeData) => void;
onNodePortraitTexture: (node: CanvasNodeData) => void;

View File

@@ -0,0 +1,89 @@
import type { CanvasNodeData } from "@/types/canvas";
export const MEDIA_CONVERSION_SCHEMA_VERSION = 1 as const;
export const MEDIA_CONVERSION_NODE_TYPE = "media-conversion" as const;
export type MediaConversionOperation = "grayscale" | "edge-canny" | "lineart" | "depth" | "pose" | "cutout";
export type MediaConversionStatus = "idle" | "processing" | "completed" | "stale" | "skipped" | "unavailable" | "error";
export type MediaConversionOutputKind = "image" | "video";
export type MediaConversionNodeState = {
schemaVersion: typeof MEDIA_CONVERSION_SCHEMA_VERSION;
operation: MediaConversionOperation;
status: MediaConversionStatus;
sourceNodeId?: string;
sourceFingerprint?: string;
outputKind?: MediaConversionOutputKind;
resultStorageKey?: string;
resultWidth?: number;
resultHeight?: number;
detectedPeople?: number;
errorCode?: string;
errorMessage?: string;
startedAt?: string;
updatedAt?: string;
};
export const MEDIA_CONVERSION_OPERATION_LABELS: Record<MediaConversionOperation, string> = {
grayscale: "灰度图",
"edge-canny": "Canny 边缘",
lineart: "AI 线稿",
depth: "深度图",
pose: "姿态骨架",
cutout: "透明抠图",
};
export const MEDIA_CONVERSION_OPERATION_DESCRIPTIONS: Record<MediaConversionOperation, string> = {
grayscale: "普通图像算法,适合先检查明暗层次",
"edge-canny": "普通图像算法,提取清晰轮廓",
lineart: "本地 ControlNet Aux 线稿预处理,首次运行需要加载模型",
depth: "本地 Depth Anything V2 Small,首次运行需要加载模型",
pose: "本地 OpenPose 人体骨架预处理,未检测到人物时会跳过",
cutout: "需要安装并验证本地抠图模型",
};
export function createDefaultMediaConversionState(): MediaConversionNodeState {
return {
schemaVersion: MEDIA_CONVERSION_SCHEMA_VERSION,
operation: "edge-canny",
status: "idle",
};
}
export function mediaConversionSourceFingerprint(node: CanvasNodeData) {
const metadata = node.metadata;
const source = [
node.id,
metadata?.storageKey || "",
compactSource(metadata?.content || metadata?.previewContent || ""),
metadata?.mimeType || "",
metadata?.bytes || "",
metadata?.naturalWidth || "",
metadata?.naturalHeight || "",
metadata?.durationMs || "",
].join("|");
let hash = 2166136261;
for (let index = 0; index < source.length; index += 1) {
hash ^= source.charCodeAt(index);
hash = Math.imul(hash, 16777619);
}
return `v1-${(hash >>> 0).toString(16).padStart(8, "0")}`;
}
export function mediaConversionOperationLabel(operation: MediaConversionOperation) {
return MEDIA_CONVERSION_OPERATION_LABELS[operation];
}
export function mediaConversionOperationDescription(operation: MediaConversionOperation) {
return MEDIA_CONVERSION_OPERATION_DESCRIPTIONS[operation];
}
export function isLocalImageOperation(operation: MediaConversionOperation) {
return operation === "grayscale" || operation === "edge-canny";
}
function compactSource(value: string) {
if (value.length <= 2048) return value;
return `${value.slice(0, 1024)}|${value.slice(-1024)}|${value.length}`;
}

View File

@@ -0,0 +1,228 @@
import type { MediaConversionOperation } from "./contracts";
export type LocalImageConversionResult = {
blob: Blob;
width: number;
height: number;
};
export class LocalImageConversionError extends Error {
constructor(readonly code: "model_missing" | "source_unreadable" | "browser_unsupported" | "aborted", message: string) {
super(message);
this.name = "LocalImageConversionError";
}
}
export async function convertImageLocally(sourceUrl: string, operation: MediaConversionOperation, options: { signal?: AbortSignal; maxDimension?: number } = {}): Promise<LocalImageConversionResult> {
throwIfAborted(options.signal);
if (operation !== "grayscale" && operation !== "edge-canny") {
throw new LocalImageConversionError("model_missing", "该转换需要先安装并验证本地模型");
}
const image = await loadImage(sourceUrl, options.signal);
throwIfAborted(options.signal);
const maxDimension = Math.max(256, options.maxDimension || 1024);
const scale = Math.min(1, maxDimension / Math.max(image.naturalWidth, image.naturalHeight));
const width = Math.max(1, Math.round(image.naturalWidth * scale));
const height = Math.max(1, Math.round(image.naturalHeight * scale));
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d", { willReadFrequently: true });
if (!context) throw new LocalImageConversionError("browser_unsupported", "当前浏览器无法创建图片处理画布");
context.drawImage(image, 0, 0, width, height);
const input = context.getImageData(0, 0, width, height);
const output = transformPixels(input.data, width, height, operation, options.signal);
const outputImage = new ImageData(output, width, height);
context.putImageData(outputImage, 0, 0);
const blob = await canvasToBlob(canvas, options.signal);
throwIfAborted(options.signal);
return { blob, width, height };
}
/** Pure pixel transform used by the browser executor and unit tests. */
export function transformPixels(source: Uint8ClampedArray, width: number, height: number, operation: "grayscale" | "edge-canny", signal?: AbortSignal) {
if (source.length !== width * height * 4) throw new Error("图片像素尺寸不匹配");
if (operation === "grayscale") return grayscalePixels(source, width, height, signal);
return cannyPixels(source, width, height, signal);
}
function grayscalePixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
const output = new Uint8ClampedArray(source);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
const value = Math.round(source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114);
output[index] = value;
output[index + 1] = value;
output[index + 2] = value;
}
}
return output;
}
function cannyPixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
const gray = new Float32Array(width * height);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
gray[y * width + x] = source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114;
}
}
const blurred = gaussianBlur(gray, width, height, signal);
const magnitude = new Float32Array(width * height);
const direction = new Float32Array(width * height);
let maximum = 0;
for (let y = 1; y < height - 1; y += 1) {
throwIfAborted(signal);
for (let x = 1; x < width - 1; x += 1) {
const index = y * width + x;
const horizontal = blurred[index + 1] - blurred[index - 1];
const vertical = blurred[index + width] - blurred[index - width];
const value = Math.hypot(horizontal, vertical);
magnitude[index] = value;
direction[index] = Math.atan2(vertical, horizontal);
maximum = Math.max(maximum, value);
}
}
const thinned = nonMaximumSuppression(magnitude, direction, width, height, signal);
const high = maximum * 0.28;
const low = high * 0.45;
const edges = new Uint8Array(width * height);
const stack: number[] = [];
for (let index = 0; index < thinned.length; index += 1) {
if (thinned[index] >= high && high > 0) {
edges[index] = 2;
stack.push(index);
} else if (thinned[index] >= low && low > 0) {
edges[index] = 1;
}
}
while (stack.length) {
throwIfAborted(signal);
const index = stack.pop()!;
const x = index % width;
const y = Math.floor(index / width);
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
if (!offsetX && !offsetY) continue;
const nextX = x + offsetX;
const nextY = y + offsetY;
if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) continue;
const next = nextY * width + nextX;
if (edges[next] !== 1) continue;
edges[next] = 2;
stack.push(next);
}
}
}
const output = new Uint8ClampedArray(source.length);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = y * width + x;
const outputIndex = index * 4;
const value = edges[index] === 2 ? 0 : 255;
output[outputIndex] = value;
output[outputIndex + 1] = value;
output[outputIndex + 2] = value;
output[outputIndex + 3] = source[outputIndex + 3];
}
}
return output;
}
function gaussianBlur(source: Float32Array, width: number, height: number, signal?: AbortSignal) {
const output = new Float32Array(source.length);
const kernel = [1, 2, 1, 2, 4, 2, 1, 2, 1];
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
let value = 0;
let weight = 0;
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
const nextX = Math.min(width - 1, Math.max(0, x + offsetX));
const nextY = Math.min(height - 1, Math.max(0, y + offsetY));
const kernelWeight = kernel[(offsetY + 1) * 3 + offsetX + 1];
value += source[nextY * width + nextX] * kernelWeight;
weight += kernelWeight;
}
}
output[y * width + x] = value / weight;
}
}
return output;
}
function nonMaximumSuppression(magnitude: Float32Array, direction: Float32Array, width: number, height: number, signal?: AbortSignal) {
const output = new Float32Array(magnitude.length);
for (let y = 1; y < height - 1; y += 1) {
throwIfAborted(signal);
for (let x = 1; x < width - 1; x += 1) {
const index = y * width + x;
let angle = (direction[index] * 180) / Math.PI;
if (angle < 0) angle += 180;
const value = magnitude[index];
let before = 0;
let after = 0;
if (angle < 22.5 || angle >= 157.5) {
before = magnitude[index - 1];
after = magnitude[index + 1];
} else if (angle < 67.5) {
before = magnitude[index - width + 1];
after = magnitude[index + width - 1];
} else if (angle < 112.5) {
before = magnitude[index - width];
after = magnitude[index + width];
} else {
before = magnitude[index - width - 1];
after = magnitude[index + width + 1];
}
output[index] = value >= before && value >= after ? value : 0;
}
}
return output;
}
async function loadImage(sourceUrl: string, signal?: AbortSignal) {
if (!sourceUrl) throw new LocalImageConversionError("source_unreadable", "没有可读取的图片输入");
const image = new Image();
image.crossOrigin = "anonymous";
return await new Promise<HTMLImageElement>((resolve, reject) => {
const abort = () => {
image.src = "";
reject(new LocalImageConversionError("aborted", "转换已取消"));
};
if (signal?.aborted) return abort();
signal?.addEventListener("abort", abort, { once: true });
image.onload = () => {
signal?.removeEventListener("abort", abort);
resolve(image);
};
image.onerror = () => {
signal?.removeEventListener("abort", abort);
reject(new LocalImageConversionError("source_unreadable", "图片无法读取,可能是跨域资源或已失效"));
};
image.src = sourceUrl;
});
}
function canvasToBlob(canvas: HTMLCanvasElement, signal?: AbortSignal) {
throwIfAborted(signal);
return new Promise<Blob>((resolve, reject) => {
canvas.toBlob((blob) => {
if (blob) resolve(blob);
else reject(new LocalImageConversionError("browser_unsupported", "当前浏览器无法导出图片结果"));
}, "image/png");
});
}
function throwIfAborted(signal?: AbortSignal): asserts signal is AbortSignal | undefined {
if (signal?.aborted) throw new LocalImageConversionError("aborted", "转换已取消");
}

View File

@@ -3,4 +3,5 @@ import "./prompt-optimizer";
import "./workflows";
import "./portrait-clearance";
import "./ai-art-critique";
import "./media-conversion";
import "./editor/editor-shell";

View File

@@ -0,0 +1,37 @@
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
import { registerPlugin } from "../plugin-registry";
import { PLUGIN_API_VERSION, type PluginManifest, type RegisteredPlugin } from "../plugin-types";
export const MEDIA_CONVERSION_PLUGIN_ID = "media-conversion";
const manifest: PluginManifest = {
apiVersion: PLUGIN_API_VERSION,
id: MEDIA_CONVERSION_PLUGIN_ID,
name: "媒体转换节点",
version: "0.1.0",
publishedAt: "2026-09-08",
updatedAt: "2026-09-08",
description: "在画布中提供图片转换节点,支持灰度、Canny 边缘、AI 线稿、本地 Depth Anything V2 深度图和 OpenPose 姿态骨架。",
documentation:
"# 媒体转换节点\n\n连接图片到转换节点后,可以在节点内选择灰度图、Canny 边缘、AI 线稿、深度图或姿态骨架。AI 线稿使用本机 ControlNet Aux 预处理模型,深度图使用本机已安装的 Depth Anything V2 Small 模型,姿态骨架使用 ControlNet Aux 的 OpenPose 人体预处理模型运行;这些操作都不会加载 Stable Diffusion 重绘管线。姿态转换只绘制人体骨架,未检测到人物时会跳过并提示。视频转换和透明抠图模型仍在验证,暂时会明确提示不可用。\n\n该插件只读取当前节点的媒体输入,并把结果保存回当前画布节点的本地素材存储。",
author: "影策团队",
surfaces: ["node"],
permissions: ["canvas.read", "canvas.write", "media.read"],
trusted: true,
runtime: { web: "declarative" },
contributes: {
transforms: [
{
id: MEDIA_CONVERSION_NODE_TYPE,
input: "media",
output: "media",
runtime: "declarative",
},
],
},
};
export const mediaConversionPlugin: RegisteredPlugin = { manifest };
registerPlugin(mediaConversionPlugin);

View File

@@ -103,8 +103,10 @@ export type PluginCanvasNodeContribution = {
defaultSize: { width: number; height: number };
schema: Record<string, unknown>;
renderer: "declarative" | "sandbox";
/** Optional input contract for nodes that consume one media kind. */
acceptsInputKind?: "image" | "video" | "audio" | "text";
/** Optional input contract for nodes that consume one or more media kinds. */
acceptsInputKind?: "image" | "video" | "audio" | "text" | Array<"image" | "video" | "audio" | "text">;
/** Optional maximum number of direct inputs. */
maxInputCount?: number;
/** Analysis/sink nodes can hide the right-side output connection. */
showOutputConnection?: boolean;
};

View File

@@ -10,6 +10,7 @@ import "@/lib/plugins/builtin";
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion";
import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry";
import type { PluginManifest, PluginManifestV2 } from "@/lib/plugins/plugin-types";
import { fetchAdminPlugins, setPluginPlatformAvailability, uninstallPlugin, uploadPlugin, type AdminPluginState, type BackendPlugin, type PluginManagement } from "@/services/api/plugins";
@@ -26,7 +27,7 @@ type AdminPluginItem = {
error?: string;
};
const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance"]);
const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", MEDIA_CONVERSION_PLUGIN_ID]);
export default function AdminPluginsPage() {
const { message, modal } = App.useApp();

View File

@@ -72,7 +72,7 @@ import { CanvasLocalAgentPanel } from "@/components/canvas/canvas-local-agent-pa
import { useFocusMode } from "@/hooks/use-focus-mode";
import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store";
import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays";
import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay, type PendingConnectionCreate } from "@/components/canvas/canvas-workspace-overlays";
import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer";
import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer";
import { CanvasFreeformEmptyState, CanvasLinkedProjectEmptyState, CanvasShortDramaEmptyState, CanvasShortDramaGuide, CanvasStoryInputNodeContent, CanvasStylePlaceholderNodeContent } from "@/components/canvas/canvas-short-drama-entry";
@@ -1018,6 +1018,20 @@ function InfiniteCanvasPage() {
.filter((node) => selectedNodeIds.has(node.id) && !batchSourceRestriction(node))
.map((node) => node.id), [nodes, selectedNodeIds]);
const createConversionFromSource = useCallback((source: CanvasNodeData) => {
if (source.type !== CanvasNodeType.Image && source.type !== CanvasNodeType.Video) return;
const spec = getNodeSpec(CanvasNodeType.MediaConversion);
const pending: PendingConnectionCreate = {
connection: { nodeId: source.id, handleType: "source", anchorRatio: 0.5 },
position: {
x: source.position.x + source.width + 96 + spec.width / 2,
y: source.position.y + source.height / 2,
},
quick: true,
};
void createConnectedNode(CanvasNodeType.MediaConversion, pending);
}, [createConnectedNode]);
const handleCanvasSelectionStart = useCallback(() => {
setContextMenu(null);
}, []);
@@ -1048,6 +1062,8 @@ function InfiniteCanvasPage() {
} else if (node.type === ART_CRITIQUE_NODE_TYPE) {
setDialogNodeId(null);
setArtCritiqueNodeId(node.id);
} else if (node.type === CanvasNodeType.MediaConversion) {
setDialogNodeId(null);
} else {
// 选择参考媒体时保留当前工作流配置面板,避免点击图片后配置“返回/消失”。
// 没有工作流配置面板时,媒体节点仍按原逻辑打开自己的面板。
@@ -1065,7 +1081,7 @@ function InfiniteCanvasPage() {
const handleNodeDragEnd = useCallback((nodeId: string) => {
const node = nodesRef.current.find((item) => item.id === nodeId);
if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) {
if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) {
setDialogNodeId(null);
return;
}
@@ -1895,7 +1911,7 @@ function InfiniteCanvasPage() {
const renderCanvasNodePanel = useCallback(
(panelNode: CanvasNodeData) => {
if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing) return null;
if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing || panelNode.type === CanvasNodeType.MediaConversion) return null;
return panelNode.type === CanvasNodeType.Config ? (
<CanvasConfigComposer
value={panelNode.metadata?.composerContent ?? panelNode.metadata?.prompt ?? ""}
@@ -2572,6 +2588,7 @@ function InfiniteCanvasPage() {
onUpload={(node) => handleUploadRequest(node.id)}
onDownload={downloadNodeImage}
onSaveAsset={(node) => void saveNodeAsset(node)}
onCreateConversion={createConversionFromSource}
onAnnotate={(node) => setAnnotationNodeId(node.id)}
onMaskEdit={(node) => setMaskEditNodeId(node.id)}
onEmotion={(node) => {

View File

@@ -200,7 +200,7 @@ export function useCanvasConnectionController({
setContextMenu(null);
}, [config, connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]);
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
const nodeType = type;
if (nodeType === CanvasNodeType.Drawing && !isDrawingEngineAvailable(defaultDrawingEngine, tldrawLicenseKey)) {
message.error("当前生产构建未配置 tldraw License Key,不能创建 tldraw 绘图");
@@ -279,6 +279,12 @@ export function useCanvasConnectionController({
setConnecting(null);
return;
}
if (batchSourceNodeIds.length && nodeType === CanvasNodeType.MediaConversion) {
message.error("转换节点一次只能连接一个输入,请先连接到普通节点");
closeConnectionCreateMenu();
setConnecting(null);
return;
}
const batchPlan = batchSourceNodeIds.length
? planBatchConnections({ sourceNodeIds: batchSourceNodeIds, targetNodeId: newNode.id, nodes: [...nodesRef.current, newNode], connections: connectionsRef.current, config, allowCapacityOverflow: true })
: null;
@@ -298,7 +304,7 @@ export function useCanvasConnectionController({
setConnections(nextConnections);
setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null);
if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id);
const skippedCount = batchPlan.skipped.length;
const duplicateCount = batchPlan.duplicates.length;
const suffix = skippedCount || duplicateCount ? `,跳过 ${skippedCount + duplicateCount} 个` : "";
@@ -364,17 +370,18 @@ export function useCanvasConnectionController({
setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null);
if (nodeType === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id);
else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id);
closeConnectionCreateMenu();
setConnecting(null);
}, [closeConnectionCreateMenu, config, connectionsRef, defaultDrawingEngine, message, nodesRef, projectId, runtimeStatuses, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, tldrawLicenseKey]);
const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
const nodeType = type;
if (nodeType === CanvasNodeType.Config) {
if (workflowProvider && !workflowProviderPluginEnabled(runtimeStatuses, workflowProvider)) return `${workflowProvider === "runninghub" ? "RunningHub" : "ComfyUI"} 工作流插件未启用`;
}
if (pending.batchSourceNodeIds?.length) {
if (nodeType === CanvasNodeType.MediaConversion) return "转换节点一次只能连接一个输入";
if (nodeType === CanvasNodeType.Drawing || nodeType === CanvasNodeType.Config) return "批量连接暂不支持此节点类型";
const pendingNode: CanvasNodeData = { id: "__pending-connection-node__", type: nodeType, title: "", position: pending.position, width: getNodeSpec(nodeType).width, height: getNodeSpec(nodeType).height };
const plan = planBatchConnections({ sourceNodeIds: pending.batchSourceNodeIds, targetNodeId: pendingNode.id, nodes: [...nodesRef.current, pendingNode], connections: connectionsRef.current, config, allowCapacityOverflow: true });

View File

@@ -153,7 +153,7 @@ export function useCanvasNodeOperations({
if (workflowTitle) node.title = workflowTitle;
commitNodes([...nodesRef.current, node]);
selectNodes(new Set([node.id]));
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id);
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== CanvasNodeType.MediaConversion && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id);
}, [commitNodes, defaultDrawingEngine, effectiveConfig.comfyBridge.enabled, effectiveConfig.comfyBridge.workflows.length, effectiveConfig.runningHub.enabled, effectiveConfig.runningHub.workflows.length, getCanvasCenter, message, nodesRef, runtimeStatuses, selectNodes, setDialogNodeId, tldrawLicenseKey]);
const createFolder = useCallback((position?: Position, linked?: { id: string; projectId: string; title: string; style: CanvasFolderStyle; theme: CanvasFolderTheme; createdAt: string }) => {
@@ -400,7 +400,7 @@ export function useCanvasNodeOperations({
const sourceNode = sourceByTargetId.get(targetNode.id);
if (sourceNode) cloneDrawingForNode(sourceNode, targetNode, "绘图副本保存失败,请重新复制");
});
if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id);
if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== CanvasNodeType.MediaConversion && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id);
}, [cloneDrawingForNode, commitConnections, commitNodes, connectionsRef, nodesRef, selectNodes, setDialogNodeId]);
const setPrimaryVersion = useCallback((nodeId: string) => {

View File

@@ -124,7 +124,7 @@ export function useCanvasViewportController({
const scale = Math.min(1.18, Math.max(viewportRef.current.k, 0.72));
transitionViewportTo({ x: size.width / 2 - (node.position.x + node.width / 2) * scale, y: size.height / 2 - (node.position.y + node.height / 2) * scale, k: scale });
selectFocusedNode(node.id);
setDialogNodeId(node.type === CanvasNodeType.Drawing ? null : node.id);
setDialogNodeId(node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion ? null : node.id);
}, [nodesRef, selectFocusedNode, setDialogNodeId, size.height, size.width, transitionViewportTo, viewportRef]);
const setZoomScale = useCallback((scale: number) => {

View File

@@ -9,6 +9,7 @@ import "@/lib/plugins/builtin";
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion";
import { ART_CRITIQUE_PLUGIN_ID } from "@/lib/art-critique/contracts";
import type { PluginManifest, PluginManifestV2, RegisteredPlugin } from "@/lib/plugins/plugin-types";
import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle";
@@ -623,7 +624,7 @@ function toRegisteredPlugin(plugin: BackendPlugin): RegisteredPlugin {
}
function isOfficialApplicationPlugin(pluginId: string) {
return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID].includes(pluginId);
return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID, MEDIA_CONVERSION_PLUGIN_ID].includes(pluginId);
}
function pluginSourceLabel(plugin: RegisteredPlugin, state?: PluginState) {

View File

@@ -0,0 +1,202 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import type { LocalRuntimeTransport } from "@/services/local-runtime";
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LocalDepthResult = {
blob: Blob;
width: number;
height: number;
modelId: string;
device: "cpu";
};
export type LocalDepthRuntimeStatus = {
ok: true;
module: "depth-estimation";
apiVersion: 1;
ready: boolean;
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
export async function runLocalDepthEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalDepthResult> {
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
const value = await requestJson<unknown>("/depth-estimation/run", {
method: "POST",
body: JSON.stringify({ schemaVersion: 1, operation: "depth", dataUrl }),
headers: { "content-type": "application/json" },
signal,
});
if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果响应无效");
}
return parseResult(value.result);
}
export async function readLocalDepthRuntimeStatus(signal?: AbortSignal) {
return parseStatus(await requestJson<unknown>("/depth-estimation/status", { method: "GET", signal }));
}
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
if (sourceUrl.startsWith("data:")) {
const match = DATA_URL_PATTERN.exec(sourceUrl);
if (!match) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = bytesFromBase64(match[2]!);
assertImageSize(bytes);
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
}
let response: Response;
try {
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
} catch (error) {
if (signal?.aborted) throw error;
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取");
}
if (!response.ok) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取", response.status);
const mimeType = imageMime(response.headers.get("content-type") || response.type);
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) {
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
}
const bytes = new Uint8Array(await response.arrayBuffer());
assertImageSize(bytes);
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
}
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
if (!response.ok) {
const code = isRecord(body) && typeof body.code === "string" ? body.code : "depth_runtime_unavailable";
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机深度运行时不可用";
throw new LocalRuntimeClientError(code, message, response.status);
}
return body as T;
}
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
let refreshed = false;
while (true) {
let response: Response;
try {
response = await (await ensureTransport()).request(path, init);
} catch (error) {
if (refreshed || !isSessionRefreshError(error)) throw error;
await reconnectTransport();
refreshed = true;
continue;
}
if (!refreshed && response.status === 401) {
await reconnectTransport();
refreshed = true;
continue;
}
if (response.status === 403) {
const body = await parseJsonResponse(response);
if (!refreshed && shouldRefreshSession(response, body)) {
await reconnectTransport();
refreshed = true;
continue;
}
return { response, body };
}
return { response };
}
}
async function ensureTransport(): Promise<LocalRuntimeTransport> {
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") await state.connect();
const current = useLocalRuntimeStore.getState();
if (current.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", current.error || "本机深度运行时不可用");
return getLocalRuntimeSessionClient();
}
async function reconnectTransport() {
const client = getLocalRuntimeSessionClient();
client.revokeLocalSession();
await useLocalRuntimeStore.getState().connect();
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", state.error || "本机深度运行时不可用");
}
async function parseJsonResponse(response: Response) {
const text = await boundedText(response, MAX_RESPONSE_BYTES);
try {
return text ? JSON.parse(text) as unknown : {};
} catch {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应无效", response.status);
}
}
async function boundedText(response: Response, limit: number) {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status);
const text = await response.text();
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status);
return text;
}
function parseResult(value: Record<string, unknown>): LocalDepthResult {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果无效");
}
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机深度结果无效");
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果过大");
return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" };
}
function parseStatus(value: unknown): LocalDepthRuntimeStatus {
if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度状态无效");
}
return value as LocalDepthRuntimeStatus;
}
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
}
function assertImageSize(bytes: Uint8Array) {
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
}
function bytesToBase64(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary);
}
function bytesFromBase64(value: string, code = "depth_input_invalid", message = "深度转换输入图片无效") {
let binary: string;
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "depth_input_invalid" ? 400 : 0); }
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
return bytes;
}
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
}
function shouldRefreshSession(response: Response, body: unknown) {
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}

View File

@@ -0,0 +1,202 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import type { LocalRuntimeTransport } from "@/services/local-runtime";
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LocalLineartResult = {
blob: Blob;
width: number;
height: number;
modelId: string;
device: "cpu";
};
export type LocalLineartRuntimeStatus = {
ok: true;
module: "lineart-estimation";
apiVersion: 1;
ready: boolean;
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
export async function runLocalLineartEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalLineartResult> {
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
const value = await requestJson<unknown>("/lineart-estimation/run", {
method: "POST",
body: JSON.stringify({ schemaVersion: 1, operation: "lineart", dataUrl }),
headers: { "content-type": "application/json" },
signal,
});
if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果响应无效");
}
return parseResult(value.result);
}
export async function readLocalLineartRuntimeStatus(signal?: AbortSignal) {
return parseStatus(await requestJson<unknown>("/lineart-estimation/status", { method: "GET", signal }));
}
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
if (sourceUrl.startsWith("data:")) {
const match = DATA_URL_PATTERN.exec(sourceUrl);
if (!match) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = bytesFromBase64(match[2]!);
assertImageSize(bytes);
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
}
let response: Response;
try {
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
} catch (error) {
if (signal?.aborted) throw error;
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取");
}
if (!response.ok) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取", response.status);
const mimeType = imageMime(response.headers.get("content-type") || response.type);
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) {
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
}
const bytes = new Uint8Array(await response.arrayBuffer());
assertImageSize(bytes);
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
}
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
if (!response.ok) {
const code = isRecord(body) && typeof body.code === "string" ? body.code : "lineart_runtime_unavailable";
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机 AI 线稿运行时不可用";
throw new LocalRuntimeClientError(code, message, response.status);
}
return body as T;
}
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
let refreshed = false;
while (true) {
let response: Response;
try {
response = await (await ensureTransport()).request(path, init);
} catch (error) {
if (refreshed || !isSessionRefreshError(error)) throw error;
await reconnectTransport();
refreshed = true;
continue;
}
if (!refreshed && response.status === 401) {
await reconnectTransport();
refreshed = true;
continue;
}
if (response.status === 403) {
const body = await parseJsonResponse(response);
if (!refreshed && shouldRefreshSession(response, body)) {
await reconnectTransport();
refreshed = true;
continue;
}
return { response, body };
}
return { response };
}
}
async function ensureTransport(): Promise<LocalRuntimeTransport> {
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") await state.connect();
const current = useLocalRuntimeStore.getState();
if (current.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", current.error || "本机 AI 线稿运行时不可用");
return getLocalRuntimeSessionClient();
}
async function reconnectTransport() {
const client = getLocalRuntimeSessionClient();
client.revokeLocalSession();
await useLocalRuntimeStore.getState().connect();
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", state.error || "本机 AI 线稿运行时不可用");
}
async function parseJsonResponse(response: Response) {
const text = await boundedText(response, MAX_RESPONSE_BYTES);
try {
return text ? JSON.parse(text) as unknown : {};
} catch {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应无效", response.status);
}
}
async function boundedText(response: Response, limit: number) {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status);
const text = await response.text();
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status);
return text;
}
function parseResult(value: Record<string, unknown>): LocalLineartResult {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果无效");
}
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机 AI 线稿结果无效");
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果过大");
return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" };
}
function parseStatus(value: unknown): LocalLineartRuntimeStatus {
if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿状态无效");
}
return value as LocalLineartRuntimeStatus;
}
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
}
function assertImageSize(bytes: Uint8Array) {
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
}
function bytesToBase64(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary);
}
function bytesFromBase64(value: string, code = "lineart_input_invalid", message = "AI 线稿输入图片无效") {
let binary: string;
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "lineart_input_invalid" ? 400 : 0); }
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
return bytes;
}
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
}
function shouldRefreshSession(response: Response, body: unknown) {
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}

View File

@@ -1,7 +1,7 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance";
export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read";
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read" | "depth:status" | "depth:run" | "lineart:status" | "lineart:run" | "pose:status" | "pose:run";
export type LocalRuntimeModuleDescriptor = {
id: LocalRuntimeModuleId;
@@ -28,6 +28,9 @@ const MODULE_SCOPES: Record<LocalRuntimeModuleId, ReadonlySet<LocalRuntimeScope>
"canvas-agent": new Set(["canvas:connect"]),
dreamina: new Set(["dreamina:status", "dreamina:login", "dreamina:logout", "dreamina:run", "dreamina:models", "dreamina:generate"]),
"portrait-clearance": new Set(["portrait:status", "portrait:model", "portrait:run", "portrait:read"]),
"depth-estimation": new Set(["depth:status", "depth:run"]),
"lineart-estimation": new Set(["lineart:status", "lineart:run"]),
"pose-estimation": new Set(["pose:status", "pose:run"]),
};
export async function readLocalRuntimeStatus(client: LocalRuntimeTransport, signal?: AbortSignal): Promise<LocalRuntimeStatus> {
@@ -136,7 +139,7 @@ function isRecord(value: unknown): value is Record<string, unknown> {
}
function isModuleId(value: unknown): value is LocalRuntimeModuleId {
return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance";
return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance" || value === "depth-estimation" || value === "lineart-estimation" || value === "pose-estimation";
}
function invalidResponse(status: number) {

View File

@@ -0,0 +1,209 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import type { LocalRuntimeTransport } from "@/services/local-runtime";
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LocalPoseResult = {
blob: Blob;
width: number;
height: number;
modelId: string;
device: "cpu";
backend: "openpose-body";
personCount: number;
people?: unknown[];
};
export type LocalPoseRuntimeStatus = {
ok: true;
module: "pose-estimation";
apiVersion: 1;
ready: boolean;
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
backend: "openpose-body";
reason?: string;
};
export async function runLocalPoseEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalPoseResult> {
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
const value = await requestJson<unknown>("/pose-estimation/run", {
method: "POST",
body: JSON.stringify({ schemaVersion: 1, operation: "pose", dataUrl }),
headers: { "content-type": "application/json" },
signal,
});
if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果响应无效");
}
return parseResult(value.result);
}
export async function readLocalPoseRuntimeStatus(signal?: AbortSignal) {
return parseStatus(await requestJson<unknown>("/pose-estimation/status", { method: "GET", signal }));
}
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
if (sourceUrl.startsWith("data:")) {
const match = DATA_URL_PATTERN.exec(sourceUrl);
if (!match) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = bytesFromBase64(match[2]!);
assertImageSize(bytes);
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
}
let response: Response;
try {
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
} catch (error) {
if (signal?.aborted) throw error;
throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取");
}
if (!response.ok) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取", response.status);
const mimeType = imageMime(response.headers.get("content-type") || response.type);
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
const bytes = new Uint8Array(await response.arrayBuffer());
assertImageSize(bytes);
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
}
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
if (!response.ok) {
const code = isRecord(body) && typeof body.code === "string" ? body.code : "pose_runtime_unavailable";
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机姿态运行时不可用";
throw new LocalRuntimeClientError(code, message, response.status);
}
return body as T;
}
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
let refreshed = false;
while (true) {
let response: Response;
try {
response = await (await ensureTransport()).request(path, init);
} catch (error) {
if (refreshed || !isSessionRefreshError(error)) throw error;
await reconnectTransport();
refreshed = true;
continue;
}
if (!refreshed && response.status === 401) {
await reconnectTransport();
refreshed = true;
continue;
}
if (response.status === 403) {
const body = await parseJsonResponse(response);
if (!refreshed && shouldRefreshSession(response, body)) {
await reconnectTransport();
refreshed = true;
continue;
}
return { response, body };
}
return { response };
}
}
async function ensureTransport(): Promise<LocalRuntimeTransport> {
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") await state.connect();
const current = useLocalRuntimeStore.getState();
if (current.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", current.error || "本机姿态运行时不可用");
return getLocalRuntimeSessionClient();
}
async function reconnectTransport() {
const client = getLocalRuntimeSessionClient();
client.revokeLocalSession();
await useLocalRuntimeStore.getState().connect();
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", state.error || "本机姿态运行时不可用");
}
async function parseJsonResponse(response: Response) {
const text = await boundedText(response, MAX_RESPONSE_BYTES);
try { return text ? JSON.parse(text) as unknown : {}; } catch { throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应无效", response.status); }
}
async function boundedText(response: Response, limit: number) {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status);
const text = await response.text();
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status);
return text;
}
function parseResult(value: Record<string, unknown>): LocalPoseResult {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || value.backend !== "openpose-body" || !isPositiveInteger(value.personCount) || typeof value.pngBase64 !== "string") {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果无效");
}
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机姿态结果无效");
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果过大");
return {
blob: new Blob([bytes], { type: "image/png" }),
width: value.width,
height: value.height,
modelId: value.modelId,
device: "cpu",
backend: "openpose-body",
personCount: value.personCount,
people: Array.isArray(value.people) ? value.people : undefined,
};
}
function parseStatus(value: unknown): LocalPoseRuntimeStatus {
if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || value.backend !== "openpose-body" || (value.reason !== undefined && typeof value.reason !== "string")) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态状态无效");
}
return value as LocalPoseRuntimeStatus;
}
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
}
function assertImageSize(bytes: Uint8Array) {
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
}
function bytesToBase64(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary);
}
function bytesFromBase64(value: string, code = "pose_input_invalid", message = "姿态转换输入图片无效") {
let binary: string;
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "pose_input_invalid" ? 400 : 0); }
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
return bytes;
}
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
}
function shouldRefreshSession(response: Response, body: unknown) {
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}

View File

@@ -1,4 +1,5 @@
import type { CanvasColorGrade } from "@/lib/canvas/canvas-color-grade";
import type { MediaConversionNodeState } from "@/lib/media-conversion/contracts";
import type { AssetCategory } from "@/lib/asset-category";
import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture";
import type { StyleExecutionPlan } from "@/lib/canvas/style-profile";
@@ -34,6 +35,7 @@ export enum CanvasNodeType {
Compare = "compare",
Chart = "chart",
ColorGrade = "colorgrade",
MediaConversion = "media-conversion",
}
/** Runtime IDs contributed by plugins share the persisted node type field. */
@@ -362,6 +364,8 @@ export type CanvasNodeMetadata = {
chartKind?: "bar" | "line";
/** 调色节点的参数;缺省视为未调色。 */
colorGrade?: CanvasColorGrade;
/** 本地图片/视频转换节点的参数、来源指纹和结果状态。 */
mediaConversion?: MediaConversionNodeState;
/** 用户手动拉伸过尺寸;图片按真实比例自动适配时避让它。 */
manualSize?: boolean;
storyboard?: StoryboardData;