tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): 图片转换插件 - 增加本地媒体转换节点与运行时

Browse Source
This commit is contained in:
QAyong
2026-09-08 22:55:44 +08:00
parent 8360b2c933
commit f755d6db97
51 changed files with 3284 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
web/src/lib/canvas/canvas-agent-ops.ts
View File

@@ -399,6 +399,7 @@ function canvasNodeTypeLabel(type?: CanvasNodeType) {
if (type === CanvasNodeType.Frame) return "背板";
if (type === CanvasNodeType.Drawing) return "绘图节点";
if (type === CanvasNodeType.Skill) return "技能节点";
if (type === CanvasNodeType.MediaConversion) return "转换节点";
return "文本节点";
}

26
web/src/lib/canvas/canvas-connection-policy.ts
View File

@@ -1,7 +1,7 @@
import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection";
import { getNodeAcceptedInputKind, getNodeGenerationMode, getNodeInputKind } from "@/lib/canvas/node-registry";
import { getNodeAcceptedInputKinds, getNodeGenerationMode, getNodeInputKind, getNodeMaxInputCount } from "@/lib/canvas/node-registry";
import type { AiConfig } from "@/stores/use-config-store";
import { type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
type ConnectionCandidate = Pick<CanvasConnection, "fromNodeId" | "toNodeId">;
type CanvasConnectionPolicyOptions = {
@@ -12,11 +12,27 @@ type CanvasConnectionPolicyOptions = {
export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) {
const target = nodes.find((node) => node.id === candidate.toNodeId);
if (!target) return "找不到连线目标节点";
const acceptedInputKind = getNodeAcceptedInputKind(target.type);
if (acceptedInputKind) {
const acceptedInputKinds = getNodeAcceptedInputKinds(target.type);
if (acceptedInputKinds.length) {
const source = nodes.find((node) => node.id === candidate.fromNodeId);
const sourceKind = source ? getNodeInputKind(source.type) : undefined;
if (sourceKind !== acceptedInputKind) return `${acceptedInputKindLabel(acceptedInputKind)}节点只接受${acceptedInputKindLabel(acceptedInputKind)}输入`;
const isMediaConversion = target.type === CanvasNodeType.MediaConversion;
const hasAcceptedSource = isMediaConversion
? source?.type === CanvasNodeType.Image || source?.type === CanvasNodeType.Video
: Boolean(sourceKind && acceptedInputKinds.includes(sourceKind));
if (!sourceKind || !hasAcceptedSource) {
const labels = acceptedInputKinds.map(acceptedInputKindLabel).join("或");
return `${isMediaConversion ? "转换" : labels}节点只接受${labels}输入`;
}
const maxInputCount = getNodeMaxInputCount(target.type);
if (maxInputCount) {
const inputCount = new Set(
[...connections, { id: "candidate", ...candidate }]
.filter((connection) => connection.toNodeId === target.id)
.map((connection) => connection.fromNodeId),
).size;
if (inputCount > maxInputCount) return `${isMediaConversion ? "转换" : "当前"}节点最多连接 ${maxInputCount} 个输入`;
}
}
const mode = getNodeGenerationMode(target);
if (!mode) return "";

20
web/src/lib/canvas/node-registry/definitions/builtin-nodes.tsx
View File

@@ -1,4 +1,4 @@
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video } from "lucide-react";
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video, WandSparkles } from "lucide-react";
import { NODE_SPECS } from "@/constant/canvas";
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
@@ -25,7 +25,7 @@ const BUILTIN_NODE_TRAITS = {
minSize: MEDIA_NODE_MIN_SIZE,
keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize,
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "image" : null),
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "image" : null),
generationMode: () => "image",
inputKind: "image",
},
@@ -82,7 +82,7 @@ const BUILTIN_NODE_TRAITS = {
minSize: MEDIA_NODE_MIN_SIZE,
keepAspectRatio: () => true,
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "video" : null),
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "video" : null),
generationMode: () => "video",
inputKind: "video",
},
@@ -165,6 +165,20 @@ const BUILTIN_NODE_TRAITS = {
resourceKind: () => "image",
inputKind: "image",
},
[CanvasNodeType.MediaConversion]: {
label: "转换",
icon: <WandSparkles />,
minSize: { width: 320, height: 360 },
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => {
const conversion = node.metadata?.mediaConversion;
if (conversion?.status !== "completed" || !conversion.resultStorageKey) return null;
return conversion.outputKind === "video" ? "video" : "image";
},
acceptsInputKind: ["image", "video"],
maxInputCount: 1,
inputKind: "image",
},
} satisfies Record<string, Omit<CanvasNodeDefinition, "type" | "defaultTitle" | "defaultSize" | "defaultMetadata">>;
export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => {

2
web/src/lib/canvas/node-registry/index.ts
View File

@@ -1,3 +1,3 @@
export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition";
export { getNodeAcceptedInputKind, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
export { getNodeAcceptedInputKind, getNodeAcceptedInputKinds, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMaxInputCount, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
import "./definitions";

7
web/src/lib/canvas/node-registry/node-definition.ts
View File

@@ -43,8 +43,10 @@ export type CanvasNodeDefinition = {
generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null;
/** 是否显示右侧输出连接点；缺省为 true，消费型终点节点可关闭。 */
showOutputConnection?: boolean;
/** 若设置，作为目标时只接受这一类上游节点。 */
acceptsInputKind?: CanvasNodeInputKind;
/** 若设置，作为目标时只接受这些上游节点类型。 */
acceptsInputKind?: CanvasNodeInputKind | CanvasNodeInputKind[];
/** 限制直接输入数量；转换节点等一进一出节点通常为 1。 */
maxInputCount?: number;
/**
* 作为上游输入被参考素材容量校验计数时的类别；
* 不设表示不参与计数（生成配置、背板）。与 resourceKind 不同，计数不看内容。
@@ -69,6 +71,7 @@ export function canvasNodeDefinitionFromPlugin(pluginId: string, contribution: P
showInCreateMenu: true,
showOutputConnection: contribution.showOutputConnection,
acceptsInputKind: contribution.acceptsInputKind,
maxInputCount: contribution.maxInputCount,
plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema },
};
}

10
web/src/lib/canvas/node-registry/node-registry.ts
View File

@@ -100,3 +100,13 @@ export function getNodeInputKind(type: CanvasNodeTypeId) {
export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) {
return definitions.get(type)?.acceptsInputKind;
}
export function getNodeAcceptedInputKinds(type: CanvasNodeTypeId) {
const accepted = definitions.get(type)?.acceptsInputKind;
if (!accepted) return [];
return Array.isArray(accepted) ? accepted : [accepted];
}
export function getNodeMaxInputCount(type: CanvasNodeTypeId) {
return definitions.get(type)?.maxInputCount;
}

1
web/src/lib/canvas/tool-registry/definitions/add-node-menu-tools.tsx
View File

@@ -20,6 +20,7 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }),
// 导演台落在节点分区，但它开的是导演工作台、不是某种画布节点，故不走注册表。
{ id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() },
nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }),

15
web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx
View File

@@ -1,4 +1,4 @@
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video, WandSparkles } from "lucide-react";
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
@@ -139,6 +139,19 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx),
run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!),
},
{
id: "createConversion",
toolbar: "node-hover",
category: "node-state",
label: "创建转换节点",
displayLabel: "转换",
icon: <WandSparkles className="size-3.5" />,
defaultVisible: true,
defaultOrder: 65,
nodeToolbar: { group: "primary", order: 15, description: "在右侧创建转换节点并自动连接" },
applicable: (ctx) => (hasImage(ctx) || hasVideo(ctx)) && Boolean(ctx.handlers.onNodeCreateConversion),
run: (ctx) => ctx.handlers.onNodeCreateConversion?.(ctx.node!),
},
{
id: "edit",
toolbar: "node-hover",

2
web/src/lib/canvas/tool-registry/tool-definition.ts
View File

@@ -81,6 +81,8 @@ export type ToolbarHandlers = {
onNodeUpload: (node: CanvasNodeData) => void;
onNodeDownload: (node: CanvasNodeData) => void;
onNodeSaveAsset: (node: CanvasNodeData) => void;
/** 从已有图片或视频创建并自动连接一个转换节点。 */
onNodeCreateConversion?: (node: CanvasNodeData) => void;
onNodeMaskEdit: (node: CanvasNodeData) => void;
onNodeEmotion: (node: CanvasNodeData) => void;
onNodePortraitTexture: (node: CanvasNodeData) => void;

89
web/src/lib/media-conversion/contracts.ts Normal file
View File

@@ -0,0 +1,89 @@
import type { CanvasNodeData } from "@/types/canvas";
export const MEDIA_CONVERSION_SCHEMA_VERSION = 1 as const;
export const MEDIA_CONVERSION_NODE_TYPE = "media-conversion" as const;
export type MediaConversionOperation = "grayscale" | "edge-canny" | "lineart" | "depth" | "pose" | "cutout";
export type MediaConversionStatus = "idle" | "processing" | "completed" | "stale" | "skipped" | "unavailable" | "error";
export type MediaConversionOutputKind = "image" | "video";
export type MediaConversionNodeState = {
schemaVersion: typeof MEDIA_CONVERSION_SCHEMA_VERSION;
operation: MediaConversionOperation;
status: MediaConversionStatus;
sourceNodeId?: string;
sourceFingerprint?: string;
outputKind?: MediaConversionOutputKind;
resultStorageKey?: string;
resultWidth?: number;
resultHeight?: number;
detectedPeople?: number;
errorCode?: string;
errorMessage?: string;
startedAt?: string;
updatedAt?: string;
};
export const MEDIA_CONVERSION_OPERATION_LABELS: Record<MediaConversionOperation, string> = {
grayscale: "灰度图",
"edge-canny": "Canny 边缘",
lineart: "AI 线稿",
depth: "深度图",
pose: "姿态骨架",
cutout: "透明抠图",
};
export const MEDIA_CONVERSION_OPERATION_DESCRIPTIONS: Record<MediaConversionOperation, string> = {
grayscale: "普通图像算法，适合先检查明暗层次",
"edge-canny": "普通图像算法，提取清晰轮廓",
lineart: "本地 ControlNet Aux 线稿预处理，首次运行需要加载模型",
depth: "本地 Depth Anything V2 Small，首次运行需要加载模型",
pose: "本地 OpenPose 人体骨架预处理，未检测到人物时会跳过",
cutout: "需要安装并验证本地抠图模型",
};
export function createDefaultMediaConversionState(): MediaConversionNodeState {
return {
schemaVersion: MEDIA_CONVERSION_SCHEMA_VERSION,
operation: "edge-canny",
status: "idle",
};
}
export function mediaConversionSourceFingerprint(node: CanvasNodeData) {
const metadata = node.metadata;
const source = [
node.id,
metadata?.storageKey || "",
compactSource(metadata?.content || metadata?.previewContent || ""),
metadata?.mimeType || "",
metadata?.bytes || "",
metadata?.naturalWidth || "",
metadata?.naturalHeight || "",
metadata?.durationMs || "",
].join("|");
let hash = 2166136261;
for (let index = 0; index < source.length; index += 1) {
hash ^= source.charCodeAt(index);
hash = Math.imul(hash, 16777619);
}
return `v1-${(hash >>> 0).toString(16).padStart(8, "0")}`;
}
export function mediaConversionOperationLabel(operation: MediaConversionOperation) {
return MEDIA_CONVERSION_OPERATION_LABELS[operation];
}
export function mediaConversionOperationDescription(operation: MediaConversionOperation) {
return MEDIA_CONVERSION_OPERATION_DESCRIPTIONS[operation];
}
export function isLocalImageOperation(operation: MediaConversionOperation) {
return operation === "grayscale" || operation === "edge-canny";
}
function compactSource(value: string) {
if (value.length <= 2048) return value;
return `${value.slice(0, 1024)}|${value.slice(-1024)}|${value.length}`;
}

228
web/src/lib/media-conversion/local-converter.ts Normal file
View File

@@ -0,0 +1,228 @@
import type { MediaConversionOperation } from "./contracts";
export type LocalImageConversionResult = {
blob: Blob;
width: number;
height: number;
};
export class LocalImageConversionError extends Error {
constructor(readonly code: "model_missing" | "source_unreadable" | "browser_unsupported" | "aborted", message: string) {
super(message);
this.name = "LocalImageConversionError";
}
}
export async function convertImageLocally(sourceUrl: string, operation: MediaConversionOperation, options: { signal?: AbortSignal; maxDimension?: number } = {}): Promise<LocalImageConversionResult> {
throwIfAborted(options.signal);
if (operation !== "grayscale" && operation !== "edge-canny") {
throw new LocalImageConversionError("model_missing", "该转换需要先安装并验证本地模型");
}
const image = await loadImage(sourceUrl, options.signal);
throwIfAborted(options.signal);
const maxDimension = Math.max(256, options.maxDimension || 1024);
const scale = Math.min(1, maxDimension / Math.max(image.naturalWidth, image.naturalHeight));
const width = Math.max(1, Math.round(image.naturalWidth * scale));
const height = Math.max(1, Math.round(image.naturalHeight * scale));
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d", { willReadFrequently: true });
if (!context) throw new LocalImageConversionError("browser_unsupported", "当前浏览器无法创建图片处理画布");
context.drawImage(image, 0, 0, width, height);
const input = context.getImageData(0, 0, width, height);
const output = transformPixels(input.data, width, height, operation, options.signal);
const outputImage = new ImageData(output, width, height);
context.putImageData(outputImage, 0, 0);
const blob = await canvasToBlob(canvas, options.signal);
throwIfAborted(options.signal);
return { blob, width, height };
}
/** Pure pixel transform used by the browser executor and unit tests. */
export function transformPixels(source: Uint8ClampedArray, width: number, height: number, operation: "grayscale" | "edge-canny", signal?: AbortSignal) {
if (source.length !== width * height * 4) throw new Error("图片像素尺寸不匹配");
if (operation === "grayscale") return grayscalePixels(source, width, height, signal);
return cannyPixels(source, width, height, signal);
}
function grayscalePixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
const output = new Uint8ClampedArray(source);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
const value = Math.round(source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114);
output[index] = value;
output[index + 1] = value;
output[index + 2] = value;
}
}
return output;
}
function cannyPixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
const gray = new Float32Array(width * height);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
gray[y * width + x] = source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114;
}
}
const blurred = gaussianBlur(gray, width, height, signal);
const magnitude = new Float32Array(width * height);
const direction = new Float32Array(width * height);
let maximum = 0;
for (let y = 1; y < height - 1; y += 1) {
throwIfAborted(signal);
for (let x = 1; x < width - 1; x += 1) {
const index = y * width + x;
const horizontal = blurred[index + 1] - blurred[index - 1];
const vertical = blurred[index + width] - blurred[index - width];
const value = Math.hypot(horizontal, vertical);
magnitude[index] = value;
direction[index] = Math.atan2(vertical, horizontal);
maximum = Math.max(maximum, value);
}
}
const thinned = nonMaximumSuppression(magnitude, direction, width, height, signal);
const high = maximum * 0.28;
const low = high * 0.45;
const edges = new Uint8Array(width * height);
const stack: number[] = [];
for (let index = 0; index < thinned.length; index += 1) {
if (thinned[index] >= high && high > 0) {
edges[index] = 2;
stack.push(index);
} else if (thinned[index] >= low && low > 0) {
edges[index] = 1;
}
}
while (stack.length) {
throwIfAborted(signal);
const index = stack.pop()!;
const x = index % width;
const y = Math.floor(index / width);
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
if (!offsetX && !offsetY) continue;
const nextX = x + offsetX;
const nextY = y + offsetY;
if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) continue;
const next = nextY * width + nextX;
if (edges[next] !== 1) continue;
edges[next] = 2;
stack.push(next);
}
}
}
const output = new Uint8ClampedArray(source.length);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = y * width + x;
const outputIndex = index * 4;
const value = edges[index] === 2 ? 0 : 255;
output[outputIndex] = value;
output[outputIndex + 1] = value;
output[outputIndex + 2] = value;
output[outputIndex + 3] = source[outputIndex + 3];
}
}
return output;
}
function gaussianBlur(source: Float32Array, width: number, height: number, signal?: AbortSignal) {
const output = new Float32Array(source.length);
const kernel = [1, 2, 1, 2, 4, 2, 1, 2, 1];
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
let value = 0;
let weight = 0;
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
const nextX = Math.min(width - 1, Math.max(0, x + offsetX));
const nextY = Math.min(height - 1, Math.max(0, y + offsetY));
const kernelWeight = kernel[(offsetY + 1) * 3 + offsetX + 1];
value += source[nextY * width + nextX] * kernelWeight;
weight += kernelWeight;
}
}
output[y * width + x] = value / weight;
}
}
return output;
}
function nonMaximumSuppression(magnitude: Float32Array, direction: Float32Array, width: number, height: number, signal?: AbortSignal) {
const output = new Float32Array(magnitude.length);
for (let y = 1; y < height - 1; y += 1) {
throwIfAborted(signal);
for (let x = 1; x < width - 1; x += 1) {
const index = y * width + x;
let angle = (direction[index] * 180) / Math.PI;
if (angle < 0) angle += 180;
const value = magnitude[index];
let before = 0;
let after = 0;
if (angle < 22.5 || angle >= 157.5) {
before = magnitude[index - 1];
after = magnitude[index + 1];
} else if (angle < 67.5) {
before = magnitude[index - width + 1];
after = magnitude[index + width - 1];
} else if (angle < 112.5) {
before = magnitude[index - width];
after = magnitude[index + width];
} else {
before = magnitude[index - width - 1];
after = magnitude[index + width + 1];
}
output[index] = value >= before && value >= after ? value : 0;
}
}
return output;
}
async function loadImage(sourceUrl: string, signal?: AbortSignal) {
if (!sourceUrl) throw new LocalImageConversionError("source_unreadable", "没有可读取的图片输入");
const image = new Image();
image.crossOrigin = "anonymous";
return await new Promise<HTMLImageElement>((resolve, reject) => {
const abort = () => {
image.src = "";
reject(new LocalImageConversionError("aborted", "转换已取消"));
};
if (signal?.aborted) return abort();
signal?.addEventListener("abort", abort, { once: true });
image.onload = () => {
signal?.removeEventListener("abort", abort);
resolve(image);
};
image.onerror = () => {
signal?.removeEventListener("abort", abort);
reject(new LocalImageConversionError("source_unreadable", "图片无法读取，可能是跨域资源或已失效"));
};
image.src = sourceUrl;
});
}
function canvasToBlob(canvas: HTMLCanvasElement, signal?: AbortSignal) {
throwIfAborted(signal);
return new Promise<Blob>((resolve, reject) => {
canvas.toBlob((blob) => {
if (blob) resolve(blob);
else reject(new LocalImageConversionError("browser_unsupported", "当前浏览器无法导出图片结果"));
}, "image/png");
});
}
function throwIfAborted(signal?: AbortSignal): asserts signal is AbortSignal | undefined {
if (signal?.aborted) throw new LocalImageConversionError("aborted", "转换已取消");
}

1
web/src/lib/plugins/builtin/index.ts
View File

@@ -3,4 +3,5 @@ import "./prompt-optimizer";
import "./workflows";
import "./portrait-clearance";
import "./ai-art-critique";
import "./media-conversion";
import "./editor/editor-shell";

37
web/src/lib/plugins/builtin/media-conversion.ts Normal file
View File

@@ -0,0 +1,37 @@
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
import { registerPlugin } from "../plugin-registry";
import { PLUGIN_API_VERSION, type PluginManifest, type RegisteredPlugin } from "../plugin-types";
export const MEDIA_CONVERSION_PLUGIN_ID = "media-conversion";
const manifest: PluginManifest = {
apiVersion: PLUGIN_API_VERSION,
id: MEDIA_CONVERSION_PLUGIN_ID,
name: "媒体转换节点",
version: "0.1.0",
publishedAt: "2026-09-08",
updatedAt: "2026-09-08",
description: "在画布中提供图片转换节点，支持灰度、Canny 边缘、AI 线稿、本地 Depth Anything V2 深度图和 OpenPose 姿态骨架。",
documentation:
"# 媒体转换节点\n\n连接图片到转换节点后，可以在节点内选择灰度图、Canny 边缘、AI 线稿、深度图或姿态骨架。AI 线稿使用本机 ControlNet Aux 预处理模型，深度图使用本机已安装的 Depth Anything V2 Small 模型，姿态骨架使用 ControlNet Aux 的 OpenPose 人体预处理模型运行；这些操作都不会加载 Stable Diffusion 重绘管线。姿态转换只绘制人体骨架，未检测到人物时会跳过并提示。视频转换和透明抠图模型仍在验证，暂时会明确提示不可用。\n\n该插件只读取当前节点的媒体输入，并把结果保存回当前画布节点的本地素材存储。",
author: "影策团队",
surfaces: ["node"],
permissions: ["canvas.read", "canvas.write", "media.read"],
trusted: true,
runtime: { web: "declarative" },
contributes: {
transforms: [
{
id: MEDIA_CONVERSION_NODE_TYPE,
input: "media",
output: "media",
runtime: "declarative",
},
],
},
};
export const mediaConversionPlugin: RegisteredPlugin = { manifest };
registerPlugin(mediaConversionPlugin);

6
web/src/lib/plugins/plugin-types.ts
View File

@@ -103,8 +103,10 @@ export type PluginCanvasNodeContribution = {
defaultSize: { width: number; height: number };
schema: Record<string, unknown>;
renderer: "declarative" | "sandbox";
/** Optional input contract for nodes that consume one media kind. */
acceptsInputKind?: "image" | "video" | "audio" | "text";
/** Optional input contract for nodes that consume one or more media kinds. */
acceptsInputKind?: "image" | "video" | "audio" | "text" | Array<"image" | "video" | "audio" | "text">;
/** Optional maximum number of direct inputs. */
maxInputCount?: number;
/** Analysis/sink nodes can hide the right-side output connection. */
showOutputConnection?: boolean;
};