feat(canvas): 画布节点 - 恢复批量创作表与转换能力
This commit is contained in:
134
web/src/lib/canvas/canvas-batch-table.ts
Normal file
134
web/src/lib/canvas/canvas-batch-table.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import type { CanvasBatchOperation, CanvasBatchReferenceColumn, CanvasBatchRow, CanvasBatchTableData, CanvasConnection, CanvasNodeData } from "@/types/canvas";
|
||||
|
||||
export const TRY_ON_BATCH_PROMPT = "参考图1是人物原图,参考图2是目标服装。保持人物身份、五官、姿态和背景不变,将人物服装替换为参考图2中的款式。准确还原服装版型、颜色、材质、纹理和装饰细节,穿着关系自然,光影与原图一致。";
|
||||
export const CREATIVE_BATCH_PROMPT = "基于参考图创作一张新的商业图片,保留主体身份和关键产品细节,画面构图完整,光影自然。";
|
||||
export const BATCH_REFERENCE_HANDLE_PREFIX = "batch-reference:";
|
||||
export const BATCH_REFERENCE_HANDLE_TOP = 112;
|
||||
export const BATCH_REFERENCE_HANDLE_GAP = 38;
|
||||
export const MAX_BATCH_REFERENCE_COLUMNS = 6;
|
||||
|
||||
export function defaultBatchReferenceColumns(): CanvasBatchReferenceColumn[] {
|
||||
return [
|
||||
{ id: "reference-1", label: "参考图 1" },
|
||||
{ id: "reference-2", label: "参考图 2" },
|
||||
{ id: "reference-3", label: "参考图 3" },
|
||||
];
|
||||
}
|
||||
|
||||
export function batchReferenceColumns(table?: CanvasBatchTableData) {
|
||||
const columns = table?.referenceColumns;
|
||||
if (!columns?.length) return defaultBatchReferenceColumns();
|
||||
// Older tables were created before the second default slot existed. They
|
||||
// never had a remove-column action, so extending this exact legacy shape is
|
||||
// a safe migration and makes reference slot 2 immediately selectable.
|
||||
if (columns.length === 1 && columns[0].id === "reference-1") return defaultBatchReferenceColumns();
|
||||
return columns;
|
||||
}
|
||||
|
||||
export function batchReferenceHandleId(columnId: string) {
|
||||
return `${BATCH_REFERENCE_HANDLE_PREFIX}${columnId}`;
|
||||
}
|
||||
|
||||
export function batchReferenceMentionToken(index: number) {
|
||||
return `@参考图${index + 1}`;
|
||||
}
|
||||
|
||||
export function batchReferenceColumnId(handleId?: string) {
|
||||
return handleId?.startsWith(BATCH_REFERENCE_HANDLE_PREFIX) ? handleId.slice(BATCH_REFERENCE_HANDLE_PREFIX.length) : undefined;
|
||||
}
|
||||
|
||||
export function batchReferenceHandleY(node: CanvasNodeData, handleId?: string) {
|
||||
if (node.type !== "batch-table") return undefined;
|
||||
const columnId = batchReferenceColumnId(handleId);
|
||||
const columns = batchReferenceColumns(node.metadata?.batchTable);
|
||||
const index = columnId ? columns.findIndex((column) => column.id === columnId) : 0;
|
||||
if (index < 0) return undefined;
|
||||
return node.position.y + BATCH_REFERENCE_HANDLE_TOP + index * BATCH_REFERENCE_HANDLE_GAP;
|
||||
}
|
||||
|
||||
export function batchReferenceHandleAtY(node: CanvasNodeData, worldY: number, hitRadius = 18) {
|
||||
if (node.type !== "batch-table") return undefined;
|
||||
const columns = batchReferenceColumns(node.metadata?.batchTable);
|
||||
let nearestIndex = -1;
|
||||
let nearestDistance = Number.POSITIVE_INFINITY;
|
||||
columns.forEach((_, index) => {
|
||||
const distance = Math.abs(worldY - (node.position.y + BATCH_REFERENCE_HANDLE_TOP + index * BATCH_REFERENCE_HANDLE_GAP));
|
||||
if (distance < nearestDistance) {
|
||||
nearestDistance = distance;
|
||||
nearestIndex = index;
|
||||
}
|
||||
});
|
||||
return nearestIndex >= 0 && nearestDistance <= hitRadius ? batchReferenceHandleId(columns[nearestIndex].id) : undefined;
|
||||
}
|
||||
|
||||
export function batchInputColumns(node: CanvasNodeData, connections: CanvasConnection[]) {
|
||||
const columns = batchReferenceColumns(node.metadata?.batchTable);
|
||||
const indexById = new Map(columns.map((column, index) => [column.id, index]));
|
||||
const result = columns.map(() => [] as string[]);
|
||||
connections.filter((connection) => connection.toNodeId === node.id && connection.relation !== "batch-output").forEach((connection) => {
|
||||
const columnId = batchReferenceColumnId(connection.toHandleId);
|
||||
const index = columnId ? indexById.get(columnId) : 0;
|
||||
if (index === undefined || result[index].includes(connection.fromNodeId)) return;
|
||||
result[index].push(connection.fromNodeId);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export function createBatchRow(operation: CanvasBatchOperation, inputNodeIds: string[] = []): CanvasBatchRow {
|
||||
return {
|
||||
id: `batch-row-${nanoid()}`,
|
||||
enabled: true,
|
||||
inputNodeIds,
|
||||
prompt: operation === "try_on" ? TRY_ON_BATCH_PROMPT : CREATIVE_BATCH_PROMPT,
|
||||
};
|
||||
}
|
||||
|
||||
/** 手工追加任务时沿用上一行的参考图,但重新使用当前任务类型的默认提示词。 */
|
||||
export function createInheritedBatchRow(operation: CanvasBatchOperation, rows: CanvasBatchRow[]) {
|
||||
return createBatchRow(operation, [...(rows.at(-1)?.inputNodeIds || [])]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connected try-on inputs follow the reference layout: all model/person images first,
|
||||
* followed by one shared garment image. Users can still edit any row afterwards.
|
||||
*/
|
||||
export function createBatchRowsFromInputs(operation: CanvasBatchOperation, inputNodeIds: string[]) {
|
||||
if (operation === "creative") return inputNodeIds.map((id) => createBatchRow(operation, [id]));
|
||||
if (inputNodeIds.length < 2) return [];
|
||||
const garmentId = inputNodeIds.at(-1)!;
|
||||
return inputNodeIds.slice(0, -1).map((personId) => createBatchRow(operation, [personId, garmentId]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Zip equally sized columns and broadcast singleton columns across every row.
|
||||
*
|
||||
* 同步连线是增量操作:匹配到的行更新参考图,未被本次连线覆盖的手工行继续保留。
|
||||
* 这样用户在表格内追加或编辑的任务不会因为再次点击“同步连线”而被删除。
|
||||
*/
|
||||
export function createBatchRowsFromColumns(operation: CanvasBatchOperation, columns: string[][], previousRows: CanvasBatchRow[] = []) {
|
||||
const rowCount = Math.max(0, ...columns.map((column) => column.length));
|
||||
const unmatchedRows = [...previousRows];
|
||||
const synchronizedRows = Array.from({ length: rowCount }, (_, rowIndex) => {
|
||||
const inputNodeIds = columns.flatMap((column) => {
|
||||
const input = column.length === 1 ? column[0] : column[rowIndex];
|
||||
return input ? [input] : [];
|
||||
});
|
||||
let previousIndex = unmatchedRows.findIndex((row) => sameBatchInputs(row.inputNodeIds, inputNodeIds));
|
||||
if (previousIndex < 0 && inputNodeIds[0]) previousIndex = unmatchedRows.findIndex((row) => row.inputNodeIds[0] === inputNodeIds[0]);
|
||||
const previous = previousIndex >= 0 ? unmatchedRows.splice(previousIndex, 1)[0] : undefined;
|
||||
const inputsUnchanged = previous && previous.inputNodeIds.length === inputNodeIds.length && previous.inputNodeIds.every((id, index) => id === inputNodeIds[index]);
|
||||
return {
|
||||
...createBatchRow(operation, inputNodeIds),
|
||||
...(previous ? { id: previous.id, enabled: previous.enabled, prompt: previous.prompt } : {}),
|
||||
...(inputsUnchanged && previous?.outputNodeId ? { outputNodeId: previous.outputNodeId } : {}),
|
||||
inputNodeIds,
|
||||
};
|
||||
});
|
||||
return [...synchronizedRows, ...unmatchedRows];
|
||||
}
|
||||
|
||||
function sameBatchInputs(left: string[], right: string[]) {
|
||||
return left.length === right.length && left.every((id, index) => id === right[index]);
|
||||
}
|
||||
@@ -16,10 +16,14 @@ export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[],
|
||||
if (acceptedInputKinds.length) {
|
||||
const source = nodes.find((node) => node.id === candidate.fromNodeId);
|
||||
const sourceKind = source ? getNodeInputKind(source.type) : undefined;
|
||||
const hasAcceptedSource = Boolean(sourceKind && acceptedInputKinds.includes(sourceKind));
|
||||
const isMediaConversion = target.type === CanvasNodeType.MediaConversion;
|
||||
const hasAcceptedSource = isMediaConversion
|
||||
? source?.type === CanvasNodeType.Image || source?.type === CanvasNodeType.Video
|
||||
: Boolean(sourceKind && acceptedInputKinds.includes(sourceKind));
|
||||
if (!sourceKind || !hasAcceptedSource) {
|
||||
const labels = acceptedInputKinds.map(acceptedInputKindLabel).join("或");
|
||||
return `${labels}节点只接受${labels}输入`;
|
||||
const targetLabel = isMediaConversion ? "转换" : target.type === CanvasNodeType.BatchTable ? "批量创作表" : labels;
|
||||
return `${targetLabel}节点只接受${labels}输入`;
|
||||
}
|
||||
const maxInputCount = getNodeMaxInputCount(target.type);
|
||||
if (maxInputCount) {
|
||||
@@ -28,7 +32,7 @@ export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[],
|
||||
.filter((connection) => connection.toNodeId === target.id)
|
||||
.map((connection) => connection.fromNodeId),
|
||||
).size;
|
||||
if (inputCount > maxInputCount) return `当前节点最多连接 ${maxInputCount} 个输入`;
|
||||
if (inputCount > maxInputCount) return `${isMediaConversion ? "转换" : "当前"}节点最多连接 ${maxInputCount} 个输入`;
|
||||
}
|
||||
}
|
||||
const mode = getNodeGenerationMode(target);
|
||||
@@ -50,10 +54,11 @@ export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[],
|
||||
return "";
|
||||
}
|
||||
|
||||
function acceptedInputKindLabel(kind: "image" | "video" | "audio" | "text") {
|
||||
function acceptedInputKindLabel(kind: "image" | "video" | "audio" | "text" | "table_data") {
|
||||
if (kind === "image") return "图片";
|
||||
if (kind === "video") return "视频";
|
||||
if (kind === "audio") return "音频";
|
||||
if (kind === "table_data") return "多维表格";
|
||||
return "文本";
|
||||
}
|
||||
|
||||
@@ -69,7 +74,7 @@ export function connectionInputSummary(targetNodeId: string, nodes: CanvasNodeDa
|
||||
if (!inputKind) return;
|
||||
// 角色卡是跨类型覆盖:落在可计数类型上时改记为角色。
|
||||
if (source.metadata?.workflowKind === "character") input.characterCount += 1;
|
||||
else input[`${inputKind}Count`] += 1;
|
||||
else if (inputKind !== "table_data") input[`${inputKind}Count`] += 1;
|
||||
});
|
||||
return input;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import { isFrameNode } from "@/lib/canvas/canvas-frame";
|
||||
import { nodeSizeFromRatio } from "@/lib/canvas/canvas-node-size";
|
||||
import { canvasNodeMentionToken, canvasResourceMentionToken, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
|
||||
import { getNodeDefinition } from "@/lib/canvas/node-registry";
|
||||
import { batchReferenceHandleY } from "@/lib/canvas/canvas-batch-table";
|
||||
import { scopedLocalStorage } from "@/lib/user-scope";
|
||||
import type { GenerationTask } from "@/services/api/task-center";
|
||||
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData, type CanvasNodeMetadata, type CanvasNodeTypeId, type ConnectionHandle, type Position, type StoryboardColumn, type StoryboardRow } from "@/types/canvas";
|
||||
@@ -166,7 +167,7 @@ function resetGenerationParamsOnModelSwitch(node: CanvasNodeData, patch: Partial
|
||||
export function getConnectionTargetAnchor(node: CanvasNodeData, current: ConnectionHandle, handleId?: string, scrollTop = 0) {
|
||||
return {
|
||||
x: current.handleType === "source" ? node.position.x : node.position.x + node.width,
|
||||
y: storyboardHandleY(node, handleId, scrollTop) ?? node.position.y + node.height / 2,
|
||||
y: batchReferenceHandleY(node, handleId) ?? storyboardHandleY(node, handleId, scrollTop) ?? node.position.y + node.height / 2,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -232,6 +232,19 @@ export function buildImageGenerationMetadata(type: CanvasImageGenerationType, co
|
||||
}
|
||||
|
||||
export function nodeReferenceImage(node: CanvasNodeData): ReferenceImage | null {
|
||||
if (node.type === CanvasNodeType.MediaConversion) {
|
||||
// 转换结果只在成功物化后作为生成参考;运行中、跳过和失败状态都不能透传旧结果。
|
||||
const conversion = node.metadata?.mediaConversion;
|
||||
const storageKey = conversion?.resultStorageKey || node.metadata?.storageKey;
|
||||
if (conversion?.status !== "completed" || !storageKey) return null;
|
||||
return {
|
||||
id: node.id,
|
||||
name: node.title || `conversion-${node.id}.png`,
|
||||
type: node.metadata?.mimeType || "image/png",
|
||||
dataUrl: node.metadata?.content || "",
|
||||
storageKey,
|
||||
};
|
||||
}
|
||||
if (node.type !== CanvasNodeType.Image || (!node.metadata?.content && !node.metadata?.storageKey)) return null;
|
||||
return {
|
||||
id: node.id,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video, WandSparkles } from "lucide-react";
|
||||
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Table2, Type, Video, WandSparkles } from "lucide-react";
|
||||
|
||||
import { NODE_SPECS } from "@/constant/canvas";
|
||||
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
|
||||
@@ -154,6 +154,8 @@ const BUILTIN_NODE_TRAITS = {
|
||||
showInCreateMenu: true,
|
||||
inputKind: "text",
|
||||
},
|
||||
[CanvasNodeType.BatchTable]: { label: "批量创作表", icon: <Table2 />, minSize: { width: 720, height: 360 }, showInCreateMenu: true, acceptsInputKind: "image", inputKind: "text", resourceKind: () => null },
|
||||
[CanvasNodeType.MediaConversion]: { label: "转换", icon: <WandSparkles />, minSize: { width: 400, height: 360 }, showInCreateMenu: true, resourceKind: (node: CanvasNodeData) => { const c = node.metadata?.mediaConversion; return c?.status === "completed" && c.resultStorageKey ? (c.outputKind === "video" ? "video" : "image") : null; }, acceptsInputKind: ["image", "video"], maxInputCount: 1, inputKind: "image" },
|
||||
[CanvasNodeType.ColorGrade]: {
|
||||
label: "调色",
|
||||
icon: <Palette />,
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata, CanvasNo
|
||||
import type { PluginCanvasNodeContribution } from "@/lib/plugins/plugin-types";
|
||||
|
||||
/** 作为上游输入被容量校验计数时归入的类别 */
|
||||
export type CanvasNodeInputKind = "image" | "video" | "audio" | "text";
|
||||
export type CanvasNodeInputKind = "image" | "video" | "audio" | "text" | "table_data";
|
||||
|
||||
/**
|
||||
* 节点定义——注册表的基本单元。
|
||||
|
||||
@@ -20,6 +20,8 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
|
||||
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
|
||||
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
|
||||
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
|
||||
nodeCommand(CanvasNodeType.BatchTable, { defaultOrder: 66, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.BatchTable) }),
|
||||
nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }),
|
||||
// 导演台落在节点分区,但它开的是导演工作台、不是某种画布节点,故不走注册表。
|
||||
{ id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() },
|
||||
nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }),
|
||||
|
||||
89
web/src/lib/media-conversion/contracts.ts
Normal file
89
web/src/lib/media-conversion/contracts.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import type { CanvasNodeData } from "@/types/canvas";
|
||||
|
||||
export const MEDIA_CONVERSION_SCHEMA_VERSION = 1 as const;
|
||||
|
||||
export const MEDIA_CONVERSION_NODE_TYPE = "media-conversion" as const;
|
||||
|
||||
export type MediaConversionOperation = "grayscale" | "edge-canny" | "lineart" | "depth" | "pose" | "cutout";
|
||||
export type MediaConversionStatus = "idle" | "processing" | "completed" | "stale" | "skipped" | "unavailable" | "error";
|
||||
export type MediaConversionOutputKind = "image" | "video";
|
||||
|
||||
export type MediaConversionNodeState = {
|
||||
schemaVersion: typeof MEDIA_CONVERSION_SCHEMA_VERSION;
|
||||
operation: MediaConversionOperation;
|
||||
status: MediaConversionStatus;
|
||||
sourceNodeId?: string;
|
||||
sourceFingerprint?: string;
|
||||
outputKind?: MediaConversionOutputKind;
|
||||
resultStorageKey?: string;
|
||||
resultWidth?: number;
|
||||
resultHeight?: number;
|
||||
detectedPeople?: number;
|
||||
errorCode?: string;
|
||||
errorMessage?: string;
|
||||
startedAt?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export const MEDIA_CONVERSION_OPERATION_LABELS: Record<MediaConversionOperation, string> = {
|
||||
grayscale: "灰度图",
|
||||
"edge-canny": "Canny 边缘",
|
||||
lineart: "AI 线稿",
|
||||
depth: "深度图",
|
||||
pose: "姿态骨架",
|
||||
cutout: "透明抠图",
|
||||
};
|
||||
|
||||
export const MEDIA_CONVERSION_OPERATION_DESCRIPTIONS: Record<MediaConversionOperation, string> = {
|
||||
grayscale: "普通图像算法,适合先检查明暗层次",
|
||||
"edge-canny": "普通图像算法,提取清晰轮廓",
|
||||
lineart: "本地 ControlNet Aux 线稿预处理,首次运行需要加载模型",
|
||||
depth: "本地 Depth Anything V2 Small,首次运行需要加载模型",
|
||||
pose: "本地 OpenPose 人体骨架预处理,未检测到人物时会跳过",
|
||||
cutout: "需要安装并验证本地抠图模型",
|
||||
};
|
||||
|
||||
export function createDefaultMediaConversionState(): MediaConversionNodeState {
|
||||
return {
|
||||
schemaVersion: MEDIA_CONVERSION_SCHEMA_VERSION,
|
||||
operation: "edge-canny",
|
||||
status: "idle",
|
||||
};
|
||||
}
|
||||
|
||||
export function mediaConversionSourceFingerprint(node: CanvasNodeData) {
|
||||
const metadata = node.metadata;
|
||||
const source = [
|
||||
node.id,
|
||||
metadata?.storageKey || "",
|
||||
compactSource(metadata?.content || metadata?.previewContent || ""),
|
||||
metadata?.mimeType || "",
|
||||
metadata?.bytes || "",
|
||||
metadata?.naturalWidth || "",
|
||||
metadata?.naturalHeight || "",
|
||||
metadata?.durationMs || "",
|
||||
].join("|");
|
||||
let hash = 2166136261;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
hash ^= source.charCodeAt(index);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return `v1-${(hash >>> 0).toString(16).padStart(8, "0")}`;
|
||||
}
|
||||
|
||||
export function mediaConversionOperationLabel(operation: MediaConversionOperation) {
|
||||
return MEDIA_CONVERSION_OPERATION_LABELS[operation];
|
||||
}
|
||||
|
||||
export function mediaConversionOperationDescription(operation: MediaConversionOperation) {
|
||||
return MEDIA_CONVERSION_OPERATION_DESCRIPTIONS[operation];
|
||||
}
|
||||
|
||||
export function isLocalImageOperation(operation: MediaConversionOperation) {
|
||||
return operation === "grayscale" || operation === "edge-canny";
|
||||
}
|
||||
|
||||
function compactSource(value: string) {
|
||||
if (value.length <= 2048) return value;
|
||||
return `${value.slice(0, 1024)}|${value.slice(-1024)}|${value.length}`;
|
||||
}
|
||||
228
web/src/lib/media-conversion/local-converter.ts
Normal file
228
web/src/lib/media-conversion/local-converter.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import type { MediaConversionOperation } from "./contracts";
|
||||
|
||||
export type LocalImageConversionResult = {
|
||||
blob: Blob;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
export class LocalImageConversionError extends Error {
|
||||
constructor(readonly code: "model_missing" | "source_unreadable" | "browser_unsupported" | "aborted", message: string) {
|
||||
super(message);
|
||||
this.name = "LocalImageConversionError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function convertImageLocally(sourceUrl: string, operation: MediaConversionOperation, options: { signal?: AbortSignal; maxDimension?: number } = {}): Promise<LocalImageConversionResult> {
|
||||
throwIfAborted(options.signal);
|
||||
if (operation !== "grayscale" && operation !== "edge-canny") {
|
||||
throw new LocalImageConversionError("model_missing", "该转换需要先安装并验证本地模型");
|
||||
}
|
||||
|
||||
const image = await loadImage(sourceUrl, options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
const maxDimension = Math.max(256, options.maxDimension || 1024);
|
||||
const scale = Math.min(1, maxDimension / Math.max(image.naturalWidth, image.naturalHeight));
|
||||
const width = Math.max(1, Math.round(image.naturalWidth * scale));
|
||||
const height = Math.max(1, Math.round(image.naturalHeight * scale));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const context = canvas.getContext("2d", { willReadFrequently: true });
|
||||
if (!context) throw new LocalImageConversionError("browser_unsupported", "当前浏览器无法创建图片处理画布");
|
||||
context.drawImage(image, 0, 0, width, height);
|
||||
const input = context.getImageData(0, 0, width, height);
|
||||
const output = transformPixels(input.data, width, height, operation, options.signal);
|
||||
const outputImage = new ImageData(output, width, height);
|
||||
context.putImageData(outputImage, 0, 0);
|
||||
const blob = await canvasToBlob(canvas, options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
return { blob, width, height };
|
||||
}
|
||||
|
||||
/** Pure pixel transform used by the browser executor and unit tests. */
|
||||
export function transformPixels(source: Uint8ClampedArray, width: number, height: number, operation: "grayscale" | "edge-canny", signal?: AbortSignal) {
|
||||
if (source.length !== width * height * 4) throw new Error("图片像素尺寸不匹配");
|
||||
if (operation === "grayscale") return grayscalePixels(source, width, height, signal);
|
||||
return cannyPixels(source, width, height, signal);
|
||||
}
|
||||
|
||||
function grayscalePixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
|
||||
const output = new Uint8ClampedArray(source);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
throwIfAborted(signal);
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const index = (y * width + x) * 4;
|
||||
const value = Math.round(source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114);
|
||||
output[index] = value;
|
||||
output[index + 1] = value;
|
||||
output[index + 2] = value;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function cannyPixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
|
||||
const gray = new Float32Array(width * height);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
throwIfAborted(signal);
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const index = (y * width + x) * 4;
|
||||
gray[y * width + x] = source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114;
|
||||
}
|
||||
}
|
||||
|
||||
const blurred = gaussianBlur(gray, width, height, signal);
|
||||
const magnitude = new Float32Array(width * height);
|
||||
const direction = new Float32Array(width * height);
|
||||
let maximum = 0;
|
||||
for (let y = 1; y < height - 1; y += 1) {
|
||||
throwIfAborted(signal);
|
||||
for (let x = 1; x < width - 1; x += 1) {
|
||||
const index = y * width + x;
|
||||
const horizontal = blurred[index + 1] - blurred[index - 1];
|
||||
const vertical = blurred[index + width] - blurred[index - width];
|
||||
const value = Math.hypot(horizontal, vertical);
|
||||
magnitude[index] = value;
|
||||
direction[index] = Math.atan2(vertical, horizontal);
|
||||
maximum = Math.max(maximum, value);
|
||||
}
|
||||
}
|
||||
|
||||
const thinned = nonMaximumSuppression(magnitude, direction, width, height, signal);
|
||||
const high = maximum * 0.28;
|
||||
const low = high * 0.45;
|
||||
const edges = new Uint8Array(width * height);
|
||||
const stack: number[] = [];
|
||||
for (let index = 0; index < thinned.length; index += 1) {
|
||||
if (thinned[index] >= high && high > 0) {
|
||||
edges[index] = 2;
|
||||
stack.push(index);
|
||||
} else if (thinned[index] >= low && low > 0) {
|
||||
edges[index] = 1;
|
||||
}
|
||||
}
|
||||
while (stack.length) {
|
||||
throwIfAborted(signal);
|
||||
const index = stack.pop()!;
|
||||
const x = index % width;
|
||||
const y = Math.floor(index / width);
|
||||
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
|
||||
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
|
||||
if (!offsetX && !offsetY) continue;
|
||||
const nextX = x + offsetX;
|
||||
const nextY = y + offsetY;
|
||||
if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) continue;
|
||||
const next = nextY * width + nextX;
|
||||
if (edges[next] !== 1) continue;
|
||||
edges[next] = 2;
|
||||
stack.push(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const output = new Uint8ClampedArray(source.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
throwIfAborted(signal);
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const index = y * width + x;
|
||||
const outputIndex = index * 4;
|
||||
const value = edges[index] === 2 ? 0 : 255;
|
||||
output[outputIndex] = value;
|
||||
output[outputIndex + 1] = value;
|
||||
output[outputIndex + 2] = value;
|
||||
output[outputIndex + 3] = source[outputIndex + 3];
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function gaussianBlur(source: Float32Array, width: number, height: number, signal?: AbortSignal) {
|
||||
const output = new Float32Array(source.length);
|
||||
const kernel = [1, 2, 1, 2, 4, 2, 1, 2, 1];
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
throwIfAborted(signal);
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let value = 0;
|
||||
let weight = 0;
|
||||
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
|
||||
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
|
||||
const nextX = Math.min(width - 1, Math.max(0, x + offsetX));
|
||||
const nextY = Math.min(height - 1, Math.max(0, y + offsetY));
|
||||
const kernelWeight = kernel[(offsetY + 1) * 3 + offsetX + 1];
|
||||
value += source[nextY * width + nextX] * kernelWeight;
|
||||
weight += kernelWeight;
|
||||
}
|
||||
}
|
||||
output[y * width + x] = value / weight;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function nonMaximumSuppression(magnitude: Float32Array, direction: Float32Array, width: number, height: number, signal?: AbortSignal) {
|
||||
const output = new Float32Array(magnitude.length);
|
||||
for (let y = 1; y < height - 1; y += 1) {
|
||||
throwIfAborted(signal);
|
||||
for (let x = 1; x < width - 1; x += 1) {
|
||||
const index = y * width + x;
|
||||
let angle = (direction[index] * 180) / Math.PI;
|
||||
if (angle < 0) angle += 180;
|
||||
const value = magnitude[index];
|
||||
let before = 0;
|
||||
let after = 0;
|
||||
if (angle < 22.5 || angle >= 157.5) {
|
||||
before = magnitude[index - 1];
|
||||
after = magnitude[index + 1];
|
||||
} else if (angle < 67.5) {
|
||||
before = magnitude[index - width + 1];
|
||||
after = magnitude[index + width - 1];
|
||||
} else if (angle < 112.5) {
|
||||
before = magnitude[index - width];
|
||||
after = magnitude[index + width];
|
||||
} else {
|
||||
before = magnitude[index - width - 1];
|
||||
after = magnitude[index + width + 1];
|
||||
}
|
||||
output[index] = value >= before && value >= after ? value : 0;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
async function loadImage(sourceUrl: string, signal?: AbortSignal) {
|
||||
if (!sourceUrl) throw new LocalImageConversionError("source_unreadable", "没有可读取的图片输入");
|
||||
const image = new Image();
|
||||
image.crossOrigin = "anonymous";
|
||||
return await new Promise<HTMLImageElement>((resolve, reject) => {
|
||||
const abort = () => {
|
||||
image.src = "";
|
||||
reject(new LocalImageConversionError("aborted", "转换已取消"));
|
||||
};
|
||||
if (signal?.aborted) return abort();
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
image.onload = () => {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
resolve(image);
|
||||
};
|
||||
image.onerror = () => {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
reject(new LocalImageConversionError("source_unreadable", "图片无法读取,可能是跨域资源或已失效"));
|
||||
};
|
||||
image.src = sourceUrl;
|
||||
});
|
||||
}
|
||||
|
||||
function canvasToBlob(canvas: HTMLCanvasElement, signal?: AbortSignal) {
|
||||
throwIfAborted(signal);
|
||||
return new Promise<Blob>((resolve, reject) => {
|
||||
canvas.toBlob((blob) => {
|
||||
if (blob) resolve(blob);
|
||||
else reject(new LocalImageConversionError("browser_unsupported", "当前浏览器无法导出图片结果"));
|
||||
}, "image/png");
|
||||
});
|
||||
}
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): asserts signal is AbortSignal | undefined {
|
||||
if (signal?.aborted) throw new LocalImageConversionError("aborted", "转换已取消");
|
||||
}
|
||||
@@ -2,4 +2,5 @@ import "./eagle";
|
||||
import "./prompt-optimizer";
|
||||
import "./workflows";
|
||||
import "./ai-art-critique";
|
||||
import "./media-conversion";
|
||||
import "./editor/editor-shell";
|
||||
|
||||
37
web/src/lib/plugins/builtin/media-conversion.ts
Normal file
37
web/src/lib/plugins/builtin/media-conversion.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
|
||||
|
||||
import { registerPlugin } from "../plugin-registry";
|
||||
import { PLUGIN_API_VERSION, type PluginManifest, type RegisteredPlugin } from "../plugin-types";
|
||||
|
||||
export const MEDIA_CONVERSION_PLUGIN_ID = "media-conversion";
|
||||
|
||||
const manifest: PluginManifest = {
|
||||
apiVersion: PLUGIN_API_VERSION,
|
||||
id: MEDIA_CONVERSION_PLUGIN_ID,
|
||||
name: "媒体转换节点",
|
||||
version: "0.1.0",
|
||||
publishedAt: "2026-09-08",
|
||||
updatedAt: "2026-09-08",
|
||||
description: "在画布中提供图片转换节点,支持灰度、Canny 边缘、AI 线稿、本地 Depth Anything V2 深度图和 OpenPose 姿态骨架。",
|
||||
documentation:
|
||||
"# 媒体转换节点\n\n连接图片到转换节点后,可以在节点内选择灰度图、Canny 边缘、AI 线稿、深度图或姿态骨架。AI 线稿使用本机 ControlNet Aux 预处理模型,深度图使用本机已安装的 Depth Anything V2 Small 模型,姿态骨架使用 ControlNet Aux 的 OpenPose 人体预处理模型运行;这些操作都不会加载 Stable Diffusion 重绘管线。姿态转换只绘制人体骨架,未检测到人物时会跳过并提示。视频转换和透明抠图模型仍在验证,暂时会明确提示不可用。\n\n该插件只读取当前节点的媒体输入,并把结果保存回当前画布节点的本地素材存储。",
|
||||
author: "影策团队",
|
||||
surfaces: ["node"],
|
||||
permissions: ["canvas.read", "canvas.write", "media.read"],
|
||||
trusted: true,
|
||||
runtime: { web: "declarative" },
|
||||
contributes: {
|
||||
transforms: [
|
||||
{
|
||||
id: MEDIA_CONVERSION_NODE_TYPE,
|
||||
input: "media",
|
||||
output: "media",
|
||||
runtime: "declarative",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export const mediaConversionPlugin: RegisteredPlugin = { manifest };
|
||||
|
||||
registerPlugin(mediaConversionPlugin);
|
||||
Reference in New Issue
Block a user