feat(canvas): 图片转换插件 - 增加本地媒体转换节点与运行时

canvas-agent/src/http-server.ts

@@ -4,7 +4,7 @@ import { CanvasSession } from "./canvas-session.js";
 import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js";
 import { createLocalRuntimeApp } from "./local-runtime.js";
 import { startLocalRuntime } from "./local-runtime-host.js";
-import { LocalRuntimeSessionManager } from "./local-runtime-session.js";
+import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./local-runtime-session.js";
 import {
     createCanvasAgentHttpModule,
     type CanvasAgentSession,
@@ -14,6 +14,9 @@ import {
     type DreaminaHttpModuleOptions,
 } from "./modules/dreamina-http.js";
 import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
+import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
+import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
+import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
 
 export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">;
 
@@ -34,21 +37,29 @@ export function createHttpApp(
     dependencies: CanvasAgentHttpDependencies = {},
 ): Express {
     const endpoint = config.url;
+    const modules = [
+        createCanvasAgentHttpModule(config, session),
+        createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
+        createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
+        createDepthEstimationHttpModule(),
+        createLineartEstimationHttpModule(),
+        createPoseEstimationHttpModule(),
+    ];
     const manager = new LocalRuntimeSessionManager({
         endpoint,
         trustedOrigins: config.trustedWebOrigins,
         registrations: config.browserRegistrations,
+        scopes: [...new Set([
+            ...LOCAL_RUNTIME_DEFAULT_SCOPES,
+            ...modules.flatMap((module) => module.descriptor.scopes),
+        ])],
     });
     return createLocalRuntimeApp({
         authority: new URL(endpoint).host,
         endpoint,
         version: "0.1.0",
         sessionManager: manager,
-        modules: [
-            createCanvasAgentHttpModule(config, session),
-            createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
-            createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
-        ],
+        modules,
         legacyMasterToken: config.token,
         legacyOrigins: config.origins ?? [],
     });

canvas-agent/src/local-runtime-contract.ts

@@ -6,7 +6,7 @@ export const LOCAL_RUNTIME_ENDPOINT = "http://127.0.0.1:17371";
 export const LOCAL_RUNTIME_ID = "framefield-local-runtime";
 export const LOCAL_RUNTIME_API_VERSION = 2;
 
-export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance";
+export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
 export type LocalRuntimeScope =
     | "runtime:status"
     | "runtime:revoke"
@@ -20,7 +20,13 @@ export type LocalRuntimeScope =
     | "portrait:status"
     | "portrait:model"
     | "portrait:run"
-    | "portrait:read";
+    | "portrait:read"
+    | "depth:status"
+    | "depth:run"
+    | "lineart:status"
+    | "lineart:run"
+    | "pose:status"
+    | "pose:run";
 
 export type DreaminaModelOperation =
     | "text-to-image"

canvas-agent/src/local-runtime-host.ts

@@ -17,6 +17,9 @@ import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./loca
 import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js";
 import { createDreaminaHttpModule } from "./modules/dreamina-http.js";
 import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
+import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
+import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
+import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
 
 export type StartLocalRuntimeOptions = {
     config?: LocalRuntimeConfig;
@@ -46,6 +49,9 @@ export function createDefaultLocalRuntimeModules(config: LocalRuntimeConfig): Lo
             ],
         }),
         createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }),
+        createDepthEstimationHttpModule(),
+        createLineartEstimationHttpModule(),
+        createPoseEstimationHttpModule(),
     ];
 }
 

canvas-agent/src/local-runtime.ts

@@ -185,7 +185,7 @@ function validateModules(modules: readonly LocalRuntimeModule[]) {
     const ids = new Set<LocalRuntimeModuleId>();
     const routes = new Set<string>();
     for (const module of modules) {
-        if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance") {
+        if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance" && module.descriptor.id !== "depth-estimation" && module.descriptor.id !== "lineart-estimation" && module.descriptor.id !== "pose-estimation") {
             throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`);
         }
         if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`);

canvas-agent/src/modules/depth-estimation-http.ts

@@ -0,0 +1,327 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+
+import type { Request, RequestHandler, Response } from "express";
+
+import type { LocalRuntimeModule } from "../local-runtime.js";
+
+const DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf";
+const MAX_INPUT_BYTES = 12 * 1024 * 1024;
+const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
+const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
+
+export type DepthRuntimePaths = {
+    pythonPath: string;
+    scriptPath: string;
+    hfHome: string;
+    modelId: string;
+    configured: boolean;
+    modelCached: boolean;
+    reason?: "python_missing" | "worker_missing";
+};
+
+export type DepthRuntimeStatus = {
+    configured: boolean;
+    modelCached: boolean;
+    loaded: boolean;
+    device: "cpu";
+    modelId: string;
+    reason?: string;
+};
+
+type DepthWorkerSuccess = {
+    ok: true;
+    requestId: string;
+    mimeType: "image/png";
+    width: number;
+    height: number;
+    modelId: string;
+    device: "cpu";
+    pngBase64: string;
+};
+
+export class DepthRuntimeError extends Error {
+    constructor(readonly code: string, message: string, readonly statusCode = 503) {
+        super(message);
+        this.name = "DepthRuntimeError";
+    }
+}
+
+export function createDepthEstimationHttpModule(): LocalRuntimeModule {
+    const worker = new DepthEstimationWorker();
+    return {
+        descriptor: {
+            id: "depth-estimation",
+            displayName: "Depth Anything V2 本机深度",
+            apiVersion: 1,
+            scopes: ["depth:status", "depth:run"],
+        },
+        routes: [
+            {
+                method: "GET",
+                path: "/depth-estimation/status",
+                scope: "depth:status",
+                handler: asyncRoute(async (_request, response) => {
+                    const status = worker.status();
+                    response.json({
+                        ok: true,
+                        module: "depth-estimation",
+                        apiVersion: 1,
+                        ready: status.configured && status.modelCached,
+                        ...status,
+                    });
+                }),
+            },
+            {
+                method: "POST",
+                path: "/depth-estimation/run",
+                scope: "depth:run",
+                handler: asyncRoute(async (request, response) => {
+                    const body = parseBody(request);
+                    const dataUrl = parseDepthInput(body);
+                    const result = await worker.run({ dataUrl });
+                    response.json({
+                        ok: true,
+                        module: "depth-estimation",
+                        apiVersion: 1,
+                        result,
+                    });
+                }),
+            },
+        ],
+        dispose: () => worker.dispose(),
+        publicHealth: () => {
+            const status = worker.status();
+            return { depthEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
+        },
+    };
+}
+
+export function resolveDepthRuntimePaths(env: NodeJS.ProcessEnv = process.env): DepthRuntimePaths {
+    const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+    const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
+    const projectRoots = [
+        configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
+        path.resolve(moduleRoot, ".."),
+        path.resolve(process.cwd()),
+    ].filter((value): value is string => Boolean(value));
+    const projectRoot = configuredProjectRoot
+        ? path.resolve(configuredProjectRoot)
+        : projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
+    const scriptCandidates = [
+        path.join(moduleRoot, "python", "depth_anything_runtime.py"),
+        path.join(projectRoot, "canvas-agent", "python", "depth_anything_runtime.py"),
+    ];
+    const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
+    const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
+        ? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
+        : process.platform === "win32"
+            ? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
+            : [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
+    const pythonPath = pythonCandidates[0]!;
+    const hfHome = path.resolve(env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
+    const modelId = env.CANVAS_DEPTH_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
+    const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
+    const workerExists = fs.existsSync(scriptPath);
+    const pythonExists = fs.existsSync(pythonPath);
+    return {
+        pythonPath,
+        scriptPath,
+        hfHome,
+        modelId,
+        configured: workerExists && pythonExists,
+        modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
+        ...(pythonExists ? {} : { reason: "python_missing" as const }),
+        ...(!workerExists ? { reason: "worker_missing" as const } : {}),
+    };
+}
+
+export class DepthEstimationWorker {
+    private paths = resolveDepthRuntimePaths();
+    private child?: ChildProcessWithoutNullStreams;
+    private stdoutBuffer = "";
+    private stderrBuffer = "";
+    private requestCounter = 0;
+    private loaded = false;
+    private readonly pending = new Map<string, { resolve: (value: DepthWorkerSuccess) => void; reject: (error: unknown) => void }>();
+
+    status(): DepthRuntimeStatus {
+        const paths = this.refreshPaths();
+        return {
+            configured: paths.configured,
+            modelCached: paths.modelCached,
+            loaded: this.loaded,
+            device: "cpu",
+            modelId: paths.modelId,
+            ...(paths.reason ? { reason: paths.reason } : {}),
+        };
+    }
+
+    async run(input: { dataUrl: string }) {
+        validateDepthDataUrl(input.dataUrl);
+        const paths = this.refreshPaths();
+        if (!paths.configured) {
+            throw new DepthRuntimeError(
+                "depth_runtime_unavailable",
+                paths.reason === "worker_missing" ? "本机深度运行文件不存在" : "未找到本机深度 Python 环境",
+            );
+        }
+        const child = this.ensureChild();
+        const requestId = `depth-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
+        const response = new Promise<DepthWorkerSuccess>((resolve, reject) => {
+            this.pending.set(requestId, { resolve, reject });
+        });
+        try {
+            child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
+        } catch (error) {
+            this.pending.delete(requestId);
+            throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法写入");
+        }
+        return response;
+    }
+
+    dispose() {
+        const child = this.child;
+        this.child = undefined;
+        this.loaded = false;
+        this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已停止"));
+        if (child && !child.killed) child.kill();
+    }
+
+    private ensureChild() {
+        if (this.child && !this.child.killed) return this.child;
+        let child: ChildProcessWithoutNullStreams;
+        try {
+            child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
+                cwd: path.dirname(this.paths.scriptPath),
+                env: {
+                    ...process.env,
+                    HF_HOME: this.paths.hfHome,
+                    HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
+                    HF_HUB_OFFLINE: "1",
+                    TRANSFORMERS_OFFLINE: "1",
+                    HF_HUB_DISABLE_TELEMETRY: "1",
+                    HF_HUB_DISABLE_PROGRESS_BARS: "1",
+                    TRANSFORMERS_VERBOSITY: "error",
+                    CANVAS_DEPTH_MODEL_ID: this.paths.modelId,
+                },
+                stdio: ["pipe", "pipe", "pipe"],
+                windowsHide: true,
+            });
+        } catch (error) {
+            throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法启动");
+        }
+        this.child = child;
+        child.stdout.setEncoding("utf8");
+        child.stderr.setEncoding("utf8");
+        child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
+        child.stderr.on("data", (chunk: string) => {
+            this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
+        });
+        child.on("error", () => this.handleChildExit(child));
+        child.on("exit", () => this.handleChildExit(child));
+        return child;
+    }
+
+    private refreshPaths() {
+        if (!this.child) this.paths = resolveDepthRuntimePaths();
+        return this.paths;
+    }
+
+    private consumeStdout(chunk: string) {
+        this.stdoutBuffer += chunk;
+        if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
+            this.child?.kill();
+            this.handleChildExit(this.child);
+            return;
+        }
+        while (true) {
+            const newline = this.stdoutBuffer.indexOf("\n");
+            if (newline < 0) return;
+            const line = this.stdoutBuffer.slice(0, newline).trim();
+            this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
+            if (!line) continue;
+            let value: unknown;
+            try { value = JSON.parse(line); } catch { continue; }
+            if (!isRecord(value) || typeof value.requestId !== "string") continue;
+            const pending = this.pending.get(value.requestId);
+            if (!pending) continue;
+            this.pending.delete(value.requestId);
+            if (value.ok === true) {
+                if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
+                    pending.reject(new DepthRuntimeError("depth_response_invalid", "本机深度响应无效", 500));
+                    continue;
+                }
+                this.loaded = true;
+                pending.resolve(value as unknown as DepthWorkerSuccess);
+                continue;
+            }
+            const code = typeof value.code === "string" ? value.code : "depth_inference_failed";
+            const statusCode = code === "depth_model_missing" ? 503 : code === "depth_input_invalid" ? 400 : 500;
+            pending.reject(new DepthRuntimeError(code, typeof value.message === "string" ? value.message : "本机深度推理失败", statusCode));
+        }
+    }
+
+    private handleChildExit(child?: ChildProcessWithoutNullStreams) {
+        if (!child || this.child !== child) return;
+        this.child = undefined;
+        this.loaded = false;
+        this.stderrBuffer = "";
+        this.stdoutBuffer = "";
+        this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已退出"));
+    }
+
+    private rejectPending(error: DepthRuntimeError) {
+        for (const { reject } of this.pending.values()) reject(error);
+        this.pending.clear();
+    }
+}
+
+export function validateDepthDataUrl(value: string) {
+    const match = DATA_URL_PATTERN.exec(value);
+    if (!match) throw new DepthRuntimeError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
+    const bytes = Buffer.from(match[2]!, "base64");
+    if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new DepthRuntimeError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
+}
+
+function parseDepthInput(body: Record<string, unknown>) {
+    const keys = Object.keys(body).sort();
+    if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "depth" || typeof body.dataUrl !== "string") {
+        throw new DepthRuntimeError("depth_input_invalid", "深度转换请求字段无效", 400);
+    }
+    validateDepthDataUrl(body.dataUrl);
+    return body.dataUrl;
+}
+
+function parseBody(request: Request): Record<string, unknown> {
+    if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
+    try {
+        const value = JSON.parse(request.body.toString("utf8")) as unknown;
+        return isRecord(value) ? value : {};
+    } catch {
+        return {};
+    }
+}
+
+function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
+    return (request, response, next) => {
+        void action(request, response).catch((error) => {
+            if (response.headersSent) return next(error);
+            if (error instanceof DepthRuntimeError) {
+                response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
+                return;
+            }
+            response.status(503).json({ ok: false, code: "depth_runtime_unavailable", message: "本机深度运行时不可用" });
+        });
+    };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+    return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+
+function isPositiveInteger(value: unknown): value is number {
+    return typeof value === "number" && Number.isInteger(value) && value > 0;
+}

canvas-agent/src/modules/lineart-estimation-http.ts

@@ -0,0 +1,327 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+
+import type { Request, RequestHandler, Response } from "express";
+
+import type { LocalRuntimeModule } from "../local-runtime.js";
+
+const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
+const MAX_INPUT_BYTES = 12 * 1024 * 1024;
+const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
+const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
+
+export type LineartRuntimePaths = {
+    pythonPath: string;
+    scriptPath: string;
+    hfHome: string;
+    modelId: string;
+    configured: boolean;
+    modelCached: boolean;
+    reason?: "python_missing" | "worker_missing";
+};
+
+export type LineartRuntimeStatus = {
+    configured: boolean;
+    modelCached: boolean;
+    loaded: boolean;
+    device: "cpu";
+    modelId: string;
+    reason?: string;
+};
+
+type LineartWorkerSuccess = {
+    ok: true;
+    requestId: string;
+    mimeType: "image/png";
+    width: number;
+    height: number;
+    modelId: string;
+    device: "cpu";
+    pngBase64: string;
+};
+
+export class LineartRuntimeError extends Error {
+    constructor(readonly code: string, message: string, readonly statusCode = 503) {
+        super(message);
+        this.name = "LineartRuntimeError";
+    }
+}
+
+export function createLineartEstimationHttpModule(): LocalRuntimeModule {
+    const worker = new LineartEstimationWorker();
+    return {
+        descriptor: {
+            id: "lineart-estimation",
+            displayName: "ControlNet Aux 本机 AI 线稿",
+            apiVersion: 1,
+            scopes: ["lineart:status", "lineart:run"],
+        },
+        routes: [
+            {
+                method: "GET",
+                path: "/lineart-estimation/status",
+                scope: "lineart:status",
+                handler: asyncRoute(async (_request, response) => {
+                    const status = worker.status();
+                    response.json({
+                        ok: true,
+                        module: "lineart-estimation",
+                        apiVersion: 1,
+                        ready: status.configured && status.modelCached,
+                        ...status,
+                    });
+                }),
+            },
+            {
+                method: "POST",
+                path: "/lineart-estimation/run",
+                scope: "lineart:run",
+                handler: asyncRoute(async (request, response) => {
+                    const body = parseBody(request);
+                    const dataUrl = parseLineartInput(body);
+                    const result = await worker.run({ dataUrl });
+                    response.json({
+                        ok: true,
+                        module: "lineart-estimation",
+                        apiVersion: 1,
+                        result,
+                    });
+                }),
+            },
+        ],
+        dispose: () => worker.dispose(),
+        publicHealth: () => {
+            const status = worker.status();
+            return { lineartEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
+        },
+    };
+}
+
+export function resolveLineartRuntimePaths(env: NodeJS.ProcessEnv = process.env): LineartRuntimePaths {
+    const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+    const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
+    const projectRoots = [
+        configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
+        path.resolve(moduleRoot, ".."),
+        path.resolve(process.cwd()),
+    ].filter((value): value is string => Boolean(value));
+    const projectRoot = configuredProjectRoot
+        ? path.resolve(configuredProjectRoot)
+        : projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
+    const scriptCandidates = [
+        path.join(moduleRoot, "python", "lineart_runtime.py"),
+        path.join(projectRoot, "canvas-agent", "python", "lineart_runtime.py"),
+    ];
+    const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
+    const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
+        ? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
+        : process.platform === "win32"
+            ? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
+            : [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
+    const pythonPath = pythonCandidates[0]!;
+    const hfHome = path.resolve(env.CANVAS_LINEART_HF_HOME?.trim() || env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
+    const modelId = env.CANVAS_LINEART_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
+    const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
+    const workerExists = fs.existsSync(scriptPath);
+    const pythonExists = fs.existsSync(pythonPath);
+    return {
+        pythonPath,
+        scriptPath,
+        hfHome,
+        modelId,
+        configured: workerExists && pythonExists,
+        modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
+        ...(pythonExists ? {} : { reason: "python_missing" as const }),
+        ...(!workerExists ? { reason: "worker_missing" as const } : {}),
+    };
+}
+
+export class LineartEstimationWorker {
+    private paths = resolveLineartRuntimePaths();
+    private child?: ChildProcessWithoutNullStreams;
+    private stdoutBuffer = "";
+    private stderrBuffer = "";
+    private requestCounter = 0;
+    private loaded = false;
+    private readonly pending = new Map<string, { resolve: (value: LineartWorkerSuccess) => void; reject: (error: unknown) => void }>();
+
+    status(): LineartRuntimeStatus {
+        const paths = this.refreshPaths();
+        return {
+            configured: paths.configured,
+            modelCached: paths.modelCached,
+            loaded: this.loaded,
+            device: "cpu",
+            modelId: paths.modelId,
+            ...(paths.reason ? { reason: paths.reason } : {}),
+        };
+    }
+
+    async run(input: { dataUrl: string }) {
+        validateLineartDataUrl(input.dataUrl);
+        const paths = this.refreshPaths();
+        if (!paths.configured) {
+            throw new LineartRuntimeError(
+                "lineart_runtime_unavailable",
+                paths.reason === "worker_missing" ? "本机 AI 线稿运行文件不存在" : "未找到本机 AI 线稿 Python 环境",
+            );
+        }
+        const child = this.ensureChild();
+        const requestId = `lineart-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
+        const response = new Promise<LineartWorkerSuccess>((resolve, reject) => {
+            this.pending.set(requestId, { resolve, reject });
+        });
+        try {
+            child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
+        } catch {
+            this.pending.delete(requestId);
+            throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法写入");
+        }
+        return response;
+    }
+
+    dispose() {
+        const child = this.child;
+        this.child = undefined;
+        this.loaded = false;
+        this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已停止"));
+        if (child && !child.killed) child.kill();
+    }
+
+    private ensureChild() {
+        if (this.child && !this.child.killed) return this.child;
+        let child: ChildProcessWithoutNullStreams;
+        try {
+            child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
+                cwd: path.dirname(this.paths.scriptPath),
+                env: {
+                    ...process.env,
+                    HF_HOME: this.paths.hfHome,
+                    HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
+                    HF_HUB_OFFLINE: "1",
+                    TRANSFORMERS_OFFLINE: "1",
+                    HF_HUB_DISABLE_TELEMETRY: "1",
+                    HF_HUB_DISABLE_PROGRESS_BARS: "1",
+                    TRANSFORMERS_VERBOSITY: "error",
+                    CANVAS_LINEART_MODEL_ID: this.paths.modelId,
+                },
+                stdio: ["pipe", "pipe", "pipe"],
+                windowsHide: true,
+            });
+        } catch {
+            throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法启动");
+        }
+        this.child = child;
+        child.stdout.setEncoding("utf8");
+        child.stderr.setEncoding("utf8");
+        child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
+        child.stderr.on("data", (chunk: string) => {
+            this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
+        });
+        child.on("error", () => this.handleChildExit(child));
+        child.on("exit", () => this.handleChildExit(child));
+        return child;
+    }
+
+    private refreshPaths() {
+        if (!this.child) this.paths = resolveLineartRuntimePaths();
+        return this.paths;
+    }
+
+    private consumeStdout(chunk: string) {
+        this.stdoutBuffer += chunk;
+        if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
+            this.child?.kill();
+            this.handleChildExit(this.child);
+            return;
+        }
+        while (true) {
+            const newline = this.stdoutBuffer.indexOf("\n");
+            if (newline < 0) return;
+            const line = this.stdoutBuffer.slice(0, newline).trim();
+            this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
+            if (!line) continue;
+            let value: unknown;
+            try { value = JSON.parse(line); } catch { continue; }
+            if (!isRecord(value) || typeof value.requestId !== "string") continue;
+            const pending = this.pending.get(value.requestId);
+            if (!pending) continue;
+            this.pending.delete(value.requestId);
+            if (value.ok === true) {
+                if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
+                    pending.reject(new LineartRuntimeError("lineart_response_invalid", "本机 AI 线稿响应无效", 500));
+                    continue;
+                }
+                this.loaded = true;
+                pending.resolve(value as unknown as LineartWorkerSuccess);
+                continue;
+            }
+            const code = typeof value.code === "string" ? value.code : "lineart_inference_failed";
+            const statusCode = code === "lineart_model_missing" ? 503 : code === "lineart_input_invalid" ? 400 : 500;
+            pending.reject(new LineartRuntimeError(code, typeof value.message === "string" ? value.message : "本机 AI 线稿推理失败", statusCode));
+        }
+    }
+
+    private handleChildExit(child?: ChildProcessWithoutNullStreams) {
+        if (!child || this.child !== child) return;
+        this.child = undefined;
+        this.loaded = false;
+        this.stderrBuffer = "";
+        this.stdoutBuffer = "";
+        this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已退出"));
+    }
+
+    private rejectPending(error: LineartRuntimeError) {
+        for (const { reject } of this.pending.values()) reject(error);
+        this.pending.clear();
+    }
+}
+
+export function validateLineartDataUrl(value: string) {
+    const match = DATA_URL_PATTERN.exec(value);
+    if (!match) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
+    const bytes = Buffer.from(match[2]!, "base64");
+    if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
+}
+
+function parseLineartInput(body: Record<string, unknown>) {
+    const keys = Object.keys(body).sort();
+    if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "lineart" || typeof body.dataUrl !== "string") {
+        throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换请求字段无效", 400);
+    }
+    validateLineartDataUrl(body.dataUrl);
+    return body.dataUrl;
+}
+
+function parseBody(request: Request): Record<string, unknown> {
+    if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
+    try {
+        const value = JSON.parse(request.body.toString("utf8")) as unknown;
+        return isRecord(value) ? value : {};
+    } catch {
+        return {};
+    }
+}
+
+function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
+    return (request, response, next) => {
+        void action(request, response).catch((error) => {
+            if (response.headersSent) return next(error);
+            if (error instanceof LineartRuntimeError) {
+                response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
+                return;
+            }
+            response.status(503).json({ ok: false, code: "lineart_runtime_unavailable", message: "本机 AI 线稿运行时不可用" });
+        });
+    };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+    return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+
+function isPositiveInteger(value: unknown): value is number {
+    return typeof value === "number" && Number.isInteger(value) && value > 0;
+}

canvas-agent/src/modules/pose-estimation-http.ts

@@ -0,0 +1,357 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+
+import type { Request, RequestHandler, Response } from "express";
+
+import type { LocalRuntimeModule } from "../local-runtime.js";
+
+const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
+const DEFAULT_BACKEND = "openpose-body" as const;
+const MAX_INPUT_BYTES = 12 * 1024 * 1024;
+const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
+const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
+
+export type PoseRuntimePaths = {
+    pythonPath: string;
+    scriptPath: string;
+    hfHome: string;
+    modelId: string;
+    configured: boolean;
+    modelCached: boolean;
+    reason?: "python_missing" | "worker_missing";
+};
+
+export type PoseRuntimeStatus = {
+    configured: boolean;
+    modelCached: boolean;
+    loaded: boolean;
+    device: "cpu";
+    modelId: string;
+    backend: typeof DEFAULT_BACKEND;
+    reason?: string;
+};
+
+type PoseWorkerSuccess = {
+    ok: true;
+    requestId: string;
+    mimeType: "image/png";
+    width: number;
+    height: number;
+    modelId: string;
+    device: "cpu";
+    backend: typeof DEFAULT_BACKEND;
+    personCount: number;
+    people?: unknown[];
+    pngBase64: string;
+};
+
+export class PoseRuntimeError extends Error {
+    constructor(readonly code: string, message: string, readonly statusCode = 503) {
+        super(message);
+        this.name = "PoseRuntimeError";
+    }
+}
+
+export function createPoseEstimationHttpModule(): LocalRuntimeModule {
+    const worker = new PoseEstimationWorker();
+    return {
+        descriptor: {
+            id: "pose-estimation",
+            displayName: "OpenPose 本机姿态骨架",
+            apiVersion: 1,
+            scopes: ["pose:status", "pose:run"],
+        },
+        routes: [
+            {
+                method: "GET",
+                path: "/pose-estimation/status",
+                scope: "pose:status",
+                handler: asyncRoute(async (_request, response) => {
+                    const status = worker.status();
+                    response.json({
+                        ok: true,
+                        module: "pose-estimation",
+                        apiVersion: 1,
+                        ready: status.configured && status.modelCached,
+                        ...status,
+                    });
+                }),
+            },
+            {
+                method: "POST",
+                path: "/pose-estimation/run",
+                scope: "pose:run",
+                handler: asyncRoute(async (request, response) => {
+                    const body = parseBody(request);
+                    const dataUrl = parsePoseInput(body);
+                    const result = await worker.run({ dataUrl });
+                    response.json({
+                        ok: true,
+                        module: "pose-estimation",
+                        apiVersion: 1,
+                        result,
+                    });
+                }),
+            },
+        ],
+        dispose: () => worker.dispose(),
+        publicHealth: () => {
+            const status = worker.status();
+            return { poseEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
+        },
+    };
+}
+
+export function resolvePoseRuntimePaths(env: NodeJS.ProcessEnv = process.env): PoseRuntimePaths {
+    const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+    const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
+    const projectRoots = [
+        configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
+        path.resolve(moduleRoot, ".."),
+        path.resolve(process.cwd()),
+    ].filter((value): value is string => Boolean(value));
+    const projectRoot = configuredProjectRoot
+        ? path.resolve(configuredProjectRoot)
+        : projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
+    const scriptCandidates = [
+        path.join(moduleRoot, "python", "pose_runtime.py"),
+        path.join(projectRoot, "canvas-agent", "python", "pose_runtime.py"),
+    ];
+    const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
+    const pythonCandidates = env.CANVAS_POSE_PYTHON?.trim()
+        ? [path.resolve(env.CANVAS_POSE_PYTHON.trim())]
+        : process.platform === "win32"
+            ? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
+            : [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
+    const pythonPath = pythonCandidates[0]!;
+    const hfHome = path.resolve(
+        env.CANVAS_POSE_HF_HOME?.trim()
+        || env.CANVAS_LINEART_HF_HOME?.trim()
+        || env.CANVAS_DEPTH_HF_HOME?.trim()
+        || path.join(projectRoot, ".local", "cache", "huggingface"),
+    );
+    const modelId = env.CANVAS_POSE_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
+    const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
+    const workerExists = fs.existsSync(scriptPath);
+    const pythonExists = fs.existsSync(pythonPath);
+    return {
+        pythonPath,
+        scriptPath,
+        hfHome,
+        modelId,
+        configured: workerExists && pythonExists,
+        modelCached: workerExists && pythonExists && findCachedModelFile(modelCacheDir, "body_pose_model.pth"),
+        ...(pythonExists ? {} : { reason: "python_missing" as const }),
+        ...(!workerExists ? { reason: "worker_missing" as const } : {}),
+    };
+}
+
+export function findCachedModelFile(modelCacheDir: string, filename: string) {
+    const snapshotsDir = path.join(modelCacheDir, "snapshots");
+    try {
+        return fs.readdirSync(snapshotsDir, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .some((entry) => fs.existsSync(path.join(snapshotsDir, entry.name, filename)));
+    } catch {
+        return false;
+    }
+}
+
+export class PoseEstimationWorker {
+    private paths = resolvePoseRuntimePaths();
+    private child?: ChildProcessWithoutNullStreams;
+    private stdoutBuffer = "";
+    private stderrBuffer = "";
+    private requestCounter = 0;
+    private loaded = false;
+    private readonly pending = new Map<string, { resolve: (value: PoseWorkerSuccess) => void; reject: (error: unknown) => void }>();
+
+    status(): PoseRuntimeStatus {
+        const paths = this.refreshPaths();
+        return {
+            configured: paths.configured,
+            modelCached: paths.modelCached,
+            loaded: this.loaded,
+            device: "cpu",
+            modelId: paths.modelId,
+            backend: DEFAULT_BACKEND,
+            ...(paths.reason ? { reason: paths.reason } : {}),
+        };
+    }
+
+    async run(input: { dataUrl: string }) {
+        validatePoseDataUrl(input.dataUrl);
+        const paths = this.refreshPaths();
+        if (!paths.configured) {
+            throw new PoseRuntimeError(
+                "pose_runtime_unavailable",
+                paths.reason === "worker_missing" ? "本机姿态运行文件不存在" : "未找到本机姿态 Python 环境",
+            );
+        }
+        const child = this.ensureChild();
+        const requestId = `pose-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
+        const response = new Promise<PoseWorkerSuccess>((resolve, reject) => {
+            this.pending.set(requestId, { resolve, reject });
+        });
+        try {
+            child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
+        } catch {
+            this.pending.delete(requestId);
+            throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法写入");
+        }
+        return response;
+    }
+
+    dispose() {
+        const child = this.child;
+        this.child = undefined;
+        this.loaded = false;
+        this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已停止"));
+        if (child && !child.killed) child.kill();
+    }
+
+    private ensureChild() {
+        if (this.child && !this.child.killed) return this.child;
+        let child: ChildProcessWithoutNullStreams;
+        try {
+            child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
+                cwd: path.dirname(this.paths.scriptPath),
+                env: {
+                    ...process.env,
+                    HF_HOME: this.paths.hfHome,
+                    HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
+                    HF_HUB_OFFLINE: "1",
+                    TRANSFORMERS_OFFLINE: "1",
+                    HF_HUB_DISABLE_TELEMETRY: "1",
+                    HF_HUB_DISABLE_PROGRESS_BARS: "1",
+                    TRANSFORMERS_VERBOSITY: "error",
+                    CANVAS_POSE_MODEL_ID: this.paths.modelId,
+                },
+                stdio: ["pipe", "pipe", "pipe"],
+                windowsHide: true,
+            });
+        } catch {
+            throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法启动");
+        }
+        this.child = child;
+        child.stdout.setEncoding("utf8");
+        child.stderr.setEncoding("utf8");
+        child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
+        child.stderr.on("data", (chunk: string) => {
+            this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
+        });
+        child.on("error", () => this.handleChildExit(child));
+        child.on("exit", () => this.handleChildExit(child));
+        return child;
+    }
+
+    private refreshPaths() {
+        if (!this.child) this.paths = resolvePoseRuntimePaths();
+        return this.paths;
+    }
+
+    private consumeStdout(chunk: string) {
+        this.stdoutBuffer += chunk;
+        if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
+            this.child?.kill();
+            this.handleChildExit(this.child);
+            return;
+        }
+        while (true) {
+            const newline = this.stdoutBuffer.indexOf("\n");
+            if (newline < 0) return;
+            const line = this.stdoutBuffer.slice(0, newline).trim();
+            this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
+            if (!line) continue;
+            let value: unknown;
+            try { value = JSON.parse(line); } catch { continue; }
+            if (!isRecord(value) || typeof value.requestId !== "string") continue;
+            const pending = this.pending.get(value.requestId);
+            if (!pending) continue;
+            this.pending.delete(value.requestId);
+            if (value.ok === true) {
+                if (value.mimeType !== "image/png"
+                    || !isPositiveInteger(value.width)
+                    || !isPositiveInteger(value.height)
+                    || typeof value.modelId !== "string"
+                    || value.device !== "cpu"
+                    || value.backend !== DEFAULT_BACKEND
+                    || !isPositiveInteger(value.personCount)
+                    || typeof value.pngBase64 !== "string"
+                    || !value.pngBase64) {
+                    pending.reject(new PoseRuntimeError("pose_response_invalid", "本机姿态响应无效", 500));
+                    continue;
+                }
+                this.loaded = true;
+                pending.resolve(value as unknown as PoseWorkerSuccess);
+                continue;
+            }
+            const code = typeof value.code === "string" ? value.code : "pose_inference_failed";
+            const statusCode = code === "pose_model_missing" ? 503 : code === "pose_input_invalid" ? 400 : code === "pose_no_person" ? 422 : 500;
+            pending.reject(new PoseRuntimeError(code, typeof value.message === "string" ? value.message : "本机姿态推理失败", statusCode));
+        }
+    }
+
+    private handleChildExit(child?: ChildProcessWithoutNullStreams) {
+        if (!child || this.child !== child) return;
+        this.child = undefined;
+        this.loaded = false;
+        this.stderrBuffer = "";
+        this.stdoutBuffer = "";
+        this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已退出"));
+    }
+
+    private rejectPending(error: PoseRuntimeError) {
+        for (const { reject } of this.pending.values()) reject(error);
+        this.pending.clear();
+    }
+}
+
+export function validatePoseDataUrl(value: string) {
+    const match = DATA_URL_PATTERN.exec(value);
+    if (!match) throw new PoseRuntimeError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
+    const bytes = Buffer.from(match[2]!, "base64");
+    if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new PoseRuntimeError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
+}
+
+function parsePoseInput(body: Record<string, unknown>) {
+    const keys = Object.keys(body).sort();
+    if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "pose" || typeof body.dataUrl !== "string") {
+        throw new PoseRuntimeError("pose_input_invalid", "姿态转换请求字段无效", 400);
+    }
+    validatePoseDataUrl(body.dataUrl);
+    return body.dataUrl;
+}
+
+function parseBody(request: Request): Record<string, unknown> {
+    if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
+    try {
+        const value = JSON.parse(request.body.toString("utf8")) as unknown;
+        return isRecord(value) ? value : {};
+    } catch {
+        return {};
+    }
+}
+
+function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
+    return (request, response, next) => {
+        void action(request, response).catch((error) => {
+            if (response.headersSent) return next(error);
+            if (error instanceof PoseRuntimeError) {
+                response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
+                return;
+            }
+            response.status(503).json({ ok: false, code: "pose_runtime_unavailable", message: "本机姿态运行时不可用" });
+        });
+    };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+    return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+
+function isPositiveInteger(value: unknown): value is number {
+    return typeof value === "number" && Number.isInteger(value) && value > 0;
+}