feat(canvas): 图片转换插件 - 增加本地媒体转换节点与运行时

This commit is contained in:
QAyong
2026-09-08 22:55:44 +08:00
parent 8360b2c933
commit f755d6db97
51 changed files with 3284 additions and 44 deletions

View File

@@ -76,6 +76,58 @@ Canvas Agent 内置 `portrait-clearance` Local Runtime 模块。它只接收签
本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`,安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时,模块返回 `portrait_model_missing`,不会伪造低风险结果。
## Depth Anything V2 本机深度模块
画布“转换 → 深度图”使用 `depth-estimation` Local Runtime 模块。模块只接受签名的本机请求,Node Runtime 启动一个常驻 Python worker;模型首次调用时加载,后续转换复用同一进程。worker 以离线模式读取本地 Hugging Face 缓存,不会把图片发送到云端。
使用仓库根目录的 `start-yingce-local.cmd` 时,Canvas Agent 会和前端、后端一起自动启动,转换节点不需要用户另外打开 Agent 窗口或手工填写地址。只有手工分别启动前端和后端时,才需要在本目录执行 `node dist/index.js`。
当前仓库开发环境默认查找:
```text
.local/depth-anything-v2/venv/Scripts/python.exe # Windows
.local/depth-anything-v2/venv/bin/python # Linux/macOS
.local/cache/huggingface/ # 模型缓存
```
发布包会携带 `python/depth_anything_runtime.py`,但不会携带 Python、依赖或模型权重。自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_DEPTH_HF_HOME`;需要指定工作区时设置 `CANVAS_PROJECT_ROOT`。模型缺失、Python 环境缺失或 worker 退出时,转换节点显示具体失败状态并保留输入,不会把失败标记为完成。
## ControlNet Aux 本机 AI 线稿模块
画布“转换 → AI 线稿”使用 `lineart-estimation` Local Runtime 模块。它只运行 ControlNet Aux 的 `LineartDetector` 预处理器,把图片转换为线稿控制图;不会加载 Stable Diffusion、ControlNet 生图管线,也不会把图片上传到云端。Node Runtime 会复用同一个常驻 Python worker,首次转换加载模型,后续请求复用已加载模型。
该模块与深度模块共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 模型缓存。首次设置或缺少依赖时,在仓库根目录执行:
```powershell
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
& $python -m pip install -r canvas-agent/python/requirements-lineart.txt
```
然后用下面的命令把线稿预处理器权重缓存到本地;下载完成后,正常转换会强制离线运行:
```powershell
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
& $python -c "from controlnet_aux import LineartDetector; LineartDetector.from_pretrained('lllyasviel/Annotators')"
```
自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_LINEART_HF_HOME` 和 `CANVAS_LINEART_MODEL_ID`;未找到 Python、依赖或模型时,节点显示“本地不可用”并保留输入,不会伪造已完成结果。
## ControlNet Aux 本机姿态骨架模块
画布“转换 → 姿态骨架”使用 `pose-estimation` Local Runtime 模块。当前笔记本方案采用 ControlNet Aux 的 **OpenPose body-only** 预处理器:只提取人体 18 点骨架并输出彩色姿态图,不加载 Stable Diffusion,也不上传图片。Node Runtime 会复用常驻 Python worker;未检测到人物时返回 `pose_no_person`,转换节点显示“已跳过”,不会保存伪造结果。
该模块与深度、线稿共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 缓存。安装 `requirements-lineart.txt` 后缓存人体权重:
```powershell
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
& $python -c "from huggingface_hub import hf_hub_download; hf_hub_download('lllyasviel/Annotators', 'body_pose_model.pth', cache_dir='$env:HF_HUB_CACHE')"
```
自定义安装位置时设置 `CANVAS_POSE_PYTHON`、`CANVAS_POSE_HF_HOME` 和 `CANVAS_POSE_MODEL_ID`。DWPose(包含手部、脸部和更完整关键点)仍可在后续按需替换,但它需要额外的 MMDetection/MMPose 依赖;当前实现优先保证普通 CPU 笔记本可运行。
## 发布
`canvas-agent` 使用自己的 `package.json` 版本号,不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。

View File

@@ -9,6 +9,7 @@
},
"files": [
"dist",
"python",
"README.md"
],
"scripts": {

View File

@@ -0,0 +1,134 @@
"""Small persistent worker for the local Depth Anything V2 Small model.
The Node Local Runtime owns the process and sends one JSON request per line.
Keeping the pipeline alive avoids loading the model for every canvas conversion.
The worker is deliberately offline-only: the model must already exist in the
configured Hugging Face cache.
"""
from __future__ import annotations
import base64
import contextlib
import io
import json
import os
import re
import sys
from typing import Any
from PIL import Image
MODEL_ID = os.environ.get("CANVAS_DEPTH_MODEL_ID", "depth-anything/Depth-Anything-V2-Small-hf")
MAX_INPUT_BYTES = 12 * 1024 * 1024
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
PIPELINE: Any = None
def emit(value: dict[str, Any]) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def load_pipeline() -> Any:
global PIPELINE
if PIPELINE is not None:
return PIPELINE
try:
from transformers import pipeline
from transformers.utils import logging as transformers_logging
transformers_logging.set_verbosity_error()
# Transformers/tqdm can write progress text while loading weights.
# Keep stdout reserved for the JSON-lines protocol.
with contextlib.redirect_stdout(sys.stderr):
PIPELINE = pipeline(
"depth-estimation",
model=MODEL_ID,
device=-1,
local_files_only=True,
)
except Exception as error: # pragma: no cover - depends on local model files
raise RuntimeError("depth_model_missing") from error
return PIPELINE
def image_from_data_url(value: Any) -> Image.Image:
if not isinstance(value, str):
raise ValueError("depth_input_invalid")
match = DATA_URL_PATTERN.fullmatch(value)
if not match:
raise ValueError("depth_input_invalid")
try:
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
except Exception as error:
raise ValueError("depth_input_invalid") from error
if not raw or len(raw) > MAX_INPUT_BYTES:
raise ValueError("depth_input_invalid")
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
image.load()
return image
except Exception as error:
raise ValueError("depth_input_invalid") from error
def encode_png(image: Image.Image) -> str:
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return base64.b64encode(output.getvalue()).decode("ascii")
def handle(request: dict[str, Any]) -> dict[str, Any]:
image = image_from_data_url(request.get("dataUrl"))
try:
result = load_pipeline()(image)
depth = result.get("depth") if isinstance(result, dict) else None
if not isinstance(depth, Image.Image):
raise RuntimeError("depth_inference_failed")
return {
"ok": True,
"requestId": request.get("requestId"),
"mimeType": "image/png",
"width": depth.width,
"height": depth.height,
"modelId": MODEL_ID,
"device": "cpu",
"pngBase64": encode_png(depth),
}
except RuntimeError:
raise
except Exception as error: # pragma: no cover - depends on local runtime
raise RuntimeError("depth_inference_failed") from error
def main() -> None:
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
for line in sys.stdin:
line = line.strip()
if not line:
continue
request_id: Any = None
try:
request = json.loads(line)
if not isinstance(request, dict):
raise ValueError("depth_input_invalid")
request_id = request.get("requestId")
emit(handle(request))
except ValueError as error:
emit({"ok": False, "requestId": request_id, "code": str(error) or "depth_input_invalid", "message": "深度转换输入无效"})
except RuntimeError as error:
code = str(error) or "depth_inference_failed"
emit({"ok": False, "requestId": request_id, "code": code, "message": "本机深度模型不可用" if code == "depth_model_missing" else "本机深度推理失败"})
except Exception:
emit({"ok": False, "requestId": request_id, "code": "depth_inference_failed", "message": "本机深度推理失败"})
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,132 @@
"""Small persistent worker for the local ControlNet Aux lineart detector.
This worker only extracts a lineart control image. It does not load Stable
Diffusion or a ControlNet generation pipeline, so it stays suitable for a
CPU-only notebook. The Node Local Runtime owns the process and sends one JSON
request per line; stdout is reserved for that protocol.
"""
from __future__ import annotations
import base64
import contextlib
import io
import json
import os
import re
import sys
from typing import Any
from PIL import Image
MODEL_ID = os.environ.get("CANVAS_LINEART_MODEL_ID", "lllyasviel/Annotators")
MAX_INPUT_BYTES = 12 * 1024 * 1024
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
DETECTOR: Any = None
def emit(value: dict[str, Any]) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def load_detector() -> Any:
global DETECTOR
if DETECTOR is not None:
return DETECTOR
try:
from controlnet_aux import LineartDetector
# The detector downloads only the lineart preprocessor weights from the
# configured Hugging Face cache. It is intentionally not a diffusion
# pipeline and never loads Stable Diffusion or ControlNet weights.
with contextlib.redirect_stdout(sys.stderr):
DETECTOR = LineartDetector.from_pretrained(MODEL_ID)
except Exception as error: # pragma: no cover - depends on local files
raise RuntimeError("lineart_model_missing") from error
return DETECTOR
def image_from_data_url(value: Any) -> Image.Image:
if not isinstance(value, str):
raise ValueError("lineart_input_invalid")
match = DATA_URL_PATTERN.fullmatch(value)
if not match:
raise ValueError("lineart_input_invalid")
try:
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
except Exception as error:
raise ValueError("lineart_input_invalid") from error
if not raw or len(raw) > MAX_INPUT_BYTES:
raise ValueError("lineart_input_invalid")
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
image.load()
return image
except Exception as error:
raise ValueError("lineart_input_invalid") from error
def encode_png(image: Image.Image) -> str:
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return base64.b64encode(output.getvalue()).decode("ascii")
def handle(request: dict[str, Any]) -> dict[str, Any]:
image = image_from_data_url(request.get("dataUrl"))
try:
lineart = load_detector()(image)
if not isinstance(lineart, Image.Image):
raise RuntimeError("lineart_inference_failed")
return {
"ok": True,
"requestId": request.get("requestId"),
"mimeType": "image/png",
"width": lineart.width,
"height": lineart.height,
"modelId": MODEL_ID,
"device": "cpu",
"pngBase64": encode_png(lineart.convert("RGB")),
}
except RuntimeError:
raise
except Exception as error: # pragma: no cover - depends on local runtime
raise RuntimeError("lineart_inference_failed") from error
def main() -> None:
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
for line in sys.stdin:
# PowerShell and a few Windows launchers may prepend a UTF-8 BOM to
# the first piped line. The Node runtime sends plain UTF-8, but
# accepting the marker keeps the worker protocol tolerant for manual
# diagnostics as well.
line = line.lstrip("\ufeff").strip()
if not line:
continue
request_id: Any = None
try:
request = json.loads(line)
if not isinstance(request, dict):
raise ValueError("lineart_input_invalid")
request_id = request.get("requestId")
emit(handle(request))
except ValueError as error:
emit({"ok": False, "requestId": request_id, "code": str(error) or "lineart_input_invalid", "message": "线稿转换输入无效"})
except RuntimeError as error:
code = str(error) or "lineart_inference_failed"
message = "本地 AI 线稿模型或依赖尚未安装" if code == "lineart_model_missing" else "本地 AI 线稿推理失败"
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
except Exception:
emit({"ok": False, "requestId": request_id, "code": "lineart_inference_failed", "message": "本地 AI 线稿推理失败"})
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,204 @@
"""Persistent CPU pose preprocessor for the local media conversion node.
The first notebook-friendly implementation uses the body-only OpenPose
preprocessor from ControlNet Aux. It produces a colored skeleton map and a
small list of normalized keypoints; it does not load a diffusion model.
"""
from __future__ import annotations
import base64
import contextlib
import io
import json
import os
import re
import sys
from typing import Any
from PIL import Image
MODEL_ID = os.environ.get("CANVAS_POSE_MODEL_ID", "lllyasviel/Annotators")
MAX_INPUT_BYTES = 12 * 1024 * 1024
DETECT_RESOLUTION = max(256, min(1024, int(os.environ.get("CANVAS_POSE_DETECT_RESOLUTION", "512"))))
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
DETECTOR: Any = None
def emit(value: dict[str, Any]) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def load_detector() -> Any:
global DETECTOR
if DETECTOR is not None:
return DETECTOR
try:
import numpy as np
from controlnet_aux.open_pose.body import Body
from huggingface_hub import hf_hub_download
cache_dir = os.environ.get("HF_HUB_CACHE") or None
with contextlib.redirect_stdout(sys.stderr):
model_path = hf_hub_download(
repo_id=MODEL_ID,
filename="body_pose_model.pth",
cache_dir=cache_dir,
local_files_only=True,
)
DETECTOR = (Body(model_path), np)
except Exception as error: # pragma: no cover - depends on local model files
raise RuntimeError("pose_model_missing") from error
return DETECTOR
def image_from_data_url(value: Any) -> Image.Image:
if not isinstance(value, str):
raise ValueError("pose_input_invalid")
match = DATA_URL_PATTERN.fullmatch(value)
if not match:
raise ValueError("pose_input_invalid")
try:
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
except Exception as error:
raise ValueError("pose_input_invalid") from error
if not raw or len(raw) > MAX_INPUT_BYTES:
raise ValueError("pose_input_invalid")
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
image.load()
return image
except Exception as error:
raise ValueError("pose_input_invalid") from error
def encode_png(image: Image.Image) -> str:
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return base64.b64encode(output.getvalue()).decode("ascii")
def normalize_people(bodies: list[Any], width: int, height: int) -> list[dict[str, Any]]:
people: list[dict[str, Any]] = []
for body in bodies:
keypoints = []
visible = []
for keypoint in body.keypoints:
if keypoint is None:
keypoints.append(None)
else:
normalized_x = max(0.0, min(1.0, float(keypoint.x) / float(width)))
normalized_y = max(0.0, min(1.0, float(keypoint.y) / float(height)))
keypoints.append({
"x": normalized_x,
"y": normalized_y,
"score": float(keypoint.score),
})
visible.append((normalized_x, normalized_y, float(keypoint.score)))
if visible:
xs = [point[0] for point in visible]
ys = [point[1] for point in visible]
confidence = sum(point[2] for point in visible) / len(visible)
box = {"x": min(xs), "y": min(ys), "width": max(xs) - min(xs), "height": max(ys) - min(ys)}
else:
confidence = 0.0
box = {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}
people.append({
"keypoints": keypoints,
"score": float(body.total_score),
"confidence": confidence,
"box": box,
"parts": int(body.total_parts),
})
return people
def handle(request: dict[str, Any]) -> dict[str, Any]:
image = image_from_data_url(request.get("dataUrl"))
try:
import cv2
import numpy as np
from controlnet_aux.open_pose.util import draw_bodypose
from controlnet_aux.util import HWC3, resize_image
detector, _ = load_detector()
source = HWC3(np.asarray(image, dtype=np.uint8))
detected = resize_image(source, DETECT_RESOLUTION)
candidate, subset = detector(detected[:, :, ::-1].copy())
bodies = detector.format_body_result(candidate, subset)
if not bodies:
raise RuntimeError("pose_no_person")
height, width = detected.shape[:2]
people = normalize_people(bodies, width, height)
canvas = np.zeros_like(detected, dtype=np.uint8)
for body in bodies:
normalized = [
None if keypoint is None else type(keypoint)(
x=float(keypoint.x) / float(width),
y=float(keypoint.y) / float(height),
score=keypoint.score,
id=keypoint.id,
)
for keypoint in body.keypoints
]
draw_bodypose(canvas, normalized)
output = cv2.resize(canvas, (image.width, image.height), interpolation=cv2.INTER_LINEAR)
output_image = Image.fromarray(output.astype(np.uint8), mode="RGB")
return {
"ok": True,
"requestId": request.get("requestId"),
"mimeType": "image/png",
"width": image.width,
"height": image.height,
"modelId": MODEL_ID,
"device": "cpu",
"backend": "openpose-body",
"personCount": len(bodies),
"people": people,
"pngBase64": encode_png(output_image),
}
except RuntimeError:
raise
except Exception as error: # pragma: no cover - depends on local runtime
raise RuntimeError("pose_inference_failed") from error
def main() -> None:
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
for line in sys.stdin:
line = line.lstrip("\ufeff").strip()
if not line:
continue
request_id: Any = None
try:
request = json.loads(line)
if not isinstance(request, dict):
raise ValueError("pose_input_invalid")
request_id = request.get("requestId")
emit(handle(request))
except ValueError as error:
emit({"ok": False, "requestId": request_id, "code": str(error) or "pose_input_invalid", "message": "姿态转换输入无效"})
except RuntimeError as error:
code = str(error) or "pose_inference_failed"
if code == "pose_model_missing":
message = "本地姿态模型或依赖尚未安装"
elif code == "pose_no_person":
message = "未检测到可用人物姿态"
else:
message = "本地姿态推理失败"
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
except Exception:
emit({"ok": False, "requestId": request_id, "code": "pose_inference_failed", "message": "本地姿态推理失败"})
if __name__ == "__main__":
main()

View File

@@ -0,0 +1 @@
controlnet-aux==0.0.10

View File

@@ -4,7 +4,7 @@ import { CanvasSession } from "./canvas-session.js";
import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js";
import { createLocalRuntimeApp } from "./local-runtime.js";
import { startLocalRuntime } from "./local-runtime-host.js";
import { LocalRuntimeSessionManager } from "./local-runtime-session.js";
import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./local-runtime-session.js";
import {
createCanvasAgentHttpModule,
type CanvasAgentSession,
@@ -14,6 +14,9 @@ import {
type DreaminaHttpModuleOptions,
} from "./modules/dreamina-http.js";
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">;
@@ -34,21 +37,29 @@ export function createHttpApp(
dependencies: CanvasAgentHttpDependencies = {},
): Express {
const endpoint = config.url;
const modules = [
createCanvasAgentHttpModule(config, session),
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
createDepthEstimationHttpModule(),
createLineartEstimationHttpModule(),
createPoseEstimationHttpModule(),
];
const manager = new LocalRuntimeSessionManager({
endpoint,
trustedOrigins: config.trustedWebOrigins,
registrations: config.browserRegistrations,
scopes: [...new Set([
...LOCAL_RUNTIME_DEFAULT_SCOPES,
...modules.flatMap((module) => module.descriptor.scopes),
])],
});
return createLocalRuntimeApp({
authority: new URL(endpoint).host,
endpoint,
version: "0.1.0",
sessionManager: manager,
modules: [
createCanvasAgentHttpModule(config, session),
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
],
modules,
legacyMasterToken: config.token,
legacyOrigins: config.origins ?? [],
});

View File

@@ -6,7 +6,7 @@ export const LOCAL_RUNTIME_ENDPOINT = "http://127.0.0.1:17371";
export const LOCAL_RUNTIME_ID = "framefield-local-runtime";
export const LOCAL_RUNTIME_API_VERSION = 2;
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance";
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
export type LocalRuntimeScope =
| "runtime:status"
| "runtime:revoke"
@@ -20,7 +20,13 @@ export type LocalRuntimeScope =
| "portrait:status"
| "portrait:model"
| "portrait:run"
| "portrait:read";
| "portrait:read"
| "depth:status"
| "depth:run"
| "lineart:status"
| "lineart:run"
| "pose:status"
| "pose:run";
export type DreaminaModelOperation =
| "text-to-image"

View File

@@ -17,6 +17,9 @@ import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./loca
import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js";
import { createDreaminaHttpModule } from "./modules/dreamina-http.js";
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
export type StartLocalRuntimeOptions = {
config?: LocalRuntimeConfig;
@@ -46,6 +49,9 @@ export function createDefaultLocalRuntimeModules(config: LocalRuntimeConfig): Lo
],
}),
createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }),
createDepthEstimationHttpModule(),
createLineartEstimationHttpModule(),
createPoseEstimationHttpModule(),
];
}

View File

@@ -185,7 +185,7 @@ function validateModules(modules: readonly LocalRuntimeModule[]) {
const ids = new Set<LocalRuntimeModuleId>();
const routes = new Set<string>();
for (const module of modules) {
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance") {
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance" && module.descriptor.id !== "depth-estimation" && module.descriptor.id !== "lineart-estimation" && module.descriptor.id !== "pose-estimation") {
throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`);
}
if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`);

View File

@@ -0,0 +1,327 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { Request, RequestHandler, Response } from "express";
import type { LocalRuntimeModule } from "../local-runtime.js";
const DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf";
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type DepthRuntimePaths = {
pythonPath: string;
scriptPath: string;
hfHome: string;
modelId: string;
configured: boolean;
modelCached: boolean;
reason?: "python_missing" | "worker_missing";
};
export type DepthRuntimeStatus = {
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
type DepthWorkerSuccess = {
ok: true;
requestId: string;
mimeType: "image/png";
width: number;
height: number;
modelId: string;
device: "cpu";
pngBase64: string;
};
export class DepthRuntimeError extends Error {
constructor(readonly code: string, message: string, readonly statusCode = 503) {
super(message);
this.name = "DepthRuntimeError";
}
}
export function createDepthEstimationHttpModule(): LocalRuntimeModule {
const worker = new DepthEstimationWorker();
return {
descriptor: {
id: "depth-estimation",
displayName: "Depth Anything V2 本机深度",
apiVersion: 1,
scopes: ["depth:status", "depth:run"],
},
routes: [
{
method: "GET",
path: "/depth-estimation/status",
scope: "depth:status",
handler: asyncRoute(async (_request, response) => {
const status = worker.status();
response.json({
ok: true,
module: "depth-estimation",
apiVersion: 1,
ready: status.configured && status.modelCached,
...status,
});
}),
},
{
method: "POST",
path: "/depth-estimation/run",
scope: "depth:run",
handler: asyncRoute(async (request, response) => {
const body = parseBody(request);
const dataUrl = parseDepthInput(body);
const result = await worker.run({ dataUrl });
response.json({
ok: true,
module: "depth-estimation",
apiVersion: 1,
result,
});
}),
},
],
dispose: () => worker.dispose(),
publicHealth: () => {
const status = worker.status();
return { depthEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
},
};
}
export function resolveDepthRuntimePaths(env: NodeJS.ProcessEnv = process.env): DepthRuntimePaths {
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
const projectRoots = [
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
path.resolve(moduleRoot, ".."),
path.resolve(process.cwd()),
].filter((value): value is string => Boolean(value));
const projectRoot = configuredProjectRoot
? path.resolve(configuredProjectRoot)
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
const scriptCandidates = [
path.join(moduleRoot, "python", "depth_anything_runtime.py"),
path.join(projectRoot, "canvas-agent", "python", "depth_anything_runtime.py"),
];
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
: process.platform === "win32"
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
const pythonPath = pythonCandidates[0]!;
const hfHome = path.resolve(env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
const modelId = env.CANVAS_DEPTH_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
const workerExists = fs.existsSync(scriptPath);
const pythonExists = fs.existsSync(pythonPath);
return {
pythonPath,
scriptPath,
hfHome,
modelId,
configured: workerExists && pythonExists,
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
...(pythonExists ? {} : { reason: "python_missing" as const }),
...(!workerExists ? { reason: "worker_missing" as const } : {}),
};
}
export class DepthEstimationWorker {
private paths = resolveDepthRuntimePaths();
private child?: ChildProcessWithoutNullStreams;
private stdoutBuffer = "";
private stderrBuffer = "";
private requestCounter = 0;
private loaded = false;
private readonly pending = new Map<string, { resolve: (value: DepthWorkerSuccess) => void; reject: (error: unknown) => void }>();
status(): DepthRuntimeStatus {
const paths = this.refreshPaths();
return {
configured: paths.configured,
modelCached: paths.modelCached,
loaded: this.loaded,
device: "cpu",
modelId: paths.modelId,
...(paths.reason ? { reason: paths.reason } : {}),
};
}
async run(input: { dataUrl: string }) {
validateDepthDataUrl(input.dataUrl);
const paths = this.refreshPaths();
if (!paths.configured) {
throw new DepthRuntimeError(
"depth_runtime_unavailable",
paths.reason === "worker_missing" ? "本机深度运行文件不存在" : "未找到本机深度 Python 环境",
);
}
const child = this.ensureChild();
const requestId = `depth-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
const response = new Promise<DepthWorkerSuccess>((resolve, reject) => {
this.pending.set(requestId, { resolve, reject });
});
try {
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
} catch (error) {
this.pending.delete(requestId);
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法写入");
}
return response;
}
dispose() {
const child = this.child;
this.child = undefined;
this.loaded = false;
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已停止"));
if (child && !child.killed) child.kill();
}
private ensureChild() {
if (this.child && !this.child.killed) return this.child;
let child: ChildProcessWithoutNullStreams;
try {
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
cwd: path.dirname(this.paths.scriptPath),
env: {
...process.env,
HF_HOME: this.paths.hfHome,
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
HF_HUB_OFFLINE: "1",
TRANSFORMERS_OFFLINE: "1",
HF_HUB_DISABLE_TELEMETRY: "1",
HF_HUB_DISABLE_PROGRESS_BARS: "1",
TRANSFORMERS_VERBOSITY: "error",
CANVAS_DEPTH_MODEL_ID: this.paths.modelId,
},
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
} catch (error) {
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法启动");
}
this.child = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
child.stderr.on("data", (chunk: string) => {
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
});
child.on("error", () => this.handleChildExit(child));
child.on("exit", () => this.handleChildExit(child));
return child;
}
private refreshPaths() {
if (!this.child) this.paths = resolveDepthRuntimePaths();
return this.paths;
}
private consumeStdout(chunk: string) {
this.stdoutBuffer += chunk;
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
this.child?.kill();
this.handleChildExit(this.child);
return;
}
while (true) {
const newline = this.stdoutBuffer.indexOf("\n");
if (newline < 0) return;
const line = this.stdoutBuffer.slice(0, newline).trim();
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
if (!line) continue;
let value: unknown;
try { value = JSON.parse(line); } catch { continue; }
if (!isRecord(value) || typeof value.requestId !== "string") continue;
const pending = this.pending.get(value.requestId);
if (!pending) continue;
this.pending.delete(value.requestId);
if (value.ok === true) {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
pending.reject(new DepthRuntimeError("depth_response_invalid", "本机深度响应无效", 500));
continue;
}
this.loaded = true;
pending.resolve(value as unknown as DepthWorkerSuccess);
continue;
}
const code = typeof value.code === "string" ? value.code : "depth_inference_failed";
const statusCode = code === "depth_model_missing" ? 503 : code === "depth_input_invalid" ? 400 : 500;
pending.reject(new DepthRuntimeError(code, typeof value.message === "string" ? value.message : "本机深度推理失败", statusCode));
}
}
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
if (!child || this.child !== child) return;
this.child = undefined;
this.loaded = false;
this.stderrBuffer = "";
this.stdoutBuffer = "";
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已退出"));
}
private rejectPending(error: DepthRuntimeError) {
for (const { reject } of this.pending.values()) reject(error);
this.pending.clear();
}
}
export function validateDepthDataUrl(value: string) {
const match = DATA_URL_PATTERN.exec(value);
if (!match) throw new DepthRuntimeError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = Buffer.from(match[2]!, "base64");
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new DepthRuntimeError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
}
function parseDepthInput(body: Record<string, unknown>) {
const keys = Object.keys(body).sort();
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "depth" || typeof body.dataUrl !== "string") {
throw new DepthRuntimeError("depth_input_invalid", "深度转换请求字段无效", 400);
}
validateDepthDataUrl(body.dataUrl);
return body.dataUrl;
}
function parseBody(request: Request): Record<string, unknown> {
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
try {
const value = JSON.parse(request.body.toString("utf8")) as unknown;
return isRecord(value) ? value : {};
} catch {
return {};
}
}
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
return (request, response, next) => {
void action(request, response).catch((error) => {
if (response.headersSent) return next(error);
if (error instanceof DepthRuntimeError) {
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
return;
}
response.status(503).json({ ok: false, code: "depth_runtime_unavailable", message: "本机深度运行时不可用" });
});
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}

View File

@@ -0,0 +1,327 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { Request, RequestHandler, Response } from "express";
import type { LocalRuntimeModule } from "../local-runtime.js";
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LineartRuntimePaths = {
pythonPath: string;
scriptPath: string;
hfHome: string;
modelId: string;
configured: boolean;
modelCached: boolean;
reason?: "python_missing" | "worker_missing";
};
export type LineartRuntimeStatus = {
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
type LineartWorkerSuccess = {
ok: true;
requestId: string;
mimeType: "image/png";
width: number;
height: number;
modelId: string;
device: "cpu";
pngBase64: string;
};
export class LineartRuntimeError extends Error {
constructor(readonly code: string, message: string, readonly statusCode = 503) {
super(message);
this.name = "LineartRuntimeError";
}
}
export function createLineartEstimationHttpModule(): LocalRuntimeModule {
const worker = new LineartEstimationWorker();
return {
descriptor: {
id: "lineart-estimation",
displayName: "ControlNet Aux 本机 AI 线稿",
apiVersion: 1,
scopes: ["lineart:status", "lineart:run"],
},
routes: [
{
method: "GET",
path: "/lineart-estimation/status",
scope: "lineart:status",
handler: asyncRoute(async (_request, response) => {
const status = worker.status();
response.json({
ok: true,
module: "lineart-estimation",
apiVersion: 1,
ready: status.configured && status.modelCached,
...status,
});
}),
},
{
method: "POST",
path: "/lineart-estimation/run",
scope: "lineart:run",
handler: asyncRoute(async (request, response) => {
const body = parseBody(request);
const dataUrl = parseLineartInput(body);
const result = await worker.run({ dataUrl });
response.json({
ok: true,
module: "lineart-estimation",
apiVersion: 1,
result,
});
}),
},
],
dispose: () => worker.dispose(),
publicHealth: () => {
const status = worker.status();
return { lineartEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
},
};
}
export function resolveLineartRuntimePaths(env: NodeJS.ProcessEnv = process.env): LineartRuntimePaths {
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
const projectRoots = [
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
path.resolve(moduleRoot, ".."),
path.resolve(process.cwd()),
].filter((value): value is string => Boolean(value));
const projectRoot = configuredProjectRoot
? path.resolve(configuredProjectRoot)
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
const scriptCandidates = [
path.join(moduleRoot, "python", "lineart_runtime.py"),
path.join(projectRoot, "canvas-agent", "python", "lineart_runtime.py"),
];
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
: process.platform === "win32"
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
const pythonPath = pythonCandidates[0]!;
const hfHome = path.resolve(env.CANVAS_LINEART_HF_HOME?.trim() || env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
const modelId = env.CANVAS_LINEART_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
const workerExists = fs.existsSync(scriptPath);
const pythonExists = fs.existsSync(pythonPath);
return {
pythonPath,
scriptPath,
hfHome,
modelId,
configured: workerExists && pythonExists,
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
...(pythonExists ? {} : { reason: "python_missing" as const }),
...(!workerExists ? { reason: "worker_missing" as const } : {}),
};
}
export class LineartEstimationWorker {
private paths = resolveLineartRuntimePaths();
private child?: ChildProcessWithoutNullStreams;
private stdoutBuffer = "";
private stderrBuffer = "";
private requestCounter = 0;
private loaded = false;
private readonly pending = new Map<string, { resolve: (value: LineartWorkerSuccess) => void; reject: (error: unknown) => void }>();
status(): LineartRuntimeStatus {
const paths = this.refreshPaths();
return {
configured: paths.configured,
modelCached: paths.modelCached,
loaded: this.loaded,
device: "cpu",
modelId: paths.modelId,
...(paths.reason ? { reason: paths.reason } : {}),
};
}
async run(input: { dataUrl: string }) {
validateLineartDataUrl(input.dataUrl);
const paths = this.refreshPaths();
if (!paths.configured) {
throw new LineartRuntimeError(
"lineart_runtime_unavailable",
paths.reason === "worker_missing" ? "本机 AI 线稿运行文件不存在" : "未找到本机 AI 线稿 Python 环境",
);
}
const child = this.ensureChild();
const requestId = `lineart-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
const response = new Promise<LineartWorkerSuccess>((resolve, reject) => {
this.pending.set(requestId, { resolve, reject });
});
try {
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
} catch {
this.pending.delete(requestId);
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法写入");
}
return response;
}
dispose() {
const child = this.child;
this.child = undefined;
this.loaded = false;
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已停止"));
if (child && !child.killed) child.kill();
}
private ensureChild() {
if (this.child && !this.child.killed) return this.child;
let child: ChildProcessWithoutNullStreams;
try {
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
cwd: path.dirname(this.paths.scriptPath),
env: {
...process.env,
HF_HOME: this.paths.hfHome,
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
HF_HUB_OFFLINE: "1",
TRANSFORMERS_OFFLINE: "1",
HF_HUB_DISABLE_TELEMETRY: "1",
HF_HUB_DISABLE_PROGRESS_BARS: "1",
TRANSFORMERS_VERBOSITY: "error",
CANVAS_LINEART_MODEL_ID: this.paths.modelId,
},
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
} catch {
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法启动");
}
this.child = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
child.stderr.on("data", (chunk: string) => {
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
});
child.on("error", () => this.handleChildExit(child));
child.on("exit", () => this.handleChildExit(child));
return child;
}
private refreshPaths() {
if (!this.child) this.paths = resolveLineartRuntimePaths();
return this.paths;
}
private consumeStdout(chunk: string) {
this.stdoutBuffer += chunk;
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
this.child?.kill();
this.handleChildExit(this.child);
return;
}
while (true) {
const newline = this.stdoutBuffer.indexOf("\n");
if (newline < 0) return;
const line = this.stdoutBuffer.slice(0, newline).trim();
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
if (!line) continue;
let value: unknown;
try { value = JSON.parse(line); } catch { continue; }
if (!isRecord(value) || typeof value.requestId !== "string") continue;
const pending = this.pending.get(value.requestId);
if (!pending) continue;
this.pending.delete(value.requestId);
if (value.ok === true) {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
pending.reject(new LineartRuntimeError("lineart_response_invalid", "本机 AI 线稿响应无效", 500));
continue;
}
this.loaded = true;
pending.resolve(value as unknown as LineartWorkerSuccess);
continue;
}
const code = typeof value.code === "string" ? value.code : "lineart_inference_failed";
const statusCode = code === "lineart_model_missing" ? 503 : code === "lineart_input_invalid" ? 400 : 500;
pending.reject(new LineartRuntimeError(code, typeof value.message === "string" ? value.message : "本机 AI 线稿推理失败", statusCode));
}
}
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
if (!child || this.child !== child) return;
this.child = undefined;
this.loaded = false;
this.stderrBuffer = "";
this.stdoutBuffer = "";
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已退出"));
}
private rejectPending(error: LineartRuntimeError) {
for (const { reject } of this.pending.values()) reject(error);
this.pending.clear();
}
}
export function validateLineartDataUrl(value: string) {
const match = DATA_URL_PATTERN.exec(value);
if (!match) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = Buffer.from(match[2]!, "base64");
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
}
function parseLineartInput(body: Record<string, unknown>) {
const keys = Object.keys(body).sort();
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "lineart" || typeof body.dataUrl !== "string") {
throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换请求字段无效", 400);
}
validateLineartDataUrl(body.dataUrl);
return body.dataUrl;
}
function parseBody(request: Request): Record<string, unknown> {
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
try {
const value = JSON.parse(request.body.toString("utf8")) as unknown;
return isRecord(value) ? value : {};
} catch {
return {};
}
}
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
return (request, response, next) => {
void action(request, response).catch((error) => {
if (response.headersSent) return next(error);
if (error instanceof LineartRuntimeError) {
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
return;
}
response.status(503).json({ ok: false, code: "lineart_runtime_unavailable", message: "本机 AI 线稿运行时不可用" });
});
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}

View File

@@ -0,0 +1,357 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { Request, RequestHandler, Response } from "express";
import type { LocalRuntimeModule } from "../local-runtime.js";
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
const DEFAULT_BACKEND = "openpose-body" as const;
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type PoseRuntimePaths = {
pythonPath: string;
scriptPath: string;
hfHome: string;
modelId: string;
configured: boolean;
modelCached: boolean;
reason?: "python_missing" | "worker_missing";
};
export type PoseRuntimeStatus = {
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
backend: typeof DEFAULT_BACKEND;
reason?: string;
};
type PoseWorkerSuccess = {
ok: true;
requestId: string;
mimeType: "image/png";
width: number;
height: number;
modelId: string;
device: "cpu";
backend: typeof DEFAULT_BACKEND;
personCount: number;
people?: unknown[];
pngBase64: string;
};
export class PoseRuntimeError extends Error {
constructor(readonly code: string, message: string, readonly statusCode = 503) {
super(message);
this.name = "PoseRuntimeError";
}
}
export function createPoseEstimationHttpModule(): LocalRuntimeModule {
const worker = new PoseEstimationWorker();
return {
descriptor: {
id: "pose-estimation",
displayName: "OpenPose 本机姿态骨架",
apiVersion: 1,
scopes: ["pose:status", "pose:run"],
},
routes: [
{
method: "GET",
path: "/pose-estimation/status",
scope: "pose:status",
handler: asyncRoute(async (_request, response) => {
const status = worker.status();
response.json({
ok: true,
module: "pose-estimation",
apiVersion: 1,
ready: status.configured && status.modelCached,
...status,
});
}),
},
{
method: "POST",
path: "/pose-estimation/run",
scope: "pose:run",
handler: asyncRoute(async (request, response) => {
const body = parseBody(request);
const dataUrl = parsePoseInput(body);
const result = await worker.run({ dataUrl });
response.json({
ok: true,
module: "pose-estimation",
apiVersion: 1,
result,
});
}),
},
],
dispose: () => worker.dispose(),
publicHealth: () => {
const status = worker.status();
return { poseEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
},
};
}
export function resolvePoseRuntimePaths(env: NodeJS.ProcessEnv = process.env): PoseRuntimePaths {
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
const projectRoots = [
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
path.resolve(moduleRoot, ".."),
path.resolve(process.cwd()),
].filter((value): value is string => Boolean(value));
const projectRoot = configuredProjectRoot
? path.resolve(configuredProjectRoot)
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
const scriptCandidates = [
path.join(moduleRoot, "python", "pose_runtime.py"),
path.join(projectRoot, "canvas-agent", "python", "pose_runtime.py"),
];
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
const pythonCandidates = env.CANVAS_POSE_PYTHON?.trim()
? [path.resolve(env.CANVAS_POSE_PYTHON.trim())]
: process.platform === "win32"
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
const pythonPath = pythonCandidates[0]!;
const hfHome = path.resolve(
env.CANVAS_POSE_HF_HOME?.trim()
|| env.CANVAS_LINEART_HF_HOME?.trim()
|| env.CANVAS_DEPTH_HF_HOME?.trim()
|| path.join(projectRoot, ".local", "cache", "huggingface"),
);
const modelId = env.CANVAS_POSE_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
const workerExists = fs.existsSync(scriptPath);
const pythonExists = fs.existsSync(pythonPath);
return {
pythonPath,
scriptPath,
hfHome,
modelId,
configured: workerExists && pythonExists,
modelCached: workerExists && pythonExists && findCachedModelFile(modelCacheDir, "body_pose_model.pth"),
...(pythonExists ? {} : { reason: "python_missing" as const }),
...(!workerExists ? { reason: "worker_missing" as const } : {}),
};
}
export function findCachedModelFile(modelCacheDir: string, filename: string) {
const snapshotsDir = path.join(modelCacheDir, "snapshots");
try {
return fs.readdirSync(snapshotsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.some((entry) => fs.existsSync(path.join(snapshotsDir, entry.name, filename)));
} catch {
return false;
}
}
export class PoseEstimationWorker {
private paths = resolvePoseRuntimePaths();
private child?: ChildProcessWithoutNullStreams;
private stdoutBuffer = "";
private stderrBuffer = "";
private requestCounter = 0;
private loaded = false;
private readonly pending = new Map<string, { resolve: (value: PoseWorkerSuccess) => void; reject: (error: unknown) => void }>();
status(): PoseRuntimeStatus {
const paths = this.refreshPaths();
return {
configured: paths.configured,
modelCached: paths.modelCached,
loaded: this.loaded,
device: "cpu",
modelId: paths.modelId,
backend: DEFAULT_BACKEND,
...(paths.reason ? { reason: paths.reason } : {}),
};
}
async run(input: { dataUrl: string }) {
validatePoseDataUrl(input.dataUrl);
const paths = this.refreshPaths();
if (!paths.configured) {
throw new PoseRuntimeError(
"pose_runtime_unavailable",
paths.reason === "worker_missing" ? "本机姿态运行文件不存在" : "未找到本机姿态 Python 环境",
);
}
const child = this.ensureChild();
const requestId = `pose-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
const response = new Promise<PoseWorkerSuccess>((resolve, reject) => {
this.pending.set(requestId, { resolve, reject });
});
try {
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
} catch {
this.pending.delete(requestId);
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法写入");
}
return response;
}
dispose() {
const child = this.child;
this.child = undefined;
this.loaded = false;
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已停止"));
if (child && !child.killed) child.kill();
}
private ensureChild() {
if (this.child && !this.child.killed) return this.child;
let child: ChildProcessWithoutNullStreams;
try {
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
cwd: path.dirname(this.paths.scriptPath),
env: {
...process.env,
HF_HOME: this.paths.hfHome,
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
HF_HUB_OFFLINE: "1",
TRANSFORMERS_OFFLINE: "1",
HF_HUB_DISABLE_TELEMETRY: "1",
HF_HUB_DISABLE_PROGRESS_BARS: "1",
TRANSFORMERS_VERBOSITY: "error",
CANVAS_POSE_MODEL_ID: this.paths.modelId,
},
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
} catch {
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法启动");
}
this.child = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
child.stderr.on("data", (chunk: string) => {
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
});
child.on("error", () => this.handleChildExit(child));
child.on("exit", () => this.handleChildExit(child));
return child;
}
private refreshPaths() {
if (!this.child) this.paths = resolvePoseRuntimePaths();
return this.paths;
}
private consumeStdout(chunk: string) {
this.stdoutBuffer += chunk;
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
this.child?.kill();
this.handleChildExit(this.child);
return;
}
while (true) {
const newline = this.stdoutBuffer.indexOf("\n");
if (newline < 0) return;
const line = this.stdoutBuffer.slice(0, newline).trim();
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
if (!line) continue;
let value: unknown;
try { value = JSON.parse(line); } catch { continue; }
if (!isRecord(value) || typeof value.requestId !== "string") continue;
const pending = this.pending.get(value.requestId);
if (!pending) continue;
this.pending.delete(value.requestId);
if (value.ok === true) {
if (value.mimeType !== "image/png"
|| !isPositiveInteger(value.width)
|| !isPositiveInteger(value.height)
|| typeof value.modelId !== "string"
|| value.device !== "cpu"
|| value.backend !== DEFAULT_BACKEND
|| !isPositiveInteger(value.personCount)
|| typeof value.pngBase64 !== "string"
|| !value.pngBase64) {
pending.reject(new PoseRuntimeError("pose_response_invalid", "本机姿态响应无效", 500));
continue;
}
this.loaded = true;
pending.resolve(value as unknown as PoseWorkerSuccess);
continue;
}
const code = typeof value.code === "string" ? value.code : "pose_inference_failed";
const statusCode = code === "pose_model_missing" ? 503 : code === "pose_input_invalid" ? 400 : code === "pose_no_person" ? 422 : 500;
pending.reject(new PoseRuntimeError(code, typeof value.message === "string" ? value.message : "本机姿态推理失败", statusCode));
}
}
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
if (!child || this.child !== child) return;
this.child = undefined;
this.loaded = false;
this.stderrBuffer = "";
this.stdoutBuffer = "";
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已退出"));
}
private rejectPending(error: PoseRuntimeError) {
for (const { reject } of this.pending.values()) reject(error);
this.pending.clear();
}
}
export function validatePoseDataUrl(value: string) {
const match = DATA_URL_PATTERN.exec(value);
if (!match) throw new PoseRuntimeError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = Buffer.from(match[2]!, "base64");
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new PoseRuntimeError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
}
function parsePoseInput(body: Record<string, unknown>) {
const keys = Object.keys(body).sort();
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "pose" || typeof body.dataUrl !== "string") {
throw new PoseRuntimeError("pose_input_invalid", "姿态转换请求字段无效", 400);
}
validatePoseDataUrl(body.dataUrl);
return body.dataUrl;
}
function parseBody(request: Request): Record<string, unknown> {
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
try {
const value = JSON.parse(request.body.toString("utf8")) as unknown;
return isRecord(value) ? value : {};
} catch {
return {};
}
}
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
return (request, response, next) => {
void action(request, response).catch((error) => {
if (response.headersSent) return next(error);
if (error instanceof PoseRuntimeError) {
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
return;
}
response.status(503).json({ ok: false, code: "pose_runtime_unavailable", message: "本机姿态运行时不可用" });
});
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}

View File

@@ -0,0 +1,36 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createDepthEstimationHttpModule, DepthRuntimeError, resolveDepthRuntimePaths, validateDepthDataUrl } from "../src/modules/depth-estimation-http.js";
test("depth runtime module declares signed status and run routes", () => {
const module = createDepthEstimationHttpModule();
assert.equal(module.descriptor.id, "depth-estimation");
assert.deepEqual(module.descriptor.scopes, ["depth:status", "depth:run"]);
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
"GET /depth-estimation/status",
"POST /depth-estimation/run",
]);
module.dispose?.();
});
test("depth input guard accepts image data URLs and rejects unsafe input", () => {
assert.doesNotThrow(() => validateDepthDataUrl("data:image/png;base64,AQID"));
assert.throws(
() => validateDepthDataUrl("data:text/plain;base64,AQID"),
(error: unknown) => error instanceof DepthRuntimeError && error.code === "depth_input_invalid" && error.statusCode === 400,
);
});
test("depth runtime paths honor explicit project and Python overrides", () => {
const paths = resolveDepthRuntimePaths({
CANVAS_PROJECT_ROOT: "C:\\depth-project",
CANVAS_DEPTH_PYTHON: "C:\\depth-project\\venv\\python.exe",
CANVAS_DEPTH_HF_HOME: "C:\\depth-cache",
CANVAS_DEPTH_MODEL_ID: "example/depth-model",
});
assert.equal(paths.pythonPath, "C:\\depth-project\\venv\\python.exe");
assert.equal(paths.hfHome, "C:\\depth-cache");
assert.equal(paths.modelId, "example/depth-model");
assert.equal(paths.configured, false);
});

View File

@@ -0,0 +1,36 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createLineartEstimationHttpModule, LineartRuntimeError, resolveLineartRuntimePaths, validateLineartDataUrl } from "../src/modules/lineart-estimation-http.js";
test("lineart runtime module declares signed status and run routes", () => {
const module = createLineartEstimationHttpModule();
assert.equal(module.descriptor.id, "lineart-estimation");
assert.deepEqual(module.descriptor.scopes, ["lineart:status", "lineart:run"]);
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
"GET /lineart-estimation/status",
"POST /lineart-estimation/run",
]);
module.dispose?.();
});
test("lineart input guard accepts image data URLs and rejects unsafe input", () => {
assert.doesNotThrow(() => validateLineartDataUrl("data:image/png;base64,AQID"));
assert.throws(
() => validateLineartDataUrl("data:text/plain;base64,AQID"),
(error: unknown) => error instanceof LineartRuntimeError && error.code === "lineart_input_invalid" && error.statusCode === 400,
);
});
test("lineart runtime paths honor explicit project and Python overrides", () => {
const paths = resolveLineartRuntimePaths({
CANVAS_PROJECT_ROOT: "C:\\lineart-project",
CANVAS_DEPTH_PYTHON: "C:\\lineart-project\\venv\\python.exe",
CANVAS_LINEART_HF_HOME: "C:\\lineart-cache",
CANVAS_LINEART_MODEL_ID: "example/lineart-model",
});
assert.equal(paths.pythonPath, "C:\\lineart-project\\venv\\python.exe");
assert.equal(paths.hfHome, "C:\\lineart-cache");
assert.equal(paths.modelId, "example/lineart-model");
assert.equal(paths.configured, false);
});

View File

@@ -0,0 +1,37 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createPoseEstimationHttpModule, PoseRuntimeError, resolvePoseRuntimePaths, validatePoseDataUrl } from "../src/modules/pose-estimation-http.js";
test("pose runtime module declares signed status and run routes", () => {
const module = createPoseEstimationHttpModule();
assert.equal(module.descriptor.id, "pose-estimation");
assert.deepEqual(module.descriptor.scopes, ["pose:status", "pose:run"]);
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
"GET /pose-estimation/status",
"POST /pose-estimation/run",
]);
module.dispose?.();
});
test("pose input guard accepts image data URLs and rejects unsafe input", () => {
assert.doesNotThrow(() => validatePoseDataUrl("data:image/png;base64,AQID"));
assert.throws(
() => validatePoseDataUrl("data:text/plain;base64,AQID"),
(error: unknown) => error instanceof PoseRuntimeError && error.code === "pose_input_invalid" && error.statusCode === 400,
);
});
test("pose runtime paths honor explicit project and Python overrides", () => {
const paths = resolvePoseRuntimePaths({
CANVAS_PROJECT_ROOT: "C:\\pose-project",
CANVAS_POSE_PYTHON: "C:\\pose-project\\venv\\python.exe",
CANVAS_POSE_HF_HOME: "C:\\pose-cache",
CANVAS_POSE_MODEL_ID: "example/pose-model",
});
assert.equal(paths.pythonPath, "C:\\pose-project\\venv\\python.exe");
assert.equal(paths.hfHome, "C:\\pose-cache");
assert.equal(paths.modelId, "example/pose-model");
assert.equal(paths.configured, false);
assert.equal(paths.modelCached, false);
});