feat(canvas): 图片转换插件 - 增加本地媒体转换节点与运行时
This commit is contained in:
@@ -76,6 +76,58 @@ Canvas Agent 内置 `portrait-clearance` Local Runtime 模块。它只接收签
|
||||
|
||||
本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`,安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时,模块返回 `portrait_model_missing`,不会伪造低风险结果。
|
||||
|
||||
## Depth Anything V2 本机深度模块
|
||||
|
||||
画布“转换 → 深度图”使用 `depth-estimation` Local Runtime 模块。模块只接受签名的本机请求,Node Runtime 启动一个常驻 Python worker;模型首次调用时加载,后续转换复用同一进程。worker 以离线模式读取本地 Hugging Face 缓存,不会把图片发送到云端。
|
||||
|
||||
使用仓库根目录的 `start-yingce-local.cmd` 时,Canvas Agent 会和前端、后端一起自动启动,转换节点不需要用户另外打开 Agent 窗口或手工填写地址。只有手工分别启动前端和后端时,才需要在本目录执行 `node dist/index.js`。
|
||||
|
||||
当前仓库开发环境默认查找:
|
||||
|
||||
```text
|
||||
.local/depth-anything-v2/venv/Scripts/python.exe # Windows
|
||||
.local/depth-anything-v2/venv/bin/python # Linux/macOS
|
||||
.local/cache/huggingface/ # 模型缓存
|
||||
```
|
||||
|
||||
发布包会携带 `python/depth_anything_runtime.py`,但不会携带 Python、依赖或模型权重。自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_DEPTH_HF_HOME`;需要指定工作区时设置 `CANVAS_PROJECT_ROOT`。模型缺失、Python 环境缺失或 worker 退出时,转换节点显示具体失败状态并保留输入,不会把失败标记为完成。
|
||||
|
||||
## ControlNet Aux 本机 AI 线稿模块
|
||||
|
||||
画布“转换 → AI 线稿”使用 `lineart-estimation` Local Runtime 模块。它只运行 ControlNet Aux 的 `LineartDetector` 预处理器,把图片转换为线稿控制图;不会加载 Stable Diffusion、ControlNet 生图管线,也不会把图片上传到云端。Node Runtime 会复用同一个常驻 Python worker,首次转换加载模型,后续请求复用已加载模型。
|
||||
|
||||
该模块与深度模块共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 模型缓存。首次设置或缺少依赖时,在仓库根目录执行:
|
||||
|
||||
```powershell
|
||||
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
|
||||
& $python -m pip install -r canvas-agent/python/requirements-lineart.txt
|
||||
```
|
||||
|
||||
然后用下面的命令把线稿预处理器权重缓存到本地;下载完成后,正常转换会强制离线运行:
|
||||
|
||||
```powershell
|
||||
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
|
||||
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
|
||||
& $python -c "from controlnet_aux import LineartDetector; LineartDetector.from_pretrained('lllyasviel/Annotators')"
|
||||
```
|
||||
|
||||
自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_LINEART_HF_HOME` 和 `CANVAS_LINEART_MODEL_ID`;未找到 Python、依赖或模型时,节点显示“本地不可用”并保留输入,不会伪造已完成结果。
|
||||
|
||||
## ControlNet Aux 本机姿态骨架模块
|
||||
|
||||
画布“转换 → 姿态骨架”使用 `pose-estimation` Local Runtime 模块。当前笔记本方案采用 ControlNet Aux 的 **OpenPose body-only** 预处理器:只提取人体 18 点骨架并输出彩色姿态图,不加载 Stable Diffusion,也不上传图片。Node Runtime 会复用常驻 Python worker;未检测到人物时返回 `pose_no_person`,转换节点显示“已跳过”,不会保存伪造结果。
|
||||
|
||||
该模块与深度、线稿共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 缓存。安装 `requirements-lineart.txt` 后缓存人体权重:
|
||||
|
||||
```powershell
|
||||
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
|
||||
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
|
||||
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
|
||||
& $python -c "from huggingface_hub import hf_hub_download; hf_hub_download('lllyasviel/Annotators', 'body_pose_model.pth', cache_dir='$env:HF_HUB_CACHE')"
|
||||
```
|
||||
|
||||
自定义安装位置时设置 `CANVAS_POSE_PYTHON`、`CANVAS_POSE_HF_HOME` 和 `CANVAS_POSE_MODEL_ID`。DWPose(包含手部、脸部和更完整关键点)仍可在后续按需替换,但它需要额外的 MMDetection/MMPose 依赖;当前实现优先保证普通 CPU 笔记本可运行。
|
||||
|
||||
## 发布
|
||||
|
||||
`canvas-agent` 使用自己的 `package.json` 版本号,不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"python",
|
||||
"README.md"
|
||||
],
|
||||
"scripts": {
|
||||
|
||||
134
canvas-agent/python/depth_anything_runtime.py
Normal file
134
canvas-agent/python/depth_anything_runtime.py
Normal file
@@ -0,0 +1,134 @@
|
||||
"""Small persistent worker for the local Depth Anything V2 Small model.
|
||||
|
||||
The Node Local Runtime owns the process and sends one JSON request per line.
|
||||
Keeping the pipeline alive avoids loading the model for every canvas conversion.
|
||||
The worker is deliberately offline-only: the model must already exist in the
|
||||
configured Hugging Face cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
MODEL_ID = os.environ.get("CANVAS_DEPTH_MODEL_ID", "depth-anything/Depth-Anything-V2-Small-hf")
|
||||
MAX_INPUT_BYTES = 12 * 1024 * 1024
|
||||
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
|
||||
PIPELINE: Any = None
|
||||
|
||||
|
||||
def emit(value: dict[str, Any]) -> None:
|
||||
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def load_pipeline() -> Any:
|
||||
global PIPELINE
|
||||
if PIPELINE is not None:
|
||||
return PIPELINE
|
||||
|
||||
try:
|
||||
from transformers import pipeline
|
||||
from transformers.utils import logging as transformers_logging
|
||||
|
||||
transformers_logging.set_verbosity_error()
|
||||
# Transformers/tqdm can write progress text while loading weights.
|
||||
# Keep stdout reserved for the JSON-lines protocol.
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
PIPELINE = pipeline(
|
||||
"depth-estimation",
|
||||
model=MODEL_ID,
|
||||
device=-1,
|
||||
local_files_only=True,
|
||||
)
|
||||
except Exception as error: # pragma: no cover - depends on local model files
|
||||
raise RuntimeError("depth_model_missing") from error
|
||||
return PIPELINE
|
||||
|
||||
|
||||
def image_from_data_url(value: Any) -> Image.Image:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("depth_input_invalid")
|
||||
match = DATA_URL_PATTERN.fullmatch(value)
|
||||
if not match:
|
||||
raise ValueError("depth_input_invalid")
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
|
||||
except Exception as error:
|
||||
raise ValueError("depth_input_invalid") from error
|
||||
if not raw or len(raw) > MAX_INPUT_BYTES:
|
||||
raise ValueError("depth_input_invalid")
|
||||
try:
|
||||
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||
image.load()
|
||||
return image
|
||||
except Exception as error:
|
||||
raise ValueError("depth_input_invalid") from error
|
||||
|
||||
|
||||
def encode_png(image: Image.Image) -> str:
|
||||
output = io.BytesIO()
|
||||
image.save(output, format="PNG", optimize=True)
|
||||
return base64.b64encode(output.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def handle(request: dict[str, Any]) -> dict[str, Any]:
|
||||
image = image_from_data_url(request.get("dataUrl"))
|
||||
try:
|
||||
result = load_pipeline()(image)
|
||||
depth = result.get("depth") if isinstance(result, dict) else None
|
||||
if not isinstance(depth, Image.Image):
|
||||
raise RuntimeError("depth_inference_failed")
|
||||
return {
|
||||
"ok": True,
|
||||
"requestId": request.get("requestId"),
|
||||
"mimeType": "image/png",
|
||||
"width": depth.width,
|
||||
"height": depth.height,
|
||||
"modelId": MODEL_ID,
|
||||
"device": "cpu",
|
||||
"pngBase64": encode_png(depth),
|
||||
}
|
||||
except RuntimeError:
|
||||
raise
|
||||
except Exception as error: # pragma: no cover - depends on local runtime
|
||||
raise RuntimeError("depth_inference_failed") from error
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
||||
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
||||
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
request_id: Any = None
|
||||
try:
|
||||
request = json.loads(line)
|
||||
if not isinstance(request, dict):
|
||||
raise ValueError("depth_input_invalid")
|
||||
request_id = request.get("requestId")
|
||||
emit(handle(request))
|
||||
except ValueError as error:
|
||||
emit({"ok": False, "requestId": request_id, "code": str(error) or "depth_input_invalid", "message": "深度转换输入无效"})
|
||||
except RuntimeError as error:
|
||||
code = str(error) or "depth_inference_failed"
|
||||
emit({"ok": False, "requestId": request_id, "code": code, "message": "本机深度模型不可用" if code == "depth_model_missing" else "本机深度推理失败"})
|
||||
except Exception:
|
||||
emit({"ok": False, "requestId": request_id, "code": "depth_inference_failed", "message": "本机深度推理失败"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
132
canvas-agent/python/lineart_runtime.py
Normal file
132
canvas-agent/python/lineart_runtime.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""Small persistent worker for the local ControlNet Aux lineart detector.
|
||||
|
||||
This worker only extracts a lineart control image. It does not load Stable
|
||||
Diffusion or a ControlNet generation pipeline, so it stays suitable for a
|
||||
CPU-only notebook. The Node Local Runtime owns the process and sends one JSON
|
||||
request per line; stdout is reserved for that protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
MODEL_ID = os.environ.get("CANVAS_LINEART_MODEL_ID", "lllyasviel/Annotators")
|
||||
MAX_INPUT_BYTES = 12 * 1024 * 1024
|
||||
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
|
||||
DETECTOR: Any = None
|
||||
|
||||
|
||||
def emit(value: dict[str, Any]) -> None:
|
||||
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def load_detector() -> Any:
|
||||
global DETECTOR
|
||||
if DETECTOR is not None:
|
||||
return DETECTOR
|
||||
|
||||
try:
|
||||
from controlnet_aux import LineartDetector
|
||||
|
||||
# The detector downloads only the lineart preprocessor weights from the
|
||||
# configured Hugging Face cache. It is intentionally not a diffusion
|
||||
# pipeline and never loads Stable Diffusion or ControlNet weights.
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
DETECTOR = LineartDetector.from_pretrained(MODEL_ID)
|
||||
except Exception as error: # pragma: no cover - depends on local files
|
||||
raise RuntimeError("lineart_model_missing") from error
|
||||
return DETECTOR
|
||||
|
||||
|
||||
def image_from_data_url(value: Any) -> Image.Image:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("lineart_input_invalid")
|
||||
match = DATA_URL_PATTERN.fullmatch(value)
|
||||
if not match:
|
||||
raise ValueError("lineart_input_invalid")
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
|
||||
except Exception as error:
|
||||
raise ValueError("lineart_input_invalid") from error
|
||||
if not raw or len(raw) > MAX_INPUT_BYTES:
|
||||
raise ValueError("lineart_input_invalid")
|
||||
try:
|
||||
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||
image.load()
|
||||
return image
|
||||
except Exception as error:
|
||||
raise ValueError("lineart_input_invalid") from error
|
||||
|
||||
|
||||
def encode_png(image: Image.Image) -> str:
|
||||
output = io.BytesIO()
|
||||
image.save(output, format="PNG", optimize=True)
|
||||
return base64.b64encode(output.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def handle(request: dict[str, Any]) -> dict[str, Any]:
|
||||
image = image_from_data_url(request.get("dataUrl"))
|
||||
try:
|
||||
lineart = load_detector()(image)
|
||||
if not isinstance(lineart, Image.Image):
|
||||
raise RuntimeError("lineart_inference_failed")
|
||||
return {
|
||||
"ok": True,
|
||||
"requestId": request.get("requestId"),
|
||||
"mimeType": "image/png",
|
||||
"width": lineart.width,
|
||||
"height": lineart.height,
|
||||
"modelId": MODEL_ID,
|
||||
"device": "cpu",
|
||||
"pngBase64": encode_png(lineart.convert("RGB")),
|
||||
}
|
||||
except RuntimeError:
|
||||
raise
|
||||
except Exception as error: # pragma: no cover - depends on local runtime
|
||||
raise RuntimeError("lineart_inference_failed") from error
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
||||
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
||||
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||
|
||||
for line in sys.stdin:
|
||||
# PowerShell and a few Windows launchers may prepend a UTF-8 BOM to
|
||||
# the first piped line. The Node runtime sends plain UTF-8, but
|
||||
# accepting the marker keeps the worker protocol tolerant for manual
|
||||
# diagnostics as well.
|
||||
line = line.lstrip("\ufeff").strip()
|
||||
if not line:
|
||||
continue
|
||||
request_id: Any = None
|
||||
try:
|
||||
request = json.loads(line)
|
||||
if not isinstance(request, dict):
|
||||
raise ValueError("lineart_input_invalid")
|
||||
request_id = request.get("requestId")
|
||||
emit(handle(request))
|
||||
except ValueError as error:
|
||||
emit({"ok": False, "requestId": request_id, "code": str(error) or "lineart_input_invalid", "message": "线稿转换输入无效"})
|
||||
except RuntimeError as error:
|
||||
code = str(error) or "lineart_inference_failed"
|
||||
message = "本地 AI 线稿模型或依赖尚未安装" if code == "lineart_model_missing" else "本地 AI 线稿推理失败"
|
||||
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
|
||||
except Exception:
|
||||
emit({"ok": False, "requestId": request_id, "code": "lineart_inference_failed", "message": "本地 AI 线稿推理失败"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
204
canvas-agent/python/pose_runtime.py
Normal file
204
canvas-agent/python/pose_runtime.py
Normal file
@@ -0,0 +1,204 @@
|
||||
"""Persistent CPU pose preprocessor for the local media conversion node.
|
||||
|
||||
The first notebook-friendly implementation uses the body-only OpenPose
|
||||
preprocessor from ControlNet Aux. It produces a colored skeleton map and a
|
||||
small list of normalized keypoints; it does not load a diffusion model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
MODEL_ID = os.environ.get("CANVAS_POSE_MODEL_ID", "lllyasviel/Annotators")
|
||||
MAX_INPUT_BYTES = 12 * 1024 * 1024
|
||||
DETECT_RESOLUTION = max(256, min(1024, int(os.environ.get("CANVAS_POSE_DETECT_RESOLUTION", "512"))))
|
||||
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
|
||||
DETECTOR: Any = None
|
||||
|
||||
|
||||
def emit(value: dict[str, Any]) -> None:
|
||||
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def load_detector() -> Any:
|
||||
global DETECTOR
|
||||
if DETECTOR is not None:
|
||||
return DETECTOR
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
from controlnet_aux.open_pose.body import Body
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
cache_dir = os.environ.get("HF_HUB_CACHE") or None
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
model_path = hf_hub_download(
|
||||
repo_id=MODEL_ID,
|
||||
filename="body_pose_model.pth",
|
||||
cache_dir=cache_dir,
|
||||
local_files_only=True,
|
||||
)
|
||||
DETECTOR = (Body(model_path), np)
|
||||
except Exception as error: # pragma: no cover - depends on local model files
|
||||
raise RuntimeError("pose_model_missing") from error
|
||||
return DETECTOR
|
||||
|
||||
|
||||
def image_from_data_url(value: Any) -> Image.Image:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("pose_input_invalid")
|
||||
match = DATA_URL_PATTERN.fullmatch(value)
|
||||
if not match:
|
||||
raise ValueError("pose_input_invalid")
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
|
||||
except Exception as error:
|
||||
raise ValueError("pose_input_invalid") from error
|
||||
if not raw or len(raw) > MAX_INPUT_BYTES:
|
||||
raise ValueError("pose_input_invalid")
|
||||
try:
|
||||
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||
image.load()
|
||||
return image
|
||||
except Exception as error:
|
||||
raise ValueError("pose_input_invalid") from error
|
||||
|
||||
|
||||
def encode_png(image: Image.Image) -> str:
|
||||
output = io.BytesIO()
|
||||
image.save(output, format="PNG", optimize=True)
|
||||
return base64.b64encode(output.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def normalize_people(bodies: list[Any], width: int, height: int) -> list[dict[str, Any]]:
|
||||
people: list[dict[str, Any]] = []
|
||||
for body in bodies:
|
||||
keypoints = []
|
||||
visible = []
|
||||
for keypoint in body.keypoints:
|
||||
if keypoint is None:
|
||||
keypoints.append(None)
|
||||
else:
|
||||
normalized_x = max(0.0, min(1.0, float(keypoint.x) / float(width)))
|
||||
normalized_y = max(0.0, min(1.0, float(keypoint.y) / float(height)))
|
||||
keypoints.append({
|
||||
"x": normalized_x,
|
||||
"y": normalized_y,
|
||||
"score": float(keypoint.score),
|
||||
})
|
||||
visible.append((normalized_x, normalized_y, float(keypoint.score)))
|
||||
if visible:
|
||||
xs = [point[0] for point in visible]
|
||||
ys = [point[1] for point in visible]
|
||||
confidence = sum(point[2] for point in visible) / len(visible)
|
||||
box = {"x": min(xs), "y": min(ys), "width": max(xs) - min(xs), "height": max(ys) - min(ys)}
|
||||
else:
|
||||
confidence = 0.0
|
||||
box = {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}
|
||||
people.append({
|
||||
"keypoints": keypoints,
|
||||
"score": float(body.total_score),
|
||||
"confidence": confidence,
|
||||
"box": box,
|
||||
"parts": int(body.total_parts),
|
||||
})
|
||||
return people
|
||||
|
||||
|
||||
def handle(request: dict[str, Any]) -> dict[str, Any]:
|
||||
image = image_from_data_url(request.get("dataUrl"))
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np
|
||||
from controlnet_aux.open_pose.util import draw_bodypose
|
||||
from controlnet_aux.util import HWC3, resize_image
|
||||
|
||||
detector, _ = load_detector()
|
||||
source = HWC3(np.asarray(image, dtype=np.uint8))
|
||||
detected = resize_image(source, DETECT_RESOLUTION)
|
||||
candidate, subset = detector(detected[:, :, ::-1].copy())
|
||||
bodies = detector.format_body_result(candidate, subset)
|
||||
if not bodies:
|
||||
raise RuntimeError("pose_no_person")
|
||||
|
||||
height, width = detected.shape[:2]
|
||||
people = normalize_people(bodies, width, height)
|
||||
canvas = np.zeros_like(detected, dtype=np.uint8)
|
||||
for body in bodies:
|
||||
normalized = [
|
||||
None if keypoint is None else type(keypoint)(
|
||||
x=float(keypoint.x) / float(width),
|
||||
y=float(keypoint.y) / float(height),
|
||||
score=keypoint.score,
|
||||
id=keypoint.id,
|
||||
)
|
||||
for keypoint in body.keypoints
|
||||
]
|
||||
draw_bodypose(canvas, normalized)
|
||||
|
||||
output = cv2.resize(canvas, (image.width, image.height), interpolation=cv2.INTER_LINEAR)
|
||||
output_image = Image.fromarray(output.astype(np.uint8), mode="RGB")
|
||||
return {
|
||||
"ok": True,
|
||||
"requestId": request.get("requestId"),
|
||||
"mimeType": "image/png",
|
||||
"width": image.width,
|
||||
"height": image.height,
|
||||
"modelId": MODEL_ID,
|
||||
"device": "cpu",
|
||||
"backend": "openpose-body",
|
||||
"personCount": len(bodies),
|
||||
"people": people,
|
||||
"pngBase64": encode_png(output_image),
|
||||
}
|
||||
except RuntimeError:
|
||||
raise
|
||||
except Exception as error: # pragma: no cover - depends on local runtime
|
||||
raise RuntimeError("pose_inference_failed") from error
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
||||
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
||||
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.lstrip("\ufeff").strip()
|
||||
if not line:
|
||||
continue
|
||||
request_id: Any = None
|
||||
try:
|
||||
request = json.loads(line)
|
||||
if not isinstance(request, dict):
|
||||
raise ValueError("pose_input_invalid")
|
||||
request_id = request.get("requestId")
|
||||
emit(handle(request))
|
||||
except ValueError as error:
|
||||
emit({"ok": False, "requestId": request_id, "code": str(error) or "pose_input_invalid", "message": "姿态转换输入无效"})
|
||||
except RuntimeError as error:
|
||||
code = str(error) or "pose_inference_failed"
|
||||
if code == "pose_model_missing":
|
||||
message = "本地姿态模型或依赖尚未安装"
|
||||
elif code == "pose_no_person":
|
||||
message = "未检测到可用人物姿态"
|
||||
else:
|
||||
message = "本地姿态推理失败"
|
||||
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
|
||||
except Exception:
|
||||
emit({"ok": False, "requestId": request_id, "code": "pose_inference_failed", "message": "本地姿态推理失败"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1
canvas-agent/python/requirements-lineart.txt
Normal file
1
canvas-agent/python/requirements-lineart.txt
Normal file
@@ -0,0 +1 @@
|
||||
controlnet-aux==0.0.10
|
||||
@@ -4,7 +4,7 @@ import { CanvasSession } from "./canvas-session.js";
|
||||
import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js";
|
||||
import { createLocalRuntimeApp } from "./local-runtime.js";
|
||||
import { startLocalRuntime } from "./local-runtime-host.js";
|
||||
import { LocalRuntimeSessionManager } from "./local-runtime-session.js";
|
||||
import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./local-runtime-session.js";
|
||||
import {
|
||||
createCanvasAgentHttpModule,
|
||||
type CanvasAgentSession,
|
||||
@@ -14,6 +14,9 @@ import {
|
||||
type DreaminaHttpModuleOptions,
|
||||
} from "./modules/dreamina-http.js";
|
||||
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
|
||||
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
|
||||
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
|
||||
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
|
||||
|
||||
export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">;
|
||||
|
||||
@@ -34,21 +37,29 @@ export function createHttpApp(
|
||||
dependencies: CanvasAgentHttpDependencies = {},
|
||||
): Express {
|
||||
const endpoint = config.url;
|
||||
const modules = [
|
||||
createCanvasAgentHttpModule(config, session),
|
||||
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
|
||||
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
|
||||
createDepthEstimationHttpModule(),
|
||||
createLineartEstimationHttpModule(),
|
||||
createPoseEstimationHttpModule(),
|
||||
];
|
||||
const manager = new LocalRuntimeSessionManager({
|
||||
endpoint,
|
||||
trustedOrigins: config.trustedWebOrigins,
|
||||
registrations: config.browserRegistrations,
|
||||
scopes: [...new Set([
|
||||
...LOCAL_RUNTIME_DEFAULT_SCOPES,
|
||||
...modules.flatMap((module) => module.descriptor.scopes),
|
||||
])],
|
||||
});
|
||||
return createLocalRuntimeApp({
|
||||
authority: new URL(endpoint).host,
|
||||
endpoint,
|
||||
version: "0.1.0",
|
||||
sessionManager: manager,
|
||||
modules: [
|
||||
createCanvasAgentHttpModule(config, session),
|
||||
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
|
||||
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
|
||||
],
|
||||
modules,
|
||||
legacyMasterToken: config.token,
|
||||
legacyOrigins: config.origins ?? [],
|
||||
});
|
||||
|
||||
@@ -6,7 +6,7 @@ export const LOCAL_RUNTIME_ENDPOINT = "http://127.0.0.1:17371";
|
||||
export const LOCAL_RUNTIME_ID = "framefield-local-runtime";
|
||||
export const LOCAL_RUNTIME_API_VERSION = 2;
|
||||
|
||||
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance";
|
||||
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
|
||||
export type LocalRuntimeScope =
|
||||
| "runtime:status"
|
||||
| "runtime:revoke"
|
||||
@@ -20,7 +20,13 @@ export type LocalRuntimeScope =
|
||||
| "portrait:status"
|
||||
| "portrait:model"
|
||||
| "portrait:run"
|
||||
| "portrait:read";
|
||||
| "portrait:read"
|
||||
| "depth:status"
|
||||
| "depth:run"
|
||||
| "lineart:status"
|
||||
| "lineart:run"
|
||||
| "pose:status"
|
||||
| "pose:run";
|
||||
|
||||
export type DreaminaModelOperation =
|
||||
| "text-to-image"
|
||||
|
||||
@@ -17,6 +17,9 @@ import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./loca
|
||||
import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js";
|
||||
import { createDreaminaHttpModule } from "./modules/dreamina-http.js";
|
||||
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
|
||||
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
|
||||
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
|
||||
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
|
||||
|
||||
export type StartLocalRuntimeOptions = {
|
||||
config?: LocalRuntimeConfig;
|
||||
@@ -46,6 +49,9 @@ export function createDefaultLocalRuntimeModules(config: LocalRuntimeConfig): Lo
|
||||
],
|
||||
}),
|
||||
createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }),
|
||||
createDepthEstimationHttpModule(),
|
||||
createLineartEstimationHttpModule(),
|
||||
createPoseEstimationHttpModule(),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -185,7 +185,7 @@ function validateModules(modules: readonly LocalRuntimeModule[]) {
|
||||
const ids = new Set<LocalRuntimeModuleId>();
|
||||
const routes = new Set<string>();
|
||||
for (const module of modules) {
|
||||
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance") {
|
||||
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance" && module.descriptor.id !== "depth-estimation" && module.descriptor.id !== "lineart-estimation" && module.descriptor.id !== "pose-estimation") {
|
||||
throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`);
|
||||
}
|
||||
if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`);
|
||||
|
||||
327
canvas-agent/src/modules/depth-estimation-http.ts
Normal file
327
canvas-agent/src/modules/depth-estimation-http.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
|
||||
import type { Request, RequestHandler, Response } from "express";
|
||||
|
||||
import type { LocalRuntimeModule } from "../local-runtime.js";
|
||||
|
||||
const DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf";
|
||||
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
|
||||
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
|
||||
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||
|
||||
export type DepthRuntimePaths = {
|
||||
pythonPath: string;
|
||||
scriptPath: string;
|
||||
hfHome: string;
|
||||
modelId: string;
|
||||
configured: boolean;
|
||||
modelCached: boolean;
|
||||
reason?: "python_missing" | "worker_missing";
|
||||
};
|
||||
|
||||
export type DepthRuntimeStatus = {
|
||||
configured: boolean;
|
||||
modelCached: boolean;
|
||||
loaded: boolean;
|
||||
device: "cpu";
|
||||
modelId: string;
|
||||
reason?: string;
|
||||
};
|
||||
|
||||
type DepthWorkerSuccess = {
|
||||
ok: true;
|
||||
requestId: string;
|
||||
mimeType: "image/png";
|
||||
width: number;
|
||||
height: number;
|
||||
modelId: string;
|
||||
device: "cpu";
|
||||
pngBase64: string;
|
||||
};
|
||||
|
||||
export class DepthRuntimeError extends Error {
|
||||
constructor(readonly code: string, message: string, readonly statusCode = 503) {
|
||||
super(message);
|
||||
this.name = "DepthRuntimeError";
|
||||
}
|
||||
}
|
||||
|
||||
export function createDepthEstimationHttpModule(): LocalRuntimeModule {
|
||||
const worker = new DepthEstimationWorker();
|
||||
return {
|
||||
descriptor: {
|
||||
id: "depth-estimation",
|
||||
displayName: "Depth Anything V2 本机深度",
|
||||
apiVersion: 1,
|
||||
scopes: ["depth:status", "depth:run"],
|
||||
},
|
||||
routes: [
|
||||
{
|
||||
method: "GET",
|
||||
path: "/depth-estimation/status",
|
||||
scope: "depth:status",
|
||||
handler: asyncRoute(async (_request, response) => {
|
||||
const status = worker.status();
|
||||
response.json({
|
||||
ok: true,
|
||||
module: "depth-estimation",
|
||||
apiVersion: 1,
|
||||
ready: status.configured && status.modelCached,
|
||||
...status,
|
||||
});
|
||||
}),
|
||||
},
|
||||
{
|
||||
method: "POST",
|
||||
path: "/depth-estimation/run",
|
||||
scope: "depth:run",
|
||||
handler: asyncRoute(async (request, response) => {
|
||||
const body = parseBody(request);
|
||||
const dataUrl = parseDepthInput(body);
|
||||
const result = await worker.run({ dataUrl });
|
||||
response.json({
|
||||
ok: true,
|
||||
module: "depth-estimation",
|
||||
apiVersion: 1,
|
||||
result,
|
||||
});
|
||||
}),
|
||||
},
|
||||
],
|
||||
dispose: () => worker.dispose(),
|
||||
publicHealth: () => {
|
||||
const status = worker.status();
|
||||
return { depthEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveDepthRuntimePaths(env: NodeJS.ProcessEnv = process.env): DepthRuntimePaths {
|
||||
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
|
||||
const projectRoots = [
|
||||
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
|
||||
path.resolve(moduleRoot, ".."),
|
||||
path.resolve(process.cwd()),
|
||||
].filter((value): value is string => Boolean(value));
|
||||
const projectRoot = configuredProjectRoot
|
||||
? path.resolve(configuredProjectRoot)
|
||||
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
|
||||
const scriptCandidates = [
|
||||
path.join(moduleRoot, "python", "depth_anything_runtime.py"),
|
||||
path.join(projectRoot, "canvas-agent", "python", "depth_anything_runtime.py"),
|
||||
];
|
||||
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
|
||||
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
|
||||
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
|
||||
: process.platform === "win32"
|
||||
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
|
||||
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
|
||||
const pythonPath = pythonCandidates[0]!;
|
||||
const hfHome = path.resolve(env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
|
||||
const modelId = env.CANVAS_DEPTH_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
|
||||
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
|
||||
const workerExists = fs.existsSync(scriptPath);
|
||||
const pythonExists = fs.existsSync(pythonPath);
|
||||
return {
|
||||
pythonPath,
|
||||
scriptPath,
|
||||
hfHome,
|
||||
modelId,
|
||||
configured: workerExists && pythonExists,
|
||||
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
|
||||
...(pythonExists ? {} : { reason: "python_missing" as const }),
|
||||
...(!workerExists ? { reason: "worker_missing" as const } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export class DepthEstimationWorker {
|
||||
private paths = resolveDepthRuntimePaths();
|
||||
private child?: ChildProcessWithoutNullStreams;
|
||||
private stdoutBuffer = "";
|
||||
private stderrBuffer = "";
|
||||
private requestCounter = 0;
|
||||
private loaded = false;
|
||||
private readonly pending = new Map<string, { resolve: (value: DepthWorkerSuccess) => void; reject: (error: unknown) => void }>();
|
||||
|
||||
status(): DepthRuntimeStatus {
|
||||
const paths = this.refreshPaths();
|
||||
return {
|
||||
configured: paths.configured,
|
||||
modelCached: paths.modelCached,
|
||||
loaded: this.loaded,
|
||||
device: "cpu",
|
||||
modelId: paths.modelId,
|
||||
...(paths.reason ? { reason: paths.reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async run(input: { dataUrl: string }) {
|
||||
validateDepthDataUrl(input.dataUrl);
|
||||
const paths = this.refreshPaths();
|
||||
if (!paths.configured) {
|
||||
throw new DepthRuntimeError(
|
||||
"depth_runtime_unavailable",
|
||||
paths.reason === "worker_missing" ? "本机深度运行文件不存在" : "未找到本机深度 Python 环境",
|
||||
);
|
||||
}
|
||||
const child = this.ensureChild();
|
||||
const requestId = `depth-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
|
||||
const response = new Promise<DepthWorkerSuccess>((resolve, reject) => {
|
||||
this.pending.set(requestId, { resolve, reject });
|
||||
});
|
||||
try {
|
||||
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
|
||||
} catch (error) {
|
||||
this.pending.delete(requestId);
|
||||
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法写入");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
dispose() {
|
||||
const child = this.child;
|
||||
this.child = undefined;
|
||||
this.loaded = false;
|
||||
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已停止"));
|
||||
if (child && !child.killed) child.kill();
|
||||
}
|
||||
|
||||
private ensureChild() {
|
||||
if (this.child && !this.child.killed) return this.child;
|
||||
let child: ChildProcessWithoutNullStreams;
|
||||
try {
|
||||
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
|
||||
cwd: path.dirname(this.paths.scriptPath),
|
||||
env: {
|
||||
...process.env,
|
||||
HF_HOME: this.paths.hfHome,
|
||||
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
|
||||
HF_HUB_OFFLINE: "1",
|
||||
TRANSFORMERS_OFFLINE: "1",
|
||||
HF_HUB_DISABLE_TELEMETRY: "1",
|
||||
HF_HUB_DISABLE_PROGRESS_BARS: "1",
|
||||
TRANSFORMERS_VERBOSITY: "error",
|
||||
CANVAS_DEPTH_MODEL_ID: this.paths.modelId,
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法启动");
|
||||
}
|
||||
this.child = child;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
|
||||
});
|
||||
child.on("error", () => this.handleChildExit(child));
|
||||
child.on("exit", () => this.handleChildExit(child));
|
||||
return child;
|
||||
}
|
||||
|
||||
private refreshPaths() {
|
||||
if (!this.child) this.paths = resolveDepthRuntimePaths();
|
||||
return this.paths;
|
||||
}
|
||||
|
||||
private consumeStdout(chunk: string) {
|
||||
this.stdoutBuffer += chunk;
|
||||
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
|
||||
this.child?.kill();
|
||||
this.handleChildExit(this.child);
|
||||
return;
|
||||
}
|
||||
while (true) {
|
||||
const newline = this.stdoutBuffer.indexOf("\n");
|
||||
if (newline < 0) return;
|
||||
const line = this.stdoutBuffer.slice(0, newline).trim();
|
||||
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||
if (!line) continue;
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(line); } catch { continue; }
|
||||
if (!isRecord(value) || typeof value.requestId !== "string") continue;
|
||||
const pending = this.pending.get(value.requestId);
|
||||
if (!pending) continue;
|
||||
this.pending.delete(value.requestId);
|
||||
if (value.ok === true) {
|
||||
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
|
||||
pending.reject(new DepthRuntimeError("depth_response_invalid", "本机深度响应无效", 500));
|
||||
continue;
|
||||
}
|
||||
this.loaded = true;
|
||||
pending.resolve(value as unknown as DepthWorkerSuccess);
|
||||
continue;
|
||||
}
|
||||
const code = typeof value.code === "string" ? value.code : "depth_inference_failed";
|
||||
const statusCode = code === "depth_model_missing" ? 503 : code === "depth_input_invalid" ? 400 : 500;
|
||||
pending.reject(new DepthRuntimeError(code, typeof value.message === "string" ? value.message : "本机深度推理失败", statusCode));
|
||||
}
|
||||
}
|
||||
|
||||
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
|
||||
if (!child || this.child !== child) return;
|
||||
this.child = undefined;
|
||||
this.loaded = false;
|
||||
this.stderrBuffer = "";
|
||||
this.stdoutBuffer = "";
|
||||
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已退出"));
|
||||
}
|
||||
|
||||
private rejectPending(error: DepthRuntimeError) {
|
||||
for (const { reject } of this.pending.values()) reject(error);
|
||||
this.pending.clear();
|
||||
}
|
||||
}
|
||||
|
||||
export function validateDepthDataUrl(value: string) {
|
||||
const match = DATA_URL_PATTERN.exec(value);
|
||||
if (!match) throw new DepthRuntimeError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||
const bytes = Buffer.from(match[2]!, "base64");
|
||||
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new DepthRuntimeError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
|
||||
}
|
||||
|
||||
function parseDepthInput(body: Record<string, unknown>) {
|
||||
const keys = Object.keys(body).sort();
|
||||
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "depth" || typeof body.dataUrl !== "string") {
|
||||
throw new DepthRuntimeError("depth_input_invalid", "深度转换请求字段无效", 400);
|
||||
}
|
||||
validateDepthDataUrl(body.dataUrl);
|
||||
return body.dataUrl;
|
||||
}
|
||||
|
||||
function parseBody(request: Request): Record<string, unknown> {
|
||||
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
|
||||
try {
|
||||
const value = JSON.parse(request.body.toString("utf8")) as unknown;
|
||||
return isRecord(value) ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
|
||||
return (request, response, next) => {
|
||||
void action(request, response).catch((error) => {
|
||||
if (response.headersSent) return next(error);
|
||||
if (error instanceof DepthRuntimeError) {
|
||||
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
|
||||
return;
|
||||
}
|
||||
response.status(503).json({ ok: false, code: "depth_runtime_unavailable", message: "本机深度运行时不可用" });
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function isPositiveInteger(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||
}
|
||||
327
canvas-agent/src/modules/lineart-estimation-http.ts
Normal file
327
canvas-agent/src/modules/lineart-estimation-http.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
|
||||
import type { Request, RequestHandler, Response } from "express";
|
||||
|
||||
import type { LocalRuntimeModule } from "../local-runtime.js";
|
||||
|
||||
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
|
||||
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
|
||||
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
|
||||
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||
|
||||
export type LineartRuntimePaths = {
|
||||
pythonPath: string;
|
||||
scriptPath: string;
|
||||
hfHome: string;
|
||||
modelId: string;
|
||||
configured: boolean;
|
||||
modelCached: boolean;
|
||||
reason?: "python_missing" | "worker_missing";
|
||||
};
|
||||
|
||||
export type LineartRuntimeStatus = {
|
||||
configured: boolean;
|
||||
modelCached: boolean;
|
||||
loaded: boolean;
|
||||
device: "cpu";
|
||||
modelId: string;
|
||||
reason?: string;
|
||||
};
|
||||
|
||||
type LineartWorkerSuccess = {
|
||||
ok: true;
|
||||
requestId: string;
|
||||
mimeType: "image/png";
|
||||
width: number;
|
||||
height: number;
|
||||
modelId: string;
|
||||
device: "cpu";
|
||||
pngBase64: string;
|
||||
};
|
||||
|
||||
export class LineartRuntimeError extends Error {
|
||||
constructor(readonly code: string, message: string, readonly statusCode = 503) {
|
||||
super(message);
|
||||
this.name = "LineartRuntimeError";
|
||||
}
|
||||
}
|
||||
|
||||
export function createLineartEstimationHttpModule(): LocalRuntimeModule {
|
||||
const worker = new LineartEstimationWorker();
|
||||
return {
|
||||
descriptor: {
|
||||
id: "lineart-estimation",
|
||||
displayName: "ControlNet Aux 本机 AI 线稿",
|
||||
apiVersion: 1,
|
||||
scopes: ["lineart:status", "lineart:run"],
|
||||
},
|
||||
routes: [
|
||||
{
|
||||
method: "GET",
|
||||
path: "/lineart-estimation/status",
|
||||
scope: "lineart:status",
|
||||
handler: asyncRoute(async (_request, response) => {
|
||||
const status = worker.status();
|
||||
response.json({
|
||||
ok: true,
|
||||
module: "lineart-estimation",
|
||||
apiVersion: 1,
|
||||
ready: status.configured && status.modelCached,
|
||||
...status,
|
||||
});
|
||||
}),
|
||||
},
|
||||
{
|
||||
method: "POST",
|
||||
path: "/lineart-estimation/run",
|
||||
scope: "lineart:run",
|
||||
handler: asyncRoute(async (request, response) => {
|
||||
const body = parseBody(request);
|
||||
const dataUrl = parseLineartInput(body);
|
||||
const result = await worker.run({ dataUrl });
|
||||
response.json({
|
||||
ok: true,
|
||||
module: "lineart-estimation",
|
||||
apiVersion: 1,
|
||||
result,
|
||||
});
|
||||
}),
|
||||
},
|
||||
],
|
||||
dispose: () => worker.dispose(),
|
||||
publicHealth: () => {
|
||||
const status = worker.status();
|
||||
return { lineartEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveLineartRuntimePaths(env: NodeJS.ProcessEnv = process.env): LineartRuntimePaths {
|
||||
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
|
||||
const projectRoots = [
|
||||
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
|
||||
path.resolve(moduleRoot, ".."),
|
||||
path.resolve(process.cwd()),
|
||||
].filter((value): value is string => Boolean(value));
|
||||
const projectRoot = configuredProjectRoot
|
||||
? path.resolve(configuredProjectRoot)
|
||||
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
|
||||
const scriptCandidates = [
|
||||
path.join(moduleRoot, "python", "lineart_runtime.py"),
|
||||
path.join(projectRoot, "canvas-agent", "python", "lineart_runtime.py"),
|
||||
];
|
||||
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
|
||||
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
|
||||
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
|
||||
: process.platform === "win32"
|
||||
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
|
||||
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
|
||||
const pythonPath = pythonCandidates[0]!;
|
||||
const hfHome = path.resolve(env.CANVAS_LINEART_HF_HOME?.trim() || env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
|
||||
const modelId = env.CANVAS_LINEART_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
|
||||
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
|
||||
const workerExists = fs.existsSync(scriptPath);
|
||||
const pythonExists = fs.existsSync(pythonPath);
|
||||
return {
|
||||
pythonPath,
|
||||
scriptPath,
|
||||
hfHome,
|
||||
modelId,
|
||||
configured: workerExists && pythonExists,
|
||||
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
|
||||
...(pythonExists ? {} : { reason: "python_missing" as const }),
|
||||
...(!workerExists ? { reason: "worker_missing" as const } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export class LineartEstimationWorker {
|
||||
private paths = resolveLineartRuntimePaths();
|
||||
private child?: ChildProcessWithoutNullStreams;
|
||||
private stdoutBuffer = "";
|
||||
private stderrBuffer = "";
|
||||
private requestCounter = 0;
|
||||
private loaded = false;
|
||||
private readonly pending = new Map<string, { resolve: (value: LineartWorkerSuccess) => void; reject: (error: unknown) => void }>();
|
||||
|
||||
status(): LineartRuntimeStatus {
|
||||
const paths = this.refreshPaths();
|
||||
return {
|
||||
configured: paths.configured,
|
||||
modelCached: paths.modelCached,
|
||||
loaded: this.loaded,
|
||||
device: "cpu",
|
||||
modelId: paths.modelId,
|
||||
...(paths.reason ? { reason: paths.reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async run(input: { dataUrl: string }) {
|
||||
validateLineartDataUrl(input.dataUrl);
|
||||
const paths = this.refreshPaths();
|
||||
if (!paths.configured) {
|
||||
throw new LineartRuntimeError(
|
||||
"lineart_runtime_unavailable",
|
||||
paths.reason === "worker_missing" ? "本机 AI 线稿运行文件不存在" : "未找到本机 AI 线稿 Python 环境",
|
||||
);
|
||||
}
|
||||
const child = this.ensureChild();
|
||||
const requestId = `lineart-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
|
||||
const response = new Promise<LineartWorkerSuccess>((resolve, reject) => {
|
||||
this.pending.set(requestId, { resolve, reject });
|
||||
});
|
||||
try {
|
||||
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
|
||||
} catch {
|
||||
this.pending.delete(requestId);
|
||||
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法写入");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
dispose() {
|
||||
const child = this.child;
|
||||
this.child = undefined;
|
||||
this.loaded = false;
|
||||
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已停止"));
|
||||
if (child && !child.killed) child.kill();
|
||||
}
|
||||
|
||||
private ensureChild() {
|
||||
if (this.child && !this.child.killed) return this.child;
|
||||
let child: ChildProcessWithoutNullStreams;
|
||||
try {
|
||||
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
|
||||
cwd: path.dirname(this.paths.scriptPath),
|
||||
env: {
|
||||
...process.env,
|
||||
HF_HOME: this.paths.hfHome,
|
||||
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
|
||||
HF_HUB_OFFLINE: "1",
|
||||
TRANSFORMERS_OFFLINE: "1",
|
||||
HF_HUB_DISABLE_TELEMETRY: "1",
|
||||
HF_HUB_DISABLE_PROGRESS_BARS: "1",
|
||||
TRANSFORMERS_VERBOSITY: "error",
|
||||
CANVAS_LINEART_MODEL_ID: this.paths.modelId,
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch {
|
||||
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法启动");
|
||||
}
|
||||
this.child = child;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
|
||||
});
|
||||
child.on("error", () => this.handleChildExit(child));
|
||||
child.on("exit", () => this.handleChildExit(child));
|
||||
return child;
|
||||
}
|
||||
|
||||
private refreshPaths() {
|
||||
if (!this.child) this.paths = resolveLineartRuntimePaths();
|
||||
return this.paths;
|
||||
}
|
||||
|
||||
private consumeStdout(chunk: string) {
|
||||
this.stdoutBuffer += chunk;
|
||||
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
|
||||
this.child?.kill();
|
||||
this.handleChildExit(this.child);
|
||||
return;
|
||||
}
|
||||
while (true) {
|
||||
const newline = this.stdoutBuffer.indexOf("\n");
|
||||
if (newline < 0) return;
|
||||
const line = this.stdoutBuffer.slice(0, newline).trim();
|
||||
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||
if (!line) continue;
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(line); } catch { continue; }
|
||||
if (!isRecord(value) || typeof value.requestId !== "string") continue;
|
||||
const pending = this.pending.get(value.requestId);
|
||||
if (!pending) continue;
|
||||
this.pending.delete(value.requestId);
|
||||
if (value.ok === true) {
|
||||
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
|
||||
pending.reject(new LineartRuntimeError("lineart_response_invalid", "本机 AI 线稿响应无效", 500));
|
||||
continue;
|
||||
}
|
||||
this.loaded = true;
|
||||
pending.resolve(value as unknown as LineartWorkerSuccess);
|
||||
continue;
|
||||
}
|
||||
const code = typeof value.code === "string" ? value.code : "lineart_inference_failed";
|
||||
const statusCode = code === "lineart_model_missing" ? 503 : code === "lineart_input_invalid" ? 400 : 500;
|
||||
pending.reject(new LineartRuntimeError(code, typeof value.message === "string" ? value.message : "本机 AI 线稿推理失败", statusCode));
|
||||
}
|
||||
}
|
||||
|
||||
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
|
||||
if (!child || this.child !== child) return;
|
||||
this.child = undefined;
|
||||
this.loaded = false;
|
||||
this.stderrBuffer = "";
|
||||
this.stdoutBuffer = "";
|
||||
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已退出"));
|
||||
}
|
||||
|
||||
private rejectPending(error: LineartRuntimeError) {
|
||||
for (const { reject } of this.pending.values()) reject(error);
|
||||
this.pending.clear();
|
||||
}
|
||||
}
|
||||
|
||||
export function validateLineartDataUrl(value: string) {
|
||||
const match = DATA_URL_PATTERN.exec(value);
|
||||
if (!match) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||
const bytes = Buffer.from(match[2]!, "base64");
|
||||
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
|
||||
}
|
||||
|
||||
function parseLineartInput(body: Record<string, unknown>) {
|
||||
const keys = Object.keys(body).sort();
|
||||
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "lineart" || typeof body.dataUrl !== "string") {
|
||||
throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换请求字段无效", 400);
|
||||
}
|
||||
validateLineartDataUrl(body.dataUrl);
|
||||
return body.dataUrl;
|
||||
}
|
||||
|
||||
function parseBody(request: Request): Record<string, unknown> {
|
||||
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
|
||||
try {
|
||||
const value = JSON.parse(request.body.toString("utf8")) as unknown;
|
||||
return isRecord(value) ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
|
||||
return (request, response, next) => {
|
||||
void action(request, response).catch((error) => {
|
||||
if (response.headersSent) return next(error);
|
||||
if (error instanceof LineartRuntimeError) {
|
||||
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
|
||||
return;
|
||||
}
|
||||
response.status(503).json({ ok: false, code: "lineart_runtime_unavailable", message: "本机 AI 线稿运行时不可用" });
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function isPositiveInteger(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||
}
|
||||
357
canvas-agent/src/modules/pose-estimation-http.ts
Normal file
357
canvas-agent/src/modules/pose-estimation-http.ts
Normal file
@@ -0,0 +1,357 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
|
||||
import type { Request, RequestHandler, Response } from "express";
|
||||
|
||||
import type { LocalRuntimeModule } from "../local-runtime.js";
|
||||
|
||||
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
|
||||
const DEFAULT_BACKEND = "openpose-body" as const;
|
||||
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
|
||||
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
|
||||
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||
|
||||
export type PoseRuntimePaths = {
|
||||
pythonPath: string;
|
||||
scriptPath: string;
|
||||
hfHome: string;
|
||||
modelId: string;
|
||||
configured: boolean;
|
||||
modelCached: boolean;
|
||||
reason?: "python_missing" | "worker_missing";
|
||||
};
|
||||
|
||||
export type PoseRuntimeStatus = {
|
||||
configured: boolean;
|
||||
modelCached: boolean;
|
||||
loaded: boolean;
|
||||
device: "cpu";
|
||||
modelId: string;
|
||||
backend: typeof DEFAULT_BACKEND;
|
||||
reason?: string;
|
||||
};
|
||||
|
||||
type PoseWorkerSuccess = {
|
||||
ok: true;
|
||||
requestId: string;
|
||||
mimeType: "image/png";
|
||||
width: number;
|
||||
height: number;
|
||||
modelId: string;
|
||||
device: "cpu";
|
||||
backend: typeof DEFAULT_BACKEND;
|
||||
personCount: number;
|
||||
people?: unknown[];
|
||||
pngBase64: string;
|
||||
};
|
||||
|
||||
export class PoseRuntimeError extends Error {
|
||||
constructor(readonly code: string, message: string, readonly statusCode = 503) {
|
||||
super(message);
|
||||
this.name = "PoseRuntimeError";
|
||||
}
|
||||
}
|
||||
|
||||
export function createPoseEstimationHttpModule(): LocalRuntimeModule {
|
||||
const worker = new PoseEstimationWorker();
|
||||
return {
|
||||
descriptor: {
|
||||
id: "pose-estimation",
|
||||
displayName: "OpenPose 本机姿态骨架",
|
||||
apiVersion: 1,
|
||||
scopes: ["pose:status", "pose:run"],
|
||||
},
|
||||
routes: [
|
||||
{
|
||||
method: "GET",
|
||||
path: "/pose-estimation/status",
|
||||
scope: "pose:status",
|
||||
handler: asyncRoute(async (_request, response) => {
|
||||
const status = worker.status();
|
||||
response.json({
|
||||
ok: true,
|
||||
module: "pose-estimation",
|
||||
apiVersion: 1,
|
||||
ready: status.configured && status.modelCached,
|
||||
...status,
|
||||
});
|
||||
}),
|
||||
},
|
||||
{
|
||||
method: "POST",
|
||||
path: "/pose-estimation/run",
|
||||
scope: "pose:run",
|
||||
handler: asyncRoute(async (request, response) => {
|
||||
const body = parseBody(request);
|
||||
const dataUrl = parsePoseInput(body);
|
||||
const result = await worker.run({ dataUrl });
|
||||
response.json({
|
||||
ok: true,
|
||||
module: "pose-estimation",
|
||||
apiVersion: 1,
|
||||
result,
|
||||
});
|
||||
}),
|
||||
},
|
||||
],
|
||||
dispose: () => worker.dispose(),
|
||||
publicHealth: () => {
|
||||
const status = worker.status();
|
||||
return { poseEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolvePoseRuntimePaths(env: NodeJS.ProcessEnv = process.env): PoseRuntimePaths {
|
||||
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
|
||||
const projectRoots = [
|
||||
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
|
||||
path.resolve(moduleRoot, ".."),
|
||||
path.resolve(process.cwd()),
|
||||
].filter((value): value is string => Boolean(value));
|
||||
const projectRoot = configuredProjectRoot
|
||||
? path.resolve(configuredProjectRoot)
|
||||
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
|
||||
const scriptCandidates = [
|
||||
path.join(moduleRoot, "python", "pose_runtime.py"),
|
||||
path.join(projectRoot, "canvas-agent", "python", "pose_runtime.py"),
|
||||
];
|
||||
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
|
||||
const pythonCandidates = env.CANVAS_POSE_PYTHON?.trim()
|
||||
? [path.resolve(env.CANVAS_POSE_PYTHON.trim())]
|
||||
: process.platform === "win32"
|
||||
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
|
||||
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
|
||||
const pythonPath = pythonCandidates[0]!;
|
||||
const hfHome = path.resolve(
|
||||
env.CANVAS_POSE_HF_HOME?.trim()
|
||||
|| env.CANVAS_LINEART_HF_HOME?.trim()
|
||||
|| env.CANVAS_DEPTH_HF_HOME?.trim()
|
||||
|| path.join(projectRoot, ".local", "cache", "huggingface"),
|
||||
);
|
||||
const modelId = env.CANVAS_POSE_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
|
||||
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
|
||||
const workerExists = fs.existsSync(scriptPath);
|
||||
const pythonExists = fs.existsSync(pythonPath);
|
||||
return {
|
||||
pythonPath,
|
||||
scriptPath,
|
||||
hfHome,
|
||||
modelId,
|
||||
configured: workerExists && pythonExists,
|
||||
modelCached: workerExists && pythonExists && findCachedModelFile(modelCacheDir, "body_pose_model.pth"),
|
||||
...(pythonExists ? {} : { reason: "python_missing" as const }),
|
||||
...(!workerExists ? { reason: "worker_missing" as const } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function findCachedModelFile(modelCacheDir: string, filename: string) {
|
||||
const snapshotsDir = path.join(modelCacheDir, "snapshots");
|
||||
try {
|
||||
return fs.readdirSync(snapshotsDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.some((entry) => fs.existsSync(path.join(snapshotsDir, entry.name, filename)));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export class PoseEstimationWorker {
|
||||
private paths = resolvePoseRuntimePaths();
|
||||
private child?: ChildProcessWithoutNullStreams;
|
||||
private stdoutBuffer = "";
|
||||
private stderrBuffer = "";
|
||||
private requestCounter = 0;
|
||||
private loaded = false;
|
||||
private readonly pending = new Map<string, { resolve: (value: PoseWorkerSuccess) => void; reject: (error: unknown) => void }>();
|
||||
|
||||
status(): PoseRuntimeStatus {
|
||||
const paths = this.refreshPaths();
|
||||
return {
|
||||
configured: paths.configured,
|
||||
modelCached: paths.modelCached,
|
||||
loaded: this.loaded,
|
||||
device: "cpu",
|
||||
modelId: paths.modelId,
|
||||
backend: DEFAULT_BACKEND,
|
||||
...(paths.reason ? { reason: paths.reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async run(input: { dataUrl: string }) {
|
||||
validatePoseDataUrl(input.dataUrl);
|
||||
const paths = this.refreshPaths();
|
||||
if (!paths.configured) {
|
||||
throw new PoseRuntimeError(
|
||||
"pose_runtime_unavailable",
|
||||
paths.reason === "worker_missing" ? "本机姿态运行文件不存在" : "未找到本机姿态 Python 环境",
|
||||
);
|
||||
}
|
||||
const child = this.ensureChild();
|
||||
const requestId = `pose-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
|
||||
const response = new Promise<PoseWorkerSuccess>((resolve, reject) => {
|
||||
this.pending.set(requestId, { resolve, reject });
|
||||
});
|
||||
try {
|
||||
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
|
||||
} catch {
|
||||
this.pending.delete(requestId);
|
||||
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法写入");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
dispose() {
|
||||
const child = this.child;
|
||||
this.child = undefined;
|
||||
this.loaded = false;
|
||||
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已停止"));
|
||||
if (child && !child.killed) child.kill();
|
||||
}
|
||||
|
||||
private ensureChild() {
|
||||
if (this.child && !this.child.killed) return this.child;
|
||||
let child: ChildProcessWithoutNullStreams;
|
||||
try {
|
||||
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
|
||||
cwd: path.dirname(this.paths.scriptPath),
|
||||
env: {
|
||||
...process.env,
|
||||
HF_HOME: this.paths.hfHome,
|
||||
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
|
||||
HF_HUB_OFFLINE: "1",
|
||||
TRANSFORMERS_OFFLINE: "1",
|
||||
HF_HUB_DISABLE_TELEMETRY: "1",
|
||||
HF_HUB_DISABLE_PROGRESS_BARS: "1",
|
||||
TRANSFORMERS_VERBOSITY: "error",
|
||||
CANVAS_POSE_MODEL_ID: this.paths.modelId,
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch {
|
||||
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法启动");
|
||||
}
|
||||
this.child = child;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
|
||||
});
|
||||
child.on("error", () => this.handleChildExit(child));
|
||||
child.on("exit", () => this.handleChildExit(child));
|
||||
return child;
|
||||
}
|
||||
|
||||
private refreshPaths() {
|
||||
if (!this.child) this.paths = resolvePoseRuntimePaths();
|
||||
return this.paths;
|
||||
}
|
||||
|
||||
private consumeStdout(chunk: string) {
|
||||
this.stdoutBuffer += chunk;
|
||||
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
|
||||
this.child?.kill();
|
||||
this.handleChildExit(this.child);
|
||||
return;
|
||||
}
|
||||
while (true) {
|
||||
const newline = this.stdoutBuffer.indexOf("\n");
|
||||
if (newline < 0) return;
|
||||
const line = this.stdoutBuffer.slice(0, newline).trim();
|
||||
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||
if (!line) continue;
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(line); } catch { continue; }
|
||||
if (!isRecord(value) || typeof value.requestId !== "string") continue;
|
||||
const pending = this.pending.get(value.requestId);
|
||||
if (!pending) continue;
|
||||
this.pending.delete(value.requestId);
|
||||
if (value.ok === true) {
|
||||
if (value.mimeType !== "image/png"
|
||||
|| !isPositiveInteger(value.width)
|
||||
|| !isPositiveInteger(value.height)
|
||||
|| typeof value.modelId !== "string"
|
||||
|| value.device !== "cpu"
|
||||
|| value.backend !== DEFAULT_BACKEND
|
||||
|| !isPositiveInteger(value.personCount)
|
||||
|| typeof value.pngBase64 !== "string"
|
||||
|| !value.pngBase64) {
|
||||
pending.reject(new PoseRuntimeError("pose_response_invalid", "本机姿态响应无效", 500));
|
||||
continue;
|
||||
}
|
||||
this.loaded = true;
|
||||
pending.resolve(value as unknown as PoseWorkerSuccess);
|
||||
continue;
|
||||
}
|
||||
const code = typeof value.code === "string" ? value.code : "pose_inference_failed";
|
||||
const statusCode = code === "pose_model_missing" ? 503 : code === "pose_input_invalid" ? 400 : code === "pose_no_person" ? 422 : 500;
|
||||
pending.reject(new PoseRuntimeError(code, typeof value.message === "string" ? value.message : "本机姿态推理失败", statusCode));
|
||||
}
|
||||
}
|
||||
|
||||
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
|
||||
if (!child || this.child !== child) return;
|
||||
this.child = undefined;
|
||||
this.loaded = false;
|
||||
this.stderrBuffer = "";
|
||||
this.stdoutBuffer = "";
|
||||
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已退出"));
|
||||
}
|
||||
|
||||
private rejectPending(error: PoseRuntimeError) {
|
||||
for (const { reject } of this.pending.values()) reject(error);
|
||||
this.pending.clear();
|
||||
}
|
||||
}
|
||||
|
||||
export function validatePoseDataUrl(value: string) {
|
||||
const match = DATA_URL_PATTERN.exec(value);
|
||||
if (!match) throw new PoseRuntimeError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||
const bytes = Buffer.from(match[2]!, "base64");
|
||||
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new PoseRuntimeError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
|
||||
}
|
||||
|
||||
function parsePoseInput(body: Record<string, unknown>) {
|
||||
const keys = Object.keys(body).sort();
|
||||
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "pose" || typeof body.dataUrl !== "string") {
|
||||
throw new PoseRuntimeError("pose_input_invalid", "姿态转换请求字段无效", 400);
|
||||
}
|
||||
validatePoseDataUrl(body.dataUrl);
|
||||
return body.dataUrl;
|
||||
}
|
||||
|
||||
function parseBody(request: Request): Record<string, unknown> {
|
||||
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
|
||||
try {
|
||||
const value = JSON.parse(request.body.toString("utf8")) as unknown;
|
||||
return isRecord(value) ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
|
||||
return (request, response, next) => {
|
||||
void action(request, response).catch((error) => {
|
||||
if (response.headersSent) return next(error);
|
||||
if (error instanceof PoseRuntimeError) {
|
||||
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
|
||||
return;
|
||||
}
|
||||
response.status(503).json({ ok: false, code: "pose_runtime_unavailable", message: "本机姿态运行时不可用" });
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function isPositiveInteger(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||
}
|
||||
36
canvas-agent/test/depth-estimation.test.ts
Normal file
36
canvas-agent/test/depth-estimation.test.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { createDepthEstimationHttpModule, DepthRuntimeError, resolveDepthRuntimePaths, validateDepthDataUrl } from "../src/modules/depth-estimation-http.js";
|
||||
|
||||
test("depth runtime module declares signed status and run routes", () => {
|
||||
const module = createDepthEstimationHttpModule();
|
||||
assert.equal(module.descriptor.id, "depth-estimation");
|
||||
assert.deepEqual(module.descriptor.scopes, ["depth:status", "depth:run"]);
|
||||
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
|
||||
"GET /depth-estimation/status",
|
||||
"POST /depth-estimation/run",
|
||||
]);
|
||||
module.dispose?.();
|
||||
});
|
||||
|
||||
test("depth input guard accepts image data URLs and rejects unsafe input", () => {
|
||||
assert.doesNotThrow(() => validateDepthDataUrl("data:image/png;base64,AQID"));
|
||||
assert.throws(
|
||||
() => validateDepthDataUrl("data:text/plain;base64,AQID"),
|
||||
(error: unknown) => error instanceof DepthRuntimeError && error.code === "depth_input_invalid" && error.statusCode === 400,
|
||||
);
|
||||
});
|
||||
|
||||
test("depth runtime paths honor explicit project and Python overrides", () => {
|
||||
const paths = resolveDepthRuntimePaths({
|
||||
CANVAS_PROJECT_ROOT: "C:\\depth-project",
|
||||
CANVAS_DEPTH_PYTHON: "C:\\depth-project\\venv\\python.exe",
|
||||
CANVAS_DEPTH_HF_HOME: "C:\\depth-cache",
|
||||
CANVAS_DEPTH_MODEL_ID: "example/depth-model",
|
||||
});
|
||||
assert.equal(paths.pythonPath, "C:\\depth-project\\venv\\python.exe");
|
||||
assert.equal(paths.hfHome, "C:\\depth-cache");
|
||||
assert.equal(paths.modelId, "example/depth-model");
|
||||
assert.equal(paths.configured, false);
|
||||
});
|
||||
36
canvas-agent/test/lineart-estimation.test.ts
Normal file
36
canvas-agent/test/lineart-estimation.test.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { createLineartEstimationHttpModule, LineartRuntimeError, resolveLineartRuntimePaths, validateLineartDataUrl } from "../src/modules/lineart-estimation-http.js";
|
||||
|
||||
test("lineart runtime module declares signed status and run routes", () => {
|
||||
const module = createLineartEstimationHttpModule();
|
||||
assert.equal(module.descriptor.id, "lineart-estimation");
|
||||
assert.deepEqual(module.descriptor.scopes, ["lineart:status", "lineart:run"]);
|
||||
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
|
||||
"GET /lineart-estimation/status",
|
||||
"POST /lineart-estimation/run",
|
||||
]);
|
||||
module.dispose?.();
|
||||
});
|
||||
|
||||
test("lineart input guard accepts image data URLs and rejects unsafe input", () => {
|
||||
assert.doesNotThrow(() => validateLineartDataUrl("data:image/png;base64,AQID"));
|
||||
assert.throws(
|
||||
() => validateLineartDataUrl("data:text/plain;base64,AQID"),
|
||||
(error: unknown) => error instanceof LineartRuntimeError && error.code === "lineart_input_invalid" && error.statusCode === 400,
|
||||
);
|
||||
});
|
||||
|
||||
test("lineart runtime paths honor explicit project and Python overrides", () => {
|
||||
const paths = resolveLineartRuntimePaths({
|
||||
CANVAS_PROJECT_ROOT: "C:\\lineart-project",
|
||||
CANVAS_DEPTH_PYTHON: "C:\\lineart-project\\venv\\python.exe",
|
||||
CANVAS_LINEART_HF_HOME: "C:\\lineart-cache",
|
||||
CANVAS_LINEART_MODEL_ID: "example/lineart-model",
|
||||
});
|
||||
assert.equal(paths.pythonPath, "C:\\lineart-project\\venv\\python.exe");
|
||||
assert.equal(paths.hfHome, "C:\\lineart-cache");
|
||||
assert.equal(paths.modelId, "example/lineart-model");
|
||||
assert.equal(paths.configured, false);
|
||||
});
|
||||
37
canvas-agent/test/pose-estimation.test.ts
Normal file
37
canvas-agent/test/pose-estimation.test.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { createPoseEstimationHttpModule, PoseRuntimeError, resolvePoseRuntimePaths, validatePoseDataUrl } from "../src/modules/pose-estimation-http.js";
|
||||
|
||||
test("pose runtime module declares signed status and run routes", () => {
|
||||
const module = createPoseEstimationHttpModule();
|
||||
assert.equal(module.descriptor.id, "pose-estimation");
|
||||
assert.deepEqual(module.descriptor.scopes, ["pose:status", "pose:run"]);
|
||||
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
|
||||
"GET /pose-estimation/status",
|
||||
"POST /pose-estimation/run",
|
||||
]);
|
||||
module.dispose?.();
|
||||
});
|
||||
|
||||
test("pose input guard accepts image data URLs and rejects unsafe input", () => {
|
||||
assert.doesNotThrow(() => validatePoseDataUrl("data:image/png;base64,AQID"));
|
||||
assert.throws(
|
||||
() => validatePoseDataUrl("data:text/plain;base64,AQID"),
|
||||
(error: unknown) => error instanceof PoseRuntimeError && error.code === "pose_input_invalid" && error.statusCode === 400,
|
||||
);
|
||||
});
|
||||
|
||||
test("pose runtime paths honor explicit project and Python overrides", () => {
|
||||
const paths = resolvePoseRuntimePaths({
|
||||
CANVAS_PROJECT_ROOT: "C:\\pose-project",
|
||||
CANVAS_POSE_PYTHON: "C:\\pose-project\\venv\\python.exe",
|
||||
CANVAS_POSE_HF_HOME: "C:\\pose-cache",
|
||||
CANVAS_POSE_MODEL_ID: "example/pose-model",
|
||||
});
|
||||
assert.equal(paths.pythonPath, "C:\\pose-project\\venv\\python.exe");
|
||||
assert.equal(paths.hfHome, "C:\\pose-cache");
|
||||
assert.equal(paths.modelId, "example/pose-model");
|
||||
assert.equal(paths.configured, false);
|
||||
assert.equal(paths.modelCached, false);
|
||||
});
|
||||
Reference in New Issue
Block a user