From b13d84e7e6c2c90284bc43dab8cbb54a696e950b Mon Sep 17 00:00:00 2001 From: QAyong Date: Tue, 8 Sep 2026 22:55:44 +0800 Subject: [PATCH] =?UTF-8?q?feat(canvas):=20=E5=9B=BE=E7=89=87=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=E6=8F=92=E4=BB=B6=20-=20=E5=A2=9E=E5=8A=A0=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E5=AA=92=E4=BD=93=E8=BD=AC=E6=8D=A2=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E4=B8=8E=E8=BF=90=E8=A1=8C=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/service/plugin_management.go | 13 +- canvas-agent/README.md | 52 +++ canvas-agent/package.json | 1 + canvas-agent/python/depth_anything_runtime.py | 134 +++++++ canvas-agent/python/lineart_runtime.py | 132 +++++++ canvas-agent/python/pose_runtime.py | 204 ++++++++++ canvas-agent/python/requirements-lineart.txt | 1 + canvas-agent/src/http-server.ts | 23 +- canvas-agent/src/local-runtime-contract.ts | 10 +- canvas-agent/src/local-runtime-host.ts | 6 + canvas-agent/src/local-runtime.ts | 2 +- .../src/modules/depth-estimation-http.ts | 327 ++++++++++++++++ .../src/modules/lineart-estimation-http.ts | 327 ++++++++++++++++ .../src/modules/pose-estimation-http.ts | 357 +++++++++++++++++ canvas-agent/test/depth-estimation.test.ts | 36 ++ canvas-agent/test/lineart-estimation.test.ts | 36 ++ canvas-agent/test/pose-estimation.test.ts | 37 ++ .../components/canvas/canvas-context-menu.tsx | 1 + .../components/canvas/canvas-node-content.tsx | 3 + .../canvas/canvas-node-generation.ts | 41 ++ .../components/canvas/canvas-node-toolbar.tsx | 6 +- web/src/components/canvas/canvas-node.tsx | 3 +- .../canvas/canvas-workspace-overlays.tsx | 3 +- .../canvas/nodes/media-conversion-node.tsx | 360 ++++++++++++++++++ web/src/constant/canvas.ts | 5 + web/src/lib/canvas/canvas-agent-ops.ts | 1 + .../lib/canvas/canvas-connection-policy.ts | 26 +- .../definitions/builtin-nodes.tsx | 20 +- web/src/lib/canvas/node-registry/index.ts | 2 +- .../canvas/node-registry/node-definition.ts | 7 +- .../lib/canvas/node-registry/node-registry.ts | 10 + .../definitions/add-node-menu-tools.tsx | 1 + .../definitions/node-hover-tools.tsx | 15 +- .../canvas/tool-registry/tool-definition.ts | 2 + web/src/lib/media-conversion/contracts.ts | 89 +++++ .../lib/media-conversion/local-converter.ts | 228 +++++++++++ web/src/lib/plugins/builtin/index.ts | 1 + .../lib/plugins/builtin/media-conversion.ts | 37 ++ web/src/lib/plugins/plugin-types.ts | 6 +- web/src/pages/admin/plugins/plugins-page.tsx | 3 +- web/src/pages/canvas/project.tsx | 23 +- .../use-canvas-connection-controller.ts | 15 +- .../canvas/use-canvas-node-operations.ts | 4 +- .../canvas/use-canvas-viewport-controller.ts | 2 +- web/src/pages/plugins/index.tsx | 3 +- web/src/services/depth-runtime.ts | 202 ++++++++++ web/src/services/lineart-runtime.ts | 202 ++++++++++ web/src/services/local-runtime.ts | 9 +- web/src/services/pose-runtime.ts | 209 ++++++++++ web/src/types/canvas.ts | 4 + web/test/media-conversion.test.ts | 87 +++++ 51 files changed, 3284 insertions(+), 44 deletions(-) create mode 100644 canvas-agent/python/depth_anything_runtime.py create mode 100644 canvas-agent/python/lineart_runtime.py create mode 100644 canvas-agent/python/pose_runtime.py create mode 100644 canvas-agent/python/requirements-lineart.txt create mode 100644 canvas-agent/src/modules/depth-estimation-http.ts create mode 100644 canvas-agent/src/modules/lineart-estimation-http.ts create mode 100644 canvas-agent/src/modules/pose-estimation-http.ts create mode 100644 canvas-agent/test/depth-estimation.test.ts create mode 100644 canvas-agent/test/lineart-estimation.test.ts create mode 100644 canvas-agent/test/pose-estimation.test.ts create mode 100644 web/src/components/canvas/nodes/media-conversion-node.tsx create mode 100644 web/src/lib/media-conversion/contracts.ts create mode 100644 web/src/lib/media-conversion/local-converter.ts create mode 100644 web/src/lib/plugins/builtin/media-conversion.ts create mode 100644 web/src/services/depth-runtime.ts create mode 100644 web/src/services/lineart-runtime.ts create mode 100644 web/src/services/pose-runtime.ts create mode 100644 web/test/media-conversion.test.ts diff --git a/backend/internal/service/plugin_management.go b/backend/internal/service/plugin_management.go index e88f30179..9b27c1848 100644 --- a/backend/internal/service/plugin_management.go +++ b/backend/internal/service/plugin_management.go @@ -13,6 +13,7 @@ const ( PluginPromptOptimizer = "prompt-optimizer" PluginPortraitClearance = "portrait-clearance" PluginAIArtCritique = "ai-art-critique" + PluginMediaConversion = "media-conversion" PluginOriginOfficial = "official" PluginOriginSystem = "system" @@ -80,6 +81,10 @@ var officialApplicationPolicies = map[string]PluginManagementView{ Origin: PluginOriginOfficial, Kind: PluginKindApplication, ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone, }, + PluginMediaConversion: { + Origin: PluginOriginOfficial, Kind: PluginKindApplication, + ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone, + }, } var systemPaymentPolicies = map[string]PluginManagementView{ @@ -196,8 +201,12 @@ func (s *Service) pluginStateForUser(actor *model.User, pluginID string, items [ // Preserve the old globally-enabled workflow behavior until each user // explicitly saves a personal choice. Other official applications were // already controlled by each user's local installation state. - if !userConfigured && hasRuntime && isLegacyWorkflowPlugin(pluginID) { - userEnabled = runtimePlugin.Status == "enabled" + if !userConfigured { + if pluginID == PluginMediaConversion { + userEnabled = true + } else if hasRuntime && isLegacyWorkflowPlugin(pluginID) { + userEnabled = runtimePlugin.Status == "enabled" + } } } effective := platformAvailable diff --git a/canvas-agent/README.md b/canvas-agent/README.md index 8e8ed83b5..a433dca44 100644 --- a/canvas-agent/README.md +++ b/canvas-agent/README.md @@ -76,6 +76,58 @@ Canvas Agent 内置 `portrait-clearance` Local Runtime 模块。它只接收签 本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`，安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时，模块返回 `portrait_model_missing`，不会伪造低风险结果。 +## Depth Anything V2 本机深度模块 + +画布“转换 → 深度图”使用 `depth-estimation` Local Runtime 模块。模块只接受签名的本机请求，Node Runtime 启动一个常驻 Python worker；模型首次调用时加载，后续转换复用同一进程。worker 以离线模式读取本地 Hugging Face 缓存，不会把图片发送到云端。 + +使用仓库根目录的 `start-yingce-local.cmd` 时，Canvas Agent 会和前端、后端一起自动启动，转换节点不需要用户另外打开 Agent 窗口或手工填写地址。只有手工分别启动前端和后端时，才需要在本目录执行 `node dist/index.js`。 + +当前仓库开发环境默认查找： + +```text +.local/depth-anything-v2/venv/Scripts/python.exe # Windows +.local/depth-anything-v2/venv/bin/python # Linux/macOS +.local/cache/huggingface/ # 模型缓存 +``` + +发布包会携带 `python/depth_anything_runtime.py`，但不会携带 Python、依赖或模型权重。自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_DEPTH_HF_HOME`；需要指定工作区时设置 `CANVAS_PROJECT_ROOT`。模型缺失、Python 环境缺失或 worker 退出时，转换节点显示具体失败状态并保留输入，不会把失败标记为完成。 + +## ControlNet Aux 本机 AI 线稿模块 + +画布“转换 → AI 线稿”使用 `lineart-estimation` Local Runtime 模块。它只运行 ControlNet Aux 的 `LineartDetector` 预处理器，把图片转换为线稿控制图；不会加载 Stable Diffusion、ControlNet 生图管线，也不会把图片上传到云端。Node Runtime 会复用同一个常驻 Python worker，首次转换加载模型，后续请求复用已加载模型。 + +该模块与深度模块共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 模型缓存。首次设置或缺少依赖时，在仓库根目录执行： + +```powershell +$python = ".local/depth-anything-v2/venv/Scripts/python.exe" +& $python -m pip install -r canvas-agent/python/requirements-lineart.txt +``` + +然后用下面的命令把线稿预处理器权重缓存到本地；下载完成后，正常转换会强制离线运行： + +```powershell +$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path +$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub" +& $python -c "from controlnet_aux import LineartDetector; LineartDetector.from_pretrained('lllyasviel/Annotators')" +``` + +自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_LINEART_HF_HOME` 和 `CANVAS_LINEART_MODEL_ID`；未找到 Python、依赖或模型时，节点显示“本地不可用”并保留输入，不会伪造已完成结果。 + +## ControlNet Aux 本机姿态骨架模块 + +画布“转换 → 姿态骨架”使用 `pose-estimation` Local Runtime 模块。当前笔记本方案采用 ControlNet Aux 的 **OpenPose body-only** 预处理器：只提取人体 18 点骨架并输出彩色姿态图，不加载 Stable Diffusion，也不上传图片。Node Runtime 会复用常驻 Python worker；未检测到人物时返回 `pose_no_person`，转换节点显示“已跳过”，不会保存伪造结果。 + +该模块与深度、线稿共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 缓存。安装 `requirements-lineart.txt` 后缓存人体权重： + +```powershell +$python = ".local/depth-anything-v2/venv/Scripts/python.exe" +$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path +$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub" +& $python -c "from huggingface_hub import hf_hub_download; hf_hub_download('lllyasviel/Annotators', 'body_pose_model.pth', cache_dir='$env:HF_HUB_CACHE')" +``` + +自定义安装位置时设置 `CANVAS_POSE_PYTHON`、`CANVAS_POSE_HF_HOME` 和 `CANVAS_POSE_MODEL_ID`。DWPose（包含手部、脸部和更完整关键点）仍可在后续按需替换，但它需要额外的 MMDetection/MMPose 依赖；当前实现优先保证普通 CPU 笔记本可运行。 + ## 发布 `canvas-agent` 使用自己的 `package.json` 版本号，不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。 diff --git a/canvas-agent/package.json b/canvas-agent/package.json index 0073cc7f1..9ac54c6cd 100644 --- a/canvas-agent/package.json +++ b/canvas-agent/package.json @@ -9,6 +9,7 @@ }, "files": [ "dist", + "python", "README.md" ], "scripts": { diff --git a/canvas-agent/python/depth_anything_runtime.py b/canvas-agent/python/depth_anything_runtime.py new file mode 100644 index 000000000..a5c389431 --- /dev/null +++ b/canvas-agent/python/depth_anything_runtime.py @@ -0,0 +1,134 @@ +"""Small persistent worker for the local Depth Anything V2 Small model. + +The Node Local Runtime owns the process and sends one JSON request per line. +Keeping the pipeline alive avoids loading the model for every canvas conversion. +The worker is deliberately offline-only: the model must already exist in the +configured Hugging Face cache. +""" + +from __future__ import annotations + +import base64 +import contextlib +import io +import json +import os +import re +import sys +from typing import Any + +from PIL import Image + + +MODEL_ID = os.environ.get("CANVAS_DEPTH_MODEL_ID", "depth-anything/Depth-Anything-V2-Small-hf") +MAX_INPUT_BYTES = 12 * 1024 * 1024 +DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$") +PIPELINE: Any = None + + +def emit(value: dict[str, Any]) -> None: + sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n") + sys.stdout.flush() + + +def load_pipeline() -> Any: + global PIPELINE + if PIPELINE is not None: + return PIPELINE + + try: + from transformers import pipeline + from transformers.utils import logging as transformers_logging + + transformers_logging.set_verbosity_error() + # Transformers/tqdm can write progress text while loading weights. + # Keep stdout reserved for the JSON-lines protocol. + with contextlib.redirect_stdout(sys.stderr): + PIPELINE = pipeline( + "depth-estimation", + model=MODEL_ID, + device=-1, + local_files_only=True, + ) + except Exception as error: # pragma: no cover - depends on local model files + raise RuntimeError("depth_model_missing") from error + return PIPELINE + + +def image_from_data_url(value: Any) -> Image.Image: + if not isinstance(value, str): + raise ValueError("depth_input_invalid") + match = DATA_URL_PATTERN.fullmatch(value) + if not match: + raise ValueError("depth_input_invalid") + try: + raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4)) + except Exception as error: + raise ValueError("depth_input_invalid") from error + if not raw or len(raw) > MAX_INPUT_BYTES: + raise ValueError("depth_input_invalid") + try: + image = Image.open(io.BytesIO(raw)).convert("RGB") + image.load() + return image + except Exception as error: + raise ValueError("depth_input_invalid") from error + + +def encode_png(image: Image.Image) -> str: + output = io.BytesIO() + image.save(output, format="PNG", optimize=True) + return base64.b64encode(output.getvalue()).decode("ascii") + + +def handle(request: dict[str, Any]) -> dict[str, Any]: + image = image_from_data_url(request.get("dataUrl")) + try: + result = load_pipeline()(image) + depth = result.get("depth") if isinstance(result, dict) else None + if not isinstance(depth, Image.Image): + raise RuntimeError("depth_inference_failed") + return { + "ok": True, + "requestId": request.get("requestId"), + "mimeType": "image/png", + "width": depth.width, + "height": depth.height, + "modelId": MODEL_ID, + "device": "cpu", + "pngBase64": encode_png(depth), + } + except RuntimeError: + raise + except Exception as error: # pragma: no cover - depends on local runtime + raise RuntimeError("depth_inference_failed") from error + + +def main() -> None: + os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") + os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1") + os.environ.setdefault("HF_HUB_OFFLINE", "1") + os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") + + for line in sys.stdin: + line = line.strip() + if not line: + continue + request_id: Any = None + try: + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("depth_input_invalid") + request_id = request.get("requestId") + emit(handle(request)) + except ValueError as error: + emit({"ok": False, "requestId": request_id, "code": str(error) or "depth_input_invalid", "message": "深度转换输入无效"}) + except RuntimeError as error: + code = str(error) or "depth_inference_failed" + emit({"ok": False, "requestId": request_id, "code": code, "message": "本机深度模型不可用" if code == "depth_model_missing" else "本机深度推理失败"}) + except Exception: + emit({"ok": False, "requestId": request_id, "code": "depth_inference_failed", "message": "本机深度推理失败"}) + + +if __name__ == "__main__": + main() diff --git a/canvas-agent/python/lineart_runtime.py b/canvas-agent/python/lineart_runtime.py new file mode 100644 index 000000000..5474719d0 --- /dev/null +++ b/canvas-agent/python/lineart_runtime.py @@ -0,0 +1,132 @@ +"""Small persistent worker for the local ControlNet Aux lineart detector. + +This worker only extracts a lineart control image. It does not load Stable +Diffusion or a ControlNet generation pipeline, so it stays suitable for a +CPU-only notebook. The Node Local Runtime owns the process and sends one JSON +request per line; stdout is reserved for that protocol. +""" + +from __future__ import annotations + +import base64 +import contextlib +import io +import json +import os +import re +import sys +from typing import Any + +from PIL import Image + + +MODEL_ID = os.environ.get("CANVAS_LINEART_MODEL_ID", "lllyasviel/Annotators") +MAX_INPUT_BYTES = 12 * 1024 * 1024 +DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$") +DETECTOR: Any = None + + +def emit(value: dict[str, Any]) -> None: + sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n") + sys.stdout.flush() + + +def load_detector() -> Any: + global DETECTOR + if DETECTOR is not None: + return DETECTOR + + try: + from controlnet_aux import LineartDetector + + # The detector downloads only the lineart preprocessor weights from the + # configured Hugging Face cache. It is intentionally not a diffusion + # pipeline and never loads Stable Diffusion or ControlNet weights. + with contextlib.redirect_stdout(sys.stderr): + DETECTOR = LineartDetector.from_pretrained(MODEL_ID) + except Exception as error: # pragma: no cover - depends on local files + raise RuntimeError("lineart_model_missing") from error + return DETECTOR + + +def image_from_data_url(value: Any) -> Image.Image: + if not isinstance(value, str): + raise ValueError("lineart_input_invalid") + match = DATA_URL_PATTERN.fullmatch(value) + if not match: + raise ValueError("lineart_input_invalid") + try: + raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4)) + except Exception as error: + raise ValueError("lineart_input_invalid") from error + if not raw or len(raw) > MAX_INPUT_BYTES: + raise ValueError("lineart_input_invalid") + try: + image = Image.open(io.BytesIO(raw)).convert("RGB") + image.load() + return image + except Exception as error: + raise ValueError("lineart_input_invalid") from error + + +def encode_png(image: Image.Image) -> str: + output = io.BytesIO() + image.save(output, format="PNG", optimize=True) + return base64.b64encode(output.getvalue()).decode("ascii") + + +def handle(request: dict[str, Any]) -> dict[str, Any]: + image = image_from_data_url(request.get("dataUrl")) + try: + lineart = load_detector()(image) + if not isinstance(lineart, Image.Image): + raise RuntimeError("lineart_inference_failed") + return { + "ok": True, + "requestId": request.get("requestId"), + "mimeType": "image/png", + "width": lineart.width, + "height": lineart.height, + "modelId": MODEL_ID, + "device": "cpu", + "pngBase64": encode_png(lineart.convert("RGB")), + } + except RuntimeError: + raise + except Exception as error: # pragma: no cover - depends on local runtime + raise RuntimeError("lineart_inference_failed") from error + + +def main() -> None: + os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") + os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1") + os.environ.setdefault("HF_HUB_OFFLINE", "1") + os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") + + for line in sys.stdin: + # PowerShell and a few Windows launchers may prepend a UTF-8 BOM to + # the first piped line. The Node runtime sends plain UTF-8, but + # accepting the marker keeps the worker protocol tolerant for manual + # diagnostics as well. + line = line.lstrip("\ufeff").strip() + if not line: + continue + request_id: Any = None + try: + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("lineart_input_invalid") + request_id = request.get("requestId") + emit(handle(request)) + except ValueError as error: + emit({"ok": False, "requestId": request_id, "code": str(error) or "lineart_input_invalid", "message": "线稿转换输入无效"}) + except RuntimeError as error: + code = str(error) or "lineart_inference_failed" + message = "本地 AI 线稿模型或依赖尚未安装" if code == "lineart_model_missing" else "本地 AI 线稿推理失败" + emit({"ok": False, "requestId": request_id, "code": code, "message": message}) + except Exception: + emit({"ok": False, "requestId": request_id, "code": "lineart_inference_failed", "message": "本地 AI 线稿推理失败"}) + + +if __name__ == "__main__": + main() diff --git a/canvas-agent/python/pose_runtime.py b/canvas-agent/python/pose_runtime.py new file mode 100644 index 000000000..16baa73b0 --- /dev/null +++ b/canvas-agent/python/pose_runtime.py @@ -0,0 +1,204 @@ +"""Persistent CPU pose preprocessor for the local media conversion node. + +The first notebook-friendly implementation uses the body-only OpenPose +preprocessor from ControlNet Aux. It produces a colored skeleton map and a +small list of normalized keypoints; it does not load a diffusion model. +""" + +from __future__ import annotations + +import base64 +import contextlib +import io +import json +import os +import re +import sys +from typing import Any + +from PIL import Image + + +MODEL_ID = os.environ.get("CANVAS_POSE_MODEL_ID", "lllyasviel/Annotators") +MAX_INPUT_BYTES = 12 * 1024 * 1024 +DETECT_RESOLUTION = max(256, min(1024, int(os.environ.get("CANVAS_POSE_DETECT_RESOLUTION", "512")))) +DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$") +DETECTOR: Any = None + + +def emit(value: dict[str, Any]) -> None: + sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n") + sys.stdout.flush() + + +def load_detector() -> Any: + global DETECTOR + if DETECTOR is not None: + return DETECTOR + + try: + import numpy as np + from controlnet_aux.open_pose.body import Body + from huggingface_hub import hf_hub_download + + cache_dir = os.environ.get("HF_HUB_CACHE") or None + with contextlib.redirect_stdout(sys.stderr): + model_path = hf_hub_download( + repo_id=MODEL_ID, + filename="body_pose_model.pth", + cache_dir=cache_dir, + local_files_only=True, + ) + DETECTOR = (Body(model_path), np) + except Exception as error: # pragma: no cover - depends on local model files + raise RuntimeError("pose_model_missing") from error + return DETECTOR + + +def image_from_data_url(value: Any) -> Image.Image: + if not isinstance(value, str): + raise ValueError("pose_input_invalid") + match = DATA_URL_PATTERN.fullmatch(value) + if not match: + raise ValueError("pose_input_invalid") + try: + raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4)) + except Exception as error: + raise ValueError("pose_input_invalid") from error + if not raw or len(raw) > MAX_INPUT_BYTES: + raise ValueError("pose_input_invalid") + try: + image = Image.open(io.BytesIO(raw)).convert("RGB") + image.load() + return image + except Exception as error: + raise ValueError("pose_input_invalid") from error + + +def encode_png(image: Image.Image) -> str: + output = io.BytesIO() + image.save(output, format="PNG", optimize=True) + return base64.b64encode(output.getvalue()).decode("ascii") + + +def normalize_people(bodies: list[Any], width: int, height: int) -> list[dict[str, Any]]: + people: list[dict[str, Any]] = [] + for body in bodies: + keypoints = [] + visible = [] + for keypoint in body.keypoints: + if keypoint is None: + keypoints.append(None) + else: + normalized_x = max(0.0, min(1.0, float(keypoint.x) / float(width))) + normalized_y = max(0.0, min(1.0, float(keypoint.y) / float(height))) + keypoints.append({ + "x": normalized_x, + "y": normalized_y, + "score": float(keypoint.score), + }) + visible.append((normalized_x, normalized_y, float(keypoint.score))) + if visible: + xs = [point[0] for point in visible] + ys = [point[1] for point in visible] + confidence = sum(point[2] for point in visible) / len(visible) + box = {"x": min(xs), "y": min(ys), "width": max(xs) - min(xs), "height": max(ys) - min(ys)} + else: + confidence = 0.0 + box = {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0} + people.append({ + "keypoints": keypoints, + "score": float(body.total_score), + "confidence": confidence, + "box": box, + "parts": int(body.total_parts), + }) + return people + + +def handle(request: dict[str, Any]) -> dict[str, Any]: + image = image_from_data_url(request.get("dataUrl")) + try: + import cv2 + import numpy as np + from controlnet_aux.open_pose.util import draw_bodypose + from controlnet_aux.util import HWC3, resize_image + + detector, _ = load_detector() + source = HWC3(np.asarray(image, dtype=np.uint8)) + detected = resize_image(source, DETECT_RESOLUTION) + candidate, subset = detector(detected[:, :, ::-1].copy()) + bodies = detector.format_body_result(candidate, subset) + if not bodies: + raise RuntimeError("pose_no_person") + + height, width = detected.shape[:2] + people = normalize_people(bodies, width, height) + canvas = np.zeros_like(detected, dtype=np.uint8) + for body in bodies: + normalized = [ + None if keypoint is None else type(keypoint)( + x=float(keypoint.x) / float(width), + y=float(keypoint.y) / float(height), + score=keypoint.score, + id=keypoint.id, + ) + for keypoint in body.keypoints + ] + draw_bodypose(canvas, normalized) + + output = cv2.resize(canvas, (image.width, image.height), interpolation=cv2.INTER_LINEAR) + output_image = Image.fromarray(output.astype(np.uint8), mode="RGB") + return { + "ok": True, + "requestId": request.get("requestId"), + "mimeType": "image/png", + "width": image.width, + "height": image.height, + "modelId": MODEL_ID, + "device": "cpu", + "backend": "openpose-body", + "personCount": len(bodies), + "people": people, + "pngBase64": encode_png(output_image), + } + except RuntimeError: + raise + except Exception as error: # pragma: no cover - depends on local runtime + raise RuntimeError("pose_inference_failed") from error + + +def main() -> None: + os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") + os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1") + os.environ.setdefault("HF_HUB_OFFLINE", "1") + os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") + + for line in sys.stdin: + line = line.lstrip("\ufeff").strip() + if not line: + continue + request_id: Any = None + try: + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("pose_input_invalid") + request_id = request.get("requestId") + emit(handle(request)) + except ValueError as error: + emit({"ok": False, "requestId": request_id, "code": str(error) or "pose_input_invalid", "message": "姿态转换输入无效"}) + except RuntimeError as error: + code = str(error) or "pose_inference_failed" + if code == "pose_model_missing": + message = "本地姿态模型或依赖尚未安装" + elif code == "pose_no_person": + message = "未检测到可用人物姿态" + else: + message = "本地姿态推理失败" + emit({"ok": False, "requestId": request_id, "code": code, "message": message}) + except Exception: + emit({"ok": False, "requestId": request_id, "code": "pose_inference_failed", "message": "本地姿态推理失败"}) + + +if __name__ == "__main__": + main() diff --git a/canvas-agent/python/requirements-lineart.txt b/canvas-agent/python/requirements-lineart.txt new file mode 100644 index 000000000..ad37a231d --- /dev/null +++ b/canvas-agent/python/requirements-lineart.txt @@ -0,0 +1 @@ +controlnet-aux==0.0.10 diff --git a/canvas-agent/src/http-server.ts b/canvas-agent/src/http-server.ts index 02397a569..0b4a381cd 100644 --- a/canvas-agent/src/http-server.ts +++ b/canvas-agent/src/http-server.ts @@ -4,7 +4,7 @@ import { CanvasSession } from "./canvas-session.js"; import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js"; import { createLocalRuntimeApp } from "./local-runtime.js"; import { startLocalRuntime } from "./local-runtime-host.js"; -import { LocalRuntimeSessionManager } from "./local-runtime-session.js"; +import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./local-runtime-session.js"; import { createCanvasAgentHttpModule, type CanvasAgentSession, @@ -14,6 +14,9 @@ import { type DreaminaHttpModuleOptions, } from "./modules/dreamina-http.js"; import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js"; +import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js"; +import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js"; +import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js"; export type CanvasAgentHttpDependencies = Pick; @@ -34,21 +37,29 @@ export function createHttpApp( dependencies: CanvasAgentHttpDependencies = {}, ): Express { const endpoint = config.url; + const modules = [ + createCanvasAgentHttpModule(config, session), + createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }), + createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }), + createDepthEstimationHttpModule(), + createLineartEstimationHttpModule(), + createPoseEstimationHttpModule(), + ]; const manager = new LocalRuntimeSessionManager({ endpoint, trustedOrigins: config.trustedWebOrigins, registrations: config.browserRegistrations, + scopes: [...new Set([ + ...LOCAL_RUNTIME_DEFAULT_SCOPES, + ...modules.flatMap((module) => module.descriptor.scopes), + ])], }); return createLocalRuntimeApp({ authority: new URL(endpoint).host, endpoint, version: "0.1.0", sessionManager: manager, - modules: [ - createCanvasAgentHttpModule(config, session), - createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }), - createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }), - ], + modules, legacyMasterToken: config.token, legacyOrigins: config.origins ?? [], }); diff --git a/canvas-agent/src/local-runtime-contract.ts b/canvas-agent/src/local-runtime-contract.ts index c17521b32..fe3559dae 100644 --- a/canvas-agent/src/local-runtime-contract.ts +++ b/canvas-agent/src/local-runtime-contract.ts @@ -6,7 +6,7 @@ export const LOCAL_RUNTIME_ENDPOINT = "http://127.0.0.1:17371"; export const LOCAL_RUNTIME_ID = "framefield-local-runtime"; export const LOCAL_RUNTIME_API_VERSION = 2; -export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance"; +export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation"; export type LocalRuntimeScope = | "runtime:status" | "runtime:revoke" @@ -20,7 +20,13 @@ export type LocalRuntimeScope = | "portrait:status" | "portrait:model" | "portrait:run" - | "portrait:read"; + | "portrait:read" + | "depth:status" + | "depth:run" + | "lineart:status" + | "lineart:run" + | "pose:status" + | "pose:run"; export type DreaminaModelOperation = | "text-to-image" diff --git a/canvas-agent/src/local-runtime-host.ts b/canvas-agent/src/local-runtime-host.ts index 1ce839336..8b11f6af8 100644 --- a/canvas-agent/src/local-runtime-host.ts +++ b/canvas-agent/src/local-runtime-host.ts @@ -17,6 +17,9 @@ import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./loca import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js"; import { createDreaminaHttpModule } from "./modules/dreamina-http.js"; import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js"; +import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js"; +import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js"; +import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js"; export type StartLocalRuntimeOptions = { config?: LocalRuntimeConfig; @@ -46,6 +49,9 @@ export function createDefaultLocalRuntimeModules(config: LocalRuntimeConfig): Lo ], }), createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }), + createDepthEstimationHttpModule(), + createLineartEstimationHttpModule(), + createPoseEstimationHttpModule(), ]; } diff --git a/canvas-agent/src/local-runtime.ts b/canvas-agent/src/local-runtime.ts index 65f08d029..629bb100e 100644 --- a/canvas-agent/src/local-runtime.ts +++ b/canvas-agent/src/local-runtime.ts @@ -185,7 +185,7 @@ function validateModules(modules: readonly LocalRuntimeModule[]) { const ids = new Set(); const routes = new Set(); for (const module of modules) { - if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance") { + if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance" && module.descriptor.id !== "depth-estimation" && module.descriptor.id !== "lineart-estimation" && module.descriptor.id !== "pose-estimation") { throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`); } if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`); diff --git a/canvas-agent/src/modules/depth-estimation-http.ts b/canvas-agent/src/modules/depth-estimation-http.ts new file mode 100644 index 000000000..8005ad32b --- /dev/null +++ b/canvas-agent/src/modules/depth-estimation-http.ts @@ -0,0 +1,327 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; + +import type { Request, RequestHandler, Response } from "express"; + +import type { LocalRuntimeModule } from "../local-runtime.js"; + +const DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf"; +const MAX_INPUT_BYTES = 12 * 1024 * 1024; +const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024; +const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/; + +export type DepthRuntimePaths = { + pythonPath: string; + scriptPath: string; + hfHome: string; + modelId: string; + configured: boolean; + modelCached: boolean; + reason?: "python_missing" | "worker_missing"; +}; + +export type DepthRuntimeStatus = { + configured: boolean; + modelCached: boolean; + loaded: boolean; + device: "cpu"; + modelId: string; + reason?: string; +}; + +type DepthWorkerSuccess = { + ok: true; + requestId: string; + mimeType: "image/png"; + width: number; + height: number; + modelId: string; + device: "cpu"; + pngBase64: string; +}; + +export class DepthRuntimeError extends Error { + constructor(readonly code: string, message: string, readonly statusCode = 503) { + super(message); + this.name = "DepthRuntimeError"; + } +} + +export function createDepthEstimationHttpModule(): LocalRuntimeModule { + const worker = new DepthEstimationWorker(); + return { + descriptor: { + id: "depth-estimation", + displayName: "Depth Anything V2 本机深度", + apiVersion: 1, + scopes: ["depth:status", "depth:run"], + }, + routes: [ + { + method: "GET", + path: "/depth-estimation/status", + scope: "depth:status", + handler: asyncRoute(async (_request, response) => { + const status = worker.status(); + response.json({ + ok: true, + module: "depth-estimation", + apiVersion: 1, + ready: status.configured && status.modelCached, + ...status, + }); + }), + }, + { + method: "POST", + path: "/depth-estimation/run", + scope: "depth:run", + handler: asyncRoute(async (request, response) => { + const body = parseBody(request); + const dataUrl = parseDepthInput(body); + const result = await worker.run({ dataUrl }); + response.json({ + ok: true, + module: "depth-estimation", + apiVersion: 1, + result, + }); + }), + }, + ], + dispose: () => worker.dispose(), + publicHealth: () => { + const status = worker.status(); + return { depthEstimation: status.configured && status.modelCached ? "available" : "unavailable" }; + }, + }; +} + +export function resolveDepthRuntimePaths(env: NodeJS.ProcessEnv = process.env): DepthRuntimePaths { + const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim(); + const projectRoots = [ + configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined, + path.resolve(moduleRoot, ".."), + path.resolve(process.cwd()), + ].filter((value): value is string => Boolean(value)); + const projectRoot = configuredProjectRoot + ? path.resolve(configuredProjectRoot) + : projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!; + const scriptCandidates = [ + path.join(moduleRoot, "python", "depth_anything_runtime.py"), + path.join(projectRoot, "canvas-agent", "python", "depth_anything_runtime.py"), + ]; + const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!; + const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim() + ? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())] + : process.platform === "win32" + ? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")] + : [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")]; + const pythonPath = pythonCandidates[0]!; + const hfHome = path.resolve(env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface")); + const modelId = env.CANVAS_DEPTH_MODEL_ID?.trim() || DEFAULT_MODEL_ID; + const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`); + const workerExists = fs.existsSync(scriptPath); + const pythonExists = fs.existsSync(pythonPath); + return { + pythonPath, + scriptPath, + hfHome, + modelId, + configured: workerExists && pythonExists, + modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir), + ...(pythonExists ? {} : { reason: "python_missing" as const }), + ...(!workerExists ? { reason: "worker_missing" as const } : {}), + }; +} + +export class DepthEstimationWorker { + private paths = resolveDepthRuntimePaths(); + private child?: ChildProcessWithoutNullStreams; + private stdoutBuffer = ""; + private stderrBuffer = ""; + private requestCounter = 0; + private loaded = false; + private readonly pending = new Map void; reject: (error: unknown) => void }>(); + + status(): DepthRuntimeStatus { + const paths = this.refreshPaths(); + return { + configured: paths.configured, + modelCached: paths.modelCached, + loaded: this.loaded, + device: "cpu", + modelId: paths.modelId, + ...(paths.reason ? { reason: paths.reason } : {}), + }; + } + + async run(input: { dataUrl: string }) { + validateDepthDataUrl(input.dataUrl); + const paths = this.refreshPaths(); + if (!paths.configured) { + throw new DepthRuntimeError( + "depth_runtime_unavailable", + paths.reason === "worker_missing" ? "本机深度运行文件不存在" : "未找到本机深度 Python 环境", + ); + } + const child = this.ensureChild(); + const requestId = `depth-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`; + const response = new Promise((resolve, reject) => { + this.pending.set(requestId, { resolve, reject }); + }); + try { + child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n"); + } catch (error) { + this.pending.delete(requestId); + throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法写入"); + } + return response; + } + + dispose() { + const child = this.child; + this.child = undefined; + this.loaded = false; + this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已停止")); + if (child && !child.killed) child.kill(); + } + + private ensureChild() { + if (this.child && !this.child.killed) return this.child; + let child: ChildProcessWithoutNullStreams; + try { + child = spawn(this.paths.pythonPath, [this.paths.scriptPath], { + cwd: path.dirname(this.paths.scriptPath), + env: { + ...process.env, + HF_HOME: this.paths.hfHome, + HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"), + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + HF_HUB_DISABLE_TELEMETRY: "1", + HF_HUB_DISABLE_PROGRESS_BARS: "1", + TRANSFORMERS_VERBOSITY: "error", + CANVAS_DEPTH_MODEL_ID: this.paths.modelId, + }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + } catch (error) { + throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法启动"); + } + this.child = child; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk)); + child.stderr.on("data", (chunk: string) => { + this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000); + }); + child.on("error", () => this.handleChildExit(child)); + child.on("exit", () => this.handleChildExit(child)); + return child; + } + + private refreshPaths() { + if (!this.child) this.paths = resolveDepthRuntimePaths(); + return this.paths; + } + + private consumeStdout(chunk: string) { + this.stdoutBuffer += chunk; + if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) { + this.child?.kill(); + this.handleChildExit(this.child); + return; + } + while (true) { + const newline = this.stdoutBuffer.indexOf("\n"); + if (newline < 0) return; + const line = this.stdoutBuffer.slice(0, newline).trim(); + this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1); + if (!line) continue; + let value: unknown; + try { value = JSON.parse(line); } catch { continue; } + if (!isRecord(value) || typeof value.requestId !== "string") continue; + const pending = this.pending.get(value.requestId); + if (!pending) continue; + this.pending.delete(value.requestId); + if (value.ok === true) { + if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) { + pending.reject(new DepthRuntimeError("depth_response_invalid", "本机深度响应无效", 500)); + continue; + } + this.loaded = true; + pending.resolve(value as unknown as DepthWorkerSuccess); + continue; + } + const code = typeof value.code === "string" ? value.code : "depth_inference_failed"; + const statusCode = code === "depth_model_missing" ? 503 : code === "depth_input_invalid" ? 400 : 500; + pending.reject(new DepthRuntimeError(code, typeof value.message === "string" ? value.message : "本机深度推理失败", statusCode)); + } + } + + private handleChildExit(child?: ChildProcessWithoutNullStreams) { + if (!child || this.child !== child) return; + this.child = undefined; + this.loaded = false; + this.stderrBuffer = ""; + this.stdoutBuffer = ""; + this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已退出")); + } + + private rejectPending(error: DepthRuntimeError) { + for (const { reject } of this.pending.values()) reject(error); + this.pending.clear(); + } +} + +export function validateDepthDataUrl(value: string) { + const match = DATA_URL_PATTERN.exec(value); + if (!match) throw new DepthRuntimeError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400); + const bytes = Buffer.from(match[2]!, "base64"); + if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new DepthRuntimeError("depth_input_invalid", "深度转换图片不能超过 12MB", 400); +} + +function parseDepthInput(body: Record) { + const keys = Object.keys(body).sort(); + if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "depth" || typeof body.dataUrl !== "string") { + throw new DepthRuntimeError("depth_input_invalid", "深度转换请求字段无效", 400); + } + validateDepthDataUrl(body.dataUrl); + return body.dataUrl; +} + +function parseBody(request: Request): Record { + if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {}; + try { + const value = JSON.parse(request.body.toString("utf8")) as unknown; + return isRecord(value) ? value : {}; + } catch { + return {}; + } +} + +function asyncRoute(action: (request: Request, response: Response) => Promise): RequestHandler { + return (request, response, next) => { + void action(request, response).catch((error) => { + if (response.headersSent) return next(error); + if (error instanceof DepthRuntimeError) { + response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message }); + return; + } + response.status(503).json({ ok: false, code: "depth_runtime_unavailable", message: "本机深度运行时不可用" }); + }); + }; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0; +} diff --git a/canvas-agent/src/modules/lineart-estimation-http.ts b/canvas-agent/src/modules/lineart-estimation-http.ts new file mode 100644 index 000000000..d6fbee882 --- /dev/null +++ b/canvas-agent/src/modules/lineart-estimation-http.ts @@ -0,0 +1,327 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; + +import type { Request, RequestHandler, Response } from "express"; + +import type { LocalRuntimeModule } from "../local-runtime.js"; + +const DEFAULT_MODEL_ID = "lllyasviel/Annotators"; +const MAX_INPUT_BYTES = 12 * 1024 * 1024; +const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024; +const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/; + +export type LineartRuntimePaths = { + pythonPath: string; + scriptPath: string; + hfHome: string; + modelId: string; + configured: boolean; + modelCached: boolean; + reason?: "python_missing" | "worker_missing"; +}; + +export type LineartRuntimeStatus = { + configured: boolean; + modelCached: boolean; + loaded: boolean; + device: "cpu"; + modelId: string; + reason?: string; +}; + +type LineartWorkerSuccess = { + ok: true; + requestId: string; + mimeType: "image/png"; + width: number; + height: number; + modelId: string; + device: "cpu"; + pngBase64: string; +}; + +export class LineartRuntimeError extends Error { + constructor(readonly code: string, message: string, readonly statusCode = 503) { + super(message); + this.name = "LineartRuntimeError"; + } +} + +export function createLineartEstimationHttpModule(): LocalRuntimeModule { + const worker = new LineartEstimationWorker(); + return { + descriptor: { + id: "lineart-estimation", + displayName: "ControlNet Aux 本机 AI 线稿", + apiVersion: 1, + scopes: ["lineart:status", "lineart:run"], + }, + routes: [ + { + method: "GET", + path: "/lineart-estimation/status", + scope: "lineart:status", + handler: asyncRoute(async (_request, response) => { + const status = worker.status(); + response.json({ + ok: true, + module: "lineart-estimation", + apiVersion: 1, + ready: status.configured && status.modelCached, + ...status, + }); + }), + }, + { + method: "POST", + path: "/lineart-estimation/run", + scope: "lineart:run", + handler: asyncRoute(async (request, response) => { + const body = parseBody(request); + const dataUrl = parseLineartInput(body); + const result = await worker.run({ dataUrl }); + response.json({ + ok: true, + module: "lineart-estimation", + apiVersion: 1, + result, + }); + }), + }, + ], + dispose: () => worker.dispose(), + publicHealth: () => { + const status = worker.status(); + return { lineartEstimation: status.configured && status.modelCached ? "available" : "unavailable" }; + }, + }; +} + +export function resolveLineartRuntimePaths(env: NodeJS.ProcessEnv = process.env): LineartRuntimePaths { + const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim(); + const projectRoots = [ + configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined, + path.resolve(moduleRoot, ".."), + path.resolve(process.cwd()), + ].filter((value): value is string => Boolean(value)); + const projectRoot = configuredProjectRoot + ? path.resolve(configuredProjectRoot) + : projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!; + const scriptCandidates = [ + path.join(moduleRoot, "python", "lineart_runtime.py"), + path.join(projectRoot, "canvas-agent", "python", "lineart_runtime.py"), + ]; + const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!; + const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim() + ? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())] + : process.platform === "win32" + ? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")] + : [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")]; + const pythonPath = pythonCandidates[0]!; + const hfHome = path.resolve(env.CANVAS_LINEART_HF_HOME?.trim() || env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface")); + const modelId = env.CANVAS_LINEART_MODEL_ID?.trim() || DEFAULT_MODEL_ID; + const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`); + const workerExists = fs.existsSync(scriptPath); + const pythonExists = fs.existsSync(pythonPath); + return { + pythonPath, + scriptPath, + hfHome, + modelId, + configured: workerExists && pythonExists, + modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir), + ...(pythonExists ? {} : { reason: "python_missing" as const }), + ...(!workerExists ? { reason: "worker_missing" as const } : {}), + }; +} + +export class LineartEstimationWorker { + private paths = resolveLineartRuntimePaths(); + private child?: ChildProcessWithoutNullStreams; + private stdoutBuffer = ""; + private stderrBuffer = ""; + private requestCounter = 0; + private loaded = false; + private readonly pending = new Map void; reject: (error: unknown) => void }>(); + + status(): LineartRuntimeStatus { + const paths = this.refreshPaths(); + return { + configured: paths.configured, + modelCached: paths.modelCached, + loaded: this.loaded, + device: "cpu", + modelId: paths.modelId, + ...(paths.reason ? { reason: paths.reason } : {}), + }; + } + + async run(input: { dataUrl: string }) { + validateLineartDataUrl(input.dataUrl); + const paths = this.refreshPaths(); + if (!paths.configured) { + throw new LineartRuntimeError( + "lineart_runtime_unavailable", + paths.reason === "worker_missing" ? "本机 AI 线稿运行文件不存在" : "未找到本机 AI 线稿 Python 环境", + ); + } + const child = this.ensureChild(); + const requestId = `lineart-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`; + const response = new Promise((resolve, reject) => { + this.pending.set(requestId, { resolve, reject }); + }); + try { + child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n"); + } catch { + this.pending.delete(requestId); + throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法写入"); + } + return response; + } + + dispose() { + const child = this.child; + this.child = undefined; + this.loaded = false; + this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已停止")); + if (child && !child.killed) child.kill(); + } + + private ensureChild() { + if (this.child && !this.child.killed) return this.child; + let child: ChildProcessWithoutNullStreams; + try { + child = spawn(this.paths.pythonPath, [this.paths.scriptPath], { + cwd: path.dirname(this.paths.scriptPath), + env: { + ...process.env, + HF_HOME: this.paths.hfHome, + HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"), + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + HF_HUB_DISABLE_TELEMETRY: "1", + HF_HUB_DISABLE_PROGRESS_BARS: "1", + TRANSFORMERS_VERBOSITY: "error", + CANVAS_LINEART_MODEL_ID: this.paths.modelId, + }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + } catch { + throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法启动"); + } + this.child = child; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk)); + child.stderr.on("data", (chunk: string) => { + this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000); + }); + child.on("error", () => this.handleChildExit(child)); + child.on("exit", () => this.handleChildExit(child)); + return child; + } + + private refreshPaths() { + if (!this.child) this.paths = resolveLineartRuntimePaths(); + return this.paths; + } + + private consumeStdout(chunk: string) { + this.stdoutBuffer += chunk; + if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) { + this.child?.kill(); + this.handleChildExit(this.child); + return; + } + while (true) { + const newline = this.stdoutBuffer.indexOf("\n"); + if (newline < 0) return; + const line = this.stdoutBuffer.slice(0, newline).trim(); + this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1); + if (!line) continue; + let value: unknown; + try { value = JSON.parse(line); } catch { continue; } + if (!isRecord(value) || typeof value.requestId !== "string") continue; + const pending = this.pending.get(value.requestId); + if (!pending) continue; + this.pending.delete(value.requestId); + if (value.ok === true) { + if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) { + pending.reject(new LineartRuntimeError("lineart_response_invalid", "本机 AI 线稿响应无效", 500)); + continue; + } + this.loaded = true; + pending.resolve(value as unknown as LineartWorkerSuccess); + continue; + } + const code = typeof value.code === "string" ? value.code : "lineart_inference_failed"; + const statusCode = code === "lineart_model_missing" ? 503 : code === "lineart_input_invalid" ? 400 : 500; + pending.reject(new LineartRuntimeError(code, typeof value.message === "string" ? value.message : "本机 AI 线稿推理失败", statusCode)); + } + } + + private handleChildExit(child?: ChildProcessWithoutNullStreams) { + if (!child || this.child !== child) return; + this.child = undefined; + this.loaded = false; + this.stderrBuffer = ""; + this.stdoutBuffer = ""; + this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已退出")); + } + + private rejectPending(error: LineartRuntimeError) { + for (const { reject } of this.pending.values()) reject(error); + this.pending.clear(); + } +} + +export function validateLineartDataUrl(value: string) { + const match = DATA_URL_PATTERN.exec(value); + if (!match) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400); + const bytes = Buffer.from(match[2]!, "base64"); + if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400); +} + +function parseLineartInput(body: Record) { + const keys = Object.keys(body).sort(); + if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "lineart" || typeof body.dataUrl !== "string") { + throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换请求字段无效", 400); + } + validateLineartDataUrl(body.dataUrl); + return body.dataUrl; +} + +function parseBody(request: Request): Record { + if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {}; + try { + const value = JSON.parse(request.body.toString("utf8")) as unknown; + return isRecord(value) ? value : {}; + } catch { + return {}; + } +} + +function asyncRoute(action: (request: Request, response: Response) => Promise): RequestHandler { + return (request, response, next) => { + void action(request, response).catch((error) => { + if (response.headersSent) return next(error); + if (error instanceof LineartRuntimeError) { + response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message }); + return; + } + response.status(503).json({ ok: false, code: "lineart_runtime_unavailable", message: "本机 AI 线稿运行时不可用" }); + }); + }; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0; +} diff --git a/canvas-agent/src/modules/pose-estimation-http.ts b/canvas-agent/src/modules/pose-estimation-http.ts new file mode 100644 index 000000000..972961d64 --- /dev/null +++ b/canvas-agent/src/modules/pose-estimation-http.ts @@ -0,0 +1,357 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; + +import type { Request, RequestHandler, Response } from "express"; + +import type { LocalRuntimeModule } from "../local-runtime.js"; + +const DEFAULT_MODEL_ID = "lllyasviel/Annotators"; +const DEFAULT_BACKEND = "openpose-body" as const; +const MAX_INPUT_BYTES = 12 * 1024 * 1024; +const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024; +const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/; + +export type PoseRuntimePaths = { + pythonPath: string; + scriptPath: string; + hfHome: string; + modelId: string; + configured: boolean; + modelCached: boolean; + reason?: "python_missing" | "worker_missing"; +}; + +export type PoseRuntimeStatus = { + configured: boolean; + modelCached: boolean; + loaded: boolean; + device: "cpu"; + modelId: string; + backend: typeof DEFAULT_BACKEND; + reason?: string; +}; + +type PoseWorkerSuccess = { + ok: true; + requestId: string; + mimeType: "image/png"; + width: number; + height: number; + modelId: string; + device: "cpu"; + backend: typeof DEFAULT_BACKEND; + personCount: number; + people?: unknown[]; + pngBase64: string; +}; + +export class PoseRuntimeError extends Error { + constructor(readonly code: string, message: string, readonly statusCode = 503) { + super(message); + this.name = "PoseRuntimeError"; + } +} + +export function createPoseEstimationHttpModule(): LocalRuntimeModule { + const worker = new PoseEstimationWorker(); + return { + descriptor: { + id: "pose-estimation", + displayName: "OpenPose 本机姿态骨架", + apiVersion: 1, + scopes: ["pose:status", "pose:run"], + }, + routes: [ + { + method: "GET", + path: "/pose-estimation/status", + scope: "pose:status", + handler: asyncRoute(async (_request, response) => { + const status = worker.status(); + response.json({ + ok: true, + module: "pose-estimation", + apiVersion: 1, + ready: status.configured && status.modelCached, + ...status, + }); + }), + }, + { + method: "POST", + path: "/pose-estimation/run", + scope: "pose:run", + handler: asyncRoute(async (request, response) => { + const body = parseBody(request); + const dataUrl = parsePoseInput(body); + const result = await worker.run({ dataUrl }); + response.json({ + ok: true, + module: "pose-estimation", + apiVersion: 1, + result, + }); + }), + }, + ], + dispose: () => worker.dispose(), + publicHealth: () => { + const status = worker.status(); + return { poseEstimation: status.configured && status.modelCached ? "available" : "unavailable" }; + }, + }; +} + +export function resolvePoseRuntimePaths(env: NodeJS.ProcessEnv = process.env): PoseRuntimePaths { + const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim(); + const projectRoots = [ + configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined, + path.resolve(moduleRoot, ".."), + path.resolve(process.cwd()), + ].filter((value): value is string => Boolean(value)); + const projectRoot = configuredProjectRoot + ? path.resolve(configuredProjectRoot) + : projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!; + const scriptCandidates = [ + path.join(moduleRoot, "python", "pose_runtime.py"), + path.join(projectRoot, "canvas-agent", "python", "pose_runtime.py"), + ]; + const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!; + const pythonCandidates = env.CANVAS_POSE_PYTHON?.trim() + ? [path.resolve(env.CANVAS_POSE_PYTHON.trim())] + : process.platform === "win32" + ? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")] + : [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")]; + const pythonPath = pythonCandidates[0]!; + const hfHome = path.resolve( + env.CANVAS_POSE_HF_HOME?.trim() + || env.CANVAS_LINEART_HF_HOME?.trim() + || env.CANVAS_DEPTH_HF_HOME?.trim() + || path.join(projectRoot, ".local", "cache", "huggingface"), + ); + const modelId = env.CANVAS_POSE_MODEL_ID?.trim() || DEFAULT_MODEL_ID; + const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`); + const workerExists = fs.existsSync(scriptPath); + const pythonExists = fs.existsSync(pythonPath); + return { + pythonPath, + scriptPath, + hfHome, + modelId, + configured: workerExists && pythonExists, + modelCached: workerExists && pythonExists && findCachedModelFile(modelCacheDir, "body_pose_model.pth"), + ...(pythonExists ? {} : { reason: "python_missing" as const }), + ...(!workerExists ? { reason: "worker_missing" as const } : {}), + }; +} + +export function findCachedModelFile(modelCacheDir: string, filename: string) { + const snapshotsDir = path.join(modelCacheDir, "snapshots"); + try { + return fs.readdirSync(snapshotsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .some((entry) => fs.existsSync(path.join(snapshotsDir, entry.name, filename))); + } catch { + return false; + } +} + +export class PoseEstimationWorker { + private paths = resolvePoseRuntimePaths(); + private child?: ChildProcessWithoutNullStreams; + private stdoutBuffer = ""; + private stderrBuffer = ""; + private requestCounter = 0; + private loaded = false; + private readonly pending = new Map void; reject: (error: unknown) => void }>(); + + status(): PoseRuntimeStatus { + const paths = this.refreshPaths(); + return { + configured: paths.configured, + modelCached: paths.modelCached, + loaded: this.loaded, + device: "cpu", + modelId: paths.modelId, + backend: DEFAULT_BACKEND, + ...(paths.reason ? { reason: paths.reason } : {}), + }; + } + + async run(input: { dataUrl: string }) { + validatePoseDataUrl(input.dataUrl); + const paths = this.refreshPaths(); + if (!paths.configured) { + throw new PoseRuntimeError( + "pose_runtime_unavailable", + paths.reason === "worker_missing" ? "本机姿态运行文件不存在" : "未找到本机姿态 Python 环境", + ); + } + const child = this.ensureChild(); + const requestId = `pose-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`; + const response = new Promise((resolve, reject) => { + this.pending.set(requestId, { resolve, reject }); + }); + try { + child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n"); + } catch { + this.pending.delete(requestId); + throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法写入"); + } + return response; + } + + dispose() { + const child = this.child; + this.child = undefined; + this.loaded = false; + this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已停止")); + if (child && !child.killed) child.kill(); + } + + private ensureChild() { + if (this.child && !this.child.killed) return this.child; + let child: ChildProcessWithoutNullStreams; + try { + child = spawn(this.paths.pythonPath, [this.paths.scriptPath], { + cwd: path.dirname(this.paths.scriptPath), + env: { + ...process.env, + HF_HOME: this.paths.hfHome, + HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"), + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + HF_HUB_DISABLE_TELEMETRY: "1", + HF_HUB_DISABLE_PROGRESS_BARS: "1", + TRANSFORMERS_VERBOSITY: "error", + CANVAS_POSE_MODEL_ID: this.paths.modelId, + }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + } catch { + throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法启动"); + } + this.child = child; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk)); + child.stderr.on("data", (chunk: string) => { + this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000); + }); + child.on("error", () => this.handleChildExit(child)); + child.on("exit", () => this.handleChildExit(child)); + return child; + } + + private refreshPaths() { + if (!this.child) this.paths = resolvePoseRuntimePaths(); + return this.paths; + } + + private consumeStdout(chunk: string) { + this.stdoutBuffer += chunk; + if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) { + this.child?.kill(); + this.handleChildExit(this.child); + return; + } + while (true) { + const newline = this.stdoutBuffer.indexOf("\n"); + if (newline < 0) return; + const line = this.stdoutBuffer.slice(0, newline).trim(); + this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1); + if (!line) continue; + let value: unknown; + try { value = JSON.parse(line); } catch { continue; } + if (!isRecord(value) || typeof value.requestId !== "string") continue; + const pending = this.pending.get(value.requestId); + if (!pending) continue; + this.pending.delete(value.requestId); + if (value.ok === true) { + if (value.mimeType !== "image/png" + || !isPositiveInteger(value.width) + || !isPositiveInteger(value.height) + || typeof value.modelId !== "string" + || value.device !== "cpu" + || value.backend !== DEFAULT_BACKEND + || !isPositiveInteger(value.personCount) + || typeof value.pngBase64 !== "string" + || !value.pngBase64) { + pending.reject(new PoseRuntimeError("pose_response_invalid", "本机姿态响应无效", 500)); + continue; + } + this.loaded = true; + pending.resolve(value as unknown as PoseWorkerSuccess); + continue; + } + const code = typeof value.code === "string" ? value.code : "pose_inference_failed"; + const statusCode = code === "pose_model_missing" ? 503 : code === "pose_input_invalid" ? 400 : code === "pose_no_person" ? 422 : 500; + pending.reject(new PoseRuntimeError(code, typeof value.message === "string" ? value.message : "本机姿态推理失败", statusCode)); + } + } + + private handleChildExit(child?: ChildProcessWithoutNullStreams) { + if (!child || this.child !== child) return; + this.child = undefined; + this.loaded = false; + this.stderrBuffer = ""; + this.stdoutBuffer = ""; + this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已退出")); + } + + private rejectPending(error: PoseRuntimeError) { + for (const { reject } of this.pending.values()) reject(error); + this.pending.clear(); + } +} + +export function validatePoseDataUrl(value: string) { + const match = DATA_URL_PATTERN.exec(value); + if (!match) throw new PoseRuntimeError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400); + const bytes = Buffer.from(match[2]!, "base64"); + if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new PoseRuntimeError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400); +} + +function parsePoseInput(body: Record) { + const keys = Object.keys(body).sort(); + if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "pose" || typeof body.dataUrl !== "string") { + throw new PoseRuntimeError("pose_input_invalid", "姿态转换请求字段无效", 400); + } + validatePoseDataUrl(body.dataUrl); + return body.dataUrl; +} + +function parseBody(request: Request): Record { + if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {}; + try { + const value = JSON.parse(request.body.toString("utf8")) as unknown; + return isRecord(value) ? value : {}; + } catch { + return {}; + } +} + +function asyncRoute(action: (request: Request, response: Response) => Promise): RequestHandler { + return (request, response, next) => { + void action(request, response).catch((error) => { + if (response.headersSent) return next(error); + if (error instanceof PoseRuntimeError) { + response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message }); + return; + } + response.status(503).json({ ok: false, code: "pose_runtime_unavailable", message: "本机姿态运行时不可用" }); + }); + }; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0; +} diff --git a/canvas-agent/test/depth-estimation.test.ts b/canvas-agent/test/depth-estimation.test.ts new file mode 100644 index 000000000..912c6f08d --- /dev/null +++ b/canvas-agent/test/depth-estimation.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createDepthEstimationHttpModule, DepthRuntimeError, resolveDepthRuntimePaths, validateDepthDataUrl } from "../src/modules/depth-estimation-http.js"; + +test("depth runtime module declares signed status and run routes", () => { + const module = createDepthEstimationHttpModule(); + assert.equal(module.descriptor.id, "depth-estimation"); + assert.deepEqual(module.descriptor.scopes, ["depth:status", "depth:run"]); + assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [ + "GET /depth-estimation/status", + "POST /depth-estimation/run", + ]); + module.dispose?.(); +}); + +test("depth input guard accepts image data URLs and rejects unsafe input", () => { + assert.doesNotThrow(() => validateDepthDataUrl("data:image/png;base64,AQID")); + assert.throws( + () => validateDepthDataUrl("data:text/plain;base64,AQID"), + (error: unknown) => error instanceof DepthRuntimeError && error.code === "depth_input_invalid" && error.statusCode === 400, + ); +}); + +test("depth runtime paths honor explicit project and Python overrides", () => { + const paths = resolveDepthRuntimePaths({ + CANVAS_PROJECT_ROOT: "C:\\depth-project", + CANVAS_DEPTH_PYTHON: "C:\\depth-project\\venv\\python.exe", + CANVAS_DEPTH_HF_HOME: "C:\\depth-cache", + CANVAS_DEPTH_MODEL_ID: "example/depth-model", + }); + assert.equal(paths.pythonPath, "C:\\depth-project\\venv\\python.exe"); + assert.equal(paths.hfHome, "C:\\depth-cache"); + assert.equal(paths.modelId, "example/depth-model"); + assert.equal(paths.configured, false); +}); diff --git a/canvas-agent/test/lineart-estimation.test.ts b/canvas-agent/test/lineart-estimation.test.ts new file mode 100644 index 000000000..8ecde2974 --- /dev/null +++ b/canvas-agent/test/lineart-estimation.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createLineartEstimationHttpModule, LineartRuntimeError, resolveLineartRuntimePaths, validateLineartDataUrl } from "../src/modules/lineart-estimation-http.js"; + +test("lineart runtime module declares signed status and run routes", () => { + const module = createLineartEstimationHttpModule(); + assert.equal(module.descriptor.id, "lineart-estimation"); + assert.deepEqual(module.descriptor.scopes, ["lineart:status", "lineart:run"]); + assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [ + "GET /lineart-estimation/status", + "POST /lineart-estimation/run", + ]); + module.dispose?.(); +}); + +test("lineart input guard accepts image data URLs and rejects unsafe input", () => { + assert.doesNotThrow(() => validateLineartDataUrl("data:image/png;base64,AQID")); + assert.throws( + () => validateLineartDataUrl("data:text/plain;base64,AQID"), + (error: unknown) => error instanceof LineartRuntimeError && error.code === "lineart_input_invalid" && error.statusCode === 400, + ); +}); + +test("lineart runtime paths honor explicit project and Python overrides", () => { + const paths = resolveLineartRuntimePaths({ + CANVAS_PROJECT_ROOT: "C:\\lineart-project", + CANVAS_DEPTH_PYTHON: "C:\\lineart-project\\venv\\python.exe", + CANVAS_LINEART_HF_HOME: "C:\\lineart-cache", + CANVAS_LINEART_MODEL_ID: "example/lineart-model", + }); + assert.equal(paths.pythonPath, "C:\\lineart-project\\venv\\python.exe"); + assert.equal(paths.hfHome, "C:\\lineart-cache"); + assert.equal(paths.modelId, "example/lineart-model"); + assert.equal(paths.configured, false); +}); diff --git a/canvas-agent/test/pose-estimation.test.ts b/canvas-agent/test/pose-estimation.test.ts new file mode 100644 index 000000000..fac0d1b94 --- /dev/null +++ b/canvas-agent/test/pose-estimation.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createPoseEstimationHttpModule, PoseRuntimeError, resolvePoseRuntimePaths, validatePoseDataUrl } from "../src/modules/pose-estimation-http.js"; + +test("pose runtime module declares signed status and run routes", () => { + const module = createPoseEstimationHttpModule(); + assert.equal(module.descriptor.id, "pose-estimation"); + assert.deepEqual(module.descriptor.scopes, ["pose:status", "pose:run"]); + assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [ + "GET /pose-estimation/status", + "POST /pose-estimation/run", + ]); + module.dispose?.(); +}); + +test("pose input guard accepts image data URLs and rejects unsafe input", () => { + assert.doesNotThrow(() => validatePoseDataUrl("data:image/png;base64,AQID")); + assert.throws( + () => validatePoseDataUrl("data:text/plain;base64,AQID"), + (error: unknown) => error instanceof PoseRuntimeError && error.code === "pose_input_invalid" && error.statusCode === 400, + ); +}); + +test("pose runtime paths honor explicit project and Python overrides", () => { + const paths = resolvePoseRuntimePaths({ + CANVAS_PROJECT_ROOT: "C:\\pose-project", + CANVAS_POSE_PYTHON: "C:\\pose-project\\venv\\python.exe", + CANVAS_POSE_HF_HOME: "C:\\pose-cache", + CANVAS_POSE_MODEL_ID: "example/pose-model", + }); + assert.equal(paths.pythonPath, "C:\\pose-project\\venv\\python.exe"); + assert.equal(paths.hfHome, "C:\\pose-cache"); + assert.equal(paths.modelId, "example/pose-model"); + assert.equal(paths.configured, false); + assert.equal(paths.modelCached, false); +}); diff --git a/web/src/components/canvas/canvas-context-menu.tsx b/web/src/components/canvas/canvas-context-menu.tsx index f332c6fea..e21d48cc4 100644 --- a/web/src/components/canvas/canvas-context-menu.tsx +++ b/web/src/components/canvas/canvas-context-menu.tsx @@ -374,6 +374,7 @@ function nodeTypeLabel(node?: CanvasNodeData | null) { if (node.type === CanvasNodeType.Video) return "视频节点"; if (node.type === CanvasNodeType.Audio) return "音频节点"; if (node.type === CanvasNodeType.Drawing) return "绘图节点"; + if (node.type === CanvasNodeType.MediaConversion) return "转换节点"; if (node.type === CanvasNodeType.Frame) return isCanvasFolderNode(node) ? "文件夹" : "背板"; return "生成配置节点"; } diff --git a/web/src/components/canvas/canvas-node-content.tsx b/web/src/components/canvas/canvas-node-content.tsx index 1e3935ba6..7fa0e4f6c 100644 --- a/web/src/components/canvas/canvas-node-content.tsx +++ b/web/src/components/canvas/canvas-node-content.tsx @@ -37,6 +37,8 @@ import { PanoramaNodeContent } from "./nodes/panorama-node"; import { SvgNodeContent } from "./nodes/svg-node"; import { PortraitClearanceNodeContent } from "./nodes/portrait-clearance-node"; import { ArtCritiqueNodeContent } from "./nodes/ai-art-critique-node"; +import { MediaConversionNodeContent } from "./nodes/media-conversion-node"; +import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts"; export type CanvasNodeContentProps = { node: CanvasNodeData; @@ -74,6 +76,7 @@ export function CanvasNodeContent(props: CanvasNodeContentProps) { if (hasCustomContent && props.renderNodeContent) return props.renderNodeContent(props.node); if (props.node.type === PORTRAIT_CLEARANCE_NODE_TYPE) return ; if (props.node.type === ART_CRITIQUE_NODE_TYPE) return ; + if (props.node.type === MEDIA_CONVERSION_NODE_TYPE) return ; if (props.isBatchRoot) return ; if (props.node.metadata?.status === "loading") return ; if (props.node.metadata?.status === "error") return ; diff --git a/web/src/components/canvas/canvas-node-generation.ts b/web/src/components/canvas/canvas-node-generation.ts index 68c32d9d6..50ee6883b 100644 --- a/web/src/components/canvas/canvas-node-generation.ts +++ b/web/src/components/canvas/canvas-node-generation.ts @@ -8,6 +8,7 @@ import { getGenerationResourceNodes, getContextResourceNodes, getMentionResource import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview"; import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade"; import { getNodeResourceKind } from "@/lib/canvas/node-registry"; +import { mediaConversionSourceFingerprint } from "@/lib/media-conversion/contracts"; import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference"; import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference"; import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation"; @@ -64,6 +65,12 @@ export type NodeGenerationInput = { }; export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, assets: Asset[], promptOnly = false): NodeGenerationContext { + const pendingConversion = findPendingMediaConversionInput(nodeId, nodes, connections); + if (pendingConversion) { + const status = pendingConversion.metadata?.mediaConversion?.status; + const reason = status === "stale" ? "输入或方式已变化" : status === "processing" ? "仍在处理中" : "尚未完成本地转换"; + throw new Error(`转换节点「${pendingConversion.title || pendingConversion.id}」${reason}，完成后才能执行下游生成`); + } const connectedInputs = buildNodeGenerationInputs(nodeId, nodes, connections); const sourceNode = nodes.find((node) => node.id === nodeId); const portraitTextureInput = sourceNode?.type === CanvasNodeType.Image && sourceNode.metadata?.content && sourceNode.metadata?.portraitTexture @@ -120,6 +127,40 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData }; } +/** + * 转换节点允许提前连到下游，但只有已物化的结果才能进入生成请求。 + * 沿上游链路检查也覆盖“生成节点 -> 配置节点 -> 转换节点”的常见接法。 + */ +export function findPendingMediaConversionInput(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) { + const nodesById = new Map(nodes.map((node) => [node.id, node])); + const queue = connections.filter((connection) => connection.toNodeId === nodeId).map((connection) => connection.fromNodeId); + const visited = new Set(); + while (queue.length) { + const sourceId = queue.shift()!; + if (visited.has(sourceId)) continue; + visited.add(sourceId); + const source = nodesById.get(sourceId); + if (!source) continue; + if (source.type === CanvasNodeType.MediaConversion) { + const state = source.metadata?.mediaConversion; + const sourceInputs = connections + .filter((connection) => connection.toNodeId === source.id) + .map((connection) => nodesById.get(connection.fromNodeId)) + .filter((input): input is CanvasNodeData => Boolean(input && (input.type === CanvasNodeType.Image || input.type === CanvasNodeType.Video))); + const sourceInput = sourceInputs[0]; + const ready = state?.status === "completed" + && Boolean(state.resultStorageKey) + && sourceInputs.length === 1 + && Boolean(sourceInput) + && state.sourceNodeId === sourceInput?.id + && state.sourceFingerprint === mediaConversionSourceFingerprint(sourceInput!); + if (!ready) return source; + } + connections.filter((connection) => connection.toNodeId === sourceId).forEach((connection) => queue.push(connection.fromNodeId)); + } + return null; +} + function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]) { let next = prompt.trim(); let removed = true; diff --git a/web/src/components/canvas/canvas-node-toolbar.tsx b/web/src/components/canvas/canvas-node-toolbar.tsx index 879fcedf0..6ef0c05ad 100644 --- a/web/src/components/canvas/canvas-node-toolbar.tsx +++ b/web/src/components/canvas/canvas-node-toolbar.tsx @@ -32,6 +32,7 @@ type CanvasNodeToolbarProps = { onUpload: (node: CanvasNodeData) => void; onDownload: (node: CanvasNodeData) => void; onSaveAsset: (node: CanvasNodeData) => void; + onCreateConversion?: (node: CanvasNodeData) => void; onMaskEdit: (node: CanvasNodeData) => void; onEmotion: (node: CanvasNodeData) => void; onPortraitTexture: (node: CanvasNodeData) => void; @@ -91,6 +92,7 @@ export function CanvasNodeToolbar({ onUpload, onDownload, onSaveAsset, + onCreateConversion, onMaskEdit, onEmotion, onPortraitTexture, @@ -221,7 +223,7 @@ export function CanvasNodeToolbar({ const nodeHoverHandlers = { onNodeInfo: onInfo, onNodeDelete: onDelete, onNodeRetry: onRetry, onNodeEditText: onEditText, onNodeDecreaseFont: onDecreaseFont, onNodeIncreaseFont: onIncreaseFont, onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload, - onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop, + onNodeSaveAsset: onSaveAsset, onNodeCreateConversion: onCreateConversion, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop, onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage, onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt, @@ -400,7 +402,7 @@ export function CanvasNodeInfoModal({ node, open, onClose, onMetadataChange, rea const [assetCategory, setAssetCategory] = useState("other"); const imageBytes = node?.type === CanvasNodeType.Image && node.metadata?.content ? getDataUrlByteSize(node.metadata.content) : 0; const batchCount = node?.type === CanvasNodeType.Image ? node.metadata?.batchChildIds?.length || 0 : 0; - const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : "生成配置"; + const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : node?.type === CanvasNodeType.MediaConversion ? "转换" : "生成配置"; useEffect(() => { setAssetTags(node?.metadata?.assetTags || []); setAssetTagInput(""); diff --git a/web/src/components/canvas/canvas-node.tsx b/web/src/components/canvas/canvas-node.tsx index 2086d5775..74b82d421 100644 --- a/web/src/components/canvas/canvas-node.tsx +++ b/web/src/components/canvas/canvas-node.tsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import type { ReactNode } from "react"; -import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video } from "lucide-react"; +import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video, WandSparkles } from "lucide-react"; import { useCanvasNodeActions } from "./canvas-node-action-context"; @@ -709,6 +709,7 @@ function nodeTypeIcon(type: CanvasNodeTypeId) { if (type === CanvasNodeType.Drawing) return Pencil; if (type === CanvasNodeType.Script) return Clapperboard; if (type === CanvasNodeType.Config) return Settings2; + if (type === CanvasNodeType.MediaConversion) return WandSparkles; if (type === CanvasNodeType.Skill) return BookOpenCheck; if (type === PORTRAIT_CLEARANCE_NODE_TYPE) return PortraitClearanceIcon; if (type === ART_CRITIQUE_NODE_TYPE) return ScanSearch; diff --git a/web/src/components/canvas/canvas-workspace-overlays.tsx b/web/src/components/canvas/canvas-workspace-overlays.tsx index b05c57f00..c3bef84da 100644 --- a/web/src/components/canvas/canvas-workspace-overlays.tsx +++ b/web/src/components/canvas/canvas-workspace-overlays.tsx @@ -153,7 +153,7 @@ function resolveNodePanelWidth(node: CanvasNodeData, viewport: ViewportTransform return clamp(Math.round(node.width * viewport.k * 1.5), 680, 920); } -export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) { +export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) { const theme = canvasThemes[useThemeStore((state) => state.theme)]; const reducedMotion = useReducedMotion(); const menuRef = useRef(null); @@ -213,6 +213,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co {canCreateDrawing ? } title="绘图" disabledReason={getDisabledReason(CanvasNodeType.Drawing)} onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null} } title="视频生成" disabledReason={getDisabledReason(CanvasNodeType.Video)} onClick={() => onCreate(CanvasNodeType.Video)} /> } title="音频参考" disabledReason={getDisabledReason(CanvasNodeType.Audio)} onClick={() => onCreate(CanvasNodeType.Audio)} /> + } title="转换" description="本地处理图片或视频" disabledReason={getDisabledReason(CanvasNodeType.MediaConversion)} onClick={() => onCreate(CanvasNodeType.MediaConversion)} /> ); diff --git a/web/src/components/canvas/nodes/media-conversion-node.tsx b/web/src/components/canvas/nodes/media-conversion-node.tsx new file mode 100644 index 000000000..22df85491 --- /dev/null +++ b/web/src/components/canvas/nodes/media-conversion-node.tsx @@ -0,0 +1,360 @@ +import { CheckCircle2, CircleAlert, Download, Image as ImageIcon, LoaderCircle, RefreshCw, Video, WandSparkles } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; + +import { getNodeInputKind } from "@/lib/canvas/node-registry"; +import type { CanvasTheme } from "@/lib/canvas-theme"; +import { + createDefaultMediaConversionState, + isLocalImageOperation, + mediaConversionOperationDescription, + mediaConversionOperationLabel, + mediaConversionSourceFingerprint, + MEDIA_CONVERSION_OPERATION_LABELS, + type MediaConversionNodeState, + type MediaConversionOperation, +} from "@/lib/media-conversion/contracts"; +import { convertImageLocally, LocalImageConversionError } from "@/lib/media-conversion/local-converter"; +import { getActiveUserScope } from "@/lib/user-scope"; +import { runLocalDepthEstimation } from "@/services/depth-runtime"; +import { runLocalLineartEstimation } from "@/services/lineart-runtime"; +import { runLocalPoseEstimation } from "@/services/pose-runtime"; +import { resolveImageUrl, setImageBlob } from "@/services/image-storage"; +import { resolveMediaUrl } from "@/services/file-storage"; +import { LocalRuntimeClientError } from "@/services/local-runtime-session"; +import { useLocalRuntimeStore } from "@/stores/use-local-runtime-store"; +import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas"; + +import { useCanvasNodeActions } from "../canvas-node-action-context"; +import { useUpstreamNodes } from "../canvas-node-graph-context"; + +type MediaConversionNodeContentProps = { + node: CanvasNodeData; + theme: CanvasTheme; +}; + +const OPERATIONS = Object.keys(MEDIA_CONVERSION_OPERATION_LABELS) as MediaConversionOperation[]; + +export function MediaConversionNodeContent({ node, theme }: MediaConversionNodeContentProps) { + const { updateMetadata } = useCanvasNodeActions(); + const upstream = useUpstreamNodes(node.id); + const localRuntimeConnection = useLocalRuntimeStore((runtime) => runtime.connection); + const localRuntimeConnecting = useLocalRuntimeStore((runtime) => runtime.connecting); + const reconnectLocalRuntime = useLocalRuntimeStore((runtime) => runtime.connect); + const mediaInputs = upstream.filter((item) => item.type === CanvasNodeType.Image || item.type === CanvasNodeType.Video); + const input = mediaInputs[0]; + const inputKind = input ? getNodeInputKind(input.type) : undefined; + const storedState = node.metadata?.mediaConversion; + const state = storedState || createDefaultMediaConversionState(); + const currentFingerprint = input ? mediaConversionSourceFingerprint(input) : ""; + const hasMultipleInputs = mediaInputs.length > 1; + const isStale = Boolean(state.status === "completed" && (!currentFingerprint || state.sourceFingerprint !== currentFingerprint)); + const status = isStale ? "stale" : state.status; + const [sourceUrl, setSourceUrl] = useState(""); + const [resultUrl, setResultUrl] = useState(""); + const [showResult, setShowResult] = useState(status === "completed" || status === "stale"); + const [notice, setNotice] = useState(""); + const abortRef = useRef(null); + + useEffect(() => { + let active = true; + const fallback = input?.metadata?.previewContent || input?.metadata?.content || ""; + setSourceUrl(fallback); + if (!input?.metadata?.storageKey) return () => { active = false; }; + const resolver = input.type === CanvasNodeType.Image ? resolveImageUrl : resolveMediaUrl; + void resolver(input.metadata.storageKey, fallback) + .then((resolved) => { if (active) setSourceUrl(resolved || fallback); }) + .catch(() => { if (active) setSourceUrl(fallback); }); + return () => { active = false; }; + }, [input?.id, input?.metadata?.content, input?.metadata?.previewContent, input?.metadata?.storageKey, input?.type]); + + useEffect(() => { + let active = true; + setResultUrl(""); + if (!state.resultStorageKey) return () => { active = false; }; + void resolveImageUrl(state.resultStorageKey) + .then((resolved) => { if (active) setResultUrl(resolved); }) + .catch(() => { if (active) setResultUrl(""); }); + return () => { active = false; }; + }, [state.resultStorageKey]); + + useEffect(() => { + if (!updateMetadata || state.status !== "completed" || (currentFingerprint && state.sourceFingerprint === currentFingerprint)) return; + updateMetadata(node.id, { + mediaConversion: { ...state, status: "stale", updatedAt: new Date().toISOString() }, + }); + }, [currentFingerprint, node.id, state, updateMetadata]); + + useEffect(() => { + if (status === "stale") setShowResult(false); + }, [status]); + + useEffect(() => () => abortRef.current?.abort(), []); + + const updateState = (patch: Partial) => { + updateMetadata?.(node.id, { mediaConversion: { ...state, ...patch, schemaVersion: 1 } }); + }; + + const selectOperation = (operation: MediaConversionOperation) => { + if (operation === state.operation) return; + setNotice(""); + updateState({ + operation, + status: state.resultStorageKey ? "stale" : "idle", + errorCode: undefined, + errorMessage: undefined, + updatedAt: new Date().toISOString(), + }); + setShowResult(false); + }; + + const run = async () => { + setNotice(""); + if (!input || !currentFingerprint) { + setNotice("请先连接一张图片或一个视频"); + return; + } + if (hasMultipleInputs) { + setNotice("转换节点只能连接一个输入，请移除多余连线"); + return; + } + if (inputKind === "video") { + updateState({ status: "unavailable", outputKind: "video", errorCode: "video_not_ready", errorMessage: "本地视频转换还在进行短片性能验证", updatedAt: new Date().toISOString() }); + setNotice("视频转换暂未开启，先完成单图模型验证"); + return; + } + if (!isLocalImageOperation(state.operation) && state.operation !== "depth" && state.operation !== "lineart" && state.operation !== "pose") { + updateState({ status: "unavailable", outputKind: "image", errorCode: "model_missing", errorMessage: "该转换需要先安装并验证本地模型", updatedAt: new Date().toISOString() }); + setNotice(mediaConversionOperationDescription(state.operation)); + return; + } + if (!sourceUrl) { + updateState({ status: "error", errorCode: "source_unreadable", errorMessage: "图片预览尚未准备好，请稍后重试", updatedAt: new Date().toISOString() }); + setNotice("图片预览尚未准备好，请稍后重试"); + return; + } + + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + const startedAt = new Date().toISOString(); + updateState({ status: "processing", sourceNodeId: input.id, sourceFingerprint: currentFingerprint, outputKind: "image", errorCode: undefined, errorMessage: undefined, startedAt, updatedAt: startedAt }); + try { + const result = state.operation === "depth" + ? await runLocalDepthEstimation(sourceUrl, controller.signal) + : state.operation === "lineart" + ? await runLocalLineartEstimation(sourceUrl, controller.signal) + : state.operation === "pose" + ? await runLocalPoseEstimation(sourceUrl, controller.signal) + : await convertImageLocally(sourceUrl, state.operation, { signal: controller.signal, maxDimension: 1024 }); + if (controller.signal.aborted) return; + const storageKey = localConversionStorageKey(node.id, currentFingerprint, state.operation); + const url = await setImageBlob(storageKey, result.blob); + const completedAt = new Date().toISOString(); + const detectedPeople = state.operation === "pose" && "personCount" in result && typeof result.personCount === "number" ? result.personCount : undefined; + updateMetadata?.(node.id, { + content: "", + previewContent: "", + storageKey, + mimeType: result.blob.type || "image/png", + bytes: result.blob.size, + naturalWidth: result.width, + naturalHeight: result.height, + mediaConversion: { + ...state, + schemaVersion: 1, + status: "completed", + sourceNodeId: input.id, + sourceFingerprint: currentFingerprint, + outputKind: "image", + resultStorageKey: storageKey, + resultWidth: result.width, + resultHeight: result.height, + ...(detectedPeople !== undefined ? { detectedPeople } : {}), + errorCode: undefined, + errorMessage: undefined, + updatedAt: completedAt, + }, + }); + setResultUrl(url); + setShowResult(true); + } catch (error) { + if (controller.signal.aborted) return; + const aborted = error instanceof LocalImageConversionError && error.code === "aborted"; + if (aborted) return; + if (error instanceof LocalRuntimeClientError) { + if (error.code === "pose_no_person") { + updateState({ status: "skipped", errorCode: error.code, errorMessage: error.message, detectedPeople: 0, updatedAt: new Date().toISOString() }); + setNotice("未检测到可用人物姿态，可更换含人物的图片或选择其他转换方式"); + return; + } + const nextStatus = ["depth_model_missing", "depth_runtime_unavailable", "lineart_model_missing", "lineart_runtime_unavailable", "pose_model_missing", "pose_runtime_unavailable"].includes(error.code) ? "unavailable" : "error"; + updateState({ status: nextStatus, errorCode: error.code, errorMessage: error.message, updatedAt: new Date().toISOString() }); + setNotice(localRuntimeNotice(error, useLocalRuntimeStore.getState().connection)); + return; + } + const conversionError = error instanceof LocalImageConversionError ? error : new LocalImageConversionError("source_unreadable", "本地图片转换失败"); + const nextStatus = conversionError.code === "model_missing" ? "unavailable" : "error"; + updateState({ status: nextStatus, errorCode: conversionError.code, errorMessage: conversionError.message, updatedAt: new Date().toISOString() }); + setNotice(conversionError.message); + } + }; + + const downloadResult = () => { + if (!resultUrl) return; + const link = document.createElement("a"); + link.href = resultUrl; + link.download = `${node.title || "转换结果"}-${state.operation}.png`; + link.click(); + }; + + const effectiveResult = Boolean(resultUrl && (status === "completed" || status === "stale")); + const displayResult = showResult && effectiveResult; + const buttonDisabled = !input || hasMultipleInputs || status === "processing"; + const statusText = mediaConversionStatusLabel(status); + const previewStatusText = state.operation === "pose" && status === "completed" && typeof state.detectedPeople === "number" + ? `${statusText} · ${state.detectedPeople} 人` + : statusText; + const showRuntimeRecovery = (state.operation === "depth" || state.operation === "lineart" || state.operation === "pose") && (localRuntimeConnection === "unreachable" || state.errorCode === "depth_runtime_unavailable" || state.errorCode === "lineart_runtime_unavailable" || state.errorCode === "pose_runtime_unavailable"); + const retryLocalRuntime = async () => { + setNotice(""); + await reconnectLocalRuntime(); + const next = useLocalRuntimeStore.getState(); + if (next.connection === "connected") { + setNotice("本地模型服务已连接，可以重新开始转换"); + } else { + setNotice(next.error || "本地模型服务仍未连接，请先运行 start-yingce-local.cmd"); + } + }; + + return ( +
event.stopPropagation()} + onWheel={(event) => event.stopPropagation()} + > +
+ + +
+
{node.title || "转换"}
+
本地处理 · 单图优先
+
+ +
+ +
+
+ {displayResult ? ( + {`${mediaConversionOperationLabel(state.operation)}结果`} + ) : input?.type === CanvasNodeType.Image && sourceUrl ? ( + 转换输入 + ) : input?.type === CanvasNodeType.Video && sourceUrl ? ( +
+ + {effectiveResult ? ( +
+ setShowResult(false)} /> + setShowResult(true)} /> + +
+ ) : null} +
+ +
event.stopPropagation()}> + + + +
+ +
+ {notice || mediaConversionOperationDescription(state.operation)} + {showRuntimeRecovery ? ( + + ) : null} +
+
+ ); +} + +function PreviewToggle({ theme, pressed, label, onClick }: { theme: CanvasTheme; pressed: boolean; label: string; onClick: () => void }) { + return ; +} + +function StatusBadge({ status }: { status: MediaConversionNodeState["status"] | "stale" }) { + if (status === "processing") return ; + if (status === "completed") return ; + if (status === "stale") return ; + if (status === "error") return ; + if (status === "unavailable") return 本地不可用; + if (status === "skipped") return 已跳过; + return 未开始; +} + +function mediaConversionStatusLabel(status: MediaConversionNodeState["status"] | "stale") { + if (status === "processing") return "正在处理"; + if (status === "completed") return "已完成"; + if (status === "stale") return "输入已变化，结果待更新"; + if (status === "unavailable") return "本地能力不可用"; + if (status === "skipped") return "未检测到人物姿态"; + if (status === "error") return "处理失败"; + return "尚未转换"; +} + +function localRuntimeNotice(error: LocalRuntimeClientError, connection: ReturnType["connection"]) { + if (connection === "unreachable") return "本地模型服务未启动，请双击 start-yingce-local.cmd 后重试"; + if (error.code === "depth_model_missing") return "深度模型尚未安装，请先完成本地模型安装"; + if (error.code === "lineart_model_missing") return "AI 线稿依赖或模型尚未安装，请先完成本地线稿模型安装"; + if (error.code === "pose_model_missing") return "姿态模型或依赖尚未安装，请先完成本地姿态模型安装"; + return error.message; +} + +function localConversionStorageKey(nodeId: string, fingerprint: string, operation: MediaConversionOperation) { + return `image:${getActiveUserScope()}:media-conversion:${nodeId}:${fingerprint}:${operation}`; +} diff --git a/web/src/constant/canvas.ts b/web/src/constant/canvas.ts index 91793c88d..aa76c9cd6 100644 --- a/web/src/constant/canvas.ts +++ b/web/src/constant/canvas.ts @@ -26,6 +26,7 @@ export const NODE_DEFAULT_SIZE = { [CanvasNodeType.Compare]: { width: 520, height: 320, title: "对比" }, [CanvasNodeType.Chart]: { width: 480, height: 320, title: "图表" }, [CanvasNodeType.ColorGrade]: { width: 420, height: 360, title: "调色" }, + [CanvasNodeType.MediaConversion]: { width: 380, height: 440, title: "转换" }, } satisfies Record; export const NODE_SPECS = { @@ -101,6 +102,10 @@ export const NODE_SPECS = { ...NODE_DEFAULT_SIZE[CanvasNodeType.ColorGrade], metadata: { status: "idle" }, }, + [CanvasNodeType.MediaConversion]: { + ...NODE_DEFAULT_SIZE[CanvasNodeType.MediaConversion], + metadata: { status: "idle" }, + }, } satisfies Record; export function getNodeSpec(type: CanvasNodeType) { diff --git a/web/src/lib/canvas/canvas-agent-ops.ts b/web/src/lib/canvas/canvas-agent-ops.ts index b02a70951..24a3b8912 100644 --- a/web/src/lib/canvas/canvas-agent-ops.ts +++ b/web/src/lib/canvas/canvas-agent-ops.ts @@ -399,6 +399,7 @@ function canvasNodeTypeLabel(type?: CanvasNodeType) { if (type === CanvasNodeType.Frame) return "背板"; if (type === CanvasNodeType.Drawing) return "绘图节点"; if (type === CanvasNodeType.Skill) return "技能节点"; + if (type === CanvasNodeType.MediaConversion) return "转换节点"; return "文本节点"; } diff --git a/web/src/lib/canvas/canvas-connection-policy.ts b/web/src/lib/canvas/canvas-connection-policy.ts index 569fa1e53..207bbda06 100644 --- a/web/src/lib/canvas/canvas-connection-policy.ts +++ b/web/src/lib/canvas/canvas-connection-policy.ts @@ -1,7 +1,7 @@ import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection"; -import { getNodeAcceptedInputKind, getNodeGenerationMode, getNodeInputKind } from "@/lib/canvas/node-registry"; +import { getNodeAcceptedInputKinds, getNodeGenerationMode, getNodeInputKind, getNodeMaxInputCount } from "@/lib/canvas/node-registry"; import type { AiConfig } from "@/stores/use-config-store"; -import { type CanvasConnection, type CanvasNodeData } from "@/types/canvas"; +import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas"; type ConnectionCandidate = Pick; type CanvasConnectionPolicyOptions = { @@ -12,11 +12,27 @@ type CanvasConnectionPolicyOptions = { export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) { const target = nodes.find((node) => node.id === candidate.toNodeId); if (!target) return "找不到连线目标节点"; - const acceptedInputKind = getNodeAcceptedInputKind(target.type); - if (acceptedInputKind) { + const acceptedInputKinds = getNodeAcceptedInputKinds(target.type); + if (acceptedInputKinds.length) { const source = nodes.find((node) => node.id === candidate.fromNodeId); const sourceKind = source ? getNodeInputKind(source.type) : undefined; - if (sourceKind !== acceptedInputKind) return `${acceptedInputKindLabel(acceptedInputKind)}节点只接受${acceptedInputKindLabel(acceptedInputKind)}输入`; + const isMediaConversion = target.type === CanvasNodeType.MediaConversion; + const hasAcceptedSource = isMediaConversion + ? source?.type === CanvasNodeType.Image || source?.type === CanvasNodeType.Video + : Boolean(sourceKind && acceptedInputKinds.includes(sourceKind)); + if (!sourceKind || !hasAcceptedSource) { + const labels = acceptedInputKinds.map(acceptedInputKindLabel).join("或"); + return `${isMediaConversion ? "转换" : labels}节点只接受${labels}输入`; + } + const maxInputCount = getNodeMaxInputCount(target.type); + if (maxInputCount) { + const inputCount = new Set( + [...connections, { id: "candidate", ...candidate }] + .filter((connection) => connection.toNodeId === target.id) + .map((connection) => connection.fromNodeId), + ).size; + if (inputCount > maxInputCount) return `${isMediaConversion ? "转换" : "当前"}节点最多连接 ${maxInputCount} 个输入`; + } } const mode = getNodeGenerationMode(target); if (!mode) return ""; diff --git a/web/src/lib/canvas/node-registry/definitions/builtin-nodes.tsx b/web/src/lib/canvas/node-registry/definitions/builtin-nodes.tsx index 8cadef136..a5802e00f 100644 --- a/web/src/lib/canvas/node-registry/definitions/builtin-nodes.tsx +++ b/web/src/lib/canvas/node-registry/definitions/builtin-nodes.tsx @@ -1,4 +1,4 @@ -import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video } from "lucide-react"; +import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video, WandSparkles } from "lucide-react"; import { NODE_SPECS } from "@/constant/canvas"; import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size"; @@ -25,7 +25,7 @@ const BUILTIN_NODE_TRAITS = { minSize: MEDIA_NODE_MIN_SIZE, keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize, showInCreateMenu: true, - resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "image" : null), + resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "image" : null), generationMode: () => "image", inputKind: "image", }, @@ -82,7 +82,7 @@ const BUILTIN_NODE_TRAITS = { minSize: MEDIA_NODE_MIN_SIZE, keepAspectRatio: () => true, showInCreateMenu: true, - resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "video" : null), + resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "video" : null), generationMode: () => "video", inputKind: "video", }, @@ -165,6 +165,20 @@ const BUILTIN_NODE_TRAITS = { resourceKind: () => "image", inputKind: "image", }, + [CanvasNodeType.MediaConversion]: { + label: "转换", + icon: , + minSize: { width: 320, height: 360 }, + showInCreateMenu: true, + resourceKind: (node: CanvasNodeData) => { + const conversion = node.metadata?.mediaConversion; + if (conversion?.status !== "completed" || !conversion.resultStorageKey) return null; + return conversion.outputKind === "video" ? "video" : "image"; + }, + acceptsInputKind: ["image", "video"], + maxInputCount: 1, + inputKind: "image", + }, } satisfies Record>; export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => { diff --git a/web/src/lib/canvas/node-registry/index.ts b/web/src/lib/canvas/node-registry/index.ts index 0f243e580..09ac25261 100644 --- a/web/src/lib/canvas/node-registry/index.ts +++ b/web/src/lib/canvas/node-registry/index.ts @@ -1,3 +1,3 @@ export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition"; -export { getNodeAcceptedInputKind, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry"; +export { getNodeAcceptedInputKind, getNodeAcceptedInputKinds, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMaxInputCount, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry"; import "./definitions"; diff --git a/web/src/lib/canvas/node-registry/node-definition.ts b/web/src/lib/canvas/node-registry/node-definition.ts index a692fef53..6126e9e81 100644 --- a/web/src/lib/canvas/node-registry/node-definition.ts +++ b/web/src/lib/canvas/node-registry/node-definition.ts @@ -43,8 +43,10 @@ export type CanvasNodeDefinition = { generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null; /** 是否显示右侧输出连接点；缺省为 true，消费型终点节点可关闭。 */ showOutputConnection?: boolean; - /** 若设置，作为目标时只接受这一类上游节点。 */ - acceptsInputKind?: CanvasNodeInputKind; + /** 若设置，作为目标时只接受这些上游节点类型。 */ + acceptsInputKind?: CanvasNodeInputKind | CanvasNodeInputKind[]; + /** 限制直接输入数量；转换节点等一进一出节点通常为 1。 */ + maxInputCount?: number; /** * 作为上游输入被参考素材容量校验计数时的类别； * 不设表示不参与计数（生成配置、背板）。与 resourceKind 不同，计数不看内容。 @@ -69,6 +71,7 @@ export function canvasNodeDefinitionFromPlugin(pluginId: string, contribution: P showInCreateMenu: true, showOutputConnection: contribution.showOutputConnection, acceptsInputKind: contribution.acceptsInputKind, + maxInputCount: contribution.maxInputCount, plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema }, }; } diff --git a/web/src/lib/canvas/node-registry/node-registry.ts b/web/src/lib/canvas/node-registry/node-registry.ts index f32bf5af6..7b9510aa1 100644 --- a/web/src/lib/canvas/node-registry/node-registry.ts +++ b/web/src/lib/canvas/node-registry/node-registry.ts @@ -100,3 +100,13 @@ export function getNodeInputKind(type: CanvasNodeTypeId) { export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) { return definitions.get(type)?.acceptsInputKind; } + +export function getNodeAcceptedInputKinds(type: CanvasNodeTypeId) { + const accepted = definitions.get(type)?.acceptsInputKind; + if (!accepted) return []; + return Array.isArray(accepted) ? accepted : [accepted]; +} + +export function getNodeMaxInputCount(type: CanvasNodeTypeId) { + return definitions.get(type)?.maxInputCount; +} diff --git a/web/src/lib/canvas/tool-registry/definitions/add-node-menu-tools.tsx b/web/src/lib/canvas/tool-registry/definitions/add-node-menu-tools.tsx index a8b9d16ee..15be3b3f1 100644 --- a/web/src/lib/canvas/tool-registry/definitions/add-node-menu-tools.tsx +++ b/web/src/lib/canvas/tool-registry/definitions/add-node-menu-tools.tsx @@ -20,6 +20,7 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [ { id: "folder", label: "文件夹", icon: , badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() }, nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }), nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }), + nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }), // 导演台落在节点分区，但它开的是导演工作台、不是某种画布节点，故不走注册表。 { id: "director", label: "导演台", icon: , badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() }, nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }), diff --git a/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx b/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx index 3c29b3ceb..66915fcfc 100644 --- a/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx +++ b/web/src/lib/canvas/tool-registry/definitions/node-hover-tools.tsx @@ -1,4 +1,4 @@ -import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react"; +import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video, WandSparkles } from "lucide-react"; import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error"; import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry"; @@ -139,6 +139,19 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [ applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx), run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!), }, + { + id: "createConversion", + toolbar: "node-hover", + category: "node-state", + label: "创建转换节点", + displayLabel: "转换", + icon: , + defaultVisible: true, + defaultOrder: 65, + nodeToolbar: { group: "primary", order: 15, description: "在右侧创建转换节点并自动连接" }, + applicable: (ctx) => (hasImage(ctx) || hasVideo(ctx)) && Boolean(ctx.handlers.onNodeCreateConversion), + run: (ctx) => ctx.handlers.onNodeCreateConversion?.(ctx.node!), + }, { id: "edit", toolbar: "node-hover", diff --git a/web/src/lib/canvas/tool-registry/tool-definition.ts b/web/src/lib/canvas/tool-registry/tool-definition.ts index aa2635645..fcb5183ba 100644 --- a/web/src/lib/canvas/tool-registry/tool-definition.ts +++ b/web/src/lib/canvas/tool-registry/tool-definition.ts @@ -81,6 +81,8 @@ export type ToolbarHandlers = { onNodeUpload: (node: CanvasNodeData) => void; onNodeDownload: (node: CanvasNodeData) => void; onNodeSaveAsset: (node: CanvasNodeData) => void; + /** 从已有图片或视频创建并自动连接一个转换节点。 */ + onNodeCreateConversion?: (node: CanvasNodeData) => void; onNodeMaskEdit: (node: CanvasNodeData) => void; onNodeEmotion: (node: CanvasNodeData) => void; onNodePortraitTexture: (node: CanvasNodeData) => void; diff --git a/web/src/lib/media-conversion/contracts.ts b/web/src/lib/media-conversion/contracts.ts new file mode 100644 index 000000000..55c156e85 --- /dev/null +++ b/web/src/lib/media-conversion/contracts.ts @@ -0,0 +1,89 @@ +import type { CanvasNodeData } from "@/types/canvas"; + +export const MEDIA_CONVERSION_SCHEMA_VERSION = 1 as const; + +export const MEDIA_CONVERSION_NODE_TYPE = "media-conversion" as const; + +export type MediaConversionOperation = "grayscale" | "edge-canny" | "lineart" | "depth" | "pose" | "cutout"; +export type MediaConversionStatus = "idle" | "processing" | "completed" | "stale" | "skipped" | "unavailable" | "error"; +export type MediaConversionOutputKind = "image" | "video"; + +export type MediaConversionNodeState = { + schemaVersion: typeof MEDIA_CONVERSION_SCHEMA_VERSION; + operation: MediaConversionOperation; + status: MediaConversionStatus; + sourceNodeId?: string; + sourceFingerprint?: string; + outputKind?: MediaConversionOutputKind; + resultStorageKey?: string; + resultWidth?: number; + resultHeight?: number; + detectedPeople?: number; + errorCode?: string; + errorMessage?: string; + startedAt?: string; + updatedAt?: string; +}; + +export const MEDIA_CONVERSION_OPERATION_LABELS: Record = { + grayscale: "灰度图", + "edge-canny": "Canny 边缘", + lineart: "AI 线稿", + depth: "深度图", + pose: "姿态骨架", + cutout: "透明抠图", +}; + +export const MEDIA_CONVERSION_OPERATION_DESCRIPTIONS: Record = { + grayscale: "普通图像算法，适合先检查明暗层次", + "edge-canny": "普通图像算法，提取清晰轮廓", + lineart: "本地 ControlNet Aux 线稿预处理，首次运行需要加载模型", + depth: "本地 Depth Anything V2 Small，首次运行需要加载模型", + pose: "本地 OpenPose 人体骨架预处理，未检测到人物时会跳过", + cutout: "需要安装并验证本地抠图模型", +}; + +export function createDefaultMediaConversionState(): MediaConversionNodeState { + return { + schemaVersion: MEDIA_CONVERSION_SCHEMA_VERSION, + operation: "edge-canny", + status: "idle", + }; +} + +export function mediaConversionSourceFingerprint(node: CanvasNodeData) { + const metadata = node.metadata; + const source = [ + node.id, + metadata?.storageKey || "", + compactSource(metadata?.content || metadata?.previewContent || ""), + metadata?.mimeType || "", + metadata?.bytes || "", + metadata?.naturalWidth || "", + metadata?.naturalHeight || "", + metadata?.durationMs || "", + ].join("|"); + let hash = 2166136261; + for (let index = 0; index < source.length; index += 1) { + hash ^= source.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return `v1-${(hash >>> 0).toString(16).padStart(8, "0")}`; +} + +export function mediaConversionOperationLabel(operation: MediaConversionOperation) { + return MEDIA_CONVERSION_OPERATION_LABELS[operation]; +} + +export function mediaConversionOperationDescription(operation: MediaConversionOperation) { + return MEDIA_CONVERSION_OPERATION_DESCRIPTIONS[operation]; +} + +export function isLocalImageOperation(operation: MediaConversionOperation) { + return operation === "grayscale" || operation === "edge-canny"; +} + +function compactSource(value: string) { + if (value.length <= 2048) return value; + return `${value.slice(0, 1024)}|${value.slice(-1024)}|${value.length}`; +} diff --git a/web/src/lib/media-conversion/local-converter.ts b/web/src/lib/media-conversion/local-converter.ts new file mode 100644 index 000000000..44547da00 --- /dev/null +++ b/web/src/lib/media-conversion/local-converter.ts @@ -0,0 +1,228 @@ +import type { MediaConversionOperation } from "./contracts"; + +export type LocalImageConversionResult = { + blob: Blob; + width: number; + height: number; +}; + +export class LocalImageConversionError extends Error { + constructor(readonly code: "model_missing" | "source_unreadable" | "browser_unsupported" | "aborted", message: string) { + super(message); + this.name = "LocalImageConversionError"; + } +} + +export async function convertImageLocally(sourceUrl: string, operation: MediaConversionOperation, options: { signal?: AbortSignal; maxDimension?: number } = {}): Promise { + throwIfAborted(options.signal); + if (operation !== "grayscale" && operation !== "edge-canny") { + throw new LocalImageConversionError("model_missing", "该转换需要先安装并验证本地模型"); + } + + const image = await loadImage(sourceUrl, options.signal); + throwIfAborted(options.signal); + const maxDimension = Math.max(256, options.maxDimension || 1024); + const scale = Math.min(1, maxDimension / Math.max(image.naturalWidth, image.naturalHeight)); + const width = Math.max(1, Math.round(image.naturalWidth * scale)); + const height = Math.max(1, Math.round(image.naturalHeight * scale)); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) throw new LocalImageConversionError("browser_unsupported", "当前浏览器无法创建图片处理画布"); + context.drawImage(image, 0, 0, width, height); + const input = context.getImageData(0, 0, width, height); + const output = transformPixels(input.data, width, height, operation, options.signal); + const outputImage = new ImageData(output, width, height); + context.putImageData(outputImage, 0, 0); + const blob = await canvasToBlob(canvas, options.signal); + throwIfAborted(options.signal); + return { blob, width, height }; +} + +/** Pure pixel transform used by the browser executor and unit tests. */ +export function transformPixels(source: Uint8ClampedArray, width: number, height: number, operation: "grayscale" | "edge-canny", signal?: AbortSignal) { + if (source.length !== width * height * 4) throw new Error("图片像素尺寸不匹配"); + if (operation === "grayscale") return grayscalePixels(source, width, height, signal); + return cannyPixels(source, width, height, signal); +} + +function grayscalePixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) { + const output = new Uint8ClampedArray(source); + for (let y = 0; y < height; y += 1) { + throwIfAborted(signal); + for (let x = 0; x < width; x += 1) { + const index = (y * width + x) * 4; + const value = Math.round(source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114); + output[index] = value; + output[index + 1] = value; + output[index + 2] = value; + } + } + return output; +} + +function cannyPixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) { + const gray = new Float32Array(width * height); + for (let y = 0; y < height; y += 1) { + throwIfAborted(signal); + for (let x = 0; x < width; x += 1) { + const index = (y * width + x) * 4; + gray[y * width + x] = source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114; + } + } + + const blurred = gaussianBlur(gray, width, height, signal); + const magnitude = new Float32Array(width * height); + const direction = new Float32Array(width * height); + let maximum = 0; + for (let y = 1; y < height - 1; y += 1) { + throwIfAborted(signal); + for (let x = 1; x < width - 1; x += 1) { + const index = y * width + x; + const horizontal = blurred[index + 1] - blurred[index - 1]; + const vertical = blurred[index + width] - blurred[index - width]; + const value = Math.hypot(horizontal, vertical); + magnitude[index] = value; + direction[index] = Math.atan2(vertical, horizontal); + maximum = Math.max(maximum, value); + } + } + + const thinned = nonMaximumSuppression(magnitude, direction, width, height, signal); + const high = maximum * 0.28; + const low = high * 0.45; + const edges = new Uint8Array(width * height); + const stack: number[] = []; + for (let index = 0; index < thinned.length; index += 1) { + if (thinned[index] >= high && high > 0) { + edges[index] = 2; + stack.push(index); + } else if (thinned[index] >= low && low > 0) { + edges[index] = 1; + } + } + while (stack.length) { + throwIfAborted(signal); + const index = stack.pop()!; + const x = index % width; + const y = Math.floor(index / width); + for (let offsetY = -1; offsetY <= 1; offsetY += 1) { + for (let offsetX = -1; offsetX <= 1; offsetX += 1) { + if (!offsetX && !offsetY) continue; + const nextX = x + offsetX; + const nextY = y + offsetY; + if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) continue; + const next = nextY * width + nextX; + if (edges[next] !== 1) continue; + edges[next] = 2; + stack.push(next); + } + } + } + + const output = new Uint8ClampedArray(source.length); + for (let y = 0; y < height; y += 1) { + throwIfAborted(signal); + for (let x = 0; x < width; x += 1) { + const index = y * width + x; + const outputIndex = index * 4; + const value = edges[index] === 2 ? 0 : 255; + output[outputIndex] = value; + output[outputIndex + 1] = value; + output[outputIndex + 2] = value; + output[outputIndex + 3] = source[outputIndex + 3]; + } + } + return output; +} + +function gaussianBlur(source: Float32Array, width: number, height: number, signal?: AbortSignal) { + const output = new Float32Array(source.length); + const kernel = [1, 2, 1, 2, 4, 2, 1, 2, 1]; + for (let y = 0; y < height; y += 1) { + throwIfAborted(signal); + for (let x = 0; x < width; x += 1) { + let value = 0; + let weight = 0; + for (let offsetY = -1; offsetY <= 1; offsetY += 1) { + for (let offsetX = -1; offsetX <= 1; offsetX += 1) { + const nextX = Math.min(width - 1, Math.max(0, x + offsetX)); + const nextY = Math.min(height - 1, Math.max(0, y + offsetY)); + const kernelWeight = kernel[(offsetY + 1) * 3 + offsetX + 1]; + value += source[nextY * width + nextX] * kernelWeight; + weight += kernelWeight; + } + } + output[y * width + x] = value / weight; + } + } + return output; +} + +function nonMaximumSuppression(magnitude: Float32Array, direction: Float32Array, width: number, height: number, signal?: AbortSignal) { + const output = new Float32Array(magnitude.length); + for (let y = 1; y < height - 1; y += 1) { + throwIfAborted(signal); + for (let x = 1; x < width - 1; x += 1) { + const index = y * width + x; + let angle = (direction[index] * 180) / Math.PI; + if (angle < 0) angle += 180; + const value = magnitude[index]; + let before = 0; + let after = 0; + if (angle < 22.5 || angle >= 157.5) { + before = magnitude[index - 1]; + after = magnitude[index + 1]; + } else if (angle < 67.5) { + before = magnitude[index - width + 1]; + after = magnitude[index + width - 1]; + } else if (angle < 112.5) { + before = magnitude[index - width]; + after = magnitude[index + width]; + } else { + before = magnitude[index - width - 1]; + after = magnitude[index + width + 1]; + } + output[index] = value >= before && value >= after ? value : 0; + } + } + return output; +} + +async function loadImage(sourceUrl: string, signal?: AbortSignal) { + if (!sourceUrl) throw new LocalImageConversionError("source_unreadable", "没有可读取的图片输入"); + const image = new Image(); + image.crossOrigin = "anonymous"; + return await new Promise((resolve, reject) => { + const abort = () => { + image.src = ""; + reject(new LocalImageConversionError("aborted", "转换已取消")); + }; + if (signal?.aborted) return abort(); + signal?.addEventListener("abort", abort, { once: true }); + image.onload = () => { + signal?.removeEventListener("abort", abort); + resolve(image); + }; + image.onerror = () => { + signal?.removeEventListener("abort", abort); + reject(new LocalImageConversionError("source_unreadable", "图片无法读取，可能是跨域资源或已失效")); + }; + image.src = sourceUrl; + }); +} + +function canvasToBlob(canvas: HTMLCanvasElement, signal?: AbortSignal) { + throwIfAborted(signal); + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (blob) resolve(blob); + else reject(new LocalImageConversionError("browser_unsupported", "当前浏览器无法导出图片结果")); + }, "image/png"); + }); +} + +function throwIfAborted(signal?: AbortSignal): asserts signal is AbortSignal | undefined { + if (signal?.aborted) throw new LocalImageConversionError("aborted", "转换已取消"); +} diff --git a/web/src/lib/plugins/builtin/index.ts b/web/src/lib/plugins/builtin/index.ts index c19c74ff4..04d78328b 100644 --- a/web/src/lib/plugins/builtin/index.ts +++ b/web/src/lib/plugins/builtin/index.ts @@ -3,4 +3,5 @@ import "./prompt-optimizer"; import "./workflows"; import "./portrait-clearance"; import "./ai-art-critique"; +import "./media-conversion"; import "./editor/editor-shell"; diff --git a/web/src/lib/plugins/builtin/media-conversion.ts b/web/src/lib/plugins/builtin/media-conversion.ts new file mode 100644 index 000000000..9803e1fe4 --- /dev/null +++ b/web/src/lib/plugins/builtin/media-conversion.ts @@ -0,0 +1,37 @@ +import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts"; + +import { registerPlugin } from "../plugin-registry"; +import { PLUGIN_API_VERSION, type PluginManifest, type RegisteredPlugin } from "../plugin-types"; + +export const MEDIA_CONVERSION_PLUGIN_ID = "media-conversion"; + +const manifest: PluginManifest = { + apiVersion: PLUGIN_API_VERSION, + id: MEDIA_CONVERSION_PLUGIN_ID, + name: "媒体转换节点", + version: "0.1.0", + publishedAt: "2026-09-08", + updatedAt: "2026-09-08", + description: "在画布中提供图片转换节点，支持灰度、Canny 边缘、AI 线稿、本地 Depth Anything V2 深度图和 OpenPose 姿态骨架。", + documentation: + "# 媒体转换节点\n\n连接图片到转换节点后，可以在节点内选择灰度图、Canny 边缘、AI 线稿、深度图或姿态骨架。AI 线稿使用本机 ControlNet Aux 预处理模型，深度图使用本机已安装的 Depth Anything V2 Small 模型，姿态骨架使用 ControlNet Aux 的 OpenPose 人体预处理模型运行；这些操作都不会加载 Stable Diffusion 重绘管线。姿态转换只绘制人体骨架，未检测到人物时会跳过并提示。视频转换和透明抠图模型仍在验证，暂时会明确提示不可用。\n\n该插件只读取当前节点的媒体输入，并把结果保存回当前画布节点的本地素材存储。", + author: "影策团队", + surfaces: ["node"], + permissions: ["canvas.read", "canvas.write", "media.read"], + trusted: true, + runtime: { web: "declarative" }, + contributes: { + transforms: [ + { + id: MEDIA_CONVERSION_NODE_TYPE, + input: "media", + output: "media", + runtime: "declarative", + }, + ], + }, +}; + +export const mediaConversionPlugin: RegisteredPlugin = { manifest }; + +registerPlugin(mediaConversionPlugin); diff --git a/web/src/lib/plugins/plugin-types.ts b/web/src/lib/plugins/plugin-types.ts index 09c420b28..374a16b9a 100644 --- a/web/src/lib/plugins/plugin-types.ts +++ b/web/src/lib/plugins/plugin-types.ts @@ -103,8 +103,10 @@ export type PluginCanvasNodeContribution = { defaultSize: { width: number; height: number }; schema: Record; renderer: "declarative" | "sandbox"; - /** Optional input contract for nodes that consume one media kind. */ - acceptsInputKind?: "image" | "video" | "audio" | "text"; + /** Optional input contract for nodes that consume one or more media kinds. */ + acceptsInputKind?: "image" | "video" | "audio" | "text" | Array<"image" | "video" | "audio" | "text">; + /** Optional maximum number of direct inputs. */ + maxInputCount?: number; /** Analysis/sink nodes can hide the right-side output connection. */ showOutputConnection?: boolean; }; diff --git a/web/src/pages/admin/plugins/plugins-page.tsx b/web/src/pages/admin/plugins/plugins-page.tsx index 53d640e8e..39b9bbe58 100644 --- a/web/src/pages/admin/plugins/plugins-page.tsx +++ b/web/src/pages/admin/plugins/plugins-page.tsx @@ -10,6 +10,7 @@ import "@/lib/plugins/builtin"; import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle"; import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer"; import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows"; +import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion"; import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry"; import type { PluginManifest, PluginManifestV2 } from "@/lib/plugins/plugin-types"; import { fetchAdminPlugins, setPluginPlatformAvailability, uninstallPlugin, uploadPlugin, type AdminPluginState, type BackendPlugin, type PluginManagement } from "@/services/api/plugins"; @@ -26,7 +27,7 @@ type AdminPluginItem = { error?: string; }; -const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance"]); +const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", MEDIA_CONVERSION_PLUGIN_ID]); export default function AdminPluginsPage() { const { message, modal } = App.useApp(); diff --git a/web/src/pages/canvas/project.tsx b/web/src/pages/canvas/project.tsx index 8a17176ae..6da4c3dd0 100644 --- a/web/src/pages/canvas/project.tsx +++ b/web/src/pages/canvas/project.tsx @@ -72,7 +72,7 @@ import { CanvasLocalAgentPanel } from "@/components/canvas/canvas-local-agent-pa import { useFocusMode } from "@/hooks/use-focus-mode"; import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store"; import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; -import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays"; +import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay, type PendingConnectionCreate } from "@/components/canvas/canvas-workspace-overlays"; import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer"; import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer"; import { CanvasFreeformEmptyState, CanvasLinkedProjectEmptyState, CanvasShortDramaEmptyState, CanvasShortDramaGuide, CanvasStoryInputNodeContent, CanvasStylePlaceholderNodeContent } from "@/components/canvas/canvas-short-drama-entry"; @@ -1018,6 +1018,20 @@ function InfiniteCanvasPage() { .filter((node) => selectedNodeIds.has(node.id) && !batchSourceRestriction(node)) .map((node) => node.id), [nodes, selectedNodeIds]); + const createConversionFromSource = useCallback((source: CanvasNodeData) => { + if (source.type !== CanvasNodeType.Image && source.type !== CanvasNodeType.Video) return; + const spec = getNodeSpec(CanvasNodeType.MediaConversion); + const pending: PendingConnectionCreate = { + connection: { nodeId: source.id, handleType: "source", anchorRatio: 0.5 }, + position: { + x: source.position.x + source.width + 96 + spec.width / 2, + y: source.position.y + source.height / 2, + }, + quick: true, + }; + void createConnectedNode(CanvasNodeType.MediaConversion, pending); + }, [createConnectedNode]); + const handleCanvasSelectionStart = useCallback(() => { setContextMenu(null); }, []); @@ -1048,6 +1062,8 @@ function InfiniteCanvasPage() { } else if (node.type === ART_CRITIQUE_NODE_TYPE) { setDialogNodeId(null); setArtCritiqueNodeId(node.id); + } else if (node.type === CanvasNodeType.MediaConversion) { + setDialogNodeId(null); } else { // 选择参考媒体时保留当前工作流配置面板，避免点击图片后配置“返回/消失”。 // 没有工作流配置面板时，媒体节点仍按原逻辑打开自己的面板。 @@ -1065,7 +1081,7 @@ function InfiniteCanvasPage() { const handleNodeDragEnd = useCallback((nodeId: string) => { const node = nodesRef.current.find((item) => item.id === nodeId); - if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) { + if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) { setDialogNodeId(null); return; } @@ -1895,7 +1911,7 @@ function InfiniteCanvasPage() { const renderCanvasNodePanel = useCallback( (panelNode: CanvasNodeData) => { - if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing) return null; + if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing || panelNode.type === CanvasNodeType.MediaConversion) return null; return panelNode.type === CanvasNodeType.Config ? ( handleUploadRequest(node.id)} onDownload={downloadNodeImage} onSaveAsset={(node) => void saveNodeAsset(node)} + onCreateConversion={createConversionFromSource} onAnnotate={(node) => setAnnotationNodeId(node.id)} onMaskEdit={(node) => setMaskEditNodeId(node.id)} onEmotion={(node) => { diff --git a/web/src/pages/canvas/use-canvas-connection-controller.ts b/web/src/pages/canvas/use-canvas-connection-controller.ts index e08d1410e..2a99079e0 100644 --- a/web/src/pages/canvas/use-canvas-connection-controller.ts +++ b/web/src/pages/canvas/use-canvas-connection-controller.ts @@ -200,7 +200,7 @@ export function useCanvasConnectionController({ setContextMenu(null); }, [config, connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]); - const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => { + const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => { const nodeType = type; if (nodeType === CanvasNodeType.Drawing && !isDrawingEngineAvailable(defaultDrawingEngine, tldrawLicenseKey)) { message.error("当前生产构建未配置 tldraw License Key，不能创建 tldraw 绘图"); @@ -279,6 +279,12 @@ export function useCanvasConnectionController({ setConnecting(null); return; } + if (batchSourceNodeIds.length && nodeType === CanvasNodeType.MediaConversion) { + message.error("转换节点一次只能连接一个输入，请先连接到普通节点"); + closeConnectionCreateMenu(); + setConnecting(null); + return; + } const batchPlan = batchSourceNodeIds.length ? planBatchConnections({ sourceNodeIds: batchSourceNodeIds, targetNodeId: newNode.id, nodes: [...nodesRef.current, newNode], connections: connectionsRef.current, config, allowCapacityOverflow: true }) : null; @@ -298,7 +304,7 @@ export function useCanvasConnectionController({ setConnections(nextConnections); setSelectedNodeIds(new Set([newNode.id])); setSelectedConnectionId(null); - if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id); + if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id); const skippedCount = batchPlan.skipped.length; const duplicateCount = batchPlan.duplicates.length; const suffix = skippedCount || duplicateCount ? `，跳过 ${skippedCount + duplicateCount} 个` : ""; @@ -364,17 +370,18 @@ export function useCanvasConnectionController({ setSelectedNodeIds(new Set([newNode.id])); setSelectedConnectionId(null); if (nodeType === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id); - else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id); + else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id); closeConnectionCreateMenu(); setConnecting(null); }, [closeConnectionCreateMenu, config, connectionsRef, defaultDrawingEngine, message, nodesRef, projectId, runtimeStatuses, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, tldrawLicenseKey]); - const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => { + const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => { const nodeType = type; if (nodeType === CanvasNodeType.Config) { if (workflowProvider && !workflowProviderPluginEnabled(runtimeStatuses, workflowProvider)) return `${workflowProvider === "runninghub" ? "RunningHub" : "ComfyUI"} 工作流插件未启用`; } if (pending.batchSourceNodeIds?.length) { + if (nodeType === CanvasNodeType.MediaConversion) return "转换节点一次只能连接一个输入"; if (nodeType === CanvasNodeType.Drawing || nodeType === CanvasNodeType.Config) return "批量连接暂不支持此节点类型"; const pendingNode: CanvasNodeData = { id: "__pending-connection-node__", type: nodeType, title: "", position: pending.position, width: getNodeSpec(nodeType).width, height: getNodeSpec(nodeType).height }; const plan = planBatchConnections({ sourceNodeIds: pending.batchSourceNodeIds, targetNodeId: pendingNode.id, nodes: [...nodesRef.current, pendingNode], connections: connectionsRef.current, config, allowCapacityOverflow: true }); diff --git a/web/src/pages/canvas/use-canvas-node-operations.ts b/web/src/pages/canvas/use-canvas-node-operations.ts index 3431f14ec..0e16916c7 100644 --- a/web/src/pages/canvas/use-canvas-node-operations.ts +++ b/web/src/pages/canvas/use-canvas-node-operations.ts @@ -153,7 +153,7 @@ export function useCanvasNodeOperations({ if (workflowTitle) node.title = workflowTitle; commitNodes([...nodesRef.current, node]); selectNodes(new Set([node.id])); - if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id); + if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== CanvasNodeType.MediaConversion && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id); }, [commitNodes, defaultDrawingEngine, effectiveConfig.comfyBridge.enabled, effectiveConfig.comfyBridge.workflows.length, effectiveConfig.runningHub.enabled, effectiveConfig.runningHub.workflows.length, getCanvasCenter, message, nodesRef, runtimeStatuses, selectNodes, setDialogNodeId, tldrawLicenseKey]); const createFolder = useCallback((position?: Position, linked?: { id: string; projectId: string; title: string; style: CanvasFolderStyle; theme: CanvasFolderTheme; createdAt: string }) => { @@ -400,7 +400,7 @@ export function useCanvasNodeOperations({ const sourceNode = sourceByTargetId.get(targetNode.id); if (sourceNode) cloneDrawingForNode(sourceNode, targetNode, "绘图副本保存失败，请重新复制"); }); - if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id); + if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== CanvasNodeType.MediaConversion && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id); }, [cloneDrawingForNode, commitConnections, commitNodes, connectionsRef, nodesRef, selectNodes, setDialogNodeId]); const setPrimaryVersion = useCallback((nodeId: string) => { diff --git a/web/src/pages/canvas/use-canvas-viewport-controller.ts b/web/src/pages/canvas/use-canvas-viewport-controller.ts index 7dca654c1..e06eee97a 100644 --- a/web/src/pages/canvas/use-canvas-viewport-controller.ts +++ b/web/src/pages/canvas/use-canvas-viewport-controller.ts @@ -124,7 +124,7 @@ export function useCanvasViewportController({ const scale = Math.min(1.18, Math.max(viewportRef.current.k, 0.72)); transitionViewportTo({ x: size.width / 2 - (node.position.x + node.width / 2) * scale, y: size.height / 2 - (node.position.y + node.height / 2) * scale, k: scale }); selectFocusedNode(node.id); - setDialogNodeId(node.type === CanvasNodeType.Drawing ? null : node.id); + setDialogNodeId(node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion ? null : node.id); }, [nodesRef, selectFocusedNode, setDialogNodeId, size.height, size.width, transitionViewportTo, viewportRef]); const setZoomScale = useCallback((scale: number) => { diff --git a/web/src/pages/plugins/index.tsx b/web/src/pages/plugins/index.tsx index ce67f370b..d99be8184 100644 --- a/web/src/pages/plugins/index.tsx +++ b/web/src/pages/plugins/index.tsx @@ -9,6 +9,7 @@ import "@/lib/plugins/builtin"; import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle"; import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer"; import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows"; +import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion"; import { ART_CRITIQUE_PLUGIN_ID } from "@/lib/art-critique/contracts"; import type { PluginManifest, PluginManifestV2, RegisteredPlugin } from "@/lib/plugins/plugin-types"; import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle"; @@ -623,7 +624,7 @@ function toRegisteredPlugin(plugin: BackendPlugin): RegisteredPlugin { } function isOfficialApplicationPlugin(pluginId: string) { - return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID].includes(pluginId); + return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID, MEDIA_CONVERSION_PLUGIN_ID].includes(pluginId); } function pluginSourceLabel(plugin: RegisteredPlugin, state?: PluginState) { diff --git a/web/src/services/depth-runtime.ts b/web/src/services/depth-runtime.ts new file mode 100644 index 000000000..44f620c16 --- /dev/null +++ b/web/src/services/depth-runtime.ts @@ -0,0 +1,202 @@ +import { LocalRuntimeClientError } from "@/services/local-runtime-session"; +import type { LocalRuntimeTransport } from "@/services/local-runtime"; +import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store"; + +const MAX_IMAGE_BYTES = 12 * 1024 * 1024; +const MAX_RESPONSE_BYTES = 32 * 1024 * 1024; +const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/; + +export type LocalDepthResult = { + blob: Blob; + width: number; + height: number; + modelId: string; + device: "cpu"; +}; + +export type LocalDepthRuntimeStatus = { + ok: true; + module: "depth-estimation"; + apiVersion: 1; + ready: boolean; + configured: boolean; + modelCached: boolean; + loaded: boolean; + device: "cpu"; + modelId: string; + reason?: string; +}; + +export async function runLocalDepthEstimation(sourceUrl: string, signal?: AbortSignal): Promise { + const dataUrl = await sourceToDataUrl(sourceUrl, signal); + const value = await requestJson("/depth-estimation/run", { + method: "POST", + body: JSON.stringify({ schemaVersion: 1, operation: "depth", dataUrl }), + headers: { "content-type": "application/json" }, + signal, + }); + if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果响应无效"); + } + return parseResult(value.result); +} + +export async function readLocalDepthRuntimeStatus(signal?: AbortSignal) { + return parseStatus(await requestJson("/depth-estimation/status", { method: "GET", signal })); +} + +async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) { + if (sourceUrl.startsWith("data:")) { + const match = DATA_URL_PATTERN.exec(sourceUrl); + if (!match) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400); + const bytes = bytesFromBase64(match[2]!); + assertImageSize(bytes); + return `data:${match[1]};base64,${bytesToBase64(bytes)}`; + } + let response: Response; + try { + response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal }); + } catch (error) { + if (signal?.aborted) throw error; + throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取"); + } + if (!response.ok) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取", response.status); + const mimeType = imageMime(response.headers.get("content-type") || response.type); + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) { + throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400); + } + const bytes = new Uint8Array(await response.arrayBuffer()); + assertImageSize(bytes); + return `data:${mimeType};base64,${bytesToBase64(bytes)}`; +} + +async function requestJson(path: string, init: RequestInit = {}) { + const { response, body: responseBody } = await requestRuntimeResponse(path, init); + const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody; + if (!response.ok) { + const code = isRecord(body) && typeof body.code === "string" ? body.code : "depth_runtime_unavailable"; + const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机深度运行时不可用"; + throw new LocalRuntimeClientError(code, message, response.status); + } + return body as T; +} + +async function requestRuntimeResponse(path: string, init: RequestInit = {}) { + let refreshed = false; + while (true) { + let response: Response; + try { + response = await (await ensureTransport()).request(path, init); + } catch (error) { + if (refreshed || !isSessionRefreshError(error)) throw error; + await reconnectTransport(); + refreshed = true; + continue; + } + if (!refreshed && response.status === 401) { + await reconnectTransport(); + refreshed = true; + continue; + } + if (response.status === 403) { + const body = await parseJsonResponse(response); + if (!refreshed && shouldRefreshSession(response, body)) { + await reconnectTransport(); + refreshed = true; + continue; + } + return { response, body }; + } + return { response }; + } +} + +async function ensureTransport(): Promise { + const state = useLocalRuntimeStore.getState(); + if (state.connection !== "connected") await state.connect(); + const current = useLocalRuntimeStore.getState(); + if (current.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", current.error || "本机深度运行时不可用"); + return getLocalRuntimeSessionClient(); +} + +async function reconnectTransport() { + const client = getLocalRuntimeSessionClient(); + client.revokeLocalSession(); + await useLocalRuntimeStore.getState().connect(); + const state = useLocalRuntimeStore.getState(); + if (state.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", state.error || "本机深度运行时不可用"); +} + +async function parseJsonResponse(response: Response) { + const text = await boundedText(response, MAX_RESPONSE_BYTES); + try { + return text ? JSON.parse(text) as unknown : {}; + } catch { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应无效", response.status); + } +} + +async function boundedText(response: Response, limit: number) { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status); + const text = await response.text(); + if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status); + return text; +} + +function parseResult(value: Record): LocalDepthResult { + if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果无效"); + } + const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机深度结果无效"); + if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果过大"); + return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" }; +} + +function parseStatus(value: unknown): LocalDepthRuntimeStatus { + if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度状态无效"); + } + return value as LocalDepthRuntimeStatus; +} + +function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" { + const mime = value.startsWith("image/") ? value.split(";", 1)[0] : ""; + if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime; + throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400); +} + +function assertImageSize(bytes: Uint8Array) { + if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400); +} + +function bytesToBase64(bytes: Uint8Array) { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +function bytesFromBase64(value: string, code = "depth_input_invalid", message = "深度转换输入图片无效") { + let binary: string; + try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "depth_input_invalid" ? 400 : 0); } + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return bytes; +} + +function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError { + return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code); +} + +function shouldRefreshSession(response: Response, body: unknown) { + return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied"); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0; +} diff --git a/web/src/services/lineart-runtime.ts b/web/src/services/lineart-runtime.ts new file mode 100644 index 000000000..17f892d06 --- /dev/null +++ b/web/src/services/lineart-runtime.ts @@ -0,0 +1,202 @@ +import { LocalRuntimeClientError } from "@/services/local-runtime-session"; +import type { LocalRuntimeTransport } from "@/services/local-runtime"; +import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store"; + +const MAX_IMAGE_BYTES = 12 * 1024 * 1024; +const MAX_RESPONSE_BYTES = 32 * 1024 * 1024; +const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/; + +export type LocalLineartResult = { + blob: Blob; + width: number; + height: number; + modelId: string; + device: "cpu"; +}; + +export type LocalLineartRuntimeStatus = { + ok: true; + module: "lineart-estimation"; + apiVersion: 1; + ready: boolean; + configured: boolean; + modelCached: boolean; + loaded: boolean; + device: "cpu"; + modelId: string; + reason?: string; +}; + +export async function runLocalLineartEstimation(sourceUrl: string, signal?: AbortSignal): Promise { + const dataUrl = await sourceToDataUrl(sourceUrl, signal); + const value = await requestJson("/lineart-estimation/run", { + method: "POST", + body: JSON.stringify({ schemaVersion: 1, operation: "lineart", dataUrl }), + headers: { "content-type": "application/json" }, + signal, + }); + if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果响应无效"); + } + return parseResult(value.result); +} + +export async function readLocalLineartRuntimeStatus(signal?: AbortSignal) { + return parseStatus(await requestJson("/lineart-estimation/status", { method: "GET", signal })); +} + +async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) { + if (sourceUrl.startsWith("data:")) { + const match = DATA_URL_PATTERN.exec(sourceUrl); + if (!match) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400); + const bytes = bytesFromBase64(match[2]!); + assertImageSize(bytes); + return `data:${match[1]};base64,${bytesToBase64(bytes)}`; + } + let response: Response; + try { + response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal }); + } catch (error) { + if (signal?.aborted) throw error; + throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取"); + } + if (!response.ok) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取", response.status); + const mimeType = imageMime(response.headers.get("content-type") || response.type); + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) { + throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400); + } + const bytes = new Uint8Array(await response.arrayBuffer()); + assertImageSize(bytes); + return `data:${mimeType};base64,${bytesToBase64(bytes)}`; +} + +async function requestJson(path: string, init: RequestInit = {}) { + const { response, body: responseBody } = await requestRuntimeResponse(path, init); + const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody; + if (!response.ok) { + const code = isRecord(body) && typeof body.code === "string" ? body.code : "lineart_runtime_unavailable"; + const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机 AI 线稿运行时不可用"; + throw new LocalRuntimeClientError(code, message, response.status); + } + return body as T; +} + +async function requestRuntimeResponse(path: string, init: RequestInit = {}) { + let refreshed = false; + while (true) { + let response: Response; + try { + response = await (await ensureTransport()).request(path, init); + } catch (error) { + if (refreshed || !isSessionRefreshError(error)) throw error; + await reconnectTransport(); + refreshed = true; + continue; + } + if (!refreshed && response.status === 401) { + await reconnectTransport(); + refreshed = true; + continue; + } + if (response.status === 403) { + const body = await parseJsonResponse(response); + if (!refreshed && shouldRefreshSession(response, body)) { + await reconnectTransport(); + refreshed = true; + continue; + } + return { response, body }; + } + return { response }; + } +} + +async function ensureTransport(): Promise { + const state = useLocalRuntimeStore.getState(); + if (state.connection !== "connected") await state.connect(); + const current = useLocalRuntimeStore.getState(); + if (current.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", current.error || "本机 AI 线稿运行时不可用"); + return getLocalRuntimeSessionClient(); +} + +async function reconnectTransport() { + const client = getLocalRuntimeSessionClient(); + client.revokeLocalSession(); + await useLocalRuntimeStore.getState().connect(); + const state = useLocalRuntimeStore.getState(); + if (state.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", state.error || "本机 AI 线稿运行时不可用"); +} + +async function parseJsonResponse(response: Response) { + const text = await boundedText(response, MAX_RESPONSE_BYTES); + try { + return text ? JSON.parse(text) as unknown : {}; + } catch { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应无效", response.status); + } +} + +async function boundedText(response: Response, limit: number) { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status); + const text = await response.text(); + if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status); + return text; +} + +function parseResult(value: Record): LocalLineartResult { + if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果无效"); + } + const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机 AI 线稿结果无效"); + if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果过大"); + return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" }; +} + +function parseStatus(value: unknown): LocalLineartRuntimeStatus { + if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿状态无效"); + } + return value as LocalLineartRuntimeStatus; +} + +function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" { + const mime = value.startsWith("image/") ? value.split(";", 1)[0] : ""; + if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime; + throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400); +} + +function assertImageSize(bytes: Uint8Array) { + if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400); +} + +function bytesToBase64(bytes: Uint8Array) { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +function bytesFromBase64(value: string, code = "lineart_input_invalid", message = "AI 线稿输入图片无效") { + let binary: string; + try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "lineart_input_invalid" ? 400 : 0); } + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return bytes; +} + +function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError { + return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code); +} + +function shouldRefreshSession(response: Response, body: unknown) { + return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied"); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0; +} diff --git a/web/src/services/local-runtime.ts b/web/src/services/local-runtime.ts index 01bf8058c..ab1d68766 100644 --- a/web/src/services/local-runtime.ts +++ b/web/src/services/local-runtime.ts @@ -1,7 +1,7 @@ import { LocalRuntimeClientError } from "@/services/local-runtime-session"; -export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance"; -export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read"; +export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation"; +export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read" | "depth:status" | "depth:run" | "lineart:status" | "lineart:run" | "pose:status" | "pose:run"; export type LocalRuntimeModuleDescriptor = { id: LocalRuntimeModuleId; @@ -28,6 +28,9 @@ const MODULE_SCOPES: Record "canvas-agent": new Set(["canvas:connect"]), dreamina: new Set(["dreamina:status", "dreamina:login", "dreamina:logout", "dreamina:run", "dreamina:models", "dreamina:generate"]), "portrait-clearance": new Set(["portrait:status", "portrait:model", "portrait:run", "portrait:read"]), + "depth-estimation": new Set(["depth:status", "depth:run"]), + "lineart-estimation": new Set(["lineart:status", "lineart:run"]), + "pose-estimation": new Set(["pose:status", "pose:run"]), }; export async function readLocalRuntimeStatus(client: LocalRuntimeTransport, signal?: AbortSignal): Promise { @@ -136,7 +139,7 @@ function isRecord(value: unknown): value is Record { } function isModuleId(value: unknown): value is LocalRuntimeModuleId { - return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance"; + return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance" || value === "depth-estimation" || value === "lineart-estimation" || value === "pose-estimation"; } function invalidResponse(status: number) { diff --git a/web/src/services/pose-runtime.ts b/web/src/services/pose-runtime.ts new file mode 100644 index 000000000..c5c53c47b --- /dev/null +++ b/web/src/services/pose-runtime.ts @@ -0,0 +1,209 @@ +import { LocalRuntimeClientError } from "@/services/local-runtime-session"; +import type { LocalRuntimeTransport } from "@/services/local-runtime"; +import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store"; + +const MAX_IMAGE_BYTES = 12 * 1024 * 1024; +const MAX_RESPONSE_BYTES = 32 * 1024 * 1024; +const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/; + +export type LocalPoseResult = { + blob: Blob; + width: number; + height: number; + modelId: string; + device: "cpu"; + backend: "openpose-body"; + personCount: number; + people?: unknown[]; +}; + +export type LocalPoseRuntimeStatus = { + ok: true; + module: "pose-estimation"; + apiVersion: 1; + ready: boolean; + configured: boolean; + modelCached: boolean; + loaded: boolean; + device: "cpu"; + modelId: string; + backend: "openpose-body"; + reason?: string; +}; + +export async function runLocalPoseEstimation(sourceUrl: string, signal?: AbortSignal): Promise { + const dataUrl = await sourceToDataUrl(sourceUrl, signal); + const value = await requestJson("/pose-estimation/run", { + method: "POST", + body: JSON.stringify({ schemaVersion: 1, operation: "pose", dataUrl }), + headers: { "content-type": "application/json" }, + signal, + }); + if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果响应无效"); + } + return parseResult(value.result); +} + +export async function readLocalPoseRuntimeStatus(signal?: AbortSignal) { + return parseStatus(await requestJson("/pose-estimation/status", { method: "GET", signal })); +} + +async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) { + if (sourceUrl.startsWith("data:")) { + const match = DATA_URL_PATTERN.exec(sourceUrl); + if (!match) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400); + const bytes = bytesFromBase64(match[2]!); + assertImageSize(bytes); + return `data:${match[1]};base64,${bytesToBase64(bytes)}`; + } + let response: Response; + try { + response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal }); + } catch (error) { + if (signal?.aborted) throw error; + throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取"); + } + if (!response.ok) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取", response.status); + const mimeType = imageMime(response.headers.get("content-type") || response.type); + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400); + const bytes = new Uint8Array(await response.arrayBuffer()); + assertImageSize(bytes); + return `data:${mimeType};base64,${bytesToBase64(bytes)}`; +} + +async function requestJson(path: string, init: RequestInit = {}) { + const { response, body: responseBody } = await requestRuntimeResponse(path, init); + const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody; + if (!response.ok) { + const code = isRecord(body) && typeof body.code === "string" ? body.code : "pose_runtime_unavailable"; + const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机姿态运行时不可用"; + throw new LocalRuntimeClientError(code, message, response.status); + } + return body as T; +} + +async function requestRuntimeResponse(path: string, init: RequestInit = {}) { + let refreshed = false; + while (true) { + let response: Response; + try { + response = await (await ensureTransport()).request(path, init); + } catch (error) { + if (refreshed || !isSessionRefreshError(error)) throw error; + await reconnectTransport(); + refreshed = true; + continue; + } + if (!refreshed && response.status === 401) { + await reconnectTransport(); + refreshed = true; + continue; + } + if (response.status === 403) { + const body = await parseJsonResponse(response); + if (!refreshed && shouldRefreshSession(response, body)) { + await reconnectTransport(); + refreshed = true; + continue; + } + return { response, body }; + } + return { response }; + } +} + +async function ensureTransport(): Promise { + const state = useLocalRuntimeStore.getState(); + if (state.connection !== "connected") await state.connect(); + const current = useLocalRuntimeStore.getState(); + if (current.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", current.error || "本机姿态运行时不可用"); + return getLocalRuntimeSessionClient(); +} + +async function reconnectTransport() { + const client = getLocalRuntimeSessionClient(); + client.revokeLocalSession(); + await useLocalRuntimeStore.getState().connect(); + const state = useLocalRuntimeStore.getState(); + if (state.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", state.error || "本机姿态运行时不可用"); +} + +async function parseJsonResponse(response: Response) { + const text = await boundedText(response, MAX_RESPONSE_BYTES); + try { return text ? JSON.parse(text) as unknown : {}; } catch { throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应无效", response.status); } +} + +async function boundedText(response: Response, limit: number) { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status); + const text = await response.text(); + if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status); + return text; +} + +function parseResult(value: Record): LocalPoseResult { + if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || value.backend !== "openpose-body" || !isPositiveInteger(value.personCount) || typeof value.pngBase64 !== "string") { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果无效"); + } + const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机姿态结果无效"); + if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果过大"); + return { + blob: new Blob([bytes], { type: "image/png" }), + width: value.width, + height: value.height, + modelId: value.modelId, + device: "cpu", + backend: "openpose-body", + personCount: value.personCount, + people: Array.isArray(value.people) ? value.people : undefined, + }; +} + +function parseStatus(value: unknown): LocalPoseRuntimeStatus { + if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || value.backend !== "openpose-body" || (value.reason !== undefined && typeof value.reason !== "string")) { + throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态状态无效"); + } + return value as LocalPoseRuntimeStatus; +} + +function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" { + const mime = value.startsWith("image/") ? value.split(";", 1)[0] : ""; + if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime; + throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400); +} + +function assertImageSize(bytes: Uint8Array) { + if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400); +} + +function bytesToBase64(bytes: Uint8Array) { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +function bytesFromBase64(value: string, code = "pose_input_invalid", message = "姿态转换输入图片无效") { + let binary: string; + try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "pose_input_invalid" ? 400 : 0); } + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return bytes; +} + +function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError { + return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code); +} + +function shouldRefreshSession(response: Response, body: unknown) { + return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied"); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0; +} diff --git a/web/src/types/canvas.ts b/web/src/types/canvas.ts index c0115739b..dd771b3f6 100644 --- a/web/src/types/canvas.ts +++ b/web/src/types/canvas.ts @@ -1,4 +1,5 @@ import type { CanvasColorGrade } from "@/lib/canvas/canvas-color-grade"; +import type { MediaConversionNodeState } from "@/lib/media-conversion/contracts"; import type { AssetCategory } from "@/lib/asset-category"; import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture"; import type { StyleExecutionPlan } from "@/lib/canvas/style-profile"; @@ -34,6 +35,7 @@ export enum CanvasNodeType { Compare = "compare", Chart = "chart", ColorGrade = "colorgrade", + MediaConversion = "media-conversion", } /** Runtime IDs contributed by plugins share the persisted node type field. */ @@ -362,6 +364,8 @@ export type CanvasNodeMetadata = { chartKind?: "bar" | "line"; /** 调色节点的参数；缺省视为未调色。 */ colorGrade?: CanvasColorGrade; + /** 本地图片/视频转换节点的参数、来源指纹和结果状态。 */ + mediaConversion?: MediaConversionNodeState; /** 用户手动拉伸过尺寸；图片按真实比例自动适配时避让它。 */ manualSize?: boolean; storyboard?: StoryboardData; diff --git a/web/test/media-conversion.test.ts b/web/test/media-conversion.test.ts new file mode 100644 index 000000000..27d9d8787 --- /dev/null +++ b/web/test/media-conversion.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, test } from "bun:test"; + +import { canvasConnectionError } from "../src/lib/canvas/canvas-connection-policy"; +import { findPendingMediaConversionInput } from "../src/components/canvas/canvas-node-generation"; +import { getNodeAcceptedInputKinds, getNodeMaxInputCount, getNodeResourceKind } from "../src/lib/canvas/node-registry"; +import { mediaConversionSourceFingerprint } from "../src/lib/media-conversion/contracts"; +import { convertImageLocally, LocalImageConversionError, transformPixels } from "../src/lib/media-conversion/local-converter"; +import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas"; +import type { AiConfig } from "../src/stores/use-config-store"; + +function node(type: CanvasNodeType, id: string, metadata: CanvasNodeData["metadata"] = {}): CanvasNodeData { + return { id, type, title: id, position: { x: 0, y: 0 }, width: 380, height: 440, metadata }; +} + +const config = {} as AiConfig; + +describe("本地转换节点注册", () => { + test("只允许一个图片或视频输入，完成后才作为图片素材", () => { + expect(getNodeAcceptedInputKinds(CanvasNodeType.MediaConversion)).toEqual(["image", "video"]); + expect(getNodeMaxInputCount(CanvasNodeType.MediaConversion)).toBe(1); + expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion"))).toBeNull(); + expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion", { + mediaConversion: { schemaVersion: 1, operation: "grayscale", status: "completed", resultStorageKey: "image:scope:result" }, + }))).toBe("image"); + expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion", { + mediaConversion: { schemaVersion: 1, operation: "pose", status: "skipped", detectedPeople: 0 }, + }))).toBeNull(); + }); + + test("连接规则接受图片和视频，拒绝没有可读媒体的节点及第二个输入", () => { + const image = node(CanvasNodeType.Image, "image"); + const video = node(CanvasNodeType.Video, "video"); + const text = node(CanvasNodeType.Text, "text"); + const conversion = node(CanvasNodeType.MediaConversion, "conversion"); + const nodes = [image, video, text, conversion]; + const candidate = (fromNodeId: string): CanvasConnection => ({ id: `candidate-${fromNodeId}`, fromNodeId, toNodeId: conversion.id }); + + expect(canvasConnectionError(config, nodes, [], candidate(image.id))).toBe(""); + expect(canvasConnectionError(config, nodes, [], candidate(video.id))).toBe(""); + expect(canvasConnectionError(config, nodes, [], candidate(text.id))).toContain("转换节点只接受图片或视频输入"); + expect(canvasConnectionError(config, nodes, [{ id: "existing", fromNodeId: image.id, toNodeId: conversion.id }], candidate(video.id))).toContain("转换节点最多连接 1 个输入"); + }); + + test("下游生成会阻止未完成或待更新的转换结果", () => { + const conversion = node(CanvasNodeType.MediaConversion, "conversion", { + mediaConversion: { schemaVersion: 1, operation: "depth", status: "unavailable" }, + }); + const input = node(CanvasNodeType.Image, "input", { content: "data:image/png;base64,source" }); + const target = node(CanvasNodeType.Image, "target"); + const connections: CanvasConnection[] = [ + { id: "input-conversion", fromNodeId: input.id, toNodeId: conversion.id }, + { id: "conversion-target", fromNodeId: conversion.id, toNodeId: target.id }, + ]; + expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)?.id).toBe(conversion.id); + conversion.metadata = { mediaConversion: { schemaVersion: 1, operation: "grayscale", status: "completed", sourceNodeId: input.id, sourceFingerprint: "v1-unknown", resultStorageKey: "image:scope:result" } }; + expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)?.id).toBe(conversion.id); + conversion.metadata.mediaConversion.sourceFingerprint = mediaConversionSourceFingerprint(input); + expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)).toBeNull(); + }); +}); + +describe("图片本地转换算法", () => { + test("灰度转换保留 alpha 并使用感知亮度", () => { + const pixels = new Uint8ClampedArray([255, 0, 0, 200, 0, 255, 0, 100]); + const output = transformPixels(pixels, 2, 1, "grayscale"); + expect(Array.from(output)).toEqual([76, 76, 76, 200, 150, 150, 150, 100]); + }); + + test("Canny 转换输出黑白边缘图并保留 alpha", () => { + const pixels = new Uint8ClampedArray([ + 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255, + 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255, + 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255, + ]); + const output = transformPixels(pixels, 3, 3, "edge-canny"); + for (let index = 0; index < output.length; index += 4) { + expect([0, 255]).toContain(output[index]); + expect(output[index]).toBe(output[index + 1]); + expect(output[index]).toBe(output[index + 2]); + expect(output[index + 3]).toBe(255); + } + }); + + test("浏览器算法转换把模型操作交给本机 Runtime", async () => { + await expect(convertImageLocally("", "depth")).rejects.toMatchObject({ code: "model_missing" }); + }); +});