feat(canvas): 图片转换插件 - 合并 PR #446 本地媒体转换节点与运行时
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
commit
99f50f85eb
51 files changed
+3286
-46
No files matched your search
@@ -13,6 +13,7 @@ const (
|
|||||||
PluginPromptOptimizer = "prompt-optimizer"
|
PluginPromptOptimizer = "prompt-optimizer"
|
||||||
PluginPortraitClearance = "portrait-clearance"
|
PluginPortraitClearance = "portrait-clearance"
|
||||||
PluginAIArtCritique = "ai-art-critique"
|
PluginAIArtCritique = "ai-art-critique"
|
||||||
|
PluginMediaConversion = "media-conversion"
|
||||||
|
|
||||||
PluginOriginOfficial = "official"
|
PluginOriginOfficial = "official"
|
||||||
PluginOriginSystem = "system"
|
PluginOriginSystem = "system"
|
||||||
@@ -80,6 +81,10 @@ var officialApplicationPolicies = map[string]PluginManagementView{
|
|||||||
Origin: PluginOriginOfficial, Kind: PluginKindApplication,
|
Origin: PluginOriginOfficial, Kind: PluginKindApplication,
|
||||||
ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone,
|
ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone,
|
||||||
},
|
},
|
||||||
|
PluginMediaConversion: {
|
||||||
|
Origin: PluginOriginOfficial, Kind: PluginKindApplication,
|
||||||
|
ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var systemPaymentPolicies = map[string]PluginManagementView{
|
var systemPaymentPolicies = map[string]PluginManagementView{
|
||||||
@@ -196,8 +201,12 @@ func (s *Service) pluginStateForUser(actor *model.User, pluginID string, items [
|
|||||||
// Preserve the old globally-enabled workflow behavior until each user
|
// Preserve the old globally-enabled workflow behavior until each user
|
||||||
// explicitly saves a personal choice. Other official applications were
|
// explicitly saves a personal choice. Other official applications were
|
||||||
// already controlled by each user's local installation state.
|
// already controlled by each user's local installation state.
|
||||||
if !userConfigured && hasRuntime && isLegacyWorkflowPlugin(pluginID) {
|
if !userConfigured {
|
||||||
userEnabled = runtimePlugin.Status == "enabled"
|
if pluginID == PluginMediaConversion {
|
||||||
|
userEnabled = true
|
||||||
|
} else if hasRuntime && isLegacyWorkflowPlugin(pluginID) {
|
||||||
|
userEnabled = runtimePlugin.Status == "enabled"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
effective := platformAvailable
|
effective := platformAvailable
|
||||||
|
|||||||
@@ -76,6 +76,58 @@ Canvas Agent 内置 `portrait-clearance` Local Runtime 模块。它只接收签
|
|||||||
|
|
||||||
本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`,安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时,模块返回 `portrait_model_missing`,不会伪造低风险结果。
|
本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`,安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时,模块返回 `portrait_model_missing`,不会伪造低风险结果。
|
||||||
|
|
||||||
|
## Depth Anything V2 本机深度模块
|
||||||
|
|
||||||
|
画布“转换 → 深度图”使用 `depth-estimation` Local Runtime 模块。模块只接受签名的本机请求,Node Runtime 启动一个常驻 Python worker;模型首次调用时加载,后续转换复用同一进程。worker 以离线模式读取本地 Hugging Face 缓存,不会把图片发送到云端。
|
||||||
|
|
||||||
|
使用仓库根目录的 `start-yingce-local.cmd` 时,Canvas Agent 会和前端、后端一起自动启动,转换节点不需要用户另外打开 Agent 窗口或手工填写地址。只有手工分别启动前端和后端时,才需要在本目录执行 `node dist/index.js`。
|
||||||
|
|
||||||
|
当前仓库开发环境默认查找:
|
||||||
|
|
||||||
|
```text
|
||||||
|
.local/depth-anything-v2/venv/Scripts/python.exe # Windows
|
||||||
|
.local/depth-anything-v2/venv/bin/python # Linux/macOS
|
||||||
|
.local/cache/huggingface/ # 模型缓存
|
||||||
|
```
|
||||||
|
|
||||||
|
发布包会携带 `python/depth_anything_runtime.py`,但不会携带 Python、依赖或模型权重。自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_DEPTH_HF_HOME`;需要指定工作区时设置 `CANVAS_PROJECT_ROOT`。模型缺失、Python 环境缺失或 worker 退出时,转换节点显示具体失败状态并保留输入,不会把失败标记为完成。
|
||||||
|
|
||||||
|
## ControlNet Aux 本机 AI 线稿模块
|
||||||
|
|
||||||
|
画布“转换 → AI 线稿”使用 `lineart-estimation` Local Runtime 模块。它只运行 ControlNet Aux 的 `LineartDetector` 预处理器,把图片转换为线稿控制图;不会加载 Stable Diffusion、ControlNet 生图管线,也不会把图片上传到云端。Node Runtime 会复用同一个常驻 Python worker,首次转换加载模型,后续请求复用已加载模型。
|
||||||
|
|
||||||
|
该模块与深度模块共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 模型缓存。首次设置或缺少依赖时,在仓库根目录执行:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
|
||||||
|
& $python -m pip install -r canvas-agent/python/requirements-lineart.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
然后用下面的命令把线稿预处理器权重缓存到本地;下载完成后,正常转换会强制离线运行:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
|
||||||
|
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
|
||||||
|
& $python -c "from controlnet_aux import LineartDetector; LineartDetector.from_pretrained('lllyasviel/Annotators')"
|
||||||
|
```
|
||||||
|
|
||||||
|
自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_LINEART_HF_HOME` 和 `CANVAS_LINEART_MODEL_ID`;未找到 Python、依赖或模型时,节点显示“本地不可用”并保留输入,不会伪造已完成结果。
|
||||||
|
|
||||||
|
## ControlNet Aux 本机姿态骨架模块
|
||||||
|
|
||||||
|
画布“转换 → 姿态骨架”使用 `pose-estimation` Local Runtime 模块。当前笔记本方案采用 ControlNet Aux 的 **OpenPose body-only** 预处理器:只提取人体 18 点骨架并输出彩色姿态图,不加载 Stable Diffusion,也不上传图片。Node Runtime 会复用常驻 Python worker;未检测到人物时返回 `pose_no_person`,转换节点显示“已跳过”,不会保存伪造结果。
|
||||||
|
|
||||||
|
该模块与深度、线稿共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 缓存。安装 `requirements-lineart.txt` 后缓存人体权重:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
|
||||||
|
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
|
||||||
|
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
|
||||||
|
& $python -c "from huggingface_hub import hf_hub_download; hf_hub_download('lllyasviel/Annotators', 'body_pose_model.pth', cache_dir='$env:HF_HUB_CACHE')"
|
||||||
|
```
|
||||||
|
|
||||||
|
自定义安装位置时设置 `CANVAS_POSE_PYTHON`、`CANVAS_POSE_HF_HOME` 和 `CANVAS_POSE_MODEL_ID`。DWPose(包含手部、脸部和更完整关键点)仍可在后续按需替换,但它需要额外的 MMDetection/MMPose 依赖;当前实现优先保证普通 CPU 笔记本可运行。
|
||||||
|
|
||||||
## 发布
|
## 发布
|
||||||
|
|
||||||
`canvas-agent` 使用自己的 `package.json` 版本号,不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。
|
`canvas-agent` 使用自己的 `package.json` 版本号,不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"dist",
|
"dist",
|
||||||
|
"python",
|
||||||
"README.md"
|
"README.md"
|
||||||
],
|
],
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""Small persistent worker for the local Depth Anything V2 Small model.
|
||||||
|
|
||||||
|
The Node Local Runtime owns the process and sends one JSON request per line.
|
||||||
|
Keeping the pipeline alive avoids loading the model for every canvas conversion.
|
||||||
|
The worker is deliberately offline-only: the model must already exist in the
|
||||||
|
configured Hugging Face cache.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import contextlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
|
MODEL_ID = os.environ.get("CANVAS_DEPTH_MODEL_ID", "depth-anything/Depth-Anything-V2-Small-hf")
|
||||||
|
MAX_INPUT_BYTES = 12 * 1024 * 1024
|
||||||
|
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
|
||||||
|
PIPELINE: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def emit(value: dict[str, Any]) -> None:
|
||||||
|
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def load_pipeline() -> Any:
|
||||||
|
global PIPELINE
|
||||||
|
if PIPELINE is not None:
|
||||||
|
return PIPELINE
|
||||||
|
|
||||||
|
try:
|
||||||
|
from transformers import pipeline
|
||||||
|
from transformers.utils import logging as transformers_logging
|
||||||
|
|
||||||
|
transformers_logging.set_verbosity_error()
|
||||||
|
# Transformers/tqdm can write progress text while loading weights.
|
||||||
|
# Keep stdout reserved for the JSON-lines protocol.
|
||||||
|
with contextlib.redirect_stdout(sys.stderr):
|
||||||
|
PIPELINE = pipeline(
|
||||||
|
"depth-estimation",
|
||||||
|
model=MODEL_ID,
|
||||||
|
device=-1,
|
||||||
|
local_files_only=True,
|
||||||
|
)
|
||||||
|
except Exception as error: # pragma: no cover - depends on local model files
|
||||||
|
raise RuntimeError("depth_model_missing") from error
|
||||||
|
return PIPELINE
|
||||||
|
|
||||||
|
|
||||||
|
def image_from_data_url(value: Any) -> Image.Image:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError("depth_input_invalid")
|
||||||
|
match = DATA_URL_PATTERN.fullmatch(value)
|
||||||
|
if not match:
|
||||||
|
raise ValueError("depth_input_invalid")
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
|
||||||
|
except Exception as error:
|
||||||
|
raise ValueError("depth_input_invalid") from error
|
||||||
|
if not raw or len(raw) > MAX_INPUT_BYTES:
|
||||||
|
raise ValueError("depth_input_invalid")
|
||||||
|
try:
|
||||||
|
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||||
|
image.load()
|
||||||
|
return image
|
||||||
|
except Exception as error:
|
||||||
|
raise ValueError("depth_input_invalid") from error
|
||||||
|
|
||||||
|
|
||||||
|
def encode_png(image: Image.Image) -> str:
|
||||||
|
output = io.BytesIO()
|
||||||
|
image.save(output, format="PNG", optimize=True)
|
||||||
|
return base64.b64encode(output.getvalue()).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def handle(request: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
image = image_from_data_url(request.get("dataUrl"))
|
||||||
|
try:
|
||||||
|
result = load_pipeline()(image)
|
||||||
|
depth = result.get("depth") if isinstance(result, dict) else None
|
||||||
|
if not isinstance(depth, Image.Image):
|
||||||
|
raise RuntimeError("depth_inference_failed")
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"requestId": request.get("requestId"),
|
||||||
|
"mimeType": "image/png",
|
||||||
|
"width": depth.width,
|
||||||
|
"height": depth.height,
|
||||||
|
"modelId": MODEL_ID,
|
||||||
|
"device": "cpu",
|
||||||
|
"pngBase64": encode_png(depth),
|
||||||
|
}
|
||||||
|
except RuntimeError:
|
||||||
|
raise
|
||||||
|
except Exception as error: # pragma: no cover - depends on local runtime
|
||||||
|
raise RuntimeError("depth_inference_failed") from error
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
||||||
|
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
||||||
|
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||||
|
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||||
|
|
||||||
|
for line in sys.stdin:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
request_id: Any = None
|
||||||
|
try:
|
||||||
|
request = json.loads(line)
|
||||||
|
if not isinstance(request, dict):
|
||||||
|
raise ValueError("depth_input_invalid")
|
||||||
|
request_id = request.get("requestId")
|
||||||
|
emit(handle(request))
|
||||||
|
except ValueError as error:
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": str(error) or "depth_input_invalid", "message": "深度转换输入无效"})
|
||||||
|
except RuntimeError as error:
|
||||||
|
code = str(error) or "depth_inference_failed"
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": code, "message": "本机深度模型不可用" if code == "depth_model_missing" else "本机深度推理失败"})
|
||||||
|
except Exception:
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": "depth_inference_failed", "message": "本机深度推理失败"})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
"""Small persistent worker for the local ControlNet Aux lineart detector.
|
||||||
|
|
||||||
|
This worker only extracts a lineart control image. It does not load Stable
|
||||||
|
Diffusion or a ControlNet generation pipeline, so it stays suitable for a
|
||||||
|
CPU-only notebook. The Node Local Runtime owns the process and sends one JSON
|
||||||
|
request per line; stdout is reserved for that protocol.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import contextlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
|
MODEL_ID = os.environ.get("CANVAS_LINEART_MODEL_ID", "lllyasviel/Annotators")
|
||||||
|
MAX_INPUT_BYTES = 12 * 1024 * 1024
|
||||||
|
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
|
||||||
|
DETECTOR: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def emit(value: dict[str, Any]) -> None:
|
||||||
|
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def load_detector() -> Any:
|
||||||
|
global DETECTOR
|
||||||
|
if DETECTOR is not None:
|
||||||
|
return DETECTOR
|
||||||
|
|
||||||
|
try:
|
||||||
|
from controlnet_aux import LineartDetector
|
||||||
|
|
||||||
|
# The detector downloads only the lineart preprocessor weights from the
|
||||||
|
# configured Hugging Face cache. It is intentionally not a diffusion
|
||||||
|
# pipeline and never loads Stable Diffusion or ControlNet weights.
|
||||||
|
with contextlib.redirect_stdout(sys.stderr):
|
||||||
|
DETECTOR = LineartDetector.from_pretrained(MODEL_ID)
|
||||||
|
except Exception as error: # pragma: no cover - depends on local files
|
||||||
|
raise RuntimeError("lineart_model_missing") from error
|
||||||
|
return DETECTOR
|
||||||
|
|
||||||
|
|
||||||
|
def image_from_data_url(value: Any) -> Image.Image:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError("lineart_input_invalid")
|
||||||
|
match = DATA_URL_PATTERN.fullmatch(value)
|
||||||
|
if not match:
|
||||||
|
raise ValueError("lineart_input_invalid")
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
|
||||||
|
except Exception as error:
|
||||||
|
raise ValueError("lineart_input_invalid") from error
|
||||||
|
if not raw or len(raw) > MAX_INPUT_BYTES:
|
||||||
|
raise ValueError("lineart_input_invalid")
|
||||||
|
try:
|
||||||
|
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||||
|
image.load()
|
||||||
|
return image
|
||||||
|
except Exception as error:
|
||||||
|
raise ValueError("lineart_input_invalid") from error
|
||||||
|
|
||||||
|
|
||||||
|
def encode_png(image: Image.Image) -> str:
|
||||||
|
output = io.BytesIO()
|
||||||
|
image.save(output, format="PNG", optimize=True)
|
||||||
|
return base64.b64encode(output.getvalue()).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def handle(request: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
image = image_from_data_url(request.get("dataUrl"))
|
||||||
|
try:
|
||||||
|
lineart = load_detector()(image)
|
||||||
|
if not isinstance(lineart, Image.Image):
|
||||||
|
raise RuntimeError("lineart_inference_failed")
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"requestId": request.get("requestId"),
|
||||||
|
"mimeType": "image/png",
|
||||||
|
"width": lineart.width,
|
||||||
|
"height": lineart.height,
|
||||||
|
"modelId": MODEL_ID,
|
||||||
|
"device": "cpu",
|
||||||
|
"pngBase64": encode_png(lineart.convert("RGB")),
|
||||||
|
}
|
||||||
|
except RuntimeError:
|
||||||
|
raise
|
||||||
|
except Exception as error: # pragma: no cover - depends on local runtime
|
||||||
|
raise RuntimeError("lineart_inference_failed") from error
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
||||||
|
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
||||||
|
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||||
|
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||||
|
|
||||||
|
for line in sys.stdin:
|
||||||
|
# PowerShell and a few Windows launchers may prepend a UTF-8 BOM to
|
||||||
|
# the first piped line. The Node runtime sends plain UTF-8, but
|
||||||
|
# accepting the marker keeps the worker protocol tolerant for manual
|
||||||
|
# diagnostics as well.
|
||||||
|
line = line.lstrip("\ufeff").strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
request_id: Any = None
|
||||||
|
try:
|
||||||
|
request = json.loads(line)
|
||||||
|
if not isinstance(request, dict):
|
||||||
|
raise ValueError("lineart_input_invalid")
|
||||||
|
request_id = request.get("requestId")
|
||||||
|
emit(handle(request))
|
||||||
|
except ValueError as error:
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": str(error) or "lineart_input_invalid", "message": "线稿转换输入无效"})
|
||||||
|
except RuntimeError as error:
|
||||||
|
code = str(error) or "lineart_inference_failed"
|
||||||
|
message = "本地 AI 线稿模型或依赖尚未安装" if code == "lineart_model_missing" else "本地 AI 线稿推理失败"
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
|
||||||
|
except Exception:
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": "lineart_inference_failed", "message": "本地 AI 线稿推理失败"})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
"""Persistent CPU pose preprocessor for the local media conversion node.
|
||||||
|
|
||||||
|
The first notebook-friendly implementation uses the body-only OpenPose
|
||||||
|
preprocessor from ControlNet Aux. It produces a colored skeleton map and a
|
||||||
|
small list of normalized keypoints; it does not load a diffusion model.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import contextlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
|
MODEL_ID = os.environ.get("CANVAS_POSE_MODEL_ID", "lllyasviel/Annotators")
|
||||||
|
MAX_INPUT_BYTES = 12 * 1024 * 1024
|
||||||
|
DETECT_RESOLUTION = max(256, min(1024, int(os.environ.get("CANVAS_POSE_DETECT_RESOLUTION", "512"))))
|
||||||
|
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
|
||||||
|
DETECTOR: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def emit(value: dict[str, Any]) -> None:
|
||||||
|
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def load_detector() -> Any:
|
||||||
|
global DETECTOR
|
||||||
|
if DETECTOR is not None:
|
||||||
|
return DETECTOR
|
||||||
|
|
||||||
|
try:
|
||||||
|
import numpy as np
|
||||||
|
from controlnet_aux.open_pose.body import Body
|
||||||
|
from huggingface_hub import hf_hub_download
|
||||||
|
|
||||||
|
cache_dir = os.environ.get("HF_HUB_CACHE") or None
|
||||||
|
with contextlib.redirect_stdout(sys.stderr):
|
||||||
|
model_path = hf_hub_download(
|
||||||
|
repo_id=MODEL_ID,
|
||||||
|
filename="body_pose_model.pth",
|
||||||
|
cache_dir=cache_dir,
|
||||||
|
local_files_only=True,
|
||||||
|
)
|
||||||
|
DETECTOR = (Body(model_path), np)
|
||||||
|
except Exception as error: # pragma: no cover - depends on local model files
|
||||||
|
raise RuntimeError("pose_model_missing") from error
|
||||||
|
return DETECTOR
|
||||||
|
|
||||||
|
|
||||||
|
def image_from_data_url(value: Any) -> Image.Image:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError("pose_input_invalid")
|
||||||
|
match = DATA_URL_PATTERN.fullmatch(value)
|
||||||
|
if not match:
|
||||||
|
raise ValueError("pose_input_invalid")
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
|
||||||
|
except Exception as error:
|
||||||
|
raise ValueError("pose_input_invalid") from error
|
||||||
|
if not raw or len(raw) > MAX_INPUT_BYTES:
|
||||||
|
raise ValueError("pose_input_invalid")
|
||||||
|
try:
|
||||||
|
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||||
|
image.load()
|
||||||
|
return image
|
||||||
|
except Exception as error:
|
||||||
|
raise ValueError("pose_input_invalid") from error
|
||||||
|
|
||||||
|
|
||||||
|
def encode_png(image: Image.Image) -> str:
|
||||||
|
output = io.BytesIO()
|
||||||
|
image.save(output, format="PNG", optimize=True)
|
||||||
|
return base64.b64encode(output.getvalue()).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_people(bodies: list[Any], width: int, height: int) -> list[dict[str, Any]]:
|
||||||
|
people: list[dict[str, Any]] = []
|
||||||
|
for body in bodies:
|
||||||
|
keypoints = []
|
||||||
|
visible = []
|
||||||
|
for keypoint in body.keypoints:
|
||||||
|
if keypoint is None:
|
||||||
|
keypoints.append(None)
|
||||||
|
else:
|
||||||
|
normalized_x = max(0.0, min(1.0, float(keypoint.x) / float(width)))
|
||||||
|
normalized_y = max(0.0, min(1.0, float(keypoint.y) / float(height)))
|
||||||
|
keypoints.append({
|
||||||
|
"x": normalized_x,
|
||||||
|
"y": normalized_y,
|
||||||
|
"score": float(keypoint.score),
|
||||||
|
})
|
||||||
|
visible.append((normalized_x, normalized_y, float(keypoint.score)))
|
||||||
|
if visible:
|
||||||
|
xs = [point[0] for point in visible]
|
||||||
|
ys = [point[1] for point in visible]
|
||||||
|
confidence = sum(point[2] for point in visible) / len(visible)
|
||||||
|
box = {"x": min(xs), "y": min(ys), "width": max(xs) - min(xs), "height": max(ys) - min(ys)}
|
||||||
|
else:
|
||||||
|
confidence = 0.0
|
||||||
|
box = {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}
|
||||||
|
people.append({
|
||||||
|
"keypoints": keypoints,
|
||||||
|
"score": float(body.total_score),
|
||||||
|
"confidence": confidence,
|
||||||
|
"box": box,
|
||||||
|
"parts": int(body.total_parts),
|
||||||
|
})
|
||||||
|
return people
|
||||||
|
|
||||||
|
|
||||||
|
def handle(request: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
image = image_from_data_url(request.get("dataUrl"))
|
||||||
|
try:
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
from controlnet_aux.open_pose.util import draw_bodypose
|
||||||
|
from controlnet_aux.util import HWC3, resize_image
|
||||||
|
|
||||||
|
detector, _ = load_detector()
|
||||||
|
source = HWC3(np.asarray(image, dtype=np.uint8))
|
||||||
|
detected = resize_image(source, DETECT_RESOLUTION)
|
||||||
|
candidate, subset = detector(detected[:, :, ::-1].copy())
|
||||||
|
bodies = detector.format_body_result(candidate, subset)
|
||||||
|
if not bodies:
|
||||||
|
raise RuntimeError("pose_no_person")
|
||||||
|
|
||||||
|
height, width = detected.shape[:2]
|
||||||
|
people = normalize_people(bodies, width, height)
|
||||||
|
canvas = np.zeros_like(detected, dtype=np.uint8)
|
||||||
|
for body in bodies:
|
||||||
|
normalized = [
|
||||||
|
None if keypoint is None else type(keypoint)(
|
||||||
|
x=float(keypoint.x) / float(width),
|
||||||
|
y=float(keypoint.y) / float(height),
|
||||||
|
score=keypoint.score,
|
||||||
|
id=keypoint.id,
|
||||||
|
)
|
||||||
|
for keypoint in body.keypoints
|
||||||
|
]
|
||||||
|
draw_bodypose(canvas, normalized)
|
||||||
|
|
||||||
|
output = cv2.resize(canvas, (image.width, image.height), interpolation=cv2.INTER_LINEAR)
|
||||||
|
output_image = Image.fromarray(output.astype(np.uint8), mode="RGB")
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"requestId": request.get("requestId"),
|
||||||
|
"mimeType": "image/png",
|
||||||
|
"width": image.width,
|
||||||
|
"height": image.height,
|
||||||
|
"modelId": MODEL_ID,
|
||||||
|
"device": "cpu",
|
||||||
|
"backend": "openpose-body",
|
||||||
|
"personCount": len(bodies),
|
||||||
|
"people": people,
|
||||||
|
"pngBase64": encode_png(output_image),
|
||||||
|
}
|
||||||
|
except RuntimeError:
|
||||||
|
raise
|
||||||
|
except Exception as error: # pragma: no cover - depends on local runtime
|
||||||
|
raise RuntimeError("pose_inference_failed") from error
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
||||||
|
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
||||||
|
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||||
|
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||||
|
|
||||||
|
for line in sys.stdin:
|
||||||
|
line = line.lstrip("\ufeff").strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
request_id: Any = None
|
||||||
|
try:
|
||||||
|
request = json.loads(line)
|
||||||
|
if not isinstance(request, dict):
|
||||||
|
raise ValueError("pose_input_invalid")
|
||||||
|
request_id = request.get("requestId")
|
||||||
|
emit(handle(request))
|
||||||
|
except ValueError as error:
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": str(error) or "pose_input_invalid", "message": "姿态转换输入无效"})
|
||||||
|
except RuntimeError as error:
|
||||||
|
code = str(error) or "pose_inference_failed"
|
||||||
|
if code == "pose_model_missing":
|
||||||
|
message = "本地姿态模型或依赖尚未安装"
|
||||||
|
elif code == "pose_no_person":
|
||||||
|
message = "未检测到可用人物姿态"
|
||||||
|
else:
|
||||||
|
message = "本地姿态推理失败"
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
|
||||||
|
except Exception:
|
||||||
|
emit({"ok": False, "requestId": request_id, "code": "pose_inference_failed", "message": "本地姿态推理失败"})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
controlnet-aux==0.0.10
|
||||||
@@ -4,7 +4,7 @@ import { CanvasSession } from "./canvas-session.js";
|
|||||||
import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js";
|
import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js";
|
||||||
import { createLocalRuntimeApp } from "./local-runtime.js";
|
import { createLocalRuntimeApp } from "./local-runtime.js";
|
||||||
import { startLocalRuntime } from "./local-runtime-host.js";
|
import { startLocalRuntime } from "./local-runtime-host.js";
|
||||||
import { LocalRuntimeSessionManager } from "./local-runtime-session.js";
|
import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./local-runtime-session.js";
|
||||||
import {
|
import {
|
||||||
createCanvasAgentHttpModule,
|
createCanvasAgentHttpModule,
|
||||||
type CanvasAgentSession,
|
type CanvasAgentSession,
|
||||||
@@ -14,6 +14,9 @@ import {
|
|||||||
type DreaminaHttpModuleOptions,
|
type DreaminaHttpModuleOptions,
|
||||||
} from "./modules/dreamina-http.js";
|
} from "./modules/dreamina-http.js";
|
||||||
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
|
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
|
||||||
|
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
|
||||||
|
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
|
||||||
|
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
|
||||||
|
|
||||||
export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">;
|
export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">;
|
||||||
|
|
||||||
@@ -34,21 +37,29 @@ export function createHttpApp(
|
|||||||
dependencies: CanvasAgentHttpDependencies = {},
|
dependencies: CanvasAgentHttpDependencies = {},
|
||||||
): Express {
|
): Express {
|
||||||
const endpoint = config.url;
|
const endpoint = config.url;
|
||||||
|
const modules = [
|
||||||
|
createCanvasAgentHttpModule(config, session),
|
||||||
|
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
|
||||||
|
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
|
||||||
|
createDepthEstimationHttpModule(),
|
||||||
|
createLineartEstimationHttpModule(),
|
||||||
|
createPoseEstimationHttpModule(),
|
||||||
|
];
|
||||||
const manager = new LocalRuntimeSessionManager({
|
const manager = new LocalRuntimeSessionManager({
|
||||||
endpoint,
|
endpoint,
|
||||||
trustedOrigins: config.trustedWebOrigins,
|
trustedOrigins: config.trustedWebOrigins,
|
||||||
registrations: config.browserRegistrations,
|
registrations: config.browserRegistrations,
|
||||||
|
scopes: [...new Set([
|
||||||
|
...LOCAL_RUNTIME_DEFAULT_SCOPES,
|
||||||
|
...modules.flatMap((module) => module.descriptor.scopes),
|
||||||
|
])],
|
||||||
});
|
});
|
||||||
return createLocalRuntimeApp({
|
return createLocalRuntimeApp({
|
||||||
authority: new URL(endpoint).host,
|
authority: new URL(endpoint).host,
|
||||||
endpoint,
|
endpoint,
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
sessionManager: manager,
|
sessionManager: manager,
|
||||||
modules: [
|
modules,
|
||||||
createCanvasAgentHttpModule(config, session),
|
|
||||||
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
|
|
||||||
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
|
|
||||||
],
|
|
||||||
legacyMasterToken: config.token,
|
legacyMasterToken: config.token,
|
||||||
legacyOrigins: config.origins ?? [],
|
legacyOrigins: config.origins ?? [],
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ export const LOCAL_RUNTIME_ENDPOINT = "http://127.0.0.1:17371";
|
|||||||
export const LOCAL_RUNTIME_ID = "framefield-local-runtime";
|
export const LOCAL_RUNTIME_ID = "framefield-local-runtime";
|
||||||
export const LOCAL_RUNTIME_API_VERSION = 2;
|
export const LOCAL_RUNTIME_API_VERSION = 2;
|
||||||
|
|
||||||
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance";
|
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
|
||||||
export type LocalRuntimeScope =
|
export type LocalRuntimeScope =
|
||||||
| "runtime:status"
|
| "runtime:status"
|
||||||
| "runtime:revoke"
|
| "runtime:revoke"
|
||||||
@@ -20,7 +20,13 @@ export type LocalRuntimeScope =
|
|||||||
| "portrait:status"
|
| "portrait:status"
|
||||||
| "portrait:model"
|
| "portrait:model"
|
||||||
| "portrait:run"
|
| "portrait:run"
|
||||||
| "portrait:read";
|
| "portrait:read"
|
||||||
|
| "depth:status"
|
||||||
|
| "depth:run"
|
||||||
|
| "lineart:status"
|
||||||
|
| "lineart:run"
|
||||||
|
| "pose:status"
|
||||||
|
| "pose:run";
|
||||||
|
|
||||||
export type DreaminaModelOperation =
|
export type DreaminaModelOperation =
|
||||||
| "text-to-image"
|
| "text-to-image"
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./loca
|
|||||||
import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js";
|
import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js";
|
||||||
import { createDreaminaHttpModule } from "./modules/dreamina-http.js";
|
import { createDreaminaHttpModule } from "./modules/dreamina-http.js";
|
||||||
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
|
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
|
||||||
|
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
|
||||||
|
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
|
||||||
|
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
|
||||||
|
|
||||||
export type StartLocalRuntimeOptions = {
|
export type StartLocalRuntimeOptions = {
|
||||||
config?: LocalRuntimeConfig;
|
config?: LocalRuntimeConfig;
|
||||||
@@ -46,6 +49,9 @@ export function createDefaultLocalRuntimeModules(config: LocalRuntimeConfig): Lo
|
|||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }),
|
createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }),
|
||||||
|
createDepthEstimationHttpModule(),
|
||||||
|
createLineartEstimationHttpModule(),
|
||||||
|
createPoseEstimationHttpModule(),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ function validateModules(modules: readonly LocalRuntimeModule[]) {
|
|||||||
const ids = new Set<LocalRuntimeModuleId>();
|
const ids = new Set<LocalRuntimeModuleId>();
|
||||||
const routes = new Set<string>();
|
const routes = new Set<string>();
|
||||||
for (const module of modules) {
|
for (const module of modules) {
|
||||||
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance") {
|
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance" && module.descriptor.id !== "depth-estimation" && module.descriptor.id !== "lineart-estimation" && module.descriptor.id !== "pose-estimation") {
|
||||||
throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`);
|
throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`);
|
||||||
}
|
}
|
||||||
if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`);
|
if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`);
|
||||||
|
|||||||
@@ -0,0 +1,327 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||||
|
|
||||||
|
import type { Request, RequestHandler, Response } from "express";
|
||||||
|
|
||||||
|
import type { LocalRuntimeModule } from "../local-runtime.js";
|
||||||
|
|
||||||
|
const DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf";
|
||||||
|
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
|
||||||
|
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
|
||||||
|
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||||
|
|
||||||
|
export type DepthRuntimePaths = {
|
||||||
|
pythonPath: string;
|
||||||
|
scriptPath: string;
|
||||||
|
hfHome: string;
|
||||||
|
modelId: string;
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
reason?: "python_missing" | "worker_missing";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type DepthRuntimeStatus = {
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
loaded: boolean;
|
||||||
|
device: "cpu";
|
||||||
|
modelId: string;
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type DepthWorkerSuccess = {
|
||||||
|
ok: true;
|
||||||
|
requestId: string;
|
||||||
|
mimeType: "image/png";
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
modelId: string;
|
||||||
|
device: "cpu";
|
||||||
|
pngBase64: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class DepthRuntimeError extends Error {
|
||||||
|
constructor(readonly code: string, message: string, readonly statusCode = 503) {
|
||||||
|
super(message);
|
||||||
|
this.name = "DepthRuntimeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createDepthEstimationHttpModule(): LocalRuntimeModule {
|
||||||
|
const worker = new DepthEstimationWorker();
|
||||||
|
return {
|
||||||
|
descriptor: {
|
||||||
|
id: "depth-estimation",
|
||||||
|
displayName: "Depth Anything V2 本机深度",
|
||||||
|
apiVersion: 1,
|
||||||
|
scopes: ["depth:status", "depth:run"],
|
||||||
|
},
|
||||||
|
routes: [
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
path: "/depth-estimation/status",
|
||||||
|
scope: "depth:status",
|
||||||
|
handler: asyncRoute(async (_request, response) => {
|
||||||
|
const status = worker.status();
|
||||||
|
response.json({
|
||||||
|
ok: true,
|
||||||
|
module: "depth-estimation",
|
||||||
|
apiVersion: 1,
|
||||||
|
ready: status.configured && status.modelCached,
|
||||||
|
...status,
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
path: "/depth-estimation/run",
|
||||||
|
scope: "depth:run",
|
||||||
|
handler: asyncRoute(async (request, response) => {
|
||||||
|
const body = parseBody(request);
|
||||||
|
const dataUrl = parseDepthInput(body);
|
||||||
|
const result = await worker.run({ dataUrl });
|
||||||
|
response.json({
|
||||||
|
ok: true,
|
||||||
|
module: "depth-estimation",
|
||||||
|
apiVersion: 1,
|
||||||
|
result,
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
dispose: () => worker.dispose(),
|
||||||
|
publicHealth: () => {
|
||||||
|
const status = worker.status();
|
||||||
|
return { depthEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveDepthRuntimePaths(env: NodeJS.ProcessEnv = process.env): DepthRuntimePaths {
|
||||||
|
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||||
|
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
|
||||||
|
const projectRoots = [
|
||||||
|
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
|
||||||
|
path.resolve(moduleRoot, ".."),
|
||||||
|
path.resolve(process.cwd()),
|
||||||
|
].filter((value): value is string => Boolean(value));
|
||||||
|
const projectRoot = configuredProjectRoot
|
||||||
|
? path.resolve(configuredProjectRoot)
|
||||||
|
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
|
||||||
|
const scriptCandidates = [
|
||||||
|
path.join(moduleRoot, "python", "depth_anything_runtime.py"),
|
||||||
|
path.join(projectRoot, "canvas-agent", "python", "depth_anything_runtime.py"),
|
||||||
|
];
|
||||||
|
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
|
||||||
|
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
|
||||||
|
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
|
||||||
|
: process.platform === "win32"
|
||||||
|
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
|
||||||
|
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
|
||||||
|
const pythonPath = pythonCandidates[0]!;
|
||||||
|
const hfHome = path.resolve(env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
|
||||||
|
const modelId = env.CANVAS_DEPTH_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
|
||||||
|
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
|
||||||
|
const workerExists = fs.existsSync(scriptPath);
|
||||||
|
const pythonExists = fs.existsSync(pythonPath);
|
||||||
|
return {
|
||||||
|
pythonPath,
|
||||||
|
scriptPath,
|
||||||
|
hfHome,
|
||||||
|
modelId,
|
||||||
|
configured: workerExists && pythonExists,
|
||||||
|
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
|
||||||
|
...(pythonExists ? {} : { reason: "python_missing" as const }),
|
||||||
|
...(!workerExists ? { reason: "worker_missing" as const } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DepthEstimationWorker {
|
||||||
|
private paths = resolveDepthRuntimePaths();
|
||||||
|
private child?: ChildProcessWithoutNullStreams;
|
||||||
|
private stdoutBuffer = "";
|
||||||
|
private stderrBuffer = "";
|
||||||
|
private requestCounter = 0;
|
||||||
|
private loaded = false;
|
||||||
|
private readonly pending = new Map<string, { resolve: (value: DepthWorkerSuccess) => void; reject: (error: unknown) => void }>();
|
||||||
|
|
||||||
|
status(): DepthRuntimeStatus {
|
||||||
|
const paths = this.refreshPaths();
|
||||||
|
return {
|
||||||
|
configured: paths.configured,
|
||||||
|
modelCached: paths.modelCached,
|
||||||
|
loaded: this.loaded,
|
||||||
|
device: "cpu",
|
||||||
|
modelId: paths.modelId,
|
||||||
|
...(paths.reason ? { reason: paths.reason } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async run(input: { dataUrl: string }) {
|
||||||
|
validateDepthDataUrl(input.dataUrl);
|
||||||
|
const paths = this.refreshPaths();
|
||||||
|
if (!paths.configured) {
|
||||||
|
throw new DepthRuntimeError(
|
||||||
|
"depth_runtime_unavailable",
|
||||||
|
paths.reason === "worker_missing" ? "本机深度运行文件不存在" : "未找到本机深度 Python 环境",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const child = this.ensureChild();
|
||||||
|
const requestId = `depth-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
|
||||||
|
const response = new Promise<DepthWorkerSuccess>((resolve, reject) => {
|
||||||
|
this.pending.set(requestId, { resolve, reject });
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
|
||||||
|
} catch (error) {
|
||||||
|
this.pending.delete(requestId);
|
||||||
|
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法写入");
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
dispose() {
|
||||||
|
const child = this.child;
|
||||||
|
this.child = undefined;
|
||||||
|
this.loaded = false;
|
||||||
|
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已停止"));
|
||||||
|
if (child && !child.killed) child.kill();
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensureChild() {
|
||||||
|
if (this.child && !this.child.killed) return this.child;
|
||||||
|
let child: ChildProcessWithoutNullStreams;
|
||||||
|
try {
|
||||||
|
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
|
||||||
|
cwd: path.dirname(this.paths.scriptPath),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
HF_HOME: this.paths.hfHome,
|
||||||
|
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
|
||||||
|
HF_HUB_OFFLINE: "1",
|
||||||
|
TRANSFORMERS_OFFLINE: "1",
|
||||||
|
HF_HUB_DISABLE_TELEMETRY: "1",
|
||||||
|
HF_HUB_DISABLE_PROGRESS_BARS: "1",
|
||||||
|
TRANSFORMERS_VERBOSITY: "error",
|
||||||
|
CANVAS_DEPTH_MODEL_ID: this.paths.modelId,
|
||||||
|
},
|
||||||
|
stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法启动");
|
||||||
|
}
|
||||||
|
this.child = child;
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
|
||||||
|
child.stderr.on("data", (chunk: string) => {
|
||||||
|
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
|
||||||
|
});
|
||||||
|
child.on("error", () => this.handleChildExit(child));
|
||||||
|
child.on("exit", () => this.handleChildExit(child));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
private refreshPaths() {
|
||||||
|
if (!this.child) this.paths = resolveDepthRuntimePaths();
|
||||||
|
return this.paths;
|
||||||
|
}
|
||||||
|
|
||||||
|
private consumeStdout(chunk: string) {
|
||||||
|
this.stdoutBuffer += chunk;
|
||||||
|
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
|
||||||
|
this.child?.kill();
|
||||||
|
this.handleChildExit(this.child);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (true) {
|
||||||
|
const newline = this.stdoutBuffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
const line = this.stdoutBuffer.slice(0, newline).trim();
|
||||||
|
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||||
|
if (!line) continue;
|
||||||
|
let value: unknown;
|
||||||
|
try { value = JSON.parse(line); } catch { continue; }
|
||||||
|
if (!isRecord(value) || typeof value.requestId !== "string") continue;
|
||||||
|
const pending = this.pending.get(value.requestId);
|
||||||
|
if (!pending) continue;
|
||||||
|
this.pending.delete(value.requestId);
|
||||||
|
if (value.ok === true) {
|
||||||
|
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
|
||||||
|
pending.reject(new DepthRuntimeError("depth_response_invalid", "本机深度响应无效", 500));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
this.loaded = true;
|
||||||
|
pending.resolve(value as unknown as DepthWorkerSuccess);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const code = typeof value.code === "string" ? value.code : "depth_inference_failed";
|
||||||
|
const statusCode = code === "depth_model_missing" ? 503 : code === "depth_input_invalid" ? 400 : 500;
|
||||||
|
pending.reject(new DepthRuntimeError(code, typeof value.message === "string" ? value.message : "本机深度推理失败", statusCode));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
|
||||||
|
if (!child || this.child !== child) return;
|
||||||
|
this.child = undefined;
|
||||||
|
this.loaded = false;
|
||||||
|
this.stderrBuffer = "";
|
||||||
|
this.stdoutBuffer = "";
|
||||||
|
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已退出"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private rejectPending(error: DepthRuntimeError) {
|
||||||
|
for (const { reject } of this.pending.values()) reject(error);
|
||||||
|
this.pending.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateDepthDataUrl(value: string) {
|
||||||
|
const match = DATA_URL_PATTERN.exec(value);
|
||||||
|
if (!match) throw new DepthRuntimeError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
const bytes = Buffer.from(match[2]!, "base64");
|
||||||
|
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new DepthRuntimeError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseDepthInput(body: Record<string, unknown>) {
|
||||||
|
const keys = Object.keys(body).sort();
|
||||||
|
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "depth" || typeof body.dataUrl !== "string") {
|
||||||
|
throw new DepthRuntimeError("depth_input_invalid", "深度转换请求字段无效", 400);
|
||||||
|
}
|
||||||
|
validateDepthDataUrl(body.dataUrl);
|
||||||
|
return body.dataUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseBody(request: Request): Record<string, unknown> {
|
||||||
|
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(request.body.toString("utf8")) as unknown;
|
||||||
|
return isRecord(value) ? value : {};
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
|
||||||
|
return (request, response, next) => {
|
||||||
|
void action(request, response).catch((error) => {
|
||||||
|
if (response.headersSent) return next(error);
|
||||||
|
if (error instanceof DepthRuntimeError) {
|
||||||
|
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.status(503).json({ ok: false, code: "depth_runtime_unavailable", message: "本机深度运行时不可用" });
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPositiveInteger(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,327 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||||
|
|
||||||
|
import type { Request, RequestHandler, Response } from "express";
|
||||||
|
|
||||||
|
import type { LocalRuntimeModule } from "../local-runtime.js";
|
||||||
|
|
||||||
|
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
|
||||||
|
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
|
||||||
|
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
|
||||||
|
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||||
|
|
||||||
|
export type LineartRuntimePaths = {
|
||||||
|
pythonPath: string;
|
||||||
|
scriptPath: string;
|
||||||
|
hfHome: string;
|
||||||
|
modelId: string;
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
reason?: "python_missing" | "worker_missing";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type LineartRuntimeStatus = {
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
loaded: boolean;
|
||||||
|
device: "cpu";
|
||||||
|
modelId: string;
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type LineartWorkerSuccess = {
|
||||||
|
ok: true;
|
||||||
|
requestId: string;
|
||||||
|
mimeType: "image/png";
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
modelId: string;
|
||||||
|
device: "cpu";
|
||||||
|
pngBase64: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class LineartRuntimeError extends Error {
|
||||||
|
constructor(readonly code: string, message: string, readonly statusCode = 503) {
|
||||||
|
super(message);
|
||||||
|
this.name = "LineartRuntimeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createLineartEstimationHttpModule(): LocalRuntimeModule {
|
||||||
|
const worker = new LineartEstimationWorker();
|
||||||
|
return {
|
||||||
|
descriptor: {
|
||||||
|
id: "lineart-estimation",
|
||||||
|
displayName: "ControlNet Aux 本机 AI 线稿",
|
||||||
|
apiVersion: 1,
|
||||||
|
scopes: ["lineart:status", "lineart:run"],
|
||||||
|
},
|
||||||
|
routes: [
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
path: "/lineart-estimation/status",
|
||||||
|
scope: "lineart:status",
|
||||||
|
handler: asyncRoute(async (_request, response) => {
|
||||||
|
const status = worker.status();
|
||||||
|
response.json({
|
||||||
|
ok: true,
|
||||||
|
module: "lineart-estimation",
|
||||||
|
apiVersion: 1,
|
||||||
|
ready: status.configured && status.modelCached,
|
||||||
|
...status,
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
path: "/lineart-estimation/run",
|
||||||
|
scope: "lineart:run",
|
||||||
|
handler: asyncRoute(async (request, response) => {
|
||||||
|
const body = parseBody(request);
|
||||||
|
const dataUrl = parseLineartInput(body);
|
||||||
|
const result = await worker.run({ dataUrl });
|
||||||
|
response.json({
|
||||||
|
ok: true,
|
||||||
|
module: "lineart-estimation",
|
||||||
|
apiVersion: 1,
|
||||||
|
result,
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
dispose: () => worker.dispose(),
|
||||||
|
publicHealth: () => {
|
||||||
|
const status = worker.status();
|
||||||
|
return { lineartEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveLineartRuntimePaths(env: NodeJS.ProcessEnv = process.env): LineartRuntimePaths {
|
||||||
|
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||||
|
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
|
||||||
|
const projectRoots = [
|
||||||
|
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
|
||||||
|
path.resolve(moduleRoot, ".."),
|
||||||
|
path.resolve(process.cwd()),
|
||||||
|
].filter((value): value is string => Boolean(value));
|
||||||
|
const projectRoot = configuredProjectRoot
|
||||||
|
? path.resolve(configuredProjectRoot)
|
||||||
|
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
|
||||||
|
const scriptCandidates = [
|
||||||
|
path.join(moduleRoot, "python", "lineart_runtime.py"),
|
||||||
|
path.join(projectRoot, "canvas-agent", "python", "lineart_runtime.py"),
|
||||||
|
];
|
||||||
|
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
|
||||||
|
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
|
||||||
|
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
|
||||||
|
: process.platform === "win32"
|
||||||
|
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
|
||||||
|
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
|
||||||
|
const pythonPath = pythonCandidates[0]!;
|
||||||
|
const hfHome = path.resolve(env.CANVAS_LINEART_HF_HOME?.trim() || env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
|
||||||
|
const modelId = env.CANVAS_LINEART_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
|
||||||
|
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
|
||||||
|
const workerExists = fs.existsSync(scriptPath);
|
||||||
|
const pythonExists = fs.existsSync(pythonPath);
|
||||||
|
return {
|
||||||
|
pythonPath,
|
||||||
|
scriptPath,
|
||||||
|
hfHome,
|
||||||
|
modelId,
|
||||||
|
configured: workerExists && pythonExists,
|
||||||
|
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
|
||||||
|
...(pythonExists ? {} : { reason: "python_missing" as const }),
|
||||||
|
...(!workerExists ? { reason: "worker_missing" as const } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class LineartEstimationWorker {
|
||||||
|
private paths = resolveLineartRuntimePaths();
|
||||||
|
private child?: ChildProcessWithoutNullStreams;
|
||||||
|
private stdoutBuffer = "";
|
||||||
|
private stderrBuffer = "";
|
||||||
|
private requestCounter = 0;
|
||||||
|
private loaded = false;
|
||||||
|
private readonly pending = new Map<string, { resolve: (value: LineartWorkerSuccess) => void; reject: (error: unknown) => void }>();
|
||||||
|
|
||||||
|
status(): LineartRuntimeStatus {
|
||||||
|
const paths = this.refreshPaths();
|
||||||
|
return {
|
||||||
|
configured: paths.configured,
|
||||||
|
modelCached: paths.modelCached,
|
||||||
|
loaded: this.loaded,
|
||||||
|
device: "cpu",
|
||||||
|
modelId: paths.modelId,
|
||||||
|
...(paths.reason ? { reason: paths.reason } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async run(input: { dataUrl: string }) {
|
||||||
|
validateLineartDataUrl(input.dataUrl);
|
||||||
|
const paths = this.refreshPaths();
|
||||||
|
if (!paths.configured) {
|
||||||
|
throw new LineartRuntimeError(
|
||||||
|
"lineart_runtime_unavailable",
|
||||||
|
paths.reason === "worker_missing" ? "本机 AI 线稿运行文件不存在" : "未找到本机 AI 线稿 Python 环境",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const child = this.ensureChild();
|
||||||
|
const requestId = `lineart-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
|
||||||
|
const response = new Promise<LineartWorkerSuccess>((resolve, reject) => {
|
||||||
|
this.pending.set(requestId, { resolve, reject });
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
|
||||||
|
} catch {
|
||||||
|
this.pending.delete(requestId);
|
||||||
|
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法写入");
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
dispose() {
|
||||||
|
const child = this.child;
|
||||||
|
this.child = undefined;
|
||||||
|
this.loaded = false;
|
||||||
|
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已停止"));
|
||||||
|
if (child && !child.killed) child.kill();
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensureChild() {
|
||||||
|
if (this.child && !this.child.killed) return this.child;
|
||||||
|
let child: ChildProcessWithoutNullStreams;
|
||||||
|
try {
|
||||||
|
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
|
||||||
|
cwd: path.dirname(this.paths.scriptPath),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
HF_HOME: this.paths.hfHome,
|
||||||
|
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
|
||||||
|
HF_HUB_OFFLINE: "1",
|
||||||
|
TRANSFORMERS_OFFLINE: "1",
|
||||||
|
HF_HUB_DISABLE_TELEMETRY: "1",
|
||||||
|
HF_HUB_DISABLE_PROGRESS_BARS: "1",
|
||||||
|
TRANSFORMERS_VERBOSITY: "error",
|
||||||
|
CANVAS_LINEART_MODEL_ID: this.paths.modelId,
|
||||||
|
},
|
||||||
|
stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法启动");
|
||||||
|
}
|
||||||
|
this.child = child;
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
|
||||||
|
child.stderr.on("data", (chunk: string) => {
|
||||||
|
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
|
||||||
|
});
|
||||||
|
child.on("error", () => this.handleChildExit(child));
|
||||||
|
child.on("exit", () => this.handleChildExit(child));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
private refreshPaths() {
|
||||||
|
if (!this.child) this.paths = resolveLineartRuntimePaths();
|
||||||
|
return this.paths;
|
||||||
|
}
|
||||||
|
|
||||||
|
private consumeStdout(chunk: string) {
|
||||||
|
this.stdoutBuffer += chunk;
|
||||||
|
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
|
||||||
|
this.child?.kill();
|
||||||
|
this.handleChildExit(this.child);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (true) {
|
||||||
|
const newline = this.stdoutBuffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
const line = this.stdoutBuffer.slice(0, newline).trim();
|
||||||
|
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||||
|
if (!line) continue;
|
||||||
|
let value: unknown;
|
||||||
|
try { value = JSON.parse(line); } catch { continue; }
|
||||||
|
if (!isRecord(value) || typeof value.requestId !== "string") continue;
|
||||||
|
const pending = this.pending.get(value.requestId);
|
||||||
|
if (!pending) continue;
|
||||||
|
this.pending.delete(value.requestId);
|
||||||
|
if (value.ok === true) {
|
||||||
|
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
|
||||||
|
pending.reject(new LineartRuntimeError("lineart_response_invalid", "本机 AI 线稿响应无效", 500));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
this.loaded = true;
|
||||||
|
pending.resolve(value as unknown as LineartWorkerSuccess);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const code = typeof value.code === "string" ? value.code : "lineart_inference_failed";
|
||||||
|
const statusCode = code === "lineart_model_missing" ? 503 : code === "lineart_input_invalid" ? 400 : 500;
|
||||||
|
pending.reject(new LineartRuntimeError(code, typeof value.message === "string" ? value.message : "本机 AI 线稿推理失败", statusCode));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
|
||||||
|
if (!child || this.child !== child) return;
|
||||||
|
this.child = undefined;
|
||||||
|
this.loaded = false;
|
||||||
|
this.stderrBuffer = "";
|
||||||
|
this.stdoutBuffer = "";
|
||||||
|
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已退出"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private rejectPending(error: LineartRuntimeError) {
|
||||||
|
for (const { reject } of this.pending.values()) reject(error);
|
||||||
|
this.pending.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateLineartDataUrl(value: string) {
|
||||||
|
const match = DATA_URL_PATTERN.exec(value);
|
||||||
|
if (!match) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
const bytes = Buffer.from(match[2]!, "base64");
|
||||||
|
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseLineartInput(body: Record<string, unknown>) {
|
||||||
|
const keys = Object.keys(body).sort();
|
||||||
|
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "lineart" || typeof body.dataUrl !== "string") {
|
||||||
|
throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换请求字段无效", 400);
|
||||||
|
}
|
||||||
|
validateLineartDataUrl(body.dataUrl);
|
||||||
|
return body.dataUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseBody(request: Request): Record<string, unknown> {
|
||||||
|
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(request.body.toString("utf8")) as unknown;
|
||||||
|
return isRecord(value) ? value : {};
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
|
||||||
|
return (request, response, next) => {
|
||||||
|
void action(request, response).catch((error) => {
|
||||||
|
if (response.headersSent) return next(error);
|
||||||
|
if (error instanceof LineartRuntimeError) {
|
||||||
|
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.status(503).json({ ok: false, code: "lineart_runtime_unavailable", message: "本机 AI 线稿运行时不可用" });
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPositiveInteger(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,357 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||||
|
|
||||||
|
import type { Request, RequestHandler, Response } from "express";
|
||||||
|
|
||||||
|
import type { LocalRuntimeModule } from "../local-runtime.js";
|
||||||
|
|
||||||
|
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
|
||||||
|
const DEFAULT_BACKEND = "openpose-body" as const;
|
||||||
|
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
|
||||||
|
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
|
||||||
|
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||||
|
|
||||||
|
export type PoseRuntimePaths = {
|
||||||
|
pythonPath: string;
|
||||||
|
scriptPath: string;
|
||||||
|
hfHome: string;
|
||||||
|
modelId: string;
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
reason?: "python_missing" | "worker_missing";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type PoseRuntimeStatus = {
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
loaded: boolean;
|
||||||
|
device: "cpu";
|
||||||
|
modelId: string;
|
||||||
|
backend: typeof DEFAULT_BACKEND;
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type PoseWorkerSuccess = {
|
||||||
|
ok: true;
|
||||||
|
requestId: string;
|
||||||
|
mimeType: "image/png";
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
modelId: string;
|
||||||
|
device: "cpu";
|
||||||
|
backend: typeof DEFAULT_BACKEND;
|
||||||
|
personCount: number;
|
||||||
|
people?: unknown[];
|
||||||
|
pngBase64: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class PoseRuntimeError extends Error {
|
||||||
|
constructor(readonly code: string, message: string, readonly statusCode = 503) {
|
||||||
|
super(message);
|
||||||
|
this.name = "PoseRuntimeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createPoseEstimationHttpModule(): LocalRuntimeModule {
|
||||||
|
const worker = new PoseEstimationWorker();
|
||||||
|
return {
|
||||||
|
descriptor: {
|
||||||
|
id: "pose-estimation",
|
||||||
|
displayName: "OpenPose 本机姿态骨架",
|
||||||
|
apiVersion: 1,
|
||||||
|
scopes: ["pose:status", "pose:run"],
|
||||||
|
},
|
||||||
|
routes: [
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
path: "/pose-estimation/status",
|
||||||
|
scope: "pose:status",
|
||||||
|
handler: asyncRoute(async (_request, response) => {
|
||||||
|
const status = worker.status();
|
||||||
|
response.json({
|
||||||
|
ok: true,
|
||||||
|
module: "pose-estimation",
|
||||||
|
apiVersion: 1,
|
||||||
|
ready: status.configured && status.modelCached,
|
||||||
|
...status,
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
path: "/pose-estimation/run",
|
||||||
|
scope: "pose:run",
|
||||||
|
handler: asyncRoute(async (request, response) => {
|
||||||
|
const body = parseBody(request);
|
||||||
|
const dataUrl = parsePoseInput(body);
|
||||||
|
const result = await worker.run({ dataUrl });
|
||||||
|
response.json({
|
||||||
|
ok: true,
|
||||||
|
module: "pose-estimation",
|
||||||
|
apiVersion: 1,
|
||||||
|
result,
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
dispose: () => worker.dispose(),
|
||||||
|
publicHealth: () => {
|
||||||
|
const status = worker.status();
|
||||||
|
return { poseEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolvePoseRuntimePaths(env: NodeJS.ProcessEnv = process.env): PoseRuntimePaths {
|
||||||
|
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||||
|
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
|
||||||
|
const projectRoots = [
|
||||||
|
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
|
||||||
|
path.resolve(moduleRoot, ".."),
|
||||||
|
path.resolve(process.cwd()),
|
||||||
|
].filter((value): value is string => Boolean(value));
|
||||||
|
const projectRoot = configuredProjectRoot
|
||||||
|
? path.resolve(configuredProjectRoot)
|
||||||
|
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
|
||||||
|
const scriptCandidates = [
|
||||||
|
path.join(moduleRoot, "python", "pose_runtime.py"),
|
||||||
|
path.join(projectRoot, "canvas-agent", "python", "pose_runtime.py"),
|
||||||
|
];
|
||||||
|
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
|
||||||
|
const pythonCandidates = env.CANVAS_POSE_PYTHON?.trim()
|
||||||
|
? [path.resolve(env.CANVAS_POSE_PYTHON.trim())]
|
||||||
|
: process.platform === "win32"
|
||||||
|
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
|
||||||
|
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
|
||||||
|
const pythonPath = pythonCandidates[0]!;
|
||||||
|
const hfHome = path.resolve(
|
||||||
|
env.CANVAS_POSE_HF_HOME?.trim()
|
||||||
|
|| env.CANVAS_LINEART_HF_HOME?.trim()
|
||||||
|
|| env.CANVAS_DEPTH_HF_HOME?.trim()
|
||||||
|
|| path.join(projectRoot, ".local", "cache", "huggingface"),
|
||||||
|
);
|
||||||
|
const modelId = env.CANVAS_POSE_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
|
||||||
|
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
|
||||||
|
const workerExists = fs.existsSync(scriptPath);
|
||||||
|
const pythonExists = fs.existsSync(pythonPath);
|
||||||
|
return {
|
||||||
|
pythonPath,
|
||||||
|
scriptPath,
|
||||||
|
hfHome,
|
||||||
|
modelId,
|
||||||
|
configured: workerExists && pythonExists,
|
||||||
|
modelCached: workerExists && pythonExists && findCachedModelFile(modelCacheDir, "body_pose_model.pth"),
|
||||||
|
...(pythonExists ? {} : { reason: "python_missing" as const }),
|
||||||
|
...(!workerExists ? { reason: "worker_missing" as const } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function findCachedModelFile(modelCacheDir: string, filename: string) {
|
||||||
|
const snapshotsDir = path.join(modelCacheDir, "snapshots");
|
||||||
|
try {
|
||||||
|
return fs.readdirSync(snapshotsDir, { withFileTypes: true })
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.some((entry) => fs.existsSync(path.join(snapshotsDir, entry.name, filename)));
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class PoseEstimationWorker {
|
||||||
|
private paths = resolvePoseRuntimePaths();
|
||||||
|
private child?: ChildProcessWithoutNullStreams;
|
||||||
|
private stdoutBuffer = "";
|
||||||
|
private stderrBuffer = "";
|
||||||
|
private requestCounter = 0;
|
||||||
|
private loaded = false;
|
||||||
|
private readonly pending = new Map<string, { resolve: (value: PoseWorkerSuccess) => void; reject: (error: unknown) => void }>();
|
||||||
|
|
||||||
|
status(): PoseRuntimeStatus {
|
||||||
|
const paths = this.refreshPaths();
|
||||||
|
return {
|
||||||
|
configured: paths.configured,
|
||||||
|
modelCached: paths.modelCached,
|
||||||
|
loaded: this.loaded,
|
||||||
|
device: "cpu",
|
||||||
|
modelId: paths.modelId,
|
||||||
|
backend: DEFAULT_BACKEND,
|
||||||
|
...(paths.reason ? { reason: paths.reason } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async run(input: { dataUrl: string }) {
|
||||||
|
validatePoseDataUrl(input.dataUrl);
|
||||||
|
const paths = this.refreshPaths();
|
||||||
|
if (!paths.configured) {
|
||||||
|
throw new PoseRuntimeError(
|
||||||
|
"pose_runtime_unavailable",
|
||||||
|
paths.reason === "worker_missing" ? "本机姿态运行文件不存在" : "未找到本机姿态 Python 环境",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const child = this.ensureChild();
|
||||||
|
const requestId = `pose-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
|
||||||
|
const response = new Promise<PoseWorkerSuccess>((resolve, reject) => {
|
||||||
|
this.pending.set(requestId, { resolve, reject });
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
|
||||||
|
} catch {
|
||||||
|
this.pending.delete(requestId);
|
||||||
|
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法写入");
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
dispose() {
|
||||||
|
const child = this.child;
|
||||||
|
this.child = undefined;
|
||||||
|
this.loaded = false;
|
||||||
|
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已停止"));
|
||||||
|
if (child && !child.killed) child.kill();
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensureChild() {
|
||||||
|
if (this.child && !this.child.killed) return this.child;
|
||||||
|
let child: ChildProcessWithoutNullStreams;
|
||||||
|
try {
|
||||||
|
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
|
||||||
|
cwd: path.dirname(this.paths.scriptPath),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
HF_HOME: this.paths.hfHome,
|
||||||
|
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
|
||||||
|
HF_HUB_OFFLINE: "1",
|
||||||
|
TRANSFORMERS_OFFLINE: "1",
|
||||||
|
HF_HUB_DISABLE_TELEMETRY: "1",
|
||||||
|
HF_HUB_DISABLE_PROGRESS_BARS: "1",
|
||||||
|
TRANSFORMERS_VERBOSITY: "error",
|
||||||
|
CANVAS_POSE_MODEL_ID: this.paths.modelId,
|
||||||
|
},
|
||||||
|
stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法启动");
|
||||||
|
}
|
||||||
|
this.child = child;
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
|
||||||
|
child.stderr.on("data", (chunk: string) => {
|
||||||
|
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
|
||||||
|
});
|
||||||
|
child.on("error", () => this.handleChildExit(child));
|
||||||
|
child.on("exit", () => this.handleChildExit(child));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
private refreshPaths() {
|
||||||
|
if (!this.child) this.paths = resolvePoseRuntimePaths();
|
||||||
|
return this.paths;
|
||||||
|
}
|
||||||
|
|
||||||
|
private consumeStdout(chunk: string) {
|
||||||
|
this.stdoutBuffer += chunk;
|
||||||
|
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
|
||||||
|
this.child?.kill();
|
||||||
|
this.handleChildExit(this.child);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (true) {
|
||||||
|
const newline = this.stdoutBuffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
const line = this.stdoutBuffer.slice(0, newline).trim();
|
||||||
|
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||||
|
if (!line) continue;
|
||||||
|
let value: unknown;
|
||||||
|
try { value = JSON.parse(line); } catch { continue; }
|
||||||
|
if (!isRecord(value) || typeof value.requestId !== "string") continue;
|
||||||
|
const pending = this.pending.get(value.requestId);
|
||||||
|
if (!pending) continue;
|
||||||
|
this.pending.delete(value.requestId);
|
||||||
|
if (value.ok === true) {
|
||||||
|
if (value.mimeType !== "image/png"
|
||||||
|
|| !isPositiveInteger(value.width)
|
||||||
|
|| !isPositiveInteger(value.height)
|
||||||
|
|| typeof value.modelId !== "string"
|
||||||
|
|| value.device !== "cpu"
|
||||||
|
|| value.backend !== DEFAULT_BACKEND
|
||||||
|
|| !isPositiveInteger(value.personCount)
|
||||||
|
|| typeof value.pngBase64 !== "string"
|
||||||
|
|| !value.pngBase64) {
|
||||||
|
pending.reject(new PoseRuntimeError("pose_response_invalid", "本机姿态响应无效", 500));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
this.loaded = true;
|
||||||
|
pending.resolve(value as unknown as PoseWorkerSuccess);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const code = typeof value.code === "string" ? value.code : "pose_inference_failed";
|
||||||
|
const statusCode = code === "pose_model_missing" ? 503 : code === "pose_input_invalid" ? 400 : code === "pose_no_person" ? 422 : 500;
|
||||||
|
pending.reject(new PoseRuntimeError(code, typeof value.message === "string" ? value.message : "本机姿态推理失败", statusCode));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
|
||||||
|
if (!child || this.child !== child) return;
|
||||||
|
this.child = undefined;
|
||||||
|
this.loaded = false;
|
||||||
|
this.stderrBuffer = "";
|
||||||
|
this.stdoutBuffer = "";
|
||||||
|
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已退出"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private rejectPending(error: PoseRuntimeError) {
|
||||||
|
for (const { reject } of this.pending.values()) reject(error);
|
||||||
|
this.pending.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validatePoseDataUrl(value: string) {
|
||||||
|
const match = DATA_URL_PATTERN.exec(value);
|
||||||
|
if (!match) throw new PoseRuntimeError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
const bytes = Buffer.from(match[2]!, "base64");
|
||||||
|
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new PoseRuntimeError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parsePoseInput(body: Record<string, unknown>) {
|
||||||
|
const keys = Object.keys(body).sort();
|
||||||
|
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "pose" || typeof body.dataUrl !== "string") {
|
||||||
|
throw new PoseRuntimeError("pose_input_invalid", "姿态转换请求字段无效", 400);
|
||||||
|
}
|
||||||
|
validatePoseDataUrl(body.dataUrl);
|
||||||
|
return body.dataUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseBody(request: Request): Record<string, unknown> {
|
||||||
|
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(request.body.toString("utf8")) as unknown;
|
||||||
|
return isRecord(value) ? value : {};
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
|
||||||
|
return (request, response, next) => {
|
||||||
|
void action(request, response).catch((error) => {
|
||||||
|
if (response.headersSent) return next(error);
|
||||||
|
if (error instanceof PoseRuntimeError) {
|
||||||
|
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.status(503).json({ ok: false, code: "pose_runtime_unavailable", message: "本机姿态运行时不可用" });
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPositiveInteger(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "node:test";
|
||||||
|
|
||||||
|
import { createDepthEstimationHttpModule, DepthRuntimeError, resolveDepthRuntimePaths, validateDepthDataUrl } from "../src/modules/depth-estimation-http.js";
|
||||||
|
|
||||||
|
test("depth runtime module declares signed status and run routes", () => {
|
||||||
|
const module = createDepthEstimationHttpModule();
|
||||||
|
assert.equal(module.descriptor.id, "depth-estimation");
|
||||||
|
assert.deepEqual(module.descriptor.scopes, ["depth:status", "depth:run"]);
|
||||||
|
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
|
||||||
|
"GET /depth-estimation/status",
|
||||||
|
"POST /depth-estimation/run",
|
||||||
|
]);
|
||||||
|
module.dispose?.();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("depth input guard accepts image data URLs and rejects unsafe input", () => {
|
||||||
|
assert.doesNotThrow(() => validateDepthDataUrl("data:image/png;base64,AQID"));
|
||||||
|
assert.throws(
|
||||||
|
() => validateDepthDataUrl("data:text/plain;base64,AQID"),
|
||||||
|
(error: unknown) => error instanceof DepthRuntimeError && error.code === "depth_input_invalid" && error.statusCode === 400,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("depth runtime paths honor explicit project and Python overrides", () => {
|
||||||
|
const paths = resolveDepthRuntimePaths({
|
||||||
|
CANVAS_PROJECT_ROOT: "C:\\depth-project",
|
||||||
|
CANVAS_DEPTH_PYTHON: "C:\\depth-project\\venv\\python.exe",
|
||||||
|
CANVAS_DEPTH_HF_HOME: "C:\\depth-cache",
|
||||||
|
CANVAS_DEPTH_MODEL_ID: "example/depth-model",
|
||||||
|
});
|
||||||
|
assert.equal(paths.pythonPath, "C:\\depth-project\\venv\\python.exe");
|
||||||
|
assert.equal(paths.hfHome, "C:\\depth-cache");
|
||||||
|
assert.equal(paths.modelId, "example/depth-model");
|
||||||
|
assert.equal(paths.configured, false);
|
||||||
|
});
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "node:test";
|
||||||
|
|
||||||
|
import { createLineartEstimationHttpModule, LineartRuntimeError, resolveLineartRuntimePaths, validateLineartDataUrl } from "../src/modules/lineart-estimation-http.js";
|
||||||
|
|
||||||
|
test("lineart runtime module declares signed status and run routes", () => {
|
||||||
|
const module = createLineartEstimationHttpModule();
|
||||||
|
assert.equal(module.descriptor.id, "lineart-estimation");
|
||||||
|
assert.deepEqual(module.descriptor.scopes, ["lineart:status", "lineart:run"]);
|
||||||
|
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
|
||||||
|
"GET /lineart-estimation/status",
|
||||||
|
"POST /lineart-estimation/run",
|
||||||
|
]);
|
||||||
|
module.dispose?.();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("lineart input guard accepts image data URLs and rejects unsafe input", () => {
|
||||||
|
assert.doesNotThrow(() => validateLineartDataUrl("data:image/png;base64,AQID"));
|
||||||
|
assert.throws(
|
||||||
|
() => validateLineartDataUrl("data:text/plain;base64,AQID"),
|
||||||
|
(error: unknown) => error instanceof LineartRuntimeError && error.code === "lineart_input_invalid" && error.statusCode === 400,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("lineart runtime paths honor explicit project and Python overrides", () => {
|
||||||
|
const paths = resolveLineartRuntimePaths({
|
||||||
|
CANVAS_PROJECT_ROOT: "C:\\lineart-project",
|
||||||
|
CANVAS_DEPTH_PYTHON: "C:\\lineart-project\\venv\\python.exe",
|
||||||
|
CANVAS_LINEART_HF_HOME: "C:\\lineart-cache",
|
||||||
|
CANVAS_LINEART_MODEL_ID: "example/lineart-model",
|
||||||
|
});
|
||||||
|
assert.equal(paths.pythonPath, "C:\\lineart-project\\venv\\python.exe");
|
||||||
|
assert.equal(paths.hfHome, "C:\\lineart-cache");
|
||||||
|
assert.equal(paths.modelId, "example/lineart-model");
|
||||||
|
assert.equal(paths.configured, false);
|
||||||
|
});
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "node:test";
|
||||||
|
|
||||||
|
import { createPoseEstimationHttpModule, PoseRuntimeError, resolvePoseRuntimePaths, validatePoseDataUrl } from "../src/modules/pose-estimation-http.js";
|
||||||
|
|
||||||
|
test("pose runtime module declares signed status and run routes", () => {
|
||||||
|
const module = createPoseEstimationHttpModule();
|
||||||
|
assert.equal(module.descriptor.id, "pose-estimation");
|
||||||
|
assert.deepEqual(module.descriptor.scopes, ["pose:status", "pose:run"]);
|
||||||
|
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
|
||||||
|
"GET /pose-estimation/status",
|
||||||
|
"POST /pose-estimation/run",
|
||||||
|
]);
|
||||||
|
module.dispose?.();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("pose input guard accepts image data URLs and rejects unsafe input", () => {
|
||||||
|
assert.doesNotThrow(() => validatePoseDataUrl("data:image/png;base64,AQID"));
|
||||||
|
assert.throws(
|
||||||
|
() => validatePoseDataUrl("data:text/plain;base64,AQID"),
|
||||||
|
(error: unknown) => error instanceof PoseRuntimeError && error.code === "pose_input_invalid" && error.statusCode === 400,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("pose runtime paths honor explicit project and Python overrides", () => {
|
||||||
|
const paths = resolvePoseRuntimePaths({
|
||||||
|
CANVAS_PROJECT_ROOT: "C:\\pose-project",
|
||||||
|
CANVAS_POSE_PYTHON: "C:\\pose-project\\venv\\python.exe",
|
||||||
|
CANVAS_POSE_HF_HOME: "C:\\pose-cache",
|
||||||
|
CANVAS_POSE_MODEL_ID: "example/pose-model",
|
||||||
|
});
|
||||||
|
assert.equal(paths.pythonPath, "C:\\pose-project\\venv\\python.exe");
|
||||||
|
assert.equal(paths.hfHome, "C:\\pose-cache");
|
||||||
|
assert.equal(paths.modelId, "example/pose-model");
|
||||||
|
assert.equal(paths.configured, false);
|
||||||
|
assert.equal(paths.modelCached, false);
|
||||||
|
});
|
||||||
@@ -373,6 +373,7 @@ function nodeTypeLabel(node?: CanvasNodeData | null) {
|
|||||||
if (node.type === CanvasNodeType.Video) return "视频节点";
|
if (node.type === CanvasNodeType.Video) return "视频节点";
|
||||||
if (node.type === CanvasNodeType.Audio) return "音频节点";
|
if (node.type === CanvasNodeType.Audio) return "音频节点";
|
||||||
if (node.type === CanvasNodeType.Drawing) return "绘图节点";
|
if (node.type === CanvasNodeType.Drawing) return "绘图节点";
|
||||||
|
if (node.type === CanvasNodeType.MediaConversion) return "转换节点";
|
||||||
if (node.type === CanvasNodeType.Frame) return isCanvasFolderNode(node) ? "文件夹" : "背板";
|
if (node.type === CanvasNodeType.Frame) return isCanvasFolderNode(node) ? "文件夹" : "背板";
|
||||||
return "生成配置节点";
|
return "生成配置节点";
|
||||||
}
|
}
|
||||||
@@ -37,6 +37,8 @@ import { PanoramaNodeContent } from "./nodes/panorama-node";
|
|||||||
import { SvgNodeContent } from "./nodes/svg-node";
|
import { SvgNodeContent } from "./nodes/svg-node";
|
||||||
import { PortraitClearanceNodeContent } from "./nodes/portrait-clearance-node";
|
import { PortraitClearanceNodeContent } from "./nodes/portrait-clearance-node";
|
||||||
import { ArtCritiqueNodeContent } from "./nodes/ai-art-critique-node";
|
import { ArtCritiqueNodeContent } from "./nodes/ai-art-critique-node";
|
||||||
|
import { MediaConversionNodeContent } from "./nodes/media-conversion-node";
|
||||||
|
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
|
||||||
|
|
||||||
export type CanvasNodeContentProps = {
|
export type CanvasNodeContentProps = {
|
||||||
node: CanvasNodeData;
|
node: CanvasNodeData;
|
||||||
@@ -74,6 +76,7 @@ export function CanvasNodeContent(props: CanvasNodeContentProps) {
|
|||||||
if (hasCustomContent && props.renderNodeContent) return props.renderNodeContent(props.node);
|
if (hasCustomContent && props.renderNodeContent) return props.renderNodeContent(props.node);
|
||||||
if (props.node.type === PORTRAIT_CLEARANCE_NODE_TYPE) return <PortraitClearanceNodeContent node={props.node} />;
|
if (props.node.type === PORTRAIT_CLEARANCE_NODE_TYPE) return <PortraitClearanceNodeContent node={props.node} />;
|
||||||
if (props.node.type === ART_CRITIQUE_NODE_TYPE) return <ArtCritiqueNodeContent node={props.node} />;
|
if (props.node.type === ART_CRITIQUE_NODE_TYPE) return <ArtCritiqueNodeContent node={props.node} />;
|
||||||
|
if (props.node.type === MEDIA_CONVERSION_NODE_TYPE) return <MediaConversionNodeContent node={props.node} theme={props.theme} />;
|
||||||
if (props.isBatchRoot) return <ImageNodeContent {...props} />;
|
if (props.isBatchRoot) return <ImageNodeContent {...props} />;
|
||||||
if (props.node.metadata?.status === "loading") return <LoadingContent node={props.node} theme={props.theme} onOpenTaskDetails={props.onOpenTaskDetails} />;
|
if (props.node.metadata?.status === "loading") return <LoadingContent node={props.node} theme={props.theme} onOpenTaskDetails={props.onOpenTaskDetails} />;
|
||||||
if (props.node.metadata?.status === "error") return <ErrorContent node={props.node} theme={props.theme} onRetry={props.onRetry} onReloadResource={props.onReloadResource} />;
|
if (props.node.metadata?.status === "error") return <ErrorContent node={props.node} theme={props.theme} onRetry={props.onRetry} onReloadResource={props.onReloadResource} />;
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import { getGenerationResourceNodes, getContextResourceNodes, getMentionResource
|
|||||||
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
|
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
|
||||||
import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade";
|
import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade";
|
||||||
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
|
import { getNodeResourceKind } from "@/lib/canvas/node-registry";
|
||||||
|
import { mediaConversionSourceFingerprint } from "@/lib/media-conversion/contracts";
|
||||||
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
|
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
|
||||||
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
|
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
|
||||||
import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation";
|
import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation";
|
||||||
@@ -64,6 +65,12 @@ export type NodeGenerationInput = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, assets: Asset[], promptOnly = false): NodeGenerationContext {
|
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, assets: Asset[], promptOnly = false): NodeGenerationContext {
|
||||||
|
const pendingConversion = findPendingMediaConversionInput(nodeId, nodes, connections);
|
||||||
|
if (pendingConversion) {
|
||||||
|
const status = pendingConversion.metadata?.mediaConversion?.status;
|
||||||
|
const reason = status === "stale" ? "输入或方式已变化" : status === "processing" ? "仍在处理中" : "尚未完成本地转换";
|
||||||
|
throw new Error(`转换节点「${pendingConversion.title || pendingConversion.id}」${reason},完成后才能执行下游生成`);
|
||||||
|
}
|
||||||
const connectedInputs = buildNodeGenerationInputs(nodeId, nodes, connections);
|
const connectedInputs = buildNodeGenerationInputs(nodeId, nodes, connections);
|
||||||
const sourceNode = nodes.find((node) => node.id === nodeId);
|
const sourceNode = nodes.find((node) => node.id === nodeId);
|
||||||
const portraitTextureInput = sourceNode?.type === CanvasNodeType.Image && sourceNode.metadata?.content && sourceNode.metadata?.portraitTexture
|
const portraitTextureInput = sourceNode?.type === CanvasNodeType.Image && sourceNode.metadata?.content && sourceNode.metadata?.portraitTexture
|
||||||
@@ -120,6 +127,40 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 转换节点允许提前连到下游,但只有已物化的结果才能进入生成请求。
|
||||||
|
* 沿上游链路检查也覆盖“生成节点 -> 配置节点 -> 转换节点”的常见接法。
|
||||||
|
*/
|
||||||
|
export function findPendingMediaConversionInput(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
|
||||||
|
const nodesById = new Map(nodes.map((node) => [node.id, node]));
|
||||||
|
const queue = connections.filter((connection) => connection.toNodeId === nodeId).map((connection) => connection.fromNodeId);
|
||||||
|
const visited = new Set<string>();
|
||||||
|
while (queue.length) {
|
||||||
|
const sourceId = queue.shift()!;
|
||||||
|
if (visited.has(sourceId)) continue;
|
||||||
|
visited.add(sourceId);
|
||||||
|
const source = nodesById.get(sourceId);
|
||||||
|
if (!source) continue;
|
||||||
|
if (source.type === CanvasNodeType.MediaConversion) {
|
||||||
|
const state = source.metadata?.mediaConversion;
|
||||||
|
const sourceInputs = connections
|
||||||
|
.filter((connection) => connection.toNodeId === source.id)
|
||||||
|
.map((connection) => nodesById.get(connection.fromNodeId))
|
||||||
|
.filter((input): input is CanvasNodeData => Boolean(input && (input.type === CanvasNodeType.Image || input.type === CanvasNodeType.Video)));
|
||||||
|
const sourceInput = sourceInputs[0];
|
||||||
|
const ready = state?.status === "completed"
|
||||||
|
&& Boolean(state.resultStorageKey)
|
||||||
|
&& sourceInputs.length === 1
|
||||||
|
&& Boolean(sourceInput)
|
||||||
|
&& state.sourceNodeId === sourceInput?.id
|
||||||
|
&& state.sourceFingerprint === mediaConversionSourceFingerprint(sourceInput!);
|
||||||
|
if (!ready) return source;
|
||||||
|
}
|
||||||
|
connections.filter((connection) => connection.toNodeId === sourceId).forEach((connection) => queue.push(connection.fromNodeId));
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]) {
|
function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]) {
|
||||||
let next = prompt.trim();
|
let next = prompt.trim();
|
||||||
let removed = true;
|
let removed = true;
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ type CanvasNodeToolbarProps = {
|
|||||||
onUpload: (node: CanvasNodeData) => void;
|
onUpload: (node: CanvasNodeData) => void;
|
||||||
onDownload: (node: CanvasNodeData) => void;
|
onDownload: (node: CanvasNodeData) => void;
|
||||||
onSaveAsset: (node: CanvasNodeData) => void;
|
onSaveAsset: (node: CanvasNodeData) => void;
|
||||||
|
onCreateConversion?: (node: CanvasNodeData) => void;
|
||||||
onMaskEdit: (node: CanvasNodeData) => void;
|
onMaskEdit: (node: CanvasNodeData) => void;
|
||||||
onEmotion: (node: CanvasNodeData) => void;
|
onEmotion: (node: CanvasNodeData) => void;
|
||||||
onPortraitTexture: (node: CanvasNodeData) => void;
|
onPortraitTexture: (node: CanvasNodeData) => void;
|
||||||
@@ -91,6 +92,7 @@ export function CanvasNodeToolbar({
|
|||||||
onUpload,
|
onUpload,
|
||||||
onDownload,
|
onDownload,
|
||||||
onSaveAsset,
|
onSaveAsset,
|
||||||
|
onCreateConversion,
|
||||||
onMaskEdit,
|
onMaskEdit,
|
||||||
onEmotion,
|
onEmotion,
|
||||||
onPortraitTexture,
|
onPortraitTexture,
|
||||||
@@ -221,7 +223,7 @@ export function CanvasNodeToolbar({
|
|||||||
const nodeHoverHandlers = {
|
const nodeHoverHandlers = {
|
||||||
onNodeInfo: onInfo, onNodeDelete: onDelete, onNodeRetry: onRetry, onNodeEditText: onEditText, onNodeDecreaseFont: onDecreaseFont, onNodeIncreaseFont: onIncreaseFont,
|
onNodeInfo: onInfo, onNodeDelete: onDelete, onNodeRetry: onRetry, onNodeEditText: onEditText, onNodeDecreaseFont: onDecreaseFont, onNodeIncreaseFont: onIncreaseFont,
|
||||||
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
|
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
|
||||||
onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
|
onNodeSaveAsset: onSaveAsset, onNodeCreateConversion: onCreateConversion, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
|
||||||
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
|
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
|
||||||
onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
|
onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
|
||||||
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
|
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
|
||||||
@@ -400,7 +402,7 @@ export function CanvasNodeInfoModal({ node, open, onClose, onMetadataChange, rea
|
|||||||
const [assetCategory, setAssetCategory] = useState<CanvasAssetCategory>("other");
|
const [assetCategory, setAssetCategory] = useState<CanvasAssetCategory>("other");
|
||||||
const imageBytes = node?.type === CanvasNodeType.Image && node.metadata?.content ? getDataUrlByteSize(node.metadata.content) : 0;
|
const imageBytes = node?.type === CanvasNodeType.Image && node.metadata?.content ? getDataUrlByteSize(node.metadata.content) : 0;
|
||||||
const batchCount = node?.type === CanvasNodeType.Image ? node.metadata?.batchChildIds?.length || 0 : 0;
|
const batchCount = node?.type === CanvasNodeType.Image ? node.metadata?.batchChildIds?.length || 0 : 0;
|
||||||
const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : "生成配置";
|
const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : node?.type === CanvasNodeType.MediaConversion ? "转换" : "生成配置";
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setAssetTags(node?.metadata?.assetTags || []);
|
setAssetTags(node?.metadata?.assetTags || []);
|
||||||
setAssetTagInput("");
|
setAssetTagInput("");
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import React, { useCallback, useEffect, useRef, useState } from "react";
|
import React, { useCallback, useEffect, useRef, useState } from "react";
|
||||||
import type { ReactNode } from "react";
|
import type { ReactNode } from "react";
|
||||||
import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video } from "lucide-react";
|
import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video, WandSparkles } from "lucide-react";
|
||||||
|
|
||||||
import { useCanvasNodeActions } from "./canvas-node-action-context";
|
import { useCanvasNodeActions } from "./canvas-node-action-context";
|
||||||
|
|
||||||
@@ -709,6 +709,7 @@ function nodeTypeIcon(type: CanvasNodeTypeId) {
|
|||||||
if (type === CanvasNodeType.Drawing) return Pencil;
|
if (type === CanvasNodeType.Drawing) return Pencil;
|
||||||
if (type === CanvasNodeType.Script) return Clapperboard;
|
if (type === CanvasNodeType.Script) return Clapperboard;
|
||||||
if (type === CanvasNodeType.Config) return Settings2;
|
if (type === CanvasNodeType.Config) return Settings2;
|
||||||
|
if (type === CanvasNodeType.MediaConversion) return WandSparkles;
|
||||||
if (type === CanvasNodeType.Skill) return BookOpenCheck;
|
if (type === CanvasNodeType.Skill) return BookOpenCheck;
|
||||||
if (type === PORTRAIT_CLEARANCE_NODE_TYPE) return PortraitClearanceIcon;
|
if (type === PORTRAIT_CLEARANCE_NODE_TYPE) return PortraitClearanceIcon;
|
||||||
if (type === ART_CRITIQUE_NODE_TYPE) return ScanSearch;
|
if (type === ART_CRITIQUE_NODE_TYPE) return ScanSearch;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { motion, useReducedMotion } from "motion/react";
|
import { motion, useReducedMotion } from "motion/react";
|
||||||
import { useLayoutEffect, useRef, useState, type CSSProperties, type ReactNode, type RefObject } from "react";
|
import { useLayoutEffect, useRef, useState, type CSSProperties, type ReactNode, type RefObject } from "react";
|
||||||
import { Clapperboard, Image as ImageIcon, List, Music2, Pencil, Video, Workflow as WorkflowIcon } from "lucide-react";
|
import { Clapperboard, Image as ImageIcon, List, Music2, Pencil, Video, WandSparkles, Workflow as WorkflowIcon } from "lucide-react";
|
||||||
|
|
||||||
import { useCanvasOverlayLayer } from "@/components/canvas/canvas-overlay-layer";
|
import { useCanvasOverlayLayer } from "@/components/canvas/canvas-overlay-layer";
|
||||||
import { canvasThemes } from "@/lib/canvas-theme";
|
import { canvasThemes } from "@/lib/canvas-theme";
|
||||||
@@ -152,7 +152,7 @@ function resolveNodePanelWidth(node: CanvasNodeData, viewport: ViewportTransform
|
|||||||
return clamp(Math.round(node.width * viewport.k * 1.5), 680, 920);
|
return clamp(Math.round(node.width * viewport.k * 1.5), 680, 920);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) {
|
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) {
|
||||||
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
||||||
const reducedMotion = useReducedMotion();
|
const reducedMotion = useReducedMotion();
|
||||||
const menuRef = useRef<HTMLDivElement>(null);
|
const menuRef = useRef<HTMLDivElement>(null);
|
||||||
@@ -160,7 +160,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co
|
|||||||
const lastPointerRef = useRef<Position | null>(null);
|
const lastPointerRef = useRef<Position | null>(null);
|
||||||
const { bringToFront, zIndex } = useCanvasOverlayLayer("connection-create-menu", "var(--z-modal-overlay)");
|
const { bringToFront, zIndex } = useCanvasOverlayLayer("connection-create-menu", "var(--z-modal-overlay)");
|
||||||
const menuWidth = Math.min(288, viewportSize.width - 24);
|
const menuWidth = Math.min(288, viewportSize.width - 24);
|
||||||
const menuHeight = canCreateDrawing ? 360 : 316;
|
const menuHeight = canCreateDrawing ? 404 : 360;
|
||||||
const gap = 12;
|
const gap = 12;
|
||||||
const initialPosition = getConnectionMenuPosition(pending.position, viewport, viewportSize, menuWidth, menuHeight, gap);
|
const initialPosition = getConnectionMenuPosition(pending.position, viewport, viewportSize, menuWidth, menuHeight, gap);
|
||||||
|
|
||||||
@@ -231,6 +231,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co
|
|||||||
{canCreateDrawing ? <ConnectionCreateOption expanded={activeOption === "绘图"} motionEnabled={!reducedMotion} icon={<Pencil className="size-4" />} title="绘图" description="以参考图片为底图,自由绘制和标注" disabledReason={getDisabledReason(CanvasNodeType.Drawing)} onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null}
|
{canCreateDrawing ? <ConnectionCreateOption expanded={activeOption === "绘图"} motionEnabled={!reducedMotion} icon={<Pencil className="size-4" />} title="绘图" description="以参考图片为底图,自由绘制和标注" disabledReason={getDisabledReason(CanvasNodeType.Drawing)} onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null}
|
||||||
<ConnectionCreateOption expanded={activeOption === "视频生成"} motionEnabled={!reducedMotion} icon={<Video className="size-4" />} title="视频生成" description="结合提示词与参考素材,生成动态视频" disabledReason={getDisabledReason(CanvasNodeType.Video)} onClick={() => onCreate(CanvasNodeType.Video)} />
|
<ConnectionCreateOption expanded={activeOption === "视频生成"} motionEnabled={!reducedMotion} icon={<Video className="size-4" />} title="视频生成" description="结合提示词与参考素材,生成动态视频" disabledReason={getDisabledReason(CanvasNodeType.Video)} onClick={() => onCreate(CanvasNodeType.Video)} />
|
||||||
<ConnectionCreateOption expanded={activeOption === "音频参考"} motionEnabled={!reducedMotion} icon={<Music2 className="size-4" />} title="音频参考" description="连接文本或角色卡,创建音频生成节点" disabledReason={getDisabledReason(CanvasNodeType.Audio)} onClick={() => onCreate(CanvasNodeType.Audio)} />
|
<ConnectionCreateOption expanded={activeOption === "音频参考"} motionEnabled={!reducedMotion} icon={<Music2 className="size-4" />} title="音频参考" description="连接文本或角色卡,创建音频生成节点" disabledReason={getDisabledReason(CanvasNodeType.Audio)} onClick={() => onCreate(CanvasNodeType.Audio)} />
|
||||||
|
<ConnectionCreateOption expanded={activeOption === "转换"} motionEnabled={!reducedMotion} icon={<WandSparkles className="size-4" />} title="转换" description="本地处理图片或视频" disabledReason={getDisabledReason(CanvasNodeType.MediaConversion)} onClick={() => onCreate(CanvasNodeType.MediaConversion)} />
|
||||||
</div>
|
</div>
|
||||||
</motion.div>
|
</motion.div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,360 @@
|
|||||||
|
import { CheckCircle2, CircleAlert, Download, Image as ImageIcon, LoaderCircle, RefreshCw, Video, WandSparkles } from "lucide-react";
|
||||||
|
import { useEffect, useRef, useState } from "react";
|
||||||
|
|
||||||
|
import { getNodeInputKind } from "@/lib/canvas/node-registry";
|
||||||
|
import type { CanvasTheme } from "@/lib/canvas-theme";
|
||||||
|
import {
|
||||||
|
createDefaultMediaConversionState,
|
||||||
|
isLocalImageOperation,
|
||||||
|
mediaConversionOperationDescription,
|
||||||
|
mediaConversionOperationLabel,
|
||||||
|
mediaConversionSourceFingerprint,
|
||||||
|
MEDIA_CONVERSION_OPERATION_LABELS,
|
||||||
|
type MediaConversionNodeState,
|
||||||
|
type MediaConversionOperation,
|
||||||
|
} from "@/lib/media-conversion/contracts";
|
||||||
|
import { convertImageLocally, LocalImageConversionError } from "@/lib/media-conversion/local-converter";
|
||||||
|
import { getActiveUserScope } from "@/lib/user-scope";
|
||||||
|
import { runLocalDepthEstimation } from "@/services/depth-runtime";
|
||||||
|
import { runLocalLineartEstimation } from "@/services/lineart-runtime";
|
||||||
|
import { runLocalPoseEstimation } from "@/services/pose-runtime";
|
||||||
|
import { resolveImageUrl, setImageBlob } from "@/services/image-storage";
|
||||||
|
import { resolveMediaUrl } from "@/services/file-storage";
|
||||||
|
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
|
||||||
|
import { useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
|
||||||
|
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
|
||||||
|
|
||||||
|
import { useCanvasNodeActions } from "../canvas-node-action-context";
|
||||||
|
import { useUpstreamNodes } from "../canvas-node-graph-context";
|
||||||
|
|
||||||
|
type MediaConversionNodeContentProps = {
|
||||||
|
node: CanvasNodeData;
|
||||||
|
theme: CanvasTheme;
|
||||||
|
};
|
||||||
|
|
||||||
|
const OPERATIONS = Object.keys(MEDIA_CONVERSION_OPERATION_LABELS) as MediaConversionOperation[];
|
||||||
|
|
||||||
|
export function MediaConversionNodeContent({ node, theme }: MediaConversionNodeContentProps) {
|
||||||
|
const { updateMetadata } = useCanvasNodeActions();
|
||||||
|
const upstream = useUpstreamNodes(node.id);
|
||||||
|
const localRuntimeConnection = useLocalRuntimeStore((runtime) => runtime.connection);
|
||||||
|
const localRuntimeConnecting = useLocalRuntimeStore((runtime) => runtime.connecting);
|
||||||
|
const reconnectLocalRuntime = useLocalRuntimeStore((runtime) => runtime.connect);
|
||||||
|
const mediaInputs = upstream.filter((item) => item.type === CanvasNodeType.Image || item.type === CanvasNodeType.Video);
|
||||||
|
const input = mediaInputs[0];
|
||||||
|
const inputKind = input ? getNodeInputKind(input.type) : undefined;
|
||||||
|
const storedState = node.metadata?.mediaConversion;
|
||||||
|
const state = storedState || createDefaultMediaConversionState();
|
||||||
|
const currentFingerprint = input ? mediaConversionSourceFingerprint(input) : "";
|
||||||
|
const hasMultipleInputs = mediaInputs.length > 1;
|
||||||
|
const isStale = Boolean(state.status === "completed" && (!currentFingerprint || state.sourceFingerprint !== currentFingerprint));
|
||||||
|
const status = isStale ? "stale" : state.status;
|
||||||
|
const [sourceUrl, setSourceUrl] = useState("");
|
||||||
|
const [resultUrl, setResultUrl] = useState("");
|
||||||
|
const [showResult, setShowResult] = useState(status === "completed" || status === "stale");
|
||||||
|
const [notice, setNotice] = useState("");
|
||||||
|
const abortRef = useRef<AbortController | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let active = true;
|
||||||
|
const fallback = input?.metadata?.previewContent || input?.metadata?.content || "";
|
||||||
|
setSourceUrl(fallback);
|
||||||
|
if (!input?.metadata?.storageKey) return () => { active = false; };
|
||||||
|
const resolver = input.type === CanvasNodeType.Image ? resolveImageUrl : resolveMediaUrl;
|
||||||
|
void resolver(input.metadata.storageKey, fallback)
|
||||||
|
.then((resolved) => { if (active) setSourceUrl(resolved || fallback); })
|
||||||
|
.catch(() => { if (active) setSourceUrl(fallback); });
|
||||||
|
return () => { active = false; };
|
||||||
|
}, [input?.id, input?.metadata?.content, input?.metadata?.previewContent, input?.metadata?.storageKey, input?.type]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let active = true;
|
||||||
|
setResultUrl("");
|
||||||
|
if (!state.resultStorageKey) return () => { active = false; };
|
||||||
|
void resolveImageUrl(state.resultStorageKey)
|
||||||
|
.then((resolved) => { if (active) setResultUrl(resolved); })
|
||||||
|
.catch(() => { if (active) setResultUrl(""); });
|
||||||
|
return () => { active = false; };
|
||||||
|
}, [state.resultStorageKey]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!updateMetadata || state.status !== "completed" || (currentFingerprint && state.sourceFingerprint === currentFingerprint)) return;
|
||||||
|
updateMetadata(node.id, {
|
||||||
|
mediaConversion: { ...state, status: "stale", updatedAt: new Date().toISOString() },
|
||||||
|
});
|
||||||
|
}, [currentFingerprint, node.id, state, updateMetadata]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (status === "stale") setShowResult(false);
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
|
useEffect(() => () => abortRef.current?.abort(), []);
|
||||||
|
|
||||||
|
const updateState = (patch: Partial<MediaConversionNodeState>) => {
|
||||||
|
updateMetadata?.(node.id, { mediaConversion: { ...state, ...patch, schemaVersion: 1 } });
|
||||||
|
};
|
||||||
|
|
||||||
|
const selectOperation = (operation: MediaConversionOperation) => {
|
||||||
|
if (operation === state.operation) return;
|
||||||
|
setNotice("");
|
||||||
|
updateState({
|
||||||
|
operation,
|
||||||
|
status: state.resultStorageKey ? "stale" : "idle",
|
||||||
|
errorCode: undefined,
|
||||||
|
errorMessage: undefined,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
setShowResult(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
setNotice("");
|
||||||
|
if (!input || !currentFingerprint) {
|
||||||
|
setNotice("请先连接一张图片或一个视频");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (hasMultipleInputs) {
|
||||||
|
setNotice("转换节点只能连接一个输入,请移除多余连线");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (inputKind === "video") {
|
||||||
|
updateState({ status: "unavailable", outputKind: "video", errorCode: "video_not_ready", errorMessage: "本地视频转换还在进行短片性能验证", updatedAt: new Date().toISOString() });
|
||||||
|
setNotice("视频转换暂未开启,先完成单图模型验证");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isLocalImageOperation(state.operation) && state.operation !== "depth" && state.operation !== "lineart" && state.operation !== "pose") {
|
||||||
|
updateState({ status: "unavailable", outputKind: "image", errorCode: "model_missing", errorMessage: "该转换需要先安装并验证本地模型", updatedAt: new Date().toISOString() });
|
||||||
|
setNotice(mediaConversionOperationDescription(state.operation));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!sourceUrl) {
|
||||||
|
updateState({ status: "error", errorCode: "source_unreadable", errorMessage: "图片预览尚未准备好,请稍后重试", updatedAt: new Date().toISOString() });
|
||||||
|
setNotice("图片预览尚未准备好,请稍后重试");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
abortRef.current?.abort();
|
||||||
|
const controller = new AbortController();
|
||||||
|
abortRef.current = controller;
|
||||||
|
const startedAt = new Date().toISOString();
|
||||||
|
updateState({ status: "processing", sourceNodeId: input.id, sourceFingerprint: currentFingerprint, outputKind: "image", errorCode: undefined, errorMessage: undefined, startedAt, updatedAt: startedAt });
|
||||||
|
try {
|
||||||
|
const result = state.operation === "depth"
|
||||||
|
? await runLocalDepthEstimation(sourceUrl, controller.signal)
|
||||||
|
: state.operation === "lineart"
|
||||||
|
? await runLocalLineartEstimation(sourceUrl, controller.signal)
|
||||||
|
: state.operation === "pose"
|
||||||
|
? await runLocalPoseEstimation(sourceUrl, controller.signal)
|
||||||
|
: await convertImageLocally(sourceUrl, state.operation, { signal: controller.signal, maxDimension: 1024 });
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
const storageKey = localConversionStorageKey(node.id, currentFingerprint, state.operation);
|
||||||
|
const url = await setImageBlob(storageKey, result.blob);
|
||||||
|
const completedAt = new Date().toISOString();
|
||||||
|
const detectedPeople = state.operation === "pose" && "personCount" in result && typeof result.personCount === "number" ? result.personCount : undefined;
|
||||||
|
updateMetadata?.(node.id, {
|
||||||
|
content: "",
|
||||||
|
previewContent: "",
|
||||||
|
storageKey,
|
||||||
|
mimeType: result.blob.type || "image/png",
|
||||||
|
bytes: result.blob.size,
|
||||||
|
naturalWidth: result.width,
|
||||||
|
naturalHeight: result.height,
|
||||||
|
mediaConversion: {
|
||||||
|
...state,
|
||||||
|
schemaVersion: 1,
|
||||||
|
status: "completed",
|
||||||
|
sourceNodeId: input.id,
|
||||||
|
sourceFingerprint: currentFingerprint,
|
||||||
|
outputKind: "image",
|
||||||
|
resultStorageKey: storageKey,
|
||||||
|
resultWidth: result.width,
|
||||||
|
resultHeight: result.height,
|
||||||
|
...(detectedPeople !== undefined ? { detectedPeople } : {}),
|
||||||
|
errorCode: undefined,
|
||||||
|
errorMessage: undefined,
|
||||||
|
updatedAt: completedAt,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
setResultUrl(url);
|
||||||
|
setShowResult(true);
|
||||||
|
} catch (error) {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
const aborted = error instanceof LocalImageConversionError && error.code === "aborted";
|
||||||
|
if (aborted) return;
|
||||||
|
if (error instanceof LocalRuntimeClientError) {
|
||||||
|
if (error.code === "pose_no_person") {
|
||||||
|
updateState({ status: "skipped", errorCode: error.code, errorMessage: error.message, detectedPeople: 0, updatedAt: new Date().toISOString() });
|
||||||
|
setNotice("未检测到可用人物姿态,可更换含人物的图片或选择其他转换方式");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const nextStatus = ["depth_model_missing", "depth_runtime_unavailable", "lineart_model_missing", "lineart_runtime_unavailable", "pose_model_missing", "pose_runtime_unavailable"].includes(error.code) ? "unavailable" : "error";
|
||||||
|
updateState({ status: nextStatus, errorCode: error.code, errorMessage: error.message, updatedAt: new Date().toISOString() });
|
||||||
|
setNotice(localRuntimeNotice(error, useLocalRuntimeStore.getState().connection));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const conversionError = error instanceof LocalImageConversionError ? error : new LocalImageConversionError("source_unreadable", "本地图片转换失败");
|
||||||
|
const nextStatus = conversionError.code === "model_missing" ? "unavailable" : "error";
|
||||||
|
updateState({ status: nextStatus, errorCode: conversionError.code, errorMessage: conversionError.message, updatedAt: new Date().toISOString() });
|
||||||
|
setNotice(conversionError.message);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const downloadResult = () => {
|
||||||
|
if (!resultUrl) return;
|
||||||
|
const link = document.createElement("a");
|
||||||
|
link.href = resultUrl;
|
||||||
|
link.download = `${node.title || "转换结果"}-${state.operation}.png`;
|
||||||
|
link.click();
|
||||||
|
};
|
||||||
|
|
||||||
|
const effectiveResult = Boolean(resultUrl && (status === "completed" || status === "stale"));
|
||||||
|
const displayResult = showResult && effectiveResult;
|
||||||
|
const buttonDisabled = !input || hasMultipleInputs || status === "processing";
|
||||||
|
const statusText = mediaConversionStatusLabel(status);
|
||||||
|
const previewStatusText = state.operation === "pose" && status === "completed" && typeof state.detectedPeople === "number"
|
||||||
|
? `${statusText} · ${state.detectedPeople} 人`
|
||||||
|
: statusText;
|
||||||
|
const showRuntimeRecovery = (state.operation === "depth" || state.operation === "lineart" || state.operation === "pose") && (localRuntimeConnection === "unreachable" || state.errorCode === "depth_runtime_unavailable" || state.errorCode === "lineart_runtime_unavailable" || state.errorCode === "pose_runtime_unavailable");
|
||||||
|
const retryLocalRuntime = async () => {
|
||||||
|
setNotice("");
|
||||||
|
await reconnectLocalRuntime();
|
||||||
|
const next = useLocalRuntimeStore.getState();
|
||||||
|
if (next.connection === "connected") {
|
||||||
|
setNotice("本地模型服务已连接,可以重新开始转换");
|
||||||
|
} else {
|
||||||
|
setNotice(next.error || "本地模型服务仍未连接,请先运行 start-yingce-local.cmd");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="flex h-full min-h-0 w-full flex-col gap-2.5 overflow-hidden rounded-[inherit] p-3"
|
||||||
|
style={{ background: theme.node.panel, color: theme.node.text }}
|
||||||
|
onPointerDown={(event) => event.stopPropagation()}
|
||||||
|
onWheel={(event) => event.stopPropagation()}
|
||||||
|
>
|
||||||
|
<div className="flex min-w-0 items-center gap-2">
|
||||||
|
<span className="grid size-8 shrink-0 place-items-center rounded-[var(--r-md)]" style={{ background: theme.toolbar.itemHover, color: theme.accent.primary }}>
|
||||||
|
<WandSparkles className="size-4" aria-hidden="true" />
|
||||||
|
</span>
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<div className="truncate text-[var(--fs-label)] font-semibold">{node.title || "转换"}</div>
|
||||||
|
<div className="truncate text-[var(--fs-tiny)]" style={{ color: theme.node.muted }}>本地处理 · 单图优先</div>
|
||||||
|
</div>
|
||||||
|
<StatusBadge status={status} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex min-h-0 flex-1 flex-col gap-2">
|
||||||
|
<div className="relative min-h-0 flex-1 overflow-hidden rounded-[var(--r-md)] border" style={{ borderColor: theme.node.edge, background: theme.node.fill }}>
|
||||||
|
{displayResult ? (
|
||||||
|
<img src={resultUrl} alt={`${mediaConversionOperationLabel(state.operation)}结果`} className="absolute inset-0 size-full object-contain" draggable={false} />
|
||||||
|
) : input?.type === CanvasNodeType.Image && sourceUrl ? (
|
||||||
|
<img src={sourceUrl} alt="转换输入" className="absolute inset-0 size-full object-contain" draggable={false} />
|
||||||
|
) : input?.type === CanvasNodeType.Video && sourceUrl ? (
|
||||||
|
<video src={sourceUrl} className="absolute inset-0 size-full object-contain" controls preload="metadata" />
|
||||||
|
) : (
|
||||||
|
<div className="absolute inset-0 grid place-items-center" style={{ color: theme.node.muted }}>
|
||||||
|
{input?.type === CanvasNodeType.Video ? <Video className="size-8 opacity-35" aria-hidden="true" /> : <ImageIcon className="size-8 opacity-35" aria-hidden="true" />}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex items-end justify-between gap-2 bg-gradient-to-t from-black/75 to-transparent px-2.5 pb-2 pt-8 text-white">
|
||||||
|
<div className="min-w-0">
|
||||||
|
<div className="truncate text-[var(--fs-micro)] font-semibold">{displayResult ? (status === "stale" ? "上次结果 · 待更新" : "转换结果") : input?.title || "等待输入"}</div>
|
||||||
|
<div className="mt-0.5 truncate text-[var(--fs-micro)] opacity-75">{hasMultipleInputs ? "已连接多个输入" : input ? `${input.type === CanvasNodeType.Video ? "视频" : "图片"} · ${previewStatusText}` : "连接图片或视频节点"}</div>
|
||||||
|
</div>
|
||||||
|
{state.resultWidth && state.resultHeight && displayResult ? <span className="shrink-0 text-[var(--fs-micro)] tabular-nums">{state.resultWidth} × {state.resultHeight}</span> : null}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{effectiveResult ? (
|
||||||
|
<div className="flex gap-1" data-canvas-no-zoom>
|
||||||
|
<PreviewToggle theme={theme} pressed={!showResult} label="原素材" onClick={() => setShowResult(false)} />
|
||||||
|
<PreviewToggle theme={theme} pressed={showResult} label="结果" onClick={() => setShowResult(true)} />
|
||||||
|
<button type="button" aria-label="下载转换结果" title="下载转换结果" className="grid h-6 w-8 shrink-0 place-items-center rounded-[var(--r-sm)] outline-none transition hover:bg-black/5 focus-visible:ring-2 dark:hover:bg-white/10" style={{ color: theme.node.muted, outlineColor: theme.accent.primary }} onClick={(event) => { event.stopPropagation(); downloadResult(); }} onMouseDown={(event) => event.stopPropagation()}>
|
||||||
|
<Download className="size-3.5" aria-hidden="true" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center gap-2" data-canvas-no-zoom onWheel={(event) => event.stopPropagation()}>
|
||||||
|
<label className="sr-only" htmlFor={`${node.id}-operation`}>转换方式</label>
|
||||||
|
<select
|
||||||
|
id={`${node.id}-operation`}
|
||||||
|
value={state.operation}
|
||||||
|
disabled={status === "processing"}
|
||||||
|
className="h-8 min-w-0 flex-1 rounded-[var(--r-sm)] border bg-transparent px-2 text-[var(--fs-tiny)] outline-none focus-visible:ring-2"
|
||||||
|
style={{ borderColor: theme.node.edge, color: theme.node.text, outlineColor: theme.accent.primary }}
|
||||||
|
onChange={(event) => selectOperation(event.target.value as MediaConversionOperation)}
|
||||||
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
|
>
|
||||||
|
{OPERATIONS.map((operation) => <option key={operation} value={operation}>{mediaConversionOperationLabel(operation)}</option>)}
|
||||||
|
</select>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
data-canvas-no-zoom
|
||||||
|
className="inline-flex h-8 shrink-0 items-center gap-1.5 rounded-[var(--r-sm)] px-3 text-[var(--fs-tiny)] font-semibold outline-none transition hover:brightness-105 focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
|
style={{ background: theme.accent.primary, color: theme.accent.onPrimary, outlineColor: theme.accent.primary }}
|
||||||
|
disabled={buttonDisabled}
|
||||||
|
onClick={(event) => { event.stopPropagation(); void run(); }}
|
||||||
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
|
>
|
||||||
|
{status === "processing" ? <LoaderCircle className="size-3.5 animate-spin" aria-hidden="true" /> : null}
|
||||||
|
{status === "processing" ? "处理中" : status === "completed" || status === "stale" ? "重新转换" : "开始转换"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex min-h-4 items-center gap-2 text-[var(--fs-micro)]" style={{ color: status === "unavailable" || status === "skipped" ? "var(--status-warning)" : notice || status === "error" ? "var(--status-error)" : theme.node.muted }}>
|
||||||
|
<span className="min-w-0 flex-1 truncate" title={notice || mediaConversionOperationDescription(state.operation)}>{notice || mediaConversionOperationDescription(state.operation)}</span>
|
||||||
|
{showRuntimeRecovery ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="shrink-0 rounded-[var(--r-sm)] px-1.5 py-0.5 font-semibold outline-none transition hover:bg-black/5 focus-visible:ring-2 dark:hover:bg-white/10"
|
||||||
|
style={{ color: theme.accent.primary, outlineColor: theme.accent.primary }}
|
||||||
|
disabled={localRuntimeConnecting}
|
||||||
|
onClick={(event) => { event.stopPropagation(); void retryLocalRuntime(); }}
|
||||||
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
|
>
|
||||||
|
{localRuntimeConnecting ? "连接中" : "重试连接"}
|
||||||
|
</button>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function PreviewToggle({ theme, pressed, label, onClick }: { theme: CanvasTheme; pressed: boolean; label: string; onClick: () => void }) {
|
||||||
|
return <button type="button" aria-pressed={pressed} className="h-6 flex-1 rounded-[var(--r-sm)] px-2 text-[var(--fs-micro)] font-medium outline-none transition focus-visible:ring-2" style={{ background: pressed ? theme.accent.primary : "color-mix(in oklch, var(--foreground-muted) 10%, transparent)", color: pressed ? theme.accent.onPrimary : "var(--foreground-muted)", outlineColor: theme.accent.primary }} onClick={(event) => { event.stopPropagation(); onClick(); }} onMouseDown={(event) => event.stopPropagation()}>{label}</button>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function StatusBadge({ status }: { status: MediaConversionNodeState["status"] | "stale" }) {
|
||||||
|
if (status === "processing") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-info)", background: "color-mix(in oklch, var(--status-info) 15%, transparent)" }}><LoaderCircle className="size-3 animate-spin" aria-hidden="true" />处理中</span>;
|
||||||
|
if (status === "completed") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-success)", background: "color-mix(in oklch, var(--status-success) 15%, transparent)" }}><CheckCircle2 className="size-3" aria-hidden="true" />已完成</span>;
|
||||||
|
if (status === "stale") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}><RefreshCw className="size-3" aria-hidden="true" />待更新</span>;
|
||||||
|
if (status === "error") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-error)", background: "color-mix(in oklch, var(--status-error) 15%, transparent)" }}><CircleAlert className="size-3" aria-hidden="true" />失败</span>;
|
||||||
|
if (status === "unavailable") return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}>本地不可用</span>;
|
||||||
|
if (status === "skipped") return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}>已跳过</span>;
|
||||||
|
return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--foreground-muted)", background: "color-mix(in oklch, var(--foreground-muted) 12%, transparent)" }}>未开始</span>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mediaConversionStatusLabel(status: MediaConversionNodeState["status"] | "stale") {
|
||||||
|
if (status === "processing") return "正在处理";
|
||||||
|
if (status === "completed") return "已完成";
|
||||||
|
if (status === "stale") return "输入已变化,结果待更新";
|
||||||
|
if (status === "unavailable") return "本地能力不可用";
|
||||||
|
if (status === "skipped") return "未检测到人物姿态";
|
||||||
|
if (status === "error") return "处理失败";
|
||||||
|
return "尚未转换";
|
||||||
|
}
|
||||||
|
|
||||||
|
function localRuntimeNotice(error: LocalRuntimeClientError, connection: ReturnType<typeof useLocalRuntimeStore.getState>["connection"]) {
|
||||||
|
if (connection === "unreachable") return "本地模型服务未启动,请双击 start-yingce-local.cmd 后重试";
|
||||||
|
if (error.code === "depth_model_missing") return "深度模型尚未安装,请先完成本地模型安装";
|
||||||
|
if (error.code === "lineart_model_missing") return "AI 线稿依赖或模型尚未安装,请先完成本地线稿模型安装";
|
||||||
|
if (error.code === "pose_model_missing") return "姿态模型或依赖尚未安装,请先完成本地姿态模型安装";
|
||||||
|
return error.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
function localConversionStorageKey(nodeId: string, fingerprint: string, operation: MediaConversionOperation) {
|
||||||
|
return `image:${getActiveUserScope()}:media-conversion:${nodeId}:${fingerprint}:${operation}`;
|
||||||
|
}
|
||||||
@@ -26,6 +26,7 @@ export const NODE_DEFAULT_SIZE = {
|
|||||||
[CanvasNodeType.Compare]: { width: 520, height: 320, title: "对比" },
|
[CanvasNodeType.Compare]: { width: 520, height: 320, title: "对比" },
|
||||||
[CanvasNodeType.Chart]: { width: 480, height: 320, title: "图表" },
|
[CanvasNodeType.Chart]: { width: 480, height: 320, title: "图表" },
|
||||||
[CanvasNodeType.ColorGrade]: { width: 420, height: 360, title: "调色" },
|
[CanvasNodeType.ColorGrade]: { width: 420, height: 360, title: "调色" },
|
||||||
|
[CanvasNodeType.MediaConversion]: { width: 380, height: 440, title: "转换" },
|
||||||
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
|
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
|
||||||
|
|
||||||
export const NODE_SPECS = {
|
export const NODE_SPECS = {
|
||||||
@@ -101,6 +102,10 @@ export const NODE_SPECS = {
|
|||||||
...NODE_DEFAULT_SIZE[CanvasNodeType.ColorGrade],
|
...NODE_DEFAULT_SIZE[CanvasNodeType.ColorGrade],
|
||||||
metadata: { status: "idle" },
|
metadata: { status: "idle" },
|
||||||
},
|
},
|
||||||
|
[CanvasNodeType.MediaConversion]: {
|
||||||
|
...NODE_DEFAULT_SIZE[CanvasNodeType.MediaConversion],
|
||||||
|
metadata: { status: "idle" },
|
||||||
|
},
|
||||||
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
|
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
|
||||||
|
|
||||||
export function getNodeSpec(type: CanvasNodeType) {
|
export function getNodeSpec(type: CanvasNodeType) {
|
||||||
|
|||||||
@@ -399,6 +399,7 @@ function canvasNodeTypeLabel(type?: CanvasNodeType) {
|
|||||||
if (type === CanvasNodeType.Frame) return "背板";
|
if (type === CanvasNodeType.Frame) return "背板";
|
||||||
if (type === CanvasNodeType.Drawing) return "绘图节点";
|
if (type === CanvasNodeType.Drawing) return "绘图节点";
|
||||||
if (type === CanvasNodeType.Skill) return "技能节点";
|
if (type === CanvasNodeType.Skill) return "技能节点";
|
||||||
|
if (type === CanvasNodeType.MediaConversion) return "转换节点";
|
||||||
return "文本节点";
|
return "文本节点";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection";
|
import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection";
|
||||||
import { getNodeAcceptedInputKind, getNodeGenerationMode, getNodeInputKind } from "@/lib/canvas/node-registry";
|
import { getNodeAcceptedInputKinds, getNodeGenerationMode, getNodeInputKind, getNodeMaxInputCount } from "@/lib/canvas/node-registry";
|
||||||
import type { AiConfig } from "@/stores/use-config-store";
|
import type { AiConfig } from "@/stores/use-config-store";
|
||||||
import { type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
|
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
|
||||||
|
|
||||||
type ConnectionCandidate = Pick<CanvasConnection, "fromNodeId" | "toNodeId">;
|
type ConnectionCandidate = Pick<CanvasConnection, "fromNodeId" | "toNodeId">;
|
||||||
type CanvasConnectionPolicyOptions = {
|
type CanvasConnectionPolicyOptions = {
|
||||||
@@ -12,11 +12,27 @@ type CanvasConnectionPolicyOptions = {
|
|||||||
export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) {
|
export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) {
|
||||||
const target = nodes.find((node) => node.id === candidate.toNodeId);
|
const target = nodes.find((node) => node.id === candidate.toNodeId);
|
||||||
if (!target) return "找不到连线目标节点";
|
if (!target) return "找不到连线目标节点";
|
||||||
const acceptedInputKind = getNodeAcceptedInputKind(target.type);
|
const acceptedInputKinds = getNodeAcceptedInputKinds(target.type);
|
||||||
if (acceptedInputKind) {
|
if (acceptedInputKinds.length) {
|
||||||
const source = nodes.find((node) => node.id === candidate.fromNodeId);
|
const source = nodes.find((node) => node.id === candidate.fromNodeId);
|
||||||
const sourceKind = source ? getNodeInputKind(source.type) : undefined;
|
const sourceKind = source ? getNodeInputKind(source.type) : undefined;
|
||||||
if (sourceKind !== acceptedInputKind) return `${acceptedInputKindLabel(acceptedInputKind)}节点只接受${acceptedInputKindLabel(acceptedInputKind)}输入`;
|
const isMediaConversion = target.type === CanvasNodeType.MediaConversion;
|
||||||
|
const hasAcceptedSource = isMediaConversion
|
||||||
|
? source?.type === CanvasNodeType.Image || source?.type === CanvasNodeType.Video
|
||||||
|
: Boolean(sourceKind && acceptedInputKinds.includes(sourceKind));
|
||||||
|
if (!sourceKind || !hasAcceptedSource) {
|
||||||
|
const labels = acceptedInputKinds.map(acceptedInputKindLabel).join("或");
|
||||||
|
return `${isMediaConversion ? "转换" : labels}节点只接受${labels}输入`;
|
||||||
|
}
|
||||||
|
const maxInputCount = getNodeMaxInputCount(target.type);
|
||||||
|
if (maxInputCount) {
|
||||||
|
const inputCount = new Set(
|
||||||
|
[...connections, { id: "candidate", ...candidate }]
|
||||||
|
.filter((connection) => connection.toNodeId === target.id)
|
||||||
|
.map((connection) => connection.fromNodeId),
|
||||||
|
).size;
|
||||||
|
if (inputCount > maxInputCount) return `${isMediaConversion ? "转换" : "当前"}节点最多连接 ${maxInputCount} 个输入`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const mode = getNodeGenerationMode(target);
|
const mode = getNodeGenerationMode(target);
|
||||||
if (!mode) return "";
|
if (!mode) return "";
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video } from "lucide-react";
|
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video, WandSparkles } from "lucide-react";
|
||||||
|
|
||||||
import { NODE_SPECS } from "@/constant/canvas";
|
import { NODE_SPECS } from "@/constant/canvas";
|
||||||
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
|
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
|
||||||
@@ -25,7 +25,7 @@ const BUILTIN_NODE_TRAITS = {
|
|||||||
minSize: MEDIA_NODE_MIN_SIZE,
|
minSize: MEDIA_NODE_MIN_SIZE,
|
||||||
keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize,
|
keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize,
|
||||||
showInCreateMenu: true,
|
showInCreateMenu: true,
|
||||||
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "image" : null),
|
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "image" : null),
|
||||||
generationMode: () => "image",
|
generationMode: () => "image",
|
||||||
inputKind: "image",
|
inputKind: "image",
|
||||||
},
|
},
|
||||||
@@ -82,7 +82,7 @@ const BUILTIN_NODE_TRAITS = {
|
|||||||
minSize: MEDIA_NODE_MIN_SIZE,
|
minSize: MEDIA_NODE_MIN_SIZE,
|
||||||
keepAspectRatio: () => true,
|
keepAspectRatio: () => true,
|
||||||
showInCreateMenu: true,
|
showInCreateMenu: true,
|
||||||
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "video" : null),
|
resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "video" : null),
|
||||||
generationMode: () => "video",
|
generationMode: () => "video",
|
||||||
inputKind: "video",
|
inputKind: "video",
|
||||||
},
|
},
|
||||||
@@ -165,6 +165,20 @@ const BUILTIN_NODE_TRAITS = {
|
|||||||
resourceKind: () => "image",
|
resourceKind: () => "image",
|
||||||
inputKind: "image",
|
inputKind: "image",
|
||||||
},
|
},
|
||||||
|
[CanvasNodeType.MediaConversion]: {
|
||||||
|
label: "转换",
|
||||||
|
icon: <WandSparkles />,
|
||||||
|
minSize: { width: 320, height: 360 },
|
||||||
|
showInCreateMenu: true,
|
||||||
|
resourceKind: (node: CanvasNodeData) => {
|
||||||
|
const conversion = node.metadata?.mediaConversion;
|
||||||
|
if (conversion?.status !== "completed" || !conversion.resultStorageKey) return null;
|
||||||
|
return conversion.outputKind === "video" ? "video" : "image";
|
||||||
|
},
|
||||||
|
acceptsInputKind: ["image", "video"],
|
||||||
|
maxInputCount: 1,
|
||||||
|
inputKind: "image",
|
||||||
|
},
|
||||||
} satisfies Record<string, Omit<CanvasNodeDefinition, "type" | "defaultTitle" | "defaultSize" | "defaultMetadata">>;
|
} satisfies Record<string, Omit<CanvasNodeDefinition, "type" | "defaultTitle" | "defaultSize" | "defaultMetadata">>;
|
||||||
|
|
||||||
export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => {
|
export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => {
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition";
|
export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition";
|
||||||
export { getNodeAcceptedInputKind, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
|
export { getNodeAcceptedInputKind, getNodeAcceptedInputKinds, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMaxInputCount, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
|
||||||
import "./definitions";
|
import "./definitions";
|
||||||
@@ -43,8 +43,10 @@ export type CanvasNodeDefinition = {
|
|||||||
generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null;
|
generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null;
|
||||||
/** 是否显示右侧输出连接点;缺省为 true,消费型终点节点可关闭。 */
|
/** 是否显示右侧输出连接点;缺省为 true,消费型终点节点可关闭。 */
|
||||||
showOutputConnection?: boolean;
|
showOutputConnection?: boolean;
|
||||||
/** 若设置,作为目标时只接受这一类上游节点。 */
|
/** 若设置,作为目标时只接受这些上游节点类型。 */
|
||||||
acceptsInputKind?: CanvasNodeInputKind;
|
acceptsInputKind?: CanvasNodeInputKind | CanvasNodeInputKind[];
|
||||||
|
/** 限制直接输入数量;转换节点等一进一出节点通常为 1。 */
|
||||||
|
maxInputCount?: number;
|
||||||
/**
|
/**
|
||||||
* 作为上游输入被参考素材容量校验计数时的类别;
|
* 作为上游输入被参考素材容量校验计数时的类别;
|
||||||
* 不设表示不参与计数(生成配置、背板)。与 resourceKind 不同,计数不看内容。
|
* 不设表示不参与计数(生成配置、背板)。与 resourceKind 不同,计数不看内容。
|
||||||
@@ -69,6 +71,7 @@ export function canvasNodeDefinitionFromPlugin(pluginId: string, contribution: P
|
|||||||
showInCreateMenu: true,
|
showInCreateMenu: true,
|
||||||
showOutputConnection: contribution.showOutputConnection,
|
showOutputConnection: contribution.showOutputConnection,
|
||||||
acceptsInputKind: contribution.acceptsInputKind,
|
acceptsInputKind: contribution.acceptsInputKind,
|
||||||
|
maxInputCount: contribution.maxInputCount,
|
||||||
plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema },
|
plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema },
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -100,3 +100,13 @@ export function getNodeInputKind(type: CanvasNodeTypeId) {
|
|||||||
export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) {
|
export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) {
|
||||||
return definitions.get(type)?.acceptsInputKind;
|
return definitions.get(type)?.acceptsInputKind;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getNodeAcceptedInputKinds(type: CanvasNodeTypeId) {
|
||||||
|
const accepted = definitions.get(type)?.acceptsInputKind;
|
||||||
|
if (!accepted) return [];
|
||||||
|
return Array.isArray(accepted) ? accepted : [accepted];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getNodeMaxInputCount(type: CanvasNodeTypeId) {
|
||||||
|
return definitions.get(type)?.maxInputCount;
|
||||||
|
}
|
||||||
@@ -20,6 +20,7 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
|
|||||||
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
|
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
|
||||||
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
|
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
|
||||||
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
|
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
|
||||||
|
nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }),
|
||||||
// 导演台落在节点分区,但它开的是导演工作台、不是某种画布节点,故不走注册表。
|
// 导演台落在节点分区,但它开的是导演工作台、不是某种画布节点,故不走注册表。
|
||||||
{ id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() },
|
{ id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() },
|
||||||
nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }),
|
nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }),
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react";
|
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video, WandSparkles } from "lucide-react";
|
||||||
|
|
||||||
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
|
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
|
||||||
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
|
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
|
||||||
@@ -139,6 +139,19 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
|
|||||||
applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx),
|
applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx),
|
||||||
run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!),
|
run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: "createConversion",
|
||||||
|
toolbar: "node-hover",
|
||||||
|
category: "node-state",
|
||||||
|
label: "创建转换节点",
|
||||||
|
displayLabel: "转换",
|
||||||
|
icon: <WandSparkles className="size-3.5" />,
|
||||||
|
defaultVisible: true,
|
||||||
|
defaultOrder: 65,
|
||||||
|
nodeToolbar: { group: "primary", order: 15, description: "在右侧创建转换节点并自动连接" },
|
||||||
|
applicable: (ctx) => (hasImage(ctx) || hasVideo(ctx)) && Boolean(ctx.handlers.onNodeCreateConversion),
|
||||||
|
run: (ctx) => ctx.handlers.onNodeCreateConversion?.(ctx.node!),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: "edit",
|
id: "edit",
|
||||||
toolbar: "node-hover",
|
toolbar: "node-hover",
|
||||||
|
|||||||
@@ -81,6 +81,8 @@ export type ToolbarHandlers = {
|
|||||||
onNodeUpload: (node: CanvasNodeData) => void;
|
onNodeUpload: (node: CanvasNodeData) => void;
|
||||||
onNodeDownload: (node: CanvasNodeData) => void;
|
onNodeDownload: (node: CanvasNodeData) => void;
|
||||||
onNodeSaveAsset: (node: CanvasNodeData) => void;
|
onNodeSaveAsset: (node: CanvasNodeData) => void;
|
||||||
|
/** 从已有图片或视频创建并自动连接一个转换节点。 */
|
||||||
|
onNodeCreateConversion?: (node: CanvasNodeData) => void;
|
||||||
onNodeMaskEdit: (node: CanvasNodeData) => void;
|
onNodeMaskEdit: (node: CanvasNodeData) => void;
|
||||||
onNodeEmotion: (node: CanvasNodeData) => void;
|
onNodeEmotion: (node: CanvasNodeData) => void;
|
||||||
onNodePortraitTexture: (node: CanvasNodeData) => void;
|
onNodePortraitTexture: (node: CanvasNodeData) => void;
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
import type { CanvasNodeData } from "@/types/canvas";
|
||||||
|
|
||||||
|
export const MEDIA_CONVERSION_SCHEMA_VERSION = 1 as const;
|
||||||
|
|
||||||
|
export const MEDIA_CONVERSION_NODE_TYPE = "media-conversion" as const;
|
||||||
|
|
||||||
|
export type MediaConversionOperation = "grayscale" | "edge-canny" | "lineart" | "depth" | "pose" | "cutout";
|
||||||
|
export type MediaConversionStatus = "idle" | "processing" | "completed" | "stale" | "skipped" | "unavailable" | "error";
|
||||||
|
export type MediaConversionOutputKind = "image" | "video";
|
||||||
|
|
||||||
|
export type MediaConversionNodeState = {
|
||||||
|
schemaVersion: typeof MEDIA_CONVERSION_SCHEMA_VERSION;
|
||||||
|
operation: MediaConversionOperation;
|
||||||
|
status: MediaConversionStatus;
|
||||||
|
sourceNodeId?: string;
|
||||||
|
sourceFingerprint?: string;
|
||||||
|
outputKind?: MediaConversionOutputKind;
|
||||||
|
resultStorageKey?: string;
|
||||||
|
resultWidth?: number;
|
||||||
|
resultHeight?: number;
|
||||||
|
detectedPeople?: number;
|
||||||
|
errorCode?: string;
|
||||||
|
errorMessage?: string;
|
||||||
|
startedAt?: string;
|
||||||
|
updatedAt?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const MEDIA_CONVERSION_OPERATION_LABELS: Record<MediaConversionOperation, string> = {
|
||||||
|
grayscale: "灰度图",
|
||||||
|
"edge-canny": "Canny 边缘",
|
||||||
|
lineart: "AI 线稿",
|
||||||
|
depth: "深度图",
|
||||||
|
pose: "姿态骨架",
|
||||||
|
cutout: "透明抠图",
|
||||||
|
};
|
||||||
|
|
||||||
|
export const MEDIA_CONVERSION_OPERATION_DESCRIPTIONS: Record<MediaConversionOperation, string> = {
|
||||||
|
grayscale: "普通图像算法,适合先检查明暗层次",
|
||||||
|
"edge-canny": "普通图像算法,提取清晰轮廓",
|
||||||
|
lineart: "本地 ControlNet Aux 线稿预处理,首次运行需要加载模型",
|
||||||
|
depth: "本地 Depth Anything V2 Small,首次运行需要加载模型",
|
||||||
|
pose: "本地 OpenPose 人体骨架预处理,未检测到人物时会跳过",
|
||||||
|
cutout: "需要安装并验证本地抠图模型",
|
||||||
|
};
|
||||||
|
|
||||||
|
export function createDefaultMediaConversionState(): MediaConversionNodeState {
|
||||||
|
return {
|
||||||
|
schemaVersion: MEDIA_CONVERSION_SCHEMA_VERSION,
|
||||||
|
operation: "edge-canny",
|
||||||
|
status: "idle",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mediaConversionSourceFingerprint(node: CanvasNodeData) {
|
||||||
|
const metadata = node.metadata;
|
||||||
|
const source = [
|
||||||
|
node.id,
|
||||||
|
metadata?.storageKey || "",
|
||||||
|
compactSource(metadata?.content || metadata?.previewContent || ""),
|
||||||
|
metadata?.mimeType || "",
|
||||||
|
metadata?.bytes || "",
|
||||||
|
metadata?.naturalWidth || "",
|
||||||
|
metadata?.naturalHeight || "",
|
||||||
|
metadata?.durationMs || "",
|
||||||
|
].join("|");
|
||||||
|
let hash = 2166136261;
|
||||||
|
for (let index = 0; index < source.length; index += 1) {
|
||||||
|
hash ^= source.charCodeAt(index);
|
||||||
|
hash = Math.imul(hash, 16777619);
|
||||||
|
}
|
||||||
|
return `v1-${(hash >>> 0).toString(16).padStart(8, "0")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mediaConversionOperationLabel(operation: MediaConversionOperation) {
|
||||||
|
return MEDIA_CONVERSION_OPERATION_LABELS[operation];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mediaConversionOperationDescription(operation: MediaConversionOperation) {
|
||||||
|
return MEDIA_CONVERSION_OPERATION_DESCRIPTIONS[operation];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isLocalImageOperation(operation: MediaConversionOperation) {
|
||||||
|
return operation === "grayscale" || operation === "edge-canny";
|
||||||
|
}
|
||||||
|
|
||||||
|
function compactSource(value: string) {
|
||||||
|
if (value.length <= 2048) return value;
|
||||||
|
return `${value.slice(0, 1024)}|${value.slice(-1024)}|${value.length}`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
import type { MediaConversionOperation } from "./contracts";
|
||||||
|
|
||||||
|
export type LocalImageConversionResult = {
|
||||||
|
blob: Blob;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class LocalImageConversionError extends Error {
|
||||||
|
constructor(readonly code: "model_missing" | "source_unreadable" | "browser_unsupported" | "aborted", message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "LocalImageConversionError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function convertImageLocally(sourceUrl: string, operation: MediaConversionOperation, options: { signal?: AbortSignal; maxDimension?: number } = {}): Promise<LocalImageConversionResult> {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
if (operation !== "grayscale" && operation !== "edge-canny") {
|
||||||
|
throw new LocalImageConversionError("model_missing", "该转换需要先安装并验证本地模型");
|
||||||
|
}
|
||||||
|
|
||||||
|
const image = await loadImage(sourceUrl, options.signal);
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const maxDimension = Math.max(256, options.maxDimension || 1024);
|
||||||
|
const scale = Math.min(1, maxDimension / Math.max(image.naturalWidth, image.naturalHeight));
|
||||||
|
const width = Math.max(1, Math.round(image.naturalWidth * scale));
|
||||||
|
const height = Math.max(1, Math.round(image.naturalHeight * scale));
|
||||||
|
const canvas = document.createElement("canvas");
|
||||||
|
canvas.width = width;
|
||||||
|
canvas.height = height;
|
||||||
|
const context = canvas.getContext("2d", { willReadFrequently: true });
|
||||||
|
if (!context) throw new LocalImageConversionError("browser_unsupported", "当前浏览器无法创建图片处理画布");
|
||||||
|
context.drawImage(image, 0, 0, width, height);
|
||||||
|
const input = context.getImageData(0, 0, width, height);
|
||||||
|
const output = transformPixels(input.data, width, height, operation, options.signal);
|
||||||
|
const outputImage = new ImageData(output, width, height);
|
||||||
|
context.putImageData(outputImage, 0, 0);
|
||||||
|
const blob = await canvasToBlob(canvas, options.signal);
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
return { blob, width, height };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pure pixel transform used by the browser executor and unit tests. */
|
||||||
|
export function transformPixels(source: Uint8ClampedArray, width: number, height: number, operation: "grayscale" | "edge-canny", signal?: AbortSignal) {
|
||||||
|
if (source.length !== width * height * 4) throw new Error("图片像素尺寸不匹配");
|
||||||
|
if (operation === "grayscale") return grayscalePixels(source, width, height, signal);
|
||||||
|
return cannyPixels(source, width, height, signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
function grayscalePixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
|
||||||
|
const output = new Uint8ClampedArray(source);
|
||||||
|
for (let y = 0; y < height; y += 1) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
for (let x = 0; x < width; x += 1) {
|
||||||
|
const index = (y * width + x) * 4;
|
||||||
|
const value = Math.round(source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114);
|
||||||
|
output[index] = value;
|
||||||
|
output[index + 1] = value;
|
||||||
|
output[index + 2] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cannyPixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
|
||||||
|
const gray = new Float32Array(width * height);
|
||||||
|
for (let y = 0; y < height; y += 1) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
for (let x = 0; x < width; x += 1) {
|
||||||
|
const index = (y * width + x) * 4;
|
||||||
|
gray[y * width + x] = source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const blurred = gaussianBlur(gray, width, height, signal);
|
||||||
|
const magnitude = new Float32Array(width * height);
|
||||||
|
const direction = new Float32Array(width * height);
|
||||||
|
let maximum = 0;
|
||||||
|
for (let y = 1; y < height - 1; y += 1) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
for (let x = 1; x < width - 1; x += 1) {
|
||||||
|
const index = y * width + x;
|
||||||
|
const horizontal = blurred[index + 1] - blurred[index - 1];
|
||||||
|
const vertical = blurred[index + width] - blurred[index - width];
|
||||||
|
const value = Math.hypot(horizontal, vertical);
|
||||||
|
magnitude[index] = value;
|
||||||
|
direction[index] = Math.atan2(vertical, horizontal);
|
||||||
|
maximum = Math.max(maximum, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const thinned = nonMaximumSuppression(magnitude, direction, width, height, signal);
|
||||||
|
const high = maximum * 0.28;
|
||||||
|
const low = high * 0.45;
|
||||||
|
const edges = new Uint8Array(width * height);
|
||||||
|
const stack: number[] = [];
|
||||||
|
for (let index = 0; index < thinned.length; index += 1) {
|
||||||
|
if (thinned[index] >= high && high > 0) {
|
||||||
|
edges[index] = 2;
|
||||||
|
stack.push(index);
|
||||||
|
} else if (thinned[index] >= low && low > 0) {
|
||||||
|
edges[index] = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
while (stack.length) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
const index = stack.pop()!;
|
||||||
|
const x = index % width;
|
||||||
|
const y = Math.floor(index / width);
|
||||||
|
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
|
||||||
|
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
|
||||||
|
if (!offsetX && !offsetY) continue;
|
||||||
|
const nextX = x + offsetX;
|
||||||
|
const nextY = y + offsetY;
|
||||||
|
if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) continue;
|
||||||
|
const next = nextY * width + nextX;
|
||||||
|
if (edges[next] !== 1) continue;
|
||||||
|
edges[next] = 2;
|
||||||
|
stack.push(next);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const output = new Uint8ClampedArray(source.length);
|
||||||
|
for (let y = 0; y < height; y += 1) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
for (let x = 0; x < width; x += 1) {
|
||||||
|
const index = y * width + x;
|
||||||
|
const outputIndex = index * 4;
|
||||||
|
const value = edges[index] === 2 ? 0 : 255;
|
||||||
|
output[outputIndex] = value;
|
||||||
|
output[outputIndex + 1] = value;
|
||||||
|
output[outputIndex + 2] = value;
|
||||||
|
output[outputIndex + 3] = source[outputIndex + 3];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function gaussianBlur(source: Float32Array, width: number, height: number, signal?: AbortSignal) {
|
||||||
|
const output = new Float32Array(source.length);
|
||||||
|
const kernel = [1, 2, 1, 2, 4, 2, 1, 2, 1];
|
||||||
|
for (let y = 0; y < height; y += 1) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
for (let x = 0; x < width; x += 1) {
|
||||||
|
let value = 0;
|
||||||
|
let weight = 0;
|
||||||
|
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
|
||||||
|
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
|
||||||
|
const nextX = Math.min(width - 1, Math.max(0, x + offsetX));
|
||||||
|
const nextY = Math.min(height - 1, Math.max(0, y + offsetY));
|
||||||
|
const kernelWeight = kernel[(offsetY + 1) * 3 + offsetX + 1];
|
||||||
|
value += source[nextY * width + nextX] * kernelWeight;
|
||||||
|
weight += kernelWeight;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output[y * width + x] = value / weight;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nonMaximumSuppression(magnitude: Float32Array, direction: Float32Array, width: number, height: number, signal?: AbortSignal) {
|
||||||
|
const output = new Float32Array(magnitude.length);
|
||||||
|
for (let y = 1; y < height - 1; y += 1) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
for (let x = 1; x < width - 1; x += 1) {
|
||||||
|
const index = y * width + x;
|
||||||
|
let angle = (direction[index] * 180) / Math.PI;
|
||||||
|
if (angle < 0) angle += 180;
|
||||||
|
const value = magnitude[index];
|
||||||
|
let before = 0;
|
||||||
|
let after = 0;
|
||||||
|
if (angle < 22.5 || angle >= 157.5) {
|
||||||
|
before = magnitude[index - 1];
|
||||||
|
after = magnitude[index + 1];
|
||||||
|
} else if (angle < 67.5) {
|
||||||
|
before = magnitude[index - width + 1];
|
||||||
|
after = magnitude[index + width - 1];
|
||||||
|
} else if (angle < 112.5) {
|
||||||
|
before = magnitude[index - width];
|
||||||
|
after = magnitude[index + width];
|
||||||
|
} else {
|
||||||
|
before = magnitude[index - width - 1];
|
||||||
|
after = magnitude[index + width + 1];
|
||||||
|
}
|
||||||
|
output[index] = value >= before && value >= after ? value : 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadImage(sourceUrl: string, signal?: AbortSignal) {
|
||||||
|
if (!sourceUrl) throw new LocalImageConversionError("source_unreadable", "没有可读取的图片输入");
|
||||||
|
const image = new Image();
|
||||||
|
image.crossOrigin = "anonymous";
|
||||||
|
return await new Promise<HTMLImageElement>((resolve, reject) => {
|
||||||
|
const abort = () => {
|
||||||
|
image.src = "";
|
||||||
|
reject(new LocalImageConversionError("aborted", "转换已取消"));
|
||||||
|
};
|
||||||
|
if (signal?.aborted) return abort();
|
||||||
|
signal?.addEventListener("abort", abort, { once: true });
|
||||||
|
image.onload = () => {
|
||||||
|
signal?.removeEventListener("abort", abort);
|
||||||
|
resolve(image);
|
||||||
|
};
|
||||||
|
image.onerror = () => {
|
||||||
|
signal?.removeEventListener("abort", abort);
|
||||||
|
reject(new LocalImageConversionError("source_unreadable", "图片无法读取,可能是跨域资源或已失效"));
|
||||||
|
};
|
||||||
|
image.src = sourceUrl;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function canvasToBlob(canvas: HTMLCanvasElement, signal?: AbortSignal) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
return new Promise<Blob>((resolve, reject) => {
|
||||||
|
canvas.toBlob((blob) => {
|
||||||
|
if (blob) resolve(blob);
|
||||||
|
else reject(new LocalImageConversionError("browser_unsupported", "当前浏览器无法导出图片结果"));
|
||||||
|
}, "image/png");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfAborted(signal?: AbortSignal): asserts signal is AbortSignal | undefined {
|
||||||
|
if (signal?.aborted) throw new LocalImageConversionError("aborted", "转换已取消");
|
||||||
|
}
|
||||||
@@ -3,4 +3,5 @@ import "./prompt-optimizer";
|
|||||||
import "./workflows";
|
import "./workflows";
|
||||||
import "./portrait-clearance";
|
import "./portrait-clearance";
|
||||||
import "./ai-art-critique";
|
import "./ai-art-critique";
|
||||||
|
import "./media-conversion";
|
||||||
import "./editor/editor-shell";
|
import "./editor/editor-shell";
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
|
||||||
|
|
||||||
|
import { registerPlugin } from "../plugin-registry";
|
||||||
|
import { PLUGIN_API_VERSION, type PluginManifest, type RegisteredPlugin } from "../plugin-types";
|
||||||
|
|
||||||
|
export const MEDIA_CONVERSION_PLUGIN_ID = "media-conversion";
|
||||||
|
|
||||||
|
const manifest: PluginManifest = {
|
||||||
|
apiVersion: PLUGIN_API_VERSION,
|
||||||
|
id: MEDIA_CONVERSION_PLUGIN_ID,
|
||||||
|
name: "媒体转换节点",
|
||||||
|
version: "0.1.0",
|
||||||
|
publishedAt: "2026-09-08",
|
||||||
|
updatedAt: "2026-09-08",
|
||||||
|
description: "在画布中提供图片转换节点,支持灰度、Canny 边缘、AI 线稿、本地 Depth Anything V2 深度图和 OpenPose 姿态骨架。",
|
||||||
|
documentation:
|
||||||
|
"# 媒体转换节点\n\n连接图片到转换节点后,可以在节点内选择灰度图、Canny 边缘、AI 线稿、深度图或姿态骨架。AI 线稿使用本机 ControlNet Aux 预处理模型,深度图使用本机已安装的 Depth Anything V2 Small 模型,姿态骨架使用 ControlNet Aux 的 OpenPose 人体预处理模型运行;这些操作都不会加载 Stable Diffusion 重绘管线。姿态转换只绘制人体骨架,未检测到人物时会跳过并提示。视频转换和透明抠图模型仍在验证,暂时会明确提示不可用。\n\n该插件只读取当前节点的媒体输入,并把结果保存回当前画布节点的本地素材存储。",
|
||||||
|
author: "影策团队",
|
||||||
|
surfaces: ["node"],
|
||||||
|
permissions: ["canvas.read", "canvas.write", "media.read"],
|
||||||
|
trusted: true,
|
||||||
|
runtime: { web: "declarative" },
|
||||||
|
contributes: {
|
||||||
|
transforms: [
|
||||||
|
{
|
||||||
|
id: MEDIA_CONVERSION_NODE_TYPE,
|
||||||
|
input: "media",
|
||||||
|
output: "media",
|
||||||
|
runtime: "declarative",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export const mediaConversionPlugin: RegisteredPlugin = { manifest };
|
||||||
|
|
||||||
|
registerPlugin(mediaConversionPlugin);
|
||||||
@@ -103,8 +103,10 @@ export type PluginCanvasNodeContribution = {
|
|||||||
defaultSize: { width: number; height: number };
|
defaultSize: { width: number; height: number };
|
||||||
schema: Record<string, unknown>;
|
schema: Record<string, unknown>;
|
||||||
renderer: "declarative" | "sandbox";
|
renderer: "declarative" | "sandbox";
|
||||||
/** Optional input contract for nodes that consume one media kind. */
|
/** Optional input contract for nodes that consume one or more media kinds. */
|
||||||
acceptsInputKind?: "image" | "video" | "audio" | "text";
|
acceptsInputKind?: "image" | "video" | "audio" | "text" | Array<"image" | "video" | "audio" | "text">;
|
||||||
|
/** Optional maximum number of direct inputs. */
|
||||||
|
maxInputCount?: number;
|
||||||
/** Analysis/sink nodes can hide the right-side output connection. */
|
/** Analysis/sink nodes can hide the right-side output connection. */
|
||||||
showOutputConnection?: boolean;
|
showOutputConnection?: boolean;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import "@/lib/plugins/builtin";
|
|||||||
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
|
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
|
||||||
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
|
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
|
||||||
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
|
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
|
||||||
|
import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion";
|
||||||
import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry";
|
import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry";
|
||||||
import type { PluginManifest, PluginManifestV2 } from "@/lib/plugins/plugin-types";
|
import type { PluginManifest, PluginManifestV2 } from "@/lib/plugins/plugin-types";
|
||||||
import { fetchAdminPlugins, setPluginPlatformAvailability, uninstallPlugin, uploadPlugin, type AdminPluginState, type BackendPlugin, type PluginManagement } from "@/services/api/plugins";
|
import { fetchAdminPlugins, setPluginPlatformAvailability, uninstallPlugin, uploadPlugin, type AdminPluginState, type BackendPlugin, type PluginManagement } from "@/services/api/plugins";
|
||||||
@@ -26,7 +27,7 @@ type AdminPluginItem = {
|
|||||||
error?: string;
|
error?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance"]);
|
const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", MEDIA_CONVERSION_PLUGIN_ID]);
|
||||||
|
|
||||||
export default function AdminPluginsPage() {
|
export default function AdminPluginsPage() {
|
||||||
const { message, modal } = App.useApp();
|
const { message, modal } = App.useApp();
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ import { CanvasLocalAgentPanel } from "@/components/canvas/canvas-local-agent-pa
|
|||||||
import { useFocusMode } from "@/hooks/use-focus-mode";
|
import { useFocusMode } from "@/hooks/use-focus-mode";
|
||||||
import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store";
|
import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store";
|
||||||
import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
|
import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
|
||||||
import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays";
|
import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay, type PendingConnectionCreate } from "@/components/canvas/canvas-workspace-overlays";
|
||||||
import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer";
|
import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer";
|
||||||
import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer";
|
import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer";
|
||||||
import { CanvasFreeformEmptyState, CanvasLinkedProjectEmptyState, CanvasShortDramaEmptyState, CanvasShortDramaGuide, CanvasStoryInputNodeContent, CanvasStylePlaceholderNodeContent } from "@/components/canvas/canvas-short-drama-entry";
|
import { CanvasFreeformEmptyState, CanvasLinkedProjectEmptyState, CanvasShortDramaEmptyState, CanvasShortDramaGuide, CanvasStoryInputNodeContent, CanvasStylePlaceholderNodeContent } from "@/components/canvas/canvas-short-drama-entry";
|
||||||
@@ -1018,6 +1018,20 @@ function InfiniteCanvasPage() {
|
|||||||
.filter((node) => selectedNodeIds.has(node.id) && !batchSourceRestriction(node))
|
.filter((node) => selectedNodeIds.has(node.id) && !batchSourceRestriction(node))
|
||||||
.map((node) => node.id), [nodes, selectedNodeIds]);
|
.map((node) => node.id), [nodes, selectedNodeIds]);
|
||||||
|
|
||||||
|
const createConversionFromSource = useCallback((source: CanvasNodeData) => {
|
||||||
|
if (source.type !== CanvasNodeType.Image && source.type !== CanvasNodeType.Video) return;
|
||||||
|
const spec = getNodeSpec(CanvasNodeType.MediaConversion);
|
||||||
|
const pending: PendingConnectionCreate = {
|
||||||
|
connection: { nodeId: source.id, handleType: "source", anchorRatio: 0.5 },
|
||||||
|
position: {
|
||||||
|
x: source.position.x + source.width + 96 + spec.width / 2,
|
||||||
|
y: source.position.y + source.height / 2,
|
||||||
|
},
|
||||||
|
quick: true,
|
||||||
|
};
|
||||||
|
void createConnectedNode(CanvasNodeType.MediaConversion, pending);
|
||||||
|
}, [createConnectedNode]);
|
||||||
|
|
||||||
const handleCanvasSelectionStart = useCallback(() => {
|
const handleCanvasSelectionStart = useCallback(() => {
|
||||||
setContextMenu(null);
|
setContextMenu(null);
|
||||||
}, []);
|
}, []);
|
||||||
@@ -1048,6 +1062,8 @@ function InfiniteCanvasPage() {
|
|||||||
} else if (node.type === ART_CRITIQUE_NODE_TYPE) {
|
} else if (node.type === ART_CRITIQUE_NODE_TYPE) {
|
||||||
setDialogNodeId(null);
|
setDialogNodeId(null);
|
||||||
setArtCritiqueNodeId(node.id);
|
setArtCritiqueNodeId(node.id);
|
||||||
|
} else if (node.type === CanvasNodeType.MediaConversion) {
|
||||||
|
setDialogNodeId(null);
|
||||||
} else {
|
} else {
|
||||||
// 选择参考媒体时保留当前工作流配置面板,避免点击图片后配置“返回/消失”。
|
// 选择参考媒体时保留当前工作流配置面板,避免点击图片后配置“返回/消失”。
|
||||||
// 没有工作流配置面板时,媒体节点仍按原逻辑打开自己的面板。
|
// 没有工作流配置面板时,媒体节点仍按原逻辑打开自己的面板。
|
||||||
@@ -1065,7 +1081,7 @@ function InfiniteCanvasPage() {
|
|||||||
|
|
||||||
const handleNodeDragEnd = useCallback((nodeId: string) => {
|
const handleNodeDragEnd = useCallback((nodeId: string) => {
|
||||||
const node = nodesRef.current.find((item) => item.id === nodeId);
|
const node = nodesRef.current.find((item) => item.id === nodeId);
|
||||||
if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) {
|
if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) {
|
||||||
setDialogNodeId(null);
|
setDialogNodeId(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1895,7 +1911,7 @@ function InfiniteCanvasPage() {
|
|||||||
|
|
||||||
const renderCanvasNodePanel = useCallback(
|
const renderCanvasNodePanel = useCallback(
|
||||||
(panelNode: CanvasNodeData) => {
|
(panelNode: CanvasNodeData) => {
|
||||||
if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing) return null;
|
if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing || panelNode.type === CanvasNodeType.MediaConversion) return null;
|
||||||
return panelNode.type === CanvasNodeType.Config ? (
|
return panelNode.type === CanvasNodeType.Config ? (
|
||||||
<CanvasConfigComposer
|
<CanvasConfigComposer
|
||||||
value={panelNode.metadata?.composerContent ?? panelNode.metadata?.prompt ?? ""}
|
value={panelNode.metadata?.composerContent ?? panelNode.metadata?.prompt ?? ""}
|
||||||
@@ -2572,6 +2588,7 @@ function InfiniteCanvasPage() {
|
|||||||
onUpload={(node) => handleUploadRequest(node.id)}
|
onUpload={(node) => handleUploadRequest(node.id)}
|
||||||
onDownload={downloadNodeImage}
|
onDownload={downloadNodeImage}
|
||||||
onSaveAsset={(node) => void saveNodeAsset(node)}
|
onSaveAsset={(node) => void saveNodeAsset(node)}
|
||||||
|
onCreateConversion={createConversionFromSource}
|
||||||
onAnnotate={(node) => setAnnotationNodeId(node.id)}
|
onAnnotate={(node) => setAnnotationNodeId(node.id)}
|
||||||
onMaskEdit={(node) => setMaskEditNodeId(node.id)}
|
onMaskEdit={(node) => setMaskEditNodeId(node.id)}
|
||||||
onEmotion={(node) => {
|
onEmotion={(node) => {
|
||||||
|
|||||||
@@ -200,7 +200,7 @@ export function useCanvasConnectionController({
|
|||||||
setContextMenu(null);
|
setContextMenu(null);
|
||||||
}, [config, connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]);
|
}, [config, connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]);
|
||||||
|
|
||||||
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
|
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
|
||||||
const nodeType = type;
|
const nodeType = type;
|
||||||
if (nodeType === CanvasNodeType.Drawing && !isDrawingEngineAvailable(defaultDrawingEngine, tldrawLicenseKey)) {
|
if (nodeType === CanvasNodeType.Drawing && !isDrawingEngineAvailable(defaultDrawingEngine, tldrawLicenseKey)) {
|
||||||
message.error("当前生产构建未配置 tldraw License Key,不能创建 tldraw 绘图");
|
message.error("当前生产构建未配置 tldraw License Key,不能创建 tldraw 绘图");
|
||||||
@@ -279,6 +279,12 @@ export function useCanvasConnectionController({
|
|||||||
setConnecting(null);
|
setConnecting(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (batchSourceNodeIds.length && nodeType === CanvasNodeType.MediaConversion) {
|
||||||
|
message.error("转换节点一次只能连接一个输入,请先连接到普通节点");
|
||||||
|
closeConnectionCreateMenu();
|
||||||
|
setConnecting(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const batchPlan = batchSourceNodeIds.length
|
const batchPlan = batchSourceNodeIds.length
|
||||||
? planBatchConnections({ sourceNodeIds: batchSourceNodeIds, targetNodeId: newNode.id, nodes: [...nodesRef.current, newNode], connections: connectionsRef.current, config, allowCapacityOverflow: true })
|
? planBatchConnections({ sourceNodeIds: batchSourceNodeIds, targetNodeId: newNode.id, nodes: [...nodesRef.current, newNode], connections: connectionsRef.current, config, allowCapacityOverflow: true })
|
||||||
: null;
|
: null;
|
||||||
@@ -298,7 +304,7 @@ export function useCanvasConnectionController({
|
|||||||
setConnections(nextConnections);
|
setConnections(nextConnections);
|
||||||
setSelectedNodeIds(new Set([newNode.id]));
|
setSelectedNodeIds(new Set([newNode.id]));
|
||||||
setSelectedConnectionId(null);
|
setSelectedConnectionId(null);
|
||||||
if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
|
if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id);
|
||||||
const skippedCount = batchPlan.skipped.length;
|
const skippedCount = batchPlan.skipped.length;
|
||||||
const duplicateCount = batchPlan.duplicates.length;
|
const duplicateCount = batchPlan.duplicates.length;
|
||||||
const suffix = skippedCount || duplicateCount ? `,跳过 ${skippedCount + duplicateCount} 个` : "";
|
const suffix = skippedCount || duplicateCount ? `,跳过 ${skippedCount + duplicateCount} 个` : "";
|
||||||
@@ -364,17 +370,18 @@ export function useCanvasConnectionController({
|
|||||||
setSelectedNodeIds(new Set([newNode.id]));
|
setSelectedNodeIds(new Set([newNode.id]));
|
||||||
setSelectedConnectionId(null);
|
setSelectedConnectionId(null);
|
||||||
if (nodeType === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id);
|
if (nodeType === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id);
|
||||||
else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
|
else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id);
|
||||||
closeConnectionCreateMenu();
|
closeConnectionCreateMenu();
|
||||||
setConnecting(null);
|
setConnecting(null);
|
||||||
}, [closeConnectionCreateMenu, config, connectionsRef, defaultDrawingEngine, message, nodesRef, projectId, runtimeStatuses, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, tldrawLicenseKey]);
|
}, [closeConnectionCreateMenu, config, connectionsRef, defaultDrawingEngine, message, nodesRef, projectId, runtimeStatuses, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, tldrawLicenseKey]);
|
||||||
|
|
||||||
const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
|
const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
|
||||||
const nodeType = type;
|
const nodeType = type;
|
||||||
if (nodeType === CanvasNodeType.Config) {
|
if (nodeType === CanvasNodeType.Config) {
|
||||||
if (workflowProvider && !workflowProviderPluginEnabled(runtimeStatuses, workflowProvider)) return `${workflowProvider === "runninghub" ? "RunningHub" : "ComfyUI"} 工作流插件未启用`;
|
if (workflowProvider && !workflowProviderPluginEnabled(runtimeStatuses, workflowProvider)) return `${workflowProvider === "runninghub" ? "RunningHub" : "ComfyUI"} 工作流插件未启用`;
|
||||||
}
|
}
|
||||||
if (pending.batchSourceNodeIds?.length) {
|
if (pending.batchSourceNodeIds?.length) {
|
||||||
|
if (nodeType === CanvasNodeType.MediaConversion) return "转换节点一次只能连接一个输入";
|
||||||
if (nodeType === CanvasNodeType.Drawing || nodeType === CanvasNodeType.Config) return "批量连接暂不支持此节点类型";
|
if (nodeType === CanvasNodeType.Drawing || nodeType === CanvasNodeType.Config) return "批量连接暂不支持此节点类型";
|
||||||
const pendingNode: CanvasNodeData = { id: "__pending-connection-node__", type: nodeType, title: "", position: pending.position, width: getNodeSpec(nodeType).width, height: getNodeSpec(nodeType).height };
|
const pendingNode: CanvasNodeData = { id: "__pending-connection-node__", type: nodeType, title: "", position: pending.position, width: getNodeSpec(nodeType).width, height: getNodeSpec(nodeType).height };
|
||||||
const plan = planBatchConnections({ sourceNodeIds: pending.batchSourceNodeIds, targetNodeId: pendingNode.id, nodes: [...nodesRef.current, pendingNode], connections: connectionsRef.current, config, allowCapacityOverflow: true });
|
const plan = planBatchConnections({ sourceNodeIds: pending.batchSourceNodeIds, targetNodeId: pendingNode.id, nodes: [...nodesRef.current, pendingNode], connections: connectionsRef.current, config, allowCapacityOverflow: true });
|
||||||
|
|||||||
@@ -153,7 +153,7 @@ export function useCanvasNodeOperations({
|
|||||||
if (workflowTitle) node.title = workflowTitle;
|
if (workflowTitle) node.title = workflowTitle;
|
||||||
commitNodes([...nodesRef.current, node]);
|
commitNodes([...nodesRef.current, node]);
|
||||||
selectNodes(new Set([node.id]));
|
selectNodes(new Set([node.id]));
|
||||||
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id);
|
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== CanvasNodeType.MediaConversion && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id);
|
||||||
}, [commitNodes, defaultDrawingEngine, effectiveConfig.comfyBridge.enabled, effectiveConfig.comfyBridge.workflows.length, effectiveConfig.runningHub.enabled, effectiveConfig.runningHub.workflows.length, getCanvasCenter, message, nodesRef, runtimeStatuses, selectNodes, setDialogNodeId, tldrawLicenseKey]);
|
}, [commitNodes, defaultDrawingEngine, effectiveConfig.comfyBridge.enabled, effectiveConfig.comfyBridge.workflows.length, effectiveConfig.runningHub.enabled, effectiveConfig.runningHub.workflows.length, getCanvasCenter, message, nodesRef, runtimeStatuses, selectNodes, setDialogNodeId, tldrawLicenseKey]);
|
||||||
|
|
||||||
const createFolder = useCallback((position?: Position, linked?: { id: string; projectId: string; title: string; style: CanvasFolderStyle; theme: CanvasFolderTheme; createdAt: string }) => {
|
const createFolder = useCallback((position?: Position, linked?: { id: string; projectId: string; title: string; style: CanvasFolderStyle; theme: CanvasFolderTheme; createdAt: string }) => {
|
||||||
@@ -400,7 +400,7 @@ export function useCanvasNodeOperations({
|
|||||||
const sourceNode = sourceByTargetId.get(targetNode.id);
|
const sourceNode = sourceByTargetId.get(targetNode.id);
|
||||||
if (sourceNode) cloneDrawingForNode(sourceNode, targetNode, "绘图副本保存失败,请重新复制");
|
if (sourceNode) cloneDrawingForNode(sourceNode, targetNode, "绘图副本保存失败,请重新复制");
|
||||||
});
|
});
|
||||||
if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id);
|
if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== CanvasNodeType.MediaConversion && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id);
|
||||||
}, [cloneDrawingForNode, commitConnections, commitNodes, connectionsRef, nodesRef, selectNodes, setDialogNodeId]);
|
}, [cloneDrawingForNode, commitConnections, commitNodes, connectionsRef, nodesRef, selectNodes, setDialogNodeId]);
|
||||||
|
|
||||||
const setPrimaryVersion = useCallback((nodeId: string) => {
|
const setPrimaryVersion = useCallback((nodeId: string) => {
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ export function useCanvasViewportController({
|
|||||||
const scale = Math.min(1.18, Math.max(viewportRef.current.k, 0.72));
|
const scale = Math.min(1.18, Math.max(viewportRef.current.k, 0.72));
|
||||||
transitionViewportTo({ x: size.width / 2 - (node.position.x + node.width / 2) * scale, y: size.height / 2 - (node.position.y + node.height / 2) * scale, k: scale });
|
transitionViewportTo({ x: size.width / 2 - (node.position.x + node.width / 2) * scale, y: size.height / 2 - (node.position.y + node.height / 2) * scale, k: scale });
|
||||||
selectFocusedNode(node.id);
|
selectFocusedNode(node.id);
|
||||||
setDialogNodeId(node.type === CanvasNodeType.Drawing ? null : node.id);
|
setDialogNodeId(node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion ? null : node.id);
|
||||||
}, [nodesRef, selectFocusedNode, setDialogNodeId, size.height, size.width, transitionViewportTo, viewportRef]);
|
}, [nodesRef, selectFocusedNode, setDialogNodeId, size.height, size.width, transitionViewportTo, viewportRef]);
|
||||||
|
|
||||||
const setZoomScale = useCallback((scale: number) => {
|
const setZoomScale = useCallback((scale: number) => {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import "@/lib/plugins/builtin";
|
|||||||
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
|
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
|
||||||
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
|
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
|
||||||
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
|
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
|
||||||
|
import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion";
|
||||||
import { ART_CRITIQUE_PLUGIN_ID } from "@/lib/art-critique/contracts";
|
import { ART_CRITIQUE_PLUGIN_ID } from "@/lib/art-critique/contracts";
|
||||||
import type { PluginManifest, PluginManifestV2, RegisteredPlugin } from "@/lib/plugins/plugin-types";
|
import type { PluginManifest, PluginManifestV2, RegisteredPlugin } from "@/lib/plugins/plugin-types";
|
||||||
import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle";
|
import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle";
|
||||||
@@ -623,7 +624,7 @@ function toRegisteredPlugin(plugin: BackendPlugin): RegisteredPlugin {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function isOfficialApplicationPlugin(pluginId: string) {
|
function isOfficialApplicationPlugin(pluginId: string) {
|
||||||
return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID].includes(pluginId);
|
return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID, MEDIA_CONVERSION_PLUGIN_ID].includes(pluginId);
|
||||||
}
|
}
|
||||||
|
|
||||||
function pluginSourceLabel(plugin: RegisteredPlugin, state?: PluginState) {
|
function pluginSourceLabel(plugin: RegisteredPlugin, state?: PluginState) {
|
||||||
|
|||||||
@@ -0,0 +1,202 @@
|
|||||||
|
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
|
||||||
|
import type { LocalRuntimeTransport } from "@/services/local-runtime";
|
||||||
|
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
|
||||||
|
|
||||||
|
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
|
||||||
|
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
|
||||||
|
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||||
|
|
||||||
|
export type LocalDepthResult = {
|
||||||
|
blob: Blob;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
modelId: string;
|
||||||
|
device: "cpu";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type LocalDepthRuntimeStatus = {
|
||||||
|
ok: true;
|
||||||
|
module: "depth-estimation";
|
||||||
|
apiVersion: 1;
|
||||||
|
ready: boolean;
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
loaded: boolean;
|
||||||
|
device: "cpu";
|
||||||
|
modelId: string;
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function runLocalDepthEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalDepthResult> {
|
||||||
|
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
|
||||||
|
const value = await requestJson<unknown>("/depth-estimation/run", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ schemaVersion: 1, operation: "depth", dataUrl }),
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果响应无效");
|
||||||
|
}
|
||||||
|
return parseResult(value.result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function readLocalDepthRuntimeStatus(signal?: AbortSignal) {
|
||||||
|
return parseStatus(await requestJson<unknown>("/depth-estimation/status", { method: "GET", signal }));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
|
||||||
|
if (sourceUrl.startsWith("data:")) {
|
||||||
|
const match = DATA_URL_PATTERN.exec(sourceUrl);
|
||||||
|
if (!match) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
const bytes = bytesFromBase64(match[2]!);
|
||||||
|
assertImageSize(bytes);
|
||||||
|
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
|
||||||
|
}
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取");
|
||||||
|
}
|
||||||
|
if (!response.ok) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取", response.status);
|
||||||
|
const mimeType = imageMime(response.headers.get("content-type") || response.type);
|
||||||
|
const declaredLength = Number(response.headers.get("content-length"));
|
||||||
|
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) {
|
||||||
|
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||||
|
assertImageSize(bytes);
|
||||||
|
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
|
||||||
|
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
|
||||||
|
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
|
||||||
|
if (!response.ok) {
|
||||||
|
const code = isRecord(body) && typeof body.code === "string" ? body.code : "depth_runtime_unavailable";
|
||||||
|
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机深度运行时不可用";
|
||||||
|
throw new LocalRuntimeClientError(code, message, response.status);
|
||||||
|
}
|
||||||
|
return body as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
|
||||||
|
let refreshed = false;
|
||||||
|
while (true) {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await (await ensureTransport()).request(path, init);
|
||||||
|
} catch (error) {
|
||||||
|
if (refreshed || !isSessionRefreshError(error)) throw error;
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!refreshed && response.status === 401) {
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (response.status === 403) {
|
||||||
|
const body = await parseJsonResponse(response);
|
||||||
|
if (!refreshed && shouldRefreshSession(response, body)) {
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return { response, body };
|
||||||
|
}
|
||||||
|
return { response };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureTransport(): Promise<LocalRuntimeTransport> {
|
||||||
|
const state = useLocalRuntimeStore.getState();
|
||||||
|
if (state.connection !== "connected") await state.connect();
|
||||||
|
const current = useLocalRuntimeStore.getState();
|
||||||
|
if (current.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", current.error || "本机深度运行时不可用");
|
||||||
|
return getLocalRuntimeSessionClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function reconnectTransport() {
|
||||||
|
const client = getLocalRuntimeSessionClient();
|
||||||
|
client.revokeLocalSession();
|
||||||
|
await useLocalRuntimeStore.getState().connect();
|
||||||
|
const state = useLocalRuntimeStore.getState();
|
||||||
|
if (state.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", state.error || "本机深度运行时不可用");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function parseJsonResponse(response: Response) {
|
||||||
|
const text = await boundedText(response, MAX_RESPONSE_BYTES);
|
||||||
|
try {
|
||||||
|
return text ? JSON.parse(text) as unknown : {};
|
||||||
|
} catch {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应无效", response.status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function boundedText(response: Response, limit: number) {
|
||||||
|
const declared = Number(response.headers.get("content-length"));
|
||||||
|
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status);
|
||||||
|
const text = await response.text();
|
||||||
|
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status);
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResult(value: Record<string, unknown>): LocalDepthResult {
|
||||||
|
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果无效");
|
||||||
|
}
|
||||||
|
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机深度结果无效");
|
||||||
|
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果过大");
|
||||||
|
return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseStatus(value: unknown): LocalDepthRuntimeStatus {
|
||||||
|
if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度状态无效");
|
||||||
|
}
|
||||||
|
return value as LocalDepthRuntimeStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
|
||||||
|
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
|
||||||
|
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
|
||||||
|
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertImageSize(bytes: Uint8Array) {
|
||||||
|
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bytesToBase64(bytes: Uint8Array) {
|
||||||
|
let binary = "";
|
||||||
|
for (const byte of bytes) binary += String.fromCharCode(byte);
|
||||||
|
return btoa(binary);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bytesFromBase64(value: string, code = "depth_input_invalid", message = "深度转换输入图片无效") {
|
||||||
|
let binary: string;
|
||||||
|
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "depth_input_invalid" ? 400 : 0); }
|
||||||
|
const bytes = new Uint8Array(binary.length);
|
||||||
|
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
|
||||||
|
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldRefreshSession(response: Response, body: unknown) {
|
||||||
|
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPositiveInteger(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
|
||||||
|
import type { LocalRuntimeTransport } from "@/services/local-runtime";
|
||||||
|
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
|
||||||
|
|
||||||
|
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
|
||||||
|
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
|
||||||
|
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||||
|
|
||||||
|
export type LocalLineartResult = {
|
||||||
|
blob: Blob;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
modelId: string;
|
||||||
|
device: "cpu";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type LocalLineartRuntimeStatus = {
|
||||||
|
ok: true;
|
||||||
|
module: "lineart-estimation";
|
||||||
|
apiVersion: 1;
|
||||||
|
ready: boolean;
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
loaded: boolean;
|
||||||
|
device: "cpu";
|
||||||
|
modelId: string;
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function runLocalLineartEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalLineartResult> {
|
||||||
|
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
|
||||||
|
const value = await requestJson<unknown>("/lineart-estimation/run", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ schemaVersion: 1, operation: "lineart", dataUrl }),
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果响应无效");
|
||||||
|
}
|
||||||
|
return parseResult(value.result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function readLocalLineartRuntimeStatus(signal?: AbortSignal) {
|
||||||
|
return parseStatus(await requestJson<unknown>("/lineart-estimation/status", { method: "GET", signal }));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
|
||||||
|
if (sourceUrl.startsWith("data:")) {
|
||||||
|
const match = DATA_URL_PATTERN.exec(sourceUrl);
|
||||||
|
if (!match) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
const bytes = bytesFromBase64(match[2]!);
|
||||||
|
assertImageSize(bytes);
|
||||||
|
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
|
||||||
|
}
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取");
|
||||||
|
}
|
||||||
|
if (!response.ok) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取", response.status);
|
||||||
|
const mimeType = imageMime(response.headers.get("content-type") || response.type);
|
||||||
|
const declaredLength = Number(response.headers.get("content-length"));
|
||||||
|
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) {
|
||||||
|
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||||
|
assertImageSize(bytes);
|
||||||
|
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
|
||||||
|
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
|
||||||
|
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
|
||||||
|
if (!response.ok) {
|
||||||
|
const code = isRecord(body) && typeof body.code === "string" ? body.code : "lineart_runtime_unavailable";
|
||||||
|
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机 AI 线稿运行时不可用";
|
||||||
|
throw new LocalRuntimeClientError(code, message, response.status);
|
||||||
|
}
|
||||||
|
return body as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
|
||||||
|
let refreshed = false;
|
||||||
|
while (true) {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await (await ensureTransport()).request(path, init);
|
||||||
|
} catch (error) {
|
||||||
|
if (refreshed || !isSessionRefreshError(error)) throw error;
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!refreshed && response.status === 401) {
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (response.status === 403) {
|
||||||
|
const body = await parseJsonResponse(response);
|
||||||
|
if (!refreshed && shouldRefreshSession(response, body)) {
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return { response, body };
|
||||||
|
}
|
||||||
|
return { response };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureTransport(): Promise<LocalRuntimeTransport> {
|
||||||
|
const state = useLocalRuntimeStore.getState();
|
||||||
|
if (state.connection !== "connected") await state.connect();
|
||||||
|
const current = useLocalRuntimeStore.getState();
|
||||||
|
if (current.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", current.error || "本机 AI 线稿运行时不可用");
|
||||||
|
return getLocalRuntimeSessionClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function reconnectTransport() {
|
||||||
|
const client = getLocalRuntimeSessionClient();
|
||||||
|
client.revokeLocalSession();
|
||||||
|
await useLocalRuntimeStore.getState().connect();
|
||||||
|
const state = useLocalRuntimeStore.getState();
|
||||||
|
if (state.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", state.error || "本机 AI 线稿运行时不可用");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function parseJsonResponse(response: Response) {
|
||||||
|
const text = await boundedText(response, MAX_RESPONSE_BYTES);
|
||||||
|
try {
|
||||||
|
return text ? JSON.parse(text) as unknown : {};
|
||||||
|
} catch {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应无效", response.status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function boundedText(response: Response, limit: number) {
|
||||||
|
const declared = Number(response.headers.get("content-length"));
|
||||||
|
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status);
|
||||||
|
const text = await response.text();
|
||||||
|
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status);
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResult(value: Record<string, unknown>): LocalLineartResult {
|
||||||
|
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果无效");
|
||||||
|
}
|
||||||
|
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机 AI 线稿结果无效");
|
||||||
|
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果过大");
|
||||||
|
return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseStatus(value: unknown): LocalLineartRuntimeStatus {
|
||||||
|
if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿状态无效");
|
||||||
|
}
|
||||||
|
return value as LocalLineartRuntimeStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
|
||||||
|
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
|
||||||
|
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
|
||||||
|
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertImageSize(bytes: Uint8Array) {
|
||||||
|
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bytesToBase64(bytes: Uint8Array) {
|
||||||
|
let binary = "";
|
||||||
|
for (const byte of bytes) binary += String.fromCharCode(byte);
|
||||||
|
return btoa(binary);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bytesFromBase64(value: string, code = "lineart_input_invalid", message = "AI 线稿输入图片无效") {
|
||||||
|
let binary: string;
|
||||||
|
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "lineart_input_invalid" ? 400 : 0); }
|
||||||
|
const bytes = new Uint8Array(binary.length);
|
||||||
|
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
|
||||||
|
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldRefreshSession(response: Response, body: unknown) {
|
||||||
|
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPositiveInteger(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
|
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
|
||||||
|
|
||||||
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance";
|
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
|
||||||
export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read";
|
export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read" | "depth:status" | "depth:run" | "lineart:status" | "lineart:run" | "pose:status" | "pose:run";
|
||||||
|
|
||||||
export type LocalRuntimeModuleDescriptor = {
|
export type LocalRuntimeModuleDescriptor = {
|
||||||
id: LocalRuntimeModuleId;
|
id: LocalRuntimeModuleId;
|
||||||
@@ -28,6 +28,9 @@ const MODULE_SCOPES: Record<LocalRuntimeModuleId, ReadonlySet<LocalRuntimeScope>
|
|||||||
"canvas-agent": new Set(["canvas:connect"]),
|
"canvas-agent": new Set(["canvas:connect"]),
|
||||||
dreamina: new Set(["dreamina:status", "dreamina:login", "dreamina:logout", "dreamina:run", "dreamina:models", "dreamina:generate"]),
|
dreamina: new Set(["dreamina:status", "dreamina:login", "dreamina:logout", "dreamina:run", "dreamina:models", "dreamina:generate"]),
|
||||||
"portrait-clearance": new Set(["portrait:status", "portrait:model", "portrait:run", "portrait:read"]),
|
"portrait-clearance": new Set(["portrait:status", "portrait:model", "portrait:run", "portrait:read"]),
|
||||||
|
"depth-estimation": new Set(["depth:status", "depth:run"]),
|
||||||
|
"lineart-estimation": new Set(["lineart:status", "lineart:run"]),
|
||||||
|
"pose-estimation": new Set(["pose:status", "pose:run"]),
|
||||||
};
|
};
|
||||||
|
|
||||||
export async function readLocalRuntimeStatus(client: LocalRuntimeTransport, signal?: AbortSignal): Promise<LocalRuntimeStatus> {
|
export async function readLocalRuntimeStatus(client: LocalRuntimeTransport, signal?: AbortSignal): Promise<LocalRuntimeStatus> {
|
||||||
@@ -136,7 +139,7 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function isModuleId(value: unknown): value is LocalRuntimeModuleId {
|
function isModuleId(value: unknown): value is LocalRuntimeModuleId {
|
||||||
return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance";
|
return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance" || value === "depth-estimation" || value === "lineart-estimation" || value === "pose-estimation";
|
||||||
}
|
}
|
||||||
|
|
||||||
function invalidResponse(status: number) {
|
function invalidResponse(status: number) {
|
||||||
|
|||||||
@@ -0,0 +1,209 @@
|
|||||||
|
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
|
||||||
|
import type { LocalRuntimeTransport } from "@/services/local-runtime";
|
||||||
|
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
|
||||||
|
|
||||||
|
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
|
||||||
|
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
|
||||||
|
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
|
||||||
|
|
||||||
|
export type LocalPoseResult = {
|
||||||
|
blob: Blob;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
modelId: string;
|
||||||
|
device: "cpu";
|
||||||
|
backend: "openpose-body";
|
||||||
|
personCount: number;
|
||||||
|
people?: unknown[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type LocalPoseRuntimeStatus = {
|
||||||
|
ok: true;
|
||||||
|
module: "pose-estimation";
|
||||||
|
apiVersion: 1;
|
||||||
|
ready: boolean;
|
||||||
|
configured: boolean;
|
||||||
|
modelCached: boolean;
|
||||||
|
loaded: boolean;
|
||||||
|
device: "cpu";
|
||||||
|
modelId: string;
|
||||||
|
backend: "openpose-body";
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function runLocalPoseEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalPoseResult> {
|
||||||
|
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
|
||||||
|
const value = await requestJson<unknown>("/pose-estimation/run", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ schemaVersion: 1, operation: "pose", dataUrl }),
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果响应无效");
|
||||||
|
}
|
||||||
|
return parseResult(value.result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function readLocalPoseRuntimeStatus(signal?: AbortSignal) {
|
||||||
|
return parseStatus(await requestJson<unknown>("/pose-estimation/status", { method: "GET", signal }));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
|
||||||
|
if (sourceUrl.startsWith("data:")) {
|
||||||
|
const match = DATA_URL_PATTERN.exec(sourceUrl);
|
||||||
|
if (!match) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
const bytes = bytesFromBase64(match[2]!);
|
||||||
|
assertImageSize(bytes);
|
||||||
|
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
|
||||||
|
}
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取");
|
||||||
|
}
|
||||||
|
if (!response.ok) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取", response.status);
|
||||||
|
const mimeType = imageMime(response.headers.get("content-type") || response.type);
|
||||||
|
const declaredLength = Number(response.headers.get("content-length"));
|
||||||
|
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
|
||||||
|
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||||
|
assertImageSize(bytes);
|
||||||
|
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
|
||||||
|
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
|
||||||
|
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
|
||||||
|
if (!response.ok) {
|
||||||
|
const code = isRecord(body) && typeof body.code === "string" ? body.code : "pose_runtime_unavailable";
|
||||||
|
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机姿态运行时不可用";
|
||||||
|
throw new LocalRuntimeClientError(code, message, response.status);
|
||||||
|
}
|
||||||
|
return body as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
|
||||||
|
let refreshed = false;
|
||||||
|
while (true) {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await (await ensureTransport()).request(path, init);
|
||||||
|
} catch (error) {
|
||||||
|
if (refreshed || !isSessionRefreshError(error)) throw error;
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!refreshed && response.status === 401) {
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (response.status === 403) {
|
||||||
|
const body = await parseJsonResponse(response);
|
||||||
|
if (!refreshed && shouldRefreshSession(response, body)) {
|
||||||
|
await reconnectTransport();
|
||||||
|
refreshed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return { response, body };
|
||||||
|
}
|
||||||
|
return { response };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureTransport(): Promise<LocalRuntimeTransport> {
|
||||||
|
const state = useLocalRuntimeStore.getState();
|
||||||
|
if (state.connection !== "connected") await state.connect();
|
||||||
|
const current = useLocalRuntimeStore.getState();
|
||||||
|
if (current.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", current.error || "本机姿态运行时不可用");
|
||||||
|
return getLocalRuntimeSessionClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function reconnectTransport() {
|
||||||
|
const client = getLocalRuntimeSessionClient();
|
||||||
|
client.revokeLocalSession();
|
||||||
|
await useLocalRuntimeStore.getState().connect();
|
||||||
|
const state = useLocalRuntimeStore.getState();
|
||||||
|
if (state.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", state.error || "本机姿态运行时不可用");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function parseJsonResponse(response: Response) {
|
||||||
|
const text = await boundedText(response, MAX_RESPONSE_BYTES);
|
||||||
|
try { return text ? JSON.parse(text) as unknown : {}; } catch { throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应无效", response.status); }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function boundedText(response: Response, limit: number) {
|
||||||
|
const declared = Number(response.headers.get("content-length"));
|
||||||
|
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status);
|
||||||
|
const text = await response.text();
|
||||||
|
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status);
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResult(value: Record<string, unknown>): LocalPoseResult {
|
||||||
|
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || value.backend !== "openpose-body" || !isPositiveInteger(value.personCount) || typeof value.pngBase64 !== "string") {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果无效");
|
||||||
|
}
|
||||||
|
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机姿态结果无效");
|
||||||
|
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果过大");
|
||||||
|
return {
|
||||||
|
blob: new Blob([bytes], { type: "image/png" }),
|
||||||
|
width: value.width,
|
||||||
|
height: value.height,
|
||||||
|
modelId: value.modelId,
|
||||||
|
device: "cpu",
|
||||||
|
backend: "openpose-body",
|
||||||
|
personCount: value.personCount,
|
||||||
|
people: Array.isArray(value.people) ? value.people : undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseStatus(value: unknown): LocalPoseRuntimeStatus {
|
||||||
|
if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || value.backend !== "openpose-body" || (value.reason !== undefined && typeof value.reason !== "string")) {
|
||||||
|
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态状态无效");
|
||||||
|
}
|
||||||
|
return value as LocalPoseRuntimeStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
|
||||||
|
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
|
||||||
|
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
|
||||||
|
throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertImageSize(bytes: Uint8Array) {
|
||||||
|
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bytesToBase64(bytes: Uint8Array) {
|
||||||
|
let binary = "";
|
||||||
|
for (const byte of bytes) binary += String.fromCharCode(byte);
|
||||||
|
return btoa(binary);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bytesFromBase64(value: string, code = "pose_input_invalid", message = "姿态转换输入图片无效") {
|
||||||
|
let binary: string;
|
||||||
|
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "pose_input_invalid" ? 400 : 0); }
|
||||||
|
const bytes = new Uint8Array(binary.length);
|
||||||
|
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
|
||||||
|
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldRefreshSession(response: Response, body: unknown) {
|
||||||
|
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPositiveInteger(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { CanvasColorGrade } from "@/lib/canvas/canvas-color-grade";
|
import type { CanvasColorGrade } from "@/lib/canvas/canvas-color-grade";
|
||||||
|
import type { MediaConversionNodeState } from "@/lib/media-conversion/contracts";
|
||||||
import type { AssetCategory } from "@/lib/asset-category";
|
import type { AssetCategory } from "@/lib/asset-category";
|
||||||
import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture";
|
import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture";
|
||||||
import type { StyleExecutionPlan } from "@/lib/canvas/style-profile";
|
import type { StyleExecutionPlan } from "@/lib/canvas/style-profile";
|
||||||
@@ -34,6 +35,7 @@ export enum CanvasNodeType {
|
|||||||
Compare = "compare",
|
Compare = "compare",
|
||||||
Chart = "chart",
|
Chart = "chart",
|
||||||
ColorGrade = "colorgrade",
|
ColorGrade = "colorgrade",
|
||||||
|
MediaConversion = "media-conversion",
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Runtime IDs contributed by plugins share the persisted node type field. */
|
/** Runtime IDs contributed by plugins share the persisted node type field. */
|
||||||
@@ -362,6 +364,8 @@ export type CanvasNodeMetadata = {
|
|||||||
chartKind?: "bar" | "line";
|
chartKind?: "bar" | "line";
|
||||||
/** 调色节点的参数;缺省视为未调色。 */
|
/** 调色节点的参数;缺省视为未调色。 */
|
||||||
colorGrade?: CanvasColorGrade;
|
colorGrade?: CanvasColorGrade;
|
||||||
|
/** 本地图片/视频转换节点的参数、来源指纹和结果状态。 */
|
||||||
|
mediaConversion?: MediaConversionNodeState;
|
||||||
/** 用户手动拉伸过尺寸;图片按真实比例自动适配时避让它。 */
|
/** 用户手动拉伸过尺寸;图片按真实比例自动适配时避让它。 */
|
||||||
manualSize?: boolean;
|
manualSize?: boolean;
|
||||||
storyboard?: StoryboardData;
|
storyboard?: StoryboardData;
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
|
||||||
|
import { canvasConnectionError } from "../src/lib/canvas/canvas-connection-policy";
|
||||||
|
import { findPendingMediaConversionInput } from "../src/components/canvas/canvas-node-generation";
|
||||||
|
import { getNodeAcceptedInputKinds, getNodeMaxInputCount, getNodeResourceKind } from "../src/lib/canvas/node-registry";
|
||||||
|
import { mediaConversionSourceFingerprint } from "../src/lib/media-conversion/contracts";
|
||||||
|
import { convertImageLocally, LocalImageConversionError, transformPixels } from "../src/lib/media-conversion/local-converter";
|
||||||
|
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
|
||||||
|
import type { AiConfig } from "../src/stores/use-config-store";
|
||||||
|
|
||||||
|
function node(type: CanvasNodeType, id: string, metadata: CanvasNodeData["metadata"] = {}): CanvasNodeData {
|
||||||
|
return { id, type, title: id, position: { x: 0, y: 0 }, width: 380, height: 440, metadata };
|
||||||
|
}
|
||||||
|
|
||||||
|
const config = {} as AiConfig;
|
||||||
|
|
||||||
|
describe("本地转换节点注册", () => {
|
||||||
|
test("只允许一个图片或视频输入,完成后才作为图片素材", () => {
|
||||||
|
expect(getNodeAcceptedInputKinds(CanvasNodeType.MediaConversion)).toEqual(["image", "video"]);
|
||||||
|
expect(getNodeMaxInputCount(CanvasNodeType.MediaConversion)).toBe(1);
|
||||||
|
expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion"))).toBeNull();
|
||||||
|
expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion", {
|
||||||
|
mediaConversion: { schemaVersion: 1, operation: "grayscale", status: "completed", resultStorageKey: "image:scope:result" },
|
||||||
|
}))).toBe("image");
|
||||||
|
expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion", {
|
||||||
|
mediaConversion: { schemaVersion: 1, operation: "pose", status: "skipped", detectedPeople: 0 },
|
||||||
|
}))).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("连接规则接受图片和视频,拒绝没有可读媒体的节点及第二个输入", () => {
|
||||||
|
const image = node(CanvasNodeType.Image, "image");
|
||||||
|
const video = node(CanvasNodeType.Video, "video");
|
||||||
|
const text = node(CanvasNodeType.Text, "text");
|
||||||
|
const conversion = node(CanvasNodeType.MediaConversion, "conversion");
|
||||||
|
const nodes = [image, video, text, conversion];
|
||||||
|
const candidate = (fromNodeId: string): CanvasConnection => ({ id: `candidate-${fromNodeId}`, fromNodeId, toNodeId: conversion.id });
|
||||||
|
|
||||||
|
expect(canvasConnectionError(config, nodes, [], candidate(image.id))).toBe("");
|
||||||
|
expect(canvasConnectionError(config, nodes, [], candidate(video.id))).toBe("");
|
||||||
|
expect(canvasConnectionError(config, nodes, [], candidate(text.id))).toContain("转换节点只接受图片或视频输入");
|
||||||
|
expect(canvasConnectionError(config, nodes, [{ id: "existing", fromNodeId: image.id, toNodeId: conversion.id }], candidate(video.id))).toContain("转换节点最多连接 1 个输入");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("下游生成会阻止未完成或待更新的转换结果", () => {
|
||||||
|
const conversion = node(CanvasNodeType.MediaConversion, "conversion", {
|
||||||
|
mediaConversion: { schemaVersion: 1, operation: "depth", status: "unavailable" },
|
||||||
|
});
|
||||||
|
const input = node(CanvasNodeType.Image, "input", { content: "data:image/png;base64,source" });
|
||||||
|
const target = node(CanvasNodeType.Image, "target");
|
||||||
|
const connections: CanvasConnection[] = [
|
||||||
|
{ id: "input-conversion", fromNodeId: input.id, toNodeId: conversion.id },
|
||||||
|
{ id: "conversion-target", fromNodeId: conversion.id, toNodeId: target.id },
|
||||||
|
];
|
||||||
|
expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)?.id).toBe(conversion.id);
|
||||||
|
conversion.metadata = { mediaConversion: { schemaVersion: 1, operation: "grayscale", status: "completed", sourceNodeId: input.id, sourceFingerprint: "v1-unknown", resultStorageKey: "image:scope:result" } };
|
||||||
|
expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)?.id).toBe(conversion.id);
|
||||||
|
conversion.metadata.mediaConversion.sourceFingerprint = mediaConversionSourceFingerprint(input);
|
||||||
|
expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("图片本地转换算法", () => {
|
||||||
|
test("灰度转换保留 alpha 并使用感知亮度", () => {
|
||||||
|
const pixels = new Uint8ClampedArray([255, 0, 0, 200, 0, 255, 0, 100]);
|
||||||
|
const output = transformPixels(pixels, 2, 1, "grayscale");
|
||||||
|
expect(Array.from(output)).toEqual([76, 76, 76, 200, 150, 150, 150, 100]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Canny 转换输出黑白边缘图并保留 alpha", () => {
|
||||||
|
const pixels = new Uint8ClampedArray([
|
||||||
|
0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255,
|
||||||
|
0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255,
|
||||||
|
0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255,
|
||||||
|
]);
|
||||||
|
const output = transformPixels(pixels, 3, 3, "edge-canny");
|
||||||
|
for (let index = 0; index < output.length; index += 4) {
|
||||||
|
expect([0, 255]).toContain(output[index]);
|
||||||
|
expect(output[index]).toBe(output[index + 1]);
|
||||||
|
expect(output[index]).toBe(output[index + 2]);
|
||||||
|
expect(output[index + 3]).toBe(255);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("浏览器算法转换把模型操作交给本机 Runtime", async () => {
|
||||||
|
await expect(convertImageLocally("", "depth")).rejects.toMatchObject<LocalImageConversionError>({ code: "model_missing" });
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user