feat(canvas): 图片转换插件 - 合并 PR #446 本地媒体转换节点与运行时

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ddCatandCursor committed 2026-09-08 23:10:26 +08:00
commit 99f50f85eb
51 files changed
+3286 -46

No files matched your search

+11 -2
View File
@@ -13,6 +13,7 @@ const (
PluginPromptOptimizer = "prompt-optimizer" PluginPromptOptimizer = "prompt-optimizer"
PluginPortraitClearance = "portrait-clearance" PluginPortraitClearance = "portrait-clearance"
PluginAIArtCritique = "ai-art-critique" PluginAIArtCritique = "ai-art-critique"
PluginMediaConversion = "media-conversion"
PluginOriginOfficial = "official" PluginOriginOfficial = "official"
PluginOriginSystem = "system" PluginOriginSystem = "system"
@@ -80,6 +81,10 @@ var officialApplicationPolicies = map[string]PluginManagementView{
Origin: PluginOriginOfficial, Kind: PluginKindApplication, Origin: PluginOriginOfficial, Kind: PluginKindApplication,
ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone, ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone,
}, },
PluginMediaConversion: {
Origin: PluginOriginOfficial, Kind: PluginKindApplication,
ActivationScope: PluginScopeUser, ConfigurationScope: PluginConfigurationNone,
},
} }
var systemPaymentPolicies = map[string]PluginManagementView{ var systemPaymentPolicies = map[string]PluginManagementView{
@@ -196,8 +201,12 @@ func (s *Service) pluginStateForUser(actor *model.User, pluginID string, items [
// Preserve the old globally-enabled workflow behavior until each user // Preserve the old globally-enabled workflow behavior until each user
// explicitly saves a personal choice. Other official applications were // explicitly saves a personal choice. Other official applications were
// already controlled by each user's local installation state. // already controlled by each user's local installation state.
if !userConfigured && hasRuntime && isLegacyWorkflowPlugin(pluginID) { if !userConfigured {
userEnabled = runtimePlugin.Status == "enabled" if pluginID == PluginMediaConversion {
userEnabled = true
} else if hasRuntime && isLegacyWorkflowPlugin(pluginID) {
userEnabled = runtimePlugin.Status == "enabled"
}
} }
} }
effective := platformAvailable effective := platformAvailable
+52
View File
@@ -76,6 +76,58 @@ Canvas Agent 内置 `portrait-clearance` Local Runtime 模块。它只接收签
本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`,安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时,模块返回 `portrait_model_missing`,不会伪造低风险结果。 本机模型不会随 npm 包发布。用户必须在肖像排查工作台中显式安装并校验 `buffalo_l` 所需的 `det_10g.onnx` 与 `w600k_r50.onnx`,安装过程保留现有可用模型并拒绝校验失败的临时文件。缺少模型时,模块返回 `portrait_model_missing`,不会伪造低风险结果。
## Depth Anything V2 本机深度模块
画布“转换 → 深度图”使用 `depth-estimation` Local Runtime 模块。模块只接受签名的本机请求,Node Runtime 启动一个常驻 Python worker;模型首次调用时加载,后续转换复用同一进程。worker 以离线模式读取本地 Hugging Face 缓存,不会把图片发送到云端。
使用仓库根目录的 `start-yingce-local.cmd` 时,Canvas Agent 会和前端、后端一起自动启动,转换节点不需要用户另外打开 Agent 窗口或手工填写地址。只有手工分别启动前端和后端时,才需要在本目录执行 `node dist/index.js`。
当前仓库开发环境默认查找:
```text
.local/depth-anything-v2/venv/Scripts/python.exe # Windows
.local/depth-anything-v2/venv/bin/python # Linux/macOS
.local/cache/huggingface/ # 模型缓存
```
发布包会携带 `python/depth_anything_runtime.py`,但不会携带 Python、依赖或模型权重。自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_DEPTH_HF_HOME`;需要指定工作区时设置 `CANVAS_PROJECT_ROOT`。模型缺失、Python 环境缺失或 worker 退出时,转换节点显示具体失败状态并保留输入,不会把失败标记为完成。
## ControlNet Aux 本机 AI 线稿模块
画布“转换 → AI 线稿”使用 `lineart-estimation` Local Runtime 模块。它只运行 ControlNet Aux 的 `LineartDetector` 预处理器,把图片转换为线稿控制图;不会加载 Stable Diffusion、ControlNet 生图管线,也不会把图片上传到云端。Node Runtime 会复用同一个常驻 Python worker,首次转换加载模型,后续请求复用已加载模型。
该模块与深度模块共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 模型缓存。首次设置或缺少依赖时,在仓库根目录执行:
```powershell
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
& $python -m pip install -r canvas-agent/python/requirements-lineart.txt
```
然后用下面的命令把线稿预处理器权重缓存到本地;下载完成后,正常转换会强制离线运行:
```powershell
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
& $python -c "from controlnet_aux import LineartDetector; LineartDetector.from_pretrained('lllyasviel/Annotators')"
```
自定义安装位置时设置 `CANVAS_DEPTH_PYTHON`、`CANVAS_LINEART_HF_HOME` 和 `CANVAS_LINEART_MODEL_ID`;未找到 Python、依赖或模型时,节点显示“本地不可用”并保留输入,不会伪造已完成结果。
## ControlNet Aux 本机姿态骨架模块
画布“转换 → 姿态骨架”使用 `pose-estimation` Local Runtime 模块。当前笔记本方案采用 ControlNet Aux 的 **OpenPose body-only** 预处理器:只提取人体 18 点骨架并输出彩色姿态图,不加载 Stable Diffusion,也不上传图片。Node Runtime 会复用常驻 Python worker;未检测到人物时返回 `pose_no_person`,转换节点显示“已跳过”,不会保存伪造结果。
该模块与深度、线稿共用 `.local/depth-anything-v2/venv` Python 环境和 `.local/cache/huggingface` 缓存。安装 `requirements-lineart.txt` 后缓存人体权重:
```powershell
$python = ".local/depth-anything-v2/venv/Scripts/python.exe"
$env:HF_HOME = (Resolve-Path ".local/cache/huggingface").Path
$env:HF_HUB_CACHE = Join-Path $env:HF_HOME "hub"
& $python -c "from huggingface_hub import hf_hub_download; hf_hub_download('lllyasviel/Annotators', 'body_pose_model.pth', cache_dir='$env:HF_HUB_CACHE')"
```
自定义安装位置时设置 `CANVAS_POSE_PYTHON`、`CANVAS_POSE_HF_HOME` 和 `CANVAS_POSE_MODEL_ID`。DWPose(包含手部、脸部和更完整关键点)仍可在后续按需替换,但它需要额外的 MMDetection/MMPose 依赖;当前实现优先保证普通 CPU 笔记本可运行。
## 发布 ## 发布
`canvas-agent` 使用自己的 `package.json` 版本号,不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。 `canvas-agent` 使用自己的 `package.json` 版本号,不跟仓库根目录 `VERSION` 绑定。发布包名为 `@ddcat666/open-ai-canvas-agent`。
+1
View File
@@ -9,6 +9,7 @@
}, },
"files": [ "files": [
"dist", "dist",
"python",
"README.md" "README.md"
], ],
"scripts": { "scripts": {
@@ -0,0 +1,134 @@
"""Small persistent worker for the local Depth Anything V2 Small model.
The Node Local Runtime owns the process and sends one JSON request per line.
Keeping the pipeline alive avoids loading the model for every canvas conversion.
The worker is deliberately offline-only: the model must already exist in the
configured Hugging Face cache.
"""
from __future__ import annotations
import base64
import contextlib
import io
import json
import os
import re
import sys
from typing import Any
from PIL import Image
MODEL_ID = os.environ.get("CANVAS_DEPTH_MODEL_ID", "depth-anything/Depth-Anything-V2-Small-hf")
MAX_INPUT_BYTES = 12 * 1024 * 1024
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
PIPELINE: Any = None
def emit(value: dict[str, Any]) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def load_pipeline() -> Any:
global PIPELINE
if PIPELINE is not None:
return PIPELINE
try:
from transformers import pipeline
from transformers.utils import logging as transformers_logging
transformers_logging.set_verbosity_error()
# Transformers/tqdm can write progress text while loading weights.
# Keep stdout reserved for the JSON-lines protocol.
with contextlib.redirect_stdout(sys.stderr):
PIPELINE = pipeline(
"depth-estimation",
model=MODEL_ID,
device=-1,
local_files_only=True,
)
except Exception as error: # pragma: no cover - depends on local model files
raise RuntimeError("depth_model_missing") from error
return PIPELINE
def image_from_data_url(value: Any) -> Image.Image:
if not isinstance(value, str):
raise ValueError("depth_input_invalid")
match = DATA_URL_PATTERN.fullmatch(value)
if not match:
raise ValueError("depth_input_invalid")
try:
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
except Exception as error:
raise ValueError("depth_input_invalid") from error
if not raw or len(raw) > MAX_INPUT_BYTES:
raise ValueError("depth_input_invalid")
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
image.load()
return image
except Exception as error:
raise ValueError("depth_input_invalid") from error
def encode_png(image: Image.Image) -> str:
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return base64.b64encode(output.getvalue()).decode("ascii")
def handle(request: dict[str, Any]) -> dict[str, Any]:
image = image_from_data_url(request.get("dataUrl"))
try:
result = load_pipeline()(image)
depth = result.get("depth") if isinstance(result, dict) else None
if not isinstance(depth, Image.Image):
raise RuntimeError("depth_inference_failed")
return {
"ok": True,
"requestId": request.get("requestId"),
"mimeType": "image/png",
"width": depth.width,
"height": depth.height,
"modelId": MODEL_ID,
"device": "cpu",
"pngBase64": encode_png(depth),
}
except RuntimeError:
raise
except Exception as error: # pragma: no cover - depends on local runtime
raise RuntimeError("depth_inference_failed") from error
def main() -> None:
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
for line in sys.stdin:
line = line.strip()
if not line:
continue
request_id: Any = None
try:
request = json.loads(line)
if not isinstance(request, dict):
raise ValueError("depth_input_invalid")
request_id = request.get("requestId")
emit(handle(request))
except ValueError as error:
emit({"ok": False, "requestId": request_id, "code": str(error) or "depth_input_invalid", "message": "深度转换输入无效"})
except RuntimeError as error:
code = str(error) or "depth_inference_failed"
emit({"ok": False, "requestId": request_id, "code": code, "message": "本机深度模型不可用" if code == "depth_model_missing" else "本机深度推理失败"})
except Exception:
emit({"ok": False, "requestId": request_id, "code": "depth_inference_failed", "message": "本机深度推理失败"})
if __name__ == "__main__":
main()
+132
View File
@@ -0,0 +1,132 @@
"""Small persistent worker for the local ControlNet Aux lineart detector.
This worker only extracts a lineart control image. It does not load Stable
Diffusion or a ControlNet generation pipeline, so it stays suitable for a
CPU-only notebook. The Node Local Runtime owns the process and sends one JSON
request per line; stdout is reserved for that protocol.
"""
from __future__ import annotations
import base64
import contextlib
import io
import json
import os
import re
import sys
from typing import Any
from PIL import Image
MODEL_ID = os.environ.get("CANVAS_LINEART_MODEL_ID", "lllyasviel/Annotators")
MAX_INPUT_BYTES = 12 * 1024 * 1024
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
DETECTOR: Any = None
def emit(value: dict[str, Any]) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def load_detector() -> Any:
global DETECTOR
if DETECTOR is not None:
return DETECTOR
try:
from controlnet_aux import LineartDetector
# The detector downloads only the lineart preprocessor weights from the
# configured Hugging Face cache. It is intentionally not a diffusion
# pipeline and never loads Stable Diffusion or ControlNet weights.
with contextlib.redirect_stdout(sys.stderr):
DETECTOR = LineartDetector.from_pretrained(MODEL_ID)
except Exception as error: # pragma: no cover - depends on local files
raise RuntimeError("lineart_model_missing") from error
return DETECTOR
def image_from_data_url(value: Any) -> Image.Image:
if not isinstance(value, str):
raise ValueError("lineart_input_invalid")
match = DATA_URL_PATTERN.fullmatch(value)
if not match:
raise ValueError("lineart_input_invalid")
try:
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
except Exception as error:
raise ValueError("lineart_input_invalid") from error
if not raw or len(raw) > MAX_INPUT_BYTES:
raise ValueError("lineart_input_invalid")
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
image.load()
return image
except Exception as error:
raise ValueError("lineart_input_invalid") from error
def encode_png(image: Image.Image) -> str:
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return base64.b64encode(output.getvalue()).decode("ascii")
def handle(request: dict[str, Any]) -> dict[str, Any]:
image = image_from_data_url(request.get("dataUrl"))
try:
lineart = load_detector()(image)
if not isinstance(lineart, Image.Image):
raise RuntimeError("lineart_inference_failed")
return {
"ok": True,
"requestId": request.get("requestId"),
"mimeType": "image/png",
"width": lineart.width,
"height": lineart.height,
"modelId": MODEL_ID,
"device": "cpu",
"pngBase64": encode_png(lineart.convert("RGB")),
}
except RuntimeError:
raise
except Exception as error: # pragma: no cover - depends on local runtime
raise RuntimeError("lineart_inference_failed") from error
def main() -> None:
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
for line in sys.stdin:
# PowerShell and a few Windows launchers may prepend a UTF-8 BOM to
# the first piped line. The Node runtime sends plain UTF-8, but
# accepting the marker keeps the worker protocol tolerant for manual
# diagnostics as well.
line = line.lstrip("\ufeff").strip()
if not line:
continue
request_id: Any = None
try:
request = json.loads(line)
if not isinstance(request, dict):
raise ValueError("lineart_input_invalid")
request_id = request.get("requestId")
emit(handle(request))
except ValueError as error:
emit({"ok": False, "requestId": request_id, "code": str(error) or "lineart_input_invalid", "message": "线稿转换输入无效"})
except RuntimeError as error:
code = str(error) or "lineart_inference_failed"
message = "本地 AI 线稿模型或依赖尚未安装" if code == "lineart_model_missing" else "本地 AI 线稿推理失败"
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
except Exception:
emit({"ok": False, "requestId": request_id, "code": "lineart_inference_failed", "message": "本地 AI 线稿推理失败"})
if __name__ == "__main__":
main()
+204
View File
@@ -0,0 +1,204 @@
"""Persistent CPU pose preprocessor for the local media conversion node.
The first notebook-friendly implementation uses the body-only OpenPose
preprocessor from ControlNet Aux. It produces a colored skeleton map and a
small list of normalized keypoints; it does not load a diffusion model.
"""
from __future__ import annotations
import base64
import contextlib
import io
import json
import os
import re
import sys
from typing import Any
from PIL import Image
MODEL_ID = os.environ.get("CANVAS_POSE_MODEL_ID", "lllyasviel/Annotators")
MAX_INPUT_BYTES = 12 * 1024 * 1024
DETECT_RESOLUTION = max(256, min(1024, int(os.environ.get("CANVAS_POSE_DETECT_RESOLUTION", "512"))))
DATA_URL_PATTERN = re.compile(r"^data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$")
DETECTOR: Any = None
def emit(value: dict[str, Any]) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def load_detector() -> Any:
global DETECTOR
if DETECTOR is not None:
return DETECTOR
try:
import numpy as np
from controlnet_aux.open_pose.body import Body
from huggingface_hub import hf_hub_download
cache_dir = os.environ.get("HF_HUB_CACHE") or None
with contextlib.redirect_stdout(sys.stderr):
model_path = hf_hub_download(
repo_id=MODEL_ID,
filename="body_pose_model.pth",
cache_dir=cache_dir,
local_files_only=True,
)
DETECTOR = (Body(model_path), np)
except Exception as error: # pragma: no cover - depends on local model files
raise RuntimeError("pose_model_missing") from error
return DETECTOR
def image_from_data_url(value: Any) -> Image.Image:
if not isinstance(value, str):
raise ValueError("pose_input_invalid")
match = DATA_URL_PATTERN.fullmatch(value)
if not match:
raise ValueError("pose_input_invalid")
try:
raw = base64.urlsafe_b64decode(match.group(2) + "=" * (-len(match.group(2)) % 4))
except Exception as error:
raise ValueError("pose_input_invalid") from error
if not raw or len(raw) > MAX_INPUT_BYTES:
raise ValueError("pose_input_invalid")
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
image.load()
return image
except Exception as error:
raise ValueError("pose_input_invalid") from error
def encode_png(image: Image.Image) -> str:
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return base64.b64encode(output.getvalue()).decode("ascii")
def normalize_people(bodies: list[Any], width: int, height: int) -> list[dict[str, Any]]:
people: list[dict[str, Any]] = []
for body in bodies:
keypoints = []
visible = []
for keypoint in body.keypoints:
if keypoint is None:
keypoints.append(None)
else:
normalized_x = max(0.0, min(1.0, float(keypoint.x) / float(width)))
normalized_y = max(0.0, min(1.0, float(keypoint.y) / float(height)))
keypoints.append({
"x": normalized_x,
"y": normalized_y,
"score": float(keypoint.score),
})
visible.append((normalized_x, normalized_y, float(keypoint.score)))
if visible:
xs = [point[0] for point in visible]
ys = [point[1] for point in visible]
confidence = sum(point[2] for point in visible) / len(visible)
box = {"x": min(xs), "y": min(ys), "width": max(xs) - min(xs), "height": max(ys) - min(ys)}
else:
confidence = 0.0
box = {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}
people.append({
"keypoints": keypoints,
"score": float(body.total_score),
"confidence": confidence,
"box": box,
"parts": int(body.total_parts),
})
return people
def handle(request: dict[str, Any]) -> dict[str, Any]:
image = image_from_data_url(request.get("dataUrl"))
try:
import cv2
import numpy as np
from controlnet_aux.open_pose.util import draw_bodypose
from controlnet_aux.util import HWC3, resize_image
detector, _ = load_detector()
source = HWC3(np.asarray(image, dtype=np.uint8))
detected = resize_image(source, DETECT_RESOLUTION)
candidate, subset = detector(detected[:, :, ::-1].copy())
bodies = detector.format_body_result(candidate, subset)
if not bodies:
raise RuntimeError("pose_no_person")
height, width = detected.shape[:2]
people = normalize_people(bodies, width, height)
canvas = np.zeros_like(detected, dtype=np.uint8)
for body in bodies:
normalized = [
None if keypoint is None else type(keypoint)(
x=float(keypoint.x) / float(width),
y=float(keypoint.y) / float(height),
score=keypoint.score,
id=keypoint.id,
)
for keypoint in body.keypoints
]
draw_bodypose(canvas, normalized)
output = cv2.resize(canvas, (image.width, image.height), interpolation=cv2.INTER_LINEAR)
output_image = Image.fromarray(output.astype(np.uint8), mode="RGB")
return {
"ok": True,
"requestId": request.get("requestId"),
"mimeType": "image/png",
"width": image.width,
"height": image.height,
"modelId": MODEL_ID,
"device": "cpu",
"backend": "openpose-body",
"personCount": len(bodies),
"people": people,
"pngBase64": encode_png(output_image),
}
except RuntimeError:
raise
except Exception as error: # pragma: no cover - depends on local runtime
raise RuntimeError("pose_inference_failed") from error
def main() -> None:
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
for line in sys.stdin:
line = line.lstrip("\ufeff").strip()
if not line:
continue
request_id: Any = None
try:
request = json.loads(line)
if not isinstance(request, dict):
raise ValueError("pose_input_invalid")
request_id = request.get("requestId")
emit(handle(request))
except ValueError as error:
emit({"ok": False, "requestId": request_id, "code": str(error) or "pose_input_invalid", "message": "姿态转换输入无效"})
except RuntimeError as error:
code = str(error) or "pose_inference_failed"
if code == "pose_model_missing":
message = "本地姿态模型或依赖尚未安装"
elif code == "pose_no_person":
message = "未检测到可用人物姿态"
else:
message = "本地姿态推理失败"
emit({"ok": False, "requestId": request_id, "code": code, "message": message})
except Exception:
emit({"ok": False, "requestId": request_id, "code": "pose_inference_failed", "message": "本地姿态推理失败"})
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
controlnet-aux==0.0.10
+17 -6
View File
@@ -4,7 +4,7 @@ import { CanvasSession } from "./canvas-session.js";
import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js"; import { CONFIG_DIR, type LocalRuntimeConfig } from "./config.js";
import { createLocalRuntimeApp } from "./local-runtime.js"; import { createLocalRuntimeApp } from "./local-runtime.js";
import { startLocalRuntime } from "./local-runtime-host.js"; import { startLocalRuntime } from "./local-runtime-host.js";
import { LocalRuntimeSessionManager } from "./local-runtime-session.js"; import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./local-runtime-session.js";
import { import {
createCanvasAgentHttpModule, createCanvasAgentHttpModule,
type CanvasAgentSession, type CanvasAgentSession,
@@ -14,6 +14,9 @@ import {
type DreaminaHttpModuleOptions, type DreaminaHttpModuleOptions,
} from "./modules/dreamina-http.js"; } from "./modules/dreamina-http.js";
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js"; import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">; export type CanvasAgentHttpDependencies = Pick<DreaminaHttpModuleOptions, "dreamina">;
@@ -34,21 +37,29 @@ export function createHttpApp(
dependencies: CanvasAgentHttpDependencies = {}, dependencies: CanvasAgentHttpDependencies = {},
): Express { ): Express {
const endpoint = config.url; const endpoint = config.url;
const modules = [
createCanvasAgentHttpModule(config, session),
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
createDepthEstimationHttpModule(),
createLineartEstimationHttpModule(),
createPoseEstimationHttpModule(),
];
const manager = new LocalRuntimeSessionManager({ const manager = new LocalRuntimeSessionManager({
endpoint, endpoint,
trustedOrigins: config.trustedWebOrigins, trustedOrigins: config.trustedWebOrigins,
registrations: config.browserRegistrations, registrations: config.browserRegistrations,
scopes: [...new Set([
...LOCAL_RUNTIME_DEFAULT_SCOPES,
...modules.flatMap((module) => module.descriptor.scopes),
])],
}); });
return createLocalRuntimeApp({ return createLocalRuntimeApp({
authority: new URL(endpoint).host, authority: new URL(endpoint).host,
endpoint, endpoint,
version: "0.1.0", version: "0.1.0",
sessionManager: manager, sessionManager: manager,
modules: [ modules,
createCanvasAgentHttpModule(config, session),
createDreaminaHttpModule({ ownerId: config.ownerId!, ...dependencies }),
createPortraitClearanceHttpModule({ ownerId: config.ownerId!, configDir: CONFIG_DIR }),
],
legacyMasterToken: config.token, legacyMasterToken: config.token,
legacyOrigins: config.origins ?? [], legacyOrigins: config.origins ?? [],
}); });
+8 -2
View File
@@ -6,7 +6,7 @@ export const LOCAL_RUNTIME_ENDPOINT = "http://127.0.0.1:17371";
export const LOCAL_RUNTIME_ID = "framefield-local-runtime"; export const LOCAL_RUNTIME_ID = "framefield-local-runtime";
export const LOCAL_RUNTIME_API_VERSION = 2; export const LOCAL_RUNTIME_API_VERSION = 2;
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance"; export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
export type LocalRuntimeScope = export type LocalRuntimeScope =
| "runtime:status" | "runtime:status"
| "runtime:revoke" | "runtime:revoke"
@@ -20,7 +20,13 @@ export type LocalRuntimeScope =
| "portrait:status" | "portrait:status"
| "portrait:model" | "portrait:model"
| "portrait:run" | "portrait:run"
| "portrait:read"; | "portrait:read"
| "depth:status"
| "depth:run"
| "lineart:status"
| "lineart:run"
| "pose:status"
| "pose:run";
export type DreaminaModelOperation = export type DreaminaModelOperation =
| "text-to-image" | "text-to-image"
+6
View File
@@ -17,6 +17,9 @@ import { LOCAL_RUNTIME_DEFAULT_SCOPES, LocalRuntimeSessionManager } from "./loca
import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js"; import { createCanvasAgentHttpModule } from "./modules/canvas-agent-http.js";
import { createDreaminaHttpModule } from "./modules/dreamina-http.js"; import { createDreaminaHttpModule } from "./modules/dreamina-http.js";
import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js"; import { createPortraitClearanceHttpModule } from "./modules/portrait-clearance-http.js";
import { createDepthEstimationHttpModule } from "./modules/depth-estimation-http.js";
import { createLineartEstimationHttpModule } from "./modules/lineart-estimation-http.js";
import { createPoseEstimationHttpModule } from "./modules/pose-estimation-http.js";
export type StartLocalRuntimeOptions = { export type StartLocalRuntimeOptions = {
config?: LocalRuntimeConfig; config?: LocalRuntimeConfig;
@@ -46,6 +49,9 @@ export function createDefaultLocalRuntimeModules(config: LocalRuntimeConfig): Lo
], ],
}), }),
createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }), createPortraitClearanceHttpModule({ ownerId: ensureRuntimeOwnerId(config), configDir: CONFIG_DIR }),
createDepthEstimationHttpModule(),
createLineartEstimationHttpModule(),
createPoseEstimationHttpModule(),
]; ];
} }
+1 -1
View File
@@ -185,7 +185,7 @@ function validateModules(modules: readonly LocalRuntimeModule[]) {
const ids = new Set<LocalRuntimeModuleId>(); const ids = new Set<LocalRuntimeModuleId>();
const routes = new Set<string>(); const routes = new Set<string>();
for (const module of modules) { for (const module of modules) {
if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance") { if (module.descriptor.id !== "canvas-agent" && module.descriptor.id !== "dreamina" && module.descriptor.id !== "portrait-clearance" && module.descriptor.id !== "depth-estimation" && module.descriptor.id !== "lineart-estimation" && module.descriptor.id !== "pose-estimation") {
throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`); throw new Error(`Unsupported Local Runtime module id: ${module.descriptor.id}`);
} }
if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`); if (ids.has(module.descriptor.id)) throw new Error(`Duplicate Local Runtime module id: ${module.descriptor.id}`);
@@ -0,0 +1,327 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { Request, RequestHandler, Response } from "express";
import type { LocalRuntimeModule } from "../local-runtime.js";
const DEFAULT_MODEL_ID = "depth-anything/Depth-Anything-V2-Small-hf";
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type DepthRuntimePaths = {
pythonPath: string;
scriptPath: string;
hfHome: string;
modelId: string;
configured: boolean;
modelCached: boolean;
reason?: "python_missing" | "worker_missing";
};
export type DepthRuntimeStatus = {
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
type DepthWorkerSuccess = {
ok: true;
requestId: string;
mimeType: "image/png";
width: number;
height: number;
modelId: string;
device: "cpu";
pngBase64: string;
};
export class DepthRuntimeError extends Error {
constructor(readonly code: string, message: string, readonly statusCode = 503) {
super(message);
this.name = "DepthRuntimeError";
}
}
export function createDepthEstimationHttpModule(): LocalRuntimeModule {
const worker = new DepthEstimationWorker();
return {
descriptor: {
id: "depth-estimation",
displayName: "Depth Anything V2 本机深度",
apiVersion: 1,
scopes: ["depth:status", "depth:run"],
},
routes: [
{
method: "GET",
path: "/depth-estimation/status",
scope: "depth:status",
handler: asyncRoute(async (_request, response) => {
const status = worker.status();
response.json({
ok: true,
module: "depth-estimation",
apiVersion: 1,
ready: status.configured && status.modelCached,
...status,
});
}),
},
{
method: "POST",
path: "/depth-estimation/run",
scope: "depth:run",
handler: asyncRoute(async (request, response) => {
const body = parseBody(request);
const dataUrl = parseDepthInput(body);
const result = await worker.run({ dataUrl });
response.json({
ok: true,
module: "depth-estimation",
apiVersion: 1,
result,
});
}),
},
],
dispose: () => worker.dispose(),
publicHealth: () => {
const status = worker.status();
return { depthEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
},
};
}
export function resolveDepthRuntimePaths(env: NodeJS.ProcessEnv = process.env): DepthRuntimePaths {
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
const projectRoots = [
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
path.resolve(moduleRoot, ".."),
path.resolve(process.cwd()),
].filter((value): value is string => Boolean(value));
const projectRoot = configuredProjectRoot
? path.resolve(configuredProjectRoot)
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
const scriptCandidates = [
path.join(moduleRoot, "python", "depth_anything_runtime.py"),
path.join(projectRoot, "canvas-agent", "python", "depth_anything_runtime.py"),
];
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
: process.platform === "win32"
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
const pythonPath = pythonCandidates[0]!;
const hfHome = path.resolve(env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
const modelId = env.CANVAS_DEPTH_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
const workerExists = fs.existsSync(scriptPath);
const pythonExists = fs.existsSync(pythonPath);
return {
pythonPath,
scriptPath,
hfHome,
modelId,
configured: workerExists && pythonExists,
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
...(pythonExists ? {} : { reason: "python_missing" as const }),
...(!workerExists ? { reason: "worker_missing" as const } : {}),
};
}
export class DepthEstimationWorker {
private paths = resolveDepthRuntimePaths();
private child?: ChildProcessWithoutNullStreams;
private stdoutBuffer = "";
private stderrBuffer = "";
private requestCounter = 0;
private loaded = false;
private readonly pending = new Map<string, { resolve: (value: DepthWorkerSuccess) => void; reject: (error: unknown) => void }>();
status(): DepthRuntimeStatus {
const paths = this.refreshPaths();
return {
configured: paths.configured,
modelCached: paths.modelCached,
loaded: this.loaded,
device: "cpu",
modelId: paths.modelId,
...(paths.reason ? { reason: paths.reason } : {}),
};
}
async run(input: { dataUrl: string }) {
validateDepthDataUrl(input.dataUrl);
const paths = this.refreshPaths();
if (!paths.configured) {
throw new DepthRuntimeError(
"depth_runtime_unavailable",
paths.reason === "worker_missing" ? "本机深度运行文件不存在" : "未找到本机深度 Python 环境",
);
}
const child = this.ensureChild();
const requestId = `depth-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
const response = new Promise<DepthWorkerSuccess>((resolve, reject) => {
this.pending.set(requestId, { resolve, reject });
});
try {
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
} catch (error) {
this.pending.delete(requestId);
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法写入");
}
return response;
}
dispose() {
const child = this.child;
this.child = undefined;
this.loaded = false;
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已停止"));
if (child && !child.killed) child.kill();
}
private ensureChild() {
if (this.child && !this.child.killed) return this.child;
let child: ChildProcessWithoutNullStreams;
try {
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
cwd: path.dirname(this.paths.scriptPath),
env: {
...process.env,
HF_HOME: this.paths.hfHome,
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
HF_HUB_OFFLINE: "1",
TRANSFORMERS_OFFLINE: "1",
HF_HUB_DISABLE_TELEMETRY: "1",
HF_HUB_DISABLE_PROGRESS_BARS: "1",
TRANSFORMERS_VERBOSITY: "error",
CANVAS_DEPTH_MODEL_ID: this.paths.modelId,
},
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
} catch (error) {
throw new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程无法启动");
}
this.child = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
child.stderr.on("data", (chunk: string) => {
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
});
child.on("error", () => this.handleChildExit(child));
child.on("exit", () => this.handleChildExit(child));
return child;
}
private refreshPaths() {
if (!this.child) this.paths = resolveDepthRuntimePaths();
return this.paths;
}
private consumeStdout(chunk: string) {
this.stdoutBuffer += chunk;
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
this.child?.kill();
this.handleChildExit(this.child);
return;
}
while (true) {
const newline = this.stdoutBuffer.indexOf("\n");
if (newline < 0) return;
const line = this.stdoutBuffer.slice(0, newline).trim();
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
if (!line) continue;
let value: unknown;
try { value = JSON.parse(line); } catch { continue; }
if (!isRecord(value) || typeof value.requestId !== "string") continue;
const pending = this.pending.get(value.requestId);
if (!pending) continue;
this.pending.delete(value.requestId);
if (value.ok === true) {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
pending.reject(new DepthRuntimeError("depth_response_invalid", "本机深度响应无效", 500));
continue;
}
this.loaded = true;
pending.resolve(value as unknown as DepthWorkerSuccess);
continue;
}
const code = typeof value.code === "string" ? value.code : "depth_inference_failed";
const statusCode = code === "depth_model_missing" ? 503 : code === "depth_input_invalid" ? 400 : 500;
pending.reject(new DepthRuntimeError(code, typeof value.message === "string" ? value.message : "本机深度推理失败", statusCode));
}
}
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
if (!child || this.child !== child) return;
this.child = undefined;
this.loaded = false;
this.stderrBuffer = "";
this.stdoutBuffer = "";
this.rejectPending(new DepthRuntimeError("depth_runtime_unavailable", "本机深度进程已退出"));
}
private rejectPending(error: DepthRuntimeError) {
for (const { reject } of this.pending.values()) reject(error);
this.pending.clear();
}
}
export function validateDepthDataUrl(value: string) {
const match = DATA_URL_PATTERN.exec(value);
if (!match) throw new DepthRuntimeError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = Buffer.from(match[2]!, "base64");
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new DepthRuntimeError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
}
function parseDepthInput(body: Record<string, unknown>) {
const keys = Object.keys(body).sort();
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "depth" || typeof body.dataUrl !== "string") {
throw new DepthRuntimeError("depth_input_invalid", "深度转换请求字段无效", 400);
}
validateDepthDataUrl(body.dataUrl);
return body.dataUrl;
}
function parseBody(request: Request): Record<string, unknown> {
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
try {
const value = JSON.parse(request.body.toString("utf8")) as unknown;
return isRecord(value) ? value : {};
} catch {
return {};
}
}
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
return (request, response, next) => {
void action(request, response).catch((error) => {
if (response.headersSent) return next(error);
if (error instanceof DepthRuntimeError) {
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
return;
}
response.status(503).json({ ok: false, code: "depth_runtime_unavailable", message: "本机深度运行时不可用" });
});
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}
@@ -0,0 +1,327 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { Request, RequestHandler, Response } from "express";
import type { LocalRuntimeModule } from "../local-runtime.js";
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LineartRuntimePaths = {
pythonPath: string;
scriptPath: string;
hfHome: string;
modelId: string;
configured: boolean;
modelCached: boolean;
reason?: "python_missing" | "worker_missing";
};
export type LineartRuntimeStatus = {
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
type LineartWorkerSuccess = {
ok: true;
requestId: string;
mimeType: "image/png";
width: number;
height: number;
modelId: string;
device: "cpu";
pngBase64: string;
};
export class LineartRuntimeError extends Error {
constructor(readonly code: string, message: string, readonly statusCode = 503) {
super(message);
this.name = "LineartRuntimeError";
}
}
export function createLineartEstimationHttpModule(): LocalRuntimeModule {
const worker = new LineartEstimationWorker();
return {
descriptor: {
id: "lineart-estimation",
displayName: "ControlNet Aux 本机 AI 线稿",
apiVersion: 1,
scopes: ["lineart:status", "lineart:run"],
},
routes: [
{
method: "GET",
path: "/lineart-estimation/status",
scope: "lineart:status",
handler: asyncRoute(async (_request, response) => {
const status = worker.status();
response.json({
ok: true,
module: "lineart-estimation",
apiVersion: 1,
ready: status.configured && status.modelCached,
...status,
});
}),
},
{
method: "POST",
path: "/lineart-estimation/run",
scope: "lineart:run",
handler: asyncRoute(async (request, response) => {
const body = parseBody(request);
const dataUrl = parseLineartInput(body);
const result = await worker.run({ dataUrl });
response.json({
ok: true,
module: "lineart-estimation",
apiVersion: 1,
result,
});
}),
},
],
dispose: () => worker.dispose(),
publicHealth: () => {
const status = worker.status();
return { lineartEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
},
};
}
export function resolveLineartRuntimePaths(env: NodeJS.ProcessEnv = process.env): LineartRuntimePaths {
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
const projectRoots = [
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
path.resolve(moduleRoot, ".."),
path.resolve(process.cwd()),
].filter((value): value is string => Boolean(value));
const projectRoot = configuredProjectRoot
? path.resolve(configuredProjectRoot)
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
const scriptCandidates = [
path.join(moduleRoot, "python", "lineart_runtime.py"),
path.join(projectRoot, "canvas-agent", "python", "lineart_runtime.py"),
];
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
const pythonCandidates = env.CANVAS_DEPTH_PYTHON?.trim()
? [path.resolve(env.CANVAS_DEPTH_PYTHON.trim())]
: process.platform === "win32"
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
const pythonPath = pythonCandidates[0]!;
const hfHome = path.resolve(env.CANVAS_LINEART_HF_HOME?.trim() || env.CANVAS_DEPTH_HF_HOME?.trim() || path.join(projectRoot, ".local", "cache", "huggingface"));
const modelId = env.CANVAS_LINEART_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
const workerExists = fs.existsSync(scriptPath);
const pythonExists = fs.existsSync(pythonPath);
return {
pythonPath,
scriptPath,
hfHome,
modelId,
configured: workerExists && pythonExists,
modelCached: workerExists && pythonExists && fs.existsSync(modelCacheDir),
...(pythonExists ? {} : { reason: "python_missing" as const }),
...(!workerExists ? { reason: "worker_missing" as const } : {}),
};
}
export class LineartEstimationWorker {
private paths = resolveLineartRuntimePaths();
private child?: ChildProcessWithoutNullStreams;
private stdoutBuffer = "";
private stderrBuffer = "";
private requestCounter = 0;
private loaded = false;
private readonly pending = new Map<string, { resolve: (value: LineartWorkerSuccess) => void; reject: (error: unknown) => void }>();
status(): LineartRuntimeStatus {
const paths = this.refreshPaths();
return {
configured: paths.configured,
modelCached: paths.modelCached,
loaded: this.loaded,
device: "cpu",
modelId: paths.modelId,
...(paths.reason ? { reason: paths.reason } : {}),
};
}
async run(input: { dataUrl: string }) {
validateLineartDataUrl(input.dataUrl);
const paths = this.refreshPaths();
if (!paths.configured) {
throw new LineartRuntimeError(
"lineart_runtime_unavailable",
paths.reason === "worker_missing" ? "本机 AI 线稿运行文件不存在" : "未找到本机 AI 线稿 Python 环境",
);
}
const child = this.ensureChild();
const requestId = `lineart-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
const response = new Promise<LineartWorkerSuccess>((resolve, reject) => {
this.pending.set(requestId, { resolve, reject });
});
try {
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
} catch {
this.pending.delete(requestId);
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法写入");
}
return response;
}
dispose() {
const child = this.child;
this.child = undefined;
this.loaded = false;
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已停止"));
if (child && !child.killed) child.kill();
}
private ensureChild() {
if (this.child && !this.child.killed) return this.child;
let child: ChildProcessWithoutNullStreams;
try {
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
cwd: path.dirname(this.paths.scriptPath),
env: {
...process.env,
HF_HOME: this.paths.hfHome,
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
HF_HUB_OFFLINE: "1",
TRANSFORMERS_OFFLINE: "1",
HF_HUB_DISABLE_TELEMETRY: "1",
HF_HUB_DISABLE_PROGRESS_BARS: "1",
TRANSFORMERS_VERBOSITY: "error",
CANVAS_LINEART_MODEL_ID: this.paths.modelId,
},
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
} catch {
throw new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程无法启动");
}
this.child = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
child.stderr.on("data", (chunk: string) => {
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
});
child.on("error", () => this.handleChildExit(child));
child.on("exit", () => this.handleChildExit(child));
return child;
}
private refreshPaths() {
if (!this.child) this.paths = resolveLineartRuntimePaths();
return this.paths;
}
private consumeStdout(chunk: string) {
this.stdoutBuffer += chunk;
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
this.child?.kill();
this.handleChildExit(this.child);
return;
}
while (true) {
const newline = this.stdoutBuffer.indexOf("\n");
if (newline < 0) return;
const line = this.stdoutBuffer.slice(0, newline).trim();
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
if (!line) continue;
let value: unknown;
try { value = JSON.parse(line); } catch { continue; }
if (!isRecord(value) || typeof value.requestId !== "string") continue;
const pending = this.pending.get(value.requestId);
if (!pending) continue;
this.pending.delete(value.requestId);
if (value.ok === true) {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string" || !value.pngBase64) {
pending.reject(new LineartRuntimeError("lineart_response_invalid", "本机 AI 线稿响应无效", 500));
continue;
}
this.loaded = true;
pending.resolve(value as unknown as LineartWorkerSuccess);
continue;
}
const code = typeof value.code === "string" ? value.code : "lineart_inference_failed";
const statusCode = code === "lineart_model_missing" ? 503 : code === "lineart_input_invalid" ? 400 : 500;
pending.reject(new LineartRuntimeError(code, typeof value.message === "string" ? value.message : "本机 AI 线稿推理失败", statusCode));
}
}
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
if (!child || this.child !== child) return;
this.child = undefined;
this.loaded = false;
this.stderrBuffer = "";
this.stdoutBuffer = "";
this.rejectPending(new LineartRuntimeError("lineart_runtime_unavailable", "本机 AI 线稿进程已退出"));
}
private rejectPending(error: LineartRuntimeError) {
for (const { reject } of this.pending.values()) reject(error);
this.pending.clear();
}
}
export function validateLineartDataUrl(value: string) {
const match = DATA_URL_PATTERN.exec(value);
if (!match) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = Buffer.from(match[2]!, "base64");
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
}
function parseLineartInput(body: Record<string, unknown>) {
const keys = Object.keys(body).sort();
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "lineart" || typeof body.dataUrl !== "string") {
throw new LineartRuntimeError("lineart_input_invalid", "AI 线稿转换请求字段无效", 400);
}
validateLineartDataUrl(body.dataUrl);
return body.dataUrl;
}
function parseBody(request: Request): Record<string, unknown> {
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
try {
const value = JSON.parse(request.body.toString("utf8")) as unknown;
return isRecord(value) ? value : {};
} catch {
return {};
}
}
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
return (request, response, next) => {
void action(request, response).catch((error) => {
if (response.headersSent) return next(error);
if (error instanceof LineartRuntimeError) {
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
return;
}
response.status(503).json({ ok: false, code: "lineart_runtime_unavailable", message: "本机 AI 线稿运行时不可用" });
});
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}
@@ -0,0 +1,357 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { Request, RequestHandler, Response } from "express";
import type { LocalRuntimeModule } from "../local-runtime.js";
const DEFAULT_MODEL_ID = "lllyasviel/Annotators";
const DEFAULT_BACKEND = "openpose-body" as const;
const MAX_INPUT_BYTES = 12 * 1024 * 1024;
const MAX_PROTOCOL_LINE_BYTES = 48 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type PoseRuntimePaths = {
pythonPath: string;
scriptPath: string;
hfHome: string;
modelId: string;
configured: boolean;
modelCached: boolean;
reason?: "python_missing" | "worker_missing";
};
export type PoseRuntimeStatus = {
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
backend: typeof DEFAULT_BACKEND;
reason?: string;
};
type PoseWorkerSuccess = {
ok: true;
requestId: string;
mimeType: "image/png";
width: number;
height: number;
modelId: string;
device: "cpu";
backend: typeof DEFAULT_BACKEND;
personCount: number;
people?: unknown[];
pngBase64: string;
};
export class PoseRuntimeError extends Error {
constructor(readonly code: string, message: string, readonly statusCode = 503) {
super(message);
this.name = "PoseRuntimeError";
}
}
export function createPoseEstimationHttpModule(): LocalRuntimeModule {
const worker = new PoseEstimationWorker();
return {
descriptor: {
id: "pose-estimation",
displayName: "OpenPose 本机姿态骨架",
apiVersion: 1,
scopes: ["pose:status", "pose:run"],
},
routes: [
{
method: "GET",
path: "/pose-estimation/status",
scope: "pose:status",
handler: asyncRoute(async (_request, response) => {
const status = worker.status();
response.json({
ok: true,
module: "pose-estimation",
apiVersion: 1,
ready: status.configured && status.modelCached,
...status,
});
}),
},
{
method: "POST",
path: "/pose-estimation/run",
scope: "pose:run",
handler: asyncRoute(async (request, response) => {
const body = parseBody(request);
const dataUrl = parsePoseInput(body);
const result = await worker.run({ dataUrl });
response.json({
ok: true,
module: "pose-estimation",
apiVersion: 1,
result,
});
}),
},
],
dispose: () => worker.dispose(),
publicHealth: () => {
const status = worker.status();
return { poseEstimation: status.configured && status.modelCached ? "available" : "unavailable" };
},
};
}
export function resolvePoseRuntimePaths(env: NodeJS.ProcessEnv = process.env): PoseRuntimePaths {
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const configuredProjectRoot = env.CANVAS_PROJECT_ROOT?.trim();
const projectRoots = [
configuredProjectRoot ? path.resolve(configuredProjectRoot) : undefined,
path.resolve(moduleRoot, ".."),
path.resolve(process.cwd()),
].filter((value): value is string => Boolean(value));
const projectRoot = configuredProjectRoot
? path.resolve(configuredProjectRoot)
: projectRoots.find((value) => fs.existsSync(path.join(value, ".local"))) || projectRoots[0]!;
const scriptCandidates = [
path.join(moduleRoot, "python", "pose_runtime.py"),
path.join(projectRoot, "canvas-agent", "python", "pose_runtime.py"),
];
const scriptPath = scriptCandidates.find((value) => fs.existsSync(value)) || scriptCandidates[0]!;
const pythonCandidates = env.CANVAS_POSE_PYTHON?.trim()
? [path.resolve(env.CANVAS_POSE_PYTHON.trim())]
: process.platform === "win32"
? [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "Scripts", "python.exe")]
: [path.join(projectRoot, ".local", "depth-anything-v2", "venv", "bin", "python")];
const pythonPath = pythonCandidates[0]!;
const hfHome = path.resolve(
env.CANVAS_POSE_HF_HOME?.trim()
|| env.CANVAS_LINEART_HF_HOME?.trim()
|| env.CANVAS_DEPTH_HF_HOME?.trim()
|| path.join(projectRoot, ".local", "cache", "huggingface"),
);
const modelId = env.CANVAS_POSE_MODEL_ID?.trim() || DEFAULT_MODEL_ID;
const modelCacheDir = path.join(hfHome, "hub", `models--${modelId.replaceAll("/", "--")}`);
const workerExists = fs.existsSync(scriptPath);
const pythonExists = fs.existsSync(pythonPath);
return {
pythonPath,
scriptPath,
hfHome,
modelId,
configured: workerExists && pythonExists,
modelCached: workerExists && pythonExists && findCachedModelFile(modelCacheDir, "body_pose_model.pth"),
...(pythonExists ? {} : { reason: "python_missing" as const }),
...(!workerExists ? { reason: "worker_missing" as const } : {}),
};
}
export function findCachedModelFile(modelCacheDir: string, filename: string) {
const snapshotsDir = path.join(modelCacheDir, "snapshots");
try {
return fs.readdirSync(snapshotsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.some((entry) => fs.existsSync(path.join(snapshotsDir, entry.name, filename)));
} catch {
return false;
}
}
export class PoseEstimationWorker {
private paths = resolvePoseRuntimePaths();
private child?: ChildProcessWithoutNullStreams;
private stdoutBuffer = "";
private stderrBuffer = "";
private requestCounter = 0;
private loaded = false;
private readonly pending = new Map<string, { resolve: (value: PoseWorkerSuccess) => void; reject: (error: unknown) => void }>();
status(): PoseRuntimeStatus {
const paths = this.refreshPaths();
return {
configured: paths.configured,
modelCached: paths.modelCached,
loaded: this.loaded,
device: "cpu",
modelId: paths.modelId,
backend: DEFAULT_BACKEND,
...(paths.reason ? { reason: paths.reason } : {}),
};
}
async run(input: { dataUrl: string }) {
validatePoseDataUrl(input.dataUrl);
const paths = this.refreshPaths();
if (!paths.configured) {
throw new PoseRuntimeError(
"pose_runtime_unavailable",
paths.reason === "worker_missing" ? "本机姿态运行文件不存在" : "未找到本机姿态 Python 环境",
);
}
const child = this.ensureChild();
const requestId = `pose-${Date.now().toString(36)}-${(++this.requestCounter).toString(36)}`;
const response = new Promise<PoseWorkerSuccess>((resolve, reject) => {
this.pending.set(requestId, { resolve, reject });
});
try {
child.stdin.write(JSON.stringify({ requestId, dataUrl: input.dataUrl }) + "\n");
} catch {
this.pending.delete(requestId);
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法写入");
}
return response;
}
dispose() {
const child = this.child;
this.child = undefined;
this.loaded = false;
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已停止"));
if (child && !child.killed) child.kill();
}
private ensureChild() {
if (this.child && !this.child.killed) return this.child;
let child: ChildProcessWithoutNullStreams;
try {
child = spawn(this.paths.pythonPath, [this.paths.scriptPath], {
cwd: path.dirname(this.paths.scriptPath),
env: {
...process.env,
HF_HOME: this.paths.hfHome,
HF_HUB_CACHE: path.join(this.paths.hfHome, "hub"),
HF_HUB_OFFLINE: "1",
TRANSFORMERS_OFFLINE: "1",
HF_HUB_DISABLE_TELEMETRY: "1",
HF_HUB_DISABLE_PROGRESS_BARS: "1",
TRANSFORMERS_VERBOSITY: "error",
CANVAS_POSE_MODEL_ID: this.paths.modelId,
},
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
} catch {
throw new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程无法启动");
}
this.child = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => this.consumeStdout(chunk));
child.stderr.on("data", (chunk: string) => {
this.stderrBuffer = `${this.stderrBuffer}${chunk}`.slice(-4_000);
});
child.on("error", () => this.handleChildExit(child));
child.on("exit", () => this.handleChildExit(child));
return child;
}
private refreshPaths() {
if (!this.child) this.paths = resolvePoseRuntimePaths();
return this.paths;
}
private consumeStdout(chunk: string) {
this.stdoutBuffer += chunk;
if (Buffer.byteLength(this.stdoutBuffer, "utf8") > MAX_PROTOCOL_LINE_BYTES) {
this.child?.kill();
this.handleChildExit(this.child);
return;
}
while (true) {
const newline = this.stdoutBuffer.indexOf("\n");
if (newline < 0) return;
const line = this.stdoutBuffer.slice(0, newline).trim();
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
if (!line) continue;
let value: unknown;
try { value = JSON.parse(line); } catch { continue; }
if (!isRecord(value) || typeof value.requestId !== "string") continue;
const pending = this.pending.get(value.requestId);
if (!pending) continue;
this.pending.delete(value.requestId);
if (value.ok === true) {
if (value.mimeType !== "image/png"
|| !isPositiveInteger(value.width)
|| !isPositiveInteger(value.height)
|| typeof value.modelId !== "string"
|| value.device !== "cpu"
|| value.backend !== DEFAULT_BACKEND
|| !isPositiveInteger(value.personCount)
|| typeof value.pngBase64 !== "string"
|| !value.pngBase64) {
pending.reject(new PoseRuntimeError("pose_response_invalid", "本机姿态响应无效", 500));
continue;
}
this.loaded = true;
pending.resolve(value as unknown as PoseWorkerSuccess);
continue;
}
const code = typeof value.code === "string" ? value.code : "pose_inference_failed";
const statusCode = code === "pose_model_missing" ? 503 : code === "pose_input_invalid" ? 400 : code === "pose_no_person" ? 422 : 500;
pending.reject(new PoseRuntimeError(code, typeof value.message === "string" ? value.message : "本机姿态推理失败", statusCode));
}
}
private handleChildExit(child?: ChildProcessWithoutNullStreams) {
if (!child || this.child !== child) return;
this.child = undefined;
this.loaded = false;
this.stderrBuffer = "";
this.stdoutBuffer = "";
this.rejectPending(new PoseRuntimeError("pose_runtime_unavailable", "本机姿态进程已退出"));
}
private rejectPending(error: PoseRuntimeError) {
for (const { reject } of this.pending.values()) reject(error);
this.pending.clear();
}
}
export function validatePoseDataUrl(value: string) {
const match = DATA_URL_PATTERN.exec(value);
if (!match) throw new PoseRuntimeError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = Buffer.from(match[2]!, "base64");
if (!bytes.length || bytes.length > MAX_INPUT_BYTES) throw new PoseRuntimeError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
}
function parsePoseInput(body: Record<string, unknown>) {
const keys = Object.keys(body).sort();
if (keys.length !== 3 || keys[0] !== "dataUrl" || keys[1] !== "operation" || keys[2] !== "schemaVersion" || body.schemaVersion !== 1 || body.operation !== "pose" || typeof body.dataUrl !== "string") {
throw new PoseRuntimeError("pose_input_invalid", "姿态转换请求字段无效", 400);
}
validatePoseDataUrl(body.dataUrl);
return body.dataUrl;
}
function parseBody(request: Request): Record<string, unknown> {
if (!Buffer.isBuffer(request.body) || request.body.length === 0) return {};
try {
const value = JSON.parse(request.body.toString("utf8")) as unknown;
return isRecord(value) ? value : {};
} catch {
return {};
}
}
function asyncRoute(action: (request: Request, response: Response) => Promise<void>): RequestHandler {
return (request, response, next) => {
void action(request, response).catch((error) => {
if (response.headersSent) return next(error);
if (error instanceof PoseRuntimeError) {
response.status(error.statusCode).json({ ok: false, code: error.code, message: error.message });
return;
}
response.status(503).json({ ok: false, code: "pose_runtime_unavailable", message: "本机姿态运行时不可用" });
});
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}
@@ -0,0 +1,36 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createDepthEstimationHttpModule, DepthRuntimeError, resolveDepthRuntimePaths, validateDepthDataUrl } from "../src/modules/depth-estimation-http.js";
test("depth runtime module declares signed status and run routes", () => {
const module = createDepthEstimationHttpModule();
assert.equal(module.descriptor.id, "depth-estimation");
assert.deepEqual(module.descriptor.scopes, ["depth:status", "depth:run"]);
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
"GET /depth-estimation/status",
"POST /depth-estimation/run",
]);
module.dispose?.();
});
test("depth input guard accepts image data URLs and rejects unsafe input", () => {
assert.doesNotThrow(() => validateDepthDataUrl("data:image/png;base64,AQID"));
assert.throws(
() => validateDepthDataUrl("data:text/plain;base64,AQID"),
(error: unknown) => error instanceof DepthRuntimeError && error.code === "depth_input_invalid" && error.statusCode === 400,
);
});
test("depth runtime paths honor explicit project and Python overrides", () => {
const paths = resolveDepthRuntimePaths({
CANVAS_PROJECT_ROOT: "C:\\depth-project",
CANVAS_DEPTH_PYTHON: "C:\\depth-project\\venv\\python.exe",
CANVAS_DEPTH_HF_HOME: "C:\\depth-cache",
CANVAS_DEPTH_MODEL_ID: "example/depth-model",
});
assert.equal(paths.pythonPath, "C:\\depth-project\\venv\\python.exe");
assert.equal(paths.hfHome, "C:\\depth-cache");
assert.equal(paths.modelId, "example/depth-model");
assert.equal(paths.configured, false);
});
@@ -0,0 +1,36 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createLineartEstimationHttpModule, LineartRuntimeError, resolveLineartRuntimePaths, validateLineartDataUrl } from "../src/modules/lineart-estimation-http.js";
test("lineart runtime module declares signed status and run routes", () => {
const module = createLineartEstimationHttpModule();
assert.equal(module.descriptor.id, "lineart-estimation");
assert.deepEqual(module.descriptor.scopes, ["lineart:status", "lineart:run"]);
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
"GET /lineart-estimation/status",
"POST /lineart-estimation/run",
]);
module.dispose?.();
});
test("lineart input guard accepts image data URLs and rejects unsafe input", () => {
assert.doesNotThrow(() => validateLineartDataUrl("data:image/png;base64,AQID"));
assert.throws(
() => validateLineartDataUrl("data:text/plain;base64,AQID"),
(error: unknown) => error instanceof LineartRuntimeError && error.code === "lineart_input_invalid" && error.statusCode === 400,
);
});
test("lineart runtime paths honor explicit project and Python overrides", () => {
const paths = resolveLineartRuntimePaths({
CANVAS_PROJECT_ROOT: "C:\\lineart-project",
CANVAS_DEPTH_PYTHON: "C:\\lineart-project\\venv\\python.exe",
CANVAS_LINEART_HF_HOME: "C:\\lineart-cache",
CANVAS_LINEART_MODEL_ID: "example/lineart-model",
});
assert.equal(paths.pythonPath, "C:\\lineart-project\\venv\\python.exe");
assert.equal(paths.hfHome, "C:\\lineart-cache");
assert.equal(paths.modelId, "example/lineart-model");
assert.equal(paths.configured, false);
});
+37
View File
@@ -0,0 +1,37 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createPoseEstimationHttpModule, PoseRuntimeError, resolvePoseRuntimePaths, validatePoseDataUrl } from "../src/modules/pose-estimation-http.js";
test("pose runtime module declares signed status and run routes", () => {
const module = createPoseEstimationHttpModule();
assert.equal(module.descriptor.id, "pose-estimation");
assert.deepEqual(module.descriptor.scopes, ["pose:status", "pose:run"]);
assert.deepEqual(module.routes.map((route) => `${route.method} ${route.path}`), [
"GET /pose-estimation/status",
"POST /pose-estimation/run",
]);
module.dispose?.();
});
test("pose input guard accepts image data URLs and rejects unsafe input", () => {
assert.doesNotThrow(() => validatePoseDataUrl("data:image/png;base64,AQID"));
assert.throws(
() => validatePoseDataUrl("data:text/plain;base64,AQID"),
(error: unknown) => error instanceof PoseRuntimeError && error.code === "pose_input_invalid" && error.statusCode === 400,
);
});
test("pose runtime paths honor explicit project and Python overrides", () => {
const paths = resolvePoseRuntimePaths({
CANVAS_PROJECT_ROOT: "C:\\pose-project",
CANVAS_POSE_PYTHON: "C:\\pose-project\\venv\\python.exe",
CANVAS_POSE_HF_HOME: "C:\\pose-cache",
CANVAS_POSE_MODEL_ID: "example/pose-model",
});
assert.equal(paths.pythonPath, "C:\\pose-project\\venv\\python.exe");
assert.equal(paths.hfHome, "C:\\pose-cache");
assert.equal(paths.modelId, "example/pose-model");
assert.equal(paths.configured, false);
assert.equal(paths.modelCached, false);
});
@@ -373,6 +373,7 @@ function nodeTypeLabel(node?: CanvasNodeData | null) {
if (node.type === CanvasNodeType.Video) return "视频节点"; if (node.type === CanvasNodeType.Video) return "视频节点";
if (node.type === CanvasNodeType.Audio) return "音频节点"; if (node.type === CanvasNodeType.Audio) return "音频节点";
if (node.type === CanvasNodeType.Drawing) return "绘图节点"; if (node.type === CanvasNodeType.Drawing) return "绘图节点";
if (node.type === CanvasNodeType.MediaConversion) return "转换节点";
if (node.type === CanvasNodeType.Frame) return isCanvasFolderNode(node) ? "文件夹" : "背板"; if (node.type === CanvasNodeType.Frame) return isCanvasFolderNode(node) ? "文件夹" : "背板";
return "生成配置节点"; return "生成配置节点";
} }
@@ -37,6 +37,8 @@ import { PanoramaNodeContent } from "./nodes/panorama-node";
import { SvgNodeContent } from "./nodes/svg-node"; import { SvgNodeContent } from "./nodes/svg-node";
import { PortraitClearanceNodeContent } from "./nodes/portrait-clearance-node"; import { PortraitClearanceNodeContent } from "./nodes/portrait-clearance-node";
import { ArtCritiqueNodeContent } from "./nodes/ai-art-critique-node"; import { ArtCritiqueNodeContent } from "./nodes/ai-art-critique-node";
import { MediaConversionNodeContent } from "./nodes/media-conversion-node";
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
export type CanvasNodeContentProps = { export type CanvasNodeContentProps = {
node: CanvasNodeData; node: CanvasNodeData;
@@ -74,6 +76,7 @@ export function CanvasNodeContent(props: CanvasNodeContentProps) {
if (hasCustomContent && props.renderNodeContent) return props.renderNodeContent(props.node); if (hasCustomContent && props.renderNodeContent) return props.renderNodeContent(props.node);
if (props.node.type === PORTRAIT_CLEARANCE_NODE_TYPE) return <PortraitClearanceNodeContent node={props.node} />; if (props.node.type === PORTRAIT_CLEARANCE_NODE_TYPE) return <PortraitClearanceNodeContent node={props.node} />;
if (props.node.type === ART_CRITIQUE_NODE_TYPE) return <ArtCritiqueNodeContent node={props.node} />; if (props.node.type === ART_CRITIQUE_NODE_TYPE) return <ArtCritiqueNodeContent node={props.node} />;
if (props.node.type === MEDIA_CONVERSION_NODE_TYPE) return <MediaConversionNodeContent node={props.node} theme={props.theme} />;
if (props.isBatchRoot) return <ImageNodeContent {...props} />; if (props.isBatchRoot) return <ImageNodeContent {...props} />;
if (props.node.metadata?.status === "loading") return <LoadingContent node={props.node} theme={props.theme} onOpenTaskDetails={props.onOpenTaskDetails} />; if (props.node.metadata?.status === "loading") return <LoadingContent node={props.node} theme={props.theme} onOpenTaskDetails={props.onOpenTaskDetails} />;
if (props.node.metadata?.status === "error") return <ErrorContent node={props.node} theme={props.theme} onRetry={props.onRetry} onReloadResource={props.onReloadResource} />; if (props.node.metadata?.status === "error") return <ErrorContent node={props.node} theme={props.theme} onRetry={props.onRetry} onReloadResource={props.onReloadResource} />;
@@ -8,6 +8,7 @@ import { getGenerationResourceNodes, getContextResourceNodes, getMentionResource
import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview"; import { canvasNodeVideoPreviewUrl, canvasVideoAssetPreviewUrl } from "@/lib/canvas/canvas-media-preview";
import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade"; import { isNeutralColorGrade, resolveCanvasColorGradeReference } from "@/lib/canvas/canvas-color-grade";
import { getNodeResourceKind } from "@/lib/canvas/node-registry"; import { getNodeResourceKind } from "@/lib/canvas/node-registry";
import { mediaConversionSourceFingerprint } from "@/lib/media-conversion/contracts";
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference"; import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference"; import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation"; import { nodeReferenceImage } from "@/lib/canvas/canvas-project-generation";
@@ -64,6 +65,12 @@ export type NodeGenerationInput = {
}; };
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, assets: Asset[], promptOnly = false): NodeGenerationContext { export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, assets: Asset[], promptOnly = false): NodeGenerationContext {
const pendingConversion = findPendingMediaConversionInput(nodeId, nodes, connections);
if (pendingConversion) {
const status = pendingConversion.metadata?.mediaConversion?.status;
const reason = status === "stale" ? "输入或方式已变化" : status === "processing" ? "仍在处理中" : "尚未完成本地转换";
throw new Error(`转换节点「${pendingConversion.title || pendingConversion.id}」${reason},完成后才能执行下游生成`);
}
const connectedInputs = buildNodeGenerationInputs(nodeId, nodes, connections); const connectedInputs = buildNodeGenerationInputs(nodeId, nodes, connections);
const sourceNode = nodes.find((node) => node.id === nodeId); const sourceNode = nodes.find((node) => node.id === nodeId);
const portraitTextureInput = sourceNode?.type === CanvasNodeType.Image && sourceNode.metadata?.content && sourceNode.metadata?.portraitTexture const portraitTextureInput = sourceNode?.type === CanvasNodeType.Image && sourceNode.metadata?.content && sourceNode.metadata?.portraitTexture
@@ -120,6 +127,40 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
}; };
} }
/**
* 转换节点允许提前连到下游,但只有已物化的结果才能进入生成请求。
* 沿上游链路检查也覆盖“生成节点 -> 配置节点 -> 转换节点”的常见接法。
*/
export function findPendingMediaConversionInput(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
const nodesById = new Map(nodes.map((node) => [node.id, node]));
const queue = connections.filter((connection) => connection.toNodeId === nodeId).map((connection) => connection.fromNodeId);
const visited = new Set<string>();
while (queue.length) {
const sourceId = queue.shift()!;
if (visited.has(sourceId)) continue;
visited.add(sourceId);
const source = nodesById.get(sourceId);
if (!source) continue;
if (source.type === CanvasNodeType.MediaConversion) {
const state = source.metadata?.mediaConversion;
const sourceInputs = connections
.filter((connection) => connection.toNodeId === source.id)
.map((connection) => nodesById.get(connection.fromNodeId))
.filter((input): input is CanvasNodeData => Boolean(input && (input.type === CanvasNodeType.Image || input.type === CanvasNodeType.Video)));
const sourceInput = sourceInputs[0];
const ready = state?.status === "completed"
&& Boolean(state.resultStorageKey)
&& sourceInputs.length === 1
&& Boolean(sourceInput)
&& state.sourceNodeId === sourceInput?.id
&& state.sourceFingerprint === mediaConversionSourceFingerprint(sourceInput!);
if (!ready) return source;
}
connections.filter((connection) => connection.toNodeId === sourceId).forEach((connection) => queue.push(connection.fromNodeId));
}
return null;
}
function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]) { function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]) {
let next = prompt.trim(); let next = prompt.trim();
let removed = true; let removed = true;
@@ -32,6 +32,7 @@ type CanvasNodeToolbarProps = {
onUpload: (node: CanvasNodeData) => void; onUpload: (node: CanvasNodeData) => void;
onDownload: (node: CanvasNodeData) => void; onDownload: (node: CanvasNodeData) => void;
onSaveAsset: (node: CanvasNodeData) => void; onSaveAsset: (node: CanvasNodeData) => void;
onCreateConversion?: (node: CanvasNodeData) => void;
onMaskEdit: (node: CanvasNodeData) => void; onMaskEdit: (node: CanvasNodeData) => void;
onEmotion: (node: CanvasNodeData) => void; onEmotion: (node: CanvasNodeData) => void;
onPortraitTexture: (node: CanvasNodeData) => void; onPortraitTexture: (node: CanvasNodeData) => void;
@@ -91,6 +92,7 @@ export function CanvasNodeToolbar({
onUpload, onUpload,
onDownload, onDownload,
onSaveAsset, onSaveAsset,
onCreateConversion,
onMaskEdit, onMaskEdit,
onEmotion, onEmotion,
onPortraitTexture, onPortraitTexture,
@@ -221,7 +223,7 @@ export function CanvasNodeToolbar({
const nodeHoverHandlers = { const nodeHoverHandlers = {
onNodeInfo: onInfo, onNodeDelete: onDelete, onNodeRetry: onRetry, onNodeEditText: onEditText, onNodeDecreaseFont: onDecreaseFont, onNodeIncreaseFont: onIncreaseFont, onNodeInfo: onInfo, onNodeDelete: onDelete, onNodeRetry: onRetry, onNodeEditText: onEditText, onNodeDecreaseFont: onDecreaseFont, onNodeIncreaseFont: onIncreaseFont,
onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload, onNodeToggleDialog: onToggleDialog, onNodeAnnotate: onAnnotate, onNodeGenerateImage: onGenerateImage, onNodeUpload: onUpload, onNodeDownload: onDownload,
onNodeSaveAsset: onSaveAsset, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop, onNodeSaveAsset: onSaveAsset, onNodeCreateConversion: onCreateConversion, onNodeMaskEdit: onMaskEdit, onNodeEmotion: onEmotion, onNodePortraitTexture: onPortraitTexture, onNodeCrop: onCrop,
onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage, onNodeSplit: onSplit, onNodeUpscale: onUpscale, onNodeSuperResolve: onSuperResolve, onNodeAngle: onAngle, onNodeViewImage: onViewImage,
onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize, onNodeExtractVideoFrames: onExtractVideoFrames, onNodeExtractAudioFromVideo: onExtractAudioFromVideo, onNodeTrimVideoSegments: onTrimVideoSegments, onNodeReversePrompt: onReversePrompt, onNodeToggleFreeResize: onToggleFreeResize,
onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt, onNodeSubtitles: onSubtitles, onNodeTimeline: onTimeline, onNodeToggleLocked: onToggleLocked, onNodeCopyPrompt: copyImagePrompt,
@@ -400,7 +402,7 @@ export function CanvasNodeInfoModal({ node, open, onClose, onMetadataChange, rea
const [assetCategory, setAssetCategory] = useState<CanvasAssetCategory>("other"); const [assetCategory, setAssetCategory] = useState<CanvasAssetCategory>("other");
const imageBytes = node?.type === CanvasNodeType.Image && node.metadata?.content ? getDataUrlByteSize(node.metadata.content) : 0; const imageBytes = node?.type === CanvasNodeType.Image && node.metadata?.content ? getDataUrlByteSize(node.metadata.content) : 0;
const batchCount = node?.type === CanvasNodeType.Image ? node.metadata?.batchChildIds?.length || 0 : 0; const batchCount = node?.type === CanvasNodeType.Image ? node.metadata?.batchChildIds?.length || 0 : 0;
const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : "生成配置"; const nodeTypeLabel = node?.type === CanvasNodeType.Text ? "文本" : node?.type === CanvasNodeType.Script ? "分镜脚本" : node?.type === CanvasNodeType.Skill ? "技能" : node?.type === CanvasNodeType.Image ? "图片" : node?.type === CanvasNodeType.Video ? "视频" : node?.type === CanvasNodeType.Audio ? "音频" : node?.type === CanvasNodeType.Drawing ? "绘图" : node?.type === CanvasNodeType.Frame ? "背板" : node?.type === CanvasNodeType.MediaConversion ? "转换" : "生成配置";
useEffect(() => { useEffect(() => {
setAssetTags(node?.metadata?.assetTags || []); setAssetTags(node?.metadata?.assetTags || []);
setAssetTagInput(""); setAssetTagInput("");
+2 -1
View File
@@ -1,6 +1,6 @@
import React, { useCallback, useEffect, useRef, useState } from "react"; import React, { useCallback, useEffect, useRef, useState } from "react";
import type { ReactNode } from "react"; import type { ReactNode } from "react";
import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video } from "lucide-react"; import { AlertCircle, BookOpenCheck, CheckCircle2, ChevronRight, Clapperboard, Copy, Download, Image as ImageIcon, Lock, Maximize2, Music2, Pencil, RefreshCw, ScanSearch, Settings2, Star, Trash2, Type, Video, WandSparkles } from "lucide-react";
import { useCanvasNodeActions } from "./canvas-node-action-context"; import { useCanvasNodeActions } from "./canvas-node-action-context";
@@ -709,6 +709,7 @@ function nodeTypeIcon(type: CanvasNodeTypeId) {
if (type === CanvasNodeType.Drawing) return Pencil; if (type === CanvasNodeType.Drawing) return Pencil;
if (type === CanvasNodeType.Script) return Clapperboard; if (type === CanvasNodeType.Script) return Clapperboard;
if (type === CanvasNodeType.Config) return Settings2; if (type === CanvasNodeType.Config) return Settings2;
if (type === CanvasNodeType.MediaConversion) return WandSparkles;
if (type === CanvasNodeType.Skill) return BookOpenCheck; if (type === CanvasNodeType.Skill) return BookOpenCheck;
if (type === PORTRAIT_CLEARANCE_NODE_TYPE) return PortraitClearanceIcon; if (type === PORTRAIT_CLEARANCE_NODE_TYPE) return PortraitClearanceIcon;
if (type === ART_CRITIQUE_NODE_TYPE) return ScanSearch; if (type === ART_CRITIQUE_NODE_TYPE) return ScanSearch;
@@ -1,6 +1,6 @@
import { motion, useReducedMotion } from "motion/react"; import { motion, useReducedMotion } from "motion/react";
import { useLayoutEffect, useRef, useState, type CSSProperties, type ReactNode, type RefObject } from "react"; import { useLayoutEffect, useRef, useState, type CSSProperties, type ReactNode, type RefObject } from "react";
import { Clapperboard, Image as ImageIcon, List, Music2, Pencil, Video, Workflow as WorkflowIcon } from "lucide-react"; import { Clapperboard, Image as ImageIcon, List, Music2, Pencil, Video, WandSparkles, Workflow as WorkflowIcon } from "lucide-react";
import { useCanvasOverlayLayer } from "@/components/canvas/canvas-overlay-layer"; import { useCanvasOverlayLayer } from "@/components/canvas/canvas-overlay-layer";
import { canvasThemes } from "@/lib/canvas-theme"; import { canvasThemes } from "@/lib/canvas-theme";
@@ -152,7 +152,7 @@ function resolveNodePanelWidth(node: CanvasNodeData, viewport: ViewportTransform
return clamp(Math.round(node.width * viewport.k * 1.5), 680, 920); return clamp(Math.round(node.width * viewport.k * 1.5), 680, 920);
} }
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) { export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, getDisabledReason, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; getDisabledReason: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => string; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, provider?: "runninghub" | "comfyui") => void; onClose: () => void }) {
const theme = canvasThemes[useThemeStore((state) => state.theme)]; const theme = canvasThemes[useThemeStore((state) => state.theme)];
const reducedMotion = useReducedMotion(); const reducedMotion = useReducedMotion();
const menuRef = useRef<HTMLDivElement>(null); const menuRef = useRef<HTMLDivElement>(null);
@@ -160,7 +160,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co
const lastPointerRef = useRef<Position | null>(null); const lastPointerRef = useRef<Position | null>(null);
const { bringToFront, zIndex } = useCanvasOverlayLayer("connection-create-menu", "var(--z-modal-overlay)"); const { bringToFront, zIndex } = useCanvasOverlayLayer("connection-create-menu", "var(--z-modal-overlay)");
const menuWidth = Math.min(288, viewportSize.width - 24); const menuWidth = Math.min(288, viewportSize.width - 24);
const menuHeight = canCreateDrawing ? 360 : 316; const menuHeight = canCreateDrawing ? 404 : 360;
const gap = 12; const gap = 12;
const initialPosition = getConnectionMenuPosition(pending.position, viewport, viewportSize, menuWidth, menuHeight, gap); const initialPosition = getConnectionMenuPosition(pending.position, viewport, viewportSize, menuWidth, menuHeight, gap);
@@ -231,6 +231,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co
{canCreateDrawing ? <ConnectionCreateOption expanded={activeOption === "绘图"} motionEnabled={!reducedMotion} icon={<Pencil className="size-4" />} title="绘图" description="以参考图片为底图,自由绘制和标注" disabledReason={getDisabledReason(CanvasNodeType.Drawing)} onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null} {canCreateDrawing ? <ConnectionCreateOption expanded={activeOption === "绘图"} motionEnabled={!reducedMotion} icon={<Pencil className="size-4" />} title="绘图" description="以参考图片为底图,自由绘制和标注" disabledReason={getDisabledReason(CanvasNodeType.Drawing)} onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null}
<ConnectionCreateOption expanded={activeOption === "视频生成"} motionEnabled={!reducedMotion} icon={<Video className="size-4" />} title="视频生成" description="结合提示词与参考素材,生成动态视频" disabledReason={getDisabledReason(CanvasNodeType.Video)} onClick={() => onCreate(CanvasNodeType.Video)} /> <ConnectionCreateOption expanded={activeOption === "视频生成"} motionEnabled={!reducedMotion} icon={<Video className="size-4" />} title="视频生成" description="结合提示词与参考素材,生成动态视频" disabledReason={getDisabledReason(CanvasNodeType.Video)} onClick={() => onCreate(CanvasNodeType.Video)} />
<ConnectionCreateOption expanded={activeOption === "音频参考"} motionEnabled={!reducedMotion} icon={<Music2 className="size-4" />} title="音频参考" description="连接文本或角色卡,创建音频生成节点" disabledReason={getDisabledReason(CanvasNodeType.Audio)} onClick={() => onCreate(CanvasNodeType.Audio)} /> <ConnectionCreateOption expanded={activeOption === "音频参考"} motionEnabled={!reducedMotion} icon={<Music2 className="size-4" />} title="音频参考" description="连接文本或角色卡,创建音频生成节点" disabledReason={getDisabledReason(CanvasNodeType.Audio)} onClick={() => onCreate(CanvasNodeType.Audio)} />
<ConnectionCreateOption expanded={activeOption === "转换"} motionEnabled={!reducedMotion} icon={<WandSparkles className="size-4" />} title="转换" description="本地处理图片或视频" disabledReason={getDisabledReason(CanvasNodeType.MediaConversion)} onClick={() => onCreate(CanvasNodeType.MediaConversion)} />
</div> </div>
</motion.div> </motion.div>
); );
@@ -0,0 +1,360 @@
import { CheckCircle2, CircleAlert, Download, Image as ImageIcon, LoaderCircle, RefreshCw, Video, WandSparkles } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { getNodeInputKind } from "@/lib/canvas/node-registry";
import type { CanvasTheme } from "@/lib/canvas-theme";
import {
createDefaultMediaConversionState,
isLocalImageOperation,
mediaConversionOperationDescription,
mediaConversionOperationLabel,
mediaConversionSourceFingerprint,
MEDIA_CONVERSION_OPERATION_LABELS,
type MediaConversionNodeState,
type MediaConversionOperation,
} from "@/lib/media-conversion/contracts";
import { convertImageLocally, LocalImageConversionError } from "@/lib/media-conversion/local-converter";
import { getActiveUserScope } from "@/lib/user-scope";
import { runLocalDepthEstimation } from "@/services/depth-runtime";
import { runLocalLineartEstimation } from "@/services/lineart-runtime";
import { runLocalPoseEstimation } from "@/services/pose-runtime";
import { resolveImageUrl, setImageBlob } from "@/services/image-storage";
import { resolveMediaUrl } from "@/services/file-storage";
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import { useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
import { useCanvasNodeActions } from "../canvas-node-action-context";
import { useUpstreamNodes } from "../canvas-node-graph-context";
type MediaConversionNodeContentProps = {
node: CanvasNodeData;
theme: CanvasTheme;
};
const OPERATIONS = Object.keys(MEDIA_CONVERSION_OPERATION_LABELS) as MediaConversionOperation[];
export function MediaConversionNodeContent({ node, theme }: MediaConversionNodeContentProps) {
const { updateMetadata } = useCanvasNodeActions();
const upstream = useUpstreamNodes(node.id);
const localRuntimeConnection = useLocalRuntimeStore((runtime) => runtime.connection);
const localRuntimeConnecting = useLocalRuntimeStore((runtime) => runtime.connecting);
const reconnectLocalRuntime = useLocalRuntimeStore((runtime) => runtime.connect);
const mediaInputs = upstream.filter((item) => item.type === CanvasNodeType.Image || item.type === CanvasNodeType.Video);
const input = mediaInputs[0];
const inputKind = input ? getNodeInputKind(input.type) : undefined;
const storedState = node.metadata?.mediaConversion;
const state = storedState || createDefaultMediaConversionState();
const currentFingerprint = input ? mediaConversionSourceFingerprint(input) : "";
const hasMultipleInputs = mediaInputs.length > 1;
const isStale = Boolean(state.status === "completed" && (!currentFingerprint || state.sourceFingerprint !== currentFingerprint));
const status = isStale ? "stale" : state.status;
const [sourceUrl, setSourceUrl] = useState("");
const [resultUrl, setResultUrl] = useState("");
const [showResult, setShowResult] = useState(status === "completed" || status === "stale");
const [notice, setNotice] = useState("");
const abortRef = useRef<AbortController | null>(null);
useEffect(() => {
let active = true;
const fallback = input?.metadata?.previewContent || input?.metadata?.content || "";
setSourceUrl(fallback);
if (!input?.metadata?.storageKey) return () => { active = false; };
const resolver = input.type === CanvasNodeType.Image ? resolveImageUrl : resolveMediaUrl;
void resolver(input.metadata.storageKey, fallback)
.then((resolved) => { if (active) setSourceUrl(resolved || fallback); })
.catch(() => { if (active) setSourceUrl(fallback); });
return () => { active = false; };
}, [input?.id, input?.metadata?.content, input?.metadata?.previewContent, input?.metadata?.storageKey, input?.type]);
useEffect(() => {
let active = true;
setResultUrl("");
if (!state.resultStorageKey) return () => { active = false; };
void resolveImageUrl(state.resultStorageKey)
.then((resolved) => { if (active) setResultUrl(resolved); })
.catch(() => { if (active) setResultUrl(""); });
return () => { active = false; };
}, [state.resultStorageKey]);
useEffect(() => {
if (!updateMetadata || state.status !== "completed" || (currentFingerprint && state.sourceFingerprint === currentFingerprint)) return;
updateMetadata(node.id, {
mediaConversion: { ...state, status: "stale", updatedAt: new Date().toISOString() },
});
}, [currentFingerprint, node.id, state, updateMetadata]);
useEffect(() => {
if (status === "stale") setShowResult(false);
}, [status]);
useEffect(() => () => abortRef.current?.abort(), []);
const updateState = (patch: Partial<MediaConversionNodeState>) => {
updateMetadata?.(node.id, { mediaConversion: { ...state, ...patch, schemaVersion: 1 } });
};
const selectOperation = (operation: MediaConversionOperation) => {
if (operation === state.operation) return;
setNotice("");
updateState({
operation,
status: state.resultStorageKey ? "stale" : "idle",
errorCode: undefined,
errorMessage: undefined,
updatedAt: new Date().toISOString(),
});
setShowResult(false);
};
const run = async () => {
setNotice("");
if (!input || !currentFingerprint) {
setNotice("请先连接一张图片或一个视频");
return;
}
if (hasMultipleInputs) {
setNotice("转换节点只能连接一个输入,请移除多余连线");
return;
}
if (inputKind === "video") {
updateState({ status: "unavailable", outputKind: "video", errorCode: "video_not_ready", errorMessage: "本地视频转换还在进行短片性能验证", updatedAt: new Date().toISOString() });
setNotice("视频转换暂未开启,先完成单图模型验证");
return;
}
if (!isLocalImageOperation(state.operation) && state.operation !== "depth" && state.operation !== "lineart" && state.operation !== "pose") {
updateState({ status: "unavailable", outputKind: "image", errorCode: "model_missing", errorMessage: "该转换需要先安装并验证本地模型", updatedAt: new Date().toISOString() });
setNotice(mediaConversionOperationDescription(state.operation));
return;
}
if (!sourceUrl) {
updateState({ status: "error", errorCode: "source_unreadable", errorMessage: "图片预览尚未准备好,请稍后重试", updatedAt: new Date().toISOString() });
setNotice("图片预览尚未准备好,请稍后重试");
return;
}
abortRef.current?.abort();
const controller = new AbortController();
abortRef.current = controller;
const startedAt = new Date().toISOString();
updateState({ status: "processing", sourceNodeId: input.id, sourceFingerprint: currentFingerprint, outputKind: "image", errorCode: undefined, errorMessage: undefined, startedAt, updatedAt: startedAt });
try {
const result = state.operation === "depth"
? await runLocalDepthEstimation(sourceUrl, controller.signal)
: state.operation === "lineart"
? await runLocalLineartEstimation(sourceUrl, controller.signal)
: state.operation === "pose"
? await runLocalPoseEstimation(sourceUrl, controller.signal)
: await convertImageLocally(sourceUrl, state.operation, { signal: controller.signal, maxDimension: 1024 });
if (controller.signal.aborted) return;
const storageKey = localConversionStorageKey(node.id, currentFingerprint, state.operation);
const url = await setImageBlob(storageKey, result.blob);
const completedAt = new Date().toISOString();
const detectedPeople = state.operation === "pose" && "personCount" in result && typeof result.personCount === "number" ? result.personCount : undefined;
updateMetadata?.(node.id, {
content: "",
previewContent: "",
storageKey,
mimeType: result.blob.type || "image/png",
bytes: result.blob.size,
naturalWidth: result.width,
naturalHeight: result.height,
mediaConversion: {
...state,
schemaVersion: 1,
status: "completed",
sourceNodeId: input.id,
sourceFingerprint: currentFingerprint,
outputKind: "image",
resultStorageKey: storageKey,
resultWidth: result.width,
resultHeight: result.height,
...(detectedPeople !== undefined ? { detectedPeople } : {}),
errorCode: undefined,
errorMessage: undefined,
updatedAt: completedAt,
},
});
setResultUrl(url);
setShowResult(true);
} catch (error) {
if (controller.signal.aborted) return;
const aborted = error instanceof LocalImageConversionError && error.code === "aborted";
if (aborted) return;
if (error instanceof LocalRuntimeClientError) {
if (error.code === "pose_no_person") {
updateState({ status: "skipped", errorCode: error.code, errorMessage: error.message, detectedPeople: 0, updatedAt: new Date().toISOString() });
setNotice("未检测到可用人物姿态,可更换含人物的图片或选择其他转换方式");
return;
}
const nextStatus = ["depth_model_missing", "depth_runtime_unavailable", "lineart_model_missing", "lineart_runtime_unavailable", "pose_model_missing", "pose_runtime_unavailable"].includes(error.code) ? "unavailable" : "error";
updateState({ status: nextStatus, errorCode: error.code, errorMessage: error.message, updatedAt: new Date().toISOString() });
setNotice(localRuntimeNotice(error, useLocalRuntimeStore.getState().connection));
return;
}
const conversionError = error instanceof LocalImageConversionError ? error : new LocalImageConversionError("source_unreadable", "本地图片转换失败");
const nextStatus = conversionError.code === "model_missing" ? "unavailable" : "error";
updateState({ status: nextStatus, errorCode: conversionError.code, errorMessage: conversionError.message, updatedAt: new Date().toISOString() });
setNotice(conversionError.message);
}
};
const downloadResult = () => {
if (!resultUrl) return;
const link = document.createElement("a");
link.href = resultUrl;
link.download = `${node.title || "转换结果"}-${state.operation}.png`;
link.click();
};
const effectiveResult = Boolean(resultUrl && (status === "completed" || status === "stale"));
const displayResult = showResult && effectiveResult;
const buttonDisabled = !input || hasMultipleInputs || status === "processing";
const statusText = mediaConversionStatusLabel(status);
const previewStatusText = state.operation === "pose" && status === "completed" && typeof state.detectedPeople === "number"
? `${statusText} · ${state.detectedPeople} 人`
: statusText;
const showRuntimeRecovery = (state.operation === "depth" || state.operation === "lineart" || state.operation === "pose") && (localRuntimeConnection === "unreachable" || state.errorCode === "depth_runtime_unavailable" || state.errorCode === "lineart_runtime_unavailable" || state.errorCode === "pose_runtime_unavailable");
const retryLocalRuntime = async () => {
setNotice("");
await reconnectLocalRuntime();
const next = useLocalRuntimeStore.getState();
if (next.connection === "connected") {
setNotice("本地模型服务已连接,可以重新开始转换");
} else {
setNotice(next.error || "本地模型服务仍未连接,请先运行 start-yingce-local.cmd");
}
};
return (
<div
className="flex h-full min-h-0 w-full flex-col gap-2.5 overflow-hidden rounded-[inherit] p-3"
style={{ background: theme.node.panel, color: theme.node.text }}
onPointerDown={(event) => event.stopPropagation()}
onWheel={(event) => event.stopPropagation()}
>
<div className="flex min-w-0 items-center gap-2">
<span className="grid size-8 shrink-0 place-items-center rounded-[var(--r-md)]" style={{ background: theme.toolbar.itemHover, color: theme.accent.primary }}>
<WandSparkles className="size-4" aria-hidden="true" />
</span>
<div className="min-w-0 flex-1">
<div className="truncate text-[var(--fs-label)] font-semibold">{node.title || "转换"}</div>
<div className="truncate text-[var(--fs-tiny)]" style={{ color: theme.node.muted }}>本地处理 · 单图优先</div>
</div>
<StatusBadge status={status} />
</div>
<div className="flex min-h-0 flex-1 flex-col gap-2">
<div className="relative min-h-0 flex-1 overflow-hidden rounded-[var(--r-md)] border" style={{ borderColor: theme.node.edge, background: theme.node.fill }}>
{displayResult ? (
<img src={resultUrl} alt={`${mediaConversionOperationLabel(state.operation)}结果`} className="absolute inset-0 size-full object-contain" draggable={false} />
) : input?.type === CanvasNodeType.Image && sourceUrl ? (
<img src={sourceUrl} alt="转换输入" className="absolute inset-0 size-full object-contain" draggable={false} />
) : input?.type === CanvasNodeType.Video && sourceUrl ? (
<video src={sourceUrl} className="absolute inset-0 size-full object-contain" controls preload="metadata" />
) : (
<div className="absolute inset-0 grid place-items-center" style={{ color: theme.node.muted }}>
{input?.type === CanvasNodeType.Video ? <Video className="size-8 opacity-35" aria-hidden="true" /> : <ImageIcon className="size-8 opacity-35" aria-hidden="true" />}
</div>
)}
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex items-end justify-between gap-2 bg-gradient-to-t from-black/75 to-transparent px-2.5 pb-2 pt-8 text-white">
<div className="min-w-0">
<div className="truncate text-[var(--fs-micro)] font-semibold">{displayResult ? (status === "stale" ? "上次结果 · 待更新" : "转换结果") : input?.title || "等待输入"}</div>
<div className="mt-0.5 truncate text-[var(--fs-micro)] opacity-75">{hasMultipleInputs ? "已连接多个输入" : input ? `${input.type === CanvasNodeType.Video ? "视频" : "图片"} · ${previewStatusText}` : "连接图片或视频节点"}</div>
</div>
{state.resultWidth && state.resultHeight && displayResult ? <span className="shrink-0 text-[var(--fs-micro)] tabular-nums">{state.resultWidth} × {state.resultHeight}</span> : null}
</div>
</div>
{effectiveResult ? (
<div className="flex gap-1" data-canvas-no-zoom>
<PreviewToggle theme={theme} pressed={!showResult} label="原素材" onClick={() => setShowResult(false)} />
<PreviewToggle theme={theme} pressed={showResult} label="结果" onClick={() => setShowResult(true)} />
<button type="button" aria-label="下载转换结果" title="下载转换结果" className="grid h-6 w-8 shrink-0 place-items-center rounded-[var(--r-sm)] outline-none transition hover:bg-black/5 focus-visible:ring-2 dark:hover:bg-white/10" style={{ color: theme.node.muted, outlineColor: theme.accent.primary }} onClick={(event) => { event.stopPropagation(); downloadResult(); }} onMouseDown={(event) => event.stopPropagation()}>
<Download className="size-3.5" aria-hidden="true" />
</button>
</div>
) : null}
</div>
<div className="flex items-center gap-2" data-canvas-no-zoom onWheel={(event) => event.stopPropagation()}>
<label className="sr-only" htmlFor={`${node.id}-operation`}>转换方式</label>
<select
id={`${node.id}-operation`}
value={state.operation}
disabled={status === "processing"}
className="h-8 min-w-0 flex-1 rounded-[var(--r-sm)] border bg-transparent px-2 text-[var(--fs-tiny)] outline-none focus-visible:ring-2"
style={{ borderColor: theme.node.edge, color: theme.node.text, outlineColor: theme.accent.primary }}
onChange={(event) => selectOperation(event.target.value as MediaConversionOperation)}
onMouseDown={(event) => event.stopPropagation()}
>
{OPERATIONS.map((operation) => <option key={operation} value={operation}>{mediaConversionOperationLabel(operation)}</option>)}
</select>
<button
type="button"
data-canvas-no-zoom
className="inline-flex h-8 shrink-0 items-center gap-1.5 rounded-[var(--r-sm)] px-3 text-[var(--fs-tiny)] font-semibold outline-none transition hover:brightness-105 focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-50"
style={{ background: theme.accent.primary, color: theme.accent.onPrimary, outlineColor: theme.accent.primary }}
disabled={buttonDisabled}
onClick={(event) => { event.stopPropagation(); void run(); }}
onMouseDown={(event) => event.stopPropagation()}
>
{status === "processing" ? <LoaderCircle className="size-3.5 animate-spin" aria-hidden="true" /> : null}
{status === "processing" ? "处理中" : status === "completed" || status === "stale" ? "重新转换" : "开始转换"}
</button>
</div>
<div className="flex min-h-4 items-center gap-2 text-[var(--fs-micro)]" style={{ color: status === "unavailable" || status === "skipped" ? "var(--status-warning)" : notice || status === "error" ? "var(--status-error)" : theme.node.muted }}>
<span className="min-w-0 flex-1 truncate" title={notice || mediaConversionOperationDescription(state.operation)}>{notice || mediaConversionOperationDescription(state.operation)}</span>
{showRuntimeRecovery ? (
<button
type="button"
className="shrink-0 rounded-[var(--r-sm)] px-1.5 py-0.5 font-semibold outline-none transition hover:bg-black/5 focus-visible:ring-2 dark:hover:bg-white/10"
style={{ color: theme.accent.primary, outlineColor: theme.accent.primary }}
disabled={localRuntimeConnecting}
onClick={(event) => { event.stopPropagation(); void retryLocalRuntime(); }}
onMouseDown={(event) => event.stopPropagation()}
>
{localRuntimeConnecting ? "连接中" : "重试连接"}
</button>
) : null}
</div>
</div>
);
}
function PreviewToggle({ theme, pressed, label, onClick }: { theme: CanvasTheme; pressed: boolean; label: string; onClick: () => void }) {
return <button type="button" aria-pressed={pressed} className="h-6 flex-1 rounded-[var(--r-sm)] px-2 text-[var(--fs-micro)] font-medium outline-none transition focus-visible:ring-2" style={{ background: pressed ? theme.accent.primary : "color-mix(in oklch, var(--foreground-muted) 10%, transparent)", color: pressed ? theme.accent.onPrimary : "var(--foreground-muted)", outlineColor: theme.accent.primary }} onClick={(event) => { event.stopPropagation(); onClick(); }} onMouseDown={(event) => event.stopPropagation()}>{label}</button>;
}
function StatusBadge({ status }: { status: MediaConversionNodeState["status"] | "stale" }) {
if (status === "processing") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-info)", background: "color-mix(in oklch, var(--status-info) 15%, transparent)" }}><LoaderCircle className="size-3 animate-spin" aria-hidden="true" />处理中</span>;
if (status === "completed") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-success)", background: "color-mix(in oklch, var(--status-success) 15%, transparent)" }}><CheckCircle2 className="size-3" aria-hidden="true" />已完成</span>;
if (status === "stale") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}><RefreshCw className="size-3" aria-hidden="true" />待更新</span>;
if (status === "error") return <span className="inline-flex shrink-0 items-center gap-1 rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-error)", background: "color-mix(in oklch, var(--status-error) 15%, transparent)" }}><CircleAlert className="size-3" aria-hidden="true" />失败</span>;
if (status === "unavailable") return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}>本地不可用</span>;
if (status === "skipped") return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--status-warning)", background: "color-mix(in oklch, var(--status-warning) 15%, transparent)" }}>已跳过</span>;
return <span className="inline-flex shrink-0 items-center rounded-full px-2 py-1 text-[var(--fs-micro)] font-semibold" style={{ color: "var(--foreground-muted)", background: "color-mix(in oklch, var(--foreground-muted) 12%, transparent)" }}>未开始</span>;
}
function mediaConversionStatusLabel(status: MediaConversionNodeState["status"] | "stale") {
if (status === "processing") return "正在处理";
if (status === "completed") return "已完成";
if (status === "stale") return "输入已变化,结果待更新";
if (status === "unavailable") return "本地能力不可用";
if (status === "skipped") return "未检测到人物姿态";
if (status === "error") return "处理失败";
return "尚未转换";
}
function localRuntimeNotice(error: LocalRuntimeClientError, connection: ReturnType<typeof useLocalRuntimeStore.getState>["connection"]) {
if (connection === "unreachable") return "本地模型服务未启动,请双击 start-yingce-local.cmd 后重试";
if (error.code === "depth_model_missing") return "深度模型尚未安装,请先完成本地模型安装";
if (error.code === "lineart_model_missing") return "AI 线稿依赖或模型尚未安装,请先完成本地线稿模型安装";
if (error.code === "pose_model_missing") return "姿态模型或依赖尚未安装,请先完成本地姿态模型安装";
return error.message;
}
function localConversionStorageKey(nodeId: string, fingerprint: string, operation: MediaConversionOperation) {
return `image:${getActiveUserScope()}:media-conversion:${nodeId}:${fingerprint}:${operation}`;
}
+5
View File
@@ -26,6 +26,7 @@ export const NODE_DEFAULT_SIZE = {
[CanvasNodeType.Compare]: { width: 520, height: 320, title: "对比" }, [CanvasNodeType.Compare]: { width: 520, height: 320, title: "对比" },
[CanvasNodeType.Chart]: { width: 480, height: 320, title: "图表" }, [CanvasNodeType.Chart]: { width: 480, height: 320, title: "图表" },
[CanvasNodeType.ColorGrade]: { width: 420, height: 360, title: "调色" }, [CanvasNodeType.ColorGrade]: { width: 420, height: 360, title: "调色" },
[CanvasNodeType.MediaConversion]: { width: 380, height: 440, title: "转换" },
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>; } satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
export const NODE_SPECS = { export const NODE_SPECS = {
@@ -101,6 +102,10 @@ export const NODE_SPECS = {
...NODE_DEFAULT_SIZE[CanvasNodeType.ColorGrade], ...NODE_DEFAULT_SIZE[CanvasNodeType.ColorGrade],
metadata: { status: "idle" }, metadata: { status: "idle" },
}, },
[CanvasNodeType.MediaConversion]: {
...NODE_DEFAULT_SIZE[CanvasNodeType.MediaConversion],
metadata: { status: "idle" },
},
} satisfies Record<CanvasNodeType, CanvasNodeSpec>; } satisfies Record<CanvasNodeType, CanvasNodeSpec>;
export function getNodeSpec(type: CanvasNodeType) { export function getNodeSpec(type: CanvasNodeType) {
+1
View File
@@ -399,6 +399,7 @@ function canvasNodeTypeLabel(type?: CanvasNodeType) {
if (type === CanvasNodeType.Frame) return "背板"; if (type === CanvasNodeType.Frame) return "背板";
if (type === CanvasNodeType.Drawing) return "绘图节点"; if (type === CanvasNodeType.Drawing) return "绘图节点";
if (type === CanvasNodeType.Skill) return "技能节点"; if (type === CanvasNodeType.Skill) return "技能节点";
if (type === CanvasNodeType.MediaConversion) return "转换节点";
return "文本节点"; return "文本节点";
} }
+21 -5
View File
@@ -1,7 +1,7 @@
import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection"; import { maxModelInputCapacity, type ModelInputSummary } from "@/lib/model-selection";
import { getNodeAcceptedInputKind, getNodeGenerationMode, getNodeInputKind } from "@/lib/canvas/node-registry"; import { getNodeAcceptedInputKinds, getNodeGenerationMode, getNodeInputKind, getNodeMaxInputCount } from "@/lib/canvas/node-registry";
import type { AiConfig } from "@/stores/use-config-store"; import type { AiConfig } from "@/stores/use-config-store";
import { type CanvasConnection, type CanvasNodeData } from "@/types/canvas"; import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
type ConnectionCandidate = Pick<CanvasConnection, "fromNodeId" | "toNodeId">; type ConnectionCandidate = Pick<CanvasConnection, "fromNodeId" | "toNodeId">;
type CanvasConnectionPolicyOptions = { type CanvasConnectionPolicyOptions = {
@@ -12,11 +12,27 @@ type CanvasConnectionPolicyOptions = {
export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) { export function canvasConnectionError(config: AiConfig, nodes: CanvasNodeData[], connections: CanvasConnection[], candidate: ConnectionCandidate, options: CanvasConnectionPolicyOptions = {}) {
const target = nodes.find((node) => node.id === candidate.toNodeId); const target = nodes.find((node) => node.id === candidate.toNodeId);
if (!target) return "找不到连线目标节点"; if (!target) return "找不到连线目标节点";
const acceptedInputKind = getNodeAcceptedInputKind(target.type); const acceptedInputKinds = getNodeAcceptedInputKinds(target.type);
if (acceptedInputKind) { if (acceptedInputKinds.length) {
const source = nodes.find((node) => node.id === candidate.fromNodeId); const source = nodes.find((node) => node.id === candidate.fromNodeId);
const sourceKind = source ? getNodeInputKind(source.type) : undefined; const sourceKind = source ? getNodeInputKind(source.type) : undefined;
if (sourceKind !== acceptedInputKind) return `${acceptedInputKindLabel(acceptedInputKind)}节点只接受${acceptedInputKindLabel(acceptedInputKind)}输入`; const isMediaConversion = target.type === CanvasNodeType.MediaConversion;
const hasAcceptedSource = isMediaConversion
? source?.type === CanvasNodeType.Image || source?.type === CanvasNodeType.Video
: Boolean(sourceKind && acceptedInputKinds.includes(sourceKind));
if (!sourceKind || !hasAcceptedSource) {
const labels = acceptedInputKinds.map(acceptedInputKindLabel).join("或");
return `${isMediaConversion ? "转换" : labels}节点只接受${labels}输入`;
}
const maxInputCount = getNodeMaxInputCount(target.type);
if (maxInputCount) {
const inputCount = new Set(
[...connections, { id: "candidate", ...candidate }]
.filter((connection) => connection.toNodeId === target.id)
.map((connection) => connection.fromNodeId),
).size;
if (inputCount > maxInputCount) return `${isMediaConversion ? "转换" : "当前"}节点最多连接 ${maxInputCount} 个输入`;
}
} }
const mode = getNodeGenerationMode(target); const mode = getNodeGenerationMode(target);
if (!mode) return ""; if (!mode) return "";
@@ -1,4 +1,4 @@
import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video } from "lucide-react"; import { ChartColumn, Clapperboard, Code, Columns2, FileText, Globe, Image as ImageIcon, Music2, PanelTop, Palette, Pencil, Settings2, Shapes, Sparkles, Type, Video, WandSparkles } from "lucide-react";
import { NODE_SPECS } from "@/constant/canvas"; import { NODE_SPECS } from "@/constant/canvas";
import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size"; import { MEDIA_NODE_MIN_SIZE } from "@/lib/canvas/canvas-node-size";
@@ -25,7 +25,7 @@ const BUILTIN_NODE_TRAITS = {
minSize: MEDIA_NODE_MIN_SIZE, minSize: MEDIA_NODE_MIN_SIZE,
keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize, keepAspectRatio: (node: CanvasNodeData) => !node.metadata?.freeResize,
showInCreateMenu: true, showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "image" : null), resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "image" : null),
generationMode: () => "image", generationMode: () => "image",
inputKind: "image", inputKind: "image",
}, },
@@ -82,7 +82,7 @@ const BUILTIN_NODE_TRAITS = {
minSize: MEDIA_NODE_MIN_SIZE, minSize: MEDIA_NODE_MIN_SIZE,
keepAspectRatio: () => true, keepAspectRatio: () => true,
showInCreateMenu: true, showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => (node.metadata?.content ? "video" : null), resourceKind: (node: CanvasNodeData) => (node.metadata?.content || node.metadata?.storageKey ? "video" : null),
generationMode: () => "video", generationMode: () => "video",
inputKind: "video", inputKind: "video",
}, },
@@ -165,6 +165,20 @@ const BUILTIN_NODE_TRAITS = {
resourceKind: () => "image", resourceKind: () => "image",
inputKind: "image", inputKind: "image",
}, },
[CanvasNodeType.MediaConversion]: {
label: "转换",
icon: <WandSparkles />,
minSize: { width: 320, height: 360 },
showInCreateMenu: true,
resourceKind: (node: CanvasNodeData) => {
const conversion = node.metadata?.mediaConversion;
if (conversion?.status !== "completed" || !conversion.resultStorageKey) return null;
return conversion.outputKind === "video" ? "video" : "image";
},
acceptsInputKind: ["image", "video"],
maxInputCount: 1,
inputKind: "image",
},
} satisfies Record<string, Omit<CanvasNodeDefinition, "type" | "defaultTitle" | "defaultSize" | "defaultMetadata">>; } satisfies Record<string, Omit<CanvasNodeDefinition, "type" | "defaultTitle" | "defaultSize" | "defaultMetadata">>;
export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => { export const BUILTIN_NODE_DEFINITIONS: CanvasNodeDefinition[] = (Object.keys(BUILTIN_NODE_TRAITS) as CanvasNodeType[]).map((type) => {
+1 -1
View File
@@ -1,3 +1,3 @@
export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition"; export type { CanvasNodeDefinition, CanvasNodeInputKind } from "./node-definition";
export { getNodeAcceptedInputKind, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry"; export { getNodeAcceptedInputKind, getNodeAcceptedInputKinds, getNodeDefinition, getNodeGenerationMode, getNodeIcon, getNodeInputKind, getNodeLabel, getNodeListLabel, getNodeMaxInputCount, getNodeMinSize, getNodeOwnerId, getNodeResourceKind, listCreatableNodeDefinitions, listNodeDefinitions, registerNodeDefinitions, registerPluginCanvasNodes, shouldKeepAspectRatio, unregisterNodeDefinitions } from "./node-registry";
import "./definitions"; import "./definitions";
@@ -43,8 +43,10 @@ export type CanvasNodeDefinition = {
generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null; generationMode?: (node: CanvasNodeData) => CanvasGenerationMode | null;
/** 是否显示右侧输出连接点;缺省为 true,消费型终点节点可关闭。 */ /** 是否显示右侧输出连接点;缺省为 true,消费型终点节点可关闭。 */
showOutputConnection?: boolean; showOutputConnection?: boolean;
/** 若设置,作为目标时只接受这一类上游节点。 */ /** 若设置,作为目标时只接受这些上游节点类型。 */
acceptsInputKind?: CanvasNodeInputKind; acceptsInputKind?: CanvasNodeInputKind | CanvasNodeInputKind[];
/** 限制直接输入数量;转换节点等一进一出节点通常为 1。 */
maxInputCount?: number;
/** /**
* 作为上游输入被参考素材容量校验计数时的类别; * 作为上游输入被参考素材容量校验计数时的类别;
* 不设表示不参与计数(生成配置、背板)。与 resourceKind 不同,计数不看内容。 * 不设表示不参与计数(生成配置、背板)。与 resourceKind 不同,计数不看内容。
@@ -69,6 +71,7 @@ export function canvasNodeDefinitionFromPlugin(pluginId: string, contribution: P
showInCreateMenu: true, showInCreateMenu: true,
showOutputConnection: contribution.showOutputConnection, showOutputConnection: contribution.showOutputConnection,
acceptsInputKind: contribution.acceptsInputKind, acceptsInputKind: contribution.acceptsInputKind,
maxInputCount: contribution.maxInputCount,
plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema }, plugin: { pluginId, renderer: contribution.renderer, schema: contribution.schema },
}; };
} }
@@ -100,3 +100,13 @@ export function getNodeInputKind(type: CanvasNodeTypeId) {
export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) { export function getNodeAcceptedInputKind(type: CanvasNodeTypeId) {
return definitions.get(type)?.acceptsInputKind; return definitions.get(type)?.acceptsInputKind;
} }
export function getNodeAcceptedInputKinds(type: CanvasNodeTypeId) {
const accepted = definitions.get(type)?.acceptsInputKind;
if (!accepted) return [];
return Array.isArray(accepted) ? accepted : [accepted];
}
export function getNodeMaxInputCount(type: CanvasNodeTypeId) {
return definitions.get(type)?.maxInputCount;
}
@@ -20,6 +20,7 @@ export const addNodeMenuCommands: AddNodeMenuCommand[] = [
{ id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() }, { id: "folder", label: "文件夹", icon: <Folder />, badge: "6 款", section: "node", defaultOrder: 45, run: (ctx) => ctx.handlers.onAddFolder() },
nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }), nodeCommand(CanvasNodeType.Image, { defaultOrder: 50, run: (ctx) => ctx.handlers.onAddImage() }),
nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }), nodeCommand(CanvasNodeType.Video, { defaultOrder: 60, run: (ctx) => ctx.handlers.onAddVideo() }),
nodeCommand(CanvasNodeType.MediaConversion, { badge: "本地", defaultOrder: 65, run: (ctx) => ctx.handlers.onAddExtensionNode(CanvasNodeType.MediaConversion) }),
// 导演台落在节点分区,但它开的是导演工作台、不是某种画布节点,故不走注册表。 // 导演台落在节点分区,但它开的是导演工作台、不是某种画布节点,故不走注册表。
{ id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() }, { id: "director", label: "导演台", icon: <Layers3 />, badge: "3D", section: "node", defaultOrder: 70, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onOpenDirector() },
nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }), nodeCommand(CanvasNodeType.Audio, { defaultOrder: 80, applicable: (ctx) => ctx.workspaceMode !== "simple", run: (ctx) => ctx.handlers.onAddAudio() }),
@@ -1,4 +1,4 @@
import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video } from "lucide-react"; import { AudioLines, Captions, Clapperboard, Download, FolderPlus, Images, Image as ImageIcon, Info, LoaderCircle, Lock, Maximize2, MessageSquare, Minus, Music2, Plus, RefreshCw, Scissors, Settings2, Trash2, Unlock, Upload, UserRound, Video, WandSparkles } from "lucide-react";
import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error"; import { CONTENT_MODERATION_ERROR_CODE, isContentModerationError } from "@/lib/generation-error";
import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry"; import { registerToolbarTools, type ToolContext, type ToolDefinition } from "@/lib/canvas/tool-registry";
@@ -139,6 +139,19 @@ export const nodeHoverToolbarTools: ToolDefinition[] = [
applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx), applicable: (ctx) => hasImage(ctx) || hasVideo(ctx) || hasAudio(ctx),
run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!), run: (ctx) => ctx.handlers.onNodeDownload(ctx.node!),
}, },
{
id: "createConversion",
toolbar: "node-hover",
category: "node-state",
label: "创建转换节点",
displayLabel: "转换",
icon: <WandSparkles className="size-3.5" />,
defaultVisible: true,
defaultOrder: 65,
nodeToolbar: { group: "primary", order: 15, description: "在右侧创建转换节点并自动连接" },
applicable: (ctx) => (hasImage(ctx) || hasVideo(ctx)) && Boolean(ctx.handlers.onNodeCreateConversion),
run: (ctx) => ctx.handlers.onNodeCreateConversion?.(ctx.node!),
},
{ {
id: "edit", id: "edit",
toolbar: "node-hover", toolbar: "node-hover",
@@ -81,6 +81,8 @@ export type ToolbarHandlers = {
onNodeUpload: (node: CanvasNodeData) => void; onNodeUpload: (node: CanvasNodeData) => void;
onNodeDownload: (node: CanvasNodeData) => void; onNodeDownload: (node: CanvasNodeData) => void;
onNodeSaveAsset: (node: CanvasNodeData) => void; onNodeSaveAsset: (node: CanvasNodeData) => void;
/** 从已有图片或视频创建并自动连接一个转换节点。 */
onNodeCreateConversion?: (node: CanvasNodeData) => void;
onNodeMaskEdit: (node: CanvasNodeData) => void; onNodeMaskEdit: (node: CanvasNodeData) => void;
onNodeEmotion: (node: CanvasNodeData) => void; onNodeEmotion: (node: CanvasNodeData) => void;
onNodePortraitTexture: (node: CanvasNodeData) => void; onNodePortraitTexture: (node: CanvasNodeData) => void;
+89
View File
@@ -0,0 +1,89 @@
import type { CanvasNodeData } from "@/types/canvas";
export const MEDIA_CONVERSION_SCHEMA_VERSION = 1 as const;
export const MEDIA_CONVERSION_NODE_TYPE = "media-conversion" as const;
export type MediaConversionOperation = "grayscale" | "edge-canny" | "lineart" | "depth" | "pose" | "cutout";
export type MediaConversionStatus = "idle" | "processing" | "completed" | "stale" | "skipped" | "unavailable" | "error";
export type MediaConversionOutputKind = "image" | "video";
export type MediaConversionNodeState = {
schemaVersion: typeof MEDIA_CONVERSION_SCHEMA_VERSION;
operation: MediaConversionOperation;
status: MediaConversionStatus;
sourceNodeId?: string;
sourceFingerprint?: string;
outputKind?: MediaConversionOutputKind;
resultStorageKey?: string;
resultWidth?: number;
resultHeight?: number;
detectedPeople?: number;
errorCode?: string;
errorMessage?: string;
startedAt?: string;
updatedAt?: string;
};
export const MEDIA_CONVERSION_OPERATION_LABELS: Record<MediaConversionOperation, string> = {
grayscale: "灰度图",
"edge-canny": "Canny 边缘",
lineart: "AI 线稿",
depth: "深度图",
pose: "姿态骨架",
cutout: "透明抠图",
};
export const MEDIA_CONVERSION_OPERATION_DESCRIPTIONS: Record<MediaConversionOperation, string> = {
grayscale: "普通图像算法,适合先检查明暗层次",
"edge-canny": "普通图像算法,提取清晰轮廓",
lineart: "本地 ControlNet Aux 线稿预处理,首次运行需要加载模型",
depth: "本地 Depth Anything V2 Small,首次运行需要加载模型",
pose: "本地 OpenPose 人体骨架预处理,未检测到人物时会跳过",
cutout: "需要安装并验证本地抠图模型",
};
export function createDefaultMediaConversionState(): MediaConversionNodeState {
return {
schemaVersion: MEDIA_CONVERSION_SCHEMA_VERSION,
operation: "edge-canny",
status: "idle",
};
}
export function mediaConversionSourceFingerprint(node: CanvasNodeData) {
const metadata = node.metadata;
const source = [
node.id,
metadata?.storageKey || "",
compactSource(metadata?.content || metadata?.previewContent || ""),
metadata?.mimeType || "",
metadata?.bytes || "",
metadata?.naturalWidth || "",
metadata?.naturalHeight || "",
metadata?.durationMs || "",
].join("|");
let hash = 2166136261;
for (let index = 0; index < source.length; index += 1) {
hash ^= source.charCodeAt(index);
hash = Math.imul(hash, 16777619);
}
return `v1-${(hash >>> 0).toString(16).padStart(8, "0")}`;
}
export function mediaConversionOperationLabel(operation: MediaConversionOperation) {
return MEDIA_CONVERSION_OPERATION_LABELS[operation];
}
export function mediaConversionOperationDescription(operation: MediaConversionOperation) {
return MEDIA_CONVERSION_OPERATION_DESCRIPTIONS[operation];
}
export function isLocalImageOperation(operation: MediaConversionOperation) {
return operation === "grayscale" || operation === "edge-canny";
}
function compactSource(value: string) {
if (value.length <= 2048) return value;
return `${value.slice(0, 1024)}|${value.slice(-1024)}|${value.length}`;
}
@@ -0,0 +1,228 @@
import type { MediaConversionOperation } from "./contracts";
export type LocalImageConversionResult = {
blob: Blob;
width: number;
height: number;
};
export class LocalImageConversionError extends Error {
constructor(readonly code: "model_missing" | "source_unreadable" | "browser_unsupported" | "aborted", message: string) {
super(message);
this.name = "LocalImageConversionError";
}
}
export async function convertImageLocally(sourceUrl: string, operation: MediaConversionOperation, options: { signal?: AbortSignal; maxDimension?: number } = {}): Promise<LocalImageConversionResult> {
throwIfAborted(options.signal);
if (operation !== "grayscale" && operation !== "edge-canny") {
throw new LocalImageConversionError("model_missing", "该转换需要先安装并验证本地模型");
}
const image = await loadImage(sourceUrl, options.signal);
throwIfAborted(options.signal);
const maxDimension = Math.max(256, options.maxDimension || 1024);
const scale = Math.min(1, maxDimension / Math.max(image.naturalWidth, image.naturalHeight));
const width = Math.max(1, Math.round(image.naturalWidth * scale));
const height = Math.max(1, Math.round(image.naturalHeight * scale));
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d", { willReadFrequently: true });
if (!context) throw new LocalImageConversionError("browser_unsupported", "当前浏览器无法创建图片处理画布");
context.drawImage(image, 0, 0, width, height);
const input = context.getImageData(0, 0, width, height);
const output = transformPixels(input.data, width, height, operation, options.signal);
const outputImage = new ImageData(output, width, height);
context.putImageData(outputImage, 0, 0);
const blob = await canvasToBlob(canvas, options.signal);
throwIfAborted(options.signal);
return { blob, width, height };
}
/** Pure pixel transform used by the browser executor and unit tests. */
export function transformPixels(source: Uint8ClampedArray, width: number, height: number, operation: "grayscale" | "edge-canny", signal?: AbortSignal) {
if (source.length !== width * height * 4) throw new Error("图片像素尺寸不匹配");
if (operation === "grayscale") return grayscalePixels(source, width, height, signal);
return cannyPixels(source, width, height, signal);
}
function grayscalePixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
const output = new Uint8ClampedArray(source);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
const value = Math.round(source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114);
output[index] = value;
output[index + 1] = value;
output[index + 2] = value;
}
}
return output;
}
function cannyPixels(source: Uint8ClampedArray, width: number, height: number, signal?: AbortSignal) {
const gray = new Float32Array(width * height);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
gray[y * width + x] = source[index] * 0.299 + source[index + 1] * 0.587 + source[index + 2] * 0.114;
}
}
const blurred = gaussianBlur(gray, width, height, signal);
const magnitude = new Float32Array(width * height);
const direction = new Float32Array(width * height);
let maximum = 0;
for (let y = 1; y < height - 1; y += 1) {
throwIfAborted(signal);
for (let x = 1; x < width - 1; x += 1) {
const index = y * width + x;
const horizontal = blurred[index + 1] - blurred[index - 1];
const vertical = blurred[index + width] - blurred[index - width];
const value = Math.hypot(horizontal, vertical);
magnitude[index] = value;
direction[index] = Math.atan2(vertical, horizontal);
maximum = Math.max(maximum, value);
}
}
const thinned = nonMaximumSuppression(magnitude, direction, width, height, signal);
const high = maximum * 0.28;
const low = high * 0.45;
const edges = new Uint8Array(width * height);
const stack: number[] = [];
for (let index = 0; index < thinned.length; index += 1) {
if (thinned[index] >= high && high > 0) {
edges[index] = 2;
stack.push(index);
} else if (thinned[index] >= low && low > 0) {
edges[index] = 1;
}
}
while (stack.length) {
throwIfAborted(signal);
const index = stack.pop()!;
const x = index % width;
const y = Math.floor(index / width);
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
if (!offsetX && !offsetY) continue;
const nextX = x + offsetX;
const nextY = y + offsetY;
if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) continue;
const next = nextY * width + nextX;
if (edges[next] !== 1) continue;
edges[next] = 2;
stack.push(next);
}
}
}
const output = new Uint8ClampedArray(source.length);
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
const index = y * width + x;
const outputIndex = index * 4;
const value = edges[index] === 2 ? 0 : 255;
output[outputIndex] = value;
output[outputIndex + 1] = value;
output[outputIndex + 2] = value;
output[outputIndex + 3] = source[outputIndex + 3];
}
}
return output;
}
function gaussianBlur(source: Float32Array, width: number, height: number, signal?: AbortSignal) {
const output = new Float32Array(source.length);
const kernel = [1, 2, 1, 2, 4, 2, 1, 2, 1];
for (let y = 0; y < height; y += 1) {
throwIfAborted(signal);
for (let x = 0; x < width; x += 1) {
let value = 0;
let weight = 0;
for (let offsetY = -1; offsetY <= 1; offsetY += 1) {
for (let offsetX = -1; offsetX <= 1; offsetX += 1) {
const nextX = Math.min(width - 1, Math.max(0, x + offsetX));
const nextY = Math.min(height - 1, Math.max(0, y + offsetY));
const kernelWeight = kernel[(offsetY + 1) * 3 + offsetX + 1];
value += source[nextY * width + nextX] * kernelWeight;
weight += kernelWeight;
}
}
output[y * width + x] = value / weight;
}
}
return output;
}
function nonMaximumSuppression(magnitude: Float32Array, direction: Float32Array, width: number, height: number, signal?: AbortSignal) {
const output = new Float32Array(magnitude.length);
for (let y = 1; y < height - 1; y += 1) {
throwIfAborted(signal);
for (let x = 1; x < width - 1; x += 1) {
const index = y * width + x;
let angle = (direction[index] * 180) / Math.PI;
if (angle < 0) angle += 180;
const value = magnitude[index];
let before = 0;
let after = 0;
if (angle < 22.5 || angle >= 157.5) {
before = magnitude[index - 1];
after = magnitude[index + 1];
} else if (angle < 67.5) {
before = magnitude[index - width + 1];
after = magnitude[index + width - 1];
} else if (angle < 112.5) {
before = magnitude[index - width];
after = magnitude[index + width];
} else {
before = magnitude[index - width - 1];
after = magnitude[index + width + 1];
}
output[index] = value >= before && value >= after ? value : 0;
}
}
return output;
}
async function loadImage(sourceUrl: string, signal?: AbortSignal) {
if (!sourceUrl) throw new LocalImageConversionError("source_unreadable", "没有可读取的图片输入");
const image = new Image();
image.crossOrigin = "anonymous";
return await new Promise<HTMLImageElement>((resolve, reject) => {
const abort = () => {
image.src = "";
reject(new LocalImageConversionError("aborted", "转换已取消"));
};
if (signal?.aborted) return abort();
signal?.addEventListener("abort", abort, { once: true });
image.onload = () => {
signal?.removeEventListener("abort", abort);
resolve(image);
};
image.onerror = () => {
signal?.removeEventListener("abort", abort);
reject(new LocalImageConversionError("source_unreadable", "图片无法读取,可能是跨域资源或已失效"));
};
image.src = sourceUrl;
});
}
function canvasToBlob(canvas: HTMLCanvasElement, signal?: AbortSignal) {
throwIfAborted(signal);
return new Promise<Blob>((resolve, reject) => {
canvas.toBlob((blob) => {
if (blob) resolve(blob);
else reject(new LocalImageConversionError("browser_unsupported", "当前浏览器无法导出图片结果"));
}, "image/png");
});
}
function throwIfAborted(signal?: AbortSignal): asserts signal is AbortSignal | undefined {
if (signal?.aborted) throw new LocalImageConversionError("aborted", "转换已取消");
}
+1
View File
@@ -3,4 +3,5 @@ import "./prompt-optimizer";
import "./workflows"; import "./workflows";
import "./portrait-clearance"; import "./portrait-clearance";
import "./ai-art-critique"; import "./ai-art-critique";
import "./media-conversion";
import "./editor/editor-shell"; import "./editor/editor-shell";
@@ -0,0 +1,37 @@
import { MEDIA_CONVERSION_NODE_TYPE } from "@/lib/media-conversion/contracts";
import { registerPlugin } from "../plugin-registry";
import { PLUGIN_API_VERSION, type PluginManifest, type RegisteredPlugin } from "../plugin-types";
export const MEDIA_CONVERSION_PLUGIN_ID = "media-conversion";
const manifest: PluginManifest = {
apiVersion: PLUGIN_API_VERSION,
id: MEDIA_CONVERSION_PLUGIN_ID,
name: "媒体转换节点",
version: "0.1.0",
publishedAt: "2026-09-08",
updatedAt: "2026-09-08",
description: "在画布中提供图片转换节点,支持灰度、Canny 边缘、AI 线稿、本地 Depth Anything V2 深度图和 OpenPose 姿态骨架。",
documentation:
"# 媒体转换节点\n\n连接图片到转换节点后,可以在节点内选择灰度图、Canny 边缘、AI 线稿、深度图或姿态骨架。AI 线稿使用本机 ControlNet Aux 预处理模型,深度图使用本机已安装的 Depth Anything V2 Small 模型,姿态骨架使用 ControlNet Aux 的 OpenPose 人体预处理模型运行;这些操作都不会加载 Stable Diffusion 重绘管线。姿态转换只绘制人体骨架,未检测到人物时会跳过并提示。视频转换和透明抠图模型仍在验证,暂时会明确提示不可用。\n\n该插件只读取当前节点的媒体输入,并把结果保存回当前画布节点的本地素材存储。",
author: "影策团队",
surfaces: ["node"],
permissions: ["canvas.read", "canvas.write", "media.read"],
trusted: true,
runtime: { web: "declarative" },
contributes: {
transforms: [
{
id: MEDIA_CONVERSION_NODE_TYPE,
input: "media",
output: "media",
runtime: "declarative",
},
],
},
};
export const mediaConversionPlugin: RegisteredPlugin = { manifest };
registerPlugin(mediaConversionPlugin);
+4 -2
View File
@@ -103,8 +103,10 @@ export type PluginCanvasNodeContribution = {
defaultSize: { width: number; height: number }; defaultSize: { width: number; height: number };
schema: Record<string, unknown>; schema: Record<string, unknown>;
renderer: "declarative" | "sandbox"; renderer: "declarative" | "sandbox";
/** Optional input contract for nodes that consume one media kind. */ /** Optional input contract for nodes that consume one or more media kinds. */
acceptsInputKind?: "image" | "video" | "audio" | "text"; acceptsInputKind?: "image" | "video" | "audio" | "text" | Array<"image" | "video" | "audio" | "text">;
/** Optional maximum number of direct inputs. */
maxInputCount?: number;
/** Analysis/sink nodes can hide the right-side output connection. */ /** Analysis/sink nodes can hide the right-side output connection. */
showOutputConnection?: boolean; showOutputConnection?: boolean;
}; };
+2 -1
View File
@@ -10,6 +10,7 @@ import "@/lib/plugins/builtin";
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle"; import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer"; import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows"; import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion";
import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry"; import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry";
import type { PluginManifest, PluginManifestV2 } from "@/lib/plugins/plugin-types"; import type { PluginManifest, PluginManifestV2 } from "@/lib/plugins/plugin-types";
import { fetchAdminPlugins, setPluginPlatformAvailability, uninstallPlugin, uploadPlugin, type AdminPluginState, type BackendPlugin, type PluginManagement } from "@/services/api/plugins"; import { fetchAdminPlugins, setPluginPlatformAvailability, uninstallPlugin, uploadPlugin, type AdminPluginState, type BackendPlugin, type PluginManagement } from "@/services/api/plugins";
@@ -26,7 +27,7 @@ type AdminPluginItem = {
error?: string; error?: string;
}; };
const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance"]); const officialApplicationIds = new Set([RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", MEDIA_CONVERSION_PLUGIN_ID]);
export default function AdminPluginsPage() { export default function AdminPluginsPage() {
const { message, modal } = App.useApp(); const { message, modal } = App.useApp();
+20 -3
View File
@@ -72,7 +72,7 @@ import { CanvasLocalAgentPanel } from "@/components/canvas/canvas-local-agent-pa
import { useFocusMode } from "@/hooks/use-focus-mode"; import { useFocusMode } from "@/hooks/use-focus-mode";
import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store"; import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store";
import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays"; import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay, type PendingConnectionCreate } from "@/components/canvas/canvas-workspace-overlays";
import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer"; import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer";
import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer"; import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer";
import { CanvasFreeformEmptyState, CanvasLinkedProjectEmptyState, CanvasShortDramaEmptyState, CanvasShortDramaGuide, CanvasStoryInputNodeContent, CanvasStylePlaceholderNodeContent } from "@/components/canvas/canvas-short-drama-entry"; import { CanvasFreeformEmptyState, CanvasLinkedProjectEmptyState, CanvasShortDramaEmptyState, CanvasShortDramaGuide, CanvasStoryInputNodeContent, CanvasStylePlaceholderNodeContent } from "@/components/canvas/canvas-short-drama-entry";
@@ -1018,6 +1018,20 @@ function InfiniteCanvasPage() {
.filter((node) => selectedNodeIds.has(node.id) && !batchSourceRestriction(node)) .filter((node) => selectedNodeIds.has(node.id) && !batchSourceRestriction(node))
.map((node) => node.id), [nodes, selectedNodeIds]); .map((node) => node.id), [nodes, selectedNodeIds]);
const createConversionFromSource = useCallback((source: CanvasNodeData) => {
if (source.type !== CanvasNodeType.Image && source.type !== CanvasNodeType.Video) return;
const spec = getNodeSpec(CanvasNodeType.MediaConversion);
const pending: PendingConnectionCreate = {
connection: { nodeId: source.id, handleType: "source", anchorRatio: 0.5 },
position: {
x: source.position.x + source.width + 96 + spec.width / 2,
y: source.position.y + source.height / 2,
},
quick: true,
};
void createConnectedNode(CanvasNodeType.MediaConversion, pending);
}, [createConnectedNode]);
const handleCanvasSelectionStart = useCallback(() => { const handleCanvasSelectionStart = useCallback(() => {
setContextMenu(null); setContextMenu(null);
}, []); }, []);
@@ -1048,6 +1062,8 @@ function InfiniteCanvasPage() {
} else if (node.type === ART_CRITIQUE_NODE_TYPE) { } else if (node.type === ART_CRITIQUE_NODE_TYPE) {
setDialogNodeId(null); setDialogNodeId(null);
setArtCritiqueNodeId(node.id); setArtCritiqueNodeId(node.id);
} else if (node.type === CanvasNodeType.MediaConversion) {
setDialogNodeId(null);
} else { } else {
// 选择参考媒体时保留当前工作流配置面板,避免点击图片后配置“返回/消失”。 // 选择参考媒体时保留当前工作流配置面板,避免点击图片后配置“返回/消失”。
// 没有工作流配置面板时,媒体节点仍按原逻辑打开自己的面板。 // 没有工作流配置面板时,媒体节点仍按原逻辑打开自己的面板。
@@ -1065,7 +1081,7 @@ function InfiniteCanvasPage() {
const handleNodeDragEnd = useCallback((nodeId: string) => { const handleNodeDragEnd = useCallback((nodeId: string) => {
const node = nodesRef.current.find((item) => item.id === nodeId); const node = nodesRef.current.find((item) => item.id === nodeId);
if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) { if (!node || node.type === CanvasNodeType.Script || node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion || node.type === PORTRAIT_CLEARANCE_NODE_TYPE || node.type === ART_CRITIQUE_NODE_TYPE) {
setDialogNodeId(null); setDialogNodeId(null);
return; return;
} }
@@ -1895,7 +1911,7 @@ function InfiniteCanvasPage() {
const renderCanvasNodePanel = useCallback( const renderCanvasNodePanel = useCallback(
(panelNode: CanvasNodeData) => { (panelNode: CanvasNodeData) => {
if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing) return null; if (panelNode.type === CanvasNodeType.Script || panelNode.type === CanvasNodeType.Drawing || panelNode.type === CanvasNodeType.MediaConversion) return null;
return panelNode.type === CanvasNodeType.Config ? ( return panelNode.type === CanvasNodeType.Config ? (
<CanvasConfigComposer <CanvasConfigComposer
value={panelNode.metadata?.composerContent ?? panelNode.metadata?.prompt ?? ""} value={panelNode.metadata?.composerContent ?? panelNode.metadata?.prompt ?? ""}
@@ -2572,6 +2588,7 @@ function InfiniteCanvasPage() {
onUpload={(node) => handleUploadRequest(node.id)} onUpload={(node) => handleUploadRequest(node.id)}
onDownload={downloadNodeImage} onDownload={downloadNodeImage}
onSaveAsset={(node) => void saveNodeAsset(node)} onSaveAsset={(node) => void saveNodeAsset(node)}
onCreateConversion={createConversionFromSource}
onAnnotate={(node) => setAnnotationNodeId(node.id)} onAnnotate={(node) => setAnnotationNodeId(node.id)}
onMaskEdit={(node) => setMaskEditNodeId(node.id)} onMaskEdit={(node) => setMaskEditNodeId(node.id)}
onEmotion={(node) => { onEmotion={(node) => {
@@ -200,7 +200,7 @@ export function useCanvasConnectionController({
setContextMenu(null); setContextMenu(null);
}, [config, connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]); }, [config, connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]);
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => { const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
const nodeType = type; const nodeType = type;
if (nodeType === CanvasNodeType.Drawing && !isDrawingEngineAvailable(defaultDrawingEngine, tldrawLicenseKey)) { if (nodeType === CanvasNodeType.Drawing && !isDrawingEngineAvailable(defaultDrawingEngine, tldrawLicenseKey)) {
message.error("当前生产构建未配置 tldraw License Key,不能创建 tldraw 绘图"); message.error("当前生产构建未配置 tldraw License Key,不能创建 tldraw 绘图");
@@ -279,6 +279,12 @@ export function useCanvasConnectionController({
setConnecting(null); setConnecting(null);
return; return;
} }
if (batchSourceNodeIds.length && nodeType === CanvasNodeType.MediaConversion) {
message.error("转换节点一次只能连接一个输入,请先连接到普通节点");
closeConnectionCreateMenu();
setConnecting(null);
return;
}
const batchPlan = batchSourceNodeIds.length const batchPlan = batchSourceNodeIds.length
? planBatchConnections({ sourceNodeIds: batchSourceNodeIds, targetNodeId: newNode.id, nodes: [...nodesRef.current, newNode], connections: connectionsRef.current, config, allowCapacityOverflow: true }) ? planBatchConnections({ sourceNodeIds: batchSourceNodeIds, targetNodeId: newNode.id, nodes: [...nodesRef.current, newNode], connections: connectionsRef.current, config, allowCapacityOverflow: true })
: null; : null;
@@ -298,7 +304,7 @@ export function useCanvasConnectionController({
setConnections(nextConnections); setConnections(nextConnections);
setSelectedNodeIds(new Set([newNode.id])); setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null); setSelectedConnectionId(null);
if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id); if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id);
const skippedCount = batchPlan.skipped.length; const skippedCount = batchPlan.skipped.length;
const duplicateCount = batchPlan.duplicates.length; const duplicateCount = batchPlan.duplicates.length;
const suffix = skippedCount || duplicateCount ? `,跳过 ${skippedCount + duplicateCount} 个` : ""; const suffix = skippedCount || duplicateCount ? `,跳过 ${skippedCount + duplicateCount} 个` : "";
@@ -364,17 +370,18 @@ export function useCanvasConnectionController({
setSelectedNodeIds(new Set([newNode.id])); setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null); setSelectedConnectionId(null);
if (nodeType === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id); if (nodeType === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id);
else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio) setDialogNodeId(newNode.id); else if (nodeType !== CanvasNodeType.Text && nodeType !== CanvasNodeType.Script && nodeType !== CanvasNodeType.Audio && nodeType !== CanvasNodeType.MediaConversion) setDialogNodeId(newNode.id);
closeConnectionCreateMenu(); closeConnectionCreateMenu();
setConnecting(null); setConnecting(null);
}, [closeConnectionCreateMenu, config, connectionsRef, defaultDrawingEngine, message, nodesRef, projectId, runtimeStatuses, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, tldrawLicenseKey]); }, [closeConnectionCreateMenu, config, connectionsRef, defaultDrawingEngine, message, nodesRef, projectId, runtimeStatuses, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds, tldrawLicenseKey]);
const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => { const getConnectionCreateDisabledReason = useCallback((type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing | CanvasNodeType.Config | CanvasNodeType.MediaConversion, pending: PendingConnectionCreate, workflowProvider?: "runninghub" | "comfyui") => {
const nodeType = type; const nodeType = type;
if (nodeType === CanvasNodeType.Config) { if (nodeType === CanvasNodeType.Config) {
if (workflowProvider && !workflowProviderPluginEnabled(runtimeStatuses, workflowProvider)) return `${workflowProvider === "runninghub" ? "RunningHub" : "ComfyUI"} 工作流插件未启用`; if (workflowProvider && !workflowProviderPluginEnabled(runtimeStatuses, workflowProvider)) return `${workflowProvider === "runninghub" ? "RunningHub" : "ComfyUI"} 工作流插件未启用`;
} }
if (pending.batchSourceNodeIds?.length) { if (pending.batchSourceNodeIds?.length) {
if (nodeType === CanvasNodeType.MediaConversion) return "转换节点一次只能连接一个输入";
if (nodeType === CanvasNodeType.Drawing || nodeType === CanvasNodeType.Config) return "批量连接暂不支持此节点类型"; if (nodeType === CanvasNodeType.Drawing || nodeType === CanvasNodeType.Config) return "批量连接暂不支持此节点类型";
const pendingNode: CanvasNodeData = { id: "__pending-connection-node__", type: nodeType, title: "", position: pending.position, width: getNodeSpec(nodeType).width, height: getNodeSpec(nodeType).height }; const pendingNode: CanvasNodeData = { id: "__pending-connection-node__", type: nodeType, title: "", position: pending.position, width: getNodeSpec(nodeType).width, height: getNodeSpec(nodeType).height };
const plan = planBatchConnections({ sourceNodeIds: pending.batchSourceNodeIds, targetNodeId: pendingNode.id, nodes: [...nodesRef.current, pendingNode], connections: connectionsRef.current, config, allowCapacityOverflow: true }); const plan = planBatchConnections({ sourceNodeIds: pending.batchSourceNodeIds, targetNodeId: pendingNode.id, nodes: [...nodesRef.current, pendingNode], connections: connectionsRef.current, config, allowCapacityOverflow: true });
@@ -153,7 +153,7 @@ export function useCanvasNodeOperations({
if (workflowTitle) node.title = workflowTitle; if (workflowTitle) node.title = workflowTitle;
commitNodes([...nodesRef.current, node]); commitNodes([...nodesRef.current, node]);
selectNodes(new Set([node.id])); selectNodes(new Set([node.id]));
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id); if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Frame && type !== CanvasNodeType.Drawing && type !== CanvasNodeType.MediaConversion && type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(node.id);
}, [commitNodes, defaultDrawingEngine, effectiveConfig.comfyBridge.enabled, effectiveConfig.comfyBridge.workflows.length, effectiveConfig.runningHub.enabled, effectiveConfig.runningHub.workflows.length, getCanvasCenter, message, nodesRef, runtimeStatuses, selectNodes, setDialogNodeId, tldrawLicenseKey]); }, [commitNodes, defaultDrawingEngine, effectiveConfig.comfyBridge.enabled, effectiveConfig.comfyBridge.workflows.length, effectiveConfig.runningHub.enabled, effectiveConfig.runningHub.workflows.length, getCanvasCenter, message, nodesRef, runtimeStatuses, selectNodes, setDialogNodeId, tldrawLicenseKey]);
const createFolder = useCallback((position?: Position, linked?: { id: string; projectId: string; title: string; style: CanvasFolderStyle; theme: CanvasFolderTheme; createdAt: string }) => { const createFolder = useCallback((position?: Position, linked?: { id: string; projectId: string; title: string; style: CanvasFolderStyle; theme: CanvasFolderTheme; createdAt: string }) => {
@@ -400,7 +400,7 @@ export function useCanvasNodeOperations({
const sourceNode = sourceByTargetId.get(targetNode.id); const sourceNode = sourceByTargetId.get(targetNode.id);
if (sourceNode) cloneDrawingForNode(sourceNode, targetNode, "绘图副本保存失败,请重新复制"); if (sourceNode) cloneDrawingForNode(sourceNode, targetNode, "绘图副本保存失败,请重新复制");
}); });
if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id); if (!isFrameNode(source) && source.type !== CanvasNodeType.Drawing && source.type !== CanvasNodeType.MediaConversion && source.type !== PORTRAIT_CLEARANCE_NODE_TYPE) setDialogNodeId(id);
}, [cloneDrawingForNode, commitConnections, commitNodes, connectionsRef, nodesRef, selectNodes, setDialogNodeId]); }, [cloneDrawingForNode, commitConnections, commitNodes, connectionsRef, nodesRef, selectNodes, setDialogNodeId]);
const setPrimaryVersion = useCallback((nodeId: string) => { const setPrimaryVersion = useCallback((nodeId: string) => {
@@ -124,7 +124,7 @@ export function useCanvasViewportController({
const scale = Math.min(1.18, Math.max(viewportRef.current.k, 0.72)); const scale = Math.min(1.18, Math.max(viewportRef.current.k, 0.72));
transitionViewportTo({ x: size.width / 2 - (node.position.x + node.width / 2) * scale, y: size.height / 2 - (node.position.y + node.height / 2) * scale, k: scale }); transitionViewportTo({ x: size.width / 2 - (node.position.x + node.width / 2) * scale, y: size.height / 2 - (node.position.y + node.height / 2) * scale, k: scale });
selectFocusedNode(node.id); selectFocusedNode(node.id);
setDialogNodeId(node.type === CanvasNodeType.Drawing ? null : node.id); setDialogNodeId(node.type === CanvasNodeType.Drawing || node.type === CanvasNodeType.MediaConversion ? null : node.id);
}, [nodesRef, selectFocusedNode, setDialogNodeId, size.height, size.width, transitionViewportTo, viewportRef]); }, [nodesRef, selectFocusedNode, setDialogNodeId, size.height, size.width, transitionViewportTo, viewportRef]);
const setZoomScale = useCallback((scale: number) => { const setZoomScale = useCallback((scale: number) => {
+2 -1
View File
@@ -9,6 +9,7 @@ import "@/lib/plugins/builtin";
import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle"; import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle";
import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer"; import { PROMPT_OPTIMIZER_PLUGIN_ID } from "@/lib/plugins/builtin/prompt-optimizer";
import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows"; import { COMFYUI_PLUGIN_ID, RUNNINGHUB_PLUGIN_ID } from "@/lib/plugins/builtin/workflows";
import { MEDIA_CONVERSION_PLUGIN_ID } from "@/lib/plugins/builtin/media-conversion";
import { ART_CRITIQUE_PLUGIN_ID } from "@/lib/art-critique/contracts"; import { ART_CRITIQUE_PLUGIN_ID } from "@/lib/art-critique/contracts";
import type { PluginManifest, PluginManifestV2, RegisteredPlugin } from "@/lib/plugins/plugin-types"; import type { PluginManifest, PluginManifestV2, RegisteredPlugin } from "@/lib/plugins/plugin-types";
import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle"; import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle";
@@ -623,7 +624,7 @@ function toRegisteredPlugin(plugin: BackendPlugin): RegisteredPlugin {
} }
function isOfficialApplicationPlugin(pluginId: string) { function isOfficialApplicationPlugin(pluginId: string) {
return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID].includes(pluginId); return [RUNNINGHUB_PLUGIN_ID, COMFYUI_PLUGIN_ID, EAGLE_PLUGIN_ID, PROMPT_OPTIMIZER_PLUGIN_ID, "portrait-clearance", ART_CRITIQUE_PLUGIN_ID, MEDIA_CONVERSION_PLUGIN_ID].includes(pluginId);
} }
function pluginSourceLabel(plugin: RegisteredPlugin, state?: PluginState) { function pluginSourceLabel(plugin: RegisteredPlugin, state?: PluginState) {
+202
View File
@@ -0,0 +1,202 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import type { LocalRuntimeTransport } from "@/services/local-runtime";
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LocalDepthResult = {
blob: Blob;
width: number;
height: number;
modelId: string;
device: "cpu";
};
export type LocalDepthRuntimeStatus = {
ok: true;
module: "depth-estimation";
apiVersion: 1;
ready: boolean;
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
export async function runLocalDepthEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalDepthResult> {
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
const value = await requestJson<unknown>("/depth-estimation/run", {
method: "POST",
body: JSON.stringify({ schemaVersion: 1, operation: "depth", dataUrl }),
headers: { "content-type": "application/json" },
signal,
});
if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果响应无效");
}
return parseResult(value.result);
}
export async function readLocalDepthRuntimeStatus(signal?: AbortSignal) {
return parseStatus(await requestJson<unknown>("/depth-estimation/status", { method: "GET", signal }));
}
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
if (sourceUrl.startsWith("data:")) {
const match = DATA_URL_PATTERN.exec(sourceUrl);
if (!match) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = bytesFromBase64(match[2]!);
assertImageSize(bytes);
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
}
let response: Response;
try {
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
} catch (error) {
if (signal?.aborted) throw error;
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取");
}
if (!response.ok) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换输入图片无法读取", response.status);
const mimeType = imageMime(response.headers.get("content-type") || response.type);
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) {
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
}
const bytes = new Uint8Array(await response.arrayBuffer());
assertImageSize(bytes);
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
}
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
if (!response.ok) {
const code = isRecord(body) && typeof body.code === "string" ? body.code : "depth_runtime_unavailable";
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机深度运行时不可用";
throw new LocalRuntimeClientError(code, message, response.status);
}
return body as T;
}
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
let refreshed = false;
while (true) {
let response: Response;
try {
response = await (await ensureTransport()).request(path, init);
} catch (error) {
if (refreshed || !isSessionRefreshError(error)) throw error;
await reconnectTransport();
refreshed = true;
continue;
}
if (!refreshed && response.status === 401) {
await reconnectTransport();
refreshed = true;
continue;
}
if (response.status === 403) {
const body = await parseJsonResponse(response);
if (!refreshed && shouldRefreshSession(response, body)) {
await reconnectTransport();
refreshed = true;
continue;
}
return { response, body };
}
return { response };
}
}
async function ensureTransport(): Promise<LocalRuntimeTransport> {
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") await state.connect();
const current = useLocalRuntimeStore.getState();
if (current.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", current.error || "本机深度运行时不可用");
return getLocalRuntimeSessionClient();
}
async function reconnectTransport() {
const client = getLocalRuntimeSessionClient();
client.revokeLocalSession();
await useLocalRuntimeStore.getState().connect();
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") throw new LocalRuntimeClientError("depth_runtime_unavailable", state.error || "本机深度运行时不可用");
}
async function parseJsonResponse(response: Response) {
const text = await boundedText(response, MAX_RESPONSE_BYTES);
try {
return text ? JSON.parse(text) as unknown : {};
} catch {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应无效", response.status);
}
}
async function boundedText(response: Response, limit: number) {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status);
const text = await response.text();
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度响应过大", response.status);
return text;
}
function parseResult(value: Record<string, unknown>): LocalDepthResult {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果无效");
}
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机深度结果无效");
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度结果过大");
return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" };
}
function parseStatus(value: unknown): LocalDepthRuntimeStatus {
if (!isRecord(value) || value.ok !== true || value.module !== "depth-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机深度状态无效");
}
return value as LocalDepthRuntimeStatus;
}
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
throw new LocalRuntimeClientError("depth_input_invalid", "深度转换只支持 PNG、JPEG 或 WebP 图片", 400);
}
function assertImageSize(bytes: Uint8Array) {
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("depth_input_invalid", "深度转换图片不能超过 12MB", 400);
}
function bytesToBase64(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary);
}
function bytesFromBase64(value: string, code = "depth_input_invalid", message = "深度转换输入图片无效") {
let binary: string;
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "depth_input_invalid" ? 400 : 0); }
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
return bytes;
}
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
}
function shouldRefreshSession(response: Response, body: unknown) {
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}
+202
View File
@@ -0,0 +1,202 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import type { LocalRuntimeTransport } from "@/services/local-runtime";
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LocalLineartResult = {
blob: Blob;
width: number;
height: number;
modelId: string;
device: "cpu";
};
export type LocalLineartRuntimeStatus = {
ok: true;
module: "lineart-estimation";
apiVersion: 1;
ready: boolean;
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
reason?: string;
};
export async function runLocalLineartEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalLineartResult> {
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
const value = await requestJson<unknown>("/lineart-estimation/run", {
method: "POST",
body: JSON.stringify({ schemaVersion: 1, operation: "lineart", dataUrl }),
headers: { "content-type": "application/json" },
signal,
});
if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果响应无效");
}
return parseResult(value.result);
}
export async function readLocalLineartRuntimeStatus(signal?: AbortSignal) {
return parseStatus(await requestJson<unknown>("/lineart-estimation/status", { method: "GET", signal }));
}
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
if (sourceUrl.startsWith("data:")) {
const match = DATA_URL_PATTERN.exec(sourceUrl);
if (!match) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = bytesFromBase64(match[2]!);
assertImageSize(bytes);
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
}
let response: Response;
try {
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
} catch (error) {
if (signal?.aborted) throw error;
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取");
}
if (!response.ok) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿输入图片无法读取", response.status);
const mimeType = imageMime(response.headers.get("content-type") || response.type);
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) {
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
}
const bytes = new Uint8Array(await response.arrayBuffer());
assertImageSize(bytes);
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
}
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
if (!response.ok) {
const code = isRecord(body) && typeof body.code === "string" ? body.code : "lineart_runtime_unavailable";
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机 AI 线稿运行时不可用";
throw new LocalRuntimeClientError(code, message, response.status);
}
return body as T;
}
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
let refreshed = false;
while (true) {
let response: Response;
try {
response = await (await ensureTransport()).request(path, init);
} catch (error) {
if (refreshed || !isSessionRefreshError(error)) throw error;
await reconnectTransport();
refreshed = true;
continue;
}
if (!refreshed && response.status === 401) {
await reconnectTransport();
refreshed = true;
continue;
}
if (response.status === 403) {
const body = await parseJsonResponse(response);
if (!refreshed && shouldRefreshSession(response, body)) {
await reconnectTransport();
refreshed = true;
continue;
}
return { response, body };
}
return { response };
}
}
async function ensureTransport(): Promise<LocalRuntimeTransport> {
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") await state.connect();
const current = useLocalRuntimeStore.getState();
if (current.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", current.error || "本机 AI 线稿运行时不可用");
return getLocalRuntimeSessionClient();
}
async function reconnectTransport() {
const client = getLocalRuntimeSessionClient();
client.revokeLocalSession();
await useLocalRuntimeStore.getState().connect();
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") throw new LocalRuntimeClientError("lineart_runtime_unavailable", state.error || "本机 AI 线稿运行时不可用");
}
async function parseJsonResponse(response: Response) {
const text = await boundedText(response, MAX_RESPONSE_BYTES);
try {
return text ? JSON.parse(text) as unknown : {};
} catch {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应无效", response.status);
}
}
async function boundedText(response: Response, limit: number) {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status);
const text = await response.text();
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿响应过大", response.status);
return text;
}
function parseResult(value: Record<string, unknown>): LocalLineartResult {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || typeof value.pngBase64 !== "string") {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果无效");
}
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机 AI 线稿结果无效");
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿结果过大");
return { blob: new Blob([bytes], { type: "image/png" }), width: value.width, height: value.height, modelId: value.modelId, device: "cpu" };
}
function parseStatus(value: unknown): LocalLineartRuntimeStatus {
if (!isRecord(value) || value.ok !== true || value.module !== "lineart-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || (value.reason !== undefined && typeof value.reason !== "string")) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机 AI 线稿状态无效");
}
return value as LocalLineartRuntimeStatus;
}
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿转换只支持 PNG、JPEG 或 WebP 图片", 400);
}
function assertImageSize(bytes: Uint8Array) {
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("lineart_input_invalid", "AI 线稿图片不能超过 12MB", 400);
}
function bytesToBase64(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary);
}
function bytesFromBase64(value: string, code = "lineart_input_invalid", message = "AI 线稿输入图片无效") {
let binary: string;
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "lineart_input_invalid" ? 400 : 0); }
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
return bytes;
}
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
}
function shouldRefreshSession(response: Response, body: unknown) {
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}
+6 -3
View File
@@ -1,7 +1,7 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session"; import { LocalRuntimeClientError } from "@/services/local-runtime-session";
export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance"; export type LocalRuntimeModuleId = "canvas-agent" | "dreamina" | "portrait-clearance" | "depth-estimation" | "lineart-estimation" | "pose-estimation";
export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read"; export type LocalRuntimeScope = "runtime:status" | "runtime:revoke" | "canvas:connect" | "dreamina:status" | "dreamina:login" | "dreamina:logout" | "dreamina:run" | "dreamina:models" | "dreamina:generate" | "portrait:status" | "portrait:model" | "portrait:run" | "portrait:read" | "depth:status" | "depth:run" | "lineart:status" | "lineart:run" | "pose:status" | "pose:run";
export type LocalRuntimeModuleDescriptor = { export type LocalRuntimeModuleDescriptor = {
id: LocalRuntimeModuleId; id: LocalRuntimeModuleId;
@@ -28,6 +28,9 @@ const MODULE_SCOPES: Record<LocalRuntimeModuleId, ReadonlySet<LocalRuntimeScope>
"canvas-agent": new Set(["canvas:connect"]), "canvas-agent": new Set(["canvas:connect"]),
dreamina: new Set(["dreamina:status", "dreamina:login", "dreamina:logout", "dreamina:run", "dreamina:models", "dreamina:generate"]), dreamina: new Set(["dreamina:status", "dreamina:login", "dreamina:logout", "dreamina:run", "dreamina:models", "dreamina:generate"]),
"portrait-clearance": new Set(["portrait:status", "portrait:model", "portrait:run", "portrait:read"]), "portrait-clearance": new Set(["portrait:status", "portrait:model", "portrait:run", "portrait:read"]),
"depth-estimation": new Set(["depth:status", "depth:run"]),
"lineart-estimation": new Set(["lineart:status", "lineart:run"]),
"pose-estimation": new Set(["pose:status", "pose:run"]),
}; };
export async function readLocalRuntimeStatus(client: LocalRuntimeTransport, signal?: AbortSignal): Promise<LocalRuntimeStatus> { export async function readLocalRuntimeStatus(client: LocalRuntimeTransport, signal?: AbortSignal): Promise<LocalRuntimeStatus> {
@@ -136,7 +139,7 @@ function isRecord(value: unknown): value is Record<string, unknown> {
} }
function isModuleId(value: unknown): value is LocalRuntimeModuleId { function isModuleId(value: unknown): value is LocalRuntimeModuleId {
return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance"; return value === "canvas-agent" || value === "dreamina" || value === "portrait-clearance" || value === "depth-estimation" || value === "lineart-estimation" || value === "pose-estimation";
} }
function invalidResponse(status: number) { function invalidResponse(status: number) {
+209
View File
@@ -0,0 +1,209 @@
import { LocalRuntimeClientError } from "@/services/local-runtime-session";
import type { LocalRuntimeTransport } from "@/services/local-runtime";
import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store";
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const DATA_URL_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,([A-Za-z0-9+/=_-]+)$/;
export type LocalPoseResult = {
blob: Blob;
width: number;
height: number;
modelId: string;
device: "cpu";
backend: "openpose-body";
personCount: number;
people?: unknown[];
};
export type LocalPoseRuntimeStatus = {
ok: true;
module: "pose-estimation";
apiVersion: 1;
ready: boolean;
configured: boolean;
modelCached: boolean;
loaded: boolean;
device: "cpu";
modelId: string;
backend: "openpose-body";
reason?: string;
};
export async function runLocalPoseEstimation(sourceUrl: string, signal?: AbortSignal): Promise<LocalPoseResult> {
const dataUrl = await sourceToDataUrl(sourceUrl, signal);
const value = await requestJson<unknown>("/pose-estimation/run", {
method: "POST",
body: JSON.stringify({ schemaVersion: 1, operation: "pose", dataUrl }),
headers: { "content-type": "application/json" },
signal,
});
if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || !isRecord(value.result)) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果响应无效");
}
return parseResult(value.result);
}
export async function readLocalPoseRuntimeStatus(signal?: AbortSignal) {
return parseStatus(await requestJson<unknown>("/pose-estimation/status", { method: "GET", signal }));
}
async function sourceToDataUrl(sourceUrl: string, signal?: AbortSignal) {
if (sourceUrl.startsWith("data:")) {
const match = DATA_URL_PATTERN.exec(sourceUrl);
if (!match) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
const bytes = bytesFromBase64(match[2]!);
assertImageSize(bytes);
return `data:${match[1]};base64,${bytesToBase64(bytes)}`;
}
let response: Response;
try {
response = await fetch(sourceUrl, { credentials: "include", redirect: "error", cache: "no-store", signal });
} catch (error) {
if (signal?.aborted) throw error;
throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取");
}
if (!response.ok) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换输入图片无法读取", response.status);
const mimeType = imageMime(response.headers.get("content-type") || response.type);
const declaredLength = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
const bytes = new Uint8Array(await response.arrayBuffer());
assertImageSize(bytes);
return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
}
async function requestJson<T = unknown>(path: string, init: RequestInit = {}) {
const { response, body: responseBody } = await requestRuntimeResponse(path, init);
const body = responseBody === undefined ? await parseJsonResponse(response) : responseBody;
if (!response.ok) {
const code = isRecord(body) && typeof body.code === "string" ? body.code : "pose_runtime_unavailable";
const message = isRecord(body) && typeof body.message === "string" ? body.message : "本机姿态运行时不可用";
throw new LocalRuntimeClientError(code, message, response.status);
}
return body as T;
}
async function requestRuntimeResponse(path: string, init: RequestInit = {}) {
let refreshed = false;
while (true) {
let response: Response;
try {
response = await (await ensureTransport()).request(path, init);
} catch (error) {
if (refreshed || !isSessionRefreshError(error)) throw error;
await reconnectTransport();
refreshed = true;
continue;
}
if (!refreshed && response.status === 401) {
await reconnectTransport();
refreshed = true;
continue;
}
if (response.status === 403) {
const body = await parseJsonResponse(response);
if (!refreshed && shouldRefreshSession(response, body)) {
await reconnectTransport();
refreshed = true;
continue;
}
return { response, body };
}
return { response };
}
}
async function ensureTransport(): Promise<LocalRuntimeTransport> {
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") await state.connect();
const current = useLocalRuntimeStore.getState();
if (current.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", current.error || "本机姿态运行时不可用");
return getLocalRuntimeSessionClient();
}
async function reconnectTransport() {
const client = getLocalRuntimeSessionClient();
client.revokeLocalSession();
await useLocalRuntimeStore.getState().connect();
const state = useLocalRuntimeStore.getState();
if (state.connection !== "connected") throw new LocalRuntimeClientError("pose_runtime_unavailable", state.error || "本机姿态运行时不可用");
}
async function parseJsonResponse(response: Response) {
const text = await boundedText(response, MAX_RESPONSE_BYTES);
try { return text ? JSON.parse(text) as unknown : {}; } catch { throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应无效", response.status); }
}
async function boundedText(response: Response, limit: number) {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status);
const text = await response.text();
if (text.length > limit) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态响应过大", response.status);
return text;
}
function parseResult(value: Record<string, unknown>): LocalPoseResult {
if (value.mimeType !== "image/png" || !isPositiveInteger(value.width) || !isPositiveInteger(value.height) || typeof value.modelId !== "string" || value.device !== "cpu" || value.backend !== "openpose-body" || !isPositiveInteger(value.personCount) || typeof value.pngBase64 !== "string") {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果无效");
}
const bytes = bytesFromBase64(value.pngBase64, "runtime_response_invalid", "本机姿态结果无效");
if (!bytes.length || bytes.length > MAX_RESPONSE_BYTES) throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态结果过大");
return {
blob: new Blob([bytes], { type: "image/png" }),
width: value.width,
height: value.height,
modelId: value.modelId,
device: "cpu",
backend: "openpose-body",
personCount: value.personCount,
people: Array.isArray(value.people) ? value.people : undefined,
};
}
function parseStatus(value: unknown): LocalPoseRuntimeStatus {
if (!isRecord(value) || value.ok !== true || value.module !== "pose-estimation" || value.apiVersion !== 1 || typeof value.ready !== "boolean" || typeof value.configured !== "boolean" || typeof value.modelCached !== "boolean" || typeof value.loaded !== "boolean" || value.device !== "cpu" || typeof value.modelId !== "string" || value.backend !== "openpose-body" || (value.reason !== undefined && typeof value.reason !== "string")) {
throw new LocalRuntimeClientError("runtime_response_invalid", "本机姿态状态无效");
}
return value as LocalPoseRuntimeStatus;
}
function imageMime(value: string): "image/jpeg" | "image/png" | "image/webp" {
const mime = value.startsWith("image/") ? value.split(";", 1)[0] : "";
if (mime === "image/jpeg" || mime === "image/png" || mime === "image/webp") return mime;
throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换只支持 PNG、JPEG 或 WebP 图片", 400);
}
function assertImageSize(bytes: Uint8Array) {
if (!bytes.byteLength || bytes.byteLength > MAX_IMAGE_BYTES) throw new LocalRuntimeClientError("pose_input_invalid", "姿态转换图片不能超过 12MB", 400);
}
function bytesToBase64(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary);
}
function bytesFromBase64(value: string, code = "pose_input_invalid", message = "姿态转换输入图片无效") {
let binary: string;
try { binary = atob(value.replaceAll("-", "+").replaceAll("_", "/")); } catch { throw new LocalRuntimeClientError(code, message, code === "pose_input_invalid" ? 400 : 0); }
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
return bytes;
}
function isSessionRefreshError(error: unknown): error is LocalRuntimeClientError {
return error instanceof LocalRuntimeClientError && ["session_required", "session_invalid", "scope_denied"].includes(error.code);
}
function shouldRefreshSession(response: Response, body: unknown) {
return response.status === 401 || (response.status === 403 && isRecord(body) && body.code === "scope_denied");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === "object" && !Array.isArray(value));
}
function isPositiveInteger(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value > 0;
}
+4
View File
@@ -1,4 +1,5 @@
import type { CanvasColorGrade } from "@/lib/canvas/canvas-color-grade"; import type { CanvasColorGrade } from "@/lib/canvas/canvas-color-grade";
import type { MediaConversionNodeState } from "@/lib/media-conversion/contracts";
import type { AssetCategory } from "@/lib/asset-category"; import type { AssetCategory } from "@/lib/asset-category";
import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture"; import type { PortraitTextureSettings } from "@/lib/canvas/canvas-portrait-texture";
import type { StyleExecutionPlan } from "@/lib/canvas/style-profile"; import type { StyleExecutionPlan } from "@/lib/canvas/style-profile";
@@ -34,6 +35,7 @@ export enum CanvasNodeType {
Compare = "compare", Compare = "compare",
Chart = "chart", Chart = "chart",
ColorGrade = "colorgrade", ColorGrade = "colorgrade",
MediaConversion = "media-conversion",
} }
/** Runtime IDs contributed by plugins share the persisted node type field. */ /** Runtime IDs contributed by plugins share the persisted node type field. */
@@ -362,6 +364,8 @@ export type CanvasNodeMetadata = {
chartKind?: "bar" | "line"; chartKind?: "bar" | "line";
/** 调色节点的参数;缺省视为未调色。 */ /** 调色节点的参数;缺省视为未调色。 */
colorGrade?: CanvasColorGrade; colorGrade?: CanvasColorGrade;
/** 本地图片/视频转换节点的参数、来源指纹和结果状态。 */
mediaConversion?: MediaConversionNodeState;
/** 用户手动拉伸过尺寸;图片按真实比例自动适配时避让它。 */ /** 用户手动拉伸过尺寸;图片按真实比例自动适配时避让它。 */
manualSize?: boolean; manualSize?: boolean;
storyboard?: StoryboardData; storyboard?: StoryboardData;
+87
View File
@@ -0,0 +1,87 @@
import { describe, expect, test } from "bun:test";
import { canvasConnectionError } from "../src/lib/canvas/canvas-connection-policy";
import { findPendingMediaConversionInput } from "../src/components/canvas/canvas-node-generation";
import { getNodeAcceptedInputKinds, getNodeMaxInputCount, getNodeResourceKind } from "../src/lib/canvas/node-registry";
import { mediaConversionSourceFingerprint } from "../src/lib/media-conversion/contracts";
import { convertImageLocally, LocalImageConversionError, transformPixels } from "../src/lib/media-conversion/local-converter";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
import type { AiConfig } from "../src/stores/use-config-store";
function node(type: CanvasNodeType, id: string, metadata: CanvasNodeData["metadata"] = {}): CanvasNodeData {
return { id, type, title: id, position: { x: 0, y: 0 }, width: 380, height: 440, metadata };
}
const config = {} as AiConfig;
describe("本地转换节点注册", () => {
test("只允许一个图片或视频输入,完成后才作为图片素材", () => {
expect(getNodeAcceptedInputKinds(CanvasNodeType.MediaConversion)).toEqual(["image", "video"]);
expect(getNodeMaxInputCount(CanvasNodeType.MediaConversion)).toBe(1);
expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion"))).toBeNull();
expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion", {
mediaConversion: { schemaVersion: 1, operation: "grayscale", status: "completed", resultStorageKey: "image:scope:result" },
}))).toBe("image");
expect(getNodeResourceKind(node(CanvasNodeType.MediaConversion, "conversion", {
mediaConversion: { schemaVersion: 1, operation: "pose", status: "skipped", detectedPeople: 0 },
}))).toBeNull();
});
test("连接规则接受图片和视频,拒绝没有可读媒体的节点及第二个输入", () => {
const image = node(CanvasNodeType.Image, "image");
const video = node(CanvasNodeType.Video, "video");
const text = node(CanvasNodeType.Text, "text");
const conversion = node(CanvasNodeType.MediaConversion, "conversion");
const nodes = [image, video, text, conversion];
const candidate = (fromNodeId: string): CanvasConnection => ({ id: `candidate-${fromNodeId}`, fromNodeId, toNodeId: conversion.id });
expect(canvasConnectionError(config, nodes, [], candidate(image.id))).toBe("");
expect(canvasConnectionError(config, nodes, [], candidate(video.id))).toBe("");
expect(canvasConnectionError(config, nodes, [], candidate(text.id))).toContain("转换节点只接受图片或视频输入");
expect(canvasConnectionError(config, nodes, [{ id: "existing", fromNodeId: image.id, toNodeId: conversion.id }], candidate(video.id))).toContain("转换节点最多连接 1 个输入");
});
test("下游生成会阻止未完成或待更新的转换结果", () => {
const conversion = node(CanvasNodeType.MediaConversion, "conversion", {
mediaConversion: { schemaVersion: 1, operation: "depth", status: "unavailable" },
});
const input = node(CanvasNodeType.Image, "input", { content: "data:image/png;base64,source" });
const target = node(CanvasNodeType.Image, "target");
const connections: CanvasConnection[] = [
{ id: "input-conversion", fromNodeId: input.id, toNodeId: conversion.id },
{ id: "conversion-target", fromNodeId: conversion.id, toNodeId: target.id },
];
expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)?.id).toBe(conversion.id);
conversion.metadata = { mediaConversion: { schemaVersion: 1, operation: "grayscale", status: "completed", sourceNodeId: input.id, sourceFingerprint: "v1-unknown", resultStorageKey: "image:scope:result" } };
expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)?.id).toBe(conversion.id);
conversion.metadata.mediaConversion.sourceFingerprint = mediaConversionSourceFingerprint(input);
expect(findPendingMediaConversionInput(target.id, [input, conversion, target], connections)).toBeNull();
});
});
describe("图片本地转换算法", () => {
test("灰度转换保留 alpha 并使用感知亮度", () => {
const pixels = new Uint8ClampedArray([255, 0, 0, 200, 0, 255, 0, 100]);
const output = transformPixels(pixels, 2, 1, "grayscale");
expect(Array.from(output)).toEqual([76, 76, 76, 200, 150, 150, 150, 100]);
});
test("Canny 转换输出黑白边缘图并保留 alpha", () => {
const pixels = new Uint8ClampedArray([
0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255,
0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255,
0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255, 255,
]);
const output = transformPixels(pixels, 3, 3, "edge-canny");
for (let index = 0; index < output.length; index += 4) {
expect([0, 255]).toContain(output[index]);
expect(output[index]).toBe(output[index + 1]);
expect(output[index]).toBe(output[index + 2]);
expect(output[index + 3]).toBe(255);
}
});
test("浏览器算法转换把模型操作交给本机 Runtime", async () => {
await expect(convertImageLocally("", "depth")).rejects.toMatchObject<LocalImageConversionError>({ code: "model_missing" });
});
});