From af41893678749998c5ee2d777757b8ac7d4cafd5 Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Sun, 23 Aug 2026 23:19:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(*):=20=E6=8F=92=E4=BB=B6=E4=B8=AD=E5=BF=83?= =?UTF-8?q?=E4=B8=8E=E6=A8=A1=E5=9E=8B=E5=8D=8F=E8=AE=AE=20-=20=E5=AE=8C?= =?UTF-8?q?=E5=96=84=E5=A3=B0=E6=98=8E=E5=BC=8F=E5=8D=8F=E8=AE=AE=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E6=97=B6=E5=B9=B6=E5=A2=9E=E5=8A=A0=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E5=BC=80=E6=94=BE=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + backend/cmd/server/main.go | 2 + backend/internal/handler/auth.go | 65 +- backend/internal/handler/custom_proxy.go | 3 + backend/internal/handler/custom_proxy_test.go | 39 + backend/internal/handler/plugin.go | 126 +- backend/internal/handler/protocols.go | 35 + backend/internal/handler/security.go | 14 +- backend/internal/handler/security_test.go | 3 + .../handler/system_proxy_path_test.go | 25 + backend/internal/model/models.go | 2 + backend/internal/protocol/adapter.go | 67 ++ backend/internal/protocol/builtin.go | 876 ++++++++++++++ backend/internal/protocol/builtin_test.go | 370 ++++++ backend/internal/protocol/docs/agnes-video.md | 46 + backend/internal/protocol/docs/async-audio.md | 52 + .../internal/protocol/docs/chat-completion.md | 62 + backend/internal/protocol/docs/claude-api.md | 65 ++ .../internal/protocol/docs/gemini-image.md | 61 + backend/internal/protocol/docs/gemini-veo.md | 72 ++ backend/internal/protocol/docs/grok-image.md | 60 + .../internal/protocol/docs/minimax-video.md | 59 + .../protocol/docs/newapi-channel-1.md | 57 + .../protocol/docs/newapi-channel-2.md | 63 + backend/internal/protocol/docs/newapi.md | 66 ++ .../internal/protocol/docs/novita-video.md | 57 + .../internal/protocol/docs/openai-audio.md | 50 + .../internal/protocol/docs/openai-image.md | 73 ++ .../internal/protocol/docs/openai-response.md | 71 ++ .../protocol/docs/volcengine-ark-image.md | 61 + .../protocol/docs/volcengine-ark-video.md | 57 + .../protocol/docs/volcengine-jimeng-image.md | 57 + .../protocol/docs/volcengine-jimeng-video.md | 55 + backend/internal/protocol/docs/xai-video.md | 55 + backend/internal/protocol/documentation.go | 113 ++ backend/internal/protocol/errors.go | 19 + backend/internal/protocol/manifest.go | 474 ++++++++ .../internal/protocol/manifest_agent_test.go | 87 ++ backend/internal/protocol/registry.go | 134 +++ backend/internal/protocol/types.go | 140 +++ backend/internal/service/admin.go | 9 - backend/internal/service/channel_models.go | 43 +- backend/internal/service/coordination.go | 3 + .../internal/service/feature_availability.go | 42 +- .../service/feature_availability_test.go | 42 +- backend/internal/service/model_catalog.go | 2 + backend/internal/service/protocol_plugins.go | 399 +++++++ backend/internal/service/protocol_registry.go | 135 +++ .../internal/service/protocol_runtime_test.go | 114 ++ backend/internal/service/provider.go | 636 ++++++++++- .../internal/service/provider_claude_test.go | 39 + .../service/provider_resource_helpers.go | 22 + backend/internal/service/provider_test.go | 108 ++ backend/internal/service/service.go | 5 +- .../internal/service/storyboard_workflow.go | 1 + canvas-agent/README.md | 6 + canvas-agent/src/agents.ts | 4 + canvas-agent/src/canvas-context.ts | 273 +++++ canvas-agent/src/canvas-session.ts | 13 +- canvas-agent/src/config.ts | 23 +- canvas-agent/src/schemas.ts | 14 +- canvas-agent/src/tools.ts | 7 +- canvas-agent/test/canvas-context.test.ts | 44 + plugins/yingce/README.md | 25 + plugins/yingce/skills/README.md | 13 + .../skills/asset-aware-generation/SKILL.md | 15 + plugins/yingce/skills/canvas-context/SKILL.md | 16 + plugins/yingce/skills/canvas-editing/SKILL.md | 14 + plugins/yingce/skills/canvas/SKILL.md | 6 +- web/bun.lock | 1 + web/package.json | 1 + web/src/components/auth/require-feature.tsx | 11 +- .../canvas/canvas-assistant-panel.tsx | 27 +- .../canvas/canvas-local-agent-panel.tsx | 15 +- .../layout/workspace-sidebar-nav.tsx | 2 +- web/src/components/model-protocol-picker.tsx | 26 +- web/src/lib/canvas/canvas-agent-context.ts | 141 +++ web/src/lib/model-protocols.ts | 132 +-- web/src/lib/plugins/builtin/eagle.ts | 81 ++ web/src/lib/plugins/plugin-types.ts | 21 +- web/src/lib/user-session.ts | 5 +- web/src/pages/admin/admin-route-pages.tsx | 2 +- .../components/channel-model-manager.tsx | 25 +- .../components/feature-availability-panel.tsx | 10 +- web/src/pages/create/index.tsx | 4 +- web/src/pages/plugins/index.tsx | 626 +++++++--- .../pages/plugins/plugin-development-guide.md | 167 +++ .../plugins/plugin-documentation-modals.tsx | 92 ++ web/src/pages/plugins/plugin-documentation.ts | 35 + web/src/pages/plugins/plugin-markdown.tsx | 27 + web/src/pages/plugins/plugins.css | 659 ++++++++++- .../pages/settings/channel-video-pricing.tsx | 35 +- web/src/pages/tasks/index.tsx | 11 - web/src/router.tsx | 4 +- web/src/services/api/auth.ts | 2 +- web/src/services/api/custom-channel-relay.ts | 2 +- web/src/services/api/generation-task.ts | 15 +- web/src/services/api/image-protocols.ts | 34 + web/src/services/api/image-streaming.ts | 66 ++ web/src/services/api/image.ts | 10 +- web/src/services/api/logical-models.ts | 1 + web/src/services/api/plugins.ts | 56 + web/src/services/api/protocols.ts | 35 + web/src/stores/use-config-store.ts | 48 +- web/src/stores/use-user-store.ts | 4 + 模型接口协议插件化改造调研报告.md | 1017 +++++++++++++++++ 106 files changed, 8817 insertions(+), 500 deletions(-) create mode 100644 backend/internal/handler/protocols.go create mode 100644 backend/internal/handler/system_proxy_path_test.go create mode 100644 backend/internal/protocol/adapter.go create mode 100644 backend/internal/protocol/builtin.go create mode 100644 backend/internal/protocol/builtin_test.go create mode 100644 backend/internal/protocol/docs/agnes-video.md create mode 100644 backend/internal/protocol/docs/async-audio.md create mode 100644 backend/internal/protocol/docs/chat-completion.md create mode 100644 backend/internal/protocol/docs/claude-api.md create mode 100644 backend/internal/protocol/docs/gemini-image.md create mode 100644 backend/internal/protocol/docs/gemini-veo.md create mode 100644 backend/internal/protocol/docs/grok-image.md create mode 100644 backend/internal/protocol/docs/minimax-video.md create mode 100644 backend/internal/protocol/docs/newapi-channel-1.md create mode 100644 backend/internal/protocol/docs/newapi-channel-2.md create mode 100644 backend/internal/protocol/docs/newapi.md create mode 100644 backend/internal/protocol/docs/novita-video.md create mode 100644 backend/internal/protocol/docs/openai-audio.md create mode 100644 backend/internal/protocol/docs/openai-image.md create mode 100644 backend/internal/protocol/docs/openai-response.md create mode 100644 backend/internal/protocol/docs/volcengine-ark-image.md create mode 100644 backend/internal/protocol/docs/volcengine-ark-video.md create mode 100644 backend/internal/protocol/docs/volcengine-jimeng-image.md create mode 100644 backend/internal/protocol/docs/volcengine-jimeng-video.md create mode 100644 backend/internal/protocol/docs/xai-video.md create mode 100644 backend/internal/protocol/documentation.go create mode 100644 backend/internal/protocol/errors.go create mode 100644 backend/internal/protocol/manifest.go create mode 100644 backend/internal/protocol/manifest_agent_test.go create mode 100644 backend/internal/protocol/registry.go create mode 100644 backend/internal/protocol/types.go create mode 100644 backend/internal/service/protocol_plugins.go create mode 100644 backend/internal/service/protocol_registry.go create mode 100644 backend/internal/service/protocol_runtime_test.go create mode 100644 backend/internal/service/provider_claude_test.go create mode 100644 backend/internal/service/provider_resource_helpers.go create mode 100644 canvas-agent/src/canvas-context.ts create mode 100644 canvas-agent/test/canvas-context.test.ts create mode 100644 plugins/yingce/skills/README.md create mode 100644 plugins/yingce/skills/asset-aware-generation/SKILL.md create mode 100644 plugins/yingce/skills/canvas-context/SKILL.md create mode 100644 plugins/yingce/skills/canvas-editing/SKILL.md create mode 100644 web/src/lib/canvas/canvas-agent-context.ts create mode 100644 web/src/pages/plugins/plugin-development-guide.md create mode 100644 web/src/pages/plugins/plugin-documentation-modals.tsx create mode 100644 web/src/pages/plugins/plugin-documentation.ts create mode 100644 web/src/pages/plugins/plugin-markdown.tsx create mode 100644 web/src/services/api/plugins.ts create mode 100644 web/src/services/api/protocols.ts create mode 100644 模型接口协议插件化改造调研报告.md diff --git a/.gitignore b/.gitignore index 0d5087118..2dccf1f1b 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ assets/prototypes/.*.drawio.bkp .spec-workflow/ .vscode/ camera_language_guide.md +.superdesign diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index e2d033e3c..9bd9b83c9 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -98,10 +98,12 @@ func main() { handler.RegisterSkillRoutes(api, svc) handler.RegisterUserDataRoutes(api, svc) handler.RegisterPluginRoutes(api, svc) + handler.RegisterProtocolRoutes(api, svc) projectAPI := api.Group("") projectAPI.Use(handler.RequireFeature(svc, service.FeatureShortDrama)) handler.RegisterProjectRoutes(projectAPI, svc) handler.RegisterCanvasShareRoutes(api, svc) + r.NoRoute(handler.SystemProxyNoRouteHandler(svc)) log.Printf("影策 backend listening on %s", addr) if err := r.Run(addr); err != nil { diff --git a/backend/internal/handler/auth.go b/backend/internal/handler/auth.go index f7b221f76..1bea54155 100644 --- a/backend/internal/handler/auth.go +++ b/backend/internal/handler/auth.go @@ -736,13 +736,71 @@ func RegisterSystemProxyRoutes(r *gin.RouterGroup, svc *service.Service) { }) } +// SystemProxyNoRouteHandler handles the short public proxy form +// /api/{channelId}/{providerPath}. It deliberately runs from NoRoute so it +// cannot shadow existing business routes such as /api/tasks or /api/plugins. +func SystemProxyNoRouteHandler(svc *service.Service) gin.HandlerFunc { + return func(c *gin.Context) { + channelID, providerPath, ok := shortSystemProxyPath(c.Request.URL.Path) + if !ok { + fail(c, http.StatusNotFound, errors.New("请求不存在")) + return + } + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + channel, err := svc.SystemChannel(channelID) + if err != nil { + fail(c, http.StatusNotFound, errors.New("系统渠道不存在或已停用")) + return + } + proxySystemRequestPath(c, svc, user, channel, providerPath) + } +} + +func shortSystemProxyPath(rawPath string) (string, string, bool) { + const prefix = "/api/" + if !strings.HasPrefix(rawPath, prefix) { + return "", "", false + } + remainder := strings.TrimPrefix(rawPath, prefix) + separator := strings.IndexByte(remainder, '/') + if separator <= 0 || separator == len(remainder)-1 { + return "", "", false + } + channelID := remainder[:separator] + providerPath := remainder[separator:] + if strings.ContainsAny(channelID, "?#\\") || strings.Contains(providerPath, "\\") || isReservedAPIPathPrefix(channelID) { + return "", "", false + } + return channelID, providerPath, true +} + +// NoRoute also sees unmatched descendants of registered API routes. Keep the +// short proxy from interpreting /api/tasks/unknown (or /api/plugins/unknown) +// as a channel request when a business route returns 404. +func isReservedAPIPathPrefix(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "admin", "ai", "announcements", "assets", "auth", "canvas-projects", "channels", "features", "files", "model-catalog", "models", "oauth", "plugins", "projects", "protocols", "public", "resources", "sessions", "settings", "skills", "style-profiles", "tasks", "user-data", "voice-profiles", "wallet": + return true + default: + return false + } +} + func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User, channel *model.ModelChannel) { + proxySystemRequestPath(c, svc, user, channel, c.Param("path")) +} + +func proxySystemRequestPath(c *gin.Context, svc *service.Service, user *model.User, channel *model.ModelChannel, providerPath string) { startedAt := time.Now() policy, available := loadRuntimePolicy(c, svc) if !available || !enforceRateLimit(c, "system-proxy:"+user.ID, policy.Request.SystemRelayPerMinute, time.Minute) { return } - path := c.Param("path") + path := providerPath if path == "" { path = "/" } @@ -796,7 +854,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User, for _, key := range []string{"key", "api_key", "access_token", "token"} { query.Del(key) } - target := strings.TrimRight(channel.BaseURL, "/") + path + target := service.ChannelAPIURLForProtocol(channel.BaseURL, path, protocol) if encodedQuery := query.Encode(); encodedQuery != "" { target += "?" + encodedQuery } @@ -852,6 +910,9 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User, service.ApplyDefaultOutboundHeaders(upstreamReq) if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage { upstreamReq.Header.Set("x-goog-api-key", channel.APIKey) + } else if protocol == model.ChannelInterfaceClaudeAPI { + upstreamReq.Header.Set("x-api-key", channel.APIKey) + upstreamReq.Header.Set("anthropic-version", "2023-06-01") } else { upstreamReq.Header.Set("Authorization", "Bearer "+channel.APIKey) } diff --git a/backend/internal/handler/custom_proxy.go b/backend/internal/handler/custom_proxy.go index 63707b310..a0980c863 100644 --- a/backend/internal/handler/custom_proxy.go +++ b/backend/internal/handler/custom_proxy.go @@ -120,6 +120,9 @@ func proxyCustomRelayRequestWithService(c *gin.Context, policy service.RuntimeRe service.ApplyDefaultOutboundHeaders(upstreamReq) if apiFormat == "gemini" { upstreamReq.Header.Set("x-goog-api-key", apiKey) + } else if apiFormat == "claude" { + upstreamReq.Header.Set("x-api-key", apiKey) + upstreamReq.Header.Set("anthropic-version", "2023-06-01") } else { upstreamReq.Header.Set("Authorization", "Bearer "+apiKey) } diff --git a/backend/internal/handler/custom_proxy_test.go b/backend/internal/handler/custom_proxy_test.go index 8eb3705e9..528bbbbd2 100644 --- a/backend/internal/handler/custom_proxy_test.go +++ b/backend/internal/handler/custom_proxy_test.go @@ -126,6 +126,45 @@ func TestCustomRelayConvertsGeminiAuthentication(t *testing.T) { } } +func TestCustomRelayConvertsClaudeAuthentication(t *testing.T) { + gin.SetMode(gin.TestMode) + const apiKey = "claude-secret-key" + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/messages" { + t.Errorf("path = %q", r.URL.Path) + } + if r.Header.Get("x-api-key") != apiKey { + t.Errorf("x-api-key = %q", r.Header.Get("x-api-key")) + } + if r.Header.Get("Authorization") != "" { + t.Errorf("Authorization should not be forwarded, got %q", r.Header.Get("Authorization")) + } + if r.Header.Get("anthropic-version") != "2023-06-01" { + t.Errorf("anthropic-version = %q", r.Header.Get("anthropic-version")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"content":[{"type":"text","text":"claude relay works"}]}`) + })) + defer upstream.Close() + useCustomRelayTestClient(t, upstream.Client()) + t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1") + + request := httptest.NewRequest(http.MethodPost, "/api/ai/custom", strings.NewReader(`{"model":"claude-test","messages":[]}`)) + request.Header.Set("Authorization", "Bearer "+apiKey) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Canvas-Upstream-URL", upstream.URL+"/v1/messages") + request.Header.Set("X-Canvas-Upstream-Format", "claude") + markDesktopLocalRelayTestRequest(request, upstream.URL) + response := httptest.NewRecorder() + context, _ := gin.CreateTestContext(response) + context.Request = request + + proxyCustomRelayRequestWithCapabilities(context, defaultCustomRelayTestPolicy(), true) + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), "claude relay works") { + t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + } +} + func TestCustomRelayStreamsBeforeUpstreamCompletes(t *testing.T) { gin.SetMode(gin.TestMode) const apiKey = "stream-secret" diff --git a/backend/internal/handler/plugin.go b/backend/internal/handler/plugin.go index 6846e0a1d..457635d9a 100644 --- a/backend/internal/handler/plugin.go +++ b/backend/internal/handler/plugin.go @@ -1,16 +1,89 @@ package handler import ( + "io" "net/http" "strconv" + "strings" + "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/service" "github.com/gin-gonic/gin" ) func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { - r.GET("/plugins/eagle/library", func(c *gin.Context) { + pluginRoutes := r.Group("/plugins") + pluginRoutes.Use(requirePluginCenterAccess(svc)) + // The frontend plugin center is the single management surface. Protocol + // plugins are returned as kind=protocol records alongside UI plugins. + pluginRoutes.GET("", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + plugins, err := svc.PluginsForUser(user) + if err != nil { + failService(c, err) + return + } + ok(c, gin.H{"plugins": plugins}) + }) + pluginRoutes.POST("", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + if err := svc.RequireAdmin(user); err != nil { + failService(c, err) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 512<<10) + var data []byte + if strings.HasPrefix(strings.ToLower(c.GetHeader("Content-Type")), "multipart/form-data") { + file, _, fileErr := c.Request.FormFile("file") + if fileErr != nil { + fail(c, http.StatusBadRequest, fileErr) + return + } + defer file.Close() + data, err = io.ReadAll(io.LimitReader(file, 512<<10+1)) + } else { + data, err = io.ReadAll(io.LimitReader(c.Request.Body, 512<<10+1)) + } + if err != nil { + fail(c, http.StatusBadRequest, err) + return + } + plugin, err := svc.InstallPlugin(data) + if err != nil { + failService(c, err) + return + } + ok(c, gin.H{"plugin": plugin}) + }) + pluginRoutes.POST("/:id/enable", pluginToggle(svc, true)) + pluginRoutes.POST("/:id/disable", pluginToggle(svc, false)) + pluginRoutes.DELETE("/:id", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + if err := svc.RequireAdmin(user); err != nil { + failService(c, err) + return + } + if err := svc.UninstallPlugin(c.Param("id")); err != nil { + failService(c, err) + return + } + ok(c, gin.H{"deleted": true}) + }) + + pluginRoutes.GET("/eagle/library", func(c *gin.Context) { if _, err := currentUser(c, svc); err != nil { failService(c, err) return @@ -23,7 +96,7 @@ func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { library.LibraryPath = "" ok(c, gin.H{"library": library}) }) - r.GET("/plugins/eagle/items", func(c *gin.Context) { + pluginRoutes.GET("/eagle/items", func(c *gin.Context) { if _, err := currentUser(c, svc); err != nil { failService(c, err) return @@ -37,7 +110,7 @@ func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { } ok(c, gin.H{"items": items}) }) - r.GET("/plugins/eagle/items/:itemId/file", func(c *gin.Context) { + pluginRoutes.GET("/eagle/items/:itemId/file", func(c *gin.Context) { if _, err := currentUser(c, svc); err != nil { failService(c, err) return @@ -53,7 +126,7 @@ func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { c.Header("Content-Disposition", "attachment; filename=\""+file.Name+"\"") c.DataFromReader(http.StatusOK, file.Size, file.MimeType, file.Body, nil) }) - r.GET("/plugins/eagle/items/:itemId/thumbnail", func(c *gin.Context) { + pluginRoutes.GET("/eagle/items/:itemId/thumbnail", func(c *gin.Context) { if _, err := currentUser(c, svc); err != nil { failService(c, err) return @@ -68,7 +141,7 @@ func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { c.Header("X-Content-Type-Options", "nosniff") c.DataFromReader(http.StatusOK, file.Size, file.MimeType, file.Body, nil) }) - r.POST("/plugins/eagle/items", func(c *gin.Context) { + pluginRoutes.POST("/eagle/items", func(c *gin.Context) { if _, err := currentUser(c, svc); err != nil { failService(c, err) return @@ -86,7 +159,7 @@ func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { } ok(c, gin.H{"item": item}) }) - r.POST("/plugins/eagle/folders", func(c *gin.Context) { + pluginRoutes.POST("/eagle/folders", func(c *gin.Context) { if _, err := currentUser(c, svc); err != nil { failService(c, err) return @@ -107,3 +180,44 @@ func RegisterPluginRoutes(r *gin.RouterGroup, svc *service.Service) { ok(c, gin.H{"created": true}) }) } + +func requirePluginCenterAccess(svc *service.Service) gin.HandlerFunc { + return func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + c.Abort() + return + } + if user.Role == model.UserRoleAdmin { + c.Next() + return + } + if err := svc.RequireFeature(service.FeaturePluginCenter); err != nil { + failService(c, err) + c.Abort() + return + } + c.Next() + } +} + +func pluginToggle(svc *service.Service, enabled bool) gin.HandlerFunc { + return func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + if err := svc.RequireAdmin(user); err != nil { + failService(c, err) + return + } + plugin, err := svc.SetPluginEnabled(c.Param("id"), enabled) + if err != nil { + failService(c, err) + return + } + ok(c, gin.H{"plugin": plugin}) + } +} diff --git a/backend/internal/handler/protocols.go b/backend/internal/handler/protocols.go new file mode 100644 index 000000000..f1e2381b0 --- /dev/null +++ b/backend/internal/handler/protocols.go @@ -0,0 +1,35 @@ +package handler + +import ( + "strings" + + "infinite-canvas/backend/internal/service" + + "github.com/gin-gonic/gin" +) + +func RegisterProtocolRoutes(r *gin.RouterGroup, svc *service.Service) { + r.GET("/protocols", func(c *gin.Context) { + if _, err := currentUser(c, svc); err != nil { + failService(c, err) + return + } + scope := strings.TrimSpace(c.DefaultQuery("scope", "user.custom-channel")) + capability := strings.TrimSpace(c.Query("capability")) + ok(c, gin.H{"protocols": svc.ProtocolCatalog(scope, capability, false)}) + }) + r.GET("/admin/protocols", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + if err := svc.RequireAdmin(user); err != nil { + failService(c, err) + return + } + includeUnavailable := c.Query("includeUnavailable") == "1" + capability := strings.TrimSpace(c.Query("capability")) + ok(c, gin.H{"protocols": svc.ProtocolCatalog("admin.system-channel", capability, includeUnavailable)}) + }) +} diff --git a/backend/internal/handler/security.go b/backend/internal/handler/security.go index 65a8b62f2..d5d20c68d 100644 --- a/backend/internal/handler/security.go +++ b/backend/internal/handler/security.go @@ -36,7 +36,7 @@ var ( systemMiniMaxTaskPath = regexp.MustCompile(`^/v2/query/video_generation/[^/]+$`) openAIPostEndpoints = map[string]bool{ "/responses": true, "/chat/completions": true, "/images/generations": true, "/images/edits": true, - "/audio/speech": true, + "/audio/speech": true, "/messages": true, } ) @@ -61,7 +61,7 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont } apiFormat = strings.ToLower(strings.TrimSpace(apiFormat)) - if apiFormat != "openai" && apiFormat != "gemini" { + if apiFormat != "openai" && apiFormat != "gemini" && apiFormat != "claude" { return errors.New("自定义渠道调用格式无效") } if method == http.MethodGet { @@ -74,7 +74,7 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont allowed := requestPath == "/models" || strings.HasSuffix(requestPath, "/models") if apiFormat == "openai" { allowed = allowed || customVideoTaskPath.MatchString(requestPath) || customXAIVideoTaskPath.MatchString(requestPath) || customVideoContentPath.MatchString(requestPath) || customArkVideoTaskPath.MatchString(requestPath) || customMiniMaxTaskPath.MatchString(requestPath) - } else { + } else if apiFormat == "gemini" { allowed = allowed || customGeminiOperationPath.MatchString(requestPath) } if len(query) != 0 || !allowed { @@ -97,6 +97,12 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont } return nil } + if apiFormat == "claude" { + if mediaType != "application/json" || !strings.HasSuffix(requestPath, "/messages") { + return errors.New("Claude 自定义渠道只允许 application/json 的 /messages 请求") + } + return nil + } if mediaType != "application/json" { return errors.New("Gemini 自定义渠道生成请求必须使用 application/json") } @@ -209,6 +215,8 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP return requestPath == "/chat/completions" case model.ChannelInterfaceOpenAIResponse: return requestPath == "/responses" + case model.ChannelInterfaceClaudeAPI: + return requestPath == "/messages" case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage: return requestPath == "/images/generations" || requestPath == "/images/edits" case model.ChannelInterfaceVolcengineArkImage: diff --git a/backend/internal/handler/security_test.go b/backend/internal/handler/security_test.go index 57d9c1991..80145402d 100644 --- a/backend/internal/handler/security_test.go +++ b/backend/internal/handler/security_test.go @@ -36,6 +36,7 @@ func TestAuthorizeCustomRelayAllowsModelsAndAgentEndpoints(t *testing.T) { {method: http.MethodGet, target: "https://api.example.com/v1/models", apiFormat: "openai"}, {method: http.MethodPost, target: "https://api.example.com/v1/responses", apiFormat: "openai", contentType: "application/json"}, {method: http.MethodPost, target: "https://api.example.com/v1/chat/completions", apiFormat: "openai", contentType: "application/json; charset=utf-8"}, + {method: http.MethodPost, target: "https://api.anthropic.com/v1/messages", apiFormat: "claude", contentType: "application/json"}, {method: http.MethodPost, target: "https://api.example.com/v1/audio/speech", apiFormat: "openai", contentType: "application/json"}, {method: http.MethodPost, target: "https://api.example.com/v1/images/edits", apiFormat: "openai", contentType: "multipart/form-data; boundary=test"}, {method: http.MethodPost, target: "https://api.example.com/v1/images/edits", apiFormat: "openai", contentType: "application/json"}, @@ -77,6 +78,8 @@ func TestAuthorizeCustomRelayRejectsArbitraryRequestsAndCredentialQueries(t *tes {method: http.MethodGet, target: "https://api.example.com/account", apiFormat: "openai"}, {method: http.MethodGet, target: "https://api.example.com/v1/models?api_key=secret", apiFormat: "openai"}, {method: http.MethodPost, target: "https://api.example.com/v1/responses", apiFormat: "openai", contentType: "text/plain"}, + {method: http.MethodPost, target: "https://api.anthropic.com/v1/messages", apiFormat: "claude", contentType: "text/plain"}, + {method: http.MethodGet, target: "https://api.anthropic.com/v1/messages", apiFormat: "claude"}, {method: http.MethodPost, target: "https://api.example.com/v1/account", apiFormat: "openai", contentType: "multipart/form-data; boundary=test"}, {method: http.MethodPost, target: "https://api.example.com/v1/../account/chat/completions", apiFormat: "openai", contentType: "application/json"}, {method: http.MethodPost, target: "https://api.example.com/v1/models/gemini:streamGenerateContent?alt=sse&token=secret", apiFormat: "gemini", contentType: "application/json"}, diff --git a/backend/internal/handler/system_proxy_path_test.go b/backend/internal/handler/system_proxy_path_test.go new file mode 100644 index 000000000..ea031b99b --- /dev/null +++ b/backend/internal/handler/system_proxy_path_test.go @@ -0,0 +1,25 @@ +package handler + +import "testing" + +func TestShortSystemProxyPath(t *testing.T) { + tests := []struct { + path string + channel string + provider string + ok bool + }{ + {path: "/api/channel-1/chat/completions", channel: "channel-1", provider: "/chat/completions", ok: true}, + {path: "/api/channel-1/v1/models", channel: "channel-1", provider: "/v1/models", ok: true}, + {path: "/api/tasks/123", ok: false}, + {path: "/api/channel-1", ok: false}, + {path: "/api//chat/completions", ok: false}, + {path: "/other/channel-1/chat/completions", ok: false}, + } + for _, test := range tests { + channel, provider, ok := shortSystemProxyPath(test.path) + if ok != test.ok || channel != test.channel || provider != test.provider { + t.Errorf("shortSystemProxyPath(%q) = (%q, %q, %v), want (%q, %q, %v)", test.path, channel, provider, ok, test.channel, test.provider, test.ok) + } + } +} diff --git a/backend/internal/model/models.go b/backend/internal/model/models.go index 830f014bc..d3d280e07 100644 --- a/backend/internal/model/models.go +++ b/backend/internal/model/models.go @@ -64,6 +64,7 @@ const ( ChannelInterfaceChatCompletion ChannelInterfaceType = "chat-completion" ChannelInterfaceOpenAIResponse ChannelInterfaceType = "openai-response" + ChannelInterfaceClaudeAPI ChannelInterfaceType = "claude-api" ChannelInterfaceOpenAIImage ChannelInterfaceType = "openai-image" ChannelInterfaceGrokImage ChannelInterfaceType = "grok-image" ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image" @@ -80,6 +81,7 @@ const ( ChannelInterfaceGeminiVeo ChannelInterfaceType = "gemini-veo" ChannelInterfaceNovitaVideo ChannelInterfaceType = "novita-video" ChannelInterfaceMiniMaxVideo ChannelInterfaceType = "minimax-video" + ChannelInterfaceAgnesVideo ChannelInterfaceType = "agnes-video" ApiCallStatusSucceeded ApiCallStatus = "succeeded" ApiCallStatusFailed ApiCallStatus = "failed" diff --git a/backend/internal/protocol/adapter.go b/backend/internal/protocol/adapter.go new file mode 100644 index 000000000..8d4372cbd --- /dev/null +++ b/backend/internal/protocol/adapter.go @@ -0,0 +1,67 @@ +package protocol + +import "context" + +// Adapter owns only protocol translation. The host owns credentials, outbound +// security, polling leases, billing, result downloads and task recovery. +type Adapter interface { + Metadata() Metadata + BuildCreate(context.Context, RequestContext) (RequestSpec, error) + ParseCreate(context.Context, []byte) (CreateResult, error) + BuildPoll(context.Context, PollContext) (RequestSpec, error) + ParsePoll(context.Context, PollContext, []byte) (PollResult, error) + BuildCancel(context.Context, PollContext) (RequestSpec, error) +} + +// AgentAdapter is the optional protocol surface for tool-capable text calls. +// The host still owns credentials, outbound policy and billing; a plugin only +// maps the platform's agent request into the provider payload and parses the +// provider response back into the platform tool-call contract. +type AgentAdapter interface { + BuildAgent(context.Context, AgentRequestContext) (RequestSpec, error) + ParseAgent(context.Context, []byte) (AgentResult, error) +} + +type AgentRequestContext struct { + BaseURL string + Model string + Request map[string]any +} + +type AgentToolCall struct { + ID string + Name string + Arguments string +} + +type AgentResult struct { + Text string + Reasoning string + ToolCalls []AgentToolCall +} + +type UnavailableAdapter struct { + Info Metadata +} + +func (a UnavailableAdapter) Metadata() Metadata { return a.Info } + +func (a UnavailableAdapter) BuildCreate(context.Context, RequestContext) (RequestSpec, error) { + return RequestSpec{}, unavailable(a.Info) +} + +func (a UnavailableAdapter) ParseCreate(context.Context, []byte) (CreateResult, error) { + return CreateResult{}, unavailable(a.Info) +} + +func (a UnavailableAdapter) BuildPoll(context.Context, PollContext) (RequestSpec, error) { + return RequestSpec{}, unavailable(a.Info) +} + +func (a UnavailableAdapter) ParsePoll(context.Context, PollContext, []byte) (PollResult, error) { + return PollResult{}, unavailable(a.Info) +} + +func (a UnavailableAdapter) BuildCancel(context.Context, PollContext) (RequestSpec, error) { + return RequestSpec{}, unavailable(a.Info) +} diff --git a/backend/internal/protocol/builtin.go b/backend/internal/protocol/builtin.go new file mode 100644 index 000000000..19777a361 --- /dev/null +++ b/backend/internal/protocol/builtin.go @@ -0,0 +1,876 @@ +package protocol + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "sync" +) + +type builtinAdapter struct { + info Metadata + create func(GenerationRequest) (RequestSpec, error) + parseCreate func(map[string]any) (CreateResult, error) + poll func(PollContext) (RequestSpec, error) + parsePoll func(PollContext, map[string]any) (PollResult, error) + cancel func(PollContext) (RequestSpec, error) +} + +func (a builtinAdapter) Metadata() Metadata { return a.info } +func (a builtinAdapter) BuildCreate(_ context.Context, c RequestContext) (RequestSpec, error) { + if a.create == nil { + return RequestSpec{}, unavailable(a.info) + } + return a.create(c.Request) +} +func (a builtinAdapter) ParseCreate(_ context.Context, body []byte) (CreateResult, error) { + if a.parseCreate == nil { + return CreateResult{}, unavailable(a.info) + } + payload, err := decodeObject(body) + if err != nil { + return CreateResult{}, err + } + return a.parseCreate(payload) +} +func (a builtinAdapter) BuildPoll(_ context.Context, c PollContext) (RequestSpec, error) { + if a.poll == nil { + return RequestSpec{}, unavailable(a.info) + } + return a.poll(c) +} +func (a builtinAdapter) ParsePoll(_ context.Context, c PollContext, body []byte) (PollResult, error) { + if a.parsePoll == nil { + return PollResult{}, unavailable(a.info) + } + payload, err := decodeObject(body) + if err != nil { + return PollResult{}, err + } + return a.parsePoll(c, payload) +} +func (a builtinAdapter) BuildCancel(_ context.Context, c PollContext) (RequestSpec, error) { + if a.cancel == nil { + return RequestSpec{}, fmt.Errorf("protocol %s does not support cancellation", a.info.ID) + } + return a.cancel(c) +} + +var builtinRegistry *Registry +var builtinRegistryMu sync.Mutex + +// Builtins returns the immutable process-wide registry of shipped protocol plugins. +// The registry itself is safe for reads and can be extended by an installer during startup. +func Builtins() *Registry { + builtinRegistryMu.Lock() + defer builtinRegistryMu.Unlock() + if builtinRegistry != nil { + return builtinRegistry + } + registry, err := NewRegistry( + openAIChatAdapter(), openAIResponsesAdapter(), claudeAdapter(), + openAIImagesAdapter(), grokImagesAdapter(), arkImagesAdapter(), jimengImagesAdapter(), geminiImagesAdapter(), + openAIVideosAdapter(), newAPIChannel1Adapter(), newAPIVideosAdapter(), xAIVideosAdapter(), arkVideosAdapter(), jimengVideosAdapter(), geminiVeoAdapter(), novitaVideosAdapter(), miniMaxVideosAdapter(), + openAIAudioAdapter(), asyncAudioAdapter(), agnesAdapter(), + ) + if err != nil { + panic(err) + } + builtinRegistry = registry + return registry +} + +func allSurfaces() []Surface { + return []Surface{SurfaceAdminSystemChannel, SurfaceUserCustomChannel, SurfaceCanvas, SurfaceCreation, SurfaceAgent} +} + +func metadata(id, name, vendor string, capability Capability, create, poll, contentType string) Metadata { + return Metadata{ID: id, Version: "1.0.0", Name: name, Vendor: vendor, Categories: []Capability{capability}, Scopes: allSurfaces(), Create: create, Poll: poll, ContentType: contentType, Enabled: true, Installable: true} +} + +func mediaParams() []Parameter { + return []Parameter{{Name: "model", Type: "string", Required: true, Mapping: "model"}, {Name: "prompt", Type: "string", Required: true, Mapping: "prompt"}, {Name: "images", Type: "media[]", Mapping: "provider-specific"}, {Name: "imageCount", Type: "integer", Mapping: "provider-specific"}, {Name: "aspectRatio", Type: "string", Mapping: "provider-specific"}, {Name: "resolution", Type: "string", Mapping: "provider-specific"}, {Name: "quality", Type: "string", Mapping: "provider-specific"}} +} + +func videoParams() []Parameter { + return []Parameter{ + {Name: "model", Type: "string", Required: true, Mapping: "model", Description: "上游模型标识。"}, + {Name: "prompt", Type: "string", Required: true, Mapping: "prompt", Description: "视频生成提示词。"}, + {Name: "images", Type: "media[]", Mapping: "provider-specific", Description: "参考图片，由协议适配器转换为上游要求的 URL、Data URL 或 multipart 文件。"}, + {Name: "videos", Type: "media[]", Mapping: "provider-specific", Description: "参考视频；仅在上游协议支持时发送。"}, + {Name: "audios", Type: "media[]", Mapping: "provider-specific", Description: "参考音频或声音；仅在上游协议支持时发送。"}, + {Name: "duration", Type: "integer", Mapping: "provider-specific", Description: "视频时长，单位由上游协议决定。"}, + {Name: "aspectRatio", Type: "string", Mapping: "provider-specific", Description: "画幅比例或上游尺寸字段。"}, + {Name: "resolution", Type: "string", Mapping: "provider-specific", Description: "输出分辨率档位。"}, + {Name: "generateAudio", Type: "boolean", Mapping: "provider-specific", Description: "是否同时生成音频。"}, + {Name: "watermark", Type: "boolean", Mapping: "provider-specific", Description: "是否添加水印。"}, + } +} + +func openAIChatAdapter() Adapter { + info := metadata("chat-completion", "OpenAI Chat Completions", "OpenAI", CapabilityText, "POST /v1/chat/completions", "", "application/json") + info.Parameters = []Parameter{ + {Name: "model", Type: "string", Required: true, Mapping: "model", Description: "上游模型标识。"}, + {Name: "prompt", Type: "string", Required: true, Mapping: "messages[0].content", Description: "默认转换为一条 user message；extra.messages 可替换完整消息数组。"}, + {Name: "stream", Type: "boolean", Mapping: "stream", Description: "是否请求流式响应。"}, + } + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "messages": []any{map[string]any{"role": "user", "content": r.Prompt}}} + mergeExtra(body, r.Extra, "messages", "temperature", "top_p", "max_tokens", "stream", "tools", "response_format") + return jsonSpec(http.MethodPost, "/v1/chat/completions", body), nil + }, + parseCreate: parseChatResponse, + } +} + +func openAIResponsesAdapter() Adapter { + info := metadata("openai-response", "OpenAI Responses", "OpenAI", CapabilityText, "POST /v1/responses", "", "application/json") + info.Parameters = []Parameter{ + {Name: "model", Type: "string", Required: true, Mapping: "model", Description: "上游模型标识。"}, + {Name: "prompt", Type: "string", Required: true, Mapping: "input", Description: "默认作为 input 字符串；extra.input 可提供结构化输入。"}, + {Name: "stream", Type: "boolean", Mapping: "stream", Description: "是否请求流式响应。"}, + } + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + input := any(r.Prompt) + body := map[string]any{"model": r.Model, "input": input} + mergeExtra(body, r.Extra, "input", "instructions", "temperature", "top_p", "max_output_tokens", "stream", "tools", "text") + return jsonSpec(http.MethodPost, "/v1/responses", body), nil + }, + parseCreate: parseResponsesResponse, + } +} + +func claudeAdapter() Adapter { + info := metadata("claude-api", "Claude API", "Anthropic", CapabilityText, "POST /v1/messages", "", "application/json") + info.Parameters = []Parameter{ + {Name: "model", Type: "string", Required: true, Mapping: "model", Description: "Anthropic 模型标识。"}, + {Name: "prompt", Type: "string", Required: true, Mapping: "messages[0].content", Description: "默认转换为一条 user message；extra.messages 可替换完整消息数组。"}, + {Name: "stream", Type: "boolean", Mapping: "stream", Description: "是否请求流式响应。"}, + } + info.LegacyAliases = []string{"anthropic-messages", "claude-messages"} + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "max_tokens": 4096, "messages": []any{map[string]any{"role": "user", "content": r.Prompt}}} + mergeExtra(body, r.Extra, "max_tokens", "system", "messages", "temperature", "top_p", "top_k", "tools", "tool_choice", "stream") + spec := jsonSpec(http.MethodPost, "/v1/messages", body) + spec.Headers = map[string]string{"anthropic-version": "2023-06-01"} + return spec, nil + }, + parseCreate: parseClaudeResponse, + } +} + +func openAIImagesAdapter() Adapter { + info := metadata("openai-image", "OpenAI Images", "OpenAI", CapabilityImage, "POST /v1/images/generations", "", "application/json or multipart/form-data") + info.Parameters = mediaParams() + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "prompt": r.Prompt} + if r.ImageCount > 0 { + body["n"] = r.ImageCount + } + copyIf(body, "size", r.AspectRatio) + copyIf(body, "quality", r.Quality) + mergeExtra(body, r.Extra, "size", "quality", "background", "response_format", "output_format", "style", "n") + return jsonSpec(http.MethodPost, "/v1/images/generations", body), nil + }, + parseCreate: parseImageResponse, + } +} + +func grokImagesAdapter() Adapter { + info := metadata("grok-image", "Grok Images", "xAI", CapabilityImage, "POST /v1/images/generations", "", "application/json") + info.Parameters = mediaParams() + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "prompt": r.Prompt, "aspect_ratio": defaultValue(r.AspectRatio, "1:1"), "resolution": defaultValue(r.Quality, "2k")} + if len(r.Images) > 0 { + body["image"] = mediaValue(r.Images[0]) + } + return jsonSpec(http.MethodPost, "/v1/images/generations", body), nil + }, + parseCreate: parseImageResponse, + } +} + +func arkImagesAdapter() Adapter { + info := metadata("volcengine-ark-image", "火山方舟图片", "Volcengine Ark", CapabilityImage, "POST /api/v3/images/generations", "", "application/json") + info.Parameters = mediaParams() + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "prompt": r.Prompt} + if size := strings.TrimSpace(r.AspectRatio); size != "" { + body["size"] = size + } + if len(r.Images) > 0 { + body["image"] = mediaValues(r.Images) + } + mergeExtra(body, r.Extra, "size", "sequential_image_generation", "sequential_image_generation_options", "watermark") + return jsonSpec(http.MethodPost, "/api/v3/images/generations", body), nil + }, + parseCreate: parseImageResponse, + } +} + +func jimengImagesAdapter() Adapter { + info := metadata("volcengine-jimeng-image", "即梦官方图片", "Volcengine Jimeng", CapabilityImage, "POST CVSync2AsyncSubmitTask", "POST CVSync2AsyncGetResult", "application/json + AK/SK signature") + info.Parameters = mediaParams() + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"req_key": r.Model, "prompt": r.Prompt} + if len(r.Images) > 0 { + body["image_urls"] = mediaValues(r.Images) + } + mergeExtra(body, r.Extra, "req_key", "prompt", "image_urls", "seed", "width", "height") + return jsonSpec(http.MethodPost, "/CVSync2AsyncSubmitTask", body), nil + }, + parseCreate: parseJiMengCreate, + poll: func(c PollContext) (RequestSpec, error) { + return jsonSpec(http.MethodPost, "/CVSync2AsyncGetResult", map[string]any{"req_key": c.Model, "task_id": c.TaskID}), nil + }, + parsePoll: parseJiMengPoll, + } +} + +func geminiImagesAdapter() Adapter { + info := metadata("gemini-image", "Gemini Images", "Google", CapabilityImage, "POST /v1beta/models/{model}:generateContent", "", "application/json") + info.Parameters = mediaParams() + return builtinAdapter{info: info, + create: func(r GenerationRequest) (RequestSpec, error) { + parts := []any{map[string]any{"text": r.Prompt}} + for _, image := range r.Images { + if image.DataURL != "" { + parts = append(parts, map[string]any{"inline_data": map[string]any{"mime_type": dataMIME(image.DataURL), "data": dataPayload(image.DataURL)}}) + } else if image.URL != "" { + parts = append(parts, map[string]any{"file_data": map[string]any{"file_uri": image.URL}}) + } + } + body := map[string]any{"contents": []any{map[string]any{"role": "user", "parts": parts}}} + if r.AspectRatio != "" || r.Quality != "" { + body["generationConfig"] = map[string]any{"imageConfig": compactMap(map[string]any{"aspectRatio": r.AspectRatio, "imageSize": r.Quality})} + } + mergeExtra(body, r.Extra, "generationConfig", "safetySettings", "systemInstruction") + return jsonSpec(http.MethodPost, "/v1beta/models/"+url.PathEscape(r.Model)+":generateContent", body), nil + }, + parseCreate: parseGeminiImageResponse, + } +} + +func openAIVideosAdapter() Adapter { + info := metadata("newapi", "OpenAI Videos", "OpenAI compatible", CapabilityVideo, "POST /v1/videos", "GET /v1/videos/{task_id}", "multipart/form-data") + info.LegacyAliases = []string{"openai-video", "openai-videos"} + info.Parameters = openAIVideoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "prompt": r.Prompt} + if r.Duration > 0 { + body["seconds"] = strconv.Itoa(r.Duration) + } + if r.AspectRatio != "" { + body["size"] = r.AspectRatio + } + if len(r.Images) > 0 { + body["input_reference"] = mediaValues(r.Images) + } + return RequestSpec{Method: http.MethodPost, Path: "/v1/videos", ContentType: "multipart/form-data", Body: body}, nil + }) +} + +func openAIVideoParams() []Parameter { + return []Parameter{ + {Name: "model", Type: "string", Required: true, Mapping: "model", Description: "Sora 等视频模型标识。"}, + {Name: "prompt", Type: "string", Required: true, Mapping: "prompt", Description: "视频生成提示词。"}, + {Name: "images", Type: "media[]", Mapping: "input_reference", Description: "可选参考图，作为 multipart 文件上传到 input_reference。"}, + {Name: "duration", Type: "integer", Mapping: "seconds", Description: "视频时长，单位为秒。"}, + {Name: "aspectRatio", Type: "string", Mapping: "size", Description: "视频尺寸，例如 1280x720；该协议使用 size 而不是独立的 aspect_ratio。"}, + {Name: "videos", Type: "media[]", Mapping: "unsupported", Description: "OpenAI Videos 当前适配器不发送参考视频。"}, + {Name: "audios", Type: "media[]", Mapping: "unsupported", Description: "OpenAI Videos 当前适配器不发送参考音频。"}, + {Name: "resolution", Type: "string", Mapping: "unsupported", Description: "分辨率通过 size 表达。"}, + {Name: "generateAudio", Type: "boolean", Mapping: "unsupported", Description: "当前适配器不额外发送该字段。"}, + {Name: "watermark", Type: "boolean", Mapping: "unsupported", Description: "当前适配器不额外发送该字段。"}, + } +} + +func newAPIVideosAdapter() Adapter { + info := metadata("newapi-channel-2", "NewAPI Video Generations", "NewAPI", CapabilityVideo, "POST /v1/video/generations", "GET /v1/video/generations/{task_id}", "application/json") + info.LegacyAliases = []string{"newapi-video-generations"} + info.Parameters = videoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "prompt": r.Prompt, "seconds": strconv.Itoa(defaultInt(r.Duration, 6)), "aspect_ratio": defaultValue(r.AspectRatio, "16:9")} + copyIf(body, "resolution", r.Resolution) + if r.GenerateAudio { + body["generate_audio"] = true + } + if len(r.Images) > 0 { + body["image_urls"] = mediaValues(r.Images) + } + if len(r.Videos) > 0 { + body["video_urls"] = mediaValues(r.Videos) + } + if len(r.Audios) > 0 { + body["audio_urls"] = mediaValues(r.Audios) + } + return jsonSpec(http.MethodPost, "/v1/video/generations", body), nil + }) +} + +func newAPIChannel1Adapter() Adapter { + info := metadata("newapi-channel-1", "NewAPI 媒体任务", "NewAPI", CapabilityVideo, "POST /v1/videos", "GET /v1/videos/{task_id}", "application/json") + info.Parameters = videoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + media := make([]map[string]string, 0, len(r.Images)+len(r.Videos)+len(r.Audios)) + for _, image := range r.Images { + media = append(media, map[string]string{"type": "reference_image", "url": mediaValue(image)}) + } + for _, video := range r.Videos { + media = append(media, map[string]string{"type": "reference_video", "url": mediaValue(video)}) + } + for _, audio := range r.Audios { + media = append(media, map[string]string{"type": "reference_voice", "url": mediaValue(audio)}) + } + input := map[string]any{"prompt": r.Prompt} + if len(media) > 0 { + input["media"] = media + } + parameters := map[string]any{"resolution": defaultValue(r.Resolution, "720P"), "ratio": defaultValue(r.AspectRatio, "16:9"), "duration": defaultInt(r.Duration, 5)} + if r.Watermark { + parameters["watermark"] = true + } + return jsonSpec(http.MethodPost, "/v1/videos", map[string]any{"model": r.Model, "input": input, "parameters": parameters}), nil + }) +} + +func xAIVideosAdapter() Adapter { + info := metadata("xai-video", "xAI 官方视频", "xAI", CapabilityVideo, "POST /v1/videos/generations", "GET /v1/videos/{request_id}", "application/json") + info.Parameters = videoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model": r.Model, "prompt": r.Prompt, "duration": defaultInt(r.Duration, 6), "aspect_ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p")} + if len(r.Images) == 1 { + body["image"] = map[string]any{"url": mediaValue(r.Images[0])} + } else if len(r.Images) > 1 { + refs := make([]any, 0, len(r.Images)) + for _, image := range r.Images { + refs = append(refs, map[string]any{"url": mediaValue(image)}) + } + body["reference_images"] = refs + } + return jsonSpec(http.MethodPost, "/v1/videos/generations", body), nil + }) +} + +func arkVideosAdapter() Adapter { + info := metadata("volcengine-ark-video", "火山方舟视频", "Volcengine Ark", CapabilityVideo, "POST /api/v3/contents/generations/tasks", "GET /api/v3/contents/generations/tasks/{task_id}", "application/json") + info.Parameters = videoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + content := []any{map[string]any{"type": "text", "text": r.Prompt}} + for _, image := range r.Images { + content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"}) + } + body := map[string]any{"model": r.Model, "content": content, "ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p"), "duration": defaultInt(r.Duration, 5)} + if r.GenerateAudio { + body["generate_audio"] = true + } + if r.Watermark { + body["watermark"] = true + } + return jsonSpec(http.MethodPost, "/api/v3/contents/generations/tasks", body), nil + }) +} + +func jimengVideosAdapter() Adapter { + info := metadata("volcengine-jimeng-video", "即梦官方视频", "Volcengine Jimeng", CapabilityVideo, "POST CVSync2AsyncSubmitTask", "POST CVSync2AsyncGetResult", "application/json + AK/SK signature") + info.Parameters = videoParams() + return videoAdapterWithPoll(info, func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"req_key": r.Model, "prompt": r.Prompt} + if len(r.Images) > 0 { + body["image_urls"] = mediaValues(r.Images) + } + mergeExtra(body, r.Extra, "req_key", "prompt", "image_urls", "duration", "ratio", "resolution") + return jsonSpec(http.MethodPost, "/CVSync2AsyncSubmitTask", body), nil + }, func(c PollContext) (RequestSpec, error) { + return jsonSpec(http.MethodPost, "/CVSync2AsyncGetResult", map[string]any{"req_key": c.Model, "task_id": c.TaskID}), nil + }) +} + +func geminiVeoAdapter() Adapter { + info := metadata("gemini-veo", "Gemini Veo", "Google", CapabilityVideo, "POST /v1beta/models/{model}:predictLongRunning", "GET /v1beta/{operation_name}", "application/json") + info.Parameters = videoParams() + create := func(r GenerationRequest) (RequestSpec, error) { + instance := map[string]any{"prompt": r.Prompt} + if len(r.Images) > 0 { + image := r.Images[0] + instance["image"] = map[string]any{"inlineData": map[string]any{"mimeType": dataMIME(image.DataURL), "data": dataPayload(image.DataURL)}} + } + body := map[string]any{"instances": []any{instance}, "parameters": compactMap(map[string]any{"aspectRatio": r.AspectRatio, "durationSeconds": r.Duration, "resolution": r.Resolution, "generateAudio": r.GenerateAudio})} + return jsonSpec(http.MethodPost, "/v1beta/models/"+url.PathEscape(r.Model)+":predictLongRunning", body), nil + } + poll := func(c PollContext) (RequestSpec, error) { + operation := strings.TrimPrefix(c.TaskID, "/") + if !strings.HasPrefix(operation, "operations/") { + operation = "operations/" + operation + } + return RequestSpec{Method: http.MethodGet, Path: "/v1beta/" + operation}, nil + } + return builtinAdapter{info: info, create: create, parseCreate: parseGeminiVeoCreate, poll: poll, parsePoll: parseGeminiVeoPoll} +} + +func novitaVideosAdapter() Adapter { + info := metadata("novita-video", "Novita 视频", "Novita", CapabilityVideo, "POST /v3/video/create", "GET /v3/async/task-result?task_id={id}", "application/json") + info.Parameters = videoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + body := map[string]any{"model_name": r.Model, "prompt": r.Prompt, "duration": defaultInt(r.Duration, 5), "aspect_ratio": defaultValue(r.AspectRatio, "16:9")} + if len(r.Images) > 0 { + body["image_url"] = mediaValue(r.Images[0]) + } + return jsonSpec(http.MethodPost, "/v3/video/create", body), nil + }) +} + +func miniMaxVideosAdapter() Adapter { + info := metadata("minimax-video", "MiniMax 视频", "MiniMax", CapabilityVideo, "POST /v2/video_generation", "GET /v2/query/video_generation/{task_id}", "application/json") + info.Parameters = videoParams() + return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + content := []any{map[string]any{"type": "text", "text": r.Prompt}} + for _, image := range r.Images { + content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"}) + } + body := map[string]any{"model": r.Model, "prompt": r.Prompt, "duration": defaultInt(r.Duration, 6), "content": content} + mergeExtra(body, r.Extra, "model", "prompt", "duration", "content", "resolution", "aspect_ratio") + return jsonSpec(http.MethodPost, "/v2/video_generation", body), nil + }) +} + +func openAIAudioAdapter() Adapter { + info := metadata("openai-audio", "OpenAI Audio", "OpenAI", CapabilityAudio, "POST /v1/audio/speech", "", "application/json") + return builtinAdapter{info: info, create: func(r GenerationRequest) (RequestSpec, error) { + return jsonSpec(http.MethodPost, "/v1/audio/speech", map[string]any{"model": r.Model, "input": r.Prompt}), nil + }, parseCreate: parseAudioResponse} +} + +func asyncAudioAdapter() Adapter { + info := metadata("async-audio", "异步音频任务", "OpenAI compatible", CapabilityAudio, "POST /v1/audio/tasks", "GET /v1/audio/tasks/{task_id}", "application/json") + return asyncMediaAdapter(info, CapabilityAudio, func(r GenerationRequest) (RequestSpec, error) { + return jsonSpec(http.MethodPost, "/v1/audio/tasks", map[string]any{"model": r.Model, "prompt": r.Prompt}), nil + }) +} + +func agnesAdapter() Adapter { + info := metadata("agnes-video", "Agnes", "Agnes", CapabilityVideo, "", "", "") + info.Enabled = false + info.Installable = false + info.UnavailableReason = "未提供可验证的官方请求、鉴权、轮询和响应协议资料" + return UnavailableAdapter{Info: info} +} + +func videoAdapter(info Metadata, create func(GenerationRequest) (RequestSpec, error)) Adapter { + return asyncMediaAdapterWithPoll(info, CapabilityVideo, create, func(c PollContext) (RequestSpec, error) { + path := strings.TrimSpace(info.Poll) + if index := strings.IndexByte(path, ' '); index > 0 { + path = strings.TrimSpace(path[index+1:]) + } + path = strings.Replace(path, "{task_id}", url.PathEscape(c.TaskID), 1) + path = strings.Replace(path, "{request_id}", url.PathEscape(c.TaskID), 1) + path = strings.Replace(path, "{id}", url.QueryEscape(c.TaskID), 1) + return RequestSpec{Method: http.MethodGet, Path: path}, nil + }) +} + +func videoAdapterWithPoll(info Metadata, create func(GenerationRequest) (RequestSpec, error), poll func(PollContext) (RequestSpec, error)) Adapter { + return asyncMediaAdapterWithPoll(info, CapabilityVideo, create, poll) + +} + +func asyncMediaAdapter(info Metadata, capability Capability, create func(GenerationRequest) (RequestSpec, error)) Adapter { + return asyncMediaAdapterWithPoll(info, capability, create, func(c PollContext) (RequestSpec, error) { + path := strings.TrimSpace(info.Poll) + if index := strings.IndexByte(path, ' '); index > 0 { + path = strings.TrimSpace(path[index+1:]) + } + path = strings.Replace(path, "{task_id}", url.PathEscape(c.TaskID), 1) + return RequestSpec{Method: http.MethodGet, Path: path}, nil + }) +} + +func asyncMediaAdapterWithPoll(info Metadata, capability Capability, create func(GenerationRequest) (RequestSpec, error), poll func(PollContext) (RequestSpec, error)) Adapter { + return builtinAdapter{ + info: info, create: create, parseCreate: parseAsyncCreate, poll: poll, + parsePoll: func(c PollContext, payload map[string]any) (PollResult, error) { + return parseAsyncMediaPoll(c, payload, capability) + }, + } +} + +func jsonSpec(method, path string, body any) RequestSpec { + return RequestSpec{Method: method, Path: path, ContentType: "application/json", Body: body} +} + +func parseChatResponse(payload map[string]any) (CreateResult, error) { + choices, _ := payload["choices"].([]any) + if len(choices) == 0 { + return CreateResult{}, fmt.Errorf("chat response has no choices") + } + choice, _ := choices[0].(map[string]any) + message, _ := choice["message"].(map[string]any) + return CreateResult{Status: StatusSucceeded, Result: &Result{Text: firstString(message, "content"), Usage: object(payload["usage"])}}, nil +} + +func parseResponsesResponse(payload map[string]any) (CreateResult, error) { + text := firstString(payload, "output_text", "text") + if text == "" { + if output, ok := payload["output"].([]any); ok { + for _, item := range output { + if m, ok := item.(map[string]any); ok { + text = firstString(m, "text", "content") + if text != "" { + break + } + } + } + } + } + return CreateResult{Status: StatusSucceeded, Result: &Result{Text: text, Usage: object(payload["usage"])}}, nil +} + +func parseClaudeResponse(payload map[string]any) (CreateResult, error) { + content, _ := payload["content"].([]any) + text := "" + for _, item := range content { + if m, ok := item.(map[string]any); ok && firstString(m, "type") == "text" { + text += firstString(m, "text") + } + } + return CreateResult{Status: StatusSucceeded, Result: &Result{Text: text, Usage: object(payload["usage"])}}, nil +} + +func parseImageResponse(payload map[string]any) (CreateResult, error) { + values, _ := payload["data"].([]any) + images := make([]MediaReference, 0, len(values)) + for _, item := range values { + media, _ := item.(map[string]any) + if value := firstString(media, "url"); value != "" { + images = append(images, MediaReference{URL: value, Kind: "image"}) + continue + } + if value := firstString(media, "b64_json", "data"); value != "" { + if !strings.HasPrefix(value, "data:") { + value = "data:image/png;base64," + value + } + images = append(images, MediaReference{DataURL: value, Kind: "image"}) + } + } + if len(images) == 0 { + return CreateResult{}, fmt.Errorf("image response has no image data") + } + return CreateResult{Status: StatusSucceeded, Result: &Result{Images: images}}, nil +} + +func parseGeminiImageResponse(payload map[string]any) (CreateResult, error) { + images := make([]MediaReference, 0) + if candidates, ok := payload["candidates"].([]any); ok { + for _, candidate := range candidates { + if c, ok := candidate.(map[string]any); ok { + content, _ := c["content"].(map[string]any) + parts, _ := content["parts"].([]any) + for _, part := range parts { + if p, ok := part.(map[string]any); ok { + data, _ := p["inlineData"].(map[string]any) + if data == nil { + data, _ = p["inline_data"].(map[string]any) + } + if data != nil { + images = append(images, MediaReference{DataURL: "data:" + firstString(data, "mimeType", "mime_type") + ";base64," + firstString(data, "data")}) + } + } + } + } + } + } + if len(images) == 0 { + return CreateResult{}, fmt.Errorf("Gemini response has no inline image data") + } + return CreateResult{Status: StatusSucceeded, Result: &Result{Images: images}}, nil +} + +func parseJiMengCreate(payload map[string]any) (CreateResult, error) { + id := firstString(payload, "task_id", "taskId", "id") + if data := object(payload["data"]); data != nil { + id = defaultValue(firstString(data, "task_id", "taskId", "id"), id) + } + if id == "" { + return CreateResult{}, fmt.Errorf("Jimeng response has no task id") + } + return CreateResult{TaskID: id, Status: StatusPending}, nil +} +func parseJiMengPoll(c PollContext, payload map[string]any) (PollResult, error) { + return parseAsyncMediaPoll(c, payload, CapabilityImage) +} + +func parseGeminiVeoCreate(payload map[string]any) (CreateResult, error) { + id := firstString(payload, "name") + if id == "" { + return CreateResult{}, fmt.Errorf("Gemini Veo response has no operation name") + } + return CreateResult{TaskID: id, Status: StatusPending}, nil +} + +func parseGeminiVeoPoll(c PollContext, payload map[string]any) (PollResult, error) { + id := defaultValue(firstString(payload, "name"), c.TaskID) + if failure := object(payload["error"]); failure != nil { + message := firstString(failure, "message", "status") + if message == "" { + message = "Gemini Veo operation failed" + } + return PollResult{TaskID: id, Status: StatusFailed, Message: message}, nil + } + done, _ := payload["done"].(bool) + if !done { + return PollResult{TaskID: id, Status: StatusProcessing}, nil + } + response := object(payload["response"]) + videos := geminiVeoVideos(response) + if len(videos) == 0 { + return PollResult{TaskID: id, Status: StatusSucceeded}, nil + } + return PollResult{TaskID: id, Status: StatusSucceeded, Result: &Result{Videos: videos}}, nil +} + +func geminiVeoVideos(response map[string]any) []MediaReference { + if response == nil { + return nil + } + containers := []map[string]any{response} + for _, key := range []string{"generateVideoResponse", "generate_video_response"} { + if nested := object(response[key]); nested != nil { + containers = append(containers, nested) + } + } + result := make([]MediaReference, 0) + for _, container := range containers { + for _, key := range []string{"generatedVideos", "generated_videos", "generatedSamples", "generated_samples", "videos"} { + items, _ := container[key].([]any) + for _, item := range items { + entry := object(item) + video := object(entry["video"]) + if video == nil { + video = entry + } + if value := firstString(video, "uri", "url", "videoUri", "video_url"); value != "" { + result = append(result, MediaReference{URL: value, Kind: string(CapabilityVideo)}) + } + } + } + } + return result +} + +func parseAsyncCreate(payload map[string]any) (CreateResult, error) { + id := firstString(payload, "id", "task_id", "taskId", "request_id") + status := normalizeStatus(firstString(payload, "status", "state")) + if task := object(payload["task"]); task != nil { + id = defaultValue(firstString(task, "id", "task_id", "taskId", "request_id"), id) + status = normalizeStatus(defaultValue(firstString(task, "status", "state"), string(status))) + } + if data := object(payload["data"]); data != nil { + id = defaultValue(firstString(data, "id", "task_id", "taskId", "request_id"), id) + status = normalizeStatus(defaultValue(firstString(data, "status", "state"), string(status))) + } + if id == "" { + return CreateResult{}, fmt.Errorf("async response has no task id") + } + if status == "" { + status = StatusPending + } + return CreateResult{TaskID: id, Status: status, Message: firstString(payload, "message", "error")}, nil +} + +func parseAsyncMediaPoll(c PollContext, payload map[string]any, capability Capability) (PollResult, error) { + if capability != CapabilityImage && capability != CapabilityVideo && capability != CapabilityAudio { + return PollResult{}, fmt.Errorf("unsupported async media capability %q", capability) + } + status := normalizeStatus(firstString(payload, "status", "state", "task_status")) + if task := object(payload["task"]); task != nil { + status = normalizeStatus(defaultValue(firstString(task, "status", "state"), string(status))) + payload = mergeMaps(task, payload) + } + if data := object(payload["data"]); data != nil { + payload = mergeMaps(data, payload) + status = normalizeStatus(defaultValue(firstString(data, "status", "state"), string(status))) + } + if status == "" { + status = StatusProcessing + } + result := &Result{} + resultKind := string(capability) + references := mediaFromArray(payload[resultKind+"s"], capability) + references = append(references, mediaFromArray(payload["data"], capability)...) + if content := object(payload["content"]); content != nil { + references = append(references, mediaFromArray([]any{content}, capability)...) + } + keys := []string{resultKind + "_url", resultKind + "Url", "result_url", "url"} + if resultKind == "video" { + keys = append(keys, "object") + } + if value := firstString(payload, keys...); value != "" { + references = append(references, MediaReference{URL: value, Kind: resultKind}) + } + switch capability { + case CapabilityImage: + result.Images = references + case CapabilityAudio: + result.Audios = references + case CapabilityVideo: + result.Videos = references + } + if status == StatusSucceeded && len(references) == 0 { + result = nil + } + return PollResult{TaskID: defaultValue(firstString(payload, "id", "task_id", "taskId"), c.TaskID), Status: status, Result: result, Message: firstString(payload, "message", "error", "fail_reason")}, nil +} + +func parseAudioResponse(payload map[string]any) (CreateResult, error) { + value := firstString(payload, "url", "audio_url", "data") + if value == "" { + return CreateResult{}, fmt.Errorf("audio response has no result") + } + return CreateResult{Status: StatusSucceeded, Result: &Result{Audios: []MediaReference{{URL: value, Kind: "audio"}}}}, nil +} + +func normalizeStatus(raw string) Status { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "queued", "pending", "created", "submitted", "in_queue", "task_status_queued": + return StatusPending + case "running", "processing", "in_progress", "executing", "task_status_running": + return StatusProcessing + case "succeeded", "success", "completed", "complete", "done", "task_status_succeed": + return StatusSucceeded + case "failed", "failure", "error", "cancelled", "canceled", "expired": + return StatusFailed + default: + return "" + } +} + +func decodeObject(body []byte) (map[string]any, error) { + var payload map[string]any + if err := json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Errorf("protocol response is not JSON: %w", err) + } + return payload, nil +} +func object(value any) map[string]any { result, _ := value.(map[string]any); return result } +func mergeMaps(primary, fallback map[string]any) map[string]any { + result := make(map[string]any, len(fallback)+len(primary)) + for k, v := range fallback { + result[k] = v + } + for k, v := range primary { + result[k] = v + } + return result +} +func firstString(payload map[string]any, keys ...string) string { + for _, key := range keys { + if value, ok := payload[key].(string); ok && strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} +func defaultValue(value, fallback string) string { + if strings.TrimSpace(value) == "" { + return fallback + } + return value +} +func defaultInt(value, fallback int) int { + if value <= 0 { + return fallback + } + return value +} +func copyIf(body map[string]any, key, value string) { + if strings.TrimSpace(value) != "" { + body[key] = value + } +} +func mergeExtra(body map[string]any, extra map[string]any, keys ...string) { + for _, key := range keys { + if value, ok := extra[key]; ok { + body[key] = value + } + } +} +func compactMap(body map[string]any) map[string]any { + result := make(map[string]any) + for key, value := range body { + switch item := value.(type) { + case string: + if strings.TrimSpace(item) != "" { + result[key] = value + } + case int: + if item > 0 { + result[key] = value + } + case bool: + result[key] = value + case nil: + default: + result[key] = item + } + } + return result +} +func mediaValue(media MediaReference) string { return defaultValue(media.URL, media.DataURL) } +func mediaValues(values []MediaReference) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + if item := mediaValue(value); item != "" { + result = append(result, item) + } + } + return result +} +func mediaFromArray(value any, capability Capability) []MediaReference { + values, _ := value.([]any) + result := make([]MediaReference, 0, len(values)) + kind := string(capability) + for _, item := range values { + if value, ok := item.(string); ok && strings.TrimSpace(value) != "" { + result = append(result, MediaReference{URL: strings.TrimSpace(value), Kind: kind}) + continue + } + if m, ok := item.(map[string]any); ok { + if capability == CapabilityImage { + if value := firstString(m, "b64_json", "data"); value != "" { + if !strings.HasPrefix(value, "data:") { + value = "data:image/png;base64," + value + } + result = append(result, MediaReference{DataURL: value, Kind: kind}) + continue + } + } + if value := firstString(m, "url", "image_url", "video_url", "audio_url"); value != "" { + result = append(result, MediaReference{URL: value, Kind: kind}) + } + } + } + return result +} +func dataMIME(value string) string { + if index := strings.Index(value, ";"); strings.HasPrefix(value, "data:") && index > 5 { + return strings.TrimPrefix(value[5:index], "") + } + return "application/octet-stream" +} +func dataPayload(value string) string { + if index := strings.Index(value, ","); strings.HasPrefix(value, "data:") && index >= 0 { + return value[index+1:] + } + return value +} diff --git a/backend/internal/protocol/builtin_test.go b/backend/internal/protocol/builtin_test.go new file mode 100644 index 000000000..3200a76e5 --- /dev/null +++ b/backend/internal/protocol/builtin_test.go @@ -0,0 +1,370 @@ +package protocol + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" +) + +func TestBuiltinCatalogContainsRequestedProtocols(t *testing.T) { + registry := Builtins() + expected := []string{ + "chat-completion", "openai-response", "claude-api", + "openai-image", "grok-image", "volcengine-ark-image", "volcengine-jimeng-image", "gemini-image", + "newapi", "newapi-channel-2", "xai-video", "volcengine-ark-video", "volcengine-jimeng-video", "gemini-veo", "novita-video", "minimax-video", "agnes-video", + } + for _, id := range expected { + if _, ok := registry.Get(id); !ok { + t.Fatalf("missing builtin protocol %q", id) + } + } + if _, ok := registry.Resolve("openai-video"); !ok { + t.Fatal("legacy OpenAI video alias was not registered") + } + available := registry.List(SurfaceUserCustomChannel, CapabilityVideo, false) + for _, item := range available { + if item.ID == "agnes-video" { + t.Fatal("unavailable Agnes adapter was selectable") + } + } + all := registry.List(SurfaceAdminSystemChannel, CapabilityVideo, true) + var found bool + for _, item := range all { + if item.ID == "agnes-video" { + found = true + if item.UnavailableReason == "" || item.Enabled { + t.Fatalf("Agnes metadata must explain that it is disabled: %#v", item) + } + } + } + if !found { + t.Fatal("Agnes metadata was omitted from the administrator catalog") + } +} + +func TestTextAdaptersMapRequestsAndResponses(t *testing.T) { + cases := []struct { + id string + path string + body map[string]any + resp string + want string + }{ + {"chat-completion", "/v1/chat/completions", map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "hello"}}}, `{"choices":[{"message":{"content":"chat answer"}}]}`, "chat answer"}, + {"openai-response", "/v1/responses", map[string]any{"model": "o-test", "input": "hello"}, `{"output_text":"response answer"}`, "response answer"}, + {"claude-api", "/v1/messages", map[string]any{"model": "claude-test", "max_tokens": float64(4096)}, `{"content":[{"type":"text","text":"claude answer"}]}`, "claude answer"}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + adapter, ok := Builtins().Get(tc.id) + if !ok { + t.Fatal("adapter missing") + } + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: tc.body["model"].(string), Prompt: "hello"}}) + if err != nil { + t.Fatal(err) + } + if spec.Method != http.MethodPost || spec.Path != tc.path { + t.Fatalf("request = %#v", spec) + } + body, _ := json.Marshal(spec.Body) + var got map[string]any + if err := json.Unmarshal(body, &got); err != nil { + t.Fatal(err) + } + for key, value := range tc.body { + if key == "messages" { + continue + } + if got[key] == nil { + t.Fatalf("request missing %q: %#v", key, got) + } + _ = value + } + result, err := adapter.ParseCreate(context.Background(), []byte(tc.resp)) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusSucceeded || result.Result == nil || result.Result.Text != tc.want { + t.Fatalf("result = %#v", result) + } + }) + } +} + +func TestClaudeAdapterDeclaresAnthropicVersion(t *testing.T) { + adapter, _ := Builtins().Get("claude-api") + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: "claude-test", Prompt: "hello"}}) + if err != nil { + t.Fatal(err) + } + if spec.Headers["anthropic-version"] != "2023-06-01" { + t.Fatalf("headers = %#v", spec.Headers) + } +} + +func TestImageAndVideoAdaptersMapProviderShapes(t *testing.T) { + cases := []struct { + id, path, poll string + request GenerationRequest + }{ + {"openai-image", "/v1/images/generations", "", GenerationRequest{Model: "dall-e-test", Prompt: "a still", ImageCount: 2, AspectRatio: "1024x1024"}}, + {"grok-image", "/v1/images/generations", "", GenerationRequest{Model: "grok-imagine-image", Prompt: "a still", AspectRatio: "16:9"}}, + {"volcengine-ark-image", "/api/v3/images/generations", "", GenerationRequest{Model: "doubao-image", Prompt: "a still"}}, + {"volcengine-jimeng-image", "/CVSync2AsyncSubmitTask", "/CVSync2AsyncGetResult", GenerationRequest{Model: "jimeng_t2i_v40", Prompt: "a still"}}, + {"gemini-image", "/v1beta/models/gemini-image:generateContent", "", GenerationRequest{Model: "gemini-image", Prompt: "a still"}}, + {"newapi", "/v1/videos", "/v1/videos/video-1", GenerationRequest{Model: "video-model", Prompt: "a clip", Duration: 6}}, + {"newapi-channel-2", "/v1/video/generations", "/v1/video/generations/video-1", GenerationRequest{Model: "video-model", Prompt: "a clip", Duration: 6, AspectRatio: "16:9"}}, + {"xai-video", "/v1/videos/generations", "/v1/videos/video-1", GenerationRequest{Model: "grok-video", Prompt: "a clip"}}, + {"volcengine-ark-video", "/api/v3/contents/generations/tasks", "/api/v3/contents/generations/tasks/video-1", GenerationRequest{Model: "seedance", Prompt: "a clip"}}, + {"volcengine-jimeng-video", "/CVSync2AsyncSubmitTask", "/CVSync2AsyncGetResult", GenerationRequest{Model: "jimeng_video", Prompt: "a clip"}}, + {"gemini-veo", "/v1beta/models/veo-test:predictLongRunning", "/v1beta/operations/video-1", GenerationRequest{Model: "veo-test", Prompt: "a clip"}}, + {"novita-video", "/v3/video/create", "/v3/async/task-result?task_id=video-1", GenerationRequest{Model: "novita", Prompt: "a clip"}}, + {"minimax-video", "/v2/video_generation", "/v2/query/video_generation/video-1", GenerationRequest{Model: "MiniMax-H3", Prompt: "a clip"}}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + adapter, ok := Builtins().Get(tc.id) + if !ok { + t.Fatal("adapter missing") + } + create, err := adapter.BuildCreate(context.Background(), RequestContext{Request: tc.request}) + if err != nil { + t.Fatal(err) + } + if create.Path != tc.path { + t.Fatalf("create path = %q, want %q", create.Path, tc.path) + } + if tc.poll != "" { + poll, err := adapter.BuildPoll(context.Background(), PollContext{Model: tc.request.Model, TaskID: "video-1"}) + if err != nil { + t.Fatal(err) + } + if poll.Path != tc.poll { + t.Fatalf("poll path = %q, want %q", poll.Path, tc.poll) + } + } + }) + } +} + +func TestAsyncVideoResponseNormalizesStatusAndResult(t *testing.T) { + adapter, _ := Builtins().Get("minimax-video") + created, err := adapter.ParseCreate(context.Background(), []byte(`{"task":{"id":"mm-1","status":"processing"}}`)) + if err != nil { + t.Fatal(err) + } + if created.TaskID != "mm-1" || created.Status != StatusProcessing { + t.Fatalf("created = %#v", created) + } + polled, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: created.TaskID}, []byte(`{"task":{"id":"mm-1","status":"succeeded","content":{"url":"https://cdn.example/video.mp4"}}}`)) + if err != nil { + t.Fatal(err) + } + if polled.Status != StatusSucceeded || polled.Result == nil || len(polled.Result.Videos) != 1 || polled.Result.Videos[0].URL != "https://cdn.example/video.mp4" { + t.Fatalf("polled = %#v", polled) + } +} + +func TestAgnesIsExplicitlyUnavailable(t *testing.T) { + adapter, _ := Builtins().Get("agnes-video") + _, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: "agnes", Prompt: "test"}}) + if err == nil { + t.Fatal("Agnes unexpectedly accepted a request") + } + if _, ok := err.(UnavailableError); !ok { + t.Fatalf("error = %T %v", err, err) + } +} + +func TestDeclarativeManifestMapsFieldsAndResponses(t *testing.T) { + manifest := []byte(`{ + "apiVersion":"v1", + "metadata":{"id":"example-video","version":"1.0.0","name":"Example Video","vendor":"Example","categories":["video"],"scopes":["admin.system-channel","user.custom-channel"],"documentation":"# Example Video"}, + "create":{"method":"POST","path":"/v1/tasks","fields":{"model":"request.model","input.prompt":"request.prompt","input.seconds":"request.duration"}}, + "poll":{"method":"GET","path":"/v1/tasks/{{taskId}}"}, + "response":{"taskIdPaths":["id","data.id"],"statusPaths":["status","data.status"],"resultUrlPaths":["result.video_url","data.result.video_url"],"resultKind":"video"} + }`) + adapter, err := LoadManifest(manifest) + if err != nil { + t.Fatal(err) + } + create, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: "example", Prompt: "hello", Duration: 5}}) + if err != nil { + t.Fatal(err) + } + if create.Path != "/v1/tasks" || create.Method != http.MethodPost { + t.Fatalf("create = %#v", create) + } + body, _ := json.Marshal(create.Body) + var got map[string]any + if err := json.Unmarshal(body, &got); err != nil { + t.Fatal(err) + } + input := got["input"].(map[string]any) + if input["prompt"] != "hello" || input["seconds"].(float64) != 5 { + t.Fatalf("body = %#v", got) + } + poll, err := adapter.BuildPoll(context.Background(), PollContext{Model: "example", TaskID: "task-1"}) + if err != nil { + t.Fatal(err) + } + if poll.Path != "/v1/tasks/task-1" { + t.Fatalf("poll = %#v", poll) + } + result, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: "task-1"}, []byte(`{"data":{"id":"task-1","status":"succeeded","result":{"video_url":"https://cdn.example/clip.mp4"}}}`)) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusSucceeded || result.Result == nil || len(result.Result.Videos) != 1 || result.Result.Videos[0].URL != "https://cdn.example/clip.mp4" { + t.Fatalf("result = %#v", result) + } +} + +func TestDeclarativeManifestRequiresDocumentation(t *testing.T) { + manifest := Manifest{ + APIVersion: "v1", + Metadata: Metadata{ + ID: "missing-docs", Version: "1.0.0", Name: "Missing Docs", + Categories: []Capability{CapabilityVideo}, + Scopes: []Surface{SurfaceAdminSystemChannel}, + }, + Create: ManifestOperation{Method: http.MethodPost, Path: "/v1/tasks"}, + } + if err := ValidateManifest(manifest); err == nil { + t.Fatal("manifest without documentation was accepted") + } +} + +func TestBuiltinDocumentationMatchesTextRequestShape(t *testing.T) { + cases := []struct { + id, mapping, requestField string + }{ + {"chat-completion", "messages[0].content", `"messages"`}, + {"openai-response", "input", `"input"`}, + {"claude-api", "messages[0].content", `"messages"`}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + adapter, ok := Builtins().Get(tc.id) + if !ok { + t.Fatal("adapter missing") + } + metadata := adapter.Metadata() + AttachDocumentation(&metadata) + if !strings.Contains(metadata.Documentation, tc.mapping) || !strings.Contains(metadata.Documentation, tc.requestField) { + t.Fatalf("documentation does not match request shape:\n%s", metadata.Documentation) + } + }) + } +} + +func TestEveryBuiltinHasDetailedDocumentation(t *testing.T) { + requiredSections := []string{"## 接口", "## 模型", "## 参数", "## 官方", "## 影策运行时合同"} + for _, metadata := range Builtins().List("", "", true) { + t.Run(metadata.ID, func(t *testing.T) { + AttachDocumentation(&metadata) + document := metadata.Documentation + if len([]rune(document)) < 900 { + t.Fatalf("documentation is too short: %d runes", len([]rune(document))) + } + for _, section := range requiredSections { + if !strings.Contains(document, section) { + t.Errorf("documentation is missing section %q", section) + } + } + for _, operation := range []string{metadata.Create, metadata.Poll} { + parts := strings.SplitN(operation, " ", 2) + if len(parts) == 2 && !strings.Contains(document, parts[1]) { + t.Errorf("documentation is missing operation path %q", parts[1]) + } + } + for _, parameter := range metadata.Parameters { + if !strings.Contains(document, "`"+parameter.Name+"`") { + t.Errorf("documentation is missing parameter %q", parameter.Name) + } + } + if strings.Contains(document, "{{") { + t.Errorf("documentation contains an unresolved template marker") + } + }) + } +} + +func TestImageResponseKeepsBase64AsDataURL(t *testing.T) { + adapter, _ := Builtins().Get("openai-image") + result, err := adapter.ParseCreate(context.Background(), []byte(`{"data":[{"b64_json":"aW1hZ2U="}]}`)) + if err != nil { + t.Fatal(err) + } + if result.Result == nil || len(result.Result.Images) != 1 || result.Result.Images[0].URL != "" || result.Result.Images[0].DataURL != "data:image/png;base64,aW1hZ2U=" { + t.Fatalf("images = %#v", result.Result) + } +} + +func TestAsyncMediaPollKeepsResultKind(t *testing.T) { + cases := []struct { + id, payload, want string + }{ + {"volcengine-jimeng-image", `{"data":{"status":"completed","images":["https://cdn.example/image.png"]}}`, "image"}, + {"async-audio", `{"task":{"id":"audio-1","status":"completed","audio_url":"https://cdn.example/audio.mp3"}}`, "audio"}, + {"minimax-video", `{"task":{"id":"video-1","status":"succeeded","content":{"url":"https://cdn.example/video.mp4"}}}`, "video"}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + adapter, _ := Builtins().Get(tc.id) + result, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: "fallback"}, []byte(tc.payload)) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusSucceeded || result.Result == nil { + t.Fatalf("result = %#v", result) + } + switch tc.want { + case "image": + if len(result.Result.Images) != 1 || len(result.Result.Videos) != 0 || len(result.Result.Audios) != 0 { + t.Fatalf("media = %#v", result.Result) + } + case "audio": + if len(result.Result.Audios) != 1 || len(result.Result.Images) != 0 || len(result.Result.Videos) != 0 { + t.Fatalf("media = %#v", result.Result) + } + case "video": + if len(result.Result.Videos) != 1 || len(result.Result.Images) != 0 || len(result.Result.Audios) != 0 { + t.Fatalf("media = %#v", result.Result) + } + } + }) + } +} + +func TestGeminiVeoParsesLongRunningOperation(t *testing.T) { + adapter, _ := Builtins().Get("gemini-veo") + created, err := adapter.ParseCreate(context.Background(), []byte(`{"name":"operations/veo-1"}`)) + if err != nil { + t.Fatal(err) + } + if created.TaskID != "operations/veo-1" || created.Status != StatusPending { + t.Fatalf("created = %#v", created) + } + + result, err := adapter.ParsePoll(context.Background(), PollContext{TaskID: created.TaskID}, []byte(`{ + "name":"operations/veo-1", + "done":true, + "response":{"generateVideoResponse":{"generatedSamples":[{"video":{"uri":"https://cdn.example/veo.mp4"}}]}} + }`)) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusSucceeded || result.Result == nil || len(result.Result.Videos) != 1 || result.Result.Videos[0].URL != "https://cdn.example/veo.mp4" { + t.Fatalf("result = %#v", result) + } +} + +func TestAsyncMediaPollRejectsUnsupportedCapability(t *testing.T) { + if _, err := parseAsyncMediaPoll(PollContext{TaskID: "task-1"}, map[string]any{"status": "completed"}, CapabilityText); err == nil { + t.Fatal("text capability was accepted by async media parser") + } +} diff --git a/backend/internal/protocol/docs/agnes-video.md b/backend/internal/protocol/docs/agnes-video.md new file mode 100644 index 000000000..9e5843a38 --- /dev/null +++ b/backend/internal/protocol/docs/agnes-video.md @@ -0,0 +1,46 @@ +# Agnes 视频协议（当前不可用） + +Agnes 作为内置占位插件保留，是为了让历史模型配置能显示明确的不可用原因，而不是静默消失。当前仓库没有可验证的官方接口、鉴权、模型、轮询和响应资料，因此插件被强制禁用，也不可安装或选择。 + +## 接口与当前状态 + +{{OPERATIONS}} + +创建、轮询、取消路径均为空。调用 `BuildCreate` 会返回 `UnavailableError`：`未提供可验证的官方请求、鉴权、轮询和响应协议资料`。页面展示该条目不代表可以发送任务。 + +## 模型与能力状态 + +当前没有任何经过验证的模型 ID、能力范围或计费规则。任何 Agnes 模型名都不可提交；只有取得官方模型目录并完成真实账户验证后，才能写入渠道模型配置。 + +## 参数与字段状态 + +当前 metadata 没有参数映射，创建请求也不存在。`model`、`prompt`、参考素材、时长、比例、分辨率、音频和水印均不得推测或透传。 + +## 启用前必须补齐的事实 + +| 项目 | 必须取得的证据 | 验收方式 | +| --- | --- | --- | +| 服务入口 | 官方 Base URL、版本、区域 | 官方文档链接和最小健康探测 | +| 鉴权 | Header、Token/签名算法、权限范围 | 不在 URL/日志泄露密钥的集成测试 | +| 模型 | 精确模型 ID、能力和停用策略 | 真实账户模型发现或官方表 | +| 创建 | Method、Path、Content-Type、完整字段 | 请求 fixture 与真实 2xx 响应 | +| 轮询 | 任务 ID、状态枚举、失败结构 | queued -> running -> completed/failed fixture | +| 结果 | 视频 URL、文件接口、有效期 | 下载并存入影策资源的冒烟测试 | +| 取消与账务 | 是否支持取消、失败退款语义 | 官方合同与错误场景验证 | + +## 禁止的临时处理 + +- 不得根据插件名猜测 `/v1/videos`、模型名或 Bearer 鉴权。 +- 不得复用其他视频供应商的字段并把 200 响应当兼容。 +- 不得用空任务 ID、默认成功状态或假 URL 让页面“看起来能用”。 +- 不得仅在前端隐藏；后端 registry 和写路径必须继续拒绝不可用协议。 + +## 未来实现结构 + +取得官方资料后，应新增独立 adapter、专属 create/poll/result fixture、错误码映射和一份替换本文的完整文档。若协议实际是已有供应商的别名，应迁移历史配置到明确的 canonical ID，而不是永久保留第二套执行逻辑。 + +## 官方资料 + +当前未找到可验证且与本插件对应的官方开发者资料。为避免误导，本页不引用搜索聚合、论坛转述或其他同名产品文档。 + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/async-audio.md b/backend/internal/protocol/docs/async-audio.md new file mode 100644 index 000000000..6a89e05e4 --- /dev/null +++ b/backend/internal/protocol/docs/async-audio.md @@ -0,0 +1,52 @@ +# 异步音频任务兼容协议 + +该插件定义一个简化的 OpenAI 风格异步音频合同：JSON 创建任务，轮询任务资源，成功后读取媒体 URL。它不是 OpenAI 官方 Speech API，也不代表某个固定供应商。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/audio/tasks +GET {channel_base_url}/v1/audio/tasks/{task_id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型、声音与计费 + +模型 ID、任务类型、声音、语言、格式、时长限制、音色克隆权限和计费完全由实现该兼容接口的渠道决定。插件没有供应商白名单；管理员必须用该部署的接口文档校对。 + +## 参数与字段映射 + +{{PARAMETERS}} + +当前创建正文只包含： + +```json +{"model":"YOUR_ASYNC_AUDIO_MODEL","prompt":"低沉克制的男声旁白：风暴即将到来。"} +``` + +没有映射 `voice`、`format`、`speed`、参考音频、歌词或音乐结构。若模型依赖这些字段，当前插件不适用。 + +## 创建、轮询与结果 + +```bash +curl "{channel_base_url}/v1/audio/tasks" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"model":"YOUR_ASYNC_AUDIO_MODEL","prompt":"雨声环境音，逐渐增强"}' +``` + +创建响应需包含任务 ID，查询路径使用该 ID。轮询 parser 按音频类型提取 `audios`、`audio_url`、`audioUrl`、`result_url` 或 `url`，不会把音频误写入视频结果。若部署返回其他深层结构，必须增加该渠道 fixture 和明确映射。 + +## 错误与安全 + +任务不存在、模型不支持、输入过长、声音授权或内容审核失败应原样返回。涉及真人声音克隆时必须确认授权；参考音频和生成链接不得写入公开日志。结果 URL 应尽快转存。 + +## 官方资料与边界 + +- 本页以影策当前 adapter 为唯一字段事实源。 +- 渠道部署方必须提供真实的模型、声音、错误码和计费文档；没有这些资料时不能对用户承诺能力。 + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/chat-completion.md b/backend/internal/protocol/docs/chat-completion.md new file mode 100644 index 000000000..ad8db1c68 --- /dev/null +++ b/backend/internal/protocol/docs/chat-completion.md @@ -0,0 +1,62 @@ +# OpenAI Chat Completions + +Chat Completions 是基于 `messages` 的同步或流式文本协议。影策默认把提示词包装成一条 `user` 消息；需要多轮上下文、工具调用或结构化输出时，通过模型扩展参数传入完整 OpenAI 字段。 + +## 接口、鉴权与请求模式 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/chat/completions +Authorization: Bearer +Content-Type: application/json +``` + +官方服务的 Base URL 通常以 `/v1` 结尾；在影策中应以渠道实际配置为准。同步请求返回单个 JSON；`stream: true` 时上游返回 SSE，影策文本任务流负责转发和组装事件。 + +## 模型与兼容边界 + +模型名不会由插件维护白名单，必须填写渠道实际暴露的模型 ID。OpenAI 新项目更推荐 Responses API；现有 Chat Completions 模型和兼容渠道仍可使用本插件。兼容渠道可能只实现字段子集，不能因为路径相同就假定支持工具、视觉输入或 JSON Schema。 + +## 参数与字段映射 + +{{PARAMETERS}} + +可通过 `extra` 透传 `messages`、`temperature`、`top_p`、`max_tokens`、`stream`、`tools` 和 `response_format`。传入 `extra.messages` 时会替换默认单条消息，而不是追加。 + +## 完整请求示例 + +```bash +curl "{channel_base_url}/v1/chat/completions" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model": "YOUR_MODEL", + "messages": [ + {"role":"system","content":"你是影视分镜助理。"}, + {"role":"user","content":"把这场雨夜追逐拆成 6 个镜头。"} + ], + "temperature": 0.7 + }' +``` + +```json +{ + "id": "chatcmpl_xxx", + "choices": [{"message":{"role":"assistant","content":"1. 全景……"},"finish_reason":"stop"}], + "usage": {"prompt_tokens":32,"completion_tokens":95,"total_tokens":127} +} +``` + +## 响应解析、流式与错误 + +同步解析只读取 `choices[0].message.content`，并保留顶层 `usage`。当前解析器不把 `tool_calls` 当文本结果；只有工具调用而没有正文时，应由调用方进入工具回合，不能当作普通文本成功。流式分片、结束标记和断线续传由文本任务 SSE 链路处理，不使用此同步 JSON 解析器硬拼。 + +常见失败包括模型不存在、上下文超限、字段不受兼容服务支持、额度或速率限制。应保留上游 HTTP 状态和错误正文，不改写为成功。 + +## 官方资料 + +- [OpenAI Chat Completions API Reference](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create) +- [OpenAI Text generation guide](https://developers.openai.com/api/docs/guides/text) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/claude-api.md b/backend/internal/protocol/docs/claude-api.md new file mode 100644 index 000000000..1f32ea7d9 --- /dev/null +++ b/backend/internal/protocol/docs/claude-api.md @@ -0,0 +1,65 @@ +# Claude Messages API + +Claude Messages 使用无状态 `messages` 请求生成内容。影策默认创建一条 `user` 消息、设置 `max_tokens: 4096`，并自动添加 `anthropic-version: 2023-06-01`。系统提示使用顶层 `system`，不能伪装成 `system` 角色消息。 + +## 接口、鉴权与版本头 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/messages +x-api-key: +anthropic-version: 2023-06-01 +Content-Type: application/json +``` + +影策的通用渠道中转也可能以 Bearer 方式连接兼容网关；直连 Anthropic 时必须按官方 `x-api-key` 规则配置渠道。版本头由适配器固定添加，Beta 能力需要额外头部时当前插件不会自动猜测。 + +## 模型与兼容边界 + +模型 ID 必须使用账户和区域实际可用的 Anthropic 模型名。AWS Bedrock、Google Vertex AI 与 Anthropic 原生 API 的鉴权、路径和模型名不同，不能把原生 Messages 插件直接当作云厂商协议使用，除非中间网关已完成协议转换。 + +## 参数与字段映射 + +{{PARAMETERS}} + +`extra` 可透传 `max_tokens`、`system`、`messages`、`temperature`、`top_p`、`top_k`、`tools`、`tool_choice` 和 `stream`。`extra.messages` 替换默认消息数组。`max_tokens` 是必需的上游字段，未覆盖时影策发送 4096。 + +## 请求示例 + +```bash +curl "{channel_base_url}/v1/messages" \ + -H "x-api-key: " \ + -H "anthropic-version: 2023-06-01" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "YOUR_CLAUDE_MODEL", + "max_tokens": 1200, + "system": "你是影视剧本顾问。", + "messages": [{"role":"user","content":"指出这一场戏的冲突升级点。"}] + }' +``` + +```json +{ + "id":"msg_xxx", + "type":"message", + "role":"assistant", + "content":[{"type":"text","text":"冲突分为三个阶段……"}], + "stop_reason":"end_turn", + "usage":{"input_tokens":31,"output_tokens":148} +} +``` + +## 多内容块、工具与流式 + +响应解析遍历 `content[]`，只拼接 `type: text` 的块。`tool_use`、thinking、引用等非文本块不会被误当成正文，但当前统一文本结果也不会保存这些块；需要工具循环或完整块语义时应走专用 Agent 执行层。`stream: true` 的事件流由文本任务链路处理，同步解析器只接收完整 JSON。 + +常见错误包括缺少版本头、模型 ID 无权限、交替角色结构不合法、`max_tokens` 缺失或上下文超限。兼容网关可能接受 Bearer 但不接受 `x-api-key`，需要在渠道层确认，不在插件里双发密钥。 + +## 官方资料 + +- [Anthropic Messages API](https://platform.claude.com/docs/en/api/messages) +- [Anthropic Messages examples](https://platform.claude.com/docs/en/build-with-claude/working-with-messages) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/gemini-image.md b/backend/internal/protocol/docs/gemini-image.md new file mode 100644 index 000000000..1c43c1ee3 --- /dev/null +++ b/backend/internal/protocol/docs/gemini-image.md @@ -0,0 +1,61 @@ +# Gemini 图片生成 + +Gemini 图片插件调用 `generateContent`，把提示词和参考图组织为一个 `user` content。Data URL 被拆为 `inline_data`，外部 URL 被发送为 `file_data.file_uri`；响应从候选内容中的内联图片解析。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1beta/models/{model}:generateContent +x-goog-api-key: +Content-Type: application/json +``` + +通过后端渠道中转时，API Key 必须留在服务端头部，不能放进浏览器 URL。Vertex AI 使用不同 Base URL、OAuth 与资源路径，除非网关转换协议，否则不能直接复用此插件。 + +## 模型与输出模式 + +只有支持原生图片输出的 Gemini 模型才能返回内联图片。模型 ID 原样放进 URL 并进行路径转义。插件不维护模型白名单、图片尺寸表或计费，因为这些应由官方模型页和渠道模型配置共同确定。 + +## 参数与字段映射 + +{{PARAMETERS}} + +`aspectRatio -> generationConfig.imageConfig.aspectRatio`，`quality -> generationConfig.imageConfig.imageSize`。`extra` 可替换 `generationConfig`、`safetySettings` 和 `systemInstruction`；替换完整配置时调用方必须保留所需图片输出配置。 + +## 文生图请求 + +```bash +curl "{channel_base_url}/v1beta/models/YOUR_MODEL:generateContent" \ + -H "x-goog-api-key: " \ + -H "Content-Type: application/json" \ + -d '{ + "contents":[{"role":"user","parts":[{"text":"复古科幻电影海报，中文标题清晰"}]}], + "generationConfig":{"imageConfig":{"aspectRatio":"3:4","imageSize":"2K"}} + }' +``` + +## 参考图请求 + +```json +{ + "contents":[{"role":"user","parts":[ + {"text":"保持人物一致，改为冬季服装"}, + {"inline_data":{"mime_type":"image/png","data":""}} + ]}] +} +``` + +外部 URL 会被构造成 `file_data.file_uri`。该 URI 是否允许任意 HTTP URL 取决于上游；如果官方只接受 Files API URI，必须先上传文件，当前插件不会自动代传。 + +## 响应解析与安全 + +解析器遍历 `candidates[].content.parts[]`，识别 `inlineData` 或 `inline_data`，再生成 `data:;base64,...`。只有文本、finishReason 或安全反馈而没有内联图片时请求会失败。大图片不能进入 localStorage，应立即转存到影策资源存储。 + +## 官方资料 + +- [Gemini native image generation](https://ai.google.dev/gemini-api/docs/image-generation) +- [Gemini generateContent API](https://ai.google.dev/api/generate-content) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/gemini-veo.md b/backend/internal/protocol/docs/gemini-veo.md new file mode 100644 index 000000000..a78392d98 --- /dev/null +++ b/backend/internal/protocol/docs/gemini-veo.md @@ -0,0 +1,72 @@ +# Gemini Veo 长任务 + +Veo 插件通过 `predictLongRunning` 创建 Google long-running operation，再查询 operation 直到完成。提示词位于 `instances[0]`，当前最多取第一张参考图并以内联 Base64 发送。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1beta/models/{model}:predictLongRunning +GET {channel_base_url}/v1beta/{operation_name} +x-goog-api-key: +Content-Type: application/json +``` + +API Key 必须通过后端头部中转，不能放入浏览器 URL。Vertex AI Veo 的 OAuth、项目、区域和发布者路径不同，不能直接套用 Gemini Developer API Base URL。 + +## 模型与能力限制 + +模型 ID 原样放入创建路径。允许时长、比例、分辨率、音频、参考图类型、并发、结果保留期和计费以 Google 官方模型页及账户区域为准。插件不维护白名单。当前 adapter 只使用第一张图片；视频/音频参考和水印不发送。 + +## 参数与字段映射 + +{{PARAMETERS}} + +实际映射为 `aspectRatio`、`durationSeconds`、`resolution`、`generateAudio`，均放入 `parameters`。参考图固定为 `instances[0].image.inlineData`；没有 data URL 时 MIME 和数据会退化为空/通用值，因此远程 URL 应先下载并转为 data URL。 + +## 创建任务示例 + +```bash +curl "{channel_base_url}/v1beta/models/YOUR_VEO_MODEL:predictLongRunning" \ + -H "x-goog-api-key: " \ + -H "Content-Type: application/json" \ + -d '{ + "instances":[{ + "prompt":"夜间列车驶过雪原，航拍镜头缓慢下降", + "image":{"inlineData":{"mimeType":"image/png","data":""}} + }], + "parameters":{"aspectRatio":"16:9","durationSeconds":8,"resolution":"720p","generateAudio":true} + }' +``` + +## Operation 轮询与结果 + +创建响应返回 operation `name`。影策会移除开头斜杠，并在缺少 `operations/` 前缀时自动补齐，然后请求 `/v1beta/operations/...`。 + +```json +{"name":"operations/op_xxx","done":false} +``` + +该插件使用独立的 Google LRO parser，不经过通用 `status/state` 兼容分支：`done:false` 视为处理中，顶层 `error` 视为失败，`done:true` 后从 `response.generatedVideos[]` 或 `response.generateVideoResponse.generatedSamples[]` 的 `video.uri` 读取视频。成功但缺少视频地址会作为真实错误返回，不把 operation 完成误报为媒体已保存。 + +```json +{ + "name":"operations/op_xxx", + "done":true, + "response":{ + "generateVideoResponse":{ + "generatedSamples":[ + {"video":{"uri":"https://generativelanguage.googleapis.com/v1beta/files/..."}} + ] + } + } +} +``` + +## 官方资料 + +- [Gemini API video generation with Veo](https://ai.google.dev/gemini-api/docs/video) +- [Google long-running operations](https://ai.google.dev/api/files#method:-operations.get) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/grok-image.md b/backend/internal/protocol/docs/grok-image.md new file mode 100644 index 000000000..143a13898 --- /dev/null +++ b/backend/internal/protocol/docs/grok-image.md @@ -0,0 +1,60 @@ +# xAI Grok Images + +Grok Images 插件使用 OpenAI 风格的图片生成路径，但请求字段按 xAI 图片协议构造：`aspect_ratio`、`resolution`，并在有参考图时发送单个 `image`。它不是 OpenAI Images 插件的别名。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/images/generations +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与支持边界 + +模型名由 xAI 账户当前可用模型决定。插件不写死模型、价格、并发或内容限制。当前实现只发送第一张参考图；多参考图、编辑专用路径、mask 和批量生成没有实现，不能因上游可能支持就展示为已支持。 + +## 参数与字段映射 + +{{PARAMETERS}} + +实际构造为：`aspectRatio -> aspect_ratio`，为空时默认 `1:1`；`quality -> resolution`，为空时默认 `2k`；`images[0] -> image`。`imageCount`、`resolution` 平台字段及其他图片不会发送。 + +## 文生图请求 + +```bash +curl "{channel_base_url}/v1/images/generations" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_GROK_IMAGE_MODEL", + "prompt":"太空港的清晨，宽银幕概念设计", + "aspect_ratio":"16:9", + "resolution":"2k" + }' +``` + +## 单图参考请求 + +```json +{ + "model":"YOUR_GROK_IMAGE_MODEL", + "prompt":"保持主体轮廓，改成铅笔分镜风格", + "aspect_ratio":"16:9", + "resolution":"2k", + "image":"https://example.com/reference.png" +} +``` + +## 响应与错误 + +插件按 OpenAI Images 风格解析 `data[]` 中的 `url`、`b64_json` 或 `data`。如果 xAI 的实际响应结构发生变化或只返回异步任务，当前解析会失败而不是伪造结果。素材不可达、比例/分辨率不受模型支持、模型无权限和审核拒绝应原样暴露。 + +## 官方资料 + +- [xAI Image Generation](https://docs.x.ai/docs/guides/image-generation) +- [xAI API Reference](https://docs.x.ai/docs/api-reference) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/minimax-video.md b/backend/internal/protocol/docs/minimax-video.md new file mode 100644 index 000000000..39eed23b9 --- /dev/null +++ b/backend/internal/protocol/docs/minimax-video.md @@ -0,0 +1,59 @@ +# MiniMax 视频生成 + +MiniMax 视频插件使用 `/v2/video_generation` 创建异步任务。它同时发送顶层 `prompt` 和结构化 `content[]`：文本为 text item，参考图片为带 `reference_image` role 的 image URL item。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v2/video_generation +GET {channel_base_url}/v2/query/video_generation/{task_id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与约束 + +当前 MiniMax V2 官方文档列出 `MiniMax-H3`，时长为 4–15 秒整数，分辨率为 `768P` 或 `2K`，比例支持 `adaptive`、`21:9`、`16:9`、`4:3`、`1:1`、`3:4`、`9:16`。当前 adapter 默认 6 秒；模型名和其他限制仍以账户实际开放能力为准。 + +## 参数与字段映射 + +{{PARAMETERS}} + +每张图片构造为 `content` 中的 `image_url` item。`extra` 可覆盖 `model`、`prompt`、`duration`、`content`、`resolution`、`aspect_ratio`。参考视频、参考音频、音频生成和水印当前不发送。 + +## 创建任务示例 + +```bash +curl "{channel_base_url}/v2/video_generation" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_MINIMAX_VIDEO_MODEL", + "prompt":"保持角色一致，人物转身看向镜头", + "duration":6, + "content":[ + {"type":"text","text":"保持角色一致，人物转身看向镜头"}, + {"type":"image_url","image_url":{"url":"https://example.com/character.png"},"role":"reference_image"} + ], + "resolution":"720P", + "aspect_ratio":"16:9" + }' +``` + +## 查询、文件与结果 + +```bash +curl "{channel_base_url}/v2/query/video_generation/task_xxx" \ + -H "Authorization: Bearer " +``` + +当前 V2 官方成功响应在 `task.content.url` 返回视频地址，通用 parser 会先合并 `task`，再从 `content.url` 提取结果。官方仅允许查询最近 7 天任务，视频 URL 有时效，应立即转存。当前 adapter 尚未发送参考视频、参考音频、`callback_url` 或 `aigc_watermark`；虽然 V2 上游支持这些字段，影策不能宣称已支持。 + +## 官方资料 + +- [MiniMax 创建视频生成任务](https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create) +- [MiniMax 查询任务](https://platform.minimaxi.com/docs/api-reference/video-generation-v2-query) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/newapi-channel-1.md b/backend/internal/protocol/docs/newapi-channel-1.md new file mode 100644 index 000000000..4039a03c3 --- /dev/null +++ b/backend/internal/protocol/docs/newapi-channel-1.md @@ -0,0 +1,57 @@ +# NewAPI 媒体任务协议 + +该插件把多模态参考素材统一包装为 `input.media[]`，把分辨率、比例、时长和水印放入 `parameters`，使用 `/v1/videos` 创建与查询异步任务。它是影策适配的渠道合同，不是某一家模型厂商的官方协议。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/videos +GET {channel_base_url}/v1/videos/{task_id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型、计费与素材上限 + +模型清单、计费、时长范围、素材数量和单文件大小均由当前 NewAPI 部署及其上游渠道决定。此插件不能把某个渠道的 Seedance、Veo 或 Kling 限制推广给所有部署。管理员应在模型配置中记录经过实际调用验证的模型 ID。 + +## 参数与字段映射 + +{{PARAMETERS}} + +实际映射：图片 -> `{type:"reference_image"}`，视频 -> `{type:"reference_video"}`，音频 -> `{type:"reference_voice"}`；均放入 `input.media`。默认 `resolution=720P`、`ratio=16:9`、`duration=5`。只有 `watermark=true` 时才发送水印字段；`generateAudio` 当前不发送。 + +## 多模态创建示例 + +```bash +curl "{channel_base_url}/v1/videos" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_CHANNEL_MODEL", + "input":{ + "prompt":"保持人物外观，动作参考视频，节奏参考声音", + "media":[ + {"type":"reference_image","url":"https://example.com/character.png"}, + {"type":"reference_video","url":"https://example.com/action.mp4"}, + {"type":"reference_voice","url":"https://example.com/beat.mp3"} + ] + }, + "parameters":{"resolution":"720P","ratio":"16:9","duration":8,"watermark":false} + }' +``` + +## 响应、轮询与排错 + +创建响应必须在顶层、`task` 或 `data` 中包含 `id/task_id/taskId/request_id`。查询使用 URL 编码后的任务 ID。完成时解析视频数组或 `video_url/videoUrl/result_url/url`。如果渠道返回自己的深层字段，必须新增专属解析，不能把空结果标记成功。 + +常见问题包括渠道未映射模型、素材 URL 无法由上游下载、媒体类型命名不被该部署接受、默认参数超出模型范围。先检查部署版本和渠道日志，再修改适配器；不要反复添加兼容别名掩盖协议差异。 + +## 官方资料与边界 + +- NewAPI 项目与部署文档由具体运维方提供；本页以影策当前 adapter 的请求事实为准。 +- 上游模型官方文档只用于确认模型自身能力，不能替代 NewAPI 网关字段合同。 + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/newapi-channel-2.md b/backend/internal/protocol/docs/newapi-channel-2.md new file mode 100644 index 000000000..6fe47d990 --- /dev/null +++ b/backend/internal/protocol/docs/newapi-channel-2.md @@ -0,0 +1,63 @@ +# NewAPI Video Generations + +该插件使用 `/v1/video/generations` JSON 异步接口，直接发送图片、视频和音频 URL 数组。它适用于实现这一特定路径和字段集合的 NewAPI 渠道，不等同于 `/v1/videos` 媒体任务协议。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/video/generations +GET {channel_base_url}/v1/video/generations/{task_id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与约束来源 + +可用模型、每种素材上限、时长范围、音频能力、分辨率和计费由该 NewAPI 部署映射的上游决定。插件没有通用白名单。用户示例中的 Seedance 限制只可用于实际连接该服务且管理员验证一致的渠道。 + +## 参数与字段映射 + +{{PARAMETERS}} + +实际字段为 `model`、`prompt`、`seconds`、`aspect_ratio`、可选 `resolution`、`generate_audio`、`image_urls`、`video_urls`、`audio_urls`。时长默认 6 秒并发送为字符串，比例默认 `16:9`。水印当前不发送。 + +## 完整多模态请求 + +```bash +curl "{channel_base_url}/v1/video/generations" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_VIDEO_MODEL", + "prompt":"人物参考第一张图，镜头运动参考视频，节奏跟随音频", + "seconds":"10", + "aspect_ratio":"9:16", + "resolution":"720p", + "generate_audio":true, + "image_urls":["https://example.com/subject.png"], + "video_urls":["https://example.com/motion.mp4"], + "audio_urls":["https://example.com/beat.mp3"] + }' +``` + +## 任务状态与结果 + +```json +{"task_id":"task_xxx","status":"queued"} +``` + +```bash +curl "{channel_base_url}/v1/video/generations/task_xxx" \ + -H "Authorization: Bearer " +``` + +状态会归一为等待、处理中、成功或失败。成功结果可来自视频数组或常见顶层 URL；当前没有取消路径。任务失败后的退款、额度释放和结果保留期属于部署方账务合同，影策不会在没有接口证据时承诺。 + +## 官方资料与边界 + +- 以连接的 NewAPI 部署接口文档、版本和渠道配置为准。 +- 素材必须能被网关和最终上游访问；网关可访问不代表最终供应商可访问。 + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/newapi.md b/backend/internal/protocol/docs/newapi.md new file mode 100644 index 000000000..bdaf8dd8d --- /dev/null +++ b/backend/internal/protocol/docs/newapi.md @@ -0,0 +1,66 @@ +# OpenAI Videos 兼容协议 + +该插件实现 `/v1/videos` 异步视频合同：以 multipart 创建任务，取得任务 ID 后查询同一路径下的任务资源。它用于 Sora/OpenAI Videos 风格渠道；模型、尺寸、时长、价格和保留期由实际渠道决定。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/videos +GET {channel_base_url}/v1/videos/{task_id} +Authorization: Bearer +Content-Type: multipart/form-data +``` + +## 模型与能力边界 + +模型名必须是部署服务实际开放的视频模型。插件不把任意 `sora-*` 或 `sd-*` 名称视为可用，也不提供价格表。当前实现支持文生视频和单组参考图输入，不发送参考视频、参考音频、独立音频开关或水印。 + +## 参数与字段映射 + +{{PARAMETERS}} + +`duration -> seconds`，发送为十进制字符串；`aspectRatio -> size`，这里应填写上游尺寸而不是只写 `16:9`；`images -> input_reference` multipart 部件。空字段不发送。 + +## 文生视频示例 + +```bash +curl -X POST "{channel_base_url}/v1/videos" \ + -H "Authorization: Bearer " \ + -F "model=YOUR_VIDEO_MODEL" \ + -F "prompt=清晨薄雾中的竹林，镜头缓慢向前推进" \ + -F "seconds=8" \ + -F "size=1280x720" +``` + +## 图片参考示例 + +```bash +curl -X POST "{channel_base_url}/v1/videos" \ + -H "Authorization: Bearer " \ + -F "model=YOUR_VIDEO_MODEL" \ + -F "prompt=保持参考图人物一致，镜头从中景推进到特写" \ + -F "seconds=8" \ + -F "size=720x1280" \ + -F "input_reference=@character.png" +``` + +## 创建、轮询与下载 + +```json +{"id":"task_xxx","status":"queued"} +``` + +```bash +curl "{channel_base_url}/v1/videos/task_xxx" -H "Authorization: Bearer " +``` + +完成响应可在顶层或嵌套对象提供 `url`、`video_url`、`result_url` 或视频数组。失败消息从 `message`、`error`、`fail_reason` 提取。当前插件没有取消接口，也不会自动删除上游任务；轮询间隔和下载有效期必须服从渠道说明。 + +## 官方与兼容资料 + +- [OpenAI Videos API reference](https://developers.openai.com/api/reference/resources/videos) +- 若渠道是 NewAPI 或其他兼容实现，还必须核对该部署的版本说明；路径相同不等于字段和模型完全一致。 + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/novita-video.md b/backend/internal/protocol/docs/novita-video.md new file mode 100644 index 000000000..2764ee929 --- /dev/null +++ b/backend/internal/protocol/docs/novita-video.md @@ -0,0 +1,57 @@ +# Novita 视频异步 API + +Novita 视频插件使用 `/v3/video/create` 创建任务，以 query 参数携带任务 ID 查询统一异步结果。模型字段名为 `model_name`，不是通用 `model`。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v3/video/create +GET {channel_base_url}/v3/async/task-result?task_id={id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与约束 + +Novita 聚合多个视频模型，不同 `model_name` 的字段、图片要求、时长、比例和价格可能不同。当前插件只提供共同子集：提示词、单张参考图、时长和比例；不能据此认定所有 Novita 视频模型都接受相同参数。 + +## 参数与字段映射 + +{{PARAMETERS}} + +`model -> model_name`，`images[0] -> image_url`，时长默认 5 秒，比例默认 `16:9`。`resolution`、参考视频、参考音频、音频生成和水印当前不发送。 + +## 创建请求示例 + +```bash +curl "{channel_base_url}/v3/video/create" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model_name":"YOUR_NOVITA_VIDEO_MODEL", + "prompt":"产品在黑色展台上缓慢旋转，轮廓光", + "duration":5, + "aspect_ratio":"16:9", + "image_url":"https://example.com/product.png" + }' +``` + +## 查询与下载 + +创建响应应返回 `task_id/id`。查询时任务 ID会进行 URL query 编码： + +```bash +curl "{channel_base_url}/v3/async/task-result?task_id=task_xxx" \ + -H "Authorization: Bearer " +``` + +成功结果由统一 parser 从视频数组或 URL 字段读取。Novita 若针对某模型返回独特嵌套结构，应建立该模型 fixture，而不是继续扩展全局模糊字段。当前没有取消接口；任务退款和结果保留期以 Novita 当前规则为准。 + +## 官方资料 + +- [Novita Unified Video Generation](https://novita.ai/docs/api-reference/reference-unified-video-generation) +- [Novita Task Result Query](https://novita.ai/docs/api-reference/model-apis-task-result) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/openai-audio.md b/backend/internal/protocol/docs/openai-audio.md new file mode 100644 index 000000000..10b6068a6 --- /dev/null +++ b/backend/internal/protocol/docs/openai-audio.md @@ -0,0 +1,50 @@ +# OpenAI Audio Speech + +该插件调用 OpenAI 风格的同步文字转语音接口，把影策提示词映射为 `input`。当前只发送模型与文本，声音、格式、语速和其他 TTS 选项尚未纳入 adapter。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/audio/speech +Authorization: Bearer +Content-Type: application/json +``` + +## 模型、声音与输出格式 + +模型、voice、格式、单次文本长度、流式能力和价格以 OpenAI 或兼容渠道实际文档为准。当前影策 metadata 没有声明参数表，这意味着只有 `model` 和 `prompt` 的内部映射稳定，不意味着上游没有必填 `voice`。若上游要求 voice，管理员必须先扩展 adapter，不能依赖未白名单的字段自动透传。 + +## 参数与字段映射 + +{{PARAMETERS}} + +实际请求正文固定为： + +```json +{"model":"YOUR_TTS_MODEL","input":"请在午夜前离开这座城市。"} +``` + +完整上游常见请求参考： + +```bash +curl "{channel_base_url}/v1/audio/speech" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"model":"YOUR_TTS_MODEL","voice":"YOUR_VOICE","input":"请在午夜前离开这座城市。","response_format":"mp3"}' \ + --output speech.mp3 +``` + +第二个示例包含当前 adapter 尚未发送的 `voice` 和 `response_format`，用于说明必须补齐的协议字段。 + +## 响应与当前实现差异 + +OpenAI 官方 Speech API 通常直接返回音频二进制流；当前 `parseAudioResponse` 却按 JSON 中的 `url/audio_url/data` 解析。因此本插件当前更适合“返回音频 URL JSON”的兼容渠道，不能宣称已经正确处理 OpenAI 官方二进制响应。直连官方服务前必须增加二进制响应合同、MIME/文件名处理和资源保存测试。 + +## 官方资料 + +- [OpenAI Text-to-speech guide](https://developers.openai.com/api/docs/guides/text-to-speech) +- [OpenAI Audio Speech reference](https://developers.openai.com/api/reference/resources/audio/subresources/speech/methods/create) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/openai-image.md b/backend/internal/protocol/docs/openai-image.md new file mode 100644 index 000000000..a23c3669d --- /dev/null +++ b/backend/internal/protocol/docs/openai-image.md @@ -0,0 +1,73 @@ +# OpenAI Images + +OpenAI Images 插件实现 JSON 文生图请求，并解析 OpenAI Images 风格的 `data[]` 响应。上游官方还提供 multipart 图像编辑接口，但当前影策适配器的创建路径只构造 `/v1/images/generations`；详情中列出编辑接口是为了说明协议能力，不代表当前画布已发送编辑请求。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/images/generations +Authorization: Bearer +Content-Type: application/json +``` + +官方编辑接口为 `POST /v1/images/edits`、`multipart/form-data`。当前适配器未在有参考图时切换到该路径，这是明确的实现缺口。 + +## 模型、尺寸与质量 + +模型、允许尺寸、质量枚举、单次张数和计费会随上游服务变化，插件不维护可能过期的白名单。渠道若是 OpenAI 兼容实现，必须分别验证它是否支持 `n`、`quality`、`background`、`output_format` 和 Base64 响应。 + +## 参数与字段映射 + +{{PARAMETERS}} + +当前实现：`imageCount -> n`、`aspectRatio -> size`、`quality -> quality`；`extra` 可透传 `size`、`quality`、`background`、`response_format`、`output_format`、`style`、`n`。`images` 和 `resolution` 当前不会进入文生图请求。 + +## 文生图请求 + +```bash +curl "{channel_base_url}/v1/images/generations" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_IMAGE_MODEL", + "prompt":"雨夜便利店门口，青绿色霓虹，电影剧照", + "size":"1536x1024", + "quality":"high", + "n":1 + }' +``` + +URL 响应： + +```json +{"created":1780000000,"data":[{"url":"https://cdn.example/generated.png"}]} +``` + +Base64 响应： + +```json +{"created":1780000000,"data":[{"b64_json":"iVBORw0KGgo..."}]} +``` + +解析器遍历 `data[]`，优先识别 `url`，也接受 `b64_json` 或 `data`。URL 应立即下载；裸 Base64 会转为 `data:image/png;base64,...`，已带 data URL 前缀的值保持不变。若渠道使用 WebP/JPEG Base64 且响应不返回 MIME，当前协议无法可靠推断格式，应优先请求 URL 或补充带 MIME 的响应合同。 + +## 图像编辑参考请求 + +```bash +curl "{channel_base_url}/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=YOUR_IMAGE_MODEL" \ + -F "image=@reference.png" \ + -F "prompt=保持人物一致，改成雨夜街道" +``` + +该示例是上游协议参考；在影策中使用前必须补 multipart 构造、参考图数量与 MIME 校验、编辑响应测试。 + +## 官方资料 + +- [OpenAI Image generation guide](https://developers.openai.com/api/docs/guides/image-generation) +- [OpenAI Images API reference](https://developers.openai.com/api/reference/resources/images) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/openai-response.md b/backend/internal/protocol/docs/openai-response.md new file mode 100644 index 000000000..4ea2ea757 --- /dev/null +++ b/backend/internal/protocol/docs/openai-response.md @@ -0,0 +1,71 @@ +# OpenAI Responses API + +Responses API 是 OpenAI 当前统一的文本与工具调用接口，输入使用 `input`，可同时承载简单字符串、结构化消息和工具回合。影策默认发送字符串提示词，并从同步响应的 `output_text` 或 `output[]` 中提取文本。 + +## 接口、鉴权与请求模式 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/responses +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与能力发现 + +插件不硬编码模型表。请使用渠道模型管理中已验证支持 Responses API 的模型 ID。兼容服务即使实现 `/v1/responses`，也可能不支持内置工具、状态延续、结构化输出或多模态输入，能力必须以该服务文档和实际探测为准。 + +## 参数与字段映射 + +{{PARAMETERS}} + +`extra` 可透传 `input`、`instructions`、`temperature`、`top_p`、`max_output_tokens`、`stream`、`tools` 和 `text`。`extra.input` 会替换默认字符串；复杂输入必须使用上游规定的 item 结构，不应把 Chat Completions 的 `messages` 原样塞进来。 + +## 请求示例 + +```bash +curl "{channel_base_url}/v1/responses" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model": "YOUR_MODEL", + "instructions": "输出可直接拍摄的镜头清单。", + "input": "一名女孩在停电的商场里寻找出口。", + "max_output_tokens": 1200 + }' +``` + +```json +{ + "id": "resp_xxx", + "status": "completed", + "output_text": "镜头 1……", + "usage": {"input_tokens":28,"output_tokens":174,"total_tokens":202} +} +``` + +## 结构化输入示例 + +```json +{ + "model": "YOUR_MODEL", + "input": [{ + "role": "user", + "content": [{"type":"input_text","text":"分析这段剧情的节奏问题"}] + }] +} +``` + +## 响应解析与限制 + +解析器优先读取顶层 `output_text`；缺失时遍历 `output`，尝试读取 item 的 `text` 或 `content`，并保留 `usage`。当前实现没有完整执行 function call、computer use、file search 等输出 item，也不通过 `previous_response_id` 自动续接状态。需要这些能力时必须在 Agent 调用链中显式实现，不能只靠文档声称支持。 + +流式响应由文本事件链路处理。上游拒绝、模型不支持、输入结构错误、输出 token 超限和工具参数校验失败均应作为失败返回。 + +## 官方资料 + +- [OpenAI Responses API Reference](https://developers.openai.com/api/reference/resources/responses/methods/create) +- [OpenAI Responses migration guide](https://developers.openai.com/api/docs/guides/migrate-to-responses) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/volcengine-ark-image.md b/backend/internal/protocol/docs/volcengine-ark-image.md new file mode 100644 index 000000000..73585a76b --- /dev/null +++ b/backend/internal/protocol/docs/volcengine-ark-image.md @@ -0,0 +1,61 @@ +# 火山方舟图片生成 + +本插件面向火山方舟 Ark 图片生成 JSON 接口。提示词直接发送，参考图数组映射到 `image`，连续生成和水印参数可从模型扩展字段透传。 + +## 接口、鉴权与区域 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/api/v3/images/generations +Authorization: Bearer +Content-Type: application/json +``` + +Base URL、鉴权方式和可用模型受火山方舟控制台、区域与接入方式影响。若使用 AK/SK 签名网关，应由渠道层完成签名，不能把 Secret Key 写入插件文档或浏览器。 + +## 模型与支持边界 + +模型 ID、尺寸范围与能力以火山方舟控制台为准。 + +## 参数与字段映射 + +{{PARAMETERS}} + +当前实现：`aspectRatio -> size`，所有 `images -> image[]`；`extra` 可透传 `size`、`sequential_image_generation`、`sequential_image_generation_options`、`watermark`。`imageCount`、`quality` 与 `resolution` 没有通用映射，必须按模型接口另行适配。 + +## 文生图示例 + +```bash +curl "{channel_base_url}/api/v3/images/generations" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_ARK_ENDPOINT_OR_MODEL", + "prompt":"古城屋顶追逐，低机位，动态构图", + "size":"2048x1152", + "watermark":false + }' +``` + +## 参考图示例 + +```json +{ + "model":"YOUR_ARK_ENDPOINT_OR_MODEL", + "prompt":"保持人物服装和发型，生成侧面中景", + "image":["https://example.com/character.png"], + "size":"2048x1152" +} +``` + +## 响应解析与限制 + +响应按 `data[]` 图片数组解析，元素需包含 `url`、`b64_json` 或 `data`。当前插件没有为 Ark 特有错误码、任务模式或内容过滤结果做专属解包；渠道返回非兼容结构时会明确失败。模型尺寸、参考图数量、序列图上限、水印和计费以控制台当前模型说明为准，不能从插件默认值推断。 + +## 官方资料 + +- [火山方舟视觉模型 API 文档](https://www.volcengine.com/docs/82379) +- [火山方舟 API 接入与鉴权文档](https://www.volcengine.com/docs/82379/1263279) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/volcengine-ark-video.md b/backend/internal/protocol/docs/volcengine-ark-video.md new file mode 100644 index 000000000..15d0b5754 --- /dev/null +++ b/backend/internal/protocol/docs/volcengine-ark-video.md @@ -0,0 +1,57 @@ +# 火山方舟视频任务 + +本插件使用 Ark Contents Generations Tasks 异步接口。提示词和参考图组成 `content[]`；比例、分辨率、时长、音频和水印位于顶层任务参数。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/api/v3/contents/generations/tasks +GET {channel_base_url}/api/v3/contents/generations/tasks/{task_id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与约束 + +模型通常对应控制台可用模型或 endpoint ID。允许时长、分辨率、画幅、参考图数量、是否可生成音频及计费由具体模型决定。插件默认 5 秒、`16:9`、`720p`，默认值不是上游能力承诺。 + +## 参数与字段映射 + +{{PARAMETERS}} + +每张图片构造为 `{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`。当前不发送参考视频与参考音频；`generate_audio` 和 `watermark` 只在 true 时发送。 + +## 创建任务示例 + +```bash +curl "{channel_base_url}/api/v3/contents/generations/tasks" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_ARK_VIDEO_MODEL", + "content":[ + {"type":"text","text":"保持人物一致，镜头缓慢环绕"}, + {"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"} + ], + "ratio":"16:9", + "resolution":"720p", + "duration":5, + "generate_audio":true, + "watermark":false + }' +``` + +## 查询、状态与结果 + +创建响应任务 ID 由统一解析器从顶层、`task` 或 `data` 提取。查询成功后读取视频数组或常见 URL 字段。若 Ark 把结果放在其他嵌套路径，必须用真实 fixture 增加专属解析，不允许只根据 `completed` 返回空成功。 + +任务失败可能来自 endpoint 不可用、素材下载失败、参数组合不受支持、内容审核、并发或额度限制。上游 request ID 应保留用于排查，API Key 和素材正文不得进入日志。 + +## 官方资料 + +- [火山方舟视频生成 API](https://www.volcengine.com/docs/82379) +- [火山方舟内容生成任务文档](https://www.volcengine.com/docs/82379/1520757) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/volcengine-jimeng-image.md b/backend/internal/protocol/docs/volcengine-jimeng-image.md new file mode 100644 index 000000000..81b265189 --- /dev/null +++ b/backend/internal/protocol/docs/volcengine-jimeng-image.md @@ -0,0 +1,57 @@ +# 即梦官方图片异步 API + +本插件使用即梦视觉异步任务模式：提交任务取得 `task_id`，再以相同 `req_key` 查询结果。接口由火山引擎视觉服务的 AK/SK 签名体系保护，不是 Bearer 兼容接口。 + +## 接口与签名 + +{{OPERATIONS}} + +逻辑操作名为 `CVSync2AsyncSubmitTask` 和 `CVSync2AsyncGetResult`。实际主机、Query 参数、Service/Region 与签名头由渠道的火山签名实现负责；不能只对这两个字符串执行普通 HTTP POST。 + +## 模型与 req_key + +影策模型 ID 映射为 `req_key`，必须与已开通的即梦能力完全一致。插件不提供可能过期的 `req_key` 白名单，也不根据名字推断文生图、图生图或版本能力。 + +## 参数与字段映射 + +{{PARAMETERS}} + +当前实现发送 `req_key`、`prompt`，有参考图时发送 `image_urls`。`extra` 可覆盖 `req_key`、`prompt`、`image_urls`、`seed`、`width`、`height`。平台的 `aspectRatio`、`quality`、`resolution` 和 `imageCount` 没有自动转换。 + +## 创建请求正文 + +```json +{ + "req_key":"YOUR_JIMENG_REQ_KEY", + "prompt":"悬疑短剧分镜，楼道尽头的逆光人影", + "image_urls":["https://example.com/reference.png"], + "width":1536, + "height":1024, + "seed":42 +} +``` + +创建响应至少需要可识别任务 ID： + +```json +{"data":{"task_id":"task_xxx"}} +``` + +## 轮询请求与结果 + +```json +{"req_key":"YOUR_JIMENG_REQ_KEY","task_id":"task_xxx"} +``` + +轮询响应从顶层、`task` 或 `data` 读取状态，并按图片类型提取 `images`、`image_url`、`imageUrl`、`result_url` 或 `url`；图片数组可包含 URL 字符串或对象。若结果只出现在即梦其他专属嵌套字段中，需要补 fixture 和专属解析，不能把“拿到 completed”当作已经保存图片。 + +## 错误与排查 + +签名时间偏差、Region/Service 错误、`req_key` 未开通、素材不可下载、宽高不支持和内容审核都会导致失败。日志可以记录请求 ID 和错误码，不得记录 Secret Key、完整签名或私有素材 data URL。 + +## 官方资料 + +- [火山引擎视觉智能开放平台](https://www.volcengine.com/docs/85621) +- [火山引擎签名方法](https://www.volcengine.com/docs/6369/67269) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/volcengine-jimeng-video.md b/backend/internal/protocol/docs/volcengine-jimeng-video.md new file mode 100644 index 000000000..77e2e0abe --- /dev/null +++ b/backend/internal/protocol/docs/volcengine-jimeng-video.md @@ -0,0 +1,55 @@ +# 即梦官方视频异步 API + +即梦视频插件使用 `CVSync2AsyncSubmitTask` 创建、`CVSync2AsyncGetResult` 查询，并由渠道的火山 AK/SK 实现完成签名。创建与查询均为 POST，模型能力键通过 `req_key` 传递。 + +## 接口与签名 + +{{OPERATIONS}} + +这两个名称是视觉服务 Action，实际 URL、Query、Region、Service、时间戳和签名头必须按火山官方签名规范生成。把 AK/SK 放入 JSON 或浏览器会造成密钥泄漏。 + +## 模型、版本与素材限制 + +`req_key` 必须是账户已开通的即梦视频能力键。每个版本对首帧/尾帧、参考图数量、时长、比例和分辨率的约束不同；插件不维护一张可能过期的模型表。当前只映射参考图片，不发送参考视频、音频、生成音频或水印。 + +## 参数与字段映射 + +{{PARAMETERS}} + +基础请求包含 `req_key`、`prompt`、可选 `image_urls`。`extra` 可透传并覆盖 `req_key`、`prompt`、`image_urls`、`duration`、`ratio`、`resolution`，适合管理员按已确认的官方版本补参数。 + +## 创建任务正文 + +```json +{ + "req_key":"YOUR_JIMENG_VIDEO_REQ_KEY", + "prompt":"保持参考人物，镜头从全景移动到近景", + "image_urls":["https://example.com/subject.png"], + "duration":5, + "ratio":"16:9", + "resolution":"720p" +} +``` + +创建响应： + +```json +{"data":{"task_id":"task_xxx"}} +``` + +查询正文： + +```json +{"req_key":"YOUR_JIMENG_VIDEO_REQ_KEY","task_id":"task_xxx"} +``` + +## 轮询、结果与失败 + +轮询响应由统一异步 parser 读取状态和视频 URL。查询必须继续携带创建时的模型 `req_key`，不能只传任务 ID。签名时间偏差、Region 错误、能力未开通、素材 URL 失效、参数不匹配和审核均会失败。当前没有取消接口，失败后的额度处理以官方账务为准。 + +## 官方资料 + +- [火山引擎视觉智能开放平台](https://www.volcengine.com/docs/85621) +- [火山引擎 API 签名方法](https://www.volcengine.com/docs/6369/67269) + +{{CONTRACT}} diff --git a/backend/internal/protocol/docs/xai-video.md b/backend/internal/protocol/docs/xai-video.md new file mode 100644 index 000000000..a0f4c0ad8 --- /dev/null +++ b/backend/internal/protocol/docs/xai-video.md @@ -0,0 +1,55 @@ +# xAI 视频生成 + +xAI 视频插件创建异步 generation request，再按 request ID 查询。单张参考图发送为 `image`，多张参考图发送为 `reference_images`；时长、画幅和分辨率使用 xAI 风格字段。 + +## 接口与鉴权 + +{{OPERATIONS}} + +```http +POST {channel_base_url}/v1/videos/generations +GET {channel_base_url}/v1/videos/{request_id} +Authorization: Bearer +Content-Type: application/json +``` + +## 模型与当前支持 + +模型 ID、允许时长、比例、分辨率、参考图数量、审核规则和计费以 xAI 控制台及官方文档为准。当前 adapter 默认 6 秒、`16:9`、`720p`。视频参考、音频参考、音频生成开关和水印均不会发送。 + +## 参数与字段映射 + +{{PARAMETERS}} + +单图结构：`"image":{"url":"..."}`；多图结构：`"reference_images":[{"url":"..."}]`。URL 或 data URL 是否被具体模型接受，应在调用前验证。 + +## 请求示例 + +```bash +curl "{channel_base_url}/v1/videos/generations" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "model":"YOUR_XAI_VIDEO_MODEL", + "prompt":"雨夜都市追逐，手持摄影，真实运动模糊", + "duration":8, + "aspect_ratio":"16:9", + "resolution":"720p", + "image":{"url":"https://example.com/character.png"} + }' +``` + +## 创建、查询与下载 + +创建响应需包含 `request_id` 或其他统一解析器支持的任务字段。查询完成后从视频数组或 `video_url/result_url/url` 读取媒体。当前没有 cancel adapter；关闭影策弹窗不会取消上游生成。结果 URL 应及时转存，不能假定永久有效。 + +## 错误处理 + +模型权限、无效尺寸、参考素材不可访问、内容审核、速率或余额限制必须作为真实失败展示。若官方响应结构与通用解析器不一致，应补 xAI 专属 parser 和 fixture 测试，而不是继续堆叠模糊字段猜测。 + +## 官方资料 + +- [xAI Video Generation guide](https://docs.x.ai/docs/guides/video-generation) +- [xAI API Reference](https://docs.x.ai/docs/api-reference) + +{{CONTRACT}} diff --git a/backend/internal/protocol/documentation.go b/backend/internal/protocol/documentation.go new file mode 100644 index 000000000..679a4af95 --- /dev/null +++ b/backend/internal/protocol/documentation.go @@ -0,0 +1,113 @@ +package protocol + +import ( + "embed" + "fmt" + "strings" +) + +//go:embed docs/*.md +var builtinDocumentationFS embed.FS + +func attachDocumentation(info *Metadata) { + if info == nil || strings.TrimSpace(info.Documentation) != "" { + return + } + document, err := builtinDocumentationFS.ReadFile("docs/" + info.ID + ".md") + if err != nil { + return + } + info.Documentation = renderBuiltinDocumentation(*info, string(document)) +} + +// AttachDocumentation fills the reference document for a shipped protocol. +func AttachDocumentation(info *Metadata) { + attachDocumentation(info) +} + +func renderBuiltinDocumentation(info Metadata, document string) string { + replacements := map[string]string{ + "{{NAME}}": info.Name, + "{{OPERATIONS}}": renderOperations(info), + "{{PARAMETERS}}": renderParameters(info.Parameters), + "{{CONTRACT}}": renderRuntimeContract(info), + } + for marker, replacement := range replacements { + document = strings.ReplaceAll(document, marker, replacement) + } + return strings.TrimSpace(document) + "\n" +} + +func renderOperations(info Metadata) string { + var b strings.Builder + b.WriteString("| 阶段 | 方法与相对路径 | 请求类型 | 影策行为 |\n| --- | --- | --- | --- |\n") + writeOperation(&b, "创建", info.Create, firstNonEmpty(info.ContentType, "application/json"), "提交请求；同步协议直接解析结果，异步协议读取任务 ID") + if strings.TrimSpace(info.Poll) != "" { + writeOperation(&b, "轮询", info.Poll, "application/json", "查询状态，成功后提取媒体地址") + } + if strings.TrimSpace(info.Cancel) != "" { + writeOperation(&b, "取消", info.Cancel, "application/json", "取消仍在运行的任务") + } + return strings.TrimSpace(b.String()) +} + +func renderParameters(parameters []Parameter) string { + if len(parameters) == 0 { + return "> 当前适配器没有声明可配置参数；这不表示上游没有参数，只表示影策尚未建立稳定映射。" + } + var b strings.Builder + b.WriteString("| 影策字段 | 类型 | 必填 | 实际上游映射 | 说明 |\n| --- | --- | --- | --- | --- |\n") + for _, parameter := range parameters { + description := strings.TrimSpace(parameter.Description) + if len(parameter.Values) > 0 { + description = strings.TrimSpace(description + " 可选值：" + strings.Join(parameter.Values, "、") + "。") + } + fmt.Fprintf(&b, "| `%s` | `%s` | %s | `%s` | %s |\n", markdownCell(parameter.Name), markdownCell(parameter.Type), yesNo(parameter.Required), markdownCell(firstNonEmpty(parameter.Mapping, "未映射")), markdownCell(firstNonEmpty(description, "由当前协议适配器按请求类型转换。"))) + } + return strings.TrimSpace(b.String()) +} + +func renderRuntimeContract(info Metadata) string { + var b strings.Builder + b.WriteString("## 影策运行时合同\n\n") + b.WriteString("- Base URL 只填写协议服务根地址，表中的相对路径由适配器拼接；不要把同一路径重复写进 Base URL。\n") + b.WriteString("- 密钥由后端渠道中转读取，插件详情和浏览器请求不保存密钥。Bearer、供应商签名或专用版本头以本文鉴权章节为准。\n") + b.WriteString("- 模型名来自渠道模型配置并原样发送。此插件不暗改模型名，也不根据名称猜测价格、额度或能力。\n") + b.WriteString("- HTTP 非 2xx、非 JSON 响应、缺少任务 ID、成功状态却没有可识别结果都会作为真实错误向上返回。\n") + if strings.TrimSpace(info.Poll) != "" { + b.WriteString("- 异步状态统一为：`queued/pending/created/submitted` -> 等待，`running/processing/in_progress` -> 处理中，`succeeded/completed/done` -> 成功，`failed/error/expired/cancelled` -> 失败。\n") + b.WriteString("- 轮询频率应遵守上游速率限制；媒体 URL 通常有有效期，成功后应立即进入影策资源保存流程。\n") + } + b.WriteString("- 参考素材只接受上游可访问的 HTTP(S) URL 或带 MIME 前缀的 data URL；裸 Base64、登录后才能访问的链接和 Cookie 保护地址不可用。\n") + b.WriteString("- 插件只保证下方“当前实现”明确列出的字段。上游新增能力不会自动获得影策支持，必须补适配、解析和测试。\n") + return strings.TrimSpace(b.String()) +} + +func writeOperation(b *strings.Builder, phase, value, contentType, purpose string) { + parts := strings.SplitN(strings.TrimSpace(value), " ", 2) + method, path := "", strings.TrimSpace(value) + if len(parts) == 2 { + method, path = parts[0], parts[1] + } + fmt.Fprintf(b, "| %s | `%s %s` | `%s` | %s |\n", phase, markdownCell(method), markdownCell(path), markdownCell(contentType), purpose) +} + +func markdownCell(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(value, "|", "\\|"), "\n", " ") +} + +func yesNo(value bool) string { + if value { + return "是" + } + return "否" +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} diff --git a/backend/internal/protocol/errors.go b/backend/internal/protocol/errors.go new file mode 100644 index 000000000..a2d0607dc --- /dev/null +++ b/backend/internal/protocol/errors.go @@ -0,0 +1,19 @@ +package protocol + +import "fmt" + +type UnavailableError struct { + Protocol string + Reason string +} + +func (e UnavailableError) Error() string { + if e.Reason == "" { + return fmt.Sprintf("protocol %s is unavailable", e.Protocol) + } + return fmt.Sprintf("protocol %s is unavailable: %s", e.Protocol, e.Reason) +} + +func unavailable(metadata Metadata) error { + return UnavailableError{Protocol: metadata.ID, Reason: metadata.UnavailableReason} +} diff --git a/backend/internal/protocol/manifest.go b/backend/internal/protocol/manifest.go new file mode 100644 index 000000000..0029829dd --- /dev/null +++ b/backend/internal/protocol/manifest.go @@ -0,0 +1,474 @@ +package protocol + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// Manifest is the declarative format used by future protocol packages. It is +// intentionally limited to relative HTTP paths and JSON field mappings; a +// plugin cannot inject code or bypass the host outbound policy. +type Manifest struct { + APIVersion string `json:"apiVersion"` + Metadata Metadata `json:"metadata"` + Create ManifestOperation `json:"create"` + Agent *ManifestOperation `json:"agent,omitempty"` + Poll *ManifestOperation `json:"poll,omitempty"` + Cancel *ManifestOperation `json:"cancel,omitempty"` + Response ManifestResponse `json:"response"` + AgentResponse *ManifestAgentResponse `json:"agentResponse,omitempty"` +} + +type ManifestOperation struct { + Method string `json:"method"` + Path string `json:"path"` + ContentType string `json:"contentType,omitempty"` + Fields map[string]string `json:"fields,omitempty"` +} + +type ManifestResponse struct { + TaskIDPaths []string `json:"taskIdPaths,omitempty"` + StatusPaths []string `json:"statusPaths,omitempty"` + MessagePaths []string `json:"messagePaths,omitempty"` + TextPaths []string `json:"textPaths,omitempty"` + ReasoningPaths []string `json:"reasoningPaths,omitempty"` + ResultURLPaths []string `json:"resultUrlPaths,omitempty"` + ResultKind string `json:"resultKind,omitempty"` +} + +// ManifestAgentResponse describes the provider response shape for a +// tool-capable text request. Tool call paths are resolved against each item in +// ToolCallsPath. +type ManifestAgentResponse struct { + TextPaths []string `json:"textPaths,omitempty"` + ReasoningPaths []string `json:"reasoningPaths,omitempty"` + ToolCallsPath string `json:"toolCallsPath,omitempty"` + ToolCallIDPaths []string `json:"toolCallIdPaths,omitempty"` + ToolCallNamePaths []string `json:"toolCallNamePaths,omitempty"` + ToolCallArgsPaths []string `json:"toolCallArgumentsPaths,omitempty"` +} + +// AdapterResolver is used by the host to bind a shipped execution engine to a +// manifest. Uploaded plugins must use the declarative path; host bindings are +// reserved for manifests shipped with the application. +type AdapterResolver func(string) (Adapter, bool) + +func LoadInstalledPlugin(data []byte, resolve AdapterResolver) (Adapter, error) { + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("decode protocol plugin: %w", err) + } + if strings.HasPrefix(strings.TrimSpace(manifest.Metadata.Execution), "host:") { + if resolve == nil { + return nil, fmt.Errorf("protocol plugin %q requires a host execution engine", manifest.Metadata.ID) + } + name := strings.TrimPrefix(strings.TrimSpace(manifest.Metadata.Execution), "host:") + adapter, ok := resolve(name) + if !ok { + return nil, fmt.Errorf("protocol plugin host execution %q is unavailable", name) + } + if err := validatePluginMetadata(manifest.Metadata); err != nil { + return nil, err + } + return metadataAdapter{metadata: manifest.Metadata, delegate: adapter}, nil + } + return LoadManifest(data) +} + +func validatePluginMetadata(metadata Metadata) error { + if strings.TrimSpace(metadata.ID) == "" || strings.TrimSpace(metadata.Version) == "" || !validManifestIdentifier(metadata.ID) { + return fmt.Errorf("protocol plugin metadata is invalid") + } + if len(metadata.Categories) == 0 || len(metadata.Scopes) == 0 { + return fmt.Errorf("protocol plugin metadata requires categories and scopes") + } + for _, capability := range metadata.Categories { + if capability != CapabilityText && capability != CapabilityImage && capability != CapabilityVideo && capability != CapabilityAudio { + return fmt.Errorf("unsupported protocol capability %q", capability) + } + } + for _, scope := range metadata.Scopes { + switch scope { + case SurfaceAdminSystemChannel, SurfaceUserCustomChannel, SurfaceCanvas, SurfaceCreation, SurfaceAgent: + default: + return fmt.Errorf("unsupported protocol scope %q", scope) + } + } + return nil +} + +type metadataAdapter struct { + metadata Metadata + delegate Adapter +} + +func (a metadataAdapter) Metadata() Metadata { return a.metadata } +func (a metadataAdapter) BuildCreate(ctx context.Context, c RequestContext) (RequestSpec, error) { + return a.delegate.BuildCreate(ctx, c) +} +func (a metadataAdapter) ParseCreate(ctx context.Context, body []byte) (CreateResult, error) { + return a.delegate.ParseCreate(ctx, body) +} +func (a metadataAdapter) BuildPoll(ctx context.Context, c PollContext) (RequestSpec, error) { + return a.delegate.BuildPoll(ctx, c) +} +func (a metadataAdapter) ParsePoll(ctx context.Context, c PollContext, body []byte) (PollResult, error) { + return a.delegate.ParsePoll(ctx, c, body) +} +func (a metadataAdapter) BuildCancel(ctx context.Context, c PollContext) (RequestSpec, error) { + return a.delegate.BuildCancel(ctx, c) +} +func (a metadataAdapter) BuildAgent(ctx context.Context, c AgentRequestContext) (RequestSpec, error) { + adapter, ok := a.delegate.(AgentAdapter) + if !ok { + return RequestSpec{}, unavailable(a.metadata) + } + return adapter.BuildAgent(ctx, c) +} +func (a metadataAdapter) ParseAgent(ctx context.Context, body []byte) (AgentResult, error) { + adapter, ok := a.delegate.(AgentAdapter) + if !ok { + return AgentResult{}, unavailable(a.metadata) + } + return adapter.ParseAgent(ctx, body) +} + +func LoadManifest(data []byte) (Adapter, error) { + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("decode protocol manifest: %w", err) + } + if err := ValidateManifest(manifest); err != nil { + return nil, err + } + if strings.TrimSpace(manifest.Metadata.Execution) == "" { + manifest.Metadata.Execution = "declarative" + } + return manifestAdapter{manifest: manifest}, nil +} + +func ValidateManifest(manifest Manifest) error { + if strings.TrimSpace(manifest.APIVersion) != "v1" { + return fmt.Errorf("unsupported protocol manifest apiVersion %q", manifest.APIVersion) + } + if strings.TrimSpace(manifest.Metadata.ID) == "" || strings.TrimSpace(manifest.Metadata.Version) == "" { + return fmt.Errorf("protocol manifest metadata requires id and version") + } + if strings.TrimSpace(manifest.Metadata.Name) == "" { + return fmt.Errorf("protocol manifest metadata requires name") + } + if strings.TrimSpace(manifest.Metadata.Documentation) == "" { + return fmt.Errorf("protocol manifest metadata requires Markdown documentation") + } + if !validManifestIdentifier(manifest.Metadata.ID) { + return fmt.Errorf("protocol manifest metadata id is invalid") + } + if len(manifest.Metadata.Name) > 160 || len(manifest.Metadata.Vendor) > 120 { + return fmt.Errorf("protocol manifest metadata name or vendor is too long") + } + if len(manifest.Metadata.Categories) == 0 || len(manifest.Metadata.Scopes) == 0 { + return fmt.Errorf("protocol manifest metadata requires categories and scopes") + } + for _, capability := range manifest.Metadata.Categories { + if capability != CapabilityText && capability != CapabilityImage && capability != CapabilityVideo && capability != CapabilityAudio { + return fmt.Errorf("unsupported protocol capability %q", capability) + } + } + for _, scope := range manifest.Metadata.Scopes { + switch scope { + case SurfaceAdminSystemChannel, SurfaceUserCustomChannel, SurfaceCanvas, SurfaceCreation, SurfaceAgent: + default: + return fmt.Errorf("unsupported protocol scope %q", scope) + } + } + if err := validateManifestOperation(manifest.Create); err != nil { + return fmt.Errorf("create operation: %w", err) + } + if manifest.Agent != nil { + if err := validateManifestOperation(*manifest.Agent); err != nil { + return fmt.Errorf("agent operation: %w", err) + } + if manifest.AgentResponse == nil { + return fmt.Errorf("agent response mapping is required when agent operation is declared") + } + } + if manifest.Poll != nil { + if err := validateManifestOperation(*manifest.Poll); err != nil { + return fmt.Errorf("poll operation: %w", err) + } + } + if manifest.Cancel != nil { + if err := validateManifestOperation(*manifest.Cancel); err != nil { + return fmt.Errorf("cancel operation: %w", err) + } + } + return nil +} + +func validateManifestOperation(operation ManifestOperation) error { + method := strings.ToUpper(strings.TrimSpace(operation.Method)) + if method != http.MethodGet && method != http.MethodPost && method != http.MethodDelete && method != http.MethodPut { + return fmt.Errorf("unsupported HTTP method %q", operation.Method) + } + if !isRelativePath(operation.Path) { + return fmt.Errorf("path must be relative: %q", operation.Path) + } + return nil +} + +type manifestAdapter struct{ manifest Manifest } + +func (a manifestAdapter) Metadata() Metadata { return a.manifest.Metadata } +func (a manifestAdapter) BuildCreate(_ context.Context, c RequestContext) (RequestSpec, error) { + return buildManifestOperation(a.manifest.Create, c.Request, ""), nil +} +func (a manifestAdapter) BuildAgent(_ context.Context, c AgentRequestContext) (RequestSpec, error) { + if a.manifest.Agent == nil { + return RequestSpec{}, fmt.Errorf("protocol %s has no agent operation", a.manifest.Metadata.ID) + } + request := GenerationRequest{Model: c.Model, Extra: map[string]any{"agent": c.Request}} + return buildManifestOperation(*a.manifest.Agent, request, ""), nil +} +func (a manifestAdapter) ParseAgent(_ context.Context, body []byte) (AgentResult, error) { + if a.manifest.AgentResponse == nil { + return AgentResult{}, fmt.Errorf("protocol %s has no agent response mapping", a.manifest.Metadata.ID) + } + payload, err := decodeObject(body) + if err != nil { + return AgentResult{}, err + } + response := a.manifest.AgentResponse + result := AgentResult{Text: firstPathValue(payload, response.TextPaths...), Reasoning: firstPathValue(payload, response.ReasoningPaths...)} + if response.ToolCallsPath == "" { + return result, nil + } + for _, item := range arrayValue(pathValue(payload, response.ToolCallsPath)) { + object, ok := item.(map[string]any) + if !ok { + continue + } + result.ToolCalls = append(result.ToolCalls, AgentToolCall{ + ID: firstPathValue(object, response.ToolCallIDPaths...), + Name: firstPathValue(object, response.ToolCallNamePaths...), + Arguments: firstPathJSONValue(object, response.ToolCallArgsPaths...), + }) + } + return result, nil +} +func (a manifestAdapter) ParseCreate(_ context.Context, body []byte) (CreateResult, error) { + payload, err := decodeObject(body) + if err != nil { + return CreateResult{}, err + } + return a.parse(payload, PollContext{}), nil +} +func (a manifestAdapter) BuildPoll(_ context.Context, c PollContext) (RequestSpec, error) { + if a.manifest.Poll == nil { + return RequestSpec{}, fmt.Errorf("protocol %s has no poll operation", a.manifest.Metadata.ID) + } + request := c.Request + request.Model = c.Model + return buildManifestOperation(*a.manifest.Poll, request, c.TaskID), nil +} +func (a manifestAdapter) ParsePoll(_ context.Context, c PollContext, body []byte) (PollResult, error) { + payload, err := decodeObject(body) + if err != nil { + return PollResult{}, err + } + result := a.parse(payload, c) + return PollResult{TaskID: result.TaskID, Status: result.Status, Result: result.Result, Message: result.Message}, nil +} +func (a manifestAdapter) BuildCancel(_ context.Context, c PollContext) (RequestSpec, error) { + if a.manifest.Cancel == nil { + return RequestSpec{}, fmt.Errorf("protocol %s does not support cancellation", a.manifest.Metadata.ID) + } + request := c.Request + request.Model = c.Model + return buildManifestOperation(*a.manifest.Cancel, request, c.TaskID), nil +} + +func (a manifestAdapter) parse(payload map[string]any, c PollContext) CreateResult { + id := firstPathValue(payload, a.manifest.Response.TaskIDPaths...) + if id == "" { + id = c.TaskID + } + status := normalizeStatus(firstPathValue(payload, a.manifest.Response.StatusPaths...)) + if status == "" { + status = StatusPending + } + message := firstPathValue(payload, a.manifest.Response.MessagePaths...) + result := &Result{ + Text: firstPathValue(payload, a.manifest.Response.TextPaths...), + Reasoning: firstPathValue(payload, a.manifest.Response.ReasoningPaths...), + } + for _, path := range a.manifest.Response.ResultURLPaths { + if value := firstPathValue(payload, path); value != "" { + item := MediaReference{URL: value, Kind: a.manifest.Response.ResultKind} + switch a.manifest.Response.ResultKind { + case "image": + result.Images = append(result.Images, item) + case "audio": + result.Audios = append(result.Audios, item) + default: + result.Videos = append(result.Videos, item) + } + } + } + if status == StatusPending && (result.Text != "" || len(result.Images) > 0 || len(result.Videos) > 0 || len(result.Audios) > 0) { + status = StatusSucceeded + } + if result.Text == "" && result.Reasoning == "" && len(result.Images) == 0 && len(result.Videos) == 0 && len(result.Audios) == 0 { + result = nil + } + return CreateResult{TaskID: id, Status: status, Result: result, Message: message} +} + +func buildManifestOperation(operation ManifestOperation, request GenerationRequest, taskID string) RequestSpec { + var body map[string]any + if len(operation.Fields) > 0 { + body = make(map[string]any, len(operation.Fields)) + } + for key, expression := range operation.Fields { + setMapPath(body, key, manifestExpressionValue(expression, request, taskID)) + } + path := strings.ReplaceAll(operation.Path, "{{taskId}}", url.PathEscape(taskID)) + path = strings.ReplaceAll(path, "{{model}}", url.PathEscape(request.Model)) + return RequestSpec{Method: strings.ToUpper(operation.Method), Path: path, ContentType: defaultValue(operation.ContentType, "application/json"), Body: body} +} + +func manifestExpressionValue(expression string, request GenerationRequest, taskID string) any { + expression = strings.TrimSpace(expression) + switch expression { + case "request.model": + return request.Model + case "request.prompt": + return request.Prompt + case "request.images": + return request.Images + case "request.videos": + return request.Videos + case "request.audios": + return request.Audios + case "request.imageCount": + return request.ImageCount + case "request.duration": + return request.Duration + case "request.aspectRatio": + return request.AspectRatio + case "request.resolution": + return request.Resolution + case "request.quality": + return request.Quality + case "request.generateAudio": + return request.GenerateAudio + case "request.watermark": + return request.Watermark + case "request.operation": + return request.Operation + case "taskId": + return taskID + } + if strings.HasPrefix(expression, "request.extra.") { + return pathValue(request.Extra, strings.TrimPrefix(expression, "request.extra.")) + } + return expression +} + +func pathValue(payload map[string]any, path string) any { + var value any = payload + for _, part := range strings.Split(strings.Trim(path, "."), ".") { + switch current := value.(type) { + case map[string]any: + value = current[part] + case []any: + index, err := strconv.Atoi(part) + if err != nil || index < 0 || index >= len(current) { + return nil + } + value = current[index] + default: + return nil + } + } + return value +} + +func arrayValue(value any) []any { + items, _ := value.([]any) + return items +} + +func firstPathValue(payload map[string]any, paths ...string) string { + for _, path := range paths { + value := pathValue(payload, path) + if text, ok := value.(string); ok && strings.TrimSpace(text) != "" { + return strings.TrimSpace(text) + } + } + return "" +} + +// firstPathJSONValue keeps scalar response fields string-like while allowing +// providers to return tool arguments as an object instead of a JSON string. +// The platform tool contract always stores arguments as JSON text. +func firstPathJSONValue(payload map[string]any, paths ...string) string { + for _, path := range paths { + value := pathValue(payload, path) + switch typed := value.(type) { + case string: + if strings.TrimSpace(typed) != "" { + return strings.TrimSpace(typed) + } + case nil: + continue + default: + encoded, err := json.Marshal(typed) + if err == nil && len(encoded) > 0 { + return string(encoded) + } + } + } + return "" +} + +func setMapPath(target map[string]any, path string, value any) { + parts := strings.Split(strings.Trim(path, "."), ".") + current := target + for _, part := range parts[:len(parts)-1] { + next, ok := current[part].(map[string]any) + if !ok { + next = map[string]any{} + current[part] = next + } + current = next + } + if len(parts) > 0 { + current[parts[len(parts)-1]] = value + } +} + +func isRelativePath(path string) bool { + parsed, err := url.Parse(strings.TrimSpace(path)) + return err == nil && parsed.Path != "" && strings.HasPrefix(parsed.Path, "/") && !strings.HasPrefix(parsed.Path, "//") && parsed.Host == "" && parsed.User == nil && parsed.Scheme == "" +} + +func validManifestIdentifier(value string) bool { + if len(value) == 0 || len(value) > 96 { + return false + } + for index, char := range value { + if (char >= 'a' && char <= 'z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' { + if index == 0 && (char == '-' || char == '_' || char == '.') { + return false + } + continue + } + return false + } + return true +} diff --git a/backend/internal/protocol/manifest_agent_test.go b/backend/internal/protocol/manifest_agent_test.go new file mode 100644 index 000000000..7425709d0 --- /dev/null +++ b/backend/internal/protocol/manifest_agent_test.go @@ -0,0 +1,87 @@ +package protocol + +import ( + "context" + "encoding/json" + "testing" +) + +func TestManifestAgentMapsRequestAndResponse(t *testing.T) { + manifest := []byte(`{ + "apiVersion":"v1", + "metadata":{"id":"agent-test","version":"1.0.0","name":"Agent Test","vendor":"Test","categories":["text"],"scopes":["agent"],"documentation":"# Agent Test"}, + "create":{"method":"POST","path":"/create"}, + "agent":{"method":"POST","path":"/messages","fields":{"model":"request.model","messages":"request.extra.agent.chatCompletion.messages","tools":"request.extra.agent.chatCompletion.tools","tool_choice":"request.extra.agent.chatCompletion.tool_choice"}}, + "agentResponse":{"textPaths":["choices.0.message.content"],"toolCallsPath":"choices.0.message.tool_calls","toolCallIdPaths":["id"],"toolCallNamePaths":["function.name"],"toolCallArgumentsPaths":["function.arguments"]}, + "response":{} + }`) + adapter, err := LoadManifest(manifest) + if err != nil { + t.Fatal(err) + } + agent, ok := adapter.(AgentAdapter) + if !ok { + t.Fatal("manifest adapter does not expose AgentAdapter") + } + request := map[string]any{ + "chatCompletion": map[string]any{ + "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "tools": []any{map[string]any{"type": "function"}}, + "tool_choice": "required", + }, + "responses": map[string]any{}, + } + spec, err := agent.BuildAgent(context.Background(), AgentRequestContext{Model: "model-a", Request: request}) + if err != nil { + t.Fatal(err) + } + body, ok := spec.Body.(map[string]any) + if !ok || body["model"] != "model-a" { + t.Fatalf("request body = %#v", spec.Body) + } + if _, ok := body["messages"].([]any); !ok { + t.Fatalf("messages were not mapped: %#v", body) + } + result, err := agent.ParseAgent(context.Background(), []byte(`{"choices":[{"message":{"content":"answer","tool_calls":[{"id":"call-1","function":{"name":"search","arguments":"{\"q\":\"x\"}"}}]}}]}`)) + if err != nil { + t.Fatal(err) + } + if result.Text != "answer" || len(result.ToolCalls) != 1 || result.ToolCalls[0].Name != "search" { + t.Fatalf("agent result = %#v", result) + } + if !json.Valid([]byte(result.ToolCalls[0].Arguments)) { + t.Fatalf("tool arguments are not JSON: %q", result.ToolCalls[0].Arguments) + } + + result, err = agent.ParseAgent(context.Background(), []byte(`{"choices":[{"message":{"tool_calls":[{"id":"call-2","function":{"name":"search","arguments":{"q":"object"}}}]}}]}`)) + if err != nil { + t.Fatal(err) + } + if got := result.ToolCalls[0].Arguments; got != `{"q":"object"}` { + t.Fatalf("object tool arguments = %q", got) + } +} + +func TestManifestMapsSynchronousTextResponse(t *testing.T) { + manifest := []byte(`{ + "apiVersion":"v1", + "metadata":{"id":"text-test","version":"1.0.0","name":"Text Test","vendor":"Test","categories":["text"],"scopes":["canvas"],"documentation":"# Text Test"}, + "create":{"method":"POST","path":"/chat","fields":{"prompt":"request.prompt"}}, + "response":{"textPaths":["choices.0.message.content"],"reasoningPaths":["choices.0.message.reasoning_content"]} + }`) + adapter, err := LoadManifest(manifest) + if err != nil { + t.Fatal(err) + } + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Prompt: "hello"}}) + if err != nil || spec.Body == nil { + t.Fatalf("build create = %#v, %v", spec, err) + } + result, err := adapter.ParseCreate(context.Background(), []byte(`{"choices":[{"message":{"content":"answer","reasoning_content":"thinking"}}]}`)) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusSucceeded || result.Result == nil || result.Result.Text != "answer" || result.Result.Reasoning != "thinking" { + t.Fatalf("text result = %#v", result) + } +} diff --git a/backend/internal/protocol/registry.go b/backend/internal/protocol/registry.go new file mode 100644 index 000000000..ceb85224b --- /dev/null +++ b/backend/internal/protocol/registry.go @@ -0,0 +1,134 @@ +package protocol + +import ( + "fmt" + "sort" + "strings" + "sync" +) + +type Registry struct { + mu sync.RWMutex + adapters map[string]Adapter +} + +func NewRegistry(adapters ...Adapter) (*Registry, error) { + r := &Registry{adapters: make(map[string]Adapter, len(adapters))} + for _, adapter := range adapters { + if err := r.Register(adapter); err != nil { + return nil, err + } + } + return r, nil +} + +func (r *Registry) Register(adapter Adapter) error { + if adapter == nil { + return fmt.Errorf("protocol adapter is nil") + } + metadata := adapter.Metadata() + metadata.ID = strings.TrimSpace(metadata.ID) + metadata.Version = strings.TrimSpace(metadata.Version) + if metadata.ID == "" || metadata.Version == "" { + return fmt.Errorf("protocol adapter metadata requires id and version") + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.adapters[metadata.ID]; exists { + return fmt.Errorf("protocol adapter %q is already registered", metadata.ID) + } + r.adapters[metadata.ID] = adapter + return nil +} + +// Unregister removes an installed protocol from the live registry. The caller +// owns persistence; the registry only represents the current plugin snapshot. +func (r *Registry) Unregister(id string) bool { + r.mu.Lock() + defer r.mu.Unlock() + id = strings.TrimSpace(id) + if _, ok := r.adapters[id]; !ok { + return false + } + delete(r.adapters, id) + return true +} + +func (r *Registry) RegisterManifest(data []byte) error { + adapter, err := LoadManifest(data) + if err != nil { + return err + } + return r.Register(adapter) +} + +func (r *Registry) Get(id string) (Adapter, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + adapter, ok := r.adapters[strings.TrimSpace(id)] + return adapter, ok +} + +func (r *Registry) Resolve(id string) (Adapter, bool) { + if adapter, ok := r.Get(id); ok { + return adapter, true + } + r.mu.RLock() + defer r.mu.RUnlock() + for _, adapter := range r.adapters { + for _, alias := range adapter.Metadata().LegacyAliases { + if strings.TrimSpace(alias) == strings.TrimSpace(id) { + return adapter, true + } + } + } + return nil, false +} + +func (r *Registry) List(surface Surface, capability Capability, includeUnavailable bool) []Metadata { + r.mu.RLock() + items := make([]Metadata, 0, len(r.adapters)) + for _, adapter := range r.adapters { + metadata := adapter.Metadata() + if !includeUnavailable && (!metadata.Enabled || metadata.UnavailableReason != "") { + continue + } + if capability != "" && !containsCapability(metadata.Categories, capability) { + continue + } + if surface != "" && !containsSurface(metadata.Scopes, surface) { + continue + } + items = append(items, metadata) + } + r.mu.RUnlock() + sort.Slice(items, func(i, j int) bool { return items[i].ID < items[j].ID }) + return items +} + +func (r *Registry) IsCapability(id string, capability Capability) bool { + adapter, ok := r.Resolve(id) + if !ok { + return false + } + metadata := adapter.Metadata() + return metadata.Enabled && metadata.UnavailableReason == "" && containsCapability(metadata.Categories, capability) +} + +func containsCapability(values []Capability, value Capability) bool { + for _, item := range values { + if item == value { + return true + } + } + return false +} + +func containsSurface(values []Surface, value Surface) bool { + for _, item := range values { + if item == value { + return true + } + } + return false +} diff --git a/backend/internal/protocol/types.go b/backend/internal/protocol/types.go new file mode 100644 index 000000000..ac7c98042 --- /dev/null +++ b/backend/internal/protocol/types.go @@ -0,0 +1,140 @@ +package protocol + +import "fmt" + +// Capability is the kind of model operation handled by a protocol adapter. +type Capability string + +const ( + CapabilityText Capability = "text" + CapabilityImage Capability = "image" + CapabilityVideo Capability = "video" + CapabilityAudio Capability = "audio" +) + +// Surface describes where an installed protocol may be selected. +type Surface string + +const ( + SurfaceAdminSystemChannel Surface = "admin.system-channel" + SurfaceUserCustomChannel Surface = "user.custom-channel" + SurfaceCanvas Surface = "canvas" + SurfaceCreation Surface = "creation" + SurfaceAgent Surface = "agent" +) + +type Status string + +const ( + StatusPending Status = "pending" + StatusProcessing Status = "processing" + StatusSucceeded Status = "succeeded" + StatusFailed Status = "failed" + StatusCancelled Status = "cancelled" +) + +type MediaReference struct { + URL string `json:"url,omitempty"` + DataURL string `json:"dataUrl,omitempty"` + Kind string `json:"kind,omitempty"` +} + +// GenerationRequest is the platform contract shared by canvas, creation and agent. +// Provider-specific fields belong in Extra and are never interpreted by the host. +type GenerationRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt,omitempty"` + Images []MediaReference `json:"images,omitempty"` + Videos []MediaReference `json:"videos,omitempty"` + Audios []MediaReference `json:"audios,omitempty"` + Duration int `json:"duration,omitempty"` + AspectRatio string `json:"aspectRatio,omitempty"` + Resolution string `json:"resolution,omitempty"` + Quality string `json:"quality,omitempty"` + GenerateAudio bool `json:"generateAudio,omitempty"` + Watermark bool `json:"watermark,omitempty"` + ImageCount int `json:"imageCount,omitempty"` + Operation string `json:"operation,omitempty"` + Extra map[string]any `json:"extra,omitempty"` +} + +type RequestContext struct { + BaseURL string + Request GenerationRequest +} + +type PollContext struct { + BaseURL string + Model string + Request GenerationRequest + TaskID string + Metadata map[string]any +} + +type RequestSpec struct { + Method string `json:"method"` + Path string `json:"path"` + ContentType string `json:"contentType"` + Headers map[string]string `json:"headers,omitempty"` + Body any `json:"body,omitempty"` +} + +type CreateResult struct { + TaskID string `json:"taskId,omitempty"` + Status Status `json:"status"` + Result *Result `json:"result,omitempty"` + Message string `json:"message,omitempty"` +} + +type PollResult struct { + TaskID string `json:"taskId,omitempty"` + Status Status `json:"status"` + Result *Result `json:"result,omitempty"` + Message string `json:"message,omitempty"` +} + +type Result struct { + Images []MediaReference `json:"images,omitempty"` + Videos []MediaReference `json:"videos,omitempty"` + Audios []MediaReference `json:"audios,omitempty"` + Text string `json:"text,omitempty"` + Reasoning string `json:"reasoning,omitempty"` + Usage map[string]any `json:"usage,omitempty"` +} + +type Parameter struct { + Name string `json:"name"` + Type string `json:"type"` + Required bool `json:"required,omitempty"` + Description string `json:"description,omitempty"` + Values []string `json:"values,omitempty"` + Mapping string `json:"mapping,omitempty"` +} + +type Metadata struct { + ID string `json:"id"` + Version string `json:"version"` + Name string `json:"name"` + Vendor string `json:"vendor"` + Description string `json:"description,omitempty"` + Categories []Capability `json:"categories"` + Scopes []Surface `json:"scopes"` + Parameters []Parameter `json:"parameters,omitempty"` + Create string `json:"create,omitempty"` + Poll string `json:"poll,omitempty"` + Cancel string `json:"cancel,omitempty"` + ContentType string `json:"contentType,omitempty"` + Documentation string `json:"documentation,omitempty"` + LegacyAliases []string `json:"legacyAliases,omitempty"` + Enabled bool `json:"enabled"` + Installable bool `json:"installable"` + Execution string `json:"execution,omitempty"` + UnavailableReason string `json:"unavailableReason,omitempty"` +} + +func (r RequestSpec) Validate() error { + if r.Method == "" || r.Path == "" { + return fmt.Errorf("protocol request spec is incomplete") + } + return nil +} diff --git a/backend/internal/service/admin.go b/backend/internal/service/admin.go index e7bf4443c..06770554e 100644 --- a/backend/internal/service/admin.go +++ b/backend/internal/service/admin.go @@ -817,15 +817,6 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel return req } -func validChannelInterfaceType(value model.ChannelInterfaceType) bool { - switch value { - case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceNovitaVideo, model.ChannelInterfaceMiniMaxVideo: - return true - default: - return false - } -} - func publicChannel(channel model.ModelChannel, admin bool, channelModels []model.ChannelModel) PublicModelChannel { models := make([]string, 0, len(channelModels)) modelCosts := make([]PublicChannelModelPrice, 0, len(channelModels)) diff --git a/backend/internal/service/channel_models.go b/backend/internal/service/channel_models.go index 6b6345d32..5f891ae53 100644 --- a/backend/internal/service/channel_models.go +++ b/backend/internal/service/channel_models.go @@ -11,6 +11,7 @@ import ( "time" "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/protocol" "infinite-canvas/backend/internal/repository" "gorm.io/gorm" @@ -190,7 +191,7 @@ func (s *Service) SaveAdminChannelModel(actor *model.User, channelID string, id if err != nil { return nil, err } - modelKey, providerModelKey, capability, protocol, err := normalizeChannelModelContract(channel, req) + modelKey, providerModelKey, capability, protocol, err := s.normalizeChannelModelContract(channel, req) if err != nil { return nil, err } @@ -529,7 +530,7 @@ func (s *Service) TestAdminChannelModel(ctx context.Context, actor *model.User, if err != nil { return nil, err } - modelKey, providerModelKey, capability, protocol, err := normalizeChannelModelContract(channel, req) + modelKey, providerModelKey, capability, protocol, err := s.normalizeChannelModelContract(channel, req) if err != nil { return nil, err } @@ -610,6 +611,7 @@ func (s *Service) TestAdminChannelModel(ctx context.Context, actor *model.User, Operation: "admin_model_test", Model: modelKey, VideoSeconds: videoSecondsValue, }) testCtx = withProviderOutboundPolicy(testCtx, input.Config) + testCtx = withProtocolRegistry(testCtx, s.protocolRegistry()) startedAt := time.Now() switch capability { case "text": @@ -648,6 +650,14 @@ func imageTestDefaults(profile *ImageCapabilityConfig) (string, string) { } func normalizeChannelModelContract(channel *model.ModelChannel, req ChannelModelRequest) (string, string, string, model.ChannelInterfaceType, error) { + return normalizeChannelModelContractWithRegistry(protocol.Builtins(), channel, req) +} + +func (s *Service) normalizeChannelModelContract(channel *model.ModelChannel, req ChannelModelRequest) (string, string, string, model.ChannelInterfaceType, error) { + return normalizeChannelModelContractWithRegistry(s.protocolRegistry(), channel, req) +} + +func normalizeChannelModelContractWithRegistry(registry *protocol.Registry, channel *model.ModelChannel, req ChannelModelRequest) (string, string, string, model.ChannelInterfaceType, error) { modelKey := strings.TrimPrefix(strings.TrimSpace(req.ModelKey), "models/") if modelKey == "" { return "", "", "", "", BadAuthRequest("请填写模型标识") @@ -660,11 +670,12 @@ func normalizeChannelModelContract(channel *model.ModelChannel, req ChannelModel if capability == "" { return "", "", "", "", BadAuthRequest("请选择模型能力") } - protocol := model.ChannelInterfaceType(strings.TrimSpace(req.Protocol)) - if !validChannelInterfaceType(protocol) { + adapter, ok := registry.Resolve(strings.TrimSpace(req.Protocol)) + if !ok || !adapter.Metadata().Enabled || adapter.Metadata().UnavailableReason != "" { return "", "", "", "", BadAuthRequest("请选择有效的模型请求协议") } - if expected := capabilityForProtocol(protocol); expected != "" && expected != capability { + protocol := model.ChannelInterfaceType(adapter.Metadata().ID) + if expected := protocolCapabilityFromMetadata(adapter.Metadata()); expected != "" && expected != capability { return "", "", "", "", BadAuthRequest("模型能力与请求协议不匹配") } if (protocol == model.ChannelInterfaceVolcengineJiMengImage || protocol == model.ChannelInterfaceVolcengineJiMengVideo) && (strings.TrimSpace(channel.APIKey) == "" || strings.TrimSpace(channel.SecretKey) == "") { @@ -813,17 +824,17 @@ func (s *Service) syncChannelModelNames(channel *model.ModelChannel) error { return s.repo.Save(channel) } -func capabilityForProtocol(protocol model.ChannelInterfaceType) string { - switch protocol { - case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage: - return "image" - case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio: - return "audio" - case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceNovitaVideo, model.ChannelInterfaceMiniMaxVideo: - return "video" - case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse: - return "text" - default: +func (s *Service) capabilityForProtocol(protocol model.ChannelInterfaceType) string { + metadata, ok := s.channelProtocolMetadata(string(protocol)) + if !ok { return "" } + return protocolCapabilityFromMetadata(metadata) +} + +func protocolCapabilityFromMetadata(metadata protocol.Metadata) string { + if len(metadata.Categories) == 0 { + return "" + } + return string(metadata.Categories[0]) } diff --git a/backend/internal/service/coordination.go b/backend/internal/service/coordination.go index 0b3814d53..b390f2db4 100644 --- a/backend/internal/service/coordination.go +++ b/backend/internal/service/coordination.go @@ -311,6 +311,9 @@ func (s *Service) AcquireChannelSlot(ctx context.Context, channelID string, fall } func (s *Service) ValidateRuntime() error { + if s.pluginRuntimeErr != nil { + return s.pluginRuntimeErr + } return s.runtimeErr } diff --git a/backend/internal/service/feature_availability.go b/backend/internal/service/feature_availability.go index 58e9f32be..a322789c4 100644 --- a/backend/internal/service/feature_availability.go +++ b/backend/internal/service/feature_availability.go @@ -14,19 +14,23 @@ import ( const featureAvailabilitySettingKey = "feature_availability" const ( - FeatureShortDrama = "shortDrama" - FeatureTaskCenter = "taskCenter" - FeatureCredits = "credits" - FeatureCustomChannels = "customChannels" - FeatureFrontendModels = "frontendModels" + FeatureShortDrama = "shortDrama" + FeatureTaskCenter = "taskCenter" + FeatureCredits = "credits" + FeatureCustomChannels = "customChannels" + FeatureFrontendModels = "frontendModels" + FeaturePluginCenter = "pluginCenter" + FeatureSystemPlugins = "systemPluginsVisibleToUsers" ) type FeatureAvailability struct { - ShortDramaEnabled bool `json:"shortDramaEnabled"` - TaskCenterEnabled bool `json:"taskCenterEnabled"` - CreditsEnabled bool `json:"creditsEnabled"` - CustomChannelsEnabled bool `json:"customChannelsEnabled"` - FrontendModelsEnabled bool `json:"frontendModelsEnabled"` + ShortDramaEnabled bool `json:"shortDramaEnabled"` + TaskCenterEnabled bool `json:"taskCenterEnabled"` + CreditsEnabled bool `json:"creditsEnabled"` + CustomChannelsEnabled bool `json:"customChannelsEnabled"` + FrontendModelsEnabled bool `json:"frontendModelsEnabled"` + PluginCenterEnabled bool `json:"pluginCenterEnabled"` + SystemPluginsVisibleToUsers bool `json:"systemPluginsVisibleToUsers"` } type PublicFeatureAvailability struct { @@ -39,7 +43,15 @@ type PublicFeatureAvailability struct { func defaultFeatureAvailability() FeatureAvailability { // 缺少配置代表尚未由运维接管；前台模型需要明确配置后才开放。 - return FeatureAvailability{ShortDramaEnabled: true, TaskCenterEnabled: true, CreditsEnabled: true, CustomChannelsEnabled: true, FrontendModelsEnabled: false} + return FeatureAvailability{ + ShortDramaEnabled: true, + TaskCenterEnabled: true, + CreditsEnabled: true, + CustomChannelsEnabled: true, + FrontendModelsEnabled: false, + PluginCenterEnabled: true, + SystemPluginsVisibleToUsers: true, + } } func (s *Service) FeatureAvailability() (*PublicFeatureAvailability, error) { @@ -98,6 +110,10 @@ func (s *Service) FeatureEnabled(feature string) (bool, error) { return value.CustomChannelsEnabled, nil case FeatureFrontendModels: return value.FrontendModelsEnabled, nil + case FeaturePluginCenter: + return value.PluginCenterEnabled, nil + case FeatureSystemPlugins: + return value.SystemPluginsVisibleToUsers, nil default: return false, errors.New("未知功能开放配置") } @@ -122,6 +138,10 @@ func (s *Service) RequireFeature(feature string) error { return Forbidden("自定义渠道暂未开放") case FeatureFrontendModels: return Forbidden("前台模型目录暂未开放") + case FeaturePluginCenter: + return Forbidden("插件中心暂未开放") + case FeatureSystemPlugins: + return Forbidden("系统插件暂未向普通用户展示") default: return Forbidden("该功能暂未开放") } diff --git a/backend/internal/service/feature_availability_test.go b/backend/internal/service/feature_availability_test.go index fa6f9c888..4164c69c2 100644 --- a/backend/internal/service/feature_availability_test.go +++ b/backend/internal/service/feature_availability_test.go @@ -5,6 +5,7 @@ import ( "testing" "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/protocol" "infinite-canvas/backend/internal/repository" "gorm.io/driver/sqlite" @@ -18,7 +19,7 @@ func TestFeatureAvailabilityDefaultsToDisableFrontendModels(t *testing.T) { if err != nil { t.Fatal(err) } - if setting.Configured || !setting.ShortDramaEnabled || !setting.TaskCenterEnabled || !setting.CreditsEnabled || !setting.CustomChannelsEnabled || setting.FrontendModelsEnabled { + if setting.Configured || !setting.ShortDramaEnabled || !setting.TaskCenterEnabled || !setting.CreditsEnabled || !setting.CustomChannelsEnabled || setting.FrontendModelsEnabled || !setting.PluginCenterEnabled || !setting.SystemPluginsVisibleToUsers { t.Fatalf("FeatureAvailability() = %#v", setting) } } @@ -34,7 +35,7 @@ func TestFeatureAvailabilityLegacySettingKeepsCustomChannelsEnabled(t *testing.T if err != nil { t.Fatal(err) } - if setting.ShortDramaEnabled || !setting.CustomChannelsEnabled || setting.FrontendModelsEnabled { + if setting.ShortDramaEnabled || !setting.CustomChannelsEnabled || setting.FrontendModelsEnabled || !setting.PluginCenterEnabled || !setting.SystemPluginsVisibleToUsers { t.Fatalf("FeatureAvailability() = %#v", setting) } } @@ -42,15 +43,18 @@ func TestFeatureAvailabilityLegacySettingKeepsCustomChannelsEnabled(t *testing.T func TestUpdateFeatureAvailabilityPersistsAndAudits(t *testing.T) { svc, db := newFeatureAvailabilityTestService(t) actor := &model.User{ID: "admin-1", Role: model.UserRoleAdmin} - want := FeatureAvailability{ShortDramaEnabled: false, TaskCenterEnabled: true, CreditsEnabled: false, CustomChannelsEnabled: true} + want := FeatureAvailability{ShortDramaEnabled: false, TaskCenterEnabled: true, CreditsEnabled: false, CustomChannelsEnabled: true, PluginCenterEnabled: false, SystemPluginsVisibleToUsers: false} setting, err := svc.UpdateFeatureAvailability(actor, want) if err != nil { t.Fatal(err) } - if !setting.Configured || setting.ShortDramaEnabled || !setting.TaskCenterEnabled || setting.CreditsEnabled || !setting.CustomChannelsEnabled { + if !setting.Configured || setting.ShortDramaEnabled || !setting.TaskCenterEnabled || setting.CreditsEnabled || !setting.CustomChannelsEnabled || setting.PluginCenterEnabled || setting.SystemPluginsVisibleToUsers { t.Fatalf("UpdateFeatureAvailability() = %#v", setting) } + if enabled, err := svc.FeatureEnabled(FeaturePluginCenter); err != nil || enabled { + t.Fatalf("FeatureEnabled(pluginCenter) = %v, %v", enabled, err) + } if err := svc.RequireFeature(FeatureShortDrama); err == nil { t.Fatal("RequireFeature(shortDrama) error = nil") } else { @@ -68,6 +72,36 @@ func TestUpdateFeatureAvailabilityPersistsAndAudits(t *testing.T) { } } +func TestPluginsForUserHidesBundledPluginsOnlyForOrdinaryUsers(t *testing.T) { + svc, _ := newFeatureAvailabilityTestService(t) + admin := &model.User{ID: "admin-1", Role: model.UserRoleAdmin} + if _, err := svc.UpdateFeatureAvailability(admin, FeatureAvailability{ + ShortDramaEnabled: true, TaskCenterEnabled: true, CreditsEnabled: true, + CustomChannelsEnabled: true, PluginCenterEnabled: true, SystemPluginsVisibleToUsers: false, + }); err != nil { + t.Fatal(err) + } + svc.pluginRuntime = &pluginRuntime{plugins: map[string]pluginRecord{ + "bundled": {Source: "bundled", Metadata: protocol.Metadata{ID: "bundled", Name: "系统协议", Version: "1"}}, + "uploaded": {Source: "uploaded", Metadata: protocol.Metadata{ID: "uploaded", Name: "用户协议", Version: "1"}}, + }} + + visibleToUser, err := svc.PluginsForUser(&model.User{ID: "user-1", Role: model.UserRoleUser}) + if err != nil { + t.Fatal(err) + } + if len(visibleToUser) != 1 || visibleToUser[0].Source != "uploaded" { + t.Fatalf("PluginsForUser(user) = %#v", visibleToUser) + } + visibleToAdmin, err := svc.PluginsForUser(admin) + if err != nil { + t.Fatal(err) + } + if len(visibleToAdmin) != 2 { + t.Fatalf("PluginsForUser(admin) = %#v", visibleToAdmin) + } +} + func TestCustomChannelTaskInputRequiresFeature(t *testing.T) { svc, _ := newFeatureAvailabilityTestService(t) actor := &model.User{ID: "admin-1", Role: model.UserRoleAdmin} diff --git a/backend/internal/service/model_catalog.go b/backend/internal/service/model_catalog.go index 6dc7e0760..49fa66b91 100644 --- a/backend/internal/service/model_catalog.go +++ b/backend/internal/service/model_catalog.go @@ -36,6 +36,7 @@ type PublicChannelModel struct { ModelKey string `json:"modelKey"` DisplayName string `json:"displayName"` Capability string `json:"capability"` + Protocol model.ChannelInterfaceType `json:"protocol"` CapabilityConfig map[string]any `json:"capabilityConfig,omitempty"` PriceTiers []PublicChannelModelPriceTier `json:"priceTiers"` PricingMode string `json:"pricingMode"` @@ -180,6 +181,7 @@ func (s *Service) sanitizeChannelModel(cm *model.ChannelModel) PublicChannelMode ModelKey: cm.ModelKey, DisplayName: cm.DisplayName, Capability: cm.Capability, + Protocol: cm.Protocol, CapabilityConfig: capabilityConfig, PriceTiers: publicTiers, PricingMode: pricingMode, diff --git a/backend/internal/service/protocol_plugins.go b/backend/internal/service/protocol_plugins.go new file mode 100644 index 000000000..b00423073 --- /dev/null +++ b/backend/internal/service/protocol_plugins.go @@ -0,0 +1,399 @@ +package service + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "infinite-canvas/backend/internal/protocol" +) + +const protocolPluginMaxBytes = 512 << 10 + +// PluginView is the backend representation consumed by the single frontend +// plugin center. Protocol-specific runtime data is nested under Protocol so +// the public plugin contract can grow without adding another center API. +type PluginView struct { + Manifest PluginManifestView `json:"manifest"` + Source string `json:"source"` + FileName string `json:"fileName"` + SHA256 string `json:"sha256"` + InstalledAt time.Time `json:"installedAt"` + UpdatedAt time.Time `json:"updatedAt"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +type PluginManifestView struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + APIVersion string `json:"apiVersion"` + Category string `json:"category"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + Surfaces []string `json:"surfaces"` + Permissions []string `json:"permissions"` + Trusted bool `json:"trusted"` + Kind string `json:"kind"` + Protocol PluginProtocolView `json:"protocol"` +} + +type PluginProtocolView struct { + Categories []string `json:"categories"` + Scopes []string `json:"scopes"` + Create string `json:"create,omitempty"` + Poll string `json:"poll,omitempty"` + Cancel string `json:"cancel,omitempty"` + ContentType string `json:"contentType,omitempty"` + Documentation string `json:"documentation,omitempty"` + Parameters []protocol.Parameter `json:"parameters,omitempty"` +} + +type pluginRecord struct { + Raw []byte + Metadata protocol.Metadata + Source string + FileName string + SHA256 string + InstalledAt time.Time + UpdatedAt time.Time + Status string + Error string +} + +type pluginRuntime struct { + mu sync.RWMutex + mutationMu sync.Mutex + dir string + plugins map[string]pluginRecord + registry *protocol.Registry +} + +func newPluginRuntime(dataDir string) (*pluginRuntime, error) { + dir := filepath.Join(dataDir, "plugins") + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create plugin directory: %w", err) + } + center := &pluginRuntime{dir: dir, plugins: make(map[string]pluginRecord)} + if err := center.bootstrapBundledPlugins(); err != nil { + return nil, err + } + if err := center.reload(); err != nil { + return nil, err + } + return center, nil +} + +func (c *pluginRuntime) bootstrapBundledPlugins() error { + items := protocol.Builtins().List("", "", true) + for _, metadata := range items { + protocol.AttachDocumentation(&metadata) + metadata.Execution = "host:" + metadata.ID + metadata.Installable = true + path := filepath.Join(c.dir, metadata.ID+".json") + existing, err := os.ReadFile(path) + if err == nil { + var installed protocol.Manifest + if err := json.Unmarshal(existing, &installed); err != nil { + return fmt.Errorf("decode bundled protocol %s: %w", metadata.ID, err) + } + if installed.Metadata.Execution != metadata.Execution { + return fmt.Errorf("protocol id %q is reserved by a bundled plugin", metadata.ID) + } + if metadata.Enabled { + metadata.Enabled = installed.Metadata.Enabled + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + manifest := protocol.Manifest{ + APIVersion: "v1", + Metadata: metadata, + Create: protocol.ManifestOperation{Method: "POST", Path: "/__host__/" + metadata.ID}, + Response: protocol.ManifestResponse{}, + } + data, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("encode bundled protocol %s: %w", metadata.ID, err) + } + if bytes.Equal(existing, data) { + continue + } + if err := writePluginFile(path, data); err != nil { + return fmt.Errorf("install bundled protocol %s: %w", metadata.ID, err) + } + } + return nil +} + +func (c *pluginRuntime) reload() error { + c.mu.Lock() + defer c.mu.Unlock() + entries, err := os.ReadDir(c.dir) + if err != nil { + return err + } + plugins := make(map[string]pluginRecord) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + path := filepath.Join(c.dir, entry.Name()) + data, readErr := os.ReadFile(path) + if readErr != nil { + return fmt.Errorf("read protocol plugin %s: %w", entry.Name(), readErr) + } + if len(data) > protocolPluginMaxBytes { + return fmt.Errorf("protocol plugin %s exceeds %d bytes", entry.Name(), protocolPluginMaxBytes) + } + var manifest protocol.Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return fmt.Errorf("decode protocol plugin %s: %w", entry.Name(), err) + } + metadata := manifest.Metadata + if strings.TrimSpace(metadata.ID) == "" { + return fmt.Errorf("protocol plugin %s has no metadata id", entry.Name()) + } + if _, exists := plugins[metadata.ID]; exists { + return fmt.Errorf("duplicate installed protocol %q", metadata.ID) + } + now := time.Now().UTC() + stat, _ := os.Stat(path) + updated := now + if stat != nil { + updated = stat.ModTime().UTC() + } + plugins[metadata.ID] = pluginRecord{Raw: data, Metadata: metadata, Source: pluginSource(metadata), FileName: entry.Name(), SHA256: pluginHash(data), InstalledAt: updated, UpdatedAt: updated, Status: "invalid"} + } + registry, err := protocol.NewRegistry() + if err != nil { + return err + } + for id, record := range plugins { + adapter, loadErr := protocol.LoadInstalledPlugin(record.Raw, func(execution string) (protocol.Adapter, bool) { + return protocol.Builtins().Resolve(execution) + }) + if loadErr != nil { + record.Metadata.Enabled = false + record.Metadata.UnavailableReason = loadErr.Error() + record.Error = loadErr.Error() + _ = registry.Register(protocol.UnavailableAdapter{Info: record.Metadata}) + plugins[id] = record + continue + } + if !record.Metadata.Enabled { + record.Status = "disabled" + _ = registry.Register(protocol.UnavailableAdapter{Info: record.Metadata}) + plugins[id] = record + continue + } + if err := registry.Register(adapter); err != nil { + record.Error = err.Error() + plugins[id] = record + continue + } + record.Status = "enabled" + plugins[id] = record + } + c.plugins = plugins + c.registry = registry + return nil +} + +func (c *pluginRuntime) list() []PluginView { + c.mu.RLock() + defer c.mu.RUnlock() + items := make([]PluginView, 0, len(c.plugins)) + for _, item := range c.plugins { + items = append(items, PluginView{Manifest: pluginManifestView(item.Metadata, item.Source), Source: item.Source, FileName: item.FileName, SHA256: item.SHA256, InstalledAt: item.InstalledAt, UpdatedAt: item.UpdatedAt, Status: item.Status, Error: item.Error}) + } + sort.Slice(items, func(i, j int) bool { return items[i].Manifest.ID < items[j].Manifest.ID }) + return items +} + +func (c *pluginRuntime) registrySnapshot() *protocol.Registry { + c.mu.RLock() + defer c.mu.RUnlock() + return c.registry +} + +func (c *pluginRuntime) install(data []byte) (PluginView, error) { + c.mutationMu.Lock() + defer c.mutationMu.Unlock() + if len(data) == 0 || len(data) > protocolPluginMaxBytes { + return PluginView{}, fmt.Errorf("plugin manifest must be between 1 and %d bytes", protocolPluginMaxBytes) + } + var manifest protocol.Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return PluginView{}, fmt.Errorf("插件清单 JSON 无效：%w", err) + } + if strings.HasPrefix(strings.TrimSpace(manifest.Metadata.Execution), "host:") { + return PluginView{}, errors.New("上传插件不能使用宿主内置执行器") + } + if _, err := protocol.LoadManifest(data); err != nil { + return PluginView{}, err + } + c.mu.RLock() + _, exists := c.plugins[manifest.Metadata.ID] + c.mu.RUnlock() + if exists { + return PluginView{}, fmt.Errorf("插件 ID %q 已存在；当前上传入口不支持覆盖安装", manifest.Metadata.ID) + } + manifest.Metadata.Enabled = true + data, err := json.Marshal(manifest) + if err != nil { + return PluginView{}, err + } + path := filepath.Join(c.dir, manifest.Metadata.ID+".json") + if err := c.persistAndReload(path, data, nil); err != nil { + return PluginView{}, fmt.Errorf("保存插件失败：%w", err) + } + for _, item := range c.list() { + if item.Manifest.ID == manifest.Metadata.ID { + return item, nil + } + } + return PluginView{}, errors.New("插件保存后未加载") +} + +func (c *pluginRuntime) setEnabled(id string, enabled bool) (PluginView, error) { + c.mutationMu.Lock() + defer c.mutationMu.Unlock() + c.mu.RLock() + record, ok := c.plugins[strings.TrimSpace(id)] + c.mu.RUnlock() + if !ok { + return PluginView{}, fmt.Errorf("插件 %q 不存在", id) + } + var manifest protocol.Manifest + if err := json.Unmarshal(record.Raw, &manifest); err != nil { + return PluginView{}, err + } + manifest.Metadata.Enabled = enabled + data, err := json.Marshal(manifest) + if err != nil { + return PluginView{}, err + } + if err := c.persistAndReload(filepath.Join(c.dir, record.FileName), data, record.Raw); err != nil { + return PluginView{}, err + } + for _, item := range c.list() { + if item.Manifest.ID == manifest.Metadata.ID { + return item, nil + } + } + return PluginView{}, errors.New("插件状态更新后未加载") +} + +func pluginManifestView(metadata protocol.Metadata, source string) PluginManifestView { + if source == "bundled" { + if adapter, ok := protocol.Builtins().Get(metadata.ID); ok { + current := adapter.Metadata() + current.Enabled = current.Enabled && metadata.Enabled + metadata = current + } + protocol.AttachDocumentation(&metadata) + } + categories := make([]string, 0, len(metadata.Categories)) + for _, item := range metadata.Categories { + categories = append(categories, string(item)) + } + scopes := make([]string, 0, len(metadata.Scopes)) + for _, item := range metadata.Scopes { + scopes = append(scopes, string(item)) + } + return PluginManifestView{ + ID: metadata.ID, Name: metadata.Name, Version: metadata.Version, APIVersion: "v1", + Category: "protocol", Description: metadata.Description, Author: metadata.Vendor, + Surfaces: []string{"hybrid"}, Permissions: []string{"generation.run"}, + Trusted: source == "bundled", Kind: "protocol", + Protocol: PluginProtocolView{Categories: categories, Scopes: scopes, Create: metadata.Create, Poll: metadata.Poll, Cancel: metadata.Cancel, ContentType: metadata.ContentType, Documentation: metadata.Documentation, Parameters: metadata.Parameters}, + } +} + +func (c *pluginRuntime) uninstall(id string) error { + c.mutationMu.Lock() + defer c.mutationMu.Unlock() + c.mu.RLock() + record, ok := c.plugins[strings.TrimSpace(id)] + c.mu.RUnlock() + if !ok { + return fmt.Errorf("协议插件 %q 不存在", id) + } + if record.Source == "bundled" { + return fmt.Errorf("内置协议插件 %q 不能卸载，可停用该插件", id) + } + path := filepath.Join(c.dir, record.FileName) + if err := os.Remove(path); err != nil { + return err + } + if err := c.reload(); err != nil { + _ = writePluginFile(path, record.Raw) + _ = c.reload() + return err + } + return nil +} + +func (c *pluginRuntime) persistAndReload(path string, data, rollback []byte) error { + if err := writePluginFile(path, data); err != nil { + return err + } + if err := c.reload(); err != nil { + if rollback == nil { + _ = os.Remove(path) + } else { + _ = writePluginFile(path, rollback) + } + _ = c.reload() + return err + } + return nil +} + +func writePluginFile(path string, data []byte) error { + temporary, err := os.CreateTemp(filepath.Dir(path), ".plugin-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryPath, path) +} + +func pluginSource(metadata protocol.Metadata) string { + if strings.HasPrefix(strings.TrimSpace(metadata.Execution), "host:") { + return "bundled" + } + return "uploaded" +} + +func pluginHash(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} diff --git a/backend/internal/service/protocol_registry.go b/backend/internal/service/protocol_registry.go new file mode 100644 index 000000000..2325de263 --- /dev/null +++ b/backend/internal/service/protocol_registry.go @@ -0,0 +1,135 @@ +package service + +import ( + "context" + "fmt" + "strings" + + "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/protocol" +) + +type protocolRegistryContextKey struct{} + +func withProtocolRegistry(ctx context.Context, registry *protocol.Registry) context.Context { + return context.WithValue(ctx, protocolRegistryContextKey{}, registry) +} + +func declarativeProtocolAdapterForContext(ctx context.Context, id string) (protocol.Adapter, bool) { + registry, _ := ctx.Value(protocolRegistryContextKey{}).(*protocol.Registry) + if registry == nil { + registry = protocol.Builtins() + } + adapter, ok := registry.Resolve(strings.TrimSpace(id)) + if !ok || adapter.Metadata().Execution != "declarative" { + return nil, false + } + return adapter, true +} + +func agentProtocolAdapterForContext(ctx context.Context, id string) (protocol.AgentAdapter, bool) { + registry, _ := ctx.Value(protocolRegistryContextKey{}).(*protocol.Registry) + if registry == nil { + registry = protocol.Builtins() + } + adapter, ok := registry.Resolve(strings.TrimSpace(id)) + if !ok || adapter.Metadata().Execution != "declarative" { + return nil, false + } + agentAdapter, ok := adapter.(protocol.AgentAdapter) + return agentAdapter, ok +} + +// ProtocolCatalog is the backend source of truth for the frontend plugin +// center's protocol filtering. +// includeUnavailable is reserved for administrators so incomplete plugins remain visible +// with an explicit reason instead of silently becoming a selectable option. +func (s *Service) ProtocolCatalog(scope, capability string, includeUnavailable bool) []protocol.Metadata { + return s.protocolRegistry().List(protocol.Surface(strings.TrimSpace(scope)), protocol.Capability(strings.TrimSpace(capability)), includeUnavailable) +} + +func (s *Service) protocolRegistry() *protocol.Registry { + if s.pluginRuntime != nil { + if registry := s.pluginRuntime.registrySnapshot(); registry != nil { + return registry + } + } + return protocol.Builtins() +} + +func (s *Service) protocolMetadata(id string) (protocol.Metadata, bool) { + adapter, ok := s.protocolRegistry().Resolve(strings.TrimSpace(id)) + if !ok { + return protocol.Metadata{}, false + } + return adapter.Metadata(), true +} + +func (s *Service) channelProtocolMetadata(id string) (protocol.Metadata, bool) { + return s.protocolMetadata(id) +} + +func (s *Service) canonicalProtocolID(id string) (string, bool) { + adapter, ok := s.protocolRegistry().Resolve(strings.TrimSpace(id)) + if !ok { + return "", false + } + return adapter.Metadata().ID, true +} + +func (s *Service) protocolIsSelectable(id string) bool { + metadata, ok := s.channelProtocolMetadata(id) + return ok && metadata.Enabled && metadata.UnavailableReason == "" +} + +func (s *Service) Plugins() []PluginView { + if s.pluginRuntime == nil { + return []PluginView{} + } + return s.pluginRuntime.list() +} + +// PluginsForUser keeps the plugin center response aligned with the public +// feature switch. Administrators must still be able to inspect and recover +// bundled plugins even when ordinary users cannot see them. +func (s *Service) PluginsForUser(actor *model.User) ([]PluginView, error) { + items := s.Plugins() + if actor != nil && actor.Role == model.UserRoleAdmin { + return items, nil + } + visible, err := s.FeatureEnabled(FeatureSystemPlugins) + if err != nil { + return nil, err + } + if visible { + return items, nil + } + filtered := make([]PluginView, 0, len(items)) + for _, item := range items { + if item.Source != "bundled" { + filtered = append(filtered, item) + } + } + return filtered, nil +} + +func (s *Service) InstallPlugin(data []byte) (PluginView, error) { + if s.pluginRuntime == nil { + return PluginView{}, fmt.Errorf("插件运行时未初始化") + } + return s.pluginRuntime.install(data) +} + +func (s *Service) SetPluginEnabled(id string, enabled bool) (PluginView, error) { + if s.pluginRuntime == nil { + return PluginView{}, fmt.Errorf("插件运行时未初始化") + } + return s.pluginRuntime.setEnabled(id, enabled) +} + +func (s *Service) UninstallPlugin(id string) error { + if s.pluginRuntime == nil { + return fmt.Errorf("插件运行时未初始化") + } + return s.pluginRuntime.uninstall(id) +} diff --git a/backend/internal/service/protocol_runtime_test.go b/backend/internal/service/protocol_runtime_test.go new file mode 100644 index 000000000..f55df1bd5 --- /dev/null +++ b/backend/internal/service/protocol_runtime_test.go @@ -0,0 +1,114 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "infinite-canvas/backend/internal/protocol" +) + +func TestPluginViewIncludesDocumentationForEveryBundledProtocol(t *testing.T) { + center, err := newPluginRuntime(t.TempDir()) + if err != nil { + t.Fatal(err) + } + plugins := center.list() + if len(plugins) != len(protocol.Builtins().List("", "", true)) { + t.Fatalf("plugin views = %d, builtins = %d", len(plugins), len(protocol.Builtins().List("", "", true))) + } + for _, plugin := range plugins { + if plugin.Source != "bundled" { + continue + } + document := strings.TrimSpace(plugin.Manifest.Protocol.Documentation) + if document == "" || !strings.Contains(document, "## 影策运行时合同") { + t.Errorf("bundled plugin %q has no complete documentation in PluginView", plugin.Manifest.ID) + } + } +} + +func TestPluginRuntimeIsTheProtocolSourceOfTruth(t *testing.T) { + dataDir := t.TempDir() + center, err := newPluginRuntime(dataDir) + if err != nil { + t.Fatal(err) + } + if got := center.registrySnapshot().List("", "", false); len(got) == 0 { + t.Fatal("bundled protocols were not reconciled into a new plugin directory") + } + manifest := []byte(`{"apiVersion":"v1","metadata":{"id":"uploaded-runtime","version":"1.0.0","name":"Uploaded Runtime","vendor":"Test","categories":["video"],"scopes":["canvas"],"documentation":"# Uploaded Runtime"},"create":{"method":"POST","path":"/tasks","fields":{"prompt":"request.prompt"}},"response":{"statusPaths":["status"]}}`) + plugin, err := center.install(manifest) + if err != nil { + t.Fatal(err) + } + if plugin.Status != "enabled" || !center.registrySnapshot().IsCapability("uploaded-runtime", protocol.CapabilityVideo) { + t.Fatalf("installed plugin was not activated: %#v", plugin) + } + if _, err := center.install(manifest); err == nil { + t.Fatal("duplicate plugin id overwrote an installed plugin") + } + if _, err := center.setEnabled("uploaded-runtime", false); err != nil { + t.Fatal(err) + } + if _, ok := center.registrySnapshot().Resolve("uploaded-runtime"); !ok { + t.Fatal("disabled plugin was removed from registry snapshot instead of being represented as unavailable") + } + if center.registrySnapshot().IsCapability("uploaded-runtime", protocol.CapabilityVideo) { + t.Fatal("disabled plugin remained selectable") + } + if err := center.uninstall("uploaded-runtime"); err != nil { + t.Fatal(err) + } + if _, ok := center.registrySnapshot().Resolve("uploaded-runtime"); ok { + t.Fatal("uninstalled plugin remained selectable") + } +} + +func TestDeclarativeProtocolRuntimeExecutesCreatePollAndDownload(t *testing.T) { + manifest := []byte(`{ + "apiVersion":"v1", + "metadata":{"id":"test-declarative-video-runtime","version":"1.0.0","name":"Test Declarative Video","vendor":"Test","categories":["video"],"scopes":["canvas"],"documentation":"# Test Declarative Video"}, + "create":{"method":"POST","path":"/tasks","fields":{"model":"request.model","prompt":"request.prompt","seconds":"request.duration"}}, + "poll":{"method":"GET","path":"/tasks/{{taskId}}"}, + "response":{"taskIdPaths":["id"],"statusPaths":["status"],"resultUrlPaths":["video_url"],"resultKind":"video"} + }`) + center, err := newPluginRuntime(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := center.install(manifest); err != nil { + t.Fatal(err) + } + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/tasks": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"task-1","status":"pending"}`)) + case "/v1/tasks/task-1": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"task-1","status":"succeeded","video_url":"` + server.URL + `/media.mp4"}`)) + case "/media.mp4": + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("video")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + config := providerConfig{BaseURL: server.URL + "/v1", APIKey: "key", Model: "test-model", APIFormat: "openai", InterfaceType: "test-declarative-video-runtime", AllowLocalChannel: true} + ctx := withProviderOutboundPolicy(context.Background(), config) + ctx = withProtocolRegistry(ctx, center.registrySnapshot()) + result, err := runDeclarativeProtocolTask(ctx, canvasGenerationInput{Mode: "video", Prompt: "a clip", Config: config}) + if err != nil { + t.Fatal(err) + } + if result["mode"] != "video" { + t.Fatalf("result = %#v", result) + } +} diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index 0aa52f4d8..faa02cb31 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -22,6 +22,7 @@ import ( "time" "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/protocol" "gorm.io/gorm" ) @@ -253,6 +254,7 @@ func providerPayloadErrorMessage(raw string) string { } func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string, taskProjectID string, taskType string, fallbackPrompt string, rawInput string) (map[string]interface{}, error) { + ctx = withProtocolRegistry(ctx, s.protocolRegistry()) var input canvasGenerationInput if err := json.Unmarshal([]byte(rawInput), &input); err != nil { return nil, fmt.Errorf("任务输入解析失败：%w", err) @@ -293,7 +295,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string if strings.TrimSpace(input.Config.BaseURL) == "" || strings.TrimSpace(input.Config.APIKey) == "" || strings.TrimSpace(input.Config.Model) == "" { return nil, errors.New("后端生成任务缺少 Base URL、API Key 或模型名") } - if err := validateGenerationInterface(input.Mode, input.Config.InterfaceType); err != nil { + if err := s.validateGenerationInterface(input.Mode, input.Config.InterfaceType); err != nil { return nil, err } if isVolcengineJiMengProtocol(input.Config.InterfaceType) && strings.TrimSpace(input.Config.SecretKey) == "" { @@ -345,6 +347,12 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string } func runAgentToolTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if adapter, ok := agentProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok { + return runDeclarativeAgentTask(ctx, input, adapter) + } + if input.AgentRequests == nil { + return nil, errors.New("画布 Agent 工具请求缺少协议参数") + } request := input.AgentRequests.ChatCompletion path := "/chat/completions" protocol := "chat-completion" @@ -352,11 +360,17 @@ func runAgentToolTask(ctx context.Context, input canvasGenerationInput) (map[str request = input.AgentRequests.Responses path = "/responses" protocol = "responses" + } else if input.Config.InterfaceType == string(model.ChannelInterfaceClaudeAPI) { + path = "/messages" + protocol = "claude-api" } if request == nil { return nil, errors.New("画布 Agent 工具请求缺少协议参数") } body := cloneStringAnyMap(request) + if protocol == "claude-api" { + body = claudeAgentBody(body) + } body["model"] = input.Config.Model var payload map[string]interface{} err := postJSON(ctx, input.Config, path, body, &payload) @@ -380,6 +394,152 @@ func runAgentToolTask(ctx context.Context, input canvasGenerationInput) (map[str return parseAgentToolPayload(payload, protocol) } +func runDeclarativeAgentTask(ctx context.Context, input canvasGenerationInput, adapter protocol.AgentAdapter) (map[string]interface{}, error) { + if input.AgentRequests == nil { + return nil, errors.New("画布 Agent 工具请求缺少协议参数") + } + request := map[string]any{ + "chatCompletion": input.AgentRequests.ChatCompletion, + "responses": input.AgentRequests.Responses, + } + spec, err := adapter.BuildAgent(ctx, protocol.AgentRequestContext{BaseURL: input.Config.BaseURL, Model: input.Config.Model, Request: request}) + if err != nil { + return nil, err + } + body, err := executeProtocolRequest(ctx, input.Config, spec) + if err != nil { + return nil, err + } + parsed, err := adapter.ParseAgent(ctx, body) + if err != nil { + return nil, err + } + result := map[string]interface{}{"mode": "text", "text": parsed.Text, "toolCalls": []interface{}{}} + if parsed.Reasoning != "" { + result["reasoning"] = parsed.Reasoning + } + calls := make([]interface{}, 0, len(parsed.ToolCalls)) + for _, call := range parsed.ToolCalls { + calls = append(calls, map[string]interface{}{ + "id": call.ID, + "type": "function", + "function": map[string]interface{}{"name": call.Name, "arguments": call.Arguments}, + }) + } + result["toolCalls"] = calls + if strings.TrimSpace(parsed.Text) == "" && len(calls) == 0 { + return nil, errors.New("声明式 Agent 接口没有返回内容") + } + return result, nil +} + +func claudeAgentBody(request map[string]interface{}) map[string]interface{} { + body := map[string]interface{}{"max_tokens": 4096} + if messages, ok := request["messages"].([]interface{}); ok { + claudeMessages := make([]interface{}, 0, len(messages)) + var system []string + for _, value := range messages { + message, _ := value.(map[string]interface{}) + role := strings.ToLower(strings.TrimSpace(stringField(message, "role"))) + if role == "system" { + if content := strings.TrimSpace(fmt.Sprint(message["content"])); content != "" { + system = append(system, content) + } + continue + } + if role == "tool" { + claudeMessages = append(claudeMessages, map[string]interface{}{"role": "user", "content": []interface{}{map[string]interface{}{ + "type": "tool_result", "tool_use_id": stringField(message, "tool_call_id"), "content": fmt.Sprint(message["content"]), + }}}) + continue + } + role = mapClaudeMessageRole(role) + content := message["content"] + if content == nil { + content = "" + } + if toolCalls, ok := message["tool_calls"].([]interface{}); ok && len(toolCalls) > 0 { + blocks := make([]interface{}, 0, len(toolCalls)) + for _, value := range toolCalls { + toolCall, _ := value.(map[string]interface{}) + function, _ := toolCall["function"].(map[string]interface{}) + blocks = append(blocks, map[string]interface{}{ + "type": "tool_use", "id": stringField(toolCall, "id"), "name": stringField(function, "name"), "input": claudeToolInput(function["arguments"]), + }) + } + content = blocks + } + claudeMessages = append(claudeMessages, map[string]interface{}{"role": role, "content": content}) + } + body["messages"] = claudeMessages + if len(system) > 0 { + body["system"] = strings.Join(system, "\n\n") + } + } + if tools, ok := request["tools"].([]interface{}); ok && len(tools) > 0 { + claudeTools := make([]interface{}, 0, len(tools)) + for _, value := range tools { + tool, _ := value.(map[string]interface{}) + function, _ := tool["function"].(map[string]interface{}) + if len(function) == 0 { + continue + } + claudeTools = append(claudeTools, map[string]interface{}{ + "name": stringField(function, "name"), "description": stringField(function, "description"), "input_schema": function["parameters"], + }) + } + if len(claudeTools) > 0 { + body["tools"] = claudeTools + } + } + if choice, ok := request["tool_choice"]; ok { + body["tool_choice"] = claudeToolChoice(choice) + } + return body +} + +func mapClaudeMessageRole(role string) string { + if role == "assistant" { + return "assistant" + } + return "user" +} + +func claudeToolInput(value interface{}) interface{} { + if raw, ok := value.(string); ok { + var parsed interface{} + if json.Unmarshal([]byte(raw), &parsed) == nil && parsed != nil { + return parsed + } + } + if value != nil { + return value + } + return map[string]interface{}{} +} + +func claudeToolChoice(value interface{}) interface{} { + switch choice := value.(type) { + case string: + switch strings.ToLower(strings.TrimSpace(choice)) { + case "required": + return map[string]interface{}{"type": "any"} + case "none": + return map[string]interface{}{"type": "auto"} + default: + return map[string]interface{}{"type": "auto"} + } + case map[string]interface{}: + if function, ok := choice["function"].(map[string]interface{}); ok && stringField(function, "name") != "" { + return map[string]interface{}{"type": "tool", "name": stringField(function, "name")} + } + if name := stringField(choice, "name"); name != "" { + return map[string]interface{}{"type": "tool", "name": name} + } + } + return map[string]interface{}{"type": "auto"} +} + func isAgentToolChoiceCompatibilityError(err error) bool { if err == nil { return false @@ -418,6 +578,28 @@ func parseAgentToolPayload(payload map[string]interface{}, protocol string) (map result["toolCalls"] = calls return result, nil } + if protocol == "claude-api" { + content := interfaceSlice(payload["content"]) + calls := make([]interface{}, 0) + for _, value := range content { + item, _ := value.(map[string]interface{}) + switch stringField(item, "type") { + case "text": + result["text"] = result["text"].(string) + stringField(item, "text") + case "tool_use": + arguments, err := json.Marshal(item["input"]) + if err != nil { + return nil, err + } + calls = append(calls, map[string]interface{}{"id": stringField(item, "id"), "type": "function", "function": map[string]interface{}{"name": stringField(item, "name"), "arguments": string(arguments)}}) + } + } + result["toolCalls"] = calls + if result["text"] == "" && len(calls) == 0 { + return nil, errors.New("Claude Agent 接口没有返回内容") + } + return result, nil + } choices := interfaceSlice(payload["choices"]) if len(choices) == 0 { return nil, errors.New("画布 Agent 接口没有返回 choices") @@ -877,6 +1059,8 @@ func (s *Service) resolveProviderConfig(config providerConfig) (providerConfig, // 模型协议是实际请求契约；混合渠道中鉴权格式也必须随模型协议切换。 if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) || config.InterfaceType == string(model.ChannelInterfaceGeminiImage) { config.APIFormat = "gemini" + } else if config.InterfaceType == string(model.ChannelInterfaceClaudeAPI) { + config.APIFormat = "claude" } else if config.InterfaceType != "" { config.APIFormat = "openai" } @@ -900,21 +1084,37 @@ func providerChannelModelKey(config providerConfig) string { func systemChannelIDFromBaseURL(baseURL string) string { value := strings.TrimSpace(baseURL) - for _, marker := range []string{"/api/ai/system/", "api/ai/system/"} { - index := strings.Index(value, marker) + lowerValue := strings.ToLower(value) + for _, marker := range []string{"/api/ai/system/", "/api/"} { + index := strings.LastIndex(lowerValue, marker) if index < 0 { continue } id := strings.Trim(value[index+len(marker):], "/") - if slash := strings.Index(id, "/"); slash >= 0 { - id = id[:slash] + if queryIndex := strings.IndexAny(id, "?#"); queryIndex >= 0 { + id = id[:queryIndex] + } + if slash := strings.Index(id, "/"); slash >= 0 { + continue + } + id = strings.TrimSpace(id) + if id == "" { + continue + } + switch strings.ToLower(id) { + case "v1", "v1beta", "v2", "v3", "plan", "ai": + continue + default: + return id } - return strings.TrimSpace(id) } return "" } func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if _, ok := declarativeProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok { + return runDeclarativeProtocolTask(ctx, input) + } if input.Config.InterfaceType == string(model.ChannelInterfaceGrokImage) { return runGrokImageTask(ctx, input) } @@ -1390,11 +1590,16 @@ func normalizeVolcengineArkImageSize(value string) string { } func runTextTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if _, ok := declarativeProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok { + return runDeclarativeProtocolTask(ctx, input) + } switch input.Config.InterfaceType { case "chat-completion": return runChatCompletionsTextTask(ctx, input) case "openai-response": return runResponsesTextTask(ctx, input) + case string(model.ChannelInterfaceClaudeAPI): + return runClaudeTextTask(ctx, input) } return runLegacyTextTask(ctx, input) } @@ -1451,6 +1656,68 @@ func runChatCompletionsTextTask(ctx context.Context, input canvasGenerationInput return map[string]interface{}{"mode": "text", "text": text}, nil } +func runClaudeTextTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if len(input.ReferenceVideos) > 0 { + return nil, errors.New("Claude API 当前不支持视频参考输入") + } + messages := make([]map[string]interface{}, 0, len(input.TextHistory)+1) + for _, message := range validatedTextHistory(input.TextHistory) { + messages = append(messages, message) + } + content, err := claudeTextContent(input) + if err != nil { + return nil, err + } + messages = append(messages, map[string]interface{}{"role": "user", "content": content}) + body := map[string]interface{}{"model": input.Config.Model, "max_tokens": 4096, "messages": messages} + if systemPrompt := strings.TrimSpace(input.Config.SystemPrompt); systemPrompt != "" { + body["system"] = systemPrompt + } + text, err := requestTextProvider(ctx, input.Config, "/messages", body, "claude-api", input.StreamText) + if err != nil { + return nil, err + } + return map[string]interface{}{"mode": "text", "text": text}, nil +} + +func claudeTextContent(input canvasGenerationInput) (interface{}, error) { + if len(input.ReferenceImages) == 0 { + return input.Prompt, nil + } + content := []map[string]interface{}{{"type": "text", "text": input.Prompt}} + for _, image := range input.ReferenceImages { + value, err := openAIImageInputURL(image) + if err != nil { + return nil, err + } + if strings.HasPrefix(value, "data:") { + mimeType, data, ok := splitDataURL(value) + if !ok { + return nil, errors.New("Claude 参考图片 data URL 无效") + } + content = append(content, map[string]interface{}{"type": "image", "source": map[string]interface{}{"type": "base64", "media_type": mimeType, "data": data}}) + } else { + content = append(content, map[string]interface{}{"type": "image", "source": map[string]interface{}{"type": "url", "url": value}}) + } + } + return content, nil +} + +func splitDataURL(value string) (string, string, bool) { + if !strings.HasPrefix(value, "data:") { + return "", "", false + } + separator := strings.Index(value, ",") + if separator <= len("data:") { + return "", "", false + } + header := strings.TrimPrefix(value[:separator], "data:") + if !strings.HasSuffix(header, ";base64") { + return "", "", false + } + return strings.TrimSuffix(header, ";base64"), value[separator+1:], value[separator+1:] != "" +} + func textResponseInput(input canvasGenerationInput) (interface{}, error) { systemPrompt := strings.TrimSpace(input.Config.SystemPrompt) if len(input.TextHistory) == 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 { @@ -1573,6 +1840,9 @@ func shouldFallbackTextToChat(err error) bool { } func runAudioTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if _, ok := declarativeProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok { + return runDeclarativeProtocolTask(ctx, input) + } if resolved, ok := input.Metadata["resolvedCharacterVersions"].([]interface{}); ok && len(resolved) > 0 { voiceKey := metadataString(input.Metadata, "resolvedCharacterVoiceKey") if voiceKey == "" || strings.TrimSpace(input.Config.AudioVoice) != voiceKey { @@ -1607,6 +1877,215 @@ func runAudioTask(ctx context.Context, input canvasGenerationInput) (map[string] return map[string]interface{}{"mode": "audio", "audio": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType, "format": format}}, nil } +// runDeclarativeProtocolTask is the host runtime for JSON manifest plugins. +// Manifest code only describes request/response mapping; credentials, outbound +// policy, polling and result downloads remain owned by the host. +func runDeclarativeProtocolTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + adapter, ok := declarativeProtocolAdapterForContext(ctx, input.Config.InterfaceType) + if !ok { + return nil, fmt.Errorf("接口类型 %s 未安装声明式适配器", input.Config.InterfaceType) + } + request := protocolRequestFromInput(input) + taskID := resumedProviderRequestID(ctx) + var created protocol.CreateResult + if taskID == "" { + spec, err := adapter.BuildCreate(ctx, protocol.RequestContext{BaseURL: input.Config.BaseURL, Request: request}) + if err != nil { + return nil, err + } + body, err := executeProtocolRequest(ctx, input.Config, spec) + if err != nil { + return nil, err + } + created, err = adapter.ParseCreate(ctx, body) + if err != nil { + return nil, err + } + taskID = created.TaskID + if created.Status == protocol.StatusFailed || created.Status == protocol.StatusCancelled { + return nil, protocolResultError(created.Message, taskID) + } + if created.Status == protocol.StatusSucceeded { + return finishProtocolResult(ctx, input.Config, input.Mode, created.Result) + } + if taskID == "" { + return nil, errors.New("声明式协议创建请求没有返回任务 ID") + } + } + + for deadline := providerPollingDeadline(ctx); time.Now().Before(deadline); { + spec, err := adapter.BuildPoll(ctx, protocol.PollContext{BaseURL: input.Config.BaseURL, Model: request.Model, Request: request, TaskID: taskID}) + if err != nil { + return nil, err + } + body, err := executeProtocolRequest(ctx, input.Config, spec) + if err != nil { + return nil, err + } + state, err := adapter.ParsePoll(ctx, protocol.PollContext{BaseURL: input.Config.BaseURL, Model: request.Model, Request: request, TaskID: taskID}, body) + if err != nil { + return nil, err + } + if state.TaskID != "" { + taskID = state.TaskID + } + switch state.Status { + case protocol.StatusSucceeded: + return finishProtocolResult(ctx, input.Config, input.Mode, state.Result) + case protocol.StatusFailed, protocol.StatusCancelled: + return nil, protocolResultError(state.Message, taskID) + } + if err := sleepContext(ctx, 2500*time.Millisecond); err != nil { + return nil, err + } + } + return nil, fmt.Errorf("声明式协议任务超时（任务 %s）", taskID) +} + +func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRequest { + request := protocol.GenerationRequest{ + Model: input.Config.Model, + Prompt: input.Prompt, + Images: protocolMediaReferences(input.ReferenceImages, "image"), + Videos: protocolMediaReferences(input.ReferenceVideos, "video"), + Audios: protocolMediaReferences(input.ReferenceAudios, "audio"), + AspectRatio: input.Config.Size, + Resolution: input.Config.VQuality, + Quality: input.Config.Quality, + GenerateAudio: parseBool(input.Config.VideoGenerateAudio, false), + Watermark: parseBool(input.Config.VideoWatermark, false), + Operation: metadataString(input.Metadata, "videoOperation"), + Extra: map[string]any{ + "videoSeconds": input.Config.VideoSeconds, + "audioVoice": input.Config.AudioVoice, + "audioFormat": input.Config.AudioFormat, + "count": input.Config.Count, + }, + } + if duration, err := strconv.Atoi(strings.TrimSpace(input.Config.VideoSeconds)); err == nil && duration > 0 { + request.Duration = duration + } + if count, err := strconv.Atoi(strings.TrimSpace(input.Config.Count)); err == nil && count > 0 { + request.ImageCount = count + } + return request +} + +func protocolMediaReferences(values []providerMedia, kind string) []protocol.MediaReference { + result := make([]protocol.MediaReference, 0, len(values)) + for _, value := range values { + item := protocol.MediaReference{URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), Kind: kind} + if item.URL != "" || item.DataURL != "" { + result = append(result, item) + } + } + return result +} + +func executeProtocolRequest(ctx context.Context, config providerConfig, spec protocol.RequestSpec) ([]byte, error) { + if err := spec.Validate(); err != nil { + return nil, err + } + method := strings.ToUpper(strings.TrimSpace(spec.Method)) + var body io.Reader + if spec.Body != nil { + contentType := strings.ToLower(strings.TrimSpace(strings.Split(spec.ContentType, ";")[0])) + if contentType != "" && contentType != "application/json" { + return nil, fmt.Errorf("声明式协议暂不支持 %s 请求体", spec.ContentType) + } + data, err := json.Marshal(spec.Body) + if err != nil { + return nil, err + } + body = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, apiURL(config.BaseURL, spec.Path), body) + if err != nil { + return nil, err + } + applyProviderAuth(req, config) + if spec.Body != nil { + req.Header.Set("Content-Type", "application/json") + } + for name, value := range spec.Headers { + req.Header.Set(name, value) + } + ApplyOutboundHeaders(req, config.Headers) + data, _, err := doBinary(req) + return data, err +} + +func finishProtocolResult(ctx context.Context, config providerConfig, mode string, result *protocol.Result) (map[string]interface{}, error) { + if result == nil { + return nil, errors.New("声明式协议已完成但没有返回结果") + } + if mode == "text" { + output := map[string]interface{}{"mode": "text", "text": result.Text} + if strings.TrimSpace(result.Reasoning) != "" { + output["reasoning"] = result.Reasoning + } + return output, nil + } + var references []protocol.MediaReference + switch mode { + case "image": + references = result.Images + case "video": + references = result.Videos + case "audio": + references = result.Audios + default: + return nil, fmt.Errorf("声明式协议不支持生成模式 %s", mode) + } + if len(references) == 0 { + return nil, errors.New("声明式协议已完成但没有返回媒体地址") + } + items := make([]interface{}, 0, len(references)) + for _, reference := range references { + data, mimeType, err := protocolMediaBytes(ctx, config, reference) + if err != nil { + return nil, err + } + item := map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType} + items = append(items, item) + } + switch mode { + case "image": + return map[string]interface{}{"mode": "image", "images": items}, nil + case "video": + return map[string]interface{}{"mode": "video", "video": items[0]}, nil + default: + return map[string]interface{}{"mode": "audio", "audio": items[0]}, nil + } +} + +func protocolMediaBytes(ctx context.Context, config providerConfig, reference protocol.MediaReference) ([]byte, string, error) { + if strings.TrimSpace(reference.DataURL) != "" { + mimeType, data, err := decodeProviderDataURL(reference.DataURL) + return data, mimeType, err + } + value := strings.TrimSpace(reference.URL) + if value == "" { + return nil, "", errors.New("声明式协议媒体结果地址为空") + } + data, mimeType, err := getProviderExternalBinary(withProviderRequestKind(ctx, "download"), config, value) + if err != nil { + return nil, "", fmt.Errorf("声明式协议媒体结果下载失败：%w", err) + } + return data, normalizedMediaMimeType(mimeType, data), nil +} + +func protocolResultError(message, taskID string) error { + message = strings.TrimSpace(message) + if message == "" { + message = "上游返回失败状态" + } + if taskID == "" { + return errors.New(message) + } + return fmt.Errorf("声明式协议任务失败（任务 %s）：%s", taskID, message) +} + func runAsyncAudioTask(ctx context.Context, input canvasGenerationInput, body map[string]interface{}, format string) (map[string]interface{}, error) { id := resumedProviderRequestID(ctx) var state map[string]interface{} @@ -1808,6 +2287,9 @@ func audioFormatMimeType(format string) string { } func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if _, ok := declarativeProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok { + return runDeclarativeProtocolTask(ctx, input) + } if input.Config.InterfaceType == string(model.ChannelInterfaceMiniMaxVideo) { return runMiniMaxVideoTask(ctx, input) } @@ -2306,11 +2788,7 @@ func getGeminiBinary(ctx context.Context, config providerConfig, rawURL string) } func geminiVeoURL(baseURL string, path string) string { - base := strings.TrimRight(strings.TrimSpace(baseURL), "/") - if !strings.HasSuffix(strings.ToLower(base), "/v1beta") { - base += "/v1beta" - } - return base + "/" + strings.TrimLeft(path, "/") + return apiURLWithDefaultPrefix(baseURL, path, "/v1beta") } func findProviderMediaURL(value interface{}) string { @@ -2798,22 +3276,41 @@ func firstNonEmptyString(values ...string) string { } func validateGenerationInterface(mode string, interfaceType string) error { + return validateGenerationInterfaceWithRegistry(protocol.Builtins(), mode, interfaceType) +} + +func (s *Service) validateGenerationInterface(mode string, interfaceType string) error { + return validateGenerationInterfaceWithRegistry(s.protocolRegistry(), mode, interfaceType) +} + +func validateGenerationInterfaceWithRegistry(registry *protocol.Registry, mode string, interfaceType string) error { interfaceType = strings.TrimSpace(interfaceType) if interfaceType == "" { return nil } - allowed := map[string]map[string]bool{ - "text": {"chat-completion": true, "openai-response": true}, - "image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true, "gemini-image": true}, - "video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true, "novita-video": true, "minimax-video": true}, - "audio": {"openai-audio": true, "async-audio": true}, + adapter, ok := registry.Resolve(interfaceType) + if !ok { + return fmt.Errorf("接口类型 %s 未安装", interfaceType) } - if allowed[mode] != nil && !allowed[mode][interfaceType] { + metadata := adapter.Metadata() + if !metadata.Enabled || metadata.UnavailableReason != "" { + return fmt.Errorf("接口类型 %s 当前不可用：%s", interfaceType, metadata.UnavailableReason) + } + if mode != "" && !protocolCapabilityMatches(metadata, protocol.Capability(mode)) { return fmt.Errorf("接口类型 %s 不支持%s生成", interfaceType, mode) } return nil } +func protocolCapabilityMatches(metadata protocol.Metadata, capability protocol.Capability) bool { + for _, item := range metadata.Categories { + if item == capability { + return true + } + } + return false +} + func grokVideoBody(input canvasGenerationInput) (map[string]interface{}, error) { if input.Config.InterfaceType == "xai-video" { body, err := xaiVideoRequestBody(input) @@ -3076,6 +3573,17 @@ func postStreamingText(ctx context.Context, config providerConfig, path string, } func extractTextPayload(payload map[string]interface{}, protocol string) string { + if protocol == "claude-api" { + content, _ := payload["content"].([]interface{}) + var result strings.Builder + for _, item := range content { + record, _ := item.(map[string]interface{}) + if stringField(record, "type") == "text" { + result.WriteString(stringField(record, "text")) + } + } + return result.String() + } if protocol == "responses" { text := stringField(payload, "output_text") if text == "" { @@ -3131,6 +3639,11 @@ func parseTextEventStream(data []byte, protocol string) (string, error) { } if protocol == "responses" { text.WriteString(stringField(payload, "delta")) + } else if protocol == "claude-api" { + delta, _ := payload["delta"].(map[string]interface{}) + if stringField(delta, "type") == "text_delta" { + text.WriteString(stringField(delta, "text")) + } } else { choices, _ := payload["choices"].([]interface{}) for _, choice := range choices { @@ -3193,7 +3706,7 @@ func postStreamingBinary(ctx context.Context, config providerConfig, path string if err != nil { return nil, "", err } - req.Header.Set("Authorization", "Bearer "+config.APIKey) + applyProviderAuth(req, config) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "text/event-stream") ApplyOutboundHeaders(req, config.Headers) @@ -3206,12 +3719,25 @@ func postJSON(ctx context.Context, config providerConfig, path string, body inte if err != nil { return err } - req.Header.Set("Authorization", "Bearer "+config.APIKey) + applyProviderAuth(req, config) req.Header.Set("Content-Type", "application/json") ApplyOutboundHeaders(req, config.Headers) return doJSON(req, target) } +func applyProviderAuth(req *http.Request, config providerConfig) { + if config.APIFormat == "claude" { + req.Header.Set("x-api-key", config.APIKey) + req.Header.Set("anthropic-version", "2023-06-01") + return + } + if config.APIFormat == "gemini" { + req.Header.Set("x-goog-api-key", config.APIKey) + return + } + req.Header.Set("Authorization", "Bearer "+config.APIKey) +} + func postForm(ctx context.Context, config providerConfig, path string, contentType string, body io.Reader, target interface{}) error { req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL(config.BaseURL, path), body) if err != nil { @@ -3269,7 +3795,7 @@ func getProviderExternalBinary(ctx context.Context, config providerConfig, rawUR return nil, "", err } if sameProviderOrigin(config.BaseURL, rawURL) { - req.Header.Set("Authorization", "Bearer "+config.APIKey) + applyProviderAuth(req, config) ApplyOutboundHeaders(req, config.Headers) } return doBinary(req) @@ -3548,17 +4074,73 @@ func providerRequestKind(method string, path string) string { } func apiURL(baseURL string, path string) string { + return apiURLWithDefaultPrefix(baseURL, path, "/v1") +} + +// ChannelAPIURL is the single URL join point for provider requests. A channel +// may be configured as host, host/, host/v1 or host/v1/; callers should pass +// protocol paths without hard-coding a version prefix whenever possible. +func ChannelAPIURL(baseURL string, path string) string { + return apiURL(baseURL, path) +} + +// ChannelAPIURLForProtocol keeps protocol-specific defaults at the transport +// boundary. In particular, Gemini uses v1beta while OpenAI-compatible APIs +// conventionally use v1. An explicit version in either input wins. +func ChannelAPIURLForProtocol(baseURL string, path string, interfaceType model.ChannelInterfaceType) string { + defaultPrefix := "/v1" + if interfaceType == model.ChannelInterfaceGeminiVeo || interfaceType == model.ChannelInterfaceGeminiImage { + defaultPrefix = "/v1beta" + } + return apiURLWithDefaultPrefix(baseURL, path, defaultPrefix) +} + +var channelAPIPrefixes = []string{"/api/plan/v3", "/api/v3", "/v1beta", "/v1", "/v2", "/v3"} + +func apiURLWithDefaultPrefix(baseURL string, path string, defaultPrefix string) string { base := strings.TrimRight(strings.TrimSpace(baseURL), "/") - if strings.HasPrefix(path, "/v2/") { - if strings.HasSuffix(base, "/v2") { - return base + strings.TrimPrefix(path, "/v2") + requestPath := strings.TrimSpace(path) + if requestPath == "" { + return base + } + if !strings.HasPrefix(requestPath, "/") { + requestPath = "/" + requestPath + } + + requestPrefix := requestAPIPathPrefix(requestPath) + basePrefix := baseAPIPathPrefix(base) + if requestPrefix != "" { + if basePrefix == requestPrefix { + return base + strings.TrimPrefix(requestPath, requestPrefix) } - return base + path + // An explicit request version takes precedence over a version accidentally + // left on the configured base URL (for example base=/v1, path=/v2/...). + return strings.TrimSuffix(base, basePrefix) + requestPath } - if strings.HasSuffix(base, "/v1") || strings.HasSuffix(base, "/v1beta") || strings.HasSuffix(base, "/v2") || strings.HasSuffix(base, "/api/v3") || strings.HasSuffix(base, "/api/plan/v3") { - return base + path + if basePrefix != "" { + return base + requestPath } - return base + "/v1" + path + return base + defaultPrefix + requestPath +} + +func requestAPIPathPrefix(value string) string { + lower := strings.ToLower(value) + for _, prefix := range channelAPIPrefixes { + if lower == prefix || strings.HasPrefix(lower, prefix+"/") || strings.HasPrefix(lower, prefix+"?") || strings.HasPrefix(lower, prefix+"#") { + return prefix + } + } + return "" +} + +func baseAPIPathPrefix(value string) string { + lower := strings.ToLower(strings.TrimRight(value, "/")) + for _, prefix := range channelAPIPrefixes { + if lower == prefix || strings.HasSuffix(lower, prefix) { + return prefix + } + } + return "" } func writeField(writer *multipart.Writer, key string, value string) { diff --git a/backend/internal/service/provider_claude_test.go b/backend/internal/service/provider_claude_test.go new file mode 100644 index 000000000..143af0afe --- /dev/null +++ b/backend/internal/service/provider_claude_test.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "infinite-canvas/backend/internal/model" +) + +func TestRunClaudeTextTaskUsesMessagesAndAPIKeyHeader(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/messages" { + t.Fatalf("path = %q", r.URL.Path) + } + if r.Header.Get("x-api-key") != "claude-key" || r.Header.Get("Authorization") != "" { + t.Fatalf("auth headers = x-api-key %q authorization %q", r.Header.Get("x-api-key"), r.Header.Get("Authorization")) + } + if r.Header.Get("anthropic-version") != "2023-06-01" { + t.Fatalf("anthropic-version = %q", r.Header.Get("anthropic-version")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"content":[{"type":"text","text":"claude works"}]}`)) + })) + defer server.Close() + + config := providerConfig{ + BaseURL: server.URL + "/v1", APIKey: "claude-key", Model: "claude-test", APIFormat: "claude", InterfaceType: string(model.ChannelInterfaceClaudeAPI), + AllowLocalChannel: true, + } + result, err := runTextTask(withProviderOutboundPolicy(context.Background(), config), canvasGenerationInput{Mode: "text", Prompt: "hello", Config: config}) + if err != nil { + t.Fatal(err) + } + if result["text"] != "claude works" { + t.Fatalf("result = %#v", result) + } +} diff --git a/backend/internal/service/provider_resource_helpers.go b/backend/internal/service/provider_resource_helpers.go new file mode 100644 index 000000000..b3b0ab955 --- /dev/null +++ b/backend/internal/service/provider_resource_helpers.go @@ -0,0 +1,22 @@ +package service + +import ( + "errors" + "time" + + "infinite-canvas/backend/internal/model" +) + +// ResourceForUser is the service-layer ownership check used by provider workers. +func (s *Service) ResourceForUser(actor *model.User, id string) (*model.Resource, error) { + if actor == nil || actor.ID == "" { + return nil, errors.New("用户身份无效") + } + return s.repo.ResourceForUser(actor.ID, id) +} + +// providerResourceURL gives an upstream a short-lived URL after the resource +// ownership and ready-state checks have already been performed. +func (s *Service) providerResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) { + return s.directResourceURL(resource, expiresAt) +} diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index 69dcf7250..bd1b7b58c 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -42,6 +42,50 @@ func TestProviderRequestErrorDetails(t *testing.T) { } } +func TestChannelAPIURLNormalizesConfiguredVersionPrefix(t *testing.T) { + tests := []struct { + name string + base string + path string + want string + }{ + {name: "host", base: "http://provider.test:8000", path: "/chat/completions", want: "http://provider.test:8000/v1/chat/completions"}, + {name: "host slash", base: "http://provider.test:8000/", path: "/chat/completions", want: "http://provider.test:8000/v1/chat/completions"}, + {name: "v1", base: "http://provider.test:8000/v1", path: "/chat/completions", want: "http://provider.test:8000/v1/chat/completions"}, + {name: "v1 slash", base: "http://provider.test:8000/v1/", path: "/chat/completions", want: "http://provider.test:8000/v1/chat/completions"}, + {name: "path carries v1beta", base: "http://provider.test:8000", path: "/v1beta/models/model", want: "http://provider.test:8000/v1beta/models/model"}, + {name: "same v1beta is not duplicated", base: "http://provider.test:8000/v1beta", path: "/v1beta/models/model", want: "http://provider.test:8000/v1beta/models/model"}, + {name: "path carries v2", base: "http://provider.test:8000/v1", path: "/v2/tasks", want: "http://provider.test:8000/v2/tasks"}, + {name: "ark v3", base: "https://ark.example.com/api/v3", path: "/images/generations", want: "https://ark.example.com/api/v3/images/generations"}, + {name: "path carries ark v3", base: "https://ark.example.com", path: "/api/v3/images/generations", want: "https://ark.example.com/api/v3/images/generations"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ChannelAPIURL(tt.base, tt.path); got != tt.want { + t.Fatalf("ChannelAPIURL(%q, %q) = %q, want %q", tt.base, tt.path, got, tt.want) + } + }) + } +} + +func TestChannelAPIURLForProtocolUsesGeminiDefault(t *testing.T) { + if got := ChannelAPIURLForProtocol("https://generativelanguage.googleapis.com", "/models/gemini:generateContent", model.ChannelInterfaceGeminiVeo); got != "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent" { + t.Fatalf("Gemini URL = %q", got) + } +} + +func TestSystemChannelIDFromBaseURLSupportsShortAndLegacyProxyPaths(t *testing.T) { + for _, test := range []struct{ base, want string }{ + {base: "/api/channel-1", want: "channel-1"}, + {base: "/api/ai/system/channel-2", want: "channel-2"}, + {base: "https://canvas.example.com/api/channel-3", want: "channel-3"}, + } { + if got := systemChannelIDFromBaseURL(test.base); got != test.want { + t.Fatalf("systemChannelIDFromBaseURL(%q) = %q, want %q", test.base, got, test.want) + } + } +} + func TestWriteMediaPartSanitizesFilenameAndSetsMimeType(t *testing.T) { var body bytes.Buffer writer := multipart.NewWriter(&body) @@ -105,6 +149,17 @@ data: [DONE] if got, err := parseTextEventStream(chat, "chat-completion"); err != nil || got != "第一镜：远景" { t.Fatalf("Chat stream = %q, err = %v", got, err) } + + claude := []byte(`event: content_block_delta +data: {"delta":{"type":"text_delta","text":"第一镜"}} + +event: content_block_delta +data: {"delta":{"type":"text_delta","text":"：远景"}} + +`) + if got, err := parseTextEventStream(claude, "claude-api"); err != nil || got != "第一镜：远景" { + t.Fatalf("Claude stream = %q, err = %v", got, err) + } } func TestParseAgentToolPayloadSupportsChatCompletions(t *testing.T) { @@ -164,6 +219,59 @@ func TestParseAgentToolPayloadSupportsResponses(t *testing.T) { } } +func TestParseAgentToolPayloadSupportsClaude(t *testing.T) { + result, err := parseAgentToolPayload(map[string]interface{}{ + "content": []interface{}{ + map[string]interface{}{"type": "text", "text": "开始操作"}, + map[string]interface{}{"type": "tool_use", "id": "call-3", "name": "canvas_get_state", "input": map[string]interface{}{}}, + }, + }, "claude-api") + if err != nil { + t.Fatalf("parseAgentToolPayload() error = %v", err) + } + if result["text"] != "开始操作" { + t.Fatalf("text = %v", result["text"]) + } + calls, _ := result["toolCalls"].([]interface{}) + if len(calls) != 1 { + t.Fatalf("toolCalls = %#v", result["toolCalls"]) + } + call, _ := calls[0].(map[string]interface{}) + function, _ := call["function"].(map[string]interface{}) + if call["id"] != "call-3" || function["name"] != "canvas_get_state" || function["arguments"] != "{}" { + t.Fatalf("tool call = %#v", call) + } +} + +func TestClaudeAgentBodyMapsOpenAIStyleTools(t *testing.T) { + body := claudeAgentBody(map[string]interface{}{ + "messages": []interface{}{ + map[string]interface{}{"role": "system", "content": "You are concise."}, + map[string]interface{}{"role": "user", "content": "读取画布"}, + map[string]interface{}{"role": "assistant", "content": nil, "tool_calls": []interface{}{map[string]interface{}{ + "id": "call-4", "function": map[string]interface{}{"name": "canvas_get_state", "arguments": `{}`}, + }}}, + map[string]interface{}{"role": "tool", "tool_call_id": "call-4", "content": `{"nodes":[]}`}, + }, + "tools": []interface{}{map[string]interface{}{"type": "function", "function": map[string]interface{}{ + "name": "canvas_get_state", "description": "读取画布", "parameters": map[string]interface{}{"type": "object"}, + }}}, + "tool_choice": "required", + }) + if body["system"] != "You are concise." || body["max_tokens"] != 4096 { + t.Fatalf("body = %#v", body) + } + messages, _ := body["messages"].([]interface{}) + if len(messages) != 3 { + t.Fatalf("messages = %#v", messages) + } + tools, _ := body["tools"].([]interface{}) + tool, _ := tools[0].(map[string]interface{}) + if tool["name"] != "canvas_get_state" || body["tool_choice"].(map[string]interface{})["type"] != "any" { + t.Fatalf("tools/choice = %#v / %#v", body["tools"], body["tool_choice"]) + } +} + func TestRunAgentToolTaskFallsBackToolChoice(t *testing.T) { t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") var choices []interface{} diff --git a/backend/internal/service/service.go b/backend/internal/service/service.go index 0e5055e0a..455495065 100644 --- a/backend/internal/service/service.go +++ b/backend/internal/service/service.go @@ -37,6 +37,8 @@ type Service struct { sessionCreationCoordinator *sessionCreationCoordinator sessionUploadCoordinator *sessionUploadCoordinator runtimeErr error + pluginRuntime *pluginRuntime + pluginRuntimeErr error workerID string routeCatalogMu sync.RWMutex routeCatalogRefreshMu sync.Mutex @@ -95,7 +97,8 @@ func New(repo *repository.Repository, dataDir string) *Service { func NewWithRuntimeCapabilities(repo *repository.Repository, dataDir string, capabilities RuntimeCapabilities) *Service { coordinator, err := newRuntimeCoordinator(repo.Dialect()) - service := &Service{repo: repo, dataDir: dataDir, runtimeCapabilities: capabilities, activeCancels: make(map[string]context.CancelFunc), coordinator: coordinator, runtimeErr: err, workerID: newID(), routeCatalogTTL: 30 * time.Second, routeCatalogMaxStale: 5 * time.Minute, routeHealthBlocked: make(map[string]time.Time)} + pluginRuntime, pluginRuntimeErr := newPluginRuntime(dataDir) + service := &Service{repo: repo, dataDir: dataDir, runtimeCapabilities: capabilities, activeCancels: make(map[string]context.CancelFunc), coordinator: coordinator, runtimeErr: err, pluginRuntime: pluginRuntime, pluginRuntimeErr: pluginRuntimeErr, workerID: newID(), routeCatalogTTL: 30 * time.Second, routeCatalogMaxStale: 5 * time.Minute, routeHealthBlocked: make(map[string]time.Time)} service.taskBillingCoordinator = newTaskBillingCoordinator(service.repo) service.taskTerminalCoordinator = newTaskTerminalCoordinator(service) service.taskRouteExecutor = newTaskRouteExecutor(service) diff --git a/backend/internal/service/storyboard_workflow.go b/backend/internal/service/storyboard_workflow.go index abc39a75a..8c624e087 100644 --- a/backend/internal/service/storyboard_workflow.go +++ b/backend/internal/service/storyboard_workflow.go @@ -112,6 +112,7 @@ func parseStoryboardTaskInput(task model.Task, label string) (agentStoryboardInp } func (s *Service) generateStoryboardPlan(ctx context.Context, task model.Task, input agentStoryboardInput, shotDuration int, shotCount int) (agentStoryboardPlan, []storyboardAsset, error) { + ctx = withProtocolRegistry(ctx, s.protocolRegistry()) if !providerConfigReady(input.Config) { return agentStoryboardPlan{}, nil, errors.New("请先配置可用的文本模型") } diff --git a/canvas-agent/README.md b/canvas-agent/README.md index 6689b41dc..70feff44a 100644 --- a/canvas-agent/README.md +++ b/canvas-agent/README.md @@ -91,6 +91,10 @@ default_tools_approval_mode = "approve" 可用工具： - `canvas_get_state` +- `canvas_get_context` +- `canvas_find_nodes` +- `canvas_get_resources` +- `canvas_validate_ops` - `canvas_get_selection` - `canvas_export_snapshot` - `canvas_apply_ops` @@ -113,6 +117,8 @@ default_tools_approval_mode = "approve" } ``` +推荐的 Agent 工作流是：先调用 `canvas_get_context` 读取语义化上下文和 `stateHash`；不知道节点 id 时调用 `canvas_find_nodes`；涉及图片、视频或音频参考时调用 `canvas_get_resources`；复杂写操作先调用 `canvas_validate_ops`，通过后再调用 `canvas_apply_ops`。这样 Agent 不需要猜测节点 id，也不会把 loading/error/占位媒体误判成可用资源。 + ## 侧边栏 Codex 本地面板会把提示词发送给 Canvas Agent。Canvas Agent 使用官方 `@openai/codex` CLI 的 `codex app-server --stdio` 启动并复用同一个 Codex thread，启动时会注入 `yingce` MCP 配置并自动放行 MCP 审批，真正执行画布修改前仍由网页侧边栏二次确认。 diff --git a/canvas-agent/src/agents.ts b/canvas-agent/src/agents.ts index 977c9eb8f..41f19e5c8 100644 --- a/canvas-agent/src/agents.ts +++ b/canvas-agent/src/agents.ts @@ -419,6 +419,10 @@ function stringOrNull(value: unknown) { function toolName(name: string) { if (name === "canvas_apply_ops") return "画布操作"; if (name === "canvas_get_state") return "读取画布"; + if (name === "canvas_get_context") return "读取画布上下文"; + if (name === "canvas_find_nodes") return "检索画布节点"; + if (name === "canvas_get_resources") return "读取画布资源"; + if (name === "canvas_validate_ops") return "校验画布操作"; if (name === "canvas_get_selection") return "读取选区"; if (name === "canvas_export_snapshot") return "导出快照"; if (name === "canvas_create_text_node") return "创建文本"; diff --git a/canvas-agent/src/canvas-context.ts b/canvas-agent/src/canvas-context.ts new file mode 100644 index 000000000..db4ccb262 --- /dev/null +++ b/canvas-agent/src/canvas-context.ts @@ -0,0 +1,273 @@ +import crypto from "node:crypto"; + +import type { CanvasConnection, CanvasNode, CanvasSnapshot } from "./types.js"; + +export type CanvasResourceSummary = { + nodeId: string; + nodeTitle: string; + nodeType: CanvasNode["type"]; + status?: string; + resourceId?: string; + storageKey?: string; + assetId?: string; + assetCategory?: string; + mimeType?: string; + bytes?: number; + width?: number; + height?: number; + durationMs?: number; + isReady: boolean; +}; + +export type CanvasContext = { + schemaVersion: 1; + stateHash: string; + canvas: { + projectId?: string; + domainProjectId?: string; + title?: string; + viewport?: CanvasSnapshot["viewport"]; + nodeCount: number; + connectionCount: number; + selectedNodeCount: number; + nodeTypeCounts: Record; + }; + selection: ReturnType[]; + nodes: ReturnType[]; + connections: Array<{ + id: string; + fromNodeId: string; + fromTitle: string; + toNodeId: string; + toTitle: string; + fromHandleId?: string; + toHandleId?: string; + }>; + resources: CanvasResourceSummary[]; + warnings: string[]; +}; + +export function buildCanvasContext(state: CanvasSnapshot | null): CanvasContext { + if (!state) throw new Error("当前没有已连接画布"); + const nodes = state.nodes || []; + const nodeById = new Map(nodes.map((node) => [node.id, node])); + const selectedIds = new Set(state.selectedNodeIds || []); + const nodeTypeCounts = nodes.reduce>((counts, node) => { + counts[node.type] = (counts[node.type] || 0) + 1; + return counts; + }, {}); + const resources = nodes.flatMap((node) => { + const resource = resourceSummary(node); + return resource ? [resource] : []; + }); + const warnings: string[] = []; + if (!nodes.length) warnings.push("画布为空；创建前先确认用户要放置的区域或使用默认网格布局。"); + if (nodes.some((node) => node.metadata?.status === "error")) warnings.push("画布中存在生成失败节点；重试前应读取节点的 errorDetails 或 generationErrorCode。"); + if (resources.some((resource) => !resource.isReady)) warnings.push("存在尚未就绪或缺少持久化资源引用的媒体节点；不要把占位节点当作可用参考素材。"); + + return { + schemaVersion: 1, + stateHash: hashState(state), + canvas: { + projectId: state.projectId, + domainProjectId: state.domainProjectId, + title: state.title, + viewport: state.viewport, + nodeCount: nodes.length, + connectionCount: (state.connections || []).length, + selectedNodeCount: selectedIds.size, + nodeTypeCounts, + }, + selection: nodes.filter((node) => selectedIds.has(node.id)).map(compactContextNode), + nodes: nodes.map(compactContextNode), + connections: (state.connections || []).map((connection) => connectionSummary(connection, nodeById)), + resources, + warnings, + }; +} + +export function findCanvasNodes(state: CanvasSnapshot | null, input: { query?: string; ids?: string[]; types?: string[]; statuses?: string[]; resourceOnly?: boolean; limit?: number }) { + if (!state) throw new Error("当前没有已连接画布"); + const query = input.query?.trim().toLocaleLowerCase(); + const ids = input.ids?.length ? new Set(input.ids) : null; + const types = input.types?.length ? new Set(input.types) : null; + const statuses = input.statuses?.length ? new Set(input.statuses) : null; + const limit = Math.min(Math.max(input.limit || 50, 1), 200); + const nodes = (state.nodes || []).filter((node) => { + const metadata = node.metadata || {}; + if (ids && !ids.has(node.id)) return false; + if (types && !types.has(node.type)) return false; + if (statuses && !statuses.has(String(metadata.status || "idle"))) return false; + if (input.resourceOnly && !resourceSummary(node)) return false; + if (!query) return true; + return [node.id, node.title, metadata.content, metadata.prompt, metadata.composerContent, metadata.assetId, Array.isArray(metadata.assetTags) ? metadata.assetTags.join(" ") : "", metadata.workflowKind, metadata.workflowTitle, metadata.characterName] + .some((value) => String(value || "").toLocaleLowerCase().includes(query)); + }); + return { query: input.query || "", total: nodes.length, truncated: nodes.length > limit, nodes: nodes.slice(0, limit).map(compactContextNode) }; +} + +export function getCanvasResources(state: CanvasSnapshot | null, input: { nodeIds?: string[]; status?: string; limit?: number }) { + if (!state) throw new Error("当前没有已连接画布"); + const nodeIds = input.nodeIds?.length ? new Set(input.nodeIds) : null; + const limit = Math.min(Math.max(input.limit || 100, 1), 300); + const resources = (state.nodes || []).flatMap((node) => { + if (nodeIds && !nodeIds.has(node.id)) return []; + const resource = resourceSummary(node); + if (!resource || (input.status && resource.status !== input.status)) return []; + return [resource]; + }); + return { total: resources.length, truncated: resources.length > limit, resources: resources.slice(0, limit) }; +} + +export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[]) { + if (!state) throw new Error("当前没有已连接画布"); + const nodes = state.nodes || []; + const nodeIds = new Set(nodes.map((node) => node.id)); + const issues: Array<{ index: number; severity: "error" | "warning"; message: string }> = []; + const addedIds = new Set(); + const requireNode = (index: number, id: unknown, label: string) => { + if (typeof id !== "string" || !id) issues.push({ index, severity: "error", message: `${label} 缺少节点 id` }); + else if (!nodeIds.has(id) && !addedIds.has(id)) issues.push({ index, severity: "error", message: `${label}「${id}」不在当前画布状态中；先重新读取 canvas_get_context 或 canvas_find_nodes` }); + }; + ops.forEach((raw, index) => { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + issues.push({ index, severity: "error", message: "操作必须是对象" }); + return; + } + const op = raw as Record; + switch (op.type) { + case "add_node": + if (typeof op.id === "string") { + if (nodeIds.has(op.id) || addedIds.has(op.id)) issues.push({ index, severity: "error", message: `新增节点 id「${op.id}」重复` }); + addedIds.add(op.id); + } + break; + case "update_node": + case "run_generation": + requireNode(index, op.nodeId ?? op.id, op.type === "run_generation" ? "生成目标" : "更新目标"); + break; + case "delete_node": + if (Array.isArray(op.ids)) op.ids.forEach((id) => requireNode(index, id, "删除目标")); + else requireNode(index, op.id, "删除目标"); + break; + case "connect_nodes": + requireNode(index, op.fromNodeId, "连接起点"); + requireNode(index, op.toNodeId, "连接终点"); + if (op.fromNodeId === op.toNodeId) issues.push({ index, severity: "error", message: "不能连接节点自身" }); + break; + case "select_nodes": + if (Array.isArray(op.ids)) op.ids.forEach((id) => requireNode(index, id, "选区节点")); + break; + case "delete_connections": + if (!op.all && !op.id && !(Array.isArray(op.ids) && op.ids.length)) issues.push({ index, severity: "error", message: "删除连线必须提供 id、ids 或 all=true" }); + break; + case "set_viewport": + if (!op.viewport || typeof op.viewport !== "object") issues.push({ index, severity: "error", message: "视口参数无效" }); + break; + default: + issues.push({ index, severity: "error", message: `不支持的操作类型「${String(op.type)}」` }); + } + }); + const errors = issues.filter((item) => item.severity === "error"); + return { ok: errors.length === 0, issues, operationCount: ops.length, currentStateHash: hashState(state) }; +} + +export function hashState(state: CanvasSnapshot) { + return crypto.createHash("sha256").update(stableStringify({ + projectId: state.projectId, + domainProjectId: state.domainProjectId, + title: state.title, + nodes: state.nodes || [], + connections: state.connections || [], + selectedNodeIds: state.selectedNodeIds || [], + viewport: state.viewport, + })).digest("hex").slice(0, 16); +} + +function compactContextNode(node: CanvasNode) { + const metadata = node.metadata || {}; + const resource = resourceSummary(node); + return { + id: node.id, + type: node.type, + title: node.title || "未命名节点", + position: node.position, + size: { width: node.width, height: node.height }, + parentId: node.parentId, + status: String(metadata.status || "idle"), + content: preview(metadata.content, 240), + prompt: preview(metadata.prompt || metadata.composerContent, 300), + generation: metadata.generationMode || metadata.workflowKind ? { + mode: metadata.generationMode, + model: metadata.model, + workflowKind: metadata.workflowKind, + workflowTitle: metadata.workflowTitle, + } : undefined, + asset: metadata.assetId || metadata.characterAssetId ? { + assetId: metadata.assetId || metadata.characterAssetId, + versionId: metadata.characterVersionId, + category: metadata.assetCategory, + tags: metadata.assetTags, + characterName: metadata.characterName, + } : undefined, + resource: resource ? { resourceId: resource.resourceId, storageKey: resource.storageKey, mimeType: resource.mimeType, bytes: resource.bytes, width: resource.width, height: resource.height, durationMs: resource.durationMs, ready: resource.isReady } : undefined, + }; +} + +function resourceSummary(node: CanvasNode): CanvasResourceSummary | null { + const metadata = node.metadata || {}; + const storageKey = stringValue(metadata.storageKey); + const resourceId = storageKey?.startsWith("resource:") ? storageKey.slice("resource:".length) : undefined; + const hasResourceSignal = Boolean(storageKey || metadata.resourceId || metadata.assetId || metadata.primaryImageId || metadata.mimeType || ["image", "video", "audio"].includes(node.type)); + if (!hasResourceSignal) return null; + const status = stringValue(metadata.status) || "idle"; + return { + nodeId: node.id, + nodeTitle: node.title || "未命名节点", + nodeType: node.type, + status, + resourceId: resourceId || stringValue(metadata.resourceId), + storageKey, + assetId: stringValue(metadata.assetId || metadata.characterAssetId), + assetCategory: stringValue(metadata.assetCategory), + mimeType: stringValue(metadata.mimeType), + bytes: numberValue(metadata.bytes), + width: numberValue(metadata.naturalWidth), + height: numberValue(metadata.naturalHeight), + durationMs: numberValue(metadata.durationMs), + isReady: status === "success" && Boolean(storageKey || metadata.resourceId || metadata.primaryImageId), + }; +} + +function connectionSummary(connection: CanvasConnection, nodeById: Map) { + return { + id: connection.id, + fromNodeId: connection.fromNodeId, + fromTitle: nodeById.get(connection.fromNodeId)?.title || "未知节点", + toNodeId: connection.toNodeId, + toTitle: nodeById.get(connection.toNodeId)?.title || "未知节点", + fromHandleId: connection.fromHandleId, + toHandleId: connection.toHandleId, + }; +} + +function preview(value: unknown, limit: number) { + if (typeof value !== "string") return undefined; + const text = value.trim(); + if (!text) return undefined; + return text.length > limit ? `${text.slice(0, limit)}…` : text; +} + +function stringValue(value: unknown) { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function numberValue(value: unknown) { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function stableStringify(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + if (value && typeof value === "object") return `{${Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`).join(",")}}`; + return JSON.stringify(value); +} diff --git a/canvas-agent/src/canvas-session.ts b/canvas-agent/src/canvas-session.ts index 216e1b8b5..76c94e035 100644 --- a/canvas-agent/src/canvas-session.ts +++ b/canvas-agent/src/canvas-session.ts @@ -2,6 +2,7 @@ import crypto from "node:crypto"; import type { ServerResponse } from "node:http"; import { CANVAS_GENERATION_CONTINUATION_TIMEOUT_MS } from "./canvas-tool-timeouts.js"; +import { buildCanvasContext, findCanvasNodes, getCanvasResources, validateCanvasOps } from "./canvas-context.js"; import { type ToolName } from "./schemas.js"; import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js"; import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; @@ -88,9 +89,13 @@ export class CanvasSession { if (!input.projectId) throw new Error("当前画布没有关联短剧项目"); return await this.requestCanvasTool(tool, input); } - const readTool = ["canvas_get_state", "canvas_get_selection", "canvas_export_snapshot"].includes(tool); + const readTool = ["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"].includes(tool); if (readTool && (!this.clients.size || !this.canvasState)) throw new Error("当前没有已连接画布"); if (tool === "canvas_get_state" || tool === "canvas_export_snapshot") return compactCanvasState(this.canvasState); + if (tool === "canvas_get_context") return buildCanvasContext(this.canvasState); + if (tool === "canvas_find_nodes") return findCanvasNodes(this.canvasState, input as Parameters[1]); + if (tool === "canvas_get_resources") return getCanvasResources(this.canvasState, input as Parameters[1]); + if (tool === "canvas_validate_ops") return validateCanvasOps(this.canvasState, (input as { ops: unknown[] }).ops); if (tool === "canvas_get_selection") { const ids = new Set(this.canvasState?.selectedNodeIds || []); return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) }; @@ -176,6 +181,12 @@ export class CanvasSession { } if (tool !== "canvas_apply_ops") throw new Error(`未知工具：${tool}`); if (!this.clients.size) throw new Error("当前没有已连接画布"); + const expectedStateHash = typeof input.expectedStateHash === "string" ? input.expectedStateHash : ""; + if (expectedStateHash && expectedStateHash !== buildCanvasContext(this.canvasState).stateHash) { + throw new Error("画布状态已变化，请重新读取 canvas_get_context 后再执行写操作"); + } + const validation = validateCanvasOps(this.canvasState, (input as { ops: unknown[] }).ops); + if (!validation.ok) throw new Error(`画布操作校验失败：${validation.issues.filter((item) => item.severity === "error").map((item) => item.message).join("；")}`); return await this.requestCanvasTool(tool, input); } diff --git a/canvas-agent/src/config.ts b/canvas-agent/src/config.ts index 05632e710..da3db00e6 100644 --- a/canvas-agent/src/config.ts +++ b/canvas-agent/src/config.ts @@ -11,7 +11,28 @@ export const DEFAULT_PORT = LOCAL_RUNTIME_DEFAULT_PORT; export const CONFIG_DIR = startupConfigDirectory(); export const CONFIG_FILE = path.join(CONFIG_DIR, "canvas-agent.json"); export const VERSION = readPackageVersion(); -export const AGENT_PROMPT = "你正在帮助用户操作影策网页画布。需要改动画布时优先使用已配置的 yingce MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。即使用户明确点名 Dreamina/即梦，本机 Canvas Agent 也必须通过 canvas_generate_image 或 canvas_generate_video 进入共享 GenerationTask；本机图片模型使用 model=local:dreamina-cli:5.0 这类产品模型值，用户选择自动分辨率时使用 quality=auto；禁止调用 direct dreamina_cli provider tool。不要模拟鼠标点击，不要要求用户手动复制 JSON。"; +export const AGENT_PROMPT = `你是影策的画布执行 Agent，不是只会生成 JSON 的聊天机器人。你的第一责任是基于真实画布状态完成可验证的结果。 + +【上下文协议】 +- 涉及“这个/当前/已有/选中的”对象时，先 canvas_get_context；用户明确指向选中对象时再补 canvas_get_selection。 +- 不要从记忆或用户描述猜节点 id。需要找节点时用 canvas_find_nodes；需要判断媒体能否作为参考时用 canvas_get_resources。 +- canvas_get_context 返回 stateHash、语义化节点、连接关系和资源就绪状态。资源 ready=false、status=loading/error 或只有占位 metadata 时，必须明确说明，不要把它当成可用素材。 + +【执行协议】 +- 任何写操作前先读取上下文；复杂批量写操作先调用 canvas_validate_ops，再调用 canvas_apply_ops。 +- 写操作只使用当前上下文中真实存在的 id；新增节点要避免重叠，优先沿现有内容的右侧或下方网格布局。 +- 操作完成后检查工具返回的真实结果；如果没有改变、部分失败或生成仍在进行，必须如实报告，不要说“已完成”。 +- 删除、覆盖、批量移动、触发生成属于高影响操作，先给出简短计划并等待网页侧确认；不要用模拟鼠标点击绕过确认。 +- 优先使用语义化工具（canvas_create_text_node、canvas_generate_*、canvas_update_node_text 等），只有确实需要批量事务时才使用 canvas_apply_ops。 + +【资源与生成】 +- 生成前先检查已有提示词、参考节点、资产引用和就绪状态；有合适资源就复用真实 node id，不要重复上传或创建孤立副本。 +- 图片、视频、音频生成必须通过 canvas_generate_image、canvas_generate_video、canvas_generate_audio 进入共享 GenerationTask；禁止调用 direct dreamina_cli provider tool。 +- 即使用户点名 Dreamina/即梦，也使用 model=local:dreamina-cli:5.0 这类产品模型值；自动分辨率使用 quality=auto。 + +【交互边界】 +- 不要求用户手动复制 JSON、URL、token 或节点 id；不编造工具结果；不把媒体 URL、API key 或 data URL 放进回复。 +- 页面文案和画布节点内容默认使用中文。`; export type CanvasWorkspaceConfig = { workspacePath: string; activeThreadId?: string; pinnedThreadIds?: string[] }; export type LocalRuntimeConfig = { diff --git a/canvas-agent/src/schemas.ts b/canvas-agent/src/schemas.ts index a7e162f1f..c239fd296 100644 --- a/canvas-agent/src/schemas.ts +++ b/canvas-agent/src/schemas.ts @@ -11,6 +11,10 @@ const projectShotSchema = z.object({ id: z.string().optional(), unitId: z.string export const toolNames = [ "canvas_get_state", + "canvas_get_context", + "canvas_find_nodes", + "canvas_get_resources", + "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot", "canvas_apply_ops", @@ -90,9 +94,13 @@ const generationFlowSchema = z.object({ export const toolInputSchemas = { canvas_get_state: z.object({}).passthrough(), + canvas_get_context: z.object({}).passthrough(), + canvas_find_nodes: z.object({ query: z.string().optional(), ids: z.array(z.string()).optional(), types: z.array(z.string()).optional(), statuses: z.array(z.string()).optional(), resourceOnly: z.boolean().optional(), limit: z.number().int().min(1).max(200).optional() }), + canvas_get_resources: z.object({ nodeIds: z.array(z.string()).optional(), status: z.string().optional(), limit: z.number().int().min(1).max(300).optional() }), + canvas_validate_ops: z.object({ ops: z.array(canvasOpSchema) }), canvas_get_selection: z.object({}).passthrough(), canvas_export_snapshot: z.object({}).passthrough(), - canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema) }), + canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema), expectedStateHash: z.string().min(1).optional() }), canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }), canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }), canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }), @@ -125,6 +133,10 @@ export const toolInputSchemas = { export const toolDescriptions: Record = { canvas_get_state: "读取当前网页画布的节点、连线、选区和视口。", + canvas_get_context: "读取面向 Agent 的语义化画布上下文：节点用途、连接关系、选区、资源引用、生成状态和状态哈希。优先于直接猜测节点 metadata。", + canvas_find_nodes: "按标题、内容、提示词、类型、状态、资产或工作流检索真实节点，返回可用于后续写操作的节点 id。", + canvas_get_resources: "读取当前画布引用的媒体资源清单，包括资源/资产引用、类型、尺寸、大小、时长和就绪状态；不会返回媒体 URL。", + canvas_validate_ops: "在真正写入前校验批量操作中的节点 id、连接关系和参数，避免对不存在节点或错误资源执行操作。", canvas_get_selection: "读取当前网页画布选中的节点。", canvas_export_snapshot: "导出当前画布快照，用于理解布局。", canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。", diff --git a/canvas-agent/src/tools.ts b/canvas-agent/src/tools.ts index 1f5539df9..346b5f4e1 100644 --- a/canvas-agent/src/tools.ts +++ b/canvas-agent/src/tools.ts @@ -16,8 +16,11 @@ export function compactCanvasState(state: CanvasSnapshot | null) { export function compactNode(node: CanvasNode) { const metadata = { ...(node.metadata || {}) }; - if (typeof metadata.content === "string" && metadata.content.length > 240) metadata.content = `${metadata.content.slice(0, 120)}...`; - return { id: node.id, type: node.type, title: node.title, position: node.position, width: node.width, height: node.height, metadata }; + for (const key of ["url", "dataUrl", "previewUrl", "coverUrl"]) delete metadata[key]; + for (const key of ["content", "prompt", "composerContent", "errorDetails"]) { + if (typeof metadata[key] === "string" && metadata[key].length > 360) metadata[key] = `${metadata[key].slice(0, 340)}...`; + } + return { id: node.id, type: node.type, title: node.title, position: node.position, width: node.width, height: node.height, parentId: node.parentId, metadata }; } export function nextCanvasX(state: CanvasSnapshot | null) { diff --git a/canvas-agent/test/canvas-context.test.ts b/canvas-agent/test/canvas-context.test.ts new file mode 100644 index 000000000..cf72ea557 --- /dev/null +++ b/canvas-agent/test/canvas-context.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { buildCanvasContext, findCanvasNodes, getCanvasResources, validateCanvasOps } from "../src/canvas-context.js"; +import type { CanvasSnapshot } from "../src/types.js"; + +const state: CanvasSnapshot = { + projectId: "canvas-1", + title: "分镜画布", + viewport: { x: 0, y: 0, k: 1 }, + selectedNodeIds: ["ref-1"], + nodes: [ + { id: "prompt-1", type: "text", title: "提示词", position: { x: 0, y: 0 }, width: 340, height: 240, metadata: { content: "夜雨中的城市", status: "success" } }, + { id: "ref-1", type: "image", title: "角色参考", position: { x: 420, y: 0 }, width: 320, height: 320, metadata: { status: "success", storageKey: "resource:abc", mimeType: "image/png", assetId: "asset-1", assetCategory: "character", naturalWidth: 1024, naturalHeight: 1024 } }, + { id: "loading-1", type: "video", title: "待生成视频", position: { x: 840, y: 0 }, width: 480, height: 270, metadata: { status: "loading" } }, + ], + connections: [{ id: "c-1", fromNodeId: "prompt-1", toNodeId: "ref-1" }], +}; + +test("builds semantic canvas context without media URLs", () => { + const context = buildCanvasContext(state); + assert.equal(context.canvas.nodeCount, 3); + assert.equal(context.selection[0]?.id, "ref-1"); + assert.equal(context.resources[0]?.resourceId, "abc"); + assert.equal(context.resources[0]?.isReady, true); + assert.equal(context.resources[1]?.isReady, false); + assert.equal((context.nodes[1] as { resource?: { ready?: boolean } }).resource?.ready, true); + assert.equal("url" in context.nodes[1], false); + assert.match(context.stateHash, /^[a-f0-9]{16}$/); +}); + +test("finds real nodes and resources by semantic query", () => { + assert.equal(findCanvasNodes(state, { query: "角色", resourceOnly: true }).nodes[0]?.id, "ref-1"); + assert.equal(getCanvasResources(state, { status: "loading" }).resources[0]?.nodeId, "loading-1"); +}); + +test("rejects stale or unsafe operations before dispatch", () => { + const result = validateCanvasOps(state, [ + { type: "update_node", id: "missing" }, + { type: "connect_nodes", fromNodeId: "prompt-1", toNodeId: "prompt-1" }, + ]); + assert.equal(result.ok, false); + assert.equal(result.issues.length, 2); +}); diff --git a/plugins/yingce/README.md b/plugins/yingce/README.md index 4349f924b..2d1ff3242 100644 --- a/plugins/yingce/README.md +++ b/plugins/yingce/README.md @@ -87,3 +87,28 @@ CANVAS_URL=http://localhost:3000 npx -y @ddcat666/open-ai-canvas-agent ``` 手动排查时先从 Agent 输出或 `http://127.0.0.1:17371/config` 读取本地地址和 token，然后直接打开 `<画布网页地址>/canvas?mode=new&agentUrl=&agentToken=`。不要通过页面点击来新建画布；`mode=new` 会让网页自动创建具体画布并连接本地 Agent。 + +## 技能库 + +插件的 `skills/` 是可按需加载的 Codex 技能库，不需要把整套规则复制到每次对话里。安装插件后，建议新建一个 Codex 线程；Codex 会根据请求自动发现并加载对应技能： + +- `canvas-context`：先读语义化画布上下文、选区、连接和资源就绪状态。 +- `canvas-editing`：写入前校验真实节点 id，批量操作后复核结果。 +- `asset-aware-generation`：复用已有角色、场景、道具、风格和媒体资源创建生成流程。 + +安装/更新流程： + +```bash +cd /path/to/open-ai-canvas +codex plugin marketplace add "$(pwd)" +codex plugin add yingce@yingce-local +# 更新插件后开启一个新的 Codex 对话 +``` + +验证技能和 MCP 是否已加载，可以在新对话中直接说： + +```text +读取当前画布上下文，列出可用媒体资源，并说明哪些资源可以作为生图参考。 +``` + +如果回答没有调用 `canvas_get_context` / `canvas_get_resources`，先确认当前对话是安装插件后新建的，并检查 `codex mcp list` 中的 `yingce`。 diff --git a/plugins/yingce/skills/README.md b/plugins/yingce/skills/README.md new file mode 100644 index 000000000..93a374554 --- /dev/null +++ b/plugins/yingce/skills/README.md @@ -0,0 +1,13 @@ +# 影策技能库 + +插件安装后，Codex 会在新线程中自动发现 `skills/` 下的技能。技能不是一组需要用户手动粘贴的 prompt，而是按请求匹配、按需加载的工作流约束。 + +当前技能： + +- `open-canvas`：打开并连接本地画布。 +- `canvas`：通用画布操作路由。 +- `canvas-context`：读取语义上下文和资源状态。 +- `canvas-editing`：可靠写入、批量校验和结果复核。 +- `asset-aware-generation`：复用角色/场景/道具/风格资源进行生成。 + +安装或更新技能后，建议新建 Codex 线程，让技能和 MCP 工具重新加载。 diff --git a/plugins/yingce/skills/asset-aware-generation/SKILL.md b/plugins/yingce/skills/asset-aware-generation/SKILL.md new file mode 100644 index 000000000..8bfd5ca54 --- /dev/null +++ b/plugins/yingce/skills/asset-aware-generation/SKILL.md @@ -0,0 +1,15 @@ +--- +name: asset-aware-generation +description: 基于影策画布已有角色、场景、道具、风格和媒体资源创建生成流程；适用于生图、视频、音频或分镜资产工作。 +--- + +# 资源感知生成 + +1. 用 `canvas_get_context` 读取已有提示词、工作流和资产引用。 +2. 用 `canvas_get_resources` 或 `canvas_find_nodes({resourceOnly:true})` 找到可复用的真实参考节点。 +3. 生成前检查资源 `ready`、`status`、`mimeType`、尺寸/时长和资产/版本引用；未就绪资源不能作为参考。 +4. 复用真实 `referenceNodeIds`，不要重复上传同一素材，也不要只在 prompt 里写一个无法定位的资源名称。 +5. 通过 `canvas_create_generation_flow` 或对应 `canvas_generate_*` 工具创建清晰的提示词节点、参考节点和目标节点关系。 +6. 返回结果后确认目标节点、连线和任务状态；生成任务未完成时不要声称成片已完成。 + +生成任务必须走影策共享 GenerationTask；不要直接调用 provider、模拟点击或把密钥放进 URL。 diff --git a/plugins/yingce/skills/canvas-context/SKILL.md b/plugins/yingce/skills/canvas-context/SKILL.md new file mode 100644 index 000000000..a9041b1a6 --- /dev/null +++ b/plugins/yingce/skills/canvas-context/SKILL.md @@ -0,0 +1,16 @@ +--- +name: canvas-context +description: 理解影策当前画布的语义结构、选区、连接关系和媒体资源状态；适用于任何需要基于已有画布继续工作的请求。 +--- + +# 画布上下文协议 + +不要把画布当成一段需要猜测的 JSON。先读取事实，再决定动作： + +1. `canvas_get_context`：读取语义化节点、真实 id、连接关系、选区、资源清单和 `stateHash`。 +2. 用户说“这个/选中内容”时，补 `canvas_get_selection`。 +3. 不知道节点 id 时用 `canvas_find_nodes`，不要猜 id。 +4. 涉及图片、视频、音频参考时用 `canvas_get_resources`；只有 `ready=true` 且有持久化引用的资源才可作为可用素材。 +5. 看到 `loading`、`error`、缺少 `storageKey/resourceId` 的节点时，向用户说明它是未就绪或占位状态。 + +上下文只作为事实来源，不要把 `storageKey`、内部 id 或资源状态编造成媒体 URL。工具结果返回后，以结果为准继续下一步。 diff --git a/plugins/yingce/skills/canvas-editing/SKILL.md b/plugins/yingce/skills/canvas-editing/SKILL.md new file mode 100644 index 000000000..d5b9d3d61 --- /dev/null +++ b/plugins/yingce/skills/canvas-editing/SKILL.md @@ -0,0 +1,14 @@ +--- +name: canvas-editing +description: 在影策画布上可靠地创建、更新、移动、连接、删除节点或调整视口；适用于需要实际修改画布的请求。 +--- + +# 可靠画布编辑 + +- 写操作前必须先读取 `canvas_get_context`；已有节点只使用真实 id。 +- 复杂批量修改先用 `canvas_validate_ops`，通过后再用 `canvas_apply_ops`。 +- 单个文本优先 `canvas_create_text_node`；生成流程优先 `canvas_generate_text/image/video/audio`；只有批量事务才直接组装 ops。 +- 新节点沿现有内容右侧或下方网格布局，保留间距，避免遮挡和重叠。 +- 删除、覆盖、批量移动和触发生成属于高影响动作，依赖网页侧确认，不要绕过确认。 +- 执行后检查返回结果。没有变化、部分失败、任务仍在运行时必须如实说明。 +- 媒体节点尽量保留原始比例；不要为了“看起来完成”随意改尺寸或创建孤立占位节点。 diff --git a/plugins/yingce/skills/canvas/SKILL.md b/plugins/yingce/skills/canvas/SKILL.md index 7c8d427b5..eb79f43f6 100644 --- a/plugins/yingce/skills/canvas/SKILL.md +++ b/plugins/yingce/skills/canvas/SKILL.md @@ -10,7 +10,9 @@ description: 操作影策当前网页画布，读取节点、选区、创建文 ## 工作流 - 如果用户还没有打开或连接网页画布，使用 `open-canvas` 技能打开影策，不要要求用户手动复制 URL 或 token。 -- 操作前先用 `canvas_get_state` 读取当前画布；如果用户明确提到选中内容、当前节点或“这个”，先用 `canvas_get_selection`。 +- 操作前先用 `canvas_get_context` 读取语义化画布和资源状态；如果用户明确提到选中内容、当前节点或“这个”，再用 `canvas_get_selection`。 +- 不知道真实节点 id 时使用 `canvas_find_nodes`；涉及媒体参考时使用 `canvas_get_resources`。 +- 复杂批量操作先用 `canvas_validate_ops`，再用 `canvas_apply_ops`。 - 创建单个文本内容优先用 `canvas_create_text_node`。 - 创建生成内容优先用 `canvas_generate_text`、`canvas_generate_image`、`canvas_generate_video`、`canvas_generate_audio`。 - 需要把提示词、参考素材和生成目标节点串成流程时，使用 `canvas_create_generation_flow` 或项目已有的流程工具。 @@ -18,6 +20,8 @@ description: 操作影策当前网页画布，读取节点、选区、创建文 - 不要模拟鼠标点击，不要要求用户手动复制 JSON。 - 写入画布的操作会由网页侧边栏做二次确认，按当前工具结果继续推进即可。 +更具体的资源感知生成和可靠编辑流程分别见 `asset-aware-generation`、`canvas-editing`。 + ## 风格 - 页面文案和画布节点内容默认使用中文。 diff --git a/web/bun.lock b/web/bun.lock index e3f9f5b08..d134851cc 100644 --- a/web/bun.lock +++ b/web/bun.lock @@ -50,6 +50,7 @@ "react-markdown": "^10.1.0", "react-router": "^8.2.0", "recharts": "^3.9.2", + "remark-gfm": "^4.0.1", "shadcn": "^4.13.1", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", diff --git a/web/package.json b/web/package.json index 612961150..b19618375 100644 --- a/web/package.json +++ b/web/package.json @@ -58,6 +58,7 @@ "react-markdown": "^10.1.0", "react-router": "^8.2.0", "recharts": "^3.9.2", + "remark-gfm": "^4.0.1", "shadcn": "^4.13.1", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", diff --git a/web/src/components/auth/require-feature.tsx b/web/src/components/auth/require-feature.tsx index a973ad9cc..cfb150475 100644 --- a/web/src/components/auth/require-feature.tsx +++ b/web/src/components/auth/require-feature.tsx @@ -7,21 +7,24 @@ import { WorkspaceErrorState, WorkspaceLoadingState, WorkspaceState } from "@/co import { refreshFeatureAvailability } from "@/lib/user-session"; import { useUserStore } from "@/stores/use-user-store"; -type FeatureKey = "shortDramaEnabled" | "taskCenterEnabled" | "creditsEnabled" | "frontendModelsEnabled"; +type FeatureKey = "shortDramaEnabled" | "taskCenterEnabled" | "creditsEnabled" | "frontendModelsEnabled" | "pluginCenterEnabled"; const featureNames: Record = { shortDramaEnabled: "短剧创作", taskCenterEnabled: "任务中心", creditsEnabled: "积分中心", frontendModelsEnabled: "前台模型", + pluginCenterEnabled: "插件中心", }; let featureAvailabilityCheckedOnce = false; export function RequireFeature({ feature, children }: { feature: FeatureKey; children: ReactNode }) { const navigate = useNavigate(); + const user = useUserStore((state) => state.user); const features = useUserStore((state) => state.features); - const [checking, setChecking] = useState(() => !useUserStore.getState().features[feature]); + const adminBypass = feature === "pluginCenterEnabled" && user?.role === "admin"; + const [checking, setChecking] = useState(() => !adminBypass && !useUserStore.getState().features[feature]); const [error, setError] = useState(""); useEffect(() => { @@ -43,9 +46,9 @@ export function RequireFeature({ feature, children }: { feature: FeatureKey; chi if (checking) return ; if (error) return navigate("/create", { replace: true })} />; - if (!features[feature]) { + if (!adminBypass && !features[feature]) { // 管理员页面返回到管理后台首页，用户页面返回到创作台 - const isAdminFeature = feature === "frontendModelsEnabled"; + const isAdminFeature = feature === "frontendModelsEnabled" || (feature === "pluginCenterEnabled" && user?.role === "admin"); const backPath = isAdminFeature ? "/admin" : "/create"; const backLabel = isAdminFeature ? "返回管理后台" : "返回创作台"; diff --git a/web/src/components/canvas/canvas-assistant-panel.tsx b/web/src/components/canvas/canvas-assistant-panel.tsx index cc1734227..2a0e1cf5b 100644 --- a/web/src/components/canvas/canvas-assistant-panel.tsx +++ b/web/src/components/canvas/canvas-assistant-panel.tsx @@ -8,7 +8,7 @@ import { modelDisplayName, modelIcon, normalizeModelOptionValue, resolveModelCha import { canvasThemes } from "@/lib/canvas-theme"; import { nanoid } from "nanoid"; import { requestToolResponse, type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; -import { logicalModelIDForConfig, runBackendToolGenerationTask } from "@/services/api/generation-task"; +import { backendModelRuntimeRequired, runBackendToolGenerationTask } from "@/services/api/generation-task"; import { imageToDataUrl } from "@/services/image-storage"; import { isCanvasGenerationDurableAckError, persistCanvasCinematicSessionContinuationEffect } from "@/services/canvas-generation-consumer"; import { consumeGenerationTaskAgent } from "@/services/project-asset-sync"; @@ -30,6 +30,7 @@ import { NODE_DEFAULT_SIZE } from "@/constant/canvas"; import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantPendingBackendSession, type CanvasAssistantReference, type CanvasAssistantSession, type CanvasNodeData } from "@/types/canvas"; import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store"; import { previewCanvasAgentOps, summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentOperationImpact, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; +import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context"; import { canvasAgentPromptCacheKey } from "@/lib/openai-prompt-cache"; import { resolveStoryboardGenerationContext } from "@/lib/canvas/canvas-storyboard-context"; @@ -37,7 +38,7 @@ export const CANVAS_AGENT_PANEL_MOTION_MS = 500; const PANEL_MOTION_SECONDS = CANVAS_AGENT_PANEL_MOTION_MS / 1000; const ONLINE_AGENT_MAX_STEPS = 4; const ONLINE_AGENT_PROMPT = - "你是影策网页内置在线画布助手。当前画布 JSON 会随用户消息提供。首轮必须调用工具：只读问题调用 canvas_get_state，需要改动画布时调用和本地 Agent 一致的 infinite-canvas 工具。需要生成内容时直接调用 canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio 或 canvas_create_generation_flow；需要精确批量操作时调用 canvas_apply_ops。不要输出 JSON ops，不要编造执行结果。工具参数涉及已有节点时必须使用当前画布 JSON 中真实存在的 id；缺少必要 id 或用户意图不明确时直接说明需要用户明确选择或说明，不要猜测。工具返回结果后，再根据真实结果回答用户。"; + "你是影策网页内置在线画布助手。首轮必须先调用 canvas_get_context；涉及已有节点时用 canvas_find_nodes 获取真实 id，涉及媒体参考时用 canvas_get_resources。复杂写操作先 canvas_validate_ops，再执行 canvas_apply_ops。不要输出 JSON ops、不要猜 id、不要把未就绪资源当作可用素材、不要编造执行结果；工具返回真实结果后再回答。"; const JSON_RECORD_SCHEMA = { type: "object", additionalProperties: true }; const POSITION_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false }; const VIEWPORT_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, k: { type: "number" } }, required: ["x", "y", "k"], additionalProperties: false }; @@ -85,7 +86,7 @@ const CANVAS_OP_SCHEMA = { required: ["type"], additionalProperties: false, }; -const ONLINE_READ_TOOLS = new Set(["canvas_get_state", "canvas_get_selection", "canvas_export_snapshot"]); +const ONLINE_READ_TOOLS = new Set(["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"]); function toolDefinition(name: string, description: string, properties: Record, required: string[] = [], strict = false): ResponseFunctionTool { return { type: "function", function: { name, description, parameters: { type: "object", properties, required, additionalProperties: false }, strict } }; @@ -111,12 +112,16 @@ function generationToolDefinition(name: string, description: string, mode?: "tex const ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [ toolDefinition("canvas_get_state", "读取当前网页画布的节点、连线、选区和视口。", {}), + toolDefinition("canvas_get_context", "读取语义化画布上下文、真实节点 id、连接关系、资源就绪状态和状态哈希。", {}), + toolDefinition("canvas_find_nodes", "按标题、内容、提示词、类型、状态或资产检索真实节点。", { query: { type: "string" }, ids: { type: "array", items: { type: "string" } }, types: { type: "array", items: { type: "string" } }, statuses: { type: "array", items: { type: "string" } }, resourceOnly: { type: "boolean" }, limit: { type: "number" } }), + toolDefinition("canvas_get_resources", "读取画布媒体资源引用、类型、尺寸、大小、时长和就绪状态，不返回媒体 URL。", { nodeIds: { type: "array", items: { type: "string" } }, status: { type: "string" }, limit: { type: "number" } }), + toolDefinition("canvas_validate_ops", "在写入前校验节点 id、连接关系和批量操作参数。", { ops: { type: "array", items: CANVAS_OP_SCHEMA } }, ["ops"]), toolDefinition("canvas_get_selection", "读取当前网页画布选中的节点。", {}), toolDefinition("canvas_export_snapshot", "导出当前画布快照，用于理解布局。", {}), toolDefinition( "canvas_apply_ops", - "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。", - { ops: { type: "array", items: CANVAS_OP_SCHEMA } }, + "批量操作当前网页画布。复杂写操作应先 canvas_validate_ops；可传 canvas_get_context 返回的 expectedStateHash 防止基于过期状态写入。", + { ops: { type: "array", items: CANVAS_OP_SCHEMA }, expectedStateHash: { type: "string" } }, ["ops"], false, ), @@ -708,7 +713,16 @@ export function CanvasAssistantPanel({ const executeOnlineTool = async (sessionId: string, name: string, args: Record, messageId?: string): Promise => { const current = snapshotRef.current; try { + const expectedStateHash = typeof args.expectedStateHash === "string" ? args.expectedStateHash : ""; + if (expectedStateHash && expectedStateHash !== buildCanvasAgentContext(current).stateHash) return { ok: false, message: "画布状态已变化，请重新读取 canvas_get_context 后再执行写操作。" }; if (name === "canvas_get_state") return { ok: true, message: describeCanvasSnapshot(current), data: compactSnapshot(current) }; + if (name === "canvas_get_context") return { ok: true, message: "已读取语义化画布上下文。", data: buildCanvasAgentContext(current) }; + if (name === "canvas_find_nodes") return { ok: true, message: "已按条件检索真实节点。", data: findCanvasAgentNodes(current, args as Parameters[1]) }; + if (name === "canvas_get_resources") return { ok: true, message: "已读取画布资源清单。", data: getCanvasAgentResources(current, args as Parameters[1]) }; + if (name === "canvas_validate_ops") { + const result = validateCanvasAgentOps(current, requireOps(args.ops)); + return { ok: result.ok, message: result.ok ? "操作校验通过。" : "操作校验失败。", data: result }; + } if (name === "canvas_export_snapshot") return { ok: true, message: describeCanvasSnapshot(current), data: compactSnapshot(current) }; if (name === "canvas_get_selection") { const ids = new Set(current.selectedNodeIds || []); @@ -1579,7 +1593,7 @@ function toolCallToResponseInput(call: ResponseToolCall): ResponseInputMessage { } async function requestOnlineAgentModel(config: AiConfig, messages: ResponseInputMessage[], toolChoice: "auto" | "required", prompt: string, onDelta: (text: string) => void, promptCacheKey?: string) { - if (logicalModelIDForConfig(config)) { + if (backendModelRuntimeRequired(config)) { const result = await runBackendToolGenerationTask({ prompt, config, messages, tools: ONLINE_AGENT_TOOLS, toolChoice }); if (result.content.trim()) onDelta(result.content); return result; @@ -1876,6 +1890,7 @@ function compactMetadata(metadata: CanvasNodeData["metadata"]) { function backendAgentProviderConfig(config: ReturnType) { return { + channelId: config.channelId, apiFormat: config.apiFormat, interfaceType: config.interfaceType, baseUrl: config.baseUrl, diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index e5e255ce8..b8c07bd0b 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -23,6 +23,7 @@ import { type AgentThreadSummary, } from "@/stores/canvas/use-canvas-agent-store"; import { previewCanvasAgentOps, summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; +import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context"; import { buildCanvasResourceReferences } from "@/lib/canvas/canvas-resource-references"; import { listAddedSkills, type Skill } from "@/services/api/skills"; import { isProjectAgentReadTool, isProjectAgentToolName, runProjectAgentTool } from "@/services/api/project-agent-tools"; @@ -383,9 +384,17 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({ const result = payload.name === "canvas_apply_ops" ? await onApplyOpsRef.current((input.ops || []) as CanvasAgentOp[], { source: "local", conversationId: activeThreadId || clientIdRef.current, messageId: payload.requestId }) - : projectToolName - ? await runProjectAgentTool(projectToolName, input, snapshotRef.current.domainProjectId) - : snapshotRef.current; + : payload.name === "canvas_get_context" + ? buildCanvasAgentContext(snapshotRef.current) + : payload.name === "canvas_find_nodes" + ? findCanvasAgentNodes(snapshotRef.current, input as Parameters[1]) + : payload.name === "canvas_get_resources" + ? getCanvasAgentResources(snapshotRef.current, input as Parameters[1]) + : payload.name === "canvas_validate_ops" + ? validateCanvasAgentOps(snapshotRef.current, (input.ops || []) as CanvasAgentOp[]) + : projectToolName + ? await runProjectAgentTool(projectToolName, input, snapshotRef.current.domainProjectId) + : snapshotRef.current; await postToolResult(clientIdRef.current, { requestId: payload.requestId, result }); if (payload.name === "canvas_apply_ops") syncState(clientIdRef.current, result as CanvasAgentSnapshot); setAgentState({ activity: "工具完成", waiting: true }); diff --git a/web/src/components/layout/workspace-sidebar-nav.tsx b/web/src/components/layout/workspace-sidebar-nav.tsx index 09638ae13..63f6627c4 100644 --- a/web/src/components/layout/workspace-sidebar-nav.tsx +++ b/web/src/components/layout/workspace-sidebar-nav.tsx @@ -56,7 +56,7 @@ function buildNav(features: FeatureAvailability, balance: string, isAdmin: boole heading: "工作台管理", items: [ toolItem("skills", "/skills"), - toolItem("plugins", "/plugins"), + ...(features.pluginCenterEnabled || isAdmin ? [toolItem("plugins", "/plugins")] : []), ...(features.creditsEnabled ? [{ ...toolItem("wallet", "/wallet"), badge: balance }] : []), ], }, diff --git a/web/src/components/model-protocol-picker.tsx b/web/src/components/model-protocol-picker.tsx index 22b63bdd1..170341dc9 100644 --- a/web/src/components/model-protocol-picker.tsx +++ b/web/src/components/model-protocol-picker.tsx @@ -3,7 +3,7 @@ import { Button, Modal } from "antd"; import { useEffect, useState, type ReactNode } from "react"; import { ModelIcon } from "@/components/model-picker"; -import { MODEL_PROTOCOLS, modelProtocolLabel, type ModelProtocol, type ModelProtocolDefinition, type ProtocolCapability } from "@/lib/model-protocols"; +import { modelProtocolLabel, type ModelProtocol, type ModelProtocolDefinition, type ProtocolCapability } from "@/lib/model-protocols"; import { cn } from "@/lib/utils"; export type ModelCapabilityChoice = ProtocolCapability; @@ -57,11 +57,11 @@ export function CapabilityCardPicker({ value, onChange, density = "comfortable" ); } -export function ProtocolCardPicker({ capability, value, onChange, density = "comfortable" }: { capability?: ModelCapabilityChoice; value?: ModelProtocol; onChange?: (value: ModelProtocol) => void; density?: PickerDensity }) { - const protocols = MODEL_PROTOCOLS.filter((item) => item.capability === capability); +export function ProtocolCardPicker({ capability, value, onChange, density = "comfortable", protocols = [] }: { capability?: ModelCapabilityChoice; value?: ModelProtocol; onChange?: (value: ModelProtocol) => void; density?: PickerDensity; protocols?: ModelProtocolDefinition[] }) { + const availableProtocols = protocols.filter((item) => item.capability === capability && item.enabled !== false); return (
- {protocols.map((protocol) => { + {availableProtocols.map((protocol) => { const selected = value === protocol.value; return (
请求协议
- setDraft((current) => ({ ...current, protocol }))} /> + setDraft((current) => ({ ...current, protocol }))} />
@@ -163,26 +163,26 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap function ProtocolBrandMark({ protocol, compact = false }: { protocol: ModelProtocolDefinition; compact?: boolean }) { const iconSize = compact ? "size-6" : "size-8"; - if (protocol.value === "chat-completion") return ; - if (protocol.value.startsWith("volcengine-jimeng-")) + const vendor = `${protocol.vendor || ""} ${protocol.label}`.toLowerCase(); + if (vendor.includes("jimeng") || vendor.includes("即梦")) return ( ); - if (protocol.value.startsWith("volcengine-")) + if (vendor.includes("volcengine") || vendor.includes("火山方舟")) return ( ); - if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video") + if (vendor.includes("newapi") || vendor.includes("novita")) return ( ); - const brand = protocol.value === "gemini-veo" || protocol.value === "gemini-image" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai"; + const brand = vendor.includes("gemini") || vendor.includes("google") ? "gemini" : vendor.includes("grok") || vendor.includes("xai") ? "grok" : vendor.includes("openai") ? "openai" : "openai"; return ; } diff --git a/web/src/lib/canvas/canvas-agent-context.ts b/web/src/lib/canvas/canvas-agent-context.ts new file mode 100644 index 000000000..4f9d6d8ad --- /dev/null +++ b/web/src/lib/canvas/canvas-agent-context.ts @@ -0,0 +1,141 @@ +import type { CanvasAgentOp, CanvasAgentSnapshot } from "./canvas-agent-ops"; +import type { CanvasNodeData } from "@/types/canvas"; + +export type CanvasAgentResource = { + nodeId: string; + nodeTitle: string; + nodeType: CanvasNodeData["type"]; + status: string; + resourceId?: string; + storageKey?: string; + assetId?: string; + assetCategory?: string; + mimeType?: string; + bytes?: number; + width?: number; + height?: number; + durationMs?: number; + ready: boolean; +}; + +export function buildCanvasAgentContext(snapshot: CanvasAgentSnapshot) { + const nodes = snapshot.nodes || []; + const selectedIds = new Set(snapshot.selectedNodeIds || []); + const nodeById = new Map(nodes.map((node) => [node.id, node])); + const resources = nodes.flatMap((node) => { + const resource = resourceFromNode(node); + return resource ? [resource] : []; + }); + const nodeTypeCounts = nodes.reduce>((counts, node) => { + counts[node.type] = (counts[node.type] || 0) + 1; + return counts; + }, {}); + return { + schemaVersion: 1, + stateHash: hashSnapshot(snapshot), + canvas: { projectId: snapshot.projectId, domainProjectId: snapshot.domainProjectId, title: snapshot.title, viewport: snapshot.viewport, nodeCount: nodes.length, connectionCount: snapshot.connections.length, selectedNodeCount: selectedIds.size, nodeTypeCounts }, + selection: nodes.filter((node) => selectedIds.has(node.id)).map(compactNode), + nodes: nodes.map(compactNode), + connections: snapshot.connections.map((connection) => ({ id: connection.id, fromNodeId: connection.fromNodeId, fromTitle: nodeById.get(connection.fromNodeId)?.title || "未知节点", toNodeId: connection.toNodeId, toTitle: nodeById.get(connection.toNodeId)?.title || "未知节点", fromHandleId: connection.fromHandleId, toHandleId: connection.toHandleId })), + resources, + warnings: [ + ...(nodes.some((node) => node.metadata?.status === "error") ? ["画布中存在生成失败节点；重试前先检查错误信息。"] : []), + ...(resources.some((resource) => !resource.ready) ? ["存在未就绪或缺少持久化引用的媒体节点，不要把占位节点当作可用参考素材。"] : []), + ], + }; +} + +export function findCanvasAgentNodes(snapshot: CanvasAgentSnapshot, input: { query?: string; ids?: string[]; types?: string[]; statuses?: string[]; resourceOnly?: boolean; limit?: number }) { + const query = input.query?.trim().toLocaleLowerCase(); + const ids = input.ids?.length ? new Set(input.ids) : null; + const types = input.types?.length ? new Set(input.types) : null; + const statuses = input.statuses?.length ? new Set(input.statuses) : null; + const limit = Math.min(Math.max(input.limit || 50, 1), 200); + const nodes = snapshot.nodes.filter((node) => { + const metadata = (node.metadata || {}) as Record; + if (ids && !ids.has(node.id)) return false; + if (types && !types.has(node.type)) return false; + if (statuses && !statuses.has(String(metadata.status || "idle"))) return false; + if (input.resourceOnly && !resourceFromNode(node)) return false; + if (!query) return true; + return [node.id, node.title, metadata.content, metadata.prompt, metadata.composerContent, metadata.assetId, Array.isArray(metadata.assetTags) ? metadata.assetTags.join(" ") : "", metadata.workflowKind, metadata.workflowTitle, metadata.characterName] + .some((value) => String(value || "").toLocaleLowerCase().includes(query)); + }); + return { query: input.query || "", total: nodes.length, truncated: nodes.length > limit, nodes: nodes.slice(0, limit).map(compactNode) }; +} + +export function getCanvasAgentResources(snapshot: CanvasAgentSnapshot, input: { nodeIds?: string[]; status?: string; limit?: number }) { + const nodeIds = input.nodeIds?.length ? new Set(input.nodeIds) : null; + const limit = Math.min(Math.max(input.limit || 100, 1), 300); + const resources = snapshot.nodes.flatMap((node) => { + if (nodeIds && !nodeIds.has(node.id)) return []; + const resource = resourceFromNode(node); + return resource && (!input.status || resource.status === input.status) ? [resource] : []; + }); + return { total: resources.length, truncated: resources.length > limit, resources: resources.slice(0, limit) }; +} + +export function validateCanvasAgentOps(snapshot: CanvasAgentSnapshot, ops: CanvasAgentOp[]) { + const nodeIds = new Set(snapshot.nodes.map((node) => node.id)); + const addedIds = new Set(); + const issues: Array<{ index: number; severity: "error" | "warning"; message: string }> = []; + const requireNode = (index: number, id: unknown, label: string) => { + if (typeof id !== "string" || !id) issues.push({ index, severity: "error", message: `${label} 缺少节点 id` }); + else if (!nodeIds.has(id) && !addedIds.has(id)) issues.push({ index, severity: "error", message: `${label}「${id}」不存在，请先重新读取画布上下文` }); + }; + ops.forEach((op, index) => { + if (op.type === "add_node") { + if (op.id && (nodeIds.has(op.id) || addedIds.has(op.id))) issues.push({ index, severity: "error", message: `新增节点 id「${op.id}」重复` }); + if (op.id) addedIds.add(op.id); + } else if (op.type === "update_node") requireNode(index, op.id, "更新目标"); + else if (op.type === "run_generation") requireNode(index, op.nodeId, "生成目标"); + else if (op.type === "delete_node") (op.ids || (op.id ? [op.id] : [])).forEach((id) => requireNode(index, id, "删除目标")); + else if (op.type === "connect_nodes") { requireNode(index, op.fromNodeId, "连接起点"); requireNode(index, op.toNodeId, "连接终点"); } + else if (op.type === "select_nodes") op.ids.forEach((id) => requireNode(index, id, "选区节点")); + }); + return { ok: !issues.some((issue) => issue.severity === "error"), issues, operationCount: ops.length, currentStateHash: hashSnapshot(snapshot) }; +} + +function compactNode(node: CanvasNodeData) { + const metadata = (node.metadata || {}) as Record; + const resource = resourceFromNode(node); + return { + id: node.id, + type: node.type, + title: node.title || "未命名节点", + position: node.position, + size: { width: node.width, height: node.height }, + parentId: node.parentId, + status: String(metadata.status || "idle"), + content: preview(metadata.content, 240), + prompt: preview(metadata.prompt || metadata.composerContent, 300), + generation: metadata.generationMode || metadata.workflowKind ? { mode: metadata.generationMode, model: metadata.model, workflowKind: metadata.workflowKind, workflowTitle: metadata.workflowTitle } : undefined, + asset: metadata.assetId || metadata.characterAssetId ? { assetId: metadata.assetId || metadata.characterAssetId, versionId: metadata.characterVersionId, category: metadata.assetCategory, tags: metadata.assetTags, characterName: metadata.characterName } : undefined, + resource: resource ? { resourceId: resource.resourceId, storageKey: resource.storageKey, mimeType: resource.mimeType, bytes: resource.bytes, width: resource.width, height: resource.height, durationMs: resource.durationMs, ready: resource.ready } : undefined, + }; +} + +function resourceFromNode(node: CanvasNodeData): CanvasAgentResource | null { + const metadata = (node.metadata || {}) as Record; + const storageKey = typeof metadata.storageKey === "string" && metadata.storageKey.trim() ? metadata.storageKey.trim() : undefined; + const resourceId = storageKey?.startsWith("resource:") ? storageKey.slice("resource:".length) : stringValue(metadata.resourceId); + const hasSignal = Boolean(storageKey || resourceId || metadata.assetId || metadata.primaryImageId || metadata.mimeType || ["image", "video", "audio"].includes(node.type)); + if (!hasSignal) return null; + const status = String(metadata.status || "idle"); + return { nodeId: node.id, nodeTitle: node.title || "未命名节点", nodeType: node.type, status, resourceId, storageKey, assetId: stringValue(metadata.assetId || metadata.characterAssetId), assetCategory: stringValue(metadata.assetCategory), mimeType: stringValue(metadata.mimeType), bytes: numberValue(metadata.bytes), width: numberValue(metadata.naturalWidth), height: numberValue(metadata.naturalHeight), durationMs: numberValue(metadata.durationMs), ready: status === "success" && Boolean(storageKey || resourceId || metadata.primaryImageId) }; +} + +function hashSnapshot(snapshot: CanvasAgentSnapshot) { + let hash = 2166136261; + const text = JSON.stringify({ projectId: snapshot.projectId, title: snapshot.title, nodes: snapshot.nodes, connections: snapshot.connections, selectedNodeIds: snapshot.selectedNodeIds, viewport: snapshot.viewport }); + for (let index = 0; index < text.length; index += 1) hash = Math.imul(hash ^ text.charCodeAt(index), 16777619); + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +function preview(value: unknown, limit: number) { + if (typeof value !== "string" || !value.trim()) return undefined; + const text = value.trim(); + return text.length > limit ? `${text.slice(0, limit)}…` : text; +} +function stringValue(value: unknown) { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function numberValue(value: unknown) { return typeof value === "number" && Number.isFinite(value) ? value : undefined; } diff --git a/web/src/lib/model-protocols.ts b/web/src/lib/model-protocols.ts index 520e3f98c..c7db65153 100644 --- a/web/src/lib/model-protocols.ts +++ b/web/src/lib/model-protocols.ts @@ -1,125 +1,19 @@ -export type ModelProtocol = - | "chat-completion" - | "openai-response" - | "openai-image" - | "grok-image" - | "volcengine-ark-image" - | "volcengine-jimeng-image" - | "gemini-image" - | "openai-audio" - | "async-audio" - | "newapi" - | "newapi-channel-1" - | "newapi-channel-2" - | "xai-video" - | "volcengine-ark-video" - | "volcengine-jimeng-video" - | "gemini-veo" - | "novita-video" - | "minimax-video"; - +export type ModelProtocol = string; export type ProtocolCapability = "text" | "image" | "video" | "audio"; +export type ModelProtocolDefinition = { value: ModelProtocol; label: string; vendor?: string; capability: ProtocolCapability; create: string; contentType: string; poll?: string; media: string; enabled?: boolean }; -export type ModelProtocolDefinition = { - value: ModelProtocol; - label: string; - capability: ProtocolCapability; - create: string; - contentType: string; - poll?: string; - media: string; -}; - -export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [ - { value: "chat-completion", label: "OpenAI Chat Completions", capability: "text", create: "POST /v1/chat/completions", contentType: "application/json", media: "文本与多模态消息" }, - { value: "openai-response", label: "OpenAI Responses", capability: "text", create: "POST /v1/responses", contentType: "application/json", media: "文本与多模态输入" }, - { value: "openai-image", label: "OpenAI Images", capability: "image", create: "POST /v1/images/generations", contentType: "application/json / multipart", media: "生成、编辑与参考图" }, - { value: "grok-image", label: "Grok Images", capability: "image", create: "POST /v1/images/generations / edits", contentType: "application/json", media: "文生图与单张 URL 参考图，不支持蒙版" }, - { value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图，不支持蒙版" }, - { value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图，模型标识填写 req_key" }, - { value: "gemini-image", label: "Gemini Images", capability: "image", create: "POST /v1beta/models/{model}:generateContent", contentType: "application/json", media: "文生图、图片编辑与多张内嵌参考图" }, - { value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" }, - { value: "async-audio", label: "异步音频任务", capability: "audio", create: "POST /v1/audio/tasks", poll: "GET /v1/audio/tasks/{task_id}", contentType: "application/json", media: "语音、音效与音乐生成" }, - { value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" }, - { value: "newapi-channel-1", label: "NewAPI 媒体任务", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "application/json", media: "图片、视频、音频公网 URL" }, - { - value: "newapi-channel-2", - label: "NewAPI Video Generations", - capability: "video", - create: "POST /v1/video/generations", - poll: "GET /v1/video/generations/{task_id}", - contentType: "application/json", - media: "image_urls（首帧、尾帧、其他参考图） / video_urls / audio_urls", - }, - { value: "xai-video", label: "xAI 官方视频", capability: "video", create: "POST /v1/videos/generations", poll: "GET /v1/videos/{request_id}", contentType: "application/json", media: "单张起始图" }, - { - value: "volcengine-ark-video", - label: "火山方舟视频", - capability: "video", - create: "POST /api/v3/contents/generations/tasks", - poll: "GET /api/v3/contents/generations/tasks/{task_id}", - contentType: "application/json", - media: "图片、视频、音频参考素材", - }, - { value: "volcengine-jimeng-video", label: "即梦官方视频", capability: "video", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "文本或一张首帧图，模型标识填写 req_key" }, - { value: "gemini-veo", label: "Gemini Veo", capability: "video", create: "POST /v1beta/models/{model}:predictLongRunning", poll: "GET /v1beta/{operation_name}", contentType: "application/json", media: "文本与单张起始图" }, - { value: "novita-video", label: "Novita 视频", capability: "video", create: "POST /v3/video/create", poll: "GET /v3/async/task-result?task_id={id}", contentType: "application/json", media: "文本或单张起始图" }, - { value: "minimax-video", label: "MiniMax 视频", capability: "video", create: "POST /v2/video_generation", poll: "GET /v2/query/video_generation/{task_id}", contentType: "application/json", media: "文本、图片、视频、音频参考素材" }, -]; - -export const MODEL_PROTOCOL_OPTIONS = protocolGroups(MODEL_PROTOCOLS.filter((item) => !item.value.startsWith("volcengine-jimeng-"))); - -export const SYSTEM_MODEL_PROTOCOL_OPTIONS = protocolGroups(MODEL_PROTOCOLS); - -function protocolGroups(protocols: ModelProtocolDefinition[]) { - return [ - { label: "文本", options: protocolOptions("text") }, - { label: "图片", options: protocolOptions("image") }, - { label: "视频", options: protocolOptions("video") }, - { label: "音频", options: protocolOptions("audio") }, - ].map((group) => ({ ...group, options: group.options.filter((option) => protocols.some((protocol) => protocol.value === option.value)) })); +export function protocolGroups(protocols: ModelProtocolDefinition[]) { + return (["text", "image", "video", "audio"] as ProtocolCapability[]).map((capability) => ({ label: { text: "文本", image: "图片", video: "视频", audio: "音频" }[capability], options: protocols.filter((item) => item.capability === capability && item.enabled !== false).map((item) => ({ label: `${item.label} · ${item.create.replace(/^POST /, "")}`, value: item.value })) })); } +export function modelProtocolDefinition(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { return definitions.find((item) => item.value === value); } +export function modelProtocolLabel(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { return modelProtocolDefinition(value, definitions)?.label || (value ? value : "未安装协议"); } +export function modelProtocolCapability(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { return modelProtocolDefinition(value, definitions)?.capability; } +export function modelProtocolSupportsTokenBilling(capability?: string, _protocol?: string) { return capability === "text"; } -export function modelProtocolDefinition(value?: string) { - return MODEL_PROTOCOLS.find((item) => item.value === value); -} - -export function modelProtocolLabel(value?: string) { - return modelProtocolDefinition(value)?.label || "自动识别"; -} - -export function modelProtocolCapability(value?: string) { - return modelProtocolDefinition(value)?.capability; -} - -export function modelProtocolSupportsTokenBilling(capability?: string, protocol?: string) { - return capability === "text" || (capability === "video" && protocol === "volcengine-ark-video"); -} - -export function protocolForModelCatalog(endpointTypes: string[] = []): ModelProtocol | undefined { - const normalized = new Set(endpointTypes.map((value) => value.trim().toLowerCase())); - if (normalized.has("openai-chat") || normalized.has("chat-completion") || normalized.has("chat")) return "chat-completion"; - if (normalized.has("openai-response") || normalized.has("responses")) return "openai-response"; - if (normalized.has("gemini-image") || normalized.has("gemini-images")) return "gemini-image"; - if (normalized.has("openai-image") || normalized.has("image")) return "openai-image"; - if (normalized.has("openai-video") || normalized.has("video")) return "newapi-channel-2"; - if (normalized.has("openai-audio") || normalized.has("audio")) return "openai-audio"; - if (normalized.has("grok-image")) return "grok-image"; - if (normalized.has("gemini-veo") || normalized.has("gemini-video") || normalized.has("veo")) return "gemini-veo"; - if (normalized.has("volcengine-ark-video") || normalized.has("volcengine-ark")) return "volcengine-ark-video"; +export function protocolForModelCatalog(_endpointTypes: string[] = []): ModelProtocol | undefined { + // A provider catalog cannot invent a protocol ID. The channel's selected + // plugin or an explicit model configuration must supply it. return undefined; } - -export function modelProtocolSummary(value?: string) { - const protocol = modelProtocolDefinition(value); - if (!protocol) return "保存模型时选择协议后，将在这里显示实际请求方式。"; - return [protocol.create, protocol.contentType, protocol.poll, protocol.media].filter(Boolean).join(" · "); -} - -export function normalizeModelProtocol(value: unknown): ModelProtocol | undefined { - return typeof value === "string" && MODEL_PROTOCOLS.some((item) => item.value === value) ? (value as ModelProtocol) : undefined; -} - -function protocolOptions(capability: ProtocolCapability) { - return MODEL_PROTOCOLS.filter((item) => item.capability === capability).map((item) => ({ label: `${item.label} · ${item.create.replace("POST ", "")}`, value: item.value })); -} +export function modelProtocolSummary(value: string | undefined, definitions: ModelProtocolDefinition[] = []) { const protocol = modelProtocolDefinition(value, definitions); return protocol ? [protocol.create, protocol.contentType, protocol.poll, protocol.media].filter(Boolean).join(" · ") : "当前协议未安装或尚未选择。"; } +export function normalizeModelProtocol(value: unknown): ModelProtocol | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } diff --git a/web/src/lib/plugins/builtin/eagle.ts b/web/src/lib/plugins/builtin/eagle.ts index 5045e1521..a60104f88 100644 --- a/web/src/lib/plugins/builtin/eagle.ts +++ b/web/src/lib/plugins/builtin/eagle.ts @@ -10,6 +10,85 @@ import type { AssetSourceProvider, ExternalAssetItem, PluginHostContext, Registe export const EAGLE_PLUGIN_ID = "eagle-asset-connector"; export const EAGLE_DEFAULT_BASE_URL = "http://127.0.0.1:41595"; +const eaglePluginDocumentation = `# Eagle 素材库 + +Eagle 插件把桌面 Eagle 资料库接入影策素材面板。浏览器不会直接访问 Eagle：所有读取、搜索、下载和写入请求先到影策后端，再由后端访问管理员配置的 Eagle Local API 地址。 + +## 使用前准备 + +1. 安装并启动 Eagle 桌面客户端，打开要使用的资料库。 +2. 确认 Eagle Local API 可用；默认地址为 \`${EAGLE_DEFAULT_BASE_URL}\`。 +3. 以管理员身份进入插件设置，填写 Base URL 并读取文件夹。 +4. 保存配置后，在素材库选择 Eagle 来源，即可浏览和导入素材。 + +> Eagle 必须保持运行。关闭客户端、切换资料库或修改本地 API 端口后，影策会收到连接失败，不能用空列表掩盖。 + +## 影策后端接口 + +| 操作 | 影策接口 | 用途 | +| --- | --- | --- | +| 读取资料库 | \`GET /api/plugins/eagle/library?baseUrl=...\` | 获取 Eagle 版本、资料库名称和文件夹树 | +| 列出素材 | \`GET /api/plugins/eagle/items\` | 按文件夹、关键词和分页读取素材 | +| 缩略图 | \`GET /api/plugins/eagle/items/{id}/thumbnail\` | 代理 Eagle 缩略图 | +| 原文件 | \`GET /api/plugins/eagle/items/{id}/file\` | 下载并导入影策资源存储 | +| 写入素材 | \`POST /api/plugins/eagle/items?baseUrl=...\` | 把图片、视频或音频写回 Eagle | +| 创建文件夹 | \`POST /api/plugins/eagle/folders?baseUrl=...\` | 在 Eagle 中创建目录 | + +这些是影策内部登录态 API，使用 Cookie 鉴权；不要把它们当作 Eagle 官方接口直接调用。 + +## 浏览与搜索 + +插件读取 Eagle 文件夹树并生成完整层级路径。素材列表支持 \`folderId\`、\`keyword\`、\`limit\`、\`offset\`。文件夹列表在当前插件实例内缓存；创建文件夹后会失效并重新读取。切换 Eagle 资料库时应重新打开插件设置确认目录。 + +## 导入到影策 + +导入时，后端代理下载 Eagle 原文件，前端再写入影策自己的图片或媒体存储。图片保留可识别的宽高和 MIME；视频、音频和其他模型文件保留字节数、扩展名、Eagle item ID、文件夹和来源元数据。 + +| Eagle 内容 | 影策结果 | 当前限制 | +| --- | --- | --- | +| 图片 | 图片素材 | 使用原图，不把缩略图当成品 | +| 视频 | 视频素材 | 元数据缺宽高时使用 1280×720 展示兜底，不改写原文件 | +| 音频 | 音频素材 | 时长以浏览器媒体探测结果为准 | +| 其他文件 | 模型素材 | 仅作为文件资源保存 | +| 文本/实体 | 不支持 | 导入会明确报错 | + +## 写回 Eagle + +插件可把影策图片、视频或音频转换为 data URL，再调用 Eagle 添加素材。已有本地 Blob 优先直接读取；只有没有本地副本时才下载远程生成地址。默认单个手动上传文件不超过 96 MB，且 MIME 必须以 \`image/\`、\`video/\` 或 \`audio/\` 开头。 + +启用“自动上传生成结果”后，生成资产可以写入配置的目标文件夹。自动写回失败必须显示为写回失败，不应把“影策已保存”误报为“Eagle 已保存”。 + +## 配置字段 + +| 字段 | 说明 | 默认值 | +| --- | --- | --- | +| \`baseUrl\` | Eagle Local API 地址，只允许明确的 HTTP(S) 地址 | \`${EAGLE_DEFAULT_BASE_URL}\` | +| \`autoUploadGenerated\` | 是否把支持的生成结果自动写回 Eagle | 开启 | +| \`generatedFolderId\` | 自动写回目标文件夹 ID | 空，使用 Eagle 默认位置 | + +## 安全边界 + +- Eagle 通常运行在本机。后端必须继续执行私网目标校验，只允许管理员明确配置的本机服务，不能把该代理变成任意 URL 抓取器。 +- \`baseUrl\` 只保存服务地址，不包含 Token、Cookie 或查询密钥。 +- 原文件和 data URL 不进入日志、localStorage 或错误上报。 +- 多用户部署中，服务器的 \`127.0.0.1\` 指服务器自身，不是访问者电脑；这种部署需要本地 Agent/隧道方案，不能假装直连可用。 + +## 常见问题 + +| 现象 | 检查项 | +| --- | --- | +| 无法读取资料库 | Eagle 是否运行、Base URL/端口、后端能否访问该主机 | +| 文件夹为空 | Eagle 当前打开的资料库、筛选条件、文件夹层级 | +| 有缩略图但导入失败 | 原文件是否仍存在、文件权限、后端响应状态 | +| 写回失败 | 文件大小/MIME、目标文件夹是否存在、远程生成链接是否过期 | +| 部署服务器连接不到本机 Eagle | 网络拓扑不成立，需要本地桥接而不是更换前端 URL | + +## 官方资料 + +- [Eagle API Documentation](https://api.eagle.cool/) +- [Eagle 产品与下载](https://eagle.cool/) +`; + export const eagleAssetPlugin: RegisteredPlugin = { manifest: { id: EAGLE_PLUGIN_ID, @@ -20,10 +99,12 @@ export const eagleAssetPlugin: RegisteredPlugin = { apiVersion: "1", category: "asset-source", description: "把 Eagle 作为影策的外部素材来源，直接浏览原始文件夹并读写 Eagle 文件。", + documentation: eaglePluginDocumentation, author: "影策社区", surfaces: ["asset-source"], permissions: ["asset.read", "asset.search", "asset.upload", "external.open"], trusted: true, + configuration: { fields: ["baseUrl", "autoUploadGenerated", "generatedFolderId"] }, }, createAssetSource: ({ config }: PluginHostContext) => { const configuredBaseUrl = config.baseUrl; diff --git a/web/src/lib/plugins/plugin-types.ts b/web/src/lib/plugins/plugin-types.ts index 1e497008c..500fc3394 100644 --- a/web/src/lib/plugins/plugin-types.ts +++ b/web/src/lib/plugins/plugin-types.ts @@ -2,8 +2,20 @@ import type { Asset } from "@/stores/use-asset-store"; export const PLUGIN_API_VERSION = "1" as const; -export type PluginCategory = "asset-source" | "canvas-node" | "workflow" | "ai-capability" | "import-export" | "agent"; +export type PluginCategory = "asset-source" | "canvas-node" | "workflow" | "ai-capability" | "import-export" | "agent" | "protocol"; export type PluginSurface = "node" | "fullscreen" | "hybrid" | "asset-source"; +export type ProtocolCapability = "text" | "image" | "video" | "audio"; +export type ProtocolScope = "admin.system-channel" | "user.custom-channel" | "canvas" | "creation" | "agent" | string; +export type ProtocolPluginInfo = { + categories: ProtocolCapability[]; + scopes: ProtocolScope[]; + create?: string; + poll?: string; + cancel?: string; + contentType?: string; + documentation?: string; + parameters?: Array<{ name: string; type: string; required?: boolean; description?: string; values?: string[]; mapping?: string }>; +}; export type PluginPermission = | "canvas.read" | "canvas.write" @@ -23,11 +35,17 @@ export type PluginManifest = { apiVersion: string; category: PluginCategory; description: string; + documentation?: string; author?: string; entry?: string; surfaces: PluginSurface[]; permissions: PluginPermission[]; trusted?: boolean; + kind?: "ui" | "protocol"; + configuration?: { + fields: string[]; + }; + protocol?: ProtocolPluginInfo; }; export type PluginStorage = { @@ -96,6 +114,7 @@ export type AssetSourceProvider = { export type RegisteredPlugin = { manifest: PluginManifest; + source?: "bundled" | "uploaded" | string; activate?: (context: PluginHostContext) => Promise | void; deactivate?: (context: PluginHostContext) => Promise | void; createAssetSource?: (context: PluginHostContext) => AssetSourceProvider; diff --git a/web/src/lib/user-session.ts b/web/src/lib/user-session.ts index 69748b478..8ad6e317b 100644 --- a/web/src/lib/user-session.ts +++ b/web/src/lib/user-session.ts @@ -135,7 +135,9 @@ function systemChannelModelChannels(channels: PublicChannelCatalog[]): ModelChan return { id: channel.id, name: channel.displayName, - baseUrl: "/api", + // 系统渠道必须走带渠道 ID 的站内代理；/api 只是业务 API 根路径， + // 不能作为模型请求的运行时 Base URL 传给 channelRequest。 + baseUrl: `/api/${channel.id}`, apiKey: "system", apiFormat: "openai", scope: "system" as const, @@ -167,6 +169,7 @@ function systemChannelModelChannels(channels: PublicChannelCatalog[]): ModelChan description: "", icon: "", capability: model.capability as ModelCapability, + protocol: model.protocol as any, pricePolicy: "channel" as const, billingMode: billingMode as any, unitPriceMicrocredits: unitPrice, diff --git a/web/src/pages/admin/admin-route-pages.tsx b/web/src/pages/admin/admin-route-pages.tsx index 571e7e67c..b787c3c9c 100644 --- a/web/src/pages/admin/admin-route-pages.tsx +++ b/web/src/pages/admin/admin-route-pages.tsx @@ -37,5 +37,5 @@ export function EmailSettingsPage() { } export function FeatureAvailabilityPage() { - return }>; + return }>; } diff --git a/web/src/pages/admin/components/channel-model-manager.tsx b/web/src/pages/admin/components/channel-model-manager.tsx index b6b990ca6..4f1d53ca0 100644 --- a/web/src/pages/admin/components/channel-model-manager.tsx +++ b/web/src/pages/admin/components/channel-model-manager.tsx @@ -8,7 +8,8 @@ import { ModelIcon } from "@/components/model-picker"; import { ModelCapabilityEditor } from "@/components/model-capability-editor"; import { CapabilityCardPicker, ProtocolCardPicker, type ModelCapabilityChoice } from "@/components/model-protocol-picker"; import { defaultModelCapabilityConfig, normalizeModelCapabilityConfig, type ModelCapabilityConfig } from "@/lib/model-capabilities"; -import { MODEL_PROTOCOLS, modelProtocolCapability, modelProtocolDefinition, modelProtocolLabel, modelProtocolSupportsTokenBilling, type ModelProtocol } from "@/lib/model-protocols"; +import { modelProtocolCapability, modelProtocolDefinition, modelProtocolLabel, modelProtocolSupportsTokenBilling, type ModelProtocol } from "@/lib/model-protocols"; +import { fetchProtocolCatalog } from "@/services/api/protocols"; import { createAdminChannelModel, deleteAdminChannelModel, fetchAdminChannelModels, listAdminChannelModels, testAdminChannelModel, updateAdminChannelModel, type ChannelModel, type ChannelModelPriceTier } from "@/services/api/wallet"; import type { ModelChannel } from "@/stores/use-config-store"; import { AdminPageFrame } from "./admin-shell"; @@ -21,7 +22,7 @@ type FormValues = { providerModelKey?: string; displayName?: string; capability: EditableCapability; - protocol: ModelProtocol; + protocol?: ModelProtocol; priceTiers: PriceTierFormValues[]; enabled: boolean; capabilityConfig?: ModelCapabilityConfig; @@ -58,6 +59,7 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel: const [status, setStatus] = useState<"all" | "enabled" | "disabled">("all"); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(20); + const [availableProtocols, setAvailableProtocols] = useState([]); const [form] = Form.useForm(); const modelCapability = Form.useWatch("capability", form); const modelProtocol = Form.useWatch("protocol", form); @@ -79,6 +81,7 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel: useEffect(() => { void reload(); + void fetchProtocolCatalog("admin.system-channel").then(setAvailableProtocols).catch(() => setAvailableProtocols([])); setEditing(null); setEditorOpen(false); setKeyword(""); @@ -111,10 +114,10 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel: providerModelKey: "", displayName: "", capability: "text", - protocol: "chat-completion", + protocol: availableProtocols.find((item) => item.capability === "text")?.value, priceTiers: [defaultPriceTier()], enabled: true, - capabilityConfig: defaultModelCapabilityConfig("chat-completion", ""), + capabilityConfig: defaultModelCapabilityConfig(availableProtocols.find((item) => item.capability === "text")?.value, ""), }); setEditorOpen(true); }; @@ -220,8 +223,8 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel: } if (!changed.capability) return; const current = form.getFieldValue("protocol") as ModelProtocol | undefined; - if (modelProtocolCapability(current) !== changed.capability) { - const nextProtocol = MODEL_PROTOCOLS.find((item) => item.capability === changed.capability)?.value; + if (modelProtocolCapability(current, availableProtocols) !== changed.capability) { + const nextProtocol = availableProtocols.find((item) => item.capability === changed.capability)?.value; form.setFieldValue("protocol", nextProtocol); form.setFieldValue("capabilityConfig", changed.capability === "text" || changed.capability === "image" || changed.capability === "video" ? defaultModelCapabilityConfig(nextProtocol, form.getFieldValue("modelKey")) : undefined); } @@ -262,8 +265,8 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel: render: (value: ModelProtocol) => value ? (
-
{modelProtocolLabel(value)}
-
{modelProtocolDefinition(value)?.create}
+
{modelProtocolLabel(value, availableProtocols)}
+
{modelProtocolDefinition(value, availableProtocols)?.create}
) : ( @@ -423,9 +426,9 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel: - - - + {availableProtocols.length ? + + : null} {modelCapability === "text" || modelCapability === "image" || modelCapability === "video" ? ( diff --git a/web/src/pages/admin/components/feature-availability-panel.tsx b/web/src/pages/admin/components/feature-availability-panel.tsx index 9d285cb53..22d8f13f4 100644 --- a/web/src/pages/admin/components/feature-availability-panel.tsx +++ b/web/src/pages/admin/components/feature-availability-panel.tsx @@ -1,18 +1,20 @@ import { useEffect, useState, type ReactNode } from "react"; import { App, Switch } from "antd"; -import { Clapperboard, Coins, ListChecks, RadioTower, ToggleLeft, Sparkles, Settings } from "lucide-react"; +import { Clapperboard, Coins, ListChecks, PlugZap, RadioTower, ToggleLeft, Sparkles, Settings } from "lucide-react"; import { getAdminFeatureAvailability, updateAdminFeatureAvailability } from "@/services/api/auth"; import { useUserStore, type FeatureAvailability } from "@/stores/use-user-store"; import { SettingsSectionCard } from "./admin-ui"; -type FeatureKey = "shortDramaEnabled" | "taskCenterEnabled" | "creditsEnabled" | "customChannelsEnabled" | "frontendModelsEnabled"; +type FeatureKey = "shortDramaEnabled" | "taskCenterEnabled" | "creditsEnabled" | "customChannelsEnabled" | "frontendModelsEnabled" | "pluginCenterEnabled" | "systemPluginsVisibleToUsers"; const userFeatureRows: Array<{ key: FeatureKey; title: string; menu: string; description: string; icon: ReactNode }> = [ { key: "shortDramaEnabled", title: "短剧创作", menu: "/projects", description: "关闭后隐藏短剧入口，并拦截项目列表、详情和项目 API。已有项目数据不会删除。", icon: }, { key: "taskCenterEnabled", title: "任务", menu: "/tasks", description: "关闭后仅隐藏并拦截任务中心页面；生成任务仍会创建、执行、记录和恢复。", icon: }, { key: "creditsEnabled", title: "积分中心", menu: "/wallet", description: "关闭后隐藏用户积分入口，新创建的任务和系统渠道请求不再冻结或消费积分。", icon: }, { key: "customChannelsEnabled", title: "自定义渠道", menu: "/settings?section=channels", description: "关闭后隐藏用户自定义渠道入口，并拦截模型目录拉取、渠道中转和新的生成任务。已有渠道配置不会删除。", icon: }, + { key: "pluginCenterEnabled", title: "插件中心", menu: "/plugins", description: "关闭后普通用户无法进入插件中心或调用插件中心相关接口；管理员仍可进入后台恢复开关。已有插件不会删除。", icon: }, + { key: "systemPluginsVisibleToUsers", title: "向普通用户显示系统插件（不可编辑）", menu: "/plugins", description: "仅在插件中心开启时生效。关闭后普通用户看不到系统插件，用户插件不受影响。", icon: }, ]; const adminFeatureRows: Array<{ key: FeatureKey; title: string; menu: string; description: string; icon: ReactNode }> = [ @@ -48,6 +50,8 @@ export default function FeatureAvailabilityPanel() { creditsEnabled: next.creditsEnabled, customChannelsEnabled: next.customChannelsEnabled, frontendModelsEnabled: next.frontendModelsEnabled, + pluginCenterEnabled: next.pluginCenterEnabled, + systemPluginsVisibleToUsers: next.systemPluginsVisibleToUsers, }); setFeatures(result.features); setGlobalFeatures(result.features); @@ -108,7 +112,7 @@ export default function FeatureAvailabilityPanel() { {item.menu} - toggle(item.key, checked)} aria-label={`开放${item.title}`} /> + toggle(item.key, checked)} aria-label={`开放${item.title}`} /> ))} diff --git a/web/src/pages/create/index.tsx b/web/src/pages/create/index.tsx index 1b8225f94..f889731b3 100644 --- a/web/src/pages/create/index.tsx +++ b/web/src/pages/create/index.tsx @@ -21,7 +21,7 @@ import { buildImageResolutionOptions, formatImageResolutionSize, imageRatioForSi import { VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options"; import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, videoDurationAllowed, videoDurationOptions, type ImageCapabilityConfig, type VideoCapabilityConfig } from "@/lib/model-capabilities"; import { resolveCompatibleModel, mergedImageCapabilityConfig, type ModelRequirements } from "@/lib/model-selection"; -import { isGenerationTaskCancelled, logicalModelIDForConfig, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task"; +import { backendModelRuntimeRequired, isGenerationTaskCancelled, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task"; import { requestImageQuestion, type AiTextContentPart } from "@/services/api/image"; import { listAddedSkills, type Skill } from "@/services/api/skills"; import { subscribeGenerationTasks, type GenerationTask } from "@/services/api/task-center"; @@ -538,7 +538,7 @@ export default function CreatePage() { }; try { if (mode === "text") { - if (logicalModelIDForConfig(requestConfig)) { + if (backendModelRuntimeRequired(requestConfig)) { const result = await runGenerationOperationOnce(retryContext?.clientOperationId, () => runBackendGenerationTask({ mode: "text", prompt: expandedPrompt, diff --git a/web/src/pages/plugins/index.tsx b/web/src/pages/plugins/index.tsx index 24ee6c491..68ec35703 100644 --- a/web/src/pages/plugins/index.tsx +++ b/web/src/pages/plugins/index.tsx @@ -1,14 +1,18 @@ import { App, Button, Input, Modal, Select, Switch, Typography } from "antd"; -import { CalendarDays, CheckCircle2, Clock3, ExternalLink, FolderOpen, PlugZap, Search, Settings2, ShieldCheck, SlidersHorizontal } from "lucide-react"; +import { AudioLines, CalendarDays, CheckCircle2, Clock3, CloudUpload, ExternalLink, Film, FolderOpen, Image as ImageIcon, MessageSquareText, PlugZap, RefreshCw, Search, Settings2, ShieldCheck, SlidersHorizontal } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { useNavigate } from "react-router"; import { listRegisteredPlugins } from "@/lib/plugins/plugin-registry"; import "@/lib/plugins/builtin"; import { EAGLE_PLUGIN_ID } from "@/lib/plugins/builtin/eagle"; +import type { PluginManifest, RegisteredPlugin } from "@/lib/plugins/plugin-types"; import { getEagleLibrary, type EagleFolder } from "@/services/api/eagle"; +import { fetchPlugins, setPluginEnabled, uploadPlugin, type BackendPlugin } from "@/services/api/plugins"; import { usePluginStore } from "@/stores/use-plugin-store"; +import { useUserStore } from "@/stores/use-user-store"; +import { PluginDetailsModal, UploadPluginModal } from "./plugin-documentation-modals"; import "./plugins.css"; const categoryLabels: Record = { @@ -18,6 +22,7 @@ const categoryLabels: Record = { "ai-capability": "AI 能力", "import-export": "导入导出", agent: "智能体", + protocol: "请求协议", }; const surfaceLabels: Record = { @@ -40,15 +45,29 @@ const permissionLabels: Record = { const pluginDateFormatter = new Intl.DateTimeFormat("zh-CN", { year: "numeric", month: "short", day: "numeric" }); +const protocolSectionMeta = [ + { key: "text", label: "文本协议", description: "对话、推理与流式响应", icon: MessageSquareText }, + { key: "image", label: "图片协议", description: "生成、编辑与参考图", icon: ImageIcon }, + { key: "video", label: "视频协议", description: "创建任务、轮询与媒体结果", icon: Film }, + { key: "audio", label: "音频协议", description: "语音与异步音频任务", icon: AudioLines }, +] as const; + export default function PluginsPage() { const { message } = App.useApp(); const navigate = useNavigate(); + const user = useUserStore((state) => state.user); + const features = useUserStore((state) => state.features); const installations = usePluginStore((state) => state.installations); const ensurePlugin = usePluginStore((state) => state.ensurePlugin); const setEnabled = usePluginStore((state) => state.setEnabled); const updateConfig = usePluginStore((state) => state.updateConfig); - const registeredPlugins = useMemo(() => listRegisteredPlugins(), []); + const builtinPlugins = useMemo(() => listRegisteredPlugins(), []); + const [backendPlugins, setBackendPlugins] = useState([]); + const [backendPluginsLoading, setBackendPluginsLoading] = useState(false); const [settingsPluginId, setSettingsPluginId] = useState(null); + const [detailsPluginId, setDetailsPluginId] = useState(null); + const [detailsRestoreFocus, setDetailsRestoreFocus] = useState(false); + const [uploadModalOpen, setUploadModalOpen] = useState(false); const [search, setSearch] = useState(""); const [categoryFilter, setCategoryFilter] = useState("all"); const [statusFilter, setStatusFilter] = useState<"all" | "enabled" | "disabled">("all"); @@ -61,8 +80,28 @@ export default function PluginsPage() { const [eagleFoldersError, setEagleFoldersError] = useState(""); useEffect(() => { - for (const plugin of registeredPlugins) ensurePlugin(plugin.manifest); - }, [ensurePlugin, registeredPlugins]); + for (const plugin of builtinPlugins) ensurePlugin(plugin.manifest); + }, [builtinPlugins, ensurePlugin]); + + const reloadBackendPlugins = async () => { + setBackendPluginsLoading(true); + try { + setBackendPlugins(await fetchPlugins()); + } catch (error) { + message.error(error instanceof Error ? error.message : "读取插件中心失败"); + setBackendPlugins([]); + } finally { + setBackendPluginsLoading(false); + } + }; + + useEffect(() => { + void reloadBackendPlugins(); + }, []); + + const remotePlugins = useMemo(() => backendPlugins.map(toRegisteredPlugin), [backendPlugins]); + const registeredPlugins = useMemo(() => [...builtinPlugins, ...remotePlugins], [builtinPlugins, remotePlugins]); + const backendPluginById = useMemo(() => new Map(backendPlugins.map((plugin) => [plugin.manifest.id, plugin])), [backendPlugins]); const eagle = installations.find((item) => item.manifest.id === EAGLE_PLUGIN_ID); @@ -75,29 +114,86 @@ export default function PluginsPage() { setEagleGeneratedFolderId(typeof folderId === "string" ? folderId : ""); }, [eagle?.config.baseUrl, eagle?.config.autoUploadGenerated, eagle?.config.generatedFolderId]); - const categoryOptions = useMemo(() => [ - { value: "all", label: "全部分类" }, - ...Array.from(new Set(registeredPlugins.map((plugin) => plugin.manifest.category))).map((category) => ({ value: category, label: categoryLabels[category] ?? category })), - ], [registeredPlugins]); - const filteredPlugins = useMemo(() => { const normalizedSearch = search.trim().toLocaleLowerCase(); return registeredPlugins.filter((plugin) => { + const isSystemPlugin = plugin.source !== "uploaded"; + if (user?.role !== "admin" && !features.systemPluginsVisibleToUsers && isSystemPlugin) return false; const installation = installations.find((item) => item.manifest.id === plugin.manifest.id); + const enabled = plugin.manifest.kind === "protocol" ? backendPluginById.get(plugin.manifest.id)?.status === "enabled" : Boolean(installation?.enabled); const manifest = plugin.manifest; const searchableText = [manifest.name, manifest.description, manifest.author, manifest.id, categoryLabels[manifest.category]].filter(Boolean).join(" ").toLocaleLowerCase(); if (normalizedSearch && !searchableText.includes(normalizedSearch)) return false; - if (categoryFilter !== "all" && manifest.category !== categoryFilter) return false; + if (categoryFilter !== "all") { + const matchesCapability = manifest.kind === "protocol" && manifest.protocol?.categories.includes(categoryFilter as "text" | "image" | "video" | "audio"); + const matchesApp = categoryFilter === "other" && manifest.kind !== "protocol"; + if (!matchesCapability && !matchesApp) return false; + } if (trustFilter === "trusted" && !manifest.trusted) return false; - if (statusFilter === "enabled" && !installation?.enabled) return false; - if (statusFilter === "disabled" && installation?.enabled) return false; + if (statusFilter === "enabled" && !enabled) return false; + if (statusFilter === "disabled" && enabled) return false; return true; }); - }, [categoryFilter, installations, registeredPlugins, search, statusFilter, trustFilter]); + }, [backendPluginById, categoryFilter, features.systemPluginsVisibleToUsers, installations, registeredPlugins, search, statusFilter, trustFilter, user?.role]); + + const pluginSections = useMemo( + () => [ + ...protocolSectionMeta.map((section) => ({ ...section, plugins: filteredPlugins.filter((plugin) => plugin.manifest.kind === "protocol" && plugin.manifest.protocol?.categories.includes(section.key)) })), + { key: "other", label: "应用插件", description: "画布、素材与工作流扩展", icon: PlugZap, plugins: filteredPlugins.filter((plugin) => plugin.manifest.kind !== "protocol") }, + ], + [filteredPlugins], + ); + + const categoryCounts = useMemo(() => { + const visiblePlugins = registeredPlugins.filter((plugin) => user?.role === "admin" || features.systemPluginsVisibleToUsers || plugin.source === "uploaded"); + const counts: Record = { all: visiblePlugins.length, text: 0, image: 0, video: 0, audio: 0, other: 0 }; + for (const plugin of visiblePlugins) { + if (plugin.manifest.kind !== "protocol") { + counts.other += 1; + continue; + } + for (const capability of plugin.manifest.protocol?.categories || []) { + counts[capability] = (counts[capability] || 0) + 1; + } + } + return counts; + }, [features.systemPluginsVisibleToUsers, registeredPlugins, user?.role]); const settingsPlugin = settingsPluginId ? registeredPlugins.find((plugin) => plugin.manifest.id === settingsPluginId) : undefined; const settingsInstallation = settingsPlugin ? installations.find((item) => item.manifest.id === settingsPlugin.manifest.id) : undefined; - const settingsEnabled = Boolean(settingsInstallation?.enabled); + const settingsEnabled = settingsPlugin?.manifest.kind === "protocol" ? backendPluginById.get(settingsPlugin.manifest.id)?.status === "enabled" : Boolean(settingsInstallation?.enabled); + const detailsPlugin = detailsPluginId ? registeredPlugins.find((plugin) => plugin.manifest.id === detailsPluginId) : undefined; + + const hasPluginConfiguration = (plugin: RegisteredPlugin) => Boolean(plugin.manifest.configuration?.fields.length); + const canConfigurePlugin = (plugin: RegisteredPlugin) => hasPluginConfiguration(plugin) && user?.role === "admin"; + + const isPluginEnabled = (plugin: RegisteredPlugin, installation = installations.find((item) => item.manifest.id === plugin.manifest.id)) => + plugin.manifest.kind === "protocol" ? backendPluginById.get(plugin.manifest.id)?.status === "enabled" : Boolean(installation?.enabled); + + const togglePlugin = async (plugin: RegisteredPlugin, enabled: boolean) => { + if (plugin.manifest.kind !== "protocol") { + setEnabled(plugin.manifest.id, enabled); + return; + } + try { + const next = await setPluginEnabled(plugin.manifest.id, enabled); + setBackendPlugins((items) => items.map((item) => (item.manifest.id === next.manifest.id ? next : item))); + message.success(`${plugin.manifest.name}${enabled ? "已启用" : "已停用"}`); + } catch (error) { + message.error(error instanceof Error ? error.message : "更新插件状态失败"); + } + }; + + const handleUpload = async (file: File) => { + try { + const plugin = await uploadPlugin(file); + setBackendPlugins((items) => [...items.filter((item) => item.manifest.id !== plugin.manifest.id), plugin]); + setUploadModalOpen(false); + message.success("插件已安装并立即生效"); + } catch (error) { + message.error(error instanceof Error ? error.message : "安装插件失败"); + } + }; const loadEagleFolders = async (url = eagleBaseUrl) => { setEagleFoldersLoading(true); @@ -126,173 +222,329 @@ export default function PluginsPage() { return (
-
-
- } - value={search} - allowClear - placeholder="搜索插件名称、描述或作者" - onChange={(event) => setSearch(event.target.value)} - /> - setStatusFilter(value as "all" | "enabled" | "disabled")} - aria-label="按状态筛选" - /> - } + value={search} + allowClear + placeholder="搜索插件名称、描述或作者" + onChange={(event) => setSearch(event.target.value)} + /> + setTrustFilter(value as "all" | "trusted")} + aria-label="按来源筛选" + /> + + {user?.role === "admin" ? ( +
+ +
+ ) : null} +
- {settingsPlugin.manifest.id === EAGLE_PLUGIN_ID ? ( - <> -
-
- - setEagleBaseUrl(event.target.value)} placeholder="http://localhost:41595" /> -

Eagle 必须在本机运行；影策通过插件直接读取和写入 Eagle 原始文件。

-
-
-
- - + {filteredPlugins.length ? ( +
+ {pluginSections.map((section) => { + if (!section.plugins.length) return null; + const SectionIcon = section.icon; + return ( +
+
+ + + +
+

{section.label}

+

{section.description}

-

图片、视频和音频生成成功后，自动写入 Eagle；影策本地素材仍会保留。

-
-
-
- - -
- setEagleBaseUrl(event.target.value)} placeholder="http://localhost:41595" /> +

Eagle 必须在本机运行；影策通过插件直接读取和写入 Eagle 原始文件。

+
+
+
+ + +
+

图片、视频和音频生成成功后，自动写入 Eagle；影策本地素材仍会保留。

+
+
+
+ + +
+