diff --git a/backend/Dockerfile b/backend/Dockerfile index 17523da40..b2b0390fd 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -37,6 +37,7 @@ RUN mkdir -p /data && chown -R app:app /data COPY --from=backend-build /out/infinite-canvas-backend /usr/local/bin/infinite-canvas-backend COPY --from=backend-build /out/migrate-schema /usr/local/bin/migrate-schema COPY --from=backend-build /out/migrate-sqlite-postgres /usr/local/bin/migrate-sqlite-postgres +COPY plugin-packages /app/plugin-packages USER app EXPOSE 8080 diff --git a/backend/internal/protocol/adapter.go b/backend/internal/protocol/adapter.go index 8d4372cbd..37f2d0999 100644 --- a/backend/internal/protocol/adapter.go +++ b/backend/internal/protocol/adapter.go @@ -22,6 +22,23 @@ type AgentAdapter interface { ParseAgent(context.Context, []byte) (AgentResult, error) } +// AgentCapability lets the host distinguish a declarative provider that +// actually declares an agent operation from one that only satisfies the Go +// method set through the shared manifest adapter implementation. +type AgentCapability interface { + AgentAvailable() bool +} + +// ResultAdapter optionally resolves an authenticated binary result after an +// asynchronous task succeeds without returning a public media URL. +type ResultAdapter interface { + BuildResult(context.Context, PollContext) (RequestSpec, error) +} + +type ResultCapability interface { + ResultAvailable() bool +} + type AgentRequestContext struct { BaseURL string Model string @@ -29,9 +46,10 @@ type AgentRequestContext struct { } type AgentToolCall struct { - ID string - Name string - Arguments string + ID string + Name string + Arguments string + ThoughtSignature string } type AgentResult struct { diff --git a/backend/internal/protocol/builtin.go b/backend/internal/protocol/builtin.go index 88c5b773a..61399d5d0 100644 --- a/backend/internal/protocol/builtin.go +++ b/backend/internal/protocol/builtin.go @@ -1087,7 +1087,9 @@ func normalizeStatus(raw string) Status { return StatusProcessing case "succeeded", "success", "completed", "complete", "done", "task_status_succeed": return StatusSucceeded - case "failed", "failure", "error", "cancelled", "canceled", "expired": + case "cancelled", "canceled", "aborted": + return StatusCancelled + case "failed", "failure", "error", "expired": return StatusFailed default: return "" diff --git a/backend/internal/protocol/expression.go b/backend/internal/protocol/expression.go new file mode 100644 index 000000000..44837f3be --- /dev/null +++ b/backend/internal/protocol/expression.go @@ -0,0 +1,613 @@ +package protocol + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" +) + +// Manifest expressions are JSON values with a deliberately small set of +// $-prefixed operators. They can construct provider payloads without running +// plugin code or exposing host objects. +func evaluateManifestValue(template any, env map[string]any) (any, error) { + switch value := template.(type) { + case nil, bool, float64, float32, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return value, nil + case string: + trimmed := strings.TrimSpace(value) + if strings.HasPrefix(trimmed, "${") && strings.HasSuffix(trimmed, "}") { + return manifestPathValue(env, strings.TrimSpace(trimmed[2:len(trimmed)-1])), nil + } + return value, nil + case []any: + result := make([]any, 0, len(value)) + for _, item := range value { + evaluated, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + if evaluated != nil { + result = append(result, evaluated) + } + } + return result, nil + case map[string]any: + if len(value) == 1 { + for operator, operand := range value { + if strings.HasPrefix(operator, "$") { + return evaluateManifestOperator(operator, operand, env) + } + } + } + result := make(map[string]any, len(value)) + for key, item := range value { + evaluated, err := evaluateManifestValue(item, env) + if err != nil { + return nil, fmt.Errorf("evaluate %s: %w", key, err) + } + if evaluated != nil { + result[key] = evaluated + } + } + return result, nil + default: + encoded, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("unsupported manifest template value %T", template) + } + var normalized any + if err := json.Unmarshal(encoded, &normalized); err != nil { + return nil, err + } + return evaluateManifestValue(normalized, env) + } +} + +func evaluateManifestOperator(operator string, operand any, env map[string]any) (any, error) { + switch operator { + case "$ref": + path, ok := operand.(string) + if !ok { + return nil, fmt.Errorf("$ref requires a string path") + } + return manifestPathValue(env, path), nil + case "$literal": + return operand, nil + case "$coalesce", "$default": + values, ok := operand.([]any) + if !ok { + return nil, fmt.Errorf("%s requires an array", operator) + } + for _, item := range values { + value, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + if !manifestEmpty(value) { + return value, nil + } + } + return nil, nil + case "$concat": + values, ok := operand.([]any) + if !ok { + return nil, fmt.Errorf("$concat requires an array") + } + var result strings.Builder + for _, item := range values { + value, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + if value != nil { + result.WriteString(manifestString(value)) + } + } + return result.String(), nil + case "$concatArrays": + values, ok := operand.([]any) + if !ok { + return nil, fmt.Errorf("$concatArrays requires an array") + } + result := make([]any, 0) + for _, item := range values { + value, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + result = append(result, manifestArray(value)...) + } + return result, nil + case "$map", "$filter": + spec, ok := operand.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s requires an object", operator) + } + from, err := evaluateManifestValue(spec["from"], env) + if err != nil { + return nil, err + } + alias := strings.TrimSpace(manifestString(spec["as"])) + if alias == "" { + alias = "item" + } + items := manifestArray(from) + result := make([]any, 0, len(items)) + for index, item := range items { + child := cloneManifestEnv(env) + child[alias] = item + child[alias+"Index"] = index + if operator == "$filter" { + condition, err := evaluateManifestValue(spec["where"], child) + if err != nil { + return nil, err + } + if manifestTruthy(condition) { + result = append(result, item) + } + continue + } + mapped, err := evaluateManifestValue(spec["in"], child) + if err != nil { + return nil, err + } + if mapped != nil { + result = append(result, mapped) + } + } + return result, nil + case "$indexObject": + spec, ok := operand.(map[string]any) + if !ok { + return nil, fmt.Errorf("$indexObject requires an object") + } + from, err := evaluateManifestValue(spec["from"], env) + if err != nil { + return nil, err + } + alias := strings.TrimSpace(manifestString(spec["as"])) + if alias == "" { + alias = "item" + } + prefixValue, err := evaluateManifestValue(spec["prefix"], env) + if err != nil { + return nil, err + } + prefix := manifestString(prefixValue) + maxItems := len(manifestArray(from)) + if spec["max"] != nil { + value, err := evaluateManifestValue(spec["max"], env) + if err != nil { + return nil, err + } + if configured := manifestInt(value); configured >= 0 && configured < maxItems { + maxItems = configured + } + } + result := make(map[string]any, maxItems) + for index, item := range manifestArray(from)[:maxItems] { + child := cloneManifestEnv(env) + child[alias] = item + child[alias+"Index"] = index + value, err := evaluateManifestValue(spec["value"], child) + if err != nil { + return nil, err + } + if value != nil && !manifestEmpty(value) { + result[prefix+strconv.Itoa(index)] = value + } + } + return result, nil + case "$first", "$last": + value, err := evaluateManifestValue(operand, env) + if err != nil { + return nil, err + } + items := manifestArray(value) + if len(items) == 0 { + return nil, nil + } + if operator == "$last" { + return items[len(items)-1], nil + } + return items[0], nil + case "$at": + values, ok := operand.([]any) + if !ok || len(values) != 2 { + return nil, fmt.Errorf("$at requires [array, index]") + } + value, err := evaluateManifestValue(values[0], env) + if err != nil { + return nil, err + } + indexValue, err := evaluateManifestValue(values[1], env) + if err != nil { + return nil, err + } + items := manifestArray(value) + index := manifestInt(indexValue) + if index < 0 || index >= len(items) { + return nil, nil + } + return items[index], nil + case "$len": + value, err := evaluateManifestValue(operand, env) + if err != nil { + return nil, err + } + switch typed := value.(type) { + case []any: + return len(typed), nil + case map[string]any: + return len(typed), nil + case string: + return len([]rune(typed)), nil + default: + return 0, nil + } + case "$if": + spec, ok := operand.(map[string]any) + if !ok { + return nil, fmt.Errorf("$if requires an object") + } + condition, err := evaluateManifestValue(spec["condition"], env) + if err != nil { + return nil, err + } + if manifestTruthy(condition) { + return evaluateManifestValue(spec["then"], env) + } + return evaluateManifestValue(spec["else"], env) + case "$switch": + spec, ok := operand.(map[string]any) + if !ok { + return nil, fmt.Errorf("$switch requires an object") + } + for _, rawCase := range manifestArray(spec["cases"]) { + item, _ := rawCase.(map[string]any) + when, err := evaluateManifestValue(item["when"], env) + if err != nil { + return nil, err + } + if manifestTruthy(when) { + return evaluateManifestValue(item["then"], env) + } + } + return evaluateManifestValue(spec["default"], env) + case "$omitEmpty": + value, err := evaluateManifestValue(operand, env) + if err != nil || manifestEmpty(value) { + return nil, err + } + return value, nil + case "$lower", "$upper", "$trim", "$toString", "$toInt", "$toBool", "$dataMime", "$dataPayload", "$json": + value, err := evaluateManifestValue(operand, env) + if err != nil { + return nil, err + } + text := manifestString(value) + switch operator { + case "$lower": + return strings.ToLower(text), nil + case "$upper": + return strings.ToUpper(text), nil + case "$trim": + return strings.TrimSpace(text), nil + case "$toString": + return text, nil + case "$toInt": + return manifestInt(value), nil + case "$toBool": + return manifestTruthy(value), nil + case "$dataMime": + return dataMIME(text), nil + case "$dataPayload": + return dataPayload(text), nil + default: + encoded, err := json.Marshal(value) + return string(encoded), err + } + case "$merge": + values, ok := operand.([]any) + if !ok { + return nil, fmt.Errorf("$merge requires an array") + } + result := make(map[string]any) + for _, item := range values { + value, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + for key, entry := range manifestObject(value) { + result[key] = entry + } + } + return result, nil + case "$sortByOrder": + value, err := evaluateManifestValue(operand, env) + if err != nil { + return nil, err + } + items := append([]any(nil), manifestArray(value)...) + sort.SliceStable(items, func(i, j int) bool { + return manifestInt(manifestPathValue(items[i], "order")) < manifestInt(manifestPathValue(items[j], "order")) + }) + return items, nil + case "$add", "$multiply", "$min", "$max": + values, ok := operand.([]any) + if !ok || len(values) == 0 { + return nil, fmt.Errorf("%s requires an array", operator) + } + result := 0.0 + if operator == "$multiply" { + result = 1 + } else if operator == "$min" { + result = math.Inf(1) + } else if operator == "$max" { + result = math.Inf(-1) + } + for _, item := range values { + value, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + number := manifestFloat(value) + switch operator { + case "$multiply": + result *= manifestFloat(value) + case "$min": + result = math.Min(result, number) + case "$max": + result = math.Max(result, number) + default: + result += manifestFloat(value) + } + } + return normalizeManifestNumber(result), nil + case "$ceilStep": + values, ok := operand.([]any) + if !ok || len(values) != 2 { + return nil, fmt.Errorf("$ceilStep requires [value, step]") + } + value, err := evaluateManifestValue(values[0], env) + if err != nil { + return nil, err + } + step, err := evaluateManifestValue(values[1], env) + if err != nil { + return nil, err + } + stepValue := manifestFloat(step) + if stepValue <= 0 { + return nil, fmt.Errorf("$ceilStep step must be positive") + } + return normalizeManifestNumber(math.Ceil(manifestFloat(value)/stepValue) * stepValue), nil + case "$eq", "$ne", "$gt", "$gte", "$lt", "$lte", "$in", "$and", "$or": + return evaluateManifestComparison(operator, operand, env) + case "$not": + value, err := evaluateManifestValue(operand, env) + return !manifestTruthy(value), err + default: + return nil, fmt.Errorf("unsupported manifest operator %q", operator) + } +} + +func evaluateManifestComparison(operator string, operand any, env map[string]any) (any, error) { + values, ok := operand.([]any) + if !ok { + return nil, fmt.Errorf("%s requires an array", operator) + } + evaluated := make([]any, 0, len(values)) + for _, item := range values { + value, err := evaluateManifestValue(item, env) + if err != nil { + return nil, err + } + evaluated = append(evaluated, value) + } + if operator == "$and" || operator == "$or" { + result := operator == "$and" + for _, value := range evaluated { + if operator == "$and" { + result = result && manifestTruthy(value) + } else { + result = result || manifestTruthy(value) + } + } + return result, nil + } + if len(evaluated) != 2 { + return nil, fmt.Errorf("%s requires two operands", operator) + } + left, right := evaluated[0], evaluated[1] + switch operator { + case "$eq", "$ne": + leftJSON, _ := json.Marshal(left) + rightJSON, _ := json.Marshal(right) + equal := string(leftJSON) == string(rightJSON) + if operator == "$ne" { + return !equal, nil + } + return equal, nil + case "$gt": + return manifestFloat(left) > manifestFloat(right), nil + case "$gte": + return manifestFloat(left) >= manifestFloat(right), nil + case "$lt": + return manifestFloat(left) < manifestFloat(right), nil + case "$lte": + return manifestFloat(left) <= manifestFloat(right), nil + case "$in": + for _, item := range manifestArray(right) { + leftJSON, _ := json.Marshal(left) + itemJSON, _ := json.Marshal(item) + if string(leftJSON) == string(itemJSON) { + return true, nil + } + } + return false, nil + } + return false, nil +} + +func interpolateManifestString(value string, env map[string]any) string { + result := value + for { + start := strings.Index(result, "{{") + if start < 0 { + return result + } + endOffset := strings.Index(result[start+2:], "}}") + if endOffset < 0 { + return result + } + end := start + 2 + endOffset + path := strings.TrimSpace(result[start+2 : end]) + result = result[:start] + manifestString(manifestPathValue(env, path)) + result[end+2:] + } +} + +func manifestPathValue(root any, path string) any { + if strings.TrimSpace(path) == "" { + return root + } + value := root + for _, part := range strings.Split(strings.Trim(path, "."), ".") { + switch current := value.(type) { + case map[string]any: + value = current[part] + case []any: + index, err := strconv.Atoi(part) + if err != nil || index < 0 || index >= len(current) { + return nil + } + value = current[index] + default: + return nil + } + } + return value +} + +func cloneManifestEnv(env map[string]any) map[string]any { + result := make(map[string]any, len(env)+2) + for key, value := range env { + result[key] = value + } + return result +} + +func manifestArray(value any) []any { + if value == nil { + return nil + } + if items, ok := value.([]any); ok { + return items + } + return []any{value} +} + +func manifestObject(value any) map[string]any { + result, _ := value.(map[string]any) + return result +} + +func manifestEmpty(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + return strings.TrimSpace(typed) == "" + case []any: + return len(typed) == 0 + case map[string]any: + return len(typed) == 0 + default: + return false + } +} + +func manifestTruthy(value any) bool { + switch typed := value.(type) { + case nil: + return false + case bool: + return typed + case string: + normalized := strings.ToLower(strings.TrimSpace(typed)) + return normalized != "" && normalized != "false" && normalized != "0" && normalized != "no" && normalized != "null" + case float64: + return typed != 0 + case float32: + return typed != 0 + case int: + return typed != 0 + case int64: + return typed != 0 + case []any: + return len(typed) > 0 + case map[string]any: + return len(typed) > 0 + default: + return true + } +} + +func manifestString(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case json.Number: + return typed.String() + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(typed), 'f', -1, 32) + case int: + return strconv.Itoa(typed) + case int64: + return strconv.FormatInt(typed, 10) + case bool: + return strconv.FormatBool(typed) + default: + encoded, _ := json.Marshal(value) + return string(encoded) + } +} + +func manifestFloat(value any) float64 { + switch typed := value.(type) { + case float64: + return typed + case float32: + return float64(typed) + case int: + return float64(typed) + case int64: + return float64(typed) + case json.Number: + result, _ := typed.Float64() + return result + case string: + result, _ := strconv.ParseFloat(strings.TrimSpace(typed), 64) + return result + default: + return 0 + } +} + +func manifestInt(value any) int { + return int(manifestFloat(value)) +} + +func normalizeManifestNumber(value float64) any { + if math.Trunc(value) == value { + return int(value) + } + return value +} diff --git a/backend/internal/protocol/manifest.go b/backend/internal/protocol/manifest.go index d52f9403f..f0e5906c5 100644 --- a/backend/internal/protocol/manifest.go +++ b/backend/internal/protocol/manifest.go @@ -2,6 +2,7 @@ package protocol import ( "context" + "crypto/sha256" "encoding/json" "fmt" "net/http" @@ -12,10 +13,24 @@ import ( ) type ManifestOperation struct { - Method string `json:"method"` - Path string `json:"path"` - ContentType string `json:"contentType,omitempty"` - Fields map[string]string `json:"fields,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + PathTemplate any `json:"pathTemplate,omitempty"` + OriginPath bool `json:"originPath,omitempty"` + ContentType string `json:"contentType,omitempty"` + ContentTypeTemplate any `json:"contentTypeTemplate,omitempty"` + Headers map[string]any `json:"headers,omitempty"` + Query map[string]any `json:"query,omitempty"` + Body any `json:"body,omitempty"` + Fields map[string]string `json:"fields,omitempty"` + Files []ManifestFilePart `json:"files,omitempty"` +} + +type ManifestFilePart struct { + Name string `json:"name"` + Source any `json:"source"` + Filename any `json:"filename,omitempty"` + MIMEType any `json:"mimeType,omitempty"` } type ManifestResponse struct { @@ -29,18 +44,28 @@ type ManifestResponse struct { ResultPaths []string `json:"resultPaths,omitempty"` ResultKind string `json:"resultKind,omitempty"` ResultEphemeral bool `json:"resultEphemeral,omitempty"` + TaskID any `json:"taskId,omitempty"` + Status any `json:"status,omitempty"` + Message any `json:"message,omitempty"` + Text any `json:"text,omitempty"` + Reasoning any `json:"reasoning,omitempty"` + Images any `json:"images,omitempty"` + Videos any `json:"videos,omitempty"` + Audios any `json:"audios,omitempty"` + Usage any `json:"usage,omitempty"` } // ManifestAgentResponse describes the provider response shape for a // tool-capable text request. Tool call paths are resolved against each item in // ToolCallsPath. type ManifestAgentResponse struct { - TextPaths []string `json:"textPaths,omitempty"` - ReasoningPaths []string `json:"reasoningPaths,omitempty"` - ToolCallsPath string `json:"toolCallsPath,omitempty"` - ToolCallIDPaths []string `json:"toolCallIdPaths,omitempty"` - ToolCallNamePaths []string `json:"toolCallNamePaths,omitempty"` - ToolCallArgsPaths []string `json:"toolCallArgumentsPaths,omitempty"` + TextPaths []string `json:"textPaths,omitempty"` + ReasoningPaths []string `json:"reasoningPaths,omitempty"` + ToolCallsPath string `json:"toolCallsPath,omitempty"` + ToolCallIDPaths []string `json:"toolCallIdPaths,omitempty"` + ToolCallNamePaths []string `json:"toolCallNamePaths,omitempty"` + ToolCallArgsPaths []string `json:"toolCallArgumentsPaths,omitempty"` + ToolCallSignaturePaths []string `json:"toolCallThoughtSignaturePaths,omitempty"` } // AdapterResolver is used by the host to bind a shipped execution engine to a @@ -133,6 +158,14 @@ type metadataAdapter struct { } func (a metadataAdapter) Metadata() Metadata { return a.metadata } +func (a metadataAdapter) AgentAvailable() bool { + capability, ok := a.delegate.(AgentCapability) + return ok && capability.AgentAvailable() +} +func (a metadataAdapter) ResultAvailable() bool { + capability, ok := a.delegate.(ResultCapability) + return ok && capability.ResultAvailable() +} func (a metadataAdapter) BuildCreate(ctx context.Context, c RequestContext) (RequestSpec, error) { return a.delegate.BuildCreate(ctx, c) } @@ -148,6 +181,13 @@ func (a metadataAdapter) ParsePoll(ctx context.Context, c PollContext, body []by func (a metadataAdapter) BuildCancel(ctx context.Context, c PollContext) (RequestSpec, error) { return a.delegate.BuildCancel(ctx, c) } +func (a metadataAdapter) BuildResult(ctx context.Context, c PollContext) (RequestSpec, error) { + adapter, ok := a.delegate.(ResultAdapter) + if !ok { + return RequestSpec{}, unavailable(a.metadata) + } + return adapter.BuildResult(ctx, c) +} func (a metadataAdapter) BuildAgent(ctx context.Context, c AgentRequestContext) (RequestSpec, error) { adapter, ok := a.delegate.(AgentAdapter) if !ok { @@ -199,7 +239,7 @@ func loadDeclarativeManifestProvider(manifest Manifest, index int) (Adapter, err } func ValidateManifest(manifest Manifest) error { - if strings.TrimSpace(manifest.APIVersion) != "yingce.plugin/v1" { + if version := strings.TrimSpace(manifest.APIVersion); version != "yingce.plugin/v1" && version != "yingce.plugin/v2" { return fmt.Errorf("unsupported protocol manifest apiVersion %q", manifest.APIVersion) } if strings.TrimSpace(manifest.Metadata.ID) == "" || strings.TrimSpace(manifest.Metadata.Version) == "" { @@ -220,44 +260,60 @@ func ValidateManifest(manifest Manifest) error { if len(manifest.Contributes.Providers) == 0 { return nil } - provider := manifest.Contributes.Providers[0] - if strings.TrimSpace(provider.ID) == "" || strings.TrimSpace(provider.Label) == "" { - return fmt.Errorf("provider contribution requires id and label") - } - if len(provider.Capabilities) == 0 || len(provider.Scopes) == 0 { - return fmt.Errorf("provider contribution requires capabilities and scopes") - } - for _, capability := range provider.Capabilities { - if capability != CapabilityText && capability != CapabilityImage && capability != CapabilityVideo && capability != CapabilityAudio { - return fmt.Errorf("unsupported protocol capability %q", capability) + providerIDs := make(map[string]struct{}, len(manifest.Contributes.Providers)) + for index, provider := range manifest.Contributes.Providers { + if strings.TrimSpace(provider.ID) == "" || strings.TrimSpace(provider.Label) == "" || !validManifestIdentifier(provider.ID) { + return fmt.Errorf("provider contribution %d requires a valid id and label", index) } - } - for _, scope := range provider.Scopes { - switch scope { - case SurfaceAdminSystemChannel, SurfaceUserCustomChannel, SurfaceCanvas, SurfaceCreation, SurfaceAgent: - default: - return fmt.Errorf("unsupported protocol scope %q", scope) + if _, exists := providerIDs[provider.ID]; exists { + return fmt.Errorf("duplicate provider contribution %q", provider.ID) } - } - if err := validateManifestOperation(provider.Create); err != nil { - return fmt.Errorf("create operation: %w", err) - } - if provider.Agent != nil { - if err := validateManifestOperation(*provider.Agent); err != nil { - return fmt.Errorf("agent operation: %w", err) + providerIDs[provider.ID] = struct{}{} + if len(provider.Capabilities) == 0 || len(provider.Scopes) == 0 { + return fmt.Errorf("provider contribution %q requires capabilities and scopes", provider.ID) } - if provider.AgentResponse == nil { - return fmt.Errorf("agent response mapping is required when agent operation is declared") + for _, capability := range provider.Capabilities { + if capability != CapabilityText && capability != CapabilityImage && capability != CapabilityVideo && capability != CapabilityAudio { + return fmt.Errorf("unsupported protocol capability %q", capability) + } } - } - if provider.Poll != nil { - if err := validateManifestOperation(*provider.Poll); err != nil { - return fmt.Errorf("poll operation: %w", err) + for _, scope := range provider.Scopes { + switch scope { + case SurfaceAdminSystemChannel, SurfaceUserCustomChannel, SurfaceCanvas, SurfaceCreation, SurfaceAgent: + default: + return fmt.Errorf("unsupported protocol scope %q", scope) + } } - } - if provider.Cancel != nil { - if err := validateManifestOperation(*provider.Cancel); err != nil { - return fmt.Errorf("cancel operation: %w", err) + if err := validateManifestOperation(provider.Create); err != nil { + return fmt.Errorf("provider %q create operation: %w", provider.ID, err) + } + if provider.Agent != nil { + if err := validateManifestOperation(*provider.Agent); err != nil { + return fmt.Errorf("provider %q agent operation: %w", provider.ID, err) + } + if provider.AgentResponse == nil { + return fmt.Errorf("provider %q agent response mapping is required when agent operation is declared", provider.ID) + } + } + if provider.Poll != nil { + if err := validateManifestOperation(*provider.Poll); err != nil { + return fmt.Errorf("provider %q poll operation: %w", provider.ID, err) + } + } + if provider.Cancel != nil { + if err := validateManifestOperation(*provider.Cancel); err != nil { + return fmt.Errorf("provider %q cancel operation: %w", provider.ID, err) + } + } + if provider.Result != nil { + if err := validateManifestOperation(*provider.Result); err != nil { + return fmt.Errorf("provider %q result operation: %w", provider.ID, err) + } + } + for ruleIndex, rule := range provider.Validations { + if rule.Assert == nil || strings.TrimSpace(rule.Message) == "" { + return fmt.Errorf("provider %q validation %d requires assert and message", provider.ID, ruleIndex) + } } } return nil @@ -293,8 +349,11 @@ func normalizeManifestForProvider(manifest *Manifest, index int) error { manifest.Agent = provider.Agent manifest.Poll = provider.Poll manifest.Cancel = provider.Cancel + manifest.ResultOperation = provider.Result manifest.Response = provider.Response manifest.AgentResponse = provider.AgentResponse + manifest.Auth = provider.Auth + manifest.Validations = provider.Validations return nil } @@ -320,27 +379,48 @@ func validateManifestOperation(operation ManifestOperation) error { if method != http.MethodGet && method != http.MethodPost && method != http.MethodDelete && method != http.MethodPut { return fmt.Errorf("unsupported HTTP method %q", operation.Method) } - if !isRelativePath(operation.Path) { + if operation.PathTemplate == nil && !isRelativePath(operation.Path) { return fmt.Errorf("path must be relative: %q", operation.Path) } + contentType := strings.ToLower(strings.TrimSpace(strings.Split(defaultValue(operation.ContentType, "application/json"), ";")[0])) + switch contentType { + case "application/json", "multipart/form-data", "application/x-www-form-urlencoded", "application/octet-stream", "": + default: + return fmt.Errorf("unsupported content type %q", operation.ContentType) + } + if len(operation.Files) > 0 && operation.ContentTypeTemplate == nil && contentType != "multipart/form-data" { + return fmt.Errorf("file parts require multipart/form-data") + } + for _, file := range operation.Files { + if strings.TrimSpace(file.Name) == "" || file.Source == nil { + return fmt.Errorf("multipart file part requires name and source") + } + } return nil } type manifestAdapter struct{ manifest Manifest } func (a manifestAdapter) Metadata() Metadata { return a.manifest.Metadata } +func (a manifestAdapter) AgentAvailable() bool { + return a.manifest.Agent != nil && a.manifest.AgentResponse != nil +} +func (a manifestAdapter) ResultAvailable() bool { return a.manifest.ResultOperation != nil } func (a manifestAdapter) BuildCreate(_ context.Context, c RequestContext) (RequestSpec, error) { if len(a.manifest.Contributes.Providers) == 0 { return RequestSpec{}, fmt.Errorf("plugin %s does not provide a provider", a.manifest.Metadata.ID) } - return buildManifestOperation(a.manifest.Create, c.Request, ""), nil + if err := validateManifestRequest(a.manifest.Validations, c.Request); err != nil { + return RequestSpec{}, err + } + return buildManifestOperation(a.manifest.Create, a.manifest.Auth, c.Request, "") } func (a manifestAdapter) BuildAgent(_ context.Context, c AgentRequestContext) (RequestSpec, error) { if a.manifest.Agent == nil { return RequestSpec{}, fmt.Errorf("protocol %s has no agent operation", a.manifest.Metadata.ID) } request := GenerationRequest{Model: c.Model, Extra: map[string]any{"agent": c.Request}} - return buildManifestOperation(*a.manifest.Agent, request, ""), nil + return buildManifestOperation(*a.manifest.Agent, a.manifest.Auth, request, "") } func (a manifestAdapter) ParseAgent(_ context.Context, body []byte) (AgentResult, error) { if a.manifest.AgentResponse == nil { @@ -355,19 +435,31 @@ func (a manifestAdapter) ParseAgent(_ context.Context, body []byte) (AgentResult if response.ToolCallsPath == "" { return result, nil } - for _, item := range arrayValue(pathValue(payload, response.ToolCallsPath)) { + for index, item := range arrayValue(pathValue(payload, response.ToolCallsPath)) { object, ok := item.(map[string]any) if !ok { continue } - result.ToolCalls = append(result.ToolCalls, AgentToolCall{ - ID: firstPathValue(object, response.ToolCallIDPaths...), - Name: firstPathValue(object, response.ToolCallNamePaths...), - Arguments: firstPathJSONValue(object, response.ToolCallArgsPaths...), - }) + call := AgentToolCall{ + ID: firstPathValue(object, response.ToolCallIDPaths...), + Name: firstPathValue(object, response.ToolCallNamePaths...), + Arguments: firstPathJSONValue(object, response.ToolCallArgsPaths...), + ThoughtSignature: firstPathValue(object, response.ToolCallSignaturePaths...), + } + if strings.TrimSpace(call.Name) != "" { + if strings.TrimSpace(call.ID) == "" { + call.ID = syntheticAgentToolCallID(body, index) + } + result.ToolCalls = append(result.ToolCalls, call) + } } return result, nil } + +func syntheticAgentToolCallID(body []byte, index int) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d", body, index))) + return fmt.Sprintf("call_%x", sum[:8]) +} func (a manifestAdapter) ParseCreate(_ context.Context, body []byte) (CreateResult, error) { payload, err := decodeObject(body) if err != nil { @@ -381,7 +473,7 @@ func (a manifestAdapter) BuildPoll(_ context.Context, c PollContext) (RequestSpe } request := c.Request request.Model = c.Model - return buildManifestOperation(*a.manifest.Poll, request, c.TaskID), nil + return buildManifestOperation(*a.manifest.Poll, a.manifest.Auth, request, c.TaskID) } func (a manifestAdapter) ParsePoll(_ context.Context, c PollContext, body []byte) (PollResult, error) { payload, err := decodeObject(body) @@ -397,25 +489,60 @@ func (a manifestAdapter) BuildCancel(_ context.Context, c PollContext) (RequestS } request := c.Request request.Model = c.Model - return buildManifestOperation(*a.manifest.Cancel, request, c.TaskID), nil + return buildManifestOperation(*a.manifest.Cancel, a.manifest.Auth, request, c.TaskID) +} + +func (a manifestAdapter) BuildResult(_ context.Context, c PollContext) (RequestSpec, error) { + if a.manifest.ResultOperation == nil { + return RequestSpec{}, fmt.Errorf("protocol %s has no result operation", a.manifest.Metadata.ID) + } + request := c.Request + request.Model = c.Model + return buildManifestOperation(*a.manifest.ResultOperation, a.manifest.Auth, request, c.TaskID) } func (a manifestAdapter) parse(payload map[string]any, c PollContext) CreateResult { - id := firstPathValue(payload, a.manifest.Response.TaskIDPaths...) + response := a.manifest.Response + env := map[string]any{"response": payload, "taskId": c.TaskID, "request": manifestRequestValues(c.Request)} + id := manifestResponseString(response.TaskID, env) + if id == "" { + id = firstPathValue(payload, response.TaskIDPaths...) + } if id == "" { id = c.TaskID } - status := normalizeStatus(firstPathValue(payload, a.manifest.Response.StatusPaths...)) + statusText := manifestResponseString(response.Status, env) + if statusText == "" { + statusText = firstPathValue(payload, response.StatusPaths...) + } + status := normalizeStatus(statusText) if status == "" { status = StatusPending } - message := firstPathValue(payload, a.manifest.Response.MessagePaths...) - if manifestError(payload, a.manifest.Response.ErrorPaths...) { + message := manifestResponseString(response.Message, env) + if message == "" { + message = firstPathValue(payload, response.MessagePaths...) + } + if manifestError(payload, response.ErrorPaths...) { status = StatusFailed } result := &Result{ - Text: firstPathValue(payload, a.manifest.Response.TextPaths...), - Reasoning: firstPathValue(payload, a.manifest.Response.ReasoningPaths...), + Text: manifestResponseString(response.Text, env), + Reasoning: manifestResponseString(response.Reasoning, env), + } + if result.Text == "" { + result.Text = firstPathValue(payload, response.TextPaths...) + } + if result.Reasoning == "" { + result.Reasoning = firstPathValue(payload, response.ReasoningPaths...) + } + result.Images = manifestResponseMedia(response.Images, env, "image", response.ResultEphemeral) + result.Videos = manifestResponseMedia(response.Videos, env, "video", response.ResultEphemeral) + result.Audios = manifestResponseMedia(response.Audios, env, "audio", response.ResultEphemeral) + if response.Usage != nil { + if value, err := evaluateManifestValue(response.Usage, env); err == nil { + result.Usage = manifestObject(value) + } } paths := append([]string{}, a.manifest.Response.ResultURLPaths...) paths = append(paths, a.manifest.Response.ResultPaths...) @@ -441,6 +568,88 @@ func (a manifestAdapter) parse(payload map[string]any, c PollContext) CreateResu return CreateResult{TaskID: id, Status: status, Result: result, Message: message} } +func validateManifestRequest(rules []ManifestValidation, request GenerationRequest) error { + if len(rules) == 0 { + return nil + } + env := map[string]any{"request": manifestRequestValues(request)} + for _, rule := range rules { + value, err := evaluateManifestValue(rule.Assert, env) + if err != nil { + return fmt.Errorf("协议参数校验表达式错误：%w", err) + } + if !manifestTruthy(value) { + return fmt.Errorf("%s", strings.TrimSpace(rule.Message)) + } + } + return nil +} + +func manifestResponseString(template any, env map[string]any) string { + if template == nil { + return "" + } + value, err := evaluateManifestValue(template, env) + if err != nil { + return "" + } + items := manifestArray(value) + parts := make([]string, 0, len(items)) + for _, item := range items { + if text := strings.TrimSpace(manifestString(item)); text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, "") +} + +func manifestResponseMedia(template any, env map[string]any, kind string, ephemeral bool) []MediaReference { + if template == nil { + return nil + } + value, err := evaluateManifestValue(template, env) + if err != nil { + return nil + } + return mediaReferencesFromManifestValue(value, kind, ephemeral) +} + +func mediaReferencesFromManifestValue(value any, kind string, ephemeral bool) []MediaReference { + result := make([]MediaReference, 0) + for _, item := range manifestArray(value) { + switch typed := item.(type) { + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" { + continue + } + reference := MediaReference{Kind: kind, Ephemeral: ephemeral} + if strings.HasPrefix(trimmed, "data:") { + reference.DataURL = trimmed + } else { + reference.URL = trimmed + } + result = append(result, reference) + case map[string]any: + reference := MediaReference{ + ID: manifestString(typed["id"]), URL: manifestString(typed["url"]), DataURL: manifestString(typed["dataUrl"]), + Kind: defaultValue(manifestString(typed["kind"]), kind), Role: manifestString(typed["role"]), MIMEType: manifestString(typed["mimeType"]), + Name: manifestString(typed["name"]), Order: manifestInt(typed["order"]), Weight: manifestFloat(typed["weight"]), Ephemeral: ephemeral || manifestTruthy(typed["ephemeral"]), + } + if reference.URL == "" { + reference.URL = firstString(typed, "file_url", "fileUrl", "image_url", "imageUrl", "video_url", "videoUrl", "audio_url", "audioUrl", "uri") + } + if reference.DataURL == "" { + reference.DataURL = firstString(typed, "data_url", "b64_json") + } + if reference.URL != "" || reference.DataURL != "" { + result = append(result, reference) + } + } + } + return result +} + var ( manifestMDImageRegex = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`) manifestHTMLVideoRegex = regexp.MustCompile(`(?i)]+src=['"]([^'"]+)['"]`) @@ -494,22 +703,135 @@ func mediaPathValues(payload map[string]any, path string) []string { return values } -func buildManifestOperation(operation ManifestOperation, request GenerationRequest, taskID string) RequestSpec { - var body map[string]any - if len(operation.Fields) > 0 { - body = make(map[string]any, len(operation.Fields)) - } - for key, expression := range operation.Fields { - value := manifestExpressionValue(expression, request, taskID) - if value == nil { - continue +func buildManifestOperation(operation ManifestOperation, auth ManifestAuth, request GenerationRequest, taskID string) (RequestSpec, error) { + requestValues := manifestRequestValues(request) + env := map[string]any{"request": requestValues, "taskId": taskID} + var body any + if operation.Body != nil { + value, err := evaluateManifestValue(operation.Body, env) + if err != nil { + return RequestSpec{}, fmt.Errorf("evaluate request body: %w", err) } - setMapPath(body, key, value) + body = normalizeManifestValue(value) + } else if len(operation.Fields) > 0 { + legacyBody := make(map[string]any, len(operation.Fields)) + for key, expression := range operation.Fields { + value := manifestExpressionValue(expression, request, taskID) + if value == nil { + continue + } + setMapPath(legacyBody, key, value) + } + body = normalizeManifestValue(legacyBody) } - normalizedBody := normalizeManifestValue(body).(map[string]any) - path := strings.ReplaceAll(operation.Path, "{{taskId}}", url.PathEscape(taskID)) + pathValue := any(operation.Path) + if operation.PathTemplate != nil { + pathValue = operation.PathTemplate + } + evaluatedPath, err := evaluateManifestValue(pathValue, env) + if err != nil { + return RequestSpec{}, fmt.Errorf("evaluate request path: %w", err) + } + path := strings.ReplaceAll(manifestString(evaluatedPath), "{{taskId}}", url.PathEscape(taskID)) path = strings.ReplaceAll(path, "{{model}}", url.PathEscape(request.Model)) - return RequestSpec{Method: strings.ToUpper(operation.Method), Path: path, ContentType: defaultValue(operation.ContentType, "application/json"), Body: normalizedBody} + path = interpolateManifestString(path, env) + if !isRelativePath(path) { + return RequestSpec{}, fmt.Errorf("evaluated request path must be relative: %q", path) + } + headers, err := evaluateManifestStringMap(operation.Headers, env) + if err != nil { + return RequestSpec{}, fmt.Errorf("evaluate request headers: %w", err) + } + query, err := evaluateManifestQuery(operation.Query, env) + if err != nil { + return RequestSpec{}, fmt.Errorf("evaluate request query: %w", err) + } + files, err := evaluateManifestFiles(operation.Files, env) + if err != nil { + return RequestSpec{}, err + } + contentTypeValue := any(operation.ContentType) + if operation.ContentTypeTemplate != nil { + contentTypeValue = operation.ContentTypeTemplate + } + evaluatedContentType, err := evaluateManifestValue(contentTypeValue, env) + if err != nil { + return RequestSpec{}, fmt.Errorf("evaluate request content type: %w", err) + } + contentType := defaultValue(manifestString(evaluatedContentType), "application/json") + return RequestSpec{Method: strings.ToUpper(operation.Method), Path: path, OriginPath: operation.OriginPath, ContentType: contentType, Headers: headers, Query: query, Body: body, Files: files, Auth: auth}, nil +} + +func evaluateManifestStringMap(values map[string]any, env map[string]any) (map[string]string, error) { + if len(values) == 0 { + return nil, nil + } + result := make(map[string]string, len(values)) + for key, template := range values { + value, err := evaluateManifestValue(template, env) + if err != nil { + return nil, err + } + if text := strings.TrimSpace(manifestString(value)); text != "" { + result[key] = text + } + } + return result, nil +} + +func evaluateManifestQuery(values map[string]any, env map[string]any) (map[string][]string, error) { + if len(values) == 0 { + return nil, nil + } + result := make(map[string][]string, len(values)) + for key, template := range values { + value, err := evaluateManifestValue(template, env) + if err != nil { + return nil, err + } + for _, item := range manifestArray(value) { + if text := strings.TrimSpace(manifestString(item)); text != "" { + result[key] = append(result[key], text) + } + } + } + return result, nil +} + +func evaluateManifestFiles(parts []ManifestFilePart, env map[string]any) ([]RequestFilePart, error) { + result := make([]RequestFilePart, 0) + for _, part := range parts { + source, err := evaluateManifestValue(part.Source, env) + if err != nil { + return nil, fmt.Errorf("evaluate multipart file %q: %w", part.Name, err) + } + filename, err := evaluateManifestValue(part.Filename, env) + if err != nil { + return nil, fmt.Errorf("evaluate multipart filename %q: %w", part.Name, err) + } + mimeType, err := evaluateManifestValue(part.MIMEType, env) + if err != nil { + return nil, fmt.Errorf("evaluate multipart MIME type %q: %w", part.Name, err) + } + for index, value := range manifestArray(source) { + references := mediaReferencesFromManifestValue(value, "file", false) + for _, reference := range references { + name := strings.TrimSpace(manifestString(filename)) + if name == "" { + name = reference.Name + } + if name == "" { + name = fmt.Sprintf("%s-%d", part.Name, index+1) + } + contentType := strings.TrimSpace(manifestString(mimeType)) + if contentType == "" { + contentType = reference.MIMEType + } + result = append(result, RequestFilePart{Name: part.Name, Filename: name, MIMEType: contentType, Reference: reference}) + } + } + } + return result, nil } func normalizeManifestValue(value any) any { @@ -601,31 +923,81 @@ func manifestRequestValues(request GenerationRequest) map[string]any { userContent = append(userContent, map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": val}}) } } - messages := make([]any, 0) + messages := make([]any, 0, len(request.Messages)+1) + for _, message := range request.Messages { + if strings.TrimSpace(message.Role) == "" || message.Content == nil { + continue + } + messages = append(messages, map[string]any{"role": message.Role, "content": message.Content}) + } if len(userContent) > 0 { - messages = append(messages, map[string]any{ - "role": "user", - "content": userContent, - }) + content := any(userContent) + if len(userContent) == 1 && len(request.Images)+len(request.Videos)+len(request.Audios) == 0 { + content = request.Prompt + } + messages = append(messages, map[string]any{"role": "user", "content": content}) } - return map[string]any{ - "model": request.Model, - "prompt": request.Prompt, - "messages": messages, - "images": manifestMediaValues(request.Images), - "videos": manifestMediaValues(request.Videos), - "audios": manifestMediaValues(request.Audios), - "imageCount": request.ImageCount, - "duration": request.Duration, - "aspectRatio": request.AspectRatio, - "resolution": request.Resolution, - "quality": request.Quality, - "generateAudio": request.GenerateAudio, - "watermark": request.Watermark, - "operation": request.Operation, - "extra": request.Extra, + inputs := request.Inputs + if len(inputs) == 0 { + inputs = append(inputs, request.Images...) + inputs = append(inputs, request.Videos...) + inputs = append(inputs, request.Audios...) } + output := request.Output + if output.Count == 0 { + output.Count = request.ImageCount + } + if output.Duration == 0 { + output.Duration = request.Duration + } + if output.AspectRatio == "" { + output.AspectRatio = request.AspectRatio + } + if output.Resolution == "" { + output.Resolution = request.Resolution + } + if output.Quality == "" { + output.Quality = request.Quality + } + output.GenerateAudio = output.GenerateAudio || request.GenerateAudio + output.Watermark = output.Watermark || request.Watermark + outputValue, _ := requestAsManifestValue(output) + + return map[string]any{ + "capability": request.Capability, + "model": request.Model, + "prompt": request.Prompt, + "instructions": request.Instructions, + "messages": messages, + "inputs": manifestMediaValues(inputs), + "images": manifestMediaValues(request.Images), + "videos": manifestMediaValues(request.Videos), + "audios": manifestMediaValues(request.Audios), + "imageCount": request.ImageCount, + "duration": request.Duration, + "aspectRatio": request.AspectRatio, + "resolution": request.Resolution, + "quality": request.Quality, + "generateAudio": request.GenerateAudio, + "watermark": request.Watermark, + "operation": request.Operation, + "output": outputValue, + "providerOptions": request.ProviderOptions, + "extra": request.Extra, + } +} + +func requestAsManifestValue(value any) (any, error) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, err + } + var result any + if err := json.Unmarshal(encoded, &result); err != nil { + return nil, err + } + return result, nil } func manifestModelID(request GenerationRequest) string { @@ -638,17 +1010,45 @@ func manifestModelID(request GenerationRequest) string { func manifestMediaValues(values []MediaReference) []any { result := make([]any, 0, len(values)) - for _, value := range values { + for index, value := range values { + order := value.Order + if order == 0 && index > 0 { + order = index + } + resolved := defaultValue(value.URL, value.DataURL) result = append(result, map[string]any{ - "url": value.URL, - "dataUrl": value.DataURL, - "kind": value.Kind, + "id": value.ID, + "url": value.URL, + "dataUrl": value.DataURL, + "value": resolved, + "kind": value.Kind, + "role": value.Role, + "mimeType": value.MIMEType, + "name": value.Name, + "order": order, + "weight": value.Weight, + "metadata": value.Metadata, + "source": map[string]any{ + "type": manifestMediaSourceType(value), + "value": resolved, + "mimeType": value.MIMEType, + }, "ephemeral": value.Ephemeral, }) } return result } +func manifestMediaSourceType(value MediaReference) string { + if value.DataURL != "" { + return "data" + } + if value.URL != "" { + return "url" + } + return "unknown" +} + func applyManifestTransform(value any, transform string, request GenerationRequest) any { transform = strings.ToLower(strings.TrimSpace(transform)) if transform == "bool" || transform == "boolean" { @@ -771,22 +1171,7 @@ func normalizeLegacyManifestVideoResolution(value string) string { } func pathValue(payload map[string]any, path string) any { - var value any = payload - for _, part := range strings.Split(strings.Trim(path, "."), ".") { - switch current := value.(type) { - case map[string]any: - value = current[part] - case []any: - index, err := strconv.Atoi(part) - if err != nil || index < 0 || index >= len(current) { - return nil - } - value = current[index] - default: - return nil - } - } - return value + return manifestPathValue(payload, path) } func arrayValue(value any) []any { diff --git a/backend/internal/protocol/official_catalog_test.go b/backend/internal/protocol/official_catalog_test.go new file mode 100644 index 000000000..68c8c04c5 --- /dev/null +++ b/backend/internal/protocol/official_catalog_test.go @@ -0,0 +1,428 @@ +package protocol + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const ( + manifestContractStart = "" + manifestContractEnd = "" + manifestDocumentationPlaceholder = "<当前插件的完整 documentation，由 README.md 与 docs/interface.md 拼接而成；为避免 JSON 递归，此处不重复展开正文。>" +) + +func TestOfficialProtocolPackagesAreSelfContainedDeclarativePlugins(t *testing.T) { + paths, err := filepath.Glob(filepath.Join("..", "..", "..", "plugin-packages", "*.yingce-plugin")) + if err != nil { + t.Fatal(err) + } + if len(paths) < 70 { + t.Fatalf("official protocol packages = %d, want at least 70", len(paths)) + } + providerOwners := map[string]string{} + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + pkg, err := ParsePluginPackage(data) + if err != nil { + t.Fatalf("parse %s: %v", filepath.Base(path), err) + } + if pkg.Manifest.APIVersion != "yingce.plugin/v2" { + t.Fatalf("%s apiVersion = %q", filepath.Base(path), pkg.Manifest.APIVersion) + } + if strings.HasPrefix(strings.TrimSpace(pkg.Manifest.Runtime.Backend), "host:") { + t.Fatalf("%s depends on host runtime", filepath.Base(path)) + } + if len(pkg.Files["README.md"]) == 0 || len(pkg.Files["docs/interface.md"]) == 0 { + t.Fatalf("%s must contain README.md and docs/interface.md", filepath.Base(path)) + } + interfaceDocs := string(pkg.Files["docs/interface.md"]) + expectedDocumentation := strings.TrimSpace(string(pkg.Files["README.md"])) + "\n\n---\n\n" + strings.TrimSpace(interfaceDocs) + if strings.TrimSpace(pkg.Manifest.Metadata.Documentation) != expectedDocumentation { + t.Fatalf("%s manifest.documentation does not contain the packaged README and interface document", filepath.Base(path)) + } + assertManifestContractMatchesPackage(t, filepath.Base(path), pkg.ManifestRaw, interfaceDocs) + if !strings.Contains(interfaceDocs, "响应") { + t.Fatalf("%s interface documentation must describe response mapping", filepath.Base(path)) + } + for _, field := range pkg.Manifest.Configuration.Fields { + if !strings.Contains(interfaceDocs, field.Name) { + t.Fatalf("%s interface documentation is missing configuration field %q", filepath.Base(path), field.Name) + } + } + adapters, err := LoadInstalledProviders(pkg.ManifestRaw, nil) + if err != nil { + t.Fatalf("load %s: %v", filepath.Base(path), err) + } + if len(adapters) != len(pkg.Manifest.Contributes.Providers) { + t.Fatalf("%s adapters = %d, providers = %d", filepath.Base(path), len(adapters), len(pkg.Manifest.Contributes.Providers)) + } + for _, provider := range pkg.Manifest.Contributes.Providers { + if owner, duplicate := providerOwners[provider.ID]; duplicate { + t.Fatalf("provider id %q is shared by %s and %s", provider.ID, owner, filepath.Base(path)) + } + providerOwners[provider.ID] = filepath.Base(path) + for _, parameter := range provider.Parameters { + if strings.TrimSpace(parameter.Name) == "" || strings.TrimSpace(parameter.Type) == "" || strings.TrimSpace(parameter.Mapping) == "" || strings.TrimSpace(parameter.Description) == "" { + t.Fatalf("%s provider %s has incomplete parameter documentation: %#v", filepath.Base(path), provider.ID, parameter) + } + if !strings.Contains(interfaceDocs, parameter.Name) { + t.Fatalf("%s interface documentation is missing provider parameter %q", filepath.Base(path), parameter.Name) + } + } + } + } +} + +func assertManifestContractMatchesPackage(t *testing.T, packageName string, manifestRaw []byte, interfaceDocs string) { + t.Helper() + start := strings.Index(interfaceDocs, manifestContractStart) + end := strings.Index(interfaceDocs, manifestContractEnd) + if start < 0 || end < start { + t.Fatalf("%s interface documentation has no complete manifest contract", packageName) + } + contractSection := interfaceDocs[start:end] + jsonStart := strings.Index(contractSection, "```json\n") + if jsonStart < 0 { + t.Fatalf("%s manifest contract has no JSON block", packageName) + } + jsonStart += len("```json\n") + jsonEnd := strings.Index(contractSection[jsonStart:], "\n```") + if jsonEnd < 0 { + t.Fatalf("%s manifest contract JSON block is not terminated", packageName) + } + + var actual map[string]any + if err := json.Unmarshal(manifestRaw, &actual); err != nil { + t.Fatalf("decode %s manifest: %v", packageName, err) + } + actual["documentation"] = manifestDocumentationPlaceholder + var documented map[string]any + if err := json.Unmarshal([]byte(contractSection[jsonStart:jsonStart+jsonEnd]), &documented); err != nil { + t.Fatalf("decode %s documented manifest contract: %v", packageName, err) + } + if !reflect.DeepEqual(documented, actual) { + t.Fatalf("%s documented manifest contract does not exactly match manifest.json", packageName) + } +} + +func TestOfficialAgentProfilesMapToolRequestsAndResponses(t *testing.T) { + requests := map[string]any{ + "chatCompletion": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "inspect"}}, "tools": []any{map[string]any{"type": "function"}}, "tool_choice": "required", "marker": "chat"}, + "responses": map[string]any{"input": []any{map[string]any{"role": "user", "content": "inspect"}}, "tools": []any{map[string]any{"type": "function"}}, "tool_choice": "required", "marker": "responses"}, + "claude": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "inspect"}}, "tools": []any{map[string]any{"name": "canvas_get_state"}}, "tool_choice": map[string]any{"type": "any"}, "max_tokens": 512, "marker": "claude"}, + "gemini": map[string]any{"contents": []any{map[string]any{"role": "user", "parts": []any{map[string]any{"text": "inspect"}}}}, "tools": []any{map[string]any{"functionDeclarations": []any{}}}, "toolConfig": map[string]any{"functionCallingConfig": map[string]any{"mode": "ANY"}}, "marker": "gemini"}, + } + tests := []struct { + name, packageName, providerID, marker, wantPath, response, wantID, wantSignature string + }{ + { + name: "openai-chat", packageName: "openai-chat-completions.yingce-plugin", providerID: "chat-completion", marker: "chat", wantPath: "/v1/chat/completions", wantID: "call-chat", + response: `{"choices":[{"message":{"content":"chat answer","tool_calls":[{"id":"call-chat","function":{"name":"canvas_get_state","arguments":"{\"scope\":\"all\"}"}}]}}]}`, + }, + { + name: "openai-responses", packageName: "openai-responses.yingce-plugin", providerID: "openai-response", marker: "responses", wantPath: "/v1/responses", wantID: "call-responses", + response: `{"output_text":"responses answer","output":[{"type":"message"},{"type":"function_call","call_id":"call-responses","name":"canvas_get_state","arguments":"{\"scope\":\"all\"}"}]}`, + }, + { + name: "anthropic", packageName: "anthropic-messages.yingce-plugin", providerID: "claude-api", marker: "claude", wantPath: "/v1/messages", wantID: "call-claude", + response: `{"content":[{"type":"text","text":"claude answer"},{"type":"tool_use","id":"call-claude","name":"canvas_get_state","input":{"scope":"all"}}]}`, + }, + { + name: "gemini", packageName: "google-gemini-generate-content.yingce-plugin", providerID: "gemini-generate-content", marker: "gemini", wantPath: "/v1beta/models/gemini-test:generateContent", wantSignature: "signature-1", + response: `{"candidates":[{"content":{"parts":[{"text":"gemini answer"},{"functionCall":{"name":"canvas_get_state","args":{"scope":"all"}},"thoughtSignature":"signature-1"}]}}]}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + adapter := officialPackageAdapter(t, test.packageName, test.providerID) + capability, ok := adapter.(AgentCapability) + if !ok || !capability.AgentAvailable() { + t.Fatalf("%s does not expose its declared agent operation", test.providerID) + } + agent, ok := adapter.(AgentAdapter) + if !ok { + t.Fatalf("%s does not implement AgentAdapter", test.providerID) + } + spec, err := agent.BuildAgent(context.Background(), AgentRequestContext{Model: "gemini-test", Request: requests}) + if err != nil { + t.Fatal(err) + } + if spec.Path != test.wantPath { + t.Fatalf("request path = %q, want %q", spec.Path, test.wantPath) + } + body := manifestTestBody(t, spec) + if body["marker"] != test.marker { + t.Fatalf("request selected wrong protocol body: %#v", body) + } + if test.providerID != "gemini-generate-content" && body["model"] != "gemini-test" { + t.Fatalf("request model = %#v", body["model"]) + } + result, err := agent.ParseAgent(context.Background(), []byte(test.response)) + if err != nil { + t.Fatal(err) + } + if len(result.ToolCalls) != 1 || result.ToolCalls[0].Name != "canvas_get_state" || result.ToolCalls[0].Arguments != `{"scope":"all"}` { + t.Fatalf("agent result = %#v", result) + } + if test.wantID != "" && result.ToolCalls[0].ID != test.wantID { + t.Fatalf("tool call id = %q, want %q", result.ToolCalls[0].ID, test.wantID) + } + if test.wantID == "" && !strings.HasPrefix(result.ToolCalls[0].ID, "call_") { + t.Fatalf("provider omitted tool call id and no stable id was synthesized: %#v", result.ToolCalls[0]) + } + if result.ToolCalls[0].ThoughtSignature != test.wantSignature { + t.Fatalf("thought signature = %q, want %q", result.ToolCalls[0].ThoughtSignature, test.wantSignature) + } + }) + } +} + +func TestOfficialVideoProfilesPreserveExplicitMediaRoles(t *testing.T) { + tests := []struct { + packageName string + providerID string + model string + resolution string + assert func(t *testing.T, spec RequestSpec) + }{ + { + packageName: "minimax-hailuo-video-v2.yingce-plugin", providerID: "minimax-video", model: "MiniMax-H3", resolution: "1080P", + assert: func(t *testing.T, spec RequestSpec) { + body := manifestTestBody(t, spec) + content, _ := body["content"].([]any) + if len(content) != 3 { + t.Fatalf("MiniMax content = %#v", content) + } + first, _ := content[1].(map[string]any) + last, _ := content[2].(map[string]any) + if first["role"] != "first_frame" || last["role"] != "last_frame" || body["ratio"] != "adaptive" || body["resolution"] != "1080P" { + t.Fatalf("MiniMax role payload = %#v", body) + } + }, + }, + { + packageName: "newapi-media-task-v1.yingce-plugin", providerID: "newapi-channel-1", model: "minimax-h3", resolution: "720P", + assert: func(t *testing.T, spec RequestSpec) { + body := manifestTestBody(t, spec) + input, _ := body["input"].(map[string]any) + media, _ := input["media"].([]any) + first, _ := media[0].(map[string]any) + last, _ := media[1].(map[string]any) + if first["type"] != "first_frame" || last["type"] != "last_frame" { + t.Fatalf("NewAPI media = %#v", media) + } + }, + }, + { + packageName: "agnes-video-25.yingce-plugin", providerID: "agnes-video", model: "agnes-video-2.5", resolution: "720P", + assert: func(t *testing.T, spec RequestSpec) { + body := manifestTestBody(t, spec) + if body["mode"] != "keyframe" || body["first_frame"] != "https://cdn.example/first.png" || body["last_frame"] != "https://cdn.example/last.png" { + t.Fatalf("Agnes keyframe payload = %#v", body) + } + }, + }, + { + packageName: "dashscope-wan-video.yingce-plugin", providerID: "dashscope-wan-video", model: "wan2.2-kf2v-flash", resolution: "720P", + assert: func(t *testing.T, spec RequestSpec) { + body := manifestTestBody(t, spec) + input, _ := body["input"].(map[string]any) + if input["first_frame_url"] != "https://cdn.example/first.png" || input["last_frame_url"] != "https://cdn.example/last.png" { + t.Fatalf("Wan keyframe input = %#v", input) + } + if _, duplicated := input["img_url"]; duplicated { + t.Fatalf("Wan keyframe input must not also send img_url: %#v", input) + } + }, + }, + } + for _, test := range tests { + t.Run(test.providerID, func(t *testing.T) { + adapter := officialPackageAdapter(t, test.packageName, test.providerID) + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Model: test.model, Prompt: "镜头推进", Duration: 6, AspectRatio: "16:9", Resolution: test.resolution, + Images: []MediaReference{ + {URL: "https://cdn.example/last.png", Role: "last_frame", Order: 2}, + {URL: "https://cdn.example/first.png", Role: "first_frame", Order: 1}, + }, + }}) + if err != nil { + t.Fatal(err) + } + test.assert(t, spec) + }) + } +} + +func TestOfficialWanProfilesUseMutuallyExclusiveReferenceFields(t *testing.T) { + video := officialPackageAdapter(t, "dashscope-wan-video.yingce-plugin", "dashscope-wan-video") + videoSpec, err := video.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Model: "wan2.6-r2v", Prompt: "保持角色一致", Operation: "reference_to_video", + Images: []MediaReference{ + {URL: "https://cdn.example/second.png", Role: "subject_reference", Order: 2}, + {URL: "https://cdn.example/first.png", Role: "reference_image", Order: 1}, + }, + }}) + if err != nil { + t.Fatal(err) + } + videoBody := manifestTestBody(t, videoSpec) + videoInput, _ := videoBody["input"].(map[string]any) + if _, duplicated := videoInput["img_url"]; duplicated { + t.Fatalf("Wan reference mode must not also send img_url: %#v", videoInput) + } + references, _ := videoInput["reference_images"].([]any) + if len(references) != 2 || references[0] != "https://cdn.example/first.png" || references[1] != "https://cdn.example/second.png" { + t.Fatalf("Wan reference_images = %#v", references) + } + + image := officialPackageAdapter(t, "dashscope-wanx-image.yingce-plugin", "dashscope-wanx-image") + for _, test := range []struct { + name string + images []MediaReference + single bool + }{ + {name: "single", images: []MediaReference{{URL: "https://cdn.example/one.png", Role: "reference_image", Order: 1}}, single: true}, + {name: "multiple", images: []MediaReference{{URL: "https://cdn.example/two.png", Role: "reference_image", Order: 2}, {URL: "https://cdn.example/one.png", Role: "reference_image", Order: 1}}}, + } { + t.Run("wanx-"+test.name, func(t *testing.T) { + spec, err := image.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: "wanx2.1-t2i-turbo", Prompt: "角色设定", Images: test.images}}) + if err != nil { + t.Fatal(err) + } + body := manifestTestBody(t, spec) + input, _ := body["input"].(map[string]any) + _, hasSingle := input["ref_img"] + _, hasMultiple := input["ref_images"] + if hasSingle != test.single || hasMultiple == test.single { + t.Fatalf("Wanx reference fields are not mutually exclusive: %#v", input) + } + }) + } +} + +func TestOfficialRollDekWanVideoUsesVideosLifecycle(t *testing.T) { + adapter := officialPackageAdapter(t, "rolldek-wan-video.yingce-plugin", "rolldek-wan-video") + request := GenerationRequest{ + Model: "wan3.0-video-prime-1080p", Prompt: "保持角色一致", Duration: 11, AspectRatio: "16:9", Resolution: "1080p", + Images: []MediaReference{{URL: "https://cdn.example/character.png", Role: "reference_image"}}, + Videos: []MediaReference{{URL: "https://cdn.example/motion.mp4", Role: "reference_video", Metadata: map[string]any{"durationMs": 5250}}}, + Audios: []MediaReference{{URL: "https://cdn.example/voice.mp3", Role: "reference_audio"}}, + } + create, err := adapter.BuildCreate(context.Background(), RequestContext{Request: request}) + if err != nil { + t.Fatal(err) + } + if create.Method != "POST" || create.Path != "/v1/videos" || create.ContentType != "application/json" { + t.Fatalf("RollDek create = %#v", create) + } + body := manifestTestBody(t, create) + if body["model"] != request.Model || body["seconds"] != "11" || body["size"] != "1080P" || body["aspect_ratio"] != "16:9" { + t.Fatalf("RollDek create body = %#v", body) + } + images, _ := body["reference_images"].([]any) + image, _ := images[0].(map[string]any) + videos, _ := body["reference_videos"].([]any) + video, _ := videos[0].(map[string]any) + audios, _ := body["reference_audios"].([]any) + audio, _ := audios[0].(map[string]any) + if image["url"] != request.Images[0].URL || image["role"] != "reference_image" || video["url"] != request.Videos[0].URL || video["duration"] != 5.25 || audio["url"] != request.Audios[0].URL { + t.Fatalf("RollDek references = images:%#v videos:%#v audios:%#v", images, videos, audios) + } + + created, err := adapter.ParseCreate(context.Background(), []byte(`{"id":"task-roll-1","task_id":"task-roll-1","status":"queued"}`)) + if err != nil { + t.Fatal(err) + } + if created.TaskID != "task-roll-1" || created.Status != StatusPending { + t.Fatalf("RollDek create result = %#v", created) + } + poll, err := adapter.BuildPoll(context.Background(), PollContext{Request: request, TaskID: created.TaskID}) + if err != nil { + t.Fatal(err) + } + if poll.Method != "GET" || poll.Path != "/v1/videos/task-roll-1" { + t.Fatalf("RollDek poll = %#v", poll) + } + state, err := adapter.ParsePoll(context.Background(), PollContext{Request: request, TaskID: created.TaskID}, []byte(`{"id":"task-roll-1","status":"completed","metadata":{"url":"https://cdn.example/result.mp4"}}`)) + if err != nil { + t.Fatal(err) + } + if state.Status != StatusSucceeded || state.Result == nil || len(state.Result.Videos) != 1 || state.Result.Videos[0].URL != "https://cdn.example/result.mp4" { + t.Fatalf("RollDek poll result = %#v", state) + } + resultAdapter, ok := adapter.(ResultAdapter) + if !ok { + t.Fatal("RollDek adapter does not expose result download") + } + result, err := resultAdapter.BuildResult(context.Background(), PollContext{Request: request, TaskID: created.TaskID}) + if err != nil { + t.Fatal(err) + } + if result.Method != "GET" || result.Path != "/v1/videos/task-roll-1/content" { + t.Fatalf("RollDek result = %#v", result) + } +} + +func TestOfficialOpenAIVideosDeclaresAuthenticatedResultDownload(t *testing.T) { + adapter := officialPackageAdapter(t, "openai-videos.yingce-plugin", "newapi") + capability, ok := adapter.(ResultCapability) + if !ok || !capability.ResultAvailable() { + t.Fatal("OpenAI Videos result operation is unavailable") + } + resultAdapter, ok := adapter.(ResultAdapter) + if !ok { + t.Fatal("OpenAI Videos does not implement ResultAdapter") + } + spec, err := resultAdapter.BuildResult(context.Background(), PollContext{Model: "sora-2", TaskID: "video-1"}) + if err != nil { + t.Fatal(err) + } + if spec.Method != "GET" || spec.Path != "/v1/videos/video-1/content" || spec.Headers["Accept"] != "video/mp4" || spec.Auth.Type != "bearer" { + t.Fatalf("result request = %#v", spec) + } +} + +func officialPackageAdapter(t *testing.T, packageName, providerID string) Adapter { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "..", "plugin-packages", packageName)) + if err != nil { + t.Fatal(err) + } + pkg, err := ParsePluginPackage(data) + if err != nil { + t.Fatal(err) + } + adapters, err := LoadInstalledProviders(pkg.ManifestRaw, nil) + if err != nil { + t.Fatal(err) + } + for _, adapter := range adapters { + if adapter.Metadata().ID == providerID { + return adapter + } + } + t.Fatalf("provider %s is missing from %s", providerID, packageName) + return nil +} + +func manifestTestBody(t *testing.T, spec RequestSpec) map[string]any { + t.Helper() + data, err := json.Marshal(spec.Body) + if err != nil { + t.Fatal(err) + } + var body map[string]any + if err := json.Unmarshal(data, &body); err != nil { + t.Fatal(err) + } + return body +} diff --git a/backend/internal/protocol/types.go b/backend/internal/protocol/types.go index e97f7b64c..af3ae59a1 100644 --- a/backend/internal/protocol/types.go +++ b/backend/internal/protocol/types.go @@ -37,31 +37,65 @@ const ( ) type MediaReference struct { - ID string `json:"id,omitempty"` - URL string `json:"url,omitempty"` - DataURL string `json:"dataUrl,omitempty"` - Kind string `json:"kind,omitempty"` - Role string `json:"role,omitempty"` - Ephemeral bool `json:"ephemeral,omitempty"` + ID string `json:"id,omitempty"` + URL string `json:"url,omitempty"` + DataURL string `json:"dataUrl,omitempty"` + Kind string `json:"kind,omitempty"` + Role string `json:"role,omitempty"` + MIMEType string `json:"mimeType,omitempty"` + Name string `json:"name,omitempty"` + Order int `json:"order,omitempty"` + Weight float64 `json:"weight,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` + Ephemeral bool `json:"ephemeral,omitempty"` +} + +// OutputOptions is the provider-neutral output contract. Legacy top-level +// fields remain on GenerationRequest while callers migrate; manifests receive +// both projections and should prefer request.output for new plugins. +type OutputOptions struct { + Count int `json:"count,omitempty"` + Duration int `json:"duration,omitempty"` + AspectRatio string `json:"aspectRatio,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + Resolution string `json:"resolution,omitempty"` + Quality string `json:"quality,omitempty"` + FPS float64 `json:"fps,omitempty"` + GenerateAudio bool `json:"generateAudio,omitempty"` + Watermark bool `json:"watermark,omitempty"` + Format string `json:"format,omitempty"` + Options map[string]any `json:"options,omitempty"` +} + +type Message struct { + Role string `json:"role"` + Content any `json:"content"` } // GenerationRequest is the platform contract shared by canvas, creation and agent. // Provider-specific fields belong in Extra and are never interpreted by the host. type GenerationRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt,omitempty"` - Images []MediaReference `json:"images,omitempty"` - Videos []MediaReference `json:"videos,omitempty"` - Audios []MediaReference `json:"audios,omitempty"` - Duration int `json:"duration,omitempty"` - AspectRatio string `json:"aspectRatio,omitempty"` - Resolution string `json:"resolution,omitempty"` - Quality string `json:"quality,omitempty"` - GenerateAudio bool `json:"generateAudio,omitempty"` - Watermark bool `json:"watermark,omitempty"` - ImageCount int `json:"imageCount,omitempty"` - Operation string `json:"operation,omitempty"` - Extra map[string]any `json:"extra,omitempty"` + Capability Capability `json:"capability,omitempty"` + Model string `json:"model"` + Prompt string `json:"prompt,omitempty"` + Instructions string `json:"instructions,omitempty"` + Messages []Message `json:"messages,omitempty"` + Inputs []MediaReference `json:"inputs,omitempty"` + Images []MediaReference `json:"images,omitempty"` + Videos []MediaReference `json:"videos,omitempty"` + Audios []MediaReference `json:"audios,omitempty"` + Duration int `json:"duration,omitempty"` + AspectRatio string `json:"aspectRatio,omitempty"` + Resolution string `json:"resolution,omitempty"` + Quality string `json:"quality,omitempty"` + GenerateAudio bool `json:"generateAudio,omitempty"` + Watermark bool `json:"watermark,omitempty"` + ImageCount int `json:"imageCount,omitempty"` + Operation string `json:"operation,omitempty"` + Output OutputOptions `json:"output,omitempty"` + ProviderOptions map[string]map[string]any `json:"providerOptions,omitempty"` + Extra map[string]any `json:"extra,omitempty"` } type RequestContext struct { @@ -78,12 +112,22 @@ type PollContext struct { } type RequestSpec struct { - Method string `json:"method"` - Path string `json:"path"` - OriginPath bool `json:"originPath,omitempty"` - ContentType string `json:"contentType"` - Headers map[string]string `json:"headers,omitempty"` - Body any `json:"body,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + OriginPath bool `json:"originPath,omitempty"` + ContentType string `json:"contentType"` + Headers map[string]string `json:"headers,omitempty"` + Query map[string][]string `json:"query,omitempty"` + Body any `json:"body,omitempty"` + Files []RequestFilePart `json:"files,omitempty"` + Auth ManifestAuth `json:"auth,omitempty"` +} + +type RequestFilePart struct { + Name string `json:"name"` + Filename string `json:"filename,omitempty"` + MIMEType string `json:"mimeType,omitempty"` + Reference MediaReference `json:"reference"` } type CreateResult struct { @@ -156,12 +200,15 @@ type Manifest struct { // These fields are the normalized provider projection used by the runtime. // They are never serialized; provider execution is described only under // contributes.providers. - Create ManifestOperation `json:"-"` - Agent *ManifestOperation `json:"-"` - Poll *ManifestOperation `json:"-"` - Cancel *ManifestOperation `json:"-"` - Response ManifestResponse `json:"-"` - AgentResponse *ManifestAgentResponse `json:"-"` + Create ManifestOperation `json:"-"` + Agent *ManifestOperation `json:"-"` + Poll *ManifestOperation `json:"-"` + Cancel *ManifestOperation `json:"-"` + ResultOperation *ManifestOperation `json:"-"` + Response ManifestResponse `json:"-"` + AgentResponse *ManifestAgentResponse `json:"-"` + Auth ManifestAuth `json:"-"` + Validations []ManifestValidation `json:"-"` } // Manifest JSON is the public plugin contract. Metadata is an internal @@ -259,18 +306,31 @@ type ManifestProvider struct { RequiresPublicMediaURLs bool `json:"requiresPublicMediaUrls,omitempty"` Auth ManifestAuth `json:"auth,omitempty"` Parameters []Parameter `json:"parameters,omitempty"` + Validations []ManifestValidation `json:"validations,omitempty"` Create ManifestOperation `json:"create"` Agent *ManifestOperation `json:"agent,omitempty"` Poll *ManifestOperation `json:"poll,omitempty"` Cancel *ManifestOperation `json:"cancel,omitempty"` + Result *ManifestOperation `json:"result,omitempty"` Response ManifestResponse `json:"response"` AgentResponse *ManifestAgentResponse `json:"agentResponse,omitempty"` } type ManifestAuth struct { - Type string `json:"type"` - Field string `json:"field"` - Header string `json:"header,omitempty"` + Type string `json:"type"` + Field string `json:"field"` + SecretField string `json:"secretField,omitempty"` + Header string `json:"header,omitempty"` + Prefix string `json:"prefix,omitempty"` + Query string `json:"query,omitempty"` + Username string `json:"username,omitempty"` + Service string `json:"service,omitempty"` + Region string `json:"region,omitempty"` +} + +type ManifestValidation struct { + Assert any `json:"assert"` + Message string `json:"message"` } type ManifestWorkflow struct { @@ -307,5 +367,8 @@ func (r RequestSpec) Validate() error { if r.Method == "" || r.Path == "" { return fmt.Errorf("protocol request spec is incomplete") } + if len(r.Files) > 0 && r.ContentType != "multipart/form-data" { + return fmt.Errorf("protocol file parts require multipart/form-data") + } return nil } diff --git a/backend/internal/repository/finance.go b/backend/internal/repository/finance.go index 07369c895..820b39795 100644 --- a/backend/internal/repository/finance.go +++ b/backend/internal/repository/finance.go @@ -759,6 +759,111 @@ func (r *Repository) SettleBillingOrder(id string, providerRequestID string) err return err } +// RestoreRefundedBillingOrder compensates a billing order that was refunded +// before an operator confirmed the upstream task had actually succeeded. +// +// The original reservation has already been returned to available credits, so +// this transition charges the final amount directly from available credits. +// The conditional refunded -> settled update keeps concurrent/repeated manual +// recovery requests idempotent. +func (r *Repository) RestoreRefundedBillingOrder(id string, providerRequestID string) error { + return r.db.Transaction(func(tx *gorm.DB) error { + var order model.BillingOrder + if err := tx.First(&order, "id = ?", id).Error; err != nil { + return err + } + if order.Status == model.BillingStatusSettled { + return nil + } + if order.Status != model.BillingStatusRefunded { + return ErrBillingStateConflict + } + + actual := order.AmountMicrocredits + var usage *BillingUsage + if order.BillingMode == "token" { + if zeroPricedTokenOrder(order) { + actual = 0 + } else { + var err error + usage, err = billingUsage(tx, id) + if err != nil { + return err + } + actual, err = tokenUsageAmount(order, usage) + if err != nil { + return err + } + } + } + if actual < 0 { + return errors.New("invalid restored billing amount") + } + + updated := tx.Model(&model.CreditAccount{}). + Where("user_id = ?", order.UserID). + Updates(map[string]any{ + "available_microcredits": gorm.Expr("available_microcredits - ?", actual), + "version": gorm.Expr("version + 1"), + "updated_at": time.Now(), + }) + if updated.Error != nil { + return updated.Error + } + if updated.RowsAffected != 1 { + return errors.New("credit account does not exist") + } + + var account model.CreditAccount + if err := tx.First(&account, "user_id = ?", order.UserID).Error; err != nil { + return err + } + now := time.Now() + orderUpdates := map[string]any{ + "status": model.BillingStatusSettled, + "actual_amount_microcredits": actual, + "refunded_amount_microcredits": max(order.AmountMicrocredits-actual, int64(0)), + "refunded_at": nil, + "settled_at": &now, + "error": "", + "updated_at": now, + } + if providerRequestID != "" { + orderUpdates["provider_request_id"] = providerRequestID + } + if usage != nil { + orderUpdates["input_tokens"] = usage.InputTokens + orderUpdates["output_tokens"] = usage.OutputTokens + orderUpdates["cached_tokens"] = usage.CachedTokens + orderUpdates["usage_available"] = true + } + orderUpdate := tx.Model(&model.BillingOrder{}). + Where("id = ? AND status = ?", order.ID, model.BillingStatusRefunded). + Updates(orderUpdates) + if orderUpdate.Error != nil { + return orderUpdate.Error + } + if orderUpdate.RowsAffected != 1 { + return ErrBillingStateConflict + } + + return tx.Create(&model.CreditLedgerEntry{ + ID: newRepositoryID(), + UserID: order.UserID, + Type: model.CreditLedgerConsume, + AmountMicrocredits: -actual, + AvailableDeltaMicrocredits: -actual, + AvailableAfterMicrocredits: account.AvailableMicrocredits, + ReservedAfterMicrocredits: account.ReservedMicrocredits, + BillingOrderID: order.ID, + Model: order.Model, + ChannelID: order.ChannelID, + Scene: order.Scene, + Note: "人工查询确认上游成功，退款订单重新扣费", + }).Error + }) +} + func zeroPricedTokenOrder(order model.BillingOrder) bool { return order.BillingMode == "token" && order.InputTokenPriceMicrocredits == 0 && diff --git a/backend/internal/repository/finance_recovery_test.go b/backend/internal/repository/finance_recovery_test.go new file mode 100644 index 000000000..2bb2602ce --- /dev/null +++ b/backend/internal/repository/finance_recovery_test.go @@ -0,0 +1,109 @@ +package repository + +import ( + "testing" + "time" + + "infinite-canvas/backend/internal/model" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func newRefundedBillingRecoveryRepository(t *testing.T) (*Repository, *gorm.DB) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+newRepositoryID()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.CreditAccount{}, &model.BillingOrder{}, &model.ApiCallLog{}, &model.CreditLedgerEntry{}); err != nil { + t.Fatal(err) + } + return &Repository{db: db}, db +} + +func TestRestoreRefundedBillingOrderChargesFixedOrderOnce(t *testing.T) { + repo, db := newRefundedBillingRecoveryRepository(t) + now := time.Now() + if err := db.Create(&model.CreditAccount{UserID: "user-1", AvailableMicrocredits: 9_000_000}).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.BillingOrder{ + ID: "order-1", UserID: "user-1", IdempotencyKey: "task:task-1", TaskID: "task-1", + Model: "Wan 3.0", Capability: "video", BillingMode: "fixed", AmountMicrocredits: 3_000_000, + ReservedAmountMicrocredits: 3_000_000, RefundedAmountMicrocredits: 3_000_000, + Status: model.BillingStatusRefunded, RefundedAt: &now, + }).Error; err != nil { + t.Fatal(err) + } + + for range 2 { + if err := repo.RestoreRefundedBillingOrder("order-1", "provider-task-1"); err != nil { + t.Fatalf("RestoreRefundedBillingOrder() error = %v", err) + } + } + + var order model.BillingOrder + if err := db.First(&order, "id = ?", "order-1").Error; err != nil { + t.Fatal(err) + } + if order.Status != model.BillingStatusSettled || order.ActualAmountMicrocredits != 3_000_000 || order.RefundedAmountMicrocredits != 0 || order.RefundedAt != nil || order.ProviderRequestID != "provider-task-1" { + t.Fatalf("restored order = %#v", order) + } + var account model.CreditAccount + if err := db.First(&account, "user_id = ?", "user-1").Error; err != nil { + t.Fatal(err) + } + if account.AvailableMicrocredits != 6_000_000 || account.ReservedMicrocredits != 0 { + t.Fatalf("restored account = %#v", account) + } + var consumeCount int64 + if err := db.Model(&model.CreditLedgerEntry{}).Where("billing_order_id = ? AND type = ?", "order-1", model.CreditLedgerConsume).Count(&consumeCount).Error; err != nil { + t.Fatal(err) + } + if consumeCount != 1 { + t.Fatalf("consume ledger count = %d, want 1", consumeCount) + } +} + +func TestRestoreRefundedBillingOrderUsesObservedTokenAmount(t *testing.T) { + repo, db := newRefundedBillingRecoveryRepository(t) + now := time.Now() + if err := db.Create(&model.CreditAccount{UserID: "user-1", AvailableMicrocredits: 100_000}).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.BillingOrder{ + ID: "order-1", UserID: "user-1", IdempotencyKey: "task:task-1", TaskID: "task-1", + Capability: "video", BillingMode: "token", AmountMicrocredits: 1_916_640, + ReservedAmountMicrocredits: 1_916_640, RefundedAmountMicrocredits: 1_916_640, + OutputTokenPriceMicrocredits: 16_000_000, MultiplierBasisPoints: 10_000, + Status: model.BillingStatusRefunded, RefundedAt: &now, + }).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.ApiCallLog{ + ID: "poll-log-1", BillingOrderID: "order-1", RequestKind: "poll", Billable: false, + Status: model.ApiCallStatusSucceeded, UsageAvailable: true, OutputTokens: 108900, + }).Error; err != nil { + t.Fatal(err) + } + + if err := repo.RestoreRefundedBillingOrder("order-1", "provider-task-1"); err != nil { + t.Fatalf("RestoreRefundedBillingOrder() error = %v", err) + } + + var order model.BillingOrder + if err := db.First(&order, "id = ?", "order-1").Error; err != nil { + t.Fatal(err) + } + if order.ActualAmountMicrocredits != 1_742_400 || order.RefundedAmountMicrocredits != 174_240 || order.OutputTokens != 108900 || !order.UsageAvailable { + t.Fatalf("restored token order = %#v", order) + } + var account model.CreditAccount + if err := db.First(&account, "user_id = ?", "user-1").Error; err != nil { + t.Fatal(err) + } + if account.AvailableMicrocredits != -1_642_400 { + t.Fatalf("available balance = %d, want -1642400", account.AvailableMicrocredits) + } +} diff --git a/backend/internal/service/protocol_plugins.go b/backend/internal/service/protocol_plugins.go index ec016eaf5..a1d386024 100644 --- a/backend/internal/service/protocol_plugins.go +++ b/backend/internal/service/protocol_plugins.go @@ -1,7 +1,6 @@ package service import ( - "bytes" "crypto/sha256" "encoding/hex" "encoding/json" @@ -95,7 +94,7 @@ func newPluginRuntime(dataDir string) (*pluginRuntime, error) { return nil, fmt.Errorf("create plugin package directory: %w", err) } center := &pluginRuntime{registryPath: filepath.Join(dataDir, "plugin_registry.json"), packageDir: packageDir, plugins: make(map[string]pluginRecord)} - if err := center.bootstrapBundledPlugins(); err != nil { + if err := center.bootstrapOfficialPlugins(); err != nil { return nil, err } if err := center.reload(); err != nil { @@ -104,7 +103,7 @@ func newPluginRuntime(dataDir string) (*pluginRuntime, error) { return center, nil } -func (c *pluginRuntime) bootstrapBundledPlugins() error { +func (c *pluginRuntime) bootstrapOfficialPlugins() error { stored, err := c.readRegistry() if err != nil { return err @@ -113,68 +112,68 @@ func (c *pluginRuntime) bootstrapBundledPlugins() error { for _, record := range stored { byID[record.ID] = record } - items := protocol.Builtins().List("", "", true) - bundledIDs := make(map[string]struct{}, len(items)+2) - for _, metadata := range items { - bundledIDs[metadata.ID] = struct{}{} - protocol.AttachDocumentation(&metadata) - metadata.Installable = true - manifest, declarative := protocol.BundledManifest(metadata.ID) - existing := []byte(nil) - if record, ok := byID[metadata.ID]; ok { - existing = record.Raw - var installed protocol.Manifest - if err := json.Unmarshal(existing, &installed); err != nil { - return fmt.Errorf("decode bundled plugin %s: %w", metadata.ID, err) - } - expectedBackend := "host:" + metadata.ID - if declarative { - expectedBackend = "declarative" - } - if installed.Runtime.Backend != expectedBackend { - return fmt.Errorf("protocol id %q is reserved by a bundled plugin", metadata.ID) - } - if metadata.Enabled { - metadata.Enabled = installed.Metadata.Enabled - } + officialDir, err := officialPluginPackageDir() + if err != nil { + return err + } + entries, err := os.ReadDir(officialDir) + if err != nil { + return fmt.Errorf("读取官方插件目录失败：%w", err) + } + officialIDs := make(map[string]struct{}, len(entries)+2) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".yingce-plugin") { + continue } - if declarative { - manifest.Metadata.Enabled = metadata.Enabled - manifest.Metadata.Installable = true - } else { - metadata.Execution = "host:" + metadata.ID - manifest = protocol.Manifest{ - APIVersion: "yingce.plugin/v1", - Metadata: metadata, - Runtime: protocol.ManifestRuntime{Backend: "host:" + metadata.ID}, - Permissions: []string{"generation.run"}, - Contributes: protocol.ManifestContributions{Providers: []protocol.ManifestProvider{{ - ID: metadata.ID, Label: metadata.Name, Capabilities: metadata.Categories, Scopes: metadata.Scopes, - Parameters: metadata.Parameters, RequiresPublicMediaURLs: metadata.RequiresPublicMediaURLs, - Create: protocol.ManifestOperation{Method: "POST", Path: "/__host__/" + metadata.ID}, Response: protocol.ManifestResponse{}, - }}}, - } - } - data, err := json.Marshal(manifest) + packageData, err := os.ReadFile(filepath.Join(officialDir, entry.Name())) if err != nil { - return fmt.Errorf("encode bundled protocol %s: %w", metadata.ID, err) + return fmt.Errorf("读取官方插件包 %s：%w", entry.Name(), err) } - record := byID[metadata.ID] - if !bytes.Equal(record.Raw, data) { - now := time.Now().UTC() - if record.InstalledAt.IsZero() { - record.InstalledAt = now + pkg, err := protocol.ParsePluginPackage(packageData) + if err != nil { + return fmt.Errorf("校验官方插件包 %s：%w", entry.Name(), err) + } + if strings.HasPrefix(strings.TrimSpace(pkg.Manifest.Runtime.Backend), "host:") { + return fmt.Errorf("官方插件 %q 不能依赖 host 执行器", pkg.Manifest.Metadata.ID) + } + if _, err := protocol.LoadInstalledProviders(pkg.ManifestRaw, nil); err != nil { + return fmt.Errorf("加载官方插件 %q：%w", pkg.Manifest.Metadata.ID, err) + } + id := pkg.Manifest.Metadata.ID + if _, duplicate := officialIDs[id]; duplicate { + return fmt.Errorf("官方插件 ID %q 重复", id) + } + officialIDs[id] = struct{}{} + manifest := pkg.Manifest + record := byID[id] + if len(record.Raw) > 0 { + var previous protocol.Manifest + if err := json.Unmarshal(record.Raw, &previous); err == nil { + manifest.Metadata.Enabled = previous.Metadata.Enabled } - record.UpdatedAt = now } - record.ID, record.Raw, record.Source, record.PackagePath = metadata.ID, data, "bundled", "" - byID[metadata.ID] = record + manifestData, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("编码官方插件 %q：%w", id, err) + } + hash := pluginHash(packageData) + packageName := hash + ".yingce-plugin" + if err := writePluginFile(filepath.Join(c.packageDir, packageName), packageData); err != nil { + return fmt.Errorf("缓存官方插件 %q：%w", id, err) + } + now := time.Now().UTC() + if record.InstalledAt.IsZero() { + record.InstalledAt = now + } + record.ID, record.Raw, record.Source, record.FileName = id, manifestData, PluginOriginOfficial, entry.Name() + record.PackagePath, record.PackageSHA256, record.UpdatedAt = packageName, hash, now + byID[id] = record } for _, workflow := range bundledWorkflowPluginManifests() { - bundledIDs[workflow.Metadata.ID] = struct{}{} + officialIDs[workflow.Metadata.ID] = struct{}{} data, err := json.Marshal(workflow) if err != nil { - return fmt.Errorf("encode bundled workflow plugin %s: %w", workflow.Metadata.ID, err) + return fmt.Errorf("encode official workflow plugin %s: %w", workflow.Metadata.ID, err) } record := byID[workflow.Metadata.ID] if len(record.Raw) > 0 { @@ -188,23 +187,20 @@ func (c *pluginRuntime) bootstrapBundledPlugins() error { return fmt.Errorf("encode bundled workflow plugin %s: %w", workflow.Metadata.ID, err) } } - if !bytes.Equal(record.Raw, data) { - now := time.Now().UTC() - if record.InstalledAt.IsZero() { - record.InstalledAt = now - } - record.UpdatedAt = now + now := time.Now().UTC() + if record.InstalledAt.IsZero() { + record.InstalledAt = now } - record.ID, record.Raw, record.Source, record.PackagePath = workflow.Metadata.ID, data, "bundled", "" + record.UpdatedAt = now + record.ID, record.Raw, record.Source, record.PackagePath = workflow.Metadata.ID, data, PluginOriginOfficial, "" byID[workflow.Metadata.ID] = record } result := make([]pluginRegistryRecord, 0, len(byID)) for _, record := range byID { - if record.Source == "bundled" { - if _, exists := bundledIDs[record.ID]; !exists { - // Bundled records are derived from the current host registry. Dropping - // an obsolete record prevents a removed built-in protocol from surviving - // as a second, stale provider after a host upgrade. + if record.Source == "bundled" || record.Source == PluginOriginOfficial { + if _, exists := officialIDs[record.ID]; !exists { + // Official records are reconciled from repository packages on every + // startup, so removed packages cannot survive as stale providers. continue } } @@ -214,6 +210,36 @@ func (c *pluginRuntime) bootstrapBundledPlugins() error { return c.writeRegistry(result) } +func officialPluginPackageDir() (string, error) { + if configured := strings.TrimSpace(os.Getenv("CANVAS_OFFICIAL_PLUGIN_DIR")); configured != "" { + info, err := os.Stat(configured) + if err != nil || !info.IsDir() { + return "", fmt.Errorf("CANVAS_OFFICIAL_PLUGIN_DIR 不是可读目录：%s", configured) + } + return configured, nil + } + workingDir, err := os.Getwd() + if err != nil { + return "", err + } + candidates := []string{"/app/plugin-packages"} + current := workingDir + for range 8 { + candidates = append(candidates, filepath.Join(current, "plugin-packages")) + parent := filepath.Dir(current) + if parent == current { + break + } + current = parent + } + for _, candidate := range candidates { + if info, err := os.Stat(candidate); err == nil && info.IsDir() { + return candidate, nil + } + } + return "", errors.New("未找到官方 plugin-packages 目录；请设置 CANVAS_OFFICIAL_PLUGIN_DIR") +} + func (c *pluginRuntime) reload() error { c.mu.Lock() defer c.mu.Unlock() @@ -249,9 +275,7 @@ func (c *pluginRuntime) reload() error { return err } for id, record := range plugins { - adapters, loadErr := protocol.LoadInstalledProviders(record.Raw, func(execution string) (protocol.Adapter, bool) { - return protocol.Builtins().Resolve(execution) - }) + adapters, loadErr := protocol.LoadInstalledProviders(record.Raw, nil) if loadErr != nil { record.Metadata.Enabled = false record.Metadata.UnavailableReason = loadErr.Error() @@ -326,8 +350,8 @@ func (c *pluginRuntime) install(data []byte, fileName string) (PluginView, error c.mu.RLock() existing, exists := c.plugins[manifest.Metadata.ID] c.mu.RUnlock() - if exists && existing.Source == "bundled" { - return PluginView{}, fmt.Errorf("内置插件 %q 不能通过上传覆盖", manifest.Metadata.ID) + if exists && existing.Source == PluginOriginOfficial { + return PluginView{}, fmt.Errorf("官方插件 %q 不能通过上传覆盖", manifest.Metadata.ID) } manifest.Metadata.Enabled = !exists || existing.Metadata.Enabled manifestData, err := json.Marshal(manifest) @@ -428,21 +452,13 @@ func (c *pluginRuntime) setEnabled(id string, enabled bool) (PluginView, error) func pluginManifestView(raw []byte, metadata protocol.Metadata, source string) PluginManifestView { var manifest protocol.Manifest _ = json.Unmarshal(raw, &manifest) - if source == "bundled" { - if adapter, ok := protocol.Builtins().Get(metadata.ID); ok { - current := adapter.Metadata() - current.Enabled = current.Enabled && metadata.Enabled - metadata = current - } - protocol.AttachDocumentation(&metadata) - } if manifest.Metadata.ID == "" { manifest.Metadata = metadata } return PluginManifestView{ ID: metadata.ID, Name: metadata.Name, Version: metadata.Version, APIVersion: "yingce.plugin/v1", Entry: manifest.Entry, Surfaces: manifest.Surfaces, Description: metadata.Description, Documentation: metadata.Documentation, Author: metadata.Vendor, - Permissions: manifest.Permissions, Trusted: source == "bundled", Runtime: manifest.Runtime, + Permissions: manifest.Permissions, Trusted: source == PluginOriginOfficial, Runtime: manifest.Runtime, Configuration: manifest.Configuration, Contributes: manifest.Contributes, } } @@ -456,8 +472,8 @@ func (c *pluginRuntime) uninstall(id string) error { if !ok { return fmt.Errorf("插件 %q 不存在", id) } - if record.Source == "bundled" { - return fmt.Errorf("内置插件 %q 不能卸载，可停用该插件", id) + if record.Source == PluginOriginOfficial { + return fmt.Errorf("官方插件 %q 不能卸载，可停用该插件", id) } stored, err := c.readRegistry() if err != nil { @@ -533,9 +549,6 @@ func writePluginFile(path string, data []byte) error { } func pluginSource(metadata protocol.Metadata) string { - if _, ok := protocol.Builtins().Get(metadata.ID); ok { - return "bundled" - } return "uploaded" } diff --git a/backend/internal/service/protocol_registry.go b/backend/internal/service/protocol_registry.go index 75f21a36a..601d52b87 100644 --- a/backend/internal/service/protocol_registry.go +++ b/backend/internal/service/protocol_registry.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "infinite-canvas/backend/internal/model" @@ -14,6 +15,10 @@ import ( type protocolRegistryContextKey struct{} +var emptyProtocolRegistry, _ = protocol.NewRegistry() +var officialFallbackRegistryOnce sync.Once +var officialFallbackRegistry = emptyProtocolRegistry + func withProtocolRegistry(ctx context.Context, registry *protocol.Registry) context.Context { return context.WithValue(ctx, protocolRegistryContextKey{}, registry) } @@ -21,7 +26,7 @@ func withProtocolRegistry(ctx context.Context, registry *protocol.Registry) cont func protocolAdapterForContext(ctx context.Context, id string) (protocol.Adapter, bool) { registry, _ := ctx.Value(protocolRegistryContextKey{}).(*protocol.Registry) if registry == nil { - registry = protocol.Builtins() + registry = emptyProtocolRegistry } return registry.Resolve(strings.TrimSpace(id)) } @@ -37,14 +42,18 @@ func declarativeProtocolAdapterForContext(ctx context.Context, id string) (proto func agentProtocolAdapterForContext(ctx context.Context, id string) (protocol.AgentAdapter, bool) { registry, _ := ctx.Value(protocolRegistryContextKey{}).(*protocol.Registry) if registry == nil { - registry = protocol.Builtins() + registry = emptyProtocolRegistry } adapter, ok := registry.Resolve(strings.TrimSpace(id)) if !ok || adapter.Metadata().Execution != "declarative" { return nil, false } agentAdapter, ok := adapter.(protocol.AgentAdapter) - return agentAdapter, ok + if !ok { + return nil, false + } + capability, ok := adapter.(protocol.AgentCapability) + return agentAdapter, ok && capability.AgentAvailable() } type PluginProviderCatalogItem struct { @@ -91,9 +100,6 @@ func (s *Service) PluginProviderCatalog(scope, capability string, includeUnavail } func canonicalProviderAdapter(registry *protocol.Registry, id string) (protocol.Adapter, bool) { - if adapter, ok := protocol.Builtins().Resolve(id); ok { - return adapter, true - } return registry.Resolve(id) } @@ -141,7 +147,44 @@ func (s *Service) protocolRegistry() *protocol.Registry { return registry } } - return protocol.Builtins() + return loadOfficialFallbackRegistry() +} + +func loadOfficialFallbackRegistry() *protocol.Registry { + officialFallbackRegistryOnce.Do(func() { + directory, err := officialPluginPackageDir() + if err != nil { + return + } + entries, err := os.ReadDir(directory) + if err != nil { + return + } + adapters := make([]protocol.Adapter, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".yingce-plugin") { + continue + } + data, err := os.ReadFile(filepath.Join(directory, entry.Name())) + if err != nil { + return + } + pkg, err := protocol.ParsePluginPackage(data) + if err != nil { + return + } + providers, err := protocol.LoadInstalledProviders(pkg.ManifestRaw, nil) + if err != nil { + return + } + adapters = append(adapters, providers...) + } + registry, err := protocol.NewRegistry(adapters...) + if err == nil { + officialFallbackRegistry = registry + } + }) + return officialFallbackRegistry } func (s *Service) protocolMetadata(id string) (protocol.Metadata, bool) { diff --git a/backend/internal/service/protocol_runtime_test.go b/backend/internal/service/protocol_runtime_test.go index 62f39bdf5..896b9cf76 100644 --- a/backend/internal/service/protocol_runtime_test.go +++ b/backend/internal/service/protocol_runtime_test.go @@ -15,26 +15,102 @@ import ( "infinite-canvas/backend/internal/protocol" ) -func TestPluginViewIncludesDocumentationForEveryBundledProtocol(t *testing.T) { +func TestPluginViewIncludesDocumentationForEveryOfficialProtocol(t *testing.T) { center, err := newPluginRuntime(t.TempDir()) if err != nil { t.Fatal(err) } plugins := center.list() - if len(plugins) != len(protocol.Builtins().List("", "", true))+2 { - t.Fatalf("plugin views = %d, builtins plus workflow plugins = %d", len(plugins), len(protocol.Builtins().List("", "", true))+2) - } + pluginsByID := make(map[string]PluginView, len(plugins)) for _, plugin := range plugins { - if plugin.Source != "bundled" { + pluginsByID[plugin.Manifest.ID] = plugin + } + packages, err := filepath.Glob(filepath.Join("..", "..", "..", "plugin-packages", "*.yingce-plugin")) + if err != nil { + t.Fatal(err) + } + if len(plugins) != len(packages)+2 { + t.Fatalf("plugin views = %d, official packages plus workflow plugins = %d", len(plugins), len(packages)+2) + } + for _, packagePath := range packages { + data, err := os.ReadFile(packagePath) + if err != nil { + t.Fatal(err) + } + pkg, err := protocol.ParsePluginPackage(data) + if err != nil { + t.Fatalf("parse %s: %v", filepath.Base(packagePath), err) + } + plugin, ok := pluginsByID[pkg.Manifest.Metadata.ID] + if !ok { + t.Errorf("fresh runtime did not install official plugin %q", pkg.Manifest.Metadata.ID) continue } - document := strings.TrimSpace(plugin.Manifest.Documentation) - if document == "" || !strings.Contains(document, "## 影策运行时合同") { - t.Errorf("bundled plugin %q has no complete documentation in PluginView", plugin.Manifest.ID) + if plugin.Source != PluginOriginOfficial { + t.Errorf("plugin %q source = %q, want official", plugin.Manifest.ID, plugin.Source) + } + expected := strings.TrimSpace(string(pkg.Files["README.md"])) + "\n\n---\n\n" + strings.TrimSpace(string(pkg.Files["docs/interface.md"])) + if strings.TrimSpace(plugin.Manifest.Documentation) != expected { + t.Errorf("freshly installed plugin %q did not expose its complete packaged documentation", plugin.Manifest.ID) + } + if !strings.Contains(plugin.Manifest.Documentation, "## Manifest 完整接口定义") { + t.Errorf("freshly installed plugin %q has no embedded full manifest contract", plugin.Manifest.ID) } } } +func TestPluginRuntimeRefreshesExistingOfficialPackageDocumentation(t *testing.T) { + packagePath := filepath.Join("..", "..", "..", "plugin-packages", "openai-images.yingce-plugin") + packageData, err := os.ReadFile(packagePath) + if err != nil { + t.Fatal(err) + } + pkg, err := protocol.ParsePluginPackage(packageData) + if err != nil { + t.Fatal(err) + } + stale := pkg.Manifest + stale.Metadata.Documentation = "# OpenAI Images\n\n旧版本摘要。" + stale.Metadata.Enabled = false + staleRaw, err := json.Marshal(stale) + if err != nil { + t.Fatal(err) + } + registryData, err := json.Marshal([]pluginRegistryRecord{{ + ID: pkg.Manifest.Metadata.ID, Raw: staleRaw, Source: PluginOriginOfficial, + FileName: filepath.Base(packagePath), PackageSHA256: "stale-package-hash", + }}) + if err != nil { + t.Fatal(err) + } + dataDir := t.TempDir() + if err := os.WriteFile(filepath.Join(dataDir, "plugin_registry.json"), registryData, 0o600); err != nil { + t.Fatal(err) + } + + center, err := newPluginRuntime(dataDir) + if err != nil { + t.Fatal(err) + } + for _, plugin := range center.list() { + if plugin.Manifest.ID != pkg.Manifest.Metadata.ID { + continue + } + expected := strings.TrimSpace(string(pkg.Files["README.md"])) + "\n\n---\n\n" + strings.TrimSpace(string(pkg.Files["docs/interface.md"])) + if strings.TrimSpace(plugin.Manifest.Documentation) != expected { + t.Fatal("existing official plugin kept stale documentation instead of importing the rebuilt package") + } + if plugin.Status != "disabled" { + t.Fatalf("official plugin enabled state was not preserved during refresh: %s", plugin.Status) + } + if plugin.SHA256 == "" || plugin.SHA256 == "stale-package-hash" { + t.Fatalf("official plugin package hash was not refreshed: %q", plugin.SHA256) + } + return + } + t.Fatal("refreshed OpenAI Images plugin is missing") +} + func TestBundledWorkflowPluginsControlAvailability(t *testing.T) { center, err := newPluginRuntime(t.TempDir()) if err != nil { @@ -121,16 +197,9 @@ func TestPluginRuntimeIsTheProtocolSourceOfTruth(t *testing.T) { } } -func TestPluginRuntimeDropsRemovedBundledProtocol(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "..", "..", "plugin-packages", "autodl-comfyui.yingce-plugin")) - if err != nil { - t.Fatal(err) - } - pkg, err := protocol.ParsePluginPackage(data) - if err != nil { - t.Fatal(err) - } - registryData, err := json.Marshal([]pluginRegistryRecord{{ID: "autodl-comfyui", Raw: pkg.ManifestRaw, Source: "bundled"}}) +func TestPluginRuntimeDropsRemovedOfficialProtocol(t *testing.T) { + staleManifest := json.RawMessage(`{"apiVersion":"yingce.plugin/v2","id":"removed-official-protocol","version":"1.0.0","name":"Removed Official Protocol","author":"Test","documentation":"# Removed\n\n## 影策运行时合同","contributes":{"providers":[{"id":"removed-official-protocol","label":"Removed","capabilities":["video"],"scopes":["canvas"],"create":{"method":"POST","path":"/tasks","body":{"prompt":{"$ref":"request.prompt"}}},"response":{"status":"pending"}}]}}`) + registryData, err := json.Marshal([]pluginRegistryRecord{{ID: "removed-official-protocol", Raw: staleManifest, Source: PluginOriginOfficial}}) if err != nil { t.Fatal(err) } @@ -142,26 +211,26 @@ func TestPluginRuntimeDropsRemovedBundledProtocol(t *testing.T) { if err != nil { t.Fatal(err) } - if _, ok := center.registrySnapshot().Resolve("autodl-comfyui"); ok { - t.Fatal("removed bundled AutoDL protocol survived bootstrap") + if _, ok := center.registrySnapshot().Resolve("removed-official-protocol"); ok { + t.Fatal("removed official protocol survived bootstrap") } } -func TestAutoDLPluginPackageInstallsThroughUploadRuntime(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "..", "..", "plugin-packages", "autodl-comfyui.yingce-plugin")) - if err != nil { - t.Fatal(err) - } +func TestAutoDLPluginPackageLoadsAsOfficialRuntime(t *testing.T) { center, err := newPluginRuntime(t.TempDir()) if err != nil { t.Fatal(err) } - plugin, err := center.install(data, "autodl-comfyui.yingce-plugin") - if err != nil { - t.Fatal(err) + var plugin *PluginView + for _, item := range center.list() { + if item.Manifest.ID == "autodl-comfyui" { + copy := item + plugin = © + break + } } - if plugin.Status != "enabled" || plugin.Manifest.ID != "autodl-comfyui" || plugin.Package != protocol.PluginPackageFormat { - t.Fatalf("installed AutoDL plugin = %#v", plugin) + if plugin == nil || plugin.Status != "enabled" || plugin.Source != PluginOriginOfficial || plugin.Package != protocol.PluginPackageFormat { + t.Fatalf("official AutoDL plugin = %#v", plugin) } if !center.registrySnapshot().IsCapability("autodl-comfyui", protocol.CapabilityVideo) { t.Fatal("AutoDL package provider was not registered") @@ -225,6 +294,58 @@ func TestDeclarativeProtocolRuntimeExecutesCreatePollAndDownload(t *testing.T) { } } +func TestDeclarativeProtocolRecoveryQueriesExistingTaskWithoutCreating(t *testing.T) { + manifest := []byte(`{ + "apiVersion":"yingce.plugin/v2", + "id":"test-declarative-video-recovery","version":"1.0.0","name":"Test Declarative Video Recovery","author":"Test","documentation":"# Test Declarative Video Recovery", + "contributes":{"providers":[{"id":"test-declarative-video-recovery","label":"Test Declarative Video Recovery","capabilities":["video"],"scopes":["canvas"],"create":{"method":"POST","path":"/tasks","body":{"model":{"$ref":"request.model"}}},"poll":{"method":"GET","path":"/videos/{{taskId}}"},"result":{"method":"GET","path":"/videos/{{taskId}}/content"},"response":{"taskId":{"$coalesce":[{"$ref":"response.id"},{"$ref":"taskId"}]},"status":{"$coalesce":[{"$ref":"response.status"},"pending"]}}}]} + }`) + center, err := newPluginRuntime(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := center.install(testPluginPackage(t, manifest), "test-declarative-video-recovery.yingce-plugin"); err != nil { + t.Fatal(err) + } + + createCalls := 0 + pollCalls := 0 + downloadCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/tasks": + createCalls++ + http.Error(w, "recovery must not create", http.StatusConflict) + case "/v1/videos/existing-task": + pollCalls++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"existing-task","status":"completed"}`)) + case "/v1/videos/existing-task/content": + downloadCalls++ + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("video")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + config := providerConfig{BaseURL: server.URL + "/v1", APIKey: "key", Model: "test-model", APIFormat: "openai", InterfaceType: "test-declarative-video-recovery", AllowLocalChannel: true} + ctx := withProviderOutboundPolicy(context.Background(), config) + ctx = withProtocolRegistry(ctx, center.registrySnapshot()) + adapter, ok := declarativeProtocolAdapterForContext(ctx, config.InterfaceType) + if !ok { + t.Fatal("declarative recovery adapter is unavailable") + } + result, status, err := queryProtocolAdapterVideoTask(ctx, canvasGenerationInput{Mode: "video", Prompt: "a clip", Config: config}, adapter, "existing-task") + if err != nil { + t.Fatal(err) + } + if status != string(protocol.StatusSucceeded) || result["mode"] != "video" || createCalls != 0 || pollCalls != 1 || downloadCalls != 1 { + t.Fatalf("recovery result=%#v status=%q calls=create:%d poll:%d download:%d", result, status, createCalls, pollCalls, downloadCalls) + } +} + func TestDeclarativeProtocolRuntimeMapsReferenceImageURL(t *testing.T) { manifest := []byte(`{ "apiVersion":"yingce.plugin/v1", diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index eb8fd9ca9..fb5fa3006 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -4,7 +4,10 @@ import ( "bufio" "bytes" "context" + "crypto/hmac" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -26,6 +29,7 @@ import ( "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/protocol" + "github.com/volcengine/volc-sdk-golang/base" "gorm.io/gorm" ) @@ -52,6 +56,8 @@ type canvasGenerationInput struct { type agentToolRequests struct { Responses map[string]interface{} `json:"responses"` ChatCompletion map[string]interface{} `json:"chatCompletion"` + Claude map[string]interface{} `json:"claude"` + Gemini map[string]interface{} `json:"gemini"` } type providerTextMessage struct { @@ -355,7 +361,10 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string } } if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiImage) { - return nil, errors.New("后端任务队列暂不支持 Gemini 调用格式，请使用 OpenAI 兼容渠道") + _, hasDeclarativeAgent := agentProtocolAdapterForContext(ctx, input.Config.InterfaceType) + if input.AgentRequests == nil || !hasDeclarativeAgent { + return nil, errors.New("后端任务队列暂不支持该 Gemini 调用格式，请选择已安装的 Gemini 协议插件") + } } if strings.TrimSpace(input.Config.BaseURL) == "" || strings.TrimSpace(input.Config.APIKey) == "" || strings.TrimSpace(input.Config.Model) == "" { return nil, errors.New("后端生成任务缺少 Base URL、API Key 或模型名") @@ -466,6 +475,8 @@ func runDeclarativeAgentTask(ctx context.Context, input canvasGenerationInput, a request := map[string]any{ "chatCompletion": input.AgentRequests.ChatCompletion, "responses": input.AgentRequests.Responses, + "claude": input.AgentRequests.Claude, + "gemini": input.AgentRequests.Gemini, } spec, err := adapter.BuildAgent(ctx, protocol.AgentRequestContext{BaseURL: input.Config.BaseURL, Model: input.Config.Model, Request: request}) if err != nil { @@ -485,11 +496,15 @@ func runDeclarativeAgentTask(ctx context.Context, input canvasGenerationInput, a } calls := make([]interface{}, 0, len(parsed.ToolCalls)) for _, call := range parsed.ToolCalls { - calls = append(calls, map[string]interface{}{ + mapped := map[string]interface{}{ "id": call.ID, "type": "function", "function": map[string]interface{}{"name": call.Name, "arguments": call.Arguments}, - }) + } + if call.ThoughtSignature != "" { + mapped["thoughtSignature"] = call.ThoughtSignature + } + calls = append(calls, mapped) } result["toolCalls"] = calls if strings.TrimSpace(parsed.Text) == "" && len(calls) == 0 { @@ -2288,7 +2303,7 @@ func runProtocolAdapterTask(ctx context.Context, input canvasGenerationInput, ad if err != nil { return nil, err } - body, err := executeProtocolRequest(ctx, input.Config, spec) + body, err := executeProtocolRequest(withProviderRequestKind(ctx, "create"), input.Config, spec) if err != nil { return nil, err } @@ -2301,7 +2316,7 @@ func runProtocolAdapterTask(ctx context.Context, input canvasGenerationInput, ad return nil, protocolResultError(created.Message, taskID) } if created.Status == protocol.StatusSucceeded { - return finishProtocolResult(ctx, input.Config, input.Mode, created.Result) + return finishProtocolAdapterResult(ctx, input, adapter, request, taskID, created.Result) } if taskID == "" { return nil, errors.New("声明式协议创建请求没有返回任务 ID") @@ -2313,7 +2328,7 @@ func runProtocolAdapterTask(ctx context.Context, input canvasGenerationInput, ad if err != nil { return nil, err } - body, err := executeProtocolRequest(ctx, input.Config, spec) + body, err := executeProtocolRequest(withProviderRequestKind(ctx, "poll"), input.Config, spec) if err != nil { return nil, err } @@ -2326,7 +2341,7 @@ func runProtocolAdapterTask(ctx context.Context, input canvasGenerationInput, ad } switch state.Status { case protocol.StatusSucceeded: - return finishProtocolResult(ctx, input.Config, input.Mode, state.Result) + return finishProtocolAdapterResult(ctx, input, adapter, request, taskID, state.Result) case protocol.StatusFailed, protocol.StatusCancelled: return nil, protocolResultError(state.Message, taskID) } @@ -2337,11 +2352,45 @@ func runProtocolAdapterTask(ctx context.Context, input canvasGenerationInput, ad return nil, fmt.Errorf("声明式协议任务超时（任务 %s）", taskID) } +// queryProtocolAdapterVideoTask performs exactly one read of an existing +// declarative provider task. Manual recovery uses this path so it can never +// create a second billable generation while checking a failed local task. +func queryProtocolAdapterVideoTask(ctx context.Context, input canvasGenerationInput, adapter protocol.Adapter, taskID string) (map[string]interface{}, string, error) { + request := protocolRequestFromInput(input) + pollContext := protocol.PollContext{BaseURL: input.Config.BaseURL, Model: request.Model, Request: request, TaskID: taskID} + spec, err := adapter.BuildPoll(ctx, pollContext) + if err != nil { + return nil, "", err + } + body, err := executeProtocolRequest(withProviderRequestKind(ctx, "poll"), input.Config, spec) + if err != nil { + return nil, "", err + } + state, err := adapter.ParsePoll(ctx, pollContext, body) + if err != nil { + return nil, "", err + } + providerStatus := string(state.Status) + switch state.Status { + case protocol.StatusSucceeded: + result, err := finishProtocolAdapterResult(ctx, input, adapter, request, taskID, state.Result) + return result, providerStatus, err + case protocol.StatusFailed, protocol.StatusCancelled: + return nil, providerStatus, protocolResultError(state.Message, taskID) + case protocol.StatusPending, protocol.StatusProcessing: + return nil, providerStatus, nil + default: + return nil, providerStatus, fmt.Errorf("声明式协议任务 %s 返回未知状态：%s", taskID, providerStatus) + } +} + func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRequest { request := protocol.GenerationRequest{ + Capability: protocol.Capability(input.Mode), Model: input.Config.Model, Prompt: input.Prompt, - Images: protocolVideoImageReferences(input), + Instructions: strings.TrimSpace(input.Config.SystemPrompt), + Images: protocolImageReferences(input), Videos: protocolMediaReferences(input.ReferenceVideos, "video"), Audios: protocolMediaReferences(input.ReferenceAudios, "audio"), AspectRatio: input.Config.Size, @@ -2357,6 +2406,18 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe "count": input.Config.Count, }, } + for _, message := range input.TextHistory { + role := strings.ToLower(strings.TrimSpace(message.Role)) + if role != "user" && role != "assistant" && role != "system" { + continue + } + if content := strings.TrimSpace(message.Content); content != "" { + request.Messages = append(request.Messages, protocol.Message{Role: role, Content: content}) + } + } + request.Inputs = append(request.Inputs, request.Images...) + request.Inputs = append(request.Inputs, request.Videos...) + request.Inputs = append(request.Inputs, request.Audios...) if input.MaxOutputTokens > 0 { request.Extra["max_output_tokens"] = input.MaxOutputTokens request.Extra["max_tokens"] = input.MaxOutputTokens @@ -2367,23 +2428,56 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe if count, err := strconv.Atoi(strings.TrimSpace(input.Config.Count)); err == nil && count > 0 { request.ImageCount = count } + request.Output = protocol.OutputOptions{ + Count: request.ImageCount, Duration: request.Duration, AspectRatio: request.AspectRatio, + Resolution: request.Resolution, Quality: request.Quality, GenerateAudio: request.GenerateAudio, + Watermark: request.Watermark, Format: input.Config.AudioFormat, + } + request.ProviderOptions = make(map[string]map[string]any) + if configured, ok := input.Metadata["providerOptions"].(map[string]any); ok { + for namespace, raw := range configured { + if options, ok := raw.(map[string]any); ok { + request.ProviderOptions[strings.TrimSpace(namespace)] = options + } + } + } return request } +func protocolImageReferences(input canvasGenerationInput) []protocol.MediaReference { + if input.Mode == "video" { + return protocolVideoImageReferences(input) + } + result := make([]protocol.MediaReference, 0, len(input.ReferenceImages)+1) + for index, value := range input.ReferenceImages { + item := protocolMediaReference(value, "image", index) + item.Role = "reference_image" + if input.Mode == "image" { + item.Role = "edit_source" + } + if item.URL != "" || item.DataURL != "" { + result = append(result, item) + } + } + if input.Mask != nil { + mask := protocolMediaReference(*input.Mask, "image", len(result)) + mask.Role = "mask" + if mask.URL != "" || mask.DataURL != "" { + result = append(result, mask) + } + } + return result +} + func protocolVideoImageReferences(input canvasGenerationInput) []protocol.MediaReference { result := make([]protocol.MediaReference, 0, len(input.ReferenceImages)) fallbackRole := "" if metadataString(input.Metadata, "videoStartFrameNodeId") != "" || metadataString(input.Metadata, "videoEndFrameNodeId") != "" { fallbackRole = "reference_image" } - for _, value := range input.ReferenceImages { - item := protocol.MediaReference{ - ID: strings.TrimSpace(value.ID), - URL: strings.TrimSpace(value.URL), - DataURL: strings.TrimSpace(value.DataURL), - Kind: "image", - Role: videoImageRoleOrDefault(input, value, fallbackRole), - } + for index, value := range input.ReferenceImages { + item := protocolMediaReference(value, "image", index) + item.Role = videoImageRoleOrDefault(input, value, fallbackRole) if item.URL != "" || item.DataURL != "" { result = append(result, item) } @@ -2393,8 +2487,13 @@ func protocolVideoImageReferences(input canvasGenerationInput) []protocol.MediaR func protocolMediaReferences(values []providerMedia, kind string) []protocol.MediaReference { result := make([]protocol.MediaReference, 0, len(values)) - for _, value := range values { - item := protocol.MediaReference{ID: strings.TrimSpace(value.ID), URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), Kind: kind} + for index, value := range values { + item := protocolMediaReference(value, kind, index) + if kind == "video" { + item.Role = "reference_video" + } else if kind == "audio" { + item.Role = "reference_audio" + } if item.URL != "" || item.DataURL != "" { result = append(result, item) } @@ -2402,46 +2501,414 @@ func protocolMediaReferences(values []providerMedia, kind string) []protocol.Med return result } +func protocolMediaReference(value providerMedia, kind string, order int) protocol.MediaReference { + return protocol.MediaReference{ + ID: strings.TrimSpace(value.ID), URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), + Kind: kind, MIMEType: firstNonEmpty(strings.TrimSpace(value.MimeType), strings.TrimSpace(value.Type)), Name: strings.TrimSpace(value.Name), Order: order, + Metadata: map[string]any{"bytes": value.Bytes, "width": value.Width, "height": value.Height, "durationMs": value.DurationMs, "storageKey": strings.TrimSpace(value.StorageKey)}, + } +} + func executeProtocolRequest(ctx context.Context, config providerConfig, spec protocol.RequestSpec) ([]byte, error) { + data, _, err := executeProtocolBinaryRequest(ctx, config, spec) + return data, err +} + +func executeProtocolBinaryRequest(ctx context.Context, config providerConfig, spec protocol.RequestSpec) ([]byte, string, error) { if err := spec.Validate(); err != nil { - return nil, err + return nil, "", err } method := strings.ToUpper(strings.TrimSpace(spec.Method)) - var body io.Reader - if spec.Body != nil { - contentType := strings.ToLower(strings.TrimSpace(strings.Split(spec.ContentType, ";")[0])) - if contentType != "" && contentType != "application/json" { - return nil, fmt.Errorf("声明式协议暂不支持 %s 请求体", spec.ContentType) - } - data, err := json.Marshal(spec.Body) - if err != nil { - return nil, err - } - body = bytes.NewReader(data) + body, contentType, err := protocolRequestBody(ctx, config, spec) + if err != nil { + return nil, "", err } requestURL, err := protocolRequestURL(config.BaseURL, spec) if err != nil { - return nil, err + return nil, "", err } req, err := http.NewRequestWithContext(ctx, method, requestURL, body) if err != nil { - return nil, err + return nil, "", err } - applyProviderAuth(req, config) - if spec.Body != nil { - req.Header.Set("Content-Type", "application/json") + if contentType != "" { + req.Header.Set("Content-Type", contentType) } for name, value := range spec.Headers { req.Header.Set(name, value) } ApplyOutboundHeaders(req, config.Headers) - data, _, err := doBinary(req) + if err := applyProtocolAuth(req, config, spec.Auth); err != nil { + return nil, "", err + } + return doBinary(req) +} + +func protocolRequestBody(ctx context.Context, config providerConfig, spec protocol.RequestSpec) (io.Reader, string, error) { + contentType := strings.ToLower(strings.TrimSpace(strings.Split(spec.ContentType, ";")[0])) + if spec.Body == nil && len(spec.Files) == 0 { + return nil, "", nil + } + switch contentType { + case "", "application/json": + data, err := json.Marshal(spec.Body) + if err != nil { + return nil, "", err + } + return bytes.NewReader(data), "application/json", nil + case "application/x-www-form-urlencoded": + values := url.Values{} + for key, value := range protocolBodyObject(spec.Body) { + for _, item := range protocolFormValues(value) { + values.Add(key, item) + } + } + return strings.NewReader(values.Encode()), contentType, nil + case "multipart/form-data": + var body bytes.Buffer + writer := multipart.NewWriter(&body) + keys := make([]string, 0) + for key := range protocolBodyObject(spec.Body) { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + for _, item := range protocolFormValues(protocolBodyObject(spec.Body)[key]) { + if err := writer.WriteField(key, item); err != nil { + _ = writer.Close() + return nil, "", err + } + } + } + for _, file := range spec.Files { + data, detectedMIME, err := protocolMediaBytes(ctx, config, file.Reference) + if err != nil { + _ = writer.Close() + return nil, "", fmt.Errorf("读取 multipart 文件 %s 失败：%w", file.Name, err) + } + filename := safeProtocolFilename(file.Filename) + mimeType := strings.TrimSpace(file.MIMEType) + if mimeType == "" { + mimeType = detectedMIME + } + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, file.Name, filename)) + header.Set("Content-Type", defaultString(mimeType, "application/octet-stream")) + part, err := writer.CreatePart(header) + if err != nil { + _ = writer.Close() + return nil, "", err + } + if _, err := part.Write(data); err != nil { + _ = writer.Close() + return nil, "", err + } + } + if err := writer.Close(); err != nil { + return nil, "", err + } + return bytes.NewReader(body.Bytes()), writer.FormDataContentType(), nil + case "application/octet-stream": + switch value := spec.Body.(type) { + case []byte: + return bytes.NewReader(value), contentType, nil + case string: + if strings.HasPrefix(value, "data:") { + mimeType, data, err := decodeProviderDataURL(value) + if err != nil { + return nil, "", err + } + return bytes.NewReader(data), defaultString(mimeType, contentType), nil + } + return strings.NewReader(value), contentType, nil + default: + return nil, "", fmt.Errorf("二进制协议请求体必须是字节或字符串") + } + default: + return nil, "", fmt.Errorf("声明式协议暂不支持 %s 请求体", spec.ContentType) + } +} + +func protocolBodyObject(value any) map[string]any { + result, _ := value.(map[string]any) + return result +} + +func protocolFormValues(value any) []string { + switch typed := value.(type) { + case nil: + return nil + case []any: + result := make([]string, 0, len(typed)) + for _, item := range typed { + result = append(result, protocolFormValues(item)...) + } + return result + case string: + return []string{typed} + case bool: + return []string{strconv.FormatBool(typed)} + case float64: + return []string{strconv.FormatFloat(typed, 'f', -1, 64)} + case int: + return []string{strconv.Itoa(typed)} + default: + data, err := json.Marshal(typed) + if err != nil { + return nil + } + return []string{string(data)} + } +} + +func safeProtocolFilename(value string) string { + value = strings.TrimSpace(value) + if index := strings.LastIndexAny(value, `/\\`); index >= 0 { + value = value[index+1:] + } + value = strings.Map(func(r rune) rune { + if r < 32 || r == 127 || r == '"' { + return -1 + } + return r + }, value) + if value == "" { + return "upload.bin" + } + return value +} + +func applyProtocolAuth(req *http.Request, config providerConfig, auth protocol.ManifestAuth) error { + typeName := strings.ToLower(strings.TrimSpace(auth.Type)) + if typeName == "" { + applyProviderAuth(req, config) + return nil + } + credential := protocolCredentialField(config, auth.Field) + switch typeName { + case "none": + return nil + case "bearer": + header := defaultString(strings.TrimSpace(auth.Header), "Authorization") + prefix := auth.Prefix + if prefix == "" { + prefix = "Bearer " + } + req.Header.Set(header, prefix+credential) + return nil + case "header", "api-key", "apikey": + header := strings.TrimSpace(auth.Header) + if header == "" { + return errors.New("插件 header 鉴权缺少 header 名称") + } + req.Header.Set(header, auth.Prefix+credential) + return nil + case "query": + name := defaultString(strings.TrimSpace(auth.Query), strings.TrimSpace(auth.Field)) + if name == "" { + return errors.New("插件 query 鉴权缺少参数名") + } + query := req.URL.Query() + query.Set(name, auth.Prefix+credential) + req.URL.RawQuery = query.Encode() + return nil + case "basic": + username := auth.Username + if username == "" { + username = credential + } + password := protocolCredentialField(config, auth.SecretField) + req.SetBasicAuth(username, password) + return nil + case "anthropic": + req.Header.Set(defaultString(auth.Header, "x-api-key"), credential) + if req.Header.Get("anthropic-version") == "" { + req.Header.Set("anthropic-version", "2023-06-01") + } + return nil + case "google-api-key", "gemini": + req.Header.Set(defaultString(auth.Header, "x-goog-api-key"), credential) + return nil + case "volcengine-v4": + secret := protocolCredentialField(config, auth.SecretField) + if credential == "" || secret == "" { + return errors.New("火山引擎 V4 鉴权需要 Access Key 和 Secret Key") + } + credentials := base.Credentials{ + AccessKeyID: credential, SecretAccessKey: secret, + Region: defaultString(strings.TrimSpace(auth.Region), "cn-north-1"), + Service: strings.TrimSpace(auth.Service), + } + if credentials.Service == "" { + return errors.New("火山引擎 V4 鉴权缺少 service") + } + signed := credentials.Sign(req) + *req = *signed + return nil + case "aws-sigv4": + secret := protocolCredentialField(config, auth.SecretField) + return signProtocolAWSV4(req, credential, secret, auth) + case "tc3": + secret := protocolCredentialField(config, auth.SecretField) + return signProtocolTC3(req, credential, secret, auth) + default: + return fmt.Errorf("插件声明了尚未启用的鉴权驱动 %s", auth.Type) + } +} + +func signProtocolAWSV4(req *http.Request, accessKey, secretKey string, auth protocol.ManifestAuth) error { + if strings.TrimSpace(accessKey) == "" || strings.TrimSpace(secretKey) == "" { + return errors.New("AWS SigV4 鉴权需要 Access Key ID 和 Secret Access Key") + } + serviceName := defaultString(strings.TrimSpace(auth.Service), "bedrock") + region := strings.TrimSpace(auth.Region) + if region == "" { + parts := strings.Split(strings.ToLower(req.URL.Hostname()), ".") + for index, part := range parts { + if strings.HasPrefix(part, serviceName) && index+1 < len(parts) { + region = parts[index+1] + break + } + } + } + if region == "" { + return errors.New("AWS SigV4 鉴权无法从 Base URL 推断 region，请使用包含区域的 Bedrock Runtime 地址") + } + payload, err := protocolRequestPayload(req) + if err != nil { + return err + } + now := time.Now().UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + payloadHash := sha256Hex(payload) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonicalHeaders, signedHeaders := protocolCanonicalHeaders(req) + canonicalRequest := strings.Join([]string{ + req.Method, + defaultString(req.URL.EscapedPath(), "/"), + req.URL.Query().Encode(), + canonicalHeaders, + signedHeaders, + payloadHash, + }, "\n") + scope := strings.Join([]string{dateStamp, region, serviceName, "aws4_request"}, "/") + stringToSign := strings.Join([]string{"AWS4-HMAC-SHA256", amzDate, scope, sha256Hex([]byte(canonicalRequest))}, "\n") + dateKey := protocolHMAC([]byte("AWS4"+secretKey), dateStamp) + regionKey := protocolHMAC(dateKey, region) + serviceKey := protocolHMAC(regionKey, serviceName) + signingKey := protocolHMAC(serviceKey, "aws4_request") + signature := hex.EncodeToString(protocolHMAC(signingKey, stringToSign)) + req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", accessKey, scope, signedHeaders, signature)) + return nil +} + +func signProtocolTC3(req *http.Request, secretID, secretKey string, auth protocol.ManifestAuth) error { + if strings.TrimSpace(secretID) == "" || strings.TrimSpace(secretKey) == "" { + return errors.New("腾讯云 TC3 鉴权需要 SecretId 和 SecretKey") + } + serviceName := defaultString(strings.TrimSpace(auth.Service), "hunyuan") + payload, err := protocolRequestPayload(req) + if err != nil { + return err + } + now := time.Now().UTC() + timestamp := now.Unix() + dateStamp := now.Format("2006-01-02") + contentType := defaultString(req.Header.Get("Content-Type"), "application/json") + req.Header.Set("Content-Type", contentType) + req.Header.Set("X-TC-Timestamp", strconv.FormatInt(timestamp, 10)) + if region := strings.TrimSpace(auth.Region); region != "" { + req.Header.Set("X-TC-Region", region) + } + canonicalHeaders := "content-type:" + strings.ToLower(strings.TrimSpace(contentType)) + "\n" + "host:" + strings.ToLower(req.URL.Host) + "\n" + signedHeaders := "content-type;host" + canonicalRequest := strings.Join([]string{req.Method, defaultString(req.URL.EscapedPath(), "/"), req.URL.Query().Encode(), canonicalHeaders, signedHeaders, sha256Hex(payload)}, "\n") + scope := dateStamp + "/" + serviceName + "/tc3_request" + stringToSign := strings.Join([]string{"TC3-HMAC-SHA256", strconv.FormatInt(timestamp, 10), scope, sha256Hex([]byte(canonicalRequest))}, "\n") + secretDate := protocolHMAC([]byte("TC3"+secretKey), dateStamp) + secretService := protocolHMAC(secretDate, serviceName) + secretSigning := protocolHMAC(secretService, "tc3_request") + signature := hex.EncodeToString(protocolHMAC(secretSigning, stringToSign)) + req.Header.Set("Authorization", fmt.Sprintf("TC3-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", secretID, scope, signedHeaders, signature)) + return nil +} + +func protocolRequestPayload(req *http.Request) ([]byte, error) { + if req.Body == nil { + return nil, nil + } + var reader io.ReadCloser + var err error + if req.GetBody != nil { + reader, err = req.GetBody() + } else { + reader = req.Body + } + if err != nil { + return nil, err + } + data, err := io.ReadAll(reader) + if req.GetBody != nil { + _ = reader.Close() + } else { + req.Body = io.NopCloser(bytes.NewReader(data)) + } return data, err } +func protocolCanonicalHeaders(req *http.Request) (string, string) { + values := map[string]string{"host": strings.ToLower(req.URL.Host)} + for name, entries := range req.Header { + lower := strings.ToLower(strings.TrimSpace(name)) + if lower == "authorization" || lower == "user-agent" || lower == "content-length" || lower == "expect" { + continue + } + cleaned := make([]string, 0, len(entries)) + for _, entry := range entries { + cleaned = append(cleaned, strings.Join(strings.Fields(entry), " ")) + } + values[lower] = strings.Join(cleaned, ",") + } + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + var canonical strings.Builder + for _, key := range keys { + canonical.WriteString(key) + canonical.WriteByte(':') + canonical.WriteString(values[key]) + canonical.WriteByte('\n') + } + return canonical.String(), strings.Join(keys, ";") +} + +func protocolHMAC(key []byte, value string) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write([]byte(value)) + return mac.Sum(nil) +} + +func sha256Hex(value []byte) string { + hash := sha256.Sum256(value) + return hex.EncodeToString(hash[:]) +} + +func protocolCredentialField(config providerConfig, field string) string { + switch strings.ToLower(strings.TrimSpace(field)) { + case "secretkey", "secret_key", "secret": + return strings.TrimSpace(config.SecretKey) + default: + return strings.TrimSpace(config.APIKey) + } +} + func protocolRequestURL(baseURL string, spec protocol.RequestSpec) (string, error) { if !spec.OriginPath { - return apiURL(baseURL, spec.Path), nil + return appendProtocolQuery(apiURL(baseURL, spec.Path), spec.Query) } base, err := url.Parse(strings.TrimSpace(baseURL)) if err != nil || base.Scheme == "" || base.Host == "" { @@ -2455,7 +2922,25 @@ func protocolRequestURL(baseURL string, spec protocol.RequestSpec) (string, erro base.RawPath = requestPath.RawPath base.RawQuery = requestPath.RawQuery base.Fragment = "" - return base.String(), nil + return appendProtocolQuery(base.String(), spec.Query) +} + +func appendProtocolQuery(rawURL string, values map[string][]string) (string, error) { + if len(values) == 0 { + return rawURL, nil + } + parsed, err := url.Parse(rawURL) + if err != nil { + return "", err + } + query := parsed.Query() + for key, items := range values { + for _, item := range items { + query.Add(key, item) + } + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil } func finishProtocolResult(ctx context.Context, config providerConfig, mode string, result *protocol.Result) (map[string]interface{}, error) { @@ -2502,6 +2987,63 @@ func finishProtocolResult(ctx context.Context, config providerConfig, mode strin } } +func finishProtocolAdapterResult(ctx context.Context, input canvasGenerationInput, adapter protocol.Adapter, request protocol.GenerationRequest, taskID string, result *protocol.Result) (map[string]interface{}, error) { + if protocolResultHasOutput(input.Mode, result) { + return finishProtocolResult(ctx, input.Config, input.Mode, result) + } + resultAdapter, ok := adapter.(protocol.ResultAdapter) + capability, hasCapability := adapter.(protocol.ResultCapability) + if !ok || !hasCapability || !capability.ResultAvailable() { + return finishProtocolResult(ctx, input.Config, input.Mode, result) + } + spec, err := resultAdapter.BuildResult(ctx, protocol.PollContext{BaseURL: input.Config.BaseURL, Model: request.Model, Request: request, TaskID: taskID}) + if err != nil { + return nil, err + } + data, mimeType, err := executeProtocolBinaryRequest(withProviderRequestKind(ctx, "download"), input.Config, spec) + if err != nil { + return nil, fmt.Errorf("声明式协议结果下载失败：%w", err) + } + if len(data) == 0 { + return nil, errors.New("声明式协议结果下载返回空内容") + } + mimeType = strings.TrimSpace(strings.Split(mimeType, ";")[0]) + if mimeType == "" { + mimeType = "application/octet-stream" + } + reference := protocol.MediaReference{DataURL: dataURL(mimeType, data), MIMEType: mimeType} + downloaded := &protocol.Result{} + switch input.Mode { + case "image": + downloaded.Images = []protocol.MediaReference{reference} + case "video": + downloaded.Videos = []protocol.MediaReference{reference} + case "audio": + downloaded.Audios = []protocol.MediaReference{reference} + default: + return nil, fmt.Errorf("声明式协议结果下载不支持生成模式 %s", input.Mode) + } + return finishProtocolResult(ctx, input.Config, input.Mode, downloaded) +} + +func protocolResultHasOutput(mode string, result *protocol.Result) bool { + if result == nil { + return false + } + switch mode { + case "text": + return strings.TrimSpace(result.Text) != "" + case "image": + return len(result.Images) > 0 + case "video": + return len(result.Videos) > 0 + case "audio": + return len(result.Audios) > 0 + default: + return false + } +} + func protocolMediaBytes(ctx context.Context, config providerConfig, reference protocol.MediaReference) ([]byte, string, error) { if strings.TrimSpace(reference.DataURL) != "" { mimeType, data, err := decodeProviderDataURL(reference.DataURL) @@ -2511,11 +3053,39 @@ func protocolMediaBytes(ctx context.Context, config providerConfig, reference pr if value == "" { return nil, "", errors.New("声明式协议媒体结果地址为空") } - data, mimeType, err := getProviderExternalBinary(withProviderRequestKind(ctx, "download"), config, value) - if err != nil { - return nil, "", fmt.Errorf("声明式协议媒体结果下载失败：%w", err) + var data []byte + var mimeType string + var err error + for attempt := 0; attempt < 3; attempt++ { + data, mimeType, err = getProviderExternalBinary(withProviderRequestKind(ctx, "download"), config, value) + if err == nil { + return data, normalizedMediaMimeType(mimeType, data), nil + } + if attempt == 2 || !retryableProtocolMediaDownload(err) { + break + } + if waitErr := sleepContext(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { + return nil, "", fmt.Errorf("声明式协议媒体结果下载失败：%w", waitErr) + } } - return data, normalizedMediaMimeType(mimeType, data), nil + return nil, "", fmt.Errorf("声明式协议媒体结果下载失败：%w", err) +} + +func retryableProtocolMediaDownload(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + var networkError net.Error + if errors.As(err, &networkError) && (networkError.Timeout() || networkError.Temporary()) { + return true + } + message := strings.ToLower(err.Error()) + for _, marker := range []string{"tls handshake timeout", "connection reset", "unexpected eof", "broken pipe"} { + if strings.Contains(message, marker) { + return true + } + } + return false } func protocolResultError(message, taskID string) error { diff --git a/backend/internal/service/provider_task_recovery.go b/backend/internal/service/provider_task_recovery.go index 80d9bb7ea..a6a6512ef 100644 --- a/backend/internal/service/provider_task_recovery.go +++ b/backend/internal/service/provider_task_recovery.go @@ -64,6 +64,7 @@ func (s *Service) AdminQueryFailedVideoTask(ctx context.Context, actor *model.Us func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, claimUserID string) (*ProviderTaskQueryResult, error) { billing := s.taskBilling() + ctx = withProtocolRegistry(ctx, s.protocolRegistry()) if task == nil || task.ID == "" { return nil, BadAuthRequest("任务不存在") } @@ -79,6 +80,7 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl if providerRequestID == "" { return nil, BadAuthRequest("该任务没有可恢复的上游任务 ID") } + wasRefunded := false if task.BillingOrderID != "" { order, err := s.repo.BillingOrder(task.BillingOrderID) if err != nil { @@ -87,9 +89,7 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl if order.UserID != task.UserID || order.TaskID != task.ID { return nil, BadAuthRequest("任务与计费订单归属不一致") } - if order.Status == model.BillingStatusRefunded { - return nil, BadAuthRequest("该任务已退款，不能恢复并重新扣费") - } + wasRefunded = order.Status == model.BillingStatusRefunded } decryptedInput, err := s.decryptTaskInputJSON(task.InputJSON) @@ -104,8 +104,9 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl if err != nil { return nil, err } - if config.InterfaceType != string(model.ChannelInterfaceNewAPIChannel2) { - return nil, BadAuthRequest("该任务不使用 NewAPI Video Generations 协议") + adapter, declarative := declarativeProtocolAdapterForContext(ctx, config.InterfaceType) + if !declarative && config.InterfaceType != string(model.ChannelInterfaceNewAPIChannel2) { + return nil, BadAuthRequest("该任务的请求协议不支持安全查询上游状态") } input.Config = config task.InputJSON = decryptedInput @@ -128,9 +129,20 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl } }() - queryCtx := withProviderAnalytics(ctx, s, *task) + // 上游已成功后的完整媒体下载和本地入库可能持续数十秒。浏览器关闭抽屉、 + // 页面刷新或代理断开都不应中断这项运维恢复，否则任务会再次停在 + // failed/refunded。保留请求值用于审计，但把执行生命周期交给恢复租约控制。 + recoveryCtx, cancelRecovery := providerTaskRecoveryContext(ctx) + defer cancelRecovery() + queryCtx := withProviderAnalytics(recoveryCtx, s, *task) queryCtx = withProviderOutboundPolicy(queryCtx, input.Config) - result, providerStatus, err := queryNewAPIChannel2VideoTask(queryCtx, input, providerRequestID) + var result map[string]interface{} + var providerStatus string + if declarative { + result, providerStatus, err = queryProtocolAdapterVideoTask(queryCtx, input, adapter, providerRequestID) + } else { + result, providerStatus, err = queryNewAPIChannel2VideoTask(queryCtx, input, providerRequestID) + } if err != nil { _ = s.log(task.UserID, task.ID, "error", "人工查询上游视频任务失败", err.Error()) return nil, err @@ -160,19 +172,30 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl } return nil, err } + billingSettled := true + var billingErr error + if wasRefunded { + billingErr = billing.RestoreRefundedBilling(task.BillingOrderID, providerRequestID) + } else { + billingErr = billing.SettleBilling(task.BillingOrderID, providerRequestID) + } + if billingErr != nil { + billingSettled = false + uncertainErr := billing.MarkBillingUncertain(task.BillingOrderID, "人工查询确认生成成功，但积分结算失败："+billingErr.Error()) + _ = s.log(task.UserID, task.ID, "error", "任务恢复成功但积分结算失败，已进入待核对", billingErr.Error()) + if uncertainErr != nil { + return nil, errors.Join(billingErr, fmt.Errorf("记录任务恢复后的计费待核对状态失败：%w", uncertainErr)) + } + return nil, billingErr + } if err := s.RegisterTaskOutputFromTask(*task); err != nil { _ = s.log(task.UserID, task.ID, "error", "任务恢复成功但项目产物登记失败", err.Error()) + return nil, fmt.Errorf("任务已恢复并完成扣费，但项目素材登记失败：%w", err) } - billingSettled := true - if err := billing.SettleBilling(task.BillingOrderID, providerRequestID); err != nil { - billingSettled = false - uncertainErr := billing.MarkBillingUncertain(task.BillingOrderID, "人工查询确认生成成功，但积分结算失败："+err.Error()) - _ = s.log(task.UserID, task.ID, "error", "任务恢复成功但积分结算失败，已进入待核对", err.Error()) - if uncertainErr != nil { - return nil, errors.Join(err, fmt.Errorf("记录任务恢复后的计费待核对状态失败：%w", uncertainErr)) - } - } else { - _ = s.log(task.UserID, task.ID, "info", "人工查询确认生成成功，任务已恢复并完成结算", providerStatus) - } + _ = s.log(task.UserID, task.ID, "info", "人工查询确认生成成功，任务已恢复、完成结算并登记项目产物", providerStatus) return &ProviderTaskQueryResult{Task: taskForOutput(*task), ProviderStatus: providerStatus, Recovered: true, BillingSettled: billingSettled}, nil } + +func providerTaskRecoveryContext(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(parent), providerTaskRecoveryLeaseDuration) +} diff --git a/backend/internal/service/provider_task_recovery_test.go b/backend/internal/service/provider_task_recovery_test.go new file mode 100644 index 000000000..ab4a3e5f4 --- /dev/null +++ b/backend/internal/service/provider_task_recovery_test.go @@ -0,0 +1,42 @@ +package service + +import ( + "context" + "errors" + "testing" +) + +func TestProviderTaskRecoveryContextSurvivesClientCancellation(t *testing.T) { + type contextKey string + parent, cancelParent := context.WithCancel(context.WithValue(context.Background(), contextKey("trace"), "trace-1")) + cancelParent() + + recovery, cancelRecovery := providerTaskRecoveryContext(parent) + defer cancelRecovery() + if recovery.Err() != nil { + t.Fatalf("recovery context inherited cancellation: %v", recovery.Err()) + } + if recovery.Value(contextKey("trace")) != "trace-1" { + t.Fatal("recovery context did not preserve request values") + } + if _, ok := recovery.Deadline(); !ok { + t.Fatal("recovery context has no bounded deadline") + } +} + +func TestRetryableProtocolMediaDownload(t *testing.T) { + for _, err := range []error{ + errors.New("net/http: TLS handshake timeout"), + errors.New("read: connection reset by peer"), + errors.New("unexpected EOF"), + } { + if !retryableProtocolMediaDownload(err) { + t.Fatalf("retryableProtocolMediaDownload(%v) = false", err) + } + } + for _, err := range []error{context.Canceled, context.DeadlineExceeded, errors.New("HTTP 404")} { + if retryableProtocolMediaDownload(err) { + t.Fatalf("retryableProtocolMediaDownload(%v) = true", err) + } + } +} diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index d425dcf6c..7c20e93ac 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -94,7 +94,7 @@ func TestProtocolRequestURLCanResolveSameOriginRootPath(t *testing.T) { } } -func TestRunVideoTaskUsesHostBackedAgnesJSONProtocol(t *testing.T) { +func TestRunVideoTaskUsesDeclarativeAgnesJSONProtocol(t *testing.T) { t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") center, err := newPluginRuntime(t.TempDir()) if err != nil { @@ -102,9 +102,9 @@ func TestRunVideoTaskUsesHostBackedAgnesJSONProtocol(t *testing.T) { } adapter, ok := center.registrySnapshot().Resolve("agnes-video") if !ok { - t.Fatal("host-backed Agnes adapter is missing") + t.Fatal("declarative Agnes adapter is missing") } - if metadata := adapter.Metadata(); metadata.Version != "1.2.0" || metadata.Execution != "host:agnes-video" || !metadata.RequiresPublicMediaURLs { + if metadata := adapter.Metadata(); metadata.Version != "2.0.0" || metadata.Execution != "declarative" || !metadata.RequiresPublicMediaURLs { t.Fatalf("Agnes runtime metadata = %#v", metadata) } @@ -165,12 +165,57 @@ func TestRunVideoTaskUsesHostBackedAgnesJSONProtocol(t *testing.T) { if !ok || video["dataUrl"] != "data:video/mp4;base64,dmlkZW8=" { t.Fatalf("video = %#v", result["video"]) } - wantPaths := "POST /v1/videos,GET /agnesapi?video_id=video-1&model_name=agnes-video-2.5,GET /video.mp4" + wantPaths := "POST /v1/videos,GET /agnesapi?model_name=agnes-video-2.5&video_id=video-1,GET /video.mp4" if got := strings.Join(paths, ","); got != wantPaths { t.Fatalf("paths = %q, want %q", got, wantPaths) } } +func TestRunVideoTaskDownloadsAuthenticatedDeclarativeResult(t *testing.T) { + t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") + center, err := newPluginRuntime(t.TempDir()) + if err != nil { + t.Fatal(err) + } + paths := make([]string, 0, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.Method+" "+r.URL.Path) + switch r.URL.Path { + case "/v1/videos": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"video-1","status":"completed"}`)) + case "/v1/videos/video-1/content": + if got := r.Header.Get("Authorization"); got != "Bearer test-key" { + t.Errorf("Authorization = %q", got) + } + if got := r.Header.Get("Accept"); got != "video/mp4" { + t.Errorf("Accept = %q", got) + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("video")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + ctx := withProtocolRegistry(context.Background(), center.registrySnapshot()) + result, err := runVideoTask(ctx, canvasGenerationInput{ + Mode: "video", Prompt: "cinematic shot", + Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", InterfaceType: "newapi", Model: "sora-2", VideoSeconds: "5", Size: "1280x720"}, + }) + if err != nil { + t.Fatal(err) + } + video, _ := result["video"].(map[string]interface{}) + if video["dataUrl"] != "data:video/mp4;base64,dmlkZW8=" { + t.Fatalf("video = %#v", video) + } + if got := strings.Join(paths, ","); got != "POST /v1/videos,GET /v1/videos/video-1/content" { + t.Fatalf("paths = %q", got) + } +} + func TestSystemChannelIDFromBaseURLSupportsShortAndLegacyProxyPaths(t *testing.T) { for _, test := range []struct{ base, want string }{ {base: "/api/channel-1", want: "channel-1"}, diff --git a/backend/internal/service/task_billing.go b/backend/internal/service/task_billing.go index bd2ff7d2b..1905606e7 100644 --- a/backend/internal/service/task_billing.go +++ b/backend/internal/service/task_billing.go @@ -20,6 +20,7 @@ type taskBillingCoordinator struct { type taskBillingRepository interface { MarkBillingRunning(orderID string) error SettleBillingOrder(orderID string, providerRequestID string) error + RestoreRefundedBillingOrder(orderID string, providerRequestID string) error RefundBillingOrder(orderID string, errorText string) error MarkBillingUncertain(orderID string, errorText string) error BillingOrder(orderID string) (*model.BillingOrder, error) @@ -49,6 +50,13 @@ func (c *taskBillingCoordinator) SettleBilling(orderID string, providerRequestID return c.repo.SettleBillingOrder(orderID, providerRequestID) } +func (c *taskBillingCoordinator) RestoreRefundedBilling(orderID string, providerRequestID string) error { + if orderID == "" { + return nil + } + return c.repo.RestoreRefundedBillingOrder(orderID, providerRequestID) +} + func (c *taskBillingCoordinator) RefundBilling(orderID string, errorText string) error { if orderID == "" { return nil diff --git a/backend/internal/service/task_billing_test.go b/backend/internal/service/task_billing_test.go index 155ab51ac..3708d536c 100644 --- a/backend/internal/service/task_billing_test.go +++ b/backend/internal/service/task_billing_test.go @@ -11,6 +11,7 @@ import ( type taskBillingRepositoryStub struct { runningCalls []string settleCalls []string + restoreCalls []string refundCalls []string uncertainCalls []string order *model.BillingOrder @@ -29,6 +30,11 @@ func (r *taskBillingRepositoryStub) SettleBillingOrder(orderID string, providerR return nil } +func (r *taskBillingRepositoryStub) RestoreRefundedBillingOrder(orderID string, providerRequestID string) error { + r.restoreCalls = append(r.restoreCalls, orderID+":"+providerRequestID) + return nil +} + func (r *taskBillingRepositoryStub) RefundBillingOrder(orderID string, errorText string) error { r.refundCalls = append(r.refundCalls, orderID+":"+errorText) return nil @@ -57,6 +63,9 @@ func TestTaskBillingCoordinatorEmptyOrderIsNoOp(t *testing.T) { if err := billing.SettleBilling("", "provider-1"); err != nil { t.Fatalf("SettleBilling() error = %v", err) } + if err := billing.RestoreRefundedBilling("", "provider-1"); err != nil { + t.Fatalf("RestoreRefundedBilling() error = %v", err) + } if err := billing.RefundBilling("", "refund"); err != nil { t.Fatalf("RefundBilling() error = %v", err) } @@ -66,7 +75,7 @@ func TestTaskBillingCoordinatorEmptyOrderIsNoOp(t *testing.T) { if billing.BillingFailureRequiresReview("", "task-1", errors.New("timeout")) { t.Fatal("BillingFailureRequiresReview() = true for empty order") } - if len(repo.runningCalls)+len(repo.settleCalls)+len(repo.refundCalls)+len(repo.uncertainCalls) != 0 { + if len(repo.runningCalls)+len(repo.settleCalls)+len(repo.restoreCalls)+len(repo.refundCalls)+len(repo.uncertainCalls) != 0 { t.Fatalf("empty order should not reach repository: %#v", repo) } } diff --git a/docs/design/model-request-protocol-landscape.md b/docs/design/model-request-protocol-landscape.md new file mode 100644 index 000000000..e77ef62e1 --- /dev/null +++ b/docs/design/model-request-protocol-landscape.md @@ -0,0 +1,543 @@ +# LLM、Image、Video 主流请求协议全景与影策兼容性调查 + +> 调查快照：2026-08-31 +> 文档性质：协议调研与现状审计，不是最终统一协议设计稿 +> 目标：为后续“影策通用生成请求标准底座”和 provider 插件体系提供事实清单、差异维度与验收边界。 + +## 1. 结论先行 + +1. **模型名、厂商名、请求协议不是同一个概念。** Wan、Seedance、Veo、Grok、MiniMax H3 是模型或产品族；同一模型可能通过官方原生 API、OpenAI-compatible 网关、NewAPI、fal、Replicate、ComfyUI 或私有中转暴露，实际请求结构完全不同。 +2. **“OpenAI-compatible”通常只表示局部兼容。** 常见情况是只兼容 `/v1/chat/completions` 的路径和基础 `messages`，但不兼容 Responses、工具调用、图片输入、文件、SSE 事件、usage、错误码、JSON Schema、视频任务或图片编辑。 +3. **文本、图片、视频不能共用一个扁平字段表。** 文本主要是消息/内容块与流式事件；图片同时存在 JSON、multipart、同步 base64、异步任务；视频普遍是异步任务，并且参考图的“顺序”和“语义角色”会改变协议结构。 +4. **参考素材必须以语义角色建模，不能只以数组下标建模。** `images[0]` 可能代表首帧、主参考图、被编辑原图或第一张角色图；`images[1]` 可能代表尾帧、第二角色、风格图或 mask。数组下标只能作为稳定顺序，不能承担业务语义。 +5. **插件不能只做字段重命名。** 主流协议需要数组遍历、按 role 过滤、条件分支、对象/数组构造、枚举转换、multipart、签名、轮询、取消、SSE 解析、结果下载和错误归一化。 +6. **影策当前已经有统一请求雏形，但声明式插件能力不足。** `GenerationRequest` 已有 `Images/Videos/Audios/Operation/Extra`，`MediaReference` 也已有 `Role`；但声明式插件看到的 `request.images` 丢失了 `id` 与 `role`，只能写 `request.images.0.url` 之类固定编号。 +7. **MiniMax H3 的现有问题不是单一参数名问题。** 当前前端旧直连与后端内置适配器对 MiniMax 视频的 payload 不一致：分辨率、比例、水印、校验范围、content 结构及顶层 prompt 的处理均有差异；通过中转时还可能再套一层 NewAPI 结构。 + +## 2. 调查范围与分类方法 + +“市面所有协议”没有封闭集合：模型、聚合平台和私有中转持续新增，部分产品没有公开稳定 API。本文采用以下覆盖标准： + +- 国际主流基础模型厂商及云平台； +- 国内主流基础模型和创作模型厂商； +- 影像生成领域常见商业 API； +- OpenAI-compatible、NewAPI、LiteLLM、OpenRouter、fal、Replicate 等聚合/运行时协议； +- ComfyUI、RunningHub、AutoDL 等工作流协议； +- 影策仓库已经出现或社区明确反馈的协议。 + +每个条目区分四层： + +| 层 | 含义 | 示例 | +| --- | --- | --- | +| 模型族 | 生成能力和模型约束 | GPT、Claude、Gemini、Seedance、Wan、Hailuo H3 | +| 原生线协议 | 厂商官方 HTTP/事件协议 | Anthropic Messages、Gemini generateContent、Ark contents tasks | +| 网关协议 | 聚合平台暴露的兼容入口 | NewAPI、OpenRouter、LiteLLM、SiliconFlow | +| 工作流协议 | 提交图或工作流并轮询 | ComfyUI prompt/history/view、RunningHub workflow | + +## 3. 所有协议都要调查的共同维度 + +后续统一底座至少要保存以下事实，不能只保存 `model + prompt + images`： + +| 维度 | 必须表达的内容 | +| --- | --- | +| endpoint | Base URL、版本前缀、模型是否在 path、创建/查询/取消/下载路径 | +| 鉴权 | Bearer、`x-api-key`、`anthropic-version`、`x-goog-api-key`、AK/SK 签名、云 IAM | +| 编码 | JSON、multipart/form-data、二进制上传、文件 ID、URL、data URL、base64 | +| 操作 | text/chat/response、image generation/edit、text/image/reference/frame/video/audio-to-video | +| 内容结构 | messages、input items、contents/parts、content blocks、instances、workflow graph | +| 素材语义 | first frame、last frame、subject/style/reference、mask、control、motion、audio、voice | +| 输出控制 | 尺寸、比例、分辨率档、时长、帧率、数量、质量、seed、水印、音频 | +| 生命周期 | 同步、SSE、异步任务、长操作、回调、轮询间隔、超时、取消 | +| 状态 | queued/running/succeeded/failed/cancelled/expired 及厂商别名 | +| 结果 | 文本块、工具调用、URL、base64、file ID、临时签名 URL、二进制下载 | +| 错误 | HTTP 状态、业务 code、error object、风控、内容审核、配额、可重试性 | +| 计费 | token、图片张数、视频秒数、分辨率、是否带音频、工作流机器时长 | +| 幂等与追踪 | idempotency key、request ID、task ID、operation name、trace ID | +| 能力约束 | 模型级输入组合、数量、格式、大小、时长、地域和版本限制 | + +## 4. LLM 主流协议 + +### 4.1 原生协议族 + +| 协议族 | 典型入口 | 核心请求结构 | 流式/工具 | 关键差异与插件注意事项 | +| --- | --- | --- | --- | --- | +| OpenAI Chat Completions | `POST /v1/chat/completions` | `model`, `messages[]`, `tools`, `tool_choice`, `response_format` | SSE，`choices[].delta`；工具参数按增量拼接 | 多模态位于 message content parts；旧 `max_tokens` 与新字段并存；兼容网关经常只实现子集 | +| OpenAI Responses | `POST /v1/responses` | `input` 字符串或 items、`instructions`、`tools`、`text` | 事件类型化 SSE；输出为 `output[]` items | 与 Chat Completions 不是字段替换关系；包含 reasoning、computer/tool、file、previous response 等状态语义 | +| Anthropic Messages | `POST /v1/messages` | `system` 独立、`messages[]`、content blocks、`max_tokens` | SSE 事件；`tool_use` / `tool_result` block | `anthropic-version` 必需；assistant/user 角色与工具回填方式不同；图片常用 base64/url source | +| Google Gemini generateContent | `.../models/{model}:generateContent` | `contents[].parts[]`, `systemInstruction`, `generationConfig`, `tools` | `streamGenerateContent`；functionCall/functionResponse | role 使用 `user/model`；文件用 `inlineData/fileData`；思考签名和函数回填不能按 OpenAI tool call 直接搬运 | +| Cohere Chat v2 | `POST /v2/chat` | `messages`, `tools`, `response_format`, documents | SSE 事件 | citation/documents、tool plan 和 OpenAI 结构不同；v1/v2 不能混用 | +| AWS Bedrock Converse | `Converse` / `ConverseStream` | `modelId` 在请求目标，`messages`, `system`, `inferenceConfig`, `toolConfig` | AWS event stream | IAM/SigV4；统一包裹多个模型，但 provider-specific 字段进入 `additionalModelRequestFields` | +| AWS Bedrock InvokeModel | `InvokeModel` / `InvokeModelWithResponseStream` | body 完全由目标模型决定 | 可流式 | 只是传输容器，不是统一内容协议；Anthropic、Titan、Llama 等 payload 各异 | +| Vertex AI Gemini | publisher model generateContent | 与 Gemini 接近，资源名和鉴权是 GCP 风格 | 流式、长操作视能力 | OAuth/IAM、project/location、文件 URI 与 AI Studio 不同 | +| Azure OpenAI | `/openai/deployments/{deployment}/...?...api-version=` 或新统一入口 | 大体遵循 OpenAI，但 model/deployment 和 api-version 规则不同 | SSE | endpoint、deployment、API version、Entra ID；不能假设 OpenAI Base URL 拼接规则 | +| 阿里云 DashScope/Qwen 原生 | `/api/v1/services/aigc/...`、Generation/MultimodalConversation | `input` + `parameters` | SSE/异步视 API | 同一平台同时提供原生协议和 OpenAI-compatible；图片/视频任务使用 `X-DashScope-Async`、task-id | +| 火山方舟 Ark 原生 | `/api/v3/...` | Chat/Responses/图片/contents task 分属不同结构 | SSE、异步任务 | endpoint/model ID、region、签名/Token；Seedance 全模态 content 有 role 语义 | +| 百度智能云千帆 | model/endpoint 路径或兼容入口 | messages 或模型专用 input | SSE | access token/IAM、服务实例 endpoint；不同代际接口并存 | +| 腾讯混元 | 腾讯云 API action/version 或兼容入口 | 云 API 公共参数 + messages/input | 流式/异步 | TC3-HMAC-SHA256 签名；Action/Version/Region 不是普通 REST Base URL | +| MiniMax 文本原生 | `/v1/text/chatcompletion_v2` 等版本化接口 | messages、tokens_to_generate、工具/函数字段随代际变化 | SSE | 海内外域名、旧 GroupId 参数与新版 Bearer 方式并存；另有 Anthropic/OpenAI 兼容入口时不能混认 | +| Ollama | `POST /api/chat`, `/api/generate` | `messages` 或 `prompt`, `options`, `format`, `keep_alive` | 默认 NDJSON | 不是 SSE；本机模型、图片字段、结构化输出和工具支持依版本变化 | + +### 4.2 OpenAI-compatible 厂商与网关 + +以下产品通常暴露 `/v1/chat/completions`，但兼容度需要按能力矩阵验证： + +| 厂商/网关 | 主要定位 | 常见扩展或差异 | +| --- | --- | --- | +| xAI Grok | 原生模型 API | 文本高度接近 OpenAI；实时搜索、图片/视频是独立能力和独立参数 | +| DeepSeek | 模型 API | Chat Completions 风格；reasoner 可能返回 `reasoning_content`，上下文续写规则需保留 | +| Mistral | 模型 API | OpenAI 风格但工具、JSON、Fim、OCR/Agents 有独立入口 | +| 月之暗面 Kimi | 模型 API | OpenAI-compatible；长上下文、联网/文件能力随产品入口不同 | +| 智谱 GLM | 模型 API | Chat Completions 风格；`web_search`、thinking、视频/图片有独立异步协议 | +| 百川 | 模型 API | OpenAI 风格子集，模型参数和工具能力需逐模型验证 | +| 零一万物 Yi | 模型 API | OpenAI 风格子集；以实际开放平台版本为准 | +| SiliconFlow | 聚合/推理平台 | Chat、Images、Audio、Rerank 并列；模型来自多个厂商，非文本协议不等于 OpenAI 官方 | +| Together AI | 聚合/推理平台 | Chat/Completions/Images/Embeddings；响应和模型扩展字段不同 | +| Groq | 高速推理平台 | Chat Completions 为主；部分 OpenAI 参数不支持或受模型限制 | +| Fireworks AI | 推理平台 | Chat/Completions/OpenAI-compatible；支持 provider 扩展和部署名 | +| NVIDIA NIM | 模型微服务 | 多数文本 NIM 暴露 OpenAI 风格；视觉、检索、生成模型可能是专用 API | +| OpenRouter | 多厂商路由网关 | `provider`, `transforms`, route、fallback；响应会带 provider/usage 扩展，模型 ID 含厂商前缀 | +| LiteLLM Proxy | 协议翻译网关 | OpenAI 入口转多厂商；支持预算、路由、fallback，但不保证目标厂商全部原生特性可透传 | +| NewAPI/OneAPI | 聚合、计费和协议转换 | 同时可能提供 OpenAI、Claude、Gemini、图片和多种视频入口；版本/渠道配置会改变最终 payload | +| vLLM | 自托管推理服务 | OpenAI-compatible server；支持参数取决于模型模板和启动参数 | +| LocalAI | 自托管兼容服务 | 文本、图片、音频等多能力共域名，但后端实现和字段支持由模型配置决定 | + +### 4.3 LLM 不能被抹平的差异 + +- system 指令可能在 `messages`、独立 `system`、`instructions` 或 `systemInstruction`。 +- 内容块命名不同：`text`、`input_text`、`image_url`、`inlineData`、`document`、`tool_use`。 +- 工具调用 ID、参数类型、并行工具调用、tool result 回填位置不同。 +- 流式不是统一的 `data: {json}`：OpenAI SSE、Anthropic 类型化事件、Gemini JSON/SSE、Ollama NDJSON、AWS event stream 均不同。 +- structured output 可能是 `response_format`、`text.format`、`generationConfig.responseSchema` 或模型专用参数。 +- reasoning 可能是隐藏计费 token、可见字段、独立 block 或签名数据，不能只抽成普通文本。 + +## 5. Image 主流协议 + +| 厂商/协议 | 创建入口与编码 | 参考图/编辑表达 | 返回与生命周期 | 关键差异 | +| --- | --- | --- | --- | --- | +| OpenAI Images | `/v1/images/generations` JSON；`/v1/images/edits` multipart | edit 使用一个或多个 image 文件，可有 mask | 通常同步，URL 或 `b64_json` | generation 与 edit 是两个 transport；尺寸、quality、background、output format 随模型变化 | +| xAI Grok Images | `/v1/images/generations` JSON | 常见 `image` 作为编辑/参考输入 | 同步 URL/base64 风格 | `aspect_ratio`、`resolution` 与 OpenAI `size/quality` 不同；不能只复用 OpenAI Images 字段 | +| Gemini 原生图片生成 | `generateContent` JSON | 图片作为 `inlineData/fileData` part，与 prompt 混排 | candidates.parts 内 inline image | 图片是多模态内容结果，不是 `data[]`；可能同时返回文本与图片 | +| Vertex Imagen | `:predict` / 专用生成接口 | `instances` + `parameters`；编辑/扩图有专用字段 | 同步或长操作视能力 | GCP 资源路径、GCS URI、safety/person generation 配置 | +| 火山方舟 Seedream/图片 | `/api/v3/images/generations` JSON | `image` 可为字符串或数组；连续图有专用配置 | 同步 URL/base64 | `size` 可能接受比例、像素或档位；watermark 和 sequential generation 是扩展字段 | +| 即梦官方图片 | `CVSync2AsyncSubmitTask` + `CVSync2AsyncGetResult` | `image_urls`, `req_key` | 异步 task | AK/SK 签名、提交和查询均 POST；不能套普通 Bearer + GET poll | +| 阿里云通义万相/Wanx | DashScope 图像生成 task | `input.prompt`、参考图 URL、`parameters` | 常见异步 task-id | 需 `X-DashScope-Async: enable`；结果 URL 有有效期 | +| MiniMax Image | MiniMax 图片生成入口 | prompt、reference/image、subject reference 随模型版本 | 同步或异步视接口 | 模型版本字段和参考图能力变化快，应按模型 profile 管理 | +| Kling Image | Kling 开放平台图片任务 | image/reference、strength、negative prompt 等 | 异步 task | access key 签名/JWT、task status、结果数组 | +| 智谱 CogView | `/images/generations` 风格入口 | 部分版本支持 image input/quality/size | 同步或任务式视版本 | OpenAI-like 但模型字段和返回 URL 生命周期不同 | +| 腾讯混元图片 | 腾讯云 Action API | prompt、resolution、style、reference | 异步 task | 云签名、Action/Version；错误码和 task 查询是腾讯云格式 | +| 百度千帆图片 | 千帆图片生成 API | prompt、参考图/控制参数因模型而异 | 同步或异步 | access token/IAM，模型服务路径和审核错误需单独解析 | +| Stability AI | `/v2beta/stable-image/...` 多为 multipart | image、mask、control image、strength | 常见同步二进制或 JSON | endpoint 按 generate/edit/control/upscale 划分；`Accept` 决定输出编码 | +| Black Forest Labs FLUX API | 创建 task JSON | prompt、image、mask、width/height 或比例 | 异步 polling URL | create 返回 task/polling 信息；输出 URL 临时；模型参数差异大 | +| Ideogram | `/v1/ideogram-v3/generate` multipart 等 | image、mask、style/reference | 同步/任务视版本 | magic prompt、rendering speed、style type、color palette 是专用参数 | +| Recraft | `/v1/images/generations`、编辑/矢量专用入口 | image、mask、style id、controls | 同步 URL | 支持 raster/vector；style/substyle 和尺寸枚举不可丢失 | +| Adobe Firefly | Firefly Services 版本化接口 | reference image、mask、structure/style reference | 任务/同步视版本 | Adobe IMS 鉴权、Content Credentials、存储/结果策略 | +| Runway Image | Runway API image endpoint | reference images 常带 tag/URI | 异步 task | 与 Runway video 共用 task lifecycle，但请求 schema 按模型区分 | +| fal.ai Queue | `queue.fal.run/{model}` | schema 由模型定义，URL/base64/file upload | queue submit/status/result | fal 是运行时协议，不是模型协议；每个 endpoint 的 input schema 必须保留 | +| Replicate Predictions | `/v1/predictions` | `version/model` + 任意 `input` | 异步 prediction/webhook | input/output 完全由模型 schema 定义；支持 cancel/webhook，不能硬编码图片字段 | +| ComfyUI | `/prompt` 提交 workflow JSON | 图片先上传，再把文件名映射到节点 input | `/history/{prompt_id}` + `/view` | 请求是节点图，不是固定模型字段；映射目标是 node id + input name | + +### 5.1 无稳定公开通用 API 的重要产品 + +- **Midjourney**：长期没有面向所有开发者的稳定官方通用生成 API；市场上的 `/imagine`、Discord 自动化和第三方 task API 属于不同供应商协议，必须明确来源，不能命名成“Midjourney 官方协议”。 +- **部分消费级图片产品**：Leonardo、Krea、即梦 Web、Canva 等可能有合作 API、企业 API 或第三方封装；如果没有公开稳定契约，应标为 provider-specific，不纳入通用字段猜测。 + +## 6. Video 主流协议 + +视频协议的核心不是“POST 一个 prompt”，而是**操作语义 + 参考媒体角色 + 异步生命周期**。 + +| 厂商/协议 | 创建/查询 | 典型请求形态 | 参考素材语义 | 结果与注意事项 | +| --- | --- | --- | --- | --- | +| OpenAI Videos/Sora | `POST /v1/videos`，`GET /v1/videos/{id}`，另有 content/cancel | multipart：model、prompt、seconds、size、`input_reference` | 参考图通常是上传文件；不是 JSON `image_url` | 异步 task；完成后可能需单独下载 content；OpenAI-compatible 网关未必支持 multipart | +| xAI Video | `POST /v1/videos/generations`，`GET /v1/videos/{request_id}` | JSON：model、prompt、duration、aspect_ratio、resolution | 单起始图常用 `image:{url}`；角色参考用 `reference_images:[{url}]` | 首帧与角色参考不能混为一个数组；尾帧支持需按当前官方能力确认 | +| Gemini Veo | `models/{model}:predictLongRunning`，查询 `operations/{name}` | `instances[]` + `parameters` | 常见单图作为 instance.image；不同版本可能有 reference images/last frame 扩展 | Google long-running operation；结果位于嵌套 response，文件需下载/复制 | +| Vertex AI Veo | Vertex publisher model 长操作 | instances/parameters + GCS URI | 首帧、尾帧、参考资产能力按 Veo 版本 | project/location/IAM；与 Gemini Developer API 路径不同 | +| 火山方舟 Seedance | `POST /api/v3/contents/generations/tasks`，GET task | `content[]` 混合 text/image_url/video_url/audio_url，媒体 block 带 `role` | `first_frame`、`last_frame`、`reference_image`、`reference_video`、`reference_audio` | 全模态组合、数量和时长有限制；纯音频组合可能非法；支持 generate_audio/watermark | +| Seedance 兼容 `/videos` | `POST /v1/videos` 或供应商自定义 `/videos` | prompt + `image_url` / `image_urls` / `reference_image_urls` + media arrays | 单主图、首尾帧数组、角色参考数组是不同字段 | 这是网关约定，不等于方舟原生 content 协议；同名 Seedance 模型需选正确 profile | +| 即梦官方视频 | SubmitTask + GetResult，均为签名 JSON | `req_key`, prompt, image_urls, duration/ratio/resolution | 常按图片顺序表达关键帧 | 异步、AK/SK；模型 req_key 决定字段能力 | +| MiniMax/Hailuo H3 | `POST /v2/video_generation`，`GET /v2/query/video_generation/{id}` | `content[]` text/image/video/audio blocks；另有 duration/resolution/ratio/watermark 等 | 图片 block 需 role；首尾帧模式不能与普通 reference_image 混用 | H3 是全模态视频协议，不应退化为 `image_urls[0/1]`；中转可能改字段名或再包一层 | +| 阿里 Wan/通义万相视频 | DashScope video synthesis task | `model`, `input`, `parameters`；image_url/first_frame/last_frame 随模型 | 文生、图生、首尾帧、参考视频需按 Wan 版本区分 | 常用异步 header、task-id 查询；Wan 3.x 仍应按官方当前模型 schema 建 profile，不以“Wan3”推断协议 | +| Kling Video | 创建 video task + 查询 task | text2video/image2video/multi-image 等独立 endpoint/schema | image、tail image、elements/subject、motion brush 等 | 模型版本和模式决定字段；JWT/签名；结果 URL 和 task status | +| Agnes Video 2.5 | `POST /v1/videos`，`GET /agnesapi?video_id=...&model_name=...` | JSON，`mode=text|keyframe|reference`；seconds 是字符串；size 是档位 | keyframe 使用 first_frame/last_frame；reference 使用 images/videos/audios | 2.5、2.5 Flash、V2.0 是同厂商不同协议 profile；Flash 限制更严格 | +| Agnes Video V2.0 | `POST /v1/videos` + Agnes 查询 | image 或 `extra_body.image`、mode/num_frames/frame_rate | 关键帧数组，无角色/风格参考模式 | duration 需换算帧数；不能照搬 2.5 的 seconds/size/aspect_ratio | +| Runway | `/v1/...` 创建，task 查询/取消 | 模型专用 JSON，promptImage/promptText、ratio/duration | 首帧、尾帧、references 按 generation 类型 | 标准异步 task；状态、失败原因和 output URL 统一程度较高 | +| Luma Dream Machine | `/dream-machine/v1/generations` 等 | prompt、model、keyframes 对象 | `frame0/frame1` 等 keyframes，可扩展图像/视频 | 异步 generation；状态和 assets 嵌套 | +| Vidu | Vidu 开放平台 task API | text2video/image2video/reference2video | reference images、first frame、subject consistency | 异步；不同模型/模板参数独立 | +| PixVerse | 平台 video generation task | prompt、model、duration/quality、image | image-to-video、transition/effects 等 | 异步；模板/effect 字段不能塞入全局标准字段 | +| Pika | 官方/合作 API 或聚合平台 endpoint | 依接入渠道而异 | image/video/reference/effect | 市面常见的是 fal/Replicate/第三方协议；必须记录实际 provider，不应假称统一 Pika 原生协议 | +| 智谱 CogVideoX | 视频生成 task API | model、prompt、image_url、quality/with_audio 等 | 常见单图图生视频 | 异步 task；字段以开放平台当前版本为准 | +| 腾讯混元视频 | 腾讯云 Action task | 云 API 参数 + prompt/image | 图生/首帧等按 Action | TC3 签名，创建和查询 Action 分离 | +| 百度视频生成 | 千帆/智能云对应模型 task | 模型专用 input | 参考图/视频按模型 | 异步、审核和错误码需单独适配 | +| Novita Video | `/v3/video/create`，`/v3/async/task-result` | `model_name`, prompt, duration, aspect_ratio, image_url | 常见单起始图 | 适合作为简单异步协议，不等于支持角色参考 | +| NewAPI Channel 1 媒体任务 | `POST /v1/videos`，GET `/v1/videos/{id}` | `{model,input:{prompt,media[]},parameters:{...}}` | `media[].type` 表达 first_frame/last_frame/reference_image/reference_video/reference_voice | 是 NewAPI 的统一媒体包装之一，不是 OpenAI Videos | +| NewAPI Channel 2 Video Generations | `POST /v1/video/generations`，GET 同路径 task | 扁平 JSON：seconds、aspect_ratio、resolution、image_urls/video_urls/audio_urls | 常以数组和顺序表达，role 能力取决于具体版本 | 与 Channel 1 不兼容；同一 NewAPI 部署可能因版本或渠道插件行为不同 | +| fal.ai Queue | submit/status/result | 每个模型自定义 schema | 由模型 endpoint schema 决定 | queue runtime 统一，模型请求不统一 | +| Replicate Predictions | predictions + webhook/cancel | 任意 model input | 由模型版本 schema 决定 | 可统一生命周期，不能统一字段名 | +| ComfyUI | `/prompt`, `/history`, `/view` | 完整 workflow graph | 素材上传后映射节点 input；首尾帧取决于 workflow | 插件应映射工作流 schema，不应伪装成固定视频 provider | +| RunningHub/AutoDL 工作流 | workflow/app task create + query | workflow ID + node/input override | 由工作流字段发现结果决定 | 需要字段发现、可覆盖性、安全上传、工作流版本，不是普通 REST 字段映射 | + +## 7. 参考素材语义：必须成为通用底座的一等公民 + +### 7.1 建议保留的角色词汇 + +| kind | role 示例 | 说明 | +| --- | --- | --- | +| image | `first_frame` | 视频起始关键帧 | +| image | `last_frame` | 视频结束关键帧 | +| image | `reference_image` | 泛化角色/主体/风格参考，不能自动当首帧 | +| image | `subject_reference` | 主体一致性参考；有些协议还需要 subject id/type | +| image | `style_reference` | 风格参考，可能带 weight | +| image | `edit_source` | 图片编辑原图 | +| image | `mask` | 编辑蒙版，与普通 reference 不同 | +| image | `control` | pose/depth/canny/layout 等控制图 | +| video | `reference_video` | 动作、镜头、风格或重绘参考 | +| video | `motion_reference` | 明确的运动参考 | +| audio | `reference_audio` | 音乐/环境/节奏参考 | +| audio | `reference_voice` | 音色/说话人参考 | + +建议媒体项至少具备： + +```json +{ + "id": "canvas-node-or-resource-id", + "kind": "image", + "role": "first_frame", + "source": { + "type": "resource|url|data|file", + "value": "...", + "mimeType": "image/png" + }, + "order": 0, + "label": "可选的人类可读名称", + "weight": 1, + "metadata": {} +} +``` + +### 7.2 为什么 `0`、`1` 编号无法承担语义 + +- 单图 + `image_to_video` 时，`0` 通常是首帧。 +- 单图 + `reference_to_video` 时，`0` 是角色/风格参考，不能发送到 `first_frame`。 +- 两图可能是首尾帧，也可能是两个角色参考，也可能是编辑原图 + 风格图。 +- 加入参考视频/音频后，图片通常自动变成 reference，而不是 frame。 +- 用户在画布重连或排序后，下标可能变化；节点 ID 和显式 role 才是稳定语义。 +- 有些上游要求 prompt 内使用 `@Image1`、`image_0` 或其他占位符；这是 provider 渲染规则，不应污染通用请求。 + +因此通用层应保存 `order`，插件应消费 `role`；只有协议明确以顺序定义首尾帧时，插件才把 role 排序后生成数组。 + +## 8. 影策当前实现审计 + +### 8.1 已有的正确基础 + +后端 `backend/internal/protocol/types.go` 已定义： + +- `GenerationRequest`：model、prompt、images、videos、audios、duration、aspectRatio、resolution、quality、generateAudio、watermark、imageCount、operation、extra； +- `MediaReference`：id、url、dataUrl、kind、role、ephemeral； +- `RequestSpec`、`CreateResult`、`PollResult`、统一 status/result； +- provider registry 和内置 adapter； +- 声明式插件 create/poll/cancel/response path mapping。 + +这说明不需要从零创建“统一请求”，而是需要补齐语义和插件执行能力。 + +### 8.2 声明式插件的关键缺口 + +#### 缺口 A：`role` 和 `id` 在插件视图中丢失 + +`MediaReference` 本身有 `ID`、`Role`，但 `manifestMediaValues()` 只暴露： + +```text +url, dataUrl, kind, ephemeral +``` + +因此插件无法读取： + +```text +request.images.0.role +request.images.0.id +``` + +只能固定读取： + +```text +request.images.0.url +request.images.1.url +``` + +这正是“画布参考图带 0/1 编号后无法直接插件映射”的结构性根因之一。 + +#### 缺口 B：不能动态遍历或按 role 过滤 + +当前 `fields` 是“目标 path -> 单一表达式”，支持固定路径和少量字符串变换，不支持： + +- `map(request.images, ...)`； +- `filter(role == first_frame)`； +- `first(role == last_frame)`； +- 根据 operation/model 选择完全不同的 body； +- 动态生成 `content[]` block； +- 把同一数组分别映射为 `first_frame`、`last_frame`、`reference_images`。 + +#### 缺口 C：声明式 runtime 只允许 JSON body + +`executeProtocolRequest()` 对非 `application/json` 直接报错，因此以下协议不能仅靠当前声明式插件实现： + +- OpenAI Images edits multipart； +- OpenAI Videos multipart `input_reference`； +- Stability/Ideogram 等 multipart 图片 API； +- 需要直接上传二进制或文件 part 的协议。 + +#### 缺口 D：transport 和解析表达能力不足 + +当前还不能完整声明： + +- 自定义 query/header/body 签名和 AK/SK； +- SSE、NDJSON、AWS event stream； +- 上传文件 -> 获得 file id -> 创建任务的多阶段流程； +- webhook、幂等 header、Retry-After； +- cancel 的 body/query 变体； +- 完成后再请求 content/download endpoint； +- 状态码到 retryable/permanent/moderation/quota 的错误分类； +- 临时 URL 有效期和需要平台代理下载的结果。 + +### 8.3 当前 role 传递还存在的边界 + +- 前端 `resolveVideoImageReferences()` 在旧创作路径中会按数量推断：单图首帧、双图首尾帧、其余参考图。 +- 后端 `protocolVideoImageReferences()` 主要依赖 `videoStartFrameNodeId`、`videoEndFrameNodeId` 和 `videoEditOperation`；没有显式 metadata 时 role 可能为空。 +- 即使后端 request 中 role 正确，声明式 manifest projection 仍会把 role 丢掉。 + +后续应明确：**role 在画布/任务创建阶段确定；协议插件只能转换，不应重新猜业务语义。** + +## 9. MiniMax H3 / 中转参数问题专项 + +### 9.1 当前前端旧直连 payload + +`web/src/services/api/video-provider-minimax.ts` 当前构造： + +```json +{ + "model": "...", + "content": [ + { "type": "text", "text": "..." }, + { "type": "image_url", "image_url": { "url": "..." }, "role": "first_frame|last_frame|reference_image" } + ], + "resolution": "768P|2K", + "duration": 4, + "ratio": "adaptive|16:9|...", + "aigc_watermark": false +} +``` + +并且有以下前端约束： + +- 图片最多取 9 张、视频 3 个、音频 3 个； +- duration 归一到 4-15 秒； +- 首尾帧模式比例强制 `adaptive`； +- 首尾帧模式禁止混入普通 `reference_image`； +- 本地素材转换为 data URL 或要求公网 URL。 + +### 9.2 当前后端内置 MiniMax adapter + +`backend/internal/protocol/builtin.go` 当前构造： + +```json +{ + "model": "...", + "prompt": "...", + "duration": 6, + "content": [ + { "type": "text", "text": "..." }, + { "type": "image_url", "image_url": { "url": "..." }, "role": "..." } + ] +} +``` + +它只从 `Extra` 合并 `resolution` 和 `aspect_ratio`，但 `protocolRequestFromInput()` 把比例和分辨率放在 `GenerationRequest.AspectRatio/Resolution`，没有放入 `Extra`。因此当前后端路径存在以下风险： + +1. `Resolution` 没有实际写入 MiniMax body； +2. `AspectRatio` 没有实际写入 body； +3. 字段候选写成 `aspect_ratio`，而旧直连使用 `ratio`； +4. `aigc_watermark` 没有映射； +5. `GenerateAudio` 没有映射； +6. 前端已有的数量、组合、duration、frame mode 校验没有在后端 adapter 等价实现； +7. 后端额外同时发送顶层 `prompt` 和 content text，是否被目标中转接受取决于中转实现。 + +### 9.3 通过 NewAPI/其他中转时的第二层差异 + +MiniMax H3 可能出现至少三种入口： + +1. MiniMax 原生 `/v2/video_generation` content blocks； +2. NewAPI Channel 1：`input.media[] + parameters`； +3. NewAPI Channel 2：`image_urls/video_urls/audio_urls + seconds/aspect_ratio/resolution`； +4. 中转商自定义“OpenAI video”入口：`/v1/videos` multipart 或 JSON； +5. fal/Replicate 模型 endpoint：完全由该 endpoint schema 决定。 + +因此“模型是 MiniMax H3”不能决定协议。正确选择键应至少是： + +```text +provider protocol profile + model capability profile + operation +``` + +而不是： + +```text +if model name contains minimax/h3 +``` + +## 10. 后续通用请求标准底座必须具备的能力 + +本节只列设计约束，不在本文冻结最终 JSON Schema。 + +### 10.1 通用请求层 + +- 显式 `capability` 与 `operation`； +- 多模态 `inputs[]`，每项保存 kind、role、source、mime、order、metadata； +- 输出配置按语义拆开：aspect ratio、pixel size、resolution tier、duration、fps、count、quality； +- `providerOptions` 必须命名空间化，例如 `providerOptions.minimax`，避免任意 `Extra` 污染公共字段； +- 模型 capability profile 声明输入组合、数量、格式、范围和默认值； +- 不在通用层把 `720P`、`1280x720`、`16:9` 混成同一个 `size` 字符串。 + +### 10.2 provider 插件映射层 + +至少需要安全的声明式能力： + +- 对象、数组、常量构造； +- `map/filter/find/first/count/sortByRole`； +- `if/switch/coalesce`； +- 枚举、大小写、数值、秒/帧、比例/像素/档位转换； +- 按 model/operation/capability profile 分支； +- JSON、multipart、form-urlencoded、binary； +- path/query/header/body 模板； +- 文件上传和临时公网 URL 策略； +- create/poll/cancel/download 多阶段 workflow； +- SSE/NDJSON/event-stream parser； +- response path + 状态映射 + 错误分类； +- webhook、幂等和重试策略； +- secret 字段隔离与日志脱敏。 + +### 10.3 插件不应负责的事项 + +- 不应从图片数量猜首尾帧业务意图； +- 不应访问浏览器中的第三方密钥； +- 不应把本地 blob URL直接发给远端； +- 不应绕过后端任务、计费、配额、取消和审计生命周期； +- 不应为适配某厂商修改全局通用字段语义。 + +## 11. 建议的插件拆分方式 + +插件应按“线协议 profile”拆分，而不是按营销品牌笼统拆分： + +| 推荐插件 ID 示例 | 覆盖范围 | +| --- | --- | +| `openai-chat-completions` | OpenAI Chat 线协议及严格兼容网关 | +| `openai-responses` | Responses 事件和 item 协议 | +| `anthropic-messages` | Claude Messages/tool/SSE | +| `google-gemini-generate-content` | Gemini contents/parts/functions | +| `openai-images` | generations + multipart edits | +| `google-gemini-image` | generateContent 图片输入输出 | +| `openai-videos` | multipart create/task/content/cancel | +| `xai-video` | xAI JSON video + reference_images | +| `volcengine-ark-seedance` | Ark contents task + multimodal roles | +| `minimax-hailuo-video-v2` | MiniMax `/v2/video_generation` | +| `dashscope-wan-video` | DashScope async Wan task | +| `newapi-media-task-v1` | NewAPI Channel 1 包装 | +| `newapi-video-generations-v1` | NewAPI Channel 2 包装 | +| `agnes-video-25` | 2.5/Flash profile | +| `agnes-video-v20` | frame-based V2.0 profile | +| `replicate-prediction` | prediction lifecycle + model schema | +| `fal-queue` | queue lifecycle + endpoint schema | +| `comfyui-workflow` | workflow graph/upload/history/view | + +一个安装包可以贡献多个 provider profile，但运行时必须能区分每个 profile 的 endpoint、transport、能力和模型约束。 + +## 12. 后续设计与验收清单 + +### P0：先消除当前阻塞 + +- [ ] 声明式插件暴露 media `id`、`role`、`mimeType`、`metadata`； +- [ ] 支持按 role 选取首帧、尾帧和 reference 数组； +- [ ] 修正后端 MiniMax H3 的 resolution、ratio、watermark 和校验映射； +- [ ] 为 MiniMax 原生、NewAPI Channel 1、Channel 2 分别建独立 profile； +- [ ] 加入 payload snapshot 测试，覆盖 0/1/多图、首尾帧、角色参考、图+音视频组合。 + +### P1：通用插件底座 + +- [ ] JSON 数组/对象构造和条件表达式； +- [ ] multipart 与文件 part； +- [ ] create/poll/cancel/download 工作流； +- [ ] 状态和错误分类； +- [ ] provider capability profile 和参数 UI schema； +- [ ] 统一 conformance fixture：请求输入 -> 期望上游 payload -> 上游响应 -> 统一结果。 + +### P2：主流协议插件化 + +- [ ] LLM：OpenAI Chat、Responses、Anthropic、Gemini； +- [ ] Image：OpenAI、Gemini、Ark/Seedream、Stability、FLUX； +- [ ] Video：OpenAI、xAI、Veo、Seedance、MiniMax、Wan、Kling、Agnes、Runway； +- [ ] Gateway：NewAPI、LiteLLM/OpenRouter 类兼容入口； +- [ ] Workflow：ComfyUI、RunningHub/AutoDL。 + +## 13. 官方与一手资料入口 + +以下链接用于继续核对当前版本。部分官方站点为 JavaScript 文档或需要登录，字段上线后仍应以实际账号下的最新 API Reference、OpenAPI schema 和真实响应 fixture 为准。 + +### LLM + +- OpenAI API Reference: +- Anthropic Messages API: +- Gemini API: +- xAI Docs: +- Cohere v2 Chat: +- Mistral API: +- DeepSeek API: +- AWS Bedrock Runtime: +- Azure OpenAI: +- Vertex AI generative AI: +- DashScope/Model Studio: +- 火山方舟: +- Ollama API: +- vLLM OpenAI-compatible server: +- LiteLLM Proxy: +- OpenRouter API: +- NewAPI: + +### Image + +- OpenAI Images: +- Gemini image generation: +- Stability API: +- Black Forest Labs API: +- Ideogram API: +- Recraft API: +- Adobe Firefly Services: +- Replicate HTTP API: +- fal model APIs: + +### Video / Workflow + +- OpenAI Videos: +- Gemini/Veo video generation: +- Seedance 产品与能力: +- MiniMax API Platform: +- Wan: +- Runway API: +- Luma API: +- Agnes Docs: +- ComfyUI API: +- NewAPI source: + +## 14. 本仓库一手证据 + +- `backend/internal/protocol/types.go`：当前统一请求、媒体 role、任务结果合同； +- `backend/internal/protocol/manifest.go`：声明式字段映射、固定数组路径、media projection 和 JSON-only runtime； +- `backend/internal/protocol/builtin.go`：现有 OpenAI/Claude/Gemini/图片/视频内置 adapter； +- `backend/internal/service/provider.go`：画布任务到 `GenerationRequest` 的转换和 role 解析； +- `web/src/services/api/video-reference-roles.ts`：前端首帧、尾帧、参考图角色推断； +- `web/src/services/api/video-provider-minimax.ts`：前端旧 MiniMax 直连 payload； +- `web/src/services/api/video-provider-seedance.ts`：Seedance 两类入口及全模态素材映射； +- `web/src/services/api/video-provider-agnes.ts`：Agnes 版本分支； +- `web/src/pages/plugins/plugin-development-guide.md`：当前插件字段表达能力说明。 + +## 15. 调查结论 + +- **现状是：** 影策已有统一任务生命周期和初版 `GenerationRequest`，内置 adapter 能处理一部分复杂协议；上传的声明式插件仍主要是固定 JSON path 映射。 +- **关键约束是：** 协议兼容的难点集中在媒体 role、动态数组、条件结构、transport、异步生命周期和模型级能力，不是 endpoint 和字段改名。 +- **此前不明显但现在确认的是：** `MediaReference.Role` 已存在，却在 manifest projection 中丢失；MiniMax 后端 adapter 与前端旧直连 payload 也存在实质差异。 +- **基于以上，判断是：** 下一阶段应先升级通用请求和插件执行底座，再批量制作主流 provider 插件；如果直接继续新增固定 `fields` 映射，会不断把协议差异硬编码进宿主，并重复出现参考图编号和 H3 参数问题。 diff --git a/docs/index.md b/docs/index.md index 3be1eee4b..f94dac8b4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,6 +14,8 @@ - [肖像权可识别性排查画布插件实施规格](design/portrait-clearance-canvas-plugin.mdx)：TypeScript 重写、画布节点与全屏工作台、项目模型复用、本机 ONNX/百度识图/候选去重、任务合同、报告和验收方案。 +- [LLM、Image、Video 主流请求协议全景与影策兼容性调查](design/model-request-protocol-landscape.md)：主流原生协议、聚合网关、图片/视频异步任务、参考素材 role、当前插件映射缺口与 MiniMax H3 专项审计。 + ## 本地协作文档（不随仓库分发） - [beautifului 创作设计](beautifului-creation-design.md)：本地设计参考，未纳入版本控制。 diff --git a/plugin-packages/adobe-firefly.yingce-plugin b/plugin-packages/adobe-firefly.yingce-plugin new file mode 100644 index 000000000..2554ea10f Binary files /dev/null and b/plugin-packages/adobe-firefly.yingce-plugin differ diff --git a/plugin-packages/agnes-video-25.yingce-plugin b/plugin-packages/agnes-video-25.yingce-plugin new file mode 100644 index 000000000..9e2be1bea Binary files /dev/null and b/plugin-packages/agnes-video-25.yingce-plugin differ diff --git a/plugin-packages/agnes-video-v20.yingce-plugin b/plugin-packages/agnes-video-v20.yingce-plugin new file mode 100644 index 000000000..f2061d362 Binary files /dev/null and b/plugin-packages/agnes-video-v20.yingce-plugin differ diff --git a/plugin-packages/anthropic-messages.yingce-plugin b/plugin-packages/anthropic-messages.yingce-plugin new file mode 100644 index 000000000..7f32d4382 Binary files /dev/null and b/plugin-packages/anthropic-messages.yingce-plugin differ diff --git a/plugin-packages/autodl-comfyui.yingce-plugin b/plugin-packages/autodl-comfyui.yingce-plugin index d9820fe1b..5adb0bce3 100644 Binary files a/plugin-packages/autodl-comfyui.yingce-plugin and b/plugin-packages/autodl-comfyui.yingce-plugin differ diff --git a/plugin-packages/autodl-comfyui/manifest.json b/plugin-packages/autodl-comfyui/manifest.json deleted file mode 100644 index 59c713651..000000000 --- a/plugin-packages/autodl-comfyui/manifest.json +++ /dev/null @@ -1,312 +0,0 @@ -{ - "apiVersion": "yingce.plugin/v1", - "id": "autodl-comfyui", - "name": "AutoDL ComfyUI 视频与音频", - "version": "1.0.3", - "author": "影策社区", - "description": "通过 AutoDL.Art ComfyUI 工作流 API 接入异步视频与音频生成，支持 H3 视频系列及 IndexTTS 语音工作流。", - "documentation": "# AutoDL ComfyUI 工作流\n\n## 接口\n\n- 创建任务：`POST /api/v1/comfyui/comfyui_workflow/{workflow_id}`\n- 查询结果：`GET /api/v1/comfyui/comfyui_workflow/result/{task_id}`\n\n## 鉴权\n\n在渠道密钥中配置 AutoDL ComfyUI 令牌（Authorization: ）。\n\n## 工作流清单（8 个工作流）\n\n1. `minimax_h3_image_audio_to_video_v2_15s` - H3 多图多音频生视频 15 秒\n2. `minimax_h3_lightx2v_v5_15s` - H3 多图生视频 15 秒\n3. `minimax_h3_image_audio_to_video_v2` - H3 多图多音频生视频\n4. `minimax_h3_image_audio_to_video` - H3 图生视频音频同步\n5. `minimax_h3_lightx2v_v5` - H3 多图参考生视频\n6. `minimax_h3_lightx2v_no_pic` - H3 文生视频\n7. `minimax_h3_lightx2v` - H3 首尾帧生成视频\n8. `indextts2-v1` - IndexTTS2 语音\n", - "permissions": [ - "generation.run", - "media.read" - ], - "configuration": { - "fields": [ - { - "name": "apiKey", - "type": "secret", - "label": "AutoDL Token", - "required": true, - "description": "填写 AutoDL 大模型令牌管理中创建的 ComfyUI 分组 Token。" - } - ] - }, - "contributes": { - "providers": [ - { - "id": "autodl-comfyui", - "label": "AutoDL ComfyUI 视频", - "capabilities": [ - "video" - ], - "scopes": [ - "admin.system-channel", - "user.custom-channel", - "canvas", - "creation" - ], - "baseUrl": "https://autodl.art", - "requiresPublicMediaUrls": true, - "auth": { - "type": "header", - "field": "apiKey", - "header": "Authorization" - }, - "parameters": [ - { - "name": "prompt", - "type": "string", - "required": true, - "mapping": "prompt", - "description": "视频生成提示词。" - }, - { - "name": "duration", - "type": "integer", - "mapping": "duration", - "description": "视频时长（秒）。" - }, - { - "name": "resolution", - "type": "string", - "mapping": "resolution", - "description": "输出分辨率档位，如 480p竖、768p竖、1080p竖、480p横、768p横、1080p横、480p(1:1)、768p(1:1) 等。" - } - ], - "create": { - "method": "POST", - "path": "/api/v1/comfyui/comfyui_workflow/{{model}}", - "contentType": "application/json", - "fields": { - "prompt": "request.prompt|omit_if_model_equals:minimax_h3_image_audio_to_video", - "duration": "request.duration|omit_if_model_equals:minimax_h3_image_audio_to_video", - "audio_duration": "request.duration|omit_unless_model_equals:minimax_h3_image_audio_to_video", - "resolution": "request.resolution|lower", - "ref_image_0": "request.images.0.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_1": "request.images.1.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_2": "request.images.2.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_3": "request.images.3.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_4": "request.images.4.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_5": "request.images.5.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_6": "request.images.6.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_7": "request.images.7.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "ref_image_8": "request.images.8.url|omit_if_model_equals:minimax_h3_lightx2v|omit_if_model_equals:minimax_h3_b99_002|omit_if_model_equals:minimax_h3_image_audio_to_video|omit_if_model_contains:no_pic|omit_if_model_contains:b99_001", - "first_frame": "request.images.0.url|omit_if_model_contains:v5|omit_if_model_contains:no_pic|omit_if_model_contains:image_audio_to_video|omit_if_model_contains:b99_001|omit_if_model_contains:b99_003", - "last_frame": "request.images.1.url|omit_if_model_contains:v5|omit_if_model_contains:no_pic|omit_if_model_contains:image_audio_to_video|omit_if_model_contains:b99_001|omit_if_model_contains:b99_003", - "ref_audio_0": "request.audios.0.url|omit_unless_model_contains:image_audio_to_video", - "ref_audio_1": "request.audios.1.url|omit_unless_model_contains:image_audio_to_video_v2", - "ref_audio_2": "request.audios.2.url|omit_unless_model_contains:image_audio_to_video_v2" - } - }, - "poll": { - "method": "GET", - "path": "/api/v1/comfyui/comfyui_workflow/result/{{taskId}}" - }, - "response": { - "taskIdPaths": [ - "data.task_id" - ], - "statusPaths": [ - "data.status" - ], - "errorPaths": [ - "code" - ], - "messagePaths": [ - "data.message", - "msg" - ], - "resultPaths": [ - "data.results" - ], - "resultKind": "video", - "resultEphemeral": true - } - }, - { - "id": "autodl-comfyui-audio", - "label": "AutoDL ComfyUI 语音", - "capabilities": [ - "audio" - ], - "scopes": [ - "admin.system-channel", - "user.custom-channel", - "canvas", - "creation" - ], - "baseUrl": "https://autodl.art", - "requiresPublicMediaUrls": true, - "auth": { - "type": "header", - "field": "apiKey", - "header": "Authorization" - }, - "parameters": [ - { - "name": "prompt_text", - "type": "string", - "required": true, - "mapping": "prompt_text", - "description": "语音合成文本。" - } - ], - "create": { - "method": "POST", - "path": "/api/v1/comfyui/comfyui_workflow/{{model}}", - "contentType": "application/json", - "fields": { - "prompt_text": "request.prompt", - "prompt_simple": "request.audios.0.url", - "emo_control_method": "与音色参考音频相同" - } - }, - "poll": { - "method": "GET", - "path": "/api/v1/comfyui/comfyui_workflow/result/{{taskId}}" - }, - "response": { - "taskIdPaths": [ - "data.task_id" - ], - "statusPaths": [ - "data.status" - ], - "errorPaths": [ - "code" - ], - "messagePaths": [ - "data.message", - "msg" - ], - "resultPaths": [ - "data.results" - ], - "resultKind": "audio", - "resultEphemeral": true - } - } - ], - "workflows": [ - { - "id": "minimax_h3_b99_002", - "label": "H3 首尾帧生成视频", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["736p竖", "736p横", "736p(1:1)"] } - ], - "defaults": { "duration": 5, "resolution": "736p竖" } - }, - { - "id": "minimax_h3_b99_001", - "label": "H3 文生视频", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["736p竖", "736p横", "736p(1:1)"] } - ], - "defaults": { "duration": 5, "resolution": "736p竖" } - }, - { - "id": "minimax_h3_b99_003_12s", - "label": "H3 多图生视频 12 秒", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~12 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["736p竖", "736p横", "736p(1:1)"] } - ], - "defaults": { "duration": 5, "resolution": "736p竖" } - }, - { - "id": "minimax_h3_image_audio_to_video_v2_15s", - "label": "H3 多图多音频生视频 15 秒", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "480p横", "768p横"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "minimax_h3_lightx2v_v5_15s", - "label": "H3 多图生视频 15 秒", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "480p横", "768p横", "480p(1:1)", "768p(1:1)"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "minimax_h3_image_audio_to_video_v2", - "label": "H3 多图多音频生视频", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~10 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "1080p竖", "480p横", "768p横", "1080p横"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "minimax_h3_image_audio_to_video", - "label": "H3 图生视频音频同步", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "audio_duration", "type": "integer", "mapping": "duration", "description": "音频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "1080p竖", "480p横", "768p横", "1080p横"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "minimax_h3_lightx2v_v5", - "label": "H3 多图参考生视频", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~10 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "1080p竖", "480p横", "768p横", "1080p横", "480p(1:1)", "768p(1:1)", "1080p(1:1)"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "minimax_h3_lightx2v_no_pic", - "label": "H3 文生视频", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "480p横", "768p横"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "minimax_h3_lightx2v", - "label": "H3 首尾帧生成视频", - "providerId": "autodl-comfyui", - "capability": "video", - "parameters": [ - { "name": "prompt", "type": "string", "required": true, "mapping": "prompt" }, - { "name": "duration", "type": "integer", "mapping": "duration", "description": "视频时长 1~15 秒" }, - { "name": "resolution", "type": "string", "mapping": "resolution", "values": ["480p竖", "768p竖", "480p横", "768p横"] } - ], - "defaults": { "duration": 5, "resolution": "768p竖" } - }, - { - "id": "indextts2-v1", - "label": "IndexTTS2 语音", - "providerId": "autodl-comfyui-audio", - "capability": "audio", - "parameters": [ - { "name": "prompt_text", "type": "string", "required": true, "mapping": "prompt" } - ], - "defaults": {} - } - ] - } -} diff --git a/plugin-packages/aws-bedrock-converse.yingce-plugin b/plugin-packages/aws-bedrock-converse.yingce-plugin new file mode 100644 index 000000000..aee0b57a9 Binary files /dev/null and b/plugin-packages/aws-bedrock-converse.yingce-plugin differ diff --git a/plugin-packages/aws-bedrock-invoke-model.yingce-plugin b/plugin-packages/aws-bedrock-invoke-model.yingce-plugin new file mode 100644 index 000000000..b7895d883 Binary files /dev/null and b/plugin-packages/aws-bedrock-invoke-model.yingce-plugin differ diff --git a/plugin-packages/azure-openai.yingce-plugin b/plugin-packages/azure-openai.yingce-plugin new file mode 100644 index 000000000..6742f078d Binary files /dev/null and b/plugin-packages/azure-openai.yingce-plugin differ diff --git a/plugin-packages/baichuan-chat.yingce-plugin b/plugin-packages/baichuan-chat.yingce-plugin new file mode 100644 index 000000000..ce3c83ba5 Binary files /dev/null and b/plugin-packages/baichuan-chat.yingce-plugin differ diff --git a/plugin-packages/baidu-qianfan-image.yingce-plugin b/plugin-packages/baidu-qianfan-image.yingce-plugin new file mode 100644 index 000000000..219d8392d Binary files /dev/null and b/plugin-packages/baidu-qianfan-image.yingce-plugin differ diff --git a/plugin-packages/baidu-video.yingce-plugin b/plugin-packages/baidu-video.yingce-plugin new file mode 100644 index 000000000..eaef9bf49 Binary files /dev/null and b/plugin-packages/baidu-video.yingce-plugin differ diff --git a/plugin-packages/bfl-flux.yingce-plugin b/plugin-packages/bfl-flux.yingce-plugin new file mode 100644 index 000000000..d1664c1b2 Binary files /dev/null and b/plugin-packages/bfl-flux.yingce-plugin differ diff --git a/plugin-packages/build-packages.sh b/plugin-packages/build-packages.sh new file mode 100755 index 000000000..8b04e36f5 --- /dev/null +++ b/plugin-packages/build-packages.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu + +root_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +node "$root_dir/embed-documentation.mjs" + +for manifest in "$root_dir"/*/manifest.json; do + package_dir=${manifest%/manifest.json} + package_id=${package_dir##*/} + output_file="$root_dir/$package_id.yingce-plugin" + temporary_file="$root_dir/.$package_id.yingce-plugin.tmp" + rm -f "$temporary_file" + ( + cd "$package_dir" + find manifest.json README.md docs assets web LICENSE -type f 2>/dev/null | LC_ALL=C sort | zip -X -q "$temporary_file" -@ + ) + mv "$temporary_file" "$output_file" +done diff --git a/plugin-packages/cohere-chat-v2.yingce-plugin b/plugin-packages/cohere-chat-v2.yingce-plugin new file mode 100644 index 000000000..4c8347405 Binary files /dev/null and b/plugin-packages/cohere-chat-v2.yingce-plugin differ diff --git a/plugin-packages/comfyui-workflow.yingce-plugin b/plugin-packages/comfyui-workflow.yingce-plugin new file mode 100644 index 000000000..b27a5df6e Binary files /dev/null and b/plugin-packages/comfyui-workflow.yingce-plugin differ diff --git a/plugin-packages/dashscope-qwen-native.yingce-plugin b/plugin-packages/dashscope-qwen-native.yingce-plugin new file mode 100644 index 000000000..cd0208c40 Binary files /dev/null and b/plugin-packages/dashscope-qwen-native.yingce-plugin differ diff --git a/plugin-packages/dashscope-wan-video.yingce-plugin b/plugin-packages/dashscope-wan-video.yingce-plugin new file mode 100644 index 000000000..97a9d44f6 Binary files /dev/null and b/plugin-packages/dashscope-wan-video.yingce-plugin differ diff --git a/plugin-packages/dashscope-wanx-image.yingce-plugin b/plugin-packages/dashscope-wanx-image.yingce-plugin new file mode 100644 index 000000000..4908a4354 Binary files /dev/null and b/plugin-packages/dashscope-wanx-image.yingce-plugin differ diff --git a/plugin-packages/deepseek-chat.yingce-plugin b/plugin-packages/deepseek-chat.yingce-plugin new file mode 100644 index 000000000..d5b6cd5c9 Binary files /dev/null and b/plugin-packages/deepseek-chat.yingce-plugin differ diff --git a/plugin-packages/embed-documentation.mjs b/plugin-packages/embed-documentation.mjs new file mode 100644 index 000000000..529f81de2 --- /dev/null +++ b/plugin-packages/embed-documentation.mjs @@ -0,0 +1,70 @@ +import { readFile, readdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = dirname(fileURLToPath(import.meta.url)); +const contractStart = ""; +const contractEnd = ""; +const recursiveDocumentationPlaceholder = + "<当前插件的完整 documentation，由 README.md 与 docs/interface.md 拼接而成；为避免 JSON 递归，此处不重复展开正文。>"; + +function withoutGeneratedContract(document) { + const start = document.indexOf(contractStart); + if (start < 0) return document.trimEnd(); + const end = document.indexOf(contractEnd, start); + if (end < 0) { + throw new Error("docs/interface.md contains an unterminated generated manifest contract"); + } + return `${document.slice(0, start)}${document.slice(end + contractEnd.length)}`.trimEnd(); +} + +function renderContract(manifest) { + const contract = JSON.parse(JSON.stringify(manifest)); + contract.documentation = recursiveDocumentationPlaceholder; + return `${contractStart} +## Manifest 完整接口定义 + +以下 JSON 与插件包内实际 \`manifest.json\` 逐字段一致，覆盖插件身份、权限、配置、鉴权、参数、校验、创建、Agent、查询、取消、结果下载、响应和 Agent 响应映射。\`documentation\` 字段的值就是当前完整文档；为避免文档在自身内部无限递归，JSON 中仅用等义占位文本表示正文。 + +\`\`\`json +${JSON.stringify(contract, null, 2)} +\`\`\` +${contractEnd}`; +} + +const entries = await readdir(root, { withFileTypes: true }); +const requestedPackageIDs = new Set(process.argv.slice(2).map((value) => value.trim()).filter(Boolean)); +const packageDirectories = entries + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .filter((name) => requestedPackageIDs.size === 0 || requestedPackageIDs.has(name)) + .sort(); + +let updated = 0; +for (const packageID of packageDirectories) { + const packageRoot = join(root, packageID); + let manifestSource; + try { + manifestSource = await readFile(join(packageRoot, "manifest.json"), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + + const manifest = JSON.parse(manifestSource); + const readme = (await readFile(join(packageRoot, "README.md"), "utf8")).trim(); + const interfacePath = join(packageRoot, "docs", "interface.md"); + const interfaceSource = await readFile(interfacePath, "utf8"); + const interfaceDocument = `${withoutGeneratedContract(interfaceSource)}\n\n${renderContract(manifest)}\n`; + manifest.documentation = `${readme}\n\n---\n\n${interfaceDocument.trim()}\n`; + + await writeFile(interfacePath, interfaceDocument); + await writeFile(join(packageRoot, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + updated += 1; +} + +if (updated === 0) { + throw new Error("no protocol plugin manifests were found"); +} + +console.log(`embedded complete documentation into ${updated} protocol packages`); diff --git a/plugin-packages/fal-queue-image.yingce-plugin b/plugin-packages/fal-queue-image.yingce-plugin new file mode 100644 index 000000000..2ce24b855 Binary files /dev/null and b/plugin-packages/fal-queue-image.yingce-plugin differ diff --git a/plugin-packages/fal-queue-video.yingce-plugin b/plugin-packages/fal-queue-video.yingce-plugin new file mode 100644 index 000000000..e326eb336 Binary files /dev/null and b/plugin-packages/fal-queue-video.yingce-plugin differ diff --git a/plugin-packages/fireworks-chat.yingce-plugin b/plugin-packages/fireworks-chat.yingce-plugin new file mode 100644 index 000000000..6f5ece165 Binary files /dev/null and b/plugin-packages/fireworks-chat.yingce-plugin differ diff --git a/plugin-packages/generate-catalog.mjs b/plugin-packages/generate-catalog.mjs new file mode 100644 index 000000000..10cacab48 --- /dev/null +++ b/plugin-packages/generate-catalog.mjs @@ -0,0 +1,805 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = dirname(fileURLToPath(import.meta.url)); +const scopes = ["admin.system-channel", "user.custom-channel", "canvas", "creation", "agent"]; +const ref = (path) => ({ $ref: path }); +const omit = (value) => ({ $omitEmpty: value }); +const coalesce = (...values) => ({ $coalesce: values }); +const map = (from, as, body) => ({ $map: { from, as, in: body } }); +const filter = (from, as, where) => ({ $filter: { from, as, where } }); +const eq = (left, right) => ({ $eq: [left, right] }); +const ne = (left, right) => ({ $ne: [left, right] }); +const gt = (left, right) => ({ $gt: [left, right] }); +const len = (value) => ({ $len: value }); +const conditional = (condition, thenValue, elseValue = null) => ({ $if: { condition, then: thenValue, else: elseValue } }); +const first = (value) => ({ $first: value }); +const sorted = (value) => ({ $sortByOrder: value }); +const mediaWithRoles = (path, roles) => filter(sorted(ref(path)), "media", { $in: [ref("media.role"), roles] }); +const firstMediaWithRoles = (path, roles) => first(mediaWithRoles(path, roles)); +const firstMediaFieldWithRoles = (path, roles, field) => first(map(mediaWithRoles(path, roles), "media", ref(`media.${field}`))); + +const textParams = [ + ["model", "string", true, "model", "上游模型 ID。"], + ["messages", "message[]", true, "provider message container", "包含历史消息和当前用户输入。"], + ["instructions", "string", false, "system/instructions", "系统指令。"], + ["temperature", "number", false, "temperature", "采样温度。"], + ["top_p", "number", false, "top_p", "核采样参数。"], + ["max_tokens", "integer", false, "max_tokens/max_output_tokens", "最大输出 token。"], + ["tools", "array", false, "tools/toolConfig", "工具定义。"], + ["tool_choice", "object|string", false, "tool_choice", "工具选择策略。"], + ["response_format", "object", false, "response_format/text", "结构化输出配置。"], + ["stream", "boolean", false, "stream", "流式开关；后台任务当前以最终响应归一。"] +]; + +const imageParams = [ + ["model", "string", true, "model", "图片模型 ID。"], + ["prompt", "string", true, "prompt", "图片提示词。"], + ["images", "media[]", false, "provider image/reference fields", "参考图或编辑源图，role 由业务层确定。"], + ["imageCount", "integer", false, "n/sample_count", "输出数量。"], + ["aspectRatio", "string", false, "size/aspect_ratio", "比例或尺寸，语义按协议说明。"], + ["resolution", "string", false, "resolution/imageSize", "分辨率档位。"], + ["quality", "string", false, "quality", "质量档位。"], + ["providerOptions", "object", false, "provider-specific fields", "插件命名空间内的厂商扩展字段。"] +]; + +const videoParams = [ + ["model", "string", true, "model", "视频模型 ID。"], + ["prompt", "string", true, "prompt/content/input", "视频提示词。"], + ["images", "media[]", false, "first/last/reference image", "显式 role 图片输入。"], + ["videos", "media[]", false, "reference video", "参考视频。"], + ["audios", "media[]", false, "reference audio/voice", "参考音频或音色。"], + ["duration", "integer", false, "duration/seconds", "时长秒数。"], + ["aspectRatio", "string", false, "ratio/aspect_ratio/size", "画幅比例或尺寸。"], + ["resolution", "string", false, "resolution", "分辨率档位。"], + ["generateAudio", "boolean", false, "generate_audio", "是否生成音频。"], + ["watermark", "boolean", false, "watermark", "水印开关。"], + ["providerOptions", "object", false, "provider-specific fields", "插件命名空间内的厂商扩展字段。"] +]; + +const parameters = (items) => items.map(([name, type, required, mapping, description]) => ({ name, type, required, mapping, description })); +const config = (extra = []) => ({ + fields: [ + { name: "apiKey", type: "secret", label: "API Key", required: true }, + ...extra + ] +}); +const bearer = { type: "bearer", field: "apiKey" }; +const jsonCreate = (path, body, extra = {}) => ({ method: "POST", path, contentType: "application/json", body, ...extra }); +const asyncResponse = (kind, overrides = {}) => ({ + taskId: coalesce(ref("response.id"), ref("response.task_id"), ref("response.taskId"), ref("response.data.id"), ref("taskId")), + status: coalesce(ref("response.status"), ref("response.state"), ref("response.data.status"), "pending"), + message: coalesce(ref("response.error.message"), ref("response.message"), ref("response.fail_reason")), + [kind + "s"]: coalesce(ref(`response.${kind}_url`), ref(`response.${kind}Url`), ref("response.result_url"), ref("response.url"), ref(`response.data.${kind}_url`), ref("response.output.url")), + errorPaths: ["error.code"], + resultEphemeral: true, + ...overrides +}); + +const specs = []; + +function add(spec) { + specs.push(spec); +} + +add({ + id: "openai-chat-completions", providerId: "chat-completion", name: "OpenAI Chat Completions", vendor: "OpenAI", capability: "text", + baseUrl: "https://api.openai.com", auth: bearer, params: textParams, + create: jsonCreate("/v1/chat/completions", { + model: ref("request.model"), messages: ref("request.messages"), + temperature: omit(ref("request.providerOptions.chat-completion.temperature")), + top_p: omit(ref("request.providerOptions.chat-completion.top_p")), + max_tokens: omit(coalesce(ref("request.extra.max_tokens"), ref("request.providerOptions.chat-completion.max_tokens"))), + tools: omit(ref("request.providerOptions.chat-completion.tools")), + tool_choice: omit(ref("request.providerOptions.chat-completion.tool_choice")), + response_format: omit(ref("request.providerOptions.chat-completion.response_format")), + stream: omit(ref("request.providerOptions.chat-completion.stream")), + stop: omit(ref("request.providerOptions.chat-completion.stop")), + seed: omit(ref("request.providerOptions.chat-completion.seed")), + frequency_penalty: omit(ref("request.providerOptions.chat-completion.frequency_penalty")), + presence_penalty: omit(ref("request.providerOptions.chat-completion.presence_penalty")), + logprobs: omit(ref("request.providerOptions.chat-completion.logprobs")), + top_logprobs: omit(ref("request.providerOptions.chat-completion.top_logprobs")), + user: omit(ref("request.providerOptions.chat-completion.user")) + }), + agent: jsonCreate("/v1/chat/completions", { $merge: [ref("request.extra.agent.chatCompletion"), { model: ref("request.model") }] }), + agentResponse: { textPaths: ["choices.0.message.content", "choices.0.text"], reasoningPaths: ["choices.0.message.reasoning_content"], toolCallsPath: "choices.0.message.tool_calls", toolCallIdPaths: ["id"], toolCallNamePaths: ["function.name"], toolCallArgumentsPaths: ["function.arguments"] }, + response: { status: "succeeded", textPaths: ["choices.0.message.content", "choices.0.text"], reasoningPaths: ["choices.0.message.reasoning_content"], usage: ref("response.usage"), errorPaths: ["error.code"], messagePaths: ["error.message"] } +}); + +add({ + id: "openai-responses", providerId: "openai-response", name: "OpenAI Responses", vendor: "OpenAI", capability: "text", + baseUrl: "https://api.openai.com", auth: bearer, params: textParams, + create: jsonCreate("/v1/responses", { + model: ref("request.model"), input: ref("request.messages"), instructions: omit(ref("request.instructions")), + temperature: omit(ref("request.providerOptions.openai-response.temperature")), + top_p: omit(ref("request.providerOptions.openai-response.top_p")), + max_output_tokens: omit(coalesce(ref("request.extra.max_output_tokens"), ref("request.providerOptions.openai-response.max_output_tokens"))), + tools: omit(ref("request.providerOptions.openai-response.tools")), tool_choice: omit(ref("request.providerOptions.openai-response.tool_choice")), + text: omit(ref("request.providerOptions.openai-response.text")), reasoning: omit(ref("request.providerOptions.openai-response.reasoning")), + previous_response_id: omit(ref("request.providerOptions.openai-response.previous_response_id")), + store: omit(ref("request.providerOptions.openai-response.store")), metadata: omit(ref("request.providerOptions.openai-response.metadata")), + stream: omit(ref("request.providerOptions.openai-response.stream")), truncation: omit(ref("request.providerOptions.openai-response.truncation")), + user: omit(ref("request.providerOptions.openai-response.user")) + }), + agent: jsonCreate("/v1/responses", { $merge: [ref("request.extra.agent.responses"), { model: ref("request.model") }] }), + agentResponse: { textPaths: ["output_text"], reasoningPaths: ["reasoning.summary.0.text"], toolCallsPath: "output", toolCallIdPaths: ["call_id", "id"], toolCallNamePaths: ["name"], toolCallArgumentsPaths: ["arguments"] }, + response: { status: "succeeded", textPaths: ["output_text"], reasoningPaths: ["reasoning.summary.0.text"], usage: ref("response.usage"), errorPaths: ["error.code"], messagePaths: ["error.message"] } +}); + +add({ + id: "anthropic-messages", providerId: "claude-api", name: "Anthropic Messages", vendor: "Anthropic", capability: "text", + baseUrl: "https://api.anthropic.com", auth: { type: "anthropic", field: "apiKey" }, params: textParams, + create: jsonCreate("/v1/messages", { + model: ref("request.model"), + max_tokens: coalesce(ref("request.extra.max_tokens"), ref("request.providerOptions.claude-api.max_tokens"), 4096), + system: omit(coalesce(ref("request.instructions"), ref("request.providerOptions.claude-api.system"))), + messages: filter(ref("request.messages"), "message", ne(ref("message.role"), "system")), + temperature: omit(ref("request.providerOptions.claude-api.temperature")), top_p: omit(ref("request.providerOptions.claude-api.top_p")), + top_k: omit(ref("request.providerOptions.claude-api.top_k")), stop_sequences: omit(ref("request.providerOptions.claude-api.stop_sequences")), + tools: omit(ref("request.providerOptions.claude-api.tools")), tool_choice: omit(ref("request.providerOptions.claude-api.tool_choice")), + metadata: omit(ref("request.providerOptions.claude-api.metadata")), stream: omit(ref("request.providerOptions.claude-api.stream")), + thinking: omit(ref("request.providerOptions.claude-api.thinking")), service_tier: omit(ref("request.providerOptions.claude-api.service_tier")) + }, { headers: { "anthropic-version": coalesce(ref("request.providerOptions.claude-api.anthropic-version"), "2023-06-01"), "anthropic-beta": omit(ref("request.providerOptions.claude-api.anthropic-beta")) } }), + agent: jsonCreate("/v1/messages", { $merge: [ref("request.extra.agent.claude"), { model: ref("request.model") }] }, { headers: { "anthropic-version": "2023-06-01" } }), + agentResponse: { textPaths: ["content.0.text"], reasoningPaths: ["content.0.thinking"], toolCallsPath: "content", toolCallIdPaths: ["id"], toolCallNamePaths: ["name"], toolCallArgumentsPaths: ["input"] }, + response: { + status: "succeeded", + text: map(filter(ref("response.content"), "part", eq(ref("part.type"), "text")), "part", ref("part.text")), + reasoning: map(filter(ref("response.content"), "part", eq(ref("part.type"), "thinking")), "part", ref("part.thinking")), + usage: ref("response.usage"), errorPaths: ["error.type"], messagePaths: ["error.message"] + } +}); + +add({ + id: "google-gemini-generate-content", providerId: "gemini-generate-content", name: "Google Gemini generateContent", vendor: "Google", capability: "text", + baseUrl: "https://generativelanguage.googleapis.com", auth: { type: "google-api-key", field: "apiKey" }, params: textParams, + create: jsonCreate("/v1beta/models/{{model}}:generateContent", { + contents: map(ref("request.messages"), "message", { + role: conditional(eq(ref("message.role"), "assistant"), "model", "user"), + parts: [{ text: ref("message.content") }] + }), + systemInstruction: conditional(ref("request.instructions"), { parts: [{ text: ref("request.instructions") }] }), + generationConfig: omit(ref("request.providerOptions.gemini-generate-content.generationConfig")), + safetySettings: omit(ref("request.providerOptions.gemini-generate-content.safetySettings")), + tools: omit(ref("request.providerOptions.gemini-generate-content.tools")), + toolConfig: omit(ref("request.providerOptions.gemini-generate-content.toolConfig")), + cachedContent: omit(ref("request.providerOptions.gemini-generate-content.cachedContent")) + }), + agent: jsonCreate("/v1beta/models/{{model}}:generateContent", ref("request.extra.agent.gemini")), + agentResponse: { textPaths: ["candidates.0.content.parts.0.text"], toolCallsPath: "candidates.0.content.parts", toolCallIdPaths: ["functionCall.id"], toolCallNamePaths: ["functionCall.name"], toolCallArgumentsPaths: ["functionCall.args"], toolCallThoughtSignaturePaths: ["thoughtSignature", "thought_signature"] }, + response: { status: "succeeded", text: map(ref("response.candidates.0.content.parts"), "part", omit(ref("part.text"))), usage: ref("response.usageMetadata"), errorPaths: ["error.code"], messagePaths: ["error.message"] } +}); + +add({ + id: "cohere-chat-v2", providerId: "cohere-chat-v2", name: "Cohere Chat v2", vendor: "Cohere", capability: "text", + baseUrl: "https://api.cohere.com", auth: bearer, params: textParams, + create: jsonCreate("/v2/chat", { + model: ref("request.model"), messages: ref("request.messages"), + temperature: omit(ref("request.providerOptions.cohere-chat-v2.temperature")), max_tokens: omit(ref("request.extra.max_tokens")), + stop_sequences: omit(ref("request.providerOptions.cohere-chat-v2.stop_sequences")), tools: omit(ref("request.providerOptions.cohere-chat-v2.tools")), + tool_choice: omit(ref("request.providerOptions.cohere-chat-v2.tool_choice")), response_format: omit(ref("request.providerOptions.cohere-chat-v2.response_format")), + documents: omit(ref("request.providerOptions.cohere-chat-v2.documents")), citation_options: omit(ref("request.providerOptions.cohere-chat-v2.citation_options")), + safety_mode: omit(ref("request.providerOptions.cohere-chat-v2.safety_mode")), seed: omit(ref("request.providerOptions.cohere-chat-v2.seed")), stream: omit(ref("request.providerOptions.cohere-chat-v2.stream")) + }), + response: { status: "succeeded", textPaths: ["message.content.0.text", "text"], usage: ref("response.usage"), errorPaths: ["error.type"], messagePaths: ["message"] } +}); + +add({ + id: "ollama-chat", providerId: "ollama-chat", name: "Ollama Chat", vendor: "Ollama", capability: "text", + baseUrl: "http://127.0.0.1:11434", auth: { type: "none", field: "apiKey" }, params: textParams, + create: jsonCreate("/api/chat", { + model: ref("request.model"), messages: ref("request.messages"), stream: false, + format: omit(ref("request.providerOptions.ollama-chat.format")), options: omit(ref("request.providerOptions.ollama-chat.options")), + keep_alive: omit(ref("request.providerOptions.ollama-chat.keep_alive")), tools: omit(ref("request.providerOptions.ollama-chat.tools")), + think: omit(ref("request.providerOptions.ollama-chat.think")) + }), + response: { status: "succeeded", textPaths: ["message.content", "response"], reasoningPaths: ["message.thinking", "thinking"], errorPaths: ["error"], messagePaths: ["error"] } +}); + +for (const [id, name, vendor, baseUrl] of [ + ["xai-grok-chat", "xAI Grok Chat", "xAI", "https://api.x.ai"], + ["deepseek-chat", "DeepSeek Chat", "DeepSeek", "https://api.deepseek.com"], + ["mistral-chat", "Mistral Chat", "Mistral AI", "https://api.mistral.ai"], + ["kimi-chat", "Kimi OpenAI-Compatible", "Moonshot AI", "https://api.moonshot.cn"], + ["zhipu-glm-chat", "智谱 GLM Chat", "Zhipu AI", "https://open.bigmodel.cn/api/paas"], + ["baichuan-chat", "百川 Chat", "Baichuan", "https://api.baichuan-ai.com"], + ["yi-chat", "零一万物 Yi Chat", "01.AI", "https://api.lingyiwanwu.com"], + ["siliconflow-chat", "SiliconFlow Chat", "SiliconFlow", "https://api.siliconflow.cn"], + ["together-chat", "Together AI Chat", "Together AI", "https://api.together.xyz"], + ["groq-chat", "Groq Chat", "Groq", "https://api.groq.com/openai"], + ["fireworks-chat", "Fireworks Chat", "Fireworks AI", "https://api.fireworks.ai/inference"], + ["nvidia-nim-chat", "NVIDIA NIM Chat", "NVIDIA", "https://integrate.api.nvidia.com"], + ["openrouter-chat", "OpenRouter Chat", "OpenRouter", "https://openrouter.ai/api"], + ["litellm-proxy-chat", "LiteLLM Proxy Chat", "LiteLLM", "http://127.0.0.1:4000"], + ["newapi-chat", "NewAPI Chat", "NewAPI", "http://127.0.0.1:3000"], + ["vllm-chat", "vLLM OpenAI-Compatible", "vLLM", "http://127.0.0.1:8000"], + ["localai-chat", "LocalAI Chat", "LocalAI", "http://127.0.0.1:8080"] +]) { + add({ + id, providerId: id, name, vendor, capability: "text", baseUrl, auth: bearer, params: textParams, + notes: "该插件实现该平台公开的 OpenAI Chat Completions 线协议 profile；平台专有字段通过 providerOptions 命名空间透传，未声明支持的 OpenAI 字段仍由模型 capability profile 校验。", + create: jsonCreate("/v1/chat/completions", { + model: ref("request.model"), messages: ref("request.messages"), + temperature: omit(ref(`request.providerOptions.${id}.temperature`)), top_p: omit(ref(`request.providerOptions.${id}.top_p`)), + max_tokens: omit(coalesce(ref("request.extra.max_tokens"), ref(`request.providerOptions.${id}.max_tokens`))), + tools: omit(ref(`request.providerOptions.${id}.tools`)), tool_choice: omit(ref(`request.providerOptions.${id}.tool_choice`)), + response_format: omit(ref(`request.providerOptions.${id}.response_format`)), stream: omit(ref(`request.providerOptions.${id}.stream`)), + provider: omit(ref(`request.providerOptions.${id}.provider`)), transforms: omit(ref(`request.providerOptions.${id}.transforms`)), + extra_body: omit(ref(`request.providerOptions.${id}.extra_body`)) + }), + agent: jsonCreate("/v1/chat/completions", { $merge: [ref("request.extra.agent.chatCompletion"), { model: ref("request.model") }] }), + agentResponse: { textPaths: ["choices.0.message.content", "choices.0.text"], reasoningPaths: ["choices.0.message.reasoning_content"], toolCallsPath: "choices.0.message.tool_calls", toolCallIdPaths: ["id"], toolCallNamePaths: ["function.name"], toolCallArgumentsPaths: ["function.arguments"] }, + response: { status: "succeeded", textPaths: ["choices.0.message.content", "choices.0.text"], reasoningPaths: ["choices.0.message.reasoning_content"], usage: ref("response.usage"), errorPaths: ["error.code"], messagePaths: ["error.message"] } + }); +} + +add({ + id: "xai-grok-images", providerId: "grok-image", name: "xAI Grok Images", vendor: "xAI", capability: "image", + baseUrl: "https://api.x.ai", auth: bearer, params: imageParams, + create: jsonCreate("/v1/images/generations", { + model: ref("request.model"), prompt: ref("request.prompt"), + image: conditional(gt(len(ref("request.images")), 0), { url: firstMediaFieldWithRoles("request.images", ["edit_source", "reference_image", ""], "value") }), + n: conditional(gt(ref("request.imageCount"), 0), ref("request.imageCount"), 1), + response_format: coalesce(ref("request.providerOptions.grok-image.response_format"), "url"), + aspect_ratio: omit(ref("request.aspectRatio")), resolution: omit(coalesce(ref("request.resolution"), ref("request.quality"))), + user: omit(ref("request.providerOptions.grok-image.user")) + }), + response: { status: "succeeded", images: map(ref("response.data"), "item", { url: omit(ref("item.url")), dataUrl: conditional(ref("item.b64_json"), { $concat: ["data:image/png;base64,", ref("item.b64_json")] }) }), errorPaths: ["error.code"], messagePaths: ["error.message"] } +}); + +add({ + id: "volcengine-ark-seedream", providerId: "volcengine-ark-image", name: "Volcengine Ark Seedream Images", vendor: "Volcengine", capability: "image", + baseUrl: "https://ark.cn-beijing.volces.com", auth: bearer, params: imageParams, + create: jsonCreate("/api/v3/images/generations", { + model: ref("request.model"), prompt: ref("request.prompt"), size: omit(ref("request.aspectRatio")), + image: omit(map(ref("request.images"), "media", ref("media.value"))), + sequential_image_generation: omit(ref("request.providerOptions.volcengine-ark-image.sequential_image_generation")), + sequential_image_generation_options: omit(ref("request.providerOptions.volcengine-ark-image.sequential_image_generation_options")), + watermark: ref("request.watermark"), seed: omit(ref("request.providerOptions.volcengine-ark-image.seed")), + response_format: omit(ref("request.providerOptions.volcengine-ark-image.response_format")) + }), + response: { status: "succeeded", images: ref("response.data"), usage: ref("response.usage"), errorPaths: ["error.code"], messagePaths: ["error.message"] } +}); + +add({ + id: "google-gemini-image", providerId: "gemini-image", name: "Google Gemini Image", vendor: "Google", capability: "image", + baseUrl: "https://generativelanguage.googleapis.com", auth: { type: "google-api-key", field: "apiKey" }, params: imageParams, + create: jsonCreate("/v1beta/models/{{model}}:generateContent", { + contents: [{ role: "user", parts: { $concatArrays: [ + [{ text: ref("request.prompt") }], + map(ref("request.images"), "media", conditional(ref("media.dataUrl"), { inlineData: { mimeType: { $dataMime: ref("media.dataUrl") }, data: { $dataPayload: ref("media.dataUrl") } } }, { fileData: { mimeType: omit(ref("media.mimeType")), fileUri: ref("media.url") } })) + ] } }], + generationConfig: { + responseModalities: coalesce(ref("request.providerOptions.gemini-image.responseModalities"), ["TEXT", "IMAGE"]), + imageConfig: { aspectRatio: omit(ref("request.aspectRatio")), imageSize: omit(coalesce(ref("request.resolution"), ref("request.quality"))) }, + candidateCount: omit(ref("request.imageCount")), temperature: omit(ref("request.providerOptions.gemini-image.temperature")), topP: omit(ref("request.providerOptions.gemini-image.topP")), topK: omit(ref("request.providerOptions.gemini-image.topK")), seed: omit(ref("request.providerOptions.gemini-image.seed")) + }, + safetySettings: omit(ref("request.providerOptions.gemini-image.safetySettings")), systemInstruction: omit(ref("request.providerOptions.gemini-image.systemInstruction")) + }), + response: { + status: "succeeded", + images: map(filter(ref("response.candidates.0.content.parts"), "part", { $or: [ref("part.inlineData"), ref("part.inline_data")] }), "part", { + dataUrl: { $concat: ["data:", coalesce(ref("part.inlineData.mimeType"), ref("part.inline_data.mime_type"), "image/png"), ";base64,", coalesce(ref("part.inlineData.data"), ref("part.inline_data.data"))] } + }), + text: map(filter(ref("response.candidates.0.content.parts"), "part", ref("part.text")), "part", ref("part.text")), + usage: ref("response.usageMetadata"), errorPaths: ["error.code"], messagePaths: ["error.message"] + } +}); + +add({ + id: "volcengine-jimeng-image", providerId: "volcengine-jimeng-image", name: "Volcengine Jimeng Image", vendor: "Volcengine", capability: "image", + baseUrl: "https://visual.volcengineapi.com", auth: { type: "volcengine-v4", field: "apiKey", secretField: "secretKey", service: "cv", region: "cn-north-1" }, params: imageParams, + configuration: config([{ name: "secretKey", type: "secret", label: "Secret Key", required: true }]), + create: jsonCreate("/", { + req_key: ref("request.model"), prompt: ref("request.prompt"), image_urls: omit(map(ref("request.images"), "media", ref("media.value"))), + seed: omit(ref("request.providerOptions.volcengine-jimeng-image.seed")), width: omit(ref("request.output.width")), height: omit(ref("request.output.height")), + req_json: omit(ref("request.providerOptions.volcengine-jimeng-image.req_json")) + }, { originPath: true, query: { Action: "CVSync2AsyncSubmitTask", Version: "2022-08-31" } }), + poll: { method: "POST", path: "/", originPath: true, contentType: "application/json", query: { Action: "CVSync2AsyncGetResult", Version: "2022-08-31" }, body: { req_key: ref("request.model"), task_id: ref("taskId"), req_json: "{\"return_url\":true}" } }, + response: asyncResponse("image", { taskId: coalesce(ref("response.data.task_id"), ref("response.task_id"), ref("taskId")), status: coalesce(ref("response.data.status"), ref("response.status"), "pending"), images: coalesce(ref("response.data.image_urls"), ref("response.data.binary_data_base64")), errorPaths: ["code"], messagePaths: ["message"] }) +}); + +add({ + id: "openai-videos", providerId: "newapi", name: "OpenAI Videos / Sora", vendor: "OpenAI", capability: "video", + baseUrl: "https://api.openai.com", auth: bearer, params: videoParams, requiresPublicMediaUrls: false, + create: { + method: "POST", path: "/v1/videos", contentType: "multipart/form-data", + body: { + model: ref("request.model"), prompt: ref("request.prompt"), seconds: { $toString: ref("request.duration") }, + size: omit(ref("request.aspectRatio")), variants: omit(ref("request.providerOptions.newapi.variants")) + }, + files: [{ name: "input_reference", source: first(filter(sorted(ref("request.images")), "media", ne(ref("media.role"), "mask"))), filename: "input-reference.png" }] + }, + poll: { method: "GET", path: "/v1/videos/{{taskId}}" }, + cancel: { method: "DELETE", path: "/v1/videos/{{taskId}}" }, + result: { method: "GET", path: "/v1/videos/{{taskId}}/content", headers: { Accept: "video/mp4" } }, + response: asyncResponse("video", { videos: coalesce(ref("response.url"), ref("response.video_url"), ref("response.output.url")), errorPaths: ["error.code"], messagePaths: ["error.message"] }) +}); + +add({ + id: "newapi-media-task-v1", providerId: "newapi-channel-1", name: "NewAPI Media Task Channel 1", vendor: "NewAPI", capability: "video", + baseUrl: "http://127.0.0.1:3000", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + create: jsonCreate("/v1/videos", { + model: ref("request.model"), + input: { + prompt: ref("request.prompt"), + media: omit({ $concatArrays: [ + map(sorted(ref("request.images")), "media", { type: coalesce(ref("media.role"), "reference_image"), url: ref("media.value") }), + map(sorted(ref("request.videos")), "media", { type: coalesce(ref("media.role"), "reference_video"), url: ref("media.value") }), + map(sorted(ref("request.audios")), "media", { type: coalesce(ref("media.role"), "reference_voice"), url: ref("media.value") }) + ] }) + }, + parameters: { + resolution: coalesce(ref("request.resolution"), "720P"), ratio: coalesce(ref("request.aspectRatio"), "16:9"), + duration: conditional(gt(ref("request.duration"), 0), ref("request.duration"), 5), watermark: ref("request.watermark"), + prompt_extend: coalesce(ref("request.providerOptions.newapi-channel-1.prompt_extend"), false) + } + }), + poll: { method: "GET", path: "/v1/videos/{{taskId}}" }, response: asyncResponse("video") +}); + +add({ + id: "newapi-video-generations-v1", providerId: "newapi-channel-2", name: "NewAPI Video Generations Channel 2", vendor: "NewAPI", capability: "video", + baseUrl: "http://127.0.0.1:3000", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + create: jsonCreate("/v1/video/generations", { + model: ref("request.model"), prompt: ref("request.prompt"), seconds: { $toString: ref("request.duration") }, + aspect_ratio: coalesce(ref("request.aspectRatio"), "16:9"), resolution: omit(ref("request.resolution")), generate_audio: ref("request.generateAudio"), + image_urls: omit(map({ $sortByOrder: ref("request.images") }, "media", ref("media.value"))), + video_urls: omit(map({ $sortByOrder: ref("request.videos") }, "media", ref("media.value"))), + audio_urls: omit(map({ $sortByOrder: ref("request.audios") }, "media", ref("media.value"))) + }), + poll: { method: "GET", path: "/v1/video/generations/{{taskId}}" }, response: asyncResponse("video") +}); + +add({ + id: "rolldek-wan-video", providerId: "rolldek-wan-video", name: "RollDek WAN 3.0 Video", vendor: "RollDek", capability: "video", + baseUrl: "https://rolldek.com", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + notes: "该协议严格对应 RollDek WAN 3.0 的 JSON /v1/videos 合同。RollDek 同时暴露的其他兼容创建入口不共享任务查询路径，不能与 NewAPI Video Generations Channel 2 混用。", + create: jsonCreate("/v1/videos", { + model: ref("request.model"), prompt: ref("request.prompt"), seconds: { $toString: ref("request.duration") }, + size: omit({ $upper: ref("request.resolution") }), aspect_ratio: omit(ref("request.aspectRatio")), + reference_images: omit(map(sorted(ref("request.images")), "media", { + url: ref("media.value"), role: coalesce(ref("media.role"), "reference_image") + })), + reference_videos: omit(map(sorted(ref("request.videos")), "media", { + url: ref("media.value"), duration: omit({ $multiply: [ref("media.metadata.durationMs"), 0.001] }) + })), + reference_audios: omit(map(sorted(ref("request.audios")), "media", { url: ref("media.value") })) + }), + poll: { method: "GET", path: "/v1/videos/{{taskId}}" }, + result: { method: "GET", path: "/v1/videos/{{taskId}}/content", headers: { Accept: "video/mp4" } }, + response: asyncResponse("video", { + videos: coalesce(ref("response.metadata.url"), ref("response.data.metadata.url"), ref("response.video_url"), ref("response.url")), + errorPaths: ["error.code", "code"], messagePaths: ["error.message", "message", "fail_reason"] + }) +}); + +add({ + id: "xai-video", providerId: "xai-video", name: "xAI Video", vendor: "xAI", capability: "video", + baseUrl: "https://api.x.ai", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + validations: [ + { assert: { $lte: [len(filter(ref("request.images"), "media", { $in: [ref("media.role"), ["first_frame", "last_frame"]] })), 1] }, message: "xAI Video 最多支持一个帧输入，当前 profile 不支持尾帧" }, + { assert: { $not: { $and: [gt(len(filter(ref("request.images"), "media", { $in: [ref("media.role"), ["first_frame", "last_frame"]] })), 0), gt(len(filter(ref("request.images"), "media", { $in: [ref("media.role"), ["reference_image", "subject_reference", "style_reference"]] })), 0)] } }, message: "xAI Video 不能同时混用起始帧和角色参考图" } + ], + create: jsonCreate("/v1/videos/generations", { + model: ref("request.model"), prompt: ref("request.prompt"), duration: conditional(gt(ref("request.duration"), 0), ref("request.duration"), 6), + aspect_ratio: coalesce(ref("request.aspectRatio"), "16:9"), resolution: coalesce(ref("request.resolution"), "720p"), + image: conditional(gt(len(mediaWithRoles("request.images", ["first_frame", ""])), 0), { url: firstMediaFieldWithRoles("request.images", ["first_frame", ""], "value") }), + reference_images: omit(map(filter(ref("request.images"), "media", { $in: [ref("media.role"), ["reference_image", "subject_reference", "style_reference"]] }), "media", { url: ref("media.value") })) + }), + poll: { method: "GET", path: "/v1/videos/{{taskId}}" }, response: asyncResponse("video") +}); + +add({ + id: "google-gemini-veo", providerId: "gemini-veo", name: "Google Gemini Veo", vendor: "Google", capability: "video", + baseUrl: "https://generativelanguage.googleapis.com", auth: { type: "google-api-key", field: "apiKey" }, params: videoParams, + validations: [{ assert: { $eq: [len(filter(ref("request.images"), "media", eq(ref("media.role"), "reference_image"))), 0] }, message: "Gemini Veo 当前 profile 不支持角色参考图，请使用首帧输入或支持 reference_to_video 的协议" }], + create: jsonCreate("/v1beta/models/{{model}}:predictLongRunning", { + instances: [{ + prompt: ref("request.prompt"), + image: conditional(gt(len(mediaWithRoles("request.images", ["first_frame", ""])), 0), conditional(firstMediaFieldWithRoles("request.images", ["first_frame", ""], "dataUrl"), { inlineData: { mimeType: { $dataMime: firstMediaFieldWithRoles("request.images", ["first_frame", ""], "dataUrl") }, data: { $dataPayload: firstMediaFieldWithRoles("request.images", ["first_frame", ""], "dataUrl") } } }, { fileUri: firstMediaFieldWithRoles("request.images", ["first_frame", ""], "url"), mimeType: omit(firstMediaFieldWithRoles("request.images", ["first_frame", ""], "mimeType")) })) + }], + parameters: { + aspectRatio: omit(ref("request.aspectRatio")), durationSeconds: omit(ref("request.duration")), resolution: omit(ref("request.resolution")), + generateAudio: ref("request.generateAudio"), sampleCount: omit(ref("request.imageCount")), negativePrompt: omit(ref("request.providerOptions.gemini-veo.negativePrompt")), + personGeneration: omit(ref("request.providerOptions.gemini-veo.personGeneration")), seed: omit(ref("request.providerOptions.gemini-veo.seed")) + } + }), + poll: { method: "GET", path: "/v1beta/{{taskId}}" }, + response: asyncResponse("video", { taskId: coalesce(ref("response.name"), ref("taskId")), status: conditional(ref("response.error"), "failed", conditional(ref("response.done"), "succeeded", "processing")), videos: coalesce(ref("response.response.generateVideoResponse.generatedSamples"), ref("response.response.generatedVideos"), ref("response.response.videos")), message: ref("response.error.message") }) +}); + +add({ + id: "novita-video", providerId: "novita-video", name: "Novita Video", vendor: "Novita", capability: "video", + baseUrl: "https://api.novita.ai", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + validations: [{ assert: { $eq: [len(filter(ref("request.images"), "media", eq(ref("media.role"), "reference_image"))), 0] }, message: "Novita Video 当前 profile 只支持起始图，不支持角色参考图" }], + create: jsonCreate("/v3/video/create", { + model_name: ref("request.model"), prompt: ref("request.prompt"), duration: conditional(gt(ref("request.duration"), 0), ref("request.duration"), 5), + aspect_ratio: coalesce(ref("request.aspectRatio"), "16:9"), image_url: omit(firstMediaFieldWithRoles("request.images", ["first_frame", ""], "value")), + seed: omit(ref("request.providerOptions.novita-video.seed")), negative_prompt: omit(ref("request.providerOptions.novita-video.negative_prompt")) + }), + poll: { method: "GET", path: "/v3/async/task-result", query: { task_id: ref("taskId") } }, response: asyncResponse("video") +}); + +add({ + id: "volcengine-jimeng-video", providerId: "volcengine-jimeng-video", name: "Volcengine Jimeng Video", vendor: "Volcengine", capability: "video", + baseUrl: "https://visual.volcengineapi.com", auth: { type: "volcengine-v4", field: "apiKey", secretField: "secretKey", service: "cv", region: "cn-north-1" }, params: videoParams, + configuration: config([{ name: "secretKey", type: "secret", label: "Secret Key", required: true }]), + create: jsonCreate("/", { + req_key: ref("request.model"), prompt: ref("request.prompt"), image_urls: omit(map({ $sortByOrder: ref("request.images") }, "media", ref("media.value"))), + duration: omit(ref("request.duration")), ratio: omit(ref("request.aspectRatio")), resolution: omit(ref("request.resolution")), + req_json: omit(ref("request.providerOptions.volcengine-jimeng-video.req_json")) + }, { originPath: true, query: { Action: "CVSync2AsyncSubmitTask", Version: "2022-08-31" } }), + poll: { method: "POST", path: "/", originPath: true, contentType: "application/json", query: { Action: "CVSync2AsyncGetResult", Version: "2022-08-31" }, body: { req_key: ref("request.model"), task_id: ref("taskId") } }, + response: asyncResponse("video", { taskId: coalesce(ref("response.data.task_id"), ref("response.task_id"), ref("taskId")), status: coalesce(ref("response.data.status"), ref("response.status"), "pending"), videos: coalesce(ref("response.data.video_urls"), ref("response.data.video_url")), errorPaths: ["code"], messagePaths: ["message"] }) +}); + +const agnesFrameImages = mediaWithRoles("request.images", ["first_frame", "last_frame"]); +const agnesReferenceImages = mediaWithRoles("request.images", ["reference_image", "subject_reference", "style_reference"]); +const agnesReferenceMode = { $or: [gt(len(agnesReferenceImages), 0), gt(len(ref("request.videos")), 0), gt(len(ref("request.audios")), 0)] }; + +add({ + id: "agnes-video-25", providerId: "agnes-video", name: "Agnes Video 2.5 / 2.5 Flash", vendor: "Agnes AI", capability: "video", + baseUrl: "https://apihub.agnes-ai.com/v1", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + validations: [ + { assert: { $in: [ref("request.model"), ["agnes-video-2.5", "agnes-video-2.5-flash"]] }, message: "Agnes 2.5 插件仅支持 agnes-video-2.5 与 agnes-video-2.5-flash" }, + { assert: { $and: [{ $gte: [ref("request.duration"), 4] }, { $lte: [ref("request.duration"), 12] }] }, message: "Agnes Video 2.5 时长必须在 4–12 秒之间" }, + { assert: { $in: [{ $upper: coalesce(ref("request.resolution"), "720P") }, ["720P", "960P", "2K"]] }, message: "Agnes Video 2.5 分辨率必须是 720P、960P 或 2K" }, + { assert: { $not: { $and: [gt(len(agnesFrameImages), 0), agnesReferenceMode] } }, message: "Agnes Video 2.5 不能同时混用首尾帧和角色、视频或音频参考素材" }, + { assert: { $or: [ne(ref("request.model"), "agnes-video-2.5-flash"), eq({ $upper: coalesce(ref("request.resolution"), "720P") }, "720P")] }, message: "Agnes Video 2.5 Flash 仅支持 720P" }, + { assert: { $or: [ne(ref("request.model"), "agnes-video-2.5-flash"), { $lte: [len(ref("request.images")), 5] }] }, message: "Agnes Video 2.5 Flash 最多支持 5 张参考图" }, + { assert: { $or: [ne(ref("request.model"), "agnes-video-2.5-flash"), eq(len(ref("request.videos")), 0)] }, message: "Agnes Video 2.5 Flash 不支持参考视频" } + ], + create: jsonCreate("/videos", { + model: ref("request.model"), prompt: ref("request.prompt"), + mode: conditional(agnesReferenceMode, "reference", conditional(gt(len(ref("request.images")), 0), "keyframe", "text")), + seconds: { $toString: ref("request.duration") }, size: { $upper: coalesce(ref("request.resolution"), "720P") }, + aspect_ratio: coalesce(ref("request.aspectRatio"), "16:9"), n: coalesce(ref("request.output.count"), 1), + first_frame: conditional({ $not: agnesReferenceMode }, omit(coalesce(firstMediaFieldWithRoles("request.images", ["first_frame"], "value"), first(map(sorted(ref("request.images")), "media", ref("media.value"))))), null), + last_frame: conditional({ $not: agnesReferenceMode }, omit(firstMediaFieldWithRoles("request.images", ["last_frame"], "value")), null), + images: conditional(agnesReferenceMode, omit(map(sorted(ref("request.images")), "media", ref("media.value"))), null), + videos: conditional(agnesReferenceMode, omit(map(sorted(ref("request.videos")), "media", { url: ref("media.value") })), null), + audios: conditional(agnesReferenceMode, omit(map(sorted(ref("request.audios")), "media", ref("media.value"))), null) + }), + poll: { method: "GET", path: "/agnesapi", originPath: true, query: { video_id: ref("taskId"), model_name: ref("request.model") } }, + response: asyncResponse("video", { + taskId: coalesce(ref("response.data.video_id"), ref("response.video_id"), ref("response.data.id"), ref("response.id"), ref("taskId")), + status: coalesce(ref("response.data.status"), ref("response.status"), "pending"), + videos: coalesce(ref("response.data.metadata.url"), ref("response.metadata.url"), ref("response.data.url"), ref("response.url")), + message: coalesce(ref("response.data.error.message"), ref("response.error.message"), ref("response.message"), ref("response.detail")) + }) +}); + +add({ + id: "agnes-video-v20", providerId: "agnes-video-v20", name: "Agnes Video V2.0", vendor: "Agnes AI", capability: "video", + baseUrl: "https://apihub.agnes-ai.com/v1", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + validations: [ + { assert: eq(ref("request.model"), "agnes-video-v2.0"), message: "Agnes Video V2.0 插件仅支持 agnes-video-v2.0" }, + { assert: eq(len(ref("request.videos")), 0), message: "Agnes Video V2.0 不支持参考视频" }, + { assert: eq(len(ref("request.audios")), 0), message: "Agnes Video V2.0 不支持参考音频" }, + { assert: eq(len(agnesReferenceImages), 0), message: "Agnes Video V2.0 不支持角色或风格参考图" }, + { assert: { $lte: [len(agnesFrameImages), 2] }, message: "Agnes Video V2.0 最多支持首帧和尾帧各一张" } + ], + create: jsonCreate("/videos", { + model: ref("request.model"), prompt: ref("request.prompt"), frame_rate: 24, + num_frames: { $min: [441, { $add: [{ $ceilStep: [{ $multiply: [coalesce(ref("request.duration"), 5), 24] }, 8] }, 1] }] }, + image: conditional(eq(len(agnesFrameImages), 1), firstMediaFieldWithRoles("request.images", ["first_frame", "last_frame"], "value")), + extra_body: conditional(gt(len(agnesFrameImages), 1), { image: map(agnesFrameImages, "media", ref("media.value")), mode: "keyframes" }) + }), + poll: { method: "GET", path: "/agnesapi", originPath: true, query: { video_id: ref("taskId"), model_name: ref("request.model") } }, + response: asyncResponse("video", { + taskId: coalesce(ref("response.data.video_id"), ref("response.video_id"), ref("response.data.id"), ref("response.id"), ref("taskId")), + status: coalesce(ref("response.data.status"), ref("response.status"), "pending"), + videos: coalesce(ref("response.data.metadata.url"), ref("response.metadata.url"), ref("response.data.url"), ref("response.url")), + message: coalesce(ref("response.data.error.message"), ref("response.error.message"), ref("response.message"), ref("response.detail")) + }) +}); + +add({ + id: "stability-image", providerId: "stability-image", name: "Stability AI Image", vendor: "Stability AI", capability: "image", + baseUrl: "https://api.stability.ai", auth: bearer, params: imageParams, + create: { + method: "POST", + pathTemplate: coalesce(ref("request.providerOptions.stability-image.endpoint"), "/v2beta/stable-image/generate/core"), + contentType: "multipart/form-data", + headers: { Accept: "application/json" }, + body: { + prompt: ref("request.prompt"), negative_prompt: omit(ref("request.providerOptions.stability-image.negative_prompt")), + aspect_ratio: omit(ref("request.aspectRatio")), output_format: coalesce(ref("request.output.format"), ref("request.providerOptions.stability-image.output_format"), "png"), + seed: omit(ref("request.providerOptions.stability-image.seed")), style_preset: omit(ref("request.providerOptions.stability-image.style_preset")), + strength: omit(ref("request.providerOptions.stability-image.strength")), cfg_scale: omit(ref("request.providerOptions.stability-image.cfg_scale")), + grow_mask: omit(ref("request.providerOptions.stability-image.grow_mask")), creativity: omit(ref("request.providerOptions.stability-image.creativity")) + }, + files: [ + { name: "image", source: first(mediaWithRoles("request.images", ["edit_source", "reference_image", ""])), filename: "image.png" }, + { name: "mask", source: first(mediaWithRoles("request.images", ["mask"])), filename: "mask.png" } + ] + }, + response: { + status: conditional(ref("response.errors"), "failed", "succeeded"), + message: coalesce(ref("response.errors"), ref("response.message")), + images: conditional(ref("response.image"), [ + { dataUrl: { $concat: ["data:", coalesce(ref("response.mime_type"), "image/png"), ";base64,", ref("response.image")] } } + ]), + errorPaths: ["errors"] + } +}); + +add({ + id: "ideogram-image", providerId: "ideogram-image", name: "Ideogram Image", vendor: "Ideogram", capability: "image", + baseUrl: "https://api.ideogram.ai", auth: { type: "header", field: "apiKey", header: "Api-Key" }, params: imageParams, + create: { + method: "POST", pathTemplate: coalesce(ref("request.providerOptions.ideogram-image.endpoint"), "/v1/ideogram-v3/generate"), contentType: "multipart/form-data", + body: { + prompt: ref("request.prompt"), negative_prompt: omit(ref("request.providerOptions.ideogram-image.negative_prompt")), + aspect_ratio: omit(ref("request.aspectRatio")), resolution: omit(ref("request.resolution")), num_images: coalesce(ref("request.imageCount"), 1), + rendering_speed: omit(ref("request.providerOptions.ideogram-image.rendering_speed")), style_type: omit(ref("request.providerOptions.ideogram-image.style_type")), + magic_prompt: omit(ref("request.providerOptions.ideogram-image.magic_prompt")), seed: omit(ref("request.providerOptions.ideogram-image.seed")), + color_palette: omit(ref("request.providerOptions.ideogram-image.color_palette")), character_reference_images_mask: omit(ref("request.providerOptions.ideogram-image.character_reference_images_mask")) + }, + files: [ + { name: "image", source: first(mediaWithRoles("request.images", ["edit_source", "reference_image", "subject_reference", "style_reference", ""])), filename: "reference.png" }, + { name: "mask", source: first(mediaWithRoles("request.images", ["mask"])), filename: "mask.png" } + ] + }, + response: { status: "succeeded", images: coalesce(ref("response.data"), ref("response.images")), errorPaths: ["error.code"], messagePaths: ["error.message", "message"] } +}); + +add({ + id: "dashscope-wan-video", providerId: "dashscope-wan-video", name: "DashScope Wan Video", vendor: "Alibaba Cloud", capability: "video", + baseUrl: "https://dashscope.aliyuncs.com", auth: bearer, params: videoParams, requiresPublicMediaUrls: true, + validations: [ + { assert: { $lte: [len(mediaWithRoles("request.images", ["first_frame"])), 1] }, message: "Wan 视频最多只能有一个首帧" }, + { assert: { $lte: [len(mediaWithRoles("request.images", ["last_frame"])), 1] }, message: "Wan 视频最多只能有一个尾帧" }, + { assert: { $or: [eq(len(mediaWithRoles("request.images", ["last_frame"])), 0), eq(len(mediaWithRoles("request.images", ["first_frame"])), 1)] }, message: "Wan 视频使用尾帧时必须同时提供首帧" } + ], + create: jsonCreate("/api/v1/services/aigc/video-generation/video-synthesis", { + model: ref("request.model"), + input: { + prompt: ref("request.prompt"), + negative_prompt: omit(ref("request.providerOptions.dashscope-wan-video.negative_prompt")), + img_url: omit(conditional({ $and: [ne(ref("request.operation"), "reference_to_video"), eq(len(mediaWithRoles("request.images", ["last_frame"])), 0)] }, coalesce(firstMediaFieldWithRoles("request.images", ["first_frame"], "value"), firstMediaFieldWithRoles("request.images", ["reference_image", "edit_source", ""], "value")))), + first_frame_url: omit(conditional(gt(len(mediaWithRoles("request.images", ["last_frame"])), 0), firstMediaFieldWithRoles("request.images", ["first_frame"], "value"))), + last_frame_url: omit(firstMediaFieldWithRoles("request.images", ["last_frame"], "value")), + reference_images: omit(conditional(eq(ref("request.operation"), "reference_to_video"), map(mediaWithRoles("request.images", ["reference_image", "subject_reference", "style_reference"]), "media", ref("media.value")))), + video_url: omit(firstMediaFieldWithRoles("request.videos", ["reference_video", ""], "value")), + audio_url: omit(firstMediaFieldWithRoles("request.audios", ["reference_audio", "reference_voice", ""], "value")) + }, + parameters: { + size: omit(ref("request.resolution")), duration: omit(ref("request.duration")), + prompt_extend: omit(ref("request.providerOptions.dashscope-wan-video.prompt_extend")), + watermark: ref("request.watermark"), seed: omit(ref("request.providerOptions.dashscope-wan-video.seed")), + shot_type: omit(ref("request.providerOptions.dashscope-wan-video.shot_type")), + audio: ref("request.generateAudio"), template: omit(ref("request.providerOptions.dashscope-wan-video.template")) + } + }, { headers: { "X-DashScope-Async": "enable" } }), + poll: { method: "GET", path: "/api/v1/tasks/{{taskId}}" }, + response: asyncResponse("video", { + taskId: coalesce(ref("response.output.task_id"), ref("response.task_id"), ref("taskId")), + status: coalesce(ref("response.output.task_status"), ref("response.status"), "pending"), + videos: coalesce(ref("response.output.video_url"), ref("response.output.results"), ref("response.video_url")), + usage: ref("response.usage"), errorPaths: ["code"], messagePaths: ["message", "output.message"] + }) +}); + +add({ + id: "dashscope-wanx-image", providerId: "dashscope-wanx-image", name: "DashScope Wanx Image", vendor: "Alibaba Cloud", capability: "image", + baseUrl: "https://dashscope.aliyuncs.com", auth: bearer, params: imageParams, requiresPublicMediaUrls: true, + create: jsonCreate("/api/v1/services/aigc/text2image/image-synthesis", { + model: ref("request.model"), + input: { + prompt: ref("request.prompt"), negative_prompt: omit(ref("request.providerOptions.dashscope-wanx-image.negative_prompt")), + ref_img: omit(conditional(eq(len(ref("request.images")), 1), firstMediaFieldWithRoles("request.images", ["edit_source", "reference_image", "subject_reference", "style_reference", ""], "value"))), + ref_images: omit(conditional(gt(len(ref("request.images")), 1), map(sorted(ref("request.images")), "media", ref("media.value")))) + }, + parameters: { + size: omit(ref("request.aspectRatio")), n: coalesce(ref("request.imageCount"), 1), + seed: omit(ref("request.providerOptions.dashscope-wanx-image.seed")), style: omit(ref("request.providerOptions.dashscope-wanx-image.style")), + prompt_extend: omit(ref("request.providerOptions.dashscope-wanx-image.prompt_extend")), watermark: ref("request.watermark") + } + }, { headers: { "X-DashScope-Async": "enable" } }), + poll: { method: "GET", path: "/api/v1/tasks/{{taskId}}" }, + response: asyncResponse("image", { + taskId: coalesce(ref("response.output.task_id"), ref("response.task_id"), ref("taskId")), + status: coalesce(ref("response.output.task_status"), ref("response.status"), "pending"), + images: coalesce(ref("response.output.results"), ref("response.output.image_url")), + usage: ref("response.usage"), errorPaths: ["code"], messagePaths: ["message", "output.message"] + }) +}); + +for (const [id, name, vendor, capability, baseUrl, createPath, pollPath, requestShape] of [ + ["dashscope-qwen-native", "DashScope Qwen Native", "Alibaba Cloud", "text", "https://dashscope.aliyuncs.com", "/api/v1/services/aigc/text-generation/generation", null, "input+parameters"], + ["minimax-text-native", "MiniMax Text Native", "MiniMax", "text", "https://api.minimax.chat", "/v1/text/chatcompletion_v2", null, "messages+tokens_to_generate"], + ["azure-openai", "Azure OpenAI", "Microsoft Azure", "text", "https://{resource}.openai.azure.com", "/openai/deployments/{{model}}/chat/completions", null, "OpenAI body + api-version query"], + ["vertex-gemini", "Vertex AI Gemini", "Google Cloud", "text", "https://{location}-aiplatform.googleapis.com", "/v1/projects/{{request.providerOptions.vertex-gemini.project}}/locations/{{request.providerOptions.vertex-gemini.location}}/publishers/google/models/{{model}}:generateContent", null, "Gemini contents/parts"], + ["aws-bedrock-converse", "AWS Bedrock Converse", "AWS", "text", "https://bedrock-runtime.{region}.amazonaws.com", "/model/{{model}}/converse", null, "messages+system+inferenceConfig+toolConfig"], + ["aws-bedrock-invoke-model", "AWS Bedrock InvokeModel", "AWS", "text", "https://bedrock-runtime.{region}.amazonaws.com", "/model/{{model}}/invoke", null, "model-specific arbitrary body"], + ["vertex-imagen", "Vertex Imagen", "Google Cloud", "image", "https://{location}-aiplatform.googleapis.com", "/v1/projects/{{request.providerOptions.vertex-imagen.project}}/locations/{{request.providerOptions.vertex-imagen.location}}/publishers/google/models/{{model}}:predict", null, "instances+parameters"], + ["minimax-image", "MiniMax Image", "MiniMax", "image", "https://api.minimax.io", "/v1/image_generation", null, "model-specific image body"], + ["kling-image", "Kling Image", "Kuaishou", "image", "https://api.klingai.com", "/v1/images/generations", "/v1/images/generations/{{taskId}}", "task JSON"], + ["zhipu-cogview", "智谱 CogView", "Zhipu AI", "image", "https://open.bigmodel.cn/api/paas", "/v4/images/generations", null, "OpenAI-like image JSON"], + ["bfl-flux", "Black Forest Labs FLUX", "Black Forest Labs", "image", "https://api.bfl.ai", "/v1/{{model}}", "/v1/get_result", "async model schema"], + ["recraft-image", "Recraft Image", "Recraft", "image", "https://external.api.recraft.ai", "/v1/images/generations", null, "raster/vector image JSON"], + ["adobe-firefly", "Adobe Firefly", "Adobe", "image", "https://firefly-api.adobe.io", "/v3/images/generate", null, "versioned Firefly request"], + ["runway-image", "Runway Image", "Runway", "image", "https://api.dev.runwayml.com", "/v1/text_to_image", "/v1/tasks/{{taskId}}", "async Runway image schema"], + ["vertex-veo", "Vertex AI Veo", "Google Cloud", "video", "https://{location}-aiplatform.googleapis.com", "/v1/projects/{{request.providerOptions.vertex-veo.project}}/locations/{{request.providerOptions.vertex-veo.location}}/publishers/google/models/{{model}}:predictLongRunning", "/v1/{{taskId}}", "Vertex long-running operation"], + ["seedance-videos-compatible", "Seedance Compatible /videos", "Gateway profile", "video", "http://127.0.0.1:3000", "/v1/videos", "/v1/videos/{{taskId}}", "gateway-specific videos JSON"], + ["kling-video", "Kling Video", "Kuaishou", "video", "https://api.klingai.com", "/v1/videos/generations", "/v1/videos/generations/{{taskId}}", "text/image/multi-image task"], + ["runway-video", "Runway Video", "Runway", "video", "https://api.dev.runwayml.com", "/v1/image_to_video", "/v1/tasks/{{taskId}}", "promptImage+promptText+ratio+duration"], + ["luma-dream-machine", "Luma Dream Machine", "Luma AI", "video", "https://api.lumalabs.ai", "/dream-machine/v1/generations", "/dream-machine/v1/generations/{{taskId}}", "prompt+keyframes"], + ["vidu-video", "Vidu Video", "Vidu", "video", "https://api.vidu.com", "/v1/videos", "/v1/tasks/{{taskId}}", "text/image/reference video task"], + ["pixverse-video", "PixVerse Video", "PixVerse", "video", "https://app-api.pixverse.ai", "/openapi/v2/video/text/generate", "/openapi/v2/video/result/{{taskId}}", "model+duration+quality+effect"], + ["zhipu-cogvideox", "智谱 CogVideoX", "Zhipu AI", "video", "https://open.bigmodel.cn/api/paas", "/v4/videos/generations", "/v4/async-result/{{taskId}}", "model+prompt+image_url+quality+with_audio"], + ["baidu-video", "百度千帆视频", "Baidu AI Cloud", "video", "https://qianfan.baidubce.com", "/v2/video/generations", "/v2/video/generations/{{taskId}}", "model-specific input"], + ["tencent-hunyuan-video", "腾讯混元视频", "Tencent Cloud", "video", "https://hunyuan.tencentcloudapi.com", "/", "/", "TC3 Action task"], + ["tencent-hunyuan-image", "腾讯混元图片", "Tencent Cloud", "image", "https://hunyuan.tencentcloudapi.com", "/", "/", "TC3 Action task"], + ["baidu-qianfan-image", "百度千帆图片", "Baidu AI Cloud", "image", "https://qianfan.baidubce.com", "/v2/images/generations", null, "model-specific image input"] +]) { + const params = capability === "text" ? textParams : capability === "image" ? imageParams : videoParams; + const optionPath = `request.providerOptions.${id}`; + const standardFields = capability === "text" + ? { model: ref("request.model"), messages: ref("request.messages"), input: omit(ref("request.prompt")), parameters: omit(ref(`${optionPath}.parameters`)) } + : capability === "image" + ? { model: ref("request.model"), prompt: ref("request.prompt"), images: omit(map(ref("request.images"), "media", ref("media.value"))), parameters: omit(ref(`${optionPath}.parameters`)) } + : { model: ref("request.model"), prompt: ref("request.prompt"), images: omit(map(ref("request.images"), "media", ref("media.value"))), videos: omit(map(ref("request.videos"), "media", ref("media.value"))), audios: omit(map(ref("request.audios"), "media", ref("media.value"))), duration: omit(ref("request.duration")), aspect_ratio: omit(ref("request.aspectRatio")), resolution: omit(ref("request.resolution")), parameters: omit(ref(`${optionPath}.parameters`)) }; + const standard = { $merge: [standardFields, coalesce(ref(`${optionPath}.body`), ref(`${optionPath}.extra_body`), {})] }; + let auth = bearer; + let configuration; + let operationExtra = id.startsWith("dashscope-") ? { headers: { "X-DashScope-Async": "enable" } } : {}; + if (id.startsWith("aws-bedrock-")) { + auth = { type: "aws-sigv4", field: "apiKey", secretField: "secretKey", service: "bedrock" }; + configuration = { fields: [{ name: "apiKey", type: "secret", label: "Access Key ID", required: true }, { name: "secretKey", type: "secret", label: "Secret Access Key", required: true }] }; + } else if (id.startsWith("tencent-")) { + auth = { type: "tc3", field: "apiKey", secretField: "secretKey", service: "hunyuan" }; + configuration = { fields: [{ name: "apiKey", type: "secret", label: "SecretId", required: true }, { name: "secretKey", type: "secret", label: "SecretKey", required: true }] }; + operationExtra = { headers: { "X-TC-Action": ref(`${optionPath}.action`), "X-TC-Version": ref(`${optionPath}.version`) } }; + } else if (id === "azure-openai") { + auth = { type: "header", field: "apiKey", header: "api-key" }; + operationExtra = { query: { "api-version": coalesce(ref(`${optionPath}.api-version`), "2024-10-21") } }; + } + add({ + id, providerId: id, name, vendor, capability, baseUrl, auth, params, configuration, + notes: `该协议的模型级字段变化快或依赖云资源配置。插件固定线协议入口和统一字段，完整厂商对象通过 providerOptions.${id} 的 parameters/input/extra_body 传入；${requestShape}。云签名型入口在未配置对应鉴权驱动时会明确失败，不会伪装成 Bearer 成功。`, + create: jsonCreate(createPath, standard, operationExtra), + poll: pollPath ? { method: "GET", path: pollPath } : undefined, + response: capability === "text" + ? { status: "succeeded", textPaths: ["output.text", "output_text", "choices.0.message.content", "result"], reasoningPaths: ["reasoning_content"], usage: ref("response.usage"), errorPaths: ["error.code", "code"], messagePaths: ["error.message", "message"] } + : pollPath ? asyncResponse(capability) : { status: "succeeded", [capability + "s"]: coalesce(ref("response.data"), ref("response.output"), ref("response.url")), errorPaths: ["error.code", "code"], messagePaths: ["error.message", "message"] } + }); +} + +for (const [id, name, capability, createPath, pollPath, resultPath] of [ + ["fal-queue-image", "fal.ai Queue Image", "image", "/{{model}}", "/{{request.providerOptions.fal-queue-image.statusPath}}", "response.images"], + ["fal-queue-video", "fal.ai Queue Video", "video", "/{{model}}", "/{{request.providerOptions.fal-queue-video.statusPath}}", "response.video"], + ["replicate-prediction-image", "Replicate Predictions Image", "image", "/v1/predictions", "/v1/predictions/{{taskId}}", "response.output"], + ["replicate-prediction-video", "Replicate Predictions Video", "video", "/v1/predictions", "/v1/predictions/{{taskId}}", "response.output"], + ["comfyui-workflow", "ComfyUI Workflow", "video", "/prompt", "/history/{{taskId}}", "response.output"], + ["runninghub-workflow", "RunningHub Workflow", "video", "/task/openapi/create", "/task/openapi/status", "response.data"], + ["pika-via-fal", "Pika via fal.ai", "video", "/{{model}}", "/{{request.providerOptions.pika-via-fal.statusPath}}", "response.video"] +]) { + const optionPath = `request.providerOptions.${id}`; + add({ + id, providerId: id, name, vendor: id.includes("replicate") ? "Replicate" : id.includes("comfy") ? "ComfyUI" : id.includes("runninghub") ? "RunningHub" : "fal.ai", capability, + baseUrl: id.includes("replicate") ? "https://api.replicate.com" : id.includes("fal") || id.includes("pika") ? "https://queue.fal.run" : "http://127.0.0.1:8188", + auth: id.includes("comfy") ? { type: "none", field: "apiKey" } : bearer, + params: capability === "image" ? imageParams : videoParams, + notes: "这是运行时/工作流协议，模型字段由 endpoint、version 或 workflow schema 决定。插件不伪造固定模型字段；providerOptions.input/workflow/prompt 是完整请求对象，并由 conformance fixture 锁定实际接入版本。", + create: jsonCreate(createPath, { + version: omit(ref(`${optionPath}.version`)), model: omit(coalesce(ref(`${optionPath}.model`), ref("request.model"))), + input: omit(coalesce(ref(`${optionPath}.input`), { prompt: ref("request.prompt"), images: map(ref("request.images"), "media", ref("media.value")), videos: map(ref("request.videos"), "media", ref("media.value")), audios: map(ref("request.audios"), "media", ref("media.value")) })), + prompt: omit(ref(`${optionPath}.workflow`)), client_id: omit(ref(`${optionPath}.client_id`)), webhook: omit(ref(`${optionPath}.webhook`)), webhook_events_filter: omit(ref(`${optionPath}.webhook_events_filter`)) + }), + poll: pollPath ? { method: "GET", path: pollPath } : undefined, + cancel: id.includes("replicate") ? { method: "POST", path: "/v1/predictions/{{taskId}}/cancel" } : undefined, + response: asyncResponse(capability, { [capability + "s"]: ref(resultPath), taskId: coalesce(ref("response.id"), ref("response.request_id"), ref("response.prompt_id"), ref("taskId")) }) + }); +} + +function manifestFor(spec) { + return { + apiVersion: "yingce.plugin/v2", + id: spec.id, + name: spec.name, + version: "2.0.0", + author: `${spec.vendor} / 影策`, + description: `${spec.name} 独立请求协议插件。`, + documentation: `# ${spec.name}\n\n完整字段、映射、响应、鉴权和兼容边界见包内 README.md 与 docs/interface.md。\n\n## 影策运行时合同\n\n用户只操作统一的文本、图片或视频能力；插件负责把统一请求转换为 ${spec.name} 上游协议。`, + permissions: ["generation.run", "media.read"], + configuration: spec.configuration || config(), + contributes: { + providers: [{ + id: spec.providerId, + label: spec.name, + capabilities: [spec.capability], + scopes, + baseUrl: spec.baseUrl, + requiresPublicMediaUrls: spec.requiresPublicMediaUrls === true, + auth: spec.auth, + parameters: parameters(spec.params), + ...(spec.validations ? { validations: spec.validations } : {}), + create: spec.create, + ...(spec.agent ? { agent: spec.agent } : {}), + ...(spec.poll ? { poll: spec.poll } : {}), + ...(spec.cancel ? { cancel: spec.cancel } : {}), + ...(spec.result ? { result: spec.result } : {}), + response: spec.response, + ...(spec.agentResponse ? { agentResponse: spec.agentResponse } : {}) + }] + } + }; +} + +function collectTemplateFields(value, prefix, rows) { + if (value === undefined || value === null) return; + if (Array.isArray(value)) { + if (value.length === 0) rows.push([prefix, "[]"]); + value.forEach((item, index) => collectTemplateFields(item, `${prefix}[${index}]`, rows)); + return; + } + if (typeof value === "object") { + const keys = Object.keys(value); + if (keys.length === 1 && keys[0].startsWith("$")) { + rows.push([prefix, JSON.stringify(value)]); + return; + } + if (keys.length === 0) rows.push([prefix, "{}"]); + for (const key of keys) collectTemplateFields(value[key], prefix ? `${prefix}.${key}` : key, rows); + return; + } + rows.push([prefix, JSON.stringify(value)]); +} + +function operationFieldRows(operation, label) { + if (!operation) return []; + const rows = [ + [`${label}.method`, JSON.stringify(operation.method)], + [`${label}.path`, JSON.stringify(operation.path || "dynamic")], + [`${label}.contentType`, JSON.stringify(operation.contentType || "application/json")] + ]; + collectTemplateFields(operation.body, `${label}.body`, rows); + collectTemplateFields(operation.query, `${label}.query`, rows); + collectTemplateFields(operation.headers, `${label}.headers`, rows); + collectTemplateFields(operation.files, `${label}.files`, rows); + return rows; +} + +function docsFor(spec) { + const rows = spec.params.map(([name, type, required, mapping, description]) => `| \`${name}\` | ${type} | ${required ? "是" : "否"} | \`${mapping}\` | ${description} |`).join("\n"); + const operationRows = [ + ...operationFieldRows(spec.create, "create"), + ...operationFieldRows(spec.agent, "agent"), + ...operationFieldRows(spec.poll, "poll"), + ...operationFieldRows(spec.cancel, "cancel"), + ...operationFieldRows(spec.result, "result") + ].map(([path, expression]) => `| \`${path}\` | \`${String(expression).replaceAll("|", "\\|")}\` |`).join("\n"); + const responseRows = []; + collectTemplateFields(spec.response, "response", responseRows); + collectTemplateFields(spec.agentResponse, "agentResponse", responseRows); + const mappedResponseRows = responseRows.map(([path, expression]) => `| \`${path}\` | \`${String(expression).replaceAll("|", "\\|")}\` |`).join("\n"); + const manifestJSON = JSON.stringify({ create: spec.create, agent: spec.agent, poll: spec.poll, cancel: spec.cancel, result: spec.result, response: spec.response, agentResponse: spec.agentResponse }); + const optionRefs = [...new Set([...manifestJSON.matchAll(new RegExp(`request\\.providerOptions\\.${spec.providerId.replaceAll("-", "\\-")}\\.([A-Za-z0-9_.-]+)`, "g"))].map((match) => match[1]))].sort(); + const optionRows = optionRefs.length ? optionRefs.map((name) => `- \`providerOptions.${spec.providerId}.${name}\``).join("\n") : "- 无额外扩展键。"; + const configRows = (spec.configuration || config()).fields.map((field) => `| \`${field.name}\` | ${field.type} | ${field.required ? "是" : "否"} | ${field.label || ""} |`).join("\n"); + return `# ${spec.name} 接口字段\n\n## 协议身份\n\n- 插件 ID：\`${spec.id}\`。\n- Provider ID：\`${spec.providerId}\`。\n- 能力：\`${spec.capability}\`。\n- 默认 Base URL：\`${spec.baseUrl}\`。\n- 鉴权驱动：\`${spec.auth?.type || "默认"}\`。\n- 创建：\`${spec.create.method} ${spec.create.path || "动态路径"}\`。\n${spec.agent ? `- Agent：\`${spec.agent.method} ${spec.agent.path || "动态路径"}\`。\n` : ""}${spec.poll ? `- 查询：\`${spec.poll.method} ${spec.poll.path}\`。\n` : "- 生命周期：同步响应。\n"}${spec.cancel ? `- 取消：\`${spec.cancel.method} ${spec.cancel.path}\`。\n` : ""}\n## 配置字段\n\n| 字段 | 类型 | 必填 | 含义 |\n| --- | --- | --- | --- |\n${configRows}\n\n## 统一字段映射\n\n| 统一字段 | 类型 | 必填 | 上游映射 | 说明 |\n| --- | --- | --- | --- | --- |\n${rows}\n\n## 上游请求模板逐字段清单\n\n下表由插件请求模板生成，覆盖 body、query、headers 和 multipart 文件声明中的每个字段。\n\n| 上游位置 | 值或转换表达式 |\n| --- | --- |\n${operationRows || "| `create` | 无请求字段 |"}\n\n## Provider 扩展键\n\n${optionRows}\n\n动态模型或工作流允许使用文档声明的完整 \`parameters/input/extra_body\` 对象；该对象是协议本身的开放 schema，不会被宿主裁剪。\n\n## 响应映射逐字段清单\n\n| 映射位置 | 上游路径或转换表达式 |\n| --- | --- |\n${mappedResponseRows || "| `response` | 无显式映射 |"}\n\n## 响应与错误\n\n插件把上游 task/status/text/media/usage 映射为统一结果。临时媒体 URL 标记为 ephemeral，由宿主立即下载持久化。HTTP 错误、业务 code 和 error object 保持失败语义，不包装成成功。\n\n## 兼容边界\n\n${spec.notes || "该包只代表上述线协议 profile；同一品牌的其他 endpoint、云区域或网关包装必须使用独立插件，不能根据模型名猜测。"}\n`; +} + +const requestedPackageIDs = new Set(process.argv.slice(2).map((value) => value.trim()).filter(Boolean)); +const selectedSpecs = requestedPackageIDs.size > 0 ? specs.filter((spec) => requestedPackageIDs.has(spec.id)) : specs; +if (requestedPackageIDs.size > 0 && selectedSpecs.length !== requestedPackageIDs.size) { + const found = new Set(selectedSpecs.map((spec) => spec.id)); + const missing = [...requestedPackageIDs].filter((id) => !found.has(id)); + throw new Error(`unknown protocol package id(s): ${missing.join(", ")}`); +} + +for (const spec of selectedSpecs) { + const dir = join(root, spec.id); + await mkdir(join(dir, "docs"), { recursive: true }); + await writeFile(join(dir, "manifest.json"), JSON.stringify(manifestFor(spec), null, 2) + "\n"); + await writeFile(join(dir, "README.md"), `# ${spec.name}\n\n该目录是独立官方协议插件源码。后端从生成的 \`${spec.id}.yingce-plugin\` 包加载，不依赖系统内置 \`host:\` 适配器。\n\n完整接口见 [docs/interface.md](docs/interface.md)。\n`); + await writeFile(join(dir, "docs", "interface.md"), docsFor(spec)); +} + +console.log(`generated ${selectedSpecs.length} protocol packages`); +await import("./embed-documentation.mjs"); diff --git a/plugin-packages/google-gemini-generate-content.yingce-plugin b/plugin-packages/google-gemini-generate-content.yingce-plugin new file mode 100644 index 000000000..afccae8ff Binary files /dev/null and b/plugin-packages/google-gemini-generate-content.yingce-plugin differ diff --git a/plugin-packages/google-gemini-image.yingce-plugin b/plugin-packages/google-gemini-image.yingce-plugin new file mode 100644 index 000000000..9d7cbc662 Binary files /dev/null and b/plugin-packages/google-gemini-image.yingce-plugin differ diff --git a/plugin-packages/google-gemini-veo.yingce-plugin b/plugin-packages/google-gemini-veo.yingce-plugin new file mode 100644 index 000000000..154e89d8c Binary files /dev/null and b/plugin-packages/google-gemini-veo.yingce-plugin differ diff --git a/plugin-packages/groq-chat.yingce-plugin b/plugin-packages/groq-chat.yingce-plugin new file mode 100644 index 000000000..dd1781899 Binary files /dev/null and b/plugin-packages/groq-chat.yingce-plugin differ diff --git a/plugin-packages/ideogram-image.yingce-plugin b/plugin-packages/ideogram-image.yingce-plugin new file mode 100644 index 000000000..91b772fbe Binary files /dev/null and b/plugin-packages/ideogram-image.yingce-plugin differ diff --git a/plugin-packages/kimi-chat.yingce-plugin b/plugin-packages/kimi-chat.yingce-plugin new file mode 100644 index 000000000..2963dc249 Binary files /dev/null and b/plugin-packages/kimi-chat.yingce-plugin differ diff --git a/plugin-packages/kling-image.yingce-plugin b/plugin-packages/kling-image.yingce-plugin new file mode 100644 index 000000000..d19b147c4 Binary files /dev/null and b/plugin-packages/kling-image.yingce-plugin differ diff --git a/plugin-packages/kling-video.yingce-plugin b/plugin-packages/kling-video.yingce-plugin new file mode 100644 index 000000000..6e1bab5af Binary files /dev/null and b/plugin-packages/kling-video.yingce-plugin differ diff --git a/plugin-packages/litellm-proxy-chat.yingce-plugin b/plugin-packages/litellm-proxy-chat.yingce-plugin new file mode 100644 index 000000000..052ee2cd2 Binary files /dev/null and b/plugin-packages/litellm-proxy-chat.yingce-plugin differ diff --git a/plugin-packages/localai-chat.yingce-plugin b/plugin-packages/localai-chat.yingce-plugin new file mode 100644 index 000000000..0c54997e1 Binary files /dev/null and b/plugin-packages/localai-chat.yingce-plugin differ diff --git a/plugin-packages/luma-dream-machine.yingce-plugin b/plugin-packages/luma-dream-machine.yingce-plugin new file mode 100644 index 000000000..96830ec28 Binary files /dev/null and b/plugin-packages/luma-dream-machine.yingce-plugin differ diff --git a/plugin-packages/minimax-hailuo-video-v2.yingce-plugin b/plugin-packages/minimax-hailuo-video-v2.yingce-plugin new file mode 100644 index 000000000..c0c06fa5e Binary files /dev/null and b/plugin-packages/minimax-hailuo-video-v2.yingce-plugin differ diff --git a/plugin-packages/minimax-image.yingce-plugin b/plugin-packages/minimax-image.yingce-plugin new file mode 100644 index 000000000..6a25c0e10 Binary files /dev/null and b/plugin-packages/minimax-image.yingce-plugin differ diff --git a/plugin-packages/minimax-text-native.yingce-plugin b/plugin-packages/minimax-text-native.yingce-plugin new file mode 100644 index 000000000..e10da08c7 Binary files /dev/null and b/plugin-packages/minimax-text-native.yingce-plugin differ diff --git a/plugin-packages/mistral-chat.yingce-plugin b/plugin-packages/mistral-chat.yingce-plugin new file mode 100644 index 000000000..b5e1455e9 Binary files /dev/null and b/plugin-packages/mistral-chat.yingce-plugin differ diff --git a/plugin-packages/newapi-chat.yingce-plugin b/plugin-packages/newapi-chat.yingce-plugin new file mode 100644 index 000000000..0ce13822d Binary files /dev/null and b/plugin-packages/newapi-chat.yingce-plugin differ diff --git a/plugin-packages/newapi-media-task-v1.yingce-plugin b/plugin-packages/newapi-media-task-v1.yingce-plugin new file mode 100644 index 000000000..6b1281470 Binary files /dev/null and b/plugin-packages/newapi-media-task-v1.yingce-plugin differ diff --git a/plugin-packages/newapi-video-generations-v1.yingce-plugin b/plugin-packages/newapi-video-generations-v1.yingce-plugin new file mode 100644 index 000000000..9398b99ca Binary files /dev/null and b/plugin-packages/newapi-video-generations-v1.yingce-plugin differ diff --git a/plugin-packages/novita-video.yingce-plugin b/plugin-packages/novita-video.yingce-plugin new file mode 100644 index 000000000..06a44465a Binary files /dev/null and b/plugin-packages/novita-video.yingce-plugin differ diff --git a/plugin-packages/nvidia-nim-chat.yingce-plugin b/plugin-packages/nvidia-nim-chat.yingce-plugin new file mode 100644 index 000000000..6b822cd0e Binary files /dev/null and b/plugin-packages/nvidia-nim-chat.yingce-plugin differ diff --git a/plugin-packages/ollama-chat.yingce-plugin b/plugin-packages/ollama-chat.yingce-plugin new file mode 100644 index 000000000..f827a2bcf Binary files /dev/null and b/plugin-packages/ollama-chat.yingce-plugin differ diff --git a/plugin-packages/openai-chat-completions.yingce-plugin b/plugin-packages/openai-chat-completions.yingce-plugin new file mode 100644 index 000000000..ecf5f33e9 Binary files /dev/null and b/plugin-packages/openai-chat-completions.yingce-plugin differ diff --git a/plugin-packages/openai-images.yingce-plugin b/plugin-packages/openai-images.yingce-plugin new file mode 100644 index 000000000..08c9b1e25 Binary files /dev/null and b/plugin-packages/openai-images.yingce-plugin differ diff --git a/plugin-packages/openai-responses.yingce-plugin b/plugin-packages/openai-responses.yingce-plugin new file mode 100644 index 000000000..df88683ab Binary files /dev/null and b/plugin-packages/openai-responses.yingce-plugin differ diff --git a/plugin-packages/openai-videos.yingce-plugin b/plugin-packages/openai-videos.yingce-plugin new file mode 100644 index 000000000..4be11abca Binary files /dev/null and b/plugin-packages/openai-videos.yingce-plugin differ diff --git a/plugin-packages/openrouter-chat.yingce-plugin b/plugin-packages/openrouter-chat.yingce-plugin new file mode 100644 index 000000000..68ca5bc37 Binary files /dev/null and b/plugin-packages/openrouter-chat.yingce-plugin differ diff --git a/plugin-packages/pika-via-fal.yingce-plugin b/plugin-packages/pika-via-fal.yingce-plugin new file mode 100644 index 000000000..6d6a3e542 Binary files /dev/null and b/plugin-packages/pika-via-fal.yingce-plugin differ diff --git a/plugin-packages/pixverse-video.yingce-plugin b/plugin-packages/pixverse-video.yingce-plugin new file mode 100644 index 000000000..728d8bca7 Binary files /dev/null and b/plugin-packages/pixverse-video.yingce-plugin differ diff --git a/plugin-packages/recraft-image.yingce-plugin b/plugin-packages/recraft-image.yingce-plugin new file mode 100644 index 000000000..c1d75e30e Binary files /dev/null and b/plugin-packages/recraft-image.yingce-plugin differ diff --git a/plugin-packages/replicate-prediction-image.yingce-plugin b/plugin-packages/replicate-prediction-image.yingce-plugin new file mode 100644 index 000000000..cb1be370e Binary files /dev/null and b/plugin-packages/replicate-prediction-image.yingce-plugin differ diff --git a/plugin-packages/replicate-prediction-video.yingce-plugin b/plugin-packages/replicate-prediction-video.yingce-plugin new file mode 100644 index 000000000..04731810a Binary files /dev/null and b/plugin-packages/replicate-prediction-video.yingce-plugin differ diff --git a/plugin-packages/rolldek-wan-video.yingce-plugin b/plugin-packages/rolldek-wan-video.yingce-plugin new file mode 100644 index 000000000..7d45fe92c Binary files /dev/null and b/plugin-packages/rolldek-wan-video.yingce-plugin differ diff --git a/plugin-packages/runninghub-workflow.yingce-plugin b/plugin-packages/runninghub-workflow.yingce-plugin new file mode 100644 index 000000000..e4485f752 Binary files /dev/null and b/plugin-packages/runninghub-workflow.yingce-plugin differ diff --git a/plugin-packages/runway-image.yingce-plugin b/plugin-packages/runway-image.yingce-plugin new file mode 100644 index 000000000..0dd49f42e Binary files /dev/null and b/plugin-packages/runway-image.yingce-plugin differ diff --git a/plugin-packages/runway-video.yingce-plugin b/plugin-packages/runway-video.yingce-plugin new file mode 100644 index 000000000..57443d03b Binary files /dev/null and b/plugin-packages/runway-video.yingce-plugin differ diff --git a/plugin-packages/seedance-videos-compatible.yingce-plugin b/plugin-packages/seedance-videos-compatible.yingce-plugin new file mode 100644 index 000000000..56ef476de Binary files /dev/null and b/plugin-packages/seedance-videos-compatible.yingce-plugin differ diff --git a/plugin-packages/siliconflow-chat.yingce-plugin b/plugin-packages/siliconflow-chat.yingce-plugin new file mode 100644 index 000000000..6b3fae134 Binary files /dev/null and b/plugin-packages/siliconflow-chat.yingce-plugin differ diff --git a/plugin-packages/stability-image.yingce-plugin b/plugin-packages/stability-image.yingce-plugin new file mode 100644 index 000000000..82a98778a Binary files /dev/null and b/plugin-packages/stability-image.yingce-plugin differ diff --git a/plugin-packages/tencent-hunyuan-image.yingce-plugin b/plugin-packages/tencent-hunyuan-image.yingce-plugin new file mode 100644 index 000000000..ac8bbfa90 Binary files /dev/null and b/plugin-packages/tencent-hunyuan-image.yingce-plugin differ diff --git a/plugin-packages/tencent-hunyuan-video.yingce-plugin b/plugin-packages/tencent-hunyuan-video.yingce-plugin new file mode 100644 index 000000000..de181776f Binary files /dev/null and b/plugin-packages/tencent-hunyuan-video.yingce-plugin differ diff --git a/plugin-packages/together-chat.yingce-plugin b/plugin-packages/together-chat.yingce-plugin new file mode 100644 index 000000000..f38a66a6d Binary files /dev/null and b/plugin-packages/together-chat.yingce-plugin differ diff --git a/plugin-packages/vertex-gemini.yingce-plugin b/plugin-packages/vertex-gemini.yingce-plugin new file mode 100644 index 000000000..d15fbc947 Binary files /dev/null and b/plugin-packages/vertex-gemini.yingce-plugin differ diff --git a/plugin-packages/vertex-imagen.yingce-plugin b/plugin-packages/vertex-imagen.yingce-plugin new file mode 100644 index 000000000..7769ec177 Binary files /dev/null and b/plugin-packages/vertex-imagen.yingce-plugin differ diff --git a/plugin-packages/vertex-veo.yingce-plugin b/plugin-packages/vertex-veo.yingce-plugin new file mode 100644 index 000000000..1e060972f Binary files /dev/null and b/plugin-packages/vertex-veo.yingce-plugin differ diff --git a/plugin-packages/vidu-video.yingce-plugin b/plugin-packages/vidu-video.yingce-plugin new file mode 100644 index 000000000..03ce6ab2e Binary files /dev/null and b/plugin-packages/vidu-video.yingce-plugin differ diff --git a/plugin-packages/vllm-chat.yingce-plugin b/plugin-packages/vllm-chat.yingce-plugin new file mode 100644 index 000000000..dff880b7e Binary files /dev/null and b/plugin-packages/vllm-chat.yingce-plugin differ diff --git a/plugin-packages/volcengine-ark-seedance.yingce-plugin b/plugin-packages/volcengine-ark-seedance.yingce-plugin new file mode 100644 index 000000000..9d0700c14 Binary files /dev/null and b/plugin-packages/volcengine-ark-seedance.yingce-plugin differ diff --git a/plugin-packages/volcengine-ark-seedream.yingce-plugin b/plugin-packages/volcengine-ark-seedream.yingce-plugin new file mode 100644 index 000000000..7664c7856 Binary files /dev/null and b/plugin-packages/volcengine-ark-seedream.yingce-plugin differ diff --git a/plugin-packages/volcengine-jimeng-image.yingce-plugin b/plugin-packages/volcengine-jimeng-image.yingce-plugin new file mode 100644 index 000000000..ef192e3a8 Binary files /dev/null and b/plugin-packages/volcengine-jimeng-image.yingce-plugin differ diff --git a/plugin-packages/volcengine-jimeng-video.yingce-plugin b/plugin-packages/volcengine-jimeng-video.yingce-plugin new file mode 100644 index 000000000..8e0a4bdb2 Binary files /dev/null and b/plugin-packages/volcengine-jimeng-video.yingce-plugin differ diff --git a/plugin-packages/xai-grok-chat.yingce-plugin b/plugin-packages/xai-grok-chat.yingce-plugin new file mode 100644 index 000000000..978ea31c7 Binary files /dev/null and b/plugin-packages/xai-grok-chat.yingce-plugin differ diff --git a/plugin-packages/xai-grok-images.yingce-plugin b/plugin-packages/xai-grok-images.yingce-plugin new file mode 100644 index 000000000..d6f34ed93 Binary files /dev/null and b/plugin-packages/xai-grok-images.yingce-plugin differ diff --git a/plugin-packages/xai-video.yingce-plugin b/plugin-packages/xai-video.yingce-plugin new file mode 100644 index 000000000..55637ae27 Binary files /dev/null and b/plugin-packages/xai-video.yingce-plugin differ diff --git a/plugin-packages/yi-chat.yingce-plugin b/plugin-packages/yi-chat.yingce-plugin new file mode 100644 index 000000000..0469b4d2d Binary files /dev/null and b/plugin-packages/yi-chat.yingce-plugin differ diff --git a/plugin-packages/zhipu-cogvideox.yingce-plugin b/plugin-packages/zhipu-cogvideox.yingce-plugin new file mode 100644 index 000000000..2e1a72d49 Binary files /dev/null and b/plugin-packages/zhipu-cogvideox.yingce-plugin differ diff --git a/plugin-packages/zhipu-cogview.yingce-plugin b/plugin-packages/zhipu-cogview.yingce-plugin new file mode 100644 index 000000000..9929ed330 Binary files /dev/null and b/plugin-packages/zhipu-cogview.yingce-plugin differ diff --git a/plugin-packages/zhipu-glm-chat.yingce-plugin b/plugin-packages/zhipu-glm-chat.yingce-plugin new file mode 100644 index 000000000..4c9f06fdd Binary files /dev/null and b/plugin-packages/zhipu-glm-chat.yingce-plugin differ diff --git a/web/src/components/auth/require-feature.tsx b/web/src/components/auth/require-feature.tsx index cfb150475..64af5bfba 100644 --- a/web/src/components/auth/require-feature.tsx +++ b/web/src/components/auth/require-feature.tsx @@ -45,11 +45,11 @@ export function RequireFeature({ feature, children }: { feature: FeatureKey; chi }, []); if (checking) return ; - if (error) return navigate("/create", { replace: true })} />; + if (error) return navigate("/", { replace: true })} />; if (!adminBypass && !features[feature]) { // 管理员页面返回到管理后台首页，用户页面返回到创作台 const isAdminFeature = feature === "frontendModelsEnabled" || (feature === "pluginCenterEnabled" && user?.role === "admin"); - const backPath = isAdminFeature ? "/admin" : "/create"; + const backPath = isAdminFeature ? "/admin" : "/"; const backLabel = isAdminFeature ? "返回管理后台" : "返回创作台"; return ( diff --git a/web/src/components/canvas/canvas-assistant-panel.tsx b/web/src/components/canvas/canvas-assistant-panel.tsx index b927aaa83..73ec07e9f 100644 --- a/web/src/components/canvas/canvas-assistant-panel.tsx +++ b/web/src/components/canvas/canvas-assistant-panel.tsx @@ -7,8 +7,8 @@ import { motion } from "motion/react"; import { modelDisplayName, modelIcon, normalizeModelOptionValue, resolveModelChannel, resolveModelRequestConfig, selectableModelsByCapability, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store"; import { canvasThemes } from "@/lib/canvas-theme"; import { nanoid } from "nanoid"; -import { requestToolResponse, type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; -import { backendModelRuntimeRequired, runBackendToolGenerationTask } from "@/services/api/generation-task"; +import { type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; +import { runBackendToolGenerationTask } from "@/services/api/generation-task"; import { imageToDataUrl } from "@/services/image-storage"; import { isCanvasGenerationDurableAckError, persistCanvasCinematicSessionContinuationEffect } from "@/services/canvas-generation-consumer"; import { consumeGenerationTaskAgent } from "@/services/project-asset-sync"; @@ -32,7 +32,6 @@ import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantPendin import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store"; import { canvasAgentPostconditionMessage, previewCanvasAgentOps, summarizeCanvasAgentOps, verifyCanvasAgentOps, type CanvasAgentOp, type CanvasAgentOperationImpact, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; import { buildCanvasAgentContext, findCanvasAgentNodes, getCanvasAgentConnection, getCanvasAgentGenerationTasks, getCanvasAgentNode, getCanvasAgentResources, validateCanvasAgentOps } from "@/lib/canvas/canvas-agent-context"; -import { canvasAgentPromptCacheKey } from "@/lib/openai-prompt-cache"; import { resolveStoryboardGenerationContext } from "@/lib/canvas/canvas-storyboard-context"; import { buildCanvasWorkflowOps, looksLikeWorkflowRequest, type CanvasWorkflowInput } from "@/lib/canvas/canvas-agent-workflow"; import { listAddedSkills, type Skill } from "@/services/api/skills"; @@ -674,7 +673,7 @@ export function CanvasAssistantPanel({ const result = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, messages, "required", userMessage.text, (text) => { streamed = text; if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); - }, canvasAgentPromptCacheKey(sessionId)); + }); addOnlineLog("模型工具回复", result); if (result.toolCalls.length) { const writableCalls = result.toolCalls.filter(isWritableToolCall); @@ -733,7 +732,7 @@ export function CanvasAssistantPanel({ const next = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, nextMessages, "auto", "继续处理画布工具结果", (text) => { streamed = text; if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); - }, canvasAgentPromptCacheKey(sessionId)); + }); addOnlineLog(`Agent Tool Loop ${step + 1} 回复`, next); if (next.toolCalls.length) { const writableCalls = next.toolCalls.filter(isWritableToolCall); @@ -1649,12 +1648,8 @@ function toolCallToResponseInput(call: ResponseToolCall): ResponseInputMessage { return { type: "function_call", call_id: call.id, name: call.function.name, arguments: call.function.arguments, ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}) }; } -async function requestOnlineAgentModel(config: AiConfig, messages: ResponseInputMessage[], toolChoice: "auto" | "required", prompt: string, onDelta: (text: string) => void, promptCacheKey?: string) { - if (backendModelRuntimeRequired(config)) { - const result = await runBackendToolGenerationTask({ prompt, config, messages, tools: ONLINE_AGENT_TOOLS, toolChoice, onDelta }); - return result; - } - return requestToolResponse(config, messages, ONLINE_AGENT_TOOLS, toolChoice, onDelta, { promptCacheKey }); +async function requestOnlineAgentModel(config: AiConfig, messages: ResponseInputMessage[], toolChoice: "auto" | "required", prompt: string, onDelta: (text: string) => void) { + return runBackendToolGenerationTask({ prompt, config, messages, tools: ONLINE_AGENT_TOOLS, toolChoice, onDelta }); } function summarizeToolCalls(calls: ResponseToolCall[]) { diff --git a/web/src/components/canvas/canvas-node-content.tsx b/web/src/components/canvas/canvas-node-content.tsx index f3f268977..03a893462 100644 --- a/web/src/components/canvas/canvas-node-content.tsx +++ b/web/src/components/canvas/canvas-node-content.tsx @@ -54,7 +54,6 @@ export type CanvasNodeContentProps = { onToggleBatch?: () => void; reduceMediaEffects?: boolean; mediaActive?: boolean; - hydrateMediaPreview?: boolean; }; export function CanvasNodeContent(props: CanvasNodeContentProps) { @@ -417,10 +416,11 @@ function EmptyImageContent({ node, theme, isBatchRoot, batchCount, batchExpanded return content; } -function VideoNodeContent({ node, theme, reduceMediaEffects, mediaActive = false, hydrateMediaPreview = false }: CanvasNodeContentProps) { +function VideoNodeContent({ node, theme, reduceMediaEffects, mediaActive = false }: CanvasNodeContentProps) { const playerBoxRef = useRef(null); const { updateMediaNode } = useCanvasNodeActions(); - const { url, loading } = useNodeResourceUrl(node, mediaActive); + const hasPassivePreview = Boolean(canvasNodeVideoPreviewUrl(node)); + const { url, loading } = useNodeResourceUrl(node, mediaActive || !hasPassivePreview); const subtitleEntries = node.metadata?.subtitleEntries || []; const subtitleStyle = node.metadata?.subtitleStyle || createDefaultSubtitleStyle(); const [currentTimeMs, setCurrentTimeMs] = useState(0); @@ -449,7 +449,7 @@ function VideoNodeContent({ node, theme, reduceMediaEffects, mediaActive = false }, [node.id, node.metadata?.naturalHeight, node.metadata?.naturalWidth, subtitleEntries.length, updateMediaNode, url]); if (!node.metadata?.content) return } label="空视频节点" color={theme.node.placeholder} />; - if (!mediaActive) return ; + if (!mediaActive) return ; if (!url) return } label={loading ? "正在加载视频" : "视频资源不可用"} />; const sourceRatio = (videoSize?.width || node.metadata?.naturalWidth || node.width) / Math.max(1, videoSize?.height || node.metadata?.naturalHeight || node.height); @@ -484,18 +484,18 @@ function AudioNodeContent({ node, theme }: CanvasNodeContentProps) { return ; } -function InactiveVideoPreview({ node, theme, hydrateMediaPreview = false }: Pick) { +function InactiveVideoPreview({ node, theme, sourceUrl, sourceLoading }: Pick & { sourceUrl: string; sourceLoading: boolean }) { const previewUrl = canvasNodeVideoPreviewUrl(node); const { updateMetadata } = useCanvasNodeActions(); const updateMetadataRef = useRef(updateMetadata); - const [hydrating, setHydrating] = useState(false); + const [hydrating, setHydrating] = useState(() => !previewUrl && Boolean(node.metadata?.content)); useEffect(() => { updateMetadataRef.current = updateMetadata; }, [updateMetadata]); useEffect(() => { - if (previewUrl || !hydrateMediaPreview || !node.metadata?.content || !updateMetadataRef.current) { + if (previewUrl || !node.metadata?.content || !updateMetadataRef.current) { setHydrating(false); return; } @@ -510,12 +510,32 @@ function InactiveVideoPreview({ node, theme, hydrateMediaPreview = false }: Pick if (!controller.signal.aborted) setHydrating(false); }); return () => controller.abort(); - }, [hydrateMediaPreview, node.id, node.metadata?.content, node.metadata?.storageKey, previewUrl]); + }, [node.id, node.metadata?.content, node.metadata?.storageKey, previewUrl]); if (previewUrl) { return
{`${node.title
; } - return } title={node.title || "视频"} hint={hydrating ? "正在生成首帧" : "选择后加载视频"} theme={theme} />; + if (sourceUrl) { + return
+
; + } + return } title={node.title || "视频"} hint={sourceLoading || hydrating ? "正在加载首帧" : "视频资源暂不可用"} theme={theme} />; +} + +function primePassiveVideoFrame(video: HTMLVideoElement) { + if (video.currentTime === 0 && Number.isFinite(video.duration) && video.duration > 0) video.currentTime = Math.min(0.001, video.duration); } function InactiveMediaCard({ icon, title, hint, theme }: { icon: ReactNode; title: string; hint: string; theme: CanvasTheme }) { diff --git a/web/src/components/canvas/canvas-node-generation.ts b/web/src/components/canvas/canvas-node-generation.ts index 634e44e87..68c32d9d6 100644 --- a/web/src/components/canvas/canvas-node-generation.ts +++ b/web/src/components/canvas/canvas-node-generation.ts @@ -182,7 +182,7 @@ function buildComposerGenerationContext( if (input.type === "text") textBlocks.push(`【${label}】\n${input.text || ""}`); else selectedInputs.push(input); } - nextPrompt += input.type === "text" ? `【${label}】` : label; + nextPrompt += input.type === "text" ? `【${label}】` : `@${label}`; } else nextPrompt += match[0]; lastIndex = match.index + match[0].length; } diff --git a/web/src/components/canvas/canvas-node.tsx b/web/src/components/canvas/canvas-node.tsx index 15622618f..aa9eaba43 100644 --- a/web/src/components/canvas/canvas-node.tsx +++ b/web/src/components/canvas/canvas-node.tsx @@ -24,7 +24,6 @@ type CanvasNodeProps = { scale: number; isSelected: boolean; mediaActive?: boolean; - hydrateMediaPreview?: boolean; isRelated: boolean; isFocusRelated: boolean; isConnectionTarget: boolean; @@ -70,7 +69,6 @@ export const CanvasNode = React.memo(function CanvasNode({ scale, isSelected, mediaActive = false, - hydrateMediaPreview = false, isRelated, isFocusRelated, isConnectionTarget, @@ -376,7 +374,6 @@ export const CanvasNode = React.memo(function CanvasNode({ onToggleBatch={() => onToggleBatch?.(data.id)} reduceMediaEffects={reduceMediaEffects} mediaActive={mediaActive} - hydrateMediaPreview={hydrateMediaPreview} /> diff --git a/web/src/components/layout/app-top-nav.tsx b/web/src/components/layout/app-top-nav.tsx index 25dea844e..b0c720d7d 100644 --- a/web/src/components/layout/app-top-nav.tsx +++ b/web/src/components/layout/app-top-nav.tsx @@ -20,7 +20,7 @@ export function AppWorkspaceShell({ children }: { children: ReactNode }) { const hideChrome = pathname.startsWith("/admin") || /^\/canvas\/[^/]+/.test(pathname); const showGlobalTopBar = !hideChrome; const spatialWorkbench = isSpatialWorkbenchPath(pathname); - const creationWorkspace = pathname === "/create"; + const creationWorkspace = pathname === "/"; const isMobileViewport = () => window.innerWidth < 1024; diff --git a/web/src/components/layout/workspace-command-palette.tsx b/web/src/components/layout/workspace-command-palette.tsx index 8eaa02daa..e3646c449 100644 --- a/web/src/components/layout/workspace-command-palette.tsx +++ b/web/src/components/layout/workspace-command-palette.tsx @@ -27,8 +27,7 @@ export function WorkspaceCommandPalette({ open, onClose }: { open: boolean; onCl return { id: slug, title: tool?.label ?? slug, icon: tool?.icon ?? Home, to }; }; return [ - { id: "home", title: "首页", icon: Home, to: "/home" }, - toolEntry("create", "/create"), + { id: "home", title: "首页", icon: Home, to: "/" }, ...(features.shortDramaEnabled ? [toolEntry("projects", "/projects")] : []), toolEntry("canvas", "/canvas"), ...(features.taskCenterEnabled ? [toolEntry("tasks", "/tasks")] : []), diff --git a/web/src/components/layout/workspace-sidebar-nav.tsx b/web/src/components/layout/workspace-sidebar-nav.tsx index 63f6627c4..d9d118423 100644 --- a/web/src/components/layout/workspace-sidebar-nav.tsx +++ b/web/src/components/layout/workspace-sidebar-nav.tsx @@ -44,8 +44,7 @@ function buildNav(features: FeatureAvailability, balance: string, isAdmin: boole const groups: WorkspaceNavGroup[] = [ { items: [ - { id: "home", title: "首页", icon: Home, to: "/home" }, - toolItem("create", "/create"), + { id: "home", title: "首页", icon: Home, to: "/" }, ...(features.shortDramaEnabled ? [toolItem("projects", "/projects")] : []), toolItem("canvas", "/canvas"), ...(features.taskCenterEnabled ? [toolItem("tasks", "/tasks")] : []), @@ -136,7 +135,7 @@ function WorkspaceSwitcher({ collapsed, onNavigate, onExpand }: { collapsed: boo
{[ - { label: "首页", to: "/home" }, + { label: "首页", to: "/" }, { label: "画布", to: "/canvas" }, { label: "设置", to: "/settings" }, ].map((entry) => ( @@ -364,7 +363,7 @@ export function WorkspaceSidebarNav({ collapsed, onNavigate, onOpenSearch, onExp const { groups, footer } = useMemo(() => buildNav(features, balance, user?.role === "admin"), [features, balance, user?.role]); - const slug = pathname.split("/").filter(Boolean)[0] || "create"; + const slug = pathname.split("/").filter(Boolean)[0] || "home"; const section = searchParams.get("section"); const activeId = slug === "settings" && section ? `settings:${section}` : slug; diff --git a/web/src/lib/canvas/canvas-generation-submission.ts b/web/src/lib/canvas/canvas-generation-submission.ts index b5ba25d11..a39d66057 100644 --- a/web/src/lib/canvas/canvas-generation-submission.ts +++ b/web/src/lib/canvas/canvas-generation-submission.ts @@ -25,6 +25,15 @@ export type CanvasGenerationRequestFingerprintInput = { context: Pick; }; +/** + * 编辑器保留带 @ 槽位的原始提示词，任务请求单独保存经过技能/风格处理的模型提示词。 + * 模型提示词也保留 @图片1 等槽位标记。 + * 两者不能互相覆盖，否则刷新后富引用会退化为普通“图片1”文本。 + */ +export function canvasGenerationPromptMetadata(composerContent: string, effectivePrompt: string) { + return { composerContent, prompt: effectivePrompt }; +} + export function canvasGenerationRequestFingerprint(input: CanvasGenerationRequestFingerprintInput) { const serialized = canonicalize({ version: 1, diff --git a/web/src/pages/admin/components/admin-shell.tsx b/web/src/pages/admin/components/admin-shell.tsx index be01e1f83..d7c8a84ed 100644 --- a/web/src/pages/admin/components/admin-shell.tsx +++ b/web/src/pages/admin/components/admin-shell.tsx @@ -144,7 +144,7 @@ export function AdminShell() {
@@ -260,7 +260,7 @@ function MobileAdminNavigation() {
- + diff --git a/web/src/pages/admin/components/api-log-detail-drawer.tsx b/web/src/pages/admin/components/api-log-detail-drawer.tsx index bb36224cb..ebc848d21 100644 --- a/web/src/pages/admin/components/api-log-detail-drawer.tsx +++ b/web/src/pages/admin/components/api-log-detail-drawer.tsx @@ -91,7 +91,7 @@ function LogDetail({ log, querying, onQueryProviderTask }: { log: ApiCallLog; qu {canQueryProviderTask ? (
) : null} diff --git a/web/src/pages/auth/forgot-password.tsx b/web/src/pages/auth/forgot-password.tsx index 2d2d9ae73..10fc8edf1 100644 --- a/web/src/pages/auth/forgot-password.tsx +++ b/web/src/pages/auth/forgot-password.tsx @@ -212,6 +212,6 @@ function Notice({ icon, children }: { icon: ReactNode; children: ReactNode }) { } function safeNext(value: string | null) { - if (!value || !value.startsWith("/") || value.startsWith("//")) return "/create"; + if (!value || !value.startsWith("/") || value.startsWith("//")) return "/"; return value; } diff --git a/web/src/pages/auth/login.tsx b/web/src/pages/auth/login.tsx index 26b0185c1..b9a2ed83f 100644 --- a/web/src/pages/auth/login.tsx +++ b/web/src/pages/auth/login.tsx @@ -111,6 +111,6 @@ function AuthField({ label, htmlFor, action, children }: { label: string; htmlFo } function safeNext(value: string | null) { - if (!value || !value.startsWith("/") || value.startsWith("//")) return "/create"; + if (!value || !value.startsWith("/") || value.startsWith("//")) return "/"; return value; } diff --git a/web/src/pages/auth/register.tsx b/web/src/pages/auth/register.tsx index c26a14094..bd917b528 100644 --- a/web/src/pages/auth/register.tsx +++ b/web/src/pages/auth/register.tsx @@ -121,6 +121,6 @@ function Notice({ icon, tone, children }: { icon: ReactNode; tone: "blue" | "amb } function safeNext(value: string | null) { - if (!value || !value.startsWith("/") || value.startsWith("//")) return "/create"; + if (!value || !value.startsWith("/") || value.startsWith("//")) return "/"; return value; } diff --git a/web/src/pages/canvas/canvas-image-generation-executor.ts b/web/src/pages/canvas/canvas-image-generation-executor.ts index af4335338..46af1db0d 100644 --- a/web/src/pages/canvas/canvas-image-generation-executor.ts +++ b/web/src/pages/canvas/canvas-image-generation-executor.ts @@ -5,6 +5,7 @@ import { canGenerateImageInPlace, findAvailableGenerationGroupPosition, imageGen import { buildImageGenerationNodeTitle } from "@/lib/canvas/canvas-generation-title"; import { nodeSizeFromRatio } from "@/lib/canvas/canvas-node-size"; import { canvasImageReferenceLimitError, buildImageGenerationMetadata, getGenerationCount, isGenerationCanceled, runCanvasGenerationTaskToConsumer } from "@/lib/canvas/canvas-project-generation"; +import { canvasGenerationPromptMetadata } from "@/lib/canvas/canvas-generation-submission"; import { CONTENT_MODERATION_ERROR_CODE, generationFailureMetadata, type GenerationFailureMetadata } from "@/lib/generation-error"; import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas"; import { useCanvasStore } from "@/stores/canvas/use-canvas-store"; @@ -89,7 +90,7 @@ export async function executeImageGeneration({ height: rootHeight, metadata: { ...(reuseSourceNode ? sourceNode?.metadata || {} : {}), - prompt: effectivePrompt, + ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, size: generationConfig.size, isBatchRoot: count > 1, @@ -115,7 +116,7 @@ export async function executeImageGeneration({ width: outputNodeSize.width, height: outputNodeSize.height, metadata: { - prompt: effectivePrompt, + ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, size: generationConfig.size, batchRootId: count > 1 && !directCopiedBatch ? rootId : undefined, @@ -134,7 +135,7 @@ export async function executeImageGeneration({ const nextNodes: CanvasNodeData[] = [ ...canvasNodes.map((node) => { if (node.id !== nodeId) return node; - if (isConfigNode) return { ...node, metadata: { ...node.metadata, prompt: effectivePrompt, status: NODE_STATUS_LOADING, errorDetails: undefined } }; + if (isConfigNode) return { ...node, metadata: { ...node.metadata, ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, errorDetails: undefined } }; if (reuseSourceNode) return { ...node, position: rootNode.position, width: rootNode.width, height: rootNode.height, title: rootNode.title, metadata: { ...node.metadata, ...rootNode.metadata, errorDetails: undefined } }; if (isImageNode) return node; return { @@ -143,7 +144,7 @@ export async function executeImageGeneration({ title: prompt.slice(0, 32) || "Prompt", width: parentConfig.width, height: parentConfig.height, - metadata: { ...node.metadata, content: prompt, richText: undefined, prompt, status: NODE_STATUS_SUCCESS, fontSize: 14, errorDetails: undefined }, + metadata: { ...node.metadata, content: prompt, richText: undefined, prompt, composerContent: prompt, status: NODE_STATUS_SUCCESS, fontSize: 14, errorDetails: undefined }, }; }), ...(reuseSourceNode || directCopiedBatch ? [] : [rootNode]), diff --git a/web/src/pages/canvas/canvas-media-generation-executors.ts b/web/src/pages/canvas/canvas-media-generation-executors.ts index 465879244..e64493b5e 100644 --- a/web/src/pages/canvas/canvas-media-generation-executors.ts +++ b/web/src/pages/canvas/canvas-media-generation-executors.ts @@ -5,6 +5,7 @@ import { canGenerateMediaInPlace } from "@/lib/canvas/canvas-generation-layout"; import { nodeSizeFromRatio } from "@/lib/canvas/canvas-node-size"; import { nextCanvasVersionLabel } from "@/lib/canvas/canvas-layout"; import { buildAudioGenerationMetadata, buildVideoGenerationMetadata, generationReferenceUrls, runCanvasGenerationTaskToConsumer } from "@/lib/canvas/canvas-project-generation"; +import { canvasGenerationPromptMetadata } from "@/lib/canvas/canvas-generation-submission"; import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas"; import type { CanvasGenerationExecution } from "./canvas-generation-executor-types"; @@ -15,6 +16,7 @@ const NODE_STATUS_SUCCESS = "success" as const; export async function executeVideoGeneration({ nodeId, sourceNode, + prompt, effectivePrompt, generationConfig, generationContext, @@ -49,7 +51,7 @@ export async function executeVideoGeneration({ height: reuseSourceNode ? sourceNode!.height : spec.height, metadata: { ...(reuseSourceNode ? sourceNode?.metadata || {} : {}), - prompt: effectivePrompt, + ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, errorDetails: undefined, generationErrorCode: undefined, @@ -130,6 +132,7 @@ export async function executeVideoGeneration({ export async function executeAudioGeneration({ nodeId, sourceNode, + prompt, effectivePrompt, generationConfig, generationContext, @@ -157,7 +160,7 @@ export async function executeAudioGeneration({ position: isEmptyAudioNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y + ((sourceNode?.height || spec.height) - spec.height) / 2 }, width: isEmptyAudioNode ? sourceNode.width : spec.width, height: isEmptyAudioNode ? sourceNode.height : spec.height, - metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, ...buildAudioGenerationMetadata(generationConfig), ...skillMetadata }, + metadata: { ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, ...buildAudioGenerationMetadata(generationConfig), ...skillMetadata }, }; registerPendingNodeIds([audioId]); setNodes((current) => diff --git a/web/src/pages/canvas/canvas-project-world-layers.tsx b/web/src/pages/canvas/canvas-project-world-layers.tsx index 613889daa..38a49a3cb 100644 --- a/web/src/pages/canvas/canvas-project-world-layers.tsx +++ b/web/src/pages/canvas/canvas-project-world-layers.tsx @@ -144,7 +144,6 @@ export const CanvasProjectWorldLayers = memo(function CanvasProjectWorldLayers(p scale={viewportScale} isSelected={props.selectedNodeIds.has(node.id)} mediaActive={activeMediaNodeId === node.id} - hydrateMediaPreview={activeMediaNodeId === null && !props.reduceMediaEffects} isRelated={props.relatedNodeIds.has(node.id)} isFocusRelated={props.activeNodeId === node.id} isConnectionTarget={props.connectionTargetNodeId === node.id || props.batchConnectionPreview?.targetNodeId === node.id} diff --git a/web/src/pages/canvas/canvas-text-generation-executor.ts b/web/src/pages/canvas/canvas-text-generation-executor.ts index 3585dc8d0..3df8bbd11 100644 --- a/web/src/pages/canvas/canvas-text-generation-executor.ts +++ b/web/src/pages/canvas/canvas-text-generation-executor.ts @@ -2,6 +2,7 @@ import { nanoid } from "nanoid"; import { NODE_DEFAULT_SIZE } from "@/constant/canvas"; import { getGenerationCount, runCanvasGenerationTaskToConsumer } from "@/lib/canvas/canvas-project-generation"; +import { canvasGenerationPromptMetadata } from "@/lib/canvas/canvas-generation-submission"; import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas"; import type { CanvasGenerationExecution } from "./canvas-generation-executor-types"; @@ -51,9 +52,9 @@ export async function executeTextGeneration({ }, width: textConfig.width, height: textConfig.height, - metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, fontSize: 14, ...skillMetadata }, + metadata: { ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, fontSize: 14, ...skillMetadata }, })); - setNodes((current) => [...current.map((node) => (node.id === nodeId && isConfigNode ? { ...node, metadata: { ...node.metadata, prompt: effectivePrompt, status: NODE_STATUS_LOADING, errorDetails: undefined } } : node)), ...childNodes]); + setNodes((current) => [...current.map((node) => (node.id === nodeId && isConfigNode ? { ...node, metadata: { ...node.metadata, ...canvasGenerationPromptMetadata(prompt, effectivePrompt), status: NODE_STATUS_LOADING, errorDetails: undefined } } : node)), ...childNodes]); setConnections((current) => [...current, ...childIds.map((childId) => ({ id: nanoid(), fromNodeId: nodeId, toNodeId: childId }))]); } diff --git a/web/src/pages/canvas/use-canvas-generation-executor.ts b/web/src/pages/canvas/use-canvas-generation-executor.ts index 75f2d3441..61cac6c84 100644 --- a/web/src/pages/canvas/use-canvas-generation-executor.ts +++ b/web/src/pages/canvas/use-canvas-generation-executor.ts @@ -4,7 +4,7 @@ import { App } from "antd"; import { buildNodeGenerationContext, hydrateNodeGenerationContext } from "@/components/canvas/canvas-node-generation"; import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel"; import { buildGenerationConfig, isGenerationCanceled } from "@/lib/canvas/canvas-project-generation"; -import { canvasGenerationRequestFingerprint, runCanvasGenerationSubmissionOnce } from "@/lib/canvas/canvas-generation-submission"; +import { canvasGenerationPromptMetadata, canvasGenerationRequestFingerprint, runCanvasGenerationSubmissionOnce } from "@/lib/canvas/canvas-generation-submission"; import { isGenerationTaskCapacityError } from "@/lib/canvas/canvas-generation-batch"; import { buildPortraitTexturePrompt } from "@/lib/canvas/canvas-portrait-texture"; import { resolveCanvasStyleExecution } from "@/lib/canvas/canvas-style-execution"; @@ -242,6 +242,7 @@ export function useCanvasGenerationExecutor({ metadata: { ...node.metadata, prompt, + composerContent: prompt, status: NODE_STATUS_LOADING, taskStage: "正在准备生成任务", taskProgress: 0, @@ -266,7 +267,7 @@ export function useCanvasGenerationExecutor({ node.id === nodeId ? { ...node, - metadata: { ...node.metadata, prompt: statusPrompt, status: NODE_STATUS_LOADING, errorDetails: undefined, generationErrorCode: undefined, resourceReloadAvailable: undefined, failedPromptFingerprint: undefined }, + metadata: { ...node.metadata, ...canvasGenerationPromptMetadata(prompt, statusPrompt), status: NODE_STATUS_LOADING, errorDetails: undefined, generationErrorCode: undefined, resourceReloadAvailable: undefined, failedPromptFingerprint: undefined }, } : node, ), diff --git a/web/src/pages/create/index.tsx b/web/src/pages/create/index.tsx index a3c708c7f..8a79f351c 100644 --- a/web/src/pages/create/index.tsx +++ b/web/src/pages/create/index.tsx @@ -25,13 +25,11 @@ import { buildImageResolutionOptions, formatImageResolutionSize, imageRatioForSi import { formatVideoResolutionLabel as videoResolutionLabel, VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options"; import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, videoDurationAllowed, videoDurationOptions, type ImageCapabilityConfig, type VideoCapabilityConfig } from "@/lib/model-capabilities"; import { inferVideoOperation, resolveCompatibleModel, mergedImageCapabilityConfig, type ModelRequirements } from "@/lib/model-selection"; -import { backendModelRuntimeRequired, isGenerationTaskCancelled, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task"; -import { requestImageQuestion, type AiTextContentPart } from "@/services/api/image"; +import { isGenerationTaskCancelled, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task"; import { listAddedSkills, type Skill } from "@/services/api/skills"; import { subscribeGenerationTasks, type GenerationTask } from "@/services/api/task-center"; -import { createTextReplayPublisher } from "@/lib/creation-text-replay"; import { isLocalDreaminaWaitStopped, localDreaminaCancellationMessage } from "@/services/local-dreamina-task-projection"; -import { getMediaBlob, uploadMediaFile } from "@/services/file-storage"; +import { uploadMediaFile } from "@/services/file-storage"; import { uploadImage } from "@/services/image-storage"; import { consumeGenerationTaskMessage, generationTaskMaterializedUrls, materializeGenerationTaskAssets, projectGenerationTaskResult } from "@/services/project-asset-sync"; import { applyGenerationConsumerEffect } from "@/services/generation-consumer-dedupe"; @@ -100,7 +98,6 @@ const qualityOptions = [ ]; const resolutionOptions = VIDEO_RESOLUTION_OPTIONS.map((value) => ({ value: String(value), label: videoResolutionLabel(value) })); const countOptions = ["1", "2", "3", "4"]; -const TEXT_ATTACHMENT_MAX_BYTES = 20 * 1024 * 1024; const conversationTimeFormatter = new Intl.DateTimeFormat("zh-CN", { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", hour12: false }); const messageTimeFormatter = new Intl.DateTimeFormat("zh-CN", { hour: "2-digit", minute: "2-digit", hour12: false }); @@ -634,43 +631,22 @@ export default function CreatePage() { }; try { if (mode === "text") { - if (backendModelRuntimeRequired(requestConfig)) { - const result = await runGenerationOperationOnce(retryContext?.clientOperationId, () => runBackendGenerationTask({ - mode: "text", - prompt: expandedPrompt, - config: requestConfig, - referenceImages, - referenceVideos, - referenceAudios, - textHistory: (activeConversation.messages || []).filter((item) => item.content.trim()).map((item) => ({ role: item.role, content: item.content })), - signal: requestLifecycle.signal, - metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, ...referenceMetadata }, - onTaskUpdate: bindTask, - onTextDelta: (text) => updateOriginAssistant((item) => ({ ...item, content: text })), - ...retryContext, - })); - if (!result.text?.trim()) throw new Error("后端任务没有返回文本"); - updateOriginAssistant((item) => ({ ...item, content: result.text || "" })); - } else { - const history = await Promise.all([...(activeConversation.messages || []), userMessage].map(async (item) => ({ - role: item.role, - content: item.role === "user" - ? await buildTextMessageContent(item) - : item.content, - }))); - const replayPublisher = createTextReplayPublisher(requestConfig, text); - void replayPublisher.start(); - let finalText = ""; - await requestImageQuestion(requestConfig, history, (full) => { - finalText = full; - updateOriginAssistant((item) => ({ ...item, content: full })); - replayPublisher.publish(full); - }, { - signal: requestLifecycle.signal, - onReasoning: (reasoning) => updateOriginAssistant((item) => ({ ...item, reasoning })), - }); - replayPublisher.finish(finalText); - } + const result = await runGenerationOperationOnce(retryContext?.clientOperationId, () => runBackendGenerationTask({ + mode: "text", + prompt: expandedPrompt, + config: requestConfig, + referenceImages, + referenceVideos, + referenceAudios, + textHistory: (activeConversation.messages || []).filter((item) => item.content.trim()).map((item) => ({ role: item.role, content: item.content })), + signal: requestLifecycle.signal, + metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, ...referenceMetadata }, + onTaskUpdate: bindTask, + onTextDelta: (value) => updateOriginAssistant((item) => ({ ...item, content: value })), + ...retryContext, + })); + if (!result.text?.trim()) throw new Error("后端任务没有返回文本"); + updateOriginAssistant((item) => ({ ...item, content: result.text || "", reasoning: result.reasoning })); } else if (mode === "image") { const taskCount = Math.max(1, Math.min(imageProfile.maxOutputs, Math.floor(Number(count) || 1))); const settled = await runGenerationOperationOnce(retryContext?.clientOperationId, () => runBackendGenerationTaskBatch({ @@ -1795,44 +1771,6 @@ function conversationPreviewMessage(conversation: CreationConversation) { return fallback; } -async function buildTextMessageContent(item: CreationMessage) { - const content = expandCreationPrompt(item.content, item.references || [], item.attachments || []); - const attachments = item.attachments || []; - if (!attachments.length) return content; - const parts: AiTextContentPart[] = [{ type: "text", text: content }]; - for (const attachment of attachments) { - if (isImageAttachment(attachment)) { - parts.push({ type: "image_url", image_url: { url: attachment.dataUrl || attachment.url || "" } }); - continue; - } - const url = await creationAttachmentDataUrl(attachment); - parts.push({ type: "file_url", file_url: { url, name: attachment.name || "附件", mimeType: attachment.type || "application/octet-stream" } }); - } - return parts; -} - -async function creationAttachmentDataUrl(attachment: CreationAttachment) { - if ((attachment.bytes || 0) > TEXT_ATTACHMENT_MAX_BYTES) throw new Error(`${attachment.name} 超过 20MB，当前文本模型附件需要压缩后再上传`); - const attachmentUrl = attachment.url || ""; - if (attachmentUrl.startsWith("data:")) return attachmentUrl; - const blob = attachment.storageKey ? await getMediaBlob(attachment.storageKey) : null; - if (blob) { - if (blob.size > TEXT_ATTACHMENT_MAX_BYTES) throw new Error(`${attachment.name} 超过 20MB，当前文本模型附件需要压缩后再上传`); - return blobToDataUrl(blob); - } - if (/^https:\/\//i.test(attachmentUrl)) return attachmentUrl; - throw new Error(`${attachment.name} 无法读取，请重新上传后再试`); -} - -function blobToDataUrl(blob: Blob) { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(String(reader.result || "")); - reader.onerror = () => reject(reader.error || new Error("附件读取失败")); - reader.readAsDataURL(blob); - }); -} - function isVideoAttachment(attachment: CreationAttachment): attachment is CreationAttachment & { url: string } { return creationAttachmentKind(attachment) === "video"; } diff --git a/web/src/pages/home/index.tsx b/web/src/pages/home/index.tsx deleted file mode 100644 index ba6407269..000000000 --- a/web/src/pages/home/index.tsx +++ /dev/null @@ -1,594 +0,0 @@ -import { useMemo, useState, type ReactNode } from "react"; -import { useQuery } from "@tanstack/react-query"; -import { App, Button, Input, Pagination, Segmented, Select } from "antd"; -import { ArrowRight, FolderKanban, Images, LayoutGrid, ListTodo, Plus, Search, Sparkles } from "lucide-react"; -import { Link, useNavigate } from "react-router"; -import { Bar, BarChart, CartesianGrid, Cell, Pie, PieChart, ResponsiveContainer, Tooltip as ChartTooltip, XAxis, YAxis } from "recharts"; - -import { resolveCanvasStylePreset, resolveProjectCanvasStyle } from "@/components/canvas/canvas-style-picker-modal"; -import { WorkspaceErrorState, WorkspaceLoadingState } from "@/components/layout/workspace-state"; -import { parseStyleProfile } from "@/lib/canvas/style-profile"; -import { projectSummaryCompletion, projectSummaryStage } from "@/lib/project-workbench"; -import { listProjects, type ProjectSummary } from "@/services/api/projects"; -import { listGenerationTasks } from "@/services/api/task-center"; -import { createCanvasProjectWithRemoteSync } from "@/services/user-data-sync"; -import { useAssetStore, type AssetKind } from "@/stores/use-asset-store"; -import { useCanvasStore } from "@/stores/canvas/use-canvas-store"; -import { useUserStore } from "@/stores/use-user-store"; - -const workflow = [ - { title: "整理故事", description: "导入小说、粘贴文本或创建章节" }, - { title: "确认设定", description: "整理角色、场景、画风和参考资料" }, - { title: "制作镜头", description: "生成分镜、图片和视频候选" }, - { title: "检查结果", description: "比较版本、处理失败并整理导出" }, -]; - -const sourceTypeLabels: Record = { - blank: "空白开始", - novel: "导入小说", - text: "粘贴文本", -}; - -const ASSET_KINDS: Array<{ key: AssetKind; label: string }> = [ - { key: "image", label: "图片" }, - { key: "video", label: "视频" }, - { key: "audio", label: "音频" }, - { key: "text", label: "文本" }, - { key: "entity", label: "实体" }, - { key: "model", label: "模型" }, -]; - -const CHART_ACCENT = "var(--workspace-accent)"; -const CHART_MUTED = "color-mix(in srgb, var(--workspace-accent) 30%, transparent)"; - -export default function IndexPage() { - const { message } = App.useApp(); - const navigate = useNavigate(); - const canvasHydrated = useCanvasStore((state) => state.hydrated); - const canvasProjects = useCanvasStore((state) => state.projects); - const assets = useAssetStore((state) => state.assets); - const user = useUserStore((state) => state.user); - const userHydrated = useUserStore((state) => state.hydrated); - const shortDramaEnabled = useUserStore((state) => state.features.shortDramaEnabled); - const taskCenterEnabled = useUserStore((state) => state.features.taskCenterEnabled); - - const domainProjectsQuery = useQuery({ - queryKey: ["projects"], - queryFn: () => listProjects(), - enabled: Boolean(user && shortDramaEnabled), - }); - const domainProjects = useMemo(() => [...(domainProjectsQuery.data?.projects || [])].sort((left, right) => right.project.updatedAt.localeCompare(left.project.updatedAt)), [domainProjectsQuery.data]); - - // 仪表盘需要生成记录做活跃度与任务统计；后端不可用时降级为空数据，不影响其他区块。 - const tasksQuery = useQuery({ - queryKey: ["home-dashboard-tasks"], - queryFn: () => listGenerationTasks(300).catch(() => []), - enabled: Boolean(user && taskCenterEnabled), - }); - const tasks = useMemo(() => tasksQuery.data || [], [tasksQuery.data]); - - const createIndependentCanvas = () => { - if (!canvasHydrated) return; - if (!user) { - navigate(`/login?next=${encodeURIComponent("/canvas?mode=new")}`); - return; - } - void createCanvasProjectWithRemoteSync(`自由画布 ${canvasProjects.length + 1}`).then(({ id, syncError }) => { - if (syncError) message.warning(syncError instanceof Error ? `画布已在本地创建，云端同步失败：${syncError.message}` : "画布已在本地创建，云端同步失败"); - navigate(`/canvas/${id}`); - }); - }; - - const loadingUserWorkspace = !userHydrated || (Boolean(user && shortDramaEnabled) && domainProjectsQuery.isLoading); - return ( -
-
- {loadingUserWorkspace ? ( - - ) : user && shortDramaEnabled && domainProjectsQuery.isError ? ( - void domainProjectsQuery.refetch()} /> - ) : ( - - )} -
-
- ); -} - -function HomeDashboard({ - user, - shortDramaEnabled, - domainProjects, - canvasProjects, - assets, - tasks, - canvasHydrated, - onCreateIndependentCanvas, -}: { - user: ReturnType["user"]; - shortDramaEnabled: boolean; - domainProjects: ProjectSummary[]; - canvasProjects: ReturnType["projects"]; - assets: ReturnType["assets"]; - tasks: Awaited>; - canvasHydrated: boolean; - onCreateIndependentCanvas: () => void; -}) { - const hasProjects = Boolean(domainProjects.length); - const displayName = user?.displayName || user?.username || "创作者"; - - const activeProjects = useMemo(() => domainProjects.filter(({ project }) => project.status !== "archived"), [domainProjects]); - const archivedCount = domainProjects.length - activeProjects.length; - const totalUnits = useMemo(() => activeProjects.reduce((sum, row) => sum + row.unitCount, 0), [activeProjects]); - const activeProject = activeProjects[0]; - - const independentCanvases = useMemo(() => canvasProjects.filter((project) => !project.projectId), [canvasProjects]); - const latestCanvasUpdate = useMemo(() => canvasProjects.reduce((latest, project) => (latest === null || project.updatedAt > latest ? project.updatedAt : latest), null), [canvasProjects]); - - const weekWindow = Date.now() - 7 * 24 * 60 * 60 * 1000; - const weekTasks = useMemo(() => tasks.filter((task) => new Date(task.createdAt).getTime() >= weekWindow), [tasks, weekWindow]); - const weekSucceeded = useMemo(() => weekTasks.filter((task) => task.status === "succeeded").length, [weekTasks]); - const weekFailed = useMemo(() => weekTasks.filter((task) => task.status === "failed" || task.status === "cancelled").length, [weekTasks]); - const weekRunning = useMemo(() => weekTasks.filter((task) => task.status === "queued" || task.status === "running").length, [weekTasks]); - - const assetKindCounts = useMemo(() => { - const counts: Partial> = {}; - for (const asset of assets) counts[asset.kind] = (counts[asset.kind] || 0) + 1; - return counts; - }, [assets]); - - const [rangeDays, setRangeDays] = useState(14); - const activityData = useMemo( - () => - buildActivityBuckets( - rangeDays, - tasks.map((task) => new Date(task.createdAt).getTime()), - canvasProjects.map((project) => new Date(project.updatedAt).getTime()), - ), - [rangeDays, tasks, canvasProjects], - ); - const assetChartEntries = useMemo(() => assetChartData(assets), [assets]); - const assetTotal = useMemo(() => assets.length, [assets]); - - const [keyword, setKeyword] = useState(""); - const [status, setStatus] = useState<"all" | "active" | "archived">("all"); - const [sort, setSort] = useState<"updated" | "progress" | "name">("updated"); - const [page, setPage] = useState(1); - const rows = useMemo(() => { - const normalizedKeyword = keyword.trim().toLowerCase(); - return domainProjects - .filter(({ project }) => status === "all" || project.status === status) - .filter(({ project }) => !normalizedKeyword || `${project.name} ${project.description || ""} ${projectStyleTitle(project)}`.toLowerCase().includes(normalizedKeyword)) - .sort((left, right) => { - if (sort === "name") return left.project.name.localeCompare(right.project.name, "zh-CN"); - if (sort === "progress") return projectSummaryCompletion(right) - projectSummaryCompletion(left); - return right.project.updatedAt.localeCompare(left.project.updatedAt); - }); - }, [domainProjects, keyword, sort, status]); - const pageSize = 5; - const safePage = Math.min(page, Math.max(1, Math.ceil(rows.length / pageSize))); - const pageRows = rows.slice((safePage - 1) * pageSize, safePage * pageSize); - - const projectHref = user ? "/projects?create=1" : `/login?next=${encodeURIComponent("/projects?create=1")}`; - - return ( - <> -
-
-

{hasProjects ? `欢迎回来，${displayName}` : `欢迎使用影策，${displayName}`}

-

{hasProjects ? "回到最近制作，或从一句话故事开始一部新的短剧。" : "从一个故事开始：整理章节、确认设定、制作镜头，直到可交付的结果。"}

-
-
- {shortDramaEnabled ? ( - - - 开始创作 - - ) : null} - -
-
- -
- {shortDramaEnabled ? ( - } label="进行中项目" value={activeProjects.length} hint={activeProjects.length ? `共 ${totalUnits} 章 · ${archivedCount} 个已归档` : "创建项目开始制作"} to="/projects" /> - ) : null} - } label="自由画布" value={independentCanvases.length} hint={latestCanvasUpdate ? `最近更新 ${formatRelativeTime(latestCanvasUpdate)}` : "新建独立创作空间"} to="/canvas" /> - } label="素材资产" value={assets.length} hint={assetSummaryHint(assetKindCounts)} to="/assets" /> - } label="本周任务" value={weekTasks.length} hint={weekTasks.length ? `成功 ${weekSucceeded} · 失败 ${weekFailed} · 进行中 ${weekRunning}` : "本周还没有生成任务"} to="/tasks" /> -
- -
-
-
-
-

创作活跃度

-

按天统计生成任务与画布更新

-
- setRangeDays(value as number)} - /> -
-
- {activityData.some((item) => item.tasks > 0 || item.canvases > 0) ? ( - - - - 14 ? 4 : rangeDays > 7 ? 2 : 0} - tick={{ fill: "var(--foreground)", opacity: 0.45, fontSize: 11 }} - /> - - - - - - - ) : ( -
还没有生成记录，去画布或项目开始第一次创作。
- )} -
-
- -
-
-
-

素材构成

-

按类型统计当前素材库

-
-
-
- {assetTotal ? ( -
-
- - - - {assetChartEntries.map((entry) => ( - - ))} - - - - -
- {assetTotal} - 项素材 -
-
-
    - {assetChartEntries.map((entry) => ( -
  • - - {entry.name} - {entry.value} -
    • - ))} -
-
- ) : ( -
还没有素材资产，生成的图片、视频和音频会沉淀在这里。
- )} -
-
-
- -
-
-
-
-

最近项目

-

按更新时间排列，支持搜索、筛选与排序

-
- - 查看全部 - - -
-
- } - value={keyword} - placeholder="搜索项目名称或画风" - onChange={(event) => { - setKeyword(event.target.value); - setPage(1); - }} - aria-label="搜索项目" - /> - { - setSort(value as "updated" | "progress" | "name"); - setPage(1); - }} - options={[ - { label: "最近更新", value: "updated" }, - { label: "章节进度", value: "progress" }, - { label: "项目名称", value: "name" }, - ]} - /> -
-
-
- - {rows.length ? ( - pageRows.map((row) => ) - ) : ( -
- {domainProjects.length ? ( - <> - -

没有匹配的项目

-

调整搜索词或状态筛选后再试。

- - ) : shortDramaEnabled ? ( - <> - -

创建第一个故事项目

-

项目会集中保存章节、项目画布、角色场景和制作进度，从这里开始一部短剧。

- - - 创建项目 - - - ) : ( - <> - -

项目功能尚未启用

-

可以从自由画布开始创作，或联系管理员开启短剧项目。

- - )} -
- )} -
-
-
- - 共 {rows.length} 个项目{rows.length ? `，每页 ${pageSize} 条` : ""} - - {rows.length > pageSize ? : null} -
-
- -
-
-
-
-

快捷创建

-

常用创作入口

-
-
-
- {shortDramaEnabled ? } title="创建短剧项目" description="从空白、小说或文本建立章节流程" href={projectHref} /> : null} - } title="打开自由画布" description="适合快速试图与提示词实验" onClick={onCreateIndependentCanvas} disabled={!canvasHydrated} /> - } title="进入素材库" description="整理角色、场景与媒体资产" href="/assets" /> -
-
- -
-
-
-

从工作流开始

-

整理故事 → 确认设定 → 制作镜头 → 检查结果

-
-
-
-
- {workflow.map((item, index) => ( -
- 0{index + 1} - - {item.title} - {item.description} - -
- ))} -
-

{activeProject ? `${projectSummaryStage(activeProject).label} · ${projectSummaryCompletion(activeProject)}%` : "从一句话故事开始，逐步推进到可交付的镜头。"}

-
-
-
-
- - ); -} - -function StatCard({ icon, label, value, hint, to }: { icon: ReactNode; label: string; value: number; hint: string; to: string }) { - return ( - - - {icon} - {value} - - - {label} - {hint} - - - ); -} - -function RecentProjectRow({ row }: { row: ProjectSummary }) { - const completion = projectSummaryCompletion(row); - const stage = projectSummaryStage(row); - const archived = row.project.status === "archived"; - return ( - - - - {row.project.name} - {archived ? 已归档 : null} - - - {projectStyleTitle(row.project)} · {sourceTypeLabel(row.project.sourceType)} · {row.canvasCount} 画布 · {row.assetCount} 资产 - - - - {stage.label} - - - - - {row.completedUnitCount}/{row.unitCount} 章 - - {completion}% - - - - - - {formatRelativeTime(row.project.updatedAt)} - - - ); -} - -function QuickItem({ icon, title, description, href, onClick, disabled }: { icon: ReactNode; title: string; description: string; href?: string; onClick?: () => void; disabled?: boolean }) { - const content = ( - <> - {icon} - - {title} - {description} - - - - ); - const className = "home-quick-item group focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-60"; - return href ? ( - - {content} - - ) : ( - - ); -} - -function projectStyleTitle(project: ProjectSummary["project"]) { - const projectStyle = resolveProjectCanvasStyle(project.stylePresetId, project.styleProfileJson); - return projectStyle?.title || parseStyleProfile(project.styleProfileJson)?.title || resolveCanvasStylePreset(project.stylePresetId)?.title || (project.stylePresetId ? "自定义画风" : "未设置画风"); -} - -function sourceTypeLabel(value: string) { - return sourceTypeLabels[value] || "其他来源"; -} - -function assetSummaryHint(counts: Partial>) { - const parts = ASSET_KINDS.map(({ key, label }) => ({ label, count: counts[key] || 0 })).filter(({ count }) => count > 0); - if (!parts.length) return "从画布或项目中沉淀素材"; - return parts - .slice(0, 3) - .map(({ label, count }) => `${label} ${count}`) - .join(" · "); -} - -function assetChartData(assets: ReturnType["assets"]) { - const colors = [ - CHART_ACCENT, - "color-mix(in srgb, var(--workspace-accent) 66%, transparent)", - "color-mix(in srgb, var(--workspace-accent) 44%, transparent)", - "color-mix(in srgb, var(--workspace-accent) 28%, transparent)", - "color-mix(in srgb, var(--workspace-accent) 16%, transparent)", - ]; - const counts = new Map(); - for (const asset of assets) counts.set(asset.kind, (counts.get(asset.kind) || 0) + 1); - const entries = ASSET_KINDS.filter(({ key }) => (counts.get(key) || 0) > 0); - return entries.map(({ key, label }, index) => ({ name: label, value: counts.get(key) || 0, color: colors[index % colors.length] })); -} - -function buildActivityBuckets(days: number, taskDates: number[], canvasDates: number[]) { - const buckets: Array<{ label: string; tasks: number; canvases: number }> = []; - const today = new Date(); - today.setHours(0, 0, 0, 0); - for (let offset = days - 1; offset >= 0; offset -= 1) { - const day = new Date(today); - day.setDate(today.getDate() - offset); - const start = day.getTime(); - const end = start + 24 * 60 * 60 * 1000; - buckets.push({ - label: `${day.getMonth() + 1}/${day.getDate()}`, - tasks: taskDates.filter((time) => time >= start && time < end).length, - canvases: canvasDates.filter((time) => time >= start && time < end).length, - }); - } - return buckets; -} - -const tooltipStyle = { - background: "var(--workspace-surface-strong)", - border: "1px solid var(--workspace-border)", - borderRadius: "var(--r-md)", - boxShadow: "var(--elevation-card-hover)", - fontSize: 12, -}; - -function formatRelativeTime(value: string) { - const diffMinutes = Math.round((new Date(value).getTime() - Date.now()) / 60_000); - const formatter = new Intl.RelativeTimeFormat("zh-CN", { numeric: "auto" }); - if (Math.abs(diffMinutes) < 60) return formatter.format(diffMinutes, "minute"); - const diffHours = Math.round(diffMinutes / 60); - if (Math.abs(diffHours) < 24) return formatter.format(diffHours, "hour"); - const diffDays = Math.round(diffHours / 24); - if (Math.abs(diffDays) < 30) return formatter.format(diffDays, "day"); - return new Date(value).toLocaleDateString("zh-CN", { month: "2-digit", day: "2-digit" }); -} diff --git a/web/src/pages/projects/index.tsx b/web/src/pages/projects/index.tsx index 93015a2f0..69075a02c 100644 --- a/web/src/pages/projects/index.tsx +++ b/web/src/pages/projects/index.tsx @@ -12,7 +12,7 @@ import { ModelPicker } from "@/components/model-picker"; import { createStyleProfileSnapshot, parseStyleProfile, serializeStyleProfile } from "@/lib/canvas/style-profile"; import { projectSummaryCompletion, projectSummaryStage } from "@/lib/project-workbench"; import { settingsPath } from "@/lib/settings-navigation"; -import { requestImageQuestion } from "@/services/api/image"; +import { runBackendGenerationTask } from "@/services/api/generation-task"; import { createProject, deleteProject, importProjectUnits, listProjects, type ProjectSummary } from "@/services/api/projects"; import { modelDisplayName, useEffectiveConfig } from "@/stores/use-config-store"; @@ -89,16 +89,16 @@ export default function ProjectsPage() { try { const project = await createUniqueProjectName(story, selectedStyle); setGenerationStatus("AI 正在生成故事大纲与章节…"); - const answer = await requestImageQuestion( - { ...effectiveConfig, model: textModel, imageModel: textModel, videoModel: textModel, textModel }, - [ - { role: "system", content: `你是短剧编剧。根据用户的一句话故事，生成一部短剧的标题、一句话简介和 ${generateChapterCount} 个章节。生成要求：叙事采用${generateStructure}结构，每章约 ${generateWordCount} 字，使用${generatePerspective}视角，整体基调${generateTone}，主要角色约 ${generateCharacterScale}，章节篇幅${generateChapterLength}。只输出一个 JSON 对象，不要输出 markdown 代码块或其他文字。JSON 结构：{"title":"剧名","synopsis":"一句话简介","chapters":[{"title":"章节标题","content":"本章情节"}]}` }, - { role: "user", content: story }, - ], - (text) => { - setGenerationPreview(text); - }, - ); + const systemPrompt = `你是短剧编剧。根据用户的一句话故事，生成一部短剧的标题、一句话简介和 ${generateChapterCount} 个章节。生成要求：叙事采用${generateStructure}结构，每章约 ${generateWordCount} 字，使用${generatePerspective}视角，整体基调${generateTone}，主要角色约 ${generateCharacterScale}，章节篇幅${generateChapterLength}。只输出一个 JSON 对象，不要输出 markdown 代码块或其他文字。JSON 结构：{"title":"剧名","synopsis":"一句话简介","chapters":[{"title":"章节标题","content":"本章情节"}]}`; + const result = await runBackendGenerationTask({ + projectId: project.project.id, + mode: "text", + prompt: story, + config: { ...effectiveConfig, model: textModel, imageModel: textModel, videoModel: textModel, textModel, systemPrompt }, + metadata: { source: "project-story-generator", projectId: project.project.id }, + onTextDelta: setGenerationPreview, + }); + const answer = result.text || ""; const parsed = parseGeneratedStory(answer); if (!parsed.chapters.length) throw new Error("AI 没有返回有效的章节内容，请重试"); setGenerationStatus(`正在导入 ${parsed.chapters.length} 个章节…`); diff --git a/web/src/services/api/generation-task.ts b/web/src/services/api/generation-task.ts index 35a9596fb..21772b279 100644 --- a/web/src/services/api/generation-task.ts +++ b/web/src/services/api/generation-task.ts @@ -16,17 +16,6 @@ import { buildBackendToolRequests, type ResponseFunctionTool, type ResponseInput export { logicalModelIDForConfig }; -// 只要模型明确选择了请求协议，就统一交给后端协议运行时执行：逻辑模型 -// 由逻辑模型路由解析协议，系统渠道由 channelId 解析真实渠道模型，用户 -// 自建渠道则由所选协议插件负责第三方字段映射和结果解析。没有显式协议 -// 的旧配置继续保留前端直连兼容路径。 -export function backendModelRuntimeRequired(config: AiConfig) { - if ((config.taskWorkflowProvider || "model") !== "model") return true; - if (logicalModelIDForConfig(config)) return true; - const requestConfig = resolveModelRequestConfig(config, config.model); - return Boolean(requestConfig.channelId || requestConfig.interfaceType); -} - export type BackendGenerationMode = "text" | "image" | "video" | "audio"; export type BackendGenerationResult = { @@ -47,7 +36,7 @@ type BackendGenerationTaskOptions = { referenceImages?: ReferenceImage[]; referenceVideos?: ReferenceVideo[]; referenceAudios?: ReferenceAudio[]; - textHistory?: Array<{ role: "user" | "assistant"; content: string }>; + textHistory?: Array<{ role: "user" | "assistant" | "system"; content: string }>; mask?: ReferenceImage; signal?: AbortSignal; metadata?: Record; @@ -119,6 +108,7 @@ export async function runBackendGenerationTask( dependencies, ); } + assertBackendRuntimeConfigured(config, mode); const prepared = await prepareGenerationReferences({ referenceImages, referenceVideos, referenceAudios, mask }); throwIfAborted(signal); return createAndWaitGenerationTask({ projectId, mode, prompt, config, referenceImages, referenceVideos, referenceAudios, textHistory, signal, metadata, onTaskUpdate }, prepared, dependencies); @@ -133,6 +123,7 @@ export async function submitBackendGenerationTask( throwIfAborted(options.signal); assertClientPromptLimit(options.mode, options.prompt, options.config, options.metadata); if (usesLocalDreamina(options.config)) throw new Error("本机即梦任务暂不支持后台提交"); + assertBackendRuntimeConfigured(options.config, options.mode); const prepared = await prepareGenerationReferences(options); throwIfAborted(options.signal); return createBackendGenerationTask(options, prepared, dependencies); @@ -161,7 +152,7 @@ export async function runBackendToolGenerationTask(options: { mode: "text", prompt: options.prompt, config: backendProviderConfig(options.config), - agentRequests: buildBackendToolRequests(options.messages, options.tools, options.toolChoice), + agentRequests: buildBackendToolRequests(options.messages, options.tools, options.toolChoice, options.config), metadata: { source: "canvas-online-agent" }, }, }); @@ -390,6 +381,13 @@ function usesLocalDreamina(config: AiConfig) { return (config.taskWorkflowProvider || "model") === "model" && isLocalDreaminaModel(config.model); } +function assertBackendRuntimeConfigured(config: AiConfig, mode: BackendGenerationMode) { + if (resolveGenerationWorkflowExecution(config, mode)) return; + if (logicalModelIDForConfig(config)) return; + const requestConfig = resolveModelRequestConfig(config, config.model); + if (!requestConfig.channelId && !requestConfig.interfaceType) throw new Error("当前模型未选择可用请求协议，请先在模型设置中选择协议插件"); +} + function throwIfAborted(signal?: AbortSignal) { if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); } @@ -445,13 +443,29 @@ async function createBackendGenerationTask(options: BackendGenerationTaskOptions referenceVideos: prepared.referenceVideos, referenceAudios: prepared.referenceAudios, mask: prepared.mask, - metadata, + metadata: generationMetadata(config, metadata), }, }); onTaskUpdate?.(task); return task; } +function generationMetadata(config: AiConfig, metadata?: Record) { + const channel = resolveModelChannel(config, config.model); + const model = modelOptionName(config.model); + const modelCost = channel.modelCosts?.find((item) => item.model === model); + const protocol = modelCost?.protocol || channel.interfaceType; + const defaults = modelCost?.defaultOptions; + if (!protocol || !defaults || !Object.keys(defaults).length) return metadata; + const existing = metadata?.providerOptions && typeof metadata.providerOptions === "object" && !Array.isArray(metadata.providerOptions) + ? metadata.providerOptions as Record + : {}; + const namespace = existing[protocol] && typeof existing[protocol] === "object" && !Array.isArray(existing[protocol]) + ? existing[protocol] as Record + : {}; + return { ...metadata, providerOptions: { ...existing, [protocol]: { ...defaults, ...namespace } } }; +} + async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAudio) { if (resourceIdFromStorageKey(media.storageKey)) return backendMediaReference(media, { storageKey: media.storageKey }); const url = media.url || ""; @@ -530,6 +544,7 @@ export function backendProviderConfig(config: AiConfig, mode: BackendGenerationM audioFormat: config.audioFormat, audioSpeed: config.audioSpeed, audioInstructions: config.audioInstructions, + systemPrompt: config.systemPrompt, }; if (logicalModelIDForConfig(config)) return generationOptions; return { @@ -543,7 +558,7 @@ export function backendProviderConfig(config: AiConfig, mode: BackendGenerationM model: requestConfig.model, ...generationOptions, capabilityConfig: modelCapabilityConfigFor(config, requestConfig.model), - systemPrompt: "", + systemPrompt: config.systemPrompt, }; } @@ -583,7 +598,7 @@ function workflowProviderConfig(config: AiConfig, requestConfig: ReturnType; chatCompletion: Record; + claude?: Record; + gemini?: Record; }; export type ResponseMessageContent = AiTextMessage["content"] | string; diff --git a/web/src/services/api/image-protocols.ts b/web/src/services/api/image-protocols.ts index f3fb17024..09d425e7c 100644 --- a/web/src/services/api/image-protocols.ts +++ b/web/src/services/api/image-protocols.ts @@ -101,8 +101,8 @@ export function toChatCompletionToolChoice(toolChoice: ToolChoice) { return typeof toolChoice === "object" ? { type: "function", function: { name: toolChoice.name } } : toolChoice; } -export function buildBackendToolRequests(messages: ResponseInputMessage[], tools: ResponseFunctionTool[], toolChoice: ToolChoice): BackendToolRequests { - return { +export function buildBackendToolRequests(messages: ResponseInputMessage[], tools: ResponseFunctionTool[], toolChoice: ToolChoice, config?: AiConfig): BackendToolRequests { + const requests: BackendToolRequests = { responses: { input: toResponseInput(messages), tools: tools.map(toResponseTool), @@ -116,6 +116,14 @@ export function buildBackendToolRequests(messages: ResponseInputMessage[], tools parallel_tool_calls: false, }, }; + if (config) { + requests.claude = { + ...toClaudeBody(config, messages, tools), + tool_choice: typeof toolChoice === "object" ? { type: "tool", name: toolChoice.name } : { type: toolChoice === "required" ? "any" : "auto" }, + }; + requests.gemini = toGeminiBody(config, messages, toGeminiToolOptions(tools, toolChoice)); + } + return requests; } export function toGeminiBody(config: AiConfig, messages: ResponseInputMessage[], extra?: Record) { diff --git a/web/src/services/canvas-video-preview.ts b/web/src/services/canvas-video-preview.ts index 61c36d26e..924be0ae5 100644 --- a/web/src/services/canvas-video-preview.ts +++ b/web/src/services/canvas-video-preview.ts @@ -14,13 +14,16 @@ export function hydrateCanvasVideoPreview(node: CanvasNodeData, signal?: AbortSi const existing = previewRequests.get(requestKey); if (existing) return existing; - const request = generateCanvasVideoPreview(node, signal).catch((error: unknown) => { - if (error instanceof DOMException && error.name === "AbortError") { + const request = generateCanvasVideoPreview(node, signal) + .then((preview) => { + if (!preview) previewRequests.delete(requestKey); + return preview; + }) + .catch((error: unknown) => { previewRequests.delete(requestKey); - throw error; - } - return null; - }); + if (error instanceof DOMException && error.name === "AbortError") throw error; + return null; + }); previewRequests.set(requestKey, request); return request; } diff --git a/web/test/canvas-generation-submission.test.ts b/web/test/canvas-generation-submission.test.ts index b76fdc100..c50bde3cd 100644 --- a/web/test/canvas-generation-submission.test.ts +++ b/web/test/canvas-generation-submission.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; -import { canvasGenerationRequestFingerprint, runCanvasGenerationSubmissionOnce } from "@/lib/canvas/canvas-generation-submission"; +import { canvasGenerationPromptMetadata, canvasGenerationRequestFingerprint, runCanvasGenerationSubmissionOnce } from "@/lib/canvas/canvas-generation-submission"; function fingerprint(overrides: Partial[0]> = {}) { return canvasGenerationRequestFingerprint({ @@ -22,6 +22,13 @@ function fingerprint(overrides: Partial { + test("持久化时分离编辑器槽位提示词和模型提示词", () => { + expect(canvasGenerationPromptMetadata("自我介绍 @图片1", "电影感：自我介绍 @图片1")).toEqual({ + composerContent: "自我介绍 @图片1", + prompt: "电影感：自我介绍 @图片1", + }); + }); + test("同一节点的并发提交只执行一次", async () => { const locks = new Map>(); let executions = 0; diff --git a/web/test/canvas-media-performance.test.ts b/web/test/canvas-media-performance.test.ts index 35bd53a88..c696dc03d 100644 --- a/web/test/canvas-media-performance.test.ts +++ b/web/test/canvas-media-performance.test.ts @@ -14,6 +14,7 @@ const canvasNodeContentSource = readFileSync(resolve(import.meta.dir, "../src/co const canvasAudioPlayerSource = readFileSync(resolve(import.meta.dir, "../src/components/canvas/canvas-audio-player.tsx"), "utf8"); const canvasMentionSource = readFileSync(resolve(import.meta.dir, "../src/components/canvas/canvas-resource-mention-textarea.tsx"), "utf8"); const canvasNodeSource = readFileSync(resolve(import.meta.dir, "../src/components/canvas/canvas-node.tsx"), "utf8"); +const canvasVideoPreviewSource = readFileSync(resolve(import.meta.dir, "../src/services/canvas-video-preview.ts"), "utf8"); const videoPlayerSource = readFileSync(resolve(import.meta.dir, "../src/components/video-player.tsx"), "utf8"); const canvasProjectSource = readFileSync(resolve(import.meta.dir, "../src/pages/canvas/project.tsx"), "utf8"); const globalStylesSource = readFileSync(resolve(import.meta.dir, "../src/styles/globals.css"), "utf8"); @@ -67,6 +68,20 @@ describe("large canvas media rendering", () => { test("keeps node shells visible while panning and zooming", () => { expect(globalStylesSource).not.toMatch(/\[data-canvas-viewport-interacting="true"\]\s+\.canvas-node-shell\s*\{[^}]*content-visibility:\s*hidden/); }); + + test("lets inactive video nodes hydrate their own first-frame preview", () => { + expect(canvasNodeContentSource).toContain("if (previewUrl || !node.metadata?.content || !updateMetadataRef.current)"); + expect(canvasNodeContentSource).not.toContain("hydrateMediaPreview"); + expect(canvasNodeContentSource).toContain("mediaActive || !hasPassivePreview"); + expect(canvasNodeContentSource).toContain('muted\n playsInline\n preload="auto"'); + expect(canvasNodeContentSource).toContain("onLoadedMetadata={(event) => primePassiveVideoFrame(event.currentTarget)}"); + expect(canvasNodeContentSource).toContain("video.currentTime = Math.min(0.001, video.duration)"); + }); + + test("allows failed or empty first-frame requests to retry", () => { + expect(canvasVideoPreviewSource).toContain("if (!preview) previewRequests.delete(requestKey)"); + expect(canvasVideoPreviewSource).toContain("previewRequests.delete(requestKey);"); + }); }); describe("audio canvas interaction", () => { diff --git a/web/test/canvas-node-generation-mentions.test.ts b/web/test/canvas-node-generation-mentions.test.ts index 53b636bdc..255afba3d 100644 --- a/web/test/canvas-node-generation-mentions.test.ts +++ b/web/test/canvas-node-generation-mentions.test.ts @@ -40,7 +40,7 @@ describe("canvas node generation position mentions", () => { expect(context.referenceImages.map((image) => image.id)).toEqual([source.id]); expect(context.imageCount).toBe(1); - expect(context.prompt).toBe("将 图片1 图片变清晰"); + expect(context.prompt).toBe("将 @图片1 图片变清晰"); }); test("已有图片节点未显式引用自身时不自动退化为图生图", () => { @@ -67,8 +67,8 @@ describe("canvas node generation position mentions", () => { expect(before.referenceImages.map((image) => image.id)).toEqual(["image-a"]); expect(after.referenceImages.map((image) => image.id)).toEqual(["image-b"]); - expect(before.prompt).toBe("让 图片1 进入画面"); - expect(after.prompt).toBe("让 图片1 进入画面"); + expect(before.prompt).toBe("让 @图片1 进入画面"); + expect(after.prompt).toBe("让 @图片1 进入画面"); }); test("按类型位置选择资源，提示词出现顺序不会改变槽位含义", () => { @@ -81,7 +81,7 @@ describe("canvas node generation position mentions", () => { expect(context.referenceImages.map((image) => image.id)).toEqual(["image-b"]); expect(context.referenceAudios.map((audio) => audio.id)).toEqual(["audio-a"]); - expect(context.prompt).toBe("让 图片1 配合 音频1"); + expect(context.prompt).toBe("让 @图片1 配合 @音频1"); }); test("旧节点 token 只做读取迁移，不再进入生成提示词", () => { @@ -90,7 +90,7 @@ describe("canvas node generation position mentions", () => { const context = buildNodeGenerationContext(target.id, [image, target], [connection(image.id)], "让 @[node:image-a] 进入画面", []); expect(context.referenceImages.map((item) => item.id)).toEqual(["image-a"]); - expect(context.prompt).toBe("让 图片1 进入画面"); + expect(context.prompt).toBe("让 @图片1 进入画面"); expect(context.prompt).not.toContain("@[node:"); }); }); diff --git a/web/test/canvas-resource-mention-editor.test.ts b/web/test/canvas-resource-mention-editor.test.ts index a25e2f24b..f4092fac1 100644 --- a/web/test/canvas-resource-mention-editor.test.ts +++ b/web/test/canvas-resource-mention-editor.test.ts @@ -55,6 +55,19 @@ describe("canvas resource mention editor", () => { expect(project).not.toContain("removeCanvasResourceMention"); }); + test("keeps editable mention prompts separate from normalized generation prompts", () => { + const imageExecutor = source("../src/pages/canvas/canvas-image-generation-executor.ts"); + const mediaExecutors = source("../src/pages/canvas/canvas-media-generation-executors.ts"); + const textExecutor = source("../src/pages/canvas/canvas-text-generation-executor.ts"); + const generationExecutor = source("../src/pages/canvas/use-canvas-generation-executor.ts"); + + expect(imageExecutor.match(/canvasGenerationPromptMetadata\(prompt, effectivePrompt\)/g)?.length).toBeGreaterThanOrEqual(3); + expect(mediaExecutors.match(/canvasGenerationPromptMetadata\(prompt, effectivePrompt\)/g)?.length).toBe(2); + expect(textExecutor.match(/canvasGenerationPromptMetadata\(prompt, effectivePrompt\)/g)?.length).toBe(2); + expect(generationExecutor).toContain("composerContent: prompt"); + expect(generationExecutor).toContain("canvasGenerationPromptMetadata(prompt, statusPrompt)"); + }); + test("anchors the mention menu to the caret instead of the textarea edge", () => { const component = source("../src/components/canvas/canvas-resource-mention-textarea.tsx");