feat(plugins): 方舟 Agent Plan - 合并 PR #527 图片视频插件并阻止图片误用 asset://
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
commit
da1560aa56
49 files changed
+3263
-74
No files matched your search
@@ -438,7 +438,7 @@ func (s *Service) syncLogicalModelsFromChannelModel(actor *model.User, channelMo
|
||||
}
|
||||
|
||||
func supportsTokenBilling(capability string, protocol model.ChannelInterfaceType) bool {
|
||||
return capability == "text" || (capability == "video" && protocol == model.ChannelInterfaceVolcengineArkVideo)
|
||||
return capability == "text" || (capability == "video" && model.IsVolcengineArkVideoProtocol(protocol))
|
||||
}
|
||||
|
||||
func (s *Service) normalizeChannelModelPriceTiers(req ChannelModelRequest, capability string, protocol model.ChannelInterfaceType, fallbackProviderModelKey string) ([]model.ChannelModelPriceTier, error) {
|
||||
|
||||
@@ -153,7 +153,7 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab
|
||||
image.ResponseFormat = ParameterSupport{Supported: true}
|
||||
image.OutputFormat = ParameterSupport{Supported: false}
|
||||
image.MaxOutputs = 1
|
||||
case model.ChannelInterfaceVolcengineArkImage:
|
||||
case model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineArkAgentPlanImage:
|
||||
image.References.MaskSupported = false
|
||||
image.Quality.Supported = false
|
||||
image.TransparentBackground.Supported = false
|
||||
@@ -229,7 +229,7 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
|
||||
case model.ChannelInterfaceGeminiVeo:
|
||||
video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{4, 6, 8}, Default: 6}
|
||||
video.Resolutions = []string{"720p", "1080p"}
|
||||
case model.ChannelInterfaceVolcengineArkVideo:
|
||||
case model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineArkAgentPlanVideo:
|
||||
video.Operations = append(video.Operations, "reference_to_video", "audio_to_video")
|
||||
video.References.MaxVideos, video.References.MaxAudios = 3, 3
|
||||
video.References.MaxVideoBytes, video.References.MaxAudioBytes = 200*1024*1024, 15*1024*1024
|
||||
@@ -798,7 +798,7 @@ func validateVideoTask(profile *VideoCapabilityConfig, input canvasGenerationInp
|
||||
if len(input.ReferenceImages) > profile.References.MaxImages || len(input.ReferenceVideos) > profile.References.MaxVideos || len(input.ReferenceAudios) > profile.References.MaxAudios {
|
||||
return BadAuthRequest("参考素材数量超过当前模型限制")
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
|
||||
if model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
|
||||
return BadAuthRequest("火山方舟全模态参考不支持纯音频或文本+音频,请同时添加参考图片或参考视频")
|
||||
}
|
||||
if len(input.ReferenceImages) < profile.References.MinImages {
|
||||
|
||||
@@ -14,7 +14,7 @@ func ValidateChannelModelPrice(billingMode string, capability string, protocol m
|
||||
return unitPrice >= 0
|
||||
case "token":
|
||||
if capability == "video" {
|
||||
return protocol == model.ChannelInterfaceVolcengineArkVideo && inputPrice >= 0 && outputPrice >= 0 && cachedPrice >= 0
|
||||
return model.IsVolcengineArkVideoProtocol(protocol) && inputPrice >= 0 && outputPrice >= 0 && cachedPrice >= 0
|
||||
}
|
||||
if capability != "" && capability != "text" {
|
||||
return false
|
||||
|
||||
@@ -464,7 +464,7 @@ type providerMediaHydrationPolicy struct {
|
||||
func providerMediaHydrationPolicyFor(ctx context.Context, input canvasGenerationInput) providerMediaHydrationPolicy {
|
||||
policy := providerMediaHydrationPolicy{preferURL: providerPrefersMediaURLs(input.Config.InterfaceType, input)}
|
||||
switch strings.TrimSpace(input.Config.InterfaceType) {
|
||||
case string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceMiniMaxVideo):
|
||||
case string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo), string(model.ChannelInterfaceMiniMaxVideo):
|
||||
policy.requireURL = true
|
||||
policy.preferURL = true
|
||||
}
|
||||
@@ -488,11 +488,11 @@ func providerPrefersMediaURLs(interfaceType string, input canvasGenerationInput)
|
||||
}
|
||||
switch strings.TrimSpace(interfaceType) {
|
||||
case string(model.ChannelInterfaceChatCompletion), string(model.ChannelInterfaceOpenAIResponse), string(model.ChannelInterfaceClaudeAPI),
|
||||
string(model.ChannelInterfaceGrokImage), string(model.ChannelInterfaceVolcengineArkImage),
|
||||
string(model.ChannelInterfaceGrokImage), string(model.ChannelInterfaceVolcengineArkImage), string(model.ChannelInterfaceVolcengineArkAgentPlanImage),
|
||||
string(model.ChannelInterfaceXAIVideo), string(model.ChannelInterfaceNovitaVideo),
|
||||
string(model.ChannelInterfaceMiniMaxVideo), string(model.ChannelInterfaceNewAPIVideo),
|
||||
string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2),
|
||||
string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo):
|
||||
return true
|
||||
}
|
||||
if isGrokVideoConfig(input.Config) || isSeedanceVideoConfig(input.Config) || isArkPlanVideoConfig(input.Config) {
|
||||
|
||||
@@ -47,7 +47,8 @@ func withoutProviderAnalytics(ctx context.Context) context.Context {
|
||||
}
|
||||
|
||||
func (s *Service) prepareArkPrivateAssetReferences(ctx context.Context, userID string, input *canvasGenerationInput) error {
|
||||
if input == nil || !isArkPrivateAssetVideoConfig(input.Config) || !parseBool(input.Config.ArkPrivateAssetUpload, true) {
|
||||
// 可信素材 asset:// 仅方舟视频协议支持;Agent Plan Seedream 等图片渠道不能上传或改写。
|
||||
if input == nil || input.Mode != "video" || !isArkPrivateAssetVideoConfig(input.Config) || !parseBool(input.Config.ArkPrivateAssetUpload, true) {
|
||||
return nil
|
||||
}
|
||||
hasOwnedReference := false
|
||||
@@ -107,7 +108,12 @@ func (s *Service) prepareArkPrivateAssetReferences(ctx context.Context, userID s
|
||||
}
|
||||
|
||||
func isArkPrivateAssetVideoConfig(config providerConfig) bool {
|
||||
return config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || isArkPlanVideoConfig(config)
|
||||
iface := strings.TrimSpace(config.InterfaceType)
|
||||
// Agent Plan 图片与视频共用 /api/plan/v3;图片协议不得进入可信素材同步。
|
||||
if iface == string(model.ChannelInterfaceVolcengineArkImage) || iface == "volcengine-ark-agent-plan-image" {
|
||||
return false
|
||||
}
|
||||
return iface == string(model.ChannelInterfaceVolcengineArkVideo) || iface == "volcengine-ark-agent-plan-video" || isArkPlanVideoConfig(config)
|
||||
}
|
||||
|
||||
func arkPrivateAssetAutomaticSyncEnabled(setting arkPrivateAssetSettingValue) (bool, error) {
|
||||
|
||||
@@ -172,7 +172,7 @@ func asyncAudioErrorMessage(state map[string]interface{}) string {
|
||||
func decodeProviderDataURL(value string) (string, []byte, error) {
|
||||
header, encoded, ok := strings.Cut(value, ",")
|
||||
if !ok || !strings.HasPrefix(header, "data:") || !strings.HasSuffix(strings.ToLower(header), ";base64") {
|
||||
return "", nil, errors.New("音频 data URL 格式无效")
|
||||
return "", nil, errors.New("data URL 格式无效")
|
||||
}
|
||||
mimeType := strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64")
|
||||
data, err := base64.StdEncoding.DecodeString(encoded)
|
||||
|
||||
@@ -28,7 +28,7 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineJiMengImage) {
|
||||
return runVolcengineJiMengImageTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkImage) {
|
||||
if model.IsVolcengineArkImageProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) {
|
||||
return runVolcengineArkImageTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceGeminiImage) {
|
||||
|
||||
@@ -194,15 +194,15 @@ func (s *Service) providerCancellationInput(task *model.Task) (canvasGenerationI
|
||||
}
|
||||
|
||||
func supportsProviderCancellation(interfaceType string) bool {
|
||||
return interfaceType == string(model.ChannelInterfaceGeminiVeo) || interfaceType == string(model.ChannelInterfaceVolcengineArkVideo)
|
||||
return interfaceType == string(model.ChannelInterfaceGeminiVeo) || model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(interfaceType))
|
||||
}
|
||||
|
||||
func cancelProviderTask(ctx context.Context, config providerConfig, providerRequestID string) error {
|
||||
switch config.InterfaceType {
|
||||
case string(model.ChannelInterfaceGeminiVeo):
|
||||
switch {
|
||||
case config.InterfaceType == string(model.ChannelInterfaceGeminiVeo):
|
||||
path := "/" + strings.TrimLeft(providerRequestID, "/") + ":cancel"
|
||||
return postGeminiJSON(ctx, config, path, map[string]any{}, &map[string]any{})
|
||||
case string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
case model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(config.InterfaceType)):
|
||||
path := "/contents/generations/tasks/" + url.PathEscape(providerRequestID)
|
||||
return deleteProviderTask(ctx, config, path)
|
||||
default:
|
||||
@@ -228,8 +228,8 @@ func deleteProviderTask(ctx context.Context, config providerConfig, path string)
|
||||
}
|
||||
|
||||
func queryProviderCancellation(ctx context.Context, config providerConfig, providerRequestID string) (providerCancellationOutcome, string, error) {
|
||||
switch config.InterfaceType {
|
||||
case string(model.ChannelInterfaceGeminiVeo):
|
||||
switch {
|
||||
case config.InterfaceType == string(model.ChannelInterfaceGeminiVeo):
|
||||
// Gemini 用 operation.error 表示取消终态,此处必须保留该字段再判断,
|
||||
// 不能让通用 JSON 解包提前把它转换成请求失败。
|
||||
var operation geminiOperation
|
||||
@@ -248,7 +248,7 @@ func queryProviderCancellation(ctx context.Context, config providerConfig, provi
|
||||
return providerCancellationFailed, firstNonEmpty(message, "failed"), nil
|
||||
}
|
||||
return providerCancellationSucceeded, "succeeded", nil
|
||||
case string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
case model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(config.InterfaceType)):
|
||||
var state map[string]any
|
||||
if err := getJSON(ctx, config, "/contents/generations/tasks/"+url.PathEscape(providerRequestID), &state); err != nil {
|
||||
return "", "", err
|
||||
|
||||
@@ -2224,6 +2224,48 @@ func TestArkPlanConfigStaysSeparateFromSeedanceVideosEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestArkPlanImageConfigDoesNotUseVideoAssetPath(t *testing.T) {
|
||||
imageConfig := providerConfig{
|
||||
InterfaceType: "volcengine-ark-agent-plan-image",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3",
|
||||
Model: "doubao-seedream-4-0-250828",
|
||||
}
|
||||
if isArkPlanVideoConfig(imageConfig) {
|
||||
t.Fatal("agent plan image must not match isArkPlanVideoConfig")
|
||||
}
|
||||
if isArkPrivateAssetVideoConfig(imageConfig) {
|
||||
t.Fatal("agent plan image must not trigger ark private asset sync")
|
||||
}
|
||||
if isSeedanceVideoConfig(imageConfig) {
|
||||
t.Fatal("agent plan image must not match isSeedanceVideoConfig")
|
||||
}
|
||||
|
||||
videoConfig := providerConfig{
|
||||
InterfaceType: "volcengine-ark-agent-plan-video",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3",
|
||||
Model: "doubao-seedance-1-5-pro-251215",
|
||||
}
|
||||
if !isArkPlanVideoConfig(videoConfig) || !isArkPrivateAssetVideoConfig(videoConfig) {
|
||||
t.Fatal("agent plan video should keep video/asset path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareArkPrivateAssetReferencesSkipsImageMode(t *testing.T) {
|
||||
svc := &Service{}
|
||||
err := svc.prepareArkPrivateAssetReferences(context.Background(), "user-1", &canvasGenerationInput{
|
||||
Mode: "image",
|
||||
Config: providerConfig{
|
||||
InterfaceType: "volcengine-ark-agent-plan-image",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3",
|
||||
ArkPrivateAssetUpload: "true",
|
||||
},
|
||||
ReferenceImages: []providerMedia{{StorageKey: "resource:res-1", URL: "https://example.com/ref.png"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepareArkPrivateAssetReferences() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
paths := make([]string, 0, 3)
|
||||
|
||||
@@ -287,7 +287,7 @@ func runSeedanceVideosTask(ctx context.Context, input canvasGenerationInput, pol
|
||||
|
||||
func runSeedanceAgentPlanVideoTask(ctx context.Context, input canvasGenerationInput, pollPolicy videoPollPolicy) (map[string]interface{}, error) {
|
||||
providerName := "Seedance"
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) {
|
||||
if model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) {
|
||||
providerName = "火山方舟"
|
||||
}
|
||||
// Agent Plan 与 Videos API 使用相同的恢复合同:已有任务只查询,不重复创建。
|
||||
@@ -299,7 +299,7 @@ func runSeedanceAgentPlanVideoTask(ctx context.Context, input canvasGenerationIn
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) {
|
||||
if model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) {
|
||||
for _, item := range content {
|
||||
if item["type"] == "image_url" {
|
||||
item["role"] = "reference_image"
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
func isPublicMediaURL(value string) bool {
|
||||
@@ -13,8 +15,8 @@ func isPublicMediaURL(value string) bool {
|
||||
}
|
||||
|
||||
func isSeedanceVideoConfig(config providerConfig) bool {
|
||||
model := strings.ToLower(config.Model)
|
||||
return strings.Contains(model, "seedance") || strings.Contains(model, "doubao-seedance") || isArkPlanVideoConfig(config)
|
||||
modelName := strings.ToLower(config.Model)
|
||||
return strings.Contains(modelName, "seedance") || strings.Contains(modelName, "doubao-seedance") || isArkPlanVideoConfig(config)
|
||||
}
|
||||
|
||||
func isGrokVideoConfig(config providerConfig) bool {
|
||||
@@ -22,7 +24,15 @@ func isGrokVideoConfig(config providerConfig) bool {
|
||||
}
|
||||
|
||||
func isArkPlanVideoConfig(config providerConfig) bool {
|
||||
return strings.Contains(strings.ToLower(config.BaseURL), "/api/plan/v3")
|
||||
if !strings.Contains(strings.ToLower(config.BaseURL), "/api/plan/v3") {
|
||||
return false
|
||||
}
|
||||
// Agent Plan 图片与视频共用 /api/plan/v3;按协议排除图片,避免 Seedream 误走视频/可信素材路径。
|
||||
iface := strings.TrimSpace(config.InterfaceType)
|
||||
if iface == string(model.ChannelInterfaceVolcengineArkImage) || iface == "volcengine-ark-agent-plan-image" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func normalizeImageQuality(value string) string {
|
||||
|
||||
Reference in new issue
Block a user