feat(plugins): 方舟 Agent Plan - 合并 PR #527 图片视频插件并阻止图片误用 asset://
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
commit
60f121d04a
49 files changed
+3263
-74
No files matched your search
@@ -438,7 +438,7 @@ func (s *Service) syncLogicalModelsFromChannelModel(actor *model.User, channelMo
|
||||
}
|
||||
|
||||
func supportsTokenBilling(capability string, protocol model.ChannelInterfaceType) bool {
|
||||
return capability == "text" || (capability == "video" && protocol == model.ChannelInterfaceVolcengineArkVideo)
|
||||
return capability == "text" || (capability == "video" && model.IsVolcengineArkVideoProtocol(protocol))
|
||||
}
|
||||
|
||||
func (s *Service) normalizeChannelModelPriceTiers(req ChannelModelRequest, capability string, protocol model.ChannelInterfaceType, fallbackProviderModelKey string) ([]model.ChannelModelPriceTier, error) {
|
||||
|
||||
@@ -153,7 +153,7 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab
|
||||
image.ResponseFormat = ParameterSupport{Supported: true}
|
||||
image.OutputFormat = ParameterSupport{Supported: false}
|
||||
image.MaxOutputs = 1
|
||||
case model.ChannelInterfaceVolcengineArkImage:
|
||||
case model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineArkAgentPlanImage:
|
||||
image.References.MaskSupported = false
|
||||
image.Quality.Supported = false
|
||||
image.TransparentBackground.Supported = false
|
||||
@@ -229,7 +229,7 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
|
||||
case model.ChannelInterfaceGeminiVeo:
|
||||
video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{4, 6, 8}, Default: 6}
|
||||
video.Resolutions = []string{"720p", "1080p"}
|
||||
case model.ChannelInterfaceVolcengineArkVideo:
|
||||
case model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineArkAgentPlanVideo:
|
||||
video.Operations = append(video.Operations, "reference_to_video", "audio_to_video")
|
||||
video.References.MaxVideos, video.References.MaxAudios = 3, 3
|
||||
video.References.MaxVideoBytes, video.References.MaxAudioBytes = 200*1024*1024, 15*1024*1024
|
||||
@@ -798,7 +798,7 @@ func validateVideoTask(profile *VideoCapabilityConfig, input canvasGenerationInp
|
||||
if len(input.ReferenceImages) > profile.References.MaxImages || len(input.ReferenceVideos) > profile.References.MaxVideos || len(input.ReferenceAudios) > profile.References.MaxAudios {
|
||||
return BadAuthRequest("参考素材数量超过当前模型限制")
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
|
||||
if model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
|
||||
return BadAuthRequest("火山方舟全模态参考不支持纯音频或文本+音频,请同时添加参考图片或参考视频")
|
||||
}
|
||||
if len(input.ReferenceImages) < profile.References.MinImages {
|
||||
|
||||
@@ -14,7 +14,7 @@ func ValidateChannelModelPrice(billingMode string, capability string, protocol m
|
||||
return unitPrice >= 0
|
||||
case "token":
|
||||
if capability == "video" {
|
||||
return protocol == model.ChannelInterfaceVolcengineArkVideo && inputPrice >= 0 && outputPrice >= 0 && cachedPrice >= 0
|
||||
return model.IsVolcengineArkVideoProtocol(protocol) && inputPrice >= 0 && outputPrice >= 0 && cachedPrice >= 0
|
||||
}
|
||||
if capability != "" && capability != "text" {
|
||||
return false
|
||||
|
||||
@@ -464,7 +464,7 @@ type providerMediaHydrationPolicy struct {
|
||||
func providerMediaHydrationPolicyFor(ctx context.Context, input canvasGenerationInput) providerMediaHydrationPolicy {
|
||||
policy := providerMediaHydrationPolicy{preferURL: providerPrefersMediaURLs(input.Config.InterfaceType, input)}
|
||||
switch strings.TrimSpace(input.Config.InterfaceType) {
|
||||
case string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceMiniMaxVideo):
|
||||
case string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo), string(model.ChannelInterfaceMiniMaxVideo):
|
||||
policy.requireURL = true
|
||||
policy.preferURL = true
|
||||
}
|
||||
@@ -488,11 +488,11 @@ func providerPrefersMediaURLs(interfaceType string, input canvasGenerationInput)
|
||||
}
|
||||
switch strings.TrimSpace(interfaceType) {
|
||||
case string(model.ChannelInterfaceChatCompletion), string(model.ChannelInterfaceOpenAIResponse), string(model.ChannelInterfaceClaudeAPI),
|
||||
string(model.ChannelInterfaceGrokImage), string(model.ChannelInterfaceVolcengineArkImage),
|
||||
string(model.ChannelInterfaceGrokImage), string(model.ChannelInterfaceVolcengineArkImage), string(model.ChannelInterfaceVolcengineArkAgentPlanImage),
|
||||
string(model.ChannelInterfaceXAIVideo), string(model.ChannelInterfaceNovitaVideo),
|
||||
string(model.ChannelInterfaceMiniMaxVideo), string(model.ChannelInterfaceNewAPIVideo),
|
||||
string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2),
|
||||
string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo):
|
||||
return true
|
||||
}
|
||||
if isGrokVideoConfig(input.Config) || isSeedanceVideoConfig(input.Config) || isArkPlanVideoConfig(input.Config) {
|
||||
|
||||
@@ -47,7 +47,8 @@ func withoutProviderAnalytics(ctx context.Context) context.Context {
|
||||
}
|
||||
|
||||
func (s *Service) prepareArkPrivateAssetReferences(ctx context.Context, userID string, input *canvasGenerationInput) error {
|
||||
if input == nil || !isArkPrivateAssetVideoConfig(input.Config) || !parseBool(input.Config.ArkPrivateAssetUpload, true) {
|
||||
// 可信素材 asset:// 仅方舟视频协议支持;Agent Plan Seedream 等图片渠道不能上传或改写。
|
||||
if input == nil || input.Mode != "video" || !isArkPrivateAssetVideoConfig(input.Config) || !parseBool(input.Config.ArkPrivateAssetUpload, true) {
|
||||
return nil
|
||||
}
|
||||
hasOwnedReference := false
|
||||
@@ -107,7 +108,12 @@ func (s *Service) prepareArkPrivateAssetReferences(ctx context.Context, userID s
|
||||
}
|
||||
|
||||
func isArkPrivateAssetVideoConfig(config providerConfig) bool {
|
||||
return config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || isArkPlanVideoConfig(config)
|
||||
iface := strings.TrimSpace(config.InterfaceType)
|
||||
// Agent Plan 图片与视频共用 /api/plan/v3;图片协议不得进入可信素材同步。
|
||||
if iface == string(model.ChannelInterfaceVolcengineArkImage) || iface == "volcengine-ark-agent-plan-image" {
|
||||
return false
|
||||
}
|
||||
return iface == string(model.ChannelInterfaceVolcengineArkVideo) || iface == "volcengine-ark-agent-plan-video" || isArkPlanVideoConfig(config)
|
||||
}
|
||||
|
||||
func arkPrivateAssetAutomaticSyncEnabled(setting arkPrivateAssetSettingValue) (bool, error) {
|
||||
|
||||
@@ -172,7 +172,7 @@ func asyncAudioErrorMessage(state map[string]interface{}) string {
|
||||
func decodeProviderDataURL(value string) (string, []byte, error) {
|
||||
header, encoded, ok := strings.Cut(value, ",")
|
||||
if !ok || !strings.HasPrefix(header, "data:") || !strings.HasSuffix(strings.ToLower(header), ";base64") {
|
||||
return "", nil, errors.New("音频 data URL 格式无效")
|
||||
return "", nil, errors.New("data URL 格式无效")
|
||||
}
|
||||
mimeType := strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64")
|
||||
data, err := base64.StdEncoding.DecodeString(encoded)
|
||||
|
||||
@@ -28,7 +28,7 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineJiMengImage) {
|
||||
return runVolcengineJiMengImageTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkImage) {
|
||||
if model.IsVolcengineArkImageProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) {
|
||||
return runVolcengineArkImageTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceGeminiImage) {
|
||||
|
||||
@@ -194,15 +194,15 @@ func (s *Service) providerCancellationInput(task *model.Task) (canvasGenerationI
|
||||
}
|
||||
|
||||
func supportsProviderCancellation(interfaceType string) bool {
|
||||
return interfaceType == string(model.ChannelInterfaceGeminiVeo) || interfaceType == string(model.ChannelInterfaceVolcengineArkVideo)
|
||||
return interfaceType == string(model.ChannelInterfaceGeminiVeo) || model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(interfaceType))
|
||||
}
|
||||
|
||||
func cancelProviderTask(ctx context.Context, config providerConfig, providerRequestID string) error {
|
||||
switch config.InterfaceType {
|
||||
case string(model.ChannelInterfaceGeminiVeo):
|
||||
switch {
|
||||
case config.InterfaceType == string(model.ChannelInterfaceGeminiVeo):
|
||||
path := "/" + strings.TrimLeft(providerRequestID, "/") + ":cancel"
|
||||
return postGeminiJSON(ctx, config, path, map[string]any{}, &map[string]any{})
|
||||
case string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
case model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(config.InterfaceType)):
|
||||
path := "/contents/generations/tasks/" + url.PathEscape(providerRequestID)
|
||||
return deleteProviderTask(ctx, config, path)
|
||||
default:
|
||||
@@ -228,8 +228,8 @@ func deleteProviderTask(ctx context.Context, config providerConfig, path string)
|
||||
}
|
||||
|
||||
func queryProviderCancellation(ctx context.Context, config providerConfig, providerRequestID string) (providerCancellationOutcome, string, error) {
|
||||
switch config.InterfaceType {
|
||||
case string(model.ChannelInterfaceGeminiVeo):
|
||||
switch {
|
||||
case config.InterfaceType == string(model.ChannelInterfaceGeminiVeo):
|
||||
// Gemini 用 operation.error 表示取消终态,此处必须保留该字段再判断,
|
||||
// 不能让通用 JSON 解包提前把它转换成请求失败。
|
||||
var operation geminiOperation
|
||||
@@ -248,7 +248,7 @@ func queryProviderCancellation(ctx context.Context, config providerConfig, provi
|
||||
return providerCancellationFailed, firstNonEmpty(message, "failed"), nil
|
||||
}
|
||||
return providerCancellationSucceeded, "succeeded", nil
|
||||
case string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
case model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(config.InterfaceType)):
|
||||
var state map[string]any
|
||||
if err := getJSON(ctx, config, "/contents/generations/tasks/"+url.PathEscape(providerRequestID), &state); err != nil {
|
||||
return "", "", err
|
||||
|
||||
@@ -2224,6 +2224,48 @@ func TestArkPlanConfigStaysSeparateFromSeedanceVideosEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestArkPlanImageConfigDoesNotUseVideoAssetPath(t *testing.T) {
|
||||
imageConfig := providerConfig{
|
||||
InterfaceType: "volcengine-ark-agent-plan-image",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3",
|
||||
Model: "doubao-seedream-4-0-250828",
|
||||
}
|
||||
if isArkPlanVideoConfig(imageConfig) {
|
||||
t.Fatal("agent plan image must not match isArkPlanVideoConfig")
|
||||
}
|
||||
if isArkPrivateAssetVideoConfig(imageConfig) {
|
||||
t.Fatal("agent plan image must not trigger ark private asset sync")
|
||||
}
|
||||
if isSeedanceVideoConfig(imageConfig) {
|
||||
t.Fatal("agent plan image must not match isSeedanceVideoConfig")
|
||||
}
|
||||
|
||||
videoConfig := providerConfig{
|
||||
InterfaceType: "volcengine-ark-agent-plan-video",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3",
|
||||
Model: "doubao-seedance-1-5-pro-251215",
|
||||
}
|
||||
if !isArkPlanVideoConfig(videoConfig) || !isArkPrivateAssetVideoConfig(videoConfig) {
|
||||
t.Fatal("agent plan video should keep video/asset path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareArkPrivateAssetReferencesSkipsImageMode(t *testing.T) {
|
||||
svc := &Service{}
|
||||
err := svc.prepareArkPrivateAssetReferences(context.Background(), "user-1", &canvasGenerationInput{
|
||||
Mode: "image",
|
||||
Config: providerConfig{
|
||||
InterfaceType: "volcengine-ark-agent-plan-image",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3",
|
||||
ArkPrivateAssetUpload: "true",
|
||||
},
|
||||
ReferenceImages: []providerMedia{{StorageKey: "resource:res-1", URL: "https://example.com/ref.png"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepareArkPrivateAssetReferences() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
paths := make([]string, 0, 3)
|
||||
|
||||
@@ -287,7 +287,7 @@ func runSeedanceVideosTask(ctx context.Context, input canvasGenerationInput, pol
|
||||
|
||||
func runSeedanceAgentPlanVideoTask(ctx context.Context, input canvasGenerationInput, pollPolicy videoPollPolicy) (map[string]interface{}, error) {
|
||||
providerName := "Seedance"
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) {
|
||||
if model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) {
|
||||
providerName = "火山方舟"
|
||||
}
|
||||
// Agent Plan 与 Videos API 使用相同的恢复合同:已有任务只查询,不重复创建。
|
||||
@@ -299,7 +299,7 @@ func runSeedanceAgentPlanVideoTask(ctx context.Context, input canvasGenerationIn
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) {
|
||||
if model.IsVolcengineArkVideoProtocol(model.ChannelInterfaceType(input.Config.InterfaceType)) {
|
||||
for _, item := range content {
|
||||
if item["type"] == "image_url" {
|
||||
item["role"] = "reference_image"
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
func isPublicMediaURL(value string) bool {
|
||||
@@ -13,8 +15,8 @@ func isPublicMediaURL(value string) bool {
|
||||
}
|
||||
|
||||
func isSeedanceVideoConfig(config providerConfig) bool {
|
||||
model := strings.ToLower(config.Model)
|
||||
return strings.Contains(model, "seedance") || strings.Contains(model, "doubao-seedance") || isArkPlanVideoConfig(config)
|
||||
modelName := strings.ToLower(config.Model)
|
||||
return strings.Contains(modelName, "seedance") || strings.Contains(modelName, "doubao-seedance") || isArkPlanVideoConfig(config)
|
||||
}
|
||||
|
||||
func isGrokVideoConfig(config providerConfig) bool {
|
||||
@@ -22,7 +24,15 @@ func isGrokVideoConfig(config providerConfig) bool {
|
||||
}
|
||||
|
||||
func isArkPlanVideoConfig(config providerConfig) bool {
|
||||
return strings.Contains(strings.ToLower(config.BaseURL), "/api/plan/v3")
|
||||
if !strings.Contains(strings.ToLower(config.BaseURL), "/api/plan/v3") {
|
||||
return false
|
||||
}
|
||||
// Agent Plan 图片与视频共用 /api/plan/v3;按协议排除图片,避免 Seedream 误走视频/可信素材路径。
|
||||
iface := strings.TrimSpace(config.InterfaceType)
|
||||
if iface == string(model.ChannelInterfaceVolcengineArkImage) || iface == "volcengine-ark-agent-plan-image" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func normalizeImageQuality(value string) string {
|
||||
|
||||
@@ -70,7 +70,7 @@ func OfficialDeclarativeVideoInterface(interfaceType string) (string, bool) {
|
||||
return "NewAPI 媒体任务", true
|
||||
case string(model.ChannelInterfaceXAIVideo):
|
||||
return "xAI", true
|
||||
case string(model.ChannelInterfaceVolcengineArkVideo):
|
||||
case string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceVolcengineArkAgentPlanVideo):
|
||||
return "火山方舟", true
|
||||
case string(model.ChannelInterfaceVolcengineJiMengVideo):
|
||||
return "即梦", true
|
||||
@@ -91,6 +91,8 @@ func OfficialDeclarativeImageInterface(interfaceType string) (string, bool) {
|
||||
return "Grok Images", true
|
||||
case string(model.ChannelInterfaceVolcengineArkImage):
|
||||
return "火山方舟图片", true
|
||||
case string(model.ChannelInterfaceVolcengineArkAgentPlanImage):
|
||||
return "火山方舟 Agent Plan 图片", true
|
||||
case string(model.ChannelInterfaceVolcengineJiMengImage):
|
||||
return "即梦图片", true
|
||||
default:
|
||||
|
||||
@@ -244,11 +244,11 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
|
||||
return requestPath == "/messages"
|
||||
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage:
|
||||
return requestPath == "/images/generations" || requestPath == "/images/edits"
|
||||
case model.ChannelInterfaceVolcengineArkImage:
|
||||
case model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineArkAgentPlanImage:
|
||||
return requestPath == "/images/generations"
|
||||
case model.ChannelInterfaceOpenAIAudio:
|
||||
return requestPath == "/audio/speech"
|
||||
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceNovitaVideo, model.ChannelInterfaceMiniMaxVideo:
|
||||
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineArkAgentPlanVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceNovitaVideo, model.ChannelInterfaceMiniMaxVideo:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
|
||||
@@ -63,16 +63,18 @@ const (
|
||||
ChannelInterfaceClaudeAPI ChannelInterfaceType = "claude-api"
|
||||
ChannelInterfaceOpenAIImage ChannelInterfaceType = "openai-image"
|
||||
ChannelInterfaceGrokImage ChannelInterfaceType = "grok-image"
|
||||
ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image"
|
||||
ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image"
|
||||
ChannelInterfaceGeminiImage ChannelInterfaceType = "gemini-image"
|
||||
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"
|
||||
ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio"
|
||||
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"
|
||||
ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1"
|
||||
ChannelInterfaceNewAPIChannel2 ChannelInterfaceType = "newapi-channel-2"
|
||||
ChannelInterfaceXAIVideo ChannelInterfaceType = "xai-video"
|
||||
ChannelInterfaceVolcengineArkVideo ChannelInterfaceType = "volcengine-ark-video"
|
||||
ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image"
|
||||
ChannelInterfaceVolcengineArkAgentPlanImage ChannelInterfaceType = "volcengine-ark-agent-plan-image"
|
||||
ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image"
|
||||
ChannelInterfaceGeminiImage ChannelInterfaceType = "gemini-image"
|
||||
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"
|
||||
ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio"
|
||||
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"
|
||||
ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1"
|
||||
ChannelInterfaceNewAPIChannel2 ChannelInterfaceType = "newapi-channel-2"
|
||||
ChannelInterfaceXAIVideo ChannelInterfaceType = "xai-video"
|
||||
ChannelInterfaceVolcengineArkVideo ChannelInterfaceType = "volcengine-ark-video"
|
||||
ChannelInterfaceVolcengineArkAgentPlanVideo ChannelInterfaceType = "volcengine-ark-agent-plan-video"
|
||||
ChannelInterfaceVolcengineJiMengVideo ChannelInterfaceType = "volcengine-jimeng-video"
|
||||
ChannelInterfaceGeminiVeo ChannelInterfaceType = "gemini-veo"
|
||||
ChannelInterfaceNovitaVideo ChannelInterfaceType = "novita-video"
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
package model
|
||||
|
||||
// IsVolcengineArkImageProtocol 识别官方 Ark 图片与 Agent Plan 图片协议。
|
||||
func IsVolcengineArkImageProtocol(protocol ChannelInterfaceType) bool {
|
||||
return protocol == ChannelInterfaceVolcengineArkImage || protocol == ChannelInterfaceVolcengineArkAgentPlanImage
|
||||
}
|
||||
|
||||
// IsVolcengineArkVideoProtocol 识别官方 Ark 视频与 Agent Plan 视频协议。
|
||||
func IsVolcengineArkVideoProtocol(protocol ChannelInterfaceType) bool {
|
||||
return protocol == ChannelInterfaceVolcengineArkVideo || protocol == ChannelInterfaceVolcengineArkAgentPlanVideo
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package model
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsVolcengineArkProtocols(t *testing.T) {
|
||||
if !IsVolcengineArkImageProtocol(ChannelInterfaceVolcengineArkImage) {
|
||||
t.Fatal("official ark image should match")
|
||||
}
|
||||
if !IsVolcengineArkImageProtocol(ChannelInterfaceVolcengineArkAgentPlanImage) {
|
||||
t.Fatal("agent plan image should match")
|
||||
}
|
||||
if IsVolcengineArkImageProtocol(ChannelInterfaceOpenAIImage) {
|
||||
t.Fatal("openai image should not match ark image")
|
||||
}
|
||||
if !IsVolcengineArkVideoProtocol(ChannelInterfaceVolcengineArkVideo) {
|
||||
t.Fatal("official ark video should match")
|
||||
}
|
||||
if !IsVolcengineArkVideoProtocol(ChannelInterfaceVolcengineArkAgentPlanVideo) {
|
||||
t.Fatal("agent plan video should match")
|
||||
}
|
||||
if IsVolcengineArkVideoProtocol(ChannelInterfaceNewAPIVideo) {
|
||||
t.Fatal("newapi video should not match ark video")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
# 火山方舟 Agent Plan 图片生成
|
||||
|
||||
本插件面向火山方舟 Agent Plan 专属图片入口。请求体与官方 Ark Seedream 协议一致,但 Base URL 与 API Key 必须来自 Agent Plan 控制台。
|
||||
|
||||
## 接口、鉴权与区域
|
||||
|
||||
{{OPERATIONS}}
|
||||
|
||||
```http
|
||||
POST {channel_base_url}/api/plan/v3/images/generations
|
||||
Authorization: Bearer <AGENT_PLAN_API_KEY>
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
||||
与官方 `/api/v3` 的差异:
|
||||
|
||||
| 项 | 官方 Ark | Agent Plan |
|
||||
| --- | --- | --- |
|
||||
| Base URL | `https://ark.cn-beijing.volces.com/api/v3` | `https://ark.cn-beijing.volces.com/api/plan/v3` |
|
||||
| API Key | 推理接入控制台 Key | Agent Plan 专属 Key |
|
||||
| 计费 | 推理后付费/项目额度 | AFP 套餐抵扣 |
|
||||
| 请求体 | Seedream images/generations | 同左 |
|
||||
|
||||
不要把官方 Key 打到 `/api/plan/v3`,也不要把 Agent Plan Key 打到 `/api/v3`。
|
||||
|
||||
## 参数与字段映射
|
||||
|
||||
{{PARAMETERS}}
|
||||
|
||||
当前实现:`aspectRatio -> size`,所有 `images -> image[]`;`extra` 可透传 `size`、`sequential_image_generation`、`sequential_image_generation_options`、`watermark`。
|
||||
|
||||
## 官方资料
|
||||
|
||||
- [火山方舟 Agent Plan 文档](https://www.volcengine.com/docs/82379/2375486)
|
||||
- [火山方舟视觉模型 API 文档](https://www.volcengine.com/docs/82379)
|
||||
|
||||
{{CONTRACT}}
|
||||
@@ -0,0 +1,43 @@
|
||||
# 火山方舟 Agent Plan 视频任务
|
||||
|
||||
本插件使用 Agent Plan 专属 Contents Generations Tasks 异步接口。提示词、参考图、参考视频和参考音频组成 `content[]`;比例、分辨率、时长、生成音频和水印位于顶层任务参数。请求体与官方 Ark Seedance 一致,入口改为 `/api/plan/v3`。
|
||||
|
||||
## 接口与鉴权
|
||||
|
||||
{{OPERATIONS}}
|
||||
|
||||
```http
|
||||
POST {channel_base_url}/api/plan/v3/contents/generations/tasks
|
||||
GET {channel_base_url}/api/plan/v3/contents/generations/tasks/{task_id}
|
||||
DELETE {channel_base_url}/api/plan/v3/contents/generations/tasks/{task_id}
|
||||
Authorization: Bearer <AGENT_PLAN_API_KEY>
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
||||
与官方 `/api/v3` 的差异:
|
||||
|
||||
| 项 | 官方 Ark | Agent Plan |
|
||||
| --- | --- | --- |
|
||||
| Base URL | `https://ark.cn-beijing.volces.com/api/v3` | `https://ark.cn-beijing.volces.com/api/plan/v3` |
|
||||
| API Key | 推理接入控制台 Key | Agent Plan 专属 Key |
|
||||
| 计费 | 推理后付费/项目额度 | AFP 套餐抵扣 |
|
||||
| 请求体 | contents/generations/tasks | 同左 |
|
||||
|
||||
## 参数与字段映射
|
||||
|
||||
{{PARAMETERS}}
|
||||
|
||||
全模态参考的素材映射如下:
|
||||
|
||||
- 图片:`{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`
|
||||
- 视频:`{"type":"video_url","video_url":{"url":"..."},"role":"reference_video"}`
|
||||
- 音频:`{"type":"audio_url","audio_url":{"url":"..."},"role":"reference_audio"}`
|
||||
|
||||
单次最多发送 9 张图片、3 个视频和 3 个音频。参考素材可以组合使用,但上游不支持纯音频或“文本 + 音频”。
|
||||
|
||||
## 官方资料
|
||||
|
||||
- [火山方舟 Agent Plan 文档](https://www.volcengine.com/docs/82379/2375486)
|
||||
- [火山方舟内容生成任务文档](https://www.volcengine.com/docs/82379/1520757)
|
||||
|
||||
{{CONTRACT}}
|
||||
@@ -711,6 +711,8 @@ func mediaReferencesFromManifestValue(value any, kind string, ephemeral bool) []
|
||||
if reference.DataURL == "" {
|
||||
reference.DataURL = firstString(typed, "data_url", "b64_json")
|
||||
}
|
||||
// OpenAI / Ark 等渠道常直接返回裸 b64_json;声明式结果下载要求 data URL。
|
||||
reference.DataURL = normalizeManifestInlineDataURL(reference.DataURL, reference.Kind, firstString(typed, "output_format", "mime_type", "mimeType"))
|
||||
if reference.URL != "" || reference.DataURL != "" {
|
||||
result = append(result, reference)
|
||||
}
|
||||
@@ -719,6 +721,50 @@ func mediaReferencesFromManifestValue(value any, kind string, ephemeral bool) []
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeManifestInlineDataURL(value, kind, formatHint string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || strings.HasPrefix(value, "data:") {
|
||||
return value
|
||||
}
|
||||
return "data:" + manifestInlineMediaMIME(kind, formatHint) + ";base64," + value
|
||||
}
|
||||
|
||||
func manifestInlineMediaMIME(kind, formatHint string) string {
|
||||
hint := strings.ToLower(strings.TrimSpace(formatHint))
|
||||
switch hint {
|
||||
case "image/png", "image/jpeg", "image/webp", "image/gif", "audio/mpeg", "audio/wav", "audio/ogg", "video/mp4", "video/webm":
|
||||
return hint
|
||||
case "image/jpg":
|
||||
return "image/jpeg"
|
||||
case "audio/mp3":
|
||||
return "audio/mpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "jpeg", "jpg":
|
||||
return "image/jpeg"
|
||||
case "webp":
|
||||
return "image/webp"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "mp3", "mpeg":
|
||||
return "audio/mpeg"
|
||||
case "wav":
|
||||
return "audio/wav"
|
||||
case "mp4":
|
||||
return "video/mp4"
|
||||
case "webm":
|
||||
return "video/webm"
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(kind)) {
|
||||
case "audio":
|
||||
return "audio/mpeg"
|
||||
case "video":
|
||||
return "video/mp4"
|
||||
default:
|
||||
return "image/png"
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
manifestMDImageRegex = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`)
|
||||
manifestHTMLVideoRegex = regexp.MustCompile(`(?i)<video[^>]+src=['"]([^'"]+)['"]`)
|
||||
|
||||
@@ -599,6 +599,36 @@ func TestOfficialArkSeedreamMapsAspectRatioToPixelSize(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialArkAgentPlanPluginsUsePlanPaths(t *testing.T) {
|
||||
image := officialPackageAdapter(t, "volcengine-ark-agent-plan-seedream.yingce-plugin", "volcengine-ark-agent-plan-image")
|
||||
imageCreate, err := image.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: "doubao-seedream-5-0-260128", Prompt: "circle", AspectRatio: "1:1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if imageCreate.Path != "/api/plan/v3/images/generations" {
|
||||
t.Fatalf("agent plan image create = %#v", imageCreate)
|
||||
}
|
||||
if body := manifestTestBody(t, imageCreate); body["size"] != "2048x2048" {
|
||||
t.Fatalf("agent plan image size = %#v", body["size"])
|
||||
}
|
||||
|
||||
video := officialPackageAdapter(t, "volcengine-ark-agent-plan-seedance.yingce-plugin", "volcengine-ark-agent-plan-video")
|
||||
videoCreate, err := video.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{Model: "doubao-seedance-2-0-260128", Prompt: "walk", AspectRatio: "16:9", Resolution: "720p", Duration: 5}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if videoCreate.Path != "/api/plan/v3/contents/generations/tasks" {
|
||||
t.Fatalf("agent plan video create = %#v", videoCreate)
|
||||
}
|
||||
poll, err := video.BuildPoll(context.Background(), PollContext{TaskID: "task-1", Model: "doubao-seedance-2-0-260128"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if poll.Path != "/api/plan/v3/contents/generations/tasks/task-1" {
|
||||
t.Fatalf("agent plan video poll = %#v", poll)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialGeminiImageMapsQualityToImageSize(t *testing.T) {
|
||||
adapter := officialPackageAdapter(t, "google-gemini-image.yingce-plugin", "gemini-image")
|
||||
tests := []struct {
|
||||
@@ -754,3 +784,27 @@ func TestOfficialOpenAIAudioSpeedDefaultsInvalidAndZeroValues(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialArkSeedreamParsesB64JSONAsDataURL(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
packageName, providerID string
|
||||
}{
|
||||
{"volcengine-ark-seedream.yingce-plugin", "volcengine-ark-image"},
|
||||
{"volcengine-ark-agent-plan-seedream.yingce-plugin", "volcengine-ark-agent-plan-image"},
|
||||
} {
|
||||
t.Run(tc.providerID, func(t *testing.T) {
|
||||
adapter := officialPackageAdapter(t, tc.packageName, tc.providerID)
|
||||
result, err := adapter.ParseCreate(context.Background(), []byte(`{"created":1,"data":[{"b64_json":"aW1hZ2U=","output_format":"jpeg","size":"1824x1024"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Result == nil || len(result.Result.Images) != 1 {
|
||||
t.Fatalf("result = %#v", result.Result)
|
||||
}
|
||||
dataURL := result.Result.Images[0].DataURL
|
||||
if dataURL != "data:image/jpeg;base64,aW1hZ2U=" {
|
||||
t.Fatalf("DataURL = %q, want jpeg data URL from b64_json + output_format", dataURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user