feat(media): 视频渠道 - 新增 Novita 异步视频生成协议
Co-authored-by: jax-novita <jax-novita@users.noreply.github.com> Original-PR: https://github.com/ddcat-ai/open-ai-canvas/pull/206
This commit is contained in:
1 parent
477aebded5
commit
7f0229463e
11 files changed
+302
-15
No files matched your search
@@ -23,15 +23,16 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
runtimeService *service.Service
|
||||
geminiGeneratePath = regexp.MustCompile(`^/models/([^/:]+):(generateContent|streamGenerateContent)$`)
|
||||
customGeminiRelayPath = regexp.MustCompile(`(?:^|/)models/[^/:]+:(generateContent|streamGenerateContent|predictLongRunning)$`)
|
||||
customVideoTaskPath = regexp.MustCompile(`(?:^|/)video/generations/[^/]+$`)
|
||||
customXAIVideoTaskPath = regexp.MustCompile(`(?:^|/)videos/[^/]+$`)
|
||||
customVideoContentPath = regexp.MustCompile(`(?:^|/)videos/[^/]+/content$`)
|
||||
customArkVideoTaskPath = regexp.MustCompile(`(?:^|/)contents/generations/tasks/[^/]+$`)
|
||||
customGeminiOperationPath = regexp.MustCompile(`(?:^|/)(?:models/[^/]+/)?operations/[^/]+$`)
|
||||
openAIPostEndpoints = map[string]bool{
|
||||
runtimeService *service.Service
|
||||
geminiGeneratePath = regexp.MustCompile(`^/models/([^/:]+):(generateContent|streamGenerateContent)$`)
|
||||
customGeminiRelayPath = regexp.MustCompile(`(?:^|/)models/[^/:]+:(generateContent|streamGenerateContent|predictLongRunning)$`)
|
||||
customVideoTaskPath = regexp.MustCompile(`(?:^|/)video/generations/[^/]+$`)
|
||||
customXAIVideoTaskPath = regexp.MustCompile(`(?:^|/)videos/[^/]+$`)
|
||||
customVideoContentPath = regexp.MustCompile(`(?:^|/)videos/[^/]+/content$`)
|
||||
customArkVideoTaskPath = regexp.MustCompile(`(?:^|/)contents/generations/tasks/[^/]+$`)
|
||||
customGeminiOperationPath = regexp.MustCompile(`(?:^|/)(?:models/[^/]+/)?operations/[^/]+$`)
|
||||
customNovitaTaskResultPath = regexp.MustCompile(`(?:^|/)async/task-result$`)
|
||||
openAIPostEndpoints = map[string]bool{
|
||||
"/responses": true, "/chat/completions": true, "/images/generations": true, "/images/edits": true,
|
||||
"/audio/speech": true,
|
||||
}
|
||||
@@ -62,6 +63,12 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont
|
||||
return errors.New("自定义渠道调用格式无效")
|
||||
}
|
||||
if method == http.MethodGet {
|
||||
if customNovitaTaskResultPath.MatchString(requestPath) {
|
||||
if len(query) == 1 && len(query["task_id"]) == 1 && strings.TrimSpace(query.Get("task_id")) != "" {
|
||||
return nil
|
||||
}
|
||||
return errors.New("自定义渠道不允许访问该上游接口")
|
||||
}
|
||||
allowed := requestPath == "/models" || strings.HasSuffix(requestPath, "/models")
|
||||
if apiFormat == "openai" {
|
||||
allowed = allowed || customVideoTaskPath.MatchString(requestPath) || customXAIVideoTaskPath.MatchString(requestPath) || customVideoContentPath.MatchString(requestPath) || customArkVideoTaskPath.MatchString(requestPath)
|
||||
@@ -82,7 +89,7 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont
|
||||
}
|
||||
if apiFormat == "openai" {
|
||||
multipartAllowed := mediaType == "multipart/form-data" && (strings.HasSuffix(requestPath, "/images/edits") || strings.HasSuffix(requestPath, "/videos"))
|
||||
jsonAllowed := mediaType == "application/json" && (strings.HasSuffix(requestPath, "/responses") || strings.HasSuffix(requestPath, "/chat/completions") || strings.HasSuffix(requestPath, "/images/generations") || strings.HasSuffix(requestPath, "/images/edits") || strings.HasSuffix(requestPath, "/audio/speech") || strings.HasSuffix(requestPath, "/video/generations") || strings.HasSuffix(requestPath, "/videos/generations") || strings.HasSuffix(requestPath, "/videos") || strings.HasSuffix(requestPath, "/contents/generations/tasks"))
|
||||
jsonAllowed := mediaType == "application/json" && (strings.HasSuffix(requestPath, "/responses") || strings.HasSuffix(requestPath, "/chat/completions") || strings.HasSuffix(requestPath, "/images/generations") || strings.HasSuffix(requestPath, "/images/edits") || strings.HasSuffix(requestPath, "/audio/speech") || strings.HasSuffix(requestPath, "/video/generations") || strings.HasSuffix(requestPath, "/videos/generations") || strings.HasSuffix(requestPath, "/videos") || strings.HasSuffix(requestPath, "/contents/generations/tasks") || strings.HasSuffix(requestPath, "/video/create"))
|
||||
if len(query) != 0 || (!multipartAllowed && !jsonAllowed) {
|
||||
return errors.New("自定义渠道不允许访问该上游接口")
|
||||
}
|
||||
@@ -189,7 +196,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
|
||||
return requestPath == "/images/generations"
|
||||
case model.ChannelInterfaceOpenAIAudio:
|
||||
return requestPath == "/audio/speech"
|
||||
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
|
||||
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceNovitaVideo:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
|
||||
@@ -46,6 +46,8 @@ func TestAuthorizeCustomRelayAllowsModelsAndAgentEndpoints(t *testing.T) {
|
||||
{method: http.MethodGet, target: "https://api.example.com/v1/video/generations/task-1", apiFormat: "openai"},
|
||||
{method: http.MethodPost, target: "https://api.x.ai/v1/videos/generations", apiFormat: "openai", contentType: "application/json"},
|
||||
{method: http.MethodGet, target: "https://api.x.ai/v1/videos/request-1", apiFormat: "openai"},
|
||||
{method: http.MethodPost, target: "https://api.novita.ai/v3/video/create", apiFormat: "openai", contentType: "application/json"},
|
||||
{method: http.MethodGet, target: "https://api.novita.ai/v3/async/task-result?task_id=task-1", apiFormat: "openai"},
|
||||
{method: http.MethodPost, target: "https://ark.cn-beijing.volces.com/api/v3/images/generations", apiFormat: "openai", contentType: "application/json"},
|
||||
{method: http.MethodPost, target: "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", apiFormat: "openai", contentType: "application/json"},
|
||||
{method: http.MethodGet, target: "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks/task-1", apiFormat: "openai"},
|
||||
@@ -78,6 +80,8 @@ func TestAuthorizeCustomRelayRejectsArbitraryRequestsAndCredentialQueries(t *tes
|
||||
{method: http.MethodPost, target: "https://api.example.com/v1/account", apiFormat: "openai", contentType: "multipart/form-data; boundary=test"},
|
||||
{method: http.MethodPost, target: "https://api.example.com/v1/../account/chat/completions", apiFormat: "openai", contentType: "application/json"},
|
||||
{method: http.MethodPost, target: "https://api.example.com/v1/models/gemini:streamGenerateContent?alt=sse&token=secret", apiFormat: "gemini", contentType: "application/json"},
|
||||
{method: http.MethodGet, target: "https://api.novita.ai/v3/async/task-result?task_id=task-1&api_key=secret", apiFormat: "openai"},
|
||||
{method: http.MethodGet, target: "https://api.novita.ai/v3/async/task-result", apiFormat: "openai"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
target, err := url.Parse(test.target)
|
||||
|
||||
@@ -73,6 +73,7 @@ const (
|
||||
ChannelInterfaceVolcengineArkVideo ChannelInterfaceType = "volcengine-ark-video"
|
||||
ChannelInterfaceVolcengineJiMengVideo ChannelInterfaceType = "volcengine-jimeng-video"
|
||||
ChannelInterfaceGeminiVeo ChannelInterfaceType = "gemini-veo"
|
||||
ChannelInterfaceNovitaVideo ChannelInterfaceType = "novita-video"
|
||||
|
||||
ApiCallStatusSucceeded ApiCallStatus = "succeeded"
|
||||
ApiCallStatusFailed ApiCallStatus = "failed"
|
||||
|
||||
@@ -175,6 +175,12 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
|
||||
video.GenerateAudio = VideoBooleanConfig{Supported: true, Default: true}
|
||||
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceXAIVideo:
|
||||
video.GenerateAudio = VideoBooleanConfig{Supported: false, Default: false}
|
||||
case model.ChannelInterfaceNovitaVideo:
|
||||
video.References.MaxImages, video.References.MaxImageBytes = 1, 10*1024*1024
|
||||
video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{5, 10}, Default: 5}
|
||||
video.Ratios = []string{"16:9", "9:16", "1:1"}
|
||||
video.Resolutions = []string{"1080p"}
|
||||
video.DefaultResolution = "1080p"
|
||||
}
|
||||
return &ModelCapabilityConfig{Version: 1, Image: DefaultImageCapabilityConfig(protocol, modelName), Video: video}
|
||||
}
|
||||
|
||||
@@ -1339,6 +1339,9 @@ func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]
|
||||
if input.Config.InterfaceType == "gemini-veo" {
|
||||
return runGeminiVeoVideoTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceNovitaVideo) {
|
||||
return runNovitaVideoTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == "newapi-channel-2" {
|
||||
return runNewAPIChannel2VideoTask(ctx, input)
|
||||
}
|
||||
@@ -1507,6 +1510,126 @@ func runGeminiVeoVideoTask(ctx context.Context, input canvasGenerationInput) (ma
|
||||
return nil, fmt.Errorf("Gemini Veo 视频生成超时(任务 %s)", id)
|
||||
}
|
||||
|
||||
func runNovitaVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
|
||||
if len(input.ReferenceImages) > 1 || len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
|
||||
return nil, errors.New("Novita 视频当前只支持 1 张起始图,不支持参考视频或参考音频")
|
||||
}
|
||||
id := resumedProviderRequestID(ctx)
|
||||
if id == "" {
|
||||
body := map[string]interface{}{
|
||||
"model": input.Config.Model,
|
||||
"prompt": strings.TrimSpace(input.Prompt),
|
||||
"duration": normalizeNovitaVideoDuration(input.Config.VideoSeconds),
|
||||
}
|
||||
if len(input.ReferenceImages) == 1 {
|
||||
imageValue, err := novitaVideoImageValue(input.ReferenceImages[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body["image"] = imageValue
|
||||
} else {
|
||||
body["aspect_ratio"] = normalizeNovitaVideoRatio(input.Config.Size)
|
||||
}
|
||||
var created map[string]interface{}
|
||||
if err := postNovitaJSON(ctx, input.Config, "/video/create", body, &created); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id = strings.TrimSpace(stringField(created, "task_id"))
|
||||
}
|
||||
if id == "" {
|
||||
return nil, errors.New("Novita 视频接口没有返回任务 ID")
|
||||
}
|
||||
for deadline := providerPollingDeadline(ctx); time.Now().Before(deadline); {
|
||||
var result map[string]interface{}
|
||||
if err := getNovitaJSON(ctx, input.Config, "/async/task-result?task_id="+url.QueryEscape(id), &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
task, _ := result["task"].(map[string]interface{})
|
||||
switch stringField(task, "status") {
|
||||
case "TASK_STATUS_SUCCEED":
|
||||
videos, _ := result["videos"].([]interface{})
|
||||
if len(videos) == 0 {
|
||||
return nil, fmt.Errorf("Novita 视频任务 %s 已完成但没有返回视频", id)
|
||||
}
|
||||
first, _ := videos[0].(map[string]interface{})
|
||||
videoURL := strings.TrimSpace(stringField(first, "video_url"))
|
||||
if videoURL == "" {
|
||||
return nil, fmt.Errorf("Novita 视频任务 %s 已完成但没有返回视频地址", id)
|
||||
}
|
||||
data, mimeType, err := getExternalBinary(withProviderRequestKind(ctx, "download"), videoURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Novita 视频结果下载失败(任务 %s):%w", id, err)
|
||||
}
|
||||
mimeType = normalizedMediaMimeType(mimeType, data)
|
||||
return map[string]interface{}{"mode": "video", "video": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType}}, nil
|
||||
case "TASK_STATUS_FAILED":
|
||||
reason := firstNonEmptyString(stringField(task, "reason"), "上游返回失败")
|
||||
return nil, fmt.Errorf("Novita 视频生成失败(任务 %s):%s", id, reason)
|
||||
}
|
||||
if err := sleepContext(ctx, 5*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("Novita 视频生成超时(任务 %s)", id)
|
||||
}
|
||||
|
||||
func novitaVideoImageValue(media providerMedia) (string, error) {
|
||||
if isPublicMediaURL(media.URL) {
|
||||
if _, err := ValidateOutboundURL(media.URL); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return media.URL, nil
|
||||
}
|
||||
raw, mimeType, err := mediaBytes(media)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dataURL(mimeType, raw), nil
|
||||
}
|
||||
|
||||
func normalizeNovitaVideoDuration(value string) string {
|
||||
if normalizeSeedanceDuration(value) >= 8 {
|
||||
return "10"
|
||||
}
|
||||
return "5"
|
||||
}
|
||||
|
||||
func normalizeNovitaVideoRatio(value string) string {
|
||||
switch strings.TrimSpace(value) {
|
||||
case "16:9", "9:16", "1:1":
|
||||
return strings.TrimSpace(value)
|
||||
default:
|
||||
return "16:9"
|
||||
}
|
||||
}
|
||||
|
||||
func novitaVideoURL(baseURL string, path string) string {
|
||||
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
return base + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func postNovitaJSON(ctx context.Context, config providerConfig, path string, body interface{}, target interface{}) error {
|
||||
data, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, novitaVideoURL(config.BaseURL, path), bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+config.APIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
ApplyOutboundHeaders(req, config.Headers)
|
||||
return doJSON(req, target)
|
||||
}
|
||||
|
||||
func getNovitaJSON(ctx context.Context, config providerConfig, path string, target interface{}) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, novitaVideoURL(config.BaseURL, path), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+config.APIKey)
|
||||
ApplyOutboundHeaders(req, config.Headers)
|
||||
return doJSON(req, target)
|
||||
}
|
||||
|
||||
func postGeminiJSON(ctx context.Context, config providerConfig, path string, body interface{}, target interface{}) error {
|
||||
data, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, geminiVeoURL(config.BaseURL, path), bytes.NewReader(data))
|
||||
@@ -2027,7 +2150,7 @@ func validateGenerationInterface(mode string, interfaceType string) error {
|
||||
allowed := map[string]map[string]bool{
|
||||
"text": {"chat-completion": true, "openai-response": true},
|
||||
"image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true},
|
||||
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true},
|
||||
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true, "novita-video": true},
|
||||
"audio": {"openai-audio": true, "async-audio": true},
|
||||
}
|
||||
if allowed[mode] != nil && !allowed[mode][interfaceType] {
|
||||
|
||||
@@ -1278,3 +1278,83 @@ func TestEquivalentStyleProfileJSONIgnoresObjectKeyOrder(t *testing.T) {
|
||||
t.Fatalf("equivalentStyleProfileJSON() equal = %v, err = %v", equal, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNovitaVideoTaskDownloadsSucceededVideo(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
paths := make([]string, 0, 3)
|
||||
var server *httptest.Server
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
paths = append(paths, r.Method+" "+r.URL.String())
|
||||
switch r.Method + " " + r.URL.Path {
|
||||
case "POST /video/create":
|
||||
if auth := r.Header.Get("Authorization"); auth != "Bearer test-key" {
|
||||
t.Errorf("Authorization = %q", auth)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
if body["model"] != "kling2.5_turbo_pro_t2v" || body["prompt"] != "make it move" || body["duration"] != "5" || body["aspect_ratio"] != "16:9" {
|
||||
t.Errorf("body = %#v", body)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"task_id":"novita-task-1"}`))
|
||||
case "GET /async/task-result":
|
||||
if r.URL.Query().Get("task_id") != "novita-task-1" {
|
||||
t.Errorf("task_id = %q", r.URL.Query().Get("task_id"))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"task":{"status":"TASK_STATUS_SUCCEED"},"videos":[{"video_url":"` + server.URL + `/video.mp4"}]}`))
|
||||
case "GET /video.mp4":
|
||||
if authorization := r.Header.Get("Authorization"); authorization != "" {
|
||||
t.Errorf("file Authorization = %q, want empty", authorization)
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("video"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := runVideoTask(context.Background(), canvasGenerationInput{
|
||||
Prompt: "make it move",
|
||||
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", Model: "kling2.5_turbo_pro_t2v", InterfaceType: "novita-video", VideoSeconds: "5", Size: "16:9"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runVideoTask() error = %v", err)
|
||||
}
|
||||
video := result["video"].(map[string]interface{})
|
||||
if video["dataUrl"] != "data:video/mp4;base64,dmlkZW8=" {
|
||||
t.Fatalf("video = %#v", video)
|
||||
}
|
||||
want := "POST /video/create,GET /async/task-result?task_id=novita-task-1,GET /video.mp4"
|
||||
if got := strings.Join(paths, ","); got != want {
|
||||
t.Fatalf("paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNovitaVideoTaskReturnsFailureReason(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method + " " + r.URL.Path {
|
||||
case "POST /video/create":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"task_id":"novita-task-2"}`))
|
||||
case "GET /async/task-result":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"task":{"status":"TASK_STATUS_FAILED","reason":"content violates policy"}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := runVideoTask(context.Background(), canvasGenerationInput{
|
||||
Prompt: "make it move",
|
||||
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", Model: "kling2.5_turbo_pro_t2v", InterfaceType: "novita-video"},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "content violates policy") {
|
||||
t.Fatalf("runVideoTask() error = %v, want reason in message", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user