fix(canvas): 视频参考协议 - 统一首尾帧与角色资产语义
This commit is contained in:
1 parent
b5f8b66622
commit
37eb2b7e48
15 files changed
+479
-68
No files matched your search
@@ -2341,7 +2341,7 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe
|
||||
request := protocol.GenerationRequest{
|
||||
Model: input.Config.Model,
|
||||
Prompt: input.Prompt,
|
||||
Images: protocolMediaReferences(input.ReferenceImages, "image"),
|
||||
Images: protocolVideoImageReferences(input),
|
||||
Videos: protocolMediaReferences(input.ReferenceVideos, "video"),
|
||||
Audios: protocolMediaReferences(input.ReferenceAudios, "audio"),
|
||||
AspectRatio: input.Config.Size,
|
||||
@@ -2370,10 +2370,31 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe
|
||||
return request
|
||||
}
|
||||
|
||||
func protocolVideoImageReferences(input canvasGenerationInput) []protocol.MediaReference {
|
||||
result := make([]protocol.MediaReference, 0, len(input.ReferenceImages))
|
||||
fallbackRole := ""
|
||||
if metadataString(input.Metadata, "videoStartFrameNodeId") != "" || metadataString(input.Metadata, "videoEndFrameNodeId") != "" {
|
||||
fallbackRole = "reference_image"
|
||||
}
|
||||
for _, value := range input.ReferenceImages {
|
||||
item := protocol.MediaReference{
|
||||
ID: strings.TrimSpace(value.ID),
|
||||
URL: strings.TrimSpace(value.URL),
|
||||
DataURL: strings.TrimSpace(value.DataURL),
|
||||
Kind: "image",
|
||||
Role: videoImageRoleOrDefault(input, value, fallbackRole),
|
||||
}
|
||||
if item.URL != "" || item.DataURL != "" {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func protocolMediaReferences(values []providerMedia, kind string) []protocol.MediaReference {
|
||||
result := make([]protocol.MediaReference, 0, len(values))
|
||||
for _, value := range values {
|
||||
item := protocol.MediaReference{URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), Kind: kind}
|
||||
item := protocol.MediaReference{ID: strings.TrimSpace(value.ID), URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), Kind: kind}
|
||||
if item.URL != "" || item.DataURL != "" {
|
||||
result = append(result, item)
|
||||
}
|
||||
@@ -2868,16 +2889,9 @@ func runMiniMaxVideoTask(ctx context.Context, input canvasGenerationInput) (map[
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MiniMax 参考图无效:%w", err)
|
||||
}
|
||||
role := "reference_image"
|
||||
if frameMode {
|
||||
switch {
|
||||
case startFrameID != "" && image.ID == startFrameID:
|
||||
role = "first_frame"
|
||||
case endFrameID != "" && image.ID == endFrameID:
|
||||
role = "last_frame"
|
||||
default:
|
||||
return nil, errors.New("MiniMax 首尾帧模式不能混合未标记的参考图")
|
||||
}
|
||||
role := videoImageRole(input, image)
|
||||
if frameMode && role == "reference_image" {
|
||||
return nil, errors.New("MiniMax 首尾帧模式不能混合未标记的参考图")
|
||||
}
|
||||
content = append(content, miniMaxVideoContent{Type: "image_url", ImageURL: &miniMaxMediaURL{URL: url}, Role: role})
|
||||
}
|
||||
@@ -3014,6 +3028,9 @@ func runGeminiVeoVideoTask(ctx context.Context, input canvasGenerationInput) (ma
|
||||
if len(input.ReferenceImages) > 1 || len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
|
||||
return nil, errors.New("Gemini Veo 当前只支持 1 张起始图,不支持参考视频或音频")
|
||||
}
|
||||
if len(input.ReferenceImages) > 0 && metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" {
|
||||
return nil, errors.New("Gemini Veo 当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型")
|
||||
}
|
||||
id := resumedProviderRequestID(ctx)
|
||||
if id == "" {
|
||||
instance := geminiVeoInstance{Prompt: strings.TrimSpace(input.Prompt)}
|
||||
@@ -3074,6 +3091,9 @@ func runNovitaVideoTask(ctx context.Context, input canvasGenerationInput) (map[s
|
||||
if len(input.ReferenceImages) > 1 || len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
|
||||
return nil, errors.New("Novita 视频当前只支持 1 张起始图,不支持参考视频或参考音频")
|
||||
}
|
||||
if len(input.ReferenceImages) > 0 && metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" {
|
||||
return nil, errors.New("Novita 视频当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型")
|
||||
}
|
||||
id := resumedProviderRequestID(ctx)
|
||||
if id == "" {
|
||||
body := map[string]interface{}{
|
||||
@@ -3636,7 +3656,7 @@ func newAPIChannel1VideoBody(input canvasGenerationInput) (map[string]interface{
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
media = append(media, map[string]string{"type": seedanceImageRole(input, image), "url": url})
|
||||
media = append(media, map[string]string{"type": videoImageRole(input, image), "url": url})
|
||||
}
|
||||
}
|
||||
for _, video := range input.ReferenceVideos {
|
||||
@@ -3798,6 +3818,9 @@ func xaiVideoRequestBody(input canvasGenerationInput) (xaiVideoRequest, error) {
|
||||
return body, nil
|
||||
}
|
||||
startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId")
|
||||
if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" {
|
||||
startFrameID = ""
|
||||
}
|
||||
if startFrameID == "" {
|
||||
// 未设置首帧:所有参考图作为 R2V 参考,不受单张起始图限制。
|
||||
for index := range input.ReferenceImages {
|
||||
@@ -4874,7 +4897,7 @@ func seedanceContent(input canvasGenerationInput) ([]map[string]interface{}, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = append(content, map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": url}, "role": seedanceImageRole(input, image)})
|
||||
content = append(content, map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": url}, "role": videoImageRole(input, image)})
|
||||
}
|
||||
for _, video := range input.ReferenceVideos {
|
||||
url, err := mediaReferenceURL(video)
|
||||
@@ -4943,7 +4966,9 @@ func seedanceVideosRequestBody(input canvasGenerationInput) (seedanceVideosReque
|
||||
if err != nil {
|
||||
return seedanceVideosRequest{}, err
|
||||
}
|
||||
if len(frameImageURLs) > 0 {
|
||||
if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" {
|
||||
body.ReferenceImageURLs = imageURLs
|
||||
} else if len(frameImageURLs) > 0 {
|
||||
body.ImageURLs = frameImageURLs
|
||||
} else if len(imageURLs) > 0 {
|
||||
body.ImageURL = imageURLs[0]
|
||||
@@ -4993,17 +5018,27 @@ func seedanceVideosPromptText(input canvasGenerationInput) string {
|
||||
return strings.TrimSpace(input.Prompt)
|
||||
}
|
||||
|
||||
func seedanceImageRole(input canvasGenerationInput, image providerMedia) string {
|
||||
func videoImageRole(input canvasGenerationInput, image providerMedia) string {
|
||||
return videoImageRoleOrDefault(input, image, "reference_image")
|
||||
}
|
||||
|
||||
func videoImageRoleOrDefault(input canvasGenerationInput, image providerMedia, fallback string) string {
|
||||
if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" {
|
||||
return "reference_image"
|
||||
}
|
||||
if id := metadataString(input.Metadata, "videoStartFrameNodeId"); id != "" && image.ID == id {
|
||||
return "first_frame"
|
||||
}
|
||||
if id := metadataString(input.Metadata, "videoEndFrameNodeId"); id != "" && image.ID == id {
|
||||
return "last_frame"
|
||||
}
|
||||
return "reference_image"
|
||||
return fallback
|
||||
}
|
||||
|
||||
func videoFrameImageURLs(input canvasGenerationInput, imageURLs []string) ([]string, error) {
|
||||
if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" {
|
||||
return nil, nil
|
||||
}
|
||||
startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId")
|
||||
endFrameID := metadataString(input.Metadata, "videoEndFrameNodeId")
|
||||
if startFrameID == "" && endFrameID == "" {
|
||||
|
||||
@@ -1372,6 +1372,31 @@ func TestSeedanceVideosBodyUsesOrderedFrameImageURLsWhenConfigured(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedanceVideosBodyKeepsProjectAssetsAsReferenceImages(t *testing.T) {
|
||||
body, err := seedanceVideosRequestBody(canvasGenerationInput{
|
||||
Prompt: "keep the character consistent",
|
||||
Config: providerConfig{Model: "seedance-2.0"},
|
||||
ReferenceImages: []providerMedia{
|
||||
{ID: "character-1", DataURL: testReferenceImageDataURL},
|
||||
{ID: "character-2", DataURL: "data:image/png;base64,d29ybGQ="},
|
||||
},
|
||||
Metadata: map[string]interface{}{
|
||||
"videoEditOperation": "reference_to_video",
|
||||
"videoStartFrameNodeId": "character-1",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seedanceVideosRequestBody() error = %v", err)
|
||||
}
|
||||
want := []string{testReferenceImageDataURL, "data:image/png;base64,d29ybGQ="}
|
||||
if !reflect.DeepEqual(body.ReferenceImageURLs, want) {
|
||||
t.Fatalf("reference_image_urls = %#v, want %#v", body.ReferenceImageURLs, want)
|
||||
}
|
||||
if body.ImageURL != "" || body.ImageURLs != nil {
|
||||
t.Fatalf("reference operation leaked frame fields: %#v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunVideoTaskUsesNewAPIForAnyVideoModel(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
paths := make([]string, 0, 3)
|
||||
@@ -1731,6 +1756,22 @@ func TestXAIVideoBodyWithStartFrameKeepsOfficialImageShape(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestXAIVideoReferenceOperationIgnoresStaleStartFrameMetadata(t *testing.T) {
|
||||
body, err := xaiVideoRequestBody(canvasGenerationInput{
|
||||
Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"},
|
||||
ReferenceImages: []providerMedia{
|
||||
{ID: "character", DataURL: testReferenceImageDataURL},
|
||||
},
|
||||
Metadata: map[string]interface{}{"videoEditOperation": "reference_to_video", "videoStartFrameNodeId": "character"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("xaiVideoRequestBody() error = %v", err)
|
||||
}
|
||||
if body.Image != nil || len(body.ReferenceImages) != 1 {
|
||||
t.Fatalf("xAI reference operation body = %#v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXAIVideoBodyWithStartFrameRejectsMultipleImages(t *testing.T) {
|
||||
_, err := xaiVideoRequestBody(canvasGenerationInput{
|
||||
Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"},
|
||||
@@ -1919,6 +1960,38 @@ func TestNewAPIChannel1VideoBodyMapsFramesAndReferences(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolRequestPreservesVideoImageIDsAndRoles(t *testing.T) {
|
||||
request := protocolRequestFromInput(canvasGenerationInput{
|
||||
Mode: "video",
|
||||
ReferenceImages: []providerMedia{
|
||||
{ID: "start", URL: "https://example.com/start.png"},
|
||||
{ID: "character", URL: "https://example.com/character.png"},
|
||||
},
|
||||
Metadata: map[string]interface{}{
|
||||
"videoEditOperation": "image_to_video",
|
||||
"videoStartFrameNodeId": "start",
|
||||
},
|
||||
})
|
||||
if len(request.Images) != 2 {
|
||||
t.Fatalf("images = %#v", request.Images)
|
||||
}
|
||||
if request.Images[0].ID != "start" || request.Images[0].Role != "first_frame" {
|
||||
t.Fatalf("start image = %#v", request.Images[0])
|
||||
}
|
||||
if request.Images[1].ID != "character" || request.Images[1].Role != "reference_image" {
|
||||
t.Fatalf("unmarked image role = %#v", request.Images[1])
|
||||
}
|
||||
|
||||
request = protocolRequestFromInput(canvasGenerationInput{
|
||||
Mode: "video",
|
||||
ReferenceImages: []providerMedia{{ID: "character", URL: "https://example.com/character.png"}},
|
||||
Metadata: map[string]interface{}{"videoEditOperation": "reference_to_video", "videoStartFrameNodeId": "character"},
|
||||
})
|
||||
if request.Images[0].Role != "reference_image" {
|
||||
t.Fatalf("reference operation image = %#v", request.Images[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAPIChannel1VideoBodyRejectsInlineMedia(t *testing.T) {
|
||||
_, err := newAPIChannel1VideoBody(canvasGenerationInput{
|
||||
Prompt: "make it move",
|
||||
|
||||
Reference in new issue
Block a user