diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index c12870d2e..1e7699dbf 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -593,7 +593,7 @@ func runChatCompletionsTextTask(ctx context.Context, input canvasGenerationInput func textResponseInput(input canvasGenerationInput) (interface{}, error) { systemPrompt := strings.TrimSpace(input.Config.SystemPrompt) - if len(input.ReferenceImages) == 0 { + if len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 { return withSystemPrompt(input.Config, input.Prompt), nil } messages := make([]map[string]interface{}, 0, 2) @@ -617,11 +617,18 @@ func textResponseContent(input canvasGenerationInput) ([]map[string]interface{}, } content = append(content, map[string]interface{}{"type": "input_image", "image_url": url}) } + for _, video := range input.ReferenceVideos { + url, err := openAIVideoInputURL(video) + if err != nil { + return nil, err + } + content = append(content, map[string]interface{}{"type": "input_video", "video_url": url}) + } return content, nil } func textChatContent(input canvasGenerationInput) (interface{}, error) { - if len(input.ReferenceImages) == 0 { + if len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 { return input.Prompt, nil } content := []map[string]interface{}{{"type": "text", "text": input.Prompt}} @@ -632,6 +639,13 @@ func textChatContent(input canvasGenerationInput) (interface{}, error) { } content = append(content, map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": url}}) } + for _, video := range input.ReferenceVideos { + url, err := openAIVideoInputURL(video) + if err != nil { + return nil, err + } + content = append(content, map[string]interface{}{"type": "video_url", "video_url": map[string]interface{}{"url": url}}) + } return content, nil } @@ -653,6 +667,24 @@ func openAIImageInputURL(media providerMedia) (string, error) { return "", errors.New("OpenAI 文本多模态参考图片需要公网 URL 或 base64 data URL") } +func openAIVideoInputURL(media providerMedia) (string, error) { + value := strings.TrimSpace(media.DataURL) + if strings.HasPrefix(value, "data:video/") { + return value, nil + } + if strings.HasPrefix(value, "data:") { + return "", errors.New("参考视频 MIME 类型无效，请重新读取或上传视频") + } + value = strings.TrimSpace(media.URL) + if strings.HasPrefix(value, "data:video/") || isPublicMediaURL(value) { + return value, nil + } + if strings.HasPrefix(value, "data:") { + return "", errors.New("参考视频 MIME 类型无效，请重新读取或上传视频") + } + return "", errors.New("文本多模态参考视频需要公网 URL 或 base64 data URL") +} + func shouldFallbackTextToChat(err error) bool { var httpErr providerHTTPError if !errors.As(err, &httpErr) { diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index dd262bc4f..f0f0c94f5 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -117,13 +117,16 @@ func TestDoBinaryRejectsOversizedProviderResponse(t *testing.T) { } } -func TestTextResponseInputIncludesReferenceImages(t *testing.T) { +func TestTextResponseInputIncludesReferenceMedia(t *testing.T) { input := canvasGenerationInput{ Prompt: "describe this image", Config: providerConfig{SystemPrompt: "answer in Chinese"}, ReferenceImages: []providerMedia{ {ID: "image-1", Name: "image.png", Type: "image/png", DataURL: testReferenceImageDataURL}, }, + ReferenceVideos: []providerMedia{ + {ID: "video-1", Name: "video.mp4", Type: "video/mp4", URL: "https://example.com/reference.mp4"}, + }, } value, err := textResponseInput(input) @@ -144,8 +147,8 @@ func TestTextResponseInputIncludesReferenceImages(t *testing.T) { if !ok { t.Fatalf("user content = %T, want []map[string]interface{}", messages[1]["content"]) } - if len(content) != 2 { - t.Fatalf("len(content) = %d, want 2", len(content)) + if len(content) != 3 { + t.Fatalf("len(content) = %d, want 3", len(content)) } if content[0]["type"] != "input_text" || content[0]["text"] != "describe this image" { t.Fatalf("text content = %#v", content[0]) @@ -153,14 +156,20 @@ func TestTextResponseInputIncludesReferenceImages(t *testing.T) { if content[1]["type"] != "input_image" || content[1]["image_url"] != testReferenceImageDataURL { t.Fatalf("image content = %#v", content[1]) } + if content[2]["type"] != "input_video" || content[2]["video_url"] != "https://example.com/reference.mp4" { + t.Fatalf("video content = %#v", content[2]) + } } -func TestTextChatContentIncludesReferenceImages(t *testing.T) { +func TestTextChatContentIncludesReferenceMedia(t *testing.T) { input := canvasGenerationInput{ Prompt: "describe this image", ReferenceImages: []providerMedia{ {ID: "image-1", Name: "image.png", Type: "image/png", DataURL: testReferenceImageDataURL}, }, + ReferenceVideos: []providerMedia{ + {ID: "video-1", Name: "video.mp4", Type: "video/mp4", URL: "https://example.com/reference.mp4"}, + }, } value, err := textChatContent(input) @@ -171,8 +180,8 @@ func TestTextChatContentIncludesReferenceImages(t *testing.T) { if !ok { t.Fatalf("textChatContent() = %T, want []map[string]interface{}", value) } - if len(content) != 2 { - t.Fatalf("len(content) = %d, want 2", len(content)) + if len(content) != 3 { + t.Fatalf("len(content) = %d, want 3", len(content)) } if content[0]["type"] != "text" || content[0]["text"] != "describe this image" { t.Fatalf("text content = %#v", content[0]) @@ -184,6 +193,13 @@ func TestTextChatContentIncludesReferenceImages(t *testing.T) { if content[1]["type"] != "image_url" || imageURL["url"] != testReferenceImageDataURL { t.Fatalf("image content = %#v", content[1]) } + videoURL, ok := content[2]["video_url"].(map[string]interface{}) + if !ok { + t.Fatalf("video_url = %T, want map[string]interface{}", content[2]["video_url"]) + } + if content[2]["type"] != "video_url" || videoURL["url"] != "https://example.com/reference.mp4" { + t.Fatalf("video content = %#v", content[2]) + } } func TestTextReferenceImageRejectsInternalAssetURL(t *testing.T) { diff --git a/web/src/components/asset-media-preview.tsx b/web/src/components/asset-media-preview.tsx new file mode 100644 index 000000000..de1dbbee5 --- /dev/null +++ b/web/src/components/asset-media-preview.tsx @@ -0,0 +1,38 @@ +import type { ReactNode } from "react"; + +import type { Asset } from "@/stores/use-asset-store"; + +type AssetMediaPreviewProps = { + asset?: Asset | null; + alt: string; + className?: string; + fallback?: ReactNode; +}; + +export function AssetMediaPreview({ asset, alt, className = "", fallback = null }: AssetMediaPreviewProps) { + if (!asset) return fallback; + + if (asset.kind === "video" && asset.data.url) { + const poster = asset.coverUrl && asset.coverUrl !== asset.data.url ? asset.coverUrl : undefined; + return ( +