tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(media,canvas): 对象存储参考素材优先 URL，并同步清理图片引用提示词 (#434)

Browse Source 
* fix(media): 对象存储参考素材 - 协议支持 URL 时优先发送签名链接

开启对象存储后，支持 URL 的上游协议优先发送 OSS/COS/七牛/S3 签名链接；必须传字节的协议仍走 Base64 或 multipart。本地资源回退字节，签发失败直接报错，不自动重发生成。

* fix(canvas): 图片引用清理 - 取消引用时同步移除提示词引用
This commit is contained in:
Rou
2026-09-07 22:28:19 +08:00
committed by GitHub
parent 469627c792
commit 21b83eca2e
9 changed files with 339 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
backend/internal/service/provider.go
View File

@@ -371,7 +371,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
}
// 工作流参数由工作流字段定义校验，普通模型能力配置不能覆盖它们。
if resumedProviderRequestID(ctx) == "" {
if err := s.hydrateGenerationMedia(userID, &input, false); err != nil {
if err := s.hydrateGenerationMedia(userID, &input, providerMediaHydrationPolicy{}); err != nil {
return nil, err
}
}
@@ -408,11 +408,8 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
}
}
if resumedProviderRequestID(ctx) == "" {
requirePublicURL := input.Config.InterfaceType == "newapi-channel-1" || input.Config.InterfaceType == "newapi-channel-2" || input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || input.Config.InterfaceType == string(model.ChannelInterfaceMiniMaxVideo)
if adapter, ok := protocolAdapterForContext(ctx, input.Config.InterfaceType); ok {
requirePublicURL = requirePublicURL || adapter.Metadata().RequiresPublicMediaURLs
}
if err := s.hydrateGenerationMedia(userID, &input, requirePublicURL); err != nil {
mediaPolicy := providerMediaHydrationPolicyFor(ctx, input)
if err := s.hydrateGenerationMedia(userID, &input, mediaPolicy); err != nil {
return nil, err
}
if err := s.prepareArkPrivateAssetReferences(ctx, userID, &input); err != nil {
@@ -445,6 +442,51 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
}
}
type providerMediaHydrationPolicy struct {
requireURL bool
preferURL bool
}
func providerMediaHydrationPolicyFor(ctx context.Context, input canvasGenerationInput) providerMediaHydrationPolicy {
policy := providerMediaHydrationPolicy{preferURL: providerPrefersMediaURLs(input.Config.InterfaceType, input)}
switch strings.TrimSpace(input.Config.InterfaceType) {
case string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceMiniMaxVideo):
policy.requireURL = true
policy.preferURL = true
}
if adapter, ok := protocolAdapterForContext(ctx, input.Config.InterfaceType); ok && adapter.Metadata().RequiresPublicMediaURLs {
policy.requireURL = true
policy.preferURL = true
}
if input.Mask != nil {
policy.requireURL = false
policy.preferURL = false
}
return policy
}
// providerPrefersMediaURLs lists protocols whose media fields accept a remote
// URL. Byte-oriented protocols deliberately remain on the existing data path.
func providerPrefersMediaURLs(interfaceType string, input canvasGenerationInput) bool {
if input.Mask != nil {
// OpenAI image edits and similar multipart requests require file bytes.
return false
}
switch strings.TrimSpace(interfaceType) {
case string(model.ChannelInterfaceChatCompletion), string(model.ChannelInterfaceOpenAIResponse), string(model.ChannelInterfaceClaudeAPI),
string(model.ChannelInterfaceGrokImage), string(model.ChannelInterfaceVolcengineArkImage),
string(model.ChannelInterfaceXAIVideo), string(model.ChannelInterfaceNovitaVideo),
string(model.ChannelInterfaceMiniMaxVideo), string(model.ChannelInterfaceNewAPIVideo),
string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2),
string(model.ChannelInterfaceVolcengineArkVideo):
return true
}
if isGrokVideoConfig(input.Config) || isSeedanceVideoConfig(input.Config) || isArkPlanVideoConfig(input.Config) {
return true
}
return false
}
func runAgentToolTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
if adapter, ok := agentProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok {
return runDeclarativeAgentTask(ctx, input, adapter)
@@ -1320,40 +1362,41 @@ func metadataStringValues(value any) map[string]string {
return values
}
func (s *Service) hydrateGenerationMedia(userID string, input *canvasGenerationInput, requirePublicURL bool) error {
func (s *Service) hydrateGenerationMedia(userID string, input *canvasGenerationInput, policy providerMediaHydrationPolicy) error {
groups := [][]providerMedia{input.ReferenceImages, input.ReferenceVideos, input.ReferenceAudios}
for _, group := range groups {
for index := range group {
if err := s.hydrateProviderMedia(userID, &group[index], requirePublicURL); err != nil {
if err := s.hydrateProviderMedia(userID, &group[index], policy); err != nil {
return err
}
}
}
if input.Mask != nil {
return s.hydrateProviderMedia(userID, input.Mask, requirePublicURL)
return s.hydrateProviderMedia(userID, input.Mask, policy)
}
return nil
}
func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requirePublicURL bool) error {
func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, policy providerMediaHydrationPolicy) error {
if !strings.HasPrefix(media.StorageKey, "resource:") {
if requirePublicURL && strings.HasPrefix(strings.TrimSpace(media.DataURL), "data:") {
if policy.requireURL && strings.HasPrefix(strings.TrimSpace(media.DataURL), "data:") {
return errors.New("当前 JSON 视频协议的参考素材不能使用内嵌数据，请先上传到对象存储或提供公网素材地址")
}
return nil
}
resourceID := strings.TrimPrefix(media.StorageKey, "resource:")
if requirePublicURL {
resource, err := s.repo.ResourceForUser(userID, resourceID)
if err != nil {
return fmt.Errorf("读取任务参考资源失败：%w", err)
}
if resource.Status != "ready" {
return errors.New("任务参考资源尚未上传完成")
}
resource, err := s.repo.ResourceForUser(userID, resourceID)
if err != nil {
return fmt.Errorf("读取任务参考资源失败：%w", err)
}
if resource.Status != "ready" {
return errors.New("任务参考资源尚未上传完成")
}
useObjectURL := policy.requireURL || (policy.preferURL && resourceUsesObjectStorage(resource))
if useObjectURL {
signedURL, err := s.directResourceURL(resource, time.Now().Add(providerResourceURLTTL))
if err != nil {
return fmt.Errorf("生成 JSON 视频协议参考素材地址失败：%w", err)
return fmt.Errorf("生成参考素材地址失败：%w", err)
}
media.URL = signedURL
media.DataURL = ""
@@ -1372,17 +1415,17 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requ
return fmt.Errorf("读取任务参考资源失败：%w", err)
}
defer body.Close()
policy, err := s.RuntimePolicy()
runtimePolicy, err := s.RuntimePolicy()
if err != nil {
return err
}
resourceLimit := megabytes(policy.Resource.ResourceUploadMB)
resourceLimit := megabytes(runtimePolicy.Resource.ResourceUploadMB)
data, err := io.ReadAll(io.LimitReader(body, resourceLimit+1))
if err != nil {
return err
}
if int64(len(data)) > resourceLimit {
return fmt.Errorf("任务参考资源超过 %dMB", policy.Resource.ResourceUploadMB)
return fmt.Errorf("任务参考资源超过 %dMB", runtimePolicy.Resource.ResourceUploadMB)
}
mimeType := normalizedMediaMimeType(firstNonEmpty(media.MimeType, resource.MimeType), data)
media.DataURL = dataURL(mimeType, data)
@@ -1394,6 +1437,14 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requ
return nil
}
func resourceUsesObjectStorage(resource *model.Resource) bool {
if resource == nil {
return false
}
provider := strings.ToLower(strings.TrimSpace(resource.Provider))
return provider != "" && provider != "local"
}
func normalizedMediaMimeType(declared string, data []byte) string {
declared = strings.TrimSpace(strings.Split(declared, ";")[0])
if declared != "" && declared != "application/octet-stream" {

28
backend/internal/service/provider_test.go
View File

@@ -23,6 +23,34 @@ import (
const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8="
const testGeminiReferenceImageDataURL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
func TestProviderMediaHydrationPolicyPrefersObjectURLs(t *testing.T) {
prefer := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceGrokImage)}})
if prefer.requireURL || !prefer.preferURL {
t.Fatalf("grok image policy = %#v", prefer)
}
required := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceMiniMaxVideo)}})
if !required.requireURL || !required.preferURL {
t.Fatalf("minimax policy = %#v", required)
}
bytesOnly := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceGeminiImage)}})
if bytesOnly.requireURL || bytesOnly.preferURL {
t.Fatalf("gemini image policy = %#v", bytesOnly)
}
masked := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceOpenAIImage)}, Mask: &providerMedia{ID: "mask"}})
if masked.requireURL || masked.preferURL {
t.Fatalf("openai mask policy = %#v", masked)
}
if resourceUsesObjectStorage(&model.Resource{Provider: "aliyun"}) != true {
t.Fatal("aliyun should use object storage URLs")
}
if resourceUsesObjectStorage(&model.Resource{Provider: "tencent"}) != true || resourceUsesObjectStorage(&model.Resource{Provider: "qiniu"}) != true || resourceUsesObjectStorage(&model.Resource{Provider: "s3"}) != true {
t.Fatal("cos/kodo/s3 should use object storage URLs")
}
if resourceUsesObjectStorage(&model.Resource{Provider: "local"}) {
t.Fatal("local resources should fall back to protocol-compatible bytes")
}
}
func TestProviderRequestErrorDetails(t *testing.T) {
tests := []struct {
name string

52
backend/internal/service/resource_test.go
View File

@@ -11,6 +11,7 @@ import (
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
@@ -735,13 +736,13 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedOSSURL(t *testing.T) {
t.Fatal(err)
}
media := providerMedia{StorageKey: "resource:resource-1", DataURL: "data:image/png;base64,old"}
if err := svc.hydrateProviderMedia("user-1", &media, true); err != nil {
if err := svc.hydrateProviderMedia("user-1", &media, providerMediaHydrationPolicy{requireURL: true}); err != nil {
t.Fatalf("hydrateProviderMedia() error = %v", err)
}
if !strings.HasPrefix(media.URL, "https://private-bucket.oss-cn-test.aliyuncs.com/") || media.DataURL != "" || !strings.Contains(media.URL, "Signature=") {
t.Fatalf("media = %#v", media)
}
if err := svc.hydrateProviderMedia("other-user", &providerMedia{StorageKey: "resource:resource-1"}, true); err == nil {
if err := svc.hydrateProviderMedia("other-user", &providerMedia{StorageKey: "resource:resource-1"}, providerMediaHydrationPolicy{requireURL: true}); err == nil {
t.Fatal("hydrateProviderMedia() allowed another user's resource")
}
}
@@ -760,7 +761,7 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedLocalURL(t *testing.T) {
t.Fatal(err)
}
media := providerMedia{StorageKey: "resource:resource-local"}
if err := svc.hydrateProviderMedia("user-1", &media, true); err != nil {
if err := svc.hydrateProviderMedia("user-1", &media, providerMediaHydrationPolicy{requireURL: true}); err != nil {
t.Fatalf("hydrateProviderMedia() error = %v", err)
}
if !strings.HasPrefix(media.URL, server.URL+"/api/public/resources/resource-local/file/resource-local.png?") || !strings.Contains(media.URL, "signature=") || media.DataURL != "" {
@@ -772,6 +773,51 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedLocalURL(t *testing.T) {
}
}
func TestHydratePreferredURLUsesObjectStorageAndFallsBackLocal(t *testing.T) {
svc := newResourceTestService(t)
settingJSON, _ := json.Marshal(ossSettingValue{
Enabled: true, Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
AccessKeyID: "access-id", AccessKeySecret: "secret-value",
})
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
t.Fatal(err)
}
objectResource := model.Resource{
ID: "resource-oss", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady,
Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
ObjectKey: "users/user-1/image/prefer.png", MimeType: "image/png",
}
if err := svc.repo.CreateResource(&objectResource); err != nil {
t.Fatal(err)
}
media := providerMedia{StorageKey: "resource:resource-oss", DataURL: "data:image/png;base64,old"}
if err := svc.hydrateProviderMedia("user-1", &media, providerMediaHydrationPolicy{preferURL: true}); err != nil {
t.Fatalf("hydrateProviderMedia(prefer object) error = %v", err)
}
if !strings.HasPrefix(media.URL, "https://private-bucket.oss-cn-test.aliyuncs.com/") || media.DataURL != "" || !strings.Contains(media.URL, "Signature=") {
t.Fatalf("object media = %#v", media)
}
localDir := filepath.Join(svc.dataDir, "resources", "users", "user-1", "image")
if err := os.MkdirAll(localDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(localDir, "local.png"), []byte("png-bytes"), 0o644); err != nil {
t.Fatal(err)
}
localResource := model.Resource{ID: "resource-local-bytes", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, Provider: "local", ObjectKey: "users/user-1/image/local.png", MimeType: "image/png"}
if err := svc.repo.CreateResource(&localResource); err != nil {
t.Fatal(err)
}
localMedia := providerMedia{StorageKey: "resource:resource-local-bytes"}
if err := svc.hydrateProviderMedia("user-1", &localMedia, providerMediaHydrationPolicy{preferURL: true}); err != nil {
t.Fatalf("hydrateProviderMedia(prefer local) error = %v", err)
}
if localMedia.URL != "" || !strings.HasPrefix(localMedia.DataURL, "data:image/png;base64,") {
t.Fatalf("local media = %#v", localMedia)
}
}
func TestPublicResourceSignatureRejectsExpiredAndAlteredLinks(t *testing.T) {
svc := newResourceTestService(t)
expires := strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10)