From 487c34bfeb8dd936763f8aec73ff76e0f3134d34 Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Tue, 22 Sep 2026 21:41:00 +0800 Subject: [PATCH] =?UTF-8?q?refactor(media):=20=E8=B5=84=E6=BA=90=E5=88=86?= =?UTF-8?q?=E5=8F=91=20-=20=E7=BB=9F=E4=B8=80=E8=B5=84=E6=BA=90=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=E7=AD=96=E7=95=A5=E4=B8=8E=E4=BA=91=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E9=87=8D=E5=AE=9A=E5=90=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/app/admin_storage.go | 19 + backend/internal/app/analytics.go | 15 +- backend/internal/app/announcement.go | 34 + backend/internal/app/appearance.go | 26 + backend/internal/app/canvas_bridge.go | 8 +- backend/internal/app/provider.go | 2 +- .../app/provider_ark_private_assets.go | 2 +- .../internal/app/provider_resource_helpers.go | 7 +- backend/internal/app/resource.go | 675 +----------------- backend/internal/app/resource_access.go | 162 +++++ backend/internal/app/resource_delete.go | 58 -- backend/internal/app/resource_test.go | 114 +-- backend/internal/app/resource_url_test.go | 2 +- backend/internal/app/settings.go | 113 +-- backend/internal/app/storage_adapters.go | 45 ++ backend/internal/app/storage_location.go | 10 +- backend/internal/app/storage_location_test.go | 12 +- backend/internal/app/storage_s3_test.go | 6 +- backend/internal/assets/access.go | 155 ++++ backend/internal/assets/access_test.go | 143 ++++ backend/internal/assets/types.go | 11 +- backend/internal/assets/urls_test.go | 2 +- backend/internal/canvas/canvas_share.go | 30 +- backend/internal/canvas/host.go | 4 +- backend/internal/generation/deps.go | 3 - backend/internal/handler/admin_storage.go | 18 +- backend/internal/handler/announcement.go | 16 +- backend/internal/handler/appearance.go | 27 +- backend/internal/handler/auth.go | 41 +- backend/internal/handler/canvas_share.go | 45 +- backend/internal/handler/openapi.yaml | 37 + backend/internal/handler/resource_delivery.go | 88 +++ .../handler/resource_delivery_test.go | 96 +++ backend/internal/handler/user_data.go | 158 +--- backend/internal/service/aliases_types.go | 4 +- backend/internal/storage/delivery.go | 49 ++ backend/internal/storage/objects.go | 519 ++++++++++++++ .../{app/storage_s3.go => storage/s3.go} | 50 +- backend/internal/storage/settings.go | 85 +++ docs/content/docs/backend/http-api.mdx | 12 + docs/design/resource-delivery-architecture.md | 253 +++++++ docs/index.md | 2 + .../layout/user-oss-settings-form.tsx | 25 +- .../admin/settings/storage-settings-page.tsx | 43 +- web/src/pages/canvas/project.tsx | 27 +- web/src/services/api/resources.ts | 103 ++- web/src/services/api/video-provider-agnes.ts | 4 +- web/src/services/api/video-provider-newapi.ts | 4 +- .../services/api/video-provider-seedance.ts | 4 +- web/src/services/resource-blob-cache.ts | 10 +- 50 files changed, 2112 insertions(+), 1266 deletions(-) create mode 100644 backend/internal/app/resource_access.go create mode 100644 backend/internal/app/storage_adapters.go create mode 100644 backend/internal/assets/access.go create mode 100644 backend/internal/assets/access_test.go create mode 100644 backend/internal/handler/resource_delivery.go create mode 100644 backend/internal/handler/resource_delivery_test.go create mode 100644 backend/internal/storage/delivery.go create mode 100644 backend/internal/storage/objects.go rename backend/internal/{app/storage_s3.go => storage/s3.go} (66%) create mode 100644 backend/internal/storage/settings.go create mode 100644 docs/design/resource-delivery-architecture.md diff --git a/backend/internal/app/admin_storage.go b/backend/internal/app/admin_storage.go index 54d13c7e9..da343e0b2 100644 --- a/backend/internal/app/admin_storage.go +++ b/backend/internal/app/admin_storage.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" @@ -127,6 +128,24 @@ func (s *Service) OpenResourceRangeAsAdmin(actor *model.User, id string, rangeHe return s.openResourceRange(resource.UserID, resource, rangeHeader) } +func (s *Service) PrepareResourceDeliveryAsAdmin(actor *model.User, id string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + if err := s.RequireAdmin(actor); err != nil { + return nil, err + } + resource, err := s.repo.Resource(strings.TrimSpace(id)) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, NotFound("资源不存在") + } + return nil, err + } + resource.Provider = normalizedResourceProvider(resource.Provider) + if options.Purpose == "" { + options.Purpose = assets.PurposeDisplay + } + return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader) +} + func normalizeAdminResourceQuery(query AdminResourceQuery) (repository.AdminResourceFilter, int, int, error) { page, limit := normalizeAdminPage(query.Page, query.Limit) filter := repository.AdminResourceFilter{ diff --git a/backend/internal/app/analytics.go b/backend/internal/app/analytics.go index 1b93c091b..f984b23a5 100644 --- a/backend/internal/app/analytics.go +++ b/backend/internal/app/analytics.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" @@ -326,21 +327,15 @@ func (s *Service) OpenAdminAPICallLogMediaRange(actor *model.User, logID string, return s.openResourceRange(userID, resource, rangeHeader) } -func (s *Service) PrepareAdminAPICallLogMediaDelivery(actor *model.User, logID string, rangeHeader string) (*ResourceDelivery, error) { +func (s *Service) PrepareAdminAPICallLogMediaDelivery(actor *model.User, logID string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { userID, resource, err := s.adminAPICallLogMediaResource(actor, logID) if err != nil { return nil, err } - delivery, err := s.prepareResourceDelivery(userID, resource, ResourceDeliveryOptions{}) - if err != nil || delivery.RedirectURL != "" { - return delivery, err + if options.Purpose == "" { + options.Purpose = assets.PurposeDisplay } - stream, err := s.openResourceRange(userID, resource, rangeHeader) - if err != nil { - return nil, err - } - delivery.Stream = stream - return delivery, nil + return s.prepareResourceDelivery(userID, resource, options, rangeHeader) } func (s *Service) adminAPICallLogMediaResource(actor *model.User, logID string) (string, *model.Resource, error) { diff --git a/backend/internal/app/announcement.go b/backend/internal/app/announcement.go index 7d09df7cc..d135ded2a 100644 --- a/backend/internal/app/announcement.go +++ b/backend/internal/app/announcement.go @@ -10,6 +10,7 @@ import ( "time" "unicode/utf8" + "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" @@ -269,6 +270,39 @@ func (s *Service) OpenAnnouncementImage(actor *model.User, announcementID string return s.openResourceRange(resource.UserID, resource, rangeHeader) } +func (s *Service) PrepareAnnouncementImageDelivery(actor *model.User, announcementID string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + if actor == nil { + return nil, Unauthorized("请先登录") + } + announcement, err := s.repo.Announcement(strings.TrimSpace(announcementID)) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, NotFound("公告不存在") + } + return nil, err + } + if actor.Role != model.UserRoleAdmin && announcement.Status != model.AnnouncementStatusActive { + return nil, Forbidden("公告不可访问") + } + if announcement.ImageResourceID == "" { + return nil, NotFound("公告配图不存在") + } + resource, err := s.repo.Resource(announcement.ImageResourceID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, NotFound("公告配图不存在") + } + return nil, err + } + if resource.Kind != "image" || resource.Status != model.ResourceStatusReady { + return nil, BadAuthRequest("公告配图资源不可用") + } + if options.Purpose == "" { + options.Purpose = assets.PurposeDisplay + } + return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader) +} + func (s *Service) cleanupStaleAnnouncementImageDrafts() { for { drafts, err := s.repo.StaleAnnouncementImageDrafts(time.Now().Add(-announcementImageDraftTTL), 50) diff --git a/backend/internal/app/appearance.go b/backend/internal/app/appearance.go index 25a54397e..840f6bdba 100644 --- a/backend/internal/app/appearance.go +++ b/backend/internal/app/appearance.go @@ -16,6 +16,7 @@ import ( "unicode" "unicode/utf8" + "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" "gorm.io/gorm" @@ -317,6 +318,31 @@ func (s *Service) OpenAppearanceAsset(slot string, rangeHeader string) (*Resourc return s.openResourceRange(resource.UserID, resource, rangeHeader) } +func (s *Service) PrepareAppearanceAssetDelivery(slot string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + _, value, err := s.readAppearance() + if err != nil { + return nil, err + } + resourceID := appearanceResourceID(value, slot) + if resourceID == "" { + return nil, NotFound("未配置该外观资源") + } + resource, err := s.repo.Resource(resourceID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, NotFound("外观资源不存在") + } + return nil, err + } + if err := validateAppearanceResourceType(slot, resource); err != nil { + return nil, err + } + if options.Purpose == "" { + options.Purpose = assets.PurposeDisplay + } + return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader) +} + func (s *Service) appearanceResourceReferences(resourceIDs []string) map[string][]AdminResourceReferenceView { result := make(map[string][]AdminResourceReferenceView) _, value, err := s.readAppearance() diff --git a/backend/internal/app/canvas_bridge.go b/backend/internal/app/canvas_bridge.go index d81d97fed..0dea32007 100644 --- a/backend/internal/app/canvas_bridge.go +++ b/backend/internal/app/canvas_bridge.go @@ -49,11 +49,11 @@ func (h canvasHost) OpenResourceRange(userID string, resource *model.Resource, r return h.svc.openResourceRange(userID, resource, rangeHeader) } -func (h canvasHost) PrepareResourceDelivery(userID string, resource *model.Resource, options assets.ResourceDeliveryOptions) (*assets.ResourceDelivery, error) { +func (h canvasHost) PrepareResourceDelivery(userID string, resource *model.Resource, options assets.AccessOptions, rangeHeader string) (*assets.ResourceDelivery, error) { if h.svc == nil { return nil, nil } - return h.svc.prepareResourceDelivery(userID, resource, options) + return h.svc.prepareResourceDelivery(userID, resource, options, rangeHeader) } func (h canvasHost) WithStorageLock(fn func() error) error { @@ -150,8 +150,8 @@ func (s *Service) OpenSharedCanvasResourceRange(token string, resourceID string, return s.canvasDomain().OpenSharedCanvasResourceRange(token, resourceID, rangeHeader) } -func (s *Service) PrepareSharedCanvasResourceDelivery(token string, resourceID string, rangeHeader string) (*ResourceDelivery, error) { - return s.canvasDomain().PrepareSharedCanvasResourceDelivery(token, resourceID, rangeHeader) +func (s *Service) PrepareSharedCanvasResourceDelivery(token string, resourceID string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + return s.canvasDomain().PrepareSharedCanvasResourceDelivery(token, resourceID, options, rangeHeader) } func (s *Service) validateCanvasMediaAssets(userID string, raw json.RawMessage) error { diff --git a/backend/internal/app/provider.go b/backend/internal/app/provider.go index ee69ff5d8..b98376cdb 100644 --- a/backend/internal/app/provider.go +++ b/backend/internal/app/provider.go @@ -805,7 +805,7 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, poli } useObjectURL := policy.requireURL || (policy.preferURL && resourceUsesObjectStorage(resource)) if useObjectURL { - signedURL, err := s.directResourceURL(resource, time.Now().Add(providerResourceURLTTL)) + signedURL, err := s.providerResourceURL(resource, time.Now().Add(providerResourceURLTTL)) if err != nil { return fmt.Errorf("生成参考素材地址失败：%w", err) } diff --git a/backend/internal/app/provider_ark_private_assets.go b/backend/internal/app/provider_ark_private_assets.go index 461bfbb47..361624f35 100644 --- a/backend/internal/app/provider_ark_private_assets.go +++ b/backend/internal/app/provider_ark_private_assets.go @@ -230,7 +230,7 @@ func (s *Service) ensureArkPrivateAsset(ctx context.Context, userID string, reso } } binding.AssetGroupID = groupID - resourceURL, err := s.directResourceURL(resource, time.Now().Add(time.Hour)) + resourceURL, err := s.providerResourceURL(resource, time.Now().Add(time.Hour)) if err != nil { return "", s.failArkPrivateAssetBinding(binding, fmt.Errorf("生成方舟素材临时地址失败：%w", err)) } diff --git a/backend/internal/app/provider_resource_helpers.go b/backend/internal/app/provider_resource_helpers.go index 1038605c0..e9c0f6812 100644 --- a/backend/internal/app/provider_resource_helpers.go +++ b/backend/internal/app/provider_resource_helpers.go @@ -4,6 +4,7 @@ import ( "errors" "time" + "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" ) @@ -18,5 +19,9 @@ func (s *Service) ResourceForUser(actor *model.User, id string) (*model.Resource // providerResourceURL 只能接收已经完成用户归属和 ready 状态校验的资源， // 并为上游签发短时地址；它不是绕过权限校验的通用资源 URL 生成器。 func (s *Service) providerResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) { - return s.directResourceURL(resource, expiresAt) + access, err := s.resolveResourceAccess(resource, ResourceAccessOptions{Purpose: assets.PurposeProvider, ExpiresAt: expiresAt}) + if err != nil { + return "", err + } + return access.URL, nil } diff --git a/backend/internal/app/resource.go b/backend/internal/app/resource.go index eecf6e9e8..2f319f470 100644 --- a/backend/internal/app/resource.go +++ b/backend/internal/app/resource.go @@ -2,9 +2,7 @@ package app import ( "bytes" - "context" "crypto/hmac" - "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -17,7 +15,6 @@ import ( _ "image/png" "infinite-canvas/backend/internal/kernel" "io" - "log" "mime" "mime/multipart" "net/http" @@ -32,21 +29,14 @@ import ( "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" - "github.com/aws/aws-sdk-go/aws/credentials" - awsv4 "github.com/aws/aws-sdk-go/aws/signer/v4" - qiniuAuth "github.com/qiniu/go-sdk/v7/auth" - qiniuStorage "github.com/qiniu/go-sdk/v7/storage" - cos "github.com/tencentyun/cos-go-sdk-v5" "gorm.io/gorm" ) const providerResourceURLTTL = 4 * time.Hour -const directResourceURLTTL = 5 * time.Minute var errInvalidGeneratedDataURL = errors.New("生成内容 data URL 无效") type ResourceStream = assets.ResourceStream -type ResourceDeliveryOptions = assets.ResourceDeliveryOptions type ResourceDelivery = assets.ResourceDelivery func (s *Service) Resources(userID string, limit int) ([]model.Resource, error) { @@ -65,141 +55,6 @@ func (s *Service) Resource(userID string, id string) (*model.Resource, error) { return resource, err } -// DirectResourceURL 先校验资源归属，再按实际存储位置签发短时下载地址。 -func (s *Service) DirectResourceURL(userID string, id string) (string, error) { - resource, err := s.repo.ResourceForUser(userID, id) - if err != nil { - return "", err - } - return s.directResourceURL(resource, time.Now().Add(directResourceURLTTL)) -} - -func (s *Service) directResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) { - if resource == nil { - return "", errors.New("资源不存在") - } - if resource.Status != model.ResourceStatusReady { - return "", BadAuthRequest("资源尚未上传完成") - } - if resource.Provider == "local" { - return s.signedPublicResourceURL(resource, expiresAt) - } - setting, err := s.ossSettingForResource(resource.UserID, resource) - if err != nil { - return "", err - } - setting.Provider = firstNonEmpty(resource.Provider, setting.Provider) - setting.Endpoint = firstNonEmpty(resource.Endpoint, setting.Endpoint) - setting.Bucket = firstNonEmpty(resource.Bucket, setting.Bucket) - if setting.Provider == s3Provider && !publicHTTPSStorageEndpoint(setting.Endpoint) { - return s.signedHTTPSPublicResourceURL(resource, expiresAt) - } - return signedOSSObjectURL(setting, resource.ObjectKey, expiresAt) -} - -// PrepareResourceDelivery 统一决定浏览器资源出口：配置 CDN 时默认直连 CDN，显式代理仅用于需要同源 Blob 的内部读取。 -func (s *Service) PrepareResourceDelivery(userID string, id string, options ResourceDeliveryOptions) (*ResourceDelivery, error) { - resource, err := s.repo.ResourceForUser(userID, id) - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, NotFound("资源不存在") - } - return nil, err - } - return s.prepareResourceDelivery(userID, resource, options) -} - -func (s *Service) prepareResourceDelivery(userID string, resource *model.Resource, options ResourceDeliveryOptions) (*ResourceDelivery, error) { - if resource == nil { - return nil, errors.New("资源不存在") - } - if resource.Status != model.ResourceStatusReady { - return nil, BadAuthRequest("资源尚未上传完成") - } - if resource.Provider != "local" && !options.ForceProxy { - setting, err := s.ossSettingForResource(userID, resource) - if err != nil { - return nil, err - } - // S3 兼容 Endpoint 可能是私网服务；浏览器默认始终使用同源代理。 - // 只有明确的服务端上游需求才签发可公开访问的短时地址。 - if setting.Provider == s3Provider && !options.ForceDirect { - return &ResourceDelivery{Resource: resource}, nil - } - if setting.Provider == qiniuKodoProvider && setting.CDNBaseURL != "" { - // 七牛私有空间即使配置了绑定域名，也不能匿名访问；必须使用 - // Kodo 私有下载签名，否则浏览器会收到 NotSupportAnonymous。 - redirectURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(directResourceURLTTL)) - if err != nil { - return nil, err - } - return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil - } - if setting.CDNBaseURL != "" { - redirectURL, err := ossCDNObjectURL(setting.CDNBaseURL, resource.ObjectKey) - if err != nil { - return nil, err - } - return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil - } - if options.ForceDirect { - if setting.Provider == s3Provider && !publicHTTPSStorageEndpoint(setting.Endpoint) { - redirectURL, err := s.signedHTTPSPublicResourceURL(resource, time.Now().Add(directResourceURLTTL)) - if err != nil { - return nil, err - } - return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil - } - redirectURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(directResourceURLTTL)) - if err != nil { - return nil, err - } - return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil - } - } - return &ResourceDelivery{Resource: resource}, nil -} - -func (s *Service) signedPublicResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) { - if resource == nil { - return "", errors.New("资源不存在") - } - baseURL, err := s.publicResourceBaseURL() - if err != nil { - return "", err - } - expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10) - signature, err := s.signPublicResource(resource.ID, expires) - if err != nil { - return "", err - } - ext := resourceFileExtension(resource.ObjectKey, resource.MimeType, resource.Kind) - filename := resource.ID - if ext != "" { - if !strings.HasPrefix(ext, ".") { - ext = "." + ext - } - filename += ext - } - baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/api/public/resources/" + url.PathEscape(resource.ID) + "/file/" + url.PathEscape(filename) - query := baseURL.Query() - query.Set("expires", expires) - query.Set("signature", signature) - baseURL.RawQuery = query.Encode() - return baseURL.String(), nil -} - -func (s *Service) signedHTTPSPublicResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) { - baseURL, err := s.publicResourceBaseURL() - if err != nil { - return "", err - } - if baseURL.Scheme != "https" { - return "", BadAuthRequest("私网或 HTTP S3 用于上游资源时，服务器公开访问地址必须使用 HTTPS") - } - return s.signedPublicResourceURL(resource, expiresAt) -} - func (s *Service) verifyPublicResourceSignature(resourceID string, expires string, signature string) error { if strings.TrimSpace(signature) == "" || !decimalDigits(expires) { return Forbidden("匿名下载链接无效") @@ -448,20 +303,6 @@ func (s *Service) OpenResourceRange(userID string, id string, rangeHeader string return s.openResourceRange(userID, resource, rangeHeader) } -func (s *Service) OpenPublicResourceRange(id string, expires string, signature string, rangeHeader string) (*ResourceStream, error) { - resource, err := s.repo.Resource(id) - if err != nil { - return nil, Forbidden("匿名下载链接无效") - } - if resource.Provider != "local" && resource.Provider != s3Provider { - return nil, Forbidden("匿名下载链接无效") - } - if err := s.verifyPublicResourceSignature(resource.ID, expires, signature); err != nil { - return nil, err - } - return s.openResourceRange(resource.UserID, resource, rangeHeader) -} - func (s *Service) openResourceRange(userID string, resource *model.Resource, rangeHeader string) (*ResourceStream, error) { if resource.Status != model.ResourceStatusReady { return nil, BadAuthRequest("资源尚未上传完成") @@ -487,7 +328,7 @@ func (s *Service) openResourceRange(userID string, resource *model.Resource, ran if err != nil { return nil, err } - return &ResourceStream{Resource: resource, Body: stream.body, StatusCode: stream.statusCode, ContentLength: stream.contentLength, ContentRange: stream.contentRange, AcceptRanges: stream.acceptRanges}, nil + return &ResourceStream{Resource: resource, Body: stream.Body, StatusCode: stream.StatusCode, ContentLength: stream.ContentLength, ContentRange: stream.ContentRange, AcceptRanges: stream.AcceptRanges}, nil } func (s *Service) storeResource(userID string, kind string, fileName string, mimeType string, size int64, width int, height int, durationMs int64, body io.Reader, uploadKey *string, forceLocal bool) (*model.Resource, bool, error) { @@ -590,9 +431,7 @@ func writeLocalResourceObject(filePath string, body io.Reader) error { return closeErr } -// storeResourceObject 写入资源物理对象。对象存储不可用（配置错误、密钥失效、网络 -// 故障、设置被删）时自动降级为本地存储并同步改写资源记录，保证上传写路径不因外部 -// 存储故障整体失败。对象存储失败后 body 会被重新读取，须支持 Seek。 +// A cloud write is successful only after the configured origin accepts it. func (s *Service) storeResourceObject(resource *model.Resource, fileName string, body io.Reader) (string, error) { if resource == nil { return "", errors.New("资源不存在") @@ -600,36 +439,11 @@ func (s *Service) storeResourceObject(resource *model.Resource, fileName string, if resource.Provider == "local" { return "", writeLocalResourceObject(filepath.Join(s.dataDir, "resources", filepath.FromSlash(resource.ObjectKey)), body) } - setting, settingErr := s.ossSettingForResource(resource.UserID, resource) - var etag string - var putErr error - if settingErr == nil { - etag, putErr = putOSSObject(setting, resource.ObjectKey, resource.MimeType, resource.Size, body) + setting, err := s.ossSettingForResource(resource.UserID, resource) + if err != nil { + return "", err } - if putErr == nil && settingErr == nil { - return etag, nil - } - fallbackErr := putErr - if fallbackErr == nil { - fallbackErr = settingErr - } - if seeker, ok := body.(io.Seeker); ok { - if _, seekErr := seeker.Seek(0, io.SeekStart); seekErr != nil { - return "", errors.Join(fallbackErr, fmt.Errorf("降级本地存储时重置读取位置失败：%w", seekErr)) - } - } - localKey := localObjectKey(resource.UserID, resource.Kind, fileName, resource.MimeType, time.Now()) - resource.Provider = "local" - resource.ObjectKey = localKey - resource.Endpoint = "" - resource.Bucket = "" - resource.StorageSettingID = "" - resource.ETag = "" - if localErr := writeLocalResourceObject(filepath.Join(s.dataDir, "resources", filepath.FromSlash(localKey)), body); localErr != nil { - return "", errors.Join(fallbackErr, fmt.Errorf("降级本地存储失败：%w", localErr)) - } - log.Printf("object storage upload degraded to local storage: resource=%s error=%v", resource.ID, fallbackErr) - return "", nil + return putOSSObject(setting, resource.ObjectKey, resource.MimeType, resource.Size, body) } func (s *Service) retryStoredResource(userID string, resource *model.Resource, kind string, mimeType string, size int64, body io.Reader) (*model.Resource, error) { @@ -1015,6 +829,10 @@ func (s *Service) ossSettingForResource(userID string, resource *model.Resource) // 密钥固定在资源绑定的历史版本；只有存储位置完全一致时，才允许沿用当前 CDN。 if resourceStorageMatches(current, resource) { setting.CDNBaseURL = current.CDNBaseURL + // CDN 的鉴权方式和回退策略属于分发配置，而不是资源创建时的 + // 凭据。存储位置未变时沿用当前策略，避免历史资源因管理员 + // 刚补齐 CDN 鉴权配置而继续回源。 + setting.Delivery = current.Delivery } } } else { @@ -1129,89 +947,6 @@ func ossObjectKey(setting ossSettingValue, userID string, kind string, fileName return strings.Trim(strings.Join(nonEmptySegments(parts), "/"), "/") } -func putOSSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { - setting = normalizeOSSSetting(setting) - if setting.Provider == tencentCOSProvider { - return putCOSObject(setting, objectKey, mimeType, size, body) - } - if setting.Provider == qiniuKodoProvider { - return putQiniuObject(setting, objectKey, mimeType, size, body) - } - if setting.Provider == s3Provider { - return putS3Object(setting, objectKey, mimeType, size, body) - } - return putAliyunOSSObject(setting, objectKey, mimeType, size, body) -} - -// 阿里云 OSS 继续沿用原有 V1 签名和请求路径，避免已有部署行为发生变化。 -func putAliyunOSSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { - if mimeType == "" { - mimeType = "application/octet-stream" - } - req, err := newOSSRequest(http.MethodPut, setting, objectKey, mimeType, body) - if err != nil { - return "", err - } - if size > 0 { - req.ContentLength = size - } - resp, err := OutboundHTTPClient(2 * time.Minute).Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return "", fmt.Errorf("OSS 上传失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) - } - return strings.Trim(resp.Header.Get("ETag"), `"`), nil -} - -type ossObjectStream struct { - body io.ReadCloser - statusCode int - contentLength int64 - contentRange string - acceptRanges string -} - -func getOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { - setting = normalizeOSSSetting(setting) - if setting.Provider == s3Provider { - return getS3ObjectRange(setting, objectKey, rangeHeader) - } - if setting.CDNBaseURL != "" { - return getOSSObjectRangeViaCDN(setting, objectKey, rangeHeader) - } - if setting.Provider == tencentCOSProvider { - return getCOSObjectRange(setting, objectKey, rangeHeader) - } - if setting.Provider == qiniuKodoProvider { - return getQiniuObjectRange(setting, objectKey, rangeHeader) - } - return getAliyunOSSObjectRange(setting, objectKey, rangeHeader) -} - -func getAliyunOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { - req, err := newOSSRequest(http.MethodGet, setting, objectKey, "", nil) - if err != nil { - return nil, err - } - if rangeHeader != "" { - req.Header.Set("Range", rangeHeader) - } - resp, err := OutboundHTTPClient(2 * time.Minute).Do(req) - if err != nil { - return nil, err - } - if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { - defer resp.Body.Close() - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return nil, fmt.Errorf("OSS 读取失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) - } - return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil -} - func normalizeSingleByteRange(value string) string { value = strings.TrimSpace(value) if len(value) > 128 || !strings.HasPrefix(value, "bytes=") || strings.Contains(value, ",") { @@ -1233,396 +968,6 @@ func decimalDigits(value string) bool { return true } -func signedOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { - setting = normalizeOSSSetting(setting) - if setting.Provider == s3Provider { - return signedS3ObjectURL(setting, objectKey, expiresAt) - } - if setting.Provider == qiniuKodoProvider { - return signedQiniuObjectURL(setting, objectKey, expiresAt) - } - if setting.CDNBaseURL != "" { - return ossCDNObjectURL(setting.CDNBaseURL, objectKey) - } - if setting.Provider == tencentCOSProvider { - return signedCOSObjectURL(setting, objectKey, expiresAt) - } - return signedAliyunOSSObjectURL(setting, objectKey, expiresAt) -} - -func signedAliyunOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { - baseURL, err := ossBucketBaseURL(setting) - if err != nil { - return "", err - } - if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" { - return "", errors.New("OSS 访问密钥不可用") - } - objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") - if objectKey == "" { - return "", errors.New("OSS 对象路径为空") - } - baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/" + escapeObjectKey(objectKey) - expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10) - stringToSign := strings.Join([]string{http.MethodGet, "", "", expires, "/" + setting.Bucket + "/" + objectKey}, "\n") - mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret)) - _, _ = mac.Write([]byte(stringToSign)) - query := baseURL.Query() - query.Set("OSSAccessKeyId", setting.AccessKeyID) - query.Set("Expires", expires) - query.Set("Signature", base64.StdEncoding.EncodeToString(mac.Sum(nil))) - baseURL.RawQuery = query.Encode() - return baseURL.String(), nil -} - -func putCOSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { - client, err := newCOSClient(setting, 2*time.Minute) - if err != nil { - return "", err - } - if mimeType == "" { - mimeType = "application/octet-stream" - } - options := &cos.ObjectPutOptions{ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ContentType: mimeType, ContentLength: size}} - resp, err := client.Object.Put(context.Background(), objectKey, body, options) - if err != nil { - return "", fmt.Errorf("COS 上传失败：%w", err) - } - return strings.Trim(resp.Header.Get("ETag"), `"`), nil -} - -func putQiniuObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { - if setting.AccessKeyID == "" || setting.AccessKeySecret == "" { - return "", errors.New("七牛云 Kodo 访问密钥不可用") - } - if setting.Bucket == "" || objectKey == "" { - return "", errors.New("七牛云 Kodo Bucket 或对象路径为空") - } - config := qiniuStorage.NewConfig() - config.Region = qiniuRegion(setting.Region) - config.UpHost = strings.TrimRight(setting.Endpoint, "/") - uploader := qiniuStorage.NewFormUploader(config) - policy := qiniuStorage.PutPolicy{Scope: setting.Bucket + ":" + strings.TrimLeft(objectKey, "/"), Expires: 3600} - token := policy.UploadToken(qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret)) - ret := qiniuStorage.PutRet{} - extra := &qiniuStorage.PutExtra{MimeType: mimeType} - if size < 0 { - size = 0 - } - if err := uploader.Put(context.Background(), &ret, token, strings.TrimLeft(objectKey, "/"), body, size, extra); err != nil { - return "", fmt.Errorf("七牛云 Kodo 上传失败：%w", err) - } - return ret.Hash, nil -} - -func getCOSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { - client, err := newCOSClient(setting, 2*time.Minute) - if err != nil { - return nil, err - } - options := &cos.ObjectGetOptions{Range: rangeHeader} - resp, err := client.Object.Get(context.Background(), objectKey, options) - if err != nil { - if resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { - return &ossObjectStream{body: io.NopCloser(bytes.NewReader(nil)), statusCode: resp.StatusCode, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil - } - return nil, fmt.Errorf("COS 读取失败：%w", err) - } - return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil -} - -func getQiniuObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { - signedURL, err := signedQiniuObjectURL(setting, objectKey, time.Now().Add(directResourceURLTTL)) - if err != nil { - return nil, err - } - req, err := http.NewRequest(http.MethodGet, signedURL, nil) - if err != nil { - return nil, err - } - if rangeHeader != "" { - req.Header.Set("Range", rangeHeader) - } - ApplyDefaultOutboundHeaders(req) - resp, err := OutboundHTTPClient(2 * time.Minute).Do(req) - if err != nil { - return nil, fmt.Errorf("七牛云 Kodo 读取失败：%w", err) - } - if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { - defer resp.Body.Close() - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return nil, fmt.Errorf("七牛云 Kodo 读取失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) - } - return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil -} - -func getOSSObjectRangeViaCDN(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { - signedURL, err := signedOSSObjectURL(setting, objectKey, time.Now().Add(directResourceURLTTL)) - if err != nil { - return nil, err - } - req, err := http.NewRequest(http.MethodGet, signedURL, nil) - if err != nil { - return nil, err - } - if rangeHeader != "" { - req.Header.Set("Range", rangeHeader) - } - ApplyDefaultOutboundHeaders(req) - resp, err := OutboundHTTPClient(2 * time.Minute).Do(req) - if err != nil { - return nil, fmt.Errorf("对象存储 CDN 读取失败：%w", err) - } - if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { - defer resp.Body.Close() - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return nil, fmt.Errorf("对象存储 CDN 读取失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) - } - return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil -} - -func signedCOSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { - if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" { - return "", errors.New("COS 访问密钥不可用") - } - objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") - if objectKey == "" { - return "", errors.New("COS 对象路径为空") - } - expires := time.Until(expiresAt) - if expires <= 0 { - return "", errors.New("COS 签名有效期必须晚于当前时间") - } - client, err := newCOSClient(setting, 2*time.Minute) - if err != nil { - return "", err - } - signedURL, err := client.Object.GetPresignedURL(context.Background(), http.MethodGet, objectKey, setting.AccessKeyID, setting.AccessKeySecret, expires, nil) - if err != nil { - return "", err - } - return signedURL.String(), nil -} - -func signedQiniuObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { - if setting.AccessKeyID == "" || setting.AccessKeySecret == "" { - return "", errors.New("七牛云 Kodo 访问密钥不可用") - } - objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") - if objectKey == "" { - return "", errors.New("七牛云 Kodo 对象路径为空") - } - deadline := expiresAt.Unix() - if deadline <= time.Now().Unix() { - return "", errors.New("七牛云 Kodo 签名有效期必须晚于当前时间") - } - if setting.CDNBaseURL == "" { - return signedQiniuS3ObjectURL(setting, objectKey, expiresAt) - } - mac := qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret) - return qiniuStorage.MakePrivateURLv2(mac, strings.TrimRight(setting.CDNBaseURL, "/"), objectKey, deadline), nil -} - -// signedQiniuS3ObjectURL 用七牛兼容 S3 的 AWS Signature V4 访问私有空间。 -// 没有绑定域名时，浏览器不直接访问该地址，而是由后端代理读取并返回文件。 -func signedQiniuS3ObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { - region := qiniuS3Region(setting) - if region == "" { - return "", errors.New("七牛云 Kodo S3 Region 不可用") - } - baseURL := &url.URL{Scheme: "https", Host: setting.Bucket + ".s3." + region + ".qiniucs.com"} - // 保留对象键的原始路径，让 url.URL 和 AWS signer 只做一次 RFC 3986 转义。 - baseURL.Path = "/" + objectKey - req, err := http.NewRequest(http.MethodGet, baseURL.String(), nil) - if err != nil { - return "", err - } - credentialsValue := credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, "") - signer := awsv4.NewSigner(credentialsValue) - if _, err := signer.Presign(req, nil, "s3", region, time.Until(expiresAt), time.Now().UTC()); err != nil { - return "", fmt.Errorf("七牛云 Kodo S3 签名失败：%w", err) - } - return req.URL.String(), nil -} - -func qiniuS3Region(setting ossSettingValue) string { - region := strings.ToLower(strings.TrimSpace(setting.Region)) - if region == "" { - endpoint := strings.ToLower(setting.Endpoint) - for _, candidate := range []string{"z0", "z1", "z2", "na0", "as0", "cn-east-1", "cn-north-1", "cn-south-1", "us-north-1", "ap-southeast-1", "cn-east-2"} { - if strings.Contains(endpoint, candidate) { - region = candidate - break - } - } - } - switch region { - case "", "z0", "cn-east-1": - return "cn-east-1" - case "z1", "cn-north-1": - return "cn-north-1" - case "z2", "cn-south-1": - return "cn-south-1" - case "na0", "us-north-1": - return "us-north-1" - case "as0", "ap-southeast-1": - return "ap-southeast-1" - case "cn-east-2", "zhejiang2": - return "cn-east-2" - default: - return "" - } -} - -func qiniuRegion(region string) *qiniuStorage.Region { - switch strings.ToLower(strings.TrimSpace(region)) { - case "z1", "cn-north-1": - return &qiniuStorage.ZoneHuabei - case "z2", "cn-south-1": - return &qiniuStorage.ZoneHuanan - case "na0", "us-north-1": - return &qiniuStorage.ZoneBeimei - case "as0", "ap-southeast-1": - return &qiniuStorage.ZoneXinjiapo - case "cn-east-2", "zhejiang2": - return &qiniuStorage.ZoneHuadongZheJiang2 - default: - return &qiniuStorage.ZoneHuadong - } -} - -func newCOSClient(setting ossSettingValue, timeout time.Duration) (*cos.Client, error) { - bucketURL, err := cosBucketBaseURL(setting) - if err != nil { - return nil, err - } - httpClient := OutboundHTTPClient(timeout) - httpClient.Transport = &cos.AuthorizationTransport{SecretID: setting.AccessKeyID, SecretKey: setting.AccessKeySecret, Transport: httpClient.Transport} - return cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, httpClient), nil -} - -func cosBucketBaseURL(setting ossSettingValue) (*url.URL, error) { - setting = normalizeOSSSetting(setting) - endpoint := strings.TrimRight(setting.Endpoint, "/") - if endpoint == "" { - return nil, errors.New("COS Endpoint 为空") - } - if !strings.Contains(endpoint, "://") { - endpoint = "https://" + endpoint - } - parsed, err := url.Parse(endpoint) - if err != nil { - return nil, err - } - if parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" { - return nil, errors.New("COS Endpoint 格式不正确") - } - if setting.Bucket == "" { - return nil, errors.New("COS Bucket 为空") - } - host := strings.ToLower(parsed.Hostname()) - if strings.HasSuffix(host, ".myqcloud.com") || strings.HasSuffix(host, ".tencentcos.cn") { - if strings.HasPrefix(host, "cos.") || strings.HasPrefix(host, "cos-internal.") || strings.HasPrefix(host, "cos-website.") { - parsed.Host = setting.Bucket + "." + parsed.Host - } else if !strings.HasPrefix(host, strings.ToLower(setting.Bucket)+".") { - return nil, errors.New("COS Endpoint 中的 Bucket 与配置不一致") - } - } - return parsed, nil -} - -func ossCDNBaseURL(raw string) (*url.URL, error) { - parsed, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || parsed.Hostname() == "" { - return nil, errors.New("对象存储 CDN 加速域名格式不正确") - } - if parsed.Scheme != "https" && parsed.Scheme != "http" { - return nil, errors.New("对象存储 CDN 加速域名只支持 http/https") - } - if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" { - return nil, errors.New("对象存储 CDN 加速域名不能包含认证信息、路径、查询参数或片段") - } - parsed.Path = "" - return parsed, nil -} - -func ossCDNObjectURL(raw string, objectKey string) (string, error) { - baseURL, err := ossCDNBaseURL(raw) - if err != nil { - return "", err - } - objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") - if objectKey == "" { - return "", errors.New("对象存储对象路径为空") - } - // CDN 使用自己的访问鉴权与私有桶回源鉴权，不能携带 OSS/COS 的预签名参数。 - // url.URL.String 会负责转义 Path；这里保留未转义值，避免把 %20 再编码为 %2520。 - baseURL.Path = "/" + objectKey - return baseURL.String(), nil -} - -func newOSSRequest(method string, setting ossSettingValue, objectKey string, contentType string, body io.Reader) (*http.Request, error) { - baseURL, err := ossBucketBaseURL(setting) - if err != nil { - return nil, err - } - baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/" + escapeObjectKey(objectKey) - // 请求体必须用 no-op close 包装：服务端提前返回（如 OSS 签名 403）时 - // http.Transport 会关闭未发完的 Request.Body，若直接传入 *os.File 等 - // 调用方持有的文件，后续“降级本地存储”的 Seek 重读将因 file already - // closed 失败。NopCloser 让 Transport 的关闭成为空操作，底层文件保持可用。 - // GET/HEAD/DELETE 等无请求体的调用传入 nil body；NopCloser(nil) 会产生非 nil 的 - // Body 包装 nil reader，Go 1.26 发送前 body 探测会直接 nil 解引用崩溃。 - var reqBody io.Reader - if body != nil { - reqBody = io.NopCloser(body) - } - req, err := http.NewRequest(method, baseURL.String(), reqBody) - if err != nil { - return nil, err - } - date := time.Now().UTC().Format(http.TimeFormat) - req.Header.Set("Date", date) - if contentType != "" { - req.Header.Set("Content-Type", contentType) - } - stringToSign := strings.Join([]string{method, "", contentType, date, "/" + setting.Bucket + "/" + objectKey}, "\n") - mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret)) - _, _ = mac.Write([]byte(stringToSign)) - signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) - req.Header.Set("Authorization", "OSS "+setting.AccessKeyID+":"+signature) - return req, nil -} - -func ossBucketBaseURL(setting ossSettingValue) (*url.URL, error) { - endpoint := strings.TrimRight(setting.Endpoint, "/") - if endpoint == "" { - return nil, errors.New("OSS Endpoint 为空") - } - if !strings.Contains(endpoint, "://") { - endpoint = "https://" + endpoint - } - parsed, err := url.Parse(endpoint) - if err != nil { - return nil, err - } - if parsed.Host == "" { - return nil, errors.New("OSS Endpoint 格式不正确") - } - if !strings.HasPrefix(parsed.Host, setting.Bucket+".") { - parsed.Host = setting.Bucket + "." + parsed.Host - } - return parsed, nil -} - -func escapeObjectKey(key string) string { - parts := strings.Split(key, "/") - for i, part := range parts { - parts[i] = url.PathEscape(part) - } - return strings.Join(parts, "/") -} - func safeObjectSegment(value string) string { value = strings.TrimSpace(value) value = strings.Map(func(r rune) rune { diff --git a/backend/internal/app/resource_access.go b/backend/internal/app/resource_access.go new file mode 100644 index 000000000..27d1fc4ca --- /dev/null +++ b/backend/internal/app/resource_access.go @@ -0,0 +1,162 @@ +package app + +import ( + "errors" + "net/url" + "strconv" + "time" + + "gorm.io/gorm" + "infinite-canvas/backend/internal/assets" + "infinite-canvas/backend/internal/model" +) + +type ResourceAccessOptions = assets.AccessOptions +type ResourceAccessRequest = assets.AccessRequest +type ResourceAccess = assets.ResourceAccess + +type ResourceAccessFailure struct { + Code int `json:"code"` + Reason string `json:"reason"` + Message string `json:"msg"` +} +type ResourceAccessResult struct { + ResourceID string `json:"resourceId"` + Access *ResourceAccess `json:"access,omitempty"` + Error *ResourceAccessFailure `json:"error,omitempty"` +} + +func (s *Service) ResourceAccessBatch(userID string, requests []ResourceAccessRequest) ([]ResourceAccessResult, error) { + if userID == "" { + return nil, Unauthorized("请先登录") + } + if len(requests) == 0 || len(requests) > 100 { + return nil, BadAuthRequest("每批资源访问请求须为 1–100 项") + } + results := make([]ResourceAccessResult, 0, len(requests)) + for _, request := range requests { + item := ResourceAccessResult{ResourceID: request.ResourceID} + // The caller has already been authenticated and each item is resolved through + // ResourceForUser below. Provider input is therefore safe to expose here and + // keeps browser-side model adapters on the same access contract as every + // other resource consumer. + options, err := assets.NormalizeAccessOptions(request.AccessOptions, true) + if err == nil { + var resource *model.Resource + resource, err = s.repo.ResourceForUser(userID, request.ResourceID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err = NotFound("资源不存在或不可访问") + } + if err == nil { + item.Access, err = s.resolveResourceAccess(resource, options) + } + } + if err != nil { + item.Error = &ResourceAccessFailure{Code: 500, Reason: "resource_access_failed", Message: "资源访问失败"} + var appErr *AppError + if errors.As(err, &appErr) { + item.Error = &ResourceAccessFailure{Code: appErr.Code, Reason: string(appErr.Reason), Message: appErr.Message} + } + } + results = append(results, item) + } + return results, nil +} + +func (s *Service) resolveResourceAccess(resource *model.Resource, options ResourceAccessOptions) (*ResourceAccess, error) { + if resource == nil { + return nil, NotFound("资源不存在") + } + var setting ossSettingValue + if resource.Provider != "local" { + var err error + setting, err = s.ossSettingForResource(resource.UserID, resource) + if err != nil { + return nil, WrapAppError(503, "无法解析资源实际存储位置", err) + } + } + return assets.ResolveAccess(resource, setting, options, time.Now().UTC(), func(variant assets.ResourceVariant, expires time.Time) (string, error) { + return s.signedResourceAccessURL(resource, variant, expires, options.Purpose == assets.PurposeProvider) + }) +} + +func (s *Service) PrepareResourceDelivery(userID, id string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + options, err := assets.NormalizeAccessOptions(options, false) + if err != nil { + return nil, err + } + resource, err := s.repo.ResourceForUser(userID, id) + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, NotFound("资源不存在或不可访问") + } + if err != nil { + return nil, err + } + return s.prepareResourceDelivery(userID, resource, options, rangeHeader) +} + +func (s *Service) prepareResourceDelivery(userID string, resource *model.Resource, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + if resource == nil || resource.UserID != userID { + return nil, Forbidden("资源不可访问") + } + access, err := s.resolveResourceAccess(resource, options) + if err != nil { + return nil, err + } + delivery := &ResourceDelivery{Resource: resource, Access: access} + if options.DescribeOnly || access.Delivery == assets.DeliveryCDN || access.Delivery == assets.DeliveryOrigin { + return delivery, nil + } + if access.ActualVariant == assets.VariantPlayback { + delivery.Stream, err = s.OpenResourcePlaybackRange(userID, resource.ID) + } else { + delivery.Stream, err = s.openResourceRange(userID, resource, rangeHeader) + } + if err != nil { + return nil, err + } + delivery.Resource = delivery.Stream.Resource + return delivery, nil +} + +func (s *Service) signedResourceAccessURL(resource *model.Resource, variant assets.ResourceVariant, expires time.Time, public bool) (string, error) { + expiry := strconv.FormatInt(expires.Unix(), 10) + signature, err := s.signPublicResource(resource.ID+"\n"+string(variant), expiry) + if err != nil { + return "", err + } + u := &url.URL{Path: "/api/public/resources/" + url.PathEscape(resource.ID) + "/file"} + q := u.Query() + q.Set("expires", expiry) + q.Set("signature", signature) + q.Set("variant", string(variant)) + u.RawQuery = q.Encode() + if !public { + return u.String(), nil + } + base, err := s.publicResourceBaseURL() + if err != nil { + return "", err + } + if base.Scheme != "https" { + return "", BadAuthRequest("模型读取平台资源需要配置 HTTPS 公网访问地址") + } + return base.ResolveReference(u).String(), nil +} + +func (s *Service) PreparePublicResourceDelivery(id, expires, signature string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) { + options, err := assets.NormalizeAccessOptions(options, false) + if err != nil { + return nil, err + } + if err := s.verifyPublicResourceSignature(id+"\n"+string(options.Variant), expires, signature); err != nil { + return nil, err + } + resource, err := s.repo.Resource(id) + if err != nil { + return nil, Forbidden("匿名下载链接无效") + } + seconds, _ := strconv.ParseInt(expires, 10, 64) + options.ExpiresAt = time.Unix(seconds, 0) + return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader) +} diff --git a/backend/internal/app/resource_delete.go b/backend/internal/app/resource_delete.go index 44a8cbeab..af1818f12 100644 --- a/backend/internal/app/resource_delete.go +++ b/backend/internal/app/resource_delete.go @@ -1,11 +1,8 @@ package app import ( - "context" "errors" "fmt" - "io" - "net/http" "os" "path/filepath" "sort" @@ -16,8 +13,6 @@ import ( "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" - qiniuAuth "github.com/qiniu/go-sdk/v7/auth" - qiniuStorage "github.com/qiniu/go-sdk/v7/storage" "gorm.io/gorm" ) @@ -365,56 +360,3 @@ func (s *Service) deleteLocalResourceObject(objectKey string) error { } return nil } - -func deleteAliyunOSSObject(setting ossSettingValue, objectKey string) error { - req, err := newOSSRequest(http.MethodDelete, setting, objectKey, "", nil) - if err != nil { - return err - } - resp, err := OutboundHTTPClient(2 * time.Minute).Do(req) - if err != nil { - return fmt.Errorf("删除阿里云 OSS 对象失败：%w", err) - } - defer resp.Body.Close() - if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusNotFound { - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return fmt.Errorf("删除阿里云 OSS 对象失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) - } - return nil -} - -func deleteTencentCOSObject(setting ossSettingValue, objectKey string) error { - client, err := newCOSClient(setting, 2*time.Minute) - if err != nil { - return err - } - resp, err := client.Object.Delete(context.Background(), objectKey) - if resp != nil && resp.Body != nil { - defer resp.Body.Close() - } - if err != nil { - if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil - } - return fmt.Errorf("删除腾讯云 COS 对象失败：%w", err) - } - return nil -} - -func deleteQiniuObject(setting ossSettingValue, objectKey string) error { - if setting.AccessKeyID == "" || setting.AccessKeySecret == "" { - return errors.New("七牛云 Kodo 访问密钥不可用") - } - if setting.Bucket == "" || strings.TrimSpace(objectKey) == "" { - return errors.New("七牛云 Kodo Bucket 或对象路径为空") - } - mac := qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret) - manager := qiniuStorage.NewBucketManager(mac, &qiniuStorage.Config{Region: qiniuRegion(setting.Region), UseHTTPS: true}) - if err := manager.Delete(setting.Bucket, strings.TrimLeft(objectKey, "/")); err != nil { - if strings.Contains(strings.ToLower(err.Error()), "no such") || strings.Contains(strings.ToLower(err.Error()), "not found") { - return nil - } - return fmt.Errorf("删除七牛云 Kodo 对象失败：%w", err) - } - return nil -} diff --git a/backend/internal/app/resource_test.go b/backend/internal/app/resource_test.go index b206bbfa3..56fc19c3a 100644 --- a/backend/internal/app/resource_test.go +++ b/backend/internal/app/resource_test.go @@ -18,8 +18,10 @@ import ( "testing/iotest" "time" + "infinite-canvas/backend/internal/assets" "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" + "infinite-canvas/backend/internal/storage" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -219,12 +221,12 @@ func TestGetOSSObjectRangeSupportsTencentCOS(t *testing.T) { if err != nil { t.Fatal(err) } - defer stream.body.Close() - data, err := io.ReadAll(stream.body) + defer stream.Body.Close() + data, err := io.ReadAll(stream.Body) if err != nil { t.Fatal(err) } - if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" { + if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "data" { t.Fatalf("stream = %#v, data = %q", stream, data) } } @@ -252,12 +254,12 @@ func TestGetOSSObjectRangeUsesTencentCOSCDNBaseURL(t *testing.T) { if err != nil { t.Fatal(err) } - defer stream.body.Close() - data, err := io.ReadAll(stream.body) + defer stream.Body.Close() + data, err := io.ReadAll(stream.Body) if err != nil { t.Fatal(err) } - if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" { + if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "data" { t.Fatalf("stream = %#v, data = %q", stream, data) } } @@ -285,12 +287,12 @@ func TestGetOSSObjectRangeUsesAliyunCDNBaseURLWithoutOSSSignature(t *testing.T) if err != nil { t.Fatal(err) } - defer stream.body.Close() - data, err := io.ReadAll(stream.body) + defer stream.Body.Close() + data, err := io.ReadAll(stream.Body) if err != nil { t.Fatal(err) } - if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" { + if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "data" { t.Fatalf("stream = %#v, data = %q", stream, data) } } @@ -412,10 +414,10 @@ func TestArchivedProviderCredentialsAreEncryptedAtRest(t *testing.T) { } } -func TestDirectResourceURLChecksOwnershipAndSignsOSSResource(t *testing.T) { +func TestResourceAccessChecksOwnershipAndSignsOSSResource(t *testing.T) { svc := newResourceTestService(t) settingJSON, _ := json.Marshal(ossSettingValue{ - Enabled: true, Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + Enabled: true, Provider: "aliyun", Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value", }) if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { @@ -423,18 +425,22 @@ func TestDirectResourceURLChecksOwnershipAndSignsOSSResource(t *testing.T) { } resource := model.Resource{ ID: "resource-direct", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, - Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + Provider: "aliyun", Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket", ObjectKey: "users/user-1/image/direct.png", MimeType: "image/png", } if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - value, err := svc.DirectResourceURL("user-1", resource.ID) - if err != nil || !strings.Contains(value, "Signature=") { - t.Fatalf("DirectResourceURL() = %q, %v", value, err) + results, err := svc.ResourceAccessBatch("user-1", []ResourceAccessRequest{{ + ResourceID: resource.ID, + AccessOptions: ResourceAccessOptions{Purpose: assets.PurposeCopy}, + }}) + if err != nil || len(results) != 1 || results[0].Access == nil || !strings.Contains(results[0].Access.URL, "Signature=") { + t.Fatalf("ResourceAccessBatch() = %#v, %v", results, err) } - if _, err := svc.DirectResourceURL("other-user", resource.ID); err == nil { - t.Fatal("DirectResourceURL() allowed another user's resource") + other, err := svc.ResourceAccessBatch("other-user", []ResourceAccessRequest{{ResourceID: resource.ID, AccessOptions: ResourceAccessOptions{Purpose: assets.PurposeCopy}}}) + if err != nil || len(other) != 1 || other[0].Access != nil { + t.Fatalf("ResourceAccessBatch() allowed another user's resource: %#v, %v", other, err) } } @@ -443,6 +449,7 @@ func TestPrepareResourceDeliveryPrefersConfiguredCDN(t *testing.T) { settingJSON, _ := json.Marshal(ossSettingValue{ Enabled: true, Provider: tencentCOSProvider, Endpoint: "https://cos.ap-shanghai.myqcloud.com", CDNBaseURL: "https://media.example.com", Bucket: "private-bucket-1250000000", AccessKeyID: "secret-id", AccessKeySecret: "secret-key", + Delivery: storage.DeliverySettings{CDNAuthMode: "public"}, }) if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { t.Fatal(err) @@ -455,19 +462,19 @@ func TestPrepareResourceDeliveryPrefersConfiguredCDN(t *testing.T) { if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{}) + delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } - if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.RedirectURL != "https://media.example.com/users/user-1/image/test%20image.png" { + if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryCDN || delivery.Access.URL != "https://media.example.com/users/user-1/image/test%20image.png" { t.Fatalf("PrepareResourceDelivery() = %#v", delivery) } } -func TestPrepareResourceDeliveryAllowsExplicitProxyWithCDN(t *testing.T) { +func TestPrepareResourceDeliveryFallsBackToOriginWhenCDNAuthIsMissing(t *testing.T) { svc := newResourceTestService(t) settingJSON, _ := json.Marshal(ossSettingValue{ - Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", CDNBaseURL: "https://media.example.com", + Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", CDNBaseURL: "https://media.example.com", Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value", }) if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { @@ -475,18 +482,18 @@ func TestPrepareResourceDeliveryAllowsExplicitProxyWithCDN(t *testing.T) { } resource := model.Resource{ ID: "resource-cdn-proxy", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, - Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket", ObjectKey: "users/user-1/image/proxy.png", MimeType: "image/png", } if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{ForceProxy: true}) + delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } - if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.RedirectURL != "" { - t.Fatalf("PrepareResourceDelivery(force proxy) = %#v", delivery) + if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryOrigin || delivery.Access.FallbackReason != "cdn_auth_unconfigured" { + t.Fatalf("PrepareResourceDelivery() = %#v", delivery) } } @@ -495,6 +502,7 @@ func TestPrepareResourceDeliverySignsPrivateQiniuCDNURL(t *testing.T) { settingJSON, _ := json.Marshal(ossSettingValue{ Enabled: true, Provider: qiniuKodoProvider, Endpoint: "https://up-z0.qiniup.com", CDNBaseURL: "https://media.example.com", Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value", + Delivery: storage.DeliverySettings{CDNAuthMode: "qiniu"}, }) if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { t.Fatal(err) @@ -507,38 +515,40 @@ func TestPrepareResourceDeliverySignsPrivateQiniuCDNURL(t *testing.T) { if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{}) + delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } - if delivery.RedirectURL == "" || !strings.HasPrefix(delivery.RedirectURL, "https://media.example.com/ai/users/user-1/image/private.png?") || !strings.Contains(delivery.RedirectURL, "e=") || !strings.Contains(delivery.RedirectURL, "token=") { - t.Fatalf("PrepareResourceDelivery() = %q, want a signed Qiniu URL", delivery.RedirectURL) + if delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryCDN || !strings.HasPrefix(delivery.Access.URL, "https://media.example.com/ai/users/user-1/image/private.png?") || !strings.Contains(delivery.Access.URL, "e=") || !strings.Contains(delivery.Access.URL, "token=") { + t.Fatalf("PrepareResourceDelivery() = %#v, want a signed Qiniu URL", delivery) } } -func TestPrepareResourceDeliveryProxiesQiniuWithoutCDNBaseURL(t *testing.T) { +func TestPrepareResourceDeliveryAllowsExplicitPrivateOriginProxy(t *testing.T) { + t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") svc := newResourceTestService(t) settingJSON, _ := json.Marshal(ossSettingValue{ - Enabled: true, Provider: qiniuKodoProvider, Region: "z0", Endpoint: "https://up-z0.qiniup.com", + Enabled: true, Provider: aliyunOSSProvider, Endpoint: "http://storage.internal", Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value", + Delivery: storage.DeliverySettings{AllowPrivateProxy: true}, }) if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { t.Fatal(err) } resource := model.Resource{ ID: "resource-qiniu-proxy", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, - Provider: qiniuKodoProvider, Endpoint: "https://up-z0.qiniup.com", Bucket: "private-bucket", + Provider: aliyunOSSProvider, Endpoint: "http://storage.internal", Bucket: "private-bucket", ObjectKey: "users/user-1/image/private.png", MimeType: "image/png", } if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{}) + results, err := svc.ResourceAccessBatch("user-1", []ResourceAccessRequest{{ResourceID: resource.ID, AccessOptions: ResourceAccessOptions{Purpose: assets.PurposeDisplay}}}) if err != nil { t.Fatal(err) } - if delivery.Resource == nil || delivery.RedirectURL != "" { - t.Fatalf("PrepareResourceDelivery() = %#v, want backend proxy delivery", delivery) + if len(results) != 1 || results[0].Access == nil || results[0].Access.Delivery != assets.DeliveryProxy { + t.Fatalf("ResourceAccessBatch() = %#v, want backend proxy delivery", results) } } @@ -568,15 +578,15 @@ func TestCurrentUserCDNSettingAppliesToHistoricalResourcesInSameStorage(t *testi } if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{ Enabled: true, Provider: aliyunOSSProvider, Endpoint: server.URL, CDNBaseURL: server.URL, Bucket: "private-bucket", - AccessKeyID: "access-id", AccessKeySecret: "secret-value", + AccessKeyID: "access-id", AccessKeySecret: "secret-value", CDNAuthMode: "public", }); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceDeliveryOptions{}) + delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } - if delivery.RedirectURL != server.URL+"/users/user-1/image/historical.png" { + if delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryCDN || delivery.Access.URL != server.URL+"/users/user-1/image/historical.png" { t.Fatalf("PrepareResourceDelivery(historical user resource) = %#v", delivery) } } @@ -599,7 +609,7 @@ func TestBoundHistoricalUserResourceDoesNotFollowCurrentProviderCDN(t *testing.T actor := &model.User{ID: "user-1"} if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{ Enabled: true, Provider: aliyunOSSProvider, Endpoint: aliyunEndpoint.URL, CDNBaseURL: aliyunCDN.URL, Bucket: "aliyun-bucket", - AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret", + AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret", CDNAuthMode: "public", }); err != nil { t.Fatal(err) } @@ -609,7 +619,7 @@ func TestBoundHistoricalUserResourceDoesNotFollowCurrentProviderCDN(t *testing.T } if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{ Enabled: true, Provider: qiniuKodoProvider, Endpoint: qiniuEndpoint.URL, CDNBaseURL: qiniuCDN.URL, Bucket: "qiniu-bucket", - AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret", + AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret", CDNAuthMode: "public", }); err != nil { t.Fatal(err) } @@ -621,13 +631,13 @@ func TestBoundHistoricalUserResourceDoesNotFollowCurrentProviderCDN(t *testing.T if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceDeliveryOptions{}) + delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } want := aliyunCDN.URL + "/ai/users/user-1/image/bound.png" - if delivery.RedirectURL != want || strings.Contains(delivery.RedirectURL, qiniuCDN.URL) { - t.Fatalf("PrepareResourceDelivery(bound historical resource) = %q, want %q", delivery.RedirectURL, want) + if delivery.Access == nil || delivery.Access.URL != want || strings.Contains(delivery.Access.URL, qiniuCDN.URL) { + t.Fatalf("PrepareResourceDelivery(bound historical resource) = %#v, want %q", delivery, want) } } @@ -648,13 +658,13 @@ func TestHistoricalUserResourceWithoutStorageSettingIDKeepsItsProviderCDN(t *tes actor := &model.User{ID: "user-1"} if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{ Enabled: true, Provider: aliyunOSSProvider, Endpoint: aliyunEndpoint.URL, CDNBaseURL: aliyunCDN.URL, Bucket: "aliyun-bucket", - AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret", + AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret", CDNAuthMode: "public", }); err != nil { t.Fatal(err) } if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{ Enabled: true, Provider: qiniuKodoProvider, Endpoint: qiniuEndpoint.URL, CDNBaseURL: qiniuCDN.URL, Bucket: "qiniu-bucket", - AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret", + AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret", CDNAuthMode: "public", }); err != nil { t.Fatal(err) } @@ -666,20 +676,20 @@ func TestHistoricalUserResourceWithoutStorageSettingIDKeepsItsProviderCDN(t *tes if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceDeliveryOptions{}) + delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } want := aliyunCDN.URL + "/ai/users/user-1/image/legacy.png" - if delivery.RedirectURL != want || strings.Contains(delivery.RedirectURL, qiniuCDN.URL) { - t.Fatalf("PrepareResourceDelivery(legacy user resource) = %q, want %q", delivery.RedirectURL, want) + if delivery.Access == nil || delivery.Access.URL != want || strings.Contains(delivery.Access.URL, qiniuCDN.URL) { + t.Fatalf("PrepareResourceDelivery(legacy user resource) = %#v, want %q", delivery, want) } } -func TestPrepareResourceDeliveryKeepsForcedOriginDirectWithoutCDN(t *testing.T) { +func TestPrepareResourceDeliveryUsesSignedOriginWithoutCDN(t *testing.T) { svc := newResourceTestService(t) settingJSON, _ := json.Marshal(ossSettingValue{ - Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value", }) if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { @@ -687,17 +697,17 @@ func TestPrepareResourceDeliveryKeepsForcedOriginDirectWithoutCDN(t *testing.T) } resource := model.Resource{ ID: "resource-origin-direct", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, - Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket", ObjectKey: "users/user-1/image/direct.png", MimeType: "image/png", } if err := svc.repo.CreateResource(&resource); err != nil { t.Fatal(err) } - delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{ForceDirect: true}) + delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "") if err != nil { t.Fatal(err) } - if !strings.Contains(delivery.RedirectURL, "private-bucket.oss-cn-test.aliyuncs.com/users/user-1/image/direct.png") || !strings.Contains(delivery.RedirectURL, "Signature=") { + if delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryOrigin || !strings.Contains(delivery.Access.URL, "private-bucket.s3.amazonaws.com/users/user-1/image/direct.png") || !strings.Contains(delivery.Access.URL, "Signature=") { t.Fatalf("PrepareResourceDelivery(force direct) = %#v", delivery) } } diff --git a/backend/internal/app/resource_url_test.go b/backend/internal/app/resource_url_test.go index e2053b0c8..395283406 100644 --- a/backend/internal/app/resource_url_test.go +++ b/backend/internal/app/resource_url_test.go @@ -8,7 +8,7 @@ func TestResourceIDFromFileURL(t *testing.T) { want string }{ {"/api/resources/abc_123/file", "abc_123"}, - {"https://host.example/api/resources/abc_123/file?proxy=1", "abc_123"}, + {"https://host.example/api/resources/abc_123/file?variant=original", "abc_123"}, {"/api/projects/abc_123", ""}, {"", ""}, } diff --git a/backend/internal/app/settings.go b/backend/internal/app/settings.go index f711138e4..9df0ff9d4 100644 --- a/backend/internal/app/settings.go +++ b/backend/internal/app/settings.go @@ -18,6 +18,7 @@ import ( "time" "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/storage" "gorm.io/gorm" ) @@ -34,20 +35,23 @@ const ( ) type OSSSettingRequest struct { - Enabled bool `json:"enabled"` - Provider string `json:"provider"` - Region string `json:"region"` - Endpoint string `json:"endpoint"` - CDNBaseURL string `json:"cdnBaseUrl"` - Bucket string `json:"bucket"` - AccessKeyID string `json:"accessKeyId"` - AccessKeySecret string `json:"accessKeySecret"` - PublicBaseURL string `json:"publicBaseUrl"` - PathPrefix string `json:"pathPrefix"` - S3Preset string `json:"s3Preset"` - PathStyle bool `json:"pathStyle"` - SessionToken string `json:"sessionToken"` - AllowUserS3 bool `json:"allowUserS3"` + Enabled bool `json:"enabled"` + Provider string `json:"provider"` + Region string `json:"region"` + Endpoint string `json:"endpoint"` + CDNBaseURL string `json:"cdnBaseUrl"` + Bucket string `json:"bucket"` + AccessKeyID string `json:"accessKeyId"` + AccessKeySecret string `json:"accessKeySecret"` + PublicBaseURL string `json:"publicBaseUrl"` + PathPrefix string `json:"pathPrefix"` + S3Preset string `json:"s3Preset"` + PathStyle bool `json:"pathStyle"` + SessionToken string `json:"sessionToken"` + AllowUserS3 bool `json:"allowUserS3"` + CDNAuthMode string `json:"cdnAuthMode"` + RequireCDN bool `json:"requireCDN"` + AllowPrivateProxy bool `json:"allowPrivateProxy"` } type PublicOSSSetting struct { @@ -70,36 +74,14 @@ type PublicOSSSetting struct { HistoryCount int64 `json:"historyCount"` ReferencedResourceCount int64 `json:"referencedResourceCount"` AllowUserS3 bool `json:"allowUserS3"` + CDNAuthMode string `json:"cdnAuthMode"` + RequireCDN bool `json:"requireCDN"` + AllowPrivateProxy bool `json:"allowPrivateProxy"` UpdatedBy string `json:"updatedBy"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` } -type ossSettingValue struct { - Enabled bool `json:"enabled"` - Provider string `json:"provider"` - Region string `json:"region"` - Endpoint string `json:"endpoint"` - CDNBaseURL string `json:"cdnBaseUrl"` - Bucket string `json:"bucket"` - AccessKeyID string `json:"accessKeyId"` - AccessKeySecret string `json:"accessKeySecret"` - PublicBaseURL string `json:"publicBaseUrl"` - PathPrefix string `json:"pathPrefix"` - S3Preset string `json:"s3Preset"` - PathStyle bool `json:"pathStyle"` - SessionToken string `json:"sessionToken"` - StorageLocationID string `json:"storageLocationId"` - AllowUserS3 bool `json:"allowUserS3"` - // 平台切换云厂商后仍需读取历史资源，因此仅归档非当前厂商的访问密钥。 - ArchivedCredentials map[string]ossProviderCredentials `json:"archivedCredentials,omitempty"` -} - -type ossProviderCredentials struct { - AccessKeyID string `json:"accessKeyId"` - AccessKeySecret string `json:"accessKeySecret"` -} - func (s *Service) AdminOSSSetting(actor *model.User) (*PublicOSSSetting, error) { if err := s.RequireAdmin(actor); err != nil { return nil, err @@ -563,7 +545,11 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS PathStyle: req.PathStyle, SessionToken: strings.TrimSpace(req.SessionToken), AllowUserS3: req.AllowUserS3, + Delivery: storage.DeliverySettings{CDNAuthMode: strings.ToLower(strings.TrimSpace(req.CDNAuthMode)), RequireCDN: req.RequireCDN, AllowPrivateProxy: req.AllowPrivateProxy}, }) + if next.Delivery.CDNAuthMode != "" && next.Delivery.CDNAuthMode != "public" && next.Delivery.CDNAuthMode != "qiniu" { + return next, BadAuthRequest("CDN 鉴权方式无效，仅支持 public 或 qiniu") + } if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider && next.Provider != s3Provider { return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS、七牛云 Kodo 和通用 S3") } @@ -640,49 +626,6 @@ func archiveOSSProviderCredentials(next ossSettingValue, current ossSettingValue return next } -func cloneOSSProviderCredentials(source map[string]ossProviderCredentials) map[string]ossProviderCredentials { - if len(source) == 0 { - return nil - } - cloned := make(map[string]ossProviderCredentials, len(source)) - for provider, credentials := range source { - cloned[strings.ToLower(strings.TrimSpace(provider))] = ossProviderCredentials{ - AccessKeyID: strings.TrimSpace(credentials.AccessKeyID), - AccessKeySecret: strings.TrimSpace(credentials.AccessKeySecret), - } - } - return cloned -} - -func normalizeOSSSetting(value ossSettingValue) ossSettingValue { - value.Provider = strings.ToLower(strings.TrimSpace(value.Provider)) - if value.Provider == "" { - value.Provider = aliyunOSSProvider - } - value.Region = strings.TrimSpace(value.Region) - value.Endpoint = strings.TrimRight(strings.TrimSpace(value.Endpoint), "/") - if value.Provider == tencentCOSProvider && value.Endpoint == "" && value.Region != "" { - value.Endpoint = "https://cos." + value.Region + ".myqcloud.com" - } - value.CDNBaseURL = strings.TrimRight(strings.TrimSpace(value.CDNBaseURL), "/") - value.Bucket = strings.TrimSpace(value.Bucket) - value.AccessKeyID = strings.TrimSpace(value.AccessKeyID) - value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret) - value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/") - value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/") - if value.PathPrefix == "" { - value.PathPrefix = defaultOSSPathPrefix - } - value.S3Preset = strings.ToLower(strings.TrimSpace(value.S3Preset)) - if value.S3Preset == "" { - value.S3Preset = "custom" - } - value.SessionToken = strings.TrimSpace(value.SessionToken) - value.StorageLocationID = strings.TrimSpace(value.StorageLocationID) - value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials) - return value -} - func defaultOSSSetting() ossSettingValue { return ossSettingValue{Provider: aliyunOSSProvider, PathPrefix: defaultOSSPathPrefix} } @@ -704,6 +647,9 @@ func (s *Service) publicOSSSetting(setting *model.SystemSetting, value ossSettin HasSessionToken: value.SessionToken != "", StorageLocationID: value.StorageLocationID, AllowUserS3: value.AllowUserS3, + CDNAuthMode: value.Delivery.CDNAuthMode, + RequireCDN: value.Delivery.RequireCDN, + AllowPrivateProxy: value.Delivery.AllowPrivateProxy, } if setting != nil { result.UpdatedBy = setting.UpdatedBy @@ -733,6 +679,9 @@ func (s *Service) publicUserOSSSetting(setting *model.UserOSSSetting, value ossS HasSessionToken: value.SessionToken != "", StorageLocationID: value.StorageLocationID, AllowUserS3: allowUserS3, + CDNAuthMode: value.Delivery.CDNAuthMode, + RequireCDN: value.Delivery.RequireCDN, + AllowPrivateProxy: value.Delivery.AllowPrivateProxy, } if value.Provider == s3Provider && !allowUserS3 { result.Enabled = false diff --git a/backend/internal/app/storage_adapters.go b/backend/internal/app/storage_adapters.go new file mode 100644 index 000000000..5311a5944 --- /dev/null +++ b/backend/internal/app/storage_adapters.go @@ -0,0 +1,45 @@ +package app + +import "infinite-canvas/backend/internal/storage" + +type ossSettingValue = storage.Settings +type ossProviderCredentials = storage.Credentials +type ossObjectStream = storage.ObjectStream + +var ( + putOSSObject = storage.PutOSSObject + putAliyunOSSObject = storage.PutAliyunOSSObject + getOSSObjectRange = storage.GetOSSObjectRange + getAliyunOSSObjectRange = storage.GetAliyunOSSObjectRange + signedOSSObjectURL = storage.SignedOSSObjectURL + signedAliyunOSSObjectURL = storage.SignedAliyunOSSObjectURL + putCOSObject = storage.PutCOSObject + putQiniuObject = storage.PutQiniuObject + getCOSObjectRange = storage.GetCOSObjectRange + getQiniuObjectRange = storage.GetQiniuObjectRange + signedCOSObjectURL = storage.SignedCOSObjectURL + signedQiniuObjectURL = storage.SignedQiniuObjectURL + signedQiniuS3ObjectURL = storage.SignedQiniuS3ObjectURL + qiniuS3Region = storage.QiniuS3Region + qiniuRegion = storage.QiniuRegion + newCOSClient = storage.NewCOSClient + cosBucketBaseURL = storage.CosBucketBaseURL + ossCDNBaseURL = storage.OssCDNBaseURL + ossCDNObjectURL = storage.OssCDNObjectURL + newOSSRequest = storage.NewOSSRequest + ossBucketBaseURL = storage.OssBucketBaseURL + escapeObjectKey = storage.EscapeObjectKey + deleteAliyunOSSObject = storage.DeleteAliyunOSSObject + deleteTencentCOSObject = storage.DeleteTencentCOSObject + deleteQiniuObject = storage.DeleteQiniuObject + validateStorageEndpoint = storage.ValidateStorageEndpoint + standardAWSS3Endpoint = storage.StandardAWSS3Endpoint + publicHTTPSStorageEndpoint = storage.PublicHTTPSStorageEndpoint + newS3Client = storage.NewS3Client + putS3Object = storage.PutS3Object + getS3ObjectRange = storage.GetS3ObjectRange + signedS3ObjectURL = storage.SignedS3ObjectURL + deleteS3Object = storage.DeleteS3Object + cloneOSSProviderCredentials = storage.CloneCredentials + normalizeOSSSetting = storage.NormalizeSettings +) diff --git a/backend/internal/app/storage_location.go b/backend/internal/app/storage_location.go index 564b07aa6..fa0559b68 100644 --- a/backend/internal/app/storage_location.go +++ b/backend/internal/app/storage_location.go @@ -124,11 +124,11 @@ func verifyOSSConnection(value ossSettingValue, testKey string) error { } func verifyOSSConnectionRead(stream *ossObjectStream, payload []byte) error { - if stream == nil || stream.body == nil { + if stream == nil || stream.Body == nil { return fmt.Errorf("%w：响应为空", errOSSConnectionReadMismatch) } - data, readErr := io.ReadAll(io.LimitReader(stream.body, int64(len(payload)+1))) - closeErr := stream.body.Close() + data, readErr := io.ReadAll(io.LimitReader(stream.Body, int64(len(payload)+1))) + closeErr := stream.Body.Close() if readErr != nil { return fmt.Errorf("对象存储响应读取失败：%w", readErr) } @@ -138,7 +138,7 @@ func verifyOSSConnectionRead(stream *ossObjectStream, payload []byte) error { // HTTP 允许服务端忽略 Range 并返回 200 + 完整对象。应用的资源代理也会 // 原样透传这种响应，因此连接测试应同时接受完整读取与标准的 206 分段读取。 - switch stream.statusCode { + switch stream.StatusCode { case http.StatusPartialContent: if len(payload) >= 4 && bytes.Equal(data, payload[:4]) { return nil @@ -148,7 +148,7 @@ func verifyOSSConnectionRead(stream *ossObjectStream, payload []byte) error { return nil } } - return fmt.Errorf("%w（HTTP %d，读取 %d 字节）", errOSSConnectionReadMismatch, stream.statusCode, len(data)) + return fmt.Errorf("%w（HTTP %d，读取 %d 字节）", errOSSConnectionReadMismatch, stream.StatusCode, len(data)) } func storageConnectionTestError(provider string, operation string, cause error) error { diff --git a/backend/internal/app/storage_location_test.go b/backend/internal/app/storage_location_test.go index bd409c7b8..f62b71dc8 100644 --- a/backend/internal/app/storage_location_test.go +++ b/backend/internal/app/storage_location_test.go @@ -80,9 +80,9 @@ func TestTencentCOSConnectionTestUsesStorageEndpointInsteadOfCDN(t *testing.T) { func TestOSSConnectionReadAcceptsFullObjectWhenRangeIsIgnored(t *testing.T) { payload := []byte("yingce-storage-test") stream := &ossObjectStream{ - body: io.NopCloser(strings.NewReader(string(payload))), - statusCode: http.StatusOK, - contentLength: int64(len(payload)), + Body: io.NopCloser(strings.NewReader(string(payload))), + StatusCode: http.StatusOK, + ContentLength: int64(len(payload)), } if err := verifyOSSConnectionRead(stream, payload); err != nil { t.Fatal(err) @@ -92,9 +92,9 @@ func TestOSSConnectionReadAcceptsFullObjectWhenRangeIsIgnored(t *testing.T) { func TestOSSConnectionReadRejectsUnexpectedContent(t *testing.T) { payload := []byte("yingce-storage-test") stream := &ossObjectStream{ - body: io.NopCloser(strings.NewReader("error page")), - statusCode: http.StatusOK, - contentLength: 10, + Body: io.NopCloser(strings.NewReader("error page")), + StatusCode: http.StatusOK, + ContentLength: 10, } err := verifyOSSConnectionRead(stream, payload) if err == nil { diff --git a/backend/internal/app/storage_s3_test.go b/backend/internal/app/storage_s3_test.go index 095e8e0d1..ac0774878 100644 --- a/backend/internal/app/storage_s3_test.go +++ b/backend/internal/app/storage_s3_test.go @@ -57,9 +57,9 @@ func TestS3ObjectOperationsUsePathStyleSessionTokenAndNoManagedHeaders(t *testin if err != nil { t.Fatal(err) } - data, _ := io.ReadAll(stream.body) - _ = stream.body.Close() - if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "payl" { + data, _ := io.ReadAll(stream.Body) + _ = stream.Body.Close() + if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "payl" { t.Fatalf("stream = %#v, body = %q", stream, data) } if err := deleteS3Object(setting, "prefix/object.txt"); err != nil { diff --git a/backend/internal/assets/access.go b/backend/internal/assets/access.go new file mode 100644 index 000000000..e542d445a --- /dev/null +++ b/backend/internal/assets/access.go @@ -0,0 +1,155 @@ +package assets + +import ( + "net/http" + "time" + + "infinite-canvas/backend/internal/kernel" + "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/storage" +) + +type AccessPurpose string +type ResourceVariant string +type DeliveryMode string + +const ( + PurposeDisplay AccessPurpose = "display" + PurposeCopy AccessPurpose = "copy" + PurposeDownload AccessPurpose = "download" + PurposeProcess AccessPurpose = "browser-process" + PurposeProvider AccessPurpose = "provider-input" + VariantOriginal ResourceVariant = "original" + VariantPlayback ResourceVariant = "playback" + DeliveryCDN DeliveryMode = "cdn" + DeliveryOrigin DeliveryMode = "origin" + DeliveryLocal DeliveryMode = "platform-local" + DeliveryProxy DeliveryMode = "platform-proxy" +) + +// AccessOptions contains use intent, never a client-controlled authorization or transport override. +type AccessOptions struct { + Purpose AccessPurpose `json:"purpose"` + Variant ResourceVariant `json:"variant"` + ExpiresAt time.Time `json:"-"` + DescribeOnly bool `json:"-"` +} + +type AccessRequest struct { + ResourceID string `json:"resourceId"` + AccessOptions +} + +type ResourceAccess struct { + ResourceID string `json:"resourceId"` + RequestedVariant ResourceVariant `json:"requestedVariant"` + ActualVariant ResourceVariant `json:"actualVariant"` + URL string `json:"url"` + Delivery DeliveryMode `json:"delivery"` + IssuedAt time.Time `json:"issuedAt"` + ExpiresAt *time.Time `json:"expiresAt"` + RefreshAt time.Time `json:"refreshAt"` + Revision string `json:"revision"` + FallbackReason string `json:"fallbackReason,omitempty"` +} + +func AccessError(status int, reason, message string) *kernel.AppError { + err := kernel.NewAppError(status, message) + err.Reason = kernel.ErrorReason(reason) + return err +} + +func NormalizeAccessOptions(options AccessOptions, allowProvider bool) (AccessOptions, error) { + if options.Purpose == "" { + options.Purpose = PurposeDisplay + } + if options.Variant == "" { + options.Variant = VariantOriginal + } + switch options.Purpose { + case PurposeDisplay, PurposeCopy, PurposeDownload, PurposeProcess: + case PurposeProvider: + if !allowProvider { + return options, kernel.Forbidden("不允许请求模型输入凭据") + } + default: + return options, kernel.BadAuthRequest("资源访问用途无效") + } + if options.Variant != VariantOriginal && options.Variant != VariantPlayback { + return options, kernel.BadAuthRequest("资源变体无效") + } + if options.Purpose == PurposeCopy || options.Purpose == PurposeDownload || options.Purpose == PurposeProvider { + options.Variant = VariantOriginal + } + return options, nil +} + +// ResolveAccess is the sole delivery policy. The caller must authorize resource ownership/scene first. +// Local URL signing is injected by the composition root; storage adapters never know application keys. +func ResolveAccess(resource *model.Resource, setting storage.Settings, options AccessOptions, now time.Time, + platformURL func(ResourceVariant, time.Time) (string, error)) (*ResourceAccess, error) { + options, err := NormalizeAccessOptions(options, true) + if err != nil { + return nil, err + } + if resource == nil { + return nil, kernel.NotFound("资源不存在") + } + if resource.Status != model.ResourceStatusReady { + return nil, AccessError(http.StatusConflict, "resource_not_ready", "资源尚未上传完成") + } + ttl := 5 * time.Minute + if options.Purpose == PurposeProvider { + ttl = 4 * time.Hour + } + expires := now.Add(ttl) + if !options.ExpiresAt.IsZero() && options.ExpiresAt.Before(expires) { + expires = options.ExpiresAt + } + if !expires.After(now) { + return nil, kernel.Forbidden("资源授权已过期") + } + access := &ResourceAccess{ResourceID: resource.ID, RequestedVariant: options.Variant, ActualVariant: VariantOriginal, + IssuedAt: now, ExpiresAt: &expires, RefreshAt: now.Add(expires.Sub(now) * 4 / 5), Revision: resource.ETag + ":" + resource.PlaybackStatus + ":" + storage.DeliveryRevision(setting)} + if options.Variant == VariantPlayback { + if resource.Provider == "local" && resource.PlaybackStatus == model.PlaybackStatusReady && resource.PlaybackObjectKey != "" { + access.ActualVariant = VariantPlayback + } else { + access.FallbackReason = "playback_not_ready" + } + } + if resource.Provider == "local" { + access.Delivery = DeliveryLocal + } else if storage.CDNEnabled(setting) { + access.URL, err = storage.SignCDNURL(setting, resource.ObjectKey, expires) + access.Delivery = DeliveryCDN + if setting.Delivery.CDNAuthMode == "public" { + access.ExpiresAt = nil + } + } else if setting.Delivery.RequireCDN { + return nil, AccessError(503, "resource_cdn_unconfigured", "CDN 访问鉴权未配置，请检查存储分发设置") + } else if storage.PublicOrigin(setting) { + access.URL, err = storage.SignedOSSObjectURL(setting, resource.ObjectKey, expires) + access.Delivery = DeliveryOrigin + if setting.CDNBaseURL != "" { + access.FallbackReason = "cdn_auth_unconfigured" + } + } else if setting.Delivery.AllowPrivateProxy { + access.Delivery = DeliveryProxy + access.FallbackReason = "private_origin" + } else { + return nil, AccessError(503, "resource_origin_private", "资源源站不可公网访问，请配置 CDN 或明确启用私网源站代理") + } + if err != nil { + failure := AccessError(503, "resource_signing_failed", "资源访问地址签发失败，请检查存储分发设置") + failure.Cause = err + return nil, failure + } + if access.Delivery == DeliveryLocal || access.Delivery == DeliveryProxy { + access.URL, err = platformURL(access.ActualVariant, expires) + if err != nil { + return nil, err + } + } + return access, nil +} diff --git a/backend/internal/assets/access_test.go b/backend/internal/assets/access_test.go new file mode 100644 index 000000000..c18580ab2 --- /dev/null +++ b/backend/internal/assets/access_test.go @@ -0,0 +1,143 @@ +package assets + +import ( + "strings" + "testing" + "time" + + "infinite-canvas/backend/internal/kernel" + "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/storage" +) + +func testReadyResource(provider string) *model.Resource { + return &model.Resource{ + ID: "resource-1", UserID: "user-1", Provider: provider, ObjectKey: "users/user-1/image/test.png", + Status: model.ResourceStatusReady, MimeType: "image/png", ETag: "etag-1", + } +} + +func testPlatformURL(variants *[]ResourceVariant) func(ResourceVariant, time.Time) (string, error) { + return func(variant ResourceVariant, _ time.Time) (string, error) { + *variants = append(*variants, variant) + return "https://canvas.example/api/resources/resource-1/file", nil + } +} + +func TestResolveAccessPolicyMatrix(t *testing.T) { + now := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + tests := []struct { + name string + resource *model.Resource + setting storage.Settings + options AccessOptions + wantMode DeliveryMode + wantReason string + wantURL string + wantErr string + }{ + { + name: "local uses platform controlled URL", + resource: testReadyResource("local"), + options: AccessOptions{Purpose: PurposeDisplay}, + wantMode: DeliveryLocal, + wantURL: "https://canvas.example/api/resources/resource-1/file", + }, + { + name: "public CDN uses CDN URL", + resource: testReadyResource("aliyun"), + setting: storage.Settings{Provider: "aliyun", CDNBaseURL: "https://media.example.com", Delivery: storage.DeliverySettings{CDNAuthMode: "public"}}, + options: AccessOptions{Purpose: PurposeCopy}, + wantMode: DeliveryCDN, + wantURL: "https://media.example.com/users/user-1/image/test.png", + }, + { + name: "CDN without supported auth falls back to public origin", + resource: testReadyResource("aliyun"), + setting: storage.Settings{Provider: "aliyun", Endpoint: "https://s3.amazonaws.com", CDNBaseURL: "https://media.example.com", AccessKeyID: "id", AccessKeySecret: "secret"}, + options: AccessOptions{Purpose: PurposeDisplay}, + wantMode: DeliveryOrigin, wantReason: "cdn_auth_unconfigured", + }, + { + name: "require CDN rejects incomplete CDN auth", + resource: testReadyResource("aliyun"), + setting: storage.Settings{Provider: "aliyun", Endpoint: "https://s3.amazonaws.com", CDNBaseURL: "https://media.example.com", AccessKeyID: "id", AccessKeySecret: "secret", Delivery: storage.DeliverySettings{RequireCDN: true}}, + options: AccessOptions{Purpose: PurposeDisplay}, + wantErr: "resource_cdn_unconfigured", + }, + { + name: "private origin requires explicit proxy", + resource: testReadyResource("aliyun"), + setting: storage.Settings{Provider: "aliyun", Endpoint: "http://storage.internal", Delivery: storage.DeliverySettings{AllowPrivateProxy: true}}, + options: AccessOptions{Purpose: PurposeDisplay}, + wantMode: DeliveryProxy, wantReason: "private_origin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var variants []ResourceVariant + access, err := ResolveAccess(tt.resource, tt.setting, tt.options, now, testPlatformURL(&variants)) + if tt.wantErr != "" { + if err == nil || !strings.Contains(string(errorReason(err)), tt.wantErr) { + t.Fatalf("ResolveAccess() error = %v, want reason %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("ResolveAccess() error = %v", err) + } + if access.Delivery != tt.wantMode || access.FallbackReason != tt.wantReason { + t.Fatalf("ResolveAccess() = %#v, want mode=%q reason=%q", access, tt.wantMode, tt.wantReason) + } + if tt.wantURL != "" && access.URL != tt.wantURL { + t.Fatalf("ResolveAccess().URL = %q, want %q", access.URL, tt.wantURL) + } + }) + } +} + +func TestResolveAccessVariantAndExpiryContract(t *testing.T) { + now := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + resource := testReadyResource("local") + resource.Kind = "video" + resource.PlaybackStatus = model.PlaybackStatusReady + resource.PlaybackObjectKey = "playback/resource-1.mp4" + var variants []ResourceVariant + access, err := ResolveAccess(resource, storage.Settings{}, AccessOptions{Purpose: PurposeProcess, Variant: VariantPlayback, ExpiresAt: now.Add(90 * time.Second)}, now, testPlatformURL(&variants)) + if err != nil { + t.Fatal(err) + } + if access.ActualVariant != VariantPlayback || access.ExpiresAt == nil || !access.ExpiresAt.Equal(now.Add(90*time.Second)) { + t.Fatalf("playback access = %#v", access) + } + if len(variants) != 1 || variants[0] != VariantPlayback { + t.Fatalf("platform URL variants = %#v, want playback", variants) + } + + provider, err := ResolveAccess(resource, storage.Settings{}, AccessOptions{Purpose: PurposeProvider}, now, testPlatformURL(&variants)) + if err != nil { + t.Fatal(err) + } + if provider.ExpiresAt == nil || !provider.ExpiresAt.Equal(now.Add(4*time.Hour)) { + t.Fatalf("provider access expiry = %v, want %v", provider.ExpiresAt, now.Add(4*time.Hour)) + } +} + +func TestResolveAccessRejectsPendingResource(t *testing.T) { + resource := testReadyResource("local") + resource.Status = model.ResourceStatusPending + _, err := ResolveAccess(resource, storage.Settings{}, AccessOptions{Purpose: PurposeDisplay}, time.Now(), func(ResourceVariant, time.Time) (string, error) { + return "", nil + }) + if err == nil || !strings.Contains(string(errorReason(err)), "resource_not_ready") { + t.Fatalf("ResolveAccess() error = %v, want resource_not_ready", err) + } +} + +func errorReason(err error) string { + if appErr, ok := err.(*kernel.AppError); ok { + return string(appErr.Reason) + } + return err.Error() +} diff --git a/backend/internal/assets/types.go b/backend/internal/assets/types.go index 15812908e..e4743f885 100644 --- a/backend/internal/assets/types.go +++ b/backend/internal/assets/types.go @@ -15,13 +15,8 @@ type ResourceStream struct { AcceptRanges string } -type ResourceDeliveryOptions struct { - ForceDirect bool - ForceProxy bool -} - type ResourceDelivery struct { - Resource *model.Resource - Stream *ResourceStream - RedirectURL string + Resource *model.Resource + Stream *ResourceStream + Access *ResourceAccess } diff --git a/backend/internal/assets/urls_test.go b/backend/internal/assets/urls_test.go index 5ed817163..8c8b1c3e0 100644 --- a/backend/internal/assets/urls_test.go +++ b/backend/internal/assets/urls_test.go @@ -8,7 +8,7 @@ func TestIDFromFileURL(t *testing.T) { want string }{ {"/api/resources/abc_123/file", "abc_123"}, - {"https://host.example/api/resources/abc_123/file?proxy=1", "abc_123"}, + {"https://host.example/api/resources/abc_123/file?variant=original", "abc_123"}, {"/api/projects/abc_123", ""}, {"", ""}, } diff --git a/backend/internal/canvas/canvas_share.go b/backend/internal/canvas/canvas_share.go index b7edfb187..ac8b10f22 100644 --- a/backend/internal/canvas/canvas_share.go +++ b/backend/internal/canvas/canvas_share.go @@ -139,47 +139,41 @@ func (s *Service) OpenSharedCanvasResource(token string, resourceID string) (*mo } func (s *Service) OpenSharedCanvasResourceRange(token string, resourceID string, rangeHeader string) (*assets.ResourceStream, error) { - userID, resource, err := s.sharedCanvasResource(token, resourceID) + userID, resource, _, err := s.sharedCanvasResource(token, resourceID) if err != nil { return nil, err } return s.host.OpenResourceRange(userID, resource, rangeHeader) } -func (s *Service) PrepareSharedCanvasResourceDelivery(token string, resourceID string, rangeHeader string) (*assets.ResourceDelivery, error) { - userID, resource, err := s.sharedCanvasResource(token, resourceID) +func (s *Service) PrepareSharedCanvasResourceDelivery(token string, resourceID string, options assets.AccessOptions, rangeHeader string) (*assets.ResourceDelivery, error) { + userID, resource, share, err := s.sharedCanvasResource(token, resourceID) if err != nil { return nil, err } - delivery, err := s.host.PrepareResourceDelivery(userID, resource, assets.ResourceDeliveryOptions{}) - if err != nil || delivery.RedirectURL != "" { - return delivery, err + if share.ExpiresAt != nil { + options.ExpiresAt = *share.ExpiresAt } - stream, err := s.host.OpenResourceRange(userID, resource, rangeHeader) - if err != nil { - return nil, err - } - delivery.Stream = stream - return delivery, nil + return s.host.PrepareResourceDelivery(userID, resource, options, rangeHeader) } -func (s *Service) sharedCanvasResource(token string, resourceID string) (string, *model.Resource, error) { +func (s *Service) sharedCanvasResource(token string, resourceID string) (string, *model.Resource, *model.CanvasShare, error) { share, project, err := s.sharedCanvasProject(token) if err != nil { - return "", nil, err + return "", nil, nil, err } _, allowedResources, err := publicCanvasProject(project, token) if err != nil { - return "", nil, err + return "", nil, nil, err } if !allowedResources[resourceID] { - return "", nil, gorm.ErrRecordNotFound + return "", nil, nil, gorm.ErrRecordNotFound } resource, err := s.repo.ResourceForUser(share.UserID, resourceID) if err != nil { - return "", nil, err + return "", nil, nil, err } - return share.UserID, resource, nil + return share.UserID, resource, share, nil } func (s *Service) sharedCanvasProject(token string) (*model.CanvasShare, *model.CanvasProject, error) { diff --git a/backend/internal/canvas/host.go b/backend/internal/canvas/host.go index b35d1d71e..95567ae7a 100644 --- a/backend/internal/canvas/host.go +++ b/backend/internal/canvas/host.go @@ -11,7 +11,7 @@ type Host interface { EncryptSecret(value string) (string, error) DecryptSecret(value string) (string, error) OpenResourceRange(userID string, resource *model.Resource, rangeHeader string) (*assets.ResourceStream, error) - PrepareResourceDelivery(userID string, resource *model.Resource, options assets.ResourceDeliveryOptions) (*assets.ResourceDelivery, error) + PrepareResourceDelivery(userID string, resource *model.Resource, options assets.AccessOptions, rangeHeader string) (*assets.ResourceDelivery, error) WithStorageLock(fn func() error) error StructuredQuota(userID, kind string, creating bool, deltaBytes int64) error StructuredReplacementQuota(userID, kind string, count int, bytes int64) error @@ -27,7 +27,7 @@ func (nopHost) DecryptSecret(value string) (string, error) { return value, nil } func (nopHost) OpenResourceRange(string, *model.Resource, string) (*assets.ResourceStream, error) { return nil, nil } -func (nopHost) PrepareResourceDelivery(string, *model.Resource, assets.ResourceDeliveryOptions) (*assets.ResourceDelivery, error) { +func (nopHost) PrepareResourceDelivery(string, *model.Resource, assets.AccessOptions, string) (*assets.ResourceDelivery, error) { return nil, nil } func (nopHost) WithStorageLock(fn func() error) error { diff --git a/backend/internal/generation/deps.go b/backend/internal/generation/deps.go index bdc309355..922bc14b6 100644 --- a/backend/internal/generation/deps.go +++ b/backend/internal/generation/deps.go @@ -2,9 +2,7 @@ package generation import ( "context" - "time" - "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/protocol" ) @@ -36,7 +34,6 @@ type ChannelLimiter interface { // MediaPersister 持久化生成结果与资源 URL。 type MediaPersister interface { PersistGeneratedMediaResult(ctx context.Context, userID, taskID string, result map[string]interface{}) (map[string]interface{}, error) - DirectResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) } // Deps 是 Engine 的外部端口集合；禁止持有组合根或回环到 service。 diff --git a/backend/internal/handler/admin_storage.go b/backend/internal/handler/admin_storage.go index 83b89459a..3ba552d2e 100644 --- a/backend/internal/handler/admin_storage.go +++ b/backend/internal/handler/admin_storage.go @@ -70,25 +70,15 @@ func RegisterAdminStorageRoutes(r *gin.RouterGroup, svc *service.Service) { failService(c, err) return } - stream, err := svc.OpenResourceRangeAsAdmin(user, c.Param("id"), c.GetHeader("Range")) + delivery, err := svc.PrepareResourceDeliveryAsAdmin(user, c.Param("id"), resourceAccessOptions(c), c.GetHeader("Range")) if err != nil { failService(c, err) return } - defer stream.Body.Close() - mimeType := stream.Resource.MimeType - if mimeType == "" { - mimeType = "application/octet-stream" - } - c.Header("Cache-Control", "private, no-cache") - c.Header("Accept-Ranges", stream.AcceptRanges) - c.Header("X-Content-Type-Options", "nosniff") - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } + disposition := "" if c.Query("download") == "1" { - c.Header("Content-Disposition", "attachment") + disposition = "attachment" } - c.DataFromReader(stream.StatusCode, stream.ContentLength, mimeType, stream.Body, nil) + serveResourceDelivery(c, delivery, "private, no-cache", disposition) }) } diff --git a/backend/internal/handler/announcement.go b/backend/internal/handler/announcement.go index 7fdaed27a..f5ea3cac6 100644 --- a/backend/internal/handler/announcement.go +++ b/backend/internal/handler/announcement.go @@ -37,24 +37,12 @@ func RegisterAnnouncementRoutes(r *gin.RouterGroup, svc *service.Service) { failService(c, err) return } - stream, err := svc.OpenAnnouncementImage(user, c.Param("id"), c.GetHeader("Range")) + delivery, err := svc.PrepareAnnouncementImageDelivery(user, c.Param("id"), resourceAccessOptions(c), c.GetHeader("Range")) if err != nil { failService(c, err) return } - defer stream.Body.Close() - mimeType := stream.Resource.MimeType - if mimeType == "" { - mimeType = "application/octet-stream" - } - c.Header("Cache-Control", "private, max-age=3600") - c.Header("Referrer-Policy", "no-referrer") - c.Header("Accept-Ranges", stream.AcceptRanges) - c.Header("X-Content-Type-Options", "nosniff") - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } - c.DataFromReader(stream.StatusCode, stream.ContentLength, mimeType, stream.Body, nil) + serveResourceDelivery(c, delivery, "private, no-cache", "") }) r.POST("/announcements/read", func(c *gin.Context) { diff --git a/backend/internal/handler/appearance.go b/backend/internal/handler/appearance.go index d301f008c..c3576788c 100644 --- a/backend/internal/handler/appearance.go +++ b/backend/internal/handler/appearance.go @@ -1,7 +1,6 @@ package handler import ( - "io" "net/http" "time" @@ -23,34 +22,12 @@ func RegisterAppearanceRoutes(r *gin.RouterGroup, svc *service.Service) { }) r.GET("/public/appearance/assets/:slot", func(c *gin.Context) { - stream, err := svc.OpenAppearanceAsset(c.Param("slot"), c.GetHeader("Range")) + delivery, err := svc.PrepareAppearanceAssetDelivery(c.Param("slot"), resourceAccessOptions(c), c.GetHeader("Range")) if err != nil { failService(c, err) return } - defer stream.Body.Close() - resource := stream.Resource - mimeType := resource.MimeType - if mimeType == "" { - mimeType = "application/octet-stream" - } - if c.Query("v") != "" { - c.Header("Cache-Control", "public, max-age=31536000, immutable") - } else { - c.Header("Cache-Control", "public, no-cache") - } - c.Header("Accept-Ranges", "bytes") - c.Header("Referrer-Policy", "no-referrer") - c.Header("X-Content-Type-Options", "nosniff") - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } - if seeker, available := stream.Body.(io.ReadSeeker); available { - c.Header("Content-Type", mimeType) - http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker) - return - } - c.DataFromReader(stream.StatusCode, stream.ContentLength, mimeType, stream.Body, nil) + serveResourceDelivery(c, delivery, "public, no-cache", "") }) r.GET("/admin/settings/appearance", func(c *gin.Context) { diff --git a/backend/internal/handler/auth.go b/backend/internal/handler/auth.go index 6ed4a6972..ddd9cc77e 100644 --- a/backend/internal/handler/auth.go +++ b/backend/internal/handler/auth.go @@ -732,49 +732,16 @@ func RegisterAdminRoutes(r *gin.RouterGroup, svc *service.Service) { failService(c, err) return } - delivery, err := svc.PrepareAdminAPICallLogMediaDelivery(user, c.Param("id"), c.GetHeader("Range")) + delivery, err := svc.PrepareAdminAPICallLogMediaDelivery(user, c.Param("id"), resourceAccessOptions(c), c.GetHeader("Range")) if err != nil { failService(c, err) return } - if delivery.RedirectURL != "" { - c.Header("Cache-Control", "private, no-store") - c.Header("Referrer-Policy", "no-referrer") - c.Header("X-Content-Type-Options", "nosniff") - c.Redirect(http.StatusTemporaryRedirect, delivery.RedirectURL) - return - } - stream := delivery.Stream - defer stream.Body.Close() - resource := stream.Resource - mimeType := resource.MimeType - if mimeType == "" { - mimeType = "application/octet-stream" - } - c.Header("Cache-Control", "private, no-cache") - c.Header("Accept-Ranges", "bytes") - c.Header("X-Content-Type-Options", "nosniff") + var disposition string if c.Query("download") == "1" { - extension := filepath.Ext(resource.ObjectKey) - if len(extension) > 12 { - extension = "" - } - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=api-log-media%s", extension)) + disposition = "attachment" } - if resource.Provider == "local" { - if seeker, ok := stream.Body.(io.ReadSeeker); ok { - c.Header("Content-Type", mimeType) - http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker) - return - } - } - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } - if stream.AcceptRanges != "" { - c.Header("Accept-Ranges", stream.AcceptRanges) - } - c.DataFromReader(stream.StatusCode, stream.ContentLength, mimeType, stream.Body, nil) + serveResourceDelivery(c, delivery, "private, no-cache", disposition) }) r.GET("/admin/api-logs/:id", func(c *gin.Context) { user, err := currentUser(c, svc) diff --git a/backend/internal/handler/canvas_share.go b/backend/internal/handler/canvas_share.go index 23e6b38eb..fcdd395ad 100644 --- a/backend/internal/handler/canvas_share.go +++ b/backend/internal/handler/canvas_share.go @@ -2,7 +2,6 @@ package handler import ( "errors" - "io" "net/http" "time" @@ -78,48 +77,14 @@ func RegisterCanvasShareRoutes(r *gin.RouterGroup, svc *service.Service) { if !enforceRateLimit(c, "public-canvas-resource:"+c.ClientIP(), 300, time.Minute) { return } - delivery, err := svc.PrepareSharedCanvasResourceDelivery(c.Param("token"), c.Param("resourceId"), c.GetHeader("Range")) + opts := resourceAccessOptions(c) + delivery, err := svc.PrepareSharedCanvasResourceDelivery(c.Param("token"), c.Param("resourceId"), opts, c.GetHeader("Range")) if err != nil { fail(c, http.StatusNotFound, errors.New("分享资源不存在")) return } - if delivery.RedirectURL != "" { - c.Header("Cache-Control", "no-store") - c.Header("Content-Security-Policy", "sandbox") - c.Header("Referrer-Policy", "no-referrer") - c.Header("X-Content-Type-Options", "nosniff") - c.Header("X-Robots-Tag", "noindex, nofollow") - c.Redirect(http.StatusTemporaryRedirect, delivery.RedirectURL) - return - } - stream := delivery.Stream - defer stream.Body.Close() - resource := stream.Resource - mimeType := resource.MimeType - if mimeType == "" { - mimeType = "application/octet-stream" - } - headers := map[string]string{ - "Cache-Control": "no-store", - "Content-Security-Policy": "sandbox", - "Referrer-Policy": "no-referrer", - "X-Content-Type-Options": "nosniff", - "X-Robots-Tag": "noindex, nofollow", - "Accept-Ranges": "bytes", - } - for key, value := range headers { - c.Header(key, value) - } - if resource.Provider == "local" { - if seeker, ok := stream.Body.(io.ReadSeeker); ok { - c.Header("Content-Type", mimeType) - http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker) - return - } - } - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } - c.DataFromReader(stream.StatusCode, stream.ContentLength, mimeType, stream.Body, nil) + c.Header("Content-Security-Policy", "sandbox") + c.Header("X-Robots-Tag", "noindex, nofollow") + serveResourceDelivery(c, delivery, "no-store", "") }) } diff --git a/backend/internal/handler/openapi.yaml b/backend/internal/handler/openapi.yaml index 1e644de14..860ccc02f 100644 --- a/backend/internal/handler/openapi.yaml +++ b/backend/internal/handler/openapi.yaml @@ -1042,6 +1042,33 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" + /resources/access: + post: + tags: [resources] + summary: 批量签发当前用户资源访问描述 + description: 统一处理展示、下载、复制、浏览器字节处理和模型输入；客户端不能通过查询参数选择 direct/proxy。每项独立成功或失败。 + requestBody: + required: true + content: + application/json: + schema: + type: array + minItems: 1 + maxItems: 100 + items: + type: object + required: [resourceId] + properties: + resourceId: {type: string} + purpose: {type: string, enum: [display, copy, download, browser-process, provider-input], default: display} + variant: {type: string, enum: [original, playback], default: original} + responses: + "200": + $ref: "#/components/responses/Envelope" + "400": + $ref: "#/components/responses/Error" + "401": + $ref: "#/components/responses/Error" /resources/{id}/file: get: tags: [resources] @@ -1051,6 +1078,16 @@ paths: name: id required: true schema: { type: string } + - in: query + name: access + schema: { type: string, enum: ["1"] } + description: 仅返回统一访问描述，不返回文件正文。 + - in: query + name: variant + schema: { type: string, enum: [original, playback], default: original } + - in: query + name: purpose + schema: { type: string, enum: [display, download, browser-process], default: display } responses: "200": description: 文件流 diff --git a/backend/internal/handler/resource_delivery.go b/backend/internal/handler/resource_delivery.go new file mode 100644 index 000000000..c307adf54 --- /dev/null +++ b/backend/internal/handler/resource_delivery.go @@ -0,0 +1,88 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "strings" + + "infinite-canvas/backend/internal/assets" + "infinite-canvas/backend/internal/service" + + "github.com/gin-gonic/gin" +) + +func resourceAccessOptions(c *gin.Context) service.ResourceAccessOptions { + purpose := assets.AccessPurpose(strings.TrimSpace(c.Query("purpose"))) + if purpose == "" && c.Query("download") == "1" { + purpose = assets.PurposeDownload + } + variant := assets.ResourceVariant(strings.TrimSpace(c.Query("variant"))) + return service.ResourceAccessOptions{ + Purpose: purpose, + Variant: variant, + DescribeOnly: c.Query("access") == "1", + } +} + +// serveResourceDelivery is the only HTTP byte/redirect executor. Authorization and +// delivery selection happen in app; this function only applies the already-resolved +// contract to an HTTP response. +func serveResourceDelivery(c *gin.Context, delivery *service.ResourceDelivery, cacheControl string, disposition string) { + if delivery == nil || delivery.Access == nil { + fail(c, http.StatusServiceUnavailable, fmt.Errorf("资源分发结果无效")) + return + } + access := delivery.Access + if access.URL == "" && delivery.Stream == nil { + fail(c, http.StatusServiceUnavailable, fmt.Errorf("资源分发结果无效")) + return + } + if c.Query("access") == "1" { + c.Header("Cache-Control", "private, no-store") + c.Header("Referrer-Policy", "no-referrer") + ok(c, gin.H{"access": access}) + return + } + c.Header("Referrer-Policy", "no-referrer") + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Accept-Ranges", "bytes") + if access.Delivery == assets.DeliveryCDN || access.Delivery == assets.DeliveryOrigin { + c.Header("Cache-Control", "private, no-store") + // Do not let Gin generate its default HTML redirect body. A media request + // should transfer only the redirect metadata; the actual bytes must be + // fetched from the CDN/origin selected by the access contract. + c.Header("Location", access.URL) + c.Status(http.StatusTemporaryRedirect) + return + } + stream := delivery.Stream + if stream == nil || stream.Body == nil { + fail(c, http.StatusServiceUnavailable, fmt.Errorf("资源流不可用")) + return + } + defer stream.Body.Close() + if cacheControl != "" { + c.Header("Cache-Control", cacheControl) + } + if disposition != "" { + c.Header("Content-Disposition", disposition) + } + resource := stream.Resource + mimeType := resource.MimeType + if mimeType == "" { + mimeType = "application/octet-stream" + } + if stream.ContentRange != "" { + c.Header("Content-Range", stream.ContentRange) + } + if stream.AcceptRanges != "" { + c.Header("Accept-Ranges", stream.AcceptRanges) + } + if seeker, ok := stream.Body.(io.ReadSeeker); ok { + c.Header("Content-Type", mimeType) + http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker) + return + } + c.DataFromReader(stream.StatusCode, stream.ContentLength, mimeType, stream.Body, nil) +} diff --git a/backend/internal/handler/resource_delivery_test.go b/backend/internal/handler/resource_delivery_test.go new file mode 100644 index 000000000..c4f38908b --- /dev/null +++ b/backend/internal/handler/resource_delivery_test.go @@ -0,0 +1,96 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "infinite-canvas/backend/internal/assets" + "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/service" + + "github.com/gin-gonic/gin" +) + +func TestServeResourceDeliveryRedirectsCloudMediaWithoutWritingBody(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.GET("/resource", func(c *gin.Context) { + serveResourceDelivery(c, &service.ResourceDelivery{Access: &assets.ResourceAccess{ + ResourceID: "resource-1", + URL: "https://cdn.example.com/users/user-1/image/test.png?signature=redacted", + Delivery: assets.DeliveryCDN, + }}, "private, no-cache", "") + }) + + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/resource", nil)) + + if recorder.Code != http.StatusTemporaryRedirect { + t.Fatalf("status = %d, want 307", recorder.Code) + } + if got := recorder.Header().Get("Location"); got != "https://cdn.example.com/users/user-1/image/test.png?signature=redacted" { + t.Fatalf("location = %q", got) + } + if recorder.Body.Len() != 0 { + t.Fatalf("redirect wrote %d body bytes; platform must not send cloud media", recorder.Body.Len()) + } +} + +func TestServeResourceDeliveryReturnsAccessDescriptorWithoutRedirect(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.GET("/resource", func(c *gin.Context) { + serveResourceDelivery(c, &service.ResourceDelivery{Access: &assets.ResourceAccess{ + ResourceID: "resource-1", + URL: "https://cdn.example.com/test.png", + Delivery: assets.DeliveryCDN, + }}, "private, no-cache", "") + }) + + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/resource?access=1", nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", recorder.Code) + } + if got := recorder.Header().Get("Cache-Control"); got != "private, no-store" { + t.Fatalf("cache-control = %q", got) + } + if !strings.Contains(recorder.Body.String(), `"url":"https://cdn.example.com/test.png"`) { + t.Fatalf("access descriptor missing from body: %s", recorder.Body.String()) + } +} + +func TestServeResourceDeliveryStreamsPlatformBytesAndRangeMetadata(t *testing.T) { + gin.SetMode(gin.TestMode) + resource := &model.Resource{ID: "resource-1", MimeType: "video/mp4", UpdatedAt: time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC)} + router := gin.New() + router.GET("/resource", func(c *gin.Context) { + serveResourceDelivery(c, &service.ResourceDelivery{ + Resource: resource, + Access: &assets.ResourceAccess{ResourceID: resource.ID, URL: "/api/resources/resource-1/file", Delivery: assets.DeliveryProxy}, + Stream: &assets.ResourceStream{Resource: resource, Body: io.NopCloser(strings.NewReader("partial-media")), StatusCode: http.StatusPartialContent, ContentLength: 13, ContentRange: "bytes 10-22/100", AcceptRanges: "bytes"}, + }, "private, no-cache", `inline; filename="video.mp4"`) + }) + + recorder := httptest.NewRecorder() + recorder.HeaderMap = make(http.Header) + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/resource", nil)) + + if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "partial-media" { + t.Fatalf("response = status %d body %q", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Content-Range") != "bytes 10-22/100" || recorder.Header().Get("Accept-Ranges") != "bytes" { + t.Fatalf("range headers missing: %#v", recorder.Header()) + } + if recorder.Header().Get("Content-Type") != "video/mp4" { + t.Fatalf("content-type = %q", recorder.Header().Get("Content-Type")) + } + if recorder.Header().Get("Content-Disposition") != `inline; filename="video.mp4"` { + t.Fatalf("content-disposition = %q", recorder.Header().Get("Content-Disposition")) + } +} diff --git a/backend/internal/handler/user_data.go b/backend/internal/handler/user_data.go index e66d6e123..6e1db987e 100644 --- a/backend/internal/handler/user_data.go +++ b/backend/internal/handler/user_data.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "log" "net/http" "strconv" @@ -18,6 +17,25 @@ import ( ) func RegisterUserDataRoutes(r *gin.RouterGroup, svc *service.Service) { + r.POST("/resources/access", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10) + var req []service.ResourceAccessRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + result, err := svc.ResourceAccessBatch(user.ID, req) + if err != nil { + failService(c, err) + return + } + ok(c, gin.H{"items": result}) + }) r.POST("/assets/batch-delete", func(c *gin.Context) { user, err := currentUser(c, svc) if err != nil { @@ -277,154 +295,28 @@ func RegisterUserDataRoutes(r *gin.RouterGroup, svc *service.Service) { } ok(c, gin.H{"resource": resource}) }) - r.GET("/resources/:id/oss-url", func(c *gin.Context) { - user, err := currentUser(c, svc) - if err != nil { - failService(c, err) - return - } - resource, err := svc.Resource(user.ID, c.Param("id")) - if err != nil { - fail(c, http.StatusNotFound, err) - return - } - ossURL, err := svc.DirectResourceURL(user.ID, resource.ID) - if err != nil { - failService(c, err) - return - } - // 签名地址只用于当前复制动作，禁止浏览器或中间代理缓存。 - c.Header("Cache-Control", "private, no-store") - c.Header("Referrer-Policy", "no-referrer") - ok(c, gin.H{"url": ossURL}) - }) r.GET("/resources/:id/file", func(c *gin.Context) { user, err := currentUser(c, svc) if err != nil { failService(c, err) return } - delivery, err := svc.PrepareResourceDelivery(user.ID, c.Param("id"), service.ResourceDeliveryOptions{ - ForceDirect: c.Query("direct") == "1", - ForceProxy: c.Query("proxy") == "1", - }) + options := resourceAccessOptions(c) + delivery, err := svc.PrepareResourceDelivery(user.ID, c.Param("id"), options, c.GetHeader("Range")) if err != nil { failService(c, err) return } - if delivery.RedirectURL != "" { - // 签名链接仅有效 5 分钟；重定向缓存必须短于签名 TTL，避免命中过期地址。 - c.Header("Cache-Control", "private, max-age=240") - c.Header("Referrer-Policy", "no-referrer") - c.Header("X-Content-Type-Options", "nosniff") - c.Redirect(http.StatusTemporaryRedirect, delivery.RedirectURL) - return - } - resource := delivery.Resource - etag := resourceResponseETag(resource) - // variant=playback：serve 浏览器兼容播放副本（H.265→H.264 转码）。 - // 副本就绪时用独立 ETag 后缀，避免浏览器拿原件缓存命中 304 而继续黑屏。 - usePlayback := c.Query("variant") == "playback" && resource.Provider == "local" && - resource.PlaybackStatus == model.PlaybackStatusReady && resource.PlaybackObjectKey != "" - serveETag := etag - if usePlayback { - serveETag = etag + ":pb" - } - // 资源 ID 内容不可变（上传永远生成新 ID，不会原地覆盖）：图片可以放心交给浏览器 - // 磁盘强缓存 30 天，大画布二次打开零请求直读磁盘缓存。视频/音频涉及转码副本 - // 就绪与 Range 语义，保持逐次条件请求（304）。 - if strings.HasPrefix(resource.MimeType, "image/") { - c.Header("Cache-Control", "private, max-age=2592000, stale-while-revalidate=86400") - } else { - c.Header("Cache-Control", "private, no-cache") - } - c.Header("ETag", serveETag) - c.Header("Accept-Ranges", "bytes") - c.Header("X-Content-Type-Options", "nosniff") - if resource.Kind == "file" { - c.Header("Content-Disposition", "attachment") - c.Header("Content-Security-Policy", "sandbox") - } - if ifNoneMatch(c.GetHeader("If-None-Match"), serveETag) { - c.Status(http.StatusNotModified) - return - } - rangeHeader := c.GetHeader("Range") - if ifRange := strings.TrimSpace(c.GetHeader("If-Range")); ifRange != "" && ifRange != serveETag { - rangeHeader = "" - } - var stream *service.ResourceStream - if usePlayback { - stream, err = svc.OpenResourcePlaybackRange(user.ID, resource.ID) - if err == nil { - resource = stream.Resource // MimeType 已置 video/mp4 - } else if errors.Is(err, service.ErrPlaybackNotReady) { - // 副本尚未就绪：回退原件，并撤销 :pb 后缀，保证副本就绪后 - // 浏览器不会拿原件缓存命中 304 而继续黑屏。 - c.Header("ETag", etag) - stream, err = svc.OpenResourceRange(user.ID, resource.ID, rangeHeader) - if err != nil { - failService(c, err) - return - } - } else { - failService(c, err) - return - } - } else { - stream, err = svc.OpenResourceRange(user.ID, resource.ID, rangeHeader) - if err != nil { - failService(c, err) - return - } - } - if err != nil { - failService(c, err) - return - } - defer stream.Body.Close() - if resource.MimeType == "" { - resource.MimeType = "application/octet-stream" - } - if resource.Provider == "local" { - if seeker, ok := stream.Body.(io.ReadSeeker); ok { - c.Header("Content-Type", resource.MimeType) - http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker) - return - } - } - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } - if stream.AcceptRanges != "" { - c.Header("Accept-Ranges", stream.AcceptRanges) - } - c.DataFromReader(stream.StatusCode, stream.ContentLength, resource.MimeType, stream.Body, nil) + serveResourceDelivery(c, delivery, "private, no-cache", "") }) publicResourceHandler := func(c *gin.Context) { - stream, err := svc.OpenPublicResourceRange(c.Param("id"), c.Query("expires"), c.Query("signature"), c.GetHeader("Range")) + options := resourceAccessOptions(c) + delivery, err := svc.PreparePublicResourceDelivery(c.Param("id"), c.Query("expires"), c.Query("signature"), options, c.GetHeader("Range")) if err != nil { failService(c, err) return } - defer stream.Body.Close() - resource := stream.Resource - if resource.MimeType == "" { - resource.MimeType = "application/octet-stream" - } - c.Header("Cache-Control", "public, max-age=0, must-revalidate") - c.Header("Accept-Ranges", "bytes") - c.Header("Referrer-Policy", "no-referrer") - c.Header("X-Content-Type-Options", "nosniff") - if stream.ContentRange != "" { - c.Header("Content-Range", stream.ContentRange) - } - if seeker, ok := stream.Body.(io.ReadSeeker); ok { - c.Header("Content-Type", resource.MimeType) - http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker) - return - } - c.DataFromReader(stream.StatusCode, stream.ContentLength, resource.MimeType, stream.Body, nil) + serveResourceDelivery(c, delivery, "public, max-age=0, must-revalidate", "") } r.GET("/public/resources/:id/file", publicResourceHandler) r.GET("/public/resources/:id/file/:filename", publicResourceHandler) diff --git a/backend/internal/service/aliases_types.go b/backend/internal/service/aliases_types.go index db687daf6..57042b836 100644 --- a/backend/internal/service/aliases_types.go +++ b/backend/internal/service/aliases_types.go @@ -253,7 +253,9 @@ type ( ResolveBillingBatchResult = app.ResolveBillingBatchResult ResolveBillingRequest = app.ResolveBillingRequest ResourceDelivery = app.ResourceDelivery - ResourceDeliveryOptions = app.ResourceDeliveryOptions + ResourceAccessOptions = app.ResourceAccessOptions + ResourceAccessRequest = app.ResourceAccessRequest + ResourceAccess = app.ResourceAccess ResourceStream = app.ResourceStream ResponseInterceptionRule = app.ResponseInterceptionRule ResponseInterceptionSetting = app.ResponseInterceptionSetting diff --git a/backend/internal/storage/delivery.go b/backend/internal/storage/delivery.go new file mode 100644 index 000000000..8243f2e18 --- /dev/null +++ b/backend/internal/storage/delivery.go @@ -0,0 +1,49 @@ +package storage + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "time" +) + +// DeliverySettings is independent from origin credentials. Public CDN access must be an explicit choice. +type DeliverySettings struct { + CDNAuthMode string `json:"cdnAuthMode"` + RequireCDN bool `json:"requireCDN"` + AllowPrivateProxy bool `json:"allowPrivateProxy"` +} + +func CDNEnabled(setting Settings) bool { + if setting.CDNBaseURL == "" { + return false + } + return setting.Delivery.CDNAuthMode == "public" || setting.Delivery.CDNAuthMode == "qiniu" +} + +func SignCDNURL(setting Settings, objectKey string, expires time.Time) (string, error) { + switch setting.Delivery.CDNAuthMode { + case "public": + return OssCDNObjectURL(setting.CDNBaseURL, objectKey) + case "qiniu": + if setting.Provider == qiniuKodoProvider { + return SignedQiniuObjectURL(setting, objectKey, expires) + } + } + return "", errors.New("CDN 用户访问鉴权方式未配置或不支持") +} + +func PublicOrigin(setting Settings) bool { + if setting.Provider == qiniuKodoProvider { + return true + } // The SDK signs the public Kodo S3 origin, not its upload endpoint. + return PublicHTTPSStorageEndpoint(setting.Endpoint) +} + +func DeliveryRevision(setting Settings) string { + // Only a digest leaves the server; changes to credentials/configuration invalidate descriptors. + data, _ := json.Marshal(setting) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:8]) +} diff --git a/backend/internal/storage/objects.go b/backend/internal/storage/objects.go new file mode 100644 index 000000000..0d5174ee0 --- /dev/null +++ b/backend/internal/storage/objects.go @@ -0,0 +1,519 @@ +package storage + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws/credentials" + awsv4 "github.com/aws/aws-sdk-go/aws/signer/v4" + qiniuAuth "github.com/qiniu/go-sdk/v7/auth" + qiniuStorage "github.com/qiniu/go-sdk/v7/storage" + cos "github.com/tencentyun/cos-go-sdk-v5" + "infinite-canvas/backend/internal/kernel" + "infinite-canvas/backend/internal/outbound" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +func PutOSSObject(setting Settings, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { + setting = NormalizeSettings(setting) + if setting.Provider == tencentCOSProvider { + return PutCOSObject(setting, objectKey, mimeType, size, body) + } + if setting.Provider == qiniuKodoProvider { + return PutQiniuObject(setting, objectKey, mimeType, size, body) + } + if setting.Provider == s3Provider { + return PutS3Object(setting, objectKey, mimeType, size, body) + } + return PutAliyunOSSObject(setting, objectKey, mimeType, size, body) +} + +// 阿里云 OSS 继续沿用原有 V1 签名和请求路径，避免已有部署行为发生变化。 +func PutAliyunOSSObject(setting Settings, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { + if mimeType == "" { + mimeType = "application/octet-stream" + } + req, err := NewOSSRequest(http.MethodPut, setting, objectKey, mimeType, body) + if err != nil { + return "", err + } + if size > 0 { + req.ContentLength = size + } + resp, err := outbound.OutboundHTTPClient(2 * time.Minute).Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return "", fmt.Errorf("OSS 上传失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) + } + return strings.Trim(resp.Header.Get("ETag"), `"`), nil +} + +type ObjectStream struct { + Body io.ReadCloser + StatusCode int + ContentLength int64 + ContentRange string + AcceptRanges string +} + +func GetOSSObjectRange(setting Settings, objectKey string, rangeHeader string) (*ObjectStream, error) { + setting = NormalizeSettings(setting) + if setting.Provider == s3Provider { + return GetS3ObjectRange(setting, objectKey, rangeHeader) + } + if setting.Provider == tencentCOSProvider { + return GetCOSObjectRange(setting, objectKey, rangeHeader) + } + if setting.Provider == qiniuKodoProvider { + return GetQiniuObjectRange(setting, objectKey, rangeHeader) + } + return GetAliyunOSSObjectRange(setting, objectKey, rangeHeader) +} + +func GetAliyunOSSObjectRange(setting Settings, objectKey string, rangeHeader string) (*ObjectStream, error) { + req, err := NewOSSRequest(http.MethodGet, setting, objectKey, "", nil) + if err != nil { + return nil, err + } + if rangeHeader != "" { + req.Header.Set("Range", rangeHeader) + } + resp, err := outbound.OutboundHTTPClient(2 * time.Minute).Do(req) + if err != nil { + return nil, err + } + if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { + defer resp.Body.Close() + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("OSS 读取失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) + } + return &ObjectStream{Body: resp.Body, StatusCode: resp.StatusCode, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), AcceptRanges: kernel.FirstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil +} + +func SignedOSSObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + setting = NormalizeSettings(setting) + if setting.Provider == s3Provider { + return SignedS3ObjectURL(setting, objectKey, expiresAt) + } + if setting.Provider == qiniuKodoProvider { + return SignedQiniuS3ObjectURL(setting, objectKey, expiresAt) + } + if setting.Provider == tencentCOSProvider { + return SignedCOSObjectURL(setting, objectKey, expiresAt) + } + return SignedAliyunOSSObjectURL(setting, objectKey, expiresAt) +} + +func SignedAliyunOSSObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + baseURL, err := OssBucketBaseURL(setting) + if err != nil { + return "", err + } + if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" { + return "", errors.New("OSS 访问密钥不可用") + } + objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") + if objectKey == "" { + return "", errors.New("OSS 对象路径为空") + } + baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/" + strings.TrimLeft(objectKey, "/") + expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10) + stringToSign := strings.Join([]string{http.MethodGet, "", "", expires, "/" + setting.Bucket + "/" + objectKey}, "\n") + mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret)) + _, _ = mac.Write([]byte(stringToSign)) + query := baseURL.Query() + query.Set("OSSAccessKeyId", setting.AccessKeyID) + query.Set("Expires", expires) + query.Set("Signature", base64.StdEncoding.EncodeToString(mac.Sum(nil))) + baseURL.RawQuery = query.Encode() + return baseURL.String(), nil +} + +func PutCOSObject(setting Settings, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { + client, err := NewCOSClient(setting, 2*time.Minute) + if err != nil { + return "", err + } + if mimeType == "" { + mimeType = "application/octet-stream" + } + options := &cos.ObjectPutOptions{ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ContentType: mimeType, ContentLength: size}} + resp, err := client.Object.Put(context.Background(), objectKey, body, options) + if err != nil { + return "", fmt.Errorf("COS 上传失败：%w", err) + } + return strings.Trim(resp.Header.Get("ETag"), `"`), nil +} + +func PutQiniuObject(setting Settings, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { + if setting.AccessKeyID == "" || setting.AccessKeySecret == "" { + return "", errors.New("七牛云 Kodo 访问密钥不可用") + } + if setting.Bucket == "" || objectKey == "" { + return "", errors.New("七牛云 Kodo Bucket 或对象路径为空") + } + config := qiniuStorage.NewConfig() + config.Region = QiniuRegion(setting.Region) + config.UpHost = strings.TrimRight(setting.Endpoint, "/") + uploader := qiniuStorage.NewFormUploader(config) + policy := qiniuStorage.PutPolicy{Scope: setting.Bucket + ":" + strings.TrimLeft(objectKey, "/"), Expires: 3600} + token := policy.UploadToken(qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret)) + ret := qiniuStorage.PutRet{} + extra := &qiniuStorage.PutExtra{MimeType: mimeType} + if size < 0 { + size = 0 + } + if err := uploader.Put(context.Background(), &ret, token, strings.TrimLeft(objectKey, "/"), body, size, extra); err != nil { + return "", fmt.Errorf("七牛云 Kodo 上传失败：%w", err) + } + return ret.Hash, nil +} + +func GetCOSObjectRange(setting Settings, objectKey string, rangeHeader string) (*ObjectStream, error) { + client, err := NewCOSClient(setting, 2*time.Minute) + if err != nil { + return nil, err + } + options := &cos.ObjectGetOptions{Range: rangeHeader} + resp, err := client.Object.Get(context.Background(), objectKey, options) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + return &ObjectStream{Body: io.NopCloser(bytes.NewReader(nil)), StatusCode: resp.StatusCode, ContentRange: resp.Header.Get("Content-Range"), AcceptRanges: kernel.FirstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil + } + return nil, fmt.Errorf("COS 读取失败：%w", err) + } + return &ObjectStream{Body: resp.Body, StatusCode: resp.StatusCode, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), AcceptRanges: kernel.FirstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil +} + +func GetQiniuObjectRange(setting Settings, objectKey string, rangeHeader string) (*ObjectStream, error) { + signedURL, err := SignedQiniuS3ObjectURL(setting, objectKey, time.Now().Add(resourceAccessURLTTL)) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodGet, signedURL, nil) + if err != nil { + return nil, err + } + if rangeHeader != "" { + req.Header.Set("Range", rangeHeader) + } + outbound.ApplyDefaultOutboundHeaders(req) + resp, err := outbound.OutboundHTTPClient(2 * time.Minute).Do(req) + if err != nil { + return nil, fmt.Errorf("七牛云 Kodo 读取失败：%w", err) + } + if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { + defer resp.Body.Close() + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("七牛云 Kodo 读取失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) + } + return &ObjectStream{Body: resp.Body, StatusCode: resp.StatusCode, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), AcceptRanges: kernel.FirstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil +} + +func SignedCOSObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" { + return "", errors.New("COS 访问密钥不可用") + } + objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") + if objectKey == "" { + return "", errors.New("COS 对象路径为空") + } + expires := time.Until(expiresAt) + if expires <= 0 { + return "", errors.New("COS 签名有效期必须晚于当前时间") + } + client, err := NewCOSClient(setting, 2*time.Minute) + if err != nil { + return "", err + } + signedURL, err := client.Object.GetPresignedURL(context.Background(), http.MethodGet, objectKey, setting.AccessKeyID, setting.AccessKeySecret, expires, nil) + if err != nil { + return "", err + } + return signedURL.String(), nil +} + +func SignedQiniuObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + if setting.AccessKeyID == "" || setting.AccessKeySecret == "" { + return "", errors.New("七牛云 Kodo 访问密钥不可用") + } + objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") + if objectKey == "" { + return "", errors.New("七牛云 Kodo 对象路径为空") + } + deadline := expiresAt.Unix() + if deadline <= time.Now().Unix() { + return "", errors.New("七牛云 Kodo 签名有效期必须晚于当前时间") + } + if setting.CDNBaseURL == "" { + return SignedQiniuS3ObjectURL(setting, objectKey, expiresAt) + } + mac := qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret) + return qiniuStorage.MakePrivateURLv2(mac, strings.TrimRight(setting.CDNBaseURL, "/"), objectKey, deadline), nil +} + +// SignedQiniuS3ObjectURL 用七牛兼容 S3 的 AWS Signature V4 访问私有空间。 +// 没有绑定域名时，浏览器不直接访问该地址，而是由后端代理读取并返回文件。 +func SignedQiniuS3ObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + region := QiniuS3Region(setting) + if region == "" { + return "", errors.New("七牛云 Kodo S3 Region 不可用") + } + baseURL := &url.URL{Scheme: "https", Host: setting.Bucket + ".s3." + region + ".qiniucs.com"} + // 保留对象键的原始路径，让 url.URL 和 AWS signer 只做一次 RFC 3986 转义。 + baseURL.Path = "/" + objectKey + req, err := http.NewRequest(http.MethodGet, baseURL.String(), nil) + if err != nil { + return "", err + } + credentialsValue := credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, "") + signer := awsv4.NewSigner(credentialsValue) + if _, err := signer.Presign(req, nil, "s3", region, time.Until(expiresAt), time.Now().UTC()); err != nil { + return "", fmt.Errorf("七牛云 Kodo S3 签名失败：%w", err) + } + return req.URL.String(), nil +} + +func QiniuS3Region(setting Settings) string { + region := strings.ToLower(strings.TrimSpace(setting.Region)) + if region == "" { + endpoint := strings.ToLower(setting.Endpoint) + for _, candidate := range []string{"z0", "z1", "z2", "na0", "as0", "cn-east-1", "cn-north-1", "cn-south-1", "us-north-1", "ap-southeast-1", "cn-east-2"} { + if strings.Contains(endpoint, candidate) { + region = candidate + break + } + } + } + switch region { + case "", "z0", "cn-east-1": + return "cn-east-1" + case "z1", "cn-north-1": + return "cn-north-1" + case "z2", "cn-south-1": + return "cn-south-1" + case "na0", "us-north-1": + return "us-north-1" + case "as0", "ap-southeast-1": + return "ap-southeast-1" + case "cn-east-2", "zhejiang2": + return "cn-east-2" + default: + return "" + } +} + +func QiniuRegion(region string) *qiniuStorage.Region { + switch strings.ToLower(strings.TrimSpace(region)) { + case "z1", "cn-north-1": + return &qiniuStorage.ZoneHuabei + case "z2", "cn-south-1": + return &qiniuStorage.ZoneHuanan + case "na0", "us-north-1": + return &qiniuStorage.ZoneBeimei + case "as0", "ap-southeast-1": + return &qiniuStorage.ZoneXinjiapo + case "cn-east-2", "zhejiang2": + return &qiniuStorage.ZoneHuadongZheJiang2 + default: + return &qiniuStorage.ZoneHuadong + } +} + +func NewCOSClient(setting Settings, timeout time.Duration) (*cos.Client, error) { + bucketURL, err := CosBucketBaseURL(setting) + if err != nil { + return nil, err + } + httpClient := outbound.OutboundHTTPClient(timeout) + httpClient.Transport = &cos.AuthorizationTransport{SecretID: setting.AccessKeyID, SecretKey: setting.AccessKeySecret, Transport: httpClient.Transport} + return cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, httpClient), nil +} + +func CosBucketBaseURL(setting Settings) (*url.URL, error) { + setting = NormalizeSettings(setting) + endpoint := strings.TrimRight(setting.Endpoint, "/") + if endpoint == "" { + return nil, errors.New("COS Endpoint 为空") + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + parsed, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + if parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" { + return nil, errors.New("COS Endpoint 格式不正确") + } + if setting.Bucket == "" { + return nil, errors.New("COS Bucket 为空") + } + host := strings.ToLower(parsed.Hostname()) + if strings.HasSuffix(host, ".myqcloud.com") || strings.HasSuffix(host, ".tencentcos.cn") { + if strings.HasPrefix(host, "cos.") || strings.HasPrefix(host, "cos-internal.") || strings.HasPrefix(host, "cos-website.") { + parsed.Host = setting.Bucket + "." + parsed.Host + } else if !strings.HasPrefix(host, strings.ToLower(setting.Bucket)+".") { + return nil, errors.New("COS Endpoint 中的 Bucket 与配置不一致") + } + } + return parsed, nil +} + +func OssCDNBaseURL(raw string) (*url.URL, error) { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.Hostname() == "" { + return nil, errors.New("对象存储 CDN 加速域名格式不正确") + } + if parsed.Scheme != "https" && parsed.Scheme != "http" { + return nil, errors.New("对象存储 CDN 加速域名只支持 http/https") + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" { + return nil, errors.New("对象存储 CDN 加速域名不能包含认证信息、路径、查询参数或片段") + } + parsed.Path = "" + return parsed, nil +} + +func OssCDNObjectURL(raw string, objectKey string) (string, error) { + baseURL, err := OssCDNBaseURL(raw) + if err != nil { + return "", err + } + objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/") + if objectKey == "" { + return "", errors.New("对象存储对象路径为空") + } + // CDN 使用自己的访问鉴权与私有桶回源鉴权，不能携带 OSS/COS 的预签名参数。 + // url.URL.String 会负责转义 Path；这里保留未转义值，避免把 %20 再编码为 %2520。 + baseURL.Path = "/" + objectKey + return baseURL.String(), nil +} + +func NewOSSRequest(method string, setting Settings, objectKey string, contentType string, body io.Reader) (*http.Request, error) { + baseURL, err := OssBucketBaseURL(setting) + if err != nil { + return nil, err + } + baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/" + strings.TrimLeft(objectKey, "/") + // 请求体必须用 no-op close 包装：服务端提前返回（如 OSS 签名 403）时 + // http.Transport 会关闭未发完的 Request.Body，若直接传入 *os.File 等 + // 调用方持有的文件，后续“降级本地存储”的 Seek 重读将因 file already + // closed 失败。NopCloser 让 Transport 的关闭成为空操作，底层文件保持可用。 + // GET/HEAD/DELETE 等无请求体的调用传入 nil body；NopCloser(nil) 会产生非 nil 的 + // Body 包装 nil reader，Go 1.26 发送前 body 探测会直接 nil 解引用崩溃。 + var reqBody io.Reader + if body != nil { + reqBody = io.NopCloser(body) + } + req, err := http.NewRequest(method, baseURL.String(), reqBody) + if err != nil { + return nil, err + } + date := time.Now().UTC().Format(http.TimeFormat) + req.Header.Set("Date", date) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + stringToSign := strings.Join([]string{method, "", contentType, date, "/" + setting.Bucket + "/" + objectKey}, "\n") + mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret)) + _, _ = mac.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + req.Header.Set("Authorization", "OSS "+setting.AccessKeyID+":"+signature) + return req, nil +} + +func OssBucketBaseURL(setting Settings) (*url.URL, error) { + endpoint := strings.TrimRight(setting.Endpoint, "/") + if endpoint == "" { + return nil, errors.New("OSS Endpoint 为空") + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + parsed, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + if parsed.Host == "" { + return nil, errors.New("OSS Endpoint 格式不正确") + } + if !strings.HasPrefix(parsed.Host, setting.Bucket+".") { + parsed.Host = setting.Bucket + "." + parsed.Host + } + return parsed, nil +} + +func EscapeObjectKey(key string) string { + parts := strings.Split(key, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +func DeleteAliyunOSSObject(setting Settings, objectKey string) error { + req, err := NewOSSRequest(http.MethodDelete, setting, objectKey, "", nil) + if err != nil { + return err + } + resp, err := outbound.OutboundHTTPClient(2 * time.Minute).Do(req) + if err != nil { + return fmt.Errorf("删除阿里云 OSS 对象失败：%w", err) + } + defer resp.Body.Close() + if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusNotFound { + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return fmt.Errorf("删除阿里云 OSS 对象失败：%s %s", resp.Status, strings.TrimSpace(string(detail))) + } + return nil +} + +func DeleteTencentCOSObject(setting Settings, objectKey string) error { + client, err := NewCOSClient(setting, 2*time.Minute) + if err != nil { + return err + } + resp, err := client.Object.Delete(context.Background(), objectKey) + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil + } + return fmt.Errorf("删除腾讯云 COS 对象失败：%w", err) + } + return nil +} + +func DeleteQiniuObject(setting Settings, objectKey string) error { + if setting.AccessKeyID == "" || setting.AccessKeySecret == "" { + return errors.New("七牛云 Kodo 访问密钥不可用") + } + if setting.Bucket == "" || strings.TrimSpace(objectKey) == "" { + return errors.New("七牛云 Kodo Bucket 或对象路径为空") + } + mac := qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret) + manager := qiniuStorage.NewBucketManager(mac, &qiniuStorage.Config{Region: QiniuRegion(setting.Region), UseHTTPS: true}) + if err := manager.Delete(setting.Bucket, strings.TrimLeft(objectKey, "/")); err != nil { + if strings.Contains(strings.ToLower(err.Error()), "no such") || strings.Contains(strings.ToLower(err.Error()), "not found") { + return nil + } + return fmt.Errorf("删除七牛云 Kodo 对象失败：%w", err) + } + return nil +} diff --git a/backend/internal/app/storage_s3.go b/backend/internal/storage/s3.go similarity index 66% rename from backend/internal/app/storage_s3.go rename to backend/internal/storage/s3.go index 418d1db03..e278286c2 100644 --- a/backend/internal/app/storage_s3.go +++ b/backend/internal/storage/s3.go @@ -1,10 +1,12 @@ -package app +package storage import ( "bytes" "context" "errors" "fmt" + "infinite-canvas/backend/internal/kernel" + "infinite-canvas/backend/internal/outbound" "io" "net/http" "net/url" @@ -18,21 +20,21 @@ import ( awss3 "github.com/aws/aws-sdk-go/service/s3" ) -func validateStorageEndpoint(raw string) (*url.URL, error) { - parsed, err := ValidateOutboundURL(raw) +func ValidateStorageEndpoint(raw string) (*url.URL, error) { + parsed, err := outbound.ValidateOutboundURL(raw) if err != nil { return nil, err } if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" { - return nil, BadAuthRequest("对象存储 Endpoint 必须是服务根 URL，不能包含认证信息、路径、查询参数或片段") + return nil, kernel.BadAuthRequest("对象存储 Endpoint 必须是服务根 URL，不能包含认证信息、路径、查询参数或片段") } - if parsed.Scheme == "http" && !AllowedPrivateUpstreamHost(parsed.Hostname()) { - return nil, BadAuthRequest("对象存储 HTTP Endpoint 仅允许访问 CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS 精确放行的主机") + if parsed.Scheme == "http" && !outbound.AllowedPrivateUpstreamHost(parsed.Hostname()) { + return nil, kernel.BadAuthRequest("对象存储 HTTP Endpoint 仅允许访问 CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS 精确放行的主机") } return parsed, nil } -func standardAWSS3Endpoint(endpoint string) bool { +func StandardAWSS3Endpoint(endpoint string) bool { parsed, err := url.Parse(endpoint) if err != nil { return false @@ -41,30 +43,30 @@ func standardAWSS3Endpoint(endpoint string) bool { return host == "s3.amazonaws.com" || strings.HasPrefix(host, "s3.") && (strings.HasSuffix(host, ".amazonaws.com") || strings.HasSuffix(host, ".amazonaws.com.cn")) || strings.HasPrefix(host, "s3-") && strings.HasSuffix(host, ".amazonaws.com") } -func publicHTTPSStorageEndpoint(endpoint string) bool { +func PublicHTTPSStorageEndpoint(endpoint string) bool { parsed, err := url.Parse(strings.TrimSpace(endpoint)) if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" { return false } - return ValidateOutboundHost(parsed.Hostname()) == nil && !AllowedPrivateUpstreamHost(parsed.Hostname()) + return outbound.ValidateOutboundHost(parsed.Hostname()) == nil && !outbound.AllowedPrivateUpstreamHost(parsed.Hostname()) } -func newS3Client(setting ossSettingValue, timeout time.Duration) (*awss3.S3, error) { - setting = normalizeOSSSetting(setting) - endpoint, err := validateStorageEndpoint(setting.Endpoint) +func NewS3Client(setting Settings, timeout time.Duration) (*awss3.S3, error) { + setting = NormalizeSettings(setting) + endpoint, err := ValidateStorageEndpoint(setting.Endpoint) if err != nil { return nil, err } if setting.Region == "" || setting.Bucket == "" || setting.AccessKeyID == "" || setting.AccessKeySecret == "" { return nil, errors.New("S3 Region、Bucket 或访问密钥不完整") } - httpClient := OutboundHTTPClient(timeout) + httpClient := outbound.OutboundHTTPClient(timeout) config := aws.NewConfig(). WithRegion(setting.Region). WithEndpoint(endpoint.String()). WithCredentials(credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, setting.SessionToken)). WithHTTPClient(httpClient). - WithS3ForcePathStyle(setting.PathStyle || !standardAWSS3Endpoint(endpoint.String())). + WithS3ForcePathStyle(setting.PathStyle || !StandardAWSS3Endpoint(endpoint.String())). WithDisableSSL(endpoint.Scheme == "http") sess, err := session.NewSession(config) if err != nil { @@ -73,8 +75,8 @@ func newS3Client(setting ossSettingValue, timeout time.Duration) (*awss3.S3, err return awss3.New(sess), nil } -func putS3Object(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { - client, err := newS3Client(setting, 2*time.Minute) +func PutS3Object(setting Settings, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { + client, err := NewS3Client(setting, 2*time.Minute) if err != nil { return "", err } @@ -109,8 +111,8 @@ func putS3Object(setting ossSettingValue, objectKey string, mimeType string, siz return strings.Trim(aws.StringValue(output.ETag), `"`), nil } -func getS3ObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { - client, err := newS3Client(setting, 2*time.Minute) +func GetS3ObjectRange(setting Settings, objectKey string, rangeHeader string) (*ObjectStream, error) { + client, err := NewS3Client(setting, 2*time.Minute) if err != nil { return nil, err } @@ -121,7 +123,7 @@ func getS3ObjectRange(setting ossSettingValue, objectKey string, rangeHeader str output, err := client.GetObjectWithContext(context.Background(), input) if err != nil { if requestFailure, ok := err.(awserr.RequestFailure); ok && requestFailure.StatusCode() == http.StatusRequestedRangeNotSatisfiable { - return &ossObjectStream{body: io.NopCloser(bytes.NewReader(nil)), statusCode: http.StatusRequestedRangeNotSatisfiable, acceptRanges: "bytes"}, nil + return &ObjectStream{Body: io.NopCloser(bytes.NewReader(nil)), StatusCode: http.StatusRequestedRangeNotSatisfiable, AcceptRanges: "bytes"}, nil } return nil, fmt.Errorf("S3 读取失败：%w", err) } @@ -129,11 +131,11 @@ func getS3ObjectRange(setting ossSettingValue, objectKey string, rangeHeader str if rangeHeader != "" && aws.StringValue(output.ContentRange) != "" { status = http.StatusPartialContent } - return &ossObjectStream{body: output.Body, statusCode: status, contentLength: aws.Int64Value(output.ContentLength), contentRange: aws.StringValue(output.ContentRange), acceptRanges: firstNonEmpty(aws.StringValue(output.AcceptRanges), "bytes")}, nil + return &ObjectStream{Body: output.Body, StatusCode: status, ContentLength: aws.Int64Value(output.ContentLength), ContentRange: aws.StringValue(output.ContentRange), AcceptRanges: kernel.FirstNonEmpty(aws.StringValue(output.AcceptRanges), "bytes")}, nil } -func signedS3ObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { - client, err := newS3Client(setting, 2*time.Minute) +func SignedS3ObjectURL(setting Settings, objectKey string, expiresAt time.Time) (string, error) { + client, err := NewS3Client(setting, 2*time.Minute) if err != nil { return "", err } @@ -149,8 +151,8 @@ func signedS3ObjectURL(setting ossSettingValue, objectKey string, expiresAt time return value, nil } -func deleteS3Object(setting ossSettingValue, objectKey string) error { - client, err := newS3Client(setting, 2*time.Minute) +func DeleteS3Object(setting Settings, objectKey string) error { + client, err := NewS3Client(setting, 2*time.Minute) if err != nil { return err } diff --git a/backend/internal/storage/settings.go b/backend/internal/storage/settings.go new file mode 100644 index 000000000..625b9fbdc --- /dev/null +++ b/backend/internal/storage/settings.go @@ -0,0 +1,85 @@ +package storage + +import ( + "strings" + "time" +) + +const ( + aliyunOSSProvider = "aliyun" + tencentCOSProvider = "tencent" + qiniuKodoProvider = "qiniu" + s3Provider = "s3" + defaultOSSPathPrefix = "open-ai-canvas" + resourceAccessURLTTL = 5 * time.Minute +) + +type Settings struct { + Delivery DeliverySettings `json:"delivery"` + Enabled bool `json:"enabled"` + Provider string `json:"provider"` + Region string `json:"region"` + Endpoint string `json:"endpoint"` + CDNBaseURL string `json:"cdnBaseUrl"` + Bucket string `json:"bucket"` + AccessKeyID string `json:"accessKeyId"` + AccessKeySecret string `json:"accessKeySecret"` + PublicBaseURL string `json:"publicBaseUrl"` + PathPrefix string `json:"pathPrefix"` + S3Preset string `json:"s3Preset"` + PathStyle bool `json:"pathStyle"` + SessionToken string `json:"sessionToken"` + StorageLocationID string `json:"storageLocationId"` + AllowUserS3 bool `json:"allowUserS3"` + // 平台切换云厂商后仍需读取历史资源，因此仅归档非当前厂商的访问密钥。 + ArchivedCredentials map[string]Credentials `json:"archivedCredentials,omitempty"` +} + +type Credentials struct { + AccessKeyID string `json:"accessKeyId"` + AccessKeySecret string `json:"accessKeySecret"` +} + +func CloneCredentials(source map[string]Credentials) map[string]Credentials { + if len(source) == 0 { + return nil + } + cloned := make(map[string]Credentials, len(source)) + for provider, credentials := range source { + cloned[strings.ToLower(strings.TrimSpace(provider))] = Credentials{ + AccessKeyID: strings.TrimSpace(credentials.AccessKeyID), + AccessKeySecret: strings.TrimSpace(credentials.AccessKeySecret), + } + } + return cloned +} + +func NormalizeSettings(value Settings) Settings { + value.Provider = strings.ToLower(strings.TrimSpace(value.Provider)) + if value.Provider == "" { + value.Provider = aliyunOSSProvider + } + value.Region = strings.TrimSpace(value.Region) + value.Endpoint = strings.TrimRight(strings.TrimSpace(value.Endpoint), "/") + if value.Provider == tencentCOSProvider && value.Endpoint == "" && value.Region != "" { + value.Endpoint = "https://cos." + value.Region + ".myqcloud.com" + } + value.CDNBaseURL = strings.TrimRight(strings.TrimSpace(value.CDNBaseURL), "/") + value.Delivery.CDNAuthMode = strings.ToLower(strings.TrimSpace(value.Delivery.CDNAuthMode)) + value.Bucket = strings.TrimSpace(value.Bucket) + value.AccessKeyID = strings.TrimSpace(value.AccessKeyID) + value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret) + value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/") + value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/") + if value.PathPrefix == "" { + value.PathPrefix = defaultOSSPathPrefix + } + value.S3Preset = strings.ToLower(strings.TrimSpace(value.S3Preset)) + if value.S3Preset == "" { + value.S3Preset = "custom" + } + value.SessionToken = strings.TrimSpace(value.SessionToken) + value.StorageLocationID = strings.TrimSpace(value.StorageLocationID) + value.ArchivedCredentials = CloneCredentials(value.ArchivedCredentials) + return value +} diff --git a/docs/content/docs/backend/http-api.mdx b/docs/content/docs/backend/http-api.mdx index 3fef7f8e9..4ecd23d9b 100644 --- a/docs/content/docs/backend/http-api.mdx +++ b/docs/content/docs/backend/http-api.mdx @@ -13,6 +13,18 @@ description: 信封、错误码与 OpenAPI 模型 HTTP 失败及经过 Provider 错误转换的业务失败，在通用提示后追加经过过滤的 `；上游：具体原因`，供任务错误和画布卡片展示内容安全、提示词、尺寸等可操作信息。只读取上游 `error` / `message` / `msg` / `detail` 消息字段，不回传整个 JSON；消息中的 HTTP(S) URL 隐藏，包含余额、账务、凭据、请求诊断或 HTML 的消息不追加原文，保留通用分类提示。后台响应拦截规则仍对最终用户文案生效；不会修改历史任务错误或改变重试、退款策略。 +## 资源访问与 OSS/CDN 分发 + +资源展示、播放、下载、复制地址和模型输入统一消费资源访问合同，不在客户端通过 `direct=1`、`proxy=1` 或存储厂商规则选择出口。 + +- `POST /api/resources/access` 要求登录，请求体为 1–100 项 `{ resourceId, purpose?, variant? }`。`purpose` 支持 `display`、`copy`、`download`、`browser-process`、`provider-input`；`variant` 支持 `original`、`playback`。每项独立返回 `access` 或结构化 `error`。 +- `access.delivery` 为 `cdn`、`origin`、`platform-local` 或 `platform-proxy`。配置了可用 CDN 时返回 CDN 地址；OSS 源站签名是源站直连，不等于 CDN。`requireCDN` 开启且 CDN 鉴权不完整时失败，不静默回源；私有源站只有 `allowPrivateProxy` 显式开启才允许平台承载正文。 +- `expiresAt` / `refreshAt` 只描述当前授权有效期。`provider-input` 默认 4 小时，其余签名访问默认 5 分钟；公开 CDN 可没有 `expiresAt`。签名地址不写入画布或服务端持久化内容。 +- `GET /api/resources/:id/file` 仍是稳定平台入口，负责本地/平台代理字节流，云资源只返回 307 到 CDN 或对象存储；`?access=1` 只返回访问描述，不打开正文。`Range` 只影响字节流路径。 +- 公共资源、画布分享、外观、公告、管理员资源和日志媒体复用同一个 HTTP 执行器。公共签名链接受自身到期时间和变体绑定限制，但已签发的外部 URL 不承诺立即撤销。 + +当前自动签名能力：公开 CDN、七牛私有下载签名和对象存储源站签名。阿里云/腾讯云私有 CDN 签名尚未自动支持，不能仅填写 CDN 域名就宣称已命中 CDN。真实 CORS、Range、CDN 命中率及账单收益必须在实际云环境通过浏览器网络记录和服务端流量观测验收。 + ## 生成成功后的重试保存 `POST /tasks/:id/recover-media` 无需请求体。当前用户只能恢复自己有保存恢复记录的失败任务；重入队仍返回原任务 ID，不请求重新生成、不新建计费订单。已排队、运行或完成时幂等返回当前任务。取消任务、已退款订单、任务/账单归属不一致、项目已删除或归档时拒绝恢复。 diff --git a/docs/design/resource-delivery-architecture.md b/docs/design/resource-delivery-architecture.md new file mode 100644 index 000000000..e9f6aa6da --- /dev/null +++ b/docs/design/resource-delivery-architecture.md @@ -0,0 +1,253 @@ +# 资源访问与 OSS/CDN 分发架构重构 + +状态：已实施核心重构，待真实 OSS/CDN 链路与线上流量验收。 + +已落地：统一资源访问合同、存储分发策略、批量访问 API、共享 HTTP 执行器、用户/管理员/公开/分享/公告/外观/日志媒体迁移，以及前端 provider 输入、Blob 读取和画布复制地址迁移。未声称真实 CDN 命中率、CORS、Range 或账单收益已在本地验证。 + +## 1. 决策摘要与完成边界 + +这不是「把几个 URL 改成 OSS 地址」，而是收回散落在页面、API helper、handler、缓存和存储实现中的分发决策权。目标是：**资源身份稳定、授权按业务场景执行、分发只有一个决策入口、供应商差异封装在适配器内**。 + +- 用户展示、播放、下载、复制地址，以及分享、公开素材、后台预览和模型输入，必须使用同一资源访问合同；不同授权场景不能变成不同分发策略。 +- 对具备直连条件的云资源，浏览器直接请求 CDN 或对象存储，平台只返回访问描述或重定向，不转发媒体正文。配置了合格 CDN 时优先 CDN；对象存储直连不能计作 CDN 命中。 +- 本地资源、不可公网访问且没有可用 CDN 的资源，以及获准的特殊字节处理，是明确、有监控的例外；不能用静默代理掩盖配置错误。 +- 画布右键复制地址签发原始资源的新鲜直链，不复制平台展示入口或浏览器 Blob URL。实际为本地资源时如实提示平台地址；没有可用地址时明确失败。 +- 展示路径不默认下载完整 Blob；浏览器缓存成功不再是云端资源或生成任务成功的条件。 +- **新增统一服务不是完成。全部调用方迁入、旧策略删除、权限及跨入口验收通过，才算完成整体重构。** 可分阶段验证，但不把阶段性补丁作为最终交付。 + +本轮不重写任务引擎、计费、模型协议或转码算法，不引入通用插件平台、事件总线或为每次签名建立数据库记录。 + +## 2. 已确认问题与证据边界 + +| 现状 | 代码依据（仓库相对位置） | 架构影响 | +| --- | --- | --- | +| 节点复制地址优先取 `metadata.content`，否则拼平台文件入口，没有使用已有直链接口 | `web/src/pages/canvas/project.tsx` 的 `copyNodeMediaUrlToClipboard` | 交互层在解释媒体身份并选择出口 | +| 前端文件地址带 `direct=1`，后端生成的稳定文件地址不带；另有 `proxy=1` | `web/src/services/api/resources.ts`；`backend/internal/assets/urls.go` | 相同资源可能因调用来源而采用不同策略 | +| S3 默认代理判断在 CDN 判断前；直链签发与文件分发又是两套分支 | `backend/internal/app/resource.go` 的 `prepareResourceDelivery`、`directResourceURL`、`signedOSSObjectURL` | 供应商、部署能力与业务用途混在一起 | +| `/file` 有重定向和流式读取两条路径 | `backend/internal/handler/user_data.go` 的资源文件路由 | 平台 URL 不等于平台转发，必须看最终请求与正文发送量 | +| 显式 Blob 消费者在部分直连失败后允许走代理 | `web/src/services/api/resources.ts` 的 `getResourceBlob`；`web/src/services/resource-blob-cache.ts` | CORS、过期或网络问题可能被代理流量掩盖；不是所有失败、所有缓存入口都会回退 | +| 视频生成同步等待完整 Blob；播放后还会计划缓存整文件，展示仍使用远程 URL | `web/src/lib/canvas/canvas-generation-task-sync.ts`；`web/src/services/project-asset-sync.ts`；`web/src/services/resource-blob-cache.ts`；`web/src/components/canvas/canvas-node-content.tsx` | 播放器读取与后台完整下载可能并存，缓存失败还会影响业务成功状态 | +| 管理预览、外观、公告、公开签名读取有直接流式路径；分享和日志媒体使用默认分发选项 | `backend/internal/handler/admin_storage.go`；`backend/internal/app/appearance.go`、`announcement.go`、`analytics.go`；`backend/internal/canvas/canvas_share.go` | 仅修改用户文件接口不能覆盖全站 | +| 普通资源写入存在 OSS 失败后落本地的路径，任务媒体新链路则强校验 | `backend/internal/app/resource.go` 的 `storeResourceObject`；`backend/internal/app/task_media_storage.go` | 当前配置不代表每个对象的实际位置，不能推断所有资源已经在 OSS | +| 对象保存实际 provider、endpoint、bucket 和历史存储关联；已有位置解析逻辑 | `backend/internal/model/models_project.go`；`backend/internal/app/resource.go` 的 `ossSettingForResource` | 不能把历史对象统一套到当前全局 CDN 域名 | +| 播放副本当前只针对本地原件生成，handler 自行选择副本和 ETag | `backend/internal/app/video_transcode.go`；`backend/internal/handler/user_data.go` | 变体选择必须进入合同，不能在重定向之后才选择；不能承诺云资源已经支持云端转码 | +| 存储连接测试会清空 CDN 配置，只验证源站；现有配置没有独立的 CDN 用户访问鉴权合同 | `backend/internal/app/storage_location.go` 的 `verifyOSSConnection`；`backend/internal/app/settings.go` | 「存储连接成功」不能作为 CDN、浏览器跨域和私有内容访问策略可用的证明 | + +以上来自代码调查。尚未读取生产存储配置、实际请求日志、CDN 日志或账单，不能据此认定全部媒体都在代理，也不能量化已经多花了多少费用。线上归因需要同时记录最终域名、307/200/206、平台正文发送字节和 CDN 回源/命中情况；它用于确认收益，不替代架构重构。 + +## 3. 目标结构：身份、授权、分发、字节读取分离 + +```text +UI / 画布 / 素材 / 编辑器 + -> media-access：资源引用解析、批量请求、有效期、请求去重、账号隔离 + -> resources API（现有 http 入口） + -> handler -> service 别名 -> app 组合根 + -> 用户 / 分享 / 管理员 / 公开槽位的授权规则 + -> assets.ResourceAccessService（唯一访问决策入口） + -> 变体及实际存储位置解析 + -> DeliveryPolicy（用途、能力、鉴权、期限） + -> OriginSigner / CDNSigner + -> Redirect 或有明确原因的 StreamPlan + +需要字节的编辑 / 导出 / 缩略图计算 + -> 浏览器 media-bytes：先取得访问描述，再按需读取与缓存 + +服务端转码 / 导出 / 导入 + -> 授权与业务编排 -> OriginReader：直接读实际存储，不套浏览器 CDN 策略 +``` + +### 3.1 后端职责 + +| 边界 | 负责 | 不负责 | +| --- | --- | --- | +| `handler` | 入参、会话上下文、业务响应、执行已决定的重定向或流式 HTTP 响应 | 查存储配置、选择 provider、解释 CDN、决定强制代理 | +| `app` | 场景授权编排、依赖注入、配置装配；向现有 `canvas.Host` 提供收敛后的能力 | 再保留一套直连与代理分支 | +| `assets` | `ResourceRef`、访问合同、变体解析、分发策略、结构化失败原因 | HTTP、React、第三方 SDK 细节、业务提示文案 | +| `storage`（从现有实现提取） | 按实际存储位置执行 Put/OpenRange/Delete、源站签名；封装 local、阿里云、腾讯云、七牛及 S3 差异 | 判断当前用户是否能看画布、选择显示还是复制 | +| 分发签名适配器 | CDN 路径映射与用户访问签名；与源站签名分开实现、分开测试 | 默认把私有内容当公开内容，或用对象存储签名算法猜测 CDN 算法 | +| `repository/model` | 资源、位置、引用、配置与状态持久化 | 签发临时 URL、控制浏览器缓存 | + +复用现有 `assets` 域，不再造一个包罗上传、任务、权限和 UI 逻辑的大 service。`internal/service` 继续只做别名；域包不得反向 import `app/service`。存储适配器不依赖 `assets` 业务策略，所需接口由组合根装配，避免循环依赖。 + +### 3.2 前端职责 + +- `services/api/resources.ts`：只保留业务接口与类型，通过现有 `http` 调用，不负责选择媒体出口。 +- `services/media-access.ts`（拟）：统一访问描述生命周期，处理批量、去重、刷新、取消、账号切换；不依赖 JSX、路由或 AntD。 +- UI hook：订阅访问状态，适配图片加载和视频恢复；组件只声明资源引用、用途和变体。 +- `services/media-bytes.ts`（拟）：承接真实需要 Blob/ArrayBuffer 的操作；原缓存模块收敛为其内部缓存能力，不能自行拼代理地址或签名地址。 +- `lib` 中的引用解析为纯函数。页面、store、生成同步和编辑器不得自行拼 `/file`、判断云厂商或提取签名参数。 + +## 4. 统一访问合同 + +### 4.1 持久身份与临时访问描述 + +持久数据继续以资源 ID / `resource:` storageKey 等稳定身份关联对象；临时访问 URL 只在内存中使用，不写回画布、任务结果、素材记录或 localStorage。文本节点的 `content` 语义不受影响，不全局删除同名字段。 + +现有稳定 `/file` 引用通过已有引用解析边界提取资源 ID；它可以作为稳定入口保留，但不能保留第二套分发策略。解析只识别受控站点的资源格式，不因外部 URL 包含相似路径就将其认作受管对象。不为替换域名批量重写所有历史画布，也不以整库迁移作为本次收敛的前提。 + +前端明确区分受管资源、外部 URL、浏览器本地内容。外部 URL 不冒充 OSS 对象；本地内容在上传成功前没有云端可复制地址。已有资源 ID 时不得优先采用过期的 `content` URL。 + +建议新增 `POST /api/resources/access`，单项和批量使用同一实现；批量上限初始为 100 项。分享等场景提供自己的授权适配入口，内部调用同一访问服务，不能让客户端自行声明管理员或分享授权上下文。 + +```ts +type ResourceAccessRequest = { + resourceId: string; + variant: 'original' | 'playback'; + purpose: 'display' | 'download' | 'copy' | 'browser-process'; +}; + +type ResourceAccess = { + resourceId: string; + requestedVariant: 'original' | 'playback'; + actualVariant: 'original' | 'playback'; + url: string; + delivery: 'cdn' | 'origin' | 'platform-local' | 'platform-proxy'; + issuedAt: string; + expiresAt: string | null; + refreshAt: string; + revision: string; + fallbackReason?: string; +}; +``` + +- 返回仍遵守 `{ code, data, msg, reason }`。批量成功受理后，`data.items` 按项返回 `access` 或机器可读错误；顶层认证失败返回真实 HTTP/业务错误。逐项校验权限，不因同一批次有一个合法资源而放行其他资源。 +- `purpose` 是策略输入，不是授权。`provider-input` 只允许可信服务端调用，不开放一个供浏览器获取长效凭据的用途枚举。 +- `copy/download` 默认原件；播放可请求兼容副本。副本不就绪时按集中规则返回原件并说明 `actualVariant/fallbackReason`，不能静默伪装副本就绪。源站与副本使用各自真实位置、MIME 和 ETag。 +- 时间字段使用 UTC RFC 3339。`expiresAt` 为实际签名到期时间，只有明确公开资源可为 `null`；客户端结合 `issuedAt`、请求耗时和本地单调时钟计算剩余期限，不能假定设备时钟准确。`refreshAt` 同时受授权剩余时间、配置刷新周期和安全余量约束，不能固定缓存 4 分钟后猜测有效期。 +- `revision` 表示资源变体与分发配置版本，不含密钥。访问描述响应默认 `no-store`，客户端仅做受控内存缓存；媒体正文的浏览器缓存与 CDN 缓存另行配置。 +- 内部决策使用互斥类型 `Redirect` / `LocalStream` / `ApprovedProxy`，每种类型只携带必要字段；删除 `ForceDirect/ForceProxy` 和多个可空字段并存的隐式状态机。 +- 返回平台 URL 时使用绑定资源、变体、期限及授权范围的受控读取入口；该入口执行已验证的流式授权，不再次解析成自己的重定向目标，防止循环重定向。 + +### 4.2 期限、复制与长视频 + +当前直链为 5 分钟、模型输入为 4 小时。重构后由单一 policy 按用途管理期限，复用这两个值作为已有行为基线，而不是让 UI 各自写常量；分享签名不得超过分享有效期，私有资源不得因用途不同扩大授权范围。 + +- 复制操作在用户点击时取得新鲜原件地址，并显示「CDN/对象存储临时地址及有效期」；浏览器剪贴板失败必须报错，不能吞掉异常再提示成功。 +- 短期直链是可转交的访问凭据，不是永不过期的分享链接。需要长期分享时使用既有分享授权能力，不把 Bucket 改为公开以满足复制需求。 +- 已签发地址在剩余有效期内的撤权限制必须明示；不能承诺撤销分享后所有外部签名立即失效。已经下载的内容也无法收回。 +- 视频续期不能简单定时替换正在播放的 `src`。媒体控制器维护播放位置、暂停状态和当前请求版本；在需要续期或过期恢复时受控换源、恢复位置，最多刷新重试一次，失败可见。 +- 验收必须覆盖播放、暂停后继续、跨过签名有效期后的拖动与新 Range 请求；不能以短视频能开始播放作为长视频通过的证明。 +- 下载文件名、Content-Disposition 等由访问策略和适配器统一处理；不能仅为更改文件名而退回平台整文件转发。 + +## 5. 唯一分发策略与配置模型 + +分发顺序固定为：**校验业务授权 → 校验资源 ready → 解析变体 → 解析该对象实际位置 → 校验用途能力和访问保护 → 签发访问描述或返回结构化错误**。 + +| 实际条件 | 决策 | 限制 | +| --- | --- | --- | +| 本地对象 | `platform-local` | 如实呈现；不能生成看起来像 OSS 的假地址 | +| 对象位置绑定有效 CDN，路径映射与用户访问保护适用于该用途 | `cdn` | 先判断可用 CDN，再考虑源站是否私网或 S3 | +| 无 CDN，源站可公网安全访问且可签发读取凭据 | `origin` | 计作源站直连，不宣称使用 CDN | +| 私网源站、无可用 CDN，配置明确允许平台读取 | `platform-proxy` | 必须记录原因、流量、大小/并发限制；复制时不能称为 OSS 直链 | +| 私有内容的 CDN 用户鉴权缺失、配置损坏、签名失败或凭据失效 | 结构化失败 | 不发布无保护地址，不因捕获异常就自动转代理 | +| 字节处理需要 CORS，但分发端未满足要求 | 明确失败；仅在服务端策略授权时采用受控代理 | 浏览器不能用 `proxy=1` 自行扩大权限或绕过流量约束 | + +CDN 是分发配置，不是存储厂商别名。源站权限、CDN 回源权限、用户访问 CDN 的权限分别建模；本项目不得把「能回源」当作「私有资源可以匿名分发」。 + +复用 `StorageLocation` 与现有历史位置解析，为位置增加结构化 `delivery` 配置，区分域名/路径映射、用户访问鉴权方式、代理例外、期限和浏览器能力。优先沿用现有配置存储与加密边界，不为了抽象额外新建一套存储中心；签名密钥不返回给浏览器。 + +原有 `cdnBaseUrl` 不足以推导私有 CDN 鉴权能力。实施时盘点实际 CDN 产品和鉴权模式，补对应签名适配器；未验证配置保持不可用，不默认公开或自动猜算法。选择「必须走 CDN」时配置失败应报错，不能自动降为源站；允许源站降级则必须由同一 policy 明确配置并记录原因。 + +连接测试拆成源站读写检查与分发检查。后者覆盖浏览器 GET/HEAD/Range、所需 CORS、响应头、签名过期和未授权访问，并关联配置版本。保存配置不等于已经验证可用；不擅自修改云端 ACL、CORS、DNS 或 CDN 配置。 + +历史资源按其绑定位置寻址；改默认厂商只影响新写入。相同位置的 CDN/密钥变更可由位置解析统一生效，并使访问缓存失效；不得把另一 Bucket 的对象拼到新域名。 + +## 6. 缓存与写入成功语义 + +### 6.1 展示缓存和字节缓存分离 + +- 普通图片、视频、音频展示直接使用访问描述，交给浏览器/CDN 按媒体协议读取。删除生成完成、进入画布、开始播放后默认整文件缓存的流程。 +- Blob 只为图片编辑、像素读取、合成、导出、复制图片内容及明确离线操作等字节消费者产生。下载原件优先原生直链下载，不一律先在 JS 中抓取全量文件。 +- `media-bytes` 统一检查 `response.ok`，传递取消信号；云直连不附带平台 Cookie。凭据过期可重新解析一次，不把所有网络/CORS 错误都判为过期，更不能无声启动代理。 +- 字节缓存按用户 scope、资源、实际变体及内容版本隔离，有大小/总量限制、请求去重与 Blob URL 回收。清晰区分缓存成功、云端保存成功和生成成功。 +- 访问描述缓存另外按 scope、授权场景、资源、变体、用途去重，按服务端时间信息留余量刷新；公开/分享会话不得复用登录用户缓存。 +- 切换账号时取消在途请求、清理访问与字节缓存引用；响应提交前核对发起时的 scope，禁止旧请求写入新账号缓存。签名 URL 不落入 localforage/localStorage。 +- 采用微批量解析避免一个画布逐节点产生 N+1 请求；无视图消费者的资源不提前签发或下载。 + +### 6.2 写入与变体 + +资源成功以服务端实际对象和资源记录持久化成功为准。任务同步、项目素材同步不再等待浏览器拿到完整视频 Blob；元信息来自服务端或按需探测，读取失败不得捏造宽高、时长或成功状态。 + +资源写入的本地降级从隐式 catch 改为明确存储策略：选择 OSS 且未允许本地降级时写入失败向上抛出；允许降级时记录实际 local 位置、返回可见状态并计入例外指标。任务媒体已有的强校验不被削弱。此处只收敛存储成功语义，不修改任务计费规则。 + +已有本地资源不会因修改分发策略自动变为云资源。若要迁移，另行执行带预检、复制、完整性校验、引用切换与延迟清理的可回滚作业；未经确认不搬迁或删除现有对象。 + +现有本地播放副本纳入统一变体解析，不改变转码范围。未来新增云端副本时必须记录副本的真实位置并复用同一存储与分发接口，不能重新在播放器/handler 写一个云端特例。 + +## 7. 全入口迁移与旧实现退场清单 + +| 入口/旧职责 | 目标归属 | 删除或收敛要求 | +| --- | --- | --- | +| 画布、素材库、项目封面、任务预览、图片预览、音视频播放器、编辑器媒体展示 | `media-access` + 展示 hook | 页面不得直接拼平台或 OSS 文件地址 | +| 节点「复制地址」、复制图片失败后的地址操作、下载入口 | 统一 copy/download 用途 | 不优先复制 `metadata.content`，不继续调用旧 `oss-url` helper | +| `resourceFileUrl`、`resourceProxyFileUrl`、`playbackVariantUrl`、`resolveResourceUrl` 等 | 引用解析与访问合同 | 删除出口选择；稳定文件路由仅作为同一 policy 的 HTTP 适配 | +| `getResourceOSSUrl`、独立直链缓存、`DirectResourceURL` | 唯一访问签发服务 | 调用方清零后删除 `/oss-url` 及独立分支，不长期并行 | +| `resolveImageUrl`、`resolveMediaUrl`、文件/图片存储服务、CachedResourceImage | 统一访问生命周期 | 存储与展示不再各自解析直链或续期 | +| 生成结果恢复、任务输出、素材同步 | 持久资源引用 + 访问描述 | 不保存签名 URL，不依赖完整 Blob 才标成功 | +| 缓存、图片编辑、导出、剪贴板内容操作 | `media-bytes` | 删除自动整文件预取和 `allowProxyFallback` 隐式回退策略 | +| 用户文件、公开签名文件、管理员存储预览、调用日志媒体 | 各自授权 + 统一访问服务 + HTTP 执行器 | 删除 handler 自选 provider、直接调用云读取实现的旁路 | +| 画布分享、站点外观、公告资源 | 现有业务授权后使用同一分发策略 | 公开入口只能访问业务实际绑定资源，不能成为任意 resourceId 下载器 | +| 模型参考输入、服务端导出/转码/导入 | 外部读取签发或内部 OriginReader | 输入 URL 只在出站前解析；内部字节处理不绕经浏览器 CDN 再代理 | +| `ForceDirect/ForceProxy`、`direct/proxy` 查询参数 | 服务端集中 policy | 不再接受调用方强制选择出口，OpenAPI 与测试同步删除 | +| 存储配置、历史位置解析、连通性测试 | 位置解析 + 分发配置/能力检查 | 一次改配置影响全部入口，不修改页面，不为每个路由新增开关 | + +上述是必须覆盖的入口族，不声称静态列举已经穷尽所有调用点。实施第一阶段生成逐调用点清单，覆盖直接 import、封装转调、动态路由及外部 SDK 输入；清单中每项都要有迁入位置和验证用例，不能只搜索一个函数名就宣布完成。 + +## 8. 错误、安全与可观测性 + +- 权限仍在业务 service/域内校验，授权后的内部资源描述不得由客户端构造。管理员、分享、公开槽位各有授权规则，但只有一套分发规则。 +- 非所属资源、不存在资源、未 ready、变体不可用、分发配置无效、签名失败、分发端不可读、代理不允许分别有机器可读 reason；沿用项目错误信封，不解析错误文案。 +- 代理只读取已授权的受管对象，不接受任意 URL；保留出站 SSRF、私网精确放行、超时和取消约束，不为 S3 内网部署放开全网访问。 +- Range、206、416、If-Range、条件缓存、MIME、Content-Disposition 在统一 HTTP 执行器及签名适配器中验证；读取失败不返回伪成功空文件。 +- 每次访问决策记录场景、用途、实际位置类别、变体、delivery、结构化原因和配置版本。日志不记录签名完整 URL、查询参数、Cookie、密钥或分享凭据。 +- 平台正文发送字节按 local/proxy 及原因分类；签发接口和重定向响应单独计量。资源 ID 可作受控日志关联，但不作高基数指标标签。 +- 观察直连/代理占比、代理发送字节、重复整文件下载、签发失败、续期失败、CDN 命中与回源字节。账单比较按同等资源量/访问量归一化，不以平台请求数下降代替成本证明。 + +## 9. 实施阶段与落地顺序 + +这些是同一架构的实施阶段，不是「先只止血，后续再说」。阶段内允许未完成的代码演进，最终不保留新旧两套策略。 + +| 阶段 | 交付物 | 进入下一阶段的门槛 | +| --- | --- | --- | +| A：合同与回归基线 | 全调用点清单、资源引用/访问/错误合同、位置与授权矩阵、现有测试映射 | 能明确每个入口的授权方、策略方、字节读取方；生产未知项单列 | +| B：后端纵向收敛 | `assets` 访问服务、存储/分发适配器、批量 API、共享 HTTP 执行器、位置配置与 OpenAPI | local、各云厂商、S3 公网/私网、CDN 私有访问、变体和所有授权场景的契约测试通过 | +| C：全部消费者迁入 | 前端访问/字节层、生成同步解耦、分享/后台/公开/模型输入迁入 | 迁移清单清零；删除旧接口、布尔策略、旁路与自动整文件缓存；前后端版本配套 | +| D：架构与真实链路验收 | 依赖约束测试、最小专项回归、真实浏览器网络记录、经授权的云配置小流量验证 | 下节验收项完成；例外流量有明确来源；真实费用收益另用线上观察确认 | + +实施同步维护 HTTP API、资源存储、代码地图及部署专题；如新增数据库字段，同步数据库说明。已实现但未获得用户确认的结果进入待验收记录，不把本设计登记为已完成功能。 + +本轮不更改数据库、云配置或生产路由。若实施涉及新增分发配置，先预检现有配置并说明变更影响；保留版本化配置/发布产物用于回滚。原则上不改变既有稳定引用编码；如确需改变持久合同，必须先给出备份、迁移与旧版本可读性方案，不能只回滚二进制。 + +## 10. 硬性验收门槛 + +### 架构约束 + +- [ ] 分发决策仅存在于 `DeliveryPolicy`；供应商签名只存在于对应适配器。 +- [ ] 新增 CDN 或更换签名实现只改分发配置/适配器及其测试，不改画布、播放器、复制按钮和业务 handler。 +- [ ] 增加一个媒体展示入口只消费统一合同，不引入新的地址拼装或 provider 判断。 +- [ ] 通过依赖约束/静态扫描检查旧导出、旧查询参数及直接读取旁路；许可例外集中列出，禁止用模糊忽略规则绕过。 +- [ ] 迁移清单无未处理调用方，没有长期保留的兼容分发链。 + +### 行为与安全 + +- [ ] CDN、源站直连、local、明确批准的私网代理覆盖原件/播放副本；配置变化后历史资源仍能按实际位置读取。 +- [ ] 所有页面展示与右键复制均遵守同一策略；云资源复制为新鲜原件直链，过期提示准确；Blob/local 不被伪装为云资源。 +- [ ] 本地播放副本、副本尚未就绪/失败、Range/拖动、签名过期、重新登录及账号切换均有针对真实实现的测试。 +- [ ] 用户越权、管理员校验、分享期限/撤销边界、公开槽位越界、私有 CDN 未鉴权访问均验证拒绝语义。 +- [ ] CORS/网络失败不会静默变平台代理；必需字节读取失败和剪贴板失败有可见错误，取消不被当成普通失败重试。 +- [ ] 打开画布、生成成功、开始播放不再触发无业务必要的整视频下载；编辑/导出等真正需要字节的路径仍可用。 + +### 流量与上线 + +- [ ] 对已验证可直连的云资源样本，逐入口检查最终 CDN/源站域名；平台不发送该媒体的 200/206 正文，仅允许访问描述或重定向。 +- [ ] 平台所有媒体正文发送都有可解释的 local/approved-proxy 原因；不能因路径叫 `/api` 就统计成代理，也不能把 CDN 回源当命中。 +- [ ] 浏览器真实请求与服务端计量一致，未产生播放 Range 加后台完整下载的重复流量。 +- [ ] 小流量上线、配置异常和发布回滚有记录；没有擅自公开私有存储、删除旧对象或修改生产 ACL。 + +## 11. 本次评估结论与经验约束 + +方案具备落地基础：仓库已有资源 ID、实际位置/历史配置、权限边界和部分直链能力；主要工作是重置职责与合同、迁移全部调用方、删除重复策略，而不是重新发明对象存储系统。改动横跨资源核心流程、权限、缓存和部署配置，应按 L2 完整实施，不适合将其承诺为几个文件的小修复。 + +尚需实际环境验证的是 CDN 产品及用户访问鉴权方式、CORS/Range、历史 local 占比与各路径流量成本。它们决定具体配置、签名适配与例外范围，不改变「单一策略入口」的架构选择；本轮不把未知项当作已验证事实。 + +上一版评估虽然识别了分散决策，但没有把策略的唯一归属、旧实现删除和未来变更影响面作为硬完成条件，容易让止血成为终点。本次已按 **症状 → 职责图 → 单一合同 → 全调用方迁移 → 旧路径删除 → 防回归约束** 推进；剩余工作只属于真实云链路验收、供应商能力补齐和调用方继续收敛，不再新增第二套资源访问策略。 diff --git a/docs/index.md b/docs/index.md index 19595d369..013b3ac45 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,6 +4,8 @@ ## 设计沉淀 +- [资源访问与 OSS/CDN 分发架构重构](design/resource-delivery-architecture.md)：资源身份、场景授权与分发策略分离，统一访问合同、全入口迁移、旧实现退场及流量/安全验收门槛（核心代码已实施，待真实 OSS/CDN 链路验收）。 + - [短信渠道插件与登录注册策略](design/sms-channels-and-auth-policy.md)：阿里云/腾讯云官方 SDK 候选基线、多渠道与模板路由、发送记录、验证码安全、短信和邮箱登录注册组合及分阶段验收（设计提案，未实施）。 - [画布批量生成一致性治理](design/canvas-consistency-repair.mdx)：节点丢失、批次状态与引用解析的根因、已实施边界重构、回归证据和仍待验收的性能/同步场景。 diff --git a/web/src/components/layout/user-oss-settings-form.tsx b/web/src/components/layout/user-oss-settings-form.tsx index 8942facd2..68d0d1b74 100644 --- a/web/src/components/layout/user-oss-settings-form.tsx +++ b/web/src/components/layout/user-oss-settings-form.tsx @@ -15,6 +15,9 @@ type OSSFormValues = { region?: string; endpoint?: string; cdnBaseUrl?: string; + cdnAuthMode?: "" | "public" | "qiniu" | string; + requireCDN?: boolean; + allowPrivateProxy?: boolean; bucket?: string; accessKeyId?: string; accessKeySecret?: string; @@ -77,6 +80,9 @@ export function UserOSSSettingsForm() { region: values.region?.trim() || "", endpoint: values.endpoint?.trim() || "", cdnBaseUrl: values.cdnBaseUrl?.trim() || "", + cdnAuthMode: values.cdnAuthMode || "", + requireCDN: values.requireCDN === true, + allowPrivateProxy: values.allowPrivateProxy === true, bucket: values.bucket?.trim() || "", accessKeyId: values.accessKeyId?.trim() || "", accessKeySecret: values.accessKeySecret?.trim() || "", @@ -138,7 +144,7 @@ export function UserOSSSettingsForm() { + + @@ -213,6 +233,9 @@ function toFormValues(setting: UserOSSSetting): OSSFormValues { region: setting.region, endpoint: setting.endpoint, cdnBaseUrl: setting.cdnBaseUrl, + cdnAuthMode: setting.cdnAuthMode || "", + requireCDN: setting.requireCDN === true, + allowPrivateProxy: setting.allowPrivateProxy === true, bucket: setting.bucket, accessKeyId: setting.accessKeyId, accessKeySecret: "", diff --git a/web/src/pages/admin/settings/storage-settings-page.tsx b/web/src/pages/admin/settings/storage-settings-page.tsx index 3155baa24..a8f1f4e5a 100644 --- a/web/src/pages/admin/settings/storage-settings-page.tsx +++ b/web/src/pages/admin/settings/storage-settings-page.tsx @@ -19,6 +19,9 @@ type OSSFormValues = { region: string; endpoint: string; cdnBaseUrl: string; + cdnAuthMode: "" | "public" | "qiniu" | string; + requireCDN: boolean; + allowPrivateProxy: boolean; bucket: string; accessKeyId: string; accessKeySecret: string; @@ -29,7 +32,7 @@ type OSSFormValues = { allowUserS3: boolean; }; -type StoragePayload = Pick; +type StoragePayload = Pick; const STORAGE_MODES: Array<{ mode: StorageMode; label: string; short: string; description: string }> = [ { mode: "local", label: "服务器本地", short: "本地磁盘", description: "新增资源写入当前部署的数据目录，通过后端签名链接访问。" }, @@ -540,10 +543,29 @@ export default function StorageSettingsPage() { + +