refactor(media): 资源分发 - 统一资源访问策略与云存储重定向
This commit is contained in:
1 parent
af6633047c
commit
487c34bfeb
50 files changed
+2112
-1266
No files matched your search
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
|
||||
@@ -127,6 +128,24 @@ func (s *Service) OpenResourceRangeAsAdmin(actor *model.User, id string, rangeHe
|
||||
return s.openResourceRange(resource.UserID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) PrepareResourceDeliveryAsAdmin(actor *model.User, id string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
if err := s.RequireAdmin(actor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resource, err := s.repo.Resource(strings.TrimSpace(id))
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("资源不存在")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
resource.Provider = normalizedResourceProvider(resource.Provider)
|
||||
if options.Purpose == "" {
|
||||
options.Purpose = assets.PurposeDisplay
|
||||
}
|
||||
return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader)
|
||||
}
|
||||
|
||||
func normalizeAdminResourceQuery(query AdminResourceQuery) (repository.AdminResourceFilter, int, int, error) {
|
||||
page, limit := normalizeAdminPage(query.Page, query.Limit)
|
||||
filter := repository.AdminResourceFilter{
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
|
||||
@@ -326,21 +327,15 @@ func (s *Service) OpenAdminAPICallLogMediaRange(actor *model.User, logID string,
|
||||
return s.openResourceRange(userID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) PrepareAdminAPICallLogMediaDelivery(actor *model.User, logID string, rangeHeader string) (*ResourceDelivery, error) {
|
||||
func (s *Service) PrepareAdminAPICallLogMediaDelivery(actor *model.User, logID string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
userID, resource, err := s.adminAPICallLogMediaResource(actor, logID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delivery, err := s.prepareResourceDelivery(userID, resource, ResourceDeliveryOptions{})
|
||||
if err != nil || delivery.RedirectURL != "" {
|
||||
return delivery, err
|
||||
if options.Purpose == "" {
|
||||
options.Purpose = assets.PurposeDisplay
|
||||
}
|
||||
stream, err := s.openResourceRange(userID, resource, rangeHeader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delivery.Stream = stream
|
||||
return delivery, nil
|
||||
return s.prepareResourceDelivery(userID, resource, options, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) adminAPICallLogMediaResource(actor *model.User, logID string) (string, *model.Resource, error) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
|
||||
@@ -269,6 +270,39 @@ func (s *Service) OpenAnnouncementImage(actor *model.User, announcementID string
|
||||
return s.openResourceRange(resource.UserID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) PrepareAnnouncementImageDelivery(actor *model.User, announcementID string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
if actor == nil {
|
||||
return nil, Unauthorized("请先登录")
|
||||
}
|
||||
announcement, err := s.repo.Announcement(strings.TrimSpace(announcementID))
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("公告不存在")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if actor.Role != model.UserRoleAdmin && announcement.Status != model.AnnouncementStatusActive {
|
||||
return nil, Forbidden("公告不可访问")
|
||||
}
|
||||
if announcement.ImageResourceID == "" {
|
||||
return nil, NotFound("公告配图不存在")
|
||||
}
|
||||
resource, err := s.repo.Resource(announcement.ImageResourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("公告配图不存在")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if resource.Kind != "image" || resource.Status != model.ResourceStatusReady {
|
||||
return nil, BadAuthRequest("公告配图资源不可用")
|
||||
}
|
||||
if options.Purpose == "" {
|
||||
options.Purpose = assets.PurposeDisplay
|
||||
}
|
||||
return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) cleanupStaleAnnouncementImageDrafts() {
|
||||
for {
|
||||
drafts, err := s.repo.StaleAnnouncementImageDrafts(time.Now().Add(-announcementImageDraftTTL), 50)
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
|
||||
"gorm.io/gorm"
|
||||
@@ -317,6 +318,31 @@ func (s *Service) OpenAppearanceAsset(slot string, rangeHeader string) (*Resourc
|
||||
return s.openResourceRange(resource.UserID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) PrepareAppearanceAssetDelivery(slot string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
_, value, err := s.readAppearance()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resourceID := appearanceResourceID(value, slot)
|
||||
if resourceID == "" {
|
||||
return nil, NotFound("未配置该外观资源")
|
||||
}
|
||||
resource, err := s.repo.Resource(resourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("外观资源不存在")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := validateAppearanceResourceType(slot, resource); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if options.Purpose == "" {
|
||||
options.Purpose = assets.PurposeDisplay
|
||||
}
|
||||
return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) appearanceResourceReferences(resourceIDs []string) map[string][]AdminResourceReferenceView {
|
||||
result := make(map[string][]AdminResourceReferenceView)
|
||||
_, value, err := s.readAppearance()
|
||||
|
||||
@@ -49,11 +49,11 @@ func (h canvasHost) OpenResourceRange(userID string, resource *model.Resource, r
|
||||
return h.svc.openResourceRange(userID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (h canvasHost) PrepareResourceDelivery(userID string, resource *model.Resource, options assets.ResourceDeliveryOptions) (*assets.ResourceDelivery, error) {
|
||||
func (h canvasHost) PrepareResourceDelivery(userID string, resource *model.Resource, options assets.AccessOptions, rangeHeader string) (*assets.ResourceDelivery, error) {
|
||||
if h.svc == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return h.svc.prepareResourceDelivery(userID, resource, options)
|
||||
return h.svc.prepareResourceDelivery(userID, resource, options, rangeHeader)
|
||||
}
|
||||
|
||||
func (h canvasHost) WithStorageLock(fn func() error) error {
|
||||
@@ -150,8 +150,8 @@ func (s *Service) OpenSharedCanvasResourceRange(token string, resourceID string,
|
||||
return s.canvasDomain().OpenSharedCanvasResourceRange(token, resourceID, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) PrepareSharedCanvasResourceDelivery(token string, resourceID string, rangeHeader string) (*ResourceDelivery, error) {
|
||||
return s.canvasDomain().PrepareSharedCanvasResourceDelivery(token, resourceID, rangeHeader)
|
||||
func (s *Service) PrepareSharedCanvasResourceDelivery(token string, resourceID string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
return s.canvasDomain().PrepareSharedCanvasResourceDelivery(token, resourceID, options, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) validateCanvasMediaAssets(userID string, raw json.RawMessage) error {
|
||||
|
||||
@@ -805,7 +805,7 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, poli
|
||||
}
|
||||
useObjectURL := policy.requireURL || (policy.preferURL && resourceUsesObjectStorage(resource))
|
||||
if useObjectURL {
|
||||
signedURL, err := s.directResourceURL(resource, time.Now().Add(providerResourceURLTTL))
|
||||
signedURL, err := s.providerResourceURL(resource, time.Now().Add(providerResourceURLTTL))
|
||||
if err != nil {
|
||||
return fmt.Errorf("生成参考素材地址失败:%w", err)
|
||||
}
|
||||
|
||||
@@ -230,7 +230,7 @@ func (s *Service) ensureArkPrivateAsset(ctx context.Context, userID string, reso
|
||||
}
|
||||
}
|
||||
binding.AssetGroupID = groupID
|
||||
resourceURL, err := s.directResourceURL(resource, time.Now().Add(time.Hour))
|
||||
resourceURL, err := s.providerResourceURL(resource, time.Now().Add(time.Hour))
|
||||
if err != nil {
|
||||
return "", s.failArkPrivateAssetBinding(binding, fmt.Errorf("生成方舟素材临时地址失败:%w", err))
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
@@ -18,5 +19,9 @@ func (s *Service) ResourceForUser(actor *model.User, id string) (*model.Resource
|
||||
// providerResourceURL 只能接收已经完成用户归属和 ready 状态校验的资源,
|
||||
// 并为上游签发短时地址;它不是绕过权限校验的通用资源 URL 生成器。
|
||||
func (s *Service) providerResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) {
|
||||
return s.directResourceURL(resource, expiresAt)
|
||||
access, err := s.resolveResourceAccess(resource, ResourceAccessOptions{Purpose: assets.PurposeProvider, ExpiresAt: expiresAt})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return access.URL, nil
|
||||
}
|
||||
@@ -2,9 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
@@ -17,7 +15,6 @@ import (
|
||||
_ "image/png"
|
||||
"infinite-canvas/backend/internal/kernel"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -32,21 +29,14 @@ import (
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
awsv4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
qiniuAuth "github.com/qiniu/go-sdk/v7/auth"
|
||||
qiniuStorage "github.com/qiniu/go-sdk/v7/storage"
|
||||
cos "github.com/tencentyun/cos-go-sdk-v5"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const providerResourceURLTTL = 4 * time.Hour
|
||||
const directResourceURLTTL = 5 * time.Minute
|
||||
|
||||
var errInvalidGeneratedDataURL = errors.New("生成内容 data URL 无效")
|
||||
|
||||
type ResourceStream = assets.ResourceStream
|
||||
type ResourceDeliveryOptions = assets.ResourceDeliveryOptions
|
||||
type ResourceDelivery = assets.ResourceDelivery
|
||||
|
||||
func (s *Service) Resources(userID string, limit int) ([]model.Resource, error) {
|
||||
@@ -65,141 +55,6 @@ func (s *Service) Resource(userID string, id string) (*model.Resource, error) {
|
||||
return resource, err
|
||||
}
|
||||
|
||||
// DirectResourceURL 先校验资源归属,再按实际存储位置签发短时下载地址。
|
||||
func (s *Service) DirectResourceURL(userID string, id string) (string, error) {
|
||||
resource, err := s.repo.ResourceForUser(userID, id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.directResourceURL(resource, time.Now().Add(directResourceURLTTL))
|
||||
}
|
||||
|
||||
func (s *Service) directResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) {
|
||||
if resource == nil {
|
||||
return "", errors.New("资源不存在")
|
||||
}
|
||||
if resource.Status != model.ResourceStatusReady {
|
||||
return "", BadAuthRequest("资源尚未上传完成")
|
||||
}
|
||||
if resource.Provider == "local" {
|
||||
return s.signedPublicResourceURL(resource, expiresAt)
|
||||
}
|
||||
setting, err := s.ossSettingForResource(resource.UserID, resource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
setting.Provider = firstNonEmpty(resource.Provider, setting.Provider)
|
||||
setting.Endpoint = firstNonEmpty(resource.Endpoint, setting.Endpoint)
|
||||
setting.Bucket = firstNonEmpty(resource.Bucket, setting.Bucket)
|
||||
if setting.Provider == s3Provider && !publicHTTPSStorageEndpoint(setting.Endpoint) {
|
||||
return s.signedHTTPSPublicResourceURL(resource, expiresAt)
|
||||
}
|
||||
return signedOSSObjectURL(setting, resource.ObjectKey, expiresAt)
|
||||
}
|
||||
|
||||
// PrepareResourceDelivery 统一决定浏览器资源出口:配置 CDN 时默认直连 CDN,显式代理仅用于需要同源 Blob 的内部读取。
|
||||
func (s *Service) PrepareResourceDelivery(userID string, id string, options ResourceDeliveryOptions) (*ResourceDelivery, error) {
|
||||
resource, err := s.repo.ResourceForUser(userID, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("资源不存在")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return s.prepareResourceDelivery(userID, resource, options)
|
||||
}
|
||||
|
||||
func (s *Service) prepareResourceDelivery(userID string, resource *model.Resource, options ResourceDeliveryOptions) (*ResourceDelivery, error) {
|
||||
if resource == nil {
|
||||
return nil, errors.New("资源不存在")
|
||||
}
|
||||
if resource.Status != model.ResourceStatusReady {
|
||||
return nil, BadAuthRequest("资源尚未上传完成")
|
||||
}
|
||||
if resource.Provider != "local" && !options.ForceProxy {
|
||||
setting, err := s.ossSettingForResource(userID, resource)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// S3 兼容 Endpoint 可能是私网服务;浏览器默认始终使用同源代理。
|
||||
// 只有明确的服务端上游需求才签发可公开访问的短时地址。
|
||||
if setting.Provider == s3Provider && !options.ForceDirect {
|
||||
return &ResourceDelivery{Resource: resource}, nil
|
||||
}
|
||||
if setting.Provider == qiniuKodoProvider && setting.CDNBaseURL != "" {
|
||||
// 七牛私有空间即使配置了绑定域名,也不能匿名访问;必须使用
|
||||
// Kodo 私有下载签名,否则浏览器会收到 NotSupportAnonymous。
|
||||
redirectURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(directResourceURLTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil
|
||||
}
|
||||
if setting.CDNBaseURL != "" {
|
||||
redirectURL, err := ossCDNObjectURL(setting.CDNBaseURL, resource.ObjectKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil
|
||||
}
|
||||
if options.ForceDirect {
|
||||
if setting.Provider == s3Provider && !publicHTTPSStorageEndpoint(setting.Endpoint) {
|
||||
redirectURL, err := s.signedHTTPSPublicResourceURL(resource, time.Now().Add(directResourceURLTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil
|
||||
}
|
||||
redirectURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(directResourceURLTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil
|
||||
}
|
||||
}
|
||||
return &ResourceDelivery{Resource: resource}, nil
|
||||
}
|
||||
|
||||
func (s *Service) signedPublicResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) {
|
||||
if resource == nil {
|
||||
return "", errors.New("资源不存在")
|
||||
}
|
||||
baseURL, err := s.publicResourceBaseURL()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10)
|
||||
signature, err := s.signPublicResource(resource.ID, expires)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ext := resourceFileExtension(resource.ObjectKey, resource.MimeType, resource.Kind)
|
||||
filename := resource.ID
|
||||
if ext != "" {
|
||||
if !strings.HasPrefix(ext, ".") {
|
||||
ext = "." + ext
|
||||
}
|
||||
filename += ext
|
||||
}
|
||||
baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/api/public/resources/" + url.PathEscape(resource.ID) + "/file/" + url.PathEscape(filename)
|
||||
query := baseURL.Query()
|
||||
query.Set("expires", expires)
|
||||
query.Set("signature", signature)
|
||||
baseURL.RawQuery = query.Encode()
|
||||
return baseURL.String(), nil
|
||||
}
|
||||
|
||||
func (s *Service) signedHTTPSPublicResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) {
|
||||
baseURL, err := s.publicResourceBaseURL()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if baseURL.Scheme != "https" {
|
||||
return "", BadAuthRequest("私网或 HTTP S3 用于上游资源时,服务器公开访问地址必须使用 HTTPS")
|
||||
}
|
||||
return s.signedPublicResourceURL(resource, expiresAt)
|
||||
}
|
||||
|
||||
func (s *Service) verifyPublicResourceSignature(resourceID string, expires string, signature string) error {
|
||||
if strings.TrimSpace(signature) == "" || !decimalDigits(expires) {
|
||||
return Forbidden("匿名下载链接无效")
|
||||
@@ -448,20 +303,6 @@ func (s *Service) OpenResourceRange(userID string, id string, rangeHeader string
|
||||
return s.openResourceRange(userID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) OpenPublicResourceRange(id string, expires string, signature string, rangeHeader string) (*ResourceStream, error) {
|
||||
resource, err := s.repo.Resource(id)
|
||||
if err != nil {
|
||||
return nil, Forbidden("匿名下载链接无效")
|
||||
}
|
||||
if resource.Provider != "local" && resource.Provider != s3Provider {
|
||||
return nil, Forbidden("匿名下载链接无效")
|
||||
}
|
||||
if err := s.verifyPublicResourceSignature(resource.ID, expires, signature); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.openResourceRange(resource.UserID, resource, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) openResourceRange(userID string, resource *model.Resource, rangeHeader string) (*ResourceStream, error) {
|
||||
if resource.Status != model.ResourceStatusReady {
|
||||
return nil, BadAuthRequest("资源尚未上传完成")
|
||||
@@ -487,7 +328,7 @@ func (s *Service) openResourceRange(userID string, resource *model.Resource, ran
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ResourceStream{Resource: resource, Body: stream.body, StatusCode: stream.statusCode, ContentLength: stream.contentLength, ContentRange: stream.contentRange, AcceptRanges: stream.acceptRanges}, nil
|
||||
return &ResourceStream{Resource: resource, Body: stream.Body, StatusCode: stream.StatusCode, ContentLength: stream.ContentLength, ContentRange: stream.ContentRange, AcceptRanges: stream.AcceptRanges}, nil
|
||||
}
|
||||
|
||||
func (s *Service) storeResource(userID string, kind string, fileName string, mimeType string, size int64, width int, height int, durationMs int64, body io.Reader, uploadKey *string, forceLocal bool) (*model.Resource, bool, error) {
|
||||
@@ -590,9 +431,7 @@ func writeLocalResourceObject(filePath string, body io.Reader) error {
|
||||
return closeErr
|
||||
}
|
||||
|
||||
// storeResourceObject 写入资源物理对象。对象存储不可用(配置错误、密钥失效、网络
|
||||
// 故障、设置被删)时自动降级为本地存储并同步改写资源记录,保证上传写路径不因外部
|
||||
// 存储故障整体失败。对象存储失败后 body 会被重新读取,须支持 Seek。
|
||||
// A cloud write is successful only after the configured origin accepts it.
|
||||
func (s *Service) storeResourceObject(resource *model.Resource, fileName string, body io.Reader) (string, error) {
|
||||
if resource == nil {
|
||||
return "", errors.New("资源不存在")
|
||||
@@ -600,36 +439,11 @@ func (s *Service) storeResourceObject(resource *model.Resource, fileName string,
|
||||
if resource.Provider == "local" {
|
||||
return "", writeLocalResourceObject(filepath.Join(s.dataDir, "resources", filepath.FromSlash(resource.ObjectKey)), body)
|
||||
}
|
||||
setting, settingErr := s.ossSettingForResource(resource.UserID, resource)
|
||||
var etag string
|
||||
var putErr error
|
||||
if settingErr == nil {
|
||||
etag, putErr = putOSSObject(setting, resource.ObjectKey, resource.MimeType, resource.Size, body)
|
||||
setting, err := s.ossSettingForResource(resource.UserID, resource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if putErr == nil && settingErr == nil {
|
||||
return etag, nil
|
||||
}
|
||||
fallbackErr := putErr
|
||||
if fallbackErr == nil {
|
||||
fallbackErr = settingErr
|
||||
}
|
||||
if seeker, ok := body.(io.Seeker); ok {
|
||||
if _, seekErr := seeker.Seek(0, io.SeekStart); seekErr != nil {
|
||||
return "", errors.Join(fallbackErr, fmt.Errorf("降级本地存储时重置读取位置失败:%w", seekErr))
|
||||
}
|
||||
}
|
||||
localKey := localObjectKey(resource.UserID, resource.Kind, fileName, resource.MimeType, time.Now())
|
||||
resource.Provider = "local"
|
||||
resource.ObjectKey = localKey
|
||||
resource.Endpoint = ""
|
||||
resource.Bucket = ""
|
||||
resource.StorageSettingID = ""
|
||||
resource.ETag = ""
|
||||
if localErr := writeLocalResourceObject(filepath.Join(s.dataDir, "resources", filepath.FromSlash(localKey)), body); localErr != nil {
|
||||
return "", errors.Join(fallbackErr, fmt.Errorf("降级本地存储失败:%w", localErr))
|
||||
}
|
||||
log.Printf("object storage upload degraded to local storage: resource=%s error=%v", resource.ID, fallbackErr)
|
||||
return "", nil
|
||||
return putOSSObject(setting, resource.ObjectKey, resource.MimeType, resource.Size, body)
|
||||
}
|
||||
|
||||
func (s *Service) retryStoredResource(userID string, resource *model.Resource, kind string, mimeType string, size int64, body io.Reader) (*model.Resource, error) {
|
||||
@@ -1015,6 +829,10 @@ func (s *Service) ossSettingForResource(userID string, resource *model.Resource)
|
||||
// 密钥固定在资源绑定的历史版本;只有存储位置完全一致时,才允许沿用当前 CDN。
|
||||
if resourceStorageMatches(current, resource) {
|
||||
setting.CDNBaseURL = current.CDNBaseURL
|
||||
// CDN 的鉴权方式和回退策略属于分发配置,而不是资源创建时的
|
||||
// 凭据。存储位置未变时沿用当前策略,避免历史资源因管理员
|
||||
// 刚补齐 CDN 鉴权配置而继续回源。
|
||||
setting.Delivery = current.Delivery
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1129,89 +947,6 @@ func ossObjectKey(setting ossSettingValue, userID string, kind string, fileName
|
||||
return strings.Trim(strings.Join(nonEmptySegments(parts), "/"), "/")
|
||||
}
|
||||
|
||||
func putOSSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
if setting.Provider == tencentCOSProvider {
|
||||
return putCOSObject(setting, objectKey, mimeType, size, body)
|
||||
}
|
||||
if setting.Provider == qiniuKodoProvider {
|
||||
return putQiniuObject(setting, objectKey, mimeType, size, body)
|
||||
}
|
||||
if setting.Provider == s3Provider {
|
||||
return putS3Object(setting, objectKey, mimeType, size, body)
|
||||
}
|
||||
return putAliyunOSSObject(setting, objectKey, mimeType, size, body)
|
||||
}
|
||||
|
||||
// 阿里云 OSS 继续沿用原有 V1 签名和请求路径,避免已有部署行为发生变化。
|
||||
func putAliyunOSSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
req, err := newOSSRequest(http.MethodPut, setting, objectKey, mimeType, body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if size > 0 {
|
||||
req.ContentLength = size
|
||||
}
|
||||
resp, err := OutboundHTTPClient(2 * time.Minute).Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return "", fmt.Errorf("OSS 上传失败:%s %s", resp.Status, strings.TrimSpace(string(detail)))
|
||||
}
|
||||
return strings.Trim(resp.Header.Get("ETag"), `"`), nil
|
||||
}
|
||||
|
||||
type ossObjectStream struct {
|
||||
body io.ReadCloser
|
||||
statusCode int
|
||||
contentLength int64
|
||||
contentRange string
|
||||
acceptRanges string
|
||||
}
|
||||
|
||||
func getOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
if setting.Provider == s3Provider {
|
||||
return getS3ObjectRange(setting, objectKey, rangeHeader)
|
||||
}
|
||||
if setting.CDNBaseURL != "" {
|
||||
return getOSSObjectRangeViaCDN(setting, objectKey, rangeHeader)
|
||||
}
|
||||
if setting.Provider == tencentCOSProvider {
|
||||
return getCOSObjectRange(setting, objectKey, rangeHeader)
|
||||
}
|
||||
if setting.Provider == qiniuKodoProvider {
|
||||
return getQiniuObjectRange(setting, objectKey, rangeHeader)
|
||||
}
|
||||
return getAliyunOSSObjectRange(setting, objectKey, rangeHeader)
|
||||
}
|
||||
|
||||
func getAliyunOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
req, err := newOSSRequest(http.MethodGet, setting, objectKey, "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rangeHeader != "" {
|
||||
req.Header.Set("Range", rangeHeader)
|
||||
}
|
||||
resp, err := OutboundHTTPClient(2 * time.Minute).Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
||||
defer resp.Body.Close()
|
||||
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return nil, fmt.Errorf("OSS 读取失败:%s %s", resp.Status, strings.TrimSpace(string(detail)))
|
||||
}
|
||||
return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
|
||||
func normalizeSingleByteRange(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if len(value) > 128 || !strings.HasPrefix(value, "bytes=") || strings.Contains(value, ",") {
|
||||
@@ -1233,396 +968,6 @@ func decimalDigits(value string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func signedOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
if setting.Provider == s3Provider {
|
||||
return signedS3ObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
if setting.Provider == qiniuKodoProvider {
|
||||
return signedQiniuObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
if setting.CDNBaseURL != "" {
|
||||
return ossCDNObjectURL(setting.CDNBaseURL, objectKey)
|
||||
}
|
||||
if setting.Provider == tencentCOSProvider {
|
||||
return signedCOSObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
return signedAliyunOSSObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
|
||||
func signedAliyunOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
baseURL, err := ossBucketBaseURL(setting)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" {
|
||||
return "", errors.New("OSS 访问密钥不可用")
|
||||
}
|
||||
objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/")
|
||||
if objectKey == "" {
|
||||
return "", errors.New("OSS 对象路径为空")
|
||||
}
|
||||
baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/" + escapeObjectKey(objectKey)
|
||||
expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10)
|
||||
stringToSign := strings.Join([]string{http.MethodGet, "", "", expires, "/" + setting.Bucket + "/" + objectKey}, "\n")
|
||||
mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret))
|
||||
_, _ = mac.Write([]byte(stringToSign))
|
||||
query := baseURL.Query()
|
||||
query.Set("OSSAccessKeyId", setting.AccessKeyID)
|
||||
query.Set("Expires", expires)
|
||||
query.Set("Signature", base64.StdEncoding.EncodeToString(mac.Sum(nil)))
|
||||
baseURL.RawQuery = query.Encode()
|
||||
return baseURL.String(), nil
|
||||
}
|
||||
|
||||
func putCOSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
options := &cos.ObjectPutOptions{ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ContentType: mimeType, ContentLength: size}}
|
||||
resp, err := client.Object.Put(context.Background(), objectKey, body, options)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("COS 上传失败:%w", err)
|
||||
}
|
||||
return strings.Trim(resp.Header.Get("ETag"), `"`), nil
|
||||
}
|
||||
|
||||
func putQiniuObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return "", errors.New("七牛云 Kodo 访问密钥不可用")
|
||||
}
|
||||
if setting.Bucket == "" || objectKey == "" {
|
||||
return "", errors.New("七牛云 Kodo Bucket 或对象路径为空")
|
||||
}
|
||||
config := qiniuStorage.NewConfig()
|
||||
config.Region = qiniuRegion(setting.Region)
|
||||
config.UpHost = strings.TrimRight(setting.Endpoint, "/")
|
||||
uploader := qiniuStorage.NewFormUploader(config)
|
||||
policy := qiniuStorage.PutPolicy{Scope: setting.Bucket + ":" + strings.TrimLeft(objectKey, "/"), Expires: 3600}
|
||||
token := policy.UploadToken(qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret))
|
||||
ret := qiniuStorage.PutRet{}
|
||||
extra := &qiniuStorage.PutExtra{MimeType: mimeType}
|
||||
if size < 0 {
|
||||
size = 0
|
||||
}
|
||||
if err := uploader.Put(context.Background(), &ret, token, strings.TrimLeft(objectKey, "/"), body, size, extra); err != nil {
|
||||
return "", fmt.Errorf("七牛云 Kodo 上传失败:%w", err)
|
||||
}
|
||||
return ret.Hash, nil
|
||||
}
|
||||
|
||||
func getCOSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options := &cos.ObjectGetOptions{Range: rangeHeader}
|
||||
resp, err := client.Object.Get(context.Background(), objectKey, options)
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
|
||||
return &ossObjectStream{body: io.NopCloser(bytes.NewReader(nil)), statusCode: resp.StatusCode, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("COS 读取失败:%w", err)
|
||||
}
|
||||
return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
|
||||
func getQiniuObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
signedURL, err := signedQiniuObjectURL(setting, objectKey, time.Now().Add(directResourceURLTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodGet, signedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rangeHeader != "" {
|
||||
req.Header.Set("Range", rangeHeader)
|
||||
}
|
||||
ApplyDefaultOutboundHeaders(req)
|
||||
resp, err := OutboundHTTPClient(2 * time.Minute).Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("七牛云 Kodo 读取失败:%w", err)
|
||||
}
|
||||
if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
||||
defer resp.Body.Close()
|
||||
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return nil, fmt.Errorf("七牛云 Kodo 读取失败:%s %s", resp.Status, strings.TrimSpace(string(detail)))
|
||||
}
|
||||
return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
|
||||
func getOSSObjectRangeViaCDN(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
signedURL, err := signedOSSObjectURL(setting, objectKey, time.Now().Add(directResourceURLTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodGet, signedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rangeHeader != "" {
|
||||
req.Header.Set("Range", rangeHeader)
|
||||
}
|
||||
ApplyDefaultOutboundHeaders(req)
|
||||
resp, err := OutboundHTTPClient(2 * time.Minute).Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("对象存储 CDN 读取失败:%w", err)
|
||||
}
|
||||
if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
||||
defer resp.Body.Close()
|
||||
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return nil, fmt.Errorf("对象存储 CDN 读取失败:%s %s", resp.Status, strings.TrimSpace(string(detail)))
|
||||
}
|
||||
return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
|
||||
func signedCOSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" {
|
||||
return "", errors.New("COS 访问密钥不可用")
|
||||
}
|
||||
objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/")
|
||||
if objectKey == "" {
|
||||
return "", errors.New("COS 对象路径为空")
|
||||
}
|
||||
expires := time.Until(expiresAt)
|
||||
if expires <= 0 {
|
||||
return "", errors.New("COS 签名有效期必须晚于当前时间")
|
||||
}
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
signedURL, err := client.Object.GetPresignedURL(context.Background(), http.MethodGet, objectKey, setting.AccessKeyID, setting.AccessKeySecret, expires, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return signedURL.String(), nil
|
||||
}
|
||||
|
||||
func signedQiniuObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return "", errors.New("七牛云 Kodo 访问密钥不可用")
|
||||
}
|
||||
objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/")
|
||||
if objectKey == "" {
|
||||
return "", errors.New("七牛云 Kodo 对象路径为空")
|
||||
}
|
||||
deadline := expiresAt.Unix()
|
||||
if deadline <= time.Now().Unix() {
|
||||
return "", errors.New("七牛云 Kodo 签名有效期必须晚于当前时间")
|
||||
}
|
||||
if setting.CDNBaseURL == "" {
|
||||
return signedQiniuS3ObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
mac := qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret)
|
||||
return qiniuStorage.MakePrivateURLv2(mac, strings.TrimRight(setting.CDNBaseURL, "/"), objectKey, deadline), nil
|
||||
}
|
||||
|
||||
// signedQiniuS3ObjectURL 用七牛兼容 S3 的 AWS Signature V4 访问私有空间。
|
||||
// 没有绑定域名时,浏览器不直接访问该地址,而是由后端代理读取并返回文件。
|
||||
func signedQiniuS3ObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
region := qiniuS3Region(setting)
|
||||
if region == "" {
|
||||
return "", errors.New("七牛云 Kodo S3 Region 不可用")
|
||||
}
|
||||
baseURL := &url.URL{Scheme: "https", Host: setting.Bucket + ".s3." + region + ".qiniucs.com"}
|
||||
// 保留对象键的原始路径,让 url.URL 和 AWS signer 只做一次 RFC 3986 转义。
|
||||
baseURL.Path = "/" + objectKey
|
||||
req, err := http.NewRequest(http.MethodGet, baseURL.String(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
credentialsValue := credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, "")
|
||||
signer := awsv4.NewSigner(credentialsValue)
|
||||
if _, err := signer.Presign(req, nil, "s3", region, time.Until(expiresAt), time.Now().UTC()); err != nil {
|
||||
return "", fmt.Errorf("七牛云 Kodo S3 签名失败:%w", err)
|
||||
}
|
||||
return req.URL.String(), nil
|
||||
}
|
||||
|
||||
func qiniuS3Region(setting ossSettingValue) string {
|
||||
region := strings.ToLower(strings.TrimSpace(setting.Region))
|
||||
if region == "" {
|
||||
endpoint := strings.ToLower(setting.Endpoint)
|
||||
for _, candidate := range []string{"z0", "z1", "z2", "na0", "as0", "cn-east-1", "cn-north-1", "cn-south-1", "us-north-1", "ap-southeast-1", "cn-east-2"} {
|
||||
if strings.Contains(endpoint, candidate) {
|
||||
region = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
switch region {
|
||||
case "", "z0", "cn-east-1":
|
||||
return "cn-east-1"
|
||||
case "z1", "cn-north-1":
|
||||
return "cn-north-1"
|
||||
case "z2", "cn-south-1":
|
||||
return "cn-south-1"
|
||||
case "na0", "us-north-1":
|
||||
return "us-north-1"
|
||||
case "as0", "ap-southeast-1":
|
||||
return "ap-southeast-1"
|
||||
case "cn-east-2", "zhejiang2":
|
||||
return "cn-east-2"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func qiniuRegion(region string) *qiniuStorage.Region {
|
||||
switch strings.ToLower(strings.TrimSpace(region)) {
|
||||
case "z1", "cn-north-1":
|
||||
return &qiniuStorage.ZoneHuabei
|
||||
case "z2", "cn-south-1":
|
||||
return &qiniuStorage.ZoneHuanan
|
||||
case "na0", "us-north-1":
|
||||
return &qiniuStorage.ZoneBeimei
|
||||
case "as0", "ap-southeast-1":
|
||||
return &qiniuStorage.ZoneXinjiapo
|
||||
case "cn-east-2", "zhejiang2":
|
||||
return &qiniuStorage.ZoneHuadongZheJiang2
|
||||
default:
|
||||
return &qiniuStorage.ZoneHuadong
|
||||
}
|
||||
}
|
||||
|
||||
func newCOSClient(setting ossSettingValue, timeout time.Duration) (*cos.Client, error) {
|
||||
bucketURL, err := cosBucketBaseURL(setting)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpClient := OutboundHTTPClient(timeout)
|
||||
httpClient.Transport = &cos.AuthorizationTransport{SecretID: setting.AccessKeyID, SecretKey: setting.AccessKeySecret, Transport: httpClient.Transport}
|
||||
return cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, httpClient), nil
|
||||
}
|
||||
|
||||
func cosBucketBaseURL(setting ossSettingValue) (*url.URL, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
endpoint := strings.TrimRight(setting.Endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return nil, errors.New("COS Endpoint 为空")
|
||||
}
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" {
|
||||
return nil, errors.New("COS Endpoint 格式不正确")
|
||||
}
|
||||
if setting.Bucket == "" {
|
||||
return nil, errors.New("COS Bucket 为空")
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
if strings.HasSuffix(host, ".myqcloud.com") || strings.HasSuffix(host, ".tencentcos.cn") {
|
||||
if strings.HasPrefix(host, "cos.") || strings.HasPrefix(host, "cos-internal.") || strings.HasPrefix(host, "cos-website.") {
|
||||
parsed.Host = setting.Bucket + "." + parsed.Host
|
||||
} else if !strings.HasPrefix(host, strings.ToLower(setting.Bucket)+".") {
|
||||
return nil, errors.New("COS Endpoint 中的 Bucket 与配置不一致")
|
||||
}
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func ossCDNBaseURL(raw string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || parsed.Hostname() == "" {
|
||||
return nil, errors.New("对象存储 CDN 加速域名格式不正确")
|
||||
}
|
||||
if parsed.Scheme != "https" && parsed.Scheme != "http" {
|
||||
return nil, errors.New("对象存储 CDN 加速域名只支持 http/https")
|
||||
}
|
||||
if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" {
|
||||
return nil, errors.New("对象存储 CDN 加速域名不能包含认证信息、路径、查询参数或片段")
|
||||
}
|
||||
parsed.Path = ""
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func ossCDNObjectURL(raw string, objectKey string) (string, error) {
|
||||
baseURL, err := ossCDNBaseURL(raw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/")
|
||||
if objectKey == "" {
|
||||
return "", errors.New("对象存储对象路径为空")
|
||||
}
|
||||
// CDN 使用自己的访问鉴权与私有桶回源鉴权,不能携带 OSS/COS 的预签名参数。
|
||||
// url.URL.String 会负责转义 Path;这里保留未转义值,避免把 %20 再编码为 %2520。
|
||||
baseURL.Path = "/" + objectKey
|
||||
return baseURL.String(), nil
|
||||
}
|
||||
|
||||
func newOSSRequest(method string, setting ossSettingValue, objectKey string, contentType string, body io.Reader) (*http.Request, error) {
|
||||
baseURL, err := ossBucketBaseURL(setting)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/" + escapeObjectKey(objectKey)
|
||||
// 请求体必须用 no-op close 包装:服务端提前返回(如 OSS 签名 403)时
|
||||
// http.Transport 会关闭未发完的 Request.Body,若直接传入 *os.File 等
|
||||
// 调用方持有的文件,后续“降级本地存储”的 Seek 重读将因 file already
|
||||
// closed 失败。NopCloser 让 Transport 的关闭成为空操作,底层文件保持可用。
|
||||
// GET/HEAD/DELETE 等无请求体的调用传入 nil body;NopCloser(nil) 会产生非 nil 的
|
||||
// Body 包装 nil reader,Go 1.26 发送前 body 探测会直接 nil 解引用崩溃。
|
||||
var reqBody io.Reader
|
||||
if body != nil {
|
||||
reqBody = io.NopCloser(body)
|
||||
}
|
||||
req, err := http.NewRequest(method, baseURL.String(), reqBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date := time.Now().UTC().Format(http.TimeFormat)
|
||||
req.Header.Set("Date", date)
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
stringToSign := strings.Join([]string{method, "", contentType, date, "/" + setting.Bucket + "/" + objectKey}, "\n")
|
||||
mac := hmac.New(sha1.New, []byte(setting.AccessKeySecret))
|
||||
_, _ = mac.Write([]byte(stringToSign))
|
||||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
req.Header.Set("Authorization", "OSS "+setting.AccessKeyID+":"+signature)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func ossBucketBaseURL(setting ossSettingValue) (*url.URL, error) {
|
||||
endpoint := strings.TrimRight(setting.Endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return nil, errors.New("OSS Endpoint 为空")
|
||||
}
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parsed.Host == "" {
|
||||
return nil, errors.New("OSS Endpoint 格式不正确")
|
||||
}
|
||||
if !strings.HasPrefix(parsed.Host, setting.Bucket+".") {
|
||||
parsed.Host = setting.Bucket + "." + parsed.Host
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func escapeObjectKey(key string) string {
|
||||
parts := strings.Split(key, "/")
|
||||
for i, part := range parts {
|
||||
parts[i] = url.PathEscape(part)
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func safeObjectSegment(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
value = strings.Map(func(r rune) rune {
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
type ResourceAccessOptions = assets.AccessOptions
|
||||
type ResourceAccessRequest = assets.AccessRequest
|
||||
type ResourceAccess = assets.ResourceAccess
|
||||
|
||||
type ResourceAccessFailure struct {
|
||||
Code int `json:"code"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"msg"`
|
||||
}
|
||||
type ResourceAccessResult struct {
|
||||
ResourceID string `json:"resourceId"`
|
||||
Access *ResourceAccess `json:"access,omitempty"`
|
||||
Error *ResourceAccessFailure `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) ResourceAccessBatch(userID string, requests []ResourceAccessRequest) ([]ResourceAccessResult, error) {
|
||||
if userID == "" {
|
||||
return nil, Unauthorized("请先登录")
|
||||
}
|
||||
if len(requests) == 0 || len(requests) > 100 {
|
||||
return nil, BadAuthRequest("每批资源访问请求须为 1–100 项")
|
||||
}
|
||||
results := make([]ResourceAccessResult, 0, len(requests))
|
||||
for _, request := range requests {
|
||||
item := ResourceAccessResult{ResourceID: request.ResourceID}
|
||||
// The caller has already been authenticated and each item is resolved through
|
||||
// ResourceForUser below. Provider input is therefore safe to expose here and
|
||||
// keeps browser-side model adapters on the same access contract as every
|
||||
// other resource consumer.
|
||||
options, err := assets.NormalizeAccessOptions(request.AccessOptions, true)
|
||||
if err == nil {
|
||||
var resource *model.Resource
|
||||
resource, err = s.repo.ResourceForUser(userID, request.ResourceID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
err = NotFound("资源不存在或不可访问")
|
||||
}
|
||||
if err == nil {
|
||||
item.Access, err = s.resolveResourceAccess(resource, options)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
item.Error = &ResourceAccessFailure{Code: 500, Reason: "resource_access_failed", Message: "资源访问失败"}
|
||||
var appErr *AppError
|
||||
if errors.As(err, &appErr) {
|
||||
item.Error = &ResourceAccessFailure{Code: appErr.Code, Reason: string(appErr.Reason), Message: appErr.Message}
|
||||
}
|
||||
}
|
||||
results = append(results, item)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *Service) resolveResourceAccess(resource *model.Resource, options ResourceAccessOptions) (*ResourceAccess, error) {
|
||||
if resource == nil {
|
||||
return nil, NotFound("资源不存在")
|
||||
}
|
||||
var setting ossSettingValue
|
||||
if resource.Provider != "local" {
|
||||
var err error
|
||||
setting, err = s.ossSettingForResource(resource.UserID, resource)
|
||||
if err != nil {
|
||||
return nil, WrapAppError(503, "无法解析资源实际存储位置", err)
|
||||
}
|
||||
}
|
||||
return assets.ResolveAccess(resource, setting, options, time.Now().UTC(), func(variant assets.ResourceVariant, expires time.Time) (string, error) {
|
||||
return s.signedResourceAccessURL(resource, variant, expires, options.Purpose == assets.PurposeProvider)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) PrepareResourceDelivery(userID, id string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
options, err := assets.NormalizeAccessOptions(options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resource, err := s.repo.ResourceForUser(userID, id)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("资源不存在或不可访问")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.prepareResourceDelivery(userID, resource, options, rangeHeader)
|
||||
}
|
||||
|
||||
func (s *Service) prepareResourceDelivery(userID string, resource *model.Resource, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
if resource == nil || resource.UserID != userID {
|
||||
return nil, Forbidden("资源不可访问")
|
||||
}
|
||||
access, err := s.resolveResourceAccess(resource, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delivery := &ResourceDelivery{Resource: resource, Access: access}
|
||||
if options.DescribeOnly || access.Delivery == assets.DeliveryCDN || access.Delivery == assets.DeliveryOrigin {
|
||||
return delivery, nil
|
||||
}
|
||||
if access.ActualVariant == assets.VariantPlayback {
|
||||
delivery.Stream, err = s.OpenResourcePlaybackRange(userID, resource.ID)
|
||||
} else {
|
||||
delivery.Stream, err = s.openResourceRange(userID, resource, rangeHeader)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delivery.Resource = delivery.Stream.Resource
|
||||
return delivery, nil
|
||||
}
|
||||
|
||||
func (s *Service) signedResourceAccessURL(resource *model.Resource, variant assets.ResourceVariant, expires time.Time, public bool) (string, error) {
|
||||
expiry := strconv.FormatInt(expires.Unix(), 10)
|
||||
signature, err := s.signPublicResource(resource.ID+"\n"+string(variant), expiry)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u := &url.URL{Path: "/api/public/resources/" + url.PathEscape(resource.ID) + "/file"}
|
||||
q := u.Query()
|
||||
q.Set("expires", expiry)
|
||||
q.Set("signature", signature)
|
||||
q.Set("variant", string(variant))
|
||||
u.RawQuery = q.Encode()
|
||||
if !public {
|
||||
return u.String(), nil
|
||||
}
|
||||
base, err := s.publicResourceBaseURL()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if base.Scheme != "https" {
|
||||
return "", BadAuthRequest("模型读取平台资源需要配置 HTTPS 公网访问地址")
|
||||
}
|
||||
return base.ResolveReference(u).String(), nil
|
||||
}
|
||||
|
||||
func (s *Service) PreparePublicResourceDelivery(id, expires, signature string, options ResourceAccessOptions, rangeHeader string) (*ResourceDelivery, error) {
|
||||
options, err := assets.NormalizeAccessOptions(options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.verifyPublicResourceSignature(id+"\n"+string(options.Variant), expires, signature); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resource, err := s.repo.Resource(id)
|
||||
if err != nil {
|
||||
return nil, Forbidden("匿名下载链接无效")
|
||||
}
|
||||
seconds, _ := strconv.ParseInt(expires, 10, 64)
|
||||
options.ExpiresAt = time.Unix(seconds, 0)
|
||||
return s.prepareResourceDelivery(resource.UserID, resource, options, rangeHeader)
|
||||
}
|
||||
@@ -1,11 +1,8 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -16,8 +13,6 @@ import (
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
|
||||
qiniuAuth "github.com/qiniu/go-sdk/v7/auth"
|
||||
qiniuStorage "github.com/qiniu/go-sdk/v7/storage"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -365,56 +360,3 @@ func (s *Service) deleteLocalResourceObject(objectKey string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteAliyunOSSObject(setting ossSettingValue, objectKey string) error {
|
||||
req, err := newOSSRequest(http.MethodDelete, setting, objectKey, "", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := OutboundHTTPClient(2 * time.Minute).Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除阿里云 OSS 对象失败:%w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if (resp.StatusCode < 200 || resp.StatusCode >= 300) && resp.StatusCode != http.StatusNotFound {
|
||||
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("删除阿里云 OSS 对象失败:%s %s", resp.Status, strings.TrimSpace(string(detail)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteTencentCOSObject(setting ossSettingValue, objectKey string) error {
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := client.Object.Delete(context.Background(), objectKey)
|
||||
if resp != nil && resp.Body != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode == http.StatusNotFound {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("删除腾讯云 COS 对象失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteQiniuObject(setting ossSettingValue, objectKey string) error {
|
||||
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return errors.New("七牛云 Kodo 访问密钥不可用")
|
||||
}
|
||||
if setting.Bucket == "" || strings.TrimSpace(objectKey) == "" {
|
||||
return errors.New("七牛云 Kodo Bucket 或对象路径为空")
|
||||
}
|
||||
mac := qiniuAuth.New(setting.AccessKeyID, setting.AccessKeySecret)
|
||||
manager := qiniuStorage.NewBucketManager(mac, &qiniuStorage.Config{Region: qiniuRegion(setting.Region), UseHTTPS: true})
|
||||
if err := manager.Delete(setting.Bucket, strings.TrimLeft(objectKey, "/")); err != nil {
|
||||
if strings.Contains(strings.ToLower(err.Error()), "no such") || strings.Contains(strings.ToLower(err.Error()), "not found") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("删除七牛云 Kodo 对象失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -18,8 +18,10 @@ import (
|
||||
"testing/iotest"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/assets"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
"infinite-canvas/backend/internal/storage"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
@@ -219,12 +221,12 @@ func TestGetOSSObjectRangeSupportsTencentCOS(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stream.body.Close()
|
||||
data, err := io.ReadAll(stream.body)
|
||||
defer stream.Body.Close()
|
||||
data, err := io.ReadAll(stream.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
t.Fatalf("stream = %#v, data = %q", stream, data)
|
||||
}
|
||||
}
|
||||
@@ -252,12 +254,12 @@ func TestGetOSSObjectRangeUsesTencentCOSCDNBaseURL(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stream.body.Close()
|
||||
data, err := io.ReadAll(stream.body)
|
||||
defer stream.Body.Close()
|
||||
data, err := io.ReadAll(stream.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
t.Fatalf("stream = %#v, data = %q", stream, data)
|
||||
}
|
||||
}
|
||||
@@ -285,12 +287,12 @@ func TestGetOSSObjectRangeUsesAliyunCDNBaseURLWithoutOSSSignature(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stream.body.Close()
|
||||
data, err := io.ReadAll(stream.body)
|
||||
defer stream.Body.Close()
|
||||
data, err := io.ReadAll(stream.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
t.Fatalf("stream = %#v, data = %q", stream, data)
|
||||
}
|
||||
}
|
||||
@@ -412,10 +414,10 @@ func TestArchivedProviderCredentialsAreEncryptedAtRest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectResourceURLChecksOwnershipAndSignsOSSResource(t *testing.T) {
|
||||
func TestResourceAccessChecksOwnershipAndSignsOSSResource(t *testing.T) {
|
||||
svc := newResourceTestService(t)
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
Enabled: true, Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
|
||||
Enabled: true, Provider: "aliyun", Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket",
|
||||
AccessKeyID: "access-id", AccessKeySecret: "secret-value",
|
||||
})
|
||||
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
|
||||
@@ -423,18 +425,22 @@ func TestDirectResourceURLChecksOwnershipAndSignsOSSResource(t *testing.T) {
|
||||
}
|
||||
resource := model.Resource{
|
||||
ID: "resource-direct", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady,
|
||||
Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
|
||||
Provider: "aliyun", Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket",
|
||||
ObjectKey: "users/user-1/image/direct.png", MimeType: "image/png",
|
||||
}
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := svc.DirectResourceURL("user-1", resource.ID)
|
||||
if err != nil || !strings.Contains(value, "Signature=") {
|
||||
t.Fatalf("DirectResourceURL() = %q, %v", value, err)
|
||||
results, err := svc.ResourceAccessBatch("user-1", []ResourceAccessRequest{{
|
||||
ResourceID: resource.ID,
|
||||
AccessOptions: ResourceAccessOptions{Purpose: assets.PurposeCopy},
|
||||
}})
|
||||
if err != nil || len(results) != 1 || results[0].Access == nil || !strings.Contains(results[0].Access.URL, "Signature=") {
|
||||
t.Fatalf("ResourceAccessBatch() = %#v, %v", results, err)
|
||||
}
|
||||
if _, err := svc.DirectResourceURL("other-user", resource.ID); err == nil {
|
||||
t.Fatal("DirectResourceURL() allowed another user's resource")
|
||||
other, err := svc.ResourceAccessBatch("other-user", []ResourceAccessRequest{{ResourceID: resource.ID, AccessOptions: ResourceAccessOptions{Purpose: assets.PurposeCopy}}})
|
||||
if err != nil || len(other) != 1 || other[0].Access != nil {
|
||||
t.Fatalf("ResourceAccessBatch() allowed another user's resource: %#v, %v", other, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,6 +449,7 @@ func TestPrepareResourceDeliveryPrefersConfiguredCDN(t *testing.T) {
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
Enabled: true, Provider: tencentCOSProvider, Endpoint: "https://cos.ap-shanghai.myqcloud.com", CDNBaseURL: "https://media.example.com",
|
||||
Bucket: "private-bucket-1250000000", AccessKeyID: "secret-id", AccessKeySecret: "secret-key",
|
||||
Delivery: storage.DeliverySettings{CDNAuthMode: "public"},
|
||||
})
|
||||
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -455,19 +462,19 @@ func TestPrepareResourceDeliveryPrefersConfiguredCDN(t *testing.T) {
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{})
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.RedirectURL != "https://media.example.com/users/user-1/image/test%20image.png" {
|
||||
if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryCDN || delivery.Access.URL != "https://media.example.com/users/user-1/image/test%20image.png" {
|
||||
t.Fatalf("PrepareResourceDelivery() = %#v", delivery)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResourceDeliveryAllowsExplicitProxyWithCDN(t *testing.T) {
|
||||
func TestPrepareResourceDeliveryFallsBackToOriginWhenCDNAuthIsMissing(t *testing.T) {
|
||||
svc := newResourceTestService(t)
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", CDNBaseURL: "https://media.example.com",
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", CDNBaseURL: "https://media.example.com",
|
||||
Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value",
|
||||
})
|
||||
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
|
||||
@@ -475,18 +482,18 @@ func TestPrepareResourceDeliveryAllowsExplicitProxyWithCDN(t *testing.T) {
|
||||
}
|
||||
resource := model.Resource{
|
||||
ID: "resource-cdn-proxy", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady,
|
||||
Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
|
||||
Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket",
|
||||
ObjectKey: "users/user-1/image/proxy.png", MimeType: "image/png",
|
||||
}
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{ForceProxy: true})
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.RedirectURL != "" {
|
||||
t.Fatalf("PrepareResourceDelivery(force proxy) = %#v", delivery)
|
||||
if delivery.Resource == nil || delivery.Resource.ID != resource.ID || delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryOrigin || delivery.Access.FallbackReason != "cdn_auth_unconfigured" {
|
||||
t.Fatalf("PrepareResourceDelivery() = %#v", delivery)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -495,6 +502,7 @@ func TestPrepareResourceDeliverySignsPrivateQiniuCDNURL(t *testing.T) {
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
Enabled: true, Provider: qiniuKodoProvider, Endpoint: "https://up-z0.qiniup.com", CDNBaseURL: "https://media.example.com",
|
||||
Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value",
|
||||
Delivery: storage.DeliverySettings{CDNAuthMode: "qiniu"},
|
||||
})
|
||||
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -507,38 +515,40 @@ func TestPrepareResourceDeliverySignsPrivateQiniuCDNURL(t *testing.T) {
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{})
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery.RedirectURL == "" || !strings.HasPrefix(delivery.RedirectURL, "https://media.example.com/ai/users/user-1/image/private.png?") || !strings.Contains(delivery.RedirectURL, "e=") || !strings.Contains(delivery.RedirectURL, "token=") {
|
||||
t.Fatalf("PrepareResourceDelivery() = %q, want a signed Qiniu URL", delivery.RedirectURL)
|
||||
if delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryCDN || !strings.HasPrefix(delivery.Access.URL, "https://media.example.com/ai/users/user-1/image/private.png?") || !strings.Contains(delivery.Access.URL, "e=") || !strings.Contains(delivery.Access.URL, "token=") {
|
||||
t.Fatalf("PrepareResourceDelivery() = %#v, want a signed Qiniu URL", delivery)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResourceDeliveryProxiesQiniuWithoutCDNBaseURL(t *testing.T) {
|
||||
func TestPrepareResourceDeliveryAllowsExplicitPrivateOriginProxy(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
svc := newResourceTestService(t)
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
Enabled: true, Provider: qiniuKodoProvider, Region: "z0", Endpoint: "https://up-z0.qiniup.com",
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: "http://storage.internal",
|
||||
Bucket: "private-bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value",
|
||||
Delivery: storage.DeliverySettings{AllowPrivateProxy: true},
|
||||
})
|
||||
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resource := model.Resource{
|
||||
ID: "resource-qiniu-proxy", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady,
|
||||
Provider: qiniuKodoProvider, Endpoint: "https://up-z0.qiniup.com", Bucket: "private-bucket",
|
||||
Provider: aliyunOSSProvider, Endpoint: "http://storage.internal", Bucket: "private-bucket",
|
||||
ObjectKey: "users/user-1/image/private.png", MimeType: "image/png",
|
||||
}
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{})
|
||||
results, err := svc.ResourceAccessBatch("user-1", []ResourceAccessRequest{{ResourceID: resource.ID, AccessOptions: ResourceAccessOptions{Purpose: assets.PurposeDisplay}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery.Resource == nil || delivery.RedirectURL != "" {
|
||||
t.Fatalf("PrepareResourceDelivery() = %#v, want backend proxy delivery", delivery)
|
||||
if len(results) != 1 || results[0].Access == nil || results[0].Access.Delivery != assets.DeliveryProxy {
|
||||
t.Fatalf("ResourceAccessBatch() = %#v, want backend proxy delivery", results)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -568,15 +578,15 @@ func TestCurrentUserCDNSettingAppliesToHistoricalResourcesInSameStorage(t *testi
|
||||
}
|
||||
if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: server.URL, CDNBaseURL: server.URL, Bucket: "private-bucket",
|
||||
AccessKeyID: "access-id", AccessKeySecret: "secret-value",
|
||||
AccessKeyID: "access-id", AccessKeySecret: "secret-value", CDNAuthMode: "public",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceDeliveryOptions{})
|
||||
delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery.RedirectURL != server.URL+"/users/user-1/image/historical.png" {
|
||||
if delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryCDN || delivery.Access.URL != server.URL+"/users/user-1/image/historical.png" {
|
||||
t.Fatalf("PrepareResourceDelivery(historical user resource) = %#v", delivery)
|
||||
}
|
||||
}
|
||||
@@ -599,7 +609,7 @@ func TestBoundHistoricalUserResourceDoesNotFollowCurrentProviderCDN(t *testing.T
|
||||
actor := &model.User{ID: "user-1"}
|
||||
if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: aliyunEndpoint.URL, CDNBaseURL: aliyunCDN.URL, Bucket: "aliyun-bucket",
|
||||
AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret",
|
||||
AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret", CDNAuthMode: "public",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -609,7 +619,7 @@ func TestBoundHistoricalUserResourceDoesNotFollowCurrentProviderCDN(t *testing.T
|
||||
}
|
||||
if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{
|
||||
Enabled: true, Provider: qiniuKodoProvider, Endpoint: qiniuEndpoint.URL, CDNBaseURL: qiniuCDN.URL, Bucket: "qiniu-bucket",
|
||||
AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret",
|
||||
AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret", CDNAuthMode: "public",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -621,13 +631,13 @@ func TestBoundHistoricalUserResourceDoesNotFollowCurrentProviderCDN(t *testing.T
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceDeliveryOptions{})
|
||||
delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := aliyunCDN.URL + "/ai/users/user-1/image/bound.png"
|
||||
if delivery.RedirectURL != want || strings.Contains(delivery.RedirectURL, qiniuCDN.URL) {
|
||||
t.Fatalf("PrepareResourceDelivery(bound historical resource) = %q, want %q", delivery.RedirectURL, want)
|
||||
if delivery.Access == nil || delivery.Access.URL != want || strings.Contains(delivery.Access.URL, qiniuCDN.URL) {
|
||||
t.Fatalf("PrepareResourceDelivery(bound historical resource) = %#v, want %q", delivery, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -648,13 +658,13 @@ func TestHistoricalUserResourceWithoutStorageSettingIDKeepsItsProviderCDN(t *tes
|
||||
actor := &model.User{ID: "user-1"}
|
||||
if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: aliyunEndpoint.URL, CDNBaseURL: aliyunCDN.URL, Bucket: "aliyun-bucket",
|
||||
AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret",
|
||||
AccessKeyID: "aliyun-access", AccessKeySecret: "aliyun-secret", CDNAuthMode: "public",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := svc.UpdateUserOSSSetting(actor, OSSSettingRequest{
|
||||
Enabled: true, Provider: qiniuKodoProvider, Endpoint: qiniuEndpoint.URL, CDNBaseURL: qiniuCDN.URL, Bucket: "qiniu-bucket",
|
||||
AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret",
|
||||
AccessKeyID: "qiniu-access", AccessKeySecret: "qiniu-secret", CDNAuthMode: "public",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -666,20 +676,20 @@ func TestHistoricalUserResourceWithoutStorageSettingIDKeepsItsProviderCDN(t *tes
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceDeliveryOptions{})
|
||||
delivery, err := svc.PrepareResourceDelivery(actor.ID, resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := aliyunCDN.URL + "/ai/users/user-1/image/legacy.png"
|
||||
if delivery.RedirectURL != want || strings.Contains(delivery.RedirectURL, qiniuCDN.URL) {
|
||||
t.Fatalf("PrepareResourceDelivery(legacy user resource) = %q, want %q", delivery.RedirectURL, want)
|
||||
if delivery.Access == nil || delivery.Access.URL != want || strings.Contains(delivery.Access.URL, qiniuCDN.URL) {
|
||||
t.Fatalf("PrepareResourceDelivery(legacy user resource) = %#v, want %q", delivery, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResourceDeliveryKeepsForcedOriginDirectWithoutCDN(t *testing.T) {
|
||||
func TestPrepareResourceDeliveryUsesSignedOriginWithoutCDN(t *testing.T) {
|
||||
svc := newResourceTestService(t)
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
|
||||
Enabled: true, Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket",
|
||||
AccessKeyID: "access-id", AccessKeySecret: "secret-value",
|
||||
})
|
||||
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
|
||||
@@ -687,17 +697,17 @@ func TestPrepareResourceDeliveryKeepsForcedOriginDirectWithoutCDN(t *testing.T)
|
||||
}
|
||||
resource := model.Resource{
|
||||
ID: "resource-origin-direct", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady,
|
||||
Provider: aliyunOSSProvider, Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket",
|
||||
Provider: aliyunOSSProvider, Endpoint: "https://s3.amazonaws.com", Bucket: "private-bucket",
|
||||
ObjectKey: "users/user-1/image/direct.png", MimeType: "image/png",
|
||||
}
|
||||
if err := svc.repo.CreateResource(&resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceDeliveryOptions{ForceDirect: true})
|
||||
delivery, err := svc.PrepareResourceDelivery("user-1", resource.ID, ResourceAccessOptions{Purpose: assets.PurposeDisplay}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(delivery.RedirectURL, "private-bucket.oss-cn-test.aliyuncs.com/users/user-1/image/direct.png") || !strings.Contains(delivery.RedirectURL, "Signature=") {
|
||||
if delivery.Access == nil || delivery.Access.Delivery != assets.DeliveryOrigin || !strings.Contains(delivery.Access.URL, "private-bucket.s3.amazonaws.com/users/user-1/image/direct.png") || !strings.Contains(delivery.Access.URL, "Signature=") {
|
||||
t.Fatalf("PrepareResourceDelivery(force direct) = %#v", delivery)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ func TestResourceIDFromFileURL(t *testing.T) {
|
||||
want string
|
||||
}{
|
||||
{"/api/resources/abc_123/file", "abc_123"},
|
||||
{"https://host.example/api/resources/abc_123/file?proxy=1", "abc_123"},
|
||||
{"https://host.example/api/resources/abc_123/file?variant=original", "abc_123"},
|
||||
{"/api/projects/abc_123", ""},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/storage"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -34,20 +35,23 @@ const (
|
||||
)
|
||||
|
||||
type OSSSettingRequest struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
S3Preset string `json:"s3Preset"`
|
||||
PathStyle bool `json:"pathStyle"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
S3Preset string `json:"s3Preset"`
|
||||
PathStyle bool `json:"pathStyle"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
CDNAuthMode string `json:"cdnAuthMode"`
|
||||
RequireCDN bool `json:"requireCDN"`
|
||||
AllowPrivateProxy bool `json:"allowPrivateProxy"`
|
||||
}
|
||||
|
||||
type PublicOSSSetting struct {
|
||||
@@ -70,36 +74,14 @@ type PublicOSSSetting struct {
|
||||
HistoryCount int64 `json:"historyCount"`
|
||||
ReferencedResourceCount int64 `json:"referencedResourceCount"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
CDNAuthMode string `json:"cdnAuthMode"`
|
||||
RequireCDN bool `json:"requireCDN"`
|
||||
AllowPrivateProxy bool `json:"allowPrivateProxy"`
|
||||
UpdatedBy string `json:"updatedBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type ossSettingValue struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
S3Preset string `json:"s3Preset"`
|
||||
PathStyle bool `json:"pathStyle"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
StorageLocationID string `json:"storageLocationId"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
// 平台切换云厂商后仍需读取历史资源,因此仅归档非当前厂商的访问密钥。
|
||||
ArchivedCredentials map[string]ossProviderCredentials `json:"archivedCredentials,omitempty"`
|
||||
}
|
||||
|
||||
type ossProviderCredentials struct {
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
}
|
||||
|
||||
func (s *Service) AdminOSSSetting(actor *model.User) (*PublicOSSSetting, error) {
|
||||
if err := s.RequireAdmin(actor); err != nil {
|
||||
return nil, err
|
||||
@@ -563,7 +545,11 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
|
||||
PathStyle: req.PathStyle,
|
||||
SessionToken: strings.TrimSpace(req.SessionToken),
|
||||
AllowUserS3: req.AllowUserS3,
|
||||
Delivery: storage.DeliverySettings{CDNAuthMode: strings.ToLower(strings.TrimSpace(req.CDNAuthMode)), RequireCDN: req.RequireCDN, AllowPrivateProxy: req.AllowPrivateProxy},
|
||||
})
|
||||
if next.Delivery.CDNAuthMode != "" && next.Delivery.CDNAuthMode != "public" && next.Delivery.CDNAuthMode != "qiniu" {
|
||||
return next, BadAuthRequest("CDN 鉴权方式无效,仅支持 public 或 qiniu")
|
||||
}
|
||||
if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider && next.Provider != s3Provider {
|
||||
return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS、七牛云 Kodo 和通用 S3")
|
||||
}
|
||||
@@ -640,49 +626,6 @@ func archiveOSSProviderCredentials(next ossSettingValue, current ossSettingValue
|
||||
return next
|
||||
}
|
||||
|
||||
func cloneOSSProviderCredentials(source map[string]ossProviderCredentials) map[string]ossProviderCredentials {
|
||||
if len(source) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make(map[string]ossProviderCredentials, len(source))
|
||||
for provider, credentials := range source {
|
||||
cloned[strings.ToLower(strings.TrimSpace(provider))] = ossProviderCredentials{
|
||||
AccessKeyID: strings.TrimSpace(credentials.AccessKeyID),
|
||||
AccessKeySecret: strings.TrimSpace(credentials.AccessKeySecret),
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func normalizeOSSSetting(value ossSettingValue) ossSettingValue {
|
||||
value.Provider = strings.ToLower(strings.TrimSpace(value.Provider))
|
||||
if value.Provider == "" {
|
||||
value.Provider = aliyunOSSProvider
|
||||
}
|
||||
value.Region = strings.TrimSpace(value.Region)
|
||||
value.Endpoint = strings.TrimRight(strings.TrimSpace(value.Endpoint), "/")
|
||||
if value.Provider == tencentCOSProvider && value.Endpoint == "" && value.Region != "" {
|
||||
value.Endpoint = "https://cos." + value.Region + ".myqcloud.com"
|
||||
}
|
||||
value.CDNBaseURL = strings.TrimRight(strings.TrimSpace(value.CDNBaseURL), "/")
|
||||
value.Bucket = strings.TrimSpace(value.Bucket)
|
||||
value.AccessKeyID = strings.TrimSpace(value.AccessKeyID)
|
||||
value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret)
|
||||
value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/")
|
||||
value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/")
|
||||
if value.PathPrefix == "" {
|
||||
value.PathPrefix = defaultOSSPathPrefix
|
||||
}
|
||||
value.S3Preset = strings.ToLower(strings.TrimSpace(value.S3Preset))
|
||||
if value.S3Preset == "" {
|
||||
value.S3Preset = "custom"
|
||||
}
|
||||
value.SessionToken = strings.TrimSpace(value.SessionToken)
|
||||
value.StorageLocationID = strings.TrimSpace(value.StorageLocationID)
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
return value
|
||||
}
|
||||
|
||||
func defaultOSSSetting() ossSettingValue {
|
||||
return ossSettingValue{Provider: aliyunOSSProvider, PathPrefix: defaultOSSPathPrefix}
|
||||
}
|
||||
@@ -704,6 +647,9 @@ func (s *Service) publicOSSSetting(setting *model.SystemSetting, value ossSettin
|
||||
HasSessionToken: value.SessionToken != "",
|
||||
StorageLocationID: value.StorageLocationID,
|
||||
AllowUserS3: value.AllowUserS3,
|
||||
CDNAuthMode: value.Delivery.CDNAuthMode,
|
||||
RequireCDN: value.Delivery.RequireCDN,
|
||||
AllowPrivateProxy: value.Delivery.AllowPrivateProxy,
|
||||
}
|
||||
if setting != nil {
|
||||
result.UpdatedBy = setting.UpdatedBy
|
||||
@@ -733,6 +679,9 @@ func (s *Service) publicUserOSSSetting(setting *model.UserOSSSetting, value ossS
|
||||
HasSessionToken: value.SessionToken != "",
|
||||
StorageLocationID: value.StorageLocationID,
|
||||
AllowUserS3: allowUserS3,
|
||||
CDNAuthMode: value.Delivery.CDNAuthMode,
|
||||
RequireCDN: value.Delivery.RequireCDN,
|
||||
AllowPrivateProxy: value.Delivery.AllowPrivateProxy,
|
||||
}
|
||||
if value.Provider == s3Provider && !allowUserS3 {
|
||||
result.Enabled = false
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package app
|
||||
|
||||
import "infinite-canvas/backend/internal/storage"
|
||||
|
||||
type ossSettingValue = storage.Settings
|
||||
type ossProviderCredentials = storage.Credentials
|
||||
type ossObjectStream = storage.ObjectStream
|
||||
|
||||
var (
|
||||
putOSSObject = storage.PutOSSObject
|
||||
putAliyunOSSObject = storage.PutAliyunOSSObject
|
||||
getOSSObjectRange = storage.GetOSSObjectRange
|
||||
getAliyunOSSObjectRange = storage.GetAliyunOSSObjectRange
|
||||
signedOSSObjectURL = storage.SignedOSSObjectURL
|
||||
signedAliyunOSSObjectURL = storage.SignedAliyunOSSObjectURL
|
||||
putCOSObject = storage.PutCOSObject
|
||||
putQiniuObject = storage.PutQiniuObject
|
||||
getCOSObjectRange = storage.GetCOSObjectRange
|
||||
getQiniuObjectRange = storage.GetQiniuObjectRange
|
||||
signedCOSObjectURL = storage.SignedCOSObjectURL
|
||||
signedQiniuObjectURL = storage.SignedQiniuObjectURL
|
||||
signedQiniuS3ObjectURL = storage.SignedQiniuS3ObjectURL
|
||||
qiniuS3Region = storage.QiniuS3Region
|
||||
qiniuRegion = storage.QiniuRegion
|
||||
newCOSClient = storage.NewCOSClient
|
||||
cosBucketBaseURL = storage.CosBucketBaseURL
|
||||
ossCDNBaseURL = storage.OssCDNBaseURL
|
||||
ossCDNObjectURL = storage.OssCDNObjectURL
|
||||
newOSSRequest = storage.NewOSSRequest
|
||||
ossBucketBaseURL = storage.OssBucketBaseURL
|
||||
escapeObjectKey = storage.EscapeObjectKey
|
||||
deleteAliyunOSSObject = storage.DeleteAliyunOSSObject
|
||||
deleteTencentCOSObject = storage.DeleteTencentCOSObject
|
||||
deleteQiniuObject = storage.DeleteQiniuObject
|
||||
validateStorageEndpoint = storage.ValidateStorageEndpoint
|
||||
standardAWSS3Endpoint = storage.StandardAWSS3Endpoint
|
||||
publicHTTPSStorageEndpoint = storage.PublicHTTPSStorageEndpoint
|
||||
newS3Client = storage.NewS3Client
|
||||
putS3Object = storage.PutS3Object
|
||||
getS3ObjectRange = storage.GetS3ObjectRange
|
||||
signedS3ObjectURL = storage.SignedS3ObjectURL
|
||||
deleteS3Object = storage.DeleteS3Object
|
||||
cloneOSSProviderCredentials = storage.CloneCredentials
|
||||
normalizeOSSSetting = storage.NormalizeSettings
|
||||
)
|
||||
@@ -124,11 +124,11 @@ func verifyOSSConnection(value ossSettingValue, testKey string) error {
|
||||
}
|
||||
|
||||
func verifyOSSConnectionRead(stream *ossObjectStream, payload []byte) error {
|
||||
if stream == nil || stream.body == nil {
|
||||
if stream == nil || stream.Body == nil {
|
||||
return fmt.Errorf("%w:响应为空", errOSSConnectionReadMismatch)
|
||||
}
|
||||
data, readErr := io.ReadAll(io.LimitReader(stream.body, int64(len(payload)+1)))
|
||||
closeErr := stream.body.Close()
|
||||
data, readErr := io.ReadAll(io.LimitReader(stream.Body, int64(len(payload)+1)))
|
||||
closeErr := stream.Body.Close()
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("对象存储响应读取失败:%w", readErr)
|
||||
}
|
||||
@@ -138,7 +138,7 @@ func verifyOSSConnectionRead(stream *ossObjectStream, payload []byte) error {
|
||||
|
||||
// HTTP 允许服务端忽略 Range 并返回 200 + 完整对象。应用的资源代理也会
|
||||
// 原样透传这种响应,因此连接测试应同时接受完整读取与标准的 206 分段读取。
|
||||
switch stream.statusCode {
|
||||
switch stream.StatusCode {
|
||||
case http.StatusPartialContent:
|
||||
if len(payload) >= 4 && bytes.Equal(data, payload[:4]) {
|
||||
return nil
|
||||
@@ -148,7 +148,7 @@ func verifyOSSConnectionRead(stream *ossObjectStream, payload []byte) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%w(HTTP %d,读取 %d 字节)", errOSSConnectionReadMismatch, stream.statusCode, len(data))
|
||||
return fmt.Errorf("%w(HTTP %d,读取 %d 字节)", errOSSConnectionReadMismatch, stream.StatusCode, len(data))
|
||||
}
|
||||
|
||||
func storageConnectionTestError(provider string, operation string, cause error) error {
|
||||
|
||||
@@ -80,9 +80,9 @@ func TestTencentCOSConnectionTestUsesStorageEndpointInsteadOfCDN(t *testing.T) {
|
||||
func TestOSSConnectionReadAcceptsFullObjectWhenRangeIsIgnored(t *testing.T) {
|
||||
payload := []byte("yingce-storage-test")
|
||||
stream := &ossObjectStream{
|
||||
body: io.NopCloser(strings.NewReader(string(payload))),
|
||||
statusCode: http.StatusOK,
|
||||
contentLength: int64(len(payload)),
|
||||
Body: io.NopCloser(strings.NewReader(string(payload))),
|
||||
StatusCode: http.StatusOK,
|
||||
ContentLength: int64(len(payload)),
|
||||
}
|
||||
if err := verifyOSSConnectionRead(stream, payload); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -92,9 +92,9 @@ func TestOSSConnectionReadAcceptsFullObjectWhenRangeIsIgnored(t *testing.T) {
|
||||
func TestOSSConnectionReadRejectsUnexpectedContent(t *testing.T) {
|
||||
payload := []byte("yingce-storage-test")
|
||||
stream := &ossObjectStream{
|
||||
body: io.NopCloser(strings.NewReader("error page")),
|
||||
statusCode: http.StatusOK,
|
||||
contentLength: 10,
|
||||
Body: io.NopCloser(strings.NewReader("error page")),
|
||||
StatusCode: http.StatusOK,
|
||||
ContentLength: 10,
|
||||
}
|
||||
err := verifyOSSConnectionRead(stream, payload)
|
||||
if err == nil {
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
awss3 "github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func validateStorageEndpoint(raw string) (*url.URL, error) {
|
||||
parsed, err := ValidateOutboundURL(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" {
|
||||
return nil, BadAuthRequest("对象存储 Endpoint 必须是服务根 URL,不能包含认证信息、路径、查询参数或片段")
|
||||
}
|
||||
if parsed.Scheme == "http" && !AllowedPrivateUpstreamHost(parsed.Hostname()) {
|
||||
return nil, BadAuthRequest("对象存储 HTTP Endpoint 仅允许访问 CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS 精确放行的主机")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func standardAWSS3Endpoint(endpoint string) bool {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
return host == "s3.amazonaws.com" || strings.HasPrefix(host, "s3.") && (strings.HasSuffix(host, ".amazonaws.com") || strings.HasSuffix(host, ".amazonaws.com.cn")) || strings.HasPrefix(host, "s3-") && strings.HasSuffix(host, ".amazonaws.com")
|
||||
}
|
||||
|
||||
func publicHTTPSStorageEndpoint(endpoint string) bool {
|
||||
parsed, err := url.Parse(strings.TrimSpace(endpoint))
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" {
|
||||
return false
|
||||
}
|
||||
return ValidateOutboundHost(parsed.Hostname()) == nil && !AllowedPrivateUpstreamHost(parsed.Hostname())
|
||||
}
|
||||
|
||||
func newS3Client(setting ossSettingValue, timeout time.Duration) (*awss3.S3, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
endpoint, err := validateStorageEndpoint(setting.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if setting.Region == "" || setting.Bucket == "" || setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return nil, errors.New("S3 Region、Bucket 或访问密钥不完整")
|
||||
}
|
||||
httpClient := OutboundHTTPClient(timeout)
|
||||
config := aws.NewConfig().
|
||||
WithRegion(setting.Region).
|
||||
WithEndpoint(endpoint.String()).
|
||||
WithCredentials(credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, setting.SessionToken)).
|
||||
WithHTTPClient(httpClient).
|
||||
WithS3ForcePathStyle(setting.PathStyle || !standardAWSS3Endpoint(endpoint.String())).
|
||||
WithDisableSSL(endpoint.Scheme == "http")
|
||||
sess, err := session.NewSession(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return awss3.New(sess), nil
|
||||
}
|
||||
|
||||
func putS3Object(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
client, err := newS3Client(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var seekable io.ReadSeeker
|
||||
if reader, ok := body.(io.ReadSeeker); ok {
|
||||
seekable = reader
|
||||
} else {
|
||||
limit := size + 1
|
||||
if limit <= 0 {
|
||||
limit = 64 << 20
|
||||
}
|
||||
data, readErr := io.ReadAll(io.LimitReader(body, limit))
|
||||
if readErr != nil {
|
||||
return "", readErr
|
||||
}
|
||||
if size >= 0 && int64(len(data)) != size {
|
||||
return "", errors.New("S3 上传内容长度与资源记录不一致")
|
||||
}
|
||||
seekable = bytes.NewReader(data)
|
||||
}
|
||||
input := &awss3.PutObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/")), Body: seekable}
|
||||
if mimeType != "" {
|
||||
input.ContentType = aws.String(mimeType)
|
||||
}
|
||||
if size >= 0 {
|
||||
input.ContentLength = aws.Int64(size)
|
||||
}
|
||||
output, err := client.PutObjectWithContext(context.Background(), input)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("S3 上传失败:%w", err)
|
||||
}
|
||||
return strings.Trim(aws.StringValue(output.ETag), `"`), nil
|
||||
}
|
||||
|
||||
func getS3ObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
client, err := newS3Client(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
input := &awss3.GetObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))}
|
||||
if rangeHeader != "" {
|
||||
input.Range = aws.String(rangeHeader)
|
||||
}
|
||||
output, err := client.GetObjectWithContext(context.Background(), input)
|
||||
if err != nil {
|
||||
if requestFailure, ok := err.(awserr.RequestFailure); ok && requestFailure.StatusCode() == http.StatusRequestedRangeNotSatisfiable {
|
||||
return &ossObjectStream{body: io.NopCloser(bytes.NewReader(nil)), statusCode: http.StatusRequestedRangeNotSatisfiable, acceptRanges: "bytes"}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("S3 读取失败:%w", err)
|
||||
}
|
||||
status := http.StatusOK
|
||||
if rangeHeader != "" && aws.StringValue(output.ContentRange) != "" {
|
||||
status = http.StatusPartialContent
|
||||
}
|
||||
return &ossObjectStream{body: output.Body, statusCode: status, contentLength: aws.Int64Value(output.ContentLength), contentRange: aws.StringValue(output.ContentRange), acceptRanges: firstNonEmpty(aws.StringValue(output.AcceptRanges), "bytes")}, nil
|
||||
}
|
||||
|
||||
func signedS3ObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
client, err := newS3Client(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
duration := time.Until(expiresAt)
|
||||
if duration <= 0 {
|
||||
return "", errors.New("S3 签名有效期必须晚于当前时间")
|
||||
}
|
||||
req, _ := client.GetObjectRequest(&awss3.GetObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))})
|
||||
value, err := req.Presign(duration)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("S3 下载地址签名失败:%w", err)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func deleteS3Object(setting ossSettingValue, objectKey string) error {
|
||||
client, err := newS3Client(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = client.DeleteObjectWithContext(context.Background(), &awss3.DeleteObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))})
|
||||
if requestFailure, ok := err.(awserr.RequestFailure); ok && requestFailure.StatusCode() == http.StatusNotFound {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除 S3 对象失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -57,9 +57,9 @@ func TestS3ObjectOperationsUsePathStyleSessionTokenAndNoManagedHeaders(t *testin
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := io.ReadAll(stream.body)
|
||||
_ = stream.body.Close()
|
||||
if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "payl" {
|
||||
data, _ := io.ReadAll(stream.Body)
|
||||
_ = stream.Body.Close()
|
||||
if stream.StatusCode != http.StatusPartialContent || stream.ContentRange != "bytes 0-3/7" || string(data) != "payl" {
|
||||
t.Fatalf("stream = %#v, body = %q", stream, data)
|
||||
}
|
||||
if err := deleteS3Object(setting, "prefix/object.txt"); err != nil {
|
||||
|
||||
Reference in new issue
Block a user