支持 S3 兼容对象存储 (#336)

* feat(storage): 支持 S3 对象存储

* fix(storage): 补齐 S3 对象文件后缀
This commit is contained in:
Rou authored and GitHub committed 2026-08-27 22:16:08 +08:00
1 parent d42e392ed7
commit 21b69d8bc7
25 files changed
+1374 -88

No files matched your search

+197 -37
View File
@@ -4,13 +4,16 @@ import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@@ -21,11 +24,13 @@ import (
const ossSettingKey = "oss"
const encryptedSettingPrefix = "enc:v1:"
const defaultOSSPathPrefix = "open-ai-canvas"
const (
aliyunOSSProvider = "aliyun"
tencentCOSProvider = "tencent"
qiniuKodoProvider = "qiniu"
s3Provider = "s3"
)
type OSSSettingRequest struct {
@@ -39,35 +44,53 @@ type OSSSettingRequest struct {
AccessKeySecret string `json:"accessKeySecret"`
PublicBaseURL string `json:"publicBaseUrl"`
PathPrefix string `json:"pathPrefix"`
S3Preset string `json:"s3Preset"`
PathStyle bool `json:"pathStyle"`
SessionToken string `json:"sessionToken"`
AllowUserS3 bool `json:"allowUserS3"`
}
type PublicOSSSetting struct {
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
Region string `json:"region"`
Endpoint string `json:"endpoint"`
CDNBaseURL string `json:"cdnBaseUrl"`
Bucket string `json:"bucket"`
AccessKeyID string `json:"accessKeyId"`
HasAccessKeySecret bool `json:"hasAccessKeySecret"`
PublicBaseURL string `json:"publicBaseUrl"`
PathPrefix string `json:"pathPrefix"`
UpdatedBy string `json:"updatedBy"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
Region string `json:"region"`
Endpoint string `json:"endpoint"`
CDNBaseURL string `json:"cdnBaseUrl"`
Bucket string `json:"bucket"`
AccessKeyID string `json:"accessKeyId"`
HasAccessKeySecret bool `json:"hasAccessKeySecret"`
PublicBaseURL string `json:"publicBaseUrl"`
PathPrefix string `json:"pathPrefix"`
S3Preset string `json:"s3Preset"`
PathStyle bool `json:"pathStyle"`
HasSessionToken bool `json:"hasSessionToken"`
StorageLocationID string `json:"storageLocationId"`
TestedAt *time.Time `json:"testedAt,omitempty"`
TestedDigest string `json:"testedDigest,omitempty"`
HistoryCount int64 `json:"historyCount"`
ReferencedResourceCount int64 `json:"referencedResourceCount"`
AllowUserS3 bool `json:"allowUserS3"`
UpdatedBy string `json:"updatedBy"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type ossSettingValue struct {
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
Region string `json:"region"`
Endpoint string `json:"endpoint"`
CDNBaseURL string `json:"cdnBaseUrl"`
Bucket string `json:"bucket"`
AccessKeyID string `json:"accessKeyId"`
AccessKeySecret string `json:"accessKeySecret"`
PublicBaseURL string `json:"publicBaseUrl"`
PathPrefix string `json:"pathPrefix"`
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
Region string `json:"region"`
Endpoint string `json:"endpoint"`
CDNBaseURL string `json:"cdnBaseUrl"`
Bucket string `json:"bucket"`
AccessKeyID string `json:"accessKeyId"`
AccessKeySecret string `json:"accessKeySecret"`
PublicBaseURL string `json:"publicBaseUrl"`
PathPrefix string `json:"pathPrefix"`
S3Preset string `json:"s3Preset"`
PathStyle bool `json:"pathStyle"`
SessionToken string `json:"sessionToken"`
StorageLocationID string `json:"storageLocationId"`
AllowUserS3 bool `json:"allowUserS3"`
// 平台切换云厂商后仍需读取历史资源,因此仅归档非当前厂商的访问密钥。
ArchivedCredentials map[string]ossProviderCredentials `json:"archivedCredentials,omitempty"`
}
@@ -85,7 +108,10 @@ func (s *Service) AdminOSSSetting(actor *model.User) (*PublicOSSSetting, error)
if err != nil {
return nil, err
}
public := publicOSSSetting(setting, value)
public, err := s.publicOSSSetting(setting, value, "platform", "")
if err != nil {
return nil, err
}
return &public, nil
}
@@ -110,6 +136,15 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P
}
}
next = archiveOSSProviderCredentials(next, currentValue)
if next.Provider == s3Provider && next.Enabled {
location, err := s.requireTestedS3Location("platform", "", next)
if err != nil {
return nil, err
}
next.StorageLocationID = location.ID
} else if next.Provider == s3Provider && storageLocationDigest(next) == storageLocationDigest(currentValue) {
next.StorageLocationID = currentValue.StorageLocationID
}
stored, err := s.encryptOSSSettingSecrets(next)
if err != nil {
return nil, err
@@ -129,7 +164,19 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P
if err := s.repo.SaveSystemSetting(&setting); err != nil {
return nil, err
}
public := publicOSSSetting(&setting, next)
if next.Provider == s3Provider && next.StorageLocationID != "" {
if err := s.repo.ActivateStorageLocation("platform", "", next.StorageLocationID, next.Enabled); err != nil {
return nil, err
}
} else if currentValue.Provider == s3Provider && currentValue.StorageLocationID != "" {
if err := s.repo.ActivateStorageLocation("platform", "", "", false); err != nil {
return nil, err
}
}
public, err := s.publicOSSSetting(&setting, next, "platform", "")
if err != nil {
return nil, err
}
return &public, nil
}
@@ -141,7 +188,14 @@ func (s *Service) UserOSSSetting(actor *model.User) (*PublicOSSSetting, error) {
if err != nil {
return nil, err
}
public := publicUserOSSSetting(setting, value)
_, platform, err := s.readOSSSetting()
if err != nil {
return nil, err
}
public, err := s.publicUserOSSSetting(setting, value, actor.ID, platform.AllowUserS3)
if err != nil {
return nil, err
}
return &public, nil
}
@@ -157,6 +211,22 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest)
if err != nil {
return nil, err
}
_, platform, err := s.readOSSSetting()
if err != nil {
return nil, err
}
if next.Enabled && next.Provider == s3Provider && !platform.AllowUserS3 {
return nil, Forbidden("平台管理员尚未允许个人 S3 兼容存储")
}
if next.Provider == s3Provider && next.Enabled {
location, err := s.requireTestedS3Location("user", actor.ID, next)
if err != nil {
return nil, err
}
next.StorageLocationID = location.ID
} else if next.Provider == s3Provider && storageLocationDigest(next) == storageLocationDigest(currentValue) {
next.StorageLocationID = currentValue.StorageLocationID
}
stored, err := s.encryptOSSSettingSecrets(next)
if err != nil {
return nil, err
@@ -170,7 +240,19 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest)
if err := s.repo.CreateUserOSSSetting(&setting); err != nil {
return nil, err
}
public := publicUserOSSSetting(&setting, next)
if next.Provider == s3Provider && next.StorageLocationID != "" {
if err := s.repo.ActivateStorageLocation("user", actor.ID, next.StorageLocationID, next.Enabled); err != nil {
return nil, err
}
} else if currentValue.Provider == s3Provider && currentValue.StorageLocationID != "" {
if err := s.repo.ActivateStorageLocation("user", actor.ID, "", false); err != nil {
return nil, err
}
}
public, err := s.publicUserOSSSetting(&setting, next, actor.ID, platform.AllowUserS3)
if err != nil {
return nil, err
}
return &public, nil
}
@@ -250,6 +332,10 @@ func (s *Service) encryptOSSSettingSecrets(value ossSettingValue) (ossSettingVal
if err != nil {
return ossSettingValue{}, err
}
value.SessionToken, err = s.encryptSettingSecret(value.SessionToken)
if err != nil {
return ossSettingValue{}, err
}
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
for provider, credentials := range value.ArchivedCredentials {
credentials.AccessKeySecret, err = s.encryptSettingSecret(credentials.AccessKeySecret)
@@ -268,6 +354,13 @@ func (s *Service) decryptOSSSettingSecrets(value *ossSettingValue) (bool, error)
return false, err
}
value.AccessKeySecret = secret
if value.SessionToken != "" && !strings.HasPrefix(value.SessionToken, encryptedSettingPrefix) {
needsMigration = true
}
value.SessionToken, err = s.decryptSettingSecret(value.SessionToken)
if err != nil {
return false, err
}
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
for provider, credentials := range value.ArchivedCredentials {
if credentials.AccessKeySecret != "" && !strings.HasPrefix(credentials.AccessKeySecret, encryptedSettingPrefix) {
@@ -466,15 +559,29 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
AccessKeySecret: strings.TrimSpace(req.AccessKeySecret),
PublicBaseURL: strings.TrimRight(strings.TrimSpace(req.PublicBaseURL), "/"),
PathPrefix: strings.Trim(strings.TrimSpace(req.PathPrefix), "/"),
S3Preset: strings.TrimSpace(req.S3Preset),
PathStyle: req.PathStyle,
SessionToken: strings.TrimSpace(req.SessionToken),
AllowUserS3: req.AllowUserS3,
})
if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider {
return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS 和七牛云 Kodo")
if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider && next.Provider != s3Provider {
return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS、七牛云 Kodo 和通用 S3")
}
if next.Provider == s3Provider {
switch next.S3Preset {
case "aws", "r2", "b2", "rustfs", "custom":
default:
return next, BadAuthRequest("S3 预设无效")
}
}
current = normalizeOSSSetting(current)
// 不同云厂商的密钥不能复用;只有继续使用同一厂商时,留空才表示保留原密钥。
// 同一云厂商修改存储位置时,留空仍表示保留现有密钥;切换厂商时不能复用。
if next.AccessKeySecret == "" && next.Provider == current.Provider {
next.AccessKeySecret = current.AccessKeySecret
}
if next.SessionToken == "" && next.Provider == current.Provider {
next.SessionToken = current.SessionToken
}
if next.Enabled {
if next.Bucket == "" {
return next, BadAuthRequest("请填写对象存储 Bucket")
@@ -486,10 +593,22 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
if next.Provider == qiniuKodoProvider {
return next, BadAuthRequest("请填写七牛云 Kodo 上传 Endpoint")
}
if next.Provider == s3Provider {
return next, BadAuthRequest("请填写 S3 Endpoint 服务根 URL")
}
return next, BadAuthRequest("请填写阿里云 OSS Endpoint")
}
if _, err := ValidateOutboundURL(next.Endpoint); err != nil {
return next, err
if next.Provider == s3Provider && next.Region == "" {
return next, BadAuthRequest("请填写 S3 Region")
}
var endpointErr error
if next.Provider == s3Provider {
_, endpointErr = validateStorageEndpoint(next.Endpoint)
} else {
_, endpointErr = ValidateOutboundURL(next.Endpoint)
}
if endpointErr != nil {
return next, endpointErr
}
if next.CDNBaseURL != "" {
if _, err := ossCDNBaseURL(next.CDNBaseURL); err != nil {
@@ -551,15 +670,24 @@ func normalizeOSSSetting(value ossSettingValue) ossSettingValue {
value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret)
value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/")
value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/")
if value.PathPrefix == "" {
value.PathPrefix = defaultOSSPathPrefix
}
value.S3Preset = strings.ToLower(strings.TrimSpace(value.S3Preset))
if value.S3Preset == "" {
value.S3Preset = "custom"
}
value.SessionToken = strings.TrimSpace(value.SessionToken)
value.StorageLocationID = strings.TrimSpace(value.StorageLocationID)
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
return value
}
func defaultOSSSetting() ossSettingValue {
return ossSettingValue{Provider: aliyunOSSProvider}
return ossSettingValue{Provider: aliyunOSSProvider, PathPrefix: defaultOSSPathPrefix}
}
func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) PublicOSSSetting {
func (s *Service) publicOSSSetting(setting *model.SystemSetting, value ossSettingValue, scope string, ownerID string) (PublicOSSSetting, error) {
result := PublicOSSSetting{
Enabled: value.Enabled,
Provider: value.Provider,
@@ -571,16 +699,24 @@ func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) Publi
HasAccessKeySecret: strings.TrimSpace(value.AccessKeySecret) != "",
PublicBaseURL: value.PublicBaseURL,
PathPrefix: value.PathPrefix,
S3Preset: value.S3Preset,
PathStyle: value.PathStyle,
HasSessionToken: value.SessionToken != "",
StorageLocationID: value.StorageLocationID,
AllowUserS3: value.AllowUserS3,
}
if setting != nil {
result.UpdatedBy = setting.UpdatedBy
result.CreatedAt = setting.CreatedAt
result.UpdatedAt = setting.UpdatedAt
}
return result
if err := s.populateStorageLocationPublic(&result, scope, ownerID, value.StorageLocationID); err != nil {
return PublicOSSSetting{}, err
}
return result, nil
}
func publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue) PublicOSSSetting {
func (s *Service) publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue, ownerID string, allowUserS3 bool) (PublicOSSSetting, error) {
result := PublicOSSSetting{
Enabled: value.Enabled,
Provider: value.Provider,
@@ -592,11 +728,35 @@ func publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue)
HasAccessKeySecret: strings.TrimSpace(value.AccessKeySecret) != "",
PublicBaseURL: value.PublicBaseURL,
PathPrefix: value.PathPrefix,
S3Preset: value.S3Preset,
PathStyle: value.PathStyle,
HasSessionToken: value.SessionToken != "",
StorageLocationID: value.StorageLocationID,
AllowUserS3: allowUserS3,
}
if value.Provider == s3Provider && !allowUserS3 {
result.Enabled = false
}
if setting != nil {
result.UpdatedBy = setting.UserID
result.CreatedAt = setting.CreatedAt
result.UpdatedAt = setting.UpdatedAt
}
return result
if err := s.populateStorageLocationPublic(&result, "user", ownerID, value.StorageLocationID); err != nil {
return PublicOSSSetting{}, err
}
return result, nil
}
func storageLocationDigest(value ossSettingValue) string {
value = normalizeOSSSetting(value)
payload := strings.Join([]string{value.Provider, value.Endpoint, value.Bucket, value.Region, strconv.FormatBool(value.PathStyle), value.PathPrefix}, "\x00")
sum := sha256.Sum256([]byte(payload))
return hex.EncodeToString(sum[:])
}
func storageTestDigest(value ossSettingValue) string {
payload := strings.Join([]string{storageLocationDigest(value), value.AccessKeyID, value.AccessKeySecret, value.SessionToken}, "\x00")
sum := sha256.Sum256([]byte(payload))
return hex.EncodeToString(sum[:])
}