支持 S3 兼容对象存储 (#336)
* feat(storage): 支持 S3 对象存储 * fix(storage): 补齐 S3 对象文件后缀
This commit is contained in:
25 files changed
+1374
-88
No files matched your search
@@ -4,13 +4,16 @@ import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -21,11 +24,13 @@ import (
|
||||
|
||||
const ossSettingKey = "oss"
|
||||
const encryptedSettingPrefix = "enc:v1:"
|
||||
const defaultOSSPathPrefix = "open-ai-canvas"
|
||||
|
||||
const (
|
||||
aliyunOSSProvider = "aliyun"
|
||||
tencentCOSProvider = "tencent"
|
||||
qiniuKodoProvider = "qiniu"
|
||||
s3Provider = "s3"
|
||||
)
|
||||
|
||||
type OSSSettingRequest struct {
|
||||
@@ -39,35 +44,53 @@ type OSSSettingRequest struct {
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
S3Preset string `json:"s3Preset"`
|
||||
PathStyle bool `json:"pathStyle"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
}
|
||||
|
||||
type PublicOSSSetting struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
HasAccessKeySecret bool `json:"hasAccessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
UpdatedBy string `json:"updatedBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
HasAccessKeySecret bool `json:"hasAccessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
S3Preset string `json:"s3Preset"`
|
||||
PathStyle bool `json:"pathStyle"`
|
||||
HasSessionToken bool `json:"hasSessionToken"`
|
||||
StorageLocationID string `json:"storageLocationId"`
|
||||
TestedAt *time.Time `json:"testedAt,omitempty"`
|
||||
TestedDigest string `json:"testedDigest,omitempty"`
|
||||
HistoryCount int64 `json:"historyCount"`
|
||||
ReferencedResourceCount int64 `json:"referencedResourceCount"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
UpdatedBy string `json:"updatedBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type ossSettingValue struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
CDNBaseURL string `json:"cdnBaseUrl"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
S3Preset string `json:"s3Preset"`
|
||||
PathStyle bool `json:"pathStyle"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
StorageLocationID string `json:"storageLocationId"`
|
||||
AllowUserS3 bool `json:"allowUserS3"`
|
||||
// 平台切换云厂商后仍需读取历史资源,因此仅归档非当前厂商的访问密钥。
|
||||
ArchivedCredentials map[string]ossProviderCredentials `json:"archivedCredentials,omitempty"`
|
||||
}
|
||||
@@ -85,7 +108,10 @@ func (s *Service) AdminOSSSetting(actor *model.User) (*PublicOSSSetting, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
public := publicOSSSetting(setting, value)
|
||||
public, err := s.publicOSSSetting(setting, value, "platform", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &public, nil
|
||||
}
|
||||
|
||||
@@ -110,6 +136,15 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P
|
||||
}
|
||||
}
|
||||
next = archiveOSSProviderCredentials(next, currentValue)
|
||||
if next.Provider == s3Provider && next.Enabled {
|
||||
location, err := s.requireTestedS3Location("platform", "", next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
next.StorageLocationID = location.ID
|
||||
} else if next.Provider == s3Provider && storageLocationDigest(next) == storageLocationDigest(currentValue) {
|
||||
next.StorageLocationID = currentValue.StorageLocationID
|
||||
}
|
||||
stored, err := s.encryptOSSSettingSecrets(next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -129,7 +164,19 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P
|
||||
if err := s.repo.SaveSystemSetting(&setting); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
public := publicOSSSetting(&setting, next)
|
||||
if next.Provider == s3Provider && next.StorageLocationID != "" {
|
||||
if err := s.repo.ActivateStorageLocation("platform", "", next.StorageLocationID, next.Enabled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if currentValue.Provider == s3Provider && currentValue.StorageLocationID != "" {
|
||||
if err := s.repo.ActivateStorageLocation("platform", "", "", false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
public, err := s.publicOSSSetting(&setting, next, "platform", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &public, nil
|
||||
}
|
||||
|
||||
@@ -141,7 +188,14 @@ func (s *Service) UserOSSSetting(actor *model.User) (*PublicOSSSetting, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
public := publicUserOSSSetting(setting, value)
|
||||
_, platform, err := s.readOSSSetting()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
public, err := s.publicUserOSSSetting(setting, value, actor.ID, platform.AllowUserS3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &public, nil
|
||||
}
|
||||
|
||||
@@ -157,6 +211,22 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, platform, err := s.readOSSSetting()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if next.Enabled && next.Provider == s3Provider && !platform.AllowUserS3 {
|
||||
return nil, Forbidden("平台管理员尚未允许个人 S3 兼容存储")
|
||||
}
|
||||
if next.Provider == s3Provider && next.Enabled {
|
||||
location, err := s.requireTestedS3Location("user", actor.ID, next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
next.StorageLocationID = location.ID
|
||||
} else if next.Provider == s3Provider && storageLocationDigest(next) == storageLocationDigest(currentValue) {
|
||||
next.StorageLocationID = currentValue.StorageLocationID
|
||||
}
|
||||
stored, err := s.encryptOSSSettingSecrets(next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -170,7 +240,19 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest)
|
||||
if err := s.repo.CreateUserOSSSetting(&setting); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
public := publicUserOSSSetting(&setting, next)
|
||||
if next.Provider == s3Provider && next.StorageLocationID != "" {
|
||||
if err := s.repo.ActivateStorageLocation("user", actor.ID, next.StorageLocationID, next.Enabled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if currentValue.Provider == s3Provider && currentValue.StorageLocationID != "" {
|
||||
if err := s.repo.ActivateStorageLocation("user", actor.ID, "", false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
public, err := s.publicUserOSSSetting(&setting, next, actor.ID, platform.AllowUserS3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &public, nil
|
||||
}
|
||||
|
||||
@@ -250,6 +332,10 @@ func (s *Service) encryptOSSSettingSecrets(value ossSettingValue) (ossSettingVal
|
||||
if err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
value.SessionToken, err = s.encryptSettingSecret(value.SessionToken)
|
||||
if err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
for provider, credentials := range value.ArchivedCredentials {
|
||||
credentials.AccessKeySecret, err = s.encryptSettingSecret(credentials.AccessKeySecret)
|
||||
@@ -268,6 +354,13 @@ func (s *Service) decryptOSSSettingSecrets(value *ossSettingValue) (bool, error)
|
||||
return false, err
|
||||
}
|
||||
value.AccessKeySecret = secret
|
||||
if value.SessionToken != "" && !strings.HasPrefix(value.SessionToken, encryptedSettingPrefix) {
|
||||
needsMigration = true
|
||||
}
|
||||
value.SessionToken, err = s.decryptSettingSecret(value.SessionToken)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
for provider, credentials := range value.ArchivedCredentials {
|
||||
if credentials.AccessKeySecret != "" && !strings.HasPrefix(credentials.AccessKeySecret, encryptedSettingPrefix) {
|
||||
@@ -466,15 +559,29 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
|
||||
AccessKeySecret: strings.TrimSpace(req.AccessKeySecret),
|
||||
PublicBaseURL: strings.TrimRight(strings.TrimSpace(req.PublicBaseURL), "/"),
|
||||
PathPrefix: strings.Trim(strings.TrimSpace(req.PathPrefix), "/"),
|
||||
S3Preset: strings.TrimSpace(req.S3Preset),
|
||||
PathStyle: req.PathStyle,
|
||||
SessionToken: strings.TrimSpace(req.SessionToken),
|
||||
AllowUserS3: req.AllowUserS3,
|
||||
})
|
||||
if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider {
|
||||
return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS 和七牛云 Kodo")
|
||||
if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider && next.Provider != s3Provider {
|
||||
return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS、七牛云 Kodo 和通用 S3")
|
||||
}
|
||||
if next.Provider == s3Provider {
|
||||
switch next.S3Preset {
|
||||
case "aws", "r2", "b2", "rustfs", "custom":
|
||||
default:
|
||||
return next, BadAuthRequest("S3 预设无效")
|
||||
}
|
||||
}
|
||||
current = normalizeOSSSetting(current)
|
||||
// 不同云厂商的密钥不能复用;只有继续使用同一厂商时,留空才表示保留原密钥。
|
||||
// 同一云厂商修改存储位置时,留空仍表示保留现有密钥;切换厂商时不能复用。
|
||||
if next.AccessKeySecret == "" && next.Provider == current.Provider {
|
||||
next.AccessKeySecret = current.AccessKeySecret
|
||||
}
|
||||
if next.SessionToken == "" && next.Provider == current.Provider {
|
||||
next.SessionToken = current.SessionToken
|
||||
}
|
||||
if next.Enabled {
|
||||
if next.Bucket == "" {
|
||||
return next, BadAuthRequest("请填写对象存储 Bucket")
|
||||
@@ -486,10 +593,22 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
|
||||
if next.Provider == qiniuKodoProvider {
|
||||
return next, BadAuthRequest("请填写七牛云 Kodo 上传 Endpoint")
|
||||
}
|
||||
if next.Provider == s3Provider {
|
||||
return next, BadAuthRequest("请填写 S3 Endpoint 服务根 URL")
|
||||
}
|
||||
return next, BadAuthRequest("请填写阿里云 OSS Endpoint")
|
||||
}
|
||||
if _, err := ValidateOutboundURL(next.Endpoint); err != nil {
|
||||
return next, err
|
||||
if next.Provider == s3Provider && next.Region == "" {
|
||||
return next, BadAuthRequest("请填写 S3 Region")
|
||||
}
|
||||
var endpointErr error
|
||||
if next.Provider == s3Provider {
|
||||
_, endpointErr = validateStorageEndpoint(next.Endpoint)
|
||||
} else {
|
||||
_, endpointErr = ValidateOutboundURL(next.Endpoint)
|
||||
}
|
||||
if endpointErr != nil {
|
||||
return next, endpointErr
|
||||
}
|
||||
if next.CDNBaseURL != "" {
|
||||
if _, err := ossCDNBaseURL(next.CDNBaseURL); err != nil {
|
||||
@@ -551,15 +670,24 @@ func normalizeOSSSetting(value ossSettingValue) ossSettingValue {
|
||||
value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret)
|
||||
value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/")
|
||||
value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/")
|
||||
if value.PathPrefix == "" {
|
||||
value.PathPrefix = defaultOSSPathPrefix
|
||||
}
|
||||
value.S3Preset = strings.ToLower(strings.TrimSpace(value.S3Preset))
|
||||
if value.S3Preset == "" {
|
||||
value.S3Preset = "custom"
|
||||
}
|
||||
value.SessionToken = strings.TrimSpace(value.SessionToken)
|
||||
value.StorageLocationID = strings.TrimSpace(value.StorageLocationID)
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
return value
|
||||
}
|
||||
|
||||
func defaultOSSSetting() ossSettingValue {
|
||||
return ossSettingValue{Provider: aliyunOSSProvider}
|
||||
return ossSettingValue{Provider: aliyunOSSProvider, PathPrefix: defaultOSSPathPrefix}
|
||||
}
|
||||
|
||||
func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) PublicOSSSetting {
|
||||
func (s *Service) publicOSSSetting(setting *model.SystemSetting, value ossSettingValue, scope string, ownerID string) (PublicOSSSetting, error) {
|
||||
result := PublicOSSSetting{
|
||||
Enabled: value.Enabled,
|
||||
Provider: value.Provider,
|
||||
@@ -571,16 +699,24 @@ func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) Publi
|
||||
HasAccessKeySecret: strings.TrimSpace(value.AccessKeySecret) != "",
|
||||
PublicBaseURL: value.PublicBaseURL,
|
||||
PathPrefix: value.PathPrefix,
|
||||
S3Preset: value.S3Preset,
|
||||
PathStyle: value.PathStyle,
|
||||
HasSessionToken: value.SessionToken != "",
|
||||
StorageLocationID: value.StorageLocationID,
|
||||
AllowUserS3: value.AllowUserS3,
|
||||
}
|
||||
if setting != nil {
|
||||
result.UpdatedBy = setting.UpdatedBy
|
||||
result.CreatedAt = setting.CreatedAt
|
||||
result.UpdatedAt = setting.UpdatedAt
|
||||
}
|
||||
return result
|
||||
if err := s.populateStorageLocationPublic(&result, scope, ownerID, value.StorageLocationID); err != nil {
|
||||
return PublicOSSSetting{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue) PublicOSSSetting {
|
||||
func (s *Service) publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue, ownerID string, allowUserS3 bool) (PublicOSSSetting, error) {
|
||||
result := PublicOSSSetting{
|
||||
Enabled: value.Enabled,
|
||||
Provider: value.Provider,
|
||||
@@ -592,11 +728,35 @@ func publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue)
|
||||
HasAccessKeySecret: strings.TrimSpace(value.AccessKeySecret) != "",
|
||||
PublicBaseURL: value.PublicBaseURL,
|
||||
PathPrefix: value.PathPrefix,
|
||||
S3Preset: value.S3Preset,
|
||||
PathStyle: value.PathStyle,
|
||||
HasSessionToken: value.SessionToken != "",
|
||||
StorageLocationID: value.StorageLocationID,
|
||||
AllowUserS3: allowUserS3,
|
||||
}
|
||||
if value.Provider == s3Provider && !allowUserS3 {
|
||||
result.Enabled = false
|
||||
}
|
||||
if setting != nil {
|
||||
result.UpdatedBy = setting.UserID
|
||||
result.CreatedAt = setting.CreatedAt
|
||||
result.UpdatedAt = setting.UpdatedAt
|
||||
}
|
||||
return result
|
||||
if err := s.populateStorageLocationPublic(&result, "user", ownerID, value.StorageLocationID); err != nil {
|
||||
return PublicOSSSetting{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func storageLocationDigest(value ossSettingValue) string {
|
||||
value = normalizeOSSSetting(value)
|
||||
payload := strings.Join([]string{value.Provider, value.Endpoint, value.Bucket, value.Region, strconv.FormatBool(value.PathStyle), value.PathPrefix}, "\x00")
|
||||
sum := sha256.Sum256([]byte(payload))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func storageTestDigest(value ossSettingValue) string {
|
||||
payload := strings.Join([]string{storageLocationDigest(value), value.AccessKeyID, value.AccessKeySecret, value.SessionToken}, "\x00")
|
||||
sum := sha256.Sum256([]byte(payload))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in new issue
Block a user