feat(media): 对象存储 - 新增腾讯云 COS 与历史凭据兼容
Co-authored-by: MasterChiefCN <49179444+MasterChiefCN@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Original-PR: https://github.com/ddcat-ai/open-ai-canvas/pull/218
This commit is contained in:
1 parent
3ddc35cbe1
commit
cb17763d27
16 files changed
+497
-83
No files matched your search
@@ -491,7 +491,7 @@ func (s *Service) hydrateGenerationMedia(userID string, input *canvasGenerationI
|
||||
func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requirePublicURL bool) error {
|
||||
if !strings.HasPrefix(media.StorageKey, "resource:") {
|
||||
if requirePublicURL && strings.HasPrefix(strings.TrimSpace(media.DataURL), "data:") {
|
||||
return errors.New("当前 JSON 视频协议的参考素材不能使用内嵌数据,请先上传到 OSS 或提供公网素材地址")
|
||||
return errors.New("当前 JSON 视频协议的参考素材不能使用内嵌数据,请先上传到对象存储或提供公网素材地址")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1953,7 +1953,7 @@ func videoGenerationsMediaURL(media providerMedia) (string, error) {
|
||||
if isPublicMediaURL(value) || strings.HasPrefix(value, "data:") {
|
||||
return value, nil
|
||||
}
|
||||
return "", errors.New("NewAPI Video Generations 的参考素材需要公网 URL;私有素材请先保存到 OSS")
|
||||
return "", errors.New("NewAPI Video Generations 的参考素材需要公网 URL;私有素材请先保存到对象存储")
|
||||
}
|
||||
|
||||
func normalizeNewAPIChannel2Ratio(value string, modelName string) string {
|
||||
@@ -2108,7 +2108,7 @@ func newAPIChannel1VideoBody(input canvasGenerationInput) (map[string]interface{
|
||||
func newAPIChannel1MediaURL(media providerMedia) (string, error) {
|
||||
value := strings.TrimSpace(media.URL)
|
||||
if !isPublicMediaURL(value) {
|
||||
return "", errors.New("NewAPI 媒体任务的参考素材必须使用公网 HTTP(S) URL,请启用 OSS 或提供公网素材地址")
|
||||
return "", errors.New("NewAPI 媒体任务的参考素材必须使用公网 HTTP(S) URL,请启用对象存储或提供公网素材地址")
|
||||
}
|
||||
if _, err := ValidateOutboundURL(value); err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
@@ -26,6 +27,8 @@ import (
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
|
||||
cos "github.com/tencentyun/cos-go-sdk-v5"
|
||||
)
|
||||
|
||||
const providerResourceURLTTL = 4 * time.Hour
|
||||
@@ -278,7 +281,7 @@ func (s *Service) openResourceRange(userID string, resource *model.Resource, ran
|
||||
return nil, err
|
||||
}
|
||||
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return nil, errors.New("OSS 访问密钥不可用")
|
||||
return nil, errors.New("对象存储访问密钥不可用")
|
||||
}
|
||||
setting.Provider = firstNonEmpty(resource.Provider, setting.Provider)
|
||||
setting.Endpoint = firstNonEmpty(resource.Endpoint, setting.Endpoint)
|
||||
@@ -637,22 +640,41 @@ func (s *Service) ossSettingForResource(userID string, resource *model.Resource)
|
||||
if err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
setting.Provider = firstNonEmpty(resource.Provider, setting.Provider)
|
||||
setting, err = ossSettingForProvider(setting, firstNonEmpty(resource.Provider, setting.Provider))
|
||||
if err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
setting.Endpoint = firstNonEmpty(resource.Endpoint, setting.Endpoint)
|
||||
setting.Bucket = firstNonEmpty(resource.Bucket, setting.Bucket)
|
||||
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return ossSettingValue{}, errors.New("OSS 访问密钥不可用")
|
||||
return ossSettingValue{}, errors.New("对象存储访问密钥不可用")
|
||||
}
|
||||
return setting, nil
|
||||
}
|
||||
|
||||
func ossSettingForProvider(setting ossSettingValue, provider string) (ossSettingValue, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
if provider == "" || provider == setting.Provider {
|
||||
return setting, nil
|
||||
}
|
||||
credentials, ok := setting.ArchivedCredentials[provider]
|
||||
if !ok || credentials.AccessKeyID == "" || credentials.AccessKeySecret == "" {
|
||||
return ossSettingValue{}, errors.New("历史对象存储访问密钥不可用")
|
||||
}
|
||||
setting.Provider = provider
|
||||
setting.AccessKeyID = credentials.AccessKeyID
|
||||
setting.AccessKeySecret = credentials.AccessKeySecret
|
||||
return setting, nil
|
||||
}
|
||||
|
||||
func validateActiveOSSSetting(setting ossSettingValue, disabledMessage string, incompleteMessage string) (ossSettingValue, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
if !setting.Enabled {
|
||||
return ossSettingValue{}, BadAuthRequest(disabledMessage)
|
||||
}
|
||||
if setting.Provider != "aliyun" {
|
||||
return ossSettingValue{}, BadAuthRequest("暂时只支持阿里云 OSS")
|
||||
if setting.Provider != aliyunOSSProvider && setting.Provider != tencentCOSProvider {
|
||||
return ossSettingValue{}, BadAuthRequest("仅支持阿里云 OSS 和腾讯云 COS")
|
||||
}
|
||||
if setting.Bucket == "" || setting.Endpoint == "" || setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||
return ossSettingValue{}, BadAuthRequest(incompleteMessage)
|
||||
@@ -686,6 +708,14 @@ func ossObjectKey(setting ossSettingValue, userID string, kind string, fileName
|
||||
}
|
||||
|
||||
func putOSSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
if normalizeOSSSetting(setting).Provider == tencentCOSProvider {
|
||||
return putCOSObject(setting, objectKey, mimeType, size, body)
|
||||
}
|
||||
return putAliyunOSSObject(setting, objectKey, mimeType, size, body)
|
||||
}
|
||||
|
||||
// 阿里云 OSS 继续沿用原有 V1 签名和请求路径,避免已有部署行为发生变化。
|
||||
func putAliyunOSSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
@@ -717,6 +747,13 @@ type ossObjectStream struct {
|
||||
}
|
||||
|
||||
func getOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
if normalizeOSSSetting(setting).Provider == tencentCOSProvider {
|
||||
return getCOSObjectRange(setting, objectKey, rangeHeader)
|
||||
}
|
||||
return getAliyunOSSObjectRange(setting, objectKey, rangeHeader)
|
||||
}
|
||||
|
||||
func getAliyunOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
req, err := newOSSRequest(http.MethodGet, setting, objectKey, "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -758,6 +795,13 @@ func decimalDigits(value string) bool {
|
||||
}
|
||||
|
||||
func signedOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
if normalizeOSSSetting(setting).Provider == tencentCOSProvider {
|
||||
return signedCOSObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
return signedAliyunOSSObjectURL(setting, objectKey, expiresAt)
|
||||
}
|
||||
|
||||
func signedAliyunOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
baseURL, err := ossBucketBaseURL(setting)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -782,6 +826,101 @@ func signedOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt tim
|
||||
return baseURL.String(), nil
|
||||
}
|
||||
|
||||
func putCOSObject(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) {
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
options := &cos.ObjectPutOptions{ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ContentType: mimeType, ContentLength: size}}
|
||||
resp, err := client.Object.Put(context.Background(), objectKey, body, options)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("COS 上传失败:%w", err)
|
||||
}
|
||||
return strings.Trim(resp.Header.Get("ETag"), `"`), nil
|
||||
}
|
||||
|
||||
func getCOSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) {
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options := &cos.ObjectGetOptions{Range: rangeHeader}
|
||||
resp, err := client.Object.Get(context.Background(), objectKey, options)
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
|
||||
return &ossObjectStream{body: io.NopCloser(bytes.NewReader(nil)), statusCode: resp.StatusCode, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("COS 读取失败:%w", err)
|
||||
}
|
||||
return &ossObjectStream{body: resp.Body, statusCode: resp.StatusCode, contentLength: resp.ContentLength, contentRange: resp.Header.Get("Content-Range"), acceptRanges: firstNonEmpty(resp.Header.Get("Accept-Ranges"), "bytes")}, nil
|
||||
}
|
||||
|
||||
func signedCOSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) {
|
||||
if strings.TrimSpace(setting.AccessKeyID) == "" || strings.TrimSpace(setting.AccessKeySecret) == "" {
|
||||
return "", errors.New("COS 访问密钥不可用")
|
||||
}
|
||||
objectKey = strings.TrimLeft(strings.TrimSpace(objectKey), "/")
|
||||
if objectKey == "" {
|
||||
return "", errors.New("COS 对象路径为空")
|
||||
}
|
||||
expires := time.Until(expiresAt)
|
||||
if expires <= 0 {
|
||||
return "", errors.New("COS 签名有效期必须晚于当前时间")
|
||||
}
|
||||
client, err := newCOSClient(setting, 2*time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
signedURL, err := client.Object.GetPresignedURL(context.Background(), http.MethodGet, objectKey, setting.AccessKeyID, setting.AccessKeySecret, expires, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return signedURL.String(), nil
|
||||
}
|
||||
|
||||
func newCOSClient(setting ossSettingValue, timeout time.Duration) (*cos.Client, error) {
|
||||
bucketURL, err := cosBucketBaseURL(setting)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpClient := OutboundHTTPClient(timeout)
|
||||
httpClient.Transport = &cos.AuthorizationTransport{SecretID: setting.AccessKeyID, SecretKey: setting.AccessKeySecret, Transport: httpClient.Transport}
|
||||
return cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, httpClient), nil
|
||||
}
|
||||
|
||||
func cosBucketBaseURL(setting ossSettingValue) (*url.URL, error) {
|
||||
setting = normalizeOSSSetting(setting)
|
||||
endpoint := strings.TrimRight(setting.Endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return nil, errors.New("COS Endpoint 为空")
|
||||
}
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" {
|
||||
return nil, errors.New("COS Endpoint 格式不正确")
|
||||
}
|
||||
if setting.Bucket == "" {
|
||||
return nil, errors.New("COS Bucket 为空")
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
if strings.HasSuffix(host, ".myqcloud.com") || strings.HasSuffix(host, ".tencentcos.cn") {
|
||||
if strings.HasPrefix(host, "cos.") || strings.HasPrefix(host, "cos-internal.") || strings.HasPrefix(host, "cos-website.") {
|
||||
parsed.Host = setting.Bucket + "." + parsed.Host
|
||||
} else if !strings.HasPrefix(host, strings.ToLower(setting.Bucket)+".") {
|
||||
return nil, errors.New("COS Endpoint 中的 Bucket 与配置不一致")
|
||||
}
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func newOSSRequest(method string, setting ossSettingValue, objectKey string, contentType string, body io.Reader) (*http.Request, error) {
|
||||
baseURL, err := ossBucketBaseURL(setting)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"hash/crc64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -39,6 +42,155 @@ func TestSignedOSSObjectURLUsesExpiringQuerySignature(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignedOSSObjectURLSupportsTencentCOS(t *testing.T) {
|
||||
value, err := signedOSSObjectURL(ossSettingValue{
|
||||
Provider: tencentCOSProvider, Region: "ap-guangzhou", Bucket: "private-bucket-1250000000",
|
||||
AccessKeyID: "secret-id", AccessKeySecret: "secret-key",
|
||||
}, "users/u-1/image/test image.png", time.Now().Add(time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("signedOSSObjectURL() error = %v", err)
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := parsed.Query()
|
||||
if parsed.Host != "private-bucket-1250000000.cos.ap-guangzhou.myqcloud.com" || query.Get("q-sign-algorithm") != "sha1" || query.Get("q-ak") != "secret-id" || query.Get("q-signature") == "" {
|
||||
t.Fatalf("signed COS URL = %q", value)
|
||||
}
|
||||
if strings.Contains(value, "secret-key") {
|
||||
t.Fatalf("signed COS URL leaked secret key: %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutOSSObjectSupportsTencentCOS(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
payload := []byte("cos upload payload")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut || r.URL.Path != "/users/u-1/image/test.png" {
|
||||
t.Errorf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if r.Header.Get("Content-Type") != "image/png" {
|
||||
t.Errorf("Content-Type = %q", r.Header.Get("Content-Type"))
|
||||
}
|
||||
authorization := r.Header.Get("Authorization")
|
||||
if !strings.Contains(authorization, "q-sign-algorithm=sha1") || !strings.Contains(authorization, "q-ak=secret-id") {
|
||||
t.Errorf("Authorization = %q", authorization)
|
||||
}
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !bytes.Equal(data, payload) {
|
||||
t.Errorf("body = %q", data)
|
||||
}
|
||||
w.Header().Set("ETag", `"cos-etag"`)
|
||||
w.Header().Set("x-cos-hash-crc64ecma", strconv.FormatUint(crc64.Checksum(data, crc64.MakeTable(crc64.ECMA)), 10))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
etag, err := putOSSObject(ossSettingValue{
|
||||
Provider: tencentCOSProvider, Endpoint: server.URL, Bucket: "private-bucket-1250000000",
|
||||
AccessKeyID: "secret-id", AccessKeySecret: "secret-key",
|
||||
}, "users/u-1/image/test.png", "image/png", int64(len(payload)), bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if etag != "cos-etag" {
|
||||
t.Fatalf("ETag = %q", etag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOSSObjectRangeSupportsTencentCOS(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Range") != "bytes=0-3" {
|
||||
t.Errorf("Range = %q", r.Header.Get("Range"))
|
||||
}
|
||||
authorization := r.Header.Get("Authorization")
|
||||
if !strings.Contains(authorization, "q-sign-algorithm=sha1") || !strings.Contains(authorization, "q-ak=secret-id") {
|
||||
t.Errorf("Authorization = %q", authorization)
|
||||
}
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("Content-Range", "bytes 0-3/7")
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write([]byte("data"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
stream, err := getOSSObjectRange(ossSettingValue{
|
||||
Provider: tencentCOSProvider, Endpoint: server.URL, Bucket: "private-bucket-1250000000",
|
||||
AccessKeyID: "secret-id", AccessKeySecret: "secret-key",
|
||||
}, "users/u-1/image/test.png", "bytes=0-3")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stream.body.Close()
|
||||
data, err := io.ReadAll(stream.body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "data" {
|
||||
t.Fatalf("stream = %#v, data = %q", stream, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTencentCOSSettingDerivesEndpointAndDoesNotReuseAliyunSecret(t *testing.T) {
|
||||
normalized := normalizeOSSSetting(ossSettingValue{Provider: tencentCOSProvider, Region: "ap-shanghai"})
|
||||
if normalized.Endpoint != "https://cos.ap-shanghai.myqcloud.com" {
|
||||
t.Fatalf("Endpoint = %q", normalized.Endpoint)
|
||||
}
|
||||
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
||||
defer server.Close()
|
||||
_, err := ossSettingFromRequest(OSSSettingRequest{
|
||||
Enabled: true, Provider: tencentCOSProvider, Endpoint: server.URL, Bucket: "private-bucket-1250000000", AccessKeyID: "secret-id",
|
||||
}, ossSettingValue{Provider: aliyunOSSProvider, AccessKeySecret: "aliyun-secret"})
|
||||
if err == nil || !strings.Contains(err.Error(), "AccessKey Secret") {
|
||||
t.Fatalf("ossSettingFromRequest() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformProviderSwitchKeepsHistoricalCredentials(t *testing.T) {
|
||||
current := ossSettingValue{Provider: aliyunOSSProvider, AccessKeyID: "aliyun-id", AccessKeySecret: "aliyun-secret"}
|
||||
next := archiveOSSProviderCredentials(ossSettingValue{Provider: tencentCOSProvider, AccessKeyID: "cos-id", AccessKeySecret: "cos-secret"}, current)
|
||||
historical, err := ossSettingForProvider(next, aliyunOSSProvider)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if historical.Provider != aliyunOSSProvider || historical.AccessKeyID != "aliyun-id" || historical.AccessKeySecret != "aliyun-secret" {
|
||||
t.Fatalf("historical setting = %#v", historical)
|
||||
}
|
||||
if _, ok := next.ArchivedCredentials[tencentCOSProvider]; ok {
|
||||
t.Fatalf("active provider credentials were archived: %#v", next.ArchivedCredentials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchivedProviderCredentialsAreEncryptedAtRest(t *testing.T) {
|
||||
svc := &Service{dataDir: t.TempDir()}
|
||||
value := ossSettingValue{
|
||||
Provider: tencentCOSProvider, AccessKeyID: "cos-id", AccessKeySecret: "cos-secret",
|
||||
ArchivedCredentials: map[string]ossProviderCredentials{
|
||||
aliyunOSSProvider: {AccessKeyID: "aliyun-id", AccessKeySecret: "aliyun-secret"},
|
||||
},
|
||||
}
|
||||
stored, err := svc.encryptOSSSettingSecrets(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(stored.AccessKeySecret, encryptedSettingPrefix) || !strings.HasPrefix(stored.ArchivedCredentials[aliyunOSSProvider].AccessKeySecret, encryptedSettingPrefix) {
|
||||
t.Fatalf("stored credentials are not encrypted: %#v", stored)
|
||||
}
|
||||
if _, err := svc.decryptOSSSettingSecrets(&stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored.AccessKeySecret != "cos-secret" || stored.ArchivedCredentials[aliyunOSSProvider].AccessKeySecret != "aliyun-secret" {
|
||||
t.Fatalf("decrypted credentials = %#v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectResourceURLChecksOwnershipAndSignsOSSResource(t *testing.T) {
|
||||
svc := newResourceTestService(t)
|
||||
settingJSON, _ := json.Marshal(ossSettingValue{
|
||||
|
||||
@@ -22,6 +22,11 @@ import (
|
||||
const ossSettingKey = "oss"
|
||||
const encryptedSettingPrefix = "enc:v1:"
|
||||
|
||||
const (
|
||||
aliyunOSSProvider = "aliyun"
|
||||
tencentCOSProvider = "tencent"
|
||||
)
|
||||
|
||||
type OSSSettingRequest struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Provider string `json:"provider"`
|
||||
@@ -59,6 +64,13 @@ type ossSettingValue struct {
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
PublicBaseURL string `json:"publicBaseUrl"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
// 平台切换云厂商后仍需读取历史资源,因此仅归档非当前厂商的访问密钥。
|
||||
ArchivedCredentials map[string]ossProviderCredentials `json:"archivedCredentials,omitempty"`
|
||||
}
|
||||
|
||||
type ossProviderCredentials struct {
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
AccessKeySecret string `json:"accessKeySecret"`
|
||||
}
|
||||
|
||||
func (s *Service) AdminOSSSetting(actor *model.User) (*PublicOSSSetting, error) {
|
||||
@@ -93,8 +105,8 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P
|
||||
return nil, fmt.Errorf("服务器访问地址无效:%w", err)
|
||||
}
|
||||
}
|
||||
stored := next
|
||||
stored.AccessKeySecret, err = s.encryptSettingSecret(next.AccessKeySecret)
|
||||
next = archiveOSSProviderCredentials(next, currentValue)
|
||||
stored, err := s.encryptOSSSettingSecrets(next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -141,8 +153,7 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stored := next
|
||||
stored.AccessKeySecret, err = s.encryptSettingSecret(next.AccessKeySecret)
|
||||
stored, err := s.encryptOSSSettingSecrets(next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -173,14 +184,12 @@ func (s *Service) readOSSSetting() (*model.SystemSetting, ossSettingValue, error
|
||||
return nil, ossSettingValue{}, errors.New("平台 OSS 配置格式无效")
|
||||
}
|
||||
}
|
||||
storedSecret := value.AccessKeySecret
|
||||
value.AccessKeySecret, err = s.decryptSettingSecret(value.AccessKeySecret)
|
||||
needsMigration, err := s.decryptOSSSettingSecrets(&value)
|
||||
if err != nil {
|
||||
return nil, ossSettingValue{}, err
|
||||
}
|
||||
if storedSecret != "" && !strings.HasPrefix(storedSecret, encryptedSettingPrefix) {
|
||||
migrated := value
|
||||
migrated.AccessKeySecret, err = s.encryptSettingSecret(value.AccessKeySecret)
|
||||
if needsMigration {
|
||||
migrated, err := s.encryptOSSSettingSecrets(value)
|
||||
if err != nil {
|
||||
return nil, ossSettingValue{}, err
|
||||
}
|
||||
@@ -224,13 +233,49 @@ func (s *Service) userOSSSettingValue(setting *model.UserOSSSetting) (ossSetting
|
||||
return ossSettingValue{}, errors.New("用户 OSS 配置格式无效")
|
||||
}
|
||||
}
|
||||
secret, err := s.decryptSettingSecret(value.AccessKeySecret)
|
||||
if _, err := s.decryptOSSSettingSecrets(&value); err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
value.Enabled = setting.Enabled
|
||||
return normalizeOSSSetting(value), nil
|
||||
}
|
||||
|
||||
func (s *Service) encryptOSSSettingSecrets(value ossSettingValue) (ossSettingValue, error) {
|
||||
var err error
|
||||
value.AccessKeySecret, err = s.encryptSettingSecret(value.AccessKeySecret)
|
||||
if err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
for provider, credentials := range value.ArchivedCredentials {
|
||||
credentials.AccessKeySecret, err = s.encryptSettingSecret(credentials.AccessKeySecret)
|
||||
if err != nil {
|
||||
return ossSettingValue{}, err
|
||||
}
|
||||
value.ArchivedCredentials[provider] = credentials
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (s *Service) decryptOSSSettingSecrets(value *ossSettingValue) (bool, error) {
|
||||
needsMigration := value.AccessKeySecret != "" && !strings.HasPrefix(value.AccessKeySecret, encryptedSettingPrefix)
|
||||
secret, err := s.decryptSettingSecret(value.AccessKeySecret)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
value.AccessKeySecret = secret
|
||||
value.Enabled = setting.Enabled
|
||||
return normalizeOSSSetting(value), nil
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
for provider, credentials := range value.ArchivedCredentials {
|
||||
if credentials.AccessKeySecret != "" && !strings.HasPrefix(credentials.AccessKeySecret, encryptedSettingPrefix) {
|
||||
needsMigration = true
|
||||
}
|
||||
credentials.AccessKeySecret, err = s.decryptSettingSecret(credentials.AccessKeySecret)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
value.ArchivedCredentials[provider] = credentials
|
||||
}
|
||||
return needsMigration, nil
|
||||
}
|
||||
|
||||
func (s *Service) encryptSettingSecret(value string) (string, error) {
|
||||
@@ -397,7 +442,7 @@ func (s *Service) decryptTaskSecrets(value interface{}) error {
|
||||
}
|
||||
|
||||
func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossSettingValue, error) {
|
||||
next := ossSettingValue{
|
||||
next := normalizeOSSSetting(ossSettingValue{
|
||||
Enabled: req.Enabled,
|
||||
Provider: strings.TrimSpace(req.Provider),
|
||||
Region: strings.TrimSpace(req.Region),
|
||||
@@ -407,22 +452,24 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
|
||||
AccessKeySecret: strings.TrimSpace(req.AccessKeySecret),
|
||||
PublicBaseURL: strings.TrimRight(strings.TrimSpace(req.PublicBaseURL), "/"),
|
||||
PathPrefix: strings.Trim(strings.TrimSpace(req.PathPrefix), "/"),
|
||||
})
|
||||
if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider {
|
||||
return next, BadAuthRequest("仅支持阿里云 OSS 和腾讯云 COS")
|
||||
}
|
||||
if next.Provider == "" {
|
||||
next.Provider = "aliyun"
|
||||
}
|
||||
if next.Provider != "aliyun" {
|
||||
return next, BadAuthRequest("暂时只支持阿里云 OSS")
|
||||
}
|
||||
if next.AccessKeySecret == "" {
|
||||
current = normalizeOSSSetting(current)
|
||||
// 不同云厂商的密钥不能复用;只有继续使用同一厂商时,留空才表示保留原密钥。
|
||||
if next.AccessKeySecret == "" && next.Provider == current.Provider {
|
||||
next.AccessKeySecret = current.AccessKeySecret
|
||||
}
|
||||
if next.Enabled {
|
||||
if next.Bucket == "" {
|
||||
return next, BadAuthRequest("请填写 OSS Bucket")
|
||||
return next, BadAuthRequest("请填写对象存储 Bucket")
|
||||
}
|
||||
if next.Endpoint == "" {
|
||||
return next, BadAuthRequest("请填写 OSS Endpoint")
|
||||
if next.Provider == tencentCOSProvider {
|
||||
return next, BadAuthRequest("请填写腾讯云 COS Region 或 Endpoint")
|
||||
}
|
||||
return next, BadAuthRequest("请填写阿里云 OSS Endpoint")
|
||||
}
|
||||
if _, err := ValidateOutboundURL(next.Endpoint); err != nil {
|
||||
return next, err
|
||||
@@ -437,23 +484,53 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func archiveOSSProviderCredentials(next ossSettingValue, current ossSettingValue) ossSettingValue {
|
||||
next.ArchivedCredentials = cloneOSSProviderCredentials(current.ArchivedCredentials)
|
||||
if current.Provider != next.Provider && (current.AccessKeyID != "" || current.AccessKeySecret != "") {
|
||||
if next.ArchivedCredentials == nil {
|
||||
next.ArchivedCredentials = make(map[string]ossProviderCredentials)
|
||||
}
|
||||
next.ArchivedCredentials[current.Provider] = ossProviderCredentials{AccessKeyID: current.AccessKeyID, AccessKeySecret: current.AccessKeySecret}
|
||||
}
|
||||
delete(next.ArchivedCredentials, next.Provider)
|
||||
return next
|
||||
}
|
||||
|
||||
func cloneOSSProviderCredentials(source map[string]ossProviderCredentials) map[string]ossProviderCredentials {
|
||||
if len(source) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make(map[string]ossProviderCredentials, len(source))
|
||||
for provider, credentials := range source {
|
||||
cloned[strings.ToLower(strings.TrimSpace(provider))] = ossProviderCredentials{
|
||||
AccessKeyID: strings.TrimSpace(credentials.AccessKeyID),
|
||||
AccessKeySecret: strings.TrimSpace(credentials.AccessKeySecret),
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func normalizeOSSSetting(value ossSettingValue) ossSettingValue {
|
||||
value.Provider = strings.TrimSpace(value.Provider)
|
||||
value.Provider = strings.ToLower(strings.TrimSpace(value.Provider))
|
||||
if value.Provider == "" {
|
||||
value.Provider = "aliyun"
|
||||
value.Provider = aliyunOSSProvider
|
||||
}
|
||||
value.Region = strings.TrimSpace(value.Region)
|
||||
value.Endpoint = strings.TrimRight(strings.TrimSpace(value.Endpoint), "/")
|
||||
if value.Provider == tencentCOSProvider && value.Endpoint == "" && value.Region != "" {
|
||||
value.Endpoint = "https://cos." + value.Region + ".myqcloud.com"
|
||||
}
|
||||
value.Bucket = strings.TrimSpace(value.Bucket)
|
||||
value.AccessKeyID = strings.TrimSpace(value.AccessKeyID)
|
||||
value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret)
|
||||
value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/")
|
||||
value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/")
|
||||
value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials)
|
||||
return value
|
||||
}
|
||||
|
||||
func defaultOSSSetting() ossSettingValue {
|
||||
return ossSettingValue{Provider: "aliyun"}
|
||||
return ossSettingValue{Provider: aliyunOSSProvider}
|
||||
}
|
||||
|
||||
func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) PublicOSSSetting {
|
||||
|
||||
Reference in new issue
Block a user