From 21b69d8bc7057d52db19395078caafcd667bb78c Mon Sep 17 00:00:00 2001 From: Rou <76741805+Rou325@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:16:08 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=20S3=20=E5=85=BC=E5=AE=B9?= =?UTF-8?q?=E5=AF=B9=E8=B1=A1=E5=AD=98=E5=82=A8=20(#336)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(storage): 支持 S3 对象存储 * fix(storage): 补齐 S3 对象文件后缀 --- CHANGELOG.md | 1 + backend/cmd/migrate-sqlite-postgres/main.go | 14 ++ .../cmd/migrate-sqlite-postgres/main_test.go | 36 +++ backend/internal/database/schema.go | 1 + backend/internal/handler/auth.go | 23 ++ .../internal/handler/storage_routes_test.go | 32 +++ backend/internal/handler/user_data.go | 22 ++ backend/internal/model/models_platform.go | 16 ++ backend/internal/repository/repository.go | 55 ++++ backend/internal/service/resource.go | 89 +++++-- backend/internal/service/resource_delete.go | 6 + backend/internal/service/resource_test.go | 2 +- backend/internal/service/service.go | 4 +- backend/internal/service/settings.go | 234 +++++++++++++++--- backend/internal/service/storage_location.go | 201 +++++++++++++++ backend/internal/service/storage_s3.go | 165 ++++++++++++ backend/internal/service/storage_s3_test.go | 178 +++++++++++++ .../content/docs/backend/backend-database.mdx | 10 + docs/content/docs/progress/pending-test.mdx | 13 +- .../layout/user-oss-settings-form.tsx | 102 ++++++-- web/src/lib/oss-settings.ts | 84 +++++++ .../admin/settings/storage-settings-page.tsx | 119 +++++++-- web/src/services/api/auth.ts | 16 +- web/src/services/api/resources.ts | 19 +- web/test/oss-settings.test.ts | 20 ++ 25 files changed, 1374 insertions(+), 88 deletions(-) create mode 100644 backend/cmd/migrate-sqlite-postgres/main_test.go create mode 100644 backend/internal/handler/storage_routes_test.go create mode 100644 backend/internal/service/storage_location.go create mode 100644 backend/internal/service/storage_s3.go create mode 100644 backend/internal/service/storage_s3_test.go create mode 100644 web/src/lib/oss-settings.ts create mode 100644 web/test/oss-settings.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b311965c1..5f4c23fe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- 新增平台与个人 S3 兼容存储，支持 AWS S3、Cloudflare R2、Backblaze B2、RustFS 和自定义 Endpoint，提供 Path Style、临时凭据、连接测试、历史位置绑定及私网资源 HTTPS 代理；个人配置启用时优先于平台存储。 - 新增用户诊断包导出：设置页和失败任务详情支持导出最近 15 分钟、30 分钟、1 小时或 24 小时内的脱敏 ZIP，并通过 `traceId`、`requestId`、`taskId` 和 `providerRequestId` 关联前端、任务日志与模型上游调用。 - 建立诊断数据安全边界：前端内存事件缓冲区最多保留 500 条，后端按当前用户校验任务和项目归属，导出时排除提示词、任务输入、请求/响应体、密钥、Cookie 和 URL 查询参数，并补充脱敏与越权测试。 - 修复 Eagle 素材库在素材缺少文件夹或标签时的空集合兼容问题，避免前端读取 `folderIds` 或 `tags` 时出现异常。 diff --git a/backend/cmd/migrate-sqlite-postgres/main.go b/backend/cmd/migrate-sqlite-postgres/main.go index c6eabb442..2750be7b7 100644 --- a/backend/cmd/migrate-sqlite-postgres/main.go +++ b/backend/cmd/migrate-sqlite-postgres/main.go @@ -221,6 +221,12 @@ func migrations() []tableMigration { migrateTable[model.EmailVerificationCode]("email_verification_codes"), migrateTable[model.ModelChannel]("model_channels"), migrateTable[model.ChannelModel]("channel_models"), + migrateTable[model.ChannelModelPriceTier]("channel_model_price_tiers"), + migrateTable[model.IDSequence]("id_sequences"), + migrateTable[model.LogicalModel]("logical_models"), + migrateTable[model.LogicalModelRevision]("logical_model_revisions"), + migrateTable[model.LogicalModelRoute]("logical_model_routes"), + migrateTable[model.RouteAttempt]("route_attempts"), migrateTable[model.ApiCallLog]("api_call_logs"), migrateTable[model.ModelPricing]("model_pricings"), migrateTable[model.CreditAccount]("credit_accounts"), @@ -231,13 +237,19 @@ func migrations() []tableMigration { migrateTable[model.AdminAuditEvent]("admin_audit_events"), migrateTable[model.UserDailyActivity]("user_daily_activities"), migrateTable[model.SystemSetting]("system_settings"), + migrateTable[model.PluginPlatformState]("plugin_platform_states"), + migrateTable[model.UserPluginState]("user_plugin_states"), + migrateTable[model.ArkPrivateAssetBinding]("ark_private_asset_bindings"), migrateTable[model.UserOSSSetting]("user_oss_settings"), + migrateTable[model.StorageLocation]("storage_locations"), migrateTable[model.UserDailyUploadUsage]("user_daily_upload_usages"), migrateTable[model.Skill]("skills"), migrateTable[model.UserSkillState]("user_skill_states"), migrateTable[model.Resource]("resources"), + migrateTable[model.ResourceDeletionJob]("resource_deletion_jobs"), migrateTable[model.Asset]("assets"), migrateTable[model.ProjectAssetLink]("project_asset_links"), + migrateTable[model.ProjectAssetFolder]("project_asset_folders"), migrateTable[model.ProjectAssetCandidate]("project_asset_candidates"), migrateTable[model.AssetVersion]("asset_versions"), migrateTable[model.AssetRepresentation]("asset_representations"), @@ -266,5 +278,7 @@ func migrations() []tableMigration { migrateTable[model.TaskLog]("task_logs"), migrateTable[model.SessionFile]("session_files"), migrateTable[model.Result]("results"), + migrateTable[model.ComfyBridge]("comfy_bridges"), + migrateTable[model.ComfyBridgeRequest]("comfy_bridge_requests"), } } diff --git a/backend/cmd/migrate-sqlite-postgres/main_test.go b/backend/cmd/migrate-sqlite-postgres/main_test.go new file mode 100644 index 000000000..9399c4b21 --- /dev/null +++ b/backend/cmd/migrate-sqlite-postgres/main_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "sync" + "testing" + + "infinite-canvas/backend/internal/database" + + "gorm.io/gorm/schema" +) + +func TestMigrationListCoversSchemaModels(t *testing.T) { + want := make(map[string]bool, len(database.Models())) + cache := &sync.Map{} + for _, value := range database.Models() { + parsed, err := schema.Parse(value, cache, schema.NamingStrategy{}) + if err != nil { + t.Fatal(err) + } + want[parsed.Table] = false + } + for _, migration := range migrations() { + if _, exists := want[migration.name]; !exists { + t.Fatalf("migration list contains unknown table %q", migration.name) + } + if want[migration.name] { + t.Fatalf("migration list contains duplicate table %q", migration.name) + } + want[migration.name] = true + } + for table, covered := range want { + if !covered { + t.Errorf("migration list is missing table %q", table) + } + } +} diff --git a/backend/internal/database/schema.go b/backend/internal/database/schema.go index fea751ef3..42b104e9b 100644 --- a/backend/internal/database/schema.go +++ b/backend/internal/database/schema.go @@ -43,6 +43,7 @@ func Models() []any { &model.UserPluginState{}, &model.ArkPrivateAssetBinding{}, &model.UserOSSSetting{}, + &model.StorageLocation{}, &model.UserDailyUploadUsage{}, &model.Skill{}, &model.UserSkillState{}, diff --git a/backend/internal/handler/auth.go b/backend/internal/handler/auth.go index 333a97d09..876547f37 100644 --- a/backend/internal/handler/auth.go +++ b/backend/internal/handler/auth.go @@ -477,6 +477,7 @@ func RegisterAdminRoutes(r *gin.RouterGroup, svc *service.Service) { failService(c, err) return } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10) var req service.OSSSettingRequest if err := c.ShouldBindJSON(&req); err != nil { fail(c, http.StatusBadRequest, err) @@ -489,6 +490,28 @@ func RegisterAdminRoutes(r *gin.RouterGroup, svc *service.Service) { } ok(c, gin.H{"setting": setting}) }) + r.POST("/admin/settings/oss/test", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + if !enforceRateLimit(c, "admin-storage-test:"+user.ID, 6, time.Minute) { + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10) + var req service.OSSSettingRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + result, err := svc.TestAdminOSSSetting(user, req) + if err != nil { + failService(c, err) + return + } + ok(c, result) + }) r.GET("/admin/settings/ark-private-assets", func(c *gin.Context) { user, err := currentUser(c, svc) if err != nil { diff --git a/backend/internal/handler/storage_routes_test.go b/backend/internal/handler/storage_routes_test.go new file mode 100644 index 000000000..8e10cbc6c --- /dev/null +++ b/backend/internal/handler/storage_routes_test.go @@ -0,0 +1,32 @@ +package handler + +import ( + "testing" + + "infinite-canvas/backend/internal/service" + + "github.com/gin-gonic/gin" +) + +func TestStorageConnectionTestRoutesAreRegistered(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + group := router.Group("/api") + RegisterAdminRoutes(group, &service.Service{}) + RegisterUserDataRoutes(group, &service.Service{}) + wanted := map[string]bool{ + "POST /api/admin/settings/oss/test": false, + "POST /api/settings/oss/test": false, + } + for _, route := range router.Routes() { + key := route.Method + " " + route.Path + if _, exists := wanted[key]; exists { + wanted[key] = true + } + } + for route, found := range wanted { + if !found { + t.Errorf("route %s is not registered", route) + } + } +} diff --git a/backend/internal/handler/user_data.go b/backend/internal/handler/user_data.go index 9408e6682..37678ee9c 100644 --- a/backend/internal/handler/user_data.go +++ b/backend/internal/handler/user_data.go @@ -92,6 +92,28 @@ func RegisterUserDataRoutes(r *gin.RouterGroup, svc *service.Service) { } ok(c, gin.H{"setting": setting}) }) + r.POST("/settings/oss/test", func(c *gin.Context) { + user, err := currentUser(c, svc) + if err != nil { + failService(c, err) + return + } + if !enforceRateLimit(c, "user-storage-test:"+user.ID, 6, time.Minute) { + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10) + var req service.OSSSettingRequest + if err := c.ShouldBindJSON(&req); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + result, err := svc.TestUserOSSSetting(user, req) + if err != nil { + failService(c, err) + return + } + ok(c, result) + }) r.GET("/resources", func(c *gin.Context) { user, err := currentUser(c, svc) if err != nil { diff --git a/backend/internal/model/models_platform.go b/backend/internal/model/models_platform.go index c03dccfd3..35c8bafb4 100644 --- a/backend/internal/model/models_platform.go +++ b/backend/internal/model/models_platform.go @@ -50,6 +50,22 @@ type UserOSSSetting struct { UpdatedAt time.Time `json:"updatedAt"` } +// StorageLocation is an immutable object-storage address. Credentials may be +// rotated in place, while resources keep referring to the same location ID. +type StorageLocation struct { + ID string `json:"id" gorm:"primaryKey;size:36"` + Scope string `json:"scope" gorm:"size:16;index:idx_storage_locations_scope_owner_active,priority:1;uniqueIndex:idx_storage_locations_identity,priority:1"` + OwnerID string `json:"ownerId" gorm:"size:36;index:idx_storage_locations_scope_owner_active,priority:2;uniqueIndex:idx_storage_locations_identity,priority:2"` + Provider string `json:"provider" gorm:"size:24;index;uniqueIndex:idx_storage_locations_identity,priority:3"` + LocationDigest string `json:"-" gorm:"size:64;uniqueIndex:idx_storage_locations_identity,priority:4"` + ValueJSON string `json:"-" gorm:"type:text"` + TestedDigest string `json:"-" gorm:"size:64"` + TestedAt *time.Time `json:"testedAt,omitempty"` + Active bool `json:"active" gorm:"index:idx_storage_locations_scope_owner_active,priority:3"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + type UserDailyUploadUsage struct { ID string `json:"id" gorm:"primaryKey;size:64"` UserID string `json:"userId" gorm:"size:36;index;uniqueIndex:idx_user_daily_upload_day,priority:1"` diff --git a/backend/internal/repository/repository.go b/backend/internal/repository/repository.go index 18feefc6b..dffa6eddd 100644 --- a/backend/internal/repository/repository.go +++ b/backend/internal/repository/repository.go @@ -840,6 +840,61 @@ func (r *Repository) CreateUserOSSSetting(setting *model.UserOSSSetting) error { return r.db.Create(setting).Error } +func (r *Repository) StorageLocation(id string) (*model.StorageLocation, error) { + var location model.StorageLocation + if err := r.db.First(&location, "id = ?", id).Error; err != nil { + return nil, err + } + return &location, nil +} + +func (r *Repository) StorageLocationByDigest(scope string, ownerID string, provider string, digest string) (*model.StorageLocation, error) { + var location model.StorageLocation + if err := r.db.First(&location, "scope = ? AND owner_id = ? AND provider = ? AND location_digest = ?", scope, ownerID, provider, digest).Error; err != nil { + return nil, err + } + return &location, nil +} + +func (r *Repository) StorageLocationHistoryCount(scope string, ownerID string) (int64, error) { + var count int64 + err := r.db.Model(&model.StorageLocation{}).Where("scope = ? AND owner_id = ?", scope, ownerID).Count(&count).Error + return count, err +} + +func (r *Repository) StorageLocationResourceCount(id string) (int64, error) { + var count int64 + err := r.db.Model(&model.Resource{}).Where("storage_setting_id = ?", id).Count(&count).Error + return count, err +} + +func (r *Repository) CreateStorageLocation(location *model.StorageLocation) error { + return r.db.Create(location).Error +} + +func (r *Repository) SaveStorageLocation(location *model.StorageLocation) error { + return r.db.Save(location).Error +} + +func (r *Repository) ActivateStorageLocation(scope string, ownerID string, id string, active bool) error { + return r.db.Transaction(func(tx *gorm.DB) error { + if err := tx.Model(&model.StorageLocation{}).Where("scope = ? AND owner_id = ? AND active = ?", scope, ownerID, true).Update("active", false).Error; err != nil { + return err + } + if !active { + return nil + } + result := tx.Model(&model.StorageLocation{}).Where("id = ? AND scope = ? AND owner_id = ?", id, scope, ownerID).Update("active", true) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return gorm.ErrRecordNotFound + } + return nil + }) +} + func (r *Repository) ReserveDailyUpload(userID string, day string, size int64, limit int64) error { usage := model.UserDailyUploadUsage{ID: userID + ":" + day, UserID: userID, Day: day} return r.db.Transaction(func(tx *gorm.DB) error { diff --git a/backend/internal/service/resource.go b/backend/internal/service/resource.go index b3a41a8e0..e58a0b509 100644 --- a/backend/internal/service/resource.go +++ b/backend/internal/service/resource.go @@ -103,6 +103,9 @@ func (s *Service) directResourceURL(resource *model.Resource, expiresAt time.Tim setting.Provider = firstNonEmpty(resource.Provider, setting.Provider) setting.Endpoint = firstNonEmpty(resource.Endpoint, setting.Endpoint) setting.Bucket = firstNonEmpty(resource.Bucket, setting.Bucket) + if setting.Provider == s3Provider && !publicHTTPSStorageEndpoint(setting.Endpoint) { + return s.signedHTTPSPublicResourceURL(resource.ID, expiresAt) + } return signedOSSObjectURL(setting, resource.ObjectKey, expiresAt) } @@ -130,6 +133,11 @@ func (s *Service) prepareResourceDelivery(userID string, resource *model.Resourc if err != nil { return nil, err } + // S3 兼容 Endpoint 可能是私网服务；浏览器默认始终使用同源代理。 + // 只有明确的服务端上游需求才签发可公开访问的短时地址。 + if setting.Provider == s3Provider && !options.ForceDirect { + return &ResourceDelivery{Resource: resource}, nil + } if setting.Provider == qiniuKodoProvider && setting.CDNBaseURL != "" { // 七牛私有空间即使配置了绑定域名，也不能匿名访问；必须使用 // Kodo 私有下载签名，否则浏览器会收到 NotSupportAnonymous。 @@ -147,6 +155,13 @@ func (s *Service) prepareResourceDelivery(userID string, resource *model.Resourc return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil } if options.ForceDirect { + if setting.Provider == s3Provider && !publicHTTPSStorageEndpoint(setting.Endpoint) { + redirectURL, err := s.signedHTTPSPublicResourceURL(resource.ID, time.Now().Add(directResourceURLTTL)) + if err != nil { + return nil, err + } + return &ResourceDelivery{Resource: resource, RedirectURL: redirectURL}, nil + } redirectURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(directResourceURLTTL)) if err != nil { return nil, err @@ -175,6 +190,17 @@ func (s *Service) signedPublicResourceURL(resourceID string, expiresAt time.Time return baseURL.String(), nil } +func (s *Service) signedHTTPSPublicResourceURL(resourceID string, expiresAt time.Time) (string, error) { + baseURL, err := s.publicResourceBaseURL() + if err != nil { + return "", err + } + if baseURL.Scheme != "https" { + return "", BadAuthRequest("私网或 HTTP S3 用于上游资源时，服务器公开访问地址必须使用 HTTPS") + } + return s.signedPublicResourceURL(resourceID, expiresAt) +} + func (s *Service) verifyPublicResourceSignature(resourceID string, expires string, signature string) error { if strings.TrimSpace(signature) == "" || !decimalDigits(expires) { return Forbidden("匿名下载链接无效") @@ -323,7 +349,7 @@ func (s *Service) OpenPublicResourceRange(id string, expires string, signature s if err != nil { return nil, Forbidden("匿名下载链接无效") } - if resource.Provider != "local" { + if resource.Provider != "local" && resource.Provider != s3Provider { return nil, Forbidden("匿名下载链接无效") } if err := s.verifyPublicResourceSignature(resource.ID, expires, signature); err != nil { @@ -368,11 +394,11 @@ func (s *Service) storeResource(userID string, kind string, fileName string, mim return nil, err } provider := "local" - objectKey := localObjectKey(userID, kind, fileName, now) + objectKey := localObjectKey(userID, kind, fileName, mimeType, now) resource := model.Resource{ID: newID(), UserID: userID, Kind: kind, Status: model.ResourceStatusPending, Provider: provider, ObjectKey: objectKey, MimeType: mimeType, Size: size, Width: width, Height: height, DurationMs: durationMs, CreatedAt: now, UpdatedAt: now} if useOSS { provider = setting.Provider - objectKey = ossObjectKey(setting, userID, kind, fileName, now) + objectKey = ossObjectKey(setting, userID, kind, fileName, mimeType, now) resource.Provider = provider resource.Endpoint = setting.Endpoint resource.Bucket = setting.Bucket @@ -431,8 +457,8 @@ func (s *Service) storeResource(userID string, kind string, fileName string, mim return &resource, nil } -func localObjectKey(userID string, kind string, fileName string, now time.Time) string { - ext := strings.ToLower(filepath.Ext(fileName)) +func localObjectKey(userID string, kind string, fileName string, mimeType string, now time.Time) string { + ext := resourceFileExtension(fileName, mimeType, kind) return path.Join("users", safeObjectSegment(userID), kind, now.Format("2006/01/02"), newID()+ext) } @@ -676,6 +702,26 @@ func extensionFromMimeType(mimeType string) string { return "bin" } +func resourceFileExtension(fileName string, mimeType string, kind string) string { + if ext := strings.ToLower(filepath.Ext(strings.TrimSpace(fileName))); ext != "" && ext != "." { + return ext + } + cleanMimeType := strings.TrimSpace(strings.Split(mimeType, ";")[0]) + if extensions, err := mime.ExtensionsByType(cleanMimeType); err == nil && len(extensions) > 0 { + return strings.ToLower(extensions[0]) + } + switch kind { + case "image": + return ".png" + case "video": + return ".mp4" + case "audio": + return ".mp3" + default: + return ".bin" + } +} + func imageDimensions(data []byte) (int, int) { config, _, err := image.DecodeConfig(bytes.NewReader(data)) if err != nil { @@ -697,26 +743,30 @@ func (s *Service) activeResourceOSSSetting(userID string) (ossSettingValue, stri if err != nil { return ossSettingValue{}, "", false, err } - if userSetting != nil && value.Enabled { - value, err = validateActiveOSSSetting(value, "用户 OSS 尚未启用", "你的 OSS 配置不完整") - return value, userSetting.ID, true, err - } _, systemValue, err := s.readOSSSetting() if err != nil { return ossSettingValue{}, "", false, err } + userAllowed := value.Provider != s3Provider || systemValue.AllowUserS3 + if userSetting != nil && value.Enabled && userAllowed { + value, err = validateActiveOSSSetting(value, "用户 OSS 尚未启用", "你的 OSS 配置不完整") + return value, firstNonEmpty(value.StorageLocationID, userSetting.ID), true, err + } if !systemValue.Enabled { return ossSettingValue{}, "", false, nil } systemValue, err = validateActiveOSSSetting(systemValue, "管理员尚未启用 OSS", "平台 OSS 配置不完整，请联系管理员") - return systemValue, "", true, err + return systemValue, systemValue.StorageLocationID, true, err } func (s *Service) ossSettingForResource(userID string, resource *model.Resource) (ossSettingValue, error) { var setting ossSettingValue var err error if resource.StorageSettingID != "" { - _, setting, err = s.readUserOSSSettingByID(userID, resource.StorageSettingID) + _, setting, err = s.storageLocationValue(resource.StorageSettingID) + if errors.Is(err, gorm.ErrRecordNotFound) { + _, setting, err = s.readUserOSSSettingByID(userID, resource.StorageSettingID) + } if err == nil { _, current, currentErr := s.readUserOSSSetting(userID) if currentErr != nil { @@ -805,8 +855,8 @@ func validateActiveOSSSetting(setting ossSettingValue, disabledMessage string, i if !setting.Enabled { return ossSettingValue{}, BadAuthRequest(disabledMessage) } - if setting.Provider != aliyunOSSProvider && setting.Provider != tencentCOSProvider && setting.Provider != qiniuKodoProvider { - return ossSettingValue{}, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS 和七牛云 Kodo") + if setting.Provider != aliyunOSSProvider && setting.Provider != tencentCOSProvider && setting.Provider != qiniuKodoProvider && setting.Provider != s3Provider { + return ossSettingValue{}, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS、七牛云 Kodo 和通用 S3") } if setting.Bucket == "" || setting.Endpoint == "" || setting.AccessKeyID == "" || setting.AccessKeySecret == "" { return ossSettingValue{}, BadAuthRequest(incompleteMessage) @@ -832,8 +882,8 @@ func normalizeResourceKind(kind string, mimeType string) string { return "file" } -func ossObjectKey(setting ossSettingValue, userID string, kind string, fileName string, now time.Time) string { - ext := strings.ToLower(filepath.Ext(fileName)) +func ossObjectKey(setting ossSettingValue, userID string, kind string, fileName string, mimeType string, now time.Time) string { + ext := resourceFileExtension(fileName, mimeType, kind) name := newID() parts := []string{setting.PathPrefix, "users", safeObjectSegment(userID), kind, now.Format("2006/01/02"), name + ext} return strings.Trim(strings.Join(nonEmptySegments(parts), "/"), "/") @@ -847,6 +897,9 @@ func putOSSObject(setting ossSettingValue, objectKey string, mimeType string, si if setting.Provider == qiniuKodoProvider { return putQiniuObject(setting, objectKey, mimeType, size, body) } + if setting.Provider == s3Provider { + return putS3Object(setting, objectKey, mimeType, size, body) + } return putAliyunOSSObject(setting, objectKey, mimeType, size, body) } @@ -884,6 +937,9 @@ type ossObjectStream struct { func getOSSObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { setting = normalizeOSSSetting(setting) + if setting.Provider == s3Provider { + return getS3ObjectRange(setting, objectKey, rangeHeader) + } if setting.CDNBaseURL != "" { return getOSSObjectRangeViaCDN(setting, objectKey, rangeHeader) } @@ -939,6 +995,9 @@ func decimalDigits(value string) bool { func signedOSSObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { setting = normalizeOSSSetting(setting) + if setting.Provider == s3Provider { + return signedS3ObjectURL(setting, objectKey, expiresAt) + } if setting.Provider == qiniuKodoProvider { return signedQiniuObjectURL(setting, objectKey, expiresAt) } diff --git a/backend/internal/service/resource_delete.go b/backend/internal/service/resource_delete.go index 03b24eea7..89839da17 100644 --- a/backend/internal/service/resource_delete.go +++ b/backend/internal/service/resource_delete.go @@ -310,6 +310,12 @@ func (s *Service) deleteStoredResourceObject(userID string, resource *model.Reso return fmt.Errorf("无法读取七牛云 Kodo 配置：%w", err) } return deleteQiniuObject(setting, resource.ObjectKey) + case s3Provider: + setting, err := s.ossSettingForResource(userID, resource) + if err != nil { + return fmt.Errorf("无法读取 S3 配置：%w", err) + } + return deleteS3Object(setting, resource.ObjectKey) default: return fmt.Errorf("资源 %s 使用了不支持的存储类型 %q", resource.ID, resource.Provider) } diff --git a/backend/internal/service/resource_test.go b/backend/internal/service/resource_test.go index 51a9c7bdb..8eedc1afa 100644 --- a/backend/internal/service/resource_test.go +++ b/backend/internal/service/resource_test.go @@ -825,7 +825,7 @@ func newResourceTestService(t *testing.T) *Service { if err != nil { t.Fatal(err) } - if err := db.AutoMigrate(&model.SystemSetting{}, &model.UserOSSSetting{}, &model.UserDailyUploadUsage{}, &model.Resource{}, &model.SessionFile{}); err != nil { + if err := db.AutoMigrate(&model.SystemSetting{}, &model.UserOSSSetting{}, &model.StorageLocation{}, &model.UserDailyUploadUsage{}, &model.Resource{}, &model.SessionFile{}); err != nil { t.Fatal(err) } return &Service{repo: repository.New(db), dataDir: t.TempDir()} diff --git a/backend/internal/service/service.go b/backend/internal/service/service.go index 7687f1ea2..eab4457d7 100644 --- a/backend/internal/service/service.go +++ b/backend/internal/service/service.go @@ -25,6 +25,8 @@ type Service struct { emailCodeMu sync.Mutex redeemBatchMu sync.Mutex storageMu sync.Mutex + storageTestMu sync.Mutex + activeStorageTests map[string]bool characterTaskMu sync.Mutex activeCancels map[string]context.CancelFunc pendingStorage map[string]int64 @@ -102,7 +104,7 @@ func New(repo *repository.Repository, dataDir string) *Service { func NewWithRuntimeCapabilities(repo *repository.Repository, dataDir string, capabilities RuntimeCapabilities) *Service { coordinator, err := newRuntimeCoordinator(repo.Dialect()) pluginRuntime, pluginRuntimeErr := newPluginRuntime(dataDir) - service := &Service{repo: repo, dataDir: dataDir, runtimeCapabilities: capabilities, activeCancels: make(map[string]context.CancelFunc), coordinator: coordinator, runtimeErr: err, pluginRuntime: pluginRuntime, pluginRuntimeErr: pluginRuntimeErr, workerID: newID(), routeCatalogTTL: 30 * time.Second, routeCatalogMaxStale: 5 * time.Minute, routeHealthBlocked: make(map[string]time.Time)} + service := &Service{repo: repo, dataDir: dataDir, runtimeCapabilities: capabilities, activeStorageTests: make(map[string]bool), activeCancels: make(map[string]context.CancelFunc), coordinator: coordinator, runtimeErr: err, pluginRuntime: pluginRuntime, pluginRuntimeErr: pluginRuntimeErr, workerID: newID(), routeCatalogTTL: 30 * time.Second, routeCatalogMaxStale: 5 * time.Minute, routeHealthBlocked: make(map[string]time.Time)} service.taskBillingCoordinator = newTaskBillingCoordinator(service.repo) service.taskTerminalCoordinator = newTaskTerminalCoordinator(service) service.taskRouteExecutor = newTaskRouteExecutor(service) diff --git a/backend/internal/service/settings.go b/backend/internal/service/settings.go index d8b5175e4..7100d4b7d 100644 --- a/backend/internal/service/settings.go +++ b/backend/internal/service/settings.go @@ -4,13 +4,16 @@ import ( "crypto/aes" "crypto/cipher" "crypto/rand" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" + "strconv" "strings" "time" @@ -21,11 +24,13 @@ import ( const ossSettingKey = "oss" const encryptedSettingPrefix = "enc:v1:" +const defaultOSSPathPrefix = "open-ai-canvas" const ( aliyunOSSProvider = "aliyun" tencentCOSProvider = "tencent" qiniuKodoProvider = "qiniu" + s3Provider = "s3" ) type OSSSettingRequest struct { @@ -39,35 +44,53 @@ type OSSSettingRequest struct { AccessKeySecret string `json:"accessKeySecret"` PublicBaseURL string `json:"publicBaseUrl"` PathPrefix string `json:"pathPrefix"` + S3Preset string `json:"s3Preset"` + PathStyle bool `json:"pathStyle"` + SessionToken string `json:"sessionToken"` + AllowUserS3 bool `json:"allowUserS3"` } type PublicOSSSetting struct { - Enabled bool `json:"enabled"` - Provider string `json:"provider"` - Region string `json:"region"` - Endpoint string `json:"endpoint"` - CDNBaseURL string `json:"cdnBaseUrl"` - Bucket string `json:"bucket"` - AccessKeyID string `json:"accessKeyId"` - HasAccessKeySecret bool `json:"hasAccessKeySecret"` - PublicBaseURL string `json:"publicBaseUrl"` - PathPrefix string `json:"pathPrefix"` - UpdatedBy string `json:"updatedBy"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + Enabled bool `json:"enabled"` + Provider string `json:"provider"` + Region string `json:"region"` + Endpoint string `json:"endpoint"` + CDNBaseURL string `json:"cdnBaseUrl"` + Bucket string `json:"bucket"` + AccessKeyID string `json:"accessKeyId"` + HasAccessKeySecret bool `json:"hasAccessKeySecret"` + PublicBaseURL string `json:"publicBaseUrl"` + PathPrefix string `json:"pathPrefix"` + S3Preset string `json:"s3Preset"` + PathStyle bool `json:"pathStyle"` + HasSessionToken bool `json:"hasSessionToken"` + StorageLocationID string `json:"storageLocationId"` + TestedAt *time.Time `json:"testedAt,omitempty"` + TestedDigest string `json:"testedDigest,omitempty"` + HistoryCount int64 `json:"historyCount"` + ReferencedResourceCount int64 `json:"referencedResourceCount"` + AllowUserS3 bool `json:"allowUserS3"` + UpdatedBy string `json:"updatedBy"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } type ossSettingValue struct { - Enabled bool `json:"enabled"` - Provider string `json:"provider"` - Region string `json:"region"` - Endpoint string `json:"endpoint"` - CDNBaseURL string `json:"cdnBaseUrl"` - Bucket string `json:"bucket"` - AccessKeyID string `json:"accessKeyId"` - AccessKeySecret string `json:"accessKeySecret"` - PublicBaseURL string `json:"publicBaseUrl"` - PathPrefix string `json:"pathPrefix"` + Enabled bool `json:"enabled"` + Provider string `json:"provider"` + Region string `json:"region"` + Endpoint string `json:"endpoint"` + CDNBaseURL string `json:"cdnBaseUrl"` + Bucket string `json:"bucket"` + AccessKeyID string `json:"accessKeyId"` + AccessKeySecret string `json:"accessKeySecret"` + PublicBaseURL string `json:"publicBaseUrl"` + PathPrefix string `json:"pathPrefix"` + S3Preset string `json:"s3Preset"` + PathStyle bool `json:"pathStyle"` + SessionToken string `json:"sessionToken"` + StorageLocationID string `json:"storageLocationId"` + AllowUserS3 bool `json:"allowUserS3"` // 平台切换云厂商后仍需读取历史资源，因此仅归档非当前厂商的访问密钥。 ArchivedCredentials map[string]ossProviderCredentials `json:"archivedCredentials,omitempty"` } @@ -85,7 +108,10 @@ func (s *Service) AdminOSSSetting(actor *model.User) (*PublicOSSSetting, error) if err != nil { return nil, err } - public := publicOSSSetting(setting, value) + public, err := s.publicOSSSetting(setting, value, "platform", "") + if err != nil { + return nil, err + } return &public, nil } @@ -110,6 +136,15 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P } } next = archiveOSSProviderCredentials(next, currentValue) + if next.Provider == s3Provider && next.Enabled { + location, err := s.requireTestedS3Location("platform", "", next) + if err != nil { + return nil, err + } + next.StorageLocationID = location.ID + } else if next.Provider == s3Provider && storageLocationDigest(next) == storageLocationDigest(currentValue) { + next.StorageLocationID = currentValue.StorageLocationID + } stored, err := s.encryptOSSSettingSecrets(next) if err != nil { return nil, err @@ -129,7 +164,19 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P if err := s.repo.SaveSystemSetting(&setting); err != nil { return nil, err } - public := publicOSSSetting(&setting, next) + if next.Provider == s3Provider && next.StorageLocationID != "" { + if err := s.repo.ActivateStorageLocation("platform", "", next.StorageLocationID, next.Enabled); err != nil { + return nil, err + } + } else if currentValue.Provider == s3Provider && currentValue.StorageLocationID != "" { + if err := s.repo.ActivateStorageLocation("platform", "", "", false); err != nil { + return nil, err + } + } + public, err := s.publicOSSSetting(&setting, next, "platform", "") + if err != nil { + return nil, err + } return &public, nil } @@ -141,7 +188,14 @@ func (s *Service) UserOSSSetting(actor *model.User) (*PublicOSSSetting, error) { if err != nil { return nil, err } - public := publicUserOSSSetting(setting, value) + _, platform, err := s.readOSSSetting() + if err != nil { + return nil, err + } + public, err := s.publicUserOSSSetting(setting, value, actor.ID, platform.AllowUserS3) + if err != nil { + return nil, err + } return &public, nil } @@ -157,6 +211,22 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest) if err != nil { return nil, err } + _, platform, err := s.readOSSSetting() + if err != nil { + return nil, err + } + if next.Enabled && next.Provider == s3Provider && !platform.AllowUserS3 { + return nil, Forbidden("平台管理员尚未允许个人 S3 兼容存储") + } + if next.Provider == s3Provider && next.Enabled { + location, err := s.requireTestedS3Location("user", actor.ID, next) + if err != nil { + return nil, err + } + next.StorageLocationID = location.ID + } else if next.Provider == s3Provider && storageLocationDigest(next) == storageLocationDigest(currentValue) { + next.StorageLocationID = currentValue.StorageLocationID + } stored, err := s.encryptOSSSettingSecrets(next) if err != nil { return nil, err @@ -170,7 +240,19 @@ func (s *Service) UpdateUserOSSSetting(actor *model.User, req OSSSettingRequest) if err := s.repo.CreateUserOSSSetting(&setting); err != nil { return nil, err } - public := publicUserOSSSetting(&setting, next) + if next.Provider == s3Provider && next.StorageLocationID != "" { + if err := s.repo.ActivateStorageLocation("user", actor.ID, next.StorageLocationID, next.Enabled); err != nil { + return nil, err + } + } else if currentValue.Provider == s3Provider && currentValue.StorageLocationID != "" { + if err := s.repo.ActivateStorageLocation("user", actor.ID, "", false); err != nil { + return nil, err + } + } + public, err := s.publicUserOSSSetting(&setting, next, actor.ID, platform.AllowUserS3) + if err != nil { + return nil, err + } return &public, nil } @@ -250,6 +332,10 @@ func (s *Service) encryptOSSSettingSecrets(value ossSettingValue) (ossSettingVal if err != nil { return ossSettingValue{}, err } + value.SessionToken, err = s.encryptSettingSecret(value.SessionToken) + if err != nil { + return ossSettingValue{}, err + } value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials) for provider, credentials := range value.ArchivedCredentials { credentials.AccessKeySecret, err = s.encryptSettingSecret(credentials.AccessKeySecret) @@ -268,6 +354,13 @@ func (s *Service) decryptOSSSettingSecrets(value *ossSettingValue) (bool, error) return false, err } value.AccessKeySecret = secret + if value.SessionToken != "" && !strings.HasPrefix(value.SessionToken, encryptedSettingPrefix) { + needsMigration = true + } + value.SessionToken, err = s.decryptSettingSecret(value.SessionToken) + if err != nil { + return false, err + } value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials) for provider, credentials := range value.ArchivedCredentials { if credentials.AccessKeySecret != "" && !strings.HasPrefix(credentials.AccessKeySecret, encryptedSettingPrefix) { @@ -466,15 +559,29 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS AccessKeySecret: strings.TrimSpace(req.AccessKeySecret), PublicBaseURL: strings.TrimRight(strings.TrimSpace(req.PublicBaseURL), "/"), PathPrefix: strings.Trim(strings.TrimSpace(req.PathPrefix), "/"), + S3Preset: strings.TrimSpace(req.S3Preset), + PathStyle: req.PathStyle, + SessionToken: strings.TrimSpace(req.SessionToken), + AllowUserS3: req.AllowUserS3, }) - if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider { - return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS 和七牛云 Kodo") + if next.Provider != aliyunOSSProvider && next.Provider != tencentCOSProvider && next.Provider != qiniuKodoProvider && next.Provider != s3Provider { + return next, BadAuthRequest("仅支持阿里云 OSS、腾讯云 COS、七牛云 Kodo 和通用 S3") + } + if next.Provider == s3Provider { + switch next.S3Preset { + case "aws", "r2", "b2", "rustfs", "custom": + default: + return next, BadAuthRequest("S3 预设无效") + } } current = normalizeOSSSetting(current) - // 不同云厂商的密钥不能复用；只有继续使用同一厂商时，留空才表示保留原密钥。 + // 同一云厂商修改存储位置时，留空仍表示保留现有密钥；切换厂商时不能复用。 if next.AccessKeySecret == "" && next.Provider == current.Provider { next.AccessKeySecret = current.AccessKeySecret } + if next.SessionToken == "" && next.Provider == current.Provider { + next.SessionToken = current.SessionToken + } if next.Enabled { if next.Bucket == "" { return next, BadAuthRequest("请填写对象存储 Bucket") @@ -486,10 +593,22 @@ func ossSettingFromRequest(req OSSSettingRequest, current ossSettingValue) (ossS if next.Provider == qiniuKodoProvider { return next, BadAuthRequest("请填写七牛云 Kodo 上传 Endpoint") } + if next.Provider == s3Provider { + return next, BadAuthRequest("请填写 S3 Endpoint 服务根 URL") + } return next, BadAuthRequest("请填写阿里云 OSS Endpoint") } - if _, err := ValidateOutboundURL(next.Endpoint); err != nil { - return next, err + if next.Provider == s3Provider && next.Region == "" { + return next, BadAuthRequest("请填写 S3 Region") + } + var endpointErr error + if next.Provider == s3Provider { + _, endpointErr = validateStorageEndpoint(next.Endpoint) + } else { + _, endpointErr = ValidateOutboundURL(next.Endpoint) + } + if endpointErr != nil { + return next, endpointErr } if next.CDNBaseURL != "" { if _, err := ossCDNBaseURL(next.CDNBaseURL); err != nil { @@ -551,15 +670,24 @@ func normalizeOSSSetting(value ossSettingValue) ossSettingValue { value.AccessKeySecret = strings.TrimSpace(value.AccessKeySecret) value.PublicBaseURL = strings.TrimRight(strings.TrimSpace(value.PublicBaseURL), "/") value.PathPrefix = strings.Trim(strings.TrimSpace(value.PathPrefix), "/") + if value.PathPrefix == "" { + value.PathPrefix = defaultOSSPathPrefix + } + value.S3Preset = strings.ToLower(strings.TrimSpace(value.S3Preset)) + if value.S3Preset == "" { + value.S3Preset = "custom" + } + value.SessionToken = strings.TrimSpace(value.SessionToken) + value.StorageLocationID = strings.TrimSpace(value.StorageLocationID) value.ArchivedCredentials = cloneOSSProviderCredentials(value.ArchivedCredentials) return value } func defaultOSSSetting() ossSettingValue { - return ossSettingValue{Provider: aliyunOSSProvider} + return ossSettingValue{Provider: aliyunOSSProvider, PathPrefix: defaultOSSPathPrefix} } -func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) PublicOSSSetting { +func (s *Service) publicOSSSetting(setting *model.SystemSetting, value ossSettingValue, scope string, ownerID string) (PublicOSSSetting, error) { result := PublicOSSSetting{ Enabled: value.Enabled, Provider: value.Provider, @@ -571,16 +699,24 @@ func publicOSSSetting(setting *model.SystemSetting, value ossSettingValue) Publi HasAccessKeySecret: strings.TrimSpace(value.AccessKeySecret) != "", PublicBaseURL: value.PublicBaseURL, PathPrefix: value.PathPrefix, + S3Preset: value.S3Preset, + PathStyle: value.PathStyle, + HasSessionToken: value.SessionToken != "", + StorageLocationID: value.StorageLocationID, + AllowUserS3: value.AllowUserS3, } if setting != nil { result.UpdatedBy = setting.UpdatedBy result.CreatedAt = setting.CreatedAt result.UpdatedAt = setting.UpdatedAt } - return result + if err := s.populateStorageLocationPublic(&result, scope, ownerID, value.StorageLocationID); err != nil { + return PublicOSSSetting{}, err + } + return result, nil } -func publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue) PublicOSSSetting { +func (s *Service) publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue, ownerID string, allowUserS3 bool) (PublicOSSSetting, error) { result := PublicOSSSetting{ Enabled: value.Enabled, Provider: value.Provider, @@ -592,11 +728,35 @@ func publicUserOSSSetting(setting *model.UserOSSSetting, value ossSettingValue) HasAccessKeySecret: strings.TrimSpace(value.AccessKeySecret) != "", PublicBaseURL: value.PublicBaseURL, PathPrefix: value.PathPrefix, + S3Preset: value.S3Preset, + PathStyle: value.PathStyle, + HasSessionToken: value.SessionToken != "", + StorageLocationID: value.StorageLocationID, + AllowUserS3: allowUserS3, + } + if value.Provider == s3Provider && !allowUserS3 { + result.Enabled = false } if setting != nil { result.UpdatedBy = setting.UserID result.CreatedAt = setting.CreatedAt result.UpdatedAt = setting.UpdatedAt } - return result + if err := s.populateStorageLocationPublic(&result, "user", ownerID, value.StorageLocationID); err != nil { + return PublicOSSSetting{}, err + } + return result, nil +} + +func storageLocationDigest(value ossSettingValue) string { + value = normalizeOSSSetting(value) + payload := strings.Join([]string{value.Provider, value.Endpoint, value.Bucket, value.Region, strconv.FormatBool(value.PathStyle), value.PathPrefix}, "\x00") + sum := sha256.Sum256([]byte(payload)) + return hex.EncodeToString(sum[:]) +} + +func storageTestDigest(value ossSettingValue) string { + payload := strings.Join([]string{storageLocationDigest(value), value.AccessKeyID, value.AccessKeySecret, value.SessionToken}, "\x00") + sum := sha256.Sum256([]byte(payload)) + return hex.EncodeToString(sum[:]) } diff --git a/backend/internal/service/storage_location.go b/backend/internal/service/storage_location.go new file mode 100644 index 000000000..d0ddc6b94 --- /dev/null +++ b/backend/internal/service/storage_location.go @@ -0,0 +1,201 @@ +package service + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "path" + "strings" + "time" + + "infinite-canvas/backend/internal/model" + + "gorm.io/gorm" +) + +type OSSConnectionTestResult struct { + OK bool `json:"ok"` + Message string `json:"message,omitempty"` + TestedAt time.Time `json:"testedAt"` + TestedDigest string `json:"testedDigest"` +} + +func (s *Service) TestAdminOSSSetting(actor *model.User, req OSSSettingRequest) (*OSSConnectionTestResult, error) { + if err := s.RequireAdmin(actor); err != nil { + return nil, err + } + _, current, err := s.readOSSSetting() + if err != nil { + return nil, err + } + return s.testOSSSetting("platform", "", actor.ID, req, current) +} + +func (s *Service) TestUserOSSSetting(actor *model.User, req OSSSettingRequest) (*OSSConnectionTestResult, error) { + if actor == nil { + return nil, Unauthorized("请先登录") + } + _, platform, err := s.readOSSSetting() + if err != nil { + return nil, err + } + if strings.EqualFold(strings.TrimSpace(req.Provider), s3Provider) && !platform.AllowUserS3 { + return nil, Forbidden("平台管理员尚未允许个人 S3 兼容存储") + } + _, current, err := s.readUserOSSSetting(actor.ID) + if err != nil { + return nil, err + } + return s.testOSSSetting("user", actor.ID, actor.ID, req, current) +} + +func (s *Service) testOSSSetting(scope string, ownerID string, actorID string, req OSSSettingRequest, current ossSettingValue) (*OSSConnectionTestResult, error) { + req.Enabled = true + value, err := ossSettingFromRequest(req, current) + if err != nil { + return nil, err + } + key := scope + ":" + actorID + s.storageTestMu.Lock() + if s.activeStorageTests == nil { + s.activeStorageTests = make(map[string]bool) + } + if s.activeStorageTests[key] { + s.storageTestMu.Unlock() + return nil, NewAppError(http.StatusConflict, "对象存储连接测试正在进行") + } + s.activeStorageTests[key] = true + s.storageTestMu.Unlock() + defer func() { + s.storageTestMu.Lock() + delete(s.activeStorageTests, key) + s.storageTestMu.Unlock() + }() + + testKey := path.Join(value.PathPrefix, ".yingce-tests", scope, newID()) + payload := []byte("yingce-storage-test") + if _, err := putOSSObject(value, testKey, "application/octet-stream", int64(len(payload)), bytes.NewReader(payload)); err != nil { + return nil, err + } + stream, err := getOSSObjectRange(value, testKey, "bytes=0-3") + var rangeErr error + if err != nil { + rangeErr = err + } else { + data, readErr := io.ReadAll(io.LimitReader(stream.body, 5)) + closeErr := stream.body.Close() + if readErr != nil || closeErr != nil || string(data) != "ying" { + rangeErr = errors.New("对象存储 Range 读取测试失败") + } + } + if err := deleteOSSObject(value, testKey); err != nil { + return nil, fmt.Errorf("对象存储删除测试失败：%w", err) + } + if rangeErr != nil { + return nil, rangeErr + } + + location, err := s.upsertStorageLocation(scope, ownerID, value) + if err != nil { + return nil, err + } + now := time.Now() + location.TestedDigest = storageTestDigest(value) + location.TestedAt = &now + if err := s.repo.SaveStorageLocation(location); err != nil { + return nil, err + } + return &OSSConnectionTestResult{OK: true, Message: "连接测试通过", TestedAt: now, TestedDigest: location.TestedDigest}, nil +} + +func (s *Service) upsertStorageLocation(scope string, ownerID string, value ossSettingValue) (*model.StorageLocation, error) { + digest := storageLocationDigest(value) + location, err := s.repo.StorageLocationByDigest(scope, ownerID, value.Provider, digest) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err + } + stored, err := s.encryptOSSSettingSecrets(value) + if err != nil { + return nil, err + } + stored.Enabled = true + stored.StorageLocationID = "" + encoded, err := json.Marshal(stored) + if err != nil { + return nil, err + } + if location == nil { + location = &model.StorageLocation{ID: newID(), Scope: scope, OwnerID: ownerID, Provider: value.Provider, LocationDigest: digest, ValueJSON: string(encoded)} + if err := s.repo.CreateStorageLocation(location); err != nil { + return nil, err + } + return location, nil + } + location.ValueJSON = string(encoded) + return location, nil +} + +func (s *Service) storageLocationValue(id string) (*model.StorageLocation, ossSettingValue, error) { + location, err := s.repo.StorageLocation(id) + if err != nil { + return nil, ossSettingValue{}, err + } + var value ossSettingValue + if err := json.Unmarshal([]byte(location.ValueJSON), &value); err != nil { + return nil, ossSettingValue{}, errors.New("对象存储位置配置格式无效") + } + if _, err := s.decryptOSSSettingSecrets(&value); err != nil { + return nil, ossSettingValue{}, err + } + value.StorageLocationID = location.ID + return location, normalizeOSSSetting(value), nil +} + +func (s *Service) requireTestedS3Location(scope string, ownerID string, value ossSettingValue) (*model.StorageLocation, error) { + location, err := s.repo.StorageLocationByDigest(scope, ownerID, s3Provider, storageLocationDigest(value)) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, BadAuthRequest("S3 关键配置尚未通过连接测试") + } + return nil, err + } + if location.TestedAt == nil || location.TestedDigest != storageTestDigest(value) { + return nil, BadAuthRequest("S3 关键配置或凭据已变化，请重新连接测试") + } + return location, nil +} + +func (s *Service) populateStorageLocationPublic(result *PublicOSSSetting, scope string, ownerID string, id string) error { + count, err := s.repo.StorageLocationHistoryCount(scope, ownerID) + if err != nil { + return err + } + result.HistoryCount = count + if id == "" { + return nil + } + location, err := s.repo.StorageLocation(id) + if err != nil { + return err + } + result.TestedAt = location.TestedAt + result.TestedDigest = location.TestedDigest + result.ReferencedResourceCount, err = s.repo.StorageLocationResourceCount(id) + return err +} + +func deleteOSSObject(setting ossSettingValue, objectKey string) error { + switch normalizeOSSSetting(setting).Provider { + case s3Provider: + return deleteS3Object(setting, objectKey) + case tencentCOSProvider: + return deleteTencentCOSObject(setting, objectKey) + case qiniuKodoProvider: + return deleteQiniuObject(setting, objectKey) + default: + return deleteAliyunOSSObject(setting, objectKey) + } +} diff --git a/backend/internal/service/storage_s3.go b/backend/internal/service/storage_s3.go new file mode 100644 index 000000000..3b86bdeae --- /dev/null +++ b/backend/internal/service/storage_s3.go @@ -0,0 +1,165 @@ +package service + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + awss3 "github.com/aws/aws-sdk-go/service/s3" +) + +func validateStorageEndpoint(raw string) (*url.URL, error) { + parsed, err := ValidateOutboundURL(raw) + if err != nil { + return nil, err + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || strings.Trim(parsed.Path, "/") != "" { + return nil, BadAuthRequest("对象存储 Endpoint 必须是服务根 URL，不能包含认证信息、路径、查询参数或片段") + } + if parsed.Scheme == "http" && !allowedPrivateUpstreamHost(parsed.Hostname()) { + return nil, BadAuthRequest("对象存储 HTTP Endpoint 仅允许访问 CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS 精确放行的主机") + } + return parsed, nil +} + +func standardAWSS3Endpoint(endpoint string) bool { + parsed, err := url.Parse(endpoint) + if err != nil { + return false + } + host := strings.ToLower(parsed.Hostname()) + return host == "s3.amazonaws.com" || strings.HasPrefix(host, "s3.") && (strings.HasSuffix(host, ".amazonaws.com") || strings.HasSuffix(host, ".amazonaws.com.cn")) || strings.HasPrefix(host, "s3-") && strings.HasSuffix(host, ".amazonaws.com") +} + +func publicHTTPSStorageEndpoint(endpoint string) bool { + parsed, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" { + return false + } + return validateOutboundHost(parsed.Hostname()) == nil && !allowedPrivateUpstreamHost(parsed.Hostname()) +} + +func newS3Client(setting ossSettingValue, timeout time.Duration) (*awss3.S3, error) { + setting = normalizeOSSSetting(setting) + endpoint, err := validateStorageEndpoint(setting.Endpoint) + if err != nil { + return nil, err + } + if setting.Region == "" || setting.Bucket == "" || setting.AccessKeyID == "" || setting.AccessKeySecret == "" { + return nil, errors.New("S3 Region、Bucket 或访问密钥不完整") + } + httpClient := OutboundHTTPClient(timeout) + config := aws.NewConfig(). + WithRegion(setting.Region). + WithEndpoint(endpoint.String()). + WithCredentials(credentials.NewStaticCredentials(setting.AccessKeyID, setting.AccessKeySecret, setting.SessionToken)). + WithHTTPClient(httpClient). + WithS3ForcePathStyle(setting.PathStyle || !standardAWSS3Endpoint(endpoint.String())). + WithDisableSSL(endpoint.Scheme == "http") + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + return awss3.New(sess), nil +} + +func putS3Object(setting ossSettingValue, objectKey string, mimeType string, size int64, body io.Reader) (string, error) { + client, err := newS3Client(setting, 2*time.Minute) + if err != nil { + return "", err + } + var seekable io.ReadSeeker + if reader, ok := body.(io.ReadSeeker); ok { + seekable = reader + } else { + limit := size + 1 + if limit <= 0 { + limit = 64 << 20 + } + data, readErr := io.ReadAll(io.LimitReader(body, limit)) + if readErr != nil { + return "", readErr + } + if size >= 0 && int64(len(data)) != size { + return "", errors.New("S3 上传内容长度与资源记录不一致") + } + seekable = bytes.NewReader(data) + } + input := &awss3.PutObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/")), Body: seekable} + if mimeType != "" { + input.ContentType = aws.String(mimeType) + } + if size >= 0 { + input.ContentLength = aws.Int64(size) + } + output, err := client.PutObjectWithContext(context.Background(), input) + if err != nil { + return "", fmt.Errorf("S3 上传失败：%w", err) + } + return strings.Trim(aws.StringValue(output.ETag), `"`), nil +} + +func getS3ObjectRange(setting ossSettingValue, objectKey string, rangeHeader string) (*ossObjectStream, error) { + client, err := newS3Client(setting, 2*time.Minute) + if err != nil { + return nil, err + } + input := &awss3.GetObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))} + if rangeHeader != "" { + input.Range = aws.String(rangeHeader) + } + output, err := client.GetObjectWithContext(context.Background(), input) + if err != nil { + if requestFailure, ok := err.(awserr.RequestFailure); ok && requestFailure.StatusCode() == http.StatusRequestedRangeNotSatisfiable { + return &ossObjectStream{body: io.NopCloser(bytes.NewReader(nil)), statusCode: http.StatusRequestedRangeNotSatisfiable, acceptRanges: "bytes"}, nil + } + return nil, fmt.Errorf("S3 读取失败：%w", err) + } + status := http.StatusOK + if rangeHeader != "" && aws.StringValue(output.ContentRange) != "" { + status = http.StatusPartialContent + } + return &ossObjectStream{body: output.Body, statusCode: status, contentLength: aws.Int64Value(output.ContentLength), contentRange: aws.StringValue(output.ContentRange), acceptRanges: firstNonEmpty(aws.StringValue(output.AcceptRanges), "bytes")}, nil +} + +func signedS3ObjectURL(setting ossSettingValue, objectKey string, expiresAt time.Time) (string, error) { + client, err := newS3Client(setting, 2*time.Minute) + if err != nil { + return "", err + } + duration := time.Until(expiresAt) + if duration <= 0 { + return "", errors.New("S3 签名有效期必须晚于当前时间") + } + req, _ := client.GetObjectRequest(&awss3.GetObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))}) + value, err := req.Presign(duration) + if err != nil { + return "", fmt.Errorf("S3 下载地址签名失败：%w", err) + } + return value, nil +} + +func deleteS3Object(setting ossSettingValue, objectKey string) error { + client, err := newS3Client(setting, 2*time.Minute) + if err != nil { + return err + } + _, err = client.DeleteObjectWithContext(context.Background(), &awss3.DeleteObjectInput{Bucket: aws.String(setting.Bucket), Key: aws.String(strings.TrimLeft(objectKey, "/"))}) + if requestFailure, ok := err.(awserr.RequestFailure); ok && requestFailure.StatusCode() == http.StatusNotFound { + return nil + } + if err != nil { + return fmt.Errorf("删除 S3 对象失败：%w", err) + } + return nil +} diff --git a/backend/internal/service/storage_s3_test.go b/backend/internal/service/storage_s3_test.go new file mode 100644 index 000000000..612c6b9f7 --- /dev/null +++ b/backend/internal/service/storage_s3_test.go @@ -0,0 +1,178 @@ +package service + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +func TestS3ObjectOperationsUsePathStyleSessionTokenAndNoManagedHeaders(t *testing.T) { + t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1") + var methods []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + if r.URL.Path != "/bucket/prefix/object.txt" { + t.Errorf("path = %q", r.URL.Path) + } + if r.Header.Get("X-Amz-Security-Token") != "session-token" { + t.Errorf("X-Amz-Security-Token = %q", r.Header.Get("X-Amz-Security-Token")) + } + if r.Header.Get("X-Amz-Acl") != "" || r.Header.Get("X-Amz-Server-Side-Encryption") != "" { + t.Errorf("unexpected ACL/SSE headers: %#v", r.Header) + } + switch r.Method { + case http.MethodPut: + data, _ := io.ReadAll(r.Body) + if string(data) != "payload" { + t.Errorf("body = %q", data) + } + w.Header().Set("ETag", `"etag-value"`) + case http.MethodGet: + if r.Header.Get("Range") != "bytes=0-3" { + t.Errorf("Range = %q", r.Header.Get("Range")) + } + w.Header().Set("Content-Range", "bytes 0-3/7") + w.Header().Set("Accept-Ranges", "bytes") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "payl") + case http.MethodDelete: + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `NoSuchKeymissing`) + } + })) + defer server.Close() + + setting := ossSettingValue{Provider: s3Provider, Region: "us-east-1", Endpoint: server.URL, Bucket: "bucket", AccessKeyID: "access-id", AccessKeySecret: "secret-value", SessionToken: "session-token"} + etag, err := putS3Object(setting, "prefix/object.txt", "text/plain", 7, bytes.NewReader([]byte("payload"))) + if err != nil || etag != "etag-value" { + t.Fatalf("putS3Object() = %q, %v", etag, err) + } + stream, err := getS3ObjectRange(setting, "prefix/object.txt", "bytes=0-3") + if err != nil { + t.Fatal(err) + } + data, _ := io.ReadAll(stream.body) + _ = stream.body.Close() + if stream.statusCode != http.StatusPartialContent || stream.contentRange != "bytes 0-3/7" || string(data) != "payl" { + t.Fatalf("stream = %#v, body = %q", stream, data) + } + if err := deleteS3Object(setting, "prefix/object.txt"); err != nil { + t.Fatalf("deleteS3Object(404) error = %v", err) + } + if strings.Join(methods, ",") != "PUT,GET,DELETE" { + t.Fatalf("methods = %v", methods) + } +} + +func TestSignedS3ObjectURLUsesSDKPresignAndSessionToken(t *testing.T) { + t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1") + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + value, err := signedS3ObjectURL(ossSettingValue{ + Provider: s3Provider, Region: "us-east-1", Endpoint: server.URL, Bucket: "bucket", + AccessKeyID: "access-id", AccessKeySecret: "secret-value", SessionToken: "session-token", + }, "folder/object.png", time.Now().Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(value) + if err != nil { + t.Fatal(err) + } + if parsed.Path != "/bucket/folder/object.png" || parsed.Query().Get("X-Amz-Security-Token") != "session-token" || parsed.Query().Get("X-Amz-Signature") == "" { + t.Fatalf("presigned URL = %q", value) + } + if strings.Contains(value, "secret-value") { + t.Fatal("presigned URL leaked secret key") + } +} + +func TestS3EndpointAndDigestsEnforceStorageContract(t *testing.T) { + t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "storage.internal") + if _, err := validateStorageEndpoint("https://example.com/path"); err == nil { + t.Fatal("validateStorageEndpoint() accepted a path") + } + if _, err := validateStorageEndpoint("http://example.com"); err == nil { + t.Fatal("validateStorageEndpoint() accepted public HTTP") + } + base := ossSettingValue{Provider: s3Provider, Region: "us-east-1", Endpoint: "https://s3.example.com", Bucket: "bucket", PathPrefix: "assets", AccessKeyID: "id", AccessKeySecret: "secret"} + rotated := base + rotated.AccessKeySecret = "rotated" + moved := base + moved.PathPrefix = "other" + if storageLocationDigest(base) != storageLocationDigest(rotated) { + t.Fatal("credential rotation changed the location digest") + } + if storageTestDigest(base) == storageTestDigest(rotated) { + t.Fatal("credential rotation did not invalidate the tested digest") + } + if storageLocationDigest(base) == storageLocationDigest(moved) { + t.Fatal("location change did not create a different digest") + } +} + +func TestOSSSettingKeepsS3SecretsWhenLocationChanges(t *testing.T) { + current := ossSettingValue{ + Provider: s3Provider, + Region: "us-east-1", + Endpoint: "https://s3.example.com", + Bucket: "old-bucket", + AccessKeyID: "access-id", + AccessKeySecret: "secret-value", + SessionToken: "session-token", + PathPrefix: defaultOSSPathPrefix, + } + next, err := ossSettingFromRequest(OSSSettingRequest{ + Enabled: true, + Provider: s3Provider, + S3Preset: "custom", + Region: "us-east-1", + Endpoint: "https://s3.example.com", + Bucket: "new-bucket", + AccessKeyID: "access-id", + PathPrefix: "changed-prefix", + }, current) + if err != nil { + t.Fatal(err) + } + if next.AccessKeySecret != current.AccessKeySecret || next.SessionToken != current.SessionToken { + t.Fatalf("secrets were not retained: %#v", next) + } +} + +func TestDefaultOSSPathPrefix(t *testing.T) { + value := normalizeOSSSetting(ossSettingValue{}) + if value.PathPrefix != defaultOSSPathPrefix || defaultOSSSetting().PathPrefix != defaultOSSPathPrefix { + t.Fatalf("default path prefix = %q, default setting = %q", value.PathPrefix, defaultOSSSetting().PathPrefix) + } +} + +func TestResourceObjectKeyPreservesOrInfersExtension(t *testing.T) { + setting := ossSettingValue{PathPrefix: "assets"} + now := time.Date(2026, time.August, 27, 20, 31, 0, 0, time.UTC) + + for _, test := range []struct { + name string + fileName string + mimeType string + kind string + ext string + }{ + {name: "original filename", fileName: "photo.JPEG", mimeType: "image/jpeg", kind: "image", ext: ".jpeg"}, + {name: "mime type", mimeType: "image/png", kind: "image", ext: ".png"}, + {name: "kind fallback", kind: "video", ext: ".mp4"}, + } { + t.Run(test.name, func(t *testing.T) { + objectKey := ossObjectKey(setting, "user-1", test.kind, test.fileName, test.mimeType, now) + if !strings.HasSuffix(objectKey, test.ext) { + t.Fatalf("object key = %q, want suffix %q", objectKey, test.ext) + } + }) + } +} diff --git a/docs/content/docs/backend/backend-database.mdx b/docs/content/docs/backend/backend-database.mdx index 2839b504a..1fde37b02 100644 --- a/docs/content/docs/backend/backend-database.mdx +++ b/docs/content/docs/backend/backend-database.mdx @@ -26,3 +26,13 @@ description: 影策后端核心持久化表与状态边界 `comfy_bridge_requests` 是 Bridge 交接的真相源。后端进程重启后，未领取请求继续从数据库投递；已领取请求的迟到结果先落库，任务 worker 恢复后使用原请求 ID 读取结果，不会创建第二次本地工作流执行。请求有效期当前为两小时，过期的排队或执行记录会转为失败。 请求表保存工作流执行 payload 和结果，但不保存 Bridge Token。任务取消时，未领取请求可以确定取消；已经领取的本机工作流无法可靠中止，迟到结果会被丢弃并进入费用核对流程。 + +## 资源存储 + +| 表 | 用途 | 关键边界 | +| --- | --- | --- | +| `resources` | 保存资源元数据和实际存储位置引用 | S3 资源必须通过 `storage_setting_id` 固定绑定创建时的位置；浏览器默认经同源接口读取 | +| `storage_locations` | 保存平台或个人 S3 位置版本、加密凭据和连接测试摘要 | Endpoint、Bucket、Region、PathStyle、PathPrefix、Provider 组成不可变位置；同位置只轮换凭据，位置变化创建新记录 | +| `resource_deletion_jobs` | 在业务记录删除后可靠清理物理对象 | 固化资源的存储位置与对象键；删除失败保留任务并重试，S3 404 按幂等成功处理 | + +`storage_locations.value_json` 中的 Secret Key 和 Session Token 使用后端设置密钥加密。启用 S3 前，关键位置字段和凭据摘要必须与最近一次 Put、Range Get、Delete 连接测试结果一致；历史资源继续按绑定的位置读取和删除，不随当前配置切换。 diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 0584b511f..a1037ded9 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -170,6 +170,17 @@ description: 影策已实现功能的手工验证清单与未确认边界 - [ ] 使用真实 OSS 配置验证对象删除成功、权限失败后的退避重试和多实例租约接管。 - [ ] 浏览器真实环境验证登录快照慢响应期间工作区保持不可写，以及删除后立即刷新不复活素材。 +## S3 兼容存储 + +- [x] 后端 S3 单次上传、Range 读取、预签名地址、Session Token、幂等删除、配置版本和迁移清单自动化测试通过。 +- [x] 前端 S3 预设、关键字段重测判断、默认 `open-ai-canvas` 路径前缀、TypeScript 检查和生产构建通过。 +- [ ] 分别使用 AWS S3、Cloudflare R2、Backblaze B2 和 RustFS 完成写入、Range 读取、删除及历史资源恢复验证。 +- [ ] 验证 Path Style 开启时强制路径寻址，关闭时 AWS 使用虚拟主机寻址、其他自定义 Endpoint 自动使用路径寻址。 +- [ ] 验证平台切换到本机后只影响新上传，历史 S3 资源仍可读取和删除；个人 S3 停用后正确回退平台存储。 +- [ ] 验证同一 S3 修改 Bucket、Endpoint、Region、前缀或 Path Style 时，留空 Secret 和 Session Token 可继续复用，并要求重新连接测试。 +- [ ] 使用私网 RustFS 验证浏览器同源代理，以及提交给模型上游时转换为影策公网 HTTPS 短时地址；未配置公网 HTTPS 地址时应明确拒绝。 +- [ ] 验证管理员关闭个人 S3 后不再接受个人新上传，但已有个人 S3 资源仍可读取和删除。 + ## 3D 导演台时间轴关键帧删除与快捷键 关键帧删除入口、全局快捷键和交互控件守卫已接入，以下场景仍需要在浏览器中由用户确认： @@ -255,4 +266,4 @@ description: 影策已实现功能的手工验证清单与未确认边界 - [ ] 在引导加载期间连续点击“重新开始引导”，再模拟 IndexedDB 写入失败与恢复，确认界面不会永久隐藏或假装推进。 - [ ] 在窄窗口检查摆场、姿态、动画、摄影机四种模式：标题栏可横向到达，底部属性面板可滚动，时间轴与引导不互相遮挡。 -用户确认上述场景后，再将对应条目移动到功能文档的已验证范围。 + 用户确认上述场景后，再将对应条目移动到功能文档的已验证范围。 diff --git a/web/src/components/layout/user-oss-settings-form.tsx b/web/src/components/layout/user-oss-settings-form.tsx index 21a780e3e..50f381801 100644 --- a/web/src/components/layout/user-oss-settings-form.tsx +++ b/web/src/components/layout/user-oss-settings-form.tsx @@ -2,19 +2,23 @@ import { App, Button, Form, Input, Select, Switch, Tag } from "antd"; import { Cloud, ShieldCheck } from "lucide-react"; import { useEffect, useState } from "react"; -import { getUserOSSSetting, updateUserOSSSetting, type UserOSSSetting } from "@/services/api/resources"; +import { changesRequireOSSRetest, DEFAULT_OSS_PATH_PREFIX, getS3PresetHints, S3_PRESET_OPTIONS, type OSSConnectionTestResult, type OSSProvider, type S3Preset } from "@/lib/oss-settings"; +import { getUserOSSSetting, testUserOSSConnection, updateUserOSSSetting, type UserOSSSetting } from "@/services/api/resources"; import { useUserStore } from "@/stores/use-user-store"; type OSSFormValues = { enabled?: boolean; - provider: "aliyun" | "tencent" | "qiniu"; + provider: OSSProvider; + s3Preset?: S3Preset; region?: string; endpoint?: string; cdnBaseUrl?: string; bucket?: string; accessKeyId?: string; accessKeySecret?: string; + sessionToken?: string; pathPrefix?: string; + pathStyle?: boolean; }; export function UserOSSSettingsForm() { @@ -24,10 +28,15 @@ export function UserOSSSettingsForm() { const [setting, setSetting] = useState(null); const [loading, setLoading] = useState(false); const [saving, setSaving] = useState(false); + const [testing, setTesting] = useState(false); + const [testResult, setTestResult] = useState(null); + const [testStale, setTestStale] = useState(false); const savedAt = formatSavedAt(setting?.updatedAt); const provider = Form.useWatch("provider", form) || "aliyun"; const isTencentCOS = provider === "tencent"; const isQiniuKodo = provider === "qiniu"; + const isS3 = provider === "s3"; + const s3Preset = Form.useWatch("s3Preset", form) || "custom"; const hasCurrentProviderSecret = Boolean(setting && setting.provider === provider && setting.hasAccessKeySecret); const accessKeyIdLabel = isTencentCOS ? "SecretId" : isQiniuKodo ? "AccessKey" : "AccessKey ID"; const accessKeySecretLabel = isTencentCOS ? "SecretKey" : isQiniuKodo ? "SecretKey" : "AccessKey Secret"; @@ -41,6 +50,7 @@ export function UserOSSSettingsForm() { if (!active) return; setSetting(data.setting); form.setFieldsValue(toFormValues(data.setting)); + setTestResult(data.setting.testedAt ? { ok: true, testedAt: data.setting.testedAt, testedDigest: data.setting.testedDigest } : null); }) .catch((error) => active && message.error(error instanceof Error ? error.message : "读取个人对象存储配置失败")) .finally(() => active && setLoading(false)); @@ -55,21 +65,27 @@ export function UserOSSSettingsForm() { const save = async () => { const values = await form.validateFields(); + if (values.enabled && values.provider === "s3" && !setting?.allowUserS3) return message.error("平台管理员尚未允许个人 S3 兼容存储"); setSaving(true); try { const data = await updateUserOSSSetting({ enabled: values.enabled === true, provider: values.provider || "aliyun", + s3Preset: values.s3Preset || "custom", region: values.region?.trim() || "", endpoint: values.endpoint?.trim() || "", cdnBaseUrl: values.cdnBaseUrl?.trim() || "", bucket: values.bucket?.trim() || "", accessKeyId: values.accessKeyId?.trim() || "", accessKeySecret: values.accessKeySecret?.trim() || "", - pathPrefix: values.pathPrefix?.trim() || "", + sessionToken: values.sessionToken?.trim() || "", + pathPrefix: values.pathPrefix?.trim() || DEFAULT_OSS_PATH_PREFIX, + pathStyle: values.pathStyle === true, }); setSetting(data.setting); form.setFieldsValue(toFormValues(data.setting)); + setTestResult(data.setting.testedAt ? { ok: true, testedAt: data.setting.testedAt, testedDigest: data.setting.testedDigest } : null); + setTestStale(false); message.success(data.setting.enabled ? "个人对象存储已启用，后续上传将优先使用该存储" : "个人对象存储已停用，后续上传将使用平台存储"); } catch (error) { message.error(error instanceof Error ? error.message : "保存个人对象存储配置失败"); @@ -78,8 +94,24 @@ export function UserOSSSettingsForm() { } }; + const testConnection = async () => { + const values = await form.validateFields(); + setTesting(true); + try { + const result = await testUserOSSConnection(connectionInput(values)); + setTestResult(result); + setTestStale(false); + result.ok ? message.success(result.message || "连接测试通过") : message.error(result.message || "连接测试失败"); + } catch (error) { + setTestResult({ ok: false, message: error instanceof Error ? error.message : "连接测试失败" }); + setTestStale(false); + } finally { + setTesting(false); + } + }; + return ( -
+ changesRequireOSSRetest(changed) && setTestStale(true)}>
@@ -102,21 +134,26 @@ export function UserOSSSettingsForm() { form.setFieldsValue(getS3PresetHints(preset))} /> + + ) : null} - + - - + + - + @@ -139,13 +176,21 @@ export function UserOSSSettingsForm() { + {isS3 ? ( + <> + + + + + + + + ) : null}
- {savedAt ? `上次保存：${savedAt}` : "尚未保存个人对象存储配置"} - +
{savedAt ? `上次保存：${savedAt}` : "尚未保存个人对象存储配置"}
+
); @@ -162,12 +207,37 @@ function toFormValues(setting: UserOSSSetting): OSSFormValues { return { enabled: setting.enabled, provider: setting.provider || "aliyun", + s3Preset: setting.s3Preset || "custom", region: setting.region, endpoint: setting.endpoint, cdnBaseUrl: setting.cdnBaseUrl, bucket: setting.bucket, accessKeyId: setting.accessKeyId, accessKeySecret: "", - pathPrefix: setting.pathPrefix, + sessionToken: "", + pathPrefix: setting.pathPrefix || DEFAULT_OSS_PATH_PREFIX, + pathStyle: setting.pathStyle === true, }; } + +function connectionInput(values: OSSFormValues) { + return { + provider: values.provider, + s3Preset: values.s3Preset, + region: values.region?.trim() || "", + endpoint: values.endpoint?.trim() || "", + cdnBaseUrl: values.cdnBaseUrl?.trim() || "", + bucket: values.bucket?.trim() || "", + accessKeyId: values.accessKeyId?.trim() || "", + accessKeySecret: values.accessKeySecret?.trim() || "", + sessionToken: values.sessionToken?.trim() || "", + pathPrefix: values.pathPrefix?.trim() || DEFAULT_OSS_PATH_PREFIX, + pathStyle: values.pathStyle === true, + }; +} + +function ConnectionTestStatus({ result, stale }: { result: OSSConnectionTestResult | null; stale: boolean }) { + if (stale) return 需重新测试; + if (!result) return null; + return {result.ok ? `测试通过${result.testedAt ? ` · ${formatSavedAt(result.testedAt)}` : ""}` : result.message || "测试失败"}; +} diff --git a/web/src/lib/oss-settings.ts b/web/src/lib/oss-settings.ts new file mode 100644 index 000000000..7053d5710 --- /dev/null +++ b/web/src/lib/oss-settings.ts @@ -0,0 +1,84 @@ +export type OSSProvider = "aliyun" | "tencent" | "qiniu" | "s3"; + +export type S3Preset = "aws" | "r2" | "b2" | "rustfs" | "custom"; + +export const DEFAULT_OSS_PATH_PREFIX = "open-ai-canvas"; + +export type OSSConnectionTestInput = { + provider: OSSProvider; + s3Preset?: S3Preset; + region: string; + endpoint: string; + cdnBaseUrl: string; + bucket: string; + accessKeyId: string; + accessKeySecret?: string; + sessionToken?: string; + pathPrefix: string; + pathStyle?: boolean; +}; + +export type OSSConnectionTestResult = { + ok: boolean; + message?: string; + testedAt?: string; + testedDigest?: string; +}; + +export const S3_PRESET_OPTIONS: Array<{ label: string; value: S3Preset }> = [ + { label: "AWS S3", value: "aws" }, + { label: "Cloudflare R2", value: "r2" }, + { label: "Backblaze B2", value: "b2" }, + { label: "RustFS", value: "rustfs" }, + { label: "自定义", value: "custom" }, +]; + +const S3_PRESET_HINTS: Record = { + aws: { + region: "us-east-1", + endpoint: "https://s3.us-east-1.amazonaws.com", + help: "填写 AWS 区域对应的 S3 服务根 URL。", + }, + r2: { + region: "auto", + endpoint: "https://.r2.cloudflarestorage.com", + help: "Endpoint 使用 R2 控制台提供的账户级 S3 API 根 URL。", + }, + b2: { + region: "us-west-004", + endpoint: "https://s3.us-west-004.backblazeb2.com", + help: "Region 和 Endpoint 应与 Backblaze B2 Bucket 所在区域一致。", + }, + rustfs: { + region: "us-east-1", + endpoint: "http://127.0.0.1:9000", + help: "填写后端可访问的 RustFS S3 服务根 URL。", + }, + custom: { + region: "us-east-1", + endpoint: "https://s3.example.com", + help: "填写兼容 S3 API 的服务根 URL，不要包含 Bucket 或对象路径。", + }, +}; + +const CONNECTION_FIELDS = new Set([ + "provider", + "s3Preset", + "region", + "endpoint", + "cdnBaseUrl", + "bucket", + "accessKeyId", + "accessKeySecret", + "sessionToken", + "pathPrefix", + "pathStyle", +]); + +export function getS3PresetHints(preset?: S3Preset) { + return S3_PRESET_HINTS[preset || "custom"]; +} + +export function changesRequireOSSRetest(changedValues: Record) { + return Object.keys(changedValues).some((key) => CONNECTION_FIELDS.has(key)); +} diff --git a/web/src/pages/admin/settings/storage-settings-page.tsx b/web/src/pages/admin/settings/storage-settings-page.tsx index 7314a0209..7a2fd3bdd 100644 --- a/web/src/pages/admin/settings/storage-settings-page.tsx +++ b/web/src/pages/admin/settings/storage-settings-page.tsx @@ -1,15 +1,17 @@ -import { App, Button, Form, Input, Segmented, Space } from "antd"; -import { Cloud, Globe, HardDrive, LocateFixed } from "lucide-react"; +import { App, Button, Form, Input, Segmented, Select, Space, Switch, Tag } from "antd"; +import { Cloud, Globe, HardDrive, LocateFixed, Wifi } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; -import { getAdminOSSSetting, updateAdminOSSSetting, type AdminOSSSetting } from "@/services/api/auth"; +import { changesRequireOSSRetest, DEFAULT_OSS_PATH_PREFIX, getS3PresetHints, S3_PRESET_OPTIONS, type OSSConnectionTestResult, type S3Preset } from "@/lib/oss-settings"; +import { getAdminOSSSetting, testAdminOSSConnection, updateAdminOSSSetting, type AdminOSSSetting } from "@/services/api/auth"; import { useAdminContext } from "../admin-context"; import { AdminPageFrame } from "../components/admin-shell"; import { AdminStatusBadge, configuredSecretText, SettingsSectionCard } from "../components/admin-ui"; -type StorageMode = "local" | "aliyun" | "tencent" | "qiniu"; +type StorageMode = "local" | AdminOSSSetting["provider"]; type OSSFormValues = { mode: StorageMode; + s3Preset?: S3Preset; publicBaseUrl?: string; region?: string; endpoint?: string; @@ -17,7 +19,10 @@ type OSSFormValues = { bucket?: string; accessKeyId?: string; accessKeySecret?: string; + sessionToken?: string; pathPrefix?: string; + pathStyle?: boolean; + allowUserS3?: boolean; }; export default function StorageSettingsPage() { @@ -26,11 +31,16 @@ export default function StorageSettingsPage() { const [setting, setSetting] = useState(null); const [loading, setLoading] = useState(true); const [saving, setSaving] = useState(false); + const [testing, setTesting] = useState(false); + const [testResult, setTestResult] = useState(null); + const [testStale, setTestStale] = useState(false); const [form] = Form.useForm(); const mode = Form.useWatch("mode", form) || "local"; const isObjectStorage = mode !== "local"; const isTencentCOS = mode === "tencent"; const isQiniuKodo = mode === "qiniu"; + const isS3 = mode === "s3"; + const s3Preset = Form.useWatch("s3Preset", form) || "custom"; const accessKeyIdLabel = isTencentCOS ? "SecretId" : isQiniuKodo ? "AccessKey" : "AccessKey ID"; const accessKeySecretLabel = isTencentCOS ? "SecretKey" : isQiniuKodo ? "SecretKey" : "AccessKey Secret"; const hasCurrentProviderSecret = Boolean(setting && setting.provider === mode && setting.hasAccessKeySecret); @@ -41,6 +51,7 @@ export default function StorageSettingsPage() { .then(({ setting: value }) => { setSetting(value); form.setFieldsValue(formValues(value)); + setTestResult(value.testedAt ? { ok: true, testedAt: value.testedAt, testedDigest: value.testedDigest } : null); }) .catch((error) => message.error(error instanceof Error ? error.message : "读取对象存储配置失败")) .finally(() => setLoading(false)); @@ -56,23 +67,31 @@ export default function StorageSettingsPage() { if (values.mode === "aliyun" && !values.endpoint?.trim()) return message.error("请填写阿里云 OSS Endpoint"); if (values.mode === "tencent" && !values.endpoint?.trim() && !values.region?.trim()) return message.error("请填写腾讯云 COS Region 或 Endpoint"); if (values.mode === "qiniu" && !values.endpoint?.trim()) return message.error("请填写七牛云 Kodo 上传 Endpoint"); + if (values.mode === "s3" && !values.region?.trim()) return message.error("请填写 S3 Region"); + if (values.mode === "s3" && !values.endpoint?.trim()) return message.error("请填写 S3 Endpoint 服务根 URL"); setSaving(true); try { const result = await updateAdminOSSSetting({ enabled: values.mode !== "local", provider: values.mode === "local" ? setting?.provider || "aliyun" : values.mode, + s3Preset: values.s3Preset || "custom", region: values.region?.trim() || "", endpoint: values.endpoint?.trim() || "", cdnBaseUrl: values.cdnBaseUrl?.trim() || "", bucket: values.bucket?.trim() || "", accessKeyId: values.accessKeyId?.trim() || "", accessKeySecret: values.accessKeySecret?.trim() || "", + sessionToken: values.sessionToken?.trim() || "", + pathStyle: values.pathStyle === true, + allowUserS3: values.allowUserS3 === true, publicBaseUrl: values.publicBaseUrl?.trim() || "", - pathPrefix: values.pathPrefix?.trim() || "", + pathPrefix: values.pathPrefix?.trim() || DEFAULT_OSS_PATH_PREFIX, }); setSetting(result.setting); form.setFieldsValue(formValues(result.setting)); + setTestResult(result.setting.testedAt ? { ok: true, testedAt: result.setting.testedAt, testedDigest: result.setting.testedDigest } : null); + setTestStale(false); message.success("存储配置已保存"); } catch (error) { message.error(error instanceof Error ? error.message : "保存存储配置失败"); @@ -81,6 +100,23 @@ export default function StorageSettingsPage() { } }; + const testConnection = async () => { + const values = await form.validateFields(); + if (values.mode === "local") return; + setTesting(true); + try { + const result = await testAdminOSSConnection(connectionInput(values)); + setTestResult(result); + setTestStale(false); + result.ok ? message.success(result.message || "连接测试通过") : message.error(result.message || "连接测试失败"); + } catch (error) { + setTestResult({ ok: false, message: error instanceof Error ? error.message : "连接测试失败" }); + setTestStale(false); + } finally { + setTesting(false); + } + }; + return (
@@ -104,7 +140,7 @@ export default function StorageSettingsPage() { } > -
+ changesRequireOSSRetest(changed) && setTestStale(true)}> block @@ -113,35 +149,47 @@ export default function StorageSettingsPage() { { label: 阿里云 OSS, value: "aliyun" }, { label: 腾讯云 COS, value: "tencent" }, { label: 七牛云 Kodo, value: "qiniu" }, + { label: S3 兼容存储, value: "s3" }, ]} onChange={(value) => { const nextMode = value as StorageMode; const switchingProvider = nextMode !== "local" && ((mode !== "local" && mode !== nextMode) || (mode === "local" && setting?.provider !== nextMode)); - if (switchingProvider) form.setFieldsValue({ region: "", endpoint: "", cdnBaseUrl: "", bucket: "", accessKeyId: "", accessKeySecret: "" }); + if (switchingProvider) form.setFieldsValue({ s3Preset: "custom", region: "", endpoint: "", cdnBaseUrl: "", bucket: "", accessKeyId: "", accessKeySecret: "", sessionToken: "", pathStyle: false }); }} /> {isObjectStorage ? (
+ {isS3 ? ( + + + - +
- - + + @@ -156,6 +204,20 @@ export default function StorageSettingsPage() {
+ {isS3 ? ( +
+ + + + + + +
+ ) : null} +
+ + +
) : ( )} + + +
@@ -181,6 +246,7 @@ export default function StorageSettingsPage() { function formValues(setting?: AdminOSSSetting | null): OSSFormValues { return { mode: setting?.enabled ? setting.provider : "local", + s3Preset: setting?.s3Preset || "custom", publicBaseUrl: setting?.publicBaseUrl || "", region: setting?.region || "", endpoint: setting?.endpoint || "", @@ -188,14 +254,39 @@ function formValues(setting?: AdminOSSSetting | null): OSSFormValues { bucket: setting?.bucket || "", accessKeyId: setting?.accessKeyId || "", accessKeySecret: "", - pathPrefix: setting?.pathPrefix || "", + sessionToken: "", + pathPrefix: setting?.pathPrefix || DEFAULT_OSS_PATH_PREFIX, + pathStyle: setting?.pathStyle === true, + allowUserS3: setting?.allowUserS3 === true, }; } function storageProviderLabel(provider?: AdminOSSSetting["provider"] | StorageMode) { - return provider === "tencent" ? "腾讯云 COS" : provider === "qiniu" ? "七牛云 Kodo" : provider === "aliyun" ? "阿里云 OSS" : "服务器本地"; + return provider === "s3" ? "S3 兼容存储" : provider === "tencent" ? "腾讯云 COS" : provider === "qiniu" ? "七牛云 Kodo" : provider === "aliyun" ? "阿里云 OSS" : "服务器本地"; } function formatTime(value?: string) { return value ? new Date(value).toLocaleString("zh-CN", { hour12: false }) : "--"; } + +function connectionInput(values: OSSFormValues) { + return { + provider: values.mode === "local" ? "aliyun" as const : values.mode, + s3Preset: values.s3Preset, + region: values.region?.trim() || "", + endpoint: values.endpoint?.trim() || "", + cdnBaseUrl: values.cdnBaseUrl?.trim() || "", + bucket: values.bucket?.trim() || "", + accessKeyId: values.accessKeyId?.trim() || "", + accessKeySecret: values.accessKeySecret?.trim() || "", + sessionToken: values.sessionToken?.trim() || "", + pathPrefix: values.pathPrefix?.trim() || DEFAULT_OSS_PATH_PREFIX, + pathStyle: values.pathStyle === true, + }; +} + +function ConnectionTestStatus({ result, stale }: { result: OSSConnectionTestResult | null; stale: boolean }) { + if (stale) return 关键配置已变更，需重新测试; + if (!result) return 尚未测试连接; + return {result.ok ? `测试通过${result.testedAt ? ` · ${formatTime(result.testedAt)}` : ""}` : result.message || "测试失败"}; +} diff --git a/web/src/services/api/auth.ts b/web/src/services/api/auth.ts index ef1c425eb..21d97973c 100644 --- a/web/src/services/api/auth.ts +++ b/web/src/services/api/auth.ts @@ -5,6 +5,7 @@ import type { CanvasDrawingEngineSetting } from "@/lib/canvas/canvas-drawing-eng import type { FeatureAvailability } from "@/stores/use-user-store"; import { apiClient, request } from "@/services/api/request"; import type { PublicLogicalModel } from "@/services/api/logical-models"; +import type { OSSConnectionTestInput, OSSConnectionTestResult, OSSProvider, S3Preset } from "@/lib/oss-settings"; const api = apiClient; @@ -257,7 +258,8 @@ export type UserPromptPreference = { export type AdminOSSSetting = { enabled: boolean; - provider: "aliyun" | "tencent" | "qiniu"; + provider: OSSProvider; + s3Preset: S3Preset; region: string; endpoint: string; cdnBaseUrl: string; @@ -265,8 +267,16 @@ export type AdminOSSSetting = { accessKeyId: string; accessKeySecret?: string; hasAccessKeySecret: boolean; + sessionToken?: string; + hasSessionToken: boolean; + pathStyle: boolean; + allowUserS3: boolean; publicBaseUrl: string; pathPrefix: string; + testedAt?: string; + testedDigest?: string; + historyCount?: number; + referencedResourceCount?: number; updatedBy?: string; createdAt?: string; updatedAt?: string; @@ -501,6 +511,10 @@ export function updateAdminOSSSetting(input: Partial) { return request<{ setting: AdminOSSSetting }>(api.patch("/admin/settings/oss", input)); } +export function testAdminOSSConnection(input: OSSConnectionTestInput) { + return request(api.post("/admin/settings/oss/test", input)); +} + export function getAdminArkPrivateAssetSetting() { return request<{ setting: AdminArkPrivateAssetSetting }>(api.get("/admin/settings/ark-private-assets")); } diff --git a/web/src/services/api/resources.ts b/web/src/services/api/resources.ts index 836514ef1..fb2910ed4 100644 --- a/web/src/services/api/resources.ts +++ b/web/src/services/api/resources.ts @@ -1,6 +1,7 @@ import { getActiveUserScope } from "@/lib/user-scope"; import axios from "axios"; import { apiBaseURL, apiClient, request, type BackendEnvelope } from "@/services/api/request"; +import type { OSSConnectionTestInput, OSSConnectionTestResult, OSSProvider, S3Preset } from "@/lib/oss-settings"; export type RemoteResource = { id: string; @@ -25,20 +26,30 @@ export type RemoteResource = { export type UserOSSSetting = { enabled: boolean; - provider: "aliyun" | "tencent" | "qiniu"; + provider: OSSProvider; + s3Preset: S3Preset; region: string; endpoint: string; cdnBaseUrl: string; bucket: string; accessKeyId: string; hasAccessKeySecret: boolean; + sessionToken?: string; + hasSessionToken: boolean; + pathStyle: boolean; + allowUserS3: boolean; publicBaseUrl: string; pathPrefix: string; + testedAt?: string; + testedDigest?: string; + historyCount?: number; + referencedResourceCount?: number; updatedAt?: string; }; -export type UserOSSSettingInput = Pick & { +export type UserOSSSettingInput = Pick & { accessKeySecret?: string; + sessionToken?: string; }; export type AccountFileStorageUsage = { @@ -68,6 +79,10 @@ export function updateUserOSSSetting(input: UserOSSSettingInput) { return request<{ setting: UserOSSSetting }>(api.patch("/settings/oss", input)); } +export function testUserOSSConnection(input: OSSConnectionTestInput) { + return request(api.post("/settings/oss/test", input)); +} + export async function getAccountFileStorageUsage() { const data = await request<{ usage: AccountFileStorageUsage }>(api.get("/resources/storage-usage")); return data.usage; diff --git a/web/test/oss-settings.test.ts b/web/test/oss-settings.test.ts new file mode 100644 index 000000000..e2127e2a9 --- /dev/null +++ b/web/test/oss-settings.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; + +import { changesRequireOSSRetest, DEFAULT_OSS_PATH_PREFIX, getS3PresetHints } from "../src/lib/oss-settings"; + +describe("OSS settings helpers", () => { + test("provides editable S3 endpoint hints for known presets", () => { + expect(getS3PresetHints("r2")).toMatchObject({ region: "auto" }); + expect(getS3PresetHints("b2").endpoint).toContain("backblazeb2.com"); + }); + + test("only connection fields invalidate a previous test", () => { + expect(changesRequireOSSRetest({ endpoint: "https://s3.example.com" })).toBe(true); + expect(changesRequireOSSRetest({ enabled: true })).toBe(false); + expect(changesRequireOSSRetest({ allowUserS3: true })).toBe(false); + }); + + test("uses the product path prefix by default", () => { + expect(DEFAULT_OSS_PATH_PREFIX).toBe("open-ai-canvas"); + }); +});