feat(assets): 素材回收站与画布历史 - 合并 PR #370 并修复删除一致性
This commit is contained in:
@@ -1072,6 +1072,18 @@ func (r *Repository) DeleteAsset(userID string, id string) error {
|
||||
return r.DeleteAssetAndResources(userID, id, nil, nil)
|
||||
}
|
||||
|
||||
func (r *Repository) FindExpiredArchivedAssets(cutoff time.Time, limit int) ([]model.Asset, error) {
|
||||
var assets []model.Asset
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
err := r.db.Where("status = ? AND updated_at <= ?", model.AssetVersionStatusArchived, cutoff).
|
||||
Order("updated_at asc, id asc").
|
||||
Limit(limit).
|
||||
Find(&assets).Error
|
||||
return assets, err
|
||||
}
|
||||
|
||||
func (r *Repository) ReplaceAssets(userID string, assets []model.Asset) error {
|
||||
return r.db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Delete(&model.Asset{}, "user_id = ?", userID).Error; err != nil {
|
||||
|
||||
@@ -257,6 +257,61 @@ func TestResourceDeletionWorkerRemovesObjectAndCompletesOutbox(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiredArchivedAssetCleanupRespectsCanvasReferencesAndUsesDeletionOutbox(t *testing.T) {
|
||||
svc, db, _ := newResourceDeletionTestService(t)
|
||||
old := time.Now().Add(-45 * 24 * time.Hour)
|
||||
resource := model.Resource{
|
||||
ID: "resource-expired-archive", UserID: "user-1", Provider: "unsupported-test-provider",
|
||||
ObjectKey: "users/user-1/image/expired.png", Status: model.ResourceStatusReady,
|
||||
}
|
||||
asset := model.Asset{
|
||||
ID: "asset-expired-archive", UserID: "user-1", Title: "过期回收站素材",
|
||||
Status: model.AssetVersionStatusArchived, PayloadJSON: `{"data":{"storageKey":"resource:resource-expired-archive"}}`,
|
||||
CreatedAt: old, UpdatedAt: old,
|
||||
}
|
||||
canvas := model.CanvasProject{
|
||||
ID: "canvas-expired-archive", UserID: "user-1", Title: "仍引用回收站素材的画布",
|
||||
PayloadJSON: `{"nodes":[{"data":{"storageKey":"resource:resource-expired-archive"}}]}`,
|
||||
}
|
||||
for _, item := range []any{&resource, &asset, &canvas} {
|
||||
if err := db.Create(item).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
svc.cleanupExpiredArchivedAssets()
|
||||
var assetCount, resourceCount, jobCount int64
|
||||
if err := db.Model(&model.Asset{}).Where("id = ?", asset.ID).Count(&assetCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Model(&model.Resource{}).Where("id = ?", resource.ID).Count(&resourceCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Model(&model.ResourceDeletionJob{}).Where("resource_id = ?", resource.ID).Count(&jobCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if assetCount != 1 || resourceCount != 1 || jobCount != 0 {
|
||||
t.Fatalf("referenced archived asset cleanup changed data: asset=%d resource=%d jobs=%d", assetCount, resourceCount, jobCount)
|
||||
}
|
||||
|
||||
if err := db.Delete(&model.CanvasProject{}, "id = ? AND user_id = ?", canvas.ID, canvas.UserID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc.cleanupExpiredArchivedAssets()
|
||||
if err := db.Model(&model.Asset{}).Where("id = ?", asset.ID).Count(&assetCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Model(&model.Resource{}).Where("id = ?", resource.ID).Count(&resourceCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Model(&model.ResourceDeletionJob{}).Where("resource_id = ?", resource.ID).Count(&jobCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if assetCount != 0 || resourceCount != 0 || jobCount != 1 {
|
||||
t.Fatalf("unreferenced archived asset did not use deletion outbox: asset=%d resource=%d jobs=%d", assetCount, resourceCount, jobCount)
|
||||
}
|
||||
}
|
||||
|
||||
func newResourceDeletionTestService(t *testing.T) (*Service, *gorm.DB, string) {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open("file:"+newID()+"?mode=memory&cache=shared"), &gorm.Config{})
|
||||
|
||||
@@ -15,24 +15,56 @@ func (s *Service) startResourceDeletionWorker(ctx context.Context) {
|
||||
s.runWorkerLoop(func(ctx context.Context) {
|
||||
s.drainResourceDeletionJobs(32)
|
||||
s.cleanupStaleAnnouncementImageDrafts()
|
||||
s.cleanupExpiredArchivedAssets()
|
||||
ticker := time.NewTicker(15 * time.Second)
|
||||
defer ticker.Stop()
|
||||
lastDraftCleanup := time.Now()
|
||||
lastPeriodicCleanup := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.drainResourceDeletionJobs(32)
|
||||
if time.Since(lastDraftCleanup) >= time.Hour {
|
||||
if time.Since(lastPeriodicCleanup) >= time.Hour {
|
||||
s.cleanupStaleAnnouncementImageDrafts()
|
||||
lastDraftCleanup = time.Now()
|
||||
s.cleanupExpiredArchivedAssets()
|
||||
lastPeriodicCleanup = time.Now()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) cleanupExpiredArchivedAssets() {
|
||||
policy, err := s.RuntimePolicy()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
retentionDays := policy.Resource.RecycleBinRetentionDays
|
||||
if retentionDays <= 0 {
|
||||
return
|
||||
}
|
||||
cutoff := time.Now().Add(-time.Duration(retentionDays) * 24 * time.Hour)
|
||||
expired, err := s.repo.FindExpiredArchivedAssets(cutoff, 100)
|
||||
if err != nil {
|
||||
log.Printf("expired archived assets query failed: %v", err)
|
||||
return
|
||||
}
|
||||
deleted := 0
|
||||
for _, asset := range expired {
|
||||
// 自动清理与用户手动删除走同一条强校验路径:先检查业务引用,
|
||||
// 再以事务 + Outbox 删除资源记录和物理对象,不能从 repository 旁路。
|
||||
if err := s.DeleteUserAsset(asset.UserID, asset.ID); err != nil {
|
||||
log.Printf("expired archived asset delete failed for %s: %v", asset.ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("recycle bin cleanup: deleted %d expired assets (retention: %d days)", deleted, retentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) drainResourceDeletionJobs(limit int) {
|
||||
owner := s.workerID
|
||||
if owner == "" {
|
||||
|
||||
@@ -25,18 +25,19 @@ const (
|
||||
)
|
||||
|
||||
type RuntimeResourcePolicy struct {
|
||||
ResourceUploadMB int64 `json:"resourceUploadMB"`
|
||||
SessionUploadMB int64 `json:"sessionUploadMB"`
|
||||
GeneratedFileMB int64 `json:"generatedFileMB"`
|
||||
DailyUploadMB int64 `json:"dailyUploadMB"`
|
||||
StoredFileGB int64 `json:"storedFileGB"`
|
||||
StructuredDataMB int64 `json:"structuredDataMB"`
|
||||
TaskDataGB int64 `json:"taskDataGB"`
|
||||
AssetCount int64 `json:"assetCount"`
|
||||
CanvasCount int64 `json:"canvasCount"`
|
||||
SessionCount int64 `json:"sessionCount"`
|
||||
TaskCount int64 `json:"taskCount"`
|
||||
APICallLogCount int64 `json:"apiCallLogCount"`
|
||||
ResourceUploadMB int64 `json:"resourceUploadMB"`
|
||||
SessionUploadMB int64 `json:"sessionUploadMB"`
|
||||
GeneratedFileMB int64 `json:"generatedFileMB"`
|
||||
DailyUploadMB int64 `json:"dailyUploadMB"`
|
||||
StoredFileGB int64 `json:"storedFileGB"`
|
||||
StructuredDataMB int64 `json:"structuredDataMB"`
|
||||
TaskDataGB int64 `json:"taskDataGB"`
|
||||
AssetCount int64 `json:"assetCount"`
|
||||
CanvasCount int64 `json:"canvasCount"`
|
||||
SessionCount int64 `json:"sessionCount"`
|
||||
TaskCount int64 `json:"taskCount"`
|
||||
APICallLogCount int64 `json:"apiCallLogCount"`
|
||||
RecycleBinRetentionDays int `json:"recycleBinRetentionDays"`
|
||||
}
|
||||
|
||||
type RuntimeTaskPolicy struct {
|
||||
@@ -90,26 +91,28 @@ type PublicRuntimePolicySetting struct {
|
||||
}
|
||||
|
||||
type PublicRuntimeLimits struct {
|
||||
ActiveTaskLimit int `json:"activeTaskLimit"`
|
||||
ResourceUploadMB int64 `json:"resourceUploadMB"`
|
||||
SessionUploadMB int64 `json:"sessionUploadMB"`
|
||||
ActiveTaskLimit int `json:"activeTaskLimit"`
|
||||
ResourceUploadMB int64 `json:"resourceUploadMB"`
|
||||
SessionUploadMB int64 `json:"sessionUploadMB"`
|
||||
RecycleBinRetentionDays int `json:"recycleBinRetentionDays"`
|
||||
}
|
||||
|
||||
func defaultRuntimePolicy() RuntimePolicySetting {
|
||||
return RuntimePolicySetting{
|
||||
Resource: RuntimeResourcePolicy{
|
||||
ResourceUploadMB: 50,
|
||||
SessionUploadMB: 32,
|
||||
GeneratedFileMB: 64,
|
||||
DailyUploadMB: 200,
|
||||
StoredFileGB: 2,
|
||||
StructuredDataMB: 256,
|
||||
TaskDataGB: 1,
|
||||
AssetCount: 2_000,
|
||||
CanvasCount: 1_000,
|
||||
SessionCount: 1_000,
|
||||
TaskCount: 20_000,
|
||||
APICallLogCount: 100_000,
|
||||
ResourceUploadMB: 50,
|
||||
SessionUploadMB: 32,
|
||||
GeneratedFileMB: 64,
|
||||
DailyUploadMB: 200,
|
||||
StoredFileGB: 2,
|
||||
StructuredDataMB: 256,
|
||||
TaskDataGB: 1,
|
||||
AssetCount: 2_000,
|
||||
CanvasCount: 1_000,
|
||||
SessionCount: 1_000,
|
||||
TaskCount: 20_000,
|
||||
APICallLogCount: 100_000,
|
||||
RecycleBinRetentionDays: 30,
|
||||
},
|
||||
Task: RuntimeTaskPolicy{
|
||||
WorkerConcurrency: effectiveChannelConcurrencyLimit(envInt("CANVAS_WORKER_CONCURRENCY", taskWorkerConcurrency)),
|
||||
@@ -155,6 +158,7 @@ func selfUseRuntimePolicy() RuntimePolicySetting {
|
||||
DailyUploadMB: maxRuntimeDataMB, StoredFileGB: maxRuntimeStorageGB, StructuredDataMB: maxRuntimeDataMB,
|
||||
TaskDataGB: maxRuntimeStorageGB, AssetCount: maxRuntimeCount, CanvasCount: maxRuntimeCount,
|
||||
SessionCount: maxRuntimeCount, TaskCount: maxRuntimeCount, APICallLogCount: maxRuntimeCount,
|
||||
RecycleBinRetentionDays: 0,
|
||||
}
|
||||
value.Task = RuntimeTaskPolicy{
|
||||
WorkerConcurrency: maxRuntimeConcurrency, ChannelConcurrency: maxRuntimeConcurrency, ActiveTaskLimit: maxRuntimeConcurrency,
|
||||
@@ -194,7 +198,8 @@ func (s *Service) PublicRuntimeLimits() (*PublicRuntimeLimits, error) {
|
||||
}
|
||||
return &PublicRuntimeLimits{
|
||||
ActiveTaskLimit: policy.Task.ActiveTaskLimit, ResourceUploadMB: policy.Resource.ResourceUploadMB,
|
||||
SessionUploadMB: policy.Resource.SessionUploadMB,
|
||||
SessionUploadMB: policy.Resource.SessionUploadMB,
|
||||
RecycleBinRetentionDays: policy.Resource.RecycleBinRetentionDays,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -271,7 +276,7 @@ func (s *Service) readRuntimePolicy() (*model.SystemSetting, RuntimePolicySettin
|
||||
if err != nil {
|
||||
return nil, RuntimePolicySetting{}, err
|
||||
}
|
||||
value := RuntimePolicySetting{}
|
||||
value := defaultRuntimePolicy()
|
||||
if strings.TrimSpace(setting.ValueJSON) == "" || json.Unmarshal([]byte(setting.ValueJSON), &value) != nil {
|
||||
return nil, RuntimePolicySetting{}, errors.New("资源与请求策略配置格式无效")
|
||||
}
|
||||
@@ -312,6 +317,9 @@ func validateRuntimePolicy(value RuntimePolicySetting) error {
|
||||
return BadAuthRequest(fmt.Sprintf("%s必须是 1-%d 的整数", label, maxRuntimeCount))
|
||||
}
|
||||
}
|
||||
if resource.RecycleBinRetentionDays < 0 || resource.RecycleBinRetentionDays > 365 {
|
||||
return BadAuthRequest("回收站保留天数必须是 0-365 的整数 (0 表示不自动清理)")
|
||||
}
|
||||
task := value.Task
|
||||
for label, item := range map[string]int{
|
||||
"Worker 并发数": task.WorkerConcurrency, "全局渠道并发数": task.ChannelConcurrency, "活动任务上限": task.ActiveTaskLimit,
|
||||
|
||||
@@ -136,3 +136,39 @@ func TestRuntimePolicySaveAndResetTakeEffectImmediately(t *testing.T) {
|
||||
t.Fatalf("reset active task limit = %d, error = %v", effective.Task.ActiveTaskLimit, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimePolicyBackfillsRecycleBinRetentionForLegacyJSON(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.SystemSetting{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy := defaultRuntimePolicy()
|
||||
encoded, err := json.Marshal(legacy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value map[string]any
|
||||
if err := json.Unmarshal(encoded, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resource := value["resource"].(map[string]any)
|
||||
delete(resource, "recycleBinRetentionDays")
|
||||
encoded, err = json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.SystemSetting{Key: runtimePolicySettingKey, ValueJSON: string(encoded), UpdatedBy: "admin"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
policy, err := New(repository.New(db), t.TempDir()).RuntimePolicy()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if policy.Resource.RecycleBinRetentionDays != 30 {
|
||||
t.Fatalf("legacy recycle bin retention = %d, want 30", policy.Resource.RecycleBinRetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user