feat(assets): 素材回收站与画布历史 - 合并 PR #370 并修复删除一致性

This commit is contained in:
ddCat
2026-09-01 18:50:30 +08:00
21 changed files with 1766 additions and 342 deletions

View File

@@ -1072,6 +1072,18 @@ func (r *Repository) DeleteAsset(userID string, id string) error {
return r.DeleteAssetAndResources(userID, id, nil, nil)
}
func (r *Repository) FindExpiredArchivedAssets(cutoff time.Time, limit int) ([]model.Asset, error) {
var assets []model.Asset
if limit <= 0 {
limit = 100
}
err := r.db.Where("status = ? AND updated_at <= ?", model.AssetVersionStatusArchived, cutoff).
Order("updated_at asc, id asc").
Limit(limit).
Find(&assets).Error
return assets, err
}
func (r *Repository) ReplaceAssets(userID string, assets []model.Asset) error {
return r.db.Transaction(func(tx *gorm.DB) error {
if err := tx.Delete(&model.Asset{}, "user_id = ?", userID).Error; err != nil {

View File

@@ -257,6 +257,61 @@ func TestResourceDeletionWorkerRemovesObjectAndCompletesOutbox(t *testing.T) {
}
}
func TestExpiredArchivedAssetCleanupRespectsCanvasReferencesAndUsesDeletionOutbox(t *testing.T) {
svc, db, _ := newResourceDeletionTestService(t)
old := time.Now().Add(-45 * 24 * time.Hour)
resource := model.Resource{
ID: "resource-expired-archive", UserID: "user-1", Provider: "unsupported-test-provider",
ObjectKey: "users/user-1/image/expired.png", Status: model.ResourceStatusReady,
}
asset := model.Asset{
ID: "asset-expired-archive", UserID: "user-1", Title: "过期回收站素材",
Status: model.AssetVersionStatusArchived, PayloadJSON: `{"data":{"storageKey":"resource:resource-expired-archive"}}`,
CreatedAt: old, UpdatedAt: old,
}
canvas := model.CanvasProject{
ID: "canvas-expired-archive", UserID: "user-1", Title: "仍引用回收站素材的画布",
PayloadJSON: `{"nodes":[{"data":{"storageKey":"resource:resource-expired-archive"}}]}`,
}
for _, item := range []any{&resource, &asset, &canvas} {
if err := db.Create(item).Error; err != nil {
t.Fatal(err)
}
}
svc.cleanupExpiredArchivedAssets()
var assetCount, resourceCount, jobCount int64
if err := db.Model(&model.Asset{}).Where("id = ?", asset.ID).Count(&assetCount).Error; err != nil {
t.Fatal(err)
}
if err := db.Model(&model.Resource{}).Where("id = ?", resource.ID).Count(&resourceCount).Error; err != nil {
t.Fatal(err)
}
if err := db.Model(&model.ResourceDeletionJob{}).Where("resource_id = ?", resource.ID).Count(&jobCount).Error; err != nil {
t.Fatal(err)
}
if assetCount != 1 || resourceCount != 1 || jobCount != 0 {
t.Fatalf("referenced archived asset cleanup changed data: asset=%d resource=%d jobs=%d", assetCount, resourceCount, jobCount)
}
if err := db.Delete(&model.CanvasProject{}, "id = ? AND user_id = ?", canvas.ID, canvas.UserID).Error; err != nil {
t.Fatal(err)
}
svc.cleanupExpiredArchivedAssets()
if err := db.Model(&model.Asset{}).Where("id = ?", asset.ID).Count(&assetCount).Error; err != nil {
t.Fatal(err)
}
if err := db.Model(&model.Resource{}).Where("id = ?", resource.ID).Count(&resourceCount).Error; err != nil {
t.Fatal(err)
}
if err := db.Model(&model.ResourceDeletionJob{}).Where("resource_id = ?", resource.ID).Count(&jobCount).Error; err != nil {
t.Fatal(err)
}
if assetCount != 0 || resourceCount != 0 || jobCount != 1 {
t.Fatalf("unreferenced archived asset did not use deletion outbox: asset=%d resource=%d jobs=%d", assetCount, resourceCount, jobCount)
}
}
func newResourceDeletionTestService(t *testing.T) (*Service, *gorm.DB, string) {
t.Helper()
db, err := gorm.Open(sqlite.Open("file:"+newID()+"?mode=memory&cache=shared"), &gorm.Config{})

View File

@@ -15,24 +15,56 @@ func (s *Service) startResourceDeletionWorker(ctx context.Context) {
s.runWorkerLoop(func(ctx context.Context) {
s.drainResourceDeletionJobs(32)
s.cleanupStaleAnnouncementImageDrafts()
s.cleanupExpiredArchivedAssets()
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
lastDraftCleanup := time.Now()
lastPeriodicCleanup := time.Now()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.drainResourceDeletionJobs(32)
if time.Since(lastDraftCleanup) >= time.Hour {
if time.Since(lastPeriodicCleanup) >= time.Hour {
s.cleanupStaleAnnouncementImageDrafts()
lastDraftCleanup = time.Now()
s.cleanupExpiredArchivedAssets()
lastPeriodicCleanup = time.Now()
}
}
}
})
}
func (s *Service) cleanupExpiredArchivedAssets() {
policy, err := s.RuntimePolicy()
if err != nil {
return
}
retentionDays := policy.Resource.RecycleBinRetentionDays
if retentionDays <= 0 {
return
}
cutoff := time.Now().Add(-time.Duration(retentionDays) * 24 * time.Hour)
expired, err := s.repo.FindExpiredArchivedAssets(cutoff, 100)
if err != nil {
log.Printf("expired archived assets query failed: %v", err)
return
}
deleted := 0
for _, asset := range expired {
// 自动清理与用户手动删除走同一条强校验路径:先检查业务引用,
// 再以事务 + Outbox 删除资源记录和物理对象,不能从 repository 旁路。
if err := s.DeleteUserAsset(asset.UserID, asset.ID); err != nil {
log.Printf("expired archived asset delete failed for %s: %v", asset.ID, err)
continue
}
deleted++
}
if deleted > 0 {
log.Printf("recycle bin cleanup: deleted %d expired assets (retention: %d days)", deleted, retentionDays)
}
}
func (s *Service) drainResourceDeletionJobs(limit int) {
owner := s.workerID
if owner == "" {

View File

@@ -25,18 +25,19 @@ const (
)
type RuntimeResourcePolicy struct {
ResourceUploadMB int64 `json:"resourceUploadMB"`
SessionUploadMB int64 `json:"sessionUploadMB"`
GeneratedFileMB int64 `json:"generatedFileMB"`
DailyUploadMB int64 `json:"dailyUploadMB"`
StoredFileGB int64 `json:"storedFileGB"`
StructuredDataMB int64 `json:"structuredDataMB"`
TaskDataGB int64 `json:"taskDataGB"`
AssetCount int64 `json:"assetCount"`
CanvasCount int64 `json:"canvasCount"`
SessionCount int64 `json:"sessionCount"`
TaskCount int64 `json:"taskCount"`
APICallLogCount int64 `json:"apiCallLogCount"`
ResourceUploadMB int64 `json:"resourceUploadMB"`
SessionUploadMB int64 `json:"sessionUploadMB"`
GeneratedFileMB int64 `json:"generatedFileMB"`
DailyUploadMB int64 `json:"dailyUploadMB"`
StoredFileGB int64 `json:"storedFileGB"`
StructuredDataMB int64 `json:"structuredDataMB"`
TaskDataGB int64 `json:"taskDataGB"`
AssetCount int64 `json:"assetCount"`
CanvasCount int64 `json:"canvasCount"`
SessionCount int64 `json:"sessionCount"`
TaskCount int64 `json:"taskCount"`
APICallLogCount int64 `json:"apiCallLogCount"`
RecycleBinRetentionDays int `json:"recycleBinRetentionDays"`
}
type RuntimeTaskPolicy struct {
@@ -90,26 +91,28 @@ type PublicRuntimePolicySetting struct {
}
type PublicRuntimeLimits struct {
ActiveTaskLimit int `json:"activeTaskLimit"`
ResourceUploadMB int64 `json:"resourceUploadMB"`
SessionUploadMB int64 `json:"sessionUploadMB"`
ActiveTaskLimit int `json:"activeTaskLimit"`
ResourceUploadMB int64 `json:"resourceUploadMB"`
SessionUploadMB int64 `json:"sessionUploadMB"`
RecycleBinRetentionDays int `json:"recycleBinRetentionDays"`
}
func defaultRuntimePolicy() RuntimePolicySetting {
return RuntimePolicySetting{
Resource: RuntimeResourcePolicy{
ResourceUploadMB: 50,
SessionUploadMB: 32,
GeneratedFileMB: 64,
DailyUploadMB: 200,
StoredFileGB: 2,
StructuredDataMB: 256,
TaskDataGB: 1,
AssetCount: 2_000,
CanvasCount: 1_000,
SessionCount: 1_000,
TaskCount: 20_000,
APICallLogCount: 100_000,
ResourceUploadMB: 50,
SessionUploadMB: 32,
GeneratedFileMB: 64,
DailyUploadMB: 200,
StoredFileGB: 2,
StructuredDataMB: 256,
TaskDataGB: 1,
AssetCount: 2_000,
CanvasCount: 1_000,
SessionCount: 1_000,
TaskCount: 20_000,
APICallLogCount: 100_000,
RecycleBinRetentionDays: 30,
},
Task: RuntimeTaskPolicy{
WorkerConcurrency: effectiveChannelConcurrencyLimit(envInt("CANVAS_WORKER_CONCURRENCY", taskWorkerConcurrency)),
@@ -155,6 +158,7 @@ func selfUseRuntimePolicy() RuntimePolicySetting {
DailyUploadMB: maxRuntimeDataMB, StoredFileGB: maxRuntimeStorageGB, StructuredDataMB: maxRuntimeDataMB,
TaskDataGB: maxRuntimeStorageGB, AssetCount: maxRuntimeCount, CanvasCount: maxRuntimeCount,
SessionCount: maxRuntimeCount, TaskCount: maxRuntimeCount, APICallLogCount: maxRuntimeCount,
RecycleBinRetentionDays: 0,
}
value.Task = RuntimeTaskPolicy{
WorkerConcurrency: maxRuntimeConcurrency, ChannelConcurrency: maxRuntimeConcurrency, ActiveTaskLimit: maxRuntimeConcurrency,
@@ -194,7 +198,8 @@ func (s *Service) PublicRuntimeLimits() (*PublicRuntimeLimits, error) {
}
return &PublicRuntimeLimits{
ActiveTaskLimit: policy.Task.ActiveTaskLimit, ResourceUploadMB: policy.Resource.ResourceUploadMB,
SessionUploadMB: policy.Resource.SessionUploadMB,
SessionUploadMB: policy.Resource.SessionUploadMB,
RecycleBinRetentionDays: policy.Resource.RecycleBinRetentionDays,
}, nil
}
@@ -271,7 +276,7 @@ func (s *Service) readRuntimePolicy() (*model.SystemSetting, RuntimePolicySettin
if err != nil {
return nil, RuntimePolicySetting{}, err
}
value := RuntimePolicySetting{}
value := defaultRuntimePolicy()
if strings.TrimSpace(setting.ValueJSON) == "" || json.Unmarshal([]byte(setting.ValueJSON), &value) != nil {
return nil, RuntimePolicySetting{}, errors.New("资源与请求策略配置格式无效")
}
@@ -312,6 +317,9 @@ func validateRuntimePolicy(value RuntimePolicySetting) error {
return BadAuthRequest(fmt.Sprintf("%s必须是 1-%d 的整数", label, maxRuntimeCount))
}
}
if resource.RecycleBinRetentionDays < 0 || resource.RecycleBinRetentionDays > 365 {
return BadAuthRequest("回收站保留天数必须是 0-365 的整数 (0 表示不自动清理)")
}
task := value.Task
for label, item := range map[string]int{
"Worker 并发数": task.WorkerConcurrency, "全局渠道并发数": task.ChannelConcurrency, "活动任务上限": task.ActiveTaskLimit,

View File

@@ -136,3 +136,39 @@ func TestRuntimePolicySaveAndResetTakeEffectImmediately(t *testing.T) {
t.Fatalf("reset active task limit = %d, error = %v", effective.Task.ActiveTaskLimit, err)
}
}
func TestRuntimePolicyBackfillsRecycleBinRetentionForLegacyJSON(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.SystemSetting{}); err != nil {
t.Fatal(err)
}
legacy := defaultRuntimePolicy()
encoded, err := json.Marshal(legacy)
if err != nil {
t.Fatal(err)
}
var value map[string]any
if err := json.Unmarshal(encoded, &value); err != nil {
t.Fatal(err)
}
resource := value["resource"].(map[string]any)
delete(resource, "recycleBinRetentionDays")
encoded, err = json.Marshal(value)
if err != nil {
t.Fatal(err)
}
if err := db.Create(&model.SystemSetting{Key: runtimePolicySettingKey, ValueJSON: string(encoded), UpdatedBy: "admin"}).Error; err != nil {
t.Fatal(err)
}
policy, err := New(repository.New(db), t.TempDir()).RuntimePolicy()
if err != nil {
t.Fatal(err)
}
if policy.Resource.RecycleBinRetentionDays != 30 {
t.Fatalf("legacy recycle bin retention = %d, want 30", policy.Resource.RecycleBinRetentionDays)
}
}