refactor(*): 数据同步与资源删除 - 收敛服务端真相并保证删除可恢复

backend/internal/service/resource_delete.go

@@ -89,7 +89,7 @@ func (s *Service) deleteUserAssetWithResources(userID string, assetID string) er
 		return BadAuthRequest(message)
 	}
 
-	// 所有引用校验必须先完成；任何资源仍被占用时，一个物理文件都不会开始删除。
+	// 所有引用校验必须先完成；仍被其他资源记录共享的物理对象不会进入删除队列。
 	physicalObjects := map[string]*model.Resource{}
 	for index := range resources {
 		resource := &resources[index]
@@ -102,23 +102,38 @@ func (s *Service) deleteUserAssetWithResources(userID string, assetID string) er
 		}
 		physicalObjects[resourceStorageIdentity(resource)] = resource
 	}
-	physicalKeys := make([]string, 0, len(physicalObjects))
-	for key := range physicalObjects {
-		physicalKeys = append(physicalKeys, key)
-	}
-	sort.Strings(physicalKeys)
-	for _, key := range physicalKeys {
-		resource := physicalObjects[key]
-		if err := s.deleteStoredResourceObject(userID, resource); err != nil {
-			return fmt.Errorf("素材文件删除失败，素材记录已保留：%w", err)
-		}
-	}
-	if err := s.repo.DeleteAssetAndResources(userID, assetID, ownedIDs); err != nil {
+	deletionJobs := resourceDeletionJobs(userID, physicalObjects)
+	// 业务记录和 Outbox 必须在同一事务提交。事务失败时物理文件完全不动；
+	// 提交成功后由幂等 worker 清理，进程退出或对象存储暂时失败都可继续重试。
+	if err := s.repo.DeleteAssetAndResources(userID, assetID, ownedIDs, deletionJobs); err != nil {
 		return fmt.Errorf("素材记录删除失败，请重试：%w", err)
 	}
+	if len(deletionJobs) > 0 {
+		go s.drainResourceDeletionJobs(len(deletionJobs))
+	}
 	return nil
 }
 
+func resourceDeletionJobs(userID string, physicalObjects map[string]*model.Resource) []model.ResourceDeletionJob {
+	keys := make([]string, 0, len(physicalObjects))
+	for key := range physicalObjects {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	now := time.Now()
+	jobs := make([]model.ResourceDeletionJob, 0, len(keys))
+	for _, key := range keys {
+		resource := physicalObjects[key]
+		jobs = append(jobs, model.ResourceDeletionJob{
+			ID: newID(), UserID: userID, ResourceID: resource.ID,
+			Provider: resource.Provider, Endpoint: resource.Endpoint, Bucket: resource.Bucket,
+			StorageSettingID: resource.StorageSettingID, ObjectKey: resource.ObjectKey,
+			Status: model.ResourceDeletionStatusPending, NextAttemptAt: now,
+		})
+	}
+	return jobs
+}
+
 type resourceUsage struct {
 	Kind  string
 	ID    string
@@ -161,6 +176,14 @@ func collectOwnedAssetDocumentReferences(raw string, resourceIDs map[string]stru
 	if err := json.Unmarshal([]byte(raw), &value); err != nil {
 		return err
 	}
+	// A scalar JSON document is an explicit URL/storage-key field. Nested scalar
+	// values must carry a registered field name; free text in arrays is not a reference.
+	if scalar, ok := value.(string); ok {
+		if resourceID := canvasResourceID(scalar); resourceID != "" {
+			resourceIDs[resourceID] = struct{}{}
+		}
+		return nil
+	}
 	walkReferenceDocument(value, "", resourceIDs)
 	return nil
 }
@@ -176,13 +199,12 @@ func walkReferenceDocument(value any, parentKey string, resourceIDs map[string]s
 			walkReferenceDocument(child, parentKey, resourceIDs)
 		}
 	case string:
-		key := normalizeReferenceKey(parentKey)
-		if isResourceLocatorKey(key) {
+		if isResourceLocatorField(parentKey) {
 			if resourceID := canvasResourceID(item); resourceID != "" {
 				resourceIDs[resourceID] = struct{}{}
 			}
 		}
-		if isBareResourceIDKey(key) {
+		if isBareResourceIDField(parentKey) {
 			if resourceID := validCanvasResourceID(item); resourceID != "" {
 				resourceIDs[resourceID] = struct{}{}
 			}
@@ -190,21 +212,20 @@ func walkReferenceDocument(value any, parentKey string, resourceIDs map[string]s
 	}
 }
 
-func normalizeReferenceKey(key string) string {
-	key = strings.ToLower(strings.TrimSpace(key))
-	return strings.NewReplacer("_", "", "-", "").Replace(key)
-}
-
-func isBareResourceIDKey(key string) bool {
-	return key == "resourceid" || key == "resourceids" || key == "sampleresourceid" || key == "referenceresourceid" || key == "referenceresourceids"
-}
-
-func isResourceLocatorKey(key string) bool {
-	if key == "" || strings.HasSuffix(key, "url") || strings.HasSuffix(key, "urls") || strings.HasSuffix(key, "storagekey") {
+func isBareResourceIDField(field string) bool {
+	switch field {
+	case "resourceId", "resourceIds", "sampleResourceId", "referenceResourceId", "referenceResourceIds":
 		return true
+	default:
+		return false
 	}
-	switch key {
-	case "content", "data", "image", "images", "video", "videos", "audio", "audios", "media", "references", "result", "output", "artifactref", "providerartifactref":
+}
+
+// Resource locator fields are a schema contract, not a naming heuristic.
+// Adding or renaming a persisted field requires updating this registry and its tests.
+func isResourceLocatorField(field string) bool {
+	switch field {
+	case "storageKey", "content", "url", "dataUrl", "coverUrl", "imageUrl", "videoUrl", "audioUrl", "referenceUrl", "referenceUrls", "artifactRef", "providerArtifactRef":
 		return true
 	default:
 		return false
@@ -219,7 +240,13 @@ func documentReferencesResources(raw string, resourceIDs map[string]struct{}) bo
 	var value any
 	if err := json.Unmarshal([]byte(raw), &value); err == nil {
 		found := map[string]struct{}{}
-		walkReferenceDocument(value, "", found)
+		if scalar, ok := value.(string); ok {
+			if resourceID := canvasResourceID(scalar); resourceID != "" {
+				found[resourceID] = struct{}{}
+			}
+		} else {
+			walkReferenceDocument(value, "", found)
+		}
 		for resourceID := range found {
 			if _, exists := resourceIDs[resourceID]; exists {
 				return true
@@ -227,11 +254,10 @@ func documentReferencesResources(raw string, resourceIDs map[string]struct{}) bo
 		}
 		return false
 	}
-	// cover_url 等字段可以直接保存资源 URL，而不是 JSON。候选记录已经包含完整 ID，保守阻止删除。
-	for resourceID := range resourceIDs {
-		if strings.Contains(raw, resourceID) {
-			return true
-		}
+	// cover_url 等数据库列可以直接保存一个资源 URL，而不是 JSON。
+	if resourceID := canvasResourceID(raw); resourceID != "" {
+		_, exists := resourceIDs[resourceID]
+		return exists
 	}
 	return false
 }