fix(canvas): 素材库 - 拒绝空字节数并提供历史数据修复工具

2026-09-14 23:26:32 +08:00
parent 8714c32869
commit f81c05abdb
7 changed files with 404 additions and 1 deletions
--- a/backend/cmd/migrate-schema/main.go
+++ b/backend/cmd/migrate-schema/main.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"

+	"infinite-canvas/backend/internal/canvas"
 	"infinite-canvas/backend/internal/database"
 )

@@ -15,6 +16,14 @@ func main() {
 	if len(os.Args) > 1 {
 		command = strings.ToLower(strings.TrimSpace(os.Args[1]))
 	}
+	apply := false
+	if command == "repair-asset-bytes" {
+		if len(os.Args) == 3 && os.Args[2] == "--apply" {
+			apply = true
+		} else if len(os.Args) > 2 {
+			log.Fatal("用法：migrate-schema repair-asset-bytes [--apply]；默认只检查")
+		}
+	}
 	db, err := database.Open(database.Config{
 		Driver:  env("CANVAS_DATABASE_DRIVER", "sqlite"),
 		DSN:     os.Getenv("DATABASE_URL"),
@@ -33,6 +42,20 @@ func main() {
 	defer sqlDB.Close()

 	switch command {
+	case "repair-asset-bytes":
+		report, repairErr := canvas.RepairAssetBytes(db, apply)
+		encoded, err := json.Marshal(report)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(string(encoded))
+		if repairErr != nil {
+			log.Fatal(repairErr)
+		}
+		if len(report.Unresolved) > 0 {
+			log.Fatal("存在未修复素材，请按报告核查；未删除或填零")
+		}
+		return
 	case "up":
 		if err := database.MigrateSchema(db); err != nil {
 			log.Fatal(err)
@@ -43,7 +66,7 @@ func main() {
 			log.Fatal(err)
 		}
 	default:
-		log.Fatalf("未知命令 %q；可用命令：up、status、verify", command)
+		log.Fatalf("未知命令 %q；可用命令：up、status、verify、repair-asset-bytes", command)
 	}
 	status, err := database.ReadSchemaStatus(db)
 	if err != nil {
--- a/backend/internal/canvas/repair_asset_bytes.go
+++ b/backend/internal/canvas/repair_asset_bytes.go
@@ -0,0 +1,116 @@
+package canvas
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"infinite-canvas/backend/internal/model"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+type AssetBytesRepairIssue struct {
+	AssetID string `json:"assetId"`
+	Reason  string `json:"reason"`
+}
+
+type AssetBytesRepairReport struct {
+	Apply      bool                    `json:"apply"`
+	Scanned    int                     `json:"scanned"`
+	Repairable int                     `json:"repairable"`
+	Repaired   int                     `json:"repaired"`
+	Unresolved []AssetBytesRepairIssue `json:"unresolved"`
+}
+
+// RepairAssetBytes is an explicit offline maintenance operation, not a read fallback.
+// Only an owned, ready resource with a known positive size can repair a record.
+func RepairAssetBytes(db *gorm.DB, apply bool) (AssetBytesRepairReport, error) {
+	report := AssetBytesRepairReport{Apply: apply, Unresolved: []AssetBytesRepairIssue{}}
+	// Payloads may contain private media URLs; never include SQL parameters in logs.
+	err := db.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}).Transaction(func(tx *gorm.DB) error {
+		after := ""
+		for {
+			var rows []model.Asset
+			if err := tx.Where("id > ? AND kind IN ?", after, []string{"image", "video", "audio", "model"}).Order("id ASC").Limit(100).Find(&rows).Error; err != nil {
+				return errors.New("读取素材失败，未提交修复")
+			}
+			if len(rows) == 0 {
+				return nil
+			}
+			for _, asset := range rows {
+				after = asset.ID
+				report.Scanned++
+				issue := func(reason string) {
+					report.Unresolved = append(report.Unresolved, AssetBytesRepairIssue{AssetID: asset.ID, Reason: reason})
+				}
+				var payload map[string]json.RawMessage
+				var data map[string]json.RawMessage
+				if json.Unmarshal([]byte(asset.PayloadJSON), &payload) != nil || json.Unmarshal(payload["data"], &data) != nil || data == nil {
+					issue("素材 JSON 或 data 无效，需人工核查")
+					continue
+				}
+				if _, err := requiredJSONNumberField(data, "bytes"); err == nil {
+					continue
+				}
+				var kind, key string
+				_ = json.Unmarshal(payload["kind"], &kind)
+				_ = json.Unmarshal(data["storageKey"], &key)
+				key = strings.TrimSpace(key)
+				if strings.TrimSpace(asset.UserID) == "" || kind != asset.Kind || !strings.HasPrefix(key, "resource:") || strings.TrimPrefix(key, "resource:") == "" {
+					issue("类型不一致或缺少明确的 resource: 引用，不能推测文件大小")
+					continue
+				}
+				var resource model.Resource
+				err := tx.Where("id = ? AND user_id = ? AND kind = ? AND status = ? AND size > 0", strings.TrimPrefix(key, "resource:"), asset.UserID, kind, model.ResourceStatusReady).Take(&resource).Error
+				if errors.Is(err, gorm.ErrRecordNotFound) {
+					issue("未找到同用户、同类型、就绪且大小已知的资源")
+					continue
+				}
+				if err != nil {
+					return errors.New("读取资源失败，未提交修复")
+				}
+				data["bytes"], err = json.Marshal(resource.Size)
+				if err != nil {
+					return err
+				}
+				payload["data"], err = json.Marshal(data)
+				if err != nil {
+					return err
+				}
+				now := time.Now().UTC()
+				payload["updatedAt"], err = json.Marshal(now.Format(time.RFC3339Nano))
+				if err != nil {
+					return err
+				}
+				encoded, err := json.Marshal(payload)
+				if err != nil {
+					return err
+				}
+				if err := validateUserAssetDocument(encoded); err != nil {
+					issue("素材仍有其他合同错误，未修改，请另行核查")
+					continue
+				}
+				report.Repairable++
+				if !apply {
+					continue
+				}
+				result := tx.Model(&model.Asset{}).Where("id = ? AND user_id = ? AND payload_json = ? AND updated_at = ?", asset.ID, asset.UserID, asset.PayloadJSON, asset.UpdatedAt).Updates(map[string]any{"payload_json": string(encoded), "updated_at": now})
+				if result.Error != nil {
+					return errors.New("更新素材失败，未提交修复")
+				}
+				if result.RowsAffected != 1 {
+					return fmt.Errorf("素材 %s 已变化，整批修复回滚，请停止写入后重试", asset.ID)
+				}
+				report.Repaired++
+			}
+		}
+	})
+	if err != nil {
+		report.Repaired = 0
+	}
+	return report, err
+}
--- a/backend/internal/canvas/repair_asset_bytes_test.go
+++ b/backend/internal/canvas/repair_asset_bytes_test.go
@@ -0,0 +1,181 @@
+package canvas
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"gorm.io/driver/postgres"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"infinite-canvas/backend/internal/model"
+)
+
+func TestRepairAssetBytes(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sqlDB, _ := db.DB()
+	defer sqlDB.Close()
+	sqlDB.SetMaxOpenConns(1)
+	testRepairAssetBytes(t, db)
+}
+
+func TestRepairAssetBytesPostgres(t *testing.T) {
+	dsn := os.Getenv("CANVAS_TEST_POSTGRES_DSN")
+	if dsn == "" {
+		t.Skip("CANVAS_TEST_POSTGRES_DSN is not configured")
+	}
+	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sqlDB, _ := db.DB()
+	defer sqlDB.Close()
+	rollback := errors.New("rollback isolated test schema")
+	err = db.Transaction(func(tx *gorm.DB) error {
+		schema := fmt.Sprintf("repair_asset_bytes_%d", time.Now().UnixNano())
+		if err := tx.Exec("CREATE SCHEMA " + schema).Error; err != nil {
+			return err
+		}
+		if err := tx.Exec("SET LOCAL search_path TO " + schema).Error; err != nil {
+			return err
+		}
+		testRepairAssetBytes(t, tx)
+		return rollback
+	})
+	if !errors.Is(err, rollback) {
+		t.Fatal(err)
+	}
+}
+
+func testRepairAssetBytes(t *testing.T, db *gorm.DB) {
+	t.Helper()
+	if err := db.AutoMigrate(&model.Asset{}, &model.Resource{}); err != nil {
+		t.Fatal(err)
+	}
+	resource := model.Resource{ID: "resource-1", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, Size: 12345}
+	if err := db.Create(&resource).Error; err != nil {
+		t.Fatal(err)
+	}
+	originals := map[string]string{}
+	for i := 0; i < 106; i++ {
+		id := fmt.Sprintf("asset-%03d", i)
+		payload := testUserAssetPayload("image", nil)
+		payload["id"] = id
+		payload["customExactInteger"] = json.RawMessage("9007199254740993")
+		data := payload["data"].(map[string]any)
+		data["storageKey"] = "resource:resource-1"
+		data["bytes"] = nil
+		userID := "user-1"
+		switch i {
+		case 0:
+			data["bytes"] = 55 // Valid values must remain untouched.
+		case 1:
+			userID = "another-user"
+		case 2:
+			delete(data, "storageKey")
+		case 3:
+			data["bytes"] = "123"
+		case 4:
+			delete(data, "bytes")
+		case 5:
+			data["bytes"] = -1
+		}
+		raw, err := json.Marshal(payload)
+		if err != nil {
+			t.Fatal(err)
+		}
+		originals[id] = string(raw)
+		asset := model.Asset{ID: id, UserID: userID, Kind: "image", PayloadJSON: string(raw)}
+		if err := db.Create(&asset).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	report, err := RepairAssetBytes(db, false)
+	if err != nil || report.Scanned != 106 || report.Repairable != 103 || report.Repaired != 0 || len(report.Unresolved) != 2 {
+		t.Fatalf("dry run: %+v, %v", report, err)
+	}
+	var before []model.Asset
+	if err := db.Find(&before).Error; err != nil {
+		t.Fatal(err)
+	}
+	for _, asset := range before {
+		if asset.PayloadJSON != originals[asset.ID] {
+			t.Fatal("dry run modified data")
+		}
+	}
+	report, err = RepairAssetBytes(db, true)
+	if err != nil || report.Repaired != 103 || len(report.Unresolved) != 2 {
+		t.Fatalf("apply: %+v, %v", report, err)
+	}
+	var after []model.Asset
+	if err := db.Find(&after).Error; err != nil {
+		t.Fatal(err)
+	}
+	for _, asset := range after {
+		if asset.ID <= "asset-002" {
+			if asset.PayloadJSON != originals[asset.ID] {
+				t.Fatal("valid or unresolved asset modified")
+			}
+			continue
+		}
+		var payload map[string]json.RawMessage
+		var data map[string]json.RawMessage
+		if err := json.Unmarshal([]byte(asset.PayloadJSON), &payload); err != nil {
+			t.Fatal(err)
+		}
+		if err := json.Unmarshal(payload["data"], &data); err != nil {
+			t.Fatal(err)
+		}
+		if string(data["bytes"]) != "12345" || string(payload["customExactInteger"]) != "9007199254740993" {
+			t.Fatal("repair corrupted payload")
+		}
+		var stamp string
+		_ = json.Unmarshal(payload["updatedAt"], &stamp)
+		parsed, err := time.Parse(time.RFC3339Nano, stamp)
+		// PostgreSQL stores microseconds; compare at that precision.
+		if err != nil || parsed.Sub(asset.UpdatedAt).Abs() >= time.Microsecond {
+			t.Fatal("timestamps disagree")
+		}
+	}
+	report, err = RepairAssetBytes(db, true)
+	if err != nil || report.Repairable != 0 || report.Repaired != 0 {
+		t.Fatalf("not idempotent: %+v, %v", report, err)
+	}
+	for _, id := range []string{"asset-003", "asset-004"} {
+		if err := db.Model(&model.Asset{}).Where("id = ?", id).Update("payload_json", originals[id]).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	updates := 0
+	const callback = "test:fail-second-asset-repair"
+	if err := db.Callback().Update().Before("gorm:update").Register(callback, func(tx *gorm.DB) {
+		updates++
+		if updates == 2 {
+			tx.AddError(errors.New("injected update failure"))
+		}
+	}); err != nil {
+		t.Fatal(err)
+	}
+	report, err = RepairAssetBytes(db, true)
+	if removeErr := db.Callback().Update().Remove(callback); removeErr != nil {
+		t.Fatal(removeErr)
+	}
+	if err == nil || report.Repaired != 0 {
+		t.Fatalf("failed repair reported success: %+v, %v", report, err)
+	}
+	for _, id := range []string{"asset-003", "asset-004"} {
+		var asset model.Asset
+		if err := db.First(&asset, "id = ?", id).Error; err != nil {
+			t.Fatal(err)
+		}
+		if asset.PayloadJSON != originals[id] {
+			t.Fatal("failed batch was not rolled back")
+		}
+	}
+}
--- a/backend/internal/canvas/user_asset_document.go
+++ b/backend/internal/canvas/user_asset_document.go
@@ -176,6 +176,11 @@ func requiredJSONNumberField(object map[string]json.RawMessage, key string) (flo
 	if !ok {
 		return 0, kernel.BadAuthRequest("素材缺少 " + key + " 字段")
 	}
+	// json.Unmarshal(null, &float64) succeeds without assigning a value.
+	// Reject null explicitly so the persisted JSON obeys the client contract.
+	if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
+		return 0, kernel.BadAuthRequest("素材字段 " + key + " 必须是数字")
+	}
 	var value float64
 	if err := json.Unmarshal(raw, &value); err != nil {
 		return 0, kernel.BadAuthRequest("素材字段 " + key + " 必须是数字")
--- a/backend/internal/canvas/user_asset_document_test.go
+++ b/backend/internal/canvas/user_asset_document_test.go
@@ -0,0 +1,32 @@
+package canvas
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestUserAssetBytesRequiresNonNegativeNumber(t *testing.T) {
+	for _, kind := range []string{"image", "video", "audio", "model"} {
+		for _, raw := range []string{"null", " null ", `"12"`, "-1", "true", "{}", "[]", "1e400", "0", "12"} {
+			t.Run(kind+"/"+raw, func(t *testing.T) {
+				data := map[string]json.RawMessage{
+					"dataUrl": json.RawMessage(`"https://example.com/file"`),
+					"url":     json.RawMessage(`"https://example.com/file"`),
+					"width":   json.RawMessage("1"), "height": json.RawMessage("1"),
+					"mimeType": json.RawMessage(`"application/octet-stream"`),
+					"fileName": json.RawMessage(`"file.glb"`),
+					"bytes":    json.RawMessage(raw),
+				}
+				err := validateUserAssetData(kind, data)
+				valid := raw == "0" || raw == "12"
+				if (err == nil) != valid {
+					t.Fatalf("bytes=%s: error=%v, want valid=%v", raw, err, valid)
+				}
+				delete(data, "bytes")
+				if err := validateUserAssetData(kind, data); err == nil {
+					t.Fatal("missing bytes accepted")
+				}
+			})
+		}
+	}
+}
--- a/docs/content/docs/backend/backend-database.mdx
+++ b/docs/content/docs/backend/backend-database.mdx
@@ -29,6 +29,45 @@ description: 影策后端核心持久化表与状态边界

 版本 3 会将 `wardrobe`、`weapon`、`accessory` 迁移为 `prop`，将 `style` 迁移为 `material`，并把原先归入 `other` 的图片、视频、音频和 3D 模型迁移为 `material`。迁移只合并重复的待确认候选：保留最早记录，其余标记为 `ignored`；已经确认的资产不会被自动删除或归并。

+## 已部署 PostgreSQL 的素材 bytes 修复
+
+升级到包含修复工具的镜像后，由部署管理员执行一次，无需终端用户修改数据库、清缓存或重建画布。旧镜像不支持以下子命令；请先将部署配置中的镜像版本更新为实际发布的修复版本。此操作不增加表或改变 schema 版本，也不会在启动时静默修改素材。
+
+`migrate-schema repair-asset-bytes` 默认只检查，`--apply` 才写入。它按 100 条分批扫描媒体素材，只修复无效的 `data.bytes`，从明确的 `resource:` 引用取得同用户、同类型、就绪且大小为正的资源 `size`。JSON 其他字段、素材 ID、文件和项目引用保留，仅同步推进素材及 JSON 的 `updatedAt`。没有可靠资源或存在其他合同错误的素材保持原样并列入 `unresolved`；存在未解决记录时命令非零退出，即使其他可修复记录已成功提交，也不代表全部修复。
+
+在原部署目录使用原来的环境文件执行；若部署使用了额外 Compose 文件，所有命令必须沿用相同参数。下面对应仓库标准 PostgreSQL Compose：
+
+```bash
+# 先下载实际发布的修复版本，再只读检查；不会启动后端。
+docker compose -f docker-compose.deploy.yml pull migrate backend web
+docker compose -f docker-compose.deploy.yml run --rm --no-deps migrate repair-asset-bytes
+```
+
+审阅报告，确认可修复范围后再进入维护窗口。多副本/独立 worker 也必须全部停止写入。备份或校验失败时停止操作，不能继续执行修复：
+
+```bash
+docker compose -f docker-compose.deploy.yml stop backend
+
+# 备份必须成功才可继续。备份含用户数据，限制权限，不上传或提交仓库。
+umask 077
+backup="asset-bytes-before-$(date +%Y%m%d-%H%M%S).dump"
+docker compose -f docker-compose.deploy.yml exec -T postgres \
+  sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' > "$backup" || exit 1
+docker compose -f docker-compose.deploy.yml exec -T postgres pg_restore --list < "$backup" > /dev/null || exit 1
+```
+
+确认备份和检查均成功后执行写入，保留 JSON 报告：
+
+```bash
+docker compose -f docker-compose.deploy.yml run --rm --no-deps migrate repair-asset-bytes --apply
+docker compose -f docker-compose.deploy.yml run --rm --no-deps migrate repair-asset-bytes
+
+# 核查 repaired / unresolved，再恢复服务。
+docker compose -f docker-compose.deploy.yml up -d backend web
+```
+
+若报告仍有 `unresolved`，需按素材 ID 核查实际文件大小及引用；不要批量填零，也不要把未解决记录删除。工具不访问外部 URL，不下载媒体。应用后让用户重新加载素材库验证；不会要求清空本地存储。若数据库更新失败或检测到并发修改，整批写入事务回滚，`repaired` 为零。已成功提交后的回滚应在隔离数据库恢复备份并核查，再选择性恢复相关记录；不要在恢复写入后直接覆盖整库，以免丢失后续数据。
+
 ## 平台外观配置

 平台外观复用既有 `system_settings` 表，不新增结构迁移。键 `appearance` 的 JSON 值使用版本化合同；版本 3 保存品牌名称、英文品牌标识、登录页主标题与可选说明文案、浅色与可选深色 Logo 资源 ID、Logo 外框开关、登录页视频资源 ID、可选封面资源 ID 和预留皮肤 ID。只配置一套 Logo 时，公开投影会在浅色和深色主题复用同一资源；旧版本缺少深色 Logo 与外框字段时分别回退到单 Logo 和启用外框，不改变既有视觉。资源元数据和文件仍分别由 `resources` 表与当前资源存储承担。恢复默认外观时删除这条设置记录并回到内置值，不级联删除已经上传的资源。
--- a/docs/content/docs/progress/pending-test.mdx
+++ b/docs/content/docs/progress/pending-test.mdx
@@ -5,6 +5,13 @@ description: 当前项目已实现功能的手工验证清单与未确认边界

 # 待测试

+## 素材 bytes 数字合同
+
+- 修复后端将 `bytes: null` 按数字零通过校验的问题，避免保存后被前端素材合同拒绝；图片、视频、音频、模型素材均覆盖。
+- 部署维护工具新增 `migrate-schema repair-asset-bytes`（默认只检查）和 `--apply`：仅使用同用户、同类型、就绪且大小为正的资源元数据修复；保留素材 ID、其他 JSON 字段和引用。无可靠资源的记录输出未解决报告，不填零；并发冲突或数据库写入错误整批回滚。操作步骤见后端数据库专题。
+- 已在 SQLite 和隔离 PostgreSQL 17 测试：只读预检、跨 100 条分页修复、缺失/null/字符串/负数处理、跨用户拒绝、合法值及其他 JSON 精度保留、重复运行幂等、写入失败整批回滚；待实际部署备份后执行并验收用户素材库。
+- 待确认：用户报错素材的实际 `data.bytes` 值及写入来源。此修复只阻止新的无效写入，不自动修改或删除历史素材；历史值需依据真实资源大小修复，不能批量填零。
+
 ## 百炼多模态图像协议插件（dashscope-qwen-image）

 - 新增官方协议包 `plugin-packages/dashscope-qwen-image.yingce-plugin`，解决 `qwen-image-3.0-pro` / `wan2.7-image-pro` 无法调用的问题：这两个模型属于百炼多模态模型，走 OpenAI 图片端点会被网关判为“模型与端点不匹配”，官方错误码表现为 `url error, please check url`。