From f81c05abdb9e09bc93ceabea6e355aff8facefdf Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Mon, 14 Sep 2026 23:26:32 +0800 Subject: [PATCH] =?UTF-8?q?fix(canvas):=20=E7=B4=A0=E6=9D=90=E5=BA=93=20-?= =?UTF-8?q?=20=E6=8B=92=E7=BB=9D=E7=A9=BA=E5=AD=97=E8=8A=82=E6=95=B0?= =?UTF-8?q?=E5=B9=B6=E6=8F=90=E4=BE=9B=E5=8E=86=E5=8F=B2=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/cmd/migrate-schema/main.go | 25 ++- backend/internal/canvas/repair_asset_bytes.go | 116 +++++++++++ .../canvas/repair_asset_bytes_test.go | 181 ++++++++++++++++++ .../internal/canvas/user_asset_document.go | 5 + .../canvas/user_asset_document_test.go | 32 ++++ .../content/docs/backend/backend-database.mdx | 39 ++++ docs/content/docs/progress/pending-test.mdx | 7 + 7 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 backend/internal/canvas/repair_asset_bytes.go create mode 100644 backend/internal/canvas/repair_asset_bytes_test.go create mode 100644 backend/internal/canvas/user_asset_document_test.go diff --git a/backend/cmd/migrate-schema/main.go b/backend/cmd/migrate-schema/main.go index ed53db4a0..48233ff52 100644 --- a/backend/cmd/migrate-schema/main.go +++ b/backend/cmd/migrate-schema/main.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "infinite-canvas/backend/internal/canvas" "infinite-canvas/backend/internal/database" ) @@ -15,6 +16,14 @@ func main() { if len(os.Args) > 1 { command = strings.ToLower(strings.TrimSpace(os.Args[1])) } + apply := false + if command == "repair-asset-bytes" { + if len(os.Args) == 3 && os.Args[2] == "--apply" { + apply = true + } else if len(os.Args) > 2 { + log.Fatal("用法：migrate-schema repair-asset-bytes [--apply]；默认只检查") + } + } db, err := database.Open(database.Config{ Driver: env("CANVAS_DATABASE_DRIVER", "sqlite"), DSN: os.Getenv("DATABASE_URL"), @@ -33,6 +42,20 @@ func main() { defer sqlDB.Close() switch command { + case "repair-asset-bytes": + report, repairErr := canvas.RepairAssetBytes(db, apply) + encoded, err := json.Marshal(report) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(encoded)) + if repairErr != nil { + log.Fatal(repairErr) + } + if len(report.Unresolved) > 0 { + log.Fatal("存在未修复素材，请按报告核查；未删除或填零") + } + return case "up": if err := database.MigrateSchema(db); err != nil { log.Fatal(err) @@ -43,7 +66,7 @@ func main() { log.Fatal(err) } default: - log.Fatalf("未知命令 %q；可用命令：up、status、verify", command) + log.Fatalf("未知命令 %q；可用命令：up、status、verify、repair-asset-bytes", command) } status, err := database.ReadSchemaStatus(db) if err != nil { diff --git a/backend/internal/canvas/repair_asset_bytes.go b/backend/internal/canvas/repair_asset_bytes.go new file mode 100644 index 000000000..19b6c981a --- /dev/null +++ b/backend/internal/canvas/repair_asset_bytes.go @@ -0,0 +1,116 @@ +package canvas + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "infinite-canvas/backend/internal/model" + + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type AssetBytesRepairIssue struct { + AssetID string `json:"assetId"` + Reason string `json:"reason"` +} + +type AssetBytesRepairReport struct { + Apply bool `json:"apply"` + Scanned int `json:"scanned"` + Repairable int `json:"repairable"` + Repaired int `json:"repaired"` + Unresolved []AssetBytesRepairIssue `json:"unresolved"` +} + +// RepairAssetBytes is an explicit offline maintenance operation, not a read fallback. +// Only an owned, ready resource with a known positive size can repair a record. +func RepairAssetBytes(db *gorm.DB, apply bool) (AssetBytesRepairReport, error) { + report := AssetBytesRepairReport{Apply: apply, Unresolved: []AssetBytesRepairIssue{}} + // Payloads may contain private media URLs; never include SQL parameters in logs. + err := db.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}).Transaction(func(tx *gorm.DB) error { + after := "" + for { + var rows []model.Asset + if err := tx.Where("id > ? AND kind IN ?", after, []string{"image", "video", "audio", "model"}).Order("id ASC").Limit(100).Find(&rows).Error; err != nil { + return errors.New("读取素材失败，未提交修复") + } + if len(rows) == 0 { + return nil + } + for _, asset := range rows { + after = asset.ID + report.Scanned++ + issue := func(reason string) { + report.Unresolved = append(report.Unresolved, AssetBytesRepairIssue{AssetID: asset.ID, Reason: reason}) + } + var payload map[string]json.RawMessage + var data map[string]json.RawMessage + if json.Unmarshal([]byte(asset.PayloadJSON), &payload) != nil || json.Unmarshal(payload["data"], &data) != nil || data == nil { + issue("素材 JSON 或 data 无效，需人工核查") + continue + } + if _, err := requiredJSONNumberField(data, "bytes"); err == nil { + continue + } + var kind, key string + _ = json.Unmarshal(payload["kind"], &kind) + _ = json.Unmarshal(data["storageKey"], &key) + key = strings.TrimSpace(key) + if strings.TrimSpace(asset.UserID) == "" || kind != asset.Kind || !strings.HasPrefix(key, "resource:") || strings.TrimPrefix(key, "resource:") == "" { + issue("类型不一致或缺少明确的 resource: 引用，不能推测文件大小") + continue + } + var resource model.Resource + err := tx.Where("id = ? AND user_id = ? AND kind = ? AND status = ? AND size > 0", strings.TrimPrefix(key, "resource:"), asset.UserID, kind, model.ResourceStatusReady).Take(&resource).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + issue("未找到同用户、同类型、就绪且大小已知的资源") + continue + } + if err != nil { + return errors.New("读取资源失败，未提交修复") + } + data["bytes"], err = json.Marshal(resource.Size) + if err != nil { + return err + } + payload["data"], err = json.Marshal(data) + if err != nil { + return err + } + now := time.Now().UTC() + payload["updatedAt"], err = json.Marshal(now.Format(time.RFC3339Nano)) + if err != nil { + return err + } + encoded, err := json.Marshal(payload) + if err != nil { + return err + } + if err := validateUserAssetDocument(encoded); err != nil { + issue("素材仍有其他合同错误，未修改，请另行核查") + continue + } + report.Repairable++ + if !apply { + continue + } + result := tx.Model(&model.Asset{}).Where("id = ? AND user_id = ? AND payload_json = ? AND updated_at = ?", asset.ID, asset.UserID, asset.PayloadJSON, asset.UpdatedAt).Updates(map[string]any{"payload_json": string(encoded), "updated_at": now}) + if result.Error != nil { + return errors.New("更新素材失败，未提交修复") + } + if result.RowsAffected != 1 { + return fmt.Errorf("素材 %s 已变化，整批修复回滚，请停止写入后重试", asset.ID) + } + report.Repaired++ + } + } + }) + if err != nil { + report.Repaired = 0 + } + return report, err +} diff --git a/backend/internal/canvas/repair_asset_bytes_test.go b/backend/internal/canvas/repair_asset_bytes_test.go new file mode 100644 index 000000000..f229531d3 --- /dev/null +++ b/backend/internal/canvas/repair_asset_bytes_test.go @@ -0,0 +1,181 @@ +package canvas + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "testing" + "time" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "infinite-canvas/backend/internal/model" +) + +func TestRepairAssetBytes(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, _ := db.DB() + defer sqlDB.Close() + sqlDB.SetMaxOpenConns(1) + testRepairAssetBytes(t, db) +} + +func TestRepairAssetBytesPostgres(t *testing.T) { + dsn := os.Getenv("CANVAS_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("CANVAS_TEST_POSTGRES_DSN is not configured") + } + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, _ := db.DB() + defer sqlDB.Close() + rollback := errors.New("rollback isolated test schema") + err = db.Transaction(func(tx *gorm.DB) error { + schema := fmt.Sprintf("repair_asset_bytes_%d", time.Now().UnixNano()) + if err := tx.Exec("CREATE SCHEMA " + schema).Error; err != nil { + return err + } + if err := tx.Exec("SET LOCAL search_path TO " + schema).Error; err != nil { + return err + } + testRepairAssetBytes(t, tx) + return rollback + }) + if !errors.Is(err, rollback) { + t.Fatal(err) + } +} + +func testRepairAssetBytes(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.AutoMigrate(&model.Asset{}, &model.Resource{}); err != nil { + t.Fatal(err) + } + resource := model.Resource{ID: "resource-1", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, Size: 12345} + if err := db.Create(&resource).Error; err != nil { + t.Fatal(err) + } + originals := map[string]string{} + for i := 0; i < 106; i++ { + id := fmt.Sprintf("asset-%03d", i) + payload := testUserAssetPayload("image", nil) + payload["id"] = id + payload["customExactInteger"] = json.RawMessage("9007199254740993") + data := payload["data"].(map[string]any) + data["storageKey"] = "resource:resource-1" + data["bytes"] = nil + userID := "user-1" + switch i { + case 0: + data["bytes"] = 55 // Valid values must remain untouched. + case 1: + userID = "another-user" + case 2: + delete(data, "storageKey") + case 3: + data["bytes"] = "123" + case 4: + delete(data, "bytes") + case 5: + data["bytes"] = -1 + } + raw, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + originals[id] = string(raw) + asset := model.Asset{ID: id, UserID: userID, Kind: "image", PayloadJSON: string(raw)} + if err := db.Create(&asset).Error; err != nil { + t.Fatal(err) + } + } + report, err := RepairAssetBytes(db, false) + if err != nil || report.Scanned != 106 || report.Repairable != 103 || report.Repaired != 0 || len(report.Unresolved) != 2 { + t.Fatalf("dry run: %+v, %v", report, err) + } + var before []model.Asset + if err := db.Find(&before).Error; err != nil { + t.Fatal(err) + } + for _, asset := range before { + if asset.PayloadJSON != originals[asset.ID] { + t.Fatal("dry run modified data") + } + } + report, err = RepairAssetBytes(db, true) + if err != nil || report.Repaired != 103 || len(report.Unresolved) != 2 { + t.Fatalf("apply: %+v, %v", report, err) + } + var after []model.Asset + if err := db.Find(&after).Error; err != nil { + t.Fatal(err) + } + for _, asset := range after { + if asset.ID <= "asset-002" { + if asset.PayloadJSON != originals[asset.ID] { + t.Fatal("valid or unresolved asset modified") + } + continue + } + var payload map[string]json.RawMessage + var data map[string]json.RawMessage + if err := json.Unmarshal([]byte(asset.PayloadJSON), &payload); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(payload["data"], &data); err != nil { + t.Fatal(err) + } + if string(data["bytes"]) != "12345" || string(payload["customExactInteger"]) != "9007199254740993" { + t.Fatal("repair corrupted payload") + } + var stamp string + _ = json.Unmarshal(payload["updatedAt"], &stamp) + parsed, err := time.Parse(time.RFC3339Nano, stamp) + // PostgreSQL stores microseconds; compare at that precision. + if err != nil || parsed.Sub(asset.UpdatedAt).Abs() >= time.Microsecond { + t.Fatal("timestamps disagree") + } + } + report, err = RepairAssetBytes(db, true) + if err != nil || report.Repairable != 0 || report.Repaired != 0 { + t.Fatalf("not idempotent: %+v, %v", report, err) + } + for _, id := range []string{"asset-003", "asset-004"} { + if err := db.Model(&model.Asset{}).Where("id = ?", id).Update("payload_json", originals[id]).Error; err != nil { + t.Fatal(err) + } + } + updates := 0 + const callback = "test:fail-second-asset-repair" + if err := db.Callback().Update().Before("gorm:update").Register(callback, func(tx *gorm.DB) { + updates++ + if updates == 2 { + tx.AddError(errors.New("injected update failure")) + } + }); err != nil { + t.Fatal(err) + } + report, err = RepairAssetBytes(db, true) + if removeErr := db.Callback().Update().Remove(callback); removeErr != nil { + t.Fatal(removeErr) + } + if err == nil || report.Repaired != 0 { + t.Fatalf("failed repair reported success: %+v, %v", report, err) + } + for _, id := range []string{"asset-003", "asset-004"} { + var asset model.Asset + if err := db.First(&asset, "id = ?", id).Error; err != nil { + t.Fatal(err) + } + if asset.PayloadJSON != originals[id] { + t.Fatal("failed batch was not rolled back") + } + } +} diff --git a/backend/internal/canvas/user_asset_document.go b/backend/internal/canvas/user_asset_document.go index 00dc2a9a1..daba2009f 100644 --- a/backend/internal/canvas/user_asset_document.go +++ b/backend/internal/canvas/user_asset_document.go @@ -176,6 +176,11 @@ func requiredJSONNumberField(object map[string]json.RawMessage, key string) (flo if !ok { return 0, kernel.BadAuthRequest("素材缺少 " + key + " 字段") } + // json.Unmarshal(null, &float64) succeeds without assigning a value. + // Reject null explicitly so the persisted JSON obeys the client contract. + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return 0, kernel.BadAuthRequest("素材字段 " + key + " 必须是数字") + } var value float64 if err := json.Unmarshal(raw, &value); err != nil { return 0, kernel.BadAuthRequest("素材字段 " + key + " 必须是数字") diff --git a/backend/internal/canvas/user_asset_document_test.go b/backend/internal/canvas/user_asset_document_test.go new file mode 100644 index 000000000..92a7a314e --- /dev/null +++ b/backend/internal/canvas/user_asset_document_test.go @@ -0,0 +1,32 @@ +package canvas + +import ( + "encoding/json" + "testing" +) + +func TestUserAssetBytesRequiresNonNegativeNumber(t *testing.T) { + for _, kind := range []string{"image", "video", "audio", "model"} { + for _, raw := range []string{"null", " null ", `"12"`, "-1", "true", "{}", "[]", "1e400", "0", "12"} { + t.Run(kind+"/"+raw, func(t *testing.T) { + data := map[string]json.RawMessage{ + "dataUrl": json.RawMessage(`"https://example.com/file"`), + "url": json.RawMessage(`"https://example.com/file"`), + "width": json.RawMessage("1"), "height": json.RawMessage("1"), + "mimeType": json.RawMessage(`"application/octet-stream"`), + "fileName": json.RawMessage(`"file.glb"`), + "bytes": json.RawMessage(raw), + } + err := validateUserAssetData(kind, data) + valid := raw == "0" || raw == "12" + if (err == nil) != valid { + t.Fatalf("bytes=%s: error=%v, want valid=%v", raw, err, valid) + } + delete(data, "bytes") + if err := validateUserAssetData(kind, data); err == nil { + t.Fatal("missing bytes accepted") + } + }) + } + } +} diff --git a/docs/content/docs/backend/backend-database.mdx b/docs/content/docs/backend/backend-database.mdx index f47abac73..28faa7ca4 100644 --- a/docs/content/docs/backend/backend-database.mdx +++ b/docs/content/docs/backend/backend-database.mdx @@ -29,6 +29,45 @@ description: 影策后端核心持久化表与状态边界 版本 3 会将 `wardrobe`、`weapon`、`accessory` 迁移为 `prop`，将 `style` 迁移为 `material`，并把原先归入 `other` 的图片、视频、音频和 3D 模型迁移为 `material`。迁移只合并重复的待确认候选：保留最早记录，其余标记为 `ignored`；已经确认的资产不会被自动删除或归并。 +## 已部署 PostgreSQL 的素材 bytes 修复 + +升级到包含修复工具的镜像后，由部署管理员执行一次，无需终端用户修改数据库、清缓存或重建画布。旧镜像不支持以下子命令；请先将部署配置中的镜像版本更新为实际发布的修复版本。此操作不增加表或改变 schema 版本，也不会在启动时静默修改素材。 + +`migrate-schema repair-asset-bytes` 默认只检查，`--apply` 才写入。它按 100 条分批扫描媒体素材，只修复无效的 `data.bytes`，从明确的 `resource:` 引用取得同用户、同类型、就绪且大小为正的资源 `size`。JSON 其他字段、素材 ID、文件和项目引用保留，仅同步推进素材及 JSON 的 `updatedAt`。没有可靠资源或存在其他合同错误的素材保持原样并列入 `unresolved`；存在未解决记录时命令非零退出，即使其他可修复记录已成功提交，也不代表全部修复。 + +在原部署目录使用原来的环境文件执行；若部署使用了额外 Compose 文件，所有命令必须沿用相同参数。下面对应仓库标准 PostgreSQL Compose： + +```bash +# 先下载实际发布的修复版本，再只读检查；不会启动后端。 +docker compose -f docker-compose.deploy.yml pull migrate backend web +docker compose -f docker-compose.deploy.yml run --rm --no-deps migrate repair-asset-bytes +``` + +审阅报告，确认可修复范围后再进入维护窗口。多副本/独立 worker 也必须全部停止写入。备份或校验失败时停止操作，不能继续执行修复： + +```bash +docker compose -f docker-compose.deploy.yml stop backend + +# 备份必须成功才可继续。备份含用户数据，限制权限，不上传或提交仓库。 +umask 077 +backup="asset-bytes-before-$(date +%Y%m%d-%H%M%S).dump" +docker compose -f docker-compose.deploy.yml exec -T postgres \ + sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' > "$backup" || exit 1 +docker compose -f docker-compose.deploy.yml exec -T postgres pg_restore --list < "$backup" > /dev/null || exit 1 +``` + +确认备份和检查均成功后执行写入，保留 JSON 报告： + +```bash +docker compose -f docker-compose.deploy.yml run --rm --no-deps migrate repair-asset-bytes --apply +docker compose -f docker-compose.deploy.yml run --rm --no-deps migrate repair-asset-bytes + +# 核查 repaired / unresolved，再恢复服务。 +docker compose -f docker-compose.deploy.yml up -d backend web +``` + +若报告仍有 `unresolved`，需按素材 ID 核查实际文件大小及引用；不要批量填零，也不要把未解决记录删除。工具不访问外部 URL，不下载媒体。应用后让用户重新加载素材库验证；不会要求清空本地存储。若数据库更新失败或检测到并发修改，整批写入事务回滚，`repaired` 为零。已成功提交后的回滚应在隔离数据库恢复备份并核查，再选择性恢复相关记录；不要在恢复写入后直接覆盖整库，以免丢失后续数据。 + ## 平台外观配置 平台外观复用既有 `system_settings` 表，不新增结构迁移。键 `appearance` 的 JSON 值使用版本化合同；版本 3 保存品牌名称、英文品牌标识、登录页主标题与可选说明文案、浅色与可选深色 Logo 资源 ID、Logo 外框开关、登录页视频资源 ID、可选封面资源 ID 和预留皮肤 ID。只配置一套 Logo 时，公开投影会在浅色和深色主题复用同一资源；旧版本缺少深色 Logo 与外框字段时分别回退到单 Logo 和启用外框，不改变既有视觉。资源元数据和文件仍分别由 `resources` 表与当前资源存储承担。恢复默认外观时删除这条设置记录并回到内置值，不级联删除已经上传的资源。 diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 542245713..0e50447c4 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -5,6 +5,13 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 # 待测试 +## 素材 bytes 数字合同 + +- 修复后端将 `bytes: null` 按数字零通过校验的问题，避免保存后被前端素材合同拒绝；图片、视频、音频、模型素材均覆盖。 +- 部署维护工具新增 `migrate-schema repair-asset-bytes`（默认只检查）和 `--apply`：仅使用同用户、同类型、就绪且大小为正的资源元数据修复；保留素材 ID、其他 JSON 字段和引用。无可靠资源的记录输出未解决报告，不填零；并发冲突或数据库写入错误整批回滚。操作步骤见后端数据库专题。 +- 已在 SQLite 和隔离 PostgreSQL 17 测试：只读预检、跨 100 条分页修复、缺失/null/字符串/负数处理、跨用户拒绝、合法值及其他 JSON 精度保留、重复运行幂等、写入失败整批回滚；待实际部署备份后执行并验收用户素材库。 +- 待确认：用户报错素材的实际 `data.bytes` 值及写入来源。此修复只阻止新的无效写入，不自动修改或删除历史素材；历史值需依据真实资源大小修复，不能批量填零。 + ## 百炼多模态图像协议插件（dashscope-qwen-image） - 新增官方协议包 `plugin-packages/dashscope-qwen-image.yingce-plugin`，解决 `qwen-image-3.0-pro` / `wan2.7-image-pro` 无法调用的问题：这两个模型属于百炼多模态模型，走 OpenAI 图片端点会被网关判为“模型与端点不匹配”，官方错误码表现为 `url error, please check url`。