diff --git a/.gitignore b/.gitignore index b023579bb..ffb4b0447 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ docs/content/docs/* !docs/content/docs/plugins/ docs/content/docs/plugins/* !docs/content/docs/plugins/ai-art-critique.mdx +# Keep the task stability runbook versioned without unignoring generated docs. +!docs/content/docs/backend/ +docs/content/docs/backend/* +!docs/content/docs/backend/task-runtime-stability.mdx .env* !.env.example data/ diff --git a/backend/internal/handler/routes.go b/backend/internal/handler/routes.go index 6502609b1..97b70e1e2 100644 --- a/backend/internal/handler/routes.go +++ b/backend/internal/handler/routes.go @@ -174,7 +174,7 @@ func RegisterTaskRoutes(r *gin.RouterGroup, svc *service.Service) { fail(c, http.StatusBadRequest, err) return } - initial, err := svc.TaskTextReplay(user.ID, c.Param("id"), after) + initial, err := svc.CachedTaskTextReplay(c.Request.Context(), user.ID, c.Param("id"), after) if err != nil { failService(c, err) return @@ -308,7 +308,7 @@ func streamTaskTextEvents(c *gin.Context, svc *service.Service, userID string, t c.Writer.Flush() case <-pollTicker.C: } - next, err := svc.TaskTextReplay(userID, taskID, after) + next, err := svc.CachedTaskTextReplay(c.Request.Context(), userID, taskID, after) if err != nil { writeTaskTextSSE(c, "error", 0, map[string]string{"message": "任务文本流不可用"}) return diff --git a/backend/internal/handler/text_events_test.go b/backend/internal/handler/text_events_test.go index 25207d530..98bff5a28 100644 --- a/backend/internal/handler/text_events_test.go +++ b/backend/internal/handler/text_events_test.go @@ -1,14 +1,19 @@ package handler import ( + "context" "net/http/httptest" "strings" "testing" + "time" "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/repository" "infinite-canvas/backend/internal/service" "github.com/gin-gonic/gin" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) func TestTaskTextEventCursorPrefersQueryAndSupportsLastEventID(t *testing.T) { @@ -32,6 +37,42 @@ func TestTaskTextEventCursorPrefersQueryAndSupportsLastEventID(t *testing.T) { } } +func TestStreamTaskTextEventsCachedReplayReachesTerminalWithoutDuplicateDelta(t *testing.T) { + t.Setenv("REDIS_URL", "") + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Task{}, &model.TaskTextDelta{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.Task{ID: "task", UserID: "user", Status: model.TaskStatusRunning}).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.TaskTextDelta{ID: "delta", UserID: "user", TaskID: "task", Sequence: 8, Content: "text", ExpiresAt: time.Now().Add(time.Hour)}).Error; err != nil { + t.Fatal(err) + } + svc := service.New(repository.New(db), t.TempDir()) + t.Cleanup(func() { _ = svc.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + initial, err := svc.CachedTaskTextReplay(ctx, "user", "task", 7) + if err != nil { + t.Fatal(err) + } + if err := db.Model(&model.Task{}).Where("id = ?", "task").Updates(map[string]any{"status": model.TaskStatusSucceeded, "progress": 100}).Error; err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + c, _ := gin.CreateTestContext(response) + c.Request = httptest.NewRequest("GET", "/api/tasks/task/text-events", nil).WithContext(ctx) + streamTaskTextEvents(c, svc, "user", "task", 7, initial) + body := response.Body.String() + if strings.Count(body, "id: 8\nevent: delta") != 1 || !strings.Contains(body, "event: terminal") || !strings.Contains(body, `"progress":100`) { + t.Fatalf("unexpected cached SSE stream: %q", body) + } +} + func TestStreamTaskTextEventsWritesSequenceAndTerminalEvent(t *testing.T) { gin.SetMode(gin.TestMode) response := httptest.NewRecorder() diff --git a/backend/internal/repository/repository.go b/backend/internal/repository/repository.go index da764d171..998008a2c 100644 --- a/backend/internal/repository/repository.go +++ b/backend/internal/repository/repository.go @@ -1,6 +1,7 @@ package repository import ( + "context" "errors" "fmt" "slices" @@ -54,6 +55,10 @@ func New(db *gorm.DB) *Repository { return &Repository{db: db} } +func (r *Repository) WithContext(ctx context.Context) *Repository { + return &Repository{db: r.db.WithContext(ctx)} +} + func (r *Repository) Dialect() string { return r.db.Dialector.Name() } @@ -428,7 +433,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (* func (r *Repository) RenewTaskLease(id string, owner string, leaseDuration time.Duration) error { result := r.db.Model(&model.Task{}). - Where("id = ? AND status = ? AND lease_owner = ?", id, model.TaskStatusRunning, owner). + Where("id = ? AND status = ? AND lease_owner = ? AND lease_expires_at > ?", id, model.TaskStatusRunning, owner, time.Now()). Updates(map[string]any{"lease_expires_at": time.Now().Add(leaseDuration), "updated_at": time.Now()}) if result.Error != nil { return result.Error @@ -449,8 +454,8 @@ func (r *Repository) UpdateTaskProviderState(id string, providerRequestID string func (r *Repository) DeferRunningTaskForProviderPoll(id string, owner string, stage string, delay time.Duration) error { now := time.Now() - result := r.db.Model(&model.Task{}). - Where("id = ? AND status = ? AND lease_owner = ?", id, model.TaskStatusRunning, owner). + result := taskLeaseWriter(r.db.Model(&model.Task{}), owner). + Where("id = ? AND status = ?", id, model.TaskStatusRunning). Updates(map[string]any{ "stage": stage, "error": "", "completed_at": nil, "next_poll_at": now.Add(delay), "lease_owner": "", "lease_expires_at": nil, "updated_at": now, @@ -500,6 +505,27 @@ func (r *Repository) UpdateTaskProgress(id string, stage string, progress int) e }).Error } +func (r *Repository) UpdateTaskProgressForLease(id string, owner string, stage string, progress int) error { + result := taskLeaseWriter(r.db.Model(&model.Task{}), owner). + Where("id = ? AND status = ?", id, model.TaskStatusRunning). + Updates(map[string]any{"stage": stage, "progress": progress, "updated_at": time.Now()}) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrTaskStateConflict + } + return nil +} + +// 无租约任务仅能写无租约记录;有租约的执行者必须仍持有本次领取的有效 owner。 +func taskLeaseWriter(db *gorm.DB, owner string) *gorm.DB { + if owner == "" { + return db.Where("(lease_owner = '' OR lease_owner IS NULL)") + } + return db.Where("lease_owner = ? AND lease_expires_at > ?", owner, time.Now()) +} + // UpdateTaskProviderProgress records upstream-reported progress without allowing // a delayed or out-of-order poll response to move the public percentage backwards. func (r *Repository) UpdateTaskProviderProgress(id string, progress int) error { @@ -513,7 +539,7 @@ func (r *Repository) UpdateTaskProviderProgress(id string, progress int) error { func (r *Repository) SaveTaskCompletion(task *model.Task, expected model.TaskStatus, session *model.Session, message *model.Message, results []model.Result) error { return r.db.Transaction(func(tx *gorm.DB) error { - updated := tx.Model(&model.Task{}). + updated := taskLeaseWriter(tx.Model(&model.Task{}), task.LeaseOwner). Where("id = ? AND status = ?", task.ID, expected). Select("*").Omit("id", "created_at").Updates(task) if updated.Error != nil { @@ -541,8 +567,8 @@ func (r *Repository) SaveTaskCompletion(task *model.Task, expected model.TaskSta }) } -func (r *Repository) UpdateTaskTerminalState(id string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) { - result := r.db.Model(&model.Task{}). +func (r *Repository) UpdateTaskTerminalState(id string, owner string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) { + result := taskLeaseWriter(r.db.Model(&model.Task{}), owner). Where("id = ? AND status = ?", id, expected). Updates(map[string]any{ "status": status, "stage": stage, "error": errorText, "completed_at": &completedAt, diff --git a/backend/internal/repository/task_lease_fencing_test.go b/backend/internal/repository/task_lease_fencing_test.go new file mode 100644 index 000000000..b4b8f09bd --- /dev/null +++ b/backend/internal/repository/task_lease_fencing_test.go @@ -0,0 +1,75 @@ +package repository + +import ( + "errors" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "infinite-canvas/backend/internal/model" +) + +func TestTaskLeaseFencesExpiredAndReclaimedWriters(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Task{}, &model.Result{}, &model.Message{}, &model.Session{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.Task{ID: "task", UserID: "user", Status: model.TaskStatusQueued}).Error; err != nil { + t.Fatal(err) + } + repo := New(db) + old, err := repo.ClaimNextTask("worker:attempt-a", time.Minute) + if err != nil || old == nil { + t.Fatalf("claim: %v %v", old, err) + } + if err := db.Model(&model.Task{}).Where("id = ?", old.ID).Update("lease_expires_at", time.Now().Add(-time.Second)).Error; err != nil { + t.Fatal(err) + } + if err := repo.RenewTaskLease(old.ID, old.LeaseOwner, time.Minute); err == nil { + t.Fatal("expired owner resurrected") + } + if err := repo.DeferRunningTaskForProviderPoll(old.ID, old.LeaseOwner, "poll", time.Minute); !errors.Is(err, ErrTaskStateConflict) { + t.Fatalf("expired owner deferred task: %v", err) + } + current, err := repo.ClaimNextTask("worker:attempt-b", time.Minute) + if err != nil || current == nil { + t.Fatalf("reclaim: %v %v", current, err) + } + for _, owner := range []string{old.LeaseOwner, ""} { + if err := repo.UpdateTaskProgressForLease(old.ID, owner, "stale", 90); !errors.Is(err, ErrTaskStateConflict) { + t.Fatalf("stale progress: %v", err) + } + updated, err := repo.UpdateTaskTerminalState(old.ID, owner, model.TaskStatusRunning, model.TaskStatusFailed, "stale", "", time.Now()) + if err != nil || updated { + t.Fatalf("stale terminal: %v %v", updated, err) + } + } + old.Status = model.TaskStatusSucceeded + if err := repo.SaveTaskCompletion(old, model.TaskStatusRunning, nil, &model.Message{ID: "stale"}, []model.Result{{ID: "stale"}}); !errors.Is(err, ErrTaskStateConflict) { + t.Fatalf("stale completion: %v", err) + } + var count int64 + for _, table := range []any{&model.Result{}, &model.Message{}} { + if err := db.Model(table).Count(&count).Error; err != nil || count != 0 { + t.Fatalf("stale related row persisted: %d %v", count, err) + } + } + if err := repo.RenewTaskLease(current.ID, current.LeaseOwner, time.Minute); err != nil { + t.Fatal(err) + } + if err := repo.UpdateTaskProgressForLease(current.ID, current.LeaseOwner, "current", 60); err != nil { + t.Fatal(err) + } + current.Status = model.TaskStatusSucceeded + if err := repo.SaveTaskCompletion(current, model.TaskStatusRunning, nil, nil, []model.Result{{ID: "valid", TaskID: current.ID}}); err != nil { + t.Fatal(err) + } + stored, err := repo.Task(current.ID) + if err != nil || stored.Status != model.TaskStatusSucceeded || stored.LeaseOwner != current.LeaseOwner { + t.Fatalf("current completion: %v %v", stored, err) + } +} diff --git a/backend/internal/service/coordination.go b/backend/internal/service/coordination.go index ba5d1ee66..f9243b873 100644 --- a/backend/internal/service/coordination.go +++ b/backend/internal/service/coordination.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "log" + "math/rand/v2" "os" "strconv" "strings" @@ -74,7 +76,8 @@ var acquireSlotScript = redis.NewScript(` redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[3]) then return 0 end redis.call('ZADD', KEYS[1], ARGV[2], ARGV[4]) -redis.call('PEXPIRE', KEYS[1], ARGV[5]) +local latest = redis.call('ZREVRANGE', KEYS[1], 0, 0, 'WITHSCORES') +redis.call('PEXPIRE', KEYS[1], math.max(tonumber(ARGV[5]), tonumber(latest[2]) - tonumber(ARGV[1]) + 60000)) return 1 `) @@ -91,6 +94,9 @@ func newRuntimeCoordinator(dialect string) (*runtimeCoordinator, error) { if err != nil { return coordinator, fmt.Errorf("REDIS_URL 无效:%w", err) } + // 协调写操作不透明重试;限流/并发失败必须及时向调用方返回,不能放大故障流量。 + options.MaxRetries = -1 + options.ContextTimeoutEnabled = true coordinator.redis = redis.NewClient(options) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -122,6 +128,21 @@ func (c *runtimeCoordinator) allow(ctx context.Context, key string, limit int, w } func (c *runtimeCoordinator) acquire(ctx context.Context, scope string, limit int, ttl time.Duration) (func(), bool, error) { + lease, acquired, err := c.acquireLease(ctx, scope, limit, ttl) + if err != nil || !acquired { + return nil, acquired, err + } + return lease.release, true, nil +} + +func (c *runtimeCoordinator) acquireLease(ctx context.Context, scope string, limit int, ttl time.Duration) (*runtimeSlotLease, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + if limit <= 0 || ttl <= 0 { + return nil, false, errors.New("并发租约参数无效") + } + lease := &runtimeSlotLease{coordinator: c, scope: scope, token: c.instanceID + ":" + newID(), ttl: ttl} if c.redis == nil { c.localMu.Lock() now := time.Now() @@ -139,29 +160,90 @@ func (c *runtimeCoordinator) acquire(ctx context.Context, scope string, limit in c.localMu.Unlock() return nil, false, nil } - token := c.instanceID + ":" + newID() - slots[token] = now.Add(ttl) + slots[lease.token] = now.Add(ttl) c.localMu.Unlock() - return func() { - c.localMu.Lock() - delete(c.localSlots[scope], token) - c.localMu.Unlock() - }, true, nil + return lease, true, nil } // 有过期分数的有序集合避免实例崩溃后永久占槽,业务数据库仍保存任务与账本真相。 key := "canvas:slots:" + scope - token := c.instanceID + ":" + newID() + ctx, cancel := context.WithTimeout(ctx, runtimeCoordinationTimeout) + defer cancel() now := time.Now() - ok, err := acquireSlotScript.Run(ctx, c.redis, []string{key}, now.UnixMilli(), now.Add(ttl).UnixMilli(), limit, token, (ttl + time.Minute).Milliseconds()).Int() + ok, err := acquireSlotScript.Run(ctx, c.redis, []string{key}, now.UnixMilli(), now.Add(ttl).UnixMilli(), limit, lease.token, (ttl + time.Minute).Milliseconds()).Int() if err != nil || ok != 1 { return nil, false, err } - return func() { _ = c.redis.ZRem(context.Background(), key, token).Err() }, true, nil + return lease, true, nil +} + +const runtimeCoordinationTimeout = 2 * time.Second + +type runtimeSlotLease struct { + coordinator *runtimeCoordinator + scope string + token string + ttl time.Duration + once sync.Once +} + +var renewSlotScript = redis.NewScript(` +local expires = redis.call('ZSCORE', KEYS[1], ARGV[1]) +if not expires or tonumber(expires) <= tonumber(ARGV[2]) then return 0 end +redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1]) +local latest = redis.call('ZREVRANGE', KEYS[1], 0, 0, 'WITHSCORES') +redis.call('PEXPIRE', KEYS[1], math.max(tonumber(ARGV[4]), tonumber(latest[2]) - tonumber(ARGV[2]) + 60000)) +return 1 +`) + +func (l *runtimeSlotLease) renew(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + c := l.coordinator + now := time.Now() + if c.redis == nil { + c.localMu.Lock() + defer c.localMu.Unlock() + if !c.localSlots[l.scope][l.token].After(now) { + return errors.New("并发租约已失效") + } + c.localSlots[l.scope][l.token] = now.Add(l.ttl) + return nil + } + ctx, cancel := context.WithTimeout(ctx, runtimeCoordinationTimeout) + defer cancel() + ok, err := renewSlotScript.Run(ctx, c.redis, []string{"canvas:slots:" + l.scope}, l.token, now.UnixMilli(), now.Add(l.ttl).UnixMilli(), (l.ttl + time.Minute).Milliseconds()).Int() + if err != nil { + return err + } + if ok != 1 { + return errors.New("并发租约已失效") + } + return nil +} + +func (l *runtimeSlotLease) release() { + l.once.Do(func() { + c := l.coordinator + if c.redis == nil { + c.localMu.Lock() + defer c.localMu.Unlock() + delete(c.localSlots[l.scope], l.token) + if len(c.localSlots[l.scope]) == 0 { + delete(c.localSlots, l.scope) + } + return + } + ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout) + defer cancel() + if err := c.redis.ZRem(ctx, "canvas:slots:"+l.scope, l.token).Err(); err != nil { + log.Printf("concurrency lease release failed: scope=%s error=%v", l.scope, err) + } + }) } func (c *runtimeCoordinator) acquireWithWait(ctx context.Context, scope string, limit int, ttl time.Duration) (func(), error) { - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() + delay := 200 * time.Millisecond for { if err := ctx.Err(); err != nil { return nil, err @@ -173,14 +255,23 @@ func (c *runtimeCoordinator) acquireWithWait(ctx context.Context, scope string, if acquired { return release, nil } + // 满载期间退避并错峰,避免每个等待者固定每秒五次同步争抢 Redis。 + wait := time.NewTimer(channelSlotRetryDelay(delay)) select { case <-ctx.Done(): + wait.Stop() return nil, ctx.Err() - case <-ticker.C: + case <-wait.C: } + delay = min(delay*2, 2*time.Second) } } +func channelSlotRetryDelay(delay time.Duration) time.Duration { + // 上限 2s,下半窗口随机等待,保留首轮至少 100ms 的退让。 + return delay/2 + time.Duration(rand.Int64N(int64(delay/2)+1)) +} + func (c *runtimeCoordinator) circuitOpen(ctx context.Context, channelID string) (bool, error) { if c.redis == nil || strings.TrimSpace(channelID) == "" { return false, nil diff --git a/backend/internal/service/coordination_redis_test.go b/backend/internal/service/coordination_redis_test.go new file mode 100644 index 000000000..b1e45cdff --- /dev/null +++ b/backend/internal/service/coordination_redis_test.go @@ -0,0 +1,86 @@ +package service + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/redis/go-redis/v9" +) + +// 只运行临时 Unix socket Redis,不读取 REDIS_URL,不连接部署环境。 +func TestRuntimeCoordinatorRedisLeasePreservesLongestTTL(t *testing.T) { + binary, err := exec.LookPath("redis-server") + if err != nil { + t.Skip("redis-server unavailable; real Redis lease integration not run") + } + // macOS 的 Unix socket 路径有长度限制,不使用包含长测试名的 t.TempDir。 + dir, err := os.MkdirTemp("", "canvas-redis-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + socket := filepath.Join(dir, "r.sock") + cmd := exec.Command(binary, "--port", "0", "--unixsocket", socket, "--unixsocketperm", "700", "--save", "", "--appendonly", "no", "--dir", dir) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + client := redis.NewClient(&redis.Options{Network: "unix", Addr: socket, MaxRetries: -1, ContextTimeoutEnabled: true}) + t.Cleanup(func() { _ = client.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for client.Ping(ctx).Err() != nil { + if ctx.Err() != nil { + t.Fatal("temporary Redis did not start") + } + time.Sleep(20 * time.Millisecond) + } + c := &runtimeCoordinator{redis: client, instanceID: "isolated-test"} + long, ok, err := c.acquireLease(ctx, "workers", 2, 45*time.Minute) + if err != nil || !ok { + t.Fatalf("legacy lease: %v %v", ok, err) + } + defer long.release() + short, ok, err := c.acquireLease(ctx, "workers", 2, time.Minute) + if err != nil || !ok { + t.Fatalf("new lease: %v %v", ok, err) + } + defer short.release() + key := "canvas:slots:workers" + assertLongestTTL := func() { + t.Helper() + ttl, err := client.PTTL(ctx, key).Result() + if err != nil || ttl < 44*time.Minute { + t.Fatalf("short lease truncated shared key TTL: %s %v", ttl, err) + } + } + assertLongestTTL() + if err := short.renew(ctx); err != nil { + t.Fatal(err) + } + assertLongestTTL() + if _, ok, err := c.acquireLease(ctx, "workers", 2, time.Minute); err != nil || ok { + t.Fatalf("global capacity exceeded: %v %v", ok, err) + } + if err := client.ZAdd(ctx, key, redis.Z{Score: float64(time.Now().Add(-time.Second).UnixMilli()), Member: short.token}).Err(); err != nil { + t.Fatal(err) + } + if err := short.renew(ctx); err == nil { + t.Fatal("expired Redis lease resurrected") + } + replacement, ok, err := c.acquireLease(ctx, "workers", 2, time.Minute) + if err != nil || !ok { + t.Fatalf("abandoned slot not reclaimed: %v %v", ok, err) + } + defer replacement.release() + short.release() + short.release() + if err := replacement.renew(ctx); err != nil { + t.Fatalf("old release removed new owner: %v", err) + } + assertLongestTTL() +} diff --git a/backend/internal/service/coordination_test.go b/backend/internal/service/coordination_test.go index 81e0d7f3d..364371912 100644 --- a/backend/internal/service/coordination_test.go +++ b/backend/internal/service/coordination_test.go @@ -6,6 +6,41 @@ import ( "time" ) +func TestRuntimeCoordinatorLeaseRenewsAndCannotResurrect(t *testing.T) { + c := &runtimeCoordinator{instanceID: "test", localSlots: map[string]map[string]time.Time{}} + lease, acquired, err := c.acquireLease(context.Background(), "workers", 1, time.Minute) + if err != nil || !acquired { + t.Fatalf("acquire: %v %v", acquired, err) + } + c.localSlots["workers"][lease.token] = time.Now().Add(10 * time.Second) + if err := lease.renew(context.Background()); err != nil { + t.Fatal(err) + } + if time.Until(c.localSlots["workers"][lease.token]) < 50*time.Second { + t.Fatal("lease was not extended") + } + if _, ok, err := c.acquireLease(context.Background(), "workers", 1, time.Minute); err != nil || ok { + t.Fatalf("capacity exceeded: %v %v", ok, err) + } + c.localSlots["workers"][lease.token] = time.Now().Add(-time.Second) + if err := lease.renew(context.Background()); err == nil { + t.Fatal("expired lease resurrected") + } + replacement, acquired, err := c.acquireLease(context.Background(), "workers", 1, time.Minute) + if err != nil || !acquired { + t.Fatalf("abandoned slot not recovered: %v %v", acquired, err) + } + lease.release() + lease.release() + if err := replacement.renew(context.Background()); err != nil { + t.Fatalf("old release removed new lease: %v", err) + } + replacement.release() + if len(c.localSlots) != 0 { + t.Fatal("empty local slot scopes retained") + } +} + func TestRuntimeCoordinatorWaitsUntilChannelSlotIsReleased(t *testing.T) { coordinator := &runtimeCoordinator{instanceID: "test", localRate: map[string]localRateEntry{}, localSlots: map[string]map[string]time.Time{}} releaseFirst, acquired, err := coordinator.acquire(context.Background(), "channel:one", 1, time.Minute) @@ -52,3 +87,13 @@ func TestRuntimeCoordinatorStopsWaitingWhenContextIsCancelled(t *testing.T) { t.Fatal("acquireWithWait() error = nil after cancellation") } } + +func TestChannelSlotRetryDelayStaysWithinBackoffWindow(t *testing.T) { + for _, base := range []time.Duration{200 * time.Millisecond, 400 * time.Millisecond, 800 * time.Millisecond, 1600 * time.Millisecond, 2 * time.Second} { + for range 100 { + if got := channelSlotRetryDelay(base); got < base/2 || got > base { + t.Fatalf("delay %s outside [%s, %s]", got, base/2, base) + } + } + } +} diff --git a/backend/internal/service/model_router.go b/backend/internal/service/model_router.go index 64d1c9828..22cf25a24 100644 --- a/backend/internal/service/model_router.go +++ b/backend/internal/service/model_router.go @@ -446,15 +446,23 @@ func isProviderCapabilityOption(name string) bool { } func (s *Service) invalidateRouteCatalog() { + s.routeCatalogRefreshMu.Lock() s.routeCatalogMu.Lock() s.routeCatalog = nil s.routeCatalogVersion++ + s.routeCatalogRetryAt = time.Time{} + s.routeCatalogRefreshError = nil s.routeCatalogMu.Unlock() + s.routeCatalogRefreshMu.Unlock() if s.coordinator != nil { - if err := s.coordinator.bumpRouteCatalogVersion(context.Background()); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout) + defer cancel() + if err := s.coordinator.bumpRouteCatalogVersion(ctx); err != nil { log.Printf("logical model route catalog distributed invalidation failed: %v", err) } } + s.initReadCaches() + s.routeVersionReadCache.clear() } func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) { @@ -480,15 +488,26 @@ func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) { } s.routeCatalogMu.RUnlock() + // 刷新锁只能防止并行回源,失败后还需要冷却,否则等待者会逐个重打数据库。 + if s.routeCatalogRefreshError != nil && now.Before(s.routeCatalogRetryAt) { + if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale { + return snapshot, nil + } + return nil, s.routeCatalogRefreshError + } loaded, err := s.loadRouteCatalog() if err != nil { + s.routeCatalogRefreshError = err + s.routeCatalogRetryAt = time.Now().Add(2 * time.Second) + log.Printf("logical model route catalog refresh failed; retry cooled down: %v", err) // 已有快照过期时允许短暂继续服务,数据库首次加载失败则明确失败。 - if snapshot != nil && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale { - log.Printf("logical model route catalog refresh failed, serving stale snapshot age=%s: %v", now.Sub(snapshot.LoadedAt).Round(time.Second), err) + if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale { return snapshot, nil } return nil, err } + s.routeCatalogRefreshError = nil + s.routeCatalogRetryAt = time.Time{} s.routeCatalogMu.Lock() s.routeCatalog = loaded s.routeCatalogMu.Unlock() @@ -502,11 +521,17 @@ func (s *Service) currentRouteCatalogVersion() int64 { if s.coordinator == nil || s.coordinator.redis == nil { return localVersion } - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + s.initReadCaches() + ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout) defer cancel() - version, err := s.coordinator.routeCatalogVersion(ctx) + version, err := s.routeVersionReadCache.get(ctx, routeCatalogVersionKey, func(ctx context.Context) (int64, int, error) { + value, err := s.coordinator.routeCatalogVersion(ctx) + if err != nil { + log.Printf("logical model route catalog distributed version check failed: %v", err) + } + return value, 256, err + }) if err != nil { - log.Printf("logical model route catalog distributed version check failed: %v", err) return localVersion } if version > localVersion { @@ -516,12 +541,15 @@ func (s *Service) currentRouteCatalogVersion() int64 { } func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) { - items, err := s.repo.LogicalModels(false) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + repo := s.repo.WithContext(ctx) + items, err := repo.LogicalModels(false) if err != nil { return nil, err } snapshot := &routeCatalogSnapshot{LoadedAt: time.Now(), CatalogVersion: s.currentRouteCatalogVersion(), Models: make(map[string]cachedLogicalModel), Ordered: make([]string, 0, len(items))} - graphs, err := s.repo.LogicalModelGraphs(items, false) + graphs, err := repo.LogicalModelGraphs(items, false) if err != nil { return nil, err } @@ -534,7 +562,7 @@ func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) { systemChannelIDs = append(systemChannelIDs, channelModel.ChannelID) } } - systemChannels, err := s.repo.SystemChannelsByIDs(systemChannelIDs, false) + systemChannels, err := repo.SystemChannelsByIDs(systemChannelIDs, false) if err != nil { return nil, err } diff --git a/backend/internal/service/read_cache.go b/backend/internal/service/read_cache.go new file mode 100644 index 000000000..33531cafa --- /dev/null +++ b/backend/internal/service/read_cache.go @@ -0,0 +1,147 @@ +package service + +import ( + "container/list" + "context" + "errors" + "sync" + "time" +) + +var errReadCacheBusy = errors.New("热点读取正在回源,请稍后重试") +var errReadCacheLoadAborted = errors.New("缓存回源异常中断") + +// 仅用于可短暂延迟的读投影。限制条目、正文预算和同时回源数;失败短暂冷却, +// 不返回过期成功值,也不允许大量不同 key 将 singleflight 本身变成无界队列。 +type boundedReadCache[K comparable, V any] struct { + mu sync.Mutex + entries map[K]*readCacheEntry[K, V] + lru list.List + bytes, loads int + maxEntries, maxBytes, maxLoads int + ttl time.Duration +} + +type readCacheEntry[K comparable, V any] struct { + key K + ready chan struct{} + value V + err error + expires time.Time + bytes int + position *list.Element +} + +func newBoundedReadCache[K comparable, V any](entries, bytes, loads int, ttl time.Duration) *boundedReadCache[K, V] { + return &boundedReadCache[K, V]{entries: make(map[K]*readCacheEntry[K, V]), maxEntries: entries, maxBytes: bytes, maxLoads: loads, ttl: ttl} +} + +func (c *boundedReadCache[K, V]) get(ctx context.Context, key K, load func(context.Context) (V, int, error)) (V, error) { + var zero V + if err := ctx.Err(); err != nil { + return zero, err + } + c.mu.Lock() + if e := c.entries[key]; e != nil { + if ready := e.ready; ready != nil { + c.mu.Unlock() + select { + case <-ctx.Done(): + return zero, ctx.Err() + case <-ready: + return e.value, e.err + } + } + if time.Now().Before(e.expires) { + c.lru.MoveToFront(e.position) + value, err := e.value, e.err + c.mu.Unlock() + return value, err + } + c.remove(e) + } + if c.loads >= c.maxLoads { + c.mu.Unlock() + return zero, errReadCacheBusy + } + for len(c.entries) >= c.maxEntries { + if !c.evict() { + c.mu.Unlock() + return zero, errReadCacheBusy + } + } + e := &readCacheEntry[K, V]{key: key, ready: make(chan struct{})} + c.entries[key] = e + c.loads++ + c.mu.Unlock() + + // 让原始 panic 继续传播给调用方,同时释放等待者和容量,避免永久卡住热点 key。 + loaded := false + defer func() { + if loaded { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.loads-- + e.err = errReadCacheLoadAborted + if c.entries[key] == e { + delete(c.entries, key) + } + close(e.ready) + e.ready = nil + }() + value, bytes, err := load(ctx) + loaded = true + c.mu.Lock() + defer c.mu.Unlock() + c.loads-- + e.value, e.err = value, err + ready := e.ready + e.ready = nil + if c.entries[key] == e { + ttl := c.ttl + if err != nil { + ttl, bytes = time.Second, 256 + } + bytes = max(bytes, 256) + if bytes > c.maxBytes || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + delete(c.entries, key) + } else { + for c.bytes+bytes > c.maxBytes { + if !c.evict() { + break + } + } + e.bytes, e.expires = bytes, time.Now().Add(ttl) + e.position = c.lru.PushFront(e) + c.bytes += bytes + } + } + close(ready) + return value, err +} + +func (c *boundedReadCache[K, V]) evict() bool { + if oldest := c.lru.Back(); oldest != nil { + c.remove(oldest.Value.(*readCacheEntry[K, V])) + return true + } + return false +} + +func (c *boundedReadCache[K, V]) remove(e *readCacheEntry[K, V]) { + delete(c.entries, e.key) + if e.position != nil { + c.lru.Remove(e.position) + c.bytes -= e.bytes + } +} + +func (c *boundedReadCache[K, V]) clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.entries = make(map[K]*readCacheEntry[K, V]) + c.lru.Init() + c.bytes = 0 +} diff --git a/backend/internal/service/read_cache_integration_test.go b/backend/internal/service/read_cache_integration_test.go new file mode 100644 index 000000000..148a4310a --- /dev/null +++ b/backend/internal/service/read_cache_integration_test.go @@ -0,0 +1,165 @@ +package service + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/repository" +) + +func TestCachedTextReplayIsolatesUsersCursorsAndCopies(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+newID()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Task{}, &model.TaskTextDelta{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.Task{ID: "task", UserID: "user", Type: "canvas_text", Status: model.TaskStatusRunning}).Error; err != nil { + t.Fatal(err) + } + for i, content := range []string{"one", "two"} { + if err := db.Create(&model.TaskTextDelta{ID: content, UserID: "user", TaskID: "task", Sequence: int64(i + 1), Content: content, ExpiresAt: time.Now().Add(time.Hour)}).Error; err != nil { + t.Fatal(err) + } + } + svc := &Service{repo: repository.New(db)} + var queries atomic.Int32 + if err := db.Callback().Query().Before("gorm:query").Register("cache-count", func(*gorm.DB) { queries.Add(1) }); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for range 50 { + wg.Add(1) + go func() { + defer wg.Done() + result, err := svc.CachedTaskTextReplay(context.Background(), "user", "task", 0) + if err != nil { + t.Error(err) + return + } + if len(result.Deltas) != 2 || result.Deltas[0].Content != "one" { + t.Errorf("unexpected replay: %#v", result) + return + } + result.Deltas[0].Content = "caller mutation" + }() + } + wg.Wait() + if queries.Load() != 2 { + t.Fatalf("expected one task+delta query, got %d queries", queries.Load()) + } + if _, err := svc.CachedTaskTextReplay(context.Background(), "other", "task", 0); err == nil { + t.Fatal("cross-user cache leak") + } + result, err := svc.CachedTaskTextReplay(context.Background(), "user", "task", 1) + if err != nil || len(result.Deltas) != 1 || result.Deltas[0].Sequence != 2 { + t.Fatalf("cursor isolation: %#v %v", result, err) + } + svc.textReplayReadCache.clear() + if err := db.Model(&model.Task{}).Where("id = ?", "task").Update("status", model.TaskStatusSucceeded).Error; err != nil { + t.Fatal(err) + } + result, err = svc.CachedTaskTextReplay(context.Background(), "user", "task", 0) + if err != nil || !result.Complete { + t.Fatalf("terminal refresh: %#v %v", result, err) + } +} + +func TestRuntimeConcurrencyCacheDoesNotCacheAuthoritativePolicy(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.SystemSetting{}, &model.AdminAuditEvent{}); err != nil { + t.Fatal(err) + } + svc := &Service{repo: repository.New(db)} + actor := &model.User{ID: "admin", Role: model.UserRoleAdmin} + var queries atomic.Int32 + if err := db.Callback().Query().Before("gorm:query").Register("policy-count", func(*gorm.DB) { queries.Add(1) }); err != nil { + t.Fatal(err) + } + for range 20 { + if _, err := svc.runtimeConcurrencySetting(); err != nil { + t.Fatal(err) + } + } + if queries.Load() != 1 { + t.Fatalf("concurrency reads hit DB %d times", queries.Load()) + } + before := queries.Load() + for range 2 { + if _, err := svc.RuntimePolicy(); err != nil { + t.Fatal(err) + } + } + if queries.Load() != before+2 { + t.Fatal("authoritative policy incorrectly cached") + } + policy := defaultRuntimePolicy() + policy.Task.WorkerConcurrency = 7 + if _, err := svc.UpdateRuntimePolicySetting(actor, policy); err != nil { + t.Fatal(err) + } + setting, err := svc.runtimeConcurrencySetting() + if err != nil || setting.WorkerConcurrency != 7 { + t.Fatalf("save did not invalidate: %#v %v", setting, err) + } + if _, err := svc.ResetRuntimePolicySetting(actor); err != nil { + t.Fatal(err) + } + setting, err = svc.runtimeConcurrencySetting() + if err != nil || setting.WorkerConcurrency != defaultRuntimePolicy().Task.WorkerConcurrency { + t.Fatalf("reset did not invalidate: %#v %v", setting, err) + } +} + +func TestRouteCatalogFailureStormAndInvalidatedStaleSnapshot(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + wantErr := errors.New("injected database outage") + var queries atomic.Int32 + if err := db.Callback().Query().Before("gorm:query").Register("catalog-outage", func(tx *gorm.DB) { queries.Add(1); tx.AddError(wantErr) }); err != nil { + t.Fatal(err) + } + svc := &Service{repo: repository.New(db), routeCatalogTTL: time.Second, routeCatalogMaxStale: time.Minute} + var wg sync.WaitGroup + for range 50 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := svc.routeCatalogSnapshot(); !errors.Is(err, wantErr) { + t.Errorf("outage error lost: %v", err) + } + }() + } + wg.Wait() + if queries.Load() != 1 { + t.Fatalf("failure storm retried DB %d times", queries.Load()) + } + svc.routeCatalog = &routeCatalogSnapshot{LoadedAt: time.Now().Add(-2 * time.Second), CatalogVersion: 0} + if _, err := svc.routeCatalogSnapshot(); err != nil { + t.Fatalf("existing same-version bounded fallback lost: %v", err) + } + svc.routeCatalogVersion = 1 + if _, err := svc.routeCatalogSnapshot(); !errors.Is(err, wantErr) { + t.Fatal("explicitly invalidated version was served stale") + } + svc.invalidateRouteCatalog() + if _, err := svc.routeCatalogSnapshot(); !errors.Is(err, wantErr) { + t.Fatal(err) + } + if queries.Load() != 2 { + t.Fatal("invalidation failed to reset refresh cooldown") + } +} diff --git a/backend/internal/service/read_cache_test.go b/backend/internal/service/read_cache_test.go new file mode 100644 index 000000000..e2d2f99d4 --- /dev/null +++ b/backend/internal/service/read_cache_test.go @@ -0,0 +1,135 @@ +package service + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestReadCacheCoalescesBurstAndCoolsFailures(t *testing.T) { + for _, fail := range []bool{false, true} { + cache := newBoundedReadCache[string, int](4, 1024, 2, time.Minute) + var calls atomic.Int32 + loadErr := errors.New("database unavailable") + load := func(context.Context) (int, int, error) { + calls.Add(1) + if fail { + return 0, 0, loadErr + } + return 42, 256, nil + } + var wg sync.WaitGroup + for range 100 { + wg.Add(1) + go func() { + defer wg.Done() + value, err := cache.get(context.Background(), "hot", load) + if (fail && !errors.Is(err, loadErr)) || (!fail && (err != nil || value != 42)) { + t.Errorf("value=%d error=%v fail=%v", value, err, fail) + } + }() + } + wg.Wait() + if calls.Load() != 1 { + t.Fatalf("burst caused %d loads (fail=%v)", calls.Load(), fail) + } + } +} + +func TestReadCacheBoundsLoadsAndInvalidation(t *testing.T) { + cache := newBoundedReadCache[string, int](2, 512, 1, time.Minute) + started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + _, _ = cache.get(context.Background(), "one", func(context.Context) (int, int, error) { + close(started) + <-release + return 1, 256, nil + }) + }() + <-started + load := func(context.Context) (int, int, error) { return 2, 256, nil } + if _, err := cache.get(context.Background(), "two", load); !errors.Is(err, errReadCacheBusy) { + t.Fatalf("unbounded distinct load: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if _, err := cache.get(ctx, "one", load); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("waiting reader did not cancel: %v", err) + } + cache.clear() + if _, err := cache.get(context.Background(), "two", load); !errors.Is(err, errReadCacheBusy) { + t.Fatalf("invalidation bypassed in-flight limit: %v", err) + } + close(release) + <-done + value, err := cache.get(context.Background(), "one", load) + if err != nil || value != 2 { + t.Fatalf("in-flight load repopulated invalidated cache: %d %v", value, err) + } +} + +func TestReadCacheBudgetsExpiryAndNoStaleSuccess(t *testing.T) { + cache := newBoundedReadCache[int, int](3, 512, 1, time.Minute) + load := func(context.Context) (int, int, error) { return 1, 256, nil } + for i := range 100 { + if _, err := cache.get(context.Background(), i, load); err != nil { + t.Fatal(err) + } + } + if len(cache.entries) > 2 || cache.bytes > 512 { + t.Fatalf("budget exceeded: entries=%d bytes=%d", len(cache.entries), cache.bytes) + } + cache.entries[99].expires = time.Now().Add(-time.Second) + wantErr := errors.New("load failed") + value, err := cache.get(context.Background(), 99, func(context.Context) (int, int, error) { return 0, 0, wantErr }) + if value != 0 || !errors.Is(err, wantErr) { + t.Fatalf("stale success served: %d %v", value, err) + } + var calls int + for range 2 { + value, err = cache.get(context.Background(), 101, func(context.Context) (int, int, error) { + calls++ + return 42, 1024, nil + }) + if value != 42 || err != nil { + t.Fatalf("oversized load should still return to caller: %d %v", value, err) + } + } + if calls != 2 { + t.Fatal("oversized entry was retained") + } +} + +func TestReadCacheCancelledLoadCanBeRetried(t *testing.T) { + cache := newBoundedReadCache[string, int](1, 512, 1, time.Minute) + _, err := cache.get(context.Background(), "key", func(context.Context) (int, int, error) { return 0, 0, context.Canceled }) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + value, err := cache.get(context.Background(), "key", func(context.Context) (int, int, error) { return 7, 256, nil }) + if err != nil || value != 7 { + t.Fatalf("cancellation poisoned cache: %d %v", value, err) + } +} + +func TestReadCachePanicDoesNotLeakCapacity(t *testing.T) { + cache := newBoundedReadCache[string, int](1, 512, 1, time.Minute) + func() { + defer func() { + if recover() != "loader panic" { + t.Error("original panic was swallowed") + } + }() + _, _ = cache.get(context.Background(), "key", func(context.Context) (int, int, error) { panic("loader panic") }) + }() + if cache.loads != 0 || len(cache.entries) != 0 { + t.Fatal("panic leaked load capacity") + } + if _, err := cache.get(context.Background(), "key", func(context.Context) (int, int, error) { return 1, 256, nil }); err != nil { + t.Fatal(err) + } +} diff --git a/backend/internal/service/runtime_policy.go b/backend/internal/service/runtime_policy.go index 66df7d4e9..1ec378def 100644 --- a/backend/internal/service/runtime_policy.go +++ b/backend/internal/service/runtime_policy.go @@ -1,6 +1,7 @@ package service import ( + "context" "encoding/json" "errors" "fmt" @@ -187,8 +188,14 @@ func (s *Service) RuntimePolicy() (RuntimePolicySetting, error) { } func (s *Service) runtimeConcurrencySetting() (RuntimeTaskPolicy, error) { - policy, err := s.RuntimePolicy() - return policy.Task, err + s.initReadCaches() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + return s.concurrencyReadCache.get(ctx, runtimePolicySettingKey, func(ctx context.Context) (RuntimeTaskPolicy, int, error) { + reader := &Service{repo: s.repo.WithContext(ctx)} + policy, err := reader.RuntimePolicy() + return policy.Task, 256, err + }) } func (s *Service) PublicRuntimeLimits() (*PublicRuntimeLimits, error) { @@ -243,6 +250,8 @@ func (s *Service) UpdateRuntimePolicySetting(actor *model.User, value RuntimePol if err := s.repo.SaveSystemSetting(&setting); err != nil { return nil, err } + s.initReadCaches() + s.concurrencyReadCache.clear() if err := s.appendAdminAudit(actor, "runtime_policy.update", "system_setting", runtimePolicySettingKey, "更新资源与请求策略", map[string]any{"before": before, "after": value}); err != nil { return nil, err } @@ -260,6 +269,8 @@ func (s *Service) ResetRuntimePolicySetting(actor *model.User) (*PublicRuntimePo if err := s.repo.DeleteSystemSetting(runtimePolicySettingKey); err != nil { return nil, err } + s.initReadCaches() + s.concurrencyReadCache.clear() after := defaultRuntimePolicy() if err := s.appendAdminAudit(actor, "runtime_policy.reset", "system_setting", runtimePolicySettingKey, "重置资源与请求策略", map[string]any{"before": before, "after": after}); err != nil { return nil, err diff --git a/backend/internal/service/service.go b/backend/internal/service/service.go index 90309ffae..eeaafeb2d 100644 --- a/backend/internal/service/service.go +++ b/backend/internal/service/service.go @@ -56,6 +56,12 @@ type Service struct { workers *workerRuntime updateManager UpdateManager mailSender func(emailSettingValue, string, string, string) error + readCachesOnce sync.Once + concurrencyReadCache *boundedReadCache[string, RuntimeTaskPolicy] + textReplayReadCache *boundedReadCache[textReplayCacheKey, *TextReplayResult] + routeVersionReadCache *boundedReadCache[string, int64] + routeCatalogRetryAt time.Time + routeCatalogRefreshError error } const taskWorkerConcurrency = 3 diff --git a/backend/internal/service/storyboard_workflow.go b/backend/internal/service/storyboard_workflow.go index f92df3cb7..3f4c40de5 100644 --- a/backend/internal/service/storyboard_workflow.go +++ b/backend/internal/service/storyboard_workflow.go @@ -72,7 +72,7 @@ func (s *Service) repairStoryboardPlan(ctx context.Context, task model.Task, inp currentErr := validationErr for attempt := 1; attempt <= maxStoryboardRepairAttempts; attempt++ { _ = s.log(task.UserID, task.ID, "warn", "分镜结构校验失败", fmt.Sprintf("第 %d 次修复前:%s", attempt, currentErr.Error())) - if err := s.repo.UpdateTaskProgress(task.ID, "修复分镜结构", 55+attempt*10); err != nil { + if err := s.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, "修复分镜结构", 55+attempt*10); err != nil { return agentStoryboardPlan{}, fmt.Errorf("更新分镜修复进度失败,上游修复请求未发出:%w", err) } repairPrompt, promptErr := s.buildStoryboardRepairPrompt(task.UserID, task.Prompt, currentErr, input, currentText) diff --git a/backend/internal/service/task_terminal.go b/backend/internal/service/task_terminal.go index e34655c77..f53c7fc13 100644 --- a/backend/internal/service/task_terminal.go +++ b/backend/internal/service/task_terminal.go @@ -27,7 +27,7 @@ type taskTerminalCoordinator struct { type taskTerminalRepository interface { Task(id string) (*model.Task, error) - UpdateTaskTerminalState(id string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) + UpdateTaskTerminalState(id string, owner string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) } type taskBillingLifecycle interface { @@ -248,9 +248,13 @@ func (c *taskTerminalCoordinator) handleSuccess(task *model.Task) error { func (c *taskTerminalCoordinator) markTerminalState(task *model.Task) error { completedAt := time.Now() task.CompletedAt = &completedAt - if _, err := c.repo.UpdateTaskTerminalState(task.ID, model.TaskStatusRunning, task.Status, task.Stage, task.Error, completedAt); err != nil { + updated, err := c.repo.UpdateTaskTerminalState(task.ID, task.LeaseOwner, model.TaskStatusRunning, task.Status, task.Stage, task.Error, completedAt) + if err != nil { return fmt.Errorf("写入任务终态失败:%w", err) } + if !updated { + return repository.ErrTaskStateConflict + } return nil } diff --git a/backend/internal/service/task_terminal_test.go b/backend/internal/service/task_terminal_test.go index 43001d8b8..b2803d44f 100644 --- a/backend/internal/service/task_terminal_test.go +++ b/backend/internal/service/task_terminal_test.go @@ -7,13 +7,15 @@ import ( "time" "infinite-canvas/backend/internal/model" + "infinite-canvas/backend/internal/repository" ) type taskTerminalRepositoryStub struct { - task *model.Task - taskError error - terminalCalls int - terminalError error + task *model.Task + taskError error + terminalCalls int + terminalError error + terminalConflict bool } func (r *taskTerminalRepositoryStub) Task(string) (*model.Task, error) { @@ -27,8 +29,11 @@ func (r *taskTerminalRepositoryStub) Task(string) (*model.Task, error) { return ©, nil } -func (r *taskTerminalRepositoryStub) UpdateTaskTerminalState(_ string, _ model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) { +func (r *taskTerminalRepositoryStub) UpdateTaskTerminalState(_ string, _ string, _ model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) { r.terminalCalls++ + if r.terminalConflict { + return false, nil + } if r.terminalError != nil { return false, r.terminalError } @@ -41,6 +46,21 @@ func (r *taskTerminalRepositoryStub) UpdateTaskTerminalState(_ string, _ model.T return true, nil } +func TestTaskTerminalConflictDoesNotRefundOrFinalize(t *testing.T) { + repo := &taskTerminalRepositoryStub{terminalConflict: true} + billing, replay := &taskTerminalBillingStub{}, &taskTerminalReplayStub{} + sessions, logger := &taskTerminalSessionStub{}, &taskTerminalLoggerStub{} + c := newTaskTerminalCoordinatorForTest(repo, billing, replay, sessions, logger, &taskTerminalOutputStub{}) + task := &model.Task{ID: "task", LeaseOwner: "stale", Status: model.TaskStatusRunning} + err := c.handleExecutionFailure(task, errors.New("upstream failed"), false, false) + if !errors.Is(err, repository.ErrTaskStateConflict) { + t.Fatalf("missing conflict: %v", err) + } + if len(billing.refund)+len(billing.uncertain)+billing.settleCalls+len(replay.statuses)+len(sessions.messages) != 0 { + t.Fatal("stale worker performed terminal side effects") + } +} + type taskTerminalBillingStub struct { uncertain []string refund []string diff --git a/backend/internal/service/task_timeline.go b/backend/internal/service/task_timeline.go index 8eafbb302..9b7d80bcf 100644 --- a/backend/internal/service/task_timeline.go +++ b/backend/internal/service/task_timeline.go @@ -99,19 +99,19 @@ func (w *taskWorkerCoordinator) processTimelineTranscription(task *model.Task, c func (w *taskWorkerCoordinator) failTimelineTask(task *model.Task, stage string, message string) error { s := w.service - done, err := s.repo.UpdateTaskTerminalState(task.ID, model.TaskStatusRunning, model.TaskStatusFailed, stage, message, time.Now()) + done, err := s.repo.UpdateTaskTerminalState(task.ID, task.LeaseOwner, model.TaskStatusRunning, model.TaskStatusFailed, stage, message, time.Now()) if err != nil { return fmt.Errorf("写入转写失败态失败: %w", err) } - s.logInfo(task.UserID, task.ID, fmt.Sprintf("时间线转写失败: %s", message), "") if !done { - return fmt.Errorf("时间线转写已失败: %s", message) + return fmt.Errorf("时间线任务状态或租约已变化:%w", repository.ErrTaskStateConflict) } + s.logInfo(task.UserID, task.ID, fmt.Sprintf("时间线转写失败: %s", message), "") return nil } func (w *taskWorkerCoordinator) progress(task *model.Task, stage string, progress int) error { - if err := w.service.repo.UpdateTaskProgress(task.ID, stage, progress); err != nil { + if err := w.service.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, stage, progress); err != nil { return fmt.Errorf("更新转写进度失败: %w", err) } return nil diff --git a/backend/internal/service/task_worker.go b/backend/internal/service/task_worker.go index 6613f3626..e3614f86a 100644 --- a/backend/internal/service/task_worker.go +++ b/backend/internal/service/task_worker.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "strings" "time" @@ -16,6 +17,8 @@ type taskWorkerCoordinator struct { service *Service } +const workerSlotLeaseDuration = time.Minute + func newTaskWorkerCoordinator(service *Service) *taskWorkerCoordinator { return &taskWorkerCoordinator{service: service} } @@ -41,30 +44,40 @@ func (w *taskWorkerCoordinator) start(ctx context.Context) { } setting, err := s.runtimeConcurrencySetting() if err != nil { + log.Printf("task dispatch paused: stage=runtime_policy worker_id=%s error=%v", s.workerID, err) return } workerConcurrency := setting.WorkerConcurrency for len(slots) < workerConcurrency { - releaseGlobal, acquired, err := s.coordinator.acquire(ctx, "workers", workerConcurrency, 45*time.Minute) + globalSlot, acquired, err := s.coordinator.acquireLease(ctx, "workers", workerConcurrency, workerSlotLeaseDuration) if err != nil || !acquired { + if err != nil { + log.Printf("task dispatch paused: stage=global_slot worker_id=%s error=%v", s.workerID, err) + } return } - task, err := s.repo.ClaimNextTask(s.workerID, 45*time.Second) + claimCtx, cancelClaim := context.WithTimeout(ctx, 5*time.Second) + // 每次领取使用独立 owner,防止同一进程内旧执行者恢复后覆盖新执行者。 + task, err := s.repo.WithContext(claimCtx).ClaimNextTask(globalSlot.token, 45*time.Second) + cancelClaim() if err != nil || task == nil { - releaseGlobal() + globalSlot.release() + if err != nil { + log.Printf("task dispatch paused: stage=claim worker_id=%s error=%v", s.workerID, err) + } return } slots <- struct{}{} started := s.runWorkerTask(func() { - defer func() { <-slots; releaseGlobal() }() - if err := w.processClaimedTask(task); err != nil { + defer func() { <-slots; globalSlot.release() }() + if err := w.processClaimedTask(task, globalSlot); err != nil { _ = s.log(task.UserID, task.ID, "error", "后台任务处理失败", err.Error()) } }) if !started { <-slots - releaseGlobal() - _ = s.repo.ReleaseTaskLease(task.ID, s.workerID) + globalSlot.release() + _ = s.repo.ReleaseTaskLease(task.ID, task.LeaseOwner) return } } @@ -86,18 +99,20 @@ func (w *taskWorkerCoordinator) start(ctx context.Context) { func (w *taskWorkerCoordinator) processNextTask() error { s := w.service - task, err := s.repo.ClaimNextTask(s.workerID, 45*time.Second) + task, err := s.repo.ClaimNextTask(s.workerID+":"+newID(), 45*time.Second) if err != nil || task == nil { return err } - return w.processClaimedTask(task) + return w.processClaimedTask(task, nil) } -func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error { +func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task, globalSlot *runtimeSlotLease) error { s := w.service terminal := s.terminalCoordinator() - _ = s.log(task.UserID, task.ID, "info", "后端任务开始处理", "") - policy, err := s.RuntimePolicy() + policyCtx, cancelPolicy := context.WithTimeout(context.Background(), 3*time.Second) + reader := &Service{repo: s.repo.WithContext(policyCtx)} + policy, err := reader.RuntimePolicy() + cancelPolicy() if err != nil { return err } @@ -105,13 +120,23 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error { defer cancel() leaseDone := make(chan struct{}) leaseLost := make(chan error, 1) + taskID, leaseOwner := task.ID, task.LeaseOwner go func() { ticker := time.NewTicker(15 * time.Second) defer ticker.Stop() for { select { case <-ticker.C: - if err := s.repo.RenewTaskLease(task.ID, s.workerID, 45*time.Second); err != nil { + renewCtx, cancelRenew := context.WithTimeout(ctx, 5*time.Second) + var err error + if globalSlot != nil { + err = globalSlot.renew(renewCtx) + } + if err == nil { + err = s.repo.WithContext(renewCtx).RenewTaskLease(taskID, leaseOwner, 45*time.Second) + } + cancelRenew() + if err != nil { leaseLost <- err cancel() return @@ -122,6 +147,7 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error { } }() defer close(leaseDone) + _ = s.log(task.UserID, task.ID, "info", "后端任务开始处理", "") s.registerActiveTask(task.ID, cancel) defer s.unregisterActiveTask(task.ID) // 取消请求可能在任务领取和注册 worker context 之间到达;再次读取终态 @@ -147,7 +173,7 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error { task.Stage = "正在连接上游" task.Progress = 0 } - if err := s.repo.UpdateTaskProgress(task.ID, task.Stage, task.Progress); err != nil { + if err := s.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, task.Stage, task.Progress); err != nil { return fmt.Errorf("更新任务进度失败,任务暂未调用上游:%w", err) } routeAttempt, err := s.beginTaskRouteAttempt(task) @@ -161,6 +187,11 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error { if stateErr != nil { return stateErr } + select { + case leaseErr := <-leaseLost: + return fmt.Errorf("任务租约失效,停止保存上游结果:%w", leaseErr) + default: + } result, canvasOps, err := routeResult.result, routeResult.canvasOps, routeResult.err providerSucceeded := routeResult.providerSucceeded if err == nil { diff --git a/backend/internal/service/text_replay_cache.go b/backend/internal/service/text_replay_cache.go new file mode 100644 index 000000000..5527ead0e --- /dev/null +++ b/backend/internal/service/text_replay_cache.go @@ -0,0 +1,51 @@ +package service + +import ( + "context" + "errors" + "time" + + "infinite-canvas/backend/internal/model" +) + +type textReplayCacheKey struct { + userID, taskID string + after int64 +} + +func (s *Service) initReadCaches() { + s.readCachesOnce.Do(func() { + s.concurrencyReadCache = newBoundedReadCache[string, RuntimeTaskPolicy](1, 1024, 1, 2*time.Second) + s.textReplayReadCache = newBoundedReadCache[textReplayCacheKey, *TextReplayResult](256, 8<<20, 8, 750*time.Millisecond) + s.routeVersionReadCache = newBoundedReadCache[string, int64](1, 1024, 1, 250*time.Millisecond) + }) +} + +// SSE 展示专用,不用于写路径和权限决策;userID 进入 key,回源仍检查任务归属。 +// 游标保留在 key 中,避免重连读取到其他客户端已经越过的增量窗口。 +func (s *Service) CachedTaskTextReplay(ctx context.Context, userID, taskID string, after int64) (*TextReplayResult, error) { + s.initReadCaches() + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + value, err := s.textReplayReadCache.get(ctx, textReplayCacheKey{userID, taskID, after}, func(ctx context.Context) (*TextReplayResult, int, error) { + reader := &Service{repo: s.repo.WithContext(ctx)} + value, err := reader.TaskTextReplay(userID, taskID, after) + if err != nil { + return nil, 0, err + } + bytes := 512 + len(value.TextDraft) + len(value.FinalText) + len(value.Error) + len(value.Stage) + for _, delta := range value.Deltas { + bytes += 256 + len(delta.Content) + len(delta.ID) + len(delta.UserID) + len(delta.TaskID) + } + return value, bytes, nil + }) + if errors.Is(err, errReadCacheBusy) { + return nil, &AppError{Status: 503, Code: 503, Message: "任务状态查询繁忙,请稍后重试", Retryable: true} + } + if err != nil { + return nil, err + } + copy := *value + copy.Deltas = append([]model.TaskTextDelta(nil), value.Deltas...) + return ©, nil +} diff --git a/docs/content/docs/backend/meta.json b/docs/content/docs/backend/meta.json index ad9affd2d..da217b31d 100644 --- a/docs/content/docs/backend/meta.json +++ b/docs/content/docs/backend/meta.json @@ -6,6 +6,7 @@ "local-development", "code-map", "backend-database", + "task-runtime-stability", "system-update", "protocol-plugins", "canvas-data-structure" diff --git a/docs/content/docs/backend/task-runtime-stability.mdx b/docs/content/docs/backend/task-runtime-stability.mdx new file mode 100644 index 000000000..74e8029ea --- /dev/null +++ b/docs/content/docs/backend/task-runtime-stability.mdx @@ -0,0 +1,63 @@ +--- +title: 任务缓存与队列稳定性 +description: 热点回源保护、并发租约、故障边界和上线验收 +--- + +# 任务缓存与队列稳定性 + +## 范围与事实源 + +当前任务仍由数据库持久化、API 进程内 worker 领取和执行,Redis 负责跨实例并发协调。本轮是热点读取和租约的止血修复,不是独立任务系统迁移,也不能替代线上事故取证。 + +- 数据库继续作为任务、额度、账务和权限的事实源。没有把任务或账本迁入 Redis,没有修改表结构或新增迁移。 +- 新增缓存是每个后端实例的有界内存读缓存,不是通用 Redis 业务缓存。实例增加时,缓存容量与不同实例的回源量会随之增加。 +- handler 负责 SSE 协议和登录上下文;service 负责读缓存、并发协调和执行生命周期;repository 负责有条件的租约写入与事务。 + +## 热点读取保护 + +| 读路径 | 缓存 / 回源限制 | 故障行为 | +| --- | --- | --- | +| worker / 渠道全局并发策略 | 2 秒 TTL,1 条目、1 个同时回源,正文预算 1 KiB | 读取失败不使用默认高并发;暂停投递或返回明确错误 | +| 文本任务 SSE 回放 | 750 毫秒 TTL,256 条目、8 个不同 key 同时回源,正文预算 8 MiB | 同 key 合并回源;超过容量返回繁忙,不创建无界的不同 key 等待队列 | +| Redis 路由版本检查 | 250 毫秒 TTL,1 个同时回源 | 失败保留原有本地版本回退;不保证 Redis 故障时跨实例实时失效 | +| 模型路由目录 | 沿用 30 秒 TTL,刷新串行,失败冷却 2 秒,数据库回源期限 5 秒 | 首次失败明确返回错误;已有快照只能在同版本、最多 5 分钟的原有过期窗口内降级 | + +读缓存按 LRU 淘汰,正文预算是估算值,不等于进程总内存上限;回源中的数据、当前请求和等待中的连接不包含在正文预算内。超大结果仅返回给当前读取者,不保留为缓存。普通失败冷却 1 秒,不返回过期成功值;请求取消和超时不缓存。回源异常退出释放容量和等待者,原始 panic 仍交给原有错误处理。 + +并发策略保存或重置成功后立即清除本实例缓存,其他实例在 2 秒 TTL 后重新读取。计费、配额和其他调用 `RuntimePolicy()` 的强校验路径仍直接读取数据库。 + +SSE 缓存 key 包含用户、任务和 `after` 游标,回源仍检查归属;返回副本避免调用者改写其他连接的增量。`GET /api/tasks/:id/text-events` 的事件格式、递增序号、`Last-Event-ID` / `?after=` 和 15 秒心跳不变。终态最多受短缓存窗口延迟;已经建立的流遇到回源失败仍发送 error 并关闭。`/text-deltas` 普通查询没有纳入此缓存。 + +这是合并相同游标的轮询读取,不是事件驱动推送;不同游标和不同实例仍可能分别查询数据库。连接总量、客户端重连退避及任务删除后的缓存失效仍需后续治理。 + +## worker 与渠道槽位 + +- worker 全局槽位由固定 45 分钟 TTL 改为 1 分钟租约,每 15 秒与数据库任务租约一起续期;数据库任务租约仍为 45 秒。 +- 每次领取生成独立 owner,不再让一个进程的所有执行共享 owner。领取数据库操作有 5 秒期限,启动策略读取有 3 秒期限。 +- Redis 协调客户端关闭透明命令重试;槽位领取、续期与释放使用最多 2 秒期限。Redis 获取槽位失败时不降级成本机无限执行。 +- 槽位丢失、过期或续期失败会取消该执行的 context。已经失效的槽位不能通过续期复活;释放幂等且只移除本次 token。释放失败记录日志,由租约过期回收。 +- 共享 Redis 槽位键按集合内最晚的租约保留过期余量,避免短任务或新版本的短租约把旧长任务槽位提前删除。旧版本脚本本身不会获得这一保护,仍建议排空后发布。 +- 渠道满载时从 200 毫秒开始指数退避,上限 2 秒,每轮在半窗口至完整窗口间随机等待,避免固定每秒五次同步争抢。空闲时仍先立即尝试领取,取消能立即结束等待。 +- worker 主进度、终态、结果事务及延迟回查要求匹配 owner 且租约仍有效。过期执行者不能覆盖已被重新领取的任务;条件更新没有命中时,终态协调器不得继续退款或完成会话。 + +**边界**:租约不是外部供应商的幂等保障。上游请求不一定响应取消;提供商状态回调、部分进度和关联资源写入尚未全部传播 owner,不能宣称所有副作用都已 fencing。渠道等待仍发生在领取 worker 槽位之后,也没有新增全局积压上限或公平调度。其他用途的槽位未统一改为续期模型。 + +## 上线与回滚 + +1. 先保存故障时段的 Redis 内存、淘汰数量、连接数、延迟和重启记录,以及数据库慢查询、连接池等待、任务 queued/running 数量与最老等待时长。必要日志必须脱敏,不能导出密钥和完整用户正文。 +2. 先在测试环境验证 Redis 与 PostgreSQL 双实例:长任务超过原租期仍只占一个槽位;停止 worker 后槽位恢复;Redis 断连时暂停领取;旧 owner 写入失败;SSE 断线续传无重复或漏终态。 +3. 采用维护窗口排空旧执行者,再发布新版本。Compose 当前默认优雅退出时间由 `CANVAS_SHUTDOWN_TIMEOUT` 控制,默认 10 分钟,不足以保证最长 60 分钟渲染自然完成;发布前核对活跃任务,不能把进程退出视为任务排空成功。 +4. 保留 `canvas:slots:*` 等现有 Redis key。旧进程的 45 分钟分数可能暂时占槽,应先确认旧执行者已结束并观察租约自然回收,**禁止用 FLUSHDB、批量删槽位或批量重投收费任务来解堵**。 +5. 回滚不需要数据迁移,但旧版本仍有原来的固定租期和缺失写入保护。避免新旧版本混跑,不应把回滚视为稳定性问题已经解除。 + +本次不部署、不清理线上缓存、不修改线上任务或触发真实收费请求。仅凭代码修复和本地测试,不能认定某次线上事故根因已确认或故障已消除。 + +`TestRuntimeCoordinatorRedisLeasePreservesLongestTTL` 在 PATH 中存在 `redis-server` 时自动启动临时 Unix socket Redis,检查长短租约混用、容量、过期拒绝续期及幂等释放;它不读取 `REDIS_URL`,不连接部署环境。缺少二进制时明确跳过,不视为 Redis 集成已通过。 + +## 下一阶段 + +- 可复用业务数据的 Redis 缓存与主动失效,任务状态快照和事件通知,缓存故障时的总量保护。 +- 独立 worker 进程、可靠投递 / outbox、渠道前置排队、积压上限和公平性;按任务类型定义幂等、重试、延迟和补偿。 +- 将执行 owner 传播到所有提供商回调和副作用;明确上游已提交但本地未落库时的查询恢复,不盲目重复生成。 +- 后台手动投递、暂停、恢复和审计;完整的请求 / 任务 / 回调关联日志与脱敏正文留存。 +- 事件驱动 SSE、连接准入和重连退避,替换按连接的数据库轮询。 diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 7b869367a..7dbf568b5 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -5,6 +5,15 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 # 待测试 +## 缓存回源与任务队列止血 + +- 已加入进程内有界读缓存:SSE 按用户、任务、游标合并回源;并发策略短缓存;路由版本短缓存和目录回源失败冷却。强校验的计费、配额、权限读取不改为缓存。 +- worker 改为可续期全局槽位和每次领取独立 owner;数据库进度、终态、完成事务、延迟回查检查有效租约;旧执行者条件更新失败后不继续退款和完成会话。渠道槽位等待改为有上限的随机退避。 +- 已通过定向回归与缓存 / 租约 race 检查,覆盖突发同 key 合并、失败冷却、容量与正文预算、取消、缓存失效、用户 / 游标隔离、调用者副本和旧 owner 写入拒绝。完整 repository / handler 测试通过;完整 service 仍有 4 项既有失败,已在 HEAD 隔离副本复现(渠道私网策略 2 项、视频恢复策略 2 项),不将其标记为全绿。 +- [ ] Redis Lua 与 PostgreSQL 双实例故障注入、长任务续租、worker 强退后恢复及真实 SSE 重连验收;本地缺少 Redis 服务且 Docker daemon 未运行,尚未执行真实 Redis / PostgreSQL 集成测试。 +- [ ] 线上故障时段取证及灰度效果确认;本次没有部署、清缓存或重投任务,不能认定早间事故已修复。 +- [ ] 独立 worker、可靠投递、渠道前置队列、全部回调 owner 校验、业务 Redis 缓存、请求链路日志和事件驱动 SSE 属于后续阶段。范围、上线风险及验收步骤见“开发文档 → 任务缓存与队列稳定性”。 + ## 邮箱域名白名单与公告自动提醒(PR #417) - 新增注册邮箱域名白名单:默认常见服务商、空列表不限制、精确域名匹配;发送注册验证码及提交注册都会校验,已有账号找回密码不受限制。保存失败不覆盖既有配置。