fix(tasks): 任务队列 - 限制缓存回源并修复并发租约与过期写入

2026-09-06 00:28:45 +08:00
parent 1ff5e5cada
commit bf0c4454fa
23 changed files with 1098 additions and 59 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,10 @@ docs/content/docs/*
 !docs/content/docs/plugins/
 docs/content/docs/plugins/*
 !docs/content/docs/plugins/ai-art-critique.mdx
+# Keep the task stability runbook versioned without unignoring generated docs.
+!docs/content/docs/backend/
+docs/content/docs/backend/*
+!docs/content/docs/backend/task-runtime-stability.mdx
 .env*
 !.env.example
 data/
--- a/backend/internal/handler/routes.go
+++ b/backend/internal/handler/routes.go
@@ -174,7 +174,7 @@ func RegisterTaskRoutes(r *gin.RouterGroup, svc *service.Service) {
 			fail(c, http.StatusBadRequest, err)
 			return
 		}
-		initial, err := svc.TaskTextReplay(user.ID, c.Param("id"), after)
+		initial, err := svc.CachedTaskTextReplay(c.Request.Context(), user.ID, c.Param("id"), after)
 		if err != nil {
 			failService(c, err)
 			return
@@ -308,7 +308,7 @@ func streamTaskTextEvents(c *gin.Context, svc *service.Service, userID string, t
 			c.Writer.Flush()
 		case <-pollTicker.C:
 		}
-		next, err := svc.TaskTextReplay(userID, taskID, after)
+		next, err := svc.CachedTaskTextReplay(c.Request.Context(), userID, taskID, after)
 		if err != nil {
 			writeTaskTextSSE(c, "error", 0, map[string]string{"message": "任务文本流不可用"})
 			return
--- a/backend/internal/handler/text_events_test.go
+++ b/backend/internal/handler/text_events_test.go
@@ -1,14 +1,19 @@
 package handler

 import (
+	"context"
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"

 	"infinite-canvas/backend/internal/model"
+	"infinite-canvas/backend/internal/repository"
 	"infinite-canvas/backend/internal/service"

 	"github.com/gin-gonic/gin"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
 )

 func TestTaskTextEventCursorPrefersQueryAndSupportsLastEventID(t *testing.T) {
@@ -32,6 +37,42 @@ func TestTaskTextEventCursorPrefersQueryAndSupportsLastEventID(t *testing.T) {
 	}
 }

+func TestStreamTaskTextEventsCachedReplayReachesTerminalWithoutDuplicateDelta(t *testing.T) {
+	t.Setenv("REDIS_URL", "")
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&model.Task{}, &model.TaskTextDelta{}); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Create(&model.Task{ID: "task", UserID: "user", Status: model.TaskStatusRunning}).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Create(&model.TaskTextDelta{ID: "delta", UserID: "user", TaskID: "task", Sequence: 8, Content: "text", ExpiresAt: time.Now().Add(time.Hour)}).Error; err != nil {
+		t.Fatal(err)
+	}
+	svc := service.New(repository.New(db), t.TempDir())
+	t.Cleanup(func() { _ = svc.Close() })
+	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
+	defer cancel()
+	initial, err := svc.CachedTaskTextReplay(ctx, "user", "task", 7)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Model(&model.Task{}).Where("id = ?", "task").Updates(map[string]any{"status": model.TaskStatusSucceeded, "progress": 100}).Error; err != nil {
+		t.Fatal(err)
+	}
+	response := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(response)
+	c.Request = httptest.NewRequest("GET", "/api/tasks/task/text-events", nil).WithContext(ctx)
+	streamTaskTextEvents(c, svc, "user", "task", 7, initial)
+	body := response.Body.String()
+	if strings.Count(body, "id: 8\nevent: delta") != 1 || !strings.Contains(body, "event: terminal") || !strings.Contains(body, `"progress":100`) {
+		t.Fatalf("unexpected cached SSE stream: %q", body)
+	}
+}
+
 func TestStreamTaskTextEventsWritesSequenceAndTerminalEvent(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 	response := httptest.NewRecorder()
--- a/backend/internal/repository/repository.go
+++ b/backend/internal/repository/repository.go
@@ -1,6 +1,7 @@
 package repository

 import (
+	"context"
 	"errors"
 	"fmt"
 	"slices"
@@ -54,6 +55,10 @@ func New(db *gorm.DB) *Repository {
 	return &Repository{db: db}
 }

+func (r *Repository) WithContext(ctx context.Context) *Repository {
+	return &Repository{db: r.db.WithContext(ctx)}
+}
+
 func (r *Repository) Dialect() string {
 	return r.db.Dialector.Name()
 }
@@ -428,7 +433,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (*

 func (r *Repository) RenewTaskLease(id string, owner string, leaseDuration time.Duration) error {
 	result := r.db.Model(&model.Task{}).
-		Where("id = ? AND status = ? AND lease_owner = ?", id, model.TaskStatusRunning, owner).
+		Where("id = ? AND status = ? AND lease_owner = ? AND lease_expires_at > ?", id, model.TaskStatusRunning, owner, time.Now()).
 		Updates(map[string]any{"lease_expires_at": time.Now().Add(leaseDuration), "updated_at": time.Now()})
 	if result.Error != nil {
 		return result.Error
@@ -449,8 +454,8 @@ func (r *Repository) UpdateTaskProviderState(id string, providerRequestID string

 func (r *Repository) DeferRunningTaskForProviderPoll(id string, owner string, stage string, delay time.Duration) error {
 	now := time.Now()
-	result := r.db.Model(&model.Task{}).
-		Where("id = ? AND status = ? AND lease_owner = ?", id, model.TaskStatusRunning, owner).
+	result := taskLeaseWriter(r.db.Model(&model.Task{}), owner).
+		Where("id = ? AND status = ?", id, model.TaskStatusRunning).
 		Updates(map[string]any{
 			"stage": stage, "error": "", "completed_at": nil, "next_poll_at": now.Add(delay),
 			"lease_owner": "", "lease_expires_at": nil, "updated_at": now,
@@ -500,6 +505,27 @@ func (r *Repository) UpdateTaskProgress(id string, stage string, progress int) e
 	}).Error
 }

+func (r *Repository) UpdateTaskProgressForLease(id string, owner string, stage string, progress int) error {
+	result := taskLeaseWriter(r.db.Model(&model.Task{}), owner).
+		Where("id = ? AND status = ?", id, model.TaskStatusRunning).
+		Updates(map[string]any{"stage": stage, "progress": progress, "updated_at": time.Now()})
+	if result.Error != nil {
+		return result.Error
+	}
+	if result.RowsAffected != 1 {
+		return ErrTaskStateConflict
+	}
+	return nil
+}
+
+// 无租约任务仅能写无租约记录；有租约的执行者必须仍持有本次领取的有效 owner。
+func taskLeaseWriter(db *gorm.DB, owner string) *gorm.DB {
+	if owner == "" {
+		return db.Where("(lease_owner = '' OR lease_owner IS NULL)")
+	}
+	return db.Where("lease_owner = ? AND lease_expires_at > ?", owner, time.Now())
+}
+
 // UpdateTaskProviderProgress records upstream-reported progress without allowing
 // a delayed or out-of-order poll response to move the public percentage backwards.
 func (r *Repository) UpdateTaskProviderProgress(id string, progress int) error {
@@ -513,7 +539,7 @@ func (r *Repository) UpdateTaskProviderProgress(id string, progress int) error {

 func (r *Repository) SaveTaskCompletion(task *model.Task, expected model.TaskStatus, session *model.Session, message *model.Message, results []model.Result) error {
 	return r.db.Transaction(func(tx *gorm.DB) error {
-		updated := tx.Model(&model.Task{}).
+		updated := taskLeaseWriter(tx.Model(&model.Task{}), task.LeaseOwner).
 			Where("id = ? AND status = ?", task.ID, expected).
 			Select("*").Omit("id", "created_at").Updates(task)
 		if updated.Error != nil {
@@ -541,8 +567,8 @@ func (r *Repository) SaveTaskCompletion(task *model.Task, expected model.TaskSta
 	})
 }

-func (r *Repository) UpdateTaskTerminalState(id string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) {
-	result := r.db.Model(&model.Task{}).
+func (r *Repository) UpdateTaskTerminalState(id string, owner string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) {
+	result := taskLeaseWriter(r.db.Model(&model.Task{}), owner).
 		Where("id = ? AND status = ?", id, expected).
 		Updates(map[string]any{
 			"status": status, "stage": stage, "error": errorText, "completed_at": &completedAt,
--- a/backend/internal/repository/task_lease_fencing_test.go
+++ b/backend/internal/repository/task_lease_fencing_test.go
@@ -0,0 +1,75 @@
+package repository
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"infinite-canvas/backend/internal/model"
+)
+
+func TestTaskLeaseFencesExpiredAndReclaimedWriters(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&model.Task{}, &model.Result{}, &model.Message{}, &model.Session{}); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Create(&model.Task{ID: "task", UserID: "user", Status: model.TaskStatusQueued}).Error; err != nil {
+		t.Fatal(err)
+	}
+	repo := New(db)
+	old, err := repo.ClaimNextTask("worker:attempt-a", time.Minute)
+	if err != nil || old == nil {
+		t.Fatalf("claim: %v %v", old, err)
+	}
+	if err := db.Model(&model.Task{}).Where("id = ?", old.ID).Update("lease_expires_at", time.Now().Add(-time.Second)).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := repo.RenewTaskLease(old.ID, old.LeaseOwner, time.Minute); err == nil {
+		t.Fatal("expired owner resurrected")
+	}
+	if err := repo.DeferRunningTaskForProviderPoll(old.ID, old.LeaseOwner, "poll", time.Minute); !errors.Is(err, ErrTaskStateConflict) {
+		t.Fatalf("expired owner deferred task: %v", err)
+	}
+	current, err := repo.ClaimNextTask("worker:attempt-b", time.Minute)
+	if err != nil || current == nil {
+		t.Fatalf("reclaim: %v %v", current, err)
+	}
+	for _, owner := range []string{old.LeaseOwner, ""} {
+		if err := repo.UpdateTaskProgressForLease(old.ID, owner, "stale", 90); !errors.Is(err, ErrTaskStateConflict) {
+			t.Fatalf("stale progress: %v", err)
+		}
+		updated, err := repo.UpdateTaskTerminalState(old.ID, owner, model.TaskStatusRunning, model.TaskStatusFailed, "stale", "", time.Now())
+		if err != nil || updated {
+			t.Fatalf("stale terminal: %v %v", updated, err)
+		}
+	}
+	old.Status = model.TaskStatusSucceeded
+	if err := repo.SaveTaskCompletion(old, model.TaskStatusRunning, nil, &model.Message{ID: "stale"}, []model.Result{{ID: "stale"}}); !errors.Is(err, ErrTaskStateConflict) {
+		t.Fatalf("stale completion: %v", err)
+	}
+	var count int64
+	for _, table := range []any{&model.Result{}, &model.Message{}} {
+		if err := db.Model(table).Count(&count).Error; err != nil || count != 0 {
+			t.Fatalf("stale related row persisted: %d %v", count, err)
+		}
+	}
+	if err := repo.RenewTaskLease(current.ID, current.LeaseOwner, time.Minute); err != nil {
+		t.Fatal(err)
+	}
+	if err := repo.UpdateTaskProgressForLease(current.ID, current.LeaseOwner, "current", 60); err != nil {
+		t.Fatal(err)
+	}
+	current.Status = model.TaskStatusSucceeded
+	if err := repo.SaveTaskCompletion(current, model.TaskStatusRunning, nil, nil, []model.Result{{ID: "valid", TaskID: current.ID}}); err != nil {
+		t.Fatal(err)
+	}
+	stored, err := repo.Task(current.ID)
+	if err != nil || stored.Status != model.TaskStatusSucceeded || stored.LeaseOwner != current.LeaseOwner {
+		t.Fatalf("current completion: %v %v", stored, err)
+	}
+}
--- a/backend/internal/service/coordination.go
+++ b/backend/internal/service/coordination.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
+	"math/rand/v2"
 	"os"
 	"strconv"
 	"strings"
@@ -74,7 +76,8 @@ var acquireSlotScript = redis.NewScript(`
 redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
 if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[3]) then return 0 end
 redis.call('ZADD', KEYS[1], ARGV[2], ARGV[4])
-redis.call('PEXPIRE', KEYS[1], ARGV[5])
+local latest = redis.call('ZREVRANGE', KEYS[1], 0, 0, 'WITHSCORES')
+redis.call('PEXPIRE', KEYS[1], math.max(tonumber(ARGV[5]), tonumber(latest[2]) - tonumber(ARGV[1]) + 60000))
 return 1
 `)

@@ -91,6 +94,9 @@ func newRuntimeCoordinator(dialect string) (*runtimeCoordinator, error) {
 	if err != nil {
 		return coordinator, fmt.Errorf("REDIS_URL 无效：%w", err)
 	}
+	// 协调写操作不透明重试；限流/并发失败必须及时向调用方返回，不能放大故障流量。
+	options.MaxRetries = -1
+	options.ContextTimeoutEnabled = true
 	coordinator.redis = redis.NewClient(options)
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 	defer cancel()
@@ -122,6 +128,21 @@ func (c *runtimeCoordinator) allow(ctx context.Context, key string, limit int, w
 }

 func (c *runtimeCoordinator) acquire(ctx context.Context, scope string, limit int, ttl time.Duration) (func(), bool, error) {
+	lease, acquired, err := c.acquireLease(ctx, scope, limit, ttl)
+	if err != nil || !acquired {
+		return nil, acquired, err
+	}
+	return lease.release, true, nil
+}
+
+func (c *runtimeCoordinator) acquireLease(ctx context.Context, scope string, limit int, ttl time.Duration) (*runtimeSlotLease, bool, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, false, err
+	}
+	if limit <= 0 || ttl <= 0 {
+		return nil, false, errors.New("并发租约参数无效")
+	}
+	lease := &runtimeSlotLease{coordinator: c, scope: scope, token: c.instanceID + ":" + newID(), ttl: ttl}
 	if c.redis == nil {
 		c.localMu.Lock()
 		now := time.Now()
@@ -139,29 +160,90 @@ func (c *runtimeCoordinator) acquire(ctx context.Context, scope string, limit in
 			c.localMu.Unlock()
 			return nil, false, nil
 		}
-		token := c.instanceID + ":" + newID()
-		slots[token] = now.Add(ttl)
+		slots[lease.token] = now.Add(ttl)
 		c.localMu.Unlock()
-		return func() {
-			c.localMu.Lock()
-			delete(c.localSlots[scope], token)
-			c.localMu.Unlock()
-		}, true, nil
+		return lease, true, nil
 	}
 	// 有过期分数的有序集合避免实例崩溃后永久占槽，业务数据库仍保存任务与账本真相。
 	key := "canvas:slots:" + scope
-	token := c.instanceID + ":" + newID()
+	ctx, cancel := context.WithTimeout(ctx, runtimeCoordinationTimeout)
+	defer cancel()
 	now := time.Now()
-	ok, err := acquireSlotScript.Run(ctx, c.redis, []string{key}, now.UnixMilli(), now.Add(ttl).UnixMilli(), limit, token, (ttl + time.Minute).Milliseconds()).Int()
+	ok, err := acquireSlotScript.Run(ctx, c.redis, []string{key}, now.UnixMilli(), now.Add(ttl).UnixMilli(), limit, lease.token, (ttl + time.Minute).Milliseconds()).Int()
 	if err != nil || ok != 1 {
 		return nil, false, err
 	}
-	return func() { _ = c.redis.ZRem(context.Background(), key, token).Err() }, true, nil
+	return lease, true, nil
+}
+
+const runtimeCoordinationTimeout = 2 * time.Second
+
+type runtimeSlotLease struct {
+	coordinator *runtimeCoordinator
+	scope       string
+	token       string
+	ttl         time.Duration
+	once        sync.Once
+}
+
+var renewSlotScript = redis.NewScript(`
+local expires = redis.call('ZSCORE', KEYS[1], ARGV[1])
+if not expires or tonumber(expires) <= tonumber(ARGV[2]) then return 0 end
+redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1])
+local latest = redis.call('ZREVRANGE', KEYS[1], 0, 0, 'WITHSCORES')
+redis.call('PEXPIRE', KEYS[1], math.max(tonumber(ARGV[4]), tonumber(latest[2]) - tonumber(ARGV[2]) + 60000))
+return 1
+`)
+
+func (l *runtimeSlotLease) renew(ctx context.Context) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	c := l.coordinator
+	now := time.Now()
+	if c.redis == nil {
+		c.localMu.Lock()
+		defer c.localMu.Unlock()
+		if !c.localSlots[l.scope][l.token].After(now) {
+			return errors.New("并发租约已失效")
+		}
+		c.localSlots[l.scope][l.token] = now.Add(l.ttl)
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(ctx, runtimeCoordinationTimeout)
+	defer cancel()
+	ok, err := renewSlotScript.Run(ctx, c.redis, []string{"canvas:slots:" + l.scope}, l.token, now.UnixMilli(), now.Add(l.ttl).UnixMilli(), (l.ttl + time.Minute).Milliseconds()).Int()
+	if err != nil {
+		return err
+	}
+	if ok != 1 {
+		return errors.New("并发租约已失效")
+	}
+	return nil
+}
+
+func (l *runtimeSlotLease) release() {
+	l.once.Do(func() {
+		c := l.coordinator
+		if c.redis == nil {
+			c.localMu.Lock()
+			defer c.localMu.Unlock()
+			delete(c.localSlots[l.scope], l.token)
+			if len(c.localSlots[l.scope]) == 0 {
+				delete(c.localSlots, l.scope)
+			}
+			return
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
+		defer cancel()
+		if err := c.redis.ZRem(ctx, "canvas:slots:"+l.scope, l.token).Err(); err != nil {
+			log.Printf("concurrency lease release failed: scope=%s error=%v", l.scope, err)
+		}
+	})
 }

 func (c *runtimeCoordinator) acquireWithWait(ctx context.Context, scope string, limit int, ttl time.Duration) (func(), error) {
-	ticker := time.NewTicker(200 * time.Millisecond)
-	defer ticker.Stop()
+	delay := 200 * time.Millisecond
 	for {
 		if err := ctx.Err(); err != nil {
 			return nil, err
@@ -173,14 +255,23 @@ func (c *runtimeCoordinator) acquireWithWait(ctx context.Context, scope string,
 		if acquired {
 			return release, nil
 		}
+		// 满载期间退避并错峰，避免每个等待者固定每秒五次同步争抢 Redis。
+		wait := time.NewTimer(channelSlotRetryDelay(delay))
 		select {
 		case <-ctx.Done():
+			wait.Stop()
 			return nil, ctx.Err()
-		case <-ticker.C:
+		case <-wait.C:
 		}
+		delay = min(delay*2, 2*time.Second)
 	}
 }

+func channelSlotRetryDelay(delay time.Duration) time.Duration {
+	// 上限 2s，下半窗口随机等待，保留首轮至少 100ms 的退让。
+	return delay/2 + time.Duration(rand.Int64N(int64(delay/2)+1))
+}
+
 func (c *runtimeCoordinator) circuitOpen(ctx context.Context, channelID string) (bool, error) {
 	if c.redis == nil || strings.TrimSpace(channelID) == "" {
 		return false, nil
--- a/backend/internal/service/coordination_redis_test.go
+++ b/backend/internal/service/coordination_redis_test.go
@@ -0,0 +1,86 @@
+package service
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// 只运行临时 Unix socket Redis，不读取 REDIS_URL，不连接部署环境。
+func TestRuntimeCoordinatorRedisLeasePreservesLongestTTL(t *testing.T) {
+	binary, err := exec.LookPath("redis-server")
+	if err != nil {
+		t.Skip("redis-server unavailable; real Redis lease integration not run")
+	}
+	// macOS 的 Unix socket 路径有长度限制，不使用包含长测试名的 t.TempDir。
+	dir, err := os.MkdirTemp("", "canvas-redis-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
+	socket := filepath.Join(dir, "r.sock")
+	cmd := exec.Command(binary, "--port", "0", "--unixsocket", socket, "--unixsocketperm", "700", "--save", "", "--appendonly", "no", "--dir", dir)
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
+	client := redis.NewClient(&redis.Options{Network: "unix", Addr: socket, MaxRetries: -1, ContextTimeoutEnabled: true})
+	t.Cleanup(func() { _ = client.Close() })
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	for client.Ping(ctx).Err() != nil {
+		if ctx.Err() != nil {
+			t.Fatal("temporary Redis did not start")
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	c := &runtimeCoordinator{redis: client, instanceID: "isolated-test"}
+	long, ok, err := c.acquireLease(ctx, "workers", 2, 45*time.Minute)
+	if err != nil || !ok {
+		t.Fatalf("legacy lease: %v %v", ok, err)
+	}
+	defer long.release()
+	short, ok, err := c.acquireLease(ctx, "workers", 2, time.Minute)
+	if err != nil || !ok {
+		t.Fatalf("new lease: %v %v", ok, err)
+	}
+	defer short.release()
+	key := "canvas:slots:workers"
+	assertLongestTTL := func() {
+		t.Helper()
+		ttl, err := client.PTTL(ctx, key).Result()
+		if err != nil || ttl < 44*time.Minute {
+			t.Fatalf("short lease truncated shared key TTL: %s %v", ttl, err)
+		}
+	}
+	assertLongestTTL()
+	if err := short.renew(ctx); err != nil {
+		t.Fatal(err)
+	}
+	assertLongestTTL()
+	if _, ok, err := c.acquireLease(ctx, "workers", 2, time.Minute); err != nil || ok {
+		t.Fatalf("global capacity exceeded: %v %v", ok, err)
+	}
+	if err := client.ZAdd(ctx, key, redis.Z{Score: float64(time.Now().Add(-time.Second).UnixMilli()), Member: short.token}).Err(); err != nil {
+		t.Fatal(err)
+	}
+	if err := short.renew(ctx); err == nil {
+		t.Fatal("expired Redis lease resurrected")
+	}
+	replacement, ok, err := c.acquireLease(ctx, "workers", 2, time.Minute)
+	if err != nil || !ok {
+		t.Fatalf("abandoned slot not reclaimed: %v %v", ok, err)
+	}
+	defer replacement.release()
+	short.release()
+	short.release()
+	if err := replacement.renew(ctx); err != nil {
+		t.Fatalf("old release removed new owner: %v", err)
+	}
+	assertLongestTTL()
+}
--- a/backend/internal/service/coordination_test.go
+++ b/backend/internal/service/coordination_test.go
@@ -6,6 +6,41 @@ import (
 	"time"
 )

+func TestRuntimeCoordinatorLeaseRenewsAndCannotResurrect(t *testing.T) {
+	c := &runtimeCoordinator{instanceID: "test", localSlots: map[string]map[string]time.Time{}}
+	lease, acquired, err := c.acquireLease(context.Background(), "workers", 1, time.Minute)
+	if err != nil || !acquired {
+		t.Fatalf("acquire: %v %v", acquired, err)
+	}
+	c.localSlots["workers"][lease.token] = time.Now().Add(10 * time.Second)
+	if err := lease.renew(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if time.Until(c.localSlots["workers"][lease.token]) < 50*time.Second {
+		t.Fatal("lease was not extended")
+	}
+	if _, ok, err := c.acquireLease(context.Background(), "workers", 1, time.Minute); err != nil || ok {
+		t.Fatalf("capacity exceeded: %v %v", ok, err)
+	}
+	c.localSlots["workers"][lease.token] = time.Now().Add(-time.Second)
+	if err := lease.renew(context.Background()); err == nil {
+		t.Fatal("expired lease resurrected")
+	}
+	replacement, acquired, err := c.acquireLease(context.Background(), "workers", 1, time.Minute)
+	if err != nil || !acquired {
+		t.Fatalf("abandoned slot not recovered: %v %v", acquired, err)
+	}
+	lease.release()
+	lease.release()
+	if err := replacement.renew(context.Background()); err != nil {
+		t.Fatalf("old release removed new lease: %v", err)
+	}
+	replacement.release()
+	if len(c.localSlots) != 0 {
+		t.Fatal("empty local slot scopes retained")
+	}
+}
+
 func TestRuntimeCoordinatorWaitsUntilChannelSlotIsReleased(t *testing.T) {
 	coordinator := &runtimeCoordinator{instanceID: "test", localRate: map[string]localRateEntry{}, localSlots: map[string]map[string]time.Time{}}
 	releaseFirst, acquired, err := coordinator.acquire(context.Background(), "channel:one", 1, time.Minute)
@@ -52,3 +87,13 @@ func TestRuntimeCoordinatorStopsWaitingWhenContextIsCancelled(t *testing.T) {
 		t.Fatal("acquireWithWait() error = nil after cancellation")
 	}
 }
+
+func TestChannelSlotRetryDelayStaysWithinBackoffWindow(t *testing.T) {
+	for _, base := range []time.Duration{200 * time.Millisecond, 400 * time.Millisecond, 800 * time.Millisecond, 1600 * time.Millisecond, 2 * time.Second} {
+		for range 100 {
+			if got := channelSlotRetryDelay(base); got < base/2 || got > base {
+				t.Fatalf("delay %s outside [%s, %s]", got, base/2, base)
+			}
+		}
+	}
+}
--- a/backend/internal/service/model_router.go
+++ b/backend/internal/service/model_router.go
@@ -446,15 +446,23 @@ func isProviderCapabilityOption(name string) bool {
 }

 func (s *Service) invalidateRouteCatalog() {
+	s.routeCatalogRefreshMu.Lock()
 	s.routeCatalogMu.Lock()
 	s.routeCatalog = nil
 	s.routeCatalogVersion++
+	s.routeCatalogRetryAt = time.Time{}
+	s.routeCatalogRefreshError = nil
 	s.routeCatalogMu.Unlock()
+	s.routeCatalogRefreshMu.Unlock()
 	if s.coordinator != nil {
-		if err := s.coordinator.bumpRouteCatalogVersion(context.Background()); err != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
+		defer cancel()
+		if err := s.coordinator.bumpRouteCatalogVersion(ctx); err != nil {
 			log.Printf("logical model route catalog distributed invalidation failed: %v", err)
 		}
 	}
+	s.initReadCaches()
+	s.routeVersionReadCache.clear()
 }

 func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) {
@@ -480,15 +488,26 @@ func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) {
 	}
 	s.routeCatalogMu.RUnlock()

+	// 刷新锁只能防止并行回源，失败后还需要冷却，否则等待者会逐个重打数据库。
+	if s.routeCatalogRefreshError != nil && now.Before(s.routeCatalogRetryAt) {
+		if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
+			return snapshot, nil
+		}
+		return nil, s.routeCatalogRefreshError
+	}
 	loaded, err := s.loadRouteCatalog()
 	if err != nil {
+		s.routeCatalogRefreshError = err
+		s.routeCatalogRetryAt = time.Now().Add(2 * time.Second)
+		log.Printf("logical model route catalog refresh failed; retry cooled down: %v", err)
 		// 已有快照过期时允许短暂继续服务，数据库首次加载失败则明确失败。
-		if snapshot != nil && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
-			log.Printf("logical model route catalog refresh failed, serving stale snapshot age=%s: %v", now.Sub(snapshot.LoadedAt).Round(time.Second), err)
+		if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
 			return snapshot, nil
 		}
 		return nil, err
 	}
+	s.routeCatalogRefreshError = nil
+	s.routeCatalogRetryAt = time.Time{}
 	s.routeCatalogMu.Lock()
 	s.routeCatalog = loaded
 	s.routeCatalogMu.Unlock()
@@ -502,11 +521,17 @@ func (s *Service) currentRouteCatalogVersion() int64 {
 	if s.coordinator == nil || s.coordinator.redis == nil {
 		return localVersion
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	s.initReadCaches()
+	ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
 	defer cancel()
-	version, err := s.coordinator.routeCatalogVersion(ctx)
+	version, err := s.routeVersionReadCache.get(ctx, routeCatalogVersionKey, func(ctx context.Context) (int64, int, error) {
+		value, err := s.coordinator.routeCatalogVersion(ctx)
+		if err != nil {
+			log.Printf("logical model route catalog distributed version check failed: %v", err)
+		}
+		return value, 256, err
+	})
 	if err != nil {
-		log.Printf("logical model route catalog distributed version check failed: %v", err)
 		return localVersion
 	}
 	if version > localVersion {
@@ -516,12 +541,15 @@ func (s *Service) currentRouteCatalogVersion() int64 {
 }

 func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) {
-	items, err := s.repo.LogicalModels(false)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	repo := s.repo.WithContext(ctx)
+	items, err := repo.LogicalModels(false)
 	if err != nil {
 		return nil, err
 	}
 	snapshot := &routeCatalogSnapshot{LoadedAt: time.Now(), CatalogVersion: s.currentRouteCatalogVersion(), Models: make(map[string]cachedLogicalModel), Ordered: make([]string, 0, len(items))}
-	graphs, err := s.repo.LogicalModelGraphs(items, false)
+	graphs, err := repo.LogicalModelGraphs(items, false)
 	if err != nil {
 		return nil, err
 	}
@@ -534,7 +562,7 @@ func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) {
 			systemChannelIDs = append(systemChannelIDs, channelModel.ChannelID)
 		}
 	}
-	systemChannels, err := s.repo.SystemChannelsByIDs(systemChannelIDs, false)
+	systemChannels, err := repo.SystemChannelsByIDs(systemChannelIDs, false)
 	if err != nil {
 		return nil, err
 	}
--- a/backend/internal/service/read_cache.go
+++ b/backend/internal/service/read_cache.go
@@ -0,0 +1,147 @@
+package service
+
+import (
+	"container/list"
+	"context"
+	"errors"
+	"sync"
+	"time"
+)
+
+var errReadCacheBusy = errors.New("热点读取正在回源，请稍后重试")
+var errReadCacheLoadAborted = errors.New("缓存回源异常中断")
+
+// 仅用于可短暂延迟的读投影。限制条目、正文预算和同时回源数；失败短暂冷却，
+// 不返回过期成功值，也不允许大量不同 key 将 singleflight 本身变成无界队列。
+type boundedReadCache[K comparable, V any] struct {
+	mu                             sync.Mutex
+	entries                        map[K]*readCacheEntry[K, V]
+	lru                            list.List
+	bytes, loads                   int
+	maxEntries, maxBytes, maxLoads int
+	ttl                            time.Duration
+}
+
+type readCacheEntry[K comparable, V any] struct {
+	key      K
+	ready    chan struct{}
+	value    V
+	err      error
+	expires  time.Time
+	bytes    int
+	position *list.Element
+}
+
+func newBoundedReadCache[K comparable, V any](entries, bytes, loads int, ttl time.Duration) *boundedReadCache[K, V] {
+	return &boundedReadCache[K, V]{entries: make(map[K]*readCacheEntry[K, V]), maxEntries: entries, maxBytes: bytes, maxLoads: loads, ttl: ttl}
+}
+
+func (c *boundedReadCache[K, V]) get(ctx context.Context, key K, load func(context.Context) (V, int, error)) (V, error) {
+	var zero V
+	if err := ctx.Err(); err != nil {
+		return zero, err
+	}
+	c.mu.Lock()
+	if e := c.entries[key]; e != nil {
+		if ready := e.ready; ready != nil {
+			c.mu.Unlock()
+			select {
+			case <-ctx.Done():
+				return zero, ctx.Err()
+			case <-ready:
+				return e.value, e.err
+			}
+		}
+		if time.Now().Before(e.expires) {
+			c.lru.MoveToFront(e.position)
+			value, err := e.value, e.err
+			c.mu.Unlock()
+			return value, err
+		}
+		c.remove(e)
+	}
+	if c.loads >= c.maxLoads {
+		c.mu.Unlock()
+		return zero, errReadCacheBusy
+	}
+	for len(c.entries) >= c.maxEntries {
+		if !c.evict() {
+			c.mu.Unlock()
+			return zero, errReadCacheBusy
+		}
+	}
+	e := &readCacheEntry[K, V]{key: key, ready: make(chan struct{})}
+	c.entries[key] = e
+	c.loads++
+	c.mu.Unlock()
+
+	// 让原始 panic 继续传播给调用方，同时释放等待者和容量，避免永久卡住热点 key。
+	loaded := false
+	defer func() {
+		if loaded {
+			return
+		}
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		c.loads--
+		e.err = errReadCacheLoadAborted
+		if c.entries[key] == e {
+			delete(c.entries, key)
+		}
+		close(e.ready)
+		e.ready = nil
+	}()
+	value, bytes, err := load(ctx)
+	loaded = true
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.loads--
+	e.value, e.err = value, err
+	ready := e.ready
+	e.ready = nil
+	if c.entries[key] == e {
+		ttl := c.ttl
+		if err != nil {
+			ttl, bytes = time.Second, 256
+		}
+		bytes = max(bytes, 256)
+		if bytes > c.maxBytes || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+			delete(c.entries, key)
+		} else {
+			for c.bytes+bytes > c.maxBytes {
+				if !c.evict() {
+					break
+				}
+			}
+			e.bytes, e.expires = bytes, time.Now().Add(ttl)
+			e.position = c.lru.PushFront(e)
+			c.bytes += bytes
+		}
+	}
+	close(ready)
+	return value, err
+}
+
+func (c *boundedReadCache[K, V]) evict() bool {
+	if oldest := c.lru.Back(); oldest != nil {
+		c.remove(oldest.Value.(*readCacheEntry[K, V]))
+		return true
+	}
+	return false
+}
+
+func (c *boundedReadCache[K, V]) remove(e *readCacheEntry[K, V]) {
+	delete(c.entries, e.key)
+	if e.position != nil {
+		c.lru.Remove(e.position)
+		c.bytes -= e.bytes
+	}
+}
+
+func (c *boundedReadCache[K, V]) clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.entries = make(map[K]*readCacheEntry[K, V])
+	c.lru.Init()
+	c.bytes = 0
+}
--- a/backend/internal/service/read_cache_integration_test.go
+++ b/backend/internal/service/read_cache_integration_test.go
@@ -0,0 +1,165 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"infinite-canvas/backend/internal/model"
+	"infinite-canvas/backend/internal/repository"
+)
+
+func TestCachedTextReplayIsolatesUsersCursorsAndCopies(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open("file:"+newID()+"?mode=memory&cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&model.Task{}, &model.TaskTextDelta{}); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Create(&model.Task{ID: "task", UserID: "user", Type: "canvas_text", Status: model.TaskStatusRunning}).Error; err != nil {
+		t.Fatal(err)
+	}
+	for i, content := range []string{"one", "two"} {
+		if err := db.Create(&model.TaskTextDelta{ID: content, UserID: "user", TaskID: "task", Sequence: int64(i + 1), Content: content, ExpiresAt: time.Now().Add(time.Hour)}).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	svc := &Service{repo: repository.New(db)}
+	var queries atomic.Int32
+	if err := db.Callback().Query().Before("gorm:query").Register("cache-count", func(*gorm.DB) { queries.Add(1) }); err != nil {
+		t.Fatal(err)
+	}
+	var wg sync.WaitGroup
+	for range 50 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			result, err := svc.CachedTaskTextReplay(context.Background(), "user", "task", 0)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			if len(result.Deltas) != 2 || result.Deltas[0].Content != "one" {
+				t.Errorf("unexpected replay: %#v", result)
+				return
+			}
+			result.Deltas[0].Content = "caller mutation"
+		}()
+	}
+	wg.Wait()
+	if queries.Load() != 2 {
+		t.Fatalf("expected one task+delta query, got %d queries", queries.Load())
+	}
+	if _, err := svc.CachedTaskTextReplay(context.Background(), "other", "task", 0); err == nil {
+		t.Fatal("cross-user cache leak")
+	}
+	result, err := svc.CachedTaskTextReplay(context.Background(), "user", "task", 1)
+	if err != nil || len(result.Deltas) != 1 || result.Deltas[0].Sequence != 2 {
+		t.Fatalf("cursor isolation: %#v %v", result, err)
+	}
+	svc.textReplayReadCache.clear()
+	if err := db.Model(&model.Task{}).Where("id = ?", "task").Update("status", model.TaskStatusSucceeded).Error; err != nil {
+		t.Fatal(err)
+	}
+	result, err = svc.CachedTaskTextReplay(context.Background(), "user", "task", 0)
+	if err != nil || !result.Complete {
+		t.Fatalf("terminal refresh: %#v %v", result, err)
+	}
+}
+
+func TestRuntimeConcurrencyCacheDoesNotCacheAuthoritativePolicy(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&model.SystemSetting{}, &model.AdminAuditEvent{}); err != nil {
+		t.Fatal(err)
+	}
+	svc := &Service{repo: repository.New(db)}
+	actor := &model.User{ID: "admin", Role: model.UserRoleAdmin}
+	var queries atomic.Int32
+	if err := db.Callback().Query().Before("gorm:query").Register("policy-count", func(*gorm.DB) { queries.Add(1) }); err != nil {
+		t.Fatal(err)
+	}
+	for range 20 {
+		if _, err := svc.runtimeConcurrencySetting(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if queries.Load() != 1 {
+		t.Fatalf("concurrency reads hit DB %d times", queries.Load())
+	}
+	before := queries.Load()
+	for range 2 {
+		if _, err := svc.RuntimePolicy(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if queries.Load() != before+2 {
+		t.Fatal("authoritative policy incorrectly cached")
+	}
+	policy := defaultRuntimePolicy()
+	policy.Task.WorkerConcurrency = 7
+	if _, err := svc.UpdateRuntimePolicySetting(actor, policy); err != nil {
+		t.Fatal(err)
+	}
+	setting, err := svc.runtimeConcurrencySetting()
+	if err != nil || setting.WorkerConcurrency != 7 {
+		t.Fatalf("save did not invalidate: %#v %v", setting, err)
+	}
+	if _, err := svc.ResetRuntimePolicySetting(actor); err != nil {
+		t.Fatal(err)
+	}
+	setting, err = svc.runtimeConcurrencySetting()
+	if err != nil || setting.WorkerConcurrency != defaultRuntimePolicy().Task.WorkerConcurrency {
+		t.Fatalf("reset did not invalidate: %#v %v", setting, err)
+	}
+}
+
+func TestRouteCatalogFailureStormAndInvalidatedStaleSnapshot(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantErr := errors.New("injected database outage")
+	var queries atomic.Int32
+	if err := db.Callback().Query().Before("gorm:query").Register("catalog-outage", func(tx *gorm.DB) { queries.Add(1); tx.AddError(wantErr) }); err != nil {
+		t.Fatal(err)
+	}
+	svc := &Service{repo: repository.New(db), routeCatalogTTL: time.Second, routeCatalogMaxStale: time.Minute}
+	var wg sync.WaitGroup
+	for range 50 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if _, err := svc.routeCatalogSnapshot(); !errors.Is(err, wantErr) {
+				t.Errorf("outage error lost: %v", err)
+			}
+		}()
+	}
+	wg.Wait()
+	if queries.Load() != 1 {
+		t.Fatalf("failure storm retried DB %d times", queries.Load())
+	}
+	svc.routeCatalog = &routeCatalogSnapshot{LoadedAt: time.Now().Add(-2 * time.Second), CatalogVersion: 0}
+	if _, err := svc.routeCatalogSnapshot(); err != nil {
+		t.Fatalf("existing same-version bounded fallback lost: %v", err)
+	}
+	svc.routeCatalogVersion = 1
+	if _, err := svc.routeCatalogSnapshot(); !errors.Is(err, wantErr) {
+		t.Fatal("explicitly invalidated version was served stale")
+	}
+	svc.invalidateRouteCatalog()
+	if _, err := svc.routeCatalogSnapshot(); !errors.Is(err, wantErr) {
+		t.Fatal(err)
+	}
+	if queries.Load() != 2 {
+		t.Fatal("invalidation failed to reset refresh cooldown")
+	}
+}
--- a/backend/internal/service/read_cache_test.go
+++ b/backend/internal/service/read_cache_test.go
@@ -0,0 +1,135 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestReadCacheCoalescesBurstAndCoolsFailures(t *testing.T) {
+	for _, fail := range []bool{false, true} {
+		cache := newBoundedReadCache[string, int](4, 1024, 2, time.Minute)
+		var calls atomic.Int32
+		loadErr := errors.New("database unavailable")
+		load := func(context.Context) (int, int, error) {
+			calls.Add(1)
+			if fail {
+				return 0, 0, loadErr
+			}
+			return 42, 256, nil
+		}
+		var wg sync.WaitGroup
+		for range 100 {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				value, err := cache.get(context.Background(), "hot", load)
+				if (fail && !errors.Is(err, loadErr)) || (!fail && (err != nil || value != 42)) {
+					t.Errorf("value=%d error=%v fail=%v", value, err, fail)
+				}
+			}()
+		}
+		wg.Wait()
+		if calls.Load() != 1 {
+			t.Fatalf("burst caused %d loads (fail=%v)", calls.Load(), fail)
+		}
+	}
+}
+
+func TestReadCacheBoundsLoadsAndInvalidation(t *testing.T) {
+	cache := newBoundedReadCache[string, int](2, 512, 1, time.Minute)
+	started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(done)
+		_, _ = cache.get(context.Background(), "one", func(context.Context) (int, int, error) {
+			close(started)
+			<-release
+			return 1, 256, nil
+		})
+	}()
+	<-started
+	load := func(context.Context) (int, int, error) { return 2, 256, nil }
+	if _, err := cache.get(context.Background(), "two", load); !errors.Is(err, errReadCacheBusy) {
+		t.Fatalf("unbounded distinct load: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+	defer cancel()
+	if _, err := cache.get(ctx, "one", load); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("waiting reader did not cancel: %v", err)
+	}
+	cache.clear()
+	if _, err := cache.get(context.Background(), "two", load); !errors.Is(err, errReadCacheBusy) {
+		t.Fatalf("invalidation bypassed in-flight limit: %v", err)
+	}
+	close(release)
+	<-done
+	value, err := cache.get(context.Background(), "one", load)
+	if err != nil || value != 2 {
+		t.Fatalf("in-flight load repopulated invalidated cache: %d %v", value, err)
+	}
+}
+
+func TestReadCacheBudgetsExpiryAndNoStaleSuccess(t *testing.T) {
+	cache := newBoundedReadCache[int, int](3, 512, 1, time.Minute)
+	load := func(context.Context) (int, int, error) { return 1, 256, nil }
+	for i := range 100 {
+		if _, err := cache.get(context.Background(), i, load); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if len(cache.entries) > 2 || cache.bytes > 512 {
+		t.Fatalf("budget exceeded: entries=%d bytes=%d", len(cache.entries), cache.bytes)
+	}
+	cache.entries[99].expires = time.Now().Add(-time.Second)
+	wantErr := errors.New("load failed")
+	value, err := cache.get(context.Background(), 99, func(context.Context) (int, int, error) { return 0, 0, wantErr })
+	if value != 0 || !errors.Is(err, wantErr) {
+		t.Fatalf("stale success served: %d %v", value, err)
+	}
+	var calls int
+	for range 2 {
+		value, err = cache.get(context.Background(), 101, func(context.Context) (int, int, error) {
+			calls++
+			return 42, 1024, nil
+		})
+		if value != 42 || err != nil {
+			t.Fatalf("oversized load should still return to caller: %d %v", value, err)
+		}
+	}
+	if calls != 2 {
+		t.Fatal("oversized entry was retained")
+	}
+}
+
+func TestReadCacheCancelledLoadCanBeRetried(t *testing.T) {
+	cache := newBoundedReadCache[string, int](1, 512, 1, time.Minute)
+	_, err := cache.get(context.Background(), "key", func(context.Context) (int, int, error) { return 0, 0, context.Canceled })
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal(err)
+	}
+	value, err := cache.get(context.Background(), "key", func(context.Context) (int, int, error) { return 7, 256, nil })
+	if err != nil || value != 7 {
+		t.Fatalf("cancellation poisoned cache: %d %v", value, err)
+	}
+}
+
+func TestReadCachePanicDoesNotLeakCapacity(t *testing.T) {
+	cache := newBoundedReadCache[string, int](1, 512, 1, time.Minute)
+	func() {
+		defer func() {
+			if recover() != "loader panic" {
+				t.Error("original panic was swallowed")
+			}
+		}()
+		_, _ = cache.get(context.Background(), "key", func(context.Context) (int, int, error) { panic("loader panic") })
+	}()
+	if cache.loads != 0 || len(cache.entries) != 0 {
+		t.Fatal("panic leaked load capacity")
+	}
+	if _, err := cache.get(context.Background(), "key", func(context.Context) (int, int, error) { return 1, 256, nil }); err != nil {
+		t.Fatal(err)
+	}
+}
--- a/backend/internal/service/runtime_policy.go
+++ b/backend/internal/service/runtime_policy.go
@@ -1,6 +1,7 @@
 package service

 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -187,8 +188,14 @@ func (s *Service) RuntimePolicy() (RuntimePolicySetting, error) {
 }

 func (s *Service) runtimeConcurrencySetting() (RuntimeTaskPolicy, error) {
-	policy, err := s.RuntimePolicy()
-	return policy.Task, err
+	s.initReadCaches()
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	return s.concurrencyReadCache.get(ctx, runtimePolicySettingKey, func(ctx context.Context) (RuntimeTaskPolicy, int, error) {
+		reader := &Service{repo: s.repo.WithContext(ctx)}
+		policy, err := reader.RuntimePolicy()
+		return policy.Task, 256, err
+	})
 }

 func (s *Service) PublicRuntimeLimits() (*PublicRuntimeLimits, error) {
@@ -243,6 +250,8 @@ func (s *Service) UpdateRuntimePolicySetting(actor *model.User, value RuntimePol
 	if err := s.repo.SaveSystemSetting(&setting); err != nil {
 		return nil, err
 	}
+	s.initReadCaches()
+	s.concurrencyReadCache.clear()
 	if err := s.appendAdminAudit(actor, "runtime_policy.update", "system_setting", runtimePolicySettingKey, "更新资源与请求策略", map[string]any{"before": before, "after": value}); err != nil {
 		return nil, err
 	}
@@ -260,6 +269,8 @@ func (s *Service) ResetRuntimePolicySetting(actor *model.User) (*PublicRuntimePo
 	if err := s.repo.DeleteSystemSetting(runtimePolicySettingKey); err != nil {
 		return nil, err
 	}
+	s.initReadCaches()
+	s.concurrencyReadCache.clear()
 	after := defaultRuntimePolicy()
 	if err := s.appendAdminAudit(actor, "runtime_policy.reset", "system_setting", runtimePolicySettingKey, "重置资源与请求策略", map[string]any{"before": before, "after": after}); err != nil {
 		return nil, err
--- a/backend/internal/service/service.go
+++ b/backend/internal/service/service.go
@@ -56,6 +56,12 @@ type Service struct {
 	workers                    *workerRuntime
 	updateManager              UpdateManager
 	mailSender                 func(emailSettingValue, string, string, string) error
+	readCachesOnce             sync.Once
+	concurrencyReadCache       *boundedReadCache[string, RuntimeTaskPolicy]
+	textReplayReadCache        *boundedReadCache[textReplayCacheKey, *TextReplayResult]
+	routeVersionReadCache      *boundedReadCache[string, int64]
+	routeCatalogRetryAt        time.Time
+	routeCatalogRefreshError   error
 }

 const taskWorkerConcurrency = 3
--- a/backend/internal/service/storyboard_workflow.go
+++ b/backend/internal/service/storyboard_workflow.go
@@ -72,7 +72,7 @@ func (s *Service) repairStoryboardPlan(ctx context.Context, task model.Task, inp
 	currentErr := validationErr
 	for attempt := 1; attempt <= maxStoryboardRepairAttempts; attempt++ {
 		_ = s.log(task.UserID, task.ID, "warn", "分镜结构校验失败", fmt.Sprintf("第 %d 次修复前：%s", attempt, currentErr.Error()))
-		if err := s.repo.UpdateTaskProgress(task.ID, "修复分镜结构", 55+attempt*10); err != nil {
+		if err := s.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, "修复分镜结构", 55+attempt*10); err != nil {
 			return agentStoryboardPlan{}, fmt.Errorf("更新分镜修复进度失败，上游修复请求未发出：%w", err)
 		}
 		repairPrompt, promptErr := s.buildStoryboardRepairPrompt(task.UserID, task.Prompt, currentErr, input, currentText)
--- a/backend/internal/service/task_terminal.go
+++ b/backend/internal/service/task_terminal.go
@@ -27,7 +27,7 @@ type taskTerminalCoordinator struct {

 type taskTerminalRepository interface {
 	Task(id string) (*model.Task, error)
-	UpdateTaskTerminalState(id string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error)
+	UpdateTaskTerminalState(id string, owner string, expected model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error)
 }

 type taskBillingLifecycle interface {
@@ -248,9 +248,13 @@ func (c *taskTerminalCoordinator) handleSuccess(task *model.Task) error {
 func (c *taskTerminalCoordinator) markTerminalState(task *model.Task) error {
 	completedAt := time.Now()
 	task.CompletedAt = &completedAt
-	if _, err := c.repo.UpdateTaskTerminalState(task.ID, model.TaskStatusRunning, task.Status, task.Stage, task.Error, completedAt); err != nil {
+	updated, err := c.repo.UpdateTaskTerminalState(task.ID, task.LeaseOwner, model.TaskStatusRunning, task.Status, task.Stage, task.Error, completedAt)
+	if err != nil {
 		return fmt.Errorf("写入任务终态失败：%w", err)
 	}
+	if !updated {
+		return repository.ErrTaskStateConflict
+	}
 	return nil
 }

--- a/backend/internal/service/task_terminal_test.go
+++ b/backend/internal/service/task_terminal_test.go
@@ -7,13 +7,15 @@ import (
 	"time"

 	"infinite-canvas/backend/internal/model"
+	"infinite-canvas/backend/internal/repository"
 )

 type taskTerminalRepositoryStub struct {
-	task          *model.Task
-	taskError     error
-	terminalCalls int
-	terminalError error
+	task             *model.Task
+	taskError        error
+	terminalCalls    int
+	terminalError    error
+	terminalConflict bool
 }

 func (r *taskTerminalRepositoryStub) Task(string) (*model.Task, error) {
@@ -27,8 +29,11 @@ func (r *taskTerminalRepositoryStub) Task(string) (*model.Task, error) {
 	return &copy, nil
 }

-func (r *taskTerminalRepositoryStub) UpdateTaskTerminalState(_ string, _ model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) {
+func (r *taskTerminalRepositoryStub) UpdateTaskTerminalState(_ string, _ string, _ model.TaskStatus, status model.TaskStatus, stage string, errorText string, completedAt time.Time) (bool, error) {
 	r.terminalCalls++
+	if r.terminalConflict {
+		return false, nil
+	}
 	if r.terminalError != nil {
 		return false, r.terminalError
 	}
@@ -41,6 +46,21 @@ func (r *taskTerminalRepositoryStub) UpdateTaskTerminalState(_ string, _ model.T
 	return true, nil
 }

+func TestTaskTerminalConflictDoesNotRefundOrFinalize(t *testing.T) {
+	repo := &taskTerminalRepositoryStub{terminalConflict: true}
+	billing, replay := &taskTerminalBillingStub{}, &taskTerminalReplayStub{}
+	sessions, logger := &taskTerminalSessionStub{}, &taskTerminalLoggerStub{}
+	c := newTaskTerminalCoordinatorForTest(repo, billing, replay, sessions, logger, &taskTerminalOutputStub{})
+	task := &model.Task{ID: "task", LeaseOwner: "stale", Status: model.TaskStatusRunning}
+	err := c.handleExecutionFailure(task, errors.New("upstream failed"), false, false)
+	if !errors.Is(err, repository.ErrTaskStateConflict) {
+		t.Fatalf("missing conflict: %v", err)
+	}
+	if len(billing.refund)+len(billing.uncertain)+billing.settleCalls+len(replay.statuses)+len(sessions.messages) != 0 {
+		t.Fatal("stale worker performed terminal side effects")
+	}
+}
+
 type taskTerminalBillingStub struct {
 	uncertain      []string
 	refund         []string
--- a/backend/internal/service/task_timeline.go
+++ b/backend/internal/service/task_timeline.go
@@ -99,19 +99,19 @@ func (w *taskWorkerCoordinator) processTimelineTranscription(task *model.Task, c

 func (w *taskWorkerCoordinator) failTimelineTask(task *model.Task, stage string, message string) error {
 	s := w.service
-	done, err := s.repo.UpdateTaskTerminalState(task.ID, model.TaskStatusRunning, model.TaskStatusFailed, stage, message, time.Now())
+	done, err := s.repo.UpdateTaskTerminalState(task.ID, task.LeaseOwner, model.TaskStatusRunning, model.TaskStatusFailed, stage, message, time.Now())
 	if err != nil {
 		return fmt.Errorf("写入转写失败态失败: %w", err)
 	}
-	s.logInfo(task.UserID, task.ID, fmt.Sprintf("时间线转写失败: %s", message), "")
 	if !done {
-		return fmt.Errorf("时间线转写已失败: %s", message)
+		return fmt.Errorf("时间线任务状态或租约已变化：%w", repository.ErrTaskStateConflict)
 	}
+	s.logInfo(task.UserID, task.ID, fmt.Sprintf("时间线转写失败: %s", message), "")
 	return nil
 }

 func (w *taskWorkerCoordinator) progress(task *model.Task, stage string, progress int) error {
-	if err := w.service.repo.UpdateTaskProgress(task.ID, stage, progress); err != nil {
+	if err := w.service.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, stage, progress); err != nil {
 		return fmt.Errorf("更新转写进度失败: %w", err)
 	}
 	return nil
--- a/backend/internal/service/task_worker.go
+++ b/backend/internal/service/task_worker.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"strings"
 	"time"

@@ -16,6 +17,8 @@ type taskWorkerCoordinator struct {
 	service *Service
 }

+const workerSlotLeaseDuration = time.Minute
+
 func newTaskWorkerCoordinator(service *Service) *taskWorkerCoordinator {
 	return &taskWorkerCoordinator{service: service}
 }
@@ -41,30 +44,40 @@ func (w *taskWorkerCoordinator) start(ctx context.Context) {
 			}
 			setting, err := s.runtimeConcurrencySetting()
 			if err != nil {
+				log.Printf("task dispatch paused: stage=runtime_policy worker_id=%s error=%v", s.workerID, err)
 				return
 			}
 			workerConcurrency := setting.WorkerConcurrency
 			for len(slots) < workerConcurrency {
-				releaseGlobal, acquired, err := s.coordinator.acquire(ctx, "workers", workerConcurrency, 45*time.Minute)
+				globalSlot, acquired, err := s.coordinator.acquireLease(ctx, "workers", workerConcurrency, workerSlotLeaseDuration)
 				if err != nil || !acquired {
+					if err != nil {
+						log.Printf("task dispatch paused: stage=global_slot worker_id=%s error=%v", s.workerID, err)
+					}
 					return
 				}
-				task, err := s.repo.ClaimNextTask(s.workerID, 45*time.Second)
+				claimCtx, cancelClaim := context.WithTimeout(ctx, 5*time.Second)
+				// 每次领取使用独立 owner，防止同一进程内旧执行者恢复后覆盖新执行者。
+				task, err := s.repo.WithContext(claimCtx).ClaimNextTask(globalSlot.token, 45*time.Second)
+				cancelClaim()
 				if err != nil || task == nil {
-					releaseGlobal()
+					globalSlot.release()
+					if err != nil {
+						log.Printf("task dispatch paused: stage=claim worker_id=%s error=%v", s.workerID, err)
+					}
 					return
 				}
 				slots <- struct{}{}
 				started := s.runWorkerTask(func() {
-					defer func() { <-slots; releaseGlobal() }()
-					if err := w.processClaimedTask(task); err != nil {
+					defer func() { <-slots; globalSlot.release() }()
+					if err := w.processClaimedTask(task, globalSlot); err != nil {
 						_ = s.log(task.UserID, task.ID, "error", "后台任务处理失败", err.Error())
 					}
 				})
 				if !started {
 					<-slots
-					releaseGlobal()
-					_ = s.repo.ReleaseTaskLease(task.ID, s.workerID)
+					globalSlot.release()
+					_ = s.repo.ReleaseTaskLease(task.ID, task.LeaseOwner)
 					return
 				}
 			}
@@ -86,18 +99,20 @@ func (w *taskWorkerCoordinator) start(ctx context.Context) {

 func (w *taskWorkerCoordinator) processNextTask() error {
 	s := w.service
-	task, err := s.repo.ClaimNextTask(s.workerID, 45*time.Second)
+	task, err := s.repo.ClaimNextTask(s.workerID+":"+newID(), 45*time.Second)
 	if err != nil || task == nil {
 		return err
 	}
-	return w.processClaimedTask(task)
+	return w.processClaimedTask(task, nil)
 }

-func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
+func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task, globalSlot *runtimeSlotLease) error {
 	s := w.service
 	terminal := s.terminalCoordinator()
-	_ = s.log(task.UserID, task.ID, "info", "后端任务开始处理", "")
-	policy, err := s.RuntimePolicy()
+	policyCtx, cancelPolicy := context.WithTimeout(context.Background(), 3*time.Second)
+	reader := &Service{repo: s.repo.WithContext(policyCtx)}
+	policy, err := reader.RuntimePolicy()
+	cancelPolicy()
 	if err != nil {
 		return err
 	}
@@ -105,13 +120,23 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 	defer cancel()
 	leaseDone := make(chan struct{})
 	leaseLost := make(chan error, 1)
+	taskID, leaseOwner := task.ID, task.LeaseOwner
 	go func() {
 		ticker := time.NewTicker(15 * time.Second)
 		defer ticker.Stop()
 		for {
 			select {
 			case <-ticker.C:
-				if err := s.repo.RenewTaskLease(task.ID, s.workerID, 45*time.Second); err != nil {
+				renewCtx, cancelRenew := context.WithTimeout(ctx, 5*time.Second)
+				var err error
+				if globalSlot != nil {
+					err = globalSlot.renew(renewCtx)
+				}
+				if err == nil {
+					err = s.repo.WithContext(renewCtx).RenewTaskLease(taskID, leaseOwner, 45*time.Second)
+				}
+				cancelRenew()
+				if err != nil {
 					leaseLost <- err
 					cancel()
 					return
@@ -122,6 +147,7 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 		}
 	}()
 	defer close(leaseDone)
+	_ = s.log(task.UserID, task.ID, "info", "后端任务开始处理", "")
 	s.registerActiveTask(task.ID, cancel)
 	defer s.unregisterActiveTask(task.ID)
 	// 取消请求可能在任务领取和注册 worker context 之间到达；再次读取终态
@@ -147,7 +173,7 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 		task.Stage = "正在连接上游"
 		task.Progress = 0
 	}
-	if err := s.repo.UpdateTaskProgress(task.ID, task.Stage, task.Progress); err != nil {
+	if err := s.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, task.Stage, task.Progress); err != nil {
 		return fmt.Errorf("更新任务进度失败，任务暂未调用上游：%w", err)
 	}
 	routeAttempt, err := s.beginTaskRouteAttempt(task)
@@ -161,6 +187,11 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 	if stateErr != nil {
 		return stateErr
 	}
+	select {
+	case leaseErr := <-leaseLost:
+		return fmt.Errorf("任务租约失效，停止保存上游结果：%w", leaseErr)
+	default:
+	}
 	result, canvasOps, err := routeResult.result, routeResult.canvasOps, routeResult.err
 	providerSucceeded := routeResult.providerSucceeded
 	if err == nil {
--- a/backend/internal/service/text_replay_cache.go
+++ b/backend/internal/service/text_replay_cache.go
@@ -0,0 +1,51 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"infinite-canvas/backend/internal/model"
+)
+
+type textReplayCacheKey struct {
+	userID, taskID string
+	after          int64
+}
+
+func (s *Service) initReadCaches() {
+	s.readCachesOnce.Do(func() {
+		s.concurrencyReadCache = newBoundedReadCache[string, RuntimeTaskPolicy](1, 1024, 1, 2*time.Second)
+		s.textReplayReadCache = newBoundedReadCache[textReplayCacheKey, *TextReplayResult](256, 8<<20, 8, 750*time.Millisecond)
+		s.routeVersionReadCache = newBoundedReadCache[string, int64](1, 1024, 1, 250*time.Millisecond)
+	})
+}
+
+// SSE 展示专用，不用于写路径和权限决策；userID 进入 key，回源仍检查任务归属。
+// 游标保留在 key 中，避免重连读取到其他客户端已经越过的增量窗口。
+func (s *Service) CachedTaskTextReplay(ctx context.Context, userID, taskID string, after int64) (*TextReplayResult, error) {
+	s.initReadCaches()
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+	value, err := s.textReplayReadCache.get(ctx, textReplayCacheKey{userID, taskID, after}, func(ctx context.Context) (*TextReplayResult, int, error) {
+		reader := &Service{repo: s.repo.WithContext(ctx)}
+		value, err := reader.TaskTextReplay(userID, taskID, after)
+		if err != nil {
+			return nil, 0, err
+		}
+		bytes := 512 + len(value.TextDraft) + len(value.FinalText) + len(value.Error) + len(value.Stage)
+		for _, delta := range value.Deltas {
+			bytes += 256 + len(delta.Content) + len(delta.ID) + len(delta.UserID) + len(delta.TaskID)
+		}
+		return value, bytes, nil
+	})
+	if errors.Is(err, errReadCacheBusy) {
+		return nil, &AppError{Status: 503, Code: 503, Message: "任务状态查询繁忙，请稍后重试", Retryable: true}
+	}
+	if err != nil {
+		return nil, err
+	}
+	copy := *value
+	copy.Deltas = append([]model.TaskTextDelta(nil), value.Deltas...)
+	return &copy, nil
+}
--- a/docs/content/docs/backend/meta.json
+++ b/docs/content/docs/backend/meta.json
@@ -6,6 +6,7 @@
    "local-development",
    "code-map",
    "backend-database",
+    "task-runtime-stability",
    "system-update",
    "protocol-plugins",
    "canvas-data-structure"
--- a/docs/content/docs/backend/task-runtime-stability.mdx
+++ b/docs/content/docs/backend/task-runtime-stability.mdx
@@ -0,0 +1,63 @@
+---
+title: 任务缓存与队列稳定性
+description: 热点回源保护、并发租约、故障边界和上线验收
+---
+
+# 任务缓存与队列稳定性
+
+## 范围与事实源
+
+当前任务仍由数据库持久化、API 进程内 worker 领取和执行，Redis 负责跨实例并发协调。本轮是热点读取和租约的止血修复，不是独立任务系统迁移，也不能替代线上事故取证。
+
+- 数据库继续作为任务、额度、账务和权限的事实源。没有把任务或账本迁入 Redis，没有修改表结构或新增迁移。
+- 新增缓存是每个后端实例的有界内存读缓存，不是通用 Redis 业务缓存。实例增加时，缓存容量与不同实例的回源量会随之增加。
+- handler 负责 SSE 协议和登录上下文；service 负责读缓存、并发协调和执行生命周期；repository 负责有条件的租约写入与事务。
+
+## 热点读取保护
+
+| 读路径 | 缓存 / 回源限制 | 故障行为 |
+| --- | --- | --- |
+| worker / 渠道全局并发策略 | 2 秒 TTL，1 条目、1 个同时回源，正文预算 1 KiB | 读取失败不使用默认高并发；暂停投递或返回明确错误 |
+| 文本任务 SSE 回放 | 750 毫秒 TTL，256 条目、8 个不同 key 同时回源，正文预算 8 MiB | 同 key 合并回源；超过容量返回繁忙，不创建无界的不同 key 等待队列 |
+| Redis 路由版本检查 | 250 毫秒 TTL，1 个同时回源 | 失败保留原有本地版本回退；不保证 Redis 故障时跨实例实时失效 |
+| 模型路由目录 | 沿用 30 秒 TTL，刷新串行，失败冷却 2 秒，数据库回源期限 5 秒 | 首次失败明确返回错误；已有快照只能在同版本、最多 5 分钟的原有过期窗口内降级 |
+
+读缓存按 LRU 淘汰，正文预算是估算值，不等于进程总内存上限；回源中的数据、当前请求和等待中的连接不包含在正文预算内。超大结果仅返回给当前读取者，不保留为缓存。普通失败冷却 1 秒，不返回过期成功值；请求取消和超时不缓存。回源异常退出释放容量和等待者，原始 panic 仍交给原有错误处理。
+
+并发策略保存或重置成功后立即清除本实例缓存，其他实例在 2 秒 TTL 后重新读取。计费、配额和其他调用 `RuntimePolicy()` 的强校验路径仍直接读取数据库。
+
+SSE 缓存 key 包含用户、任务和 `after` 游标，回源仍检查归属；返回副本避免调用者改写其他连接的增量。`GET /api/tasks/:id/text-events` 的事件格式、递增序号、`Last-Event-ID` / `?after=` 和 15 秒心跳不变。终态最多受短缓存窗口延迟；已经建立的流遇到回源失败仍发送 error 并关闭。`/text-deltas` 普通查询没有纳入此缓存。
+
+这是合并相同游标的轮询读取，不是事件驱动推送；不同游标和不同实例仍可能分别查询数据库。连接总量、客户端重连退避及任务删除后的缓存失效仍需后续治理。
+
+## worker 与渠道槽位
+
+- worker 全局槽位由固定 45 分钟 TTL 改为 1 分钟租约，每 15 秒与数据库任务租约一起续期；数据库任务租约仍为 45 秒。
+- 每次领取生成独立 owner，不再让一个进程的所有执行共享 owner。领取数据库操作有 5 秒期限，启动策略读取有 3 秒期限。
+- Redis 协调客户端关闭透明命令重试；槽位领取、续期与释放使用最多 2 秒期限。Redis 获取槽位失败时不降级成本机无限执行。
+- 槽位丢失、过期或续期失败会取消该执行的 context。已经失效的槽位不能通过续期复活；释放幂等且只移除本次 token。释放失败记录日志，由租约过期回收。
+- 共享 Redis 槽位键按集合内最晚的租约保留过期余量，避免短任务或新版本的短租约把旧长任务槽位提前删除。旧版本脚本本身不会获得这一保护，仍建议排空后发布。
+- 渠道满载时从 200 毫秒开始指数退避，上限 2 秒，每轮在半窗口至完整窗口间随机等待，避免固定每秒五次同步争抢。空闲时仍先立即尝试领取，取消能立即结束等待。
+- worker 主进度、终态、结果事务及延迟回查要求匹配 owner 且租约仍有效。过期执行者不能覆盖已被重新领取的任务；条件更新没有命中时，终态协调器不得继续退款或完成会话。
+
+**边界**：租约不是外部供应商的幂等保障。上游请求不一定响应取消；提供商状态回调、部分进度和关联资源写入尚未全部传播 owner，不能宣称所有副作用都已 fencing。渠道等待仍发生在领取 worker 槽位之后，也没有新增全局积压上限或公平调度。其他用途的槽位未统一改为续期模型。
+
+## 上线与回滚
+
+1. 先保存故障时段的 Redis 内存、淘汰数量、连接数、延迟和重启记录，以及数据库慢查询、连接池等待、任务 queued/running 数量与最老等待时长。必要日志必须脱敏，不能导出密钥和完整用户正文。
+2. 先在测试环境验证 Redis 与 PostgreSQL 双实例：长任务超过原租期仍只占一个槽位；停止 worker 后槽位恢复；Redis 断连时暂停领取；旧 owner 写入失败；SSE 断线续传无重复或漏终态。
+3. 采用维护窗口排空旧执行者，再发布新版本。Compose 当前默认优雅退出时间由 `CANVAS_SHUTDOWN_TIMEOUT` 控制，默认 10 分钟，不足以保证最长 60 分钟渲染自然完成；发布前核对活跃任务，不能把进程退出视为任务排空成功。
+4. 保留 `canvas:slots:*` 等现有 Redis key。旧进程的 45 分钟分数可能暂时占槽，应先确认旧执行者已结束并观察租约自然回收，**禁止用 FLUSHDB、批量删槽位或批量重投收费任务来解堵**。
+5. 回滚不需要数据迁移，但旧版本仍有原来的固定租期和缺失写入保护。避免新旧版本混跑，不应把回滚视为稳定性问题已经解除。
+
+本次不部署、不清理线上缓存、不修改线上任务或触发真实收费请求。仅凭代码修复和本地测试，不能认定某次线上事故根因已确认或故障已消除。
+
+`TestRuntimeCoordinatorRedisLeasePreservesLongestTTL` 在 PATH 中存在 `redis-server` 时自动启动临时 Unix socket Redis，检查长短租约混用、容量、过期拒绝续期及幂等释放；它不读取 `REDIS_URL`，不连接部署环境。缺少二进制时明确跳过，不视为 Redis 集成已通过。
+
+## 下一阶段
+
+- 可复用业务数据的 Redis 缓存与主动失效，任务状态快照和事件通知，缓存故障时的总量保护。
+- 独立 worker 进程、可靠投递 / outbox、渠道前置排队、积压上限和公平性；按任务类型定义幂等、重试、延迟和补偿。
+- 将执行 owner 传播到所有提供商回调和副作用；明确上游已提交但本地未落库时的查询恢复，不盲目重复生成。
+- 后台手动投递、暂停、恢复和审计；完整的请求 / 任务 / 回调关联日志与脱敏正文留存。
+- 事件驱动 SSE、连接准入和重连退避，替换按连接的数据库轮询。
--- a/docs/content/docs/progress/pending-test.mdx
+++ b/docs/content/docs/progress/pending-test.mdx
@@ -5,6 +5,15 @@ description: 当前项目已实现功能的手工验证清单与未确认边界

 # 待测试

+## 缓存回源与任务队列止血
+
+- 已加入进程内有界读缓存：SSE 按用户、任务、游标合并回源；并发策略短缓存；路由版本短缓存和目录回源失败冷却。强校验的计费、配额、权限读取不改为缓存。
+- worker 改为可续期全局槽位和每次领取独立 owner；数据库进度、终态、完成事务、延迟回查检查有效租约；旧执行者条件更新失败后不继续退款和完成会话。渠道槽位等待改为有上限的随机退避。
+- 已通过定向回归与缓存 / 租约 race 检查，覆盖突发同 key 合并、失败冷却、容量与正文预算、取消、缓存失效、用户 / 游标隔离、调用者副本和旧 owner 写入拒绝。完整 repository / handler 测试通过；完整 service 仍有 4 项既有失败，已在 HEAD 隔离副本复现（渠道私网策略 2 项、视频恢复策略 2 项），不将其标记为全绿。
+- [ ] Redis Lua 与 PostgreSQL 双实例故障注入、长任务续租、worker 强退后恢复及真实 SSE 重连验收；本地缺少 Redis 服务且 Docker daemon 未运行，尚未执行真实 Redis / PostgreSQL 集成测试。
+- [ ] 线上故障时段取证及灰度效果确认；本次没有部署、清缓存或重投任务，不能认定早间事故已修复。
+- [ ] 独立 worker、可靠投递、渠道前置队列、全部回调 owner 校验、业务 Redis 缓存、请求链路日志和事件驱动 SSE 属于后续阶段。范围、上线风险及验收步骤见“开发文档 → 任务缓存与队列稳定性”。
+
 ## 邮箱域名白名单与公告自动提醒（PR #417）

 - 新增注册邮箱域名白名单：默认常见服务商、空列表不限制、精确域名匹配；发送注册验证码及提交注册都会校验，已有账号找回密码不受限制。保存失败不覆盖既有配置。