fix(tasks): 任务队列 - 限制缓存回源并修复并发租约与过期写入

backend/internal/service/task_worker.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"strings"
 	"time"
 
@@ -16,6 +17,8 @@ type taskWorkerCoordinator struct {
 	service *Service
 }
 
+const workerSlotLeaseDuration = time.Minute
+
 func newTaskWorkerCoordinator(service *Service) *taskWorkerCoordinator {
 	return &taskWorkerCoordinator{service: service}
 }
@@ -41,30 +44,40 @@ func (w *taskWorkerCoordinator) start(ctx context.Context) {
 			}
 			setting, err := s.runtimeConcurrencySetting()
 			if err != nil {
+				log.Printf("task dispatch paused: stage=runtime_policy worker_id=%s error=%v", s.workerID, err)
 				return
 			}
 			workerConcurrency := setting.WorkerConcurrency
 			for len(slots) < workerConcurrency {
-				releaseGlobal, acquired, err := s.coordinator.acquire(ctx, "workers", workerConcurrency, 45*time.Minute)
+				globalSlot, acquired, err := s.coordinator.acquireLease(ctx, "workers", workerConcurrency, workerSlotLeaseDuration)
 				if err != nil || !acquired {
+					if err != nil {
+						log.Printf("task dispatch paused: stage=global_slot worker_id=%s error=%v", s.workerID, err)
+					}
 					return
 				}
-				task, err := s.repo.ClaimNextTask(s.workerID, 45*time.Second)
+				claimCtx, cancelClaim := context.WithTimeout(ctx, 5*time.Second)
+				// 每次领取使用独立 owner，防止同一进程内旧执行者恢复后覆盖新执行者。
+				task, err := s.repo.WithContext(claimCtx).ClaimNextTask(globalSlot.token, 45*time.Second)
+				cancelClaim()
 				if err != nil || task == nil {
-					releaseGlobal()
+					globalSlot.release()
+					if err != nil {
+						log.Printf("task dispatch paused: stage=claim worker_id=%s error=%v", s.workerID, err)
+					}
 					return
 				}
 				slots <- struct{}{}
 				started := s.runWorkerTask(func() {
-					defer func() { <-slots; releaseGlobal() }()
-					if err := w.processClaimedTask(task); err != nil {
+					defer func() { <-slots; globalSlot.release() }()
+					if err := w.processClaimedTask(task, globalSlot); err != nil {
 						_ = s.log(task.UserID, task.ID, "error", "后台任务处理失败", err.Error())
 					}
 				})
 				if !started {
 					<-slots
-					releaseGlobal()
-					_ = s.repo.ReleaseTaskLease(task.ID, s.workerID)
+					globalSlot.release()
+					_ = s.repo.ReleaseTaskLease(task.ID, task.LeaseOwner)
 					return
 				}
 			}
@@ -86,18 +99,20 @@ func (w *taskWorkerCoordinator) start(ctx context.Context) {
 
 func (w *taskWorkerCoordinator) processNextTask() error {
 	s := w.service
-	task, err := s.repo.ClaimNextTask(s.workerID, 45*time.Second)
+	task, err := s.repo.ClaimNextTask(s.workerID+":"+newID(), 45*time.Second)
 	if err != nil || task == nil {
 		return err
 	}
-	return w.processClaimedTask(task)
+	return w.processClaimedTask(task, nil)
 }
 
-func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
+func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task, globalSlot *runtimeSlotLease) error {
 	s := w.service
 	terminal := s.terminalCoordinator()
-	_ = s.log(task.UserID, task.ID, "info", "后端任务开始处理", "")
-	policy, err := s.RuntimePolicy()
+	policyCtx, cancelPolicy := context.WithTimeout(context.Background(), 3*time.Second)
+	reader := &Service{repo: s.repo.WithContext(policyCtx)}
+	policy, err := reader.RuntimePolicy()
+	cancelPolicy()
 	if err != nil {
 		return err
 	}
@@ -105,13 +120,23 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 	defer cancel()
 	leaseDone := make(chan struct{})
 	leaseLost := make(chan error, 1)
+	taskID, leaseOwner := task.ID, task.LeaseOwner
 	go func() {
 		ticker := time.NewTicker(15 * time.Second)
 		defer ticker.Stop()
 		for {
 			select {
 			case <-ticker.C:
-				if err := s.repo.RenewTaskLease(task.ID, s.workerID, 45*time.Second); err != nil {
+				renewCtx, cancelRenew := context.WithTimeout(ctx, 5*time.Second)
+				var err error
+				if globalSlot != nil {
+					err = globalSlot.renew(renewCtx)
+				}
+				if err == nil {
+					err = s.repo.WithContext(renewCtx).RenewTaskLease(taskID, leaseOwner, 45*time.Second)
+				}
+				cancelRenew()
+				if err != nil {
 					leaseLost <- err
 					cancel()
 					return
@@ -122,6 +147,7 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 		}
 	}()
 	defer close(leaseDone)
+	_ = s.log(task.UserID, task.ID, "info", "后端任务开始处理", "")
 	s.registerActiveTask(task.ID, cancel)
 	defer s.unregisterActiveTask(task.ID)
 	// 取消请求可能在任务领取和注册 worker context 之间到达；再次读取终态
@@ -147,7 +173,7 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 		task.Stage = "正在连接上游"
 		task.Progress = 0
 	}
-	if err := s.repo.UpdateTaskProgress(task.ID, task.Stage, task.Progress); err != nil {
+	if err := s.repo.UpdateTaskProgressForLease(task.ID, task.LeaseOwner, task.Stage, task.Progress); err != nil {
 		return fmt.Errorf("更新任务进度失败，任务暂未调用上游：%w", err)
 	}
 	routeAttempt, err := s.beginTaskRouteAttempt(task)
@@ -161,6 +187,11 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task) error {
 	if stateErr != nil {
 		return stateErr
 	}
+	select {
+	case leaseErr := <-leaseLost:
+		return fmt.Errorf("任务租约失效，停止保存上游结果：%w", leaseErr)
+	default:
+	}
 	result, canvasOps, err := routeResult.result, routeResult.canvasOps, routeResult.err
 	providerSucceeded := routeResult.providerSucceeded
 	if err == nil {