fix(runtime): 本地任务恢复 - 恢复本地画布和延迟视频任务
This commit is contained in:
1 parent
ea657f179d
commit
bda114b29e
13 files changed
+739
-67
No files matched your search
@@ -294,7 +294,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (*
|
||||
now := time.Now()
|
||||
leaseExpiresAt := now.Add(leaseDuration)
|
||||
err := r.db.Transaction(func(tx *gorm.DB) error {
|
||||
query := tx.Where("status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))", model.TaskStatusQueued, model.TaskStatusRunning, now).
|
||||
query := tx.Where("(status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))) AND (next_poll_at IS NULL OR next_poll_at <= ?)", model.TaskStatusQueued, model.TaskStatusRunning, now, now).
|
||||
Order("created_at asc").Limit(1)
|
||||
if r.Dialect() == "postgres" {
|
||||
query = query.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"})
|
||||
@@ -309,7 +309,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (*
|
||||
}
|
||||
claim := tx.Model(&model.Task{}).Where("id = ?", task.ID)
|
||||
if r.Dialect() != "postgres" {
|
||||
claim = claim.Where("status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))", model.TaskStatusQueued, model.TaskStatusRunning, now)
|
||||
claim = claim.Where("(status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))) AND (next_poll_at IS NULL OR next_poll_at <= ?)", model.TaskStatusQueued, model.TaskStatusRunning, now, now)
|
||||
}
|
||||
updated := claim.
|
||||
Updates(map[string]any{
|
||||
@@ -320,6 +320,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (*
|
||||
"started_at": gorm.Expr("COALESCE(started_at, ?)", now),
|
||||
"lease_owner": owner,
|
||||
"lease_expires_at": leaseExpiresAt,
|
||||
"next_poll_at": nil,
|
||||
"updated_at": now,
|
||||
})
|
||||
if updated.Error != nil {
|
||||
@@ -358,6 +359,23 @@ func (r *Repository) UpdateTaskProviderState(id string, providerRequestID string
|
||||
return r.db.Model(&model.Task{}).Where("id = ?", id).Updates(updates).Error
|
||||
}
|
||||
|
||||
func (r *Repository) DeferRunningTaskForProviderPoll(id string, owner string, stage string, delay time.Duration) error {
|
||||
now := time.Now()
|
||||
result := r.db.Model(&model.Task{}).
|
||||
Where("id = ? AND status = ? AND lease_owner = ?", id, model.TaskStatusRunning, owner).
|
||||
Updates(map[string]any{
|
||||
"stage": stage, "error": "", "completed_at": nil, "next_poll_at": now.Add(delay),
|
||||
"lease_owner": "", "lease_expires_at": nil, "updated_at": now,
|
||||
})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return ErrTaskStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 人工恢复仅锁定失败任务;旧 worker 的租约可覆盖,但未过期的人工恢复租约不能并发抢占。
|
||||
func (r *Repository) ClaimFailedTaskProviderRecovery(id string, userID string, owner string, leaseDuration time.Duration) error {
|
||||
now := time.Now()
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestDeferredProviderPollKeepsOriginalTaskIdentityWithoutImmediateReclaim(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:provider-poll-defer?mode=memory&cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Task{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := model.Task{
|
||||
ID: "task-1", UserID: "user-1", Type: "canvas_video", Status: model.TaskStatusRunning,
|
||||
LeaseOwner: "worker-1", LeaseExpiresAt: ptrTime(time.Now().Add(time.Minute)), ProviderRequestID: "provider-task-1",
|
||||
}
|
||||
if err := db.Create(&task).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := New(db)
|
||||
if err := repo.DeferRunningTaskForProviderPoll(task.ID, task.LeaseOwner, "后台仍在生成", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := repo.ClaimNextTask("worker-2", 45*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claimed != nil {
|
||||
t.Fatalf("task reclaimed before next poll: %#v", claimed)
|
||||
}
|
||||
if err := db.Model(&model.Task{}).Where("id = ?", task.ID).Update("next_poll_at", time.Now().Add(-time.Second)).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err = repo.ClaimNextTask("worker-2", 45*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claimed == nil || claimed.ID != task.ID || claimed.ProviderRequestID != task.ProviderRequestID {
|
||||
t.Fatalf("reclaimed task = %#v", claimed)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrTime(value time.Time) *time.Time { return &value }
|
||||
Reference in new issue
Block a user