fix(runtime): 运行恢复整合 - 合并 PR #253 本地任务恢复改动
This commit is contained in:
commit
f7c5b3b257
13 files changed
+736
-67
No files matched your search
@@ -1854,7 +1854,10 @@ func runNewAPIChannel2VideoTask(ctx context.Context, input canvasGenerationInput
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("NewAPI Video Generations 视频生成超时(任务 %s)", id)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
|
||||
// 单次查询只读取既有上游任务,不创建新任务;自动轮询和人工恢复共用这条安全边界。
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -12,6 +13,9 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8="
|
||||
@@ -1306,6 +1310,59 @@ func TestRunNewAPIChannel2VideoTaskDownloadsTemporaryResult(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNewAPIChannel2VideoTaskResumesOriginalProviderTaskWithoutAnotherPost(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
paths := make([]string, 0, 2)
|
||||
var server *httptest.Server
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
paths = append(paths, r.Method+" "+r.URL.Path)
|
||||
switch r.Method + " " + r.URL.Path {
|
||||
case "GET /v1/video/generations/existing-provider-task":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":"success","data":{"task_id":"existing-provider-task","status":"SUCCESS","result_url":"` + server.URL + `/video.mp4"}}`))
|
||||
case "GET /video.mp4":
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("video"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
input := canvasGenerationInput{
|
||||
Mode: "video",
|
||||
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", Model: "video-model", InterfaceType: string(model.ChannelInterfaceNewAPIChannel2)},
|
||||
}
|
||||
inputJSON, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := withProviderAnalytics(context.Background(), nil, model.Task{
|
||||
ID: "task-1", Type: "canvas_video", ProviderRequestID: "existing-provider-task", InputJSON: string(inputJSON),
|
||||
})
|
||||
result, err := runNewAPIChannel2VideoTask(ctx, input)
|
||||
if err != nil {
|
||||
t.Fatalf("runNewAPIChannel2VideoTask() error = %v", err)
|
||||
}
|
||||
if result["video"] == nil {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
want := "GET /v1/video/generations/existing-provider-task,GET /video.mp4"
|
||||
if got := strings.Join(paths, ","); got != want {
|
||||
t.Fatalf("paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNewAPIChannel2VideoTaskReturnsTypedDeadlineWhenPollingWindowEnds(t *testing.T) {
|
||||
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
defer cancel()
|
||||
ctx = withProviderAnalytics(ctx, nil, model.Task{ID: "task-1", Type: "canvas_video", ProviderRequestID: "existing-provider-task"})
|
||||
_, err := runNewAPIChannel2VideoTask(ctx, canvasGenerationInput{})
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("runNewAPIChannel2VideoTask() error = %v, want context deadline exceeded", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunGeminiVeoVideoTaskUsesLongRunningOperation(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
paths := make([]string, 0, 3)
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
@@ -10,6 +13,76 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestVideoTaskTimeoutHasFiveMinuteSafetyFloor(t *testing.T) {
|
||||
policy := defaultRuntimePolicy().Task
|
||||
policy.VideoTimeoutMinutes = 1
|
||||
policy.ImageTimeoutMinutes = 1
|
||||
if got := taskExecutionTimeoutWithPolicy("canvas_video", policy); got != 5*time.Minute {
|
||||
t.Fatalf("video timeout = %s, want 5m", got)
|
||||
}
|
||||
if got := taskExecutionTimeoutWithPolicy("canvas_image", policy); got != time.Minute {
|
||||
t.Fatalf("image timeout = %s, want 1m", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyResumableNewAPIChannel2VideoDeadlinesStayRunning(t *testing.T) {
|
||||
svc := &Service{}
|
||||
input, err := json.Marshal(canvasGenerationInput{Mode: "video", Config: providerConfig{BaseURL: "https://example.com", InterfaceType: string(model.ChannelInterfaceNewAPIChannel2)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := model.Task{ID: "task-1", Type: "canvas_video", ProviderRequestID: "provider-task-1"}
|
||||
if !svc.shouldDeferVideoProviderTask(base, string(input), context.DeadlineExceeded) {
|
||||
t.Fatal("resumable NewAPI Channel 2 deadline should remain running")
|
||||
}
|
||||
if svc.shouldDeferVideoProviderTask(model.Task{ID: "task-2", Type: "canvas_video"}, string(input), context.DeadlineExceeded) {
|
||||
t.Fatal("missing provider task id must not be deferred")
|
||||
}
|
||||
if svc.shouldDeferVideoProviderTask(base, string(input), context.Canceled) {
|
||||
t.Fatal("explicit cancellation must not be deferred")
|
||||
}
|
||||
other, err := json.Marshal(canvasGenerationInput{Mode: "video", Config: providerConfig{BaseURL: "https://example.com", InterfaceType: string(model.ChannelInterfaceNewAPIVideo)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if svc.shouldDeferVideoProviderTask(base, string(other), context.DeadlineExceeded) {
|
||||
t.Fatal("providers without a verified query-only resume contract must fail closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableVideoDeadlineUsesResolvedSystemChannelProtocol(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+newID()+"?mode=memory&cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.ModelChannel{}, &model.ChannelModel{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
channel := model.ModelChannel{
|
||||
ID: "channel-video", UserID: "admin", Scope: model.ChannelScopeSystem, Enabled: true, Name: "Video",
|
||||
BaseURL: "https://example.com", APIKey: "test-key", APIFormat: "openai", ModelsJSON: `["video-model"]`,
|
||||
}
|
||||
channelModel := model.ChannelModel{
|
||||
ID: "model-video", ChannelID: channel.ID, ModelKey: "video-model", Capability: "video",
|
||||
Protocol: model.ChannelInterfaceNewAPIChannel2, Enabled: true, PriceConfigured: true,
|
||||
}
|
||||
if err := db.Create(&channel).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&channelModel).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input, err := json.Marshal(canvasGenerationInput{Mode: "video", Config: providerConfig{ChannelID: channel.ID, Model: "video-model"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := New(repository.New(db), t.TempDir())
|
||||
task := model.Task{ID: "task-system", Type: "canvas_video", ProviderRequestID: "provider-task-1"}
|
||||
if !svc.shouldDeferVideoProviderTask(task, string(input), context.DeadlineExceeded) {
|
||||
t.Fatal("system-channel task must use the resolved model protocol when deciding query-only recovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimePolicyDefaultsAndSelfUseModeValidate(t *testing.T) {
|
||||
if err := validateRuntimePolicy(defaultRuntimePolicy()); err != nil {
|
||||
t.Fatalf("default runtime policy error = %v", err)
|
||||
|
||||
@@ -990,6 +990,14 @@ func (s *Service) processClaimedTask(task *model.Task) error {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
decryptedInput, decryptErr := s.decryptTaskInputJSON(task.InputJSON)
|
||||
if decryptErr == nil && s.shouldDeferVideoProviderTask(*task, decryptedInput, err) {
|
||||
if deferErr := s.repo.DeferRunningTaskForProviderPoll(task.ID, task.LeaseOwner, "后台仍在生成", 15*time.Second); deferErr != nil {
|
||||
return deferErr
|
||||
}
|
||||
_ = s.log(task.UserID, task.ID, "info", "前台等待结束,上游视频仍在生成,将继续回查原任务", task.PollStage)
|
||||
return nil
|
||||
}
|
||||
err = errors.New(taskTimeoutMessage(task.Type))
|
||||
}
|
||||
task.Status = model.TaskStatusFailed
|
||||
@@ -1079,7 +1087,7 @@ func taskExecutionTimeoutWithPolicy(taskType string, policy RuntimeTaskPolicy) t
|
||||
case taskType == "agent_storyboard" || taskType == "agent_storyboard_rows":
|
||||
return time.Duration(policy.StoryboardTimeoutMinutes) * time.Minute
|
||||
case strings.HasPrefix(taskType, "canvas_video") || strings.HasPrefix(taskType, "video_"):
|
||||
return time.Duration(policy.VideoTimeoutMinutes) * time.Minute
|
||||
return max(time.Duration(policy.VideoTimeoutMinutes)*time.Minute, 5*time.Minute)
|
||||
case strings.HasPrefix(taskType, "canvas_image"):
|
||||
return time.Duration(policy.ImageTimeoutMinutes) * time.Minute
|
||||
case strings.HasPrefix(taskType, "canvas_audio"):
|
||||
@@ -1091,6 +1099,18 @@ func taskExecutionTimeoutWithPolicy(taskType string, policy RuntimeTaskPolicy) t
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) shouldDeferVideoProviderTask(task model.Task, decryptedInput string, err error) bool {
|
||||
if !errors.Is(err, context.DeadlineExceeded) || strings.TrimSpace(task.ProviderRequestID) == "" || (!strings.HasPrefix(task.Type, "canvas_video") && !strings.HasPrefix(task.Type, "video_")) {
|
||||
return false
|
||||
}
|
||||
var input canvasGenerationInput
|
||||
if json.Unmarshal([]byte(decryptedInput), &input) != nil {
|
||||
return false
|
||||
}
|
||||
resolved, resolveErr := s.resolveProviderConfig(input.Config)
|
||||
return resolveErr == nil && resolved.InterfaceType == string(model.ChannelInterfaceNewAPIChannel2)
|
||||
}
|
||||
|
||||
func taskTimeoutMessage(taskType string) string {
|
||||
if strings.HasPrefix(taskType, "canvas_video") || strings.HasPrefix(taskType, "video_") {
|
||||
return "视频生成等待超时,请稍后到任务中心查看或重试。"
|
||||
|
||||
Reference in new issue
Block a user