feat(media): 作品保存恢复 - 按原任务重试交付并避免重复生成计费
This commit is contained in:
@@ -750,7 +750,7 @@ func (s *Service) LogAPICall(log model.ApiCallLog) error {
|
||||
stdlog.Printf("provider billing request id update failed: billing_order_id=%s provider_request_id=%s error=%v", log.BillingOrderID, log.ProviderRequestID, err)
|
||||
}
|
||||
}
|
||||
if log.TaskID != "" {
|
||||
if log.TaskID != "" && (log.RequestKind == "create" || log.RequestKind == "poll" || log.RequestKind == "cancel") {
|
||||
stage := log.RequestKind
|
||||
var nextPollAt *time.Time
|
||||
if stage == "create" && log.Status == model.ApiCallStatusSucceeded && log.ProviderRequestID != "" {
|
||||
@@ -798,7 +798,9 @@ func (s *Service) LogAPICall(log model.ApiCallLog) error {
|
||||
}
|
||||
|
||||
func (s *Service) mergeVideoAPICallLog(log model.ApiCallLog) (bool, error) {
|
||||
if log.Capability != "video" || (log.RequestKind != "poll" && log.RequestKind != "download") {
|
||||
// Delivery is a separate outcome: a failed download must not rewrite a
|
||||
// successful generation request as failed.
|
||||
if log.Capability != "video" || log.RequestKind != "poll" {
|
||||
return false, nil
|
||||
}
|
||||
if log.TaskID == "" && log.ProviderRequestID == "" {
|
||||
|
||||
@@ -246,7 +246,7 @@ func (s *Service) decorateAPICallLogs(logs []model.ApiCallLog) error {
|
||||
billingOrderIDs = append(billingOrderIDs, log.BillingOrderID)
|
||||
}
|
||||
}
|
||||
if (log.Capability != "image" && log.Capability != "video") || log.TaskID == "" {
|
||||
if (log.Capability != "image" && log.Capability != "video" && log.Capability != "audio") || log.TaskID == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seenTaskIDs[log.TaskID]; exists {
|
||||
@@ -303,6 +303,7 @@ func (s *Service) decorateAPICallLogs(logs []model.ApiCallLog) error {
|
||||
}
|
||||
if task, exists := taskByID[logs[index].TaskID]; exists && task.UserID == logs[index].UserID {
|
||||
logs[index].TaskStatus = task.Status
|
||||
logs[index].MediaStage = task.MediaStage
|
||||
previewURL, previewKind := taskMediaPreview(task.ResultJSON, task.Type)
|
||||
if canvasResourceID(previewURL) != "" {
|
||||
logs[index].MediaPreviewURL = "/api/admin/api-logs/" + logs[index].ID + "/media"
|
||||
|
||||
@@ -789,6 +789,9 @@ func finishProtocolResult(ctx context.Context, config providerConfig, mode strin
|
||||
if len(references) == 0 {
|
||||
return nil, errors.New("声明式协议已完成但没有返回媒体地址")
|
||||
}
|
||||
if output, handled, err := recoverProtocolMedia(ctx, config, mode, references); handled {
|
||||
return output, err
|
||||
}
|
||||
items := make([]interface{}, 0, len(references))
|
||||
for _, reference := range references {
|
||||
var data []byte
|
||||
|
||||
@@ -71,6 +71,9 @@ func (s *Service) queryFailedVideoTask(ctx context.Context, task *model.Task, cl
|
||||
if task.Status != model.TaskStatusFailed {
|
||||
return nil, BadAuthRequest("只能人工查询状态为失败的任务")
|
||||
}
|
||||
if task.MediaRecoveryJSON != "" {
|
||||
return nil, BadAuthRequest("该任务已生成作品,请在任务中心重试保存,无需重新查询生成")
|
||||
}
|
||||
if !strings.HasPrefix(task.Type, "canvas_video") && !strings.HasPrefix(task.Type, "video_") {
|
||||
return nil, BadAuthRequest("该任务不是视频生成任务")
|
||||
}
|
||||
|
||||
@@ -491,6 +491,10 @@ func (s *Service) openResourceRange(userID string, resource *model.Resource, ran
|
||||
}
|
||||
|
||||
func (s *Service) storeResource(userID string, kind string, fileName string, mimeType string, size int64, width int, height int, durationMs int64, body io.Reader, uploadKey *string, forceLocal bool) (*model.Resource, bool, error) {
|
||||
return s.storeResourceWithWriter(userID, kind, fileName, mimeType, size, width, height, durationMs, body, uploadKey, forceLocal, s.storeResourceObject)
|
||||
}
|
||||
|
||||
func (s *Service) storeResourceWithWriter(userID string, kind string, fileName string, mimeType string, size int64, width int, height int, durationMs int64, body io.Reader, uploadKey *string, forceLocal bool, writeObject func(*model.Resource, string, io.Reader) (string, error)) (*model.Resource, bool, error) {
|
||||
if existing, err := s.resourceForUploadKey(userID, uploadKey); err != nil {
|
||||
return nil, false, err
|
||||
} else if existing != nil {
|
||||
@@ -536,7 +540,7 @@ func (s *Service) storeResource(userID string, kind string, fileName string, mim
|
||||
return nil, false, err
|
||||
}
|
||||
var etag string
|
||||
etag, err = s.storeResourceObject(&resource, fileName, body)
|
||||
etag, err = writeObject(&resource, fileName, body)
|
||||
resource.UpdatedAt = time.Now()
|
||||
if err != nil {
|
||||
resource.Status = model.ResourceStatusFailed
|
||||
|
||||
@@ -150,7 +150,18 @@ func (s *Service) saveTaskCompletionWithinStorageQuota(task *model.Task, resultJ
|
||||
completed.ResultJSON = string(resultJSON)
|
||||
completed.InputJSON = publicInputJSON
|
||||
completed.CompletedAt = ptr(time.Now())
|
||||
if err := s.repo.SaveTaskCompletion(&completed, expectedStatus, results); err != nil {
|
||||
var register func(*repository.Repository) error
|
||||
if task.MediaRecoveryJSON != "" {
|
||||
completed.MediaStage = "completed"
|
||||
register = func(repo *repository.Repository) error {
|
||||
writer := &Service{repo: repo, dataDir: s.dataDir}
|
||||
if err := writer.RegisterTaskOutputFromTask(completed); err != nil {
|
||||
return err
|
||||
}
|
||||
return writer.registerRecoveredMediaAssets(completed)
|
||||
}
|
||||
}
|
||||
if err := s.repo.SaveTaskCompletionWithRegistration(&completed, expectedStatus, results, register); err != nil {
|
||||
return err
|
||||
}
|
||||
*task = completed
|
||||
|
||||
@@ -22,6 +22,7 @@ func (s *Service) processTask(ctx context.Context, task model.Task) (map[string]
|
||||
task.InputJSON = decryptedInput
|
||||
ctx = withTaskExecutionID(ctx, task.ID)
|
||||
ctx = withProviderAnalytics(ctx, s, task)
|
||||
ctx = context.WithValue(ctx, mediaExecutionTaskKey{}, task)
|
||||
|
||||
if task.Type == "canvas_text" || task.Type == "canvas_image" || task.Type == "canvas_video" || task.Type == "canvas_audio" {
|
||||
result, err := s.processCanvasGenerationTask(ctx, task.UserID, task.ProjectID, task.Type, task.Prompt, task.InputJSON)
|
||||
|
||||
@@ -39,6 +39,9 @@ func (w *taskLifecycleCoordinator) retryTask(userID string, id string) (*model.T
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if task.MediaRecoveryJSON != "" {
|
||||
return nil, BadAuthRequest("作品已生成,请使用重试保存,不要重复生成")
|
||||
}
|
||||
if task.CreationSubmissionID != nil {
|
||||
return nil, creationConflict("智能创作重做需要新的报价批准,请回到创作会话继续")
|
||||
}
|
||||
|
||||
129
backend/internal/app/task_media_assets.go
Normal file
129
backend/internal/app/task_media_assets.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
// A deleted canvas/project must not be resurrected by an old download worker.
|
||||
func (s *Service) mediaTaskProject(task *model.Task) (string, error) {
|
||||
id := task.ProjectID
|
||||
if id == "" {
|
||||
return "", nil
|
||||
}
|
||||
canvas, err := s.repo.CanvasProjectForUser(task.UserID, id)
|
||||
if err == nil {
|
||||
id = canvas.ProjectID
|
||||
if id == "" {
|
||||
return "", nil
|
||||
}
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return "", err
|
||||
}
|
||||
project, err := s.repo.ProjectForUser(task.UserID, id)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return "", NotFound("任务所属画布或项目已不存在")
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if project.Status == model.ProjectStatusArchived {
|
||||
return "", BadAuthRequest("项目已归档,无法恢复作品")
|
||||
}
|
||||
return project.ID, nil
|
||||
}
|
||||
|
||||
// Uses the same effect key and ID as browser materialization, so reconnecting
|
||||
// the canvas cannot create a second asset for a server-delivered output.
|
||||
func (s *Service) registerRecoveredMediaAssets(task model.Task) error {
|
||||
projectID, err := s.mediaTaskProject(&task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checkpoint, err := s.decodeMediaCheckpoint(&task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
representations, err := s.repo.AssetRepresentationsForTask(task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
policy, err := s.RuntimePolicy()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for index, item := range checkpoint.Items {
|
||||
registered := false
|
||||
for _, representation := range representations {
|
||||
if representation.ResourceID == item.ResourceID && representation.AssetVersionID != "" {
|
||||
registered = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if registered {
|
||||
continue
|
||||
}
|
||||
resource, err := s.repo.ResourceForUser(task.UserID, item.ResourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resource.Status != model.ResourceStatusReady {
|
||||
return errors.New("作品文件尚未保存完成")
|
||||
}
|
||||
effectKey := fmt.Sprintf("materialize:%s:%d", task.ID, index)
|
||||
sum := sha256.Sum256([]byte(effectKey))
|
||||
assetID := "generation_" + hex.EncodeToString(sum[:])
|
||||
// Preserve existing user edits on replay; the deterministic key is unique.
|
||||
if _, err := s.repo.AssetForUser(task.UserID, assetID); err == nil {
|
||||
continue
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
url := resourceFileURL(resource.ID)
|
||||
data := map[string]any{"storageKey": "resource:" + resource.ID, "mimeType": resource.MimeType, "bytes": resource.Size, "width": resource.Width, "height": resource.Height, "durationMs": resource.DurationMs}
|
||||
if checkpoint.Mode == "image" {
|
||||
data["dataUrl"] = url
|
||||
} else {
|
||||
data["url"] = url
|
||||
}
|
||||
title := "生成作品"
|
||||
metadata := map[string]any{"source": "generation-task", "generationEffectKey": effectKey, "taskId": task.ID, "outputIndex": index}
|
||||
if projectID != "" {
|
||||
metadata["projectIds"] = []string{projectID}
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{"id": assetID, "kind": checkpoint.Mode, "category": model.AssetCategoryMaterial, "status": model.AssetVersionStatusConfirmed, "title": title, "coverUrl": url, "tags": []string{"生成"}, "createdAt": now.UTC().Format(time.RFC3339Nano), "updatedAt": now.UTC().Format(time.RFC3339Nano), "data": data, "metadata": metadata})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
usage, err := s.repo.UserStorageUsage(task.UserID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateStructuredStorageQuotaWithPolicy(usage, "asset", true, int64(len(payload)), policy.Resource); err != nil {
|
||||
return err
|
||||
}
|
||||
asset := &model.Asset{ID: assetID, UserID: task.UserID, Kind: checkpoint.Mode, Category: model.AssetCategoryMaterial, Status: model.AssetVersionStatusConfirmed, Title: title, PayloadJSON: string(payload), CreatedAt: now, UpdatedAt: now}
|
||||
if err := s.repo.UpsertAsset(asset); err != nil {
|
||||
return err
|
||||
}
|
||||
if projectID != "" {
|
||||
position, err := s.repo.NextProjectAssetPosition(projectID, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.repo.LinkProjectAsset(asset, nil, &model.ProjectAssetLink{ID: newID(), ProjectID: projectID, AssetID: assetID, Position: position, CreatedAt: now})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
428
backend/internal/app/task_media_recovery.go
Normal file
428
backend/internal/app/task_media_recovery.go
Normal file
@@ -0,0 +1,428 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/protocol"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
)
|
||||
|
||||
// Recovery is keyed by the task, not a second generation or billing order.
|
||||
type mediaCheckpoint struct {
|
||||
Mode string `json:"mode"`
|
||||
Items []mediaCheckpointItem `json:"items"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
Attempts int `json:"attempts"`
|
||||
ManualAttempts int `json:"manualAttempts"`
|
||||
LastManualAt time.Time `json:"lastManualAt"`
|
||||
}
|
||||
|
||||
type mediaCheckpointItem struct {
|
||||
Reference protocol.MediaReference `json:"reference"`
|
||||
TempName string `json:"tempName,omitempty"`
|
||||
MIMEType string `json:"mimeType,omitempty"`
|
||||
ResourceID string `json:"resourceId,omitempty"`
|
||||
}
|
||||
|
||||
type mediaRecoveryError struct {
|
||||
stage string
|
||||
retryable bool
|
||||
cause error
|
||||
}
|
||||
type mediaExecutionTaskKey struct{}
|
||||
|
||||
func (e *mediaRecoveryError) Error() string {
|
||||
return "作品已生成,但保存未完成:" + mediaStageLabel(e.stage)
|
||||
}
|
||||
func (e *mediaRecoveryError) Unwrap() error { return e.cause }
|
||||
|
||||
var mediaRecoverySlots = make(chan struct{}, 2)
|
||||
var mediaRecoveryDelays = []time.Duration{15 * time.Second, time.Minute, 3 * time.Minute, 10 * time.Minute}
|
||||
|
||||
func mediaStageLabel(stage string) string {
|
||||
switch stage {
|
||||
case "download":
|
||||
return "下载结果失败"
|
||||
case "upload":
|
||||
return "上传 OSS 失败"
|
||||
case "local_save":
|
||||
return "保存本地文件失败"
|
||||
case "register":
|
||||
return "登记作品失败"
|
||||
default:
|
||||
return "记录恢复信息失败"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) decodeMediaCheckpoint(task *model.Task) (*mediaCheckpoint, error) {
|
||||
plain, err := s.decryptSettingSecret(task.MediaRecoveryJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var checkpoint mediaCheckpoint
|
||||
if err := json.Unmarshal([]byte(plain), &checkpoint); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(checkpoint.Items) == 0 || len(checkpoint.Items) > 32 || (checkpoint.Mode != "image" && checkpoint.Mode != "video" && checkpoint.Mode != "audio") {
|
||||
return nil, errors.New("作品恢复信息无效")
|
||||
}
|
||||
return &checkpoint, nil
|
||||
}
|
||||
|
||||
func (s *Service) saveMediaCheckpoint(task *model.Task, checkpoint *mediaCheckpoint, stage string) error {
|
||||
data, err := json.Marshal(checkpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := s.encryptSettingSecret(string(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.repo.SaveTaskMediaCheckpoint(task, encoded, stage)
|
||||
}
|
||||
|
||||
func recoverProtocolMedia(ctx context.Context, config providerConfig, mode string, references []protocol.MediaReference) (map[string]interface{}, bool, error) {
|
||||
metadata, ok := ctx.Value(providerAnalyticsKey{}).(providerAnalyticsContext)
|
||||
if !ok || metadata.Service == nil || metadata.TaskID == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
s := metadata.Service
|
||||
executing, ok := ctx.Value(mediaExecutionTaskKey{}).(model.Task)
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
task := &executing
|
||||
var err error
|
||||
// Manual video queries retain their existing recovery lease and completion path.
|
||||
if task.Status != model.TaskStatusRunning {
|
||||
return nil, false, nil
|
||||
}
|
||||
if len(references) == 0 || len(references) > 32 {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: errors.New("生成结果数量超过恢复限制")}
|
||||
}
|
||||
if mode != "image" {
|
||||
references = references[:1]
|
||||
}
|
||||
if task.MediaRecoveryJSON != "" {
|
||||
result, err := s.resumeTaskMedia(ctx, task)
|
||||
return result, true, err
|
||||
}
|
||||
checkpoint := &mediaCheckpoint{Mode: mode, StartedAt: time.Now()}
|
||||
for _, reference := range references {
|
||||
if len(reference.URL) > 16_384 {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: errors.New("上游媒体地址过长")}
|
||||
}
|
||||
item := mediaCheckpointItem{Reference: protocol.MediaReference{URL: reference.URL, MIMEType: reference.MIMEType}}
|
||||
// Inline bytes are staged, never stored as a large base64 DB checkpoint.
|
||||
if reference.DataURL != "" {
|
||||
mimeType, data, decodeErr := decodeProviderDataURL(reference.DataURL)
|
||||
if decodeErr != nil {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: decodeErr}
|
||||
}
|
||||
policy, policyErr := s.RuntimePolicy()
|
||||
if policyErr != nil {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: policyErr}
|
||||
}
|
||||
if int64(len(data)) > min(megabytes(policy.Resource.GeneratedFileMB), mediaStagingBudget) || !strings.HasPrefix(mimeType, mode+"/") {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: errors.New("生成文件类型或大小不符合限制")}
|
||||
}
|
||||
item.TempName, err = s.stageInlineMedia(data)
|
||||
if err != nil {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: err}
|
||||
}
|
||||
item.MIMEType = mimeType
|
||||
item.Reference.DataURL = ""
|
||||
}
|
||||
checkpoint.Items = append(checkpoint.Items, item)
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "download"); err != nil {
|
||||
return nil, true, &mediaRecoveryError{stage: "checkpoint", cause: err}
|
||||
}
|
||||
result, err := s.materializeTaskMedia(ctx, task, config, checkpoint)
|
||||
return result, true, err
|
||||
}
|
||||
|
||||
func (s *Service) resumeTaskMedia(ctx context.Context, task *model.Task) (map[string]interface{}, error) {
|
||||
checkpoint, err := s.decodeMediaCheckpoint(task)
|
||||
if err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "checkpoint", cause: err}
|
||||
}
|
||||
raw, err := s.decryptTaskInputJSON(task.InputJSON)
|
||||
if err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "checkpoint", cause: err}
|
||||
}
|
||||
var input canvasGenerationInput
|
||||
if err := json.Unmarshal([]byte(raw), &input); err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "checkpoint", cause: err}
|
||||
}
|
||||
return s.materializeTaskMedia(withProviderAnalytics(ctx, s, *task), task, input.Config, checkpoint)
|
||||
}
|
||||
|
||||
func (s *Service) materializeTaskMedia(ctx context.Context, task *model.Task, config providerConfig, checkpoint *mediaCheckpoint) (result map[string]interface{}, err error) {
|
||||
defer func() {
|
||||
var failure *mediaRecoveryError
|
||||
if err != nil && !errors.As(err, &failure) {
|
||||
err = &mediaRecoveryError{stage: task.MediaStage, cause: err}
|
||||
}
|
||||
}()
|
||||
// Do not occupy a worker waiting for a download slot.
|
||||
select {
|
||||
case mediaRecoverySlots <- struct{}{}:
|
||||
defer func() { <-mediaRecoverySlots }()
|
||||
default:
|
||||
return nil, &mediaRecoveryError{stage: task.MediaStage, retryable: true, cause: errMediaRecoveryBusy}
|
||||
}
|
||||
if _, err := s.mediaTaskProject(task); err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "register", cause: err}
|
||||
}
|
||||
items := make([]interface{}, 0, len(checkpoint.Items))
|
||||
for index := range checkpoint.Items {
|
||||
item := &checkpoint.Items[index]
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, &mediaRecoveryError{stage: task.MediaStage, retryable: true, cause: err}
|
||||
}
|
||||
resource, err := s.resourceForUploadKey(task.UserID, mediaUploadKey(task.ID, index))
|
||||
if err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "register", retryable: true, cause: err}
|
||||
}
|
||||
if resource == nil || resource.Status != model.ResourceStatusReady {
|
||||
path := s.mediaTempPath(item.TempName)
|
||||
info, statErr := os.Lstat(path)
|
||||
if path == "" || statErr != nil || !info.Mode().IsRegular() {
|
||||
if item.Reference.URL == "" {
|
||||
return nil, &mediaRecoveryError{stage: "download", cause: errors.New("暂存结果已过期且上游没有可重新下载的地址")}
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "download"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config, err = s.resolveProviderConfig(config)
|
||||
if err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "download", cause: err}
|
||||
}
|
||||
item.TempName, item.MIMEType, err = s.downloadTaskMedia(ctx, config, item.Reference.URL, checkpoint.Mode)
|
||||
if err != nil {
|
||||
return nil, &mediaRecoveryError{stage: "download", retryable: retryableMediaRecovery(err), cause: err}
|
||||
}
|
||||
path = s.mediaTempPath(item.TempName)
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "download"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
stage := "local_save"
|
||||
_, _, oss, settingErr := s.activeResourceOSSSetting(task.UserID)
|
||||
if settingErr != nil {
|
||||
return nil, &mediaRecoveryError{stage: "upload", cause: settingErr}
|
||||
}
|
||||
if oss || (resource != nil && resource.Provider != "local") {
|
||||
stage = "upload"
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, stage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
started := time.Now()
|
||||
resource, err = s.storeTaskMediaFile(task, index, path, item.MIMEType, checkpoint.Mode, resource)
|
||||
s.logMediaStage(*task, stage, started, err)
|
||||
if err != nil {
|
||||
return nil, &mediaRecoveryError{stage: stage, retryable: retryableMediaRecovery(err), cause: err}
|
||||
}
|
||||
}
|
||||
item.ResourceID = resource.ID
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "register"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
url := resourceFileURL(resource.ID)
|
||||
items = append(items, map[string]interface{}{"dataUrl": url, "url": url, "resourceId": resource.ID, "storageKey": "resource:" + resource.ID, "mimeType": resource.MimeType, "bytes": resource.Size, "width": resource.Width, "height": resource.Height, "durationMs": resource.DurationMs})
|
||||
}
|
||||
result = map[string]interface{}{"mode": checkpoint.Mode}
|
||||
if checkpoint.Mode == "image" {
|
||||
result["images"] = items
|
||||
} else {
|
||||
result[checkpoint.Mode] = items[0]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var errMediaRecoveryBusy = errors.New("作品保存并发已满")
|
||||
|
||||
func retryableMediaRecovery(err error) bool {
|
||||
if retryableProtocolMediaDownload(err) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var upstream providerHTTPError
|
||||
return errors.As(err, &upstream) && (upstream.StatusCode == 429 || upstream.StatusCode == 408 || upstream.StatusCode >= 500)
|
||||
}
|
||||
|
||||
// Called before the generic execution failure path, so result delivery cannot be
|
||||
// mistaken for model failure or trigger a second generation.
|
||||
func (s *Service) handleMediaRecoveryFailure(task *model.Task, cause error) error {
|
||||
latest, err := s.repo.Task(task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if latest.Status != model.TaskStatusRunning || latest.LeaseOwner != task.LeaseOwner {
|
||||
return repository.ErrTaskStateConflict
|
||||
}
|
||||
task.MediaRecoveryJSON, task.MediaStage = latest.MediaRecoveryJSON, latest.MediaStage
|
||||
var failure *mediaRecoveryError
|
||||
if !errors.As(cause, &failure) {
|
||||
failure = &mediaRecoveryError{stage: task.MediaStage, retryable: true, cause: cause}
|
||||
}
|
||||
if task.MediaRecoveryJSON != "" {
|
||||
checkpoint, decodeErr := s.decodeMediaCheckpoint(task)
|
||||
if decodeErr != nil {
|
||||
return decodeErr
|
||||
}
|
||||
if errors.Is(cause, errMediaRecoveryBusy) && time.Since(checkpoint.StartedAt) < 20*time.Minute {
|
||||
return s.repo.DeferRunningTaskForProviderPoll(task.ID, task.LeaseOwner, "作品已生成,等待保存", 5*time.Second)
|
||||
}
|
||||
if failure.retryable && checkpoint.Attempts < len(mediaRecoveryDelays) && time.Since(checkpoint.StartedAt) < 20*time.Minute {
|
||||
delay := mediaRecoveryDelays[checkpoint.Attempts] + time.Duration(rand.IntN(5_000))*time.Millisecond
|
||||
var upstream providerHTTPError
|
||||
if errors.As(cause, &upstream) {
|
||||
delay = max(delay, min(upstream.RetryAfter, 10*time.Minute))
|
||||
}
|
||||
checkpoint.Attempts++
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, failure.stage); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.repo.DeferRunningTaskForProviderPoll(task.ID, task.LeaseOwner, "作品已生成,保存遇到问题,正在自动恢复", delay)
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, failure.stage); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Billing belongs to the original generation. A delivery failure cannot refund
|
||||
// or submit it again implicitly; the existing reconciliation path is retained.
|
||||
return s.terminalCoordinator().handleResultPersistenceFailureError(task, failure)
|
||||
}
|
||||
|
||||
func (s *Service) RecoverTaskMedia(userID, id string) (*model.Task, error) {
|
||||
if s.IsDraining() {
|
||||
return nil, BadAuthRequest("服务正在维护,请稍后重试保存")
|
||||
}
|
||||
task, err := s.repo.TaskForUser(userID, strings.TrimSpace(id))
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, NotFound("任务不存在")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if task.MediaRecoveryJSON == "" {
|
||||
return nil, BadAuthRequest("该任务没有可恢复的作品,请先核对上游结果")
|
||||
}
|
||||
if task.Status == model.TaskStatusRunning || task.Status == model.TaskStatusQueued || task.Status == model.TaskStatusSucceeded {
|
||||
return taskForOutput(*task), nil
|
||||
}
|
||||
if task.Status != model.TaskStatusFailed {
|
||||
return nil, BadAuthRequest("仅保存失败的作品可以恢复")
|
||||
}
|
||||
if _, err := s.mediaTaskProject(task); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.taskBilling().CheckRetryEligibility(task.BillingOrderID); err != nil && !errors.Is(err, errTaskBillingReview) {
|
||||
return nil, err
|
||||
}
|
||||
// Never silently restore a refunded order (the old video recovery does that).
|
||||
if task.BillingOrderID != "" {
|
||||
order, err := s.repo.BillingOrder(task.BillingOrderID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if order.UserID != task.UserID || order.TaskID != task.ID {
|
||||
return nil, BadAuthRequest("任务与计费订单归属不一致")
|
||||
}
|
||||
if order.Status == model.BillingStatusRefunded {
|
||||
return nil, BadAuthRequest("原任务已退款,请联系管理员处理作品,不会自动重新扣费")
|
||||
}
|
||||
}
|
||||
checkpoint, err := s.decodeMediaCheckpoint(task)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if checkpoint.ManualAttempts >= 3 {
|
||||
return nil, BadAuthRequest("已达到重试保存上限,请联系管理员核对渠道结果")
|
||||
}
|
||||
if time.Since(checkpoint.LastManualAt) < time.Minute {
|
||||
return nil, BadAuthRequest("请稍后再试,重试保存间隔至少一分钟")
|
||||
}
|
||||
checkpoint.ManualAttempts++
|
||||
checkpoint.LastManualAt = time.Now()
|
||||
// Manual recovery is one attempt, not another full automatic retry budget.
|
||||
checkpoint.Attempts = len(mediaRecoveryDelays)
|
||||
data, err := json.Marshal(checkpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encrypted, err := s.encryptSettingSecret(string(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.repo.RequeueTaskMediaRecovery(task, encrypted); err != nil {
|
||||
if errors.Is(err, repository.ErrTaskStateConflict) {
|
||||
if latest, readErr := s.repo.TaskForUser(userID, task.ID); readErr == nil && (latest.Status == model.TaskStatusQueued || latest.Status == model.TaskStatusRunning || latest.Status == model.TaskStatusSucceeded) {
|
||||
return taskForOutput(*latest), nil
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
next, err := s.repo.TaskForUser(userID, task.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return taskForOutput(*next), nil
|
||||
}
|
||||
|
||||
func (s *Service) logMediaStage(task model.Task, stage string, started time.Time, err error) {
|
||||
status := model.ApiCallStatusSucceeded
|
||||
code, message := "", ""
|
||||
if err != nil {
|
||||
status = model.ApiCallStatusFailed
|
||||
code = "media_" + stage + "_failed"
|
||||
message = mediaStageLabel(stage)
|
||||
}
|
||||
if logErr := s.LogAPICall(model.ApiCallLog{UserID: task.UserID, TaskID: task.ID, BillingOrderID: task.BillingOrderID, ProviderRequestID: task.ProviderRequestID, Source: "backend-task", Capability: capabilityFromTaskType(task.Type), Model: task.Model, RequestKind: stage, Method: "INTERNAL", Path: "task/media/" + stage, Status: status, ErrorCode: code, Error: message, DurationMs: time.Since(started).Milliseconds()}); logErr != nil {
|
||||
_ = s.log(task.UserID, task.ID, "error", "记录作品保存阶段失败", logErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) mediaTempPath(name string) string {
|
||||
if !strings.HasPrefix(name, "result-") || filepath.Base(name) != name || strings.ContainsAny(name, `/\\`) {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(s.dataDir, "media-staging", name)
|
||||
}
|
||||
|
||||
func (c *taskTerminalCoordinator) handleResultPersistenceFailureError(task *model.Task, err error) error {
|
||||
_, failure := c.handleResultPersistenceFailure(task, err)
|
||||
return failure
|
||||
}
|
||||
|
||||
func (s *Service) finishTaskMediaRecovery(task *model.Task, result map[string]interface{}, err error) error {
|
||||
if err != nil {
|
||||
return s.handleMediaRecoveryFailure(task, err)
|
||||
}
|
||||
started := time.Now()
|
||||
if _, err = s.finalizeCharacterTurnaroundTask(*task, result); err == nil {
|
||||
var encoded []byte
|
||||
encoded, err = json.Marshal(result)
|
||||
if err == nil {
|
||||
err = s.saveTaskCompletionWithinStorageQuota(task, encoded, nil, false)
|
||||
}
|
||||
}
|
||||
s.logMediaStage(*task, "register", started, err)
|
||||
if err != nil {
|
||||
return s.handleMediaRecoveryFailure(task, &mediaRecoveryError{stage: "register", cause: err})
|
||||
}
|
||||
s.cleanupMediaCheckpointFiles(task)
|
||||
return s.terminalCoordinator().handleSuccess(task)
|
||||
}
|
||||
411
backend/internal/app/task_media_recovery_test.go
Normal file
411
backend/internal/app/task_media_recovery_test.go
Normal file
@@ -0,0 +1,411 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"image"
|
||||
"image/png"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"infinite-canvas/backend/internal/database"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/protocol"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
)
|
||||
|
||||
func newMediaRecoveryTestService(t *testing.T) (*Service, *gorm.DB) {
|
||||
t.Helper()
|
||||
db := newSQLiteTestDB(t)
|
||||
if err := database.MigrateSchema(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &Service{repo: repository.New(db), dataDir: t.TempDir(), activeCancels: make(map[string]context.CancelFunc)}, db
|
||||
}
|
||||
|
||||
func mediaTestPNG(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
if err := png.Encode(&buffer, image.NewRGBA(image.Rect(0, 0, 2, 3))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func seedMediaTask(t *testing.T, db *gorm.DB, config providerConfig) *model.Task {
|
||||
t.Helper()
|
||||
input, err := json.Marshal(canvasGenerationInput{Mode: "image", Config: config, Prompt: "test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expires := time.Now().Add(time.Hour)
|
||||
task := &model.Task{ID: newID(), UserID: "media-user", Type: "canvas_image", Status: model.TaskStatusRunning, InputJSON: string(input), LeaseOwner: "worker-a", LeaseExpiresAt: &expires, RouteRun: 1, CreatedAt: time.Now()}
|
||||
if err := db.Create(task).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
func TestMediaRecoveryDownloadFailureResumesSameTaskAfterRestart(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1")
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
picture := mediaTestPNG(t)
|
||||
var downloads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
t.Errorf("recovery submitted generation: %s", r.Method)
|
||||
}
|
||||
if downloads.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(picture)
|
||||
}))
|
||||
defer server.Close()
|
||||
config := providerConfig{BaseURL: server.URL}
|
||||
task := seedMediaTask(t, db, config)
|
||||
ctx := context.WithValue(withProviderAnalytics(context.Background(), s, *task), mediaExecutionTaskKey{}, *task)
|
||||
_, handled, failure := recoverProtocolMedia(ctx, config, "image", []protocol.MediaReference{{URL: server.URL + "/result?token=private-test-token"}})
|
||||
if !handled || failure == nil {
|
||||
t.Fatalf("handled=%v failure=%v", handled, failure)
|
||||
}
|
||||
if err := s.handleMediaRecoveryFailure(task, failure); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, err := s.repo.Task(task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.Status != model.TaskStatusRunning || saved.NextPollAt == nil || saved.LeaseOwner != "" {
|
||||
t.Fatalf("not deferred: %+v", saved)
|
||||
}
|
||||
if saved.MediaRecoveryJSON == "" || strings.Contains(saved.MediaRecoveryJSON, "private-test-token") {
|
||||
t.Fatal("missing/encrypted checkpoint")
|
||||
}
|
||||
public, _ := json.Marshal(taskForOutput(*saved))
|
||||
if bytes.Contains(public, []byte("mediaRecovery")) || bytes.Contains(public, []byte("private-test-token")) {
|
||||
t.Fatal("checkpoint leaked")
|
||||
}
|
||||
if err := db.Model(&model.Task{}).Where("id = ?", task.ID).Update("next_poll_at", time.Now().Add(-time.Second)).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A fresh service has no in-memory state or downloaded bytes from generation.
|
||||
restarted := &Service{repo: s.repo, dataDir: s.dataDir, activeCancels: make(map[string]context.CancelFunc)}
|
||||
claimed, err := restarted.repo.ClaimNextTask("worker-b", time.Minute)
|
||||
if err != nil || claimed == nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
if err := restarted.taskWorker().processClaimedTask(claimed, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed, _ := s.repo.Task(task.ID)
|
||||
if completed.Status != model.TaskStatusSucceeded || completed.MediaStage != "completed" {
|
||||
t.Fatalf("not delivered: %s %s", completed.Status, completed.MediaStage)
|
||||
}
|
||||
if downloads.Load() != 2 {
|
||||
t.Fatalf("downloads=%d", downloads.Load())
|
||||
}
|
||||
var assets []model.Asset
|
||||
if err := db.Find(&assets, "user_id = ?", task.UserID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(assets) != 1 || !strings.HasPrefix(assets[0].ID, "generation_") {
|
||||
t.Fatalf("assets: %+v", assets)
|
||||
}
|
||||
if err := restarted.registerRecoveredMediaAssets(*completed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int64
|
||||
db.Model(&model.Asset{}).Where("user_id = ?", task.UserID).Count(&count)
|
||||
if count != 1 {
|
||||
t.Fatalf("duplicate assets: %d", count)
|
||||
}
|
||||
db.Model(&model.BillingOrder{}).Count(&count)
|
||||
if count != 0 {
|
||||
t.Fatal("recovery created a billing order")
|
||||
}
|
||||
files, err := os.ReadDir(filepath.Join(s.dataDir, "media-staging"))
|
||||
if err != nil || len(files) != 0 {
|
||||
t.Fatalf("staging not cleaned: %v %v", files, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryUploadFailureReusesStagedFileAndResource(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOWED_PRIVATE_UPSTREAM_HOSTS", "127.0.0.1")
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
seedOSSEnabled(t, db, "media-user", "http://127.0.0.1:1")
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
temp, err := s.stageInlineMedia(mediaTestPNG(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkpoint := &mediaCheckpoint{Mode: "image", StartedAt: time.Now(), Items: []mediaCheckpointItem{{TempName: temp, MIMEType: "image/png"}}}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "upload"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, failure := s.materializeTaskMedia(context.Background(), task, providerConfig{}, checkpoint)
|
||||
var delivery *mediaRecoveryError
|
||||
if !errors.As(failure, &delivery) || delivery.stage != "upload" {
|
||||
t.Fatalf("expected OSS failure, got %v", failure)
|
||||
}
|
||||
resource, err := s.resourceForUploadKey(task.UserID, mediaUploadKey(task.ID, 0))
|
||||
if err != nil || resource == nil || resource.Status == model.ResourceStatusReady || resource.Provider == "local" {
|
||||
t.Fatalf("silently degraded upload: %+v %v", resource, err)
|
||||
}
|
||||
if _, err := os.Stat(s.mediaTempPath(temp)); err != nil {
|
||||
t.Fatal("upload failure discarded download", err)
|
||||
}
|
||||
// Simulate fixing storage configuration on the same existing object identity.
|
||||
resource.Provider, resource.ObjectKey = "local", "recovered/picture.png"
|
||||
if err := s.repo.SaveResource(resource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output, err := s.resumeTaskMedia(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.finishTaskMediaRecovery(task, output, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ready, _ := s.resourceForUploadKey(task.UserID, mediaUploadKey(task.ID, 0))
|
||||
if ready.ID != resource.ID || ready.Status != model.ResourceStatusReady {
|
||||
t.Fatal("resource identity changed")
|
||||
}
|
||||
var count int64
|
||||
db.Model(&model.Resource{}).Where("user_id = ?", task.UserID).Count(&count)
|
||||
if count != 1 {
|
||||
t.Fatalf("duplicate resource count=%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryRegistrationRollbackAndRetryWithoutDownload(t *testing.T) {
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
temp, err := s.stageInlineMedia(mediaTestPNG(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkpoint := &mediaCheckpoint{Mode: "image", StartedAt: time.Now(), Items: []mediaCheckpointItem{{TempName: temp, MIMEType: "image/png"}}}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "local_save"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output, err := s.materializeTaskMedia(context.Background(), task, providerConfig{}, checkpoint)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, _ := json.Marshal(output)
|
||||
if err := db.Exec(`CREATE TRIGGER reject_media_asset BEFORE INSERT ON assets BEGIN SELECT RAISE(ABORT, 'injected asset failure'); END`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.saveTaskCompletionWithinStorageQuota(task, encoded, nil, false); err == nil {
|
||||
t.Fatal("registration failure was ignored")
|
||||
}
|
||||
saved, _ := s.repo.Task(task.ID)
|
||||
if saved.Status != model.TaskStatusRunning || saved.ResultJSON != "" {
|
||||
t.Fatal("false succeeded task escaped transaction")
|
||||
}
|
||||
if err := db.Exec(`DROP TRIGGER reject_media_asset`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(s.mediaTempPath(temp)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output, err = s.resumeTaskMedia(context.Background(), saved)
|
||||
if err != nil {
|
||||
t.Fatal("ready resource must not need staged file or URL", err)
|
||||
}
|
||||
if err := s.finishTaskMediaRecovery(saved, output, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryFencesLeasesCancellationAndManualRetries(t *testing.T) {
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
checkpoint := &mediaCheckpoint{Mode: "image", StartedAt: time.Now(), Items: []mediaCheckpointItem{{Reference: protocol.MediaReference{URL: "https://example.test/result"}}}}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "download"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale := *task
|
||||
stale.LeaseOwner = "obsolete-worker"
|
||||
if err := s.saveMediaCheckpoint(&stale, checkpoint, "upload"); !errors.Is(err, repository.ErrTaskStateConflict) {
|
||||
t.Fatalf("stale write: %v", err)
|
||||
}
|
||||
if err := db.Model(task).Updates(map[string]any{"status": model.TaskStatusFailed, "lease_owner": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.RecoverTaskMedia("other-user", task.ID); err == nil {
|
||||
t.Fatal("cross-user recovery")
|
||||
}
|
||||
queued, err := s.RecoverTaskMedia(task.UserID, task.ID)
|
||||
if err != nil || queued.ID != task.ID {
|
||||
t.Fatalf("manual recovery: %v", err)
|
||||
}
|
||||
duplicate, err := s.RecoverTaskMedia(task.UserID, task.ID)
|
||||
if err != nil || duplicate.ID != task.ID {
|
||||
t.Fatalf("duplicate click: %v", err)
|
||||
}
|
||||
if err := db.Model(task).Update("status", model.TaskStatusCancelled).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.RecoverTaskMedia(task.UserID, task.ID); err == nil {
|
||||
t.Fatal("cancelled task revived")
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "register"); !errors.Is(err, repository.ErrTaskStateConflict) {
|
||||
t.Fatalf("cancelled write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryDoesNotFailOverGenerationRoute(t *testing.T) {
|
||||
p := &taskRouteExecutionPortStub{nextAttempts: map[string]*model.RouteAttempt{"first": {RouteID: "second"}}, processResults: map[string]taskRouteExecutionStubResult{"first": {err: &mediaRecoveryError{stage: "download", retryable: true, cause: errors.New("connection reset")}}}}
|
||||
result, err := (&taskRouteExecutor{port: p}).execute(context.Background(), &model.Task{RouteID: "first"}, &model.RouteAttempt{RouteID: "first"})
|
||||
if err != nil || !result.providerSucceeded || len(p.processCalls) != 1 || p.finished[0] != "first:succeeded" {
|
||||
t.Fatalf("delivery entered model failover: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryTerminalStagesLimitsAndPublicList(t *testing.T) {
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
attempts int
|
||||
age time.Duration
|
||||
failure *mediaRecoveryError
|
||||
}{
|
||||
{"permanent", 0, 0, &mediaRecoveryError{stage: "upload", cause: providerHTTPError{StatusCode: 403}}},
|
||||
{"budget", 4, 0, &mediaRecoveryError{stage: "download", retryable: true, cause: errors.New("reset")}},
|
||||
{"expired", 0, time.Hour, &mediaRecoveryError{stage: "register", retryable: true, cause: errors.New("database unavailable")}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
checkpoint := &mediaCheckpoint{Mode: "image", StartedAt: time.Now().Add(-tc.age), Attempts: tc.attempts, Items: []mediaCheckpointItem{{Reference: protocol.MediaReference{URL: "https://example.test/private"}}}}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "download"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.handleMediaRecoveryFailure(task, tc.failure); err == nil {
|
||||
t.Fatal("terminal failure swallowed")
|
||||
}
|
||||
saved, err := s.repo.Task(task.ID)
|
||||
if err != nil || saved.Status != model.TaskStatusFailed || saved.MediaStage != tc.failure.stage {
|
||||
t.Fatalf("terminal stage: %+v %v", saved, err)
|
||||
}
|
||||
if !strings.Contains(saved.Error, "作品已生成") {
|
||||
t.Fatalf("wrong user message: %s", saved.Error)
|
||||
}
|
||||
listed, err := s.repo.Tasks(task.UserID, 100, "", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range taskSummariesForOutput(listed) {
|
||||
if item.ID == task.ID {
|
||||
found = item.CanRecoverMedia && item.MediaStage == tc.failure.stage
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("list lost recovery capability")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryManualGuardsDoNotChangeBilling(t *testing.T) {
|
||||
for _, name := range []string{"refunded", "wrong-owner", "limit", "cooldown", "deleted-project"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
checkpoint := &mediaCheckpoint{Mode: "image", StartedAt: time.Now(), Items: []mediaCheckpointItem{{Reference: protocol.MediaReference{URL: "https://example.test/result"}}}}
|
||||
order := model.BillingOrder{ID: newID(), TaskID: task.ID, UserID: task.UserID, IdempotencyKey: newID(), Status: model.BillingStatusUncertain, ReservedAmountMicrocredits: 100}
|
||||
switch name {
|
||||
case "refunded":
|
||||
order.Status = model.BillingStatusRefunded
|
||||
case "wrong-owner":
|
||||
order.UserID = "another-user"
|
||||
case "limit":
|
||||
checkpoint.ManualAttempts = 3
|
||||
case "cooldown":
|
||||
checkpoint.LastManualAt = time.Now()
|
||||
case "deleted-project":
|
||||
task.ProjectID = "deleted-project"
|
||||
}
|
||||
if err := db.Create(&order).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "upload"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Model(task).Updates(map[string]any{"status": model.TaskStatusFailed, "lease_owner": "", "lease_expires_at": nil, "billing_order_id": order.ID, "project_id": task.ProjectID}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.RecoverTaskMedia(task.UserID, task.ID); err == nil {
|
||||
t.Fatal("guard bypassed")
|
||||
}
|
||||
saved, _ := s.repo.Task(task.ID)
|
||||
if saved.Status != model.TaskStatusFailed {
|
||||
t.Fatal("guard changed task")
|
||||
}
|
||||
unchanged, err := s.repo.BillingOrder(order.ID)
|
||||
if err != nil || unchanged.Status != order.Status || unchanged.ReservedAmountMicrocredits != 100 {
|
||||
t.Fatalf("billing changed: %+v %v", unchanged, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryVideoDownloadDoesNotRewriteGenerationLog(t *testing.T) {
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
for _, item := range []model.ApiCallLog{
|
||||
{ID: "media-create", UserID: task.UserID, TaskID: task.ID, Capability: "video", RequestKind: "create", Method: "POST", Billable: true, Status: model.ApiCallStatusSucceeded, StatusCode: 200},
|
||||
{ID: "media-download", UserID: task.UserID, TaskID: task.ID, Capability: "video", RequestKind: "download", Method: "GET", Status: model.ApiCallStatusFailed, StatusCode: 503},
|
||||
} {
|
||||
if err := s.LogAPICall(item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var logs []model.ApiCallLog
|
||||
if err := db.Where("task_id = ?", task.ID).Order("id").Find(&logs).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(logs) != 2 || logs[0].Status != model.ApiCallStatusSucceeded || logs[1].Status != model.ApiCallStatusFailed || logs[1].Billable {
|
||||
t.Fatalf("merged delivery into generation: %+v", logs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRecoveryExpiredLeaseAndStagingLimits(t *testing.T) {
|
||||
s, db := newMediaRecoveryTestService(t)
|
||||
task := seedMediaTask(t, db, providerConfig{})
|
||||
checkpoint := &mediaCheckpoint{Mode: "image", StartedAt: time.Now(), Items: []mediaCheckpointItem{{}}}
|
||||
if err := db.Model(task).Update("lease_expires_at", time.Now().Add(-time.Second)).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.saveMediaCheckpoint(task, checkpoint, "download"); !errors.Is(err, repository.ErrTaskStateConflict) {
|
||||
t.Fatalf("expired lease wrote: %v", err)
|
||||
}
|
||||
for _, name := range []string{"../result-secret", "other", "", "result-dir/file"} {
|
||||
if s.mediaTempPath(name) != "" {
|
||||
t.Fatalf("unsafe staging name %s", name)
|
||||
}
|
||||
}
|
||||
if _, err := s.newMediaTemp(mediaStagingBudget + 1); err == nil {
|
||||
t.Fatal("oversized reservation accepted")
|
||||
}
|
||||
file, err := s.newMediaTemp(mediaStagingBudget)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
if _, err := s.newMediaTemp(1); err == nil {
|
||||
t.Fatal("staging budget exceeded")
|
||||
}
|
||||
}
|
||||
301
backend/internal/app/task_media_storage.go
Normal file
301
backend/internal/app/task_media_storage.go
Normal file
@@ -0,0 +1,301 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
const mediaStagingBudget int64 = 512 << 20
|
||||
const mediaStagingTTL = 24 * time.Hour
|
||||
|
||||
var mediaStagingMu sync.Mutex
|
||||
|
||||
// Reserve the maximum response size before opening a file. Temporary files are
|
||||
// sparse reservations, so concurrent downloads cannot oversubscribe this budget.
|
||||
func (s *Service) newMediaTemp(limit int64) (*os.File, error) {
|
||||
mediaStagingMu.Lock()
|
||||
defer mediaStagingMu.Unlock()
|
||||
dir := filepath.Join(s.dataDir, "media-staging")
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var used int64
|
||||
for _, entry := range entries {
|
||||
if !strings.HasPrefix(entry.Name(), "result-") || !entry.Type().IsRegular() {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if time.Since(info.ModTime()) > mediaStagingTTL {
|
||||
if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
used += info.Size()
|
||||
}
|
||||
if limit <= 0 || limit > mediaStagingBudget || used > mediaStagingBudget-limit {
|
||||
return nil, errors.New("作品暂存空间不足,请稍后重试保存")
|
||||
}
|
||||
file, err := os.CreateTemp(dir, "result-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := file.Truncate(limit); err != nil {
|
||||
file.Close()
|
||||
os.Remove(file.Name())
|
||||
return nil, err
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (s *Service) stageInlineMedia(data []byte) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", errors.New("上游返回空作品")
|
||||
}
|
||||
file, err := s.newMediaTemp(int64(len(data)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = file.Write(data)
|
||||
if err == nil {
|
||||
err = file.Sync()
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(file.Name())
|
||||
return "", err
|
||||
}
|
||||
return filepath.Base(file.Name()), nil
|
||||
}
|
||||
|
||||
func (s *Service) downloadTaskMedia(ctx context.Context, config providerConfig, url, mode string) (name, mimeType string, err error) {
|
||||
policy, err := s.RuntimePolicy()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
limit := min(megabytes(policy.Resource.GeneratedFileMB), mediaStagingBudget)
|
||||
downloadCtx, cancel := context.WithTimeout(withProviderRequestKind(ctx, "download"), 3*time.Minute)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
started := time.Now()
|
||||
statusCode := 0
|
||||
defer func() {
|
||||
// Signed paths and net/url errors can contain bearer credentials. Keep
|
||||
// the real error for retry classification, never in request logs.
|
||||
logged := req.Clone(req.Context())
|
||||
redactedURL := *req.URL
|
||||
redactedURL.Path, redactedURL.RawPath, redactedURL.RawQuery, redactedURL.Fragment = "/task-media", "", "", ""
|
||||
redactedURL.User = nil
|
||||
logged.URL = &redactedURL
|
||||
var safeErr error
|
||||
if err != nil {
|
||||
safeErr = errors.New("结果下载失败")
|
||||
}
|
||||
recordProviderRequest(logged, started, statusCode, nil, safeErr)
|
||||
}()
|
||||
if _, err = ValidateOutboundURL(url); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if sameProviderOrigin(config.BaseURL, url) {
|
||||
applyProviderAuth(req, config)
|
||||
ApplyOutboundHeaders(req, config.Headers)
|
||||
}
|
||||
ApplyDefaultOutboundHeaders(req)
|
||||
response, err := OutboundHTTPClient(3 * time.Minute).Do(req)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
statusCode = response.StatusCode
|
||||
if statusCode < 200 || statusCode >= 300 {
|
||||
return "", "", providerHTTPError{StatusCode: statusCode, Status: response.Status, RetryAfter: parseRetryAfter(response.Header.Get("Retry-After"), time.Now())}
|
||||
}
|
||||
if response.ContentLength > limit {
|
||||
return "", "", errors.New("生成文件超过大小限制")
|
||||
}
|
||||
if response.ContentLength > 0 {
|
||||
limit = response.ContentLength
|
||||
}
|
||||
file, err := s.newMediaTemp(limit)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
if err != nil {
|
||||
_ = os.Remove(file.Name())
|
||||
}
|
||||
}()
|
||||
size, err := io.Copy(file, io.LimitReader(response.Body, limit+1))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if size == 0 || size > limit {
|
||||
return "", "", errors.New("生成文件为空或超过大小限制")
|
||||
}
|
||||
if response.ContentLength >= 0 && size != response.ContentLength {
|
||||
return "", "", io.ErrUnexpectedEOF
|
||||
}
|
||||
if err = file.Truncate(size); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if _, err = file.Seek(0, io.SeekStart); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
head := make([]byte, min(size, 512))
|
||||
if _, err = io.ReadFull(file, head); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
mimeType = normalizedMediaMimeType(response.Header.Get("Content-Type"), head)
|
||||
if mode == "audio" && mimeType == "application/ogg" {
|
||||
mimeType = "audio/ogg"
|
||||
}
|
||||
if !strings.HasPrefix(mimeType, mode+"/") {
|
||||
return "", "", errors.New("上游返回的内容不是有效的" + mode + "媒体")
|
||||
}
|
||||
if err = file.Sync(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = file.Close(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return filepath.Base(file.Name()), mimeType, nil
|
||||
}
|
||||
|
||||
func mediaUploadKey(taskID string, index int) *string {
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("task-media:%s:%d", taskID, index)))
|
||||
key := hex.EncodeToString(sum[:])
|
||||
return &key
|
||||
}
|
||||
|
||||
func (s *Service) storeTaskMediaFile(task *model.Task, index int, path, mimeType, kind string, existing *model.Resource) (*model.Resource, error) {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, errors.New("暂存作品不是普通文件")
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err = file.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, errors.New("暂存作品不是普通文件")
|
||||
}
|
||||
width, height := 0, 0
|
||||
if kind == "image" {
|
||||
config, _, err := image.DecodeConfig(file)
|
||||
if err != nil {
|
||||
// The standard library does not decode WebP/AVIF. Verify their
|
||||
// container signatures instead of trusting a response MIME header.
|
||||
head := make([]byte, 64)
|
||||
n, _ := file.ReadAt(head, 0)
|
||||
head = head[:n]
|
||||
webp := mimeType == "image/webp" && n >= 12 && string(head[:4]) == "RIFF" && string(head[8:12]) == "WEBP"
|
||||
avif := mimeType == "image/avif" && n >= 16 && string(head[4:8]) == "ftyp" && (strings.Contains(string(head[8:]), "avif") || strings.Contains(string(head[8:]), "avis"))
|
||||
if !webp && !avif {
|
||||
return nil, fmt.Errorf("图片内容校验失败:%w", err)
|
||||
}
|
||||
}
|
||||
width, height = config.Width, config.Height
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
day, err := s.reserveGeneratedResourceQuota(task.UserID, info.Size())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
s.releaseUserUploadQuota(task.UserID, day, info.Size())
|
||||
}
|
||||
}()
|
||||
var resource *model.Resource
|
||||
if existing == nil {
|
||||
resource, _, err = s.storeResourceWithWriter(task.UserID, kind, "generated."+extensionFromMimeType(mimeType), mimeType, info.Size(), width, height, 0, file, mediaUploadKey(task.ID, index), false, s.storeTaskMediaObject)
|
||||
} else {
|
||||
// Reuse the same object key after a failed upload; no orphan per retry.
|
||||
resource = existing
|
||||
if resource.UserID != task.UserID || resource.Size != info.Size() || resource.MimeType != mimeType {
|
||||
return nil, errors.New("恢复文件与原始资源不一致")
|
||||
}
|
||||
resource.ETag, err = s.storeTaskMediaObject(resource, "generated."+extensionFromMimeType(mimeType), file)
|
||||
if err == nil {
|
||||
resource.Status, resource.Error, resource.UpdatedAt = model.ResourceStatusReady, "", time.Now()
|
||||
err = s.repo.SaveResource(resource)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.commitUserUploadQuota(task.UserID, info.Size())
|
||||
committed = true
|
||||
if existing != nil {
|
||||
s.recordActivity(task.UserID, "resource", 1)
|
||||
s.maybeStartPlaybackTranscode(resource)
|
||||
}
|
||||
return resource, nil
|
||||
}
|
||||
|
||||
// Unlike ordinary uploads, generated output preserves an OSS failure for
|
||||
// explicit recovery instead of silently changing the storage destination.
|
||||
func (s *Service) storeTaskMediaObject(resource *model.Resource, _ string, body io.Reader) (string, error) {
|
||||
if resource.Provider == "local" {
|
||||
return "", writeLocalResourceObject(filepath.Join(s.dataDir, "resources", filepath.FromSlash(resource.ObjectKey)), body)
|
||||
}
|
||||
setting, err := s.ossSettingForResource(resource.UserID, resource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return putOSSObject(setting, resource.ObjectKey, resource.MimeType, resource.Size, body)
|
||||
}
|
||||
|
||||
// Cleanup is best effort only after durable completion. Expired leftovers are
|
||||
// also collected on staging allocation, never by deleting resource objects.
|
||||
func (s *Service) cleanupMediaCheckpointFiles(task *model.Task) {
|
||||
checkpoint, err := s.decodeMediaCheckpoint(task)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, item := range checkpoint.Items {
|
||||
if path := s.mediaTempPath(item.TempName); path != "" {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
_ = s.log(task.UserID, task.ID, "warn", "清理作品暂存文件失败", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ type TaskSummary struct {
|
||||
Type string `json:"type"`
|
||||
Status model.TaskStatus `json:"status"`
|
||||
Stage string `json:"stage"`
|
||||
MediaStage string `json:"mediaStage,omitempty"`
|
||||
CanRecoverMedia bool `json:"canRecoverMedia,omitempty"`
|
||||
Progress int `json:"progress"`
|
||||
Prompt string `json:"prompt"`
|
||||
Operation string `json:"operation,omitempty"`
|
||||
@@ -107,6 +109,8 @@ func taskSummaryForOutput(task model.Task) TaskSummary {
|
||||
Type: task.Type,
|
||||
Status: task.Status,
|
||||
Stage: task.Stage,
|
||||
MediaStage: task.MediaStage,
|
||||
CanRecoverMedia: task.MediaRecoveryJSON != "" && task.Status == model.TaskStatusFailed,
|
||||
Progress: task.Progress,
|
||||
Prompt: truncateRunes(task.Prompt, 500),
|
||||
Operation: task.Operation,
|
||||
@@ -285,6 +289,7 @@ func truncateRunes(value string, limit int) string {
|
||||
}
|
||||
|
||||
func taskForOutput(task model.Task) *model.Task {
|
||||
task.CanRecoverMedia = task.MediaRecoveryJSON != "" && task.Status == model.TaskStatusFailed
|
||||
task.Diagnostic = taskExecutionDiagnostic(&task)
|
||||
task.InputJSON = publicTaskInputJSON(task.InputJSON)
|
||||
// 普通任务接口只暴露前台模型身份;渠道模型和供应线路属于管理员内部信息。
|
||||
|
||||
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
@@ -51,6 +52,13 @@ func (e *taskRouteExecutor) execute(ctx context.Context, task *model.Task, attem
|
||||
if stateErr := e.port.refreshTaskProviderState(task); stateErr != nil {
|
||||
return taskRouteExecutionResult{}, stateErr
|
||||
}
|
||||
var deliveryFailure *mediaRecoveryError
|
||||
if errors.As(execution.err, &deliveryFailure) {
|
||||
// The provider succeeded. Delivery must never enter route failover.
|
||||
e.port.finishTaskRouteAttempt(attempt, task, nil)
|
||||
execution.providerSucceeded = true
|
||||
return execution, nil
|
||||
}
|
||||
e.port.finishTaskRouteAttempt(attempt, task, execution.err)
|
||||
if execution.err == nil {
|
||||
break
|
||||
|
||||
@@ -218,7 +218,7 @@ func (c *taskTerminalCoordinator) handleSuccess(task *model.Task) error {
|
||||
if fetchErr != nil {
|
||||
completionErr = fmt.Errorf("任务成功后读取任务产物失败:%w", fetchErr)
|
||||
_ = c.logger.log(task.UserID, task.ID, "error", "任务成功但读取任务产物失败", fetchErr.Error())
|
||||
} else {
|
||||
} else if completedTask.MediaRecoveryJSON == "" {
|
||||
if registerErr := c.outputs.RegisterTaskOutputFromTask(*completedTask); registerErr != nil {
|
||||
// 任务成功与产物登记分开记账;登记失败保持步骤异常,允许项目页幂等补登记。
|
||||
_ = c.logger.log(task.UserID, task.ID, "error", "任务成功但项目产物登记失败", registerErr.Error())
|
||||
|
||||
@@ -185,6 +185,10 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task, globalSlot
|
||||
if task.Type == model.TaskTypeTimelineRender {
|
||||
return w.processTimelineRender(task, ctx)
|
||||
}
|
||||
if task.MediaRecoveryJSON != "" {
|
||||
result, recoveryErr := s.resumeTaskMedia(ctx, task)
|
||||
return s.finishTaskMediaRecovery(task, result, recoveryErr)
|
||||
}
|
||||
|
||||
s.markAgentMemoryCompactRunning(*task)
|
||||
task.Stage = "调用生成模型"
|
||||
@@ -215,6 +219,15 @@ func (w *taskWorkerCoordinator) processClaimedTask(task *model.Task, globalSlot
|
||||
default:
|
||||
}
|
||||
result, canvasOps, err := routeResult.result, routeResult.canvasOps, routeResult.err
|
||||
latestMedia, readErr := s.repo.Task(task.ID)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
task.MediaRecoveryJSON, task.MediaStage = latestMedia.MediaRecoveryJSON, latestMedia.MediaStage
|
||||
var deliveryFailure *mediaRecoveryError
|
||||
if task.MediaRecoveryJSON != "" || errors.As(err, &deliveryFailure) {
|
||||
return s.finishTaskMediaRecovery(task, result, err)
|
||||
}
|
||||
providerSucceeded := routeResult.providerSucceeded
|
||||
if err == nil {
|
||||
result, err = s.persistGeneratedMediaResult(task.UserID, result)
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const CurrentSchemaVersion int64 = 33
|
||||
const CurrentSchemaVersion int64 = 34
|
||||
|
||||
const baselineSchemaChecksum = "sha256:open-ai-canvas-schema-v1-20260830"
|
||||
const schemaMigrationAppliedAtIndexChecksum = "sha256:schema-migrations-applied-at-index-v2-20260830"
|
||||
@@ -107,6 +107,16 @@ var schemaMigrations = []migration{
|
||||
}},
|
||||
{version: 32, name: "channel_model_tags", checksum: "sha256:channel-model-tags-v32", apply: migrateChannelModelTags},
|
||||
{version: 33, name: "oauth_state_accepted_terms", checksum: "sha256:oauth-state-accepted-terms-v33", apply: migrateOAuthStateAcceptedTerms},
|
||||
{version: 34, name: "task_media_recovery", checksum: "sha256:task-media-recovery-v34", apply: func(tx *gorm.DB) error {
|
||||
for _, field := range []string{"MediaRecoveryJSON", "MediaStage"} {
|
||||
if !tx.Migrator().HasColumn(&model.Task{}, field) {
|
||||
if err := tx.Migrator().AddColumn(&model.Task{}, field); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
}
|
||||
|
||||
func migrateChannelModelTags(tx *gorm.DB) error {
|
||||
|
||||
38
backend/internal/database/task_media_recovery_test.go
Normal file
38
backend/internal/database/task_media_recovery_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTaskMediaRecoveryMigrationPreservesHistoricalTasks(t *testing.T) {
|
||||
db, err := Open(Config{Driver: "sqlite", DSN: ":memory:"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
if err := db.Exec(`CREATE TABLE tasks (id TEXT PRIMARY KEY, status TEXT, error TEXT)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec(`INSERT INTO tasks VALUES ('old-task', 'failed', 'download failed')`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := schemaMigrations[33].apply(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var task model.Task
|
||||
if err := db.First(&task, "id = ?", "old-task").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.Status != model.TaskStatusFailed || task.Error != "download failed" || task.MediaRecoveryJSON != "" || task.MediaStage != "" {
|
||||
t.Fatalf("migration rewrote historical task: %+v", task)
|
||||
}
|
||||
for _, field := range []string{"media_stage", "media_recovery_json"} {
|
||||
if !db.Migrator().HasColumn(&model.Task{}, field) {
|
||||
t.Fatalf("missing %s", field)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,10 +17,11 @@ func TestRegisterCanvasAPIExposesOpenAPIAndProjects(t *testing.T) {
|
||||
RegisterCanvasAPI(router.Group("/api"), &service.Service{})
|
||||
|
||||
wanted := map[string]bool{
|
||||
"GET /api/openapi.yaml": false,
|
||||
"GET /api/projects": false,
|
||||
"POST /api/tasks": false,
|
||||
"GET /api/resources": false,
|
||||
"GET /api/openapi.yaml": false,
|
||||
"GET /api/projects": false,
|
||||
"POST /api/tasks": false,
|
||||
"POST /api/tasks/:id/recover-media": false,
|
||||
"GET /api/resources": false,
|
||||
}
|
||||
for _, route := range router.Routes() {
|
||||
key := route.Method + " " + route.Path
|
||||
|
||||
@@ -1067,6 +1067,25 @@ paths:
|
||||
$ref: "#/components/responses/Envelope"
|
||||
"400":
|
||||
$ref: "#/components/responses/Error"
|
||||
/tasks/{id}/recover-media:
|
||||
post:
|
||||
tags: [tasks]
|
||||
summary: 按原任务 ID 重试作品保存,不重新生成或创建计费订单
|
||||
description: 仅当前用户有恢复记录的保存失败任务可重入队;已在队列或运行中时幂等返回。每任务最多手动恢复三次,间隔至少一分钟。已取消、已退款或归属项目不可用时拒绝。
|
||||
parameters:
|
||||
- in: path
|
||||
name: id
|
||||
required: true
|
||||
schema: { type: string }
|
||||
responses:
|
||||
"200":
|
||||
$ref: "#/components/responses/Envelope"
|
||||
"400":
|
||||
$ref: "#/components/responses/Error"
|
||||
"401":
|
||||
$ref: "#/components/responses/Error"
|
||||
"404":
|
||||
$ref: "#/components/responses/Error"
|
||||
/tasks/{id}/text-events:
|
||||
get:
|
||||
tags: [tasks]
|
||||
|
||||
@@ -202,6 +202,23 @@ func RegisterTaskRoutes(r *gin.RouterGroup, svc *service.Service) {
|
||||
}
|
||||
ok(c, task)
|
||||
})
|
||||
r.POST("/tasks/:id/recover-media", func(c *gin.Context) {
|
||||
user, err := currentUser(c, svc)
|
||||
if err != nil {
|
||||
failService(c, err)
|
||||
return
|
||||
}
|
||||
policy, available := loadRuntimePolicy(c, svc)
|
||||
if !available || !enforceRateLimit(c, "task-media-recovery:"+user.ID, policy.Request.TaskCreatePerMinute, time.Minute) {
|
||||
return
|
||||
}
|
||||
task, err := svc.RecoverTaskMedia(user.ID, c.Param("id"))
|
||||
if err != nil {
|
||||
failService(c, err)
|
||||
return
|
||||
}
|
||||
ok(c, task)
|
||||
})
|
||||
r.POST("/tasks/:id/query-provider", func(c *gin.Context) {
|
||||
user, err := currentUser(c, svc)
|
||||
if err != nil {
|
||||
|
||||
64
backend/internal/handler/task_media_recovery_test.go
Normal file
64
backend/internal/handler/task_media_recovery_test.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"infinite-canvas/backend/internal/auth"
|
||||
"infinite-canvas/backend/internal/database"
|
||||
"infinite-canvas/backend/internal/model"
|
||||
"infinite-canvas/backend/internal/repository"
|
||||
"infinite-canvas/backend/internal/service"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTaskMediaRecoveryHTTPAuthenticationAndOwnership(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := database.Open(database.Config{Driver: "sqlite", DSN: ":memory:"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
if err := database.MigrateSchema(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, item := range []any{
|
||||
&model.User{ID: "media-user", Username: "media-user", Email: "media@example.invalid", Role: model.UserRoleUser, Status: model.UserStatusActive},
|
||||
&model.AuthSession{ID: "media-session", UserID: "media-user", TokenHash: auth.HashToken("test-token"), ExpiresAt: time.Now().Add(time.Hour)},
|
||||
&model.Task{ID: "legacy-task", UserID: "media-user", Status: model.TaskStatusFailed},
|
||||
&model.Task{ID: "foreign-task", UserID: "other-user", Status: model.TaskStatusFailed},
|
||||
} {
|
||||
if err := db.Create(item).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
router := gin.New()
|
||||
svc := service.New(repository.New(db), t.TempDir())
|
||||
previous := runtimeService
|
||||
ConfigureRuntime(svc)
|
||||
t.Cleanup(func() { ConfigureRuntime(previous) })
|
||||
RegisterTaskRoutes(router.Group("/api"), svc)
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
auth bool
|
||||
want int
|
||||
}{
|
||||
{"legacy-task", false, http.StatusUnauthorized},
|
||||
{"legacy-task", true, http.StatusBadRequest},
|
||||
{"foreign-task", true, http.StatusNotFound},
|
||||
{"missing-task", true, http.StatusNotFound},
|
||||
} {
|
||||
r := httptest.NewRequest(http.MethodPost, "/api/tasks/"+tc.id+"/recover-media", nil)
|
||||
if tc.auth {
|
||||
r.AddCookie(&http.Cookie{Name: service.SessionCookieName, Value: "media-session.test-token"})
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, r)
|
||||
if w.Code != tc.want {
|
||||
t.Fatalf("%s auth=%v: %d %s", tc.id, tc.auth, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -105,6 +105,7 @@ type ApiCallLog struct {
|
||||
ChannelName string `json:"channelName" gorm:"-"`
|
||||
TaskID string `json:"taskId,omitempty" gorm:"index;size:36"`
|
||||
TaskStatus TaskStatus `json:"taskStatus,omitempty" gorm:"-"`
|
||||
MediaStage string `json:"mediaStage,omitempty" gorm:"-"`
|
||||
BillingOrderID string `json:"billingOrderId,omitempty" gorm:"index;size:36"`
|
||||
BillingStatus BillingStatus `json:"billingStatus,omitempty" gorm:"-"`
|
||||
BillingAmount int64 `json:"billingAmountMicrocredits" gorm:"-"`
|
||||
|
||||
@@ -46,13 +46,17 @@ type Task struct {
|
||||
LeaseExpiresAt *time.Time `json:"-" gorm:"index;index:idx_tasks_claim,priority:2"`
|
||||
InputJSON string `json:"inputJson" gorm:"type:text"`
|
||||
ResultJSON string `json:"resultJson" gorm:"type:text"`
|
||||
TextDraft string `json:"textDraft,omitempty" gorm:"type:text"`
|
||||
Error string `json:"error"`
|
||||
Attempts int `json:"attempts"`
|
||||
StartedAt *time.Time `json:"startedAt"`
|
||||
CompletedAt *time.Time `json:"completedAt"`
|
||||
CreatedAt time.Time `json:"createdAt" gorm:"index:idx_tasks_user_created,priority:2;index:idx_tasks_status_created,priority:2;index:idx_tasks_claim,priority:3;index:idx_tasks_user_project_created,priority:3"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
// MediaRecoveryJSON is an encrypted server-only checkpoint, never a public result.
|
||||
MediaRecoveryJSON string `json:"-" gorm:"type:text"`
|
||||
MediaStage string `json:"mediaStage,omitempty" gorm:"size:24"`
|
||||
CanRecoverMedia bool `json:"canRecoverMedia,omitempty" gorm:"-"`
|
||||
TextDraft string `json:"textDraft,omitempty" gorm:"type:text"`
|
||||
Error string `json:"error"`
|
||||
Attempts int `json:"attempts"`
|
||||
StartedAt *time.Time `json:"startedAt"`
|
||||
CompletedAt *time.Time `json:"completedAt"`
|
||||
CreatedAt time.Time `json:"createdAt" gorm:"index:idx_tasks_user_created,priority:2;index:idx_tasks_status_created,priority:2;index:idx_tasks_claim,priority:3;index:idx_tasks_user_project_created,priority:3"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// TaskTextDelta 只保存可回放窗口内的文本增量;最终正文和失败草稿分别归并到 Task.ResultJSON 与 Task.TextDraft。
|
||||
|
||||
@@ -144,7 +144,7 @@ func (r *Repository) filteredAPICallLogQuery(filter APICallLogFilter) *gorm.DB {
|
||||
query = query.Where("api_call_logs.request_kind = ?", "download")
|
||||
case "all":
|
||||
default:
|
||||
query = visibleAPICallLogQuery(query).Where("COALESCE(api_call_logs.request_kind, '') <> ?", "download")
|
||||
query = visibleAPICallLogQuery(query).Where("COALESCE(api_call_logs.request_kind, '') NOT IN ?", []string{"download", "upload", "local_save", "register"})
|
||||
}
|
||||
if value := strings.TrimSpace(filter.Keyword); value != "" {
|
||||
pattern := "%" + strings.ToLower(value) + "%"
|
||||
@@ -167,7 +167,7 @@ func (r *Repository) APICallLogTasks(ids []string) ([]model.Task, error) {
|
||||
return []model.Task{}, nil
|
||||
}
|
||||
var tasks []model.Task
|
||||
err := r.db.Select("id", "user_id", "type", "status", "result_json").Where("id IN ?", ids).Find(&tasks).Error
|
||||
err := r.db.Select("id", "user_id", "type", "status", "result_json", "media_stage").Where("id IN ?", ids).Find(&tasks).Error
|
||||
return tasks, err
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,9 @@ func TestAPICallLogRecordTypeFiltersListAndExport(t *testing.T) {
|
||||
{ID: "poll", Capability: "video", RequestKind: "poll", CreatedAt: now},
|
||||
{ID: "video-download", Capability: "video", RequestKind: "download", CreatedAt: now},
|
||||
{ID: "image-download", Capability: "image", RequestKind: "download", CreatedAt: now},
|
||||
{ID: "upload", Capability: "image", RequestKind: "upload", CreatedAt: now},
|
||||
{ID: "local-save", Capability: "image", RequestKind: "local_save", CreatedAt: now},
|
||||
{ID: "register", Capability: "image", RequestKind: "register", CreatedAt: now},
|
||||
} {
|
||||
if err := db.Create(&item).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -41,7 +44,7 @@ func TestAPICallLogRecordTypeFiltersListAndExport(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
kind string
|
||||
count int
|
||||
}{{"", 1}, {"request", 1}, {"download", 2}, {"all", 4}} {
|
||||
}{{"", 1}, {"request", 1}, {"download", 2}, {"all", 7}} {
|
||||
filter := APICallLogFilter{AnalyticsFilter: AnalyticsFilter{From: now.Add(-time.Hour), To: now.Add(time.Hour)}, RecordType: tc.kind}
|
||||
logs, total, err := repo.QueryAPICallLogs(filter)
|
||||
if err != nil {
|
||||
|
||||
@@ -110,7 +110,7 @@ func (r *Repository) UserStorageUsage(userID string) (UserStorageUsage, error) {
|
||||
(SELECT COUNT(*) FROM canvas_projects WHERE user_id = ?) AS canvas_count,
|
||||
(SELECT COALESCE(SUM(length(CAST(COALESCE(payload_json, '') AS BLOB))), 0) FROM canvas_projects WHERE user_id = ?) AS canvas_bytes,
|
||||
(SELECT COUNT(*) FROM tasks WHERE user_id = ?) AS task_count,
|
||||
(SELECT COALESCE(SUM(length(CAST(COALESCE(prompt, '') AS BLOB)) + length(CAST(COALESCE(input_json, '') AS BLOB)) + length(CAST(COALESCE(result_json, '') AS BLOB)) + length(CAST(COALESCE(text_draft, '') AS BLOB)) + length(CAST(COALESCE(error, '') AS BLOB))), 0) FROM tasks WHERE user_id = ?)
|
||||
(SELECT COALESCE(SUM(length(CAST(COALESCE(prompt, '') AS BLOB)) + length(CAST(COALESCE(input_json, '') AS BLOB)) + length(CAST(COALESCE(result_json, '') AS BLOB)) + length(CAST(COALESCE(media_recovery_json, '') AS BLOB)) + length(CAST(COALESCE(text_draft, '') AS BLOB)) + length(CAST(COALESCE(error, '') AS BLOB))), 0) FROM tasks WHERE user_id = ?)
|
||||
+ (SELECT COALESCE(SUM(length(CAST(COALESCE(message, '') AS BLOB)) + length(CAST(COALESCE(payload, '') AS BLOB))), 0) FROM task_logs WHERE user_id = ?)
|
||||
+ (SELECT COALESCE(SUM(length(CAST(COALESCE(url, '') AS BLOB)) + length(CAST(COALESCE(payload, '') AS BLOB))), 0) FROM results WHERE user_id = ?)
|
||||
+ (SELECT COALESCE(SUM(byte_count), 0) FROM task_text_delta WHERE user_id = ?)
|
||||
@@ -536,6 +536,12 @@ func (r *Repository) UpdateTaskProviderProgress(id string, progress int) error {
|
||||
}
|
||||
|
||||
func (r *Repository) SaveTaskCompletion(task *model.Task, expected model.TaskStatus, results []model.Result) error {
|
||||
return r.SaveTaskCompletionWithRegistration(task, expected, results, nil)
|
||||
}
|
||||
|
||||
// Registration and terminal success commit together; a failed asset write leaves
|
||||
// the task recoverable and never exposes a false success to the canvas.
|
||||
func (r *Repository) SaveTaskCompletionWithRegistration(task *model.Task, expected model.TaskStatus, results []model.Result, register func(*Repository) error) error {
|
||||
return r.db.Transaction(func(tx *gorm.DB) error {
|
||||
updated := taskLeaseWriter(tx.Model(&model.Task{}), task.LeaseOwner).
|
||||
Where("id = ? AND status = ?", task.ID, expected).
|
||||
@@ -551,6 +557,9 @@ func (r *Repository) SaveTaskCompletion(task *model.Task, expected model.TaskSta
|
||||
return err
|
||||
}
|
||||
}
|
||||
if register != nil {
|
||||
return register(New(tx))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -696,7 +705,7 @@ func (r *Repository) Tasks(userID string, limit int, projectID string, activeOnl
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 50
|
||||
}
|
||||
query := r.db.Select("id", "project_id", "type", "status", "stage", "progress", "prompt", "operation", "provider", "model", "input_json", "result_json", "billing_order_id", "provider_request_id", "provider_cancel_status", "provider_cancel_error", "provider_cancel_attempts", "provider_cancel_requested_at", "provider_cancelled_at", "provider_cancel_next_check_at", "attempts", "started_at", "completed_at", "created_at", "updated_at", "agent_run_id", "generation_id", "approval_id", "authorized_charge_microcredits", "execution_diagnostic_json", "cancellation_source", "cancellation_actor_id", "cancellation_requested_at").
|
||||
query := r.db.Select("id", "project_id", "type", "status", "stage", "media_stage", "media_recovery_json", "progress", "prompt", "operation", "provider", "model", "input_json", "result_json", "billing_order_id", "provider_request_id", "provider_cancel_status", "provider_cancel_error", "provider_cancel_attempts", "provider_cancel_requested_at", "provider_cancelled_at", "provider_cancel_next_check_at", "attempts", "started_at", "completed_at", "created_at", "updated_at", "agent_run_id", "generation_id", "approval_id", "authorized_charge_microcredits", "execution_diagnostic_json", "cancellation_source", "cancellation_actor_id", "cancellation_requested_at").
|
||||
Where("user_id = ?", userID)
|
||||
if strings.TrimSpace(projectID) != "" {
|
||||
query = query.Where("project_id = ?", strings.TrimSpace(projectID))
|
||||
|
||||
37
backend/internal/repository/task_media_recovery.go
Normal file
37
backend/internal/repository/task_media_recovery.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
)
|
||||
|
||||
// The existing task lease fences every checkpoint write, including after cancellation.
|
||||
func (r *Repository) SaveTaskMediaCheckpoint(task *model.Task, encrypted, stage string) error {
|
||||
result := taskLeaseWriter(r.db.Model(&model.Task{}), task.LeaseOwner).
|
||||
Where("id = ? AND user_id = ? AND status = ? AND route_run = ?", task.ID, task.UserID, model.TaskStatusRunning, task.RouteRun).
|
||||
Updates(map[string]any{"media_recovery_json": encrypted, "media_stage": stage, "stage": "作品已生成,正在保存", "updated_at": time.Now()})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return ErrTaskStateConflict
|
||||
}
|
||||
task.MediaRecoveryJSON, task.MediaStage = encrypted, stage
|
||||
return nil
|
||||
}
|
||||
|
||||
// Manual recovery requeues the same task and order, never a new generation.
|
||||
func (r *Repository) RequeueTaskMediaRecovery(task *model.Task, encrypted string) error {
|
||||
now := time.Now()
|
||||
result := r.db.Model(&model.Task{}).
|
||||
Where("id = ? AND user_id = ? AND status = ? AND media_recovery_json = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?)", task.ID, task.UserID, model.TaskStatusFailed, task.MediaRecoveryJSON, now).
|
||||
Updates(map[string]any{"status": model.TaskStatusQueued, "stage": "等待恢复作品保存", "error": "", "completed_at": nil, "next_poll_at": nil, "lease_owner": "", "lease_expires_at": nil, "media_recovery_json": encrypted, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return ErrTaskStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user