fix(platform): 渠道并发 - 支持后台热配置并等待空闲槽位 (#34)

Co-authored-by: ddCat <ddcat666@88.com>
This commit is contained in:
ddcat666
2026-07-23 18:35:51 +08:00
committed by GitHub
parent f26f9983bc
commit 93316dd3d7
25 changed files with 703 additions and 112 deletions

View File

@@ -439,6 +439,37 @@ func RegisterAdminRoutes(r *gin.RouterGroup, svc *service.Service) {
}
ok(c, gin.H{"setting": setting})
})
r.GET("/admin/settings/concurrency", func(c *gin.Context) {
user, err := currentUser(c, svc)
if err != nil {
failService(c, err)
return
}
setting, err := svc.AdminRuntimeConcurrencySetting(user)
if err != nil {
failService(c, err)
return
}
ok(c, gin.H{"setting": setting})
})
r.PATCH("/admin/settings/concurrency", func(c *gin.Context) {
user, err := currentUser(c, svc)
if err != nil {
failService(c, err)
return
}
var req service.RuntimeConcurrencySettingRequest
if err := c.ShouldBindJSON(&req); err != nil {
fail(c, http.StatusBadRequest, err)
return
}
setting, err := svc.UpdateRuntimeConcurrencySetting(user, req)
if err != nil {
failService(c, err)
return
}
ok(c, gin.H{"setting": setting})
})
r.GET("/admin/api-logs", func(c *gin.Context) {
user, err := currentUser(c, svc)
if err != nil {
@@ -524,12 +555,35 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
}
modelName := proxyRequestModel(c.GetHeader("Content-Type"), body)
billingOrderID := ""
query := c.Request.URL.Query()
for _, key := range []string{"key", "api_key", "access_token", "token"} {
query.Del(key)
}
target := strings.TrimRight(channel.BaseURL, "/") + path
if encodedQuery := query.Encode(); encodedQuery != "" {
target += "?" + encodedQuery
}
if _, err := service.ValidateOutboundURL(target); err != nil {
_ = svc.RefundBilling(billingOrderID, "系统渠道地址校验失败")
failService(c, err)
return
}
// 同步代理与后台任务必须共享渠道槽位,否则两条入口会共同超过供应商并发上限。
releaseChannel, concurrencyLimit, err := svc.AcquireChannelSlot(c.Request.Context(), channel.ID, "", 36*time.Minute)
if err != nil {
log := apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, model.ApiCallStatusFailed, 0, time.Since(startedAt), err.Error(), concurrencyLimit)
log.ErrorCode, log.Error = service.ChannelSlotFailureDetails(err)
logSystemProxyCall(svc, log, nil)
fail(c, http.StatusServiceUnavailable, err)
return
}
defer releaseChannel()
if c.Request.Method == http.MethodPost {
capability := "text"
switch channel.InterfaceType {
case model.ChannelInterfaceOpenAIImage:
capability = "image"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2:
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
capability = "video"
}
order, err := svc.ReserveProxyBilling(user.ID, channel.ID, strings.TrimPrefix(modelName, "models/"), capability, c.GetHeader("X-Canvas-Scene"), c.GetHeader("X-Idempotency-Key"))
@@ -544,19 +598,6 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
return
}
}
query := c.Request.URL.Query()
for _, key := range []string{"key", "api_key", "access_token", "token"} {
query.Del(key)
}
target := strings.TrimRight(channel.BaseURL, "/") + path
if encodedQuery := query.Encode(); encodedQuery != "" {
target += "?" + encodedQuery
}
if _, err := service.ValidateOutboundURL(target); err != nil {
_ = svc.RefundBilling(billingOrderID, "系统渠道地址校验失败")
failService(c, err)
return
}
upstreamReq, err := http.NewRequestWithContext(c.Request.Context(), c.Request.Method, target, bytes.NewReader(body))
if err != nil {
_ = svc.RefundBilling(billingOrderID, "系统渠道请求构造失败")
@@ -583,7 +624,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
status = model.ApiCallStatusFailed
errorText = err.Error()
_ = svc.MarkBillingUncertain(billingOrderID, "系统渠道连接中断,费用状态待核对")
logSystemProxyCall(svc, apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, status, statusCode, time.Since(startedAt), errorText), nil)
logSystemProxyCall(svc, apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, status, statusCode, time.Since(startedAt), errorText, concurrencyLimit), nil)
fail(c, http.StatusBadGateway, errors.New("系统渠道连接失败"))
return
}
@@ -597,7 +638,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
status = model.ApiCallStatusFailed
errorText = readErr.Error()
_ = svc.MarkBillingUncertain(billingOrderID, "系统渠道响应读取失败,费用状态待核对")
logSystemProxyCall(svc, apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, status, statusCode, time.Since(startedAt), errorText), nil)
logSystemProxyCall(svc, apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, status, statusCode, time.Since(startedAt), errorText, concurrencyLimit), nil)
fail(c, http.StatusBadGateway, errors.New("系统渠道响应读取失败"))
return
}
@@ -615,7 +656,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
} else {
_ = svc.RefundBilling(billingOrderID, "上游明确返回失败")
}
logSystemProxyCall(svc, apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, status, statusCode, time.Since(startedAt), errorText), responseBody)
logSystemProxyCall(svc, apiCallLog(user, channel, billingOrderID, c.Request.Method, path, target, body, status, statusCode, time.Since(startedAt), errorText, concurrencyLimit), responseBody)
for _, key := range []string{"Content-Type", "Cache-Control", "Content-Disposition"} {
if value := resp.Header.Get(key); value != "" {
c.Header(key, value)
@@ -625,12 +666,12 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
c.Data(resp.StatusCode, resp.Header.Get("Content-Type"), responseBody)
}
func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID string, method string, path string, target string, body []byte, status model.ApiCallStatus, statusCode int, duration time.Duration, errorText string) model.ApiCallLog {
func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID string, method string, path string, target string, body []byte, status model.ApiCallStatus, statusCode int, duration time.Duration, errorText string, concurrencyLimit int) model.ApiCallLog {
capability := "text"
switch channel.InterfaceType {
case model.ChannelInterfaceOpenAIImage:
capability = "image"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2:
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
capability = "video"
}
requestKind := "create"
@@ -641,22 +682,23 @@ func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID st
}
}
return model.ApiCallLog{
UserID: user.ID,
ChannelID: channel.ID,
BillingOrderID: billingOrderID,
Source: "system-channel",
Capability: capability,
RequestKind: requestKind,
Billable: method == http.MethodPost,
APIFormat: channel.APIFormat,
Method: method,
Path: path,
Model: readPayloadModel(body),
Status: status,
StatusCode: statusCode,
DurationMs: duration.Milliseconds(),
Error: errorText,
UpstreamURL: target,
UserID: user.ID,
ChannelID: channel.ID,
BillingOrderID: billingOrderID,
Source: "system-channel",
Capability: capability,
RequestKind: requestKind,
Billable: method == http.MethodPost,
APIFormat: channel.APIFormat,
Method: method,
Path: path,
Model: readPayloadModel(body),
Status: status,
StatusCode: statusCode,
DurationMs: duration.Milliseconds(),
Error: errorText,
ConcurrencyLimit: concurrencyLimit,
UpstreamURL: target,
}
}

View File

@@ -147,18 +147,19 @@ type EmailVerificationCode struct {
}
type ModelChannel struct {
ID string `json:"id" gorm:"primaryKey;size:36"`
UserID string `json:"userId" gorm:"index;size:36"`
Scope ChannelScope `json:"scope" gorm:"index;size:24"`
Enabled bool `json:"enabled" gorm:"index"`
Name string `json:"name" gorm:"size:80"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"-"`
APIFormat string `json:"apiFormat" gorm:"size:24"`
InterfaceType ChannelInterfaceType `json:"interfaceType" gorm:"size:32"`
ModelsJSON string `json:"modelsJson" gorm:"type:text"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id" gorm:"primaryKey;size:36"`
UserID string `json:"userId" gorm:"index;size:36"`
Scope ChannelScope `json:"scope" gorm:"index;size:24"`
Enabled bool `json:"enabled" gorm:"index"`
Name string `json:"name" gorm:"size:80"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"-"`
APIFormat string `json:"apiFormat" gorm:"size:24"`
InterfaceType ChannelInterfaceType `json:"interfaceType" gorm:"size:32"`
ConcurrencyLimit int `json:"concurrencyLimit"`
ModelsJSON string `json:"modelsJson" gorm:"type:text"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type ChannelModel struct {
@@ -207,6 +208,7 @@ type ApiCallLog struct {
Currency string `json:"currency" gorm:"size:12"`
ErrorCode string `json:"errorCode,omitempty" gorm:"index;size:80"`
Error string `json:"error"`
ConcurrencyLimit int `json:"concurrencyLimit"`
UpstreamURL string `json:"upstreamUrl"`
CreatedAt time.Time `json:"createdAt" gorm:"index;index:idx_api_logs_user_created,priority:2;index:idx_api_logs_channel_created,priority:2;index:idx_api_logs_model_created,priority:2;index:idx_api_logs_status_created,priority:2"`
}

View File

@@ -75,29 +75,32 @@ type AdminReferenceData struct {
}
type ChannelRequest struct {
Name string `json:"name"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"apiKey"`
InterfaceType string `json:"interfaceType"`
Models []string `json:"models"`
Enabled *bool `json:"enabled"`
Name string `json:"name"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"apiKey"`
InterfaceType string `json:"interfaceType"`
ConcurrencyLimit *int `json:"concurrencyLimit"`
UseGlobalConcurrency *bool `json:"useGlobalConcurrency"`
Models []string `json:"models"`
Enabled *bool `json:"enabled"`
}
type PublicModelChannel struct {
ID string `json:"id"`
UserID string `json:"userId"`
Scope model.ChannelScope `json:"scope"`
Enabled bool `json:"enabled"`
Name string `json:"name"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"apiKey"`
APIFormat string `json:"apiFormat"`
InterfaceType model.ChannelInterfaceType `json:"interfaceType"`
Models []string `json:"models"`
ModelCosts []PublicChannelModelPrice `json:"modelCosts"`
HasAPIKey bool `json:"hasApiKey"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
UserID string `json:"userId"`
Scope model.ChannelScope `json:"scope"`
Enabled bool `json:"enabled"`
Name string `json:"name"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"apiKey"`
APIFormat string `json:"apiFormat"`
InterfaceType model.ChannelInterfaceType `json:"interfaceType"`
ConcurrencyLimit int `json:"concurrencyLimit"`
Models []string `json:"models"`
ModelCosts []PublicChannelModelPrice `json:"modelCosts"`
HasAPIKey bool `json:"hasApiKey"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type PublicChannelModelPrice struct {
@@ -530,6 +533,16 @@ func channelFromRequest(req ChannelRequest, channel model.ModelChannel) (model.M
// 系统渠道均由后端按已声明的接口类型分发,调用格式固定为 Bearer/OpenAI 兼容鉴权。
channel.APIFormat = "openai"
channel.InterfaceType = interfaceType
if req.UseGlobalConcurrency != nil && *req.UseGlobalConcurrency {
channel.ConcurrencyLimit = 0
} else if req.ConcurrencyLimit != nil {
if *req.ConcurrencyLimit < minChannelConcurrencyLimit || *req.ConcurrencyLimit > maxChannelConcurrencyLimit {
return channel, BadAuthRequest("最大并发数必须是 1-100 的整数")
}
channel.ConcurrencyLimit = *req.ConcurrencyLimit
} else if req.UseGlobalConcurrency != nil {
return channel, BadAuthRequest("请填写渠道最大并发数")
}
channel.ModelsJSON = string(modelsJSON)
if req.Enabled != nil {
channel.Enabled = *req.Enabled
@@ -611,20 +624,21 @@ func publicChannel(channel model.ModelChannel, admin bool, channelModels []model
interfaceType = inferChannelInterfaceType(models)
}
return PublicModelChannel{
ID: channel.ID,
UserID: channel.UserID,
Scope: channel.Scope,
Enabled: channel.Enabled,
Name: channel.Name,
BaseURL: baseURL,
APIKey: apiKey,
APIFormat: channel.APIFormat,
InterfaceType: interfaceType,
Models: models,
ModelCosts: modelCosts,
HasAPIKey: strings.TrimSpace(channel.APIKey) != "",
CreatedAt: channel.CreatedAt,
UpdatedAt: channel.UpdatedAt,
ID: channel.ID,
UserID: channel.UserID,
Scope: channel.Scope,
Enabled: channel.Enabled,
Name: channel.Name,
BaseURL: baseURL,
APIKey: apiKey,
APIFormat: channel.APIFormat,
InterfaceType: interfaceType,
ConcurrencyLimit: channel.ConcurrencyLimit,
Models: models,
ModelCosts: modelCosts,
HasAPIKey: strings.TrimSpace(channel.APIKey) != "",
CreatedAt: channel.CreatedAt,
UpdatedAt: channel.UpdatedAt,
}
}

View File

@@ -14,11 +14,12 @@ func TestChannelFromRequestStoresInterfaceType(t *testing.T) {
defer server.Close()
channel, err := channelFromRequest(ChannelRequest{
Name: "NewAPI 渠道 1",
BaseURL: server.URL + "/v1",
APIKey: "secret",
InterfaceType: "newapi-channel-1",
Models: []string{"seedance-2.0"},
Name: "NewAPI 渠道 1",
BaseURL: server.URL + "/v1",
APIKey: "secret",
InterfaceType: "newapi-channel-1",
ConcurrencyLimit: intPtr(6),
Models: []string{"seedance-2.0"},
}, model.ModelChannel{})
if err != nil {
t.Fatalf("channelFromRequest() error = %v", err)
@@ -29,6 +30,9 @@ func TestChannelFromRequestStoresInterfaceType(t *testing.T) {
if channel.APIFormat != "openai" {
t.Fatalf("APIFormat = %q, want openai", channel.APIFormat)
}
if channel.ConcurrencyLimit != 6 {
t.Fatalf("ConcurrencyLimit = %d, want 6", channel.ConcurrencyLimit)
}
}
func TestChannelFromRequestStoresXAIVideoInterfaceType(t *testing.T) {
@@ -70,3 +74,29 @@ func TestChannelFromRequestRejectsUnknownInterfaceType(t *testing.T) {
t.Fatal("channelFromRequest() error = nil")
}
}
func TestChannelFromRequestRejectsInvalidConcurrencyLimit(t *testing.T) {
for _, limit := range []int{0, 101} {
_, err := channelFromRequest(ChannelRequest{Name: "Bad", BaseURL: "https://example.com/v1", InterfaceType: "newapi", ConcurrencyLimit: &limit}, model.ModelChannel{})
if err == nil {
t.Fatalf("channelFromRequest() concurrencyLimit = %d, error = nil", limit)
}
}
}
func TestRuntimeConcurrencyUsesEnvironmentFallback(t *testing.T) {
t.Setenv("CANVAS_CHANNEL_CONCURRENCY", "7")
t.Setenv("CANVAS_WORKER_CONCURRENCY", "9")
setting := runtimeConcurrencyFromEnvironment()
if setting.ChannelConcurrency != 7 || setting.WorkerConcurrency != 9 {
t.Fatalf("runtimeConcurrencyFromEnvironment() = %#v", setting)
}
useGlobal := true
channel, err := channelFromRequest(ChannelRequest{Name: "Global", BaseURL: "https://example.com/v1", InterfaceType: "newapi", UseGlobalConcurrency: &useGlobal}, model.ModelChannel{ConcurrencyLimit: 4})
if err != nil || channel.ConcurrencyLimit != 0 {
t.Fatalf("global concurrency channel = %#v, error = %v", channel, err)
}
}
func intPtr(value int) *int { return &value }

View File

@@ -245,9 +245,9 @@ func (s *Service) AdminAPICallLogsCSV(actor *model.User, query APICallLogQuery)
var buffer bytes.Buffer
buffer.WriteString("\xEF\xBB\xBF")
writer := csv.NewWriter(&buffer)
_ = writer.Write([]string{"时间", "用户ID", "渠道ID", "任务ID", "计费单ID", "能力", "请求阶段", "模型", "状态", "HTTP状态", "耗时毫秒", "供应商任务ID", "错误码", "错误"})
_ = writer.Write([]string{"时间", "用户ID", "渠道ID", "任务ID", "计费单ID", "能力", "请求阶段", "模型", "状态", "HTTP状态", "耗时毫秒", "渠道并发上限", "供应商任务ID", "错误码", "错误"})
for _, log := range logs {
_ = writer.Write([]string{log.CreatedAt.UTC().Format(time.RFC3339), log.UserID, log.ChannelID, log.TaskID, log.BillingOrderID, log.Capability, log.RequestKind, log.Model, string(log.Status), strconv.Itoa(log.StatusCode), strconv.FormatInt(log.DurationMs, 10), log.ProviderRequestID, log.ErrorCode, log.Error})
_ = writer.Write([]string{log.CreatedAt.UTC().Format(time.RFC3339), log.UserID, log.ChannelID, log.TaskID, log.BillingOrderID, log.Capability, log.RequestKind, log.Model, string(log.Status), strconv.Itoa(log.StatusCode), strconv.FormatInt(log.DurationMs, 10), strconv.Itoa(log.ConcurrencyLimit), log.ProviderRequestID, log.ErrorCode, log.Error})
}
writer.Flush()
if err := writer.Error(); err != nil {

View File

@@ -0,0 +1,122 @@
package service
import (
"encoding/json"
"errors"
"strings"
"time"
"infinite-canvas/backend/internal/model"
"gorm.io/gorm"
)
const runtimeConcurrencySettingKey = "runtime_concurrency"
type RuntimeConcurrencySettingRequest struct {
WorkerConcurrency int `json:"workerConcurrency"`
ChannelConcurrency int `json:"channelConcurrency"`
}
type PublicRuntimeConcurrencySetting struct {
WorkerConcurrency int `json:"workerConcurrency"`
ChannelConcurrency int `json:"channelConcurrency"`
UpdatedBy string `json:"updatedBy"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type runtimeConcurrencySettingValue struct {
WorkerConcurrency int `json:"workerConcurrency"`
ChannelConcurrency int `json:"channelConcurrency"`
}
func (s *Service) AdminRuntimeConcurrencySetting(actor *model.User) (*PublicRuntimeConcurrencySetting, error) {
if err := s.RequireAdmin(actor); err != nil {
return nil, err
}
setting, value, err := s.readRuntimeConcurrencySetting()
if err != nil {
return nil, err
}
return publicRuntimeConcurrencySetting(setting, value), nil
}
func (s *Service) UpdateRuntimeConcurrencySetting(actor *model.User, req RuntimeConcurrencySettingRequest) (*PublicRuntimeConcurrencySetting, error) {
if err := s.RequireAdmin(actor); err != nil {
return nil, err
}
value := runtimeConcurrencySettingValue{WorkerConcurrency: req.WorkerConcurrency, ChannelConcurrency: req.ChannelConcurrency}
if err := validateRuntimeConcurrencySetting(value); err != nil {
return nil, err
}
encoded, err := json.Marshal(value)
if err != nil {
return nil, err
}
current, _, err := s.readRuntimeConcurrencySetting()
if err != nil {
return nil, err
}
setting := model.SystemSetting{Key: runtimeConcurrencySettingKey, ValueJSON: string(encoded), UpdatedBy: actor.ID}
if current != nil {
setting.CreatedAt = current.CreatedAt
}
if err := s.repo.SaveSystemSetting(&setting); err != nil {
return nil, err
}
if err := s.appendAdminAudit(actor, "runtime_concurrency.update", "system_setting", runtimeConcurrencySettingKey, "更新任务并发配置", value); err != nil {
return nil, err
}
return publicRuntimeConcurrencySetting(&setting, value), nil
}
func (s *Service) runtimeConcurrencySetting() (runtimeConcurrencySettingValue, error) {
_, value, err := s.readRuntimeConcurrencySetting()
return value, err
}
func (s *Service) readRuntimeConcurrencySetting() (*model.SystemSetting, runtimeConcurrencySettingValue, error) {
setting, err := s.repo.SystemSetting(runtimeConcurrencySettingKey)
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, runtimeConcurrencyFromEnvironment(), nil
}
if err != nil {
return nil, runtimeConcurrencySettingValue{}, err
}
value := runtimeConcurrencySettingValue{}
if strings.TrimSpace(setting.ValueJSON) == "" || json.Unmarshal([]byte(setting.ValueJSON), &value) != nil {
return nil, runtimeConcurrencySettingValue{}, errors.New("任务并发配置格式无效")
}
if err := validateRuntimeConcurrencySetting(value); err != nil {
return nil, runtimeConcurrencySettingValue{}, err
}
return setting, value, nil
}
func runtimeConcurrencyFromEnvironment() runtimeConcurrencySettingValue {
return runtimeConcurrencySettingValue{
WorkerConcurrency: effectiveChannelConcurrencyLimit(envInt("CANVAS_WORKER_CONCURRENCY", taskWorkerConcurrency)),
ChannelConcurrency: defaultChannelConcurrencyLimit(),
}
}
func validateRuntimeConcurrencySetting(value runtimeConcurrencySettingValue) error {
if value.WorkerConcurrency < minChannelConcurrencyLimit || value.WorkerConcurrency > maxChannelConcurrencyLimit {
return BadAuthRequest("Worker 并发数必须是 1-100 的整数")
}
if value.ChannelConcurrency < minChannelConcurrencyLimit || value.ChannelConcurrency > maxChannelConcurrencyLimit {
return BadAuthRequest("全局渠道并发数必须是 1-100 的整数")
}
return nil
}
func publicRuntimeConcurrencySetting(setting *model.SystemSetting, value runtimeConcurrencySettingValue) *PublicRuntimeConcurrencySetting {
result := &PublicRuntimeConcurrencySetting{WorkerConcurrency: value.WorkerConcurrency, ChannelConcurrency: value.ChannelConcurrency}
if setting != nil {
result.UpdatedBy = setting.UpdatedBy
result.CreatedAt = setting.CreatedAt
result.UpdatedAt = setting.UpdatedAt
}
return result
}

View File

@@ -0,0 +1,63 @@
package service
import (
"context"
"testing"
"time"
"infinite-canvas/backend/internal/model"
"infinite-canvas/backend/internal/repository"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestRuntimeConcurrencySettingPersistsAndChannelOverrideWins(t *testing.T) {
t.Setenv("REDIS_URL", "")
t.Setenv("CANVAS_WORKER_CONCURRENCY", "3")
t.Setenv("CANVAS_CHANNEL_CONCURRENCY", "3")
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if sqlDB, err := db.DB(); err == nil {
sqlDB.SetMaxOpenConns(1)
}
if err := db.AutoMigrate(&model.SystemSetting{}, &model.AdminAuditEvent{}, &model.ModelChannel{}); err != nil {
t.Fatal(err)
}
svc := New(repository.New(db), t.TempDir())
actor := &model.User{ID: "admin-1", Role: model.UserRoleAdmin}
setting, err := svc.UpdateRuntimeConcurrencySetting(actor, RuntimeConcurrencySettingRequest{WorkerConcurrency: 8, ChannelConcurrency: 6})
if err != nil {
t.Fatal(err)
}
if setting.WorkerConcurrency != 8 || setting.ChannelConcurrency != 6 {
t.Fatalf("setting = %#v", setting)
}
channel := model.ModelChannel{ID: "channel-1", Scope: model.ChannelScopeSystem, Enabled: true, ConcurrencyLimit: 0}
if err := db.Create(&channel).Error; err != nil {
t.Fatal(err)
}
release, limit, err := svc.AcquireChannelSlot(context.Background(), channel.ID, "", time.Minute)
if err != nil {
t.Fatal(err)
}
release()
if limit != 6 {
t.Fatalf("global channel limit = %d, want 6", limit)
}
if err := db.Model(&channel).Update("concurrency_limit", 9).Error; err != nil {
t.Fatal(err)
}
release, limit, err = svc.AcquireChannelSlot(context.Background(), channel.ID, "", time.Minute)
if err != nil {
t.Fatal(err)
}
release()
if limit != 9 {
t.Fatalf("overridden channel limit = %d, want 9", limit)
}
}

View File

@@ -18,11 +18,50 @@ type localRateEntry struct {
count int
}
const (
minChannelConcurrencyLimit = 1
maxChannelConcurrencyLimit = 100
defaultChannelConcurrencyValue = 3
)
type channelSlotError struct {
scope string
limit int
err error
}
func (e channelSlotError) Error() string {
if errors.Is(e.err, context.DeadlineExceeded) {
return fmt.Sprintf("等待渠道并发槽位超时(渠道 %s,并发上限 %d)", e.scope, e.limit)
}
if errors.Is(e.err, context.Canceled) {
return fmt.Sprintf("等待渠道并发槽位已取消(渠道 %s,并发上限 %d)", e.scope, e.limit)
}
return fmt.Sprintf("获取渠道并发配额失败(渠道 %s,并发上限 %d):%v", e.scope, e.limit, e.err)
}
func (e channelSlotError) Unwrap() error { return e.err }
func ChannelSlotFailureDetails(err error) (string, string) {
var slotErr channelSlotError
if !errors.As(err, &slotErr) {
return "", ""
}
if errors.Is(slotErr, context.DeadlineExceeded) {
return "channel_concurrency_wait_timeout", slotErr.Error()
}
if errors.Is(slotErr, context.Canceled) {
return "channel_concurrency_wait_cancelled", slotErr.Error()
}
return "channel_concurrency_unavailable", slotErr.Error()
}
type runtimeCoordinator struct {
redis *redis.Client
instanceID string
localMu sync.Mutex
localRate map[string]localRateEntry
localSlots map[string]map[string]time.Time
}
var fixedWindowScript = redis.NewScript(`
@@ -40,7 +79,7 @@ return 1
`)
func newRuntimeCoordinator(dialect string) (*runtimeCoordinator, error) {
coordinator := &runtimeCoordinator{instanceID: newID(), localRate: map[string]localRateEntry{}}
coordinator := &runtimeCoordinator{instanceID: newID(), localRate: map[string]localRateEntry{}, localSlots: map[string]map[string]time.Time{}}
redisURL := strings.TrimSpace(os.Getenv("REDIS_URL"))
if redisURL == "" {
if dialect == "postgres" {
@@ -84,7 +123,30 @@ func (c *runtimeCoordinator) allow(ctx context.Context, key string, limit int, w
func (c *runtimeCoordinator) acquire(ctx context.Context, scope string, limit int, ttl time.Duration) (func(), bool, error) {
if c.redis == nil {
return func() {}, true, nil
c.localMu.Lock()
now := time.Now()
slots := c.localSlots[scope]
if slots == nil {
slots = map[string]time.Time{}
c.localSlots[scope] = slots
}
for token, expiresAt := range slots {
if !expiresAt.After(now) {
delete(slots, token)
}
}
if len(slots) >= limit {
c.localMu.Unlock()
return nil, false, nil
}
token := c.instanceID + ":" + newID()
slots[token] = now.Add(ttl)
c.localMu.Unlock()
return func() {
c.localMu.Lock()
delete(c.localSlots[scope], token)
c.localMu.Unlock()
}, true, nil
}
// 有过期分数的有序集合避免实例崩溃后永久占槽,业务数据库仍保存任务与账本真相。
key := "canvas:slots:" + scope
@@ -97,6 +159,28 @@ func (c *runtimeCoordinator) acquire(ctx context.Context, scope string, limit in
return func() { _ = c.redis.ZRem(context.Background(), key, token).Err() }, true, nil
}
func (c *runtimeCoordinator) acquireWithWait(ctx context.Context, scope string, limit int, ttl time.Duration) (func(), error) {
ticker := time.NewTicker(200 * time.Millisecond)
defer ticker.Stop()
for {
if err := ctx.Err(); err != nil {
return nil, err
}
release, acquired, err := c.acquire(ctx, scope, limit, ttl)
if err != nil {
return nil, err
}
if acquired {
return release, nil
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-ticker.C:
}
}
}
func (c *runtimeCoordinator) circuitOpen(ctx context.Context, channelID string) (bool, error) {
if c.redis == nil || strings.TrimSpace(channelID) == "" {
return false, nil
@@ -133,6 +217,52 @@ func envInt(key string, fallback int) int {
return value
}
func defaultChannelConcurrencyLimit() int {
return effectiveChannelConcurrencyLimit(envInt("CANVAS_CHANNEL_CONCURRENCY", defaultChannelConcurrencyValue))
}
func effectiveChannelConcurrencyLimit(configured int) int {
if configured < minChannelConcurrencyLimit || configured > maxChannelConcurrencyLimit {
return defaultChannelConcurrencyValue
}
return configured
}
func (s *Service) AcquireChannelSlot(ctx context.Context, channelID string, fallbackScope string, ttl time.Duration) (func(), int, error) {
setting, err := s.runtimeConcurrencySetting()
limit := defaultChannelConcurrencyLimit()
if err != nil {
return nil, limit, channelSlotError{scope: firstNonEmpty(strings.TrimSpace(channelID), strings.TrimSpace(fallbackScope), "unknown"), limit: limit, err: fmt.Errorf("读取全局并发配置失败:%w", err)}
}
limit = setting.ChannelConcurrency
scope := strings.TrimSpace(channelID)
if scope != "" {
channel, err := s.repo.SystemChannel(scope)
if err != nil {
return nil, limit, channelSlotError{scope: scope, limit: limit, err: fmt.Errorf("读取渠道并发配置失败:%w", err)}
}
if channel.ConcurrencyLimit > 0 {
if channel.ConcurrencyLimit < minChannelConcurrencyLimit || channel.ConcurrencyLimit > maxChannelConcurrencyLimit {
return nil, limit, channelSlotError{scope: scope, limit: limit, err: errors.New("渠道并发配置超出 1-100 范围")}
}
limit = channel.ConcurrencyLimit
}
} else {
scope = strings.TrimSpace(fallbackScope)
}
if scope == "" {
return nil, limit, channelSlotError{scope: "unknown", limit: limit, err: errors.New("渠道并发范围为空")}
}
if s.coordinator == nil {
return nil, limit, channelSlotError{scope: scope, limit: limit, err: errors.New("运行时协调器未初始化")}
}
release, err := s.coordinator.acquireWithWait(ctx, "channel:"+scope, limit, ttl)
if err != nil {
return nil, limit, channelSlotError{scope: scope, limit: limit, err: err}
}
return release, limit, nil
}
func (s *Service) ValidateRuntime() error {
return s.runtimeErr
}

View File

@@ -0,0 +1,54 @@
package service
import (
"context"
"testing"
"time"
)
func TestRuntimeCoordinatorWaitsUntilChannelSlotIsReleased(t *testing.T) {
coordinator := &runtimeCoordinator{instanceID: "test", localRate: map[string]localRateEntry{}, localSlots: map[string]map[string]time.Time{}}
releaseFirst, acquired, err := coordinator.acquire(context.Background(), "channel:one", 1, time.Minute)
if err != nil || !acquired {
t.Fatalf("first acquire = (%v, %v), want acquired", acquired, err)
}
result := make(chan error, 1)
go func() {
releaseSecond, waitErr := coordinator.acquireWithWait(context.Background(), "channel:one", 1, time.Minute)
if waitErr == nil {
releaseSecond()
}
result <- waitErr
}()
select {
case err := <-result:
t.Fatalf("second acquire returned before release: %v", err)
case <-time.After(50 * time.Millisecond):
}
releaseFirst()
select {
case err := <-result:
if err != nil {
t.Fatalf("second acquire after release: %v", err)
}
case <-time.After(time.Second):
t.Fatal("second acquire did not resume after release")
}
}
func TestRuntimeCoordinatorStopsWaitingWhenContextIsCancelled(t *testing.T) {
coordinator := &runtimeCoordinator{instanceID: "test", localRate: map[string]localRateEntry{}, localSlots: map[string]map[string]time.Time{}}
release, acquired, err := coordinator.acquire(context.Background(), "channel:one", 1, time.Minute)
if err != nil || !acquired {
t.Fatalf("first acquire = (%v, %v), want acquired", acquired, err)
}
defer release()
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := coordinator.acquireWithWait(ctx, "channel:one", 1, time.Minute); err == nil {
t.Fatal("acquireWithWait() error = nil after cancellation")
}
}

View File

@@ -97,6 +97,7 @@ type providerAnalyticsContext struct {
VideoSeconds int
RequestKind string
ProviderRequestID string
ConcurrencyLimit int
}
func withProviderAnalytics(ctx context.Context, service *Service, task model.Task) context.Context {
@@ -1274,17 +1275,17 @@ func doBinary(req *http.Request) ([]byte, string, error) {
if open {
return nil, "", errors.New("当前渠道连续失败,已暂时熔断,请稍后重试")
}
var acquired bool
slotID := channelID
if slotID == "" {
slotID = "custom:" + strings.ToLower(req.URL.Host)
}
release, acquired, err = coordinator.acquire(req.Context(), "channel:"+slotID, envInt("CANVAS_CHANNEL_CONCURRENCY", 2), providerHTTPTimeout+time.Minute)
var concurrencyLimit int
release, concurrencyLimit, err = metadata.Service.AcquireChannelSlot(req.Context(), channelID, slotID, providerHTTPTimeout+time.Minute)
metadata.ConcurrencyLimit = concurrencyLimit
req = req.WithContext(context.WithValue(req.Context(), providerAnalyticsKey{}, metadata))
if err != nil {
return nil, "", fmt.Errorf("获取渠道并发配额失败:%w", err)
}
if !acquired {
return nil, "", errors.New("当前渠道并发已满,请稍后重试")
recordProviderRequest(req, startedAt, 0, nil, err)
return nil, "", err
}
defer release()
}
@@ -1356,7 +1357,13 @@ func recordProviderRequest(req *http.Request, startedAt time.Time, statusCode in
RequestKind: requestKind, Billable: req.Method == http.MethodPost,
APIFormat: "openai", Method: req.Method, Path: req.URL.Path, Model: metadata.Model,
Status: status, StatusCode: statusCode, DurationMs: time.Since(startedAt).Milliseconds(),
Error: errorText, UpstreamURL: req.URL.Scheme + "://" + req.URL.Host + req.URL.Path,
Error: errorText, ConcurrencyLimit: metadata.ConcurrencyLimit, UpstreamURL: req.URL.Scheme + "://" + req.URL.Host + req.URL.Path,
}
channelSlotFailure := false
if code, message := ChannelSlotFailureDetails(requestErr); code != "" {
channelSlotFailure = true
log.ErrorCode = code
log.Error = message
}
if requestKind == "create" && metadata.Capability == "video" {
log.VideoSeconds = metadata.VideoSeconds
@@ -1370,7 +1377,9 @@ func recordProviderRequest(req *http.Request, startedAt time.Time, statusCode in
}
metadata.Service.EnrichAPICallLog(&log, responseBody)
if err := metadata.Service.LogAPICall(log); err != nil {
_ = metadata.Service.MarkBillingUncertain(metadata.BillingOrderID, "上游调用日志写入失败,费用状态待核对")
if !channelSlotFailure {
_ = metadata.Service.MarkBillingUncertain(metadata.BillingOrderID, "上游调用日志写入失败,费用状态待核对")
}
}
}

View File

@@ -160,10 +160,14 @@ func New(repo *repository.Repository, dataDir string) *Service {
func (s *Service) StartWorker() {
go func() {
workerConcurrency := envInt("CANVAS_WORKER_CONCURRENCY", taskWorkerConcurrency)
slots := make(chan struct{}, workerConcurrency)
slots := make(chan struct{}, maxChannelConcurrencyLimit)
dispatch := func() {
for len(slots) < cap(slots) {
setting, err := s.runtimeConcurrencySetting()
if err != nil {
return
}
workerConcurrency := setting.WorkerConcurrency
for len(slots) < workerConcurrency {
releaseGlobal, acquired, err := s.coordinator.acquire(context.Background(), "workers", workerConcurrency, 45*time.Minute)
if err != nil || !acquired {
return
@@ -697,6 +701,10 @@ func (s *Service) processClaimedTask(task *model.Task) error {
result, err = s.persistGeneratedMediaResult(task.UserID, result)
}
if err != nil {
channelSlotFailedBeforeRequest := false
if code, _ := ChannelSlotFailureDetails(err); code != "" {
channelSlotFailedBeforeRequest = true
}
select {
case leaseErr := <-leaseLost:
_ = s.log(task.UserID, task.ID, "warn", "任务租约失效,等待其他 worker 恢复", leaseErr.Error())
@@ -709,7 +717,11 @@ func (s *Service) processClaimedTask(task *model.Task) error {
task.Error = "任务已取消"
task.CompletedAt = ptr(time.Now())
_ = s.repo.Save(task)
_ = s.MarkBillingUncertain(task.BillingOrderID, "任务取消时上游费用状态不明确")
if channelSlotFailedBeforeRequest {
_ = s.RefundBilling(task.BillingOrderID, "等待渠道槽位期间取消,上游请求未发出")
} else {
_ = s.MarkBillingUncertain(task.BillingOrderID, "任务取消时上游费用状态不明确")
}
_ = s.markSessionFailed(*task, "会话任务已取消。")
_ = s.log(task.UserID, task.ID, "warn", "任务已取消", "")
return nil
@@ -722,7 +734,7 @@ func (s *Service) processClaimedTask(task *model.Task) error {
task.Error = taskFailureMessage(err)
task.CompletedAt = ptr(time.Now())
_ = s.repo.Save(task)
if providerSucceeded || s.BillingFailureRequiresReview(task.BillingOrderID, task.ID, err) {
if providerSucceeded || (!channelSlotFailedBeforeRequest && s.BillingFailureRequiresReview(task.BillingOrderID, task.ID, err)) {
_ = s.MarkBillingUncertain(task.BillingOrderID, task.Error)
} else {
_ = s.RefundBilling(task.BillingOrderID, task.Error)