fix(*): 平台稳定性 - 修复会话隔离、运行时缓存与部署镜像更新链路
This commit is contained in:
1 parent
88abdd1b75
commit
e77c26d2b0
32 files changed
+788
-116
No files matched your search
@@ -188,6 +188,18 @@ func (s *Service) AdminReferences(actor *model.User) (*AdminReferenceData, error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
channelIDs := make([]string, 0, len(channels))
|
||||
for _, channel := range channels {
|
||||
channelIDs = append(channelIDs, channel.ID)
|
||||
}
|
||||
channelModels, err := s.repo.ChannelModelReferences(channelIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
modelsByChannel := make(map[string][]model.ChannelModel, len(channels))
|
||||
for _, item := range channelModels {
|
||||
modelsByChannel[item.ChannelID] = append(modelsByChannel[item.ChannelID], item)
|
||||
}
|
||||
result := &AdminReferenceData{
|
||||
Users: make([]AdminUserReference, 0, len(users)),
|
||||
Channels: make([]AdminChannelReference, 0, len(channels)),
|
||||
@@ -196,10 +208,7 @@ func (s *Service) AdminReferences(actor *model.User) (*AdminReferenceData, error
|
||||
result.Users = append(result.Users, AdminUserReference{ID: user.ID, Username: user.Username, DisplayName: user.DisplayName})
|
||||
}
|
||||
for _, channel := range channels {
|
||||
items, itemErr := s.repo.ChannelModels(channel.ID, true)
|
||||
if itemErr != nil {
|
||||
return nil, itemErr
|
||||
}
|
||||
items := modelsByChannel[channel.ID]
|
||||
models := make([]string, 0, len(items))
|
||||
displayNames := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
|
||||
@@ -73,7 +73,7 @@ func TestCachedTextReplayIsolatesUsersCursorsAndCopies(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeConcurrencyCacheDoesNotCacheAuthoritativePolicy(t *testing.T) {
|
||||
func TestRuntimeConcurrencyCacheCachesAuthoritativePolicy(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -101,8 +101,8 @@ func TestRuntimeConcurrencyCacheDoesNotCacheAuthoritativePolicy(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if queries.Load() != before+2 {
|
||||
t.Fatal("authoritative policy incorrectly cached")
|
||||
if queries.Load() != before {
|
||||
t.Fatalf("authoritative policy cache missed: queries=%d before=%d", queries.Load(), before)
|
||||
}
|
||||
policy := defaultRuntimePolicy()
|
||||
policy.Task.WorkerConcurrency = 7
|
||||
|
||||
@@ -164,11 +164,6 @@ func RegisterAuthRoutes(r *gin.RouterGroup, svc *service.Service) {
|
||||
failService(c, err)
|
||||
return
|
||||
}
|
||||
logicalModels, logicalModelsErr := svc.PublicLogicalModels(nil)
|
||||
if logicalModelsErr != nil {
|
||||
failService(c, logicalModelsErr)
|
||||
return
|
||||
}
|
||||
limits, err := svc.PublicRuntimeLimits()
|
||||
if err != nil {
|
||||
failService(c, err)
|
||||
@@ -184,7 +179,7 @@ func RegisterAuthRoutes(r *gin.RouterGroup, svc *service.Service) {
|
||||
failService(c, err)
|
||||
return
|
||||
}
|
||||
ok(c, gin.H{"user": publicUser, "logicalModels": logicalModels, "runtimeLimits": limits, "drawingEngine": drawingEngine, "features": features})
|
||||
ok(c, gin.H{"user": publicUser, "runtimeLimits": limits, "drawingEngine": drawingEngine, "features": features})
|
||||
})
|
||||
r.GET("/channels/system", func(c *gin.Context) {
|
||||
actor, err := currentUser(c, svc)
|
||||
|
||||
@@ -264,13 +264,19 @@ func (m *Manager) runUpdate(fromVersion, targetVersion string) {
|
||||
m.failWithoutRollback(PhaseFailed, fmt.Errorf("保存旧 Compose 配置:%w", err))
|
||||
return
|
||||
}
|
||||
if err := replaceFile(m.envPath(), m.previousEnvPath(), 0o600); err != nil {
|
||||
m.failWithoutRollback(PhaseFailed, fmt.Errorf("保存旧部署环境:%w", err))
|
||||
return
|
||||
}
|
||||
|
||||
m.setPhase(PhasePulling, "拉取目标版本镜像")
|
||||
if err := m.compose(nextCompose, targetVersion, m.config.StepTimeout, nil, "pull", "backend", "web"); err != nil {
|
||||
targetImages := immutableImageRefs(m.config.Repository, targetVersion)
|
||||
if err := m.composeWithImages(nextCompose, targetVersion, targetImages, m.config.StepTimeout, nil, "pull", "backend", "web"); err != nil {
|
||||
m.failWithoutRollback(PhaseFailed, err)
|
||||
return
|
||||
}
|
||||
if err := m.verifyImages(targetVersion); err != nil {
|
||||
targetImages, err = m.verifyImages(targetVersion)
|
||||
if err != nil {
|
||||
m.failWithoutRollback(PhaseFailed, err)
|
||||
return
|
||||
}
|
||||
@@ -282,13 +288,13 @@ func (m *Manager) runUpdate(fromVersion, targetVersion string) {
|
||||
}
|
||||
|
||||
m.setPhase(PhaseMigrating, "执行目标版本数据库迁移")
|
||||
if err := m.compose(nextCompose, targetVersion, m.config.StepTimeout, nil, "run", "--rm", "migrate"); err != nil {
|
||||
if err := m.composeWithImages(nextCompose, targetVersion, targetImages, m.config.StepTimeout, nil, "run", "--rm", "migrate"); err != nil {
|
||||
m.failWithRollback(err, fromVersion, backup)
|
||||
return
|
||||
}
|
||||
|
||||
m.setPhase(PhaseSwitching, "切换 Compose 配置与镜像版本")
|
||||
if err := setEnvValue(m.envPath(), "CANVAS_IMAGE_TAG", strings.TrimPrefix(targetVersion, "v")); err != nil {
|
||||
if err := setDeploymentImages(m.envPath(), targetVersion, targetImages); err != nil {
|
||||
m.failWithRollback(err, fromVersion, backup)
|
||||
return
|
||||
}
|
||||
@@ -297,7 +303,7 @@ func (m *Manager) runUpdate(fromVersion, targetVersion string) {
|
||||
m.failWithRollback(err, fromVersion, backup)
|
||||
return
|
||||
}
|
||||
if err := m.compose(m.composePath(), targetVersion, m.config.StepTimeout, nil, "up", "-d", "--remove-orphans", "--wait", "--wait-timeout", "600"); err != nil {
|
||||
if err := m.composeWithImages(m.composePath(), targetVersion, targetImages, m.config.StepTimeout, nil, "up", "-d", "--remove-orphans", "--wait", "--wait-timeout", "600"); err != nil {
|
||||
m.failWithRollback(err, fromVersion, backup)
|
||||
return
|
||||
}
|
||||
@@ -368,10 +374,12 @@ func (m *Manager) runRollback(targetVersion string, backup Backup, automatic boo
|
||||
failures = append(failures, fmt.Errorf("恢复旧 Compose 配置:%w", err))
|
||||
}
|
||||
}
|
||||
if err := m.restoreDatabase(backup); err != nil {
|
||||
failures = append(failures, err)
|
||||
if _, err := os.Stat(m.previousEnvPath()); err == nil {
|
||||
if err := replaceFile(m.previousEnvPath(), m.envPath(), 0o600); err != nil {
|
||||
failures = append(failures, fmt.Errorf("恢复旧部署环境:%w", err))
|
||||
}
|
||||
}
|
||||
if err := setEnvValue(m.envPath(), "CANVAS_IMAGE_TAG", strings.TrimPrefix(targetVersion, "v")); err != nil {
|
||||
if err := m.restoreDatabase(backup); err != nil {
|
||||
failures = append(failures, err)
|
||||
}
|
||||
started := true
|
||||
@@ -513,6 +521,9 @@ func (m *Manager) envPath() string { return filepath.Join(m.config.InstallDir, m
|
||||
func (m *Manager) previousComposePath() string {
|
||||
return filepath.Join(m.config.StateDir, "previous-compose.yml")
|
||||
}
|
||||
func (m *Manager) previousEnvPath() string {
|
||||
return filepath.Join(m.config.StateDir, "previous.env")
|
||||
}
|
||||
|
||||
func randomID() string {
|
||||
buffer := make([]byte, 12)
|
||||
|
||||
@@ -133,7 +133,7 @@ func (m *Manager) preflight(composePath, targetVersion string) error {
|
||||
if err := checkBackupDiskSpace(m.config.BackupDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.compose(composePath, targetVersion, 2*time.Minute, nil, "config", "--quiet"); err != nil {
|
||||
if err := m.composeWithImages(composePath, targetVersion, immutableImageRefs(m.config.Repository, targetVersion), 2*time.Minute, nil, "config", "--quiet"); err != nil {
|
||||
return fmt.Errorf("目标 Compose 校验失败:%w", err)
|
||||
}
|
||||
current, err := m.currentVersion()
|
||||
@@ -243,7 +243,25 @@ func (m *Manager) restartSelf() {
|
||||
_ = m.runner.Run(ctx, "systemctl", []string{"restart", m.config.ServiceName}, nil, io.Discard, io.Discard)
|
||||
}
|
||||
|
||||
type deploymentImages struct {
|
||||
backend string
|
||||
web string
|
||||
}
|
||||
|
||||
func immutableImageRefs(repository, version string) deploymentImages {
|
||||
owner := strings.SplitN(repository, "/", 2)[0]
|
||||
tag := strings.TrimPrefix(version, "v")
|
||||
return deploymentImages{
|
||||
backend: "ghcr.io/" + owner + "/open-ai-canvas-backend:" + tag,
|
||||
web: "ghcr.io/" + owner + "/open-ai-canvas-web:" + tag,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) compose(composePath, imageTag string, timeout time.Duration, stdout io.Writer, arguments ...string) error {
|
||||
return m.composeWithImages(composePath, imageTag, deploymentImages{}, timeout, stdout, arguments...)
|
||||
}
|
||||
|
||||
func (m *Manager) composeWithImages(composePath, imageTag string, images deploymentImages, timeout time.Duration, stdout io.Writer, arguments ...string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
args := []string{"compose", "--env-file", m.envPath(), "-f", composePath}
|
||||
@@ -252,7 +270,11 @@ func (m *Manager) compose(composePath, imageTag string, timeout time.Duration, s
|
||||
if stdout == nil {
|
||||
stdout = io.Discard
|
||||
}
|
||||
err := m.runner.Run(ctx, "docker", args, []string{"CANVAS_IMAGE_TAG=" + strings.TrimPrefix(imageTag, "v")}, stdout, &stderr)
|
||||
environment := []string{"CANVAS_IMAGE_TAG=" + strings.TrimPrefix(imageTag, "v")}
|
||||
if images.backend != "" {
|
||||
environment = append(environment, "CANVAS_BACKEND_IMAGE="+images.backend, "CANVAS_WEB_IMAGE="+images.web)
|
||||
}
|
||||
err := m.runner.Run(ctx, "docker", args, environment, stdout, &stderr)
|
||||
if err != nil {
|
||||
message := strings.TrimSpace(stderr.String())
|
||||
if len(message) > 1000 {
|
||||
@@ -266,22 +288,47 @@ func (m *Manager) compose(composePath, imageTag string, timeout time.Duration, s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) verifyImages(targetVersion string) error {
|
||||
owner := strings.SplitN(m.config.Repository, "/", 2)[0]
|
||||
for _, image := range []string{"ghcr.io/" + owner + "/open-ai-canvas-backend:", "ghcr.io/" + owner + "/open-ai-canvas-web:"} {
|
||||
func (m *Manager) verifyImages(targetVersion string) (deploymentImages, error) {
|
||||
images := immutableImageRefs(m.config.Repository, targetVersion)
|
||||
refs := []string{images.backend, images.web}
|
||||
digests := make([]string, 0, len(refs))
|
||||
for _, image := range refs {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
var output bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
err := m.runner.Run(ctx, "docker", []string{"image", "inspect", image + strings.TrimPrefix(targetVersion, "v"), "--format", "{{json .RepoDigests}}"}, nil, &output, &stderr)
|
||||
err := m.runner.Run(ctx, "docker", []string{"image", "inspect", image, "--format", "{{json .RepoDigests}}"}, nil, &output, &stderr)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("校验目标镜像摘要失败:%s", strings.TrimSpace(stderr.String()))
|
||||
return deploymentImages{}, fmt.Errorf("校验目标镜像摘要失败:%s", strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
if !strings.Contains(output.String(), "@sha256:") {
|
||||
return fmt.Errorf("目标镜像 %s 未包含仓库摘要", image+targetVersion)
|
||||
var repositoryDigests []string
|
||||
if err := json.Unmarshal(output.Bytes(), &repositoryDigests); err != nil {
|
||||
return deploymentImages{}, fmt.Errorf("解析目标镜像摘要失败:%w", err)
|
||||
}
|
||||
repository := strings.Split(image, ":")[0]
|
||||
var digest string
|
||||
for _, candidate := range repositoryDigests {
|
||||
if strings.HasPrefix(candidate, repository+"@sha256:") {
|
||||
digest = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if digest == "" {
|
||||
return deploymentImages{}, fmt.Errorf("目标镜像 %s 未包含仓库摘要", image)
|
||||
}
|
||||
digests = append(digests, digest)
|
||||
}
|
||||
return nil
|
||||
return deploymentImages{backend: digests[0], web: digests[1]}, nil
|
||||
}
|
||||
|
||||
func setDeploymentImages(path, version string, images deploymentImages) error {
|
||||
if err := setEnvValue(path, "CANVAS_IMAGE_TAG", strings.TrimPrefix(version, "v")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := setEnvValue(path, "CANVAS_BACKEND_IMAGE", images.backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return setEnvValue(path, "CANVAS_WEB_IMAGE", images.web)
|
||||
}
|
||||
|
||||
func (m *Manager) createBackup(version string) (Backup, error) {
|
||||
|
||||
@@ -28,10 +28,11 @@ func (nopHost) ChannelConcurrencyLimit(string) (int, error) {
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
repo *repository.Repository
|
||||
host Host
|
||||
Coordinator *Coordinator
|
||||
concurrencyCache *BoundedReadCache[string, RuntimeTaskPolicy]
|
||||
repo *repository.Repository
|
||||
host Host
|
||||
Coordinator *Coordinator
|
||||
concurrencyCache *BoundedReadCache[string, RuntimeTaskPolicy]
|
||||
runtimePolicyCache *BoundedReadCache[string, RuntimePolicySetting]
|
||||
}
|
||||
|
||||
func New(repo *repository.Repository, coordinator *Coordinator, host Host) *Service {
|
||||
@@ -39,10 +40,11 @@ func New(repo *repository.Repository, coordinator *Coordinator, host Host) *Serv
|
||||
host = nopHost{}
|
||||
}
|
||||
return &Service{
|
||||
repo: repo,
|
||||
host: host,
|
||||
Coordinator: coordinator,
|
||||
concurrencyCache: NewBoundedReadCache[string, RuntimeTaskPolicy](1, 1024, 1, 2*time.Second),
|
||||
repo: repo,
|
||||
host: host,
|
||||
Coordinator: coordinator,
|
||||
concurrencyCache: NewBoundedReadCache[string, RuntimeTaskPolicy](1, 1024, 1, 2*time.Second),
|
||||
runtimePolicyCache: NewBoundedReadCache[string, RuntimePolicySetting](1, 8192, 1, 2*time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"infinite-canvas/backend/internal/model"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const runtimePolicySettingKey = "runtime_policy"
|
||||
@@ -216,8 +214,21 @@ func selfUseRuntimePolicy() RuntimePolicySetting {
|
||||
func SelfUseRuntimePolicy() RuntimePolicySetting { return selfUseRuntimePolicy() }
|
||||
|
||||
func (s *Service) RuntimePolicy() (RuntimePolicySetting, error) {
|
||||
_, value, err := s.readRuntimePolicy()
|
||||
return value, err
|
||||
if s.runtimePolicyCache == nil {
|
||||
_, value, err := s.readRuntimePolicy()
|
||||
return value, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
return s.runtimePolicyCache.Get(ctx, runtimePolicySettingKey, func(ctx context.Context) (RuntimePolicySetting, int, error) {
|
||||
reader := &Service{
|
||||
repo: s.repo.WithContext(ctx),
|
||||
host: s.host,
|
||||
Coordinator: s.Coordinator,
|
||||
}
|
||||
_, value, err := reader.readRuntimePolicy()
|
||||
return value, 2048, err
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) RuntimeConcurrencySetting() (RuntimeTaskPolicy, error) {
|
||||
@@ -225,7 +236,7 @@ func (s *Service) RuntimeConcurrencySetting() (RuntimeTaskPolicy, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
return s.concurrencyCache.Get(ctx, runtimePolicySettingKey, func(ctx context.Context) (RuntimeTaskPolicy, int, error) {
|
||||
reader := &Service{repo: s.repo.WithContext(ctx), host: s.host, Coordinator: s.Coordinator, concurrencyCache: s.concurrencyCache}
|
||||
reader := &Service{repo: s.repo.WithContext(ctx), host: s.host, Coordinator: s.Coordinator, concurrencyCache: s.concurrencyCache, runtimePolicyCache: s.runtimePolicyCache}
|
||||
policy, err := reader.RuntimePolicy()
|
||||
return policy.Task, 256, err
|
||||
})
|
||||
@@ -284,6 +295,9 @@ func (s *Service) UpdateRuntimePolicySetting(actor *model.User, value RuntimePol
|
||||
}
|
||||
|
||||
s.concurrencyCache.Clear()
|
||||
if s.runtimePolicyCache != nil {
|
||||
s.runtimePolicyCache.Clear()
|
||||
}
|
||||
if err := s.appendAdminAudit(actor, "runtime_policy.update", "system_setting", runtimePolicySettingKey, "更新资源与请求策略", map[string]any{"before": before, "after": value}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -303,6 +317,9 @@ func (s *Service) ResetRuntimePolicySetting(actor *model.User) (*PublicRuntimePo
|
||||
}
|
||||
|
||||
s.concurrencyCache.Clear()
|
||||
if s.runtimePolicyCache != nil {
|
||||
s.runtimePolicyCache.Clear()
|
||||
}
|
||||
after := DefaultRuntimePolicy()
|
||||
if err := s.appendAdminAudit(actor, "runtime_policy.reset", "system_setting", runtimePolicySettingKey, "重置资源与请求策略", map[string]any{"before": before, "after": after}); err != nil {
|
||||
return nil, err
|
||||
@@ -311,14 +328,14 @@ func (s *Service) ResetRuntimePolicySetting(actor *model.User) (*PublicRuntimePo
|
||||
}
|
||||
|
||||
func (s *Service) readRuntimePolicy() (*model.SystemSetting, RuntimePolicySetting, error) {
|
||||
setting, err := s.repo.SystemSetting(runtimePolicySettingKey)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
value := DefaultRuntimePolicy()
|
||||
return nil, value, validateRuntimePolicy(value)
|
||||
}
|
||||
setting, err := s.repo.SystemSettingOptional(runtimePolicySettingKey)
|
||||
if err != nil {
|
||||
return nil, RuntimePolicySetting{}, err
|
||||
}
|
||||
if setting == nil {
|
||||
value := DefaultRuntimePolicy()
|
||||
return nil, value, validateRuntimePolicy(value)
|
||||
}
|
||||
value := DefaultRuntimePolicy()
|
||||
if strings.TrimSpace(setting.ValueJSON) == "" || json.Unmarshal([]byte(setting.ValueJSON), &value) != nil {
|
||||
return nil, RuntimePolicySetting{}, errors.New("资源与请求策略配置格式无效")
|
||||
|
||||
@@ -86,6 +86,20 @@ func (r *Repository) ChannelModels(channelID string, includeDisabled bool) ([]mo
|
||||
return items, r.attachChannelModelPriceTiers(pointers)
|
||||
}
|
||||
|
||||
func (r *Repository) ChannelModelReferences(channelIDs []string) ([]model.ChannelModel, error) {
|
||||
if len(channelIDs) == 0 {
|
||||
return []model.ChannelModel{}, nil
|
||||
}
|
||||
var items []model.ChannelModel
|
||||
if err := r.db.Select("id", "channel_id", "model_key", "display_name", "sort_order", "enabled").
|
||||
Where("channel_id IN ?", channelIDs).
|
||||
Order("channel_id asc, sort_order asc, created_at asc, id asc").
|
||||
Find(&items).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r *Repository) CreateDuplicatedSystemChannel(channel *model.ModelChannel, channelModels []model.ChannelModel, priceTiers []model.ChannelModelPriceTier) error {
|
||||
return r.db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Create(channel).Error; err != nil {
|
||||
|
||||
@@ -835,6 +835,17 @@ func (r *Repository) SystemSetting(key string) (*model.SystemSetting, error) {
|
||||
return &setting, nil
|
||||
}
|
||||
|
||||
func (r *Repository) SystemSettingOptional(key string) (*model.SystemSetting, error) {
|
||||
var setting model.SystemSetting
|
||||
if err := r.db.Where("key = ?", key).Limit(1).Find(&setting).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if setting.Key == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return &setting, nil
|
||||
}
|
||||
|
||||
func (r *Repository) SaveSystemSetting(setting *model.SystemSetting) error {
|
||||
return r.db.Save(setting).Error
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user