fix(tasks): 任务队列 - 限制缓存回源并修复并发租约与过期写入

This commit is contained in:
ddCat committed 2026-09-06 00:28:45 +08:00
1 parent 1ff5e5cada
commit bf0c4454fa
23 files changed
+1098 -59

No files matched your search

+37 -9
View File
@@ -446,15 +446,23 @@ func isProviderCapabilityOption(name string) bool {
}
func (s *Service) invalidateRouteCatalog() {
s.routeCatalogRefreshMu.Lock()
s.routeCatalogMu.Lock()
s.routeCatalog = nil
s.routeCatalogVersion++
s.routeCatalogRetryAt = time.Time{}
s.routeCatalogRefreshError = nil
s.routeCatalogMu.Unlock()
s.routeCatalogRefreshMu.Unlock()
if s.coordinator != nil {
if err := s.coordinator.bumpRouteCatalogVersion(context.Background()); err != nil {
ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
defer cancel()
if err := s.coordinator.bumpRouteCatalogVersion(ctx); err != nil {
log.Printf("logical model route catalog distributed invalidation failed: %v", err)
}
}
s.initReadCaches()
s.routeVersionReadCache.clear()
}
func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) {
@@ -480,15 +488,26 @@ func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) {
}
s.routeCatalogMu.RUnlock()
// 刷新锁只能防止并行回源,失败后还需要冷却,否则等待者会逐个重打数据库。
if s.routeCatalogRefreshError != nil && now.Before(s.routeCatalogRetryAt) {
if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
return snapshot, nil
}
return nil, s.routeCatalogRefreshError
}
loaded, err := s.loadRouteCatalog()
if err != nil {
s.routeCatalogRefreshError = err
s.routeCatalogRetryAt = time.Now().Add(2 * time.Second)
log.Printf("logical model route catalog refresh failed; retry cooled down: %v", err)
// 已有快照过期时允许短暂继续服务,数据库首次加载失败则明确失败。
if snapshot != nil && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
log.Printf("logical model route catalog refresh failed, serving stale snapshot age=%s: %v", now.Sub(snapshot.LoadedAt).Round(time.Second), err)
if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
return snapshot, nil
}
return nil, err
}
s.routeCatalogRefreshError = nil
s.routeCatalogRetryAt = time.Time{}
s.routeCatalogMu.Lock()
s.routeCatalog = loaded
s.routeCatalogMu.Unlock()
@@ -502,11 +521,17 @@ func (s *Service) currentRouteCatalogVersion() int64 {
if s.coordinator == nil || s.coordinator.redis == nil {
return localVersion
}
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
s.initReadCaches()
ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
defer cancel()
version, err := s.coordinator.routeCatalogVersion(ctx)
version, err := s.routeVersionReadCache.get(ctx, routeCatalogVersionKey, func(ctx context.Context) (int64, int, error) {
value, err := s.coordinator.routeCatalogVersion(ctx)
if err != nil {
log.Printf("logical model route catalog distributed version check failed: %v", err)
}
return value, 256, err
})
if err != nil {
log.Printf("logical model route catalog distributed version check failed: %v", err)
return localVersion
}
if version > localVersion {
@@ -516,12 +541,15 @@ func (s *Service) currentRouteCatalogVersion() int64 {
}
func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) {
items, err := s.repo.LogicalModels(false)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
repo := s.repo.WithContext(ctx)
items, err := repo.LogicalModels(false)
if err != nil {
return nil, err
}
snapshot := &routeCatalogSnapshot{LoadedAt: time.Now(), CatalogVersion: s.currentRouteCatalogVersion(), Models: make(map[string]cachedLogicalModel), Ordered: make([]string, 0, len(items))}
graphs, err := s.repo.LogicalModelGraphs(items, false)
graphs, err := repo.LogicalModelGraphs(items, false)
if err != nil {
return nil, err
}
@@ -534,7 +562,7 @@ func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) {
systemChannelIDs = append(systemChannelIDs, channelModel.ChannelID)
}
}
systemChannels, err := s.repo.SystemChannelsByIDs(systemChannelIDs, false)
systemChannels, err := repo.SystemChannelsByIDs(systemChannelIDs, false)
if err != nil {
return nil, err
}