fix(tasks): 任务队列 - 限制缓存回源并修复并发租约与过期写入
This commit is contained in:
1 parent
1ff5e5cada
commit
bf0c4454fa
23 files changed
+1098
-59
No files matched your search
@@ -446,15 +446,23 @@ func isProviderCapabilityOption(name string) bool {
|
||||
}
|
||||
|
||||
func (s *Service) invalidateRouteCatalog() {
|
||||
s.routeCatalogRefreshMu.Lock()
|
||||
s.routeCatalogMu.Lock()
|
||||
s.routeCatalog = nil
|
||||
s.routeCatalogVersion++
|
||||
s.routeCatalogRetryAt = time.Time{}
|
||||
s.routeCatalogRefreshError = nil
|
||||
s.routeCatalogMu.Unlock()
|
||||
s.routeCatalogRefreshMu.Unlock()
|
||||
if s.coordinator != nil {
|
||||
if err := s.coordinator.bumpRouteCatalogVersion(context.Background()); err != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
|
||||
defer cancel()
|
||||
if err := s.coordinator.bumpRouteCatalogVersion(ctx); err != nil {
|
||||
log.Printf("logical model route catalog distributed invalidation failed: %v", err)
|
||||
}
|
||||
}
|
||||
s.initReadCaches()
|
||||
s.routeVersionReadCache.clear()
|
||||
}
|
||||
|
||||
func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) {
|
||||
@@ -480,15 +488,26 @@ func (s *Service) routeCatalogSnapshot() (*routeCatalogSnapshot, error) {
|
||||
}
|
||||
s.routeCatalogMu.RUnlock()
|
||||
|
||||
// 刷新锁只能防止并行回源,失败后还需要冷却,否则等待者会逐个重打数据库。
|
||||
if s.routeCatalogRefreshError != nil && now.Before(s.routeCatalogRetryAt) {
|
||||
if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
|
||||
return snapshot, nil
|
||||
}
|
||||
return nil, s.routeCatalogRefreshError
|
||||
}
|
||||
loaded, err := s.loadRouteCatalog()
|
||||
if err != nil {
|
||||
s.routeCatalogRefreshError = err
|
||||
s.routeCatalogRetryAt = time.Now().Add(2 * time.Second)
|
||||
log.Printf("logical model route catalog refresh failed; retry cooled down: %v", err)
|
||||
// 已有快照过期时允许短暂继续服务,数据库首次加载失败则明确失败。
|
||||
if snapshot != nil && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
|
||||
log.Printf("logical model route catalog refresh failed, serving stale snapshot age=%s: %v", now.Sub(snapshot.LoadedAt).Round(time.Second), err)
|
||||
if snapshot != nil && snapshot.CatalogVersion == version && now.Sub(snapshot.LoadedAt) <= s.routeCatalogMaxStale {
|
||||
return snapshot, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
s.routeCatalogRefreshError = nil
|
||||
s.routeCatalogRetryAt = time.Time{}
|
||||
s.routeCatalogMu.Lock()
|
||||
s.routeCatalog = loaded
|
||||
s.routeCatalogMu.Unlock()
|
||||
@@ -502,11 +521,17 @@ func (s *Service) currentRouteCatalogVersion() int64 {
|
||||
if s.coordinator == nil || s.coordinator.redis == nil {
|
||||
return localVersion
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
s.initReadCaches()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runtimeCoordinationTimeout)
|
||||
defer cancel()
|
||||
version, err := s.coordinator.routeCatalogVersion(ctx)
|
||||
version, err := s.routeVersionReadCache.get(ctx, routeCatalogVersionKey, func(ctx context.Context) (int64, int, error) {
|
||||
value, err := s.coordinator.routeCatalogVersion(ctx)
|
||||
if err != nil {
|
||||
log.Printf("logical model route catalog distributed version check failed: %v", err)
|
||||
}
|
||||
return value, 256, err
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("logical model route catalog distributed version check failed: %v", err)
|
||||
return localVersion
|
||||
}
|
||||
if version > localVersion {
|
||||
@@ -516,12 +541,15 @@ func (s *Service) currentRouteCatalogVersion() int64 {
|
||||
}
|
||||
|
||||
func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) {
|
||||
items, err := s.repo.LogicalModels(false)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
repo := s.repo.WithContext(ctx)
|
||||
items, err := repo.LogicalModels(false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
snapshot := &routeCatalogSnapshot{LoadedAt: time.Now(), CatalogVersion: s.currentRouteCatalogVersion(), Models: make(map[string]cachedLogicalModel), Ordered: make([]string, 0, len(items))}
|
||||
graphs, err := s.repo.LogicalModelGraphs(items, false)
|
||||
graphs, err := repo.LogicalModelGraphs(items, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -534,7 +562,7 @@ func (s *Service) loadRouteCatalog() (*routeCatalogSnapshot, error) {
|
||||
systemChannelIDs = append(systemChannelIDs, channelModel.ChannelID)
|
||||
}
|
||||
}
|
||||
systemChannels, err := s.repo.SystemChannelsByIDs(systemChannelIDs, false)
|
||||
systemChannels, err := repo.SystemChannelsByIDs(systemChannelIDs, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user