347 lines
14 KiB
Go
347 lines
14 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"os"
|
||
"runtime"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"infinite-canvas/backend/internal/buildinfo"
|
||
"infinite-canvas/backend/internal/model"
|
||
"infinite-canvas/backend/internal/repository"
|
||
|
||
"github.com/redis/go-redis/v9"
|
||
)
|
||
|
||
var systemProcessStartedAt = time.Now()
|
||
|
||
const systemPerformanceTimeout = 3 * time.Second
|
||
|
||
type AdminSystemPerformance struct {
|
||
CollectedAt time.Time `json:"collectedAt"`
|
||
Status string `json:"status"`
|
||
Host SystemPerformanceHost `json:"host"`
|
||
Memory SystemPerformanceMemory `json:"memory"`
|
||
Disk SystemPerformanceDisk `json:"disk"`
|
||
Database repository.DatabaseRuntimeStats `json:"database"`
|
||
Redis SystemPerformanceRedis `json:"redis"`
|
||
Build buildinfo.Info `json:"build"`
|
||
}
|
||
|
||
type SystemPerformanceHost struct {
|
||
Hostname string `json:"hostname"`
|
||
OS string `json:"os"`
|
||
Arch string `json:"arch"`
|
||
CPUCores int `json:"cpuCores"`
|
||
GOMAXPROCS int `json:"gomaxprocs"`
|
||
ProcessID int `json:"processId"`
|
||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||
Goroutines int `json:"goroutines"`
|
||
ActiveWorkerTasks int64 `json:"activeWorkerTasks"`
|
||
LoadAverage [3]float64 `json:"loadAverage"`
|
||
LoadAverageAvailable bool `json:"loadAverageAvailable"`
|
||
}
|
||
|
||
type SystemPerformanceMemory struct {
|
||
SystemAvailable bool `json:"systemAvailable"`
|
||
TotalBytes uint64 `json:"totalBytes"`
|
||
UsedBytes uint64 `json:"usedBytes"`
|
||
AvailableBytes uint64 `json:"availableBytes"`
|
||
UsagePercent float64 `json:"usagePercent"`
|
||
HeapAllocBytes uint64 `json:"heapAllocBytes"`
|
||
HeapSysBytes uint64 `json:"heapSysBytes"`
|
||
SysBytes uint64 `json:"sysBytes"`
|
||
HeapObjects uint64 `json:"heapObjects"`
|
||
GCCount uint32 `json:"gcCount"`
|
||
LastGCAt *time.Time `json:"lastGCAt,omitempty"`
|
||
}
|
||
|
||
type SystemPerformanceDisk struct {
|
||
Available bool `json:"available"`
|
||
Writable bool `json:"writable"`
|
||
TotalBytes uint64 `json:"totalBytes"`
|
||
UsedBytes uint64 `json:"usedBytes"`
|
||
FreeBytes uint64 `json:"freeBytes"`
|
||
UsagePercent float64 `json:"usagePercent"`
|
||
}
|
||
|
||
type SystemPerformanceRedisPool struct {
|
||
Hits uint32 `json:"hits"`
|
||
Misses uint32 `json:"misses"`
|
||
Timeouts uint32 `json:"timeouts"`
|
||
TotalConnections uint32 `json:"totalConnections"`
|
||
IdleConnections uint32 `json:"idleConnections"`
|
||
StaleConnections uint32 `json:"staleConnections"`
|
||
PendingRequests uint32 `json:"pendingRequests"`
|
||
}
|
||
|
||
type SystemPerformanceCacheGroup struct {
|
||
ID string `json:"id"`
|
||
Label string `json:"label"`
|
||
Keys int64 `json:"keys"`
|
||
Clearable bool `json:"clearable"`
|
||
}
|
||
|
||
type SystemPerformanceRedis struct {
|
||
Configured bool `json:"configured"`
|
||
Connected bool `json:"connected"`
|
||
Mode string `json:"mode"`
|
||
LatencyMs int64 `json:"latencyMs,omitempty"`
|
||
Version string `json:"version,omitempty"`
|
||
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
|
||
UsedMemoryBytes int64 `json:"usedMemoryBytes,omitempty"`
|
||
PeakMemoryBytes int64 `json:"peakMemoryBytes,omitempty"`
|
||
MaxMemoryBytes int64 `json:"maxMemoryBytes,omitempty"`
|
||
MaxMemoryPolicy string `json:"maxMemoryPolicy,omitempty"`
|
||
Keys int64 `json:"keys,omitempty"`
|
||
Clients int64 `json:"clients,omitempty"`
|
||
BlockedClients int64 `json:"blockedClients,omitempty"`
|
||
OpsPerSecond int64 `json:"opsPerSecond,omitempty"`
|
||
HitRate float64 `json:"hitRate,omitempty"`
|
||
ExpiredKeys int64 `json:"expiredKeys,omitempty"`
|
||
EvictedKeys int64 `json:"evictedKeys,omitempty"`
|
||
Pool SystemPerformanceRedisPool `json:"pool"`
|
||
CacheGroups []SystemPerformanceCacheGroup `json:"cacheGroups"`
|
||
StatusMessage string `json:"statusMessage,omitempty"`
|
||
}
|
||
|
||
type AdminCacheClearRequest struct {
|
||
Scope string `json:"scope"`
|
||
}
|
||
|
||
type AdminCacheClearGroupResult struct {
|
||
ID string `json:"id"`
|
||
DeletedKeys int64 `json:"deletedKeys"`
|
||
}
|
||
|
||
type AdminCacheClearResult struct {
|
||
Scope string `json:"scope"`
|
||
DeletedKeys int64 `json:"deletedKeys"`
|
||
MemoryEntries int `json:"memoryEntries"`
|
||
Groups []AdminCacheClearGroupResult `json:"groups"`
|
||
ClearedAt time.Time `json:"clearedAt"`
|
||
}
|
||
|
||
type runtimeCacheDefinition struct {
|
||
id, label, pattern string
|
||
}
|
||
|
||
var runtimeCacheDefinitions = []runtimeCacheDefinition{
|
||
{id: "rateLimits", label: "请求频控", pattern: "canvas:rate:*"},
|
||
{id: "circuitFailures", label: "渠道失败计数", pattern: "canvas:circuit:failures:*"},
|
||
{id: "circuitOpen", label: "渠道熔断状态", pattern: "canvas:circuit:open:*"},
|
||
{id: "routeHealth", label: "线路临时禁用", pattern: "canvas:logical-route-health:*"},
|
||
}
|
||
|
||
func (s *Service) AdminSystemPerformance(ctx context.Context, actor *model.User) (*AdminSystemPerformance, error) {
|
||
if err := s.RequireAdmin(actor); err != nil {
|
||
return nil, err
|
||
}
|
||
ctx, cancel := context.WithTimeout(ctx, systemPerformanceTimeout)
|
||
defer cancel()
|
||
|
||
hostname, _ := os.Hostname()
|
||
load, loadAvailable := platformLoadAverage()
|
||
memory := collectSystemMemory()
|
||
disk := collectSystemDisk(s.dataDir)
|
||
databaseStats, databaseErr := s.repo.DatabaseRuntimeStats(ctx)
|
||
redisStats := s.collectRedisPerformance(ctx)
|
||
status := "healthy"
|
||
if databaseErr != nil || !databaseStats.Connected || !disk.Available || !disk.Writable || s.ValidateRuntime() != nil || (redisStats.Configured && !redisStats.Connected) {
|
||
status = "degraded"
|
||
}
|
||
return &AdminSystemPerformance{
|
||
CollectedAt: time.Now(), Status: status,
|
||
Host: SystemPerformanceHost{Hostname: hostname, OS: runtime.GOOS, Arch: runtime.GOARCH, CPUCores: runtime.NumCPU(), GOMAXPROCS: runtime.GOMAXPROCS(0), ProcessID: os.Getpid(), UptimeSeconds: int64(time.Since(systemProcessStartedAt).Seconds()), Goroutines: runtime.NumGoroutine(), ActiveWorkerTasks: s.ActiveWorkerTasks(), LoadAverage: load, LoadAverageAvailable: loadAvailable},
|
||
Memory: memory, Disk: disk, Database: databaseStats, Redis: redisStats, Build: buildinfo.Current(),
|
||
}, nil
|
||
}
|
||
|
||
func collectSystemMemory() SystemPerformanceMemory {
|
||
var stats runtime.MemStats
|
||
runtime.ReadMemStats(&stats)
|
||
total, available, ok := platformMemory()
|
||
used := uint64(0)
|
||
percentage := float64(0)
|
||
if ok && total >= available {
|
||
used = total - available
|
||
if total > 0 {
|
||
percentage = float64(used) / float64(total) * 100
|
||
}
|
||
}
|
||
result := SystemPerformanceMemory{SystemAvailable: ok, TotalBytes: total, UsedBytes: used, AvailableBytes: available, UsagePercent: percentage, HeapAllocBytes: stats.HeapAlloc, HeapSysBytes: stats.HeapSys, SysBytes: stats.Sys, HeapObjects: stats.HeapObjects, GCCount: stats.NumGC}
|
||
if stats.LastGC > 0 {
|
||
last := time.Unix(0, int64(stats.LastGC))
|
||
result.LastGCAt = &last
|
||
}
|
||
return result
|
||
}
|
||
|
||
func (s *Service) collectRedisPerformance(ctx context.Context) SystemPerformanceRedis {
|
||
result := SystemPerformanceRedis{Mode: "local", StatusMessage: "未配置 Redis，当前使用单实例本地协调", CacheGroups: make([]SystemPerformanceCacheGroup, 0, len(runtimeCacheDefinitions))}
|
||
if s.coordinator == nil {
|
||
result.CacheGroups = append(result.CacheGroups, SystemPerformanceCacheGroup{ID: "rateLimits", Label: "本地请求频控", Clearable: true})
|
||
return result
|
||
}
|
||
if s.coordinator.redis == nil {
|
||
s.coordinator.localMu.Lock()
|
||
localRateKeys := int64(len(s.coordinator.localRate))
|
||
s.coordinator.localMu.Unlock()
|
||
result.CacheGroups = append(result.CacheGroups, SystemPerformanceCacheGroup{ID: "rateLimits", Label: "本地请求频控", Keys: localRateKeys, Clearable: true})
|
||
return result
|
||
}
|
||
result.Configured, result.Mode = true, "redis"
|
||
started := time.Now()
|
||
if err := s.coordinator.redis.Ping(ctx).Err(); err != nil {
|
||
result.StatusMessage = "Redis 暂时不可用"
|
||
return result
|
||
}
|
||
result.Connected = true
|
||
result.LatencyMs = max(1, time.Since(started).Milliseconds())
|
||
result.StatusMessage = "Redis 连接正常"
|
||
if size, err := s.coordinator.redis.DBSize(ctx).Result(); err == nil {
|
||
result.Keys = size
|
||
}
|
||
if info, err := s.coordinator.redis.Info(ctx, "server", "clients", "memory", "stats").Result(); err == nil {
|
||
values := parseRedisInfo(info)
|
||
result.Version = values["redis_version"]
|
||
result.UptimeSeconds = redisInfoInt(values, "uptime_in_seconds")
|
||
result.UsedMemoryBytes = redisInfoInt(values, "used_memory")
|
||
result.PeakMemoryBytes = redisInfoInt(values, "used_memory_peak")
|
||
result.MaxMemoryBytes = redisInfoInt(values, "maxmemory")
|
||
result.MaxMemoryPolicy = values["maxmemory_policy"]
|
||
result.Clients = redisInfoInt(values, "connected_clients")
|
||
result.BlockedClients = redisInfoInt(values, "blocked_clients")
|
||
result.OpsPerSecond = redisInfoInt(values, "instantaneous_ops_per_sec")
|
||
result.ExpiredKeys = redisInfoInt(values, "expired_keys")
|
||
result.EvictedKeys = redisInfoInt(values, "evicted_keys")
|
||
hits, misses := redisInfoInt(values, "keyspace_hits"), redisInfoInt(values, "keyspace_misses")
|
||
if hits+misses > 0 {
|
||
result.HitRate = float64(hits) / float64(hits+misses) * 100
|
||
}
|
||
}
|
||
pool := s.coordinator.redis.PoolStats()
|
||
result.Pool = SystemPerformanceRedisPool{Hits: pool.Hits, Misses: pool.Misses, Timeouts: pool.Timeouts, TotalConnections: pool.TotalConns, IdleConnections: pool.IdleConns, StaleConnections: pool.StaleConns, PendingRequests: pool.PendingRequests}
|
||
for _, definition := range runtimeCacheDefinitions {
|
||
count, _ := scanRedisKeyCount(ctx, s.coordinator.redis, definition.pattern)
|
||
result.CacheGroups = append(result.CacheGroups, SystemPerformanceCacheGroup{ID: definition.id, Label: definition.label, Keys: count, Clearable: true})
|
||
}
|
||
return result
|
||
}
|
||
|
||
func (s *Service) ClearAdminRuntimeCache(ctx context.Context, actor *model.User, request AdminCacheClearRequest) (*AdminCacheClearResult, error) {
|
||
if err := s.RequireAdmin(actor); err != nil {
|
||
return nil, err
|
||
}
|
||
if strings.TrimSpace(request.Scope) != "runtime" {
|
||
return nil, BadAuthRequest("仅支持清理运行时缓存")
|
||
}
|
||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||
defer cancel()
|
||
result := &AdminCacheClearResult{Scope: "runtime", ClearedAt: time.Now(), Groups: make([]AdminCacheClearGroupResult, 0, len(runtimeCacheDefinitions))}
|
||
if s.coordinator != nil {
|
||
if s.coordinator.redis != nil {
|
||
for _, definition := range runtimeCacheDefinitions {
|
||
deleted, err := deleteRedisKeysByPattern(ctx, s.coordinator.redis, definition.pattern)
|
||
if err != nil {
|
||
return nil, errors.New("清理 Redis 运行时缓存失败")
|
||
}
|
||
result.Groups = append(result.Groups, AdminCacheClearGroupResult{ID: definition.id, DeletedKeys: deleted})
|
||
result.DeletedKeys += deleted
|
||
}
|
||
} else {
|
||
s.coordinator.localMu.Lock()
|
||
count := len(s.coordinator.localRate)
|
||
s.coordinator.localRate = map[string]localRateEntry{}
|
||
s.coordinator.localMu.Unlock()
|
||
result.DeletedKeys += int64(count)
|
||
result.Groups = append(result.Groups, AdminCacheClearGroupResult{ID: "rateLimits", DeletedKeys: int64(count)})
|
||
}
|
||
}
|
||
s.initReadCaches()
|
||
result.MemoryEntries += s.concurrencyReadCache.clearAndCount()
|
||
result.MemoryEntries += s.textReplayReadCache.clearAndCount()
|
||
result.MemoryEntries += s.routeVersionReadCache.clearAndCount()
|
||
s.routeHealthMu.Lock()
|
||
result.MemoryEntries += len(s.routeHealthBlocked)
|
||
s.routeHealthBlocked = make(map[string]time.Time)
|
||
s.routeHealthMu.Unlock()
|
||
// 与路由目录刷新保持同一加锁顺序，避免清理动作和回源刷新并发写入重试状态。
|
||
s.routeCatalogRefreshMu.Lock()
|
||
s.routeCatalogMu.Lock()
|
||
if s.routeCatalog != nil {
|
||
result.MemoryEntries++
|
||
}
|
||
s.routeCatalog = nil
|
||
s.routeCatalogVersion = 0
|
||
s.routeCatalogRetryAt = time.Time{}
|
||
s.routeCatalogRefreshError = nil
|
||
s.routeCatalogMu.Unlock()
|
||
s.routeCatalogRefreshMu.Unlock()
|
||
if err := s.appendAdminAudit(actor, "system.runtime_cache.clear", "system", "runtime-cache", "清理安全运行时缓存", map[string]any{"deletedKeys": result.DeletedKeys, "memoryEntries": result.MemoryEntries}); err != nil {
|
||
return nil, err
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func parseRedisInfo(info string) map[string]string {
|
||
values := make(map[string]string)
|
||
for _, line := range strings.Split(info, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
key, value, found := strings.Cut(line, ":")
|
||
if found {
|
||
values[key] = strings.TrimSpace(value)
|
||
}
|
||
}
|
||
return values
|
||
}
|
||
|
||
func redisInfoInt(values map[string]string, key string) int64 {
|
||
value, _ := strconv.ParseInt(values[key], 10, 64)
|
||
return value
|
||
}
|
||
|
||
func scanRedisKeyCount(ctx context.Context, client *redis.Client, pattern string) (int64, error) {
|
||
var cursor uint64
|
||
var total int64
|
||
for {
|
||
keys, next, err := client.Scan(ctx, cursor, pattern, 200).Result()
|
||
if err != nil {
|
||
return total, err
|
||
}
|
||
total += int64(len(keys))
|
||
cursor = next
|
||
if cursor == 0 {
|
||
return total, nil
|
||
}
|
||
}
|
||
}
|
||
|
||
func deleteRedisKeysByPattern(ctx context.Context, client *redis.Client, pattern string) (int64, error) {
|
||
var cursor uint64
|
||
var deleted int64
|
||
for {
|
||
keys, next, err := client.Scan(ctx, cursor, pattern, 200).Result()
|
||
if err != nil {
|
||
return deleted, err
|
||
}
|
||
if len(keys) > 0 {
|
||
count, err := client.Unlink(ctx, keys...).Result()
|
||
if err != nil {
|
||
return deleted, err
|
||
}
|
||
deleted += count
|
||
}
|
||
cursor = next
|
||
if cursor == 0 {
|
||
return deleted, nil
|
||
}
|
||
}
|
||
}
|