package timer import ( "fmt" "math/rand" "time" "github.com/didi/nightingale/v5/cache" "github.com/didi/nightingale/v5/models" "github.com/toolkits/pkg/logger" ) func SyncAlertRules() { if err := syncAlertRules(); err != nil { fmt.Println(err) exit(1) } go loopSyncAlertRules() } func loopSyncAlertRules() { randtime := rand.Intn(9000) fmt.Printf("timer: sync alert rules: random sleep %dms\n", randtime) time.Sleep(time.Duration(randtime) * time.Millisecond) interval := time.Duration(9) * time.Second for { time.Sleep(interval) if err := syncAlertRules(); err != nil { logger.Warning(err) } } } func syncAlertRules() error { start := time.Now() // 上次同步的时候同步了多少条rule,数据库中最近的更新时间,如果这俩信息都没变,说明DB中数据没变 // 数据库中数据没变,那就不用再做操作了 lastMaxUpdateTs := cache.AlertRulesByMetric.MaxUpdateTs ruleNum := cache.AlertRulesByMetric.RuleNum statistic, err := models.GetAlertRuleStatistic() if err != nil { return fmt.Errorf("sync alertRules getAlertRuleStatistics err: %v", err) } if statistic.Count == ruleNum && statistic.MaxUpdateAt == lastMaxUpdateTs { lastMaxUpdateStr := time.Unix(lastMaxUpdateTs, 0).Format("2006-01-02 15:04:05") logger.Debugf("[no_change_not_sync][LastUpdateAt:%+v][ruleNum:%+v]:", lastMaxUpdateStr, ruleNum) return nil } // 数据库中的记录和上次拉取的数据相比,发生变化,重新从数据库拉取最新数据 logger.Debugf("[alert_rule_change_start_sync][last_num:%d this_num:%d][last_max_update_ts:%d this_max_update_ts:%d]:", ruleNum, statistic.Count, lastMaxUpdateTs, statistic.MaxUpdateAt) alertRules, err := models.AllAlertRules() alertRulesMap := make(map[int64]*models.AlertRule) if err != nil { return fmt.Errorf("sync alertRules [type=all] err: %v", err) } metricAlertRulesMap := make(map[string][]*models.AlertRule) for i := range alertRules { if err := alertRules[i].Decode(); err != nil { // 单个rule无法decode,直接忽略继续处理别的,等后面用户修复好了,数据库last_update信息变化,这里自然能感知 logger.Warningf("syncAlertRule %v err:%v", alertRules[i], err) continue } alertRulesMap[alertRules[i].Id] = alertRules[i] if alertRules[i].Type == models.PUSH { metricAlertRulesMap[alertRules[i].FirstMetric] = append(metricAlertRulesMap[alertRules[i].FirstMetric], alertRules[i]) } } cache.AlertRules.SetAll(alertRulesMap) cache.AlertRulesByMetric.SetAll(metricAlertRulesMap, statistic.MaxUpdateAt, statistic.Count, start.UnixNano()) logger.Infof("[timer] sync alert rules done, found %d records, cost: %dms", statistic.Count, time.Since(start).Milliseconds()) return nil }