p58036791
/
nightingale
forked from pneymrl2f/nightingale
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: sort event tags & change extra

Browse Source
master
710leo 5 years ago
parent 322eb97a45
commit d9b89f24b5
11 changed files with 75 additions and 62 deletions
Show Stats Download Patch File Download Diff File

22
src/dataobj/event.go
Unescape View File

@ -20,6 +20,24 @@ type History struct {
Metric string `json:"metric"` // 指标名 Metric string `json:"metric"` // 指标名
Tags map[string]string `json:"tags,omitempty"` // endpoint/counter Tags map[string]string `json:"tags,omitempty"` // endpoint/counter
Granularity int `json:"-"` // alarm补齐数据时需要 Granularity int `json:"-"` // alarm补齐数据时需要
Points []*RRDData `json:"points"` // 现场值 Points []*HistoryData `json:"points"` // 现场值
Extra string `json:"extra"` }
type HistoryData struct {
Timestamp int64 `json:"timestamp"`
Value JsonFloat `json:"value"`
Extra string `json:"extra"`
}
func RRDData2HistoryData(datas []*RRDData) []*HistoryData {
historyDatas := make([]*HistoryData, len(datas))
for i := range datas {
historyData := &HistoryData{
Timestamp: datas[i].Timestamp,
Value: datas[i].Value,
}
historyDatas[i] = historyData
}
return historyDatas
} }

6
src/dataobj/judge.go
Unescape View File

@ -28,9 +28,3 @@ func (j *JudgeItem) PrimaryKey() string {
func (j *JudgeItem) MD5() string { func (j *JudgeItem) MD5() string {
return gstr.MD5(str.PK(strconv.FormatInt(j.Sid, 16), j.Endpoint, j.Metric, str.SortedTags(j.TagsMap))) return gstr.MD5(str.PK(strconv.FormatInt(j.Sid, 16), j.Endpoint, j.Metric, str.SortedTags(j.TagsMap)))
} }
//告警现场的值
type HistoryData struct {
Timestamp int64 `json:"timestamp"`
Value float64 `json:"value"`
}

2
src/model/event.go
Unescape View File

@ -39,12 +39,12 @@ type EventDetail struct {
Tags map[string]string `json:"tags"` Tags map[string]string `json:"tags"`
Points []*EventDetailPoint `json:"points"` Points []*EventDetailPoint `json:"points"`
PredPoints []*EventDetailPoint `json:"pred_points,omitempty"` // 预测值, 预测值不为空时, 现场值对应的是实际值 PredPoints []*EventDetailPoint `json:"pred_points,omitempty"` // 预测值, 预测值不为空时, 现场值对应的是实际值
Extra string `json:"extra"`
} }
type EventDetailPoint struct { type EventDetailPoint struct {
Timestamp int64 `json:"timestamp"` Timestamp int64 `json:"timestamp"`
Value float64 `json:"value"` Value float64 `json:"value"`
Extra string `json:"extra"`
} }
type EventAlertUpgrade struct { type EventAlertUpgrade struct {

2
src/model/event_cur.go
Unescape View File

@ -232,7 +232,7 @@ func EventCurGet(col string, value interface{}) (*EventCur, error) {
} }
func (e *EventCur) EventIgnore() error { func (e *EventCur) EventIgnore() error {
_, err := DB["mon"].Exec("update event_cur set ignore_alert=1 where id=?", e.Id) _, err := DB["mon"].Where("id=?", e.Id).Delete(new(EventCur))
return err return err
} }

2
src/modules/judge/cache/index.go vendored
Unescape View File

@ -71,7 +71,7 @@ func (i *IndexMap) Clean() {
} }
for key, series := range index { for key, series := range index {
if now-series.TS > 3600 { if now-series.TS > 300 {
delete(i.Data[id], key) delete(i.Data[id], key)
} }
} }

20
src/modules/judge/cache/linkedlist.go vendored
Unescape View File

@ -54,22 +54,22 @@ func (ll *SafeLinkedList) PushFrontAndMaintain(v *dataobj.JudgeItem, maxCount in
// @param limit 至多返回这些，如果不够，有多少返回多少 // @param limit 至多返回这些，如果不够，有多少返回多少
// @return bool isEnough // @return bool isEnough
func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) { func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.HistoryData, bool) {
if limit < 1 { if limit < 1 {
// 其实limit不合法，此处也返回false吧，上层代码要注意 // 其实limit不合法，此处也返回false吧，上层代码要注意
// 因为false通常使上层代码进入异常分支，这样就统一了 // 因为false通常使上层代码进入异常分支，这样就统一了
return []*dataobj.RRDData{}, false return []*dataobj.HistoryData{}, false
} }
size := ll.Len() size := ll.Len()
if size == 0 { if size == 0 {
return []*dataobj.RRDData{}, false return []*dataobj.HistoryData{}, false
} }
firstElement := ll.Front() firstElement := ll.Front()
firstItem := firstElement.Value.(*dataobj.JudgeItem) firstItem := firstElement.Value.(*dataobj.JudgeItem)
var vs []*dataobj.RRDData var vs []*dataobj.HistoryData
isEnough := true isEnough := true
judgeType := firstItem.DsType[0] judgeType := firstItem.DsType[0]
@ -79,15 +79,21 @@ func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) {
limit = size limit = size
isEnough = false isEnough = false
} }
vs = make([]*dataobj.RRDData, limit) vs = make([]*dataobj.HistoryData, limit)
vs[0] = &dataobj.RRDData{Timestamp: firstItem.Timestamp, Value: dataobj.JsonFloat(firstItem.Value)} vs[0] = &dataobj.HistoryData{
Timestamp: firstItem.Timestamp,
Value: dataobj.JsonFloat(firstItem.Value),
Extra: firstItem.Extra,
}
i := 1 i := 1
currentElement := firstElement currentElement := firstElement
for i < limit { for i < limit {
nextElement := currentElement.Next() nextElement := currentElement.Next()
vs[i] = &dataobj.RRDData{ vs[i] = &dataobj.HistoryData{
Timestamp: nextElement.Value.(*dataobj.JudgeItem).Timestamp, Timestamp: nextElement.Value.(*dataobj.JudgeItem).Timestamp,
Value: dataobj.JsonFloat(nextElement.Value.(*dataobj.JudgeItem).Value), Value: dataobj.JsonFloat(nextElement.Value.(*dataobj.JudgeItem).Value),
Extra: nextElement.Value.(*dataobj.JudgeItem).Extra,
} }
i++ i++
currentElement = nextElement currentElement = nextElement

28
src/modules/judge/judge/func.go
Unescape View File

@ -8,7 +8,7 @@ import (
) )
type Function interface { type Function interface {
Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool)
} }
type MaxFunction struct { type MaxFunction struct {
@ -18,7 +18,7 @@ type MaxFunction struct {
RightValue float64 RightValue float64
} }
func (this MaxFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this MaxFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -42,7 +42,7 @@ type MinFunction struct {
RightValue float64 RightValue float64
} }
func (this MinFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this MinFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -66,7 +66,7 @@ type AllFunction struct {
RightValue float64 RightValue float64
} }
func (this AllFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this AllFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -90,7 +90,7 @@ type SumFunction struct {
RightValue float64 RightValue float64
} }
func (this SumFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this SumFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -112,7 +112,7 @@ type AvgFunction struct {
RightValue float64 RightValue float64
} }
func (this AvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this AvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -135,7 +135,7 @@ type DiffFunction struct {
} }
// 只要有一个点的diff触发阈值，就报警 // 只要有一个点的diff触发阈值，就报警
func (this DiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this DiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -163,7 +163,7 @@ type PDiffFunction struct {
RightValue float64 RightValue float64
} }
func (this PDiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this PDiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -193,7 +193,7 @@ type HappenFunction struct {
RightValue float64 RightValue float64
} }
func (this HappenFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this HappenFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
for n, i := 0, 0; i < len(vs); i++ { for n, i := 0, 0; i < len(vs); i++ {
if checkIsTriggered(vs[i].Value, this.Operator, this.RightValue) { if checkIsTriggered(vs[i].Value, this.Operator, this.RightValue) {
n++ n++
@ -211,7 +211,7 @@ type NodataFunction struct {
Function Function
} }
func (this NodataFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this NodataFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
for _, value := range vs { for _, value := range vs {
if !math.IsNaN(float64(value.Value)) { if !math.IsNaN(float64(value.Value)) {
return value.Value, false return value.Value, false
@ -228,7 +228,7 @@ type CAvgAbsFunction struct {
CompareValue float64 CompareValue float64
} }
func (this CAvgAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this CAvgAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -253,7 +253,7 @@ type CAvgFunction struct {
CompareValue float64 CompareValue float64
} }
func (this CAvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this CAvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -277,7 +277,7 @@ type CAvgRateAbsFunction struct {
CompareValue float64 CompareValue float64
} }
func (this CAvgRateAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this CAvgRateAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }
@ -302,7 +302,7 @@ type CAvgRateFunction struct {
CompareValue float64 CompareValue float64
} }
func (this CAvgRateFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { func (this CAvgRateFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit { if len(vs) < this.Limit {
return return
} }

22
src/modules/judge/judge/judge.go
Unescape View File

@ -62,7 +62,7 @@ func ToJudge(historyMap *cache.JudgeItemMap, key string, val *dataobj.JudgeItem,
Judge(stra, stra.Exprs, historyData, val, now, history, "", "", "", []bool{}) Judge(stra, stra.Exprs, historyData, val, now, history, "", "", "", []bool{})
} }
func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) { func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
stats.Counter.Set("running", 1) stats.Counter.Set("running", 1)
if len(exps) < 1 { if len(exps) < 1 {
@ -86,10 +86,6 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
Granularity: int(firstItem.Step), Granularity: int(firstItem.Step),
Points: historyData, Points: historyData,
} }
if len(history) == 0 {
//只有第一个指标是push的模式，可以获取到extra字段
h.Extra = firstItem.Extra
}
history = append(history, h) history = append(history, h)
defer func() { defer func() {
@ -135,7 +131,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
Tags: "", Tags: "",
DsType: "GAUGE", DsType: "GAUGE",
} }
Judge(stra, exps[1:], []*dataobj.RRDData{}, judgeItem, now, history, info, value, extra, status) Judge(stra, exps[1:], []*dataobj.HistoryData{}, judgeItem, now, history, info, value, extra, status)
return return
} }
@ -143,7 +139,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
firstItem.Endpoint = respData[i].Endpoint firstItem.Endpoint = respData[i].Endpoint
firstItem.Tags = getTags(respData[i].Counter) firstItem.Tags = getTags(respData[i].Counter)
firstItem.Step = respData[i].Step firstItem.Step = respData[i].Step
Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status) Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status)
} }
} else { } else {
@ -162,13 +158,13 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
firstItem.Endpoint = respData[i].Endpoint firstItem.Endpoint = respData[i].Endpoint
firstItem.Tags = getTags(respData[i].Counter) firstItem.Tags = getTags(respData[i].Counter)
firstItem.Step = respData[i].Step firstItem.Step = respData[i].Step
Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status) Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status)
} }
} }
} }
} }
func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.RRDData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) { func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.HistoryData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) {
straFunc := exp.Func straFunc := exp.Func
straParam := []interface{}{} straParam := []interface{}{}
@ -298,7 +294,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
stats.Counter.Set("get.index", 1) stats.Counter.Set("get.index", 1)
indexsData, err := query.Xclude(req) indexsData, err := query.Xclude(req)
if err != nil { if err != nil {
return reqs, err logger.Warning("get index err:", err)
} }
lostSeries := []cache.Series{} lostSeries := []cache.Series{}
@ -345,6 +341,10 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
} }
seriess := cache.SeriesMap.Get(stra.Id) seriess := cache.SeriesMap.Get(stra.Id)
if len(seriess) == 0 && err != nil {
return reqs, err
}
step := 0 step := 0
if len(seriess) > 1 { if len(seriess) > 1 {
step = seriess[0].Step step = seriess[0].Step
@ -389,7 +389,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
return reqs, nil return reqs, nil
} }
func sendEventIfNeed(historyData []*dataobj.RRDData, status []bool, event *dataobj.Event, stra *model.Stra) { func sendEventIfNeed(historyData []*dataobj.HistoryData, status []bool, event *dataobj.Event, stra *model.Stra) {
isTriggered := true isTriggered := true
for _, s := range status { for _, s := range status {
isTriggered = isTriggered && s isTriggered = isTriggered && s

6
src/modules/judge/judge/nodata.go
Unescape View File

@ -51,7 +51,7 @@ func nodataJudge() {
} }
nodataJob.Acquire() nodataJob.Acquire()
go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.RRDData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{}) go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.HistoryData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
} }
return return
} }
@ -80,12 +80,12 @@ func nodataJudge() {
} }
nodataJob.Acquire() nodataJob.Acquire()
go AsyncJudge(nodataJob, stra, stra.Exprs, data.Values, judgeItem, now, []dataobj.History{}, "", "", "", []bool{}) go AsyncJudge(nodataJob, stra, stra.Exprs, dataobj.RRDData2HistoryData(data.Values), judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
} }
} }
} }
func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) { func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
defer sema.Release() defer sema.Release()
Judge(stra, exps, historyData, firstItem, now, history, info, value, extra, status) Judge(stra, exps, historyData, firstItem, now, history, info, value, extra, status)
} }

10
src/modules/monapi/cron/event_consumer.go
Unescape View File

@ -220,10 +220,9 @@ func isInConverge(event *model.Event) bool {
return false return false
} }
// 三种情况，不需要升级报警 // 两种情况，不需要升级报警
// 1，认领的报警不需要升级 // 1，认领的报警不需要升级
// 2，忽略的报警不需要升级 // 2，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注
// 3，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注
func needUpgrade(event *model.Event) bool { func needUpgrade(event *model.Event) bool {
alertUpgradeKey := PrefixAlertUpgrade + fmt.Sprint(event.HashId) alertUpgradeKey := PrefixAlertUpgrade + fmt.Sprint(event.HashId)
eventAlertKey := PrefixAlertTime + fmt.Sprint(event.HashId) eventAlertKey := PrefixAlertTime + fmt.Sprint(event.HashId)
@ -282,11 +281,6 @@ func needUpgrade(event *model.Event) bool {
return false return false
} }
// 告警已经忽略了
if eventCur.IgnoreAlert == 1 {
return false
}
// 告警之后，比如30分钟没有处理，就需要升级，那首先得知道首次告警时间 // 告警之后，比如30分钟没有处理，就需要升级，那首先得知道首次告警时间
if !redisc.HasKey(eventAlertKey) { if !redisc.HasKey(eventAlertKey) {
err := redisc.SetWithTTL(eventAlertKey, now, 30*24*3600) err := redisc.SetWithTTL(eventAlertKey, now, 30*24*3600)

17
src/modules/monapi/http/routes/event.go
Unescape View File

@ -9,6 +9,7 @@ import (
"github.com/toolkits/pkg/errors" "github.com/toolkits/pkg/errors"
"github.com/didi/nightingale/src/dataobj"
"github.com/didi/nightingale/src/model" "github.com/didi/nightingale/src/model"
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
) )
@ -79,9 +80,9 @@ func eventCurGets(c *gin.Context) {
err = json.Unmarshal([]byte(events[i].Detail), &detail) err = json.Unmarshal([]byte(events[i].Detail), &detail)
errors.Dangerous(err) errors.Dangerous(err)
tagsList := []string{} var tags string
for k, v := range detail[0].Tags { if len(detail) > 0 {
tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v)) tags = dataobj.SortedTags(detail[0].Tags)
} }
alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade) alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade)
@ -106,7 +107,7 @@ func eventCurGets(c *gin.Context) {
Etime: events[i].Etime, Etime: events[i].Etime,
Value: events[i].Value, Value: events[i].Value,
Info: events[i].Info, Info: events[i].Info,
Tags: strings.Join(tagsList, ","), Tags: tags,
Created: events[i].Created, Created: events[i].Created,
Nid: events[i].Nid, Nid: events[i].Nid,
Users: users, Users: users,
@ -164,9 +165,9 @@ func eventHisGets(c *gin.Context) {
err = json.Unmarshal([]byte(events[i].Detail), &detail) err = json.Unmarshal([]byte(events[i].Detail), &detail)
errors.Dangerous(err) errors.Dangerous(err)
tagsList := []string{} var tags string
for k, v := range detail[0].Tags { if len(detail) > 0 {
tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v)) tags = dataobj.SortedTags(detail[0].Tags)
} }
alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade) alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade)
@ -191,7 +192,7 @@ func eventHisGets(c *gin.Context) {
Etime: events[i].Etime, Etime: events[i].Etime,
Value: events[i].Value, Value: events[i].Value,
Info: events[i].Info, Info: events[i].Info,
Tags: strings.Join(tagsList, ","), Tags: tags,
Created: events[i].Created, Created: events[i].Created,
Nid: events[i].Nid, Nid: events[i].Nid,
Users: users, Users: users,

Write Preview
Loading…
Cancel
Save