pneymrl2f
/
nightingale
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

add alerts and dashboard templates for Elasticsearch, MongoDB and Linux Process (#904)

Browse Source 
Co-authored-by: yushuangyu <yushuangyu@flashcat.cloud>
master
ysyneu 3 years ago committed by GitHub
parent ac24e8b028
commit 8187334ef6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1419 additions and 0 deletions
Show Stats Download Patch File Download Diff File

392
etc/alerts/elasticsearch_by_exporter.json
View File

File diff suppressed because it is too large Load Diff

242
etc/alerts/mongo_by_exporter.json
Unescape View File

@ -0,0 +1,242 @@
[
{
"name": "Mongo出现Assert错误",
"note": "",
"severity": 3,
"disabled": 0,
"prom_for_duration": 1800,
"prom_ql": "rate(mongodb_asserts_total{type=~\"regular|message\"}[5m]) > 0",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoAssertsDetected"
]
},
{
"name": "Mongo出现游标超时",
"note": "",
"severity": 2,
"disabled": 0,
"prom_for_duration": 1800,
"prom_ql": "rate(mongodb_mongod_metrics_cursor_timed_out_total[5m]) > 0",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoRecurrentCursorTimeout"
]
},
{
"name": "Mongo出现页错误中断",
"note": "",
"severity": 2,
"disabled": 0,
"prom_for_duration": 1800,
"prom_ql": "rate(mongodb_extra_info_page_faults_total[5m]) > 0",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoRecurrentMemoryPageFaults"
]
},
{
"name": "Mongo刚刚有重启，请注意",
"note": "",
"severity": 3,
"disabled": 0,
"prom_for_duration": 0,
"prom_ql": "mongodb_instance_uptime_seconds < 60",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoRestarted"
]
},
{
"name": "Mongo副本集主从延迟超过30s",
"note": "",
"severity": 1,
"disabled": 0,
"prom_for_duration": 60,
"prom_ql": "mongodb_mongod_replset_member_replication_lag > 30",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoSlaveReplicationLag(>30s)"
]
},
{
"name": "Mongo实例挂了",
"note": "",
"severity": 1,
"disabled": 0,
"prom_for_duration": 0,
"prom_ql": "MongoServerDown",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoServerDown"
]
},
{
"name": "Mongo操作平均耗时超过250秒",
"note": "",
"severity": 2,
"disabled": 0,
"prom_for_duration": 600,
"prom_ql": "rate(mongodb_mongod_op_latencies_latency_total[5m]) / rate(mongodb_mongod_op_latencies_ops_total[5m]) > 250000",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoOperationHighLatency"
]
},
{
"name": "Mongo连接数已超过80%",
"note": "",
"severity": 2,
"disabled": 0,
"prom_for_duration": 120,
"prom_ql": "avg by (instance) (mongodb_connections{state=\"current\"}) / avg by (instance) (mongodb_connections{state=\"available\"}) * 100 > 80",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=MongoTooManyConnections(>80%)"
]
}
]

92
etc/alerts/process_by_exporter.json
Unescape View File

@ -0,0 +1,92 @@
[
{
"name": "Process X high number of open files",
"note": "",
"severity": 2,
"disabled": 0,
"prom_for_duration": 60,
"prom_ql": "avg by (instance) (namedprocess_namegroup_worst_fd_ratio{groupname=\"X\"}) * 100 > 80",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=ProcessHighOpenFiles"
]
},
{
"name": "Process X is down",
"note": "",
"severity": 1,
"disabled": 0,
"prom_for_duration": 0,
"prom_ql": "sum by (instance) (namedprocess_namegroup_num_procs{groupname=\"X\"}) == 0",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=ProcessNotRunning"
]
},
{
"name": "Process X is restarted",
"note": "",
"severity": 3,
"disabled": 0,
"prom_for_duration": 0,
"prom_ql": "namedprocess_namegroup_oldest_start_time_seconds{groupname=\"X\"} > time() - 60 ",
"prom_eval_interval": 15,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": [
"1",
"2",
"3",
"4",
"5",
"6",
"0"
],
"enable_in_bg": 0,
"notify_recovered": 1,
"notify_channels": [],
"notify_repeat_step": 60,
"recover_duration": 0,
"callbacks": [],
"runbook_url": "",
"append_tags": [
"alertname=ProcessRestarted"
]
}
]

431
etc/dashboards/elasticsearch_by_exporter.json
View File

File diff suppressed because one or more lines are too long

109
etc/dashboards/mongo_by_exporter.json
View File

File diff suppressed because one or more lines are too long

153
etc/dashboards/process_by_exporter.json
View File

File diff suppressed because one or more lines are too long
Write Preview
Loading…
Cancel
Save