Provide new attributes in health conf files (#10961)
* read and store new attributes (class, component, type) from health conf files. Replace family variable in info strings * provide the attributes to jsons * remove extra semicolon * populate conf files with new attributes * added newline * remove extra defines from health.h * remove empty line * remove realloc * use helper variables for find_and_replace. Adjust position for next strstr * remove comments * Add type to mysql.conf and vcsa.conf * fix formatting * add parenthesis * remove extra assignment * changes to mysql_galera_cluster_state from master * add type Errors to unbound_request_list_overwritten * fix identation for info strings spawning more than one line * check for null, replace with empty string if true * add class, component, type to systemdunits.conf
Emmanuel Vasilakis committed
Apr 20, 2021 at 16:24 UTC
f5bd20e60ae8a65b9c709fcd28d08c1bea268f2d
93 files changed
+4284
-3155
database/rrd.h
+4
@@ -666,6 +666,10 @@ struct alarm_entry {
666
667
char *family;
668
669
+ char *class;
670
+ char *component;
671
+ char *type;
672
+
673
char *exec;
674
char *recipient;
675
time_t exec_run_timestamp;
database/rrdcalc.c
+17
@@ -91,6 +91,9 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
91
rc->name,
92
rc->rrdset->id,
93
rc->rrdset->family,
94
+ rc->class,
95
+ rc->component,
96
+ rc->type,
97
rc->exec,
98
rc->recipient,
99
now - rc->last_status_change,
@@ -165,6 +168,9 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
168
rc->name,
169
rc->rrdset->id,
170
rc->rrdset->family,
171
+ rc->class,
172
+ rc->component,
173
+ rc->type,
174
rc->exec,
175
rc->recipient,
176
now - rc->last_status_change,
@@ -428,6 +434,10 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
434
if(rt->units) rc->units = strdupz(rt->units);
435
if(rt->info) rc->info = strdupz(rt->info);
436
437
+ if (rt->class) rc->class = strdupz(rt->class);
438
+ if (rt->component) rc->component = strdupz(rt->component);
439
+ if (rt->type) rc->type = strdupz(rt->type);
440
+
441
if(rt->calculation) {
442
rc->calculation = expression_parse(rt->calculation->source, NULL, NULL);
443
if(!rc->calculation)
@@ -535,6 +545,10 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch
545
if(rc->units) newrc->units = strdupz(rc->units);
546
if(rc->info) newrc->info = strdupz(rc->info);
547
548
+ if (rc->class) newrc->class = strdupz(rc->class);
549
+ if (rc->component) newrc->component = strdupz(rc->component);
550
+ if (rc->type) newrc->type = strdupz(rc->type);
551
+
552
if(rc->calculation) {
553
newrc->calculation = expression_parse(rc->calculation->source, NULL, NULL);
554
if(!newrc->calculation)
@@ -573,6 +587,9 @@ void rrdcalc_free(RRDCALC *rc) {
587
freez(rc->source);
588
freez(rc->units);
589
freez(rc->info);
590
+ freez(rc->class);
591
+ freez(rc->component);
592
+ freez(rc->type);
593
simple_pattern_free(rc->spdim);
594
freez(rc->labels);
595
simple_pattern_free(rc->splabels);
database/rrdcalc.h
+4
-1
@@ -42,10 +42,13 @@ struct rrdcalc {
42
char *exec; // the command to execute when this alarm switches state
43
char *recipient; // the recipient of the alarm (the first parameter to exec)
44
45
+ char *class; // the class that this alarm belongs
46
+ char *component; // the component that this alarm refers to
47
+ char *type; // type of the alarm
48
+
49
char *chart; // the chart id this should be linked to
50
uint32_t hash_chart;
51
48
-
52
char *plugin_match; //the plugin name that should be linked to
53
SIMPLE_PATTERN *plugin_pattern;
54
database/rrdcalctemplate.h
+4
@@ -15,6 +15,10 @@ struct rrdcalctemplate {
15
char *exec;
16
char *recipient;
17
18
+ char *class;
19
+ char *component;
20
+ char *type;
21
+
22
char *context;
23
uint32_t hash_context;
24
health/health.c
+2
-2
@@ -930,7 +930,7 @@ void *health_main(void *ptr) {
930
if(likely(!rrdcalc_isrepeating(rc))) {
931
ALARM_ENTRY *ae = health_create_alarm_entry(
932
host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,
933
- rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change,
933
+ rc->rrdset->family, rc->class, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
934
rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,
935
rc->delay_last,
936
(
@@ -980,7 +980,7 @@ void *health_main(void *ptr) {
980
rc->last_repeat = now;
981
ALARM_ENTRY *ae = health_create_alarm_entry(
982
host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,
983
- rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change,
983
+ rc->rrdset->family, rc->class, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
984
rc->old_value, rc->value, rc->old_status, rc->status, rc->source, rc->units, rc->info,
985
rc->delay_last,
986
(
health/health.d/adaptec_raid.conf
+24
-18
@@ -1,24 +1,30 @@
1
2
# logical device status check
3
4
-template: adaptec_raid_ld_status
5
- on: adaptec_raid.ld_status
6
- lookup: max -10s foreach *
7
- units: bool
8
- every: 10s
9
- crit: $this > 0
10
- delay: down 5m multiplier 1.5 max 1h
11
- info: logical device status is failed or degraded
12
- to: sysadmin
4
+ template: adaptec_raid_ld_status
5
+ on: adaptec_raid.ld_status
6
+ class: System
7
+component: RAID
8
+ type: Errors
9
+ lookup: max -10s foreach *
10
+ units: bool
11
+ every: 10s
12
+ crit: $this > 0
13
+ delay: down 5m multiplier 1.5 max 1h
14
+ info: logical device status is failed or degraded
15
+ to: sysadmin
16
17
# physical device state check
18
16
-template: adaptec_raid_pd_state
17
- on: adaptec_raid.pd_state
18
- lookup: max -10s foreach *
19
- units: bool
20
- every: 10s
21
- crit: $this > 0
22
- delay: down 5m multiplier 1.5 max 1h
23
- info: physical device state is not online
24
- to: sysadmin
19
+ template: adaptec_raid_pd_state
20
+ on: adaptec_raid.pd_state
21
+ class: System
22
+component: RAID
23
+ type: Errors
24
+ lookup: max -10s foreach *
25
+ units: bool
26
+ every: 10s
27
+ crit: $this > 0
28
+ delay: down 5m multiplier 1.5 max 1h
29
+ info: physical device state is not online
30
+ to: sysadmin
health/health.d/am2320.conf
+13
-10
@@ -1,12 +1,15 @@
1
# make sure am2320 is sending stats
2
3
-template: am2320_last_collected_secs
4
- on: am2320.temperature
5
- calc: $now - $last_collected_t
6
- units: seconds ago
7
- every: 10s
8
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
9
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
10
- delay: down 5m multiplier 1.5 max 1h
11
- info: number of seconds since the last successful data collection
12
- to: webmaster
\ No newline at end of file
3
+ template: am2320_last_collected_secs
4
+ on: am2320.temperature
5
+ class: Other
6
+component: Sensors
7
+ type: Latency
8
+ calc: $now - $last_collected_t
9
+ units: seconds ago
10
+ every: 10s
11
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
12
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
13
+ delay: down 5m multiplier 1.5 max 1h
14
+ info: number of seconds since the last successful data collection
15
+ to: webmaster
health/health.d/anomalies.conf
+18
-12
@@ -1,17 +1,23 @@
1
# raise a warning alarm if an anomaly probability is consistently above 50%
2
3
-template: anomalies_anomaly_probabilities
4
- on: anomalies.probability
5
- lookup: average -2m foreach *
6
- every: 1m
7
- warn: $this > 50
8
- info: average anomaly probability over the last 2 minutes
3
+ template: anomalies_anomaly_probabilities
4
+ on: anomalies.probability
5
+ class: Netdata
6
+component: ML
7
+ type: Errors
8
+ lookup: average -2m foreach *
9
+ every: 1m
10
+ warn: $this > 50
11
+ info: average anomaly probability over the last 2 minutes
12
13
# raise a warning alarm if an anomaly flag is consistently firing
14
12
-template: anomalies_anomaly_flags
13
- on: anomalies.anomaly
14
- lookup: sum -2m foreach *
15
- every: 1m
16
- warn: $this > 10
17
- info: number of anomalies in the last 2 minutes
15
+ template: anomalies_anomaly_flags
16
+ on: anomalies.anomaly
17
+ class: Netdata
18
+component: ML
19
+ type: Errors
20
+ lookup: sum -2m foreach *
21
+ every: 1m
22
+ warn: $this > 10
23
+ info: number of anomalies in the last 2 minutes
health/health.d/apache.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure apache is running
3
4
-template: apache_last_collected_secs
5
- on: apache.requests
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: apache_last_collected_secs
5
+ on: apache.requests
6
+ class: Web Server
7
+component: Apache
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
health/health.d/apcupsd.conf
+43
-34
@@ -1,40 +1,49 @@
1
# you can disable an alarm notification by setting the 'to' line to: silent
2
3
-template: apcupsd_10min_ups_load
4
- on: apcupsd.load
5
- os: *
6
- hosts: *
7
- lookup: average -10m unaligned of percentage
8
- units: %
9
- every: 1m
10
- warn: $this > (($status >= $WARNING) ? (70) : (80))
11
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
12
- delay: down 10m multiplier 1.5 max 1h
13
- info: average UPS load over the last 10 minutes
14
- to: sitemgr
3
+ template: apcupsd_10min_ups_load
4
+ on: apcupsd.load
5
+ class: Power Supply
6
+component: UPS
7
+ type: Utilization
8
+ os: *
9
+ hosts: *
10
+ lookup: average -10m unaligned of percentage
11
+ units: %
12
+ every: 1m
13
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
14
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
15
+ delay: down 10m multiplier 1.5 max 1h
16
+ info: average UPS load over the last 10 minutes
17
+ to: sitemgr
18
19
# Discussion in https://github.com/netdata/netdata/pull/3928:
20
# Fire the alarm as soon as it's going on battery (99% charge) and clear only when full.
18
-template: apcupsd_ups_charge
19
- on: apcupsd.charge
20
- os: *
21
- hosts: *
22
- lookup: average -60s unaligned of charge
23
- units: %
24
- every: 60s
25
- warn: $this < 100
26
- crit: $this < (($status == $CRITICAL) ? (60) : (50))
27
- delay: down 10m multiplier 1.5 max 1h
28
- info: average UPS charge over the last minute
29
- to: sitemgr
21
+ template: apcupsd_ups_charge
22
+ on: apcupsd.charge
23
+ class: Power Supply
24
+component: UPS
25
+ type: Errors
26
+ os: *
27
+ hosts: *
28
+ lookup: average -60s unaligned of charge
29
+ units: %
30
+ every: 60s
31
+ warn: $this < 100
32
+ crit: $this < (($status == $CRITICAL) ? (60) : (50))
33
+ delay: down 10m multiplier 1.5 max 1h
34
+ info: average UPS charge over the last minute
35
+ to: sitemgr
36
31
-template: apcupsd_last_collected_secs
32
- on: apcupsd.load
33
- calc: $now - $last_collected_t
34
- every: 10s
35
- units: seconds ago
36
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
37
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
38
- delay: down 5m multiplier 1.5 max 1h
39
- info: number of seconds since the last successful data collection
40
- to: sitemgr
37
+ template: apcupsd_last_collected_secs
38
+ on: apcupsd.load
39
+ class: Power Supply
40
+component: UPS device
41
+ type: Latency
42
+ calc: $now - $last_collected_t
43
+ every: 10s
44
+ units: seconds ago
45
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
46
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
47
+ delay: down 5m multiplier 1.5 max 1h
48
+ info: number of seconds since the last successful data collection
49
+ to: sitemgr
health/health.d/backend.conf
+37
-28
@@ -1,33 +1,42 @@
1
# Alert that backends subsystem will be disabled soon
2
- alarm: backend_metrics_eol
3
- on: netdata.backend_metrics
4
- units: boolean
5
- calc: $now - $last_collected_t
6
- every: 1m
7
- warn: $this > 0
8
- delay: down 5m multiplier 1.5 max 1h
9
- info: the backends subsystem is deprecated and will be removed soon. Migrate your configuration to exporting.conf.
10
- to: sysadmin
2
+ alarm: backend_metrics_eol
3
+ on: netdata.backend_metrics
4
+ class: Netdata
5
+component: Exporting engine
6
+ type: Errors
7
+ units: boolean
8
+ calc: $now - $last_collected_t
9
+ every: 1m
10
+ warn: $this > 0
11
+ delay: down 5m multiplier 1.5 max 1h
12
+ info: the backends subsystem is deprecated and will be removed soon. Migrate your configuration to exporting.conf.
13
+ to: sysadmin
14
15
# make sure we are sending data to backend
16
14
- alarm: backend_last_buffering
15
- on: netdata.backend_metrics
16
- calc: $now - $last_collected_t
17
- units: seconds ago
18
- every: 10s
19
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
20
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
21
- delay: down 5m multiplier 1.5 max 1h
22
- info: number of seconds since the last successful buffering of backend data
23
- to: dba
17
+ alarm: backend_last_buffering
18
+ on: netdata.backend_metrics
19
+ class: Netdata
20
+component: Exporting engine
21
+ type: Latency
22
+ calc: $now - $last_collected_t
23
+ units: seconds ago
24
+ every: 10s
25
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
26
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
27
+ delay: down 5m multiplier 1.5 max 1h
28
+ info: number of seconds since the last successful buffering of backend data
29
+ to: dba
30
25
- alarm: backend_metrics_sent
26
- on: netdata.backend_metrics
27
- units: %
28
- calc: abs($sent) * 100 / abs($buffered)
29
- every: 10s
30
- warn: $this != 100
31
- delay: down 5m multiplier 1.5 max 1h
32
- info: percentage of metrics sent to the backend server
33
- to: dba
31
+ alarm: backend_metrics_sent
32
+ on: netdata.backend_metrics
33
+ class: Netdata
34
+component: Exporting engine
35
+ type: Workload
36
+ units: %
37
+ calc: abs($sent) * 100 / abs($buffered)
38
+ every: 10s
39
+ warn: $this != 100
40
+ delay: down 5m multiplier 1.5 max 1h
41
+ info: percentage of metrics sent to the backend server
42
+ to: dba
health/health.d/bcache.conf
+28
-22
@@ -1,24 +1,30 @@
1
2
-template: bcache_cache_errors
3
- on: disk.bcache_cache_read_races
4
- lookup: sum -1m unaligned absolute
5
- units: errors
6
- every: 1m
7
- warn: $this > 0
8
- delay: up 2m down 1h multiplier 1.5 max 2h
9
- info: number of times data was read from the cache, \
10
- the bucket was reused and invalidated in the last 10 minutes \
11
- (when this occurs the data is reread from the backing device)
12
- to: sysadmin
2
+ template: bcache_cache_errors
3
+ on: disk.bcache_cache_read_races
4
+ class: System
5
+component: Disk
6
+ type: Errors
7
+ lookup: sum -1m unaligned absolute
8
+ units: errors
9
+ every: 1m
10
+ warn: $this > 0
11
+ delay: up 2m down 1h multiplier 1.5 max 2h
12
+ info: number of times data was read from the cache, \
13
+ the bucket was reused and invalidated in the last 10 minutes \
14
+ (when this occurs the data is reread from the backing device)
15
+ to: sysadmin
16
14
-template: bcache_cache_dirty
15
- on: disk.bcache_cache_alloc
16
- calc: $dirty + $metadata + $undefined
17
- units: %
18
- every: 1m
19
- warn: $this > ( ($status >= $WARNING ) ? ( 70 ) : ( 90 ) )
20
- crit: $this > ( ($status == $CRITICAL) ? ( 90 ) : ( 95 ) )
21
- delay: up 1m down 1h multiplier 1.5 max 2h
22
- info: percentage of cache space used for dirty data and metadata \
23
- (this usually means your SSD cache is too small)
24
- to: sysadmin
17
+ template: bcache_cache_dirty
18
+ on: disk.bcache_cache_alloc
19
+ class: System
20
+component: Disk
21
+ type: Utilization
22
+ calc: $dirty + $metadata + $undefined
23
+ units: %
24
+ every: 1m
25
+ warn: $this > ( ($status >= $WARNING ) ? ( 70 ) : ( 90 ) )
26
+ crit: $this > ( ($status == $CRITICAL) ? ( 90 ) : ( 95 ) )
27
+ delay: up 1m down 1h multiplier 1.5 max 2h
28
+ info: percentage of cache space used for dirty data and metadata \
29
+ (this usually means your SSD cache is too small)
30
+ to: sysadmin
health/health.d/beanstalkd.conf
+15
-12
@@ -1,17 +1,20 @@
1
# get the number of buried jobs in all queues
2
3
-template: beanstalk_server_buried_jobs
4
- on: beanstalk.current_jobs
5
- calc: $buried
6
- units: jobs
7
- every: 10s
8
- warn: $this > 0
9
- crit: $this > 10
10
- delay: up 0 down 5m multiplier 1.2 max 1h
11
- info: number of buried jobs across all tubes. \
12
- You need to manually kick them so they can be processed. \
13
- Presence of buried jobs in a tube does not affect new jobs.
14
- to: sysadmin
3
+ template: beanstalk_server_buried_jobs
4
+ on: beanstalk.current_jobs
5
+ class: Messaging
6
+component: Beanstalk
7
+ type: Workload
8
+ calc: $buried
9
+ units: jobs
10
+ every: 10s
11
+ warn: $this > 0
12
+ crit: $this > 10
13
+ delay: up 0 down 5m multiplier 1.2 max 1h
14
+ info: number of buried jobs across all tubes. \
15
+ You need to manually kick them so they can be processed. \
16
+ Presence of buried jobs in a tube does not affect new jobs.
17
+ to: sysadmin
18
19
# get the number of buried jobs per queue
20
health/health.d/bind_rndc.conf
+12
-9
@@ -1,9 +1,12 @@
1
-template: bind_rndc_stats_file_size
2
- on: bind_rndc.stats_size
3
- units: megabytes
4
- every: 60
5
- calc: $stats_size
6
- warn: $this > 512
7
- crit: $this > 1024
8
- info: BIND statistics-file size
9
- to: sysadmin
1
+ template: bind_rndc_stats_file_size
2
+ on: bind_rndc.stats_size
3
+ class: DNS
4
+component: BIND
5
+ type: Utilization
6
+ units: megabytes
7
+ every: 60
8
+ calc: $stats_size
9
+ warn: $this > 512
10
+ crit: $this > 1024
11
+ info: BIND statistics-file size
12
+ to: sysadmin
health/health.d/boinc.conf
+65
-53
@@ -1,62 +1,74 @@
1
# Alarms for various BOINC issues.
2
3
# Warn on any compute errors encountered.
4
-template: boinc_compute_errors
5
- on: boinc.states
6
- os: *
7
- hosts: *
8
-families: *
9
- lookup: average -10m unaligned of comperror
10
- units: tasks
11
- every: 1m
12
- warn: $this > 0
13
- crit: $this > 1
14
- delay: up 1m down 5m multiplier 1.5 max 1h
15
- info: average number of compute errors over the last 10 minutes
16
- to: sysadmin
4
+ template: boinc_compute_errors
5
+ on: boinc.states
6
+ class: Computing
7
+component: BOINC
8
+ type: Errors
9
+ os: *
10
+ hosts: *
11
+ families: *
12
+ lookup: average -10m unaligned of comperror
13
+ units: tasks
14
+ every: 1m
15
+ warn: $this > 0
16
+ crit: $this > 1
17
+ delay: up 1m down 5m multiplier 1.5 max 1h
18
+ info: average number of compute errors over the last 10 minutes
19
+ to: sysadmin
20
21
# Warn on lots of upload errors
19
-template: boinc_upload_errors
20
- on: boinc.states
21
- os: *
22
- hosts: *
23
-families: *
24
- lookup: average -10m unaligned of upload_failed
25
- units: tasks
26
- every: 1m
27
- warn: $this > 0
28
- crit: $this > 1
29
- delay: up 1m down 5m multiplier 1.5 max 1h
30
- info: average number of failed uploads over the last 10 minutes
31
- to: sysadmin
22
+ template: boinc_upload_errors
23
+ on: boinc.states
24
+ class: Computing
25
+component: BOINC
26
+ type: Errors
27
+ os: *
28
+ hosts: *
29
+ families: *
30
+ lookup: average -10m unaligned of upload_failed
31
+ units: tasks
32
+ every: 1m
33
+ warn: $this > 0
34
+ crit: $this > 1
35
+ delay: up 1m down 5m multiplier 1.5 max 1h
36
+ info: average number of failed uploads over the last 10 minutes
37
+ to: sysadmin
38
39
# Warn on the task queue being empty
34
-template: boinc_total_tasks
35
- on: boinc.tasks
36
- os: *
37
- hosts: *
38
-families: *
39
- lookup: average -10m unaligned of total
40
- units: tasks
41
- every: 1m
42
- warn: $this < 1
43
- crit: $this < 0.1
44
- delay: up 5m down 10m multiplier 1.5 max 1h
45
- info: average number of total tasks over the last 10 minutes
46
- to: sysadmin
40
+ template: boinc_total_tasks
41
+ on: boinc.tasks
42
+ class: Computing
43
+component: BOINC
44
+ type: Utilization
45
+ os: *
46
+ hosts: *
47
+ families: *
48
+ lookup: average -10m unaligned of total
49
+ units: tasks
50
+ every: 1m
51
+ warn: $this < 1
52
+ crit: $this < 0.1
53
+ delay: up 5m down 10m multiplier 1.5 max 1h
54
+ info: average number of total tasks over the last 10 minutes
55
+ to: sysadmin
56
57
# Warn on no active tasks with a non-empty queue
49
-template: boinc_active_tasks
50
- on: boinc.tasks
51
- os: *
52
- hosts: *
53
-families: *
54
- lookup: average -10m unaligned of active
55
- calc: ($boinc_total_tasks >= 1) ? ($this) : (inf)
56
- units: tasks
57
- every: 1m
58
- warn: $this < 1
59
- crit: $this < 0.1
60
- delay: up 5m down 10m multiplier 1.5 max 1h
61
- info: average number of active tasks over the last 10 minutes
62
- to: sysadmin
58
+ template: boinc_active_tasks
59
+ on: boinc.tasks
60
+ class: Computing
61
+component: BOINC
62
+ type: Utilization
63
+ os: *
64
+ hosts: *
65
+ families: *
66
+ lookup: average -10m unaligned of active
67
+ calc: ($boinc_total_tasks >= 1) ? ($this) : (inf)
68
+ units: tasks
69
+ every: 1m
70
+ warn: $this < 1
71
+ crit: $this < 0.1
72
+ delay: up 5m down 10m multiplier 1.5 max 1h
73
+ info: average number of active tasks over the last 10 minutes
74
+ to: sysadmin
health/health.d/btrfs.conf
+64
-52
@@ -1,56 +1,68 @@
1
2
-template: btrfs_allocated
3
- on: btrfs.disk
4
- os: *
5
- hosts: *
6
-families: *
7
- calc: 100 - ($unallocated * 100 / ($unallocated + $data_used + $data_free + $meta_used + $meta_free + $sys_used + $sys_free))
8
- units: %
9
- every: 10s
10
- warn: $this > (($status >= $WARNING) ? (90) : (95))
11
- crit: $this > (($status == $CRITICAL) ? (95) : (98))
12
- delay: up 1m down 15m multiplier 1.5 max 1h
13
- info: percentage of allocated BTRFS physical disk space
14
- to: sysadmin
2
+ template: btrfs_allocated
3
+ on: btrfs.disk
4
+ class: System
5
+component: File system
6
+ type: Utilization
7
+ os: *
8
+ hosts: *
9
+ families: *
10
+ calc: 100 - ($unallocated * 100 / ($unallocated + $data_used + $data_free + $meta_used + $meta_free + $sys_used + $sys_free))
11
+ units: %
12
+ every: 10s
13
+ warn: $this > (($status >= $WARNING) ? (90) : (95))
14
+ crit: $this > (($status == $CRITICAL) ? (95) : (98))
15
+ delay: up 1m down 15m multiplier 1.5 max 1h
16
+ info: percentage of allocated BTRFS physical disk space
17
+ to: sysadmin
18
16
-template: btrfs_data
17
- on: btrfs.data
18
- os: *
19
- hosts: *
20
-families: *
21
- calc: $used * 100 / ($used + $free)
22
- units: %
23
- every: 10s
24
- warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
25
- crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
26
- delay: up 1m down 15m multiplier 1.5 max 1h
27
- info: utilization of BTRFS data space
28
- to: sysadmin
19
+ template: btrfs_data
20
+ on: btrfs.data
21
+ class: System
22
+component: File system
23
+ type: Utilization
24
+ os: *
25
+ hosts: *
26
+ families: *
27
+ calc: $used * 100 / ($used + $free)
28
+ units: %
29
+ every: 10s
30
+ warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
31
+ crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
32
+ delay: up 1m down 15m multiplier 1.5 max 1h
33
+ info: utilization of BTRFS data space
34
+ to: sysadmin
35
30
-template: btrfs_metadata
31
- on: btrfs.metadata
32
- os: *
33
- hosts: *
34
-families: *
35
- calc: ($used + $reserved) * 100 / ($used + $free + $reserved)
36
- units: %
37
- every: 10s
38
- warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
39
- crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
40
- delay: up 1m down 15m multiplier 1.5 max 1h
41
- info: utilization of BTRFS metadata space
42
- to: sysadmin
36
+ template: btrfs_metadata
37
+ on: btrfs.metadata
38
+ class: System
39
+component: File system
40
+ type: Utilization
41
+ os: *
42
+ hosts: *
43
+ families: *
44
+ calc: ($used + $reserved) * 100 / ($used + $free + $reserved)
45
+ units: %
46
+ every: 10s
47
+ warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
48
+ crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
49
+ delay: up 1m down 15m multiplier 1.5 max 1h
50
+ info: utilization of BTRFS metadata space
51
+ to: sysadmin
52
44
-template: btrfs_system
45
- on: btrfs.system
46
- os: *
47
- hosts: *
48
-families: *
49
- calc: $used * 100 / ($used + $free)
50
- units: %
51
- every: 10s
52
- warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
53
- crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
54
- delay: up 1m down 15m multiplier 1.5 max 1h
55
- info: utilization of BTRFS system space
56
- to: sysadmin
53
+ template: btrfs_system
54
+ on: btrfs.system
55
+ class: System
56
+component: File system
57
+ type: Utilization
58
+ os: *
59
+ hosts: *
60
+ families: *
61
+ calc: $used * 100 / ($used + $free)
62
+ units: %
63
+ every: 10s
64
+ warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
65
+ crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
66
+ delay: up 1m down 15m multiplier 1.5 max 1h
67
+ info: utilization of BTRFS system space
68
+ to: sysadmin
health/health.d/ceph.conf
+13
-10
@@ -1,12 +1,15 @@
1
# low ceph disk available
2
3
-template: ceph_cluster_space_usage
4
- on: ceph.general_usage
5
- calc: $used * 100 / ($used + $avail)
6
- units: %
7
- every: 1m
8
- warn: $this > (($status >= $WARNING ) ? (85) : (90))
9
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
10
- delay: down 5m multiplier 1.2 max 1h
11
- info: cluster disk space utilization
12
- to: sysadmin
3
+ template: ceph_cluster_space_usage
4
+ on: ceph.general_usage
5
+ class: Storage
6
+component: Ceph
7
+ type: Utilization
8
+ calc: $used * 100 / ($used + $avail)
9
+ units: %
10
+ every: 1m
11
+ warn: $this > (($status >= $WARNING ) ? (85) : (90))
12
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
13
+ delay: down 5m multiplier 1.2 max 1h
14
+ info: cluster disk space utilization
15
+ to: sysadmin
health/health.d/cgroups.conf
+30
-24
@@ -1,28 +1,34 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
-template: cgroup_10min_cpu_usage
5
- on: cgroup.cpu_limit
6
- os: linux
7
- hosts: *
8
- lookup: average -10m unaligned
9
- units: %
10
- every: 1m
11
- warn: $this > (($status >= $WARNING) ? (75) : (85))
12
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
13
- delay: down 15m multiplier 1.5 max 1h
14
- info: average cgroup CPU utilization over the last 10 minutes
15
- to: sysadmin
4
+ template: cgroup_10min_cpu_usage
5
+ on: cgroup.cpu_limit
6
+ class: Cgroups
7
+component: CPU
8
+ type: Utilization
9
+ os: linux
10
+ hosts: *
11
+ lookup: average -10m unaligned
12
+ units: %
13
+ every: 1m
14
+ warn: $this > (($status >= $WARNING) ? (75) : (85))
15
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
16
+ delay: down 15m multiplier 1.5 max 1h
17
+ info: average cgroup CPU utilization over the last 10 minutes
18
+ to: sysadmin
19
17
-template: cgroup_ram_in_use
18
- on: cgroup.mem_usage
19
- os: linux
20
- hosts: *
21
- calc: ($ram) * 100 / $memory_limit
22
- units: %
23
- every: 10s
24
- warn: $this > (($status >= $WARNING) ? (80) : (90))
25
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
26
- delay: down 15m multiplier 1.5 max 1h
27
- info: cgroup memory utilization
28
- to: sysadmin
20
+ template: cgroup_ram_in_use
21
+ on: cgroup.mem_usage
22
+ class: Cgroups
23
+component: Memory
24
+ type: Utilization
25
+ os: linux
26
+ hosts: *
27
+ calc: ($ram) * 100 / $memory_limit
28
+ units: %
29
+ every: 10s
30
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
31
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
32
+ delay: down 15m multiplier 1.5 max 1h
33
+ info: cgroup memory utilization
34
+ to: sysadmin
health/health.d/cockroachdb.conf
+97
-73
@@ -1,91 +1,115 @@
1
2
# Availability
3
4
-template: cockroachdb_last_collected_secs
5
- on: cockroachdb.live_nodes
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: cockroachdb_last_collected_secs
5
+ on: cockroachdb.live_nodes
6
+ class: Database
7
+component: CockroachDB
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
17
18
# Capacity
19
17
-template: cockroachdb_used_storage_capacity
18
- on: cockroachdb.storage_used_capacity_percentage
19
- calc: $capacity_used_percent
20
- units: %
21
- every: 10s
22
- warn: $this > (($status >= $WARNING) ? (80) : (85))
23
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
24
- delay: down 15m multiplier 1.5 max 1h
25
- info: storage capacity utilization
26
- to: dba
20
+ template: cockroachdb_used_storage_capacity
21
+ on: cockroachdb.storage_used_capacity_percentage
22
+ class: Database
23
+component: CockroachDB
24
+ type: Utilization
25
+ calc: $capacity_used_percent
26
+ units: %
27
+ every: 10s
28
+ warn: $this > (($status >= $WARNING) ? (80) : (85))
29
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
30
+ delay: down 15m multiplier 1.5 max 1h
31
+ info: storage capacity utilization
32
+ to: dba
33
28
-template: cockroachdb_used_usable_storage_capacity
29
- on: cockroachdb.storage_used_capacity_percentage
30
- calc: $capacity_usable_used_percent
31
- units: %
32
- every: 10s
33
- warn: $this > (($status >= $WARNING) ? (80) : (85))
34
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
35
- delay: down 15m multiplier 1.5 max 1h
36
- info: storage usable space utilization
37
- to: dba
34
+ template: cockroachdb_used_usable_storage_capacity
35
+ on: cockroachdb.storage_used_capacity_percentage
36
+ class: Database
37
+component: CockroachDB
38
+ type: Utilization
39
+ calc: $capacity_usable_used_percent
40
+ units: %
41
+ every: 10s
42
+ warn: $this > (($status >= $WARNING) ? (80) : (85))
43
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
44
+ delay: down 15m multiplier 1.5 max 1h
45
+ info: storage usable space utilization
46
+ to: dba
47
48
# Replication
49
41
-template: cockroachdb_unavailable_ranges
42
- on: cockroachdb.ranges_replication_problem
43
- calc: $ranges_unavailable
44
- units: num
45
- every: 10s
46
- warn: $this > 0
47
- delay: down 15m multiplier 1.5 max 1h
48
- info: number of ranges with fewer live replicas than the replication target
49
- to: dba
50
+ template: cockroachdb_unavailable_ranges
51
+ on: cockroachdb.ranges_replication_problem
52
+ class: Database
53
+component: CockroachDB
54
+ type: Utilization
55
+ calc: $ranges_unavailable
56
+ units: num
57
+ every: 10s
58
+ warn: $this > 0
59
+ delay: down 15m multiplier 1.5 max 1h
60
+ info: number of ranges with fewer live replicas than the replication target
61
+ to: dba
62
51
-template: cockroachdb_replicas_leaders_not_leaseholders
52
- on: cockroachdb.replicas_leaders
53
- calc: $replicas_leaders_not_leaseholders
54
- units: num
55
- every: 10s
56
- warn: $this > 0
57
- delay: down 15m multiplier 1.5 max 1h
58
- info: number of replicas that are Raft leaders whose range lease is held by another store
59
- to: dba
63
+ template: cockroachdb_replicas_leaders_not_leaseholders
64
+ on: cockroachdb.replicas_leaders
65
+ class: Database
66
+component: CockroachDB
67
+ type: Utilization
68
+ calc: $replicas_leaders_not_leaseholders
69
+ units: num
70
+ every: 10s
71
+ warn: $this > 0
72
+ delay: down 15m multiplier 1.5 max 1h
73
+ info: number of replicas that are Raft leaders whose range lease is held by another store
74
+ to: dba
75
76
# FD
77
63
-template: cockroachdb_open_file_descriptors_limit
64
- on: cockroachdb.process_file_descriptors
65
- calc: $sys_fd_open/$sys_fd_softlimit * 100
66
- units: %
67
- every: 10s
68
- warn: $this > 80
69
- delay: down 15m multiplier 1.5 max 1h
70
- info: open file descriptors utilization (against softlimit)
71
- to: dba
78
+ template: cockroachdb_open_file_descriptors_limit
79
+ on: cockroachdb.process_file_descriptors
80
+ class: Database
81
+component: CockroachDB
82
+ type: Utilization
83
+ calc: $sys_fd_open/$sys_fd_softlimit * 100
84
+ units: %
85
+ every: 10s
86
+ warn: $this > 80
87
+ delay: down 15m multiplier 1.5 max 1h
88
+ info: open file descriptors utilization (against softlimit)
89
+ to: dba
90
91
# SQL
92
75
-template: cockroachdb_sql_active_connections
76
- on: cockroachdb.sql_connections
77
- calc: $sql_conns
78
- units: active connections
79
- every: 10s
80
- info: number of active SQL connections
81
- to: dba
93
+ template: cockroachdb_sql_active_connections
94
+ on: cockroachdb.sql_connections
95
+ class: Database
96
+component: CockroachDB
97
+ type: Utilization
98
+ calc: $sql_conns
99
+ units: active connections
100
+ every: 10s
101
+ info: number of active SQL connections
102
+ to: dba
103
83
-template: cockroachdb_sql_executed_statements_total_last_5m
84
- on: cockroachdb.sql_statements_total
85
- lookup: sum -5m absolute of sql_query_count
86
- units: statements
87
- every: 10s
88
- warn: $this == 0 AND $cockroachdb_sql_active_connections != 0
89
- delay: down 15m up 30s multiplier 1.5 max 1h
90
- info: number of executed SQL statements in the last 5 minutes
91
- to: dba
104
+ template: cockroachdb_sql_executed_statements_total_last_5m
105
+ on: cockroachdb.sql_statements_total
106
+ class: Database
107
+component: CockroachDB
108
+ type: Workload
109
+ lookup: sum -5m absolute of sql_query_count
110
+ units: statements
111
+ every: 10s
112
+ warn: $this == 0 AND $cockroachdb_sql_active_connections != 0
113
+ delay: down 15m up 30s multiplier 1.5 max 1h
114
+ info: number of executed SQL statements in the last 5 minutes
115
+ to: dba
health/health.d/couchdb.conf
+13
-10
@@ -1,13 +1,16 @@
1
2
# make sure couchdb is running
3
4
-template: couchdb_last_collected_secs
5
- on: couchdb.request_methods
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: couchdb_last_collected_secs
5
+ on: couchdb.request_methods
6
+ class: Database
7
+component: CouchDB
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
health/health.d/cpu.conf
+60
-48
@@ -1,55 +1,67 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
-template: 10min_cpu_usage
5
- on: system.cpu
6
- os: linux
7
- hosts: *
8
- lookup: average -10m unaligned of user,system,softirq,irq,guest
9
- units: %
10
- every: 1m
11
- warn: $this > (($status >= $WARNING) ? (75) : (85))
12
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
13
- delay: down 15m multiplier 1.5 max 1h
14
- info: average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
15
- to: sysadmin
4
+ template: 10min_cpu_usage
5
+ on: system.cpu
6
+ class: System
7
+component: CPU
8
+ type: Utilization
9
+ os: linux
10
+ hosts: *
11
+ lookup: average -10m unaligned of user,system,softirq,irq,guest
12
+ units: %
13
+ every: 1m
14
+ warn: $this > (($status >= $WARNING) ? (75) : (85))
15
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
16
+ delay: down 15m multiplier 1.5 max 1h
17
+ info: average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
18
+ to: sysadmin
19
17
-template: 10min_cpu_iowait
18
- on: system.cpu
19
- os: linux
20
- hosts: *
21
- lookup: average -10m unaligned of iowait
22
- units: %
23
- every: 1m
24
- warn: $this > (($status >= $WARNING) ? (20) : (40))
25
- crit: $this > (($status == $CRITICAL) ? (40) : (50))
26
- delay: down 15m multiplier 1.5 max 1h
27
- info: average CPU iowait time over the last 10 minutes
28
- to: sysadmin
20
+ template: 10min_cpu_iowait
21
+ on: system.cpu
22
+ class: System
23
+component: CPU
24
+ type: Utilization
25
+ os: linux
26
+ hosts: *
27
+ lookup: average -10m unaligned of iowait
28
+ units: %
29
+ every: 1m
30
+ warn: $this > (($status >= $WARNING) ? (20) : (40))
31
+ crit: $this > (($status == $CRITICAL) ? (40) : (50))
32
+ delay: down 15m multiplier 1.5 max 1h
33
+ info: average CPU iowait time over the last 10 minutes
34
+ to: sysadmin
35
30
-template: 20min_steal_cpu
31
- on: system.cpu
32
- os: linux
33
- hosts: *
34
- lookup: average -20m unaligned of steal
35
- units: %
36
- every: 5m
37
- warn: $this > (($status >= $WARNING) ? (5) : (10))
38
- crit: $this > (($status == $CRITICAL) ? (20) : (30))
39
- delay: down 1h multiplier 1.5 max 2h
40
- info: average CPU steal time over the last 20 minutes
41
- to: sysadmin
36
+ template: 20min_steal_cpu
37
+ on: system.cpu
38
+ class: System
39
+component: CPU
40
+ type: Latency
41
+ os: linux
42
+ hosts: *
43
+ lookup: average -20m unaligned of steal
44
+ units: %
45
+ every: 5m
46
+ warn: $this > (($status >= $WARNING) ? (5) : (10))
47
+ crit: $this > (($status == $CRITICAL) ? (20) : (30))
48
+ delay: down 1h multiplier 1.5 max 2h
49
+ info: average CPU steal time over the last 20 minutes
50
+ to: sysadmin
51
52
## FreeBSD
44
-template: 10min_cpu_usage
45
- on: system.cpu
46
- os: freebsd
47
- hosts: *
48
- lookup: average -10m unaligned of user,system,interrupt
49
- units: %
50
- every: 1m
51
- warn: $this > (($status >= $WARNING) ? (75) : (85))
52
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
53
- delay: down 15m multiplier 1.5 max 1h
54
- info: average CPU utilization over the last 10 minutes (excluding nice)
55
- to: sysadmin
53
+ template: 10min_cpu_usage
54
+ on: system.cpu
55
+ class: System
56
+component: CPU
57
+ type: Utilization
58
+ os: freebsd
59
+ hosts: *
60
+ lookup: average -10m unaligned of user,system,interrupt
61
+ units: %
62
+ every: 1m
63
+ warn: $this > (($status >= $WARNING) ? (75) : (85))
64
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
65
+ delay: down 15m multiplier 1.5 max 1h
66
+ info: average CPU utilization over the last 10 minutes (excluding nice)
67
+ to: sysadmin
health/health.d/dbengine.conf
+58
-46
@@ -1,52 +1,64 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: 10min_dbengine_global_fs_errors
5
- on: netdata.dbengine_global_errors
6
- os: linux freebsd macos
7
- hosts: *
8
-lookup: sum -10m unaligned of fs_errors
9
- units: errors
10
- every: 10s
11
- crit: $this > 0
12
- delay: down 15m multiplier 1.5 max 1h
13
- info: number of filesystem errors in the last 10 minutes (too many open files, wrong permissions, etc)
14
- to: sysadmin
4
+ alarm: 10min_dbengine_global_fs_errors
5
+ on: netdata.dbengine_global_errors
6
+ class: Netdata
7
+component: DB engine
8
+ type: Errors
9
+ os: linux freebsd macos
10
+ hosts: *
11
+ lookup: sum -10m unaligned of fs_errors
12
+ units: errors
13
+ every: 10s
14
+ crit: $this > 0
15
+ delay: down 15m multiplier 1.5 max 1h
16
+ info: number of filesystem errors in the last 10 minutes (too many open files, wrong permissions, etc)
17
+ to: sysadmin
18
16
- alarm: 10min_dbengine_global_io_errors
17
- on: netdata.dbengine_global_errors
18
- os: linux freebsd macos
19
- hosts: *
20
-lookup: sum -10m unaligned of io_errors
21
- units: errors
22
- every: 10s
23
- crit: $this > 0
24
- delay: down 1h multiplier 1.5 max 3h
25
- info: number of IO errors in the last 10 minutes (CRC errors, out of space, bad disk, etc)
26
- to: sysadmin
19
+ alarm: 10min_dbengine_global_io_errors
20
+ on: netdata.dbengine_global_errors
21
+ class: Netdata
22
+component: DB engine
23
+ type: Errors
24
+ os: linux freebsd macos
25
+ hosts: *
26
+ lookup: sum -10m unaligned of io_errors
27
+ units: errors
28
+ every: 10s
29
+ crit: $this > 0
30
+ delay: down 1h multiplier 1.5 max 3h
31
+ info: number of IO errors in the last 10 minutes (CRC errors, out of space, bad disk, etc)
32
+ to: sysadmin
33
28
- alarm: 10min_dbengine_global_flushing_warnings
29
- on: netdata.dbengine_global_errors
30
- os: linux freebsd macos
31
- hosts: *
32
-lookup: sum -10m unaligned of pg_cache_over_half_dirty_events
33
- units: errors
34
- every: 10s
35
- warn: $this > 0
36
- delay: down 1h multiplier 1.5 max 3h
37
- info: number of times when dbengine dirty pages were over 50% of the instance's page cache in the last 10 minutes. \
38
- Metric data are at risk of not being stored in the database. To remedy, reduce disk load or use faster disks.
39
- to: sysadmin
34
+ alarm: 10min_dbengine_global_flushing_warnings
35
+ on: netdata.dbengine_global_errors
36
+ class: Netdata
37
+component: DB engine
38
+ type: Errors
39
+ os: linux freebsd macos
40
+ hosts: *
41
+ lookup: sum -10m unaligned of pg_cache_over_half_dirty_events
42
+ units: errors
43
+ every: 10s
44
+ warn: $this > 0
45
+ delay: down 1h multiplier 1.5 max 3h
46
+ info: number of times when dbengine dirty pages were over 50% of the instance's page cache in the last 10 minutes. \
47
+ Metric data are at risk of not being stored in the database. To remedy, reduce disk load or use faster disks.
48
+ to: sysadmin
49
41
- alarm: 10min_dbengine_global_flushing_errors
42
- on: netdata.dbengine_long_term_page_stats
43
- os: linux freebsd macos
44
- hosts: *
45
-lookup: sum -10m unaligned of flushing_pressure_deletions
46
- units: pages
47
- every: 10s
48
- crit: $this != 0
49
- delay: down 1h multiplier 1.5 max 3h
50
- info: number of pages deleted due to failure to flush data to disk in the last 10 minutes. \
51
- Metric data were lost to unblock data collection. To fix, reduce disk load or use faster disks.
52
- to: sysadmin
50
+ alarm: 10min_dbengine_global_flushing_errors
51
+ on: netdata.dbengine_long_term_page_stats
52
+ class: Netdata
53
+component: DB engine
54
+ type: Errors
55
+ os: linux freebsd macos
56
+ hosts: *
57
+ lookup: sum -10m unaligned of flushing_pressure_deletions
58
+ units: pages
59
+ every: 10s
60
+ crit: $this != 0
61
+ delay: down 1h multiplier 1.5 max 3h
62
+ info: number of pages deleted due to failure to flush data to disk in the last 10 minutes. \
63
+ Metric data were lost to unblock data collection. To fix, reduce disk load or use faster disks.
64
+ to: sysadmin
health/health.d/disks.conf
+69
-57
@@ -9,33 +9,39 @@
9
# raise an alarm if the disk is low on
10
# available disk space
11
12
-template: disk_space_usage
13
- on: disk.space
14
- os: linux freebsd
15
- hosts: *
16
-families: !/dev !/dev/* !/run !/run/* *
17
- calc: $used * 100 / ($avail + $used)
18
- units: %
19
- every: 1m
20
- warn: $this > (($status >= $WARNING ) ? (80) : (90))
21
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
22
- delay: up 1m down 15m multiplier 1.5 max 1h
23
- info: disk space utilization
24
- to: sysadmin
25
-
26
-template: disk_inode_usage
27
- on: disk.inodes
28
- os: linux freebsd
29
- hosts: *
30
-families: !/dev !/dev/* !/run !/run/* *
31
- calc: $used * 100 / ($avail + $used)
32
- units: %
33
- every: 1m
34
- warn: $this > (($status >= $WARNING) ? (80) : (90))
35
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
36
- delay: up 1m down 15m multiplier 1.5 max 1h
37
- info: disk inode utilization
38
- to: sysadmin
12
+ template: disk_space_usage
13
+ on: disk.space
14
+ class: System
15
+component: Disk
16
+ type: Utilization
17
+ os: linux freebsd
18
+ hosts: *
19
+ families: !/dev !/dev/* !/run !/run/* *
20
+ calc: $used * 100 / ($avail + $used)
21
+ units: %
22
+ every: 1m
23
+ warn: $this > (($status >= $WARNING ) ? (80) : (90))
24
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
25
+ delay: up 1m down 15m multiplier 1.5 max 1h
26
+ info: disk $family space utilization
27
+ to: sysadmin
28
+
29
+ template: disk_inode_usage
30
+ on: disk.inodes
31
+ class: System
32
+component: Disk
33
+ type: Utilization
34
+ os: linux freebsd
35
+ hosts: *
36
+ families: !/dev !/dev/* !/run !/run/* *
37
+ calc: $used * 100 / ($avail + $used)
38
+ units: %
39
+ every: 1m
40
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
41
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
42
+ delay: up 1m down 15m multiplier 1.5 max 1h
43
+ info: disk $family inode utilization
44
+ to: sysadmin
45
46
47
# -----------------------------------------------------------------------------
@@ -128,21 +134,24 @@ families: !/dev !/dev/* !/run !/run/* *
134
# by calculating the average disk utilization
135
# for the last 10 minutes
136
131
-template: 10min_disk_utilization
132
- on: disk.util
133
- os: linux freebsd
134
- hosts: *
135
-families: *
136
- lookup: average -10m unaligned
137
- units: %
138
- every: 1m
139
- green: 90
140
- red: 98
141
- warn: $this > $green * (($status >= $WARNING) ? (0.7) : (1))
142
- crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
143
- delay: down 15m multiplier 1.2 max 1h
144
- info: average percentage of time the disk was busy over the last 10 minutes
145
- to: silent
137
+ template: 10min_disk_utilization
138
+ on: disk.util
139
+ class: System
140
+component: Disk
141
+ type: Utilization
142
+ os: linux freebsd
143
+ hosts: *
144
+ families: *
145
+ lookup: average -10m unaligned
146
+ units: %
147
+ every: 1m
148
+ green: 90
149
+ red: 98
150
+ warn: $this > $green * (($status >= $WARNING) ? (0.7) : (1))
151
+ crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
152
+ delay: down 15m multiplier 1.2 max 1h
153
+ info: average percentage of time $family disk was busy over the last 10 minutes
154
+ to: silent
155
156
157
# raise an alarm if the disk backlog
@@ -150,18 +159,21 @@ families: *
159
# for 10 minutes
160
# (i.e. the disk cannot catch up)
161
153
-template: 10min_disk_backlog
154
- on: disk.backlog
155
- os: linux
156
- hosts: *
157
-families: *
158
- lookup: average -10m unaligned
159
- units: ms
160
- every: 1m
161
- green: 2000
162
- red: 5000
163
- warn: $this > $green * (($status >= $WARNING) ? (0.7) : (1))
164
- crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
165
- delay: down 15m multiplier 1.2 max 1h
166
- info: average disk backlog size over the last 10 minutes
167
- to: silent
162
+ template: 10min_disk_backlog
163
+ on: disk.backlog
164
+ class: System
165
+component: Disk
166
+ type: Latency
167
+ os: linux
168
+ hosts: *
169
+ families: *
170
+ lookup: average -10m unaligned
171
+ units: ms
172
+ every: 1m
173
+ green: 2000
174
+ red: 5000
175
+ warn: $this > $green * (($status >= $WARNING) ? (0.7) : (1))
176
+ crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
177
+ delay: down 15m multiplier 1.2 max 1h
178
+ info: average backlog size of the $family disk over the last 10 minutes
179
+ to: silent
health/health.d/dns_query.conf
+12
-9
@@ -1,12 +1,15 @@
1
2
# detect dns query failure
3
4
-template: dns_query_time_query_time
5
- on: dns_query_time.query_time
6
- lookup: average -10s unaligned foreach *
7
- units: ms
8
- every: 10s
9
- warn: $this == nan
10
- delay: up 20s down 5m multiplier 1.5 max 1h
11
- info: average DNS query round trip time over the last 10 seconds
12
- to: sysadmin
4
+ template: dns_query_time_query_time
5
+ on: dns_query_time.query_time
6
+ class: DNS
7
+component: DNS
8
+ type: Latency
9
+ lookup: average -10s unaligned foreach *
10
+ units: ms
11
+ every: 10s
12
+ warn: $this == nan
13
+ delay: up 20s down 5m multiplier 1.5 max 1h
14
+ info: average DNS query round trip time over the last 10 seconds
15
+ to: sysadmin
health/health.d/dnsmasq_dhcp.conf
+13
-10
@@ -1,12 +1,15 @@
1
# dhcp-range utilization
2
3
-template: dnsmasq_dhcp_dhcp_range_utilization
4
- on: dnsmasq_dhcp.dhcp_range_utilization
5
- every: 10s
6
- units: %
7
- calc: $used
8
- warn: $this > ( ($status >= $WARNING ) ? ( 80 ) : ( 90 ) )
9
- crit: $this > ( ($status == $CRITICAL) ? ( 90 ) : ( 95 ) )
10
- delay: down 5m
11
- info: DHCP range utilization
12
- to: sysadmin
3
+ template: dnsmasq_dhcp_dhcp_range_utilization
4
+ on: dnsmasq_dhcp.dhcp_range_utilization
5
+ class: DHCP
6
+component: Dnsmasq
7
+ type: Utilization
8
+ every: 10s
9
+ units: %
10
+ calc: $used
11
+ warn: $this > ( ($status >= $WARNING ) ? ( 80 ) : ( 90 ) )
12
+ crit: $this > ( ($status == $CRITICAL) ? ( 90 ) : ( 95 ) )
13
+ delay: down 5m
14
+ info: DHCP range utilization
15
+ to: sysadmin
health/health.d/dockerd.conf
+11
-8
@@ -1,8 +1,11 @@
1
-template: docker_unhealthy_containers
2
- on: docker.unhealthy_containers
3
- units: unhealthy containers
4
- every: 10s
5
- lookup: average -10s
6
- crit: $this > 0
7
- info: average number of unhealthy docker containers over the last 10 seconds
8
- to: sysadmin
1
+ template: docker_unhealthy_containers
2
+ on: docker.unhealthy_containers
3
+ class: Containers
4
+component: Docker
5
+ type: Errors
6
+ units: unhealthy containers
7
+ every: 10s
8
+ lookup: average -10s
9
+ crit: $this > 0
10
+ info: average number of unhealthy docker containers over the last 10 seconds
11
+ to: sysadmin
health/health.d/elasticsearch.conf
+12
-9
@@ -1,12 +1,15 @@
1
2
# make sure elasticsearch is running
3
4
-template: elasticsearch_last_collected
5
- on: elasticsearch.cluster_health_status
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- info: number of seconds since the last successful data collection
12
- to: sysadmin
4
+ template: elasticsearch_last_collected
5
+ on: elasticsearch.cluster_health_status
6
+ class: Search engine
7
+component: Elasticsearch
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ info: number of seconds since the last successful data collection
15
+ to: sysadmin
health/health.d/entropy.conf
+14
-11
@@ -3,14 +3,17 @@
3
# the alarm is checked every 1 minute
4
# and examines the last hour of data
5
6
- alarm: lowest_entropy
7
- on: system.entropy
8
- os: linux
9
- hosts: *
10
- lookup: min -5m unaligned
11
- units: entries
12
- every: 5m
13
- warn: $this < (($status >= $WARNING) ? (200) : (100))
14
- delay: down 1h multiplier 1.5 max 2h
15
- info: minimum number of entries in the random numbers pool in the last 5 minutes
16
- to: silent
6
+ alarm: lowest_entropy
7
+ on: system.entropy
8
+ class: System
9
+component: Cryptography
10
+ type: Utilization
11
+ os: linux
12
+ hosts: *
13
+ lookup: min -5m unaligned
14
+ units: entries
15
+ every: 5m
16
+ warn: $this < (($status >= $WARNING) ? (200) : (100))
17
+ delay: down 1h multiplier 1.5 max 2h
18
+ info: minimum number of entries in the random numbers pool in the last 5 minutes
19
+ to: silent
health/health.d/exporting.conf
+13
-10
@@ -11,13 +11,16 @@ families: *
11
info: number of seconds since the last successful buffering of exporting data
12
to: dba
13
14
-template: exporting_metrics_sent
15
-families: *
16
- on: exporting_data_size
17
- units: %
18
- calc: abs($sent) * 100 / abs($buffered)
19
- every: 10s
20
- warn: $this != 100
21
- delay: down 5m multiplier 1.5 max 1h
22
- info: percentage of metrics sent to the external database server
23
- to: dba
14
+ template: exporting_metrics_sent
15
+ families: *
16
+ on: exporting_data_size
17
+ class: Netdata
18
+component: Exporting engine
19
+ type: Workload
20
+ units: %
21
+ calc: abs($sent) * 100 / abs($buffered)
22
+ every: 10s
23
+ warn: $this != 100
24
+ delay: down 5m multiplier 1.5 max 1h
25
+ info: percentage of metrics sent to the external database server
26
+ to: dba
health/health.d/fping.conf
+60
-48
@@ -1,52 +1,64 @@
1
2
-template: fping_last_collected_secs
3
-families: *
4
- on: fping.latency
5
- calc: $now - $last_collected_t
6
- units: seconds ago
7
- every: 10s
8
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
9
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
10
- delay: down 5m multiplier 1.5 max 1h
11
- info: number of seconds since the last successful data collection
12
- to: sysadmin
2
+ template: fping_last_collected_secs
3
+ families: *
4
+ on: fping.latency
5
+ class: Other
6
+component: Network
7
+ type: Latency
8
+ calc: $now - $last_collected_t
9
+ units: seconds ago
10
+ every: 10s
11
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
12
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
13
+ delay: down 5m multiplier 1.5 max 1h
14
+ info: number of seconds since the last successful data collection
15
+ to: sysadmin
16
14
-template: fping_host_reachable
15
-families: *
16
- on: fping.latency
17
- calc: $average != nan
18
- units: up/down
19
- every: 10s
20
- crit: $this == 0
21
- delay: down 30m multiplier 1.5 max 2h
22
- info: reachability status of the network host (0: unreachable, 1: reachable)
23
- to: sysadmin
17
+ template: fping_host_reachable
18
+ families: *
19
+ on: fping.latency
20
+ class: Other
21
+component: Network
22
+ type: Errors
23
+ calc: $average != nan
24
+ units: up/down
25
+ every: 10s
26
+ crit: $this == 0
27
+ delay: down 30m multiplier 1.5 max 2h
28
+ info: reachability status of the network host (0: unreachable, 1: reachable)
29
+ to: sysadmin
30
25
-template: fping_host_latency
26
-families: *
27
- on: fping.latency
28
- lookup: average -10s unaligned of average
29
- units: ms
30
- every: 10s
31
- green: 500
32
- red: 1000
33
- warn: $this > $green OR $max > $red
34
- crit: $this > $red
35
- delay: down 30m multiplier 1.5 max 2h
36
- info: average latency to the network host over the last 10 seconds
37
- to: sysadmin
31
+ template: fping_host_latency
32
+ families: *
33
+ on: fping.latency
34
+ class: Other
35
+component: Network
36
+ type: Latency
37
+ lookup: average -10s unaligned of average
38
+ units: ms
39
+ every: 10s
40
+ green: 500
41
+ red: 1000
42
+ warn: $this > $green OR $max > $red
43
+ crit: $this > $red
44
+ delay: down 30m multiplier 1.5 max 2h
45
+ info: average latency to the network host over the last 10 seconds
46
+ to: sysadmin
47
39
-template: fping_packet_loss
40
-families: *
41
- on: fping.quality
42
- lookup: average -10m unaligned of returned
43
- calc: 100 - $this
44
- green: 1
45
- red: 10
46
- units: %
47
- every: 10s
48
- warn: $this > $green
49
- crit: $this > $red
50
- delay: down 30m multiplier 1.5 max 2h
51
- info: packet loss ratio to the network host over the last 10 minutes
52
- to: sysadmin
48
+ template: fping_packet_loss
49
+ families: *
50
+ on: fping.quality
51
+ class: System
52
+component: Network
53
+ type: Errors
54
+ lookup: average -10m unaligned of returned
55
+ calc: 100 - $this
56
+ green: 1
57
+ red: 10
58
+ units: %
59
+ every: 10s
60
+ warn: $this > $green
61
+ crit: $this > $red
62
+ delay: down 30m multiplier 1.5 max 2h
63
+ info: packet loss ratio to the network host over the last 10 minutes
64
+ to: sysadmin
health/health.d/fronius.conf
+14
-11
@@ -1,11 +1,14 @@
1
-template: fronius_last_collected_secs
2
-families: *
3
- on: fronius.power
4
- calc: $now - $last_collected_t
5
- every: 10s
6
- units: seconds ago
7
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
- delay: down 5m multiplier 1.5 max 1h
10
- info: number of seconds since the last successful data collection
11
- to: sitemgr
1
+ template: fronius_last_collected_secs
2
+ families: *
3
+ on: fronius.power
4
+ class: Power Supply
5
+component: Solar
6
+ type: Latency
7
+ calc: $now - $last_collected_t
8
+ every: 10s
9
+ units: seconds ago
10
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: number of seconds since the last successful data collection
14
+ to: sitemgr
health/health.d/gearman.conf
+26
-20
@@ -1,22 +1,28 @@
1
# make sure Gearman is running
2
-template: gearman_last_collected_secs
3
- on: gearman.total_jobs
4
- calc: $now - $last_collected_t
5
- units: seconds ago
6
- every: 10s
7
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
- delay: down 5m multiplier 1.5 max 1h
10
- info: number of seconds since the last successful data collection
11
- to: sysadmin
2
+ template: gearman_last_collected_secs
3
+ on: gearman.total_jobs
4
+ class: Computing
5
+component: Gearman
6
+ type: Latency
7
+ calc: $now - $last_collected_t
8
+ units: seconds ago
9
+ every: 10s
10
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: number of seconds since the last successful data collection
14
+ to: sysadmin
15
13
-template: gearman_workers_queued
14
- on: gearman.single_job
15
- lookup: average -10m unaligned match-names of Queued
16
- units: workers
17
- every: 10s
18
- warn: $this > 30000
19
- crit: $this > 100000
20
- delay: down 5m multiplier 1.5 max 1h
21
- info: average number of queued jobs over the last 10 minutes
22
- to: sysadmin
16
+ template: gearman_workers_queued
17
+ on: gearman.single_job
18
+ class: Computing
19
+component: Gearman
20
+ type: Latency
21
+ lookup: average -10m unaligned match-names of Queued
22
+ units: workers
23
+ every: 10s
24
+ warn: $this > 30000
25
+ crit: $this > 100000
26
+ delay: down 5m multiplier 1.5 max 1h
27
+ info: average number of queued jobs over the last 10 minutes
28
+ to: sysadmin
health/health.d/haproxy.conf
+34
-25
@@ -1,27 +1,36 @@
1
-template: haproxy_backend_server_status
2
- on: haproxy_hs.down
3
- units: failed servers
4
- every: 10s
5
- lookup: average -10s
6
- crit: $this > 0
7
- info: average number of failed haproxy backend servers over the last 10 seconds
8
- to: sysadmin
1
+ template: haproxy_backend_server_status
2
+ on: haproxy_hs.down
3
+ class: Web Proxy
4
+component: HAProxy
5
+ type: Errors
6
+ units: failed servers
7
+ every: 10s
8
+ lookup: average -10s
9
+ crit: $this > 0
10
+ info: average number of failed haproxy backend servers over the last 10 seconds
11
+ to: sysadmin
12
10
-template: haproxy_backend_status
11
- on: haproxy_hb.down
12
- units: failed backend
13
- every: 10s
14
- lookup: average -10s
15
- crit: $this > 0
16
- info: average number of failed haproxy backends over the last 10 seconds
17
- to: sysadmin
13
+ template: haproxy_backend_status
14
+ on: haproxy_hb.down
15
+ class: Web Proxy
16
+component: HAProxy
17
+ type: Errors
18
+ units: failed backend
19
+ every: 10s
20
+ lookup: average -10s
21
+ crit: $this > 0
22
+ info: average number of failed haproxy backends over the last 10 seconds
23
+ to: sysadmin
24
19
-template: haproxy_last_collected
20
- on: haproxy_hb.down
21
- calc: $now - $last_collected_t
22
- units: seconds ago
23
- every: 10s
24
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
25
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
26
- info: number of seconds since the last successful data collection
27
- to: sysadmin
25
+ template: haproxy_last_collected
26
+ on: haproxy_hb.down
27
+ class: Web Proxy
28
+component: HAProxy
29
+ type: Latency
30
+ calc: $now - $last_collected_t
31
+ units: seconds ago
32
+ every: 10s
33
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
34
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
35
+ info: number of seconds since the last successful data collection
36
+ to: sysadmin
health/health.d/hdfs.conf
+74
-56
@@ -1,75 +1,93 @@
1
2
# make sure hdfs is running
3
4
-template: hdfs_last_collected_secs
5
- on: hdfs.heap_memory
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: hdfs_last_collected_secs
5
+ on: hdfs.heap_memory
6
+ class: Storage
7
+component: HDFS
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
18
19
# Common
20
18
-template: hdfs_capacity_usage
19
- on: hdfs.capacity
20
- calc: ($used) * 100 / ($used + $remaining)
21
- units: %
22
- every: 10s
23
- warn: $this > (($status >= $WARNING) ? (70) : (80))
24
- crit: $this > (($status == $CRITICAL) ? (80) : (98))
25
- delay: down 15m multiplier 1.5 max 1h
26
- info: summary datanodes space capacity utilization
27
- to: sysadmin
21
+ template: hdfs_capacity_usage
22
+ on: hdfs.capacity
23
+ class: Storage
24
+component: HDFS
25
+ type: Utilization
26
+ calc: ($used) * 100 / ($used + $remaining)
27
+ units: %
28
+ every: 10s
29
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
30
+ crit: $this > (($status == $CRITICAL) ? (80) : (98))
31
+ delay: down 15m multiplier 1.5 max 1h
32
+ info: summary datanodes space capacity utilization
33
+ to: sysadmin
34
35
36
# NameNode
37
32
-template: hdfs_missing_blocks
33
- on: hdfs.blocks
34
- calc: $missing
35
- units: missing blocks
36
- every: 10s
37
- warn: $this > 0
38
- delay: down 15m multiplier 1.5 max 1h
39
- info: number of missing blocks
40
- to: sysadmin
38
+ template: hdfs_missing_blocks
39
+ on: hdfs.blocks
40
+ class: Storage
41
+component: HDFS
42
+ type: Errors
43
+ calc: $missing
44
+ units: missing blocks
45
+ every: 10s
46
+ warn: $this > 0
47
+ delay: down 15m multiplier 1.5 max 1h
48
+ info: number of missing blocks
49
+ to: sysadmin
50
51
43
-template: hdfs_stale_nodes
44
- on: hdfs.data_nodes
45
- calc: $stale
46
- units: dead nodes
47
- every: 10s
48
- warn: $this > 0
49
- delay: down 15m multiplier 1.5 max 1h
50
- info: number of datanodes marked stale due to delayed heartbeat
51
- to: sysadmin
52
+ template: hdfs_stale_nodes
53
+ on: hdfs.data_nodes
54
+ class: Storage
55
+component: HDFS
56
+ type: Errors
57
+ calc: $stale
58
+ units: dead nodes
59
+ every: 10s
60
+ warn: $this > 0
61
+ delay: down 15m multiplier 1.5 max 1h
62
+ info: number of datanodes marked stale due to delayed heartbeat
63
+ to: sysadmin
64
65
54
-template: hdfs_dead_nodes
55
- on: hdfs.data_nodes
56
- calc: $dead
57
- units: dead nodes
58
- every: 10s
59
- crit: $this > 0
60
- delay: down 15m multiplier 1.5 max 1h
61
- info: number of datanodes which are currently dead
62
- to: sysadmin
66
+ template: hdfs_dead_nodes
67
+ on: hdfs.data_nodes
68
+ class: Storage
69
+component: HDFS
70
+ type: Errors
71
+ calc: $dead
72
+ units: dead nodes
73
+ every: 10s
74
+ crit: $this > 0
75
+ delay: down 15m multiplier 1.5 max 1h
76
+ info: number of datanodes which are currently dead
77
+ to: sysadmin
78
79
80
# DataNode
81
67
-template: hdfs_num_failed_volumes
68
- on: hdfs.num_failed_volumes
69
- calc: $fsds_num_failed_volumes
70
- units: failed volumes
71
- every: 10s
72
- warn: $this > 0
73
- delay: down 15m multiplier 1.5 max 1h
74
- info: number of failed volumes
75
- to: sysadmin
82
+ template: hdfs_num_failed_volumes
83
+ on: hdfs.num_failed_volumes
84
+ class: Storage
85
+component: HDFS
86
+ type: Errors
87
+ calc: $fsds_num_failed_volumes
88
+ units: failed volumes
89
+ every: 10s
90
+ warn: $this > 0
91
+ delay: down 15m multiplier 1.5 max 1h
92
+ info: number of failed volumes
93
+ to: sysadmin
health/health.d/httpcheck.conf
+116
-89
@@ -1,99 +1,126 @@
1
-template: httpcheck_last_collected_secs
2
-families: *
3
- on: httpcheck.status
4
- calc: $now - $last_collected_t
5
- every: 10s
6
- units: seconds ago
7
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
- delay: down 5m multiplier 1.5 max 1h
10
- info: number of seconds since the last successful data collection
11
- to: sysadmin
1
+ template: httpcheck_last_collected_secs
2
+ families: *
3
+ on: httpcheck.status
4
+ class: Other
5
+component: HTTP endpoint
6
+ type: Latency
7
+ calc: $now - $last_collected_t
8
+ every: 10s
9
+ units: seconds ago
10
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: number of seconds since the last successful data collection
14
+ to: sysadmin
15
16
# This is a fast-reacting no-notification alarm ideal for custom dashboards or badges
14
-template: httpcheck_web_service_up
15
-families: *
16
- on: httpcheck.status
17
- lookup: average -1m unaligned percentage of success
18
- calc: ($this < 75) ? (0) : ($this)
19
- every: 5s
20
- units: up/down
21
- info: average ratio of successful HTTP requests over the last minute (at least 75%)
22
- to: silent
17
+ template: httpcheck_web_service_up
18
+ families: *
19
+ on: httpcheck.status
20
+ class: Web Server
21
+component: HTTP endpoint
22
+ type: Utilization
23
+ lookup: average -1m unaligned percentage of success
24
+ calc: ($this < 75) ? (0) : ($this)
25
+ every: 5s
26
+ units: up/down
27
+ info: average ratio of successful HTTP requests over the last minute (at least 75%)
28
+ to: silent
29
24
-template: httpcheck_web_service_bad_content
25
-families: *
26
- on: httpcheck.status
27
- lookup: average -5m unaligned percentage of bad_content
28
- every: 10s
29
- units: %
30
- warn: $this >= 10 AND $this < 40
31
- crit: $this >= 40
32
- delay: down 5m multiplier 1.5 max 1h
33
- info: average ratio of HTTP responses with unexpected content over the last 5 minutes
34
- options: no-clear-notification
35
- to: webmaster
30
+ template: httpcheck_web_service_bad_content
31
+ families: *
32
+ on: httpcheck.status
33
+ class: Web Server
34
+component: HTTP endpoint
35
+ type: Workload
36
+ lookup: average -5m unaligned percentage of bad_content
37
+ every: 10s
38
+ units: %
39
+ warn: $this >= 10 AND $this < 40
40
+ crit: $this >= 40
41
+ delay: down 5m multiplier 1.5 max 1h
42
+ info: average ratio of HTTP responses with unexpected content over the last 5 minutes
43
+ options: no-clear-notification
44
+ to: webmaster
45
37
-template: httpcheck_web_service_bad_status
38
-families: *
39
- on: httpcheck.status
40
- lookup: average -5m unaligned percentage of bad_status
41
- every: 10s
42
- units: %
43
- warn: $this >= 10 AND $this < 40
44
- crit: $this >= 40
45
- delay: down 5m multiplier 1.5 max 1h
46
- info: average ratio of HTTP responses with unexpected status over the last 5 minutes
47
- options: no-clear-notification
48
- to: webmaster
46
+ template: httpcheck_web_service_bad_status
47
+ families: *
48
+ on: httpcheck.status
49
+ class: Web Server
50
+component: HTTP endpoint
51
+ type: Workload
52
+ lookup: average -5m unaligned percentage of bad_status
53
+ every: 10s
54
+ units: %
55
+ warn: $this >= 10 AND $this < 40
56
+ crit: $this >= 40
57
+ delay: down 5m multiplier 1.5 max 1h
58
+ info: average ratio of HTTP responses with unexpected status over the last 5 minutes
59
+ options: no-clear-notification
60
+ to: webmaster
61
50
-template: httpcheck_web_service_timeouts
51
-families: *
52
- on: httpcheck.status
53
- lookup: average -5m unaligned percentage of timeout
54
- every: 10s
55
- units: %
56
- info: average ratio of HTTP request timeouts over the last 5 minutes
62
+ template: httpcheck_web_service_timeouts
63
+ families: *
64
+ on: httpcheck.status
65
+ class: Web Server
66
+component: HTTP endpoint
67
+ type: Latency
68
+ lookup: average -5m unaligned percentage of timeout
69
+ every: 10s
70
+ units: %
71
+ info: average ratio of HTTP request timeouts over the last 5 minutes
72
58
-template: httpcheck_no_web_service_connections
59
-families: *
60
- on: httpcheck.status
61
- lookup: average -5m unaligned percentage of no_connection
62
- every: 10s
63
- units: %
64
- info: average ratio of failed requests during the last 5 minutes
73
+ template: httpcheck_no_web_service_connections
74
+ families: *
75
+ on: httpcheck.status
76
+ class: Other
77
+component: HTTP endpoint
78
+ type: Errors
79
+ lookup: average -5m unaligned percentage of no_connection
80
+ every: 10s
81
+ units: %
82
+ info: average ratio of failed requests during the last 5 minutes
83
84
# combined timeout & no connection alarm
67
-template: httpcheck_web_service_unreachable
68
-families: *
69
- on: httpcheck.status
70
- calc: ($httpcheck_no_web_service_connections >= $httpcheck_web_service_timeouts) ? ($httpcheck_no_web_service_connections) : ($httpcheck_web_service_timeouts)
71
- units: %
72
- every: 10s
73
- warn: ($httpcheck_no_web_service_connections >= 10 OR $httpcheck_web_service_timeouts >= 10) AND ($httpcheck_no_web_service_connections < 40 OR $httpcheck_web_service_timeouts < 40)
74
- crit: $httpcheck_no_web_service_connections >= 40 OR $httpcheck_web_service_timeouts >= 40
75
- delay: down 5m multiplier 1.5 max 1h
76
- info: ratio of failed requests either due to timeouts or no connection over the last 5 minutes
77
- options: no-clear-notification
78
- to: webmaster
85
+ template: httpcheck_web_service_unreachable
86
+ families: *
87
+ on: httpcheck.status
88
+ class: Web Server
89
+component: HTTP endpoint
90
+ type: Errors
91
+ calc: ($httpcheck_no_web_service_connections >= $httpcheck_web_service_timeouts) ? ($httpcheck_no_web_service_connections) : ($httpcheck_web_service_timeouts)
92
+ units: %
93
+ every: 10s
94
+ warn: ($httpcheck_no_web_service_connections >= 10 OR $httpcheck_web_service_timeouts >= 10) AND ($httpcheck_no_web_service_connections < 40 OR $httpcheck_web_service_timeouts < 40)
95
+ crit: $httpcheck_no_web_service_connections >= 40 OR $httpcheck_web_service_timeouts >= 40
96
+ delay: down 5m multiplier 1.5 max 1h
97
+ info: ratio of failed requests either due to timeouts or no connection over the last 5 minutes
98
+ options: no-clear-notification
99
+ to: webmaster
100
80
-template: httpcheck_1h_web_service_response_time
81
-families: *
82
- on: httpcheck.responsetime
83
- lookup: average -1h unaligned of time
84
- every: 30s
85
- units: ms
86
- info: average HTTP response time over the last hour
101
+ template: httpcheck_1h_web_service_response_time
102
+ families: *
103
+ on: httpcheck.responsetime
104
+ class: Other
105
+component: HTTP endpoint
106
+ type: Latency
107
+ lookup: average -1h unaligned of time
108
+ every: 30s
109
+ units: ms
110
+ info: average HTTP response time over the last hour
111
88
-template: httpcheck_web_service_slow
89
-families: *
90
- on: httpcheck.responsetime
91
- lookup: average -3m unaligned of time
92
- units: ms
93
- every: 10s
94
- warn: ($this > ($httpcheck_1h_web_service_response_time * 2) )
95
- crit: ($this > ($httpcheck_1h_web_service_response_time * 3) )
96
- delay: down 5m multiplier 1.5 max 1h
97
- info: average HTTP response time over the last 3 minutes, compared to the average over the last hour
98
- options: no-clear-notification
99
- to: webmaster
112
+ template: httpcheck_web_service_slow
113
+ families: *
114
+ on: httpcheck.responsetime
115
+ class: Web Server
116
+component: HTTP endpoint
117
+ type: Latency
118
+ lookup: average -3m unaligned of time
119
+ units: ms
120
+ every: 10s
121
+ warn: ($this > ($httpcheck_1h_web_service_response_time * 2) )
122
+ crit: ($this > ($httpcheck_1h_web_service_response_time * 3) )
123
+ delay: down 5m multiplier 1.5 max 1h
124
+ info: average HTTP response time over the last 3 minutes, compared to the average over the last hour
125
+ options: no-clear-notification
126
+ to: webmaster
health/health.d/ioping.conf
+16
-13
@@ -1,13 +1,16 @@
1
-template: ioping_disk_latency
2
-families: *
3
- on: ioping.latency
4
- lookup: average -10s unaligned of average
5
- units: ms
6
- every: 10s
7
- green: 500
8
- red: 1000
9
- warn: $this > $green OR $max > $red
10
- crit: $this > $red
11
- delay: down 30m multiplier 1.5 max 2h
12
- info: average I/O latency over the last 10 seconds
13
- to: sysadmin
1
+ template: ioping_disk_latency
2
+ families: *
3
+ on: ioping.latency
4
+ class: System
5
+component: Disk
6
+ type: Latency
7
+ lookup: average -10s unaligned of average
8
+ units: ms
9
+ every: 10s
10
+ green: 500
11
+ red: 1000
12
+ warn: $this > $green OR $max > $red
13
+ crit: $this > $red
14
+ delay: down 30m multiplier 1.5 max 2h
15
+ info: average I/O latency over the last 10 seconds
16
+ to: sysadmin
health/health.d/ipc.conf
+30
-24
@@ -1,28 +1,34 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: semaphores_used
5
- on: system.ipc_semaphores
6
- os: linux
7
- hosts: *
8
- calc: $semaphores * 100 / $ipc_semaphores_max
9
- units: %
10
- every: 10s
11
- warn: $this > (($status >= $WARNING) ? (70) : (80))
12
- crit: $this > (($status == $CRITICAL) ? (70) : (90))
13
- delay: down 5m multiplier 1.5 max 1h
14
- info: IPC semaphore utilization
15
- to: sysadmin
4
+ alarm: semaphores_used
5
+ on: system.ipc_semaphores
6
+ class: System
7
+component: IPC
8
+ type: Utilization
9
+ os: linux
10
+ hosts: *
11
+ calc: $semaphores * 100 / $ipc_semaphores_max
12
+ units: %
13
+ every: 10s
14
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
15
+ crit: $this > (($status == $CRITICAL) ? (70) : (90))
16
+ delay: down 5m multiplier 1.5 max 1h
17
+ info: IPC semaphore utilization
18
+ to: sysadmin
19
17
- alarm: semaphore_arrays_used
18
- on: system.ipc_semaphore_arrays
19
- os: linux
20
- hosts: *
21
- calc: $arrays * 100 / $ipc_semaphores_arrays_max
22
- units: %
23
- every: 10s
24
- warn: $this > (($status >= $WARNING) ? (70) : (80))
25
- crit: $this > (($status == $CRITICAL) ? (70) : (90))
26
- delay: down 5m multiplier 1.5 max 1h
27
- info: IPC semaphore arrays utilization
28
- to: sysadmin
20
+ alarm: semaphore_arrays_used
21
+ on: system.ipc_semaphore_arrays
22
+ class: System
23
+component: IPC
24
+ type: Utilization
25
+ os: linux
26
+ hosts: *
27
+ calc: $arrays * 100 / $ipc_semaphores_arrays_max
28
+ units: %
29
+ every: 10s
30
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
31
+ crit: $this > (($status == $CRITICAL) ? (70) : (90))
32
+ delay: down 5m multiplier 1.5 max 1h
33
+ info: IPC semaphore arrays utilization
34
+ to: sysadmin
health/health.d/ipfs.conf
+13
-10
@@ -1,11 +1,14 @@
1
2
-template: ipfs_datastore_usage
3
- on: ipfs.repo_size
4
- calc: $size * 100 / $avail
5
- units: %
6
- every: 10s
7
- warn: $this > (($status >= $WARNING) ? (80) : (90))
8
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
9
- delay: down 15m multiplier 1.5 max 1h
10
- info: IPFS datastore utilization
11
- to: sysadmin
2
+ template: ipfs_datastore_usage
3
+ on: ipfs.repo_size
4
+ class: Data Sharing
5
+component: IPFS
6
+ type: Utilization
7
+ calc: $size * 100 / $avail
8
+ units: %
9
+ every: 10s
10
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
11
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
12
+ delay: down 15m multiplier 1.5 max 1h
13
+ info: IPFS datastore utilization
14
+ to: sysadmin
health/health.d/ipmi.conf
+25
-19
@@ -1,20 +1,26 @@
1
- alarm: ipmi_sensors_states
2
- on: ipmi.sensors_states
3
- calc: $warning + $critical
4
- units: sensors
5
- every: 10s
6
- warn: $this > 0
7
- crit: $critical > 0
8
- delay: up 5m down 15m multiplier 1.5 max 1h
9
- info: number of IPMI sensors in non-nominal state
10
- to: sysadmin
1
+ alarm: ipmi_sensors_states
2
+ on: ipmi.sensors_states
3
+ class: System
4
+component: IPMI
5
+ type: Errors
6
+ calc: $warning + $critical
7
+ units: sensors
8
+ every: 10s
9
+ warn: $this > 0
10
+ crit: $critical > 0
11
+ delay: up 5m down 15m multiplier 1.5 max 1h
12
+ info: number of IPMI sensors in non-nominal state
13
+ to: sysadmin
14
12
- alarm: ipmi_events
13
- on: ipmi.events
14
- calc: $events
15
- units: events
16
- every: 10s
17
- warn: $this > 0
18
- delay: up 5m down 15m multiplier 1.5 max 1h
19
- info: number of events in the IPMI System Event Log (SEL)
20
- to: sysadmin
15
+ alarm: ipmi_events
16
+ on: ipmi.events
17
+ class: System
18
+component: IPMI
19
+ type: Utilization
20
+ calc: $events
21
+ units: events
22
+ every: 10s
23
+ warn: $this > 0
24
+ delay: up 5m down 15m multiplier 1.5 max 1h
25
+ info: number of events in the IPMI System Event Log (SEL)
26
+ to: sysadmin
health/health.d/kubelet.conf
+111
-84
@@ -4,39 +4,48 @@
4
5
# True (1) if the node is experiencing a configuration-related error, false (0) otherwise.
6
7
- template: kubelet_node_config_error
8
- on: k8s_kubelet.kubelet_node_config_error
9
- calc: $kubelet_node_config_error
10
- units: bool
11
- every: 10s
12
- warn: $this == 1
13
- delay: down 1m multiplier 1.5 max 2h
14
- info: the node is experiencing a configuration-related error (0: false, 1: true)
15
- to: sysadmin
7
+ template: kubelet_node_config_error
8
+ on: k8s_kubelet.kubelet_node_config_error
9
+ class: Kubernetes
10
+component: Kubelet
11
+ type: Errors
12
+ calc: $kubelet_node_config_error
13
+ units: bool
14
+ every: 10s
15
+ warn: $this == 1
16
+ delay: down 1m multiplier 1.5 max 2h
17
+ info: the node is experiencing a configuration-related error (0: false, 1: true)
18
+ to: sysadmin
19
20
# Failed Token() requests to the alternate token source
21
19
- template: kubelet_token_requests
20
- lookup: sum -10s of token_fail_count
21
- on: k8s_kubelet.kubelet_token_requests
22
- units: failed requests
23
- every: 10s
24
- warn: $this > 0
25
- delay: down 1m multiplier 1.5 max 2h
26
- info: number of failed Token() requests to the alternate token source
27
- to: sysadmin
22
+ template: kubelet_token_requests
23
+ lookup: sum -10s of token_fail_count
24
+ on: k8s_kubelet.kubelet_token_requests
25
+ class: Kubernetes
26
+component: Kubelet
27
+ type: Errors
28
+ units: failed requests
29
+ every: 10s
30
+ warn: $this > 0
31
+ delay: down 1m multiplier 1.5 max 2h
32
+ info: number of failed Token() requests to the alternate token source
33
+ to: sysadmin
34
35
# Docker and runtime operation errors
36
31
- template: kubelet_operations_error
32
- lookup: sum -1m
33
- on: k8s_kubelet.kubelet_operations_errors
34
- units: errors
35
- every: 10s
36
- warn: $this > (($status >= $WARNING) ? (0) : (20))
37
- delay: up 30s down 1m multiplier 1.5 max 2h
38
- info: number of Docker or runtime operation errors
39
- to: sysadmin
37
+ template: kubelet_operations_error
38
+ lookup: sum -1m
39
+ on: k8s_kubelet.kubelet_operations_errors
40
+ class: Kubernetes
41
+component: Kubelet
42
+ type: Errors
43
+ units: errors
44
+ every: 10s
45
+ warn: $this > (($status >= $WARNING) ? (0) : (20))
46
+ delay: up 30s down 1m multiplier 1.5 max 2h
47
+ info: number of Docker or runtime operation errors
48
+ to: sysadmin
49
50
# -----------------------------------------------------------------------------
51
@@ -53,66 +62,84 @@
62
63
# quantile 0.5
64
56
-template: kubelet_1m_pleg_relist_latency_quantile_05
57
- on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
58
- lookup: average -1m unaligned of kubelet_pleg_relist_latency_05
59
- units: microseconds
60
- every: 10s
61
- info: average Pod Lifecycle Event Generator relisting latency over the last minute (quantile 0.5)
62
-
63
-template: kubelet_10s_pleg_relist_latency_quantile_05
64
- on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
65
- lookup: average -10s unaligned of kubelet_pleg_relist_latency_05
66
- calc: $this * 100 / (($kubelet_1m_pleg_relist_latency_quantile_05 < 1000)?(1000):($kubelet_1m_pleg_relist_latency_quantile_05))
67
- every: 10s
68
- units: %
69
- warn: $this > (($status >= $WARNING)?(100):(200))
70
- crit: $this > (($status >= $WARNING)?(200):(400))
71
- delay: down 1m multiplier 1.5 max 2h
72
- info: ratio of average Pod Lifecycle Event Generator relisting latency over the last 10 seconds, \
73
- compared to the last minute (quantile 0.5)
74
- to: sysadmin
65
+ template: kubelet_1m_pleg_relist_latency_quantile_05
66
+ on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
67
+ class: Kubernetes
68
+component: Kubelet
69
+ type: Latency
70
+ lookup: average -1m unaligned of kubelet_pleg_relist_latency_05
71
+ units: microseconds
72
+ every: 10s
73
+ info: average Pod Lifecycle Event Generator relisting latency over the last minute (quantile 0.5)
74
+
75
+ template: kubelet_10s_pleg_relist_latency_quantile_05
76
+ on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
77
+ class: Kubernetes
78
+component: Kubelet
79
+ type: Latency
80
+ lookup: average -10s unaligned of kubelet_pleg_relist_latency_05
81
+ calc: $this * 100 / (($kubelet_1m_pleg_relist_latency_quantile_05 < 1000)?(1000):($kubelet_1m_pleg_relist_latency_quantile_05))
82
+ every: 10s
83
+ units: %
84
+ warn: $this > (($status >= $WARNING)?(100):(200))
85
+ crit: $this > (($status >= $WARNING)?(200):(400))
86
+ delay: down 1m multiplier 1.5 max 2h
87
+ info: ratio of average Pod Lifecycle Event Generator relisting latency over the last 10 seconds, \
88
+ compared to the last minute (quantile 0.5)
89
+ to: sysadmin
90
91
# quantile 0.9
92
78
-template: kubelet_1m_pleg_relist_latency_quantile_09
79
- on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
80
- lookup: average -1m unaligned of kubelet_pleg_relist_latency_09
81
- units: microseconds
82
- every: 10s
83
- info: average Pod Lifecycle Event Generator relisting latency over the last minute (quantile 0.9)
84
-
85
-template: kubelet_10s_pleg_relist_latency_quantile_09
86
- on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
87
- lookup: average -10s unaligned of kubelet_pleg_relist_latency_09
88
- calc: $this * 100 / (($kubelet_1m_pleg_relist_latency_quantile_09 < 1000)?(1000):($kubelet_1m_pleg_relist_latency_quantile_09))
89
- every: 10s
90
- units: %
91
- warn: $this > (($status >= $WARNING)?(200):(400))
92
- crit: $this > (($status >= $WARNING)?(400):(800))
93
- delay: down 1m multiplier 1.5 max 2h
94
- info: ratio of average Pod Lifecycle Event Generator relisting latency over the last 10 seconds, \
95
- compared to the last minute (quantile 0.9)
96
- to: sysadmin
93
+ template: kubelet_1m_pleg_relist_latency_quantile_09
94
+ on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
95
+ class: Kubernetes
96
+component: Kubelet
97
+ type: Latency
98
+ lookup: average -1m unaligned of kubelet_pleg_relist_latency_09
99
+ units: microseconds
100
+ every: 10s
101
+ info: average Pod Lifecycle Event Generator relisting latency over the last minute (quantile 0.9)
102
+
103
+ template: kubelet_10s_pleg_relist_latency_quantile_09
104
+ on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
105
+ class: Kubernetes
106
+component: Kubelet
107
+ type: Latency
108
+ lookup: average -10s unaligned of kubelet_pleg_relist_latency_09
109
+ calc: $this * 100 / (($kubelet_1m_pleg_relist_latency_quantile_09 < 1000)?(1000):($kubelet_1m_pleg_relist_latency_quantile_09))
110
+ every: 10s
111
+ units: %
112
+ warn: $this > (($status >= $WARNING)?(200):(400))
113
+ crit: $this > (($status >= $WARNING)?(400):(800))
114
+ delay: down 1m multiplier 1.5 max 2h
115
+ info: ratio of average Pod Lifecycle Event Generator relisting latency over the last 10 seconds, \
116
+ compared to the last minute (quantile 0.9)
117
+ to: sysadmin
118
119
# quantile 0.99
120
100
-template: kubelet_1m_pleg_relist_latency_quantile_099
101
- on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
102
- lookup: average -1m unaligned of kubelet_pleg_relist_latency_099
103
- units: microseconds
104
- every: 10s
105
- info: average Pod Lifecycle Event Generator relisting latency over the last minute (quantile 0.99)
106
-
107
-template: kubelet_10s_pleg_relist_latency_quantile_099
108
- on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
109
- lookup: average -10s unaligned of kubelet_pleg_relist_latency_099
110
- calc: $this * 100 / (($kubelet_1m_pleg_relist_latency_quantile_099 < 1000)?(1000):($kubelet_1m_pleg_relist_latency_quantile_099))
111
- every: 10s
112
- units: %
113
- warn: $this > (($status >= $WARNING)?(400):(800))
114
- crit: $this > (($status >= $WARNING)?(800):(1200))
115
- delay: down 1m multiplier 1.5 max 2h
116
- info: ratio of average Pod Lifecycle Event Generator relisting latency over the last 10 seconds, \
117
- compared to the last minute (quantile 0.99)
118
- to: sysadmin
121
+ template: kubelet_1m_pleg_relist_latency_quantile_099
122
+ on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
123
+ class: Kubernetes
124
+component: Kubelet
125
+ type: Latency
126
+ lookup: average -1m unaligned of kubelet_pleg_relist_latency_099
127
+ units: microseconds
128
+ every: 10s
129
+ info: average Pod Lifecycle Event Generator relisting latency over the last minute (quantile 0.99)
130
+
131
+ template: kubelet_10s_pleg_relist_latency_quantile_099
132
+ on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
133
+ class: Kubernetes
134
+component: Kubelet
135
+ type: Latency
136
+ lookup: average -10s unaligned of kubelet_pleg_relist_latency_099
137
+ calc: $this * 100 / (($kubelet_1m_pleg_relist_latency_quantile_099 < 1000)?(1000):($kubelet_1m_pleg_relist_latency_quantile_099))
138
+ every: 10s
139
+ units: %
140
+ warn: $this > (($status >= $WARNING)?(400):(800))
141
+ crit: $this > (($status >= $WARNING)?(800):(1200))
142
+ delay: down 1m multiplier 1.5 max 2h
143
+ info: ratio of average Pod Lifecycle Event Generator relisting latency over the last 10 seconds, \
144
+ compared to the last minute (quantile 0.99)
145
+ to: sysadmin
health/health.d/lighttpd.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure lighttpd is running
3
4
-template: lighttpd_last_collected_secs
5
- on: lighttpd.requests
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: lighttpd_last_collected_secs
5
+ on: lighttpd.requests
6
+ class: Web Server
7
+component: Lighttpd
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
health/health.d/linux_power_supply.conf
+13
-10
@@ -1,12 +1,15 @@
1
# Alert on low battery capacity.
2
3
-template: linux_power_supply_capacity
4
- on: powersupply.capacity
5
- calc: $capacity
6
- units: %
7
- every: 10s
8
- warn: $this < 10
9
- crit: $this < 5
10
- delay: up 30s down 5m multiplier 1.2 max 1h
11
- info: percentage of remaining power supply capacity
12
- to: sysadmin
3
+ template: linux_power_supply_capacity
4
+ on: powersupply.capacity
5
+ class: Power Supply
6
+component: Battery
7
+ type: Utilization
8
+ calc: $capacity
9
+ units: %
10
+ every: 10s
11
+ warn: $this < 10
12
+ crit: $this < 5
13
+ delay: up 30s down 5m multiplier 1.2 max 1h
14
+ info: percentage of remaining power supply capacity
15
+ to: sysadmin
health/health.d/load.conf
+53
-41
@@ -4,51 +4,63 @@
4
# Calculate the base trigger point for the load average alarms.
5
# This is the maximum number of CPU's in the system over the past 1
6
# minute, with a special case for a single CPU of setting the trigger at 2.
7
- alarm: load_cpu_number
8
- on: system.load
9
- os: linux
10
- hosts: *
11
- calc: ($active_processors == nan or $active_processors == inf or $active_processors < 2) ? ( 2 ) : ( $active_processors )
12
- units: cpus
13
- every: 1m
14
- info: number of active CPU cores in the system
7
+ alarm: load_cpu_number
8
+ on: system.load
9
+ class: System
10
+component: Load
11
+ type: Utilization
12
+ os: linux
13
+ hosts: *
14
+ calc: ($active_processors == nan or $active_processors == inf or $active_processors < 2) ? ( 2 ) : ( $active_processors )
15
+ units: cpus
16
+ every: 1m
17
+ info: number of active CPU cores in the system
18
19
# Send alarms if the load average is unusually high.
20
# These intentionally _do not_ calculate the average over the sampled
21
# time period because the values being checked already are averages.
22
20
- alarm: load_average_15
21
- on: system.load
22
- os: linux
23
- hosts: *
24
- lookup: max -1m unaligned of load15
25
- units: load
26
- every: 1m
27
- warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
28
- delay: down 15m multiplier 1.5 max 1h
29
- info: system fifteen-minute load average
30
- to: sysadmin
23
+ alarm: load_average_15
24
+ on: system.load
25
+ class: System
26
+component: Load
27
+ type: Utilization
28
+ os: linux
29
+ hosts: *
30
+ lookup: max -1m unaligned of load15
31
+ units: load
32
+ every: 1m
33
+ warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
34
+ delay: down 15m multiplier 1.5 max 1h
35
+ info: system fifteen-minute load average
36
+ to: sysadmin
37
32
- alarm: load_average_5
33
- on: system.load
34
- os: linux
35
- hosts: *
36
- lookup: max -1m unaligned of load5
37
- units: load
38
- every: 1m
39
- warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
40
- delay: down 15m multiplier 1.5 max 1h
41
- info: system five-minute load average
42
- to: sysadmin
38
+ alarm: load_average_5
39
+ on: system.load
40
+ class: System
41
+component: Load
42
+ type: Utilization
43
+ os: linux
44
+ hosts: *
45
+ lookup: max -1m unaligned of load5
46
+ units: load
47
+ every: 1m
48
+ warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
49
+ delay: down 15m multiplier 1.5 max 1h
50
+ info: system five-minute load average
51
+ to: sysadmin
52
44
- alarm: load_average_1
45
- on: system.load
46
- os: linux
47
- hosts: *
48
- lookup: max -1m unaligned of load1
49
- units: load
50
- every: 1m
51
- warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
52
- delay: down 15m multiplier 1.5 max 1h
53
- info: system one-minute load average
54
- to: sysadmin
53
+ alarm: load_average_1
54
+ on: system.load
55
+ class: System
56
+component: Load
57
+ type: Utilization
58
+ os: linux
59
+ hosts: *
60
+ lookup: max -1m unaligned of load1
61
+ units: load
62
+ every: 1m
63
+ warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
64
+ delay: down 15m multiplier 1.5 max 1h
65
+ info: system one-minute load average
66
+ to: sysadmin
health/health.d/mdstat.conf
+48
-36
@@ -1,39 +1,51 @@
1
-template: mdstat_last_collected
2
- on: md.disks
3
- calc: $now - $last_collected_t
4
- units: seconds ago
5
- every: 10s
6
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
7
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
8
- info: number of seconds since the last successful data collection
9
- to: sysadmin
1
+ template: mdstat_last_collected
2
+ on: md.disks
3
+ class: System
4
+component: RAID
5
+ type: Latency
6
+ calc: $now - $last_collected_t
7
+ units: seconds ago
8
+ every: 10s
9
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
+ info: number of seconds since the last successful data collection
12
+ to: sysadmin
13
11
-template: mdstat_disks
12
- on: md.disks
13
- units: failed devices
14
- every: 10s
15
- calc: $down
16
- crit: $this > 0
17
- info: number of devices in the down state. \
18
- Any number > 0 indicates that the array is degraded.
19
- to: sysadmin
14
+ template: mdstat_disks
15
+ on: md.disks
16
+ class: System
17
+component: RAID
18
+ type: Errors
19
+ units: failed devices
20
+ every: 10s
21
+ calc: $down
22
+ crit: $this > 0
23
+ info: number of devices in the down state for the $family array. \
24
+ Any number > 0 indicates that the array is degraded.
25
+ to: sysadmin
26
21
-template: mdstat_mismatch_cnt
22
- on: md.mismatch_cnt
23
- units: unsynchronized blocks
24
- calc: $count
25
- every: 60s
26
- warn: $this > 1024
27
- delay: up 30m
28
- info: number of unsynchronized blocks
29
- to: sysadmin
27
+ template: mdstat_mismatch_cnt
28
+ on: md.mismatch_cnt
29
+ class: System
30
+component: RAID
31
+ type: Errors
32
+ units: unsynchronized blocks
33
+ calc: $count
34
+ every: 60s
35
+ warn: $this > 1024
36
+ delay: up 30m
37
+ info: number of unsynchronized blocks for the $family array
38
+ to: sysadmin
39
31
-template: mdstat_nonredundant_last_collected
32
- on: md.nonredundant
33
- calc: $now - $last_collected_t
34
- units: seconds ago
35
- every: 10s
36
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
37
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
38
- info: number of seconds since the last successful data collection
39
- to: sysadmin
40
+ template: mdstat_nonredundant_last_collected
41
+ on: md.nonredundant
42
+ class: System
43
+component: RAID
44
+ type: Latency
45
+ calc: $now - $last_collected_t
46
+ units: seconds ago
47
+ every: 10s
48
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
49
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
50
+ info: number of seconds since the last successful data collection
51
+ to: sysadmin
health/health.d/megacli.conf
+62
-47
@@ -1,56 +1,71 @@
1
2
## Adapters (controllers)
3
4
-template: megacli_adapter_state
5
- on: megacli.adapter_degraded
6
- lookup: max -10s foreach *
7
- units: boolean
8
- every: 10s
9
- crit: $this > 0
10
- delay: down 5m multiplier 2 max 10m
11
- info: adapter is in the degraded state (0: false, 1: true)
12
- to: sysadmin
4
+ template: megacli_adapter_state
5
+ on: megacli.adapter_degraded
6
+ class: System
7
+component: RAID
8
+ type: Errors
9
+ lookup: max -10s foreach *
10
+ units: boolean
11
+ every: 10s
12
+ crit: $this > 0
13
+ delay: down 5m multiplier 2 max 10m
14
+ info: adapter is in the degraded state (0: false, 1: true)
15
+ to: sysadmin
16
17
## Physical Disks
18
16
-template: megacli_pd_predictive_failures
17
- on: megacli.pd_predictive_failure
18
- lookup: sum -10s foreach *
19
- units: predictive failures
20
- every: 10s
21
- warn: $this > 0
22
- delay: up 1m down 5m multiplier 2 max 10m
23
- info: number of physical drive predictive failures
24
- to: sysadmin
25
-
26
-template: megacli_pd_media_errors
27
- on: megacli.pd_media_error
28
- lookup: sum -10s foreach *
29
- units: media errors
30
- every: 10s
31
- warn: $this > 0
32
- delay: up 1m down 5m multiplier 2 max 10m
33
- info: number of physical drive media errors
34
- to: sysadmin
19
+ template: megacli_pd_predictive_failures
20
+ on: megacli.pd_predictive_failure
21
+ class: System
22
+component: RAID
23
+ type: Errors
24
+ lookup: sum -10s foreach *
25
+ units: predictive failures
26
+ every: 10s
27
+ warn: $this > 0
28
+ delay: up 1m down 5m multiplier 2 max 10m
29
+ info: number of physical drive predictive failures
30
+ to: sysadmin
31
+
32
+ template: megacli_pd_media_errors
33
+ on: megacli.pd_media_error
34
+ class: System
35
+component: RAID
36
+ type: Errors
37
+ lookup: sum -10s foreach *
38
+ units: media errors
39
+ every: 10s
40
+ warn: $this > 0
41
+ delay: up 1m down 5m multiplier 2 max 10m
42
+ info: number of physical drive media errors
43
+ to: sysadmin
44
45
## Battery Backup Units (BBU)
46
38
-template: megacli_bbu_relative_charge
39
- on: megacli.bbu_relative_charge
40
- lookup: average -10s
41
- units: percent
42
- every: 10s
43
- warn: $this <= (($status >= $WARNING) ? (85) : (80))
44
- crit: $this <= (($status == $CRITICAL) ? (50) : (40))
45
- info: average battery backup unit (BBU) relative state of charge over the last 10 seconds
46
- to: sysadmin
47
-
48
-template: megacli_bbu_cycle_count
49
- on: megacli.bbu_cycle_count
50
- lookup: average -10s
51
- units: cycles
52
- every: 10s
53
- warn: $this >= 100
54
- crit: $this >= 500
55
- info: average battery backup unit (BBU) charge cycles count over the last 10 seconds
56
- to: sysadmin
47
+ template: megacli_bbu_relative_charge
48
+ on: megacli.bbu_relative_charge
49
+ class: System
50
+component: RAID
51
+ type: Workload
52
+ lookup: average -10s
53
+ units: percent
54
+ every: 10s
55
+ warn: $this <= (($status >= $WARNING) ? (85) : (80))
56
+ crit: $this <= (($status == $CRITICAL) ? (50) : (40))
57
+ info: average battery backup unit (BBU) relative state of charge over the last 10 seconds
58
+ to: sysadmin
59
+
60
+ template: megacli_bbu_cycle_count
61
+ on: megacli.bbu_cycle_count
62
+ class: System
63
+component: RAID
64
+ type: Workload
65
+ lookup: average -10s
66
+ units: cycles
67
+ every: 10s
68
+ warn: $this >= 100
69
+ crit: $this >= 500
70
+ info: average battery backup unit (BBU) charge cycles count over the last 10 seconds
71
+ to: sysadmin
health/health.d/memcached.conf
+50
-38
@@ -1,53 +1,65 @@
1
2
# make sure memcached is running
3
4
-template: memcached_last_collected_secs
5
- on: memcached.cache
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: memcached_last_collected_secs
5
+ on: memcached.cache
6
+ class: KV Storage
7
+component: Memcached
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
17
18
19
# detect if memcached cache is full
20
18
-template: memcached_cache_memory_usage
19
- on: memcached.cache
20
- calc: $used * 100 / ($used + $available)
21
- units: %
22
- every: 10s
23
- warn: $this > (($status >= $WARNING) ? (70) : (80))
24
- crit: $this > (($status == $CRITICAL) ? (80) : (90))
25
- delay: up 0 down 15m multiplier 1.5 max 1h
26
- info: cache memory utilization
27
- to: dba
21
+ template: memcached_cache_memory_usage
22
+ on: memcached.cache
23
+ class: KV Storage
24
+component: Memcached
25
+ type: Utilization
26
+ calc: $used * 100 / ($used + $available)
27
+ units: %
28
+ every: 10s
29
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
30
+ crit: $this > (($status == $CRITICAL) ? (80) : (90))
31
+ delay: up 0 down 15m multiplier 1.5 max 1h
32
+ info: cache memory utilization
33
+ to: dba
34
35
36
# find the rate memcached cache is filling
37
32
-template: memcached_cache_fill_rate
33
- on: memcached.cache
34
- lookup: min -10m at -50m unaligned of available
35
- calc: ($this - $available) / (($now - $after) / 3600)
36
- units: KB/hour
37
- every: 1m
38
- info: average rate the cache fills up (positive), or frees up (negative) space over the last hour
38
+ template: memcached_cache_fill_rate
39
+ on: memcached.cache
40
+ class: KV Storage
41
+component: Memcached
42
+ type: Utilization
43
+ lookup: min -10m at -50m unaligned of available
44
+ calc: ($this - $available) / (($now - $after) / 3600)
45
+ units: KB/hour
46
+ every: 1m
47
+ info: average rate the cache fills up (positive), or frees up (negative) space over the last hour
48
49
50
# find the hours remaining until memcached cache is full
51
43
-template: memcached_out_of_cache_space_time
44
- on: memcached.cache
45
- calc: ($memcached_cache_fill_rate > 0) ? ($available / $memcached_cache_fill_rate) : (inf)
46
- units: hours
47
- every: 10s
48
- warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
49
- crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
50
- delay: down 15m multiplier 1.5 max 1h
51
- info: estimated time the cache will run out of space \
52
- if the system continues to add data at the same rate as the past hour
53
- to: dba
52
+ template: memcached_out_of_cache_space_time
53
+ on: memcached.cache
54
+ class: KV Storage
55
+component: Memcached
56
+ type: Utilization
57
+ calc: ($memcached_cache_fill_rate > 0) ? ($available / $memcached_cache_fill_rate) : (inf)
58
+ units: hours
59
+ every: 10s
60
+ warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
61
+ crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
62
+ delay: down 15m multiplier 1.5 max 1h
63
+ info: estimated time the cache will run out of space \
64
+ if the system continues to add data at the same rate as the past hour
65
+ to: dba
health/health.d/memory.conf
+42
-33
@@ -1,38 +1,47 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: 1hour_ecc_memory_correctable
5
- on: mem.ecc_ce
6
- os: linux
7
- hosts: *
8
- lookup: sum -10m unaligned
9
- units: errors
10
- every: 1m
11
- warn: $this > 0
12
- delay: down 1h multiplier 1.5 max 1h
13
- info: number of ECC correctable errors in the last 10 minutes
14
- to: sysadmin
4
+ alarm: 1hour_ecc_memory_correctable
5
+ on: mem.ecc_ce
6
+ class: System
7
+component: Memory
8
+ type: Errors
9
+ os: linux
10
+ hosts: *
11
+ lookup: sum -10m unaligned
12
+ units: errors
13
+ every: 1m
14
+ warn: $this > 0
15
+ delay: down 1h multiplier 1.5 max 1h
16
+ info: number of ECC correctable errors in the last 10 minutes
17
+ to: sysadmin
18
16
- alarm: 1hour_ecc_memory_uncorrectable
17
- on: mem.ecc_ue
18
- os: linux
19
- hosts: *
20
- lookup: sum -10m unaligned
21
- units: errors
22
- every: 1m
23
- crit: $this > 0
24
- delay: down 1h multiplier 1.5 max 1h
25
- info: number of ECC uncorrectable errors in the last 10 minutes
26
- to: sysadmin
19
+ alarm: 1hour_ecc_memory_uncorrectable
20
+ on: mem.ecc_ue
21
+ class: System
22
+component: Memory
23
+ type: Errors
24
+ os: linux
25
+ hosts: *
26
+ lookup: sum -10m unaligned
27
+ units: errors
28
+ every: 1m
29
+ crit: $this > 0
30
+ delay: down 1h multiplier 1.5 max 1h
31
+ info: number of ECC uncorrectable errors in the last 10 minutes
32
+ to: sysadmin
33
28
- alarm: 1hour_memory_hw_corrupted
29
- on: mem.hwcorrupt
30
- os: linux
31
- hosts: *
32
- calc: $HardwareCorrupted
33
- units: MB
34
- every: 10s
35
- warn: $this > 0
36
- delay: down 1h multiplier 1.5 max 1h
37
- info: amount of memory corrupted due to a hardware failure
38
- to: sysadmin
34
+ alarm: 1hour_memory_hw_corrupted
35
+ on: mem.hwcorrupt
36
+ class: System
37
+component: Memory
38
+ type: Errors
39
+ os: linux
40
+ hosts: *
41
+ calc: $HardwareCorrupted
42
+ units: MB
43
+ every: 10s
44
+ warn: $this > 0
45
+ delay: down 1h multiplier 1.5 max 1h
46
+ info: amount of memory corrupted due to a hardware failure
47
+ to: sysadmin
health/health.d/mongodb.conf
+13
-10
@@ -1,13 +1,16 @@
1
2
# make sure mongodb is running
3
4
-template: mongodb_last_collected_secs
5
- on: mongodb.read_operations
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: mongodb_last_collected_secs
5
+ on: mongodb.read_operations
6
+ class: Database
7
+component: MongoDB
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
health/health.d/mysql.conf
+151
-115
@@ -1,150 +1,186 @@
1
2
# make sure mysql is running
3
4
-template: mysql_last_collected_secs
5
- on: mysql.queries
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: mysql_last_collected_secs
5
+ on: mysql.queries
6
+ class: Database
7
+component: MySQL
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
17
18
19
# -----------------------------------------------------------------------------
20
# slow queries
21
19
-template: mysql_10s_slow_queries
20
- on: mysql.queries
21
- lookup: sum -10s of slow_queries
22
- units: slow queries
23
- every: 10s
24
- warn: $this > (($status >= $WARNING) ? (5) : (10))
25
- crit: $this > (($status == $CRITICAL) ? (10) : (20))
26
- delay: down 5m multiplier 1.5 max 1h
27
- info: number of slow queries in the last 10 seconds
28
- to: dba
22
+ template: mysql_10s_slow_queries
23
+ on: mysql.queries
24
+ class: Database
25
+component: MySQL
26
+ type: Latency
27
+ lookup: sum -10s of slow_queries
28
+ units: slow queries
29
+ every: 10s
30
+ warn: $this > (($status >= $WARNING) ? (5) : (10))
31
+ crit: $this > (($status == $CRITICAL) ? (10) : (20))
32
+ delay: down 5m multiplier 1.5 max 1h
33
+ info: number of slow queries in the last 10 seconds
34
+ to: dba
35
36
37
# -----------------------------------------------------------------------------
38
# lock waits
39
34
-template: mysql_10s_table_locks_immediate
35
- on: mysql.table_locks
36
- lookup: sum -10s absolute of immediate
37
- units: immediate locks
38
- every: 10s
39
- info: number of table immediate locks in the last 10 seconds
40
- to: dba
41
-
42
-template: mysql_10s_table_locks_waited
43
- on: mysql.table_locks
44
- lookup: sum -10s absolute of waited
45
- units: waited locks
46
- every: 10s
47
- info: number of table waited locks in the last 10 seconds
48
- to: dba
49
-
50
-template: mysql_10s_waited_locks_ratio
51
- on: mysql.table_locks
52
- calc: ( ($mysql_10s_table_locks_waited + $mysql_10s_table_locks_immediate) > 0 ) ? (($mysql_10s_table_locks_waited * 100) / ($mysql_10s_table_locks_waited + $mysql_10s_table_locks_immediate)) : 0
53
- units: %
54
- every: 10s
55
- warn: $this > (($status >= $WARNING) ? (10) : (25))
56
- crit: $this > (($status == $CRITICAL) ? (25) : (50))
57
- delay: down 30m multiplier 1.5 max 1h
58
- info: ratio of waited table locks over the last 10 seconds
59
- to: dba
40
+ template: mysql_10s_table_locks_immediate
41
+ on: mysql.table_locks
42
+ class: Database
43
+component: MySQL
44
+ type: Utilization
45
+ lookup: sum -10s absolute of immediate
46
+ units: immediate locks
47
+ every: 10s
48
+ info: number of table immediate locks in the last 10 seconds
49
+ to: dba
50
+
51
+ template: mysql_10s_table_locks_waited
52
+ on: mysql.table_locks
53
+ class: Database
54
+component: MySQL
55
+ type: Latency
56
+ lookup: sum -10s absolute of waited
57
+ units: waited locks
58
+ every: 10s
59
+ info: number of table waited locks in the last 10 seconds
60
+ to: dba
61
+
62
+ template: mysql_10s_waited_locks_ratio
63
+ on: mysql.table_locks
64
+ class: Database
65
+component: MySQL
66
+ type: Latency
67
+ calc: ( ($mysql_10s_table_locks_waited + $mysql_10s_table_locks_immediate) > 0 ) ? (($mysql_10s_table_locks_waited * 100) / ($mysql_10s_table_locks_waited + $mysql_10s_table_locks_immediate)) : 0
68
+ units: %
69
+ every: 10s
70
+ warn: $this > (($status >= $WARNING) ? (10) : (25))
71
+ crit: $this > (($status == $CRITICAL) ? (25) : (50))
72
+ delay: down 30m multiplier 1.5 max 1h
73
+ info: ratio of waited table locks over the last 10 seconds
74
+ to: dba
75
76
77
# -----------------------------------------------------------------------------
78
# connections
79
65
-template: mysql_connections
66
- on: mysql.connections_active
67
- calc: $active * 100 / $limit
68
- units: %
69
- every: 10s
70
- warn: $this > (($status >= $WARNING) ? (60) : (70))
71
- crit: $this > (($status == $CRITICAL) ? (80) : (90))
72
- delay: down 15m multiplier 1.5 max 1h
73
- info: client connections utilization
74
- to: dba
80
+ template: mysql_connections
81
+ on: mysql.connections_active
82
+ class: Database
83
+component: MySQL
84
+ type: Utilization
85
+ calc: $active * 100 / $limit
86
+ units: %
87
+ every: 10s
88
+ warn: $this > (($status >= $WARNING) ? (60) : (70))
89
+ crit: $this > (($status == $CRITICAL) ? (80) : (90))
90
+ delay: down 15m multiplier 1.5 max 1h
91
+ info: client connections utilization
92
+ to: dba
93
94
95
# -----------------------------------------------------------------------------
96
# replication
97
80
-template: mysql_replication
81
- on: mysql.slave_status
82
- calc: ($sql_running <= 0 OR $io_running <= 0)?0:1
83
- units: ok/failed
84
- every: 10s
85
- crit: $this == 0
86
- delay: down 5m multiplier 1.5 max 1h
87
- info: replication status (0: stopped, 1: working)
88
- to: dba
89
-
90
-template: mysql_replication_lag
91
- on: mysql.slave_behind
92
- calc: $seconds
93
- units: seconds
94
- every: 10s
95
- warn: $this > (($status >= $WARNING) ? (5) : (10))
96
- crit: $this > (($status == $CRITICAL) ? (10) : (30))
97
- delay: down 15m multiplier 1.5 max 1h
98
- info: difference between the timestamp of the latest transaction processed by the SQL thread and \
99
- the timestamp of the same transaction when it was processed on the master
100
- to: dba
98
+ template: mysql_replication
99
+ on: mysql.slave_status
100
+ class: Database
101
+component: MySQL
102
+ type: Errors
103
+ calc: ($sql_running <= 0 OR $io_running <= 0)?0:1
104
+ units: ok/failed
105
+ every: 10s
106
+ crit: $this == 0
107
+ delay: down 5m multiplier 1.5 max 1h
108
+ info: replication status (0: stopped, 1: working)
109
+ to: dba
110
+
111
+ template: mysql_replication_lag
112
+ on: mysql.slave_behind
113
+ class: Database
114
+component: MySQL
115
+ type: Errors
116
+ calc: $seconds
117
+ units: seconds
118
+ every: 10s
119
+ warn: $this > (($status >= $WARNING) ? (5) : (10))
120
+ crit: $this > (($status == $CRITICAL) ? (10) : (30))
121
+ delay: down 15m multiplier 1.5 max 1h
122
+ info: difference between the timestamp of the latest transaction processed by the SQL thread and \
123
+ the timestamp of the same transaction when it was processed on the master
124
+ to: dba
125
126
127
# -----------------------------------------------------------------------------
128
# galera cluster size
129
106
-template: mysql_galera_cluster_size_max_2m
107
- on: mysql.galera_cluster_size
108
- lookup: max -2m absolute
109
- units: nodes
110
- every: 10s
111
- info: maximum galera cluster size in the last 2 minutes
112
- to: dba
113
-
114
-template: mysql_galera_cluster_size
115
- on: mysql.galera_cluster_size
116
- calc: $nodes
117
- units: nodes
118
- every: 10s
119
- warn: $this > $mysql_galera_cluster_size_max_2m
120
- crit: $this < $mysql_galera_cluster_size_max_2m
121
- delay: up 20s down 5m multiplier 1.5 max 1h
122
- info: current galera cluster size, compared to the maximum size in the last 2 minutes
123
- to: dba
130
+ template: mysql_galera_cluster_size_max_2m
131
+ on: mysql.galera_cluster_size
132
+ class: Database
133
+component: MySQL
134
+ type: Utilization
135
+ lookup: max -2m absolute
136
+ units: nodes
137
+ every: 10s
138
+ info: maximum galera cluster size in the last 2 minutes
139
+ to: dba
140
+
141
+ template: mysql_galera_cluster_size
142
+ on: mysql.galera_cluster_size
143
+ class: Database
144
+component: MySQL
145
+ type: Utilization
146
+ calc: $nodes
147
+ units: nodes
148
+ every: 10s
149
+ warn: $this > $mysql_galera_cluster_size_max_2m
150
+ crit: $this < $mysql_galera_cluster_size_max_2m
151
+ delay: up 20s down 5m multiplier 1.5 max 1h
152
+ info: current galera cluster size, compared to the maximum size in the last 2 minutes
153
+ to: dba
154
155
# galera node state
156
127
-template: mysql_galera_cluster_state
128
- on: mysql.galera_cluster_state
129
- calc: $state
130
- every: 10s
131
- warn: $this == 2 OR $this == 3
132
- crit: $this == 0 OR $this == 1 OR $this >= 5
133
- delay: up 30s down 5m multiplier 1.5 max 1h
134
- info: galera node state \
135
- (0: Undefined, 1: Joining, 2: Donor/Desynced, 3: Joined, 4: Synced, 5: Inconsistent)
136
- to: dba
157
+ template: mysql_galera_cluster_state
158
+ on: mysql.galera_cluster_state
159
+ class: Database
160
+component: MySQL
161
+ type: Errors
162
+ calc: $state
163
+ every: 10s
164
+ warn: $this == 2 OR $this == 3
165
+ crit: $this == 0 OR $this == 1 OR $this >= 5
166
+ delay: up 30s down 5m multiplier 1.5 max 1h
167
+ info: galera node state \
168
+ (0: Undefined, 1: Joining, 2: Donor/Desynced, 3: Joined, 4: Synced, 5: Inconsistent)
169
+ to: dba
170
171
172
# galera node status
173
141
-template: mysql_galera_cluster_status
142
- on: mysql.galera_cluster_status
143
- calc: $wsrep_cluster_status
144
- every: 10s
145
- crit: $mysql_galera_cluster_state != nan AND $this != 0
146
- delay: up 30s down 5m multiplier 1.5 max 1h
147
- info: galera node cluster component status \
148
- (-1: unknown, 0: primary/quorum present, 1: non-primary/quorum lost, 2: disconnected). \
149
- Any other value than primary indicates that the node is part of a nonoperational component.
150
- to: dba
174
+ template: mysql_galera_cluster_status
175
+ on: mysql.galera_cluster_status
176
+ class: Database
177
+component: MySQL
178
+ type: Errors
179
+ calc: $wsrep_cluster_status
180
+ every: 10s
181
+ crit: $mysql_galera_cluster_state != nan AND $this != 0
182
+ delay: up 30s down 5m multiplier 1.5 max 1h
183
+ info: galera node cluster component status \
184
+ (-1: unknown, 0: primary/quorum present, 1: non-primary/quorum lost, 2: disconnected). \
185
+ Any other value than primary indicates that the node is part of a nonoperational component.
186
+ to: dba
health/health.d/named.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure named is running
3
4
-template: named_last_collected_secs
5
- on: named.global_queries
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: domainadmin
4
+ template: named_last_collected_secs
5
+ on: named.global_queries
6
+ class: DNS
7
+component: BIND
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: domainadmin
17
health/health.d/net.conf
+182
-140
@@ -6,16 +6,22 @@
6
7
template: interface_speed
8
on: net.net
9
+ class: System
10
+component: Network
11
+ type: Latency
12
os: *
13
hosts: *
14
families: *
15
calc: ( $nic_speed_max > 0 ) ? ( $nic_speed_max) : ( nan )
16
units: Mbit
17
every: 10s
15
- info: network interface current speed
18
+ info: network interface $family current speed
19
20
template: 1m_received_traffic_overflow
21
on: net.net
22
+ class: System
23
+component: Network
24
+ type: Workload
25
os: linux
26
hosts: *
27
families: *
@@ -25,11 +31,14 @@
31
every: 10s
32
warn: $this > (($status >= $WARNING) ? (85) : (90))
33
delay: up 1m down 1m multiplier 1.5 max 1h
28
- info: average inbound utilization for the network interface over the last minute
34
+ info: average inbound utilization for the network interface $family over the last minute
35
to: sysadmin
36
37
template: 1m_sent_traffic_overflow
38
on: net.net
39
+ class: System
40
+component: Network
41
+ type: Workload
42
os: linux
43
hosts: *
44
families: *
@@ -39,7 +48,7 @@
48
every: 10s
49
warn: $this > (($status >= $WARNING) ? (85) : (90))
50
delay: up 1m down 1m multiplier 1.5 max 1h
42
- info: average outbound utilization for the network interface over the last minute
51
+ info: average outbound utilization for the network interface $family over the last minute
52
to: sysadmin
53
54
# -----------------------------------------------------------------------------
@@ -52,110 +61,134 @@
61
# it is possible to have expected packet drops on an interface for some network configurations
62
# look at the Monitoring Network Interfaces section in the proc.plugin documentation for more information
63
55
-template: inbound_packets_dropped
56
- on: net.drops
57
- os: linux
58
- hosts: *
59
-families: !net* *
60
- lookup: sum -10m unaligned absolute of inbound
61
- units: packets
62
- every: 1m
63
- info: number of inbound dropped packets for the network interface in the last 10 minutes
64
-
65
-template: outbound_packets_dropped
66
- on: net.drops
67
- os: linux
68
- hosts: *
69
-families: !net* *
70
- lookup: sum -10m unaligned absolute of outbound
71
- units: packets
72
- every: 1m
73
- info: number of outbound dropped packets for the network interface in the last 10 minutes
74
-
75
-template: inbound_packets_dropped_ratio
76
- on: net.packets
77
- os: linux
78
- hosts: *
79
-families: !net* !wl* *
80
- lookup: sum -10m unaligned absolute of received
81
- calc: (($inbound_packets_dropped != nan AND $this > 1000) ? ($inbound_packets_dropped * 100 / $this) : (0))
82
- units: %
83
- every: 1m
84
- warn: $this >= 2
85
- delay: up 1m down 1h multiplier 1.5 max 2h
86
- info: ratio of inbound dropped packets for the network interface over the last 10 minutes
87
- to: sysadmin
88
-
89
-template: outbound_packets_dropped_ratio
90
- on: net.packets
91
- os: linux
92
- hosts: *
93
-families: !net* !wl* *
94
- lookup: sum -10m unaligned absolute of sent
95
- calc: (($outbound_packets_dropped != nan AND $this > 1000) ? ($outbound_packets_dropped * 100 / $this) : (0))
96
- units: %
97
- every: 1m
98
- warn: $this >= 2
99
- delay: up 1m down 1h multiplier 1.5 max 2h
100
- info: ratio of outbound dropped packets for the network interface over the last 10 minutes
101
- to: sysadmin
102
-
103
-template: wifi_inbound_packets_dropped_ratio
104
- on: net.packets
105
- os: linux
106
- hosts: *
107
-families: wl*
108
- lookup: sum -10m unaligned absolute of received
109
- calc: (($inbound_packets_dropped != nan AND $this > 1000) ? ($inbound_packets_dropped * 100 / $this) : (0))
110
- units: %
111
- every: 1m
112
- warn: $this >= 10
113
- delay: up 1m down 1h multiplier 1.5 max 2h
114
- info: ratio of inbound dropped packets for the network interface over the last 10 minutes
115
- to: sysadmin
116
-
117
-template: wifi_outbound_packets_dropped_ratio
118
- on: net.packets
119
- os: linux
120
- hosts: *
121
-families: wl*
122
- lookup: sum -10m unaligned absolute of sent
123
- calc: (($outbound_packets_dropped != nan AND $this > 1000) ? ($outbound_packets_dropped * 100 / $this) : (0))
124
- units: %
125
- every: 1m
126
- warn: $this >= 10
127
- delay: up 1m down 1h multiplier 1.5 max 2h
128
- info: ratio of outbound dropped packets for the network interface over the last 10 minutes
129
- to: sysadmin
64
+ template: inbound_packets_dropped
65
+ on: net.drops
66
+ class: System
67
+component: Network
68
+ type: Errors
69
+ os: linux
70
+ hosts: *
71
+ families: !net* *
72
+ lookup: sum -10m unaligned absolute of inbound
73
+ units: packets
74
+ every: 1m
75
+ info: number of inbound dropped packets for the network interface $family in the last 10 minutes
76
+
77
+ template: outbound_packets_dropped
78
+ on: net.drops
79
+ class: System
80
+component: Network
81
+ type: Errors
82
+ os: linux
83
+ hosts: *
84
+ families: !net* *
85
+ lookup: sum -10m unaligned absolute of outbound
86
+ units: packets
87
+ every: 1m
88
+ info: number of outbound dropped packets for the network interface $family in the last 10 minutes
89
+
90
+ template: inbound_packets_dropped_ratio
91
+ on: net.packets
92
+ class: System
93
+component: Network
94
+ type: Errors
95
+ os: linux
96
+ hosts: *
97
+ families: !net* !wl* *
98
+ lookup: sum -10m unaligned absolute of received
99
+ calc: (($inbound_packets_dropped != nan AND $this > 1000) ? ($inbound_packets_dropped * 100 / $this) : (0))
100
+ units: %
101
+ every: 1m
102
+ warn: $this >= 2
103
+ delay: up 1m down 1h multiplier 1.5 max 2h
104
+ info: ratio of inbound dropped packets for the network interface $family over the last 10 minutes
105
+ to: sysadmin
106
+
107
+ template: outbound_packets_dropped_ratio
108
+ on: net.packets
109
+ class: System
110
+component: Network
111
+ type: Errors
112
+ os: linux
113
+ hosts: *
114
+ families: !net* !wl* *
115
+ lookup: sum -10m unaligned absolute of sent
116
+ calc: (($outbound_packets_dropped != nan AND $this > 1000) ? ($outbound_packets_dropped * 100 / $this) : (0))
117
+ units: %
118
+ every: 1m
119
+ warn: $this >= 2
120
+ delay: up 1m down 1h multiplier 1.5 max 2h
121
+ info: ratio of outbound dropped packets for the network interface $family over the last 10 minutes
122
+ to: sysadmin
123
+
124
+ template: wifi_inbound_packets_dropped_ratio
125
+ on: net.packets
126
+ class: System
127
+component: Network
128
+ type: Errors
129
+ os: linux
130
+ hosts: *
131
+ families: wl*
132
+ lookup: sum -10m unaligned absolute of received
133
+ calc: (($inbound_packets_dropped != nan AND $this > 1000) ? ($inbound_packets_dropped * 100 / $this) : (0))
134
+ units: %
135
+ every: 1m
136
+ warn: $this >= 10
137
+ delay: up 1m down 1h multiplier 1.5 max 2h
138
+ info: ratio of inbound dropped packets for the network interface $family over the last 10 minutes
139
+ to: sysadmin
140
+
141
+ template: wifi_outbound_packets_dropped_ratio
142
+ on: net.packets
143
+ class: System
144
+component: Network
145
+ type: Errors
146
+ os: linux
147
+ hosts: *
148
+ families: wl*
149
+ lookup: sum -10m unaligned absolute of sent
150
+ calc: (($outbound_packets_dropped != nan AND $this > 1000) ? ($outbound_packets_dropped * 100 / $this) : (0))
151
+ units: %
152
+ every: 1m
153
+ warn: $this >= 10
154
+ delay: up 1m down 1h multiplier 1.5 max 2h
155
+ info: ratio of outbound dropped packets for the network interface $family over the last 10 minutes
156
+ to: sysadmin
157
158
# -----------------------------------------------------------------------------
159
# interface errors
160
134
-template: interface_inbound_errors
135
- on: net.errors
136
- os: freebsd
137
- hosts: *
138
-families: *
139
- lookup: sum -10m unaligned absolute of inbound
140
- units: errors
141
- every: 1m
142
- warn: $this >= 5
143
- delay: down 1h multiplier 1.5 max 2h
144
- info: number of inbound errors for the network interface in the last 10 minutes
145
- to: sysadmin
146
-
147
-template: interface_outbound_errors
148
- on: net.errors
149
- os: freebsd
150
- hosts: *
151
-families: *
152
- lookup: sum -10m unaligned absolute of outbound
153
- units: errors
154
- every: 1m
155
- warn: $this >= 5
156
- delay: down 1h multiplier 1.5 max 2h
157
- info: number of outbound errors for the network interface in the last 10 minutes
158
- to: sysadmin
161
+ template: interface_inbound_errors
162
+ on: net.errors
163
+ class: System
164
+component: Network
165
+ type: Errors
166
+ os: freebsd
167
+ hosts: *
168
+ families: *
169
+ lookup: sum -10m unaligned absolute of inbound
170
+ units: errors
171
+ every: 1m
172
+ warn: $this >= 5
173
+ delay: down 1h multiplier 1.5 max 2h
174
+ info: number of inbound errors for the network interface $family in the last 10 minutes
175
+ to: sysadmin
176
+
177
+ template: interface_outbound_errors
178
+ on: net.errors
179
+ class: System
180
+component: Network
181
+ type: Errors
182
+ os: freebsd
183
+ hosts: *
184
+ families: *
185
+ lookup: sum -10m unaligned absolute of outbound
186
+ units: errors
187
+ every: 1m
188
+ warn: $this >= 5
189
+ delay: down 1h multiplier 1.5 max 2h
190
+ info: number of outbound errors for the network interface $family in the last 10 minutes
191
+ to: sysadmin
192
193
# -----------------------------------------------------------------------------
194
# FIFO errors
@@ -165,18 +198,21 @@ families: *
198
# the alarm is checked every 1 minute
199
# and examines the last 10 minutes of data
200
168
-template: 10min_fifo_errors
169
- on: net.fifo
170
- os: linux
171
- hosts: *
172
-families: *
173
- lookup: sum -10m unaligned absolute
174
- units: errors
175
- every: 1m
176
- warn: $this > 0
177
- delay: down 1h multiplier 1.5 max 2h
178
- info: number of FIFO errors for the network interface in the last 10 minutes
179
- to: sysadmin
201
+ template: 10min_fifo_errors
202
+ on: net.fifo
203
+ class: System
204
+component: Network
205
+ type: Errors
206
+ os: linux
207
+ hosts: *
208
+ families: *
209
+ lookup: sum -10m unaligned absolute
210
+ units: errors
211
+ every: 1m
212
+ warn: $this > 0
213
+ delay: down 1h multiplier 1.5 max 2h
214
+ info: number of FIFO errors for the network interface $family in the last 10 minutes
215
+ to: sysadmin
216
217
# -----------------------------------------------------------------------------
218
# check for packet storms
@@ -187,28 +223,34 @@ families: *
223
# we assume the minimum packet storm should at least have
224
# 10000 packets/s, average of the last 10 seconds
225
190
-template: 1m_received_packets_rate
191
- on: net.packets
192
- os: linux freebsd
193
- hosts: *
194
-families: *
195
- lookup: average -1m unaligned of received
196
- units: packets
197
- every: 10s
198
- info: average number of packets received by the network interface over the last minute
199
-
200
-template: 10s_received_packets_storm
201
- on: net.packets
202
- os: linux freebsd
203
- hosts: *
204
-families: *
205
- lookup: average -10s unaligned of received
206
- calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
207
- every: 10s
208
- units: %
209
- warn: $this > (($status >= $WARNING)?(200):(5000))
210
- crit: $this > (($status == $CRITICAL)?(5000):(6000))
211
- options: no-clear-notification
212
- info: ratio of average number of received packets for the network interface over the last 10 seconds, \
213
- compared to the rate over the last minute
214
- to: sysadmin
226
+ template: 1m_received_packets_rate
227
+ on: net.packets
228
+ class: System
229
+component: Network
230
+ type: Workload
231
+ os: linux freebsd
232
+ hosts: *
233
+ families: *
234
+ lookup: average -1m unaligned of received
235
+ units: packets
236
+ every: 10s
237
+ info: average number of packets received by the network interface $family over the last minute
238
+
239
+ template: 10s_received_packets_storm
240
+ on: net.packets
241
+ class: System
242
+component: Network
243
+ type: Workload
244
+ os: linux freebsd
245
+ hosts: *
246
+ families: *
247
+ lookup: average -10s unaligned of received
248
+ calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
249
+ every: 10s
250
+ units: %
251
+ warn: $this > (($status >= $WARNING)?(200):(5000))
252
+ crit: $this > (($status == $CRITICAL)?(5000):(6000))
253
+ options: no-clear-notification
254
+ info: ratio of average number of received packets for the network interface $family over the last 10 seconds, \
255
+ compared to the rate over the last minute
256
+ to: sysadmin
health/health.d/netfilter.conf
+16
-13
@@ -1,16 +1,19 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: netfilter_conntrack_full
5
- on: netfilter.conntrack_sockets
6
- os: linux
7
- hosts: *
8
- lookup: max -10s unaligned of connections
9
- calc: $this * 100 / $netfilter_conntrack_max
10
- units: %
11
- every: 10s
12
- warn: $this > (($status >= $WARNING) ? (85) : (90))
13
- crit: $this > (($status == $CRITICAL) ? (90) : (95))
14
- delay: down 5m multiplier 1.5 max 1h
15
- info: netfilter connection tracker table size utilization
16
- to: sysadmin
4
+ alarm: netfilter_conntrack_full
5
+ on: netfilter.conntrack_sockets
6
+ class: System
7
+component: Network
8
+ type: Workload
9
+ os: linux
10
+ hosts: *
11
+ lookup: max -10s unaligned of connections
12
+ calc: $this * 100 / $netfilter_conntrack_max
13
+ units: %
14
+ every: 10s
15
+ warn: $this > (($status >= $WARNING) ? (85) : (90))
16
+ crit: $this > (($status == $CRITICAL) ? (90) : (95))
17
+ delay: down 5m multiplier 1.5 max 1h
18
+ info: netfilter connection tracker table size utilization
19
+ to: sysadmin
health/health.d/nginx.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure nginx is running
3
4
-template: nginx_last_collected_secs
5
- on: nginx.requests
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: nginx_last_collected_secs
5
+ on: nginx.requests
6
+ class: Web Server
7
+component: NGINX
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
health/health.d/nginx_plus.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure nginx_plus is running
3
4
-template: nginx_plus_last_collected_secs
5
- on: nginx_plus.requests_total
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: nginx_plus_last_collected_secs
5
+ on: nginx_plus.requests_total
6
+ class: Web Server
7
+component: NGINX Plus
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
health/health.d/phpfpm.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure phpfpm is running
3
4
-template: phpfpm_last_collected_secs
5
- on: phpfpm.requests
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: phpfpm_last_collected_secs
5
+ on: phpfpm.requests
6
+ class: Web Server
7
+component: PHP-FPM
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
health/health.d/pihole.conf
+62
-47
@@ -1,65 +1,80 @@
1
2
# Make sure Pi-hole is responding.
3
4
-template: pihole_last_collected_secs
5
- on: pihole.dns_queries_total
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: pihole_last_collected_secs
5
+ on: pihole.dns_queries_total
6
+ class: Ad Filtering
7
+component: Pi-hole
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
18
# Blocked DNS queries.
19
17
-template: pihole_blocked_queries
18
- on: pihole.dns_queries_percentage
19
- every: 10s
20
- units: %
21
- calc: $blocked
22
- warn: $this > ( ($status >= $WARNING ) ? ( 45 ) : ( 55 ) )
23
- crit: $this > ( ($status == $CRITICAL) ? ( 55 ) : ( 75 ) )
24
- delay: up 2m down 5m
25
- info: percentage of blocked dns queries over the last 24 hour
26
- to: sysadmin
20
+ template: pihole_blocked_queries
21
+ on: pihole.dns_queries_percentage
22
+ class: Ad Filtering
23
+component: Pi-hole
24
+ type: Errors
25
+ every: 10s
26
+ units: %
27
+ calc: $blocked
28
+ warn: $this > ( ($status >= $WARNING ) ? ( 45 ) : ( 55 ) )
29
+ crit: $this > ( ($status == $CRITICAL) ? ( 55 ) : ( 75 ) )
30
+ delay: up 2m down 5m
31
+ info: percentage of blocked dns queries over the last 24 hour
32
+ to: sysadmin
33
34
35
# Blocklist last update time.
36
# Default update interval is a week.
37
32
-template: pihole_blocklist_last_update
33
- on: pihole.blocklist_last_update
34
- every: 10s
35
- units: seconds
36
- calc: $ago
37
- warn: $this > 60 * 60 * 24 * 8
38
- crit: $this > 60 * 60 * 24 * 8 * 2
39
- info: gravity.list (blocklist) file last update time
40
- to: sysadmin
38
+ template: pihole_blocklist_last_update
39
+ on: pihole.blocklist_last_update
40
+ class: Ad Filtering
41
+component: Pi-hole
42
+ type: Errors
43
+ every: 10s
44
+ units: seconds
45
+ calc: $ago
46
+ warn: $this > 60 * 60 * 24 * 8
47
+ crit: $this > 60 * 60 * 24 * 8 * 2
48
+ info: gravity.list (blocklist) file last update time
49
+ to: sysadmin
50
51
# Gravity file check (gravity.list).
52
44
-template: pihole_blocklist_gravity_file
45
- on: pihole.blocklist_last_update
46
- every: 10s
47
- units: boolean
48
- calc: $file_exists
49
- crit: $this != 1
50
- delay: up 2m down 5m
51
- info: gravity.list (blocklist) file existence state (0: exists, 1: not-exists)
52
- to: sysadmin
53
+ template: pihole_blocklist_gravity_file
54
+ on: pihole.blocklist_last_update
55
+ class: Ad Filtering
56
+component: Pi-hole
57
+ type: Errors
58
+ every: 10s
59
+ units: boolean
60
+ calc: $file_exists
61
+ crit: $this != 1
62
+ delay: up 2m down 5m
63
+ info: gravity.list (blocklist) file existence state (0: exists, 1: not-exists)
64
+ to: sysadmin
65
66
# Pi-hole's ability to block unwanted domains.
67
# Should be enabled. The whole point of Pi-hole!
68
57
-template: pihole_status
58
- on: pihole.unwanted_domains_blocking_status
59
- every: 10s
60
- units: boolean
61
- calc: $enabled
62
- warn: $this != 1
63
- delay: up 2m down 5m
64
- info: unwanted domains blocking status (0: enabled, 1: disabled)
65
- to: sysadmin
69
+ template: pihole_status
70
+ on: pihole.unwanted_domains_blocking_status
71
+ class: Ad Filtering
72
+component: Pi-hole
73
+ type: Errors
74
+ every: 10s
75
+ units: boolean
76
+ calc: $enabled
77
+ warn: $this != 1
78
+ delay: up 2m down 5m
79
+ info: unwanted domains blocking status (0: enabled, 1: disabled)
80
+ to: sysadmin
health/health.d/portcheck.conf
+54
-42
@@ -1,46 +1,58 @@
1
-template: portcheck_last_collected_secs
2
-families: *
3
- on: portcheck.status
4
- calc: $now - $last_collected_t
5
- every: 10s
6
- units: seconds ago
7
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
- delay: down 5m multiplier 1.5 max 1h
10
- info: number of seconds since the last successful data collection
11
- to: sysadmin
1
+ template: portcheck_last_collected_secs
2
+ families: *
3
+ on: portcheck.status
4
+ class: Other
5
+component: TCP endpoint
6
+ type: Latency
7
+ calc: $now - $last_collected_t
8
+ every: 10s
9
+ units: seconds ago
10
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: number of seconds since the last successful data collection
14
+ to: sysadmin
15
16
# This is a fast-reacting no-notification alarm ideal for custom dashboards or badges
14
-template: portcheck_service_reachable
15
-families: *
16
- on: portcheck.status
17
- lookup: average -1m unaligned percentage of success
18
- calc: ($this < 75) ? (0) : ($this)
19
- every: 5s
20
- units: up/down
21
- info: average ratio of successful connections over the last minute (at least 75%)
22
- to: silent
17
+ template: portcheck_service_reachable
18
+ families: *
19
+ on: portcheck.status
20
+ class: Other
21
+component: TCP endpoint
22
+ type: Workload
23
+ lookup: average -1m unaligned percentage of success
24
+ calc: ($this < 75) ? (0) : ($this)
25
+ every: 5s
26
+ units: up/down
27
+ info: average ratio of successful connections over the last minute (at least 75%)
28
+ to: silent
29
24
-template: portcheck_connection_timeouts
25
-families: *
26
- on: portcheck.status
27
- lookup: average -5m unaligned percentage of timeout
28
- every: 10s
29
- units: %
30
- warn: $this >= 10 AND $this < 40
31
- crit: $this >= 40
32
- delay: down 5m multiplier 1.5 max 1h
33
- info: average ratio of timeouts over the last 5 minutes
34
- to: sysadmin
30
+ template: portcheck_connection_timeouts
31
+ families: *
32
+ on: portcheck.status
33
+ class: Other
34
+component: TCP endpoint
35
+ type: Errors
36
+ lookup: average -5m unaligned percentage of timeout
37
+ every: 10s
38
+ units: %
39
+ warn: $this >= 10 AND $this < 40
40
+ crit: $this >= 40
41
+ delay: down 5m multiplier 1.5 max 1h
42
+ info: average ratio of timeouts over the last 5 minutes
43
+ to: sysadmin
44
36
-template: portcheck_connection_fails
37
-families: *
38
- on: portcheck.status
39
- lookup: average -5m unaligned percentage of no_connection,failed
40
- every: 10s
41
- units: %
42
- warn: $this >= 10 AND $this < 40
43
- crit: $this >= 40
44
- delay: down 5m multiplier 1.5 max 1h
45
- info: average ratio of failed connections over the last 5 minutes
46
- to: sysadmin
45
+ template: portcheck_connection_fails
46
+ families: *
47
+ on: portcheck.status
48
+ class: Other
49
+component: TCP endpoint
50
+ type: Errors
51
+ lookup: average -5m unaligned percentage of no_connection,failed
52
+ every: 10s
53
+ units: %
54
+ warn: $this >= 10 AND $this < 40
55
+ crit: $this >= 40
56
+ delay: down 5m multiplier 1.5 max 1h
57
+ info: average ratio of failed connections over the last 5 minutes
58
+ to: sysadmin
health/health.d/postgres.conf
+13
-10
@@ -1,13 +1,16 @@
1
2
# make sure postgres is running
3
4
-template: postgres_last_collected_secs
5
- on: postgres.db_stat_transactions
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: postgres_last_collected_secs
5
+ on: postgres.db_stat_transactions
6
+ class: Database
7
+component: PostgreSQL
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
health/health.d/processes.conf
+14
-11
@@ -1,13 +1,16 @@
1
# you can disable an alarm notification by setting the 'to' line to: silent
2
3
- alarm: active_processes
4
- on: system.active_processes
5
- hosts: *
6
- calc: $active * 100 / $pidmax
7
- units: %
8
- every: 5s
9
- warn: $this > (($status >= $WARNING) ? (85) : (90))
10
- crit: $this > (($status == $CRITICAL) ? (90) : (95))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: system process IDs (PID) space utilization
13
- to: sysadmin
3
+ alarm: active_processes
4
+ on: system.active_processes
5
+ class: System
6
+component: Processes
7
+ type: Workload
8
+ hosts: *
9
+ calc: $active * 100 / $pidmax
10
+ units: %
11
+ every: 5s
12
+ warn: $this > (($status >= $WARNING) ? (85) : (90))
13
+ crit: $this > (($status == $CRITICAL) ? (90) : (95))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: system process IDs (PID) space utilization
16
+ to: sysadmin
health/health.d/pulsar.conf
+13
-10
@@ -1,13 +1,16 @@
1
2
# Availability
3
4
-template: pulsar_last_collected_secs
5
- on: pulsar.broker_components
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: sysadmin
4
+ template: pulsar_last_collected_secs
5
+ on: pulsar.broker_components
6
+ class: Messaging
7
+component: Pulsar
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: sysadmin
health/health.d/ram.conf
+72
-57
@@ -1,41 +1,50 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: used_ram_to_ignore
5
- on: system.ram
6
- os: linux freebsd
7
- hosts: *
8
- calc: ($zfs.arc_size.arcsz = nan)?(0):($zfs.arc_size.arcsz - $zfs.arc_size.min)
9
- every: 10s
10
- info: amount of memory reported as used, \
11
- but it is actually capable for resizing itself based on the system needs (eg. ZFS ARC)
4
+ alarm: used_ram_to_ignore
5
+ on: system.ram
6
+ class: System
7
+component: Memory
8
+ type: Utilization
9
+ os: linux freebsd
10
+ hosts: *
11
+ calc: ($zfs.arc_size.arcsz = nan)?(0):($zfs.arc_size.arcsz - $zfs.arc_size.min)
12
+ every: 10s
13
+ info: amount of memory reported as used, \
14
+ but it is actually capable for resizing itself based on the system needs (eg. ZFS ARC)
15
13
- alarm: ram_in_use
14
- on: system.ram
15
- os: linux
16
- hosts: *
17
-# calc: $used * 100 / ($used + $cached + $free)
18
- calc: ($used - $used_ram_to_ignore) * 100 / ($used + $cached + $free)
19
- units: %
20
- every: 10s
21
- warn: $this > (($status >= $WARNING) ? (80) : (90))
22
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
23
- delay: down 15m multiplier 1.5 max 1h
24
- info: system memory utilization
25
- to: sysadmin
16
+ alarm: ram_in_use
17
+ on: system.ram
18
+ class: System
19
+component: Memory
20
+ type: Utilization
21
+ os: linux
22
+ hosts: *
23
+# calc: $used * 100 / ($used + $cached + $free)
24
+ calc: ($used - $used_ram_to_ignore) * 100 / ($used + $cached + $free)
25
+ units: %
26
+ every: 10s
27
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
28
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
29
+ delay: down 15m multiplier 1.5 max 1h
30
+ info: system memory utilization
31
+ to: sysadmin
32
27
- alarm: ram_available
28
- on: mem.available
29
- os: linux
30
- hosts: *
31
- calc: ($avail + $system.ram.used_ram_to_ignore) * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers)
32
- units: %
33
- every: 10s
34
- warn: $this < (($status >= $WARNING) ? (15) : (10))
35
- crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
36
- delay: down 15m multiplier 1.5 max 1h
37
- info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
38
- to: sysadmin
33
+ alarm: ram_available
34
+ on: mem.available
35
+ class: System
36
+component: Memory
37
+ type: Utilization
38
+ os: linux
39
+ hosts: *
40
+ calc: ($avail + $system.ram.used_ram_to_ignore) * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers)
41
+ units: %
42
+ every: 10s
43
+ warn: $this < (($status >= $WARNING) ? (15) : (10))
44
+ crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
45
+ delay: down 15m multiplier 1.5 max 1h
46
+ info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
47
+ to: sysadmin
48
49
alarm: oom_kill
50
on: mem.oom_kill
@@ -50,28 +59,34 @@
59
to: sysadmin
60
61
## FreeBSD
53
- alarm: ram_in_use
54
- on: system.ram
55
- os: freebsd
56
- hosts: *
57
- calc: ($active + $wired + $laundry + $buffers - $used_ram_to_ignore) * 100 / ($active + $wired + $laundry + $buffers - $used_ram_to_ignore + $cache + $free + $inactive)
58
- units: %
59
- every: 10s
60
- warn: $this > (($status >= $WARNING) ? (80) : (90))
61
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
62
- delay: down 15m multiplier 1.5 max 1h
63
- info: system memory utilization
64
- to: sysadmin
62
+ alarm: ram_in_use
63
+ on: system.ram
64
+ class: System
65
+component: Memory
66
+ type: Utilization
67
+ os: freebsd
68
+ hosts: *
69
+ calc: ($active + $wired + $laundry + $buffers - $used_ram_to_ignore) * 100 / ($active + $wired + $laundry + $buffers - $used_ram_to_ignore + $cache + $free + $inactive)
70
+ units: %
71
+ every: 10s
72
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
73
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
74
+ delay: down 15m multiplier 1.5 max 1h
75
+ info: system memory utilization
76
+ to: sysadmin
77
66
- alarm: ram_available
67
- on: system.ram
68
- os: freebsd
69
- hosts: *
70
- calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $laundry + $buffers)
71
- units: %
72
- every: 10s
73
- warn: $this < (($status >= $WARNING) ? (15) : (10))
74
- crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
75
- delay: down 15m multiplier 1.5 max 1h
76
- info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
77
- to: sysadmin
78
+ alarm: ram_available
79
+ on: system.ram
80
+ class: System
81
+component: Memory
82
+ type: Utilization
83
+ os: freebsd
84
+ hosts: *
85
+ calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $laundry + $buffers)
86
+ units: %
87
+ every: 10s
88
+ warn: $this < (($status >= $WARNING) ? (15) : (10))
89
+ crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
90
+ delay: down 15m multiplier 1.5 max 1h
91
+ info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
92
+ to: sysadmin
health/health.d/redis.conf
+38
-29
@@ -1,34 +1,43 @@
1
2
# make sure redis is running
3
4
-template: redis_last_collected_secs
5
- on: redis.operations
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: dba
4
+ template: redis_last_collected_secs
5
+ on: redis.operations
6
+ class: KV Storage
7
+component: Redis
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: dba
17
15
-template: redis_bgsave_broken
16
-families: *
17
- on: redis.bgsave_health
18
- every: 10s
19
- crit: $rdb_last_bgsave_status != 0
20
- units: ok/failed
21
- info: status of the last RDB save operation (0: ok, 1: error)
22
- delay: down 5m multiplier 1.5 max 1h
23
- to: dba
18
+ template: redis_bgsave_broken
19
+ families: *
20
+ on: redis.bgsave_health
21
+ class: KV Storage
22
+component: Redis
23
+ type: Errors
24
+ every: 10s
25
+ crit: $rdb_last_bgsave_status != 0
26
+ units: ok/failed
27
+ info: status of the last RDB save operation (0: ok, 1: error)
28
+ delay: down 5m multiplier 1.5 max 1h
29
+ to: dba
30
25
-template: redis_bgsave_slow
26
-families: *
27
- on: redis.bgsave_now
28
- every: 10s
29
- warn: $rdb_bgsave_in_progress > 600
30
- crit: $rdb_bgsave_in_progress > 1200
31
- units: seconds
32
- info: duration of the on-going RDB save operation
33
- delay: down 5m multiplier 1.5 max 1h
34
- to: dba
31
+ template: redis_bgsave_slow
32
+ families: *
33
+ on: redis.bgsave_now
34
+ class: KV Storage
35
+component: Redis
36
+ type: Latency
37
+ every: 10s
38
+ warn: $rdb_bgsave_in_progress > 600
39
+ crit: $rdb_bgsave_in_progress > 1200
40
+ units: seconds
41
+ info: duration of the on-going RDB save operation
42
+ delay: down 5m multiplier 1.5 max 1h
43
+ to: dba
health/health.d/retroshare.conf
+26
-20
@@ -1,25 +1,31 @@
1
# make sure RetroShare is running
2
3
-template: retroshare_last_collected_secs
4
- on: retroshare.peers
5
- calc: $now - $last_collected_t
6
- units: seconds ago
7
- every: 10s
8
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
9
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
10
- delay: down 5m multiplier 1.5 max 1h
11
- info: number of seconds since the last successful data collection
12
- to: sysadmin
3
+ template: retroshare_last_collected_secs
4
+ on: retroshare.peers
5
+ class: Data Sharing
6
+component: Retroshare
7
+ type: Latency
8
+ calc: $now - $last_collected_t
9
+ units: seconds ago
10
+ every: 10s
11
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
12
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
13
+ delay: down 5m multiplier 1.5 max 1h
14
+ info: number of seconds since the last successful data collection
15
+ to: sysadmin
16
17
# make sure the DHT is fine when active
18
16
-template: retroshare_dht_working
17
- on: retroshare.dht
18
- calc: $dht_size_all
19
- units: peers
20
- every: 1m
21
- warn: $this < (($status >= $WARNING) ? (120) : (100))
22
- crit: $this < (($status == $CRITICAL) ? (10) : (1))
23
- delay: up 0 down 15m multiplier 1.5 max 1h
24
- info: number of DHT peers
25
- to: sysadmin
19
+ template: retroshare_dht_working
20
+ on: retroshare.dht
21
+ class: Data Sharing
22
+component: Retroshare
23
+ type: Utilization
24
+ calc: $dht_size_all
25
+ units: peers
26
+ every: 1m
27
+ warn: $this < (($status >= $WARNING) ? (120) : (100))
28
+ crit: $this < (($status == $CRITICAL) ? (10) : (1))
29
+ delay: up 0 down 15m multiplier 1.5 max 1h
30
+ info: number of DHT peers
31
+ to: sysadmin
health/health.d/riakkv.conf
+90
-69
@@ -1,86 +1,107 @@
1
# Ensure that Riak is running. template: riak_last_collected_secs
2
-template: riakkv_last_collected_secs
3
- on: riak.kv.throughput
4
- calc: $now - $last_collected_t
5
- units: seconds ago
6
- every: 10s
7
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
- delay: down 5m multiplier 1.5 max 1h
10
- info: number of seconds since the last successful data collection
11
- to: dba
2
+ template: riakkv_last_collected_secs
3
+ on: riak.kv.throughput
4
+ class: Database
5
+component: Riak KV
6
+ type: Latency
7
+ calc: $now - $last_collected_t
8
+ units: seconds ago
9
+ every: 10s
10
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: number of seconds since the last successful data collection
14
+ to: dba
15
16
# Warn if a list keys operation is running.
14
-template: riakkv_list_keys_active
15
- on: riak.core.fsm_active
16
- calc: $list_fsm_active
17
- units: state machines
18
- every: 10s
19
- warn: $list_fsm_active > 0
20
- info: number of currently running list keys finite state machines
21
- to: dba
17
+ template: riakkv_list_keys_active
18
+ on: riak.core.fsm_active
19
+ class: Database
20
+component: Riak KV
21
+ type: Utilization
22
+ calc: $list_fsm_active
23
+ units: state machines
24
+ every: 10s
25
+ warn: $list_fsm_active > 0
26
+ info: number of currently running list keys finite state machines
27
+ to: dba
28
29
30
## Timing healthchecks
31
# KV GET
26
-template: riakkv_1h_kv_get_mean_latency
27
- on: riak.kv.latency.get
28
- calc: $node_get_fsm_time_mean
29
- lookup: average -1h unaligned of time
30
- every: 30s
31
- units: ms
32
- info: average time between reception of client GET request and \
33
- subsequent response to client over the last hour
32
+ template: riakkv_1h_kv_get_mean_latency
33
+ on: riak.kv.latency.get
34
+ class: Database
35
+component: Riak KV
36
+ type: Latency
37
+ calc: $node_get_fsm_time_mean
38
+ lookup: average -1h unaligned of time
39
+ every: 30s
40
+ units: ms
41
+ info: average time between reception of client GET request and \
42
+ subsequent response to client over the last hour
43
35
-template: riakkv_kv_get_slow
36
- on: riak.kv.latency.get
37
- calc: $mean
38
- lookup: average -3m unaligned of time
39
- units: ms
40
- every: 10s
41
- warn: ($this > ($riakkv_1h_kv_get_mean_latency * 2) )
42
- crit: ($this > ($riakkv_1h_kv_get_mean_latency * 3) )
43
- info: average time between reception of client GET request and \
44
- subsequent response to the client over the last 3 minutes, \
45
- compared to the average over the last hour
46
- delay: down 5m multiplier 1.5 max 1h
47
- to: dba
44
+ template: riakkv_kv_get_slow
45
+ on: riak.kv.latency.get
46
+ class: Database
47
+component: Riak KV
48
+ type: Latency
49
+ calc: $mean
50
+ lookup: average -3m unaligned of time
51
+ units: ms
52
+ every: 10s
53
+ warn: ($this > ($riakkv_1h_kv_get_mean_latency * 2) )
54
+ crit: ($this > ($riakkv_1h_kv_get_mean_latency * 3) )
55
+ info: average time between reception of client GET request and \
56
+ subsequent response to the client over the last 3 minutes, \
57
+ compared to the average over the last hour
58
+ delay: down 5m multiplier 1.5 max 1h
59
+ to: dba
60
61
# KV PUT
50
-template: riakkv_1h_kv_put_mean_latency
51
- on: riak.kv.latency.put
52
- calc: $node_put_fsm_time_mean
53
- lookup: average -1h unaligned of time
54
- every: 30s
55
- units: ms
56
- info: average time between reception of client PUT request and \
57
- subsequent response to the client over the last hour
62
+ template: riakkv_1h_kv_put_mean_latency
63
+ on: riak.kv.latency.put
64
+ class: Database
65
+component: Riak KV
66
+ type: Latency
67
+ calc: $node_put_fsm_time_mean
68
+ lookup: average -1h unaligned of time
69
+ every: 30s
70
+ units: ms
71
+ info: average time between reception of client PUT request and \
72
+ subsequent response to the client over the last hour
73
59
-template: riakkv_kv_put_slow
60
- on: riak.kv.latency.put
61
- calc: $mean
62
- lookup: average -3m unaligned of time
63
- units: ms
64
- every: 10s
65
- warn: ($this > ($riakkv_1h_kv_put_mean_latency * 2) )
66
- crit: ($this > ($riakkv_1h_kv_put_mean_latency * 3) )
67
- info: average time between reception of client PUT request and \
68
- subsequent response to the client over the last 3 minutes, \
69
- compared to the average over the last hour
70
- delay: down 5m multiplier 1.5 max 1h
71
- to: dba
74
+ template: riakkv_kv_put_slow
75
+ on: riak.kv.latency.put
76
+ class: Database
77
+component: Riak KV
78
+ type: Latency
79
+ calc: $mean
80
+ lookup: average -3m unaligned of time
81
+ units: ms
82
+ every: 10s
83
+ warn: ($this > ($riakkv_1h_kv_put_mean_latency * 2) )
84
+ crit: ($this > ($riakkv_1h_kv_put_mean_latency * 3) )
85
+ info: average time between reception of client PUT request and \
86
+ subsequent response to the client over the last 3 minutes, \
87
+ compared to the average over the last hour
88
+ delay: down 5m multiplier 1.5 max 1h
89
+ to: dba
90
91
92
## VM healthchecks
93
94
# Default Erlang VM process limit: 262144
95
# On systems observed, this is < 2000, but may grow depending on load.
78
-template: riakkv_vm_high_process_count
79
- on: riak.vm
80
- calc: $sys_process_count
81
- units: processes
82
- every: 10s
83
- warn: $this > 10000
84
- crit: $this > 100000
85
- info: number of processes running in the Erlang VM
86
- to: dba
96
+ template: riakkv_vm_high_process_count
97
+ on: riak.vm
98
+ class: Database
99
+component: Riak KV
100
+ type: Utilization
101
+ calc: $sys_process_count
102
+ units: processes
103
+ every: 10s
104
+ warn: $this > 10000
105
+ crit: $this > 100000
106
+ info: number of processes running in the Erlang VM
107
+ to: dba
health/health.d/scaleio.conf
+37
-28
@@ -1,38 +1,47 @@
1
2
# make sure scaleio is running
3
4
-template: scaleio_last_collected_secs
5
- on: scaleio.system_capacity_total
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: sysadmin
4
+ template: scaleio_last_collected_secs
5
+ on: scaleio.system_capacity_total
6
+ class: Storage
7
+component: ScaleIO
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: sysadmin
17
18
# make sure Storage Pool capacity utilization is under limit
19
17
-template: scaleio_storage_pool_capacity_utilization
18
- on: scaleio.storage_pool_capacity_utilization
19
- calc: $used
20
- units: %
21
- every: 10s
22
- warn: $this > (($status >= $WARNING) ? (80) : (90))
23
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
24
- delay: down 15m multiplier 1.5 max 1h
25
- info: storage pool capacity utilization
26
- to: sysadmin
20
+ template: scaleio_storage_pool_capacity_utilization
21
+ on: scaleio.storage_pool_capacity_utilization
22
+ class: Storage
23
+component: ScaleIO
24
+ type: Utilization
25
+ calc: $used
26
+ units: %
27
+ every: 10s
28
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
29
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
30
+ delay: down 15m multiplier 1.5 max 1h
31
+ info: storage pool capacity utilization
32
+ to: sysadmin
33
34
35
# make sure Sdc is connected to MDM
36
31
-template: scaleio_sdc_mdm_connection_state
32
- on: scaleio.sdc_mdm_connection_state
33
- calc: $connected
34
- every: 10s
35
- warn: $this != 1
36
- delay: up 30s down 5m multiplier 1.5 max 1h
37
- info: Data Client (SDC) to Metadata Manager (MDM) connection state (0: disconnected, 1: connected)
38
- to: sysadmin
37
+ template: scaleio_sdc_mdm_connection_state
38
+ on: scaleio.sdc_mdm_connection_state
39
+ class: Storage
40
+component: ScaleIO
41
+ type: Utilization
42
+ calc: $connected
43
+ every: 10s
44
+ warn: $this != 1
45
+ delay: up 30s down 5m multiplier 1.5 max 1h
46
+ info: Data Client (SDC) to Metadata Manager (MDM) connection state (0: disconnected, 1: connected)
47
+ to: sysadmin
health/health.d/softnet.conf
+47
-38
@@ -3,43 +3,52 @@
3
4
# check for common /proc/net/softnet_stat errors
5
6
- alarm: 1min_netdev_backlog_exceeded
7
- on: system.softnet_stat
8
- os: linux
9
- hosts: *
10
- lookup: average -1m unaligned absolute of dropped
11
- units: packets
12
- every: 10s
13
- warn: $this > (($status >= $WARNING) ? (0) : (10))
14
- delay: down 1h multiplier 1.5 max 2h
15
- info: average number of dropped packets in the last minute \
16
- due to exceeded net.core.netdev_max_backlog
17
- to: sysadmin
6
+ alarm: 1min_netdev_backlog_exceeded
7
+ on: system.softnet_stat
8
+ class: System
9
+component: Network
10
+ type: Errors
11
+ os: linux
12
+ hosts: *
13
+ lookup: average -1m unaligned absolute of dropped
14
+ units: packets
15
+ every: 10s
16
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
17
+ delay: down 1h multiplier 1.5 max 2h
18
+ info: average number of dropped packets in the last minute \
19
+ due to exceeded net.core.netdev_max_backlog
20
+ to: sysadmin
21
19
- alarm: 1min_netdev_budget_ran_outs
20
- on: system.softnet_stat
21
- os: linux
22
- hosts: *
23
- lookup: average -1m unaligned absolute of squeezed
24
- units: events
25
- every: 10s
26
- warn: $this > (($status >= $WARNING) ? (0) : (10))
27
- delay: down 1h multiplier 1.5 max 2h
28
- info: average number of times ksoftirq ran out of sysctl net.core.netdev_budget or \
29
- net.core.netdev_budget_usecs with work remaining over the last minute \
30
- (this can be a cause for dropped packets)
31
- to: silent
22
+ alarm: 1min_netdev_budget_ran_outs
23
+ on: system.softnet_stat
24
+ class: System
25
+component: Network
26
+ type: Errors
27
+ os: linux
28
+ hosts: *
29
+ lookup: average -1m unaligned absolute of squeezed
30
+ units: events
31
+ every: 10s
32
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
33
+ delay: down 1h multiplier 1.5 max 2h
34
+ info: average number of times ksoftirq ran out of sysctl net.core.netdev_budget or \
35
+ net.core.netdev_budget_usecs with work remaining over the last minute \
36
+ (this can be a cause for dropped packets)
37
+ to: silent
38
33
- alarm: 10min_netisr_backlog_exceeded
34
- on: system.softnet_stat
35
- os: freebsd
36
- hosts: *
37
- lookup: average -1m unaligned absolute of qdrops
38
- units: packets
39
- every: 10s
40
- warn: $this > (($status >= $WARNING) ? (0) : (10))
41
- delay: down 1h multiplier 1.5 max 2h
42
- info: average number of drops in the last minute \
43
- due to exceeded sysctl net.route.netisr_maxqlen \
44
- (this can be a cause for dropped packets)
45
- to: sysadmin
39
+ alarm: 10min_netisr_backlog_exceeded
40
+ on: system.softnet_stat
41
+ class: System
42
+component: Network
43
+ type: Errors
44
+ os: freebsd
45
+ hosts: *
46
+ lookup: average -1m unaligned absolute of qdrops
47
+ units: packets
48
+ every: 10s
49
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
50
+ delay: down 1h multiplier 1.5 max 2h
51
+ info: average number of drops in the last minute \
52
+ due to exceeded sysctl net.route.netisr_maxqlen \
53
+ (this can be a cause for dropped packets)
54
+ to: sysadmin
health/health.d/squid.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure squid is running
3
4
-template: squid_last_collected_secs
5
- on: squid.clients_requests
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: proxyadmin
4
+ template: squid_last_collected_secs
5
+ on: squid.clients_requests
6
+ class: Web Proxy
7
+component: Squid
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: proxyadmin
17
health/health.d/stiebeleltron.conf
+14
-11
@@ -1,11 +1,14 @@
1
-template: stiebeleltron_last_collected_secs
2
-families: *
3
- on: stiebeleltron.heating.hc1
4
- calc: $now - $last_collected_t
5
- every: 10s
6
- units: seconds ago
7
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
- delay: down 5m multiplier 1.5 max 1h
10
- info: number of seconds since the last successful data collection
11
- to: sitemgr
1
+ template: stiebeleltron_last_collected_secs
2
+ families: *
3
+ on: stiebeleltron.heating.hc1
4
+ class: Other
5
+component: Sensors
6
+ type: Latency
7
+ calc: $now - $last_collected_t
8
+ every: 10s
9
+ units: seconds ago
10
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: number of seconds since the last successful data collection
14
+ to: sitemgr
health/health.d/swap.conf
+31
-25
@@ -1,29 +1,35 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: 30min_ram_swapped_out
5
- on: system.swapio
6
- os: linux freebsd
7
- hosts: *
8
- lookup: sum -30m unaligned absolute of out
9
- # we have to convert KB to MB by dividing $this (i.e. the result of the lookup) with 1024
10
- calc: $this / 1024 * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
11
- units: % of RAM
12
- every: 1m
13
- warn: $this > (($status >= $WARNING) ? (20) : (30))
14
- delay: down 15m multiplier 1.5 max 1h
15
- info: percentage of the system RAM swapped in the last 30 minutes
16
- to: sysadmin
4
+ alarm: 30min_ram_swapped_out
5
+ on: system.swapio
6
+ class: System
7
+component: Memory
8
+ type: Workload
9
+ os: linux freebsd
10
+ hosts: *
11
+ lookup: sum -30m unaligned absolute of out
12
+ # we have to convert KB to MB by dividing $this (i.e. the result of the lookup) with 1024
13
+ calc: $this / 1024 * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
14
+ units: % of RAM
15
+ every: 1m
16
+ warn: $this > (($status >= $WARNING) ? (20) : (30))
17
+ delay: down 15m multiplier 1.5 max 1h
18
+ info: percentage of the system RAM swapped in the last 30 minutes
19
+ to: sysadmin
20
18
- alarm: used_swap
19
- on: system.swap
20
- os: linux freebsd
21
- hosts: *
22
- calc: $used * 100 / ( $used + $free )
23
- units: %
24
- every: 10s
25
- warn: $this > (($status >= $WARNING) ? (80) : (90))
26
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
27
- delay: up 30s down 15m multiplier 1.5 max 1h
28
- info: swap memory utilization
29
- to: sysadmin
21
+ alarm: used_swap
22
+ on: system.swap
23
+ class: System
24
+component: Memory
25
+ type: Utilization
26
+ os: linux freebsd
27
+ hosts: *
28
+ calc: $used * 100 / ( $used + $free )
29
+ units: %
30
+ every: 10s
31
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
32
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
33
+ delay: up 30s down 15m multiplier 1.5 max 1h
34
+ info: swap memory utilization
35
+ to: sysadmin
health/health.d/systemdunits.conf
+120
-90
@@ -2,111 +2,141 @@
2
## States: 1 - active, 2 - inactive, 3 - activating, 4 - deactivating, 5 - failed.
3
4
## Service units
5
-template: systemd_service_units_state
6
- on: systemd.service_units_state
7
- lookup: max -1s min2max
8
- units: ok/failed
9
- every: 10s
10
- warn: $this != nan AND $this == 5
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: one or more systemd service units are in the failed state
13
- to: sysadmin
5
+ template: systemd_service_units_state
6
+ on: systemd.service_units_state
7
+ class: Linux
8
+component: Systemd units
9
+ type: Errors
10
+ lookup: max -1s min2max
11
+ units: ok/failed
12
+ every: 10s
13
+ warn: $this != nan AND $this == 5
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: one or more systemd service units are in the failed state
16
+ to: sysadmin
17
18
## Socket units
16
-template: systemd_socket_units_state
17
- on: systemd.socket_unit_state
18
- lookup: max -1s min2max
19
- units: ok/failed
20
- every: 10s
21
- warn: $this != nan AND $this == 5
22
- delay: down 5m multiplier 1.5 max 1h
23
- info: one or more systemd socket units are in the failed state
24
- to: sysadmin
19
+ template: systemd_socket_units_state
20
+ on: systemd.socket_unit_state
21
+ class: Linux
22
+component: Systemd units
23
+ type: Errors
24
+ lookup: max -1s min2max
25
+ units: ok/failed
26
+ every: 10s
27
+ warn: $this != nan AND $this == 5
28
+ delay: down 5m multiplier 1.5 max 1h
29
+ info: one or more systemd socket units are in the failed state
30
+ to: sysadmin
31
32
## Target units
27
-template: systemd_target_units_state
28
- on: systemd.target_unit_state
29
- lookup: max -1s min2max
30
- units: ok/failed
31
- every: 10s
32
- warn: $this != nan AND $this == 5
33
- delay: down 5m multiplier 1.5 max 1h
34
- info: one or more systemd target units are in the failed state
35
- to: sysadmin
33
+ template: systemd_target_units_state
34
+ on: systemd.target_unit_state
35
+ class: Linux
36
+component: Systemd units
37
+ type: Errors
38
+ lookup: max -1s min2max
39
+ units: ok/failed
40
+ every: 10s
41
+ warn: $this != nan AND $this == 5
42
+ delay: down 5m multiplier 1.5 max 1h
43
+ info: one or more systemd target units are in the failed state
44
+ to: sysadmin
45
46
## Path units
38
-template: systemd_path_units_state
39
- on: systemd.path_unit_state
40
- lookup: max -1s min2max
41
- units: ok/failed
42
- every: 10s
43
- warn: $this != nan AND $this == 5
44
- delay: down 5m multiplier 1.5 max 1h
45
- info: one or more systemd path units are in the failed state
46
- to: sysadmin
47
+ template: systemd_path_units_state
48
+ on: systemd.path_unit_state
49
+ class: Linux
50
+component: Systemd units
51
+ type: Errors
52
+ lookup: max -1s min2max
53
+ units: ok/failed
54
+ every: 10s
55
+ warn: $this != nan AND $this == 5
56
+ delay: down 5m multiplier 1.5 max 1h
57
+ info: one or more systemd path units are in the failed state
58
+ to: sysadmin
59
60
## Device units
49
-template: systemd_device_units_state
50
- on: systemd.device_unit_state
51
- lookup: max -1s min2max
52
- units: ok/failed
53
- every: 10s
54
- warn: $this != nan AND $this == 5
55
- delay: down 5m multiplier 1.5 max 1h
56
- info: one or more the systemd device units are in the failed state
57
- to: sysadmin
61
+ template: systemd_device_units_state
62
+ on: systemd.device_unit_state
63
+ class: Linux
64
+component: Systemd units
65
+ type: Errors
66
+ lookup: max -1s min2max
67
+ units: ok/failed
68
+ every: 10s
69
+ warn: $this != nan AND $this == 5
70
+ delay: down 5m multiplier 1.5 max 1h
71
+ info: one or more the systemd device units are in the failed state
72
+ to: sysadmin
73
74
## Mount units
60
-template: systemd_mount_units_state
61
- on: systemd.mount_unit_state
62
- lookup: max -1s min2max
63
- units: ok/failed
64
- every: 10s
65
- warn: $this != nan AND $this == 5
66
- delay: down 5m multiplier 1.5 max 1h
67
- info: one or more the systemd mount units are in the failed state
68
- to: sysadmin
75
+ template: systemd_mount_units_state
76
+ on: systemd.mount_unit_state
77
+ class: Linux
78
+component: Systemd units
79
+ type: Errors
80
+ lookup: max -1s min2max
81
+ units: ok/failed
82
+ every: 10s
83
+ warn: $this != nan AND $this == 5
84
+ delay: down 5m multiplier 1.5 max 1h
85
+ info: one or more the systemd mount units are in the failed state
86
+ to: sysadmin
87
88
## Automount units
71
-template: systemd_automount_units_state
72
- on: systemd.automount_unit_state
73
- lookup: max -1s min2max
74
- units: ok/failed
75
- every: 10s
76
- warn: $this != nan AND $this == 5
77
- delay: down 5m multiplier 1.5 max 1h
78
- info: one or more systemd automount units are in the failed state
79
- to: sysadmin
89
+ template: systemd_automount_units_state
90
+ on: systemd.automount_unit_state
91
+ class: Linux
92
+component: Systemd units
93
+ type: Errors
94
+ lookup: max -1s min2max
95
+ units: ok/failed
96
+ every: 10s
97
+ warn: $this != nan AND $this == 5
98
+ delay: down 5m multiplier 1.5 max 1h
99
+ info: one or more systemd automount units are in the failed state
100
+ to: sysadmin
101
102
## Swap units
82
-template: systemd_swap_units_state
83
- on: systemd.swap_unit_state
84
- lookup: max -1s min2max
85
- units: ok/failed
86
- every: 10s
87
- warn: $this != nan AND $this == 5
88
- delay: down 5m multiplier 1.5 max 1h
89
- info: one or more systemd swap units are in the failed state
90
- to: sysadmin
103
+ template: systemd_swap_units_state
104
+ on: systemd.swap_unit_state
105
+ class: Linux
106
+component: Systemd units
107
+ type: Errors
108
+ lookup: max -1s min2max
109
+ units: ok/failed
110
+ every: 10s
111
+ warn: $this != nan AND $this == 5
112
+ delay: down 5m multiplier 1.5 max 1h
113
+ info: one or more systemd swap units are in the failed state
114
+ to: sysadmin
115
116
## Scope units
93
-template: systemd_scope_units_state
94
- on: systemd.scope_unit_state
95
- lookup: max -1s min2max
96
- units: ok/failed
97
- every: 10s
98
- warn: $this != nan AND $this == 5
99
- delay: down 5m multiplier 1.5 max 1h
100
- info: one or more systemd scope units are in the failed state
101
- to: sysadmin
117
+ template: systemd_scope_units_state
118
+ on: systemd.scope_unit_state
119
+ class: Linux
120
+component: Systemd units
121
+ type: Errors
122
+ lookup: max -1s min2max
123
+ units: ok/failed
124
+ every: 10s
125
+ warn: $this != nan AND $this == 5
126
+ delay: down 5m multiplier 1.5 max 1h
127
+ info: one or more systemd scope units are in the failed state
128
+ to: sysadmin
129
130
## Slice units
104
-template: systemd_slice_units_state
105
- on: systemd.slice_unit_state
106
- lookup: max -1s min2max
107
- units: ok/failed
108
- every: 10s
109
- warn: $this != nan AND $this == 5
110
- delay: down 5m multiplier 1.5 max 1h
111
- info: one or more systemd slice units are in the failed state
112
- to: sysadmin
131
+ template: systemd_slice_units_state
132
+ on: systemd.slice_unit_state
133
+ class: Linux
134
+component: Systemd units
135
+ type: Errors
136
+ lookup: max -1s min2max
137
+ units: ok/failed
138
+ every: 10s
139
+ warn: $this != nan AND $this == 5
140
+ delay: down 5m multiplier 1.5 max 1h
141
+ info: one or more systemd slice units are in the failed state
142
+ to: sysadmin
health/health.d/tcp_conn.conf
+15
-12
@@ -5,15 +5,18 @@
5
# In this case, the alarm will always be zero.
6
#
7
8
- alarm: tcp_connections
9
- on: ipv4.tcpsock
10
- os: linux
11
- hosts: *
12
- calc: (${tcp_max_connections} > 0) ? ( ${connections} * 100 / ${tcp_max_connections} ) : 0
13
- units: %
14
- every: 10s
15
- warn: $this > (($status >= $WARNING ) ? ( 60 ) : ( 80 ))
16
- crit: $this > (($status == $CRITICAL) ? ( 80 ) : ( 90 ))
17
- delay: up 0 down 5m multiplier 1.5 max 1h
18
- info: IPv4 TCP connections utilization
19
- to: sysadmin
8
+ alarm: tcp_connections
9
+ on: ipv4.tcpsock
10
+ class: System
11
+component: Network
12
+ type: Workload
13
+ os: linux
14
+ hosts: *
15
+ calc: (${tcp_max_connections} > 0) ? ( ${connections} * 100 / ${tcp_max_connections} ) : 0
16
+ units: %
17
+ every: 10s
18
+ warn: $this > (($status >= $WARNING ) ? ( 60 ) : ( 80 ))
19
+ crit: $this > (($status == $CRITICAL) ? ( 80 ) : ( 90 ))
20
+ delay: up 0 down 5m multiplier 1.5 max 1h
21
+ info: IPv4 TCP connections utilization
22
+ to: sysadmin
health/health.d/tcp_listen.conf
+61
-49
@@ -18,33 +18,39 @@
18
# -----------------------------------------------------------------------------
19
# tcp accept queue (at the kernel)
20
21
- alarm: 1m_tcp_accept_queue_overflows
22
- on: ip.tcp_accept_queue
23
- os: linux
24
- hosts: *
25
- lookup: average -60s unaligned absolute of ListenOverflows
26
- units: overflows
27
- every: 10s
28
- warn: $this > 1
29
- crit: $this > (($status == $CRITICAL) ? (1) : (5))
30
- delay: up 0 down 5m multiplier 1.5 max 1h
31
- info: average number of overflows in the TCP accept queue over the last minute
32
- to: sysadmin
21
+ alarm: 1m_tcp_accept_queue_overflows
22
+ on: ip.tcp_accept_queue
23
+ class: System
24
+component: Network
25
+ type: Workload
26
+ os: linux
27
+ hosts: *
28
+ lookup: average -60s unaligned absolute of ListenOverflows
29
+ units: overflows
30
+ every: 10s
31
+ warn: $this > 1
32
+ crit: $this > (($status == $CRITICAL) ? (1) : (5))
33
+ delay: up 0 down 5m multiplier 1.5 max 1h
34
+ info: average number of overflows in the TCP accept queue over the last minute
35
+ to: sysadmin
36
37
# THIS IS TOO GENERIC
38
# CHECK: https://github.com/netdata/netdata/issues/3234#issuecomment-423935842
36
- alarm: 1m_tcp_accept_queue_drops
37
- on: ip.tcp_accept_queue
38
- os: linux
39
- hosts: *
40
- lookup: average -60s unaligned absolute of ListenDrops
41
- units: drops
42
- every: 10s
43
- warn: $this > 1
44
- crit: $this > (($status == $CRITICAL) ? (1) : (5))
45
- delay: up 0 down 5m multiplier 1.5 max 1h
46
- info: average number of dropped packets in the TCP accept queue over the last minute
47
- to: sysadmin
39
+ alarm: 1m_tcp_accept_queue_drops
40
+ on: ip.tcp_accept_queue
41
+ class: System
42
+component: Network
43
+ type: Workload
44
+ os: linux
45
+ hosts: *
46
+ lookup: average -60s unaligned absolute of ListenDrops
47
+ units: drops
48
+ every: 10s
49
+ warn: $this > 1
50
+ crit: $this > (($status == $CRITICAL) ? (1) : (5))
51
+ delay: up 0 down 5m multiplier 1.5 max 1h
52
+ info: average number of dropped packets in the TCP accept queue over the last minute
53
+ to: sysadmin
54
55
56
# -----------------------------------------------------------------------------
@@ -55,30 +61,36 @@
61
# enabled or not. In both cases this probably indicates a SYN flood attack,
62
# so i guess a notification should be sent.
63
58
- alarm: 1m_tcp_syn_queue_drops
59
- on: ip.tcp_syn_queue
60
- os: linux
61
- hosts: *
62
- lookup: average -60s unaligned absolute of TCPReqQFullDrop
63
- units: drops
64
- every: 10s
65
- warn: $this > 1
66
- crit: $this > (($status == $CRITICAL) ? (0) : (5))
67
- delay: up 10 down 5m multiplier 1.5 max 1h
68
- info: average number of SYN requests was dropped due to the full TCP SYN queue over the last minute \
69
- (SYN cookies were not enabled)
70
- to: sysadmin
64
+ alarm: 1m_tcp_syn_queue_drops
65
+ on: ip.tcp_syn_queue
66
+ class: System
67
+component: Network
68
+ type: Workload
69
+ os: linux
70
+ hosts: *
71
+ lookup: average -60s unaligned absolute of TCPReqQFullDrop
72
+ units: drops
73
+ every: 10s
74
+ warn: $this > 1
75
+ crit: $this > (($status == $CRITICAL) ? (0) : (5))
76
+ delay: up 10 down 5m multiplier 1.5 max 1h
77
+ info: average number of SYN requests was dropped due to the full TCP SYN queue over the last minute \
78
+ (SYN cookies were not enabled)
79
+ to: sysadmin
80
72
- alarm: 1m_tcp_syn_queue_cookies
73
- on: ip.tcp_syn_queue
74
- os: linux
75
- hosts: *
76
- lookup: average -60s unaligned absolute of TCPReqQFullDoCookies
77
- units: cookies
78
- every: 10s
79
- warn: $this > 1
80
- crit: $this > (($status == $CRITICAL) ? (0) : (5))
81
- delay: up 10 down 5m multiplier 1.5 max 1h
82
- info: average number of sent SYN cookies due to the full TCP SYN queue over the last minute
83
- to: sysadmin
81
+ alarm: 1m_tcp_syn_queue_cookies
82
+ on: ip.tcp_syn_queue
83
+ class: System
84
+component: Network
85
+ type: Workload
86
+ os: linux
87
+ hosts: *
88
+ lookup: average -60s unaligned absolute of TCPReqQFullDoCookies
89
+ units: cookies
90
+ every: 10s
91
+ warn: $this > 1
92
+ crit: $this > (($status == $CRITICAL) ? (0) : (5))
93
+ delay: up 10 down 5m multiplier 1.5 max 1h
94
+ info: average number of sent SYN cookies due to the full TCP SYN queue over the last minute
95
+ to: sysadmin
96
health/health.d/tcp_mem.conf
+15
-12
@@ -6,15 +6,18 @@
6
# and a critical when TCP is 90% of its upper memory limit
7
#
8
9
- alarm: tcp_memory
10
- on: ipv4.sockstat_tcp_mem
11
- os: linux
12
- hosts: *
13
- calc: ${mem} * 100 / ${tcp_mem_high}
14
- units: %
15
- every: 10s
16
- warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} ))
17
- crit: ${mem} > (($status == $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
18
- delay: up 0 down 5m multiplier 1.5 max 1h
19
- info: TCP memory utilization
20
- to: sysadmin
9
+ alarm: tcp_memory
10
+ on: ipv4.sockstat_tcp_mem
11
+ class: System
12
+component: Network
13
+ type: Utilization
14
+ os: linux
15
+ hosts: *
16
+ calc: ${mem} * 100 / ${tcp_mem_high}
17
+ units: %
18
+ every: 10s
19
+ warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} ))
20
+ crit: ${mem} > (($status == $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
21
+ delay: up 0 down 5m multiplier 1.5 max 1h
22
+ info: TCP memory utilization
23
+ to: sysadmin
health/health.d/tcp_orphans.conf
+15
-12
@@ -7,15 +7,18 @@
7
# so we alarm warning at 25% and critical at 50%
8
#
9
10
- alarm: tcp_orphans
11
- on: ipv4.sockstat_tcp_sockets
12
- os: linux
13
- hosts: *
14
- calc: ${orphan} * 100 / ${tcp_max_orphans}
15
- units: %
16
- every: 10s
17
- warn: $this > (($status >= $WARNING ) ? ( 20 ) : ( 25 ))
18
- crit: $this > (($status == $CRITICAL) ? ( 25 ) : ( 50 ))
19
- delay: up 0 down 5m multiplier 1.5 max 1h
20
- info: orphan IPv4 TCP sockets utilization
21
- to: sysadmin
10
+ alarm: tcp_orphans
11
+ on: ipv4.sockstat_tcp_sockets
12
+ class: System
13
+component: Network
14
+ type: Errors
15
+ os: linux
16
+ hosts: *
17
+ calc: ${orphan} * 100 / ${tcp_max_orphans}
18
+ units: %
19
+ every: 10s
20
+ warn: $this > (($status >= $WARNING ) ? ( 20 ) : ( 25 ))
21
+ crit: $this > (($status == $CRITICAL) ? ( 25 ) : ( 50 ))
22
+ delay: up 0 down 5m multiplier 1.5 max 1h
23
+ info: orphan IPv4 TCP sockets utilization
24
+ to: sysadmin
health/health.d/tcp_resets.conf
+57
-45
@@ -4,54 +4,66 @@
4
# -----------------------------------------------------------------------------
5
# tcp resets this host sends
6
7
- alarm: 1m_ipv4_tcp_resets_sent
8
- on: ipv4.tcphandshake
9
- os: linux
10
- hosts: *
11
- lookup: average -1m at -10s unaligned absolute of OutRsts
12
- units: tcp resets/s
13
- every: 10s
14
- info: average number of sent TCP RESETS over the last minute
7
+ alarm: 1m_ipv4_tcp_resets_sent
8
+ on: ipv4.tcphandshake
9
+ class: System
10
+component: Network
11
+ type: Errors
12
+ os: linux
13
+ hosts: *
14
+ lookup: average -1m at -10s unaligned absolute of OutRsts
15
+ units: tcp resets/s
16
+ every: 10s
17
+ info: average number of sent TCP RESETS over the last minute
18
16
- alarm: 10s_ipv4_tcp_resets_sent
17
- on: ipv4.tcphandshake
18
- os: linux
19
- hosts: *
20
- lookup: average -10s unaligned absolute of OutRsts
21
- units: tcp resets/s
22
- every: 10s
23
- warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (20)))
24
- delay: up 20s down 60m multiplier 1.2 max 2h
25
- options: no-clear-notification
26
- info: average number of sent TCP RESETS over the last 10 seconds. \
27
- This can indicate a port scan, \
28
- or that a service running on this host has crashed. \
29
- Netdata will not send a clear notification for this alarm.
30
- to: sysadmin
19
+ alarm: 10s_ipv4_tcp_resets_sent
20
+ on: ipv4.tcphandshake
21
+ class: System
22
+component: Network
23
+ type: Errors
24
+ os: linux
25
+ hosts: *
26
+ lookup: average -10s unaligned absolute of OutRsts
27
+ units: tcp resets/s
28
+ every: 10s
29
+ warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (20)))
30
+ delay: up 20s down 60m multiplier 1.2 max 2h
31
+ options: no-clear-notification
32
+ info: average number of sent TCP RESETS over the last 10 seconds. \
33
+ This can indicate a port scan, \
34
+ or that a service running on this host has crashed. \
35
+ Netdata will not send a clear notification for this alarm.
36
+ to: sysadmin
37
38
# -----------------------------------------------------------------------------
39
# tcp resets this host receives
40
35
- alarm: 1m_ipv4_tcp_resets_received
36
- on: ipv4.tcphandshake
37
- os: linux freebsd
38
- hosts: *
39
- lookup: average -1m at -10s unaligned absolute of AttemptFails
40
- units: tcp resets/s
41
- every: 10s
42
- info: average number of received TCP RESETS over the last minute
41
+ alarm: 1m_ipv4_tcp_resets_received
42
+ on: ipv4.tcphandshake
43
+ class: System
44
+component: Network
45
+ type: Errors
46
+ os: linux freebsd
47
+ hosts: *
48
+ lookup: average -1m at -10s unaligned absolute of AttemptFails
49
+ units: tcp resets/s
50
+ every: 10s
51
+ info: average number of received TCP RESETS over the last minute
52
44
- alarm: 10s_ipv4_tcp_resets_received
45
- on: ipv4.tcphandshake
46
- os: linux freebsd
47
- hosts: *
48
- lookup: average -10s unaligned absolute of AttemptFails
49
- units: tcp resets/s
50
- every: 10s
51
- warn: $this > ((($1m_ipv4_tcp_resets_received < 5)?(5):($1m_ipv4_tcp_resets_received)) * (($status >= $WARNING) ? (1) : (10)))
52
- delay: up 20s down 60m multiplier 1.2 max 2h
53
- options: no-clear-notification
54
- info: average number of received TCP RESETS over the last 10 seconds. \
55
- This can be an indication that a service this host needs has crashed. \
56
- Netdata will not send a clear notification for this alarm.
57
- to: sysadmin
53
+ alarm: 10s_ipv4_tcp_resets_received
54
+ on: ipv4.tcphandshake
55
+ class: System
56
+component: Network
57
+ type: Errors
58
+ os: linux freebsd
59
+ hosts: *
60
+ lookup: average -10s unaligned absolute of AttemptFails
61
+ units: tcp resets/s
62
+ every: 10s
63
+ warn: $this > ((($1m_ipv4_tcp_resets_received < 5)?(5):($1m_ipv4_tcp_resets_received)) * (($status >= $WARNING) ? (1) : (10)))
64
+ delay: up 20s down 60m multiplier 1.2 max 2h
65
+ options: no-clear-notification
66
+ info: average number of received TCP RESETS over the last 10 seconds. \
67
+ This can be an indication that a service this host needs has crashed. \
68
+ Netdata will not send a clear notification for this alarm.
69
+ to: sysadmin
health/health.d/udp_errors.conf
+28
-22
@@ -4,29 +4,35 @@
4
# -----------------------------------------------------------------------------
5
# UDP receive buffer errors
6
7
- alarm: 1m_ipv4_udp_receive_buffer_errors
8
- on: ipv4.udperrors
9
- os: linux freebsd
10
- hosts: *
11
- lookup: average -1m unaligned absolute of RcvbufErrors
12
- units: errors
13
- every: 10s
14
- warn: $this > (($status >= $WARNING) ? (0) : (10))
15
- info: average number of UDP receive buffer errors over the last minute
16
- delay: up 1m down 60m multiplier 1.2 max 2h
17
- to: sysadmin
7
+ alarm: 1m_ipv4_udp_receive_buffer_errors
8
+ on: ipv4.udperrors
9
+ class: System
10
+component: Network
11
+ type: Errors
12
+ os: linux freebsd
13
+ hosts: *
14
+ lookup: average -1m unaligned absolute of RcvbufErrors
15
+ units: errors
16
+ every: 10s
17
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
18
+ info: average number of UDP receive buffer errors over the last minute
19
+ delay: up 1m down 60m multiplier 1.2 max 2h
20
+ to: sysadmin
21
22
# -----------------------------------------------------------------------------
23
# UDP send buffer errors
24
22
- alarm: 1m_ipv4_udp_send_buffer_errors
23
- on: ipv4.udperrors
24
- os: linux
25
- hosts: *
26
- lookup: average -1m unaligned absolute of SndbufErrors
27
- units: errors
28
- every: 10s
29
- warn: $this > (($status >= $WARNING) ? (0) : (10))
30
- info: average number of UDP send buffer errors over the last minute
31
- delay: up 1m down 60m multiplier 1.2 max 2h
32
- to: sysadmin
25
+ alarm: 1m_ipv4_udp_send_buffer_errors
26
+ on: ipv4.udperrors
27
+ class: System
28
+component: Network
29
+ type: Errors
30
+ os: linux
31
+ hosts: *
32
+ lookup: average -1m unaligned absolute of SndbufErrors
33
+ units: errors
34
+ every: 10s
35
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
36
+ info: average number of UDP send buffer errors over the last minute
37
+ delay: up 1m down 60m multiplier 1.2 max 2h
38
+ to: sysadmin
health/health.d/unbound.conf
+37
-28
@@ -1,35 +1,44 @@
1
2
# make sure unbound is running
3
4
-template: unbound_last_collected_secs
5
- on: unbound.queries
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: sysadmin
4
+ template: unbound_last_collected_secs
5
+ on: unbound.queries
6
+ class: DNS
7
+component: Unbound
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: sysadmin
17
18
# make sure there is no overwritten/dropped queries in the request-list
19
17
-template: unbound_request_list_overwritten
18
- on: unbound.request_list_jostle_list
19
- lookup: average -60s unaligned absolute match-names of overwritten
20
- units: queries
21
- every: 10s
22
- warn: $this > 5
23
- delay: up 10 down 5m multiplier 1.5 max 1h
24
- info: number of overwritten queries in the request-list
25
- to: sysadmin
20
+ template: unbound_request_list_overwritten
21
+ on: unbound.request_list_jostle_list
22
+ class: DNS
23
+component: Unbound
24
+ type: Errors
25
+ lookup: average -60s unaligned absolute match-names of overwritten
26
+ units: queries
27
+ every: 10s
28
+ warn: $this > 5
29
+ delay: up 10 down 5m multiplier 1.5 max 1h
30
+ info: number of overwritten queries in the request-list
31
+ to: sysadmin
32
27
-template: unbound_request_list_dropped
28
- on: unbound.request_list_jostle_list
29
- lookup: average -60s unaligned absolute match-names of dropped
30
- units: queries
31
- every: 10s
32
- warn: $this > 0
33
- delay: up 10 down 5m multiplier 1.5 max 1h
34
- info: number of dropped queries in the request-list
35
- to: sysadmin
33
+ template: unbound_request_list_dropped
34
+ on: unbound.request_list_jostle_list
35
+ class: DNS
36
+component: Unbound
37
+ type: Errors
38
+ lookup: average -60s unaligned absolute match-names of dropped
39
+ units: queries
40
+ every: 10s
41
+ warn: $this > 0
42
+ delay: up 10 down 5m multiplier 1.5 max 1h
43
+ info: number of dropped queries in the request-list
44
+ to: sysadmin
health/health.d/varnish.conf
+12
-9
@@ -1,9 +1,12 @@
1
- alarm: varnish_last_collected
2
- on: varnish.uptime
3
- calc: $now - $last_collected_t
4
- units: seconds ago
5
- every: 10s
6
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
7
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
8
- info: number of seconds since the last successful data collection
9
- to: sysadmin
1
+ alarm: varnish_last_collected
2
+ on: varnish.uptime
3
+ class: Web Proxy
4
+component: Varnish
5
+ type: Latency
6
+ calc: $now - $last_collected_t
7
+ units: seconds ago
8
+ every: 10s
9
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
+ info: number of seconds since the last successful data collection
12
+ to: sysadmin
health/health.d/vcsa.conf
+125
-98
@@ -1,16 +1,19 @@
1
2
# make sure vcsa is running and responding
3
4
-template: vcsa_last_collected_secs
5
- on: vcsa.system_health
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: sysadmin
4
+ template: vcsa_last_collected_secs
5
+ on: vcsa.system_health
6
+ class: Virtual Machine
7
+component: VMware vCenter
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: sysadmin
17
18
# Overall system health:
19
# - 0: all components are healthy.
@@ -19,17 +22,20 @@ template: vcsa_last_collected_secs
22
# - 3: one or more components might be in an unusable status and the appliance might become unresponsive soon.
23
# - 4: no health data is available.
24
22
-template: vcsa_system_health
23
- on: vcsa.system_health
24
- lookup: max -10s unaligned of system
25
- units: status
26
- every: 10s
27
- warn: ($this == 1) || ($this == 2)
28
- crit: $this == 3
29
- delay: down 1m multiplier 1.5 max 1h
30
- info: overall system health status \
31
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
32
- to: sysadmin
25
+ template: vcsa_system_health
26
+ on: vcsa.system_health
27
+ class: Virtual Machine
28
+component: VMware vCenter
29
+ type: Errors
30
+ lookup: max -10s unaligned of system
31
+ units: status
32
+ every: 10s
33
+ warn: ($this == 1) || ($this == 2)
34
+ crit: $this == 3
35
+ delay: down 1m multiplier 1.5 max 1h
36
+ info: overall system health status \
37
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
38
+ to: sysadmin
39
40
# Components health:
41
# - 0: healthy.
@@ -38,77 +44,95 @@ template: vcsa_system_health
44
# - 3: unavailable, or will stop functioning soon.
45
# - 4: no health data is available.
46
41
-template: vcsa_swap_health
42
- on: vcsa.components_health
43
- lookup: max -10s unaligned of swap
44
- units: status
45
- every: 10s
46
- warn: $this == 1
47
- crit: ($this == 2) || ($this == 3)
48
- delay: down 1m multiplier 1.5 max 1h
49
- info: swap health status \
50
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
51
- to: sysadmin
47
+ template: vcsa_swap_health
48
+ on: vcsa.components_health
49
+ class: Virtual Machine
50
+component: VMware vCenter
51
+ type: Errors
52
+ lookup: max -10s unaligned of swap
53
+ units: status
54
+ every: 10s
55
+ warn: $this == 1
56
+ crit: ($this == 2) || ($this == 3)
57
+ delay: down 1m multiplier 1.5 max 1h
58
+ info: swap health status \
59
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
60
+ to: sysadmin
61
53
-template: vcsa_storage_health
54
- on: vcsa.components_health
55
- lookup: max -10s unaligned of storage
56
- units: status
57
- every: 10s
58
- warn: $this == 1
59
- crit: ($this == 2) || ($this == 3)
60
- delay: down 1m multiplier 1.5 max 1h
61
- info: storage health status \
62
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
63
- to: sysadmin
62
+ template: vcsa_storage_health
63
+ on: vcsa.components_health
64
+ class: Virtual Machine
65
+component: VMware vCenter
66
+ type: Errors
67
+ lookup: max -10s unaligned of storage
68
+ units: status
69
+ every: 10s
70
+ warn: $this == 1
71
+ crit: ($this == 2) || ($this == 3)
72
+ delay: down 1m multiplier 1.5 max 1h
73
+ info: storage health status \
74
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
75
+ to: sysadmin
76
65
-template: vcsa_mem_health
66
- on: vcsa.components_health
67
- lookup: max -10s unaligned of mem
68
- units: status
69
- every: 10s
70
- warn: $this == 1
71
- crit: ($this == 2) || ($this == 3)
72
- delay: down 1m multiplier 1.5 max 1h
73
- info: memory health status \
74
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
75
- to: sysadmin
77
+ template: vcsa_mem_health
78
+ on: vcsa.components_health
79
+ class: Virtual Machine
80
+component: VMware vCenter
81
+ type: Errors
82
+ lookup: max -10s unaligned of mem
83
+ units: status
84
+ every: 10s
85
+ warn: $this == 1
86
+ crit: ($this == 2) || ($this == 3)
87
+ delay: down 1m multiplier 1.5 max 1h
88
+ info: memory health status \
89
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
90
+ to: sysadmin
91
77
-template: vcsa_load_health
78
- on: vcsa.components_health
79
- lookup: max -10s unaligned of load
80
- units: status
81
- every: 10s
82
- warn: $this == 1
83
- crit: ($this == 2) || ($this == 3)
84
- delay: down 1m multiplier 1.5 max 1h
85
- info: load health status \
86
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
87
- to: sysadmin
92
+ template: vcsa_load_health
93
+ on: vcsa.components_health
94
+ class: Virtual Machine
95
+component: VMware vCenter
96
+ type: Utilization
97
+ lookup: max -10s unaligned of load
98
+ units: status
99
+ every: 10s
100
+ warn: $this == 1
101
+ crit: ($this == 2) || ($this == 3)
102
+ delay: down 1m multiplier 1.5 max 1h
103
+ info: load health status \
104
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
105
+ to: sysadmin
106
89
-template: vcsa_database_storage_health
90
- on: vcsa.components_health
91
- lookup: max -10s unaligned of database_storage
92
- units: status
93
- every: 10s
94
- warn: $this == 1
95
- crit: ($this == 2) || ($this == 3)
96
- delay: down 1m multiplier 1.5 max 1h
97
- info: database storage health status \
98
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
99
- to: sysadmin
107
+ template: vcsa_database_storage_health
108
+ on: vcsa.components_health
109
+ class: Virtual Machine
110
+component: VMware vCenter
111
+ type: Errors
112
+ lookup: max -10s unaligned of database_storage
113
+ units: status
114
+ every: 10s
115
+ warn: $this == 1
116
+ crit: ($this == 2) || ($this == 3)
117
+ delay: down 1m multiplier 1.5 max 1h
118
+ info: database storage health status \
119
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
120
+ to: sysadmin
121
101
-template: vcsa_applmgmt_health
102
- on: vcsa.components_health
103
- lookup: max -10s unaligned of applmgmt
104
- units: status
105
- every: 10s
106
- warn: $this == 1
107
- crit: ($this == 2) || ($this == 3)
108
- delay: down 1m multiplier 1.5 max 1h
109
- info: applmgmt health status \
110
- (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
111
- to: sysadmin
122
+ template: vcsa_applmgmt_health
123
+ on: vcsa.components_health
124
+ class: Virtual Machine
125
+component: VMware vCenter
126
+ type: Errors
127
+ lookup: max -10s unaligned of applmgmt
128
+ units: status
129
+ every: 10s
130
+ warn: $this == 1
131
+ crit: ($this == 2) || ($this == 3)
132
+ delay: down 1m multiplier 1.5 max 1h
133
+ info: applmgmt health status \
134
+ (-1: unknown, 0: green, 1: yellow, 2: orange, 3: red, 4: grey)
135
+ to: sysadmin
136
137
138
# Software updates health:
@@ -117,14 +141,17 @@ template: vcsa_applmgmt_health
141
# - 3: security updates are available.
142
# - 4: an error retrieving information on software updates.
143
120
-template: vcsa_software_updates_health
121
- on: vcsa.software_updates_health
122
- lookup: max -10s unaligned of software_packages
123
- units: status
124
- every: 10s
125
- warn: $this == 4
126
- crit: $this == 3
127
- delay: down 1m multiplier 1.5 max 1h
128
- info: software updates availability status \
129
- (-1: unknown, 0: green, 2: orange, 3: red, 4: grey)
130
- to: sysadmin
144
+ template: vcsa_software_updates_health
145
+ on: vcsa.software_updates_health
146
+ class: Virtual Machine
147
+component: VMware vCenter
148
+ type: Errors
149
+ lookup: max -10s unaligned of software_packages
150
+ units: status
151
+ every: 10s
152
+ warn: $this == 4
153
+ crit: $this == 3
154
+ delay: down 1m multiplier 1.5 max 1h
155
+ info: software updates availability status \
156
+ (-1: unknown, 0: green, 2: orange, 3: red, 4: grey)
157
+ to: sysadmin
health/health.d/vernemq.conf
+339
-258
@@ -1,300 +1,381 @@
1
2
# Availability
3
4
-template: vernemq_last_collected_secs
5
- on: vernemq.node_uptime
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: sysadmin
4
+ template: vernemq_last_collected_secs
5
+ on: vernemq.node_uptime
6
+ class: Messaging
7
+component: VerneMQ
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: sysadmin
17
18
# Socket errors
19
17
-template: vernemq_socket_errors
18
- on: vernemq.socket_errors
19
- lookup: sum -1m unaligned absolute of socket_error
20
- units: errors
21
- every: 1m
22
- warn: $this > (($status >= $WARNING) ? (0) : (5))
23
- delay: up 2m down 5m multiplier 1.5 max 2h
24
- info: number of socket errors in the last minute
25
- to: sysadmin
20
+ template: vernemq_socket_errors
21
+ on: vernemq.socket_errors
22
+ class: Messaging
23
+component: VerneMQ
24
+ type: Errors
25
+ lookup: sum -1m unaligned absolute of socket_error
26
+ units: errors
27
+ every: 1m
28
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
29
+ delay: up 2m down 5m multiplier 1.5 max 2h
30
+ info: number of socket errors in the last minute
31
+ to: sysadmin
32
33
# Queues dropped/expired/unhandled PUBLISH messages
34
29
-template: vernemq_queue_message_drop
30
- on: vernemq.queue_undelivered_messages
31
- lookup: sum -1m unaligned absolute of queue_message_drop
32
- units: dropped messages
33
- every: 1m
34
- warn: $this > (($status >= $WARNING) ? (0) : (5))
35
- delay: up 5m down 5m multiplier 1.5 max 2h
36
- info: number of dropped messaged due to full queues in the last minute
37
- to: sysadmin
38
-
39
-template: vernemq_queue_message_expired
40
- on: vernemq.queue_undelivered_messages
41
- lookup: sum -1m unaligned absolute of queue_message_expired
42
- units: expired messages
43
- every: 1m
44
- warn: $this > (($status >= $WARNING) ? (0) : (15))
45
- delay: up 5m down 5m multiplier 1.5 max 2h
46
- info: number of messages which expired before delivery in the last minute
47
- to: sysadmin
48
-
49
-template: vernemq_queue_message_unhandled
50
- on: vernemq.queue_undelivered_messages
51
- lookup: sum -1m unaligned absolute of queue_message_unhandled
52
- units: unhandled messages
53
- every: 1m
54
- warn: $this > (($status >= $WARNING) ? (0) : (5))
55
- delay: up 5m down 5m multiplier 1.5 max 2h
56
- info: number of unhandled messages (connections with clean session=true) in the last minute
57
- to: sysadmin
35
+ template: vernemq_queue_message_drop
36
+ on: vernemq.queue_undelivered_messages
37
+ class: Messaging
38
+component: VerneMQ
39
+ type: Errors
40
+ lookup: sum -1m unaligned absolute of queue_message_drop
41
+ units: dropped messages
42
+ every: 1m
43
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
44
+ delay: up 5m down 5m multiplier 1.5 max 2h
45
+ info: number of dropped messaged due to full queues in the last minute
46
+ to: sysadmin
47
+
48
+ template: vernemq_queue_message_expired
49
+ on: vernemq.queue_undelivered_messages
50
+ class: Messaging
51
+component: VerneMQ
52
+ type: Latency
53
+ lookup: sum -1m unaligned absolute of queue_message_expired
54
+ units: expired messages
55
+ every: 1m
56
+ warn: $this > (($status >= $WARNING) ? (0) : (15))
57
+ delay: up 5m down 5m multiplier 1.5 max 2h
58
+ info: number of messages which expired before delivery in the last minute
59
+ to: sysadmin
60
+
61
+ template: vernemq_queue_message_unhandled
62
+ on: vernemq.queue_undelivered_messages
63
+ class: Messaging
64
+component: VerneMQ
65
+ type: Latency
66
+ lookup: sum -1m unaligned absolute of queue_message_unhandled
67
+ units: unhandled messages
68
+ every: 1m
69
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
70
+ delay: up 5m down 5m multiplier 1.5 max 2h
71
+ info: number of unhandled messages (connections with clean session=true) in the last minute
72
+ to: sysadmin
73
74
# Erlang VM
75
61
-template: vernemq_average_scheduler_utilization
62
- on: vernemq.average_scheduler_utilization
63
- lookup: average -10m unaligned
64
- units: %
65
- every: 1m
66
- warn: $this > (($status >= $WARNING) ? (75) : (85))
67
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
68
- delay: down 15m multiplier 1.5 max 1h
69
- info: average scheduler utilization over the last 10 minutes
70
- to: sysadmin
76
+ template: vernemq_average_scheduler_utilization
77
+ on: vernemq.average_scheduler_utilization
78
+ class: Messaging
79
+component: VerneMQ
80
+ type: Utilization
81
+ lookup: average -10m unaligned
82
+ units: %
83
+ every: 1m
84
+ warn: $this > (($status >= $WARNING) ? (75) : (85))
85
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
86
+ delay: down 15m multiplier 1.5 max 1h
87
+ info: average scheduler utilization over the last 10 minutes
88
+ to: sysadmin
89
90
# Cluster communication and netsplits
91
74
-template: vernemq_cluster_dropped
75
- on: vernemq.cluster_dropped
76
- lookup: sum -1m unaligned
77
- units: KiB
78
- every: 1m
79
- warn: $this > 0
80
- delay: up 5m down 5m multiplier 1.5 max 1h
81
- info: amount of traffic dropped during communication with the cluster nodes in the last minute
82
- to: sysadmin
83
-
84
-template: vernemq_netsplits
85
- on: vernemq.netsplits
86
- lookup: sum -1m unaligned absolute of netsplit_detected
87
- units: netsplits
88
- every: 10s
89
- warn: $this > 0
90
- delay: down 5m multiplier 1.5 max 2h
91
- info: number of detected netsplits (split brain situation) in the last minute
92
- to: sysadmin
92
+ template: vernemq_cluster_dropped
93
+ on: vernemq.cluster_dropped
94
+ class: Messaging
95
+component: VerneMQ
96
+ type: Errors
97
+ lookup: sum -1m unaligned
98
+ units: KiB
99
+ every: 1m
100
+ warn: $this > 0
101
+ delay: up 5m down 5m multiplier 1.5 max 1h
102
+ info: amount of traffic dropped during communication with the cluster nodes in the last minute
103
+ to: sysadmin
104
+
105
+ template: vernemq_netsplits
106
+ on: vernemq.netsplits
107
+ class: Messaging
108
+component: VerneMQ
109
+ type: Workload
110
+ lookup: sum -1m unaligned absolute of netsplit_detected
111
+ units: netsplits
112
+ every: 10s
113
+ warn: $this > 0
114
+ delay: down 5m multiplier 1.5 max 2h
115
+ info: number of detected netsplits (split brain situation) in the last minute
116
+ to: sysadmin
117
118
# Unsuccessful CONNACK
119
96
-template: vernemq_mqtt_connack_sent_reason_unsuccessful
97
- on: vernemq.mqtt_connack_sent_reason
98
- lookup: sum -1m unaligned absolute match-names of !success,*
99
- units: packets
100
- every: 1m
101
- warn: $this > (($status >= $WARNING) ? (0) : (5))
102
- delay: up 5m down 5m multiplier 1.5 max 2h
103
- info: number of sent unsuccessful v3/v5 CONNACK packets in the last minute
104
- to: sysadmin
120
+ template: vernemq_mqtt_connack_sent_reason_unsuccessful
121
+ on: vernemq.mqtt_connack_sent_reason
122
+ class: Messaging
123
+component: VerneMQ
124
+ type: Errors
125
+ lookup: sum -1m unaligned absolute match-names of !success,*
126
+ units: packets
127
+ every: 1m
128
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
129
+ delay: up 5m down 5m multiplier 1.5 max 2h
130
+ info: number of sent unsuccessful v3/v5 CONNACK packets in the last minute
131
+ to: sysadmin
132
133
# Not normal DISCONNECT
134
108
-template: vernemq_mqtt_disconnect_received_reason_not_normal
109
- on: vernemq.mqtt_disconnect_received_reason
110
- lookup: sum -1m unaligned absolute match-names of !normal_disconnect,*
111
- units: packets
112
- every: 1m
113
- warn: $this > (($status >= $WARNING) ? (0) : (5))
114
- delay: up 5m down 5m multiplier 1.5 max 2h
115
- info: number of received not normal v5 DISCONNECT packets in the last minute
116
- to: sysadmin
117
-
118
-template: vernemq_mqtt_disconnect_sent_reason_not_normal
119
- on: vernemq.mqtt_disconnect_sent_reason
120
- lookup: sum -1m unaligned absolute match-names of !normal_disconnect,*
121
- units: packets
122
- every: 1m
123
- warn: $this > (($status >= $WARNING) ? (0) : (5))
124
- delay: up 5m down 5m multiplier 1.5 max 2h
125
- info: number of sent not normal v5 DISCONNECT packets in the last minute
126
- to: sysadmin
135
+ template: vernemq_mqtt_disconnect_received_reason_not_normal
136
+ on: vernemq.mqtt_disconnect_received_reason
137
+ class: Messaging
138
+component: VerneMQ
139
+ type: Workload
140
+ lookup: sum -1m unaligned absolute match-names of !normal_disconnect,*
141
+ units: packets
142
+ every: 1m
143
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
144
+ delay: up 5m down 5m multiplier 1.5 max 2h
145
+ info: number of received not normal v5 DISCONNECT packets in the last minute
146
+ to: sysadmin
147
+
148
+ template: vernemq_mqtt_disconnect_sent_reason_not_normal
149
+ on: vernemq.mqtt_disconnect_sent_reason
150
+ class: Messaging
151
+component: VerneMQ
152
+ type: Errors
153
+ lookup: sum -1m unaligned absolute match-names of !normal_disconnect,*
154
+ units: packets
155
+ every: 1m
156
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
157
+ delay: up 5m down 5m multiplier 1.5 max 2h
158
+ info: number of sent not normal v5 DISCONNECT packets in the last minute
159
+ to: sysadmin
160
161
# SUBSCRIBE errors and unauthorized attempts
162
130
-template: vernemq_mqtt_subscribe_error
131
- on: vernemq.mqtt_subscribe_error
132
- lookup: sum -1m unaligned absolute
133
- units: failed ops
134
- every: 1m
135
- warn: $this > (($status >= $WARNING) ? (0) : (5))
136
- delay: up 5m down 5m multiplier 1.5 max 2h
137
- info: number of failed v3/v5 SUBSCRIBE operations in the last minute
138
- to: sysadmin
139
-
140
-template: vernemq_mqtt_subscribe_auth_error
141
- on: vernemq.mqtt_subscribe_auth_error
142
- lookup: sum -1m unaligned absolute
143
- units: attempts
144
- every: 1m
145
- warn: $this > (($status >= $WARNING) ? (0) : (5))
146
- delay: up 5m down 5m multiplier 1.5 max 2h
147
- info: number of unauthorized v3/v5 SUBSCRIBE attempts in the last minute
148
- to: sysadmin
163
+ template: vernemq_mqtt_subscribe_error
164
+ on: vernemq.mqtt_subscribe_error
165
+ class: Messaging
166
+component: VerneMQ
167
+ type: Errors
168
+ lookup: sum -1m unaligned absolute
169
+ units: failed ops
170
+ every: 1m
171
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
172
+ delay: up 5m down 5m multiplier 1.5 max 2h
173
+ info: number of failed v3/v5 SUBSCRIBE operations in the last minute
174
+ to: sysadmin
175
+
176
+ template: vernemq_mqtt_subscribe_auth_error
177
+ on: vernemq.mqtt_subscribe_auth_error
178
+ class: Messaging
179
+component: VerneMQ
180
+ type: Workload
181
+ lookup: sum -1m unaligned absolute
182
+ units: attempts
183
+ every: 1m
184
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
185
+ delay: up 5m down 5m multiplier 1.5 max 2h
186
+ info: number of unauthorized v3/v5 SUBSCRIBE attempts in the last minute
187
+ to: sysadmin
188
189
# UNSUBSCRIBE errors
190
152
-template: vernemq_mqtt_unsubscribe_error
153
- on: vernemq.mqtt_unsubscribe_error
154
- lookup: sum -1m unaligned absolute
155
- units: failed ops
156
- every: 1m
157
- warn: $this > (($status >= $WARNING) ? (0) : (5))
158
- delay: up 5m down 5m multiplier 1.5 max 2h
159
- info: number of failed v3/v5 UNSUBSCRIBE operations in the last minute
160
- to: sysadmin
191
+ template: vernemq_mqtt_unsubscribe_error
192
+ on: vernemq.mqtt_unsubscribe_error
193
+ class: Messaging
194
+component: VerneMQ
195
+ type: Errors
196
+ lookup: sum -1m unaligned absolute
197
+ units: failed ops
198
+ every: 1m
199
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
200
+ delay: up 5m down 5m multiplier 1.5 max 2h
201
+ info: number of failed v3/v5 UNSUBSCRIBE operations in the last minute
202
+ to: sysadmin
203
204
# PUBLISH errors and unauthorized attempts
205
164
-template: vernemq_mqtt_publish_errors
165
- on: vernemq.mqtt_publish_errors
166
- lookup: sum -1m unaligned absolute
167
- units: failed ops
168
- every: 1m
169
- warn: $this > (($status >= $WARNING) ? (0) : (5))
170
- delay: up 5m down 5m multiplier 1.5 max 2h
171
- info: number of failed v3/v5 PUBLISH operations in the last minute
172
- to: sysadmin
173
-
174
-template: vernemq_mqtt_publish_auth_errors
175
- on: vernemq.mqtt_publish_auth_errors
176
- lookup: sum -1m unaligned absolute
177
- units: attempts
178
- every: 1m
179
- warn: $this > (($status >= $WARNING) ? (0) : (5))
180
- delay: up 5m down 5m multiplier 1.5 max 2h
181
- info: number of unauthorized v3/v5 PUBLISH attempts in the last minute
182
- to: sysadmin
206
+ template: vernemq_mqtt_publish_errors
207
+ on: vernemq.mqtt_publish_errors
208
+ class: Messaging
209
+component: VerneMQ
210
+ type: Errors
211
+ lookup: sum -1m unaligned absolute
212
+ units: failed ops
213
+ every: 1m
214
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
215
+ delay: up 5m down 5m multiplier 1.5 max 2h
216
+ info: number of failed v3/v5 PUBLISH operations in the last minute
217
+ to: sysadmin
218
+
219
+ template: vernemq_mqtt_publish_auth_errors
220
+ on: vernemq.mqtt_publish_auth_errors
221
+ class: Messaging
222
+component: VerneMQ
223
+ type: Workload
224
+ lookup: sum -1m unaligned absolute
225
+ units: attempts
226
+ every: 1m
227
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
228
+ delay: up 5m down 5m multiplier 1.5 max 2h
229
+ info: number of unauthorized v3/v5 PUBLISH attempts in the last minute
230
+ to: sysadmin
231
232
# Unsuccessful and unexpected PUBACK
233
186
-template: vernemq_mqtt_puback_received_reason_unsuccessful
187
- on: vernemq.mqtt_puback_received_reason
188
- lookup: sum -1m unaligned absolute match-names of !success,*
189
- units: packets
190
- every: 1m
191
- warn: $this > (($status >= $WARNING) ? (0) : (5))
192
- delay: up 5m down 5m multiplier 1.5 max 2h
193
- info: number of received unsuccessful v5 PUBACK packets in the last minute
194
- to: sysadmin
195
-
196
-template: vernemq_mqtt_puback_sent_reason_unsuccessful
197
- on: vernemq.mqtt_puback_sent_reason
198
- lookup: sum -1m unaligned absolute match-names of !success,*
199
- units: packets
200
- every: 1m
201
- warn: $this > (($status >= $WARNING) ? (0) : (5))
202
- delay: up 5m down 5m multiplier 1.5 max 2h
203
- info: number of sent unsuccessful v5 PUBACK packets in the last minute
204
- to: sysadmin
205
-
206
-template: vernemq_mqtt_puback_unexpected
207
- on: vernemq.mqtt_puback_invalid_error
208
- lookup: sum -1m unaligned absolute
209
- units: messages
210
- every: 1m
211
- warn: $this > (($status >= $WARNING) ? (0) : (5))
212
- delay: up 5m down 5m multiplier 1.5 max 2h
213
- info: number of received unexpected v3/v5 PUBACK packets in the last minute
214
- to: sysadmin
234
+ template: vernemq_mqtt_puback_received_reason_unsuccessful
235
+ on: vernemq.mqtt_puback_received_reason
236
+ class: Messaging
237
+component: VerneMQ
238
+ type: Errors
239
+ lookup: sum -1m unaligned absolute match-names of !success,*
240
+ units: packets
241
+ every: 1m
242
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
243
+ delay: up 5m down 5m multiplier 1.5 max 2h
244
+ info: number of received unsuccessful v5 PUBACK packets in the last minute
245
+ to: sysadmin
246
+
247
+ template: vernemq_mqtt_puback_sent_reason_unsuccessful
248
+ on: vernemq.mqtt_puback_sent_reason
249
+ class: Messaging
250
+component: VerneMQ
251
+ type: Errors
252
+ lookup: sum -1m unaligned absolute match-names of !success,*
253
+ units: packets
254
+ every: 1m
255
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
256
+ delay: up 5m down 5m multiplier 1.5 max 2h
257
+ info: number of sent unsuccessful v5 PUBACK packets in the last minute
258
+ to: sysadmin
259
+
260
+ template: vernemq_mqtt_puback_unexpected
261
+ on: vernemq.mqtt_puback_invalid_error
262
+ class: Messaging
263
+component: VerneMQ
264
+ type: Workload
265
+ lookup: sum -1m unaligned absolute
266
+ units: messages
267
+ every: 1m
268
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
269
+ delay: up 5m down 5m multiplier 1.5 max 2h
270
+ info: number of received unexpected v3/v5 PUBACK packets in the last minute
271
+ to: sysadmin
272
273
# Unsuccessful and unexpected PUBREC
274
218
-template: vernemq_mqtt_pubrec_received_reason_unsuccessful
219
- on: vernemq.mqtt_pubrec_received_reason
220
- lookup: sum -1m unaligned absolute match-names of !success,*
221
- units: packets
222
- every: 1m
223
- warn: $this > (($status >= $WARNING) ? (0) : (5))
224
- delay: up 5m down 5m multiplier 1.5 max 2h
225
- info: number of received unsuccessful v5 PUBREC packets in the last minute
226
- to: sysadmin
227
-
228
-template: vernemq_mqtt_pubrec_sent_reason_unsuccessful
229
- on: vernemq.mqtt_pubrec_sent_reason
230
- lookup: sum -1m unaligned absolute match-names of !success,*
231
- units: packets
232
- every: 1m
233
- warn: $this > (($status >= $WARNING) ? (0) : (5))
234
- delay: up 5m down 5m multiplier 1.5 max 2h
235
- info: number of sent unsuccessful v5 PUBREC packets in the last minute
236
- to: sysadmin
237
-
238
-template: vernemq_mqtt_pubrec_invalid_error
239
- on: vernemq.mqtt_pubrec_invalid_error
240
- lookup: sum -1m unaligned absolute
241
- units: messages
242
- every: 1m
243
- warn: $this > (($status >= $WARNING) ? (0) : (5))
244
- delay: up 5m down 5m multiplier 1.5 max 2h
245
- info: number of received unexpected v3 PUBREC packets in the last minute
246
- to: sysadmin
275
+ template: vernemq_mqtt_pubrec_received_reason_unsuccessful
276
+ on: vernemq.mqtt_pubrec_received_reason
277
+ class: Messaging
278
+component: VerneMQ
279
+ type: Errors
280
+ lookup: sum -1m unaligned absolute match-names of !success,*
281
+ units: packets
282
+ every: 1m
283
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
284
+ delay: up 5m down 5m multiplier 1.5 max 2h
285
+ info: number of received unsuccessful v5 PUBREC packets in the last minute
286
+ to: sysadmin
287
+
288
+ template: vernemq_mqtt_pubrec_sent_reason_unsuccessful
289
+ on: vernemq.mqtt_pubrec_sent_reason
290
+ class: Messaging
291
+component: VerneMQ
292
+ type: Errors
293
+ lookup: sum -1m unaligned absolute match-names of !success,*
294
+ units: packets
295
+ every: 1m
296
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
297
+ delay: up 5m down 5m multiplier 1.5 max 2h
298
+ info: number of sent unsuccessful v5 PUBREC packets in the last minute
299
+ to: sysadmin
300
+
301
+ template: vernemq_mqtt_pubrec_invalid_error
302
+ on: vernemq.mqtt_pubrec_invalid_error
303
+ class: Messaging
304
+component: VerneMQ
305
+ type: Workload
306
+ lookup: sum -1m unaligned absolute
307
+ units: messages
308
+ every: 1m
309
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
310
+ delay: up 5m down 5m multiplier 1.5 max 2h
311
+ info: number of received unexpected v3 PUBREC packets in the last minute
312
+ to: sysadmin
313
314
# Unsuccessful PUBREL
315
250
-template: vernemq_mqtt_pubrel_received_reason_unsuccessful
251
- on: vernemq.mqtt_pubrel_received_reason
252
- lookup: sum -1m unaligned absolute match-names of !success,*
253
- units: packets
254
- every: 1m
255
- warn: $this > (($status >= $WARNING) ? (0) : (5))
256
- delay: up 5m down 5m multiplier 1.5 max 2h
257
- info: number of received unsuccessful v5 PUBREL packets in the last minute
258
- to: sysadmin
259
-
260
-template: vernemq_mqtt_pubrel_sent_reason_unsuccessful
261
- on: vernemq.mqtt_pubrel_sent_reason
262
- lookup: sum -1m unaligned absolute match-names of !success,*
263
- units: packets
264
- every: 1m
265
- warn: $this > (($status >= $WARNING) ? (0) : (5))
266
- delay: up 5m down 5m multiplier 1.5 max 2h
267
- info: number of sent unsuccessful v5 PUBREL packets in the last minute
268
- to: sysadmin
316
+ template: vernemq_mqtt_pubrel_received_reason_unsuccessful
317
+ on: vernemq.mqtt_pubrel_received_reason
318
+ class: Messaging
319
+component: VerneMQ
320
+ type: Errors
321
+ lookup: sum -1m unaligned absolute match-names of !success,*
322
+ units: packets
323
+ every: 1m
324
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
325
+ delay: up 5m down 5m multiplier 1.5 max 2h
326
+ info: number of received unsuccessful v5 PUBREL packets in the last minute
327
+ to: sysadmin
328
+
329
+ template: vernemq_mqtt_pubrel_sent_reason_unsuccessful
330
+ on: vernemq.mqtt_pubrel_sent_reason
331
+ class: Messaging
332
+component: VerneMQ
333
+ type: Errors
334
+ lookup: sum -1m unaligned absolute match-names of !success,*
335
+ units: packets
336
+ every: 1m
337
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
338
+ delay: up 5m down 5m multiplier 1.5 max 2h
339
+ info: number of sent unsuccessful v5 PUBREL packets in the last minute
340
+ to: sysadmin
341
342
# Unsuccessful and unexpected PUBCOMP
343
272
-template: vernemq_mqtt_pubcomp_received_reason_unsuccessful
273
- on: vernemq.mqtt_pubcomp_received_reason
274
- lookup: sum -1m unaligned absolute match-names of !success,*
275
- units: packets
276
- every: 1m
277
- warn: $this > (($status >= $WARNING) ? (0) : (5))
278
- delay: up 5m down 5m multiplier 1.5 max 2h
279
- info: number of received unsuccessful v5 PUBCOMP packets in the last minute
280
- to: sysadmin
281
-
282
-template: vernemq_mqtt_pubcomp_sent_reason_unsuccessful
283
- on: vernemq.mqtt_pubcomp_sent_reason
284
- lookup: sum -1m unaligned absolute match-names of !success,*
285
- units: packets
286
- every: 1m
287
- warn: $this > (($status >= $WARNING) ? (0) : (5))
288
- delay: up 5m down 5m multiplier 1.5 max 2h
289
- info: number of sent unsuccessful v5 PUBCOMP packets in the last minute
290
- to: sysadmin
291
-
292
-template: vernemq_mqtt_pubcomp_unexpected
293
- on: vernemq.mqtt_pubcomp_invalid_error
294
- lookup: sum -1m unaligned absolute
295
- units: messages
296
- every: 1m
297
- warn: $this > (($status >= $WARNING) ? (0) : (5))
298
- delay: up 5m down 5m multiplier 1.5 max 2h
299
- info: number of received unexpected v3/v5 PUBCOMP packets in the last minute
300
- to: sysadmin
344
+ template: vernemq_mqtt_pubcomp_received_reason_unsuccessful
345
+ on: vernemq.mqtt_pubcomp_received_reason
346
+ class: Messaging
347
+component: VerneMQ
348
+ type: Errors
349
+ lookup: sum -1m unaligned absolute match-names of !success,*
350
+ units: packets
351
+ every: 1m
352
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
353
+ delay: up 5m down 5m multiplier 1.5 max 2h
354
+ info: number of received unsuccessful v5 PUBCOMP packets in the last minute
355
+ to: sysadmin
356
+
357
+ template: vernemq_mqtt_pubcomp_sent_reason_unsuccessful
358
+ on: vernemq.mqtt_pubcomp_sent_reason
359
+ class: Messaging
360
+component: VerneMQ
361
+ type: Errors
362
+ lookup: sum -1m unaligned absolute match-names of !success,*
363
+ units: packets
364
+ every: 1m
365
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
366
+ delay: up 5m down 5m multiplier 1.5 max 2h
367
+ info: number of sent unsuccessful v5 PUBCOMP packets in the last minute
368
+ to: sysadmin
369
+
370
+ template: vernemq_mqtt_pubcomp_unexpected
371
+ on: vernemq.mqtt_pubcomp_invalid_error
372
+ class: Messaging
373
+component: VerneMQ
374
+ type: Workload
375
+ lookup: sum -1m unaligned absolute
376
+ units: messages
377
+ every: 1m
378
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
379
+ delay: up 5m down 5m multiplier 1.5 max 2h
380
+ info: number of received unexpected v3/v5 PUBCOMP packets in the last minute
381
+ to: sysadmin
health/health.d/vsphere.conf
+148
-115
@@ -4,138 +4,171 @@
4
# -----------------------------------------------VM Specific------------------------------------------------------------
5
# Memory
6
7
-template: vsphere_vm_mem_usage
8
- on: vsphere.vm_mem_usage_percentage
9
- hosts: *
10
- calc: $used
11
- units: %
12
- every: 20s
13
- warn: $this > (($status >= $WARNING) ? (80) : (90))
14
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
15
- delay: down 15m multiplier 1.5 max 1h
16
- info: virtual machine memory utilization
7
+ template: vsphere_vm_mem_usage
8
+ on: vsphere.vm_mem_usage_percentage
9
+ class: Virtual Machine
10
+component: Memory
11
+ type: Utilization
12
+ hosts: *
13
+ calc: $used
14
+ units: %
15
+ every: 20s
16
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
17
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
18
+ delay: down 15m multiplier 1.5 max 1h
19
+ info: virtual machine memory utilization
20
21
# -----------------------------------------------HOST Specific----------------------------------------------------------
22
# Memory
23
21
-template: vsphere_host_mem_usage
22
- on: vsphere.host_mem_usage_percentage
23
- hosts: *
24
- calc: $used
25
- units: %
26
- every: 20s
27
- warn: $this > (($status >= $WARNING) ? (80) : (90))
28
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
29
- delay: down 15m multiplier 1.5 max 1h
30
- info: host memory utilization
24
+ template: vsphere_host_mem_usage
25
+ on: vsphere.host_mem_usage_percentage
26
+ class: Virtual Machine
27
+component: Memory
28
+ type: Utilization
29
+ hosts: *
30
+ calc: $used
31
+ units: %
32
+ every: 20s
33
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
34
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
35
+ delay: down 15m multiplier 1.5 max 1h
36
+ info: host memory utilization
37
38
# Network errors
39
34
-template: vsphere_inbound_packets_errors
35
- on: vsphere.net_errors_total
36
- hosts: *
37
-families: *
38
- lookup: sum -10m unaligned absolute match-names of rx
39
- units: packets
40
- every: 1m
41
- info: number of inbound errors for the network interface in the last 10 minutes
42
-
43
-template: vsphere_outbound_packets_errors
44
- on: vsphere.net_errors_total
45
- hosts: *
46
-families: *
47
- lookup: sum -10m unaligned absolute match-names of tx
48
- units: packets
49
- every: 1m
50
- info: number of outbound errors for the network interface in the last 10 minutes
40
+ template: vsphere_inbound_packets_errors
41
+ on: vsphere.net_errors_total
42
+ class: Virtual Machine
43
+component: Network
44
+ type: Errors
45
+ hosts: *
46
+ families: *
47
+ lookup: sum -10m unaligned absolute match-names of rx
48
+ units: packets
49
+ every: 1m
50
+ info: number of inbound errors for the network interface in the last 10 minutes
51
+
52
+ template: vsphere_outbound_packets_errors
53
+ on: vsphere.net_errors_total
54
+ class: Virtual Machine
55
+component: Network
56
+ type: Errors
57
+ hosts: *
58
+ families: *
59
+ lookup: sum -10m unaligned absolute match-names of tx
60
+ units: packets
61
+ every: 1m
62
+ info: number of outbound errors for the network interface in the last 10 minutes
63
64
# Network errors ratio
65
54
-template: vsphere_inbound_packets_errors_ratio
55
- on: vsphere.net_packets_total
56
- hosts: *
57
-families: *
58
- lookup: sum -10m unaligned absolute match-names of rx
59
- calc: (($vsphere_inbound_packets_errors != nan AND $this > 1000) ? ($vsphere_inbound_packets_errors * 100 / $this) : (0))
60
- units: %
61
- every: 1m
62
- warn: $this >= 2
63
- delay: up 1m down 1h multiplier 1.5 max 2h
64
- info: ratio of inbound errors for the network interface over the last 10 minutes
65
- to: sysadmin
66
-
67
-template: vsphere_outbound_packets_errors_ratio
68
- on: vsphere.net_packets_total
69
- hosts: *
70
-families: *
71
- lookup: sum -10m unaligned absolute match-names of tx
72
- calc: (($vsphere_outbound_packets_errors != nan AND $this > 1000) ? ($vsphere_outbound_packets_errors * 100 / $this) : (0))
73
- units: %
74
- every: 1m
75
- warn: $this >= 2
76
- delay: up 1m down 1h multiplier 1.5 max 2h
77
- info: ratio of outbound errors for the network interface over the last 10 minutes
78
- to: sysadmin
66
+ template: vsphere_inbound_packets_errors_ratio
67
+ on: vsphere.net_packets_total
68
+ class: Virtual Machine
69
+component: Network
70
+ type: Errors
71
+ hosts: *
72
+ families: *
73
+ lookup: sum -10m unaligned absolute match-names of rx
74
+ calc: (($vsphere_inbound_packets_errors != nan AND $this > 1000) ? ($vsphere_inbound_packets_errors * 100 / $this) : (0))
75
+ units: %
76
+ every: 1m
77
+ warn: $this >= 2
78
+ delay: up 1m down 1h multiplier 1.5 max 2h
79
+ info: ratio of inbound errors for the network interface over the last 10 minutes
80
+ to: sysadmin
81
+
82
+ template: vsphere_outbound_packets_errors_ratio
83
+ on: vsphere.net_packets_total
84
+ class: Virtual Machine
85
+component: Network
86
+ type: Errors
87
+ hosts: *
88
+ families: *
89
+ lookup: sum -10m unaligned absolute match-names of tx
90
+ calc: (($vsphere_outbound_packets_errors != nan AND $this > 1000) ? ($vsphere_outbound_packets_errors * 100 / $this) : (0))
91
+ units: %
92
+ every: 1m
93
+ warn: $this >= 2
94
+ delay: up 1m down 1h multiplier 1.5 max 2h
95
+ info: ratio of outbound errors for the network interface over the last 10 minutes
96
+ to: sysadmin
97
98
# -----------------------------------------------Common-------------------------------------------------------------------
99
# CPU
100
83
-template: vsphere_cpu_usage
84
- on: vsphere.cpu_usage_total
85
- hosts: *
86
- lookup: average -10m unaligned match-names of used
87
- units: %
88
- every: 20s
89
- warn: $this > (($status >= $WARNING) ? (75) : (85))
90
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
91
- delay: down 15m multiplier 1.5 max 1h
92
- info: average CPU utilization
93
- to: sysadmin
101
+ template: vsphere_cpu_usage
102
+ on: vsphere.cpu_usage_total
103
+ class: Virtual Machine
104
+component: CPU
105
+ type: Utilization
106
+ hosts: *
107
+ lookup: average -10m unaligned match-names of used
108
+ units: %
109
+ every: 20s
110
+ warn: $this > (($status >= $WARNING) ? (75) : (85))
111
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
112
+ delay: down 15m multiplier 1.5 max 1h
113
+ info: average CPU utilization
114
+ to: sysadmin
115
116
# Network drops
117
97
-template: vsphere_inbound_packets_dropped
98
- on: vsphere.net_drops_total
99
- hosts: *
100
-families: *
101
- lookup: sum -10m unaligned absolute match-names of rx
102
- units: packets
103
- every: 1m
104
- info: number of inbound dropped packets for the network interface in the last 10 minutes
105
-
106
-template: vsphere_outbound_packets_dropped
107
- on: vsphere.net_drops_total
108
- hosts: *
109
-families: *
110
- lookup: sum -10m unaligned absolute match-names of tx
111
- units: packets
112
- every: 1m
113
- info: number of outbound dropped packets for the network interface in the last 10 minutes
118
+ template: vsphere_inbound_packets_dropped
119
+ on: vsphere.net_drops_total
120
+ class: Virtual Machine
121
+component: Network
122
+ type: Errors
123
+ hosts: *
124
+ families: *
125
+ lookup: sum -10m unaligned absolute match-names of rx
126
+ units: packets
127
+ every: 1m
128
+ info: number of inbound dropped packets for the network interface in the last 10 minutes
129
+
130
+ template: vsphere_outbound_packets_dropped
131
+ on: vsphere.net_drops_total
132
+ class: Virtual Machine
133
+component: Network
134
+ type: Errors
135
+ hosts: *
136
+ families: *
137
+ lookup: sum -10m unaligned absolute match-names of tx
138
+ units: packets
139
+ every: 1m
140
+ info: number of outbound dropped packets for the network interface in the last 10 minutes
141
142
# Network drops ratio
143
117
-template: vsphere_inbound_packets_dropped_ratio
118
- on: vsphere.net_packets_total
119
- hosts: *
120
-families: *
121
- lookup: sum -10m unaligned absolute match-names of rx
122
- calc: (($vsphere_inbound_packets_dropped != nan AND $this > 1000) ? ($vsphere_inbound_packets_dropped * 100 / $this) : (0))
123
- units: %
124
- every: 1m
125
- warn: $this >= 2
126
- delay: up 1m down 1h multiplier 1.5 max 2h
127
- info: ratio of inbound dropped packets for the network interface over the last 10 minutes
128
- to: sysadmin
129
-
130
-template: vsphere_outbound_packets_dropped_ratio
131
- on: vsphere.net_packets_total
132
- hosts: *
133
-families: *
134
- lookup: sum -10m unaligned absolute match-names of tx
135
- calc: (($vsphere_outbound_packets_dropped != nan AND $this > 1000) ? ($vsphere_outbound_packets_dropped * 100 / $this) : (0))
136
- units: %
137
- every: 1m
138
- warn: $this >= 2
139
- delay: up 1m down 1h multiplier 1.5 max 2h
140
- info: ratio of outbound dropped packets for the network interface over the last 10 minutes
141
- to: sysadmin
144
+ template: vsphere_inbound_packets_dropped_ratio
145
+ on: vsphere.net_packets_total
146
+ class: Virtual Machine
147
+component: Network
148
+ type: Errors
149
+ hosts: *
150
+ families: *
151
+ lookup: sum -10m unaligned absolute match-names of rx
152
+ calc: (($vsphere_inbound_packets_dropped != nan AND $this > 1000) ? ($vsphere_inbound_packets_dropped * 100 / $this) : (0))
153
+ units: %
154
+ every: 1m
155
+ warn: $this >= 2
156
+ delay: up 1m down 1h multiplier 1.5 max 2h
157
+ info: ratio of inbound dropped packets for the network interface over the last 10 minutes
158
+ to: sysadmin
159
+
160
+ template: vsphere_outbound_packets_dropped_ratio
161
+ on: vsphere.net_packets_total
162
+ class: Virtual Machine
163
+component: Network
164
+ type: Errors
165
+ hosts: *
166
+ families: *
167
+ lookup: sum -10m unaligned absolute match-names of tx
168
+ calc: (($vsphere_outbound_packets_dropped != nan AND $this > 1000) ? ($vsphere_outbound_packets_dropped * 100 / $this) : (0))
169
+ units: %
170
+ every: 1m
171
+ warn: $this >= 2
172
+ delay: up 1m down 1h multiplier 1.5 max 2h
173
+ info: ratio of outbound dropped packets for the network interface over the last 10 minutes
174
+ to: sysadmin
health/health.d/web_log.conf
+364
-286
@@ -1,17 +1,20 @@
1
2
# make sure we can collect web log data
3
4
-template: last_collected_secs
5
- on: web_log.response_codes
6
-families: *
7
- calc: $now - $last_collected_t
8
- units: seconds ago
9
- every: 10s
10
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
11
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
12
- delay: down 5m multiplier 1.5 max 1h
13
- info: number of seconds since the last successful data collection
14
- to: webmaster
4
+ template: last_collected_secs
5
+ on: web_log.response_codes
6
+ class: Web Server
7
+component: Web log
8
+ type: Latency
9
+ families: *
10
+ calc: $now - $last_collected_t
11
+ units: seconds ago
12
+ every: 10s
13
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
14
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
15
+ delay: down 5m multiplier 1.5 max 1h
16
+ info: number of seconds since the last successful data collection
17
+ to: webmaster
18
19
20
# -----------------------------------------------------------------------------
@@ -24,66 +27,81 @@ families: *
27
#
28
# i.e. when there are at least 120 requests during the last minute
29
27
-template: 1m_requests
28
- on: web_log.response_statuses
29
-families: *
30
- lookup: sum -1m unaligned
31
- calc: ($this == 0)?(1):($this)
32
- units: requests
33
- every: 10s
34
- info: number of HTTP requests in the last minute
35
-
36
-template: 1m_successful
37
- on: web_log.response_statuses
38
-families: *
39
- lookup: sum -1m unaligned of successful_requests
40
- calc: $this * 100 / $1m_requests
41
- units: %
42
- every: 10s
43
- warn: ($1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
44
- crit: ($1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
45
- delay: up 2m down 15m multiplier 1.5 max 1h
46
- info: ratio of successful HTTP requests over the last minute (1xx, 2xx, 304, 401)
47
- to: webmaster
48
-
49
-template: 1m_redirects
50
- on: web_log.response_statuses
51
-families: *
52
- lookup: sum -1m unaligned of redirects
53
- calc: $this * 100 / $1m_requests
54
- units: %
55
- every: 10s
56
- warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING ) ? ( 1 ) : ( 20 )) ) : ( 0 )
57
- crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 20 ) : ( 30 )) ) : ( 0 )
58
- delay: up 2m down 15m multiplier 1.5 max 1h
59
- info: ratio of redirection HTTP requests over the last minute (3xx except 304)
60
- to: webmaster
61
-
62
-template: 1m_bad_requests
63
- on: web_log.response_statuses
64
-families: *
65
- lookup: sum -1m unaligned of bad_requests
66
- calc: $this * 100 / $1m_requests
67
- units: %
68
- every: 10s
69
- warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 10 ) : ( 30 )) ) : ( 0 )
70
- crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 30 ) : ( 50 )) ) : ( 0 )
71
- delay: up 2m down 15m multiplier 1.5 max 1h
72
- info: ratio of client error HTTP requests over the last minute (4xx except 401)
73
- to: webmaster
74
-
75
-template: 1m_internal_errors
76
- on: web_log.response_statuses
77
-families: *
78
- lookup: sum -1m unaligned of server_errors
79
- calc: $this * 100 / $1m_requests
80
- units: %
81
- every: 10s
82
- warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 2 )) ) : ( 0 )
83
- crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
84
- delay: up 2m down 15m multiplier 1.5 max 1h
85
- info: ratio of server error HTTP requests over the last minute (5xx)
86
- to: webmaster
30
+ template: 1m_requests
31
+ on: web_log.response_statuses
32
+ class: Web Server
33
+component: Web log
34
+ type: Workload
35
+ families: *
36
+ lookup: sum -1m unaligned
37
+ calc: ($this == 0)?(1):($this)
38
+ units: requests
39
+ every: 10s
40
+ info: number of HTTP requests in the last minute
41
+
42
+ template: 1m_successful
43
+ on: web_log.response_statuses
44
+ class: Web Server
45
+component: Web log
46
+ type: Workload
47
+ families: *
48
+ lookup: sum -1m unaligned of successful_requests
49
+ calc: $this * 100 / $1m_requests
50
+ units: %
51
+ every: 10s
52
+ warn: ($1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
53
+ crit: ($1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
54
+ delay: up 2m down 15m multiplier 1.5 max 1h
55
+ info: ratio of successful HTTP requests over the last minute (1xx, 2xx, 304, 401)
56
+ to: webmaster
57
+
58
+ template: 1m_redirects
59
+ on: web_log.response_statuses
60
+ class: Web Server
61
+component: Web log
62
+ type: Workload
63
+ families: *
64
+ lookup: sum -1m unaligned of redirects
65
+ calc: $this * 100 / $1m_requests
66
+ units: %
67
+ every: 10s
68
+ warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING ) ? ( 1 ) : ( 20 )) ) : ( 0 )
69
+ crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 20 ) : ( 30 )) ) : ( 0 )
70
+ delay: up 2m down 15m multiplier 1.5 max 1h
71
+ info: ratio of redirection HTTP requests over the last minute (3xx except 304)
72
+ to: webmaster
73
+
74
+ template: 1m_bad_requests
75
+ on: web_log.response_statuses
76
+ class: Web Server
77
+component: Web log
78
+ type: Errors
79
+ families: *
80
+ lookup: sum -1m unaligned of bad_requests
81
+ calc: $this * 100 / $1m_requests
82
+ units: %
83
+ every: 10s
84
+ warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 10 ) : ( 30 )) ) : ( 0 )
85
+ crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 30 ) : ( 50 )) ) : ( 0 )
86
+ delay: up 2m down 15m multiplier 1.5 max 1h
87
+ info: ratio of client error HTTP requests over the last minute (4xx except 401)
88
+ to: webmaster
89
+
90
+ template: 1m_internal_errors
91
+ on: web_log.response_statuses
92
+ class: Web Server
93
+component: Web log
94
+ type: Errors
95
+ families: *
96
+ lookup: sum -1m unaligned of server_errors
97
+ calc: $this * 100 / $1m_requests
98
+ units: %
99
+ every: 10s
100
+ warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 2 )) ) : ( 0 )
101
+ crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
102
+ delay: up 2m down 15m multiplier 1.5 max 1h
103
+ info: ratio of server error HTTP requests over the last minute (5xx)
104
+ to: webmaster
105
106
# unmatched lines
107
@@ -94,26 +112,32 @@ families: *
112
#
113
# i.e. when there are at least 120 requests during the last minute
114
97
-template: 1m_total_requests
98
- on: web_log.response_codes
99
-families: *
100
- lookup: sum -1m unaligned
101
- calc: ($this == 0)?(1):($this)
102
- units: requests
103
- every: 10s
104
- info: number of HTTP requests over the last minute
105
-
106
-template: 1m_unmatched
107
- on: web_log.response_codes
108
-families: *
109
- lookup: sum -1m unaligned of unmatched
110
- calc: $this * 100 / $1m_total_requests
111
- units: %
112
- every: 10s
113
- warn: ($1m_total_requests > 120) ? ($this > 1) : ( 0 )
114
- delay: up 1m down 5m multiplier 1.5 max 1h
115
- info: percentage of unparsed log lines over the last minute
116
- to: webmaster
115
+ template: 1m_total_requests
116
+ on: web_log.response_codes
117
+ class: Web Server
118
+component: Web log
119
+ type: Workload
120
+ families: *
121
+ lookup: sum -1m unaligned
122
+ calc: ($this == 0)?(1):($this)
123
+ units: requests
124
+ every: 10s
125
+ info: number of HTTP requests over the last minute
126
+
127
+ template: 1m_unmatched
128
+ on: web_log.response_codes
129
+ class: Web Server
130
+component: Web log
131
+ type: Errors
132
+ families: *
133
+ lookup: sum -1m unaligned of unmatched
134
+ calc: $this * 100 / $1m_total_requests
135
+ units: %
136
+ every: 10s
137
+ warn: ($1m_total_requests > 120) ? ($this > 1) : ( 0 )
138
+ delay: up 1m down 5m multiplier 1.5 max 1h
139
+ info: percentage of unparsed log lines over the last minute
140
+ to: webmaster
141
142
# -----------------------------------------------------------------------------
143
# web slow
@@ -125,28 +149,34 @@ families: *
149
#
150
# i.e. when there are at least 120 requests during the last minute
151
128
-template: 10m_response_time
129
- on: web_log.response_time
130
-families: *
131
- lookup: average -10m unaligned of avg
132
- units: ms
133
- every: 30s
134
- info: average HTTP response time over the last 10 minutes
135
-
136
-template: web_slow
137
- on: web_log.response_time
138
-families: *
139
- lookup: average -1m unaligned of avg
140
- units: ms
141
- every: 10s
142
- green: 500
143
- red: 1000
144
- warn: ($1m_requests > 120) ? ($this > $green && $this > ($10m_response_time * 2) ) : ( 0 )
145
- crit: ($1m_requests > 120) ? ($this > $red && $this > ($10m_response_time * 4) ) : ( 0 )
146
- delay: down 15m multiplier 1.5 max 1h
147
- info: average HTTP response time over the last minute
148
- options: no-clear-notification
149
- to: webmaster
152
+ template: 10m_response_time
153
+ on: web_log.response_time
154
+ class: System
155
+component: Web log
156
+ type: Latency
157
+ families: *
158
+ lookup: average -10m unaligned of avg
159
+ units: ms
160
+ every: 30s
161
+ info: average HTTP response time over the last 10 minutes
162
+
163
+ template: web_slow
164
+ on: web_log.response_time
165
+ class: Web Server
166
+component: Web log
167
+ type: Latency
168
+ families: *
169
+ lookup: average -1m unaligned of avg
170
+ units: ms
171
+ every: 10s
172
+ green: 500
173
+ red: 1000
174
+ warn: ($1m_requests > 120) ? ($this > $green && $this > ($10m_response_time * 2) ) : ( 0 )
175
+ crit: ($1m_requests > 120) ? ($this > $red && $this > ($10m_response_time * 4) ) : ( 0 )
176
+ delay: down 15m multiplier 1.5 max 1h
177
+ info: average HTTP response time over the last minute
178
+ options: no-clear-notification
179
+ to: webmaster
180
181
# -----------------------------------------------------------------------------
182
# web too many or too few requests
@@ -159,36 +189,45 @@ families: *
189
# i.e. when there were at least 120 requests during the 5 minutes starting
190
# at -10m and ending at -5m
191
162
-template: 5m_successful_old
163
- on: web_log.response_statuses
164
-families: *
165
- lookup: average -5m at -5m unaligned of successful_requests
166
- units: requests/s
167
- every: 30s
168
- info: average number of successful HTTP requests for the 5 minutes starting 10 minutes ago
169
-
170
-template: 5m_successful
171
- on: web_log.response_statuses
172
-families: *
173
- lookup: average -5m unaligned of successful_requests
174
- units: requests/s
175
- every: 30s
176
- info: average number of successful HTTP requests over the last 5 minutes
177
-
178
-template: 5m_requests_ratio
179
- on: web_log.response_codes
180
-families: *
181
- calc: ($5m_successful_old > 0)?($5m_successful * 100 / $5m_successful_old):(100)
182
- units: %
183
- every: 30s
184
- warn: ($5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
185
- crit: ($5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
186
- delay: down 15m multiplier 1.5 max 1h
187
-options: no-clear-notification
188
- info: ratio of successful HTTP requests over the last 5 minutes, \
189
- compared with the previous 5 minutes \
190
- (clear notification for this alarm will not be sent)
191
- to: webmaster
192
+ template: 5m_successful_old
193
+ on: web_log.response_statuses
194
+ class: Web Server
195
+component: Web log
196
+ type: Workload
197
+ families: *
198
+ lookup: average -5m at -5m unaligned of successful_requests
199
+ units: requests/s
200
+ every: 30s
201
+ info: average number of successful HTTP requests for the 5 minutes starting 10 minutes ago
202
+
203
+ template: 5m_successful
204
+ on: web_log.response_statuses
205
+ class: Web Server
206
+component: Web log
207
+ type: Workload
208
+ families: *
209
+ lookup: average -5m unaligned of successful_requests
210
+ units: requests/s
211
+ every: 30s
212
+ info: average number of successful HTTP requests over the last 5 minutes
213
+
214
+ template: 5m_requests_ratio
215
+ on: web_log.response_codes
216
+ class: Web Server
217
+component: Web log
218
+ type: Workload
219
+ families: *
220
+ calc: ($5m_successful_old > 0)?($5m_successful * 100 / $5m_successful_old):(100)
221
+ units: %
222
+ every: 30s
223
+ warn: ($5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
224
+ crit: ($5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
225
+ delay: down 15m multiplier 1.5 max 1h
226
+ options: no-clear-notification
227
+ info: ratio of successful HTTP requests over the last 5 minutes, \
228
+ compared with the previous 5 minutes \
229
+ (clear notification for this alarm will not be sent)
230
+ to: webmaster
231
232
233
@@ -196,17 +235,20 @@ options: no-clear-notification
235
236
# make sure we can collect web log data
237
199
-template: web_log_last_collected_secs
200
- on: web_log.requests
201
-families: *
202
- calc: $now - $last_collected_t
203
- units: seconds ago
204
- every: 10s
205
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
206
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
207
- delay: down 5m multiplier 1.5 max 1h
208
- info: number of seconds since the last successful data collection
209
- to: webmaster
238
+ template: web_log_last_collected_secs
239
+ on: web_log.requests
240
+ class: Web Server
241
+component: Web log
242
+ type: Latency
243
+ families: *
244
+ calc: $now - $last_collected_t
245
+ units: seconds ago
246
+ every: 10s
247
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
248
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
249
+ delay: down 5m multiplier 1.5 max 1h
250
+ info: number of seconds since the last successful data collection
251
+ to: webmaster
252
253
# unmatched lines
254
@@ -217,26 +259,32 @@ families: *
259
#
260
# i.e. when there are at least 120 requests during the last minute
261
220
-template: web_log_1m_total_requests
221
- on: web_log.requests
222
-families: *
223
- lookup: sum -1m unaligned
224
- calc: ($this == 0)?(1):($this)
225
- units: requests
226
- every: 10s
227
- info: number of HTTP requests in the last minute
228
-
229
-template: web_log_1m_unmatched
230
- on: web_log.excluded_requests
231
-families: *
232
- lookup: sum -1m unaligned of unmatched
233
- calc: $this * 100 / $web_log_1m_total_requests
234
- units: %
235
- every: 10s
236
- warn: ($web_log_1m_total_requests > 120) ? ($this > 1) : ( 0 )
237
- delay: up 1m down 5m multiplier 1.5 max 1h
238
- info: percentage of unparsed log lines over the last minute
239
- to: webmaster
262
+ template: web_log_1m_total_requests
263
+ on: web_log.requests
264
+ class: Web Server
265
+component: Web log
266
+ type: Workload
267
+ families: *
268
+ lookup: sum -1m unaligned
269
+ calc: ($this == 0)?(1):($this)
270
+ units: requests
271
+ every: 10s
272
+ info: number of HTTP requests in the last minute
273
+
274
+ template: web_log_1m_unmatched
275
+ on: web_log.excluded_requests
276
+ class: Web Server
277
+component: Web log
278
+ type: Errors
279
+ families: *
280
+ lookup: sum -1m unaligned of unmatched
281
+ calc: $this * 100 / $web_log_1m_total_requests
282
+ units: %
283
+ every: 10s
284
+ warn: ($web_log_1m_total_requests > 120) ? ($this > 1) : ( 0 )
285
+ delay: up 1m down 5m multiplier 1.5 max 1h
286
+ info: percentage of unparsed log lines over the last minute
287
+ to: webmaster
288
289
# -----------------------------------------------------------------------------
290
# high level response code alarms
@@ -248,66 +296,81 @@ families: *
296
#
297
# i.e. when there are at least 120 requests during the last minute
298
251
-template: web_log_1m_requests
252
- on: web_log.type_requests
253
-families: *
254
- lookup: sum -1m unaligned
255
- calc: ($this == 0)?(1):($this)
256
- units: requests
257
- every: 10s
258
- info: number of HTTP requests in the last minute
259
-
260
-template: web_log_1m_successful
261
- on: web_log.type_requests
262
-families: *
263
- lookup: sum -1m unaligned of success
264
- calc: $this * 100 / $web_log_1m_requests
265
- units: %
266
- every: 10s
267
- warn: ($web_log_1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
268
- crit: ($web_log_1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
269
- delay: up 2m down 15m multiplier 1.5 max 1h
270
- info: ratio of successful HTTP requests over the last minute (1xx, 2xx, 304, 401)
271
- to: webmaster
272
-
273
-template: web_log_1m_redirects
274
- on: web_log.type_requests
275
-families: *
276
- lookup: sum -1m unaligned of redirect
277
- calc: $this * 100 / $web_log_1m_requests
278
- units: %
279
- every: 10s
280
- warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING ) ? ( 1 ) : ( 20 )) ) : ( 0 )
281
- crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 20 ) : ( 30 )) ) : ( 0 )
282
- delay: up 2m down 15m multiplier 1.5 max 1h
283
- info: ratio of redirection HTTP requests over the last minute (3xx except 304)
284
- to: webmaster
285
-
286
-template: web_log_1m_bad_requests
287
- on: web_log.type_requests
288
-families: *
289
- lookup: sum -1m unaligned of bad
290
- calc: $this * 100 / $web_log_1m_requests
291
- units: %
292
- every: 10s
293
- warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 10 ) : ( 30 )) ) : ( 0 )
294
- crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 30 ) : ( 50 )) ) : ( 0 )
295
- delay: up 2m down 15m multiplier 1.5 max 1h
296
- info: ratio of client error HTTP requests over the last minute (4xx except 401)
297
- to: webmaster
298
-
299
-template: web_log_1m_internal_errors
300
- on: web_log.type_requests
301
-families: *
302
- lookup: sum -1m unaligned of error
303
- calc: $this * 100 / $web_log_1m_requests
304
- units: %
305
- every: 10s
306
- warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 2 )) ) : ( 0 )
307
- crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
308
- delay: up 2m down 15m multiplier 1.5 max 1h
309
- info: ratio of server error HTTP requests over the last minute (5xx)
310
- to: webmaster
299
+ template: web_log_1m_requests
300
+ on: web_log.type_requests
301
+ class: Web Server
302
+component: Web log
303
+ type: Workload
304
+ families: *
305
+ lookup: sum -1m unaligned
306
+ calc: ($this == 0)?(1):($this)
307
+ units: requests
308
+ every: 10s
309
+ info: number of HTTP requests in the last minute
310
+
311
+ template: web_log_1m_successful
312
+ on: web_log.type_requests
313
+ class: Web Server
314
+component: Web log
315
+ type: Workload
316
+ families: *
317
+ lookup: sum -1m unaligned of success
318
+ calc: $this * 100 / $web_log_1m_requests
319
+ units: %
320
+ every: 10s
321
+ warn: ($web_log_1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
322
+ crit: ($web_log_1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
323
+ delay: up 2m down 15m multiplier 1.5 max 1h
324
+ info: ratio of successful HTTP requests over the last minute (1xx, 2xx, 304, 401)
325
+ to: webmaster
326
+
327
+ template: web_log_1m_redirects
328
+ on: web_log.type_requests
329
+ class: Web Server
330
+component: Web log
331
+ type: Workload
332
+ families: *
333
+ lookup: sum -1m unaligned of redirect
334
+ calc: $this * 100 / $web_log_1m_requests
335
+ units: %
336
+ every: 10s
337
+ warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING ) ? ( 1 ) : ( 20 )) ) : ( 0 )
338
+ crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 20 ) : ( 30 )) ) : ( 0 )
339
+ delay: up 2m down 15m multiplier 1.5 max 1h
340
+ info: ratio of redirection HTTP requests over the last minute (3xx except 304)
341
+ to: webmaster
342
+
343
+ template: web_log_1m_bad_requests
344
+ on: web_log.type_requests
345
+ class: Web Server
346
+component: Web log
347
+ type: Errors
348
+ families: *
349
+ lookup: sum -1m unaligned of bad
350
+ calc: $this * 100 / $web_log_1m_requests
351
+ units: %
352
+ every: 10s
353
+ warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 10 ) : ( 30 )) ) : ( 0 )
354
+ crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 30 ) : ( 50 )) ) : ( 0 )
355
+ delay: up 2m down 15m multiplier 1.5 max 1h
356
+ info: ratio of client error HTTP requests over the last minute (4xx except 401)
357
+ to: webmaster
358
+
359
+ template: web_log_1m_internal_errors
360
+ on: web_log.type_requests
361
+ class: Web Server
362
+component: Web log
363
+ type: Errors
364
+ families: *
365
+ lookup: sum -1m unaligned of error
366
+ calc: $this * 100 / $web_log_1m_requests
367
+ units: %
368
+ every: 10s
369
+ warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 2 )) ) : ( 0 )
370
+ crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
371
+ delay: up 2m down 15m multiplier 1.5 max 1h
372
+ info: ratio of server error HTTP requests over the last minute (5xx)
373
+ to: webmaster
374
375
# -----------------------------------------------------------------------------
376
# web slow
@@ -319,28 +382,34 @@ families: *
382
#
383
# i.e. when there are at least 120 requests during the last minute
384
322
-template: web_log_10m_response_time
323
- on: web_log.request_processing_time
324
-families: *
325
- lookup: average -10m unaligned of avg
326
- units: ms
327
- every: 30s
328
- info: average HTTP response time over the last 10 minutes
329
-
330
-template: web_log_web_slow
331
- on: web_log.request_processing_time
332
-families: *
333
- lookup: average -1m unaligned of avg
334
- units: ms
335
- every: 10s
336
- green: 500
337
- red: 1000
338
- warn: ($web_log_1m_requests > 120) ? ($this > $green && $this > ($web_log_10m_response_time * 2) ) : ( 0 )
339
- crit: ($web_log_1m_requests > 120) ? ($this > $red && $this > ($web_log_10m_response_time * 4) ) : ( 0 )
340
- delay: down 15m multiplier 1.5 max 1h
341
- info: average HTTP response time over the last 1 minute
342
- options: no-clear-notification
343
- to: webmaster
385
+ template: web_log_10m_response_time
386
+ on: web_log.request_processing_time
387
+ class: System
388
+component: Web log
389
+ type: Latency
390
+ families: *
391
+ lookup: average -10m unaligned of avg
392
+ units: ms
393
+ every: 30s
394
+ info: average HTTP response time over the last 10 minutes
395
+
396
+ template: web_log_web_slow
397
+ on: web_log.request_processing_time
398
+ class: Web Server
399
+component: Web log
400
+ type: Latency
401
+ families: *
402
+ lookup: average -1m unaligned of avg
403
+ units: ms
404
+ every: 10s
405
+ green: 500
406
+ red: 1000
407
+ warn: ($web_log_1m_requests > 120) ? ($this > $green && $this > ($web_log_10m_response_time * 2) ) : ( 0 )
408
+ crit: ($web_log_1m_requests > 120) ? ($this > $red && $this > ($web_log_10m_response_time * 4) ) : ( 0 )
409
+ delay: down 15m multiplier 1.5 max 1h
410
+ info: average HTTP response time over the last 1 minute
411
+ options: no-clear-notification
412
+ to: webmaster
413
414
# -----------------------------------------------------------------------------
415
# web too many or too few requests
@@ -353,33 +422,42 @@ families: *
422
# i.e. when there were at least 120 requests during the 5 minutes starting
423
# at -10m and ending at -5m
424
356
-template: web_log_5m_successful_old
357
- on: web_log.type_requests
358
-families: *
359
- lookup: average -5m at -5m unaligned of success
360
- units: requests/s
361
- every: 30s
362
- info: average number of successful HTTP requests for the 5 minutes starting 10 minutes ago
363
-
364
-template: web_log_5m_successful
365
- on: web_log.type_requests
366
-families: *
367
- lookup: average -5m unaligned of success
368
- units: requests/s
369
- every: 30s
370
- info: average number of successful HTTP requests over the last 5 minutes
371
-
372
-template: web_log_5m_requests_ratio
373
- on: web_log.type_requests
374
-families: *
375
- calc: ($web_log_5m_successful_old > 0)?($web_log_5m_successful * 100 / $web_log_5m_successful_old):(100)
376
- units: %
377
- every: 30s
378
- warn: ($web_log_5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
379
- crit: ($web_log_5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
380
- delay: down 15m multiplier 1.5 max 1h
381
-options: no-clear-notification
382
- info: ratio of successful HTTP requests over over the last 5 minutes, \
383
- compared with the previous 5 minutes \
384
- (clear notification for this alarm will not be sent)
385
- to: webmaster
425
+ template: web_log_5m_successful_old
426
+ on: web_log.type_requests
427
+ class: Web Server
428
+component: Web log
429
+ type: Workload
430
+ families: *
431
+ lookup: average -5m at -5m unaligned of success
432
+ units: requests/s
433
+ every: 30s
434
+ info: average number of successful HTTP requests for the 5 minutes starting 10 minutes ago
435
+
436
+ template: web_log_5m_successful
437
+ on: web_log.type_requests
438
+ class: Web Server
439
+component: Web log
440
+ type: Workload
441
+ families: *
442
+ lookup: average -5m unaligned of success
443
+ units: requests/s
444
+ every: 30s
445
+ info: average number of successful HTTP requests over the last 5 minutes
446
+
447
+ template: web_log_5m_requests_ratio
448
+ on: web_log.type_requests
449
+ class: Web Server
450
+component: Web log
451
+ type: Workload
452
+ families: *
453
+ calc: ($web_log_5m_successful_old > 0)?($web_log_5m_successful * 100 / $web_log_5m_successful_old):(100)
454
+ units: %
455
+ every: 30s
456
+ warn: ($web_log_5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
457
+ crit: ($web_log_5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
458
+ delay: down 15m multiplier 1.5 max 1h
459
+ options: no-clear-notification
460
+ info: ratio of successful HTTP requests over over the last 5 minutes, \
461
+ compared with the previous 5 minutes \
462
+ (clear notification for this alarm will not be sent)
463
+ to: webmaster
health/health.d/whoisquery.conf
+25
-19
@@ -1,24 +1,30 @@
1
2
# make sure whoisquery is running
3
4
-template: whoisquery_last_collected_secs
5
- on: whoisquery.time_until_expiration
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 60s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: whoisquery_last_collected_secs
5
+ on: whoisquery.time_until_expiration
6
+ class: Other
7
+component: WHOIS
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 60s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
18
16
-template: whoisquery_days_until_expiration
17
- on: whoisquery.time_until_expiration
18
- calc: $expiry
19
- units: seconds
20
- every: 60s
21
- warn: $this < $days_until_expiration_warning*24*60*60
22
- crit: $this < $days_until_expiration_critical*24*60*60
23
- info: time until the domain name registration expires
24
- to: webmaster
19
+ template: whoisquery_days_until_expiration
20
+ on: whoisquery.time_until_expiration
21
+ class: Other
22
+component: WHOIS
23
+ type: Utilization
24
+ calc: $expiry
25
+ units: seconds
26
+ every: 60s
27
+ warn: $this < $days_until_expiration_warning*24*60*60
28
+ crit: $this < $days_until_expiration_critical*24*60*60
29
+ info: time until the domain name registration expires
30
+ to: webmaster
health/health.d/wmi.conf
+137
-110
@@ -3,128 +3,155 @@
3
4
## Availability
5
6
-template: wmi_last_collected_secs
7
- on: cpu.collector_duration
8
- calc: $now - $last_collected_t
9
- units: seconds ago
10
- every: 10s
11
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
12
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
13
- delay: down 5m multiplier 1.5 max 1h
14
- info: number of seconds since the last successful data collection
15
- to: sysadmin
6
+ template: wmi_last_collected_secs
7
+ on: cpu.collector_duration
8
+ class: Windows
9
+component: Availability
10
+ type: Latency
11
+ calc: $now - $last_collected_t
12
+ units: seconds ago
13
+ every: 10s
14
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
15
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
16
+ delay: down 5m multiplier 1.5 max 1h
17
+ info: number of seconds since the last successful data collection
18
+ to: sysadmin
19
20
## CPU
21
19
-template: wmi_10min_cpu_usage
20
- on: wmi.cpu_utilization_total
21
- os: linux
22
- hosts: *
23
- lookup: average -10m unaligned match-names of dpc,user,privileged,interrupt
24
- units: %
25
- every: 1m
26
- warn: $this > (($status >= $WARNING) ? (75) : (85))
27
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
28
- delay: down 15m multiplier 1.5 max 1h
29
- info: average CPU utilization over the last 10 minutes
30
- to: sysadmin
22
+ template: wmi_10min_cpu_usage
23
+ on: wmi.cpu_utilization_total
24
+ class: Windows
25
+component: CPU
26
+ type: Utilization
27
+ os: linux
28
+ hosts: *
29
+ lookup: average -10m unaligned match-names of dpc,user,privileged,interrupt
30
+ units: %
31
+ every: 1m
32
+ warn: $this > (($status >= $WARNING) ? (75) : (85))
33
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
34
+ delay: down 15m multiplier 1.5 max 1h
35
+ info: average CPU utilization over the last 10 minutes
36
+ to: sysadmin
37
38
39
## Memory
40
35
-template: wmi_ram_in_use
36
- on: wmi.memory_utilization
37
- os: linux
38
- hosts: *
39
- calc: ($used) * 100 / ($used + $available)
40
- units: %
41
- every: 10s
42
- warn: $this > (($status >= $WARNING) ? (80) : (90))
43
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
44
- delay: down 15m multiplier 1.5 max 1h
45
- info: memory utilization
46
- to: sysadmin
47
-
48
-template: wmi_swap_in_use
49
- on: wmi.memory_swap_utilization
50
- os: linux
51
- hosts: *
52
- calc: ($used) * 100 / ($used + $available)
53
- units: %
54
- every: 10s
55
- warn: $this > (($status >= $WARNING) ? (80) : (90))
56
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
57
- delay: down 15m multiplier 1.5 max 1h
58
- info: swap memory utilization
59
- to: sysadmin
41
+ template: wmi_ram_in_use
42
+ on: wmi.memory_utilization
43
+ class: Windows
44
+component: Memory
45
+ type: Utilization
46
+ os: linux
47
+ hosts: *
48
+ calc: ($used) * 100 / ($used + $available)
49
+ units: %
50
+ every: 10s
51
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
52
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
53
+ delay: down 15m multiplier 1.5 max 1h
54
+ info: memory utilization
55
+ to: sysadmin
56
+
57
+ template: wmi_swap_in_use
58
+ on: wmi.memory_swap_utilization
59
+ class: Windows
60
+component: Memory
61
+ type: Utilization
62
+ os: linux
63
+ hosts: *
64
+ calc: ($used) * 100 / ($used + $available)
65
+ units: %
66
+ every: 10s
67
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
68
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
69
+ delay: down 15m multiplier 1.5 max 1h
70
+ info: swap memory utilization
71
+ to: sysadmin
72
73
74
## Network
75
64
-template: wmi_inbound_packets_discarded
65
- on: wmi.net_discarded
66
- os: linux
67
- hosts: *
68
-families: *
69
- lookup: sum -10m unaligned absolute match-names of inbound
70
- units: packets
71
- every: 1m
72
- warn: $this >= 5
73
- delay: down 1h multiplier 1.5 max 2h
74
- info: number of inbound discarded packets for the network interface in the last 10 minutes
75
- to: sysadmin
76
-
77
-template: wmi_outbound_packets_discarded
78
- on: wmi.net_discarded
79
- os: linux
80
- hosts: *
81
-families: *
82
- lookup: sum -10m unaligned absolute match-names of outbound
83
- units: packets
84
- every: 1m
85
- warn: $this >= 5
86
- delay: down 1h multiplier 1.5 max 2h
87
- info: number of outbound discarded packets for the network interface in the last 10 minutes
88
- to: sysadmin
89
-
90
-template: wmi_inbound_packets_errors
91
- on: wmi.net_errors
92
- os: linux
93
- hosts: *
94
-families: *
95
- lookup: sum -10m unaligned absolute match-names of inbound
96
- units: packets
97
- every: 1m
98
- warn: $this >= 5
99
- delay: down 1h multiplier 1.5 max 2h
100
- info: number of inbound errors for the network interface in the last 10 minutes
101
- to: sysadmin
102
-
103
-template: wmi_outbound_packets_errors
104
- on: wmi.net_errors
105
- os: linux
106
- hosts: *
107
-families: *
108
- lookup: sum -10m unaligned absolute match-names of outbound
109
- units: packets
110
- every: 1m
111
- warn: $this >= 5
112
- delay: down 1h multiplier 1.5 max 2h
113
- info: number of outbound errors for the network interface in the last 10 minutes
114
- to: sysadmin
76
+ template: wmi_inbound_packets_discarded
77
+ on: wmi.net_discarded
78
+ class: Windows
79
+component: Network
80
+ type: Errors
81
+ os: linux
82
+ hosts: *
83
+ families: *
84
+ lookup: sum -10m unaligned absolute match-names of inbound
85
+ units: packets
86
+ every: 1m
87
+ warn: $this >= 5
88
+ delay: down 1h multiplier 1.5 max 2h
89
+ info: number of inbound discarded packets for the network interface in the last 10 minutes
90
+ to: sysadmin
91
+
92
+ template: wmi_outbound_packets_discarded
93
+ on: wmi.net_discarded
94
+ class: Windows
95
+component: Network
96
+ type: Errors
97
+ os: linux
98
+ hosts: *
99
+ families: *
100
+ lookup: sum -10m unaligned absolute match-names of outbound
101
+ units: packets
102
+ every: 1m
103
+ warn: $this >= 5
104
+ delay: down 1h multiplier 1.5 max 2h
105
+ info: number of outbound discarded packets for the network interface in the last 10 minutes
106
+ to: sysadmin
107
+
108
+ template: wmi_inbound_packets_errors
109
+ on: wmi.net_errors
110
+ class: Windows
111
+component: Network
112
+ type: Errors
113
+ os: linux
114
+ hosts: *
115
+ families: *
116
+ lookup: sum -10m unaligned absolute match-names of inbound
117
+ units: packets
118
+ every: 1m
119
+ warn: $this >= 5
120
+ delay: down 1h multiplier 1.5 max 2h
121
+ info: number of inbound errors for the network interface in the last 10 minutes
122
+ to: sysadmin
123
+
124
+ template: wmi_outbound_packets_errors
125
+ on: wmi.net_errors
126
+ class: Windows
127
+component: Network
128
+ type: Errors
129
+ os: linux
130
+ hosts: *
131
+ families: *
132
+ lookup: sum -10m unaligned absolute match-names of outbound
133
+ units: packets
134
+ every: 1m
135
+ warn: $this >= 5
136
+ delay: down 1h multiplier 1.5 max 2h
137
+ info: number of outbound errors for the network interface in the last 10 minutes
138
+ to: sysadmin
139
140
141
## Disk
142
119
-template: wmi_disk_in_use
120
- on: wmi.logical_disk_utilization
121
- os: linux
122
- hosts: *
123
- calc: ($used) * 100 / ($used + $free)
124
- units: %
125
- every: 10s
126
- warn: $this > (($status >= $WARNING) ? (80) : (90))
127
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
128
- delay: down 15m multiplier 1.5 max 1h
129
- info: disk space utilization
130
- to: sysadmin
143
+ template: wmi_disk_in_use
144
+ on: wmi.logical_disk_utilization
145
+ class: Windows
146
+component: Disk
147
+ type: Utilization
148
+ os: linux
149
+ hosts: *
150
+ calc: ($used) * 100 / ($used + $free)
151
+ units: %
152
+ every: 10s
153
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
154
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
155
+ delay: down 15m multiplier 1.5 max 1h
156
+ info: disk space utilization
157
+ to: sysadmin
health/health.d/x509check.conf
+35
-26
@@ -1,32 +1,41 @@
1
2
# make sure x509check is running
3
4
-template: x509check_last_collected_secs
5
- on: x509check.time_until_expiration
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 60s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: x509check_last_collected_secs
5
+ on: x509check.time_until_expiration
6
+ class: Certificates
7
+component: x509 certificates
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 60s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
18
16
-template: x509check_days_until_expiration
17
- on: x509check.time_until_expiration
18
- calc: $expiry
19
- units: seconds
20
- every: 60s
21
- warn: $this < $days_until_expiration_warning*24*60*60
22
- crit: $this < $days_until_expiration_critical*24*60*60
23
- info: time until x509 certificate expires
24
- to: webmaster
19
+ template: x509check_days_until_expiration
20
+ on: x509check.time_until_expiration
21
+ class: Certificates
22
+component: x509 certificates
23
+ type: Latency
24
+ calc: $expiry
25
+ units: seconds
26
+ every: 60s
27
+ warn: $this < $days_until_expiration_warning*24*60*60
28
+ crit: $this < $days_until_expiration_critical*24*60*60
29
+ info: time until x509 certificate expires
30
+ to: webmaster
31
26
-template: x509check_revocation_status
27
- on: x509check.revocation_status
28
- calc: $revoked
29
- every: 60s
30
- crit: $this != nan AND $this != 0
31
- info: x509 certificate revocation status (0: revoked, 1: valid)
32
- to: webmaster
32
+ template: x509check_revocation_status
33
+ on: x509check.revocation_status
34
+ class: Certificates
35
+component: x509 certificates
36
+ type: Errors
37
+ calc: $revoked
38
+ every: 60s
39
+ crit: $this != nan AND $this != 0
40
+ info: x509 certificate revocation status (0: revoked, 1: valid)
41
+ to: webmaster
health/health.d/zfs.conf
+12
-9
@@ -1,10 +1,13 @@
1
2
- alarm: zfs_memory_throttle
3
- on: zfs.memory_ops
4
- lookup: sum -10m unaligned absolute of throttled
5
- units: events
6
- every: 1m
7
- warn: $this > 0
8
- delay: down 1h multiplier 1.5 max 2h
9
- info: number of times ZFS had to limit the ARC growth in the last 10 minutes
10
- to: sysadmin
2
+ alarm: zfs_memory_throttle
3
+ on: zfs.memory_ops
4
+ class: System
5
+component: File system
6
+ type: Utilization
7
+ lookup: sum -10m unaligned absolute of throttled
8
+ units: events
9
+ every: 1m
10
+ warn: $this > 0
11
+ delay: down 1h multiplier 1.5 max 2h
12
+ info: number of times ZFS had to limit the ARC growth in the last 10 minutes
13
+ to: sysadmin
health/health.d/zookeeper.conf
+13
-10
@@ -1,14 +1,17 @@
1
2
# make sure zookeeper is running
3
4
-template: zookeeper_last_collected_secs
5
- on: zookeeper.requests
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- delay: down 5m multiplier 1.5 max 1h
12
- info: number of seconds since the last successful data collection
13
- to: webmaster
4
+ template: zookeeper_last_collected_secs
5
+ on: zookeeper.requests
6
+ class: KV Storage
7
+component: ZooKeeper
8
+ type: Latency
9
+ calc: $now - $last_collected_t
10
+ units: seconds ago
11
+ every: 10s
12
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: number of seconds since the last successful data collection
16
+ to: webmaster
17
health/health.h
+3
-14
@@ -37,20 +37,6 @@ extern unsigned int default_health_enabled;
37
#define HEALTH_LISTEN_BACKLOG 4096
38
#endif
39
40
-#define HEALTH_ON_KEY "on"
41
-#define HEALTH_EVERY_KEY "every"
42
-#define HEALTH_GREEN_KEY "green"
43
-#define HEALTH_RED_KEY "red"
44
-#define HEALTH_WARN_KEY "warn"
45
-#define HEALTH_CRIT_KEY "crit"
46
-#define HEALTH_EXEC_KEY "exec"
47
-#define HEALTH_RECIPIENT_KEY "to"
48
-#define HEALTH_UNITS_KEY "units"
49
-#define HEALTH_INFO_KEY "info"
50
-#define HEALTH_DELAY_KEY "delay"
51
-#define HEALTH_OPTIONS_KEY "options"
52
-#define HEALTH_FOREACH_KEY "foreach"
53
-
40
#define HEALTH_SILENCERS_MAX_FILE_LEN 10000
41
42
extern char *silencers_filename;
@@ -81,6 +67,9 @@ extern ALARM_ENTRY* health_create_alarm_entry(
67
const char *name,
68
const char *chart,
69
const char *family,
70
+ const char *class,
71
+ const char *component,
72
+ const char *type,
73
const char *exec,
74
const char *recipient,
75
time_t duration,
health/health_config.c
+76
@@ -23,10 +23,14 @@
23
#define HEALTH_RECIPIENT_KEY "to"
24
#define HEALTH_UNITS_KEY "units"
25
#define HEALTH_INFO_KEY "info"
26
+#define HEALTH_CLASS_KEY "class"
27
+#define HEALTH_COMPONENT_KEY "component"
28
+#define HEALTH_TYPE_KEY "type"
29
#define HEALTH_DELAY_KEY "delay"
30
#define HEALTH_OPTIONS_KEY "options"
31
#define HEALTH_REPEAT_KEY "repeat"
32
#define HEALTH_HOST_LABEL_KEY "host labels"
33
+#define HEALTH_FOREACH_KEY "foreach"
34
35
static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
36
if(!rc->chart) {
@@ -499,6 +503,9 @@ static int health_readfile(const char *filename, void *data) {
503
hash_lookup = 0,
504
hash_units = 0,
505
hash_info = 0,
506
+ hash_class = 0,
507
+ hash_component = 0,
508
+ hash_type = 0,
509
hash_recipient = 0,
510
hash_delay = 0,
511
hash_options = 0,
@@ -526,6 +533,9 @@ static int health_readfile(const char *filename, void *data) {
533
hash_every = simple_uhash(HEALTH_EVERY_KEY);
534
hash_units = simple_hash(HEALTH_UNITS_KEY);
535
hash_info = simple_hash(HEALTH_INFO_KEY);
536
+ hash_class = simple_uhash(HEALTH_CLASS_KEY);
537
+ hash_component = simple_uhash(HEALTH_COMPONENT_KEY);
538
+ hash_type = simple_uhash(HEALTH_TYPE_KEY);
539
hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
540
hash_delay = simple_uhash(HEALTH_DELAY_KEY);
541
hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
@@ -696,6 +706,39 @@ static int health_readfile(const char *filename, void *data) {
706
rc->chart = strdupz(value);
707
rc->hash_chart = simple_hash(rc->chart);
708
}
709
+ else if(hash == hash_class && !strcasecmp(key, HEALTH_CLASS_KEY)) {
710
+ if(rc->class) {
711
+ if(strcmp(rc->class, value) != 0)
712
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
713
+ line, filename, rc->name, key, rc->class, value, value);
714
+
715
+ freez(rc->class);
716
+ }
717
+ rc->class = strdupz(value);
718
+ strip_quotes(rc->class);
719
+ }
720
+ else if(hash == hash_component && !strcasecmp(key, HEALTH_COMPONENT_KEY)) {
721
+ if(rc->component) {
722
+ if(strcmp(rc->component, value) != 0)
723
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
724
+ line, filename, rc->name, key, rc->component, value, value);
725
+
726
+ freez(rc->component);
727
+ }
728
+ rc->component = strdupz(value);
729
+ strip_quotes(rc->component);
730
+ }
731
+ else if(hash == hash_type && !strcasecmp(key, HEALTH_TYPE_KEY)) {
732
+ if(rc->type) {
733
+ if(strcmp(rc->type, value) != 0)
734
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
735
+ line, filename, rc->name, key, rc->type, value, value);
736
+
737
+ freez(rc->type);
738
+ }
739
+ rc->type = strdupz(value);
740
+ strip_quotes(rc->type);
741
+ }
742
else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
743
health_parse_db_lookup(line, filename, value, &rc->group, &rc->after, &rc->before,
744
&rc->update_every, &rc->options, &rc->dimensions, &rc->foreachdim);
@@ -848,6 +891,39 @@ static int health_readfile(const char *filename, void *data) {
891
rt->context = strdupz(value);
892
rt->hash_context = simple_hash(rt->context);
893
}
894
+ else if(hash == hash_class && !strcasecmp(key, HEALTH_CLASS_KEY)) {
895
+ if(rt->class) {
896
+ if(strcmp(rt->class, value) != 0)
897
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
898
+ line, filename, rt->name, key, rt->class, value, value);
899
+
900
+ freez(rt->class);
901
+ }
902
+ rt->class = strdupz(value);
903
+ strip_quotes(rt->class);
904
+ }
905
+ else if(hash == hash_component && !strcasecmp(key, HEALTH_COMPONENT_KEY)) {
906
+ if(rt->component) {
907
+ if(strcmp(rt->component, value) != 0)
908
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
909
+ line, filename, rt->name, key, rt->component, value, value);
910
+
911
+ freez(rt->component);
912
+ }
913
+ rt->component = strdupz(value);
914
+ strip_quotes(rt->component);
915
+ }
916
+ else if(hash == hash_type && !strcasecmp(key, HEALTH_TYPE_KEY)) {
917
+ if(rt->type) {
918
+ if(strcmp(rt->type, value) != 0)
919
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
920
+ line, filename, rt->name, key, rt->type, value, value);
921
+
922
+ freez(rt->type);
923
+ }
924
+ rt->type = strdupz(value);
925
+ strip_quotes(rt->type);
926
+ }
927
else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
928
freez(rt->family_match);
929
simple_pattern_free(rt->family_pattern);
health/health_json.c
+48
-2
@@ -23,6 +23,9 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
23
"\t\t\"name\": \"%s\",\n"
24
"\t\t\"chart\": \"%s\",\n"
25
"\t\t\"family\": \"%s\",\n"
26
+ "\t\t\"class\": \"%s\",\n"
27
+ "\t\t\"component\": \"%s\",\n"
28
+ "\t\t\"type\": \"%s\",\n"
29
"\t\t\"processed\": %s,\n"
30
"\t\t\"updated\": %s,\n"
31
"\t\t\"exec_run\": %lu,\n"
@@ -52,6 +55,9 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
55
, ae->name
56
, ae->chart
57
, ae->family
58
+ , ae->class?ae->class:"Unknown"
59
+ , ae->component?ae->component:"Unknown"
60
+ , ae->type?ae->type:"Unknown"
61
, (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false"
62
, (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"
63
, (unsigned long)ae->exec_run_timestamp
@@ -76,7 +82,22 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
82
, (ae->flags & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false"
83
);
84
79
- health_string2json(wb, "\t\t", "info", ae->info?ae->info:"", ",\n");
85
+ char *replaced_info = NULL;
86
+ if (likely(ae->info)) {
87
+ char *m = NULL;
88
+ replaced_info = strdupz(ae->info);
89
+ size_t pos = 0;
90
+ while ((m = strstr(replaced_info + pos, "$family"))) {
91
+ char *buf = NULL;
92
+ pos = m - replaced_info;
93
+ buf = find_and_replace(replaced_info, "$family", ae->family ? ae->family : "", m);
94
+ freez(replaced_info);
95
+ replaced_info = strdupz(buf);
96
+ freez(buf);
97
+ }
98
+ }
99
+
100
+ health_string2json(wb, "\t\t", "info", replaced_info?replaced_info:"", ",\n");
101
102
if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
103
buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");
@@ -91,6 +112,8 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
112
buffer_strcat(wb, "\n");
113
114
buffer_strcat(wb, "\t}");
115
+
116
+ freez(replaced_info);
117
}
118
119
void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart) {
@@ -140,12 +163,30 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
163
char value_string[100 + 1];
164
format_value_and_unit(value_string, 100, rc->value, rc->units, -1);
165
166
+ char *replaced_info = NULL;
167
+ if (likely(rc->info)) {
168
+ char *m;
169
+ replaced_info = strdupz(rc->info);
170
+ size_t pos = 0;
171
+ while ((m = strstr(replaced_info + pos, "$family"))) {
172
+ char *buf = NULL;
173
+ pos = m - replaced_info;
174
+ buf = find_and_replace(replaced_info, "$family", (rc->rrdset && rc->rrdset->family) ? rc->rrdset->family : "", m);
175
+ freez(replaced_info);
176
+ replaced_info = strdupz(buf);
177
+ freez(buf);
178
+ }
179
+ }
180
+
181
buffer_sprintf(wb,
182
"\t\t\"%s.%s\": {\n"
183
"\t\t\t\"id\": %lu,\n"
184
"\t\t\t\"name\": \"%s\",\n"
185
"\t\t\t\"chart\": \"%s\",\n"
186
"\t\t\t\"family\": \"%s\",\n"
187
+ "\t\t\t\"class\": \"%s\",\n"
188
+ "\t\t\t\"component\": \"%s\",\n"
189
+ "\t\t\t\"type\": \"%s\",\n"
190
"\t\t\t\"active\": %s,\n"
191
"\t\t\t\"disabled\": %s,\n"
192
"\t\t\t\"silenced\": %s,\n"
@@ -174,6 +215,9 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
215
, rc->name
216
, rc->chart
217
, (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
218
+ , rc->class?rc->class:"Unknown"
219
+ , rc->component?rc->component:"Unknown"
220
+ , rc->type?rc->type:"Unknown"
221
, (rc->rrdset)?"true":"false"
222
, (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false"
223
, (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false"
@@ -181,7 +225,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
225
, rc->recipient?rc->recipient:host->health_default_recipient
226
, rc->source
227
, rc->units?rc->units:""
184
- , rc->info?rc->info:""
228
+ , replaced_info?replaced_info:""
229
, rrdcalc_status2string(rc->status)
230
, (unsigned long)rc->last_status_change
231
, (unsigned long)rc->last_updated
@@ -252,6 +296,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
296
buffer_strcat(wb, "\n");
297
298
buffer_strcat(wb, "\t\t}");
299
+
300
+ freez(replaced_info);
301
}
302
303
//void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
health/health_log.c
+52
-2
@@ -111,6 +111,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
111
"\t%d\t%d\t%d\t%d"
112
"\t" CALCULATED_NUMBER_FORMAT_AUTO "\t" CALCULATED_NUMBER_FORMAT_AUTO
113
"\t%016lx"
114
+ "\t%s\t%s\t%s"
115
"\n"
116
, (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
117
, host->hostname
@@ -145,6 +146,9 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
146
, ae->new_value
147
, ae->old_value
148
, (uint64_t)ae->last_repeat
149
+ , (ae->class)?ae->class:"Unknown"
150
+ , (ae->component)?ae->component:"Unknown"
151
+ , (ae->type)?ae->type:"Unknown"
152
) < 0))
153
error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", host->hostname, host->health_log_filename);
154
else {
@@ -191,7 +195,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
195
host->health_log_entries_written++;
196
line++;
197
194
- int max_entries = 30, entries = 0;
198
+ int max_entries = 33, entries = 0;
199
char *pointers[max_entries];
200
201
pointers[entries++] = s++;
@@ -364,6 +368,20 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
368
369
ae->last_repeat = last_repeat;
370
371
+ if (likely(entries > 28)) {
372
+ freez(ae->class);
373
+ ae->class = strdupz(pointers[28]);
374
+ if(!*ae->class) { freez(ae->class); ae->class = NULL; }
375
+
376
+ freez(ae->component);
377
+ ae->component = strdupz(pointers[29]);
378
+ if(!*ae->component) { freez(ae->component); ae->component = NULL; }
379
+
380
+ freez(ae->type);
381
+ ae->type = strdupz(pointers[30]);
382
+ if(!*ae->type) { freez(ae->type); ae->type = NULL; }
383
+ }
384
+
385
char value_string[100 + 1];
386
freez(ae->old_value_string);
387
freez(ae->new_value_string);
@@ -442,6 +460,9 @@ inline ALARM_ENTRY* health_create_alarm_entry(
460
const char *name,
461
const char *chart,
462
const char *family,
463
+ const char *class,
464
+ const char *component,
465
+ const char *type,
466
const char *exec,
467
const char *recipient,
468
time_t duration,
@@ -469,11 +490,19 @@ inline ALARM_ENTRY* health_create_alarm_entry(
490
if(family)
491
ae->family = strdupz(family);
492
493
+ if (class)
494
+ ae->class = strdupz(class);
495
+
496
+ if (component)
497
+ ae->component = strdupz(component);
498
+
499
+ if (type)
500
+ ae->type = strdupz(type);
501
+
502
if(exec) ae->exec = strdupz(exec);
503
if(recipient) ae->recipient = strdupz(recipient);
504
if(source) ae->source = strdupz(source);
505
if(units) ae->units = strdupz(units);
476
- if(info) ae->info = strdupz(info);
506
507
ae->unique_id = host->health_log.next_log_id++;
508
ae->alarm_id = alarm_id;
@@ -486,6 +515,24 @@ inline ALARM_ENTRY* health_create_alarm_entry(
515
ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
516
ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
517
518
+ char *replaced_info = NULL;
519
+ if (likely(info)) {
520
+ char *m;
521
+ replaced_info = strdupz(info);
522
+ size_t pos = 0;
523
+ while ((m = strstr(replaced_info + pos, "$family"))) {
524
+ char *buf = NULL;
525
+ pos = m - replaced_info;
526
+ buf = find_and_replace(replaced_info, "$family", (ae->family) ? ae->family : "", m);
527
+ freez(replaced_info);
528
+ replaced_info = strdupz(buf);
529
+ freez(buf);
530
+ }
531
+ }
532
+
533
+ if(replaced_info) ae->info = strdupz(replaced_info);
534
+ freez(replaced_info);
535
+
536
ae->old_status = old_status;
537
ae->new_status = new_status;
538
ae->duration = duration;
@@ -548,6 +595,9 @@ inline void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae) {
595
freez(ae->name);
596
freez(ae->chart);
597
freez(ae->family);
598
+ freez(ae->class);
599
+ freez(ae->component);
600
+ freez(ae->type);
601
freez(ae->exec);
602
freez(ae->recipient);
603
freez(ae->source);
libnetdata/libnetdata.c
+30
@@ -1492,3 +1492,33 @@ char *read_by_filename(char *filename, long *file_size)
1492
*file_size = size;
1493
return contents;
1494
}
1495
+
1496
+char *find_and_replace(const char *src, const char *find, const char *replace, const char *where)
1497
+{
1498
+ size_t size = strlen(src) + 1;
1499
+ size_t find_len = strlen(find);
1500
+ size_t repl_len = strlen(replace);
1501
+ char *value, *dst;
1502
+
1503
+ if (likely(where))
1504
+ size += (repl_len - find_len);
1505
+
1506
+ value = mallocz(size);
1507
+ dst = value;
1508
+
1509
+ if (likely(where)) {
1510
+ size_t count = where - src;
1511
+
1512
+ memmove(dst, src, count);
1513
+ src += count;
1514
+ dst += count;
1515
+
1516
+ memmove(dst, replace, repl_len);
1517
+ src += find_len;
1518
+ dst += repl_len;
1519
+ }
1520
+
1521
+ strcpy(dst, src);
1522
+
1523
+ return value;
1524
+}
libnetdata/libnetdata.h
+1
@@ -279,6 +279,7 @@ extern void recursive_config_double_dir_load(
279
, size_t depth
280
);
281
extern char *read_by_filename(char *filename, long *file_size);
282
+extern char *find_and_replace(const char *src, const char *find, const char *replace, const char *where);
283
284
/* fix for alpine linux */
285
#ifndef RUSAGE_THREAD