@cryptotaxi247 / netdata-1 / commits / 746ebfdbd

remove "os" "hosts" "plugin" and "module" from stock alarms (#17113)

Ilya Mashchenko committed Mar 5, 2024 at 22:37 UTC 746ebfdbd20045a6e72057736d57820caed73e5b
35 files changed +1124 -1306
src/health/health.d/apcupsd.conf
-4
@@ -5,8 +5,6 @@
5 class: Utilization
6 type: Power Supply
7 component: UPS
8 - os: *
9 - hosts: *
8 lookup: average -10m unaligned of percentage
9 units: %
10 every: 1m
@@ -23,8 +21,6 @@ component: UPS
21 class: Errors
22 type: Power Supply
23 component: UPS
26 - os: *
27 - hosts: *
24 lookup: average -60s unaligned of charge
25 units: %
26 every: 60s
src/health/health.d/boinc.conf
+1 -9
@@ -1,4 +1,4 @@
1 -# Alarms for various BOINC issues.
1 +# you can disable an alarm notification by setting the 'to' line to: silent
2
3 # Warn on any compute errors encountered.
4 template: boinc_compute_errors
@@ -6,8 +6,6 @@
6 class: Errors
7 type: Computing
8 component: BOINC
9 - os: *
10 - hosts: *
9 lookup: average -10m unaligned of comperror
10 units: tasks
11 every: 1m
@@ -23,8 +21,6 @@ component: BOINC
21 class: Errors
22 type: Computing
23 component: BOINC
26 - os: *
27 - hosts: *
24 lookup: average -10m unaligned of upload_failed
25 units: tasks
26 every: 1m
@@ -40,8 +36,6 @@ component: BOINC
36 class: Utilization
37 type: Computing
38 component: BOINC
43 - os: *
44 - hosts: *
39 lookup: average -10m unaligned of total
40 units: tasks
41 every: 1m
@@ -57,8 +51,6 @@ component: BOINC
51 class: Utilization
52 type: Computing
53 component: BOINC
60 - os: *
61 - hosts: *
54 lookup: average -10m unaligned of active
55 calc: ($boinc_total_tasks >= 1) ? ($this) : (inf)
56 units: tasks
src/health/health.d/btrfs.conf
+1 -18
@@ -1,11 +1,10 @@
1 +# you can disable an alarm notification by setting the 'to' line to: silent
2
3 template: btrfs_allocated
4 on: btrfs.disk
5 class: Utilization
6 type: System
7 component: File system
7 - os: *
8 - hosts: *
8 calc: 100 - ($unallocated * 100 / ($unallocated + $data_used + $data_free + $meta_used + $meta_free + $sys_used + $sys_free))
9 units: %
10 every: 10s
@@ -20,8 +19,6 @@ component: File system
19 class: Utilization
20 type: System
21 component: File system
23 - os: *
24 - hosts: *
22 calc: $used * 100 / ($used + $free)
23 units: %
24 every: 10s
@@ -37,8 +34,6 @@ component: File system
34 class: Utilization
35 type: System
36 component: File system
40 - os: *
41 - hosts: *
37 calc: ($used + $reserved) * 100 / ($used + $free + $reserved)
38 units: %
39 every: 10s
@@ -54,8 +49,6 @@ component: File system
49 class: Utilization
50 type: System
51 component: File system
57 - os: *
58 - hosts: *
52 calc: $used * 100 / ($used + $free)
53 units: %
54 every: 10s
@@ -71,8 +64,6 @@ component: File system
64 class: Errors
65 type: System
66 component: File system
74 - os: *
75 - hosts: *
67 units: errors
68 lookup: max -10m every 1m of read_errs
69 warn: $this > 0
@@ -86,8 +77,6 @@ component: File system
77 class: Errors
78 type: System
79 component: File system
89 - os: *
90 - hosts: *
80 units: errors
81 lookup: max -10m every 1m of write_errs
82 crit: $this > 0
@@ -101,8 +90,6 @@ component: File system
90 class: Errors
91 type: System
92 component: File system
104 - os: *
105 - hosts: *
93 units: errors
94 lookup: max -10m every 1m of flush_errs
95 crit: $this > 0
@@ -116,8 +103,6 @@ component: File system
103 class: Errors
104 type: System
105 component: File system
119 - os: *
120 - hosts: *
106 units: errors
107 lookup: max -10m every 1m of corruption_errs
108 warn: $this > 0
@@ -131,8 +116,6 @@ component: File system
116 class: Errors
117 type: System
118 component: File system
134 - os: *
135 - hosts: *
119 units: errors
120 lookup: max -10m every 1m of generation_errs
121 warn: $this > 0
src/health/health.d/cgroups.conf
+60 -65
@@ -1,72 +1,67 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
4 - template: cgroup_10min_cpu_usage
5 - on: cgroup.cpu_limit
6 - class: Utilization
7 - type: Cgroups
8 -component: CPU
9 - os: linux
10 - hosts: *
11 - lookup: average -10m unaligned
12 - units: %
13 - every: 1m
14 - warn: $this > (($status == $CRITICAL) ? (85) : (95))
15 - delay: down 15m multiplier 1.5 max 1h
16 - summary: Cgroup ${label:cgroup_name} CPU utilization
17 - info: Cgroup ${label:cgroup_name} average CPU utilization over the last 10 minutes
18 - to: silent
3 + template: cgroup_10min_cpu_usage
4 + on: cgroup.cpu_limit
5 + class: Utilization
6 + type: Cgroups
7 + component: CPU
8 +host labels: _os=linux
9 + lookup: average -10m unaligned
10 + units: %
11 + every: 1m
12 + warn: $this > (($status == $CRITICAL) ? (85) : (95))
13 + delay: down 15m multiplier 1.5 max 1h
14 + summary: Cgroup ${label:cgroup_name} CPU utilization
15 + info: Cgroup ${label:cgroup_name} average CPU utilization over the last 10 minutes
16 + to: silent
17
20 - template: cgroup_ram_in_use
21 - on: cgroup.mem_usage
22 - class: Utilization
23 - type: Cgroups
24 -component: Memory
25 - os: linux
26 - hosts: *
27 - calc: ($ram) * 100 / $memory_limit
28 - units: %
29 - every: 10s
30 - warn: $this > (($status >= $WARNING) ? (80) : (90))
31 - crit: $this > (($status == $CRITICAL) ? (90) : (98))
32 - delay: down 15m multiplier 1.5 max 1h
33 - summary: Cgroup ${label:cgroup_name} memory utilization
34 - info: Cgroup ${label:cgroup_name} memory utilization
35 - to: silent
18 + template: cgroup_ram_in_use
19 + on: cgroup.mem_usage
20 + class: Utilization
21 + type: Cgroups
22 + component: Memory
23 +host labels: _os=linux
24 + calc: ($ram) * 100 / $memory_limit
25 + units: %
26 + every: 10s
27 + warn: $this > (($status >= $WARNING) ? (80) : (90))
28 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
29 + delay: down 15m multiplier 1.5 max 1h
30 + summary: Cgroup ${label:cgroup_name} memory utilization
31 + info: Cgroup ${label:cgroup_name} memory utilization
32 + to: silent
33
34 # ---------------------------------K8s containers--------------------------------------------
35
39 - template: k8s_cgroup_10min_cpu_usage
40 - on: k8s.cgroup.cpu_limit
41 - class: Utilization
42 - type: Cgroups
43 -component: CPU
44 - os: linux
45 - hosts: *
46 - lookup: average -10m unaligned
47 - units: %
48 - every: 1m
49 - warn: $this > (($status >= $WARNING) ? (75) : (85))
50 - delay: down 15m multiplier 1.5 max 1h
51 - summary: Container ${label:k8s_container_name} pod ${label:k8s_pod_name} CPU utilization
52 - info: Container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
53 - average CPU utilization over the last 10 minutes
54 - to: silent
36 + template: k8s_cgroup_10min_cpu_usage
37 + on: k8s.cgroup.cpu_limit
38 + class: Utilization
39 + type: Cgroups
40 + component: CPU
41 +host labels: _os=linux
42 + lookup: average -10m unaligned
43 + units: %
44 + every: 1m
45 + warn: $this > (($status >= $WARNING) ? (75) : (85))
46 + delay: down 15m multiplier 1.5 max 1h
47 + summary: Container ${label:k8s_container_name} pod ${label:k8s_pod_name} CPU utilization
48 + info: Container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
49 + average CPU utilization over the last 10 minutes
50 + to: silent
51
56 - template: k8s_cgroup_ram_in_use
57 - on: k8s.cgroup.mem_usage
58 - class: Utilization
59 - type: Cgroups
60 -component: Memory
61 - os: linux
62 - hosts: *
63 - calc: ($ram) * 100 / $memory_limit
64 - units: %
65 - every: 10s
66 - warn: $this > (($status >= $WARNING) ? (80) : (90))
67 - crit: $this > (($status == $CRITICAL) ? (90) : (98))
68 - delay: down 15m multiplier 1.5 max 1h
69 - summary: Container ${label:k8s_container_name} pod ${label:k8s_pod_name} memory utilization
70 - info: container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
71 - memory utilization
72 - to: silent
52 + template: k8s_cgroup_ram_in_use
53 + on: k8s.cgroup.mem_usage
54 + class: Utilization
55 + type: Cgroups
56 + component: Memory
57 +host labels: _os=linux
58 + calc: ($ram) * 100 / $memory_limit
59 + units: %
60 + every: 10s
61 + warn: $this > (($status >= $WARNING) ? (80) : (90))
62 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
63 + delay: down 15m multiplier 1.5 max 1h
64 + summary: Container ${label:k8s_container_name} pod ${label:k8s_pod_name} memory utilization
65 + info: container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
66 + memory utilization
67 + to: silent
src/health/health.d/cpu.conf
+58 -62
@@ -1,69 +1,65 @@
1
2 # you can disable an alarm notification by setting the 'to' line to: silent
3
4 - template: 10min_cpu_usage
5 - on: system.cpu
6 - class: Utilization
7 - type: System
8 -component: CPU
9 - os: linux
10 - hosts: *
11 - lookup: average -10m unaligned of user,system,softirq,irq,guest
12 - units: %
13 - every: 1m
14 - warn: $this > (($status >= $WARNING) ? (75) : (85))
15 - crit: $this > (($status == $CRITICAL) ? (85) : (95))
16 - delay: down 15m multiplier 1.5 max 1h
17 - summary: System CPU utilization
18 - info: Average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
19 - to: silent
4 + template: 10min_cpu_usage
5 + on: system.cpu
6 + class: Utilization
7 + type: System
8 + component: CPU
9 +host labels: _os=linux
10 + lookup: average -10m unaligned of user,system,softirq,irq,guest
11 + units: %
12 + every: 1m
13 + warn: $this > (($status >= $WARNING) ? (75) : (85))
14 + crit: $this > (($status == $CRITICAL) ? (85) : (95))
15 + delay: down 15m multiplier 1.5 max 1h
16 + summary: System CPU utilization
17 + info: Average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
18 + to: silent
19
21 - template: 10min_cpu_iowait
22 - on: system.cpu
23 - class: Utilization
24 - type: System
25 -component: CPU
26 - os: linux
27 - hosts: *
28 - lookup: average -10m unaligned of iowait
29 - units: %
30 - every: 1m
31 - warn: $this > (($status >= $WARNING) ? (20) : (40))
32 - delay: up 30m down 30m multiplier 1.5 max 2h
33 - summary: System CPU iowait time
34 - info: Average CPU iowait time over the last 10 minutes
35 - to: silent
20 + template: 10min_cpu_iowait
21 + on: system.cpu
22 + class: Utilization
23 + type: System
24 + component: CPU
25 +host labels: _os=linux
26 + lookup: average -10m unaligned of iowait
27 + units: %
28 + every: 1m
29 + warn: $this > (($status >= $WARNING) ? (20) : (40))
30 + delay: up 30m down 30m multiplier 1.5 max 2h
31 + summary: System CPU iowait time
32 + info: Average CPU iowait time over the last 10 minutes
33 + to: silent
34
37 - template: 20min_steal_cpu
38 - on: system.cpu
39 - class: Latency
40 - type: System
41 -component: CPU
42 - os: linux
43 - hosts: *
44 - lookup: average -20m unaligned of steal
45 - units: %
46 - every: 5m
47 - warn: $this > (($status >= $WARNING) ? (5) : (10))
48 - delay: down 1h multiplier 1.5 max 2h
49 - summary: System CPU steal time
50 - info: Average CPU steal time over the last 20 minutes
51 - to: silent
35 + template: 20min_steal_cpu
36 + on: system.cpu
37 + class: Latency
38 + type: System
39 + component: CPU
40 +host labels: _os=linux
41 + lookup: average -20m unaligned of steal
42 + units: %
43 + every: 5m
44 + warn: $this > (($status >= $WARNING) ? (5) : (10))
45 + delay: down 1h multiplier 1.5 max 2h
46 + summary: System CPU steal time
47 + info: Average CPU steal time over the last 20 minutes
48 + to: silent
49
50 ## FreeBSD
54 - template: 10min_cpu_usage
55 - on: system.cpu
56 - class: Utilization
57 - type: System
58 -component: CPU
59 - os: freebsd
60 - hosts: *
61 - lookup: average -10m unaligned of user,system,interrupt
62 - units: %
63 - every: 1m
64 - warn: $this > (($status >= $WARNING) ? (75) : (85))
65 - crit: $this > (($status == $CRITICAL) ? (85) : (95))
66 - delay: down 15m multiplier 1.5 max 1h
67 - summary: System CPU utilization
68 - info: Average CPU utilization over the last 10 minutes (excluding nice)
69 - to: silent
51 + template: 10min_cpu_usage
52 + on: system.cpu
53 + class: Utilization
54 + type: System
55 + component: CPU
56 +host labels: _os=freebsd
57 + lookup: average -10m unaligned of user,system,interrupt
58 + units: %
59 + every: 1m
60 + warn: $this > (($status >= $WARNING) ? (75) : (85))
61 + crit: $this > (($status == $CRITICAL) ? (85) : (95))
62 + delay: down 15m multiplier 1.5 max 1h
63 + summary: System CPU utilization
64 + info: Average CPU utilization over the last 10 minutes (excluding nice)
65 + to: silent
src/health/health.d/dbengine.conf
-9
@@ -1,4 +1,3 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 alarm: 10min_dbengine_global_fs_errors
@@ -6,8 +5,6 @@
5 class: Errors
6 type: Netdata
7 component: DB engine
9 - os: linux freebsd macos
10 - hosts: *
8 lookup: sum -10m unaligned of fs_errors
9 units: errors
10 every: 10s
@@ -22,8 +19,6 @@ component: DB engine
19 class: Errors
20 type: Netdata
21 component: DB engine
25 - os: linux freebsd macos
26 - hosts: *
22 lookup: sum -10m unaligned of io_errors
23 units: errors
24 every: 10s
@@ -38,8 +33,6 @@ component: DB engine
33 class: Errors
34 type: Netdata
35 component: DB engine
41 - os: linux freebsd macos
42 - hosts: *
36 lookup: sum -10m unaligned of pg_cache_over_half_dirty_events
37 units: errors
38 every: 10s
@@ -55,8 +48,6 @@ component: DB engine
48 class: Errors
49 type: Netdata
50 component: DB engine
58 - os: linux freebsd macos
59 - hosts: *
51 lookup: sum -10m unaligned of flushing_pressure_deletions
52 units: pages
53 every: 10s
src/health/health.d/disks.conf
+101 -112
@@ -1,7 +1,5 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
4 -
3 # -----------------------------------------------------------------------------
4 # low disk space
5
@@ -9,41 +7,39 @@
7 # raise an alarm if the disk is low on
8 # available disk space
9
12 - template: disk_space_usage
13 - on: disk.space
14 - class: Utilization
15 - type: System
16 -component: Disk
17 - os: linux freebsd
18 - hosts: *
10 + template: disk_space_usage
11 + on: disk.space
12 + class: Utilization
13 + type: System
14 + component: Disk
15 + host labels: _os=linux freebsd
16 chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
20 - calc: $used * 100 / ($avail + $used)
21 - units: %
22 - every: 1m
23 - warn: $this > (($status >= $WARNING ) ? (80) : (90))
24 - crit: ($this > (($status == $CRITICAL) ? (90) : (98))) && $avail < 5
25 - delay: up 1m down 15m multiplier 1.5 max 1h
26 - summary: Disk ${label:mount_point} space usage
27 - info: Total space utilization of disk ${label:mount_point}
28 - to: sysadmin
29 -
30 - template: disk_inode_usage
31 - on: disk.inodes
32 - class: Utilization
33 - type: System
34 -component: Disk
35 - os: linux freebsd
36 - hosts: *
17 + calc: $used * 100 / ($avail + $used)
18 + units: %
19 + every: 1m
20 + warn: $this > (($status >= $WARNING ) ? (80) : (90))
21 + crit: ($this > (($status == $CRITICAL) ? (90) : (98))) && $avail < 5
22 + delay: up 1m down 15m multiplier 1.5 max 1h
23 + summary: Disk ${label:mount_point} space usage
24 + info: Total space utilization of disk ${label:mount_point}
25 + to: sysadmin
26 +
27 + template: disk_inode_usage
28 + on: disk.inodes
29 + class: Utilization
30 + type: System
31 + component: Disk
32 + host labels: _os=linux freebsd
33 chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
38 - calc: $used * 100 / ($avail + $used)
39 - units: %
40 - every: 1m
41 - warn: $this > (($status >= $WARNING) ? (80) : (90))
42 - crit: $this > (($status == $CRITICAL) ? (90) : (98))
43 - delay: up 1m down 15m multiplier 1.5 max 1h
44 - summary: Disk ${label:mount_point} inode usage
45 - info: Total inode utilization of disk ${label:mount_point}
46 - to: sysadmin
34 + calc: $used * 100 / ($avail + $used)
35 + units: %
36 + every: 1m
37 + warn: $this > (($status >= $WARNING) ? (80) : (90))
38 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
39 + delay: up 1m down 15m multiplier 1.5 max 1h
40 + summary: Disk ${label:mount_point} inode usage
41 + info: Total inode utilization of disk ${label:mount_point}
42 + to: sysadmin
43
44
45 # -----------------------------------------------------------------------------
@@ -57,33 +53,30 @@ chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
53 # we will use it in the next template to find
54 # the hours remaining
55
60 -template: disk_fill_rate
61 - on: disk.space
62 - os: linux freebsd
63 - hosts: *
64 - lookup: min -10m at -50m unaligned of avail
65 - calc: ($this - $avail) / (($now - $after) / 3600)
66 - every: 1m
67 - units: GB/hour
68 - info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
56 + template: disk_fill_rate
57 + on: disk.space
58 +host labels: _os=linux freebsd
59 + lookup: min -10m at -50m unaligned of avail
60 + calc: ($this - $avail) / (($now - $after) / 3600)
61 + every: 1m
62 + units: GB/hour
63 + info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
64
65 # calculate the hours remaining
71 -# if the disk continues to fill
72 -# in this rate
73 -
74 -template: out_of_disk_space_time
75 - on: disk.space
76 - os: linux freebsd
77 - hosts: *
78 - calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
79 - units: hours
80 - every: 10s
81 - warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
82 - crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
83 - delay: down 15m multiplier 1.2 max 1h
84 - summary: Disk ${label:mount_point} estimation of lack of space
85 - info: Estimated time the disk ${label:mount_point} will run out of space, if the system continues to add data with the rate of the last hour
86 - to: silent
66 +# if the disk continues to fill in this rate
67 +
68 + template: out_of_disk_space_time
69 + on: disk.space
70 +host labels: _os=linux freebsd
71 + calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
72 + units: hours
73 + every: 10s
74 + warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
75 + crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
76 + delay: down 15m multiplier 1.2 max 1h
77 + summary: Disk ${label:mount_point} estimation of lack of space
78 + info: Estimated time the disk ${label:mount_point} will run out of space, if the system continues to add data with the rate of the last hour
79 + to: silent
80
81
82 # -----------------------------------------------------------------------------
@@ -97,33 +90,31 @@ template: out_of_disk_space_time
90 # we will use it in the next template to find
91 # the hours remaining
92
100 -template: disk_inode_rate
101 - on: disk.inodes
102 - os: linux freebsd
103 - hosts: *
104 - lookup: min -10m at -50m unaligned of avail
105 - calc: ($this - $avail) / (($now - $after) / 3600)
106 - every: 1m
107 - units: inodes/hour
108 - info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour
93 + template: disk_inode_rate
94 + on: disk.inodes
95 +host labels: _os=linux freebsd
96 + lookup: min -10m at -50m unaligned of avail
97 + calc: ($this - $avail) / (($now - $after) / 3600)
98 + every: 1m
99 + units: inodes/hour
100 + info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour
101
102 # calculate the hours remaining
103 # if the disk inodes are allocated
104 # in this rate
105
114 -template: out_of_disk_inodes_time
115 - on: disk.inodes
116 - os: linux freebsd
117 - hosts: *
118 - calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf)
119 - units: hours
120 - every: 10s
121 - warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
122 - crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
123 - delay: down 15m multiplier 1.2 max 1h
124 - summary: Disk ${label:mount_point} estimation of lack of inodes
125 - info: Estimated time the disk ${label:mount_point} will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
126 - to: silent
106 + template: out_of_disk_inodes_time
107 + on: disk.inodes
108 +host labels: _os=linux freebsd
109 + calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf)
110 + units: hours
111 + every: 10s
112 + warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
113 + crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
114 + delay: down 15m multiplier 1.2 max 1h
115 + summary: Disk ${label:mount_point} estimation of lack of inodes
116 + info: Estimated time the disk ${label:mount_point} will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
117 + to: silent
118
119
120 # -----------------------------------------------------------------------------
@@ -133,21 +124,20 @@ template: out_of_disk_inodes_time
124 # by calculating the average disk utilization
125 # for the last 10 minutes
126
136 - template: 10min_disk_utilization
137 - on: disk.util
138 - class: Utilization
139 - type: System
140 -component: Disk
141 - os: linux freebsd
142 - hosts: *
143 - lookup: average -10m unaligned
144 - units: %
145 - every: 1m
146 - warn: $this > 98 * (($status >= $WARNING) ? (0.7) : (1))
147 - delay: down 15m multiplier 1.2 max 1h
148 - summary: Disk ${label:device} utilization
149 - info: Average percentage of time ${label:device} disk was busy over the last 10 minutes
150 - to: silent
127 + template: 10min_disk_utilization
128 + on: disk.util
129 + class: Utilization
130 + type: System
131 + component: Disk
132 +host labels: _os=linux freebsd
133 + lookup: average -10m unaligned
134 + units: %
135 + every: 1m
136 + warn: $this > 98 * (($status >= $WARNING) ? (0.7) : (1))
137 + delay: down 15m multiplier 1.2 max 1h
138 + summary: Disk ${label:device} utilization
139 + info: Average percentage of time ${label:device} disk was busy over the last 10 minutes
140 + to: silent
141
142
143 # raise an alarm if the disk backlog
@@ -155,18 +145,17 @@ component: Disk
145 # for 10 minutes
146 # (i.e. the disk cannot catch up)
147
158 - template: 10min_disk_backlog
159 - on: disk.backlog
160 - class: Latency
161 - type: System
162 -component: Disk
163 - os: linux
164 - hosts: *
165 - lookup: average -10m unaligned
166 - units: ms
167 - every: 1m
168 - warn: $this > 5000 * (($status >= $WARNING) ? (0.7) : (1))
169 - delay: down 15m multiplier 1.2 max 1h
170 - summary: Disk ${label:device} backlog
171 - info: Average backlog size of the ${label:device} disk over the last 10 minutes
172 - to: silent
148 + template: 10min_disk_backlog
149 + on: disk.backlog
150 + class: Latency
151 + type: System
152 + component: Disk
153 +host labels: _os=linux freebsd
154 + lookup: average -10m unaligned
155 + units: ms
156 + every: 1m
157 + warn: $this > 5000 * (($status >= $WARNING) ? (0.7) : (1))
158 + delay: down 15m multiplier 1.2 max 1h
159 + summary: Disk ${label:device} backlog
160 + info: Average backlog size of the ${label:device} disk over the last 10 minutes
161 + to: silent
src/health/health.d/entropy.conf
+14 -15
@@ -3,18 +3,17 @@
3 # the alarm is checked every 1 minute
4 # and examines the last hour of data
5
6 - alarm: lowest_entropy
7 - on: system.entropy
8 - class: Utilization
9 - type: System
10 -component: Cryptography
11 - os: linux
12 - hosts: *
13 - lookup: min -5m unaligned
14 - units: entries
15 - every: 5m
16 - warn: $this < (($status >= $WARNING) ? (200) : (100))
17 - delay: down 1h multiplier 1.5 max 2h
18 - summary: System entropy pool number of entries
19 - info: Minimum number of entries in the random numbers pool in the last 5 minutes
20 - to: silent
6 + alarm: lowest_entropy
7 + on: system.entropy
8 + class: Utilization
9 + type: System
10 + component: Cryptography
11 +host labels: _os=linux
12 + lookup: min -5m unaligned
13 + units: entries
14 + every: 5m
15 + warn: $this < (($status >= $WARNING) ? (200) : (100))
16 + delay: down 1h multiplier 1.5 max 2h
17 + summary: System entropy pool number of entries
18 + info: Minimum number of entries in the random numbers pool in the last 5 minutes
19 + to: silent
src/health/health.d/file_descriptors.conf
+14 -17
@@ -5,7 +5,6 @@
5 class: Utilization
6 type: System
7 component: Processes
8 - hosts: *
8 lookup: max -1m unaligned
9 units: %
10 every: 1m
@@ -15,19 +14,17 @@
14 info: System-wide utilization of open files
15 to: sysadmin
16
18 - template: apps_group_file_descriptors_utilization
19 - on: app.fds_open_limit
20 - class: Utilization
21 - type: System
22 -component: Process
23 - os: linux
24 - module: *
25 - hosts: *
26 - lookup: max -10s unaligned
27 - units: %
28 - every: 10s
29 - warn: $this > (($status >= $WARNING) ? (85) : (95))
30 - delay: down 15m multiplier 1.5 max 1h
31 - summary: App group ${label:app_group} file descriptors utilization
32 - info: Open files percentage against the processes limits, among all PIDs in application group
33 - to: sysadmin
17 + template: apps_group_file_descriptors_utilization
18 + on: app.fds_open_limit
19 + class: Utilization
20 + type: System
21 + component: Process
22 +host labels: _os=linux
23 + lookup: max -10s unaligned
24 + units: %
25 + every: 10s
26 + warn: $this > (($status >= $WARNING) ? (85) : (95))
27 + delay: down 15m multiplier 1.5 max 1h
28 + summary: App group ${label:app_group} file descriptors utilization
29 + info: Open files percentage against the processes limits, among all PIDs in application group
30 + to: sysadmin
src/health/health.d/go.d.plugin.conf
+15 -16
@@ -1,18 +1,17 @@
1 -
1 # make sure go.d.plugin data collection job is running
2
4 - template: go.d_job_last_collected_secs
5 - on: netdata.go_plugin_execution_time
6 - class: Errors
7 - type: Netdata
8 -component: go.d.plugin
9 - module: !* *
10 - calc: $now - $last_collected_t
11 - units: seconds ago
12 - every: 10s
13 - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
14 - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
15 - delay: down 5m multiplier 1.5 max 1h
16 - summary: Go.d plugin last collection
17 - info: Number of seconds since the last successful data collection
18 - to: webmaster
3 + template: go.d_job_last_collected_secs
4 + on: netdata.go_plugin_execution_time
5 + class: Errors
6 + type: Netdata
7 + component: go.d.plugin
8 +host labels: _hostname=!*
9 + calc: $now - $last_collected_t
10 + units: seconds ago
11 + every: 10s
12 + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13 + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14 + delay: down 5m multiplier 1.5 max 1h
15 + summary: Go.d plugin last collection
16 + info: Number of seconds since the last successful data collection
17 + to: webmaster
src/health/health.d/ipc.conf
+28 -30
@@ -1,34 +1,32 @@
1
2 # you can disable an alarm notification by setting the 'to' line to: silent
3
4 - alarm: semaphores_used
5 - on: system.ipc_semaphores
6 - class: Utilization
7 - type: System
8 -component: IPC
9 - os: linux
10 - hosts: *
11 - calc: $semaphores * 100 / $ipc_semaphores_max
12 - units: %
13 - every: 10s
14 - warn: $this > (($status >= $WARNING) ? (70) : (80))
15 - delay: down 5m multiplier 1.5 max 1h
16 - summary: IPC semaphores used
17 - info: IPC semaphore utilization
18 - to: sysadmin
4 + alarm: semaphores_used
5 + on: system.ipc_semaphores
6 + class: Utilization
7 + type: System
8 + component: IPC
9 +host labels: _os=linux
10 + calc: $semaphores * 100 / $ipc_semaphores_max
11 + units: %
12 + every: 10s
13 + warn: $this > (($status >= $WARNING) ? (70) : (80))
14 + delay: down 5m multiplier 1.5 max 1h
15 + summary: IPC semaphores used
16 + info: IPC semaphore utilization
17 + to: sysadmin
18
20 - alarm: semaphore_arrays_used
21 - on: system.ipc_semaphore_arrays
22 - class: Utilization
23 - type: System
24 -component: IPC
25 - os: linux
26 - hosts: *
27 - calc: $arrays * 100 / $ipc_semaphores_arrays_max
28 - units: %
29 - every: 10s
30 - warn: $this > (($status >= $WARNING) ? (70) : (80))
31 - delay: down 5m multiplier 1.5 max 1h
32 - summary: IPC semaphore arrays used
33 - info: IPC semaphore arrays utilization
34 - to: sysadmin
19 + alarm: semaphore_arrays_used
20 + on: system.ipc_semaphore_arrays
21 + class: Utilization
22 + type: System
23 + component: IPC
24 +host labels: _os=linux
25 + calc: $arrays * 100 / $ipc_semaphores_arrays_max
26 + units: %
27 + every: 10s
28 + warn: $this > (($status >= $WARNING) ? (70) : (80))
29 + delay: down 5m multiplier 1.5 max 1h
30 + summary: IPC semaphore arrays used
31 + info: IPC semaphore arrays utilization
32 + to: sysadmin
src/health/health.d/load.conf
+55 -60
@@ -1,72 +1,67 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 # Calculate the base trigger point for the load average alarms.
4 # This is the maximum number of CPU's in the system over the past 1
5 # minute, with a special case for a single CPU of setting the trigger at 2.
7 - alarm: load_cpu_number
8 - on: system.load
9 - class: Utilization
10 - type: System
11 -component: Load
12 - os: linux
13 - hosts: *
14 - calc: ($active_processors == nan or $active_processors == 0) ? (nan) : ( ($active_processors < 2) ? ( 2 ) : ( $active_processors ) )
15 - units: cpus
16 - every: 1m
17 - info: Number of active CPU cores in the system
6 + alarm: load_cpu_number
7 + on: system.load
8 + class: Utilization
9 + type: System
10 + component: Load
11 +host labels: _os=linux
12 + calc: ($active_processors == nan or $active_processors == 0) ? (nan) : ( ($active_processors < 2) ? ( 2 ) : ( $active_processors ) )
13 + units: cpus
14 + every: 1m
15 + info: Number of active CPU cores in the system
16
17 # Send alarms if the load average is unusually high.
18 # These intentionally _do not_ calculate the average over the sampled
19 # time period because the values being checked already are averages.
20
23 - alarm: load_average_15
24 - on: system.load
25 - class: Utilization
26 - type: System
27 -component: Load
28 - os: linux
29 - hosts: *
30 - lookup: max -1m unaligned of load15
31 - calc: ($load_cpu_number == nan) ? (nan) : ($this)
32 - units: load
33 - every: 1m
34 - warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
35 - delay: down 15m multiplier 1.5 max 1h
36 - summary: Host load average (15 minutes)
37 - info: System load average for the past 15 minutes
38 - to: silent
21 + alarm: load_average_15
22 + on: system.load
23 + class: Utilization
24 + type: System
25 + component: Load
26 +host labels: _os=linux
27 + lookup: max -1m unaligned of load15
28 + calc: ($load_cpu_number == nan) ? (nan) : ($this)
29 + units: load
30 + every: 1m
31 + warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
32 + delay: down 15m multiplier 1.5 max 1h
33 + summary: Host load average (15 minutes)
34 + info: System load average for the past 15 minutes
35 + to: silent
36
40 - alarm: load_average_5
41 - on: system.load
42 - class: Utilization
43 - type: System
44 -component: Load
45 - os: linux
46 - hosts: *
47 - lookup: max -1m unaligned of load5
48 - calc: ($load_cpu_number == nan) ? (nan) : ($this)
49 - units: load
50 - every: 1m
51 - warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
52 - delay: down 15m multiplier 1.5 max 1h
53 - summary: System load average (5 minutes)
54 - info: System load average for the past 5 minutes
55 - to: silent
37 + alarm: load_average_5
38 + on: system.load
39 + class: Utilization
40 + type: System
41 + component: Load
42 +host labels: _os=linux
43 + lookup: max -1m unaligned of load5
44 + calc: ($load_cpu_number == nan) ? (nan) : ($this)
45 + units: load
46 + every: 1m
47 + warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
48 + delay: down 15m multiplier 1.5 max 1h
49 + summary: System load average (5 minutes)
50 + info: System load average for the past 5 minutes
51 + to: silent
52
57 - alarm: load_average_1
58 - on: system.load
59 - class: Utilization
60 - type: System
61 -component: Load
62 - os: linux
63 - hosts: *
64 - lookup: max -1m unaligned of load1
65 - calc: ($load_cpu_number == nan) ? (nan) : ($this)
66 - units: load
67 - every: 1m
68 - warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
69 - delay: down 15m multiplier 1.5 max 1h
70 - summary: System load average (1 minute)
71 - info: System load average for the past 1 minute
72 - to: silent
53 + alarm: load_average_1
54 + on: system.load
55 + class: Utilization
56 + type: System
57 + component: Load
58 +host labels: _os=linux
59 + lookup: max -1m unaligned of load1
60 + calc: ($load_cpu_number == nan) ? (nan) : ($this)
61 + units: load
62 + every: 1m
63 + warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
64 + delay: down 15m multiplier 1.5 max 1h
65 + summary: System load average (1 minute)
66 + info: System load average for the past 1 minute
67 + to: silent
src/health/health.d/memory.conf
+66 -71
@@ -1,81 +1,76 @@
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 - alarm: 1hour_memory_hw_corrupted
4 - on: mem.hwcorrupt
5 - class: Errors
6 - type: System
7 -component: Memory
8 - os: linux
9 - hosts: *
10 - calc: $HardwareCorrupted
11 - units: MB
12 - every: 10s
13 - warn: $this > 0
14 - delay: down 1h multiplier 1.5 max 1h
15 - summary: System corrupted memory
16 - info: Amount of memory corrupted due to a hardware failure
17 - to: sysadmin
3 + alarm: 1hour_memory_hw_corrupted
4 + on: mem.hwcorrupt
5 + class: Errors
6 + type: System
7 + component: Memory
8 +host labels: _os=linux
9 + calc: $HardwareCorrupted
10 + units: MB
11 + every: 10s
12 + warn: $this > 0
13 + delay: down 1h multiplier 1.5 max 1h
14 + summary: System corrupted memory
15 + info: Amount of memory corrupted due to a hardware failure
16 + to: sysadmin
17
18 ## ECC Controller
19
21 - template: ecc_memory_mc_correctable
22 - on: mem.edac_mc_errors
23 - class: Errors
24 - type: System
25 -component: Memory
26 - os: linux
27 - hosts: *
28 - calc: $correctable + $correctable_noinfo
29 - units: errors
30 - every: 1m
31 - warn: $this > 0
32 - summary: System ECC memory ${label:controller} correctable errors
33 - info: Memory controller ${label:controller} ECC correctable errors
34 - to: sysadmin
20 + template: ecc_memory_mc_correctable
21 + on: mem.edac_mc_errors
22 + class: Errors
23 + type: System
24 + component: Memory
25 +host labels: _os=linux
26 + calc: $correctable + $correctable_noinfo
27 + units: errors
28 + every: 1m
29 + warn: $this > 0
30 + summary: System ECC memory ${label:controller} correctable errors
31 + info: Memory controller ${label:controller} ECC correctable errors
32 + to: sysadmin
33
36 - template: ecc_memory_mc_uncorrectable
37 - on: mem.edac_mc_errors
38 - class: Errors
39 - type: System
40 -component: Memory
41 - os: linux
42 - hosts: *
43 - calc: $uncorrectable + $uncorrectable_noinfo
44 - units: errors
45 - every: 1m
46 - crit: $this > 0
47 - summary: System ECC memory ${label:controller} uncorrectable errors
48 - info: Memory controller ${label:controller} ECC uncorrectable errors
49 - to: sysadmin
34 + template: ecc_memory_mc_uncorrectable
35 + on: mem.edac_mc_errors
36 + class: Errors
37 + type: System
38 + component: Memory
39 +host labels: _os=linux
40 + calc: $uncorrectable + $uncorrectable_noinfo
41 + units: errors
42 + every: 1m
43 + crit: $this > 0
44 + summary: System ECC memory ${label:controller} uncorrectable errors
45 + info: Memory controller ${label:controller} ECC uncorrectable errors
46 + to: sysadmin
47
48 ## ECC DIMM
49
53 - template: ecc_memory_dimm_correctable
54 - on: mem.edac_mc_dimm_errors
55 - class: Errors
56 - type: System
57 -component: Memory
58 - os: linux
59 - hosts: *
60 - calc: $correctable
61 - units: errors
62 - every: 1m
63 - warn: $this > 0
64 - summary: System ECC memory DIMM ${label:dimm} correctable errors
65 - info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC correctable errors
66 - to: sysadmin
50 + template: ecc_memory_dimm_correctable
51 + on: mem.edac_mc_dimm_errors
52 + class: Errors
53 + type: System
54 + component: Memory
55 +host labels: _os=linux
56 + calc: $correctable
57 + units: errors
58 + every: 1m
59 + warn: $this > 0
60 + summary: System ECC memory DIMM ${label:dimm} correctable errors
61 + info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC correctable errors
62 + to: sysadmin
63
68 - template: ecc_memory_dimm_uncorrectable
69 - on: mem.edac_mc_dimm_errors
70 - class: Errors
71 - type: System
72 -component: Memory
73 - os: linux
74 - hosts: *
75 - calc: $uncorrectable
76 - units: errors
77 - every: 1m
78 - crit: $this > 0
79 - summary: System ECC memory DIMM ${label:dimm} uncorrectable errors
80 - info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC uncorrectable errors
81 - to: sysadmin
64 + template: ecc_memory_dimm_uncorrectable
65 + on: mem.edac_mc_dimm_errors
66 + class: Errors
67 + type: System
68 + component: Memory
69 +host labels: _os=linux
70 + calc: $uncorrectable
71 + units: errors
72 + every: 1m
73 + crit: $this > 0
74 + summary: System ECC memory DIMM ${label:dimm} uncorrectable errors
75 + info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC uncorrectable errors
76 + to: sysadmin
src/health/health.d/ml.conf
-7
@@ -13,8 +13,6 @@
13 class: Workload
14 type: System
15 component: ML
16 - os: *
17 - hosts: *
16 lookup: average -1m of anomaly_rate
17 calc: $this
18 units: %
@@ -29,8 +27,6 @@ component: ML
27 # if anomaly rate is above 20% then critical (pick your own threshold that works best via tial and error).
28 # template: ml_5min_cpu_dims
29 # on: system.cpu
32 -# os: linux
33 -# hosts: *
30 # lookup: average -5m anomaly-bit foreach *
31 # calc: $this
32 # units: %
@@ -44,8 +40,6 @@ component: ML
40 # if anomaly rate is above 20% then critical (pick your own threshold that works best via tial and error).
41 # template: ml_5min_cpu_chart
42 # on: system.cpu
47 -# os: linux
48 -# hosts: *
43 # lookup: average -5m anomaly-bit of *
44 # calc: $this
45 # units: %
@@ -53,4 +47,3 @@ component: ML
47 # warn: $this > (($status >= $WARNING) ? (5) : (20))
48 # crit: $this > (($status == $CRITICAL) ? (20) : (100))
49 # info: rolling 5min anomaly rate for system.cpu chart
56 -
src/health/health.d/net.conf
+163 -182
@@ -9,46 +9,42 @@
9 class: Latency
10 type: System
11 component: Network
12 - os: *
13 - hosts: *
12 calc: ( $nic_speed_max > 0 ) ? ( $nic_speed_max / 1000) : ( nan )
13 units: Mbit
14 every: 10s
15 info: Network interface ${label:device} current speed
16
19 - template: 1m_received_traffic_overflow
20 - on: net.net
21 - class: Workload
22 - type: System
23 -component: Network
24 - os: linux
25 - hosts: *
26 - lookup: average -1m unaligned absolute of received
27 - calc: ($interface_speed > 0) ? ($this * 100 / ($interface_speed * 1000)) : ( nan )
28 - units: %
29 - every: 10s
30 - warn: $this > (($status >= $WARNING) ? (85) : (90))
31 - delay: up 1m down 1m multiplier 1.5 max 1h
32 - summary: System network interface ${label:device} inbound utilization
33 - info: Average inbound utilization for the network interface ${label:device} over the last minute
34 - to: silent
35 -
36 - template: 1m_sent_traffic_overflow
37 - on: net.net
38 - class: Workload
39 - type: System
40 -component: Network
41 - os: linux
42 - hosts: *
43 - lookup: average -1m unaligned absolute of sent
44 - calc: ($interface_speed > 0) ? ($this * 100 / ($interface_speed * 1000)) : ( nan )
45 - units: %
46 - every: 10s
47 - warn: $this > (($status >= $WARNING) ? (85) : (90))
48 - delay: up 1m down 1m multiplier 1.5 max 1h
49 - summary: System network interface ${label:device} outbound utilization
50 - info: Average outbound utilization for the network interface ${label:device} over the last minute
51 - to: silent
17 + template: 1m_received_traffic_overflow
18 + on: net.net
19 + class: Workload
20 + type: System
21 + component: Network
22 +host labels: _os=linux
23 + lookup: average -1m unaligned absolute of received
24 + calc: ($interface_speed > 0) ? ($this * 100 / ($interface_speed * 1000)) : ( nan )
25 + units: %
26 + every: 10s
27 + warn: $this > (($status >= $WARNING) ? (85) : (90))
28 + delay: up 1m down 1m multiplier 1.5 max 1h
29 + summary: System network interface ${label:device} inbound utilization
30 + info: Average inbound utilization for the network interface ${label:device} over the last minute
31 + to: silent
32 +
33 + template: 1m_sent_traffic_overflow
34 + on: net.net
35 + class: Workload
36 + type: System
37 + component: Network
38 +host labels: _os=linux
39 + lookup: average -1m unaligned absolute of sent
40 + calc: ($interface_speed > 0) ? ($this * 100 / ($interface_speed * 1000)) : ( nan )
41 + units: %
42 + every: 10s
43 + warn: $this > (($status >= $WARNING) ? (85) : (90))
44 + delay: up 1m down 1m multiplier 1.5 max 1h
45 + summary: System network interface ${label:device} outbound utilization
46 + info: Average outbound utilization for the network interface ${label:device} over the last minute
47 + to: silent
48
49 # -----------------------------------------------------------------------------
50 # dropped packets
@@ -65,8 +61,6 @@ component: Network
61 class: Workload
62 type: System
63 component: Network
68 - os: *
69 - hosts: *
64 lookup: sum -10m unaligned absolute of received
65 units: packets
66 every: 1m
@@ -78,120 +72,110 @@ component: Network
72 class: Workload
73 type: System
74 component: Network
81 - os: *
82 - hosts: *
75 lookup: sum -10m unaligned absolute of sent
76 units: packets
77 every: 1m
78 summary: Network interface ${label:device} sent packets
79 info: Sent packets for the network interface ${label:device} in the last 10 minutes
80
89 - template: inbound_packets_dropped_ratio
90 - on: net.drops
91 - class: Errors
92 - type: System
93 -component: Network
94 - os: *
95 - hosts: *
81 + template: inbound_packets_dropped_ratio
82 + on: net.drops
83 + class: Errors
84 + type: System
85 + component: Network
86 chart labels: device=!wl* *
97 - lookup: sum -10m unaligned absolute of inbound
98 - calc: (($net_interface_inbound_packets > 10000) ? ($this * 100 / $net_interface_inbound_packets) : (0))
99 - units: %
100 - every: 1m
101 - warn: $this >= 2
102 - delay: up 1m down 1h multiplier 1.5 max 2h
103 - summary: System network interface ${label:device} inbound drops
104 - info: Ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
105 - to: silent
106 -
107 - template: outbound_packets_dropped_ratio
108 - on: net.drops
109 - class: Errors
110 - type: System
111 -component: Network
112 - os: *
113 - hosts: *
87 + lookup: sum -10m unaligned absolute of inbound
88 + calc: (($net_interface_inbound_packets > 10000) ? ($this * 100 / $net_interface_inbound_packets) : (0))
89 + units: %
90 + every: 1m
91 + warn: $this >= 2
92 + delay: up 1m down 1h multiplier 1.5 max 2h
93 + summary: System network interface ${label:device} inbound drops
94 + info: Ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
95 + to: silent
96 +
97 + template: outbound_packets_dropped_ratio
98 + on: net.drops
99 + class: Errors
100 + type: System
101 + component: Network
102 chart labels: device=!wl* *
115 - lookup: sum -10m unaligned absolute of outbound
116 - calc: (($net_interface_outbound_packets > 1000) ? ($this * 100 / $net_interface_outbound_packets) : (0))
117 - units: %
118 - every: 1m
119 - warn: $this >= 2
120 - delay: up 1m down 1h multiplier 1.5 max 2h
121 - summary: System network interface ${label:device} outbound drops
122 - info: Ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
123 - to: silent
124 -
125 - template: wifi_inbound_packets_dropped_ratio
126 - on: net.drops
127 - class: Errors
128 - type: System
129 -component: Network
130 - os: linux
131 - hosts: *
103 + lookup: sum -10m unaligned absolute of outbound
104 + calc: (($net_interface_outbound_packets > 1000) ? ($this * 100 / $net_interface_outbound_packets) : (0))
105 + units: %
106 + every: 1m
107 + warn: $this >= 2
108 + delay: up 1m down 1h multiplier 1.5 max 2h
109 + summary: System network interface ${label:device} outbound drops
110 + info: Ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
111 + to: silent
112 +
113 + template: wifi_inbound_packets_dropped_ratio
114 + on: net.drops
115 + class: Errors
116 + type: System
117 + component: Network
118 + host labels: _os=linux
119 chart labels: device=wl*
133 - lookup: sum -10m unaligned absolute of received
134 - calc: (($net_interface_inbound_packets > 10000) ? ($this * 100 / $net_interface_inbound_packets) : (0))
135 - units: %
136 - every: 1m
137 - warn: $this >= 10
138 - delay: up 1m down 1h multiplier 1.5 max 2h
139 - summary: System network interface ${label:device} inbound drops ratio
140 - info: Ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
141 - to: silent
142 -
143 - template: wifi_outbound_packets_dropped_ratio
144 - on: net.drops
145 - class: Errors
146 - type: System
147 -component: Network
148 - os: linux
149 - hosts: *
120 + lookup: sum -10m unaligned absolute of received
121 + calc: (($net_interface_inbound_packets > 10000) ? ($this * 100 / $net_interface_inbound_packets) : (0))
122 + units: %
123 + every: 1m
124 + warn: $this >= 10
125 + delay: up 1m down 1h multiplier 1.5 max 2h
126 + summary: System network interface ${label:device} inbound drops ratio
127 + info: Ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
128 + to: silent
129 +
130 + template: wifi_outbound_packets_dropped_ratio
131 + on: net.drops
132 + class: Errors
133 + type: System
134 + component: Network
135 + host labels: _os=linux
136 chart labels: device=wl*
151 - lookup: sum -10m unaligned absolute of sent
152 - calc: (($net_interface_outbound_packets > 1000) ? ($this * 100 / $net_interface_outbound_packets) : (0))
153 - units: %
154 - every: 1m
155 - warn: $this >= 10
156 - delay: up 1m down 1h multiplier 1.5 max 2h
157 - summary: System network interface ${label:device} outbound drops ratio
158 - info: Ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
159 - to: silent
137 + lookup: sum -10m unaligned absolute of sent
138 + calc: (($net_interface_outbound_packets > 1000) ? ($this * 100 / $net_interface_outbound_packets) : (0))
139 + units: %
140 + every: 1m
141 + warn: $this >= 10
142 + delay: up 1m down 1h multiplier 1.5 max 2h
143 + summary: System network interface ${label:device} outbound drops ratio
144 + info: Ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
145 + to: silent
146
147 # -----------------------------------------------------------------------------
148 # interface errors
149
164 - template: interface_inbound_errors
165 - on: net.errors
166 - class: Errors
167 - type: System
168 -component: Network
169 - os: freebsd
170 - hosts: *
171 - lookup: sum -10m unaligned absolute of inbound
172 - units: errors
173 - every: 1m
174 - warn: $this >= 5
175 - delay: down 1h multiplier 1.5 max 2h
176 - summary: System network interface ${label:device} inbound errors
177 - info: Number of inbound errors for the network interface ${label:device} in the last 10 minutes
178 - to: silent
179 -
180 - template: interface_outbound_errors
181 - on: net.errors
182 - class: Errors
183 - type: System
184 -component: Network
185 - os: freebsd
186 - hosts: *
187 - lookup: sum -10m unaligned absolute of outbound
188 - units: errors
189 - every: 1m
190 - warn: $this >= 5
191 - delay: down 1h multiplier 1.5 max 2h
192 - summary: System network interface ${label:device} outbound errors
193 - info: Number of outbound errors for the network interface ${label:device} in the last 10 minutes
194 - to: silent
150 + template: interface_inbound_errors
151 + on: net.errors
152 + class: Errors
153 + type: System
154 + component: Network
155 +host labels: _os=freebsd
156 + lookup: sum -10m unaligned absolute of inbound
157 + units: errors
158 + every: 1m
159 + warn: $this >= 5
160 + delay: down 1h multiplier 1.5 max 2h
161 + summary: System network interface ${label:device} inbound errors
162 + info: Number of inbound errors for the network interface ${label:device} in the last 10 minutes
163 + to: silent
164 +
165 + template: interface_outbound_errors
166 + on: net.errors
167 + class: Errors
168 + type: System
169 + component: Network
170 +host labels: _os=freebsd
171 + lookup: sum -10m unaligned absolute of outbound
172 + units: errors
173 + every: 1m
174 + warn: $this >= 5
175 + delay: down 1h multiplier 1.5 max 2h
176 + summary: System network interface ${label:device} outbound errors
177 + info: Number of outbound errors for the network interface ${label:device} in the last 10 minutes
178 + to: silent
179
180 # -----------------------------------------------------------------------------
181 # FIFO errors
@@ -201,21 +185,20 @@ component: Network
185 # the alarm is checked every 1 minute
186 # and examines the last 10 minutes of data
187
204 - template: 10min_fifo_errors
205 - on: net.fifo
206 - class: Errors
207 - type: System
208 -component: Network
209 - os: linux
210 - hosts: *
211 - lookup: sum -10m unaligned absolute
212 - units: errors
213 - every: 1m
214 - warn: $this > 0
215 - delay: down 1h multiplier 1.5 max 2h
216 - summary: System network interface ${label:device} FIFO errors
217 - info: Number of FIFO errors for the network interface ${label:device} in the last 10 minutes
218 - to: silent
188 + template: 10min_fifo_errors
189 + on: net.fifo
190 + class: Errors
191 + type: System
192 + component: Network
193 +host labels: _os=linux
194 + lookup: sum -10m unaligned absolute
195 + units: errors
196 + every: 1m
197 + warn: $this > 0
198 + delay: down 1h multiplier 1.5 max 2h
199 + summary: System network interface ${label:device} FIFO errors
200 + info: Number of FIFO errors for the network interface ${label:device} in the last 10 minutes
201 + to: silent
202
203 # -----------------------------------------------------------------------------
204 # check for packet storms
@@ -226,33 +209,31 @@ component: Network
209 # we assume the minimum packet storm should at least have
210 # 10000 packets/s, average of the last 10 seconds
211
229 - template: 1m_received_packets_rate
230 - on: net.packets
231 - class: Workload
232 - type: System
233 -component: Network
234 - os: linux freebsd
235 - hosts: *
236 - lookup: average -1m unaligned of received
237 - units: packets
238 - every: 10s
239 - info: Average number of packets received by the network interface ${label:device} over the last minute
240 -
241 - template: 10s_received_packets_storm
242 - on: net.packets
243 - class: Workload
244 - type: System
245 -component: Network
246 - os: linux freebsd
247 - hosts: *
248 - lookup: average -10s unaligned of received
249 - calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
250 - every: 10s
251 - units: %
252 - warn: $this > (($status >= $WARNING)?(200):(5000))
253 - crit: $this > (($status == $CRITICAL)?(5000):(6000))
254 - options: no-clear-notification
255 - summary: System network interface ${label:device} inbound packet storm
256 - info: Ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
257 - compared to the rate over the last minute
258 - to: silent
212 + template: 1m_received_packets_rate
213 + on: net.packets
214 + class: Workload
215 + type: System
216 + component: Network
217 +host labels: _os=linux freebsd
218 + lookup: average -1m unaligned of received
219 + units: packets
220 + every: 10s
221 + info: Average number of packets received by the network interface ${label:device} over the last minute
222 +
223 + template: 10s_received_packets_storm
224 + on: net.packets
225 + class: Workload
226 + type: System
227 + component: Network
228 +host labels: _os=linux freebsd
229 + lookup: average -10s unaligned of received
230 + calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
231 + every: 10s
232 + units: %
233 + warn: $this > (($status >= $WARNING)?(200):(5000))
234 + crit: $this > (($status == $CRITICAL)?(5000):(6000))
235 + options: no-clear-notification
236 + summary: System network interface ${label:device} inbound packet storm
237 + info: Ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
238 + compared to the rate over the last minute
239 + to: silent
src/health/health.d/netfilter.conf
+16 -18
@@ -1,20 +1,18 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
4 - alarm: netfilter_conntrack_full
5 - on: netfilter.conntrack_sockets
6 - class: Workload
7 - type: System
8 -component: Network
9 - os: linux
10 - hosts: *
11 - lookup: max -10s unaligned of connections
12 - calc: $this * 100 / $netfilter_conntrack_max
13 - units: %
14 - every: 10s
15 - warn: $this > (($status >= $WARNING) ? (85) : (90))
16 - crit: $this > (($status == $CRITICAL) ? (90) : (95))
17 - delay: down 5m multiplier 1.5 max 1h
18 - summary: System Netfilter connection tracker utilization
19 - info: Netfilter connection tracker table size utilization
20 - to: sysadmin
3 + alarm: netfilter_conntrack_full
4 + on: netfilter.conntrack_sockets
5 + class: Workload
6 + type: System
7 + component: Network
8 +host labels: _os=linux
9 + lookup: max -10s unaligned of connections
10 + calc: $this * 100 / $netfilter_conntrack_max
11 + units: %
12 + every: 10s
13 + warn: $this > (($status >= $WARNING) ? (85) : (90))
14 + crit: $this > (($status == $CRITICAL) ? (90) : (95))
15 + delay: down 5m multiplier 1.5 max 1h
16 + summary: System Netfilter connection tracker utilization
17 + info: Netfilter connection tracker table size utilization
18 + to: sysadmin
src/health/health.d/postgres.conf
+2 -14
@@ -5,7 +5,6 @@
5 class: Utilization
6 type: Database
7 component: PostgreSQL
8 - hosts: *
8 lookup: average -1m unaligned of used
9 units: %
10 every: 1m
@@ -21,7 +20,6 @@ component: PostgreSQL
20 class: Utilization
21 type: Database
22 component: PostgreSQL
24 - hosts: *
23 lookup: average -1m unaligned of used
24 units: %
25 every: 1m
@@ -36,7 +34,6 @@ component: PostgreSQL
34 class: Utilization
35 type: Database
36 component: PostgreSQL
39 - hosts: *
37 calc: $txid_exhaustion
38 units: %
39 every: 1m
@@ -53,7 +50,6 @@ component: PostgreSQL
50 class: Workload
51 type: Database
52 component: PostgreSQL
56 - hosts: *
53 lookup: average -1m unaligned of miss
54 calc: 100 - $this
55 units: %
@@ -70,7 +66,6 @@ component: PostgreSQL
66 class: Workload
67 type: Database
68 component: PostgreSQL
73 - hosts: *
69 lookup: average -5m unaligned of rollback
70 units: %
71 every: 1m
@@ -85,7 +80,6 @@ component: PostgreSQL
80 class: Errors
81 type: Database
82 component: PostgreSQL
88 - hosts: *
83 lookup: sum -1m unaligned of deadlocks
84 units: deadlocks
85 every: 1m
@@ -102,7 +96,6 @@ component: PostgreSQL
96 class: Workload
97 type: Database
98 component: PostgreSQL
105 - hosts: *
99 lookup: average -1m unaligned of miss
100 calc: 100 - $this
101 units: %
@@ -119,7 +112,6 @@ component: PostgreSQL
112 class: Workload
113 type: Database
114 component: PostgreSQL
122 - hosts: *
115 lookup: average -1m unaligned of miss
116 calc: 100 - $this
117 units: %
@@ -136,7 +128,6 @@ component: PostgreSQL
128 class: Workload
129 type: Database
130 component: PostgreSQL
139 - hosts: *
131 lookup: average -1m unaligned of miss
132 calc: 100 - $this
133 units: %
@@ -153,7 +144,6 @@ component: PostgreSQL
144 class: Workload
145 type: Database
146 component: PostgreSQL
156 - hosts: *
147 lookup: average -1m unaligned of miss
148 calc: 100 - $this
149 units: %
@@ -170,7 +160,6 @@ component: PostgreSQL
160 class: Errors
161 type: Database
162 component: PostgreSQL
173 - hosts: *
163 calc: ($table_size > (1024 * 1024 * 100)) ? ($bloat) : (0)
164 units: %
165 every: 1m
@@ -186,7 +175,7 @@ component: PostgreSQL
175 class: Errors
176 type: Database
177 component: PostgreSQL
189 - hosts: !*
178 +host labels: _hostname=!*
179 calc: $time
180 units: seconds
181 every: 1m
@@ -200,7 +189,7 @@ component: PostgreSQL
189 class: Errors
190 type: Database
191 component: PostgreSQL
203 - hosts: !*
192 +host labels: _hostname=!*
193 calc: $time
194 units: seconds
195 every: 1m
@@ -216,7 +205,6 @@ component: PostgreSQL
205 class: Errors
206 type: Database
207 component: PostgreSQL
219 - hosts: *
208 calc: ($index_size > (1024 * 1024 * 10)) ? ($bloat) : (0)
209 units: %
210 every: 1m
src/health/health.d/processes.conf
-1
@@ -5,7 +5,6 @@
5 class: Workload
6 type: System
7 component: Processes
8 - hosts: *
8 calc: $active * 100 / $pidmax
9 units: %
10 every: 5s
src/health/health.d/python.d.plugin.conf
+15 -16
@@ -1,18 +1,17 @@
1 -
1 # make sure python.d.plugin data collection job is running
2
4 - template: python.d_job_last_collected_secs
5 - on: netdata.pythond_runtime
6 - class: Errors
7 - type: Netdata
8 -component: python.d.plugin
9 - module: !* *
10 - calc: $now - $last_collected_t
11 - units: seconds ago
12 - every: 10s
13 - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
14 - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
15 - delay: down 5m multiplier 1.5 max 1h
16 - summary: Python.d plugin last collection
17 - info: Number of seconds since the last successful data collection
18 - to: webmaster
3 + template: python.d_job_last_collected_secs
4 + on: netdata.pythond_runtime
5 + class: Errors
6 + type: Netdata
7 + component: python.d.plugin
8 +host labels: _hostname=!*
9 + calc: $now - $last_collected_t
10 + units: seconds ago
11 + every: 10s
12 + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
13 + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
14 + delay: down 5m multiplier 1.5 max 1h
15 + summary: Python.d plugin last collection
16 + info: Number of seconds since the last successful data collection
17 + to: webmaster
src/health/health.d/qos.conf
+10 -12
@@ -1,18 +1,16 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 # check if a QoS class is dropping packets
4 # the alarm is checked every 10 seconds
5 # and examines the last minute of data
6
8 -template: 10min_qos_packet_drops
9 - on: tc.qos_dropped
10 - os: linux
11 - hosts: *
12 - lookup: sum -5m unaligned absolute
13 - every: 30s
14 - warn: $this > 0
15 - units: packets
16 - summary: QOS packet drops
17 - info: Dropped packets in the last 5 minutes
18 - to: silent
7 + template: 10min_qos_packet_drops
8 + on: tc.qos_dropped
9 +host labels: _os=linux
10 + lookup: sum -5m unaligned absolute
11 + every: 30s
12 + warn: $this > 0
13 + units: packets
14 + summary: QOS packet drops
15 + info: Dropped packets in the last 5 minutes
16 + to: silent
src/health/health.d/ram.conf
+69 -75
@@ -1,82 +1,76 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
4 - alarm: ram_in_use
5 - on: system.ram
6 - class: Utilization
7 - type: System
8 -component: Memory
9 - os: linux
10 - hosts: *
11 - calc: $used * 100 / ($used + $cached + $free + $buffers)
12 - units: %
13 - every: 10s
14 - warn: $this > (($status >= $WARNING) ? (80) : (90))
15 - crit: $this > (($status == $CRITICAL) ? (90) : (98))
16 - delay: down 15m multiplier 1.5 max 1h
17 - summary: System memory utilization
18 - info: System memory utilization
19 - to: sysadmin
3 + alarm: ram_in_use
4 + on: system.ram
5 + class: Utilization
6 + type: System
7 + component: Memory
8 +host labels: _os=linux
9 + calc: $used * 100 / ($used + $cached + $free + $buffers)
10 + units: %
11 + every: 10s
12 + warn: $this > (($status >= $WARNING) ? (80) : (90))
13 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
14 + delay: down 15m multiplier 1.5 max 1h
15 + summary: System memory utilization
16 + info: System memory utilization
17 + to: sysadmin
18
21 - alarm: ram_available
22 - on: mem.available
23 - class: Utilization
24 - type: System
25 -component: Memory
26 - os: linux
27 - hosts: *
28 - calc: $avail * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers)
29 - units: %
30 - every: 10s
31 - warn: $this < (($status >= $WARNING) ? (15) : (10))
32 - delay: down 15m multiplier 1.5 max 1h
33 - summary: System available memory
34 - info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
35 - to: silent
19 + alarm: ram_available
20 + on: mem.available
21 + class: Utilization
22 + type: System
23 + component: Memory
24 +host labels: _os=linux
25 + calc: $avail * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers)
26 + units: %
27 + every: 10s
28 + warn: $this < (($status >= $WARNING) ? (15) : (10))
29 + delay: down 15m multiplier 1.5 max 1h
30 + summary: System available memory
31 + info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
32 + to: silent
33
37 - alarm: oom_kill
38 - on: mem.oom_kill
39 - os: linux
40 - hosts: *
41 - lookup: sum -30m unaligned
42 - units: kills
43 - every: 5m
44 - warn: $this > 0
45 - delay: down 10m
46 - summary: System OOM kills
47 - info: Number of out of memory kills in the last 30 minutes
48 - to: silent
34 + alarm: oom_kill
35 + on: mem.oom_kill
36 +host labels: _os=linux
37 + lookup: sum -30m unaligned
38 + units: kills
39 + every: 5m
40 + warn: $this > 0
41 + delay: down 10m
42 + summary: System OOM kills
43 + info: Number of out of memory kills in the last 30 minutes
44 + to: silent
45
46 ## FreeBSD
51 - alarm: ram_in_use
52 - on: system.ram
53 - class: Utilization
54 - type: System
55 -component: Memory
56 - os: freebsd
57 - hosts: *
58 - calc: ($active + $wired + $laundry + $buffers) * 100 / ($active + $wired + $laundry + $buffers + $cache + $free + $inactive)
59 - units: %
60 - every: 10s
61 - warn: $this > (($status >= $WARNING) ? (80) : (90))
62 - crit: $this > (($status == $CRITICAL) ? (90) : (98))
63 - delay: down 15m multiplier 1.5 max 1h
64 - summary: System memory utilization
65 - info: System memory utilization
66 - to: sysadmin
47 + alarm: ram_in_use
48 + on: system.ram
49 + class: Utilization
50 + type: System
51 + component: Memory
52 +host labels: _os=freebsd
53 + calc: ($active + $wired + $laundry + $buffers) * 100 / ($active + $wired + $laundry + $buffers + $cache + $free + $inactive)
54 + units: %
55 + every: 10s
56 + warn: $this > (($status >= $WARNING) ? (80) : (90))
57 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
58 + delay: down 15m multiplier 1.5 max 1h
59 + summary: System memory utilization
60 + info: System memory utilization
61 + to: sysadmin
62
68 - alarm: ram_available
69 - on: mem.available
70 - class: Utilization
71 - type: System
72 -component: Memory
73 - os: freebsd
74 - hosts: *
75 - calc: $avail * 100 / ($system.ram.free + $system.ram.active + $system.ram.inactive + $system.ram.wired + $system.ram.cache + $system.ram.laundry + $system.ram.buffers)
76 - units: %
77 - every: 10s
78 - warn: $this < (($status >= $WARNING) ? (15) : (10))
79 - delay: down 15m multiplier 1.5 max 1h
80 - summary: System available memory
81 - info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
82 - to: silent
63 + alarm: ram_available
64 + on: mem.available
65 + class: Utilization
66 + type: System
67 + component: Memory
68 +host labels: _os=freebsd
69 + calc: $avail * 100 / ($system.ram.free + $system.ram.active + $system.ram.inactive + $system.ram.wired + $system.ram.cache + $system.ram.laundry + $system.ram.buffers)
70 + units: %
71 + every: 10s
72 + warn: $this < (($status >= $WARNING) ? (15) : (10))
73 + delay: down 15m multiplier 1.5 max 1h
74 + summary: System available memory
75 + info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
76 + to: silent
src/health/health.d/softnet.conf
+47 -51
@@ -1,57 +1,53 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 # check for common /proc/net/softnet_stat errors
4
6 - alarm: 1min_netdev_backlog_exceeded
7 - on: system.softnet_stat
8 - class: Errors
9 - type: System
10 -component: Network
11 - os: linux
12 - hosts: *
13 - lookup: average -1m unaligned absolute of dropped
14 - units: packets
15 - every: 10s
16 - warn: $this > (($status >= $WARNING) ? (0) : (10))
17 - delay: down 1h multiplier 1.5 max 2h
18 - summary: System netdev dropped packets
19 - info: Average number of dropped packets in the last minute \
20 - due to exceeded net.core.netdev_max_backlog
21 - to: silent
5 + alarm: 1min_netdev_backlog_exceeded
6 + on: system.softnet_stat
7 + class: Errors
8 + type: System
9 + component: Network
10 +host labels: _os=linux
11 + lookup: average -1m unaligned absolute of dropped
12 + units: packets
13 + every: 10s
14 + warn: $this > (($status >= $WARNING) ? (0) : (10))
15 + delay: down 1h multiplier 1.5 max 2h
16 + summary: System netdev dropped packets
17 + info: Average number of dropped packets in the last minute \
18 + due to exceeded net.core.netdev_max_backlog
19 + to: silent
20
23 - alarm: 1min_netdev_budget_ran_outs
24 - on: system.softnet_stat
25 - class: Errors
26 - type: System
27 -component: Network
28 - os: linux
29 - hosts: *
30 - lookup: average -1m unaligned absolute of squeezed
31 - units: events
32 - every: 10s
33 - warn: $this > (($status >= $WARNING) ? (0) : (10))
34 - delay: down 1h multiplier 1.5 max 2h
35 - summary: System netdev budget run outs
36 - info: Average number of times ksoftirq ran out of sysctl net.core.netdev_budget or \
37 - net.core.netdev_budget_usecs with work remaining over the last minute \
38 - (this can be a cause for dropped packets)
39 - to: silent
21 + alarm: 1min_netdev_budget_ran_outs
22 + on: system.softnet_stat
23 + class: Errors
24 + type: System
25 + component: Network
26 +host labels: _os=linux
27 + lookup: average -1m unaligned absolute of squeezed
28 + units: events
29 + every: 10s
30 + warn: $this > (($status >= $WARNING) ? (0) : (10))
31 + delay: down 1h multiplier 1.5 max 2h
32 + summary: System netdev budget run outs
33 + info: Average number of times ksoftirq ran out of sysctl net.core.netdev_budget or \
34 + net.core.netdev_budget_usecs with work remaining over the last minute \
35 + (this can be a cause for dropped packets)
36 + to: silent
37
41 - alarm: 10min_netisr_backlog_exceeded
42 - on: system.softnet_stat
43 - class: Errors
44 - type: System
45 -component: Network
46 - os: freebsd
47 - hosts: *
48 - lookup: average -1m unaligned absolute of qdrops
49 - units: packets
50 - every: 10s
51 - warn: $this > (($status >= $WARNING) ? (0) : (10))
52 - delay: down 1h multiplier 1.5 max 2h
53 - summary: System netisr drops
54 - info: Average number of drops in the last minute \
55 - due to exceeded sysctl net.route.netisr_maxqlen \
56 - (this can be a cause for dropped packets)
57 - to: silent
38 + alarm: 10min_netisr_backlog_exceeded
39 + on: system.softnet_stat
40 + class: Errors
41 + type: System
42 + component: Network
43 +host labels: _os=freebsd
44 + lookup: average -1m unaligned absolute of qdrops
45 + units: packets
46 + every: 10s
47 + warn: $this > (($status >= $WARNING) ? (0) : (10))
48 + delay: down 1h multiplier 1.5 max 2h
49 + summary: System netisr drops
50 + info: Average number of drops in the last minute \
51 + due to exceeded sysctl net.route.netisr_maxqlen \
52 + (this can be a cause for dropped packets)
53 + to: silent
src/health/health.d/swap.conf
+31 -34
@@ -1,37 +1,34 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
4 - alarm: 30min_ram_swapped_out
5 - on: mem.swapio
6 - class: Workload
7 - type: System
8 -component: Memory
9 - os: linux freebsd
10 - hosts: *
11 - lookup: sum -30m unaligned absolute of out
12 - # we have to convert KB to MB by dividing $this (i.e. the result of the lookup) with 1024
13 - calc: $this / 1024 * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
14 - units: % of RAM
15 - every: 1m
16 - warn: $this > (($status >= $WARNING) ? (20) : (30))
17 - delay: down 15m multiplier 1.5 max 1h
18 - summary: System memory swapped out
19 - info: Percentage of the system RAM swapped in the last 30 minutes
20 - to: silent
3 + alarm: 30min_ram_swapped_out
4 + on: mem.swapio
5 + class: Workload
6 + type: System
7 + component: Memory
8 +host labels: _os=linux freebsd
9 + lookup: sum -30m unaligned absolute of out
10 + # we have to convert KB to MB by dividing $this (i.e. the result of the lookup) with 1024
11 + calc: $this / 1024 * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
12 + units: % of RAM
13 + every: 1m
14 + warn: $this > (($status >= $WARNING) ? (20) : (30))
15 + delay: down 15m multiplier 1.5 max 1h
16 + summary: System memory swapped out
17 + info: Percentage of the system RAM swapped in the last 30 minutes
18 + to: silent
19
22 - alarm: used_swap
23 - on: mem.swap
24 - class: Utilization
25 - type: System
26 -component: Memory
27 - os: linux freebsd
28 - hosts: *
29 - calc: (($used + $free) > 0) ? ($used * 100 / ($used + $free)) : 0
30 - units: %
31 - every: 10s
32 - warn: $this > (($status >= $WARNING) ? (80) : (90))
33 - crit: $this > (($status == $CRITICAL) ? (90) : (98))
34 - delay: up 30s down 15m multiplier 1.5 max 1h
35 - summary: System swap memory utilization
36 - info: Swap memory utilization
37 - to: sysadmin
20 + alarm: used_swap
21 + on: mem.swap
22 + class: Utilization
23 + type: System
24 + component: Memory
25 +host labels: _os=linux freebsd
26 + calc: (($used + $free) > 0) ? ($used * 100 / ($used + $free)) : 0
27 + units: %
28 + every: 10s
29 + warn: $this > (($status >= $WARNING) ? (80) : (90))
30 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
31 + delay: up 30s down 15m multiplier 1.5 max 1h
32 + summary: System swap memory utilization
33 + info: Swap memory utilization
34 + to: sysadmin
src/health/health.d/synchronization.conf
-1
@@ -2,7 +2,6 @@
2 on: mem.sync
3 lookup: sum -1m of sync
4 units: calls
5 - plugin: ebpf.plugin
5 every: 1m
6 warn: $this > 6
7 delay: up 1m down 10m multiplier 1.5 max 1h
src/health/health.d/systemdunits.conf
+154 -154
@@ -1,177 +1,177 @@
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 ## Service units
4 - template: systemd_service_unit_failed_state
5 - on: systemd.service_unit_state
6 - class: Errors
7 - type: Linux
8 -component: Systemd units
9 - module: !* *
10 - calc: $failed
11 - units: state
12 - every: 10s
13 - warn: $this != nan AND $this == 1
14 - delay: down 5m multiplier 1.5 max 1h
15 - summary: systemd unit ${label:unit_name} state
16 - info: systemd service unit in the failed state
17 - to: sysadmin
4 + template: systemd_service_unit_failed_state
5 + on: systemd.service_unit_state
6 + class: Errors
7 + type: Linux
8 + component: Systemd units
9 +chart labels: unit_name=!*
10 + calc: $failed
11 + units: state
12 + every: 10s
13 + warn: $this != nan AND $this == 1
14 + delay: down 5m multiplier 1.5 max 1h
15 + summary: systemd unit ${label:unit_name} state
16 + info: systemd service unit in the failed state
17 + to: sysadmin
18
19 ## Socket units
20 - template: systemd_socket_unit_failed_state
21 - on: systemd.socket_unit_state
22 - class: Errors
23 - type: Linux
24 -component: Systemd units
25 - module: !* *
26 - calc: $failed
27 - units: state
28 - every: 10s
29 - warn: $this != nan AND $this == 1
30 - delay: down 5m multiplier 1.5 max 1h
31 - summary: systemd unit ${label:unit_name} state
32 - info: systemd socket unit in the failed state
33 - to: sysadmin
20 + template: systemd_socket_unit_failed_state
21 + on: systemd.socket_unit_state
22 + class: Errors
23 + type: Linux
24 + component: Systemd units
25 +chart labels: unit_name=!*
26 + calc: $failed
27 + units: state
28 + every: 10s
29 + warn: $this != nan AND $this == 1
30 + delay: down 5m multiplier 1.5 max 1h
31 + summary: systemd unit ${label:unit_name} state
32 + info: systemd socket unit in the failed state
33 + to: sysadmin
34
35 ## Target units
36 - template: systemd_target_unit_failed_state
37 - on: systemd.target_unit_state
38 - class: Errors
39 - type: Linux
40 -component: Systemd units
41 - module: !* *
42 - calc: $failed
43 - units: state
44 - every: 10s
45 - warn: $this != nan AND $this == 1
46 - delay: down 5m multiplier 1.5 max 1h
47 - summary: systemd unit ${label:unit_name} state
48 - info: systemd target unit in the failed state
49 - to: sysadmin
36 + template: systemd_target_unit_failed_state
37 + on: systemd.target_unit_state
38 + class: Errors
39 + type: Linux
40 + component: Systemd units
41 +chart labels: unit_name=!*
42 + calc: $failed
43 + units: state
44 + every: 10s
45 + warn: $this != nan AND $this == 1
46 + delay: down 5m multiplier 1.5 max 1h
47 + summary: systemd unit ${label:unit_name} state
48 + info: systemd target unit in the failed state
49 + to: sysadmin
50
51 ## Path units
52 - template: systemd_path_unit_failed_state
53 - on: systemd.path_unit_state
54 - class: Errors
55 - type: Linux
56 -component: Systemd units
57 - module: !* *
58 - calc: $failed
59 - units: state
60 - every: 10s
61 - warn: $this != nan AND $this == 1
62 - delay: down 5m multiplier 1.5 max 1h
63 - summary: systemd unit ${label:unit_name} state
64 - info: systemd path unit in the failed state
65 - to: sysadmin
52 + template: systemd_path_unit_failed_state
53 + on: systemd.path_unit_state
54 + class: Errors
55 + type: Linux
56 + component: Systemd units
57 +chart labels: unit_name=!*
58 + calc: $failed
59 + units: state
60 + every: 10s
61 + warn: $this != nan AND $this == 1
62 + delay: down 5m multiplier 1.5 max 1h
63 + summary: systemd unit ${label:unit_name} state
64 + info: systemd path unit in the failed state
65 + to: sysadmin
66
67 ## Device units
68 - template: systemd_device_unit_failed_state
69 - on: systemd.device_unit_state
70 - class: Errors
71 - type: Linux
72 -component: Systemd units
73 - module: !* *
74 - calc: $failed
75 - units: state
76 - every: 10s
77 - warn: $this != nan AND $this == 1
78 - delay: down 5m multiplier 1.5 max 1h
79 - summary: systemd unit ${label:unit_name} state
80 - info: systemd device unit in the failed state
81 - to: sysadmin
68 + template: systemd_device_unit_failed_state
69 + on: systemd.device_unit_state
70 + class: Errors
71 + type: Linux
72 + component: Systemd units
73 +chart labels: unit_name=!*
74 + calc: $failed
75 + units: state
76 + every: 10s
77 + warn: $this != nan AND $this == 1
78 + delay: down 5m multiplier 1.5 max 1h
79 + summary: systemd unit ${label:unit_name} state
80 + info: systemd device unit in the failed state
81 + to: sysadmin
82
83 ## Mount units
84 - template: systemd_mount_unit_failed_state
85 - on: systemd.mount_unit_state
86 - class: Errors
87 - type: Linux
88 -component: Systemd units
89 - module: !* *
90 - calc: $failed
91 - units: state
92 - every: 10s
93 - warn: $this != nan AND $this == 1
94 - delay: down 5m multiplier 1.5 max 1h
95 - summary: systemd unit ${label:unit_name} state
96 - info: systemd mount units in the failed state
97 - to: sysadmin
84 + template: systemd_mount_unit_failed_state
85 + on: systemd.mount_unit_state
86 + class: Errors
87 + type: Linux
88 + component: Systemd units
89 +chart labels: unit_name=!*
90 + calc: $failed
91 + units: state
92 + every: 10s
93 + warn: $this != nan AND $this == 1
94 + delay: down 5m multiplier 1.5 max 1h
95 + summary: systemd unit ${label:unit_name} state
96 + info: systemd mount units in the failed state
97 + to: sysadmin
98
99 ## Automount units
100 - template: systemd_automount_unit_failed_state
101 - on: systemd.automount_unit_state
102 - class: Errors
103 - type: Linux
104 -component: Systemd units
105 - module: !* *
106 - calc: $failed
107 - units: state
108 - every: 10s
109 - warn: $this != nan AND $this == 1
110 - delay: down 5m multiplier 1.5 max 1h
111 - summary: systemd unit ${label:unit_name} state
112 - info: systemd automount unit in the failed state
113 - to: sysadmin
100 + template: systemd_automount_unit_failed_state
101 + on: systemd.automount_unit_state
102 + class: Errors
103 + type: Linux
104 + component: Systemd units
105 +chart labels: unit_name=!*
106 + calc: $failed
107 + units: state
108 + every: 10s
109 + warn: $this != nan AND $this == 1
110 + delay: down 5m multiplier 1.5 max 1h
111 + summary: systemd unit ${label:unit_name} state
112 + info: systemd automount unit in the failed state
113 + to: sysadmin
114
115 ## Swap units
116 - template: systemd_swap_unit_failed_state
117 - on: systemd.swap_unit_state
118 - class: Errors
119 - type: Linux
120 -component: Systemd units
121 - module: !* *
122 - calc: $failed
123 - units: state
124 - every: 10s
125 - warn: $this != nan AND $this == 1
126 - delay: down 5m multiplier 1.5 max 1h
127 - summary: systemd unit ${label:unit_name} state
128 - info: systemd swap units in the failed state
129 - to: sysadmin
116 + template: systemd_swap_unit_failed_state
117 + on: systemd.swap_unit_state
118 + class: Errors
119 + type: Linux
120 + component: Systemd units
121 +chart labels: unit_name=!*
122 + calc: $failed
123 + units: state
124 + every: 10s
125 + warn: $this != nan AND $this == 1
126 + delay: down 5m multiplier 1.5 max 1h
127 + summary: systemd unit ${label:unit_name} state
128 + info: systemd swap units in the failed state
129 + to: sysadmin
130
131 ## Scope units
132 - template: systemd_scope_unit_failed_state
133 - on: systemd.scope_unit_state
134 - class: Errors
135 - type: Linux
136 -component: Systemd units
137 - module: !* *
138 - calc: $failed
139 - units: state
140 - every: 10s
141 - warn: $this != nan AND $this == 1
142 - delay: down 5m multiplier 1.5 max 1h
143 - summary: systemd unit ${label:unit_name} state
144 - info: systemd scope units in the failed state
145 - to: sysadmin
132 + template: systemd_scope_unit_failed_state
133 + on: systemd.scope_unit_state
134 + class: Errors
135 + type: Linux
136 + component: Systemd units
137 +chart labels: unit_name=!*
138 + calc: $failed
139 + units: state
140 + every: 10s
141 + warn: $this != nan AND $this == 1
142 + delay: down 5m multiplier 1.5 max 1h
143 + summary: systemd unit ${label:unit_name} state
144 + info: systemd scope units in the failed state
145 + to: sysadmin
146
147 ## Slice units
148 - template: systemd_slice_unit_failed_state
149 - on: systemd.slice_unit_state
150 - class: Errors
151 - type: Linux
152 -component: Systemd units
153 - module: !* *
154 - calc: $failed
155 - units: state
156 - every: 10s
157 - warn: $this != nan AND $this == 1
158 - delay: down 5m multiplier 1.5 max 1h
159 - summary: systemd unit ${label:unit_name} state
160 - info: systemd slice units in the failed state
161 - to: sysadmin
148 + template: systemd_slice_unit_failed_state
149 + on: systemd.slice_unit_state
150 + class: Errors
151 + type: Linux
152 + component: Systemd units
153 +chart labels: unit_name=!*
154 + calc: $failed
155 + units: state
156 + every: 10s
157 + warn: $this != nan AND $this == 1
158 + delay: down 5m multiplier 1.5 max 1h
159 + summary: systemd unit ${label:unit_name} state
160 + info: systemd slice units in the failed state
161 + to: sysadmin
162
163 ## Timer units
164 - template: systemd_timer_unit_failed_state
165 - on: systemd.timer_unit_state
166 - class: Errors
167 - type: Linux
168 -component: Systemd units
169 - module: !* *
170 - calc: $failed
171 - units: state
172 - every: 10s
173 - warn: $this != nan AND $this == 1
174 - delay: down 5m multiplier 1.5 max 1h
175 - summary: systemd unit ${label:unit_name} state
176 - info: systemd timer unit in the failed state
177 - to: sysadmin
164 + template: systemd_timer_unit_failed_state
165 + on: systemd.timer_unit_state
166 + class: Errors
167 + type: Linux
168 + component: Systemd units
169 +chart labels: unit_name=!*
170 + calc: $failed
171 + units: state
172 + every: 10s
173 + warn: $this != nan AND $this == 1
174 + delay: down 5m multiplier 1.5 max 1h
175 + summary: systemd unit ${label:unit_name} state
176 + info: systemd timer unit in the failed state
177 + to: sysadmin
src/health/health.d/tcp_conn.conf
+16 -18
@@ -1,23 +1,21 @@
1 +# you can disable an alarm notification by setting the 'to' line to: silent
2
2 -#
3 # ${tcp_max_connections} may be nan or -1 if the system
4 # supports dynamic threshold for TCP connections.
5 # In this case, the alarm will always be zero.
6 -#
6
8 - alarm: tcp_connections
9 - on: ip.tcpsock
10 - class: Workload
11 - type: System
12 -component: Network
13 - os: linux
14 - hosts: *
15 - calc: (${tcp_max_connections} > 0) ? ( ${connections} * 100 / ${tcp_max_connections} ) : 0
16 - units: %
17 - every: 10s
18 - warn: $this > (($status >= $WARNING ) ? ( 60 ) : ( 80 ))
19 - crit: $this > (($status == $CRITICAL) ? ( 80 ) : ( 90 ))
20 - delay: up 0 down 5m multiplier 1.5 max 1h
21 - summary: System TCP connections utilization
22 - info: IPv4 TCP connections utilization
23 - to: sysadmin
7 + alarm: tcp_connections
8 + on: ip.tcpsock
9 + class: Workload
10 + type: System
11 + component: Network
12 +host labels: _os=linux
13 + calc: (${tcp_max_connections} > 0) ? ( ${connections} * 100 / ${tcp_max_connections} ) : 0
14 + units: %
15 + every: 10s
16 + warn: $this > (($status >= $WARNING ) ? ( 60 ) : ( 80 ))
17 + crit: $this > (($status == $CRITICAL) ? ( 80 ) : ( 90 ))
18 + delay: up 0 down 5m multiplier 1.5 max 1h
19 + summary: System TCP connections utilization
20 + info: IPv4 TCP connections utilization
21 + to: sysadmin
src/health/health.d/tcp_listen.conf
+61 -68
@@ -1,4 +1,3 @@
1 -#
1 # There are two queues involved when incoming TCP connections are handled
2 # (both at the kernel):
3 #
@@ -18,42 +17,39 @@
17 # -----------------------------------------------------------------------------
18 # tcp accept queue (at the kernel)
19
21 - alarm: 1m_tcp_accept_queue_overflows
22 - on: ip.tcp_accept_queue
23 - class: Workload
24 - type: System
25 -component: Network
26 - os: linux
27 - hosts: *
28 - lookup: average -60s unaligned absolute of ListenOverflows
29 - units: overflows
30 - every: 10s
31 - warn: $this > 1
32 - crit: $this > (($status == $CRITICAL) ? (1) : (5))
33 - delay: up 0 down 5m multiplier 1.5 max 1h
34 - summary: System TCP accept queue overflows
35 - info: Average number of overflows in the TCP accept queue over the last minute
36 - to: silent
20 + alarm: 1m_tcp_accept_queue_overflows
21 + on: ip.tcp_accept_queue
22 + class: Workload
23 + type: System
24 + component: Network
25 +host labels: _os=linux
26 + lookup: average -60s unaligned absolute of ListenOverflows
27 + units: overflows
28 + every: 10s
29 + warn: $this > 1
30 + crit: $this > (($status == $CRITICAL) ? (1) : (5))
31 + delay: up 0 down 5m multiplier 1.5 max 1h
32 + summary: System TCP accept queue overflows
33 + info: Average number of overflows in the TCP accept queue over the last minute
34 + to: silent
35
36 # THIS IS TOO GENERIC
37 # CHECK: https://github.com/netdata/netdata/issues/3234#issuecomment-423935842
40 - alarm: 1m_tcp_accept_queue_drops
41 - on: ip.tcp_accept_queue
42 - class: Workload
43 - type: System
44 -component: Network
45 - os: linux
46 - hosts: *
47 - lookup: average -60s unaligned absolute of ListenDrops
48 - units: drops
49 - every: 10s
50 - warn: $this > 1
51 - crit: $this > (($status == $CRITICAL) ? (1) : (5))
52 - delay: up 0 down 5m multiplier 1.5 max 1h
53 - summary: System TCP accept queue dropped packets
54 - info: Average number of dropped packets in the TCP accept queue over the last minute
55 - to: silent
56 -
38 + alarm: 1m_tcp_accept_queue_drops
39 + on: ip.tcp_accept_queue
40 + class: Workload
41 + type: System
42 + component: Network
43 +host labels: _os=linux
44 + lookup: average -60s unaligned absolute of ListenDrops
45 + units: drops
46 + every: 10s
47 + warn: $this > 1
48 + crit: $this > (($status == $CRITICAL) ? (1) : (5))
49 + delay: up 0 down 5m multiplier 1.5 max 1h
50 + summary: System TCP accept queue dropped packets
51 + info: Average number of dropped packets in the TCP accept queue over the last minute
52 + to: silent
53
54 # -----------------------------------------------------------------------------
55 # tcp SYN queue (at the kernel)
@@ -63,38 +59,35 @@ component: Network
59 # enabled or not. In both cases this probably indicates a SYN flood attack,
60 # so i guess a notification should be sent.
61
66 - alarm: 1m_tcp_syn_queue_drops
67 - on: ip.tcp_syn_queue
68 - class: Workload
69 - type: System
70 -component: Network
71 - os: linux
72 - hosts: *
73 - lookup: average -60s unaligned absolute of TCPReqQFullDrop
74 - units: drops
75 - every: 10s
76 - warn: $this > 1
77 - crit: $this > (($status == $CRITICAL) ? (0) : (5))
78 - delay: up 10 down 5m multiplier 1.5 max 1h
79 - summary: System TCP SYN queue drops
80 - info: Average number of SYN requests was dropped due to the full TCP SYN queue over the last minute \
81 - (SYN cookies were not enabled)
82 - to: silent
83 -
84 - alarm: 1m_tcp_syn_queue_cookies
85 - on: ip.tcp_syn_queue
86 - class: Workload
87 - type: System
88 -component: Network
89 - os: linux
90 - hosts: *
91 - lookup: average -60s unaligned absolute of TCPReqQFullDoCookies
92 - units: cookies
93 - every: 10s
94 - warn: $this > 1
95 - crit: $this > (($status == $CRITICAL) ? (0) : (5))
96 - delay: up 10 down 5m multiplier 1.5 max 1h
97 - summary: System TCP SYN queue cookies
98 - info: Average number of sent SYN cookies due to the full TCP SYN queue over the last minute
99 - to: silent
62 + alarm: 1m_tcp_syn_queue_drops
63 + on: ip.tcp_syn_queue
64 + class: Workload
65 + type: System
66 + component: Network
67 +host labels: _os=linux
68 + lookup: average -60s unaligned absolute of TCPReqQFullDrop
69 + units: drops
70 + every: 10s
71 + warn: $this > 1
72 + crit: $this > (($status == $CRITICAL) ? (0) : (5))
73 + delay: up 10 down 5m multiplier 1.5 max 1h
74 + summary: System TCP SYN queue drops
75 + info: Average number of SYN requests was dropped due to the full TCP SYN queue over the last minute \
76 + (SYN cookies were not enabled)
77 + to: silent
78
79 + alarm: 1m_tcp_syn_queue_cookies
80 + on: ip.tcp_syn_queue
81 + class: Workload
82 + type: System
83 + component: Network
84 +host labels: _os=linux
85 + lookup: average -60s unaligned absolute of TCPReqQFullDoCookies
86 + units: cookies
87 + every: 10s
88 + warn: $this > 1
89 + crit: $this > (($status == $CRITICAL) ? (0) : (5))
90 + delay: up 10 down 5m multiplier 1.5 max 1h
91 + summary: System TCP SYN queue cookies
92 + info: Average number of sent SYN cookies due to the full TCP SYN queue over the last minute
93 + to: silent
src/health/health.d/tcp_mem.conf
+15 -17
@@ -1,4 +1,3 @@
1 -#
1 # check
2 # http://blog.tsunanet.net/2011/03/out-of-socket-memory.html
3 #
@@ -6,19 +5,18 @@
5 # and a critical when TCP is 90% of its upper memory limit
6 #
7
9 - alarm: tcp_memory
10 - on: ipv4.sockstat_tcp_mem
11 - class: Utilization
12 - type: System
13 -component: Network
14 - os: linux
15 - hosts: *
16 - calc: ${mem} * 100 / ${tcp_mem_high}
17 - units: %
18 - every: 10s
19 - warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} ))
20 - crit: ${mem} > (($status == $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
21 - delay: up 0 down 5m multiplier 1.5 max 1h
22 - summary: System TCP memory utilization
23 - info: TCP memory utilization
24 - to: silent
8 + alarm: tcp_memory
9 + on: ipv4.sockstat_tcp_mem
10 + class: Utilization
11 + type: System
12 + component: Network
13 +host labels: _os=linux
14 + calc: ${mem} * 100 / ${tcp_mem_high}
15 + units: %
16 + every: 10s
17 + warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} ))
18 + crit: ${mem} > (($status == $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
19 + delay: up 0 down 5m multiplier 1.5 max 1h
20 + summary: System TCP memory utilization
21 + info: TCP memory utilization
22 + to: silent
src/health/health.d/tcp_orphans.conf
+15 -18
@@ -1,5 +1,3 @@
1 -
2 -#
1 # check
2 # http://blog.tsunanet.net/2011/03/out-of-socket-memory.html
3 #
@@ -7,19 +5,18 @@
5 # so we alarm warning at 25% and critical at 50%
6 #
7
10 - alarm: tcp_orphans
11 - on: ipv4.sockstat_tcp_sockets
12 - class: Errors
13 - type: System
14 -component: Network
15 - os: linux
16 - hosts: *
17 - calc: ${orphan} * 100 / ${tcp_max_orphans}
18 - units: %
19 - every: 10s
20 - warn: $this > (($status >= $WARNING ) ? ( 20 ) : ( 25 ))
21 - crit: $this > (($status == $CRITICAL) ? ( 25 ) : ( 50 ))
22 - delay: up 0 down 5m multiplier 1.5 max 1h
23 - summary: System TCP orphan sockets utilization
24 - info: Orphan IPv4 TCP sockets utilization
25 - to: silent
8 + alarm: tcp_orphans
9 + on: ipv4.sockstat_tcp_sockets
10 + class: Errors
11 + type: System
12 + component: Network
13 +host labels: _os=linux
14 + calc: ${orphan} * 100 / ${tcp_max_orphans}
15 + units: %
16 + every: 10s
17 + warn: $this > (($status >= $WARNING ) ? ( 20 ) : ( 25 ))
18 + crit: $this > (($status == $CRITICAL) ? ( 25 ) : ( 50 ))
19 + delay: up 0 down 5m multiplier 1.5 max 1h
20 + summary: System TCP orphan sockets utilization
21 + info: Orphan IPv4 TCP sockets utilization
22 + to: silent
src/health/health.d/tcp_resets.conf
+55 -60
@@ -1,71 +1,66 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 # -----------------------------------------------------------------------------
4 # tcp resets this host sends
5
7 - alarm: 1m_ip_tcp_resets_sent
8 - on: ip.tcphandshake
9 - class: Errors
10 - type: System
11 -component: Network
12 - os: linux
13 - hosts: *
14 - lookup: average -1m at -10s unaligned absolute of OutRsts
15 - units: tcp resets/s
16 - every: 10s
17 - info: average number of sent TCP RESETS over the last minute
6 + alarm: 1m_ip_tcp_resets_sent
7 + on: ip.tcphandshake
8 + class: Errors
9 + type: System
10 + component: Network
11 +host labels: _os=linux
12 + lookup: average -1m at -10s unaligned absolute of OutRsts
13 + units: tcp resets/s
14 + every: 10s
15 + info: average number of sent TCP RESETS over the last minute
16
19 - alarm: 10s_ip_tcp_resets_sent
20 - on: ip.tcphandshake
21 - class: Errors
22 - type: System
23 -component: Network
24 - os: linux
25 - hosts: *
26 - lookup: average -10s unaligned absolute of OutRsts
27 - units: tcp resets/s
28 - every: 10s
29 - warn: $netdata.uptime.uptime > (1 * 60) AND $this > ((($1m_ip_tcp_resets_sent < 5)?(5):($1m_ip_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (10)))
30 - delay: up 20s down 60m multiplier 1.2 max 2h
31 - options: no-clear-notification
32 - summary: System TCP outbound resets
33 - info: Average number of sent TCP RESETS over the last 10 seconds. \
34 - This can indicate a port scan, \
35 - or that a service running on this host has crashed. \
36 - Netdata will not send a clear notification for this alarm.
37 - to: silent
17 + alarm: 10s_ip_tcp_resets_sent
18 + on: ip.tcphandshake
19 + class: Errors
20 + type: System
21 + component: Network
22 +host labels: _os=linux
23 + lookup: average -10s unaligned absolute of OutRsts
24 + units: tcp resets/s
25 + every: 10s
26 + warn: $netdata.uptime.uptime > (1 * 60) AND $this > ((($1m_ip_tcp_resets_sent < 5)?(5):($1m_ip_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (10)))
27 + delay: up 20s down 60m multiplier 1.2 max 2h
28 + options: no-clear-notification
29 + summary: System TCP outbound resets
30 + info: Average number of sent TCP RESETS over the last 10 seconds. \
31 + This can indicate a port scan, \
32 + or that a service running on this host has crashed. \
33 + Netdata will not send a clear notification for this alarm.
34 + to: silent
35
36 # -----------------------------------------------------------------------------
37 # tcp resets this host receives
38
42 - alarm: 1m_ip_tcp_resets_received
43 - on: ip.tcphandshake
44 - class: Errors
45 - type: System
46 -component: Network
47 - os: linux freebsd
48 - hosts: *
49 - lookup: average -1m at -10s unaligned absolute of AttemptFails
50 - units: tcp resets/s
51 - every: 10s
52 - info: average number of received TCP RESETS over the last minute
39 + alarm: 1m_ip_tcp_resets_received
40 + on: ip.tcphandshake
41 + class: Errors
42 + type: System
43 + component: Network
44 +host labels: _os=linux freebsd
45 + lookup: average -1m at -10s unaligned absolute of AttemptFails
46 + units: tcp resets/s
47 + every: 10s
48 + info: average number of received TCP RESETS over the last minute
49
54 - alarm: 10s_ip_tcp_resets_received
55 - on: ip.tcphandshake
56 - class: Errors
57 - type: System
58 -component: Network
59 - os: linux freebsd
60 - hosts: *
61 - lookup: average -10s unaligned absolute of AttemptFails
62 - units: tcp resets/s
63 - every: 10s
64 - warn: $netdata.uptime.uptime > (1 * 60) AND $this > ((($1m_ip_tcp_resets_received < 5)?(5):($1m_ip_tcp_resets_received)) * (($status >= $WARNING) ? (1) : (10)))
65 - delay: up 20s down 60m multiplier 1.2 max 2h
66 - options: no-clear-notification
67 - summary: System TCP inbound resets
68 - info: average number of received TCP RESETS over the last 10 seconds. \
69 - This can be an indication that a service this host needs has crashed. \
70 - Netdata will not send a clear notification for this alarm.
71 - to: silent
50 + alarm: 10s_ip_tcp_resets_received
51 + on: ip.tcphandshake
52 + class: Errors
53 + type: System
54 + component: Network
55 +host labels: _os=linux freebsd
56 + lookup: average -10s unaligned absolute of AttemptFails
57 + units: tcp resets/s
58 + every: 10s
59 + warn: $netdata.uptime.uptime > (1 * 60) AND $this > ((($1m_ip_tcp_resets_received < 5)?(5):($1m_ip_tcp_resets_received)) * (($status >= $WARNING) ? (1) : (10)))
60 + delay: up 20s down 60m multiplier 1.2 max 2h
61 + options: no-clear-notification
62 + summary: System TCP inbound resets
63 + info: average number of received TCP RESETS over the last 10 seconds. \
64 + This can be an indication that a service this host needs has crashed. \
65 + Netdata will not send a clear notification for this alarm.
66 + to: silent
src/health/health.d/timex.conf
+14 -15
@@ -1,18 +1,17 @@
1 -
1 # It can take several minutes before ntpd selects a server to synchronize with;
2 # try checking after 17 minutes (1024 seconds).
3
5 - alarm: system_clock_sync_state
6 - on: system.clock_sync_state
7 - os: linux
8 - class: Errors
9 - type: System
10 -component: Clock
11 - calc: $state
12 - units: synchronization state
13 - every: 10s
14 - warn: $system.uptime.uptime > 17 * 60 AND $this == 0
15 - delay: down 5m
16 - summary: System clock sync state
17 - info: When set to 0, the system kernel believes the system clock is not properly synchronized to a reliable server
18 - to: silent
4 + alarm: system_clock_sync_state
5 + on: system.clock_sync_state
6 + class: Errors
7 + type: System
8 + component: Clock
9 +host labels: _os=linux
10 + calc: $state
11 + units: synchronization state
12 + every: 10s
13 + warn: $system.uptime.uptime > 17 * 60 AND $this == 0
14 + delay: down 5m
15 + summary: System clock sync state
16 + info: When set to 0, the system kernel believes the system clock is not properly synchronized to a reliable server
17 + to: silent
src/health/health.d/udp_errors.conf
+28 -31
@@ -1,40 +1,37 @@
1 -
1 # you can disable an alarm notification by setting the 'to' line to: silent
2
3 # -----------------------------------------------------------------------------
4 # UDP receive buffer errors
5
7 - alarm: 1m_ipv4_udp_receive_buffer_errors
8 - on: ipv4.udperrors
9 - class: Errors
10 - type: System
11 -component: Network
12 - os: linux freebsd
13 - hosts: *
14 - lookup: average -1m unaligned absolute of RcvbufErrors
15 - units: errors
16 - every: 10s
17 - warn: $this > (($status >= $WARNING) ? (0) : (10))
18 - summary: System UDP receive buffer errors
19 - info: Average number of UDP receive buffer errors over the last minute
20 - delay: up 1m down 60m multiplier 1.2 max 2h
21 - to: silent
6 + alarm: 1m_ipv4_udp_receive_buffer_errors
7 + on: ipv4.udperrors
8 + class: Errors
9 + type: System
10 + component: Network
11 +host labels: _os=linux freebsd
12 + lookup: average -1m unaligned absolute of RcvbufErrors
13 + units: errors
14 + every: 10s
15 + warn: $this > (($status >= $WARNING) ? (0) : (10))
16 + summary: System UDP receive buffer errors
17 + info: Average number of UDP receive buffer errors over the last minute
18 + delay: up 1m down 60m multiplier 1.2 max 2h
19 + to: silent
20
21 # -----------------------------------------------------------------------------
22 # UDP send buffer errors
23
26 - alarm: 1m_ipv4_udp_send_buffer_errors
27 - on: ipv4.udperrors
28 - class: Errors
29 - type: System
30 -component: Network
31 - os: linux
32 - hosts: *
33 - lookup: average -1m unaligned absolute of SndbufErrors
34 - units: errors
35 - every: 10s
36 - warn: $this > (($status >= $WARNING) ? (0) : (10))
37 - summary: System UDP send buffer errors
38 - info: Average number of UDP send buffer errors over the last minute
39 - delay: up 1m down 60m multiplier 1.2 max 2h
40 - to: silent
24 + alarm: 1m_ipv4_udp_send_buffer_errors
25 + on: ipv4.udperrors
26 + class: Errors
27 + type: System
28 + component: Network
29 +host labels: _os=linux
30 + lookup: average -1m unaligned absolute of SndbufErrors
31 + units: errors
32 + every: 10s
33 + warn: $this > (($status >= $WARNING) ? (0) : (10))
34 + summary: System UDP send buffer errors
35 + info: Average number of UDP send buffer errors over the last minute
36 + delay: up 1m down 60m multiplier 1.2 max 2h
37 + to: silent
src/health/health.d/upsd.conf
-4
@@ -5,8 +5,6 @@
5 class: Utilization
6 type: Power Supply
7 component: UPS
8 - os: *
9 - hosts: *
8 lookup: average -10m unaligned of load
9 units: %
10 every: 1m
@@ -22,8 +20,6 @@ component: UPS
20 class: Errors
21 type: Power Supply
22 component: UPS
25 - os: *
26 - hosts: *
23 lookup: average -60s unaligned of charge
24 units: %
25 every: 60s
src/health/health.d/vsphere.conf
-4
@@ -8,7 +8,6 @@
8 class: Utilization
9 type: Virtual Machine
10 component: CPU
11 - hosts: *
11 lookup: average -10m unaligned match-names of used
12 units: %
13 every: 20s
@@ -24,7 +23,6 @@ component: CPU
23 class: Utilization
24 type: Virtual Machine
25 component: Memory
27 - hosts: *
26 calc: $used
27 units: %
28 every: 20s
@@ -42,7 +40,6 @@ component: Memory
40 class: Utilization
41 type: Virtual Machine
42 component: CPU
45 - hosts: *
43 lookup: average -10m unaligned match-names of used
44 units: %
45 every: 20s
@@ -58,7 +55,6 @@ component: CPU
55 class: Utilization
56 type: Virtual Machine
57 component: Memory
61 - hosts: *
58 calc: $used
59 units: %
60 every: 20s
src/health/health.d/windows.conf
-18
@@ -1,4 +1,3 @@
1 -
1 ## CPU
2
3 template: windows_10min_cpu_usage
@@ -6,8 +5,6 @@
5 class: Utilization
6 type: Windows
7 component: CPU
9 - os: *
10 - hosts: *
8 lookup: average -10m unaligned match-names of dpc,user,privileged,interrupt
9 units: %
10 every: 1m
@@ -18,7 +15,6 @@ component: CPU
15 info: Average CPU utilization over the last 10 minutes
16 to: silent
17
21 -
18 ## Memory
19
20 template: windows_ram_in_use
@@ -26,8 +22,6 @@ component: CPU
22 class: Utilization
23 type: Windows
24 component: Memory
29 - os: *
30 - hosts: *
25 calc: ($used) * 100 / ($used + $available)
26 units: %
27 every: 10s
@@ -38,7 +32,6 @@ component: Memory
32 info: Memory utilization
33 to: sysadmin
34
41 -
35 ## Network
36
37 template: windows_inbound_packets_discarded
@@ -46,8 +39,6 @@ component: Memory
39 class: Errors
40 type: Windows
41 component: Network
49 - os: *
50 - hosts: *
42 lookup: sum -10m unaligned absolute match-names of inbound
43 units: packets
44 every: 1m
@@ -62,8 +53,6 @@ component: Network
53 class: Errors
54 type: Windows
55 component: Network
65 - os: *
66 - hosts: *
56 lookup: sum -10m unaligned absolute match-names of outbound
57 units: packets
58 every: 1m
@@ -78,8 +67,6 @@ component: Network
67 class: Errors
68 type: Windows
69 component: Network
81 - os: *
82 - hosts: *
70 lookup: sum -10m unaligned absolute match-names of inbound
71 units: packets
72 every: 1m
@@ -94,8 +81,6 @@ component: Network
81 class: Errors
82 type: Windows
83 component: Network
97 - os: *
98 - hosts: *
84 lookup: sum -10m unaligned absolute match-names of outbound
85 units: packets
86 every: 1m
@@ -105,7 +90,6 @@ component: Network
90 info: Number of outbound errors for the network interface in the last 10 minutes
91 to: silent
92
108 -
93 ## Disk
94
95 template: windows_disk_in_use
@@ -113,8 +97,6 @@ component: Network
97 class: Utilization
98 type: Windows
99 component: Disk
116 - os: *
117 - hosts: *
100 calc: ($used) * 100 / ($used + $free)
101 units: %
102 every: 10s