feat(health): add system_reboot_detection alarm (#19624)
* feat(health): add system_reboot_detection alarm * add a note for ram mode nodes
Ilya Mashchenko committed
Feb 12, 2025 at 13:04 UTC
0b33fcac1757175d8d5fc91255045ab8b4baa4c2
2 files changed
+35
-15
src/health/health.d/reboot.conf
new
+35
@@ -0,0 +1,35 @@
1
+# you can disable an alarm notification by setting the 'to' line to: silent
2
+
3
+ alarm: system_post_update_reboot_status
4
+ on: system.post_update_reboot_status
5
+ class: Errors
6
+ type: System
7
+ component: OS
8
+host labels: _os=linux
9
+ calc: $required
10
+ units: status
11
+ every: 10s
12
+ warn: $this == 1
13
+ summary: System requires reboot after package updates
14
+ info: System requires reboot after package updates
15
+ to: sysadmin
16
+
17
+
18
+# Detects system reboots by monitoring uptime
19
+# - Uses 2-hour history (15 min offset) to identify real reboots vs new systems
20
+# - Alerts when uptime is under 10 minutes
21
+# Note: Will not work on nodes with DB mode RAM as they have no history after reboot
22
+
23
+ alarm: system_reboot_detection
24
+ on: system.uptime
25
+ class: Errors
26
+ type: System
27
+ component: OS
28
+ lookup: max -2h at -15m unaligned
29
+ calc: ($this != nan AND $uptime < 60 * 10) ? 1 : 0
30
+ units: status
31
+ every: 1m
32
+ warn: $this == 1
33
+ summary: System has rebooted within the last 10 minutes
34
+ info: System has rebooted within the last 10 minutes
35
+ to: silent
src/health/health.d/reboot_required.conf
deleted
-15
@@ -1,15 +0,0 @@
1
-# you can disable an alarm notification by setting the 'to' line to: silent
2
-
3
- alarm: system_post_update_reboot_status
4
- on: system.post_update_reboot_status
5
- class: Errors
6
- type: System
7
- component: OS
8
-host labels: _os=linux
9
- calc: $required
10
- units: status
11
- every: 10s
12
- warn: $this == 1
13
- summary: System requires reboot after package updates
14
- info: System requires reboot after package updates
15
- to: sysadmin