Add a chart for out of memory kills (#10880)
Co-authored-by: Joel Hans <joel.g.hans@gmail.com> Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>
Vladimir Kobal committed
Apr 14, 2021 at 12:17 UTC
9f63bef4db69ee67735078570af62dfed208e510
4 files changed
+105
-17
collectors/all.h
+3
-2
@@ -80,8 +80,9 @@
80
// Memory Section - 1xxx
81
82
#define NETDATA_CHART_PRIO_MEM_SYSTEM_AVAILABLE 1010
83
-#define NETDATA_CHART_PRIO_MEM_SYSTEM_COMMITTED 1020
84
-#define NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS 1030
83
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_OOM_KILL 1020
84
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_COMMITTED 1030
85
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS 1040
86
#define NETDATA_CHART_PRIO_MEM_KERNEL 1100
87
#define NETDATA_CHART_PRIO_MEM_SLAB 1200
88
#define NETDATA_CHART_PRIO_MEM_HUGEPAGES 1250
collectors/proc.plugin/README.md
+22
@@ -291,6 +291,28 @@ each state.
291
292
`schedstat filename to monitor`, `cpuidle name filename to monitor`, and `cpuidle time filename to monitor` in the `[plugin:proc:/proc/stat]` configuration section
293
294
+## Monitoring memory
295
+
296
+### Monitored memory metrics
297
+
298
+- Amount of memory swapped in/out
299
+- Amount of memory paged from/to disk
300
+- Number of memory page faults
301
+- Number of out of memory kills
302
+- Number of NUMA events
303
+
304
+### Configuration
305
+
306
+```conf
307
+[plugin:proc:/proc/vmstat]
308
+ filename to monitor = /proc/vmstat
309
+ swap i/o = auto
310
+ disk i/o = yes
311
+ memory page faults = yes
312
+ out of memory kills = yes
313
+ system-wide numa metric summary = auto
314
+```
315
+
316
## Monitoring Network Interfaces
317
318
### Monitored network interface metrics
collectors/proc.plugin/proc_vmstat.c
+68
-15
@@ -4,11 +4,13 @@
4
5
#define PLUGIN_PROC_MODULE_VMSTAT_NAME "/proc/vmstat"
6
7
+#define OOM_KILL_STRING "oom_kill"
8
+
9
int do_proc_vmstat(int update_every, usec_t dt) {
10
(void)dt;
11
12
static procfile *ff = NULL;
11
- static int do_swapio = -1, do_io = -1, do_pgfaults = -1, do_numa = -1;
13
+ static int do_swapio = -1, do_io = -1, do_pgfaults = -1, do_oom_kill = -1, do_numa = -1;
14
static int has_numa = -1;
15
16
static ARL_BASE *arl_base = NULL;
@@ -27,11 +29,25 @@ int do_proc_vmstat(int update_every, usec_t dt) {
29
static unsigned long long pgpgout = 0ULL;
30
static unsigned long long pswpin = 0ULL;
31
static unsigned long long pswpout = 0ULL;
32
+ static unsigned long long oom_kill = 0ULL;
33
+
34
+ if(unlikely(!ff)) {
35
+ char filename[FILENAME_MAX + 1];
36
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/vmstat");
37
+ ff = procfile_open(config_get("plugin:proc:/proc/vmstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
38
+ if(unlikely(!ff)) return 1;
39
+ }
40
+
41
+ ff = procfile_readall(ff);
42
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
43
+
44
+ size_t lines = procfile_lines(ff), l;
45
46
if(unlikely(!arl_base)) {
47
do_swapio = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "swap i/o", CONFIG_BOOLEAN_AUTO);
33
- do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", 1);
34
- do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", 1);
48
+ do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", CONFIG_BOOLEAN_YES);
49
+ do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", CONFIG_BOOLEAN_YES);
50
+ do_oom_kill = config_get_boolean("plugin:proc:/proc/vmstat", "out of memory kills", CONFIG_BOOLEAN_AUTO);
51
do_numa = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "system-wide numa metric summary", CONFIG_BOOLEAN_AUTO);
52
53
@@ -43,6 +59,20 @@ int do_proc_vmstat(int update_every, usec_t dt) {
59
arl_expect(arl_base, "pswpin", &pswpin);
60
arl_expect(arl_base, "pswpout", &pswpout);
61
62
+ int has_oom_kill = 0;
63
+
64
+ for (l = 0; l < lines; l++) {
65
+ if (!strcmp(procfile_lineword(ff, l, 0), OOM_KILL_STRING)) {
66
+ has_oom_kill = 1;
67
+ break;
68
+ }
69
+ }
70
+
71
+ if (has_oom_kill)
72
+ arl_expect(arl_base, OOM_KILL_STRING, &oom_kill);
73
+ else
74
+ do_oom_kill = CONFIG_BOOLEAN_NO;
75
+
76
if(do_numa == CONFIG_BOOLEAN_YES || (do_numa == CONFIG_BOOLEAN_AUTO &&
77
(get_numa_node_count() >= 2 ||
78
netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
@@ -66,18 +96,6 @@ int do_proc_vmstat(int update_every, usec_t dt) {
96
}
97
}
98
69
- if(unlikely(!ff)) {
70
- char filename[FILENAME_MAX + 1];
71
- snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/vmstat");
72
- ff = procfile_open(config_get("plugin:proc:/proc/vmstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
73
- if(unlikely(!ff)) return 1;
74
- }
75
-
76
- ff = procfile_readall(ff);
77
- if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
78
-
79
- size_t lines = procfile_lines(ff), l;
80
-
99
arl_begin(arl_base);
100
for(l = 0; l < lines ;l++) {
101
size_t words = procfile_linewords(ff, l);
@@ -193,6 +211,41 @@ int do_proc_vmstat(int update_every, usec_t dt) {
211
rrdset_done(st_pgfaults);
212
}
213
214
+ // --------------------------------------------------------------------
215
+
216
+ if (do_oom_kill == CONFIG_BOOLEAN_YES ||
217
+ (do_oom_kill == CONFIG_BOOLEAN_AUTO && (oom_kill || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
218
+ static RRDSET *st_oom_kill = NULL;
219
+ static RRDDIM *rd_oom_kill = NULL;
220
+
221
+ do_oom_kill = CONFIG_BOOLEAN_YES;
222
+
223
+ if(unlikely(!st_oom_kill)) {
224
+ st_oom_kill = rrdset_create_localhost(
225
+ "mem"
226
+ , "oom_kill"
227
+ , NULL
228
+ , "system"
229
+ , NULL
230
+ , "Out of Memory Kills"
231
+ , "kills/s"
232
+ , PLUGIN_PROC_NAME
233
+ , PLUGIN_PROC_MODULE_VMSTAT_NAME
234
+ , NETDATA_CHART_PRIO_MEM_SYSTEM_OOM_KILL
235
+ , update_every
236
+ , RRDSET_TYPE_LINE
237
+ );
238
+
239
+ rrdset_flag_set(st_oom_kill, RRDSET_FLAG_DETAIL);
240
+
241
+ rd_oom_kill = rrddim_add(st_oom_kill, "kills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
242
+ }
243
+ else rrdset_next(st_oom_kill);
244
+
245
+ rrddim_set_by_pointer(st_oom_kill, rd_oom_kill, oom_kill);
246
+ rrdset_done(st_oom_kill);
247
+ }
248
+
249
// --------------------------------------------------------------------
250
251
// Ondemand criteria for NUMA. Since this won't change at run time, we
health/health.d/ram.conf
+12
@@ -37,6 +37,18 @@
37
info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
38
to: sysadmin
39
40
+ alarm: oom_kill
41
+ on: mem.oom_kill
42
+ os: linux
43
+ hosts: *
44
+ lookup: sum -1m unaligned
45
+ units: kills
46
+ every: 10s
47
+ warn: $this > 0
48
+ delay: down 5m
49
+ info: number of out of memory kills in the last minute
50
+ to: sysadmin
51
+
52
## FreeBSD
53
alarm: ram_in_use
54
on: system.ram