eBPF OOM kill tracking (#11470)
Uman Shahzad committed
Sep 16, 2021 at 02:47 UTC
67ddc2bca29d9121f28417a450b64193d84ff989
16 files changed
+325
-9
CMakeLists.txt
+2
@@ -500,6 +500,8 @@ set(EBPF_PROCESS_PLUGIN_FILES
500
collectors/ebpf.plugin/ebpf_mount.h
501
collectors/ebpf.plugin/ebpf_filesystem.c
502
collectors/ebpf.plugin/ebpf_filesystem.h
503
+ collectors/ebpf.plugin/ebpf_oomkill.c
504
+ collectors/ebpf.plugin/ebpf_oomkill.h
505
collectors/ebpf.plugin/ebpf_process.c
506
collectors/ebpf.plugin/ebpf_process.h
507
collectors/ebpf.plugin/ebpf_socket.c
Makefile.am
+2
@@ -304,6 +304,8 @@ EBPF_PLUGIN_FILES = \
304
collectors/ebpf.plugin/ebpf_hardirq.h \
305
collectors/ebpf.plugin/ebpf_mount.c \
306
collectors/ebpf.plugin/ebpf_mount.h \
307
+ collectors/ebpf.plugin/ebpf_oomkill.c \
308
+ collectors/ebpf.plugin/ebpf_oomkill.h \
309
collectors/ebpf.plugin/ebpf_process.c \
310
collectors/ebpf.plugin/ebpf_process.h \
311
collectors/ebpf.plugin/ebpf_socket.c \
collectors/ebpf.plugin/Makefile.am
+1
@@ -31,6 +31,7 @@ dist_ebpfconfig_DATA = \
31
ebpf.d/hardirq.conf \
32
ebpf.d/mount.conf \
33
ebpf.d/network.conf \
34
+ ebpf.d/oomkill.conf \
35
ebpf.d/process.conf \
36
ebpf.d/softirq.conf \
37
ebpf.d/sync.conf \
collectors/ebpf.plugin/README.md
+4
@@ -322,6 +322,10 @@ The eBPF collector enables and runs the following eBPF programs by default:
322
time spent servicing individual hardware interrupt requests (hard IRQs).
323
- `softirq`: This eBPF program creates charts that show information about
324
time spent servicing individual software interrupt requests (soft IRQs).
325
+- `oomkill`: This eBPF program creates a chart that shows OOM kills for all
326
+ applications recognized via the `apps.plugin` integration. Note that this
327
+ program will show application charts regardless of whether apps integration
328
+ is turned on or off.
329
330
You can also enable the following eBPF programs:
331
- `cachestat`: Netdata's eBPF data collector creates charts about the memory page cache. When the integration with
collectors/ebpf.plugin/ebpf.c
+26
-1
@@ -140,6 +140,11 @@ ebpf_module_t ebpf_modules[] = {
140
.optional = 0, .apps_routine = NULL, .maps = NULL,
141
.pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &softirq_config,
142
.config_file = NETDATA_SOFTIRQ_CONFIG_FILE},
143
+ { .thread_name = "oomkill", .config_name = "oomkill", .enabled = 0, .start_routine = ebpf_oomkill_thread,
144
+ .update_time = 1, .global_charts = 1, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY,
145
+ .optional = 0, .apps_routine = ebpf_oomkill_create_apps_charts, .maps = NULL,
146
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &oomkill_config,
147
+ .config_file = NETDATA_OOMKILL_CONFIG_FILE},
148
{ .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1,
149
.global_charts = 0, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY,
150
.optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0, .names = NULL,
@@ -730,6 +735,8 @@ void ebpf_print_help()
735
"\n"
736
" --net or -n Enable network viewer charts.\n"
737
"\n"
738
+ " --oomkill or -o Enable chart related to OOM kill tracking.\n"
739
+ "\n"
740
" --process or -p Enable charts related to process run time.\n"
741
"\n"
742
" --return or -r Run the collector in return mode.\n"
@@ -1168,6 +1175,13 @@ static void read_collector_values(int *disable_apps)
1175
started++;
1176
}
1177
1178
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "oomkill",
1179
+ CONFIG_BOOLEAN_YES);
1180
+ if (enabled) {
1181
+ ebpf_enable_chart(EBPF_MODULE_OOMKILL_IDX, *disable_apps);
1182
+ started++;
1183
+ }
1184
+
1185
if (!started){
1186
ebpf_enable_all_charts(*disable_apps);
1187
// Read network viewer section
@@ -1258,6 +1272,7 @@ static void parse_args(int argc, char **argv)
1272
{"hardirq", no_argument, 0, 'q' },
1273
{"mount", no_argument, 0, 'm' },
1274
{"net", no_argument, 0, 'n' },
1275
+ {"oomkill", no_argument, 0, 'o' },
1276
{"process", no_argument, 0, 'p' },
1277
{"return", no_argument, 0, 'r' },
1278
{"softirq", no_argument, 0, 't' },
@@ -1278,7 +1293,7 @@ static void parse_args(int argc, char **argv)
1293
}
1294
1295
while (1) {
1281
- int c = getopt_long(argc, argv, "hvgacdkieqmnprtswf", long_options, &option_index);
1296
+ int c = getopt_long(argc, argv, "hvgacdkieqmnoprtswf", long_options, &option_index);
1297
if (c == -1)
1298
break;
1299
@@ -1370,6 +1385,14 @@ static void parse_args(int argc, char **argv)
1385
ebpf_enable_chart(EBPF_MODULE_SOCKET_IDX, disable_apps);
1386
#ifdef NETDATA_INTERNAL_CHECKS
1387
info("EBPF enabling \"NET\" charts, because it was started with the option \"--net\" or \"-n\".");
1388
+#endif
1389
+ break;
1390
+ }
1391
+ case 'o': {
1392
+ enabled = 1;
1393
+ ebpf_enable_chart(EBPF_MODULE_OOMKILL_IDX, disable_apps);
1394
+#ifdef NETDATA_INTERNAL_CHECKS
1395
+ info("EBPF enabling \"oomkill\" chart, because it was started with the option \"--oomkill\" or \"-o\".");
1396
#endif
1397
break;
1398
}
@@ -1705,6 +1728,8 @@ int main(int argc, char **argv)
1728
NULL, NULL, ebpf_modules[EBPF_MODULE_HARDIRQ_IDX].start_routine},
1729
{"EBPF SOFTIRQ" , NULL, NULL, 1,
1730
NULL, NULL, ebpf_modules[EBPF_MODULE_SOFTIRQ_IDX].start_routine},
1731
+ {"EBPF OOMKILL" , NULL, NULL, 1,
1732
+ NULL, NULL, ebpf_modules[EBPF_MODULE_OOMKILL_IDX].start_routine},
1733
{NULL , NULL, NULL, 0,
1734
NULL, NULL, NULL}
1735
};
collectors/ebpf.plugin/ebpf.d.conf
+2
@@ -32,6 +32,7 @@
32
# `mount` : Monitor calls for syscalls mount and umount
33
# `filesystem`: Monitor calls for functions used to manipulate specific filesystems
34
# `hardirq` : Monitor latency of serving hardware interrupt requests (hard IRQs).
35
+# `oomkill` : This eBPF program creates a chart that shows which process got OOM killed and when.
36
# `process` : This eBPF program creates charts that show information about process life.
37
# `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
38
# bandwidth consumed by each.
@@ -48,6 +49,7 @@
49
filesystem = no
50
hardirq = yes
51
mount = yes
52
+ oomkill = yes
53
process = yes
54
socket = yes
55
softirq = yes
collectors/ebpf.plugin/ebpf.d/oomkill.conf
new
+7
@@ -0,0 +1,7 @@
1
+# The `ebpf load mode` option accepts the following values :
2
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
3
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
4
+# new charts for the return of these functions, such as errors.
5
+[global]
6
+ ebpf load mode = entry
7
+ update every = 1
collectors/ebpf.plugin/ebpf.h
+3
-1
@@ -86,7 +86,8 @@ enum ebpf_module_indexes {
86
EBPF_MODULE_MOUNT_IDX,
87
EBPF_MODULE_FD_IDX,
88
EBPF_MODULE_HARDIRQ_IDX,
89
- EBPF_MODULE_SOFTIRQ_IDX
89
+ EBPF_MODULE_SOFTIRQ_IDX,
90
+ EBPF_MODULE_OOMKILL_IDX
91
};
92
93
typedef struct ebpf_tracepoint {
@@ -225,6 +226,7 @@ extern uint32_t ebpf_enable_tracepoints(ebpf_tracepoint_t *tps);
226
#define EBPF_COMMON_DIMENSION_PACKETS "packets"
227
#define EBPF_COMMON_DIMENSION_FILES "files"
228
#define EBPF_COMMON_DIMENSION_MILLISECONDS "milliseconds"
229
+#define EBPF_COMMON_DIMENSION_KILLS "kills"
230
231
// Common variables
232
extern int debug_enabled;
collectors/ebpf.plugin/ebpf_apps.c
+27
@@ -909,6 +909,33 @@ static inline void del_pid_entry(pid_t pid)
909
all_pids_count--;
910
}
911
912
+/**
913
+ * Get command string associated with a PID.
914
+ * This can only safely be used when holding the `collect_data_mutex` lock.
915
+ *
916
+ * @param pid the pid to search the data.
917
+ * @param n the maximum amount of bytes to copy into dest.
918
+ * if this is greater than the size of the command, it is clipped.
919
+ * @param dest the target memory buffer to write the command into.
920
+ * @return -1 if the PID hasn't been scraped yet, 0 otherwise.
921
+ */
922
+int get_pid_comm(pid_t pid, size_t n, char *dest)
923
+{
924
+ struct pid_stat *stat;
925
+
926
+ stat = all_pids[pid];
927
+ if (unlikely(stat == NULL)) {
928
+ return -1;
929
+ }
930
+
931
+ if (unlikely(n > sizeof(stat->comm))) {
932
+ n = sizeof(stat->comm);
933
+ }
934
+
935
+ strncpyz(dest, stat->comm, n);
936
+ return 0;
937
+}
938
+
939
/**
940
* Cleanup variable from other threads
941
*
collectors/ebpf.plugin/ebpf_apps.h
+3
@@ -24,6 +24,7 @@
24
#include "ebpf_hardirq.h"
25
#include "ebpf_cachestat.h"
26
#include "ebpf_mount.h"
27
+#include "ebpf_oomkill.h"
28
#include "ebpf_softirq.h"
29
#include "ebpf_sync.h"
30
#include "ebpf_swap.h"
@@ -418,6 +419,8 @@ extern void cleanup_exited_pids();
419
420
extern int ebpf_read_hash_table(void *ep, int fd, uint32_t pid);
421
422
+extern int get_pid_comm(pid_t pid, size_t n, char *dest);
423
+
424
extern size_t read_processes_statistic_using_pid_on_target(ebpf_process_stat_t **ep,
425
int fd,
426
struct pid_on_target *pids);
collectors/ebpf.plugin/ebpf_hardirq.h
-1
@@ -69,6 +69,5 @@ typedef struct hardirq_static_val {
69
70
extern struct config hardirq_config;
71
extern void *ebpf_hardirq_thread(void *ptr);
72
-extern void ebpf_hardirq_create_apps_charts(struct ebpf_module *em, void *ptr);
72
73
#endif /* NETDATA_EBPF_HARDIRQ_H */
collectors/ebpf.plugin/ebpf_oomkill.c
new
+217
@@ -0,0 +1,217 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "ebpf.h"
4
+#include "ebpf_oomkill.h"
5
+
6
+struct config oomkill_config = { .first_section = NULL,
7
+ .last_section = NULL,
8
+ .mutex = NETDATA_MUTEX_INITIALIZER,
9
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
10
+ .rwlock = AVL_LOCK_INITIALIZER } };
11
+
12
+#define OOMKILL_MAP_KILLCNT 0
13
+static ebpf_local_maps_t oomkill_maps[] = {
14
+ {
15
+ .name = "tbl_oomkill",
16
+ .internal_input = NETDATA_OOMKILL_MAX_ENTRIES,
17
+ .user_input = 0,
18
+ .type = NETDATA_EBPF_MAP_STATIC,
19
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
20
+ },
21
+ /* end */
22
+ {
23
+ .name = NULL,
24
+ .internal_input = 0,
25
+ .user_input = 0,
26
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
27
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
28
+ }
29
+};
30
+
31
+static ebpf_data_t oomkill_data;
32
+
33
+static ebpf_tracepoint_t oomkill_tracepoints[] = {
34
+ {.enabled = false, .class = "oom", .event = "mark_victim"},
35
+ /* end */
36
+ {.enabled = false, .class = NULL, .event = NULL}
37
+};
38
+
39
+static struct bpf_link **probe_links = NULL;
40
+static struct bpf_object *objects = NULL;
41
+
42
+/**
43
+ * Clean up the main thread.
44
+ *
45
+ * @param ptr thread data.
46
+ */
47
+static void oomkill_cleanup(void *ptr)
48
+{
49
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
50
+ if (!em->enabled) {
51
+ return;
52
+ }
53
+
54
+ if (probe_links) {
55
+ struct bpf_program *prog;
56
+ size_t i = 0 ;
57
+ bpf_object__for_each_program(prog, objects) {
58
+ bpf_link__destroy(probe_links[i]);
59
+ i++;
60
+ }
61
+ bpf_object__close(objects);
62
+ }
63
+}
64
+
65
+static void oomkill_write_data()
66
+{
67
+ // the first `i` entries of `keys` will contain the currently active PIDs
68
+ // in the eBPF map.
69
+ uint32_t i = 0;
70
+ int32_t keys[NETDATA_OOMKILL_MAX_ENTRIES] = {0};
71
+
72
+ uint32_t curr_key = 0;
73
+ uint32_t key = 0;
74
+ int mapfd = oomkill_maps[OOMKILL_MAP_KILLCNT].map_fd;
75
+ while (bpf_map_get_next_key(mapfd, &curr_key, &key) == 0) {
76
+ curr_key = key;
77
+
78
+ keys[i] = key;
79
+ i += 1;
80
+
81
+ // delete this key now that we've recorded its existence. there's no
82
+ // race here, as the same PID will only get OOM killed once.
83
+ int test = bpf_map_delete_elem(mapfd, &key);
84
+ if (unlikely(test < 0)) {
85
+ // since there's only 1 thread doing these deletions, it should be
86
+ // impossible to get this condition.
87
+ error("key unexpectedly not available for deletion.");
88
+ }
89
+ }
90
+
91
+ // for each app, see if it was OOM killed. record as 1 if so otherwise 0.
92
+ struct target *w;
93
+ for (w = apps_groups_root_target; w != NULL; w = w->next) {
94
+ if (likely(w->exposed && w->processes)) {
95
+ bool was_oomkilled = false;
96
+ struct pid_on_target *pids = w->root_pid;
97
+ while (pids) {
98
+ uint32_t j;
99
+ for (j = 0; j < i; j++) {
100
+ if (pids->pid == keys[j]) {
101
+ was_oomkilled = true;
102
+ // set to 0 so we consider it "done".
103
+ keys[j] = 0;
104
+ goto write_dim;
105
+ }
106
+ }
107
+ pids = pids->next;
108
+ }
109
+
110
+ write_dim:;
111
+ write_chart_dimension(w->name, was_oomkilled);
112
+ }
113
+ }
114
+
115
+ // for any remaining keys for which we couldn't find a group, this could be
116
+ // for various reasons, but the primary one is that the PID has not yet
117
+ // been picked up by the process thread when parsing the proc filesystem.
118
+ // since it's been OOM killed, it will never be parsed in the future, so
119
+ // we have no choice but to dump it into `other`.
120
+ uint32_t j;
121
+ uint32_t rem_count = 0;
122
+ for (j = 0; j < i; j++) {
123
+ int32_t key = keys[j];
124
+ if (key != 0) {
125
+ rem_count += 1;
126
+ }
127
+ }
128
+ if (rem_count > 0) {
129
+ write_chart_dimension("other", rem_count);
130
+ }
131
+}
132
+
133
+/**
134
+* Main loop for this collector.
135
+*/
136
+static void oomkill_collector(ebpf_module_t *em)
137
+{
138
+ UNUSED(em);
139
+
140
+ // loop and read until ebpf plugin is closed.
141
+ while (!close_ebpf_plugin) {
142
+ pthread_mutex_lock(&collect_data_mutex);
143
+ pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex);
144
+ pthread_mutex_lock(&lock);
145
+
146
+ // write everything from the ebpf map.
147
+ write_begin_chart(NETDATA_APPS_FAMILY, "oomkills");
148
+ oomkill_write_data();
149
+ write_end_chart();
150
+
151
+ pthread_mutex_unlock(&lock);
152
+ pthread_mutex_unlock(&collect_data_mutex);
153
+ }
154
+}
155
+
156
+/**
157
+ * Create apps charts
158
+ *
159
+ * Call ebpf_create_chart to create the charts on apps submenu.
160
+ *
161
+ * @param em a pointer to the structure with the default values.
162
+ */
163
+void ebpf_oomkill_create_apps_charts(struct ebpf_module *em, void *ptr)
164
+{
165
+ UNUSED(em);
166
+
167
+ struct target *root = ptr;
168
+ ebpf_create_charts_on_apps("oomkills",
169
+ "OOM kills",
170
+ EBPF_COMMON_DIMENSION_KILLS,
171
+ "mem",
172
+ NETDATA_EBPF_CHART_TYPE_STACKED,
173
+ 20020,
174
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
175
+ root, NETDATA_EBPF_MODULE_NAME_OOMKILL);
176
+}
177
+
178
+/**
179
+ * OOM kill tracking thread.
180
+ *
181
+ * @param ptr a `ebpf_module_t *`.
182
+ * @return always NULL.
183
+ */
184
+void *ebpf_oomkill_thread(void *ptr)
185
+{
186
+ netdata_thread_cleanup_push(oomkill_cleanup, ptr);
187
+
188
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
189
+ em->maps = oomkill_maps;
190
+
191
+ fill_ebpf_data(&oomkill_data);
192
+
193
+ if (!em->enabled) {
194
+ goto endoomkill;
195
+ }
196
+
197
+ if (ebpf_update_kernel(&oomkill_data)) {
198
+ goto endoomkill;
199
+ }
200
+
201
+ if (ebpf_enable_tracepoints(oomkill_tracepoints) == 0) {
202
+ em->enabled = CONFIG_BOOLEAN_NO;
203
+ goto endoomkill;
204
+ }
205
+
206
+ probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, oomkill_data.map_fd);
207
+ if (!probe_links) {
208
+ goto endoomkill;
209
+ }
210
+
211
+ oomkill_collector(em);
212
+
213
+endoomkill:
214
+ netdata_thread_cleanup_pop(1);
215
+
216
+ return NULL;
217
+}
collectors/ebpf.plugin/ebpf_oomkill.h
new
+27
@@ -0,0 +1,27 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EBPF_OOMKILL_H
4
+#define NETDATA_EBPF_OOMKILL_H 1
5
+
6
+/*****************************************************************
7
+ * copied from kernel-collectors repo, with modifications needed
8
+ * for inclusion here.
9
+ *****************************************************************/
10
+
11
+#define NETDATA_OOMKILL_MAX_ENTRIES 64
12
+
13
+typedef uint8_t oomkill_ebpf_val_t;
14
+
15
+/*****************************************************************
16
+ * below this is eBPF plugin-specific code.
17
+ *****************************************************************/
18
+
19
+#define NETDATA_EBPF_MODULE_NAME_OOMKILL "oomkill"
20
+#define NETDATA_OOMKILL_SLEEP_MS 650000ULL
21
+#define NETDATA_OOMKILL_CONFIG_FILE "oomkill.conf"
22
+
23
+extern struct config oomkill_config;
24
+extern void *ebpf_oomkill_thread(void *ptr);
25
+extern void ebpf_oomkill_create_apps_charts(struct ebpf_module *em, void *ptr);
26
+
27
+#endif /* NETDATA_EBPF_OOMKILL_H */
collectors/ebpf.plugin/ebpf_softirq.h
-1
@@ -30,6 +30,5 @@ typedef struct sofirq_val {
30
31
extern struct config softirq_config;
32
extern void *ebpf_softirq_thread(void *ptr);
33
-extern void ebpf_softirq_create_apps_charts(struct ebpf_module *em, void *ptr);
33
34
#endif /* NETDATA_EBPF_SOFTIRQ_H */
packaging/ebpf.checksums
+3
-3
@@ -1,3 +1,3 @@
1
-8080f817431a7a039ec322ea2d6cdd28ad69c0ee469e05b4f15115df054a3d4e netdata-kernel-collector-glibc-v0.7.9.1.tar.xz
2
-049453d48678e751966633d6e16b138edf78c1d4ff70bb3d7e679837bf7b1227 netdata-kernel-collector-musl-v0.7.9.1.tar.xz
3
-a62bcbcf86981eb5e3f46d06c2d84bc82b6d664b61a610b56fe0718a22368bb0 netdata-kernel-collector-static-v0.7.9.1.tar.xz
1
+826c2b1e17df316d5667ae78cab0fbc7aacbb65479afe601175a29fdb6499853 netdata-kernel-collector-glibc-v0.7.10.tar.xz
2
+e7d74be56d80c8a999bf0428c059e8dbfbcaed62eeb247f99970c4b01808ab6d netdata-kernel-collector-musl-v0.7.10.tar.xz
3
+f2dbf0402fcaf172b09bacf1e60dbb2ccd69cea5bc4af2a8f66be2e0e37d7025 netdata-kernel-collector-static-v0.7.10.tar.xz
packaging/ebpf.version
+1
-2
@@ -1,2 +1 @@
1
-v0.7.9.1
2
-
1
+v0.7.10