eBPF OOMKills adjust and fixes. (#17734)
thiagoftsm committed
May 31, 2024 at 17:16 UTC
a440fb39062b3f454595457eae9f1f8100b92cd7
3 files changed
+70
-60
src/collectors/ebpf.plugin/ebpf.h
-1
@@ -338,7 +338,6 @@ void ebpf_pid_file(char *filename, size_t length);
338
#define EBPF_COMMON_DIMENSION_PACKETS "packets"
339
#define EBPF_COMMON_DIMENSION_FILES "files"
340
#define EBPF_COMMON_DIMENSION_MILLISECONDS "milliseconds"
341
-#define EBPF_COMMON_DIMENSION_KILLS "kills"
341
342
#define EBPF_CHART_ALGORITHM_ABSOLUTE "absolute"
343
#define EBPF_CHART_ALGORITHM_INCREMENTAL "incremental"
src/collectors/ebpf.plugin/ebpf_oomkill.c
+67
-58
@@ -40,11 +40,11 @@ static ebpf_tracepoint_t oomkill_tracepoints[] = {
40
{.enabled = false, .class = NULL, .event = NULL}
41
};
42
43
-static netdata_publish_syscall_t oomkill_publish_aggregated = {.name = "oomkill", .dimension = "oomkill",
43
+static netdata_publish_syscall_t oomkill_publish_aggregated = {.name = "kills", .dimension = "kills",
44
.algorithm = "absolute",
45
.next = NULL};
46
47
-static void ebpf_create_specific_oomkill_charts(char *type, int update_every);
47
+static void ebpf_obsolete_specific_oomkill_charts(char *type, int update_every);
48
49
/**
50
* Obsolete services
@@ -58,11 +58,11 @@ static void ebpf_obsolete_oomkill_services(ebpf_module_t *em, char *id)
58
ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
59
id,
60
NETDATA_OOMKILL_CHART,
61
- "OOM kills. This chart is provided by eBPF plugin.",
62
- EBPF_COMMON_DIMENSION_KILLS,
61
+ "Systemd service OOM kills.",
62
+ EBPF_OOMKILL_UNIT_KILLS,
63
NETDATA_EBPF_MEMORY_GROUP,
64
- NETDATA_EBPF_CHART_TYPE_LINE,
65
- NULL,
64
+ NETDATA_EBPF_CHART_TYPE_STACKED,
65
+ NETDATA_CGROUP_OOMKILLS_CONTEXT,
66
20191,
67
em->update_every);
68
}
@@ -86,7 +86,7 @@ static inline void ebpf_obsolete_oomkill_cgroup_charts(ebpf_module_t *em)
86
continue;
87
}
88
89
- ebpf_create_specific_oomkill_charts(ect->name, em->update_every);
89
+ ebpf_obsolete_specific_oomkill_charts(ect->name, em->update_every);
90
}
91
pthread_mutex_unlock(&mutex_cgroup_shm);
92
}
@@ -109,13 +109,13 @@ static void ebpf_obsolete_oomkill_apps(ebpf_module_t *em)
109
110
ebpf_write_chart_obsolete(NETDATA_APP_FAMILY,
111
w->clean_name,
112
- "_app_oomkill",
113
- "OOM kills.",
114
- EBPF_COMMON_DIMENSION_KILLS,
112
+ NETDATA_OOMKILL_CHART,
113
+ "Processes OOM kills.",
114
+ EBPF_OOMKILL_UNIT_KILLS,
115
NETDATA_EBPF_MEMORY_GROUP,
116
NETDATA_EBPF_CHART_TYPE_STACKED,
117
"ebpf.app_oomkill",
118
- 20020,
118
+ 20072,
119
update_every);
120
121
w->charts_created &= ~(1<<EBPF_MODULE_OOMKILL_IDX);
@@ -164,8 +164,10 @@ static void oomkill_write_data(int32_t *keys, uint32_t total)
164
{
165
// for each app, see if it was OOM killed. record as 1 if so otherwise 0.
166
struct ebpf_target *w;
167
+ uint32_t used_pid = 0;
168
+ pthread_mutex_lock(&collect_data_mutex);
169
for (w = apps_groups_root_target; w != NULL; w = w->next) {
168
- if (unlikely(!(w->charts_created & (1<<EBPF_MODULE_OOMKILL_IDX))))
170
+ if (unlikely(!(w->charts_created & (1 << EBPF_MODULE_OOMKILL_IDX))))
171
continue;
172
173
bool was_oomkilled = false;
@@ -175,6 +177,7 @@ static void oomkill_write_data(int32_t *keys, uint32_t total)
177
uint32_t j;
178
for (j = 0; j < total; j++) {
179
if (pids->pid == keys[j]) {
180
+ used_pid++;
181
was_oomkilled = true;
182
// set to 0 so we consider it "done".
183
keys[j] = 0;
@@ -185,27 +188,24 @@ static void oomkill_write_data(int32_t *keys, uint32_t total)
188
}
189
}
190
write_dim:
188
- ebpf_write_begin_chart(NETDATA_APP_FAMILY, w->clean_name, "_ebpf_oomkill");
189
- write_chart_dimension(EBPF_COMMON_DIMENSION_KILLS, was_oomkilled);
191
+ ebpf_write_begin_chart(NETDATA_APP_FAMILY, w->clean_name, NETDATA_OOMKILL_CHART);
192
+ write_chart_dimension(oomkill_publish_aggregated.dimension, was_oomkilled);
193
ebpf_write_end_chart();
194
}
195
193
- // for any remaining keys for which we couldn't find a group, this could be
194
- // for various reasons, but the primary one is that the PID has not yet
195
- // been picked up by the process thread when parsing the proc filesystem.
196
- // since it's been OOM killed, it will never be parsed in the future, so
197
- // we have no choice but to dump it into `other`.
198
- uint32_t j;
199
- uint32_t rem_count = 0;
200
- for (j = 0; j < total; j++) {
201
- int32_t key = keys[j];
202
- if (key != 0) {
203
- rem_count += 1;
204
- }
205
- }
206
- if (rem_count > 0) {
207
- write_chart_dimension("other", rem_count);
196
+ if (total != used_pid) {
197
+ // for any remaining keys for which we couldn't find a group, this could be
198
+ // for various reasons, but the primary one is that the PID has not yet
199
+ // been picked up by the process thread when parsing the proc filesystem.
200
+ // since it's been OOM killed, it will never be parsed in the future, so
201
+ // we have no choice but to dump it into `other`.
202
+ uint32_t rem_count = total - used_pid;
203
+ ebpf_write_begin_chart(NETDATA_APP_FAMILY, "other", NETDATA_OOMKILL_CHART);
204
+ write_chart_dimension(oomkill_publish_aggregated.dimension, rem_count);
205
+ ebpf_write_end_chart();
206
}
207
+
208
+ pthread_mutex_unlock(&collect_data_mutex);
209
}
210
211
/**
@@ -218,8 +218,8 @@ write_dim:
218
*/
219
static void ebpf_create_specific_oomkill_charts(char *type, int update_every)
220
{
221
- ebpf_create_chart(type, NETDATA_OOMKILL_CHART, "OOM kills. This chart is provided by eBPF plugin.",
222
- EBPF_COMMON_DIMENSION_KILLS, NETDATA_EBPF_MEMORY_GROUP,
221
+ ebpf_create_chart(type, NETDATA_OOMKILL_CHART, "Cgroup OOM kills.",
222
+ EBPF_OOMKILL_UNIT_KILLS, NETDATA_EBPF_MEMORY_GROUP,
223
NETDATA_CGROUP_OOMKILLS_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
224
NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5600,
225
ebpf_create_global_dimension,
@@ -236,17 +236,17 @@ static void ebpf_create_specific_oomkill_charts(char *type, int update_every)
236
static void ebpf_create_systemd_oomkill_charts(int update_every)
237
{
238
static ebpf_systemd_args_t data_oom = {
239
- .title = "OOM kills. This chart is provided by eBPF plugin.",
240
- .units = EBPF_COMMON_DIMENSION_KILLS,
239
+ .title = "Systemd service OOM kills.",
240
+ .units = EBPF_OOMKILL_UNIT_KILLS,
241
.family = NETDATA_EBPF_MEMORY_GROUP,
242
.charttype = NETDATA_EBPF_CHART_TYPE_STACKED,
243
.order = 20191,
244
.algorithm = EBPF_CHART_ALGORITHM_INCREMENTAL,
245
- .context = NETDATA_EBPF_MODULE_NAME_OOMKILL,
246
- .module = NETDATA_EBPF_MODULE_NAME_SWAP,
245
+ .context = NETDATA_CGROUP_OOMKILLS_CONTEXT,
246
+ .module = NETDATA_EBPF_MODULE_NAME_OOMKILL,
247
.update_every = 0,
248
.suffix = NETDATA_OOMKILL_CHART,
249
- .dimension = "oom"
249
+ .dimension = "kills"
250
};
251
252
if (!data_oom.update_every)
@@ -277,7 +277,7 @@ static void ebpf_send_systemd_oomkill_charts()
277
continue;
278
}
279
ebpf_write_begin_chart(NETDATA_SERVICE_FAMILY, ect->name, NETDATA_OOMKILL_CHART);
280
- write_chart_dimension("oom", (long long) ect->oomkill);
280
+ write_chart_dimension(oomkill_publish_aggregated.dimension, (long long) ect->oomkill);
281
ect->oomkill = 0;
282
ebpf_write_end_chart();
283
}
@@ -294,7 +294,7 @@ static void ebpf_send_systemd_oomkill_charts()
294
static void ebpf_send_specific_oomkill_data(char *type, int value)
295
{
296
ebpf_write_begin_chart(type, NETDATA_OOMKILL_CHART, "");
297
- write_chart_dimension(oomkill_publish_aggregated.name, (long long)value);
297
+ write_chart_dimension(oomkill_publish_aggregated.dimension, (long long)value);
298
ebpf_write_end_chart();
299
}
300
@@ -308,8 +308,8 @@ static void ebpf_send_specific_oomkill_data(char *type, int value)
308
*/
309
static void ebpf_obsolete_specific_oomkill_charts(char *type, int update_every)
310
{
311
- ebpf_write_chart_obsolete(type, NETDATA_OOMKILL_CHART, "", "OOM kills. This chart is provided by eBPF plugin.",
312
- EBPF_COMMON_DIMENSION_KILLS, NETDATA_EBPF_MEMORY_GROUP,
311
+ ebpf_write_chart_obsolete(type, NETDATA_OOMKILL_CHART, "", "Cgroup OOM kills.",
312
+ EBPF_OOMKILL_UNIT_KILLS, NETDATA_EBPF_MEMORY_GROUP,
313
NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_OOMKILLS_CONTEXT,
314
NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5600, update_every);
315
}
@@ -340,11 +340,13 @@ void ebpf_oomkill_send_cgroup_data(int update_every)
340
ect->flags |= NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART;
341
}
342
343
- if (ect->flags & NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART && ect->updated) {
344
- ebpf_send_specific_oomkill_data(ect->name, ect->oomkill);
345
- } else {
346
- ebpf_obsolete_specific_oomkill_charts(ect->name, update_every);
347
- ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART;
343
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART) {
344
+ if (ect->updated) {
345
+ ebpf_send_specific_oomkill_data(ect->name, ect->oomkill);
346
+ } else {
347
+ ebpf_obsolete_specific_oomkill_charts(ect->name, update_every);
348
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART;
349
+ }
350
}
351
}
352
@@ -369,6 +371,7 @@ static uint32_t oomkill_read_data(int32_t *keys)
371
uint32_t curr_key = 0;
372
uint32_t key = 0;
373
int mapfd = oomkill_maps[OOMKILL_MAP_KILLCNT].map_fd;
374
+ uint32_t limit = NETDATA_OOMKILL_MAX_ENTRIES -1;
375
while (bpf_map_get_next_key(mapfd, &curr_key, &key) == 0) {
376
curr_key = key;
377
@@ -383,6 +386,8 @@ static uint32_t oomkill_read_data(int32_t *keys)
386
// impossible to get this condition.
387
netdata_log_error("key unexpectedly not available for deletion.");
388
}
389
+ if (i > limit)
390
+ break;
391
}
392
393
return i;
@@ -399,6 +404,7 @@ static uint32_t oomkill_read_data(int32_t *keys)
404
static void ebpf_update_oomkill_cgroup(int32_t *keys, uint32_t total)
405
{
406
ebpf_cgroup_target_t *ect;
407
+ pthread_mutex_lock(&mutex_cgroup_shm);
408
for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
409
ect->oomkill = 0;
410
struct pid_on_target2 *pids;
@@ -413,6 +419,7 @@ static void ebpf_update_oomkill_cgroup(int32_t *keys, uint32_t total)
419
}
420
}
421
}
422
+ pthread_mutex_unlock(&mutex_cgroup_shm);
423
}
424
425
/**
@@ -470,18 +477,18 @@ static void oomkill_collector(ebpf_module_t *em)
477
stats[NETDATA_CONTROLLER_PID_TABLE_ADD] += (uint64_t) count;
478
stats[NETDATA_CONTROLLER_PID_TABLE_DEL] += (uint64_t) count;
479
473
- pthread_mutex_lock(&lock);
474
- if (cgroups && shm_ebpf_cgroup.header) {
480
+ if (cgroups && shm_ebpf_cgroup.header)
481
ebpf_update_oomkill_cgroup(keys, count);
476
- // write everything from the ebpf map.
482
+
483
+ netdata_apps_integration_flags_t apps = em->apps_charts;
484
+ pthread_mutex_lock(&lock);
485
+ // write everything from the ebpf map.
486
+ if (cgroups && shm_ebpf_cgroup.header)
487
ebpf_oomkill_send_cgroup_data(update_every);
478
- }
488
480
- if (em->apps_charts & NETDATA_EBPF_APPS_FLAG_CHART_CREATED) {
481
- pthread_mutex_lock(&collect_data_mutex);
489
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
490
oomkill_write_data(keys, count);
483
- pthread_mutex_unlock(&collect_data_mutex);
484
- }
491
+
492
pthread_mutex_unlock(&lock);
493
494
running_time = ebpf_update_oomkill_period(running_time, em);
@@ -506,18 +513,20 @@ void ebpf_oomkill_create_apps_charts(struct ebpf_module *em, void *ptr)
513
514
ebpf_write_chart_cmd(NETDATA_APP_FAMILY,
515
w->clean_name,
509
- "_ebpf_oomkill",
510
- "OOM kills.",
511
- EBPF_COMMON_DIMENSION_KILLS,
516
+ NETDATA_OOMKILL_CHART,
517
+ "Processes OOM kills.",
518
+ EBPF_OOMKILL_UNIT_KILLS,
519
NETDATA_EBPF_MEMORY_GROUP,
520
NETDATA_EBPF_CHART_TYPE_STACKED,
521
"app.ebpf_oomkill",
522
20072,
523
update_every,
524
NETDATA_EBPF_MODULE_NAME_OOMKILL);
518
- ebpf_create_chart_labels("app_group", w->name, RRDLABEL_SRC_AUTO);
525
+ ebpf_create_chart_labels("app_group", w->clean_name, RRDLABEL_SRC_AUTO);
526
ebpf_commit_label();
520
- fprintf(stdout, "DIMENSION kills '' %s 1 1\n", ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]);
527
+ fprintf(stdout, "DIMENSION '%s' '' %s 1 1\n",
528
+ oomkill_publish_aggregated.dimension,
529
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]);
530
531
w->charts_created |= 1<<EBPF_MODULE_OOMKILL_IDX;
532
}
src/collectors/ebpf.plugin/ebpf_oomkill.h
+3
-1
@@ -22,7 +22,9 @@ typedef uint8_t oomkill_ebpf_val_t;
22
#define NETDATA_EBPF_MODULE_NAME_OOMKILL "oomkill"
23
#define NETDATA_OOMKILL_CONFIG_FILE "oomkill.conf"
24
25
-#define NETDATA_OOMKILL_CHART "oomkills"
25
+#define NETDATA_OOMKILL_CHART "_ebpf_oomkill"
26
+
27
+#define EBPF_OOMKILL_UNIT_KILLS "kills/s"
28
29
// Contexts
30
#define NETDATA_CGROUP_OOMKILLS_CONTEXT "cgroup.oomkills"