Pulse extended memory statistics, now report glibc allocations (#19379)
* add malloc_info() to netdata * report number of arenas * arenas line chart
Costa Tsaousis committed
Jan 11, 2025 at 18:23 UTC
c54c40605a44c2283aa62cfbbad15c7491cb26a6
3 files changed
+187
CMakeLists.txt
+8
@@ -484,6 +484,14 @@ int main() {
484
}
485
" HAVE_C_MALLOC_TRIM)
486
487
+check_c_source_compiles("
488
+#include <malloc.h>
489
+int main() {
490
+ malloc_info(0, stdout);
491
+ return 0;
492
+}
493
+" HAVE_C_MALLOC_INFO)
494
+
495
check_c_source_compiles("
496
#define _GNU_SOURCE
497
#include <stdio.h>
packaging/cmake/config.cmake.h.in
+1
@@ -98,6 +98,7 @@
98
#cmakedefine HAVE_C__GENERIC
99
#cmakedefine HAVE_C_MALLOPT
100
#cmakedefine HAVE_C_MALLOC_TRIM
101
+#cmakedefine HAVE_C_MALLOC_INFO
102
#cmakedefine HAVE_SETNS
103
#cmakedefine HAVE_STRNDUP
104
#cmakedefine SSL_HAS_PENDING
src/daemon/pulse/pulse-daemon-memory.c
+178
@@ -9,6 +9,116 @@
9
10
struct netdata_buffers_statistics netdata_buffers_statistics = { 0 };
11
12
+#ifdef HAVE_C_MALLOC_INFO
13
+#include <malloc.h>
14
+
15
+// Helper function to find the last occurrence of a substring in a string
16
+static char *find_last(const char *haystack, const char *needle, size_t *found) {
17
+ *found = 0;
18
+
19
+ char *last = NULL;
20
+ char *current = strstr(haystack, needle);
21
+ while (current) {
22
+ (*found)++;
23
+ last = current;
24
+ current = strstr(current + 1, needle);
25
+ }
26
+ return last;
27
+}
28
+
29
+static bool parse_malloc_info(size_t *arenas, size_t *allocated_memory, size_t *used_fast, size_t *used_rest, size_t *used_mmap, size_t *unused_memory) {
30
+ int found = 0;
31
+
32
+ *arenas = 0;
33
+ *allocated_memory = 0;
34
+ *used_fast = 0;
35
+ *used_rest = 0;
36
+ *used_mmap = 0;
37
+ *unused_memory = 0;
38
+
39
+ // Buffer for malloc_info XML
40
+ char *buffer = NULL;
41
+ size_t size = 0;
42
+ FILE *meminfo = open_memstream(&buffer, &size);
43
+ if (!meminfo)
44
+ goto cleanup;
45
+
46
+ char *t = malloc(1024);
47
+
48
+ // Generate malloc_info XML
49
+ if(malloc_info(0, meminfo) != 0)
50
+ goto cleanup;
51
+
52
+ fflush(meminfo);
53
+ fclose(meminfo);
54
+ meminfo = NULL;
55
+
56
+ free(t);
57
+
58
+ if(!size || !buffer)
59
+ goto cleanup;
60
+
61
+ // make sure it is terminated
62
+ buffer[size - 1] = '\0';
63
+
64
+ // Find the last </heap>
65
+ char *last_heap_end = find_last(buffer, "</heap>", arenas);
66
+ if (!last_heap_end)
67
+ goto cleanup;
68
+
69
+ // Move past the last </heap>
70
+ char *summary_section = last_heap_end + strlen("</heap>");
71
+
72
+ // Parse the summary section using strstr
73
+ const char *fast_key = "<total type=\"fast\"";
74
+ const char *rest_key = "<total type=\"rest\"";
75
+ const char *mmap_key = "<total type=\"mmap\"";
76
+ const char *system_key = "<system type=\"current\"";
77
+ char *size_pos;
78
+
79
+ // Extract Used Fast
80
+ char *fast_pos = strstr(summary_section, fast_key);
81
+ if(!fast_pos || !(size_pos = strstr(fast_pos, "size=\"")))
82
+ goto cleanup;
83
+
84
+ *used_fast = strtoull(size_pos + 6, NULL, 10);
85
+ found++;
86
+
87
+ // Extract Used Rest
88
+ char *rest_pos = strstr(summary_section, rest_key);
89
+ if (!rest_pos || !(size_pos = strstr(rest_pos, "size=\"")))
90
+ goto cleanup;
91
+
92
+ *used_rest = strtoull(size_pos + 6, NULL, 10);
93
+ found++;
94
+
95
+ // Extract Used Mmap
96
+ char *mmap_pos = strstr(summary_section, mmap_key);
97
+ if (!mmap_pos || !(size_pos = strstr(mmap_pos, "size=\"")))
98
+ goto cleanup;
99
+
100
+ *used_mmap = strtoull(size_pos + 6, NULL, 10);
101
+ found++;
102
+
103
+ // Extract Allocated Memory
104
+ char *system_pos = strstr(summary_section, system_key);
105
+ if (!system_pos || !(size_pos = strstr(system_pos, "size=\"")))
106
+ goto cleanup;
107
+
108
+ *allocated_memory = strtoull(size_pos + 6, NULL, 10);
109
+ found++;
110
+
111
+ // Calculate Unused Memory
112
+ *unused_memory = *allocated_memory > (*used_fast + *used_rest + *used_mmap) ?
113
+ *allocated_memory - (*used_fast + *used_rest + *used_mmap) : 0;
114
+
115
+cleanup:
116
+ if(meminfo) fclose(meminfo);
117
+ if(buffer) free(buffer);
118
+ return found == 4;
119
+}
120
+#endif // HAVE_C_MALLOC_INFO
121
+
122
void pulse_daemon_memory_do(bool extended) {
123
{
124
static RRDSET *st_memory = NULL;
@@ -285,4 +395,72 @@ void pulse_daemon_memory_do(bool extended) {
395
396
// ----------------------------------------------------------------------------------------------------------------
397
398
+#ifdef HAVE_C_MALLOC_INFO
399
+ size_t glibc_arenas, glibc_allocated_memory, glibc_used_fast, glibc_used_rest, glibc_used_mmap, glibc_unused_memory;
400
+ if(parse_malloc_info(&glibc_arenas, &glibc_allocated_memory, &glibc_used_fast, &glibc_used_rest, &glibc_used_mmap, &glibc_unused_memory)) {
401
+ if (glibc_arenas) {
402
+ static RRDSET *st_arenas = NULL;
403
+ static RRDDIM *rd_arenas = NULL;
404
+
405
+ if (unlikely(!st_arenas)) {
406
+ st_arenas = rrdset_create_localhost(
407
+ "netdata",
408
+ "glibc_arenas",
409
+ NULL,
410
+ "Memory Usage",
411
+ NULL,
412
+ "Glibc Memory Arenas",
413
+ "bytes",
414
+ "netdata",
415
+ "pulse",
416
+ 130104,
417
+ localhost->rrd_update_every,
418
+ RRDSET_TYPE_LINE);
419
+
420
+ rd_arenas = rrddim_add(st_arenas, "arenas", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
421
+ }
422
+
423
+ // the sum of all these needs to be above at the total buffers calculation
424
+ rrddim_set_by_pointer(st_arenas, rd_arenas, (collected_number)glibc_arenas);
425
+ rrdset_done(st_arenas);
426
+ }
427
+
428
+ if (glibc_allocated_memory > 0 && glibc_used_fast + glibc_used_rest + glibc_used_mmap + glibc_unused_memory > 0) {
429
+ static RRDSET *st_malloc = NULL;
430
+ static RRDDIM *rd_unused = NULL;
431
+ static RRDDIM *rd_fast = NULL;
432
+ static RRDDIM *rd_rest = NULL;
433
+ static RRDDIM *rd_mmap = NULL;
434
+
435
+ if (unlikely(!st_malloc)) {
436
+ st_malloc = rrdset_create_localhost(
437
+ "netdata",
438
+ "glibc_memory",
439
+ NULL,
440
+ "Memory Usage",
441
+ NULL,
442
+ "Glibc Memory Usage",
443
+ "bytes",
444
+ "netdata",
445
+ "pulse",
446
+ 130105,
447
+ localhost->rrd_update_every,
448
+ RRDSET_TYPE_STACKED);
449
+
450
+ rd_unused = rrddim_add(st_malloc, "unused", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
451
+ rd_fast = rrddim_add(st_malloc, "fast", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
452
+ rd_rest = rrddim_add(st_malloc, "rest", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
453
+ rd_mmap = rrddim_add(st_malloc, "mmap", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
454
+ }
455
+
456
+ // the sum of all these needs to be above at the total buffers calculation
457
+ rrddim_set_by_pointer(st_malloc, rd_unused, (collected_number)glibc_unused_memory);
458
+ rrddim_set_by_pointer(st_malloc, rd_fast, (collected_number)glibc_used_fast);
459
+ rrddim_set_by_pointer(st_malloc, rd_rest, (collected_number)glibc_used_rest);
460
+ rrddim_set_by_pointer(st_malloc, rd_mmap, (collected_number)glibc_used_mmap);
461
+ rrdset_done(st_malloc);
462
+ }
463
+ }
464
+#endif
465
+
466
}