procfile: more comfortable initial settings and faster/fewer reallocs (#12791)
Costa Tsaousis committed
May 2, 2022 at 14:33 UTC
43b9fdc213ee043c38e248d9ff404ae02cae480d
3 files changed
+53
-51
libnetdata/libnetdata.c
+30
-41
@@ -31,6 +31,8 @@ const char *program_version = VERSION;
31
// routines.
32
33
#ifdef NETDATA_LOG_ALLOCATIONS
34
+#warning NETDATA_LOG_ALLOCATIONS ENABLED - set log_thread_memory_allocations=1 on any thread to log all its allocations - or use log_allocations() to log them on demand
35
+
36
static __thread struct memory_statistics {
37
volatile ssize_t malloc_calls_made;
38
volatile ssize_t calloc_calls_made;
@@ -44,23 +46,14 @@ static __thread struct memory_statistics {
46
47
__thread size_t log_thread_memory_allocations = 0;
48
47
-static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) {
49
+inline void log_allocations_int(const char *file, const char *function, const unsigned long line) {
50
static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
51
50
- fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n",
51
- netdata_thread_tag(),
52
- log_thread_memory_allocations,
53
- line, file, function,
54
- type, size
55
- );
56
-
57
- fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
52
+ fprintf(stderr, "%s MEMORY ALLOCATIONS: (%04lu@%s:%s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
53
netdata_thread_tag(),
59
- log_thread_memory_allocations,
54
line, file, function,
55
(memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
56
(memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
63
- type,
57
memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
58
memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
59
memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
@@ -79,12 +72,12 @@ static inline void mmap_accounting(size_t size) {
72
}
73
74
void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
82
- if(log_thread_memory_allocations) {
83
- memory_statistics.memory_calls_made++;
84
- memory_statistics.malloc_calls_made++;
85
- memory_statistics.allocated_memory += size;
86
- print_allocations(file, function, line, "malloc()", size);
87
- }
75
+ memory_statistics.memory_calls_made++;
76
+ memory_statistics.malloc_calls_made++;
77
+ memory_statistics.allocated_memory += size;
78
+
79
+ if(log_thread_memory_allocations)
80
+ log_allocations_int(file, function, line);
81
82
size_t *n = (size_t *)malloc(sizeof(size_t) + size);
83
if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size);
@@ -95,12 +88,11 @@ void *mallocz_int(const char *file, const char *function, const unsigned long li
88
void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
89
size = nmemb * size;
90
98
- if(log_thread_memory_allocations) {
99
- memory_statistics.memory_calls_made++;
100
- memory_statistics.calloc_calls_made++;
101
- memory_statistics.allocated_memory += size;
102
- print_allocations(file, function, line, "calloc()", size);
103
- }
91
+ memory_statistics.memory_calls_made++;
92
+ memory_statistics.calloc_calls_made++;
93
+ memory_statistics.allocated_memory += size;
94
+ if(log_thread_memory_allocations)
95
+ log_allocations_int(file, function, line);
96
97
size_t *n = (size_t *)calloc(1, sizeof(size_t) + size);
98
if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size);
@@ -118,12 +110,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l
110
n = realloc(n, sizeof(size_t) + size);
111
if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size);
112
121
- if(log_thread_memory_allocations) {
122
- memory_statistics.memory_calls_made++;
123
- memory_statistics.realloc_calls_made++;
124
- memory_statistics.allocated_memory += (size - old_size);
125
- print_allocations(file, function, line, "realloc()", size - old_size);
126
- }
113
+ memory_statistics.memory_calls_made++;
114
+ memory_statistics.realloc_calls_made++;
115
+ memory_statistics.allocated_memory += (size - old_size);
116
+ if(log_thread_memory_allocations)
117
+ log_allocations_int(file, function, line);
118
119
*n = size;
120
return (void *)&n[1];
@@ -132,12 +123,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l
123
char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
124
size_t size = strlen(s) + 1;
125
135
- if(log_thread_memory_allocations) {
136
- memory_statistics.memory_calls_made++;
137
- memory_statistics.strdup_calls_made++;
138
- memory_statistics.allocated_memory += size;
139
- print_allocations(file, function, line, "strdup()", size);
140
- }
126
+ memory_statistics.memory_calls_made++;
127
+ memory_statistics.strdup_calls_made++;
128
+ memory_statistics.allocated_memory += size;
129
+ if(log_thread_memory_allocations)
130
+ log_allocations_int(file, function, line);
131
132
size_t *n = (size_t *)malloc(sizeof(size_t) + size);
133
if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size);
@@ -155,12 +145,11 @@ void freez_int(const char *file, const char *function, const unsigned long line,
145
n--;
146
size_t size = *n;
147
158
- if(log_thread_memory_allocations) {
159
- memory_statistics.memory_calls_made++;
160
- memory_statistics.free_calls_made++;
161
- memory_statistics.allocated_memory -= size;
162
- print_allocations(file, function, line, "free()", size);
163
- }
148
+ memory_statistics.memory_calls_made++;
149
+ memory_statistics.free_calls_made++;
150
+ memory_statistics.allocated_memory -= size;
151
+ if(log_thread_memory_allocations)
152
+ log_allocations_int(file, function, line);
153
154
free(n);
155
}
libnetdata/libnetdata.h
+3
@@ -233,12 +233,15 @@ extern __thread size_t log_thread_memory_allocations;
233
#define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size)
234
#define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size)
235
#define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr)
236
+#define log_allocations() log_allocations_int(__FILE__, __FUNCTION__, __LINE__)
237
238
extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s);
239
extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size);
240
extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size);
241
extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size);
242
extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr);
243
+extern void log_allocations_int(const char *file, const char *function, const unsigned long line);
244
+
245
#else // NETDATA_LOG_ALLOCATIONS
246
extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL;
247
extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;
libnetdata/procfile/procfile.c
+20
-10
@@ -4,9 +4,9 @@
4
5
#define PF_PREFIX "PROCFILE"
6
7
-#define PFWORDS_INCREASE_STEP 200
8
-#define PFLINES_INCREASE_STEP 10
9
-#define PROCFILE_INCREMENT_BUFFER 512
7
+#define PFWORDS_INCREASE_STEP 2000
8
+#define PFLINES_INCREASE_STEP 200
9
+#define PROCFILE_INCREMENT_BUFFER 4096
10
11
int procfile_open_flags = O_RDONLY;
12
@@ -48,9 +48,12 @@ static inline void pfwords_add(procfile *ff, char *str) {
48
pfwords *fw = ff->words;
49
if(unlikely(fw->len == fw->size)) {
50
// debug(D_PROCFILE, PF_PREFIX ": expanding words");
51
+ size_t minimum = PFWORDS_INCREASE_STEP;
52
+ size_t optimal = fw->size / 2;
53
+ size_t wanted = (optimal > minimum)?optimal:minimum;
54
52
- ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *));
53
- fw->size += PFWORDS_INCREASE_STEP;
55
+ ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *));
56
+ fw->size += wanted;
57
}
58
59
fw->words[fw->len++] = str;
@@ -90,9 +93,12 @@ static inline size_t *pflines_add(procfile *ff) {
93
pflines *fl = ff->lines;
94
if(unlikely(fl->len == fl->size)) {
95
// debug(D_PROCFILE, PF_PREFIX ": expanding lines");
96
+ size_t minimum = PFLINES_INCREASE_STEP;
97
+ size_t optimal = fl->size / 2;
98
+ size_t wanted = (optimal > minimum)?optimal:minimum;
99
94
- ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline));
95
- fl->size += PFLINES_INCREASE_STEP;
100
+ ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline));
101
+ fl->size += wanted;
102
}
103
104
ffline *ffl = &fl->lines[fl->len++];
@@ -272,9 +278,13 @@ procfile *procfile_readall(procfile *ff) {
278
ssize_t x = ff->size - s;
279
280
if(unlikely(!x)) {
275
- debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", procfile_filename(ff));
276
- ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER);
277
- ff->size += PROCFILE_INCREMENT_BUFFER;
281
+ size_t minimum = PROCFILE_INCREMENT_BUFFER;
282
+ size_t optimal = ff->size / 2;
283
+ size_t wanted = (optimal > minimum)?optimal:minimum;
284
+
285
+ debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted);
286
+ ff = reallocz(ff, sizeof(procfile) + ff->size + wanted);
287
+ ff->size += wanted;
288
}
289
290
debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s));