added support for re-usable procfile, to eliminate the repeating memory allocations on apps.plugin
Costa Tsaousis (ktsaou) committed
Apr 2, 2015 at 23:11 UTC
e8b3c968037dff4a35de75272998333ac382e51d
3 files changed
+73
-29
src/plugins.d/apps_plugin.c
+19
-15
@@ -197,15 +197,17 @@ char *strdup_debug(const char *file, int line, const char *function, const char
197
// ----------------------------------------------------------------------------
198
// helper functions
199
200
+procfile *ff = NULL;
201
+
202
long get_processors(void) {
203
int processors = 0;
204
203
- procfile *ff = procfile_open("/proc/stat", "");
205
+ ff = procfile_reopen(ff, "/proc/stat", "");
206
if(!ff) return 1;
207
208
ff = procfile_readall(ff);
209
if(!ff) {
208
- procfile_close(ff);
210
+ // procfile_close(ff);
211
return 1;
212
}
213
@@ -218,26 +220,26 @@ long get_processors(void) {
220
processors--;
221
if(processors < 1) processors = 1;
222
221
- procfile_close(ff);
223
+ // procfile_close(ff);
224
return processors;
225
}
226
227
long get_pid_max(void) {
228
long mpid = 32768;
229
228
- procfile *ff = procfile_open("/proc/sys/kernel/pid_max", "");
230
+ ff = procfile_reopen(ff, "/proc/sys/kernel/pid_max", "");
231
if(!ff) return mpid;
232
233
ff = procfile_readall(ff);
234
if(!ff) {
233
- procfile_close(ff);
235
+ // procfile_close(ff);
236
return mpid;
237
}
238
239
mpid = atol(procfile_lineword(ff, 0, 0));
240
if(mpid) mpid = 32768;
241
240
- procfile_close(ff);
242
+ // procfile_close(ff);
243
return mpid;
244
}
245
@@ -608,12 +610,12 @@ int read_proc_pid_stat(struct pid_stat *p) {
610
611
snprintf(filename, FILENAME_MAX, "/proc/%d/stat", p->pid);
612
611
- procfile *ff = procfile_open(filename, "");
613
+ ff = procfile_reopen(ff, filename, "");
614
if(!ff) return 1;
615
616
ff = procfile_readall(ff);
617
if(!ff) {
616
- procfile_close(ff);
618
+ // procfile_close(ff);
619
return 1;
620
}
621
@@ -688,7 +690,7 @@ int read_proc_pid_stat(struct pid_stat *p) {
690
691
if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: VALUES: %s utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu, threads=%d\n", p->comm, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt, p->num_threads);
692
691
- procfile_close(ff);
693
+ // procfile_close(ff);
694
return 0;
695
}
696
@@ -697,12 +699,12 @@ int read_proc_pid_statm(struct pid_stat *p) {
699
700
snprintf(filename, FILENAME_MAX, "/proc/%d/statm", p->pid);
701
700
- procfile *ff = procfile_open(filename, "");
702
+ ff = procfile_reopen(ff, filename, "");
703
if(!ff) return 1;
704
705
ff = procfile_readall(ff);
706
if(!ff) {
705
- procfile_close(ff);
707
+ // procfile_close(ff);
708
return 1;
709
}
710
@@ -716,7 +718,7 @@ int read_proc_pid_statm(struct pid_stat *p) {
718
p->statm_data = strtoull(procfile_lineword(ff, 0, 5), NULL, 10);
719
p->statm_dirty = strtoull(procfile_lineword(ff, 0, 6), NULL, 10);
720
719
- procfile_close(ff);
721
+ // procfile_close(ff);
722
return 0;
723
}
724
@@ -725,12 +727,12 @@ int read_proc_pid_io(struct pid_stat *p) {
727
728
snprintf(filename, FILENAME_MAX, "/proc/%d/io", p->pid);
729
728
- procfile *ff = procfile_open(filename, ":");
730
+ ff = procfile_reopen(ff, filename, ":");
731
if(!ff) return 1;
732
733
ff = procfile_readall(ff);
734
if(!ff) {
733
- procfile_close(ff);
735
+ // procfile_close(ff);
736
return 1;
737
}
738
@@ -744,7 +746,7 @@ int read_proc_pid_io(struct pid_stat *p) {
746
p->io_storage_bytes_written = strtoull(procfile_lineword(ff, 5, 1), NULL, 10);
747
p->io_cancelled_write_bytes = strtoull(procfile_lineword(ff, 6, 1), NULL, 10);
748
747
- procfile_close(ff);
749
+ // procfile_close(ff);
750
return 0;
751
}
752
@@ -1893,6 +1895,8 @@ int main(int argc, char **argv)
1895
1896
info("apps.plugin: starting...");
1897
1898
+ procfile_adaptive_initial_allocation = 1;
1899
+
1900
unsigned long started_t = time(NULL), current_t;
1901
Hertz = get_hertz();
1902
pid_max = get_pid_max();
src/procfile.c
+45
-12
@@ -20,10 +20,21 @@
20
21
#define PF_PREFIX "PROCFILE"
22
23
+#define PFWORDS_INCREASE_STEP 200
24
+#define PFLINES_INCREASE_STEP 10
25
+#define PROCFILE_INCREMENT_BUFFER 512
26
+
27
+int procfile_adaptive_initial_allocation = 0;
28
+
29
+// if adaptive allocation is set, these store the
30
+// max values we have seen so far
31
+uint32_t procfile_max_lines = PFLINES_INCREASE_STEP;
32
+uint32_t procfile_max_words = PFWORDS_INCREASE_STEP;
33
+size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER;
34
+
35
// ----------------------------------------------------------------------------
36
// An array of words
37
26
-#define PFWORDS_INCREASE_STEP 200
38
39
pfwords *pfwords_add(pfwords *fw, char *str) {
40
// debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str);
@@ -49,11 +60,13 @@ pfwords *pfwords_add(pfwords *fw, char *str) {
60
pfwords *pfwords_new(void) {
61
// debug(D_PROCFILE, PF_PREFIX ": initializing words");
62
52
- pfwords *new = malloc(sizeof(pfwords) + PFWORDS_INCREASE_STEP * sizeof(char *));
63
+ uint32_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP;
64
+
65
+ pfwords *new = malloc(sizeof(pfwords) + size * sizeof(char *));
66
if(!new) return NULL;
67
68
new->len = 0;
56
- new->size = PFWORDS_INCREASE_STEP;
69
+ new->size = size;
70
return new;
71
}
72
@@ -72,8 +85,6 @@ void pfwords_free(pfwords *fw) {
85
// ----------------------------------------------------------------------------
86
// An array of lines
87
75
-#define PFLINES_INCREASE_STEP 10
76
-
88
pflines *pflines_add(pflines *fl, uint32_t first_word) {
89
// debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word);
90
@@ -99,11 +110,13 @@ pflines *pflines_add(pflines *fl, uint32_t first_word) {
110
pflines *pflines_new(void) {
111
// debug(D_PROCFILE, PF_PREFIX ": initializing lines");
112
102
- pflines *new = malloc(sizeof(pflines) + PFLINES_INCREASE_STEP * sizeof(ffline));
113
+ uint32_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFLINES_INCREASE_STEP;
114
+
115
+ pflines *new = malloc(sizeof(pflines) + size * sizeof(ffline));
116
if(!new) return NULL;
117
118
new->len = 0;
106
- new->size = PFLINES_INCREASE_STEP;
119
+ new->size = size;
120
return new;
121
}
122
@@ -123,9 +136,6 @@ void pflines_free(pflines *fl) {
136
// ----------------------------------------------------------------------------
137
// The procfile
138
126
-#define PROCFILE_INITIAL_BUFFER 256
127
-#define PROCFILE_INCREMENT_BUFFER 512
128
-
139
#define PF_CHAR_IS_SEPARATOR 0
140
#define PF_CHAR_IS_NEWLINE 1
141
#define PF_CHAR_IS_WORD 2
@@ -264,6 +274,12 @@ procfile *procfile_readall(procfile *ff) {
274
275
ff = procfile_parser(ff);
276
277
+ if(procfile_adaptive_initial_allocation) {
278
+ if(ff->len > procfile_max_allocation) procfile_max_allocation = ff->len;
279
+ if(ff->lines->len > procfile_max_lines) procfile_max_lines = ff->lines->len;
280
+ if(ff->words->len > procfile_max_words) procfile_max_words = ff->words->len;
281
+ }
282
+
283
debug(D_PROCFILE, "File '%s' updated.", ff->filename);
284
return ff;
285
}
@@ -277,7 +293,8 @@ procfile *procfile_open(const char *filename, const char *separators) {
293
return NULL;
294
}
295
280
- procfile *ff = malloc(sizeof(procfile) + PROCFILE_INITIAL_BUFFER);
296
+ size_t size = (procfile_adaptive_initial_allocation) ? procfile_max_allocation : PROCFILE_INCREMENT_BUFFER;
297
+ procfile *ff = malloc(sizeof(procfile) + size);
298
if(!ff) {
299
error(PF_PREFIX ": Cannot allocate memory for file '%s'. Reason: %s", filename, strerror(errno));
300
close(fd);
@@ -288,7 +305,7 @@ procfile *procfile_open(const char *filename, const char *separators) {
305
ff->filename[FILENAME_MAX] = '\0';
306
307
ff->fd = fd;
291
- ff->size = PROCFILE_INITIAL_BUFFER;
308
+ ff->size = size;
309
ff->len = 0;
310
311
ff->lines = pflines_new();
@@ -315,6 +332,22 @@ procfile *procfile_open(const char *filename, const char *separators) {
332
return ff;
333
}
334
335
+procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators) {
336
+ if(!ff) return procfile_open(filename, separators);
337
+
338
+ if(ff->fd != -1) close(ff->fd);
339
+
340
+ ff->fd = open(filename, O_RDONLY, 0666);
341
+ if(ff->fd == -1) {
342
+ procfile_close(ff);
343
+ return NULL;
344
+ }
345
+
346
+ strncpy(ff->filename, filename, FILENAME_MAX);
347
+ ff->filename[FILENAME_MAX] = '\0';
348
+
349
+ return ff;
350
+}
351
352
// ----------------------------------------------------------------------------
353
// example parsing of procfile data
src/procfile.h
+9
-2
@@ -58,8 +58,8 @@ typedef struct {
58
typedef struct {
59
char filename[FILENAME_MAX + 1];
60
int fd; // the file desriptor
61
- ssize_t len; // the bytes we have placed into data
62
- ssize_t size; // the bytes we have allocated for data
61
+ size_t len; // the bytes we have placed into data
62
+ size_t size; // the bytes we have allocated for data
63
pflines *lines;
64
pfwords *words;
65
char separators[256];
@@ -75,11 +75,18 @@ extern procfile *procfile_readall(procfile *ff);
75
// open a /proc or /sys file
76
extern procfile *procfile_open(const char *filename, const char *separators);
77
78
+// re-open a file
79
+// the separators argument is only used if ff == NULL
80
+extern procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators);
81
+
82
// example walk-through a procfile parsed file
83
extern void procfile_print(procfile *ff);
84
85
// ----------------------------------------------------------------------------
86
87
+// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far
88
+extern int procfile_adaptive_initial_allocation;
89
+
90
// return the number of lines present
91
#define procfile_lines(ff) (ff->lines->len)
92