@cryptotaxi247 / netdata-1 / commits / 7311d8368

code cleanup; interpolation had issues - solved all them; added --unittest option to verify interpolation works as expected

Costa Tsaousis (ktsaou) committed May 24, 2014 at 04:49 UTC 7311d8368b4205d2d485a72b758e7392694b3ae8
2 files changed +499 -355
apps_plugin.c
+241 -234
@@ -30,6 +30,104 @@ long processors = 1;
30 long pid_max = 32768;
31 int debug = 0;
32
33 +int update_every = 1;
34 +unsigned long long file_counter = 0;
35 +
36 +#define PROC_BUFFER 4096
37 +
38 +
39 +// ----------------------------------------------------------------------------
40 +// helper functions
41 +
42 +unsigned long long usecdiff(struct timeval *now, struct timeval *last) {
43 + return ((((now->tv_sec * 1000000ULL) + now->tv_usec) - ((last->tv_sec * 1000000ULL) + last->tv_usec)));
44 +}
45 +
46 +char *trim(char *s)
47 +{
48 + // skip leading spaces
49 + while(*s && isspace(*s)) s++;
50 + if(!*s) return NULL;
51 +
52 + // skip tailing spaces
53 + int c = strlen(s) - 1;
54 + while(c >= 0 && isspace(s[c])) {
55 + s[c] = '\0';
56 + c--;
57 + }
58 + if(c < 0) return NULL;
59 + if(!*s) return NULL;
60 + return s;
61 +}
62 +
63 +unsigned long simple_hash(const char *name)
64 +{
65 + int i, len = strlen(name);
66 + unsigned long hash = 0;
67 +
68 + for(i = 0; i < len ;i++) hash += (i * name[i]) + i + name[i];
69 +
70 + return hash;
71 +}
72 +
73 +long get_processors(void)
74 +{
75 + char buffer[1025], *s;
76 + int processors = 0;
77 +
78 + FILE *fp = fopen("/proc/stat", "r");
79 + if(!fp) return 1;
80 +
81 + while((s = fgets(buffer, 1024, fp))) {
82 + if(strncmp(buffer, "cpu", 3) == 0) processors++;
83 + }
84 + fclose(fp);
85 + processors--;
86 + if(processors < 1) processors = 1;
87 + return processors;
88 +}
89 +
90 +long get_pid_max(void)
91 +{
92 + char buffer[1025], *s;
93 + long mpid = 32768;
94 +
95 + FILE *fp = fopen("/proc/sys/kernel/pid_max", "r");
96 + if(!fp) return 1;
97 +
98 + s = fgets(buffer, 1024, fp);
99 + if(s) mpid = atol(buffer);
100 + fclose(fp);
101 + if(mpid < 32768) mpid = 32768;
102 + return mpid;
103 +}
104 +
105 +unsigned long long get_hertz(void)
106 +{
107 + unsigned long long hz = 1;
108 +
109 +#ifdef _SC_CLK_TCK
110 + if((hz = sysconf(_SC_CLK_TCK)) > 0) {
111 + return hz;
112 + }
113 +#endif
114 +
115 +#ifdef HZ
116 + hz = (unsigned long long)HZ; /* <asm/param.h> */
117 +#else
118 + /* If 32-bit or big-endian (not Alpha or ia64), assume HZ is 100. */
119 + hz = (sizeof(long)==sizeof(int) || htons(999)==999) ? 100UL : 1024UL;
120 +#endif
121 +
122 + fprintf(stderr, "apps.plugin: ERROR: unknown HZ value. Assuming %llu.\n", hz);
123 + return hz;
124 +}
125 +
126 +
127 +// ----------------------------------------------------------------------------
128 +// target
129 +// target is the point to aggregate a process tree values
130 +
131 struct target {
132 char compare[MAX_COMPARE_NAME + 1];
133 char id[MAX_NAME + 1];
@@ -99,9 +197,8 @@ struct target {
197 struct target *next;
198 } *target_root = NULL, *default_target = NULL;
199
102 -int update_every = 1;
103 -unsigned long long file_counter = 0;
104 -
200 +// find or create a target
201 +// there are targets that are just agregated to other target (the second argument)
202 struct target *get_target(const char *id, struct target *target)
203 {
204 const char *nid = id;
@@ -132,23 +229,7 @@ struct target *get_target(const char *id, struct target *target)
229 return w;
230 }
231
135 -char *trim(char *s)
136 -{
137 - // skip leading spaces
138 - while(*s && isspace(*s)) s++;
139 - if(!*s) return NULL;
140 -
141 - // skip tailing spaces
142 - int c = strlen(s) - 1;
143 - while(c >= 0 && isspace(s[c])) {
144 - s[c] = '\0';
145 - c--;
146 - }
147 - if(c < 0) return NULL;
148 - if(!*s) return NULL;
149 - return s;
150 -}
151 -
232 +// read the process groups file
233 int read_process_groups(const char *name)
234 {
235 char buffer[4096+1];
@@ -230,44 +311,11 @@ int read_process_groups(const char *name)
311 return 0;
312 }
313
233 -void parse_args(int argc, char **argv)
234 -{
235 - int i, freq = 0;
236 - char *name = NULL;
237 -
238 - for(i = 1; i < argc; i++) {
239 - if(!freq) {
240 - int n = atoi(argv[i]);
241 - if(n > 0) {
242 - freq = n;
243 - continue;
244 - }
245 - }
246 -
247 - if(strcmp("debug", argv[i]) == 0) {
248 - debug = 1;
249 - continue;
250 - }
251 -
252 - if(!name) {
253 - name = argv[i];
254 - continue;
255 - }
256 -
257 - fprintf(stderr, "apps.plugin: ERROR: cannot understand option %s\n", argv[i]);
258 - exit(1);
259 - }
260 -
261 - if(freq > 0) update_every = freq;
262 - if(!name) name = "groups";
263 -
264 - if(read_process_groups(name)) {
265 - fprintf(stderr, "apps.plugin: ERROR: cannot read process groups %s\n", name);
266 - exit(1);
267 - }
268 -}
314
315 +// ----------------------------------------------------------------------------
316 +// data to store for each pid
317 // see: man proc
318 +
319 struct pid_stat {
320 int32_t pid;
321 char comm[MAX_COMPARE_NAME + 1];
@@ -330,7 +378,8 @@ struct pid_stat {
378 unsigned long long io_storage_bytes_written;
379 unsigned long long io_cancelled_write_bytes;
380
333 -/* unsigned long long old_utime;
381 +#ifdef INCLUDE_CHILDS
382 + unsigned long long old_utime;
383 unsigned long long old_stime;
384 unsigned long long old_minflt;
385 unsigned long long old_majflt;
@@ -349,24 +398,20 @@ struct pid_stat {
398 unsigned long long diff_cstime;
399 unsigned long long diff_cminflt;
400 unsigned long long diff_cmajflt;
352 -*/
401 +#endif
402
354 - int *fds;
355 - int fds_size;
403 + int *fds; // array of fds it uses
404 + int fds_size; // the size of the fds array
405
357 - int childs; // number of processes directly referencing this
358 - int updated;
359 - int merged;
406 + int childs; // number of processes directly referencing this
407 + int updated; // 1 when update
408 + int merged; // 1 when it has been merged to its parent
409 int new_entry;
410 struct target *target;
411 struct pid_stat *parent;
412 struct pid_stat *prev;
413 struct pid_stat *next;
365 -} *root = NULL;
366 -
367 -struct pid_stat **all_pids;
368 -
369 -#define PROC_BUFFER 4096
414 +} *root = NULL, **all_pids;
415
416 struct pid_stat *get_entry(pid_t pid)
417 {
@@ -410,16 +455,49 @@ void del_entry(pid_t pid)
455 all_pids[pid] = NULL;
456 }
457
458 +#ifdef INCLUDE_CHILDS
459 +// print a tree view of all processes
460 +int walk_down(pid_t pid, int level) {
461 + struct pid_stat *p = NULL;
462 + char b[level+3];
463 + int i, ret = 0;
464
414 -unsigned long simple_hash(const char *name)
415 -{
416 - int i, len = strlen(name);
417 - unsigned long hash = 0;
465 + for(i = 0; i < level; i++) b[i] = '\t';
466 + b[level] = '|';
467 + b[level+1] = '-';
468 + b[level+2] = '\0';
469
419 - for(i = 0; i < len ;i++) hash += (i * name[i]) + i + name[i];
470 + for(p = root; p ; p = p->next) {
471 + if(p->ppid == pid) {
472 + ret += walk_down(p->pid, level+1);
473 + }
474 + }
475
421 - return hash;
476 + p = all_pids[pid];
477 + if(p) {
478 + if(!p->updated) ret += 1;
479 + if(ret) fprintf(stderr, "%s %s %d [%s, %s] c=%d u=%llu+%llu, s=%llu+%llu, cu=%llu+%llu, cs=%llu+%llu, n=%llu+%llu, j=%llu+%llu, cn=%llu+%llu, cj=%llu+%llu\n"
480 + , b, p->comm, p->pid, p->updated?"OK":"KILLED", p->target->name, p->childs
481 + , p->utime, p->utime - p->old_utime
482 + , p->stime, p->stime - p->old_stime
483 + , p->cutime, p->cutime - p->old_cutime
484 + , p->cstime, p->cstime - p->old_cstime
485 + , p->minflt, p->minflt - p->old_minflt
486 + , p->majflt, p->majflt - p->old_majflt
487 + , p->cminflt, p->cminflt - p->old_cminflt
488 + , p->cmajflt, p->cmajflt - p->old_cmajflt
489 + );
490 + }
491 +
492 + return ret;
493 }
494 +#endif
495 +
496 +
497 +// ----------------------------------------------------------------------------
498 +// file descriptor
499 +// this is used to keep a global list of all open files of the system
500 +// it is needed in order to figure out the unique files a process tree has open
501
502 struct file_descriptor {
503 unsigned long hash;
@@ -528,6 +606,26 @@ unsigned long file_descriptor_find_or_add(const char *name)
606 return c;
607 }
608
609 +
610 +// ----------------------------------------------------------------------------
611 +// update pids from proc
612 +
613 +// 1. read all files in /proc
614 +// 2. for each numeric directory:
615 +// i. read /proc/pid/stat
616 +// ii. read /proc/pid/statm
617 +// iii. read /proc/pid/io (requires root access)
618 +// iii. read the entries in directory /proc/pid/fd (requires root access)
619 +// for each entry:
620 +// a. find or create a struct file_descriptor
621 +// b. cleanup any old/unused file_descriptors
622 +
623 +// after all these, some pids may be linked to targets, while others may not
624 +
625 +// in case of errors, only 1 every 1000 errors is printed
626 +// to avoid filling up all disk space
627 +// if debug is enabled, all errors are printed
628 +
629 int update_from_proc(void)
630 {
631 static long count_errors = 0;
@@ -576,7 +674,6 @@ int update_from_proc(void)
674 if(bytes == -1) {
675 if(!count_errors++ || debug)
676 fprintf(stderr, "apps.plugin: ERROR: cannot read from file '%s' (%s).\n", filename, strerror(errno));
579 -
677 continue;
678 }
679
@@ -821,42 +918,22 @@ int update_from_proc(void)
918
919 return 1;
920 }
824 -/*
825 -int walk_down(pid_t pid, int level) {
826 - struct pid_stat *p = NULL;
827 - char b[level+3];
828 - int i, ret = 0;
921
830 - for(i = 0; i < level; i++) b[i] = '\t';
831 - b[level] = '|';
832 - b[level+1] = '-';
833 - b[level+2] = '\0';
922
835 - for(p = root; p ; p = p->next) {
836 - if(p->ppid == pid) {
837 - ret += walk_down(p->pid, level+1);
838 - }
839 - }
923 +// ----------------------------------------------------------------------------
924 +// update statistics on the targets
925
841 - p = all_pids[pid];
842 - if(p) {
843 - if(!p->updated) ret += 1;
844 - if(ret) fprintf(stderr, "%s %s %d [%s, %s] c=%d u=%llu+%llu, s=%llu+%llu, cu=%llu+%llu, cs=%llu+%llu, n=%llu+%llu, j=%llu+%llu, cn=%llu+%llu, cj=%llu+%llu\n"
845 - , b, p->comm, p->pid, p->updated?"OK":"KILLED", p->target->name, p->childs
846 - , p->utime, p->utime - p->old_utime
847 - , p->stime, p->stime - p->old_stime
848 - , p->cutime, p->cutime - p->old_cutime
849 - , p->cstime, p->cstime - p->old_cstime
850 - , p->minflt, p->minflt - p->old_minflt
851 - , p->majflt, p->majflt - p->old_majflt
852 - , p->cminflt, p->cminflt - p->old_cminflt
853 - , p->cmajflt, p->cmajflt - p->old_cmajflt
854 - );
855 - }
856 -
857 - return ret;
858 -}
859 -*/
926 +// 1. link all childs to their parents
927 +// 2. go from bottom to top, marking as merged all childs to their parents
928 +// this step links all parents without a target to the child target, if any
929 +// 3. link all top level processes (the ones not merged) to the default target
930 +// 4. go from top to bottom, linking all childs without a target, to their parent target
931 +// after this step, all processes have a target
932 +// [5. for each killed pid (updated = 0), remove its usage from its target]
933 +// 6. zero all targets
934 +// 7. concentrate all values on the targets
935 +// 8. remove all killed processes
936 +// 9. find the unique file count for each target
937
938 void update_statistics(void)
939 {
@@ -911,13 +988,15 @@ void update_statistics(void)
988 // then is is a top level process
989 if(!p->merged && !p->target) p->target = default_target;
990
914 -/* // by the way, update the diffs
991 +#ifdef INCLUDE_CHILDS
992 + // by the way, update the diffs
993 // will be used later for substracting killed process times
994 p->diff_cutime = p->utime - p->cutime;
995 p->diff_cstime = p->stime - p->cstime;
996 p->diff_cminflt = p->minflt - p->cminflt;
997 p->diff_cmajflt = p->majflt - p->cmajflt;
920 -*/ }
998 +#endif
999 + }
1000
1001 // give a target to all merged child processes
1002 found = 1;
@@ -931,7 +1010,8 @@ void update_statistics(void)
1010 }
1011 }
1012
934 -/* // for each killed process, remove its values from the parents
1013 +#ifdef INCLUDE_CHILDS
1014 + // for each killed process, remove its values from the parents
1015 // sums (we had already added them in a previous loop)
1016 for(p = root; p ; p = p->next) {
1017 if(p->updated) continue;
@@ -983,7 +1063,8 @@ void update_statistics(void)
1063 if(diff_minflt) fprintf(stderr, "apps.plugin: \t cannot fix up minflt %llu\n", diff_minflt);
1064 if(diff_majflt) fprintf(stderr, "apps.plugin: \t cannot fix up majflt %llu\n", diff_majflt);
1065 }
986 -*/
1066 +#endif
1067 +
1068 // zero all the targets
1069 struct target *w;
1070 for (w = target_root; w ; w = w->next) {
@@ -1020,8 +1101,10 @@ void update_statistics(void)
1101 w->io_cancelled_write_bytes = 0;
1102 }
1103
1023 -/* walk_down(0, 1);
1024 -*/
1104 +#ifdef INCLUDE_CHILDS
1105 + if(debug) walk_down(0, 1);
1106 +#endif
1107 +
1108 // concentrate everything on the targets
1109 for(p = root; p ; p = p->next) {
1110 if(!p->target) {
@@ -1080,7 +1163,8 @@ void update_statistics(void)
1163 if(p->majflt - p->old_majflt > 5000) fprintf(stderr, "BIG CHANGE: %d %s majflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->majflt - p->old_majflt, p->old_majflt, p->majflt);
1164 if(p->cminflt - p->old_cminflt > 15000) fprintf(stderr, "BIG CHANGE: %d %s cminflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cminflt - p->old_cminflt, p->old_cminflt, p->cminflt);
1165 if(p->cmajflt - p->old_cmajflt > 15000) fprintf(stderr, "BIG CHANGE: %d %s cmajflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cmajflt - p->old_cmajflt, p->old_cmajflt, p->cmajflt);
1083 -
1166 +*/
1167 +#ifdef INCLUDE_CHILDS
1168 p->old_utime = p->utime;
1169 p->old_cutime = p->cutime;
1170 p->old_stime = p->stime;
@@ -1089,7 +1173,8 @@ void update_statistics(void)
1173 p->old_majflt = p->majflt;
1174 p->old_cminflt = p->cminflt;
1175 p->old_cmajflt = p->cmajflt;
1092 -*/ }
1176 +#endif
1177 + }
1178 else {
1179 // since the process has exited, the user
1180 // will see a drop in our charts, because the incremental
@@ -1116,13 +1201,12 @@ void update_statistics(void)
1201 }
1202 }
1203
1119 -/* fprintf(stderr, "\n");
1120 -*/
1204 +// fprintf(stderr, "\n");
1205 // cleanup all un-updated processed (exited, killed, etc)
1206 for(p = root, c = 0; p ; c++) {
1207 if(!p->updated) {
1124 -/* fprintf(stderr, "\tEXITED %d %s [parent %d %s, target %s] utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->pid, p->comm, p->parent->pid, p->parent->comm, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
1125 -*/
1208 +// fprintf(stderr, "\tEXITED %d %s [parent %d %s, target %s] utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->pid, p->comm, p->parent->pid, p->parent->comm, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
1209 +
1210 pid_t r = p->pid;
1211 p = p->next;
1212 del_entry(r);
@@ -1185,9 +1269,8 @@ void update_statistics(void)
1269 }
1270 }
1271
1188 -unsigned long long usecdiff(struct timeval *now, struct timeval *last) {
1189 - return ((((now->tv_sec * 1000000ULL) + now->tv_usec) - ((last->tv_sec * 1000000ULL) + last->tv_usec)));
1190 -}
1272 +// ----------------------------------------------------------------------------
1273 +// update chart dimensions
1274
1275 void show_dimensions(void)
1276 {
@@ -1345,42 +1428,11 @@ void show_dimensions(void)
1428 }
1429 fprintf(stdout, "END\n");
1430
1348 - fprintf(stdout, "BEGIN apps.inotify %llu\n", usec);
1349 - for (w = target_root; w ; w = w->next) {
1350 - if(w->target || (!w->processes && !w->exposed)) continue;
1351 -
1352 - fprintf(stdout, "SET %s = %llu\n", w->name, w->openinotifies);
1353 - }
1354 - fprintf(stdout, "END\n");
1355 -
1356 - fprintf(stdout, "BEGIN apps.eventfd %llu\n", usec);
1357 - for (w = target_root; w ; w = w->next) {
1358 - if(w->target || (!w->processes && !w->exposed)) continue;
1359 -
1360 - fprintf(stdout, "SET %s = %llu\n", w->name, w->openeventfds + w->openeventpolls);
1361 - }
1362 - fprintf(stdout, "END\n");
1363 -
1364 - fprintf(stdout, "BEGIN apps.timerfd %llu\n", usec);
1365 - for (w = target_root; w ; w = w->next) {
1366 - if(w->target || (!w->processes && !w->exposed)) continue;
1367 -
1368 - fprintf(stdout, "SET %s = %llu\n", w->name, w->opentimerfds);
1369 - }
1370 - fprintf(stdout, "END\n");
1371 -
1372 - fprintf(stdout, "BEGIN apps.signalfd %llu\n", usec);
1373 - for (w = target_root; w ; w = w->next) {
1374 - if(w->target || (!w->processes && !w->exposed)) continue;
1375 -
1376 - fprintf(stdout, "SET %s = %llu\n", w->name, w->opensignalfds);
1377 - }
1378 - fprintf(stdout, "END\n");
1379 -
1431 fprintf(stdout, "BEGIN netdata.apps_cpu %llu\n", usec);
1432 fprintf(stdout, "SET user = %llu\n", cpuuser);
1433 fprintf(stdout, "SET system = %llu\n", cpusyst);
1434 fprintf(stdout, "END\n");
1435 +
1436 fprintf(stdout, "BEGIN netdata.apps_files %llu\n", usec);
1437 fprintf(stdout, "SET files = %llu\n", file_counter);
1438 fprintf(stdout, "END\n");
@@ -1388,6 +1440,10 @@ void show_dimensions(void)
1440 fflush(stdout);
1441 }
1442
1443 +
1444 +// ----------------------------------------------------------------------------
1445 +// generate the charts
1446 +
1447 void show_charts(void)
1448 {
1449 struct target *w;
@@ -1510,34 +1566,6 @@ void show_charts(void)
1566 fprintf(stdout, "DIMENSION %s '' absolute-no-interpolation 1 1\n", w->name);
1567 }
1568
1513 - fprintf(stdout, "CHART apps.inotify '' 'Apps inotifies' 'open inotifies' apps none stacked 20054 %d\n", update_every);
1514 - for (w = target_root; w ; w = w->next) {
1515 - if(w->target || (!w->processes && !w->exposed)) continue;
1516 -
1517 - fprintf(stdout, "DIMENSION %s '' absolute-no-interpolation 1 1\n", w->name);
1518 - }
1519 -
1520 - fprintf(stdout, "CHART apps.eventfd '' 'Apps Event File Descriptors' 'open event fds' apps none stacked 20055 %d\n", update_every);
1521 - for (w = target_root; w ; w = w->next) {
1522 - if(w->target || (!w->processes && !w->exposed)) continue;
1523 -
1524 - fprintf(stdout, "DIMENSION %s '' absolute-no-interpolation 1 1\n", w->name);
1525 - }
1526 -
1527 - fprintf(stdout, "CHART apps.timerfd '' 'Apps Timer File Descriptors' 'open timer fds' apps none stacked 20057 %d\n", update_every);
1528 - for (w = target_root; w ; w = w->next) {
1529 - if(w->target || (!w->processes && !w->exposed)) continue;
1530 -
1531 - fprintf(stdout, "DIMENSION %s '' absolute-no-interpolation 1 1\n", w->name);
1532 - }
1533 -
1534 - fprintf(stdout, "CHART apps.signalfd '' 'Apps Timer File Descriptors' 'open timer fds' apps none stacked 20058 %d\n", update_every);
1535 - for (w = target_root; w ; w = w->next) {
1536 - if(w->target || (!w->processes && !w->exposed)) continue;
1537 -
1538 - fprintf(stdout, "DIMENSION %s '' absolute-no-interpolation 1 1\n", w->name);
1539 - }
1540 -
1569 fprintf(stdout, "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' netdata netdata stacked 10000 %d\n", update_every);
1570 fprintf(stdout, "DIMENSION user '' incremental 1 1000\n");
1571 fprintf(stdout, "DIMENSION system '' incremental 1 1000\n");
@@ -1548,57 +1576,45 @@ void show_charts(void)
1576 fflush(stdout);
1577 }
1578
1551 -unsigned long long get_hertz(void)
1552 -{
1553 - unsigned long long hz = 1;
1579
1555 -#ifdef _SC_CLK_TCK
1556 - if((hz = sysconf(_SC_CLK_TCK)) > 0) {
1557 - return hz;
1558 - }
1559 -#endif
1580 +// ----------------------------------------------------------------------------
1581 +// parse command line arguments
1582
1561 -#ifdef HZ
1562 - hz = (unsigned long long)HZ; /* <asm/param.h> */
1563 -#else
1564 - /* If 32-bit or big-endian (not Alpha or ia64), assume HZ is 100. */
1565 - hz = (sizeof(long)==sizeof(int) || htons(999)==999) ? 100UL : 1024UL;
1566 -#endif
1583 +void parse_args(int argc, char **argv)
1584 +{
1585 + int i, freq = 0;
1586 + char *name = NULL;
1587
1568 - fprintf(stderr, "apps.plugin: ERROR: unknown HZ value. Assuming %llu.\n", hz);
1569 - return hz;
1570 -}
1588 + for(i = 1; i < argc; i++) {
1589 + if(!freq) {
1590 + int n = atoi(argv[i]);
1591 + if(n > 0) {
1592 + freq = n;
1593 + continue;
1594 + }
1595 + }
1596
1572 -long get_processors(void)
1573 -{
1574 - char buffer[1025], *s;
1575 - int processors = 0;
1597 + if(strcmp("debug", argv[i]) == 0) {
1598 + debug = 1;
1599 + continue;
1600 + }
1601
1577 - FILE *fp = fopen("/proc/stat", "r");
1578 - if(!fp) return 1;
1602 + if(!name) {
1603 + name = argv[i];
1604 + continue;
1605 + }
1606
1580 - while((s = fgets(buffer, 1024, fp))) {
1581 - if(strncmp(buffer, "cpu", 3) == 0) processors++;
1607 + fprintf(stderr, "apps.plugin: ERROR: cannot understand option %s\n", argv[i]);
1608 + exit(1);
1609 }
1583 - fclose(fp);
1584 - processors--;
1585 - if(processors < 1) processors = 1;
1586 - return processors;
1587 -}
1588 -
1589 -long get_pid_max(void)
1590 -{
1591 - char buffer[1025], *s;
1592 - long mpid = 32768;
1610
1594 - FILE *fp = fopen("/proc/sys/kernel/pid_max", "r");
1595 - if(!fp) return 1;
1611 + if(freq > 0) update_every = freq;
1612 + if(!name) name = "groups";
1613
1597 - s = fgets(buffer, 1024, fp);
1598 - if(s) mpid = atol(buffer);
1599 - fclose(fp);
1600 - if(mpid < 32768) mpid = 32768;
1601 - return mpid;
1614 + if(read_process_groups(name)) {
1615 + fprintf(stderr, "apps.plugin: ERROR: cannot read process groups %s\n", name);
1616 + exit(1);
1617 + }
1618 }
1619
1620 int main(int argc, char **argv)
@@ -1631,7 +1647,6 @@ int main(int argc, char **argv)
1647 update_statistics();
1648 show_charts(); // this is smart enough to show only newly added apps, when needed
1649 show_dimensions();
1634 - // printf("FLUSH xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n");
1650
1651 if(debug) fprintf(stderr, "apps.plugin: done Loop No %llu\n", counter);
1652 fflush(NULL);
@@ -1645,15 +1660,7 @@ int main(int argc, char **argv)
1660 if(usec < (update_every * 1000000ULL / 2)) susec = (update_every * 1000000ULL) - usec;
1661 else susec = update_every * 1000000ULL / 2;
1662
1648 - //fprintf(stderr, "apps.plugin: sleeping for %llu...", susec);
1649 - //fflush(NULL);
1650 - //struct timeval before, after;
1651 - //gettimeofday(&before, NULL);
1663 usleep(susec);
1653 - //gettimeofday(&after, NULL);
1654 - //fprintf(stderr, "apps.plugin: sleeped for %llu\n", usecdiff(&after, &before));
1655 - //fflush(NULL);
1656 -
1664 bcopy(&now, &last, sizeof(struct timeval));
1665 }
1666 }
netdata.c
+258 -121
@@ -1107,13 +1107,15 @@ struct rrd_dimension {
1107 long multiplier;
1108 long divisor;
1109
1110 - struct timeval last_collected; // when was this dimension last updated
1110 + struct timeval last_collected_time; // when was this dimension last updated
1111 + // this is only used to detect un-updated dimensions
1112 + // which are removed after some time
1113
1114 calculated_number calculated_value;
1115 calculated_number last_calculated_value;
1116
1115 - collected_number collected_value;
1116 - collected_number last_collected_value;
1117 + collected_number collected_value; // the value collected at this round
1118 + collected_number last_collected_value; // the value that was collected at the last round
1119
1120 struct rrd_dimension *next; // linking of dimensions within the same data set
1121
@@ -1153,8 +1155,8 @@ struct rrd_stats {
1155
1156 int update_every; // every how many seconds is this updated?
1157 unsigned long long first_entry_t; // the timestamp (in microseconds) of the oldest entry in the db
1156 - struct timeval last_updated; // when this data set was last updated
1157 - struct timeval last_collected;
1158 + struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)
1159 + struct timeval last_collected_time; //
1160 unsigned long long usec_since_last_update;
1161
1162 total_number absolute_total;
@@ -1322,8 +1324,8 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
1324
1325 st->debug = 0;
1326
1325 - st->last_collected.tv_sec = 0;
1326 - st->last_collected.tv_usec = 0;
1327 + st->last_collected_time.tv_sec = 0;
1328 + st->last_collected_time.tv_usec = 0;
1329
1330 pthread_rwlock_init(&st->rwlock, NULL);
1331 pthread_rwlock_wrlock(&root_rwlock);
@@ -1597,11 +1599,9 @@ int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
1599
1600 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
1601 {
1600 - if(!st->last_collected.tv_sec) gettimeofday(&st->last_collected, NULL);
1601 -
1602 - rd->last_collected.tv_sec = st->last_collected.tv_sec;
1603 - rd->last_collected.tv_usec = st->last_collected.tv_usec;
1604 -
1602 + if(st) {;}
1603 +
1604 + gettimeofday(&rd->last_collected_time, NULL);
1605 rd->collected_value = value;
1606 }
1607
@@ -1617,64 +1617,15 @@ int rrd_stats_dimension_set(RRD_STATS *st, char *id, collected_number value)
1617 return 0;
1618 }
1619
1620 -void rrd_stats_next_internal(RRD_STATS *st)
1621 -{
1622 - // a read lock is OK here
1623 - pthread_rwlock_rdlock(&st->rwlock);
1624 -
1625 - RRD_DIMENSION *rd;
1626 - for( rd = st->dimensions; rd ; rd = rd->next ) {
1627 - rd->last_collected_value = rd->collected_value;
1628 - rd->collected_value = 0;
1629 - }
1630 -
1631 - pthread_rwlock_unlock(&st->rwlock);
1632 -}
1633 -
1634 -void rrd_stats_next_timeval(RRD_STATS *st, struct timeval *now)
1635 -{
1636 - if(!st->last_collected.tv_sec) {
1637 - gettimeofday(&st->last_collected, NULL);
1638 - unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec - st->update_every * 1000000ULL;
1639 - st->last_collected.tv_sec = ut / 1000000ULL;
1640 - st->last_collected.tv_usec = ut % 1000000ULL;
1641 - }
1642 -
1643 - st->usec_since_last_update = usecdiff(now, &st->last_collected);
1644 - st->last_collected.tv_sec = now->tv_sec;
1645 - st->last_collected.tv_usec = now->tv_usec;
1646 -
1647 - rrd_stats_next_internal(st);
1648 -}
1649 -
1620 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
1621 {
1652 - if(!st->last_collected.tv_sec) {
1653 - gettimeofday(&st->last_collected, NULL);
1654 - unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec;
1655 - st->last_collected.tv_sec = ut / 1000000ULL;
1656 - st->last_collected.tv_usec = ut % 1000000ULL;
1657 - }
1658 - else {
1659 - unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec + microseconds;
1660 - st->last_collected.tv_sec = ut / 1000000ULL;
1661 - st->last_collected.tv_usec = ut % 1000000ULL;
1662 - }
1622 + if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
1623 st->usec_since_last_update = microseconds;
1664 -
1665 - rrd_stats_next_internal(st);
1624 }
1625
1626 void rrd_stats_next(RRD_STATS *st)
1627 {
1670 - if(st->last_collected.tv_sec) {
1671 - struct timeval now;
1672 - gettimeofday(&now, NULL);
1673 -
1674 - rrd_stats_next_timeval(st, &now);
1675 - }
1676 - else
1677 - rrd_stats_next_usec(st, st->update_every * 1000000ULL);
1628 + rrd_stats_next_usec(st, st->update_every * 1000000ULL);
1629 }
1630
1631 void rrd_stats_next_plugins(RRD_STATS *st)
@@ -1693,29 +1644,32 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1644 // a read lock is OK here
1645 pthread_rwlock_rdlock(&st->rwlock);
1646
1647 + if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
1648 +
1649 + if(!st->last_collected_time.tv_sec) gettimeofday(&st->last_collected_time, NULL);
1650 + else {
1651 + unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
1652 + st->last_collected_time.tv_sec = ut / 1000000ULL;
1653 + st->last_collected_time.tv_usec = ut % 1000000ULL;
1654 + }
1655 +
1656 if(!st->last_updated.tv_sec) {
1697 - unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec - st->usec_since_last_update;
1657 + unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
1658 st->last_updated.tv_sec = ut / 1000000ULL;
1659 st->last_updated.tv_usec = ut % 1000000ULL;
1660 +
1661 + if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
1662 }
1663
1664 unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
1703 - unsigned long long now_ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec + st->usec_since_last_update;
1665 + unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
1666 unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
1667
1706 - st->counter_done++;
1707 - if(st->counter_done == 1 || now_ut < next_ut) {
1708 - if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
1709 - // we don't have any usable data yet
1710 - pthread_rwlock_unlock(&st->rwlock);
1711 -
1712 - if(pthread_setcancelstate(oldstate, NULL) != 0)
1713 - error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1714 -
1715 - return(st->usec_since_last_update);
1716 - }
1668 + if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
1669 + if(st->debug) debug(D_RRD_STATS, "%s: now ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
1670 + if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
1671
1718 - if(st->debug) debug(D_RRD_STATS, "microseconds since last update: %llu", st->usec_since_last_update);
1672 + st->counter_done++;
1673
1674 // calculate totals and count the dimensions
1675 int dimensions;
@@ -1724,7 +1678,9 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1678 for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
1679 st->absolute_total += rd->collected_value;
1680
1727 - // process all dimensions to calculate its values
1681 + // process all dimensions to calculate their values
1682 + // based on the collected figures only
1683 + // at this stage we do not interpolate anything
1684 for( rd = st->dimensions ; rd ; rd = rd->next ) {
1685 switch(rd->algorithm) {
1686 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
@@ -1737,7 +1693,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1693 / (calculated_number)(st->absolute_total - st->last_absolute_total);
1694
1695 if(st->debug)
1740 - debug(D_RRD_STATS, "%s/%s: CALC "
1696 + debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
1697 CALCULATED_NUMBER_FORMAT " = 100"
1698 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1699 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
@@ -1759,7 +1715,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1715 / (calculated_number)st->absolute_total;
1716
1717 if(st->debug)
1762 - debug(D_RRD_STATS, "%s/%s: CALC "
1718 + debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
1719 CALCULATED_NUMBER_FORMAT " = 100"
1720 " * " COLLECTED_NUMBER_FORMAT
1721 " / " COLLECTED_NUMBER_FORMAT
@@ -1779,21 +1735,32 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1735 // to reset the calculation (it will give zero as the calculation for this second)
1736 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
1737
1782 - rd->calculated_value =
1783 - (calculated_number)1000000
1784 - * (calculated_number)(rd->collected_value - rd->last_collected_value)
1785 - / (calculated_number)st->usec_since_last_update;
1738 + if(!st->usec_since_last_update) {
1739 + rd->calculated_value = 0;
1740 + if(st->debug)
1741 + debug(D_RRD_STATS, "%s/%s: CALC INC-ZERO "
1742 + CALCULATED_NUMBER_FORMAT " = 0"
1743 + , st->id, rd->name
1744 + , rd->calculated_value
1745 + );
1746 + }
1747 + else {
1748 + rd->calculated_value =
1749 + (calculated_number)1000000
1750 + * (calculated_number)(rd->collected_value - rd->last_collected_value)
1751 + / (calculated_number)st->usec_since_last_update;
1752
1787 - if(st->debug)
1788 - debug(D_RRD_STATS, "%s/%s: CALC "
1789 - CALCULATED_NUMBER_FORMAT " = 1000000"
1790 - " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1791 - " / %llu"
1792 - , st->id, rd->name
1793 - , rd->calculated_value
1794 - , rd->collected_value, rd->last_collected_value
1795 - , st->usec_since_last_update
1796 - );
1753 + if(st->debug)
1754 + debug(D_RRD_STATS, "%s/%s: CALC INC "
1755 + CALCULATED_NUMBER_FORMAT " = 1000000"
1756 + " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1757 + " / %llu"
1758 + , st->id, rd->name
1759 + , rd->calculated_value
1760 + , rd->collected_value, rd->last_collected_value
1761 + , st->usec_since_last_update
1762 + );
1763 + }
1764 break;
1765
1766 case RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION:
@@ -1805,16 +1772,27 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1772 // to reset the calculation (it will give zero as the calculation for this second)
1773 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
1774
1808 - rd->calculated_value = (calculated_number)(rd->collected_value - rd->last_collected_value);
1775 + if(!st->usec_since_last_update) {
1776 + rd->calculated_value = 0;
1777 + if(st->debug)
1778 + debug(D_RRD_STATS, "%s/%s: CALC INC-NO-IN-ZERO "
1779 + CALCULATED_NUMBER_FORMAT " = 0"
1780 + , st->id, rd->name
1781 + , rd->calculated_value
1782 + );
1783 + }
1784 + else {
1785 + rd->calculated_value = (calculated_number)(rd->collected_value - rd->last_collected_value);
1786
1810 - if(st->debug)
1811 - debug(D_RRD_STATS, "%s/%s: CALC "
1812 - CALCULATED_NUMBER_FORMAT " = "
1813 - COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
1814 - , st->id, rd->name
1815 - , rd->calculated_value
1816 - , rd->collected_value, rd->last_collected_value
1817 - );
1787 + if(st->debug)
1788 + debug(D_RRD_STATS, "%s/%s: CALC INC-NO-IN "
1789 + CALCULATED_NUMBER_FORMAT " = "
1790 + COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
1791 + , st->id, rd->name
1792 + , rd->calculated_value
1793 + , rd->collected_value, rd->last_collected_value
1794 + );
1795 + }
1796 break;
1797
1798 case RRD_DIMENSION_ABSOLUTE:
@@ -1822,7 +1800,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1800 rd->calculated_value = (calculated_number)rd->collected_value;
1801
1802 if(st->debug)
1825 - debug(D_RRD_STATS, "%s/%s: CALC "
1803 + debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
1804 CALCULATED_NUMBER_FORMAT " = "
1805 COLLECTED_NUMBER_FORMAT
1806 , st->id, rd->name
@@ -1845,9 +1823,30 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1823 break;
1824 }
1825 }
1826 + // at this point we have all the calculated values ready
1827 +
1828 + if(st->counter_done == 1 || now_ut < next_ut) {
1829 + // we don't have any usable data yet
1830 + if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
1831 +
1832 + for( rd = st->dimensions; rd ; rd = rd->next ) {
1833 + rd->last_collected_value = rd->collected_value;
1834 + rd->last_calculated_value = rd->calculated_value;
1835 + rd->collected_value = 0;
1836 + }
1837 +
1838 + pthread_rwlock_unlock(&st->rwlock);
1839 + if(pthread_setcancelstate(oldstate, NULL) != 0)
1840 + error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1841 +
1842 + return(st->usec_since_last_update);
1843 + }
1844
1845 + // it is now time to interpolate values on a second boundary
1846 + unsigned long long first_ut = last_ut;
1847 for( ; next_ut < now_ut ; next_ut += st->update_every * 1000000ULL ) {
1850 - unsigned long long np = next_ut - last_ut;
1848 + if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
1849 + if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
1850
1851 st->last_updated.tv_sec = next_ut / 1000000ULL;
1852 st->last_updated.tv_usec = 0;
@@ -1864,24 +1863,46 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1863 new_value = rd->calculated_value;
1864
1865 if(st->debug)
1867 - debug(D_RRD_STATS, "%s/%s: CALC2 "
1866 + debug(D_RRD_STATS, "%s/%s: CALC2 NO-IN "
1867 + CALCULATED_NUMBER_FORMAT
1868 + , st->id, rd->name
1869 + , new_value
1870 + );
1871 + break;
1872 +
1873 + case RRD_DIMENSION_INCREMENTAL:
1874 + new_value = (calculated_number)
1875 + ( rd->calculated_value
1876 + * (calculated_number)(next_ut - last_ut)
1877 + / (calculated_number)(st->update_every * 1000000)
1878 + );
1879 +
1880 + if(st->debug)
1881 + debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
1882 + CALCULATED_NUMBER_FORMAT " = "
1883 CALCULATED_NUMBER_FORMAT
1884 + " * %llu"
1885 + " / %llu"
1886 , st->id, rd->name
1887 , new_value
1888 + , rd->calculated_value
1889 + , (next_ut - last_ut)
1890 + , (st->update_every * 1000000)
1891 );
1892 break;
1893
1894 + case RRD_DIMENSION_ABSOLUTE:
1895 default:
1896 new_value = (calculated_number)
1897 ( ( (rd->calculated_value - rd->last_calculated_value)
1877 - * (calculated_number)np
1878 - / (calculated_number)(now_ut - last_ut)
1898 + * (calculated_number)(next_ut - first_ut)
1899 + / (calculated_number)(now_ut - first_ut)
1900 )
1901 + rd->last_calculated_value
1902 );
1903
1904 if(st->debug)
1884 - debug(D_RRD_STATS, "%s/%s: CALC2 "
1905 + debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
1906 CALCULATED_NUMBER_FORMAT " = ((("
1907 "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
1908 " * %llu"
@@ -1889,10 +1910,11 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1910 , st->id, rd->name
1911 , new_value
1912 , rd->calculated_value, rd->last_calculated_value
1892 - , np
1893 - , (now_ut - last_ut), rd->last_calculated_value
1913 + , (next_ut - first_ut)
1914 + , (now_ut - first_ut), rd->last_calculated_value
1915 );
1916 break;
1917 +
1918 }
1919
1920 rd->values[st->current_entry] = (storage_number)
@@ -1915,14 +1937,13 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1937 , rd->divisor
1938 );
1939
1918 - if((rd->algorithm == RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION || rd->algorithm == RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION)
1919 - && (next_ut + st->update_every * 1000000ULL < now_ut)) {
1940 + if(next_ut + st->update_every * 1000000ULL < now_ut) {
1941 // there is another iteration
1942 // do not change the anything
1943 ;
1944 }
1945 else
1925 - rd->last_calculated_value = rd->calculated_value = new_value;
1946 + rd->calculated_value = new_value;
1947 }
1948
1949 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
@@ -1937,13 +1958,19 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1958 last_ut = next_ut;
1959 }
1960
1961 + for( rd = st->dimensions; rd ; rd = rd->next ) {
1962 + rd->last_collected_value = rd->collected_value;
1963 + rd->last_calculated_value = rd->calculated_value;
1964 + rd->collected_value = 0;
1965 + }
1966 +
1967 // ALL DONE ABOUT THE DATA UPDATE
1968 // --------------------------------------------------------------------
1969
1970
1971 // find if there are any obsolete dimensions (not updated recently)
1972 for( rd = st->dimensions; rd ; rd = rd->next )
1946 - if((rd->last_collected.tv_sec + (10 * st->update_every)) < st->last_collected.tv_sec)
1973 + if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
1974 break;
1975
1976 if(rd) {
@@ -1953,7 +1980,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1980 pthread_rwlock_wrlock(&st->rwlock);
1981
1982 for( rd = st->dimensions, last = NULL ; rd ; ) {
1956 - if((rd->last_collected.tv_sec + (10 * st->update_every)) < st->last_collected.tv_sec) { // remove it only it is not updated in 10 seconds
1983 + if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
1984 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1985
1986 if(!last) {
@@ -2359,7 +2386,7 @@ unsigned long rrd_stats_one_json(RRD_STATS *st, char *options, struct web_buffer
2386 , algorithm_name(rd->algorithm)
2387 , rd->multiplier
2388 , rd->divisor
2362 - , rd->last_collected.tv_sec
2389 + , rd->last_collected_time.tv_sec
2390 , rd->collected_value
2391 , rd->calculated_value
2392 , rd->last_collected_value
@@ -6432,9 +6459,9 @@ void *checks_main(void *ptr)
6459 if(!apps_cpu) apps_cpu = rrd_stats_find("apps.cpu");
6460 if(check3->counter_done) rrd_stats_next_usec(check3, loop_usec);
6461 gettimeofday(&loop, NULL);
6435 - rrd_stats_dimension_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected));
6436 - rrd_stats_dimension_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected));
6437 - if(apps_cpu) rrd_stats_dimension_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected));
6462 + rrd_stats_dimension_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected_time));
6463 + rrd_stats_dimension_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected_time));
6464 + if(apps_cpu) rrd_stats_dimension_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected_time));
6465 rrd_stats_done(check3);
6466 }
6467
@@ -7223,12 +7250,121 @@ int become_daemon(int close_all_files, const char *input, const char *output, co
7250 return(0);
7251 }
7252
7253 +void unit_test(void)
7254 +{
7255 + debug_flags = 0xffffffff;
7256 +
7257 + int do_abs = 1;
7258 + int do_inc = 1;
7259 + int do_abs2 = 1;
7260 + int do_inc2 = 1;
7261 +
7262 + RRD_STATS *st = rrd_stats_find_bytype("netdata", "unittest");
7263 + if(!st) st = rrd_stats_create("netdata", "unittest", "unittest", "netdata", "Unit Testing", "a value", 1, 1, CHART_TYPE_LINE);
7264 + st->debug = 1;
7265 +
7266 + RRD_DIMENSION *rdabs = NULL;
7267 + RRD_DIMENSION *rdinc = NULL;
7268 + RRD_DIMENSION *rdabs2 = NULL;
7269 + RRD_DIMENSION *rdinc2 = NULL;
7270 +
7271 + if(do_abs) rdabs = rrd_stats_dimension_add(st, "absolute", "absolute", 1, 1, RRD_DIMENSION_ABSOLUTE);
7272 + if(do_inc) rdinc = rrd_stats_dimension_add(st, "incremental", "incremental", 1, 1, RRD_DIMENSION_INCREMENTAL);
7273 + if(do_abs2) rdabs2 = rrd_stats_dimension_add(st, "absolute-no-interpolation", "absolute-no-interpolation", 1, 1, RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION);
7274 + if(do_inc2) rdinc2 = rrd_stats_dimension_add(st, "incremental-no-interpolation", "incremental-no-interpolation", 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
7275 +
7276 + long delay = 2000000;
7277 + collected_number i = 1000;
7278 +
7279 + fprintf(stderr, "\n\nINIT, VALUE = " COLLECTED_NUMBER_FORMAT "\n", i);
7280 + if(do_abs) rrd_stats_dimension_set(st, "absolute", i);
7281 + if(do_inc) rrd_stats_dimension_set(st, "incremental", i);
7282 + if(do_abs2) rrd_stats_dimension_set(st, "absolute-no-interpolation", i);
7283 + if(do_inc2) rrd_stats_dimension_set(st, "incremental-no-interpolation", i);
7284 +
7285 + gettimeofday(&st->last_collected_time, NULL);
7286 + st->last_collected_time.tv_usec = 500000;
7287 + rrd_stats_done(st);
7288 + //usleep(delay);
7289 + i += 1000;
7290 + fprintf(stderr, "\n\nDELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", delay, i);
7291 + rrd_stats_next_usec(st, delay);
7292 +
7293 + if(do_abs) rrd_stats_dimension_set(st, "absolute", i);
7294 + if(do_inc) rrd_stats_dimension_set(st, "incremental", i);
7295 + if(do_abs2) rrd_stats_dimension_set(st, "absolute-no-interpolation", i);
7296 + if(do_inc2) rrd_stats_dimension_set(st, "incremental-no-interpolation", i);
7297 +
7298 + rrd_stats_done(st);
7299 + //usleep(delay);
7300 + i += 1000;
7301 + fprintf(stderr, "\n\nDELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", delay, i);
7302 + rrd_stats_next_usec(st, delay);
7303 +
7304 + if(do_abs) rrd_stats_dimension_set(st, "absolute", i);
7305 + if(do_inc) rrd_stats_dimension_set(st, "incremental", i);
7306 + if(do_abs2) rrd_stats_dimension_set(st, "absolute-no-interpolation", i);
7307 + if(do_inc2) rrd_stats_dimension_set(st, "incremental-no-interpolation", i);
7308 +
7309 + rrd_stats_done(st);
7310 + //usleep(delay);
7311 + i += 1000;
7312 + fprintf(stderr, "\n\nDELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", delay, i);
7313 + rrd_stats_next_usec(st, delay);
7314 +
7315 + if(do_abs) rrd_stats_dimension_set(st, "absolute", i);
7316 + if(do_inc) rrd_stats_dimension_set(st, "incremental", i);
7317 + if(do_abs2) rrd_stats_dimension_set(st, "absolute-no-interpolation", i);
7318 + if(do_inc2) rrd_stats_dimension_set(st, "incremental-no-interpolation", i);
7319 +
7320 + rrd_stats_done(st);
7321 + //usleep(delay);
7322 + i += 1000;
7323 + fprintf(stderr, "\n\nDELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", delay, i);
7324 + rrd_stats_next_usec(st, delay);
7325 +
7326 + if(do_abs) rrd_stats_dimension_set(st, "absolute", i);
7327 + if(do_inc) rrd_stats_dimension_set(st, "incremental", i);
7328 + if(do_abs2) rrd_stats_dimension_set(st, "absolute-no-interpolation", i);
7329 + if(do_inc2) rrd_stats_dimension_set(st, "incremental-no-interpolation", i);
7330 +
7331 + rrd_stats_done(st);
7332 +
7333 + fprintf(stderr, "\n\nDONE\n");
7334 +
7335 + int ret = 0;
7336 + unsigned long c;
7337 + storage_number v;
7338 + for(c = 0 ; c < st->counter ; c++) {
7339 + fprintf(stderr, "\nPOSITION: %lu\n", c);
7340 +
7341 + RRD_DIMENSION *rd;
7342 + for(rd = st->dimensions ; rd ; rd = rd->next) {
7343 + fprintf(stderr, "\t %s " STORAGE_NUMBER_FORMAT " ==> ", rd->id, rd->values[c]);
7344 +
7345 + if(rd == rdabs) {
7346 + v = 12500 + c * 5000;
7347 + }
7348 + else if(rd == rdinc) {
7349 + if(c == 0) v = 2500;
7350 + else v = 5000;
7351 + }
7352 + else if(rd == rdinc2) v = c?10000:0;
7353 + else if(rd == rdabs2) v = 10000 + (c + 1) / 2 * 10000;
7354 +
7355 + if(v == rd->values[c]) fprintf(stderr, "PASSED!\n");
7356 + else { fprintf(stderr, "ERROR! (expected " STORAGE_NUMBER_FORMAT ")\n", v); ret = 1; }
7357 + }
7358 + }
7359 +
7360 + exit(ret);
7361 +}
7362 +
7363 int main(int argc, char **argv)
7364 {
7365 int i;
7366 int config_loaded = 0;
7367
7231 -
7368 // parse the arguments
7369 for(i = 1; i < argc ; i++) {
7370 if(strcmp(argv[i], "-c") == 0 && (i+1) < argc) {
@@ -7247,6 +7383,7 @@ int main(int argc, char **argv)
7383 else if(strcmp(argv[i], "-u") == 0 && (i+1) < argc) { config_set("global", "run as user", argv[i+1]); i++; }
7384 else if(strcmp(argv[i], "-l") == 0 && (i+1) < argc) { config_set("global", "history", argv[i+1]); i++; }
7385 else if(strcmp(argv[i], "-t") == 0 && (i+1) < argc) { config_set("global", "update every", argv[i+1]); i++; }
7386 + else if(strcmp(argv[i], "--unittest") == 0) { unit_test(); }
7387 else {
7388 fprintf(stderr, "Cannot understand option '%s'.\n", argv[i]);
7389 fprintf(stderr, "\nUSAGE: %s [-d] [-l LINES_TO_SAVE] [-u UPDATE_TIMER] [-p LISTEN_PORT] [-dl debug log file] [-df debug flags].\n\n", argv[0]);