@cryptotaxi247 / netdata-1 / commits / ff7e81d3c

files, sockets etc are now uniquely counted for the whole process tree; also lowered cpu resources by 30% by not reading all the sockets, files, pipes for every loop

Costa Tsaousis (ktsaou) committed May 23, 2014 at 02:45 UTC ff7e81d3c7981ef05d12cdfb4029168edee3cdfa
1 file changed +242 -71
apps_plugin.c
+242 -71
@@ -79,6 +79,7 @@ struct target {
79 unsigned long long fix_io_storage_bytes_written;
80 unsigned long long fix_io_cancelled_write_bytes;
81
82 + int *fds;
83 unsigned long long openfiles;
84 unsigned long long openpipes;
85 unsigned long long opensockets;
@@ -87,7 +88,6 @@ struct target {
88 unsigned long long opentimerfds;
89 unsigned long long opensignalfds;
90 unsigned long long openeventpolls;
90 - unsigned long long openanoninodes;
91 unsigned long long openother;
92
93 unsigned long processes; // how many processes have been merged to this
@@ -351,16 +351,8 @@ struct pid_stat {
351 unsigned long long diff_cmajflt;
352 */
353
354 - unsigned long long openfiles;
355 - unsigned long long openpipes;
356 - unsigned long long opensockets;
357 - unsigned long long openinotifies;
358 - unsigned long long openeventfds;
359 - unsigned long long opentimerfds;
360 - unsigned long long opensignalfds;
361 - unsigned long long openeventpolls;
362 - unsigned long long openanoninodes;
363 - unsigned long long openother;
354 + int *fds;
355 + int fds_size;
356
357 int childs; // number of processes directly referencing this
358 int updated;
@@ -389,6 +381,11 @@ struct pid_stat *get_entry(pid_t pid)
381 return NULL;
382 }
383
384 + all_pids[pid]->fds = calloc(sizeof(int), 100);
385 + if(!all_pids[pid]->fds)
386 + fprintf(stderr, "apps.plugin: ERROR: Cannot allocate %ld bytes of memory\n", sizeof(int) * 100);
387 + else all_pids[pid]->fds_size = 100;
388 +
389 if(root) root->prev = all_pids[pid];
390 all_pids[pid]->next = root;
391 root = all_pids[pid];
@@ -408,10 +405,129 @@ void del_entry(pid_t pid)
405 if(all_pids[pid]->next) all_pids[pid]->next->prev = all_pids[pid]->prev;
406 if(all_pids[pid]->prev) all_pids[pid]->prev->next = all_pids[pid]->next;
407
408 + if(all_pids[pid]->fds) free(all_pids[pid]->fds);
409 free(all_pids[pid]);
410 all_pids[pid] = NULL;
411 }
412
413 +
414 +unsigned long simple_hash(const char *name)
415 +{
416 + int i, len = strlen(name);
417 + unsigned long hash = 0;
418 +
419 + for(i = 0; i < len ;i++) hash += (i * name[i]) + i + name[i];
420 +
421 + return hash;
422 +}
423 +
424 +struct file_descriptor {
425 + unsigned long hash;
426 + char *name;
427 + int type;
428 + long count;
429 +} *all_files = NULL;
430 +
431 +int all_files_len = 0;
432 +int all_files_size = 0;
433 +
434 +#define FILETYPE_OTHER 0
435 +#define FILETYPE_FILE 1
436 +#define FILETYPE_PIPE 2
437 +#define FILETYPE_SOCKET 3
438 +#define FILETYPE_INOTIFY 4
439 +#define FILETYPE_EVENTFD 5
440 +#define FILETYPE_EVENTPOLL 6
441 +#define FILETYPE_TIMERFD 7
442 +#define FILETYPE_SIGNALFD 8
443 +
444 +void file_descriptor_not_used(int id)
445 +{
446 + if(id > 0 && id < all_files_len) {
447 + if(all_files[id].count > 0)
448 + all_files[id].count--;
449 + else
450 + fprintf(stderr, "apps.plugin: ERROR: request to decrease counter of fd %d (%s), while the use counter is 0\n", id, all_files[id].name);
451 + }
452 + else fprintf(stderr, "apps.plugin: ERROR: request to decrease counter of fd %d, which is outside the array size (1 to %d)\n", id, all_files_len);
453 +}
454 +
455 +unsigned long file_descriptor_find_or_add(const char *name)
456 +{
457 + int type = FILETYPE_OTHER;
458 +
459 + if(name[0] == '/') type = FILETYPE_FILE;
460 + else if(strncmp(name, "pipe:", 5) == 0) type = FILETYPE_PIPE;
461 + else if(strncmp(name, "socket:", 7) == 0) type = FILETYPE_SOCKET;
462 + else if(strcmp(name, "anon_inode:inotify") == 0 || strcmp(name, "inotify") == 0) type = FILETYPE_INOTIFY;
463 + else if(strcmp(name, "anon_inode:[eventfd]") == 0) type = FILETYPE_EVENTFD;
464 + else if(strcmp(name, "anon_inode:[eventpoll]") == 0) type = FILETYPE_EVENTPOLL;
465 + else if(strcmp(name, "anon_inode:[timerfd]") == 0) type = FILETYPE_TIMERFD;
466 + else if(strcmp(name, "anon_inode:[signalfd]") == 0) type = FILETYPE_SIGNALFD;
467 + else if(strncmp(name, "anon_inode:", 11) == 0) {
468 + if(debug) fprintf(stderr, "apps.plugin: FIXME: unknown anonymous inode: %s\n", name);
469 + type = FILETYPE_OTHER;
470 + }
471 + else {
472 + if(debug) fprintf(stderr, "apps.plugin: FIXME: cannot understand linkname: %s\n", name);
473 + type = FILETYPE_OTHER;
474 + }
475 +
476 + // init
477 + if(!all_files) {
478 + all_files = malloc(1024 * sizeof(struct file_descriptor));
479 + if(!all_files) return 0;
480 +
481 + all_files_size = 1024;
482 + all_files_len = 1; // start from 1, skip 0
483 + if(debug) fprintf(stderr, "apps.plugin: initialized fd array to %d entries\n", all_files_size);
484 + }
485 +
486 + // try to find it
487 + unsigned long hash = simple_hash(name);
488 + int c;
489 + for( c = 0 ; c < all_files_len ; c++) {
490 + if(all_files[c].hash == hash && strcmp(all_files[c].name, name) == 0) break;
491 + }
492 +
493 + // found it
494 + if(c < all_files_len) {
495 + all_files[c].count++;
496 + return c;
497 + }
498 +
499 + // not found, search for an empty slot
500 + for(c = 0 ; c < all_files_len ; c++) {
501 + if(!all_files[c].count) {
502 + if(debug) fprintf(stderr, "apps.plugin: re-using fd position %d (last name: %s)\n", c, all_files[c].name);
503 + if(all_files[c].name) free(all_files[c].name);
504 + break;
505 + }
506 + }
507 +
508 + if(c == all_files_len) {
509 + // not found any emtpty slot
510 + all_files_len++;
511 +
512 + if(c >= all_files_size) {
513 + // not enough memory - extend it
514 + if(debug) fprintf(stderr, "apps.plugin: extending fd array to %d entries\n", all_files_size + 1024);
515 + all_files = realloc(all_files, (all_files_size + 1024) * sizeof(struct file_descriptor));
516 + all_files_size += 1024;
517 + }
518 + }
519 + // else we have the slot in 'c'
520 +
521 + all_files[c].name = strdup(name);
522 + all_files[c].hash = hash;
523 + all_files[c].type = type;
524 + all_files[c].count++;
525 +
526 + if(debug) fprintf(stderr, "apps.plugin: using fd position %d (name: %s)\n", c, all_files[c].name);
527 +
528 + return c;
529 +}
530 +
531 int update_from_proc(void)
532 {
533 static long count_errors = 0;
@@ -432,17 +548,6 @@ int update_from_proc(void)
548 p->childs = 0;
549 p->merged = 0;
550 p->new_entry = 0;
435 -
436 - p->openfiles = 0;
437 - p->openpipes = 0;
438 - p->opensockets = 0;
439 - p->openinotifies = 0;
440 - p->openeventfds = 0;
441 - p->openeventpolls = 0;
442 - p->opentimerfds = 0;
443 - p->opensignalfds = 0;
444 - p->openanoninodes = 0;
445 - p->openother = 0;
551 }
552
553 while((file = readdir(dir))) {
@@ -639,44 +744,65 @@ int update_from_proc(void)
744 snprintf(filename, FILENAME_MAX, "/proc/%s/fd", file->d_name);
745 DIR *fds = opendir(filename);
746 if(fds) {
747 + int c;
748 struct dirent *de;
749 char fdname[FILENAME_MAX + 1];
750 char linkname[FILENAME_MAX + 1];
751
752 + // make the array negative
753 + for(c = 0 ; c < p->fds_size ; c++) p->fds[c] = -p->fds[c];
754 +
755 while((de = readdir(fds))) {
756 if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
757
649 - sprintf(fdname, "/proc/%s/fd/%s", file->d_name, de->d_name);
650 - int l = readlink(fdname, linkname, FILENAME_MAX);
651 - if(l == -1) {
652 - if(debug || (p->target && p->target->debug)) {
653 - if(!count_errors++ || debug || (p->target && p->target->debug))
654 - fprintf(stderr, "apps.plugin: ERROR: cannot read link %s\n", fdname);
758 + // check if the fds array is small
759 + int fdid = atol(de->d_name);
760 + if(fdid < 0) continue;
761 + if(fdid >= p->fds_size) {
762 + // it is small, extend it
763 + p->fds = realloc(p->fds, (fdid + 100) * sizeof(int));
764 + if(!p->fds) {
765 + fprintf(stderr, "apps.plugin: ERROR: cannot re-allocate fds for %s\n", p->comm);
766 + break;
767 }
656 - continue;
657 - }
658 - file_counter++;
659 -
660 - linkname[l] = '\0';
661 -
662 - if(linkname[0] == '/') p->openfiles++;
663 - else if(strncmp(linkname, "pipe:", 5) == 0) p->openpipes++;
664 - else if(strncmp(linkname, "socket:", 7) == 0) p->opensockets++;
665 - else if(strcmp(linkname, "anon_inode:inotify") == 0 || strcmp(linkname, "inotify") == 0) p->openinotifies++;
666 - else if(strcmp(linkname, "anon_inode:[eventfd]") == 0) p->openeventfds++;
667 - else if(strcmp(linkname, "anon_inode:[eventpoll]") == 0) p->openeventpolls++;
668 - else if(strcmp(linkname, "anon_inode:[timerfd]") == 0) p->opentimerfds++;
669 - else if(strcmp(linkname, "anon_inode:[signalfd]") == 0) p->opensignalfds++;
670 - else if(strncmp(linkname, "anon_inode:", 11) == 0) {
671 - if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: FIXME: unknown anonymous inode: %s\n", linkname);
672 - p->openanoninodes++;
768 +
769 + // and initialize it
770 + for(c = p->fds_size ; c <= (fdid + 100) ; c++) p->fds[c] = 0;
771 + p->fds_size = fdid + 100;
772 }
674 - else {
675 - if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: FIXME: cannot understand linkname: %s\n", linkname);
676 - p->openother++;
773 +
774 + if(p->fds[fdid] == 0) {
775 + // we don't know this fd, get it
776 +
777 + sprintf(fdname, "/proc/%s/fd/%s", file->d_name, de->d_name);
778 + int l = readlink(fdname, linkname, FILENAME_MAX);
779 + if(l == -1) {
780 + if(debug || (p->target && p->target->debug)) {
781 + if(!count_errors++ || debug || (p->target && p->target->debug))
782 + fprintf(stderr, "apps.plugin: ERROR: cannot read link %s\n", fdname);
783 + }
784 + continue;
785 + }
786 + linkname[l] = '\0';
787 + file_counter++;
788 +
789 + // if another process already has this, we will get
790 + // the same id
791 + p->fds[fdid] = file_descriptor_find_or_add(linkname);
792 }
793 +
794 + // else make it positive again, we need it
795 + // of course, the actual file may have changed, but we don't care so much
796 + // FIXME: we could compare the inode as returned by readdir direct structure
797 + else p->fds[fdid] = -p->fds[fdid];
798 }
799 closedir(fds);
800 +
801 + // remove all the negative file descriptors
802 + for(c = 0 ; c < p->fds_size ; c++) if(p->fds[c] < 0) {
803 + file_descriptor_not_used(-p->fds[c]);
804 + p->fds[c] = 0;
805 + }
806 }
807
808 // --------------------------------------------------------------------
@@ -733,6 +859,7 @@ int walk_down(pid_t pid, int level) {
859
860 void update_statistics(void)
861 {
862 + int c;
863 struct pid_stat *p = NULL;
864
865 // link all parents and update childs count
@@ -859,6 +986,10 @@ void update_statistics(void)
986 // zero all the targets
987 struct target *w;
988 for (w = target_root; w ; w = w->next) {
989 + w->fds = calloc(sizeof(int), all_files_len);
990 + if(!w->fds)
991 + fprintf(stderr, "apps.plugin: ERROR: cannot allocate memory for fds in %s\n", w->name);
992 +
993 w->minflt = 0;
994 w->majflt = 0;
995 w->utime = 0;
@@ -886,17 +1017,6 @@ void update_statistics(void)
1017 w->io_storage_bytes_read = 0;
1018 w->io_storage_bytes_written = 0;
1019 w->io_cancelled_write_bytes = 0;
889 -
890 - w->openfiles = 0;
891 - w->openpipes = 0;
892 - w->opensockets = 0;
893 - w->openinotifies = 0;
894 - w->openeventfds = 0;
895 - w->opentimerfds = 0;
896 - w->opensignalfds = 0;
897 - w->openeventpolls = 0;
898 - w->openanoninodes = 0;
899 - w->openother = 0;
1020 }
1021
1022 /* walk_down(0, 1);
@@ -938,19 +1058,17 @@ void update_statistics(void)
1058 p->target->io_storage_bytes_written += p->io_storage_bytes_written;
1059 p->target->io_cancelled_write_bytes += p->io_cancelled_write_bytes;
1060
941 - p->target->openfiles += p->openfiles;
942 - p->target->openpipes += p->openpipes;
943 - p->target->opensockets += p->opensockets;
944 - p->target->openinotifies += p->openinotifies;
945 - p->target->openeventfds += p->openeventfds;
946 - p->target->opentimerfds += p->opentimerfds;
947 - p->target->opensignalfds += p->opensignalfds;
948 - p->target->openeventpolls += p->openeventpolls;
949 - p->target->openanoninodes += p->openanoninodes;
950 - p->target->openother += p->openother;
951 -
1061 p->target->processes++;
1062
1063 + for(c = 0; c < p->fds_size ;c++) {
1064 + if(p->fds[c] == 0) continue;
1065 + if(p->fds[c] > 0 && p->fds[c] < all_files_len) {
1066 + if(p->target->fds) p->target->fds[p->fds[c]]++;
1067 + }
1068 + else
1069 + fprintf(stderr, "apps.plugin: ERROR: invalid fd number %d\n", p->fds[c]);
1070 + }
1071 +
1072 if(debug || p->target->debug) fprintf(stderr, "apps.plugin: \tAgregating %s pid %d on %s utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->comm, p->pid, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
1073
1074 /* if(p->utime - p->old_utime > 100) fprintf(stderr, "BIG CHANGE: %d %s utime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->utime - p->old_utime, p->old_utime, p->utime);
@@ -1000,7 +1118,6 @@ void update_statistics(void)
1118 /* fprintf(stderr, "\n");
1119 */
1120 // cleanup all un-updated processed (exited, killed, etc)
1003 - int c;
1121 for(p = root, c = 0; p ; c++) {
1122 if(!p->updated) {
1123 /* fprintf(stderr, "\tEXITED %d %s [parent %d %s, target %s] utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->pid, p->comm, p->parent->pid, p->parent->comm, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
@@ -1011,6 +1128,60 @@ void update_statistics(void)
1128 }
1129 else p = p->next;
1130 }
1131 +
1132 + for (w = target_root; w ; w = w->next) {
1133 + w->openfiles = 0;
1134 + w->openpipes = 0;
1135 + w->opensockets = 0;
1136 + w->openinotifies = 0;
1137 + w->openeventfds = 0;
1138 + w->opentimerfds = 0;
1139 + w->opensignalfds = 0;
1140 + w->openeventpolls = 0;
1141 + w->openother = 0;
1142 +
1143 + for(c = 1; c < all_files_len ;c++) {
1144 + if(w->fds && w->fds[c] > 0) switch(all_files[c].type) {
1145 + case FILETYPE_FILE:
1146 + w->openfiles++;
1147 + break;
1148 +
1149 + case FILETYPE_PIPE:
1150 + w->openpipes++;
1151 + break;
1152 +
1153 + case FILETYPE_SOCKET:
1154 + w->opensockets++;
1155 + break;
1156 +
1157 + case FILETYPE_INOTIFY:
1158 + w->openinotifies++;
1159 + break;
1160 +
1161 + case FILETYPE_EVENTFD:
1162 + w->openeventfds++;
1163 + break;
1164 +
1165 + case FILETYPE_TIMERFD:
1166 + w->opentimerfds++;
1167 + break;
1168 +
1169 + case FILETYPE_SIGNALFD:
1170 + w->opensignalfds++;
1171 + break;
1172 +
1173 + case FILETYPE_EVENTPOLL:
1174 + w->openeventpolls++;
1175 + break;
1176 +
1177 + default:
1178 + w->openother++;
1179 + }
1180 + }
1181 +
1182 + free(w->fds);
1183 + w->fds = NULL;
1184 + }
1185 }
1186
1187 unsigned long long usecdiff(struct timeval *now, struct timeval *last) {