@cryptotaxi247 / netdata-1 / commits / c748305f6

support for custom data collection plugins in charts.d - working example included

Costa Tsaousis (ktsaou) committed May 2, 2014 at 07:07 UTC c748305f6d612cd8bbe6584b0c628c7fd78e6cfa
2 files changed +372 -22
charts.d/skeleton-chart.sh
+6 -6
@@ -24,7 +24,7 @@ echo "MYPID $$"
24 # If you don't want to read it, just skip at the end to see the action...
25 #
26 # CREATE NEW CHART:
27 -# > CHART type.chartname title units family[=chartname] category[=type] charttype[=line] priority[=1000] update_every[=user default]
27 +# > CHART type.id name title units family[=id] category[=type] charttype[=line] priority[=1000] update_every[=user default]
28 #
29 # where
30 # - charttype = line, area or stacked
@@ -32,7 +32,7 @@ echo "MYPID $$"
32 # - category = is the home page section: any name or the word 'none' which hides the chart from the home web page
33 #
34 # create all the dimensions you need
35 -# > DIMENSION CREATE dimensionname algorithm multiplier divisor [hidden]"
35 +# > DIMENSION id name algorithm multiplier divisor [hidden]
36 #
37 # algorithms:
38 # absolute
@@ -90,10 +90,10 @@ echo "MYPID $$"
90
91
92 cat <<EOF
93 -CHART example.random "Random Numbers Example Chart" "random numbers" ExampleFamily ExampleCategory stacked 500 $update_every
94 -DIMENSION number1 absolute 1 1
95 -DIMENSION number2 absolute 1 1
96 -DIMENSION number3 absolute 1 1 hidden
93 +CHART example.random '' "Random Numbers Example Chart" "random numbers" ExampleFamily ExampleCategory stacked 500 $update_every
94 +DIMENSION number1 '' absolute 1 1
95 +DIMENSION number2 '' absolute 1 1
96 +DIMENSION number3 '' absolute 1 1 hidden
97 EOF
98
99 # You can create more charts if you like.
netdata.c
+366 -16
@@ -33,6 +33,7 @@
33 #include <zlib.h>
34 #include <malloc.h>
35 #include <inttypes.h>
36 +#include <dirent.h>
37
38
39 #define RRD_TYPE_NET "net"
@@ -85,6 +86,7 @@
86 #define D_TC_LOOP 0x00000100
87 #define D_DEFLATE 0x00000200
88 #define D_CONFIG 0x00000400
89 +#define D_CHARTSD 0x00000800
90
91 #define CT_APPLICATION_JSON 1
92 #define CT_TEXT_PLAIN 2
@@ -867,7 +869,7 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
869 snprintf(st->id, RRD_STATS_NAME_MAX, "%s.%s", type, id);
870 st->hash = simple_hash(st->id);
871
870 - if(name) {
872 + if(name && *name) {
873 char varvalue[CONFIG_MAX_VALUE + 1];
874 snprintf(varvalue, CONFIG_MAX_VALUE, "%s.%s", type?type:"", name);
875 st->name = config_get(st->id, "name", varvalue);
@@ -877,12 +879,12 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
879 {
880 char varvalue[CONFIG_MAX_VALUE + 1];
881 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
880 - st->title = config_get(st->id, "title", varvalue);
882 + st->title = config_get(st->id, "title", varvalue);
883 }
884
885 st->family = config_get(st->id, "family", family?family:st->id);
886 st->units = config_get(st->id, "units", units?units:"");
885 - st->type = config_get(st->id, "type", type?type:"");
887 + st->type = config_get(st->id, "type", type);
888 st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
889
890 st->entries = config_get_number(st->id, "history", save_history);
@@ -938,7 +940,7 @@ RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char
940 rd->hash = simple_hash(rd->id);
941
942 snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
941 - rd->name = config_get(st->id, varname, name?name:rd->id);
943 + rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
944
945 snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
946 rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
@@ -1093,7 +1095,6 @@ unsigned long long rrd_stats_next(RRD_STATS *st)
1095 st->last_absolute_total = st->absolute_total;
1096 st->absolute_total = 0;
1097 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
1096 - st->counter++;
1098
1099 // leave mutex locked
1100
@@ -1291,6 +1292,8 @@ void rrd_stats_done(RRD_STATS *st)
1292 st->last_updated.tv_sec = now.tv_sec;
1293 st->last_updated.tv_usec = now.tv_usec;
1294
1295 + st->counter++;
1296 +
1297 pthread_mutex_unlock(&st->mutex);
1298 }
1299
@@ -2134,7 +2137,7 @@ int mysendfile(struct web_client *w, char *filename)
2137 char *mystrsep(char **ptr, char *s)
2138 {
2139 char *p = "";
2137 - while ( !p[0] && *ptr ) p = strsep(ptr, s);
2140 + while ( p && !p[0] && *ptr ) p = strsep(ptr, s);
2141 return(p);
2142 }
2143
@@ -3178,11 +3181,14 @@ void *socket_listen_main(void *ptr)
3181 if(FD_ISSET(listener, &ifds)) {
3182 w = web_client_create(listener);
3183
3181 - if(pthread_create(&w->thread, NULL, new_client, w) != 0)
3182 - error("%llu: failed to create new thread.");
3183 -
3184 - if(pthread_detach(w->thread) != 0)
3185 - error("%llu: Cannot request detach of newly created thread.", w->id);
3184 + if(pthread_create(&w->thread, NULL, new_client, w) != 0) {
3185 + error("%llu: failed to create new thread for web client.");
3186 + w->obsolete = 1;
3187 + }
3188 + else if(pthread_detach(w->thread) != 0) {
3189 + error("%llu: Cannot request detach of newly created web client thread.", w->id);
3190 + w->obsolete = 1;
3191 + }
3192
3193 syslog(LOG_NOTICE, "%llu: %s connected", w->id, w->client_ip);
3194 }
@@ -5351,6 +5357,10 @@ void *tc_main(void *ptr)
5357
5358 snprintf(buffer, TC_LINE_MAX, "exec %s %d", config_get("plugin:tc", "script to run to get tc values", "./tc-all.sh"), update_every);
5359 fp = popen(buffer, "r");
5360 + if(!fp) {
5361 + error("Cannot popen(\"%s\", \"r\").", buffer);
5362 + return NULL;
5363 + }
5364
5365 while(fgets(buffer, TC_LINE_MAX, fp) != NULL) {
5366 buffer[TC_LINE_MAX] = '\0';
@@ -5497,7 +5507,318 @@ void *cpuidlejitter_main(void *ptr)
5507 return NULL;
5508 }
5509
5510 +// ----------------------------------------------------------------------------
5511 +// charts.d
5512 +
5513 +#define CHARTS_D_FILE_SUFFIX "-chart.sh"
5514 +#define CHARTS_D_FILE_SUFFIX_LEN strlen(CHARTS_D_FILE_SUFFIX)
5515 +#define CHARTSD_CMD_MAX (FILENAME_MAX*2)
5516 +#define CHARTSD_LINE_MAX 1024
5517 +
5518 +struct chartd {
5519 + char id[RRD_STATS_NAME_MAX+1]; // id
5520 + char filename[FILENAME_MAX+1]; // just the filename
5521 + char fullfilename[FILENAME_MAX+1]; // with path
5522 + char cmd[CHARTSD_CMD_MAX+1]; // the command that is executes
5523 +
5524 + pid_t pid;
5525 + pthread_t thread;
5526 +
5527 + int update_every;
5528 + int obsolete;
5529 +
5530 + struct chartd *next;
5531 +} *chartsd_root = NULL;
5532 +
5533 +// like strsep() but:
5534 +// it trims spaces before and after each value
5535 +// it accepts quoted values in single or double quotes
5536 +char *qstrsep(char **ptr)
5537 +{
5538 + if(!*ptr || !**ptr) return NULL;
5539 +
5540 + char *s, *p = *ptr;
5541 +
5542 + // skip leading spaces
5543 + while(isspace(*p)) p++;
5544 +
5545 + // if the first char is a quote, assume quoted
5546 + if(*p == '"' || *p == '\'') {
5547 + char q = *p;
5548 + s = ++p;
5549 + while(*p && *p != q) p++;
5550 +
5551 + if(*p == q) {
5552 + *p = '\0';
5553 + p++;
5554 + }
5555 +
5556 + *ptr = p;
5557 + return s;
5558 + }
5559 +
5560 + s = p;
5561 + while(*p && !isspace(*p)) p++;
5562 + if(!*p) *ptr = NULL;
5563 + else {
5564 + *p = '\0';
5565 + *ptr = ++p;
5566 + }
5567 +
5568 + return s;
5569 +}
5570 +
5571 +void *chartsd_worker_thread(void *arg)
5572 +{
5573 + struct chartd *cd = (struct chartd *)arg;
5574 + char line[CHARTSD_LINE_MAX + 1];
5575 +
5576 + cd->update_every = config_get_number(cd->id, "update every", update_every);
5577 + snprintf(cd->cmd, CHARTSD_CMD_MAX, "exec %s %d", cd->fullfilename, cd->update_every);
5578 +
5579 + while(1) {
5580 + FILE *fp = popen(cd->cmd, "r");
5581 + if(!fp) {
5582 + error("Cannot popen(\"%s\", \"r\").", cd->cmd);
5583 + break;
5584 + }
5585 +
5586 + RRD_STATS *st = NULL;
5587 +
5588 + unsigned long long count = 0;
5589 + while(fgets(line, CHARTSD_LINE_MAX, fp) != NULL) {
5590 + char *p = line;
5591 + char *s = qstrsep(&p);
5592 +
5593 + if(!strcmp(s, "SET")) {
5594 + char *dimension = qstrsep(&p);
5595 + char *equal = qstrsep(&p);
5596 + char *value = qstrsep(&p);
5597 +
5598 + if(!dimension || !equal || *equal != '=' || !value) {
5599 + error("CHARTSD: script %s is requesting a SET on chart '%s', like this: 'SET %s %s %s %s'", cd->fullfilename, st->id, dimension?dimension:"", equal?equal:"", value?value:"");
5600 + continue;
5601 + }
5602 +
5603 + if(!st) {
5604 + error("CHARTSD: script %s is requesting a SET, without a BEGIN", cd->fullfilename);
5605 + continue;
5606 + }
5607 +
5608 + debug(D_CHARTSD, "CHARTSD: script %s is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value);
5609 +
5610 + rrd_stats_dimension_set(st, dimension, atoll(value));
5611 + }
5612 + else if(!strcmp(s, "BEGIN")) {
5613 + char *id = qstrsep(&p);
5614 + if(!id) {
5615 + error("CHARTSD: script %s is requesting a BEGIN without a chart id", cd->fullfilename);
5616 + continue;
5617 + }
5618 +
5619 + st = rrd_stats_find(id);
5620 + if(!st) {
5621 + error("CHARTSD: script %s is requesting a BEGIN on chart '%s', which does not exist", cd->fullfilename, id);
5622 + continue;
5623 + }
5624 + if(st->counter) rrd_stats_next(st);
5625 + }
5626 + else if(!strcmp(s, "END")) {
5627 + if(!st) {
5628 + error("CHARTSD: script %s is requesting an END, without a BEGIN", cd->fullfilename);
5629 + continue;
5630 + }
5631 +
5632 + debug(D_CHARTSD, "CHARTSD: script %s is requesting a END on chart %s", cd->fullfilename, st->id);
5633 +
5634 + rrd_stats_done(st);
5635 + st = NULL;
5636 + }
5637 + else if(!strcmp(s, "CHART")) {
5638 + st = NULL;
5639 +
5640 + char *type = qstrsep(&p);
5641 + char *id = NULL;
5642 + if(type) {
5643 + id = strchr(type, '.');
5644 + if(id) { *id = '\0'; id++; }
5645 + }
5646 + char *name = qstrsep(&p);
5647 + char *title = qstrsep(&p);
5648 + char *units = qstrsep(&p);
5649 + char *family = qstrsep(&p);
5650 + char *category = qstrsep(&p);
5651 + char *chart = qstrsep(&p);
5652 + char *priority_s = qstrsep(&p);
5653 + char *update_every_s = qstrsep(&p);
5654 +
5655 + if(!type || !*type || !id || !*id) {
5656 + error("CHARTSD: script %s is requesting a CHART, without a type.id", cd->fullfilename);
5657 + continue;
5658 + }
5659 +
5660 + int priority = 1000;
5661 + if(priority_s) priority = atoi(priority_s);
5662 +
5663 + int update_every = cd->update_every;
5664 + if(update_every_s) update_every = atoi(update_every_s);
5665 + if(!update_every) update_every = cd->update_every;
5666 +
5667 + int chart_type = CHART_TYPE_LINE;
5668 + if(chart) chart_type = chart_type_id(chart);
5669 +
5670 + if(!name || !*name) name = id;
5671 + if(!family || !*family) family = id;
5672 + if(!category || !*category) category = type;
5673 +
5674 + debug(D_CHARTSD, "CHARTSD: Creating chart type='%s', id='%s', name='%s', family='%s', category='%s', chart='%s', priority=%d, update_every=%d"
5675 + , type, id
5676 + , name?name:""
5677 + , family?family:""
5678 + , category?category:""
5679 + , chart_type_name(chart_type)
5680 + , priority
5681 + , update_every
5682 + );
5683 +
5684 + st = rrd_stats_create(type, id, name, family, title, units, priority, update_every, chart_type);
5685 + cd->update_every = update_every;
5686 + }
5687 + else if(!strcmp(s, "DIMENSION")) {
5688 + char *id = qstrsep(&p);
5689 + char *name = qstrsep(&p);
5690 + char *algorithm = qstrsep(&p);
5691 + char *multiplier_s = qstrsep(&p);
5692 + char *divisor_s = qstrsep(&p);
5693 + char *hidden = qstrsep(&p);
5694 +
5695 + if(!id || !*id) {
5696 + error("CHARTSD: script %s is requesting a DIMENSION, without an id", cd->fullfilename);
5697 + continue;
5698 + }
5699 +
5700 + if(!st) {
5701 + error("CHARTSD: script %s is requesting an DIMENSION, without a CHART", cd->fullfilename);
5702 + continue;
5703 + }
5704 +
5705 + long multiplier = 1;
5706 + if(multiplier_s && *multiplier_s) multiplier = atol(multiplier_s);
5707 + if(!multiplier) multiplier = 1;
5708 +
5709 + long divisor = 1;
5710 + if(divisor_s && *divisor_s) divisor = atol(divisor_s);
5711 + if(!divisor) divisor = 1;
5712 +
5713 + if(!algorithm || !*algorithm) algorithm = "absolute";
5714 +
5715 + debug(D_CHARTSD, "CHARTSD: Creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'"
5716 + , st->id
5717 + , id
5718 + , name?name:""
5719 + , algorithm_name(algorithm_id(algorithm))
5720 + , multiplier
5721 + , divisor
5722 + , hidden?hidden:""
5723 + );
5724 +
5725 + RRD_DIMENSION *rd = rrd_stats_dimension_add(st, id, name, multiplier, divisor, algorithm_id(algorithm));
5726 + if(hidden && strcmp(hidden, "hidden") == 0)
5727 + rd->hidden = 1;
5728 + }
5729 + else if(!strcmp(s, "MYPID")) {
5730 + char *pid = qstrsep(&p);
5731 + cd->pid = atol(pid);
5732 + debug(D_CHARTSD, "CHARTSD: %s is on pid %d", cd->id, cd->pid);
5733 + }
5734 + else error("CHARTSD: script %s is sending command '%s' which is not known by netdata", cd->fullfilename, s);
5735 +
5736 + count++;
5737 + }
5738 + if(!count) error("Script '%s' does not generate any output.", cd->fullfilename);
5739 + pclose(fp);
5740 +
5741 + sleep(cd->update_every);
5742 + }
5743 +
5744 + cd->obsolete = 1;
5745 + return NULL;
5746 +}
5747 +
5748 +void *chartsd_main(void *ptr)
5749 +{
5750 + char *dir_name = config_get("plugin:charts.d", "charts.d directory", "charts.d");
5751 + DIR *dir = NULL;
5752 + struct dirent *file = NULL;
5753 + struct chartd *cd;
5754 +
5755 + while(1) {
5756 + dir = opendir(dir_name);
5757 + if(!dir) {
5758 + error("Cannot open directory '%s'.", dir_name);
5759 + return NULL;
5760 + }
5761 +
5762 + while((file = readdir(dir))) {
5763 + debug(D_CHARTSD, "CHARTSD: Examining file '%s'", file->d_name);
5764 +
5765 + if(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) continue;
5766 +
5767 + int len = strlen(file->d_name);
5768 + if(len <= CHARTS_D_FILE_SUFFIX_LEN) continue;
5769 + if(strcmp(CHARTS_D_FILE_SUFFIX, &file->d_name[len - CHARTS_D_FILE_SUFFIX_LEN]) != 0) {
5770 + debug(D_CHARTSD, "CHARTSD: File '%s' does not end in '%s'.", file->d_name, CHARTS_D_FILE_SUFFIX);
5771 + continue;
5772 + }
5773 +
5774 + char buf[CONFIG_MAX_NAME + 1];
5775 + snprintf(buf, CONFIG_MAX_NAME, "plugin %.*s", (int)(len - CHARTS_D_FILE_SUFFIX_LEN), file->d_name);
5776 + if(!config_get_boolean("plugin:charts.d", buf, 1)) continue;
5777 +
5778 + // check if it runs already
5779 + for(cd = chartsd_root ; cd ; cd = cd->next) {
5780 + if(strcmp(cd->filename, file->d_name) == 0) break;
5781 + }
5782 + if(cd && !cd->obsolete) {
5783 + debug(D_CHARTSD, "CHARTSD: %s is already running", cd->filename);
5784 + continue;
5785 + }
5786 +
5787 + // it is not running
5788 + // allocate a new one, or use the obsolete one
5789 + if(!cd) {
5790 + cd = calloc(sizeof(struct chartd), 1);
5791 + if(!cd) fatal("Cannot allocate memory for chart.d");
5792 +
5793 + // link it
5794 + if(chartsd_root) cd->next = chartsd_root;
5795 + chartsd_root = cd;
5796 + }
5797 +
5798 + strncpy(cd->filename, file->d_name, FILENAME_MAX);
5799 + snprintf(cd->id, RRD_STATS_NAME_MAX, "plugin:charts.d:%.*s", (int)(len - CHARTS_D_FILE_SUFFIX_LEN), file->d_name);
5800 + snprintf(cd->fullfilename, FILENAME_MAX, "%s/%s", dir_name, cd->filename);
5801 + cd->obsolete = 0;
5802 +
5803 + // spawn a new thread for it
5804 + if(pthread_create(&cd->thread, NULL, chartsd_worker_thread, cd) != 0) {
5805 + error("CHARTS.D: failed to create new thread for chart.d %s.", cd->filename);
5806 + cd->obsolete = 1;
5807 + }
5808 + else if(pthread_detach(cd->thread) != 0)
5809 + error("CHARTS.D: Cannot request detach of newly created thread for chart.d %s.", cd->filename);
5810 + }
5811 +
5812 + closedir(dir);
5813 + sleep(60);
5814 + }
5815 +
5816 + return NULL;
5817 +}
5818 +
5819
5820 +// ----------------------------------------------------------------------------
5821 +// main and related functions
5822
5823 void bye(void)
5824 {
@@ -5517,6 +5838,11 @@ void sig_handler(int signo)
5838 error("Signaled exit (signal %d).", signo);
5839 if(tc_child_pid) kill(tc_child_pid, SIGTERM);
5840 tc_child_pid = 0;
5841 +
5842 + struct chartd *cd;
5843 + for(cd = chartsd_root ; cd ; cd = cd->next)
5844 + if(cd->pid) kill(cd->pid, SIGTERM);
5845 +
5846 exit(1);
5847 break;
5848
@@ -5744,13 +6070,37 @@ int main(int argc, char **argv)
6070 for (i = 1 ; i < 65 ;i++) if(i != SIGSEGV) signal(i, sig_handler);
6071
6072
5747 - pthread_t p_proc, p_tc, p_jitter;
6073 + pthread_t p_proc, p_tc, p_jitter, p_chartsd;
6074 +
6075 + // spawn childs to collect data
6076 + if(config_get_boolean("plugins", "tc", 1)) {
6077 + if(pthread_create(&p_tc, NULL, tc_main, NULL))
6078 + error("failed to create new thread for tc.");
6079 + else if(pthread_detach(p_tc))
6080 + error("Cannot request detach of newly created tc thread.");
6081 + }
6082 +
6083 + if(config_get_boolean("plugins", "idlejitter", 1)) {
6084 + if(pthread_create(&p_jitter, NULL, cpuidlejitter_main, NULL))
6085 + error("failed to create new thread for idlejitter.");
6086 + else if(pthread_detach(p_jitter))
6087 + error("Cannot request detach of newly created idlejitter thread.");
6088 + }
6089
5749 - // spawn a child to collect data
5750 - if(config_get_boolean("plugins", "tc", 1)) pthread_create(&p_tc, NULL, tc_main, NULL);
5751 - if(config_get_boolean("plugins", "idlejitter", 1)) pthread_create(&p_jitter, NULL, cpuidlejitter_main, NULL);
5752 - if(config_get_boolean("plugins", "proc", 1)) pthread_create(&p_proc, NULL, proc_main, NULL);
6090 + if(config_get_boolean("plugins", "proc", 1)) {
6091 + if(pthread_create(&p_proc, NULL, proc_main, NULL))
6092 + error("failed to create new thread for proc.");
6093 + else if(pthread_detach(p_proc))
6094 + error("Cannot request detach of newly created proc thread.");
6095 + }
6096
6097 + if(config_get_boolean("plugins", "charts.d", 1)) {
6098 + if(pthread_create(&p_chartsd, NULL, chartsd_main, NULL))
6099 + error("failed to create new thread for charts.d.");
6100 + else if(pthread_detach(p_chartsd))
6101 + error("Cannot request detach of newly created charts.d thread.");
6102 + }
6103 +
6104 // the main process - the web server listener
6105 // this never ends
6106 socket_listen_main(NULL);