@cryptotaxi247 / netdata-1 / commits / 961e0b759

mmap() may introduce delays; a first attempt to work around them

Costa Tsaousis (ktsaou) committed May 11, 2014 at 20:11 UTC 961e0b759e6db4ef2bfdb4375c1845cfa60892df

2 files changed +37 -22

apps_plugin.c

+11

@@ -969,6 +969,8 @@ void show_dimensions(void)

969 fprintf(stdout, "SET %s = %llu\n", w->name, w->io_storage_bytes_written);

970 }

971 fprintf(stdout, "END\n");

972 +

973 + fflush(stdout);

974 }

975

976 void show_charts(void)

@@ -1071,6 +1073,8 @@ void show_charts(void)

1073

1074 fprintf(stdout, "DIMENSION %s '' incremental 1 1024\n", w->name);

1075 }

1076 +

1077 + fflush(stdout);

1078 }

1079

1080 unsigned long long get_hertz(void)

@@ -1203,7 +1207,14 @@ int main(int argc, char **argv)

1207 if(usec < (update_every * 1000000ULL / 2)) susec = (update_every * 1000000ULL) - usec;

1208 else susec = update_every * 1000000ULL / 2;

1209

1210 + //fprintf(stderr, "sleeping for %llu...", susec);

1211 + //fflush(NULL);

1212 + //struct timeval before, after;

1213 + //gettimeofday(&before, NULL);

1214 usleep(susec);

1215 + //gettimeofday(&after, NULL);

1216 + //fprintf(stderr, " sleeped for %llu\n", usecdiff(&after, &before));

1217 + //fflush(NULL);

1218

1219 bcopy(&now, &last, sizeof(struct timeval));

1220 }

netdata.c

+26 -22

@@ -952,7 +952,7 @@ void *mymmap(const char *filename, unsigned long size)

952

953 mem = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);

954 if(mem) {

955 - if(madvise(mem, size, MADV_SEQUENTIAL|MADV_DONTFORK) != 0)

955 + if(madvise(mem, size, MADV_SEQUENTIAL|MADV_DONTFORK|MADV_WILLNEED) != 0)

956 error("Cannot advise the kernel about the memory usage of file '%s'.", filename);

957 }

958 }

@@ -1479,6 +1479,7 @@ int rrd_stats_dimension_set(RRD_STATS *st, char *id, collected_number value)

1479

1480 void rrd_stats_next(RRD_STATS *st)

1481 {

1482 + // lock it to work with the dimensions

1483 pthread_mutex_lock(&st->mutex);

1484

1485 RRD_DIMENSION *rd;

@@ -1501,6 +1502,9 @@ void rrd_stats_next(RRD_STATS *st)

1502

1503 unsigned long long rrd_stats_done(RRD_STATS *st)

1504 {

1505 + struct timeval now;

1506 + gettimeofday(&now, NULL);

1507 +

1508 pthread_mutex_lock(&st->mutex);

1509

1510 RRD_DIMENSION *rd, *last;

@@ -1534,7 +1538,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)

1538 st->enabled = 0;

1539 pthread_mutex_unlock(&st->mutex);

1540 // rrd_stats_free(st);

1537 - return st->update_every * 1000000ULL;

1541 + return(st->update_every * 1000000ULL);

1542 }

1543

1544 // calculate totals and count the dimensions

@@ -1543,9 +1547,6 @@ unsigned long long rrd_stats_done(RRD_STATS *st)

1547 for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )

1548 st->absolute_total += rd->collected_value;

1549

1546 - struct timeval now;

1547 - gettimeofday(&now, NULL);

1548 -

1550 // if this is the second+ value we collect

1551 if(st->counter) {

1552

@@ -1704,7 +1705,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)

1705

1706 pthread_mutex_unlock(&st->mutex);

1707

1707 - return st->usec_since_last_update;

1708 + return(st->usec_since_last_update);

1709 }

1710

1711

@@ -6070,7 +6071,6 @@ void *pluginsd_worker_thread(void *arg)

6071 #endif

6072

6073 while(1) {

6073 - // FILE *fp = popen(cd->cmd, "r");

6074 FILE *fp = mypopen(cd->cmd, &cd->pid);

6075 if(!fp) {

6076 error("Cannot popen(\"%s\", \"r\").", cd->cmd);

@@ -6095,20 +6095,20 @@ void *pluginsd_worker_thread(void *arg)

6095 char *value = qstrsep(&p);

6096

6097 if(!dimension || !*dimension || !value) {

6098 - error("PLUGINSD: script %s is requesting a SET on chart '%s', like this: 'SET %s = %s'. Disabling it.", cd->fullfilename, st->id, dimension?dimension:"", value?value:"");

6098 + error("PLUGINSD: '%s' is requesting a SET on chart '%s', like this: 'SET %s = %s'. Disabling it.", cd->fullfilename, st->id, dimension?dimension:"", value?value:"");

6099 cd->enabled = 0;

6100 kill(cd->pid, SIGTERM);

6101 break;

6102 }

6103

6104 if(!st) {

6105 - error("PLUGINSD: script %s is requesting a SET on dimension %s with value %s, without a BEGIN. Disabling it.", cd->fullfilename, dimension, value);

6105 + error("PLUGINSD: '%s' is requesting a SET on dimension %s with value %s, without a BEGIN. Disabling it.", cd->fullfilename, dimension, value);

6106 cd->enabled = 0;

6107 kill(cd->pid, SIGTERM);

6108 break;

6109 }

6110

6111 - if(st->debug) debug(D_PLUGINSD, "PLUGINSD: script %s is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value);

6111 + if(st->debug) debug(D_PLUGINSD, "PLUGINSD: '%s' is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value);

6112 rrd_stats_dimension_set(st, dimension, atoll(value));

6113

6114 count++;

@@ -6116,7 +6116,7 @@ void *pluginsd_worker_thread(void *arg)

6116 else if(!strcmp(s, "BEGIN")) {

6117 char *id = qstrsep(&p);

6118 if(!id) {

6119 - error("PLUGINSD: script %s is requesting a BEGIN without a chart id. Disabling it.", cd->fullfilename);

6119 + error("PLUGINSD: '%s' is requesting a BEGIN without a chart id. Disabling it.", cd->fullfilename);

6120 cd->enabled = 0;

6121 kill(cd->pid, SIGTERM);

6122 break;

@@ -6124,7 +6124,7 @@ void *pluginsd_worker_thread(void *arg)

6124

6125 st = rrd_stats_find(id);

6126 if(!st) {

6127 - error("PLUGINSD: script %s is requesting a BEGIN on chart '%s', which does not exist. Disabling it.", cd->fullfilename, id);

6127 + error("PLUGINSD: '%s' is requesting a BEGIN on chart '%s', which does not exist. Disabling it.", cd->fullfilename, id);

6128 cd->enabled = 0;

6129 kill(cd->pid, SIGTERM);

6130 break;

@@ -6133,18 +6133,18 @@ void *pluginsd_worker_thread(void *arg)

6133 }

6134 else if(!strcmp(s, "END")) {

6135 if(!st) {

6136 - error("PLUGINSD: script %s is requesting an END, without a BEGIN. Disabling it.", cd->fullfilename);

6136 + error("PLUGINSD: '%s' is requesting an END, without a BEGIN. Disabling it.", cd->fullfilename);

6137 cd->enabled = 0;

6138 kill(cd->pid, SIGTERM);

6139 break;

6140 }

6141

6142 - if(st->debug) debug(D_PLUGINSD, "PLUGINSD: script %s is requesting a END on chart %s", cd->fullfilename, st->id);

6142 + if(st->debug) debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a END on chart %s", cd->fullfilename, st->id);

6143

6144 unsigned long long usec_since_last_update = rrd_stats_done(st);

6145 if((time(NULL) - cd->started_t) > 10) {

6146 if(usec_since_last_update < (cd->update_every * 1000000ULL / 10)) {

6147 - error("PLUGINSD: script '%s' (up for %lu secs) updates charts too frequently. Chart %s updated after %llu microseconds, expected %llu microseconds.", cd->fullfilename, time(NULL) - cd->started_t, st->id, usec_since_last_update, cd->update_every * 1000000ULL);

6147 + error("PLUGINSD: '%s' (up for %lu secs) updates charts too frequently. Chart %s updated after %llu microseconds, expected %llu microseconds.", cd->fullfilename, time(NULL) - cd->started_t, st->id, usec_since_last_update, cd->update_every * 1000000ULL);

6148 //cd->enabled = 0;

6149 //kill(cd->pid, SIGTERM);

6150 //break;

@@ -6153,6 +6153,10 @@ void *pluginsd_worker_thread(void *arg)

6153

6154 st = NULL;

6155 }

6156 + else if(!strcmp(s, "FLUSH")) {

6157 + debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename);

6158 + st = NULL;

6159 + }

6160 else if(!strcmp(s, "CHART")) {

6161 st = NULL;

6162

@@ -6172,7 +6176,7 @@ void *pluginsd_worker_thread(void *arg)

6176 char *update_every_s = qstrsep(&p);

6177

6178 if(!type || !*type || !id || !*id) {

6175 - error("PLUGINSD: script %s is requesting a CHART, without a type.id. Disabling it.", cd->fullfilename);

6179 + error("PLUGINSD: '%s' is requesting a CHART, without a type.id. Disabling it.", cd->fullfilename);

6180 cd->enabled = 0;

6181 kill(cd->pid, SIGTERM);

6182 break;

@@ -6220,14 +6224,14 @@ void *pluginsd_worker_thread(void *arg)

6224 char *hidden = qstrsep(&p);

6225

6226 if(!id || !*id) {

6223 - error("PLUGINSD: script %s is requesting a DIMENSION, without an id. Disabling it.", cd->fullfilename);

6227 + error("PLUGINSD: '%s' is requesting a DIMENSION, without an id. Disabling it.", cd->fullfilename);

6228 cd->enabled = 0;

6229 kill(cd->pid, SIGTERM);

6230 break;

6231 }

6232

6233 if(!st) {

6230 - error("PLUGINSD: script %s is requesting a DIMENSION, without a CHART. Disabling it.", cd->fullfilename);

6234 + error("PLUGINSD: '%s' is requesting a DIMENSION, without a CHART. Disabling it.", cd->fullfilename);

6235 cd->enabled = 0;

6236 kill(cd->pid, SIGTERM);

6237 break;

@@ -6261,7 +6265,7 @@ void *pluginsd_worker_thread(void *arg)

6265 else if(st->debug) debug(D_PLUGINSD, "PLUGINSD: dimension %s/%s already exists. Not adding it again.", st->id, id);

6266 }

6267 else if(!strcmp(s, "DISABLE")) {

6264 - error("PLUGINSD: script '%s' called DISABLE. Disabling it.", cd->fullfilename);

6268 + error("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename);

6269 cd->enabled = 0;

6270 kill(cd->pid, SIGTERM);

6271 break;

@@ -6275,7 +6279,7 @@ void *pluginsd_worker_thread(void *arg)

6279 debug(D_PLUGINSD, "PLUGINSD: %s is on pid %d", cd->id, cd->pid);

6280 }

6281 else if(!strcmp(s, "STOPPING_WAKE_ME_UP_PLEASE")) {

6278 - error("PLUGINSD: script '%s' (pid %d) called STOPPING_WAKE_ME_UP_PLEASE.", cd->fullfilename, cd->pid);

6282 + error("PLUGINSD: '%s' (pid %d) called STOPPING_WAKE_ME_UP_PLEASE.", cd->fullfilename, cd->pid);

6283

6284 gettimeofday(&now, NULL);

6285 if(!usec && !susec) {

@@ -6298,7 +6302,7 @@ void *pluginsd_worker_thread(void *arg)

6302 }

6303 #endif

6304 else {

6301 - error("PLUGINSD: script %s is sending command '%s' which is not known by netdata. Disabling it.", cd->fullfilename, s);

6305 + error("PLUGINSD: '%s' is sending command '%s' which is not known by netdata. Disabling it.", cd->fullfilename, s);

6306 cd->enabled = 0;

6307 kill(cd->pid, SIGTERM);

6308 break;

@@ -6309,7 +6313,7 @@ void *pluginsd_worker_thread(void *arg)

6313 mypclose(fp);

6314

6315 if(!count && cd->enabled) {

6312 - error("PLUGINSD: script '%s' does not generate usefull output. Disabling it.", cd->fullfilename);

6316 + error("PLUGINSD: '%s' does not generate usefull output. Disabling it.", cd->fullfilename);

6317 cd->enabled = 0;

6318 kill(cd->pid, SIGTERM);

6319 }