@cryptotaxi247 / netdata-1 / commits / 76eafaa8c

first complete API implementation - it now supports multiple different visualizations

Costa Tsaousis (ktsaou) committed Nov 29, 2015 at 03:51 UTC 76eafaa8cdae91d6c7447ccfbae63f152aaa0767
6 files changed +716 -70
src/avl.c
+1
@@ -19,6 +19,7 @@
19 #include "avl.h"
20
21 /* Private methods */
22 +int _avl_removeroot(avl_tree* t);
23
24 /* Swing to the left
25 * Warning: no balance maintainance
src/rrd2json.c
+476 -41
@@ -4,6 +4,7 @@
4 #include <pthread.h>
5 #include <sys/time.h>
6 #include <stdlib.h>
7 +#include <string.h>
8
9 #include "log.h"
10 #include "common.h"
@@ -14,8 +15,6 @@ char *hostname = "unknown";
15
16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
17 {
17 - time_t now = time(NULL);
18 -
18 pthread_rwlock_rdlock(&st->rwlock);
19
20 buffer_sprintf(wb,
@@ -261,10 +260,10 @@ void rrd_stats_all_json(BUFFER *wb)
260 // ----------------------------------------------------------------------------
261
262 // RRDR options
264 -#define RRDR_EMPTY 0x01
265 -#define RRDR_RESET 0x02
266 -#define RRDR_HIDDEN 0x04
267 -#define RRDR_NONZERO 0x08
263 +#define RRDR_EMPTY 0x01 // the dimension contains / the value is empty (null)
264 +#define RRDR_RESET 0x02 // the dimension contains / the value is reset
265 +#define RRDR_HIDDEN 0x04 // the dimension contains / the value is hidden
266 +#define RRDR_NONZERO 0x08 // the dimension contains / the value is non-zero
267
268
269 typedef struct rrdresult {
@@ -279,11 +278,14 @@ typedef struct rrdresult {
278 calculated_number *v; // array n x d values
279 uint8_t *o; // array n x d options
280
282 - int c; // current line ( 0 ~ n )
281 + int c; // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
282
283 int has_st_lock; // if st is read locked by us
284 } RRDR;
285
286 +#define rrdr_rows(r) ((r)->c + 1)
287 +
288 +/*
289 static void rrdr_dump(RRDR *r)
290 {
291 long c, i;
@@ -337,6 +339,353 @@ static void rrdr_dump(RRDR *r)
339 fprintf(stderr, "\n");
340 }
341 }
342 +*/
343 +
344 +void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
345 +{
346 + char b[strlen(dims) + 1];
347 + char *o = b, *tok;
348 + strcpy(o, dims);
349 +
350 + long c;
351 + RRDDIM *d;
352 +
353 + // disable all of them
354 + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
355 + r->od[c] |= RRDR_HIDDEN;
356 +
357 + while(o && *o && (tok = mystrsep(&o, ", |"))) {
358 + if(!*tok) continue;
359 +
360 + // find it and enable it
361 + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
362 + if(!strcmp(d->name, tok)) {
363 + r->od[c] &= ~RRDR_HIDDEN;
364 + }
365 + }
366 + }
367 +}
368 +
369 +#define JSON_DATES_JS 1
370 +#define JSON_DATES_TIMESTAMP 2
371 +
372 +static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int google)
373 +{
374 + int annotations = 0, dates = JSON_DATES_JS;
375 + char kq[2] = "", // key quote
376 + sq[2] = "", // string quote
377 + pre_label[101] = "", // before each label
378 + post_label[101] = "", // after each label
379 + pre_date[101] = "", // the beginning of line, to the date
380 + post_date[101] = "", // closing the date
381 + pre_value[101] = "", // before each value
382 + post_value[101] = "", // after each value
383 + post_line[101] = "", // at the end of each row
384 + normal_annotation[201] = "", // default row annotation
385 + overflow_annotation[201] = "", // overflow row annotation
386 + data_begin[101] = "", // between labels and values
387 + finish[101] = ""; // at the end of everything
388 +
389 + if(google) {
390 + dates = JSON_DATES_JS;
391 + kq[0] = '\0';
392 + sq[0] = '\'';
393 + annotations = 1;
394 + snprintf(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
395 + snprintf(post_date, 100, "%s}", sq);
396 + snprintf(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
397 + snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
398 + snprintf(pre_value, 100, ",{%sv%s:", kq, kq);
399 + snprintf(post_value, 100, "}");
400 + snprintf(post_line, 100, "]}");
401 + snprintf(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq);
402 + snprintf(finish, 100, "\n ]\n}\n");
403 +
404 + snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
405 + snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
406 +
407 + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
408 + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
409 + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
410 + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
411 + }
412 + else {
413 + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
414 + dates = JSON_DATES_TIMESTAMP;
415 + snprintf(pre_date, 100, " [");
416 + }
417 + else {
418 + dates = JSON_DATES_JS;
419 + snprintf(pre_date, 100, " [new ");
420 + }
421 + kq[0] = '"';
422 + sq[0] = '"';
423 + snprintf(pre_label, 100, ", \"");
424 + snprintf(post_label, 100, "\"");
425 + snprintf(pre_value, 100, ", ");
426 + snprintf(post_line, 100, "]");
427 + snprintf(data_begin, 100, "],\n %sdata%s:\n [\n", kq, kq);
428 + snprintf(finish, 100, "\n ]\n}\n");
429 +
430 + buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
431 + buffer_sprintf(wb, "%stime%s", sq, sq);
432 + }
433 +
434 + // -------------------------------------------------------------------------
435 + // print the JSON header
436 +
437 + long c, i;
438 + RRDDIM *rd;
439 +
440 + // print the csv header
441 + for(c = 0, i = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
442 + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
443 + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
444 +
445 + buffer_strcat(wb, pre_label);
446 + buffer_strcat(wb, rd->name);
447 + buffer_strcat(wb, post_label);
448 + i++;
449 + }
450 + if(!i) {
451 + buffer_strcat(wb, pre_label);
452 + buffer_strcat(wb, "no data");
453 + buffer_strcat(wb, post_label);
454 + }
455 +
456 + // print the begin of row data
457 + buffer_strcat(wb, data_begin);
458 +
459 + // if all dimensions are hidden, print a null
460 + if(!i) {
461 + buffer_strcat(wb, pre_value);
462 + if(options & RRDR_OPTION_NULL2ZERO)
463 + buffer_strcat(wb, "0");
464 + else
465 + buffer_strcat(wb, "null");
466 + buffer_strcat(wb, post_value);
467 + }
468 +
469 + long start = 0, end = rrdr_rows(r), step = 1;
470 + if((options & RRDR_OPTION_REVERSED)) {
471 + start = rrdr_rows(r) - 1;
472 + end = -1;
473 + step = -1;
474 + }
475 +
476 + // for each line in the array
477 + for(i = start; i != end ;i += step) {
478 + calculated_number *cn = &r->v[ i * r->d ];
479 + uint8_t *co = &r->o[ i * r->d ];
480 +
481 + time_t now = r->t[i];
482 +
483 + if(dates == JSON_DATES_JS) {
484 + // generate the local date time
485 + struct tm *tm = localtime(&now);
486 + if(!tm) { error("localtime() failed."); continue; }
487 +
488 + if(likely(i != start)) buffer_strcat(wb, ",\n");
489 + buffer_strcat(wb, pre_date);
490 + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
491 + buffer_strcat(wb, post_date);
492 +
493 + if(annotations) {
494 + if(co[c] & RRDR_RESET)
495 + buffer_strcat(wb, overflow_annotation);
496 + else
497 + buffer_strcat(wb, normal_annotation);
498 + }
499 + }
500 + else {
501 + // print the timestamp of the line
502 + if(likely(i != start)) buffer_strcat(wb, ",\n");
503 + buffer_strcat(wb, pre_date);
504 + buffer_rrd_value(wb, (calculated_number)r->t[i]);
505 + // in ms
506 + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
507 + buffer_strcat(wb, post_date);
508 + }
509 +
510 + // for each dimension
511 + for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
512 + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
513 + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
514 +
515 + calculated_number n = cn[c];
516 +
517 + if(co[c] & RRDR_EMPTY) {
518 + buffer_strcat(wb, pre_value);
519 + if(options & RRDR_OPTION_NULL2ZERO)
520 + buffer_strcat(wb, "0");
521 + else
522 + buffer_strcat(wb, "null");
523 + buffer_strcat(wb, post_value);
524 + }
525 + else if((options & RRDR_OPTION_ABSOLUTE)) {
526 + buffer_strcat(wb, pre_value);
527 + buffer_rrd_value(wb, (n<0)?-n:n);
528 + buffer_strcat(wb, post_value);
529 + }
530 + else {
531 + buffer_strcat(wb, pre_value);
532 + buffer_rrd_value(wb, n);
533 + buffer_strcat(wb, post_value);
534 + }
535 + }
536 +
537 + buffer_strcat(wb, post_line);
538 + }
539 +
540 + buffer_strcat(wb, finish);
541 +}
542 +
543 +static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline)
544 +{
545 + long c, i;
546 + RRDDIM *d;
547 +
548 + // print the csv header
549 + for(c = 0, i = 0, d = r->st->dimensions; d ;c++, d = d->next) {
550 + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
551 + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
552 +
553 + if(!i) {
554 + buffer_strcat(wb, startline);
555 + buffer_strcat(wb, "time");
556 + }
557 + buffer_strcat(wb, separator);
558 + buffer_strcat(wb, d->name);
559 + i++;
560 + }
561 + buffer_strcat(wb, endline);
562 +
563 + if(!i) {
564 + // no dimensions present
565 + return;
566 + }
567 +
568 + long start = 0, end = rrdr_rows(r), step = 1;
569 + if((options & RRDR_OPTION_REVERSED)) {
570 + start = rrdr_rows(r) - 1;
571 + end = -1;
572 + step = -1;
573 + }
574 +
575 + // for each line in the array
576 + for(i = start; i != end ;i += step) {
577 + calculated_number *cn = &r->v[ i * r->d ];
578 + uint8_t *co = &r->o[ i * r->d ];
579 +
580 + buffer_strcat(wb, startline);
581 +
582 + time_t now = r->t[i];
583 +
584 + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
585 + // print the timestamp of the line
586 + buffer_rrd_value(wb, (calculated_number)now);
587 + // in ms
588 + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
589 + }
590 + else {
591 + // generate the local date time
592 + struct tm *tm = localtime(&now);
593 + if(!tm) { error("localtime() failed."); continue; }
594 + buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
595 + }
596 +
597 + // for each dimension
598 + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
599 + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
600 + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
601 +
602 + buffer_strcat(wb, separator);
603 +
604 + calculated_number n = cn[c];
605 +
606 + if(co[c] & RRDR_EMPTY) {
607 + if(options & RRDR_OPTION_NULL2ZERO)
608 + buffer_strcat(wb, "0");
609 + else
610 + buffer_strcat(wb, "null");
611 + }
612 + else if((options & RRDR_OPTION_ABSOLUTE))
613 + buffer_rrd_value(wb, (n<0)?-n:n);
614 + else
615 + buffer_rrd_value(wb, n);
616 + }
617 +
618 + buffer_strcat(wb, endline);
619 + }
620 +}
621 +
622 +static void rrdr2ssv(RRDR *r, BUFFER *out, uint32_t options, const char *prefix, const char *separator, const char *suffix)
623 +{
624 + long c, i;
625 + RRDDIM *d;
626 +
627 + buffer_strcat(out, prefix);
628 + long start = 0, end = rrdr_rows(r), step = 1;
629 + if((options & RRDR_OPTION_REVERSED)) {
630 + start = rrdr_rows(r) - 1;
631 + end = -1;
632 + step = -1;
633 + }
634 +
635 + // for each line in the array
636 + for(i = start; i != end ;i += step) {
637 +
638 + calculated_number *cn = &r->v[ i * r->d ];
639 + uint8_t *co = &r->o[ i * r->d ];
640 +
641 + calculated_number sum = 0, min = 0, max = 0;
642 + int all_null = 1, init = 1;
643 +
644 + // for each dimension
645 + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
646 + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
647 + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
648 +
649 + calculated_number n = cn[c];
650 +
651 + if(unlikely(init)) {
652 + if(n > 0) {
653 + min = 0;
654 + max = n;
655 + }
656 + else {
657 + min = n;
658 + max = 0;
659 + }
660 + init = 0;
661 + }
662 +
663 + if(likely(!(co[c] & RRDR_EMPTY))) {
664 + all_null = 0;
665 + if((options & RRDR_OPTION_ABSOLUTE) && n < 0) n = -n;
666 + sum += n;
667 + }
668 +
669 + if(n < min) min = n;
670 + if(n > max) max = n;
671 + }
672 +
673 + if(likely(i != start))
674 + buffer_strcat(out, separator);
675 +
676 + if(all_null) {
677 + if(options & RRDR_OPTION_NULL2ZERO)
678 + buffer_strcat(out, "0");
679 + else
680 + buffer_strcat(out, "null");
681 + }
682 + else if(options & RRDR_OPTION_MIN2MAX)
683 + buffer_rrd_value(out, max - min);
684 + else
685 + buffer_rrd_value(out, sum);
686 + }
687 + buffer_strcat(out, suffix);
688 +}
689
690 inline static calculated_number *rrdr_line_values(RRDR *r)
691 {
@@ -442,13 +791,18 @@ cleanup:
791 return NULL;
792 }
793
445 -RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method)
794 +RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
795 {
796 int debug = st->debug;
797
798 time_t first_entry_t = rrdset_first_entry_t(st);
799 time_t last_entry_t = rrdset_last_entry_t(st);
800
801 + if(before == 0 && after == 0) {
802 + before = last_entry_t;
803 + after = first_entry_t;
804 + }
805 +
806 // allow relative for before and after
807 if(before <= st->update_every * st->entries) before = last_entry_t + before;
808 if(after <= st->update_every * st->entries) after = last_entry_t + after;
@@ -462,9 +816,9 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
816
817 // check if they are upside down
818 if(after > before) {
465 - time_t t = before;
819 + time_t tmp = before;
820 before = after;
467 - after = t;
821 + after = tmp;
822 }
823
824 // the duration of the chart
@@ -544,36 +898,49 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
898 // -------------------------------------------------------------------------
899 // the main loop
900
547 - long t = rrdset_time2slot(st, before), // rrdset_last_slot(st),
548 - stop_at_t = rrdset_time2slot(st, after),
549 - added = 0,
550 - group_count = 0,
551 - add_this = 0,
552 - stop_now = 0;
901 + long start_at_slot = rrdset_time2slot(st, before), // rrdset_last_slot(st),
902 + stop_at_slot = rrdset_time2slot(st, after);
903
554 - // align to group for proper panning of data
555 - t -= t % group;
556 -
557 - time_t now = rrdset_slot2time(st, t),
904 + time_t now = rrdset_slot2time(st, start_at_slot),
905 dt = st->update_every,
906 group_start_t = 0;
907
561 - if(debug) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop slot %ld), before_t: %lu (start slot %ld), start_t(now): %lu"
908 + if(unlikely(debug)) debug(D_RRD_STATS, "INFO %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
909 + , st->id
910 + , after
911 + , stop_at_slot
912 + , before
913 + , start_at_slot
914 + , now
915 + , st->current_entry
916 + , st->entries
917 + );
918 +
919 + // align to group for proper panning of data
920 + start_at_slot -= start_at_slot % group;
921 + stop_at_slot -= stop_at_slot % group;
922 + now = rrdset_slot2time(st, start_at_slot);
923 +
924 + if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
925 , st->id
926 , after
564 - , stop_at_t
927 + , stop_at_slot
928 , before
566 - , t
929 + , start_at_slot
930 , now
931 + , st->current_entry
932 + , st->entries
933 );
934
570 - for( ; !stop_now ; now -= dt, t--) {
571 - if(unlikely(t < 0)) c = st->entries - 1;
572 - if(t == stop_at_t) stop_now = 1;
935 + long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
936 + for(; !stop_now ; now -= dt, slot--, counter++) {
937 + if(unlikely(slot < 0)) slot = st->entries - 1;
938 + if(unlikely(slot == stop_at_slot)) stop_now = counter;
939
574 - if(debug) debug(D_RRD_STATS, "ROW %s c: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
940 + if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
941 , st->id
576 - , t
942 + , slot
943 + , counter
944 , group_count + 1
945 , added
946 , now
@@ -585,7 +952,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
952 if(unlikely(now > before)) continue;
953 if(unlikely(now < after)) break;
954
588 - if(group_count == 0) group_start_t = now;
955 + if(unlikely(group_count == 0)) group_start_t = now;
956 group_count++;
957
958 if(unlikely(group_count == group)) {
@@ -595,13 +962,13 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
962
963 // do the calculations
964 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
598 - storage_number n = rd->values[t];
965 + storage_number n = rd->values[slot];
966 if(unlikely(!does_storage_number_exist(n))) continue;
967
968 group_counts[c]++;
969
970 calculated_number value = unpack_storage_number(n);
604 - if(value != 0.0) {
971 + if(likely(value != 0.0)) {
972 group_options[c] |= RRDR_NONZERO;
973 found_non_zero[c] = 1;
974 }
@@ -625,7 +992,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
992
993 // added it
994 if(unlikely(add_this)) {
628 - if(!rrdr_line_init(r, group_start_t)) break;
995 + if(unlikely(!rrdr_line_init(r, group_start_t))) break;
996
997 calculated_number *cn = rrdr_line_values(r);
998 uint8_t *co = rrdr_line_options(r);
@@ -633,14 +1000,14 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
1000 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
1001
1002 // update the dimension options
636 - if(found_non_zero[c]) r->od[c] |= RRDR_NONZERO;
637 - if(rd->flags & RRDDIM_FLAG_HIDDEN) r->od[c] |= RRDR_HIDDEN;
1003 + if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1004 + if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] |= RRDR_HIDDEN;
1005
1006 // store the specific point options
1007 co[c] = group_options[c];
1008
1009 // store the value
643 - if(group_counts[c] == 0) {
1010 + if(unlikely(group_counts[c] == 0)) {
1011 cn[c] = 0.0;
1012 co[c] |= RRDR_EMPTY;
1013 }
@@ -663,9 +1030,80 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
1030 }
1031 }
1032
666 - rrdr_dump(r);
667 - rrdr_free(r);
668 - return NULL;
1033 + return r;
1034 +}
1035 +
1036 +int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1037 +{
1038 + RRDR *rrdr = rrd2rrdr(st, points, after, before, group_method);
1039 + if(!rrdr) {
1040 + buffer_strcat(out, "Cannot generate output with these parameters on this chart.");
1041 + return 500;
1042 + }
1043 +
1044 + if(dimensions)
1045 + rrdr_disable_not_selected_dimensions(rrdr, buffer_tostring(dimensions));
1046 +
1047 + if(latest_timestamp && rrdr_rows(rrdr) > 0)
1048 + *latest_timestamp = rrdr->t[rrdr_rows(rrdr) - 1];
1049 +
1050 + switch(format) {
1051 + case DATASOURCE_SSV:
1052 + out->contenttype = CT_TEXT_PLAIN;
1053 + rrdr2ssv(rrdr, out, options, "", " ", "");
1054 + break;
1055 +
1056 + case DATASOURCE_SSV_COMMA:
1057 + out->contenttype = CT_TEXT_PLAIN;
1058 + rrdr2ssv(rrdr, out, options, "", ",", "");
1059 + break;
1060 +
1061 + case DATASOURCE_JS_ARRAY:
1062 + out->contenttype = CT_APPLICATION_JSON;
1063 + rrdr2ssv(rrdr, out, options, "[", ",", "]");
1064 + break;
1065 +
1066 + case DATASOURCE_CSV:
1067 + out->contenttype = CT_TEXT_PLAIN;
1068 + rrdr2csv(rrdr, out, options, "", ",", "\r\n");
1069 + break;
1070 +
1071 + case DATASOURCE_TSV:
1072 + out->contenttype = CT_TEXT_PLAIN;
1073 + rrdr2csv(rrdr, out, options, "", "\t", "\r\n");
1074 + break;
1075 +
1076 + case DATASOURCE_HTML:
1077 + out->contenttype = CT_TEXT_HTML;
1078 + buffer_strcat(out, "<html>\n<center><table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">");
1079 + rrdr2csv(rrdr, out, options, "<tr><td>", "</td><td>", "</td></tr>\n");
1080 + buffer_strcat(out, "</table>\n</center>\n</html>\n");
1081 + break;
1082 +
1083 + case DATASOURCE_GOOGLE_JSONP:
1084 + out->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1085 + rrdr2json(rrdr, out, options, 1);
1086 + break;
1087 +
1088 + case DATASOURCE_GOOGLE_JSON:
1089 + out->contenttype = CT_APPLICATION_JSON;
1090 + rrdr2json(rrdr, out, options, 1);
1091 + break;
1092 +
1093 + case DATASOURCE_JSONP:
1094 + out->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1095 + rrdr2json(rrdr, out, options, 0);
1096 + break;
1097 +
1098 + case DATASOURCE_JSON:
1099 + default:
1100 + out->contenttype = CT_APPLICATION_JSON;
1101 + rrdr2json(rrdr, out, options, 0);
1102 + break;
1103 + }
1104 +
1105 + rrdr_free(rrdr);
1106 + return 200;
1107 }
1108
1109 unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero)
@@ -816,9 +1254,6 @@ unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int points, int g
1254 int annotate_reset = 0;
1255 int annotation_count = 0;
1256
819 - // the minimum line length we expect
820 - int line_size = 4096 + (dimensions * 200);
821 -
1257 long t = rrdset_time2slot(st, before),
1258 stop_at_t = rrdset_time2slot(st, after),
1259 stop_now = 0;
src/rrd2json.h
+16
@@ -16,11 +16,24 @@ extern char *hostname;
16 #define DATASOURCE_GOOGLE_JSONP 2
17 #define DATASOURCE_SSV 3
18 #define DATASOURCE_CSV 4
19 +#define DATASOURCE_JSONP 5
20 +#define DATASOURCE_TSV 6
21 +#define DATASOURCE_HTML 7
22 +#define DATASOURCE_JS_ARRAY 8
23 +#define DATASOURCE_SSV_COMMA 9
24
25 #define GROUP_AVERAGE 0
26 #define GROUP_MAX 1
27 #define GROUP_SUM 2
28
29 +#define RRDR_OPTION_NONZERO 0x00000001 // don't output dimensions will just zero values
30 +#define RRDR_OPTION_REVERSED 0x00000002 // output the rows in reverse order (oldest to newest)
31 +#define RRDR_OPTION_ABSOLUTE 0x00000004 // values positive, for DATASOURCE_SSV before summing
32 +#define RRDR_OPTION_MIN2MAX 0x00000008 // for DATASOURCE_SSV, out max - min
33 +#define RRDR_OPTION_SECONDS 0x00000010 // output seconds, instead of dates
34 +#define RRDR_OPTION_MILLISECONDS 0x00000020 // output milliseconds, instead of dates
35 +#define RRDR_OPTION_NULL2ZERO 0x00000040 // do not show nulls, convert them to zeros
36 +
37 extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
38 extern void rrd_stats_api_v1_charts(BUFFER *wb);
39
@@ -32,4 +45,7 @@ extern void rrd_stats_all_json(BUFFER *wb);
45
46 extern unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero);
47
48 +extern int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp);
49 +
50 +
51 #endif /* NETDATA_RRD2JSON_H */
src/web_buffer.c
+50 -9
@@ -71,6 +71,8 @@ void buffer_char_replace(BUFFER *wb, char from, char to)
71
72 void buffer_strcat(BUFFER *wb, const char *txt)
73 {
74 + if(unlikely(!txt || !*txt)) return;
75 +
76 if(wb->size - wb->len < 512)
77 buffer_need_bytes(wb, 512);
78
@@ -102,6 +104,8 @@ void buffer_strcat(BUFFER *wb, const char *txt)
104
105 void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...)
106 {
107 + if(unlikely(!fmt || !*fmt)) return;
108 +
109 buffer_need_bytes(wb, len+1);
110
111 va_list args;
@@ -116,6 +120,8 @@ void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...)
120
121 void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args)
122 {
123 + if(unlikely(!fmt || !*fmt)) return;
124 +
125 size_t len = wb->size - wb->len;
126 if(unlikely(!len)) return;
127
@@ -128,6 +134,8 @@ void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args)
134
135 void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
136 {
137 + if(unlikely(!fmt || !*fmt)) return;
138 +
139 if(unlikely(wb->len > wb->size)) {
140 error("web_buffer_sprintf(): already overflown length %ld, size = %ld", wb->len, wb->size);
141 }
@@ -135,7 +143,7 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
143 // if(wb->size - wb->len < 512)
144 // web_buffer_need_bytes(wb, 512);
145
138 - size_t len = wb->size - wb->len, old_len = wb->len, wrote;
146 + size_t len = wb->size - wb->len, wrote;
147
148 buffer_need_bytes(wb, len);
149
@@ -183,9 +191,9 @@ void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minu
191 {
192 // 10 20 30 = 35
193 // 01234567890123456789012345678901234
186 - // Date(2014, 04, 01, 03, 28, 20, 065)
194 + // Date(2014,04,01,03,28,20)
195
188 - buffer_need_bytes(wb, 36);
196 + buffer_need_bytes(wb, 30);
197
198 char *b = &wb->buffer[wb->len];
199
@@ -200,23 +208,18 @@ void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minu
208 b[i++]= 48 + year / 10;
209 b[i++]= 48 + year % 10;
210 b[i++]=',';
203 - //b[i++]=' ';
211 b[i]= 48 + month / 10; if(b[i] != '0') i++;
212 b[i++]= 48 + month % 10;
213 b[i++]=',';
207 - //b[i++]=' ';
214 b[i]= 48 + day / 10; if(b[i] != '0') i++;
215 b[i++]= 48 + day % 10;
216 b[i++]=',';
211 - //b[i++]=' ';
217 b[i]= 48 + hours / 10; if(b[i] != '0') i++;
218 b[i++]= 48 + hours % 10;
219 b[i++]=',';
215 - //b[i++]=' ';
220 b[i]= 48 + minutes / 10; if(b[i] != '0') i++;
221 b[i++]= 48 + minutes % 10;
222 b[i++]=',';
219 - //b[i++]=' ';
223 b[i]= 48 + seconds / 10; if(b[i] != '0') i++;
224 b[i++]= 48 + seconds % 10;
225 b[i++]=')';
@@ -225,9 +228,47 @@ void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minu
228 wb->len += i;
229
230 // terminate it
228 - buffer_need_bytes(wb, 1);
231 wb->buffer[wb->len] = '\0';
232 + buffer_overflow_check(wb);
233 +}
234 +
235 +// generate a date, the fastest possible way...
236 +void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds)
237 +{
238 + // 10 20 30 = 35
239 + // 01234567890123456789012345678901234
240 + // 2014-04-01 03:28:20
241 +
242 + buffer_need_bytes(wb, 36);
243 +
244 + char *b = &wb->buffer[wb->len];
245
246 + int i = 0;
247 + b[i++]= 48 + year / 1000; year -= (year / 1000) * 1000;
248 + b[i++]= 48 + year / 100; year -= (year / 100) * 100;
249 + b[i++]= 48 + year / 10;
250 + b[i++]= 48 + year % 10;
251 + b[i++]='-';
252 + b[i++]= 48 + month / 10;
253 + b[i++]= 48 + month % 10;
254 + b[i++]='-';
255 + b[i++]= 48 + day / 10;
256 + b[i++]= 48 + day % 10;
257 + b[i++]=' ';
258 + b[i++]= 48 + hours / 10;
259 + b[i++]= 48 + hours % 10;
260 + b[i++]=':';
261 + b[i++]= 48 + minutes / 10;
262 + b[i++]= 48 + minutes % 10;
263 + b[i++]=':';
264 + b[i++]= 48 + seconds / 10;
265 + b[i++]= 48 + seconds % 10;
266 + b[i]='\0';
267 +
268 + wb->len += i;
269 +
270 + // terminate it
271 + wb->buffer[wb->len] = '\0';
272 buffer_overflow_check(wb);
273 }
274
src/web_buffer.h
+2 -1
@@ -34,7 +34,7 @@ typedef struct web_buffer {
34 #define buffer_strlen(wb) ((wb)->len)
35 extern const char *buffer_tostring(BUFFER *wb);
36
37 -#define buffer_need_bytes(buffer, needed_free_size) do { if(unlikely((buffer)->size - (buffer)->len < (needed_free_size))) buffer_increase((buffer), (needed_free_size)); } while(0)
37 +#define buffer_need_bytes(buffer, needed_free_size) do { if(unlikely((buffer)->size - (buffer)->len < (long)(needed_free_size))) buffer_increase((buffer), (long)(needed_free_size)); } while(0)
38
39 #define buffer_flush(wb) wb->buffer[wb->len = 0] = '\0'
40 extern void buffer_reset(BUFFER *wb);
@@ -42,6 +42,7 @@ extern void buffer_reset(BUFFER *wb);
42 extern void buffer_strcat(BUFFER *wb, const char *txt);
43 extern void buffer_rrd_value(BUFFER *wb, calculated_number value);
44
45 +extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
46 extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
47
48 extern BUFFER *buffer_create(long size);
src/web_client.c
+171 -19
@@ -14,6 +14,7 @@
14 #include <netinet/tcp.h>
15 #include <malloc.h>
16 #include <pwd.h>
17 +#include <ctype.h>
18
19 #include "common.h"
20 #include "log.h"
@@ -398,9 +399,31 @@ void web_client_enable_deflate(struct web_client *w) {
399 }
400 #endif // NETDATA_WITH_ZLIB
401
401 -int rrdset_validate_dimensions(RRDSET *st, BUFFER *dimensions)
402 +uint32_t web_client_api_request_v1_data_options(char *o)
403 {
403 - ;
404 + uint32_t ret = 0x00000000;
405 + char *tok;
406 +
407 + while(o && *o && (tok = mystrsep(&o, ", |"))) {
408 + if(!*tok) continue;
409 +
410 + if(!strcmp(tok, "nonzero"))
411 + ret |= RRDR_OPTION_NONZERO;
412 + else if(!strcmp(tok, "flip") || !strcmp(tok, "reversed") || !strcmp(tok, "reverse"))
413 + ret |= RRDR_OPTION_REVERSED;
414 + else if(!strcmp(tok, "abs") || !strcmp(tok, "absolute") || !strcmp(tok, "absolute_sum") || !strcmp(tok, "absolute-sum"))
415 + ret |= RRDR_OPTION_ABSOLUTE;
416 + else if(!strcmp(tok, "min2max"))
417 + ret |= RRDR_OPTION_MIN2MAX;
418 + else if(!strcmp(tok, "seconds"))
419 + ret |= RRDR_OPTION_SECONDS;
420 + else if(!strcmp(tok, "ms") || !strcmp(tok, "milliseconds"))
421 + ret |= RRDR_OPTION_MILLISECONDS;
422 + else if(!strcmp(tok, "null2zero"))
423 + ret |= RRDR_OPTION_NULL2ZERO;
424 + }
425 +
426 + return ret;
427 }
428
429 int web_client_api_request_v1_data_format(char *name)
@@ -414,17 +437,65 @@ int web_client_api_request_v1_data_format(char *name)
437 else if(!strcmp(name, "json"))
438 return DATASOURCE_JSON;
439
440 + else if(!strcmp(name, "jsonp"))
441 + return DATASOURCE_JSONP;
442 +
443 else if(!strcmp(name, "ssv"))
444 return DATASOURCE_SSV;
445
446 else if(!strcmp(name, "csv"))
447 return DATASOURCE_CSV;
448
423 - return DATASOURCE_INVALID;
449 + else if(!strcmp(name, "tsv"))
450 + return DATASOURCE_TSV;
451 +
452 + else if(!strcmp(name, "tsv-excel"))
453 + return DATASOURCE_TSV;
454 +
455 + else if(!strcmp(name, "html"))
456 + return DATASOURCE_HTML;
457 +
458 + else if(!strcmp(name, "array"))
459 + return DATASOURCE_JS_ARRAY;
460 +
461 + else if(!strcmp(name, "ssvcomma"))
462 + return DATASOURCE_SSV_COMMA;
463 +
464 + return DATASOURCE_JSON;
465 +}
466 +
467 +int web_client_api_request_v1_data_google_format(char *name)
468 +{
469 + if(!strcmp(name, "json"))
470 + return DATASOURCE_GOOGLE_JSONP;
471 +
472 + else if(!strcmp(name, "html"))
473 + return DATASOURCE_HTML;
474 +
475 + else if(!strcmp(name, "csv"))
476 + return DATASOURCE_CSV;
477 +
478 + else if(!strcmp(name, "tsv-excel"))
479 + return DATASOURCE_TSV;
480 +
481 + return DATASOURCE_JSON;
482 +}
483 +
484 +int web_client_api_request_v1_data_group(char *name)
485 +{
486 + if(!strcmp(name, "max"))
487 + return GROUP_MAX;
488 +
489 + else if(!strcmp(name, "average"))
490 + return GROUP_AVERAGE;
491 +
492 + return GROUP_MAX;
493 }
494
495 int web_client_api_request_v1_charts(struct web_client *w, char *url)
496 {
497 + if(url) { ; }
498 +
499 buffer_flush(w->response.data);
500 rrd_stats_api_v1_charts(w->response.data);
501 return 200;
@@ -476,22 +547,32 @@ cleanup:
547 return ret;
548 }
549
550 +// returns the HTTP code
551 int web_client_api_request_v1_data(struct web_client *w, char *url)
552 {
553 + debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
554 +
555 int ret = 400;
556 BUFFER *dimensions = NULL;
557
558 buffer_flush(w->response.data);
559
560 + char *google_version = "0.6",
561 + *google_reqId = "0",
562 + *google_sig = "0",
563 + *google_out = "json",
564 + *google_responseHandler = "google.visualization.Query.setResponse",
565 + *google_outFileName = NULL;
566 +
567 + time_t last_timestamp_in_data = 0, google_timestamp = 0;
568 +
569 char *chart = NULL
487 - , *dim = NULL
570 , *before_str = NULL
571 , *after_str = NULL
490 - , *points_str = NULL
491 - , *group_str = NULL
492 - , *options = NULL;
572 + , *points_str = NULL;
573
494 - int format = DATASOURCE_JSON;
574 + int format = DATASOURCE_JSON, group = GROUP_MAX;
575 + uint32_t options = 0x00000000;
576
577 while(url) {
578 char *value = mystrsep(&url, "?&[]");
@@ -501,11 +582,13 @@ int web_client_api_request_v1_data(struct web_client *w, char *url)
582 if(!name || !*name) continue;
583 if(!value || !*value) continue;
584
585 + debug(D_WEB_CLIENT, "%llu: API v1 query param '%s' with value '%s'", w->id, name, value);
586 +
587 // name and value are now the parameters
588 // they are not null and not empty
589
590 if(!strcmp(name, "chart")) chart = value;
508 - else if(!strcmp(name, "dimension") || !strcmp(name, "dim")) {
591 + else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
592 if(!dimensions) dimensions = buffer_create(strlen(value));
593 if(dimensions) {
594 buffer_strcat(dimensions, "|");
@@ -515,16 +598,45 @@ int web_client_api_request_v1_data(struct web_client *w, char *url)
598 else if(!strcmp(name, "after")) after_str = value;
599 else if(!strcmp(name, "before")) before_str = value;
600 else if(!strcmp(name, "points")) points_str = value;
518 - else if(!strcmp(name, "group")) group_str = value;
601 + else if(!strcmp(name, "group")) {
602 + group = web_client_api_request_v1_data_group(value);
603 + }
604 else if(!strcmp(name, "format")) {
605 format = web_client_api_request_v1_data_format(value);
606 }
607 else if(!strcmp(name, "options")) {
523 - options = value;
608 + options |= web_client_api_request_v1_data_options(value);
609 }
525 - else {
526 - buffer_sprintf(w->response.data, "Unknown parameter '%s' in request.", name);
527 - goto cleanup;
610 + else if(!strcmp(name, "tqx")) {
611 + // parse Google Visualization API options
612 + // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source
613 + char *tqx_name, *tqx_value;
614 +
615 + while(value) {
616 + tqx_value = mystrsep(&value, ";");
617 + if(!tqx_value || !*tqx_value) continue;
618 +
619 + tqx_name = mystrsep(&tqx_value, ":");
620 + if(!tqx_name || !*tqx_name) continue;
621 + if(!tqx_value || !*tqx_value) continue;
622 +
623 + if(!strcmp(tqx_name, "version"))
624 + google_version = tqx_value;
625 + else if(!strcmp(tqx_name, "reqId"))
626 + google_reqId = tqx_value;
627 + else if(!strcmp(tqx_name, "sig")) {
628 + google_sig = tqx_value;
629 + google_timestamp = strtoul(google_sig, NULL, 0);
630 + }
631 + else if(!strcmp(tqx_name, "out")) {
632 + google_out = tqx_value;
633 + format = web_client_api_request_v1_data_google_format(google_out);
634 + }
635 + else if(!strcmp(tqx_name, "responseHandler"))
636 + google_responseHandler = tqx_value;
637 + else if(!strcmp(tqx_name, "outFileName"))
638 + google_outFileName = tqx_value;
639 + }
640 }
641 }
642
@@ -541,12 +653,51 @@ int web_client_api_request_v1_data(struct web_client *w, char *url)
653 goto cleanup;
654 }
655
544 - uint32_t before = (before_str && *before_str)?atol(before_str):0;
545 - uint32_t after = (after_str && *after_str) ?atol(after_str):0;
546 - int points = (points_str && *points_str)?atoi(points_str):0;
547 - int group = (group_str && *group_str) ?atoi(group_str):0;
656 + long long before = (before_str && *before_str)?atol(before_str):0;
657 + long long after = (after_str && *after_str) ?atol(after_str):0;
658 + int points = (points_str && *points_str)?atoi(points_str):0;
659 +
660 + debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%u', format '%u', options '0x%08x'"
661 + , w->id
662 + , chart
663 + , (dimensions)?buffer_tostring(dimensions):""
664 + , after
665 + , before
666 + , points
667 + , group
668 + , format
669 + , options
670 + );
671 +
672 + if(google_outFileName && *google_outFileName) {
673 + buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", google_outFileName);
674 + error("generating outfilename header: '%s'", google_outFileName);
675 + }
676 +
677 + if(format == DATASOURCE_GOOGLE_JSONP) {
678 + debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSON/JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
679 + w->id, google_version, google_reqId, google_sig, google_out, google_responseHandler, google_outFileName
680 + );
681 +
682 + buffer_sprintf(w->response.data,
683 + "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:",
684 + google_responseHandler, google_version, google_reqId, st->last_updated.tv_sec);
685 + }
686 +
687 + ret = rrd2format(st, w->response.data, dimensions, format, points, after, before, group, options, &last_timestamp_in_data);
688
549 - buffer_sprintf(w->response.data, "API command 'data' for chart '%s', dimensions '%s', after '%s', before '%s', points '%s', group '%s', format '%s', options '%s'", chart, dim, after_str, before_str, points_str, group_str, format, options);
689 + if(format == DATASOURCE_GOOGLE_JSONP) {
690 + if(google_timestamp < last_timestamp_in_data)
691 + buffer_strcat(w->response.data, "});");
692 +
693 + else {
694 + // the client already has the latest data
695 + buffer_flush(w->response.data);
696 + buffer_sprintf(w->response.data,
697 + "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
698 + google_responseHandler, google_version, google_reqId);
699 + }
700 + }
701
702 cleanup:
703 if(dimensions) buffer_free(dimensions);
@@ -732,6 +883,7 @@ int web_client_data_request(struct web_client *w, char *url, int datasource_type
883
884 else {
885 // the client already has the latest data
886 + buffer_flush(w->response.data);
887 buffer_sprintf(w->response.data,
888 "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
889 google_responseHandler, google_version, google_reqId);