json generation can now increase the web buffer as required to fit all the data
Costa Tsaousis (ktsaou) committed
Apr 6, 2014 at 04:38 UTC
c3e106c6c6adec3b89f552b99f1df5bc5687be90
1 file changed
+177
-167
netdata.c
+177
-167
@@ -729,172 +729,9 @@ long long rrd_stats_dimension_get(RRD_DIMENSION *rd, size_t position)
729
return(0);
730
}
731
732
-size_t rrd_stats_json(RRD_STATS *st, char *b, size_t length, size_t entries_to_show, size_t group_count, int group_method)
733
-{
734
- pthread_mutex_lock(&st->mutex);
735
-
736
- // check the options
737
- if(entries_to_show <= 0) entries_to_show = 1;
738
- if(group_count <= 0) group_count = 1;
739
- if(group_count > st->entries / 20) group_count = st->entries / 20;
740
-
741
- size_t i = 0; // the bytes of JSON output we have generated so far
742
- size_t printed = 0; // the lines of JSON data we have generated so far
743
-
744
- size_t current_entry = st->current_entry;
745
- size_t t, lt; // t = the current entry, lt = the lest entry of data
746
- long count = st->entries; // count down of the entries examined so far
747
- long pad = 0; // align the entries when grouping values together
748
-
749
- RRD_DIMENSION *rd;
750
- size_t c = 0; // counter for dimension loops
751
- size_t dimensions = 0; // the total number of dimensions present
752
-
753
- unsigned long long usec = 0;// usec between the entries
754
- char dtm[201]; // temp variable for storing dates
755
-
756
- int we_need_totals = 0; // if set, we should calculate totals for all dimensions
757
-
758
- // find how many dimensions we have
759
- for( rd = st->dimensions ; rd ; rd = rd->next) {
760
- dimensions++;
761
- if(rd->type == RRD_DIMENSION_PCENT_OVER_TOTAL) we_need_totals++;
762
- }
763
-
764
- if(!dimensions) {
765
- pthread_mutex_unlock(&st->mutex);
766
- return sprintf(b, "No dimensions yet.");
767
- }
768
-
769
- // temporary storage to keep track of group values and counts
770
- long double group_values[dimensions];
771
- int group_counts[dimensions];
772
- for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++)
773
- group_values[c] = group_counts[c] = 0;
774
-
775
- i += sprintf(&b[i], "{\n \"cols\":\n [\n");
776
- i += sprintf(&b[i], " {\"id\":\"\",\"label\":\"time\",\"pattern\":\"\",\"type\":\"datetime\"}");
777
-
778
- for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++)
779
- if(!rd->hidden)
780
- i += sprintf(&b[i], ",\n {\"id\":\"\",\"label\":\"%s\",\"pattern\":\"\",\"type\":\"number\"}", rd->name);
781
-
782
- i += sprintf(&b[i], " ],\n \"rows\":\n [\n");
783
-
784
- // to allow grouping on the same values, we need a pad
785
- pad = current_entry % group_count;
786
-
787
- // make sure current_entry is within limits
788
- if(current_entry < 0 || current_entry >= st->entries) current_entry = 0;
789
-
790
- // find the old entry of the round-robin
791
- t = current_entry + 1;
792
- if(t >= st->entries) t = 0;
793
- lt = t;
794
-
795
- // find the current entry
796
- t++;
797
- if(t >= st->entries) t = 0;
798
-
799
- // the loop in dimension data
800
- count -= 2;
801
- for ( ; t != current_entry && count >= 0 ; lt = t++, count--) {
802
- if(t >= st->entries) t = 0;
803
-
804
- // if the last is empty, loop again
805
- if(!st->times[lt].tv_sec) continue;
806
-
807
- // check if we may exceed the buffer provided
808
- if((length - i) < 1024) break;
809
-
810
- // prefer the most recent last entries
811
- if(((count-pad) / group_count) > entries_to_show) continue;
812
-
813
-
814
- // ok. we will use this entry!
815
- // find how much usec since the previous entry
816
-
817
- usec = usecdiff(&st->times[t], &st->times[lt]);
818
-
819
- if(((count-pad) % group_count) == 0) {
820
- if(printed >= entries_to_show) break;
821
-
822
- // generate the local date time
823
- struct tm *tm = localtime(&st->times[t].tv_sec);
824
- if(!tm) { error("localtime() failed."); continue; }
825
-
826
- sprintf(dtm, "\"Date(%d, %d, %d, %d, %d, %d, %d)\"", tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec, (int)(st->times[t].tv_usec / 1000)); // datetime
827
- // strftime(dtm, 200, "\"Date(%Y, %m, %d, %H, %M, %S)\"", tm); // datetime
828
- // strftime(dtm, 200, "[%H, %M, %S, 0]", tm); // timeofday
829
- i += sprintf(&b[i], "%s {\"c\":[{\"v\":%s}", printed?"]},\n":"", dtm);
830
-
831
- printed++;
832
- }
833
-
834
- // if we need a PCENT_OVER_TOTAL, calculate the totals for the current and the last
835
- long double total = 0, oldtotal = 0;
836
- if(we_need_totals) {
837
- for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
838
- total += (long double)rrd_stats_dimension_get(rd, t);
839
- oldtotal += (long double)rrd_stats_dimension_get(rd, lt);
840
- }
841
- }
842
-
843
- for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
844
- //long long oldvalue, value; // temp variable for storing data values
845
- long double oldvalue, value; // temp variable for storing data values
846
-
847
- value = (long double)rrd_stats_dimension_get(rd, t);
848
- oldvalue = (long double)rrd_stats_dimension_get(rd, lt);
849
-
850
- switch(rd->type) {
851
- case RRD_DIMENSION_PCENT_OVER_TOTAL:
852
- value = (long double)100 * (value - oldvalue) / (total - oldtotal);
853
- break;
854
-
855
- case RRD_DIMENSION_INCREMENTAL:
856
- if(oldvalue > value) value = 0; // detect overflows and resets
857
- else value -= oldvalue;
858
- value = value * (long double)1000000L / (long double)usec;
859
- break;
860
-
861
- default:
862
- break;
863
- }
864
-
865
- value = value * (long double)rd->multiplier / (long double)rd->divisor;
866
-
867
- group_counts[c]++;
868
- switch(group_method) {
869
- case GROUP_MAX:
870
- if(abs(value) > abs(group_values[c])) group_values[c] = value;
871
- break;
872
-
873
- default:
874
- case GROUP_AVERAGE:
875
- group_values[c] += value;
876
- if(((count - pad) % group_count) == 0) group_values[c] /= (long double)group_counts[c];
877
- break;
878
- }
879
-
880
- if(((count-pad) % group_count) == 0) {
881
- if(!rd->hidden)
882
- i += sprintf(&b[i], ",{\"v\":%0.1Lf}", group_values[c]);
883
- //i += sprintf(&b[i], ",{\"v\":%lld}", group_values[c]);
884
-
885
- group_values[c] = group_counts[c] = 0;
886
- }
887
- }
888
- }
889
- if(printed) i += sprintf(&b[i], "]}");
890
- i += sprintf(&b[i], "\n ]\n}\n");
891
-
892
- pthread_mutex_unlock(&st->mutex);
893
- return(i);
894
-}
732
733
// ----------------------------------------------------------------------------
897
-// listener (web server)
734
+// web buffer
735
736
struct web_buffer {
737
size_t size; // allocation size of buffer
@@ -1016,7 +853,7 @@ struct web_client *web_client_create(int listener)
853
strncpy(w->client_ip, inet_ntoa(w->clientaddr.sin_addr), 100);
854
w->client_ip[100] = '\0';
855
1019
- syslog(LOG_NOTICE, "%llu: New client from %s.", w->id, w->client_ip);
856
+ // syslog(LOG_NOTICE, "%llu: New client from %s.", w->id, w->client_ip);
857
debug(D_WEB_CLIENT_ACCESS, "%llu: New web client from %s on socket %d.", w->id, w->client_ip, w->ifd);
858
859
{
@@ -1059,6 +896,172 @@ struct web_client *web_client_free(struct web_client *w)
896
return(n);
897
}
898
899
+size_t rrd_stats_json(RRD_STATS *st, struct web_buffer *wb, size_t entries_to_show, size_t group_count, int group_method)
900
+{
901
+ char *b = wb->buffer;
902
+
903
+ pthread_mutex_lock(&st->mutex);
904
+
905
+ // check the options
906
+ if(entries_to_show <= 0) entries_to_show = 1;
907
+ if(group_count <= 0) group_count = 1;
908
+ if(group_count > st->entries / 20) group_count = st->entries / 20;
909
+
910
+ size_t i = 0; // the bytes of JSON output we have generated so far
911
+ size_t printed = 0; // the lines of JSON data we have generated so far
912
+
913
+ size_t current_entry = st->current_entry;
914
+ size_t t, lt; // t = the current entry, lt = the lest entry of data
915
+ long count = st->entries; // count down of the entries examined so far
916
+ long pad = 0; // align the entries when grouping values together
917
+
918
+ RRD_DIMENSION *rd;
919
+ size_t c = 0; // counter for dimension loops
920
+ size_t dimensions = 0; // the total number of dimensions present
921
+
922
+ unsigned long long usec = 0;// usec between the entries
923
+ char dtm[201]; // temp variable for storing dates
924
+
925
+ int we_need_totals = 0; // if set, we should calculate totals for all dimensions
926
+
927
+ // find how many dimensions we have
928
+ for( rd = st->dimensions ; rd ; rd = rd->next) {
929
+ dimensions++;
930
+ if(rd->type == RRD_DIMENSION_PCENT_OVER_TOTAL) we_need_totals++;
931
+ }
932
+
933
+ if(!dimensions) {
934
+ pthread_mutex_unlock(&st->mutex);
935
+ return (wb->bytes = sprintf(b, "No dimensions yet."));
936
+ }
937
+
938
+ // temporary storage to keep track of group values and counts
939
+ long double group_values[dimensions];
940
+ int group_counts[dimensions];
941
+ for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++)
942
+ group_values[c] = group_counts[c] = 0;
943
+
944
+ i += sprintf(&b[i], "{\n \"cols\":\n [\n");
945
+ i += sprintf(&b[i], " {\"id\":\"\",\"label\":\"time\",\"pattern\":\"\",\"type\":\"datetime\"}");
946
+
947
+ for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++)
948
+ if(!rd->hidden)
949
+ i += sprintf(&b[i], ",\n {\"id\":\"\",\"label\":\"%s\",\"pattern\":\"\",\"type\":\"number\"}", rd->name);
950
+
951
+ i += sprintf(&b[i], " ],\n \"rows\":\n [\n");
952
+
953
+ // to allow grouping on the same values, we need a pad
954
+ pad = current_entry % group_count;
955
+
956
+ // make sure current_entry is within limits
957
+ if(current_entry < 0 || current_entry >= st->entries) current_entry = 0;
958
+
959
+ // find the old entry of the round-robin
960
+ t = current_entry + 1;
961
+ if(t >= st->entries) t = 0;
962
+ lt = t;
963
+
964
+ // find the current entry
965
+ t++;
966
+ if(t >= st->entries) t = 0;
967
+
968
+ // the loop in dimension data
969
+ count -= 2;
970
+ for ( ; t != current_entry && count >= 0 ; lt = t++, count--) {
971
+ if(t >= st->entries) t = 0;
972
+
973
+ // if the last is empty, loop again
974
+ if(!st->times[lt].tv_sec) continue;
975
+
976
+ // check if we may exceed the buffer provided
977
+ web_buffer_increase(wb, 1024);
978
+
979
+ // prefer the most recent last entries
980
+ if(((count-pad) / group_count) > entries_to_show) continue;
981
+
982
+
983
+ // ok. we will use this entry!
984
+ // find how much usec since the previous entry
985
+
986
+ usec = usecdiff(&st->times[t], &st->times[lt]);
987
+
988
+ if(((count-pad) % group_count) == 0) {
989
+ if(printed >= entries_to_show) break;
990
+
991
+ // generate the local date time
992
+ struct tm *tm = localtime(&st->times[t].tv_sec);
993
+ if(!tm) { error("localtime() failed."); continue; }
994
+
995
+ sprintf(dtm, "\"Date(%d, %d, %d, %d, %d, %d, %d)\"", tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec, (int)(st->times[t].tv_usec / 1000)); // datetime
996
+ // strftime(dtm, 200, "\"Date(%Y, %m, %d, %H, %M, %S)\"", tm); // datetime
997
+ // strftime(dtm, 200, "[%H, %M, %S, 0]", tm); // timeofday
998
+ i += sprintf(&b[i], "%s {\"c\":[{\"v\":%s}", printed?"]},\n":"", dtm);
999
+
1000
+ printed++;
1001
+ }
1002
+
1003
+ // if we need a PCENT_OVER_TOTAL, calculate the totals for the current and the last
1004
+ long double total = 0, oldtotal = 0;
1005
+ if(we_need_totals) {
1006
+ for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1007
+ total += (long double)rrd_stats_dimension_get(rd, t);
1008
+ oldtotal += (long double)rrd_stats_dimension_get(rd, lt);
1009
+ }
1010
+ }
1011
+
1012
+ for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1013
+ //long long oldvalue, value; // temp variable for storing data values
1014
+ long double oldvalue, value; // temp variable for storing data values
1015
+
1016
+ value = (long double)rrd_stats_dimension_get(rd, t);
1017
+ oldvalue = (long double)rrd_stats_dimension_get(rd, lt);
1018
+
1019
+ switch(rd->type) {
1020
+ case RRD_DIMENSION_PCENT_OVER_TOTAL:
1021
+ value = (long double)100 * (value - oldvalue) / (total - oldtotal);
1022
+ break;
1023
+
1024
+ case RRD_DIMENSION_INCREMENTAL:
1025
+ if(oldvalue > value) value = 0; // detect overflows and resets
1026
+ else value -= oldvalue;
1027
+ value = value * (long double)1000000L / (long double)usec;
1028
+ break;
1029
+
1030
+ default:
1031
+ break;
1032
+ }
1033
+
1034
+ value = value * (long double)rd->multiplier / (long double)rd->divisor;
1035
+
1036
+ group_counts[c]++;
1037
+ switch(group_method) {
1038
+ case GROUP_MAX:
1039
+ if(abs(value) > abs(group_values[c])) group_values[c] = value;
1040
+ break;
1041
+
1042
+ default:
1043
+ case GROUP_AVERAGE:
1044
+ group_values[c] += value;
1045
+ if(((count - pad) % group_count) == 0) group_values[c] /= (long double)group_counts[c];
1046
+ break;
1047
+ }
1048
+
1049
+ if(((count-pad) % group_count) == 0) {
1050
+ if(!rd->hidden)
1051
+ i += sprintf(&b[i], ",{\"v\":%0.1Lf}", group_values[c]);
1052
+ //i += sprintf(&b[i], ",{\"v\":%lld}", group_values[c]);
1053
+
1054
+ group_values[c] = group_counts[c] = 0;
1055
+ }
1056
+ }
1057
+ }
1058
+ if(printed) i += sprintf(&b[i], "]}");
1059
+ i += sprintf(&b[i], "\n ]\n}\n");
1060
+
1061
+ pthread_mutex_unlock(&st->mutex);
1062
+ return(wb->bytes = i);
1063
+}
1064
+
1065
int mysendfile(struct web_client *w, char *filename)
1066
{
1067
debug(D_WEB_CLIENT, "%llu: Looking for file '%s'...", w->id, filename);
@@ -1322,11 +1325,12 @@ void web_client_process(struct web_client *w)
1325
else debug(D_WEB_CLIENT, "%llu: Unknown group method '%s'", w->id, tok);
1326
}
1327
1328
+ syslog(LOG_NOTICE, "%llu: Sending '%s' data to client %s.", w->id, st->name, w->client_ip);
1329
debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group_count, %d group_method).", w->id, st->name, st->id, lines, group_count, group_method);
1330
1331
code = 200;
1332
w->data->contenttype = CT_APPLICATION_JSON;
1329
- w->data->bytes = rrd_stats_json(st, w->data->buffer, w->data->size, lines, group_count, group_method);
1333
+ w->data->bytes = rrd_stats_json(st, w->data, lines, group_count, group_method);
1334
}
1335
}
1336
else if(strcmp(tok, WEB_PATH_GRAPH) == 0) { // "graph"
@@ -1345,6 +1349,7 @@ void web_client_process(struct web_client *w)
1349
}
1350
else {
1351
code = 200;
1352
+ syslog(LOG_NOTICE, "%llu: Sending '%s' graph to client %s.", w->id, st->name, w->client_ip);
1353
debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name);
1354
w->data->contenttype = CT_APPLICATION_JSON;
1355
w->data->bytes = rrd_stats_graph_json(st, url, w->data->buffer, w->data->size);
@@ -1354,6 +1359,7 @@ void web_client_process(struct web_client *w)
1359
code = 200;
1360
1361
debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
1362
+ syslog(LOG_NOTICE, "%llu: mirroring client %s.", w->id, w->client_ip);
1363
1364
// just leave the buffer as is
1365
// it will be copied back to the client
@@ -1361,6 +1367,7 @@ void web_client_process(struct web_client *w)
1367
else if(strcmp(tok, "list") == 0) {
1368
code = 200;
1369
1370
+ syslog(LOG_NOTICE, "%llu: Sending text list of all monitors to client %s.", w->id, w->client_ip);
1371
debug(D_WEB_CLIENT_ACCESS, "%llu: Sending list of RRD_STATS...", w->id);
1372
1373
w->data->bytes = 0;
@@ -1373,6 +1380,7 @@ void web_client_process(struct web_client *w)
1380
code = 200;
1381
1382
debug(D_WEB_CLIENT_ACCESS, "%llu: Sending envlist of RRD_STATS...", w->id);
1383
+ syslog(LOG_NOTICE, "%llu: Sending envlist to client %s.", w->id, w->client_ip);
1384
1385
w->data->bytes = 0;
1386
RRD_STATS *st = root;
@@ -1384,7 +1392,9 @@ void web_client_process(struct web_client *w)
1392
}
1393
else if(strcmp(tok, "all.json") == 0) {
1394
code = 200;
1387
- debug(D_WEB_CLIENT_ACCESS, "%llu: Sending all.json of RRD_STATS...", w->id);
1395
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Sending JSON list of all monitors of RRD_STATS...", w->id);
1396
+ syslog(LOG_NOTICE, "%llu: Sending list with all graphs in JSON to client %s.", w->id, w->client_ip);
1397
+
1398
w->data->contenttype = CT_APPLICATION_JSON;
1399
w->data->bytes = rrd_stats_all_json(w->data->buffer, w->data->size);
1400
}