@cryptotaxi247 / netdata-1 / commits / 9f1b98c30

properly calculate the first data collection of statsd metrics when charts are loaded from disk

Costa Tsaousis (ktsaou) committed Jul 4, 2017 at 01:26 UTC 9f1b98c30e08adcc885dadf6bfcbac9663f59ec8
4 files changed +198 -120
src/rrd.h
+8
@@ -381,6 +381,14 @@ typedef enum rrdhost_flags {
381 #define rrdhost_flag_clear(host, flag) (host)->flags &= ~flag
382 #endif
383
384 +#ifdef NETDATA_INTERNAL_CHECKS
385 +#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
386 + debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, st->name, ##args); } while(0)
387 +#else
388 +#define rrdset_debug(st, fmt, args...) debug_dummy()
389 +#endif
390 +
391 +
392 // ----------------------------------------------------------------------------
393 // RRD HOST
394
src/rrddim.c
+1 -1
@@ -226,7 +226,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
226 rd->collected_volume = 0;
227 rd->stored_volume = 0;
228 rd->last_stored_value = 0;
229 - rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
229 + rd->values[st->current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS);
230 rd->last_collected_time.tv_sec = 0;
231 rd->last_collected_time.tv_usec = 0;
232 rd->rrdset = st;
src/rrdset.c
+187 -119
@@ -675,10 +675,11 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
675 microseconds = (usec_t)since_last_usec;
676 }
677 }
678 - debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
678
680 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
681 - debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
679 + #ifdef NETDATA_INTERNAL_CHECKS
680 + debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
681 + rrdset_debug(st, "NEXT: %llu microseconds", microseconds);
682 + #endif
683
684 st->usec_since_last_update = microseconds;
685 }
@@ -687,9 +688,17 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
688 // ----------------------------------------------------------------------------
689 // RRDSET - process the collected values for all dimensions of a chart
690
690 -static inline void rrdset_init_last_collected_time(RRDSET *st) {
691 +static inline usec_t rrdset_init_last_collected_time(RRDSET *st) {
692 now_realtime_timeval(&st->last_collected_time);
693 last_collected_time_align(st);
694 +
695 + usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
696 +
697 + #ifdef NETDATA_INTERNAL_CHECKS
698 + rrdset_debug(st, "initialized last collected time to %0.3Lf", (long double)last_collect_ut / USEC_PER_SEC);
699 + #endif
700 +
701 + return last_collect_ut;
702 }
703
704 static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
@@ -697,10 +706,15 @@ static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
706 usec_t ut = last_collect_ut + st->usec_since_last_update;
707 st->last_collected_time.tv_sec = (time_t) (ut / USEC_PER_SEC);
708 st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC);
709 +
710 + #ifdef NETDATA_INTERNAL_CHECKS
711 + rrdset_debug(st, "updated last collected time to %0.3Lf", (long double)last_collect_ut / USEC_PER_SEC);
712 + #endif
713 +
714 return last_collect_ut;
715 }
716
703 -static inline void rrdset_init_last_updated_time(RRDSET *st) {
717 +static inline usec_t rrdset_init_last_updated_time(RRDSET *st) {
718 // copy the last collected time to last updated time
719 st->last_updated.tv_sec = st->last_collected_time.tv_sec;
720 st->last_updated.tv_usec = st->last_collected_time.tv_usec;
@@ -709,6 +723,14 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) {
723 st->last_updated.tv_sec -= st->update_every;
724
725 last_updated_time_align(st);
726 +
727 + usec_t last_updated_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
728 +
729 + #ifdef NETDATA_INTERNAL_CHECKS
730 + rrdset_debug(st, "initialized last updated time to %0.3Lf", (long double)last_updated_ut / USEC_PER_SEC);
731 + #endif
732 +
733 + return last_updated_ut;
734 }
735
736 static inline void rrdset_done_push_exclusive(RRDSET *st) {
@@ -761,13 +783,11 @@ static inline size_t rrdset_done_interpolate(
783 long current_entry = st->current_entry;
784
785 for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
786 +
787 #ifdef NETDATA_INTERNAL_CHECKS
788 if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
766 -
767 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
768 - debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/USEC_PER_SEC);
769 - debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/USEC_PER_SEC);
770 - }
789 + rrdset_debug(st, "last_stored_ut = %0.3Lf (last updated time)", (long double)last_stored_ut/USEC_PER_SEC);
790 + rrdset_debug(st, "next_store_ut = %0.3Lf (next interpolation point)", (long double)next_store_ut/USEC_PER_SEC);
791 #endif
792
793 last_ut = next_store_ut;
@@ -784,18 +804,17 @@ static inline size_t rrdset_done_interpolate(
804 );
805
806 #ifdef NETDATA_INTERNAL_CHECKS
787 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
788 - debug(D_RRD_STATS, "%s/%s: CALC2 INC "
807 + rrdset_debug(st, "%s: CALC2 INC "
808 CALCULATED_NUMBER_FORMAT " = "
809 CALCULATED_NUMBER_FORMAT
791 - " * %llu"
792 - " / %llu"
793 - , st->id, rd->name
810 + " * (%llu - %llu)"
811 + " / (%llu - %llu)"
812 + , rd->name
813 , new_value
814 , rd->calculated_value
796 - , (next_store_ut - last_stored_ut)
797 - , (now_collect_ut - last_stored_ut)
798 - );
815 + , next_store_ut, last_collect_ut
816 + , now_collect_ut, last_collect_ut
817 + );
818 #endif
819
820 rd->calculated_value -= new_value;
@@ -806,11 +825,10 @@ static inline size_t rrdset_done_interpolate(
825 if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
826
827 #ifdef NETDATA_INTERNAL_CHECKS
809 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
810 - debug(D_RRD_STATS, "%s/%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING",
811 - st->id, rd->name
828 + rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING",
829 + rd->name
830 , (calculated_number)(next_store_ut - last_stored_ut)
813 - );
831 + );
832 #endif
833
834 new_value = new_value * (calculated_number)(st->update_every * USEC_PER_SEC) / (calculated_number)(next_store_ut - last_stored_ut);
@@ -841,18 +859,17 @@ static inline size_t rrdset_done_interpolate(
859 );
860
861 #ifdef NETDATA_INTERNAL_CHECKS
844 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
845 - debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
862 + rrdset_debug(st, "%s: CALC2 DEF "
863 CALCULATED_NUMBER_FORMAT " = ((("
864 "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
865 " * %llu"
866 " / %llu) + " CALCULATED_NUMBER_FORMAT
850 - , st->id, rd->name
867 + , rd->name
868 , new_value
869 , rd->calculated_value, rd->last_calculated_value
870 , (next_store_ut - first_ut)
871 , (now_collect_ut - first_ut), rd->last_calculated_value
855 - );
872 + );
873 #endif
874 }
875 break;
@@ -868,24 +885,22 @@ static inline size_t rrdset_done_interpolate(
885 rd->last_stored_value = new_value;
886
887 #ifdef NETDATA_INTERNAL_CHECKS
871 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
872 - debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
888 + rrdset_debug(st, "%s: STORE[%ld] "
889 CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
874 - , st->id, rd->name
890 + , rd->name
891 , current_entry
892 , unpack_storage_number(rd->values[current_entry]), new_value
877 - );
893 + );
894 #endif
895
896 }
897 else {
898
899 #ifdef NETDATA_INTERNAL_CHECKS
884 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
885 - debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING "
886 - , st->id, rd->name
900 + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING "
901 + , rd->name
902 , current_entry
888 - );
903 + );
904 #endif
905
906 rd->values[current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS);
@@ -944,6 +959,36 @@ static inline size_t rrdset_done_interpolate(
959 return stored_entries;
960 }
961
962 +static inline void rrdset_done_fill_the_gap(RRDSET *st) {
963 + usec_t update_every_ut = st->update_every * USEC_PER_SEC;
964 + usec_t now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
965 +
966 + long c = 0, entries = st->entries;
967 + RRDDIM *rd;
968 + rrddim_foreach_read(rd, st) {
969 + usec_t next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
970 + long current_entry = st->current_entry;
971 +
972 + for(c = 0; c < entries && next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, c++) {
973 + rd->values[current_entry] = SN_EMPTY_SLOT;
974 + current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1;
975 +
976 + #ifdef NETDATA_INTERNAL_CHECKS
977 + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rd->name, current_entry);
978 + #endif
979 + }
980 + }
981 +
982 + if(c > 0) {
983 + c--;
984 + st->last_updated.tv_sec += c * st->update_every;
985 +
986 + st->current_entry += c;
987 + if(st->current_entry >= st->entries)
988 + st->current_entry -= st->entries;
989 + }
990 +}
991 +
992 void rrdset_done(RRDSET *st) {
993 if(unlikely(netdata_exit)) return;
994
@@ -978,12 +1023,6 @@ void rrdset_done(RRDSET *st) {
1023 // a read lock is OK here
1024 rrdset_rdlock(st);
1025
981 -/*
982 - // enable the chart, if it was disabled
983 - if(unlikely(rrd_delete_unupdated_dimensions) && !st->enabled)
984 - st->enabled = 1;
985 -*/
986 -
1026 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
1027 error("Chart '%s' has the OBSOLETE flag set, but it is collected.", st->id);
1028 rrdset_isnot_obsolete(st);
@@ -998,23 +1037,19 @@ void rrdset_done(RRDSET *st) {
1037 first_entry = 1;
1038 }
1039
1001 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1002 - debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
1040 + #ifdef NETDATA_INTERNAL_CHECKS
1041 + rrdset_debug(st, "microseconds since last update: %llu", st->usec_since_last_update);
1042 + #endif
1043
1044 // set last_collected_time
1045 if(unlikely(!st->last_collected_time.tv_sec)) {
1046 // it is the first entry
1047 // set the last_collected_time to now
1008 - rrdset_init_last_collected_time(st);
1009 -
1010 - last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec - update_every_ut;
1048 + last_collect_ut = rrdset_init_last_collected_time(st) - update_every_ut;
1049
1050 // the first entry should not be stored
1051 store_this_entry = 0;
1052 first_entry = 1;
1015 -
1016 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1017 - debug(D_RRD_STATS, "%s: has not set last_collected_time. Setting it now. Will not store the next entry.", st->name);
1053 }
1054 else {
1055 // it is not the first entry
@@ -1032,9 +1067,6 @@ void rrdset_done(RRDSET *st) {
1067 // the first entry should not be stored
1068 store_this_entry = 0;
1069 first_entry = 1;
1035 -
1036 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1037 - debug(D_RRD_STATS, "%s: initializing last_updated to last_collected_time - %llu microseconds. Will not store the next entry.", st->name, st->usec_since_last_update);
1070 }
1071
1072 // check if we will re-write the entire data set
@@ -1054,27 +1086,48 @@ void rrdset_done(RRDSET *st) {
1086 // last_stored_ut = the last time we added a value to the storage
1087 // now_collect_ut = the time the current value has been collected
1088 // next_store_ut = the time of the next interpolation point
1057 - last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
1089 now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
1090 + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
1091 next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
1092
1061 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
1062 - debug(D_RRD_STATS, "%s: last_collect_ut = %0.3Lf (last collection time)", st->name, (long double)last_collect_ut/USEC_PER_SEC);
1063 - debug(D_RRD_STATS, "%s: now_collect_ut = %0.3Lf (current collection time)", st->name, (long double)now_collect_ut/USEC_PER_SEC);
1064 - debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/USEC_PER_SEC);
1065 - debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/USEC_PER_SEC);
1066 - }
1067 -
1093 if(unlikely(!st->counter_done)) {
1069 - store_this_entry = 0;
1070 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1071 - debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name);
1094 + // if we have not collected metrics this session (st->counter_done == 0)
1095 + // and we have collected metrics for this chart in the past (st->counter != 0)
1096 + // fill the gap (the chart has been just loaded from disk)
1097 + if(unlikely(st->counter)) {
1098 + rrdset_done_fill_the_gap(st);
1099 + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
1100 + next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
1101 + }
1102 +
1103 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))) {
1104 + store_this_entry = 1;
1105 + last_collect_ut = next_store_ut - update_every_ut;
1106 +
1107 + #ifdef NETDATA_INTERNAL_CHECKS
1108 + rrdset_debug(st, "Fixed first entry.");
1109 + #endif
1110 + }
1111 + else {
1112 + store_this_entry = 0;
1113 +
1114 + #ifdef NETDATA_INTERNAL_CHECKS
1115 + rrdset_debug(st, "Will not store the next entry.");
1116 + #endif
1117 + }
1118 }
1119 st->counter_done++;
1120
1121 if(unlikely(st->rrdhost->rrdpush_enabled))
1122 rrdset_done_push(st);
1123
1124 + #ifdef NETDATA_INTERNAL_CHECKS
1125 + rrdset_debug(st, "last_collect_ut = %0.3Lf (last collection time)", (long double)last_collect_ut/USEC_PER_SEC);
1126 + rrdset_debug(st, "now_collect_ut = %0.3Lf (current collection time)", (long double)now_collect_ut/USEC_PER_SEC);
1127 + rrdset_debug(st, "last_stored_ut = %0.3Lf (last updated time)", (long double)last_stored_ut/USEC_PER_SEC);
1128 + rrdset_debug(st, "next_store_ut = %0.3Lf (next interpolation point)", (long double)next_store_ut/USEC_PER_SEC);
1129 + #endif
1130 +
1131 // calculate totals and count the dimensions
1132 int dimensions = 0;
1133 st->collected_total = 0;
@@ -1096,18 +1149,19 @@ void rrdset_done(RRDSET *st) {
1149 continue;
1150 }
1151
1099 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1100 - debug(D_RRD_STATS, "%s/%s: START "
1101 - " last_collected_value = " COLLECTED_NUMBER_FORMAT
1102 - " collected_value = " COLLECTED_NUMBER_FORMAT
1103 - " last_calculated_value = " CALCULATED_NUMBER_FORMAT
1104 - " calculated_value = " CALCULATED_NUMBER_FORMAT
1105 - , st->id, rd->name
1152 + #ifdef NETDATA_INTERNAL_CHECKS
1153 + rrdset_debug(st, "%s: START "
1154 + " last_collected_value = " COLLECTED_NUMBER_FORMAT
1155 + " collected_value = " COLLECTED_NUMBER_FORMAT
1156 + " last_calculated_value = " CALCULATED_NUMBER_FORMAT
1157 + " calculated_value = " CALCULATED_NUMBER_FORMAT
1158 + , rd->name
1159 , rd->last_collected_value
1160 , rd->collected_value
1161 , rd->last_calculated_value
1162 , rd->calculated_value
1110 - );
1163 + );
1164 + #endif
1165
1166 switch(rd->algorithm) {
1167 case RRD_ALGORITHM_ABSOLUTE:
@@ -1115,18 +1169,20 @@ void rrdset_done(RRDSET *st) {
1169 * (calculated_number)rd->multiplier
1170 / (calculated_number)rd->divisor;
1171
1118 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1119 - debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
1172 + #ifdef NETDATA_INTERNAL_CHECKS
1173 + rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN "
1174 CALCULATED_NUMBER_FORMAT " = "
1175 COLLECTED_NUMBER_FORMAT
1176 " * " CALCULATED_NUMBER_FORMAT
1177 " / " CALCULATED_NUMBER_FORMAT
1124 - , st->id, rd->name
1178 + , rd->name
1179 , rd->calculated_value
1180 , rd->collected_value
1181 , (calculated_number)rd->multiplier
1182 , (calculated_number)rd->divisor
1129 - );
1183 + );
1184 + #endif
1185 +
1186 break;
1187
1188 case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
@@ -1140,16 +1196,18 @@ void rrdset_done(RRDSET *st) {
1196 * (calculated_number)rd->collected_value
1197 / (calculated_number)st->collected_total;
1198
1143 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1144 - debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
1199 + #ifdef NETDATA_INTERNAL_CHECKS
1200 + rrdset_debug(st, "%s: CALC PCENT-ROW "
1201 CALCULATED_NUMBER_FORMAT " = 100"
1146 - " * " COLLECTED_NUMBER_FORMAT
1202 + " * " COLLECTED_NUMBER_FORMAT
1203 " / " COLLECTED_NUMBER_FORMAT
1148 - , st->id, rd->name
1204 + , rd->name
1205 , rd->calculated_value
1206 , rd->collected_value
1207 , st->collected_total
1152 - );
1208 + );
1209 + #endif
1210 +
1211 break;
1212
1213 case RRD_ALGORITHM_INCREMENTAL:
@@ -1177,19 +1235,21 @@ void rrdset_done(RRDSET *st) {
1235 * (calculated_number)rd->multiplier
1236 / (calculated_number)rd->divisor;
1237
1180 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1181 - debug(D_RRD_STATS, "%s/%s: CALC INC PRE "
1238 + #ifdef NETDATA_INTERNAL_CHECKS
1239 + rrdset_debug(st, "%s: CALC INC PRE "
1240 CALCULATED_NUMBER_FORMAT " = ("
1241 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
1242 ")"
1243 " * " CALCULATED_NUMBER_FORMAT
1244 " / " CALCULATED_NUMBER_FORMAT
1187 - , st->id, rd->name
1245 + , rd->name
1246 , rd->calculated_value
1247 , rd->collected_value, rd->last_collected_value
1248 , (calculated_number)rd->multiplier
1249 , (calculated_number)rd->divisor
1192 - );
1250 + );
1251 + #endif
1252 +
1253 break;
1254
1255 case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
@@ -1204,7 +1264,8 @@ void rrdset_done(RRDSET *st) {
1264 debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
1265 , st->name, rd->name
1266 , rd->last_collected_value
1207 - , rd->collected_value);
1267 + , rd->collected_value
1268 + );
1269
1270 if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)))
1271 storage_flags = SN_EXISTS_RESET;
@@ -1222,16 +1283,18 @@ void rrdset_done(RRDSET *st) {
1283 * (calculated_number)(rd->collected_value - rd->last_collected_value)
1284 / (calculated_number)(st->collected_total - st->last_collected_total);
1285
1225 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1226 - debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
1286 + #ifdef NETDATA_INTERNAL_CHECKS
1287 + rrdset_debug(st, "%s: CALC PCENT-DIFF "
1288 CALCULATED_NUMBER_FORMAT " = 100"
1228 - " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1229 - " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1230 - , st->id, rd->name
1289 + " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1290 + " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1291 + , rd->name
1292 , rd->calculated_value
1293 , rd->collected_value, rd->last_collected_value
1294 , st->collected_total, st->last_collected_total
1234 - );
1295 + );
1296 + #endif
1297 +
1298 break;
1299
1300 default:
@@ -1239,46 +1302,43 @@ void rrdset_done(RRDSET *st) {
1302 // it gets noticed when we add new types
1303 rd->calculated_value = 0;
1304
1242 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1243 - debug(D_RRD_STATS, "%s/%s: CALC "
1305 + #ifdef NETDATA_INTERNAL_CHECKS
1306 + rrdset_debug(st, "%s: CALC "
1307 CALCULATED_NUMBER_FORMAT " = 0"
1245 - , st->id, rd->name
1308 + , rd->name
1309 , rd->calculated_value
1247 - );
1310 + );
1311 + #endif
1312 +
1313 break;
1314 }
1315
1251 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1252 - debug(D_RRD_STATS, "%s/%s: PHASE2 "
1316 + #ifdef NETDATA_INTERNAL_CHECKS
1317 + rrdset_debug(st, "%s: PHASE2 "
1318 " last_collected_value = " COLLECTED_NUMBER_FORMAT
1319 " collected_value = " COLLECTED_NUMBER_FORMAT
1320 " last_calculated_value = " CALCULATED_NUMBER_FORMAT
1321 " calculated_value = " CALCULATED_NUMBER_FORMAT
1257 - , st->id, rd->name
1322 + , rd->name
1323 , rd->last_collected_value
1324 , rd->collected_value
1325 , rd->last_calculated_value
1326 , rd->calculated_value
1262 - );
1327 + );
1328 + #endif
1329
1330 }
1331
1332 // at this point we have all the calculated values ready
1333 // it is now time to interpolate values on a second boundary
1334
1335 +#ifdef NETDATA_INTERNAL_CHECKS
1336 if(unlikely(now_collect_ut < next_store_ut)) {
1337 // this is collected in the same interpolation point
1271 -
1272 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1273 - debug(D_RRD_STATS, "host '%s', chart '%s': THIS IS IN THE SAME INTERPOLATION POINT", st->rrdhost->hostname, st->name);
1274 -
1275 -#ifdef NETDATA_INTERNAL_CHECKS
1338 + rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
1339 info("INTERNAL CHECK: host '%s', chart '%s' is collected in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, next_store_ut - now_collect_ut);
1277 -#endif
1340 }
1279 -
1280 - if(unlikely(!store_this_entry && rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
1281 - store_this_entry = 1;
1341 +#endif
1342
1343 rrdset_done_interpolate(st
1344 , update_every_ut
@@ -1296,29 +1356,35 @@ void rrdset_done(RRDSET *st) {
1356 if(unlikely(!rd->updated))
1357 continue;
1358
1299 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1300 - debug(D_RRD_STATS, "%s/%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_collected_value, rd->collected_value);
1359 + #ifdef NETDATA_INTERNAL_CHECKS
1360 + rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rd->name, rd->last_collected_value, rd->collected_value);
1361 + #endif
1362
1363 rd->last_collected_value = rd->collected_value;
1364
1365 switch(rd->algorithm) {
1366 case RRD_ALGORITHM_INCREMENTAL:
1367 if(unlikely(!first_entry)) {
1307 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1308 - debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
1368 + #ifdef NETDATA_INTERNAL_CHECKS
1369 + rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
1370 + #endif
1371 +
1372 rd->last_calculated_value += rd->calculated_value;
1373 }
1374 else {
1312 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1313 - debug(D_RRD_STATS, "%s: THIS IS THE FIRST POINT", st->name);
1375 + #ifdef NETDATA_INTERNAL_CHECKS
1376 + rrdset_debug(st, "THIS IS THE FIRST POINT");
1377 + #endif
1378 }
1379 break;
1380
1381 case RRD_ALGORITHM_ABSOLUTE:
1382 case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
1383 case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
1320 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1321 - debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value, rd->calculated_value);
1384 + #ifdef NETDATA_INTERNAL_CHECKS
1385 + rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value);
1386 + #endif
1387 +
1388 rd->last_calculated_value = rd->calculated_value;
1389 break;
1390 }
@@ -1327,18 +1393,20 @@ void rrdset_done(RRDSET *st) {
1393 rd->collected_value = 0;
1394 rd->updated = 0;
1395
1330 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
1331 - debug(D_RRD_STATS, "%s/%s: END "
1396 + #ifdef NETDATA_INTERNAL_CHECKS
1397 + rrdset_debug(st, "%s: END "
1398 " last_collected_value = " COLLECTED_NUMBER_FORMAT
1399 " collected_value = " COLLECTED_NUMBER_FORMAT
1400 " last_calculated_value = " CALCULATED_NUMBER_FORMAT
1401 " calculated_value = " CALCULATED_NUMBER_FORMAT
1336 - , st->id, rd->name
1402 + , rd->name
1403 , rd->last_collected_value
1404 , rd->collected_value
1405 , rd->last_calculated_value
1406 , rd->calculated_value
1341 - );
1407 + );
1408 + #endif
1409 +
1410 }
1411
1412 // ALL DONE ABOUT THE DATA UPDATE
src/statsd.c
+2
@@ -1262,6 +1262,7 @@ static inline RRDSET *statsd_private_rrdset_create(
1262 , history
1263 );
1264 rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
1265 + // rrdset_flag_set(st, RRDSET_FLAG_DEBUG);
1266 return st;
1267 }
1268
@@ -1658,6 +1659,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
1659 );
1660
1661 rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);
1662 + // rrdset_flag_set(chart->st, RRDSET_FLAG_DEBUG);
1663 }
1664 else rrdset_next(chart->st);
1665