added flags to storage_number; added gap functionality; restored reset functionality; fixed a bug in procfile
Costa Tsaousis (ktsaou) committed
Mar 21, 2015 at 15:52 UTC
7372600140f4a9206726bdc3d376b92d3368cafb
8 files changed
+312
-131
conf.d/apps_groups.conf
+10
-6
@@ -20,9 +20,9 @@
20
# If a group_name starts with a -, the dimension will be hidden (cpu chart only)
21
#
22
23
-compile: cc1 cc1plus as gcc make automake autoconf
23
+compile: cc1 cc1plus as gcc ld make automake autoconf
24
rsync: rsync
25
-media: mplayer vlc xine mediatomb omxplayer omxplayer.bin
25
+media: mplayer vlc xine mediatomb omxplayer omxplayer.bin kodi kodi.bin xbmc xbmc.bin mediacenter eventlircd
26
squid: squid squid2 squid3
27
apache: apache apache2
28
mysql: mysqld mysql
@@ -37,9 +37,11 @@ lighttpd: lighttpd
37
ftpd: proftpd in.tftpd
38
samba: smbd nmbd winbindd
39
nfs: rpcbind rpc.statd rpc.idmapd rpc.mountd nfsd4 nfsd4_callbacks nfsd nfsiod
40
-ssh: ssh sshd
41
-X: gdm X lightdm xdm gnome-session gconfd-2 gnome-terminal gnome-screensaver gnome-settings-daemon pulseaudio
42
-named: named
40
+ssh: ssh sshd scp
41
+X: X lightdm xdm pulseaudio gkrellm
42
+xfce: xfwm4 xfdesktop xfce4-appfinder Thunar xfsettingsd xfce4-panel
43
+gnome: gnome-session gdm gconfd-2 gnome-terminal gnome-screensaver gnome-settings-daemon
44
+named: named rncd
45
clam: clamd freshclam
46
cups: cupsd cups-browsed
47
ntp: ntpq ntpd
@@ -53,6 +55,8 @@ openvpn: openvpn
55
cron: cron atd
56
ha: corosync hs_logd ha_logd stonithd
57
ipvs: ipvs_syncmaster ipvs_syncbackup
56
-kernel: kthreadd
58
+kernel: kthreadd kauditd lockd khelper kdevtmpfs khungtaskd rpciod fsnotify_mark kthrotld iscsi_eh deferwq
59
netdata: netdata apps.plugin charts.d.plugin
60
crsproxy: crsproxy
61
+wifi: hostapd wpa_supplicant
62
+system: systemd-journal systemd-udevd systemd-logind udisks-glue udisks-daemon udevd udevd connmand ipv6_addrconf dbus-daemon
src/procfile.c
+3
-3
@@ -279,7 +279,7 @@ procfile *procfile_open(const char *filename, const char *separators) {
279
280
procfile *ff = malloc(sizeof(procfile) + PROCFILE_INITIAL_BUFFER);
281
if(!ff) {
282
- error(PF_PREFIX ": Cannot allocate memory for file '%s'. Reason: %s", ff->filename, strerror(errno));
282
+ error(PF_PREFIX ": Cannot allocate memory for file '%s'. Reason: %s", filename, strerror(errno));
283
close(fd);
284
return NULL;
285
}
@@ -295,7 +295,7 @@ procfile *procfile_open(const char *filename, const char *separators) {
295
ff->words = pfwords_new();
296
297
if(!ff->lines || !ff->words) {
298
- error(PF_PREFIX ": Cannot initialize parser for file '%s'. Reason: %s", ff->filename, strerror(errno));
298
+ error(PF_PREFIX ": Cannot initialize parser for file '%s'. Reason: %s", filename, strerror(errno));
299
procfile_close(ff);
300
return NULL;
301
}
@@ -311,7 +311,7 @@ procfile *procfile_open(const char *filename, const char *separators) {
311
const char *s = separators;
312
while(*s) ff->separators[(int)*s++] = PF_CHAR_IS_SEPARATOR;
313
314
- debug(D_PROCFILE, "File '%s' opened.", ff->filename);
314
+ debug(D_PROCFILE, "File '%s' opened.", filename);
315
return ff;
316
}
317
src/rrd.c
+146
-67
@@ -177,13 +177,14 @@ void rrd_stats_reset(RRD_STATS *st)
177
st->last_updated.tv_sec = 0;
178
st->last_updated.tv_usec = 0;
179
st->current_entry = 0;
180
+ st->counter = 0;
181
st->counter_done = 0;
182
183
RRD_DIMENSION *rd;
184
for(rd = st->dimensions; rd ; rd = rd->next) {
185
rd->last_collected_time.tv_sec = 0;
186
rd->last_collected_time.tv_usec = 0;
186
- rd->current_entry = 0;
187
+ bzero(rd->values, rd->entries * sizeof(storage_number));
188
}
189
}
190
@@ -217,14 +218,15 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
218
if(st) {
219
if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
220
errno = 0;
220
- error("File %s does not have our version. Clearing it.", fullfilename);
221
+ info("Initializing file %s.", fullfilename);
222
bzero(st, size);
223
}
224
else if(strcmp(st->id, fullid) != 0) {
225
errno = 0;
225
- error("File %s does not have our id. Unmapping it.", fullfilename);
226
- munmap(st, size);
227
- st = NULL;
226
+ error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
227
+ // munmap(st, size);
228
+ // st = NULL;
229
+ bzero(st, size);
230
}
231
else if(st->memsize != size || st->entries != entries) {
232
errno = 0;
@@ -297,6 +299,8 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
299
st->last_collected_time.tv_usec = 0;
300
st->counter_done = 0;
301
302
+ st->gap_when_lost_iterations = config_get_number(st->id, "gap when lost iterations above", DEFAULT_GAP_INTERPOLATIONS);
303
+
304
pthread_rwlock_init(&st->rwlock, NULL);
305
pthread_rwlock_wrlock(&root_rwlock);
306
@@ -328,7 +332,7 @@ RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char
332
333
if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
334
errno = 0;
331
- error("File %s does not have our version. Clearing it.", fullfilename);
335
+ info("Initializing file %s.", fullfilename);
336
bzero(rd, size);
337
}
338
else if(rd->memsize != size) {
@@ -363,9 +367,10 @@ RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char
367
}
368
else if(strcmp(rd->id, id) != 0) {
369
errno = 0;
366
- error("File %s does not have our dimension id. Unmapping it.", fullfilename);
367
- munmap(rd, size);
368
- rd = NULL;
370
+ error("File %s contents are not for dimension %s. Clearing it.", fullfilename, id);
371
+ // munmap(rd, size);
372
+ // rd = NULL;
373
+ bzero(rd, size);
374
}
375
}
376
@@ -597,6 +602,7 @@ void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collec
602
603
gettimeofday(&rd->last_collected_time, NULL);
604
rd->collected_value = value;
605
+ rd->updated = 1;
606
}
607
608
int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value)
@@ -642,7 +648,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
648
debug(D_RRD_CALLS, "rrd_stats_done() for chart %s", st->name);
649
650
RRD_DIMENSION *rd, *last;
645
- int oldstate;
651
+ int oldstate, store_this_entry = 1;
652
653
if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
654
error("Cannot set pthread cancel state to DISABLE.");
@@ -650,45 +656,81 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
656
// a read lock is OK here
657
pthread_rwlock_rdlock(&st->rwlock);
658
659
+ // check if the chart has a long time to be refreshed
660
if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) {
654
- info("Chart chart %s took too long to be updated (%0.3Lf secs). Reseting chart history.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
661
+ info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
662
rrd_stats_reset(st);
663
st->usec_since_last_update = st->update_every * 1000000ULL;
664
}
658
-
665
if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
666
661
- if(!st->last_collected_time.tv_sec) gettimeofday(&st->last_collected_time, NULL);
667
+ // set last_collected_time
668
+ if(!st->last_collected_time.tv_sec) {
669
+ // it is the first entry
670
+ // set the last_collected_time to now
671
+ gettimeofday(&st->last_collected_time, NULL);
672
+
673
+ // the first entry should not be stored
674
+ store_this_entry = 0;
675
+
676
+ if(st->debug) debug(D_RRD_STATS, "%s: initializing last_collected to now. Will not store the next entry.", st->name);
677
+ }
678
else {
679
+ // it is not the first entry
680
+ // calculate the proper last_collected_time, using usec_since_last_update
681
unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
682
st->last_collected_time.tv_sec = ut / 1000000ULL;
683
st->last_collected_time.tv_usec = ut % 1000000ULL;
684
}
685
686
+ // if this set has not been updated in the past
687
+ // we fake the last_update time to be = now - usec_since_last_update
688
if(!st->last_updated.tv_sec) {
689
+ // it has never been updated before
690
+ // set a fake last_updated, in the past using usec_since_last_update
691
unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
692
st->last_updated.tv_sec = ut / 1000000ULL;
693
st->last_updated.tv_usec = ut % 1000000ULL;
694
673
- if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
695
+ // the first entry should not be stored
696
+ store_this_entry = 0;
697
+
698
+ if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
699
}
700
676
- if(usecdiff(&st->last_collected_time, &st->last_updated) > st->entries * st->update_every * 1000000ULL) {
677
- info("History of chart %s too old (last updated at %u, last collected at %u). Reseting chart.", st->name, st->last_updated.tv_sec, st->last_collected_time.tv_sec);
701
+ // check if we will re-write the entire data set
702
+ if(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL) {
703
+ info("%s: too old data (last updated at %u.%u, last collected at %u.%u). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
704
rrd_stats_reset(st);
705
+
706
st->usec_since_last_update = st->update_every * 1000000ULL;
680
- pthread_rwlock_unlock(&st->rwlock);
681
- return(st->usec_since_last_update);
707
+
708
+ gettimeofday(&st->last_collected_time, NULL);
709
+
710
+ unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
711
+ st->last_updated.tv_sec = ut / 1000000ULL;
712
+ st->last_updated.tv_usec = ut % 1000000ULL;
713
+
714
+ // the first entry should not be stored
715
+ store_this_entry = 0;
716
}
717
718
+ // these are the 3 variables that will help us in interpolation
719
+ // last_ut = the last time we added a value to the storage
720
+ // now_ut = the time the current value is taken at
721
+ // next_ut = the time of the next interpolation point
722
unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
685
- unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
723
+ unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
724
unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
725
726
if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
727
if(st->debug) debug(D_RRD_STATS, "%s: now ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
728
if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
729
730
+ if(!st->counter_done) {
731
+ store_this_entry = 0;
732
+ if(st->debug) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name);
733
+ }
734
st->counter_done++;
735
736
// calculate totals and count the dimensions
@@ -697,10 +739,13 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
739
for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
740
st->collected_total += rd->collected_value;
741
742
+ uint32_t storage_flags = SN_EXISTS;
743
+
744
// process all dimensions to calculate their values
745
// based on the collected figures only
746
// at this stage we do not interpolate anything
747
for( rd = st->dimensions ; rd ; rd = rd->next ) {
748
+
749
if(st->debug) debug(D_RRD_STATS, "%s/%s: "
750
" last_collected_value = " COLLECTED_NUMBER_FORMAT
751
" collected_value = " COLLECTED_NUMBER_FORMAT
@@ -760,7 +805,10 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
805
case RRD_DIMENSION_INCREMENTAL:
806
// if the new is smaller than the old (an overflow, or reset), set the old equal to the new
807
// to reset the calculation (it will give zero as the calculation for this second)
763
- if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
808
+ if(rd->last_collected_value > rd->collected_value) {
809
+ storage_flags = SN_EXISTS_RESET;
810
+ rd->last_collected_value = rd->collected_value;
811
+ }
812
813
rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
814
@@ -801,39 +849,16 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
849
break;
850
}
851
}
804
- // at this point we have all the calculated values ready
805
-
806
- if(st->counter_done == 1 || next_ut > now_ut) {
807
- // we don't have any usable data yet
808
- if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
809
-
810
- for( rd = st->dimensions; rd ; rd = rd->next ) {
811
- rd->last_calculated_value = rd->calculated_value;
812
- rd->last_collected_value = rd->collected_value;
852
814
- switch(rd->algorithm) {
815
- case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
816
- case RRD_DIMENSION_INCREMENTAL:
817
- if(!st->usec_since_last_update) rd->calculated_value = 0;
818
- // keep the previous values
819
- // the next time, a new incremental total will be calculated
820
- break;
821
- }
822
-
823
- rd->collected_value = 0;
824
- }
825
- st->last_collected_total = st->collected_total;
853
+ // at this point we have all the calculated values ready
854
+ // it is now time to interpolate values on a second boundary
855
827
- pthread_rwlock_unlock(&st->rwlock);
828
- if(pthread_setcancelstate(oldstate, NULL) != 0)
829
- error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
856
+ unsigned long long first_ut = last_ut;
857
+ int iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL);
858
831
- return(st->usec_since_last_update);
832
- }
859
+ for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL, iterations-- ) {
860
+ if(iterations < 0) error("iterations calculation wrapped!");
861
834
- // it is now time to interpolate values on a second boundary
835
- unsigned long long first_ut = last_ut;
836
- for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL ) {
862
if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
863
if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
864
@@ -894,28 +919,70 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
919
920
if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
921
break;
922
+ }
923
924
+ if(!store_this_entry) {
925
+ store_this_entry = 1;
926
+ continue;
927
+ }
928
+
929
+ if(rd->updated && iterations < st->gap_when_lost_iterations) {
930
+ rd->values[st->current_entry] = pack_storage_number(
931
+ new_value
932
+ * (calculated_number)rd->multiplier
933
+ / (calculated_number)rd->divisor
934
+ , storage_flags );
935
+
936
+ if(st->debug)
937
+ debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
938
+ CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
939
+ " * %ld"
940
+ " / %ld"
941
+ , st->id, rd->name
942
+ , st->current_entry
943
+ , unpack_storage_number(rd->values[st->current_entry]), new_value
944
+ , rd->multiplier
945
+ , rd->divisor
946
+ );
947
+ }
948
+ else {
949
+ if(st->debug) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING "
950
+ , st->id, rd->name
951
+ , st->current_entry
952
+ );
953
+ rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
954
}
955
956
+ if(st->debug) {
957
+ calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;
958
+ calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]);
959
+ calculated_number accuracy = accuracy_loss(t1, t2);
960
+ debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
961
+ , st->id, rd->name
962
+ , st->current_entry
963
+ , t2
964
+ , get_storage_number_flags(rd->values[st->current_entry])
965
+ , t1
966
+ , accuracy
967
+ , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
968
+ );
969
+
970
+ rd->collected_volume += t1;
971
+ rd->stored_volume += t2;
972
+ accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);
973
+ debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"
974
+ , st->id, rd->name
975
+ , st->current_entry
976
+ , rd->stored_volume
977
+ , rd->collected_volume
978
+ , accuracy
979
+ , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
980
+ );
981
901
- rd->values[st->current_entry] = pack_storage_number(
902
- new_value
903
- * (calculated_number)rd->multiplier
904
- / (calculated_number)rd->divisor
905
- );
906
-
907
- if(st->debug)
908
- debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
909
- CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
910
- " * %ld"
911
- " / %ld"
912
- , st->id, rd->name
913
- , st->current_entry
914
- , unpack_storage_number(rd->values[st->current_entry]), new_value
915
- , rd->multiplier
916
- , rd->divisor
917
- );
982
+ }
983
}
984
+ // reset the storage flags for the next point, if any;
985
+ storage_flags = SN_EXISTS;
986
987
if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
988
// the db is overwriting values
@@ -930,16 +997,28 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
997
}
998
999
for( rd = st->dimensions; rd ; rd = rd->next ) {
1000
+ if(!rd->updated) continue;
1001
rd->last_collected_value = rd->collected_value;
1002
rd->last_calculated_value = rd->calculated_value;
1003
rd->collected_value = 0;
1004
+ rd->updated = 0;
1005
+
1006
+ // if this is the first entry of incremental dimensions
1007
+ // we have to set the first calculated_value to zero
1008
+ // to eliminate the first spike
1009
+ if(st->counter_done == 1) switch(rd->algorithm) {
1010
+ case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
1011
+ case RRD_DIMENSION_INCREMENTAL:
1012
+ rd->calculated_value = 0;
1013
+ // the next time, a new incremental total will be calculated
1014
+ break;
1015
+ }
1016
}
1017
st->last_collected_total = st->collected_total;
1018
1019
// ALL DONE ABOUT THE DATA UPDATE
1020
// --------------------------------------------------------------------
1021
942
-
1022
// find if there are any obsolete dimensions (not updated recently)
1023
for( rd = st->dimensions; rd ; rd = rd->next )
1024
if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
src/rrd.h
+15
-7
@@ -14,14 +14,16 @@ extern int update_every;
14
#define HISTORY_MAX (86400*10)
15
extern int save_history;
16
17
+#define DEFAULT_GAP_INTERPOLATIONS 10
18
+
19
typedef long long total_number;
20
21
#define TOTAL_NUMBER_FORMAT "%lld"
22
23
#define RRD_STATS_NAME_MAX 1024
24
23
-#define RRD_STATS_MAGIC "NETDATA CACHE STATS FILE V009"
24
-#define RRD_DIMENSION_MAGIC "NETDATA CACHE DIMENSION FILE V009"
25
+#define RRD_STATS_MAGIC "NETDATA CACHE STATS FILE V010"
26
+#define RRD_DIMENSION_MAGIC "NETDATA CACHE DIMENSION FILE V010"
27
28
29
// ----------------------------------------------------------------------------
@@ -59,8 +61,8 @@ extern int memory_mode_id(const char *name);
61
// ----------------------------------------------------------------------------
62
// algorithms types
63
62
-#define RRD_DIMENSION_ABSOLUTE_NAME "absolute"
63
-#define RRD_DIMENSION_INCREMENTAL_NAME "incremental"
64
+#define RRD_DIMENSION_ABSOLUTE_NAME "absolute"
65
+#define RRD_DIMENSION_INCREMENTAL_NAME "incremental"
66
#define RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row"
67
#define RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row"
68
@@ -84,8 +86,8 @@ struct rrd_dimension {
86
long entries; // how many entries this dimension has
87
// this should be the same to the entries of the data set
88
87
- long current_entry; // the entry that is currently being updated
89
int update_every; // every how many seconds is this updated?
90
+ int updated; // set to 0 after each calculation, to 1 after each collected value
91
92
int hidden; // if set to non zero, this dimension will not be sent to the client
93
int mapped; // 1 if the file is mapped
@@ -96,8 +98,8 @@ struct rrd_dimension {
98
long divisor;
99
100
struct timeval last_collected_time; // when was this dimension last updated
99
- // this is only used to detect un-updated dimensions
100
- // which are removed after some time
101
+ // this is actual date time we updated the last_collected_value
102
+ // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRD_STATS
103
104
calculated_number calculated_value;
105
calculated_number last_calculated_value;
@@ -105,6 +107,9 @@ struct rrd_dimension {
107
collected_number collected_value; // the value collected at this round
108
collected_number last_collected_value; // the value that was collected at the last round
109
110
+ calculated_number collected_volume;
111
+ calculated_number stored_volume;
112
+
113
struct rrd_dimension *next; // linking of dimensions within the same data set
114
115
storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER
@@ -135,6 +140,9 @@ struct rrd_stats {
140
unsigned long hash; // a simple hash on the id, to speed up searching
141
// we first compare hashes, and only if the hashes are equal we do string comparisons
142
143
+ int gap_when_lost_iterations; // after how many lost iterations a gap should be stored
144
+ // netdata will interpolate values for gaps lower than this
145
+
146
long priority;
147
148
long entries; // total number of entries in the data set
src/rrd2json.c
+35
-14
@@ -264,12 +264,14 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
264
calculated_number print_values[dimensions]; // keep the final value to be printed
265
int print_hidden[dimensions]; // keep hidden flags
266
int found_non_zero[dimensions];
267
+ int found_non_existing[dimensions];
268
269
// initialize them
270
for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
271
group_values[c] = print_values[c] = 0;
272
print_hidden[c] = rd->hidden;
273
found_non_zero[c] = 0;
274
+ found_non_existing[c] = 0;
275
}
276
277
// -------------------------------------------------------------------------
@@ -348,6 +350,7 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
350
count++;
351
group_count++;
352
353
+ // check if we have to print this now
354
if(((count - pad) % group) == 0) {
355
if(printed >= entries_to_show) {
356
// debug(D_RRD_STATS, "Already printed all rows. Stopping.");
@@ -356,8 +359,10 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
359
360
if(group_count != group) {
361
// this is an incomplete group, skip it.
359
- for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++)
362
+ for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
363
group_values[c] = 0;
364
+ found_non_existing[c] = 0;
365
+ }
366
367
group_count = 0;
368
continue;
@@ -379,9 +384,17 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
384
print_this = 1;
385
}
386
387
+ // do the calculations
388
for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
383
- calculated_number value = unpack_storage_number(rd->values[t]);
389
+ storage_number n = rd->values[t];
390
+ calculated_number value = unpack_storage_number(n);
391
392
+ if(!does_storage_number_exist(n)) {
393
+ value = 0.0;
394
+ found_non_existing[c]++;
395
+ }
396
+ if(did_storage_number_reset(n)) annotate_reset = 1;
397
+
398
switch(group_method) {
399
case GROUP_MAX:
400
if(abs(value) > abs(group_values[c])) group_values[c] = value;
@@ -394,19 +407,12 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
407
default:
408
case GROUP_AVERAGE:
409
group_values[c] += value;
397
- if(print_this) group_values[c] /= group_count;
410
+ if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
411
break;
412
}
400
-
401
- if(print_this) {
402
- print_values[c] = group_values[c];
403
- group_values[c] = 0;
404
- }
413
}
414
415
if(print_this) {
408
- group_count = 0;
409
-
416
if(annotate_reset) {
417
annotation_count++;
418
web_buffer_strcpy(wb, overflow_annotation);
@@ -417,22 +423,36 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
423
424
pc = 0;
425
for(c = 0 ; c < dimensions ; c++) {
420
- if(!print_hidden[c]) {
426
+ if(found_non_existing[c] == group_count) {
427
+ // all entries are non-existing
428
pc++;
429
web_buffer_strcpy(wb, pre_value);
423
- web_buffer_rrd_value(wb, print_values[c]);
430
+ web_buffer_strcpy(wb, "null");
431
+ web_buffer_strcpy(wb, post_value);
432
+ }
433
+ else if(!print_hidden[c]) {
434
+ pc++;
435
+ web_buffer_strcpy(wb, pre_value);
436
+ web_buffer_rrd_value(wb, group_values[c]);
437
web_buffer_strcpy(wb, post_value);
438
426
- if(print_values[c]) found_non_zero[c]++;
439
+ if(group_values[c]) found_non_zero[c]++;
440
}
441
+
442
+ // reset them for the next loop
443
+ group_values[c] = 0;
444
+ found_non_existing[c] = 0;
445
}
446
+
447
+ // if all dimensions are hidden, print a null
448
if(!pc) {
449
web_buffer_strcpy(wb, pre_value);
431
- web_buffer_rrd_value(wb, (calculated_number)0);
450
+ web_buffer_strcpy(wb, "null");
451
web_buffer_strcpy(wb, post_value);
452
}
453
454
printed++;
455
+ group_count = 0;
456
}
457
}
458
@@ -443,6 +463,7 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
463
int changed = 0;
464
for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
465
group_values[c] = 0;
466
+ found_non_existing[c] = 0;
467
468
if(!print_hidden[c] && !found_non_zero[c]) {
469
changed = 1;
src/storage_number.c
+43
-29
@@ -13,57 +13,70 @@
13
#endif
14
#endif
15
16
-storage_number pack_storage_number(calculated_number value)
16
+storage_number pack_storage_number(calculated_number value, uint32_t flags)
17
{
18
- storage_number r = 0;
19
- if(!value) return r;
20
-
21
- // bit 32 = sign
18
+ // bit 32 = sign 0:positive, 1:negative
19
// bit 31 = 0:divide, 1:multiply
23
- // bit 30, 29, 28 = (multiplier or divider) 0-7
24
- // bit 27 to 25 = reserved for flags
20
+ // bit 30, 29, 28 = (multiplier or divider) 0-6 (7 total)
21
+ // bit 27, 26, 25 flags
22
// bit 24 to bit 1 = the value
23
27
- storage_number sign = 0, exp = 0, mul;
24
+ storage_number r = get_storage_number_flags(flags);
25
+ if(!value) return r;
26
+
27
int m = 0;
28
calculated_number n = value;
29
30
+ // if the value is negative
31
+ // add the sign bit and make it positive
32
if(n < 0) {
32
- sign = 1;
33
+ r += (1 << 31); // the sign bit 32
34
n = -n;
35
}
36
37
+ // make its integer part fit in 0x00ffffff
38
+ // by dividing it by 10 up to 7 times
39
+ // and increasing the multiplier
40
while(m < 7 && n > (calculated_number)0x00ffffff) {
41
n /= 10;
42
m++;
43
}
40
- while(m > -7 && n < (calculated_number)0x00199999) {
41
- n *= 10;
42
- m--;
43
- }
44
45
- if(m <= 0) {
46
- exp = 0;
47
- m = -m;
48
- }
49
- else exp = 1;
45
+ if(m) {
46
+ // the value was too big and we divided it
47
+ // so we add a multiplier to unpack it
48
+ r += (1 << 30) + (m << 27); // the multiplier m
49
51
- if(n > (calculated_number)0x00ffffff) {
52
- error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value);
53
- n = (calculated_number)0x00ffffff;
50
+ if(n > (calculated_number)0x00ffffff) {
51
+ error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value);
52
+ r += 0x00ffffff;
53
+ return r;
54
+ }
55
+ }
56
+ else {
57
+ // 0x0019999e is the number that can be multiplied
58
+ // by 10 to give 0x00ffffff
59
+ // while the value is below 0x0019999e we can
60
+ // multiply it by 10, up to 7 times, increasing
61
+ // the multiplier
62
+ while(m < 7 && n < (calculated_number)0x0019999e) {
63
+ n *= 10;
64
+ m++;
65
+ }
66
+
67
+ // the value was small enough and we multiplied it
68
+ // so we add a divider to unpack it
69
+ r += (0 << 30) + (m << 27); // the divider m
70
}
55
-
56
- mul = m;
71
72
#ifdef STORAGE_WITH_MATH
73
// without this there are rounding problems
74
// example: 0.9 becomes 0.89
61
- n = lrint(n);
75
+ r += lrint(n);
76
+#else
77
+ r += (storage_number)n;
78
#endif
79
64
- r = (sign << 31) + (exp << 30) + (mul << 27) + n;
65
- // fprintf(stderr, "PACK: %08X, sign = %d, exp = %d, mul = %d, n = " CALCULATED_NUMBER_FORMAT "\n", r, sign, exp, mul, n);
66
-
80
return r;
81
}
82
@@ -71,8 +84,9 @@ calculated_number unpack_storage_number(storage_number value)
84
{
85
if(!value) return 0;
86
74
- int sign = 0;
75
- int exp = 0;
87
+ int sign = 0, exp = 0;
88
+
89
+ value ^= get_storage_number_flags(value);
90
91
if(value & (1 << 31)) {
92
sign = 1;
src/storage_number.h
+22
-1
@@ -15,7 +15,24 @@ typedef int32_t storage_number;
15
typedef uint32_t ustorage_number;
16
#define STORAGE_NUMBER_FORMAT "%d"
17
18
-storage_number pack_storage_number(calculated_number value);
18
+#define SN_NOT_EXISTS (0x0 << 24)
19
+#define SN_EXISTS (0x1 << 24)
20
+#define SN_EXISTS_RESET (0x2 << 24)
21
+#define SN_EXISTS_UNDEF1 (0x3 << 24)
22
+#define SN_EXISTS_UNDEF2 (0x4 << 24)
23
+#define SN_EXISTS_UNDEF3 (0x5 << 24)
24
+#define SN_EXISTS_UNDEF4 (0x6 << 24)
25
+
26
+#define SN_FLAGS_MASK (~(0x6 << 24))
27
+
28
+// extract the flags
29
+#define get_storage_number_flags(value) ((((storage_number)value) & (1 << 24)) | (((storage_number)value) & (2 << 24)) | (((storage_number)value) & (4 << 24)))
30
+
31
+// checks
32
+#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0)
33
+#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0)
34
+
35
+storage_number pack_storage_number(calculated_number value, uint32_t flags);
36
calculated_number unpack_storage_number(storage_number value);
37
38
int print_calculated_number(char *str, calculated_number value);
@@ -25,4 +42,8 @@ int print_calculated_number(char *str, calculated_number value);
42
#define STORAGE_NUMBER_NEGATIVE_MAX -0.00001
43
#define STORAGE_NUMBER_NEGATIVE_MIN -167772150000000.0
44
45
+// accepted accuracy loss
46
+#define ACCURACY_LOSS 0.0001
47
+#define accuracy_loss(t1, t2) ((t1 == t2 || t1 == 0.0 || t2 == 0.0) ? 0.0 : (100.0 - ((t1 > t2) ? (t2 * 100.0 / t1 ) : (t1 * 100.0 / t2))))
48
+
49
#endif /* NETDATA_STORAGE_NUMBER_H */
src/unit_test.c
+38
-4
@@ -8,14 +8,18 @@
8
#include "log.h"
9
#include "web_buffer.h"
10
11
-#define ACCURACY_LOSS 0.0000001
12
-
11
int check_storage_number(calculated_number n, int debug) {
12
char buffer[100];
13
+ uint32_t flags = SN_EXISTS;
14
16
- storage_number s = pack_storage_number(n);
15
+ storage_number s = pack_storage_number(n, flags);
16
calculated_number d = unpack_storage_number(s);
17
18
+ if(!does_storage_number_exist(s)) {
19
+ fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n);
20
+ return 5;
21
+ }
22
+
23
calculated_number ddiff = d - n;
24
calculated_number dcdiff = ddiff * 100.0 / n;
25
@@ -152,7 +156,7 @@ void benchmark_storage_number(int loop, int multiplier) {
156
n *= multiplier;
157
if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN;
158
155
- s = pack_storage_number(n);
159
+ s = pack_storage_number(n, 1);
160
d = unpack_storage_number(s);
161
print_calculated_number(buffer, d);
162
}
@@ -177,8 +181,38 @@ void benchmark_storage_number(int loop, int multiplier) {
181
182
}
183
184
+static int check_storage_number_exists() {
185
+ uint32_t flags = SN_EXISTS;
186
+
187
+
188
+ for(flags = 0; flags < 7 ; flags++) {
189
+ if(get_storage_number_flags(flags << 24) != flags << 24) {
190
+ fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24));
191
+ return 1;
192
+ }
193
+ }
194
+
195
+ flags = SN_EXISTS;
196
+ calculated_number n = 0.0;
197
+
198
+ storage_number s = pack_storage_number(n, flags);
199
+ calculated_number d = unpack_storage_number(s);
200
+ if(get_storage_number_flags(s) != flags) {
201
+ fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s));
202
+ return 1;
203
+ }
204
+ if(n != d) {
205
+ fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d);
206
+ return 1;
207
+ }
208
+
209
+ return 0;
210
+}
211
+
212
int unit_test_storage()
213
{
214
+ if(check_storage_number_exists()) return 0;
215
+
216
calculated_number c, a = 0;
217
int i, j, g, r = 0;
218