fixed chart startup interpolation / extrapolation to allow the first point added to the chart be as accurate as possible; fixes #981
Costa Tsaousis (ktsaou) committed
Sep 22, 2016 at 22:50 UTC
1d97faadc071ac64173951517d53504d2fb6260a
2 files changed
+88
-20
src/rrd.c
+47
-15
@@ -5,10 +5,12 @@
5
// ----------------------------------------------------------------------------
6
// globals
7
8
+/*
9
// if not zero it gives the time (in seconds) to remove un-updated dimensions
10
// DO NOT ENABLE
11
// if dimensions are removed, the chart generation will have to run again
12
int rrd_delete_unupdated_dimensions = 0;
13
+*/
14
15
int rrd_update_every = UPDATE_EVERY;
16
int rrd_default_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
@@ -986,25 +988,42 @@ unsigned long long rrdset_done(RRDSET *st)
988
989
debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name);
990
989
- RRDDIM *rd, *last;
990
- int oldstate, store_this_entry = 1, first_entry = 0;
991
- unsigned long long last_stored_ut, now_collect_ut, last_collect_ut, next_store_ut, stored_entries = 0;
991
+ RRDDIM *rd;
992
+
993
+ int
994
+ pthreadoldcancelstate; // store the old cancelable pthread state, to restore it at the end
995
+
996
+ char
997
+ store_this_entry = 1, // boolean: 1 = store this entry, 0 = don't store this entry
998
+ first_entry = 0; // boolean: 1 = this is the first entry seen for this chart, 0 = all other entries
999
993
- if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
1000
+ unsigned int
1001
+ stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done()
1002
+
1003
+ unsigned long long
1004
+ last_collect_ut, // the timestamp in microseconds, of the last collected value
1005
+ now_collect_ut, // the timestamp in microseconds, of this collected value (this is NOW)
1006
+ last_stored_ut, // the timestamp in microseconds, of the last stored entry in the db
1007
+ next_store_ut, // the timestamp in microseconds, of the next entry to store in the db
1008
+ update_every_ut = st->update_every * 1000000ULL; // st->update_every in microseconds
1009
+
1010
+ if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0))
1011
error("Cannot set pthread cancel state to DISABLE.");
1012
1013
// a read lock is OK here
1014
pthread_rwlock_rdlock(&st->rwlock);
1015
1016
+/*
1017
// enable the chart, if it was disabled
1018
if(unlikely(rrd_delete_unupdated_dimensions) && !st->enabled)
1019
st->enabled = 1;
1020
+*/
1021
1022
// check if the chart has a long time to be updated
1004
- if(unlikely(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL)) {
1023
+ if(unlikely(st->usec_since_last_update > st->entries * update_every_ut)) {
1024
info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
1025
rrdset_reset(st);
1007
- st->usec_since_last_update = st->update_every * 1000000ULL;
1026
+ st->usec_since_last_update = update_every_ut;
1027
first_entry = 1;
1028
}
1029
if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
@@ -1014,7 +1033,7 @@ unsigned long long rrdset_done(RRDSET *st)
1033
// it is the first entry
1034
// set the last_collected_time to now
1035
gettimeofday(&st->last_collected_time, NULL);
1017
- last_collect_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - (st->update_every * 1000000);
1036
+ last_collect_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - update_every_ut;
1037
1038
// the first entry should not be stored
1039
store_this_entry = 0;
@@ -1048,11 +1067,11 @@ unsigned long long rrdset_done(RRDSET *st)
1067
}
1068
1069
// check if we will re-write the entire data set
1051
- if(unlikely(usec_dt(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) {
1070
+ if(unlikely(usec_dt(&st->last_collected_time, &st->last_updated) > st->entries * update_every_ut)) {
1071
info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
1072
rrdset_reset(st);
1073
1055
- st->usec_since_last_update = st->update_every * 1000000ULL;
1074
+ st->usec_since_last_update = update_every_ut;
1075
1076
gettimeofday(&st->last_collected_time, NULL);
1077
@@ -1272,10 +1291,10 @@ unsigned long long rrdset_done(RRDSET *st)
1291
}
1292
1293
unsigned long long first_ut = last_stored_ut;
1275
- long long iterations = (now_collect_ut - last_stored_ut) / (st->update_every * 1000000ULL);
1276
- if((now_collect_ut % (st->update_every * 1000000ULL)) == 0) iterations++;
1294
+ long long iterations = (now_collect_ut - last_stored_ut) / (update_every_ut);
1295
+ if((now_collect_ut % (update_every_ut)) == 0) iterations++;
1296
1278
- for( ; next_store_ut <= now_collect_ut ; next_store_ut += st->update_every * 1000000ULL, iterations-- ) {
1297
+ for( ; next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, iterations-- ) {
1298
#ifdef NETDATA_INTERNAL_CHECKS
1299
if(iterations < 0) { error("%s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
1300
#endif
@@ -1312,11 +1331,21 @@ unsigned long long rrdset_done(RRDSET *st)
1331
, (now_collect_ut - last_stored_ut)
1332
);
1333
1315
- last_collect_ut = next_store_ut;
1334
rd->calculated_value -= new_value;
1335
new_value += rd->last_calculated_value;
1336
rd->last_calculated_value = 0;
1337
new_value /= (calculated_number)st->update_every;
1338
+
1339
+ if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
1340
+ if(unlikely(st->debug))
1341
+ debug(D_RRD_STATS, "%s/%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING",
1342
+ st->id, rd->name
1343
+ , (calculated_number)(next_store_ut - last_stored_ut)
1344
+ );
1345
+ new_value = new_value * (calculated_number)(st->update_every * 1000000) / (calculated_number)(next_store_ut - last_stored_ut);
1346
+ }
1347
+
1348
+ last_collect_ut = next_store_ut;
1349
break;
1350
1351
case RRDDIM_ABSOLUTE:
@@ -1467,6 +1496,7 @@ unsigned long long rrdset_done(RRDSET *st)
1496
// ALL DONE ABOUT THE DATA UPDATE
1497
// --------------------------------------------------------------------
1498
1499
+/*
1500
// find if there are any obsolete dimensions (not updated recently)
1501
if(unlikely(rrd_delete_unupdated_dimensions)) {
1502
@@ -1475,6 +1505,7 @@ unsigned long long rrdset_done(RRDSET *st)
1505
break;
1506
1507
if(unlikely(rd)) {
1508
+ RRDDIM *last;
1509
// there is dimension to free
1510
// upgrade our read lock to a write lock
1511
pthread_rwlock_unlock(&st->rwlock);
@@ -1512,11 +1543,12 @@ unsigned long long rrdset_done(RRDSET *st)
1543
}
1544
}
1545
}
1546
+*/
1547
1548
pthread_rwlock_unlock(&st->rwlock);
1549
1518
- if(unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
1519
- error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1550
+ if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0))
1551
+ error("Cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate);
1552
1553
return(st->usec_since_last_update);
1554
}
src/unit_test.c
+41
-5
@@ -379,7 +379,7 @@ struct feed_values test4_feed[] = {
379
};
380
381
calculated_number test4_results[] = {
382
- 5, 10, 10, 10, 10, 10, 10, 10, 10
382
+ 10, 10, 10, 10, 10, 10, 10, 10, 10
383
};
384
385
struct test test4 = {
@@ -414,7 +414,7 @@ struct feed_values test5_feed[] = {
414
};
415
416
calculated_number test5_results[] = {
417
- 500, 500, 0, 500, 500, 0, 0, 0, 0
417
+ 1000, 500, 0, 500, 500, 0, 0, 0, 0
418
};
419
420
struct test test5 = {
@@ -455,7 +455,7 @@ struct feed_values test6_feed[] = {
455
};
456
457
calculated_number test6_results[] = {
458
- 3000, 4000, 4000, 4000
458
+ 4000, 4000, 4000, 4000
459
};
460
461
struct test test6 = {
@@ -490,7 +490,7 @@ struct feed_values test7_feed[] = {
490
};
491
492
calculated_number test7_results[] = {
493
- 250, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500
493
+ 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500
494
};
495
496
struct test test7 = {
@@ -597,7 +597,7 @@ struct feed_values test10_feed[] = {
597
};
598
599
calculated_number test10_results[] = {
600
- 500, 1000, 1000, 1000, 1000, 1000, 1000
600
+ 1000, 1000, 1000, 1000, 1000, 1000, 1000
601
};
602
603
struct test test10 = {
@@ -770,6 +770,7 @@ struct test test14 = {
770
NULL, // feed2
771
NULL // results2
772
};
773
+
774
struct feed_values test14b_feed[] = {
775
{ 0, 0 },
776
{ 13573000, 13573000 },
@@ -802,6 +803,38 @@ struct test test14b = {
803
NULL // results2
804
};
805
806
+struct feed_values test14c_feed[] = {
807
+ { 29000000, 29000000 },
808
+ { 1000000, 29000000 + 1000000 },
809
+ { 30000000, 29000000 + 1000000 + 30000000 },
810
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 },
811
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 },
812
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 },
813
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 },
814
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 },
815
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 },
816
+ { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 },
817
+};
818
+
819
+calculated_number test14c_results[] = {
820
+ 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000
821
+};
822
+
823
+struct test test14c = {
824
+ "test14c", // name
825
+ "issue #981 with dummy data, checking for late start",
826
+ 30, // update_every
827
+ 1, // multiplier
828
+ 1, // divisor
829
+ RRDDIM_INCREMENTAL, // algorithm
830
+ 10, // feed entries
831
+ 9, // result entries
832
+ test14c_feed, // feed
833
+ test14c_results, // results
834
+ NULL, // feed2
835
+ NULL // results2
836
+};
837
+
838
// --------------------------------------------------------------------------------------------------------------------
839
840
int run_test(struct test *test)
@@ -946,6 +979,9 @@ int run_all_mockup_tests(void)
979
if(run_test(&test14b))
980
return 1;
981
982
+ if(run_test(&test14c))
983
+ return 1;
984
+
985
return 0;
986
}
987