statsd metrics do not loose their first data collection due to interpolation; added external plugins API chart option "store_first" to indicate that netdata should store the first collected value, assuming the last one was zero
Costa Tsaousis (ktsaou) committed
Jul 2, 2017 at 22:42 UTC
369a67deaaa575993ac5d0ad6706123dac74ef26
6 files changed
+52
-25
src/plugins_d.c
+5
@@ -286,6 +286,11 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
286
rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
287
else
288
rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
289
+
290
+ if(strstr(options, "store_first"))
291
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
292
+ else
293
+ rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
294
}
295
}
296
else if(likely(hash == DIMENSION_HASH && !strcmp(s, PLUGINSD_KEYWORD_DIMENSION))) {
src/rrd.h
+2
-1
@@ -225,7 +225,8 @@ typedef enum rrdset_flags {
225
RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete
226
RRDSET_FLAG_BACKEND_SEND = 1 << 4, // if set, this chart should be sent to backends
227
RRDSET_FLAG_BACKEND_IGNORE = 1 << 5, // if set, this chart should not be sent to backends
228
- RRDSET_FLAG_EXPOSED_UPSTREAM = 1 << 6 // if set, we have sent this chart to netdata master (streaming)
228
+ RRDSET_FLAG_EXPOSED_UPSTREAM = 1 << 6, // if set, we have sent this chart to netdata master (streaming)
229
+ RRDSET_FLAG_STORE_FIRST = 1 << 7 // if set, do not eliminate the first collection during interpolation
230
} RRDSET_FLAGS;
231
232
#ifdef HAVE_C___ATOMIC
src/rrddim.c
+5
-2
@@ -211,8 +211,11 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
211
rd->entries = st->entries;
212
rd->update_every = st->update_every;
213
214
- // prevent incremental calculation spikes
215
- rd->collections_counter = 0;
214
+ if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
215
+ rd->collections_counter = 1;
216
+ else
217
+ rd->collections_counter = 0;
218
+
219
rd->updated = 0;
220
rd->flags = 0x00000000;
221
src/rrdpush.c
+2
-1
@@ -79,7 +79,7 @@ static inline int need_to_send_chart_definition(RRDSET *st) {
79
static inline void send_chart_definition(RRDSET *st) {
80
rrdset_flag_set(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
81
82
- buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s\"\n"
82
+ buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s\"\n"
83
, st->id
84
, st->name
85
, st->title
@@ -91,6 +91,7 @@ static inline void send_chart_definition(RRDSET *st) {
91
, st->update_every
92
, rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
93
, rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":""
94
+ , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
95
);
96
97
RRDDIM *rd;
src/rrdset.c
+33
-20
@@ -236,14 +236,18 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
236
return entries;
237
}
238
239
-static inline void last_collected_time_align(struct timeval *tv, int update_every) {
240
- tv->tv_sec -= tv->tv_sec % update_every;
241
- tv->tv_usec = 500000;
239
+static inline void last_collected_time_align(RRDSET *st) {
240
+ st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
241
+
242
+ if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
243
+ st->last_collected_time.tv_usec = 0;
244
+ else
245
+ st->last_collected_time.tv_usec = 500000;
246
}
247
244
-static inline void last_updated_time_align(struct timeval *tv, int update_every) {
245
- tv->tv_sec -= tv->tv_sec % update_every;
246
- tv->tv_usec = 0;
248
+static inline void last_updated_time_align(RRDSET *st) {
249
+ st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
250
+ st->last_updated.tv_usec = 0;
251
}
252
253
// ----------------------------------------------------------------------------
@@ -512,8 +516,10 @@ RRDSET *rrdset_create_custom(
516
}
517
518
// make sure the database is aligned
515
- if(st->last_updated.tv_sec)
516
- last_updated_time_align(&st->last_updated, update_every);
519
+ if(st->last_updated.tv_sec) {
520
+ st->update_every = update_every;
521
+ last_updated_time_align(st);
522
+ }
523
}
524
525
// make sure we have the right memory mode
@@ -654,11 +660,11 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
660
661
st->last_collected_time.tv_sec = now.tv_sec - st->update_every;
662
st->last_collected_time.tv_usec = now.tv_usec;
657
- last_collected_time_align(&st->last_collected_time, st->update_every);
663
+ last_collected_time_align(st);
664
665
st->last_updated.tv_sec = now.tv_sec - st->update_every;
666
st->last_updated.tv_usec = now.tv_usec;
661
- last_updated_time_align(&st->last_updated, st->update_every);
667
+ last_updated_time_align(st);
668
669
microseconds = st->update_every * USEC_PER_SEC;
670
}
@@ -683,7 +689,7 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
689
690
static inline void rrdset_init_last_collected_time(RRDSET *st) {
691
now_realtime_timeval(&st->last_collected_time);
686
- last_collected_time_align(&st->last_collected_time, st->update_every);
692
+ last_collected_time_align(st);
693
}
694
695
static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
@@ -698,7 +704,11 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) {
704
// copy the last collected time to last updated time
705
st->last_updated.tv_sec = st->last_collected_time.tv_sec;
706
st->last_updated.tv_usec = st->last_collected_time.tv_usec;
701
- last_updated_time_align(&st->last_updated, st->update_every);
707
+
708
+ if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
709
+ st->last_updated.tv_sec -= st->update_every;
710
+
711
+ last_updated_time_align(st);
712
}
713
714
static inline void rrdset_done_push_exclusive(RRDSET *st) {
@@ -755,8 +765,8 @@ static inline size_t rrdset_done_interpolate(
765
if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
766
767
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
758
- debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/1000000.0);
759
- debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0);
768
+ debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/USEC_PER_SEC);
769
+ debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/USEC_PER_SEC);
770
}
771
#endif
772
@@ -803,7 +813,7 @@ static inline size_t rrdset_done_interpolate(
813
);
814
#endif
815
806
- new_value = new_value * (calculated_number)(st->update_every * 1000000) / (calculated_number)(next_store_ut - last_stored_ut);
816
+ new_value = new_value * (calculated_number)(st->update_every * USEC_PER_SEC) / (calculated_number)(next_store_ut - last_stored_ut);
817
}
818
break;
819
@@ -981,7 +991,7 @@ void rrdset_done(RRDSET *st) {
991
992
// check if the chart has a long time to be updated
993
if(unlikely(st->usec_since_last_update > st->entries * update_every_ut)) {
984
- info("host '%s', chart %s: took too long to be updated (%0.3Lf secs). Resetting it.", st->rrdhost->hostname, st->name, (long double)(st->usec_since_last_update / 1000000.0));
994
+ info("host '%s', chart %s: took too long to be updated (%0.3Lf secs). Resetting it.", st->rrdhost->hostname, st->name, (long double)st->usec_since_last_update / USEC_PER_SEC);
995
rrdset_reset(st);
996
st->usec_since_last_update = update_every_ut;
997
store_this_entry = 0;
@@ -1049,10 +1059,10 @@ void rrdset_done(RRDSET *st) {
1059
next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
1060
1061
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
1052
- debug(D_RRD_STATS, "%s: last_collect_ut = %0.3Lf (last collection time)", st->name, (long double)last_collect_ut/1000000.0);
1053
- debug(D_RRD_STATS, "%s: now_collect_ut = %0.3Lf (current collection time)", st->name, (long double)now_collect_ut/1000000.0);
1054
- debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/1000000.0);
1055
- debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0);
1062
+ debug(D_RRD_STATS, "%s: last_collect_ut = %0.3Lf (last collection time)", st->name, (long double)last_collect_ut/USEC_PER_SEC);
1063
+ debug(D_RRD_STATS, "%s: now_collect_ut = %0.3Lf (current collection time)", st->name, (long double)now_collect_ut/USEC_PER_SEC);
1064
+ debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/USEC_PER_SEC);
1065
+ debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/USEC_PER_SEC);
1066
}
1067
1068
if(unlikely(!st->counter_done)) {
@@ -1267,6 +1277,9 @@ void rrdset_done(RRDSET *st) {
1277
#endif
1278
}
1279
1280
+ if(unlikely(!store_this_entry && rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
1281
+ store_this_entry = 1;
1282
+
1283
rrdset_done_interpolate(st
1284
, update_every_ut
1285
, last_stored_ut
src/statsd.c
+5
-1
@@ -1246,7 +1246,7 @@ static inline RRDSET *statsd_private_rrdset_create(
1246
}
1247
1248
statsd.private_charts++;
1249
- return rrdset_create_custom(
1249
+ RRDSET *st = rrdset_create_custom(
1250
localhost
1251
, type
1252
, id
@@ -1261,6 +1261,8 @@ static inline RRDSET *statsd_private_rrdset_create(
1261
, memory_mode
1262
, history
1263
);
1264
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
1265
+ return st;
1266
}
1267
1268
static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
@@ -1654,6 +1656,8 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
1656
, app->rrd_memory_mode
1657
, app->rrd_history_entries
1658
);
1659
+
1660
+ rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);
1661
}
1662
else rrdset_next(chart->st);
1663