recalculate last_collected_total (#13945)
Costa Tsaousis committed
Nov 3, 2022 at 11:50 UTC
c270b6067bc91233696dc038c6356c1e384e13cf
6 files changed
+36
-48
collectors/plugins.d/pluginsd_parser.c
-5
@@ -1056,8 +1056,6 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
1056
{
1057
char *last_collected_ut_str = get_word(words, num_words, 1);
1058
char *last_updated_ut_str = get_word(words, num_words, 2);
1059
- char *last_collected_total_str = get_word(words, num_words, 3);
1060
- char *collected_total_str = get_word(words, num_words, 4);
1059
1060
RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
1061
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
@@ -1082,9 +1080,6 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
1080
st->last_updated.tv_usec = last_updated_ut % USEC_PER_SEC;
1081
}
1082
1085
- st->last_collected_total = last_collected_total_str ? strtoll(last_collected_total_str, NULL, 0) : 0;
1086
- st->collected_total = collected_total_str ? strtoll(collected_total_str, NULL, 0) : 0;
1087
-
1083
st->counter++;
1084
st->counter_done++;
1085
database/rrd.h
-3
@@ -607,9 +607,6 @@ struct rrdset {
607
struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)
608
struct timeval last_collected_time; // when did this data set last collected values
609
610
- total_number collected_total; // used internally to calculate percentages
611
- total_number last_collected_total; // used internally to calculate percentages
612
-
610
size_t rrdlabels_last_saved_version;
611
612
DICTIONARY *functions_view; // collector functions this rrdset supports, can be NULL
database/rrddim.c
+4
-1
@@ -601,7 +601,7 @@ struct rrddim_map_save_v019 {
601
long double last_calculated_value; // ignored
602
long double last_stored_value; // ignored
603
long long collected_value; // ignored
604
- long long last_collected_value; // ignored
604
+ long long last_collected_value; // load and save
605
long double collected_volume; // ignored
606
long double stored_volume; // ignored
607
void *next; // ignored
@@ -624,6 +624,7 @@ void rrddim_memory_file_update(RRDDIM *rd) {
624
625
rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
626
rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;
627
+ rd_on_file->last_collected_value = rd->last_collected_value;
628
}
629
630
void rrddim_memory_file_free(RRDDIM *rd) {
@@ -703,6 +704,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
704
}
705
706
if(!reset) {
707
+ rd->last_collected_value = rd_on_file->last_collected_value;
708
+
709
if(rd_on_file->algorithm != rd->algorithm)
710
info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
711
fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
database/rrdset.c
+30
-34
@@ -1510,13 +1510,16 @@ after_first_database_work:
1510
if(unlikely(rrdhost_has_rrdpush_sender_enabled(st->rrdhost)))
1511
rrdset_done_push(st);
1512
1513
+ uint32_t has_reset_value = 0;
1514
+
1515
size_t rda_slots = dictionary_entries(st->rrddim_root_index);
1516
struct rda_item *rda_base = rrdset_thread_rda(&rda_slots);
1517
1518
size_t dim_id;
1519
size_t dimensions = 0;
1520
struct rda_item *rda = rda_base;
1519
- st->collected_total = 0;
1521
+ total_number collected_total = 0;
1522
+ total_number last_collected_total = 0;
1523
rrddim_foreach_read(rd, st) {
1524
if(rd_dfe.counter >= rda_slots)
1525
break;
@@ -1535,7 +1538,24 @@ after_first_database_work:
1538
1539
// calculate totals
1540
if(likely(rd->updated)) {
1538
- st->collected_total += rd->collected_value;
1541
+ // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1542
+ // to reset the calculation (it will give zero as the calculation for this second)
1543
+ if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->last_collected_value > rd->collected_value)) {
1544
+ debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
1545
+ , rrdset_id(st)
1546
+ , rrddim_name(rd)
1547
+ , rd->last_collected_value
1548
+ , rd->collected_value
1549
+ );
1550
+
1551
+ if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
1552
+ has_reset_value = 1;
1553
+
1554
+ rd->last_collected_value = rd->collected_value;
1555
+ }
1556
+
1557
+ last_collected_total += rd->last_collected_value;
1558
+ collected_total += rd->collected_value;
1559
1560
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
1561
error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
@@ -1554,8 +1574,6 @@ after_first_database_work:
1574
rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
1575
rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
1576
1557
- uint32_t has_reset_value = 0;
1558
-
1577
// process all dimensions to calculate their values
1578
// based on the collected figures only
1579
// at this stage we do not interpolate anything
@@ -1599,7 +1617,7 @@ after_first_database_work:
1617
break;
1618
1619
case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
1602
- if(unlikely(!st->collected_total))
1620
+ if(unlikely(!collected_total))
1621
rd->calculated_value = 0;
1622
else
1623
// the percentage of the current value
@@ -1607,7 +1625,7 @@ after_first_database_work:
1625
rd->calculated_value =
1626
(NETDATA_DOUBLE)100
1627
* (NETDATA_DOUBLE)rd->collected_value
1610
- / (NETDATA_DOUBLE)st->collected_total;
1628
+ / (NETDATA_DOUBLE)collected_total;
1629
1630
rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
1631
" * " COLLECTED_NUMBER_FORMAT
@@ -1615,7 +1633,7 @@ after_first_database_work:
1633
, rrddim_name(rd)
1634
, rd->calculated_value
1635
, rd->collected_value
1618
- , st->collected_total
1636
+ , collected_total
1637
);
1638
break;
1639
@@ -1694,31 +1712,15 @@ after_first_database_work:
1712
continue;
1713
}
1714
1697
- // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1698
- // to reset the calculation (it will give zero as the calculation for this second)
1699
- if(unlikely(rd->last_collected_value > rd->collected_value)) {
1700
- debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
1701
- , rrdset_id(st)
1702
- , rrddim_name(rd)
1703
- , rd->last_collected_value
1704
- , rd->collected_value
1705
- );
1706
-
1707
- if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
1708
- has_reset_value = 1;
1709
-
1710
- rd->last_collected_value = rd->collected_value;
1711
- }
1712
-
1715
// the percentage of the current increment
1716
// over the increment of all dimensions together
1715
- if(unlikely(st->collected_total == st->last_collected_total))
1717
+ if(unlikely(collected_total == last_collected_total))
1718
rd->calculated_value = 0;
1719
else
1720
rd->calculated_value =
1721
(NETDATA_DOUBLE)100
1722
* (NETDATA_DOUBLE)(rd->collected_value - rd->last_collected_value)
1721
- / (NETDATA_DOUBLE)(st->collected_total - st->last_collected_total);
1723
+ / (NETDATA_DOUBLE)(collected_total - last_collected_total);
1724
1725
rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
1726
" * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
@@ -1726,7 +1728,7 @@ after_first_database_work:
1728
, rrddim_name(rd)
1729
, rd->calculated_value
1730
, rd->collected_value, rd->last_collected_value
1729
- , st->collected_total, st->last_collected_total
1731
+ , collected_total, last_collected_total
1732
);
1733
break;
1734
@@ -1780,8 +1782,6 @@ after_first_database_work:
1782
);
1783
1784
after_second_database_work:
1783
- st->last_collected_total = st->collected_total;
1784
-
1785
for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
1786
rd = rda->rd;
1787
if(unlikely(!rd)) continue;
@@ -1953,8 +1953,8 @@ struct rrdset_map_save_v019 {
1953
usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
1954
struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
1955
struct timeval last_collected_time; // ignored
1956
- long long collected_total; // NEEDS TO BE UPDATED - maintained on load
1957
- long long last_collected_total; // NEEDS TO BE UPDATED - maintained on load
1956
+ long long collected_total; // ignored
1957
+ long long last_collected_total; // ignored
1958
void *rrdfamily; // ignored
1959
void *rrdhost; // ignored
1960
void *next; // ignored
@@ -1978,8 +1978,6 @@ void rrdset_memory_file_update(RRDSET *st) {
1978
st_on_file->usec_since_last_update = st->usec_since_last_update;
1979
st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
1980
st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
1981
- st_on_file->collected_total = st->collected_total;
1982
- st_on_file->last_collected_total = st->last_collected_total;
1981
}
1982
1983
const char *rrdset_cache_filename(RRDSET *st) {
@@ -2071,8 +2069,6 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo
2069
st->usec_since_last_update = st_on_file->usec_since_last_update;
2070
st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
2071
st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
2074
- st->collected_total = st_on_file->collected_total;
2075
- st->last_collected_total = st_on_file->last_collected_total;
2072
2073
// link it to st
2074
st->st_on_file = st_on_file;
health/health.c
-1
@@ -809,7 +809,6 @@ static void health_execute_delayed_initializations(RRDHOST *host) {
809
rrddimvar_index_init(st);
810
811
rrdsetvar_add_and_leave_released(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_FLAG_NONE);
812
- rrdsetvar_add_and_leave_released(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, RRDVAR_FLAG_NONE);
812
rrdsetvar_add_and_leave_released(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, RRDVAR_FLAG_NONE);
813
rrdsetvar_add_and_leave_released(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, RRDVAR_FLAG_NONE);
814
rrdsetvar_add_and_leave_released(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_FLAG_NONE);
streaming/replication.c
+2
-4
@@ -143,11 +143,9 @@ static void replicate_chart_collection_state(BUFFER *wb, RRDSET *st) {
143
}
144
rrddim_foreach_done(rd);
145
146
- buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " %llu %llu " TOTAL_NUMBER_FORMAT " " TOTAL_NUMBER_FORMAT "\n",
146
+ buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " %llu %llu\n",
147
(usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec,
148
- (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec,
149
- st->last_collected_total,
150
- st->collected_total
148
+ (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec
149
);
150
}
151