@cryptotaxi247 / netdata-1 / commits / 211914432

Fix sum on incremental dimensions for archived dims (#22580)

fix sum on incremental dimensions for archived dims query_target.c read the algorithm from the live RRDDIM, so SUM grouping on incremental dims whose RRDDIM was no longer in memory (host disconnected, chart obsoleted, parent-side child history) skipped the rate->volume scaling and returned raw rate sums, off by source_update_every. Persist RRD_ALGORITHM on RRDMETRIC, populate it from the live RRDDIM, the SQL dimension row at startup, and rrddim_set_algorithm at runtime. Switch query_target.c to read it atomically from RRDMETRIC, no live RRDDIM required. No schema migration; the dimension.algorithm column already exists.

Stelios Fragkakis committed Jun 8, 2026 at 10:59 UTC 211914432845e855c08e525270068b9e2a1a1515

8 files changed +74 -3

src/daemon/unit_test.c

+38

@@ -2,6 +2,7 @@

3 #include "common.h"

4 #include "web/api/formatters/rrd2json.h"

5 +#include "database/contexts/rrdcontext-internal.h"

7 #if defined(OS_LINUX)

8 #include "collectors/proc.plugin/plugin_proc.h"

@@ -1522,6 +1523,40 @@ static int test_incremental_sum_lookup_respects_update_every(void) {

1523 return rc;

1524 }

1525

1526 +static int test_rrdmetric_algorithm_follows_rrddim(void) {

1527 + fprintf(stderr, "%s() running...\n", __FUNCTION__);

1528 +

1529 + RRD_DB_MODE old_default_rrd_memory_mode = default_rrd_memory_mode;

1530 + default_rrd_memory_mode = RRD_DB_MODE_ALLOC;

1531 +

1532 + RRDSET *st = rrdset_create_localhost(

1533 + "netdata", "unittest-algo-track", "unittest-algo-track", "netdata", NULL,

1534 + "Unit Testing", "x", "unittest", NULL, 1,

1535 + nd_profile.update_every, RRDSET_TYPE_LINE);

1536 + RRDDIM *rd = rrddim_add(st, "d", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1537 +

1538 + int rc = 0;

1539 + RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdcontexts.rrdmetric);

1540 + if(!rm || rrdmetric_algorithm_atomic_load(rm) != RRD_ALGORITHM_INCREMENTAL) {

1541 + fprintf(stderr, "%s: after rrddim_add INCREMENTAL, rm->algorithm = %d (want %d)\n",

1542 + __FUNCTION__, rm ? (int)rrdmetric_algorithm_atomic_load(rm) : -1,

1543 + (int)RRD_ALGORITHM_INCREMENTAL);

1544 + rc = 1;

1545 + }

1546 +

1547 + rrddim_set_algorithm(st, rd, RRD_ALGORITHM_ABSOLUTE);

1548 + rm = rrdmetric_acquired_value(rd->rrdcontexts.rrdmetric);

1549 + if(!rm || rrdmetric_algorithm_atomic_load(rm) != RRD_ALGORITHM_ABSOLUTE) {

1550 + fprintf(stderr, "%s: after rrddim_set_algorithm ABSOLUTE, rm->algorithm = %d (want %d)\n",

1551 + __FUNCTION__, rm ? (int)rrdmetric_algorithm_atomic_load(rm) : -1,

1552 + (int)RRD_ALGORITHM_ABSOLUTE);

1553 + rc = 1;

1554 + }

1555 +

1556 + default_rrd_memory_mode = old_default_rrd_memory_mode;

1557 + return rc;

1558 +}

1559 +

1560 int run_all_mockup_tests(void)

1561 {

1562 fprintf(stderr, "%s() running...\n", __FUNCTION__ );

@@ -1541,6 +1576,9 @@ int run_all_mockup_tests(void)

1576 if(test_incremental_sum_lookup_respects_update_every())

1577 return 1;

1578

1579 + if(test_rrdmetric_algorithm_follows_rrddim())

1580 + return 1;

1581 +

1582 if(!test_variable_renames())

1583 return 1;

1584

src/database/contexts/query_target.c

+1 -1

@@ -270,7 +270,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON

270 } tier_retention[nd_profile.storage_tiers];

271

272 RRDDIM *rd = rrdmetric_rrddim_get_and_lock(rm);

273 - bool values_stored_as_rates = rd && rd->algorithm == RRD_ALGORITHM_INCREMENTAL;

273 + bool values_stored_as_rates = rrdmetric_algorithm_atomic_load(rm) == RRD_ALGORITHM_INCREMENTAL;

274

275 for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {

276 STORAGE_ENGINE *eng = qn->rrdhost->db[tier].eng;

src/database/contexts/rrdcontext-internal.h

+10

@@ -220,6 +220,7 @@ typedef struct rrdmetric {

220 STRING *name;

221

222 RRDDIM *rrddim;

223 + RRD_ALGORITHM algorithm; // atomic load/store; survives RRDDIM archive so query paths don't need a live rrddim

224

225 time_t first_time_s;

226 time_t last_time_s;

@@ -235,6 +236,14 @@ static ALWAYS_INLINE void rrdmetric_rrddim_atomic_store(RRDMETRIC *rm, RRDDIM *r

236 __atomic_store_n(&rm->rrddim, rd, __ATOMIC_RELEASE);

237 }

238

239 +static ALWAYS_INLINE RRD_ALGORITHM rrdmetric_algorithm_atomic_load(RRDMETRIC *rm) {

240 + return __atomic_load_n(&rm->algorithm, __ATOMIC_ACQUIRE);

241 +}

242 +

243 +static ALWAYS_INLINE void rrdmetric_algorithm_atomic_store(RRDMETRIC *rm, RRD_ALGORITHM algorithm) {

244 + __atomic_store_n(&rm->algorithm, algorithm, __ATOMIC_RELEASE);

245 +}

246 +

247 static ALWAYS_INLINE RRDDIM *rrdmetric_rrddim_get_and_lock(RRDMETRIC *rm) {

248 for(size_t retries = 0; retries < 5; retries++) {

249 RRDDIM *rd = rrdmetric_rrddim_atomic_load(rm);

@@ -437,6 +446,7 @@ static ALWAYS_INLINE void rrdmetric_release(RRDMETRIC_ACQUIRED *rma) {

446

447 void rrdmetric_rrddim_is_freed(RRDDIM *rd);

448 void rrdmetric_updated_rrddim_flags(RRDDIM *rd);

449 +void rrdmetric_updated_rrddim_algorithm(RRDDIM *rd);

450 void rrdmetric_collected_rrddim(RRDDIM *rd);

451 void rrdmetric_not_collected_rrddim(RRDDIM *rd);

452

src/database/contexts/rrdcontext-loading.c

@@ -60,6 +60,7 @@ static void rrdinstance_load_dimension_callback(SQL_DIMENSION_DATA *sd, void *da

60 .id = string_strdupz(sd->id),

61 .name = string_strdupz(sd->name),

62 .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomic

63 + .algorithm = (RRD_ALGORITHM)sd->algorithm,

64 };

65 if(sd->hidden) trm.flags |= RRD_FLAG_HIDDEN;

src/database/contexts/rrdcontext-metric.c

+20

@@ -170,6 +170,11 @@ static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus

170 rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);

171 }

172

173 + if(rrdmetric_algorithm_atomic_load(rm) != rm_new->algorithm) {

174 + rrdmetric_algorithm_atomic_store(rm, rm_new->algorithm);

175 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);

176 + }

177 +

178 if(!rm->first_time_s || (rm_new->first_time_s && rm_new->first_time_s < rm->first_time_s)) {

179 rm->first_time_s = rm_new->first_time_s;

180 rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);

@@ -256,6 +261,7 @@ void rrdmetric_from_rrddim(RRDDIM *rd) {

261 .name = string_dup(rd->name),

262 .flags = RRD_FLAG_NONE, // no need for atomics

263 .rrddim = rd,

264 + .algorithm = rd->algorithm,

265 };

266

267 RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_set_and_acquire_item(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));

@@ -314,6 +320,20 @@ inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {

320 rrdmetric_trigger_updates(rm, __FUNCTION__ );

321 }

322

323 +inline void rrdmetric_updated_rrddim_algorithm(RRDDIM *rd) {

324 + rrdmetric_not_collected_rrddim(rd);

325 +

326 + RRDMETRIC *rm = rrddim_get_rrdmetric(rd);

327 + if(unlikely(!rm)) return;

328 +

329 + if(rrdmetric_algorithm_atomic_load(rm) != rd->algorithm) {

330 + rrdmetric_algorithm_atomic_store(rm, rd->algorithm);

331 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);

332 + }

333 +

334 + rrdmetric_trigger_updates(rm, __FUNCTION__ );

335 +}

336 +

337 ALWAYS_INLINE void rrdmetric_collected_rrddim(RRDDIM *rd) {

338 if(rd->rrdcontexts.collected)

339 return;

src/database/contexts/rrdcontext.c

+1 -1

@@ -50,7 +50,7 @@ ALWAYS_INLINE void rrdcontext_removed_rrddim(RRDDIM *rd) {

50 }

52 ALWAYS_INLINE void rrdcontext_updated_rrddim_algorithm(RRDDIM *rd) {

53 - rrdmetric_updated_rrddim_flags(rd);

53 + rrdmetric_updated_rrddim_algorithm(rd);

54 }

56 ALWAYS_INLINE void rrdcontext_updated_rrddim_multiplier(RRDDIM *rd) {

src/database/sqlite/sqlite_context.c

+2 -1

@@ -130,7 +130,7 @@ done:

130 }

131

132 // Dimension list

133 -#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END, c.type||'.'||c.id, c.context " \

133 +#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END, c.type||'.'||c.id, c.context, d.algorithm " \

134 "FROM dimension d, chart c WHERE c.host_id = @host_id AND d.chart_id = c.chart_id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC"

135 void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)

136 {

@@ -162,6 +162,7 @@ void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_

162 dimension_data.hidden = sqlite3_column_int(res, 3);

163 dimension_data.chart_id = (char *) sqlite3_column_text(res, 4);

164 dimension_data.context = (char *) sqlite3_column_text(res, 5);

165 + dimension_data.algorithm = sqlite3_column_int(res, 6);

166 dict_cb(&dimension_data, data);

167 }

168

src/database/sqlite/sqlite_context.h

@@ -25,6 +25,7 @@ typedef struct ctx_dimension {

25 char *id;

26 char *name;

27 bool hidden;

28 + int algorithm;

30 char *context;

31 char *chart_id;