Fix deadlock on dimension creation (#22400)
fix(rrddim): break AB-BA deadlock on rrddim_root_index The heterogeneity peek in the insert callback iterated the same dict's items list while the inserter still held its index write-lock; the maintenance archiver (items-write + dictionary_del wanting index-write) deadlocked against it. Move the peek to the react callback, where no dict lock is held.
Stelios Fragkakis committed
May 4, 2026 at 09:12 UTC
aac8eae539f4c374468811acb9b4579dcb9b4eda
1 file changed
+39
-25
src/database/rrddim.c
+39
-25
@@ -143,31 +143,11 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
143
netdata_log_error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
144
}
145
146
- if(rrdset_number_of_dimensions(st) != 0) {
147
- RRDDIM *td;
148
- dfe_start_write(st->rrddim_root_index, td) {
149
- if(td) break;
150
- }
151
- dfe_done(td);
152
-
153
- if(td && (td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor))) {
154
- if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
155
-#ifdef NETDATA_INTERNAL_CHECKS
156
- netdata_log_info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already "
157
- "present (algorithm is '%s' vs '%s', multiplier is %d vs %d, "
158
- "divisor is %d vs %d).",
159
- rrddim_name(rd),
160
- rrdset_name(st),
161
- rrdhost_hostname(host),
162
- rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm),
163
- rd->multiplier, td->multiplier,
164
- rd->divisor, td->divisor
165
- );
166
-#endif
167
- rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS);
168
- }
169
- }
170
- }
146
+ // NOTE: the heterogeneity check has moved to rrddim_react_callback().
147
+ // Running it here calls dfe_start_*() on st->rrddim_root_index while the
148
+ // index write-lock of that dict is still held by the inserter — AB-BA
149
+ // against svc_rrdset_archive_obsolete_dimensions() which holds items-write
150
+ // and then calls dictionary_del() that wants index-write.
151
152
// let the chart resync
153
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
@@ -300,6 +280,40 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
280
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
281
}
282
283
+ if(ctr->react_action & RRDDIM_REACT_NEW) {
284
+ // heterogeneity check (in react not insert: insert still holds index-write).
285
+ // compare and flag-set inside the loop body so td stays referenced via the
286
+ // dfe iterator (no UAF after dfe_done). rd is in the items list by react
287
+ // time, so skip td == rd.
288
+ RRDDIM *td;
289
+ dfe_start_read(st->rrddim_root_index, td) {
290
+ if(td == rd)
291
+ continue;
292
+
293
+ if(td->algorithm != rd->algorithm
294
+ || ABS(td->multiplier) != ABS(rd->multiplier)
295
+ || ABS(td->divisor) != ABS(rd->divisor)) {
296
+ if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
297
+#ifdef NETDATA_INTERNAL_CHECKS
298
+ netdata_log_info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already "
299
+ "present (algorithm is '%s' vs '%s', multiplier is %d vs %d, "
300
+ "divisor is %d vs %d).",
301
+ rrddim_name(rd),
302
+ rrdset_name(st),
303
+ rrdhost_hostname(st->rrdhost),
304
+ rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm),
305
+ rd->multiplier, td->multiplier,
306
+ rd->divisor, td->divisor
307
+ );
308
+#endif
309
+ rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS);
310
+ }
311
+ }
312
+ break;
313
+ }
314
+ dfe_done(td);
315
+ }
316
+
317
rrddim_metadata_updated(rd);
318
}
319