1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "rrdcontext-internal.h"
4
+#include "../sqlite/sqlite_aclk.h"
5
6
// ----------------------------------------------------------------------------
7
// visualizing flags
201
// ----------------------------------------------------------------------------
202
// ACLK interface
203
203
-void rrdcontext_hub_checkpoint_command(void *ptr) {
204
- struct ctxs_checkpoint *cmd = ptr;
204
+static void rrdcontext_checkpoint_clear_pending_unsafe(struct aclk_sync_cfg_t *aclk_host_config) {
205
+ freez(aclk_host_config->pending_ctx_claim_id);
206
+ freez(aclk_host_config->pending_ctx_node_id);
207
+ aclk_host_config->pending_ctx_claim_id = NULL;
208
+ aclk_host_config->pending_ctx_node_id = NULL;
209
+ aclk_host_config->pending_ctx_version_hash = 0;
210
+ aclk_host_config->pending_ctx_saved_monotonic_s = 0;
211
+ __atomic_store_n(&aclk_host_config->pending_ctx_checkpoint, false, __ATOMIC_RELEASE);
212
+}
213
206
- if(!claim_id_matches(cmd->claim_id)) {
207
- CLAIM_ID claim_id = claim_id_get();
208
- nd_log(NDLS_DAEMON, NDLP_WARNING,
209
- "RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', "
210
- "but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
211
- cmd->claim_id, cmd->node_id,
212
- claim_id.str, cmd->claim_id);
214
+static uint64_t rrdcontext_checkpoint_invalidate_pending(RRDHOST *host) {
215
+ struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_ACQUIRE);
216
+ if(!aclk_host_config)
217
+ return 0;
218
214
- return;
215
- }
219
+ spinlock_lock(&aclk_host_config->pending_ctx_spinlock);
220
+ uint64_t generation = __atomic_add_fetch(&aclk_host_config->pending_ctx_generation, 1, __ATOMIC_RELEASE);
221
+ rrdcontext_checkpoint_clear_pending_unsafe(aclk_host_config);
222
+ spinlock_unlock(&aclk_host_config->pending_ctx_spinlock);
223
217
- RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id);
218
- if(!host) {
219
- nd_log(NDLS_DAEMON, NDLP_WARNING,
220
- "RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', "
221
- "but there is no node with such node id here. Ignoring command.",
222
- cmd->claim_id, cmd->node_id);
224
+ return generation;
225
+}
226
+
227
+static bool rrdcontext_checkpoint_generation_is_current(RRDHOST *host, const char *claim_id, const char *node_id, uint64_t generation) {
228
+ if(!generation)
229
+ return true;
230
+
231
+ struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_ACQUIRE);
232
+ uint64_t current_generation = aclk_host_config ?
233
+ __atomic_load_n(&aclk_host_config->pending_ctx_generation, __ATOMIC_ACQUIRE) :
234
+ 0;
235
+
236
+ if(likely(aclk_host_config && current_generation == generation))
237
+ return true;
238
+
239
+ nd_log(NDLS_DAEMON, NDLP_DEBUG,
240
+ "RRDCONTEXT: skipping stale checkpoint for host '%s', claim id '%s', node id '%s' "
241
+ "(generation %"PRIu64", current %"PRIu64").",
242
+ rrdhost_hostname(host), claim_id, node_id, generation, current_generation);
243
+ return false;
244
+}
245
+
246
+// Save a pending checkpoint to be replayed when context processing completes.
247
+// Returns true if saved successfully, false if save failed (caller should execute immediately).
248
+static bool rrdcontext_checkpoint_save_pending(RRDHOST *host, struct ctxs_checkpoint *cmd) {
249
+ struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_RELAXED);
250
+ if(!aclk_host_config)
251
+ return false;
252
+
253
+ spinlock_lock(&aclk_host_config->pending_ctx_spinlock);
254
255
+ // Pause incremental context streaming until the deferred checkpoint is replayed.
256
+ rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
257
+
258
+ __atomic_add_fetch(&aclk_host_config->pending_ctx_generation, 1, __ATOMIC_RELEASE);
259
+ rrdcontext_checkpoint_clear_pending_unsafe(aclk_host_config);
260
+ aclk_host_config->pending_ctx_claim_id = strdupz(cmd->claim_id);
261
+ aclk_host_config->pending_ctx_node_id = strdupz(cmd->node_id);
262
+ aclk_host_config->pending_ctx_version_hash = cmd->version_hash;
263
+ aclk_host_config->pending_ctx_saved_monotonic_s = now_monotonic_sec();
264
+ __atomic_store_n(&aclk_host_config->pending_ctx_checkpoint, true, __ATOMIC_RELEASE);
265
+ spinlock_unlock(&aclk_host_config->pending_ctx_spinlock);
266
+ return true;
267
+}
268
+
269
+// Execute the checkpoint: compare version hash, send snapshot if needed, enable streaming.
270
+static void rrdcontext_checkpoint_execute(RRDHOST *host, const char *claim_id, const char *node_id, uint64_t version_hash, uint64_t generation) {
271
+ if(!rrdcontext_checkpoint_generation_is_current(host, claim_id, node_id, generation))
272
return;
225
- }
273
274
if(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
275
nd_log(NDLS_DAEMON, NDLP_NOTICE,
229
- "RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', "
276
+ "RRDCONTEXT: checkpoint for claim id '%s', node id '%s', "
277
"while node '%s' has an active context streaming.",
231
- cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
278
+ claim_id, node_id, rrdhost_hostname(host));
279
280
// disable it temporarily, so that our worker will not attempt to send messages in parallel
281
rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
283
284
uint64_t our_version_hash = rrdcontext_version_hash(host);
285
239
- if(cmd->version_hash != our_version_hash) {
286
+ if(version_hash != our_version_hash) {
287
nd_log(NDLS_DAEMON, NDLP_NOTICE,
288
"RRDCONTEXT: received version hash %"PRIu64" for host '%s', does not match our version hash %"PRIu64". "
289
"Sending snapshot of all contexts.",
243
- cmd->version_hash, rrdhost_hostname(host), our_version_hash);
290
+ version_hash, rrdhost_hostname(host), our_version_hash);
291
292
// prepare the snapshot
293
char uuid_str[UUID_STR_LEN];
294
uuid_unparse_lower(host->node_id.uuid, uuid_str);
248
- contexts_snapshot_t bundle = contexts_snapshot_new(cmd->claim_id, uuid_str, our_version_hash);
295
+ contexts_snapshot_t bundle = contexts_snapshot_new(claim_id, uuid_str, our_version_hash);
296
297
// do a deep scan on every metric of the host to make sure all our data are updated
298
rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, false);
300
// calculate version hash and pack all the messages together in one go
301
our_version_hash = rrdcontext_version_hash_with_callback(host, rrdcontext_message_send_unsafe, true, bundle);
302
303
+ if(!rrdcontext_checkpoint_generation_is_current(host, claim_id, node_id, generation)) {
304
+ contexts_snapshot_delete(bundle);
305
+ return;
306
+ }
307
+
308
// update the version
309
contexts_snapshot_set_version(bundle, our_version_hash);
310
312
aclk_send_contexts_snapshot(bundle);
313
}
314
315
+ if(!rrdcontext_checkpoint_generation_is_current(host, claim_id, node_id, generation))
316
+ return;
317
+
318
nd_log(NDLS_DAEMON, NDLP_DEBUG,
319
"RRDCONTEXT: host '%s' enabling streaming of contexts",
320
rrdhost_hostname(host));
321
267
- rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
322
+ struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_ACQUIRE);
323
+ if(aclk_host_config) {
324
+ spinlock_lock(&aclk_host_config->pending_ctx_spinlock);
325
+
326
+ bool can_enable =
327
+ __atomic_load_n(&aclk_host_config->pending_ctx_generation, __ATOMIC_ACQUIRE) == generation &&
328
+ !__atomic_load_n(&aclk_host_config->pending_ctx_checkpoint, __ATOMIC_ACQUIRE);
329
+
330
+ if(can_enable)
331
+ rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
332
+
333
+ spinlock_unlock(&aclk_host_config->pending_ctx_spinlock);
334
+
335
+ if(!can_enable)
336
+ return;
337
+ }
338
+ else
339
+ rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
340
+
341
char node_str[UUID_STR_LEN];
342
uuid_unparse_lower(host->node_id.uuid, node_str);
343
nd_log(NDLS_ACCESS, NDLP_DEBUG,
345
node_str, rrdhost_hostname(host));
346
}
347
348
+void rrdcontext_hub_checkpoint_command(void *ptr) {
349
+ struct ctxs_checkpoint *cmd = ptr;
350
+
351
+ if(!claim_id_matches(cmd->claim_id)) {
352
+ CLAIM_ID claim_id = claim_id_get();
353
+ nd_log(NDLS_DAEMON, NDLP_WARNING,
354
+ "RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', "
355
+ "but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
356
+ cmd->claim_id, cmd->node_id,
357
+ claim_id.str, cmd->claim_id);
358
+
359
+ return;
360
+ }
361
+
362
+ RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id);
363
+ if(!host) {
364
+ nd_log(NDLS_DAEMON, NDLP_WARNING,
365
+ "RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', "
366
+ "but there is no node with such node id here. Ignoring command.",
367
+ cmd->claim_id, cmd->node_id);
368
+
369
+ return;
370
+ }
371
+
372
+ if(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD) ||
373
+ rrdcontext_queue_entries(&host->rrdctx.pp_queue) > 0) {
374
+ nd_log(NDLS_DAEMON, NDLP_NOTICE,
375
+ "RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', "
376
+ "but host '%s' has pending context work. Saving checkpoint for replay after processing completes.",
377
+ cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
378
+
379
+ if(rrdcontext_checkpoint_save_pending(host, cmd))
380
+ return;
381
+
382
+ nd_log(NDLS_DAEMON, NDLP_WARNING,
383
+ "RRDCONTEXT: failed to save pending checkpoint for host '%s' (no aclk config). Executing immediately.",
384
+ rrdhost_hostname(host));
385
+ }
386
+
387
+ uint64_t generation = rrdcontext_checkpoint_invalidate_pending(host);
388
+ rrdcontext_checkpoint_execute(host, cmd->claim_id, cmd->node_id, cmd->version_hash, generation);
389
+}
390
+
391
void rrdcontext_hub_stop_streaming_command(void *ptr) {
392
struct stop_streaming_ctxs *cmd = ptr;
393
412
return;
413
}
414
415
+ bool had_pending_checkpoint = false;
416
+ struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_ACQUIRE);
417
+ if(aclk_host_config)
418
+ had_pending_checkpoint = __atomic_load_n(&aclk_host_config->pending_ctx_checkpoint, __ATOMIC_ACQUIRE);
419
+
420
+ rrdcontext_checkpoint_invalidate_pending(host);
421
+
422
if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
423
nd_log(NDLS_DAEMON, NDLP_NOTICE,
424
"RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', "
302
- "but node '%s' does not have active context streaming. Ignoring command.",
303
- cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
425
+ "but node '%s' does not have active context streaming%s.",
426
+ cmd->claim_id, cmd->node_id, rrdhost_hostname(host),
427
+ had_pending_checkpoint ? "; invalidated deferred checkpoint" : "");
428
429
return;
430
}
436
rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
437
}
438
439
+#define PENDING_CTX_CHECKPOINT_MAX_AGE_S 300
440
+
441
+void rrdcontext_hub_pending_checkpoint_replay(RRDHOST *host) {
442
+ struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_RELAXED);
443
+ if(!aclk_host_config || !__atomic_load_n(&aclk_host_config->pending_ctx_checkpoint, __ATOMIC_ACQUIRE))
444
+ return;
445
+
446
+ bool pending_context_load = rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);
447
+ bool pp_queue_empty = rrdcontext_queue_entries(&host->rrdctx.pp_queue) <= 0;
448
+
449
+ spinlock_lock(&aclk_host_config->pending_ctx_spinlock);
450
+ if(!__atomic_load_n(&aclk_host_config->pending_ctx_checkpoint, __ATOMIC_RELAXED)) {
451
+ spinlock_unlock(&aclk_host_config->pending_ctx_spinlock);
452
+ return;
453
+ }
454
+
455
+ // replay when pp_queue is empty, or force replay on timeout
456
+ time_t now_s = now_monotonic_sec();
457
+ bool timed_out = (now_s - aclk_host_config->pending_ctx_saved_monotonic_s >= PENDING_CTX_CHECKPOINT_MAX_AGE_S);
458
+ if((pending_context_load || !pp_queue_empty) && !timed_out) {
459
+ spinlock_unlock(&aclk_host_config->pending_ctx_spinlock);
460
+ return;
461
+ }
462
+
463
+ char *claim_id = aclk_host_config->pending_ctx_claim_id;
464
+ char *node_id = aclk_host_config->pending_ctx_node_id;
465
+ uint64_t version_hash = aclk_host_config->pending_ctx_version_hash;
466
+ uint64_t generation = __atomic_load_n(&aclk_host_config->pending_ctx_generation, __ATOMIC_RELAXED);
467
+
468
+ aclk_host_config->pending_ctx_claim_id = NULL;
469
+ aclk_host_config->pending_ctx_node_id = NULL;
470
+ aclk_host_config->pending_ctx_version_hash = 0;
471
+ aclk_host_config->pending_ctx_saved_monotonic_s = 0;
472
+ __atomic_store_n(&aclk_host_config->pending_ctx_checkpoint, false, __ATOMIC_RELEASE);
473
+ spinlock_unlock(&aclk_host_config->pending_ctx_spinlock);
474
+
475
+ if(timed_out && (pending_context_load || !pp_queue_empty))
476
+ nd_log(NDLS_DAEMON, NDLP_WARNING,
477
+ "RRDCONTEXT: pending checkpoint for host '%s' timed out after %d sec with context work still active. Forcing replay.",
478
+ rrdhost_hostname(host), PENDING_CTX_CHECKPOINT_MAX_AGE_S);
479
+
480
+ if(!claim_id || !node_id) {
481
+ freez(claim_id);
482
+ freez(node_id);
483
+ return;
484
+ }
485
+
486
+ // verify claim id is still valid
487
+ if(!claim_id_matches(claim_id)) {
488
+ nd_log(NDLS_DAEMON, NDLP_WARNING,
489
+ "RRDCONTEXT: pending checkpoint for host '%s' has stale claim id '%s'. Discarding.",
490
+ rrdhost_hostname(host), claim_id);
491
+ freez(claim_id);
492
+ freez(node_id);
493
+ return;
494
+ }
495
+
496
+ nd_log(NDLS_DAEMON, NDLP_NOTICE,
497
+ "RRDCONTEXT: replaying deferred checkpoint for host '%s', claim id '%s', node id '%s'.",
498
+ rrdhost_hostname(host), claim_id, node_id);
499
+
500
+ rrdcontext_checkpoint_execute(host, claim_id, node_id, version_hash, generation);
501
+
502
+ freez(claim_id);
503
+ freez(node_id);
504
+}
505
+
506
ALWAYS_INLINE
507
bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t before) {
508
if(unlikely(!rca)) return false;