@cryptotaxi247 / netdata-1 / commits / 321907160

Improve agent startup time (#21891)

* Refactor mmap advisory handling and optimize MRG population - Simplify and streamline `madvise_*` logic in journalfile for direct access pattern handling. - Add access hint support with `journalfile_v2_data_acquire_with_hint()` for improved advisory control. - Introduce resume-based indexing and progress logging for efficient MRG population workflows. * Optimize mmap advisory and access patterns in journalfile - Set initial `not_needed_since_s` to `0` for consistent initialization. - Add `madvise_*` hints to improve memory mapping usage and performance. - Introduce sequential read hints during validation for better kernel prefetching. * Update `journalfile` initialization and streamline memory advisory handling - Set `not_needed_since_s` to `now_monotonic_sec()` for accurate initialization. - Replace inline `madvise_*` calls with `journalfile_v2_mounted_data_unmount()` for consistent advisory handling.

Stelios Fragkakis committed Mar 5, 2026 at 17:00 UTC 3219071609790f2a55f9faccdd4e5edb5a2a7746

3 files changed +67 -15

src/database/engine/journalfile.c

+35 -5

@@ -223,15 +223,18 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin

223 madvise_dontfork(journalfile->mmap.data, journalfile->mmap.size);

224 madvise_dontdump(journalfile->mmap.data, journalfile->mmap.size);

225 // madvise_dontneed(journalfile->mmap.data, journalfile->mmap.size);

226 - madvise_random(journalfile->mmap.data, journalfile->mmap.size);

226

227 journalfile->v2.flags |= JOURNALFILE_FLAG_IS_AVAILABLE | JOURNALFILE_FLAG_IS_MOUNTED;

228 JOURNALFILE_FLAGS flags = journalfile->v2.flags;

229

230 if(flags & JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION) {

232 - // we need the entire metrics directory into memory to process it

231 + // sequential access pattern during MRG population

232 + madvise_sequential(journalfile->mmap.data, journalfile->v2.size_of_directory);

233 madvise_willneed(journalfile->mmap.data, journalfile->v2.size_of_directory);

234 }

235 + else {

236 + madvise_random(journalfile->mmap.data, journalfile->mmap.size);

237 + }

238 }

239 }

240

@@ -332,7 +335,9 @@ void journalfile_v2_data_unmount_cleanup(time_t now_s) {

335 }

336 }

337

335 -ALWAYS_INLINE struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s) {

338 +ALWAYS_INLINE struct journal_v2_header *journalfile_v2_data_acquire_with_hint(struct rrdengine_journalfile *journalfile,

339 + size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s, JOURNALFILE_V2_ACCESS_HINT hint)

340 +{

341 spinlock_lock(&journalfile->data_spinlock);

342

343 bool has_data = (journalfile->v2.flags & JOURNALFILE_FLAG_IS_AVAILABLE);

@@ -343,16 +348,32 @@ ALWAYS_INLINE struct journal_v2_header *journalfile_v2_data_acquire(struct rrden

348 if (!wanted_first_time_s || !wanted_last_time_s ||

349 is_page_in_time_range(journalfile->v2.first_time_s, journalfile->v2.last_time_s,

350 wanted_first_time_s, wanted_last_time_s) == PAGE_IS_IN_RANGE) {

351 + bool was_sequential = (journalfile->v2.flags & JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION);

352

353 journalfile->v2.refcount++;

354

355 do_we_need_it = true;

356

351 - if (!wanted_first_time_s && !wanted_last_time_s && !is_mounted)

357 + if(hint == JOURNALFILE_V2_ACCESS_AUTO)

358 + hint = (!wanted_first_time_s && !wanted_last_time_s) ?

359 + JOURNALFILE_V2_ACCESS_SEQUENTIAL_DIRECTORY :

360 + JOURNALFILE_V2_ACCESS_RANDOM;

361 +

362 + bool want_sequential = (hint == JOURNALFILE_V2_ACCESS_SEQUENTIAL_DIRECTORY);

363 + if (want_sequential)

364 journalfile->v2.flags |= JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION;

365 else

366 journalfile->v2.flags &= ~JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION;

367

368 + if (is_mounted && journalfile->mmap.data) {

369 + if (!was_sequential && want_sequential) {

370 + madvise_sequential(journalfile->mmap.data, journalfile->v2.size_of_directory);

371 + madvise_willneed(journalfile->mmap.data, journalfile->v2.size_of_directory);

372 + }

373 + else if (was_sequential && !want_sequential)

374 + madvise_random(journalfile->mmap.data, journalfile->mmap.size);

375 + }

376 +

377 }

378 }

379 spinlock_unlock(&journalfile->data_spinlock);

@@ -363,6 +384,10 @@ ALWAYS_INLINE struct journal_v2_header *journalfile_v2_data_acquire(struct rrden

384 return NULL;

385 }

386

387 +ALWAYS_INLINE struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s) {

388 + return journalfile_v2_data_acquire_with_hint(journalfile, data_size, wanted_first_time_s, wanted_last_time_s, JOURNALFILE_V2_ACCESS_AUTO);

389 +}

390 +

391 ALWAYS_INLINE void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile) {

392 spinlock_lock(&journalfile->data_spinlock);

393

@@ -993,7 +1018,8 @@ void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, st

1018 usec_t started_ut = now_monotonic_usec();

1019

1020 size_t data_size = 0;

996 - struct journal_v2_header *j2_header = journalfile_v2_data_acquire(journalfile, &data_size, 0, 0);

1021 + struct journal_v2_header *j2_header = journalfile_v2_data_acquire_with_hint(

1022 + journalfile, &data_size, 0, 0, JOURNALFILE_V2_ACCESS_SEQUENTIAL_DIRECTORY);

1023 if(!j2_header)

1024 return;

1025

@@ -1110,6 +1136,10 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal

1136 return 1;

1137 }

1138

1139 + // validation reads the file sequentially — hint the kernel to prefetch

1140 + madvise_sequential(data_start, journal_v2_file_size);

1141 + madvise_willneed(data_start, journal_v2_file_size);

1142 +

1143 nd_log_daemon(NDLP_DEBUG, "DBENGINE: checking integrity of \"%s\"", path_v2);

1144

1145 usec_t validation_start_ut = now_monotonic_usec();

src/database/engine/journalfile.h

@@ -23,6 +23,12 @@ typedef enum __attribute__ ((__packed__)) {

23 JOURNALFILE_FLAG_METRIC_CRC_CHECK = (1 << 3),

24 } JOURNALFILE_FLAGS;

26 +typedef enum __attribute__ ((__packed__)) {

27 + JOURNALFILE_V2_ACCESS_AUTO = 0,

28 + JOURNALFILE_V2_ACCESS_RANDOM,

29 + JOURNALFILE_V2_ACCESS_SEQUENTIAL_DIRECTORY,

30 +} JOURNALFILE_V2_ACCESS_HINT;

31 +

32 struct rrdengine_journalfile {

33 SPINLOCK data_spinlock;

34 struct {

@@ -271,6 +277,7 @@ bool journalfile_v2_data_available(struct rrdengine_journalfile *journalfile);

277 size_t journalfile_v2_data_size_get(struct rrdengine_journalfile *journalfile);

278 void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd, void *journal_data, uint32_t journal_data_size);

279 struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s);

280 +struct journal_v2_header *journalfile_v2_data_acquire_with_hint(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s, JOURNALFILE_V2_ACCESS_HINT hint);

281 void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile);

282 void journalfile_v2_data_unmount_cleanup(time_t now_s);

283

src/database/engine/rrdengine.c

+25 -10

@@ -1167,7 +1167,8 @@ static time_t find_uuid_first_time(

1167

1168 bool agent_shutdown = false;

1169 while (datafile) {

1170 - struct journal_v2_header *j2_header = journalfile_v2_data_acquire(datafile->journalfile, NULL, 0, 0);

1170 + struct journal_v2_header *j2_header = journalfile_v2_data_acquire_with_hint(

1171 + datafile->journalfile, NULL, 0, 0, JOURNALFILE_V2_ACCESS_RANDOM);

1172 if (!j2_header) {

1173 datafile = datafile_release_and_acquire_next_for_retention(ctx, datafile);

1174 continue;

@@ -1342,7 +1343,8 @@ static void update_metrics_first_time_s(struct rrdengine_instance *ctx, struct r

1343 worker_is_busy(UV_EVENT_DBENGINE_FIND_ROTATED_METRICS);

1344

1345 struct rrdengine_journalfile *journalfile = datafile_to_delete->journalfile;

1345 - struct journal_v2_header *j2_header = journalfile_v2_data_acquire(journalfile, NULL, 0, 0);

1346 + struct journal_v2_header *j2_header = journalfile_v2_data_acquire_with_hint(

1347 + journalfile, NULL, 0, 0, JOURNALFILE_V2_ACCESS_SEQUENTIAL_DIRECTORY);

1348

1349 if (unlikely(!j2_header)) {

1350 if (worker)

@@ -1692,14 +1694,16 @@ static void *populate_mrg_tp_worker(

1694 }

1695

1696 size_t total = 0;

1697 + Word_t last_index = 0;

1698 + bool resume_scan = false;

1699 do {

1700 struct rrdengine_datafile *datafile = NULL;

1701

1702 // find a datafile to work on

1703 netdata_rwlock_rdlock(&ctx->datafiles.rwlock);

1700 - bool first_then_next = true;

1701 - Pvoid_t *Pvalue = NULL;

1702 - Word_t Index = 0;

1704 + Pvoid_t *Pvalue = NULL;

1705 + Word_t Index = resume_scan ? last_index : 0;

1706 + bool first_then_next = !resume_scan;

1707 while((Pvalue = JudyLFirstThenNext(ctx->datafiles.JudyL, &Index, &first_then_next))) {

1708 datafile = *Pvalue;

1709 if(!spinlock_trylock(&datafile->populate_mrg.spinlock)) {

@@ -1719,6 +1723,10 @@ static void *populate_mrg_tp_worker(

1723 if(!datafile)

1724 break;

1725

1726 + // resume next scan from current position

1727 + last_index = Index;

1728 + resume_scan = true;

1729 +

1730 uv_sem_wait(mlt->sem);

1731 struct mrg_load_thread *local_mlt = callocz(1, sizeof(struct mrg_load_thread));

1732 local_mlt->datafile = datafile;

@@ -1727,18 +1735,25 @@ static void *populate_mrg_tp_worker(

1735 local_mlt->populated_datafiles = &populated_datafiles;

1736 __atomic_add_fetch(local_mlt->total, 1, __ATOMIC_RELAXED);

1737 rrdeng_enq_cmd(ctx, RRDENG_OPCODE_MRG_LOAD, local_mlt, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);

1730 - nd_log_limit_static_thread_var(erl, 10, 0);

1731 - nd_log_limit(&erl, NDLS_DAEMON, NDLP_INFO, "DBENGINE: Tier %d MRG population completed: %.2f%% (%zu/%zu)", tier, (populated_datafiles * 100.0) / total_datafiles,

1732 - populated_datafiles, total_datafiles);

1738 + {

1739 + nd_log_limit_static_thread_var(erl, 10, 0);

1740 + size_t completed = __atomic_load_n(&populated_datafiles, __ATOMIC_RELAXED);

1741 + nd_log_limit(&erl, NDLS_DAEMON, NDLP_INFO,

1742 + "DBENGINE: Tier %d MRG population completed: %.2f%% (%zu/%zu)",

1743 + tier, (completed * 100.0) / total_datafiles, completed, total_datafiles);

1744 + }

1745 } while(1);

1746

1735 - // We've processed all datafiles. Now wait for all our threads to complete

1747 + // We've queued all datafiles. Now wait for all worker threads to complete.

1748 size_t pending;

1749 do {

1750 pending = __atomic_load_n(&total, __ATOMIC_ACQUIRE);

1751 if (pending) {

1752 nd_log_limit_static_thread_var(erl, 10, 0);

1741 - nd_log_limit(&erl, NDLS_DAEMON, NDLP_INFO, "DBENGINE: Tier %d Waiting for %zu threads", tier, total);

1753 + size_t completed = __atomic_load_n(&populated_datafiles, __ATOMIC_RELAXED);

1754 + nd_log_limit(&erl, NDLS_DAEMON, NDLP_INFO,

1755 + "DBENGINE: Tier %d MRG population completed: %.2f%% (%zu/%zu), waiting for %zu workers",

1756 + tier, (completed * 100.0) / total_datafiles, completed, total_datafiles, pending);

1757 sleep_usec(10 * USEC_PER_MS);

1758 }

1759 } while (pending > 0);