@cryptotaxi247 / netdata-1 / commits / 8a4c9df1c

Reduce flush operations during journal build (#17220)

after creating journal file v2, flush open cache only once Co-authored-by: Costa Tsaousis <costa@netdata.cloud>

Stelios Fragkakis committed Mar 21, 2024 at 20:20 UTC 8a4c9df1cc907f3ceb90d1a7824890e657ed8f7c

3 files changed +11 -11

src/database/engine/cache.c

+9 -9

@@ -1882,7 +1882,7 @@ void pgc_page_release(PGC *cache, PGC_PAGE *page) {

1882 page_release(cache, page, is_page_clean(page));

1883 }

1884

1885 -void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page) {

1885 +void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page, bool never_flush) {

1886 __atomic_add_fetch(&cache->stats.workers_hot2dirty, 1, __ATOMIC_RELAXED);

1887

1888 //#ifdef NETDATA_INTERNAL_CHECKS

@@ -1901,10 +1901,8 @@ void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page) {

1901 __atomic_sub_fetch(&cache->stats.workers_hot2dirty, 1, __ATOMIC_RELAXED);

1902

1903 // flush, if we have to

1904 - if((cache->config.options & PGC_OPTIONS_FLUSH_PAGES_INLINE) || flushing_critical(cache)) {

1905 - flush_pages(cache, cache->config.max_flushes_inline, PGC_SECTION_ALL,

1906 - false, false);

1907 - }

1904 + if(!never_flush && ((cache->config.options & PGC_OPTIONS_FLUSH_PAGES_INLINE) || flushing_critical(cache)))

1905 + flush_pages(cache, cache->config.max_flushes_inline, PGC_SECTION_ALL, false, false);

1906 }

1907

1908 bool pgc_page_to_clean_evict_or_release(PGC *cache, PGC_PAGE *page) {

@@ -2224,7 +2222,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_

2222 while ((PValue2 = JudyLFirstThenNext(mi->JudyL_pages_by_start_time, &start_time, &start_time_first))) {

2223 struct jv2_page_info *pi = *PValue2;

2224 page_transition_unlock(cache, pi->page);

2227 - pgc_page_hot_to_dirty_and_release(cache, pi->page);

2225 + pgc_page_hot_to_dirty_and_release(cache, pi->page, true);

2226 // make_acquired_page_clean_and_evict_or_page_release(cache, pi->page);

2227 aral_freez(ar_pi, pi);

2228 }

@@ -2251,6 +2249,8 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_

2249 aral_by_size_release(ar_mi);

2250

2251 __atomic_sub_fetch(&cache->stats.workers_jv2_flush, 1, __ATOMIC_RELAXED);

2252 +

2253 + flush_pages(cache, cache->config.max_flushes_inline, PGC_SECTION_ALL, false, false);

2254 }

2255

2256 static bool match_page_data(PGC_PAGE *page, void *data) {

@@ -2396,7 +2396,7 @@ void *unittest_stress_test_collector(void *ptr) {

2396 if(i % 10 == 0)

2397 pgc_page_to_clean_evict_or_release(pgc_uts.cache, pgc_uts.metrics[i]);

2398 else

2399 - pgc_page_hot_to_dirty_and_release(pgc_uts.cache, pgc_uts.metrics[i]);

2399 + pgc_page_hot_to_dirty_and_release(pgc_uts.cache, pgc_uts.metrics[i], false);

2400 }

2401 }

2402

@@ -2721,7 +2721,7 @@ int pgc_unittest(void) {

2721 }, NULL);

2722

2723 pgc_page_hot_set_end_time_s(cache, page2, 2001);

2724 - pgc_page_hot_to_dirty_and_release(cache, page2);

2724 + pgc_page_hot_to_dirty_and_release(cache, page2, false);

2725

2726 PGC_PAGE *page3 = pgc_page_add_and_acquire(cache, (PGC_ENTRY){

2727 .section = 3,

@@ -2734,7 +2734,7 @@ int pgc_unittest(void) {

2734 }, NULL);

2735

2736 pgc_page_hot_set_end_time_s(cache, page3, 2001);

2737 - pgc_page_hot_to_dirty_and_release(cache, page3);

2737 + pgc_page_hot_to_dirty_and_release(cache, page3, false);

2738

2739 pgc_destroy(cache);

2740

src/database/engine/cache.h

+1 -1

@@ -192,7 +192,7 @@ PGC_PAGE *pgc_page_dup(PGC *cache, PGC_PAGE *page);

192 void pgc_page_release(PGC *cache, PGC_PAGE *page);

193

194 // mark a hot page dirty, and release it

195 -void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page);

195 +void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page, bool never_flush);

196

197 // find a page from the cache

198 typedef enum {

src/database/engine/rrdengineapi.c

+1 -1

@@ -320,7 +320,7 @@ void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *sch) {

320 __atomic_add_fetch(&ctx->atomic.samples, add_samples, __ATOMIC_RELAXED);

321 }

322

323 - pgc_page_hot_to_dirty_and_release(main_cache, handle->pgc_page);

323 + pgc_page_hot_to_dirty_and_release(main_cache, handle->pgc_page, false);

324 }

325

326 mrg_metric_set_hot_latest_time_s(main_mrg, handle->metric, 0);