Drop dirty dbengine pages if disk cannot keep up (#7777)
* Introduce dirty page pressure handling in the dbengine page cache that invalidates pages when the disk cannot keep up with the flushing speed.
Markos Fountoulakis committed
Feb 6, 2020 at 21:58 UTC
6b119d9170fce726e9a5720edc83f6d9ac88e7ce
11 files changed
+375
-116
configs.signatures
+1
-1
@@ -381,7 +381,7 @@ declare -A configs_signatures=(
381
['7deb236ec68a512b9bdd18e6a51d76f7']='python.d/mysql.conf'
382
['7e5fc1644aa7a54f9dbb1bd102521b09']='health.d/memcached.conf'
383
['7f13631183fbdf79c21c8e5a171e9b34']='health.d/zfs.conf'
384
- ['8edc8c73a8f3ca40b32e27fe452c70f3']='health.d/dbengine.conf'
384
+ ['82f1dc0a477a175ae31d7b815411e44e']='health.d/dbengine.conf'
385
['7fb8184d56a27040e73261ed9c6fc76f']='health_alarm_notify.conf'
386
['80266bddd3df374923c750a6de91d120']='health.d/apache.conf'
387
['803a7f9dcb942eeac0fd764b9e3e38ca']='fping.conf'
daemon/global_statistics.c
+64
-22
@@ -544,7 +544,7 @@ void global_statistics_charts(void) {
544
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
545
++hosts_with_dbengine;
546
/* get localhost's DB engine's statistics */
547
- rrdeng_get_35_statistics(host->rrdeng_ctx, local_stats_array);
547
+ rrdeng_get_37_statistics(host->rrdeng_ctx, local_stats_array);
548
for (i = 0 ; i < RRDENG_NR_STATS ; ++i) {
549
/* aggregate statistics across hosts */
550
stats_array[i] += local_stats_array[i];
@@ -558,6 +558,8 @@ void global_statistics_charts(void) {
558
stats_array[30] = local_stats_array[30];
559
stats_array[31] = local_stats_array[31];
560
stats_array[32] = local_stats_array[32];
561
+ stats_array[34] = local_stats_array[34];
562
+ stats_array[36] = local_stats_array[36];
563
564
// ----------------------------------------------------------------
565
@@ -642,7 +644,6 @@ void global_statistics_charts(void) {
644
old_misses = misses;
645
646
if (hits_delta + misses_delta) {
645
- // allow negative savings
647
ratio = (hits_delta * 100 * 1000) / (hits_delta + misses_delta);
648
} else {
649
ratio = 0;
@@ -658,11 +659,10 @@ void global_statistics_charts(void) {
659
static RRDSET *st_pg_cache_pages = NULL;
660
static RRDDIM *rd_descriptors = NULL;
661
static RRDDIM *rd_populated = NULL;
661
- static RRDDIM *rd_committed = NULL;
662
- static RRDDIM *rd_insertions = NULL;
663
- static RRDDIM *rd_deletions = NULL;
662
+ static RRDDIM *rd_dirty = NULL;
663
static RRDDIM *rd_backfills = NULL;
664
static RRDDIM *rd_evictions = NULL;
665
+ static RRDDIM *rd_used_by_collectors = NULL;
666
667
if (unlikely(!st_pg_cache_pages)) {
668
st_pg_cache_pages = rrdset_create_localhost(
@@ -671,7 +671,7 @@ void global_statistics_charts(void) {
671
, NULL
672
, "dbengine"
673
, NULL
674
- , "NetData DB engine page statistics"
674
+ , "NetData dbengine page cache statistics"
675
, "pages"
676
, "netdata"
677
, "stats"
@@ -682,27 +682,68 @@ void global_statistics_charts(void) {
682
683
rd_descriptors = rrddim_add(st_pg_cache_pages, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
684
rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
685
- rd_committed = rrddim_add(st_pg_cache_pages, "committed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
686
- rd_insertions = rrddim_add(st_pg_cache_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
687
- rd_deletions = rrddim_add(st_pg_cache_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
685
+ rd_dirty = rrddim_add(st_pg_cache_pages, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
686
rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
687
rd_evictions = rrddim_add(st_pg_cache_pages, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
688
+ rd_used_by_collectors = rrddim_add(st_pg_cache_pages, "used_by_collectors", NULL, 1, 1,
689
+ RRD_ALGORITHM_ABSOLUTE);
690
}
691
else
692
rrdset_next(st_pg_cache_pages);
693
694
rrddim_set_by_pointer(st_pg_cache_pages, rd_descriptors, (collected_number)stats_array[27]);
695
rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]);
696
- rrddim_set_by_pointer(st_pg_cache_pages, rd_committed, (collected_number)stats_array[4]);
697
- rrddim_set_by_pointer(st_pg_cache_pages, rd_insertions, (collected_number)stats_array[5]);
698
- rrddim_set_by_pointer(st_pg_cache_pages, rd_deletions, (collected_number)stats_array[6]);
696
+ rrddim_set_by_pointer(st_pg_cache_pages, rd_dirty, (collected_number)stats_array[0] + stats_array[4]);
697
rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]);
698
rrddim_set_by_pointer(st_pg_cache_pages, rd_evictions, (collected_number)stats_array[10]);
699
+ rrddim_set_by_pointer(st_pg_cache_pages, rd_used_by_collectors, (collected_number)stats_array[0]);
700
rrdset_done(st_pg_cache_pages);
701
}
702
703
// ----------------------------------------------------------------
704
705
+ {
706
+ static RRDSET *st_long_term_pages = NULL;
707
+ static RRDDIM *rd_total = NULL;
708
+ static RRDDIM *rd_insertions = NULL;
709
+ static RRDDIM *rd_deletions = NULL;
710
+ static RRDDIM *rd_flushing_pressure_deletions = NULL;
711
+
712
+ if (unlikely(!st_long_term_pages)) {
713
+ st_long_term_pages = rrdset_create_localhost(
714
+ "netdata"
715
+ , "dbengine_long_term_page_stats"
716
+ , NULL
717
+ , "dbengine"
718
+ , NULL
719
+ , "NetData dbengine long-term page statistics"
720
+ , "pages"
721
+ , "netdata"
722
+ , "stats"
723
+ , 130505
724
+ , localhost->rrd_update_every
725
+ , RRDSET_TYPE_LINE
726
+ );
727
+
728
+ rd_total = rrddim_add(st_long_term_pages, "total", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
729
+ rd_insertions = rrddim_add(st_long_term_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
730
+ rd_deletions = rrddim_add(st_long_term_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
731
+ rd_flushing_pressure_deletions = rrddim_add(st_long_term_pages, "flushing_pressure_deletions", NULL, -1,
732
+ 1, RRD_ALGORITHM_INCREMENTAL);
733
+ }
734
+ else
735
+ rrdset_next(st_long_term_pages);
736
+
737
+ rrddim_set_by_pointer(st_long_term_pages, rd_total, (collected_number)stats_array[2]);
738
+ rrddim_set_by_pointer(st_long_term_pages, rd_insertions, (collected_number)stats_array[5]);
739
+ rrddim_set_by_pointer(st_long_term_pages, rd_deletions, (collected_number)stats_array[6]);
740
+ rrddim_set_by_pointer(st_long_term_pages, rd_flushing_pressure_deletions,
741
+ (collected_number)stats_array[36]);
742
+ rrdset_done(st_long_term_pages);
743
+ }
744
+
745
+ // ----------------------------------------------------------------
746
+
747
{
748
static RRDSET *st_io_stats = NULL;
749
static RRDDIM *rd_reads = NULL;
@@ -719,7 +760,7 @@ void global_statistics_charts(void) {
760
, "MiB/s"
761
, "netdata"
762
, "stats"
722
- , 130505
763
+ , 130506
764
, localhost->rrd_update_every
765
, RRDSET_TYPE_LINE
766
);
@@ -753,7 +794,7 @@ void global_statistics_charts(void) {
794
, "operations/s"
795
, "netdata"
796
, "stats"
756
- , 130506
797
+ , 130507
798
, localhost->rrd_update_every
799
, RRDSET_TYPE_LINE
800
);
@@ -775,7 +816,7 @@ void global_statistics_charts(void) {
816
static RRDSET *st_errors = NULL;
817
static RRDDIM *rd_fs_errors = NULL;
818
static RRDDIM *rd_io_errors = NULL;
778
- static RRDDIM *rd_flushing_errors = NULL;
819
+ static RRDDIM *pg_cache_over_half_dirty_events = NULL;
820
821
if (unlikely(!st_errors)) {
822
st_errors = rrdset_create_localhost(
@@ -788,21 +829,22 @@ void global_statistics_charts(void) {
829
, "errors/s"
830
, "netdata"
831
, "stats"
791
- , 130507
832
+ , 130508
833
, localhost->rrd_update_every
834
, RRDSET_TYPE_LINE
835
);
836
796
- rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
797
- rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
798
- rd_flushing_errors = rrddim_add(st_errors, "flushing errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
837
+ rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
838
+ rd_fs_errors = rrddim_add(st_errors, "fs_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
839
+ pg_cache_over_half_dirty_events = rrddim_add(st_errors, "pg_cache_over_half_dirty_events", NULL, 1, 1,
840
+ RRD_ALGORITHM_INCREMENTAL);
841
}
842
else
843
rrdset_next(st_errors);
844
845
rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
846
rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
805
- rrddim_set_by_pointer(st_errors, rd_flushing_errors, (collected_number)stats_array[34]);
847
+ rrddim_set_by_pointer(st_errors, pg_cache_over_half_dirty_events, (collected_number)stats_array[34]);
848
rrdset_done(st_errors);
849
}
850
@@ -824,7 +866,7 @@ void global_statistics_charts(void) {
866
, "descriptors"
867
, "netdata"
868
, "stats"
827
- , 130508
869
+ , 130509
870
, localhost->rrd_update_every
871
, RRDSET_TYPE_LINE
872
);
@@ -863,7 +905,7 @@ void global_statistics_charts(void) {
905
, "MiB"
906
, "netdata"
907
, "stats"
866
- , 130509
908
+ , 130510
909
, localhost->rrd_update_every
910
, RRDSET_TYPE_STACKED
911
);
daemon/unit_test.c
+3
@@ -1491,6 +1491,9 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number
1491
1492
static RRDHOST *dbengine_rrdhost_find_or_create(char *name)
1493
{
1494
+ /* We don't want to drop metrics when generating load, we prefer to block data generation itself */
1495
+ rrdeng_drop_metrics_under_page_cache_pressure = 0;
1496
+
1497
return rrdhost_find_or_create(
1498
name
1499
, name
database/engine/pagecache.c
+23
-8
@@ -217,7 +217,6 @@ static void pg_cache_release_pages(struct rrdengine_instance *ctx, unsigned numb
217
218
/*
219
* This function returns the maximum number of pages allowed in the page cache.
220
- * The caller must hold the page cache lock.
220
*/
221
unsigned long pg_cache_hard_limit(struct rrdengine_instance *ctx)
222
{
@@ -228,7 +227,6 @@ unsigned long pg_cache_hard_limit(struct rrdengine_instance *ctx)
227
/*
228
* This function returns the low watermark number of pages in the page cache. The page cache should strive to keep the
229
* number of pages below that number.
231
- * The caller must hold the page cache lock.
230
*/
231
unsigned long pg_cache_soft_limit(struct rrdengine_instance *ctx)
232
{
@@ -236,6 +234,16 @@ unsigned long pg_cache_soft_limit(struct rrdengine_instance *ctx)
234
return ctx->cache_pages_low_watermark + 2 * (unsigned long)ctx->stats.metric_API_producers;
235
}
236
237
+/*
238
+ * This function returns the maximum number of dirty pages that are committed to be written to disk allowed in the page
239
+ * cache.
240
+ */
241
+unsigned long pg_cache_committed_hard_limit(struct rrdengine_instance *ctx)
242
+{
243
+ /* We remove the active pages of the producers from the calculation and only allow 50% of the extra pinned pages */
244
+ return ctx->cache_pages_low_watermark + (unsigned long)ctx->stats.metric_API_producers / 2;
245
+}
246
+
247
/*
248
* This function will block until it reserves #number populated pages.
249
* It will trigger evictions or dirty page flushing if the pg_cache_hard_limit() limit is hit.
@@ -375,7 +383,11 @@ static int pg_cache_try_evict_one_page_unsafe(struct rrdengine_instance *ctx)
383
return 0;
384
}
385
378
-void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty)
386
+/*
387
+ * Callers of this function need to make sure they're not deleting the same descriptor concurrently
388
+ */
389
+void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
390
+ uint8_t is_exclusive_holder)
391
{
392
struct page_cache *pg_cache = &ctx->pg_cache;
393
struct page_cache_descr *pg_cache_descr = NULL;
@@ -408,11 +420,14 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
420
421
rrdeng_page_descr_mutex_lock(ctx, descr);
422
pg_cache_descr = descr->pg_cache_descr;
411
- while (!pg_cache_try_get_unsafe(descr, 1)) {
412
- debug(D_RRDENGINE, "%s: Waiting for locked page:", __func__);
413
- if (unlikely(debug_flags & D_RRDENGINE))
414
- print_page_cache_descr(descr);
415
- pg_cache_wait_event_unsafe(descr);
423
+ if (!is_exclusive_holder) {
424
+ /* If we don't hold an exclusive page reference get one */
425
+ while (!pg_cache_try_get_unsafe(descr, 1)) {
426
+ debug(D_RRDENGINE, "%s: Waiting for locked page:", __func__);
427
+ if (unlikely(debug_flags & D_RRDENGINE))
428
+ print_page_cache_descr(descr);
429
+ pg_cache_wait_event_unsafe(descr);
430
+ }
431
}
432
if (remove_dirty) {
433
pg_cache_descr->flags &= ~RRD_PAGE_DIRTY;
database/engine/pagecache.h
+3
-1
@@ -163,7 +163,8 @@ extern void pg_cache_put_unsafe(struct rrdeng_page_descr *descr);
163
extern void pg_cache_put(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr);
164
extern void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index *index,
165
struct rrdeng_page_descr *descr);
166
-extern void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty);
166
+extern void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
167
+ uint8_t is_exclusive_holder);
168
extern usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
169
usec_t start_time, usec_t end_time);
170
extern void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
@@ -185,6 +186,7 @@ extern void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index,
186
extern void pg_cache_update_metric_times(struct pg_cache_page_index *page_index);
187
extern unsigned long pg_cache_hard_limit(struct rrdengine_instance *ctx);
188
extern unsigned long pg_cache_soft_limit(struct rrdengine_instance *ctx);
189
+extern unsigned long pg_cache_committed_hard_limit(struct rrdengine_instance *ctx);
190
191
static inline void
192
pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_timep, uint32_t *page_lengthp)
database/engine/rrdengine.c
+204
-53
@@ -6,7 +6,8 @@
6
rrdeng_stats_t global_io_errors = 0;
7
rrdeng_stats_t global_fs_errors = 0;
8
rrdeng_stats_t rrdeng_reserved_file_descriptors = 0;
9
-rrdeng_stats_t global_flushing_errors = 0;
9
+rrdeng_stats_t global_pg_cache_over_half_dirty_events = 0;
10
+rrdeng_stats_t global_flushing_pressure_page_deletions = 0;
11
12
static void sanity_check(void)
13
{
@@ -248,6 +249,109 @@ static void do_commit_transaction(struct rrdengine_worker_config* wc, uint8_t ty
249
}
250
}
251
252
+static void after_invalidate_oldest_committed(struct rrdengine_worker_config* wc)
253
+{
254
+ int error;
255
+
256
+ error = uv_thread_join(wc->now_invalidating_dirty_pages);
257
+ if (error) {
258
+ error("uv_thread_join(): %s", uv_strerror(error));
259
+ }
260
+ freez(wc->now_invalidating_dirty_pages);
261
+ wc->now_invalidating_dirty_pages = NULL;
262
+ wc->cleanup_thread_invalidating_dirty_pages = 0;
263
+}
264
+
265
+static void invalidate_oldest_committed(void *arg)
266
+{
267
+ struct rrdengine_instance *ctx = arg;
268
+ struct rrdengine_worker_config *wc = &ctx->worker_config;
269
+ struct page_cache *pg_cache = &ctx->pg_cache;
270
+ int ret;
271
+ struct rrdeng_page_descr *descr;
272
+ struct page_cache_descr *pg_cache_descr;
273
+ Pvoid_t *PValue;
274
+ Word_t Index;
275
+ unsigned nr_committed_pages;
276
+
277
+ do {
278
+ uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
279
+ for (Index = 0,
280
+ PValue = JudyLFirst(pg_cache->committed_page_index.JudyL_array, &Index, PJE0),
281
+ descr = unlikely(NULL == PValue) ? NULL : *PValue;
282
+
283
+ descr != NULL;
284
+
285
+ PValue = JudyLNext(pg_cache->committed_page_index.JudyL_array, &Index, PJE0),
286
+ descr = unlikely(NULL == PValue) ? NULL : *PValue) {
287
+ assert(0 != descr->page_length);
288
+
289
+ rrdeng_page_descr_mutex_lock(ctx, descr);
290
+ pg_cache_descr = descr->pg_cache_descr;
291
+ if (!(pg_cache_descr->flags & RRD_PAGE_WRITE_PENDING) && pg_cache_try_get_unsafe(descr, 1)) {
292
+ rrdeng_page_descr_mutex_unlock(ctx, descr);
293
+
294
+ ret = JudyLDel(&pg_cache->committed_page_index.JudyL_array, Index, PJE0);
295
+ assert(1 == ret);
296
+ break;
297
+ }
298
+ rrdeng_page_descr_mutex_unlock(ctx, descr);
299
+ }
300
+ uv_rwlock_wrunlock(&pg_cache->committed_page_index.lock);
301
+
302
+ if (!descr) {
303
+ info("Failed to invalidate any dirty pages to relieve page cache pressure.");
304
+
305
+ goto out;
306
+ }
307
+ pg_cache_punch_hole(ctx, descr, 1, 1);
308
+
309
+ uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
310
+ nr_committed_pages = --pg_cache->committed_page_index.nr_committed_pages;
311
+ uv_rwlock_wrunlock(&pg_cache->committed_page_index.lock);
312
+ rrd_stat_atomic_add(&ctx->stats.flushing_pressure_page_deletions, 1);
313
+ rrd_stat_atomic_add(&global_flushing_pressure_page_deletions, 1);
314
+
315
+ } while (nr_committed_pages >= pg_cache_committed_hard_limit(ctx));
316
+out:
317
+ wc->cleanup_thread_invalidating_dirty_pages = 1;
318
+ /* wake up event loop */
319
+ assert(0 == uv_async_send(&wc->async));
320
+}
321
+
322
+void rrdeng_invalidate_oldest_committed(struct rrdengine_worker_config* wc)
323
+{
324
+ struct rrdengine_instance *ctx = wc->ctx;
325
+ struct page_cache *pg_cache = &ctx->pg_cache;
326
+ unsigned nr_committed_pages;
327
+ int error;
328
+
329
+ uv_rwlock_rdlock(&pg_cache->committed_page_index.lock);
330
+ nr_committed_pages = pg_cache->committed_page_index.nr_committed_pages;
331
+ uv_rwlock_rdunlock(&pg_cache->committed_page_index.lock);
332
+
333
+ if (nr_committed_pages >= pg_cache_committed_hard_limit(ctx)) {
334
+ /* delete the oldest page in memory */
335
+ if (wc->now_invalidating_dirty_pages) {
336
+ /* already deleting a page */
337
+ return;
338
+ }
339
+ errno = 0;
340
+ error("Failed to flush dirty buffers quickly enough in dbengine instance \"%s\". "
341
+ "Metric data are being deleted, please reduce disk load or use a faster disk.", ctx->dbfiles_path);
342
+
343
+ wc->now_invalidating_dirty_pages = mallocz(sizeof(*wc->now_invalidating_dirty_pages));
344
+ wc->cleanup_thread_invalidating_dirty_pages = 0;
345
+
346
+ error = uv_thread_create(wc->now_invalidating_dirty_pages, invalidate_oldest_committed, ctx);
347
+ if (error) {
348
+ error("uv_thread_create(): %s", uv_strerror(error));
349
+ freez(wc->now_invalidating_dirty_pages);
350
+ wc->now_invalidating_dirty_pages = NULL;
351
+ }
352
+ }
353
+}
354
+
355
void flush_pages_cb(uv_fs_t* req)
356
{
357
struct rrdengine_worker_config* wc = req->loop->data;
@@ -294,6 +398,7 @@ void flush_pages_cb(uv_fs_t* req)
398
uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
399
pg_cache->committed_page_index.nr_committed_pages -= count;
400
uv_rwlock_wrunlock(&pg_cache->committed_page_index.lock);
401
+ wc->inflight_dirty_pages -= count;
402
}
403
404
/*
@@ -366,6 +471,8 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
471
complete(completion);
472
return 0;
473
}
474
+ wc->inflight_dirty_pages += count;
475
+
476
xt_io_descr = mallocz(sizeof(*xt_io_descr));
477
payload_offset = sizeof(*header) + count * sizeof(header->descr[0]);
478
switch (compression_algorithm) {
@@ -466,17 +573,15 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
573
return ALIGN_BYTES_CEILING(size_bytes);
574
}
575
469
-static void after_delete_old_data(uv_work_t *req, int status)
576
+static void after_delete_old_data(struct rrdengine_worker_config* wc)
577
{
471
- struct rrdengine_instance *ctx = req->data;
472
- struct rrdengine_worker_config* wc = &ctx->worker_config;
578
+ struct rrdengine_instance *ctx = wc->ctx;
579
struct rrdengine_datafile *datafile;
580
struct rrdengine_journalfile *journalfile;
581
unsigned deleted_bytes, journalfile_bytes, datafile_bytes;
476
- int ret;
582
+ int ret, error;
583
char path[RRDENG_PATH_MAX];
584
479
- (void)status;
585
datafile = ctx->datafiles.first;
586
journalfile = datafile->journalfile;
587
datafile_bytes = datafile->pos;
@@ -503,15 +608,24 @@ static void after_delete_old_data(uv_work_t *req, int status)
608
ctx->disk_space -= deleted_bytes;
609
info("Reclaimed %u bytes of disk space.", deleted_bytes);
610
611
+ error = uv_thread_join(wc->now_deleting_files);
612
+ if (error) {
613
+ error("uv_thread_join(): %s", uv_strerror(error));
614
+ }
615
+ freez(wc->now_deleting_files);
616
/* unfreeze command processing */
507
- wc->now_deleting.data = NULL;
508
- /* wake up event loop */
509
- assert(0 == uv_async_send(&wc->async));
617
+ wc->now_deleting_files = NULL;
618
+
619
+ wc->cleanup_thread_deleting_files = 0;
620
+
621
+ /* interrupt event loop */
622
+ uv_stop(wc->loop);
623
}
624
512
-static void delete_old_data(uv_work_t *req)
625
+static void delete_old_data(void *arg)
626
{
514
- struct rrdengine_instance *ctx = req->data;
627
+ struct rrdengine_instance *ctx = arg;
628
+ struct rrdengine_worker_config* wc = &ctx->worker_config;
629
struct rrdengine_datafile *datafile;
630
struct extent_info *extent, *next;
631
struct rrdeng_page_descr *descr;
@@ -524,11 +638,14 @@ static void delete_old_data(uv_work_t *req)
638
count = extent->number_of_pages;
639
for (i = 0 ; i < count ; ++i) {
640
descr = extent->pages[i];
527
- pg_cache_punch_hole(ctx, descr, 0);
641
+ pg_cache_punch_hole(ctx, descr, 0, 0);
642
}
643
next = extent->next;
644
freez(extent);
645
}
646
+ wc->cleanup_thread_deleting_files = 1;
647
+ /* wake up event loop */
648
+ assert(0 == uv_async_send(&wc->async));
649
}
650
651
void rrdeng_test_quota(struct rrdengine_worker_config* wc)
@@ -537,7 +654,7 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
654
struct rrdengine_datafile *datafile;
655
unsigned current_size, target_size;
656
uint8_t out_of_space, only_one_datafile;
540
- int ret;
657
+ int ret, error;
658
659
out_of_space = 0;
660
if (unlikely(ctx->disk_space > ctx->max_disk_space)) {
@@ -559,7 +676,7 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
676
}
677
if (unlikely(out_of_space)) {
678
/* delete old data */
562
- if (wc->now_deleting.data) {
679
+ if (wc->now_deleting_files) {
680
/* already deleting data */
681
return;
682
}
@@ -571,8 +688,33 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
688
}
689
info("Deleting data file \"%s/"DATAFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL DATAFILE_EXTENSION"\".",
690
ctx->dbfiles_path, ctx->datafiles.first->tier, ctx->datafiles.first->fileno);
574
- wc->now_deleting.data = ctx;
575
- assert(0 == uv_queue_work(wc->loop, &wc->now_deleting, delete_old_data, after_delete_old_data));
691
+ wc->now_deleting_files = mallocz(sizeof(*wc->now_deleting_files));
692
+ wc->cleanup_thread_deleting_files = 0;
693
+
694
+ error = uv_thread_create(wc->now_deleting_files, delete_old_data, ctx);
695
+ if (error) {
696
+ error("uv_thread_create(): %s", uv_strerror(error));
697
+ freez(wc->now_deleting_files);
698
+ wc->now_deleting_files = NULL;
699
+ }
700
+ }
701
+}
702
+
703
+static inline int rrdeng_threads_alive(struct rrdengine_worker_config* wc)
704
+{
705
+ if (wc->now_invalidating_dirty_pages || wc->now_deleting_files) {
706
+ return 1;
707
+ }
708
+ return 0;
709
+}
710
+
711
+static void rrdeng_cleanup_finished_threads(struct rrdengine_worker_config* wc)
712
+{
713
+ if (unlikely(wc->cleanup_thread_invalidating_dirty_pages)) {
714
+ after_invalidate_oldest_committed(wc);
715
+ }
716
+ if (unlikely(wc->cleanup_thread_deleting_files)) {
717
+ after_delete_old_data(wc);
718
}
719
}
720
@@ -662,34 +804,37 @@ void timer_cb(uv_timer_t* handle)
804
uv_update_time(handle->loop);
805
rrdeng_test_quota(wc);
806
debug(D_RRDENGINE, "%s: timeout reached.", __func__);
665
- if (likely(!wc->now_deleting.data)) {
666
- /* There is free space so we can write to disk */
807
+ if (likely(!wc->now_deleting_files && !wc->now_invalidating_dirty_pages)) {
808
+ /* There is free space so we can write to disk and we are not actively deleting dirty buffers */
809
struct rrdengine_instance *ctx = wc->ctx;
810
struct page_cache *pg_cache = &ctx->pg_cache;
811
unsigned long total_bytes, bytes_written, nr_committed_pages, bytes_to_write = 0, producers, low_watermark,
812
high_watermark;
813
672
- uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
814
+ uv_rwlock_rdlock(&pg_cache->committed_page_index.lock);
815
nr_committed_pages = pg_cache->committed_page_index.nr_committed_pages;
674
- uv_rwlock_wrunlock(&pg_cache->committed_page_index.lock);
816
+ uv_rwlock_rdunlock(&pg_cache->committed_page_index.lock);
817
producers = ctx->stats.metric_API_producers;
818
/* are flushable pages more than 25% of the maximum page cache size */
819
high_watermark = (ctx->max_cache_pages * 25LLU) / 100;
820
low_watermark = (ctx->max_cache_pages * 5LLU) / 100; /* 5%, must be smaller than high_watermark */
821
680
- if (nr_committed_pages > producers &&
681
- /* committed to be written pages are more than the produced number */
682
- nr_committed_pages - producers > high_watermark) {
683
- /* Flushing speed must increase to stop page cache from filling with dirty pages */
684
- bytes_to_write = (nr_committed_pages - producers - low_watermark) * RRDENG_BLOCK_SIZE;
685
- }
686
- bytes_to_write = MAX(DATAFILE_IDEAL_IO_SIZE, bytes_to_write);
822
+ /* Flush more pages only if disk can keep up */
823
+ if (wc->inflight_dirty_pages < high_watermark + producers) {
824
+ if (nr_committed_pages > producers &&
825
+ /* committed to be written pages are more than the produced number */
826
+ nr_committed_pages - producers > high_watermark) {
827
+ /* Flushing speed must increase to stop page cache from filling with dirty pages */
828
+ bytes_to_write = (nr_committed_pages - producers - low_watermark) * RRDENG_BLOCK_SIZE;
829
+ }
830
+ bytes_to_write = MAX(DATAFILE_IDEAL_IO_SIZE, bytes_to_write);
831
688
- debug(D_RRDENGINE, "Flushing pages to disk.");
689
- for (total_bytes = bytes_written = do_flush_pages(wc, 0, NULL) ;
690
- bytes_written && (total_bytes < bytes_to_write) ;
691
- total_bytes += bytes_written) {
692
- bytes_written = do_flush_pages(wc, 0, NULL);
832
+ debug(D_RRDENGINE, "Flushing pages to disk.");
833
+ for (total_bytes = bytes_written = do_flush_pages(wc, 0, NULL);
834
+ bytes_written && (total_bytes < bytes_to_write);
835
+ total_bytes += bytes_written) {
836
+ bytes_written = do_flush_pages(wc, 0, NULL);
837
+ }
838
}
839
}
840
#ifdef NETDATA_INTERNAL_CHECKS
@@ -730,7 +875,12 @@ void rrdeng_worker(void* arg)
875
}
876
wc->async.data = wc;
877
733
- wc->now_deleting.data = NULL;
878
+ wc->now_deleting_files = NULL;
879
+ wc->cleanup_thread_deleting_files = 0;
880
+
881
+ wc->now_invalidating_dirty_pages = NULL;
882
+ wc->cleanup_thread_invalidating_dirty_pages = 0;
883
+ wc->inflight_dirty_pages = 0;
884
885
/* dirty page flushing timer */
886
ret = uv_timer_init(loop, &timer_req);
@@ -746,8 +896,9 @@ void rrdeng_worker(void* arg)
896
897
assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
898
shutdown = 0;
749
- while (shutdown == 0 || uv_loop_alive(loop)) {
899
+ while (likely(shutdown == 0 || rrdeng_threads_alive(wc))) {
900
uv_run(loop, UV_RUN_DEFAULT);
901
+ rrdeng_cleanup_finished_threads(wc);
902
903
/* wait for commands */
904
cmd_batch_size = 0;
@@ -769,14 +920,6 @@ void rrdeng_worker(void* arg)
920
break;
921
case RRDENG_SHUTDOWN:
922
shutdown = 1;
772
- /*
773
- * uv_async_send after uv_close does not seem to crash in linux at the moment,
774
- * it is however undocumented behaviour and we need to be aware if this becomes
775
- * an issue in the future.
776
- */
777
- uv_close((uv_handle_t *)&wc->async, NULL);
778
- assert(0 == uv_timer_stop(&timer_req));
779
- uv_close((uv_handle_t *)&timer_req, NULL);
923
break;
924
case RRDENG_READ_PAGE:
925
do_read_extent(wc, &cmd.read_page.page_cache_descr, 1, 0);
@@ -788,16 +931,16 @@ void rrdeng_worker(void* arg)
931
do_commit_transaction(wc, STORE_DATA, NULL);
932
break;
933
case RRDENG_FLUSH_PAGES: {
791
- unsigned bytes_written;
792
-
793
- /* First I/O should be enough to call completion */
794
- bytes_written = do_flush_pages(wc, 1, cmd.completion);
795
- if (bytes_written) {
796
- while (do_flush_pages(wc, 1, NULL) && likely(!wc->now_deleting.data)) {
797
- ; /* Force flushing of all committed pages if there is free space. */
798
- }
934
+ if (wc->now_invalidating_dirty_pages) {
935
+ /* Do not flush if the disk cannot keep up */
936
+ complete(cmd.completion);
937
+ } else {
938
+ (void)do_flush_pages(wc, 1, cmd.completion);
939
}
940
break;
941
+ case RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE:
942
+ rrdeng_invalidate_oldest_committed(wc);
943
+ break;
944
}
945
default:
946
debug(D_RRDENGINE, "%s: default.", __func__);
@@ -805,11 +948,19 @@ void rrdeng_worker(void* arg)
948
}
949
} while (opcode != RRDENG_NOOP);
950
}
951
+
952
/* cleanup operations of the event loop */
809
- if (unlikely(wc->now_deleting.data)) {
810
- info("Postponing shutting RRD engine event loop down until after datafile deletion is finished.");
811
- }
953
info("Shutting down RRD engine event loop.");
954
+
955
+ /*
956
+ * uv_async_send after uv_close does not seem to crash in linux at the moment,
957
+ * it is however undocumented behaviour and we need to be aware if this becomes
958
+ * an issue in the future.
959
+ */
960
+ uv_close((uv_handle_t *)&wc->async, NULL);
961
+ assert(0 == uv_timer_stop(&timer_req));
962
+ uv_close((uv_handle_t *)&timer_req, NULL);
963
+
964
while (do_flush_pages(wc, 1, NULL)) {
965
; /* Force flushing of all committed pages. */
966
}
database/engine/rrdengine.h
+17
-4
@@ -49,6 +49,7 @@ enum rrdeng_opcode {
49
RRDENG_COMMIT_PAGE,
50
RRDENG_FLUSH_PAGES,
51
RRDENG_SHUTDOWN,
52
+ RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE,
53
54
RRDENG_MAX_OPCODE
55
};
@@ -102,7 +103,16 @@ struct rrdengine_worker_config {
103
uv_thread_t thread;
104
uv_loop_t* loop;
105
uv_async_t async;
105
- uv_work_t now_deleting;
106
+
107
+ /* file deletion thread */
108
+ uv_thread_t *now_deleting_files;
109
+ unsigned long cleanup_thread_deleting_files; /* set to 0 when now_deleting_files is still running */
110
+
111
+ /* dirty page deletion thread */
112
+ uv_thread_t *now_invalidating_dirty_pages;
113
+ /* set to 0 when now_invalidating_dirty_pages is still running */
114
+ unsigned long cleanup_thread_invalidating_dirty_pages;
115
+ unsigned inflight_dirty_pages;
116
117
/* FIFO command queue */
118
uv_mutex_t cmd_mutex;
@@ -145,7 +155,8 @@ struct rrdengine_statistics {
155
rrdeng_stats_t page_cache_descriptors;
156
rrdeng_stats_t io_errors;
157
rrdeng_stats_t fs_errors;
148
- rrdeng_stats_t flushing_errors;
158
+ rrdeng_stats_t pg_cache_over_half_dirty_events;
159
+ rrdeng_stats_t flushing_pressure_page_deletions;
160
};
161
162
/* I/O errors global counter */
@@ -154,13 +165,15 @@ extern rrdeng_stats_t global_io_errors;
165
extern rrdeng_stats_t global_fs_errors;
166
/* number of File-Descriptors that have been reserved by dbengine */
167
extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
157
-/* inability to flush global counter */
158
-extern rrdeng_stats_t global_flushing_errors;
168
+/* inability to flush global counters */
169
+extern rrdeng_stats_t global_pg_cache_over_half_dirty_events;
170
+extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */
171
172
struct rrdengine_instance {
173
struct rrdengine_worker_config worker_config;
174
struct completion rrdengine_completion;
175
struct page_cache pg_cache;
176
+ uint8_t drop_metrics_under_page_cache_pressure; /* boolean */
177
uint8_t global_compress_alg;
178
struct transaction_commit_log commit_log;
179
struct rrdengine_datafile_list datafiles;
database/engine/rrdengineapi.c
+30
-14
@@ -6,6 +6,8 @@ static struct rrdengine_instance default_global_ctx;
6
7
int default_rrdeng_page_cache_mb = 32;
8
int default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
9
+/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
10
+uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
11
12
/*
13
* Gets a handle for storing metrics to the database.
@@ -107,7 +109,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
109
if (unlikely(debug_flags & D_RRDENGINE))
110
print_page_cache_descr(descr);
111
pg_cache_put(ctx, descr);
110
- pg_cache_punch_hole(ctx, descr, 1);
112
+ pg_cache_punch_hole(ctx, descr, 1, 0);
113
handle->prev_descr = NULL;
114
} else {
115
/* added 1 extra reference to keep 2 dirty pages pinned per metric, expected refcnt = 2 */
@@ -629,16 +631,27 @@ void rrdeng_commit_page(struct rrdengine_instance *ctx, struct rrdeng_page_descr
631
nr_committed_pages = ++pg_cache->committed_page_index.nr_committed_pages;
632
uv_rwlock_wrunlock(&pg_cache->committed_page_index.lock);
633
632
- if (nr_committed_pages >= (ctx->max_cache_pages) / 2 + (unsigned long)ctx->stats.metric_API_producers) {
633
- /* 50% of pages have not been committed yet */
634
- if (0 == (unsigned long)ctx->stats.flushing_errors) {
635
- /* only print the first time */
636
- error("Failed to flush dirty buffers quickly enough in dbengine instance \"%s\"."
637
- "Metric data at risk of not being stored in the database, "
638
- "please reduce disk load or use a faster disk.", ctx->dbfiles_path);
634
+ if (nr_committed_pages >= pg_cache_hard_limit(ctx) / 2) {
635
+ /* over 50% of pages have not been committed yet */
636
+
637
+ if (ctx->drop_metrics_under_page_cache_pressure &&
638
+ nr_committed_pages >= pg_cache_committed_hard_limit(ctx)) {
639
+ /* 100% of pages are dirty */
640
+ struct rrdeng_cmd cmd;
641
+
642
+ cmd.opcode = RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE;
643
+ rrdeng_enq_cmd(&ctx->worker_config, &cmd);
644
+ } else {
645
+ if (0 == (unsigned long) ctx->stats.pg_cache_over_half_dirty_events) {
646
+ /* only print the first time */
647
+ errno = 0;
648
+ error("Failed to flush dirty buffers quickly enough in dbengine instance \"%s\". "
649
+ "Metric data at risk of not being stored in the database, "
650
+ "please reduce disk load or use a faster disk.", ctx->dbfiles_path);
651
+ }
652
+ rrd_stat_atomic_add(&ctx->stats.pg_cache_over_half_dirty_events, 1);
653
+ rrd_stat_atomic_add(&global_pg_cache_over_half_dirty_events, 1);
654
}
640
- rrd_stat_atomic_add(&ctx->stats.flushing_errors, 1);
641
- rrd_stat_atomic_add(&global_flushing_errors, 1);
655
}
656
657
pg_cache_put(ctx, descr);
@@ -688,7 +701,7 @@ void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_i
701
* You must not change the indices of the statistics or user code will break.
702
* You must not exceed RRDENG_NR_STATS or it will crash.
703
*/
691
-void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
704
+void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
705
{
706
struct page_cache *pg_cache = &ctx->pg_cache;
707
@@ -725,9 +738,11 @@ void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long
738
array[30] = (uint64_t)global_io_errors;
739
array[31] = (uint64_t)global_fs_errors;
740
array[32] = (uint64_t)rrdeng_reserved_file_descriptors;
728
- array[33] = (uint64_t)ctx->stats.flushing_errors;
729
- array[34] = (uint64_t)global_flushing_errors;
730
- assert(RRDENG_NR_STATS == 35);
741
+ array[33] = (uint64_t)ctx->stats.pg_cache_over_half_dirty_events;
742
+ array[34] = (uint64_t)global_pg_cache_over_half_dirty_events;
743
+ array[35] = (uint64_t)ctx->stats.flushing_pressure_page_deletions;
744
+ array[36] = (uint64_t)global_flushing_pressure_page_deletions;
745
+ assert(RRDENG_NR_STATS == 37);
746
}
747
748
/* Releases reference to page */
@@ -777,6 +792,7 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
792
ctx->max_disk_space = disk_space_mb * 1048576LLU;
793
strncpyz(ctx->dbfiles_path, dbfiles_path, sizeof(ctx->dbfiles_path) - 1);
794
ctx->dbfiles_path[sizeof(ctx->dbfiles_path) - 1] = '\0';
795
+ ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;
796
797
memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
798
ctx->worker_config.ctx = ctx;
database/engine/rrdengineapi.h
+3
-2
@@ -8,12 +8,13 @@
8
#define RRDENG_MIN_PAGE_CACHE_SIZE_MB (8)
9
#define RRDENG_MIN_DISK_SPACE_MB (256)
10
11
-#define RRDENG_NR_STATS (35)
11
+#define RRDENG_NR_STATS (37)
12
13
#define RRDENG_FD_BUDGET_PER_INSTANCE (50)
14
15
extern int default_rrdeng_page_cache_mb;
16
extern int default_rrdeng_disk_quota_mb;
17
+extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;
18
19
struct rrdeng_region_info {
20
time_t start_time;
@@ -41,7 +42,7 @@ extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_han
42
extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
43
extern time_t rrdeng_metric_latest_time(RRDDIM *rd);
44
extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
44
-extern void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
45
+extern void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
46
47
/* must call once before using anything */
48
extern int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
database/engine/rrdenginelib.c
+8
-4
@@ -159,8 +159,10 @@ char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t si
159
"global_io_errors: %ld\n"
160
"global_fs_errors: %ld\n"
161
"rrdeng_reserved_file_descriptors: %ld\n"
162
- "flushing_errors: %ld\n"
163
- "global_flushing_errors: %ld\n",
162
+ "pg_cache_over_half_dirty_events: %ld\n"
163
+ "global_pg_cache_over_half_dirty_events: %ld\n"
164
+ "flushing_pressure_page_deletions: %ld\n"
165
+ "global_flushing_pressure_page_deletions: %ld\n",
166
(long)ctx->stats.metric_API_producers,
167
(long)ctx->stats.metric_API_consumers,
168
(long)pg_cache->page_descriptors,
@@ -194,8 +196,10 @@ char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t si
196
(long)global_io_errors,
197
(long)global_fs_errors,
198
(long)rrdeng_reserved_file_descriptors,
197
- (long)ctx->stats.flushing_errors,
198
- (long)global_flushing_errors
199
+ (long)ctx->stats.pg_cache_over_half_dirty_events,
200
+ (long)global_pg_cache_over_half_dirty_events,
201
+ (long)ctx->stats.flushing_pressure_page_deletions,
202
+ (long)global_flushing_pressure_page_deletions
203
);
204
return str;
205
}
health/health.d/dbengine.conf
+19
-7
@@ -5,7 +5,7 @@
5
on: netdata.dbengine_global_errors
6
os: linux freebsd macos
7
hosts: *
8
-lookup: sum -10m unaligned of FS errors
8
+lookup: sum -10m unaligned of fs_errors
9
units: errors
10
every: 10s
11
crit: $this > 0
@@ -17,7 +17,7 @@ lookup: sum -10m unaligned of FS errors
17
on: netdata.dbengine_global_errors
18
os: linux freebsd macos
19
hosts: *
20
-lookup: sum -10m unaligned of I/O errors
20
+lookup: sum -10m unaligned of io_errors
21
units: errors
22
every: 10s
23
crit: $this > 0
@@ -25,14 +25,26 @@ lookup: sum -10m unaligned of I/O errors
25
info: number of IO errors dbengine came across the last 10 minutes (CRC errors, out of space, bad disk etc)
26
to: sysadmin
27
28
- alarm: 10min_dbengine_global_flushing_errors
28
+ alarm: 10min_dbengine_global_flushing_warnings
29
on: netdata.dbengine_global_errors
30
os: linux freebsd macos
31
hosts: *
32
-lookup: sum -10m unaligned of flushing errors
32
+lookup: sum -10m unaligned of pg_cache_over_half_dirty_events
33
units: errors
34
- every: 3s
35
- crit: $this > 0
34
+ every: 10s
35
+ warn: $this > 0
36
+ delay: down 1h multiplier 1.5 max 3h
37
+ info: number of times in the last 10 minutes that dbengine dirty pages were over 50% of the instance's page cache, metric data at risk of not being stored in the database, please reduce disk load or use faster disks
38
+ to: sysadmin
39
+
40
+ alarm: 10min_dbengine_global_flushing_errors
41
+ on: netdata.dbengine_long_term_page_stats
42
+ os: linux freebsd macos
43
+ hosts: *
44
+lookup: sum -10m unaligned of flushing_pressure_deletions
45
+ units: pages
46
+ every: 10s
47
+ crit: $this != 0
48
delay: down 1h multiplier 1.5 max 3h
37
- info: number of times in the last 10 minutes that the dbengine failed to completely flush data to disk, metric data will not be stored in the database, please reduce disk load or use a faster disk
49
+ info: number of pages deleted due to failure to flush data to disk in the last 10 minutes, metric data were lost to unblock data collection, please reduce disk load or use faster disks
50
to: sysadmin