Remove hard cap from page cache size to eliminate deadlocks. (#7006)
* Remove page cache error detection and deadlock resolution * Change page cache logic to disallow deadlocks due to too many API users * Updated documentation * Changed default and minimum page cache size values to 32 and 8 MiB respectively
Markos Fountoulakis committed
Oct 7, 2019 at 11:08 UTC
852c97d412080205a7880d2cda2561c8fa40b203
8 files changed
+42
-84
configs.signatures
+1
-1
@@ -381,7 +381,7 @@ declare -A configs_signatures=(
381
['7deb236ec68a512b9bdd18e6a51d76f7']='python.d/mysql.conf'
382
['7e5fc1644aa7a54f9dbb1bd102521b09']='health.d/memcached.conf'
383
['7f13631183fbdf79c21c8e5a171e9b34']='health.d/zfs.conf'
384
- ['c87415051e896b46faba1d16d955102d']='health.d/dbengine.conf'
384
+ ['e48b89d4a97b96acf9a88970ab858c3b']='health.d/dbengine.conf'
385
['7fb8184d56a27040e73261ed9c6fc76f']='health_alarm_notify.conf'
386
['80266bddd3df374923c750a6de91d120']='health.d/apache.conf'
387
['803a7f9dcb942eeac0fd764b9e3e38ca']='fping.conf'
daemon/global_statistics.c
+1
-8
@@ -538,7 +538,7 @@ void global_statistics_charts(void) {
538
unsigned long long stats_array[RRDENG_NR_STATS];
539
540
/* get localhost's DB engine's statistics */
541
- rrdeng_get_35_statistics(localhost->rrdeng_ctx, stats_array);
541
+ rrdeng_get_33_statistics(localhost->rrdeng_ctx, stats_array);
542
543
// ----------------------------------------------------------------
544
@@ -756,8 +756,6 @@ void global_statistics_charts(void) {
756
static RRDSET *st_errors = NULL;
757
static RRDDIM *rd_fs_errors = NULL;
758
static RRDDIM *rd_io_errors = NULL;
759
- static RRDDIM *rd_pg_cache_warnings = NULL;
760
- static RRDDIM *rd_pg_cache_errors = NULL;
759
760
if (unlikely(!st_errors)) {
761
st_errors = rrdset_create_localhost(
@@ -777,17 +775,12 @@ void global_statistics_charts(void) {
775
776
rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
777
rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
780
- rd_pg_cache_warnings = rrddim_add(st_errors, "Page-Cache warnings", NULL, 1, 1,
781
- RRD_ALGORITHM_INCREMENTAL);
782
- rd_pg_cache_errors = rrddim_add(st_errors, "Page-Cache errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
778
}
779
else
780
rrdset_next(st_errors);
781
782
rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
783
rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
789
- rrddim_set_by_pointer(st_errors, rd_pg_cache_warnings, (collected_number)stats_array[33]);
790
- rrddim_set_by_pointer(st_errors, rd_pg_cache_errors, (collected_number)stats_array[34]);
784
rrdset_done(st_errors);
785
}
786
database/engine/README.md
+4
-2
@@ -57,7 +57,8 @@ The above values are the default and minimum values for Page Cache size and DB e
57
in **MiB**. All DB engine instances will allocate the configured resources separately.
58
59
The `page cache size` option determines the amount of RAM in **MiB** that is dedicated to caching Netdata metric values
60
-themselves.
60
+themselves as far as queries are concerned. The total page cache size will be greater since data collection itself will
61
+consume additional memory as is described in the [Memory requirements](#memory-requirements) section.
62
63
The `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing Netdata
64
metric values and all related metadata describing them.
@@ -88,7 +89,8 @@ available memory.
89
There are explicit memory requirements **per** DB engine **instance**, meaning **per** Netdata **node** (e.g. localhost
90
and streaming recipient nodes):
91
91
-- `page cache size` must be at least `#dimensions-being-collected x 4096 x 2` bytes.
92
+- The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what
93
+ the user configured with `page cache size`.
94
95
- an additional `#pages-on-disk x 4096 x 0.03` bytes of RAM are allocated for metadata.
96
database/engine/pagecache.c
+29
-7
@@ -209,9 +209,31 @@ static void pg_cache_release_pages(struct rrdengine_instance *ctx, unsigned numb
209
pg_cache_release_pages_unsafe(ctx, number);
210
uv_rwlock_wrunlock(&pg_cache->pg_cache_rwlock);
211
}
212
+
213
+/*
214
+ * This function returns the maximum number of pages allowed in the page cache.
215
+ * The caller must hold the page cache lock.
216
+ */
217
+static inline unsigned long pg_cache_hard_limit(struct rrdengine_instance *ctx)
218
+{
219
+ /* it's twice the number of producers since we pin 2 pages per producer */
220
+ return ctx->max_cache_pages + 2 * (unsigned long)ctx->stats.metric_API_producers;
221
+}
222
+
223
+/*
224
+ * This function returns the low watermark number of pages in the page cache. The page cache should strive to keep the
225
+ * number of pages below that number.
226
+ * The caller must hold the page cache lock.
227
+ */
228
+static inline unsigned long pg_cache_soft_limit(struct rrdengine_instance *ctx)
229
+{
230
+ /* it's twice the number of producers since we pin 2 pages per producer */
231
+ return ctx->cache_pages_low_watermark + 2 * (unsigned long)ctx->stats.metric_API_producers;
232
+}
233
+
234
/*
235
* This function will block until it reserves #number populated pages.
214
- * It will trigger evictions or dirty page flushing if the ctx->max_cache_pages limit is hit.
236
+ * It will trigger evictions or dirty page flushing if the pg_cache_hard_limit() limit is hit.
237
*/
238
static void pg_cache_reserve_pages(struct rrdengine_instance *ctx, unsigned number)
239
{
@@ -223,10 +245,10 @@ static void pg_cache_reserve_pages(struct rrdengine_instance *ctx, unsigned numb
245
assert(number < ctx->max_cache_pages);
246
247
uv_rwlock_wrlock(&pg_cache->pg_cache_rwlock);
226
- if (pg_cache->populated_pages + number >= ctx->max_cache_pages + 1)
248
+ if (pg_cache->populated_pages + number >= pg_cache_hard_limit(ctx) + 1)
249
debug(D_RRDENGINE, "==Page cache full. Reserving %u pages.==",
250
number);
229
- while (pg_cache->populated_pages + number >= ctx->max_cache_pages + 1) {
251
+ while (pg_cache->populated_pages + number >= pg_cache_hard_limit(ctx) + 1) {
252
253
if (!pg_cache_try_evict_one_page_unsafe(ctx)) {
254
/* failed to evict */
@@ -260,7 +282,7 @@ static void pg_cache_reserve_pages(struct rrdengine_instance *ctx, unsigned numb
282
283
/*
284
* This function will attempt to reserve #number populated pages.
263
- * It may trigger evictions if the ctx->cache_pages_low_watermark limit is hit.
285
+ * It may trigger evictions if the pg_cache_soft_limit() limit is hit.
286
* Returns 0 on failure and 1 on success.
287
*/
288
static int pg_cache_try_reserve_pages(struct rrdengine_instance *ctx, unsigned number)
@@ -272,7 +294,7 @@ static int pg_cache_try_reserve_pages(struct rrdengine_instance *ctx, unsigned n
294
assert(number < ctx->max_cache_pages);
295
296
uv_rwlock_wrlock(&pg_cache->pg_cache_rwlock);
275
- if (pg_cache->populated_pages + number >= ctx->cache_pages_low_watermark + 1) {
297
+ if (pg_cache->populated_pages + number >= pg_cache_soft_limit(ctx) + 1) {
298
debug(D_RRDENGINE,
299
"==Page cache full. Trying to reserve %u pages.==",
300
number);
@@ -280,11 +302,11 @@ static int pg_cache_try_reserve_pages(struct rrdengine_instance *ctx, unsigned n
302
if (!pg_cache_try_evict_one_page_unsafe(ctx))
303
break;
304
++count;
283
- } while (pg_cache->populated_pages + number >= ctx->cache_pages_low_watermark + 1);
305
+ } while (pg_cache->populated_pages + number >= pg_cache_soft_limit(ctx) + 1);
306
debug(D_RRDENGINE, "Evicted %u pages.", count);
307
}
308
287
- if (pg_cache->populated_pages + number < ctx->max_cache_pages + 1) {
309
+ if (pg_cache->populated_pages + number < pg_cache_hard_limit(ctx) + 1) {
310
pg_cache->populated_pages += number;
311
ret = 1; /* success */
312
}
database/engine/rrdengine.h
-13
@@ -148,25 +148,12 @@ struct rrdengine_statistics {
148
rrdeng_stats_t page_cache_descriptors;
149
rrdeng_stats_t io_errors;
150
rrdeng_stats_t fs_errors;
151
- rrdeng_stats_t pg_cache_warnings;
152
- rrdeng_stats_t pg_cache_errors;
151
};
152
153
/* I/O errors global counter */
154
extern rrdeng_stats_t global_io_errors;
155
/* File-System errors global counter */
156
extern rrdeng_stats_t global_fs_errors;
159
-/*
160
- * Page cache warnings global counter.
161
- * Some page cache instance is near critical utilization where metrics will fail to be stored.
162
- */
163
-extern rrdeng_stats_t global_pg_cache_warnings;
164
-/*
165
- * Page cache errors global counter.
166
- * Some page cache instance has hit critical utilization where metrics failed to be stored as a deadlock resolution
167
- * measure.
168
- */
169
-extern rrdeng_stats_t global_pg_cache_errors;
157
/* number of File-Descriptors that have been reserved by dbengine */
158
extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
159
database/engine/rrdengineapi.c
+3
-24
@@ -4,7 +4,7 @@
4
/* Default global database instance */
5
static struct rrdengine_instance default_global_ctx;
6
7
-int default_rrdeng_page_cache_mb = 128;
7
+int default_rrdeng_page_cache_mb = 32;
8
int default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
9
10
/*
@@ -192,25 +192,6 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
192
descr->start_time = point_in_time;
193
194
rrd_stat_atomic_add(&ctx->stats.metric_API_producers, 1);
195
-
196
- if (unlikely(((unsigned long)ctx->stats.metric_API_producers) >= ctx->max_cache_pages)) {
197
- if (0 == (unsigned long)ctx->stats.pg_cache_errors) {
198
- /* only print the first time */
199
- error("Deadlock detected in dbengine instance \"%s\", metric data will not be stored in the database"
200
- ", please increase page cache size.", ctx->dbfiles_path);
201
- }
202
- rrd_stat_atomic_add(&ctx->stats.pg_cache_errors, 1);
203
- rrd_stat_atomic_add(&global_pg_cache_errors, 1);
204
- /* Resolve deadlock */
205
- descr->page_length = 0; /* make sure the page descriptor is deconstructed */
206
- rrdeng_store_metric_flush_current_page(rd);
207
- rrd_stat_atomic_add(&ctx->stats.metric_API_producers, -1);
208
- return;
209
- } else if (unlikely(((unsigned long)ctx->stats.metric_API_producers) >= ctx->cache_pages_low_watermark)) {
210
- rrd_stat_atomic_add(&ctx->stats.pg_cache_warnings, 1);
211
- rrd_stat_atomic_add(&global_pg_cache_warnings, 1);
212
- }
213
-
195
pg_cache_insert(ctx, handle->page_index, descr);
196
} else {
197
pg_cache_add_new_metric_time(handle->page_index, descr);
@@ -692,7 +673,7 @@ void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_i
673
* You must not change the indices of the statistics or user code will break.
674
* You must not exceed RRDENG_NR_STATS or it will crash.
675
*/
695
-void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
676
+void rrdeng_get_33_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
677
{
678
struct page_cache *pg_cache = &ctx->pg_cache;
679
@@ -729,9 +710,7 @@ void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long
710
array[30] = (uint64_t)global_io_errors;
711
array[31] = (uint64_t)global_fs_errors;
712
array[32] = (uint64_t)rrdeng_reserved_file_descriptors;
732
- array[33] = (uint64_t)global_pg_cache_warnings;
733
- array[34] = (uint64_t)global_pg_cache_errors;
734
- assert(RRDENG_NR_STATS == 35);
713
+ assert(RRDENG_NR_STATS == 33);
714
}
715
716
/* Releases reference to page */
database/engine/rrdengineapi.h
+3
-3
@@ -5,10 +5,10 @@
5
6
#include "rrdengine.h"
7
8
-#define RRDENG_MIN_PAGE_CACHE_SIZE_MB (32)
8
+#define RRDENG_MIN_PAGE_CACHE_SIZE_MB (8)
9
#define RRDENG_MIN_DISK_SPACE_MB (256)
10
11
-#define RRDENG_NR_STATS (35)
11
+#define RRDENG_NR_STATS (33)
12
13
#define RRDENG_FD_BUDGET_PER_INSTANCE (50)
14
@@ -41,7 +41,7 @@ extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_han
41
extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
42
extern time_t rrdeng_metric_latest_time(RRDDIM *rd);
43
extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
44
-extern void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
44
+extern void rrdeng_get_33_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
45
46
/* must call once before using anything */
47
extern int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
health/health.d/dbengine.conf
+1
-26
@@ -23,29 +23,4 @@ lookup: sum -10m unaligned of I/O errors
23
crit: $this > 0
24
delay: down 1h multiplier 1.5 max 3h
25
info: number of IO errors dbengine came across the last 10 minutes (CRC errors, out of space, bad disk etc)
26
- to: sysadmin
27
-
28
- alarm: 10min_dbengine_global_page_cache_errors
29
- on: netdata.dbengine_global_errors
30
- os: linux freebsd macos
31
- hosts: *
32
- units: errors
33
- every: 10s
34
-lookup: sum -10m unaligned of Page-Cache errors
35
- crit: $this > 0
36
-repeat: warning 1h critical 1h
37
- delay: down 1h multiplier 1.5 max 3h
38
- info: number of deadlocks dbengine resolved the last 10 minutes due to insufficient page cache size, metrics have been lost
39
- to: sysadmin
40
-
41
- alarm: 10min_dbengine_global_page_cache_warnings
42
- on: netdata.dbengine_global_errors
43
- os: linux freebsd macos
44
- hosts: *
45
- units: errors
46
- every: 10s
47
-lookup: sum -10m unaligned of Page-Cache warnings
48
- warn: $this > 0
49
- delay: down 1h multiplier 1.5 max 3h
50
- info: number of times dbengine almost deadlocked the last 10 minutes due to insufficient page cache size
51
- to: sysadmin
26
+ to: sysadmin
\ No newline at end of file