Detect deadlock in dbengine page cache (#6911)
* Detect deadlock in dbengine page cache when there are too many metrics and print error message * Resolve dbengine deadlock by dropping metrics when page cache is too small and define relevant alarms * Changed printing deadlock errors to only happen once per dbengine instance
Markos Fountoulakis committed
Sep 24, 2019 at 11:59 UTC
2728be8b0614fd58d4b530a32fee164509e88fd8
7 files changed
+97
-32
configs.signatures
+1
-1
@@ -381,7 +381,7 @@ declare -A configs_signatures=(
381
['7deb236ec68a512b9bdd18e6a51d76f7']='python.d/mysql.conf'
382
['7e5fc1644aa7a54f9dbb1bd102521b09']='health.d/memcached.conf'
383
['7f13631183fbdf79c21c8e5a171e9b34']='health.d/zfs.conf'
384
- ['93674f3206872ae9c43ecbc54988413b']='health.d/dbengine.conf'
384
+ ['0fca55fc770c243ebfd8387c89059dd2']='health.d/dbengine.conf'
385
['7fb8184d56a27040e73261ed9c6fc76f']='health_alarm_notify.conf'
386
['80266bddd3df374923c750a6de91d120']='health.d/apache.conf'
387
['803a7f9dcb942eeac0fd764b9e3e38ca']='fping.conf'
daemon/global_statistics.c
+8
-1
@@ -538,7 +538,7 @@ void global_statistics_charts(void) {
538
unsigned long long stats_array[RRDENG_NR_STATS];
539
540
/* get localhost's DB engine's statistics */
541
- rrdeng_get_33_statistics(localhost->rrdeng_ctx, stats_array);
541
+ rrdeng_get_35_statistics(localhost->rrdeng_ctx, stats_array);
542
543
// ----------------------------------------------------------------
544
@@ -756,6 +756,8 @@ void global_statistics_charts(void) {
756
static RRDSET *st_errors = NULL;
757
static RRDDIM *rd_fs_errors = NULL;
758
static RRDDIM *rd_io_errors = NULL;
759
+ static RRDDIM *rd_pg_cache_warnings = NULL;
760
+ static RRDDIM *rd_pg_cache_errors = NULL;
761
762
if (unlikely(!st_errors)) {
763
st_errors = rrdset_create_localhost(
@@ -775,12 +777,17 @@ void global_statistics_charts(void) {
777
778
rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
779
rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
780
+ rd_pg_cache_warnings = rrddim_add(st_errors, "Page-Cache warnings", NULL, 1, 1,
781
+ RRD_ALGORITHM_INCREMENTAL);
782
+ rd_pg_cache_errors = rrddim_add(st_errors, "Page-Cache errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
783
}
784
else
785
rrdset_next(st_errors);
786
787
rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
788
rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
789
+ rrddim_set_by_pointer(st_errors, rd_pg_cache_warnings, (collected_number)stats_array[33]);
790
+ rrddim_set_by_pointer(st_errors, rd_pg_cache_errors, (collected_number)stats_array[34]);
791
rrdset_done(st_errors);
792
}
793
database/engine/rrdengine.c
+2
@@ -5,6 +5,8 @@
5
6
rrdeng_stats_t global_io_errors = 0;
7
rrdeng_stats_t global_fs_errors = 0;
8
+rrdeng_stats_t global_pg_cache_warnings = 0;
9
+rrdeng_stats_t global_pg_cache_errors = 0;
10
rrdeng_stats_t rrdeng_reserved_file_descriptors = 0;
11
12
void sanity_check(void)
database/engine/rrdengine.h
+13
@@ -148,12 +148,25 @@ struct rrdengine_statistics {
148
rrdeng_stats_t page_cache_descriptors;
149
rrdeng_stats_t io_errors;
150
rrdeng_stats_t fs_errors;
151
+ rrdeng_stats_t pg_cache_warnings;
152
+ rrdeng_stats_t pg_cache_errors;
153
};
154
155
/* I/O errors global counter */
156
extern rrdeng_stats_t global_io_errors;
157
/* File-System errors global counter */
158
extern rrdeng_stats_t global_fs_errors;
159
+/*
160
+ * Page cache warnings global counter.
161
+ * Some page cache instance is near critical utilization where metrics will fail to be stored.
162
+ */
163
+extern rrdeng_stats_t global_pg_cache_warnings;
164
+/*
165
+ * Page cache errors global counter.
166
+ * Some page cache instance has hit critical utilization where metrics failed to be stored as a deadlock resolution
167
+ * measure.
168
+ */
169
+extern rrdeng_stats_t global_pg_cache_errors;
170
/* number of File-Descriptors that have been reserved by dbengine */
171
extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
172
database/engine/rrdengineapi.c
+24
-6
@@ -95,9 +95,8 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
95
if (likely(descr->page_length)) {
96
int ret, page_is_empty;
97
98
-#ifdef NETDATA_INTERNAL_CHECKS
98
rrd_stat_atomic_add(&ctx->stats.metric_API_producers, -1);
100
-#endif
99
+
100
if (handle->prev_descr) {
101
/* unpin old second page */
102
pg_cache_put(ctx, handle->prev_descr);
@@ -192,9 +191,26 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
191
if (unlikely(INVALID_TIME == descr->start_time)) {
192
descr->start_time = point_in_time;
193
195
-#ifdef NETDATA_INTERNAL_CHECKS
194
rrd_stat_atomic_add(&ctx->stats.metric_API_producers, 1);
197
-#endif
195
+
196
+ if (unlikely(((unsigned long)ctx->stats.metric_API_producers) >= ctx->max_cache_pages)) {
197
+ if (0 == (unsigned long)ctx->stats.pg_cache_errors) {
198
+ /* only print the first time */
199
+ error("Deadlock detected in dbengine instance \"%s\", metric data will not be stored in the database"
200
+ ", please increase page cache size.", ctx->dbfiles_path);
201
+ }
202
+ rrd_stat_atomic_add(&ctx->stats.pg_cache_errors, 1);
203
+ rrd_stat_atomic_add(&global_pg_cache_errors, 1);
204
+ /* Resolve deadlock */
205
+ descr->page_length = 0; /* make sure the page descriptor is deconstructed */
206
+ rrdeng_store_metric_flush_current_page(rd);
207
+ rrd_stat_atomic_add(&ctx->stats.metric_API_producers, -1);
208
+ return;
209
+ } else if (unlikely(((unsigned long)ctx->stats.metric_API_producers) >= ctx->cache_pages_low_watermark)) {
210
+ rrd_stat_atomic_add(&ctx->stats.pg_cache_warnings, 1);
211
+ rrd_stat_atomic_add(&global_pg_cache_warnings, 1);
212
+ }
213
+
214
pg_cache_insert(ctx, handle->page_index, descr);
215
} else {
216
pg_cache_add_new_metric_time(handle->page_index, descr);
@@ -672,7 +688,7 @@ void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_i
688
* You must not change the indices of the statistics or user code will break.
689
* You must not exceed RRDENG_NR_STATS or it will crash.
690
*/
675
-void rrdeng_get_33_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
691
+void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
692
{
693
struct page_cache *pg_cache = &ctx->pg_cache;
694
@@ -709,7 +725,9 @@ void rrdeng_get_33_statistics(struct rrdengine_instance *ctx, unsigned long long
725
array[30] = (uint64_t)global_io_errors;
726
array[31] = (uint64_t)global_fs_errors;
727
array[32] = (uint64_t)rrdeng_reserved_file_descriptors;
712
- assert(RRDENG_NR_STATS == 33);
728
+ array[33] = (uint64_t)global_pg_cache_warnings;
729
+ array[34] = (uint64_t)global_pg_cache_errors;
730
+ assert(RRDENG_NR_STATS == 35);
731
}
732
733
/* Releases reference to page */
database/engine/rrdengineapi.h
+2
-2
@@ -8,7 +8,7 @@
8
#define RRDENG_MIN_PAGE_CACHE_SIZE_MB (32)
9
#define RRDENG_MIN_DISK_SPACE_MB (256)
10
11
-#define RRDENG_NR_STATS (33)
11
+#define RRDENG_NR_STATS (35)
12
13
#define RRDENG_FD_BUDGET_PER_INSTANCE (50)
14
@@ -41,7 +41,7 @@ extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_han
41
extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
42
extern time_t rrdeng_metric_latest_time(RRDDIM *rd);
43
extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
44
-extern void rrdeng_get_33_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
44
+extern void rrdeng_get_35_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
45
46
/* must call once before using anything */
47
extern int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
health/health.d/dbengine.conf
+47
-22
@@ -1,26 +1,51 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: 10min_dbengine_global_fs_errors
5
- on: netdata.dbengine_global_errors
6
- os: linux freebsd macos
7
- hosts: *
8
- lookup: sum -10m unaligned of FS errors
9
- units: errors
10
- every: 10s
11
- crit: $this > 0
12
- delay: down 15m multiplier 1.5 max 1h
13
- info: number of File-System errors dbengine came across the last 10 minutes (too many open files, wrong permissions etc)
14
- to: sysadmin
4
+ alarm: 10min_dbengine_global_fs_errors
5
+ on: netdata.dbengine_global_errors
6
+ os: linux freebsd macos
7
+ hosts: *
8
+lookup: sum -10m unaligned of FS errors
9
+ units: errors
10
+ every: 10s
11
+ crit: $this > 0
12
+ delay: down 15m multiplier 1.5 max 1h
13
+ info: number of File-System errors dbengine came across the last 10 minutes (too many open files, wrong permissions etc)
14
+ to: sysadmin
15
16
- alarm: 10min_dbengine_global_io_errors
17
- on: netdata.dbengine_global_errors
18
- os: linux freebsd macos
19
- hosts: *
20
- lookup: sum -10m unaligned of I/O errors
21
- units: errors
22
- every: 10s
23
- crit: $this > 0
24
- delay: down 1h multiplier 1.5 max 3h
25
- info: number of IO errors dbengine came across the last 10 minutes (CRC errors, out of space, bad disk etc)
26
- to: sysadmin
16
+ alarm: 10min_dbengine_global_io_errors
17
+ on: netdata.dbengine_global_errors
18
+ os: linux freebsd macos
19
+ hosts: *
20
+lookup: sum -10m unaligned of I/O errors
21
+ units: errors
22
+ every: 10s
23
+ crit: $this > 0
24
+ delay: down 1h multiplier 1.5 max 3h
25
+ info: number of IO errors dbengine came across the last 10 minutes (CRC errors, out of space, bad disk etc)
26
+ to: sysadmin
27
+
28
+ alarm: 10min_dbengine_global_page_cache_errors
29
+ on: netdata.dbengine_global_errors
30
+ os: linux freebsd macos
31
+ hosts: *
32
+ units: errors
33
+ every: 10s
34
+lookup: sum -10m unaligned of Page-Cache errors
35
+ crit: $this > 0
36
+repeat: warning 120s critical 10s
37
+ delay: down 1h multiplier 1.5 max 3h
38
+ info: number of deadlocks dbengine resolved the last 10 minutes due to insufficient page cache size, metrics have been lost
39
+ to: sysadmin
40
+
41
+ alarm: 10min_dbengine_global_page_cache_warnings
42
+ on: netdata.dbengine_global_errors
43
+ os: linux freebsd macos
44
+ hosts: *
45
+ units: errors
46
+ every: 10s
47
+lookup: sum -10m unaligned of Page-Cache warnings
48
+ warn: $this > 0
49
+ delay: down 1h multiplier 1.5 max 3h
50
+ info: number of times dbengine almost deadlocked the last 10 minutes due to insufficient page cache size
51
+ to: sysadmin