Fix flushing errors (#9738)
* Do not allow the pinned pages to exceed the disk space quota to avoid deadlocks. * Add support for timed wait in dbengine. * Disable previous collector page pinning for now as it leads to deadlocks.
Markos Fountoulakis committed
Aug 14, 2020 at 16:09 UTC
87fbd817994eefb40c10084fa3cee757ac1cbb8b
8 files changed
+75
-6
database/engine/README.md
+7
@@ -117,6 +117,13 @@ options.
117
You can use our [database engine calculator](https://learn.netdata.cloud/docs/agent/database/calculator) to
118
validate the memory requirements for your particular system(s) and configuration (**out-of-date**).
119
120
+### Disk space requirements
121
+
122
+There are explicit disk space requirements **per** DB engine **instance**:
123
+
124
+- The total disk space footprint will be the maximum between `#dimensions-being-collected x 4096 x 2` bytes or what
125
+ the user configured with `dbengine multihost disk space` or `dbengine disk space`.
126
+
127
### File descriptor requirements
128
129
The Database Engine may keep a **significant** amount of files open per instance (e.g. per streaming child or
database/engine/metadata_log/metadatalog.c
+8
@@ -415,3 +415,11 @@ void error_with_guid(uuid_t *uuid, char *reason)
415
errno = 0;
416
error("%s (GUID = %s)", reason, uuid_str);
417
}
418
+
419
+void info_with_guid(uuid_t *uuid, char *reason)
420
+{
421
+ char uuid_str[37];
422
+
423
+ uuid_unparse_lower(*uuid, uuid_str);
424
+ info("%s (GUID = %s)", reason, uuid_str);
425
+}
database/engine/metadata_log/metadatalog.h
+1
@@ -135,4 +135,5 @@ extern void metalog_worker(void* arg);
135
extern void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd);
136
extern struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc);
137
extern void error_with_guid(uuid_t *uuid, char *reason);
138
+extern void info_with_guid(uuid_t *uuid, char *reason);
139
#endif /* NETDATA_METADATALOG_H */
database/engine/metadata_log/metadatalogapi.c
+31
-2
@@ -392,8 +392,7 @@ void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metr
392
uint8_t empty_chart;
393
394
rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);
395
- if (!rd) { /* in 8the case of legacy UUID convert to multihost and try again */
396
- // TODO: Check what to do since we have no host
395
+ if (!rd) { /* in the case of legacy UUID convert to multihost and try again */
396
uuid_t multihost_uuid;
397
398
rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->machine_guid, metric_uuid, &multihost_uuid);
@@ -429,6 +428,36 @@ void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metr
428
}
429
}
430
431
+void metalog_print_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
432
+{
433
+ RRDDIM *rd;
434
+ RRDSET *st;
435
+ RRDHOST *host;
436
+
437
+ if (!ctx || !ctx->initialized)
438
+ return;
439
+
440
+ rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);
441
+ if (!rd) { /* in the case of legacy UUID convert to multihost and try again */
442
+ uuid_t multihost_uuid;
443
+
444
+ rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->machine_guid, metric_uuid, &multihost_uuid);
445
+ rd = metalog_get_dimension_from_uuid(ctx, &multihost_uuid);
446
+ }
447
+ if(!rd) {
448
+ error_with_guid(metric_uuid, "GUID not found, unknown metric.");
449
+ return;
450
+ }
451
+ st = rd->rrdset;
452
+ host = st->rrdhost;
453
+
454
+ error_with_guid(metric_uuid, "Host - Chart - Dimension are the below:");
455
+ error("%s %s %s.", host->hostname, st->id, rd->id);
456
+
457
+ if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE))
458
+ error_with_guid(metric_uuid, "UUID does not belong to RRD_MEMORY_MODE_DBENGINE.");
459
+}
460
+
461
/*
462
* Returns 0 on success, negative on error
463
*/
database/engine/metadata_log/metadatalogapi.h
+1
@@ -19,6 +19,7 @@ extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t
19
extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
20
extern RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *uuid);
21
extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
22
+extern void metalog_print_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
23
24
/* must call once before using anything */
25
extern int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx);
database/engine/pagecache.c
+18
@@ -122,6 +122,24 @@ void pg_cache_wait_event_unsafe(struct rrdeng_page_descr *descr)
122
--pg_cache_descr->waiters;
123
}
124
125
+/*
126
+ * The caller must hold page descriptor lock.
127
+ * The lock will be released and re-acquired. The descriptor is not guaranteed
128
+ * to exist after this function returns.
129
+ * Returns UV_ETIMEDOUT if timeout_sec seconds pass.
130
+ */
131
+int pg_cache_timedwait_event_unsafe(struct rrdeng_page_descr *descr, uint64_t timeout_sec)
132
+{
133
+ int ret;
134
+ struct page_cache_descr *pg_cache_descr = descr->pg_cache_descr;
135
+
136
+ ++pg_cache_descr->waiters;
137
+ ret = uv_cond_timedwait(&pg_cache_descr->cond, &pg_cache_descr->mutex, timeout_sec * NSEC_PER_SEC);
138
+ --pg_cache_descr->waiters;
139
+
140
+ return ret;
141
+}
142
+
143
/*
144
* Returns page flags.
145
* The lock will be released and re-acquired. The descriptor is not guaranteed
database/engine/rrdengine.c
+2
-1
@@ -669,7 +669,8 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
669
int ret, error;
670
671
out_of_space = 0;
672
- if (unlikely(ctx->disk_space > ctx->max_disk_space)) {
672
+ /* Do not allow the pinned pages to exceed the disk space quota to avoid deadlocks */
673
+ if (unlikely(ctx->disk_space > MAX(ctx->max_disk_space, 2 * ctx->metric_API_max_producers * RRDENG_BLOCK_SIZE))) {
674
out_of_space = 1;
675
}
676
datafile = ctx->datafiles.last;
database/engine/rrdengineapi.c
+7
-3
@@ -204,14 +204,18 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
204
pg_cache_punch_hole(ctx, descr, 1, 0, NULL);
205
handle->prev_descr = NULL;
206
} else {
207
+ /*
208
+ * Disable pinning for now as it leads to deadlocks. When a collector stops collecting the extra pinned page
209
+ * eventually gets rotated but it cannot be destroyed due to the extra reference.
210
+ */
211
/* added 1 extra reference to keep 2 dirty pages pinned per metric, expected refcnt = 2 */
208
- rrdeng_page_descr_mutex_lock(ctx, descr);
212
+/* rrdeng_page_descr_mutex_lock(ctx, descr);
213
ret = pg_cache_try_get_unsafe(descr, 0);
214
rrdeng_page_descr_mutex_unlock(ctx, descr);
211
- fatal_assert(1 == ret);
215
+ fatal_assert(1 == ret);*/
216
217
rrdeng_commit_page(ctx, descr, handle->page_correlation_id);
214
- handle->prev_descr = descr;
218
+ /* handle->prev_descr = descr;*/
219
}
220
} else {
221
freez(descr->pg_cache_descr->page);