Implemented default disk space size calculation for multihost db (#9504)
Implemented default disk space size calculation for multihost db (#9504)
Stelios Fragkakis committed
Jul 16, 2020 at 18:58 UTC
36f0ad220ee6fa0ef5968047b4547bb9c31bcca3
6 files changed
+94
-2
daemon/main.c
+7
@@ -557,6 +557,13 @@ static void get_netdata_configured_variables() {
557
error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
558
default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
559
}
560
+
561
+ default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "multidb disk space", compute_multidb_diskspace());
562
+ if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
563
+ error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
564
+ default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
565
+ }
566
+
567
#endif
568
// ------------------------------------------------------------------------
569
database/engine/metadata_log/metadatalog.c
+1
-1
@@ -414,4 +414,4 @@ void error_with_guid(uuid_t *uuid, char *reason)
414
uuid_unparse_lower(*uuid, uuid_str);
415
errno = 0;
416
error("%s (GUID = %s)", reason, uuid_str);
417
-}
\ No newline at end of file
417
+}
database/engine/rrdengineapi.c
+1
@@ -6,6 +6,7 @@ static struct rrdengine_instance default_global_ctx;
6
7
int default_rrdeng_page_cache_mb = 32;
8
int default_rrdeng_disk_quota_mb = 256;
9
+int default_multidb_disk_quota_mb = 256;
10
/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
11
uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
12
database/engine/rrdengineapi.h
+2
-1
@@ -14,6 +14,7 @@
14
15
extern int default_rrdeng_page_cache_mb;
16
extern int default_rrdeng_disk_quota_mb;
17
+extern int default_multidb_disk_quota_mb;
18
extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;
19
20
struct rrdeng_region_info {
@@ -58,4 +59,4 @@ extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *db
59
extern int rrdeng_exit(struct rrdengine_instance *ctx);
60
extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
61
61
-#endif /* NETDATA_RRDENGINEAPI_H */
\ No newline at end of file
62
+#endif /* NETDATA_RRDENGINEAPI_H */
database/engine/rrdenginelib.c
+81
@@ -208,3 +208,84 @@ char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t si
208
);
209
return str;
210
}
211
+
212
+int is_legacy_child(const char *machine_guid)
213
+{
214
+ uuid_t uuid;
215
+ char dbengine_file[FILENAME_MAX+1];
216
+
217
+ if (!uuid_parse(machine_guid, uuid)) {
218
+ uv_fs_t stat_req;
219
+ snprintfz(dbengine_file, FILENAME_MAX, "%s/%s/dbengine", netdata_configured_cache_dir, machine_guid);
220
+ int rc = uv_fs_stat(NULL, &stat_req, dbengine_file, NULL);
221
+ if (likely(rc == 0 && ((stat_req.statbuf.st_mode & S_IFMT) == S_IFDIR))) {
222
+ //info("Found legacy engine folder \"%s\"", dbengine_file);
223
+ return 1;
224
+ }
225
+ }
226
+ return 0;
227
+}
228
+
229
+int count_legacy_children(char *dbfiles_path)
230
+{
231
+ int ret;
232
+ uv_fs_t req;
233
+ uv_dirent_t dent;
234
+ int legacy_engines = 0;
235
+
236
+ ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
237
+ if (ret < 0) {
238
+ uv_fs_req_cleanup(&req);
239
+ error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
240
+ return ret;
241
+ }
242
+
243
+ while(UV_EOF != uv_fs_scandir_next(&req, &dent)) {
244
+ if (dent.type == UV_DIRENT_DIR) {
245
+ if (is_legacy_child(dent.name))
246
+ legacy_engines++;
247
+ }
248
+ }
249
+ uv_fs_req_cleanup(&req);
250
+ return legacy_engines;
251
+}
252
+
253
+int compute_multidb_diskspace()
254
+{
255
+ char multidb_disk_space_file[FILENAME_MAX + 1];
256
+ FILE *fp;
257
+ int computed_multidb_disk_quota_mb = -1;
258
+
259
+ snprintfz(multidb_disk_space_file, FILENAME_MAX, "%s/dbengine_multihost_size", netdata_configured_varlib_dir);
260
+ fp = fopen(multidb_disk_space_file, "r");
261
+ if (likely(fp)) {
262
+ int rc = fscanf(fp, "%d", &computed_multidb_disk_quota_mb);
263
+ fclose(fp);
264
+ if (unlikely(rc != 1 || computed_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB)) {
265
+ errno = 0;
266
+ error("File '%s' contains invalid input, it will be rebuild", multidb_disk_space_file);
267
+ computed_multidb_disk_quota_mb = -1;
268
+ }
269
+ }
270
+
271
+ if (computed_multidb_disk_quota_mb == -1) {
272
+ int rc = count_legacy_children(netdata_configured_cache_dir);
273
+ if (likely(rc >= 0)) {
274
+ computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
275
+ //info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
276
+
277
+ // TODO: will activate the next block of code when multidb is in place
278
+// fp = fopen(multidb_disk_space_file, "w");
279
+// if (likely(fp)) {
280
+// fprintf(fp, "%d", rc * default_rrdeng_disk_quota_mb);
281
+// info("Created file '%s' to store the computed value", multidb_disk_space_file);
282
+// fclose(fp);
283
+// } else
284
+// error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
285
+ }
286
+ else
287
+ computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
288
+ }
289
+
290
+ return computed_multidb_disk_quota_mb;
291
+}
database/engine/rrdenginelib.h
+2
@@ -110,5 +110,7 @@ static inline int open_file_buffered_io(char *path, int flags, uv_file *file)
110
return open_file_for_io(path, flags, file, 0);
111
}
112
extern char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t size);
113
+extern int compute_multidb_diskspace();
114
+extern int is_legacy_child(const char *machine_guid);
115
116
#endif /* NETDATA_RRDENGINELIB_H */
\ No newline at end of file