@cryptotaxi247 / netdata-1 / commits / 08cab7222

Add a thread to asynchronously process metadata updates (#13783)

* Remove old metalog text fle processing * Add metadata event loop * Move functions from sqlite_functions.c to sqlite_metadata.c Queue updates to the metadata event loop Migration to remove unused tables Cleanup unused functions * Queue chart labels to metadata * Store chart labels to metadata * During shutdown, run full speed * Add shutdown prepare Handle SHUTDOWN in the cmd queue function Add worker thread to handle host/chart/dimension metadata doing dictionary traversals * Remove unused RRDIM_FLAG_ACLK Add flags to trigger host/chart/dimension metadata processing * Incremental processing of chart metadata writes * Store host labels * Remove redundant return statements * Change unit tests / cleanup * Fix rescheduling * Schedule chart labels update by setting the RRDSET_FLAG_METADATA_UPDATE flag * Queue commands to update metadata for dimension and host labels * Make sure we do a final scan to store metadata during shutdown (if needed) * Remove unused structures Adjust queue size since we do batch processing of updates without queueing individual messages Remove pragma mmap for now Fix memory leak during sqlite unittest (minor) * Dont update if we are in archive mode * Cleanup * Build entire message payload and store * Initialize worker completion properly * Properly skip host check for pending metadata updates * Report bind param failures Add worker request inside the data payload Initialize variables to silence warnings Rebase on master * Report the chart id (not the dimension) and the dimension id when storing a dimension * Compilation warnings in 32bit * Add DEFINE for the queries * Remove commented out code * * Remove items parameter from unitest * Remove commented out code * sqlite_metadata.h contains only public items * Use sleep_usec instead of usleep * Rename metadata_database_init_cmd_queue to metadata_init_cmd_queue * Rename metadata_database_enq_cmd_noblock to metadata_enq_cmd_noblock

Stelios Fragkakis committed Oct 16, 2022 at 23:15 UTC 08cab7222478d06481c070b7f5027ead1c5d2b01
38 files changed +1793 -1912
CMakeLists.txt
+2 -10
@@ -748,6 +748,8 @@ set(RRD_PLUGIN_FILES
748 database/storage_engine.h
749 database/ram/rrddim_mem.c
750 database/ram/rrddim_mem.h
751 + database/sqlite/sqlite_metadata.c
752 + database/sqlite/sqlite_metadata.h
753 database/sqlite/sqlite_functions.c
754 database/sqlite/sqlite_functions.h
755 database/sqlite/sqlite_context.c
@@ -779,16 +781,6 @@ set(RRD_PLUGIN_FILES
781 database/engine/pagecache.h
782 database/engine/rrdenglocking.c
783 database/engine/rrdenglocking.h
782 - database/engine/metadata_log/metadatalog.h
783 - database/engine/metadata_log/metadatalogapi.c
784 - database/engine/metadata_log/metadatalogapi.h
785 - database/engine/metadata_log/logfile.h
786 - database/engine/metadata_log/logfile.c
787 - database/engine/metadata_log/metadatalogprotocol.h
788 - database/engine/metadata_log/metalogpluginsd.c
789 - database/engine/metadata_log/metalogpluginsd.h
790 - database/engine/metadata_log/compaction.c
791 - database/engine/metadata_log/compaction.h
784 database/KolmogorovSmirnovDist.c
785 database/KolmogorovSmirnovDist.h
786 )
Makefile.am
+3 -11
@@ -456,7 +456,9 @@ RRD_PLUGIN_FILES = \
456 database/sqlite/sqlite_db_migration.c \
457 database/sqlite/sqlite_db_migration.h \
458 database/sqlite/sqlite_aclk.c \
459 - database/sqlite/sqlite_aclk.h \
459 + database/sqlite/sqlite_aclk.h \
460 + database/sqlite/sqlite_metadata.c \
461 + database/sqlite/sqlite_metadata.h \
462 database/sqlite/sqlite_health.c \
463 database/sqlite/sqlite_health.h \
464 database/sqlite/sqlite_aclk_node.c \
@@ -547,16 +549,6 @@ if ENABLE_DBENGINE
549 database/engine/pagecache.h \
550 database/engine/rrdenglocking.c \
551 database/engine/rrdenglocking.h \
550 - database/engine/metadata_log/metadatalog.h \
551 - database/engine/metadata_log/metadatalogapi.c \
552 - database/engine/metadata_log/metadatalogapi.h \
553 - database/engine/metadata_log/logfile.h \
554 - database/engine/metadata_log/logfile.c \
555 - database/engine/metadata_log/metadatalogprotocol.h \
556 - database/engine/metadata_log/metalogpluginsd.c \
557 - database/engine/metadata_log/metalogpluginsd.h \
558 - database/engine/metadata_log/compaction.c \
559 - database/engine/metadata_log/compaction.h \
552 $(NULL)
553 endif
554
claim/claim.c
+1 -1
@@ -170,7 +170,7 @@ void load_claiming_state(void)
170 }
171
172 invalidate_node_instances(&localhost->host_uuid, claimed_id ? &uuid : NULL);
173 - store_claim_id(&localhost->host_uuid, claimed_id ? &uuid : NULL);
173 + metaqueue_store_claim_id(&localhost->host_uuid, claimed_id ? &uuid : NULL);
174
175 rrdhost_aclk_state_unlock(localhost);
176 if (!claimed_id) {
collectors/plugins.d/pluginsd_parser.c
+11 -5
@@ -80,7 +80,6 @@ PARSER_RC pluginsd_begin(char **words, void *user, PLUGINSD_ACTION *plugins_act
80 } else
81 rrdset_next(st);
82 }
83 -
83 return PARSER_RC_OK;
84 disable:
85 ((PARSER_USER_OBJECT *)user)->enabled = 0;
@@ -306,14 +305,14 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins
305 if (likely(unhide_dimension)) {
306 rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN);
307 if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
309 - (void)sql_set_dimension_option(&rd->metric_uuid, NULL);
308 rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
309 + metaqueue_dimension_update_flags(rd);
310 }
311 } else {
312 rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
313 if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
315 - (void)sql_set_dimension_option(&rd->metric_uuid, "hidden");
314 rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
315 + metaqueue_dimension_update_flags(rd);
316 }
317 }
318
@@ -639,7 +638,6 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
638 error("cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", name, rrdhost_hostname(host), rrdset_id(st));
639 }
640
642 -
641 return PARSER_RC_OK;
642 }
643
@@ -713,7 +711,7 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins
711 host->rrdlabels = rrdlabels_create();
712
713 rrdlabels_migrate_to_these(host->rrdlabels, (DICTIONARY *) (((PARSER_USER_OBJECT *)user)->new_host_labels));
716 - sql_store_host_labels(host);
714 + metaqueue_store_host_labels(host->machine_guid);
715
716 rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_host_labels);
717 ((PARSER_USER_OBJECT *)user)->new_host_labels = NULL;
@@ -743,6 +741,11 @@ PARSER_RC pluginsd_clabel_commit(char **words, void *user, PLUGINSD_ACTION *plu
741 UNUSED(words);
742
743 RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
744 + RRDSET *st = ((PARSER_USER_OBJECT *)user)->st;
745 +
746 + if (unlikely(!st))
747 + return PARSER_RC_OK;
748 +
749 debug(D_PLUGINSD, "requested to commit chart labels");
750
751 if(!((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily) {
@@ -752,6 +755,9 @@ PARSER_RC pluginsd_clabel_commit(char **words, void *user, PLUGINSD_ACTION *plu
755
756 rrdlabels_remove_all_unmarked(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily);
757
758 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
759 + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
760 +
761 ((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily = NULL;
762 return PARSER_RC_OK;
763 }
configure.ac
-1
@@ -1713,7 +1713,6 @@ AC_CONFIG_FILES([
1713 daemon/Makefile
1714 database/Makefile
1715 database/engine/Makefile
1716 - database/engine/metadata_log/Makefile
1716 database/ram/Makefile
1717 database/sqlite/Makefile
1718 diagrams/Makefile
daemon/global_statistics.c
+1
@@ -1832,6 +1832,7 @@ static struct worker_utilization all_workers_utilization[] = {
1832 { .name = "WEB", .family = "workers web server", .priority = 1000000 },
1833 { .name = "ACLKQUERY", .family = "workers aclk query", .priority = 1000000 },
1834 { .name = "ACLKSYNC", .family = "workers aclk host sync", .priority = 1000000 },
1835 + { .name = "METASYNC", .family = "workers metadata sync", .priority = 1000000 },
1836 { .name = "PLUGINSD", .family = "workers plugins.d", .priority = 1000000 },
1837 { .name = "STATSD", .family = "workers plugin statsd", .priority = 1000000 },
1838 { .name = "STATSDFLUSH", .family = "workers plugin statsd flush", .priority = 1000000 },
daemon/main.c
+5
@@ -60,7 +60,9 @@ void netdata_cleanup_and_exit(int ret) {
60 rrdeng_prepare_exit(multidb_ctx[tier]);
61 }
62 #endif
63 + metadata_sync_shutdown_prepare();
64 rrdhost_free_all();
65 + metadata_sync_shutdown();
66 #ifdef ENABLE_DBENGINE
67 if(dbengine_enabled) {
68 for (int tier = 0; tier < storage_tiers; tier++)
@@ -1039,6 +1041,9 @@ int main(int argc, char **argv) {
1041 else if(strcmp(optarg, "rrdlabelstest") == 0) {
1042 return rrdlabels_unittest();
1043 }
1044 + else if(strcmp(optarg, "metatest") == 0) {
1045 + return metadata_unittest();
1046 + }
1047 else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
1048 optarg += strlen(createdataset_string);
1049 unsigned history_seconds = strtoul(optarg, NULL, 0);
daemon/service.c
+1 -1
@@ -60,7 +60,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
60
61 if (tiers_available == tiers_said_yes && tiers_said_yes) {
62 /* This metric has no data and no references */
63 - delete_dimension_uuid(&rd->metric_uuid);
63 + metaqueue_delete_dimension_uuid(&rd->metric_uuid);
64 }
65 else {
66 /* Do not delete this dimension */
daemon/unit_test.c
+2
@@ -1564,8 +1564,10 @@ int test_sqlite(void) {
1564
1565 buffer_free(sql);
1566 fprintf(stderr,"SQLite is OK\n");
1567 + rc = sqlite3_close_v2(db_meta);
1568 return 0;
1569 error:
1570 + rc = sqlite3_close_v2(db_meta);
1571 fprintf(stderr,"SQLite statement failed: %s\n", buffer_tostring(sql));
1572 buffer_free(sql);
1573 fprintf(stderr,"SQLite tests failed\n");
database/engine/Makefile.am
-1
@@ -4,7 +4,6 @@ AUTOMAKE_OPTIONS = subdir-objects
4 MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
6 SUBDIRS = \
7 - metadata_log \
7 $(NULL)
8
9 dist_noinst_DATA = \
database/engine/metadata_log/Makefile.am deleted
-8
@@ -1,8 +0,0 @@
1 -# SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -AUTOMAKE_OPTIONS = subdir-objects
4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 -
6 -dist_noinst_DATA = \
7 - README.md \
8 - $(NULL)
database/engine/metadata_log/compaction.c deleted
-86
@@ -1,86 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
3 -
4 -#include "metadatalog.h"
5 -
6 -/* Return 0 on success. */
7 -int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
8 - unsigned *matched_files)
9 -{
10 - int ret;
11 - unsigned starting_fileno, fileno, i, j, recovered_files;
12 - struct metadata_logfile *metalogfile = NULL, *compactionfile = NULL, **tmp_metalogfiles;
13 - char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
14 -
15 - for (i = 0 ; i < *matched_files ; ++i) {
16 - metalogfile = metalogfiles[i];
17 - if (0 == metalogfile->starting_fileno)
18 - continue; /* skip standard metadata log files */
19 - break; /* this is a compaction temporary file */
20 - }
21 - if (i == *matched_files) /* no recovery needed */
22 - return 0;
23 - info("Starting metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
24 -
25 - if (*matched_files - i > 1) { /* Can't have more than 1 temporary compaction files */
26 - error("Metadata log files are in an invalid state. Cannot proceed.");
27 - return 1;
28 - }
29 - compactionfile = metalogfile;
30 - starting_fileno = compactionfile->starting_fileno;
31 - fileno = compactionfile->fileno;
32 - /* scratchpad space to move file pointers around */
33 - tmp_metalogfiles = callocz(*matched_files, sizeof(*tmp_metalogfiles));
34 -
35 - for (j = 0, recovered_files = 0 ; j < i ; ++j) {
36 - metalogfile = metalogfiles[j];
37 - fatal_assert(0 == metalogfile->starting_fileno);
38 - if (metalogfile->fileno < starting_fileno) {
39 - tmp_metalogfiles[recovered_files++] = metalogfile;
40 - continue;
41 - }
42 - break; /* reached compaction file serial number */
43 - }
44 -
45 - if ((j == i) /* Shouldn't be possible, invalid compaction temporary file */ ||
46 - (metalogfile->fileno == starting_fileno && metalogfile->fileno == fileno)) {
47 - error("Deleting invalid compaction temporary file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
48 - METALOG_EXTENSION"\"", dbfiles_path, starting_fileno, fileno);
49 - unlink_metadata_logfile(compactionfile);
50 - freez(compactionfile);
51 - freez(tmp_metalogfiles);
52 - --*matched_files; /* delete the last one */
53 -
54 - info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
55 - return 0;
56 - }
57 -
58 - for ( ; j < i ; ++j) { /* continue iterating through normal metadata log files */
59 - metalogfile = metalogfiles[j];
60 - fatal_assert(0 == metalogfile->starting_fileno);
61 - if (metalogfile->fileno < fileno) { /* It has already been compacted */
62 - error("Deleting invalid metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
63 - METALOG_EXTENSION"\"", dbfiles_path, 0U, metalogfile->fileno);
64 - unlink_metadata_logfile(metalogfile);
65 - freez(metalogfile);
66 - continue;
67 - }
68 - tmp_metalogfiles[recovered_files++] = metalogfile;
69 - }
70 -
71 - /* compaction temporary file is valid */
72 - tmp_metalogfiles[recovered_files++] = compactionfile;
73 - ret = rename_metadata_logfile(compactionfile, 0, starting_fileno);
74 - if (ret < 0) {
75 - error("Cannot rename temporary compaction files. Cannot proceed.");
76 - freez(tmp_metalogfiles);
77 - return 1;
78 - }
79 -
80 - memcpy(metalogfiles, tmp_metalogfiles, recovered_files * sizeof(*metalogfiles));
81 - *matched_files = recovered_files;
82 - freez(tmp_metalogfiles);
83 -
84 - info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
85 - return 0;
86 -}
database/engine/metadata_log/compaction.h deleted
-14
@@ -1,14 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_COMPACTION_H
4 -#define NETDATA_COMPACTION_H
5 -
6 -#ifndef _GNU_SOURCE
7 -#define _GNU_SOURCE
8 -#endif
9 -#include "../rrdengine.h"
10 -
11 -int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
12 - unsigned *matched_files);
13 -
14 -#endif /* NETDATA_COMPACTION_H */
database/engine/metadata_log/logfile.c deleted
-445
@@ -1,445 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -#include <database/sqlite/sqlite_functions.h>
3 -#include "metadatalog.h"
4 -#include "metalogpluginsd.h"
5 -
6 -
7 -void generate_metadata_logfile_path(struct metadata_logfile *metalogfile, char *str, size_t maxlen)
8 -{
9 - (void) snprintfz(str, maxlen, "%s/" METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION,
10 - metalogfile->ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
11 -}
12 -
13 -void metadata_logfile_init(struct metadata_logfile *metalogfile, struct metalog_instance *ctx, unsigned starting_fileno,
14 - unsigned fileno)
15 -{
16 - metalogfile->starting_fileno = starting_fileno;
17 - metalogfile->fileno = fileno;
18 - metalogfile->file = (uv_file)0;
19 - metalogfile->pos = 0;
20 - metalogfile->next = NULL;
21 - metalogfile->ctx = ctx;
22 -}
23 -
24 -int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno, unsigned new_fileno)
25 -{
26 - //struct metalog_instance *ctx = metalogfile->ctx;
27 - uv_fs_t req;
28 - int ret;
29 - char oldpath[RRDENG_PATH_MAX], newpath[RRDENG_PATH_MAX];
30 - unsigned backup_starting_fileno, backup_fileno;
31 -
32 - backup_starting_fileno = metalogfile->starting_fileno;
33 - backup_fileno = metalogfile->fileno;
34 - generate_metadata_logfile_path(metalogfile, oldpath, sizeof(oldpath));
35 - metalogfile->starting_fileno = new_starting_fileno;
36 - metalogfile->fileno = new_fileno;
37 - generate_metadata_logfile_path(metalogfile, newpath, sizeof(newpath));
38 -
39 - info("Renaming metadata log file \"%s\" to \"%s\".", oldpath, newpath);
40 - ret = uv_fs_rename(NULL, &req, oldpath, newpath, NULL);
41 - if (ret < 0) {
42 - error("uv_fs_rename(%s): %s", oldpath, uv_strerror(ret));
43 - //++ctx->stats.fs_errors; /* this is racy, may miss some errors */
44 - rrd_stat_atomic_add(&global_fs_errors, 1);
45 - /* restore previous values */
46 - metalogfile->starting_fileno = backup_starting_fileno;
47 - metalogfile->fileno = backup_fileno;
48 - }
49 - uv_fs_req_cleanup(&req);
50 -
51 - return ret;
52 -}
53 -
54 -int unlink_metadata_logfile(struct metadata_logfile *metalogfile)
55 -{
56 - //struct metalog_instance *ctx = metalogfile->ctx;
57 - uv_fs_t req;
58 - int ret;
59 - char path[RRDENG_PATH_MAX];
60 -
61 - generate_metadata_logfile_path(metalogfile, path, sizeof(path));
62 -
63 - ret = uv_fs_unlink(NULL, &req, path, NULL);
64 - if (ret < 0) {
65 - error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
66 -// ++ctx->stats.fs_errors;
67 - rrd_stat_atomic_add(&global_fs_errors, 1);
68 - }
69 - uv_fs_req_cleanup(&req);
70 -
71 - return ret;
72 -}
73 -
74 -static int check_metadata_logfile_superblock(uv_file file)
75 -{
76 - int ret;
77 - struct rrdeng_metalog_sb *superblock;
78 - uv_buf_t iov;
79 - uv_fs_t req;
80 -
81 - ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
82 - if (unlikely(ret)) {
83 - fatal("posix_memalign:%s", strerror(ret));
84 - }
85 - iov = uv_buf_init((void *)superblock, sizeof(*superblock));
86 -
87 - ret = uv_fs_read(NULL, &req, file, &iov, 1, 0, NULL);
88 - if (ret < 0) {
89 - error("uv_fs_read: %s", uv_strerror(ret));
90 - uv_fs_req_cleanup(&req);
91 - goto error;
92 - }
93 - fatal_assert(req.result >= 0);
94 - uv_fs_req_cleanup(&req);
95 -
96 - if (strncmp(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ)) {
97 - error("File has invalid superblock.");
98 - ret = UV_EINVAL;
99 - } else {
100 - ret = 0;
101 - }
102 - if (superblock->version > RRDENG_METALOG_VER) {
103 - error("File has unknown version %"PRIu16". Compatibility is not guaranteed.", superblock->version);
104 - }
105 -error:
106 - posix_memfree(superblock);
107 - return ret;
108 -}
109 -
110 -void replay_record(struct metadata_logfile *metalogfile, struct rrdeng_metalog_record_header *header, void *payload)
111 -{
112 - struct metalog_instance *ctx = metalogfile->ctx;
113 - char *line, *nextline, *record_end;
114 - int ret;
115 -
116 - debug(D_METADATALOG, "RECORD contents: %.*s", (int)header->payload_length, (char *)payload);
117 - record_end = (char *)payload + header->payload_length - 1;
118 - *record_end = '\0';
119 -
120 - for (line = payload ; line ; line = nextline) {
121 - nextline = strchr(line, '\n');
122 - if (nextline) {
123 - *nextline++ = '\0';
124 - }
125 - ret = parser_action(ctx->metalog_parser_object->parser, line);
126 - debug(D_METADATALOG, "parser_action ret:%d", ret);
127 - if (ret)
128 - return; /* skip record due to error */
129 - };
130 -}
131 -
132 -/* This function only works with buffered I/O */
133 -static inline int metalogfile_read(struct metadata_logfile *metalogfile, void *buf, size_t len, uint64_t offset)
134 -{
135 -// struct metalog_instance *ctx;
136 - uv_file file;
137 - uv_buf_t iov;
138 - uv_fs_t req;
139 - int ret;
140 -
141 -// ctx = metalogfile->ctx;
142 - file = metalogfile->file;
143 - iov = uv_buf_init(buf, len);
144 - ret = uv_fs_read(NULL, &req, file, &iov, 1, offset, NULL);
145 - if (unlikely(ret < 0 && ret != req.result)) {
146 - fatal("uv_fs_read: %s", uv_strerror(ret));
147 - }
148 - if (req.result < 0) {
149 -// ++ctx->stats.io_errors;
150 - rrd_stat_atomic_add(&global_io_errors, 1);
151 - error("%s: uv_fs_read - %s - record at offset %"PRIu64"(%u) in metadata logfile %u-%u.", __func__,
152 - uv_strerror((int)req.result), offset, (unsigned)len, metalogfile->starting_fileno, metalogfile->fileno);
153 - }
154 - uv_fs_req_cleanup(&req);
155 -// ctx->stats.io_read_bytes += len;
156 -// ++ctx->stats.io_read_requests;
157 -
158 - return ret;
159 -}
160 -
161 -/* Return 0 on success */
162 -static int metadata_record_integrity_check(void *record)
163 -{
164 - int ret;
165 - uint32_t data_size;
166 - struct rrdeng_metalog_record_header *header;
167 - struct rrdeng_metalog_record_trailer *trailer;
168 - uLong crc;
169 -
170 - header = record;
171 - data_size = header->header_length + header->payload_length;
172 - trailer = record + data_size;
173 -
174 - crc = crc32(0L, Z_NULL, 0);
175 - crc = crc32(crc, record, data_size);
176 - ret = crc32cmp(trailer->checksum, crc);
177 -
178 - return ret;
179 -}
180 -
181 -#define MAX_READ_BYTES (RRDENG_BLOCK_SIZE * 32) /* no record should be over 128KiB in this version */
182 -
183 -/*
184 - * Iterates metadata log file records and creates database objects (host/chart/dimension)
185 - */
186 -static void iterate_records(struct metadata_logfile *metalogfile)
187 -{
188 - uint32_t file_size, pos, bytes_remaining, record_size;
189 - void *buf;
190 - struct rrdeng_metalog_record_header *header;
191 - struct metalog_instance *ctx = metalogfile->ctx;
192 - struct metalog_pluginsd_state *state = ctx->metalog_parser_object->private;
193 - const size_t min_header_size = offsetof(struct rrdeng_metalog_record_header, header_length) +
194 - sizeof(header->header_length);
195 -
196 - file_size = metalogfile->pos;
197 - state->metalogfile = metalogfile;
198 -
199 - buf = mallocz(MAX_READ_BYTES);
200 -
201 - for (pos = sizeof(struct rrdeng_metalog_sb) ; pos < file_size ; pos += record_size) {
202 - bytes_remaining = file_size - pos;
203 - if (bytes_remaining < min_header_size) {
204 - error("%s: unexpected end of file in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
205 - metalogfile->fileno);
206 - break;
207 - }
208 - if (metalogfile_read(metalogfile, buf, min_header_size, pos) < 0)
209 - break;
210 - header = (struct rrdeng_metalog_record_header *)buf;
211 - if (METALOG_STORE_PADDING == header->type) {
212 - info("%s: Skipping padding in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
213 - metalogfile->fileno);
214 - record_size = ALIGN_BYTES_FLOOR(pos + RRDENG_BLOCK_SIZE) - pos;
215 - continue;
216 - }
217 - if (metalogfile_read(metalogfile, buf + min_header_size, sizeof(*header) - min_header_size,
218 - pos + min_header_size) < 0)
219 - break;
220 - record_size = header->header_length + header->payload_length + sizeof(struct rrdeng_metalog_record_trailer);
221 - if (header->header_length < min_header_size || record_size > bytes_remaining) {
222 - error("%s: Corrupted record in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
223 - metalogfile->fileno);
224 - break;
225 - }
226 - if (record_size > MAX_READ_BYTES) {
227 - error("%s: Record is too long (%u bytes) in metadata logfile %u-%u.", __func__, record_size,
228 - metalogfile->starting_fileno, metalogfile->fileno);
229 - continue;
230 - }
231 - if (metalogfile_read(metalogfile, buf + sizeof(*header), record_size - sizeof(*header),
232 - pos + sizeof(*header)) < 0)
233 - break;
234 - if (metadata_record_integrity_check(buf)) {
235 - error("%s: Record at offset %"PRIu32" was read from disk. CRC32 check: FAILED", __func__, pos);
236 - continue;
237 - }
238 - debug(D_METADATALOG, "%s: Record at offset %"PRIu32" was read from disk. CRC32 check: SUCCEEDED", __func__,
239 - pos);
240 -
241 - replay_record(metalogfile, header, buf + header->header_length);
242 - }
243 -
244 - freez(buf);
245 -}
246 -
247 -int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
248 -{
249 - UNUSED(ctx);
250 - uv_fs_t req;
251 - uv_file file;
252 - int ret, fd, error;
253 - uint64_t file_size;
254 - char path[RRDENG_PATH_MAX];
255 -
256 - generate_metadata_logfile_path(metalogfile, path, sizeof(path));
257 - if (file_is_migrated(path))
258 - return 0;
259 -
260 - fd = open_file_buffered_io(path, O_RDWR, &file);
261 - if (fd < 0) {
262 -// ++ctx->stats.fs_errors;
263 - rrd_stat_atomic_add(&global_fs_errors, 1);
264 - return fd;
265 - }
266 - info("Loading metadata log \"%s\".", path);
267 -
268 - ret = check_file_properties(file, &file_size, sizeof(struct rrdeng_metalog_sb));
269 - if (ret)
270 - goto error;
271 -
272 - ret = check_metadata_logfile_superblock(file);
273 - if (ret)
274 - goto error;
275 -// ctx->stats.io_read_bytes += sizeof(struct rrdeng_jf_sb);
276 -// ++ctx->stats.io_read_requests;
277 -
278 - metalogfile->file = file;
279 - metalogfile->pos = file_size;
280 -
281 - iterate_records(metalogfile);
282 -
283 - info("Metadata log \"%s\" migrated to the database (size:%"PRIu64").", path, file_size);
284 - add_migrated_file(path, file_size);
285 - return 0;
286 -
287 -error:
288 - error = ret;
289 - ret = uv_fs_close(NULL, &req, file, NULL);
290 - if (ret < 0) {
291 - error("uv_fs_close(%s): %s", path, uv_strerror(ret));
292 -// ++ctx->stats.fs_errors;
293 - rrd_stat_atomic_add(&global_fs_errors, 1);
294 - }
295 - uv_fs_req_cleanup(&req);
296 - return error;
297 -}
298 -
299 -static int scan_metalog_files_cmp(const void *a, const void *b)
300 -{
301 - struct metadata_logfile *file1, *file2;
302 - char path1[RRDENG_PATH_MAX], path2[RRDENG_PATH_MAX];
303 -
304 - file1 = *(struct metadata_logfile **)a;
305 - file2 = *(struct metadata_logfile **)b;
306 - generate_metadata_logfile_path(file1, path1, sizeof(path1));
307 - generate_metadata_logfile_path(file2, path2, sizeof(path2));
308 - return strcmp(path1, path2);
309 -}
310 -
311 -/* Returns number of metadata logfiles that were loaded or < 0 on error */
312 -static int scan_metalog_files(struct metalog_instance *ctx)
313 -{
314 - int ret;
315 - unsigned starting_no, no, matched_files, i, failed_to_load;
316 - static uv_fs_t req;
317 - uv_dirent_t dent;
318 - struct metadata_logfile **metalogfiles, *metalogfile;
319 - char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
320 -
321 - ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
322 - if (ret < 0) {
323 - fatal_assert(req.result < 0);
324 - uv_fs_req_cleanup(&req);
325 - error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
326 -// ++ctx->stats.fs_errors;
327 - rrd_stat_atomic_add(&global_fs_errors, 1);
328 - return ret;
329 - }
330 - info("Found %d files in path %s", ret, dbfiles_path);
331 -
332 - metalogfiles = callocz(MIN(ret, MAX_DATAFILES), sizeof(*metalogfiles));
333 - for (matched_files = 0 ; UV_EOF != uv_fs_scandir_next(&req, &dent) && matched_files < MAX_DATAFILES ; ) {
334 - info("Scanning file \"%s/%s\"", dbfiles_path, dent.name);
335 - ret = sscanf(dent.name, METALOG_PREFIX METALOG_FILE_NUMBER_SCAN_TMPL METALOG_EXTENSION, &starting_no, &no);
336 - if (2 == ret) {
337 - info("Matched file \"%s/%s\"", dbfiles_path, dent.name);
338 - metalogfile = mallocz(sizeof(*metalogfile));
339 - metadata_logfile_init(metalogfile, ctx, starting_no, no);
340 - metalogfiles[matched_files++] = metalogfile;
341 - }
342 - }
343 - uv_fs_req_cleanup(&req);
344 -
345 - if (0 == matched_files) {
346 - freez(metalogfiles);
347 - return 0;
348 - }
349 - if (matched_files == MAX_DATAFILES) {
350 - error("Warning: hit maximum database engine file limit of %d files", MAX_DATAFILES);
351 - }
352 - qsort(metalogfiles, matched_files, sizeof(*metalogfiles), scan_metalog_files_cmp);
353 - ret = compaction_failure_recovery(ctx, metalogfiles, &matched_files);
354 - if (ret) { /* If the files are corrupted fail */
355 - for (i = 0 ; i < matched_files ; ++i) {
356 - freez(metalogfiles[i]);
357 - }
358 - freez(metalogfiles);
359 - return UV_EINVAL;
360 - }
361 - //ctx->last_fileno = metalogfiles[matched_files - 1]->fileno;
362 -
363 - struct plugind cd = {
364 - .enabled = 1,
365 - .update_every = 0,
366 - .pid = 0,
367 - .serial_failures = 0,
368 - .successful_collections = 0,
369 - .obsolete = 0,
370 - .started_t = INVALID_TIME,
371 - .next = NULL,
372 - .capabilities = 0,
373 - };
374 -
375 - struct metalog_pluginsd_state metalog_parser_state;
376 - metalog_pluginsd_state_init(&metalog_parser_state, ctx);
377 -
378 - PARSER_USER_OBJECT metalog_parser_object = {
379 - .enabled = cd.enabled,
380 - .host = ctx->rrdeng_ctx->host,
381 - .cd = &cd,
382 - .trust_durations = 0,
383 - .private = &metalog_parser_state
384 - };
385 -
386 - PARSER *parser = parser_init(metalog_parser_object.host, &metalog_parser_object, NULL, NULL, PARSER_INPUT_SPLIT|PARSER_NO_ACTION_INIT);
387 - parser_add_keyword(parser, PLUGINSD_KEYWORD_HOST, metalog_pluginsd_host);
388 - parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
389 - parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
390 - parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
391 - parser->plugins_action->guid_action = &metalog_pluginsd_guid_action;
392 - parser->plugins_action->context_action = &metalog_pluginsd_context_action;
393 - parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
394 - parser->plugins_action->host_action = &metalog_pluginsd_host_action;
395 -
396 -
397 - metalog_parser_object.parser = parser;
398 - ctx->metalog_parser_object = &metalog_parser_object;
399 -
400 - for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {
401 - metalogfile = metalogfiles[i];
402 - db_lock();
403 - db_execute("BEGIN TRANSACTION;");
404 - ret = load_metadata_logfile(ctx, metalogfile);
405 - if (0 != ret) {
406 - error("Deleting invalid metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
407 - METALOG_EXTENSION"\"", dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
408 - unlink_metadata_logfile(metalogfile);
409 - ++failed_to_load;
410 - db_execute("ROLLBACK TRANSACTION;");
411 - }
412 - else
413 - db_execute("COMMIT TRANSACTION;");
414 - db_unlock();
415 - freez(metalogfile);
416 - }
417 - matched_files -= failed_to_load;
418 - debug(D_METADATALOG, "PARSER ended");
419 -
420 - parser_destroy(parser);
421 -
422 - size_t count __maybe_unused = metalog_parser_object.count;
423 -
424 - debug(D_METADATALOG, "Parsing count=%u", (unsigned)count);
425 -
426 - freez(metalogfiles);
427 - return matched_files;
428 -}
429 -
430 -/* Return 0 on success. */
431 -int init_metalog_files(struct metalog_instance *ctx)
432 -{
433 - int ret;
434 - char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
435 -
436 - ret = scan_metalog_files(ctx);
437 - if (ret < 0) {
438 - error("Failed to scan path \"%s\".", dbfiles_path);
439 - return ret;
440 - }/* else if (0 == ret) {
441 - ctx->last_fileno = 1;
442 - }*/
443 -
444 - return 0;
445 -}
database/engine/metadata_log/logfile.h deleted
-39
@@ -1,39 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_LOGFILE_H
4 -#define NETDATA_LOGFILE_H
5 -
6 -#include "metadatalogprotocol.h"
7 -#include "../rrdengine.h"
8 -
9 -/* Forward declarations */
10 -struct metadata_logfile;
11 -struct metalog_worker_config;
12 -
13 -#define METALOG_PREFIX "metadatalog-"
14 -#define METALOG_EXTENSION ".mlf"
15 -
16 -/* only one event loop is supported for now */
17 -struct metadata_logfile {
18 - unsigned fileno; /* Starts at 1 */
19 - unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
20 - uv_file file;
21 - uint64_t pos;
22 - struct metalog_instance *ctx;
23 - struct metadata_logfile *next;
24 -};
25 -
26 -struct metadata_logfile_list {
27 - struct metadata_logfile *first; /* oldest */
28 - struct metadata_logfile *last; /* newest */
29 -};
30 -
31 -void generate_metadata_logfile_path(struct metadata_logfile *metadatalog, char *str, size_t maxlen);
32 -int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno,
33 - unsigned new_fileno);
34 -int unlink_metadata_logfile(struct metadata_logfile *metalogfile);
35 -int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *logfile);
36 -int init_metalog_files(struct metalog_instance *ctx);
37 -
38 -
39 -#endif /* NETDATA_LOGFILE_H */
database/engine/metadata_log/metadatalog.h deleted
-28
@@ -1,28 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_METADATALOG_H
4 -#define NETDATA_METADATALOG_H
5 -
6 -#ifndef _GNU_SOURCE
7 -#define _GNU_SOURCE
8 -#endif
9 -#include "../rrdengine.h"
10 -#include "metadatalogprotocol.h"
11 -#include "logfile.h"
12 -#include "metadatalogapi.h"
13 -#include "compaction.h"
14 -
15 -/* Forward declarations */
16 -struct metalog_instance;
17 -struct parser_user_object;
18 -
19 -#define METALOG_FILE_NUMBER_SCAN_TMPL "%5u-%5u"
20 -#define METALOG_FILE_NUMBER_PRINT_TMPL "%5.5u-%5.5u"
21 -
22 -struct metalog_instance {
23 - struct rrdengine_instance *rrdeng_ctx;
24 - struct parser_user_object *metalog_parser_object;
25 - uint8_t initialized; /* set to 1 to mark context initialized */
26 -};
27 -
28 -#endif /* NETDATA_METADATALOG_H */
database/engine/metadata_log/metadatalogapi.c deleted
-39
@@ -1,39 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
3 -
4 -#include "metadatalog.h"
5 -
6 -/*
7 - * Returns 0 on success, negative on error
8 - */
9 -int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)
10 -{
11 - struct metalog_instance *ctx;
12 - int error;
13 -
14 - ctx = callocz(1, sizeof(*ctx));
15 - ctx->initialized = 0;
16 - rrdeng_parent_ctx->metalog_ctx = ctx;
17 -
18 - ctx->rrdeng_ctx = rrdeng_parent_ctx;
19 - error = init_metalog_files(ctx);
20 - if (error) {
21 - goto error_after_init_rrd_files;
22 - }
23 - ctx->initialized = 1; /* notify dbengine that the metadata log has finished initializing */
24 - return 0;
25 -
26 -error_after_init_rrd_files:
27 - freez(ctx);
28 - return UV_EIO;
29 -}
30 -
31 -/* This function is called by dbengine rotation logic when the metric has no writers */
32 -void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
33 -{
34 - uuid_t multihost_uuid;
35 -
36 - delete_dimension_uuid(metric_uuid);
37 - rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->machine_guid, metric_uuid, &multihost_uuid);
38 - delete_dimension_uuid(&multihost_uuid);
39 -}
database/engine/metadata_log/metadatalogapi.h deleted
-12
@@ -1,12 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_METADATALOGAPI_H
4 -#define NETDATA_METADATALOGAPI_H
5 -
6 -void metalog_commit_delete_chart(RRDSET *st);
7 -void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
8 -
9 -/* must call once before using anything */
10 -int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx);
11 -
12 -#endif /* NETDATA_METADATALOGAPI_H */
database/engine/metadata_log/metadatalogprotocol.h deleted
-53
@@ -1,53 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_METADATALOGPROTOCOL_H
4 -#define NETDATA_METADATALOGPROTOCOL_H
5 -
6 -#include "../rrddiskprotocol.h"
7 -
8 -#define RRDENG_METALOG_MAGIC "netdata-metadata-log"
9 -
10 -#define RRDENG_METALOG_VER (1)
11 -
12 -#define RRDENG_METALOG_SB_PADDING_SZ (RRDENG_BLOCK_SIZE - (RRDENG_MAGIC_SZ + sizeof(uint16_t)))
13 -/*
14 - * Metadata log persistent super-block
15 - */
16 -struct rrdeng_metalog_sb {
17 - char magic_number[RRDENG_MAGIC_SZ];
18 - uint16_t version;
19 - uint8_t padding[RRDENG_METALOG_SB_PADDING_SZ];
20 -} __attribute__ ((packed));
21 -
22 -/*
23 - * Metadata log record types
24 - */
25 -#define METALOG_STORE_PADDING (0)
26 -#define METALOG_CREATE_OBJECT (1)
27 -#define METALOG_DELETE_OBJECT (2)
28 -#define METALOG_OTHER (3) /* reserved */
29 -
30 -/*
31 - * Metadata log record header
32 - */
33 -struct rrdeng_metalog_record_header {
34 - /* when set to METALOG_STORE_PADDING jump to start of next block */
35 - uint8_t type;
36 -
37 - uint16_t header_length;
38 - uint32_t payload_length;
39 - /******************************************************
40 - * No fields above this point can ever change. *
41 - ******************************************************
42 - * All fields below this point are subject to change. *
43 - ******************************************************/
44 -} __attribute__ ((packed));
45 -
46 -/*
47 - * Metadata log record trailer
48 - */
49 -struct rrdeng_metalog_record_trailer {
50 - uint8_t checksum[CHECKSUM_SZ]; /* CRC32 */
51 -} __attribute__ ((packed));
52 -
53 -#endif /* NETDATA_METADATALOGPROTOCOL_H */
database/engine/metadata_log/metalogpluginsd.c deleted
-89
@@ -1,89 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
3 -
4 -#include "metadatalog.h"
5 -#include "metalogpluginsd.h"
6 -
7 -extern struct config stream_config;
8 -
9 -PARSER_RC metalog_pluginsd_host_action(
10 - void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone,
11 - char *tags)
12 -{
13 - struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
14 -
15 - RRDHOST *host = rrdhost_find_by_guid(machine_guid);
16 - if (host) {
17 - if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
18 - error("Archived host '%s' has memory mode '%s', but the archived one is '%s'. Ignoring archived state.",
19 - rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode),
20 - rrd_memory_mode_name(RRD_MEMORY_MODE_DBENGINE));
21 - ((PARSER_USER_OBJECT *) user)->host = NULL; /* Ignore objects if memory mode is not dbengine */
22 - }
23 - ((PARSER_USER_OBJECT *) user)->host = host;
24 - return PARSER_RC_OK;
25 - }
26 -
27 - if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) {
28 - ((PARSER_USER_OBJECT *) user)->host = host;
29 - return PARSER_RC_OK;
30 - }
31 -
32 - if (likely(!uuid_parse(machine_guid, state->host_uuid))) {
33 - int rc = sql_store_host(&state->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags, 1);
34 - if (unlikely(rc)) {
35 - errno = 0;
36 - error("Failed to store host %s with UUID %s in the database", hostname, machine_guid);
37 - }
38 - }
39 - else {
40 - errno = 0;
41 - error("Host machine GUID %s is not valid", machine_guid);
42 - }
43 -
44 - return PARSER_RC_OK;
45 -}
46 -
47 -PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid)
48 -{
49 - struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
50 -
51 - uuid_copy(state->uuid, *uuid);
52 -
53 - return PARSER_RC_OK;
54 -}
55 -
56 -PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
57 -{
58 - struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
59 -
60 - int rc = find_uuid_type(uuid);
61 -
62 - if (rc == 1) {
63 - uuid_copy(state->host_uuid, *uuid);
64 - ((PARSER_USER_OBJECT *)user)->st_exists = 0;
65 - ((PARSER_USER_OBJECT *)user)->host_exists = 1;
66 - } else if (rc == 2) {
67 - uuid_copy(state->chart_uuid, *uuid);
68 - ((PARSER_USER_OBJECT *)user)->st_exists = 1;
69 - } else
70 - uuid_copy(state->uuid, *uuid);
71 -
72 - return PARSER_RC_OK;
73 -}
74 -
75 -PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)
76 -{
77 - UNUSED(user);
78 - UNUSED(uuid);
79 -
80 - return PARSER_RC_OK;
81 -}
82 -
83 -void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx)
84 -{
85 - state->ctx = ctx;
86 - state->skip_record = 0;
87 - uuid_clear(state->uuid);
88 - state->metalogfile = NULL;
89 -}
database/engine/metadata_log/metalogpluginsd.h deleted
-29
@@ -1,29 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_METALOGPLUGINSD_H
4 -#define NETDATA_METALOGPLUGINSD_H
5 -
6 -#include "collectors/plugins.d/pluginsd_parser.h"
7 -#include "collectors/plugins.d/plugins_d.h"
8 -#include "parser/parser.h"
9 -
10 -struct metalog_pluginsd_state {
11 - struct metalog_instance *ctx;
12 - uuid_t uuid;
13 - uuid_t host_uuid;
14 - uuid_t chart_uuid;
15 - uint8_t skip_record; /* skip this record due to errors in parsing */
16 - struct metadata_logfile *metalogfile; /* current metadata log file being replayed */
17 -};
18 -
19 -void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx);
20 -PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
21 - long multiplier, long divisor, char *options,
22 - RRD_ALGORITHM algorithm_type);
23 -PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
24 -PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
25 -PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
26 -PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action);
27 -PARSER_RC metalog_pluginsd_host_action(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone, char *tags);
28 -
29 -#endif /* NETDATA_METALOGPLUGINSD_H */
database/engine/rrdengine.c
+2 -6
@@ -947,12 +947,12 @@ static void delete_old_data(void *arg)
947 for (i = 0 ; i < count ; ++i) {
948 descr = extent->pages[i];
949 can_delete_metric = pg_cache_punch_hole(ctx, descr, 0, 0, &metric_id);
950 - if (unlikely(can_delete_metric && ctx->metalog_ctx->initialized)) {
950 + if (unlikely(can_delete_metric)) {
951 /*
952 * If the metric is empty, has no active writers and if the metadata log has been initialized then
953 * attempt to delete the corresponding netdata dimension.
954 */
955 - metalog_delete_dimension_by_uuid(ctx->metalog_ctx, &metric_id);
955 + metaqueue_delete_dimension_uuid(&metric_id);
956 }
957 }
958 next = extent->next;
@@ -1201,10 +1201,6 @@ void timer_cb(uv_timer_t* handle)
1201
1202 uv_stop(handle->loop);
1203 uv_update_time(handle->loop);
1204 - if (unlikely(!ctx->metalog_ctx->initialized)) {
1205 - worker_is_idle();
1206 - return; /* Wait for the metadata log to initialize */
1207 - }
1204 rrdeng_test_quota(wc);
1205 debug(D_RRDENGINE, "%s: timeout reached.", __func__);
1206 if (likely(!wc->now_deleting_files && !wc->now_invalidating_dirty_pages)) {
database/engine/rrdengine.h
-2
@@ -17,7 +17,6 @@
17 #include "rrdenginelib.h"
18 #include "datafile.h"
19 #include "journalfile.h"
20 -#include "metadata_log/metadatalog.h"
20 #include "rrdengineapi.h"
21 #include "pagecache.h"
22 #include "rrdenglocking.h"
@@ -236,7 +235,6 @@ typedef enum {
235 } INVALID_PAGE_ID;
236
237 struct rrdengine_instance {
239 - struct metalog_instance *metalog_ctx;
238 struct rrdengine_worker_config worker_config;
239 struct completion rrdengine_completion;
240 struct page_cache pg_cache;
database/engine/rrdengineapi.c
+5 -6
@@ -1087,7 +1087,6 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
1087 ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;
1088 ctx->metric_API_max_producers = 0;
1089 ctx->quiesce = NO_QUIESCE;
1090 - ctx->metalog_ctx = NULL; /* only set this after the metadata log has finished initializing */
1090 ctx->host = host;
1091
1092 memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
@@ -1108,11 +1107,11 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
1107 if (ctx->worker_config.error) {
1108 goto error_after_rrdeng_worker;
1109 }
1111 - error = metalog_init(ctx);
1112 - if (error) {
1113 - error("Failed to initialize metadata log file event loop.");
1114 - goto error_after_rrdeng_worker;
1115 - }
1110 +// error = metalog_init(ctx);
1111 +// if (error) {
1112 +// error("Failed to initialize metadata log file event loop.");
1113 +// goto error_after_rrdeng_worker;
1114 +// }
1115
1116 return 0;
1117
database/rrd.h
+6 -1
@@ -193,7 +193,7 @@ typedef enum rrddim_flags {
193 // No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at
194 // least rrdset_free_obsolete_time seconds ago.
195 RRDDIM_FLAG_ARCHIVED = (1 << 3),
196 - RRDDIM_FLAG_ACLK = (1 << 4),
196 + RRDDIM_FLAG_METADATA_UPDATE = (1 << 4), // Metadata needs to go to the database
197
198 RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database
199
@@ -501,6 +501,8 @@ typedef enum rrdset_flags {
501 // No new values have been collected for this chart since agent start, or it was marked RRDSET_FLAG_OBSOLETE at
502 // least rrdset_free_obsolete_time seconds ago.
503 RRDSET_FLAG_ARCHIVED = (1 << 15),
504 + RRDSET_FLAG_METADATA_UPDATE = (1 << 16), // Mark that metadata needs to be stored
505 + RRDSET_FLAG_PENDING_FOREACH_ALARMS = (1 << 17), // contains dims with uninitialized foreach alarms
506 RRDSET_FLAG_ANOMALY_DETECTION = (1 << 18), // flag to identify anomaly detection charts.
507 RRDSET_FLAG_INDEXED_ID = (1 << 19), // the rrdset is indexed by its id
508 RRDSET_FLAG_INDEXED_NAME = (1 << 20), // the rrdset is indexed by its name
@@ -727,6 +729,8 @@ typedef enum rrdhost_flags {
729
730 // ACLK
731 RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 24), // when set, we should send ACLK stream context updates
732 + // Metadata
733 + RRDHOST_FLAG_METADATA_UPDATE = (1 << 25), // metadata needs to be stored in the database
734 } RRDHOST_FLAGS;
735
736 #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
@@ -1317,6 +1321,7 @@ int get_tier_grouping(int tier);
1321 #endif
1322 #include "sqlite/sqlite_functions.h"
1323 #include "sqlite/sqlite_context.h"
1324 +#include "sqlite/sqlite_metadata.h"
1325 #include "sqlite/sqlite_aclk.h"
1326 #include "sqlite/sqlite_aclk_alert.h"
1327 #include "sqlite/sqlite_aclk_node.h"
database/rrddim.c
+18 -34
@@ -86,7 +86,9 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
86 rd->aclk_live_status = -1;
87 #endif
88
89 - (void) find_dimension_uuid(st, rd, &(rd->metric_uuid));
89 + if (unlikely(find_dimension_uuid(st, rd, &(rd->metric_uuid)))) {
90 + uuid_generate(rd->metric_uuid);
91 + }
92
93 // initialize the db tiers
94 {
@@ -197,7 +199,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
199
200 if (tiers_available == tiers_said_yes && tiers_said_yes && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
201 /* This metric has no data and no references */
200 - delete_dimension_uuid(&rd->metric_uuid);
202 + metaqueue_delete_dimension_uuid(&rd->metric_uuid);
203 }
204 }
205
@@ -276,10 +278,13 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
278 RRDDIM *rd = rrddim;
279 RRDSET *st = ctr->st;
280
279 - if(ctr->react_action == RRDDIM_REACT_UPDATED) {
280 - debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rrddim_id(rd));
281 - (void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
281 + if(ctr->react_action & (RRDDIM_REACT_UPDATED | RRDDIM_REACT_NEW)) {
282 + rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE);
283 + rrdset_flag_set(rd->rrdset, RRDSET_FLAG_METADATA_UPDATE);
284 + rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
285 + }
286
287 + if(ctr->react_action == RRDDIM_REACT_UPDATED) {
288 // the chart needs to be updated to the parent
289 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
290 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
@@ -421,29 +426,6 @@ time_t rrddim_first_entry_t(RRDDIM *rd) {
426 return oldest;
427 }
428
424 -// Return either
425 -// 0 : Dimension is live
426 -// last collected time : Dimension is not live
427 -
428 -#ifdef ENABLE_ACLK
429 -#define RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER (3)
430 -#define RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER (30)
431 -time_t calc_dimension_liveness(RRDDIM *rd, time_t now)
432 -{
433 - time_t last_updated = rd->last_collected_time.tv_sec;
434 - int live;
435 -
436 - if(rd->aclk_live_status == 1) {
437 - int offline_time = RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER * rd->update_every;
438 - live = ((now - last_updated) < MIN(rrdset_free_obsolete_time, offline_time));
439 - }
440 - else
441 - live = ((now - last_updated) < RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER * rd->update_every);
442 -
443 - return live ? 0 : last_updated;
444 -}
445 -#endif
446 -
429 RRDDIM *rrddim_add_custom(RRDSET *st
430 , const char *id
431 , const char *name
@@ -499,11 +481,12 @@ int rrddim_hide(RRDSET *st, const char *id) {
481 error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
482 return 1;
483 }
502 - if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
503 - (void)sql_set_dimension_option(&rd->metric_uuid, "hidden");
484 + if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
485 + rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
486 + metaqueue_dimension_update_flags(rd);
487 + }
488
489 rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
506 - rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
490 rrdcontext_updated_rrddim_flags(rd);
491 return 0;
492 }
@@ -517,11 +500,12 @@ int rrddim_unhide(RRDSET *st, const char *id) {
500 error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
501 return 1;
502 }
520 - if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
521 - (void)sql_set_dimension_option(&rd->metric_uuid, NULL);
503 + if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
504 + rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
505 + metaqueue_dimension_update_flags(rd);
506 + }
507
508 rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN);
524 - rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
509 rrdcontext_updated_rrddim_flags(rd);
510 return 0;
511 }
database/rrdhost.c
+10 -12
@@ -46,7 +46,7 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine) {
46 // ----------------------------------------------------------------------------
47 // RRDHOST indexes management
48
49 -static DICTIONARY *rrdhost_root_index = NULL;
49 +DICTIONARY *rrdhost_root_index = NULL;
50 static DICTIONARY *rrdhost_root_index_hostname = NULL;
51
52 static inline void rrdhost_init() {
@@ -299,6 +299,8 @@ static void rrdhost_initialize_health(RRDHOST *host,
299 add_migrated_file(host->health_log_filename, 0);
300 }
301 } else {
302 + // TODO: This needs to go to the metadata thread
303 + // Health should wait before accessing the table (needs to be created by the metadata thread
304 sql_create_health_log_table(host);
305 sql_health_alarm_log_load(host);
306 }
@@ -414,14 +416,8 @@ int is_legacy = 1;
416 }
417
418 if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) {
417 - int rc;
418 -
419 - if(!archived) {
420 - rc = sql_store_host_info(host);
421 - if (unlikely(rc))
422 - error_report("Failed to store machine GUID to the database");
423 - }
424 -
419 + if(!archived)
420 + metaqueue_host_update_info(host->machine_guid);
421 sql_load_node_id(host);
422 }
423 else
@@ -541,7 +537,8 @@ int is_legacy = 1;
537 , string2str(host->health_default_exec)
538 , string2str(host->health_default_recipient)
539 );
544 - sql_store_host_system_info(&host->host_uuid, system_info);
540 + if(!archived)
541 + metaqueue_host_update_system_info(host);
542
543 rrd_hosts_available++;
544
@@ -584,7 +581,7 @@ void rrdhost_update(RRDHOST *host
581
582 rrdhost_system_info_free(host->system_info);
583 host->system_info = system_info;
587 - sql_store_host_system_info(&host->host_uuid, system_info);
584 + metaqueue_host_update_system_info(host);
585
586 rrdhost_init_os(host, os);
587 rrdhost_init_timezone(host, timezone, abbrev_timezone, utc_offset);
@@ -922,6 +919,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
919 health_init();
920
921 unittest:
922 + metadata_sync_init();
923 debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
924 rrd_wrlock();
925 localhost = rrdhost_create(
@@ -1385,7 +1383,7 @@ void reload_host_labels(void) {
1383 rrdhost_load_auto_labels();
1384
1385 rrdlabels_remove_all_unmarked(localhost->rrdlabels);
1388 - sql_store_host_labels(localhost);
1386 + metaqueue_store_host_labels(localhost->machine_guid);
1387
1388 health_label_log_save(localhost);
1389
database/rrdlabels.c
+1 -38
@@ -952,23 +952,6 @@ int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each,
952 return dictionary_walkthrough_read(labels, label_to_buffer_callback, (void *)&tmp);
953 }
954
955 -struct label_str {
956 - BUFFER *sql;
957 - int count;
958 - char uuid_str[UUID_STR_LEN];
959 -};
960 -
961 -static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
962 - struct label_str *lb = data;
963 - if (unlikely(!lb->count))
964 - buffer_sprintf(lb->sql, "INSERT OR REPLACE INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES ");
965 - else
966 - buffer_strcat(lb->sql, ", ");
967 - buffer_sprintf(lb->sql, "(u2h('%s'), %d,'%s','%s', unixepoch())", lb->uuid_str, ls, name, value);
968 - lb->count++;
969 - return 1;
970 -}
971 -
955 void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
956 if(!st->rrdlabels)
957 st->rrdlabels = rrdlabels_create();
@@ -976,27 +959,7 @@ void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
959 if (new_rrdlabels)
960 rrdlabels_migrate_to_these(st->rrdlabels, new_rrdlabels);
961
979 - rrdset_save_rrdlabels_to_sql(st);
980 -}
981 -
982 -void rrdset_save_rrdlabels_to_sql(RRDSET *st) {
983 - if(!st->rrdlabels) return;
984 -
985 - size_t old_version = st->rrdlabels_last_saved_version;
986 - size_t new_version = dictionary_version(st->rrdlabels);
987 -
988 - if(new_version != old_version) {
989 - // TODO - we should also cleanup sqlite from old new_rrdlabels that have been removed
990 -
991 - BUFFER *sql_buf = buffer_create(1024);
992 - struct label_str tmp = {.sql = sql_buf, .count = 0};
993 - uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
994 - rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
995 - db_execute(buffer_tostring(sql_buf));
996 - buffer_free(sql_buf);
997 -
998 - st->rrdlabels_last_saved_version = new_version;
999 - }
962 + metaqueue_chart_labels(st);
963 }
964
965
database/rrdset.c
+8 -8
@@ -173,10 +173,6 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
173 }
174 }
175
176 - if (find_chart_uuid(host, string2str(st->parts.type), string2str(st->parts.id), string2str(st->parts.name), &st->chart_uuid))
177 - uuid_generate(st->chart_uuid);
178 - update_chart_metadata(&st->chart_uuid, st, string2str(st->parts.id), string2str(st->parts.name));
179 -
176 // initialize the db tiers
177 {
178 RRD_MEMORY_MODE wanted_mode = ctr->memory_mode;
@@ -392,10 +388,14 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
388 rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
389 }
390
395 - if(ctr->react_action & (RRDSET_REACT_UPDATED | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
396 - debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st));
397 - if(unlikely(update_chart_metadata(&st->chart_uuid, st, ctr->id, ctr->name)))
398 - error_report("Failed to update chart metadata in the database");
391 + if(ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
392 + if (ctr->react_action & RRDSET_REACT_NEW) {
393 + if (find_chart_uuid(host, string2str(st->parts.type), string2str(st->parts.id), &st->chart_uuid)) {
394 + uuid_generate(st->chart_uuid);
395 + }
396 + }
397 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
398 + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
399 }
400
401 rrdcontext_updated_rrdset(st);
database/sqlite/sqlite_aclk.c
-42
@@ -5,28 +5,12 @@
5
6 #include "sqlite_aclk_node.h"
7
8 -#ifdef ENABLE_ACLK
9 -#include "../../aclk/aclk.h"
10 -#endif
11 -
8 void sanity_check(void) {
9 // make sure the compiler will stop on misconfigurations
10 BUILD_BUG_ON(WORKER_UTILIZATION_MAX_JOB_TYPES < ACLK_MAX_ENUMERATIONS_DEFINED);
11 }
12
13 const char *aclk_sync_config[] = {
18 - "CREATE TABLE IF NOT EXISTS dimension_delete (dimension_id blob, dimension_name text, chart_type_id text, "
19 - "dim_id blob, chart_id blob, host_id blob, date_created);",
20 -
21 - "CREATE INDEX IF NOT EXISTS ind_h1 ON dimension_delete (host_id);",
22 -
23 - "CREATE TRIGGER IF NOT EXISTS tr_dim_del AFTER DELETE ON dimension BEGIN INSERT INTO dimension_delete "
24 - "(dimension_id, dimension_name, chart_type_id, dim_id, chart_id, host_id, date_created)"
25 - " select old.id, old.name, c.type||\".\"||c.id, old.dim_id, old.chart_id, c.host_id, unixepoch() FROM"
26 - " chart c WHERE c.chart_id = old.chart_id; END;",
27 -
28 - "DELETE FROM dimension_delete WHERE host_id NOT IN"
29 - " (SELECT host_id FROM host) OR unixepoch() - date_created > 604800;",
14
15 NULL,
16 };
@@ -174,25 +158,6 @@ struct aclk_database_cmd aclk_database_deq_cmd(struct aclk_database_worker_confi
158 return ret;
159 }
160
177 -int aclk_worker_enq_cmd(char *node_id, struct aclk_database_cmd *cmd)
178 -{
179 - if (unlikely(!node_id || !cmd))
180 - return 0;
181 -
182 - uv_mutex_lock(&aclk_async_lock);
183 - struct aclk_database_worker_config *wc = aclk_thread_head;
184 -
185 - while (wc) {
186 - if (!strcmp(wc->node_id, node_id))
187 - break;
188 - wc = wc->next;
189 - }
190 - uv_mutex_unlock(&aclk_async_lock);
191 - if (wc)
192 - aclk_database_enq_cmd(wc, cmd);
193 - return (wc == NULL);
194 -}
195 -
161 struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id)
162 {
163 if (unlikely(!node_id))
@@ -466,9 +431,7 @@ void aclk_database_worker(void *arg)
431 timer_req.data = wc;
432 fatal_assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
433
469 -// wc->retry_count = 0;
434 wc->node_info_send = 1;
471 -// aclk_add_worker_thread(wc);
435 info("Starting ACLK sync thread for host %s -- scratch area %lu bytes", wc->host_guid, (unsigned long int) sizeof(*wc));
436
437 memset(&cmd, 0, sizeof(cmd));
@@ -688,9 +651,7 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
651 wc->host = host;
652 strcpy(wc->uuid_str, uuid_str);
653 strcpy(wc->host_guid, host_guid);
691 -// wc->chart_updates = 0;
654 wc->alert_updates = 0;
693 -// wc->retry_count = 0;
655 aclk_database_init_cmd_queue(wc);
656 aclk_add_worker_thread(wc);
657 fatal_assert(0 == uv_thread_create(&(wc->thread), aclk_database_worker, wc));
@@ -836,8 +797,5 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc)
797 error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
798 sqlite3_free(err_msg);
799 }
839 - db_execute("DELETE FROM dimension_delete WHERE host_id NOT IN (SELECT host_id FROM host) "
840 - " OR unixepoch() - date_created > 604800;");
841 -
800 return;
801 }
database/sqlite/sqlite_aclk.h
-23
@@ -66,19 +66,6 @@ static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)
66 out[23] = '_';
67 }
68
69 -static inline char *get_str_from_uuid(uuid_t *uuid)
70 -{
71 - char uuid_str[GUID_LEN + 1];
72 - if (unlikely(!uuid)) {
73 - uuid_t zero_uuid;
74 - uuid_clear(zero_uuid);
75 - uuid_unparse_lower(zero_uuid, uuid_str);
76 - }
77 - else
78 - uuid_unparse_lower(*uuid, uuid_str);
79 - return strdupz(uuid_str);
80 -}
81 -
69 #define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \
70 "alert_unique_id, date_created, date_submitted, date_cloud_ack, " \
71 "unique(alert_unique_id));"
@@ -104,20 +91,11 @@ enum aclk_database_opcode {
91 ACLK_MAX_ENUMERATIONS_DEFINED
92 };
93
107 -struct aclk_chart_payload_t {
108 - long sequence_id;
109 - long last_sequence_id;
110 - char *payload;
111 - struct aclk_chart_payload_t *next;
112 -};
113 -
114 -
94 struct aclk_database_cmd {
95 enum aclk_database_opcode opcode;
96 void *data;
97 void *data_param;
98 int count;
120 - uint64_t param1;
99 struct aclk_completion *completion;
100 };
101
@@ -187,7 +165,6 @@ extern sqlite3 *db_meta;
165 int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
166 void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
167 void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
190 -int aclk_worker_enq_cmd(char *node_id, struct aclk_database_cmd *cmd);
168 void sql_aclk_sync_init(void);
169 void sql_check_aclk_table_list(struct aclk_database_worker_config *wc);
170 void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
database/sqlite/sqlite_db_migration.c
+17
@@ -73,6 +73,13 @@ const char *database_migrate_v4_v5[] = {
73 NULL
74 };
75
76 +const char *database_migrate_v5_v6[] = {
77 + "DROP TRIGGER IF EXISTS tr_dim_del;",
78 + "DROP TABLE IF EXISTS dimension_delete;",
79 + NULL
80 +};
81 +
82 +
83 static int do_migration_v1_v2(sqlite3 *database, const char *name)
84 {
85 UNUSED(name);
@@ -133,6 +140,15 @@ static int do_migration_v4_v5(sqlite3 *database, const char *name)
140 return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v4_v5[0]);
141 }
142
143 +static int do_migration_v5_v6(sqlite3 *database, const char *name)
144 +{
145 + UNUSED(name);
146 + info("Running \"%s\" database migration", name);
147 +
148 + return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v5_v6[0]);
149 +}
150 +
151 +
152 static int do_migration_noop(sqlite3 *database, const char *name)
153 {
154 UNUSED(database);
@@ -181,6 +197,7 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = {
197 {.name = "v2 to v3", .func = do_migration_v2_v3},
198 {.name = "v3 to v4", .func = do_migration_v3_v4},
199 {.name = "v4 to v5", .func = do_migration_v4_v5},
200 + {.name = "v5 to v6", .func = do_migration_v5_v6},
201 // the terminator of this array
202 {.name = NULL, .func = NULL}
203 };
database/sqlite/sqlite_functions.c
+70 -815
@@ -3,7 +3,7 @@
3 #include "sqlite_functions.h"
4 #include "sqlite_db_migration.h"
5
6 -#define DB_METADATA_VERSION 5
6 +#define DB_METADATA_VERSION 6
7
8 const char *database_config[] = {
9 "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "
@@ -54,6 +54,7 @@ const char *database_cleanup[] = {
54 "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);",
55 "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);",
56 "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host);",
57 + "DROP TRIGGER IF EXISTS tr_dim_del;",
58 NULL
59 };
60
@@ -62,9 +63,6 @@ sqlite3 *db_meta = NULL;
63 #define MAX_PREPARED_STATEMENTS (32)
64 pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];
65
65 -static uv_mutex_t sqlite_transaction_lock;
66 -
67 -
66 SQLITE_API int sqlite3_exec_monitored(
67 sqlite3 *db, /* An open database */
68 const char *sql, /* SQL to be evaluated */
@@ -78,13 +76,28 @@ SQLITE_API int sqlite3_exec_monitored(
76 }
77
78 SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) {
81 - int rc = sqlite3_step(stmt);
82 -
83 - if(likely(rc == SQLITE_ROW))
84 - sqlite3_row_completed();
85 - else
86 - sqlite3_query_completed(rc == SQLITE_DONE, rc == SQLITE_BUSY, rc == SQLITE_LOCKED);
79 + int rc;
80 + int cnt = 0;
81
82 + while (cnt++ < SQL_MAX_RETRY) {
83 + rc = sqlite3_step(stmt);
84 + switch (rc) {
85 + case SQLITE_DONE:
86 + sqlite3_query_completed(1, 0, 0);
87 + break;
88 + case SQLITE_ROW:
89 + sqlite3_row_completed();
90 + break;
91 + case SQLITE_BUSY:
92 + case SQLITE_LOCKED:
93 + sqlite3_query_completed(rc == SQLITE_DONE, rc == SQLITE_BUSY, rc == SQLITE_LOCKED);
94 + usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
95 + continue;
96 + default:
97 + break;
98 + }
99 + break;
100 + }
101 return rc;
102 }
103
@@ -141,7 +154,7 @@ static void release_statement(void *statement)
154 error_report("Failed to finalize statement, rc = %d", rc);
155 }
156
144 -int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement)
157 +int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement)
158 {
159 static __thread uint32_t keys_used = 0;
160
@@ -231,7 +244,6 @@ static void rebuild_chart()
244 sqlite3_free(err_msg);
245 }
246 }
234 - return;
247 }
248
249 const char *rebuild_dimension_commands[] = {
@@ -264,7 +276,6 @@ void rebuild_dimension()
276 sqlite3_free(err_msg);
277 }
278 }
267 - return;
279 }
280
281 static int attempt_database_fix()
@@ -302,7 +313,7 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha
313 return 0;
314 }
315
305 -static void _uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)
316 +static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)
317 {
318 uuid_t uuid;
319
@@ -433,13 +444,12 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
444 if (init_database_batch(db_meta, rebuild, 0, &database_cleanup[0]))
445 return 1;
446
436 - fatal_assert(0 == uv_mutex_init(&sqlite_transaction_lock));
447 info("SQLite database initialization completed");
448
449 for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++)
450 (void)pthread_key_create(&key_pool[i], release_statement);
451
442 - rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, _uuid_parse, 0, 0);
452 + rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);
453 if (unlikely(rc != SQLITE_OK))
454 error_report("Failed to register internal u2h function");
455 return 0;
@@ -462,43 +472,30 @@ void sql_close_database(void)
472 rc = sqlite3_close_v2(db_meta);
473 if (unlikely(rc != SQLITE_OK))
474 error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));
465 - return;
475 }
476
468 -#define FIND_UUID_TYPE "select 1 from host where host_id = @uuid union select 2 from chart where chart_id = @uuid union select 3 from dimension where dim_id = @uuid;"
477
470 -int find_uuid_type(uuid_t *uuid)
478 +// Expect a UUID in column 0
479 +// Return 0 and store in store_uuid
480 +// Return 1 if it fails (store_uuid is left unchanged)
481 +static int expect_uuid_column0(sqlite3_stmt *res, uuid_t *store_uuid)
482 {
472 - static __thread sqlite3_stmt *res = NULL;
473 - int rc;
474 - int uuid_type = 3;
475 -
476 - if (unlikely(!res)) {
477 - rc = prepare_statement(db_meta, FIND_UUID_TYPE, &res);
478 - if (rc != SQLITE_OK) {
479 - error_report("Failed to bind prepare statement to find UUID type in the database");
480 - return 0;
481 - }
483 + int rc = sqlite3_step_monitored(res);
484 + if (likely(rc == SQLITE_ROW)) {
485 + uuid_copy(*store_uuid, *((uuid_t *)sqlite3_column_blob(res, 0)));
486 + return 0;
487 }
488 + return 1;
489 +}
490
484 - rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
485 - if (unlikely(rc != SQLITE_OK))
486 - goto bind_fail;
487 -
488 - rc = sqlite3_step_monitored(res);
489 - if (likely(rc == SQLITE_ROW))
490 - uuid_type = sqlite3_column_int(res, 0);
491 -
492 - rc = sqlite3_reset(res);
493 - if (unlikely(rc != SQLITE_OK))
494 - error_report("Failed to reset statement during find uuid type, rc = %d", rc);
495 -
496 - return uuid_type;
491
498 -bind_fail:
499 - return 0;
500 -}
492 +// Do a database lookup to find the uuid of a dimension
493 +// Return 0 if uuid is found and store it in *store_uuid
494 +// 1 if no uuid is found (needs to be created)
495 +//
496
497 +#define SQL_FIND_DIMENSION_UUID \
498 + "SELECT dim_id FROM dimension WHERE chart_id=@chart AND id=@id AND name=@name AND LENGTH(dim_id)=16;"
499 int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
500 {
501 static __thread sqlite3_stmt *res = NULL;
@@ -511,86 +508,41 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
508 if (unlikely(!res)) {
509 rc = prepare_statement(db_meta, SQL_FIND_DIMENSION_UUID, &res);
510 if (rc != SQLITE_OK) {
514 - error_report("Failed to bind prepare statement to lookup dimension UUID in the database");
511 + error_report("Failed to prepare statement to lookup dimension UUID in the database");
512 return 1;
513 }
514 }
515
516 rc = sqlite3_bind_blob(res, 1, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC);
517 if (unlikely(rc != SQLITE_OK))
521 - goto bind_fail;
518 + goto skip;
519
520 rc = sqlite3_bind_text(res, 2, rrddim_id(rd), -1, SQLITE_STATIC);
521 if (unlikely(rc != SQLITE_OK))
525 - goto bind_fail;
522 + goto skip;
523
524 rc = sqlite3_bind_text(res, 3, rrddim_name(rd), -1, SQLITE_STATIC);
525 if (unlikely(rc != SQLITE_OK))
529 - goto bind_fail;
526 + goto skip;
527
531 - rc = sqlite3_step_monitored(res);
532 - if (likely(rc == SQLITE_ROW)) {
533 - uuid_copy(*store_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
534 - status = 0;
535 - }
536 - else {
537 - uuid_generate(*store_uuid);
538 - status = sql_store_dimension(store_uuid, &st->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
539 - if (unlikely(status))
540 - error_report("Failed to store dimension metadata in the database");
541 - }
528 + status = expect_uuid_column0(res, store_uuid);
529
530 +skip:
531 rc = sqlite3_reset(res);
532 if (unlikely(rc != SQLITE_OK))
533 error_report("Failed to reset statement find dimension uuid, rc = %d", rc);
534 return status;
547 -
548 -bind_fail:
549 - error_report("Failed to bind input parameter to perform dimension UUID database lookup, rc = %d", rc);
550 - return 1;
535 }
536
553 -#define DELETE_DIMENSION_UUID "delete from dimension where dim_id = @uuid;"
554 -
555 -void delete_dimension_uuid(uuid_t *dimension_uuid)
556 -{
557 - static __thread sqlite3_stmt *res = NULL;
558 - int rc;
559 -
560 -#ifdef NETDATA_INTERNAL_CHECKS
561 - char uuid_str[GUID_LEN + 1];
562 - uuid_unparse_lower(*dimension_uuid, uuid_str);
563 - debug(D_METADATALOG,"Deleting dimension uuid %s", uuid_str);
564 -#endif
565 -
566 - if (unlikely(!res)) {
567 - rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res);
568 - if (rc != SQLITE_OK) {
569 - error_report("Failed to prepare statement to delete a dimension uuid");
570 - return;
571 - }
572 - }
573 -
574 - rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);
575 - if (unlikely(rc != SQLITE_OK))
576 - goto bind_fail;
577 -
578 - rc = sqlite3_step_monitored(res);
579 - if (unlikely(rc != SQLITE_DONE))
580 - error_report("Failed to delete dimension uuid, rc = %d", rc);
581 -
582 -bind_fail:
583 - rc = sqlite3_reset(res);
584 - if (unlikely(rc != SQLITE_OK))
585 - error_report("Failed to reset statement when deleting dimension UUID, rc = %d", rc);
586 - return;
587 -}
537
538 /*
539 * Do a database lookup to find the UUID of a chart
540 *
541 */
593 -int find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char *name, uuid_t *store_uuid)
542 +
543 +#define SQL_FIND_CHART_UUID "SELECT chart_id FROM chart WHERE host_id = @host AND type=@type AND id=@id AND chart_id IS NOT NULL;"
544 +
545 +int find_chart_uuid(RRDHOST *host, const char *type, const char *id, uuid_t *store_uuid)
546 {
547 static __thread sqlite3_stmt *res = NULL;
548 int rc;
@@ -609,59 +561,26 @@ int find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char
561
562 rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
563 if (unlikely(rc != SQLITE_OK))
612 - goto bind_fail;
564 + goto skip;
565
566 rc = sqlite3_bind_text(res, 2, type, -1, SQLITE_STATIC);
567 if (unlikely(rc != SQLITE_OK))
616 - goto bind_fail;
568 + goto skip;
569
570 rc = sqlite3_bind_text(res, 3, id, -1, SQLITE_STATIC);
571 if (unlikely(rc != SQLITE_OK))
620 - goto bind_fail;
572 + goto skip;
573
622 - rc = sqlite3_bind_text(res, 4, name ? name : id, -1, SQLITE_STATIC);
623 - if (unlikely(rc != SQLITE_OK))
624 - goto bind_fail;
625 -
626 - rc = sqlite3_step_monitored(res);
627 - if (likely(rc == SQLITE_ROW && sqlite3_column_bytes(res,0) == sizeof(*store_uuid))) {
628 - uuid_copy(*store_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
629 - status = 0;
630 - }
574 + status = expect_uuid_column0(res, store_uuid);
575
576 +skip:
577 rc = sqlite3_reset(res);
578 if (unlikely(rc != SQLITE_OK))
579 error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc);
635 -
580 return status;
637 -
638 -bind_fail:
639 - error_report("Failed to bind input parameter to perform chart UUID database lookup, rc = %d", rc);
640 - rc = sqlite3_reset(res);
641 - if (unlikely(rc != SQLITE_OK))
642 - error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc);
643 - return 1;
644 -}
645 -
646 -int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const char *name)
647 -{
648 - int rc;
649 -
650 - if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
651 - return 0;
652 -
653 - rc = sql_store_chart(
654 - chart_uuid, &st->rrdhost->host_uuid,
655 - rrdset_parts_type(st), id, name,
656 - rrdset_family(st), rrdset_context(st), rrdset_title(st), rrdset_units(st),
657 - rrdset_plugin_name(st), rrdset_module_name(st),
658 - st->priority, st->update_every, st->chart_type,
659 - st->rrd_memory_mode, st->entries);
660 -
661 - return rc;
581 }
582
664 -static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
583 +int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
584 {
585 int rc, result = 1;
586 sqlite3_stmt *res = NULL;
@@ -675,7 +594,7 @@ static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
594 rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
595 if (unlikely(rc != SQLITE_OK)) {
596 error_report("Failed to bind host parameter to %s, rc = %d", sql, rc);
678 - goto failed;
597 + goto skip;
598 }
599
600 rc = execute_insert(res);
@@ -684,7 +603,7 @@ static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
603 else
604 error_report("Failed to execute %s, rc = %d", sql, rc);
605
687 -failed:
606 +skip:
607 rc = sqlite3_finalize(res);
608 if (unlikely(rc != SQLITE_OK))
609 error_report("Failed to finalize statement %s, rc = %d", sql, rc);
@@ -692,445 +611,13 @@ failed:
611 }
612
613
695 -// Migrate all hosts with hops zero to this host_uuid
696 -void migrate_localhost(uuid_t *host_uuid)
697 -{
698 - int rc;
699 -
700 - rc = exec_statement_with_uuid("UPDATE chart SET host_id = @host_id WHERE host_id in (SELECT host_id FROM host where host_id <> @host_id and hops = 0); ", host_uuid);
701 - if (!rc)
702 - rc = exec_statement_with_uuid("DELETE FROM host WHERE hops = 0 AND host_id <> @host_id; ", host_uuid);
703 - if (!rc)
704 - db_execute("DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);");
705 -
706 -}
707 -
708 -int sql_store_host(
709 - uuid_t *host_uuid, const char *hostname, const char *registry_hostname, int update_every, const char *os,
710 - const char *tzone, const char *tags, int hops)
711 -{
712 - static __thread sqlite3_stmt *res = NULL;
713 - int rc;
714 -
715 - if (unlikely(!db_meta)) {
716 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
717 - return 0;
718 - error_report("Database has not been initialized");
719 - return 1;
720 - }
721 -
722 - if (unlikely((!res))) {
723 - rc = prepare_statement(db_meta, SQL_STORE_HOST, &res);
724 - if (unlikely(rc != SQLITE_OK)) {
725 - error_report("Failed to prepare statement to store host, rc = %d", rc);
726 - return 1;
727 - }
728 - }
729 -
730 - rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
731 - if (unlikely(rc != SQLITE_OK))
732 - goto bind_fail;
733 -
734 - rc = sqlite3_bind_text(res, 2, hostname, -1, SQLITE_STATIC);
735 - if (unlikely(rc != SQLITE_OK))
736 - goto bind_fail;
737 -
738 - rc = sqlite3_bind_text(res, 3, registry_hostname, -1, SQLITE_STATIC);
739 - if (unlikely(rc != SQLITE_OK))
740 - goto bind_fail;
741 -
742 - rc = sqlite3_bind_int(res, 4, update_every);
743 - if (unlikely(rc != SQLITE_OK))
744 - goto bind_fail;
745 -
746 - rc = sqlite3_bind_text(res, 5, os, -1, SQLITE_STATIC);
747 - if (unlikely(rc != SQLITE_OK))
748 - goto bind_fail;
749 -
750 - rc = sqlite3_bind_text(res, 6, tzone, -1, SQLITE_STATIC);
751 - if (unlikely(rc != SQLITE_OK))
752 - goto bind_fail;
753 -
754 - rc = sqlite3_bind_text(res, 7, tags, -1, SQLITE_STATIC);
755 - if (unlikely(rc != SQLITE_OK))
756 - goto bind_fail;
757 -
758 - rc = sqlite3_bind_int(res, 8, hops);
759 - if (unlikely(rc != SQLITE_OK))
760 - goto bind_fail;
761 -
762 - int store_rc = sqlite3_step_monitored(res);
763 - if (unlikely(store_rc != SQLITE_DONE))
764 - error_report("Failed to store host %s, rc = %d", hostname, rc);
765 -
766 - rc = sqlite3_reset(res);
767 - if (unlikely(rc != SQLITE_OK))
768 - error_report("Failed to reset statement to store host %s, rc = %d", hostname, rc);
769 -
770 - return !(store_rc == SQLITE_DONE);
771 -bind_fail:
772 - error_report("Failed to bind parameter to store host %s, rc = %d", hostname, rc);
773 - rc = sqlite3_reset(res);
774 - if (unlikely(rc != SQLITE_OK))
775 - error_report("Failed to reset statement to store host %s, rc = %d", hostname, rc);
776 - return 1;
777 -}
614
615 //
780 -// Store host and host system info information in the database
781 -#define SQL_STORE_HOST_INFO "INSERT OR REPLACE INTO host " \
782 - "(host_id, hostname, registry_hostname, update_every, os, timezone," \
783 - "tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, program_version," \
784 - "entries, health_enabled) " \
785 - "values (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, @memory_mode, " \
786 - "@abbrev_timezone, @utc_offset, @program_name, @program_version, " \
787 - "@entries, @health_enabled);"
788 -
789 -int sql_store_host_info(RRDHOST *host)
790 -{
791 - static __thread sqlite3_stmt *res = NULL;
792 - int rc;
793 -
794 - if (unlikely(!db_meta)) {
795 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
796 - return 0;
797 - error_report("Database has not been initialized");
798 - return 1;
799 - }
800 -
801 - if (unlikely((!res))) {
802 - rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res);
803 - if (unlikely(rc != SQLITE_OK)) {
804 - error_report("Failed to prepare statement to store host, rc = %d", rc);
805 - return 1;
806 - }
807 - }
808 -
809 - rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
810 - if (unlikely(rc != SQLITE_OK))
811 - goto bind_fail;
812 -
813 - rc = bind_text_null(res, 2, rrdhost_hostname(host), 0);
814 - if (unlikely(rc != SQLITE_OK))
815 - goto bind_fail;
816 -
817 - rc = bind_text_null(res, 3, rrdhost_registry_hostname(host), 1);
818 - if (unlikely(rc != SQLITE_OK))
819 - goto bind_fail;
820 -
821 - rc = sqlite3_bind_int(res, 4, host->rrd_update_every);
822 - if (unlikely(rc != SQLITE_OK))
823 - goto bind_fail;
824 -
825 - rc = bind_text_null(res, 5, rrdhost_os(host), 1);
826 - if (unlikely(rc != SQLITE_OK))
827 - goto bind_fail;
828 -
829 - rc = bind_text_null(res, 6, rrdhost_timezone(host), 1);
830 - if (unlikely(rc != SQLITE_OK))
831 - goto bind_fail;
832 -
833 - rc = bind_text_null(res, 7, rrdhost_tags(host), 1);
834 - if (unlikely(rc != SQLITE_OK))
835 - goto bind_fail;
836 -
837 - rc = sqlite3_bind_int(res, 8, host->system_info ? host->system_info->hops : 0);
838 - if (unlikely(rc != SQLITE_OK))
839 - goto bind_fail;
840 -
841 - rc = sqlite3_bind_int(res, 9, host->rrd_memory_mode);
842 - if (unlikely(rc != SQLITE_OK))
843 - goto bind_fail;
844 -
845 - rc = bind_text_null(res, 10, rrdhost_abbrev_timezone(host), 1);
846 - if (unlikely(rc != SQLITE_OK))
847 - goto bind_fail;
848 -
849 - rc = sqlite3_bind_int(res, 11, host->utc_offset);
850 - if (unlikely(rc != SQLITE_OK))
851 - goto bind_fail;
852 -
853 - rc = bind_text_null(res, 12, rrdhost_program_name(host), 1);
854 - if (unlikely(rc != SQLITE_OK))
855 - goto bind_fail;
856 -
857 - rc = bind_text_null(res, 13, rrdhost_program_version(host), 1);
858 - if (unlikely(rc != SQLITE_OK))
859 - goto bind_fail;
860 -
861 - rc = sqlite3_bind_int64(res, 14, host->rrd_history_entries);
862 - if (unlikely(rc != SQLITE_OK))
863 - goto bind_fail;
864 -
865 - rc = sqlite3_bind_int(res, 15, host->health_enabled);
866 - if (unlikely(rc != SQLITE_OK))
867 - goto bind_fail;
868 -
869 - int store_rc = sqlite3_step_monitored(res);
870 - if (unlikely(store_rc != SQLITE_DONE))
871 - error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc);
872 -
873 - rc = sqlite3_reset(res);
874 - if (unlikely(rc != SQLITE_OK))
875 - error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
876 -
877 - return !(store_rc == SQLITE_DONE);
878 -bind_fail:
879 - error_report("Failed to bind parameter to store host %s, rc = %d", rrdhost_hostname(host), rc);
880 - rc = sqlite3_reset(res);
881 - if (unlikely(rc != SQLITE_OK))
882 - error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
883 - return 1;
884 -}
885 -
886 -/*
887 - * Store a chart in the database
888 - */
889 -
890 -int sql_store_chart(
891 - uuid_t *chart_uuid, uuid_t *host_uuid, const char *type, const char *id, const char *name, const char *family,
892 - const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
893 - int update_every, int chart_type, int memory_mode, long history_entries)
894 -{
895 - static __thread sqlite3_stmt *res = NULL;
896 - int rc, param = 0;
897 -
898 - if (unlikely(!db_meta)) {
899 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
900 - return 0;
901 - error_report("Database has not been initialized");
902 - return 1;
903 - }
904 -
905 - if (unlikely(!res)) {
906 - rc = prepare_statement(db_meta, SQL_STORE_CHART, &res);
907 - if (unlikely(rc != SQLITE_OK)) {
908 - error_report("Failed to prepare statement to store chart, rc = %d", rc);
909 - return 1;
910 - }
911 - }
912 -
913 - param++;
914 - rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
915 - if (unlikely(rc != SQLITE_OK))
916 - goto bind_fail;
917 -
918 - param++;
919 - rc = sqlite3_bind_blob(res, 2, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
920 - if (unlikely(rc != SQLITE_OK))
921 - goto bind_fail;
922 -
923 - param++;
924 - rc = sqlite3_bind_text(res, 3, type, -1, SQLITE_STATIC);
925 - if (unlikely(rc != SQLITE_OK))
926 - goto bind_fail;
927 -
928 - param++;
929 - rc = sqlite3_bind_text(res, 4, id, -1, SQLITE_STATIC);
930 - if (unlikely(rc != SQLITE_OK))
931 - goto bind_fail;
932 -
933 - param++;
934 - if (name && *name)
935 - rc = sqlite3_bind_text(res, 5, name, -1, SQLITE_STATIC);
936 - else
937 - rc = sqlite3_bind_null(res, 5);
938 - if (unlikely(rc != SQLITE_OK))
939 - goto bind_fail;
940 -
941 - param++;
942 - rc = sqlite3_bind_text(res, 6, family, -1, SQLITE_STATIC);
943 - if (unlikely(rc != SQLITE_OK))
944 - goto bind_fail;
945 -
946 - param++;
947 - rc = sqlite3_bind_text(res, 7, context, -1, SQLITE_STATIC);
948 - if (unlikely(rc != SQLITE_OK))
949 - goto bind_fail;
950 -
951 - param++;
952 - rc = sqlite3_bind_text(res, 8, title, -1, SQLITE_STATIC);
953 - if (unlikely(rc != SQLITE_OK))
954 - goto bind_fail;
955 -
956 - param++;
957 - rc = sqlite3_bind_text(res, 9, units, -1, SQLITE_STATIC);
958 - if (unlikely(rc != SQLITE_OK))
959 - goto bind_fail;
960 -
961 - param++;
962 - rc = sqlite3_bind_text(res, 10, plugin, -1, SQLITE_STATIC);
963 - if (unlikely(rc != SQLITE_OK))
964 - goto bind_fail;
965 -
966 - param++;
967 - rc = sqlite3_bind_text(res, 11, module, -1, SQLITE_STATIC);
968 - if (unlikely(rc != SQLITE_OK))
969 - goto bind_fail;
970 -
971 - param++;
972 - rc = sqlite3_bind_int(res, 12, priority);
973 - if (unlikely(rc != SQLITE_OK))
974 - goto bind_fail;
975 -
976 - param++;
977 - rc = sqlite3_bind_int(res, 13, update_every);
978 - if (unlikely(rc != SQLITE_OK))
979 - goto bind_fail;
980 -
981 - param++;
982 - rc = sqlite3_bind_int(res, 14, chart_type);
983 - if (unlikely(rc != SQLITE_OK))
984 - goto bind_fail;
985 -
986 - param++;
987 - rc = sqlite3_bind_int(res, 15, memory_mode);
988 - if (unlikely(rc != SQLITE_OK))
989 - goto bind_fail;
990 -
991 - param++;
992 - rc = sqlite3_bind_int(res, 16, history_entries);
993 - if (unlikely(rc != SQLITE_OK))
994 - goto bind_fail;
995 -
996 - rc = execute_insert(res);
997 - if (unlikely(rc != SQLITE_DONE))
998 - error_report("Failed to store chart, rc = %d", rc);
999 -
1000 - rc = sqlite3_reset(res);
1001 - if (unlikely(rc != SQLITE_OK))
1002 - error_report("Failed to reset statement in chart store function, rc = %d", rc);
1003 -
1004 - return 0;
1005 -
1006 -bind_fail:
1007 - error_report("Failed to bind parameter %d to store chart, rc = %d", param, rc);
1008 - rc = sqlite3_reset(res);
1009 - if (unlikely(rc != SQLITE_OK))
1010 - error_report("Failed to reset statement in chart store function, rc = %d", rc);
1011 - return 1;
1012 -}
1013 -
1014 -/*
1015 - * Store a dimension
1016 - */
1017 -int sql_store_dimension(
1018 - uuid_t *dim_uuid, uuid_t *chart_uuid, const char *id, const char *name, collected_number multiplier,
1019 - collected_number divisor, int algorithm)
1020 -{
1021 - static __thread sqlite3_stmt *res = NULL;
1022 - int rc;
1023 -
1024 - if (unlikely(!db_meta)) {
1025 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1026 - return 0;
1027 - error_report("Database has not been initialized");
1028 - return 1;
1029 - }
1030 -
1031 - if (unlikely(!res)) {
1032 - rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res);
1033 - if (unlikely(rc != SQLITE_OK)) {
1034 - error_report("Failed to prepare statement to store dimension, rc = %d", rc);
1035 - return 1;
1036 - }
1037 - }
1038 -
1039 - rc = sqlite3_bind_blob(res, 1, dim_uuid, sizeof(*dim_uuid), SQLITE_STATIC);
1040 - if (unlikely(rc != SQLITE_OK))
1041 - goto bind_fail;
1042 -
1043 - rc = sqlite3_bind_blob(res, 2, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
1044 - if (unlikely(rc != SQLITE_OK))
1045 - goto bind_fail;
1046 -
1047 - rc = sqlite3_bind_text(res, 3, id, -1, SQLITE_STATIC);
1048 - if (unlikely(rc != SQLITE_OK))
1049 - goto bind_fail;
1050 -
1051 - rc = sqlite3_bind_text(res, 4, name, -1, SQLITE_STATIC);
1052 - if (unlikely(rc != SQLITE_OK))
1053 - goto bind_fail;
1054 -
1055 - rc = sqlite3_bind_int(res, 5, multiplier);
1056 - if (unlikely(rc != SQLITE_OK))
1057 - goto bind_fail;
1058 -
1059 - rc = sqlite3_bind_int(res, 6, divisor);
1060 - if (unlikely(rc != SQLITE_OK))
1061 - goto bind_fail;
1062 -
1063 - rc = sqlite3_bind_int(res, 7, algorithm);
1064 - if (unlikely(rc != SQLITE_OK))
1065 - goto bind_fail;
1066 -
1067 - rc = execute_insert(res);
1068 - if (unlikely(rc != SQLITE_DONE))
1069 - error_report("Failed to store dimension, rc = %d", rc);
1070 -
1071 - rc = sqlite3_reset(res);
1072 - if (unlikely(rc != SQLITE_OK))
1073 - error_report("Failed to reset statement in store dimension, rc = %d", rc);
1074 - return 0;
1075 -
1076 -bind_fail:
1077 - error_report("Failed to bind parameter to store dimension, rc = %d", rc);
1078 - rc = sqlite3_reset(res);
1079 - if (unlikely(rc != SQLITE_OK))
1080 - error_report("Failed to reset statement in store dimension, rc = %d", rc);
1081 - return 1;
1082 -}
1083 -
1084 -/*
1085 - * Store set option for a dimension
1086 - */
1087 -int sql_set_dimension_option(uuid_t *dim_uuid, char *option)
1088 -{
1089 - sqlite3_stmt *res = NULL;
1090 - int rc;
1091 -
1092 - if (unlikely(!db_meta)) {
1093 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1094 - return 0;
1095 - error_report("Database has not been initialized");
1096 - return 1;
1097 - }
1098 -
1099 - rc = sqlite3_prepare_v2(db_meta, "UPDATE dimension SET options = @options WHERE dim_id = @dim_id", -1, &res, 0);
1100 - if (unlikely(rc != SQLITE_OK)) {
1101 - error_report("Failed to prepare statement to update dimension options");
1102 - return 0;
1103 - };
1104 -
1105 - rc = sqlite3_bind_blob(res, 2, dim_uuid, sizeof(*dim_uuid), SQLITE_STATIC);
1106 - if (unlikely(rc != SQLITE_OK))
1107 - goto bind_fail;
1108 -
1109 - if (!option || !strcmp(option,"unhide"))
1110 - rc = sqlite3_bind_null(res, 1);
1111 - else
1112 - rc = sqlite3_bind_text(res, 1, option, -1, SQLITE_STATIC);
1113 - if (unlikely(rc != SQLITE_OK))
1114 - goto bind_fail;
1115 -
1116 - rc = execute_insert(res);
1117 - if (unlikely(rc != SQLITE_DONE))
1118 - error_report("Failed to update dimension option, rc = %d", rc);
1119 -
1120 -bind_fail:
1121 - rc = sqlite3_finalize(res);
1122 - if (unlikely(rc != SQLITE_OK))
1123 - error_report("Failed to finalize statement in update dimension options, rc = %d", rc);
1124 - return 0;
1125 -}
1126 -
1127 -
1128 -//
1129 -// Support for archived charts
616 +// Support for archived charts (TO BE REMOVED)
617 //
618 #define SELECT_DIMENSION "select d.id, d.name from dimension d where d.chart_id = @chart_uuid;"
619
1133 -void sql_rrdim2json(sqlite3_stmt *res_dim, uuid_t *chart_uuid, BUFFER *wb, size_t *dimensions_count)
620 +static void sql_rrdim2json(sqlite3_stmt *res_dim, uuid_t *chart_uuid, BUFFER *wb, size_t *dimensions_count)
621 {
622 int rc;
623
@@ -1326,8 +813,6 @@ failed:
813 rc = sqlite3_finalize(res_chart);
814 if (unlikely(rc != SQLITE_OK))
815 error_report("Failed to finalize the prepared statement when reading archived charts");
1329 -
1330 - return;
816 }
817
818 void db_execute(const char *cmd)
@@ -1353,19 +838,6 @@ void db_execute(const char *cmd)
838 }
839 }
840
1356 -void db_lock(void)
1357 -{
1358 - uv_mutex_lock(&sqlite_transaction_lock);
1359 - return;
1360 -}
1361 -
1362 -void db_unlock(void)
1363 -{
1364 - uv_mutex_unlock(&sqlite_transaction_lock);
1365 - return;
1366 -}
1367 -
1368 -
841 #define SELECT_MIGRATED_FILE "select 1 from metadata_migration where filename = @path;"
842
843 int file_is_migrated(char *path)
@@ -1413,7 +885,7 @@ void add_migrated_file(char *path, uint64_t file_size)
885 return;
886 }
887
1416 - rc = sqlite3_bind_int64(res, 2, file_size);
888 + rc = sqlite3_bind_int64(res, 2, (sqlite_int64) file_size);
889 if (unlikely(rc != SQLITE_OK)) {
890 error_report("Failed to bind size parameter to store migration information");
891 return;
@@ -1425,74 +897,9 @@ void add_migrated_file(char *path, uint64_t file_size)
897
898 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
899 error_report("Failed to finalize the prepared statement when checking if metadata file is migrated");
1428 -
1429 - return;
900 }
901
1432 -static int sql_store_label(sqlite3_stmt *res, uuid_t *uuid, int source_type, const char *label, const char *value)
1433 -{
1434 - int rc;
902
1436 - rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
1437 - if (unlikely(rc != SQLITE_OK)) {
1438 - error_report("Failed to bind UUID parameter to store label information");
1439 - goto skip_store;
1440 - }
1441 -
1442 - rc = sqlite3_bind_int(res, 2, source_type);
1443 - if (unlikely(rc != SQLITE_OK)) {
1444 - error_report("Failed to bind type parameter to store label information");
1445 - goto skip_store;
1446 - }
1447 -
1448 - rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC);
1449 - if (unlikely(rc != SQLITE_OK)) {
1450 - error_report("Failed to bind label parameter to store label information");
1451 - goto skip_store;
1452 - }
1453 -
1454 - rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC);
1455 - if (unlikely(rc != SQLITE_OK)) {
1456 - error_report("Failed to bind value parameter to store label information");
1457 - goto skip_store;
1458 - }
1459 -
1460 - rc = execute_insert(res);
1461 - if (unlikely(rc != SQLITE_DONE))
1462 - error_report("Failed to store label entry, rc = %d", rc);
1463 -
1464 -skip_store:
1465 - if (unlikely(sqlite3_reset(res) != SQLITE_OK))
1466 - error_report("Failed to reset the prepared statement when storing label information");
1467 -
1468 - return rc != SQLITE_DONE;
1469 -}
1470 -
1471 -#define SQL_INS_HOST_LABEL "INSERT OR REPLACE INTO host_label " \
1472 - "(host_id, source_type, label_key, label_value, date_created) " \
1473 - "values (@chart, @source, @label, @value, unixepoch());"
1474 -
1475 -static void sql_store_host_label(uuid_t *host_uuid, int source_type, const char *label, const char *value)
1476 -{
1477 - static __thread sqlite3_stmt *res = NULL;
1478 - int rc;
1479 -
1480 - if (unlikely(!db_meta)) {
1481 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
1482 - error_report("Database has not been initialized");
1483 - return;
1484 - }
1485 -
1486 - if (unlikely(!res)) {
1487 - rc = prepare_statement(db_meta, SQL_INS_HOST_LABEL, &res);
1488 - if (unlikely(rc != SQLITE_OK)) {
1489 - error_report("Failed to prepare statement store chart labels");
1490 - return;
1491 - }
1492 - }
1493 -
1494 - (void) sql_store_label(res, host_uuid, source_type, label, value);
1495 -}
903
904 #define SQL_STORE_CLAIM_ID "insert into node_instance " \
905 "(host_id, claim_id, date_created) values (@host_id, @claim_id, unixepoch()) " \
@@ -1562,7 +969,6 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
969 sql_create_aclk_table(host, &host->host_uuid, node_id);
970 else
971 uuid_unparse_lower(*node_id, wc->node_id);
1565 - return;
972 }
973
974 #define SQL_UPDATE_NODE_ID "update node_instance set node_id = @node_id where host_id = @host_id;"
@@ -1781,9 +1187,9 @@ failed:
1187 #define SQL_GET_NODE_INSTANCE_LIST "select ni.node_id, ni.host_id, h.hostname " \
1188 "from node_instance ni, host h where ni.host_id = h.host_id;"
1189
1784 -struct node_instance_list *get_node_list(void)
1190 +struct node_instance_list *get_node_list(void)
1191 {
1786 - struct node_instance_list *node_list = NULL;
1192 + struct node_instance_list *node_list = NULL;
1193 sqlite3_stmt *res = NULL;
1194 int rc;
1195
@@ -1811,6 +1217,7 @@ struct node_instance_list *get_node_list(void)
1217 node_list = callocz(row + 1, sizeof(*node_list));
1218 int max_rows = row;
1219 row = 0;
1220 + // TODO: Check to remove lock
1221 rrd_rdlock();
1222 while (sqlite3_step_monitored(res) == SQLITE_ROW) {
1223 if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t))
@@ -1878,8 +1285,6 @@ void sql_load_node_id(RRDHOST *host)
1285 failed:
1286 if (unlikely(sqlite3_reset(res) != SQLITE_OK))
1287 error_report("Failed to reset the prepared statement when loading node instance information");
1881 -
1882 - return;
1288 };
1289
1290
@@ -1900,7 +1305,7 @@ void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *sys
1305 rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
1306 if (unlikely(rc != SQLITE_OK)) {
1307 error_report("Failed to bind host parameter host information");
1903 - goto skip_loading;
1308 + goto skip;
1309 }
1310
1311 while (sqlite3_step_monitored(res) == SQLITE_ROW) {
@@ -1908,159 +1313,9 @@ void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *sys
1313 (char *) sqlite3_column_text(res, 1));
1314 }
1315
1911 -skip_loading:
1316 +skip:
1317 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
1318 error_report("Failed to finalize the prepared statement when reading host information");
1914 - return;
1915 -}
1916 -
1917 -
1918 -#define SQL_INS_HOST_SYSTEM_INFO "INSERT OR REPLACE INTO host_info " \
1919 - "(host_id, system_key, system_value, date_created) " \
1920 - "VALUES (@host, @key, @value, unixepoch());"
1921 -
1922 -void sql_store_host_system_info_key_value(uuid_t *host_id, const char *name, const char *value)
1923 -{
1924 - sqlite3_stmt *res = NULL;
1925 - int rc;
1926 -
1927 - if (unlikely(!db_meta)) {
1928 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
1929 - error_report("Database has not been initialized");
1930 - return;
1931 - }
1932 -
1933 - rc = sqlite3_prepare_v2(db_meta, SQL_INS_HOST_SYSTEM_INFO, -1, &res, 0);
1934 - if (unlikely(rc != SQLITE_OK)) {
1935 - error_report("Failed to prepare statement to store system info");
1936 - return;
1937 - }
1938 -
1939 - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
1940 - if (unlikely(rc != SQLITE_OK)) {
1941 - error_report("Failed to bind host parameter to store system information");
1942 - goto skip_store;
1943 - }
1944 -
1945 - rc = sqlite3_bind_text(res, 2, name, -1, SQLITE_STATIC);
1946 - if (unlikely(rc != SQLITE_OK)) {
1947 - error_report("Failed to bind label parameter to store name information");
1948 - goto skip_store;
1949 - }
1950 -
1951 - rc = sqlite3_bind_text(res, 3, value, -1, SQLITE_STATIC);
1952 - if (unlikely(rc != SQLITE_OK)) {
1953 - error_report("Failed to bind value parameter to store value information");
1954 - goto skip_store;
1955 - }
1956 -
1957 - rc = execute_insert(res);
1958 - if (unlikely(rc != SQLITE_DONE))
1959 - error_report("Failed to store host system info, rc = %d", rc);
1960 -
1961 -skip_store:
1962 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
1963 - error_report("Failed to finalize the prepared statement when storing host system information");
1964 -
1965 - return;
1966 -}
1967 -
1968 -
1969 -void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_info *system_info)
1970 -{
1971 - if (unlikely(!system_info))
1972 - return;
1973 -
1974 - if (system_info->container_os_name)
1975 - sql_store_host_system_info_key_value(host_id, "NETDATA_CONTAINER_OS_NAME", system_info->container_os_name);
1976 -
1977 - if (system_info->container_os_id)
1978 - sql_store_host_system_info_key_value(host_id, "NETDATA_CONTAINER_OS_ID", system_info->container_os_id);
1979 -
1980 - if (system_info->container_os_id_like)
1981 - sql_store_host_system_info_key_value(host_id, "NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like);
1982 -
1983 - if (system_info->container_os_version)
1984 - sql_store_host_system_info_key_value(host_id, "NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version);
1985 -
1986 - if (system_info->container_os_version_id)
1987 - sql_store_host_system_info_key_value(host_id, "NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id);
1988 -
1989 - if (system_info->host_os_detection)
1990 - sql_store_host_system_info_key_value(host_id, "NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection);
1991 -
1992 - if (system_info->host_os_name)
1993 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_OS_NAME", system_info->host_os_name);
1994 -
1995 - if (system_info->host_os_id)
1996 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_OS_ID", system_info->host_os_id);
1997 -
1998 - if (system_info->host_os_id_like)
1999 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like);
2000 -
2001 - if (system_info->host_os_version)
2002 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_OS_VERSION", system_info->host_os_version);
2003 -
2004 - if (system_info->host_os_version_id)
2005 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id);
2006 -
2007 - if (system_info->host_os_detection)
2008 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_OS_DETECTION", system_info->host_os_detection);
2009 -
2010 - if (system_info->kernel_name)
2011 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name);
2012 -
2013 - if (system_info->host_cores)
2014 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores);
2015 -
2016 - if (system_info->host_cpu_freq)
2017 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq);
2018 -
2019 - if (system_info->host_ram_total)
2020 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total);
2021 -
2022 - if (system_info->host_disk_space)
2023 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space);
2024 -
2025 - if (system_info->kernel_version)
2026 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version);
2027 -
2028 - if (system_info->architecture)
2029 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture);
2030 -
2031 - if (system_info->virtualization)
2032 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization);
2033 -
2034 - if (system_info->virt_detection)
2035 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection);
2036 -
2037 - if (system_info->container)
2038 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_CONTAINER", system_info->container);
2039 -
2040 - if (system_info->container_detection)
2041 - sql_store_host_system_info_key_value(host_id, "NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection);
2042 -
2043 - if (system_info->is_k8s_node)
2044 - sql_store_host_system_info_key_value(host_id, "NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node);
2045 -
2046 - return;
2047 -}
2048 -
2049 -static int save_host_label_callback(const char *name, const char *value, RRDLABEL_SRC label_source, void *data)
2050 -{
2051 - RRDHOST *host = (RRDHOST *)data;
2052 - sql_store_host_label(&host->host_uuid, (int)label_source & ~(RRDLABEL_FLAG_INTERNAL), name, value);
2053 - return 0;
2054 -}
2055 -
2056 -#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid;"
2057 -void sql_store_host_labels(RRDHOST *host)
2058 -{
2059 - int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid);
2060 - if (rc != SQLITE_OK)
2061 - error_report("Failed to remove old host labels for host %s", rrdhost_hostname(host));
2062 -
2063 - rrdlabels_walkthrough_read(host->rrdlabels, save_host_label_callback, host);
1319 }
1320
1321 #define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \
@@ -2082,7 +1337,7 @@ DICTIONARY *sql_load_host_labels(uuid_t *host_id)
1337 rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
1338 if (unlikely(rc != SQLITE_OK)) {
1339 error_report("Failed to bind host parameter host information");
2085 - goto skip_loading;
1340 + goto skip;
1341 }
1342
1343 labels = rrdlabels_create();
@@ -2095,7 +1350,7 @@ DICTIONARY *sql_load_host_labels(uuid_t *host_id)
1350 sqlite3_column_int(res, 2));
1351 }
1352
2098 -skip_loading:
1353 +skip:
1354 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
1355 error_report("Failed to finalize the prepared statement when reading host information");
1356 return labels;
database/sqlite/sqlite_functions.h
+29 -52
@@ -25,23 +25,7 @@ typedef enum db_check_action_type {
25 } db_check_action_type_t;
26
27 #define SQL_MAX_RETRY (100)
28 -#define SQLITE_INSERT_DELAY (50) // Insert delay in case of lock
29 -
30 -#define SQL_STORE_HOST "insert or replace into host (host_id,hostname,registry_hostname,update_every,os,timezone,tags, hops) " \
31 - "values (?1,?2,?3,?4,?5,?6,?7,?8);"
32 -
33 -#define SQL_STORE_CHART "insert or replace into chart (chart_id, host_id, type, id, " \
34 - "name, family, context, title, unit, plugin, module, priority, update_every , chart_type , memory_mode , " \
35 - "history_entries) values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16);"
36 -
37 -#define SQL_FIND_CHART_UUID \
38 - "select chart_id from chart where host_id = @host and type=@type and id=@id and (name is null or name=@name) and chart_id is not null;"
39 -
40 -#define SQL_STORE_DIMENSION \
41 - "INSERT OR REPLACE into dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm) values (?0001,?0002,?0003,?0004,?0005,?0006,?0007);"
42 -
43 -#define SQL_FIND_DIMENSION_UUID \
44 - "select dim_id from dimension where chart_id=@chart and id=@id and name=@name and length(dim_id)=16;"
28 +#define SQLITE_INSERT_DELAY (10) // Insert delay in case of lock
29
30 #define CHECK_SQLITE_CONNECTION(db_meta) \
31 if (unlikely(!db_meta)) { \
@@ -61,52 +45,45 @@ SQLITE_API int sqlite3_exec_monitored(
45 char **errmsg /* Error msg written here */
46 );
47
48 +// Initialization and shutdown
49 +int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[]);
50 int sql_init_database(db_check_action_type_t rebuild, int memory);
51 void sql_close_database(void);
66 -int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);
67 -int sql_store_host(uuid_t *guid, const char *hostname, const char *registry_hostname, int update_every, const char *os,
68 - const char *timezone, const char *tags, int hops);
69 -
70 -int sql_store_host_info(RRDHOST *host);
71 -
72 -int sql_store_chart(
73 - uuid_t *chart_uuid, uuid_t *host_uuid, const char *type, const char *id, const char *name, const char *family,
74 - const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
75 - int update_every, int chart_type, int memory_mode, long history_entries);
76 -int sql_store_dimension(uuid_t *dim_uuid, uuid_t *chart_uuid, const char *id, const char *name, collected_number multiplier,
77 - collected_number divisor, int algorithm);
52
79 -int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid);
80 -
81 -int find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char *name, uuid_t *store_uuid);
82 -int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const char *name);
83 -
84 -int find_uuid_type(uuid_t *uuid);
85 -
86 -void sql_rrdset2json(RRDHOST *host, BUFFER *wb);
87 -
88 -int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement);
53 +// Helpers
54 +int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);
55 +int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement);
56 int execute_insert(sqlite3_stmt *res);
90 -void db_execute(const char *cmd);
57 int file_is_migrated(char *path);
58 +int exec_statement_with_uuid(const char *sql, uuid_t *uuid);
59 void add_migrated_file(char *path, uint64_t file_size);
93 -void db_unlock(void);
94 -void db_lock(void);
95 -void delete_dimension_uuid(uuid_t *dimension_uuid);
96 -void store_claim_id(uuid_t *host_id, uuid_t *claim_id);
97 -int update_node_id(uuid_t *host_id, uuid_t *node_id);
60 +void db_execute(const char *cmd);
61 +
62 +// Look up functions
63 +int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid);
64 +int find_chart_uuid(RRDHOST *host, const char *type, const char *id, uuid_t *store_uuid);
65 int get_node_id(uuid_t *host_id, uuid_t *node_id);
66 int get_host_id(uuid_t *node_id, uuid_t *host_id);
100 -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);
67 struct node_instance_list *get_node_list(void);
68 void sql_load_node_id(RRDHOST *host);
103 -int sql_set_dimension_option(uuid_t *dim_uuid, char *option);
69 char *get_hostname_by_node_id(char *node_id);
105 -int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[]);
106 -void migrate_localhost(uuid_t *host_uuid);
107 -void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_info *system_info);
70 +
71 +// Help build archived hosts in memory when agent starts
72 void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);
109 -void sql_store_host_labels(RRDHOST *host);
110 -int sql_metadata_cache_stats(int op);
73 DICTIONARY *sql_load_host_labels(uuid_t *host_id);
74 +
75 +// For queries: To be removed when context queries are implemented
76 +RRDHOST *sql_create_host_by_uuid(char *guid);
77 +void sql_rrdset2json(RRDHOST *host, BUFFER *wb);
78 +void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart);
79 +void free_temporary_host(RRDHOST *host);
80 +
81 +// TODO: move to metadata
82 +int update_node_id(uuid_t *host_id, uuid_t *node_id);
83 +
84 +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);
85 +
86 +// Provide statistics
87 +int sql_metadata_cache_stats(int op);
88 +
89 #endif //NETDATA_SQLITE_FUNCTIONS_H
database/sqlite/sqlite_metadata.c new
+1571
@@ -0,0 +1,1571 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "sqlite_metadata.h"
4 +
5 +extern DICTIONARY *rrdhost_root_index;
6 +
7 +// SQL statements
8 +
9 +#define SQL_STORE_CLAIM_ID "insert into node_instance " \
10 + "(host_id, claim_id, date_created) values (@host_id, @claim_id, unixepoch()) " \
11 + "on conflict(host_id) do update set claim_id = excluded.claim_id;"
12 +
13 +#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid;"
14 +
15 +#define STORE_HOST_LABEL \
16 + "INSERT OR REPLACE INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "
17 +
18 +#define STORE_CHART_LABEL \
19 + "INSERT OR REPLACE INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "
20 +
21 +#define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"
22 +
23 +#define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid;"
24 +
25 +#define SQL_STORE_HOST_INFO "INSERT OR REPLACE INTO host " \
26 + "(host_id, hostname, registry_hostname, update_every, os, timezone," \
27 + "tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, program_version," \
28 + "entries, health_enabled) " \
29 + "values (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, @memory_mode, " \
30 + "@abbrev_timezone, @utc_offset, @program_name, @program_version, " \
31 + "@entries, @health_enabled);"
32 +
33 +#define SQL_STORE_CHART "insert or replace into chart (chart_id, host_id, type, id, " \
34 + "name, family, context, title, unit, plugin, module, priority, update_every , chart_type , memory_mode , " \
35 + "history_entries) values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16);"
36 +
37 +#define SQL_STORE_DIMENSION "INSERT OR REPLACE INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm) " \
38 + "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm);"
39 +
40 +#define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id"
41 +
42 +#define STORE_HOST_INFO "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES "
43 +#define STORE_HOST_INFO_VALUES "(u2h('%s'), '%s','%s', unixepoch())"
44 +
45 +#define MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID \
46 + "UPDATE chart SET host_id = @host_id WHERE host_id in (SELECT host_id FROM host where host_id <> @host_id and hops = 0);"
47 +#define DELETE_NON_EXISTING_LOCALHOST "DELETE FROM host WHERE hops = 0 AND host_id <> @host_id;"
48 +#define DELETE_MISSING_NODE_INSTANCES "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);"
49 +
50 +#define METADATA_CMD_Q_MAX_SIZE (1024) // Max queue size; callers will block until there is room
51 +#define METADATA_MAINTENANCE_FIRST_CHECK (1800) // Maintenance first run after agent startup in seconds
52 +#define METADATA_MAINTENANCE_RETRY (60) // Retry run if already running or last run did actual work
53 +#define METADATA_MAINTENANCE_INTERVAL (3600) // Repeat maintenance after latest successful
54 +
55 +#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata
56 +#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata
57 +#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write
58 +
59 +#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)
60 +#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop
61 +#define METADATA_MAX_TRANSACTION_BATCH (128) // Maximum commands to add in a transaction
62 +
63 +enum metadata_opcode {
64 + METADATA_DATABASE_NOOP = 0,
65 + METADATA_DATABASE_TIMER,
66 + METADATA_ADD_CHART,
67 + METADATA_ADD_CHART_LABEL,
68 + METADATA_ADD_DIMENSION,
69 + METADATA_DEL_DIMENSION,
70 + METADATA_ADD_DIMENSION_OPTION,
71 + METADATA_ADD_HOST_SYSTEM_INFO,
72 + METADATA_ADD_HOST_INFO,
73 + METADATA_STORE_CLAIM_ID,
74 + METADATA_STORE_HOST_LABELS,
75 + METADATA_STORE_BUFFER,
76 +
77 + METADATA_SKIP_TRANSACTION, // Dummy -- OPCODES less than this one can be in a tranasction
78 +
79 + METADATA_SCAN_HOSTS,
80 + METADATA_MAINTENANCE,
81 + METADATA_SYNC_SHUTDOWN,
82 + METADATA_UNITTEST,
83 + // leave this last
84 + // we need it to check for worker utilization
85 + METADATA_MAX_ENUMERATIONS_DEFINED
86 +};
87 +
88 +#define MAX_PARAM_LIST (2)
89 +struct metadata_cmd {
90 + enum metadata_opcode opcode;
91 + struct completion *completion;
92 + const void *param[MAX_PARAM_LIST];
93 +};
94 +
95 +struct metadata_database_cmdqueue {
96 + unsigned head, tail;
97 + struct metadata_cmd cmd_array[METADATA_CMD_Q_MAX_SIZE];
98 +};
99 +
100 +typedef enum {
101 + METADATA_FLAG_CLEANUP = (1 << 0), // Cleanup is running
102 + METADATA_FLAG_SCANNING_HOSTS = (1 << 1), // Scanning of hosts in worker thread
103 + METADATA_FLAG_SHUTDOWN = (1 << 2), // Shutting down
104 +} METADATA_FLAG;
105 +
106 +#define METADATA_WORKER_BUSY (METADATA_FLAG_CLEANUP | METADATA_FLAG_SCANNING_HOSTS)
107 +
108 +struct metadata_wc {
109 + uv_thread_t thread;
110 + time_t check_metadata_after;
111 + time_t check_hosts_after;
112 + volatile unsigned queue_size;
113 + uv_loop_t *loop;
114 + uv_async_t async;
115 + METADATA_FLAG flags;
116 + uint64_t row_id;
117 + uv_timer_t timer_req;
118 + struct completion init_complete;
119 + /* FIFO command queue */
120 + uv_mutex_t cmd_mutex;
121 + uv_cond_t cmd_cond;
122 + struct metadata_database_cmdqueue cmd_queue;
123 +};
124 +
125 +#define metadata_flag_check(target_flags, flag) (__atomic_load_n(&((target_flags)->flags), __ATOMIC_SEQ_CST) & (flag))
126 +#define metadata_flag_set(target_flags, flag) __atomic_or_fetch(&((target_flags)->flags), (flag), __ATOMIC_SEQ_CST)
127 +#define metadata_flag_clear(target_flags, flag) __atomic_and_fetch(&((target_flags)->flags), ~(flag), __ATOMIC_SEQ_CST)
128 +
129 +//
130 +// For unittest
131 +//
132 +struct thread_unittest {
133 + int join;
134 + unsigned added;
135 + unsigned processed;
136 + unsigned *done;
137 +};
138 +
139 +
140 +// Metadata functions
141 +
142 +struct query_build {
143 + BUFFER *sql;
144 + int count;
145 + char uuid_str[UUID_STR_LEN];
146 +};
147 +
148 +static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
149 + struct query_build *lb = data;
150 + if (unlikely(!lb->count))
151 + buffer_sprintf(lb->sql, STORE_HOST_LABEL);
152 + else
153 + buffer_strcat(lb->sql, ", ");
154 + buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int)ls & ~(RRDLABEL_FLAG_INTERNAL), name, value);
155 + lb->count++;
156 + return 1;
157 +}
158 +
159 +static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
160 + struct query_build *lb = data;
161 + if (unlikely(!lb->count))
162 + buffer_sprintf(lb->sql, STORE_CHART_LABEL);
163 + else
164 + buffer_strcat(lb->sql, ", ");
165 + buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, ls, name, value);
166 + lb->count++;
167 + return 1;
168 +}
169 +
170 +static void check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer)
171 +{
172 + size_t old_version = st->rrdlabels_last_saved_version;
173 + size_t new_version = dictionary_version(st->rrdlabels);
174 +
175 + if(new_version != old_version) {
176 + buffer_flush(work_buffer);
177 + struct query_build tmp = {.sql = work_buffer, .count = 0};
178 + uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
179 + rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
180 + st->rrdlabels_last_saved_version = new_version;
181 + db_execute(buffer_tostring(work_buffer));
182 + }
183 +}
184 +
185 +// Migrate all hosts with hops zero to this host_uuid
186 +void migrate_localhost(uuid_t *host_uuid)
187 +{
188 + int rc;
189 +
190 + rc = exec_statement_with_uuid(MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID, host_uuid);
191 + if (!rc)
192 + rc = exec_statement_with_uuid(DELETE_NON_EXISTING_LOCALHOST, host_uuid);
193 + if (!rc)
194 + db_execute(DELETE_MISSING_NODE_INSTANCES);
195 +
196 +}
197 +
198 +static void store_claim_id(uuid_t *host_id, uuid_t *claim_id)
199 +{
200 + sqlite3_stmt *res = NULL;
201 + int rc;
202 +
203 + if (unlikely(!db_meta)) {
204 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
205 + error_report("Database has not been initialized");
206 + return;
207 + }
208 +
209 + rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CLAIM_ID, -1, &res, 0);
210 + if (unlikely(rc != SQLITE_OK)) {
211 + error_report("Failed to prepare statement store chart labels");
212 + return;
213 + }
214 +
215 + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
216 + if (unlikely(rc != SQLITE_OK)) {
217 + error_report("Failed to bind host_id parameter to store node instance information");
218 + goto failed;
219 + }
220 +
221 + if (claim_id)
222 + rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);
223 + else
224 + rc = sqlite3_bind_null(res, 2);
225 + if (unlikely(rc != SQLITE_OK)) {
226 + error_report("Failed to bind claim_id parameter to store node instance information");
227 + goto failed;
228 + }
229 +
230 + rc = execute_insert(res);
231 + if (unlikely(rc != SQLITE_DONE))
232 + error_report("Failed to store node instance information, rc = %d", rc);
233 +
234 +failed:
235 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
236 + error_report("Failed to finalize the prepared statement when storing node instance information");
237 +}
238 +
239 +static void delete_dimension_uuid(uuid_t *dimension_uuid)
240 +{
241 + static __thread sqlite3_stmt *res = NULL;
242 + int rc;
243 +
244 + if (unlikely(!res)) {
245 + rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res);
246 + if (rc != SQLITE_OK) {
247 + error_report("Failed to prepare statement to delete a dimension uuid");
248 + return;
249 + }
250 + }
251 +
252 + rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);
253 + if (unlikely(rc != SQLITE_OK))
254 + goto skip_execution;
255 +
256 + rc = sqlite3_step_monitored(res);
257 + if (unlikely(rc != SQLITE_DONE))
258 + error_report("Failed to delete dimension uuid, rc = %d", rc);
259 +
260 +skip_execution:
261 + rc = sqlite3_reset(res);
262 + if (unlikely(rc != SQLITE_OK))
263 + error_report("Failed to reset statement when deleting dimension UUID, rc = %d", rc);
264 +}
265 +
266 +//
267 +// Store host and host system info information in the database
268 +static int sql_store_host_info(RRDHOST *host)
269 +{
270 + static __thread sqlite3_stmt *res = NULL;
271 + int rc, param = 0;
272 +
273 + if (unlikely(!db_meta)) {
274 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
275 + return 0;
276 + error_report("Database has not been initialized");
277 + return 1;
278 + }
279 +
280 + if (unlikely((!res))) {
281 + rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res);
282 + if (unlikely(rc != SQLITE_OK)) {
283 + error_report("Failed to prepare statement to store host, rc = %d", rc);
284 + return 1;
285 + }
286 + }
287 +
288 + rc = sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
289 + if (unlikely(rc != SQLITE_OK))
290 + goto bind_fail;
291 +
292 + rc = bind_text_null(res, ++param, rrdhost_hostname(host), 0);
293 + if (unlikely(rc != SQLITE_OK))
294 + goto bind_fail;
295 +
296 + rc = bind_text_null(res, ++param, rrdhost_registry_hostname(host), 1);
297 + if (unlikely(rc != SQLITE_OK))
298 + goto bind_fail;
299 +
300 + rc = sqlite3_bind_int(res, ++param, host->rrd_update_every);
301 + if (unlikely(rc != SQLITE_OK))
302 + goto bind_fail;
303 +
304 + rc = bind_text_null(res, ++param, rrdhost_os(host), 1);
305 + if (unlikely(rc != SQLITE_OK))
306 + goto bind_fail;
307 +
308 + rc = bind_text_null(res, ++param, rrdhost_timezone(host), 1);
309 + if (unlikely(rc != SQLITE_OK))
310 + goto bind_fail;
311 +
312 + rc = bind_text_null(res, ++param, rrdhost_tags(host), 1);
313 + if (unlikely(rc != SQLITE_OK))
314 + goto bind_fail;
315 +
316 + rc = sqlite3_bind_int(res, ++param, host->system_info ? host->system_info->hops : 0);
317 + if (unlikely(rc != SQLITE_OK))
318 + goto bind_fail;
319 +
320 + rc = sqlite3_bind_int(res, ++param, host->rrd_memory_mode);
321 + if (unlikely(rc != SQLITE_OK))
322 + goto bind_fail;
323 +
324 + rc = bind_text_null(res, ++param, rrdhost_abbrev_timezone(host), 1);
325 + if (unlikely(rc != SQLITE_OK))
326 + goto bind_fail;
327 +
328 + rc = sqlite3_bind_int(res, ++param, host->utc_offset);
329 + if (unlikely(rc != SQLITE_OK))
330 + goto bind_fail;
331 +
332 + rc = bind_text_null(res, ++param, rrdhost_program_name(host), 1);
333 + if (unlikely(rc != SQLITE_OK))
334 + goto bind_fail;
335 +
336 + rc = bind_text_null(res, ++param, rrdhost_program_version(host), 1);
337 + if (unlikely(rc != SQLITE_OK))
338 + goto bind_fail;
339 +
340 + rc = sqlite3_bind_int64(res, ++param, host->rrd_history_entries);
341 + if (unlikely(rc != SQLITE_OK))
342 + goto bind_fail;
343 +
344 + rc = sqlite3_bind_int(res, ++param, (int ) host->health_enabled);
345 + if (unlikely(rc != SQLITE_OK))
346 + goto bind_fail;
347 +
348 + int store_rc = sqlite3_step_monitored(res);
349 + if (unlikely(store_rc != SQLITE_DONE))
350 + error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc);
351 +
352 + rc = sqlite3_reset(res);
353 + if (unlikely(rc != SQLITE_OK))
354 + error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
355 +
356 + return !(store_rc == SQLITE_DONE);
357 +bind_fail:
358 + error_report("Failed to bind %d parameter to store host %s, rc = %d", param, rrdhost_hostname(host), rc);
359 + rc = sqlite3_reset(res);
360 + if (unlikely(rc != SQLITE_OK))
361 + error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
362 + return 1;
363 +}
364 +
365 +static void sql_store_host_system_info_key_value(const char *name, const char *value, void *data)
366 +{
367 + struct query_build *lb = data;
368 +
369 + if (unlikely(!value))
370 + return;
371 +
372 + if (unlikely(!lb->count))
373 + buffer_sprintf(
374 + lb->sql, STORE_HOST_INFO);
375 + else
376 + buffer_strcat(lb->sql, ", ");
377 + buffer_sprintf(lb->sql, STORE_HOST_INFO_VALUES, lb->uuid_str, name, value);
378 + lb->count++;
379 +}
380 +
381 +static BUFFER *sql_store_host_system_info(RRDHOST *host)
382 +{
383 + struct rrdhost_system_info *system_info = host->system_info;
384 +
385 + if (unlikely(!system_info))
386 + return NULL;
387 +
388 + BUFFER *work_buffer = buffer_create(1024);
389 +
390 + struct query_build key_data = {.sql = work_buffer, .count = 0};
391 + uuid_unparse_lower(host->host_uuid, key_data.uuid_str);
392 +
393 + sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &key_data);
394 + sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &key_data);
395 + sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &key_data);
396 + sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &key_data);
397 + sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &key_data);
398 + sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &key_data);
399 + sql_store_host_system_info_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &key_data);
400 + sql_store_host_system_info_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &key_data);
401 + sql_store_host_system_info_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &key_data);
402 + sql_store_host_system_info_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &key_data);
403 + sql_store_host_system_info_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &key_data);
404 + sql_store_host_system_info_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &key_data);
405 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &key_data);
406 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &key_data);
407 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &key_data);
408 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total, &key_data);
409 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space, &key_data);
410 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version, &key_data);
411 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture, &key_data);
412 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization, &key_data);
413 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection, &key_data);
414 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_CONTAINER", system_info->container, &key_data);
415 + sql_store_host_system_info_key_value("NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection, &key_data);
416 + sql_store_host_system_info_key_value("NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node, &key_data);
417 +
418 + return work_buffer;
419 +}
420 +
421 +
422 +/*
423 + * Store set option for a dimension
424 + */
425 +static int sql_set_dimension_option(uuid_t *dim_uuid, char *option)
426 +{
427 + sqlite3_stmt *res = NULL;
428 + int rc;
429 +
430 + if (unlikely(!db_meta)) {
431 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
432 + return 0;
433 + error_report("Database has not been initialized");
434 + return 1;
435 + }
436 +
437 + rc = sqlite3_prepare_v2(db_meta, "UPDATE dimension SET options = @options WHERE dim_id = @dim_id", -1, &res, 0);
438 + if (unlikely(rc != SQLITE_OK)) {
439 + error_report("Failed to prepare statement to update dimension options");
440 + return 0;
441 + };
442 +
443 + rc = sqlite3_bind_blob(res, 2, dim_uuid, sizeof(*dim_uuid), SQLITE_STATIC);
444 + if (unlikely(rc != SQLITE_OK))
445 + goto bind_fail;
446 +
447 + if (!option || !strcmp(option,"unhide"))
448 + rc = sqlite3_bind_null(res, 1);
449 + else
450 + rc = sqlite3_bind_text(res, 1, option, -1, SQLITE_STATIC);
451 + if (unlikely(rc != SQLITE_OK))
452 + goto bind_fail;
453 +
454 + rc = execute_insert(res);
455 + if (unlikely(rc != SQLITE_DONE))
456 + error_report("Failed to update dimension option, rc = %d", rc);
457 +
458 +bind_fail:
459 + rc = sqlite3_finalize(res);
460 + if (unlikely(rc != SQLITE_OK))
461 + error_report("Failed to finalize statement in update dimension options, rc = %d", rc);
462 + return 0;
463 +}
464 +
465 +/*
466 + * Store a chart in the database
467 + */
468 +
469 +static int sql_store_chart(
470 + uuid_t *chart_uuid, uuid_t *host_uuid, const char *type, const char *id, const char *name, const char *family,
471 + const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
472 + int update_every, int chart_type, int memory_mode, long history_entries)
473 +{
474 + static __thread sqlite3_stmt *res = NULL;
475 + int rc, param = 0;
476 +
477 + if (unlikely(!db_meta)) {
478 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
479 + return 0;
480 + error_report("Database has not been initialized");
481 + return 1;
482 + }
483 +
484 + if (unlikely(!res)) {
485 + rc = prepare_statement(db_meta, SQL_STORE_CHART, &res);
486 + if (unlikely(rc != SQLITE_OK)) {
487 + error_report("Failed to prepare statement to store chart, rc = %d", rc);
488 + return 1;
489 + }
490 + }
491 +
492 + param++;
493 + rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
494 + if (unlikely(rc != SQLITE_OK))
495 + goto bind_fail;
496 +
497 + param++;
498 + rc = sqlite3_bind_blob(res, 2, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
499 + if (unlikely(rc != SQLITE_OK))
500 + goto bind_fail;
501 +
502 + param++;
503 + rc = sqlite3_bind_text(res, 3, type, -1, SQLITE_STATIC);
504 + if (unlikely(rc != SQLITE_OK))
505 + goto bind_fail;
506 +
507 + param++;
508 + rc = sqlite3_bind_text(res, 4, id, -1, SQLITE_STATIC);
509 + if (unlikely(rc != SQLITE_OK))
510 + goto bind_fail;
511 +
512 + param++;
513 + if (name && *name)
514 + rc = sqlite3_bind_text(res, 5, name, -1, SQLITE_STATIC);
515 + else
516 + rc = sqlite3_bind_null(res, 5);
517 + if (unlikely(rc != SQLITE_OK))
518 + goto bind_fail;
519 +
520 + param++;
521 + rc = sqlite3_bind_text(res, 6, family, -1, SQLITE_STATIC);
522 + if (unlikely(rc != SQLITE_OK))
523 + goto bind_fail;
524 +
525 + param++;
526 + rc = sqlite3_bind_text(res, 7, context, -1, SQLITE_STATIC);
527 + if (unlikely(rc != SQLITE_OK))
528 + goto bind_fail;
529 +
530 + param++;
531 + rc = sqlite3_bind_text(res, 8, title, -1, SQLITE_STATIC);
532 + if (unlikely(rc != SQLITE_OK))
533 + goto bind_fail;
534 +
535 + param++;
536 + rc = sqlite3_bind_text(res, 9, units, -1, SQLITE_STATIC);
537 + if (unlikely(rc != SQLITE_OK))
538 + goto bind_fail;
539 +
540 + param++;
541 + rc = sqlite3_bind_text(res, 10, plugin, -1, SQLITE_STATIC);
542 + if (unlikely(rc != SQLITE_OK))
543 + goto bind_fail;
544 +
545 + param++;
546 + rc = sqlite3_bind_text(res, 11, module, -1, SQLITE_STATIC);
547 + if (unlikely(rc != SQLITE_OK))
548 + goto bind_fail;
549 +
550 + param++;
551 + rc = sqlite3_bind_int(res, 12, (int) priority);
552 + if (unlikely(rc != SQLITE_OK))
553 + goto bind_fail;
554 +
555 + param++;
556 + rc = sqlite3_bind_int(res, 13, update_every);
557 + if (unlikely(rc != SQLITE_OK))
558 + goto bind_fail;
559 +
560 + param++;
561 + rc = sqlite3_bind_int(res, 14, chart_type);
562 + if (unlikely(rc != SQLITE_OK))
563 + goto bind_fail;
564 +
565 + param++;
566 + rc = sqlite3_bind_int(res, 15, memory_mode);
567 + if (unlikely(rc != SQLITE_OK))
568 + goto bind_fail;
569 +
570 + param++;
571 + rc = sqlite3_bind_int(res, 16, (int) history_entries);
572 + if (unlikely(rc != SQLITE_OK))
573 + goto bind_fail;
574 +
575 + rc = execute_insert(res);
576 + if (unlikely(rc != SQLITE_DONE))
577 + error_report("Failed to store chart, rc = %d", rc);
578 +
579 + rc = sqlite3_reset(res);
580 + if (unlikely(rc != SQLITE_OK))
581 + error_report("Failed to reset statement in chart store function, rc = %d", rc);
582 +
583 + return 0;
584 +
585 +bind_fail:
586 + error_report("Failed to bind parameter %d to store chart, rc = %d", param, rc);
587 + rc = sqlite3_reset(res);
588 + if (unlikely(rc != SQLITE_OK))
589 + error_report("Failed to reset statement in chart store function, rc = %d", rc);
590 + return 1;
591 +}
592 +
593 +/*
594 + * Store a dimension
595 + */
596 +static int sql_store_dimension(
597 + uuid_t *dim_uuid, uuid_t *chart_uuid, const char *id, const char *name, collected_number multiplier,
598 + collected_number divisor, int algorithm)
599 +{
600 + static __thread sqlite3_stmt *res = NULL;
601 + int rc, param = 0;
602 +
603 + if (unlikely(!db_meta)) {
604 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
605 + return 0;
606 + error_report("Database has not been initialized");
607 + return 1;
608 + }
609 +
610 + if (unlikely(!res)) {
611 + rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res);
612 + if (unlikely(rc != SQLITE_OK)) {
613 + error_report("Failed to prepare statement to store dimension, rc = %d", rc);
614 + return 1;
615 + }
616 + }
617 +
618 + rc = sqlite3_bind_blob(res, ++param, dim_uuid, sizeof(*dim_uuid), SQLITE_STATIC);
619 + if (unlikely(rc != SQLITE_OK))
620 + goto bind_fail;
621 +
622 + rc = sqlite3_bind_blob(res, ++param, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
623 + if (unlikely(rc != SQLITE_OK))
624 + goto bind_fail;
625 +
626 + rc = sqlite3_bind_text(res, ++param, id, -1, SQLITE_STATIC);
627 + if (unlikely(rc != SQLITE_OK))
628 + goto bind_fail;
629 +
630 + rc = sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC);
631 + if (unlikely(rc != SQLITE_OK))
632 + goto bind_fail;
633 +
634 + rc = sqlite3_bind_int(res, ++param, (int) multiplier);
635 + if (unlikely(rc != SQLITE_OK))
636 + goto bind_fail;
637 +
638 + rc = sqlite3_bind_int(res, ++param, (int ) divisor);
639 + if (unlikely(rc != SQLITE_OK))
640 + goto bind_fail;
641 +
642 + rc = sqlite3_bind_int(res, ++param, algorithm);
643 + if (unlikely(rc != SQLITE_OK))
644 + goto bind_fail;
645 +
646 + rc = execute_insert(res);
647 + if (unlikely(rc != SQLITE_DONE))
648 + error_report("Failed to store dimension, rc = %d", rc);
649 +
650 + rc = sqlite3_reset(res);
651 + if (unlikely(rc != SQLITE_OK))
652 + error_report("Failed to reset statement in store dimension, rc = %d", rc);
653 + return 0;
654 +
655 +bind_fail:
656 + error_report("Failed to bind parameter %d to store dimension, rc = %d", param, rc);
657 + rc = sqlite3_reset(res);
658 + if (unlikely(rc != SQLITE_OK))
659 + error_report("Failed to reset statement in store dimension, rc = %d", rc);
660 + return 1;
661 +}
662 +
663 +static bool dimension_can_be_deleted(uuid_t *dim_uuid)
664 +{
665 +#ifdef ENABLE_DBENGINE
666 + bool no_retention = true;
667 + for (int tier = 0; tier < storage_tiers; tier++) {
668 + if (!multidb_ctx[tier])
669 + continue;
670 + time_t first_time_t = 0, last_time_t = 0;
671 + if (rrdeng_metric_retention_by_uuid((void *) multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t) == 0) {
672 + if (first_time_t > 0) {
673 + no_retention = false;
674 + break;
675 + }
676 + }
677 + }
678 + return no_retention;
679 +#else
680 + return false;
681 +#endif
682 +}
683 +
684 +static void check_dimension_metadata(struct metadata_wc *wc)
685 +{
686 + int rc;
687 + sqlite3_stmt *res = NULL;
688 +
689 + rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION_LIST, -1, &res, 0);
690 + if (unlikely(rc != SQLITE_OK)) {
691 + error_report("Failed to prepare statement to fetch host dimensions");
692 + return;
693 + }
694 +
695 + rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) wc->row_id);
696 + if (unlikely(rc != SQLITE_OK)) {
697 + error_report("Failed to row parameter");
698 + goto skip_run;
699 + }
700 +
701 + uint32_t total_checked = 0;
702 + uint32_t total_deleted= 0;
703 + uint64_t last_row_id = wc->row_id;
704 +
705 + info("METADATA: Checking dimensions starting after row %"PRIu64, wc->row_id);
706 +
707 + while (sqlite3_step_monitored(res) == SQLITE_ROW && total_deleted < MAX_METADATA_CLEANUP) {
708 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
709 + break;
710 +
711 + last_row_id = sqlite3_column_int64(res, 1);
712 + rc = dimension_can_be_deleted((uuid_t *)sqlite3_column_blob(res, 0));
713 + if (rc == true) {
714 + delete_dimension_uuid((uuid_t *)sqlite3_column_blob(res, 0));
715 + total_deleted++;
716 + }
717 + total_checked++;
718 + }
719 + wc->row_id = last_row_id;
720 + time_t now = now_realtime_sec();
721 + if (total_deleted > 0) {
722 + wc->check_metadata_after = now + METADATA_MAINTENANCE_RETRY;
723 + } else
724 + wc->row_id = 0;
725 + info("METADATA: Checked %u, deleted %u -- will resume after row %"PRIu64" in %ld seconds", total_checked, total_deleted, wc->row_id,
726 + wc->check_metadata_after - now);
727 +
728 +skip_run:
729 + rc = sqlite3_finalize(res);
730 + if (unlikely(rc != SQLITE_OK))
731 + error_report("Failed to finalize the prepared statement when reading dimensions");
732 +}
733 +
734 +
735 +//
736 +// EVENT LOOP STARTS HERE
737 +//
738 +static uv_mutex_t metadata_async_lock;
739 +
740 +static void metadata_init_cmd_queue(struct metadata_wc *wc)
741 +{
742 + wc->cmd_queue.head = wc->cmd_queue.tail = 0;
743 + wc->queue_size = 0;
744 + fatal_assert(0 == uv_cond_init(&wc->cmd_cond));
745 + fatal_assert(0 == uv_mutex_init(&wc->cmd_mutex));
746 +}
747 +
748 +int metadata_enq_cmd_noblock(struct metadata_wc *wc, struct metadata_cmd *cmd)
749 +{
750 + unsigned queue_size;
751 +
752 + /* wait for free space in queue */
753 + uv_mutex_lock(&wc->cmd_mutex);
754 +
755 + if (cmd->opcode == METADATA_SYNC_SHUTDOWN) {
756 + metadata_flag_set(wc, METADATA_FLAG_SHUTDOWN);
757 + uv_mutex_unlock(&wc->cmd_mutex);
758 + return 0;
759 + }
760 +
761 + if (unlikely((queue_size = wc->queue_size) == METADATA_CMD_Q_MAX_SIZE ||
762 + metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
763 + uv_mutex_unlock(&wc->cmd_mutex);
764 + return 1;
765 + }
766 +
767 + fatal_assert(queue_size < METADATA_CMD_Q_MAX_SIZE);
768 + /* enqueue command */
769 + wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
770 + wc->cmd_queue.tail = wc->cmd_queue.tail != METADATA_CMD_Q_MAX_SIZE - 1 ?
771 + wc->cmd_queue.tail + 1 : 0;
772 + wc->queue_size = queue_size + 1;
773 + uv_mutex_unlock(&wc->cmd_mutex);
774 + return 0;
775 +}
776 +
777 +static void metadata_enq_cmd(struct metadata_wc *wc, struct metadata_cmd *cmd)
778 +{
779 + unsigned queue_size;
780 +
781 + /* wait for free space in queue */
782 + uv_mutex_lock(&wc->cmd_mutex);
783 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
784 + uv_mutex_unlock(&wc->cmd_mutex);
785 + (void) uv_async_send(&wc->async);
786 + return;
787 + }
788 +
789 + if (cmd->opcode == METADATA_SYNC_SHUTDOWN) {
790 + metadata_flag_set(wc, METADATA_FLAG_SHUTDOWN);
791 + uv_mutex_unlock(&wc->cmd_mutex);
792 + (void) uv_async_send(&wc->async);
793 + return;
794 + }
795 +
796 + while ((queue_size = wc->queue_size) == METADATA_CMD_Q_MAX_SIZE) {
797 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
798 + uv_mutex_unlock(&wc->cmd_mutex);
799 + return;
800 + }
801 + uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
802 + }
803 + fatal_assert(queue_size < METADATA_CMD_Q_MAX_SIZE);
804 + /* enqueue command */
805 + wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
806 + wc->cmd_queue.tail = wc->cmd_queue.tail != METADATA_CMD_Q_MAX_SIZE - 1 ?
807 + wc->cmd_queue.tail + 1 : 0;
808 + wc->queue_size = queue_size + 1;
809 + uv_mutex_unlock(&wc->cmd_mutex);
810 +
811 + /* wake up event loop */
812 + (void) uv_async_send(&wc->async);
813 +}
814 +
815 +static struct metadata_cmd metadata_deq_cmd(struct metadata_wc *wc, enum metadata_opcode *next_opcode)
816 +{
817 + struct metadata_cmd ret;
818 + unsigned queue_size;
819 +
820 + uv_mutex_lock(&wc->cmd_mutex);
821 + queue_size = wc->queue_size;
822 + if (queue_size == 0) {
823 + memset(&ret, 0, sizeof(ret));
824 + ret.opcode = METADATA_DATABASE_NOOP;
825 + ret.completion = NULL;
826 + *next_opcode = METADATA_DATABASE_NOOP;
827 + } else {
828 + /* dequeue command */
829 + ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];
830 +
831 + if (queue_size == 1) {
832 + wc->cmd_queue.head = wc->cmd_queue.tail = 0;
833 + } else {
834 + wc->cmd_queue.head = wc->cmd_queue.head != METADATA_CMD_Q_MAX_SIZE - 1 ?
835 + wc->cmd_queue.head + 1 : 0;
836 + }
837 + wc->queue_size = queue_size - 1;
838 + if (wc->queue_size > 0)
839 + *next_opcode = wc->cmd_queue.cmd_array[wc->cmd_queue.head].opcode;
840 + else
841 + *next_opcode = METADATA_DATABASE_NOOP;
842 + /* wake up producers */
843 + uv_cond_signal(&wc->cmd_cond);
844 + }
845 + uv_mutex_unlock(&wc->cmd_mutex);
846 +
847 + return ret;
848 +}
849 +
850 +static void async_cb(uv_async_t *handle)
851 +{
852 + uv_stop(handle->loop);
853 + uv_update_time(handle->loop);
854 +}
855 +
856 +#define TIMER_INITIAL_PERIOD_MS (1000)
857 +#define TIMER_REPEAT_PERIOD_MS (1000)
858 +
859 +static void timer_cb(uv_timer_t* handle)
860 +{
861 + uv_stop(handle->loop);
862 + uv_update_time(handle->loop);
863 +
864 + struct metadata_wc *wc = handle->data;
865 + struct metadata_cmd cmd;
866 + memset(&cmd, 0, sizeof(cmd));
867 +
868 + time_t now = now_realtime_sec();
869 +
870 + if (wc->check_metadata_after && wc->check_metadata_after < now) {
871 + cmd.opcode = METADATA_MAINTENANCE;
872 + if (!metadata_enq_cmd_noblock(wc, &cmd))
873 + wc->check_metadata_after = now + METADATA_MAINTENANCE_INTERVAL;
874 + }
875 +
876 + if (wc->check_hosts_after && wc->check_hosts_after < now) {
877 + cmd.opcode = METADATA_SCAN_HOSTS;
878 + if (!metadata_enq_cmd_noblock(wc, &cmd))
879 + wc->check_hosts_after = now + METADATA_HOST_CHECK_INTERVAL;
880 + }
881 +}
882 +
883 +static void after_metadata_cleanup(uv_work_t *req, int status)
884 +{
885 + UNUSED(status);
886 +
887 + struct metadata_wc *wc = req->data;
888 + metadata_flag_clear(wc, METADATA_FLAG_CLEANUP);
889 +}
890 +static void start_metadata_cleanup(uv_work_t *req)
891 +{
892 + struct metadata_wc *wc = req->data;
893 + check_dimension_metadata(wc);
894 +}
895 +
896 +struct scan_metadata_payload {
897 + uv_work_t request;
898 + struct metadata_wc *wc;
899 + struct completion *completion;
900 + uint32_t max_count;
901 +};
902 +
903 +// Callback after scan of hosts is done
904 +static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)
905 +{
906 + struct scan_metadata_payload *data = req->data;
907 + struct metadata_wc *wc = data->wc;
908 +
909 + metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
910 + internal_error(true, "METADATA: scanning hosts complete");
911 + if (unlikely(data->completion)) {
912 + completion_mark_complete(data->completion);
913 + internal_error(true, "METADATA: Sending completion done");
914 + }
915 + freez(data);
916 +}
917 +
918 +static bool metadata_scan_host(RRDHOST *host, uint32_t max_count) {
919 + RRDSET *st;
920 + int rc;
921 +
922 + bool more_to_do = false;
923 + uint32_t scan_count = 1;
924 + BUFFER *work_buffer = buffer_create(1024);
925 +
926 + rrdset_foreach_reentrant(st, host) {
927 + if (scan_count == max_count) {
928 + more_to_do = true;
929 + break;
930 + }
931 + if(rrdset_flag_check(st, RRDSET_FLAG_METADATA_UPDATE)) {
932 + rrdset_flag_clear(st, RRDSET_FLAG_METADATA_UPDATE);
933 + scan_count++;
934 +
935 + check_and_update_chart_labels(st, work_buffer);
936 +
937 + rc = sql_store_chart(
938 + &st->chart_uuid,
939 + &st->rrdhost->host_uuid,
940 + string2str(st->parts.type),
941 + string2str(st->parts.id),
942 + string2str(st->parts.name),
943 + rrdset_family(st),
944 + rrdset_context(st),
945 + rrdset_title(st),
946 + rrdset_units(st),
947 + rrdset_plugin_name(st),
948 + rrdset_module_name(st),
949 + st->priority,
950 + st->update_every,
951 + st->chart_type,
952 + st->rrd_memory_mode,
953 + st->entries);
954 + if (unlikely(rc))
955 + internal_error(true, "METADATA: Failed to store chart metadata %s", string2str(st->id));
956 + }
957 +
958 + RRDDIM *rd;
959 + rrddim_foreach_read(rd, st) {
960 + if(rrddim_flag_check(rd, RRDDIM_FLAG_METADATA_UPDATE)) {
961 + rrddim_flag_clear(rd, RRDDIM_FLAG_METADATA_UPDATE);
962 +
963 + rc = sql_store_dimension(
964 + &rd->metric_uuid,
965 + &rd->rrdset->chart_uuid,
966 + string2str(rd->id),
967 + string2str(rd->name),
968 + rd->multiplier,
969 + rd->divisor,
970 + rd->algorithm);
971 +
972 + if (unlikely(rc))
973 + error_report("METADATA: Failed to store dimension %s", string2str(rd->id));
974 + }
975 + }
976 + rrddim_foreach_done(rd);
977 + }
978 + rrdset_foreach_done(st);
979 +
980 + buffer_free(work_buffer);
981 + return more_to_do;
982 +}
983 +
984 +// Worker thread to scan hosts for pending metadata to store
985 +static void start_metadata_hosts(uv_work_t *req __maybe_unused)
986 +{
987 + RRDHOST *host;
988 +
989 + struct scan_metadata_payload *data = req->data;
990 + struct metadata_wc *wc = data->wc;
991 +
992 + bool run_again = false;
993 + dfe_start_reentrant(rrdhost_root_index, host) {
994 + if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE))
995 + continue;
996 + internal_error(true, "METADATA: Scanning host %s", rrdhost_hostname(host));
997 + rrdhost_flag_clear(host,RRDHOST_FLAG_METADATA_UPDATE);
998 + if (unlikely(metadata_scan_host(host, data->max_count))) {
999 + run_again = true;
1000 + rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE);
1001 + internal_error(true,"METADATA: Rescheduling host %s to run; more charts to store", rrdhost_hostname(host));
1002 + }
1003 + }
1004 + dfe_done(host);
1005 + if (unlikely(run_again))
1006 + wc->check_hosts_after = now_realtime_sec() + METADATA_HOST_CHECK_IMMEDIATE;
1007 + else
1008 + wc->check_hosts_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL;
1009 +}
1010 +
1011 +static void metadata_event_loop(void *arg)
1012 +{
1013 + worker_register("METASYNC");
1014 + worker_register_job_name(METADATA_DATABASE_NOOP, "noop");
1015 + worker_register_job_name(METADATA_DATABASE_TIMER, "timer");
1016 + worker_register_job_name(METADATA_ADD_CHART, "add chart");
1017 + worker_register_job_name(METADATA_ADD_CHART_LABEL, "add chart label");
1018 + worker_register_job_name(METADATA_ADD_DIMENSION, "add dimension");
1019 + worker_register_job_name(METADATA_DEL_DIMENSION, "delete dimension");
1020 + worker_register_job_name(METADATA_ADD_DIMENSION_OPTION, "dimension option");
1021 + worker_register_job_name(METADATA_ADD_HOST_SYSTEM_INFO, "host system info");
1022 + worker_register_job_name(METADATA_ADD_HOST_INFO, "host info");
1023 + worker_register_job_name(METADATA_STORE_CLAIM_ID, "add claim id");
1024 + worker_register_job_name(METADATA_STORE_HOST_LABELS, "host labels");
1025 + worker_register_job_name(METADATA_MAINTENANCE, "maintenance");
1026 +
1027 +
1028 + int ret;
1029 + uv_loop_t *loop;
1030 + unsigned cmd_batch_size;
1031 + struct metadata_wc *wc = arg;
1032 + enum metadata_opcode opcode, next_opcode;
1033 + uv_work_t metadata_cleanup_worker;
1034 +
1035 + uv_thread_set_name_np(wc->thread, "METASYNC");
1036 + loop = wc->loop = mallocz(sizeof(uv_loop_t));
1037 + ret = uv_loop_init(loop);
1038 + if (ret) {
1039 + error("uv_loop_init(): %s", uv_strerror(ret));
1040 + goto error_after_loop_init;
1041 + }
1042 + loop->data = wc;
1043 +
1044 + ret = uv_async_init(wc->loop, &wc->async, async_cb);
1045 + if (ret) {
1046 + error("uv_async_init(): %s", uv_strerror(ret));
1047 + goto error_after_async_init;
1048 + }
1049 + wc->async.data = wc;
1050 +
1051 + ret = uv_timer_init(loop, &wc->timer_req);
1052 + if (ret) {
1053 + error("uv_timer_init(): %s", uv_strerror(ret));
1054 + goto error_after_timer_init;
1055 + }
1056 + wc->timer_req.data = wc;
1057 + fatal_assert(0 == uv_timer_start(&wc->timer_req, timer_cb, TIMER_INITIAL_PERIOD_MS, TIMER_REPEAT_PERIOD_MS));
1058 +
1059 + info("Starting metadata sync thread with %d entries command queue", METADATA_CMD_Q_MAX_SIZE);
1060 +
1061 + struct metadata_cmd cmd;
1062 + memset(&cmd, 0, sizeof(cmd));
1063 + metadata_flag_clear(wc, METADATA_FLAG_CLEANUP);
1064 + metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
1065 +
1066 + wc->check_metadata_after = now_realtime_sec() + METADATA_MAINTENANCE_FIRST_CHECK;
1067 + wc->check_hosts_after = now_realtime_sec() + METADATA_HOST_CHECK_FIRST_CHECK;
1068 +
1069 + int shutdown = 0;
1070 + int in_transaction = 0;
1071 + int commands_in_transaction = 0;
1072 + // This can be used in the event loop for all opcodes (not workers)
1073 + BUFFER *work_buffer = buffer_create(1024);
1074 + wc->row_id = 0;
1075 + completion_mark_complete(&wc->init_complete);
1076 +
1077 + while (shutdown == 0 || (wc->flags & METADATA_WORKER_BUSY)) {
1078 + RRDDIM *rd = NULL;
1079 + RRDSET *st = NULL;
1080 + RRDHOST *host = NULL;
1081 + DICTIONARY_ITEM *dict_item = NULL;
1082 + BUFFER *buffer = NULL;
1083 + uuid_t *uuid;
1084 + int rc;
1085 +
1086 + worker_is_idle();
1087 + uv_run(loop, UV_RUN_DEFAULT);
1088 +
1089 + /* wait for commands */
1090 + cmd_batch_size = 0;
1091 + do {
1092 + if (unlikely(cmd_batch_size >= METADATA_MAX_BATCH_SIZE))
1093 + break;
1094 +
1095 + cmd = metadata_deq_cmd(wc, &next_opcode);
1096 + opcode = cmd.opcode;
1097 +
1098 + if (unlikely(opcode == METADATA_DATABASE_NOOP && metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
1099 + shutdown = 1;
1100 + continue;
1101 + }
1102 +
1103 + ++cmd_batch_size;
1104 +
1105 + // If we are not in transaction and this command is the same with the next ; start a transaction
1106 + if (!in_transaction && opcode < METADATA_SKIP_TRANSACTION && opcode == next_opcode) {
1107 + if (opcode != METADATA_DATABASE_NOOP) {
1108 + in_transaction = 1;
1109 + db_execute("BEGIN TRANSACTION;");
1110 + }
1111 + }
1112 +
1113 + if (likely(in_transaction)) {
1114 + commands_in_transaction++;
1115 + }
1116 +
1117 + if (likely(opcode != METADATA_DATABASE_NOOP))
1118 + worker_is_busy(opcode);
1119 +
1120 + switch (opcode) {
1121 + case METADATA_DATABASE_NOOP:
1122 + case METADATA_DATABASE_TIMER:
1123 + break;
1124 + case METADATA_ADD_CHART:
1125 + dict_item = (DICTIONARY_ITEM * ) cmd.param[0];
1126 + st = (RRDSET *) dictionary_acquired_item_value(dict_item);
1127 +
1128 + rc = sql_store_chart(
1129 + &st->chart_uuid,
1130 + &st->rrdhost->host_uuid,
1131 + string2str(st->parts.type),
1132 + string2str(st->parts.id),
1133 + string2str(st->parts.name),
1134 + rrdset_family(st),
1135 + rrdset_context(st),
1136 + rrdset_title(st),
1137 + rrdset_units(st),
1138 + rrdset_plugin_name(st),
1139 + rrdset_module_name(st),
1140 + st->priority,
1141 + st->update_every,
1142 + st->chart_type,
1143 + st->rrd_memory_mode,
1144 + st->entries);
1145 +
1146 + if (unlikely(rc))
1147 + error_report("Failed to store chart %s", rrdset_id(st));
1148 +
1149 + dictionary_acquired_item_release(st->rrdhost->rrdset_root_index, dict_item);
1150 + break;
1151 + case METADATA_ADD_CHART_LABEL:
1152 + dict_item = (DICTIONARY_ITEM * ) cmd.param[0];
1153 + st = (RRDSET *) dictionary_acquired_item_value(dict_item);
1154 + check_and_update_chart_labels(st, work_buffer);
1155 + dictionary_acquired_item_release(st->rrdhost->rrdset_root_index, dict_item);
1156 + break;
1157 + case METADATA_ADD_DIMENSION:
1158 + dict_item = (DICTIONARY_ITEM * ) cmd.param[0];
1159 + rd = (RRDDIM *) dictionary_acquired_item_value(dict_item);
1160 +
1161 + rc = sql_store_dimension(
1162 + &rd->metric_uuid,
1163 + &rd->rrdset->chart_uuid,
1164 + string2str(rd->id),
1165 + string2str(rd->name),
1166 + rd->multiplier,
1167 + rd->divisor,
1168 + rd->algorithm);
1169 +
1170 + if (unlikely(rc))
1171 + error_report("Failed to store dimension %s", rrddim_id(rd));
1172 +
1173 + dictionary_acquired_item_release(rd->rrdset->rrddim_root_index, dict_item);
1174 + break;
1175 + case METADATA_DEL_DIMENSION:
1176 + uuid = (uuid_t *) cmd.param[0];
1177 + if (likely(dimension_can_be_deleted(uuid)))
1178 + delete_dimension_uuid(uuid);
1179 + freez(uuid);
1180 + break;
1181 + case METADATA_ADD_DIMENSION_OPTION:
1182 + dict_item = (DICTIONARY_ITEM * ) cmd.param[0];
1183 + rd = (RRDDIM *) dictionary_acquired_item_value(dict_item);
1184 + rc = sql_set_dimension_option(
1185 + &rd->metric_uuid, rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN) ? "hidden" : NULL);
1186 + if (unlikely(rc))
1187 + error_report("Failed to store dimension option for %s", string2str(rd->id));
1188 + dictionary_acquired_item_release(rd->rrdset->rrddim_root_index, dict_item);
1189 + break;
1190 + case METADATA_ADD_HOST_SYSTEM_INFO:
1191 + buffer = (BUFFER *) cmd.param[0];
1192 + db_execute(buffer_tostring(buffer));
1193 + buffer_free(buffer);
1194 + break;
1195 + case METADATA_ADD_HOST_INFO:
1196 + dict_item = (DICTIONARY_ITEM * ) cmd.param[0];
1197 + host = (RRDHOST *) dictionary_acquired_item_value(dict_item);
1198 + rc = sql_store_host_info(host);
1199 + if (unlikely(rc))
1200 + error_report("Failed to store host info in the database for %s", string2str(host->hostname));
1201 + dictionary_acquired_item_release(rrdhost_root_index, dict_item);
1202 + break;
1203 + case METADATA_STORE_CLAIM_ID:
1204 + store_claim_id((uuid_t *) cmd.param[0], (uuid_t *) cmd.param[1]);
1205 + freez((void *) cmd.param[0]);
1206 + freez((void *) cmd.param[1]);
1207 + break;
1208 + case METADATA_STORE_HOST_LABELS:
1209 + dict_item = (DICTIONARY_ITEM * ) cmd.param[0];
1210 + host = (RRDHOST *) dictionary_acquired_item_value(dict_item);
1211 + rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid);
1212 +
1213 + if (likely(rc == SQLITE_OK)) {
1214 + buffer_flush(work_buffer);
1215 + struct query_build tmp = {.sql = work_buffer, .count = 0};
1216 + uuid_unparse_lower(host->host_uuid, tmp.uuid_str);
1217 + rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
1218 + db_execute(buffer_tostring(work_buffer));
1219 + }
1220 +
1221 + dictionary_acquired_item_release(rrdhost_root_index, dict_item);
1222 + break;
1223 +
1224 + case METADATA_SCAN_HOSTS:
1225 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SCANNING_HOSTS)))
1226 + break;
1227 +
1228 + struct scan_metadata_payload *data = mallocz(sizeof(*data));
1229 + data->request.data = data;
1230 + data->wc = wc;
1231 + data->completion = cmd.completion; // Completion by the worker
1232 +
1233 + if (unlikely(cmd.completion)) {
1234 + data->max_count = 0; // 0 will process all pending updates
1235 + cmd.completion = NULL; // Do not complete after launching worker (worker will do)
1236 + }
1237 + else
1238 + data->max_count = 1000;
1239 +
1240 + metadata_flag_set(wc, METADATA_FLAG_SCANNING_HOSTS);
1241 + if (unlikely(
1242 + uv_queue_work(loop,&data->request,
1243 + start_metadata_hosts,
1244 + after_metadata_hosts))) {
1245 + // Failed to launch worker -- let the event loop handle completion
1246 + cmd.completion = data->completion;
1247 + freez(data);
1248 + metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
1249 + }
1250 + break;
1251 + case METADATA_STORE_BUFFER:
1252 + buffer = (BUFFER *) cmd.param[0];
1253 + db_execute(buffer_tostring(buffer));
1254 + buffer_free(buffer);
1255 + break;
1256 + case METADATA_MAINTENANCE:
1257 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_CLEANUP)))
1258 + break;
1259 +
1260 + metadata_cleanup_worker.data = wc;
1261 + metadata_flag_set(wc, METADATA_FLAG_CLEANUP);
1262 + if (unlikely(
1263 + uv_queue_work(loop, &metadata_cleanup_worker, start_metadata_cleanup, after_metadata_cleanup))) {
1264 + metadata_flag_clear(wc, METADATA_FLAG_CLEANUP);
1265 + }
1266 + break;
1267 + case METADATA_UNITTEST:;
1268 + struct thread_unittest *tu = (struct thread_unittest *) cmd.param[0];
1269 + sleep_usec(1000); // processing takes 1ms
1270 + __atomic_fetch_add(&tu->processed, 1, __ATOMIC_SEQ_CST);
1271 + break;
1272 + default:
1273 + break;
1274 + }
1275 + if (in_transaction && (commands_in_transaction >= METADATA_MAX_TRANSACTION_BATCH || opcode != next_opcode)) {
1276 + in_transaction = 0;
1277 + db_execute("COMMIT TRANSACTION;");
1278 + commands_in_transaction = 0;
1279 + }
1280 +
1281 + if (cmd.completion)
1282 + completion_mark_complete(cmd.completion);
1283 + } while (opcode != METADATA_DATABASE_NOOP);
1284 + }
1285 +
1286 + if (!uv_timer_stop(&wc->timer_req))
1287 + uv_close((uv_handle_t *)&wc->timer_req, NULL);
1288 +
1289 + /*
1290 + * uv_async_send after uv_close does not seem to crash in linux at the moment,
1291 + * it is however undocumented behaviour we need to be aware if this becomes
1292 + * an issue in the future.
1293 + */
1294 + uv_close((uv_handle_t *)&wc->async, NULL);
1295 + uv_run(loop, UV_RUN_DEFAULT);
1296 +
1297 + uv_cond_destroy(&wc->cmd_cond);
1298 + /* uv_mutex_destroy(&wc->cmd_mutex); */
1299 + //fatal_assert(0 == uv_loop_close(loop));
1300 + int rc;
1301 +
1302 + do {
1303 + rc = uv_loop_close(loop);
1304 + } while (rc != UV_EBUSY);
1305 +
1306 + freez(loop);
1307 + worker_unregister();
1308 +
1309 + buffer_free(work_buffer);
1310 + info("METADATA: Shutting down event loop");
1311 + completion_mark_complete(&wc->init_complete);
1312 + return;
1313 +
1314 +error_after_timer_init:
1315 + uv_close((uv_handle_t *)&wc->async, NULL);
1316 +error_after_async_init:
1317 + fatal_assert(0 == uv_loop_close(loop));
1318 +error_after_loop_init:
1319 + freez(loop);
1320 + worker_unregister();
1321 +}
1322 +
1323 +struct metadata_wc metasync_worker;
1324 +
1325 +void metadata_sync_shutdown(void)
1326 +{
1327 + completion_init(&metasync_worker.init_complete);
1328 +
1329 + struct metadata_cmd cmd;
1330 + memset(&cmd, 0, sizeof(cmd));
1331 + info("METADATA: Sending a shutdown command");
1332 + cmd.opcode = METADATA_SYNC_SHUTDOWN;
1333 + metadata_enq_cmd(&metasync_worker, &cmd);
1334 +
1335 + /* wait for metadata thread to shut down */
1336 + info("METADATA: Waiting for shutdown ACK");
1337 + completion_wait_for(&metasync_worker.init_complete);
1338 + completion_destroy(&metasync_worker.init_complete);
1339 + info("METADATA: Shutdown complete");
1340 +}
1341 +
1342 +void metadata_sync_shutdown_prepare(void)
1343 +{
1344 + struct metadata_cmd cmd;
1345 + memset(&cmd, 0, sizeof(cmd));
1346 +
1347 + struct completion compl;
1348 + completion_init(&compl);
1349 +
1350 + info("METADATA: Sending a scan host command");
1351 + uint32_t max_wait_iterations = 2000;
1352 + while (unlikely(metadata_flag_check(&metasync_worker, METADATA_FLAG_SCANNING_HOSTS)) && max_wait_iterations--) {
1353 + if (max_wait_iterations == 1999)
1354 + info("METADATA: Current worker is running; waiting to finish");
1355 + sleep_usec(1000);
1356 + }
1357 +
1358 + cmd.opcode = METADATA_SCAN_HOSTS;
1359 + cmd.completion = &compl;
1360 + metadata_enq_cmd(&metasync_worker, &cmd);
1361 +
1362 + info("METADATA: Waiting for host scan completion");
1363 + completion_wait_for(&compl);
1364 + completion_destroy(&compl);
1365 + info("METADATA: Host scan complete; can continue with shutdown");
1366 +}
1367 +
1368 +// -------------------------------------------------------------
1369 +// Init function called on agent startup
1370 +
1371 +void metadata_sync_init(void)
1372 +{
1373 + struct metadata_wc *wc = &metasync_worker;
1374 +
1375 + fatal_assert(0 == uv_mutex_init(&metadata_async_lock));
1376 +
1377 + memset(wc, 0, sizeof(*wc));
1378 + metadata_init_cmd_queue(wc);
1379 + completion_init(&wc->init_complete);
1380 +
1381 + fatal_assert(0 == uv_thread_create(&(wc->thread), metadata_event_loop, wc));
1382 +
1383 + completion_wait_for(&wc->init_complete);
1384 + completion_destroy(&wc->init_complete);
1385 +
1386 + info("SQLite metadata sync initialization complete");
1387 +}
1388 +
1389 +
1390 +// Helpers
1391 +
1392 +static inline void queue_metadata_cmd(enum metadata_opcode opcode, const void *param0, const void *param1)
1393 +{
1394 + struct metadata_cmd cmd;
1395 + cmd.opcode = opcode;
1396 + cmd.param[0] = param0;
1397 + cmd.param[1] = param1;
1398 + cmd.completion = NULL;
1399 + metadata_enq_cmd(&metasync_worker, &cmd);
1400 +
1401 +}
1402 +
1403 +// Public
1404 +void metaqueue_chart_update(RRDSET *st)
1405 +{
1406 + const DICTIONARY_ITEM *acquired_st = dictionary_get_and_acquire_item(st->rrdhost->rrdset_root_index, string2str(st->id));
1407 + queue_metadata_cmd(METADATA_ADD_CHART, acquired_st, NULL);
1408 +}
1409 +
1410 +//
1411 +// RD may not be collected, so we may store it needlessly
1412 +void metaqueue_dimension_update(RRDDIM *rd)
1413 +{
1414 + const DICTIONARY_ITEM *acquired_rd =
1415 + dictionary_get_and_acquire_item(rd->rrdset->rrddim_root_index, string2str(rd->id));
1416 +
1417 + if (unlikely(rrdset_flag_check(rd->rrdset, RRDSET_FLAG_METADATA_UPDATE))) {
1418 + metaqueue_chart_update(rd->rrdset);
1419 + rrdset_flag_clear(rd->rrdset, RRDSET_FLAG_METADATA_UPDATE);
1420 + }
1421 +
1422 + queue_metadata_cmd(METADATA_ADD_DIMENSION, acquired_rd, NULL);
1423 +}
1424 +
1425 +void metaqueue_dimension_update_flags(RRDDIM *rd)
1426 +{
1427 + const DICTIONARY_ITEM *acquired_rd =
1428 + dictionary_get_and_acquire_item(rd->rrdset->rrddim_root_index, string2str(rd->id));
1429 + queue_metadata_cmd(METADATA_ADD_DIMENSION_OPTION, acquired_rd, NULL);
1430 +}
1431 +
1432 +void metaqueue_host_update_system_info(RRDHOST *host)
1433 +{
1434 + BUFFER *work_buffer = sql_store_host_system_info(host);
1435 +
1436 + if (unlikely(!work_buffer))
1437 + return;
1438 +
1439 + queue_metadata_cmd(METADATA_ADD_HOST_SYSTEM_INFO, work_buffer, NULL);
1440 +}
1441 +
1442 +void metaqueue_host_update_info(const char *machine_guid)
1443 +{
1444 + const DICTIONARY_ITEM *acquired_host = dictionary_get_and_acquire_item(rrdhost_root_index, machine_guid);
1445 + queue_metadata_cmd(METADATA_ADD_HOST_INFO, acquired_host, NULL);
1446 +}
1447 +
1448 +void metaqueue_delete_dimension_uuid(uuid_t *uuid)
1449 +{
1450 + uuid_t *use_uuid = mallocz(sizeof(*uuid));
1451 + uuid_copy(*use_uuid, *uuid);
1452 + queue_metadata_cmd(METADATA_DEL_DIMENSION, use_uuid, NULL);
1453 +}
1454 +
1455 +void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid)
1456 +{
1457 + if (unlikely(!host_uuid))
1458 + return;
1459 +
1460 + uuid_t *local_host_uuid = mallocz(sizeof(*host_uuid));
1461 + uuid_t *local_claim_uuid = NULL;
1462 +
1463 + uuid_copy(*local_host_uuid, *host_uuid);
1464 + if (likely(claim_uuid)) {
1465 + local_claim_uuid = mallocz(sizeof(*claim_uuid));
1466 + uuid_copy(*local_claim_uuid, *claim_uuid);
1467 + }
1468 + queue_metadata_cmd(METADATA_STORE_CLAIM_ID, local_host_uuid, local_claim_uuid);
1469 +}
1470 +
1471 +void metaqueue_store_host_labels(const char *machine_guid)
1472 +{
1473 + const DICTIONARY_ITEM *acquired_host = dictionary_get_and_acquire_item(rrdhost_root_index, machine_guid);
1474 + queue_metadata_cmd(METADATA_STORE_HOST_LABELS, acquired_host, NULL);
1475 +}
1476 +
1477 +void metaqueue_buffer(BUFFER *buffer)
1478 +{
1479 + queue_metadata_cmd(METADATA_STORE_BUFFER, buffer, NULL);
1480 +}
1481 +
1482 +void metaqueue_chart_labels(RRDSET *st)
1483 +{
1484 + const DICTIONARY_ITEM *acquired_st = dictionary_get_and_acquire_item(st->rrdhost->rrdset_root_index, string2str(st->id));
1485 + queue_metadata_cmd(METADATA_ADD_CHART_LABEL, acquired_st, NULL);
1486 +}
1487 +
1488 +
1489 +//
1490 +// unitests
1491 +//
1492 +
1493 +static void *unittest_queue_metadata(void *arg) {
1494 + struct thread_unittest *tu = arg;
1495 +
1496 + struct metadata_cmd cmd;
1497 + cmd.opcode = METADATA_UNITTEST;
1498 + cmd.param[0] = tu;
1499 + cmd.param[1] = NULL;
1500 + cmd.completion = NULL;
1501 + metadata_enq_cmd(&metasync_worker, &cmd);
1502 +
1503 + do {
1504 + __atomic_fetch_add(&tu->added, 1, __ATOMIC_SEQ_CST);
1505 + metadata_enq_cmd(&metasync_worker, &cmd);
1506 + sleep_usec(10000);
1507 + } while (!__atomic_load_n(&tu->join, __ATOMIC_RELAXED));
1508 + return arg;
1509 +}
1510 +
1511 +static void *metadata_unittest_threads(void)
1512 +{
1513 +
1514 + unsigned done;
1515 +
1516 + struct thread_unittest tu = {
1517 + .join = 0,
1518 + .added = 0,
1519 + .processed = 0,
1520 + .done = &done,
1521 + };
1522 +
1523 + // Queue messages / Time it
1524 + time_t seconds_to_run = 5;
1525 + int threads_to_create = 4;
1526 + fprintf(
1527 + stderr,
1528 + "\nChecking metadata queue using %d threads for %ld seconds...\n",
1529 + threads_to_create,
1530 + seconds_to_run);
1531 +
1532 + netdata_thread_t threads[threads_to_create];
1533 + tu.join = 0;
1534 + for (int i = 0; i < threads_to_create; i++) {
1535 + char buf[100 + 1];
1536 + snprintf(buf, 100, "meta%d", i);
1537 + netdata_thread_create(
1538 + &threads[i],
1539 + buf,
1540 + NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE,
1541 + unittest_queue_metadata,
1542 + &tu);
1543 + }
1544 + uv_async_send(&metasync_worker.async);
1545 + sleep_usec(seconds_to_run * USEC_PER_SEC);
1546 +
1547 + __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED);
1548 + for (int i = 0; i < threads_to_create; i++) {
1549 + void *retval;
1550 + netdata_thread_join(threads[i], &retval);
1551 + }
1552 +// uv_async_send(&metasync_worker.async);
1553 + sleep_usec(5 * USEC_PER_SEC);
1554 +
1555 + fprintf(stderr, "Added %u elements, processed %u\n", tu.added, tu.processed);
1556 +
1557 + return 0;
1558 +}
1559 +
1560 +int metadata_unittest(void)
1561 +{
1562 + metadata_sync_init();
1563 +
1564 + // Queue items for a specific period of time
1565 + metadata_unittest_threads();
1566 +
1567 + fprintf(stderr, "Items still in queue %u\n", metasync_worker.queue_size);
1568 + metadata_sync_shutdown();
1569 +
1570 + return 0;
1571 +}
database/sqlite/sqlite_metadata.h new
+28
@@ -0,0 +1,28 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_SQLITE_METADATA_H
4 +#define NETDATA_SQLITE_METADATA_H
5 +
6 +#include "sqlite3.h"
7 +#include "sqlite_functions.h"
8 +
9 +// To initialize and shutdown
10 +void metadata_sync_init(void);
11 +void metadata_sync_shutdown(void);
12 +void metadata_sync_shutdown_prepare(void);
13 +
14 +void metaqueue_dimension_update(RRDDIM *rd);
15 +void metaqueue_chart_update(RRDSET *st);
16 +void metaqueue_dimension_update_flags(RRDDIM *rd);
17 +void metaqueue_host_update_system_info(RRDHOST *host);
18 +void metaqueue_host_update_info(const char *machine_guid);
19 +void metaqueue_delete_dimension_uuid(uuid_t *uuid);
20 +void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid);
21 +void metaqueue_store_host_labels(const char *machine_guid);
22 +void metaqueue_chart_labels(RRDSET *st);
23 +void migrate_localhost(uuid_t *host_uuid);
24 +void metaqueue_buffer(BUFFER *buffer);
25 +
26 +// UNIT TEST
27 +int metadata_unittest(void);
28 +#endif //NETDATA_SQLITE_METADATA_H
libnetdata/log/log.h
+1
@@ -45,6 +45,7 @@ extern "C" {
45 #define D_ACLK 0x0000000200000000
46 #define D_METADATALOG 0x0000000400000000
47 #define D_ACLK_SYNC 0x0000000800000000
48 +#define D_META_SYNC 0x0000002000000000
49 #define D_SYSTEM 0x8000000000000000
50
51 extern int web_server_is_multithreaded;
streaming/receiver.c
+1 -1
@@ -159,7 +159,7 @@ PARSER_RC streaming_claimed_id(char **words, void *user, PLUGINSD_ACTION *plugin
159 freez(host->aclk_state.claimed_id);
160 host->aclk_state.claimed_id = strcmp(words[2], "NULL") ? strdupz(words[2]) : NULL;
161
162 - store_claim_id(&host->host_uuid, host->aclk_state.claimed_id ? &uuid : NULL);
162 + metaqueue_store_claim_id(&host->host_uuid, host->aclk_state.claimed_id ? &uuid : NULL);
163
164 rrdhost_aclk_state_unlock(host);
165