@cryptotaxi247 / netdata-1 / commits / 18e729d67

Code improvements (#16104)

* Remove unused functions * No need for prepare statement because the function is not used frequently * Remove db_meta check, already assumed valid * Remove D_ACLK_SYNC and D_METADATALOG, fix log message * Reuse prepared statements per run to avoid sql parsing all the time * Keep rowid in charts and dimensions * Host and chart labels keep rowids * Don't store internal flags * Remove commented out code * Formatting * Fix algorithm when updating dimension

Stelios Fragkakis committed Oct 6, 2023 at 16:33 UTC 18e729d67007315368899fa3d6bc4deef7c4936c
5 files changed +149 -171
database/sqlite/sqlite_aclk.c
+2 -14
@@ -65,9 +65,7 @@ static void aclk_database_enq_cmd(struct aclk_database_cmd *cmd)
65 uv_mutex_unlock(&aclk_sync_config.cmd_mutex);
66
67 /* wake up event loop */
68 - int rc = uv_async_send(&aclk_sync_config.async);
69 - if (unlikely(rc))
70 - netdata_log_debug(D_ACLK_SYNC, "Failed to wake up event loop");
68 + (void) uv_async_send(&aclk_sync_config.async);
69 }
70
71 enum {
@@ -226,14 +224,8 @@ static void sql_delete_aclk_table_list(char *host_guid)
224 uuid_unparse_lower(host_uuid, host_str);
225 uuid_unparse_lower_fix(&host_uuid, uuid_str);
226
229 - netdata_log_debug(D_ACLK_SYNC, "Checking if I should delete aclk tables for node %s", host_str);
230 -
231 - if (is_host_available(&host_uuid)) {
232 - netdata_log_debug(D_ACLK_SYNC, "Host %s exists, not deleting aclk sync tables", host_str);
227 + if (is_host_available(&host_uuid))
228 return;
234 - }
235 -
236 - netdata_log_debug(D_ACLK_SYNC, "Host %s does NOT exist, can delete aclk sync tables", host_str);
229
230 sqlite3_stmt *res = NULL;
231 BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, &netdata_buffers_statistics.buffers_sqlite);
@@ -265,7 +257,6 @@ fail:
257
258 static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unused, char **argv __maybe_unused, char **column __maybe_unused)
259 {
268 - netdata_log_debug(D_ACLK_SYNC,"Scheduling aclk sync table check for node %s", (char *) argv[0]);
260 struct aclk_database_cmd cmd;
261 memset(&cmd, 0, sizeof(cmd));
262 cmd.opcode = ACLK_DATABASE_DELETE_HOST;
@@ -280,7 +271,6 @@ static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unus
271 static void sql_check_aclk_table_list(void)
272 {
273 char *err_msg = NULL;
283 - netdata_log_debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
274 int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, NULL, &err_msg);
275 if (rc != SQLITE_OK) {
276 error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
@@ -305,7 +295,6 @@ static int sql_maint_aclk_sync_database(void *data __maybe_unused, int argc __ma
295 static void sql_maint_aclk_sync_database_all(void)
296 {
297 char *err_msg = NULL;
308 - netdata_log_debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
298 int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ALERT_LIST, sql_maint_aclk_sync_database, NULL, &err_msg);
299 if (rc != SQLITE_OK) {
300 error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
@@ -444,7 +433,6 @@ static void aclk_synchronization(void *arg __maybe_unused)
433 sql_process_queue_removed_alerts_to_aclk(cmd.param[0]);
434 break;
435 default:
447 - netdata_log_debug(D_ACLK_SYNC, "%s: default.", __func__);
436 break;
437 }
438 if (cmd.completion)
database/sqlite/sqlite_functions.c
+13 -62
@@ -185,18 +185,10 @@ static void recover_database(const char *sqlite_database, const char *new_sqlite
185 int execute_insert(sqlite3_stmt *res)
186 {
187 int rc;
188 - int cnt = 0;
189 - while ((rc = sqlite3_step_monitored(res)) != SQLITE_DONE && ++cnt < SQL_MAX_RETRY && likely(!netdata_exit)) {
190 - if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) {
191 - usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
192 - error_report("Failed to insert/update, rc = %d -- attempt %d", rc, cnt);
193 - }
194 - else {
195 - if (rc == SQLITE_CORRUPT)
196 - (void) mark_database_to_recover(res, NULL);
197 - error_report("SQLite error %d", rc);
198 - break;
199 - }
188 + rc = sqlite3_step_monitored(res);
189 + if (rc == SQLITE_CORRUPT) {
190 + (void)mark_database_to_recover(res, NULL);
191 + error_report("SQLite error %d", rc);
192 }
193 return rc;
194 }
@@ -320,7 +312,6 @@ int init_database_batch(sqlite3 *database, const char *batch[])
312 int rc;
313 char *err_msg = NULL;
314 for (int i = 0; batch[i]; i++) {
323 - netdata_log_debug(D_METADATALOG, "Executing %s", batch[i]);
315 rc = sqlite3_exec_monitored(database, batch[i], 0, 0, &err_msg);
316 if (rc != SQLITE_OK) {
317 error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg);
@@ -496,7 +487,7 @@ int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
487
488 rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
489 if (unlikely(rc != SQLITE_OK)) {
499 - error_report("Failed to bind host parameter to %s, rc = %d", sql, rc);
490 + error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc);
491 goto skip;
492 }
493
@@ -618,44 +609,6 @@ failed:
609 return rc - 1;
610 }
611
621 -#define SQL_SELECT_HOST_BY_NODE_ID "select host_id from node_instance where node_id = @node_id;"
622 -
623 -int get_host_id(uuid_t *node_id, uuid_t *host_id)
624 -{
625 - static __thread sqlite3_stmt *res = NULL;
626 - int rc;
627 -
628 - if (unlikely(!db_meta)) {
629 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
630 - error_report("Database has not been initialized");
631 - return 1;
632 - }
633 -
634 - if (unlikely(!res)) {
635 - rc = prepare_statement(db_meta, SQL_SELECT_HOST_BY_NODE_ID, &res);
636 - if (unlikely(rc != SQLITE_OK)) {
637 - error_report("Failed to prepare statement to select node instance information for a node");
638 - return 1;
639 - }
640 - }
641 -
642 - rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC);
643 - if (unlikely(rc != SQLITE_OK)) {
644 - error_report("Failed to bind host_id parameter to select node instance information");
645 - goto failed;
646 - }
647 -
648 - rc = sqlite3_step_monitored(res);
649 - if (likely(rc == SQLITE_ROW && host_id))
650 - uuid_copy(*host_id, *((uuid_t *) sqlite3_column_blob(res, 0)));
651 -
652 -failed:
653 - if (unlikely(sqlite3_reset(res) != SQLITE_OK))
654 - error_report("Failed to reset the prepared statement when selecting node instance information");
655 -
656 - return (rc == SQLITE_ROW) ? 0 : -1;
657 -}
658 -
612 #define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL;"
613
614 int get_node_id(uuid_t *host_id, uuid_t *node_id)
@@ -811,7 +764,7 @@ failed:
764
765 void sql_load_node_id(RRDHOST *host)
766 {
814 - static __thread sqlite3_stmt *res = NULL;
767 + sqlite3_stmt *res = NULL;
768 int rc;
769
770 if (unlikely(!db_meta)) {
@@ -820,13 +773,11 @@ void sql_load_node_id(RRDHOST *host)
773 return;
774 }
775
823 - if (unlikely(!res)) {
824 - rc = prepare_statement(db_meta, SQL_GET_HOST_NODE_ID, &res);
825 - if (unlikely(rc != SQLITE_OK)) {
826 - error_report("Failed to prepare statement to fetch node id");
827 - return;
828 - };
829 - }
776 + rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);
777 + if (unlikely(rc != SQLITE_OK)) {
778 + error_report("Failed to prepare statement to fetch node id");
779 + return;
780 + };
781
782 rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
783 if (unlikely(rc != SQLITE_OK)) {
@@ -843,8 +794,8 @@ void sql_load_node_id(RRDHOST *host)
794 }
795
796 failed:
846 - if (unlikely(sqlite3_reset(res) != SQLITE_OK))
847 - error_report("Failed to reset the prepared statement when loading node instance information");
797 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
798 + error_report("Failed to finalize the prepared statement when loading node instance information");
799 };
800
801
database/sqlite/sqlite_functions.h
-2
@@ -62,10 +62,8 @@ void initialize_thread_key_pool(void);
62
63 // Look up functions
64 int get_node_id(uuid_t *host_id, uuid_t *node_id);
65 -int get_host_id(uuid_t *node_id, uuid_t *host_id);
65 struct node_instance_list *get_node_list(void);
66 void sql_load_node_id(RRDHOST *host);
68 -char *get_hostname_by_node_id(char *node_id);
67
68 // Help build archived hosts in memory when agent starts
69 void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);
database/sqlite/sqlite_metadata.c
+134 -90
@@ -11,37 +11,44 @@
11 #define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid;"
12
13 #define STORE_HOST_LABEL \
14 - "INSERT OR REPLACE INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "
14 + "INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "
15
16 #define STORE_CHART_LABEL \
17 - "INSERT OR REPLACE INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "
17 + "INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "
18
19 #define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"
20
21 #define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid;"
22
23 -#define SQL_STORE_HOST_INFO \
24 - "INSERT OR REPLACE INTO host " \
25 - "(host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, memory_mode, " \
26 - "abbrev_timezone, utc_offset, program_name, program_version," \
27 - "entries, health_enabled, last_connected) " \
28 - "VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, @memory_mode, " \
29 - "@abbrev_timezone, @utc_offset, @program_name, @program_version, " \
30 - "@entries, @health_enabled, @last_connected);"
31 -
32 -#define SQL_STORE_CHART "insert or replace into chart (chart_id, host_id, type, id, " \
33 - "name, family, context, title, unit, plugin, module, priority, update_every , chart_type , memory_mode , " \
34 - "history_entries) values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16);"
35 -
36 -#define SQL_STORE_DIMENSION "INSERT OR REPLACE INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \
37 - "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options);"
23 +#define SQL_STORE_HOST_INFO \
24 + "INSERT OR REPLACE INTO host (host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, " \
25 + "memory_mode, abbrev_timezone, utc_offset, program_name, program_version, entries, health_enabled, last_connected) " \
26 + "VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, " \
27 + "@memory_mode, @abbrev_tz, @utc_offset, @prog_name, @prog_version, @entries, @health_enabled, @last_connected);"
28 +
29 +#define SQL_STORE_CHART \
30 + "INSERT INTO chart (chart_id, host_id, type, id, name, family, context, title, unit, plugin, module, priority, " \
31 + "update_every, chart_type, memory_mode, history_entries) " \
32 + "values (@chart_id, @host_id, @type, @id, @name, @family, @context, @title, @unit, @plugin, @module, @priority, " \
33 + "@update_every, @chart_type, @memory_mode, @history_entries) " \
34 + "ON CONFLICT(chart_id) DO UPDATE SET type=excluded.type, id=excluded.id, name=excluded.name, " \
35 + "family=excluded.family, context=excluded.context, title=excluded.title, unit=excluded.unit, " \
36 + "plugin=excluded.plugin, module=excluded.module, priority=excluded.priority, update_every=excluded.update_every, " \
37 + "chart_type=excluded.chart_type, memory_mode = excluded.memory_mode, history_entries = excluded.history_entries"
38 +
39 +#define SQL_STORE_DIMENSION \
40 + "INSERT INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \
41 + "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options) " \
42 + "ON CONFLICT(dim_id) DO UPDATE SET id=excluded.id, name=excluded.name, multiplier=excluded.multiplier, " \
43 + "divisor=excluded.divisor, algorithm=excluded.algorithm, options=excluded.options"
44
45 #define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id"
46 #define SELECT_CHART_LIST "SELECT chart_id, rowid FROM chart WHERE rowid > @row_id"
47 #define SELECT_CHART_LABEL_LIST "SELECT chart_id, rowid FROM chart_label WHERE rowid > @row_id"
48
43 -#define SQL_STORE_HOST_SYSTEM_INFO_VALUES "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \
44 - "(@uuid, @name, @value, unixepoch())"
49 +#define SQL_STORE_HOST_SYSTEM_INFO_VALUES \
50 + "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \
51 + "(@uuid, @name, @value, UNIXEPOCH())"
52
53 #define MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID \
54 "UPDATE chart SET host_id = @host_id WHERE host_id in (SELECT host_id FROM host where host_id <> @host_id and hops = 0);"
@@ -145,7 +152,7 @@ static int host_label_store_to_sql_callback(const char *name, const char *value,
152 buffer_sprintf(lb->sql, STORE_HOST_LABEL);
153 else
154 buffer_strcat(lb->sql, ", ");
148 - buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int)ls & ~(RRDLABEL_FLAG_INTERNAL), name, value);
155 + buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);
156 lb->count++;
157 return 1;
158 }
@@ -156,7 +163,7 @@ static int chart_label_store_to_sql_callback(const char *name, const char *value
163 buffer_sprintf(lb->sql, STORE_CHART_LABEL);
164 else
165 buffer_strcat(lb->sql, ", ");
159 - buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, ls, name, value);
166 + buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);
167 lb->count++;
168 return 1;
169 }
@@ -190,6 +197,7 @@ static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t
197 struct query_build tmp = {.sql = work_buffer, .count = 0};
198 uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
199 rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
200 + buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
201 int rc = db_execute(db_meta, buffer_tostring(work_buffer));
202 if (likely(!rc)) {
203 st->rrdlabels_last_saved_version = new_version;
@@ -257,7 +265,7 @@ failed:
265 return rc != SQLITE_DONE;
266 }
267
260 -static void delete_dimension_uuid(uuid_t *dimension_uuid, bool flag __maybe_unused)
268 +static void delete_dimension_uuid(uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused)
269 {
270 static __thread sqlite3_stmt *res = NULL;
271 int rc;
@@ -270,7 +278,7 @@ static void delete_dimension_uuid(uuid_t *dimension_uuid, bool flag __maybe_unus
278 }
279 }
280
273 - rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);
281 + rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);
282 if (unlikely(rc != SQLITE_OK))
283 goto skip_execution;
284
@@ -291,13 +299,6 @@ static int store_host_metadata(RRDHOST *host)
299 static __thread sqlite3_stmt *res = NULL;
300 int rc, param = 0;
301
294 - if (unlikely(!db_meta)) {
295 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
296 - return 0;
297 - error_report("Database has not been initialized");
298 - return 1;
299 - }
300 -
302 if (unlikely((!res))) {
303 rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res);
304 if (unlikely(rc != SQLITE_OK)) {
@@ -483,13 +484,6 @@ static int store_chart_metadata(RRDSET *st)
484 static __thread sqlite3_stmt *res = NULL;
485 int rc, param = 0, store_rc = 0;
486
486 - if (unlikely(!db_meta)) {
487 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
488 - return 0;
489 - error_report("Database has not been initialized");
490 - return 1;
491 - }
492 -
487 if (unlikely(!res)) {
488 rc = prepare_statement(db_meta, SQL_STORE_CHART, &res);
489 if (unlikely(rc != SQLITE_OK)) {
@@ -592,13 +586,6 @@ static int store_dimension_metadata(RRDDIM *rd)
586 static __thread sqlite3_stmt *res = NULL;
587 int rc, param = 0;
588
595 - if (unlikely(!db_meta)) {
596 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
597 - return 0;
598 - error_report("Database has not been initialized");
599 - return 1;
600 - }
601 -
589 if (unlikely(!res)) {
590 rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res);
591 if (unlikely(rc != SQLITE_OK)) {
@@ -659,7 +646,7 @@ bind_fail:
646 return 1;
647 }
648
662 -static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, bool flag __maybe_unused)
649 +static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused)
650 {
651 #ifdef ENABLE_DBENGINE
652 if(dbengine_enabled) {
@@ -716,17 +703,16 @@ int get_database_page_count(sqlite3 *database)
703 static bool run_cleanup_loop(
704 sqlite3_stmt *res,
705 struct metadata_wc *wc,
719 - bool (*check_cb)(uuid_t *uuid, bool check_flag __maybe_unused),
720 - void (*action_cb)(uuid_t *uuid, bool action_flag __maybe_unused),
706 + bool (*check_cb)(uuid_t *, sqlite3_stmt **, bool),
707 + void (*action_cb)(uuid_t *, sqlite3_stmt **, bool),
708 uint32_t *total_checked,
709 uint32_t *total_deleted,
723 - uint32_t run_threshold,
724 - uint32_t cleanup_threshold,
710 uint64_t *row_id,
711 + sqlite3_stmt **check_stmt,
712 + sqlite3_stmt **action_stmt,
713 bool check_flag,
714 bool action_flag)
715 {
729 -
716 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
717 return true;
718
@@ -736,41 +722,45 @@ static bool run_cleanup_loop(
722
723 time_t start_running = now_monotonic_sec();
724 bool time_expired = false;
739 - while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW && *total_deleted < cleanup_threshold &&
740 - *total_checked < cleanup_threshold) {
725 + while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW &&
726 + (*total_deleted < MAX_METADATA_CLEANUP && *total_checked < MAX_METADATA_CLEANUP)) {
727 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
728 break;
729
730 *row_id = sqlite3_column_int64(res, 1);
745 - rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_flag);
731 + rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag);
732
733 if (rc == true) {
748 - action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_flag);
734 + action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag);
735 (*total_deleted)++;
736 }
737
738 (*total_checked)++;
753 - time_expired = ((now_monotonic_sec() - start_running) > run_threshold);
739 + time_expired = ((now_monotonic_sec() - start_running) > METADATA_RUNTIME_THRESHOLD);
740 }
755 - return time_expired || (*total_deleted == cleanup_threshold) || (*total_checked == cleanup_threshold);
741 + return time_expired || (*total_checked == MAX_METADATA_CLEANUP) || (*total_deleted == MAX_METADATA_CLEANUP);
742 }
743
744
745 #define SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION "SELECT count(1) FROM dimension WHERE chart_id = @chart_id"
746 #define SQL_CHECK_CHART_EXISTENCE_IN_CHART "SELECT count(1) FROM chart WHERE chart_id = @chart_id"
747
762 -static bool chart_can_be_deleted(uuid_t *chart_uuid, bool check_in_dimension)
748 +static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension)
749 {
750 int rc, result = 1;
765 - sqlite3_stmt *res = NULL;
751 + sqlite3_stmt *res = check_res ? *check_res : NULL;
752
767 - if (check_in_dimension)
768 - rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION, -1, &res, 0);
769 - else
770 - rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_CHART, -1, &res, 0);
771 - if (unlikely(rc != SQLITE_OK)) {
772 - error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
773 - return 0;
753 + if (!res) {
754 + if (check_in_dimension)
755 + rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION, -1, &res, 0);
756 + else
757 + rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_CHART, -1, &res, 0);
758 + if (unlikely(rc != SQLITE_OK)) {
759 + error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
760 + return 0;
761 + }
762 + if (check_res)
763 + *check_res = res;
764 }
765
766 rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
@@ -784,20 +774,55 @@ static bool chart_can_be_deleted(uuid_t *chart_uuid, bool check_in_dimension)
774 result = sqlite3_column_int(res, 0);
775
776 skip:
787 - rc = sqlite3_finalize(res);
777 + if (check_res)
778 + rc = sqlite3_reset(res);
779 + else
780 + rc = sqlite3_finalize(res);
781 +
782 if (unlikely(rc != SQLITE_OK))
789 - error_report("Failed to finalize statement that checks chart uuid existence rc = %d", rc);
783 + error_report("Failed to %s statement that checks chart uuid existence rc = %d", check_res ? "reset" : "finalize", rc);
784 return result == 0;
785 }
786
787 #define SQL_DELETE_CHART_BY_UUID "DELETE FROM chart WHERE chart_id = @chart_id"
788 #define SQL_DELETE_CHART_LABEL_BY_UUID "DELETE FROM chart_label WHERE chart_id = @chart_id"
789
796 -static void delete_chart_uuid(uuid_t(*chart_uuid), bool label_only)
790 +static void delete_chart_uuid(uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only)
791 {
798 - if (label_only == false)
799 - (void) exec_statement_with_uuid(SQL_DELETE_CHART_BY_UUID, chart_uuid);
800 - (void) exec_statement_with_uuid(SQL_DELETE_CHART_LABEL_BY_UUID, chart_uuid);
792 + int rc;
793 + sqlite3_stmt *res = action_res ? *action_res : NULL;
794 +
795 + if (!res) {
796 + if (label_only)
797 + rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABEL_BY_UUID, -1, &res, 0);
798 + else
799 + rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_BY_UUID, -1, &res, 0);
800 + if (unlikely(rc != SQLITE_OK)) {
801 + error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
802 + return;
803 + }
804 + if (action_res)
805 + *action_res = res;
806 + }
807 +
808 + rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
809 + if (unlikely(rc != SQLITE_OK)) {
810 + error_report("Failed to bind chart uuid parameter, rc = %d", rc);
811 + goto skip;
812 + }
813 +
814 + rc = sqlite3_step_monitored(res);
815 + if (unlikely(rc != SQLITE_DONE))
816 + error_report("Failed to delete a chart uuid from the %s table, rc = %d", label_only ? "labels" : "chart", rc);
817 +
818 +skip:
819 + if (action_res)
820 + rc = sqlite3_reset(res);
821 + else
822 + rc = sqlite3_finalize(res);
823 +
824 + if (unlikely(rc != SQLITE_OK))
825 + error_report("Failed to %s statement that deletes a chart uuid rc = %d", action_res ? "reset" : "finalize", rc);
826 }
827
828 static void check_dimension_metadata(struct metadata_wc *wc)
@@ -827,21 +852,21 @@ static void check_dimension_metadata(struct metadata_wc *wc)
852
853 internal_error(true, "METADATA: Checking dimensions starting after row %"PRIu64, last_row_id);
854
830 - bool runtime_exceeded = run_cleanup_loop(
855 + bool more_to_do = run_cleanup_loop(
856 res,
857 wc,
858 dimension_can_be_deleted,
859 delete_dimension_uuid,
860 &total_checked,
861 &total_deleted,
837 - METADATA_RUNTIME_THRESHOLD,
838 - MAX_METADATA_CLEANUP,
862 &last_row_id,
863 + NULL,
864 + NULL,
865 false,
866 false);
867
868 now = now_realtime_sec();
844 - if (total_deleted > 0 || runtime_exceeded)
869 + if (more_to_do)
870 next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
871 else {
872 last_row_id = 0;
@@ -882,25 +907,33 @@ static void check_chart_metadata(struct metadata_wc *wc)
907 }
908
909 uint32_t total_checked = 0;
885 - uint32_t total_deleted= 0;
910 + uint32_t total_deleted = 0;
911
912 internal_error(true, "METADATA: Checking charts starting after row %"PRIu64, last_row_id);
913
889 - bool runtime_exceeded = run_cleanup_loop(
914 + sqlite3_stmt *check_res = NULL;
915 + sqlite3_stmt *action_res = NULL;
916 + bool more_to_do = run_cleanup_loop(
917 res,
918 wc,
919 chart_can_be_deleted,
920 delete_chart_uuid,
921 &total_checked,
922 &total_deleted,
896 - METADATA_RUNTIME_THRESHOLD,
897 - MAX_METADATA_CLEANUP,
923 &last_row_id,
924 + &check_res,
925 + &action_res,
926 true,
927 false);
928
929 + if (check_res)
930 + sqlite3_finalize(check_res);
931 +
932 + if (action_res)
933 + sqlite3_finalize(action_res);
934 +
935 now = now_realtime_sec();
903 - if (total_deleted > 0 || runtime_exceeded)
936 + if (more_to_do)
937 next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
938 else {
939 last_row_id = 0;
@@ -942,25 +975,34 @@ static void check_label_metadata(struct metadata_wc *wc)
975 }
976
977 uint32_t total_checked = 0;
945 - uint32_t total_deleted= 0;
978 + uint32_t total_deleted = 0;
979
980 internal_error(true,"METADATA: Checking charts labels starting after row %"PRIu64, last_row_id);
981
949 - bool runtime_exceeded = run_cleanup_loop(
982 + sqlite3_stmt *check_res = NULL;
983 + sqlite3_stmt *action_res = NULL;
984 +
985 + bool more_to_do = run_cleanup_loop(
986 res,
987 wc,
988 chart_can_be_deleted,
989 delete_chart_uuid,
990 &total_checked,
991 &total_deleted,
956 - METADATA_RUNTIME_THRESHOLD,
957 - MAX_METADATA_CLEANUP,
992 &last_row_id,
993 + &check_res,
994 + &action_res,
995 false,
996 true);
997
998 + if (check_res)
999 + sqlite3_finalize(check_res);
1000 +
1001 + if (action_res)
1002 + sqlite3_finalize(action_res);
1003 +
1004 now = now_realtime_sec();
963 - if (total_deleted > 0 || runtime_exceeded)
1005 + if (more_to_do)
1006 next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
1007 else {
1008 last_row_id = 0;
@@ -1334,7 +1376,7 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_trans
1376 uint32_t scan_count = 1;
1377
1378 if (use_transaction)
1337 - (void)db_execute(db_meta, "BEGIN TRANSACTION;");
1379 + (void)db_execute(db_meta, "BEGIN TRANSACTION");
1380
1381 rrdset_foreach_reentrant(st, host) {
1382 if (scan_count == max_count) {
@@ -1383,7 +1425,7 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_trans
1425 rrdset_foreach_done(st);
1426
1427 if (use_transaction)
1386 - (void)db_execute(db_meta, "COMMIT TRANSACTION;");
1428 + (void)db_execute(db_meta, "COMMIT TRANSACTION");
1429
1430 return more_to_do;
1431 }
@@ -1450,6 +1492,7 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
1492 struct query_build tmp = {.sql = work_buffer, .count = 0};
1493 uuid_unparse_lower(host->host_uuid, tmp.uuid_str);
1494 rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
1495 + buffer_strcat(work_buffer, " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
1496 rc = db_execute(db_meta, buffer_tostring(work_buffer));
1497
1498 if (unlikely(rc)) {
@@ -1600,13 +1643,14 @@ static void metadata_event_loop(void *arg)
1643
1644 case METADATA_ML_LOAD_MODELS: {
1645 RRDDIM *rd = (RRDDIM *) cmd.param[0];
1603 - ml_dimension_load_models(rd);
1646 + if (!shutdown)
1647 + ml_dimension_load_models(rd);
1648 break;
1649 }
1650 case METADATA_DEL_DIMENSION:
1651 uuid = (uuid_t *) cmd.param[0];
1608 - if (likely(dimension_can_be_deleted(uuid, false)))
1609 - delete_dimension_uuid(uuid, false);
1652 + if (likely(dimension_can_be_deleted(uuid, NULL, false)))
1653 + delete_dimension_uuid(uuid, NULL, false);
1654 freez(uuid);
1655 break;
1656 case METADATA_STORE_CLAIM_ID:
libnetdata/log/log.h
-3
@@ -43,9 +43,6 @@ extern "C" {
43 #define D_ANALYTICS 0x0000000080000000
44 #define D_RRDENGINE 0x0000000100000000
45 #define D_ACLK 0x0000000200000000
46 -#define D_METADATALOG 0x0000000400000000
47 -#define D_ACLK_SYNC 0x0000000800000000
48 -#define D_META_SYNC 0x0000001000000000
46 #define D_REPLICATION 0x0000002000000000
47 #define D_SYSTEM 0x8000000000000000
48