Migrate data when machine GUID changes (#13232)
* Add hops column to the host table * Allow store host to take hops count and store in the database * Add a function to check existance of a column in a table Add a generic function to be used as a on-select callback with a single column (integer) * During metadata log replay (to be obsoleted) store hops = 1 * Function now uses generic return_int_cb * Add migration functions v1 to v2 * Add migrate localhost * Allocate in-memory sqlite for unittests
Stelios Fragkakis committed
Jul 4, 2022 at 14:22 UTC
f429d1b0e45f6df326643c0a4f4f1df2d5061ab0
5 files changed
+113
-21
database/engine/metadata_log/metalogpluginsd.c
+1
-1
@@ -30,7 +30,7 @@ PARSER_RC metalog_pluginsd_host_action(
30
}
31
32
if (likely(!uuid_parse(machine_guid, state->host_uuid))) {
33
- int rc = sql_store_host(&state->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags);
33
+ int rc = sql_store_host(&state->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags, 1);
34
if (unlikely(rc)) {
35
errno = 0;
36
error("Failed to store host %s with UUID %s in the database", hostname, machine_guid);
database/rrdhost.c
+5
-2
@@ -308,7 +308,8 @@ RRDHOST *rrdhost_create(const char *hostname,
308
}
309
310
if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) {
311
- int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags);
311
+ int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags,
312
+ host->system_info ? host->system_info->hops : 0);
313
if (unlikely(rc))
314
error_report("Failed to store machine GUID to the database");
315
sql_load_node_id(host);
@@ -705,7 +706,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
706
config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
707
}
708
708
- if (unlikely(sql_init_database(DB_CHECK_NONE, 0))) {
709
+ if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
710
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
711
fatal("Failed to initialize SQLite");
712
info("Skipping SQLITE metadata initialization since memory mode is not db engine");
@@ -763,6 +764,8 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
764
fatal("Failed to initialize dbengine");
765
}
766
#endif
767
+ if (likely(system_info))
768
+ migrate_localhost(&localhost->host_uuid);
769
sql_aclk_sync_init();
770
rrd_unlock();
771
database/sqlite/sqlite_db_migration.c
+46
-10
@@ -2,6 +2,50 @@
2
3
#include "sqlite_db_migration.h"
4
5
+static int return_int_cb(void *data, int argc, char **argv, char **column)
6
+{
7
+ int *status = data;
8
+ UNUSED(argc);
9
+ UNUSED(column);
10
+ *status = str2uint32_t(argv[0]);
11
+ return 0;
12
+}
13
+
14
+static int column_exists_in_table(const char *table, const char *column)
15
+{
16
+ char *err_msg = NULL;
17
+ char sql[128];
18
+
19
+ int exists = 0;
20
+
21
+ snprintf(sql, 127, "SELECT 1 FROM pragma_table_info('%s') where name = '%s';", table, column);
22
+
23
+ int rc = sqlite3_exec(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
24
+ if (rc != SQLITE_OK) {
25
+ info("Error checking column existence; %s", err_msg);
26
+ sqlite3_free(err_msg);
27
+ }
28
+
29
+ return exists;
30
+}
31
+
32
+const char *database_migrate_v1_v2[] = {
33
+ "ALTER TABLE host ADD hops INTEGER;",
34
+ NULL
35
+};
36
+
37
+static int do_migration_v1_v2(sqlite3 *database, const char *name)
38
+{
39
+ UNUSED(database);
40
+ UNUSED(name);
41
+ info("Running database migration %s", name);
42
+
43
+ if (!column_exists_in_table("host", "hops"))
44
+ return init_database_batch(DB_CHECK_NONE, 0, &database_migrate_v1_v2[0]);
45
+ return 0;
46
+}
47
+
48
+
49
static int do_migration_noop(sqlite3 *database, const char *name)
50
{
51
UNUSED(database);
@@ -15,26 +59,18 @@ static struct database_func_migration_list {
59
int (*func)(sqlite3 *database, const char *name);
60
} migration_action[] = {
61
{.name = "v0 to v1", .func = do_migration_noop},
62
+ {.name = "v1 to v2", .func = do_migration_v1_v2},
63
// the terminator of this array
64
{.name = NULL, .func = NULL}
65
};
66
67
23
-static int perform_database_migration_cb(void *data, int argc, char **argv, char **column)
24
-{
25
- int *status = data;
26
- UNUSED(argc);
27
- UNUSED(column);
28
- *status = str2uint32_t(argv[0]);
29
- return 0;
30
-}
31
-
68
int perform_database_migration(sqlite3 *database, int target_version)
69
{
70
int user_version = 0;
71
char *err_msg = NULL;
72
37
- int rc = sqlite3_exec(database, "PRAGMA user_version;", perform_database_migration_cb, (void *) &user_version, &err_msg);
73
+ int rc = sqlite3_exec(database, "PRAGMA user_version;", return_int_cb, (void *) &user_version, &err_msg);
74
if (rc != SQLITE_OK) {
75
info("Error checking the database version; %s", err_msg);
76
sqlite3_free(err_msg);
database/sqlite/sqlite_functions.c
+55
-6
@@ -3,11 +3,11 @@
3
#include "sqlite_functions.h"
4
#include "sqlite_db_migration.h"
5
6
-#define DB_METADATA_VERSION 1
6
+#define DB_METADATA_VERSION 2
7
8
const char *database_config[] = {
9
"CREATE TABLE IF NOT EXISTS host(host_id blob PRIMARY KEY, hostname text, "
10
- "registry_hostname text, update_every int, os text, timezone text, tags text);",
10
+ "registry_hostname text, update_every int, os text, timezone text, tags text, hops INT);",
11
"CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "
12
"family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "
13
"chart_type int, memory_mode int, history_entries);",
@@ -345,7 +345,7 @@ static int attempt_database_fix()
345
return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT, 0);
346
}
347
348
-static int init_database_batch(int rebuild, int init_type, const char *batch[])
348
+int init_database_batch(int rebuild, int init_type, const char *batch[])
349
{
350
int rc;
351
char *err_msg = NULL;
@@ -437,7 +437,10 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
437
char buf[1024 + 1] = "";
438
const char *list[2] = { buf, NULL };
439
440
- int target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
440
+ int target_version = DB_METADATA_VERSION;
441
+
442
+ if (likely(!memory))
443
+ target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
444
445
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
446
// PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
@@ -731,11 +734,53 @@ uuid_t *create_chart_uuid(RRDSET *st, const char *id, const char *name)
734
return uuid;
735
}
736
734
-// Functions to create host, chart, dimension in the database
737
+static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
738
+{
739
+ int rc, result = 1;
740
+ sqlite3_stmt *res = NULL;
741
+
742
+ rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
743
+ if (unlikely(rc != SQLITE_OK)) {
744
+ error_report("Failed to prepare statement %s, rc = %d", sql, rc);
745
+ return 1;
746
+ }
747
+
748
+ rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
749
+ if (unlikely(rc != SQLITE_OK)) {
750
+ error_report("Failed to bind host parameter to %s, rc = %d", sql, rc);
751
+ goto failed;
752
+ }
753
+
754
+ rc = execute_insert(res);
755
+ if (likely(rc == SQLITE_DONE))
756
+ result = 0;
757
+ else
758
+ error_report("Failed to execute %s, rc = %d", sql, rc);
759
+
760
+failed:
761
+ rc = sqlite3_finalize(res);
762
+ if (unlikely(rc != SQLITE_OK))
763
+ error_report("Failed to finalize statement %s, rc = %d", sql, rc);
764
+ return result;
765
+}
766
+
767
+
768
+// Migrate all hosts with hops zero to this host_uuid
769
+void migrate_localhost(uuid_t *host_uuid)
770
+{
771
+ int rc;
772
+
773
+ rc = exec_statement_with_uuid("UPDATE chart SET host_id = @host_id WHERE host_id in (SELECT host_id FROM host where host_id <> @host_id and hops = 0); ", host_uuid);
774
+ if (!rc)
775
+ rc = exec_statement_with_uuid("DELETE FROM host WHERE hops = 0 AND host_id <> @host_id; ", host_uuid);
776
+ if (!rc)
777
+ db_execute("DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);");
778
+
779
+}
780
781
int sql_store_host(
782
uuid_t *host_uuid, const char *hostname, const char *registry_hostname, int update_every, const char *os,
738
- const char *tzone, const char *tags)
783
+ const char *tzone, const char *tags, int hops)
784
{
785
static __thread sqlite3_stmt *res = NULL;
786
int rc;
@@ -783,6 +828,10 @@ int sql_store_host(
828
if (unlikely(rc != SQLITE_OK))
829
goto bind_fail;
830
831
+ rc = sqlite3_bind_int(res, 8, hops);
832
+ if (unlikely(rc != SQLITE_OK))
833
+ goto bind_fail;
834
+
835
int store_rc = sqlite3_step(res);
836
if (unlikely(store_rc != SQLITE_DONE))
837
error_report("Failed to store host %s, rc = %d", hostname, rc);
database/sqlite/sqlite_functions.h
+6
-2
@@ -27,7 +27,8 @@ typedef enum db_check_action_type {
27
#define SQL_MAX_RETRY (100)
28
#define SQLITE_INSERT_DELAY (50) // Insert delay in case of lock
29
30
-#define SQL_STORE_HOST "insert or replace into host (host_id,hostname,registry_hostname,update_every,os,timezone,tags) values (?1,?2,?3,?4,?5,?6,?7);"
30
+#define SQL_STORE_HOST "insert or replace into host (host_id,hostname,registry_hostname,update_every,os,timezone,tags, hops) " \
31
+ "values (?1,?2,?3,?4,?5,?6,?7,?8);"
32
33
#define SQL_STORE_CHART "insert or replace into chart (chart_id, host_id, type, id, " \
34
"name, family, context, title, unit, plugin, module, priority, update_every , chart_type , memory_mode , " \
@@ -60,7 +61,8 @@ typedef enum db_check_action_type {
61
extern int sql_init_database(db_check_action_type_t rebuild, int memory);
62
extern void sql_close_database(void);
63
63
-extern int sql_store_host(uuid_t *guid, const char *hostname, const char *registry_hostname, int update_every, const char *os, const char *timezone, const char *tags);
64
+extern int sql_store_host(uuid_t *guid, const char *hostname, const char *registry_hostname, int update_every, const char *os,
65
+ const char *timezone, const char *tags, int hops);
66
extern int sql_store_chart(
67
uuid_t *chart_uuid, uuid_t *host_uuid, const char *type, const char *id, const char *name, const char *family,
68
const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
@@ -102,4 +104,6 @@ extern void compute_chart_hash(RRDSET *st);
104
extern int sql_set_dimension_option(uuid_t *dim_uuid, char *option);
105
char *get_hostname_by_node_id(char *node_id);
106
void free_temporary_host(RRDHOST *host);
107
+int init_database_batch(int rebuild, int init_type, const char *batch[]);
108
+void migrate_localhost(uuid_t *host_uuid);
109
#endif //NETDATA_SQLITE_FUNCTIONS_H