Allow for an easy way to do metadata migrations (#13196)
Allow for an easy way to migrate metadata to a new database schema (versioning)
Stelios Fragkakis committed
Jun 22, 2022 at 19:21 UTC
ea927b87f4e8f1d3e270a9c555b81809bc0c11d6
5 files changed
+79
-2
CMakeLists.txt
+2
@@ -658,6 +658,8 @@ set(RRD_PLUGIN_FILES
658
database/ram/rrddim_mem.h
659
database/sqlite/sqlite_functions.c
660
database/sqlite/sqlite_functions.h
661
+ database/sqlite/sqlite_db_migration.c
662
+ database/sqlite/sqlite_db_migration.h
663
database/sqlite/sqlite_aclk.c
664
database/sqlite/sqlite_aclk.h
665
database/sqlite/sqlite_health.c
Makefile.am
+2
@@ -454,6 +454,8 @@ RRD_PLUGIN_FILES = \
454
database/ram/rrddim_mem.h \
455
database/sqlite/sqlite_functions.c \
456
database/sqlite/sqlite_functions.h \
457
+ database/sqlite/sqlite_db_migration.c \
458
+ database/sqlite/sqlite_db_migration.h \
459
database/sqlite/sqlite_aclk.c \
460
database/sqlite/sqlite_aclk.h \
461
database/sqlite/sqlite_health.c \
database/sqlite/sqlite_db_migration.c
new
+57
@@ -0,0 +1,57 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "sqlite_db_migration.h"
4
+
5
+static int do_migration_noop(sqlite3 *database, const char *name)
6
+{
7
+ UNUSED(database);
8
+ UNUSED(name);
9
+ info("Running database migration %s", name);
10
+ return 0;
11
+}
12
+
13
+static struct database_func_migration_list {
14
+ char *name;
15
+ int (*func)(sqlite3 *database, const char *name);
16
+} migration_action[] = {
17
+ {.name = "v0 to v1", .func = do_migration_noop},
18
+ // the terminator of this array
19
+ {.name = NULL, .func = NULL}
20
+};
21
+
22
+
23
+static int perform_database_migration_cb(void *data, int argc, char **argv, char **column)
24
+{
25
+ int *status = data;
26
+ UNUSED(argc);
27
+ UNUSED(column);
28
+ *status = str2uint32_t(argv[0]);
29
+ return 0;
30
+}
31
+
32
+int perform_database_migration(sqlite3 *database, int target_version)
33
+{
34
+ int user_version = 0;
35
+ char *err_msg = NULL;
36
+
37
+ int rc = sqlite3_exec(database, "PRAGMA user_version;", perform_database_migration_cb, (void *) &user_version, &err_msg);
38
+ if (rc != SQLITE_OK) {
39
+ info("Error checking the database version; %s", err_msg);
40
+ sqlite3_free(err_msg);
41
+ }
42
+
43
+ if (likely(user_version == target_version)) {
44
+ info("Metadata database version is %d", target_version);
45
+ return target_version;
46
+ }
47
+
48
+ info("Database version is %d, current version is %d. Running migration ...", user_version, target_version);
49
+ for (int i = user_version; migration_action[i].func && i < target_version; i++) {
50
+ rc = (migration_action[i].func)(database, migration_action[i].name);
51
+ if (unlikely(rc)) {
52
+ error_report("Database migration from version %d to version %d failed", i, i + 1);
53
+ return i;
54
+ }
55
+ }
56
+ return target_version;
57
+}
database/sqlite/sqlite_db_migration.h
new
+11
@@ -0,0 +1,11 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+#ifndef NETDATA_SQLITE_DB_MIGRATION_H
3
+#define NETDATA_SQLITE_DB_MIGRATION_H
4
+
5
+#include "daemon/common.h"
6
+#include "sqlite3.h"
7
+
8
+
9
+int perform_database_migration(sqlite3 *database, int target_version);
10
+
11
+#endif //NETDATA_SQLITE_DB_MIGRATION_H
database/sqlite/sqlite_functions.c
+7
-2
@@ -1,8 +1,9 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "sqlite_functions.h"
4
+#include "sqlite_db_migration.h"
5
5
-#define DB_METADATA_VERSION "1"
6
+#define DB_METADATA_VERSION 1
7
8
const char *database_config[] = {
9
"CREATE TABLE IF NOT EXISTS host(host_id blob PRIMARY KEY, hostname text, "
@@ -52,7 +53,6 @@ const char *database_config[] = {
53
"INSERT INTO chart_hash_map (chart_id, hash_id) values (new.chart_id, new.hash_id) "
54
"on conflict (chart_id, hash_id) do nothing; END; ",
55
55
- "PRAGMA user_version="DB_METADATA_VERSION";",
56
NULL
57
};
58
@@ -404,6 +404,8 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
404
char buf[1024 + 1] = "";
405
const char *list[2] = { buf, NULL };
406
407
+ int target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
408
+
409
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
410
// PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
411
snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
@@ -435,6 +437,9 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
437
snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
438
if(init_database_batch(rebuild, 0, list)) return 1;
439
440
+ snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
441
+ if(init_database_batch(rebuild, 0, list)) return 1;
442
+
443
if (init_database_batch(rebuild, 0, &database_config[0]))
444
return 1;
445