Skip database migration steps in new installation (#16071)
* For new installation skip database migration steps * Simplify logging * Count database tables to determine if database is empty * Report extended error message
Stelios Fragkakis committed
Oct 3, 2023 at 15:11 UTC
385d022035f0cff9102deedf0b225cc9916ef344
4 files changed
+74
-66
database/sqlite/sqlite_db_migration.c
+70
-62
@@ -29,7 +29,22 @@ static int get_auto_vaccum(sqlite3 *database)
29
return exists;
30
}
31
32
-int table_exists_in_database(const char *table)
32
+int db_table_count(sqlite3 *database)
33
+{
34
+ char *err_msg = NULL;
35
+ char sql[128];
36
+
37
+ int count = 0;
38
+ snprintf(sql, 127, "select count(1) from sqlite_schema where type = 'table'");
39
+ int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &count, &err_msg);
40
+ if (rc != SQLITE_OK) {
41
+ netdata_log_info("Error checking database table count; %s", err_msg);
42
+ sqlite3_free(err_msg);
43
+ }
44
+ return count;
45
+}
46
+
47
+int table_exists_in_database(sqlite3 *database, const char *table)
48
{
49
char *err_msg = NULL;
50
char sql[128];
@@ -38,7 +53,7 @@ int table_exists_in_database(const char *table)
53
54
snprintf(sql, 127, "select 1 from sqlite_schema where type = 'table' and name = '%s';", table);
55
41
- int rc = sqlite3_exec_monitored(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
56
+ int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
57
if (rc != SQLITE_OK) {
58
netdata_log_info("Error checking table existence; %s", err_msg);
59
sqlite3_free(err_msg);
@@ -47,7 +62,7 @@ int table_exists_in_database(const char *table)
62
return exists;
63
}
64
50
-static int column_exists_in_table(const char *table, const char *column)
65
+static int column_exists_in_table(sqlite3 *database, const char *table, const char *column)
66
{
67
char *err_msg = NULL;
68
char sql[128];
@@ -56,7 +71,7 @@ static int column_exists_in_table(const char *table, const char *column)
71
72
snprintf(sql, 127, "SELECT 1 FROM pragma_table_info('%s') where name = '%s';", table, column);
73
59
- int rc = sqlite3_exec_monitored(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
74
+ int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
75
if (rc != SQLITE_OK) {
76
netdata_log_info("Error checking column existence; %s", err_msg);
77
sqlite3_free(err_msg);
@@ -65,6 +80,17 @@ static int column_exists_in_table(const char *table, const char *column)
80
return exists;
81
}
82
83
+static int get_database_user_version(sqlite3 *database)
84
+{
85
+ int user_version = 0;
86
+
87
+ int rc = sqlite3_exec_monitored(database, "PRAGMA user_version", return_int_cb, (void *)&user_version, NULL);
88
+ if (rc != SQLITE_OK)
89
+ netdata_log_error("Failed to get user version for database");
90
+
91
+ return user_version;
92
+}
93
+
94
const char *database_migrate_v1_v2[] = {
95
"ALTER TABLE host ADD hops INTEGER NOT NULL DEFAULT 0;",
96
NULL
@@ -127,28 +153,22 @@ const char *database_migrate_v13_v14[] = {
153
NULL
154
};
155
130
-static int do_migration_v1_v2(sqlite3 *database, const char *name)
156
+static int do_migration_v1_v2(sqlite3 *database)
157
{
132
- netdata_log_info("Running \"%s\" database migration", name);
133
-
134
- if (table_exists_in_database("host") && !column_exists_in_table("host", "hops"))
158
+ if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "hops"))
159
return init_database_batch(database, &database_migrate_v1_v2[0]);
160
return 0;
161
}
162
139
-static int do_migration_v2_v3(sqlite3 *database, const char *name)
163
+static int do_migration_v2_v3(sqlite3 *database)
164
{
141
- netdata_log_info("Running \"%s\" database migration", name);
142
-
143
- if (table_exists_in_database("host") && !column_exists_in_table("host", "memory_mode"))
165
+ if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "memory_mode"))
166
return init_database_batch(database, &database_migrate_v2_v3[0]);
167
return 0;
168
}
169
148
-static int do_migration_v3_v4(sqlite3 *database, const char *name)
170
+static int do_migration_v3_v4(sqlite3 *database)
171
{
150
- netdata_log_info("Running database migration %s", name);
151
-
172
char sql[256];
173
174
int rc;
@@ -162,7 +182,7 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
182
183
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
184
char *table = strdupz((char *) sqlite3_column_text(res, 0));
165
- if (!column_exists_in_table(table, "chart_context")) {
185
+ if (!column_exists_in_table(database, table, "chart_context")) {
186
snprintfz(sql, 255, "ALTER TABLE %s ADD chart_context text", table);
187
sqlite3_exec_monitored(database, sql, 0, 0, NULL);
188
}
@@ -176,24 +196,18 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
196
return 0;
197
}
198
179
-static int do_migration_v4_v5(sqlite3 *database, const char *name)
199
+static int do_migration_v4_v5(sqlite3 *database)
200
{
181
- netdata_log_info("Running \"%s\" database migration", name);
182
-
201
return init_database_batch(database, &database_migrate_v4_v5[0]);
202
}
203
186
-static int do_migration_v5_v6(sqlite3 *database, const char *name)
204
+static int do_migration_v5_v6(sqlite3 *database)
205
{
188
- netdata_log_info("Running \"%s\" database migration", name);
189
-
206
return init_database_batch(database, &database_migrate_v5_v6[0]);
207
}
208
193
-static int do_migration_v6_v7(sqlite3 *database, const char *name)
209
+static int do_migration_v6_v7(sqlite3 *database)
210
{
195
- netdata_log_info("Running \"%s\" database migration", name);
196
-
211
char sql[256];
212
213
int rc;
@@ -207,7 +221,7 @@ static int do_migration_v6_v7(sqlite3 *database, const char *name)
221
222
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
223
char *table = strdupz((char *) sqlite3_column_text(res, 0));
210
- if (!column_exists_in_table(table, "filtered_alert_unique_id")) {
224
+ if (!column_exists_in_table(database, table, "filtered_alert_unique_id")) {
225
snprintfz(sql, 255, "ALTER TABLE %s ADD filtered_alert_unique_id", table);
226
sqlite3_exec_monitored(database, sql, 0, 0, NULL);
227
snprintfz(sql, 255, "UPDATE %s SET filtered_alert_unique_id = alert_unique_id", table);
@@ -223,10 +237,8 @@ static int do_migration_v6_v7(sqlite3 *database, const char *name)
237
return 0;
238
}
239
226
-static int do_migration_v7_v8(sqlite3 *database, const char *name)
240
+static int do_migration_v7_v8(sqlite3 *database)
241
{
228
- netdata_log_info("Running database migration %s", name);
229
-
242
char sql[256];
243
244
int rc;
@@ -240,7 +252,7 @@ static int do_migration_v7_v8(sqlite3 *database, const char *name)
252
253
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
254
char *table = strdupz((char *) sqlite3_column_text(res, 0));
243
- if (!column_exists_in_table(table, "transition_id")) {
255
+ if (!column_exists_in_table(database, table, "transition_id")) {
256
snprintfz(sql, 255, "ALTER TABLE %s ADD transition_id blob", table);
257
sqlite3_exec_monitored(database, sql, 0, 0, NULL);
258
}
@@ -254,10 +266,8 @@ static int do_migration_v7_v8(sqlite3 *database, const char *name)
266
return 0;
267
}
268
257
-static int do_migration_v8_v9(sqlite3 *database, const char *name)
269
+static int do_migration_v8_v9(sqlite3 *database)
270
{
259
- netdata_log_info("Running database migration %s", name);
260
-
271
char sql[2048];
272
int rc;
273
sqlite3_stmt *res = NULL;
@@ -328,34 +338,28 @@ static int do_migration_v8_v9(sqlite3 *database, const char *name)
338
return 0;
339
}
340
331
-static int do_migration_v9_v10(sqlite3 *database, const char *name)
341
+static int do_migration_v9_v10(sqlite3 *database)
342
{
333
- netdata_log_info("Running \"%s\" database migration", name);
334
-
335
- if (table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "chart_labels"))
343
+ if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "chart_labels"))
344
return init_database_batch(database, &database_migrate_v9_v10[0]);
345
return 0;
346
}
347
340
-static int do_migration_v10_v11(sqlite3 *database, const char *name)
348
+static int do_migration_v10_v11(sqlite3 *database)
349
{
342
- netdata_log_info("Running \"%s\" database migration", name);
343
-
344
- if (table_exists_in_database("health_log") && !column_exists_in_table("health_log", "chart_name"))
350
+ if (table_exists_in_database(database, "health_log") && !column_exists_in_table(database, "health_log", "chart_name"))
351
return init_database_batch(database, &database_migrate_v10_v11[0]);
352
353
return 0;
354
}
355
356
#define MIGR_11_12_UPD_HEALTH_LOG_DETAIL "UPDATE health_log_detail SET summary = (select name from health_log where health_log_id = health_log_detail.health_log_id);"
351
-static int do_migration_v11_v12(sqlite3 *database, const char *name)
357
+static int do_migration_v11_v12(sqlite3 *database)
358
{
359
int rc = 0;
360
355
- netdata_log_info("Running \"%s\" database migration", name);
356
-
357
- if (table_exists_in_database("health_log_detail") && !column_exists_in_table("health_log_detail", "summary") &&
358
- table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "summary"))
361
+ if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary") &&
362
+ table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary"))
363
rc = init_database_batch(database, &database_migrate_v11_v12[0]);
364
365
if (!rc)
@@ -364,28 +368,24 @@ static int do_migration_v11_v12(sqlite3 *database, const char *name)
368
return rc;
369
}
370
367
-static int do_migration_v12_v13(sqlite3 *database, const char *name)
371
+static int do_migration_v12_v13(sqlite3 *database)
372
{
373
int rc = 0;
374
371
- netdata_log_info("Running \"%s\" database migration", name);
372
-
373
- if (table_exists_in_database("health_log_detail") && !column_exists_in_table("health_log_detail", "summary")) {
375
+ if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary")) {
376
rc = init_database_batch(database, &database_migrate_v12_v13_detail[0]);
377
sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL);
378
}
379
378
- if (table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "summary"))
380
+ if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary"))
381
rc = init_database_batch(database, &database_migrate_v12_v13_hash[0]);
382
383
return rc;
384
}
385
384
-static int do_migration_v13_v14(sqlite3 *database, const char *name)
386
+static int do_migration_v13_v14(sqlite3 *database)
387
{
386
- netdata_log_info("Running \"%s\" database migration", name);
387
-
388
- if (!column_exists_in_table("host", "last_connected"))
388
+ if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "last_connected"))
389
return init_database_batch(database, &database_migrate_v13_v14[0]);
390
391
return 0;
@@ -401,25 +401,22 @@ const char *database_ml_migrate_v1_v2[] = {
401
NULL
402
};
403
404
-static int do_ml_migration_v1_v2(sqlite3 *database, const char *name)
404
+static int do_ml_migration_v1_v2(sqlite3 *database)
405
{
406
- netdata_log_info("Running \"%s\" database migration", name);
407
-
406
if (get_auto_vaccum(database) != 2)
407
return init_database_batch(database, &database_ml_migrate_v1_v2[0]);
408
return 0;
409
}
410
413
-static int do_migration_noop(sqlite3 *database, const char *name)
411
+static int do_migration_noop(sqlite3 *database)
412
{
413
UNUSED(database);
416
- netdata_log_info("Running database migration %s", name);
414
return 0;
415
}
416
417
typedef struct database_func_migration_list {
418
char *name;
422
- int (*func)(sqlite3 *database, const char *name);
419
+ int (*func)(sqlite3 *database);
420
} DATABASE_FUNC_MIGRATION_LIST;
421
422
@@ -441,7 +438,8 @@ static int migrate_database(sqlite3 *database, int target_version, char *db_name
438
439
netdata_log_info("Database version is %d, current version is %d. Running migration for %s ...", user_version, target_version, db_name);
440
for (int i = user_version; i < target_version && migration_list[i].func; i++) {
444
- rc = (migration_list[i].func)(database, migration_list[i].name);
441
+ netdata_log_info("Running database \"%s\" migration %s", db_name, migration_list[i].name);
442
+ rc = (migration_list[i].func)(database);
443
if (unlikely(rc)) {
444
error_report("Database %s migration from version %d to version %d failed", db_name, i, i + 1);
445
return i;
@@ -485,11 +483,21 @@ DATABASE_FUNC_MIGRATION_LIST ml_migration_action[] = {
483
484
int perform_database_migration(sqlite3 *database, int target_version)
485
{
486
+ int user_version = get_database_user_version(database);
487
+
488
+ if (!user_version && !db_table_count(database))
489
+ return target_version;
490
+
491
return migrate_database(database, target_version, "metadata", migration_action);
492
}
493
494
int perform_context_database_migration(sqlite3 *database, int target_version)
495
{
496
+ int user_version = get_database_user_version(database);
497
+
498
+ if (!user_version && !table_exists_in_database(database, "context"))
499
+ return target_version;
500
+
501
return migrate_database(database, target_version, "context", context_migration_action);
502
}
503
database/sqlite/sqlite_db_migration.h
+1
-1
@@ -8,7 +8,7 @@
8
9
int perform_database_migration(sqlite3 *database, int target_version);
10
int perform_context_database_migration(sqlite3 *database, int target_version);
11
-int table_exists_in_database(const char *table);
11
+int table_exists_in_database(sqlite3 *database, const char *table);
12
int perform_ml_database_migration(sqlite3 *database, int target_version);
13
14
#endif //NETDATA_SQLITE_DB_MIGRATION_H
database/sqlite/sqlite_functions.c
+2
-2
@@ -325,7 +325,7 @@ int init_database_batch(sqlite3 *database, const char *batch[])
325
if (rc != SQLITE_OK) {
326
error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg);
327
error_report("SQLite failed statement %s", batch[i]);
328
- analytics_set_data_str(&analytics_data.netdata_fail_reason, err_msg);
328
+ analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(sqlite3_extended_errcode(database)));
329
sqlite3_free(err_msg);
330
if (SQLITE_CORRUPT == rc) {
331
if (mark_database_to_recover(NULL, database))
@@ -409,7 +409,7 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
409
rc = sqlite3_open(sqlite_database, &db_meta);
410
if (rc != SQLITE_OK) {
411
error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
412
- analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(rc));
412
+ analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(sqlite3_extended_errcode(db_meta)));
413
sqlite3_close(db_meta);
414
db_meta = NULL;
415
return 1;
database/sqlite/sqlite_health.c
+1
-1
@@ -458,7 +458,7 @@ void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed) {
458
uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
459
snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);
460
461
- bool aclk_table_exists = table_exists_in_database(command);
461
+ bool aclk_table_exists = table_exists_in_database(db_meta, command);
462
463
char *sql = SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED;
464