Convert the ML database (#16046)
* Convert a db to WAL with auto vacuum * Use single sqlite configuration function * Remove UNUSED statements
Stelios Fragkakis committed
Sep 28, 2023 at 19:40 UTC
f90c2a23e9d3e5a96dcf908cbf332cfe14dc8512
9 files changed
+147
-96
database/sqlite/sqlite_context.c
+2
-33
@@ -52,39 +52,8 @@ int sql_init_context_database(int memory)
52
if (likely(!memory))
53
target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION);
54
55
- // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
56
- // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
57
- snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
58
- if(init_database_batch(db_context_meta, list)) return 1;
59
-
60
- // https://www.sqlite.org/pragma.html#pragma_synchronous
61
- // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
62
- snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
63
- if(init_database_batch(db_context_meta, list)) return 1;
64
-
65
- // https://www.sqlite.org/pragma.html#pragma_journal_mode
66
- // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
67
- snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
68
- if(init_database_batch(db_context_meta, list)) return 1;
69
-
70
- // https://www.sqlite.org/pragma.html#pragma_temp_store
71
- // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
72
- snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
73
- if(init_database_batch(db_context_meta, list)) return 1;
74
-
75
- // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
76
- // PRAGMA schema.journal_size_limit = N ;
77
- snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
78
- if(init_database_batch(db_context_meta, list)) return 1;
79
-
80
- // https://www.sqlite.org/pragma.html#pragma_cache_size
81
- // PRAGMA schema.cache_size = pages;
82
- // PRAGMA schema.cache_size = -kibibytes;
83
- snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
84
- if(init_database_batch(db_context_meta, list)) return 1;
85
-
86
- snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
87
- if(init_database_batch(db_context_meta, list)) return 1;
55
+ if (configure_sqlite_database(db_context_meta, target_version))
56
+ return 1;
57
58
if (likely(!memory))
59
snprintfz(buf, 1024, "ATTACH DATABASE \"%s/netdata-meta.db\" as meta;", netdata_configured_cache_dir);
database/sqlite/sqlite_db_migration.c
+47
-8
@@ -11,6 +11,24 @@ static int return_int_cb(void *data, int argc, char **argv, char **column)
11
return 0;
12
}
13
14
+static int get_auto_vaccum(sqlite3 *database)
15
+{
16
+ char *err_msg = NULL;
17
+ char sql[128];
18
+
19
+ int exists = 0;
20
+
21
+ snprintf(sql, 127, "PRAGMA auto_vacuum");
22
+
23
+ int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
24
+ if (rc != SQLITE_OK) {
25
+ netdata_log_info("Error checking database auto vacuum setting; %s", err_msg);
26
+ sqlite3_free(err_msg);
27
+ }
28
+
29
+ return exists;
30
+}
31
+
32
int table_exists_in_database(const char *table)
33
{
34
char *err_msg = NULL;
@@ -111,7 +129,6 @@ const char *database_migrate_v13_v14[] = {
129
130
static int do_migration_v1_v2(sqlite3 *database, const char *name)
131
{
114
- UNUSED(name);
132
netdata_log_info("Running \"%s\" database migration", name);
133
134
if (table_exists_in_database("host") && !column_exists_in_table("host", "hops"))
@@ -121,7 +138,6 @@ static int do_migration_v1_v2(sqlite3 *database, const char *name)
138
139
static int do_migration_v2_v3(sqlite3 *database, const char *name)
140
{
124
- UNUSED(name);
141
netdata_log_info("Running \"%s\" database migration", name);
142
143
if (table_exists_in_database("host") && !column_exists_in_table("host", "memory_mode"))
@@ -131,7 +147,6 @@ static int do_migration_v2_v3(sqlite3 *database, const char *name)
147
148
static int do_migration_v3_v4(sqlite3 *database, const char *name)
149
{
134
- UNUSED(name);
150
netdata_log_info("Running database migration %s", name);
151
152
char sql[256];
@@ -163,7 +178,6 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
178
179
static int do_migration_v4_v5(sqlite3 *database, const char *name)
180
{
166
- UNUSED(name);
181
netdata_log_info("Running \"%s\" database migration", name);
182
183
return init_database_batch(database, &database_migrate_v4_v5[0]);
@@ -171,7 +185,6 @@ static int do_migration_v4_v5(sqlite3 *database, const char *name)
185
186
static int do_migration_v5_v6(sqlite3 *database, const char *name)
187
{
174
- UNUSED(name);
188
netdata_log_info("Running \"%s\" database migration", name);
189
190
return init_database_batch(database, &database_migrate_v5_v6[0]);
@@ -179,7 +192,6 @@ static int do_migration_v5_v6(sqlite3 *database, const char *name)
192
193
static int do_migration_v6_v7(sqlite3 *database, const char *name)
194
{
182
- UNUSED(name);
195
netdata_log_info("Running \"%s\" database migration", name);
196
197
char sql[256];
@@ -213,7 +225,6 @@ static int do_migration_v6_v7(sqlite3 *database, const char *name)
225
226
static int do_migration_v7_v8(sqlite3 *database, const char *name)
227
{
216
- UNUSED(name);
228
netdata_log_info("Running database migration %s", name);
229
230
char sql[256];
@@ -381,10 +392,27 @@ static int do_migration_v13_v14(sqlite3 *database, const char *name)
392
}
393
394
395
+// Actions for ML migration
396
+const char *database_ml_migrate_v1_v2[] = {
397
+ "PRAGMA journal_mode=delete",
398
+ "PRAGMA journal_mode=WAL",
399
+ "PRAGMA auto_vacuum=2",
400
+ "VACUUM",
401
+ NULL
402
+};
403
+
404
+static int do_ml_migration_v1_v2(sqlite3 *database, const char *name)
405
+{
406
+ netdata_log_info("Running \"%s\" database migration", name);
407
+
408
+ if (get_auto_vaccum(database) != 2)
409
+ return init_database_batch(database, &database_ml_migrate_v1_v2[0]);
410
+ return 0;
411
+}
412
+
413
static int do_migration_noop(sqlite3 *database, const char *name)
414
{
415
UNUSED(database);
387
- UNUSED(name);
416
netdata_log_info("Running database migration %s", name);
417
return 0;
418
}
@@ -448,6 +476,12 @@ DATABASE_FUNC_MIGRATION_LIST context_migration_action[] = {
476
{.name = NULL, .func = NULL}
477
};
478
479
+DATABASE_FUNC_MIGRATION_LIST ml_migration_action[] = {
480
+ {.name = "v0 to v1", .func = do_migration_noop},
481
+ {.name = "v1 to v2", .func = do_ml_migration_v1_v2},
482
+ // the terminator of this array
483
+ {.name = NULL, .func = NULL}
484
+};
485
486
int perform_database_migration(sqlite3 *database, int target_version)
487
{
@@ -458,3 +492,8 @@ int perform_context_database_migration(sqlite3 *database, int target_version)
492
{
493
return migrate_database(database, target_version, "context", context_migration_action);
494
}
495
+
496
+int perform_ml_database_migration(sqlite3 *database, int target_version)
497
+{
498
+ return migrate_database(database, target_version, "ml", ml_migration_action);
499
+}
database/sqlite/sqlite_db_migration.h
+1
@@ -9,5 +9,6 @@
9
int perform_database_migration(sqlite3 *database, int target_version);
10
int perform_context_database_migration(sqlite3 *database, int target_version);
11
int table_exists_in_database(const char *table);
12
+int perform_ml_database_migration(sqlite3 *database, int target_version);
13
14
#endif //NETDATA_SQLITE_DB_MIGRATION_H
database/sqlite/sqlite_functions.c
+51
-36
@@ -201,6 +201,55 @@ int execute_insert(sqlite3_stmt *res)
201
return rc;
202
}
203
204
+int configure_sqlite_database(sqlite3 *database, int target_version)
205
+{
206
+ char buf[1024 + 1] = "";
207
+ const char *list[2] = { buf, NULL };
208
+
209
+ // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
210
+ // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
211
+ snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
212
+ if (init_database_batch(database, list))
213
+ return 1;
214
+
215
+ // https://www.sqlite.org/pragma.html#pragma_synchronous
216
+ // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
217
+ snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
218
+ if (init_database_batch(database, list))
219
+ return 1;
220
+
221
+ // https://www.sqlite.org/pragma.html#pragma_journal_mode
222
+ // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
223
+ snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
224
+ if (init_database_batch(database, list))
225
+ return 1;
226
+
227
+ // https://www.sqlite.org/pragma.html#pragma_temp_store
228
+ // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
229
+ snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
230
+ if (init_database_batch(database, list))
231
+ return 1;
232
+
233
+ // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
234
+ // PRAGMA schema.journal_size_limit = N ;
235
+ snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
236
+ if (init_database_batch(database, list))
237
+ return 1;
238
+
239
+ // https://www.sqlite.org/pragma.html#pragma_cache_size
240
+ // PRAGMA schema.cache_size = pages;
241
+ // PRAGMA schema.cache_size = -kibibytes;
242
+ snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
243
+ if (init_database_batch(database, list))
244
+ return 1;
245
+
246
+ snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
247
+ if (init_database_batch(database, list))
248
+ return 1;
249
+
250
+ return 0;
251
+}
252
+
253
#define MAX_OPEN_STATEMENTS (512)
254
255
static void add_stmt_to_list(sqlite3_stmt *res)
@@ -382,9 +431,6 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
431
432
netdata_log_info("SQLite database %s initialization", sqlite_database);
433
385
- char buf[1024 + 1] = "";
386
- const char *list[2] = { buf, NULL };
387
-
434
rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);
435
if (unlikely(rc != SQLITE_OK))
436
error_report("Failed to register internal u2h function");
@@ -402,39 +448,8 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
448
if (likely(!memory))
449
target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
450
405
- // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
406
- // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
407
- snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
408
- if(init_database_batch(db_meta, list)) return 1;
409
-
410
- // https://www.sqlite.org/pragma.html#pragma_synchronous
411
- // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
412
- snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
413
- if(init_database_batch(db_meta, list)) return 1;
414
-
415
- // https://www.sqlite.org/pragma.html#pragma_journal_mode
416
- // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
417
- snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
418
- if(init_database_batch(db_meta, list)) return 1;
419
-
420
- // https://www.sqlite.org/pragma.html#pragma_temp_store
421
- // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
422
- snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
423
- if(init_database_batch(db_meta, list)) return 1;
424
-
425
- // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
426
- // PRAGMA schema.journal_size_limit = N ;
427
- snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
428
- if(init_database_batch(db_meta, list)) return 1;
429
-
430
- // https://www.sqlite.org/pragma.html#pragma_cache_size
431
- // PRAGMA schema.cache_size = pages;
432
- // PRAGMA schema.cache_size = -kibibytes;
433
- snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
434
- if(init_database_batch(db_meta, list)) return 1;
435
-
436
- snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
437
- if(init_database_batch(db_meta, list)) return 1;
451
+ if (configure_sqlite_database(db_meta, target_version))
452
+ return 1;
453
454
if (init_database_batch(db_meta, &database_config[0]))
455
return 1;
database/sqlite/sqlite_functions.h
+1
@@ -50,6 +50,7 @@ SQLITE_API int sqlite3_exec_monitored(
50
int init_database_batch(sqlite3 *database, const char *batch[]);
51
int sql_init_database(db_check_action_type_t rebuild, int memory);
52
void sql_close_database(void);
53
+int configure_sqlite_database(sqlite3 *database, int target_version);
54
55
// Helpers
56
int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);
database/sqlite/sqlite_metadata.c
+26
-14
@@ -60,11 +60,12 @@
60
#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata
61
#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata
62
#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write
63
-#define METADATA_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum
64
-#define METADATA_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum
63
#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)
64
#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop
65
66
+#define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum
67
+#define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum
68
+
69
enum metadata_opcode {
70
METADATA_DATABASE_NOOP = 0,
71
METADATA_DATABASE_TIMER,
@@ -1148,6 +1149,28 @@ static void timer_cb(uv_timer_t* handle)
1149
}
1150
}
1151
1152
+void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc)
1153
+{
1154
+ int free_pages = get_free_page_count(database);
1155
+ int total_pages = get_database_page_count(database);
1156
+
1157
+ if (!threshold)
1158
+ threshold = DATABASE_FREE_PAGES_THRESHOLD_PC;
1159
+
1160
+ if (!vacuum_pc)
1161
+ vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC;
1162
+
1163
+ if (free_pages > (total_pages * threshold / 100)) {
1164
+
1165
+ int do_free_pages = (int) (free_pages * vacuum_pc / 100);
1166
+ netdata_log_info("%s: Freeing %d database pages", db_alias, do_free_pages);
1167
+
1168
+ char sql[128];
1169
+ snprintfz(sql, 127, "PRAGMA incremental_vacuum(%d)", do_free_pages);
1170
+ (void) db_execute(database, sql);
1171
+ }
1172
+}
1173
+
1174
void run_metadata_cleanup(struct metadata_wc *wc)
1175
{
1176
if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
@@ -1161,18 +1184,7 @@ void run_metadata_cleanup(struct metadata_wc *wc)
1184
if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
1185
return;
1186
1164
- int free_pages = get_free_page_count(db_meta);
1165
- int total_pages = get_database_page_count(db_meta);
1166
-
1167
- if (free_pages > (total_pages * METADATA_FREE_PAGES_THRESHOLD_PC / 100)) {
1168
-
1169
- int do_free_pages = (int) (free_pages * METADATA_FREE_PAGES_VACUUM_PC / 100);
1170
- netdata_log_info("METADATA: Freeing %d database pages", do_free_pages);
1171
-
1172
- char sql[128];
1173
- snprintfz(sql, 127, "PRAGMA incremental_vacuum(%d)", do_free_pages);
1174
- (void) db_execute(db_meta, sql);
1175
- }
1187
+ vacuum_database(db_meta, "METADATA", DATABASE_FREE_PAGES_THRESHOLD_PC, DATABASE_FREE_PAGES_VACUUM_PC);
1188
1189
(void) sqlite3_wal_checkpoint(db_meta, NULL);
1190
}
database/sqlite/sqlite_metadata.h
+1
@@ -17,6 +17,7 @@ void metaqueue_host_update_info(RRDHOST *host);
17
void metaqueue_ml_load_models(RRDDIM *rd);
18
void migrate_localhost(uuid_t *host_uuid);
19
void metadata_queue_load_host_context(RRDHOST *host);
20
+void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc);
21
22
// UNIT TEST
23
int metadata_unittest(void);
ml/ml.cc
+17
-5
@@ -9,6 +9,8 @@
9
#include "ad_charts.h"
10
#include "database/sqlite/sqlite3.h"
11
12
+#define ML_METADATA_VERSION 2
13
+
14
#define WORKER_TRAIN_QUEUE_POP 0
15
#define WORKER_TRAIN_ACQUIRE_DIMENSION 1
16
#define WORKER_TRAIN_QUERY 2
@@ -1625,6 +1627,8 @@ static void ml_flush_pending_models(ml_training_thread_t *training_thread) {
1627
training_thread->num_models_to_prune += training_thread->pending_model_info.size();
1628
}
1629
1630
+ vacuum_database(db, "ML", 0, 0);
1631
+
1632
training_thread->pending_model_info.clear();
1633
}
1634
@@ -1777,14 +1781,22 @@ void ml_init()
1781
1782
// create table
1783
if (db) {
1780
- char *err = NULL;
1781
- int rc = sqlite3_exec(db, db_models_create_table, NULL, NULL, &err);
1782
- if (rc != SQLITE_OK) {
1783
- error_report("Failed to create models table (%s, %s)", sqlite3_errstr(rc), err ? err : "");
1784
+ int target_version = perform_ml_database_migration(db, ML_METADATA_VERSION);
1785
+ if (configure_sqlite_database(db, target_version)) {
1786
+ error_report("Failed to setup ML database");
1787
sqlite3_close(db);
1785
- sqlite3_free(err);
1788
db = NULL;
1789
}
1790
+ else {
1791
+ char *err = NULL;
1792
+ int rc = sqlite3_exec(db, db_models_create_table, NULL, NULL, &err);
1793
+ if (rc != SQLITE_OK) {
1794
+ error_report("Failed to create models table (%s, %s)", sqlite3_errstr(rc), err ? err : "");
1795
+ sqlite3_close(db);
1796
+ sqlite3_free(err);
1797
+ db = NULL;
1798
+ }
1799
+ }
1800
}
1801
}
1802
ml/ml.h
+1
@@ -9,6 +9,7 @@ extern "C" {
9
10
#include "daemon/common.h"
11
#include "web/api/queries/rrdr.h"
12
+#include "database/sqlite/sqlite_db_migration.h"
13
14
bool ml_capable();
15
bool ml_enabled(RRDHOST *rh);