@cryptotaxi247 / netdata-1 / commits / 1116111ec

Reorganize and cleanup database related code (#17101)

* Rearrange code * Explicit init of sqlite library and shutdown Function to close all databases * Add functions to return db size * Prevent overflow * Address review comments * Initialize sqlite library before ml_init * Fix unittest

Stelios Fragkakis committed Mar 5, 2024 at 17:21 UTC 1116111ece3c4b84a9805d630beda7ca6d664b0a

13 files changed +799 -770

src/daemon/main.c

+11 -7

@@ -448,11 +448,10 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re

448 #endif

449 }

450

451 - sql_close_context_database();

452 - watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);

451 + sqlite_close_databases();

452 + watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);

453 + sqlite_library_shutdown();

454

454 - sql_close_database();

455 - watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);

455

456 // unlink the pid

457 if(pidfile[0]) {

@@ -1500,22 +1499,24 @@ int main(int argc, char **argv) {

1499 #endif

1500

1501 if(strcmp(optarg, "sqlite-meta-recover") == 0) {

1503 - sql_init_database(DB_CHECK_RECOVER, 0);

1502 + sql_init_meta_database(DB_CHECK_RECOVER, 0);

1503 return 0;

1504 }

1505

1506 if(strcmp(optarg, "sqlite-compact") == 0) {

1508 - sql_init_database(DB_CHECK_RECLAIM_SPACE, 0);

1507 + sql_init_meta_database(DB_CHECK_RECLAIM_SPACE, 0);

1508 return 0;

1509 }

1510

1511 if(strcmp(optarg, "sqlite-analyze") == 0) {

1513 - sql_init_database(DB_CHECK_ANALYZE, 0);

1512 + sql_init_meta_database(DB_CHECK_ANALYZE, 0);

1513 return 0;

1514 }

1515

1516 if(strcmp(optarg, "unittest") == 0) {

1517 unittest_running = true;

1518 + if (sqlite_library_init())

1519 + return 1;

1520

1521 if (pluginsd_parser_unittest()) return 1;

1522 if (unit_test_static_threads()) return 1;

@@ -1539,6 +1540,7 @@ int main(int argc, char **argv) {

1540 if (ctx_unittest()) return 1;

1541 if (uuid_unittest()) return 1;

1542 if (dyncfg_unittest()) return 1;

1543 + sqlite_library_shutdown();

1544 fprintf(stderr, "\n\nALL TESTS PASSED\n\n");

1545 return 0;

1546 }

@@ -2054,6 +2056,8 @@ int main(int argc, char **argv) {

2056 exit(1);

2057 }

2058 }

2059 + if (sqlite_library_init())

2060 + fatal("Failed to initialize sqlite library");

2061

2062 // --------------------------------------------------------------------

2063 // Initialize ML configuration

src/daemon/unit_test.c

+1 -1

@@ -2546,7 +2546,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi

2546

2547 fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n");

2548

2549 - (void) sql_init_database(DB_CHECK_NONE, 1);

2549 + (void)sql_init_meta_database(DB_CHECK_NONE, 1);

2550 host = dbengine_rrdhost_find_or_create("dbengine-stress-test");

2551 if (NULL == host)

2552 return;

src/daemon/watcher.c

+3 -6

@@ -85,8 +85,7 @@ void *watcher_main(void *arg)

85 watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);

86 watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);

87 watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);

88 - watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);

89 - watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);

88 + watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);

89 watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE);

90 watcher_wait_for_step(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);

91 watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);

@@ -146,10 +145,8 @@ void watcher_thread_start() {

145 "wait for dbengine main cache to finish flushing";

146 watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg =

147 "stop dbengine tiers";

149 - watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB].msg =

150 - "close SQL context db";

151 - watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB].msg =

152 - "close SQL main db";

148 + watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_DATABASES].msg =

149 + "close SQL databases";

150 watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg =

151 "remove pid file";

152 watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg =

src/daemon/watcher.h

+1 -2

@@ -27,8 +27,7 @@ typedef enum {

27 WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH,

28 WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING,

29 WATCHER_STEP_ID_STOP_DBENGINE_TIERS,

30 - WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB,

31 - WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB,

30 + WATCHER_STEP_ID_CLOSE_SQL_DATABASES,

31 WATCHER_STEP_ID_REMOVE_PID_FILE,

32 WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES,

33 WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE,

src/database/rrdhost.c

+1 -1

@@ -1009,7 +1009,7 @@ void dbengine_init(char *hostname) {

1009 int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {

1010 rrdhost_init();

1011

1012 - if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {

1012 + if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {

1013 if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

1014 set_late_global_environment(system_info);

1015 fatal("Failed to initialize SQLite");

src/database/sqlite/sqlite_context.c

+10 -20

@@ -71,24 +71,6 @@ int sql_init_context_database(int memory)

71 return 0;

72 }

74 -/*

75 - * Close the sqlite database

76 - */

77 -

78 -void sql_close_context_database(void)

79 -{

80 - int rc;

81 - if (unlikely(!db_context_meta))

82 - return;

83 -

84 - netdata_log_info("Closing context SQLite database");

85 -

86 - rc = sqlite3_close_v2(db_context_meta);

87 - if (unlikely(rc != SQLITE_OK))

88 - error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc));

89 - db_context_meta = NULL;

90 -}

91 -

74 //

75 // Fetching data

76 //

@@ -422,6 +404,12 @@ int sql_context_cache_stats(int op)

404 return count;

405 }

406

407 +

408 +uint64_t sqlite_get_context_space(void)

409 +{

410 + return sqlite_get_db_space(db_context_meta);

411 +}

412 +

413 //

414 // TESTING FUNCTIONS

415 //

@@ -456,7 +444,8 @@ int ctx_unittest(void)

444 uuid_t host_uuid;

445 uuid_generate(host_uuid);

446

459 - initialize_thread_key_pool();

447 + if (sqlite_library_init())

448 + return 1;

449

450 int rc = sql_init_context_database(1);

451

@@ -532,7 +521,8 @@ int ctx_unittest(void)

521 ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL);

522 netdata_log_info("List context end after delete");

523

535 - sql_close_context_database();

524 + sql_close_database(db_context_meta, "CONTEXT");

525 + sqlite_library_shutdown();

526

527 return 0;

528 }

src/database/sqlite/sqlite_context.h

@@ -65,6 +65,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);

65 int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data);

67 int sql_init_context_database(int memory);

68 +uint64_t sqlite_get_context_space(void);

69 void sql_close_context_database(void);

70 int ctx_unittest(void);

71 #endif //NETDATA_SQLITE_CONTEXT_H

src/database/sqlite/sqlite_functions.c

+73 -655

@@ -1,94 +1,6 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

3 #include "sqlite_functions.h"

4 -#include "sqlite3recover.h"

5 -#include "sqlite_db_migration.h"

6 -

7 -#define DB_METADATA_VERSION 16

8 -

9 -const char *database_config[] = {

10 - "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "

11 - "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, "

12 - "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default '',"

13 - "hops INT NOT NULL DEFAULT 0,"

14 - "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0,"

15 - "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', "

16 - "entries INT NOT NULL DEFAULT 0,"

17 - "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)",

18 -

19 - "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "

20 - "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "

21 - "chart_type int, memory_mode int, history_entries)",

22 -

23 - "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "

24 - "multiplier int, divisor int , algorithm int, options text)",

25 -

26 - "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)",

27 -

28 - "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)",

29 -

30 - "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)",

31 -

32 - "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "

33 - "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))",

34 -

35 - "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)",

36 -

37 - "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, "

38 - "on_key text, class text, component text, type text, os text, hosts text, lookup text, "

39 - "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "

40 - "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "

41 - "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "

42 - "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text)",

43 -

44 - "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "

45 - "date_created INT, PRIMARY KEY(host_id, system_key))",

46 -

47 - "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, "

48 - "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))",

49 -

50 - "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)"

51 - " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END",

52 -

53 - "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, "

54 - "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, "

55 - "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))",

56 -

57 - "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)",

58 -

59 - "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, "

60 - "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "

61 - "flags int, exec_run_timestamp int, delay_up_to_timestamp int, "

62 - "info text, exec_code int, new_status real, old_status real, delay int, "

63 - "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)",

64 -

65 - "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)",

66 - "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)",

67 - "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)",

68 - "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)",

69 - "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)",

70 - "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)",

71 -

72 - NULL

73 -};

74 -

75 -const char *database_cleanup[] = {

76 - "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)",

77 - "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)",

78 - "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)",

79 - "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)",

80 - "DROP TRIGGER IF EXISTS tr_dim_del",

81 - "DROP INDEX IF EXISTS ind_d1",

82 - "DROP INDEX IF EXISTS ind_c1",

83 - "DROP INDEX IF EXISTS ind_c2",

84 - "DROP INDEX IF EXISTS alert_hash_index",

85 - "DROP INDEX IF EXISTS health_log_d_ind_4",

86 - "DROP INDEX IF EXISTS health_log_d_ind_1",

87 - "DROP INDEX IF EXISTS health_log_d_ind_5",

88 - NULL

89 -};

90 -

91 -sqlite3 *db_meta = NULL;

5 #define MAX_PREPARED_STATEMENTS (32)

6 pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];

@@ -152,45 +64,6 @@ static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database)

64 return false;

65 }

155 -static void recover_database(const char *sqlite_database, const char *new_sqlite_database)

156 -{

157 - sqlite3 *database;

158 - int rc = sqlite3_open(sqlite_database, &database);

159 - if (rc != SQLITE_OK)

160 - return;

161 -

162 - netdata_log_info("Recover %s", sqlite_database);

163 - netdata_log_info(" to %s", new_sqlite_database);

164 -

165 - // This will remove the -shm and -wal files when we close the database

166 - (void) db_execute(database, "select count(*) from sqlite_master limit 0");

167 -

168 - sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);

169 - if (recover) {

170 -

171 - rc = sqlite3_recover_run(recover);

172 -

173 - if (rc == SQLITE_OK)

174 - netdata_log_info("Recover complete");

175 - else

176 - netdata_log_info("Recover encountered an error but the database may be usable");

177 -

178 - rc = sqlite3_recover_finish(recover);

179 -

180 - (void) sqlite3_close(database);

181 -

182 - if (rc == SQLITE_OK) {

183 - rc = rename(new_sqlite_database, sqlite_database);

184 - if (rc == 0) {

185 - netdata_log_info("Renamed %s", new_sqlite_database);

186 - netdata_log_info(" to %s", sqlite_database);

187 - }

188 - }

189 - }

190 - else

191 - (void) sqlite3_close(database);

192 -}

193 -

67 int execute_insert(sqlite3_stmt *res)

68 {

69 int rc;

@@ -279,14 +152,11 @@ static void add_stmt_to_list(sqlite3_stmt *res)

152 static void release_statement(void *statement)

153 {

154 int rc;

282 -#ifdef NETDATA_DEV_MODE

283 - netdata_log_info("Thread %d: Cleaning prepared statement on %p", gettid(), statement);

284 -#endif

155 if (unlikely(rc = sqlite3_finalize((sqlite3_stmt *) statement) != SQLITE_OK))

156 error_report("Failed to finalize statement, rc = %d", rc);

157 }

158

289 -void initialize_thread_key_pool(void)

159 +static void initialize_thread_key_pool(void)

160 {

161 for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++)

162 (void)pthread_key_create(&key_pool[i], release_statement);

@@ -303,13 +173,9 @@ int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **state

173 key = &key_pool[keys_used++];

174

175 int rc = sqlite3_prepare_v2(database, query, -1, statement, 0);

306 - if (likely(rc == SQLITE_OK)) {

307 - if (likely(key)) {

176 + if (rc == SQLITE_OK) {

177 + if (key)

178 ret = pthread_setspecific(*key, *statement);

309 -#ifdef NETDATA_DEV_MODE

310 - netdata_log_info("Thread %d: Using key %u on statement %p", gettid(), keys_used, *statement);

311 -#endif

312 - }

179 if (ret)

180 add_stmt_to_list(*statement);

181 }

@@ -355,202 +221,6 @@ int init_database_batch(sqlite3 *database, const char *batch[], const char *desc

221 return 0;

222 }

223

358 -static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)

359 -{

360 - uuid_t uuid;

361 -

362 - if ( argc != 1 ){

363 - sqlite3_result_null(context);

364 - return ;

365 - }

366 - int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid);

367 - if (rc == -1) {

368 - sqlite3_result_null(context);

369 - return ;

370 - }

371 -

372 - sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);

373 -}

374 -

375 -void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv)

376 -{

377 - if (argc != 1 ){

378 - sqlite3_result_null(context);

379 - return ;

380 - }

381 -

382 - if (sqlite3_value_int(argv[0]) != 0) {

383 - struct timespec req = {.tv_sec = 0, .tv_nsec = 1};

384 - nanosleep(&req, NULL);

385 - }

386 -

387 - sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec());

388 -}

389 -

390 -void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv)

391 -{

392 - (void)argc;

393 - (void)argv;

394 -

395 - uuid_t uuid;

396 - uuid_generate_random(uuid);

397 - sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);

398 -}

399 -

400 -/*

401 - * Initialize the SQLite database

402 - * Return 0 on success

403 - */

404 -int sql_init_database(db_check_action_type_t rebuild, int memory)

405 -{

406 - char *err_msg = NULL;

407 - char sqlite_database[FILENAME_MAX + 1];

408 - int rc;

409 -

410 - if (likely(!memory)) {

411 - snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);

412 - rc = unlink(sqlite_database);

413 - snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);

414 -

415 - if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {

416 - char new_sqlite_database[FILENAME_MAX + 1];

417 - snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);

418 - recover_database(sqlite_database, new_sqlite_database);

419 - if (rebuild & DB_CHECK_RECOVER)

420 - return 0;

421 - }

422 - }

423 - else

424 - strcpy(sqlite_database, ":memory:");

425 -

426 - rc = sqlite3_open(sqlite_database, &db_meta);

427 - if (rc != SQLITE_OK) {

428 - error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));

429 - char *error_str = get_database_extented_error(db_meta, 0, "meta_open");

430 - if (error_str)

431 - analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str);

432 - freez(error_str);

433 - sqlite3_close(db_meta);

434 - db_meta = NULL;

435 - return 1;

436 - }

437 -

438 - if (rebuild & DB_CHECK_RECLAIM_SPACE) {

439 - netdata_log_info("Reclaiming space of %s", sqlite_database);

440 - rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg);

441 - if (rc != SQLITE_OK) {

442 - error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);

443 - sqlite3_free(err_msg);

444 - }

445 - else {

446 - (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");

447 - (void) sqlite3_close(db_meta);

448 - }

449 - return 1;

450 - }

451 -

452 - if (rebuild & DB_CHECK_ANALYZE) {

453 - errno = 0;

454 - netdata_log_info("Running ANALYZE on %s", sqlite_database);

455 - rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg);

456 - if (rc != SQLITE_OK) {

457 - error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg);

458 - sqlite3_free(err_msg);

459 - }

460 - else {

461 - (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");

462 - (void) sqlite3_close(db_meta);

463 - }

464 - return 1;

465 - }

466 -

467 - netdata_log_info("SQLite database %s initialization", sqlite_database);

468 -

469 - rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);

470 - if (unlikely(rc != SQLITE_OK))

471 - error_report("Failed to register internal u2h function");

472 -

473 - rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0);

474 - if (unlikely(rc != SQLITE_OK))

475 - error_report("Failed to register internal now_usec function");

476 -

477 - rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0);

478 - if (unlikely(rc != SQLITE_OK))

479 - error_report("Failed to register internal uuid_random function");

480 -

481 - int target_version = DB_METADATA_VERSION;

482 -

483 - if (likely(!memory))

484 - target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);

485 -

486 - if (configure_sqlite_database(db_meta, target_version, "meta_config"))

487 - return 1;

488 -

489 - if (init_database_batch(db_meta, &database_config[0], "meta_init"))

490 - return 1;

491 -

492 - if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup"))

493 - return 1;

494 -

495 - netdata_log_info("SQLite database initialization completed");

496 -

497 - initialize_thread_key_pool();

498 -

499 - return 0;

500 -}

501 -

502 -/*

503 - * Close the sqlite database

504 - */

505 -

506 -void sql_close_database(void)

507 -{

508 - int rc;

509 - if (unlikely(!db_meta))

510 - return;

511 -

512 - netdata_log_info("Closing SQLite database");

513 -

514 - add_stmt_to_list(NULL);

515 -

516 - (void) db_execute(db_meta, "PRAGMA analysis_limit=10000");

517 - (void) db_execute(db_meta, "PRAGMA optimize");

518 -

519 - rc = sqlite3_close_v2(db_meta);

520 - if (unlikely(rc != SQLITE_OK))

521 - error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));

522 - db_meta = NULL;

523 -}

524 -

525 -int exec_statement_with_uuid(const char *sql, uuid_t *uuid)

526 -{

527 - int rc, result = 1;

528 - sqlite3_stmt *res = NULL;

529 -

530 - rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);

531 - if (unlikely(rc != SQLITE_OK)) {

532 - error_report("Failed to prepare statement %s, rc = %d", sql, rc);

533 - return 1;

534 - }

535 -

536 - rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);

537 - if (unlikely(rc != SQLITE_OK)) {

538 - error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc);

539 - goto skip;

540 - }

541 -

542 - rc = execute_insert(res);

543 - if (likely(rc == SQLITE_DONE))

544 - result = SQLITE_OK;

545 - else

546 - error_report("Failed to execute %s, rc = %d", sql, rc);

547 -

548 -skip:

549 - rc = sqlite3_finalize(res);

550 - if (unlikely(rc != SQLITE_OK))

551 - error_report("Failed to finalize statement %s, rc = %d", sql, rc);

552 - return result;

553 -}

224 // Return 0 OK

225 // Return 1 Failed

226 int db_execute(sqlite3 *db, const char *cmd)

@@ -580,376 +250,124 @@ int db_execute(sqlite3 *db, const char *cmd)

250 return (rc != SQLITE_OK);

251 }

252

583 -static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)

253 +// Utils

254 +int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null)

255 {

585 - if (unlikely(!host))

586 - return;

587 -

588 - if (unlikely(!node_id)) {

589 - freez(host->node_id);

590 - __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED);

591 - return;

592 - }

593 -

594 - struct aclk_sync_cfg_t *wc = host->aclk_config;

595 -

596 - if (unlikely(!host->node_id)) {

597 - uuid_t *t = mallocz(sizeof(*host->node_id));

598 - uuid_copy(*t, *node_id);

599 - __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED);

600 - }

601 - else {

602 - uuid_copy(*(host->node_id), *node_id);

603 - }

604 -

605 - if (unlikely(!wc))

606 - sql_create_aclk_table(host, &host->host_uuid, node_id);

607 - else

608 - uuid_unparse_lower(*node_id, wc->node_id);

256 + if (likely(text))

257 + return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC);

258 + if (!can_be_null)

259 + return 1;

260 + return sqlite3_bind_null(res, position);

261 }

262

611 -#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id"

263 +#define SQL_DROP_TABLE "DROP table %s"

264

613 -int update_node_id(uuid_t *host_id, uuid_t *node_id)

265 +void sql_drop_table(const char *table)

266 {

615 - sqlite3_stmt *res = NULL;

616 - RRDHOST *host = NULL;

617 - int rc = 2;

618 -

619 - char host_guid[GUID_LEN + 1];

620 - uuid_unparse_lower(*host_id, host_guid);

621 - rrd_wrlock();

622 - host = rrdhost_find_by_guid(host_guid);

623 - if (likely(host))

624 - set_host_node_id(host, node_id);

625 - rrd_unlock();

626 -

627 - if (unlikely(!db_meta)) {

628 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

629 - error_report("Database has not been initialized");

630 - return 1;

631 - }

632 -

633 - rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0);

634 - if (unlikely(rc != SQLITE_OK)) {

635 - error_report("Failed to prepare statement to store node instance information");

636 - return 1;

637 - }

267 + if (!table)

268 + return;

269

639 - rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC);

640 - if (unlikely(rc != SQLITE_OK)) {

641 - error_report("Failed to bind host_id parameter to store node instance information");

642 - goto failed;

643 - }

270 + char wstr[255];

271 + snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table);

272

645 - rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC);

646 - if (unlikely(rc != SQLITE_OK)) {

647 - error_report("Failed to bind host_id parameter to store node instance information");

648 - goto failed;

273 + int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL);

274 + if (rc != SQLITE_OK) {

275 + error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc);

276 }

650 -

651 - rc = execute_insert(res);

652 - if (unlikely(rc != SQLITE_DONE))

653 - error_report("Failed to store node instance information, rc = %d", rc);

654 - rc = sqlite3_changes(db_meta);

655 -

656 -failed:

657 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

658 - error_report("Failed to finalize the prepared statement when storing node instance information");

659 -

660 - return rc - 1;

277 }

278

663 -#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL"

664 -

665 -int get_node_id(uuid_t *host_id, uuid_t *node_id)

279 +static int get_pragma_value(sqlite3 *database, const char *sql)

280 {

281 sqlite3_stmt *res = NULL;

668 - int rc;

669 -

670 - if (unlikely(!db_meta)) {

671 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

672 - error_report("Database has not been initialized");

673 - return 1;

674 - }

675 -

676 - rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0);

677 - if (unlikely(rc != SQLITE_OK)) {

678 - error_report("Failed to prepare statement to select node instance information for a host");

679 - return 1;

680 - }

681 -

682 - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

683 - if (unlikely(rc != SQLITE_OK)) {

684 - error_report("Failed to bind host_id parameter to select node instance information");

685 - goto failed;

686 - }

282 + int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);

283 + if (unlikely(rc != SQLITE_OK))

284 + return -1;

285

286 + int result = -1;

287 rc = sqlite3_step_monitored(res);

689 - if (likely(rc == SQLITE_ROW && node_id))

690 - uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0)));

288 + if (likely(rc == SQLITE_ROW))

289 + result = sqlite3_column_int(res, 0);

290

692 -failed:

693 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

694 - error_report("Failed to finalize the prepared statement when selecting node instance information");

291 + rc = sqlite3_finalize(res);

292 + (void) rc;

293

696 - return (rc == SQLITE_ROW) ? 0 : -1;

294 + return result;

295 }

296

699 -#define SQL_INVALIDATE_NODE_INSTANCES \

700 - "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \

701 - "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))"

702 -

703 -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)

297 +int get_free_page_count(sqlite3 *database)

298 {

705 - sqlite3_stmt *res = NULL;

706 - int rc;

707 -

708 - if (unlikely(!db_meta)) {

709 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

710 - error_report("Database has not been initialized");

711 - return;

712 - }

713 -

714 - rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0);

715 - if (unlikely(rc != SQLITE_OK)) {

716 - error_report("Failed to prepare statement to invalidate node instance ids");

717 - return;

718 - }

719 -

720 - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

721 - if (unlikely(rc != SQLITE_OK)) {

722 - error_report("Failed to bind host_id parameter to invalidate node instance information");

723 - goto failed;

724 - }

725 -

726 - if (claim_id)

727 - rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);

728 - else

729 - rc = sqlite3_bind_null(res, 2);

730 -

731 - if (unlikely(rc != SQLITE_OK)) {

732 - error_report("Failed to bind claim_id parameter to invalidate node instance information");

733 - goto failed;

734 - }

735 -

736 - rc = execute_insert(res);

737 - if (unlikely(rc != SQLITE_DONE))

738 - error_report("Failed to invalidate node instance information, rc = %d", rc);

739 -

740 -failed:

741 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

742 - error_report("Failed to finalize the prepared statement when invalidating node instance information");

299 + return get_pragma_value(database, "PRAGMA freelist_count");

300 }

301

745 -#define SQL_GET_NODE_INSTANCE_LIST \

746 - "SELECT ni.node_id, ni.host_id, h.hostname " \

747 - "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0"

748 -

749 -struct node_instance_list *get_node_list(void)

302 +int get_database_page_count(sqlite3 *database)

303 {

751 - struct node_instance_list *node_list = NULL;

752 - sqlite3_stmt *res = NULL;

753 - int rc;

754 -

755 - if (unlikely(!db_meta)) {

756 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

757 - error_report("Database has not been initialized");

758 - return NULL;

759 - }

760 -

761 - rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0);

762 - if (unlikely(rc != SQLITE_OK)) {

763 - error_report("Failed to prepare statement to get node instance information");

764 - return NULL;

765 - }

766 -

767 - int row = 0;

768 - char host_guid[37];

769 - while (sqlite3_step_monitored(res) == SQLITE_ROW)

770 - row++;

771 -

772 - if (sqlite3_reset(res) != SQLITE_OK) {

773 - error_report("Failed to reset the prepared statement while fetching node instance information");

774 - goto failed;

775 - }

776 - node_list = callocz(row + 1, sizeof(*node_list));

777 - int max_rows = row;

778 - row = 0;

779 - // TODO: Check to remove lock

780 - rrd_rdlock();

781 - while (sqlite3_step_monitored(res) == SQLITE_ROW) {

782 - if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t))

783 - uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0)));

784 - if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {

785 - uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);

786 - uuid_unparse_lower(*host_id, host_guid);

787 - RRDHOST *host = rrdhost_find_by_guid(host_guid);

788 - if (!host)

789 - continue;

790 - if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {

791 - netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));

792 - continue;

793 - }

794 - uuid_copy(node_list[row].host_id, *host_id);

795 - node_list[row].queryable = 1;

796 - node_list[row].live = (host && (host == localhost || host->receiver

797 - || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;

798 - node_list[row].hops = (host && host->system_info) ? host->system_info->hops :

799 - uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0;

800 - node_list[row].hostname =

801 - sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;

802 - }

803 - row++;

804 - if (row == max_rows)

805 - break;

806 - }

807 - rrd_unlock();

808 -

809 -failed:

810 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

811 - error_report("Failed to finalize the prepared statement when fetching node instance information");

812 -

813 - return node_list;

304 + return get_pragma_value(database, "PRAGMA page_count");

305 }

306

816 -#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id"

817 -

818 -void sql_load_node_id(RRDHOST *host)

307 +uint64_t sqlite_get_db_space(sqlite3 *db)

308 {

820 - sqlite3_stmt *res = NULL;

821 - int rc;

822 -

823 - if (unlikely(!db_meta)) {

824 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

825 - error_report("Database has not been initialized");

826 - return;

827 - }

828 -

829 - rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);

830 - if (unlikely(rc != SQLITE_OK)) {

831 - error_report("Failed to prepare statement to fetch node id");

832 - return;

833 - }

834 -

835 - rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);

836 - if (unlikely(rc != SQLITE_OK)) {

837 - error_report("Failed to bind host_id parameter to load node instance information");

838 - goto failed;

839 - }

309 + if (!db)

310 + return 0;

311

841 - rc = sqlite3_step_monitored(res);

842 - if (likely(rc == SQLITE_ROW)) {

843 - if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t)))

844 - set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0));

845 - else

846 - set_host_node_id(host, NULL);

847 - }

312 + uint64_t page_size = (uint64_t) get_pragma_value(db, "PRAGMA page_size");

313 + uint64_t page_count = (uint64_t) get_pragma_value(db, "PRAGMA page_count");

314

849 -failed:

850 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

851 - error_report("Failed to finalize the prepared statement when loading node instance information");

315 + return page_size * page_count;

316 }

317

854 -#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id"

318 +/*

319 + * Close the sqlite database

320 + */

321

856 -void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info)

322 +void sql_close_database(sqlite3 *database, const char *database_name)

323 {

324 int rc;

859 -

860 - sqlite3_stmt *res = NULL;

861 -

862 - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_INFO, -1, &res, 0);

863 - if (unlikely(rc != SQLITE_OK)) {

864 - error_report("Failed to prepare statement to read host information");

325 + if (unlikely(!database))

326 return;

866 - }

867 -

868 - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

869 - if (unlikely(rc != SQLITE_OK)) {

870 - error_report("Failed to bind host parameter host information");

871 - goto skip;

872 - }

873 -

874 - while (sqlite3_step_monitored(res) == SQLITE_ROW) {

875 - rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0),

876 - (char *) sqlite3_column_text(res, 1));

877 - }

878 -

879 -skip:

880 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

881 - error_report("Failed to finalize the prepared statement when reading host information");

882 -}

883 -

884 -#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \

885 - "AND label_key IS NOT NULL AND label_value IS NOT NULL"

886 -

887 -RRDLABELS *sql_load_host_labels(uuid_t *host_id)

888 -{

889 - int rc;

327

891 - RRDLABELS *labels = NULL;

892 - sqlite3_stmt *res = NULL;

328 + (void) db_execute(database, "PRAGMA analysis_limit=10000");

329 + (void) db_execute(database, "PRAGMA optimize");

330

894 - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0);

895 - if (unlikely(rc != SQLITE_OK)) {

896 - error_report("Failed to prepare statement to read host information");

897 - return NULL;

898 - }

331 + netdata_log_info("%s: Closing sqlite database", database_name);

332

900 - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

901 - if (unlikely(rc != SQLITE_OK)) {

902 - error_report("Failed to bind host parameter host information");

903 - goto skip;

904 - }

333 +#ifdef NETDATA_DEV_MODE

334 + int t_count_used,t_count_hit,t_count_miss,t_count_full, dummy;

335 + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_USED, &dummy, &t_count_used, 0);

336 + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_HIT, &dummy,&t_count_hit, 0);

337 + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &dummy,&t_count_miss, 0);

338 + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &dummy,&t_count_full, 0);

339

906 - labels = rrdlabels_create();

340 + netdata_log_info("%s: Database lookaside allocation statistics: Used slots %d, Hit %d, Misses due to small slot size %d, Misses due to slots full %d", database_name,

341 + t_count_used,t_count_hit, t_count_miss, t_count_full);

342

908 - while (sqlite3_step_monitored(res) == SQLITE_ROW) {

909 - rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2));

910 - }

343 + (void) sqlite3_db_release_memory(database);

344 +#endif

345

912 -skip:

913 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

914 - error_report("Failed to finalize the prepared statement when reading host information");

915 - return labels;

346 + rc = sqlite3_close_v2(database);

347 + if (unlikely(rc != SQLITE_OK))

348 + error_report("%s: Error while closing the sqlite database: rc %d, error \"%s\"", database_name, rc, sqlite3_errstr(rc));

349 }

350

918 -// Utils

919 -int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null)

351 +extern sqlite3 *db_context_meta;

352 +

353 +void sqlite_close_databases(void)

354 {

921 - if (likely(text))

922 - return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC);

923 - if (!can_be_null)

924 - return 1;

925 - return sqlite3_bind_null(res, position);

355 + add_stmt_to_list(NULL);

356 +

357 + sql_close_database(db_context_meta, "CONTEXT");

358 + sql_close_database(db_meta, "METADATA");

359 }

360

928 -int sql_metadata_cache_stats(int op)

361 +int sqlite_library_init(void)

362 {

930 - int count, dummy;

363 + initialize_thread_key_pool();

364

932 - if (unlikely(!db_meta))

933 - return 0;

365 + int rc = sqlite3_initialize();

366

935 - netdata_thread_disable_cancelability();

936 - sqlite3_db_status(db_meta, op, &count, &dummy, 0);

937 - netdata_thread_enable_cancelability();

938 - return count;

367 + return (SQLITE_OK != rc);

368 }

369

941 -#define SQL_DROP_TABLE "DROP table %s"

942 -

943 -void sql_drop_table(const char *table)

370 +void sqlite_library_shutdown(void)

371 {

945 - if (!table)

946 - return;

947 -

948 - char wstr[255];

949 - snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table);

950 -

951 - int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL);

952 - if (rc != SQLITE_OK) {

953 - error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc);

954 - }

372 + (void) sqlite3_shutdown();

373 }

src/database/sqlite/sqlite_functions.h

+10 -46

@@ -8,36 +8,9 @@

9 void analytics_set_data_str(char **name, const char *value);

11 -// return a node list

12 -struct node_instance_list {

13 - uuid_t node_id;

14 - uuid_t host_id;

15 - char *hostname;

16 - int live;

17 - int queryable;

18 - int hops;

19 -};

20 -

21 -typedef enum db_check_action_type {

22 - DB_CHECK_NONE = (1 << 0),

23 - DB_CHECK_RECLAIM_SPACE = (1 << 1),

24 - DB_CHECK_ANALYZE = (1 << 2),

25 - DB_CHECK_CONT = (1 << 3),

26 - DB_CHECK_RECOVER = (1 << 4),

27 -} db_check_action_type_t;

28 -

11 #define SQL_MAX_RETRY (100)

12 #define SQLITE_INSERT_DELAY (10) // Insert delay in case of lock

32 -#define CHECK_SQLITE_CONNECTION(db_meta) \

33 - if (unlikely(!db_meta)) { \

34 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { \

35 - return 1; \

36 - } \

37 - error_report("Database has not been initialized"); \

38 - return 1; \

39 - }

40 -

14 SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt);

15 SQLITE_API int sqlite3_exec_monitored(

16 sqlite3 *db, /* An open database */

@@ -49,35 +22,26 @@ SQLITE_API int sqlite3_exec_monitored(

23 // Initialization and shutdown

24 int init_database_batch(sqlite3 *database, const char *batch[], const char *description);

52 -int sql_init_database(db_check_action_type_t rebuild, int memory);

53 -void sql_close_database(void);

25 int configure_sqlite_database(sqlite3 *database, int target_version, const char *description);

27 // Helpers

28 int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);

29 int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement);

30 int execute_insert(sqlite3_stmt *res);

60 -int exec_statement_with_uuid(const char *sql, uuid_t *uuid);

31 int db_execute(sqlite3 *database, const char *cmd);

62 -void initialize_thread_key_pool(void);

63 -

64 -// Look up functions

65 -int get_node_id(uuid_t *host_id, uuid_t *node_id);

66 -struct node_instance_list *get_node_list(void);

67 -void sql_load_node_id(RRDHOST *host);

32 +char *get_database_extented_error(sqlite3 *database, int i, const char *description);

69 -// Help build archived hosts in memory when agent starts

70 -void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);

71 -RRDLABELS *sql_load_host_labels(uuid_t *host_id);

34 +void sql_drop_table(const char *table);

35 +void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv);

73 -// TODO: move to metadata

74 -int update_node_id(uuid_t *host_id, uuid_t *node_id);

37 +uint64_t sqlite_get_db_space(sqlite3 *db);

76 -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);

39 +int get_free_page_count(sqlite3 *database);

40 +int get_database_page_count(sqlite3 *database);

78 -// Provide statistics

79 -int sql_metadata_cache_stats(int op);

42 +int sqlite_library_init(void);

43 +void sqlite_library_shutdown(void);

81 -void sql_drop_table(const char *table);

82 -void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv);

45 +void sql_close_database(sqlite3 *database, const char *database_name);

46 +void sqlite_close_databases(void);

47 #endif //NETDATA_SQLITE_FUNCTIONS_H

src/database/sqlite/sqlite_metadata.c

+647 -29

@@ -1,6 +1,91 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

3 #include "sqlite_metadata.h"

4 +#include "sqlite3recover.h"

5 +//#include "sqlite_db_migration.h"

6 +

7 +#define DB_METADATA_VERSION 16

8 +

9 +const char *database_config[] = {

10 + "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "

11 + "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, "

12 + "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default '',"

13 + "hops INT NOT NULL DEFAULT 0,"

14 + "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0,"

15 + "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', "

16 + "entries INT NOT NULL DEFAULT 0,"

17 + "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)",

18 +

19 + "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "

20 + "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "

21 + "chart_type int, memory_mode int, history_entries)",

22 +

23 + "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "

24 + "multiplier int, divisor int , algorithm int, options text)",

25 +

26 + "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)",

27 +

28 + "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "

29 + "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))",

30 +

31 + "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)",

32 +

33 + "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, "

34 + "on_key text, class text, component text, type text, os text, hosts text, lookup text, "

35 + "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "

36 + "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "

37 + "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "

38 + "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text)",

39 +

40 + "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "

41 + "date_created INT, PRIMARY KEY(host_id, system_key))",

42 +

43 + "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, "

44 + "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))",

45 +

46 + "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)"

47 + " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END",

48 +

49 + "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, "

50 + "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, "

51 + "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))",

52 +

53 + "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, "

54 + "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "

55 + "flags int, exec_run_timestamp int, delay_up_to_timestamp int, "

56 + "info text, exec_code int, new_status real, old_status real, delay int, "

57 + "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)",

58 +

59 + "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)",

60 + "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)",

61 + "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)",

62 + "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)",

63 + "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)",

64 + "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)",

65 + "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)",

66 + "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)",

67 + "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)",

68 +

69 + NULL

70 +};

71 +

72 +const char *database_cleanup[] = {

73 + "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)",

74 + "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)",

75 + "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)",

76 + "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)",

77 + "DROP TRIGGER IF EXISTS tr_dim_del",

78 + "DROP INDEX IF EXISTS ind_d1",

79 + "DROP INDEX IF EXISTS ind_c1",

80 + "DROP INDEX IF EXISTS ind_c2",

81 + "DROP INDEX IF EXISTS alert_hash_index",

82 + "DROP INDEX IF EXISTS health_log_d_ind_4",

83 + "DROP INDEX IF EXISTS health_log_d_ind_1",

84 + "DROP INDEX IF EXISTS health_log_d_ind_5",

85 + NULL

86 +};

87 +

88 +sqlite3 *db_meta = NULL;

90 // SQL statements

@@ -131,6 +216,568 @@ struct thread_unittest {

216 unsigned *done;

217 };

218

219 +int sql_metadata_cache_stats(int op)

220 +{

221 + int count, dummy;

222 +

223 + if (unlikely(!db_meta))

224 + return 0;

225 +

226 + netdata_thread_disable_cancelability();

227 + sqlite3_db_status(db_meta, op, &count, &dummy, 0);

228 + netdata_thread_enable_cancelability();

229 + return count;

230 +}

231 +

232 +static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)

233 +{

234 + if (unlikely(!host))

235 + return;

236 +

237 + if (unlikely(!node_id)) {

238 + freez(host->node_id);

239 + __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED);

240 + return;

241 + }

242 +

243 + struct aclk_sync_cfg_t *wc = host->aclk_config;

244 +

245 + if (unlikely(!host->node_id)) {

246 + uuid_t *t = mallocz(sizeof(*host->node_id));

247 + uuid_copy(*t, *node_id);

248 + __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED);

249 + }

250 + else {

251 + uuid_copy(*(host->node_id), *node_id);

252 + }

253 +

254 + if (unlikely(!wc))

255 + sql_create_aclk_table(host, &host->host_uuid, node_id);

256 + else

257 + uuid_unparse_lower(*node_id, wc->node_id);

258 +}

259 +

260 +#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id"

261 +

262 +int update_node_id(uuid_t *host_id, uuid_t *node_id)

263 +{

264 + sqlite3_stmt *res = NULL;

265 + RRDHOST *host = NULL;

266 + int rc = 2;

267 +

268 + char host_guid[GUID_LEN + 1];

269 + uuid_unparse_lower(*host_id, host_guid);

270 + rrd_wrlock();

271 + host = rrdhost_find_by_guid(host_guid);

272 + if (likely(host))

273 + set_host_node_id(host, node_id);

274 + rrd_unlock();

275 +

276 + if (unlikely(!db_meta)) {

277 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

278 + error_report("Database has not been initialized");

279 + return 1;

280 + }

281 +

282 + rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0);

283 + if (unlikely(rc != SQLITE_OK)) {

284 + error_report("Failed to prepare statement to store node instance information");

285 + return 1;

286 + }

287 +

288 + rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC);

289 + if (unlikely(rc != SQLITE_OK)) {

290 + error_report("Failed to bind host_id parameter to store node instance information");

291 + goto failed;

292 + }

293 +

294 + rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC);

295 + if (unlikely(rc != SQLITE_OK)) {

296 + error_report("Failed to bind host_id parameter to store node instance information");

297 + goto failed;

298 + }

299 +

300 + rc = execute_insert(res);

301 + if (unlikely(rc != SQLITE_DONE))

302 + error_report("Failed to store node instance information, rc = %d", rc);

303 + rc = sqlite3_changes(db_meta);

304 +

305 +failed:

306 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

307 + error_report("Failed to finalize the prepared statement when storing node instance information");

308 +

309 + return rc - 1;

310 +}

311 +

312 +#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL"

313 +

314 +int get_node_id(uuid_t *host_id, uuid_t *node_id)

315 +{

316 + sqlite3_stmt *res = NULL;

317 + int rc;

318 +

319 + if (unlikely(!db_meta)) {

320 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

321 + error_report("Database has not been initialized");

322 + return 1;

323 + }

324 +

325 + rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0);

326 + if (unlikely(rc != SQLITE_OK)) {

327 + error_report("Failed to prepare statement to select node instance information for a host");

328 + return 1;

329 + }

330 +

331 + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

332 + if (unlikely(rc != SQLITE_OK)) {

333 + error_report("Failed to bind host_id parameter to select node instance information");

334 + goto failed;

335 + }

336 +

337 + rc = sqlite3_step_monitored(res);

338 + if (likely(rc == SQLITE_ROW && node_id))

339 + uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0)));

340 +

341 +failed:

342 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

343 + error_report("Failed to finalize the prepared statement when selecting node instance information");

344 +

345 + return (rc == SQLITE_ROW) ? 0 : -1;

346 +}

347 +

348 +#define SQL_INVALIDATE_NODE_INSTANCES \

349 + "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \

350 + "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))"

351 +

352 +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)

353 +{

354 + sqlite3_stmt *res = NULL;

355 + int rc;

356 +

357 + if (unlikely(!db_meta)) {

358 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

359 + error_report("Database has not been initialized");

360 + return;

361 + }

362 +

363 + rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0);

364 + if (unlikely(rc != SQLITE_OK)) {

365 + error_report("Failed to prepare statement to invalidate node instance ids");

366 + return;

367 + }

368 +

369 + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

370 + if (unlikely(rc != SQLITE_OK)) {

371 + error_report("Failed to bind host_id parameter to invalidate node instance information");

372 + goto failed;

373 + }

374 +

375 + if (claim_id)

376 + rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);

377 + else

378 + rc = sqlite3_bind_null(res, 2);

379 +

380 + if (unlikely(rc != SQLITE_OK)) {

381 + error_report("Failed to bind claim_id parameter to invalidate node instance information");

382 + goto failed;

383 + }

384 +

385 + rc = execute_insert(res);

386 + if (unlikely(rc != SQLITE_DONE))

387 + error_report("Failed to invalidate node instance information, rc = %d", rc);

388 +

389 +failed:

390 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

391 + error_report("Failed to finalize the prepared statement when invalidating node instance information");

392 +}

393 +

394 +#define SQL_GET_NODE_INSTANCE_LIST \

395 + "SELECT ni.node_id, ni.host_id, h.hostname " \

396 + "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0"

397 +

398 +struct node_instance_list *get_node_list(void)

399 +{

400 + struct node_instance_list *node_list = NULL;

401 + sqlite3_stmt *res = NULL;

402 + int rc;

403 +

404 + if (unlikely(!db_meta)) {

405 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

406 + error_report("Database has not been initialized");

407 + return NULL;

408 + }

409 +

410 + rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0);

411 + if (unlikely(rc != SQLITE_OK)) {

412 + error_report("Failed to prepare statement to get node instance information");

413 + return NULL;

414 + }

415 +

416 + int row = 0;

417 + char host_guid[UUID_STR_LEN];

418 + while (sqlite3_step_monitored(res) == SQLITE_ROW)

419 + row++;

420 +

421 + if (sqlite3_reset(res) != SQLITE_OK) {

422 + error_report("Failed to reset the prepared statement while fetching node instance information");

423 + goto failed;

424 + }

425 + node_list = callocz(row + 1, sizeof(*node_list));

426 + int max_rows = row;

427 + row = 0;

428 + // TODO: Check to remove lock

429 + rrd_rdlock();

430 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

431 + if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t))

432 + uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0)));

433 + if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {

434 + uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);

435 + uuid_unparse_lower(*host_id, host_guid);

436 + RRDHOST *host = rrdhost_find_by_guid(host_guid);

437 + if (!host)

438 + continue;

439 + if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {

440 + netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));

441 + continue;

442 + }

443 + uuid_copy(node_list[row].host_id, *host_id);

444 + node_list[row].queryable = 1;

445 + node_list[row].live = (host && (host == localhost || host->receiver

446 + || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;

447 + node_list[row].hops = (host && host->system_info) ? host->system_info->hops :

448 + uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0;

449 + node_list[row].hostname =

450 + sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;

451 + }

452 + row++;

453 + if (row == max_rows)

454 + break;

455 + }

456 + rrd_unlock();

457 +

458 +failed:

459 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

460 + error_report("Failed to finalize the prepared statement when fetching node instance information");

461 +

462 + return node_list;

463 +}

464 +

465 +#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id"

466 +

467 +void sql_load_node_id(RRDHOST *host)

468 +{

469 + sqlite3_stmt *res = NULL;

470 + int rc;

471 +

472 + if (unlikely(!db_meta)) {

473 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

474 + error_report("Database has not been initialized");

475 + return;

476 + }

477 +

478 + rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);

479 + if (unlikely(rc != SQLITE_OK)) {

480 + error_report("Failed to prepare statement to fetch node id");

481 + return;

482 + }

483 +

484 + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);

485 + if (unlikely(rc != SQLITE_OK)) {

486 + error_report("Failed to bind host_id parameter to load node instance information");

487 + goto failed;

488 + }

489 +

490 + rc = sqlite3_step_monitored(res);

491 + if (likely(rc == SQLITE_ROW)) {

492 + if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t)))

493 + set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0));

494 + else

495 + set_host_node_id(host, NULL);

496 + }

497 +

498 +failed:

499 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

500 + error_report("Failed to finalize the prepared statement when loading node instance information");

501 +}

502 +

503 +#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id"

504 +

505 +void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info)

506 +{

507 + int rc;

508 +

509 + sqlite3_stmt *res = NULL;

510 +

511 + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_INFO, -1, &res, 0);

512 + if (unlikely(rc != SQLITE_OK)) {

513 + error_report("Failed to prepare statement to read host information");

514 + return;

515 + }

516 +

517 + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

518 + if (unlikely(rc != SQLITE_OK)) {

519 + error_report("Failed to bind host parameter host information");

520 + goto skip;

521 + }

522 +

523 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

524 + rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0),

525 + (char *) sqlite3_column_text(res, 1));

526 + }

527 +

528 +skip:

529 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

530 + error_report("Failed to finalize the prepared statement when reading host information");

531 +}

532 +

533 +#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \

534 + "AND label_key IS NOT NULL AND label_value IS NOT NULL"

535 +

536 +RRDLABELS *sql_load_host_labels(uuid_t *host_id)

537 +{

538 + int rc;

539 +

540 + RRDLABELS *labels = NULL;

541 + sqlite3_stmt *res = NULL;

542 +

543 + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0);

544 + if (unlikely(rc != SQLITE_OK)) {

545 + error_report("Failed to prepare statement to read host information");

546 + return NULL;

547 + }

548 +

549 + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

550 + if (unlikely(rc != SQLITE_OK)) {

551 + error_report("Failed to bind host parameter host information");

552 + goto skip;

553 + }

554 +

555 + labels = rrdlabels_create();

556 +

557 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

558 + rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2));

559 + }

560 +

561 +skip:

562 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

563 + error_report("Failed to finalize the prepared statement when reading host information");

564 + return labels;

565 +}

566 +

567 +static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)

568 +{

569 + int rc, result = 1;

570 + sqlite3_stmt *res = NULL;

571 +

572 + rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);

573 + if (unlikely(rc != SQLITE_OK)) {

574 + error_report("Failed to prepare statement %s, rc = %d", sql, rc);

575 + return 1;

576 + }

577 +

578 + rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);

579 + if (unlikely(rc != SQLITE_OK)) {

580 + error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc);

581 + goto skip;

582 + }

583 +

584 + rc = execute_insert(res);

585 + if (likely(rc == SQLITE_DONE))

586 + result = SQLITE_OK;

587 + else

588 + error_report("Failed to execute %s, rc = %d", sql, rc);

589 +

590 +skip:

591 + rc = sqlite3_finalize(res);

592 + if (unlikely(rc != SQLITE_OK))

593 + error_report("Failed to finalize statement %s, rc = %d", sql, rc);

594 + return result;

595 +}

596 +

597 +static void recover_database(const char *sqlite_database, const char *new_sqlite_database)

598 +{

599 + sqlite3 *database;

600 + int rc = sqlite3_open(sqlite_database, &database);

601 + if (rc != SQLITE_OK)

602 + return;

603 +

604 + netdata_log_info("Recover %s", sqlite_database);

605 + netdata_log_info(" to %s", new_sqlite_database);

606 +

607 + // This will remove the -shm and -wal files when we close the database

608 + (void) db_execute(database, "select count(*) from sqlite_master limit 0");

609 +

610 + sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);

611 + if (recover) {

612 +

613 + rc = sqlite3_recover_run(recover);

614 +

615 + if (rc == SQLITE_OK)

616 + netdata_log_info("Recover complete");

617 + else

618 + netdata_log_error("Recover encountered an error but the database may be usable");

619 +

620 + rc = sqlite3_recover_finish(recover);

621 +

622 + (void) sqlite3_close(database);

623 +

624 + if (rc == SQLITE_OK) {

625 + rc = rename(new_sqlite_database, sqlite_database);

626 + if (rc == 0) {

627 + netdata_log_info("Renamed %s", new_sqlite_database);

628 + netdata_log_info(" to %s", sqlite_database);

629 + }

630 + }

631 + else

632 + netdata_log_error("Recover failed to free resources");

633 + }

634 + else

635 + (void) sqlite3_close(database);

636 +}

637 +

638 +

639 +static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)

640 +{

641 + uuid_t uuid;

642 +

643 + if ( argc != 1 ){

644 + sqlite3_result_null(context);

645 + return ;

646 + }

647 + int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid);

648 + if (rc == -1) {

649 + sqlite3_result_null(context);

650 + return ;

651 + }

652 +

653 + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);

654 +}

655 +

656 +void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv)

657 +{

658 + if (argc != 1 ){

659 + sqlite3_result_null(context);

660 + return ;

661 + }

662 +

663 + if (sqlite3_value_int(argv[0]) != 0) {

664 + struct timespec req = {.tv_sec = 0, .tv_nsec = 1};

665 + nanosleep(&req, NULL);

666 + }

667 +

668 + sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec());

669 +}

670 +

671 +void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv)

672 +{

673 + (void)argc;

674 + (void)argv;

675 +

676 + uuid_t uuid;

677 + uuid_generate_random(uuid);

678 + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);

679 +}

680 +

681 +// Init

682 +/*

683 + * Initialize the SQLite database

684 + * Return 0 on success

685 + */

686 +int sql_init_meta_database(db_check_action_type_t rebuild, int memory)

687 +{

688 + char *err_msg = NULL;

689 + char sqlite_database[FILENAME_MAX + 1];

690 + int rc;

691 +

692 + if (likely(!memory)) {

693 + snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);

694 + rc = unlink(sqlite_database);

695 + snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);

696 +

697 + if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {

698 + char new_sqlite_database[FILENAME_MAX + 1];

699 + snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);

700 + recover_database(sqlite_database, new_sqlite_database);

701 + if (rebuild & DB_CHECK_RECOVER)

702 + return 0;

703 + }

704 + }

705 + else

706 + strncpyz(sqlite_database, ":memory:", sizeof(sqlite_database) - 1);

707 +

708 + rc = sqlite3_open(sqlite_database, &db_meta);

709 + if (rc != SQLITE_OK) {

710 + error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));

711 + char *error_str = get_database_extented_error(db_meta, 0, "meta_open");

712 + if (error_str)

713 + analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str);

714 + freez(error_str);

715 + sqlite3_close(db_meta);

716 + db_meta = NULL;

717 + return 1;

718 + }

719 +

720 + if (rebuild & DB_CHECK_RECLAIM_SPACE) {

721 + netdata_log_info("Reclaiming space of %s", sqlite_database);

722 + rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg);

723 + if (rc != SQLITE_OK) {

724 + error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);

725 + sqlite3_free(err_msg);

726 + }

727 + else {

728 + (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");

729 + (void) sqlite3_close(db_meta);

730 + }

731 + return 1;

732 + }

733 +

734 + if (rebuild & DB_CHECK_ANALYZE) {

735 + errno = 0;

736 + netdata_log_info("Running ANALYZE on %s", sqlite_database);

737 + rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg);

738 + if (rc != SQLITE_OK) {

739 + error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg);

740 + sqlite3_free(err_msg);

741 + }

742 + else {

743 + (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");

744 + (void) sqlite3_close(db_meta);

745 + }

746 + return 1;

747 + }

748 +

749 + netdata_log_info("SQLite database %s initialization", sqlite_database);

750 +

751 + rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);

752 + if (unlikely(rc != SQLITE_OK))

753 + error_report("Failed to register internal u2h function");

754 +

755 + rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0);

756 + if (unlikely(rc != SQLITE_OK))

757 + error_report("Failed to register internal now_usec function");

758 +

759 + rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0);

760 + if (unlikely(rc != SQLITE_OK))

761 + error_report("Failed to register internal uuid_random function");

762 +

763 + int target_version = DB_METADATA_VERSION;

764 +

765 + if (likely(!memory))

766 + target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);

767 +

768 + if (configure_sqlite_database(db_meta, target_version, "meta_config"))

769 + return 1;

770 +

771 + if (init_database_batch(db_meta, &database_config[0], "meta_init"))

772 + return 1;

773 +

774 + if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup"))

775 + return 1;

776 +

777 + netdata_log_info("SQLite database initialization completed");

778 +

779 + return 0;

780 +}

781

782 // Metadata functions

783

@@ -691,35 +1338,6 @@ static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_st

1338 #endif

1339 }

1340

694 -int get_pragma_value(sqlite3 *database, const char *sql)

695 -{

696 - sqlite3_stmt *res = NULL;

697 - int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);

698 - if (unlikely(rc != SQLITE_OK))

699 - return -1;

700 -

701 - int result = -1;

702 - rc = sqlite3_step_monitored(res);

703 - if (likely(rc == SQLITE_ROW))

704 - result = sqlite3_column_int(res, 0);

705 -

706 - rc = sqlite3_finalize(res);

707 - (void) rc;

708 -

709 - return result;

710 -}

711 -

712 -

713 -int get_free_page_count(sqlite3 *database)

714 -{

715 - return get_pragma_value(database, "PRAGMA freelist_count");

716 -}

717 -

718 -int get_database_page_count(sqlite3 *database)

719 -{

720 - return get_pragma_value(database, "PRAGMA page_count");

721 -}

722 -

1341 static bool run_cleanup_loop(

1342 sqlite3_stmt *res,

1343 struct metadata_wc *wc,

src/database/sqlite/sqlite_metadata.h

+34

@@ -6,6 +6,24 @@

6 #include "sqlite3.h"

7 #include "sqlite_functions.h"

9 +// return a node list

10 +struct node_instance_list {

11 + uuid_t node_id;

12 + uuid_t host_id;

13 + char *hostname;

14 + int live;

15 + int queryable;

16 + int hops;

17 +};

18 +

19 +typedef enum db_check_action_type {

20 + DB_CHECK_NONE = (1 << 0),

21 + DB_CHECK_RECLAIM_SPACE = (1 << 1),

22 + DB_CHECK_ANALYZE = (1 << 2),

23 + DB_CHECK_CONT = (1 << 3),

24 + DB_CHECK_RECOVER = (1 << 4),

25 +} db_check_action_type_t;

26 +

27 // To initialize and shutdown

28 void metadata_sync_init(void);

29 void metadata_sync_shutdown(void);

@@ -20,6 +38,22 @@ void metadata_queue_load_host_context(RRDHOST *host);

38 void metadata_delete_host_chart_labels(char *machine_guid);

39 void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc);

41 +int sql_metadata_cache_stats(int op);

42 +

43 +int get_node_id(uuid_t *host_id, uuid_t *node_id);

44 +int update_node_id(uuid_t *host_id, uuid_t *node_id);

45 +struct node_instance_list *get_node_list(void);

46 +void sql_load_node_id(RRDHOST *host);

47 +

48 +// Help build archived hosts in memory when agent starts

49 +void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);

50 +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);

51 +RRDLABELS *sql_load_host_labels(uuid_t *host_id);

52 +

53 +uint64_t sqlite_get_meta_space(void);

54 +int sql_init_meta_database(db_check_action_type_t rebuild, int memory);

55 +void sql_close_meta_database(void);

56 +

57 // UNIT TEST

58 int metadata_unittest(void);

59 #endif //NETDATA_SQLITE_METADATA_H

src/ml/ml.cc

+6 -3

@@ -1809,13 +1809,16 @@ void ml_init()

1809 }

1810 }

1811

1812 +uint64_t sqlite_get_ml_space(void)

1813 +{

1814 + return sqlite_get_db_space(db);

1815 +}

1816 +

1817 void ml_fini() {

1818 if (!Cfg.enable_anomaly_detection || !db)

1819 return;

1820

1816 - int rc = sqlite3_close_v2(db);

1817 - if (unlikely(rc != SQLITE_OK))

1818 - error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));

1821 + sql_close_database(db, "ML");

1822 db = NULL;

1823 }

1824

src/ml/ml.h

@@ -46,6 +46,7 @@ void ml_update_global_statistics_charts(uint64_t models_consulted);

47 bool ml_host_get_host_status(RRDHOST *rh, struct ml_metrics_statistics *mlm);

48 bool ml_host_running(RRDHOST *rh);

49 +uint64_t sqlite_get_ml_space(void);

51 #ifdef __cplusplus

52 };