@cryptotaxi247 / netdata-1 / commits / 7553d19a1

Verify and cleanup deleted contexts (#19389)

* Verify and cleanup deleted contexts * Remove context from cleanup list immediately when processed * Throttle cleanup if wal size is getting big * Add context check creation time

Stelios Fragkakis committed Jan 14, 2025 at 17:21 UTC 7553d19a18eab4f1c75e6edb02b9c51de55947ee

4 files changed +253 -9

src/database/sqlite/sqlite_context.c

+77 -1

@@ -13,6 +13,13 @@ const char *database_context_config[] = {

13 "last_time_t INT NOT NULL, deleted INT NOT NULL, "

14 "family TEXT, PRIMARY KEY (host_id, id))",

16 + "CREATE TABLE IF NOT EXISTS context_metadata_cleanup (id INTEGER PRIMARY KEY, host_id BLOB, context TEXT NOT NULL, date_created INT, "

17 + "UNIQUE (host_id, context))",

18 +

19 + "CREATE TRIGGER IF NOT EXISTS del_context1 AFTER DELETE ON context "

20 + "BEGIN INSERT INTO context_metadata_cleanup (host_id, context, date_created) "

21 + "VALUES (old.host_id, old.id, UNIXEPOCH()) ON CONFLICT DO UPDATE SET date_created = excluded.date_created; END",

22 +

23 NULL

24 };

@@ -261,8 +268,77 @@ done:

268 return (rc_stored != SQLITE_DONE);

269 }

270

264 -// Delete a context

271 +#define CTX_DELETE_CONTEXT_META_CLEANUP_ITEM "DELETE FROM context_metadata_cleanup WHERE host_id = @host_id AND context = @context"

272 +

273 +void ctx_delete_metadata_cleanup_context(sqlite3_stmt **res, nd_uuid_t(*host_uuid), const char *context)

274 +{

275 + if (!*res) {

276 + if (!PREPARE_STATEMENT(db_context_meta, CTX_DELETE_CONTEXT_META_CLEANUP_ITEM, res))

277 + return;

278 + }

279 +

280 + int param = 0;

281 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(*res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));

282 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, context, -1, SQLITE_STATIC));

283 +

284 + param = 0;

285 + int rc = sqlite3_step_monitored(*res);

286 + if (rc != SQLITE_DONE)

287 + error_report("Failed to delete context check entry, rc = %d", rc);

288 +

289 +done:

290 + REPORT_BIND_FAIL(*res, param);

291 + SQLITE_RESET(*res);

292 +}

293 +

294 +// Schedule context cleanup for host

295 +#define CTX_GET_CONTEXT_META_CLEANUP_LIST "SELECT context FROM context_metadata_cleanup WHERE host_id = @host_id"

296 +

297 +void ctx_get_context_list_to_cleanup(nd_uuid_t *host_uuid, void (*cleanup_cb)(Pvoid_t JudyL, void *data), void *data)

298 +{

299 + if (unlikely(!host_uuid))

300 + return;

301 +

302 + sqlite3_stmt *res = NULL;

303 +

304 + if (!PREPARE_STATEMENT(db_context_meta, CTX_GET_CONTEXT_META_CLEANUP_LIST, &res))

305 + return;

306 +

307 + int param = 0;

308 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));

309 + param = 0;

310 +

311 + const char *context;

312 + Pvoid_t CTX_JudyL = NULL;

313 + Pvoid_t *Pvalue;

314 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

315 + context = (char *) sqlite3_column_text(res, 0);

316 + STRING *ctx = string_strdupz(context);

317 + Pvalue = JudyLIns(&CTX_JudyL, (Word_t) ctx, PJE0);

318 + if (*Pvalue)

319 + string_freez(ctx);

320 + else

321 + *(int *)Pvalue = 1;

322 + }

323

324 + if (CTX_JudyL) {

325 + cleanup_cb(CTX_JudyL, data);

326 +

327 + bool first = true;

328 + Word_t Index = 0;

329 + while ((Pvalue = JudyLFirstThenNext(CTX_JudyL, &Index, &first))) {

330 + STRING *ctx = (STRING *) Index;

331 + string_freez(ctx);

332 + }

333 + }

334 + (void)JudyLFreeArray(&CTX_JudyL, PJE0);

335 +

336 +done:

337 + REPORT_BIND_FAIL(res, param);

338 + SQLITE_FINALIZE(res);

339 +}

340 +

341 +// Delete a context

342 #define CTX_DELETE_CONTEXT "DELETE FROM context WHERE host_id = @host_id AND id = @context"

343 int ctx_delete_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data)

344 {

src/database/sqlite/sqlite_context.h

@@ -63,6 +63,9 @@ void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_

64 int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);

66 +void ctx_get_context_list_to_cleanup(nd_uuid_t *host_uuid, void (*cleanup_cb)(Pvoid_t context, void *data), void *data);

67 +void ctx_delete_metadata_cleanup_context(sqlite3_stmt **context_res, nd_uuid_t(*host_uuid), const char *context);

68 +

69 #define ctx_update_context(host_uuid, context_data) ctx_store_context(host_uuid, context_data)

71 int ctx_delete_context(nd_uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data);

src/database/sqlite/sqlite_functions.c

+6 -3

@@ -5,6 +5,8 @@

5 #define MAX_PREPARED_STATEMENTS (32)

6 pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];

8 +long long def_journal_size_limit = 16777216;

9 +

10 SQLITE_API int sqlite3_exec_monitored(

11 sqlite3 *db, /* An open database */

12 const char *sql, /* SQL to be evaluated */

@@ -84,7 +86,6 @@ int configure_sqlite_database(sqlite3 *database, int target_version, const char

86 const char *def_synchronous = "NORMAL";

87 const char *def_journal_mode = "WAL";

88 const char *def_temp_store = "MEMORY";

87 - long long def_journal_size_limit = 16777216;

89 long long def_cache_size = -2000;

91 // https://www.sqlite.org/pragma.html#pragma_auto_vacuum

@@ -122,8 +123,10 @@ int configure_sqlite_database(sqlite3 *database, int target_version, const char

123 // https://www.sqlite.org/pragma.html#pragma_journal_size_limit

124 // PRAGMA schema.journal_size_limit = N ;

125 snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_size_limit=%lld", def_journal_size_limit);

125 - if (config_exists(CONFIG_SECTION_SQLITE, "journal size limit"))

126 - snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_size_limit=%lld", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", def_journal_size_limit));

126 + if (config_exists(CONFIG_SECTION_SQLITE, "journal size limit")) {

127 + def_journal_size_limit = config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", def_journal_size_limit);

128 + snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_size_limit=%lld", def_journal_size_limit);

129 + }

130 if (init_database_batch(database, list, description))

131 return 1;

132

src/database/sqlite/sqlite_metadata.c

+167 -5

@@ -8,6 +8,8 @@

9 #define DB_METADATA_VERSION 18

11 +extern long long def_journal_size_limit;

12 +

13 const char *database_config[] = {

14 "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "

15 "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, "

@@ -184,7 +186,6 @@ sqlite3 *db_meta = NULL;

186

187 enum metadata_opcode {

188 METADATA_DATABASE_NOOP = 0,

187 - METADATA_DATABASE_TIMER,

189 METADATA_DEL_DIMENSION,

190 METADATA_STORE_CLAIM_ID,

191 METADATA_ADD_HOST_INFO,

@@ -668,6 +669,31 @@ void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv

669 sqlite3_result_blob(context, &uuid, sizeof(nd_uuid_t), SQLITE_TRANSIENT);

670 }

671

672 +static int64_t sql_get_wal_size(const char *database_file)

673 +{

674 + char filename[FILENAME_MAX + 1];

675 + snprintfz(filename, sizeof(filename) - 1, "%s/%s-wal", netdata_configured_cache_dir, database_file);

676 +

677 + uv_fs_t req;

678 + int result = uv_fs_stat(NULL, &req, filename, NULL);

679 + int64_t file_size = result >= 0 ? (int64_t) req.statbuf.st_size : -1;

680 +

681 + uv_fs_req_cleanup(&req);

682 + return file_size;

683 +}

684 +

685 +#define SQLITE_METADATA_WAL_LIMIT_X (10)

686 +

687 +bool sql_metadata_wal_size_acceptable()

688 +{

689 + int64_t wal_size = sql_get_wal_size("netdata-meta.db");

690 +

691 + if (wal_size > SQLITE_METADATA_WAL_LIMIT_X * def_journal_size_limit)

692 + return false;

693 +

694 + return true;

695 +}

696 +

697 // Init

698 /*

699 * Initialize the SQLite database

@@ -898,6 +924,29 @@ done:

924 return rc != SQLITE_DONE;

925 }

926

927 +#define SQL_DELETE_DIMENSION_BY_ID "DELETE FROM dimension WHERE rowid = @dimension_row AND dim_id = @uuid"

928 +

929 +static void delete_dimension_by_rowid(sqlite3_stmt **res, int64_t dimension_id, nd_uuid_t *dim_uuid)

930 +{

931 + if (!*res) {

932 + if (!PREPARE_STATEMENT(db_meta, SQL_DELETE_DIMENSION_BY_ID, res))

933 + return;

934 + }

935 +

936 + int param = 0;

937 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(*res, ++param, dimension_id));

938 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(*res, ++param, dim_uuid, sizeof(*dim_uuid), SQLITE_STATIC));

939 +

940 + param = 0;

941 + int rc = sqlite3_step_monitored(*res);

942 + if (unlikely(rc != SQLITE_DONE))

943 + error_report("Failed to delete dimension id, rc = %d", rc);

944 +

945 +done:

946 + REPORT_BIND_FAIL(*res, param);

947 + SQLITE_RESET(*res);

948 +}

949 +

950 static void delete_dimension_uuid(nd_uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused)

951 {

952 static __thread sqlite3_stmt *res = NULL;

@@ -1542,11 +1591,127 @@ void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int

1591 }

1592 }

1593

1594 +

1595 +#define SQL_SELECT_HOST_CTX_CHART_DIM_LIST \

1596 + "SELECT d.dim_id, d.rowid FROM chart c, dimension d WHERE c.chart_id = d.chart_id AND c.rowid = @rowid"

1597 +

1598 +static bool clean_host_chart_dimensions(sqlite3_stmt **res, int64_t chart_row_id, size_t *checked, size_t *deleted)

1599 +{

1600 + struct metadata_wc *wc = &metasync_worker;

1601 +

1602 + if (!*res) {

1603 + if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_HOST_CTX_CHART_DIM_LIST, res))

1604 + return false;

1605 + }

1606 + int param = 0;

1607 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(*res, ++param, chart_row_id));

1608 + param = 0;

1609 +

1610 + sqlite3_stmt *dim_del_stmt = NULL;

1611 +

1612 + bool can_continue = true;

1613 +

1614 + while (can_continue && sqlite3_step_monitored(*res) == SQLITE_ROW) {

1615 + if (sqlite3_column_bytes(*res, 0) != sizeof(nd_uuid_t))

1616 + continue;

1617 +

1618 + nd_uuid_t *dim_uuid = (nd_uuid_t *)sqlite3_column_blob(*res, 0);

1619 + int64_t dimension_id = sqlite3_column_int64(*res, 1);

1620 +

1621 + if (dimension_can_be_deleted(dim_uuid, NULL, false)) {

1622 + delete_dimension_by_rowid(&dim_del_stmt, dimension_id, dim_uuid);

1623 + (*deleted)++;

1624 + }

1625 + (*checked)++;

1626 + can_continue = (!metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)) && sql_metadata_wal_size_acceptable();

1627 + }

1628 + SQLITE_FINALIZE(dim_del_stmt);

1629 +

1630 +done:

1631 + REPORT_BIND_FAIL(*res, param);

1632 + SQLITE_RESET(*res);

1633 + return can_continue;

1634 +}

1635 +

1636 +#define SQL_SELECT_HOST_CTX_CHART_LIST "SELECT rowid, context FROM chart WHERE host_id = @host"

1637 +

1638 +static void cleanup_host_context_metadata(Pvoid_t CTX_JudyL, void *data)

1639 +{

1640 + if (!CTX_JudyL || !data)

1641 + return;

1642 +

1643 + struct metadata_wc *wc = &metasync_worker;

1644 +

1645 + RRDHOST *host = data;

1646 +

1647 + sqlite3_stmt *res = NULL;

1648 + sqlite3_stmt *dimension_res = NULL;

1649 + sqlite3_stmt *context_res = NULL;

1650 +

1651 + if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_HOST_CTX_CHART_LIST, &res))

1652 + return;

1653 +

1654 + Word_t num_of_contexts = JudyLCount(CTX_JudyL, 0, -1, PJE0);

1655 +

1656 + nd_log_daemon(NDLP_DEBUG, "Verifying the retention of %zu contexts for host %s", num_of_contexts, rrdhost_hostname(host));

1657 +

1658 + int param = 0;

1659 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_id.uuid, sizeof(host->host_id.uuid), SQLITE_STATIC));

1660 +

1661 + param = 0;

1662 + Pvoid_t *Pvalue;

1663 + int64_t chart_row_id;

1664 +

1665 + size_t deleted = 0;

1666 + size_t checked = 0;

1667 +

1668 + bool can_continue = true;

1669 + while (can_continue && sqlite3_step_monitored(res) == SQLITE_ROW) {

1670 + chart_row_id = sqlite3_column_int64(res, 0);

1671 + const char *context = (char *)sqlite3_column_text(res, 1);

1672 + STRING *ctx = string_strdupz(context);

1673 + Pvalue = JudyLGet(CTX_JudyL, (Word_t)ctx, PJE0);

1674 + if (Pvalue) {

1675 + can_continue = clean_host_chart_dimensions(&dimension_res, chart_row_id, &checked, &deleted);

1676 + ctx_delete_metadata_cleanup_context(&context_res, &host->host_id.uuid, context);

1677 + }

1678 + string_freez(ctx);

1679 + can_continue =

1680 + can_continue && (!metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)) && sql_metadata_wal_size_acceptable();

1681 + }

1682 + SQLITE_FINALIZE(dimension_res);

1683 + SQLITE_FINALIZE(context_res);

1684 +

1685 + nd_log_daemon(

1686 + NDLP_DEBUG,

1687 + "Verified the contexts of host %s (Checked %zu metrics and removed %zu)",

1688 + rrdhost_hostname(host),

1689 + checked,

1690 + deleted);

1691 +

1692 +done:

1693 + REPORT_BIND_FAIL(res, param);

1694 + SQLITE_FINALIZE(res);

1695 +}

1696 +

1697 void run_metadata_cleanup(struct metadata_wc *wc)

1698 {

1699 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))

1700 return;

1701

1702 + if (sql_metadata_wal_size_acceptable()) {

1703 + RRDHOST *host;

1704 + dfe_start_reentrant(rrdhost_root_index, host) {

1705 + ctx_get_context_list_to_cleanup(&host->host_id.uuid, cleanup_host_context_metadata, host);

1706 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN) || false == sql_metadata_wal_size_acceptable())

1707 + break;

1708 + }

1709 + dfe_done(host);

1710 + }

1711 +

1712 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))

1713 + return;

1714 +

1715 check_dimension_metadata(wc);

1716 check_chart_metadata(wc);

1717 check_label_metadata(wc);

@@ -1956,7 +2121,7 @@ static void do_chart_label_cleanup(struct judy_list_t *cl_cleanup_data)

2121 }

2122

2123 // Worker thread to scan hosts for pending metadata to store

1959 -static void start_metadata_hosts(uv_work_t *req __maybe_unused)

2124 +static void start_metadata_hosts(uv_work_t *req)

2125 {

2126 register_libuv_worker_jobs();

2127

@@ -2072,7 +2237,6 @@ static void metadata_event_loop(void *arg)

2237 {

2238 worker_register("METASYNC");

2239 worker_register_job_name(METADATA_DATABASE_NOOP, "noop");

2075 - worker_register_job_name(METADATA_DATABASE_TIMER, "timer");

2240 worker_register_job_name(METADATA_DEL_DIMENSION, "delete dimension");

2241 worker_register_job_name(METADATA_STORE_CLAIM_ID, "add claim id");

2242 worker_register_job_name(METADATA_ADD_HOST_INFO, "add host info");

@@ -2130,7 +2294,6 @@ static void metadata_event_loop(void *arg)

2294 nd_uuid_t *uuid;

2295 RRDHOST *host = NULL;

2296 ALARM_ENTRY *ae = NULL;

2133 -// struct aclk_sync_cfg_t *host_aclk_sync;

2297

2298 worker_is_idle();

2299 uv_run(loop, UV_RUN_DEFAULT);

@@ -2156,7 +2319,6 @@ static void metadata_event_loop(void *arg)

2319

2320 switch (opcode) {

2321 case METADATA_DATABASE_NOOP:

2159 - case METADATA_DATABASE_TIMER:

2322 break;

2323 case METADATA_DEL_DIMENSION:

2324 uuid = (nd_uuid_t *) cmd.param[0];