@cryptotaxi247 / netdata-1 / commits / 0329cd404

Improve metadata cleanup (#19479)

* Add workers Do not delete labels when removing a node Complete one scan for chart and dimension metadata check * Fix logs, formatting * Remove unused function * Code cleanup * Fixed missing prepared statement check * Do checks for vacuum at most every 60 seconds

Stelios Fragkakis committed Jan 27, 2025 at 21:41 UTC 0329cd404f3c5f13b06adb0c4b699737b51f5753
5 files changed +374 -381
src/daemon/libuv_workers.c
+4
@@ -59,9 +59,13 @@ void register_libuv_worker_jobs() {
59 worker_register_job_name(UV_EVENT_CTX_CLEANUP, "metadata ctx cleanup");
60 worker_register_job_name(UV_EVENT_STORE_ALERT_TRANSITIONS, "metadata store alert transitions");
61 worker_register_job_name(UV_EVENT_CHART_LABEL_CLEANUP, "metadata chart label cleanup");
62 + worker_register_job_name(UV_EVENT_HEALTH_LOG_CLEANUP, "alert transitions cleanup");
63 worker_register_job_name(UV_EVENT_UUID_DELETION, "metadata dimension deletion");
64 worker_register_job_name(UV_EVENT_DIMENSION_CLEANUP, "metadata dimension cleanup");
65 worker_register_job_name(UV_EVENT_CHART_CLEANUP, "metadata chart cleanup");
66 + worker_register_job_name(UV_EVENT_STORE_HOST, "metadata store host");
67 + worker_register_job_name(UV_EVENT_STORE_CHART, "metadata store chart");
68 + worker_register_job_name(UV_EVENT_STORE_DIMENSION, "metadata store dimension");
69
70 // aclk_sync
71 worker_register_job_name(UV_EVENT_ACLK_NODE_INFO, "aclk host node info");
src/daemon/libuv_workers.h
+4
@@ -49,7 +49,11 @@ enum event_loop_job {
49 UV_EVENT_METADATA_ML_LOAD,
50 UV_EVENT_CTX_CLEANUP_SCHEDULE,
51 UV_EVENT_CTX_CLEANUP,
52 + UV_EVENT_STORE_HOST,
53 + UV_EVENT_STORE_CHART,
54 + UV_EVENT_STORE_DIMENSION,
55 UV_EVENT_STORE_ALERT_TRANSITIONS,
56 + UV_EVENT_HEALTH_LOG_CLEANUP,
57 UV_EVENT_CHART_LABEL_CLEANUP,
58 UV_EVENT_UUID_DELETION,
59 UV_EVENT_DIMENSION_CLEANUP,
src/database/sqlite/sqlite_aclk.c
+2 -6
@@ -298,14 +298,10 @@ static void sql_unregister_node(char *machine_guid)
298 param = 0;
299
300 rc = sqlite3_step_monitored(res);
301 - if (unlikely(rc != SQLITE_DONE)) {
301 + if (unlikely(rc != SQLITE_DONE))
302 error_report("Failed to execute command to remove host node id");
303 - } else {
304 - // node: machine guid will be freed after processing
303 + else
304 invalidate_host_last_connected(&host_uuid);
306 - metadata_delete_host_chart_labels(machine_guid);
307 - machine_guid = NULL;
308 - }
305
306 done:
307 REPORT_BIND_FAIL(res, param);
src/database/sqlite/sqlite_metadata.c
+364 -374
@@ -178,16 +178,14 @@ sqlite3 *db_meta = NULL;
178 #define METADATA_MAINTENANCE_REPEAT (60) // Repeat if last run for dimensions, charts, labels needs more work
179 #define METADATA_MAINTENANCE_CTX_CLEAN_REPEAT (300) // Repeat if last run for dimensions, charts, labels needs more work
180 #define METADATA_HEALTH_LOG_INTERVAL (3600) // Repeat maintenance for health
181 -#define METADATA_DIM_CHECK_INTERVAL (3600) // Repeat maintenance for dimensions
182 -#define METADATA_CHART_CHECK_INTERVAL (3600) // Repeat maintenance for charts
181 #define METADATA_LABEL_CHECK_INTERVAL (3600) // Repeat maintenance for labels
182 #define METADATA_RUNTIME_THRESHOLD (5) // Run time threshold for cleanup task
183
184 #define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata
185 #define METADATA_HOST_CHECK_INTERVAL (5) // Repeat check for pending metadata
188 -#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)
186 #define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop
187
188 +#define DATABASE_VACUUM_FREQUENCY_SECONDS (60)
189 #define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum
190 #define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum
191
@@ -297,11 +295,6 @@ struct host_ctx_cleanup_s {
295
296 static void ctx_delete_metadata_cleanup_context(sqlite3_stmt **res, nd_uuid_t *host_uuid, const char *context)
297 {
300 -// char host_str[UUID_STR_LEN];
301 -// uuid_unparse_lower(*host_uuid, host_str);
302 -// nd_log_daemon(NDLP_INFO, "Will delete context %s for host %s because it was checked", context, host_str);
303 -// return;
304 -
298 if (!*res) {
299 if (!PREPARE_STATEMENT(db_meta, CTX_DELETE_CONTEXT_META_CLEANUP_ITEM, res))
300 return;
@@ -926,29 +919,6 @@ struct query_build {
919 char uuid_str[UUID_STR_LEN];
920 };
921
929 -#define SQL_DELETE_CHART_LABELS_BY_HOST \
930 - "DELETE FROM chart_label WHERE chart_id in (SELECT chart_id FROM chart WHERE host_id = @host_id)"
931 -
932 -static void delete_host_chart_labels(nd_uuid_t *host_uuid)
933 -{
934 - sqlite3_stmt *res = NULL;
935 -
936 - if (!PREPARE_STATEMENT(db_meta, SQL_DELETE_CHART_LABELS_BY_HOST, &res))
937 - return;
938 -
939 - int param = 0;
940 - SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));
941 -
942 - param = 0;
943 - int rc = sqlite3_step_monitored(res);
944 - if (unlikely(rc != SQLITE_DONE))
945 - error_report("Failed to execute command to remove chart labels, rc = %d", rc);
946 -
947 -done:
948 - REPORT_BIND_FAIL(res, param);
949 - SQLITE_FINALIZE(res);
950 -}
951 -
922 static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
923 struct query_build *lb = data;
924 if (unlikely(!lb->count))
@@ -1164,86 +1134,81 @@ static bool store_host_systeminfo(RRDHOST *host)
1134 * Store a chart in the database
1135 */
1136
1167 -static int store_chart_metadata(RRDSET *st)
1137 +static int store_chart_metadata(RRDSET *st, sqlite3_stmt **res)
1138 {
1169 - static __thread sqlite3_stmt *res = NULL;
1170 -
1171 - if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_STORE_CHART, &res))
1172 - return 1;
1139 + if (!*res) {
1140 + if (!PREPARE_STATEMENT(db_meta, SQL_STORE_CHART, res))
1141 + return 1;
1142 + }
1143
1144 + int rc = SQLITE_DONE;
1145 int param = 0;
1175 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(res, ++param, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC));
1176 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(res, ++param, &st->rrdhost->host_id.uuid, sizeof(st->rrdhost->host_id.uuid), SQLITE_STATIC));
1177 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, string2str(st->parts.type), -1, SQLITE_STATIC));
1178 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, string2str(st->parts.id), -1, SQLITE_STATIC));
1146 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(*res, ++param, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC));
1147 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(*res, ++param, &st->rrdhost->host_id.uuid, sizeof(st->rrdhost->host_id.uuid), SQLITE_STATIC));
1148 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, string2str(st->parts.type), -1, SQLITE_STATIC));
1149 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, string2str(st->parts.id), -1, SQLITE_STATIC));
1150
1151 const char *name = string2str(st->parts.name);
1152 if (name && *name)
1182 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC));
1153 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, name, -1, SQLITE_STATIC));
1154 else
1184 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_null(res, ++param));
1185 -
1186 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, rrdset_family(st), -1, SQLITE_STATIC));
1187 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, rrdset_context(st), -1, SQLITE_STATIC));
1188 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, rrdset_title(st), -1, SQLITE_STATIC));
1189 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, rrdset_units(st), -1, SQLITE_STATIC));
1190 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, rrdset_plugin_name(st), -1, SQLITE_STATIC));
1191 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, rrdset_module_name(st), -1, SQLITE_STATIC));
1192 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, (int) st->priority));
1193 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, st->update_every));
1194 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, st->chart_type));
1195 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, st->rrd_memory_mode));
1196 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, (int) st->db.entries));
1197 -
1198 - int store_rc = execute_insert(res);
1199 - if (unlikely(store_rc != SQLITE_DONE))
1200 - error_report("Failed to store chart, rc = %d", store_rc);
1155 + SQLITE_BIND_FAIL(done, sqlite3_bind_null(*res, ++param));
1156 +
1157 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, rrdset_family(st), -1, SQLITE_STATIC));
1158 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, rrdset_context(st), -1, SQLITE_STATIC));
1159 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, rrdset_title(st), -1, SQLITE_STATIC));
1160 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, rrdset_units(st), -1, SQLITE_STATIC));
1161 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, rrdset_plugin_name(st), -1, SQLITE_STATIC));
1162 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, rrdset_module_name(st), -1, SQLITE_STATIC));
1163 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, (int) st->priority));
1164 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, st->update_every));
1165 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, st->chart_type));
1166 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, st->rrd_memory_mode));
1167 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, (int) st->db.entries));
1168
1202 - SQLITE_RESET(res);
1203 -
1204 - return store_rc != SQLITE_DONE;
1169 + param = 0;
1170 + rc = sqlite3_step_monitored(*res);
1171 + if (unlikely(rc != SQLITE_DONE))
1172 + error_report("Failed to store chart, rc = %d", rc);
1173
1206 -bind_fail:
1207 - REPORT_BIND_FAIL(res, param);
1208 - SQLITE_RESET(res);
1209 - return 1;
1174 +done:
1175 + REPORT_BIND_FAIL(*res, param);
1176 + SQLITE_RESET(*res);
1177 + return rc != SQLITE_DONE;
1178 }
1179
1212 -/*
1213 - * Store a dimension
1214 - */
1215 -static int store_dimension_metadata(RRDDIM *rd)
1180 +static bool store_dimension_metadata(RRDDIM *rd, sqlite3_stmt **res)
1181 {
1217 - static __thread sqlite3_stmt *res = NULL;
1218 - int rc, param = 0;
1182 + if (!*res) {
1183 + if (!PREPARE_STATEMENT(db_meta, SQL_STORE_DIMENSION, res))
1184 + return 1;
1185 + }
1186
1220 - if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_STORE_DIMENSION, &res))
1221 - return 1;
1187 + int rc = SQLITE_DONE;
1188 + int param = 0;
1189
1190 nd_uuid_t *rd_uuid = uuidmap_uuid_ptr(rd->uuid);
1224 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(res, ++param, rd_uuid, sizeof(*rd_uuid), SQLITE_STATIC));
1225 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(res, ++param, &rd->rrdset->chart_uuid, sizeof(rd->rrdset->chart_uuid), SQLITE_STATIC));
1226 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, string2str(rd->id), -1, SQLITE_STATIC));
1227 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, string2str(rd->name), -1, SQLITE_STATIC));
1228 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, (int) rd->multiplier));
1229 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, (int ) rd->divisor));
1230 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, rd->algorithm));
1191 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(*res, ++param, rd_uuid, sizeof(*rd_uuid), SQLITE_STATIC));
1192 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(*res, ++param, &rd->rrdset->chart_uuid, sizeof(rd->rrdset->chart_uuid), SQLITE_STATIC));
1193 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, string2str(rd->id), -1, SQLITE_STATIC));
1194 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, string2str(rd->name), -1, SQLITE_STATIC));
1195 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, (int) rd->multiplier));
1196 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, (int ) rd->divisor));
1197 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(*res, ++param, rd->algorithm));
1198 if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
1232 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(res, ++param, "hidden", -1, SQLITE_STATIC));
1199 + SQLITE_BIND_FAIL(done, sqlite3_bind_text(*res, ++param, "hidden", -1, SQLITE_STATIC));
1200 else
1234 - SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_null(res, ++param));
1201 + SQLITE_BIND_FAIL(done, sqlite3_bind_null(*res, ++param));
1202
1236 - rc = execute_insert(res);
1203 + param = 0;
1204 + rc = sqlite3_step_monitored(*res);
1205 if (unlikely(rc != SQLITE_DONE))
1206 error_report("Failed to store dimension, rc = %d", rc);
1207
1240 - SQLITE_RESET(res);
1241 - return 0;
1242 -
1243 -bind_fail:
1244 - REPORT_BIND_FAIL(res, param);
1245 - SQLITE_RESET(res);
1246 - return 1;
1208 +done:
1209 + REPORT_BIND_FAIL(*res, param);
1210 + SQLITE_RESET(*res);
1211 + return (rc != SQLITE_DONE);
1212 }
1213
1214 static bool dimension_can_be_deleted(nd_uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused)
@@ -1293,8 +1258,10 @@ static bool run_cleanup_loop(
1258
1259 time_t start_running = now_monotonic_sec();
1260 bool time_expired = false;
1296 - while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW &&
1297 - (*total_deleted < MAX_METADATA_CLEANUP && *total_checked < MAX_METADATA_CLEANUP)) {
1261 +
1262 + uint32_t l_checked = 0;
1263 + uint32_t l_deleted = 0;
1264 + while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW) {
1265 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
1266 break;
1267
@@ -1303,13 +1270,18 @@ static bool run_cleanup_loop(
1270
1271 if (rc == true) {
1272 action_cb((nd_uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag);
1306 - (*total_deleted)++;
1273 + l_deleted++;
1274 +// if (false == sql_metadata_wal_size_acceptable())
1275 +// (void) sqlite3_wal_checkpoint(db_meta, NULL);
1276 }
1277
1309 - (*total_checked)++;
1278 + l_checked++;
1279 time_expired = ((now_monotonic_sec() - start_running) > METADATA_RUNTIME_THRESHOLD);
1280 }
1312 - return time_expired || (*total_checked == MAX_METADATA_CLEANUP) || (*total_deleted == MAX_METADATA_CLEANUP);
1281 +
1282 + (*total_checked) += l_checked;
1283 + (*total_deleted) += l_deleted;
1284 + return time_expired;
1285 }
1286
1287
@@ -1383,23 +1355,54 @@ skip:
1355 SQLITE_FINALIZE(res);
1356 }
1357
1386 -static void check_dimension_metadata(struct metadata_wc *wc)
1358 +static uint64_t get_rowid_from_statement(const char *sql)
1359 +{
1360 + sqlite3_stmt *res = NULL;
1361 +
1362 + if (!PREPARE_STATEMENT(db_meta, sql, &res))
1363 + return 0;
1364 +
1365 + uint64_t rowid = 0;
1366 +
1367 + if (sqlite3_step_monitored(res) == SQLITE_ROW) {
1368 + rowid = sqlite3_column_int64(res, 0);
1369 + }
1370 +
1371 + SQLITE_FINALIZE(res);
1372 + return rowid;
1373 +}
1374 +
1375 +
1376 +#define SQL_GET_MAX_DIM_ROW_ID "SELECT MAX(rowid) FROM dimension"
1377 +
1378 +static bool check_dimension_metadata(struct metadata_wc *wc)
1379 {
1380 static time_t next_execution_t = 0;
1381 static uint64_t last_row_id = 0;
1382 + static uint64_t max_row_id = 0;
1383
1384 time_t now = now_realtime_sec();
1385
1393 - if (!next_execution_t)
1386 + if (!next_execution_t) {
1387 next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
1388 + max_row_id = get_rowid_from_statement(SQL_GET_MAX_DIM_ROW_ID);
1389 + nd_log(NDLS_DAEMON, NDLP_INFO, "Dimension metadata check has been scheduled to run (max id = %lu)", max_row_id);
1390 + }
1391
1392 if (next_execution_t && next_execution_t > now)
1397 - return;
1393 + return true;
1394 +
1395 + if (max_row_id && last_row_id >= max_row_id) {
1396 + nd_log_daemon(NDLP_INFO, "Dimension metadata check completed");
1397 + // For long running agents, check in a week
1398 + next_execution_t = now + 604800;
1399 + return true;
1400 + }
1401
1402 sqlite3_stmt *res = NULL;
1403
1404 if (!PREPARE_STATEMENT(db_meta, SELECT_DIMENSION_LIST, &res))
1402 - return;
1405 + return true;
1406
1407 uint32_t total_checked = 0;
1408 uint32_t total_deleted = 0;
@@ -1408,7 +1411,7 @@ static void check_dimension_metadata(struct metadata_wc *wc)
1411
1412 worker_is_busy(UV_EVENT_DIMENSION_CLEANUP);
1413
1411 - bool more_to_do = run_cleanup_loop(
1414 + (void) run_cleanup_loop(
1415 res,
1416 wc,
1417 dimension_can_be_deleted,
@@ -1422,53 +1425,63 @@ static void check_dimension_metadata(struct metadata_wc *wc)
1425 false);
1426
1427 now = now_realtime_sec();
1425 - if (more_to_do)
1426 - next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
1427 - else {
1428 - last_row_id = 0;
1429 - next_execution_t = now + METADATA_DIM_CHECK_INTERVAL;
1430 - }
1431 -
1432 - nd_log(
1433 - NDLS_DAEMON,
1428 + next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
1429 + nd_log_daemon(
1430 NDLP_DEBUG,
1435 - "Dimensions checked %u, deleted %u. Checks will %s in %lld seconds",
1431 + "Dimensions checked %u, deleted %u. Checks will resume in %d seconds",
1432 total_checked,
1433 total_deleted,
1438 - last_row_id ? "resume" : "restart",
1439 - (long long)(next_execution_t - now));
1434 + METADATA_MAINTENANCE_REPEAT);
1435
1436 SQLITE_FINALIZE(res);
1437
1438 worker_is_idle();
1439 + return false;
1440 }
1441
1446 -static void check_chart_metadata(struct metadata_wc *wc)
1442 +#define SQL_GET_MAX_CHART_ROW_ID "SELECT MAX(rowid) FROM chart"
1443 +
1444 +static bool check_chart_metadata(struct metadata_wc *wc)
1445 {
1446 static time_t next_execution_t = 0;
1447 static uint64_t last_row_id = 0;
1448 + static uint64_t max_row_id = 0;
1449 + static bool check_completed = false;
1450 +
1451 + if (check_completed)
1452 + return true;
1453
1454 time_t now = now_realtime_sec();
1455
1453 - if (!next_execution_t)
1456 + if (!next_execution_t) {
1457 next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
1458 + max_row_id = get_rowid_from_statement(SQL_GET_MAX_CHART_ROW_ID);
1459 + nd_log(NDLS_DAEMON, NDLP_INFO, "Chart metadata check has been scheduled to run (max id = %lu)", max_row_id);
1460 + }
1461
1462 if (next_execution_t && next_execution_t > now)
1457 - return;
1463 + return true;
1464 +
1465 + if (max_row_id && last_row_id >= max_row_id) {
1466 + nd_log(NDLS_DAEMON, NDLP_INFO, "Chart metadata check completed");
1467 + check_completed = true;
1468 + return true;
1469 + }
1470
1471 sqlite3_stmt *res = NULL;
1472
1473 if (!PREPARE_STATEMENT(db_meta, SELECT_CHART_LIST, &res))
1462 - return;
1474 + return true;
1475
1476 uint32_t total_checked = 0;
1477 uint32_t total_deleted = 0;
1478
1479 nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts starting after row %" PRIu64, last_row_id);
1480
1481 + worker_is_busy(UV_EVENT_CHART_CLEANUP);
1482 sqlite3_stmt *check_res = NULL;
1483 sqlite3_stmt *action_res = NULL;
1471 - bool more_to_do = run_cleanup_loop(
1484 + (void)run_cleanup_loop(
1485 res,
1486 wc,
1487 chart_can_be_deleted,
@@ -1485,42 +1498,52 @@ static void check_chart_metadata(struct metadata_wc *wc)
1498 SQLITE_FINALIZE(action_res);
1499
1500 now = now_realtime_sec();
1488 - if (more_to_do)
1489 - next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
1490 - else {
1491 - last_row_id = 0;
1492 - next_execution_t = now + METADATA_CHART_CHECK_INTERVAL;
1493 - }
1494 -
1495 - nd_log(
1496 - NDLS_DAEMON,
1501 + next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
1502 + nd_log_daemon(
1503 NDLP_DEBUG,
1498 - "Charts checked %u, deleted %u. Checks will %s in %lld seconds",
1504 + "Charts checked %u, deleted %u. Checks will resume in %d seconds",
1505 total_checked,
1506 total_deleted,
1501 - last_row_id ? "resume" : "restart",
1502 - (long long)(next_execution_t - now));
1507 + METADATA_MAINTENANCE_REPEAT);
1508
1509 SQLITE_FINALIZE(res);
1510 + worker_is_idle();
1511 + return false;
1512 }
1513
1507 -static void check_label_metadata(struct metadata_wc *wc)
1514 +#define SQL_GET_MAX_CHART_LABEL_ROW_ID "SELECT MAX(rowid) FROM chart_label"
1515 +
1516 +static bool check_label_metadata(struct metadata_wc *wc)
1517 {
1518 static time_t next_execution_t = 0;
1519 static uint64_t last_row_id = 0;
1520 + static uint64_t max_row_id = 0;
1521 + static bool check_completed = false;
1522 +
1523 + if (check_completed)
1524 + return true;
1525
1526 time_t now = now_realtime_sec();
1527
1514 - if (!next_execution_t)
1528 + if (!next_execution_t) {
1529 next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
1530 + max_row_id = get_rowid_from_statement(SQL_GET_MAX_CHART_LABEL_ROW_ID);
1531 + nd_log(NDLS_DAEMON, NDLP_INFO, "Chart label metadata check has been scheduled to run (max id = %lu)", max_row_id);
1532 + }
1533
1534 if (next_execution_t && next_execution_t > now)
1518 - return;
1535 + return true;
1536 +
1537 + if (max_row_id && last_row_id >= max_row_id) {
1538 + nd_log(NDLS_DAEMON, NDLP_INFO, "Chart label metadata check completed");
1539 + check_completed = true;
1540 + return true;
1541 + }
1542
1543 sqlite3_stmt *res = NULL;
1544
1545 if (!PREPARE_STATEMENT(db_meta, SELECT_CHART_LABEL_LIST, &res))
1523 - return;
1546 + return true;
1547
1548 uint32_t total_checked = 0;
1549 uint32_t total_deleted = 0;
@@ -1532,7 +1555,7 @@ static void check_label_metadata(struct metadata_wc *wc)
1555
1556 worker_is_busy(UV_EVENT_CHART_LABEL_CLEANUP);
1557
1535 - bool more_to_do = run_cleanup_loop(
1558 + (void )run_cleanup_loop(
1559 res,
1560 wc,
1561 chart_can_be_deleted,
@@ -1549,28 +1572,21 @@ static void check_label_metadata(struct metadata_wc *wc)
1572 SQLITE_FINALIZE(action_res);
1573
1574 now = now_realtime_sec();
1552 - if (more_to_do)
1553 - next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
1554 - else {
1555 - last_row_id = 0;
1556 - next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL;
1557 - }
1575 + next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL;
1576
1559 - nd_log(
1560 - NDLS_DAEMON,
1577 + nd_log_daemon(
1578 NDLP_DEBUG,
1562 - "Chart labels checked %u, deleted %u. Checks will %s in %lld seconds",
1579 + "Chart labels checked %u, deleted %u. Checks will resume in %d seconds",
1580 total_checked,
1581 total_deleted,
1565 - last_row_id ? "resume" : "restart",
1566 - (long long)(next_execution_t - now));
1582 + METADATA_LABEL_CHECK_INTERVAL);
1583
1584 SQLITE_FINALIZE(res);
1585
1586 worker_is_idle();
1587 + return false;
1588 }
1589
1573 -
1590 static void cleanup_health_log(struct metadata_wc *wc)
1591 {
1592 static time_t next_execution_t = 0;
@@ -1586,22 +1602,25 @@ static void cleanup_health_log(struct metadata_wc *wc)
1602 next_execution_t = now + METADATA_HEALTH_LOG_INTERVAL;
1603
1604 RRDHOST *host;
1605 + worker_is_busy(UV_EVENT_HEALTH_LOG_CLEANUP);
1606
1590 - dfe_start_reentrant(rrdhost_root_index, host){
1591 - if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
1592 - continue;
1607 + dfe_start_reentrant(rrdhost_root_index, host)
1608 + {
1609 sql_health_alarm_log_cleanup(host);
1610 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
1611 break;
1612 }
1613 dfe_done(host);
1614
1599 - if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
1615 + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
1616 + worker_is_idle();
1617 return;
1618 + }
1619
1620 (void) db_execute(db_meta,"DELETE FROM health_log WHERE host_id NOT IN (SELECT host_id FROM host)");
1621 (void) db_execute(db_meta,"DELETE FROM health_log_detail WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)");
1622 (void) db_execute(db_meta,"DELETE FROM alert_version WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)");
1623 + worker_is_idle();
1624 }
1625
1626 //
@@ -1689,26 +1708,32 @@ static void timer_cb(uv_timer_t* handle)
1708
1709 void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc)
1710 {
1692 - int free_pages = get_free_page_count(database);
1693 - int total_pages = get_database_page_count(database);
1711 + static time_t next_run = 0;
1712
1695 - if (!threshold)
1696 - threshold = DATABASE_FREE_PAGES_THRESHOLD_PC;
1713 + time_t now = now_realtime_sec();
1714 + if (next_run > now)
1715 + return;
1716
1698 - if (!vacuum_pc)
1699 - vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC;
1717 + next_run = now + DATABASE_VACUUM_FREQUENCY_SECONDS;
1718
1701 - if (free_pages > (total_pages * threshold / 100)) {
1719 + int free_pages = get_free_page_count(database);
1720 + int total_pages = get_database_page_count(database);
1721
1703 - int do_free_pages = (int) (free_pages * vacuum_pc / 100);
1704 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "%s: Freeing %d database pages", db_alias, do_free_pages);
1722 + if (!threshold)
1723 + threshold = DATABASE_FREE_PAGES_THRESHOLD_PC;
1724
1706 - char sql[128];
1707 - snprintfz(sql, sizeof(sql) - 1, "PRAGMA incremental_vacuum(%d)", do_free_pages);
1708 - (void) db_execute(database, sql);
1709 - }
1710 -}
1725 + if (!vacuum_pc)
1726 + vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC;
1727 +
1728 + if (free_pages > (total_pages * threshold / 100)) {
1729 + int do_free_pages = (int)(free_pages * vacuum_pc / 100);
1730 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "%s: Freeing %d database pages", db_alias, do_free_pages);
1731
1732 + char sql[128];
1733 + snprintfz(sql, sizeof(sql) - 1, "PRAGMA incremental_vacuum(%d)", do_free_pages);
1734 + (void)db_execute(database, sql);
1735 + }
1736 +}
1737
1738 #define SQL_SELECT_HOST_CTX_CHART_DIM_LIST \
1739 "SELECT d.dim_id, d.rowid FROM chart c, dimension d WHERE c.chart_id = d.chart_id AND c.rowid = @rowid"
@@ -1841,9 +1866,10 @@ void run_metadata_cleanup(struct metadata_wc *wc)
1866 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
1867 return;
1868
1844 - check_dimension_metadata(wc);
1845 - check_chart_metadata(wc);
1846 - check_label_metadata(wc);
1869 + if (check_dimension_metadata(wc))
1870 + if (check_chart_metadata(wc))
1871 + check_label_metadata(wc);
1872 +
1873 cleanup_health_log(wc);
1874
1875 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
@@ -1857,12 +1883,10 @@ void run_metadata_cleanup(struct metadata_wc *wc)
1883 struct scan_metadata_payload {
1884 uv_work_t request;
1885 struct metadata_wc *wc;
1860 - void *chart_label_cleanup;
1886 void *pending_alert_list;
1887 void *pending_ctx_cleanup_list;
1888 void *pending_uuid_deletion;
1889 BUFFER *work_buffer;
1865 - uint32_t max_count;
1890 };
1891
1892 struct host_context_load_thread {
@@ -2091,67 +2115,83 @@ static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)
2115 freez(data);
2116 }
2117
2094 -static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_transaction, BUFFER *work_buffer, size_t *query_counter) {
2118 +static void metadata_scan_host(RRDHOST *host, BUFFER *work_buffer, size_t *query_counter, bool shutting_down)
2119 +{
2120 + static bool skip_models = false;
2121 RRDSET *st;
2122 int rc;
2123
2098 - bool more_to_do = false;
2099 - uint32_t scan_count = 1;
2100 -
2124 sqlite3_stmt *ml_load_stmt = NULL;
2125 + sqlite3_stmt *store_dimension = NULL;
2126 + sqlite3_stmt *store_chart = NULL;
2127
2103 - bool load_ml_models = max_count;
2104 -
2105 - if (use_transaction)
2106 - (void)db_execute(db_meta, "BEGIN TRANSACTION");
2128 + bool host_need_recheck = false;
2129 + (void)db_execute(db_meta, "BEGIN TRANSACTION");
2130
2131 rrdset_foreach_reentrant(st, host) {
2109 - if (scan_count == max_count) {
2110 - more_to_do = true;
2111 - break;
2112 - }
2132 +
2133 if(rrdset_flag_check(st, RRDSET_FLAG_METADATA_UPDATE)) {
2134 (*query_counter)++;
2135
2136 rrdset_flag_clear(st, RRDSET_FLAG_METADATA_UPDATE);
2117 - scan_count++;
2137
2138 buffer_flush(work_buffer);
2139 +
2140 + worker_is_busy(UV_EVENT_STORE_CHART);
2141 rc = check_and_update_chart_labels(st, work_buffer, query_counter);
2142 if (unlikely(rc))
2143 error_report("METADATA: 'host:%s': Failed to update labels for chart %s", rrdhost_hostname(host), rrdset_name(st));
2144 else
2145 (*query_counter)++;
2146
2126 - rc = store_chart_metadata(st);
2127 - if (unlikely(rc))
2128 - error_report("METADATA: 'host:%s': Failed to store metadata for chart %s", rrdhost_hostname(host), rrdset_name(st));
2147 + rc = store_chart_metadata(st, &store_chart);
2148 + if (unlikely(rc)) {
2149 + host_need_recheck = true;
2150 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
2151 + error_report(
2152 + "METADATA: 'host:%s': Failed to store metadata for chart %s",
2153 + rrdhost_hostname(host),
2154 + rrdset_name(st));
2155 + }
2156 + else
2157 + (*query_counter)++;
2158 + worker_is_idle();
2159 }
2160
2161 RRDDIM *rd;
2162 rrddim_foreach_read(rd, st) {
2133 - if(rrddim_flag_check(rd, RRDDIM_FLAG_METADATA_UPDATE)) {
2134 - (*query_counter)++;
2163
2136 - rrddim_flag_clear(rd, RRDDIM_FLAG_METADATA_UPDATE);
2164 + if (rrddim_flag_check(rd, RRDDIM_FLAG_ML_MODEL_LOAD)) {
2165 + rrddim_flag_clear(rd, RRDDIM_FLAG_ML_MODEL_LOAD);
2166 + if (likely(!skip_models && !shutting_down)) {
2167 + worker_is_busy(UV_EVENT_METADATA_ML_LOAD);
2168 + skip_models = ml_dimension_load_models(rd, &ml_load_stmt);
2169 + worker_is_idle();
2170 + }
2171 + }
2172
2138 - if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
2139 - rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
2140 - else
2141 - rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
2173 + if(likely(!rrddim_flag_check(rd, RRDDIM_FLAG_METADATA_UPDATE)))
2174 + continue;
2175
2143 - rc = store_dimension_metadata(rd);
2144 - if (unlikely(rc))
2145 - error_report("METADATA: 'host:%s': Failed to dimension metadata for chart %s. dimension %s",
2146 - rrdhost_hostname(host), rrdset_name(st),
2147 - rrddim_name(rd));
2148 - }
2176 + rrddim_flag_clear(rd, RRDDIM_FLAG_METADATA_UPDATE);
2177
2150 - if(rrddim_flag_check(rd, RRDDIM_FLAG_ML_MODEL_LOAD)) {
2151 - rrddim_flag_clear(rd, RRDDIM_FLAG_ML_MODEL_LOAD);
2152 - if (likely(load_ml_models))
2153 - (void) ml_dimension_load_models(rd, &ml_load_stmt);
2154 - }
2178 + if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
2179 + rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
2180 + else
2181 + rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
2182 +
2183 + worker_is_busy(UV_EVENT_STORE_DIMENSION);
2184 + rc = store_dimension_metadata(rd, &store_dimension);
2185 + if (unlikely(rc)) {
2186 + host_need_recheck = true;
2187 + rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE);
2188 + error_report(
2189 + "METADATA: 'host:%s': Failed to store dimension metadata for chart %s. dimension %s",
2190 + rrdhost_hostname(host),
2191 + rrdset_name(st),
2192 + rrddim_name(rd));
2193 + } else
2194 + (*query_counter)++;
2195
2196 worker_is_idle();
2197 }
@@ -2159,17 +2199,24 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_trans
2199 }
2200 rrdset_foreach_done(st);
2201
2162 - if (use_transaction)
2163 - (void)db_execute(db_meta, "COMMIT TRANSACTION");
2202 + (void)db_execute(db_meta, "COMMIT TRANSACTION");
2203 + if (host_need_recheck)
2204 + rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE);
2205
2206 SQLITE_FINALIZE(ml_load_stmt);
2166 - ml_load_stmt = NULL;
2207 + SQLITE_FINALIZE(store_dimension);
2208
2168 - return more_to_do;
2209 + return;
2210 }
2211
2212 +
2213 static void store_host_and_system_info(RRDHOST *host, size_t *query_counter)
2214 {
2215 + if (!rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_INFO))
2216 + return;
2217 +
2218 + rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_INFO);
2219 +
2220 if (unlikely(store_host_systeminfo(host))) {
2221 error_report("METADATA: 'host:%s': Failed to store host updated system information in the database", rrdhost_hostname(host));
2222 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
@@ -2206,15 +2253,15 @@ static void do_pending_uuid_deletion(struct metadata_wc *wc, struct judy_list_t
2253 size_t entries = pending_uuid_deletion->count;
2254 Word_t Index = 0;
2255 bool first = true;
2209 - Pvoid_t *PValue;
2210 - while ((PValue = JudyLFirstThenNext(pending_uuid_deletion->JudyL, &Index, &first))) {
2211 - if (!*PValue)
2256 + Pvoid_t *Pvalue;
2257 + while ((Pvalue = JudyLFirstThenNext(pending_uuid_deletion->JudyL, &Index, &first))) {
2258 + if (!*Pvalue)
2259 continue;
2260
2261 if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
2262 break;
2263
2217 - nd_uuid_t *uuid = *PValue;
2264 + nd_uuid_t *uuid = *Pvalue;
2265 if (dimension_can_be_deleted(uuid, NULL, false))
2266 delete_dimension_uuid(uuid, NULL, false);
2267
@@ -2245,16 +2292,16 @@ static void store_ctx_cleanup_list(struct metadata_wc *wc, struct judy_list_t *p
2292 size_t entries = pending_ctx_cleanup_list->count;
2293 Word_t Index = 0;
2294 bool first = true;
2248 - Pvoid_t *PValue;
2295 + Pvoid_t *Pvalue;
2296 sqlite3_stmt *res = NULL;
2250 - while ((PValue = JudyLFirstThenNext(pending_ctx_cleanup_list->JudyL, &Index, &first))) {
2251 - if (!*PValue)
2297 + while ((Pvalue = JudyLFirstThenNext(pending_ctx_cleanup_list->JudyL, &Index, &first))) {
2298 + if (!*Pvalue)
2299 continue;
2300
2301 if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
2302 break;
2303
2257 - struct host_ctx_cleanup_s *ctx_cleanup = *PValue;
2304 + struct host_ctx_cleanup_s *ctx_cleanup = *Pvalue;
2305 sql_schedule_host_ctx_cleanup(&res, &ctx_cleanup->host_uuid, string2str(ctx_cleanup->context));
2306 string_freez(ctx_cleanup->context);
2307 freez(ctx_cleanup);
@@ -2285,12 +2332,12 @@ static void store_alert_transitions(struct judy_list_t *pending_alert_list)
2332 size_t entries = pending_alert_list->count;
2333 Word_t Index = 0;
2334 bool first = true;
2288 - Pvoid_t *PValue;
2289 - while ((PValue = JudyLFirstThenNext(pending_alert_list->JudyL, &Index, &first))) {
2290 - RRDHOST *host = *PValue;
2335 + Pvoid_t *Pvalue;
2336 + while ((Pvalue = JudyLFirstThenNext(pending_alert_list->JudyL, &Index, &first))) {
2337 + RRDHOST *host = *Pvalue;
2338
2292 - PValue = JudyLGet(pending_alert_list->JudyL, ++Index, PJE0);
2293 - ALARM_ENTRY *ae = *PValue;
2339 + Pvalue = JudyLGet(pending_alert_list->JudyL, ++Index, PJE0);
2340 + ALARM_ENTRY *ae = *Pvalue;
2341
2342 sql_health_alarm_log_save(host, ae);
2343
@@ -2311,149 +2358,121 @@ static void store_alert_transitions(struct judy_list_t *pending_alert_list)
2358 worker_is_idle();
2359 }
2360
2314 -static void do_chart_label_cleanup(struct metadata_wc *wc, struct judy_list_t *cl_cleanup_data)
2361 +static void meta_store_host_labels(RRDHOST *host, BUFFER *work_buffer, size_t *query_counter)
2362 {
2316 - if (!cl_cleanup_data)
2363 + if (likely(!rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_LABELS)))
2364 return;
2365
2319 - worker_is_busy(UV_EVENT_CHART_LABEL_CLEANUP);
2366 + rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_LABELS);
2367
2321 - Word_t Index = 0;
2322 - bool first = true;
2323 - Pvoid_t *PValue;
2324 - while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) {
2325 - char *machine_guid = *PValue;
2368 + int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_id.uuid);
2369 + if (unlikely(rc)) {
2370 + error_report("METADATA: 'host:%s': failed to delete old host labels", rrdhost_hostname(host));
2371 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
2372 + return;
2373 + }
2374
2327 - if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
2328 - break;
2375 + (*query_counter)++;
2376 + buffer_flush(work_buffer);
2377
2330 - RRDHOST *host = rrdhost_find_by_guid(machine_guid);
2331 - if (likely(!host)) {
2332 - nd_uuid_t host_uuid;
2333 - if (!uuid_parse(machine_guid, host_uuid))
2334 - delete_host_chart_labels(&host_uuid);
2335 - }
2378 + struct query_build tmp = {.sql = work_buffer, .count = 0};
2379 + uuid_unparse_lower(host->host_id.uuid, tmp.uuid_str);
2380 + rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
2381 + buffer_strcat(
2382 + work_buffer,
2383 + " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
2384 + rc = db_execute(db_meta, buffer_tostring(work_buffer));
2385 +
2386 + if (unlikely(rc)) {
2387 + error_report("METADATA: 'host:%s': failed to update metadata host labels", rrdhost_hostname(host));
2388 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
2389 + } else
2390 + (*query_counter)++;
2391 +}
2392
2337 - freez(machine_guid);
2338 - }
2339 - JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0);
2340 - freez(cl_cleanup_data);
2393 +static void store_host_claim_id(RRDHOST *host, size_t *query_counter)
2394 +{
2395 + if (likely(!rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_CLAIMID)))
2396 + return;
2397
2342 - worker_is_idle();
2398 + rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_CLAIMID);
2399 + int rc;
2400 + ND_UUID uuid = claim_id_get_uuid();
2401 + if (!UUIDiszero(uuid))
2402 + rc = store_claim_id(&host->host_id.uuid, &uuid.uuid);
2403 + else
2404 + rc = store_claim_id(&host->host_id.uuid, NULL);
2405 +
2406 + if (unlikely(rc))
2407 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID | RRDHOST_FLAG_METADATA_UPDATE);
2408 + else
2409 + (*query_counter)++;
2410 }
2411
2412 +#define COMPUTE_DURATION(var_name, unit, start, end) \
2413 + char var_name[64]; \
2414 + duration_snprintf(var_name, sizeof(var_name), \
2415 + (int64_t)((end) - (start)), unit, true)
2416 +
2417 // Worker thread to scan hosts for pending metadata to store
2418 static void start_metadata_hosts(uv_work_t *req)
2419 {
2420 register_libuv_worker_jobs();
2421
2350 - RRDHOST *host;
2351 - int transaction_started = 0;
2352 -
2422 struct scan_metadata_payload *data = req->data;
2423 struct metadata_wc *wc = data->wc;
2424
2425 + bool shutting_down = (!wc->scan_complete);
2426 +
2427 BUFFER *work_buffer = data->work_buffer;
2357 - usec_t all_started_ut = now_monotonic_usec(); (void)all_started_ut;
2358 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking all hosts started");
2359 - usec_t started_ut = now_monotonic_usec(); (void)started_ut;
2428 + usec_t all_started_ut = now_monotonic_usec();
2429
2430 store_alert_transitions((struct judy_list_t *)data->pending_alert_list);
2431 store_ctx_cleanup_list(wc, (struct judy_list_t *)data->pending_ctx_cleanup_list);
2363 - do_chart_label_cleanup(wc, (struct judy_list_t *)data->chart_label_cleanup);
2432
2365 - bool run_again = false;
2433 worker_is_busy(UV_EVENT_METADATA_STORE);
2434
2368 - if (!data->max_count)
2369 - transaction_started = !db_execute(db_meta, "BEGIN TRANSACTION");
2370 -
2435 + RRDHOST *host;
2436 dfe_start_reentrant(rrdhost_root_index, host) {
2437
2438 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE))
2439 continue;
2440
2376 - size_t query_counter = 0; (void)query_counter;
2441 + usec_t started_ut = now_monotonic_usec();
2442 + size_t query_counter = 0;
2443
2444 rrdhost_flag_clear(host,RRDHOST_FLAG_METADATA_UPDATE);
2445
2380 - if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_LABELS))) {
2381 - rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_LABELS);
2382 -
2383 - int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_id.uuid);
2384 - if (likely(!rc)) {
2385 - query_counter++;
2446 + worker_is_busy(UV_EVENT_STORE_HOST);
2447 + // Store labels (if needed)
2448 + meta_store_host_labels(host, work_buffer, &query_counter);
2449
2387 - buffer_flush(work_buffer);
2388 - struct query_build tmp = {.sql = work_buffer, .count = 0};
2389 - uuid_unparse_lower(host->host_id.uuid, tmp.uuid_str);
2390 - rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
2391 - buffer_strcat(work_buffer, " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
2392 - rc = db_execute(db_meta, buffer_tostring(work_buffer));
2450 + // Store claim id (if needed)
2451 + store_host_claim_id(host, &query_counter);
2452
2394 - if (unlikely(rc)) {
2395 - error_report("METADATA: 'host:%s': failed to update metadata host labels", rrdhost_hostname(host));
2396 - rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
2397 - }
2398 - else
2399 - query_counter++;
2400 - } else {
2401 - error_report("METADATA: 'host:%s': failed to delete old host labels", rrdhost_hostname(host));
2402 - rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
2403 - }
2404 - }
2405 -
2406 - if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_CLAIMID))) {
2407 - rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_CLAIMID);
2408 - int rc;
2409 - ND_UUID uuid = claim_id_get_uuid();
2410 - if(!UUIDiszero(uuid))
2411 - rc = store_claim_id(&host->host_id.uuid, &uuid.uuid);
2412 - else
2413 - rc = store_claim_id(&host->host_id.uuid, NULL);
2453 + // Store host and system info (if needed);
2454 + store_host_and_system_info(host, &query_counter);
2455 + worker_is_idle();
2456
2415 - if (unlikely(rc))
2416 - rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID | RRDHOST_FLAG_METADATA_UPDATE);
2417 - else
2418 - query_counter++;
2419 - }
2420 - if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_INFO))) {
2421 - rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_INFO);
2422 - store_host_and_system_info(host, &query_counter);
2423 - }
2457 + metadata_scan_host(host, work_buffer, &query_counter, shutting_down);
2458
2425 - // For clarity
2426 - bool use_transaction = data->max_count;
2427 - if (unlikely(metadata_scan_host(host, data->max_count, use_transaction, work_buffer, &query_counter))) {
2428 - run_again = true;
2429 - rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE);
2430 - }
2431 - usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
2432 - nd_log(
2433 - NDLS_DAEMON,
2459 + COMPUTE_DURATION(report_duration, "us", started_ut, now_monotonic_usec());
2460 + nd_log_daemon(
2461 NDLP_DEBUG,
2435 - "Host %s saved metadata with %zu SQL statements, in %0.2f ms",
2462 + "Host %s saved metadata with %zu SQL statements, in %s",
2463 rrdhost_hostname(host),
2464 query_counter,
2438 - (double)(ended_ut - started_ut) / USEC_PER_MS);
2465 + report_duration);
2466 }
2467 dfe_done(host);
2468
2442 - if (!data->max_count && transaction_started)
2443 - transaction_started = db_execute(db_meta, "COMMIT TRANSACTION");
2469
2445 - usec_t all_ended_ut = now_monotonic_usec(); (void)all_ended_ut;
2446 - nd_log(
2447 - NDLS_DAEMON,
2448 - NDLP_DEBUG,
2449 - "Checking all hosts completed in %0.2f ms",
2450 - (double)(all_ended_ut - all_started_ut) / USEC_PER_MS);
2470 + COMPUTE_DURATION(report_duration, "us", all_started_ut, now_monotonic_usec());
2471 + nd_log_daemon(NDLP_DEBUG, "Checking all hosts completed in %s", report_duration);
2472
2473 do_pending_uuid_deletion(wc, (struct judy_list_t *)data->pending_uuid_deletion);
2474
2454 - if (likely(!run_again)) {
2455 - run_metadata_cleanup(wc);
2456 - }
2475 + run_metadata_cleanup(wc);
2476
2477 wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL;
2478 worker_is_idle();
@@ -2468,7 +2487,6 @@ static void metadata_event_loop(void *arg)
2487 worker_register_job_name(METADATA_ADD_CTX_CLEANUP, "host ctx cleanup");
2488 worker_register_job_name(METADATA_SCAN_HOSTS, "host metadata store");
2489 worker_register_job_name(METADATA_LOAD_HOST_CONTEXT, "host load context");
2471 - worker_register_job_name(METADATA_DELETE_HOST_CHART_LABELS, "delete host labels");
2490 worker_register_job_name(METADATA_ADD_HOST_AE, "add host alert entry");
2491 worker_register_job_name(METADATA_DEL_HOST_AE, "delete host alert entry");
2492
@@ -2516,8 +2534,7 @@ static void metadata_event_loop(void *arg)
2534 completion_mark_complete(&wc->start_stop_complete);
2535 BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
2536 struct scan_metadata_payload *data;
2519 - struct judy_list_t *cl_cleanup_data = NULL;
2520 - Pvoid_t *PValue;
2537 + Pvoid_t *Pvalue;
2538 struct judy_list_t *pending_ae_list = NULL;
2539 struct judy_list_t *pending_ctx_cleanup_list = NULL;
2540 struct judy_list_t *pending_uuid_deletion = NULL;
@@ -2557,9 +2574,14 @@ static void metadata_event_loop(void *arg)
2574 if (!pending_uuid_deletion)
2575 pending_uuid_deletion = callocz(1, sizeof(*pending_uuid_deletion));
2576
2560 - PValue = JudyLIns(&pending_uuid_deletion->JudyL, ++pending_uuid_deletion->count, PJE0);
2561 - if (PValue)
2562 - *PValue = uuid;
2577 + Pvalue = JudyLIns(&pending_uuid_deletion->JudyL, ++pending_uuid_deletion->count, PJE0);
2578 + if (Pvalue != PJERR)
2579 + *Pvalue = uuid;
2580 + else {
2581 + // Failure in Judy, attempt to continue running anyway
2582 + // ignore uuid, global cleanup will take care of it
2583 + freez(uuid);
2584 + }
2585 break;
2586 case METADATA_STORE_CLAIM_ID:
2587 store_claim_id((nd_uuid_t *) cmd.param[0], (nd_uuid_t *) cmd.param[1]);
@@ -2571,9 +2593,16 @@ static void metadata_event_loop(void *arg)
2593 if (!pending_ctx_cleanup_list)
2594 pending_ctx_cleanup_list = callocz(1, sizeof(*pending_ctx_cleanup_list));
2595
2574 - PValue = JudyLIns(&pending_ctx_cleanup_list->JudyL, ++pending_ctx_cleanup_list->count, PJE0);
2575 - if (PValue)
2576 - *PValue = (void *)cmd.param[0];
2596 + struct host_ctx_cleanup_s *ctx_cleanup = (struct host_ctx_cleanup_s *) cmd.param[0];
2597 + Pvalue = JudyLIns(&pending_ctx_cleanup_list->JudyL, ++pending_ctx_cleanup_list->count, PJE0);
2598 + if (Pvalue != PJERR)
2599 + *Pvalue = ctx_cleanup;
2600 + else {
2601 + // Failure in Judy, attempt to continue running anyway
2602 + // Cleanup structure
2603 + string_freez(ctx_cleanup->context);
2604 + freez(ctx_cleanup);
2605 + }
2606 break;
2607 case METADATA_SCAN_HOSTS:
2608 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_PROCESSING)))
@@ -2585,29 +2614,22 @@ static void metadata_event_loop(void *arg)
2614 data = mallocz(sizeof(*data));
2615 data->request.data = data;
2616 data->wc = wc;
2588 - data->chart_label_cleanup = cl_cleanup_data;
2617 data->pending_alert_list = pending_ae_list;
2618 data->pending_ctx_cleanup_list = pending_ctx_cleanup_list;
2619 data->pending_uuid_deletion = pending_uuid_deletion;
2620
2621 data->work_buffer = work_buffer;
2594 - cl_cleanup_data = NULL;
2622 pending_ae_list = NULL;
2623 pending_ctx_cleanup_list = NULL;
2624 pending_uuid_deletion = NULL;
2625
2599 - if (unlikely(cmd.completion)) {
2600 - data->max_count = 0; // 0 will process all pending updates
2626 + if (unlikely(cmd.completion))
2627 cmd.completion = NULL; // Do not complete after launching worker (worker will do)
2602 - }
2603 - else
2604 - data->max_count = 5000;
2628
2629 metadata_flag_set(wc, METADATA_FLAG_PROCESSING);
2630 if (uv_queue_work(loop, &data->request, start_metadata_hosts, after_metadata_hosts)) {
2631 // Failed to launch worker -- let the event loop handle completion
2632 cmd.completion = wc->scan_complete;
2610 - cl_cleanup_data = data->chart_label_cleanup;
2633 pending_ae_list = data->pending_alert_list;
2634 pending_ctx_cleanup_list = data->pending_ctx_cleanup_list;
2635 pending_uuid_deletion = data->pending_uuid_deletion;
@@ -2625,15 +2647,6 @@ static void metadata_event_loop(void *arg)
2647 if (uv_queue_work(loop, &data->request, start_all_host_load_context, after_start_host_load_context)) {
2648 freez(data);
2649 }
2628 - break;
2629 - case METADATA_DELETE_HOST_CHART_LABELS:;
2630 - if (!cl_cleanup_data)
2631 - cl_cleanup_data = callocz(1,sizeof(*cl_cleanup_data));
2632 -
2633 - PValue = JudyLIns(&cl_cleanup_data->JudyL, (Word_t) ++cl_cleanup_data->count, PJE0);
2634 - if (PValue)
2635 - *PValue = (void *) cmd.param[0];
2636 -
2650 break;
2651 case METADATA_ADD_HOST_AE:
2652 host = (RRDHOST *) cmd.param[0];
@@ -2642,13 +2655,13 @@ static void metadata_event_loop(void *arg)
2655 if (!pending_ae_list)
2656 pending_ae_list = callocz(1, sizeof(*pending_ae_list));
2657
2645 - PValue = JudyLIns(&pending_ae_list->JudyL, ++pending_ae_list->count, PJE0);
2646 - if (PValue)
2647 - *PValue = (void *)host;
2658 + Pvalue = JudyLIns(&pending_ae_list->JudyL, ++pending_ae_list->count, PJE0);
2659 + if (Pvalue)
2660 + *Pvalue = (void *)host;
2661
2649 - PValue = JudyLIns(&pending_ae_list->JudyL, ++pending_ae_list->count, PJE0);
2650 - if (PValue)
2651 - *PValue = (void *)ae;
2662 + Pvalue = JudyLIns(&pending_ae_list->JudyL, ++pending_ae_list->count, PJE0);
2663 + if (Pvalue)
2664 + *Pvalue = (void *)ae;
2665 break;
2666 case METADATA_DEL_HOST_AE:
2667 (void) JudyLIns(&wc->ae_DelJudyL, (Word_t) (void *) cmd.param[0], PJE0);
@@ -2690,17 +2703,6 @@ static void metadata_event_loop(void *arg)
2703 Word_t Index;
2704 bool first;
2705
2693 - if (cl_cleanup_data) {
2694 - Index = 0;
2695 - first = true;
2696 - while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) {
2697 - char *machine_guid = *PValue;
2698 - freez(machine_guid);
2699 - }
2700 - JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0);
2701 - freez(cl_cleanup_data);
2702 - }
2703 -
2706 if (pending_ae_list) {
2707 (void)JudyLFreeArray(&pending_ae_list->JudyL, PJE0);
2708 freez(pending_ae_list);
@@ -2709,10 +2711,10 @@ static void metadata_event_loop(void *arg)
2711 if (pending_ctx_cleanup_list) {
2712 Index = 0;
2713 first = true;
2712 - while ((PValue = JudyLFirstThenNext(pending_ctx_cleanup_list->JudyL, &Index, &first))) {
2713 - if (!*PValue)
2714 + while ((Pvalue = JudyLFirstThenNext(pending_ctx_cleanup_list->JudyL, &Index, &first))) {
2715 + if (!*Pvalue)
2716 continue;
2715 - struct host_ctx_cleanup_s *ctx_cleanup = *PValue;
2717 + struct host_ctx_cleanup_s *ctx_cleanup = *Pvalue;
2718 string_freez(ctx_cleanup->context);
2719 freez(ctx_cleanup);
2720 }
@@ -2869,18 +2871,6 @@ void metadata_queue_load_host_context(RRDHOST *host)
2871 nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command to load host contexts");
2872 }
2873
2872 -void metadata_delete_host_chart_labels(char *machine_guid)
2873 -{
2874 - if (unlikely(!metasync_worker.loop)) {
2875 - freez(machine_guid);
2876 - return;
2877 - }
2878 -
2879 - // Node machine guid is already strdup-ed
2880 - queue_metadata_cmd(METADATA_DELETE_HOST_CHART_LABELS, machine_guid, NULL);
2881 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command delete chart labels for host %s", machine_guid);
2882 -}
2883 -
2874 void metadata_queue_ctx_host_cleanup(nd_uuid_t *host_uuid, const char *context)
2875 {
2876 if (unlikely(!metasync_worker.loop))
src/database/sqlite/sqlite_metadata.h
-1
@@ -43,7 +43,6 @@ void metaqueue_store_claim_id(nd_uuid_t *host_uuid, nd_uuid_t *claim_uuid);
43 void metaqueue_ml_load_models(RRDDIM *rd);
44 void detect_machine_guid_change(nd_uuid_t *host_uuid);
45 void metadata_queue_load_host_context(RRDHOST *host);
46 -void metadata_delete_host_chart_labels(char *machine_guid);
46 void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc);
47
48 int sql_metadata_cache_stats(int op);