@cryptotaxi247 / netdata-1 / commits / 10721be4e

Improve context load (#16659)

* Improve single thread load. Handle thread creation failure as well Remove RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS Improve chart label cleanup * Init thread index

Stelios Fragkakis committed Jan 15, 2024 at 18:25 UTC 10721be4ec3feaf72ff45856a7c447fe95a4b435

3 files changed +102 -71

database/rrd.h

+5 -6

@@ -975,19 +975,18 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {

975

976 // ACLK

977 RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 21), // when set, we should send ACLK stream context updates

978 - RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // set when the receiver part is disconnected

978 + RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // Host should stream alerts

979

980 // Metadata

981 RRDHOST_FLAG_METADATA_UPDATE = (1 << 23), // metadata needs to be stored in the database

982 RRDHOST_FLAG_METADATA_LABELS = (1 << 24), // metadata needs to be stored in the database

983 RRDHOST_FLAG_METADATA_INFO = (1 << 25), // metadata needs to be stored in the database

984 - RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 26), // metadata needs to be stored in the database

985 - RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS = (1 << 27), // metadata needs to be stored in the database

984 + RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 26), // Context needs to be loaded

985

987 - RRDHOST_FLAG_METADATA_CLAIMID = (1 << 28), // metadata needs to be stored in the database

988 - RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED = (1 << 29), // set when the receiver part is disconnected

986 + RRDHOST_FLAG_METADATA_CLAIMID = (1 << 27), // metadata needs to be stored in the database

987 + RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED = (1 << 28), // set when the receiver part is disconnected

988

990 - RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 30), // set when the host has updated global functions

989 + RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 29), // set when the host has updated global functions

990 } RRDHOST_FLAGS;

991

992 #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))

database/rrdhost.c

+1 -1

@@ -1739,7 +1739,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {

1739 s->db.instances = host->rrdctx.instances;

1740 s->db.contexts = dictionary_entries(host->rrdctx.contexts);

1741 if(!s->db.first_time_s || !s->db.last_time_s || !s->db.metrics || !s->db.instances || !s->db.contexts ||

1742 - (flags & (RRDHOST_FLAG_PENDING_CONTEXT_LOAD|RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS)))

1742 + (flags & (RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))

1743 s->db.status = RRDHOST_DB_STATUS_INITIALIZING;

1744 else

1745 s->db.status = RRDHOST_DB_STATUS_QUERYABLE;

database/sqlite/sqlite_metadata.c

+96 -64

@@ -873,7 +873,7 @@ static void check_dimension_metadata(struct metadata_wc *wc)

873 uint32_t total_checked = 0;

874 uint32_t total_deleted = 0;

875

876 - internal_error(true, "METADATA: Checking dimensions starting after row %"PRIu64, last_row_id);

876 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking dimensions starting after row %" PRIu64, last_row_id);

877

878 bool more_to_do = run_cleanup_loop(

879 res,

@@ -896,8 +896,10 @@ static void check_dimension_metadata(struct metadata_wc *wc)

896 next_execution_t = now + METADATA_DIM_CHECK_INTERVAL;

897 }

898

899 - internal_error(true,

900 - "METADATA: Dimensions checked %u, deleted %u. Checks will %s in %lld seconds",

899 + nd_log(

900 + NDLS_DAEMON,

901 + NDLP_DEBUG,

902 + "Dimensions checked %u, deleted %u. Checks will %s in %lld seconds",

903 total_checked,

904 total_deleted,

905 last_row_id ? "resume" : "restart",

@@ -932,7 +934,7 @@ static void check_chart_metadata(struct metadata_wc *wc)

934 uint32_t total_checked = 0;

935 uint32_t total_deleted = 0;

936

935 - internal_error(true, "METADATA: Checking charts starting after row %"PRIu64, last_row_id);

937 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts starting after row %" PRIu64, last_row_id);

938

939 sqlite3_stmt *check_res = NULL;

940 sqlite3_stmt *action_res = NULL;

@@ -963,8 +965,10 @@ static void check_chart_metadata(struct metadata_wc *wc)

965 next_execution_t = now + METADATA_CHART_CHECK_INTERVAL;

966 }

967

966 - internal_error(true,

967 - "METADATA: Charts checked %u, deleted %u. Checks will %s in %lld seconds",

968 + nd_log(

969 + NDLS_DAEMON,

970 + NDLP_DEBUG,

971 + "Charts checked %u, deleted %u. Checks will %s in %lld seconds",

972 total_checked,

973 total_deleted,

974 last_row_id ? "resume" : "restart",

@@ -1000,7 +1004,7 @@ static void check_label_metadata(struct metadata_wc *wc)

1004 uint32_t total_checked = 0;

1005 uint32_t total_deleted = 0;

1006

1003 - internal_error(true,"METADATA: Checking charts labels starting after row %"PRIu64, last_row_id);

1007 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts labels starting after row %" PRIu64, last_row_id);

1008

1009 sqlite3_stmt *check_res = NULL;

1010 sqlite3_stmt *action_res = NULL;

@@ -1032,8 +1036,10 @@ static void check_label_metadata(struct metadata_wc *wc)

1036 next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL;

1037 }

1038

1035 - internal_error(true,

1036 - "METADATA: Chart labels checked %u, deleted %u. Checks will %s in %lld seconds",

1039 + nd_log(

1040 + NDLS_DAEMON,

1041 + NDLP_DEBUG,

1042 + "Chart labels checked %u, deleted %u. Checks will %s in %lld seconds",

1043 total_checked,

1044 total_deleted,

1045 last_row_id ? "resume" : "restart",

@@ -1223,13 +1229,17 @@ static void restore_host_context(void *arg)

1229 rrdhost_load_rrdcontext_data(host);

1230 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

1231

1226 - rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD | RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS);

1232 + rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);

1233

1234 #ifdef ENABLE_ACLK

1235 aclk_queue_node_info(host, false);

1236 #endif

1237

1232 - internal_error(true, "METADATA: 'host:%s' context load in %0.2f ms", rrdhost_hostname(host),

1238 + nd_log(

1239 + NDLS_DAEMON,

1240 + NDLP_DEBUG,

1241 + "Contexts for host %s loaded in %0.2f ms",

1242 + rrdhost_hostname(host),

1243 (double)(ended_ut - started_ut) / USEC_PER_MS);

1244

1245 __atomic_store_n(&hclt->finished, true, __ATOMIC_RELEASE);

@@ -1246,12 +1256,15 @@ static void after_start_host_load_context(uv_work_t *req, int status __maybe_unu

1256

1257 static void cleanup_finished_threads(struct host_context_load_thread *hclt, size_t max_thread_slots, bool wait)

1258 {

1259 + if (!hclt)

1260 + return;

1261 +

1262 for (size_t index = 0; index < max_thread_slots; index++) {

1263 if (__atomic_load_n(&(hclt[index].finished), __ATOMIC_RELAXED)

1264 || (wait && __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE))) {

1265 int rc = uv_thread_join(&(hclt[index].thread));

1266 if (rc)

1254 - netdata_log_error("Failed to join thread, rc = %d",rc);

1267 + nd_log(NDLS_DAEMON, NDLP_WARNING, "Failed to join thread, rc = %d", rc);

1268 __atomic_store_n(&(hclt[index].busy), false, __ATOMIC_RELEASE);

1269 __atomic_store_n(&(hclt[index].finished), false, __ATOMIC_RELEASE);

1270 }

@@ -1290,40 +1303,50 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)

1303 size_t max_threads = MIN(get_netdata_cpus() / 2, 6);

1304 if (max_threads < 1)

1305 max_threads = 1;

1293 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Using %zu threads for context loading", max_threads);

1294 - struct host_context_load_thread *hclt = callocz(max_threads, sizeof(*hclt));

1306

1296 - size_t thread_index;

1307 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Using %zu threads for context loading", max_threads);

1308 + struct host_context_load_thread *hclt = max_threads > 1 ? callocz(max_threads, sizeof(*hclt)) : NULL;

1309 +

1310 + size_t thread_index = 0;

1311 dfe_start_reentrant(rrdhost_root_index, host) {

1298 - if (rrdhost_flag_check(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS) ||

1299 - !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))

1312 + if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))

1313 continue;

1314

1302 - rrdhost_flag_set(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS);

1303 - internal_error(true, "METADATA: 'host:%s' loading context", rrdhost_hostname(host));

1315 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host));

1316

1305 - bool found_slot = false;

1306 - do {

1307 - if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1308 - break;

1317 + int rc = 0;

1318 + if (hclt) {

1319 + bool found_slot = false;

1320 + do {

1321 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1322 + break;

1323 +

1324 + cleanup_finished_threads(hclt, max_threads, false);

1325 + found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);

1326 + } while (!found_slot);

1327

1310 - cleanup_finished_threads(hclt, max_threads, false);

1311 - found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);

1312 - } while (!found_slot);

1328 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1329 + break;

1330

1314 - if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1315 - break;

1331 + __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);

1332 + hclt[thread_index].host = host;

1333 + rc = uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]);

1334 + }

1335 + // if single thread or thread creation failed

1336 + if (rc || !hclt) {

1337 + struct host_context_load_thread hclt_sync = {.host = host};

1338 + restore_host_context(&hclt_sync);

1339

1317 - __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);

1318 - hclt[thread_index].host = host;

1319 - fatal_assert(0 == uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]));

1340 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1341 + break;

1342 + }

1343 }

1344 dfe_done(host);

1345

1346 cleanup_finished_threads(hclt, max_threads, true);

1347 freez(hclt);

1348 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

1326 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);

1349 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);

1350

1351 worker_is_idle();

1352 }

@@ -1335,11 +1358,10 @@ static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)

1358 struct metadata_wc *wc = data->wc;

1359

1360 metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);

1338 - internal_error(true, "METADATA: scanning hosts complete");

1339 - if (unlikely(wc->scan_complete)) {

1361 +

1362 + if (unlikely(wc->scan_complete))

1363 completion_mark_complete(wc->scan_complete);

1341 - internal_error(true, "METADATA: Sending completion done");

1342 - }

1364 +

1365 freez(data);

1366 }

1367

@@ -1448,6 +1470,30 @@ struct host_chart_label_cleanup {

1470 Word_t count;

1471 };

1472

1473 +static void do_chart_label_cleanup(struct host_chart_label_cleanup *cl_cleanup_data)

1474 +{

1475 + if (!cl_cleanup_data)

1476 + return;

1477 +

1478 + Word_t Index = 0;

1479 + bool first = true;

1480 + Pvoid_t *PValue;

1481 + while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) {

1482 + char *machine_guid = *PValue;

1483 +

1484 + RRDHOST *host = rrdhost_find_by_guid(machine_guid);

1485 + if (likely(!host)) {

1486 + uuid_t host_uuid;

1487 + if (!uuid_parse(machine_guid, host_uuid))

1488 + delete_host_chart_labels(&host_uuid);

1489 + }

1490 +

1491 + freez(machine_guid);

1492 + }

1493 + JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0);

1494 + freez(cl_cleanup_data);

1495 +}

1496 +

1497 // Worker thread to scan hosts for pending metadata to store

1498 static void start_metadata_hosts(uv_work_t *req __maybe_unused)

1499 {

@@ -1461,30 +1507,10 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)

1507

1508 BUFFER *work_buffer = data->work_buffer;

1509 usec_t all_started_ut = now_monotonic_usec(); (void)all_started_ut;

1464 - internal_error(true, "METADATA: checking all hosts...");

1510 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking all hosts started");

1511 usec_t started_ut = now_monotonic_usec(); (void)started_ut;

1512

1467 - struct host_chart_label_cleanup *cl_cleanup_data = data->data;

1468 -

1469 - if (cl_cleanup_data) {

1470 - Word_t Index = 0;

1471 - bool first = true;

1472 - Pvoid_t *PValue;

1473 - while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) {

1474 - char *machine_guid = *PValue;

1475 -

1476 - host = rrdhost_find_by_guid(machine_guid);

1477 - if (likely(!host)) {

1478 - uuid_t host_uuid;

1479 - if (!uuid_parse(machine_guid, host_uuid))

1480 - delete_host_chart_labels(&host_uuid);

1481 - }

1482 -

1483 - freez(machine_guid);

1484 - }

1485 - JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0);

1486 - freez(cl_cleanup_data);

1487 - }

1513 + do_chart_label_cleanup((struct host_chart_label_cleanup *) data->data);

1514

1515 bool run_again = false;

1516 worker_is_busy(UV_EVENT_METADATA_STORE);

@@ -1550,12 +1576,15 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)

1576 if (unlikely(metadata_scan_host(host, data->max_count, use_transaction, work_buffer, &query_counter))) {

1577 run_again = true;

1578 rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE);

1553 - internal_error(true,"METADATA: 'host:%s': scheduling another run, more charts to store", rrdhost_hostname(host));

1579 }

1580 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

1556 - internal_error(true, "METADATA: 'host:%s': saved metadata with %zu SQL statements, in %0.2f ms",

1557 - rrdhost_hostname(host), query_counter,

1558 - (double)(ended_ut - started_ut) / USEC_PER_MS);

1581 + nd_log(

1582 + NDLS_DAEMON,

1583 + NDLP_DEBUG,

1584 + "Host %s saved metadata with %zu SQL statements, in %0.2f ms",

1585 + rrdhost_hostname(host),

1586 + query_counter,

1587 + (double)(ended_ut - started_ut) / USEC_PER_MS);

1588 }

1589 dfe_done(host);

1590

@@ -1563,8 +1592,11 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)

1592 transaction_started = db_execute(db_meta, "COMMIT TRANSACTION");

1593

1594 usec_t all_ended_ut = now_monotonic_usec(); (void)all_ended_ut;

1566 - internal_error(true, "METADATA: checking all hosts completed in %0.2f ms",

1567 - (double)(all_ended_ut - all_started_ut) / USEC_PER_MS);

1595 + nd_log(

1596 + NDLS_DAEMON,

1597 + NDLP_DEBUG,

1598 + "Checking all hosts completed in %0.2f ms",

1599 + (double)(all_ended_ut - all_started_ut) / USEC_PER_MS);

1600

1601 if (unlikely(run_again))

1602 wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_IMMEDIATE;