@cryptotaxi247 / netdata-1 / commits / 38f1ed617

Cleanup metadata event loop (#19447)

* Remove METADATA_ADD_HOST_INFO -- will be done when storing host data Add workers Schedule dimension cleanup in a worker * Remove commented code

Stelios Fragkakis committed Jan 20, 2025 at 23:23 UTC 38f1ed617c76f68e9634ce4e94dfa891a751e356

5 files changed +112 -31

src/daemon/libuv_workers.c

@@ -55,6 +55,13 @@ void register_libuv_worker_jobs() {

55 worker_register_job_name(UV_EVENT_METADATA_STORE, "metadata store host");

56 worker_register_job_name(UV_EVENT_METADATA_CLEANUP, "metadata cleanup");

57 worker_register_job_name(UV_EVENT_METADATA_ML_LOAD, "metadata load ml models");

58 + worker_register_job_name(UV_EVENT_CTX_CLEANUP_SCHEDULE, "metadata ctx cleanup schedule");

59 + worker_register_job_name(UV_EVENT_CTX_CLEANUP, "metadata ctx cleanup");

60 + worker_register_job_name(UV_EVENT_STORE_ALERT_TRANSITIONS, "metadata store alert transitions");

61 + worker_register_job_name(UV_EVENT_CHART_LABEL_CLEANUP, "metadata chart label cleanup");

62 + worker_register_job_name(UV_EVENT_UUID_DELETION, "metadata dimension deletion");

63 + worker_register_job_name(UV_EVENT_DIMENSION_CLEANUP, "metadata dimension cleanup");

64 + worker_register_job_name(UV_EVENT_CHART_CLEANUP, "metadata chart cleanup");

66 // aclk_sync

67 worker_register_job_name(UV_EVENT_ACLK_NODE_INFO, "aclk host node info");

src/daemon/libuv_workers.h

@@ -47,6 +47,13 @@ enum event_loop_job {

47 UV_EVENT_METADATA_STORE,

48 UV_EVENT_METADATA_CLEANUP,

49 UV_EVENT_METADATA_ML_LOAD,

50 + UV_EVENT_CTX_CLEANUP_SCHEDULE,

51 + UV_EVENT_CTX_CLEANUP,

52 + UV_EVENT_STORE_ALERT_TRANSITIONS,

53 + UV_EVENT_CHART_LABEL_CLEANUP,

54 + UV_EVENT_UUID_DELETION,

55 + UV_EVENT_DIMENSION_CLEANUP,

56 + UV_EVENT_CHART_CLEANUP,

58 // aclk_sync

59 UV_EVENT_ACLK_NODE_INFO,

src/database/rrdhost.c

+1 -2

@@ -498,9 +498,8 @@ RRDHOST *rrdhost_create(

498 );

499

500 if(!archived) {

501 - metaqueue_host_update_info(host);

501 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);

502 rrdhost_load_rrdcontext_data(host);

503 -// rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);

503 ml_host_new(host);

504 } else

505 rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD | RRDHOST_FLAG_ARCHIVED | RRDHOST_FLAG_ORPHAN);

src/database/sqlite/sqlite_metadata.c

+97 -28

@@ -193,7 +193,6 @@ enum metadata_opcode {

193 METADATA_DATABASE_NOOP = 0,

194 METADATA_DEL_DIMENSION,

195 METADATA_STORE_CLAIM_ID,

196 - METADATA_ADD_HOST_INFO,

196 METADATA_SCAN_HOSTS,

197 METADATA_LOAD_HOST_CONTEXT,

198 METADATA_DELETE_HOST_CHART_LABELS,

@@ -1405,6 +1404,8 @@ static void check_dimension_metadata(struct metadata_wc *wc)

1404

1405 nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking dimensions starting after row %" PRIu64, last_row_id);

1406

1407 + worker_is_busy(UV_EVENT_DIMENSION_CLEANUP);

1408 +

1409 bool more_to_do = run_cleanup_loop(

1410 res,

1411 wc,

@@ -1436,6 +1437,8 @@ static void check_dimension_metadata(struct metadata_wc *wc)

1437 (long long)(next_execution_t - now));

1438

1439 SQLITE_FINALIZE(res);

1440 +

1441 + worker_is_idle();

1442 }

1443

1444 static void check_chart_metadata(struct metadata_wc *wc)

@@ -1525,6 +1528,8 @@ static void check_label_metadata(struct metadata_wc *wc)

1528 sqlite3_stmt *check_res = NULL;

1529 sqlite3_stmt *action_res = NULL;

1530

1531 + worker_is_busy(UV_EVENT_CHART_LABEL_CLEANUP);

1532 +

1533 bool more_to_do = run_cleanup_loop(

1534 res,

1535 wc,

@@ -1559,6 +1564,8 @@ static void check_label_metadata(struct metadata_wc *wc)

1564 (long long)(next_execution_t - now));

1565

1566 SQLITE_FINALIZE(res);

1567 +

1568 + worker_is_idle();

1569 }

1570

1571

@@ -1818,12 +1825,14 @@ void run_metadata_cleanup(struct metadata_wc *wc)

1825

1826 if (next_context_list_cleanup < now && sql_metadata_wal_size_acceptable()) {

1827 RRDHOST *host;

1828 + worker_is_busy(UV_EVENT_CTX_CLEANUP);

1829 dfe_start_reentrant(rrdhost_root_index, host) {

1830 ctx_get_context_list_to_cleanup(&host->host_id.uuid, cleanup_host_context_metadata, host);

1831 if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN) || false == sql_metadata_wal_size_acceptable())

1832 break;

1833 }

1834 dfe_done(host);

1835 + worker_is_idle();

1836 next_context_list_cleanup = now_realtime_sec() + METADATA_MAINTENANCE_CTX_CLEAN_REPEAT;

1837 }

1838

@@ -1849,6 +1858,7 @@ struct scan_metadata_payload {

1858 void *chart_label_cleanup;

1859 void *pending_alert_list;

1860 void *pending_ctx_cleanup_list;

1861 + void *pending_uuid_deletion;

1862 BUFFER *work_buffer;

1863 uint32_t max_count;

1864 };

@@ -2181,12 +2191,52 @@ struct judy_list_t {

2191 Word_t count;

2192 };

2193

2194 +static void do_pending_uuid_deletion(struct metadata_wc *wc, struct judy_list_t *pending_uuid_deletion)

2195 +{

2196 + if (!pending_uuid_deletion)

2197 + return;

2198 +

2199 + worker_is_busy(UV_EVENT_UUID_DELETION);

2200 +

2201 + usec_t started_ut = now_monotonic_usec(); (void)started_ut;

2202 +

2203 + size_t entries = pending_uuid_deletion->count;

2204 + Word_t Index = 0;

2205 + bool first = true;

2206 + Pvoid_t *PValue;

2207 + while ((PValue = JudyLFirstThenNext(pending_uuid_deletion->JudyL, &Index, &first))) {

2208 + if (!*PValue)

2209 + continue;

2210 +

2211 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

2212 + break;

2213 +

2214 + nd_uuid_t *uuid = *PValue;

2215 + if (dimension_can_be_deleted(uuid, NULL, false))

2216 + delete_dimension_uuid(uuid, NULL, false);

2217 +

2218 + freez(uuid);

2219 + }

2220 + (void) JudyLFreeArray(&pending_uuid_deletion->JudyL, PJE0);

2221 + freez(pending_uuid_deletion);

2222 +

2223 + usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

2224 + nd_log_daemon(

2225 + NDLP_DEBUG,

2226 + "Processed %zu dimension delete items in %0.2f ms",

2227 + entries,

2228 + (double)(ended_ut - started_ut) / USEC_PER_MS);

2229 +

2230 + worker_is_idle();

2231 +}

2232

2185 -static void store_ctx_cleanup_list(struct judy_list_t *pending_ctx_cleanup_list)

2233 +static void store_ctx_cleanup_list(struct metadata_wc *wc, struct judy_list_t *pending_ctx_cleanup_list)

2234 {

2235 if (!pending_ctx_cleanup_list)

2236 return;

2237

2238 + worker_is_busy(UV_EVENT_CTX_CLEANUP_SCHEDULE);

2239 +

2240 usec_t started_ut = now_monotonic_usec(); (void)started_ut;

2241

2242 size_t entries = pending_ctx_cleanup_list->count;

@@ -2198,6 +2248,9 @@ static void store_ctx_cleanup_list(struct judy_list_t *pending_ctx_cleanup_list)

2248 if (!*PValue)

2249 continue;

2250

2251 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

2252 + break;

2253 +

2254 struct host_ctx_cleanup_s *ctx_cleanup = *PValue;

2255 sql_schedule_host_ctx_cleanup(&res, &ctx_cleanup->host_uuid, string2str(ctx_cleanup->context));

2256 string_freez(ctx_cleanup->context);

@@ -2213,6 +2266,8 @@ static void store_ctx_cleanup_list(struct judy_list_t *pending_ctx_cleanup_list)

2266 "Stored %zu host context cleanup items in %0.2f ms",

2267 entries,

2268 (double)(ended_ut - started_ut) / USEC_PER_MS);

2269 +

2270 + worker_is_idle();

2271 }

2272

2273 static void store_alert_transitions(struct judy_list_t *pending_alert_list)

@@ -2220,6 +2275,8 @@ static void store_alert_transitions(struct judy_list_t *pending_alert_list)

2275 if (!pending_alert_list)

2276 return;

2277

2278 + worker_is_busy(UV_EVENT_STORE_ALERT_TRANSITIONS);

2279 +

2280 usec_t started_ut = now_monotonic_usec(); (void)started_ut;

2281

2282 size_t entries = pending_alert_list->count;

@@ -2247,19 +2304,26 @@ static void store_alert_transitions(struct judy_list_t *pending_alert_list)

2304 "Stored and processed %zu alert transitions in %0.2f ms",

2305 entries,

2306 (double)(ended_ut - started_ut) / USEC_PER_MS);

2307 +

2308 + worker_is_idle();

2309 }

2310

2252 -static void do_chart_label_cleanup(struct judy_list_t *cl_cleanup_data)

2311 +static void do_chart_label_cleanup(struct metadata_wc *wc, struct judy_list_t *cl_cleanup_data)

2312 {

2313 if (!cl_cleanup_data)

2314 return;

2315

2316 + worker_is_busy(UV_EVENT_CHART_LABEL_CLEANUP);

2317 +

2318 Word_t Index = 0;

2319 bool first = true;

2320 Pvoid_t *PValue;

2321 while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) {

2322 char *machine_guid = *PValue;

2323

2324 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

2325 + break;

2326 +

2327 RRDHOST *host = rrdhost_find_by_guid(machine_guid);

2328 if (likely(!host)) {

2329 nd_uuid_t host_uuid;

@@ -2271,6 +2335,8 @@ static void do_chart_label_cleanup(struct judy_list_t *cl_cleanup_data)

2335 }

2336 JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0);

2337 freez(cl_cleanup_data);

2338 +

2339 + worker_is_idle();

2340 }

2341

2342 // Worker thread to scan hosts for pending metadata to store

@@ -2289,9 +2355,9 @@ static void start_metadata_hosts(uv_work_t *req)

2355 nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking all hosts started");

2356 usec_t started_ut = now_monotonic_usec(); (void)started_ut;

2357

2292 - store_ctx_cleanup_list((struct judy_list_t *)data->pending_ctx_cleanup_list);

2358 store_alert_transitions((struct judy_list_t *)data->pending_alert_list);

2294 - do_chart_label_cleanup((struct judy_list_t *)data->chart_label_cleanup);

2359 + store_ctx_cleanup_list(wc, (struct judy_list_t *)data->pending_ctx_cleanup_list);

2360 + do_chart_label_cleanup(wc, (struct judy_list_t *)data->chart_label_cleanup);

2361

2362 bool run_again = false;

2363 worker_is_busy(UV_EVENT_METADATA_STORE);

@@ -2380,8 +2446,11 @@ static void start_metadata_hosts(uv_work_t *req)

2446 "Checking all hosts completed in %0.2f ms",

2447 (double)(all_ended_ut - all_started_ut) / USEC_PER_MS);

2448

2383 - if (likely(!run_again))

2449 + do_pending_uuid_deletion(wc, (struct judy_list_t *)data->pending_uuid_deletion);

2450 +

2451 + if (likely(!run_again)) {

2452 run_metadata_cleanup(wc);

2453 + }

2454

2455 wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL;

2456 worker_is_idle();

@@ -2390,12 +2459,15 @@ static void start_metadata_hosts(uv_work_t *req)

2459 static void metadata_event_loop(void *arg)

2460 {

2461 worker_register("METASYNC");

2393 - worker_register_job_name(METADATA_DATABASE_NOOP, "noop");

2394 - worker_register_job_name(METADATA_DEL_DIMENSION, "delete dimension");

2395 - worker_register_job_name(METADATA_STORE_CLAIM_ID, "add claim id");

2396 - worker_register_job_name(METADATA_ADD_HOST_INFO, "add host info");

2397 - worker_register_job_name(METADATA_ADD_CTX_CLEANUP, "host ctx cleanup");

2398 - worker_register_job_name(METADATA_MAINTENANCE, "maintenance");

2462 + worker_register_job_name(METADATA_DATABASE_NOOP, "noop");

2463 + worker_register_job_name(METADATA_DEL_DIMENSION, "delete dimension");

2464 + worker_register_job_name(METADATA_STORE_CLAIM_ID, "add claim id");

2465 + worker_register_job_name(METADATA_ADD_CTX_CLEANUP, "host ctx cleanup");

2466 + worker_register_job_name(METADATA_SCAN_HOSTS, "host metadata store");

2467 + worker_register_job_name(METADATA_LOAD_HOST_CONTEXT, "host load context");

2468 + worker_register_job_name(METADATA_DELETE_HOST_CHART_LABELS, "delete host labels");

2469 + worker_register_job_name(METADATA_ADD_HOST_AE, "add host alert entry");

2470 + worker_register_job_name(METADATA_DEL_HOST_AE, "delete host alert entry");

2471

2472 int ret;

2473 uv_loop_t *loop;

@@ -2445,6 +2517,7 @@ static void metadata_event_loop(void *arg)

2517 Pvoid_t *PValue;

2518 struct judy_list_t *pending_ae_list = NULL;

2519 struct judy_list_t *pending_ctx_cleanup_list = NULL;

2520 + struct judy_list_t *pending_uuid_deletion = NULL;

2521

2522 while (shutdown == 0 || (wc->flags & METADATA_FLAG_PROCESSING)) {

2523 nd_uuid_t *uuid;

@@ -2478,19 +2551,18 @@ static void metadata_event_loop(void *arg)

2551 break;

2552 case METADATA_DEL_DIMENSION:

2553 uuid = (nd_uuid_t *) cmd.param[0];

2481 - if (likely(dimension_can_be_deleted(uuid, NULL, false)))

2482 - delete_dimension_uuid(uuid, NULL, false);

2483 - freez(uuid);

2554 + if (!pending_uuid_deletion)

2555 + pending_uuid_deletion = callocz(1, sizeof(*pending_uuid_deletion));

2556 +

2557 + PValue = JudyLIns(&pending_uuid_deletion->JudyL, ++pending_uuid_deletion->count, PJE0);

2558 + if (PValue)

2559 + *PValue = uuid;

2560 break;

2561 case METADATA_STORE_CLAIM_ID:

2562 store_claim_id((nd_uuid_t *) cmd.param[0], (nd_uuid_t *) cmd.param[1]);

2563 freez((void *) cmd.param[0]);

2564 freez((void *) cmd.param[1]);

2565 break;

2490 - case METADATA_ADD_HOST_INFO:

2491 - host = (RRDHOST *) cmd.param[0];

2492 - store_host_and_system_info(host, NULL);

2493 - break;

2566

2567 case METADATA_ADD_CTX_CLEANUP:

2568 if (!pending_ctx_cleanup_list)

@@ -2513,10 +2585,13 @@ static void metadata_event_loop(void *arg)

2585 data->chart_label_cleanup = cl_cleanup_data;

2586 data->pending_alert_list = pending_ae_list;

2587 data->pending_ctx_cleanup_list = pending_ctx_cleanup_list;

2588 + data->pending_uuid_deletion = pending_uuid_deletion;

2589 +

2590 data->work_buffer = work_buffer;

2591 cl_cleanup_data = NULL;

2592 pending_ae_list = NULL;

2593 pending_ctx_cleanup_list = NULL;

2594 + pending_uuid_deletion = NULL;

2595

2596 if (unlikely(cmd.completion)) {

2597 data->max_count = 0; // 0 will process all pending updates

@@ -2532,11 +2607,12 @@ static void metadata_event_loop(void *arg)

2607 cl_cleanup_data = data->chart_label_cleanup;

2608 pending_ae_list = data->pending_alert_list;

2609 pending_ctx_cleanup_list = data->pending_ctx_cleanup_list;

2610 + pending_uuid_deletion = data->pending_uuid_deletion;

2611 freez(data);

2612 metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);

2613 }

2614 break;

2539 - case METADATA_LOAD_HOST_CONTEXT:;

2615 + case METADATA_LOAD_HOST_CONTEXT:

2616 if (unittest_running)

2617 break;

2618

@@ -2571,7 +2647,7 @@ static void metadata_event_loop(void *arg)

2647 if (PValue)

2648 *PValue = (void *)ae;

2649 break;

2574 - case METADATA_DEL_HOST_AE:;

2650 + case METADATA_DEL_HOST_AE:

2651 (void) JudyLIns(&wc->ae_DelJudyL, (Word_t) (void *) cmd.param[0], PJE0);

2652 break;

2653 case METADATA_UNITTEST:;

@@ -2777,13 +2853,6 @@ void metaqueue_store_claim_id(nd_uuid_t *host_uuid, nd_uuid_t *claim_uuid)

2853 queue_metadata_cmd(METADATA_STORE_CLAIM_ID, local_host_uuid, local_claim_uuid);

2854 }

2855

2780 -void metaqueue_host_update_info(RRDHOST *host)

2781 -{

2782 - if (unlikely(!metasync_worker.loop))

2783 - return;

2784 - queue_metadata_cmd(METADATA_ADD_HOST_INFO, host, NULL);

2785 -}

2786 -

2856 void metaqueue_ml_load_models(RRDDIM *rd)

2857 {

2858 rrddim_flag_set(rd, RRDDIM_FLAG_ML_MODEL_LOAD);

src/database/sqlite/sqlite_metadata.h

-1

@@ -40,7 +40,6 @@ void metadata_sync_shutdown_prepare(void);

41 void metaqueue_delete_dimension_uuid(nd_uuid_t *uuid);

42 void metaqueue_store_claim_id(nd_uuid_t *host_uuid, nd_uuid_t *claim_uuid);

43 -void metaqueue_host_update_info(RRDHOST *host);

43 void metaqueue_ml_load_models(RRDDIM *rd);

44 void detect_machine_guid_change(nd_uuid_t *host_uuid);

45 void metadata_queue_load_host_context(RRDHOST *host);