@cryptotaxi247 / netdata-1 / commits / a27aed521

Improve context load on startup (#16203)

* Retrieve last connected timestamp from the database (host->last_connected) * Improve context load performance Check for agent shutdown while context load in progress Log information about host load start and finish * Remove check for slot as it will only reach this part when a slot is found

Stelios Fragkakis committed Oct 18, 2023 at 17:21 UTC a27aed521f6117fd157e2c599566a4812d9c2dc2

3 files changed +27 -19

database/sqlite/sqlite_aclk.c

+5 -2

@@ -84,6 +84,7 @@ enum {

84 IDX_PROGRAM_VERSION,

85 IDX_ENTRIES,

86 IDX_HEALTH_ENABLED,

87 + IDX_LAST_CONNECTED,

88 };

90 static int create_host_callback(void *data, int argc, char **argv, char **column)

@@ -127,8 +128,10 @@ static int create_host_callback(void *data, int argc, char **argv, char **column

128 , system_info

129 , 1

130 );

130 - if (likely(host))

131 + if (likely(host)) {

132 host->rrdlabels = sql_load_host_labels((uuid_t *)argv[IDX_HOST_ID]);

133 + host->last_connected = (time_t) (argv[IDX_LAST_CONNECTED] ? str2uint64_t(argv[IDX_LAST_CONNECTED], NULL) : 0);

134 + }

135

136 (*number_of_chidren)++;

137

@@ -524,7 +527,7 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may

527

528 #define SQL_FETCH_ALL_HOSTS "SELECT host_id, hostname, registry_hostname, update_every, os, " \

529 "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \

527 - "program_version, entries, health_enabled FROM host WHERE hops >0;"

530 + "program_version, entries, health_enabled, last_connected FROM host WHERE hops >0;"

531

532 #define SQL_FETCH_ALL_INSTANCES "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni " \

533 "WHERE h.host_id = ni.host_id AND ni.node_id IS NOT NULL; "

database/sqlite/sqlite_context.c

+4 -4

@@ -92,7 +92,7 @@ void sql_close_context_database(void)

92 // Fetching data

93 //

94 #define CTX_GET_CHART_LIST "SELECT c.chart_id, c.type||'.'||c.id, c.name, c.context, c.title, c.unit, c.priority, " \

95 - "c.update_every, c.chart_type, c.family FROM meta.chart c WHERE c.host_id = @host_id and c.chart_id is not null; "

95 + "c.update_every, c.chart_type, c.family FROM chart c WHERE c.host_id = @host_id and c.chart_id is not null; "

97 void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data)

98 {

@@ -105,7 +105,7 @@ void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, voi

105 }

106

107 if (unlikely(!res)) {

108 - rc = prepare_statement(db_context_meta, CTX_GET_CHART_LIST, &res);

108 + rc = prepare_statement(db_meta, CTX_GET_CHART_LIST, &res);

109 if (rc != SQLITE_OK) {

110 error_report("Failed to prepare statement to fetch chart list");

111 return;

@@ -141,14 +141,14 @@ skip_load:

141

142 // Dimension list

143 #define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \

144 - "FROM meta.dimension d WHERE d.chart_id = @id and d.dim_id is not null ORDER BY d.rowid ASC;"

144 + "FROM dimension d WHERE d.chart_id = @id and d.dim_id is not null ORDER BY d.rowid ASC;"

145 void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)

146 {

147 int rc;

148 static __thread sqlite3_stmt *res = NULL;

149

150 if (unlikely(!res)) {

151 - rc = prepare_statement(db_context_meta, CTX_GET_DIMENSION_LIST, &res);

151 + rc = prepare_statement(db_meta, CTX_GET_DIMENSION_LIST, &res);

152 if (rc != SQLITE_OK) {

153 error_report("Failed to prepare statement to fetch chart dimension data");

154 return;

database/sqlite/sqlite_metadata.c

+18 -13

@@ -1270,7 +1270,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)

1270 register_libuv_worker_jobs();

1271

1272 struct scan_metadata_payload *data = req->data;

1273 - UNUSED(data);

1273 + struct metadata_wc *wc = data->wc;

1274

1275 worker_is_busy(UV_EVENT_HOST_CONTEXT_LOAD);

1276 usec_t started_ut = now_monotonic_usec(); (void)started_ut;

@@ -1278,6 +1278,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)

1278 RRDHOST *host;

1279

1280 size_t max_threads = MIN(get_netdata_cpus() / 2, 6);

1281 + netdata_log_info("METADATA: Using %zu threads for context loading", max_threads);

1282 struct host_context_load_thread *hclt = callocz(max_threads, sizeof(*hclt));

1283

1284 size_t thread_index;

@@ -1289,25 +1290,28 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)

1290 rrdhost_flag_set(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS);

1291 internal_error(true, "METADATA: 'host:%s' loading context", rrdhost_hostname(host));

1292

1292 - cleanup_finished_threads(hclt, max_threads, false);

1293 - bool found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);

1293 + bool found_slot = false;

1294 + do {

1295 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1296 + break;

1297

1295 - if (unlikely(!found_slot)) {

1296 - struct host_context_load_thread hclt_sync = {.host = host};

1297 - restore_host_context(&hclt_sync);

1298 - }

1299 - else {

1300 - __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);

1301 - hclt[thread_index].host = host;

1302 - assert(0 == uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]));

1303 - }

1298 + cleanup_finished_threads(hclt, max_threads, false);

1299 + found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);

1300 + } while (!found_slot);

1301 +

1302 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1303 + break;

1304 +

1305 + __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);

1306 + hclt[thread_index].host = host;

1307 + assert(0 == uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]));

1308 }

1309 dfe_done(host);

1310

1311 cleanup_finished_threads(hclt, max_threads, true);

1312 freez(hclt);

1313 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

1310 - internal_error(true, "METADATA: 'host:ALL' contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);

1314 + netdata_log_info("METADATA: host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);

1315

1316 worker_is_idle();

1317 }

@@ -1905,6 +1909,7 @@ void metadata_queue_load_host_context(RRDHOST *host)

1909 if (unlikely(!metasync_worker.loop))

1910 return;

1911 queue_metadata_cmd(METADATA_LOAD_HOST_CONTEXT, host, NULL);

1912 + netdata_log_info("Queued command to load host contexts");

1913 }

1914

1915 //