@cryptotaxi247 / netdata-1 / commits / 21d3e3abd

Improve context load time during startup (#19321)

* Improve context load time during startup * Remove cache for instance acquired --------- Co-authored-by: Costa Tsaousis <costa@netdata.cloud>

Stelios Fragkakis committed Jan 10, 2025 at 11:48 UTC 21d3e3abd206ba51a0bd3954d8503db763a804e3

2 files changed +145 -80

src/database/sqlite/sqlite_context.c

+13 -10

@@ -3,6 +3,7 @@

3 #include "sqlite_functions.h"

4 #include "sqlite_context.h"

5 #include "sqlite_db_migration.h"

6 +#include "database/contexts/internal.h"

8 #define DB_CONTEXT_METADATA_VERSION 1

@@ -72,6 +73,8 @@ int sql_init_context_database(int memory)

73 return 0;

74 }

76 +extern __thread sqlite3 *db_meta_thread;

77 +extern __thread sqlite3 *db_context_thread;

78 //

79 // Fetching data

80 //

@@ -80,14 +83,14 @@ int sql_init_context_database(int memory)

84 void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data)

85 {

83 - static __thread sqlite3_stmt *res = NULL;

86 + sqlite3_stmt *res = NULL;

88 if (unlikely(!host_uuid)) {

89 internal_error(true, "Requesting context chart list without host_id");

90 return;

91 }

90 - if (!PREPARE_COMPILED_STATEMENT(db_meta, CTX_GET_CHART_LIST, &res))

93 + if (!PREPARE_STATEMENT(db_meta_thread ? db_meta_thread : db_meta, CTX_GET_CHART_LIST, &res))

94 return;

96 int param = 0;

@@ -111,7 +114,7 @@ void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *,

114

115 done:

116 REPORT_BIND_FAIL(res, param);

114 - SQLITE_RESET(res);

117 + SQLITE_FINALIZE(res);

118 }

119

120 // Dimension list

@@ -119,9 +122,9 @@ done:

122 "FROM dimension d, chart c WHERE c.host_id = @host_id AND d.chart_id = c.chart_id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC"

123 void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)

124 {

122 - static __thread sqlite3_stmt *res = NULL;

125 + sqlite3_stmt *res = NULL;

126

124 - if (!PREPARE_COMPILED_STATEMENT(db_meta, CTX_GET_DIMENSION_LIST, &res))

127 + if (!PREPARE_STATEMENT(db_meta_thread ? db_meta_thread : db_meta, CTX_GET_DIMENSION_LIST, &res))

128 return;

129

130 int param = 0;

@@ -142,7 +145,7 @@ void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_

145

146 done:

147 REPORT_BIND_FAIL(res, param);

145 - SQLITE_RESET(res);

148 + SQLITE_FINALIZE(res);

149 }

150

151 // LABEL LIST

@@ -183,9 +186,9 @@ void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEX

186 if (unlikely(!host_uuid))

187 return;

188

186 - static __thread sqlite3_stmt *res = NULL;

189 + sqlite3_stmt *res = NULL;

190

188 - if (!PREPARE_COMPILED_STATEMENT(db_context_meta, CTX_GET_CONTEXT_LIST, &res))

191 + if (!PREPARE_STATEMENT(db_context_thread ? db_context_thread : db_context_meta, CTX_GET_CONTEXT_LIST, &res))

192 return;

193

194 VERSIONED_CONTEXT_DATA context_data = {0};

@@ -210,7 +213,7 @@ void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEX

213

214 done:

215 REPORT_BIND_FAIL(res, param);

213 - SQLITE_RESET(res);

216 + SQLITE_FINALIZE(res);

217 }

218

219

@@ -230,7 +233,7 @@ int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data

233 if (unlikely(!host_uuid || !context_data || !context_data->id))

234 return 0;

235

233 - if (!PREPARE_STATEMENT(db_context_meta, CTX_STORE_CONTEXT, &res))

236 + if (!PREPARE_STATEMENT(db_context_meta ? db_context_meta : db_meta, CTX_STORE_CONTEXT, &res))

237 return 1;

238

239 int param = 0;

src/database/sqlite/sqlite_metadata.c

+132 -70

@@ -1571,29 +1571,66 @@ struct scan_metadata_payload {

1571 struct host_context_load_thread {

1572 uv_thread_t thread;

1573 RRDHOST *host;

1574 + sqlite3 *db_meta_thread;

1575 + sqlite3 *db_context_thread;

1576 bool busy;

1577 bool finished;

1578 };

1579

1580 +__thread sqlite3 *db_meta_thread = NULL;

1581 +__thread sqlite3 *db_context_thread = NULL;

1582 +__thread bool main_context_thread = false;

1583 +

1584 static void restore_host_context(void *arg)

1585 {

1586 struct host_context_load_thread *hclt = arg;

1587 RRDHOST *host = hclt->host;

1588

1589 + if (!host)

1590 + return;

1591 +

1592 + if (!db_meta_thread) {

1593 + if (hclt->db_meta_thread) {

1594 + db_meta_thread = hclt->db_meta_thread;

1595 + db_context_thread = hclt->db_context_thread;

1596 + } else {

1597 + char sqlite_database[FILENAME_MAX + 1];

1598 + snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/netdata-meta.db", netdata_configured_cache_dir);

1599 + int rc = sqlite3_open_v2(sqlite_database, &db_meta_thread, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, NULL);

1600 + if (rc != SQLITE_OK) {

1601 + sqlite3_close(db_meta_thread);

1602 + db_meta_thread = NULL;

1603 + }

1604 +

1605 + snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/context-meta.db", netdata_configured_cache_dir);

1606 + rc = sqlite3_open_v2(sqlite_database, &db_context_thread, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, NULL);

1607 + if (rc != SQLITE_OK) {

1608 + sqlite3_close(db_context_thread);

1609 + db_context_thread = NULL;

1610 + }

1611 +

1612 + hclt->db_meta_thread = db_meta_thread;

1613 + hclt->db_context_thread = db_context_thread;

1614 + }

1615 + }

1616 +

1617 usec_t started_ut = now_monotonic_usec(); (void)started_ut;

1618 rrdhost_load_rrdcontext_data(host);

1619 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

1620

1621 + char load_duration[64];

1622 + duration_snprintf(load_duration, sizeof(load_duration), (int64_t)(ended_ut - started_ut), "us", true);

1623 + nd_log_daemon(NDLP_DEBUG, "Contexts for host %s loaded in %s", rrdhost_hostname(host), load_duration);

1624 +

1625 rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);

1626

1627 aclk_queue_node_info(host, false);

1628

1591 - nd_log(

1592 - NDLS_DAEMON,

1593 - NDLP_DEBUG,

1594 - "Contexts for host %s loaded in %0.2f ms",

1595 - rrdhost_hostname(host),

1596 - (double)(ended_ut - started_ut) / USEC_PER_MS);

1629 + // Check and clear the thread local variables

1630 + if (!main_context_thread) {

1631 + db_meta_thread = NULL;

1632 + db_context_thread = NULL;

1633 + }

1634

1635 __atomic_store_n(&hclt->finished, true, __ATOMIC_RELEASE);

1636 }

@@ -1605,40 +1642,41 @@ static void after_start_host_load_context(uv_work_t *req, int status __maybe_unu

1642 freez(data);

1643 }

1644

1608 -#define MAX_FIND_THREAD_RETRIES (10)

1609 -

1610 -static void cleanup_finished_threads(struct host_context_load_thread *hclt, size_t max_thread_slots, bool wait)

1645 +static bool cleanup_finished_threads(struct host_context_load_thread *hclt, size_t max_thread_slots, bool wait, size_t *free_slot)

1646 {

1647 if (!hclt)

1613 - return;

1648 + return false;

1649

1615 - for (size_t index = 0; index < max_thread_slots; index++) {

1616 - if (__atomic_load_n(&(hclt[index].finished), __ATOMIC_RELAXED)

1617 - || (wait && __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE))) {

1618 - int rc = uv_thread_join(&(hclt[index].thread));

1619 - if (rc)

1620 - nd_log(NDLS_DAEMON, NDLP_WARNING, "Failed to join thread, rc = %d", rc);

1621 - __atomic_store_n(&(hclt[index].busy), false, __ATOMIC_RELEASE);

1622 - __atomic_store_n(&(hclt[index].finished), false, __ATOMIC_RELEASE);

1623 - }

1624 - }

1625 -}

1650 + bool found_slot = false;

1651

1627 -static size_t find_available_thread_slot(struct host_context_load_thread *hclt, size_t max_thread_slots, size_t *found_index)

1628 -{

1629 - size_t retries = MAX_FIND_THREAD_RETRIES;

1630 - while (retries--) {

1631 - size_t index = 0;

1632 - while (index < max_thread_slots) {

1633 - if (false == __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE)) {

1634 - *found_index = index;

1635 - return true;

1636 - }

1637 - index++;

1638 - }

1639 - sleep_usec(10 * USEC_PER_MS);

1652 + size_t loop_count = 20;

1653 + while (loop_count--) {

1654 + for (size_t index = 0; index < max_thread_slots; index++) {

1655 + if (free_slot && false == __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE)) {

1656 + found_slot = true;

1657 + *free_slot = index;

1658 + break;

1659 + }

1660 + if (__atomic_load_n(&(hclt[index].finished), __ATOMIC_RELAXED) ||

1661 + (wait && __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE))) {

1662 +

1663 + int rc = uv_thread_join(&(hclt[index].thread));

1664 + if (rc)

1665 + nd_log_daemon(NDLP_WARNING, "Failed to join thread, rc = %d", rc);

1666 + __atomic_store_n(&(hclt[index].busy), false, __ATOMIC_RELEASE);

1667 + __atomic_store_n(&(hclt[index].finished), false, __ATOMIC_RELEASE);

1668 + found_slot = true;

1669 + if (free_slot) {

1670 + *free_slot = index;

1671 + break;

1672 + }

1673 + }

1674 + }

1675 + if (found_slot || wait)

1676 + break;

1677 + sleep_usec(10 * USEC_PER_MS);

1678 }

1641 - return false;

1679 + return found_slot || wait;

1680 }

1681

1682 static void start_all_host_load_context(uv_work_t *req __maybe_unused)

@@ -1653,7 +1691,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)

1691

1692 RRDHOST *host;

1693

1656 - size_t max_threads = netdata_conf_cpus() / 2;

1694 + size_t max_threads = netdata_conf_cpus();

1695 if (max_threads < 1)

1696 max_threads = 1;

1697

@@ -1661,45 +1699,69 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)

1699 struct host_context_load_thread *hclt = max_threads > 1 ? callocz(max_threads, sizeof(*hclt)) : NULL;

1700

1701 size_t thread_index = 0;

1702 + main_context_thread = true;

1703 + size_t host_count = 0;

1704 + size_t sync_exec = 0;

1705 + size_t async_exec = 0;

1706 dfe_start_reentrant(rrdhost_root_index, host) {

1665 - if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))

1666 - continue;

1667 -

1668 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host));

1669 -

1670 - int rc = 0;

1671 - if (hclt) {

1672 - bool found_slot = false;

1673 - do {

1674 - if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1675 - break;

1676 -

1677 - cleanup_finished_threads(hclt, max_threads, false);

1678 - found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);

1679 - } while (!found_slot);

1680 -

1681 - if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1682 - break;

1683 -

1684 - __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);

1685 - hclt[thread_index].host = host;

1686 - rc = uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]);

1687 - }

1688 - // if single thread or thread creation failed

1689 - if (rc || !hclt) {

1690 - struct host_context_load_thread hclt_sync = {.host = host};

1691 - restore_host_context(&hclt_sync);

1692 -

1693 - if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1694 - break;

1695 - }

1707 + if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))

1708 + continue;

1709 +

1710 + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))

1711 + break;

1712 +

1713 + nd_log_daemon(NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host));

1714 +

1715 + int rc = 0;

1716 + bool thread_found = cleanup_finished_threads(hclt, max_threads, false, &thread_index);

1717 + if (thread_found) {

1718 + __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);

1719 + hclt[thread_index].host = host;

1720 + rc = uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]);

1721 + async_exec += (rc == 0);

1722 + }

1723 + // if single thread, thread creation failure or failure to find slot

1724 + if (rc || !thread_found) {

1725 + sync_exec++;

1726 + struct host_context_load_thread hclt_sync = {.host = host};

1727 + restore_host_context(&hclt_sync);

1728 + }

1729 + host_count++;

1730 }

1731 dfe_done(host);

1732

1699 - cleanup_finished_threads(hclt, max_threads, true);

1700 - freez(hclt);

1733 + bool should_clean_threads = cleanup_finished_threads(hclt, max_threads, true, NULL);

1734 +

1735 + if (should_clean_threads) {

1736 + for (size_t index = 0; index < max_threads; index++) {

1737 + if (hclt[index].db_meta_thread)

1738 + sqlite3_close_v2(hclt[index].db_meta_thread);

1739 +

1740 + if (hclt[index].db_context_thread)

1741 + sqlite3_close_v2(hclt[index].db_context_thread);

1742 + }

1743 + freez(hclt);

1744 + }

1745 +

1746 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;

1702 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);

1747 + char load_duration[64];

1748 + duration_snprintf(load_duration, sizeof(load_duration), (int64_t)(ended_ut - started_ut), "us", true);

1749 +

1750 + nd_log_daemon(

1751 + NDLP_INFO,

1752 + "Contexts for %zu hosts loaded: %zu delegated to %zu threads, %zu handled directly, in %s.",

1753 + host_count,

1754 + async_exec,

1755 + max_threads,

1756 + sync_exec,

1757 + load_duration);

1758 +

1759 + if (db_meta_thread) {

1760 + sqlite3_close_v2(db_meta_thread);

1761 + sqlite3_close_v2(db_context_thread);

1762 + db_meta_thread = NULL;

1763 + db_context_thread = NULL;

1764 + }

1765

1766 worker_is_idle();

1767 }