@cryptotaxi247 / netdata-1 / commits / 440bd51e0

Refactor RRD code. (#15423)

* Storage engine. * Host indexes to rrdb * Move globals to rrdb * Move storage_tiers_backfill to rrdb * default_rrd_update_every to rrdb * default_rrd_history_entries to rrdb * gap_when_lost_iterations_above to rrdb * rrdset_free_obsolete_time_s to rrdb * libuv_worker_threads to rrdb * ieee754_doubles to rrdb * rrdhost_free_orphan_time_s to rrdb * rrd_rwlock to rrdb * localhost to rrdb * rm extern from func decls * mv rrd macro under rrd.h * default_rrdeng_page_cache_mb to rrdb * default_rrdeng_extent_cache_mb to rrdb * db_engine_journal_check to rrdb * default_rrdeng_disk_quota_mb to rrdb * default_multidb_disk_quota_mb to rrdb * multidb_ctx to rrdb * page_type_size to rrdb * tier_page_size to rrdb * No storage_engine_id in rrdim functions * storage_engine_id is provided by st * Update to fix merge conflict. * Update field name * Remove unnecessary macros from rrd.h * Rm unused type decls * Rm duplicate func decls * make internal function static * Make the rest of public dbengine funcs accept a storage_instance. * No more rrdengine_instance :) * rm rrdset_debug from rrd.h * Use rrdb to access globals in ML and ACLK Missed due to not having the submodules in the worktree. * rm total_number * rm RRDVAR_TYPE_TOTAL * rm unused inline * Rm names from typedef'd enums * rm unused header include * Move include * Rm unused header include * s/rrdhost_find_or_create/rrdhost_get_or_create/g * s/find_host_by_node_id/rrdhost_find_by_node_id/ Also, remove duplicate definition in rrdcontext.c * rm macro used only once * rm macro used only once * Reduce rrd.h api by moving funcs into a collector specific utils header * Remove unused func * Move parser specific function out of rrd.h * return storage_number instead of void pointer * move code related to rrd initialization out of rrdhost.c * Remove tier_grouping from rrdim_tier Saves 8 * storage_tiers bytes per dimension. * Fix rebase * s/rrd_update_every/update_every/ * Mark functions as static and constify args * Add license notes and file to build systems. * Remove remaining non-log/config mentions of memory mode * Move rrdlabels api to separate file. Also, move localhost functions that loads labels outside of database/ and into daemon/ * Remove function decl in rrd.h * merge rrdhost_cache_dir_for_rrdset_alloc into rrdset_cache_dir * Do not expose internal function from rrd.h * Rm NETDATA_RRD_INTERNALS Only one function decl is covered. We have more database internal functions that we currently expose for no good reason. These will be placed in a separate internal header in follow up PRs. * Add license note * Include libnetdata.h instead of aral.h * Use rrdb to access localhost * Fix builds without dbengine * Add header to build system files * Add rrdlabels.h to build systems * Move func def from rrd.h to rrdhost.c * Fix macos build * Rm non-existing function * Rebase master * Define buffer length macro in ad_charts. * Fix FreeBSD builds. * Mark functions static * Rm func decls without definitions * Rebase master * Rebase master * Properly initialize value of storage tiers. * Fix build after rebase.

vkalintiris committed Jul 26, 2023 at 15:30 UTC 440bd51e08fdfa2a4daa191fb68643456028a753
155 files changed +3063 -2918
CMakeLists.txt
+4
@@ -733,6 +733,7 @@ set(RRD_PLUGIN_FILES
733 database/rrdfunctions.h
734 database/rrdhost.c
735 database/rrdlabels.c
736 + database/rrdlabels.h
737 database/rrd.c
738 database/rrd.h
739 database/rrdset.c
@@ -742,6 +743,7 @@ set(RRD_PLUGIN_FILES
743 database/rrdvar.h
744 database/storage_engine.c
745 database/storage_engine.h
746 + database/storage_engine_types.h
747 database/ram/rrddim_mem.c
748 database/ram/rrddim_mem.h
749 database/sqlite/sqlite_metadata.c
@@ -773,6 +775,7 @@ set(RRD_PLUGIN_FILES
775 database/engine/rrdenginelib.h
776 database/engine/rrdengineapi.c
777 database/engine/rrdengineapi.h
778 + database/engine/rrddim_eng.h
779 database/engine/pagecache.c
780 database/engine/pagecache.h
781 database/engine/cache.c
@@ -1055,6 +1058,7 @@ ENDIF()
1058
1059 set(NETDATA_FILES
1060 collectors/all.h
1061 + collectors/utils.h
1062 ${DAEMON_FILES}
1063 ${API_PLUGIN_FILES}
1064 ${EXPORTING_ENGINE_FILES}
Makefile.am
+4
@@ -462,6 +462,7 @@ RRD_PLUGIN_FILES = \
462 database/rrdfamily.c \
463 database/rrdhost.c \
464 database/rrdlabels.c \
465 + database/rrdlabels.h \
466 database/rrd.c \
467 database/rrd.h \
468 database/rrdset.c \
@@ -473,6 +474,7 @@ RRD_PLUGIN_FILES = \
474 database/rrdvar.h \
475 database/storage_engine.c \
476 database/storage_engine.h \
477 + database/storage_engine_types.h \
478 database/ram/rrddim_mem.c \
479 database/ram/rrddim_mem.h \
480 database/sqlite/sqlite_functions.c \
@@ -545,6 +547,7 @@ libjudy_a-JudyLTables.$(OBJEXT) : CFLAGS += -I$(abs_top_srcdir)/libnetdata/libju
547
548 if ENABLE_DBENGINE
549 RRD_PLUGIN_FILES += \
550 + database/engine/rrddim_eng.h \
551 database/engine/rrdengine.c \
552 database/engine/rrdengine.h \
553 database/engine/rrddiskprotocol.h \
@@ -1070,6 +1073,7 @@ endif #ENABLE_H2O
1073
1074 NETDATA_FILES = \
1075 collectors/all.h \
1076 + collectors/utils.h \
1077 $(DAEMON_FILES) \
1078 $(LIBNETDATA_FILES) \
1079 $(API_PLUGIN_FILES) \
aclk/aclk.c
+18 -18
@@ -913,15 +913,15 @@ void aclk_host_state_update(RRDHOST *host, int cmd)
913 // node_id not found
914 aclk_query_t create_query;
915 create_query = aclk_query_new(REGISTER_NODE);
916 - rrdhost_aclk_state_lock(localhost);
916 + rrdhost_aclk_state_lock(rrdb.localhost);
917 node_instance_creation_t node_instance_creation = {
918 - .claim_id = localhost->aclk_state.claimed_id,
918 + .claim_id = rrdb.localhost->aclk_state.claimed_id,
919 .hops = host->system_info->hops,
920 .hostname = rrdhost_hostname(host),
921 .machine_guid = host->machine_guid};
922 create_query->data.bin_payload.payload =
923 generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation);
924 - rrdhost_aclk_state_unlock(localhost);
924 + rrdhost_aclk_state_unlock(rrdb.localhost);
925 create_query->data.bin_payload.topic = ACLK_TOPICID_CREATE_NODE;
926 create_query->data.bin_payload.msg_name = "CreateNodeInstance";
927 netdata_log_info("Registering host=%s, hops=%u", host->machine_guid, host->system_info->hops);
@@ -942,10 +942,10 @@ void aclk_host_state_update(RRDHOST *host, int cmd)
942
943 node_state_update.capabilities = aclk_get_agent_capas();
944
945 - rrdhost_aclk_state_lock(localhost);
946 - node_state_update.claim_id = localhost->aclk_state.claimed_id;
945 + rrdhost_aclk_state_lock(rrdb.localhost);
946 + node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id;
947 query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update);
948 - rrdhost_aclk_state_unlock(localhost);
948 + rrdhost_aclk_state_unlock(rrdb.localhost);
949
950 netdata_log_info("Queuing status update for node=%s, live=%d, hops=%u",(char*)node_state_update.node_id, cmd,
951 host->system_info->hops);
@@ -986,10 +986,10 @@ void aclk_send_node_instances()
986 }
987 node_state_update.capabilities = aclk_get_node_instance_capas(host);
988
989 - rrdhost_aclk_state_lock(localhost);
990 - node_state_update.claim_id = localhost->aclk_state.claimed_id;
989 + rrdhost_aclk_state_lock(rrdb.localhost);
990 + node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id;
991 query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update);
992 - rrdhost_aclk_state_unlock(localhost);
992 + rrdhost_aclk_state_unlock(rrdb.localhost);
993 netdata_log_info("Queuing status update for node=%s, live=%d, hops=%d",(char*)node_state_update.node_id,
994 list->live,
995 list->hops);
@@ -1010,10 +1010,10 @@ void aclk_send_node_instances()
1010 uuid_unparse_lower(list->host_id, (char*)node_instance_creation.machine_guid);
1011 create_query->data.bin_payload.topic = ACLK_TOPICID_CREATE_NODE;
1012 create_query->data.bin_payload.msg_name = "CreateNodeInstance";
1013 - rrdhost_aclk_state_lock(localhost);
1014 - node_instance_creation.claim_id = localhost->aclk_state.claimed_id,
1013 + rrdhost_aclk_state_lock(rrdb.localhost);
1014 + node_instance_creation.claim_id = rrdb.localhost->aclk_state.claimed_id,
1015 create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation);
1016 - rrdhost_aclk_state_unlock(localhost);
1016 + rrdhost_aclk_state_unlock(rrdb.localhost);
1017 netdata_log_info("Queuing registration for host=%s, hops=%d",(char*)node_instance_creation.machine_guid,
1018 list->hops);
1019 freez((void *)node_instance_creation.machine_guid);
@@ -1124,9 +1124,9 @@ char *aclk_state(void)
1124 buffer_sprintf(wb, "\n\tNode ID: %s\n", node_id);
1125 }
1126
1127 - buffer_sprintf(wb, "\tStreaming Hops: %d\n\tRelationship: %s", host->system_info->hops, host == localhost ? "self" : "child");
1127 + buffer_sprintf(wb, "\tStreaming Hops: %d\n\tRelationship: %s", host->system_info->hops, host == rrdb.localhost ? "self" : "child");
1128
1129 - if (host != localhost)
1129 + if (host != rrdb.localhost)
1130 buffer_sprintf(wb, "\n\tStreaming Connection Live: %s", host->receiver ? "true" : "false");
1131
1132 buffer_strcat(wb, "\n\tAlert Streaming Status:");
@@ -1270,10 +1270,10 @@ char *aclk_state_json(void)
1270 tmp = json_object_new_int(host->system_info->hops);
1271 json_object_object_add(nodeinstance, "streaming-hops", tmp);
1272
1273 - tmp = json_object_new_string(host == localhost ? "self" : "child");
1273 + tmp = json_object_new_string(host == rrdb.localhost ? "self" : "child");
1274 json_object_object_add(nodeinstance, "relationship", tmp);
1275
1276 - tmp = json_object_new_boolean((host->receiver || host == localhost));
1276 + tmp = json_object_new_boolean((host->receiver || host == rrdb.localhost));
1277 json_object_object_add(nodeinstance, "streaming-online", tmp);
1278
1279 tmp = json_object_new_object();
@@ -1292,7 +1292,7 @@ char *aclk_state_json(void)
1292 }
1293
1294 void add_aclk_host_labels(void) {
1295 - DICTIONARY *labels = localhost->rrdlabels;
1295 + DICTIONARY *labels = rrdb.localhost->rrdlabels;
1296
1297 #ifdef ENABLE_ACLK
1298 rrdlabels_add(labels, "_aclk_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
@@ -1324,5 +1324,5 @@ void aclk_queue_node_info(RRDHOST *host, bool immediate)
1324 {
1325 struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) host->aclk_sync_host_config;
1326 if (likely(wc))
1327 - wc->node_info_send_time = (host == localhost || immediate) ? 1 : now_realtime_sec();
1327 + wc->node_info_send_time = (host == rrdb.localhost || immediate) ? 1 : now_realtime_sec();
1328 }
aclk/aclk_capas.c
+3 -3
@@ -21,19 +21,19 @@ const struct capability *aclk_get_agent_capas()
21 { .name = NULL, .version = 0, .enabled = 0 }
22 };
23 agent_capabilities[2].version = ml_capable() ? 1 : 0;
24 - agent_capabilities[2].enabled = ml_enabled(localhost);
24 + agent_capabilities[2].enabled = ml_enabled(rrdb.localhost);
25
26 agent_capabilities[3].version = enable_metric_correlations ? metric_correlations_version : 0;
27 agent_capabilities[3].enabled = enable_metric_correlations;
28
29 - agent_capabilities[7].enabled = localhost->health.health_enabled;
29 + agent_capabilities[7].enabled = rrdb.localhost->health.health_enabled;
30
31 return agent_capabilities;
32 }
33
34 struct capability *aclk_get_node_instance_capas(RRDHOST *host)
35 {
36 - bool functions = (host == localhost || (host->receiver && stream_has_capability(host->receiver, STREAM_CAP_FUNCTIONS)));
36 + bool functions = (host == rrdb.localhost || (host->receiver && stream_has_capability(host->receiver, STREAM_CAP_FUNCTIONS)));
37
38 struct capability ni_caps[] = {
39 { .name = "proto", .version = 1, .enabled = 1 },
aclk/aclk_proxy.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "aclk_proxy.h"
4
5 #include "daemon/common.h"
aclk/aclk_proxy.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef ACLK_PROXY_H
4 #define ACLK_PROXY_H
5
aclk/aclk_query.c
+1 -1
@@ -128,7 +128,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
128 ACLK_STATS_UNLOCK;
129 }
130
131 - w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path);
131 + w->response.code = (short)web_client_api_request_with_node_selection(rrdb.localhost, w, path);
132 web_client_timeout_checkpoint_response_ready(w, &t);
133
134 if(buffer_strlen(w->response.data) > ACLK_MAX_WEB_RESPONSE_SIZE) {
aclk/aclk_rrdhost_state.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef ACLK_RRDHOST_STATE_H
4 #define ACLK_RRDHOST_STATE_H
5
aclk/aclk_rx_msgs.c
+4 -4
@@ -279,7 +279,7 @@ int create_node_instance_result(const char *msg, size_t msg_len)
279
280 RRDHOST *host = rrdhost_find_by_guid(res.machine_guid);
281 if (likely(host)) {
282 - if (host == localhost) {
282 + if (host == rrdb.localhost) {
283 node_state_update.live = 1;
284 node_state_update.hops = 0;
285 } else {
@@ -289,10 +289,10 @@ int create_node_instance_result(const char *msg, size_t msg_len)
289 node_state_update.capabilities = aclk_get_node_instance_capas(host);
290 }
291
292 - rrdhost_aclk_state_lock(localhost);
293 - node_state_update.claim_id = localhost->aclk_state.claimed_id;
292 + rrdhost_aclk_state_lock(rrdb.localhost);
293 + node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id;
294 query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update);
295 - rrdhost_aclk_state_unlock(localhost);
295 + rrdhost_aclk_state_unlock(rrdb.localhost);
296
297 freez((void *)node_state_update.capabilities);
298
aclk/aclk_stats.c
+25 -25
@@ -38,7 +38,7 @@ static void aclk_stats_collect(struct aclk_metrics_per_sample *per_sample, struc
38 if (unlikely(!st_aclkstats)) {
39 st_aclkstats = rrdset_create_localhost(
40 "netdata", "aclk_status", NULL, "aclk", NULL, "ACLK/Cloud connection status",
41 - "connected", "netdata", "stats", 200000, localhost->rrd_update_every, RRDSET_TYPE_LINE);
41 + "connected", "netdata", "stats", 200000, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
42
43 rd_online_status = rrddim_add(st_aclkstats, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
44 }
@@ -57,10 +57,10 @@ static void aclk_stats_query_queue(struct aclk_metrics_per_sample *per_sample)
57 if (unlikely(!st_query_thread)) {
58 st_query_thread = rrdset_create_localhost(
59 "netdata", "aclk_query_per_second", NULL, "aclk", NULL, "ACLK Queries per second", "queries/s",
60 - "netdata", "stats", 200001, localhost->rrd_update_every, RRDSET_TYPE_AREA);
60 + "netdata", "stats", 200001, rrdb.localhost->update_every, RRDSET_TYPE_AREA);
61
62 - rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
63 - rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
62 + rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
63 + rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
64 }
65
66 rrddim_set_by_pointer(st_query_thread, rd_queued, per_sample->queries_queued);
@@ -79,7 +79,7 @@ static void aclk_stats_latency(struct aclk_metrics_per_sample *per_sample)
79 if (unlikely(!st)) {
80 st = rrdset_create_localhost(
81 "netdata", "aclk_latency_mqtt", NULL, "aclk", NULL, "ACLK Message Publish Latency", "ms",
82 - "netdata", "stats", 200002, localhost->rrd_update_every, RRDSET_TYPE_LINE);
82 + "netdata", "stats", 200002, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
83
84 rd_avg = rrddim_add(st, "avg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
85 rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -105,10 +105,10 @@ static void aclk_stats_cloud_req(struct aclk_metrics_per_sample *per_sample)
105 if (unlikely(!st)) {
106 st = rrdset_create_localhost(
107 "netdata", "aclk_cloud_req", NULL, "aclk", NULL, "Requests received from cloud", "req/s",
108 - "netdata", "stats", 200005, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
108 + "netdata", "stats", 200005, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
109
110 - rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
111 - rd_rq_err = rrddim_add(st, "malformed", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
110 + rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
111 + rd_rq_err = rrddim_add(st, "malformed", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
112 }
113
114 rrddim_set_by_pointer(st, rd_rq_rcvd, per_sample->cloud_req_recvd - per_sample->cloud_req_err);
@@ -125,10 +125,10 @@ static void aclk_stats_cloud_req_type(struct aclk_metrics_per_sample *per_sample
125 if (unlikely(!st)) {
126 st = rrdset_create_localhost(
127 "netdata", "aclk_processed_query_type", NULL, "aclk", NULL, "Query thread commands processed by their type", "cmd/s",
128 - "netdata", "stats", 200006, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
128 + "netdata", "stats", 200006, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
129
130 for (int i = 0; i < ACLK_QUERY_TYPE_COUNT; i++)
131 - dims[i] = rrddim_add(st, aclk_query_get_name(i, 1), NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
131 + dims[i] = rrddim_add(st, aclk_query_get_name(i, 1), NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
132
133 }
134
@@ -167,10 +167,10 @@ static void aclk_stats_cloud_req_http_type(struct aclk_metrics_per_sample *per_s
167 if (unlikely(!st)) {
168 st = rrdset_create_localhost(
169 "netdata", "aclk_cloud_req_http_type", NULL, "aclk", NULL, "Requests received from cloud via HTTP by their type", "req/s",
170 - "netdata", "stats", 200007, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
170 + "netdata", "stats", 200007, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
171
172 for (int i = 0; i < ACLK_STATS_CLOUD_HTTP_REQ_TYPE_CNT; i++)
173 - rd_rq_types[i] = rrddim_add(st, cloud_req_http_type_names[i], NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
173 + rd_rq_types[i] = rrddim_add(st, cloud_req_http_type_names[i], NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
174 }
175
176 for (int i = 0; i < ACLK_STATS_CLOUD_HTTP_REQ_TYPE_CNT; i++)
@@ -189,12 +189,12 @@ static void aclk_stats_query_threads(uint32_t *queries_per_thread)
189 if (unlikely(!st)) {
190 st = rrdset_create_localhost(
191 "netdata", "aclk_query_threads", NULL, "aclk", NULL, "Queries Processed Per Thread", "req/s",
192 - "netdata", "stats", 200009, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
192 + "netdata", "stats", 200009, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
193
194 for (int i = 0; i < aclk_stats_cfg.query_thread_count; i++) {
195 if (snprintfz(dim_name, MAX_DIM_NAME, "Query %d", i) < 0)
196 netdata_log_error("snprintf encoding error");
197 - aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
197 + aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
198 }
199 }
200
@@ -215,11 +215,11 @@ static void aclk_stats_query_time(struct aclk_metrics_per_sample *per_sample)
215 if (unlikely(!st)) {
216 st = rrdset_create_localhost(
217 "netdata", "aclk_query_time", NULL, "aclk", NULL, "Time it took to process cloud requested DB queries", "us",
218 - "netdata", "stats", 200008, localhost->rrd_update_every, RRDSET_TYPE_LINE);
218 + "netdata", "stats", 200008, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
219
220 - rd_rq_avg = rrddim_add(st, "avg", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
221 - rd_rq_max = rrddim_add(st, "max", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
222 - rd_rq_total = rrddim_add(st, "total", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
220 + rd_rq_avg = rrddim_add(st, "avg", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
221 + rd_rq_max = rrddim_add(st, "max", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
222 + rd_rq_total = rrddim_add(st, "total", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
223 }
224
225 if(per_sample->cloud_q_process_count)
@@ -240,10 +240,10 @@ static void aclk_stats_newproto_rx(uint32_t *rx_msgs_sample)
240 if (unlikely(!st)) {
241 st = rrdset_create_localhost(
242 "netdata", "aclk_protobuf_rx_types", NULL, "aclk", NULL, "Received new cloud architecture messages by their type.", "msg/s",
243 - "netdata", "stats", 200010, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
243 + "netdata", "stats", 200010, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
244
245 for (unsigned int i = 0; i < aclk_stats_cfg.proto_hdl_cnt; i++) {
246 - aclk_stats_cfg.rx_msg_dims[i] = rrddim_add(st, rx_handler_get_name(i), NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
246 + aclk_stats_cfg.rx_msg_dims[i] = rrddim_add(st, rx_handler_get_name(i), NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
247 }
248 }
249
@@ -284,7 +284,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
284 if (unlikely(!st)) {
285 st = rrdset_create_localhost(
286 "netdata", "aclk_openssl_bytes", NULL, "aclk", NULL, "Received and Sent bytes.", "B/s",
287 - "netdata", "stats", 200011, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
287 + "netdata", "stats", 200011, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
288
289 rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
290 rd_recvd = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -293,7 +293,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
293 if (unlikely(!st_txbuf_perc)) {
294 st_txbuf_perc = rrdset_create_localhost(
295 "netdata", "aclk_mqtt_tx_perc", NULL, "aclk", NULL, "Actively used percentage of MQTT Tx Buffer,", "%",
296 - "netdata", "stats", 200012, localhost->rrd_update_every, RRDSET_TYPE_LINE);
296 + "netdata", "stats", 200012, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
297
298 rd_txbuf_perc = rrddim_add(st_txbuf_perc, "used", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
299 }
@@ -301,7 +301,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
301 if (unlikely(!st_txbuf)) {
302 st_txbuf = rrdset_create_localhost(
303 "netdata", "aclk_mqtt_tx_queue", NULL, "aclk", NULL, "State of transmit MQTT queue.", "B",
304 - "netdata", "stats", 200013, localhost->rrd_update_every, RRDSET_TYPE_LINE);
304 + "netdata", "stats", 200013, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
305
306 rd_tx_buffer_usable = rrddim_add(st_txbuf, "usable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
307 rd_tx_buffer_reclaimable = rrddim_add(st_txbuf, "reclaimable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -313,7 +313,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
313 if (unlikely(!st_timing)) {
314 st_timing = rrdset_create_localhost(
315 "netdata", "aclk_mqtt_wss_time", NULL, "aclk", NULL, "Time spent handling MQTT, WSS, SSL and network communication.", "us",
316 - "netdata", "stats", 200014, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
316 + "netdata", "stats", 200014, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
317
318 rd_keepalive = rrddim_add(st_timing, "keep-alive", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
319 rd_read_socket = rrddim_add(st_timing, "socket_read_ssl", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -379,7 +379,7 @@ void *aclk_stats_main_thread(void *ptr)
379
380 heartbeat_t hb;
381 heartbeat_init(&hb);
382 - usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
382 + usec_t step_ut = rrdb.localhost->update_every * USEC_PER_SEC;
383
384 struct aclk_metrics_per_sample per_sample;
385 struct aclk_metrics permanent;
aclk/aclk_tx_msgs.c
+15 -15
@@ -218,19 +218,19 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable
218 .capabilities = aclk_get_agent_capas()
219 };
220
221 - rrdhost_aclk_state_lock(localhost);
222 - if (unlikely(!localhost->aclk_state.claimed_id)) {
221 + rrdhost_aclk_state_lock(rrdb.localhost);
222 + if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) {
223 netdata_log_error("Internal error. Should not come here if not claimed");
224 - rrdhost_aclk_state_unlock(localhost);
224 + rrdhost_aclk_state_unlock(rrdb.localhost);
225 return 0;
226 }
227 - if (localhost->aclk_state.prev_claimed_id)
228 - conn.claim_id = localhost->aclk_state.prev_claimed_id;
227 + if (rrdb.localhost->aclk_state.prev_claimed_id)
228 + conn.claim_id = rrdb.localhost->aclk_state.prev_claimed_id;
229 else
230 - conn.claim_id = localhost->aclk_state.claimed_id;
230 + conn.claim_id = rrdb.localhost->aclk_state.claimed_id;
231
232 char *msg = generate_update_agent_connection(&len, &conn);
233 - rrdhost_aclk_state_unlock(localhost);
233 + rrdhost_aclk_state_unlock(rrdb.localhost);
234
235 if (!msg) {
236 netdata_log_error("Error generating agent::v1::UpdateAgentConnection payload");
@@ -238,9 +238,9 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable
238 }
239
240 pid = aclk_send_bin_message_subtopic_pid(client, msg, len, ACLK_TOPICID_AGENT_CONN, "UpdateAgentConnection");
241 - if (localhost->aclk_state.prev_claimed_id) {
242 - freez(localhost->aclk_state.prev_claimed_id);
243 - localhost->aclk_state.prev_claimed_id = NULL;
241 + if (rrdb.localhost->aclk_state.prev_claimed_id) {
242 + freez(rrdb.localhost->aclk_state.prev_claimed_id);
243 + rrdb.localhost->aclk_state.prev_claimed_id = NULL;
244 }
245 return pid;
246 }
@@ -253,16 +253,16 @@ char *aclk_generate_lwt(size_t *size) {
253 .capabilities = NULL
254 };
255
256 - rrdhost_aclk_state_lock(localhost);
257 - if (unlikely(!localhost->aclk_state.claimed_id)) {
256 + rrdhost_aclk_state_lock(rrdb.localhost);
257 + if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) {
258 netdata_log_error("Internal error. Should not come here if not claimed");
259 - rrdhost_aclk_state_unlock(localhost);
259 + rrdhost_aclk_state_unlock(rrdb.localhost);
260 return NULL;
261 }
262 - conn.claim_id = localhost->aclk_state.claimed_id;
262 + conn.claim_id = rrdb.localhost->aclk_state.claimed_id;
263
264 char *msg = generate_update_agent_connection(size, &conn);
265 - rrdhost_aclk_state_unlock(localhost);
265 + rrdhost_aclk_state_unlock(rrdb.localhost);
266
267 if (!msg)
268 netdata_log_error("Error generating agent::v1::UpdateAgentConnection payload for LWT");
aclk/aclk_util.c
+7 -7
@@ -183,20 +183,20 @@ static void topic_generate_final(struct aclk_topic *t) {
183 if (!replace_tag)
184 return;
185
186 - rrdhost_aclk_state_lock(localhost);
187 - if (unlikely(!localhost->aclk_state.claimed_id)) {
186 + rrdhost_aclk_state_lock(rrdb.localhost);
187 + if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) {
188 netdata_log_error("This should never be called if agent not claimed");
189 - rrdhost_aclk_state_unlock(localhost);
189 + rrdhost_aclk_state_unlock(rrdb.localhost);
190 return;
191 }
192
193 - t->topic = mallocz(strlen(t->topic_recvd) + 1 - strlen(CLAIM_ID_REPLACE_TAG) + strlen(localhost->aclk_state.claimed_id));
193 + t->topic = mallocz(strlen(t->topic_recvd) + 1 - strlen(CLAIM_ID_REPLACE_TAG) + strlen(rrdb.localhost->aclk_state.claimed_id));
194 memcpy(t->topic, t->topic_recvd, replace_tag - t->topic_recvd);
195 dest = t->topic + (replace_tag - t->topic_recvd);
196
197 - memcpy(dest, localhost->aclk_state.claimed_id, strlen(localhost->aclk_state.claimed_id));
198 - dest += strlen(localhost->aclk_state.claimed_id);
199 - rrdhost_aclk_state_unlock(localhost);
197 + memcpy(dest, rrdb.localhost->aclk_state.claimed_id, strlen(rrdb.localhost->aclk_state.claimed_id));
198 + dest += strlen(rrdb.localhost->aclk_state.claimed_id);
199 + rrdhost_aclk_state_unlock(rrdb.localhost);
200 replace_tag += strlen(CLAIM_ID_REPLACE_TAG);
201 strcpy(dest, replace_tag);
202 dest += strlen(replace_tag);
aclk/schema-wrappers/node_info.cc
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "node_info.h"
4
5 #include "proto/nodeinstance/info/v1/info.pb.h"
aclk/schema-wrappers/proto_2_json.cc
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include <google/protobuf/message.h>
4 #include <google/protobuf/util/json_util.h>
5
claim/claim.c
+7 -6
@@ -35,9 +35,9 @@ static char *claiming_errors[] = {
35 char *get_agent_claimid()
36 {
37 char *result;
38 - rrdhost_aclk_state_lock(localhost);
39 - result = (localhost->aclk_state.claimed_id == NULL) ? NULL : strdupz(localhost->aclk_state.claimed_id);
40 - rrdhost_aclk_state_unlock(localhost);
38 + rrdhost_aclk_state_lock(rrdb.localhost);
39 + result = (rrdb.localhost->aclk_state.claimed_id == NULL) ? NULL : strdupz(rrdb.localhost->aclk_state.claimed_id);
40 + rrdhost_aclk_state_unlock(rrdb.localhost);
41 return result;
42 }
43
@@ -81,7 +81,7 @@ CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, con
81 "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s -noreload %s",
82 proxy_flag,
83 netdata_configured_hostname,
84 - localhost->machine_guid,
84 + rrdb.localhost->machine_guid,
85 cloud_base_url,
86 claiming_arguments);
87
@@ -142,6 +142,7 @@ void load_claiming_state(void)
142 #if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
143 netdata_cloud_enabled = false;
144 #else
145 + RRDHOST *localhost = rrdb.localhost;
146 uuid_t uuid;
147
148 // Propagate into aclk and registry. Be kind of atomic...
@@ -312,7 +313,7 @@ void claim_reload_all(void) {
313 error_log_limit_unlimited();
314 load_claiming_state();
315 registry_update_cloud_base_url();
315 - rrdpush_send_claimed_id(localhost);
316 + rrdpush_send_claimed_id(rrdb.localhost);
317 error_log_limit_reset();
318 }
319
@@ -409,7 +410,7 @@ int api_v2_claim(struct web_client *w, char *url) {
410 int ms = 0;
411 do {
412 status = cloud_status();
412 - if (status == CLOUD_STATUS_ONLINE && __atomic_load_n(&localhost->node_id, __ATOMIC_RELAXED))
413 + if (status == CLOUD_STATUS_ONLINE && __atomic_load_n(&rrdb.localhost->node_id, __ATOMIC_RELAXED))
414 break;
415
416 sleep_usec(50 * USEC_PER_MS);
collectors/cgroups.plugin/sys_fs_cgroup.c
+3 -3
@@ -301,9 +301,9 @@ void read_cgroup_plugin_configuration() {
301 throttled_time_hash = simple_hash("throttled_time");
302 throttled_usec_hash = simple_hash("throttled_usec");
303
304 - cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every);
305 - if(cgroup_update_every < localhost->rrd_update_every)
306 - cgroup_update_every = localhost->rrd_update_every;
304 + cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", rrdb.localhost->update_every);
305 + if(cgroup_update_every < rrdb.localhost->update_every)
306 + cgroup_update_every = rrdb.localhost->update_every;
307
308 cgroup_check_for_new_every = (int)config_get_number("plugin:cgroups", "check for new cgroups every", (long long)cgroup_check_for_new_every * (long long)cgroup_update_every);
309 if(cgroup_check_for_new_every < cgroup_update_every)
collectors/cgroups.plugin/tests/test_doubles.c
+4 -4
@@ -47,7 +47,7 @@ void mountinfo_free_all(struct mountinfo *mi)
47 RRDSET *rrdset_create_custom(
48 RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context,
49 const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every,
50 - RRDSET_TYPE chart_type, RRD_MEMORY_MODE memory_mode, long history_entries)
50 + RRDSET_TYPE chart_type, STORAGE_ENGINE_ID storage_engine_id, long history_entries)
51 {
52 UNUSED(host);
53 UNUSED(type);
@@ -62,7 +62,7 @@ RRDSET *rrdset_create_custom(
62 UNUSED(priority);
63 UNUSED(update_every);
64 UNUSED(chart_type);
65 - UNUSED(memory_mode);
65 + UNUSED(storage_engine_id);
66 UNUSED(history_entries);
67
68 return NULL;
@@ -70,7 +70,7 @@ RRDSET *rrdset_create_custom(
70
71 RRDDIM *rrddim_add_custom(
72 RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor,
73 - RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode)
73 + RRD_ALGORITHM algorithm, STORAGE_ENGINE_ID storage_engine_id)
74 {
75 UNUSED(st);
76 UNUSED(id);
@@ -78,7 +78,7 @@ RRDDIM *rrddim_add_custom(
78 UNUSED(multiplier);
79 UNUSED(divisor);
80 UNUSED(algorithm);
81 - UNUSED(memory_mode);
81 + UNUSED(storage_engine_id);
82
83 return NULL;
84 }
collectors/diskspace.plugin/plugin_diskspace.c
+5 -4
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "../proc.plugin/plugin_proc.h"
3 +#include "collectors/proc.plugin/plugin_proc.h"
4 +#include "collectors/utils.h"
5
6 #define PLUGIN_DISKSPACE_NAME "diskspace.plugin"
7
@@ -626,9 +627,9 @@ void *diskspace_main(void *ptr) {
627
628 cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points);
629
629 - int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", localhost->rrd_update_every);
630 - if(update_every < localhost->rrd_update_every)
631 - update_every = localhost->rrd_update_every;
630 + int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", rrdb.localhost->update_every);
631 + if(update_every < rrdb.localhost->update_every)
632 + update_every = rrdb.localhost->update_every;
633
634 check_for_new_mountpoints_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "check for new mount points every", check_for_new_mountpoints_every);
635 if(check_for_new_mountpoints_every < update_every)
collectors/ebpf.plugin/ebpf_unittest.h
+3 -1
@@ -1,5 +1,7 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef NETDATA_EBPF_PLUGIN_UNITTEST_H_
2 -# define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1
4 +#define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1
5
6 #include "ebpf.h"
7
collectors/freebsd.plugin/plugin_freebsd.c
+2 -2
@@ -104,7 +104,7 @@ void *freebsd_main(void *ptr)
104 worker_register_job_name(i, freebsd_modules[i].dim);
105 }
106
107 - usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
107 + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
108 heartbeat_t hb;
109 heartbeat_init(&hb);
110
@@ -124,7 +124,7 @@ void *freebsd_main(void *ptr)
124 netdata_log_debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name);
125
126 worker_is_busy(i);
127 - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
127 + pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt);
128
129 if (unlikely(netdata_exit))
130 break;
collectors/freebsd.plugin/plugin_freebsd.h
+1
@@ -4,6 +4,7 @@
4 #define NETDATA_PLUGIN_FREEBSD_H 1
5
6 #include "daemon/common.h"
7 +#include "collectors/utils.h"
8
9 #include <sys/sysctl.h>
10
collectors/idlejitter.plugin/plugin_idlejitter.c
+2 -2
@@ -38,14 +38,14 @@ void *cpuidlejitter_main(void *ptr) {
38 , "idlejitter.plugin"
39 , NULL
40 , NETDATA_CHART_PRIO_SYSTEM_IDLEJITTER
41 - , localhost->rrd_update_every
41 + , rrdb.localhost->update_every
42 , RRDSET_TYPE_AREA
43 );
44 RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
45 RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
46 RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
47
48 - usec_t update_every_ut = localhost->rrd_update_every * USEC_PER_SEC;
48 + usec_t update_every_ut = rrdb.localhost->update_every * USEC_PER_SEC;
49 struct timeval before, after;
50
51 while (service_running(SERVICE_COLLECTORS)) {
collectors/macos.plugin/plugin_macos.c
+2 -2
@@ -53,7 +53,7 @@ void *macos_main(void *ptr)
53 worker_register_job_name(i, macos_modules[i].dim);
54 }
55
56 - usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
56 + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
57 heartbeat_t hb;
58 heartbeat_init(&hb);
59
@@ -69,7 +69,7 @@ void *macos_main(void *ptr)
69 netdata_log_debug(D_PROCNETDEV_LOOP, "macos calling %s.", pm->name);
70
71 worker_is_busy(i);
72 - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
72 + pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt);
73
74 if (unlikely(netdata_exit))
75 break;
collectors/macos.plugin/plugin_macos.h
+1
@@ -4,6 +4,7 @@
4 #define NETDATA_PLUGIN_MACOS_H 1
5
6 #include "daemon/common.h"
7 +#include "collectors/utils.h"
8
9 int do_macos_sysctl(int update_every, usec_t dt);
10 int do_macos_mach_smi(int update_every, usec_t dt);
collectors/plugins.d/local_listeners.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "libnetdata/libnetdata.h"
4 #include "libnetdata/required_dummies.h"
5
collectors/plugins.d/plugins_d.c
+2 -2
@@ -288,11 +288,11 @@ void *pluginsd_main(void *ptr)
288 strncpyz(cd->filename, file->d_name, FILENAME_MAX);
289 snprintfz(cd->fullfilename, FILENAME_MAX, "%s/%s", directory_name, cd->filename);
290
291 - cd->host = localhost;
291 + cd->host = rrdb.localhost;
292 cd->unsafe.enabled = enabled;
293 cd->unsafe.running = false;
294
295 - cd->update_every = (int)config_get_number(cd->id, "update every", localhost->rrd_update_every);
295 + cd->update_every = (int)config_get_number(cd->id, "update every", rrdb.localhost->update_every);
296 cd->started_t = now_realtime_sec();
297
298 char *def = "";
collectors/plugins.d/pluginsd_parser.c
+34 -7
@@ -204,7 +204,7 @@ static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, cons
204 return NULL;
205 }
206
207 - RRDSET *st = rrdset_find(host, chart);
207 + RRDSET *st = rrdset_find_by_id(host, chart);
208 if (unlikely(!st))
209 netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.",
210 rrdhost_hostname(host), chart, cmd);
@@ -375,11 +375,38 @@ static inline PARSER_RC pluginsd_host_labels(char **words, size_t num_words, PAR
375 PLUGINSD_KEYWORD_HOST_LABEL);
376 }
377
378 +static struct rrdhost_system_info *labels_to_system_info(DICTIONARY *labels) {
379 + struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
380 + info->hops = 1;
381 +
382 + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type");
383 + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type");
384 + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region");
385 + rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name");
386 + rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version");
387 + rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version");
388 + rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores");
389 + rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq");
390 + rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total");
391 + rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space");
392 + rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture");
393 + rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization");
394 + rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container");
395 + rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection");
396 + rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection");
397 + rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node");
398 + rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type");
399 + rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch");
400 + rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist");
401 +
402 + return info;
403 +}
404 +
405 static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
406 if(!parser->user.host_define.parsing_host)
407 return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE_END, "missing initialization, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");
408
382 - RRDHOST *host = rrdhost_find_or_create(
409 + RRDHOST *host = rrdhost_get_or_create(
410 string2str(parser->user.host_define.hostname),
411 string2str(parser->user.host_define.hostname),
412 parser->user.host_define.machine_guid_str,
@@ -390,9 +417,9 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
417 NULL,
418 program_name,
419 program_version,
393 - default_rrd_update_every,
394 - default_rrd_history_entries,
395 - default_rrd_memory_mode,
420 + rrdb.default_update_every,
421 + rrdb.default_rrd_history_entries,
422 + default_storage_engine_id,
423 default_health_enabled,
424 default_rrdpush_enabled,
425 default_rrdpush_destination,
@@ -401,7 +428,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
428 default_rrdpush_enable_replication,
429 default_rrdpush_seconds_to_replicate,
430 default_rrdpush_replication_step,
404 - rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels),
431 + labels_to_system_info(parser->user.host_define.rrdlabels),
432 false
433 );
434
@@ -431,7 +458,7 @@ static inline PARSER_RC pluginsd_host(char **words, size_t num_words, PARSER *pa
458 char *guid = get_word(words, num_words, 1);
459
460 if(!guid || !*guid || strcmp(guid, "localhost") == 0) {
434 - parser->user.host = localhost;
461 + parser->user.host = rrdb.localhost;
462 return PARSER_RC_OK;
463 }
464
collectors/proc.plugin/ipc.c
+6 -6
@@ -327,7 +327,7 @@ int do_ipc(int update_every, usec_t dt) {
327 , PLUGIN_PROC_NAME
328 , "ipc"
329 , NETDATA_CHART_PRIO_SYSTEM_IPC_SEMAPHORES
330 - , localhost->rrd_update_every
330 + , rrdb.localhost->update_every
331 , RRDSET_TYPE_AREA
332 );
333 rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -345,15 +345,15 @@ int do_ipc(int update_every, usec_t dt) {
345 , PLUGIN_PROC_NAME
346 , "ipc"
347 , NETDATA_CHART_PRIO_SYSTEM_IPC_SEM_ARRAYS
348 - , localhost->rrd_update_every
348 + , rrdb.localhost->update_every
349 , RRDSET_TYPE_AREA
350 );
351 rd_arrays = rrddim_add(st_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
352 }
353
354 // variables
355 - semaphores_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_max");
356 - arrays_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_arrays_max");
355 + semaphores_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "ipc_semaphores_max");
356 + arrays_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "ipc_semaphores_arrays_max");
357 }
358
359 struct stat stbuf;
@@ -373,8 +373,8 @@ int do_ipc(int update_every, usec_t dt) {
373 collector_error("Unable to fetch semaphore limits.");
374 }
375 else {
376 - if(semaphores_max) rrdvar_custom_host_variable_set(localhost, semaphores_max, limits.semmns);
377 - if(arrays_max) rrdvar_custom_host_variable_set(localhost, arrays_max, limits.semmni);
376 + if(semaphores_max) rrdvar_custom_host_variable_set(rrdb.localhost, semaphores_max, limits.semmns);
377 + if(arrays_max) rrdvar_custom_host_variable_set(rrdb.localhost, arrays_max, limits.semmni);
378
379 st_arrays->red = limits.semmni;
380 st_semaphores->red = limits.semmns;
collectors/proc.plugin/plugin_proc.c
+2 -2
@@ -161,7 +161,7 @@ void *proc_main(void *ptr)
161 worker_register_job_name(i, proc_modules[i].dim);
162 }
163
164 - usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
164 + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
165 heartbeat_t hb;
166 heartbeat_init(&hb);
167
@@ -185,7 +185,7 @@ void *proc_main(void *ptr)
185 netdata_log_debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name);
186
187 worker_is_busy(i);
188 - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
188 + pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt);
189 }
190 }
191
collectors/proc.plugin/proc_mdstat.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "plugin_proc.h"
4 +#include "collectors/utils.h"
5
6 #define PLUGIN_PROC_MODULE_MDSTAT_NAME "/proc/mdstat"
7
collectors/proc.plugin/proc_net_dev.c
+2 -2
@@ -1518,7 +1518,7 @@ void *netdev_main(void *ptr)
1518
1519 netdata_thread_cleanup_push(netdev_main_cleanup, ptr);
1520
1521 - usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
1521 + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
1522 heartbeat_t hb;
1523 heartbeat_init(&hb);
1524
@@ -1530,7 +1530,7 @@ void *netdev_main(void *ptr)
1530 break;
1531
1532 worker_is_busy(0);
1533 - if(do_proc_net_dev(localhost->rrd_update_every, hb_dt))
1533 + if(do_proc_net_dev(rrdb.localhost->update_every, hb_dt))
1534 break;
1535 }
1536
collectors/proc.plugin/proc_net_netstat.c
+2 -2
@@ -1647,7 +1647,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
1647 arl_expect(arl_udplite, "InCsumErrors", &snmp_root.udplite_InCsumErrors);
1648 arl_expect(arl_udplite, "IgnoredMulti", &snmp_root.udplite_IgnoredMulti);
1649
1650 - tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_connections");
1650 + tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_max_connections");
1651 }
1652
1653 size_t lines, l, words;
@@ -2741,7 +2741,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
2741 // snmp Tcp charts
2742
2743 // this is smart enough to update it, only when it is changed
2744 - rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn);
2744 + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn);
2745
2746 // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
2747 if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO &&
collectors/proc.plugin/proc_net_sockstat.c
+8 -8
@@ -32,9 +32,9 @@ static int read_tcp_mem(void) {
32 *tcp_mem_high_threshold = NULL;
33
34 if(unlikely(!tcp_mem_low_threshold)) {
35 - tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_low");
36 - tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_pressure");
37 - tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_high");
35 + tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_low");
36 + tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_pressure");
37 + tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_high");
38 }
39
40 if(unlikely(!filename)) {
@@ -60,9 +60,9 @@ static int read_tcp_mem(void) {
60
61 // fprintf(stderr, "TCP MEM low = %llu, pressure = %llu, high = %llu\n", low, pressure, high);
62
63 - rrdvar_custom_host_variable_set(localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0);
64 - rrdvar_custom_host_variable_set(localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0);
65 - rrdvar_custom_host_variable_set(localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0);
63 + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0);
64 + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0);
65 + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0);
66
67 return 0;
68 }
@@ -81,9 +81,9 @@ static kernel_uint_t read_tcp_max_orphans(void) {
81 if(read_single_number_file(filename, &tcp_max_orphans) == 0) {
82
83 if(unlikely(!tcp_max_orphans_var))
84 - tcp_max_orphans_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_orphans");
84 + tcp_max_orphans_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_max_orphans");
85
86 - rrdvar_custom_host_variable_set(localhost, tcp_max_orphans_var, tcp_max_orphans);
86 + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_max_orphans_var, tcp_max_orphans);
87 return tcp_max_orphans;
88 }
89
collectors/proc.plugin/proc_net_softnet_stat.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "plugin_proc.h"
4 +#include "collectors/utils.h"
5
6 #define PLUGIN_PROC_MODULE_NET_SOFTNET_NAME "/proc/net/softnet_stat"
7
collectors/proc.plugin/proc_net_stat_conntrack.c
+2 -2
@@ -50,7 +50,7 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) {
50 if(!do_sockets && !read_full)
51 return 1;
52
53 - rrdvar_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "netfilter_conntrack_max");
53 + rrdvar_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "netfilter_conntrack_max");
54 }
55
56 if(likely(read_full)) {
@@ -125,7 +125,7 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) {
125
126 unsigned long long max;
127 if(likely(!read_single_number_file(nf_conntrack_max_filename, &max)))
128 - rrdvar_custom_host_variable_set(localhost, rrdvar_max, max);
128 + rrdvar_custom_host_variable_set(rrdb.localhost, rrdvar_max, max);
129 }
130
131 // --------------------------------------------------------------------
collectors/proc.plugin/proc_net_wireless.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include <stdbool.h>
4 #include "plugin_proc.h"
5
collectors/proc.plugin/proc_stat.c
+2 -2
@@ -719,7 +719,7 @@ int do_proc_stat(int update_every, usec_t dt) {
719 }
720
721 if(unlikely(core == 0 && cpus_var == NULL))
722 - cpus_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "active_processors");
722 + cpus_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "active_processors");
723 }
724
725 rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_user, user);
@@ -1064,7 +1064,7 @@ int do_proc_stat(int update_every, usec_t dt) {
1064 }
1065
1066 if(cpus_var)
1067 - rrdvar_custom_host_variable_set(localhost, cpus_var, cores_found);
1067 + rrdvar_custom_host_variable_set(rrdb.localhost, cpus_var, cores_found);
1068
1069 return 0;
1070 }
collectors/proc.plugin/sys_devices_pci_aer.c
+1 -1
@@ -196,7 +196,7 @@ int do_proc_sys_devices_pci_aer(int update_every, usec_t dt __maybe_unused) {
196 int do_pci_slots = CONFIG_BOOLEAN_NO;
197
198 char buffer[100 + 1] = "";
199 - rrdlabels_get_value_strcpyz(localhost->rrdlabels, buffer, 100, "_virtualization");
199 + rrdlabels_get_value_strcpyz(rrdb.localhost->rrdlabels, buffer, 100, "_virtualization");
200 if(strcmp(buffer, "none") != 0) {
201 // no need to run on virtualized environments
202 do_root_ports = CONFIG_BOOLEAN_NO;
collectors/profile.plugin/plugin_profile.cc
+2 -2
@@ -195,8 +195,8 @@ extern "C" void *profile_main(void *ptr) {
195 netdata_thread_cleanup_push(profile_main_cleanup, ptr);
196
197 int UpdateEvery = (int) config_get_number(CONFIG_SECTION_PROFILE, "update every", 1);
198 - if (UpdateEvery < localhost->rrd_update_every)
199 - UpdateEvery = localhost->rrd_update_every;
198 + if (UpdateEvery < rrdb.localhost->update_every)
199 + UpdateEvery = rrdb.localhost->update_every;
200
201 // pick low-default values, in case this plugin is ever enabled accidentaly.
202 size_t NumThreads = config_get_number(CONFIG_SECTION_PROFILE, "number of threads", 2);
collectors/statsd.plugin/statsd.c
+12 -13
@@ -218,7 +218,7 @@ typedef struct statsd_app {
218 const char *name;
219 SIMPLE_PATTERN *metrics;
220 STATS_METRIC_OPTIONS default_options;
221 - RRD_MEMORY_MODE rrd_memory_mode;
221 + STORAGE_ENGINE_ID storage_engine_id;
222 int32_t rrd_history_entries;
223 DICTIONARY *dict;
224
@@ -1296,8 +1296,8 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1296 // a new app
1297 app = callocz(sizeof(STATSD_APP), 1);
1298 app->name = strdupz("unnamed");
1299 - app->rrd_memory_mode = localhost->rrd_memory_mode;
1300 - app->rrd_history_entries = localhost->rrd_history_entries;
1299 + app->storage_engine_id = rrdb.localhost->storage_engine_id;
1300 + app->rrd_history_entries = rrdb.localhost->rrd_history_entries;
1301
1302 app->next = statsd.apps;
1303 statsd.apps = app;
@@ -1409,7 +1409,6 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1409 else if (!strcmp(name, "memory mode")) {
1410 // this is not supported anymore
1411 // with the implementation of storage engines, all charts have the same storage engine always
1412 - // app->rrd_memory_mode = rrd_memory_mode_id(value);
1412 ;
1413 }
1414 else if (!strcmp(name, "history")) {
@@ -1599,7 +1598,7 @@ static inline RRDSET *statsd_private_rrdset_create(
1598 statsd.private_charts++;
1599
1600 RRDSET *st = rrdset_create_custom(
1602 - localhost // host
1601 + rrdb.localhost // host
1602 , type // type
1603 , id // id
1604 , name // name
@@ -1612,8 +1611,8 @@ static inline RRDSET *statsd_private_rrdset_create(
1611 , priority // priority
1612 , update_every // update every
1613 , chart_type // chart type
1615 - , default_rrd_memory_mode // memory mode
1616 - , default_rrd_history_entries // history
1614 + , default_storage_engine_id // storage_engine_id
1615 + , rrdb.default_rrd_history_entries // history
1616 );
1617 rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
1618
@@ -2204,7 +2203,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
2203
2204 if(!chart->st) {
2205 chart->st = rrdset_create_custom(
2207 - localhost // host
2206 + rrdb.localhost // host
2207 , app->name // type
2208 , chart->id // id
2209 , chart->name // name
@@ -2217,7 +2216,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
2216 , chart->priority // priority
2217 , statsd.update_every // update every
2218 , chart->chart_type // chart type
2220 - , app->rrd_memory_mode // memory mode
2219 + , app->storage_engine_id // storage engine id
2220 , app->rrd_history_entries // history
2221 );
2222
@@ -2432,11 +2431,11 @@ void *statsd_main(void *ptr) {
2431
2432 statsd.enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, "statsd", statsd.enabled);
2433
2435 - statsd.update_every = default_rrd_update_every;
2434 + statsd.update_every = rrdb.default_update_every;
2435 statsd.update_every = (int)config_get_number(CONFIG_SECTION_STATSD, "update every (flushInterval)", statsd.update_every);
2437 - if(statsd.update_every < default_rrd_update_every) {
2438 - collector_error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, default_rrd_update_every);
2439 - statsd.update_every = default_rrd_update_every;
2436 + if(statsd.update_every < rrdb.default_update_every) {
2437 + collector_error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, rrdb.default_update_every);
2438 + statsd.update_every = rrdb.default_update_every;
2439 }
2440
2441 #ifdef HAVE_RECVMMSG
collectors/tc.plugin/plugin_tc.c
+7 -7
@@ -414,7 +414,7 @@ static inline void tc_device_commit(struct tc_device *d) {
414 PLUGIN_TC_NAME,
415 NULL,
416 NETDATA_CHART_PRIO_TC_QOS,
417 - localhost->rrd_update_every,
417 + rrdb.localhost->update_every,
418 d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
419
420 rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -475,7 +475,7 @@ static inline void tc_device_commit(struct tc_device *d) {
475 PLUGIN_TC_NAME,
476 NULL,
477 NETDATA_CHART_PRIO_TC_QOS_PACKETS,
478 - localhost->rrd_update_every,
478 + rrdb.localhost->update_every,
479 d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
480
481 rrdlabels_add(d->st_packets->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -539,7 +539,7 @@ static inline void tc_device_commit(struct tc_device *d) {
539 PLUGIN_TC_NAME,
540 NULL,
541 NETDATA_CHART_PRIO_TC_QOS_DROPPED,
542 - localhost->rrd_update_every,
542 + rrdb.localhost->update_every,
543 d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
544
545 rrdlabels_add(d->st_dropped->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -603,7 +603,7 @@ static inline void tc_device_commit(struct tc_device *d) {
603 PLUGIN_TC_NAME,
604 NULL,
605 NETDATA_CHART_PRIO_TC_QOS_TOKENS,
606 - localhost->rrd_update_every,
606 + rrdb.localhost->update_every,
607 RRDSET_TYPE_LINE);
608
609 rrdlabels_add(d->st_tokens->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -668,7 +668,7 @@ static inline void tc_device_commit(struct tc_device *d) {
668 PLUGIN_TC_NAME,
669 NULL,
670 NETDATA_CHART_PRIO_TC_QOS_CTOKENS,
671 - localhost->rrd_update_every,
671 + rrdb.localhost->update_every,
672 RRDSET_TYPE_LINE);
673
674 rrdlabels_add(d->st_ctokens->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -935,7 +935,7 @@ void *tc_main(void *ptr) {
935 struct tc_device *device = NULL;
936 struct tc_class *class = NULL;
937
938 - snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, localhost->rrd_update_every);
938 + snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, rrdb.localhost->update_every);
939 netdata_log_debug(D_TC_LOOP, "executing '%s'", command);
940
941 fp_child_output = netdata_popen(command, (pid_t *)&tc_child_pid, &fp_child_input);
@@ -1173,7 +1173,7 @@ void *tc_main(void *ptr) {
1173 goto cleanup;
1174 }
1175
1176 - sleep((unsigned int) localhost->rrd_update_every);
1176 + sleep((unsigned int) rrdb.localhost->update_every);
1177 }
1178
1179 cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement
collectors/timex.plugin/plugin_timex.c
+2 -2
@@ -50,8 +50,8 @@ void *timex_main(void *ptr)
50 netdata_thread_cleanup_push(timex_main_cleanup, ptr);
51
52 int update_every = (int)config_get_number(CONFIG_SECTION_TIMEX, "update every", 10);
53 - if (update_every < localhost->rrd_update_every)
54 - update_every = localhost->rrd_update_every;
53 + if (update_every < rrdb.localhost->update_every)
54 + update_every = rrdb.localhost->update_every;
55
56 int do_sync = config_get_boolean(CONFIG_SECTION_TIMEX, "clock synchronization state", CONFIG_BOOLEAN_YES);
57 int do_offset = config_get_boolean(CONFIG_SECTION_TIMEX, "time offset", CONFIG_BOOLEAN_YES);
collectors/utils.h new
+42
@@ -0,0 +1,42 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef COLLECTORS_UTILS_H
4 +#define COLLECTORS_UTILS_H
5 +
6 +#include "database/rrd.h"
7 +
8 +static inline RRDSET *rrdset_find_active_localhost(const char *id)
9 +{
10 + RRDSET *st = rrdset_find_by_id(rrdb.localhost, id);
11 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
12 + return NULL;
13 + return st;
14 +}
15 +
16 +static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
17 +{
18 + RRDSET *st = rrdset_find_by_type(rrdb.localhost, type, id);
19 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
20 + return NULL;
21 + return st;
22 +}
23 +
24 +
25 +static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
26 +{
27 + RRDSET *st = rrdset_find_by_name(rrdb.localhost, name);
28 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
29 + return NULL;
30 + return st;
31 +}
32 +
33 +static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id) {
34 + RRDDIM *rd = rrddim_find_by_id(st, id);
35 +
36 + if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
37 + return NULL;
38 +
39 + return rd;
40 +}
41 +
42 +#endif /* COLLECTORS_UTILS_H */
daemon/analytics.c
+22 -22
@@ -25,7 +25,7 @@ struct array_printer {
25 void analytics_log_data(void)
26 {
27 netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_STREAM_ENABLED : [%s]", analytics_data.netdata_config_stream_enabled);
28 - netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_MEMORY_MODE : [%s]", analytics_data.netdata_config_memory_mode);
28 + netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_MEMORY_MODE : [%s]", analytics_data.netdata_config_storage_engine_name);
29 netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_EXPORTING_ENABLED : [%s]", analytics_data.netdata_config_exporting_enabled);
30 netdata_log_debug(D_ANALYTICS, "NETDATA_EXPORTING_CONNECTORS : [%s]", analytics_data.netdata_exporting_connectors);
31 netdata_log_debug(D_ANALYTICS, "NETDATA_ALLMETRICS_PROMETHEUS_USED : [%s]", analytics_data.netdata_allmetrics_prometheus_used);
@@ -71,7 +71,7 @@ void analytics_log_data(void)
71 void analytics_free_data(void)
72 {
73 freez(analytics_data.netdata_config_stream_enabled);
74 - freez(analytics_data.netdata_config_memory_mode);
74 + freez(analytics_data.netdata_config_storage_engine_name);
75 freez(analytics_data.netdata_config_exporting_enabled);
76 freez(analytics_data.netdata_exporting_connectors);
77 freez(analytics_data.netdata_allmetrics_prometheus_used);
@@ -114,7 +114,7 @@ void analytics_free_data(void)
114 /*
115 * Set a numeric/boolean data with a value
116 */
117 -void analytics_set_data(char **name, char *value)
117 +static void analytics_set_data(char **name, const char *value)
118 {
119 if (*name) {
120 analytics_data.data_length -= strlen(*name);
@@ -127,7 +127,7 @@ void analytics_set_data(char **name, char *value)
127 /*
128 * Set a string data with a value
129 */
130 -void analytics_set_data_str(char **name, char *value)
130 +static void analytics_set_data_str(char **name, const char *value)
131 {
132 size_t value_string_len;
133 if (*name) {
@@ -215,7 +215,7 @@ void analytics_mirrored_hosts(void)
215 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
216 continue;
217
218 - ((host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) ? reachable++ : unreachable++);
218 + ((host == rrdb.localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) ? reachable++ : unreachable++);
219
220 count++;
221 }
@@ -235,7 +235,7 @@ void analytics_exporters(void)
235 //decide if something else is more suitable (but probably not null)
236 BUFFER *bi = buffer_create(1000, NULL);
237 analytics_exporting_connectors(bi);
238 - analytics_set_data_str(&analytics_data.netdata_exporting_connectors, (char *)buffer_tostring(bi));
238 + analytics_set_data_str(&analytics_data.netdata_exporting_connectors, buffer_tostring(bi));
239 buffer_free(bi);
240 }
241
@@ -272,7 +272,7 @@ void analytics_collectors(void)
272 char name[500];
273 BUFFER *bt = buffer_create(1000, NULL);
274
275 - rrdset_foreach_read(st, localhost) {
275 + rrdset_foreach_read(st, rrdb.localhost) {
276 if(!rrdset_is_available_for_viewers(st))
277 continue;
278
@@ -349,7 +349,7 @@ void analytics_alarms_notifications(void)
349 }
350 freez(script);
351
352 - analytics_set_data_str(&analytics_data.netdata_notification_methods, (char *)buffer_tostring(b));
352 + analytics_set_data_str(&analytics_data.netdata_notification_methods, buffer_tostring(b));
353
354 buffer_free(b);
355 }
@@ -377,15 +377,15 @@ void analytics_https(void)
377 analytics_exporting_connectors_ssl(b);
378
379 buffer_strcat(b, netdata_ssl_streaming_sender_ctx &&
380 - rrdhost_flag_check(localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) &&
381 - SSL_connection(&localhost->sender->ssl) ? "streaming|" : "|");
380 + rrdhost_flag_check(rrdb.localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) &&
381 + SSL_connection(&rrdb.localhost->sender->ssl) ? "streaming|" : "|");
382
383 buffer_strcat(b, netdata_ssl_web_server_ctx ? "web" : "");
384 #else
385 buffer_strcat(b, "||");
386 #endif
387
388 - analytics_set_data_str(&analytics_data.netdata_config_https_available, (char *)buffer_tostring(b));
388 + analytics_set_data_str(&analytics_data.netdata_config_https_available, buffer_tostring(b));
389 buffer_free(b);
390 }
391
@@ -394,7 +394,7 @@ void analytics_charts(void)
394 RRDSET *st;
395 size_t c = 0;
396
397 - rrdset_foreach_read(st, localhost)
397 + rrdset_foreach_read(st, rrdb.localhost)
398 if(rrdset_is_available_for_viewers(st)) c++;
399 rrdset_foreach_done(st);
400
@@ -410,7 +410,7 @@ void analytics_metrics(void)
410 {
411 RRDSET *st;
412 size_t dimensions = 0;
413 - rrdset_foreach_read(st, localhost) {
413 + rrdset_foreach_read(st, rrdb.localhost) {
414 if (rrdset_is_available_for_viewers(st)) {
415 RRDDIM *rd;
416 rrddim_foreach_read(rd, st) {
@@ -436,7 +436,7 @@ void analytics_alarms(void)
436 size_t alarm_warn = 0, alarm_crit = 0, alarm_normal = 0;
437 char b[21];
438 RRDCALC *rc;
439 - foreach_rrdcalc_in_rrdhost_read(localhost, rc) {
439 + foreach_rrdcalc_in_rrdhost_read(rrdb.localhost, rc) {
440 if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
441 continue;
442
@@ -650,7 +650,7 @@ static const char *verify_or_create_required_directory(const char *dir) {
650 void set_late_global_environment(struct rrdhost_system_info *system_info)
651 {
652 analytics_set_data(&analytics_data.netdata_config_stream_enabled, default_rrdpush_enabled ? "true" : "false");
653 - analytics_set_data_str(&analytics_data.netdata_config_memory_mode, (char *)rrd_memory_mode_name(default_rrd_memory_mode));
653 + analytics_set_data_str(&analytics_data.netdata_config_storage_engine_name, storage_engine_name(default_storage_engine_id));
654
655 #ifdef DISABLE_CLOUD
656 analytics_set_data(&analytics_data.netdata_host_cloud_enabled, "false");
@@ -663,10 +663,10 @@ void set_late_global_environment(struct rrdhost_system_info *system_info)
663 #ifdef ENABLE_DBENGINE
664 {
665 char b[16];
666 - snprintfz(b, 15, "%d", default_rrdeng_page_cache_mb);
666 + snprintfz(b, 15, "%d", rrdb.default_rrdeng_page_cache_mb);
667 analytics_set_data(&analytics_data.netdata_config_page_cache_size, b);
668
669 - snprintfz(b, 15, "%d", default_multidb_disk_quota_mb);
669 + snprintfz(b, 15, "%d", rrdb.default_multidb_disk_quota_mb);
670 analytics_set_data(&analytics_data.netdata_config_multidb_disk_quota, b);
671 }
672 #endif
@@ -682,12 +682,12 @@ void set_late_global_environment(struct rrdhost_system_info *system_info)
682 else
683 analytics_set_data(&analytics_data.netdata_config_web_enabled, "true");
684
685 - analytics_set_data_str(&analytics_data.netdata_config_release_channel, (char *)get_release_channel());
685 + analytics_set_data_str(&analytics_data.netdata_config_release_channel, get_release_channel());
686
687 {
688 BUFFER *bi = buffer_create(1000, NULL);
689 analytics_build_info(bi);
690 - analytics_set_data_str(&analytics_data.netdata_buildinfo, (char *)buffer_tostring(bi));
690 + analytics_set_data_str(&analytics_data.netdata_buildinfo, buffer_tostring(bi));
691 buffer_free(bi);
692 }
693
@@ -827,7 +827,7 @@ void set_global_environment()
827 {
828 {
829 char b[16];
830 - snprintfz(b, 15, "%d", default_rrd_update_every);
830 + snprintfz(b, 15, "%d", rrdb.default_update_every);
831 setenv("NETDATA_UPDATE_EVERY", b, 1);
832 }
833
@@ -861,7 +861,7 @@ void set_global_environment()
861
862 analytics_data.data_length = 0;
863 analytics_set_data(&analytics_data.netdata_config_stream_enabled, "null");
864 - analytics_set_data(&analytics_data.netdata_config_memory_mode, "null");
864 + analytics_set_data(&analytics_data.netdata_config_storage_engine_name, "null");
865 analytics_set_data(&analytics_data.netdata_config_exporting_enabled, "null");
866 analytics_set_data(&analytics_data.netdata_exporting_connectors, "null");
867 analytics_set_data(&analytics_data.netdata_allmetrics_prometheus_used, "null");
@@ -987,7 +987,7 @@ void send_statistics(const char *action, const char *action_result, const char *
987 action_result,
988 action_data,
989 analytics_data.netdata_config_stream_enabled,
990 - analytics_data.netdata_config_memory_mode,
990 + analytics_data.netdata_config_storage_engine_name,
991 analytics_data.netdata_config_exporting_enabled,
992 analytics_data.netdata_exporting_connectors,
993 analytics_data.netdata_allmetrics_prometheus_used,
daemon/analytics.h
+1 -1
@@ -22,7 +22,7 @@
22
23 struct analytics_data {
24 char *netdata_config_stream_enabled;
25 - char *netdata_config_memory_mode;
25 + char *netdata_config_storage_engine_name;
26 char *netdata_exporting_connectors;
27 char *netdata_config_exporting_enabled;
28 char *netdata_allmetrics_prometheus_used;
daemon/buildinfo.c
+2 -2
@@ -1252,8 +1252,8 @@ static void populate_system_info(void) {
1252 struct rrdhost_system_info *system_info;
1253 bool free_system_info = false;
1254
1255 - if(localhost && localhost->system_info) {
1256 - system_info = localhost->system_info;
1255 + if(rrdb.localhost && rrdb.localhost->system_info) {
1256 + system_info = rrdb.localhost->system_info;
1257 }
1258 else {
1259 system_info = callocz(1, sizeof(struct rrdhost_system_info));
daemon/commands.c
+3 -3
@@ -52,7 +52,7 @@ static cmd_status_t cmd_dumpconfig(char *args, char **message);
52 static command_info_t command_info_array[] = {
53 {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY}, // show help menu
54 {"reload-health", cmd_reload_health_execute, CMD_TYPE_ORTHOGONAL}, // reload health configuration
55 - {"save-database", cmd_save_database_execute, CMD_TYPE_ORTHOGONAL}, // save database for memory mode save
55 + {"save-database", cmd_save_database_execute, CMD_TYPE_ORTHOGONAL}, // save database for storage engine SAVE
56 {"reopen-logs", cmd_reopen_logs_execute, CMD_TYPE_ORTHOGONAL}, // Close and reopen log files
57 {"shutdown-agent", cmd_exit_execute, CMD_TYPE_EXCLUSIVE}, // exit cleanly
58 {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY}, // exit with fatal error
@@ -218,10 +218,10 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
218 {
219 (void)args;
220 netdata_log_info("COMMAND: reloading host labels.");
221 - reload_host_labels();
221 + localhost_load_labels();
222
223 BUFFER *wb = buffer_create(10, NULL);
224 - rrdlabels_log_to_buffer(localhost->rrdlabels, wb);
224 + rrdlabels_log_to_buffer(rrdb.localhost->rrdlabels, wb);
225 (*message)=strdupz(buffer_tostring(wb));
226 buffer_free(wb);
227
daemon/global_statistics.c
+91 -91
@@ -83,7 +83,7 @@ static struct global_statistics {
83 };
84
85 void global_statistics_rrdset_done_chart_collection_completed(size_t *points_read_per_tier_array) {
86 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
86 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
87 __atomic_fetch_add(&global_statistics.db_points_stored_per_tier[tier], points_read_per_tier_array[tier], __ATOMIC_RELAXED);
88 points_read_per_tier_array[tier] = 0;
89 }
@@ -210,7 +210,7 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
210 gs->backfill_queries_made = __atomic_load_n(&global_statistics.backfill_queries_made, __ATOMIC_RELAXED);
211 gs->backfill_db_points_read = __atomic_load_n(&global_statistics.backfill_db_points_read, __ATOMIC_RELAXED);
212
213 - for(size_t tier = 0; tier < storage_tiers ;tier++)
213 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
214 gs->db_points_stored_per_tier[tier] = __atomic_load_n(&global_statistics.db_points_stored_per_tier[tier], __ATOMIC_RELAXED);
215
216 if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) {
@@ -262,7 +262,7 @@ static void global_statistics_charts(void) {
262 , "netdata"
263 , "stats"
264 , 130000
265 - , localhost->rrd_update_every
265 + , rrdb.localhost->update_every
266 , RRDSET_TYPE_STACKED
267 );
268
@@ -308,7 +308,7 @@ static void global_statistics_charts(void) {
308 "netdata",
309 "stats",
310 130100,
311 - localhost->rrd_update_every,
311 + rrdb.localhost->update_every,
312 RRDSET_TYPE_STACKED);
313
314 rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -397,7 +397,7 @@ static void global_statistics_charts(void) {
397 "netdata",
398 "stats",
399 130101,
400 - localhost->rrd_update_every,
400 + rrdb.localhost->update_every,
401 RRDSET_TYPE_STACKED);
402
403 rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -452,7 +452,7 @@ static void global_statistics_charts(void) {
452 "netdata",
453 "stats",
454 130150,
455 - localhost->rrd_update_every,
455 + rrdb.localhost->update_every,
456 RRDSET_TYPE_LINE);
457
458 rd_uptime = rrddim_add(st_uptime, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -480,7 +480,7 @@ static void global_statistics_charts(void) {
480 , "netdata"
481 , "stats"
482 , 130200
483 - , localhost->rrd_update_every
483 + , rrdb.localhost->update_every
484 , RRDSET_TYPE_LINE
485 );
486
@@ -509,7 +509,7 @@ static void global_statistics_charts(void) {
509 , "netdata"
510 , "stats"
511 , 130300
512 - , localhost->rrd_update_every
512 + , rrdb.localhost->update_every
513 , RRDSET_TYPE_LINE
514 );
515
@@ -539,7 +539,7 @@ static void global_statistics_charts(void) {
539 , "netdata"
540 , "stats"
541 , 130400
542 - , localhost->rrd_update_every
542 + , rrdb.localhost->update_every
543 , RRDSET_TYPE_AREA
544 );
545
@@ -571,7 +571,7 @@ static void global_statistics_charts(void) {
571 , "netdata"
572 , "stats"
573 , 130500
574 - , localhost->rrd_update_every
574 + , rrdb.localhost->update_every
575 , RRDSET_TYPE_LINE
576 );
577
@@ -618,7 +618,7 @@ static void global_statistics_charts(void) {
618 , "netdata"
619 , "stats"
620 , 130600
621 - , localhost->rrd_update_every
621 + , rrdb.localhost->update_every
622 , RRDSET_TYPE_LINE
623 );
624
@@ -670,7 +670,7 @@ static void global_statistics_charts(void) {
670 , "netdata"
671 , "stats"
672 , 131000
673 - , localhost->rrd_update_every
673 + , rrdb.localhost->update_every
674 , RRDSET_TYPE_STACKED
675 );
676
@@ -721,7 +721,7 @@ static void global_statistics_charts(void) {
721 , "netdata"
722 , "stats"
723 , 131001
724 - , localhost->rrd_update_every
724 + , rrdb.localhost->update_every
725 , RRDSET_TYPE_STACKED
726 );
727
@@ -770,7 +770,7 @@ static void global_statistics_charts(void) {
770 , "netdata"
771 , "stats"
772 , 131002
773 - , localhost->rrd_update_every
773 + , rrdb.localhost->update_every
774 , RRDSET_TYPE_STACKED
775 );
776
@@ -810,18 +810,18 @@ static void global_statistics_charts(void) {
810 , "netdata"
811 , "stats"
812 , 131003
813 - , localhost->rrd_update_every
813 + , rrdb.localhost->update_every
814 , RRDSET_TYPE_STACKED
815 );
816
817 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
817 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
818 char buf[30 + 1];
819 snprintfz(buf, 30, "tier%zu", tier);
820 rds[tier] = rrddim_add(st_points_stored, buf, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
821 }
822 }
823
824 - for(size_t tier = 0; tier < storage_tiers ;tier++)
824 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
825 rrddim_set_by_pointer(st_points_stored, rds[tier], (collected_number)gs.db_points_stored_per_tier[tier]);
826
827 rrdset_done(st_points_stored);
@@ -882,7 +882,7 @@ static inline void sqlite3_statistics_copy(struct sqlite3_statistics *gs) {
882 gs->sqlite3_queries_failed_locked = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
883 gs->sqlite3_rows = __atomic_load_n(&sqlite3_statistics.sqlite3_rows, __ATOMIC_RELAXED);
884
885 - usec_t timeout = default_rrd_update_every * USEC_PER_SEC + default_rrd_update_every * USEC_PER_SEC / 3;
885 + usec_t timeout = rrdb.default_update_every * USEC_PER_SEC + rrdb.default_update_every * USEC_PER_SEC / 3;
886 usec_t now = now_monotonic_usec();
887 if(!last_run)
888 last_run = now;
@@ -968,7 +968,7 @@ static void sqlite3_statistics_charts(void) {
968 , "netdata"
969 , "stats"
970 , 131100
971 - , localhost->rrd_update_every
971 + , rrdb.localhost->update_every
972 , RRDSET_TYPE_LINE
973 );
974
@@ -998,7 +998,7 @@ static void sqlite3_statistics_charts(void) {
998 , "netdata"
999 , "stats"
1000 , 131101
1001 - , localhost->rrd_update_every
1001 + , rrdb.localhost->update_every
1002 , RRDSET_TYPE_LINE
1003 );
1004
@@ -1034,7 +1034,7 @@ static void sqlite3_statistics_charts(void) {
1034 , "netdata"
1035 , "stats"
1036 , 131102
1037 - , localhost->rrd_update_every
1037 + , rrdb.localhost->update_every
1038 , RRDSET_TYPE_LINE
1039 );
1040
@@ -1065,7 +1065,7 @@ static void sqlite3_statistics_charts(void) {
1065 , "netdata"
1066 , "stats"
1067 , 131103
1068 - , localhost->rrd_update_every
1068 + , rrdb.localhost->update_every
1069 , RRDSET_TYPE_LINE
1070 );
1071
@@ -1109,7 +1109,7 @@ static void sqlite3_statistics_charts(void) {
1109 , "netdata"
1110 , "stats"
1111 , 131104
1112 - , localhost->rrd_update_every
1112 + , rrdb.localhost->update_every
1113 , RRDSET_TYPE_LINE
1114 );
1115
@@ -1237,7 +1237,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1237 "netdata",
1238 "stats",
1239 priority,
1240 - localhost->rrd_update_every,
1240 + rrdb.localhost->update_every,
1241 RRDSET_TYPE_LINE);
1242
1243 ptrs->rd_hit_ratio_closest = rrddim_add(ptrs->st_cache_hit_ratio, "closest", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
@@ -1285,7 +1285,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1285 "netdata",
1286 "stats",
1287 priority,
1288 - localhost->rrd_update_every,
1288 + rrdb.localhost->update_every,
1289 RRDSET_TYPE_LINE);
1290
1291 ptrs->rd_searches_closest = rrddim_add(ptrs->st_operations, "search closest", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1339,7 +1339,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1339 "netdata",
1340 "stats",
1341 priority,
1342 - localhost->rrd_update_every,
1342 + rrdb.localhost->update_every,
1343 RRDSET_TYPE_STACKED);
1344
1345 ptrs->rd_pgc_memory_free = rrddim_add(ptrs->st_pgc_memory, "free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1393,7 +1393,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1393 "netdata",
1394 "stats",
1395 priority,
1396 - localhost->rrd_update_every,
1396 + rrdb.localhost->update_every,
1397 RRDSET_TYPE_LINE);
1398
1399 ptrs->rd_pgc_tm_current = rrddim_add(ptrs->st_pgc_tm, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1443,7 +1443,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1443 "netdata",
1444 "stats",
1445 priority,
1446 - localhost->rrd_update_every,
1446 + rrdb.localhost->update_every,
1447 RRDSET_TYPE_LINE);
1448
1449 ptrs->rd_pgc_pages_clean = rrddim_add(ptrs->st_pgc_pages, "clean", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1487,7 +1487,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1487 "netdata",
1488 "stats",
1489 priority,
1490 - localhost->rrd_update_every,
1490 + rrdb.localhost->update_every,
1491 RRDSET_TYPE_AREA);
1492
1493 ptrs->rd_pgc_memory_new_clean = rrddim_add(ptrs->st_pgc_memory_changes, "new clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1529,7 +1529,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1529 "netdata",
1530 "stats",
1531 priority,
1532 - localhost->rrd_update_every,
1532 + rrdb.localhost->update_every,
1533 RRDSET_TYPE_AREA);
1534
1535 ptrs->rd_pgc_memory_dirty_to_clean = rrddim_add(ptrs->st_pgc_memory_migrations, "dirty to clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1569,7 +1569,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1569 "netdata",
1570 "stats",
1571 priority,
1572 - localhost->rrd_update_every,
1572 + rrdb.localhost->update_every,
1573 RRDSET_TYPE_AREA);
1574
1575 ptrs->rd_pgc_memory_evictions_aggressive = rrddim_add(ptrs->st_pgc_memory_events, "evictions aggressive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1611,7 +1611,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1611 "netdata",
1612 "stats",
1613 priority,
1614 - localhost->rrd_update_every,
1614 + rrdb.localhost->update_every,
1615 RRDSET_TYPE_LINE);
1616
1617 ptrs->rd_pgc_waste_evictions_skipped = rrddim_add(ptrs->st_pgc_waste, "evictions skipped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1663,7 +1663,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1663 "netdata",
1664 "stats",
1665 priority,
1666 - localhost->rrd_update_every,
1666 + rrdb.localhost->update_every,
1667 RRDSET_TYPE_LINE);
1668
1669 ptrs->rd_pgc_workers_searchers = rrddim_add(ptrs->st_pgc_workers, "searchers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1752,7 +1752,7 @@ static void dbengine2_statistics_charts(void) {
1752 "netdata",
1753 "stats",
1754 priority,
1755 - localhost->rrd_update_every,
1755 + rrdb.localhost->update_every,
1756 RRDSET_TYPE_STACKED);
1757
1758 rd_pgc_memory_main = rrddim_add(st_pgc_memory, "main cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1804,7 +1804,7 @@ static void dbengine2_statistics_charts(void) {
1804 "netdata",
1805 "stats",
1806 priority,
1807 - localhost->rrd_update_every,
1807 + rrdb.localhost->update_every,
1808 RRDSET_TYPE_STACKED);
1809
1810 rd_pgc_buffers_pgc = rrddim_add(st_pgc_buffers, "pgc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1863,7 +1863,7 @@ static void dbengine2_statistics_charts(void) {
1863 "netdata",
1864 "stats",
1865 priority,
1866 - localhost->rrd_update_every,
1866 + rrdb.localhost->update_every,
1867 RRDSET_TYPE_AREA);
1868
1869 rd_julyl_moved = rrddim_add(st_julyl_moved, "moved", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1897,7 +1897,7 @@ static void dbengine2_statistics_charts(void) {
1897 "netdata",
1898 "stats",
1899 priority,
1900 - localhost->rrd_update_every,
1900 + rrdb.localhost->update_every,
1901 RRDSET_TYPE_LINE);
1902
1903 rd_mrg_metrics = rrddim_add(st_mrg_metrics, "all", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1937,7 +1937,7 @@ static void dbengine2_statistics_charts(void) {
1937 "netdata",
1938 "stats",
1939 priority,
1940 - localhost->rrd_update_every,
1940 + rrdb.localhost->update_every,
1941 RRDSET_TYPE_LINE);
1942
1943 rd_mrg_add = rrddim_add(st_mrg_ops, "add", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1969,7 +1969,7 @@ static void dbengine2_statistics_charts(void) {
1969 "netdata",
1970 "stats",
1971 priority,
1972 - localhost->rrd_update_every,
1972 + rrdb.localhost->update_every,
1973 RRDSET_TYPE_LINE);
1974
1975 rd_mrg_references = rrddim_add(st_mrg_references, "references", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2000,7 +2000,7 @@ static void dbengine2_statistics_charts(void) {
2000 "netdata",
2001 "stats",
2002 priority,
2003 - localhost->rrd_update_every,
2003 + rrdb.localhost->update_every,
2004 RRDSET_TYPE_LINE);
2005
2006 rd_hit_ratio = rrddim_add(st_cache_hit_ratio, "overall", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
@@ -2077,7 +2077,7 @@ static void dbengine2_statistics_charts(void) {
2077 "netdata",
2078 "stats",
2079 priority,
2080 - localhost->rrd_update_every,
2080 + rrdb.localhost->update_every,
2081 RRDSET_TYPE_LINE);
2082
2083 rd_total = rrddim_add(st_queries, "total", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2113,7 +2113,7 @@ static void dbengine2_statistics_charts(void) {
2113 "netdata",
2114 "stats",
2115 priority,
2116 - localhost->rrd_update_every,
2116 + rrdb.localhost->update_every,
2117 RRDSET_TYPE_LINE);
2118
2119 rd_queries = rrddim_add(st_queries_running, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2143,7 +2143,7 @@ static void dbengine2_statistics_charts(void) {
2143 "netdata",
2144 "stats",
2145 priority,
2146 - localhost->rrd_update_every,
2146 + rrdb.localhost->update_every,
2147 RRDSET_TYPE_STACKED);
2148
2149 rd_cache = rrddim_add(st_query_pages_metadata_source, "cache hit", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2177,7 +2177,7 @@ static void dbengine2_statistics_charts(void) {
2177 "netdata",
2178 "stats",
2179 priority,
2180 - localhost->rrd_update_every,
2180 + rrdb.localhost->update_every,
2181 RRDSET_TYPE_STACKED);
2182
2183 rd_pages_main_cache = rrddim_add(st_query_pages_data_source, "main cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2213,7 +2213,7 @@ static void dbengine2_statistics_charts(void) {
2213 "netdata",
2214 "stats",
2215 priority,
2216 - localhost->rrd_update_every,
2216 + rrdb.localhost->update_every,
2217 RRDSET_TYPE_STACKED);
2218
2219 rd_pass4 = rrddim_add(st_query_next_page, "pass4", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2254,7 +2254,7 @@ static void dbengine2_statistics_charts(void) {
2254 "netdata",
2255 "stats",
2256 priority,
2257 - localhost->rrd_update_every,
2257 + rrdb.localhost->update_every,
2258 RRDSET_TYPE_STACKED);
2259
2260 rd_pages_zero_time = rrddim_add(st_query_page_issues, "zero timestamp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2301,7 +2301,7 @@ static void dbengine2_statistics_charts(void) {
2301 "netdata",
2302 "stats",
2303 priority,
2304 - localhost->rrd_update_every,
2304 + rrdb.localhost->update_every,
2305 RRDSET_TYPE_LINE);
2306
2307 rd_compressed = rrddim_add(st_query_pages_from_disk, "ok compressed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2353,7 +2353,7 @@ static void dbengine2_statistics_charts(void) {
2353 "netdata",
2354 "stats",
2355 priority,
2356 - localhost->rrd_update_every,
2356 + rrdb.localhost->update_every,
2357 RRDSET_TYPE_LINE);
2358
2359 rd_journal_v2_mapped = rrddim_add(st_events, "journal v2 mapped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2397,7 +2397,7 @@ static void dbengine2_statistics_charts(void) {
2397 "netdata",
2398 "stats",
2399 priority,
2400 - localhost->rrd_update_every,
2400 + rrdb.localhost->update_every,
2401 RRDSET_TYPE_STACKED);
2402
2403 rd_routing = rrddim_add(st_prep_timings, "routing", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2438,7 +2438,7 @@ static void dbengine2_statistics_charts(void) {
2438 "netdata",
2439 "stats",
2440 priority,
2441 - localhost->rrd_update_every,
2441 + rrdb.localhost->update_every,
2442 RRDSET_TYPE_STACKED);
2443
2444 rd_init = rrddim_add(st_query_timings, "init", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2460,7 +2460,7 @@ static void dbengine2_statistics_charts(void) {
2460 rrdset_done(st_query_timings);
2461 }
2462
2463 - if(netdata_rwlock_tryrdlock(&rrd_rwlock) == 0) {
2463 + if(rrd_tryrdlock() == 0) {
2464 priority = 135400;
2465
2466 RRDHOST *host;
@@ -2472,8 +2472,8 @@ static void dbengine2_statistics_charts(void) {
2472 if (!rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
2473
2474 /* get localhost's DB engine's statistics for each tier */
2475 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
2476 - if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue;
2475 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
2476 + if(host->db[tier].id != STORAGE_ENGINE_DBENGINE) continue;
2477 if(!host->db[tier].instance) continue;
2478
2479 if(is_storage_engine_shared(host->db[tier].instance)) {
@@ -2484,7 +2484,7 @@ static void dbengine2_statistics_charts(void) {
2484 }
2485
2486 ++dbengine_contexts;
2487 - rrdeng_get_37_statistics((struct rrdengine_instance *)host->db[tier].instance, local_stats_array);
2487 + rrdeng_get_37_statistics(host->db[tier].instance, local_stats_array);
2488 for (i = 0; i < RRDENG_NR_STATS; ++i) {
2489 /* aggregate statistics across hosts */
2490 stats_array[i] += local_stats_array[i];
@@ -2520,7 +2520,7 @@ static void dbengine2_statistics_charts(void) {
2520 "netdata",
2521 "stats",
2522 priority,
2523 - localhost->rrd_update_every,
2523 + rrdb.localhost->update_every,
2524 RRDSET_TYPE_LINE);
2525
2526 rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
@@ -2561,7 +2561,7 @@ static void dbengine2_statistics_charts(void) {
2561 "netdata",
2562 "stats",
2563 priority,
2564 - localhost->rrd_update_every,
2564 + rrdb.localhost->update_every,
2565 RRDSET_TYPE_LINE);
2566
2567 rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -2593,7 +2593,7 @@ static void dbengine2_statistics_charts(void) {
2593 "netdata",
2594 "stats",
2595 priority,
2596 - localhost->rrd_update_every,
2596 + rrdb.localhost->update_every,
2597 RRDSET_TYPE_LINE);
2598
2599 rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2626,7 +2626,7 @@ static void dbengine2_statistics_charts(void) {
2626 "netdata",
2627 "stats",
2628 priority,
2629 - localhost->rrd_update_every,
2629 + rrdb.localhost->update_every,
2630 RRDSET_TYPE_LINE);
2631
2632 rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2661,7 +2661,7 @@ static void dbengine2_statistics_charts(void) {
2661 "netdata",
2662 "stats",
2663 priority,
2664 - localhost->rrd_update_every,
2664 + rrdb.localhost->update_every,
2665 RRDSET_TYPE_LINE);
2666
2667 rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2701,7 +2701,7 @@ static void update_strings_charts() {
2701 , "netdata"
2702 , "stats"
2703 , 910000
2704 - , localhost->rrd_update_every
2704 + , rrdb.localhost->update_every
2705 , RRDSET_TYPE_LINE);
2706
2707 rd_ops_inserts = rrddim_add(st_ops, "inserts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2730,7 +2730,7 @@ static void update_strings_charts() {
2730 , "netdata"
2731 , "stats"
2732 , 910001
2733 - , localhost->rrd_update_every
2733 + , rrdb.localhost->update_every
2734 , RRDSET_TYPE_AREA);
2735
2736 rd_entries_entries = rrddim_add(st_entries, "entries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2753,7 +2753,7 @@ static void update_strings_charts() {
2753 , "netdata"
2754 , "stats"
2755 , 910001
2756 - , localhost->rrd_update_every
2756 + , rrdb.localhost->update_every
2757 , RRDSET_TYPE_AREA);
2758
2759 rd_mem = rrddim_add(st_mem, "memory", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2781,7 +2781,7 @@ static void update_heartbeat_charts() {
2781 , "netdata"
2782 , "stats"
2783 , 900000
2784 - , localhost->rrd_update_every
2784 + , rrdb.localhost->update_every
2785 , RRDSET_TYPE_AREA);
2786
2787 rd_heartbeat_min = rrddim_add(st_heartbeat, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2903,7 +2903,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
2903 , "netdata"
2904 , "stats"
2905 , c->priority + 0
2906 - , localhost->rrd_update_every
2906 + , rrdb.localhost->update_every
2907 , RRDSET_TYPE_LINE
2908 );
2909
@@ -2944,7 +2944,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
2944 , "netdata"
2945 , "stats"
2946 , c->priority + 1
2947 - , localhost->rrd_update_every
2947 + , rrdb.localhost->update_every
2948 , RRDSET_TYPE_LINE
2949 );
2950
@@ -2994,7 +2994,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
2994 , "netdata"
2995 , "stats"
2996 , c->priority + 2
2997 - , localhost->rrd_update_every
2997 + , rrdb.localhost->update_every
2998 , RRDSET_TYPE_LINE
2999 );
3000
@@ -3053,7 +3053,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
3053 , "netdata"
3054 , "stats"
3055 , c->priority + 3
3056 - , localhost->rrd_update_every
3056 + , rrdb.localhost->update_every
3057 , RRDSET_TYPE_LINE
3058 );
3059
@@ -3099,7 +3099,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
3099 , "netdata"
3100 , "stats"
3101 , c->priority + 4
3102 - , localhost->rrd_update_every
3102 + , rrdb.localhost->update_every
3103 , RRDSET_TYPE_STACKED
3104 );
3105
@@ -3144,7 +3144,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
3144 , "netdata"
3145 , "stats"
3146 , c->priority + 5
3147 - , localhost->rrd_update_every
3147 + , rrdb.localhost->update_every
3148 , RRDSET_TYPE_LINE
3149 );
3150
@@ -3239,7 +3239,7 @@ static void malloc_trace_statistics(void) {
3239 , "netdata"
3240 , "stats"
3241 , 900000
3242 - , localhost->rrd_update_every
3242 + , rrdb.localhost->update_every
3243 , RRDSET_TYPE_STACKED
3244 );
3245 }
@@ -3256,7 +3256,7 @@ static void malloc_trace_statistics(void) {
3256 , "netdata"
3257 , "stats"
3258 , 900001
3259 - , localhost->rrd_update_every
3259 + , rrdb.localhost->update_every
3260 , RRDSET_TYPE_LINE
3261 );
3262 }
@@ -3273,7 +3273,7 @@ static void malloc_trace_statistics(void) {
3273 , "netdata"
3274 , "stats"
3275 , 900002
3276 - , localhost->rrd_update_every
3276 + , rrdb.localhost->update_every
3277 , RRDSET_TYPE_STACKED
3278 );
3279 }
@@ -3290,7 +3290,7 @@ static void malloc_trace_statistics(void) {
3290 , "netdata"
3291 , "stats"
3292 , 900003
3293 - , localhost->rrd_update_every
3293 + , rrdb.localhost->update_every
3294 , RRDSET_TYPE_LINE
3295 );
3296 }
@@ -3462,7 +3462,7 @@ static void workers_total_cpu_utilization_chart(void) {
3462 "netdata",
3463 "stats",
3464 999000,
3465 - localhost->rrd_update_every,
3465 + rrdb.localhost->update_every,
3466 RRDSET_TYPE_STACKED);
3467 }
3468
@@ -3510,7 +3510,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3510 , "netdata"
3511 , "stats"
3512 , wu->priority
3513 - , localhost->rrd_update_every
3513 + , rrdb.localhost->update_every
3514 , RRDSET_TYPE_AREA
3515 );
3516 }
@@ -3563,7 +3563,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3563 , "netdata"
3564 , "stats"
3565 , wu->priority + 1
3566 - , localhost->rrd_update_every
3566 + , rrdb.localhost->update_every
3567 , RRDSET_TYPE_AREA
3568 );
3569 }
@@ -3614,7 +3614,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3614 , "netdata"
3615 , "stats"
3616 , wu->priority + 2
3617 - , localhost->rrd_update_every
3617 + , rrdb.localhost->update_every
3618 , RRDSET_TYPE_STACKED
3619 );
3620 }
@@ -3657,7 +3657,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3657 , "netdata"
3658 , "stats"
3659 , wu->priority + 3
3660 - , localhost->rrd_update_every
3660 + , rrdb.localhost->update_every
3661 , RRDSET_TYPE_STACKED
3662 );
3663 }
@@ -3701,7 +3701,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3701 , "netdata"
3702 , "stats"
3703 , wu->priority + 4
3704 - , localhost->rrd_update_every
3704 + , rrdb.localhost->update_every
3705 , RRDSET_TYPE_STACKED
3706 );
3707
@@ -3753,7 +3753,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3753 , "netdata"
3754 , "stats"
3755 , wu->priority + 5 + i
3756 - , localhost->rrd_update_every
3756 + , rrdb.localhost->update_every
3757 , RRDSET_TYPE_LINE
3758 );
3759
@@ -3809,7 +3809,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3809 , "netdata"
3810 , "stats"
3811 , wu->priority + 5 + i
3812 - , localhost->rrd_update_every
3812 + , rrdb.localhost->update_every
3813 , RRDSET_TYPE_LINE
3814 );
3815
@@ -4136,9 +4136,9 @@ void *global_statistics_main(void *ptr)
4136 netdata_thread_cleanup_push(global_statistics_cleanup, ptr);
4137
4138 int update_every =
4139 - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every);
4140 - if (update_every < localhost->rrd_update_every)
4141 - update_every = localhost->rrd_update_every;
4139 + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", rrdb.localhost->update_every);
4140 + if (update_every < rrdb.localhost->update_every)
4141 + update_every = rrdb.localhost->update_every;
4142
4143 usec_t step = update_every * USEC_PER_SEC;
4144 heartbeat_t hb;
@@ -4159,7 +4159,7 @@ void *global_statistics_main(void *ptr)
4159 registry_statistics();
4160
4161 #ifdef ENABLE_DBENGINE
4162 - if(dbengine_enabled) {
4162 + if(rrdb.dbengine_enabled) {
4163 worker_is_busy(WORKER_JOB_DBENGINE);
4164 dbengine2_statistics_charts();
4165 }
@@ -4209,9 +4209,9 @@ void *global_statistics_workers_main(void *ptr)
4209 netdata_thread_cleanup_push(global_statistics_workers_cleanup, ptr);
4210
4211 int update_every =
4212 - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every);
4213 - if (update_every < localhost->rrd_update_every)
4214 - update_every = localhost->rrd_update_every;
4212 + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", rrdb.localhost->update_every);
4213 + if (update_every < rrdb.localhost->update_every)
4214 + update_every = rrdb.localhost->update_every;
4215
4216 usec_t step = update_every * USEC_PER_SEC;
4217 heartbeat_t hb;
@@ -4251,9 +4251,9 @@ void *global_statistics_sqlite3_main(void *ptr)
4251 netdata_thread_cleanup_push(global_statistics_sqlite3_cleanup, ptr);
4252
4253 int update_every =
4254 - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every);
4255 - if (update_every < localhost->rrd_update_every)
4256 - update_every = localhost->rrd_update_every;
4254 + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", rrdb.localhost->update_every);
4255 + if (update_every < rrdb.localhost->update_every)
4256 + update_every = rrdb.localhost->update_every;
4257
4258 usec_t step = update_every * USEC_PER_SEC;
4259 heartbeat_t hb;
daemon/main.c
+233 -97
@@ -8,12 +8,9 @@
8 #warning COMPILING 32BIT NETDATA
9 #endif
10
11 -bool unittest_running = false;
11 int netdata_zero_metrics_enabled;
12 int netdata_anonymous_statistics_enabled;
13
15 -int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
16 -bool ieee754_doubles = false;
14 time_t netdata_start_time = 0;
15 struct netdata_static_thread *static_threads;
16
@@ -327,10 +324,10 @@ void netdata_cleanup_and_exit(int ret) {
324 (void) rename(agent_crash_file, agent_incomplete_shutdown_file);
325
326 #ifdef ENABLE_DBENGINE
330 - if(dbengine_enabled) {
327 + if (rrdb.dbengine_enabled) {
328 delta_shutdown_time("dbengine exit mode");
332 - for (size_t tier = 0; tier < storage_tiers; tier++)
333 - rrdeng_exit_mode(multidb_ctx[tier]);
329 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
330 + rrdeng_exit_mode(rrdb.multidb_ctx[tier]);
331 }
332 #endif
333
@@ -418,10 +415,10 @@ void netdata_cleanup_and_exit(int ret) {
415 // exit cleanly
416
417 #ifdef ENABLE_DBENGINE
421 - if(dbengine_enabled) {
418 + if (rrdb.dbengine_enabled) {
419 delta_shutdown_time("flush dbengine tiers");
423 - for (size_t tier = 0; tier < storage_tiers; tier++)
424 - rrdeng_prepare_exit(multidb_ctx[tier]);
420 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
421 + rrdeng_prepare_exit(rrdb.multidb_ctx[tier]);
422 }
423 #endif
424
@@ -436,14 +433,14 @@ void netdata_cleanup_and_exit(int ret) {
433 metadata_sync_shutdown();
434
435 #ifdef ENABLE_DBENGINE
439 - if(dbengine_enabled) {
436 + if (rrdb.dbengine_enabled) {
437 delta_shutdown_time("wait for dbengine collectors to finish");
438
439 size_t running = 1;
440 while(running) {
441 running = 0;
445 - for (size_t tier = 0; tier < storage_tiers; tier++)
446 - running += rrdeng_collectors_running(multidb_ctx[tier]);
442 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
443 + running += rrdeng_collectors_running(rrdb.multidb_ctx[tier]);
444
445 if(running) {
446 error_limit_static_thread_var(erl, 1, 100 * USEC_PER_MS);
@@ -460,8 +457,8 @@ void netdata_cleanup_and_exit(int ret) {
457 }
458
459 delta_shutdown_time("stop dbengine tiers");
463 - for (size_t tier = 0; tier < storage_tiers; tier++)
464 - rrdeng_exit(multidb_ctx[tier]);
460 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
461 + rrdeng_exit(rrdb.multidb_ctx[tier]);
462 }
463 #endif
464 }
@@ -660,6 +657,26 @@ static void set_nofile_limit(struct rlimit *rl) {
657 netdata_log_error("Number of open file descriptors allowed for this process is too low (RLIMIT_NOFILE=%zu)", (size_t)rl->rlim_cur);
658 }
659
660 +static void set_libuv_worker_threads()
661 +{
662 +#if defined(ENV32BIT)
663 + const int min_libuv_worker_threads = 8;
664 + const int max_libuv_worker_threads = 128;
665 +#else
666 + const int min_libuv_worker_threads = 16;
667 + const int max_libuv_worker_threads = 1024;
668 +#endif
669 +
670 + int n = (int) get_netdata_cpus() * 6;
671 + n = CLAMP(n, min_libuv_worker_threads, max_libuv_worker_threads);
672 +
673 + n = config_get_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", n);
674 + n = MAX(n, min_libuv_worker_threads);
675 + config_set_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", n);
676 +
677 + rrdb.libuv_worker_threads = n;
678 +}
679 +
680 void cancel_main_threads() {
681 error_log_limit_unlimited();
682
@@ -1065,37 +1082,37 @@ static void get_netdata_configured_variables() {
1082 // ------------------------------------------------------------------------
1083 // get default database update frequency
1084
1068 - default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY);
1069 - if(default_rrd_update_every < 1 || default_rrd_update_every > 600) {
1070 - netdata_log_error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY);
1071 - default_rrd_update_every = UPDATE_EVERY;
1072 - config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every);
1085 + rrdb.default_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", rrdb.default_update_every);
1086 + if(rrdb.default_update_every < UPDATE_EVERY_MIN || rrdb.default_update_every > UPDATE_EVERY_MAX) {
1087 + netdata_log_error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", rrdb.default_update_every, UPDATE_EVERY_MIN);
1088 + rrdb.default_update_every = UPDATE_EVERY_MIN;
1089 + config_set_number(CONFIG_SECTION_DB, "update every", rrdb.default_update_every);
1090 }
1091
1092 // ------------------------------------------------------------------------
1076 - // get default memory mode for the database
1093 + // get default storage engine for the database
1094
1095 {
1079 - const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
1080 - default_rrd_memory_mode = rrd_memory_mode_id(mode);
1081 - if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) {
1082 - netdata_log_error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode));
1083 - config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
1096 + const char *se_name = config_get(CONFIG_SECTION_DB, "mode", storage_engine_name(default_storage_engine_id));
1097 + if (!storage_engine_id(se_name, &default_storage_engine_id)) {
1098 + netdata_log_error("Invalid memory mode '%s' given. Using '%s'",
1099 + se_name, storage_engine_name(default_storage_engine_id));
1100 + config_set(CONFIG_SECTION_DB, "mode", storage_engine_name(default_storage_engine_id));
1101 }
1102 }
1103
1104 // ------------------------------------------------------------------------
1105 // get default database size
1106
1090 - if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) {
1091 - default_rrd_history_entries = (int)config_get_number(
1107 + if(default_storage_engine_id != STORAGE_ENGINE_DBENGINE && default_storage_engine_id != STORAGE_ENGINE_NONE) {
1108 + rrdb.default_rrd_history_entries = (int)config_get_number(
1109 CONFIG_SECTION_DB, "retention",
1093 - align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));
1110 + align_entries_to_pagesize(default_storage_engine_id, RRD_DEFAULT_HISTORY_ENTRIES));
1111
1095 - long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries);
1096 - if (h != default_rrd_history_entries) {
1112 + long h = align_entries_to_pagesize(default_storage_engine_id, rrdb.default_rrd_history_entries);
1113 + if (h != rrdb.default_rrd_history_entries) {
1114 config_set_number(CONFIG_SECTION_DB, "retention", h);
1098 - default_rrd_history_entries = (int)h;
1115 + rrdb.default_rrd_history_entries = (int)h;
1116 }
1117 }
1118
@@ -1122,39 +1139,39 @@ static void get_netdata_configured_variables() {
1139 // ------------------------------------------------------------------------
1140 // get default Database Engine page cache size in MiB
1141
1125 - default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
1126 - default_rrdeng_extent_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine extent cache size MB", default_rrdeng_extent_cache_mb);
1127 - db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO);
1142 + rrdb.default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", rrdb.default_rrdeng_page_cache_mb);
1143 + rrdb.default_rrdeng_extent_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine extent cache size MB", rrdb.default_rrdeng_extent_cache_mb);
1144 + rrdb.db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO);
1145
1129 - if(default_rrdeng_extent_cache_mb < 0)
1130 - default_rrdeng_extent_cache_mb = 0;
1146 + if(rrdb.default_rrdeng_extent_cache_mb < 0)
1147 + rrdb.default_rrdeng_extent_cache_mb = 0;
1148
1132 - if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
1133 - netdata_log_error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
1134 - default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
1135 - config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
1149 + if(rrdb.default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
1150 + netdata_log_error("Invalid page cache size %d given. Defaulting to %d.", rrdb.default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
1151 + rrdb.default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
1152 + config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", rrdb.default_rrdeng_page_cache_mb);
1153 }
1154
1155 // ------------------------------------------------------------------------
1156 // get default Database Engine disk space quota in MiB
1157
1141 - default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
1142 - if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1143 - netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
1144 - default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
1145 - config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
1158 + rrdb.default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", rrdb.default_rrdeng_disk_quota_mb);
1159 + if (rrdb.default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1160 + netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", rrdb.default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
1161 + rrdb.default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
1162 + config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", rrdb.default_rrdeng_disk_quota_mb);
1163 }
1164
1148 - default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
1149 - if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1150 - netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
1151 - default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
1152 - config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);
1165 + rrdb.default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
1166 + if(rrdb.default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1167 + netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", rrdb.default_multidb_disk_quota_mb, rrdb.default_rrdeng_disk_quota_mb);
1168 + rrdb.default_multidb_disk_quota_mb = rrdb.default_rrdeng_disk_quota_mb;
1169 + config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", rrdb.default_multidb_disk_quota_mb);
1170 }
1171 #else
1155 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1172 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE) {
1173 error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");
1157 - default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
1174 + default_storage_engine_id = STORAGE_ENGINE_SAVE;
1175 }
1176 #endif
1177 // ------------------------------------------------------------------------
@@ -1179,22 +1196,22 @@ static void get_netdata_configured_variables() {
1196
1197 // --------------------------------------------------------------------
1198
1182 - rrdset_free_obsolete_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);
1199 + rrdb.rrdset_free_obsolete_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdb.rrdset_free_obsolete_time_s);
1200 // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
1201 // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
1202 // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
1186 - if (rrdset_free_obsolete_time_s < 10) {
1187 - rrdset_free_obsolete_time_s = 10;
1203 + if (rrdb.rrdset_free_obsolete_time_s < 10) {
1204 + rrdb.rrdset_free_obsolete_time_s = 10;
1205 netdata_log_info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
1189 - config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);
1206 + config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdb.rrdset_free_obsolete_time_s);
1207 }
1208
1192 - gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
1193 - if (gap_when_lost_iterations_above < 1) {
1194 - gap_when_lost_iterations_above = 1;
1195 - config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
1209 + rrdb.gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", rrdb.gap_when_lost_iterations_above);
1210 + if (rrdb.gap_when_lost_iterations_above < 1) {
1211 + rrdb.gap_when_lost_iterations_above = 1;
1212 + config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", rrdb.gap_when_lost_iterations_above);
1213 }
1197 - gap_when_lost_iterations_above += 2;
1214 + rrdb.gap_when_lost_iterations_above += 2;
1215
1216 // --------------------------------------------------------------------
1217 // get various system parameters
@@ -1305,6 +1322,138 @@ int get_system_info(struct rrdhost_system_info *system_info, bool log) {
1322 return 0;
1323 }
1324
1325 +static void localhost_load_config_labels(void) {
1326 + int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
1327 + if(!status) {
1328 + char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
1329 + netdata_log_error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
1330 + }
1331 +
1332 + struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
1333 + if(co) {
1334 + config_section_wrlock(co);
1335 + struct config_option *cv;
1336 + for(cv = co->values; cv ; cv = cv->next) {
1337 + rrdlabels_add(rrdb.localhost->rrdlabels, cv->name, cv->value, RRDLABEL_SRC_CONFIG);
1338 + cv->flags |= CONFIG_VALUE_USED;
1339 + }
1340 + config_section_unlock(co);
1341 + }
1342 +}
1343 +
1344 +static void localhost_load_kubernetes_labels(void) {
1345 + char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
1346 + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
1347 +
1348 + if (unlikely(access(label_script, R_OK) != 0)) {
1349 + netdata_log_error("Kubernetes pod label fetching script %s not found.",label_script);
1350 + return;
1351 + }
1352 +
1353 + netdata_log_debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
1354 +
1355 + pid_t pid;
1356 + FILE *fp_child_input;
1357 + FILE *fp_child_output = netdata_popen(label_script, &pid, &fp_child_input);
1358 + if(!fp_child_output) return;
1359 +
1360 + char buffer[1000 + 1];
1361 + while (fgets(buffer, 1000, fp_child_output) != NULL)
1362 + rrdlabels_add_pair(rrdb.localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);
1363 +
1364 + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
1365 + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
1366 + int rc = netdata_pclose(fp_child_input, fp_child_output, pid);
1367 + if(rc)
1368 + netdata_log_error("%s exited abnormally. Failed to get kubernetes labels.", label_script);
1369 +}
1370 +
1371 +static void localhost_load_auto_labels(void) {
1372 + DICTIONARY *labels = rrdb.localhost->rrdlabels;
1373 +
1374 + if (rrdb.localhost->system_info->cloud_provider_type)
1375 + rrdlabels_add(labels, "_cloud_provider_type", rrdb.localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
1376 +
1377 + if (rrdb.localhost->system_info->cloud_instance_type)
1378 + rrdlabels_add(labels, "_cloud_instance_type", rrdb.localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
1379 +
1380 + if (rrdb.localhost->system_info->cloud_instance_region)
1381 + rrdlabels_add(labels, "_cloud_instance_region", rrdb.localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
1382 +
1383 + if (rrdb.localhost->system_info->host_os_name)
1384 + rrdlabels_add(labels, "_os_name", rrdb.localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
1385 +
1386 + if (rrdb.localhost->system_info->host_os_version)
1387 + rrdlabels_add(labels, "_os_version", rrdb.localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO);
1388 +
1389 + if (rrdb.localhost->system_info->kernel_version)
1390 + rrdlabels_add(labels, "_kernel_version", rrdb.localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO);
1391 +
1392 + if (rrdb.localhost->system_info->host_cores)
1393 + rrdlabels_add(labels, "_system_cores", rrdb.localhost->system_info->host_cores, RRDLABEL_SRC_AUTO);
1394 +
1395 + if (rrdb.localhost->system_info->host_cpu_freq)
1396 + rrdlabels_add(labels, "_system_cpu_freq", rrdb.localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO);
1397 +
1398 + if (rrdb.localhost->system_info->host_ram_total)
1399 + rrdlabels_add(labels, "_system_ram_total", rrdb.localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO);
1400 +
1401 + if (rrdb.localhost->system_info->host_disk_space)
1402 + rrdlabels_add(labels, "_system_disk_space", rrdb.localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO);
1403 +
1404 + if (rrdb.localhost->system_info->architecture)
1405 + rrdlabels_add(labels, "_architecture", rrdb.localhost->system_info->architecture, RRDLABEL_SRC_AUTO);
1406 +
1407 + if (rrdb.localhost->system_info->virtualization)
1408 + rrdlabels_add(labels, "_virtualization", rrdb.localhost->system_info->virtualization, RRDLABEL_SRC_AUTO);
1409 +
1410 + if (rrdb.localhost->system_info->container)
1411 + rrdlabels_add(labels, "_container", rrdb.localhost->system_info->container, RRDLABEL_SRC_AUTO);
1412 +
1413 + if (rrdb.localhost->system_info->container_detection)
1414 + rrdlabels_add(labels, "_container_detection", rrdb.localhost->system_info->container_detection, RRDLABEL_SRC_AUTO);
1415 +
1416 + if (rrdb.localhost->system_info->virt_detection)
1417 + rrdlabels_add(labels, "_virt_detection", rrdb.localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO);
1418 +
1419 + if (rrdb.localhost->system_info->is_k8s_node)
1420 + rrdlabels_add(labels, "_is_k8s_node", rrdb.localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO);
1421 +
1422 + if (rrdb.localhost->system_info->install_type)
1423 + rrdlabels_add(labels, "_install_type", rrdb.localhost->system_info->install_type, RRDLABEL_SRC_AUTO);
1424 +
1425 + if (rrdb.localhost->system_info->prebuilt_arch)
1426 + rrdlabels_add(labels, "_prebuilt_arch", rrdb.localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO);
1427 +
1428 + if (rrdb.localhost->system_info->prebuilt_dist)
1429 + rrdlabels_add(labels, "_prebuilt_dist", rrdb.localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO);
1430 +
1431 + add_aclk_host_labels();
1432 +
1433 + health_add_host_labels();
1434 +
1435 + rrdlabels_add(labels, "_is_parent", (rrdb.localhost->connected_children_count > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);
1436 +
1437 + if (rrdb.localhost->rrdpush_send_destination)
1438 + rrdlabels_add(labels, "_streams_to", rrdb.localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
1439 +}
1440 +
1441 +void localhost_load_labels(void) {
1442 + if(!rrdb.localhost->rrdlabels)
1443 + rrdb.localhost->rrdlabels = rrdlabels_create();
1444 +
1445 + rrdlabels_unmark_all(rrdb.localhost->rrdlabels);
1446 +
1447 + // priority is important here
1448 + localhost_load_config_labels();
1449 + localhost_load_kubernetes_labels();
1450 + localhost_load_auto_labels();
1451 +
1452 + rrdhost_flag_set(rrdb.localhost, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1453 +
1454 + rrdpush_send_host_labels(rrdb.localhost);
1455 +}
1456 +
1457 void set_silencers_filename() {
1458 char filename[FILENAME_MAX + 1];
1459 snprintfz(filename, FILENAME_MAX, "%s/health.silencers.json", netdata_configured_varlib_dir);
@@ -1451,7 +1600,7 @@ int main(int argc, char **argv) {
1600 }
1601
1602 if(strcmp(optarg, "unittest") == 0) {
1454 - unittest_running = true;
1603 + rrdb.unittest_running = true;
1604
1605 if (pluginsd_parser_unittest())
1606 return 1;
@@ -1469,10 +1618,10 @@ int main(int argc, char **argv) {
1618 // No call to load the config file on this code-path
1619 post_conf_load(&user);
1620 get_netdata_configured_variables();
1472 - default_rrd_update_every = 1;
1473 - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
1621 + rrdb.default_update_every = 1;
1622 + default_storage_engine_id = STORAGE_ENGINE_RAM;
1623 default_health_enabled = 0;
1475 - storage_tiers = 1;
1624 + rrdb.storage_tiers = 1;
1625 registry_init();
1626 if(rrd_init("unittest", NULL, true)) {
1627 fprintf(stderr, "rrd_init failed for unittest\n");
@@ -1501,52 +1650,52 @@ int main(int argc, char **argv) {
1650 return command_argument_sanitization_tests();
1651 }
1652 else if(strcmp(optarg, "dicttest") == 0) {
1504 - unittest_running = true;
1653 + rrdb.unittest_running = true;
1654 return dictionary_unittest(10000);
1655 }
1656 else if(strcmp(optarg, "araltest") == 0) {
1508 - unittest_running = true;
1657 + rrdb.unittest_running = true;
1658 return aral_unittest(10000);
1659 }
1660 else if(strcmp(optarg, "stringtest") == 0) {
1512 - unittest_running = true;
1661 + rrdb.unittest_running = true;
1662 return string_unittest(10000);
1663 }
1664 else if(strcmp(optarg, "rrdlabelstest") == 0) {
1516 - unittest_running = true;
1665 + rrdb.unittest_running = true;
1666 return rrdlabels_unittest();
1667 }
1668 else if(strcmp(optarg, "buffertest") == 0) {
1520 - unittest_running = true;
1669 + rrdb.unittest_running = true;
1670 return buffer_unittest();
1671 }
1672 #ifdef ENABLE_DBENGINE
1673 else if(strcmp(optarg, "mctest") == 0) {
1525 - unittest_running = true;
1674 + rrdb.unittest_running = true;
1675 return mc_unittest();
1676 }
1677 else if(strcmp(optarg, "ctxtest") == 0) {
1529 - unittest_running = true;
1678 + rrdb.unittest_running = true;
1679 return ctx_unittest();
1680 }
1681 else if(strcmp(optarg, "metatest") == 0) {
1533 - unittest_running = true;
1682 + rrdb.unittest_running = true;
1683 return metadata_unittest();
1684 }
1685 else if(strcmp(optarg, "pgctest") == 0) {
1537 - unittest_running = true;
1686 + rrdb.unittest_running = true;
1687 return pgc_unittest();
1688 }
1689 else if(strcmp(optarg, "mrgtest") == 0) {
1541 - unittest_running = true;
1690 + rrdb.unittest_running = true;
1691 return mrg_unittest();
1692 }
1693 else if(strcmp(optarg, "julytest") == 0) {
1545 - unittest_running = true;
1694 + rrdb.unittest_running = true;
1695 return julytest();
1696 }
1697 else if(strcmp(optarg, "parsertest") == 0) {
1549 - unittest_running = true;
1698 + rrdb.unittest_running = true;
1699 return pluginsd_parser_unittest();
1700 }
1701 else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
@@ -1554,7 +1703,7 @@ int main(int argc, char **argv) {
1703 unsigned history_seconds = strtoul(optarg, NULL, 0);
1704 post_conf_load(&user);
1705 get_netdata_configured_variables();
1557 - default_rrd_update_every = 1;
1706 + rrdb.default_update_every = 1;
1707 registry_init();
1708 if(rrd_init("dbengine-dataset", NULL, true)) {
1709 fprintf(stderr, "rrd_init failed for unittest\n");
@@ -1832,25 +1981,12 @@ int main(int argc, char **argv) {
1981 #endif
1982 #endif
1983
1835 - // set libuv worker threads
1836 - libuv_worker_threads = (int)get_netdata_cpus() * 6;
1837 -
1838 - if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS)
1839 - libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
1840 -
1841 - if(libuv_worker_threads > MAX_LIBUV_WORKER_THREADS)
1842 - libuv_worker_threads = MAX_LIBUV_WORKER_THREADS;
1843 -
1844 -
1845 - libuv_worker_threads = config_get_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", libuv_worker_threads);
1846 - if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS) {
1847 - libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
1848 - config_set_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", libuv_worker_threads);
1849 - }
1984 + // decide how many libuv workers to have on 32 vs 64 bit platforms
1985 + set_libuv_worker_threads();
1986
1987 {
1988 char buf[20 + 1];
1853 - snprintfz(buf, 20, "%d", libuv_worker_threads);
1989 + snprintfz(buf, 20, "%d", rrdb.libuv_worker_threads);
1990 setenv("UV_THREADPOOL_SIZE", buf, 1);
1991 }
1992
@@ -1898,7 +2034,7 @@ int main(int argc, char **argv) {
2034 // initialize the log files
2035 open_all_log_files();
2036
1901 - ieee754_doubles = is_system_ieee754_double();
2037 + rrdb.ieee754_doubles = is_system_ieee754_double();
2038
2039 aral_judy_init();
2040
@@ -2072,9 +2208,9 @@ int main(int argc, char **argv) {
2208
2209 error_log_limit_reset();
2210
2075 - // Load host labels
2211 + // Load localhost labels
2212 delta_startup_time("collect host labels");
2077 - reload_host_labels();
2213 + localhost_load_labels();
2214
2215 // ------------------------------------------------------------------------
2216 // spawn the threads
daemon/main.h
+2
@@ -58,4 +58,6 @@ void service_exits(void);
58 bool service_running(SERVICE_TYPE service);
59 struct service_thread *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t request_quit_callback, force_quit_t force_quit_callback, void *data, bool update __maybe_unused);
60
61 +void localhost_load_labels(void);
62 +
63 #endif /* NETDATA_MAIN_H */
daemon/service.c
+11 -11
@@ -45,17 +45,17 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
45 netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
46 }
47
48 - if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
48 + if (st->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
49 rrddimvar_delete_all(rd);
50
51 /* only a collector can mark a chart as obsolete, so we must remove the reference */
52
53 size_t tiers_available = 0, tiers_said_no_retention = 0;
54 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
54 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
55 if(rd->tiers[tier].db_collection_handle) {
56 tiers_available++;
57
58 - if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle))
58 + if(storage_engine_store_finalize(st->storage_engine_id, rd->tiers[tier].db_collection_handle))
59 tiers_said_no_retention++;
60
61 rd->tiers[tier].db_collection_handle = NULL;
@@ -87,7 +87,7 @@ static bool svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensio
87 dfe_start_write(st->rrddim_root_index, rd) {
88 if(unlikely(
89 all_dimensions ||
90 - (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && (rd->collector.last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now))
90 + (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && (rd->collector.last_collected_time.tv_sec + rrdb.rrdset_free_obsolete_time_s < now))
91 )) {
92
93 if(dictionary_acquired_item_references(rd_dfe.item) == 1) {
@@ -121,7 +121,7 @@ static void svc_rrdset_obsolete_to_archive(RRDSET *st) {
121 // has to be run after all dimensions are archived - or use-after-free will occur
122 rrdvar_delete_all(st->rrdvars);
123
124 - if(st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {
124 + if(st->storage_engine_id != STORAGE_ENGINE_DBENGINE) {
125 if(rrdhost_option_check(st->rrdhost, RRDHOST_OPTION_DELETE_OBSOLETE_CHARTS)) {
126 worker_is_busy(WORKER_JOB_DELETE_CHART);
127 rrdset_delete_files(st);
@@ -146,9 +146,9 @@ static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
146 continue;
147
148 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)
149 - && st->last_accessed_time_s + rrdset_free_obsolete_time_s < now
150 - && st->last_updated.tv_sec + rrdset_free_obsolete_time_s < now
151 - && st->last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now
149 + && st->last_accessed_time_s + rrdb.rrdset_free_obsolete_time_s < now
150 + && st->last_updated.tv_sec + rrdb.rrdset_free_obsolete_time_s < now
151 + && st->last_collected_time.tv_sec + rrdb.rrdset_free_obsolete_time_s < now
152 )) {
153 svc_rrdset_obsolete_to_archive(st);
154 }
@@ -196,7 +196,7 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
196 svc_rrdhost_cleanup_obsolete_charts(host);
197 }
198
199 - if(host != localhost
199 + if(host != rrdb.localhost
200 && host->trigger_chart_obsoletion_check
201 && (
202 (
@@ -231,7 +231,7 @@ restart_after_removal:
231
232 if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST)
233 /* don't delete multi-host DB host files */
234 - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
234 + && !(host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
235 ) {
236 worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS);
237 rrdhost_delete_charts(host);
@@ -297,7 +297,7 @@ void *service_main(void *ptr)
297 heartbeat_next(&hb, step);
298
299 svc_rrd_cleanup_obsolete_charts_from_all_hosts();
300 - svc_rrdhost_cleanup_orphan_hosts(localhost);
300 + svc_rrdhost_cleanup_orphan_hosts(rrdb.localhost);
301 }
302
303 netdata_thread_cleanup_pop(1);
daemon/unit_test.c
+31 -31
@@ -1268,8 +1268,8 @@ int run_test(struct test *test)
1268 {
1269 fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description);
1270
1271 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
1272 - default_rrd_update_every = test->update_every;
1271 + default_storage_engine_id = STORAGE_ENGINE_ALLOC;
1272 + rrdb.default_update_every = test->update_every;
1273
1274 char name[101];
1275 snprintfz(name, 100, "unittest-%s", test->name);
@@ -1537,8 +1537,8 @@ int unit_test(long delay, long shift)
1537 snprintfz(name, 100, "unittest-%d-%ld-%ld", repeat, delay, shift);
1538
1539 //debug_flags = 0xffffffff;
1540 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
1541 - default_rrd_update_every = 1;
1540 + default_storage_engine_id = STORAGE_ENGINE_ALLOC;
1541 + rrdb.default_update_every = 1;
1542
1543 int do_abs = 1;
1544 int do_inc = 1;
@@ -1826,11 +1826,11 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number
1826 if(unlikely(v > rd->collector.collected_value_max)) rd->collector.collected_value_max = v;
1827 }
1828
1829 -static RRDHOST *dbengine_rrdhost_find_or_create(char *name)
1829 +static RRDHOST *dbengine_rrdhost_get_or_create(char *name)
1830 {
1831 /* We don't want to drop metrics when generating load, we prefer to block data generation itself */
1832
1833 - return rrdhost_find_or_create(
1833 + return rrdhost_get_or_create(
1834 name
1835 , name
1836 , name
@@ -1841,9 +1841,9 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name)
1841 , ""
1842 , program_name
1843 , program_version
1844 - , default_rrd_update_every
1845 - , default_rrd_history_entries
1846 - , RRD_MEMORY_MODE_DBENGINE
1844 + , rrdb.default_update_every
1845 + , rrdb.default_rrd_history_entries
1846 + , STORAGE_ENGINE_DBENGINE
1847 , default_health_enabled
1848 , default_rrdpush_enabled
1849 , default_rrdpush_destination
@@ -1934,7 +1934,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
1934 // feed it with the test data
1935 for (i = 0 ; i < CHARTS ; ++i) {
1936 for (j = 0 ; j < DIMS ; ++j) {
1937 - storage_engine_store_change_collection_frequency(rd[i][j]->tiers[0].db_collection_handle, update_every);
1937 + storage_engine_store_change_collection_frequency(st[i]->storage_engine_id, rd[i][j]->tiers[0].db_collection_handle, update_every);
1938
1939 rd[i][j]->collector.last_collected_time.tv_sec =
1940 st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
@@ -1985,7 +1985,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1985 time_now = time_start + (c + 1) * update_every;
1986 for (i = 0 ; i < CHARTS ; ++i) {
1987 for (j = 0; j < DIMS; ++j) {
1988 - storage_engine_query_init(rd[i][j]->tiers[0].backend, rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
1988 + storage_engine_query_init(st[i]->storage_engine_id, rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
1989 for (k = 0; k < QUERY_BATCH; ++k) {
1990 last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
1991 j * REGION_POINTS[current_region] + c + k;
@@ -2120,10 +2120,10 @@ int test_dbengine(void)
2120 error_log_limit_unlimited();
2121 fprintf(stderr, "\nRunning DB-engine test\n");
2122
2123 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
2123 + default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
2124
2125 fprintf(stderr, "Initializing localhost with hostname 'unittest-dbengine'");
2126 - host = dbengine_rrdhost_find_or_create("unittest-dbengine");
2126 + host = dbengine_rrdhost_get_or_create("unittest-dbengine");
2127 if (NULL == host)
2128 return 1;
2129
@@ -2234,9 +2234,9 @@ int test_dbengine(void)
2234 }
2235
2236 rrd_wrlock();
2237 - rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
2237 + rrdeng_prepare_exit(host->db[0].instance);
2238 rrdhost_delete_charts(host);
2239 - rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
2239 + rrdeng_exit(host->db[0].instance);
2240 rrd_unlock();
2241
2242 return errors + value_errors + time_errors;
@@ -2336,17 +2336,17 @@ void generate_dbengine_dataset(unsigned history_seconds)
2336 int i;
2337 time_t time_present;
2338
2339 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
2340 - default_rrdeng_page_cache_mb = 128;
2339 + default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
2340 + rrdb.default_rrdeng_page_cache_mb = 128;
2341 // Worst case for uncompressible data
2342 - default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) /
2342 + rrdb.default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) /
2343 (1024 * 1024);
2344 - default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2344 + rrdb.default_rrdeng_disk_quota_mb -= rrdb.default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2345
2346 error_log_limit_unlimited();
2347 fprintf(stderr, "Initializing localhost with hostname 'dbengine-dataset'");
2348
2349 - host = dbengine_rrdhost_find_or_create("dbengine-dataset");
2349 + host = dbengine_rrdhost_get_or_create("dbengine-dataset");
2350 if (NULL == host)
2351 return;
2352
@@ -2381,7 +2381,7 @@ void generate_dbengine_dataset(unsigned history_seconds)
2381 }
2382 freez(thread_info);
2383 rrd_wrlock();
2384 - rrdhost_free___while_having_rrd_wrlock(localhost, true);
2384 + rrdhost_free___while_having_rrd_wrlock(rrdb.localhost, true);
2385 rrd_unlock();
2386 }
2387
@@ -2429,7 +2429,7 @@ static void query_dbengine_chart(void *arg)
2429
2430 if (thread_info->delete_old_data) {
2431 /* A time window of twice the disk space is sufficient for compression space savings of up to 50% */
2432 - time_approx_min = time_max - (default_rrdeng_disk_quota_mb * 2 * 1024 * 1024) /
2432 + time_approx_min = time_max - (rrdb.default_rrdeng_disk_quota_mb * 2 * 1024 * 1024) /
2433 (((uint64_t) DSET_DIMS * DSET_CHARTS) * sizeof(storage_number));
2434 time_min = MAX(time_min, time_approx_min);
2435 }
@@ -2441,7 +2441,7 @@ static void query_dbengine_chart(void *arg)
2441 time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
2442 }
2443
2444 - storage_engine_query_init(rd->tiers[0].backend, rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
2444 + storage_engine_query_init(st->storage_engine_id, rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
2445 ++thread_info->queries_nr;
2446 for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
2447 generatedv = generate_dbengine_chart_value(i, j, time_now);
@@ -2529,23 +2529,23 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2529 if (PAGE_CACHE_MB < RRDENG_MIN_PAGE_CACHE_SIZE_MB)
2530 PAGE_CACHE_MB = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
2531
2532 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
2533 - default_rrdeng_page_cache_mb = PAGE_CACHE_MB;
2532 + default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
2533 + rrdb.default_rrdeng_page_cache_mb = PAGE_CACHE_MB;
2534 if (DISK_SPACE_MB) {
2535 fprintf(stderr, "By setting disk space limit data are allowed to be deleted. "
2536 "Data validation is turned off for this run.\n");
2537 - default_rrdeng_disk_quota_mb = DISK_SPACE_MB;
2537 + rrdb.default_rrdeng_disk_quota_mb = DISK_SPACE_MB;
2538 } else {
2539 // Worst case for uncompressible data
2540 - default_rrdeng_disk_quota_mb =
2540 + rrdb.default_rrdeng_disk_quota_mb =
2541 (((uint64_t) DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * HISTORY_SECONDS) / (1024 * 1024);
2542 - default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2542 + rrdb.default_rrdeng_disk_quota_mb -= rrdb.default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2543 }
2544
2545 fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n");
2546
2547 (void) sql_init_database(DB_CHECK_NONE, 1);
2548 - host = dbengine_rrdhost_find_or_create("dbengine-stress-test");
2548 + host = dbengine_rrdhost_get_or_create("dbengine-stress-test");
2549 if (NULL == host)
2550 return;
2551
@@ -2643,9 +2643,9 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2643 }
2644 freez(query_threads);
2645 rrd_wrlock();
2646 - rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
2646 + rrdeng_prepare_exit(host->db[0].instance);
2647 rrdhost_delete_charts(host);
2648 - rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
2648 + rrdeng_exit(host->db[0].instance);
2649 rrd_unlock();
2650 }
2651
database/contexts/api_v2.c
+14 -15
@@ -802,7 +802,7 @@ static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdc
802 {
803 buffer_json_member_add_string(wb, "status", rrdhost_db_status_to_string(s.db.status));
804 buffer_json_member_add_string(wb, "liveness", rrdhost_db_liveness_to_string(s.db.liveness));
805 - buffer_json_member_add_string(wb, "mode", rrd_memory_mode_name(s.db.mode));
805 + buffer_json_member_add_string(wb, "mode", storage_engine_name(s.db.storage_engine_id));
806 buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s);
807 buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s);
808 buffer_json_member_add_uint64(wb, "metrics", s.db.metrics);
@@ -989,9 +989,9 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
989 else
990 buffer_json_member_add_object(wb, "agent");
991
992 - buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
993 - buffer_json_member_add_uuid(wb, "nd", localhost->node_id);
994 - buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost));
992 + buffer_json_member_add_string(wb, "mg", rrdb.localhost->machine_guid);
993 + buffer_json_member_add_uuid(wb, "nd", rrdb.localhost->node_id);
994 + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(rrdb.localhost));
995 buffer_json_member_add_time_t(wb, "now", now_s);
996
997 if(array)
@@ -1005,14 +1005,13 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
1005 buffer_json_cloud_status(wb, now_s);
1006
1007 buffer_json_member_add_array(wb, "db_size");
1008 - for (size_t tier = 0; tier < storage_tiers; tier++) {
1009 - STORAGE_ENGINE *eng = localhost->db[tier].eng;
1010 - if (!eng) continue;
1008 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
1009 + STORAGE_ENGINE_ID storage_engine_id = rrdb.localhost->db[tier].id;
1010
1012 - size_t max = storage_engine_disk_space_max(eng->backend, localhost->db[tier].instance);
1013 - size_t used = storage_engine_disk_space_used(eng->backend, localhost->db[tier].instance);
1014 - time_t first_time_s = storage_engine_global_first_time_s(eng->backend, localhost->db[tier].instance);
1015 - size_t currently_collected_metrics = storage_engine_collected_metrics(eng->backend, localhost->db[tier].instance);
1011 + size_t max = storage_engine_disk_space_max(storage_engine_id, rrdb.localhost->db[tier].instance);
1012 + size_t used = storage_engine_disk_space_used(storage_engine_id, rrdb.localhost->db[tier].instance);
1013 + time_t first_time_s = storage_engine_global_first_time_s(storage_engine_id, rrdb.localhost->db[tier].instance);
1014 + size_t currently_collected_metrics = storage_engine_collected_metrics(storage_engine_id, rrdb.localhost->db[tier].instance);
1015
1016 NETDATA_DOUBLE percent;
1017 if (used && max)
@@ -1258,7 +1257,7 @@ static void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct s
1257 {
1258 buffer_json_member_add_string(wb, "type", "agent");
1259 buffer_json_member_add_string(wb, "exec", t->notification.exec ? t->notification.exec : NULL);
1261 - buffer_json_member_add_string(wb, "to", t->notification.to_key ? t->notification.to_key : string2str(localhost->health.health_default_recipient));
1260 + buffer_json_member_add_string(wb, "to", t->notification.to_key ? t->notification.to_key : string2str(rrdb.localhost->health.health_default_recipient));
1261 buffer_json_member_add_string(wb, "delay", t->notification.delay);
1262 buffer_json_member_add_string(wb, "repeat", t->notification.repeat);
1263 buffer_json_member_add_string(wb, "options", t->notification.options);
@@ -1575,7 +1574,7 @@ static void contexts_v2_alert_transition_callback(struct sql_alert_transition_da
1574 [ATF_CLASS] = t->classification,
1575 [ATF_TYPE] = t->type,
1576 [ATF_COMPONENT] = t->component,
1578 - [ATF_ROLE] = t->recipient && *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient),
1577 + [ATF_ROLE] = t->recipient && *t->recipient ? t->recipient : string2str(rrdb.localhost->health.health_default_recipient),
1578 [ATF_NODE] = machine_guid,
1579 [ATF_ALERT_NAME] = t->alert_name,
1580 [ATF_CHART_NAME] = t->chart_name,
@@ -1745,9 +1744,9 @@ static void contexts_v2_alert_transitions_to_json(BUFFER *wb, struct rrdcontext_
1744 buffer_json_member_add_time_t(wb, "delay", t->delay);
1745 buffer_json_member_add_time_t(wb, "delay_up_to_time", t->delay_up_to_timestamp);
1746 health_entry_flags_to_json_array(wb, "flags", t->flags);
1748 - buffer_json_member_add_string(wb, "exec", *t->exec ? t->exec : string2str(localhost->health.health_default_exec));
1747 + buffer_json_member_add_string(wb, "exec", *t->exec ? t->exec : string2str(rrdb.localhost->health.health_default_exec));
1748 buffer_json_member_add_uint64(wb, "exec_code", t->exec_code);
1750 - buffer_json_member_add_string(wb, "to", *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient));
1749 + buffer_json_member_add_string(wb, "to", *t->recipient ? t->recipient : string2str(rrdb.localhost->health.health_default_recipient));
1750 }
1751 buffer_json_object_close(wb); // notification
1752 }
database/contexts/internal.h
-1
@@ -8,7 +8,6 @@
8 #include "../../aclk/schema-wrappers/context.h"
9 #include "../../aclk/aclk_contexts_api.h"
10 #include "../../aclk/aclk.h"
11 -#include "../storage_engine.h"
11
12 #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
13 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
database/contexts/query_scope.c
+1 -1
@@ -17,7 +17,7 @@ ssize_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN
17 uint64_t a_hash = 0;
18 uint64_t t_hash = 0;
19
20 - dfe_start_read(rrdhost_root_index, host) {
20 + dfe_start_read(rrdb.rrdhost_root_index, host) {
21 if(host->node_id)
22 uuid_unparse_lower(*host->node_id, host_node_id_str);
23 else
database/contexts/query_target.c
+19 -19
@@ -211,19 +211,19 @@ typedef struct query_target_locals {
211 QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
212 } QUERY_TARGET_LOCALS;
213
214 -struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
214 +STORAGE_ENGINE_ID query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
215 QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
216 - return qn->rrdhost->db[tier].eng;
216 + return qn->rrdhost->db[tier].id;
217 }
218
219 static inline void query_metric_release(QUERY_TARGET *qt, QUERY_METRIC *qm) {
220 qm->plan.used = 0;
221
222 // reset the tiers
223 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
223 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
224 if(qm->tiers[tier].db_metric_handle) {
225 - STORAGE_ENGINE *eng = query_metric_storage_engine(qt, qm, tier);
226 - eng->api.metric_release(qm->tiers[tier].db_metric_handle);
225 + STORAGE_ENGINE_ID storage_engine_id = query_metric_storage_engine(qt, qm, tier);
226 + storage_engine_metric_release(storage_engine_id, qm->tiers[tier].db_metric_handle);
227 qm->tiers[tier].db_metric_handle = NULL;
228 }
229 }
@@ -240,26 +240,26 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
240 size_t tiers_added = 0;
241
242 struct {
243 - STORAGE_ENGINE *eng;
243 + STORAGE_ENGINE_ID storage_engine_id;
244 STORAGE_METRIC_HANDLE *db_metric_handle;
245 time_t db_first_time_s;
246 time_t db_last_time_s;
247 time_t db_update_every_s;
248 - } tier_retention[storage_tiers];
248 + } tier_retention[rrdb.storage_tiers];
249
250 - for (size_t tier = 0; tier < storage_tiers; tier++) {
251 - STORAGE_ENGINE *eng = qn->rrdhost->db[tier].eng;
252 - tier_retention[tier].eng = eng;
250 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
251 + STORAGE_ENGINE_ID storage_engine_id = qn->rrdhost->db[tier].id;
252 + tier_retention[tier].storage_engine_id = storage_engine_id;
253 tier_retention[tier].db_update_every_s = (time_t) (qn->rrdhost->db[tier].tier_grouping * ri->update_every_s);
254
255 if(rm->rrddim && rm->rrddim->tiers[tier].db_metric_handle)
256 - tier_retention[tier].db_metric_handle = eng->api.metric_dup(rm->rrddim->tiers[tier].db_metric_handle);
256 + tier_retention[tier].db_metric_handle = storage_engine_metric_dup(rm->rrddim->rrdset->storage_engine_id, rm->rrddim->tiers[tier].db_metric_handle);
257 else
258 - tier_retention[tier].db_metric_handle = eng->api.metric_get(qn->rrdhost->db[tier].instance, &rm->uuid);
258 + tier_retention[tier].db_metric_handle = storage_engine_metric_get(qn->rrdhost->db[tier].id, qn->rrdhost->db[tier].instance, &rm->uuid);
259
260 if(tier_retention[tier].db_metric_handle) {
261 - tier_retention[tier].db_first_time_s = storage_engine_oldest_time_s(tier_retention[tier].eng->backend, tier_retention[tier].db_metric_handle);
262 - tier_retention[tier].db_last_time_s = storage_engine_latest_time_s(tier_retention[tier].eng->backend, tier_retention[tier].db_metric_handle);
261 + tier_retention[tier].db_first_time_s = storage_engine_oldest_time_s(tier_retention[tier].storage_engine_id, tier_retention[tier].db_metric_handle);
262 + tier_retention[tier].db_last_time_s = storage_engine_latest_time_s(tier_retention[tier].storage_engine_id, tier_retention[tier].db_metric_handle);
263
264 if(!common_first_time_s)
265 common_first_time_s = tier_retention[tier].db_first_time_s;
@@ -285,7 +285,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
285 }
286 }
287
288 - for (size_t tier = 0; tier < storage_tiers; tier++) {
288 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
289 if(!qt->db.tiers[tier].update_every || (tier_retention[tier].db_update_every_s && tier_retention[tier].db_update_every_s < qt->db.tiers[tier].update_every))
290 qt->db.tiers[tier].update_every = tier_retention[tier].db_update_every_s;
291
@@ -329,8 +329,8 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
329 if (!qt->db.last_time_s || common_last_time_s > qt->db.last_time_s)
330 qt->db.last_time_s = common_last_time_s;
331
332 - for (size_t tier = 0; tier < storage_tiers; tier++) {
333 - internal_fatal(tier_retention[tier].eng != query_metric_storage_engine(qt, qm, tier), "QUERY TARGET: storage engine mismatch");
332 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
333 + internal_fatal(tier_retention[tier].storage_engine_id != query_metric_storage_engine(qt, qm, tier), "QUERY TARGET: storage engine mismatch");
334 qm->tiers[tier].db_metric_handle = tier_retention[tier].db_metric_handle;
335 qm->tiers[tier].db_first_time_s = tier_retention[tier].db_first_time_s;
336 qm->tiers[tier].db_last_time_s = tier_retention[tier].db_last_time_s;
@@ -341,9 +341,9 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
341 }
342
343 // cleanup anything we allocated to the retention we will not use
344 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
344 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
345 if (tier_retention[tier].db_metric_handle)
346 - tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
346 + storage_engine_metric_release(tier_retention[tier].storage_engine_id, tier_retention[tier].db_metric_handle);
347 }
348
349 return false;
database/contexts/rrdcontext.c
+4 -21
@@ -199,25 +199,8 @@ int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char
199 // ACLK interface
200
201 static bool rrdhost_check_our_claim_id(const char *claim_id) {
202 - if(!localhost->aclk_state.claimed_id) return false;
203 - return (strcasecmp(claim_id, localhost->aclk_state.claimed_id) == 0) ? true : false;
204 -}
205 -
206 -static RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
207 - uuid_t uuid;
208 - if (uuid_parse(node_id, uuid))
209 - return NULL;
210 -
211 - RRDHOST *host = NULL;
212 - dfe_start_read(rrdhost_root_index, host) {
213 - if(!host->node_id) continue;
214 -
215 - if(uuid_memcmp(&uuid, host->node_id) == 0)
216 - break;
217 - }
218 - dfe_done(host);
219 -
220 - return host;
202 + if(!rrdb.localhost->aclk_state.claimed_id) return false;
203 + return (strcasecmp(claim_id, rrdb.localhost->aclk_state.claimed_id) == 0) ? true : false;
204 }
205
206 void rrdcontext_hub_checkpoint_command(void *ptr) {
@@ -226,7 +209,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) {
209 if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
210 netdata_log_error("RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
211 cmd->claim_id, cmd->node_id,
229 - localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
212 + rrdb.localhost->aclk_state.claimed_id ? rrdb.localhost->aclk_state.claimed_id : "NOT SET",
213 cmd->claim_id);
214
215 return;
@@ -288,7 +271,7 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) {
271 if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
272 netdata_log_error("RRDCONTEXT: received stop streaming command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
273 cmd->claim_id, cmd->node_id,
291 - localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
274 + rrdb.localhost->aclk_state.claimed_id ? rrdb.localhost->aclk_state.claimed_id : "NOT SET",
275 cmd->claim_id);
276
277 return;
database/contexts/rrdcontext.h
+1 -1
@@ -577,7 +577,7 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
577 return rrdmetric_acquired_name(qd->rma);
578 }
579
580 -struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
580 +STORAGE_ENGINE_ID query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
581
582 STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version);
583 STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version);
database/contexts/worker.c
+5 -7
@@ -216,7 +216,7 @@ void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool
216 static void rrdcontext_recalculate_retention_all_hosts(void) {
217 rrdcontext_next_db_rotation_ut = 0;
218 RRDHOST *host;
219 - dfe_start_reentrant(rrdhost_root_index, host) {
219 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
220 worker_is_busy(WORKER_JOB_RETENTION);
221 rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DB_ROTATION, true);
222 }
@@ -235,11 +235,9 @@ bool rrdmetric_update_retention(RRDMETRIC *rm) {
235 }
236 else {
237 RRDHOST *rrdhost = rm->ri->rc->rrdhost;
238 - for (size_t tier = 0; tier < storage_tiers; tier++) {
239 - STORAGE_ENGINE *eng = rrdhost->db[tier].eng;
240 -
238 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
239 time_t first_time_t, last_time_t;
242 - if (eng->api.metric_retention_by_uuid(rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t)) {
240 + if (storage_engine_metric_retention(rrdhost->db[tier].id, rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t)) {
241 if (first_time_t < min_first_time_t)
242 min_first_time_t = first_time_t;
243
@@ -436,7 +434,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo
434
435 static void rrdcontext_garbage_collect_for_all_hosts(void) {
436 RRDHOST *host;
439 - dfe_start_reentrant(rrdhost_root_index, host) {
437 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
438 rrdcontext_garbage_collect_single_host(host, true);
439 }
440 dfe_done(host);
@@ -1104,7 +1102,7 @@ void *rrdcontext_main(void *ptr) {
1102 size_t pp_queued_contexts_for_all_hosts = 0;
1103
1104 RRDHOST *host;
1107 - dfe_start_reentrant(rrdhost_root_index, host) {
1105 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1106 if(unlikely(!service_running(SERVICE_CONTEXT))) break;
1107
1108 worker_is_busy(WORKER_JOB_HOSTS);
database/engine/cache.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "cache.h"
4
5 /* STATES AND TRANSITIONS
database/engine/cache.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef DBENGINE_CACHE_H
4 #define DBENGINE_CACHE_H
5
database/engine/datafile.c
+2 -2
@@ -250,7 +250,7 @@ int create_data_file(struct rrdengine_datafile *datafile)
250 char path[RRDENG_PATH_MAX];
251
252 generate_datafilepath(datafile, path, sizeof(path));
253 - fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, use_direct_io);
253 + fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, rrdb.use_direct_io);
254 if (fd < 0) {
255 ctx_fs_error(ctx);
256 return fd;
@@ -333,7 +333,7 @@ static int load_data_file(struct rrdengine_datafile *datafile)
333 char path[RRDENG_PATH_MAX];
334
335 generate_datafilepath(datafile, path, sizeof(path));
336 - fd = open_file_for_io(path, O_RDWR, &file, use_direct_io);
336 + fd = open_file_for_io(path, O_RDWR, &file, rrdb.use_direct_io);
337 if (fd < 0) {
338 ctx_fs_error(ctx);
339 return fd;
database/engine/journalfile.c
+6 -6
@@ -296,8 +296,8 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo
296 void journalfile_v2_data_unmount_cleanup(time_t now_s) {
297 // DO NOT WAIT ON ANY LOCK!!!
298
299 - for(size_t tier = 0; tier < (size_t)storage_tiers ;tier++) {
300 - struct rrdengine_instance *ctx = multidb_ctx[tier];
299 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
300 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) rrdb.multidb_ctx[tier];
301 if(!ctx) continue;
302
303 struct rrdengine_datafile *datafile;
@@ -577,7 +577,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi
577 char path[RRDENG_PATH_MAX];
578
579 journalfile_v1_generate_path(datafile, path, sizeof(path));
580 - fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, use_direct_io);
580 + fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, rrdb.use_direct_io);
581 if (fd < 0) {
582 ctx_fs_error(ctx);
583 return fd;
@@ -917,7 +917,7 @@ static int journalfile_v2_validate(void *data_start, size_t journal_v2_file_size
917 rc = journalfile_check_v2_extent_list(data_start, journal_v2_file_size);
918 if (rc) return 1;
919
920 - if (!db_engine_journal_check)
920 + if (!rrdb.db_engine_journal_check)
921 return 0;
922
923 rc = journalfile_check_v2_metric_list(data_start, journal_v2_file_size);
@@ -1115,7 +1115,7 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal
1115
1116 // Initialize the journal file to be able to access the data
1117
1118 - if (!db_engine_journal_check)
1118 + if (!rrdb.db_engine_journal_check)
1119 journalfile->v2.flags |= JOURNALFILE_FLAG_METRIC_CRC_CHECK;
1120 journalfile_v2_data_set(journalfile, fd, data_start, journal_v2_file_size);
1121
@@ -1501,7 +1501,7 @@ int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfil
1501
1502 journalfile_v1_generate_path(datafile, path, sizeof(path));
1503
1504 - fd = open_file_for_io(path, O_RDWR, &file, use_direct_io);
1504 + fd = open_file_for_io(path, O_RDWR, &file, rrdb.use_direct_io);
1505 if (fd < 0) {
1506 ctx_fs_error(ctx);
1507
database/engine/metric.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "metric.h"
4
5 typedef int32_t REFCOUNT;
database/engine/metric.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef DBENGINE_METRIC_H
4 #define DBENGINE_METRIC_H
5
database/engine/pagecache.c
+4 -5
@@ -1,5 +1,4 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
2
3 #include "rrdengine.h"
4
@@ -232,7 +231,7 @@ static size_t get_page_list_from_pgc(PGC *cache, METRIC *metric, struct rrdengin
231 time_t dt_s = mrg_metric_get_update_every_s(main_mrg, metric);
232
233 if(!dt_s)
235 - dt_s = default_rrd_update_every;
234 + dt_s = rrdb.default_update_every;
235
236 time_t previous_page_end_time_s = now_s - dt_s;
237 bool first = true;
@@ -423,7 +422,7 @@ static size_t list_has_time_gaps(
422 time_t now_s = wanted_start_time_s;
423 time_t dt_s = mrg_metric_get_update_every_s(main_mrg, metric);
424 if(!dt_s)
426 - dt_s = default_rrd_update_every;
425 + dt_s = rrdb.default_update_every;
426
427 size_t pages_pass2 = 0, pages_pass3 = 0;
428 while((pd = pdc_find_page_for_time(
@@ -1085,7 +1084,7 @@ void pgc_and_mrg_initialize(void)
1084 {
1085 main_mrg = mrg_create(0);
1086
1088 - size_t target_cache_size = (size_t)default_rrdeng_page_cache_mb * 1024ULL * 1024ULL;
1087 + size_t target_cache_size = (size_t) rrdb.default_rrdeng_page_cache_mb * 1024ULL * 1024ULL;
1088 size_t main_cache_size = (target_cache_size / 100) * 95;
1089 size_t open_cache_size = 0;
1090 size_t extent_cache_size = (target_cache_size / 100) * 5;
@@ -1095,7 +1094,7 @@ void pgc_and_mrg_initialize(void)
1094 main_cache_size = target_cache_size - extent_cache_size;
1095 }
1096
1098 - extent_cache_size += (size_t)(default_rrdeng_extent_cache_mb * 1024ULL * 1024ULL);
1097 + extent_cache_size += (size_t) (rrdb.default_rrdeng_extent_cache_mb * 1024ULL * 1024ULL);
1098
1099 main_cache = pgc_create(
1100 "main_cache",
database/engine/pdc.c
+2 -2
@@ -1,5 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
2 +
3 #include "pdc.h"
4
5 struct extent_page_details_list {
@@ -667,7 +667,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
667 };
668
669 // always calculate entries by size
670 - vd.point_size = page_type_size[vd.type];
670 + vd.point_size = rrdb.page_type_size[vd.type];
671 vd.entries = page_entries_by_size(vd.page_length, vd.point_size);
672
673 // allow to be called without entries (when loading pages from disk)
database/engine/rrddim_eng.h new
+73
@@ -0,0 +1,73 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDDIM_ENG_H
4 +#define NETDATA_RRDDIM_ENG_H
5 +
6 +#include "database/storage_engine_types.h"
7 +
8 +STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
9 +
10 +void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
11 +
12 +STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
13 +void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value, uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
14 +void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
15 +void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
16 +
17 +size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
18 +
19 +size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
20 +
21 +STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
22 +STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
23 +void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
24 +STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);
25 +
26 +
27 +void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle, time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
28 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
29 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
30 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
31 +time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
32 +
33 +time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
34 +time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
35 +
36 +
37 +bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s);
38 +
39 +time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *db_instance);
40 +
41 +int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
42 +
43 +size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
44 +
45 +time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
46 +time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
47 +
48 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
49 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
50 +time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
51 +
52 +// The following functions are dbengine-specific. Include them here until
53 +// we refactor the code so that, either we don't need them, or settle to a
54 +// common API.
55 +
56 +bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance);
57 +
58 +void rrdeng_get_37_statistics(STORAGE_INSTANCE *db_instance, unsigned long long *array);
59 +
60 +int rrdeng_init(STORAGE_INSTANCE **db_instance, const char *dbfiles_path,
61 + unsigned disk_space_mb, size_t tier);
62 +
63 +void rrdeng_readiness_wait(STORAGE_INSTANCE *db_instance);
64 +void rrdeng_exit_mode(STORAGE_INSTANCE *db_instance);
65 +
66 +int rrdeng_exit(STORAGE_INSTANCE *ctx);
67 +void rrdeng_prepare_exit(STORAGE_INSTANCE *ctx);
68 +
69 +size_t rrdeng_collectors_running(STORAGE_INSTANCE *db_instance);
70 +
71 +void rrdeng_size_statistics(STORAGE_INSTANCE *db_instance, BUFFER *wb);
72 +
73 +#endif /* NETDATA_RRDDIM_ENG_H */
database/engine/rrdengine.c
+14 -11
@@ -1,5 +1,4 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
2
3 #include "rrdengine.h"
4 #include "pdc.h"
@@ -157,11 +156,15 @@ enum LIBUV_WORKERS_STATUS {
156
157 static inline enum LIBUV_WORKERS_STATUS work_request_full(void) {
158 size_t dispatched = __atomic_load_n(&rrdeng_main.work_cmd.atomics.dispatched, __ATOMIC_RELAXED);
159 + int reserved_libuv_worker_threads = 3;
160
161 - if(dispatched >= (size_t)(libuv_worker_threads))
161 + internal_fatal(rrdb.libuv_worker_threads <= reserved_libuv_worker_threads,
162 + "libuv worker threads below minimum required");
163 +
164 + if(dispatched >= (size_t)(rrdb.libuv_worker_threads))
165 return LIBUV_WORKERS_CRITICAL;
166
164 - else if(dispatched >= (size_t)(libuv_worker_threads - RESERVED_LIBUV_WORKER_THREADS))
167 + else if(dispatched >= (size_t)(rrdb.libuv_worker_threads - reserved_libuv_worker_threads))
168 return LIBUV_WORKERS_STRESSED;
169
170 return LIBUV_WORKERS_RELAXED;
@@ -354,7 +357,7 @@ static void wal_cleanup1(void) {
357 if(!spinlock_trylock(&wal_globals.protected.spinlock))
358 return;
359
357 - if(wal_globals.protected.available_items && wal_globals.protected.available > storage_tiers) {
360 + if (wal_globals.protected.available_items && wal_globals.protected.available > rrdb.storage_tiers) {
361 wal = wal_globals.protected.available_items;
362 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
363 wal_globals.protected.available--;
@@ -582,25 +585,25 @@ struct {
585 } dbengine_page_alloc_globals = {};
586
587 static inline ARAL *page_size_lookup(size_t size) {
585 - for(size_t tier = 0; tier < storage_tiers ;tier++)
586 - if(size == tier_page_size[tier])
588 + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
589 + if(size == rrdb.tier_page_size[tier])
590 return dbengine_page_alloc_globals.aral[tier];
591
592 return NULL;
593 }
594
595 static void dbengine_page_alloc_init(void) {
593 - for(size_t i = storage_tiers; i > 0 ;i--) {
594 - size_t tier = storage_tiers - i;
596 + for (size_t i = rrdb.storage_tiers; i > 0 ;i--) {
597 + size_t tier = rrdb.storage_tiers - i;
598
599 char buf[20 + 1];
600 snprintfz(buf, 20, "tier%zu-pages", tier);
601
602 dbengine_page_alloc_globals.aral[tier] = aral_create(
603 buf,
601 - tier_page_size[tier],
604 + rrdb.tier_page_size[tier],
605 64,
603 - 512 * tier_page_size[tier],
606 + 512 * rrdb.tier_page_size[tier],
607 pgc_aral_statistics(),
608 NULL, NULL, false, false);
609 }
@@ -1784,7 +1787,7 @@ void dbengine_event_loop(void* arg) {
1787 }
1788
1789 case RRDENG_OPCODE_FLUSH_INIT: {
1787 - if(rrdeng_main.flushes_running < (size_t)(libuv_worker_threads / 4)) {
1790 + if(rrdeng_main.flushes_running < (size_t)(rrdb.libuv_worker_threads / 4)) {
1791 rrdeng_main.flushes_running++;
1792 work_dispatch(NULL, NULL, NULL, opcode, cache_flush_tp_worker, after_do_cache_flush);
1793 }
database/engine/rrdengine.h
-2
@@ -184,8 +184,6 @@ typedef enum __attribute__ ((__packed__)) {
184 } RRDENG_COLLECT_PAGE_FLAGS;
185
186 struct rrdeng_collect_handle {
187 - struct storage_collect_handle common; // has to be first item
188 -
187 RRDENG_COLLECT_PAGE_FLAGS page_flags;
188 RRDENG_COLLECT_HANDLE_OPTIONS options;
189 uint8_t type;
database/engine/rrdengineapi.c
+104 -51
@@ -13,40 +13,20 @@ struct rrdengine_instance multidb_ctx_storage_tier4;
13 #if RRD_STORAGE_TIERS != 5
14 #error RRD_STORAGE_TIERS is not 5 - you need to add allocations here
15 #endif
16 -struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
16 uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER};
17
19 -#if defined(ENV32BIT)
20 -size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};
21 -#else
22 -size_t tier_page_size[RRD_STORAGE_TIERS] = {4096, 2048, 384, 384, 384};
23 -#endif
24 -
18 #if PAGE_TYPE_MAX != 1
19 #error PAGE_TYPE_MAX is not 1 - you need to add allocations here
20 #endif
28 -size_t page_type_size[256] = {sizeof(storage_number), sizeof(storage_number_tier1_t)};
21
22 __attribute__((constructor)) void initialize_multidb_ctx(void) {
31 - multidb_ctx[0] = &multidb_ctx_storage_tier0;
32 - multidb_ctx[1] = &multidb_ctx_storage_tier1;
33 - multidb_ctx[2] = &multidb_ctx_storage_tier2;
34 - multidb_ctx[3] = &multidb_ctx_storage_tier3;
35 - multidb_ctx[4] = &multidb_ctx_storage_tier4;
23 + rrdb.multidb_ctx[0] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier0;
24 + rrdb.multidb_ctx[1] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier1;
25 + rrdb.multidb_ctx[2] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier2;
26 + rrdb.multidb_ctx[3] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier3;
27 + rrdb.multidb_ctx[4] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier4;
28 }
29
38 -int db_engine_journal_check = 0;
39 -int default_rrdeng_disk_quota_mb = 256;
40 -int default_multidb_disk_quota_mb = 256;
41 -
42 -#if defined(ENV32BIT)
43 -int default_rrdeng_page_cache_mb = 16;
44 -int default_rrdeng_extent_cache_mb = 0;
45 -#else
46 -int default_rrdeng_page_cache_mb = 32;
47 -int default_rrdeng_extent_cache_mb = 0;
48 -#endif
49 -
30 // ----------------------------------------------------------------------------
31 // metrics groups
32
@@ -85,7 +65,7 @@ void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance __maybe_unused,
65 // metric handle for legacy dbs
66
67 /* This UUID is not unique across hosts */
88 -void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
68 +static void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
69 {
70 EVP_MD_CTX *evpctx;
71 unsigned char hash_value[EVP_MAX_MD_SIZE];
@@ -255,7 +235,6 @@ STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metri
235 struct rrdeng_collect_handle *handle;
236
237 handle = callocz(1, sizeof(struct rrdeng_collect_handle));
258 - handle->common.backend = STORAGE_ENGINE_BACKEND_DBENGINE;
238 handle->metric = metric;
239 handle->page = NULL;
240 handle->data = NULL;
@@ -444,7 +423,7 @@ static size_t aligned_allocation_entries(size_t max_slots, size_t target_slot, t
423 static void *rrdeng_alloc_new_metric_data(struct rrdeng_collect_handle *handle, size_t *data_size, usec_t point_in_time_ut) {
424 struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
425
447 - size_t max_size = tier_page_size[ctx->config.tier];
426 + size_t max_size = rrdb.tier_page_size[ctx->config.tier];
427 size_t max_slots = max_size / CTX_POINT_SIZE_BYTES(ctx);
428
429 size_t slots = aligned_allocation_entries(
@@ -464,7 +443,7 @@ static void *rrdeng_alloc_new_metric_data(struct rrdeng_collect_handle *handle,
443 // internal_error(true, "PAGE ALLOC %zu bytes (%zu max)", size, max_size);
444
445 internal_fatal(slots < 3 || slots > max_slots, "ooops! wrong distribution of metrics across time");
467 - internal_fatal(size > tier_page_size[ctx->config.tier] || size < CTX_POINT_SIZE_BYTES(ctx) * 2, "ooops! wrong page size");
446 + internal_fatal(size > rrdb.tier_page_size[ctx->config.tier] || size < CTX_POINT_SIZE_BYTES(ctx) * 2, "ooops! wrong page size");
447
448 *data_size = size;
449 void *d = dbengine_page_alloc(size);
@@ -767,15 +746,14 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle,
746
747 handle->dt_s = db_update_every_s;
748 if (!handle->dt_s) {
770 - handle->dt_s = default_rrd_update_every;
771 - mrg_metric_set_update_every_s_if_zero(main_mrg, metric, default_rrd_update_every);
749 + handle->dt_s = rrdb.default_update_every;
750 + mrg_metric_set_update_every_s_if_zero(main_mrg, metric, rrdb.default_update_every);
751 }
752
753 rrddim_handle->handle = (STORAGE_QUERY_HANDLE *) handle;
754 rrddim_handle->start_time_s = handle->start_time_s;
755 rrddim_handle->end_time_s = handle->end_time_s;
756 rrddim_handle->priority = priority;
778 - rrddim_handle->backend = STORAGE_ENGINE_BACKEND_DBENGINE;
757
758 pg_cache_preload(handle);
759
@@ -791,7 +769,6 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle,
769 rrddim_handle->start_time_s = handle->start_time_s;
770 rrddim_handle->end_time_s = 0;
771 rrddim_handle->priority = priority;
794 - rrddim_handle->backend = STORAGE_ENGINE_BACKEND_DBENGINE;
772 }
773 }
774
@@ -1028,8 +1005,10 @@ size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance) {
1005 * You must not change the indices of the statistics or user code will break.
1006 * You must not exceed RRDENG_NR_STATS or it will crash.
1007 */
1031 -void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
1008 +void rrdeng_get_37_statistics(STORAGE_INSTANCE *db_instance, unsigned long long *array)
1009 {
1010 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1011 +
1012 if (ctx == NULL)
1013 return;
1014
@@ -1082,12 +1061,12 @@ static void rrdeng_populate_mrg(struct rrdengine_instance *ctx) {
1061 datafiles++;
1062 uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
1063
1085 - ssize_t cpus = (ssize_t)get_netdata_cpus() / (ssize_t)storage_tiers;
1064 + ssize_t cpus = (ssize_t) get_netdata_cpus() / rrdb.storage_tiers;
1065 if(cpus > (ssize_t)datafiles)
1066 cpus = (ssize_t)datafiles;
1067
1089 - if(cpus > (ssize_t)libuv_worker_threads)
1090 - cpus = (ssize_t)libuv_worker_threads;
1068 + if(cpus > (ssize_t) rrdb.libuv_worker_threads)
1069 + cpus = (ssize_t) rrdb.libuv_worker_threads;
1070
1071 if(cpus >= (ssize_t)get_netdata_cpus() / 2)
1072 cpus = (ssize_t)(get_netdata_cpus() / 2 - 1);
@@ -1126,7 +1105,9 @@ static void rrdeng_populate_mrg(struct rrdengine_instance *ctx) {
1105 }
1106 }
1107
1129 -void rrdeng_readiness_wait(struct rrdengine_instance *ctx) {
1108 +void rrdeng_readiness_wait(STORAGE_INSTANCE *db_instance) {
1109 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1110 +
1111 for (size_t i = 0; i < ctx->loading.populate_mrg.size; i++) {
1112 completion_wait_for(&ctx->loading.populate_mrg.array[i]);
1113 completion_destroy(&ctx->loading.populate_mrg.array[i]);
@@ -1144,14 +1125,17 @@ bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance) {
1125 return ctx->config.legacy;
1126 }
1127
1147 -void rrdeng_exit_mode(struct rrdengine_instance *ctx) {
1128 +void rrdeng_exit_mode(STORAGE_INSTANCE *db_instance) {
1129 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1130 __atomic_store_n(&ctx->quiesce.exit_mode, true, __ATOMIC_RELAXED);
1131 }
1132 /*
1133 * Returns 0 on success, negative on error
1134 */
1153 -int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
1135 +int rrdeng_init(STORAGE_INSTANCE **db_instance_ptr, const char *dbfiles_path,
1136 unsigned disk_space_mb, size_t tier) {
1137 + struct rrdengine_instance **ctxp = (struct rrdengine_instance **) db_instance_ptr;
1138 +
1139 struct rrdengine_instance *ctx;
1140 uint32_t max_open_files;
1141
@@ -1171,7 +1155,7 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
1155 }
1156
1157 if(NULL == ctxp) {
1174 - ctx = multidb_ctx[tier];
1158 + ctx = (struct rrdengine_instance *) rrdb.multidb_ctx[tier];
1159 memset(ctx, 0, sizeof(*ctx));
1160 ctx->config.legacy = false;
1161 }
@@ -1211,15 +1195,18 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
1195 return UV_EIO;
1196 }
1197
1214 -size_t rrdeng_collectors_running(struct rrdengine_instance *ctx) {
1198 +size_t rrdeng_collectors_running(STORAGE_INSTANCE *db_instance) {
1199 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1200 return __atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED);
1201 }
1202
1203 /*
1204 * Returns 0 on success, 1 on error
1205 */
1221 -int rrdeng_exit(struct rrdengine_instance *ctx) {
1222 - if (NULL == ctx)
1206 +int rrdeng_exit(STORAGE_INSTANCE *db_instance) {
1207 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1208 +
1209 + if (ctx == NULL)
1210 return 1;
1211
1212 // FIXME - ktsaou - properly cleanup ctx
@@ -1229,7 +1216,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx) {
1216 // 4. then wait for completion
1217
1218 bool logged = false;
1232 - while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && !unittest_running) {
1219 + while (__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && !rrdb.unittest_running) {
1220 if(!logged) {
1221 netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", (ctx->config.legacy) ? -1 : ctx->config.tier);
1222 logged = true;
@@ -1256,7 +1243,9 @@ int rrdeng_exit(struct rrdengine_instance *ctx) {
1243 return 0;
1244 }
1245
1259 -void rrdeng_prepare_exit(struct rrdengine_instance *ctx) {
1246 +void rrdeng_prepare_exit(STORAGE_INSTANCE *db_instance) {
1247 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1248 +
1249 if (NULL == ctx)
1250 return;
1251
@@ -1306,7 +1295,7 @@ static void populate_v2_statistics(struct rrdengine_datafile *datafile, RRDENG_S
1295 if(likely(points > 1))
1296 update_every_s = (time_t) ((end_time_s - start_time_s) / (points - 1));
1297 else {
1309 - update_every_s = (time_t) (default_rrd_update_every * get_tier_grouping(datafile->ctx->config.tier));
1298 + update_every_s = (time_t) (rrdb.default_update_every * get_tier_grouping(datafile->ctx->config.tier));
1299 stats->single_point_pages++;
1300 }
1301
@@ -1335,7 +1324,7 @@ static void populate_v2_statistics(struct rrdengine_datafile *datafile, RRDENG_S
1324 journalfile_v2_data_release(datafile->journalfile);
1325 }
1326
1338 -RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
1327 +static RRDENG_SIZE_STATS rrdeng_size_statistics_internal(struct rrdengine_instance *ctx) {
1328 RRDENG_SIZE_STATS stats = { 0 };
1329
1330 uv_rwlock_rdlock(&ctx->datafiles.rwlock);
@@ -1381,18 +1370,82 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
1370 // stats.sizeof_metric = 0;
1371 stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));
1372 stats.sizeof_page_in_cache = 0; // struct_natural_alignment(sizeof(struct page_cache_descr));
1384 - stats.sizeof_point_data = page_type_size[ctx->config.page_type];
1385 - stats.sizeof_page_data = tier_page_size[ctx->config.tier];
1373 + stats.sizeof_point_data = rrdb.page_type_size[ctx->config.page_type];
1374 + stats.sizeof_page_data = rrdb.tier_page_size[ctx->config.tier];
1375 stats.pages_per_extent = rrdeng_pages_per_extent;
1376
1377 // stats.sizeof_metric_in_index = 40;
1378 // stats.sizeof_page_in_index = 24;
1379
1391 - stats.default_granularity_secs = (size_t)default_rrd_update_every * get_tier_grouping(ctx->config.tier);
1380 + stats.default_granularity_secs = (size_t) rrdb.default_update_every * get_tier_grouping(ctx->config.tier);
1381
1382 return stats;
1383 }
1384
1385 +void rrdeng_size_statistics(STORAGE_INSTANCE *db_instance, BUFFER *wb) {
1386 + struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1387 + RRDENG_SIZE_STATS stats = rrdeng_size_statistics_internal(ctx);
1388 +
1389 + buffer_sprintf(wb,
1390 + "\n\t\t\"default_granularity_secs\":%zu"
1391 + ",\n\t\t\"sizeof_datafile\":%zu"
1392 + ",\n\t\t\"sizeof_page_in_cache\":%zu"
1393 + ",\n\t\t\"sizeof_point_data\":%zu"
1394 + ",\n\t\t\"sizeof_page_data\":%zu"
1395 + ",\n\t\t\"pages_per_extent\":%zu"
1396 + ",\n\t\t\"datafiles\":%zu"
1397 + ",\n\t\t\"extents\":%zu"
1398 + ",\n\t\t\"extents_pages\":%zu"
1399 + ",\n\t\t\"points\":%zu"
1400 + ",\n\t\t\"metrics\":%zu"
1401 + ",\n\t\t\"metrics_pages\":%zu"
1402 + ",\n\t\t\"extents_compressed_bytes\":%zu"
1403 + ",\n\t\t\"pages_uncompressed_bytes\":%zu"
1404 + ",\n\t\t\"pages_duration_secs\":%lld"
1405 + ",\n\t\t\"single_point_pages\":%zu"
1406 + ",\n\t\t\"first_t\":%ld"
1407 + ",\n\t\t\"last_t\":%ld"
1408 + ",\n\t\t\"database_retention_secs\":%lld"
1409 + ",\n\t\t\"average_compression_savings\":%0.2f"
1410 + ",\n\t\t\"average_point_duration_secs\":%0.2f"
1411 + ",\n\t\t\"average_metric_retention_secs\":%0.2f"
1412 + ",\n\t\t\"ephemeral_metrics_per_day_percent\":%0.2f"
1413 + ",\n\t\t\"average_page_size_bytes\":%0.2f"
1414 + ",\n\t\t\"estimated_concurrently_collected_metrics\":%zu"
1415 + ",\n\t\t\"currently_collected_metrics\":%zu"
1416 + ",\n\t\t\"disk_space\":%zu"
1417 + ",\n\t\t\"max_disk_space\":%zu"
1418 + , stats.default_granularity_secs
1419 + , stats.sizeof_datafile
1420 + , stats.sizeof_page_in_cache
1421 + , stats.sizeof_point_data
1422 + , stats.sizeof_page_data
1423 + , stats.pages_per_extent
1424 + , stats.datafiles
1425 + , stats.extents
1426 + , stats.extents_pages
1427 + , stats.points
1428 + , stats.metrics
1429 + , stats.metrics_pages
1430 + , stats.extents_compressed_bytes
1431 + , stats.pages_uncompressed_bytes
1432 + , (long long)stats.pages_duration_secs
1433 + , stats.single_point_pages
1434 + , stats.first_time_s
1435 + , stats.last_time_s
1436 + , (long long)stats.database_retention_secs
1437 + , stats.average_compression_savings
1438 + , stats.average_point_duration_secs
1439 + , stats.average_metric_retention_secs
1440 + , stats.ephemeral_metrics_per_day_percent
1441 + , stats.average_page_size_bytes
1442 + , stats.estimated_concurrently_collected_metrics
1443 + , stats.currently_collected_metrics
1444 + , stats.disk_space
1445 + , stats.max_disk_space
1446 + );
1447 +}
1448 +
1449 struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void) {
1450 // FIXME - make cache efficiency stats atomic
1451 return rrdeng_cache_efficiency_stats;
database/engine/rrdengineapi.h
+1 -62
@@ -12,61 +12,7 @@
12
13 #define RRDENG_FD_BUDGET_PER_INSTANCE (50)
14
15 -extern int default_rrdeng_page_cache_mb;
16 -extern int default_rrdeng_extent_cache_mb;
17 -extern int db_engine_journal_check;
18 -extern int default_rrdeng_disk_quota_mb;
19 -extern int default_multidb_disk_quota_mb;
20 -extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
21 -extern size_t page_type_size[];
22 -extern size_t tier_page_size[];
23 -
24 -#define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type]
25 -
26 -void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid);
27 -
28 -STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
29 -STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
30 -void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
31 -STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);
32 -
33 -STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
34 -void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
35 -void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
36 -void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
37 - NETDATA_DOUBLE min_value,
38 - NETDATA_DOUBLE max_value,
39 - uint16_t count,
40 - uint16_t anomaly_count,
41 - SN_FLAGS flags);
42 -int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
43 -
44 -void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle,
45 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
46 -STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
47 -
48 -
49 -int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
50 -void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
51 -time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
52 -time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
53 -time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
54 -
55 -void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
56 -
57 -/* must call once before using anything */
58 -int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
59 - unsigned disk_space_mb, size_t tier);
60 -
61 -void rrdeng_readiness_wait(struct rrdengine_instance *ctx);
62 -void rrdeng_exit_mode(struct rrdengine_instance *ctx);
63 -
64 -int rrdeng_exit(struct rrdengine_instance *ctx);
65 -void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
66 -bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s);
67 -
68 -extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
69 -extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
15 +#define CTX_POINT_SIZE_BYTES(ctx) rrdb.page_type_size[(ctx)->config.page_type]
16
17 typedef struct rrdengine_size_statistics {
18 size_t default_granularity_secs;
@@ -218,11 +164,4 @@ struct rrdeng_buffer_sizes {
164 struct rrdeng_buffer_sizes rrdeng_get_buffer_sizes(void);
165 struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void);
166
221 -RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
222 -size_t rrdeng_collectors_running(struct rrdengine_instance *ctx);
223 -bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance);
224 -
225 -size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
226 -size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
227 -
167 #endif /* NETDATA_RRDENGINEAPI_H */
database/engine/rrdenginelib.c
+2 -2
@@ -142,7 +142,7 @@ int compute_multidb_diskspace()
142 if (computed_multidb_disk_quota_mb == -1) {
143 int rc = count_legacy_children(netdata_configured_cache_dir);
144 if (likely(rc >= 0)) {
145 - computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
145 + computed_multidb_disk_quota_mb = (rc + 1) * rrdb.default_rrdeng_disk_quota_mb;
146 netdata_log_info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
147
148 fp = fopen(multidb_disk_space_file, "w");
@@ -154,7 +154,7 @@ int compute_multidb_diskspace()
154 netdata_log_error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
155 }
156 else
157 - computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
157 + computed_multidb_disk_quota_mb = rrdb.default_rrdeng_disk_quota_mb;
158 }
159
160 return computed_multidb_disk_quota_mb;
database/ram/rrddim_mem.c
+22 -3
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "rrddim_mem.h"
4 -#include "Judy.h"
4 +#include "database/rrd.h"
5
6 static Pvoid_t rrddim_JudyHS_array = NULL;
7 static netdata_rwlock_t rrddim_JudyHS_rwlock = NETDATA_RWLOCK_INITIALIZER;
@@ -143,7 +143,6 @@ STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_han
143 internal_fatal((uint32_t)mh->update_every_s != update_every, "RRDDIM: update requested does not match the dimension");
144
145 struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
146 - ch->common.backend = STORAGE_ENGINE_BACKEND_RRDDIM;
146 ch->rd = rd;
147 ch->db_metric_handle = db_metric_handle;
148
@@ -341,7 +340,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_e
340 handle->start_time_s = start_time_s;
341 handle->end_time_s = end_time_s;
342 handle->priority = priority;
344 - handle->backend = STORAGE_ENGINE_BACKEND_RRDDIM;
343 +
344 struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
345 h->db_metric_handle = db_metric_handle;
346
@@ -435,3 +434,23 @@ time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle) {
434 struct mem_metric_handle *mh = (struct mem_metric_handle *)db_metric_handle;
435 return (time_t)(mh->last_updated_s - metric_duration(mh));
436 }
437 +
438 +size_t rrddim_disk_space_max(STORAGE_INSTANCE *db_instance) {
439 + UNUSED(db_instance);
440 + return 0;
441 +}
442 +
443 +size_t rrddim_disk_space_used(STORAGE_INSTANCE *db_instance) {
444 + UNUSED(db_instance);
445 + return 0;
446 +}
447 +
448 +time_t rrddim_global_first_time_s(STORAGE_INSTANCE *db_instance) {
449 + UNUSED(db_instance);
450 + return now_realtime_sec() - (time_t)(rrdb.default_rrd_history_entries * rrdb.default_update_every);
451 +}
452 +
453 +size_t rrddim_currently_collected_metrics(STORAGE_INSTANCE *db_instance) {
454 + UNUSED(db_instance);
455 + return 0;
456 +}
database/ram/rrddim_mem.h
+27 -10
@@ -3,11 +3,9 @@
3 #ifndef NETDATA_RRDDIMMEM_H
4 #define NETDATA_RRDDIMMEM_H
5
6 -#include "database/rrd.h"
6 +#include "database/storage_engine_types.h"
7
8 struct mem_collect_handle {
9 - struct storage_collect_handle common; // has to be first item
10 -
9 STORAGE_METRIC_HANDLE *db_metric_handle;
10 RRDDIM *rd;
11 };
@@ -29,11 +27,6 @@ void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
27
28 bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
29
32 -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
33 -void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
34 -
35 -STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
36 -void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
30 void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
31 NETDATA_DOUBLE min_value,
32 NETDATA_DOUBLE max_value,
@@ -43,12 +36,36 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
36 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
37 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
38
39 +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
40 +
41 +void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
42 +
43 +STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
44 +
45 +void rrddim_collect_store_metric(
46 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
47 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
48 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
49 +
50 +
51 +void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
52 +
53 +int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
54 +
55 +void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
56 +time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
57 +time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
58 void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle, time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
59 STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle);
60 int rrddim_query_is_finished(struct storage_engine_query_handle *handle);
61 void rrddim_query_finalize(struct storage_engine_query_handle *handle);
50 -time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
51 -time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
62 time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
63
64 +size_t rrddim_disk_space_max(STORAGE_INSTANCE *db_instance);
65 +size_t rrddim_disk_space_used(STORAGE_INSTANCE *db_instance);
66 +
67 +time_t rrddim_global_first_time_s(STORAGE_INSTANCE *db_instance);
68 +
69 +size_t rrddim_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
70 +
71 #endif
database/rrd.c
+354 -87
@@ -1,70 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS 1
2
3 #include "rrd.h"
5 -#include "storage_engine.h"
6 -
7 -// ----------------------------------------------------------------------------
8 -// globals
9 -
10 -/*
11 -// if not zero it gives the time (in seconds) to remove un-updated dimensions
12 -// DO NOT ENABLE
13 -// if dimensions are removed, the chart generation will have to run again
14 -int rrd_delete_unupdated_dimensions = 0;
15 -*/
16 -
17 -int default_rrd_update_every = UPDATE_EVERY;
18 -int default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
19 -#ifdef ENABLE_DBENGINE
20 -RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
21 -#else
22 -RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
23 -#endif
24 -int gap_when_lost_iterations_above = 1;
25 -
26 -
27 -// ----------------------------------------------------------------------------
28 -// RRD - memory modes
29 -
30 -inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) {
31 - switch(id) {
32 - case RRD_MEMORY_MODE_RAM:
33 - return RRD_MEMORY_MODE_RAM_NAME;
34 -
35 - case RRD_MEMORY_MODE_MAP:
36 - return RRD_MEMORY_MODE_MAP_NAME;
37 -
38 - case RRD_MEMORY_MODE_NONE:
39 - return RRD_MEMORY_MODE_NONE_NAME;
40 -
41 - case RRD_MEMORY_MODE_SAVE:
42 - return RRD_MEMORY_MODE_SAVE_NAME;
43 -
44 - case RRD_MEMORY_MODE_ALLOC:
45 - return RRD_MEMORY_MODE_ALLOC_NAME;
46 -
47 - case RRD_MEMORY_MODE_DBENGINE:
48 - return RRD_MEMORY_MODE_DBENGINE_NAME;
49 - }
50 -
51 - STORAGE_ENGINE* eng = storage_engine_get(id);
52 - if (eng) {
53 - return eng->name;
54 - }
55 -
56 - return RRD_MEMORY_MODE_SAVE_NAME;
57 -}
58 -
59 -RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) {
60 - STORAGE_ENGINE* eng = storage_engine_find(name);
61 - if (eng) {
62 - return eng->id;
63 - }
64 -
65 - return RRD_MEMORY_MODE_SAVE;
66 -}
67 -
4
5 // ----------------------------------------------------------------------------
6 // RRD - algorithms types
@@ -107,7 +43,7 @@ const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) {
43 // ----------------------------------------------------------------------------
44 // RRD - chart types
45
110 -inline RRDSET_TYPE rrdset_type_id(const char *name) {
46 +RRDSET_TYPE rrdset_type_id(const char *name) {
47 if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))
48 return RRDSET_TYPE_AREA;
49
@@ -132,28 +68,6 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type) {
68 }
69 }
70
135 -// ----------------------------------------------------------------------------
136 -// RRD - cache directory
137 -
138 -char *rrdhost_cache_dir_for_rrdset_alloc(RRDHOST *host, const char *id) {
139 - char *ret = NULL;
140 -
141 - char b[FILENAME_MAX + 1];
142 - char n[FILENAME_MAX + 1];
143 - rrdset_strncpyz_name(b, id, FILENAME_MAX);
144 -
145 - snprintfz(n, FILENAME_MAX, "%s/%s", host->cache_dir, b);
146 - ret = strdupz(n);
147 -
148 - if(host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
149 - int r = mkdir(ret, 0775);
150 - if(r != 0 && errno != EEXIST)
151 - netdata_log_error("Cannot create directory '%s'", ret);
152 - }
153 -
154 - return ret;
155 -}
156 -
71 // ----------------------------------------------------------------------------
72 // RRD - string management
73
@@ -166,3 +80,356 @@ STRING *rrd_string_strdupz(const char *s) {
80 freez(tmp);
81 return ret;
82 }
83 +
84 +// ----------------------------------------------------------------------------
85 +// rrd global / startup initialization
86 +
87 +#ifdef ENABLE_DBENGINE
88 +struct dbengine_initialization {
89 + netdata_thread_t thread;
90 + char path[FILENAME_MAX + 1];
91 + int disk_space_mb;
92 + size_t tier;
93 + int ret;
94 +};
95 +
96 +static void *dbengine_tier_init(void *ptr) {
97 + struct dbengine_initialization *dbi = ptr;
98 + dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);
99 + return ptr;
100 +}
101 +#endif
102 +
103 +static void dbengine_init(const char *hostname) {
104 +#ifdef ENABLE_DBENGINE
105 + rrdb.use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", rrdb.use_direct_io);
106 +
107 + unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
108 + if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
109 + rrdeng_pages_per_extent = read_num;
110 + else {
111 + netdata_log_error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
112 + config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
113 + }
114 +
115 + rrdb.storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
116 + if (rrdb.storage_tiers < 1) {
117 + netdata_log_error("At least 1 storage tier is required. Assuming 1.");
118 + rrdb.storage_tiers = 1;
119 + config_set_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
120 + }
121 + if (rrdb.storage_tiers > RRD_STORAGE_TIERS) {
122 + netdata_log_error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS);
123 + rrdb.storage_tiers = RRD_STORAGE_TIERS;
124 + config_set_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
125 + }
126 +
127 + bool parallel_initialization = (rrdb.storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
128 + parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
129 +
130 + struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
131 +
132 + size_t created_tiers = 0;
133 + char dbenginepath[FILENAME_MAX + 1];
134 + char dbengineconfig[200 + 1];
135 + int divisor = 1;
136 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
137 + if(tier == 0)
138 + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
139 + else
140 + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
141 +
142 + int ret = mkdir(dbenginepath, 0775);
143 + if (ret != 0 && errno != EEXIST) {
144 + netdata_log_error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
145 + break;
146 + }
147 +
148 + if(tier > 0)
149 + divisor *= 2;
150 +
151 + int disk_space_mb = rrdb.default_multidb_disk_quota_mb / divisor;
152 + size_t grouping_iterations = rrdb.storage_tiers_grouping_iterations[tier];
153 + RRD_BACKFILL backfill = rrdb.storage_tiers_backfill[tier];
154 +
155 + if(tier > 0) {
156 + snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
157 + disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
158 +
159 + snprintfz(dbengineconfig, 200, "dbengine tier %zu update every iterations", tier);
160 + grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
161 + if(grouping_iterations < 2) {
162 + grouping_iterations = 2;
163 + config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
164 + netdata_log_error("DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",
165 + hostname,
166 + tier);
167 + }
168 +
169 + snprintfz(dbengineconfig, 200, "dbengine tier %zu backfill", tier);
170 + const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
171 + if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
172 + else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
173 + else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;
174 + else {
175 + netdata_log_error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf);
176 + config_set(CONFIG_SECTION_DB, dbengineconfig, "new");
177 + backfill = RRD_BACKFILL_NEW;
178 + }
179 + }
180 +
181 + rrdb.storage_tiers_grouping_iterations[tier] = grouping_iterations;
182 + rrdb.storage_tiers_backfill[tier] = backfill;
183 +
184 + if(tier > 0 && get_tier_grouping(tier) > 65535) {
185 + rrdb.storage_tiers_grouping_iterations[tier] = 1;
186 + netdata_log_error("DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. Disabling tiers above %zu",
187 + hostname,
188 + tier,
189 + tier);
190 + break;
191 + }
192 +
193 + internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, rrdb.storage_tiers_grouping_iterations[tier]);
194 +
195 + tiers_init[tier].disk_space_mb = disk_space_mb;
196 + tiers_init[tier].tier = tier;
197 + strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
198 + tiers_init[tier].ret = 0;
199 +
200 + if(parallel_initialization) {
201 + char tag[NETDATA_THREAD_TAG_MAX + 1];
202 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier);
203 + netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE,
204 + dbengine_tier_init, &tiers_init[tier]);
205 + }
206 + else
207 + dbengine_tier_init(&tiers_init[tier]);
208 + }
209 +
210 + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
211 + void *ptr;
212 +
213 + if(parallel_initialization)
214 + netdata_thread_join(tiers_init[tier].thread, &ptr);
215 +
216 + if(tiers_init[tier].ret != 0) {
217 + netdata_log_error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
218 + hostname,
219 + tiers_init[tier].tier,
220 + tiers_init[tier].path);
221 + }
222 + else if(created_tiers == tier)
223 + created_tiers++;
224 + }
225 +
226 + if(created_tiers && created_tiers < rrdb.storage_tiers) {
227 + netdata_log_error("DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.",
228 + hostname,
229 + created_tiers,
230 + rrdb.storage_tiers,
231 + created_tiers);
232 + rrdb.storage_tiers = created_tiers;
233 + }
234 + else if(!created_tiers)
235 + fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
236 +
237 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
238 + rrdeng_readiness_wait(rrdb.multidb_ctx[tier]);
239 +
240 + rrdb.dbengine_enabled = true;
241 +#else
242 + rrdb.storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
243 + if(rrdb.storage_tiers != 1) {
244 + netdata_log_error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname);
245 + rrdb.storage_tiers = 1;
246 + config_set_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
247 + }
248 + rrdb.dbengine_enabled = false;
249 +#endif
250 +}
251 +
252 +static void init_host_indexes() {
253 + internal_fatal(rrdb.rrdhost_root_index || rrdb.rrdhost_root_index_hostname,
254 + "Host indexes have already been initialized");
255 +
256 + DICT_OPTIONS dict_opts = DICT_OPTION_NAME_LINK_DONT_CLONE |
257 + DICT_OPTION_VALUE_LINK_DONT_CLONE |
258 + DICT_OPTION_DONT_OVERWRITE_VALUE;
259 +
260 + rrdb.rrdhost_root_index = dictionary_create_advanced(dict_opts, &dictionary_stats_category_rrdhost, 0);
261 + rrdb.rrdhost_root_index_hostname = dictionary_create_advanced(dict_opts, &dictionary_stats_category_rrdhost, 0);
262 +}
263 +
264 +int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
265 + init_host_indexes();
266 +
267 + if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
268 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE) {
269 + set_late_global_environment(system_info);
270 + fatal("Failed to initialize SQLite");
271 + }
272 + netdata_log_info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
273 + }
274 +
275 + if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
276 + error_report("Failed to initialize context metadata database");
277 + }
278 +
279 + if (unlikely(unittest)) {
280 + rrdb.dbengine_enabled = true;
281 + }
282 + else {
283 + health_init();
284 + rrdpush_init();
285 +
286 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
287 + netdata_log_info("DBENGINE: Initializing ...");
288 + dbengine_init(hostname);
289 + }
290 + else {
291 + netdata_log_info("DBENGINE: Not initializing ...");
292 + rrdb.storage_tiers = 1;
293 + }
294 +
295 + if (!rrdb.dbengine_enabled) {
296 + if (rrdb.storage_tiers > 1) {
297 + netdata_log_error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
298 + rrdb.storage_tiers);
299 + rrdb.storage_tiers = 1;
300 + }
301 +
302 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE) {
303 + netdata_log_error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
304 + default_storage_engine_id = STORAGE_ENGINE_ALLOC;
305 + }
306 + }
307 + }
308 +
309 + if(!unittest)
310 + metadata_sync_init();
311 +
312 + netdata_log_debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
313 + rrdb.localhost = rrdhost_create(
314 + hostname
315 + , registry_get_this_machine_hostname()
316 + , registry_get_this_machine_guid()
317 + , os_type
318 + , netdata_configured_timezone
319 + , netdata_configured_abbrev_timezone
320 + , netdata_configured_utc_offset
321 + , ""
322 + , program_name
323 + , program_version
324 + , rrdb.default_update_every
325 + , rrdb.default_rrd_history_entries
326 + , default_storage_engine_id
327 + , default_health_enabled
328 + , default_rrdpush_enabled
329 + , default_rrdpush_destination
330 + , default_rrdpush_api_key
331 + , default_rrdpush_send_charts_matching
332 + , default_rrdpush_enable_replication
333 + , default_rrdpush_seconds_to_replicate
334 + , default_rrdpush_replication_step
335 + , system_info
336 + , 1
337 + , 0
338 + );
339 +
340 + if (unlikely(!rrdb.localhost)) {
341 + return 1;
342 + }
343 +
344 +#if NETDATA_DEV_MODE
345 + // we register this only on localhost
346 + // for the other nodes, the origin server should register it
347 + rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
348 + rrd_collector_add_function(rrdb.localhost, NULL, "streaming", 10,
349 + RRDFUNCTIONS_STREAMING_HELP, true,
350 + rrdhost_function_streaming, NULL);
351 +#endif
352 +
353 + if (likely(system_info)) {
354 + migrate_localhost(&rrdb.localhost->host_uuid);
355 + sql_aclk_sync_init();
356 + web_client_api_v1_management_init();
357 + }
358 + return rrdb.localhost == NULL;
359 +}
360 +
361 +struct rrdb rrdb = {
362 + .rrdhost_root_index = NULL,
363 + .rrdhost_root_index_hostname = NULL,
364 + .unittest_running = false,
365 + .dbengine_enabled = false,
366 + .storage_tiers = 3,
367 + .use_direct_io = true,
368 + .storage_tiers_grouping_iterations = {
369 + 1,
370 + 60,
371 + 60,
372 + 60,
373 + 60
374 + },
375 + .storage_tiers_backfill = {
376 + RRD_BACKFILL_NEW,
377 + RRD_BACKFILL_NEW,
378 + RRD_BACKFILL_NEW,
379 + RRD_BACKFILL_NEW,
380 + RRD_BACKFILL_NEW
381 + },
382 + .default_update_every = UPDATE_EVERY_MIN,
383 + .default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES,
384 + .gap_when_lost_iterations_above = 1,
385 + .rrdset_free_obsolete_time_s = RRD_DEFAULT_HISTORY_ENTRIES,
386 + .libuv_worker_threads = 8,
387 + .ieee754_doubles = false,
388 + .rrdhost_free_orphan_time_s = RRD_DEFAULT_HISTORY_ENTRIES,
389 + .rrd_rwlock = NETDATA_RWLOCK_INITIALIZER,
390 + .localhost = NULL,
391 +
392 +#if defined(ENV32BIT)
393 + .default_rrdeng_page_cache_mb = 16,
394 + .default_rrdeng_extent_cache_mb = 0,
395 +#else
396 + .default_rrdeng_page_cache_mb = 32,
397 + .default_rrdeng_extent_cache_mb = 0,
398 +#endif
399 +
400 + .db_engine_journal_check = CONFIG_BOOLEAN_NO,
401 +
402 + .default_rrdeng_disk_quota_mb = 256,
403 + .default_multidb_disk_quota_mb = 256,
404 +
405 + .multidb_ctx = {
406 + NULL,
407 + NULL,
408 + NULL,
409 + NULL,
410 + NULL,
411 + },
412 +
413 + .page_type_size = {
414 + sizeof(storage_number),
415 + sizeof(storage_number_tier1_t)
416 + },
417 +
418 +#if defined(ENV32BIT)
419 + .tier_page_size = {
420 + 2048,
421 + 1024,
422 + 192,
423 + 192,
424 + 192
425 + },
426 +#else
427 + .tier_page_size = {
428 + 4096,
429 + 2048,
430 + 384,
431 + 384,
432 + 384
433 + },
434 +#endif
435 +};
database/rrd.h
+124 -557
@@ -9,13 +9,8 @@ extern "C" {
9
10 #include "libnetdata/libnetdata.h"
11
12 -// non-existing structs instead of voids
13 -// to enable type checking at compile time
14 -typedef struct storage_instance STORAGE_INSTANCE;
15 -typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
16 -typedef struct storage_alignment STORAGE_METRICS_GROUP;
17 -
12 // forward typedefs
13 +typedef struct rrdb RRDB;
14 typedef struct rrdhost RRDHOST;
15 typedef struct rrddim RRDDIM;
16 typedef struct rrdset RRDSET;
@@ -46,57 +41,11 @@ typedef enum __attribute__ ((__packed__)) {
41 QUERY_SOURCE_UNITTEST,
42 } QUERY_SOURCE;
43
49 -typedef enum __attribute__ ((__packed__)) storage_priority {
50 - STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
51 - STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
52 -
53 - // query priorities
54 - STORAGE_PRIORITY_HIGH,
55 - STORAGE_PRIORITY_NORMAL,
56 - STORAGE_PRIORITY_LOW,
57 - STORAGE_PRIORITY_BEST_EFFORT,
58 -
59 - // synchronous query, not to be dispatched to workers or queued
60 - STORAGE_PRIORITY_SYNCHRONOUS,
61 -
62 - STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
63 -} STORAGE_PRIORITY;
44 +#define RRD_STORAGE_TIERS 5
45
46 // forward declarations
47 struct rrddim_tier;
48
68 -#ifdef ENABLE_DBENGINE
69 -struct rrdeng_page_descr;
70 -struct rrdengine_instance;
71 -struct pg_cache_page_index;
72 -#endif
73 -
74 -// ----------------------------------------------------------------------------
75 -// memory mode
76 -
77 -typedef enum __attribute__ ((__packed__)) rrd_memory_mode {
78 - RRD_MEMORY_MODE_NONE = 0,
79 - RRD_MEMORY_MODE_RAM = 1,
80 - RRD_MEMORY_MODE_MAP = 2,
81 - RRD_MEMORY_MODE_SAVE = 3,
82 - RRD_MEMORY_MODE_ALLOC = 4,
83 - RRD_MEMORY_MODE_DBENGINE = 5,
84 -
85 - // this is 8-bit
86 -} RRD_MEMORY_MODE;
87 -
88 -#define RRD_MEMORY_MODE_NONE_NAME "none"
89 -#define RRD_MEMORY_MODE_RAM_NAME "ram"
90 -#define RRD_MEMORY_MODE_MAP_NAME "map"
91 -#define RRD_MEMORY_MODE_SAVE_NAME "save"
92 -#define RRD_MEMORY_MODE_ALLOC_NAME "alloc"
93 -#define RRD_MEMORY_MODE_DBENGINE_NAME "dbengine"
94 -
95 -extern RRD_MEMORY_MODE default_rrd_memory_mode;
96 -
97 -const char *rrd_memory_mode_name(RRD_MEMORY_MODE id);
98 -RRD_MEMORY_MODE rrd_memory_mode_id(const char *name);
99 -
49 struct ml_metrics_statistics {
50 size_t anomalous;
51 size_t normal;
@@ -105,6 +54,7 @@ struct ml_metrics_statistics {
54 size_t silenced;
55 };
56
57 +#include "storage_engine.h"
58 #include "daemon/common.h"
59 #include "web/api/queries/query.h"
60 #include "web/api/queries/rrdr.h"
@@ -113,32 +63,15 @@ struct ml_metrics_statistics {
63 #include "rrddimvar.h"
64 #include "rrdcalc.h"
65 #include "rrdcalctemplate.h"
66 +#include "rrdfunctions.h"
67 +#include "rrdlabels.h"
68 #include "streaming/rrdpush.h"
69 #include "aclk/aclk_rrdhost_state.h"
118 -#include "sqlite/sqlite_health.h"
119 -
120 -typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
121 -
122 -typedef enum __attribute__ ((__packed__)) {
123 - STORAGE_ENGINE_BACKEND_RRDDIM = 1,
124 - STORAGE_ENGINE_BACKEND_DBENGINE = 2,
125 -} STORAGE_ENGINE_BACKEND;
126 -
127 -#define is_valid_backend(backend) ((backend) >= STORAGE_ENGINE_BACKEND_RRDDIM && (backend) <= STORAGE_ENGINE_BACKEND_DBENGINE)
128 -
129 -// iterator state for RRD dimension data queries
130 -struct storage_engine_query_handle {
131 - time_t start_time_s;
132 - time_t end_time_s;
133 - STORAGE_PRIORITY priority;
134 - STORAGE_ENGINE_BACKEND backend;
135 - STORAGE_QUERY_HANDLE *handle;
136 -};
70
71 // ----------------------------------------------------------------------------
72 // chart types
73
141 -typedef enum __attribute__ ((__packed__)) rrdset_type {
74 +typedef enum __attribute__ ((__packed__)) {
75 RRDSET_TYPE_LINE = 0,
76 RRDSET_TYPE_AREA = 1,
77 RRDSET_TYPE_STACKED = 2,
@@ -153,53 +86,24 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type);
86
87 #include "contexts/rrdcontext.h"
88
156 -extern bool unittest_running;
157 -extern bool dbengine_enabled;
158 -extern size_t storage_tiers;
159 -extern bool use_direct_io;
160 -extern size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
161 -
89 typedef enum __attribute__ ((__packed__)) {
90 RRD_BACKFILL_NONE = 0,
91 RRD_BACKFILL_FULL,
92 RRD_BACKFILL_NEW
93 } RRD_BACKFILL;
94
168 -extern RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
169 -
170 -#define UPDATE_EVERY 1
171 -#define UPDATE_EVERY_MAX 3600
95 +#define UPDATE_EVERY_MIN 1
96 +#define UPDATE_EVERY_MAX 600
97
98 #define RRD_DEFAULT_HISTORY_ENTRIES 3600
99 #define RRD_HISTORY_ENTRIES_MAX (86400*365)
100
176 -extern int default_rrd_update_every;
177 -extern int default_rrd_history_entries;
178 -extern int gap_when_lost_iterations_above;
179 -extern time_t rrdset_free_obsolete_time_s;
180 -
181 -#if defined(ENV32BIT)
182 -#define MIN_LIBUV_WORKER_THREADS 8
183 -#define MAX_LIBUV_WORKER_THREADS 128
184 -#define RESERVED_LIBUV_WORKER_THREADS 3
185 -#else
186 -#define MIN_LIBUV_WORKER_THREADS 16
187 -#define MAX_LIBUV_WORKER_THREADS 1024
188 -#define RESERVED_LIBUV_WORKER_THREADS 6
189 -#endif
190 -
191 -extern int libuv_worker_threads;
192 -extern bool ieee754_doubles;
193 -
101 #define RRD_ID_LENGTH_MAX 1000
102
196 -typedef long long total_number;
197 -#define TOTAL_NUMBER_FORMAT "%lld"
198 -
103 // ----------------------------------------------------------------------------
104 // algorithms types
105
202 -typedef enum __attribute__ ((__packed__)) rrd_algorithm {
106 +typedef enum __attribute__ ((__packed__)) {
107 RRD_ALGORITHM_ABSOLUTE = 0,
108 RRD_ALGORITHM_INCREMENTAL = 1,
109 RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2,
@@ -230,7 +134,7 @@ DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rf);
134 // flags & options
135
136 // options are permanent configuration options (no atomics to alter/access them)
233 -typedef enum __attribute__ ((__packed__)) rrddim_options {
137 +typedef enum __attribute__ ((__packed__)) {
138 RRDDIM_OPTION_NONE = 0,
139 RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
140 RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
@@ -246,7 +150,7 @@ typedef enum __attribute__ ((__packed__)) rrddim_options {
150 #define rrddim_option_clear(rd, option) (rd)->collector.options &= ~(option)
151
152 // flags are runtime changing status flags (atomics are required to alter/access them)
249 -typedef enum __attribute__ ((__packed__)) rrddim_flags {
153 +typedef enum __attribute__ ((__packed__)) {
154 RRDDIM_FLAG_NONE = 0,
155 RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 0),
156
@@ -265,73 +169,11 @@ typedef enum __attribute__ ((__packed__)) rrddim_flags {
169 #define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
170 #define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
171
268 -typedef enum __attribute__ ((__packed__)) rrdlabel_source {
269 - RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
270 - RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
271 - RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
272 - RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
273 -
274 - // more sources can be added here
275 -
276 - RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
277 - RRDLABEL_FLAG_OLD = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
278 - RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels
279 -} RRDLABEL_SRC;
280 -
281 -#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
282 -
283 -size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
284 -
285 -DICTIONARY *rrdlabels_create(void);
286 -void rrdlabels_destroy(DICTIONARY *labels_dict);
287 -void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
288 -void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
289 -void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
290 -void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key);
291 -void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key);
292 -void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key);
293 -STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key);
294 -STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset);
295 -void rrdlabels_flush(DICTIONARY *labels_dict);
296 -
297 -void rrdlabels_unmark_all(DICTIONARY *labels);
298 -void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
299 -
300 -int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
301 -int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
302 -
303 -void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
304 -bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
305 -
306 -bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
307 -int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
308 -void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb);
309 -
310 -void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
311 -void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
312 -
313 -void reload_host_labels(void);
314 -void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
315 -void rrdset_save_rrdlabels_to_sql(RRDSET *st);
316 -void rrdhost_set_is_parent_label(void);
317 -int rrdlabels_unittest(void);
318 -
319 -// unfortunately this break when defined in exporting_engine.h
320 -bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
321 -
322 -// ----------------------------------------------------------------------------
323 -// engine-specific iterator state for dimension data collection
324 -typedef struct storage_collect_handle {
325 - STORAGE_ENGINE_BACKEND backend;
326 -} STORAGE_COLLECT_HANDLE;
327 -
172 // ----------------------------------------------------------------------------
173 // Storage tier data for every dimension
174
175 struct rrddim_tier {
176 STORAGE_POINT virtual_point;
333 - STORAGE_ENGINE_BACKEND backend;
334 - uint32_t tier_grouping;
177 time_t next_point_end_time_s;
178 STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database
179 STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle
@@ -352,7 +194,6 @@ struct rrddim {
194 STRING *name; // the name of this dimension (as presented to user)
195
196 RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
355 - RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
197 RRDDIM_FLAGS flags; // run time changing status flags
198
199 int32_t multiplier; // the multiplier of the collected values
@@ -425,292 +266,17 @@ size_t rrddim_size(void);
266 // returns the RRDDIM cache filename, or NULL if it does not exist
267 const char *rrddim_cache_filename(RRDDIM *rd);
268
428 -// updated the header with the latest RRDDIM value, for memory mode MAP and SAVE
269 +// updated the header with the latest RRDDIM value, for storage engines MAP and SAVE
270 void rrddim_memory_file_update(RRDDIM *rd);
271
431 -// free the memory file structures for memory mode MAP and SAVE
272 +// free the memory file structures for storage engines MAP and SAVE
273 void rrddim_memory_file_free(RRDDIM *rd);
274
434 -bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode);
435 -
275 // return the v019 header size of RRDDIM files
276 size_t rrddim_memory_file_header_size(void);
277
278 void rrddim_memory_file_save(RRDDIM *rd);
279
441 -// ------------------------------------------------------------------------
442 -// DATA COLLECTION STORAGE OPS
443 -
444 -STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
445 -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
446 -static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance, uuid_t *uuid) {
447 - internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
448 -
449 -#ifdef ENABLE_DBENGINE
450 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
451 - return rrdeng_metrics_group_get(db_instance, uuid);
452 -#endif
453 - return rrddim_metrics_group_get(db_instance, uuid);
454 -}
455 -
456 -void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
457 -void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
458 -static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
459 - internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
460 -
461 -#ifdef ENABLE_DBENGINE
462 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
463 - rrdeng_metrics_group_release(db_instance, smg);
464 - else
465 -#endif
466 - rrddim_metrics_group_release(db_instance, smg);
467 -}
468 -
469 -STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
470 -STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
471 -static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {
472 - internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
473 -
474 -#ifdef ENABLE_DBENGINE
475 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
476 - return rrdeng_store_metric_init(db_metric_handle, update_every, smg);
477 -#endif
478 - return rrddim_collect_init(db_metric_handle, update_every, smg);
479 -}
480 -
481 -void rrdeng_store_metric_next(
482 - STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
483 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
484 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
485 -
486 -void rrddim_collect_store_metric(
487 - STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
488 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
489 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
490 -
491 -static inline void storage_engine_store_metric(
492 - STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
493 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
494 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {
495 - internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
496 -
497 -#ifdef ENABLE_DBENGINE
498 - if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
499 - return rrdeng_store_metric_next(collection_handle, point_in_time_ut,
500 - n, min_value, max_value,
501 - count, anomaly_count, flags);
502 -#endif
503 - return rrddim_collect_store_metric(collection_handle, point_in_time_ut,
504 - n, min_value, max_value,
505 - count, anomaly_count, flags);
506 -}
507 -
508 -size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
509 -static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
510 -#ifdef ENABLE_DBENGINE
511 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
512 - return rrdeng_disk_space_max(db_instance);
513 -#endif
514 -
515 - return 0;
516 -}
517 -
518 -size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
519 -static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
520 -#ifdef ENABLE_DBENGINE
521 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
522 - return rrdeng_disk_space_used(db_instance);
523 -#endif
524 -
525 - // TODO - calculate the total host disk space for memory mode save and map
526 - return 0;
527 -}
528 -
529 -time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *db_instance);
530 -static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
531 -#ifdef ENABLE_DBENGINE
532 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
533 - return rrdeng_global_first_time_s(db_instance);
534 -#endif
535 -
536 - return now_realtime_sec() - (time_t)(default_rrd_history_entries * default_rrd_update_every);
537 -}
538 -
539 -size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
540 -static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
541 -#ifdef ENABLE_DBENGINE
542 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
543 - return rrdeng_currently_collected_metrics(db_instance);
544 -#endif
545 -
546 - // TODO - calculate the total host disk space for memory mode save and map
547 - return 0;
548 -}
549 -
550 -void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
551 -void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
552 -static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
553 - if(unlikely(!collection_handle))
554 - return;
555 -
556 - internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
557 -
558 -#ifdef ENABLE_DBENGINE
559 - if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
560 - rrdeng_store_metric_flush_current_page(collection_handle);
561 - else
562 -#endif
563 - rrddim_store_metric_flush(collection_handle);
564 -}
565 -
566 -int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
567 -int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
568 -// a finalization function to run after collection is over
569 -// returns 1 if it's safe to delete the dimension
570 -static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
571 - internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
572 -
573 -#ifdef ENABLE_DBENGINE
574 - if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
575 - return rrdeng_store_metric_finalize(collection_handle);
576 -#endif
577 -
578 - return rrddim_collect_finalize(collection_handle);
579 -}
580 -
581 -void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
582 -void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
583 -static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every) {
584 - internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
585 -
586 -#ifdef ENABLE_DBENGINE
587 - if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
588 - rrdeng_store_metric_change_collection_frequency(collection_handle, update_every);
589 - else
590 -#endif
591 - rrddim_store_metric_change_collection_frequency(collection_handle, update_every);
592 -}
593 -
594 -
595 -// ----------------------------------------------------------------------------
596 -// STORAGE ENGINE QUERY OPS
597 -
598 -time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
599 -time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
600 -static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_METRIC_HANDLE *db_metric_handle) {
601 - internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
602 -
603 -#ifdef ENABLE_DBENGINE
604 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
605 - return rrdeng_metric_oldest_time(db_metric_handle);
606 -#endif
607 - return rrddim_query_oldest_time_s(db_metric_handle);
608 -}
609 -
610 -time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
611 -time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
612 -static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_METRIC_HANDLE *db_metric_handle) {
613 - internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
614 -
615 -#ifdef ENABLE_DBENGINE
616 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
617 - return rrdeng_metric_latest_time(db_metric_handle);
618 -#endif
619 - return rrddim_query_latest_time_s(db_metric_handle);
620 -}
621 -
622 -void rrdeng_load_metric_init(
623 - STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle,
624 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
625 -
626 -void rrddim_query_init(
627 - STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
628 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
629 -
630 -static inline void storage_engine_query_init(
631 - STORAGE_ENGINE_BACKEND backend __maybe_unused,
632 - STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
633 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {
634 - internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
635 -
636 -#ifdef ENABLE_DBENGINE
637 - if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
638 - rrdeng_load_metric_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
639 - else
640 -#endif
641 - rrddim_query_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
642 -}
643 -
644 -STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
645 -STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle);
646 -static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *handle) {
647 - internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
648 -
649 -#ifdef ENABLE_DBENGINE
650 - if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
651 - return rrdeng_load_metric_next(handle);
652 -#endif
653 - return rrddim_query_next_metric(handle);
654 -}
655 -
656 -int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
657 -int rrddim_query_is_finished(struct storage_engine_query_handle *handle);
658 -static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *handle) {
659 - internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
660 -
661 -#ifdef ENABLE_DBENGINE
662 - if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
663 - return rrdeng_load_metric_is_finished(handle);
664 -#endif
665 - return rrddim_query_is_finished(handle);
666 -}
667 -
668 -void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
669 -void rrddim_query_finalize(struct storage_engine_query_handle *handle);
670 -static inline void storage_engine_query_finalize(struct storage_engine_query_handle *handle) {
671 - internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
672 -
673 -#ifdef ENABLE_DBENGINE
674 - if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
675 - rrdeng_load_metric_finalize(handle);
676 - else
677 -#endif
678 - rrddim_query_finalize(handle);
679 -}
680 -
681 -time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
682 -time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
683 -static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *handle) {
684 - internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
685 -
686 -#ifdef ENABLE_DBENGINE
687 - if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
688 - return rrdeng_load_align_to_optimal_before(handle);
689 -#endif
690 - return rrddim_query_align_to_optimal_before(handle);
691 -}
692 -
693 -// ------------------------------------------------------------------------
694 -// function pointers for all APIs provided by a storage engine
695 -typedef struct storage_engine_api {
696 - // metric management
697 - STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *instance, uuid_t *uuid);
698 - STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *instance);
699 - void (*metric_release)(STORAGE_METRIC_HANDLE *);
700 - STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
701 - bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
702 -} STORAGE_ENGINE_API;
703 -
704 -typedef struct storage_engine {
705 - STORAGE_ENGINE_BACKEND backend;
706 - RRD_MEMORY_MODE id;
707 - const char* name;
708 - STORAGE_ENGINE_API api;
709 -} STORAGE_ENGINE;
710 -
711 -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
712 -STORAGE_ENGINE* storage_engine_find(const char* name);
713 -
280 // ----------------------------------------------------------------------------
281 // these loop macros make sure the linked list is accessed with the right lock
282
@@ -733,7 +299,7 @@ STORAGE_ENGINE* storage_engine_find(const char* name);
299 // flags are set/unset in a manner that is not thread safe
300 // and may lead to missing information.
301
736 -typedef enum __attribute__ ((__packed__)) rrdset_flags {
302 +typedef enum __attribute__ ((__packed__)) {
303 RRDSET_FLAG_DETAIL = (1 << 1), // if set, the data set should be considered as a detail of another
304 // (the master data set should be the one that has the same family and is not detail)
305 RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
@@ -813,7 +379,7 @@ struct rrdset {
379 // operational state members
380
381 RRDSET_FLAGS flags; // flags
816 - RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset
382 + STORAGE_ENGINE_ID storage_engine_id; // the db mode of this rrdset
383
384 DICTIONARY *rrddim_root_index; // dimensions index
385
@@ -941,9 +507,6 @@ void rrdset_memory_file_save(RRDSET *st);
507 void rrdset_memory_file_free(RRDSET *st);
508 void rrdset_memory_file_update(RRDSET *st);
509 const char *rrdset_cache_filename(RRDSET *st);
944 -bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
945 -
946 -#include "rrdfunctions.h"
510
511 // ----------------------------------------------------------------------------
512 // RRDHOST flags
@@ -951,7 +514,7 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE m
514 // flags are set/unset in a manner that is not thread safe
515 // and may lead to missing information.
516
954 -typedef enum __attribute__ ((__packed__)) rrdhost_flags {
517 +typedef enum __attribute__ ((__packed__)) {
518
519 // Careful not to overlap with rrdhost_options to avoid bugs if
520 // rrdhost_flags_xxx is used instead of rrdhost_option_xxx or vice-versa
@@ -995,13 +558,6 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {
558 #define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST)
559 #define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~(flag), __ATOMIC_SEQ_CST)
560
998 -#ifdef NETDATA_INTERNAL_CHECKS
999 -#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
1000 - debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0)
1001 -#else
1002 -#define rrdset_debug(st, fmt, args...) debug_dummy()
1003 -#endif
1004 -
561 typedef enum __attribute__ ((__packed__)) {
562 // Indexing
563 RRDHOST_OPTION_INDEXED_MACHINE_GUID = (1 << 0), // when set, we have indexed its machine guid
@@ -1163,8 +719,6 @@ struct rrdhost_system_info {
719 int mc_version;
720 };
721
1166 -struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels);
1167 -
722 struct rrdhost {
723 char machine_guid[GUID_LEN + 1]; // the unique ID of this host
724
@@ -1186,17 +740,16 @@ struct rrdhost {
740 RRDHOST_FLAGS flags; // runtime flags about this RRDHOST (atomics on this)
741 RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances
742
1189 - int32_t rrd_update_every; // the update frequency of the host
743 + int32_t update_every; // the update frequency of the host
744 int32_t rrd_history_entries; // the number of history entries for the host's charts
745
1192 - RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
746 + STORAGE_ENGINE_ID storage_engine_id; // the configured storage engine id for the charts of this host
747 // the actual per tier is at .db[tier].mode
748
749 char *cache_dir; // the directory to save RRD cache files
750
751 struct {
1198 - RRD_MEMORY_MODE mode; // the db mode for this tier
1199 - STORAGE_ENGINE *eng; // the storage engine API for this tier
752 + STORAGE_ENGINE_ID id; // the storage engine id for this tier
753 STORAGE_INSTANCE *instance; // the db instance for this tier
754 uint32_t tier_grouping; // tier 0 iterations aggregated on this tier
755 } db[RRD_STORAGE_TIERS];
@@ -1309,7 +862,57 @@ struct rrdhost {
862 struct rrdhost *next;
863 struct rrdhost *prev;
864 };
1312 -extern RRDHOST *localhost;
865 +
866 +// ----------------------------------------------------------------------------
867 +// RRDB
868 +
869 +struct rrdb {
870 + DICTIONARY *rrdhost_root_index;
871 + DICTIONARY *rrdhost_root_index_hostname;
872 +
873 + bool unittest_running;
874 + bool dbengine_enabled;
875 + size_t storage_tiers;
876 + bool use_direct_io;
877 +
878 + size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
879 + RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
880 +
881 + int default_update_every;
882 + int default_rrd_history_entries;
883 +
884 + int gap_when_lost_iterations_above;
885 +
886 + time_t rrdset_free_obsolete_time_s;
887 +
888 + int libuv_worker_threads;
889 +
890 + bool ieee754_doubles;
891 +
892 + time_t rrdhost_free_orphan_time_s;
893 +
894 + netdata_rwlock_t rrd_rwlock;
895 +
896 + RRDHOST *localhost;
897 +
898 + int default_rrdeng_page_cache_mb;
899 +
900 + int default_rrdeng_extent_cache_mb;
901 +
902 + int db_engine_journal_check;
903 +
904 + int default_rrdeng_disk_quota_mb;
905 +
906 + int default_multidb_disk_quota_mb;
907 +
908 + STORAGE_INSTANCE *multidb_ctx[RRD_STORAGE_TIERS];
909 +
910 + size_t page_type_size[256];
911 +
912 + size_t tier_page_size[RRD_STORAGE_TIERS];
913 +};
914 +
915 +extern struct rrdb rrdb;
916
917 #define rrdhost_hostname(host) string2str((host)->hostname)
918 #define rrdhost_registry_hostname(host) string2str((host)->registry_hostname)
@@ -1333,33 +936,33 @@ extern RRDHOST *localhost;
936 #define rrdhost_sender_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->rrdpush_sender_replicating_charts), 1, __ATOMIC_RELAXED))
937 #define rrdhost_sender_replicating_charts_zero(host) (__atomic_store_n(&((host)->rrdpush_sender_replicating_charts), 0, __ATOMIC_RELAXED))
938
1336 -#define rrdhost_is_online(host) ((host) == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
939 +#define rrdhost_is_online(host) ((host) == rrdb.localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
940 bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now);
941
1339 -extern DICTIONARY *rrdhost_root_index;
942 size_t rrdhost_hosts_available(void);
943
944 RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid);
945 RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha);
946 void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha);
947
948 +void rrdhost_set_is_parent_label(void);
949 +
950 // ----------------------------------------------------------------------------
951
952 #define rrdhost_foreach_read(var) \
1349 - for((var) = localhost; var ; (var) = (var)->next)
953 + for((var) = rrdb.localhost; var ; (var) = (var)->next)
954
955 #define rrdhost_foreach_write(var) \
1352 - for((var) = localhost; var ; (var) = (var)->next)
956 + for((var) = rrdb.localhost; var ; (var) = (var)->next)
957
958
959 // ----------------------------------------------------------------------------
960 // global lock for all RRDHOSTs
961
1358 -extern netdata_rwlock_t rrd_rwlock;
1359 -
1360 -#define rrd_rdlock() netdata_rwlock_rdlock(&rrd_rwlock)
1361 -#define rrd_wrlock() netdata_rwlock_wrlock(&rrd_rwlock)
1362 -#define rrd_unlock() netdata_rwlock_unlock(&rrd_rwlock)
962 +#define rrd_rdlock() netdata_rwlock_rdlock(&rrdb.rrd_rwlock)
963 +#define rrd_wrlock() netdata_rwlock_wrlock(&rrdb.rrd_rwlock)
964 +#define rrd_unlock() netdata_rwlock_unlock(&rrdb.rrd_rwlock)
965 +#define rrd_tryrdlock() netdata_rwlock_tryrdlock(&rrdb.rrd_rwlock)
966
967 // ----------------------------------------------------------------------------
968
@@ -1372,15 +975,39 @@ void rrddim_index_destroy(RRDSET *st);
975
976 // ----------------------------------------------------------------------------
977
1375 -extern time_t rrdhost_free_orphan_time_s;
1376 -
978 int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest);
979
980 RRDHOST *rrdhost_find_by_hostname(const char *hostname);
981 RRDHOST *rrdhost_find_by_guid(const char *guid);
1381 -RRDHOST *find_host_by_node_id(char *node_id);
1382 -
1383 -RRDHOST *rrdhost_find_or_create(
982 +RRDHOST *rrdhost_find_by_node_id(const char *node_id);
983 +
984 +RRDHOST *rrdhost_create(
985 + const char *hostname,
986 + const char *registry_hostname,
987 + const char *guid,
988 + const char *os,
989 + const char *timezone,
990 + const char *abbrev_timezone,
991 + int32_t utc_offset,
992 + const char *tags,
993 + const char *program_name,
994 + const char *program_version,
995 + int update_every,
996 + long entries,
997 + STORAGE_ENGINE_ID storage_engine_id,
998 + unsigned int health_enabled,
999 + unsigned int rrdpush_enabled,
1000 + char *rrdpush_destination,
1001 + char *rrdpush_api_key,
1002 + char *rrdpush_send_charts_matching,
1003 + bool rrdpush_enable_replication,
1004 + time_t rrdpush_seconds_to_replicate,
1005 + time_t rrdpush_replication_step,
1006 + struct rrdhost_system_info *system_info,
1007 + int is_localhost,
1008 + bool archived);
1009 +
1010 +RRDHOST *rrdhost_get_or_create(
1011 const char *hostname
1012 , const char *registry_hostname
1013 , const char *guid
@@ -1393,7 +1020,7 @@ RRDHOST *rrdhost_find_or_create(
1020 , const char *program_version
1021 , int update_every
1022 , long history
1396 - , RRD_MEMORY_MODE mode
1023 + , STORAGE_ENGINE_ID storage_engine_id
1024 , unsigned int health_enabled
1025 , unsigned int rrdpush_enabled
1026 , char *rrdpush_destination
@@ -1426,14 +1053,14 @@ RRDSET *rrdset_create_custom(RRDHOST *host
1053 , long priority
1054 , int update_every
1055 , RRDSET_TYPE chart_type
1429 - , RRD_MEMORY_MODE memory_mode
1056 + , STORAGE_ENGINE_ID storage_engine_id
1057 , long history_entries);
1058
1059 #define rrdset_create(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
1433 - rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries)
1060 + rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->storage_engine_id, (host)->rrd_history_entries)
1061
1062 #define rrdset_create_localhost(type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
1436 - rrdset_create(localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
1063 + rrdset_create(rrdb.localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
1064
1065 void rrdhost_free_all(void);
1066 void rrdhost_save_all(void);
@@ -1450,44 +1077,14 @@ void rrdset_update_heterogeneous_flag(RRDSET *st);
1077
1078 time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);
1079
1453 -RRDSET *rrdset_find(RRDHOST *host, const char *id);
1080 +RRDSET *rrdset_find_by_id(RRDHOST *host, const char *id);
1081 +RRDSET *rrdset_find_by_type(RRDHOST *host, const char *type, const char *id);
1082 +RRDSET *rrdset_find_by_name(RRDHOST *host, const char *name);
1083
1084 RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);
1085 RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);
1086 void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
1087
1459 -#define rrdset_find_localhost(id) rrdset_find(localhost, id)
1460 -/* This will not return charts that are archived */
1461 -static inline RRDSET *rrdset_find_active_localhost(const char *id)
1462 -{
1463 - RRDSET *st = rrdset_find_localhost(id);
1464 - if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
1465 - return NULL;
1466 - return st;
1467 -}
1468 -
1469 -RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
1470 -#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
1471 -/* This will not return charts that are archived */
1472 -static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
1473 -{
1474 - RRDSET *st = rrdset_find_bytype_localhost(type, id);
1475 - if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
1476 - return NULL;
1477 - return st;
1478 -}
1479 -
1480 -RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
1481 -#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)
1482 -/* This will not return charts that are archived */
1483 -static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
1484 -{
1485 - RRDSET *st = rrdset_find_byname_localhost(name);
1486 - if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
1487 - return NULL;
1488 - return st;
1489 -}
1490 -
1088 void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds);
1089 void rrdset_next_usec(RRDSET *st, usec_t microseconds);
1090 void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);
@@ -1500,7 +1097,7 @@ void rrdset_is_obsolete(RRDSET *st);
1097 void rrdset_isnot_obsolete(RRDSET *st);
1098
1099 // checks if the RRDSET should be offered to viewers
1503 -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
1100 +#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->storage_engine_id != STORAGE_ENGINE_NONE)
1101 #define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
1102 #define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
1103
@@ -1516,31 +1113,24 @@ time_t rrdset_last_entry_s_of_tier(RRDSET *st, size_t tier);
1113
1114 void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_time_s, time_t *last_time_s, time_t now_s, size_t tier);
1115
1116 +void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
1117 +
1118 // ----------------------------------------------------------------------------
1119 // RRD DIMENSION functions
1120
1522 -RRDDIM *rrddim_add_custom(RRDSET *st
1523 - , const char *id
1524 - , const char *name
1525 - , collected_number multiplier
1526 - , collected_number divisor
1527 - , RRD_ALGORITHM algorithm
1528 - , RRD_MEMORY_MODE memory_mode
1529 - );
1530 -
1531 -#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \
1532 - rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
1121 +RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name,
1122 + collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm);
1123
1124 int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name);
1125 int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
1126 int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier);
1127 int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor);
1128
1539 -RRDDIM *rrddim_find(RRDSET *st, const char *id);
1129 +RRDDIM *rrddim_find_by_id(RRDSET *st, const char *id);
1130 +
1131 RRDDIM_ACQUIRED *rrddim_find_and_acquire(RRDSET *st, const char *id);
1132 RRDDIM *rrddim_acquired_to_rrddim(RRDDIM_ACQUIRED *rda);
1133 void rrddim_acquired_release(RRDDIM_ACQUIRED *rda);
1543 -RRDDIM *rrddim_find_active(RRDSET *st, const char *id);
1134
1135 int rrddim_hide(RRDSET *st, const char *id);
1136 int rrddim_unhide(RRDSET *st, const char *id);
@@ -1557,7 +1147,7 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);
1147 void rrdhost_finalize_collection(RRDHOST *host);
1148 void rrd_finalize_collection_for_all_hosts(void);
1149
1560 -long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
1150 +long align_entries_to_pagesize(STORAGE_ENGINE_ID id, long entries);
1151
1152 #ifdef NETDATA_LOG_COLLECTION_ERRORS
1153 #define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__)
@@ -1580,45 +1170,22 @@ void rrdset_free(RRDSET *st);
1170
1171 void rrddim_free(RRDSET *st, RRDDIM *rd);
1172
1583 -#ifdef NETDATA_RRD_INTERNALS
1584 -
1585 -char *rrdhost_cache_dir_for_rrdset_alloc(RRDHOST *host, const char *id);
1173 const char *rrdset_cache_dir(RRDSET *st);
1174
1588 -void rrdset_reset(RRDSET *st);
1589 -void rrdset_delete_obsolete_dimensions(RRDSET *st);
1590 -
1591 -#endif /* NETDATA_RRD_INTERNALS */
1592 -
1175 void set_host_properties(
1594 - RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname,
1176 + RRDHOST *host, int update_every, STORAGE_ENGINE_ID storage_engine_id, const char *registry_hostname,
1177 const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
1178 const char *program_name, const char *program_version);
1179
1180 size_t get_tier_grouping(size_t tier);
1181 void store_metric_collection_completed(void);
1182
1601 -static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to) {
1602 - time_t first_time_s = 0, last_time_s = 0;
1603 - spinlock_lock(&host->retention.spinlock);
1604 - first_time_s = host->retention.first_time_s;
1605 - last_time_s = host->retention.last_time_s;
1606 - spinlock_unlock(&host->retention.spinlock);
1607 -
1608 - if(from)
1609 - *from = first_time_s;
1610 -
1611 - if(to)
1612 - *to = online ? now : last_time_s;
1613 -}
1614 -
1183 // ----------------------------------------------------------------------------
1184 // RRD DB engine declarations
1185
1186 #ifdef ENABLE_DBENGINE
1187 #include "database/engine/rrdengineapi.h"
1188 #endif
1621 -#include "sqlite/sqlite_functions.h"
1189 #include "sqlite/sqlite_context.h"
1190 #include "sqlite/sqlite_metadata.h"
1191 #include "sqlite/sqlite_aclk.h"
database/rrdcalc.c
+1 -1
@@ -802,7 +802,7 @@ void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host
802
803 void rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts() {
804 RRDHOST *host;
805 - dfe_start_reentrant(rrdhost_root_index, host) {
805 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
806 if (unlikely(!host->health.health_enabled))
807 continue;
808
database/rrddim.c
+210 -233
@@ -1,8 +1,181 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
5 -#include "storage_engine.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// compatibility layer for RRDDIM files v019
7 +
8 +#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019"
9 +
10 +struct avl_element_v019 {
11 + void *avl_link[2];
12 + signed char avl_balance;
13 +};
14 +
15 +struct rrddim_map_save_v019 {
16 + struct avl_element_v019 avl; // ignored
17 + void *id; // ignored
18 + void *name; // ignored
19 + uint32_t algorithm; // print warning on mismatch - update on load
20 + uint32_t storage_engine_id; // ignored
21 + long long multiplier; // print warning on mismatch - update on load
22 + long long divisor; // print warning on mismatch - update on load
23 + uint32_t flags; // ignored
24 + uint32_t hash; // ignored
25 + uint32_t hash_name; // ignored
26 + void *cache_filename; // ignored - we use it to keep the filename to save back
27 + size_t collections_counter; // ignored
28 + void *state; // ignored
29 + size_t unused[8]; // ignored
30 + long long collected_value_max; // ignored
31 + unsigned int updated:1; // ignored
32 + unsigned int exposed:1; // ignored
33 + struct timeval last_collected_time; // check to reset all - ignored after load
34 + long double calculated_value; // ignored
35 + long double last_calculated_value; // ignored
36 + long double last_stored_value; // ignored
37 + long long collected_value; // ignored
38 + long long last_collected_value; // load and save
39 + long double collected_volume; // ignored
40 + long double stored_volume; // ignored
41 + void *next; // ignored
42 + void *rrdset; // ignored
43 + long entries; // check to reset all - update on load
44 + int update_every; // check to reset all - update on load
45 + size_t memsize; // check to reset all - update on load
46 + char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
47 + void *variables; // ignored
48 + storage_number values[]; // the array of values
49 +};
50 +
51 +size_t rrddim_memory_file_header_size(void) {
52 + return sizeof(struct rrddim_map_save_v019);
53 +}
54 +
55 +void rrddim_memory_file_update(RRDDIM *rd) {
56 + if(!rd || !rd->db.rd_on_file) return;
57 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
58 +
59 + rd_on_file->last_collected_time.tv_sec = rd->collector.last_collected_time.tv_sec;
60 + rd_on_file->last_collected_time.tv_usec = rd->collector.last_collected_time.tv_usec;
61 + rd_on_file->last_collected_value = rd->collector.last_collected_value;
62 +}
63 +
64 +void rrddim_memory_file_free(RRDDIM *rd) {
65 + if(!rd || !rd->db.rd_on_file) return;
66 +
67 + // needed for storage engine MAP, to save the latest state
68 + rrddim_memory_file_update(rd);
69 +
70 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
71 + __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
72 + freez(rd_on_file->cache_filename);
73 + netdata_munmap(rd_on_file, rd_on_file->memsize);
74 +
75 + // remove the pointers from the RRDDIM
76 + rd->db.rd_on_file = NULL;
77 + rd->db.data = NULL;
78 +}
79 +
80 +const char *rrddim_cache_filename(RRDDIM *rd) {
81 + if(!rd || !rd->db.rd_on_file) return NULL;
82 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
83 + return rd_on_file->cache_filename;
84 +}
85 +
86 +void rrddim_memory_file_save(RRDDIM *rd) {
87 + if(!rd || !rd->db.rd_on_file) return;
88 +
89 + rrddim_memory_file_update(rd);
90 +
91 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
92 + if(rd_on_file->storage_engine_id != STORAGE_ENGINE_SAVE) return;
93 +
94 + memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
95 +}
96 +
97 +static bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd) {
98 + assert(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP);
99 +
100 + struct rrddim_map_save_v019 *rd_on_file = NULL;
101 +
102 + unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number));
103 +
104 + char filename[FILENAME_MAX + 1];
105 + char fullfilename[FILENAME_MAX + 1];
106 + rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
107 + snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename);
108 +
109 + rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(
110 + fullfilename, size, ((st->storage_engine_id == STORAGE_ENGINE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL);
111 +
112 + if(unlikely(!rd_on_file)) return false;
113 +
114 + struct timeval now;
115 + now_realtime_timeval(&now);
116 +
117 + int reset = 0;
118 + rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0';
119 + if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
120 + netdata_log_info("Initializing file %s.", fullfilename);
121 + memset(rd_on_file, 0, size);
122 + reset = 1;
123 + }
124 + else if(rd_on_file->memsize != size) {
125 + netdata_log_error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, (unsigned long int) rd_on_file->memsize);
126 + memset(rd_on_file, 0, size);
127 + reset = 1;
128 + }
129 + else if(rd_on_file->update_every != st->update_every) {
130 + netdata_log_error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
131 + memset(rd_on_file, 0, size);
132 + reset = 1;
133 + }
134 + else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
135 + netdata_log_info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
136 + memset(rd_on_file, 0, size);
137 + reset = 1;
138 + }
139 +
140 + if(!reset) {
141 + rd->collector.last_collected_value = rd_on_file->last_collected_value;
142 +
143 + if(rd_on_file->algorithm != rd->algorithm)
144 + netdata_log_info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
145 + fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
146 +
147 + if(rd_on_file->multiplier != rd->multiplier)
148 + netdata_log_info("File %s does not have the expected multiplier (expected %d, found %ld). "
149 + "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier);
150 +
151 + if(rd_on_file->divisor != rd->divisor)
152 + netdata_log_info("File %s does not have the expected divisor (expected %d, found %ld). "
153 + "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor);
154 + }
155 +
156 + // zero the entire header
157 + memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
158 +
159 + // set the important fields
160 + strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
161 + rd_on_file->algorithm = rd->algorithm;
162 + rd_on_file->multiplier = rd->multiplier;
163 + rd_on_file->divisor = rd->divisor;
164 + rd_on_file->entries = st->db.entries;
165 + rd_on_file->update_every = rd->rrdset->update_every;
166 + rd_on_file->memsize = size;
167 + rd_on_file->storage_engine_id = st->storage_engine_id;
168 + rd_on_file->cache_filename = strdupz(fullfilename);
169 +
170 + __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
171 +
172 + rd->db.data = &rd_on_file->values[0];
173 + rd->db.rd_on_file = rd_on_file;
174 + rd->db.memsize = size;
175 + rrddim_memory_file_update(rd);
176 +
177 + return true;
178 +}
179
180 // ----------------------------------------------------------------------------
181 // RRDDIM index
@@ -14,7 +187,6 @@ struct rrddim_constructor {
187 collected_number multiplier;
188 collected_number divisor;
189 RRD_ALGORITHM algorithm;
17 - RRD_MEMORY_MODE memory_mode;
190
191 enum {
192 RRDDIM_REACT_NONE = 0,
@@ -26,7 +198,7 @@ struct rrddim_constructor {
198
199 // isolated call to appear
200 // separate in statistics
29 -static void *rrddim_alloc_db(size_t entries) {
201 +static storage_number *rrddim_alloc_db(size_t entries) {
202 return callocz(entries, sizeof(storage_number));
203 }
204
@@ -51,21 +223,22 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
223 if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
224 rd->collector.counter = 1;
225
54 - if(ctr->memory_mode == RRD_MEMORY_MODE_MAP || ctr->memory_mode == RRD_MEMORY_MODE_SAVE) {
55 - if(!rrddim_memory_load_or_create_map_save(st, rd, ctr->memory_mode)) {
56 - netdata_log_info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (ctr->memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd));
57 - ctr->memory_mode = RRD_MEMORY_MODE_RAM;
226 + if (st->storage_engine_id == STORAGE_ENGINE_MAP || st->storage_engine_id == STORAGE_ENGINE_SAVE) {
227 + if(!rrddim_memory_load_or_create_map_save(st, rd)) {
228 + netdata_log_info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram",
229 + storage_engine_name(st->storage_engine_id), rrdset_name(st), rrddim_name(rd));
230 + st->storage_engine_id = STORAGE_ENGINE_RAM;
231 }
232 }
233
61 - if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) {
234 + if (st->storage_engine_id == STORAGE_ENGINE_RAM) {
235 size_t entries = st->db.entries;
236 if(!entries) entries = 5;
237
238 rd->db.data = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);
239 if(!rd->db.data) {
240 netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));
68 - ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;
241 + st->storage_engine_id = STORAGE_ENGINE_ALLOC;
242 }
243 else {
244 rd->db.memsize = entries * sizeof(storage_number);
@@ -73,7 +246,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
246 }
247 }
248
76 - if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) {
249 + if (st->storage_engine_id == STORAGE_ENGINE_ALLOC || st->storage_engine_id == STORAGE_ENGINE_NONE) {
250 size_t entries = st->db.entries;
251 if(entries < 5) entries = 5;
252
@@ -82,19 +255,14 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
255 __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);
256 }
257
85 - rd->rrd_memory_mode = ctr->memory_mode;
86 -
258 if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid))))
259 uuid_generate(rd->metric_uuid);
260
261 // initialize the db tiers
262 {
263 size_t initialized = 0;
93 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
94 - STORAGE_ENGINE *eng = host->db[tier].eng;
95 - rd->tiers[tier].backend = eng->backend;
96 - rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
97 - rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
264 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
265 + rd->tiers[tier].db_metric_handle = storage_engine_metric_get_or_create(rd, host->db[tier].id, host->db[tier].instance);
266 storage_point_unset(rd->tiers[tier].virtual_point);
267 initialized++;
268
@@ -111,10 +279,10 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
279 // initialize data collection for all tiers
280 {
281 size_t initialized = 0;
114 - for (size_t tier = 0; tier < storage_tiers; tier++) {
282 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
283 if (rd->tiers[tier].db_metric_handle) {
284 rd->tiers[tier].db_collection_handle =
117 - storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
285 + storage_metric_store_init(st->storage_engine_id, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
286 initialized++;
287 }
288 }
@@ -169,13 +337,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
337 bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
338 size_t tiers_available = 0, tiers_said_no_retention = 0;
339
172 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
340 + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
341 if(!rd->tiers[tier].db_collection_handle)
342 continue;
343
344 tiers_available++;
345
178 - if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle))
346 + if(storage_engine_store_finalize(rd->rrdset->storage_engine_id, rd->tiers[tier].db_collection_handle))
347 tiers_said_no_retention++;
348
349 rd->tiers[tier].db_collection_handle = NULL;
@@ -199,7 +367,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
367
368 netdata_log_debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
369
202 - if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
370 + if (!rrddim_finalize_collection_and_check_retention(rd) && st->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
371 /* This metric has no data and no references */
372 metaqueue_delete_dimension_uuid(&rd->metric_uuid);
373 }
@@ -215,18 +383,17 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
383 // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
384 rrddim_memory_file_free(rd);
385
218 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
386 + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
387 if(!rd->tiers[tier].db_metric_handle) continue;
388
221 - STORAGE_ENGINE* eng = host->db[tier].eng;
222 - eng->api.metric_release(rd->tiers[tier].db_metric_handle);
389 + storage_engine_metric_release(host->db[tier].id, rd->tiers[tier].db_metric_handle);
390 rd->tiers[tier].db_metric_handle = NULL;
391 }
392
393 if(rd->db.data) {
394 __atomic_sub_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);
395
229 - if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
396 + if(st->storage_engine_id == STORAGE_ENGINE_RAM)
397 netdata_munmap(rd->db.data, rd->db.memsize);
398 else
399 freez(rd->db.data);
@@ -250,10 +417,10 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
417 rc += rrddim_set_multiplier(st, rd, ctr->multiplier);
418 rc += rrddim_set_divisor(st, rd, ctr->divisor);
419
253 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
420 + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
421 if (!rd->tiers[tier].db_collection_handle)
422 rd->tiers[tier].db_collection_handle =
256 - storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
423 + storage_metric_store_init(st->storage_engine_id, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
424 }
425
426 if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@@ -290,7 +457,7 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
457 }
458
459 size_t rrddim_size(void) {
293 - return sizeof(RRDDIM) + storage_tiers * sizeof(struct rrddim_tier);
460 + return sizeof(RRDDIM) + rrdb.storage_tiers * sizeof(struct rrddim_tier);
461 }
462
463 void rrddim_index_init(RRDSET *st) {
@@ -317,7 +484,7 @@ static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id) {
484 // ----------------------------------------------------------------------------
485 // RRDDIM - find a dimension
486
320 -inline RRDDIM *rrddim_find(RRDSET *st, const char *id) {
487 +inline RRDDIM *rrddim_find_by_id(RRDSET *st, const char *id) {
488 netdata_log_debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", rrdset_name(st), id);
489
490 return rrddim_index_find(st, id);
@@ -344,15 +511,6 @@ void rrddim_acquired_release(RRDDIM_ACQUIRED *rda) {
511 dictionary_acquired_item_release(rd->rrdset->rrddim_root_index, (const DICTIONARY_ITEM *)rda);
512 }
513
347 -// This will not return dimensions that are archived
348 -RRDDIM *rrddim_find_active(RRDSET *st, const char *id) {
349 - RRDDIM *rd = rrddim_find(st, id);
350 -
351 - if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
352 - return NULL;
353 -
354 - return rd;
355 -}
514
515 // ----------------------------------------------------------------------------
516 // RRDDIM rename a dimension
@@ -419,17 +577,17 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor) {
577 // ----------------------------------------------------------------------------
578
579 time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) {
422 - if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
580 + if (unlikely(tier > rrdb.storage_tiers || !rd->tiers[tier].db_metric_handle))
581 return 0;
582
425 - return storage_engine_latest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle);
583 + return storage_engine_latest_time_s(rd->rrdset->storage_engine_id, rd->tiers[tier].db_metric_handle);
584 }
585
586 // get the timestamp of the last entry in the round-robin database
587 time_t rrddim_last_entry_s(RRDDIM *rd) {
588 time_t latest_time_s = rrddim_last_entry_s_of_tier(rd, 0);
589
432 - for(size_t tier = 1; tier < storage_tiers ;tier++) {
590 + for (size_t tier = 1; tier < rrdb.storage_tiers ;tier++) {
591 if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
592
593 time_t t = rrddim_last_entry_s_of_tier(rd, tier);
@@ -441,16 +599,16 @@ time_t rrddim_last_entry_s(RRDDIM *rd) {
599 }
600
601 time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) {
444 - if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
602 + if (unlikely(tier > rrdb.storage_tiers || !rd->tiers[tier].db_metric_handle))
603 return 0;
604
447 - return storage_engine_oldest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle);
605 + return storage_engine_oldest_time_s(rd->rrdset->storage_engine_id, rd->tiers[tier].db_metric_handle);
606 }
607
608 time_t rrddim_first_entry_s(RRDDIM *rd) {
609 time_t oldest_time_s = 0;
610
453 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
611 + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
612 time_t t = rrddim_first_entry_s_of_tier(rd, tier);
613 if(t != 0 && (oldest_time_s == 0 || t < oldest_time_s))
614 oldest_time_s = t;
@@ -459,14 +617,9 @@ time_t rrddim_first_entry_s(RRDDIM *rd) {
617 return oldest_time_s;
618 }
619
462 -RRDDIM *rrddim_add_custom(RRDSET *st
463 - , const char *id
464 - , const char *name
465 - , collected_number multiplier
466 - , collected_number divisor
467 - , RRD_ALGORITHM algorithm
468 - , RRD_MEMORY_MODE memory_mode
469 - ) {
620 +RRDDIM *rrddim_add(RRDSET *st ,const char *id, const char *name,
621 + collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm)
622 +{
623 struct rrddim_constructor tmp = {
624 .st = st,
625 .id = id,
@@ -474,7 +627,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st
627 .multiplier = multiplier,
628 .divisor = divisor,
629 .algorithm = algorithm,
477 - .memory_mode = memory_mode,
630 };
631
632 RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, rrddim_size(), &tmp);
@@ -497,7 +649,7 @@ int rrddim_hide(RRDSET *st, const char *id) {
649
650 RRDHOST *host = st->rrdhost;
651
500 - RRDDIM *rd = rrddim_find(st, id);
652 + RRDDIM *rd = rrddim_find_by_id(st, id);
653 if(unlikely(!rd)) {
654 netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
655 return 1;
@@ -516,7 +668,7 @@ int rrddim_unhide(RRDSET *st, const char *id) {
668 netdata_log_debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", rrdset_name(st), id);
669
670 RRDHOST *host = st->rrdhost;
519 - RRDDIM *rd = rrddim_find(st, id);
671 + RRDDIM *rd = rrddim_find_by_id(st, id);
672 if(unlikely(!rd)) {
673 netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
674 return 1;
@@ -580,7 +732,7 @@ collected_number rrddim_timed_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *
732
733 collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) {
734 RRDHOST *host = st->rrdhost;
583 - RRDDIM *rd = rrddim_find(st, id);
735 + RRDDIM *rd = rrddim_find_by_id(st, id);
736 if(unlikely(!rd)) {
737 netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
738 return 0;
@@ -590,178 +742,3 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
742 }
743
744
593 -// ----------------------------------------------------------------------------
594 -// compatibility layer for RRDDIM files v019
595 -
596 -#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019"
597 -
598 -struct avl_element_v019 {
599 - void *avl_link[2];
600 - signed char avl_balance;
601 -};
602 -
603 -struct rrddim_map_save_v019 {
604 - struct avl_element_v019 avl; // ignored
605 - void *id; // ignored
606 - void *name; // ignored
607 - uint32_t algorithm; // print warning on mismatch - update on load
608 - uint32_t rrd_memory_mode; // ignored
609 - long long multiplier; // print warning on mismatch - update on load
610 - long long divisor; // print warning on mismatch - update on load
611 - uint32_t flags; // ignored
612 - uint32_t hash; // ignored
613 - uint32_t hash_name; // ignored
614 - void *cache_filename; // ignored - we use it to keep the filename to save back
615 - size_t collections_counter; // ignored
616 - void *state; // ignored
617 - size_t unused[8]; // ignored
618 - long long collected_value_max; // ignored
619 - unsigned int updated:1; // ignored
620 - unsigned int exposed:1; // ignored
621 - struct timeval last_collected_time; // check to reset all - ignored after load
622 - long double calculated_value; // ignored
623 - long double last_calculated_value; // ignored
624 - long double last_stored_value; // ignored
625 - long long collected_value; // ignored
626 - long long last_collected_value; // load and save
627 - long double collected_volume; // ignored
628 - long double stored_volume; // ignored
629 - void *next; // ignored
630 - void *rrdset; // ignored
631 - long entries; // check to reset all - update on load
632 - int update_every; // check to reset all - update on load
633 - size_t memsize; // check to reset all - update on load
634 - char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
635 - void *variables; // ignored
636 - storage_number values[]; // the array of values
637 -};
638 -
639 -size_t rrddim_memory_file_header_size(void) {
640 - return sizeof(struct rrddim_map_save_v019);
641 -}
642 -
643 -void rrddim_memory_file_update(RRDDIM *rd) {
644 - if(!rd || !rd->db.rd_on_file) return;
645 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
646 -
647 - rd_on_file->last_collected_time.tv_sec = rd->collector.last_collected_time.tv_sec;
648 - rd_on_file->last_collected_time.tv_usec = rd->collector.last_collected_time.tv_usec;
649 - rd_on_file->last_collected_value = rd->collector.last_collected_value;
650 -}
651 -
652 -void rrddim_memory_file_free(RRDDIM *rd) {
653 - if(!rd || !rd->db.rd_on_file) return;
654 -
655 - // needed for memory mode map, to save the latest state
656 - rrddim_memory_file_update(rd);
657 -
658 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
659 - __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
660 - freez(rd_on_file->cache_filename);
661 - netdata_munmap(rd_on_file, rd_on_file->memsize);
662 -
663 - // remove the pointers from the RRDDIM
664 - rd->db.rd_on_file = NULL;
665 - rd->db.data = NULL;
666 -}
667 -
668 -const char *rrddim_cache_filename(RRDDIM *rd) {
669 - if(!rd || !rd->db.rd_on_file) return NULL;
670 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
671 - return rd_on_file->cache_filename;
672 -}
673 -
674 -void rrddim_memory_file_save(RRDDIM *rd) {
675 - if(!rd || !rd->db.rd_on_file) return;
676 -
677 - rrddim_memory_file_update(rd);
678 -
679 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
680 - if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
681 -
682 - memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
683 -}
684 -
685 -bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode) {
686 - if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
687 - return false;
688 -
689 - struct rrddim_map_save_v019 *rd_on_file = NULL;
690 -
691 - unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number));
692 -
693 - char filename[FILENAME_MAX + 1];
694 - char fullfilename[FILENAME_MAX + 1];
695 - rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
696 - snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename);
697 -
698 - rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(
699 - fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL);
700 -
701 - if(unlikely(!rd_on_file)) return false;
702 -
703 - struct timeval now;
704 - now_realtime_timeval(&now);
705 -
706 - int reset = 0;
707 - rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0';
708 - if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
709 - netdata_log_info("Initializing file %s.", fullfilename);
710 - memset(rd_on_file, 0, size);
711 - reset = 1;
712 - }
713 - else if(rd_on_file->memsize != size) {
714 - netdata_log_error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, (unsigned long int) rd_on_file->memsize);
715 - memset(rd_on_file, 0, size);
716 - reset = 1;
717 - }
718 - else if(rd_on_file->update_every != st->update_every) {
719 - netdata_log_error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
720 - memset(rd_on_file, 0, size);
721 - reset = 1;
722 - }
723 - else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
724 - netdata_log_info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
725 - memset(rd_on_file, 0, size);
726 - reset = 1;
727 - }
728 -
729 - if(!reset) {
730 - rd->collector.last_collected_value = rd_on_file->last_collected_value;
731 -
732 - if(rd_on_file->algorithm != rd->algorithm)
733 - netdata_log_info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
734 - fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
735 -
736 - if(rd_on_file->multiplier != rd->multiplier)
737 - netdata_log_info("File %s does not have the expected multiplier (expected %d, found %ld). "
738 - "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier);
739 -
740 - if(rd_on_file->divisor != rd->divisor)
741 - netdata_log_info("File %s does not have the expected divisor (expected %d, found %ld). "
742 - "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor);
743 - }
744 -
745 - // zero the entire header
746 - memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
747 -
748 - // set the important fields
749 - strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
750 - rd_on_file->algorithm = rd->algorithm;
751 - rd_on_file->multiplier = rd->multiplier;
752 - rd_on_file->divisor = rd->divisor;
753 - rd_on_file->entries = st->db.entries;
754 - rd_on_file->update_every = rd->rrdset->update_every;
755 - rd_on_file->memsize = size;
756 - rd_on_file->rrd_memory_mode = memory_mode;
757 - rd_on_file->cache_filename = strdupz(fullfilename);
758 -
759 - __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
760 -
761 - rd->db.data = &rd_on_file->values[0];
762 - rd->db.rd_on_file = rd_on_file;
763 - rd->db.memsize = size;
764 - rrddim_memory_file_update(rd);
765 -
766 - return true;
767 -}
database/rrdfamily.c
-1
@@ -1,6 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
4
5 typedef struct rrdfamily {
database/rrdfunctions.c
+4 -3
@@ -1,4 +1,5 @@
1 -#define NETDATA_RRD_INTERNALS
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "rrd.h"
4
5 #define MAX_FUNCTION_LENGTH (PLUGINSD_LINE_MAX - 512) // we need some space for the rest of the line
@@ -816,7 +817,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
817 wb->content_type = CT_APPLICATION_JSON;
818 buffer_json_initialize(wb, "\"", "\"", 0, true, false);
819
819 - buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
820 + buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(rrdb.localhost));
821 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
822 buffer_json_member_add_string(wb, "type", "table");
823 buffer_json_member_add_time_t(wb, "update_every", 1);
@@ -829,7 +830,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
830 size_t max_ml_anomalous = 0, max_ml_normal = 0, max_ml_trained = 0, max_ml_pending = 0, max_ml_silenced = 0;
831 {
832 RRDHOST *host;
832 - dfe_start_read(rrdhost_root_index, host) {
833 + dfe_start_read(rrdb.rrdhost_root_index, host) {
834 RRDHOST_STATUS s;
835 rrdhost_status(host, now, &s);
836 buffer_json_add_array_item_array(wb);
database/rrdfunctions.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef NETDATA_RRDFUNCTIONS_H
4 #define NETDATA_RRDFUNCTIONS_H 1
5
database/rrdhost.c
+132 -570
@@ -1,37 +1,24 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
4
5 static void rrdhost_streaming_sender_structures_init(RRDHOST *host);
6
8 -bool dbengine_enabled = false; // will become true if and when dbengine is initialized
9 -size_t storage_tiers = 3;
10 -bool use_direct_io = true;
11 -size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
12 -RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
13 -
7 #if RRD_STORAGE_TIERS != 5
8 #error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier
9 #endif
10
11 size_t get_tier_grouping(size_t tier) {
19 - if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1;
12 + if(unlikely(tier >= rrdb.storage_tiers)) tier = rrdb.storage_tiers - 1;
13
14 size_t grouping = 1;
15 // first tier is always 1 iteration of whatever update every the chart has
16 for(size_t i = 1; i <= tier ;i++)
24 - grouping *= storage_tiers_grouping_iterations[i];
17 + grouping *= rrdb.storage_tiers_grouping_iterations[i];
18
19 return grouping;
20 }
21
29 -RRDHOST *localhost = NULL;
30 -netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER;
31 -
32 -time_t rrdset_free_obsolete_time_s = 3600;
33 -time_t rrdhost_free_orphan_time_s = 3600;
34 -
22 bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
23 #ifdef ENABLE_DBENGINE
24 if(!rrdeng_is_legacy(engine))
@@ -41,14 +28,13 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
28 return false;
29 }
30
44 -RRDHOST *find_host_by_node_id(char *node_id) {
45 -
31 +RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
32 uuid_t node_uuid;
33 if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
34 return NULL;
35
36 RRDHOST *host, *ret = NULL;
51 - dfe_start_read(rrdhost_root_index, host) {
37 + dfe_start_read(rrdb.rrdhost_root_index, host) {
38 if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) {
39 ret = host;
40 break;
@@ -62,27 +48,10 @@ RRDHOST *find_host_by_node_id(char *node_id) {
48 // ----------------------------------------------------------------------------
49 // RRDHOST indexes management
50
65 -DICTIONARY *rrdhost_root_index = NULL;
66 -static DICTIONARY *rrdhost_root_index_hostname = NULL;
67 -
68 -static inline void rrdhost_init() {
69 - if(unlikely(!rrdhost_root_index)) {
70 - rrdhost_root_index = dictionary_create_advanced(
71 - DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
72 - &dictionary_stats_category_rrdhost, 0);
73 - }
74 -
75 - if(unlikely(!rrdhost_root_index_hostname)) {
76 - rrdhost_root_index_hostname = dictionary_create_advanced(
77 - DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
78 - &dictionary_stats_category_rrdhost, 0);
79 - }
80 -}
81 -
51 RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid) {
52 netdata_log_debug(D_RRD_CALLS, "rrdhost_find_and_acquire() host %s", machine_guid);
53
85 - return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdhost_root_index, machine_guid);
54 + return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdb.rrdhost_root_index, machine_guid);
55 }
56
57 RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha) {
@@ -96,22 +65,22 @@ void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha) {
65 if(unlikely(!rha))
66 return;
67
99 - dictionary_acquired_item_release(rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
68 + dictionary_acquired_item_release(rrdb.rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
69 }
70
71 // ----------------------------------------------------------------------------
72 // RRDHOST index by UUID
73
74 inline size_t rrdhost_hosts_available(void) {
106 - return dictionary_entries(rrdhost_root_index);
75 + return dictionary_entries(rrdb.rrdhost_root_index);
76 }
77
78 inline RRDHOST *rrdhost_find_by_guid(const char *guid) {
110 - return dictionary_get(rrdhost_root_index, guid);
79 + return dictionary_get(rrdb.rrdhost_root_index, guid);
80 }
81
82 static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
114 - RRDHOST *ret_machine_guid = dictionary_set(rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
83 + RRDHOST *ret_machine_guid = dictionary_set(rrdb.rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
84 if(ret_machine_guid == host)
85 rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_MACHINE_GUID);
86 else {
@@ -125,7 +94,7 @@ static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
94
95 static void rrdhost_index_del_by_guid(RRDHOST *host) {
96 if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_MACHINE_GUID)) {
128 - if(!dictionary_del(rrdhost_root_index, host->machine_guid))
97 + if(!dictionary_del(rrdb.rrdhost_root_index, host->machine_guid))
98 netdata_log_error("RRDHOST: %s() failed to delete machine guid '%s' from index",
99 __FUNCTION__, host->machine_guid);
100
@@ -138,16 +107,16 @@ static void rrdhost_index_del_by_guid(RRDHOST *host) {
107
108 inline RRDHOST *rrdhost_find_by_hostname(const char *hostname) {
109 if(unlikely(!strcmp(hostname, "localhost")))
141 - return localhost;
110 + return rrdb.localhost;
111
143 - return dictionary_get(rrdhost_root_index_hostname, hostname);
112 + return dictionary_get(rrdb.rrdhost_root_index_hostname, hostname);
113 }
114
115 static inline void rrdhost_index_del_hostname(RRDHOST *host) {
116 if(unlikely(!host->hostname)) return;
117
118 if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_HOSTNAME)) {
150 - if(!dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)))
119 + if(!dictionary_del(rrdb.rrdhost_root_index_hostname, rrdhost_hostname(host)))
120 netdata_log_error("RRDHOST: %s() failed to delete hostname '%s' from index",
121 __FUNCTION__, rrdhost_hostname(host));
122
@@ -158,7 +127,7 @@ static inline void rrdhost_index_del_hostname(RRDHOST *host) {
127 static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) {
128 if(!host->hostname) return host;
129
161 - RRDHOST *ret_hostname = dictionary_set(rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
130 + RRDHOST *ret_hostname = dictionary_set(rrdb.rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
131 if(ret_hostname == host)
132 rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
133 else {
@@ -226,14 +195,14 @@ static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, co
195 host->utc_offset = utc_offset;
196 }
197
229 -void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode,
198 +void set_host_properties(RRDHOST *host, int update_every, STORAGE_ENGINE_ID storage_engine_id,
199 const char *registry_hostname, const char *os, const char *tags,
200 const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name,
201 const char *program_version)
202 {
203
235 - host->rrd_update_every = update_every;
236 - host->rrd_memory_mode = memory_mode;
204 + host->update_every = update_every;
205 + host->storage_engine_id = storage_engine_id;
206
207 rrdhost_init_os(host, os);
208 rrdhost_init_timezone(host, tzone, abbrev_tzone, utc_offset);
@@ -277,7 +246,7 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host,
246 rrdhost_option_clear(host, RRDHOST_OPTION_SENDER_ENABLED);
247 }
248
280 -static RRDHOST *rrdhost_create(
249 +RRDHOST *rrdhost_create(
250 const char *hostname,
251 const char *registry_hostname,
252 const char *guid,
@@ -290,7 +259,7 @@ static RRDHOST *rrdhost_create(
259 const char *program_version,
260 int update_every,
261 long entries,
293 - RRD_MEMORY_MODE memory_mode,
262 + STORAGE_ENGINE_ID storage_engine_id,
263 unsigned int health_enabled,
264 unsigned int rrdpush_enabled,
265 char *rrdpush_destination,
@@ -305,31 +274,30 @@ static RRDHOST *rrdhost_create(
274 ) {
275 netdata_log_debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
276
308 - if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) {
309 - netdata_log_error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'",
310 - hostname);
311 - memory_mode = RRD_MEMORY_MODE_ALLOC;
277 + if(storage_engine_id == STORAGE_ENGINE_DBENGINE && !rrdb.dbengine_enabled) {
278 + netdata_log_error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'", hostname);
279 + storage_engine_id = STORAGE_ENGINE_ALLOC;
280 }
281
282 #ifdef ENABLE_DBENGINE
315 - int is_legacy = (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid);
283 + int is_legacy = (storage_engine_id == STORAGE_ENGINE_DBENGINE) && is_legacy_child(guid);
284 #else
285 int is_legacy = 1;
286 #endif
287
320 - int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);
288 + int is_in_multihost = (storage_engine_id == STORAGE_ENGINE_DBENGINE && !is_legacy);
289 RRDHOST *host = callocz(1, sizeof(RRDHOST));
290 __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
291
292 strncpyz(host->machine_guid, guid, GUID_LEN + 1);
293
326 - set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, registry_hostname, os,
294 + set_host_properties(host, (update_every > 0)?update_every:1, storage_engine_id, registry_hostname, os,
295 tags, timezone, abbrev_timezone, utc_offset, program_name, program_version);
296
297 rrdhost_init_hostname(host, hostname, false);
298
331 - host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
332 - host->health.health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
299 + host->rrd_history_entries = align_entries_to_pagesize(storage_engine_id, entries);
300 + host->health.health_enabled = ((storage_engine_id == STORAGE_ENGINE_NONE)) ? 0 : health_enabled;
301
302 if (likely(!archived)) {
303 rrdfunctions_init(host);
@@ -347,17 +315,16 @@ int is_legacy = 1;
315 host->rrdpush_replication_step = rrdpush_replication_step;
316 host->rrdpush_receiver_replication_percent = 100.0;
317
350 - switch(memory_mode) {
351 - default:
352 - case RRD_MEMORY_MODE_ALLOC:
353 - case RRD_MEMORY_MODE_MAP:
354 - case RRD_MEMORY_MODE_SAVE:
355 - case RRD_MEMORY_MODE_RAM:
356 - if(host->rrdpush_seconds_to_replicate > (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every)
357 - host->rrdpush_seconds_to_replicate = (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every;
318 + switch(storage_engine_id) {
319 + case STORAGE_ENGINE_DBENGINE:
320 break;
359 -
360 - case RRD_MEMORY_MODE_DBENGINE:
321 + case STORAGE_ENGINE_ALLOC:
322 + case STORAGE_ENGINE_MAP:
323 + case STORAGE_ENGINE_SAVE:
324 + case STORAGE_ENGINE_RAM:
325 + default:
326 + if(host->rrdpush_seconds_to_replicate > (time_t) host->rrd_history_entries * (time_t) host->update_every)
327 + host->rrdpush_seconds_to_replicate = (time_t) host->rrd_history_entries * (time_t) host->update_every;
328 break;
329 }
330
@@ -387,8 +354,8 @@ int is_legacy = 1;
354 host->cache_dir = strdupz(filename);
355 }
356
390 - if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE ||
391 - (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
357 + if((host->storage_engine_id == STORAGE_ENGINE_MAP || host->storage_engine_id == STORAGE_ENGINE_SAVE ||
358 + (host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_legacy))) {
359 int r = mkdir(host->cache_dir, 0775);
360 if(r != 0 && errno != EEXIST)
361 netdata_log_error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir);
@@ -408,7 +375,7 @@ int is_legacy = 1;
375 rrdcalctemplate_index_init(host);
376 rrdcalc_rrdhost_index_init(host);
377
411 - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
378 + if (host->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
379 #ifdef ENABLE_DBENGINE
380 char dbenginepath[FILENAME_MAX + 1];
381 int ret;
@@ -424,35 +391,29 @@ int is_legacy = 1;
391 if (is_legacy) {
392 // initialize legacy dbengine instance as needed
393
427 - host->db[0].mode = RRD_MEMORY_MODE_DBENGINE;
428 - host->db[0].eng = storage_engine_get(host->db[0].mode);
394 + host->db[0].id = STORAGE_ENGINE_DBENGINE;
395 host->db[0].tier_grouping = get_tier_grouping(0);
396
431 - ret = rrdeng_init(
432 - (struct rrdengine_instance **)&host->db[0].instance,
433 - dbenginepath,
434 - default_rrdeng_disk_quota_mb,
435 - 0); // may fail here for legacy dbengine initialization
397 + // may fail here for legacy dbengine initialization
398 + ret = rrdeng_init(&host->db[0].instance, dbenginepath, rrdb.default_rrdeng_disk_quota_mb, 0);
399
400 if(ret == 0) {
438 - rrdeng_readiness_wait((struct rrdengine_instance *)host->db[0].instance);
401 + rrdeng_readiness_wait(host->db[0].instance);
402
403 // assign the rest of the shared storage instances to it
404 // to allow them collect its metrics too
405
443 - for(size_t tier = 1; tier < storage_tiers ; tier++) {
444 - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
445 - host->db[tier].eng = storage_engine_get(host->db[tier].mode);
446 - host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
406 + for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
407 + host->db[tier].id= STORAGE_ENGINE_DBENGINE;
408 + host->db[tier].instance = rrdb.multidb_ctx[tier];
409 host->db[tier].tier_grouping = get_tier_grouping(tier);
410 }
411 }
412 }
413 else {
452 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
453 - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
454 - host->db[tier].eng = storage_engine_get(host->db[tier].mode);
455 - host->db[tier].instance = (STORAGE_INSTANCE *)multidb_ctx[tier];
414 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
415 + host->db[tier].id = STORAGE_ENGINE_DBENGINE;
416 + host->db[tier].instance = rrdb.multidb_ctx[tier];
417 host->db[tier].tier_grouping = get_tier_grouping(tier);
418 }
419 }
@@ -469,21 +430,19 @@ int is_legacy = 1;
430 }
431
432 #else
472 - fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform.");
433 + fatal("STORAGE_ENGINE_DBENGINE is not supported in this platform.");
434 #endif
435 }
436 else {
476 - host->db[0].mode = host->rrd_memory_mode;
477 - host->db[0].eng = storage_engine_get(host->db[0].mode);
437 + host->db[0].id = host->storage_engine_id;
438 host->db[0].instance = NULL;
439 host->db[0].tier_grouping = get_tier_grouping(0);
440
441 #ifdef ENABLE_DBENGINE
442 // the first tier is reserved for the non-dbengine modes
483 - for(size_t tier = 1; tier < storage_tiers ; tier++) {
484 - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
485 - host->db[tier].eng = storage_engine_get(host->db[tier].mode);
486 - host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
443 + for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
444 + host->db[tier].id = STORAGE_ENGINE_DBENGINE;
445 + host->db[tier].instance = rrdb.multidb_ctx[tier];
446 host->db[tier].tier_grouping = get_tier_grouping(tier);
447 }
448 #endif
@@ -517,9 +476,9 @@ int is_legacy = 1;
476 rrdhost_index_add_hostname(host);
477
478 if(is_localhost)
520 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(localhost, host, prev, next);
479 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
480 else
522 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(localhost, host, prev, next);
481 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
482
483 rrd_unlock();
484
@@ -548,8 +507,8 @@ int is_legacy = 1;
507 , rrdhost_tags(host)
508 , rrdhost_program_name(host)
509 , rrdhost_program_version(host)
551 - , host->rrd_update_every
552 - , rrd_memory_mode_name(host->rrd_memory_mode)
510 + , host->update_every
511 + , storage_engine_name(host->storage_engine_id)
512 , host->rrd_history_entries
513 , rrdhost_has_rrdpush_sender_enabled(host)?"enabled":"disabled"
514 , host->rrdpush_send_destination?host->rrdpush_send_destination:""
@@ -584,7 +543,7 @@ static void rrdhost_update(RRDHOST *host
543 , const char *program_version
544 , int update_every
545 , long history
587 - , RRD_MEMORY_MODE mode
546 + , STORAGE_ENGINE_ID storage_engine_id
547 , unsigned int health_enabled
548 , unsigned int rrdpush_enabled
549 , char *rrdpush_destination
@@ -600,7 +559,7 @@ static void rrdhost_update(RRDHOST *host
559
560 spinlock_lock(&host->rrdhost_update_lock);
561
603 - host->health.health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;
562 + host->health.health_enabled = (storage_engine_id == STORAGE_ENGINE_NONE) ? 0 : health_enabled;
563
564 {
565 struct rrdhost_system_info *old = host->system_info;
@@ -636,24 +595,20 @@ static void rrdhost_update(RRDHOST *host
595 string_freez(t);
596 }
597
639 - if(host->rrd_update_every != update_every)
598 + if(host->update_every != update_every)
599 netdata_log_error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. "
600 "Restart netdata here to apply the new settings.",
642 - rrdhost_hostname(host), host->rrd_update_every, update_every);
601 + rrdhost_hostname(host), host->update_every, update_every);
602
644 - if(host->rrd_memory_mode != mode)
603 + if(host->storage_engine_id != storage_engine_id) {
604 netdata_log_error("Host '%s' has memory mode '%s', but the wanted one is '%s'. "
605 "Restart netdata here to apply the new settings.",
647 - rrdhost_hostname(host),
648 - rrd_memory_mode_name(host->rrd_memory_mode),
649 - rrd_memory_mode_name(mode));
650 -
651 - else if(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && host->rrd_history_entries < history)
606 + rrdhost_hostname(host), storage_engine_name(host->storage_engine_id), storage_engine_name(storage_engine_id));
607 + } else if(host->storage_engine_id != STORAGE_ENGINE_DBENGINE && host->rrd_history_entries < history) {
608 netdata_log_error("Host '%s' has history of %d entries, but the wanted one is %ld entries. "
609 "Restart netdata here to apply the new settings.",
654 - rrdhost_hostname(host),
655 - host->rrd_history_entries,
656 - history);
610 + rrdhost_hostname(host), host->rrd_history_entries, history);
611 + }
612
613 // update host tags
614 rrdhost_init_tags(host, tags);
@@ -699,7 +654,7 @@ static void rrdhost_update(RRDHOST *host
654 spinlock_unlock(&host->rrdhost_update_lock);
655 }
656
702 -RRDHOST *rrdhost_find_or_create(
657 +RRDHOST *rrdhost_get_or_create(
658 const char *hostname
659 , const char *registry_hostname
660 , const char *guid
@@ -712,7 +667,7 @@ RRDHOST *rrdhost_find_or_create(
667 , const char *program_version
668 , int update_every
669 , long history
715 - , RRD_MEMORY_MODE mode
670 + , STORAGE_ENGINE_ID storage_engine_id
671 , unsigned int health_enabled
672 , unsigned int rrdpush_enabled
673 , char *rrdpush_destination
@@ -727,16 +682,16 @@ RRDHOST *rrdhost_find_or_create(
682 netdata_log_debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);
683
684 RRDHOST *host = rrdhost_find_by_guid(guid);
730 - if (unlikely(host && host->rrd_memory_mode != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
685 + if (unlikely(host && host->storage_engine_id != storage_engine_id && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
686
687 if (likely(!archived && rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
688 return host;
689
735 - /* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */
690 + /* If a legacy storage engine instantiates all dbengine state must be discarded to avoid inconsistencies */
691 netdata_log_error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",
692 rrdhost_hostname(host),
738 - rrd_memory_mode_name(host->rrd_memory_mode),
739 - rrd_memory_mode_name(mode));
693 + storage_engine_name(host->storage_engine_id),
694 + storage_engine_name(storage_engine_id));
695
696 rrd_wrlock();
697 rrdhost_free___while_having_rrd_wrlock(host, true);
@@ -758,7 +713,7 @@ RRDHOST *rrdhost_find_or_create(
713 , program_version
714 , update_every
715 , history
761 - , mode
716 + , storage_engine_id
717 , health_enabled
718 , rrdpush_enabled
719 , rrdpush_destination
@@ -787,7 +742,7 @@ RRDHOST *rrdhost_find_or_create(
742 , program_version
743 , update_every
744 , history
790 - , mode
745 + , storage_engine_id
746 , health_enabled
747 , rrdpush_enabled
748 , rrdpush_destination
@@ -804,284 +759,19 @@ RRDHOST *rrdhost_find_or_create(
759
760 inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s) {
761 if(host != protected_host
807 - && host != localhost
762 + && host != rrdb.localhost
763 && rrdhost_receiver_replicating_charts(host) == 0
764 && rrdhost_sender_replicating_charts(host) == 0
765 && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)
766 && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)
767 && !host->receiver
768 && host->child_disconnected_time
814 - && host->child_disconnected_time + rrdhost_free_orphan_time_s < now_s)
769 + && host->child_disconnected_time + rrdb.rrdhost_free_orphan_time_s < now_s)
770 return 1;
771
772 return 0;
773 }
774
820 -// ----------------------------------------------------------------------------
821 -// RRDHOST global / startup initialization
822 -
823 -#ifdef ENABLE_DBENGINE
824 -struct dbengine_initialization {
825 - netdata_thread_t thread;
826 - char path[FILENAME_MAX + 1];
827 - int disk_space_mb;
828 - size_t tier;
829 - int ret;
830 -};
831 -
832 -void *dbengine_tier_init(void *ptr) {
833 - struct dbengine_initialization *dbi = ptr;
834 - dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);
835 - return ptr;
836 -}
837 -#endif
838 -
839 -void dbengine_init(char *hostname) {
840 -#ifdef ENABLE_DBENGINE
841 - use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", use_direct_io);
842 -
843 - unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
844 - if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
845 - rrdeng_pages_per_extent = read_num;
846 - else {
847 - netdata_log_error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
848 - config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
849 - }
850 -
851 - storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
852 - if(storage_tiers < 1) {
853 - netdata_log_error("At least 1 storage tier is required. Assuming 1.");
854 - storage_tiers = 1;
855 - config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
856 - }
857 - if(storage_tiers > RRD_STORAGE_TIERS) {
858 - netdata_log_error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS);
859 - storage_tiers = RRD_STORAGE_TIERS;
860 - config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
861 - }
862 -
863 - bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
864 - parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
865 -
866 - struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
867 -
868 - size_t created_tiers = 0;
869 - char dbenginepath[FILENAME_MAX + 1];
870 - char dbengineconfig[200 + 1];
871 - int divisor = 1;
872 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
873 - if(tier == 0)
874 - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
875 - else
876 - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
877 -
878 - int ret = mkdir(dbenginepath, 0775);
879 - if (ret != 0 && errno != EEXIST) {
880 - netdata_log_error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
881 - break;
882 - }
883 -
884 - if(tier > 0)
885 - divisor *= 2;
886 -
887 - int disk_space_mb = default_multidb_disk_quota_mb / divisor;
888 - size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
889 - RRD_BACKFILL backfill = storage_tiers_backfill[tier];
890 -
891 - if(tier > 0) {
892 - snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
893 - disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
894 -
895 - snprintfz(dbengineconfig, 200, "dbengine tier %zu update every iterations", tier);
896 - grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
897 - if(grouping_iterations < 2) {
898 - grouping_iterations = 2;
899 - config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
900 - netdata_log_error("DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",
901 - hostname,
902 - tier);
903 - }
904 -
905 - snprintfz(dbengineconfig, 200, "dbengine tier %zu backfill", tier);
906 - const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
907 - if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
908 - else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
909 - else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;
910 - else {
911 - netdata_log_error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf);
912 - config_set(CONFIG_SECTION_DB, dbengineconfig, "new");
913 - backfill = RRD_BACKFILL_NEW;
914 - }
915 - }
916 -
917 - storage_tiers_grouping_iterations[tier] = grouping_iterations;
918 - storage_tiers_backfill[tier] = backfill;
919 -
920 - if(tier > 0 && get_tier_grouping(tier) > 65535) {
921 - storage_tiers_grouping_iterations[tier] = 1;
922 - netdata_log_error("DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. Disabling tiers above %zu",
923 - hostname,
924 - tier,
925 - tier);
926 - break;
927 - }
928 -
929 - internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);
930 -
931 - tiers_init[tier].disk_space_mb = disk_space_mb;
932 - tiers_init[tier].tier = tier;
933 - strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
934 - tiers_init[tier].ret = 0;
935 -
936 - if(parallel_initialization) {
937 - char tag[NETDATA_THREAD_TAG_MAX + 1];
938 - snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier);
939 - netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE,
940 - dbengine_tier_init, &tiers_init[tier]);
941 - }
942 - else
943 - dbengine_tier_init(&tiers_init[tier]);
944 - }
945 -
946 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
947 - void *ptr;
948 -
949 - if(parallel_initialization)
950 - netdata_thread_join(tiers_init[tier].thread, &ptr);
951 -
952 - if(tiers_init[tier].ret != 0) {
953 - netdata_log_error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
954 - hostname,
955 - tiers_init[tier].tier,
956 - tiers_init[tier].path);
957 - }
958 - else if(created_tiers == tier)
959 - created_tiers++;
960 - }
961 -
962 - if(created_tiers && created_tiers < storage_tiers) {
963 - netdata_log_error("DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.",
964 - hostname,
965 - created_tiers,
966 - storage_tiers,
967 - created_tiers);
968 - storage_tiers = created_tiers;
969 - }
970 - else if(!created_tiers)
971 - fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
972 -
973 - for(size_t tier = 0; tier < storage_tiers ;tier++)
974 - rrdeng_readiness_wait(multidb_ctx[tier]);
975 -
976 - dbengine_enabled = true;
977 -#else
978 - storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
979 - if(storage_tiers != 1) {
980 - netdata_log_error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname);
981 - storage_tiers = 1;
982 - config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
983 - }
984 - dbengine_enabled = false;
985 -#endif
986 -}
987 -
988 -int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
989 - rrdhost_init();
990 -
991 - if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
992 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
993 - set_late_global_environment(system_info);
994 - fatal("Failed to initialize SQLite");
995 - }
996 - netdata_log_info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
997 - }
998 -
999 - if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
1000 - error_report("Failed to initialize context metadata database");
1001 - }
1002 -
1003 - if (unlikely(unittest)) {
1004 - dbengine_enabled = true;
1005 - }
1006 - else {
1007 - health_init();
1008 - rrdpush_init();
1009 -
1010 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
1011 - netdata_log_info("DBENGINE: Initializing ...");
1012 - dbengine_init(hostname);
1013 - }
1014 - else {
1015 - netdata_log_info("DBENGINE: Not initializing ...");
1016 - storage_tiers = 1;
1017 - }
1018 -
1019 - if (!dbengine_enabled) {
1020 - if (storage_tiers > 1) {
1021 - netdata_log_error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
1022 - storage_tiers);
1023 - storage_tiers = 1;
1024 - }
1025 -
1026 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1027 - netdata_log_error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
1028 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
1029 - }
1030 - }
1031 - }
1032 -
1033 - if(!unittest)
1034 - metadata_sync_init();
1035 -
1036 - netdata_log_debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
1037 - localhost = rrdhost_create(
1038 - hostname
1039 - , registry_get_this_machine_hostname()
1040 - , registry_get_this_machine_guid()
1041 - , os_type
1042 - , netdata_configured_timezone
1043 - , netdata_configured_abbrev_timezone
1044 - , netdata_configured_utc_offset
1045 - , ""
1046 - , program_name
1047 - , program_version
1048 - , default_rrd_update_every
1049 - , default_rrd_history_entries
1050 - , default_rrd_memory_mode
1051 - , default_health_enabled
1052 - , default_rrdpush_enabled
1053 - , default_rrdpush_destination
1054 - , default_rrdpush_api_key
1055 - , default_rrdpush_send_charts_matching
1056 - , default_rrdpush_enable_replication
1057 - , default_rrdpush_seconds_to_replicate
1058 - , default_rrdpush_replication_step
1059 - , system_info
1060 - , 1
1061 - , 0
1062 - );
1063 -
1064 - if (unlikely(!localhost)) {
1065 - return 1;
1066 - }
1067 -
1068 -#ifdef NETDATA_DEV_MODE
1069 - // we register this only on localhost
1070 - // for the other nodes, the origin server should register it
1071 - rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
1072 - rrd_collector_add_function(localhost, NULL, "streaming", 10,
1073 - RRDFUNCTIONS_STREAMING_HELP, true,
1074 - rrdhost_function_streaming, NULL);
1075 -#endif
1076 -
1077 - if (likely(system_info)) {
1078 - migrate_localhost(&localhost->host_uuid);
1079 - sql_aclk_sync_init();
1080 - web_client_api_v1_management_init();
1081 - }
1082 - return localhost==NULL;
1083 -}
1084 -
775 // ----------------------------------------------------------------------------
776 // RRDHOST - free
777
@@ -1184,7 +874,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
874 rrdhost_index_del_by_guid(host);
875
876 if (host->prev)
1187 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(localhost, host, prev, next);
877 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
878 }
879
880 // ------------------------------------------------------------------------
@@ -1205,11 +895,11 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
895 // release its children resources
896
897 #ifdef ENABLE_DBENGINE
1208 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
1209 - if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
898 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
899 + if(host->db[tier].id == STORAGE_ENGINE_DBENGINE
900 && host->db[tier].instance
901 && !is_storage_engine_shared(host->db[tier].instance))
1212 - rrdeng_prepare_exit((struct rrdengine_instance *)host->db[tier].instance);
902 + rrdeng_prepare_exit(host->db[tier].instance);
903 }
904 #endif
905
@@ -1226,11 +916,11 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
916 health_alarm_log_free(host);
917
918 #ifdef ENABLE_DBENGINE
1229 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
1230 - if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
919 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
920 + if(host->db[tier].id == STORAGE_ENGINE_DBENGINE
921 && host->db[tier].instance
922 && !is_storage_engine_shared(host->db[tier].instance))
1233 - rrdeng_exit((struct rrdengine_instance *)host->db[tier].instance);
923 + rrdeng_exit(host->db[tier].instance);
924 }
925 #endif
926
@@ -1267,7 +957,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
957 rrdfamily_index_destroy(host);
958 rrdfunctions_destroy(host);
959 rrdvariables_destroy(host->rrdvars);
1270 - if (host == localhost)
960 + if (host == rrdb.localhost)
961 rrdvariables_destroy(health_rrdvars);
962
963 rrdhost_destroy_rrdcontexts(host);
@@ -1281,18 +971,18 @@ void rrdhost_free_all(void) {
971 rrd_wrlock();
972
973 /* Make sure child-hosts are released before the localhost. */
1284 - while(localhost && localhost->next)
1285 - rrdhost_free___while_having_rrd_wrlock(localhost->next, true);
974 + while (rrdb.localhost && rrdb.localhost->next)
975 + rrdhost_free___while_having_rrd_wrlock(rrdb.localhost->next, true);
976
1287 - if(localhost)
1288 - rrdhost_free___while_having_rrd_wrlock(localhost, true);
977 + if (rrdb.localhost)
978 + rrdhost_free___while_having_rrd_wrlock(rrdb.localhost, true);
979
980 rrd_unlock();
981 }
982
983 void rrd_finalize_collection_for_all_hosts(void) {
984 RRDHOST *host;
1295 - dfe_start_reentrant(rrdhost_root_index, host) {
985 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
986 rrdhost_finalize_collection(host);
987 }
988 dfe_done(host);
@@ -1316,181 +1006,22 @@ void rrdhost_save_charts(RRDHOST *host) {
1006 rrdset_foreach_done(st);
1007 }
1008
1319 -struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
1320 - struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
1321 - info->hops = 1;
1322 -
1323 - rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type");
1324 - rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type");
1325 - rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region");
1326 - rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name");
1327 - rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version");
1328 - rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version");
1329 - rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores");
1330 - rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq");
1331 - rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total");
1332 - rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space");
1333 - rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture");
1334 - rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization");
1335 - rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container");
1336 - rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection");
1337 - rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection");
1338 - rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node");
1339 - rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type");
1340 - rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch");
1341 - rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist");
1342 -
1343 - return info;
1344 -}
1345 -
1346 -static void rrdhost_load_auto_labels(void) {
1347 - DICTIONARY *labels = localhost->rrdlabels;
1348 -
1349 - if (localhost->system_info->cloud_provider_type)
1350 - rrdlabels_add(labels, "_cloud_provider_type", localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
1351 -
1352 - if (localhost->system_info->cloud_instance_type)
1353 - rrdlabels_add(labels, "_cloud_instance_type", localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
1354 -
1355 - if (localhost->system_info->cloud_instance_region)
1356 - rrdlabels_add(labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
1357 -
1358 - if (localhost->system_info->host_os_name)
1359 - rrdlabels_add(labels, "_os_name", localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
1360 -
1361 - if (localhost->system_info->host_os_version)
1362 - rrdlabels_add(labels, "_os_version", localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO);
1363 -
1364 - if (localhost->system_info->kernel_version)
1365 - rrdlabels_add(labels, "_kernel_version", localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO);
1366 -
1367 - if (localhost->system_info->host_cores)
1368 - rrdlabels_add(labels, "_system_cores", localhost->system_info->host_cores, RRDLABEL_SRC_AUTO);
1369 -
1370 - if (localhost->system_info->host_cpu_freq)
1371 - rrdlabels_add(labels, "_system_cpu_freq", localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO);
1372 -
1373 - if (localhost->system_info->host_ram_total)
1374 - rrdlabels_add(labels, "_system_ram_total", localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO);
1375 -
1376 - if (localhost->system_info->host_disk_space)
1377 - rrdlabels_add(labels, "_system_disk_space", localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO);
1378 -
1379 - if (localhost->system_info->architecture)
1380 - rrdlabels_add(labels, "_architecture", localhost->system_info->architecture, RRDLABEL_SRC_AUTO);
1381 -
1382 - if (localhost->system_info->virtualization)
1383 - rrdlabels_add(labels, "_virtualization", localhost->system_info->virtualization, RRDLABEL_SRC_AUTO);
1384 -
1385 - if (localhost->system_info->container)
1386 - rrdlabels_add(labels, "_container", localhost->system_info->container, RRDLABEL_SRC_AUTO);
1387 -
1388 - if (localhost->system_info->container_detection)
1389 - rrdlabels_add(labels, "_container_detection", localhost->system_info->container_detection, RRDLABEL_SRC_AUTO);
1390 -
1391 - if (localhost->system_info->virt_detection)
1392 - rrdlabels_add(labels, "_virt_detection", localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO);
1393 -
1394 - if (localhost->system_info->is_k8s_node)
1395 - rrdlabels_add(labels, "_is_k8s_node", localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO);
1396 -
1397 - if (localhost->system_info->install_type)
1398 - rrdlabels_add(labels, "_install_type", localhost->system_info->install_type, RRDLABEL_SRC_AUTO);
1399 -
1400 - if (localhost->system_info->prebuilt_arch)
1401 - rrdlabels_add(labels, "_prebuilt_arch", localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO);
1402 -
1403 - if (localhost->system_info->prebuilt_dist)
1404 - rrdlabels_add(labels, "_prebuilt_dist", localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO);
1405 -
1406 - add_aclk_host_labels();
1407 -
1408 - health_add_host_labels();
1409 -
1410 - rrdlabels_add(labels, "_is_parent", (localhost->connected_children_count > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);
1411 -
1412 - if (localhost->rrdpush_send_destination)
1413 - rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
1414 -}
1415 -
1009 void rrdhost_set_is_parent_label(void) {
1417 - int count = __atomic_load_n(&localhost->connected_children_count, __ATOMIC_RELAXED);
1010 + int count = __atomic_load_n(&rrdb.localhost->connected_children_count, __ATOMIC_RELAXED);
1011
1012 if (count == 0 || count == 1) {
1420 - DICTIONARY *labels = localhost->rrdlabels;
1013 + DICTIONARY *labels = rrdb.localhost->rrdlabels;
1014 rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);
1015
1016 //queue a node info
1017 #ifdef ENABLE_ACLK
1018 if (netdata_cloud_enabled) {
1426 - aclk_queue_node_info(localhost, false);
1019 + aclk_queue_node_info(rrdb.localhost, false);
1020 }
1021 #endif
1022 }
1023 }
1024
1432 -static void rrdhost_load_config_labels(void) {
1433 - int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
1434 - if(!status) {
1435 - char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
1436 - netdata_log_error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
1437 - }
1438 -
1439 - struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
1440 - if(co) {
1441 - config_section_wrlock(co);
1442 - struct config_option *cv;
1443 - for(cv = co->values; cv ; cv = cv->next) {
1444 - rrdlabels_add(localhost->rrdlabels, cv->name, cv->value, RRDLABEL_SRC_CONFIG);
1445 - cv->flags |= CONFIG_VALUE_USED;
1446 - }
1447 - config_section_unlock(co);
1448 - }
1449 -}
1450 -
1451 -static void rrdhost_load_kubernetes_labels(void) {
1452 - char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
1453 - sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
1454 -
1455 - if (unlikely(access(label_script, R_OK) != 0)) {
1456 - netdata_log_error("Kubernetes pod label fetching script %s not found.",label_script);
1457 - return;
1458 - }
1459 -
1460 - netdata_log_debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
1461 -
1462 - pid_t pid;
1463 - FILE *fp_child_input;
1464 - FILE *fp_child_output = netdata_popen(label_script, &pid, &fp_child_input);
1465 - if(!fp_child_output) return;
1466 -
1467 - char buffer[1000 + 1];
1468 - while (fgets(buffer, 1000, fp_child_output) != NULL)
1469 - rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);
1470 -
1471 - // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
1472 - // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
1473 - int rc = netdata_pclose(fp_child_input, fp_child_output, pid);
1474 - if(rc)
1475 - netdata_log_error("%s exited abnormally. Failed to get kubernetes labels.", label_script);
1476 -}
1477 -
1478 -void reload_host_labels(void) {
1479 - if(!localhost->rrdlabels)
1480 - localhost->rrdlabels = rrdlabels_create();
1481 -
1482 - rrdlabels_unmark_all(localhost->rrdlabels);
1483 -
1484 - // priority is important here
1485 - rrdhost_load_config_labels();
1486 - rrdhost_load_kubernetes_labels();
1487 - rrdhost_load_auto_labels();
1488 -
1489 - rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1490 -
1491 - rrdpush_send_host_labels(localhost);
1492 -}
1493 -
1025 void rrdhost_finalize_collection(RRDHOST *host) {
1026 netdata_log_info("RRD: 'host:%s' stopping data collection...", rrdhost_hostname(host));
1027
@@ -1510,7 +1041,7 @@ void rrdhost_delete_charts(RRDHOST *host) {
1041
1042 RRDSET *st;
1043
1513 - if(host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
1044 + if(host->storage_engine_id == STORAGE_ENGINE_SAVE || host->storage_engine_id == STORAGE_ENGINE_MAP) {
1045 // we get a write lock
1046 // to ensure only one thread is saving the database
1047 rrdset_foreach_write(st, host){
@@ -1525,6 +1056,23 @@ void rrdhost_delete_charts(RRDHOST *host) {
1056 // ----------------------------------------------------------------------------
1057 // RRDHOST - cleanup host files
1058
1059 +static void delete_obsolete_dimensions(RRDSET *st) {
1060 + RRDDIM *rd;
1061 +
1062 + netdata_log_info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
1063 +
1064 + rrddim_foreach_read(rd, st) {
1065 + if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
1066 + const char *cache_filename = rrddim_cache_filename(rd);
1067 + if(!cache_filename) continue;
1068 + netdata_log_info("Deleting dimension file '%s'.", cache_filename);
1069 + if(unlikely(unlink(cache_filename) == -1))
1070 + netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
1071 + }
1072 + }
1073 + rrddim_foreach_done(rd);
1074 +}
1075 +
1076 void rrdhost_cleanup_charts(RRDHOST *host) {
1077 if(!host) return;
1078
@@ -1541,7 +1089,7 @@ void rrdhost_cleanup_charts(RRDHOST *host) {
1089 rrdset_delete_files(st);
1090
1091 else if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))
1544 - rrdset_delete_obsolete_dimensions(st);
1092 + delete_obsolete_dimensions(st);
1093
1094 else
1095 rrdset_save(st);
@@ -1576,9 +1124,9 @@ void rrdhost_cleanup_all(void) {
1124
1125 RRDHOST *host;
1126 rrdhost_foreach_read(host) {
1579 - if (host != localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver
1127 + if (host != rrdb.localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver
1128 /* don't delete multi-host DB host files */
1581 - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
1129 + && !(host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
1130 )
1131 rrdhost_delete_charts(host);
1132 else
@@ -1743,6 +1291,20 @@ static NETDATA_DOUBLE rrdhost_sender_replication_completion_unsafe(RRDHOST *host
1291 return completion;
1292 }
1293
1294 +static void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to)
1295 +{
1296 + spinlock_lock(&host->retention.spinlock);
1297 + time_t first_time_s = host->retention.first_time_s;
1298 + time_t last_time_s = host->retention.last_time_s;
1299 + spinlock_unlock(&host->retention.spinlock);
1300 +
1301 + if (from)
1302 + *from = first_time_s;
1303 +
1304 + if (to)
1305 + *to = online ? now : last_time_s;
1306 +}
1307 +
1308 bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) {
1309 time_t first_time_s, last_time_s;
1310 rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s);
@@ -1775,7 +1337,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1337 else
1338 s->db.status = RRDHOST_DB_STATUS_QUERYABLE;
1339
1778 - s->db.mode = host->rrd_memory_mode;
1340 + s->db.storage_engine_id = host->storage_engine_id;
1341
1342 // --- ingest ---
1343
@@ -1783,7 +1345,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1345 s->ingest.reason = (online) ? STREAM_HANDSHAKE_NEVER : host->rrdpush_last_receiver_exit_reason;
1346
1347 netdata_mutex_lock(&host->receiver_lock);
1786 - s->ingest.hops = (host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
1348 + s->ingest.hops = (host->system_info ? host->system_info->hops : (host == rrdb.localhost) ? 0 : 1);
1349 bool has_receiver = false;
1350 if (host->receiver) {
1351 has_receiver = true;
@@ -1803,7 +1365,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1365 if(s->db.status == RRDHOST_DB_STATUS_INITIALIZING)
1366 s->ingest.status = RRDHOST_INGEST_STATUS_INITIALIZING;
1367
1806 - else if (host == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) {
1368 + else if (host == rrdb.localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) {
1369 s->ingest.status = RRDHOST_INGEST_STATUS_ONLINE;
1370 s->ingest.since = netdata_start_time;
1371 }
@@ -1824,7 +1386,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1386 s->ingest.status = RRDHOST_INGEST_STATUS_OFFLINE;
1387 }
1388
1827 - if(host == localhost)
1389 + if(host == rrdb.localhost)
1390 s->ingest.type = RRDHOST_INGEST_TYPE_LOCALHOST;
1391 else if(has_receiver || rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
1392 s->ingest.type = RRDHOST_INGEST_TYPE_CHILD;
database/rrdlabels.c
+1 -31
@@ -1,7 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
4 -#include "rrd.h"
3 +#include "rrdlabels.h"
4
5 // ----------------------------------------------------------------------------
6 // labels sanitization
@@ -634,23 +633,6 @@ void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) {
633 rrdlabels_add(dict, name, value, ls);
634 }
635
637 -// ----------------------------------------------------------------------------
638 -// rrdlabels_get_value_to_buffer_or_null()
639 -
640 -void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null) {
641 - if(!labels) return;
642 -
643 - const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
644 - RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
645 -
646 - if(lb && lb->label_value)
647 - buffer_sprintf(wb, "%s%s%s", quote, string2str(lb->label_value), quote);
648 - else
649 - buffer_strcat(wb, null);
650 -
651 - dictionary_acquired_item_release(labels, acquired_item);
652 -}
653 -
636 void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key) {
637 if(!labels) return;
638
@@ -1030,18 +1012,6 @@ void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb) {
1012 dfe_done(lb);
1013 }
1014
1033 -void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
1034 - if(!st->rrdlabels)
1035 - st->rrdlabels = rrdlabels_create();
1036 -
1037 - if (new_rrdlabels)
1038 - rrdlabels_migrate_to_these(st->rrdlabels, new_rrdlabels);
1039 -
1040 - rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
1041 - rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
1042 -}
1043 -
1044 -
1015 // ----------------------------------------------------------------------------
1016 // rrdlabels unit test
1017
database/rrdlabels.h new
+60
@@ -0,0 +1,60 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDLABELS_H
4 +#define NETDATA_RRDLABELS_H
5 +
6 +#include "rrd.h"
7 +
8 +typedef enum __attribute__ ((__packed__)) {
9 + RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
10 + RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
11 + RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
12 + RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
13 +
14 + // more sources can be added here
15 +
16 + RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
17 + RRDLABEL_FLAG_OLD = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
18 + RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels
19 +} RRDLABEL_SRC;
20 +
21 +#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
22 +
23 +size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
24 +
25 +DICTIONARY *rrdlabels_create(void);
26 +void rrdlabels_destroy(DICTIONARY *labels_dict);
27 +void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
28 +void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
29 +void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key);
30 +void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key);
31 +void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key);
32 +STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key);
33 +STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset);
34 +void rrdlabels_flush(DICTIONARY *labels_dict);
35 +
36 +void rrdlabels_unmark_all(DICTIONARY *labels);
37 +void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
38 +
39 +int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
40 +int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
41 +
42 +void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
43 +bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
44 +
45 +bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
46 +int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them,
47 + bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data,
48 + void (*name_sanitizer)(char *dst, const char *src, size_t dst_size),
49 + void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
50 +void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb);
51 +
52 +void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
53 +void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
54 +
55 +int rrdlabels_unittest(void);
56 +
57 +// unfortunately this break when defined in exporting_engine.h
58 +bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
59 +
60 +#endif /* NETDATA_RRDLABELS_H */
database/rrdset.c
+335 -324
@@ -1,9 +1,290 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
4 #include <sched.h>
6 -#include "storage_engine.h"
5 +
6 +#ifdef NETDATA_INTERNAL_CHECKS
7 +#define rrdset_debug(st, fmt, args...) do { \
8 + if(unlikely((debug_flags & D_RRD_STATS) && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
9 + debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); \
10 +} while(0)
11 +#else
12 +#define rrdset_debug(st, fmt, args...) debug_dummy()
13 +#endif
14 +
15 +// ----------------------------------------------------------------------------
16 +// RRDSET - helpers for rrdset_create()
17 +
18 +inline long align_entries_to_pagesize(STORAGE_ENGINE_ID id, long entries) {
19 + if(id == STORAGE_ENGINE_DBENGINE)
20 + return 0;
21 +
22 + if(id == STORAGE_ENGINE_NONE)
23 + return 5;
24 +
25 + if(entries < 5) entries = 5;
26 + if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
27 +
28 + if(id == STORAGE_ENGINE_MAP || id == STORAGE_ENGINE_SAVE || id == STORAGE_ENGINE_RAM) {
29 + long header_size = 0;
30 +
31 + if(id == STORAGE_ENGINE_MAP || id == STORAGE_ENGINE_SAVE)
32 + header_size = (long)rrddim_memory_file_header_size();
33 +
34 + long page = (long)sysconf(_SC_PAGESIZE);
35 + long size = (long)(header_size + entries * sizeof(storage_number));
36 + if (unlikely(size % page)) {
37 + size -= (size % page);
38 + size += page;
39 +
40 + long n = (long)((size - header_size) / sizeof(storage_number));
41 + return n;
42 + }
43 + }
44 +
45 + return entries;
46 +}
47 +
48 +static inline void last_collected_time_align(RRDSET *st) {
49 + st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
50 +
51 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
52 + st->last_collected_time.tv_usec = 0;
53 + else
54 + st->last_collected_time.tv_usec = 500000;
55 +}
56 +
57 +static inline void last_updated_time_align(RRDSET *st) {
58 + st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
59 + st->last_updated.tv_usec = 0;
60 +}
61 +
62 +// ----------------------------------------------------------------------------
63 +// compatibility layer for RRDSET files v019
64 +
65 +#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
66 +#define RRD_ID_LENGTH_MAX_V019 200
67 +
68 +struct avl_element_v019 {
69 + void *avl_link[2];
70 + signed char avl_balance;
71 +};
72 +
73 +struct avl_tree_type_v019 {
74 + void *root;
75 + int (*compar)(void *a, void *b);
76 +};
77 +
78 +struct avl_tree_lock_v019 {
79 + struct avl_tree_type_v019 avl_tree;
80 + pthread_rwlock_t rwlock;
81 +};
82 +
83 +struct rrdset_map_save_v019 {
84 + struct avl_element_v019 avl; // ignored
85 + struct avl_element_v019 avlname; // ignored
86 + char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
87 + void *name; // ignored
88 + void *unused_ptr; // ignored
89 + void *type; // ignored
90 + void *family; // ignored
91 + void *title; // ignored
92 + void *units; // ignored
93 + void *context; // ignored
94 + uint32_t hash_context; // ignored
95 + uint32_t chart_type; // ignored
96 + int update_every; // check to reset all - update on load
97 + long entries; // check to reset all - update on load
98 + long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
99 + uint32_t flags; // ignored
100 + void *exporting_flags; // ignored
101 + int gap_when_lost_iterations_above; // ignored
102 + long priority; // ignored
103 + uint32_t storage_engine_id; // ignored
104 + void *cache_dir; // ignored
105 + char cache_filename[FILENAME_MAX+1]; // ignored - update on load
106 + pthread_rwlock_t rrdset_rwlock; // ignored
107 + size_t counter; // NEEDS TO BE UPDATED - maintained on load
108 + size_t counter_done; // ignored
109 + union { //
110 + time_t last_accessed_time_s; // ignored
111 + time_t last_entry_s; // ignored
112 + }; //
113 + time_t upstream_resync_time; // ignored
114 + void *plugin_name; // ignored
115 + void *module_name; // ignored
116 + void *chart_uuid; // ignored
117 + void *state; // ignored
118 + size_t unused[3]; // ignored
119 + size_t rrddim_page_alignment; // ignored
120 + uint32_t hash; // ignored
121 + uint32_t hash_name; // ignored
122 + usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
123 + struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
124 + struct timeval last_collected_time; // ignored
125 + long long collected_total; // ignored
126 + long long last_collected_total; // ignored
127 + void *rrdfamily; // ignored
128 + void *rrdhost; // ignored
129 + void *next; // ignored
130 + long double green; // ignored
131 + long double red; // ignored
132 + struct avl_tree_lock_v019 rrdvar_root_index; // ignored
133 + void *variables; // ignored
134 + void *alarms; // ignored
135 + unsigned long memsize; // check to reset all - update on load
136 + char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
137 + struct avl_tree_lock_v019 dimensions_index; // ignored
138 + void *dimensions; // ignored
139 +};
140 +
141 +void rrdset_memory_file_update(RRDSET *st) {
142 + if(!st->db.st_on_file) return;
143 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
144 +
145 + st_on_file->current_entry = st->db.current_entry;
146 + st_on_file->counter = st->counter;
147 + st_on_file->usec_since_last_update = st->usec_since_last_update;
148 + st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
149 + st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
150 +}
151 +
152 +const char *rrdset_cache_filename(RRDSET *st) {
153 + if(!st->db.st_on_file) return NULL;
154 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
155 + return st_on_file->cache_filename;
156 +}
157 +
158 +const char *rrdset_cache_dir(RRDSET *st) {
159 + if (st->db.cache_dir)
160 + return st->db.cache_dir;
161 +
162 + char b[FILENAME_MAX + 1];
163 + rrdset_strncpyz_name(b, rrdset_id(st), FILENAME_MAX);
164 +
165 + char n[FILENAME_MAX + 1];
166 + snprintfz(n, FILENAME_MAX, "%s/%s", st->rrdhost->cache_dir, b);
167 +
168 + st->db.cache_dir = strdupz(n);
169 +
170 + if (st->rrdhost->storage_engine_id == STORAGE_ENGINE_MAP ||
171 + st->rrdhost->storage_engine_id == STORAGE_ENGINE_SAVE)
172 + {
173 + int r = mkdir(st->db.cache_dir, 0775);
174 + if(r != 0 && errno != EEXIST)
175 + netdata_log_error("Cannot create directory '%s'", st->db.cache_dir);
176 + }
177 +
178 + return st->db.cache_dir;
179 +}
180 +
181 +void rrdset_memory_file_free(RRDSET *st) {
182 + if(!st->db.st_on_file) return;
183 +
184 + // needed for storage engine MAP, to save the latest state
185 + rrdset_memory_file_update(st);
186 +
187 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
188 + __atomic_sub_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
189 + netdata_munmap(st_on_file, st_on_file->memsize);
190 +
191 + // remove the pointers from the RRDDIM
192 + st->db.st_on_file = NULL;
193 +}
194 +
195 +void rrdset_memory_file_save(RRDSET *st) {
196 + if(!st->db.st_on_file) return;
197 +
198 + rrdset_memory_file_update(st);
199 +
200 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
201 + if(st_on_file->storage_engine_id != STORAGE_ENGINE_SAVE) return;
202 +
203 + memory_file_save(st_on_file->cache_filename, st->db.st_on_file, st_on_file->memsize);
204 +}
205 +
206 +static bool rrdset_memory_load_or_create_map_save(RRDSET *st) {
207 + if (st->storage_engine_id != STORAGE_ENGINE_SAVE && st->storage_engine_id != STORAGE_ENGINE_MAP)
208 + return false;
209 +
210 + char fullfilename[FILENAME_MAX + 1];
211 + snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", rrdset_cache_dir(st));
212 +
213 + unsigned long size = sizeof(struct rrdset_map_save_v019);
214 + struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
215 + fullfilename, size, ((st->storage_engine_id == STORAGE_ENGINE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0, false, NULL);
216 +
217 + if(!st_on_file) return false;
218 +
219 + time_t now_s = now_realtime_sec();
220 +
221 + st_on_file->magic[sizeof(RRDSET_MAGIC_V019)] = '\0';
222 + if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
223 + netdata_log_info("Initializing file '%s'.", fullfilename);
224 + memset(st_on_file, 0, size);
225 + }
226 + else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) {
227 + netdata_log_error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));
228 + memset(st_on_file, 0, size);
229 + }
230 + else if(st_on_file->memsize != size || st_on_file->entries != st->db.entries) {
231 + netdata_log_error("File '%s' does not have the desired size. Clearing it.", fullfilename);
232 + memset(st_on_file, 0, size);
233 + }
234 + else if(st_on_file->update_every != st->update_every) {
235 + netdata_log_error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
236 + memset(st_on_file, 0, size);
237 + }
238 + else if((now_s - st_on_file->last_updated.tv_sec) > (long)st->update_every * (long)st->db.entries) {
239 + netdata_log_info("File '%s' is too old. Clearing it.", fullfilename);
240 + memset(st_on_file, 0, size);
241 + }
242 + else if(st_on_file->last_updated.tv_sec > now_s + st->update_every) {
243 + netdata_log_error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now_s));
244 + st_on_file->last_updated.tv_sec = now_s;
245 + }
246 +
247 + if(st_on_file->current_entry >= st_on_file->entries)
248 + st_on_file->current_entry = 0;
249 +
250 + // make sure the database is aligned
251 + bool align_last_updated = false;
252 + if(st_on_file->last_updated.tv_sec) {
253 + st_on_file->update_every = st->update_every;
254 + align_last_updated = true;
255 + }
256 +
257 + // copy the useful values to st
258 + st->db.current_entry = st_on_file->current_entry;
259 + st->counter = st_on_file->counter;
260 + st->usec_since_last_update = st_on_file->usec_since_last_update;
261 + st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
262 + st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
263 +
264 + // link it to st
265 + st->db.st_on_file = st_on_file;
266 +
267 + // clear everything
268 + memset(st_on_file, 0, size);
269 +
270 + // set the values we need
271 + strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019);
272 + strcpy(st_on_file->cache_filename, fullfilename);
273 + strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
274 + st_on_file->memsize = size;
275 + st_on_file->entries = st->db.entries;
276 + st_on_file->update_every = st->update_every;
277 + st_on_file->storage_engine_id = st->storage_engine_id;
278 +
279 + if(align_last_updated)
280 + last_updated_time_align(st);
281 +
282 + // copy the useful values back to st_on_file
283 + rrdset_memory_file_update(st);
284 +
285 + __atomic_add_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
286 + return true;
287 +}
288
289 // ----------------------------------------------------------------------------
290 // RRDSET name index
@@ -87,7 +368,7 @@ struct rrdset_constructor {
368 long priority;
369 int update_every;
370 RRDSET_TYPE chart_type;
90 - RRD_MEMORY_MODE memory_mode;
371 + STORAGE_ENGINE_ID storage_engine_id;
372 long history_entries;
373
374 enum {
@@ -128,9 +409,9 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
409 st->module_name = rrd_string_strdupz(ctr->module);
410 st->priority = ctr->priority;
411
131 - st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;
412 + st->db.entries = (ctr->storage_engine_id != STORAGE_ENGINE_DBENGINE) ? align_entries_to_pagesize(ctr->storage_engine_id, ctr->history_entries) : 5;
413 st->update_every = ctr->update_every;
133 - st->rrd_memory_mode = ctr->memory_mode;
414 + st->storage_engine_id = ctr->storage_engine_id;
415
416 st->chart_type = ctr->chart_type;
417 st->rrdhost = host;
@@ -145,20 +426,18 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
426
427 rw_spinlock_init(&st->alerts.spinlock);
428
148 - if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
149 - if(!rrdset_memory_load_or_create_map_save(st, st->rrd_memory_mode)) {
150 - netdata_log_info("Failed to use db mode %s for chart '%s', falling back to ram mode.", (st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st));
151 - st->rrd_memory_mode = RRD_MEMORY_MODE_RAM;
429 + if(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP) {
430 + if(!rrdset_memory_load_or_create_map_save(st)) {
431 + netdata_log_info("Failed to use db mode %s for chart '%s', falling back to ram mode.", (st->storage_engine_id == STORAGE_ENGINE_MAP)?"map":"save", rrdset_name(st));
432 + st->storage_engine_id = STORAGE_ENGINE_RAM;
433 }
434 }
435
436 // initialize the db tiers
437 {
157 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
158 - STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
159 - if(!eng) continue;
160 -
161 - st->storage_metrics_groups[tier] = storage_engine_metrics_group_get(eng->backend, host->db[tier].instance, &st->chart_uuid);
438 + for (size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
439 + STORAGE_ENGINE_ID storage_engine_id = st->rrdhost->db[tier].id;
440 + st->storage_metrics_groups[tier] = storage_engine_metrics_group_get(storage_engine_id, host->db[tier].instance, &st->chart_uuid);
441 }
442 }
443
@@ -198,12 +477,11 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
477 rrddim_foreach_done(rd);
478 }
479
201 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
202 - STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
203 - if(!eng) continue;
480 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
481 + STORAGE_ENGINE_ID storage_engine_id = st->rrdhost->db[tier].id;
482
483 if(st->storage_metrics_groups[tier]) {
206 - storage_engine_metrics_group_release(eng->backend, host->db[tier].instance, st->storage_metrics_groups[tier]);
484 + storage_engine_metrics_group_release(storage_engine_id, host->db[tier].instance, st->storage_metrics_groups[tier]);
485 st->storage_metrics_groups[tier] = NULL;
486 }
487 }
@@ -449,7 +727,7 @@ static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) {
727 // ----------------------------------------------------------------------------
728 // RRDSET - find charts
729
452 -inline RRDSET *rrdset_find(RRDHOST *host, const char *id) {
730 +inline RRDSET *rrdset_find_by_id(RRDHOST *host, const char *id) {
731 netdata_log_debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host));
732 RRDSET *st = rrdset_index_find(host, id);
733
@@ -459,7 +737,7 @@ inline RRDSET *rrdset_find(RRDHOST *host, const char *id) {
737 return(st);
738 }
739
462 -inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id) {
740 +inline RRDSET *rrdset_find_by_type(RRDHOST *host, const char *type, const char *id) {
741 netdata_log_debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host));
742
743 char buf[RRD_ID_LENGTH_MAX + 1];
@@ -468,10 +746,10 @@ inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *i
746 int len = (int) strlen(buf);
747 strncpyz(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len));
748
471 - return(rrdset_find(host, buf));
749 + return(rrdset_find_by_id(host, buf));
750 }
751
474 -inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {
752 +inline RRDSET *rrdset_find_by_name(RRDHOST *host, const char *name) {
753 netdata_log_debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host));
754 RRDSET *st = rrdset_index_find_name(host, name);
755 return(st);
@@ -597,7 +875,7 @@ time_t rrdset_first_entry_s(RRDSET *st) {
875 }
876
877 time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier) {
600 - if(unlikely(tier > storage_tiers))
878 + if(unlikely(tier > rrdb.storage_tiers))
879 return 0;
880
881 RRDDIM *rd;
@@ -741,10 +1019,7 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
1019 }
1020 }
1021
744 -// ----------------------------------------------------------------------------
745 -// RRDSET - reset a chart
746 -
747 -void rrdset_reset(RRDSET *st) {
1022 +static void rrdset_reset(RRDSET *st) {
1023 netdata_log_debug(D_RRD_CALLS, "rrdset_reset() %s", rrdset_name(st));
1024
1025 st->last_collected_time.tv_sec = 0;
@@ -762,57 +1037,15 @@ void rrdset_reset(RRDSET *st) {
1037 rd->collector.counter = 0;
1038
1039 if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
765 - for(size_t tier = 0; tier < storage_tiers ;tier++)
766 - storage_engine_store_flush(rd->tiers[tier].db_collection_handle);
1040 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
1041 + if (rd->tiers[tier].db_collection_handle)
1042 + storage_engine_store_flush(st->storage_engine_id, rd->tiers[tier].db_collection_handle);
1043 + }
1044 }
1045 }
1046 rrddim_foreach_done(rd);
1047 }
1048
772 -// ----------------------------------------------------------------------------
773 -// RRDSET - helpers for rrdset_create()
774 -
775 -inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
776 - if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
777 - if(mode == RRD_MEMORY_MODE_NONE) return 5;
778 -
779 - if(entries < 5) entries = 5;
780 - if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
781 -
782 - if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE || mode == RRD_MEMORY_MODE_RAM) {
783 - long header_size = 0;
784 -
785 - if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE)
786 - header_size = (long)rrddim_memory_file_header_size();
787 -
788 - long page = (long)sysconf(_SC_PAGESIZE);
789 - long size = (long)(header_size + entries * sizeof(storage_number));
790 - if (unlikely(size % page)) {
791 - size -= (size % page);
792 - size += page;
793 -
794 - long n = (long)((size - header_size) / sizeof(storage_number));
795 - return n;
796 - }
797 - }
798 -
799 - return entries;
800 -}
801 -
802 -static inline void last_collected_time_align(RRDSET *st) {
803 - st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
804 -
805 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
806 - st->last_collected_time.tv_usec = 0;
807 - else
808 - st->last_collected_time.tv_usec = 500000;
809 -}
810 -
811 -static inline void last_updated_time_align(RRDSET *st) {
812 - st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
813 - st->last_updated.tv_usec = 0;
814 -}
815 -
1049 // ----------------------------------------------------------------------------
1050 // RRDSET - free a chart
1051
@@ -835,7 +1068,7 @@ void rrdset_delete_files(RRDSET *st) {
1068
1069 netdata_log_info("Deleting chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
1070
838 - if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
1071 + if(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP) {
1072 const char *cache_filename = rrdset_cache_filename(st);
1073 if(cache_filename) {
1074 netdata_log_info("Deleting chart header file '%s'.", cache_filename);
@@ -860,23 +1093,6 @@ void rrdset_delete_files(RRDSET *st) {
1093 recursively_delete_dir(st->db.cache_dir, "left-over chart");
1094 }
1095
863 -void rrdset_delete_obsolete_dimensions(RRDSET *st) {
864 - RRDDIM *rd;
865 -
866 - netdata_log_info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
867 -
868 - rrddim_foreach_read(rd, st) {
869 - if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
870 - const char *cache_filename = rrddim_cache_filename(rd);
871 - if(!cache_filename) continue;
872 - netdata_log_info("Deleting dimension file '%s'.", cache_filename);
873 - if(unlikely(unlink(cache_filename) == -1))
874 - netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
875 - }
876 - }
877 - rrddim_foreach_done(rd);
878 -}
879 -
1096 // ----------------------------------------------------------------------------
1097 // RRDSET - create a chart
1098
@@ -894,10 +1110,10 @@ RRDSET *rrdset_create_custom(
1110 , long priority
1111 , int update_every
1112 , RRDSET_TYPE chart_type
897 - , RRD_MEMORY_MODE memory_mode
1113 + , STORAGE_ENGINE_ID storage_engine_id
1114 , long history_entries
1115 ) {
900 - if (host != localhost)
1116 + if (host != rrdb.localhost)
1117 host->child_last_chart_command = now_realtime_sec();
1118
1119 if(!type || !type[0])
@@ -949,7 +1165,7 @@ RRDSET *rrdset_create_custom(
1165 .priority = priority,
1166 .update_every = update_every,
1167 .chart_type = chart_type,
952 - .memory_mode = memory_mode,
1168 + .storage_engine_id = storage_engine_id,
1169 .history_entries = history_entries,
1170 };
1171
@@ -1127,14 +1343,15 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) {
1343
1344 static __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];
1345
1130 -static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, time_t now_s) {
1131 - time_t loop = (time_t)rd->rrdset->update_every * (time_t)t->tier_grouping;
1346 +static inline time_t tier_next_point_time_s(RRDDIM *rd, size_t tier, time_t now_s) {
1347 + uint32_t tier_grouping = rd->rrdset->rrdhost->db[tier].tier_grouping;
1348 + time_t loop = (time_t)rd->rrdset->update_every * (time_t) tier_grouping;
1349 return now_s + loop - ((now_s + loop) % loop);
1350 }
1351
1352 void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {
1353 if (unlikely(!t->next_point_end_time_s))
1137 - t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
1354 + t->next_point_end_time_s = tier_next_point_time_s(rd, tier, sp.end_time_s);
1355
1356 if(unlikely(sp.start_time_s >= t->next_point_end_time_s)) {
1357 // flush the virtual point, it is done
@@ -1142,6 +1359,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1359 if (likely(!storage_point_is_unset(t->virtual_point))) {
1360
1361 storage_engine_store_metric(
1362 + rd->rrdset->storage_engine_id,
1363 t->db_collection_handle,
1364 t->next_point_end_time_s * USEC_PER_SEC,
1365 t->virtual_point.sum,
@@ -1153,6 +1371,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1371 }
1372 else {
1373 storage_engine_store_metric(
1374 + rd->rrdset->storage_engine_id,
1375 t->db_collection_handle,
1376 t->next_point_end_time_s * USEC_PER_SEC,
1377 NAN,
@@ -1164,7 +1383,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1383
1384 rrdset_done_statistics_points_stored_per_tier[tier]++;
1385 t->virtual_point.count = 0; // make the point unset
1167 - t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
1386 + t->next_point_end_time_s = tier_next_point_time_s(rd, tier, sp.end_time_s);
1387 }
1388
1389 // merge the dates into our virtual point
@@ -1224,9 +1443,8 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1443 #endif // NETDATA_LOG_COLLECTION_ERRORS
1444
1445 // store the metric on tier 0
1227 - storage_engine_store_metric(rd->tiers[0].db_collection_handle, point_end_time_ut,
1228 - n, 0, 0,
1229 - 1, 0, flags);
1446 + storage_engine_store_metric(rd->rrdset->storage_engine_id, rd->tiers[0].db_collection_handle, point_end_time_ut,
1447 + n, 0, 0, 1, 0, flags);
1448
1449 rrdset_done_statistics_points_stored_per_tier[0]++;
1450
@@ -1243,7 +1461,7 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1461 .flags = flags
1462 };
1463
1246 - for(size_t tier = 1; tier < storage_tiers ;tier++) {
1464 + for(size_t tier = 1; tier < rrdb.storage_tiers ;tier++) {
1465 if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
1466
1467 struct rrddim_tier *t = &rd->tiers[tier];
@@ -1381,11 +1599,8 @@ static inline size_t rrdset_done_interpolate(
1599 new_value /= (NETDATA_DOUBLE)st->update_every;
1600
1601 if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
1384 -
1602 rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
1386 - rrddim_name(rd)
1387 - , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
1388 - );
1603 + rrddim_name(rd) , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut));
1604
1605 new_value = new_value * (NETDATA_DOUBLE)(st->update_every * USEC_PER_SEC) / (NETDATA_DOUBLE)(next_store_ut - last_stored_ut);
1606 }
@@ -1439,7 +1654,7 @@ static inline size_t rrdset_done_interpolate(
1654 continue;
1655 }
1656
1442 - if(likely(rrddim_check_updated(rd) && rd->collector.counter > 1 && iterations < gap_when_lost_iterations_above)) {
1657 + if(likely(rrddim_check_updated(rd) && rd->collector.counter > 1 && iterations < rrdb.gap_when_lost_iterations_above)) {
1658 uint32_t dim_storage_flags = storage_flags;
1659
1660 if (ml_dimension_is_anomalous(rd, current_time_s, new_value, true)) {
@@ -1583,7 +1798,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1798
1799 // check if we will re-write the entire data set
1800 if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&
1586 - st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
1801 + st->storage_engine_id != STORAGE_ENGINE_DBENGINE)) {
1802 netdata_log_info(
1803 "'%s': too old data (last updated at %"PRId64".%"PRId64", last collected at %"PRId64".%"PRId64"). "
1804 "Resetting it. Will not store the next entry.",
@@ -1643,8 +1858,8 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1858 size_t dim_id;
1859 size_t dimensions = 0;
1860 struct rda_item *rda = rda_base;
1646 - total_number collected_total = 0;
1647 - total_number last_collected_total = 0;
1861 + collected_number collected_total = 0;
1862 + collected_number last_collected_total = 0;
1863 rrddim_foreach_read(rd, st) {
1864 if(rd_dfe.counter >= rda_slots)
1865 break;
@@ -1882,14 +2097,6 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
2097 // at this point we have all the calculated values ready
2098 // it is now time to interpolate values on a second boundary
2099
1885 -// #ifdef NETDATA_INTERNAL_CHECKS
1886 -// if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) {
1887 -// // this is collected in the same interpolation point
1888 -// rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
1889 -// netdata_log_info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, rrdset_name(st), st->counter_done, next_store_ut - now_collect_ut);
1890 -// }
1891 -// #endif
1892 -
2100 rrdset_done_interpolate(
2101 &stream_buffer
2102 , st
@@ -1967,7 +2174,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
2174 // ALL DONE ABOUT THE DATA UPDATE
2175 // --------------------------------------------------------------------
2176
1970 - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
2177 + if(unlikely(st->storage_engine_id == STORAGE_ENGINE_MAP)) {
2178 // update the memory mapped files with the latest values
2179
2180 rrdset_memory_file_update(st);
@@ -2006,9 +2213,10 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
2213 // switch update every to the storage engine
2214 RRDDIM *rd;
2215 rrddim_foreach_read(rd, st) {
2009 - for (size_t tier = 0; tier < storage_tiers; tier++) {
2216 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
2217 if (rd->tiers[tier].db_collection_handle)
2218 storage_engine_store_change_collection_frequency(
2219 + st->storage_engine_id,
2220 rd->tiers[tier].db_collection_handle,
2221 (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
2222 }
@@ -2018,210 +2226,13 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
2226 return prev_update_every_s;
2227 }
2228
2021 -// ----------------------------------------------------------------------------
2022 -// compatibility layer for RRDSET files v019
2229 +void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
2230 + if(!st->rrdlabels)
2231 + st->rrdlabels = rrdlabels_create();
2232
2024 -#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
2025 -#define RRD_ID_LENGTH_MAX_V019 200
2233 + if (new_rrdlabels)
2234 + rrdlabels_migrate_to_these(st->rrdlabels, new_rrdlabels);
2235
2027 -struct avl_element_v019 {
2028 - void *avl_link[2];
2029 - signed char avl_balance;
2030 -};
2031 -struct avl_tree_type_v019 {
2032 - void *root;
2033 - int (*compar)(void *a, void *b);
2034 -};
2035 -struct avl_tree_lock_v019 {
2036 - struct avl_tree_type_v019 avl_tree;
2037 - pthread_rwlock_t rwlock;
2038 -};
2039 -struct rrdset_map_save_v019 {
2040 - struct avl_element_v019 avl; // ignored
2041 - struct avl_element_v019 avlname; // ignored
2042 - char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
2043 - void *name; // ignored
2044 - void *unused_ptr; // ignored
2045 - void *type; // ignored
2046 - void *family; // ignored
2047 - void *title; // ignored
2048 - void *units; // ignored
2049 - void *context; // ignored
2050 - uint32_t hash_context; // ignored
2051 - uint32_t chart_type; // ignored
2052 - int update_every; // check to reset all - update on load
2053 - long entries; // check to reset all - update on load
2054 - long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
2055 - uint32_t flags; // ignored
2056 - void *exporting_flags; // ignored
2057 - int gap_when_lost_iterations_above; // ignored
2058 - long priority; // ignored
2059 - uint32_t rrd_memory_mode; // ignored
2060 - void *cache_dir; // ignored
2061 - char cache_filename[FILENAME_MAX+1]; // ignored - update on load
2062 - pthread_rwlock_t rrdset_rwlock; // ignored
2063 - size_t counter; // NEEDS TO BE UPDATED - maintained on load
2064 - size_t counter_done; // ignored
2065 - union { //
2066 - time_t last_accessed_time_s; // ignored
2067 - time_t last_entry_s; // ignored
2068 - }; //
2069 - time_t upstream_resync_time; // ignored
2070 - void *plugin_name; // ignored
2071 - void *module_name; // ignored
2072 - void *chart_uuid; // ignored
2073 - void *state; // ignored
2074 - size_t unused[3]; // ignored
2075 - size_t rrddim_page_alignment; // ignored
2076 - uint32_t hash; // ignored
2077 - uint32_t hash_name; // ignored
2078 - usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
2079 - struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
2080 - struct timeval last_collected_time; // ignored
2081 - long long collected_total; // ignored
2082 - long long last_collected_total; // ignored
2083 - void *rrdfamily; // ignored
2084 - void *rrdhost; // ignored
2085 - void *next; // ignored
2086 - long double green; // ignored
2087 - long double red; // ignored
2088 - struct avl_tree_lock_v019 rrdvar_root_index; // ignored
2089 - void *variables; // ignored
2090 - void *alarms; // ignored
2091 - unsigned long memsize; // check to reset all - update on load
2092 - char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
2093 - struct avl_tree_lock_v019 dimensions_index; // ignored
2094 - void *dimensions; // ignored
2095 -};
2096 -
2097 -void rrdset_memory_file_update(RRDSET *st) {
2098 - if(!st->db.st_on_file) return;
2099 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2100 -
2101 - st_on_file->current_entry = st->db.current_entry;
2102 - st_on_file->counter = st->counter;
2103 - st_on_file->usec_since_last_update = st->usec_since_last_update;
2104 - st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
2105 - st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
2106 -}
2107 -
2108 -const char *rrdset_cache_filename(RRDSET *st) {
2109 - if(!st->db.st_on_file) return NULL;
2110 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2111 - return st_on_file->cache_filename;
2112 -}
2113 -
2114 -const char *rrdset_cache_dir(RRDSET *st) {
2115 - if(!st->db.cache_dir)
2116 - st->db.cache_dir = rrdhost_cache_dir_for_rrdset_alloc(st->rrdhost, rrdset_id(st));
2117 -
2118 - return st->db.cache_dir;
2119 -}
2120 -
2121 -void rrdset_memory_file_free(RRDSET *st) {
2122 - if(!st->db.st_on_file) return;
2123 -
2124 - // needed for memory mode map, to save the latest state
2125 - rrdset_memory_file_update(st);
2126 -
2127 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2128 - __atomic_sub_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
2129 - netdata_munmap(st_on_file, st_on_file->memsize);
2130 -
2131 - // remove the pointers from the RRDDIM
2132 - st->db.st_on_file = NULL;
2133 -}
2134 -
2135 -void rrdset_memory_file_save(RRDSET *st) {
2136 - if(!st->db.st_on_file) return;
2137 -
2138 - rrdset_memory_file_update(st);
2139 -
2140 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2141 - if(st_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
2142 -
2143 - memory_file_save(st_on_file->cache_filename, st->db.st_on_file, st_on_file->memsize);
2144 -}
2145 -
2146 -bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mode) {
2147 - if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
2148 - return false;
2149 -
2150 - char fullfilename[FILENAME_MAX + 1];
2151 - snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", rrdset_cache_dir(st));
2152 -
2153 - unsigned long size = sizeof(struct rrdset_map_save_v019);
2154 - struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
2155 - fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0, false, NULL);
2156 -
2157 - if(!st_on_file) return false;
2158 -
2159 - time_t now_s = now_realtime_sec();
2160 -
2161 - st_on_file->magic[sizeof(RRDSET_MAGIC_V019)] = '\0';
2162 - if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
2163 - netdata_log_info("Initializing file '%s'.", fullfilename);
2164 - memset(st_on_file, 0, size);
2165 - }
2166 - else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) {
2167 - netdata_log_error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));
2168 - memset(st_on_file, 0, size);
2169 - }
2170 - else if(st_on_file->memsize != size || st_on_file->entries != st->db.entries) {
2171 - netdata_log_error("File '%s' does not have the desired size. Clearing it.", fullfilename);
2172 - memset(st_on_file, 0, size);
2173 - }
2174 - else if(st_on_file->update_every != st->update_every) {
2175 - netdata_log_error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
2176 - memset(st_on_file, 0, size);
2177 - }
2178 - else if((now_s - st_on_file->last_updated.tv_sec) > (long)st->update_every * (long)st->db.entries) {
2179 - netdata_log_info("File '%s' is too old. Clearing it.", fullfilename);
2180 - memset(st_on_file, 0, size);
2181 - }
2182 - else if(st_on_file->last_updated.tv_sec > now_s + st->update_every) {
2183 - netdata_log_error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now_s));
2184 - st_on_file->last_updated.tv_sec = now_s;
2185 - }
2186 -
2187 - if(st_on_file->current_entry >= st_on_file->entries)
2188 - st_on_file->current_entry = 0;
2189 -
2190 - // make sure the database is aligned
2191 - bool align_last_updated = false;
2192 - if(st_on_file->last_updated.tv_sec) {
2193 - st_on_file->update_every = st->update_every;
2194 - align_last_updated = true;
2195 - }
2196 -
2197 - // copy the useful values to st
2198 - st->db.current_entry = st_on_file->current_entry;
2199 - st->counter = st_on_file->counter;
2200 - st->usec_since_last_update = st_on_file->usec_since_last_update;
2201 - st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
2202 - st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
2203 -
2204 - // link it to st
2205 - st->db.st_on_file = st_on_file;
2206 -
2207 - // clear everything
2208 - memset(st_on_file, 0, size);
2209 -
2210 - // set the values we need
2211 - strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019);
2212 - strcpy(st_on_file->cache_filename, fullfilename);
2213 - strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
2214 - st_on_file->memsize = size;
2215 - st_on_file->entries = st->db.entries;
2216 - st_on_file->update_every = st->update_every;
2217 - st_on_file->rrd_memory_mode = memory_mode;
2218 -
2219 - if(align_last_updated)
2220 - last_updated_time_align(st);
2221 -
2222 - // copy the useful values back to st_on_file
2223 - rrdset_memory_file_update(st);
2224 -
2225 - __atomic_add_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
2226 - return true;
2236 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
2237 + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
2238 }
database/rrdvar.c
-5
@@ -217,11 +217,6 @@ NETDATA_DOUBLE rrdvar2number(const RRDVAR_ACQUIRED *rva) {
217 return (NETDATA_DOUBLE)*n;
218 }
219
220 - case RRDVAR_TYPE_TOTAL: {
221 - total_number *n = (total_number *)rv->value;
222 - return (NETDATA_DOUBLE)*n;
223 - }
224 -
220 case RRDVAR_TYPE_INT: {
221 int *n = (int *)rv->value;
222 return *n;
database/rrdvar.h
+1 -2
@@ -9,8 +9,7 @@ typedef enum rrdvar_type {
9 RRDVAR_TYPE_CALCULATED = 1,
10 RRDVAR_TYPE_TIME_T = 2,
11 RRDVAR_TYPE_COLLECTED = 3,
12 - RRDVAR_TYPE_TOTAL = 4,
13 - RRDVAR_TYPE_INT = 5
12 + RRDVAR_TYPE_INT = 4
13
14 // this is 8 bit
15 // to increase it you have to set change the bitfield in
database/sqlite/sqlite_aclk.c
+8 -8
@@ -104,7 +104,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
104
105 sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info);
106
107 - RRDHOST *host = rrdhost_find_or_create(
107 + RRDHOST *host = rrdhost_get_or_create(
108 (const char *) argv[IDX_HOSTNAME]
109 , (const char *) argv[IDX_REGISTRY]
110 , guid
@@ -117,7 +117,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
117 , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown")
118 , argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1
119 , argv[IDX_ENTRIES] ? str2i(argv[IDX_ENTRIES]) : 0
120 - , default_rrd_memory_mode
120 + , default_storage_engine_id
121 , 0 // health
122 , 0 // rrdpush enabled
123 , NULL //destination
@@ -181,7 +181,7 @@ static int is_host_available(uuid_t *host_id)
181 int rc;
182
183 if (unlikely(!db_meta)) {
184 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
184 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
185 error_report("Database has not been initialized");
186 return 1;
187 }
@@ -319,7 +319,7 @@ static int aclk_config_parameters(void *data __maybe_unused, int argc __maybe_un
319 uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str);
320
321 RRDHOST *host = rrdhost_find_by_guid(uuid_str);
322 - if (host == localhost)
322 + if (host == rrdb.localhost)
323 return 0;
324
325 sql_create_aclk_table(host, (uuid_t *) argv[0], (uuid_t *) argv[1]);
@@ -424,7 +424,7 @@ static void aclk_synchronization(void *arg __maybe_unused)
424 // NODE STATE
425 case ACLK_DATABASE_NODE_STATE:;
426 RRDHOST *host = cmd.param[0];
427 - int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
427 + int live = (host == rrdb.localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
428 struct aclk_sync_host_config *ahc = host->aclk_sync_host_config;
429 if (unlikely(!ahc))
430 sql_create_aclk_table(host, &host->host_uuid, host->node_id);
@@ -520,7 +520,7 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may
520 strcpy(wc->uuid_str, uuid_str);
521 wc->alert_updates = 0;
522 time_t now = now_realtime_sec();
523 - wc->node_info_send_time = (host == localhost || NULL == localhost) ? now - 25 : now;
523 + wc->node_info_send_time = (host == rrdb.localhost || NULL == rrdb.localhost) ? now - 25 : now;
524 #endif
525 }
526
@@ -536,7 +536,7 @@ void sql_aclk_sync_init(void)
536 int rc;
537
538 if (unlikely(!db_meta)) {
539 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {
539 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE) {
540 return;
541 }
542 error_report("Database has not been initialized");
@@ -558,7 +558,7 @@ void sql_aclk_sync_init(void)
558
559 #ifdef ENABLE_ACLK
560 if (!number_of_children)
561 - aclk_queue_node_info(localhost, true);
561 + aclk_queue_node_info(rrdb.localhost, true);
562
563 rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_INSTANCES,aclk_config_parameters, NULL,&err_msg);
564
database/sqlite/sqlite_aclk.h
+1 -1
@@ -38,7 +38,7 @@ static inline int uuid_parse_fix(char *in, uuid_t uuid)
38
39 static inline int claimed()
40 {
41 - return localhost->aclk_state.claimed_id != NULL;
41 + return rrdb.localhost->aclk_state.claimed_id != NULL;
42 }
43
44 #define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \
database/sqlite/sqlite_aclk_alert.c
+8 -8
@@ -430,7 +430,7 @@ void aclk_push_alert_events_for_all_hosts(void)
430 {
431 RRDHOST *host;
432
433 - dfe_start_reentrant(rrdhost_root_index, host) {
433 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
434 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))
435 continue;
436
@@ -504,7 +504,7 @@ void aclk_send_alarm_configuration(char *config_hash)
504 if (unlikely(!config_hash))
505 return;
506
507 - struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) localhost->aclk_sync_host_config;
507 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) rrdb.localhost->aclk_sync_host_config;
508
509 if (unlikely(!wc))
510 return;
@@ -529,7 +529,7 @@ int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash
529 sqlite3_stmt *res = NULL;
530
531 struct aclk_sync_host_config *wc = NULL;
532 - RRDHOST *host = find_host_by_node_id(node_id);
532 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
533
534 if (unlikely(!host)) {
535 freez(config_hash);
@@ -655,7 +655,7 @@ void aclk_start_alert_streaming(char *node_id, bool resets)
655 if (uuid_parse(node_id, node_uuid))
656 return;
657
658 - RRDHOST *host = find_host_by_node_id(node_id);
658 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
659
660 if (unlikely(!host))
661 return;
@@ -689,7 +689,7 @@ void aclk_start_alert_streaming(char *node_id, bool resets)
689 void sql_process_queue_removed_alerts_to_aclk(char *node_id)
690 {
691 struct aclk_sync_host_config *wc;
692 - RRDHOST *host = find_host_by_node_id(node_id);
692 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
693 freez(node_id);
694
695 if (unlikely(!host || !(wc = host->aclk_sync_host_config)))
@@ -750,7 +750,7 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unus
750 if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
751 return;
752
753 - RRDHOST *host = find_host_by_node_id(node_id);
753 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
754 if (unlikely(!host)) {
755 netdata_log_access("ACLK STA [%s (N/A)]: ACLK node id does not exist", node_id);
756 return;
@@ -854,7 +854,7 @@ static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, uint32_t mark)
854 void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)
855 {
856 #ifdef ENABLE_ACLK
857 - RRDHOST *host = find_host_by_node_id(node_id);
857 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
858
859 if (unlikely(!host)) {
860 netdata_log_access("AC [%s (N/A)]: Node id not found", node_id);
@@ -1039,7 +1039,7 @@ void aclk_send_alarm_checkpoint(char *node_id, char *claim_id __maybe_unused)
1039 return;
1040
1041 struct aclk_sync_host_config *wc = NULL;
1042 - RRDHOST *host = find_host_by_node_id(node_id);
1042 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
1043
1044 if (unlikely(!host))
1045 return;
database/sqlite/sqlite_aclk_node.c
+6 -6
@@ -29,7 +29,7 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) {
29 static void build_node_collectors(char *node_id __maybe_unused)
30 {
31
32 - RRDHOST *host = find_host_by_node_id(node_id);
32 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
33
34 if (unlikely(!host))
35 return;
@@ -59,7 +59,7 @@ static void build_node_info(char *node_id __maybe_unused)
59 {
60 struct update_node_info node_info;
61
62 - RRDHOST *host = find_host_by_node_id(node_id);
62 + RRDHOST *host = rrdhost_find_by_node_id(node_id);
63
64 if (unlikely((!host))) {
65 freez(node_id);
@@ -77,7 +77,7 @@ static void build_node_info(char *node_id __maybe_unused)
77 node_info.node_id = wc->node_id;
78 node_info.claim_id = get_agent_claimid();
79 node_info.machine_guid = host->machine_guid;
80 - node_info.child = (wc->host != localhost);
80 + node_info.child = (wc->host != rrdb.localhost);
81 node_info.ml_info.ml_capable = ml_capable();
82 node_info.ml_info.ml_enabled = ml_enabled(wc->host);
83
@@ -86,7 +86,7 @@ static void build_node_info(char *node_id __maybe_unused)
86 now_realtime_timeval(&node_info.updated_at);
87
88 char *host_version = NULL;
89 - if (host != localhost) {
89 + if (host != rrdb.localhost) {
90 netdata_mutex_lock(&host->receiver_lock);
91 host_version = strdupz(host->receiver && host->receiver->program_version ? host->receiver->program_version : rrdhost_program_version(host));
92 netdata_mutex_unlock(&host->receiver_lock);
@@ -124,7 +124,7 @@ static void build_node_info(char *node_id __maybe_unused)
124 node_info.data.host_labels_ptr = host->rrdlabels;
125
126 aclk_update_node_info(&node_info);
127 - netdata_log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), host->machine_guid, wc->host == localhost ? "parent" : "child");
127 + netdata_log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), host->machine_guid, wc->host == rrdb.localhost ? "parent" : "child");
128
129 rrd_unlock();
130 freez(node_info.claim_id);
@@ -145,7 +145,7 @@ void aclk_check_node_info_and_collectors(void)
145 return;
146
147 size_t pending = 0;
148 - dfe_start_reentrant(rrdhost_root_index, host) {
148 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
149
150 struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
151 if (unlikely(!wc))
database/sqlite/sqlite_functions.c
+8 -8
@@ -631,7 +631,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id)
631 rrd_unlock();
632
633 if (unlikely(!db_meta)) {
634 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
634 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
635 error_report("Database has not been initialized");
636 return 1;
637 }
@@ -674,7 +674,7 @@ int get_host_id(uuid_t *node_id, uuid_t *host_id)
674 int rc;
675
676 if (unlikely(!db_meta)) {
677 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
677 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
678 error_report("Database has not been initialized");
679 return 1;
680 }
@@ -712,7 +712,7 @@ int get_node_id(uuid_t *host_id, uuid_t *node_id)
712 int rc;
713
714 if (unlikely(!db_meta)) {
715 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
715 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
716 error_report("Database has not been initialized");
717 return 1;
718 }
@@ -749,7 +749,7 @@ void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)
749 int rc;
750
751 if (unlikely(!db_meta)) {
752 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
752 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
753 error_report("Database has not been initialized");
754 return;
755 }
@@ -795,7 +795,7 @@ struct node_instance_list *get_node_list(void)
795 int rc;
796
797 if (unlikely(!db_meta)) {
798 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
798 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
799 error_report("Database has not been initialized");
800 return NULL;
801 }
@@ -833,10 +833,10 @@ struct node_instance_list *get_node_list(void)
833 }
834 uuid_copy(node_list[row].host_id, *host_id);
835 node_list[row].queryable = 1;
836 - node_list[row].live = (host && (host == localhost || host->receiver
836 + node_list[row].live = (host && (host == rrdb.localhost || host->receiver
837 || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;
838 node_list[row].hops = (host && host->system_info) ? host->system_info->hops :
839 - uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0;
839 + uuid_memcmp(host_id, &rrdb.localhost->host_uuid) ? 1 : 0;
840 node_list[row].hostname =
841 sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;
842 }
@@ -861,7 +861,7 @@ void sql_load_node_id(RRDHOST *host)
861 int rc;
862
863 if (unlikely(!db_meta)) {
864 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
864 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
865 error_report("Database has not been initialized");
866 return;
867 }
database/sqlite/sqlite_functions.h
+1 -1
@@ -29,7 +29,7 @@ typedef enum db_check_action_type {
29
30 #define CHECK_SQLITE_CONNECTION(db_meta) \
31 if (unlikely(!db_meta)) { \
32 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { \
32 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE) { \
33 return 1; \
34 } \
35 error_report("Database has not been initialized"); \
database/sqlite/sqlite_health.c
+7 -7
@@ -16,7 +16,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
16 int rc;
17
18 if (unlikely(!db_meta)) {
19 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
19 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
20 error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
21 return;
22 }
@@ -98,7 +98,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
98 uint64_t health_log_id = 0;
99
100 if (unlikely(!db_meta)) {
101 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
101 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
102 error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
103 return;
104 }
@@ -368,7 +368,7 @@ void sql_health_alarm_log_count(RRDHOST *host) {
368 int rc;
369
370 if (unlikely(!db_meta)) {
371 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
371 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
372 error_report("Database has not been initialized");
373 return;
374 }
@@ -407,7 +407,7 @@ void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host) {
407 char command[MAX_HEALTH_SQL_SIZE + 1];
408
409 if (unlikely(!db_meta)) {
410 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
410 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
411 error_report("Database has not been initialized");
412 return;
413 }
@@ -468,7 +468,7 @@ void sql_health_alarm_log_cleanup_claimed(RRDHOST *host) {
468 char command[MAX_HEALTH_SQL_SIZE + 1];
469
470 if (unlikely(!db_meta)) {
471 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
471 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
472 error_report("Database has not been initialized");
473 return;
474 }
@@ -784,7 +784,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
784 host->health.health_log_entries_written = 0;
785
786 if (unlikely(!db_meta)) {
787 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
787 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
788 error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
789 return;
790 }
@@ -1012,7 +1012,7 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
1012 int rc, param = 0;
1013
1014 if (unlikely(!db_meta)) {
1015 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1015 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
1016 return 0;
1017 error_report("Database has not been initialized");
1018 return 1;
database/sqlite/sqlite_metadata.c
+17 -17
@@ -215,7 +215,7 @@ static int store_claim_id(uuid_t *host_id, uuid_t *claim_id)
215 int rc;
216
217 if (unlikely(!db_meta)) {
218 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
218 + if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
219 error_report("Database has not been initialized");
220 return 1;
221 }
@@ -287,7 +287,7 @@ static int store_host_metadata(RRDHOST *host)
287 int rc, param = 0;
288
289 if (unlikely(!db_meta)) {
290 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
290 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
291 return 0;
292 error_report("Database has not been initialized");
293 return 1;
@@ -313,7 +313,7 @@ static int store_host_metadata(RRDHOST *host)
313 if (unlikely(rc != SQLITE_OK))
314 goto bind_fail;
315
316 - rc = sqlite3_bind_int(res, ++param, host->rrd_update_every);
316 + rc = sqlite3_bind_int(res, ++param, host->update_every);
317 if (unlikely(rc != SQLITE_OK))
318 goto bind_fail;
319
@@ -333,7 +333,7 @@ static int store_host_metadata(RRDHOST *host)
333 if (unlikely(rc != SQLITE_OK))
334 goto bind_fail;
335
336 - rc = sqlite3_bind_int(res, ++param, host->rrd_memory_mode);
336 + rc = sqlite3_bind_int(res, ++param, host->storage_engine_id);
337 if (unlikely(rc != SQLITE_OK))
338 goto bind_fail;
339
@@ -384,7 +384,7 @@ static int add_host_sysinfo_key_value(const char *name, const char *value, uuid_
384 int rc, param = 0;
385
386 if (unlikely(!db_meta)) {
387 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
387 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
388 return 0;
389 error_report("Database has not been initialized");
390 return 0;
@@ -475,7 +475,7 @@ static int store_chart_metadata(RRDSET *st)
475 int rc, param = 0, store_rc = 0;
476
477 if (unlikely(!db_meta)) {
478 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
478 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
479 return 0;
480 error_report("Database has not been initialized");
481 return 1;
@@ -549,7 +549,7 @@ static int store_chart_metadata(RRDSET *st)
549 if (unlikely(rc != SQLITE_OK))
550 goto bind_fail;
551
552 - rc = sqlite3_bind_int(res, ++param, st->rrd_memory_mode);
552 + rc = sqlite3_bind_int(res, ++param, st->storage_engine_id);
553 if (unlikely(rc != SQLITE_OK))
554 goto bind_fail;
555
@@ -584,7 +584,7 @@ static int store_dimension_metadata(RRDDIM *rd)
584 int rc, param = 0;
585
586 if (unlikely(!db_meta)) {
587 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
587 + if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
588 return 0;
589 error_report("Database has not been initialized");
590 return 1;
@@ -653,13 +653,13 @@ bind_fail:
653 static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused)
654 {
655 #ifdef ENABLE_DBENGINE
656 - if(dbengine_enabled) {
656 + if (rrdb.dbengine_enabled) {
657 bool no_retention = true;
658 - for (size_t tier = 0; tier < storage_tiers; tier++) {
659 - if (!multidb_ctx[tier])
658 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
659 + if (!rrdb.multidb_ctx[tier])
660 continue;
661 time_t first_time_t = 0, last_time_t = 0;
662 - if (rrdeng_metric_retention_by_uuid((void *) multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) {
662 + if (rrdeng_metric_retention_by_uuid((void *) rrdb.multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) {
663 if (first_time_t > 0) {
664 no_retention = false;
665 break;
@@ -728,7 +728,7 @@ skip_run:
728 static void cleanup_health_log(void)
729 {
730 RRDHOST *host;
731 - dfe_start_reentrant(rrdhost_root_index, host) {
731 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
732 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
733 continue;
734 sql_health_alarm_log_cleanup(host);
@@ -990,7 +990,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)
990 struct host_context_load_thread *hclt = callocz(max_threads, sizeof(*hclt));
991
992 size_t thread_index;
993 - dfe_start_reentrant(rrdhost_root_index, host) {
993 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
994 if (rrdhost_flag_check(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS) ||
995 !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))
996 continue;
@@ -1141,7 +1141,7 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
1141 if (!data->max_count)
1142 transaction_started = !db_execute(db_meta, "BEGIN TRANSACTION;");
1143
1144 - dfe_start_reentrant(rrdhost_root_index, host) {
1144 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1145 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE))
1146 continue;
1147
@@ -1335,7 +1335,7 @@ static void metadata_event_loop(void *arg)
1335 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SCANNING_HOSTS)))
1336 break;
1337
1338 - if (unittest_running)
1338 + if (rrdb.unittest_running)
1339 break;
1340
1341 data = mallocz(sizeof(*data));
@@ -1363,7 +1363,7 @@ static void metadata_event_loop(void *arg)
1363 }
1364 break;
1365 case METADATA_LOAD_HOST_CONTEXT:;
1366 - if (unittest_running)
1366 + if (rrdb.unittest_running)
1367 break;
1368
1369 data = callocz(1,sizeof(*data));
database/storage_engine.c
+42 -101
@@ -1,118 +1,59 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "storage_engine.h"
4 +
5 #include "ram/rrddim_mem.h"
6 #ifdef ENABLE_DBENGINE
7 #include "engine/rrdengineapi.h"
8 #endif
9
9 -static STORAGE_ENGINE engines[] = {
10 - {
11 - .id = RRD_MEMORY_MODE_NONE,
12 - .name = RRD_MEMORY_MODE_NONE_NAME,
13 - .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
14 - .api = {
15 - .metric_get = rrddim_metric_get,
16 - .metric_get_or_create = rrddim_metric_get_or_create,
17 - .metric_dup = rrddim_metric_dup,
18 - .metric_release = rrddim_metric_release,
19 - .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
20 - }
21 - },
22 - {
23 - .id = RRD_MEMORY_MODE_RAM,
24 - .name = RRD_MEMORY_MODE_RAM_NAME,
25 - .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
26 - .api = {
27 - .metric_get = rrddim_metric_get,
28 - .metric_get_or_create = rrddim_metric_get_or_create,
29 - .metric_dup = rrddim_metric_dup,
30 - .metric_release = rrddim_metric_release,
31 - .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
32 - }
33 - },
34 - {
35 - .id = RRD_MEMORY_MODE_MAP,
36 - .name = RRD_MEMORY_MODE_MAP_NAME,
37 - .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
38 - .api = {
39 - .metric_get = rrddim_metric_get,
40 - .metric_get_or_create = rrddim_metric_get_or_create,
41 - .metric_dup = rrddim_metric_dup,
42 - .metric_release = rrddim_metric_release,
43 - .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
44 - }
45 - },
46 - {
47 - .id = RRD_MEMORY_MODE_SAVE,
48 - .name = RRD_MEMORY_MODE_SAVE_NAME,
49 - .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
50 - .api = {
51 - .metric_get = rrddim_metric_get,
52 - .metric_get_or_create = rrddim_metric_get_or_create,
53 - .metric_dup = rrddim_metric_dup,
54 - .metric_release = rrddim_metric_release,
55 - .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
56 - }
57 - },
58 - {
59 - .id = RRD_MEMORY_MODE_ALLOC,
60 - .name = RRD_MEMORY_MODE_ALLOC_NAME,
61 - .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
62 - .api = {
63 - .metric_get = rrddim_metric_get,
64 - .metric_get_or_create = rrddim_metric_get_or_create,
65 - .metric_dup = rrddim_metric_dup,
66 - .metric_release = rrddim_metric_release,
67 - .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
68 - }
69 - },
10 #ifdef ENABLE_DBENGINE
71 - {
72 - .id = RRD_MEMORY_MODE_DBENGINE,
73 - .name = RRD_MEMORY_MODE_DBENGINE_NAME,
74 - .backend = STORAGE_ENGINE_BACKEND_DBENGINE,
75 - .api = {
76 - .metric_get = rrdeng_metric_get,
77 - .metric_get_or_create = rrdeng_metric_get_or_create,
78 - .metric_dup = rrdeng_metric_dup,
79 - .metric_release = rrdeng_metric_release,
80 - .metric_retention_by_uuid = rrdeng_metric_retention_by_uuid,
81 - }
82 - },
11 +STORAGE_ENGINE_ID default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
12 +#else
13 +STORAGE_ENGINE_ID default_storage_engine_id = STORAGE_ENGINE_SAVE;
14 #endif
84 - { .id = RRD_MEMORY_MODE_NONE, .name = NULL }
85 -};
15
87 -STORAGE_ENGINE* storage_engine_find(const char* name)
88 -{
89 - for (STORAGE_ENGINE* it = engines; it->name; it++) {
90 - if (strcmp(it->name, name) == 0)
91 - return it;
92 - }
93 - return NULL;
94 -}
16 +const char *storage_engine_name(STORAGE_ENGINE_ID id) {
17 + switch(id) {
18 + case STORAGE_ENGINE_RAM:
19 + return STORAGE_ENGINE_RAM_NAME;
20
96 -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode)
97 -{
98 - for (STORAGE_ENGINE* it = engines; it->name; it++) {
99 - if (it->id == mmode)
100 - return it;
101 - }
102 - return NULL;
103 -}
21 + case STORAGE_ENGINE_MAP:
22 + return STORAGE_ENGINE_MAP_NAME;
23 +
24 + case STORAGE_ENGINE_NONE:
25 + return STORAGE_ENGINE_NONE_NAME;
26
105 -STORAGE_ENGINE* storage_engine_foreach_init()
106 -{
107 - // Assuming at least one engine exists
108 - return &engines[0];
27 + case STORAGE_ENGINE_SAVE:
28 + return STORAGE_ENGINE_SAVE_NAME;
29 +
30 + case STORAGE_ENGINE_ALLOC:
31 + return STORAGE_ENGINE_ALLOC_NAME;
32 +
33 + case STORAGE_ENGINE_DBENGINE:
34 + return STORAGE_ENGINE_DBENGINE_NAME;
35 +
36 + default:
37 + __builtin_unreachable();
38 + }
39 }
40
111 -STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it)
112 -{
113 - if (!it || !it->name)
114 - return NULL;
41 +bool storage_engine_id(const char *name, STORAGE_ENGINE_ID *id) {
42 + if (!strcmp(name, STORAGE_ENGINE_RAM_NAME)) {
43 + *id = STORAGE_ENGINE_RAM;
44 + } else if (!strcmp(name, STORAGE_ENGINE_MAP_NAME)) {
45 + *id = STORAGE_ENGINE_MAP;
46 + } else if (!strcmp(name, STORAGE_ENGINE_NONE_NAME)) {
47 + *id = STORAGE_ENGINE_NONE;
48 + } else if (!strcmp(name, STORAGE_ENGINE_SAVE_NAME)) {
49 + *id = STORAGE_ENGINE_SAVE;
50 + } else if (!strcmp(name, STORAGE_ENGINE_ALLOC_NAME)) {
51 + *id = STORAGE_ENGINE_ALLOC;
52 + } else if (!strcmp(name, STORAGE_ENGINE_DBENGINE_NAME)) {
53 + *id = STORAGE_ENGINE_DBENGINE;
54 + } else {
55 + return false;
56 + }
57
116 - it++;
117 - return it->name ? it : NULL;
58 + return true;
59 }
database/storage_engine.h
+445 -6
@@ -1,12 +1,451 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#ifndef NETDATA_STORAGEENGINEAPI_H
4 -#define NETDATA_STORAGEENGINEAPI_H
3 +#ifndef NETDATA_STORAGE_ENGINE_H
4 +#define NETDATA_STORAGE_ENGINE_H
5
6 -#include "rrd.h"
6 +#include "ram/rrddim_mem.h"
7 +#include "engine/rrddim_eng.h"
8
8 -// Iterator over existing engines
9 -STORAGE_ENGINE* storage_engine_foreach_init();
10 -STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);
9 +extern STORAGE_ENGINE_ID default_storage_engine_id;
10
11 +static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance, uuid_t *uuid)
12 +{
13 + switch (id) {
14 + case STORAGE_ENGINE_NONE:
15 + case STORAGE_ENGINE_RAM:
16 + case STORAGE_ENGINE_MAP:
17 + case STORAGE_ENGINE_SAVE:
18 + case STORAGE_ENGINE_ALLOC:
19 + return rrddim_metrics_group_get(instance, uuid);
20 +#ifdef ENABLE_DBENGINE
21 + case STORAGE_ENGINE_DBENGINE:
22 + return rrdeng_metrics_group_get(instance, uuid);
23 #endif
24 + default:
25 + __builtin_unreachable();
26 + }
27 +}
28 +
29 +static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance, STORAGE_METRICS_GROUP *smg)
30 +{
31 + switch (id) {
32 + case STORAGE_ENGINE_NONE:
33 + case STORAGE_ENGINE_RAM:
34 + case STORAGE_ENGINE_MAP:
35 + case STORAGE_ENGINE_SAVE:
36 + case STORAGE_ENGINE_ALLOC:
37 + rrddim_metrics_group_release(instance, smg);
38 + break;
39 +#ifdef ENABLE_DBENGINE
40 + case STORAGE_ENGINE_DBENGINE:
41 + rrdeng_metrics_group_release(instance, smg);
42 + break;
43 +#endif
44 + default:
45 + __builtin_unreachable();
46 + }
47 +}
48 +
49 +static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg)
50 +{
51 + switch (id) {
52 + case STORAGE_ENGINE_NONE:
53 + case STORAGE_ENGINE_RAM:
54 + case STORAGE_ENGINE_MAP:
55 + case STORAGE_ENGINE_SAVE:
56 + case STORAGE_ENGINE_ALLOC:
57 + return rrddim_collect_init(metric_handle, update_every, smg);
58 +#ifdef ENABLE_DBENGINE
59 + case STORAGE_ENGINE_DBENGINE:
60 + return rrdeng_store_metric_init(metric_handle, update_every, smg);
61 +#endif
62 + default:
63 + __builtin_unreachable();
64 + }
65 +}
66 +
67 +static inline void storage_engine_store_metric(
68 + STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
69 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
70 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags)
71 +{
72 + switch (id) {
73 + case STORAGE_ENGINE_NONE:
74 + case STORAGE_ENGINE_RAM:
75 + case STORAGE_ENGINE_MAP:
76 + case STORAGE_ENGINE_SAVE:
77 + case STORAGE_ENGINE_ALLOC:
78 + return rrddim_collect_store_metric(collection_handle, point_in_time_ut,
79 + n, min_value, max_value,
80 + count, anomaly_count, flags);
81 +#ifdef ENABLE_DBENGINE
82 + case STORAGE_ENGINE_DBENGINE:
83 + return rrdeng_store_metric_next(collection_handle, point_in_time_ut,
84 + n, min_value, max_value,
85 + count, anomaly_count, flags);
86 +#endif
87 + default:
88 + __builtin_unreachable();
89 + }
90 +}
91 +
92 +static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
93 +{
94 + switch (id) {
95 + case STORAGE_ENGINE_NONE:
96 + case STORAGE_ENGINE_RAM:
97 + case STORAGE_ENGINE_MAP:
98 + case STORAGE_ENGINE_SAVE:
99 + case STORAGE_ENGINE_ALLOC:
100 + return rrddim_disk_space_max(db_instance);
101 +#ifdef ENABLE_DBENGINE
102 + case STORAGE_ENGINE_DBENGINE:
103 + return rrdeng_disk_space_max(db_instance);
104 +#endif
105 + default:
106 + __builtin_unreachable();
107 + }
108 +}
109 +
110 +static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
111 +{
112 + switch (id) {
113 + case STORAGE_ENGINE_NONE:
114 + case STORAGE_ENGINE_RAM:
115 + case STORAGE_ENGINE_MAP:
116 + case STORAGE_ENGINE_SAVE:
117 + case STORAGE_ENGINE_ALLOC:
118 + return rrddim_disk_space_used(db_instance);
119 +#ifdef ENABLE_DBENGINE
120 + case STORAGE_ENGINE_DBENGINE:
121 + return rrdeng_disk_space_max(db_instance);
122 +#endif
123 + default:
124 + __builtin_unreachable();
125 + }
126 +}
127 +
128 +static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
129 +{
130 + switch (id) {
131 + case STORAGE_ENGINE_NONE:
132 + case STORAGE_ENGINE_RAM:
133 + case STORAGE_ENGINE_MAP:
134 + case STORAGE_ENGINE_SAVE:
135 + case STORAGE_ENGINE_ALLOC:
136 + return rrddim_global_first_time_s(db_instance);
137 +#ifdef ENABLE_DBENGINE
138 + case STORAGE_ENGINE_DBENGINE:
139 + return rrdeng_global_first_time_s(db_instance);
140 +#endif
141 + default:
142 + __builtin_unreachable();
143 + }
144 +}
145 +
146 +static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
147 +{
148 + switch (id) {
149 + case STORAGE_ENGINE_NONE:
150 + case STORAGE_ENGINE_RAM:
151 + case STORAGE_ENGINE_MAP:
152 + case STORAGE_ENGINE_SAVE:
153 + case STORAGE_ENGINE_ALLOC:
154 + return rrddim_currently_collected_metrics(db_instance);
155 +#ifdef ENABLE_DBENGINE
156 + case STORAGE_ENGINE_DBENGINE:
157 + return rrdeng_currently_collected_metrics(db_instance);
158 +#endif
159 + default:
160 + __builtin_unreachable();
161 + }
162 +}
163 +
164 +static inline void storage_engine_store_flush(STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle)
165 +{
166 + switch (id) {
167 + case STORAGE_ENGINE_NONE:
168 + case STORAGE_ENGINE_RAM:
169 + case STORAGE_ENGINE_MAP:
170 + case STORAGE_ENGINE_SAVE:
171 + case STORAGE_ENGINE_ALLOC:
172 + rrddim_store_metric_flush(collection_handle);
173 + return;
174 +#ifdef ENABLE_DBENGINE
175 + case STORAGE_ENGINE_DBENGINE:
176 + rrdeng_store_metric_flush_current_page(collection_handle);
177 + return;
178 +#endif
179 + default:
180 + __builtin_unreachable();
181 + }
182 +}
183 +
184 +// a finalization function to run after collection is over
185 +// returns 1 if it's safe to delete the dimension
186 +static inline int storage_engine_store_finalize(STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle)
187 +{
188 + switch (id) {
189 + case STORAGE_ENGINE_NONE:
190 + case STORAGE_ENGINE_RAM:
191 + case STORAGE_ENGINE_MAP:
192 + case STORAGE_ENGINE_SAVE:
193 + case STORAGE_ENGINE_ALLOC:
194 + return rrddim_collect_finalize(collection_handle);
195 +#ifdef ENABLE_DBENGINE
196 + case STORAGE_ENGINE_DBENGINE:
197 + return rrdeng_store_metric_finalize(collection_handle);
198 +#endif
199 + default:
200 + __builtin_unreachable();
201 + }
202 +}
203 +
204 +static inline void storage_engine_store_change_collection_frequency(STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle, int update_every)
205 +{
206 + switch (id) {
207 + case STORAGE_ENGINE_NONE:
208 + case STORAGE_ENGINE_RAM:
209 + case STORAGE_ENGINE_MAP:
210 + case STORAGE_ENGINE_SAVE:
211 + case STORAGE_ENGINE_ALLOC:
212 + rrddim_store_metric_change_collection_frequency(collection_handle, update_every);
213 + return;
214 +#ifdef ENABLE_DBENGINE
215 + case STORAGE_ENGINE_DBENGINE:
216 + rrdeng_store_metric_change_collection_frequency(collection_handle, update_every);
217 + return;
218 +#endif
219 + default:
220 + __builtin_unreachable();
221 + }
222 +}
223 +
224 +static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
225 +{
226 + switch (id) {
227 + case STORAGE_ENGINE_NONE:
228 + case STORAGE_ENGINE_RAM:
229 + case STORAGE_ENGINE_MAP:
230 + case STORAGE_ENGINE_SAVE:
231 + case STORAGE_ENGINE_ALLOC:
232 + return rrddim_query_oldest_time_s(db_metric_handle);
233 +#ifdef ENABLE_DBENGINE
234 + case STORAGE_ENGINE_DBENGINE:
235 + return rrdeng_metric_oldest_time(db_metric_handle);
236 +#endif
237 + default:
238 + __builtin_unreachable();
239 + }
240 +}
241 +
242 +static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
243 +{
244 + switch (id) {
245 + case STORAGE_ENGINE_NONE:
246 + case STORAGE_ENGINE_RAM:
247 + case STORAGE_ENGINE_MAP:
248 + case STORAGE_ENGINE_SAVE:
249 + case STORAGE_ENGINE_ALLOC:
250 + return rrddim_query_latest_time_s(db_metric_handle);
251 +#ifdef ENABLE_DBENGINE
252 + case STORAGE_ENGINE_DBENGINE:
253 + return rrdeng_metric_latest_time(db_metric_handle);
254 +#endif
255 + default:
256 + __builtin_unreachable();
257 + }
258 +}
259 +
260 +static inline void storage_engine_query_init(
261 + STORAGE_ENGINE_ID id,
262 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
263 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority)
264 +{
265 + handle->id = id;
266 +
267 + switch (id) {
268 + case STORAGE_ENGINE_NONE:
269 + case STORAGE_ENGINE_RAM:
270 + case STORAGE_ENGINE_MAP:
271 + case STORAGE_ENGINE_SAVE:
272 + case STORAGE_ENGINE_ALLOC:
273 + rrddim_query_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
274 + return;
275 +#ifdef ENABLE_DBENGINE
276 + case STORAGE_ENGINE_DBENGINE:
277 + rrdeng_load_metric_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
278 + return;
279 +#endif
280 + default:
281 + __builtin_unreachable();
282 + }
283 +}
284 +
285 +static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *handle)
286 +{
287 + switch (handle->id) {
288 + case STORAGE_ENGINE_NONE:
289 + case STORAGE_ENGINE_RAM:
290 + case STORAGE_ENGINE_MAP:
291 + case STORAGE_ENGINE_SAVE:
292 + case STORAGE_ENGINE_ALLOC:
293 + return rrddim_query_next_metric(handle);
294 +#ifdef ENABLE_DBENGINE
295 + case STORAGE_ENGINE_DBENGINE:
296 + return rrdeng_load_metric_next(handle);
297 +#endif
298 + default:
299 + __builtin_unreachable();
300 + }
301 +}
302 +
303 +static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *handle)
304 +{
305 + switch (handle->id) {
306 + case STORAGE_ENGINE_NONE:
307 + case STORAGE_ENGINE_RAM:
308 + case STORAGE_ENGINE_MAP:
309 + case STORAGE_ENGINE_SAVE:
310 + case STORAGE_ENGINE_ALLOC:
311 + return rrddim_query_is_finished(handle);
312 +#ifdef ENABLE_DBENGINE
313 + case STORAGE_ENGINE_DBENGINE:
314 + return rrdeng_load_metric_is_finished(handle);
315 +#endif
316 + default:
317 + __builtin_unreachable();
318 + }
319 +}
320 +
321 +static inline void storage_engine_query_finalize(struct storage_engine_query_handle *handle)
322 +{
323 + switch (handle->id) {
324 + case STORAGE_ENGINE_NONE:
325 + case STORAGE_ENGINE_RAM:
326 + case STORAGE_ENGINE_MAP:
327 + case STORAGE_ENGINE_SAVE:
328 + case STORAGE_ENGINE_ALLOC:
329 + rrddim_query_finalize(handle);
330 + return;
331 +#ifdef ENABLE_DBENGINE
332 + case STORAGE_ENGINE_DBENGINE:
333 + rrdeng_load_metric_finalize(handle);
334 + return;
335 +#endif
336 + default:
337 + __builtin_unreachable();
338 + }
339 +}
340 +
341 +static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *handle)
342 +{
343 + switch (handle->id) {
344 + case STORAGE_ENGINE_NONE:
345 + case STORAGE_ENGINE_RAM:
346 + case STORAGE_ENGINE_MAP:
347 + case STORAGE_ENGINE_SAVE:
348 + case STORAGE_ENGINE_ALLOC:
349 + return rrddim_query_align_to_optimal_before(handle);
350 +#ifdef ENABLE_DBENGINE
351 + case STORAGE_ENGINE_DBENGINE:
352 + return rrdeng_load_align_to_optimal_before(handle);
353 +#endif
354 + default:
355 + __builtin_unreachable();
356 + }
357 +}
358 +
359 +static inline bool storage_engine_metric_retention(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s)
360 +{
361 + switch (id) {
362 + case STORAGE_ENGINE_NONE:
363 + case STORAGE_ENGINE_RAM:
364 + case STORAGE_ENGINE_MAP:
365 + case STORAGE_ENGINE_SAVE:
366 + case STORAGE_ENGINE_ALLOC:
367 + return rrddim_metric_retention_by_uuid(db_instance, uuid, first_entry_s, last_entry_s);
368 +#ifdef ENABLE_DBENGINE
369 + case STORAGE_ENGINE_DBENGINE:
370 + return rrdeng_metric_retention_by_uuid(db_instance, uuid, first_entry_s, last_entry_s);
371 +#endif
372 + default:
373 + __builtin_unreachable();
374 + }
375 +}
376 +
377 +static inline STORAGE_METRIC_HANDLE *storage_engine_metric_get(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance, uuid_t *uuid)
378 +{
379 + switch (id) {
380 + case STORAGE_ENGINE_NONE:
381 + case STORAGE_ENGINE_RAM:
382 + case STORAGE_ENGINE_MAP:
383 + case STORAGE_ENGINE_SAVE:
384 + case STORAGE_ENGINE_ALLOC:
385 + return rrddim_metric_get(instance, uuid);
386 +#ifdef ENABLE_DBENGINE
387 + case STORAGE_ENGINE_DBENGINE:
388 + return rrdeng_metric_get(instance, uuid);
389 +#endif
390 + default:
391 + __builtin_unreachable();
392 + }
393 +}
394 +
395 +static inline STORAGE_METRIC_HANDLE *storage_engine_metric_get_or_create(RRDDIM *rd, STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance)
396 +{
397 + switch (id) {
398 + case STORAGE_ENGINE_NONE:
399 + case STORAGE_ENGINE_RAM:
400 + case STORAGE_ENGINE_MAP:
401 + case STORAGE_ENGINE_SAVE:
402 + case STORAGE_ENGINE_ALLOC:
403 + return rrddim_metric_get_or_create(rd, instance);
404 +#ifdef ENABLE_DBENGINE
405 + case STORAGE_ENGINE_DBENGINE:
406 + return rrdeng_metric_get_or_create(rd, instance);
407 +#endif
408 + default:
409 + __builtin_unreachable();
410 + }
411 +}
412 +
413 +static inline void storage_engine_metric_release(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
414 +{
415 + switch (id) {
416 + case STORAGE_ENGINE_NONE:
417 + case STORAGE_ENGINE_RAM:
418 + case STORAGE_ENGINE_MAP:
419 + case STORAGE_ENGINE_SAVE:
420 + case STORAGE_ENGINE_ALLOC:
421 + rrddim_metric_release(db_metric_handle);
422 + break;
423 +#ifdef ENABLE_DBENGINE
424 + case STORAGE_ENGINE_DBENGINE:
425 + rrdeng_metric_release(db_metric_handle);
426 + break;
427 +#endif
428 + default:
429 + __builtin_unreachable();
430 + }
431 +}
432 +
433 +static inline STORAGE_METRIC_HANDLE *storage_engine_metric_dup(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
434 +{
435 + switch (id) {
436 + case STORAGE_ENGINE_NONE:
437 + case STORAGE_ENGINE_RAM:
438 + case STORAGE_ENGINE_MAP:
439 + case STORAGE_ENGINE_SAVE:
440 + case STORAGE_ENGINE_ALLOC:
441 + return rrddim_metric_dup(db_metric_handle);
442 +#ifdef ENABLE_DBENGINE
443 + case STORAGE_ENGINE_DBENGINE:
444 + return rrdeng_metric_dup(db_metric_handle);
445 +#endif
446 + default:
447 + __builtin_unreachable();
448 + }
449 +}
450 +
451 +#endif /* NETDATA_STORAGE_ENGINE_H */
database/storage_engine_types.h new
+56
@@ -0,0 +1,56 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STORAGE_ENGINE_TYPES_H
4 +#define NETDATA_STORAGE_ENGINE_TYPES_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +
8 +typedef struct rrddim RRDDIM;
9 +typedef struct storage_instance STORAGE_INSTANCE;
10 +typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
11 +typedef struct storage_alignment STORAGE_METRICS_GROUP;
12 +typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
13 +typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
14 +
15 +typedef enum __attribute__ ((__packed__)) {
16 + STORAGE_ENGINE_NONE = 0,
17 + STORAGE_ENGINE_RAM = 1,
18 + STORAGE_ENGINE_MAP = 2,
19 + STORAGE_ENGINE_SAVE = 3,
20 + STORAGE_ENGINE_ALLOC = 4,
21 + STORAGE_ENGINE_DBENGINE = 5,
22 +} STORAGE_ENGINE_ID;
23 +
24 +#define STORAGE_ENGINE_NONE_NAME "none"
25 +#define STORAGE_ENGINE_RAM_NAME "ram"
26 +#define STORAGE_ENGINE_MAP_NAME "map"
27 +#define STORAGE_ENGINE_SAVE_NAME "save"
28 +#define STORAGE_ENGINE_ALLOC_NAME "alloc"
29 +#define STORAGE_ENGINE_DBENGINE_NAME "dbengine"
30 +
31 +const char *storage_engine_name(STORAGE_ENGINE_ID id);
32 +bool storage_engine_id(const char *name, STORAGE_ENGINE_ID *id);
33 +
34 +typedef enum __attribute__ ((__packed__)) storage_priority {
35 + STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
36 + STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
37 +
38 + STORAGE_PRIORITY_HIGH,
39 + STORAGE_PRIORITY_NORMAL,
40 + STORAGE_PRIORITY_LOW,
41 + STORAGE_PRIORITY_BEST_EFFORT,
42 +
43 + STORAGE_PRIORITY_SYNCHRONOUS,
44 +
45 + STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
46 +} STORAGE_PRIORITY;
47 +
48 +struct storage_engine_query_handle {
49 + time_t start_time_s;
50 + time_t end_time_s;
51 + STORAGE_PRIORITY priority;
52 + STORAGE_ENGINE_ID id;
53 + STORAGE_QUERY_HANDLE *handle;
54 +};
55 +
56 +#endif /* NETDATA_STORAGE_ENGINE_TYPES_H */
\ No newline at end of file
exporting/check_filters.c
+3 -3
@@ -25,7 +25,7 @@ int rrdhost_is_exportable(struct instance *instance, RRDHOST *host)
25 RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index];
26
27 if (unlikely((*flags & (RRDHOST_FLAG_EXPORTING_SEND | RRDHOST_FLAG_EXPORTING_DONT_SEND)) == 0)) {
28 - const char *host_name = (host == localhost) ? "localhost" : rrdhost_hostname(host);
28 + const char *host_name = (host == rrdb.localhost) ? "localhost" : rrdhost_hostname(host);
29
30 if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) {
31 *flags |= RRDHOST_FLAG_EXPORTING_SEND;
@@ -79,8 +79,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)
79 return 0;
80 }
81
82 - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
83 - netdata_log_debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode));
82 + if(unlikely(st->storage_engine_id == STORAGE_ENGINE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
83 + netdata_log_debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), storage_engine_name(host->storage_engine_id));
84 return 0;
85 }
86
exporting/exporting_engine.c
+1 -1
@@ -193,7 +193,7 @@ void *exporting_main(void *ptr)
193 RRDDIM *rd_main_system = NULL;
194 create_main_rusage_chart(&st_main_rusage, &rd_main_user, &rd_main_system);
195
196 - usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
196 + usec_t step_ut = rrdb.localhost->update_every * USEC_PER_SEC;
197 heartbeat_t hb;
198 heartbeat_init(&hb);
199
exporting/graphite/graphite.c
+2 -2
@@ -135,7 +135,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
135 instance->buffer,
136 "%s.%s.%s.%s%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n",
137 instance->config.prefix,
138 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
138 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
139 chart_name,
140 dimension_name,
141 (host->tags) ? ";" : "",
@@ -181,7 +181,7 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
181 instance->buffer,
182 "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
183 instance->config.prefix,
184 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
184 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
185 chart_name,
186 dimension_name,
187 (host->tags) ? ";" : "",
exporting/json/json.c
+2 -2
@@ -186,7 +186,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
186 "\"timestamp\":%llu}",
187
188 instance->config.prefix,
189 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
189 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
190 tags_pre,
191 tags,
192 tags_post,
@@ -270,7 +270,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
270 "\"timestamp\": %llu}",
271
272 instance->config.prefix,
273 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
273 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
274 tags_pre,
275 tags,
276 tags_post,
exporting/opentsdb/opentsdb.c
+4 -4
@@ -192,7 +192,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
192 dimension_name,
193 (unsigned long long)rd->collector.last_collected_time.tv_sec,
194 rd->collector.last_collected_value,
195 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
195 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
196 (host->tags) ? " " : "",
197 (host->tags) ? rrdhost_tags(host) : "",
198 (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
@@ -238,7 +238,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
238 dimension_name,
239 (unsigned long long)last_t,
240 value,
241 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
241 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
242 (host->tags) ? " " : "",
243 (host->tags) ? rrdhost_tags(host) : "",
244 (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
@@ -334,7 +334,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
334 dimension_name,
335 (unsigned long long)rd->collector.last_collected_time.tv_sec,
336 rd->collector.last_collected_value,
337 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
337 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
338 (host->tags) ? " " : "",
339 (host->tags) ? rrdhost_tags(host) : "",
340 instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
@@ -390,7 +390,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
390 dimension_name,
391 (unsigned long long)last_t,
392 value,
393 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
393 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
394 (host->tags) ? " " : "",
395 (host->tags) ? rrdhost_tags(host) : "",
396 instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
exporting/process_data.c
+4 -4
@@ -44,7 +44,7 @@ int mark_scheduled_instances(struct engine *engine)
44
45 for (struct instance *instance = engine->instance_root; instance; instance = instance->next) {
46 if (!instance->disabled && (engine->now % instance->config.update_every >=
47 - instance->config.update_every - localhost->rrd_update_every)) {
47 + instance->config.update_every - rrdb.localhost->update_every)) {
48 instance->scheduled = 1;
49 instances_were_scheduled = 1;
50 instance->before = engine->now;
@@ -77,8 +77,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
77 time_t before = instance->before;
78
79 // find the edges of the rrd database for this chart
80 - time_t first_t = storage_engine_oldest_time_s(rd->tiers[0].backend, rd->tiers[0].db_metric_handle);
81 - time_t last_t = storage_engine_latest_time_s(rd->tiers[0].backend, rd->tiers[0].db_metric_handle);
80 + time_t first_t = storage_engine_oldest_time_s(st->storage_engine_id, rd->tiers[0].db_metric_handle);
81 + time_t last_t = storage_engine_latest_time_s(st->storage_engine_id, rd->tiers[0].db_metric_handle);
82 time_t update_every = st->update_every;
83 struct storage_engine_query_handle handle;
84
@@ -126,7 +126,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
126 size_t counter = 0;
127 NETDATA_DOUBLE sum = 0;
128
129 - for (storage_engine_query_init(rd->tiers[0].backend, rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_SYNCHRONOUS); !storage_engine_query_is_finished(&handle);) {
129 + for (storage_engine_query_init(st->storage_engine_id, rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_SYNCHRONOUS); !storage_engine_query_is_finished(&handle);) {
130 STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
131 points_read++;
132
exporting/prometheus/prometheus.c
+3 -3
@@ -60,14 +60,14 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN
60 }
61
62 if (unlikely(
63 - st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&
63 + st->storage_engine_id == STORAGE_ENGINE_NONE &&
64 !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
65 netdata_log_debug(
66 D_EXPORTING,
67 "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.",
68 rrdset_id(st),
69 rrdhost_hostname(host),
70 - rrd_memory_mode_name(host->rrd_memory_mode));
70 + storage_engine_name(host->storage_engine_id));
71 return 0;
72 }
73
@@ -957,7 +957,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
957 prometheus_exporter_instance->before,
958 output_options);
959
960 - dfe_start_reentrant(rrdhost_root_index, host)
960 + dfe_start_reentrant(rrdb.rrdhost_root_index, host)
961 {
962 rrd_stats_api_v1_charts_allmetrics_prometheus(
963 prometheus_exporter_instance, host, filter_string, wb, prefix, exporting_options, 1, output_options);
exporting/prometheus/remote_write/remote_write.c
+6 -6
@@ -170,7 +170,7 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
170 char hostname[PROMETHEUS_ELEMENT_MAX + 1];
171 prometheus_label_copy(
172 hostname,
173 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
173 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
174 PROMETHEUS_ELEMENT_MAX);
175
176 add_host_info(
@@ -249,7 +249,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
249 "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', "
250 "its last data collection (%lu) is not within our timeframe (%lu to %lu)",
251 rrddim_id(rd), rrdset_id(rd->rrdset),
252 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
252 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
253 (unsigned long)rd->collector.last_collected_time.tv_sec,
254 (unsigned long)instance->after,
255 (unsigned long)instance->before);
@@ -274,7 +274,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
274 add_metric(
275 connector_specific_data->write_request,
276 name, chart, family, dimension,
277 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
277 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
278 rd->collector.last_collected_value, timeval_msec(&rd->collector.last_collected_time));
279 } else {
280 // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
@@ -291,7 +291,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
291 add_metric(
292 connector_specific_data->write_request,
293 name, chart, family, NULL,
294 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
294 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
295 rd->collector.last_collected_value, timeval_msec(&rd->collector.last_collected_time));
296 }
297 } else {
@@ -316,7 +316,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
316 add_metric(
317 connector_specific_data->write_request,
318 name, chart, family, dimension,
319 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
319 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
320 value, last_t * MSEC_PER_SEC);
321 }
322 }
@@ -346,7 +346,7 @@ static int format_variable_prometheus_remote_write_callback(const DICTIONARY_ITE
346
347 NETDATA_DOUBLE value = rrdvar2number(rv);
348 add_variable(connector_specific_data->write_request, name,
349 - (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS);
349 + (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS);
350 }
351
352 return 0;
exporting/read_config.c
+2 -2
@@ -380,11 +380,11 @@ struct engine *read_exporting_config()
380
381 tmp_instance->config.options = exporting_parse_data_source(data_source, tmp_instance->config.options);
382 if (EXPORTING_OPTIONS_DATA_SOURCE(tmp_instance->config.options) != EXPORTING_SOURCE_DATA_AS_COLLECTED &&
383 - tmp_instance->config.update_every % localhost->rrd_update_every)
383 + tmp_instance->config.update_every % rrdb.localhost->update_every)
384 netdata_log_info(
385 "The update interval %d for instance %s is not a multiple of the database update interval %d. "
386 "Metric values will deviate at different points in time.",
387 - tmp_instance->config.update_every, tmp_instance->config.name, localhost->rrd_update_every);
387 + tmp_instance->config.update_every, tmp_instance->config.name, rrdb.localhost->update_every);
388
389 if (exporter_get_boolean(instance_name, "send configured labels", CONFIG_BOOLEAN_YES))
390 tmp_instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
exporting/send_internal_metrics.c
+1 -1
@@ -19,7 +19,7 @@ void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_
19
20 *st_rusage = rrdset_create_localhost(
21 "netdata", "exporting_main_thread_cpu", NULL, "exporting", "exporting_cpu_usage", "Netdata Main Exporting Thread CPU Usage",
22 - "milliseconds/s", "exporting", NULL, 130600, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
22 + "milliseconds/s", "exporting", NULL, 130600, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
23
24 *rd_user = rrddim_add(*st_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
25 *rd_system = rrddim_add(*st_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
exporting/tests/exporting_fixtures.c
+2 -2
@@ -64,7 +64,7 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
64 st->name = string_strdupz("chart_name");
65
66 st->update_every = 1;
67 - st->rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
67 + st->storage_engine_id = STORAGE_ENGINE_SAVE;
68
69 st->rrdhost = host;
70
@@ -77,7 +77,7 @@ int setup_rrdhost()
77 {
78 localhost = calloc(1, sizeof(RRDHOST));
79
80 - localhost->rrd_update_every = 1;
80 + localhost->update_every = 1;
81
82 localhost->tags = string_strdupz("TAG1=VALUE1 TAG2=VALUE2");
83
exporting/tests/netdata_doubles.c
+6 -6
@@ -99,7 +99,7 @@ RRDSET *rrdset_create_custom(
99 long priority,
100 int update_every,
101 RRDSET_TYPE chart_type,
102 - RRD_MEMORY_MODE memory_mode,
102 + STORAGE_ENGINE_ID storage_engine_id,
103 long history_entries)
104 {
105 check_expected_ptr(host);
@@ -115,7 +115,7 @@ RRDSET *rrdset_create_custom(
115 check_expected(priority);
116 check_expected(update_every);
117 check_expected(chart_type);
118 - UNUSED(memory_mode);
118 + UNUSED(storage_engine_id);
119 UNUSED(history_entries);
120
121 function_called();
@@ -145,7 +145,7 @@ RRDDIM *rrddim_add_custom(
145 collected_number multiplier,
146 collected_number divisor,
147 RRD_ALGORITHM algorithm,
148 - RRD_MEMORY_MODE memory_mode)
148 + STORAGE_ENGINE_ID storage_engine_id)
149 {
150 check_expected_ptr(st);
151 UNUSED(id);
@@ -153,7 +153,7 @@ RRDDIM *rrddim_add_custom(
153 check_expected(multiplier);
154 check_expected(divisor);
155 check_expected(algorithm);
156 - UNUSED(memory_mode);
156 + UNUSED(storage_engine_id);
157
158 function_called();
159
@@ -171,10 +171,10 @@ collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number
171 return 0;
172 }
173
174 -const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
174 +const char *storage_engine_name(STORAGE_ENGINE_ID id)
175 {
176 (void)id;
177 - return RRD_MEMORY_MODE_NONE_NAME;
177 + return STORAGE_ENGINE_NONE_NAME;
178 }
179
180 void rrdset_update_heterogeneous_flag(RRDSET *st)
exporting/tests/test_exporting_engine.c
+2 -2
@@ -849,7 +849,7 @@ static void test_create_main_rusage_chart(void **state)
849 expect_string(rrdset_create_custom, plugin, "exporting");
850 expect_value(rrdset_create_custom, module, NULL);
851 expect_value(rrdset_create_custom, priority, 130600);
852 - expect_value(rrdset_create_custom, update_every, localhost->rrd_update_every);
852 + expect_value(rrdset_create_custom, update_every, localhost->update_every);
853 expect_value(rrdset_create_custom, chart_type, RRDSET_TYPE_STACKED);
854 will_return(rrdset_create_custom, st_rusage);
855
@@ -1078,7 +1078,7 @@ static void test_can_send_rrdset(void **state)
1078 assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0);
1079 rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
1080
1081 - st->rrd_memory_mode = RRD_MEMORY_MODE_NONE;
1081 + st->storage_engine_id = STORAGE_ENGINE_NONE;
1082 prometheus_exporter_instance->config.options |= EXPORTING_SOURCE_DATA_AVERAGE;
1083 assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0);
1084 }
health/health.c
+8 -8
@@ -254,7 +254,7 @@ static inline void unlink_alarm_notify_in_progress(ALARM_ENTRY *ae)
254 *
255 * @return a pointer to the user config directory
256 */
257 -inline char *health_user_config_dir(void) {
257 +static const char *health_user_config_dir(void) {
258 char buffer[FILENAME_MAX + 1];
259 snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_user_config_dir);
260 return config_get(CONFIG_SECTION_DIRECTORIES, "health config", buffer);
@@ -267,7 +267,7 @@ inline char *health_user_config_dir(void) {
267 *
268 * @return a pointer to the stock config directory.
269 */
270 -inline char *health_stock_config_dir(void) {
270 +static const char *health_stock_config_dir(void) {
271 char buffer[FILENAME_MAX + 1];
272 snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_stock_config_dir);
273 return config_get(CONFIG_SECTION_DIRECTORIES, "stock health config", buffer);
@@ -344,8 +344,8 @@ static void health_reload_host(RRDHOST *host) {
344
345 netdata_log_health("[%s]: Reloading health.", rrdhost_hostname(host));
346
347 - char *user_path = health_user_config_dir();
348 - char *stock_path = health_stock_config_dir();
347 + const char *user_path = health_user_config_dir();
348 + const char *stock_path = health_stock_config_dir();
349
350 // free all running alarms
351 rrdcalc_delete_all(host);
@@ -403,7 +403,7 @@ void health_reload(void) {
403 sql_refresh_hashes();
404
405 RRDHOST *host;
406 - dfe_start_reentrant(rrdhost_root_index, host){
406 + dfe_start_reentrant(rrdb.rrdhost_root_index, host){
407 health_reload_host(host);
408 }
409 dfe_done(host);
@@ -1080,7 +1080,7 @@ void *health_main(void *ptr) {
1080 }
1081
1082 worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
1083 - dfe_start_reentrant(rrdhost_root_index, host) {
1083 + dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1084
1085 if(unlikely(!service_running(SERVICE_HEALTH)))
1086 break;
@@ -1114,7 +1114,7 @@ void *health_main(void *ptr) {
1114 }
1115
1116 // wait until cleanup of obsolete charts on children is complete
1117 - if (host != localhost) {
1117 + if (host != rrdb.localhost) {
1118 if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
1119 netdata_log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
1120 continue;
@@ -1611,7 +1611,7 @@ void *health_main(void *ptr) {
1611 }
1612
1613 void health_add_host_labels(void) {
1614 - DICTIONARY *labels = localhost->rrdlabels;
1614 + DICTIONARY *labels = rrdb.localhost->rrdlabels;
1615
1616 // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).
1617 // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.
health/health.h
-7
@@ -51,15 +51,12 @@ void health_reload(void);
51
52 void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status);
53 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);
54 -void health_alert2json_conf(RRDHOST *host, BUFFER *wb, CONTEXTS_V2_OPTIONS all);
54 void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all);
55
56 void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf);
57 void health_api_v1_chart_custom_variables2json(RRDSET *st, BUFFER *buf);
58
60 -int health_alarm_log_open(RRDHOST *host);
59 void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
62 -void health_alarm_log_load(RRDHOST *host);
60
61 ALARM_ENTRY* health_create_alarm_entry(
62 RRDHOST *host,
@@ -91,14 +88,10 @@ ALARM_ENTRY* health_create_alarm_entry(
88 void health_alarm_log_add_entry(RRDHOST *host, ALARM_ENTRY *ae);
89
90 void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath);
94 -char *health_user_config_dir(void);
95 -char *health_stock_config_dir(void);
91 void health_alarm_log_free(RRDHOST *host);
92
93 void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae);
94
100 -void *health_cmdapi_thread(void *ptr);
101 -
95 char *health_edit_command_from_source(const char *source);
96 void sql_refresh_hashes(void);
97
health/health_config.c
+1 -1
@@ -452,7 +452,7 @@ char *health_edit_command_from_source(const char *source)
452 netdata_configured_user_config_dir,
453 file_no_path + 1,
454 temp,
455 - rrdhost_registry_hostname(localhost));
455 + rrdhost_registry_hostname(rrdb.localhost));
456 } else
457 buffer[0] = '\0';
458
libnetdata/aral/aral.c
+2 -1
@@ -1,5 +1,6 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "../libnetdata.h"
2 -#include "aral.h"
4
5 #ifdef NETDATA_TRACE_ALLOCATIONS
6 #define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS , const char *file, const char *function, size_t line
libnetdata/aral/aral.h
+1
@@ -1,3 +1,4 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef ARAL_H
4 #define ARAL_H 1
libnetdata/circular_buffer/circular_buffer.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "../libnetdata.h"
4
5 struct circular_buffer *cbuffer_new(size_t initial, size_t max, size_t *statistics) {
libnetdata/circular_buffer/circular_buffer.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef CIRCULAR_BUFFER_H
4 #define CIRCULAR_BUFFER_H 1
5
libnetdata/json/jsmn.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include <stdlib.h>
4
5 #include "jsmn.h"
libnetdata/json/jsmn.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef __JSMN_H_
4 #define __JSMN_H_
5
libnetdata/libnetdata.c
+2 -2
@@ -1139,8 +1139,8 @@ void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool r
1139 {
1140 // netdata_log_info("netdata_mmap('%s', %zu", filename, size);
1141
1142 - // MAP_SHARED is used in memory mode map
1143 - // MAP_PRIVATE is used in memory mode ram and save
1142 + // MAP_SHARED is used in storage engine MAP
1143 + // MAP_PRIVATE is used in storage engine RAM and SAVE
1144
1145 if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE)))
1146 fatal("Neither MAP_SHARED or MAP_PRIVATE were given to netdata_mmap()");
libnetdata/libnetdata.h
+1 -2
@@ -248,6 +248,7 @@ size_t judy_aral_structures(void);
248 #define ABS(x) (((x) < 0)? (-(x)) : (x))
249 #define MIN(a,b) (((a)<(b))?(a):(b))
250 #define MAX(a,b) (((a)>(b))?(a):(b))
251 +#define CLAMP(value, min, max) MIN(MAX((value), (min)), (max))
252
253 #define GUID_LEN 36
254
@@ -840,8 +841,6 @@ extern char *netdata_configured_host_prefix;
841 // BEWARE: this exists in alarm-notify.sh
842 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
843
843 -#define RRD_STORAGE_TIERS 5
844 -
844 static inline size_t struct_natural_alignment(size_t size) __attribute__((const));
845
846 #define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
libnetdata/onewayalloc/onewayalloc.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "onewayalloc.h"
4
5 // https://www.gnu.org/software/libc/manual/html_node/Aligned-Memory-Blocks.html
libnetdata/onewayalloc/onewayalloc.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef ONEWAYALLOC_H
4 #define ONEWAYALLOC_H 1
5
libnetdata/socket/security.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "../libnetdata.h"
4
5 #ifdef ENABLE_HTTPS
libnetdata/socket/security.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef NETDATA_SECURITY_H
4 # define NETDATA_SECURITY_H
5
libnetdata/string/string.h
+1
@@ -1,3 +1,4 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef NETDATA_STRING_H
4 #define NETDATA_STRING_H 1
libnetdata/worker_utilization/worker_utilization.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "worker_utilization.h"
4
5 #define WORKER_IDLE 'I'
libnetdata/worker_utilization/worker_utilization.h
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #ifndef WORKER_UTILIZATION_H
4 #define WORKER_UTILIZATION_H 1
5
ml/ad_charts.cc
+58 -55
@@ -1,18 +1,21 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 +#include "database/rrd.h"
4 #include "ad_charts.h"
5
6 +#define BUF_LEN 1024
7 +
8 void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats_t &mls) {
9 /*
10 * Machine learning status
11 */
12 if (Cfg.enable_statistics_charts) {
13 if (!host->machine_learning_status_rs) {
11 - char id_buf[1024];
12 - char name_buf[1024];
14 + char id_buf[BUF_LEN + 1];
15 + char name_buf[BUF_LEN + 1];
16
14 - snprintfz(id_buf, 1024, "machine_learning_status_on_%s", localhost->machine_guid);
15 - snprintfz(name_buf, 1024, "machine_learning_status_on_%s", rrdhost_hostname(localhost));
17 + snprintfz(id_buf, BUF_LEN, "machine_learning_status_on_%s", rrdb.localhost->machine_guid);
18 + snprintfz(name_buf, BUF_LEN, "machine_learning_status_on_%s", rrdhost_hostname(rrdb.localhost));
19
20 host->machine_learning_status_rs = rrdset_create(
21 host->rh,
@@ -26,7 +29,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
29 NETDATA_ML_PLUGIN, // plugin
30 NETDATA_ML_MODULE_TRAINING, // module
31 NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
29 - localhost->rrd_update_every, // update_every
32 + rrdb.localhost->update_every, // update_every
33 RRDSET_TYPE_LINE // chart_type
34 );
35 rrdset_flag_set(host->machine_learning_status_rs , RRDSET_FLAG_ANOMALY_DETECTION);
@@ -50,11 +53,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
53 */
54 if (Cfg.enable_statistics_charts) {
55 if (!host->metric_type_rs) {
53 - char id_buf[1024];
54 - char name_buf[1024];
56 + char id_buf[BUF_LEN + 1];
57 + char name_buf[BUF_LEN + 1];
58
56 - snprintfz(id_buf, 1024, "metric_types_on_%s", localhost->machine_guid);
57 - snprintfz(name_buf, 1024, "metric_types_on_%s", rrdhost_hostname(localhost));
59 + snprintfz(id_buf, BUF_LEN, "metric_types_on_%s", rrdb.localhost->machine_guid);
60 + snprintfz(name_buf, BUF_LEN, "metric_types_on_%s", rrdhost_hostname(rrdb.localhost));
61
62 host->metric_type_rs = rrdset_create(
63 host->rh,
@@ -68,7 +71,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
71 NETDATA_ML_PLUGIN, // plugin
72 NETDATA_ML_MODULE_TRAINING, // module
73 NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
71 - localhost->rrd_update_every, // update_every
74 + rrdb.localhost->update_every, // update_every
75 RRDSET_TYPE_LINE // chart_type
76 );
77 rrdset_flag_set(host->metric_type_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -92,11 +95,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
95 */
96 if (Cfg.enable_statistics_charts) {
97 if (!host->training_status_rs) {
95 - char id_buf[1024];
96 - char name_buf[1024];
98 + char id_buf[BUF_LEN + 1];
99 + char name_buf[BUF_LEN + 1];
100
98 - snprintfz(id_buf, 1024, "training_status_on_%s", localhost->machine_guid);
99 - snprintfz(name_buf, 1024, "training_status_on_%s", rrdhost_hostname(localhost));
101 + snprintfz(id_buf, BUF_LEN, "training_status_on_%s", rrdb.localhost->machine_guid);
102 + snprintfz(name_buf, BUF_LEN, "training_status_on_%s", rrdhost_hostname(rrdb.localhost));
103
104 host->training_status_rs = rrdset_create(
105 host->rh,
@@ -110,7 +113,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
113 NETDATA_ML_PLUGIN, // plugin
114 NETDATA_ML_MODULE_TRAINING, // module
115 NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
113 - localhost->rrd_update_every, // update_every
116 + rrdb.localhost->update_every, // update_every
117 RRDSET_TYPE_LINE // chart_type
118 );
119
@@ -147,11 +150,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
150 */
151 {
152 if (!host->dimensions_rs) {
150 - char id_buf[1024];
151 - char name_buf[1024];
153 + char id_buf[BUF_LEN + 1];
154 + char name_buf[BUF_LEN + 1];
155
153 - snprintfz(id_buf, 1024, "dimensions_on_%s", localhost->machine_guid);
154 - snprintfz(name_buf, 1024, "dimensions_on_%s", rrdhost_hostname(localhost));
156 + snprintfz(id_buf, BUF_LEN, "dimensions_on_%s", rrdb.localhost->machine_guid);
157 + snprintfz(name_buf, BUF_LEN, "dimensions_on_%s", rrdhost_hostname(rrdb.localhost));
158
159 host->dimensions_rs = rrdset_create(
160 host->rh,
@@ -165,7 +168,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
168 NETDATA_ML_PLUGIN, // plugin
169 NETDATA_ML_MODULE_TRAINING, // module
170 ML_CHART_PRIO_DIMENSIONS, // priority
168 - localhost->rrd_update_every, // update_every
171 + rrdb.localhost->update_every, // update_every
172 RRDSET_TYPE_LINE // chart_type
173 );
174 rrdset_flag_set(host->dimensions_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -187,11 +190,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
190 // ML running
191 {
192 if (!host->ml_running_rs) {
190 - char id_buf[1024];
191 - char name_buf[1024];
193 + char id_buf[BUF_LEN + 1];
194 + char name_buf[BUF_LEN + 1];
195
193 - snprintfz(id_buf, 1024, "ml_running_on_%s", localhost->machine_guid);
194 - snprintfz(name_buf, 1024, "ml_running_on_%s", rrdhost_hostname(localhost));
196 + snprintfz(id_buf, BUF_LEN, "ml_running_on_%s", rrdb.localhost->machine_guid);
197 + snprintfz(name_buf, BUF_LEN, "ml_running_on_%s", rrdhost_hostname(rrdb.localhost));
198
199 host->ml_running_rs = rrdset_create(
200 host->rh,
@@ -205,7 +208,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
208 NETDATA_ML_PLUGIN, // plugin
209 NETDATA_ML_MODULE_DETECTION, // module
210 NETDATA_ML_CHART_RUNNING, // priority
208 - localhost->rrd_update_every, // update_every
211 + rrdb.localhost->update_every, // update_every
212 RRDSET_TYPE_LINE // chart_type
213 );
214 rrdset_flag_set(host->ml_running_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -226,11 +229,11 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
229 */
230 {
231 if (!host->anomaly_rate_rs) {
229 - char id_buf[1024];
230 - char name_buf[1024];
232 + char id_buf[BUF_LEN + 1];
233 + char name_buf[BUF_LEN + 1];
234
232 - snprintfz(id_buf, 1024, "anomaly_rate_on_%s", localhost->machine_guid);
233 - snprintfz(name_buf, 1024, "anomaly_rate_on_%s", rrdhost_hostname(localhost));
235 + snprintfz(id_buf, BUF_LEN, "anomaly_rate_on_%s", rrdb.localhost->machine_guid);
236 + snprintfz(name_buf, BUF_LEN, "anomaly_rate_on_%s", rrdhost_hostname(rrdb.localhost));
237
238 host->anomaly_rate_rs = rrdset_create(
239 host->rh,
@@ -244,7 +247,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
247 NETDATA_ML_PLUGIN, // plugin
248 NETDATA_ML_MODULE_DETECTION, // module
249 ML_CHART_PRIO_ANOMALY_RATE, // priority
247 - localhost->rrd_update_every, // update_every
250 + rrdb.localhost->update_every, // update_every
251 RRDSET_TYPE_LINE // chart_type
252 );
253 rrdset_flag_set(host->anomaly_rate_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -263,11 +266,11 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
266 */
267 {
268 if (!host->detector_events_rs) {
266 - char id_buf[1024];
267 - char name_buf[1024];
269 + char id_buf[BUF_LEN + 1];
270 + char name_buf[BUF_LEN + 1];
271
269 - snprintfz(id_buf, 1024, "anomaly_detection_on_%s", localhost->machine_guid);
270 - snprintfz(name_buf, 1024, "anomaly_detection_on_%s", rrdhost_hostname(localhost));
272 + snprintfz(id_buf, BUF_LEN, "anomaly_detection_on_%s", rrdb.localhost->machine_guid);
273 + snprintfz(name_buf, BUF_LEN, "anomaly_detection_on_%s", rrdhost_hostname(rrdb.localhost));
274
275 host->detector_events_rs = rrdset_create(
276 host->rh,
@@ -281,7 +284,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
284 NETDATA_ML_PLUGIN, // plugin
285 NETDATA_ML_MODULE_DETECTION, // module
286 ML_CHART_PRIO_DETECTOR_EVENTS, // priority
284 - localhost->rrd_update_every, // update_every
287 + rrdb.localhost->update_every, // update_every
288 RRDSET_TYPE_LINE // chart_type
289 );
290 rrdset_flag_set(host->detector_events_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -298,7 +301,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
301 if (host->ml_running) {
302 ONEWAYALLOC *OWA = onewayalloc_create(0);
303 time_t Now = now_realtime_sec();
301 - time_t Before = Now - host->rh->rrd_update_every;
304 + time_t Before = Now - host->rh->update_every;
305 time_t After = Before - Cfg.anomaly_detection_query_duration;
306 RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
307
@@ -353,14 +356,14 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
356 */
357 {
358 if (!training_thread->queue_stats_rs) {
356 - char id_buf[1024];
357 - char name_buf[1024];
359 + char id_buf[BUF_LEN + 1];
360 + char name_buf[BUF_LEN + 1];
361
359 - snprintfz(id_buf, 1024, "training_queue_%zu_stats", training_thread->id);
360 - snprintfz(name_buf, 1024, "training_queue_%zu_stats", training_thread->id);
362 + snprintfz(id_buf, BUF_LEN, "training_queue_%zu_stats", training_thread->id);
363 + snprintfz(name_buf, BUF_LEN, "training_queue_%zu_stats", training_thread->id);
364
365 training_thread->queue_stats_rs = rrdset_create(
363 - localhost,
366 + rrdb.localhost,
367 "netdata", // type
368 id_buf, // id
369 name_buf, // name
@@ -371,7 +374,7 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
374 NETDATA_ML_PLUGIN, // plugin
375 NETDATA_ML_MODULE_TRAINING, // module
376 NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
374 - localhost->rrd_update_every, // update_every
377 + rrdb.localhost->update_every, // update_every
378 RRDSET_TYPE_LINE// chart_type
379 );
380 rrdset_flag_set(training_thread->queue_stats_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -395,14 +398,14 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
398 */
399 {
400 if (!training_thread->training_time_stats_rs) {
398 - char id_buf[1024];
399 - char name_buf[1024];
401 + char id_buf[BUF_LEN + 1];
402 + char name_buf[BUF_LEN + 1];
403
401 - snprintfz(id_buf, 1024, "training_queue_%zu_time_stats", training_thread->id);
402 - snprintfz(name_buf, 1024, "training_queue_%zu_time_stats", training_thread->id);
404 + snprintfz(id_buf, BUF_LEN, "training_queue_%zu_time_stats", training_thread->id);
405 + snprintfz(name_buf, BUF_LEN, "training_queue_%zu_time_stats", training_thread->id);
406
407 training_thread->training_time_stats_rs = rrdset_create(
405 - localhost,
408 + rrdb.localhost,
409 "netdata", // type
410 id_buf, // id
411 name_buf, // name
@@ -413,7 +416,7 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
416 NETDATA_ML_PLUGIN, // plugin
417 NETDATA_ML_MODULE_TRAINING, // module
418 NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
416 - localhost->rrd_update_every, // update_every
419 + rrdb.localhost->update_every, // update_every
420 RRDSET_TYPE_LINE// chart_type
421 );
422 rrdset_flag_set(training_thread->training_time_stats_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -441,14 +444,14 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
444 */
445 {
446 if (!training_thread->training_results_rs) {
444 - char id_buf[1024];
445 - char name_buf[1024];
447 + char id_buf[BUF_LEN + 1];
448 + char name_buf[BUF_LEN + 1];
449
447 - snprintfz(id_buf, 1024, "training_queue_%zu_results", training_thread->id);
448 - snprintfz(name_buf, 1024, "training_queue_%zu_results", training_thread->id);
450 + snprintfz(id_buf, BUF_LEN, "training_queue_%zu_results", training_thread->id);
451 + snprintfz(name_buf, BUF_LEN, "training_queue_%zu_results", training_thread->id);
452
453 training_thread->training_results_rs = rrdset_create(
451 - localhost,
454 + rrdb.localhost,
455 "netdata", // type
456 id_buf, // id
457 name_buf, // name
@@ -459,7 +462,7 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
462 NETDATA_ML_PLUGIN, // plugin
463 NETDATA_ML_MODULE_TRAINING, // module
464 NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
462 - localhost->rrd_update_every, // update_every
465 + rrdb.localhost->update_every, // update_every
466 RRDSET_TYPE_LINE// chart_type
467 );
468 rrdset_flag_set(training_thread->training_results_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -508,7 +511,7 @@ void ml_update_global_statistics_charts(uint64_t models_consulted) {
511 , NETDATA_ML_PLUGIN // plugin
512 , NETDATA_ML_MODULE_DETECTION // module
513 , NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS // priority
511 - , localhost->rrd_update_every // update_every
514 + , rrdb.localhost->update_every // update_every
515 , RRDSET_TYPE_AREA // chart_type
516 );
517
ml/ml-dummy.c
+2
@@ -110,11 +110,13 @@ void ml_update_global_statistics_charts(uint64_t models_consulted) {
110 }
111
112 bool ml_host_get_host_status(RRDHOST *rh, struct ml_metrics_statistics *mlm) {
113 + UNUSED(rh);
114 memset(mlm, 0, sizeof(*mlm));
115 return false;
116 }
117
118 bool ml_host_running(RRDHOST *rh) {
119 + UNUSED(rh);
120 return false;
121 }
122
ml/ml.cc
+1 -1
@@ -356,7 +356,7 @@ ml_dimension_calculated_numbers(ml_training_thread_t *training_thread, ml_dimens
356 */
357 struct storage_engine_query_handle handle;
358
359 - storage_engine_query_init(dim->rd->tiers[0].backend, dim->rd->tiers[0].db_metric_handle, &handle,
359 + storage_engine_query_init(dim->rd->rrdset->storage_engine_id, dim->rd->tiers[0].db_metric_handle, &handle,
360 training_response.query_after_t, training_response.query_before_t,
361 STORAGE_PRIORITY_BEST_EFFORT);
362
registry/registry.c
+7 -7
@@ -171,10 +171,10 @@ int registry_request_hello_json(RRDHOST *host, struct web_client *w) {
171
172 buffer_json_member_add_object(w->response.data, "agent");
173 {
174 - buffer_json_member_add_string(w->response.data, "machine_guid", localhost->machine_guid);
174 + buffer_json_member_add_string(w->response.data, "machine_guid", rrdb.localhost->machine_guid);
175
176 - if(localhost->node_id)
177 - buffer_json_member_add_uuid(w->response.data, "node_id", localhost->node_id);
176 + if(rrdb.localhost->node_id)
177 + buffer_json_member_add_uuid(w->response.data, "node_id", rrdb.localhost->node_id);
178
179 char *claim_id = get_agent_claimid();
180 if (claim_id) {
@@ -195,7 +195,7 @@ int registry_request_hello_json(RRDHOST *host, struct web_client *w) {
195
196 buffer_json_member_add_array(w->response.data, "nodes");
197 RRDHOST *h;
198 - dfe_start_read(rrdhost_root_index, h) {
198 + dfe_start_read(rrdb.rrdhost_root_index, h) {
199 buffer_json_add_array_item_object(w->response.data);
200 buffer_json_member_add_string(w->response.data, "machine_guid", h->machine_guid);
201
@@ -456,7 +456,7 @@ void registry_statistics(void) {
456 , "registry"
457 , "stats"
458 , 131000
459 - , localhost->rrd_update_every
459 + , rrdb.localhost->update_every
460 , RRDSET_TYPE_LINE
461 );
462
@@ -480,7 +480,7 @@ void registry_statistics(void) {
480 , "registry"
481 , "stats"
482 , 131100
483 - , localhost->rrd_update_every
483 + , rrdb.localhost->update_every
484 , RRDSET_TYPE_LINE
485 );
486
@@ -510,7 +510,7 @@ void registry_statistics(void) {
510 , "registry"
511 , "stats"
512 , 131300
513 - , localhost->rrd_update_every
513 + , rrdb.localhost->update_every
514 , RRDSET_TYPE_STACKED
515 );
516
streaming/compression.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include "rrdpush.h"
4
5 #ifdef ENABLE_RRDPUSH_COMPRESSION
streaming/receiver.c
+35 -18
@@ -391,7 +391,7 @@ static bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
391 rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);
392
393 host->rrdpush_receiver_connection_counter++;
394 - __atomic_add_fetch(&localhost->connected_children_count, 1, __ATOMIC_RELAXED);
394 + __atomic_add_fetch(&rrdb.localhost->connected_children_count, 1, __ATOMIC_RELAXED);
395
396 host->receiver = rpt;
397 rpt->host = host;
@@ -445,7 +445,7 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
445
446 // Make sure that we detach this thread and don't kill a freshly arriving receiver
447 if(host->receiver == rpt) {
448 - __atomic_sub_fetch(&localhost->connected_children_count, 1, __ATOMIC_RELAXED);
448 + __atomic_sub_fetch(&rrdb.localhost->connected_children_count, 1, __ATOMIC_RELAXED);
449 rrdhost_flag_set(rpt->host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED);
450
451 host->trigger_chart_obsoletion_check = 0;
@@ -549,8 +549,8 @@ void rrdpush_receive_log_status(struct receiver_state *rpt, const char *msg, con
549
550 static void rrdpush_receive(struct receiver_state *rpt)
551 {
552 - rpt->config.mode = default_rrd_memory_mode;
553 - rpt->config.history = default_rrd_history_entries;
552 + rpt->config.storage_engine_id = default_storage_engine_id;
553 + rpt->config.history = rrdb.default_rrd_history_entries;
554
555 rpt->config.health_enabled = (int)default_health_enabled;
556 rpt->config.alarms_delay = 60;
@@ -572,17 +572,34 @@ static void rrdpush_receive(struct receiver_state *rpt)
572 rpt->config.history = (int)appconfig_get_number(&stream_config, rpt->machine_guid, "history", rpt->config.history);
573 if(rpt->config.history < 5) rpt->config.history = 5;
574
575 - rpt->config.mode = rrd_memory_mode_id(appconfig_get(&stream_config, rpt->key, "default memory mode", rrd_memory_mode_name(rpt->config.mode)));
576 - rpt->config.mode = rrd_memory_mode_id(appconfig_get(&stream_config, rpt->machine_guid, "memory mode", rrd_memory_mode_name(rpt->config.mode)));
575 + // figure out storage engine ids for key and/or machine guid
576 + {
577 + const char *se_name = appconfig_get(&stream_config, rpt->key,
578 + "default memory mode",
579 + storage_engine_name(rpt->config.storage_engine_id));
580 + if (!storage_engine_id(se_name, &rpt->config.storage_engine_id)) {
581 + netdata_log_error("STREAM '%s' [receive from %s:%s]: invalid default memory mode for key %s (given: '%s', will use: '%s').",
582 + rpt->hostname, rpt->client_ip, rpt->client_port,
583 + rpt->key, se_name, storage_engine_name(rpt->config.storage_engine_id));
584 + }
585
578 - if (unlikely(rpt->config.mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled)) {
579 - netdata_log_error("STREAM '%s' [receive from %s:%s]: "
580 - "dbengine is not enabled, falling back to default."
581 - , rpt->hostname
582 - , rpt->client_ip, rpt->client_port
583 - );
586 + se_name = appconfig_get(&stream_config, rpt->machine_guid,
587 + "memory mode",
588 + storage_engine_name(rpt->config.storage_engine_id));
589 +
590 + if (!storage_engine_id(se_name, &rpt->config.storage_engine_id)) {
591 + netdata_log_error("STREAM '%s' [receive from %s:%s]: invalid memory mode for machine guid %s (given: '%s', will use: '%s').",
592 + rpt->hostname, rpt->client_ip, rpt->client_port,
593 + rpt->machine_guid, se_name, storage_engine_name(rpt->config.storage_engine_id));
594 + }
595 + }
596 +
597 + if (unlikely(rpt->config.storage_engine_id == STORAGE_ENGINE_DBENGINE && !rrdb.dbengine_enabled))
598 + {
599 + netdata_log_error("STREAM '%s' [receive from %s:%s]: dbengine is not enabled, falling back to default.",
600 + rpt->hostname, rpt->client_ip, rpt->client_port);
601
585 - rpt->config.mode = default_rrd_memory_mode;
602 + rpt->config.storage_engine_id = default_storage_engine_id;
603 }
604
605 rpt->config.health_enabled = appconfig_get_boolean_ondemand(&stream_config, rpt->key, "health enabled by default", rpt->config.health_enabled);
@@ -626,7 +643,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
643 // find the host for this receiver
644 {
645 // this will also update the host with our system_info
629 - RRDHOST *host = rrdhost_find_or_create(
646 + RRDHOST *host = rrdhost_get_or_create(
647 rpt->hostname
648 , rpt->registry_hostname
649 , rpt->machine_guid
@@ -639,7 +656,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
656 , rpt->program_version
657 , rpt->config.update_every
658 , rpt->config.history
642 - , rpt->config.mode
659 + , rpt->config.storage_engine_id
660 , (unsigned int)(rpt->config.health_enabled != CONFIG_BOOLEAN_NO)
661 , (unsigned int)(rpt->config.rrdpush_enabled && rpt->config.rrdpush_destination && *rpt->config.rrdpush_destination && rpt->config.rrdpush_api_key && *rpt->config.rrdpush_api_key)
662 , rpt->config.rrdpush_destination
@@ -683,9 +700,9 @@ static void rrdpush_receive(struct receiver_state *rpt)
700 , rpt->client_port
701 , rrdhost_hostname(rpt->host)
702 , rpt->host->machine_guid
686 - , rpt->host->rrd_update_every
703 + , rpt->host->update_every
704 , rpt->host->rrd_history_entries
688 - , rrd_memory_mode_name(rpt->host->rrd_memory_mode)
705 + , storage_engine_name(rpt->host->storage_engine_id)
706 , (rpt->config.health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((rpt->config.health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto")
707 #ifdef ENABLE_HTTPS
708 , (rpt->ssl.conn != NULL) ? " SSL," : ""
@@ -698,7 +715,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
715
716
717 struct plugind cd = {
701 - .update_every = default_rrd_update_every,
718 + .update_every = rrdb.default_update_every,
719 .unsafe = {
720 .spinlock = NETDATA_SPINLOCK_INITIALIZER,
721 .running = true,
streaming/replication.c
+9 -9
@@ -96,7 +96,7 @@ struct replication_query {
96 size_t points_read;
97 size_t points_generated;
98
99 - STORAGE_ENGINE_BACKEND backend;
99 + STORAGE_ENGINE_ID id;
100 struct replication_request *rq;
101
102 size_t dimensions;
@@ -162,7 +162,7 @@ static struct replication_query *replication_query_prepare(
162 }
163 }
164
165 - q->backend = st->rrdhost->db[0].eng->backend;
165 + q->id = st->rrdhost->db[0].id;
166
167 // prepare our array of dimensions
168 size_t count = 0;
@@ -184,7 +184,7 @@ static struct replication_query *replication_query_prepare(
184 d->rda = dictionary_acquired_item_dup(rd_dfe.dict, rd_dfe.item);
185 d->rd = rd;
186
187 - storage_engine_query_init(q->backend, rd->tiers[0].db_metric_handle, &d->handle, q->query.after, q->query.before,
187 + storage_engine_query_init(q->id, rd->tiers[0].db_metric_handle, &d->handle, q->query.after, q->query.before,
188 q->query.locked_data_collection ? STORAGE_PRIORITY_HIGH : STORAGE_PRIORITY_LOW);
189 d->enabled = true;
190 d->skip = false;
@@ -1406,7 +1406,7 @@ static bool replication_execute_request(struct replication_request *rq, bool wor
1406 if(likely(workers))
1407 worker_is_busy(WORKER_JOB_FIND_CHART);
1408
1409 - rq->st = rrdset_find(rq->sender->host, string2str(rq->chart_id));
1409 + rq->st = rrdset_find_by_id(rq->sender->host, string2str(rq->chart_id));
1410 }
1411
1412 if(!rq->st) {
@@ -1601,7 +1601,7 @@ static void verify_all_hosts_charts_are_streaming_now(void) {
1601
1602 size_t errors = 0;
1603 RRDHOST *host;
1604 - dfe_start_read(rrdhost_root_index, host)
1604 + dfe_start_read(rrdb.rrdhost_root_index, host)
1605 errors += verify_host_charts_are_streaming_now(host);
1606 dfe_done(host);
1607
@@ -1695,8 +1695,8 @@ static int replication_pipeline_execute_next(void) {
1695 if(unlikely(!rtp.rqs)) {
1696 rtp.max_requests_ahead = (int)get_netdata_cpus() / 2;
1697
1698 - if(rtp.max_requests_ahead > libuv_worker_threads * 2)
1699 - rtp.max_requests_ahead = libuv_worker_threads * 2;
1698 + if(rtp.max_requests_ahead > rrdb.libuv_worker_threads * 2)
1699 + rtp.max_requests_ahead = rrdb.libuv_worker_threads * 2;
1700
1701 if(rtp.max_requests_ahead < 2)
1702 rtp.max_requests_ahead = 2;
@@ -1722,7 +1722,7 @@ static int replication_pipeline_execute_next(void) {
1722 if(rq->found) {
1723 if (!rq->st) {
1724 worker_is_busy(WORKER_JOB_FIND_CHART);
1725 - rq->st = rrdset_find(rq->sender->host, string2str(rq->chart_id));
1725 + rq->st = rrdset_find_by_id(rq->sender->host, string2str(rq->chart_id));
1726 }
1727
1728 if (rq->st && !rq->q) {
@@ -1896,7 +1896,7 @@ void *replication_thread_main(void *ptr __maybe_unused) {
1896
1897 // statistics
1898 usec_t now_mono_ut = now_monotonic_usec();
1899 - if(unlikely(now_mono_ut - last_now_mono_ut > default_rrd_update_every * USEC_PER_SEC)) {
1899 + if(unlikely(now_mono_ut - last_now_mono_ut > rrdb.default_update_every * USEC_PER_SEC)) {
1900 last_now_mono_ut = now_mono_ut;
1901
1902 worker_is_busy(WORKER_JOB_STATISTICS);
streaming/rrdpush.c
+4 -4
@@ -96,7 +96,7 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) {
96 STREAM_CAP_BINARY |
97 STREAM_CAP_INTERPOLATED |
98 STREAM_HAS_COMPRESSION |
99 - (ieee754_doubles ? STREAM_CAP_IEEE754 : 0) |
99 + (rrdb.ieee754_doubles ? STREAM_CAP_IEEE754 : 0) |
100 (ml_capability ? STREAM_CAP_DATA_WITH_ML : 0) |
101 0;
102 }
@@ -140,7 +140,7 @@ int rrdpush_init() {
140 default_rrdpush_seconds_to_replicate = config_get_number(CONFIG_SECTION_DB, "seconds to replicate", default_rrdpush_seconds_to_replicate);
141 default_rrdpush_replication_step = config_get_number(CONFIG_SECTION_DB, "seconds per replication step", default_rrdpush_replication_step);
142
143 - rrdhost_free_orphan_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup orphan hosts after secs", rrdhost_free_orphan_time_s);
143 + rrdb.rrdhost_free_orphan_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup orphan hosts after secs", rrdb.rrdhost_free_orphan_time_s);
144
145 #ifdef ENABLE_RRDPUSH_COMPRESSION
146 default_rrdpush_compression_enabled = (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM,
@@ -801,7 +801,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
801 rpt->ssl = NETDATA_SSL_UNSET_CONNECTION;
802 #endif
803
804 - rpt->config.update_every = default_rrd_update_every;
804 + rpt->config.update_every = rrdb.default_update_every;
805
806 // parse the parameters and fill rpt and rpt->system_info
807
@@ -1057,7 +1057,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
1057 }
1058 }
1059
1060 - if (strcmp(rpt->machine_guid, localhost->machine_guid) == 0) {
1060 + if (strcmp(rpt->machine_guid, rrdb.localhost->machine_guid) == 0) {
1061
1062 rrdpush_receiver_takeover_web_connection(w, rpt);
1063
streaming/rrdpush.h
+2 -2
@@ -395,7 +395,7 @@ struct receiver_state {
395 } exit;
396
397 struct {
398 - RRD_MEMORY_MODE mode;
398 + STORAGE_ENGINE_ID storage_engine_id;
399 int history;
400 int update_every;
401 int health_enabled; // CONFIG_BOOLEAN_YES, CONFIG_BOOLEAN_NO, CONFIG_BOOLEAN_AUTO
@@ -696,7 +696,7 @@ typedef struct rrdhost_status {
696 struct {
697 RRDHOST_DB_STATUS status;
698 RRDHOST_DB_LIVENESS liveness;
699 - RRD_MEMORY_MODE mode;
699 + STORAGE_ENGINE_ID storage_engine_id;
700 time_t first_time_s;
701 time_t last_time_s;
702 size_t metrics;
streaming/sender.c
+1 -1
@@ -657,7 +657,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
657 , rrdhost_hostname(host)
658 , rrdhost_registry_hostname(host)
659 , host->machine_guid
660 - , default_rrd_update_every
660 + , rrdb.default_update_every
661 , rrdhost_os(host)
662 , rrdhost_timezone(host)
663 , rrdhost_abbrev_timezone(host)
tests/ebpf/sync_tester.c
+2
@@ -1,3 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 #include <errno.h>
4 #include <string.h>
5 #include <stdio.h>
web/api/badges/web_buffer_svg.c
+3 -3
@@ -961,8 +961,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
961
962 int scale = (scale_str && *scale_str)?str2i(scale_str):100;
963
964 - st = rrdset_find(host, chart);
965 - if(!st) st = rrdset_find_byname(host, chart);
964 + st = rrdset_find_by_id(host, chart);
965 + if(!st) st = rrdset_find_by_name(host, chart);
966 if(!st) {
967 buffer_no_cacheable(w->response.data);
968 buffer_svg(w->response.data, "chart not found", NAN, "", NULL, NULL, -1, scale, 0, -1, -1, NULL, NULL);
@@ -1111,7 +1111,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
1111 ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
1112
1113 // if the collected value is too old, don't calculate its value
1114 - if (rrdset_last_entry_s(st) >= (now_realtime_sec() - (st->update_every * gap_when_lost_iterations_above)))
1114 + if (rrdset_last_entry_s(st) >= (now_realtime_sec() - (st->update_every * rrdb.gap_when_lost_iterations_above)))
1115 ret = rrdset2value_api_v1(st, w->response.data, &n,
1116 (dimensions) ? buffer_tostring(dimensions) : NULL,
1117 points, after, before, group, group_options, 0, options,
web/api/formatters/charts2json.c
+2 -2
@@ -62,9 +62,9 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
62 , get_release_channel()
63 , rrdhost_os(host)
64 , rrdhost_timezone(host)
65 - , host->rrd_update_every
65 + , host->update_every
66 , host->rrd_history_entries
67 - , rrd_memory_mode_name(host->rrd_memory_mode)
67 + , storage_engine_name(host->storage_engine_id)
68 , custom_dashboard_info_js_filename
69 );
70
web/api/formatters/json_wrapper.c
+5 -5
@@ -16,7 +16,7 @@ static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) {
16 buffer_json_array_close(wb);
17
18 buffer_json_member_add_array(wb, "tiers");
19 - for (size_t tier = 0; tier < storage_tiers; tier++) {
19 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
20 buffer_json_add_array_item_object(wb);
21 buffer_json_member_add_uint64(wb, "tr", tier);
22 buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s);
@@ -920,7 +920,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
920 buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
921
922 buffer_json_member_add_array(wb, "db_points_per_tier");
923 - for(size_t tier = 0; tier < storage_tiers ; tier++)
923 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++)
924 buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points);
925 buffer_json_array_close(wb);
926
@@ -1269,7 +1269,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
1269 void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) {
1270 buffer_json_member_add_object(wb, "versions");
1271 buffer_json_member_add_uint64(wb, "routing_hard_hash", 1);
1272 - buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index));
1272 + buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdb.rrdhost_root_index));
1273 buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash);
1274 buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash);
1275 buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash);
@@ -1498,7 +1498,7 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
1498
1499 buffer_json_member_add_object(wb, "db");
1500 {
1501 - buffer_json_member_add_uint64(wb, "tiers", storage_tiers);
1501 + buffer_json_member_add_uint64(wb, "tiers", rrdb.storage_tiers);
1502 buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
1503 buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
1504 buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
@@ -1513,7 +1513,7 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
1513 buffer_json_object_close(wb); // dimensions
1514
1515 buffer_json_member_add_array(wb, "per_tier");
1516 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
1516 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1517 buffer_json_add_array_item_object(wb);
1518 buffer_json_member_add_uint64(wb, "tier", tier);
1519 buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries);
web/api/formatters/rrd2json.c
+1 -2
@@ -1,7 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "web/api/web_api_v1.h"
4 -#include "database/storage_engine.h"
4
5 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
6 rrdset2json(st, wb, NULL, NULL, 0);
@@ -100,7 +99,7 @@ int rrdset2value_api_v1(
99 *db_points_read += r->stats.db_points_read;
100
101 if(db_points_per_tier) {
103 - for(size_t t = 0; t < storage_tiers ;t++)
102 + for(size_t t = 0; t < rrdb.storage_tiers ;t++)
103 db_points_per_tier[t] += r->internal.qt->db.tiers[t].points;
104 }
105
web/api/formatters/value/value.c
+1 -1
@@ -131,7 +131,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
131 storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points);
132 }
133
134 - for(size_t t = 0; t < storage_tiers ;t++)
134 + for(size_t t = 0; t < rrdb.storage_tiers ;t++)
135 qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;
136
137 long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
web/api/health/health_cmdapi.c
+1 -3
@@ -1,6 +1,4 @@
1 -//
2 -// Created by Christopher on 11/12/18.
3 -//
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "health_cmdapi.h"
4
web/api/queries/query.c
+37 -32
@@ -946,7 +946,7 @@ static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) {
946 }
947
948 static size_t query_metric_first_working_tier(QUERY_METRIC *qm) {
949 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
949 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
950
951 // find the db time-range for this tier for all metrics
952 STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
@@ -991,7 +991,7 @@ static long query_plan_points_coverage_weight(time_t db_first_time_s, time_t db_
991 }
992
993 static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
994 - if(unlikely(storage_tiers < 2))
994 + if(unlikely(rrdb.storage_tiers < 2))
995 return 0;
996
997 if(unlikely(after_wanted == before_wanted || points_wanted <= 0))
@@ -1004,7 +1004,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1004 time_t min_first_time_s = 0;
1005 time_t max_last_time_s = 0;
1006
1007 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
1007 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1008 time_t first_time_s = qm->tiers[tier].db_first_time_s;
1009 time_t last_time_s = qm->tiers[tier].db_last_time_s;
1010
@@ -1015,7 +1015,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1015 max_last_time_s = last_time_s;
1016 }
1017
1018 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
1018 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1019
1020 // find the db time-range for this tier for all metrics
1021 STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
@@ -1042,7 +1042,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1042 }
1043
1044 size_t best_tier = 0;
1045 - for(size_t tier = 1; tier < storage_tiers ; tier++) {
1045 + for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
1046 if(qm->tiers[tier].weight >= qm->tiers[best_tier].weight)
1047 best_tier = tier;
1048 }
@@ -1051,7 +1051,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1051 }
1052
1053 static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
1054 - if(unlikely(storage_tiers < 2))
1054 + if(unlikely(rrdb.storage_tiers < 2))
1055 return 0;
1056
1057 if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) {
@@ -1059,9 +1059,9 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
1059 return 0;
1060 }
1061
1062 - long weight[storage_tiers];
1062 + long weight[rrdb.storage_tiers];
1063
1064 - for(size_t tier = 0; tier < storage_tiers ; tier++) {
1064 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1065
1066 time_t common_first_time_s = 0;
1067 time_t common_last_time_s = 0;
@@ -1098,7 +1098,7 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
1098 }
1099
1100 size_t best_tier = 0;
1101 - for(size_t tier = 1; tier < storage_tiers ; tier++) {
1101 + for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
1102 if(weight[tier] >= weight[best_tier])
1103 best_tier = tier;
1104 }
@@ -1111,13 +1111,13 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
1111
1112 static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier) {
1113 size_t best_tier;
1114 - if((options & RRDR_OPTION_SELECTED_TIER) && tier < storage_tiers)
1114 + if((options & RRDR_OPTION_SELECTED_TIER) && tier < rrdb.storage_tiers)
1115 best_tier = tier;
1116 else
1117 best_tier = rrddim_find_best_tier_for_timeframe(qt, after_wanted, before_wanted, points_wanted);
1118
1119 // find the db minimum update every for this tier for all metrics
1120 - time_t common_update_every_s = default_rrd_update_every;
1120 + time_t common_update_every_s = rrdb.default_update_every;
1121 for(size_t i = 0, used = qt->query.used; i < used ; i++) {
1122 QUERY_METRIC *qm = query_metric(qt, i);
1123
@@ -1260,8 +1260,8 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
1260 ops->r->internal.qt->db.tiers[tier].queries++;
1261
1262 struct query_metric_tier *tier_ptr = &qm->tiers[tier];
1263 - STORAGE_ENGINE *eng = query_metric_storage_engine(ops->r->internal.qt, qm, tier);
1264 - storage_engine_query_init(eng->backend, tier_ptr->db_metric_handle, &ops->plans[p].handle,
1263 + STORAGE_ENGINE_ID storage_engine_id = query_metric_storage_engine(ops->r->internal.qt, qm, tier);
1264 + storage_engine_query_init(storage_engine_id, tier_ptr->db_metric_handle, &ops->plans[p].handle,
1265 after, before, ops->r->internal.qt->request.priority);
1266
1267 ops->plans[p].initialized = true;
@@ -1354,7 +1354,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1354 bool switch_tiers = true;
1355
1356 if((ops->r->internal.qt->window.options & RRDR_OPTION_SELECTED_TIER)
1357 - && ops->r->internal.qt->window.tier < storage_tiers
1357 + && ops->r->internal.qt->window.tier < rrdb.storage_tiers
1358 && query_metric_is_valid_tier(qm, ops->r->internal.qt->window.tier)) {
1359 selected_tier = ops->r->internal.qt->window.tier;
1360 switch_tiers = false;
@@ -1383,7 +1383,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1383 // check if our selected tier can start the query
1384 if (selected_tier_first_time_s > after_wanted) {
1385 // we need some help from other tiers
1386 - for (size_t tr = (int)selected_tier + 1; tr < storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) {
1386 + for (size_t tr = (int)selected_tier + 1; tr < rrdb.storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) {
1387 if(!query_metric_is_valid_tier(qm, tr))
1388 continue;
1389
@@ -1940,7 +1940,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1940
1941 r->stats.result_points_generated += points_added;
1942 r->stats.db_points_read += ops->db_total_points_read;
1943 - for(size_t tr = 0; tr < storage_tiers ; tr++)
1943 + for(size_t tr = 0; tr < rrdb.storage_tiers ; tr++)
1944 qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
1945 }
1946
@@ -1950,18 +1950,23 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1950 void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
1951
1952 void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {
1953 - if(unlikely(tier >= storage_tiers)) return;
1954 - if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
1953 + if (unlikely(tier >= rrdb.storage_tiers))
1954 + return;
1955 + if (rrdb.storage_tiers_backfill[tier] == RRD_BACKFILL_NONE)
1956 + return;
1957
1958 struct rrddim_tier *t = &rd->tiers[tier];
1957 - if(unlikely(!t)) return;
1959 + if (unlikely(!t))
1960 + return;
1961
1959 - time_t latest_time_s = storage_engine_latest_time_s(t->backend, t->db_metric_handle);
1960 - time_t granularity = (time_t)t->tier_grouping * (time_t)rd->rrdset->update_every;
1962 + STORAGE_ENGINE_ID storage_engine_id = rd->rrdset->storage_engine_id;
1963 + time_t latest_time_s = storage_engine_latest_time_s(storage_engine_id, t->db_metric_handle);
1964 + time_t granularity = (time_t) rd->rrdset->rrdhost->db[tier].tier_grouping * (time_t)rd->rrdset->update_every;
1965 time_t time_diff = now_s - latest_time_s;
1966
1967 // if the user wants only NEW backfilling, and we don't have any data
1964 - if(storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_s <= 0) return;
1968 + if (rrdb.storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_s <= 0)
1969 + return;
1970
1971 // there is really nothing we can do
1972 if(now_s <= latest_time_s || time_diff < granularity) return;
@@ -1970,15 +1975,15 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
1975
1976 // for each lower tier
1977 for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){
1973 - time_t smaller_tier_first_time = storage_engine_oldest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle);
1974 - time_t smaller_tier_last_time = storage_engine_latest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle);
1978 + time_t smaller_tier_first_time = storage_engine_oldest_time_s(storage_engine_id, rd->tiers[read_tier].db_metric_handle);
1979 + time_t smaller_tier_last_time = storage_engine_latest_time_s(storage_engine_id, rd->tiers[read_tier].db_metric_handle);
1980 if(smaller_tier_last_time <= latest_time_s) continue; // it is as bad as we are
1981
1982 long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s;
1983 long before_wanted = smaller_tier_last_time;
1984
1985 struct rrddim_tier *tmp = &rd->tiers[read_tier];
1981 - storage_engine_query_init(tmp->backend, tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
1986 + storage_engine_query_init(storage_engine_id, tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
1987
1988 size_t points_read = 0;
1989
@@ -2139,16 +2144,16 @@ bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now
2144 time_t absolute_minimum_time = now - (10 * 365 * 86400);
2145 time_t absolute_maximum_time = now + (1 * 365 * 86400);
2146
2142 - if (after_requested < absolute_minimum_time && !unittest_running)
2147 + if (after_requested < absolute_minimum_time && !rrdb.unittest_running)
2148 after_requested = absolute_minimum_time;
2149
2145 - if (after_requested > absolute_maximum_time && !unittest_running)
2150 + if (after_requested > absolute_maximum_time && !rrdb.unittest_running)
2151 after_requested = absolute_maximum_time;
2152
2148 - if (before_requested < absolute_minimum_time && !unittest_running)
2153 + if (before_requested < absolute_minimum_time && !rrdb.unittest_running)
2154 before_requested = absolute_minimum_time;
2155
2151 - if (before_requested > absolute_maximum_time && !unittest_running)
2156 + if (before_requested > absolute_maximum_time && !rrdb.unittest_running)
2157 before_requested = absolute_maximum_time;
2158
2159 *before = before_requested;
@@ -2286,7 +2291,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
2291 rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
2292 query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
2293
2289 - if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
2294 + if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && rrdb.storage_tiers > 1) {
2295 update_every = rrdset_find_natural_update_every_for_timeframe(
2296 qt, after_wanted, before_wanted, points_wanted, options, tier);
2297
@@ -2349,7 +2354,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
2354 query_debug_log(":max points_wanted %zu", points_wanted);
2355 }
2356
2352 - if(points_wanted > 86400 && !unittest_running) {
2357 + if(points_wanted > 86400 && !rrdb.unittest_running) {
2358 points_wanted = 86400;
2359 query_debug_log(":absolute max points_wanted %zu", points_wanted);
2360 }
@@ -3545,7 +3550,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3550 if(qt->query.used)
3551 ops = onewayalloc_callocz(owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
3552
3548 - size_t capacity = libuv_worker_threads * 10;
3553 + size_t capacity = rrdb.libuv_worker_threads * 10;
3554 size_t max_queries_to_prepare = (qt->query.used > (capacity - 1)) ? (capacity - 1) : qt->query.used;
3555 size_t queries_prepared = 0;
3556 while(queries_prepared < max_queries_to_prepare) {
web/api/queries/weights.c
+4 -4
@@ -149,7 +149,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
149
150 buffer_json_member_add_array(wb, "db_points_per_tier");
151 {
152 - for (size_t tier = 0; tier < storage_tiers; tier++)
152 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
153 buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]);
154 }
155 buffer_json_array_close(wb);
@@ -461,7 +461,7 @@ static void results_header_to_json_v2(DICTIONARY *results __maybe_unused, BUFFER
461
462 buffer_json_member_add_array(wb, "db_points_per_tier");
463 {
464 - for (size_t tier = 0; tier < storage_tiers; tier++)
464 + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
465 buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]);
466 }
467 buffer_json_array_close(wb);
@@ -1288,7 +1288,7 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
1288 stats->db_queries++;
1289 stats->result_points += r->stats.result_points_generated;
1290 stats->db_points += r->stats.db_points_read;
1291 - for(size_t tr = 0; tr < storage_tiers ; tr++)
1291 + for(size_t tr = 0; tr < rrdb.storage_tiers ; tr++)
1292 stats->db_points_per_tier[tr] += r->internal.qt->db.tiers[tr].points;
1293
1294 if(r->d != 1 || r->internal.qt->query.used != 1) {
@@ -1395,7 +1395,7 @@ static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats, si
1395 stats->db_queries += queries;
1396 stats->result_points += qv->result_points;
1397 stats->db_points += qv->points_read;
1398 - for(size_t tier = 0; tier < storage_tiers ; tier++)
1398 + for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++)
1399 stats->db_points_per_tier[tier] += qv->storage_points_per_tier[tier];
1400 }
1401
web/api/web_api.c
+1 -1
@@ -175,7 +175,7 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur
175
176 else if(!strcmp(name, "tier")) {
177 tier = str2ul(value);
178 - if(tier < storage_tiers)
178 + if(tier < rrdb.storage_tiers)
179 options |= RRDR_OPTION_SELECTED_TIER;
180 else
181 tier = 0;
web/api/web_api_v1.c
+15 -77
@@ -494,8 +494,8 @@ inline int web_client_api_request_single_chart(RRDHOST *host, struct web_client
494 goto cleanup;
495 }
496
497 - RRDSET *st = rrdset_find(host, chart);
498 - if(!st) st = rrdset_find_byname(host, chart);
497 + RRDSET *st = rrdset_find_by_id(host, chart);
498 + if(!st) st = rrdset_find_by_name(host, chart);
499 if(!st) {
500 buffer_strcat(w->response.data, "Chart is not found: ");
501 buffer_strcat_htmlescape(w->response.data, chart);
@@ -762,7 +762,7 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
762 }
763 else if(!strcmp(name, "tier")) {
764 tier = str2ul(value);
765 - if(tier < storage_tiers)
765 + if(tier < rrdb.storage_tiers)
766 options |= RRDR_OPTION_SELECTED_TIER;
767 else
768 tier = 0;
@@ -788,8 +788,8 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
788
789 if(chart && !context) {
790 // check if this is a specific chart
791 - st = rrdset_find(host, chart);
792 - if (!st) st = rrdset_find_byname(host, chart);
791 + st = rrdset_find_by_id(host, chart);
792 + if (!st) st = rrdset_find_by_name(host, chart);
793 }
794
795 long long before = (before_str && *before_str)?str2l(before_str):0;
@@ -1116,8 +1116,8 @@ static inline void web_client_api_request_v1_info_mirrored_hosts_status(BUFFER *
1116 buffer_json_add_array_item_object(wb);
1117
1118 buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
1119 - buffer_json_member_add_uint64(wb, "hops", host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
1120 - buffer_json_member_add_boolean(wb, "reachable", (host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)));
1119 + buffer_json_member_add_uint64(wb, "hops", host->system_info ? host->system_info->hops : (host == rrdb.localhost) ? 0 : 1);
1120 + buffer_json_member_add_boolean(wb, "reachable", (host == rrdb.localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)));
1121
1122 buffer_json_member_add_string(wb, "guid", host->machine_guid);
1123 buffer_json_member_add_uuid(wb, "node_id", host->node_id);
@@ -1140,12 +1140,12 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
1140
1141 buffer_json_member_add_array(wb, "mirrored_hosts_status");
1142 rrdhost_foreach_read(host) {
1143 - if ((host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1143 + if ((host == rrdb.localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1144 web_client_api_request_v1_info_mirrored_hosts_status(wb, host);
1145 }
1146 }
1147 rrdhost_foreach_read(host) {
1148 - if ((host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1148 + if ((host != rrdb.localhost && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1149 web_client_api_request_v1_info_mirrored_hosts_status(wb, host);
1150 }
1151 }
@@ -1257,10 +1257,10 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
1257 buffer_json_member_add_boolean(wb, "aclk-available", false);
1258 #endif
1259
1260 - buffer_json_member_add_string(wb, "memory-mode", rrd_memory_mode_name(host->rrd_memory_mode));
1260 + buffer_json_member_add_string(wb, "memory-mode", storage_engine_name(host->storage_engine_id));
1261 #ifdef ENABLE_DBENGINE
1262 - buffer_json_member_add_uint64(wb, "multidb-disk-quota", default_multidb_disk_quota_mb);
1263 - buffer_json_member_add_uint64(wb, "page-cache-size", default_rrdeng_page_cache_mb);
1262 + buffer_json_member_add_uint64(wb, "multidb-disk-quota", rrdb.default_multidb_disk_quota_mb);
1263 + buffer_json_member_add_uint64(wb, "page-cache-size", rrdb.default_rrdeng_page_cache_mb);
1264 #endif // ENABLE_DBENGINE
1265 buffer_json_member_add_boolean(wb, "web-enabled", web_server_mode != WEB_SERVER_MODE_NONE);
1266 buffer_json_member_add_boolean(wb, "stream-enabled", default_rrdpush_enabled);
@@ -1431,68 +1431,6 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1431 return HTTP_RESP_NOT_FOUND;
1432 }
1433 #else
1434 -static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, size_t tier) {
1435 - RRDENG_SIZE_STATS stats = rrdeng_size_statistics(multidb_ctx[tier]);
1436 -
1437 - buffer_sprintf(wb,
1438 - "\n\t\t\"default_granularity_secs\":%zu"
1439 - ",\n\t\t\"sizeof_datafile\":%zu"
1440 - ",\n\t\t\"sizeof_page_in_cache\":%zu"
1441 - ",\n\t\t\"sizeof_point_data\":%zu"
1442 - ",\n\t\t\"sizeof_page_data\":%zu"
1443 - ",\n\t\t\"pages_per_extent\":%zu"
1444 - ",\n\t\t\"datafiles\":%zu"
1445 - ",\n\t\t\"extents\":%zu"
1446 - ",\n\t\t\"extents_pages\":%zu"
1447 - ",\n\t\t\"points\":%zu"
1448 - ",\n\t\t\"metrics\":%zu"
1449 - ",\n\t\t\"metrics_pages\":%zu"
1450 - ",\n\t\t\"extents_compressed_bytes\":%zu"
1451 - ",\n\t\t\"pages_uncompressed_bytes\":%zu"
1452 - ",\n\t\t\"pages_duration_secs\":%lld"
1453 - ",\n\t\t\"single_point_pages\":%zu"
1454 - ",\n\t\t\"first_t\":%ld"
1455 - ",\n\t\t\"last_t\":%ld"
1456 - ",\n\t\t\"database_retention_secs\":%lld"
1457 - ",\n\t\t\"average_compression_savings\":%0.2f"
1458 - ",\n\t\t\"average_point_duration_secs\":%0.2f"
1459 - ",\n\t\t\"average_metric_retention_secs\":%0.2f"
1460 - ",\n\t\t\"ephemeral_metrics_per_day_percent\":%0.2f"
1461 - ",\n\t\t\"average_page_size_bytes\":%0.2f"
1462 - ",\n\t\t\"estimated_concurrently_collected_metrics\":%zu"
1463 - ",\n\t\t\"currently_collected_metrics\":%zu"
1464 - ",\n\t\t\"disk_space\":%zu"
1465 - ",\n\t\t\"max_disk_space\":%zu"
1466 - , stats.default_granularity_secs
1467 - , stats.sizeof_datafile
1468 - , stats.sizeof_page_in_cache
1469 - , stats.sizeof_point_data
1470 - , stats.sizeof_page_data
1471 - , stats.pages_per_extent
1472 - , stats.datafiles
1473 - , stats.extents
1474 - , stats.extents_pages
1475 - , stats.points
1476 - , stats.metrics
1477 - , stats.metrics_pages
1478 - , stats.extents_compressed_bytes
1479 - , stats.pages_uncompressed_bytes
1480 - , (long long)stats.pages_duration_secs
1481 - , stats.single_point_pages
1482 - , stats.first_time_s
1483 - , stats.last_time_s
1484 - , (long long)stats.database_retention_secs
1485 - , stats.average_compression_savings
1486 - , stats.average_point_duration_secs
1487 - , stats.average_metric_retention_secs
1488 - , stats.ephemeral_metrics_per_day_percent
1489 - , stats.average_page_size_bytes
1490 - , stats.estimated_concurrently_collected_metrics
1491 - , stats.currently_collected_metrics
1492 - , stats.disk_space
1493 - , stats.max_disk_space
1494 - );
1495 -}
1434 int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) {
1435 if (!netdata_ready)
1436 return HTTP_RESP_BACKEND_FETCH_FAILED;
@@ -1500,7 +1438,7 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1438 BUFFER *wb = w->response.data;
1439 buffer_flush(wb);
1440
1503 - if(!dbengine_enabled) {
1441 + if(!rrdb.dbengine_enabled) {
1442 buffer_strcat(wb, "dbengine is not enabled");
1443 return HTTP_RESP_NOT_FOUND;
1444 }
@@ -1508,9 +1446,9 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1446 wb->content_type = CT_APPLICATION_JSON;
1447 buffer_no_cacheable(wb);
1448 buffer_strcat(wb, "{");
1511 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
1449 + for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
1450 buffer_sprintf(wb, "%s\n\t\"tier%zu\": {", tier?",":"", tier);
1513 - web_client_api_v1_dbengine_stats_for_tier(wb, tier);
1451 + rrdeng_size_statistics(rrdb.multidb_ctx[tier], wb);
1452 buffer_strcat(wb, "\n\t}");
1453 }
1454 buffer_strcat(wb, "\n}");
web/api/web_api_v2.c
+5 -5
@@ -95,7 +95,7 @@ static bool verify_agent_uuids(const char *machine_guid, const char *node_id, co
95 if(!machine_guid || !node_id || !claim_id)
96 return false;
97
98 - if(strcmp(machine_guid, localhost->machine_guid) != 0)
98 + if(strcmp(machine_guid, rrdb.localhost->machine_guid) != 0)
99 return false;
100
101 char *agent_claim_id = get_agent_claimid();
@@ -103,11 +103,11 @@ static bool verify_agent_uuids(const char *machine_guid, const char *node_id, co
103 bool not_verified = (!agent_claim_id || strcmp(claim_id, agent_claim_id) != 0);
104 freez(agent_claim_id);
105
106 - if(not_verified || !localhost->node_id)
106 + if (not_verified || !rrdb.localhost->node_id)
107 return false;
108
109 char buf[UUID_STR_LEN];
110 - uuid_unparse_lower(*localhost->node_id, buf);
110 + uuid_unparse_lower(*rrdb.localhost->node_id, buf);
111
112 if(strcmp(node_id, buf) != 0)
113 return false;
@@ -193,7 +193,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
193 BUFFER *wb = w->response.data;
194 buffer_flush(wb);
195 buffer_json_initialize(wb, "\"", "\"", 0, true, false);
196 - buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
196 + buffer_json_member_add_string(wb, "mg", rrdb.localhost->machine_guid);
197 buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
198 buffer_json_member_add_uuid(wb, "token", &uuid);
199 buffer_json_member_add_time_t(wb, "expiration", expires_s);
@@ -521,7 +521,7 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
521
522 if(tier_str && *tier_str) {
523 tier = str2ul(tier_str);
524 - if(tier < storage_tiers)
524 + if(tier < rrdb.storage_tiers)
525 options |= RRDR_OPTION_SELECTED_TIER;
526 else
527 tier = 0;
web/server/h2o/http_server.c
+2 -2
@@ -231,7 +231,7 @@ static inline int _netdata_uberhandler(h2o_req_t *req, RRDHOST **host)
231 static int netdata_uberhandler(h2o_handler_t *self, h2o_req_t *req)
232 {
233 UNUSED(self);
234 - RRDHOST *host = localhost;
234 + RRDHOST *host = rrdb.localhost;
235
236 int ret = _netdata_uberhandler(req, &host);
237
@@ -245,7 +245,7 @@ static int netdata_uberhandler(h2o_handler_t *self, h2o_req_t *req)
245 ", response: %d",
246 PRINTF_H2O_IOVEC(&req->method),
247 PRINTF_H2O_IOVEC(&req->input.path),
248 - host == localhost ? "localhost" : host_uuid_str,
248 + host == rrdb.localhost ? "localhost" : host_uuid_str,
249 req->res.status);
250 } else {
251 netdata_log_access("HTTPD %d"
web/server/static/static-threaded.c
+1 -1
@@ -438,7 +438,7 @@ void *socket_listen_main_static_threaded_worker(void *ptr) {
438 , NULL
439 , web_client_first_request_timeout
440 , web_client_timeout
441 - , default_rrd_update_every * 1000 // timer_milliseconds
441 + , rrdb.default_update_every * 1000 // timer_milliseconds
442 , ptr // timer_data
443 , worker_private->max_sockets
444 );
web/server/web_client.c
+7 -13
@@ -683,12 +683,6 @@ void buffer_data_options2string(BUFFER *wb, uint32_t options) {
683 }
684
685 static inline int check_host_and_call(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) {
686 - //if(unlikely(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) {
687 - // buffer_flush(w->response.data);
688 - // buffer_strcat(w->response.data, "This host does not maintain a database");
689 - // return HTTP_RESP_BAD_REQUEST;
690 - //}
691 -
686 return func(host, w, url);
687 }
688
@@ -1282,7 +1276,7 @@ void web_client_build_http_header(struct web_client *w) {
1276 // set a proper expiration date, if not already set
1277 if(unlikely(!w->response.data->expires)) {
1278 if(w->response.data->options & WB_CONTENT_NO_CACHEABLE)
1285 - w->response.data->expires = w->response.data->date + localhost->rrd_update_every;
1279 + w->response.data->expires = w->response.data->date + rrdb.localhost->update_every;
1280 else
1281 w->response.data->expires = w->response.data->date + 86400;
1282 }
@@ -1467,7 +1461,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1461 hash_localhost = simple_hash("localhost");
1462 }
1463
1470 - if(host != localhost) {
1464 + if(host != rrdb.localhost) {
1465 buffer_flush(w->response.data);
1466 buffer_strcat(w->response.data, "Nesting of hosts is not allowed.");
1467 return HTTP_RESP_BAD_REQUEST;
@@ -1478,7 +1472,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1472 netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for host with name '%s'.", w->id, tok);
1473
1474 if(nodeid) {
1481 - host = find_host_by_node_id(tok);
1475 + host = rrdhost_find_by_node_id(tok);
1476 if(!host) {
1477 host = rrdhost_find_by_hostname(tok);
1478 if (!host)
@@ -1490,7 +1484,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1484 if(!host) {
1485 host = rrdhost_find_by_guid(tok);
1486 if (!host)
1493 - host = find_host_by_node_id(tok);
1487 + host = rrdhost_find_by_node_id(tok);
1488 }
1489 }
1490
@@ -1669,8 +1663,8 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1663 netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1664
1665 // do we have such a data set?
1672 - RRDSET *st = rrdset_find_byname(host, tok);
1673 - if(!st) st = rrdset_find(host, tok);
1666 + RRDSET *st = rrdset_find_by_name(host, tok);
1667 + if(!st) st = rrdset_find_by_id(host, tok);
1668 if(!st) {
1669 w->response.data->content_type = CT_TEXT_HTML;
1670 buffer_strcat(w->response.data, "Chart is not found: ");
@@ -1793,7 +1787,7 @@ void web_client_process_request(struct web_client *w) {
1787 }
1788 }
1789
1796 - w->response.code = (short)web_client_process_url(localhost, w, path);
1790 + w->response.code = (short)web_client_process_url(rrdb.localhost, w, path);
1791 break;
1792 }
1793 break;