@cryptotaxi247 / netdata-1 / commits / 0e230a260

Revert "Refactor RRD code. (#15423)" (#15723)

This reverts commit 440bd51e08fdfa2a4daa191fb68643456028a753. dbengine was still being used for non-zero tiers even on non-dbengine modes.

vkalintiris committed Aug 3, 2023 at 13:13 UTC 0e230a260ec7d8b4d6dc5c7165da6ab103d096b3
155 files changed +2917 -3062
CMakeLists.txt
-4
@@ -734,7 +734,6 @@ set(RRD_PLUGIN_FILES
734 database/rrdfunctions.h
735 database/rrdhost.c
736 database/rrdlabels.c
737 - database/rrdlabels.h
737 database/rrd.c
738 database/rrd.h
739 database/rrdset.c
@@ -744,7 +743,6 @@ set(RRD_PLUGIN_FILES
743 database/rrdvar.h
744 database/storage_engine.c
745 database/storage_engine.h
747 - database/storage_engine_types.h
746 database/ram/rrddim_mem.c
747 database/ram/rrddim_mem.h
748 database/sqlite/sqlite_metadata.c
@@ -776,7 +774,6 @@ set(RRD_PLUGIN_FILES
774 database/engine/rrdenginelib.h
775 database/engine/rrdengineapi.c
776 database/engine/rrdengineapi.h
779 - database/engine/rrddim_eng.h
777 database/engine/pagecache.c
778 database/engine/pagecache.h
779 database/engine/cache.c
@@ -1059,7 +1056,6 @@ ENDIF()
1056
1057 set(NETDATA_FILES
1058 collectors/all.h
1062 - collectors/utils.h
1059 ${DAEMON_FILES}
1060 ${API_PLUGIN_FILES}
1061 ${EXPORTING_ENGINE_FILES}
Makefile.am
-4
@@ -464,7 +464,6 @@ RRD_PLUGIN_FILES = \
464 database/rrdfamily.c \
465 database/rrdhost.c \
466 database/rrdlabels.c \
467 - database/rrdlabels.h \
467 database/rrd.c \
468 database/rrd.h \
469 database/rrdset.c \
@@ -476,7 +475,6 @@ RRD_PLUGIN_FILES = \
475 database/rrdvar.h \
476 database/storage_engine.c \
477 database/storage_engine.h \
479 - database/storage_engine_types.h \
478 database/ram/rrddim_mem.c \
479 database/ram/rrddim_mem.h \
480 database/sqlite/sqlite_functions.c \
@@ -549,7 +547,6 @@ libjudy_a-JudyLTables.$(OBJEXT) : CFLAGS += -I$(abs_top_srcdir)/libnetdata/libju
547
548 if ENABLE_DBENGINE
549 RRD_PLUGIN_FILES += \
552 - database/engine/rrddim_eng.h \
550 database/engine/rrdengine.c \
551 database/engine/rrdengine.h \
552 database/engine/rrddiskprotocol.h \
@@ -1075,7 +1072,6 @@ endif #ENABLE_H2O
1072
1073 NETDATA_FILES = \
1074 collectors/all.h \
1078 - collectors/utils.h \
1075 $(DAEMON_FILES) \
1076 $(LIBNETDATA_FILES) \
1077 $(API_PLUGIN_FILES) \
aclk/aclk.c
+18 -18
@@ -913,15 +913,15 @@ void aclk_host_state_update(RRDHOST *host, int cmd)
913 // node_id not found
914 aclk_query_t create_query;
915 create_query = aclk_query_new(REGISTER_NODE);
916 - rrdhost_aclk_state_lock(rrdb.localhost);
916 + rrdhost_aclk_state_lock(localhost);
917 node_instance_creation_t node_instance_creation = {
918 - .claim_id = rrdb.localhost->aclk_state.claimed_id,
918 + .claim_id = localhost->aclk_state.claimed_id,
919 .hops = host->system_info->hops,
920 .hostname = rrdhost_hostname(host),
921 .machine_guid = host->machine_guid};
922 create_query->data.bin_payload.payload =
923 generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation);
924 - rrdhost_aclk_state_unlock(rrdb.localhost);
924 + rrdhost_aclk_state_unlock(localhost);
925 create_query->data.bin_payload.topic = ACLK_TOPICID_CREATE_NODE;
926 create_query->data.bin_payload.msg_name = "CreateNodeInstance";
927 netdata_log_info("Registering host=%s, hops=%u", host->machine_guid, host->system_info->hops);
@@ -942,10 +942,10 @@ void aclk_host_state_update(RRDHOST *host, int cmd)
942
943 node_state_update.capabilities = aclk_get_agent_capas();
944
945 - rrdhost_aclk_state_lock(rrdb.localhost);
946 - node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id;
945 + rrdhost_aclk_state_lock(localhost);
946 + node_state_update.claim_id = localhost->aclk_state.claimed_id;
947 query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update);
948 - rrdhost_aclk_state_unlock(rrdb.localhost);
948 + rrdhost_aclk_state_unlock(localhost);
949
950 netdata_log_info("Queuing status update for node=%s, live=%d, hops=%u",(char*)node_state_update.node_id, cmd,
951 host->system_info->hops);
@@ -986,10 +986,10 @@ void aclk_send_node_instances()
986 }
987 node_state_update.capabilities = aclk_get_node_instance_capas(host);
988
989 - rrdhost_aclk_state_lock(rrdb.localhost);
990 - node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id;
989 + rrdhost_aclk_state_lock(localhost);
990 + node_state_update.claim_id = localhost->aclk_state.claimed_id;
991 query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update);
992 - rrdhost_aclk_state_unlock(rrdb.localhost);
992 + rrdhost_aclk_state_unlock(localhost);
993 netdata_log_info("Queuing status update for node=%s, live=%d, hops=%d",(char*)node_state_update.node_id,
994 list->live,
995 list->hops);
@@ -1010,10 +1010,10 @@ void aclk_send_node_instances()
1010 uuid_unparse_lower(list->host_id, (char*)node_instance_creation.machine_guid);
1011 create_query->data.bin_payload.topic = ACLK_TOPICID_CREATE_NODE;
1012 create_query->data.bin_payload.msg_name = "CreateNodeInstance";
1013 - rrdhost_aclk_state_lock(rrdb.localhost);
1014 - node_instance_creation.claim_id = rrdb.localhost->aclk_state.claimed_id,
1013 + rrdhost_aclk_state_lock(localhost);
1014 + node_instance_creation.claim_id = localhost->aclk_state.claimed_id,
1015 create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation);
1016 - rrdhost_aclk_state_unlock(rrdb.localhost);
1016 + rrdhost_aclk_state_unlock(localhost);
1017 netdata_log_info("Queuing registration for host=%s, hops=%d",(char*)node_instance_creation.machine_guid,
1018 list->hops);
1019 freez((void *)node_instance_creation.machine_guid);
@@ -1124,9 +1124,9 @@ char *aclk_state(void)
1124 buffer_sprintf(wb, "\n\tNode ID: %s\n", node_id);
1125 }
1126
1127 - buffer_sprintf(wb, "\tStreaming Hops: %d\n\tRelationship: %s", host->system_info->hops, host == rrdb.localhost ? "self" : "child");
1127 + buffer_sprintf(wb, "\tStreaming Hops: %d\n\tRelationship: %s", host->system_info->hops, host == localhost ? "self" : "child");
1128
1129 - if (host != rrdb.localhost)
1129 + if (host != localhost)
1130 buffer_sprintf(wb, "\n\tStreaming Connection Live: %s", host->receiver ? "true" : "false");
1131
1132 buffer_strcat(wb, "\n\tAlert Streaming Status:");
@@ -1270,10 +1270,10 @@ char *aclk_state_json(void)
1270 tmp = json_object_new_int(host->system_info->hops);
1271 json_object_object_add(nodeinstance, "streaming-hops", tmp);
1272
1273 - tmp = json_object_new_string(host == rrdb.localhost ? "self" : "child");
1273 + tmp = json_object_new_string(host == localhost ? "self" : "child");
1274 json_object_object_add(nodeinstance, "relationship", tmp);
1275
1276 - tmp = json_object_new_boolean((host->receiver || host == rrdb.localhost));
1276 + tmp = json_object_new_boolean((host->receiver || host == localhost));
1277 json_object_object_add(nodeinstance, "streaming-online", tmp);
1278
1279 tmp = json_object_new_object();
@@ -1292,7 +1292,7 @@ char *aclk_state_json(void)
1292 }
1293
1294 void add_aclk_host_labels(void) {
1295 - DICTIONARY *labels = rrdb.localhost->rrdlabels;
1295 + DICTIONARY *labels = localhost->rrdlabels;
1296
1297 #ifdef ENABLE_ACLK
1298 rrdlabels_add(labels, "_aclk_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
@@ -1324,5 +1324,5 @@ void aclk_queue_node_info(RRDHOST *host, bool immediate)
1324 {
1325 struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) host->aclk_sync_host_config;
1326 if (likely(wc))
1327 - wc->node_info_send_time = (host == rrdb.localhost || immediate) ? 1 : now_realtime_sec();
1327 + wc->node_info_send_time = (host == localhost || immediate) ? 1 : now_realtime_sec();
1328 }
aclk/aclk_capas.c
+3 -3
@@ -21,19 +21,19 @@ const struct capability *aclk_get_agent_capas()
21 { .name = NULL, .version = 0, .enabled = 0 }
22 };
23 agent_capabilities[2].version = ml_capable() ? 1 : 0;
24 - agent_capabilities[2].enabled = ml_enabled(rrdb.localhost);
24 + agent_capabilities[2].enabled = ml_enabled(localhost);
25
26 agent_capabilities[3].version = enable_metric_correlations ? metric_correlations_version : 0;
27 agent_capabilities[3].enabled = enable_metric_correlations;
28
29 - agent_capabilities[7].enabled = rrdb.localhost->health.health_enabled;
29 + agent_capabilities[7].enabled = localhost->health.health_enabled;
30
31 return agent_capabilities;
32 }
33
34 struct capability *aclk_get_node_instance_capas(RRDHOST *host)
35 {
36 - bool functions = (host == rrdb.localhost || (host->receiver && stream_has_capability(host->receiver, STREAM_CAP_FUNCTIONS)));
36 + bool functions = (host == localhost || (host->receiver && stream_has_capability(host->receiver, STREAM_CAP_FUNCTIONS)));
37
38 struct capability ni_caps[] = {
39 { .name = "proto", .version = 1, .enabled = 1 },
aclk/aclk_proxy.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "aclk_proxy.h"
2
3 #include "daemon/common.h"
aclk/aclk_proxy.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef ACLK_PROXY_H
2 #define ACLK_PROXY_H
3
aclk/aclk_query.c
+1 -1
@@ -128,7 +128,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
128 ACLK_STATS_UNLOCK;
129 }
130
131 - w->response.code = (short)web_client_api_request_with_node_selection(rrdb.localhost, w, path);
131 + w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path);
132 web_client_timeout_checkpoint_response_ready(w, &t);
133
134 if(buffer_strlen(w->response.data) > ACLK_MAX_WEB_RESPONSE_SIZE) {
aclk/aclk_rrdhost_state.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef ACLK_RRDHOST_STATE_H
2 #define ACLK_RRDHOST_STATE_H
3
aclk/aclk_rx_msgs.c
+4 -4
@@ -279,7 +279,7 @@ int create_node_instance_result(const char *msg, size_t msg_len)
279
280 RRDHOST *host = rrdhost_find_by_guid(res.machine_guid);
281 if (likely(host)) {
282 - if (host == rrdb.localhost) {
282 + if (host == localhost) {
283 node_state_update.live = 1;
284 node_state_update.hops = 0;
285 } else {
@@ -289,10 +289,10 @@ int create_node_instance_result(const char *msg, size_t msg_len)
289 node_state_update.capabilities = aclk_get_node_instance_capas(host);
290 }
291
292 - rrdhost_aclk_state_lock(rrdb.localhost);
293 - node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id;
292 + rrdhost_aclk_state_lock(localhost);
293 + node_state_update.claim_id = localhost->aclk_state.claimed_id;
294 query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update);
295 - rrdhost_aclk_state_unlock(rrdb.localhost);
295 + rrdhost_aclk_state_unlock(localhost);
296
297 freez((void *)node_state_update.capabilities);
298
aclk/aclk_stats.c
+25 -25
@@ -38,7 +38,7 @@ static void aclk_stats_collect(struct aclk_metrics_per_sample *per_sample, struc
38 if (unlikely(!st_aclkstats)) {
39 st_aclkstats = rrdset_create_localhost(
40 "netdata", "aclk_status", NULL, "aclk", NULL, "ACLK/Cloud connection status",
41 - "connected", "netdata", "stats", 200000, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
41 + "connected", "netdata", "stats", 200000, localhost->rrd_update_every, RRDSET_TYPE_LINE);
42
43 rd_online_status = rrddim_add(st_aclkstats, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
44 }
@@ -57,10 +57,10 @@ static void aclk_stats_query_queue(struct aclk_metrics_per_sample *per_sample)
57 if (unlikely(!st_query_thread)) {
58 st_query_thread = rrdset_create_localhost(
59 "netdata", "aclk_query_per_second", NULL, "aclk", NULL, "ACLK Queries per second", "queries/s",
60 - "netdata", "stats", 200001, rrdb.localhost->update_every, RRDSET_TYPE_AREA);
60 + "netdata", "stats", 200001, localhost->rrd_update_every, RRDSET_TYPE_AREA);
61
62 - rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
63 - rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
62 + rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
63 + rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
64 }
65
66 rrddim_set_by_pointer(st_query_thread, rd_queued, per_sample->queries_queued);
@@ -79,7 +79,7 @@ static void aclk_stats_latency(struct aclk_metrics_per_sample *per_sample)
79 if (unlikely(!st)) {
80 st = rrdset_create_localhost(
81 "netdata", "aclk_latency_mqtt", NULL, "aclk", NULL, "ACLK Message Publish Latency", "ms",
82 - "netdata", "stats", 200002, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
82 + "netdata", "stats", 200002, localhost->rrd_update_every, RRDSET_TYPE_LINE);
83
84 rd_avg = rrddim_add(st, "avg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
85 rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -105,10 +105,10 @@ static void aclk_stats_cloud_req(struct aclk_metrics_per_sample *per_sample)
105 if (unlikely(!st)) {
106 st = rrdset_create_localhost(
107 "netdata", "aclk_cloud_req", NULL, "aclk", NULL, "Requests received from cloud", "req/s",
108 - "netdata", "stats", 200005, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
108 + "netdata", "stats", 200005, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
109
110 - rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
111 - rd_rq_err = rrddim_add(st, "malformed", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
110 + rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
111 + rd_rq_err = rrddim_add(st, "malformed", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
112 }
113
114 rrddim_set_by_pointer(st, rd_rq_rcvd, per_sample->cloud_req_recvd - per_sample->cloud_req_err);
@@ -125,10 +125,10 @@ static void aclk_stats_cloud_req_type(struct aclk_metrics_per_sample *per_sample
125 if (unlikely(!st)) {
126 st = rrdset_create_localhost(
127 "netdata", "aclk_processed_query_type", NULL, "aclk", NULL, "Query thread commands processed by their type", "cmd/s",
128 - "netdata", "stats", 200006, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
128 + "netdata", "stats", 200006, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
129
130 for (int i = 0; i < ACLK_QUERY_TYPE_COUNT; i++)
131 - dims[i] = rrddim_add(st, aclk_query_get_name(i, 1), NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
131 + dims[i] = rrddim_add(st, aclk_query_get_name(i, 1), NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
132
133 }
134
@@ -167,10 +167,10 @@ static void aclk_stats_cloud_req_http_type(struct aclk_metrics_per_sample *per_s
167 if (unlikely(!st)) {
168 st = rrdset_create_localhost(
169 "netdata", "aclk_cloud_req_http_type", NULL, "aclk", NULL, "Requests received from cloud via HTTP by their type", "req/s",
170 - "netdata", "stats", 200007, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
170 + "netdata", "stats", 200007, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
171
172 for (int i = 0; i < ACLK_STATS_CLOUD_HTTP_REQ_TYPE_CNT; i++)
173 - rd_rq_types[i] = rrddim_add(st, cloud_req_http_type_names[i], NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
173 + rd_rq_types[i] = rrddim_add(st, cloud_req_http_type_names[i], NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
174 }
175
176 for (int i = 0; i < ACLK_STATS_CLOUD_HTTP_REQ_TYPE_CNT; i++)
@@ -189,12 +189,12 @@ static void aclk_stats_query_threads(uint32_t *queries_per_thread)
189 if (unlikely(!st)) {
190 st = rrdset_create_localhost(
191 "netdata", "aclk_query_threads", NULL, "aclk", NULL, "Queries Processed Per Thread", "req/s",
192 - "netdata", "stats", 200009, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
192 + "netdata", "stats", 200009, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
193
194 for (int i = 0; i < aclk_stats_cfg.query_thread_count; i++) {
195 if (snprintfz(dim_name, MAX_DIM_NAME, "Query %d", i) < 0)
196 netdata_log_error("snprintf encoding error");
197 - aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
197 + aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
198 }
199 }
200
@@ -215,11 +215,11 @@ static void aclk_stats_query_time(struct aclk_metrics_per_sample *per_sample)
215 if (unlikely(!st)) {
216 st = rrdset_create_localhost(
217 "netdata", "aclk_query_time", NULL, "aclk", NULL, "Time it took to process cloud requested DB queries", "us",
218 - "netdata", "stats", 200008, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
218 + "netdata", "stats", 200008, localhost->rrd_update_every, RRDSET_TYPE_LINE);
219
220 - rd_rq_avg = rrddim_add(st, "avg", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
221 - rd_rq_max = rrddim_add(st, "max", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
222 - rd_rq_total = rrddim_add(st, "total", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
220 + rd_rq_avg = rrddim_add(st, "avg", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
221 + rd_rq_max = rrddim_add(st, "max", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
222 + rd_rq_total = rrddim_add(st, "total", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
223 }
224
225 if(per_sample->cloud_q_process_count)
@@ -240,10 +240,10 @@ static void aclk_stats_newproto_rx(uint32_t *rx_msgs_sample)
240 if (unlikely(!st)) {
241 st = rrdset_create_localhost(
242 "netdata", "aclk_protobuf_rx_types", NULL, "aclk", NULL, "Received new cloud architecture messages by their type.", "msg/s",
243 - "netdata", "stats", 200010, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
243 + "netdata", "stats", 200010, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
244
245 for (unsigned int i = 0; i < aclk_stats_cfg.proto_hdl_cnt; i++) {
246 - aclk_stats_cfg.rx_msg_dims[i] = rrddim_add(st, rx_handler_get_name(i), NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE);
246 + aclk_stats_cfg.rx_msg_dims[i] = rrddim_add(st, rx_handler_get_name(i), NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
247 }
248 }
249
@@ -284,7 +284,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
284 if (unlikely(!st)) {
285 st = rrdset_create_localhost(
286 "netdata", "aclk_openssl_bytes", NULL, "aclk", NULL, "Received and Sent bytes.", "B/s",
287 - "netdata", "stats", 200011, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
287 + "netdata", "stats", 200011, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
288
289 rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
290 rd_recvd = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -293,7 +293,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
293 if (unlikely(!st_txbuf_perc)) {
294 st_txbuf_perc = rrdset_create_localhost(
295 "netdata", "aclk_mqtt_tx_perc", NULL, "aclk", NULL, "Actively used percentage of MQTT Tx Buffer,", "%",
296 - "netdata", "stats", 200012, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
296 + "netdata", "stats", 200012, localhost->rrd_update_every, RRDSET_TYPE_LINE);
297
298 rd_txbuf_perc = rrddim_add(st_txbuf_perc, "used", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
299 }
@@ -301,7 +301,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
301 if (unlikely(!st_txbuf)) {
302 st_txbuf = rrdset_create_localhost(
303 "netdata", "aclk_mqtt_tx_queue", NULL, "aclk", NULL, "State of transmit MQTT queue.", "B",
304 - "netdata", "stats", 200013, rrdb.localhost->update_every, RRDSET_TYPE_LINE);
304 + "netdata", "stats", 200013, localhost->rrd_update_every, RRDSET_TYPE_LINE);
305
306 rd_tx_buffer_usable = rrddim_add(st_txbuf, "usable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
307 rd_tx_buffer_reclaimable = rrddim_add(st_txbuf, "reclaimable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -313,7 +313,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats)
313 if (unlikely(!st_timing)) {
314 st_timing = rrdset_create_localhost(
315 "netdata", "aclk_mqtt_wss_time", NULL, "aclk", NULL, "Time spent handling MQTT, WSS, SSL and network communication.", "us",
316 - "netdata", "stats", 200014, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
316 + "netdata", "stats", 200014, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
317
318 rd_keepalive = rrddim_add(st_timing, "keep-alive", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
319 rd_read_socket = rrddim_add(st_timing, "socket_read_ssl", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -379,7 +379,7 @@ void *aclk_stats_main_thread(void *ptr)
379
380 heartbeat_t hb;
381 heartbeat_init(&hb);
382 - usec_t step_ut = rrdb.localhost->update_every * USEC_PER_SEC;
382 + usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
383
384 struct aclk_metrics_per_sample per_sample;
385 struct aclk_metrics permanent;
aclk/aclk_tx_msgs.c
+15 -15
@@ -218,19 +218,19 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable
218 .capabilities = aclk_get_agent_capas()
219 };
220
221 - rrdhost_aclk_state_lock(rrdb.localhost);
222 - if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) {
221 + rrdhost_aclk_state_lock(localhost);
222 + if (unlikely(!localhost->aclk_state.claimed_id)) {
223 netdata_log_error("Internal error. Should not come here if not claimed");
224 - rrdhost_aclk_state_unlock(rrdb.localhost);
224 + rrdhost_aclk_state_unlock(localhost);
225 return 0;
226 }
227 - if (rrdb.localhost->aclk_state.prev_claimed_id)
228 - conn.claim_id = rrdb.localhost->aclk_state.prev_claimed_id;
227 + if (localhost->aclk_state.prev_claimed_id)
228 + conn.claim_id = localhost->aclk_state.prev_claimed_id;
229 else
230 - conn.claim_id = rrdb.localhost->aclk_state.claimed_id;
230 + conn.claim_id = localhost->aclk_state.claimed_id;
231
232 char *msg = generate_update_agent_connection(&len, &conn);
233 - rrdhost_aclk_state_unlock(rrdb.localhost);
233 + rrdhost_aclk_state_unlock(localhost);
234
235 if (!msg) {
236 netdata_log_error("Error generating agent::v1::UpdateAgentConnection payload");
@@ -238,9 +238,9 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable
238 }
239
240 pid = aclk_send_bin_message_subtopic_pid(client, msg, len, ACLK_TOPICID_AGENT_CONN, "UpdateAgentConnection");
241 - if (rrdb.localhost->aclk_state.prev_claimed_id) {
242 - freez(rrdb.localhost->aclk_state.prev_claimed_id);
243 - rrdb.localhost->aclk_state.prev_claimed_id = NULL;
241 + if (localhost->aclk_state.prev_claimed_id) {
242 + freez(localhost->aclk_state.prev_claimed_id);
243 + localhost->aclk_state.prev_claimed_id = NULL;
244 }
245 return pid;
246 }
@@ -253,16 +253,16 @@ char *aclk_generate_lwt(size_t *size) {
253 .capabilities = NULL
254 };
255
256 - rrdhost_aclk_state_lock(rrdb.localhost);
257 - if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) {
256 + rrdhost_aclk_state_lock(localhost);
257 + if (unlikely(!localhost->aclk_state.claimed_id)) {
258 netdata_log_error("Internal error. Should not come here if not claimed");
259 - rrdhost_aclk_state_unlock(rrdb.localhost);
259 + rrdhost_aclk_state_unlock(localhost);
260 return NULL;
261 }
262 - conn.claim_id = rrdb.localhost->aclk_state.claimed_id;
262 + conn.claim_id = localhost->aclk_state.claimed_id;
263
264 char *msg = generate_update_agent_connection(size, &conn);
265 - rrdhost_aclk_state_unlock(rrdb.localhost);
265 + rrdhost_aclk_state_unlock(localhost);
266
267 if (!msg)
268 netdata_log_error("Error generating agent::v1::UpdateAgentConnection payload for LWT");
aclk/aclk_util.c
+7 -7
@@ -183,20 +183,20 @@ static void topic_generate_final(struct aclk_topic *t) {
183 if (!replace_tag)
184 return;
185
186 - rrdhost_aclk_state_lock(rrdb.localhost);
187 - if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) {
186 + rrdhost_aclk_state_lock(localhost);
187 + if (unlikely(!localhost->aclk_state.claimed_id)) {
188 netdata_log_error("This should never be called if agent not claimed");
189 - rrdhost_aclk_state_unlock(rrdb.localhost);
189 + rrdhost_aclk_state_unlock(localhost);
190 return;
191 }
192
193 - t->topic = mallocz(strlen(t->topic_recvd) + 1 - strlen(CLAIM_ID_REPLACE_TAG) + strlen(rrdb.localhost->aclk_state.claimed_id));
193 + t->topic = mallocz(strlen(t->topic_recvd) + 1 - strlen(CLAIM_ID_REPLACE_TAG) + strlen(localhost->aclk_state.claimed_id));
194 memcpy(t->topic, t->topic_recvd, replace_tag - t->topic_recvd);
195 dest = t->topic + (replace_tag - t->topic_recvd);
196
197 - memcpy(dest, rrdb.localhost->aclk_state.claimed_id, strlen(rrdb.localhost->aclk_state.claimed_id));
198 - dest += strlen(rrdb.localhost->aclk_state.claimed_id);
199 - rrdhost_aclk_state_unlock(rrdb.localhost);
197 + memcpy(dest, localhost->aclk_state.claimed_id, strlen(localhost->aclk_state.claimed_id));
198 + dest += strlen(localhost->aclk_state.claimed_id);
199 + rrdhost_aclk_state_unlock(localhost);
200 replace_tag += strlen(CLAIM_ID_REPLACE_TAG);
201 strcpy(dest, replace_tag);
202 dest += strlen(replace_tag);
aclk/schema-wrappers/node_info.cc
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "node_info.h"
2
3 #include "proto/nodeinstance/info/v1/info.pb.h"
aclk/schema-wrappers/proto_2_json.cc
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include <google/protobuf/message.h>
2 #include <google/protobuf/util/json_util.h>
3
claim/claim.c
+6 -7
@@ -35,9 +35,9 @@ static char *claiming_errors[] = {
35 char *get_agent_claimid()
36 {
37 char *result;
38 - rrdhost_aclk_state_lock(rrdb.localhost);
39 - result = (rrdb.localhost->aclk_state.claimed_id == NULL) ? NULL : strdupz(rrdb.localhost->aclk_state.claimed_id);
40 - rrdhost_aclk_state_unlock(rrdb.localhost);
38 + rrdhost_aclk_state_lock(localhost);
39 + result = (localhost->aclk_state.claimed_id == NULL) ? NULL : strdupz(localhost->aclk_state.claimed_id);
40 + rrdhost_aclk_state_unlock(localhost);
41 return result;
42 }
43
@@ -89,7 +89,7 @@ CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, con
89 command_exec_buffer,
90 proxy_flag,
91 netdata_configured_hostname,
92 - rrdb.localhost->machine_guid,
92 + localhost->machine_guid,
93 cloud_base_url,
94 claiming_arguments);
95
@@ -156,7 +156,6 @@ void load_claiming_state(void)
156 #if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
157 netdata_cloud_enabled = false;
158 #else
159 - RRDHOST *localhost = rrdb.localhost;
159 uuid_t uuid;
160
161 // Propagate into aclk and registry. Be kind of atomic...
@@ -327,7 +326,7 @@ void claim_reload_all(void) {
326 error_log_limit_unlimited();
327 load_claiming_state();
328 registry_update_cloud_base_url();
330 - rrdpush_send_claimed_id(rrdb.localhost);
329 + rrdpush_send_claimed_id(localhost);
330 error_log_limit_reset();
331 }
332
@@ -424,7 +423,7 @@ int api_v2_claim(struct web_client *w, char *url) {
423 int ms = 0;
424 do {
425 status = cloud_status();
427 - if (status == CLOUD_STATUS_ONLINE && __atomic_load_n(&rrdb.localhost->node_id, __ATOMIC_RELAXED))
426 + if (status == CLOUD_STATUS_ONLINE && __atomic_load_n(&localhost->node_id, __ATOMIC_RELAXED))
427 break;
428
429 sleep_usec(50 * USEC_PER_MS);
collectors/cgroups.plugin/sys_fs_cgroup.c
+3 -3
@@ -301,9 +301,9 @@ void read_cgroup_plugin_configuration() {
301 throttled_time_hash = simple_hash("throttled_time");
302 throttled_usec_hash = simple_hash("throttled_usec");
303
304 - cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", rrdb.localhost->update_every);
305 - if(cgroup_update_every < rrdb.localhost->update_every)
306 - cgroup_update_every = rrdb.localhost->update_every;
304 + cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every);
305 + if(cgroup_update_every < localhost->rrd_update_every)
306 + cgroup_update_every = localhost->rrd_update_every;
307
308 cgroup_check_for_new_every = (int)config_get_number("plugin:cgroups", "check for new cgroups every", (long long)cgroup_check_for_new_every * (long long)cgroup_update_every);
309 if(cgroup_check_for_new_every < cgroup_update_every)
collectors/cgroups.plugin/tests/test_doubles.c
+4 -4
@@ -47,7 +47,7 @@ void mountinfo_free_all(struct mountinfo *mi)
47 RRDSET *rrdset_create_custom(
48 RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context,
49 const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every,
50 - RRDSET_TYPE chart_type, STORAGE_ENGINE_ID storage_engine_id, long history_entries)
50 + RRDSET_TYPE chart_type, RRD_MEMORY_MODE memory_mode, long history_entries)
51 {
52 UNUSED(host);
53 UNUSED(type);
@@ -62,7 +62,7 @@ RRDSET *rrdset_create_custom(
62 UNUSED(priority);
63 UNUSED(update_every);
64 UNUSED(chart_type);
65 - UNUSED(storage_engine_id);
65 + UNUSED(memory_mode);
66 UNUSED(history_entries);
67
68 return NULL;
@@ -70,7 +70,7 @@ RRDSET *rrdset_create_custom(
70
71 RRDDIM *rrddim_add_custom(
72 RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor,
73 - RRD_ALGORITHM algorithm, STORAGE_ENGINE_ID storage_engine_id)
73 + RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode)
74 {
75 UNUSED(st);
76 UNUSED(id);
@@ -78,7 +78,7 @@ RRDDIM *rrddim_add_custom(
78 UNUSED(multiplier);
79 UNUSED(divisor);
80 UNUSED(algorithm);
81 - UNUSED(storage_engine_id);
81 + UNUSED(memory_mode);
82
83 return NULL;
84 }
collectors/diskspace.plugin/plugin_diskspace.c
+4 -5
@@ -1,7 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "collectors/proc.plugin/plugin_proc.h"
4 -#include "collectors/utils.h"
3 +#include "../proc.plugin/plugin_proc.h"
4
5 #define PLUGIN_DISKSPACE_NAME "diskspace.plugin"
6
@@ -627,9 +626,9 @@ void *diskspace_main(void *ptr) {
626
627 cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points);
628
630 - int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", rrdb.localhost->update_every);
631 - if(update_every < rrdb.localhost->update_every)
632 - update_every = rrdb.localhost->update_every;
629 + int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", localhost->rrd_update_every);
630 + if(update_every < localhost->rrd_update_every)
631 + update_every = localhost->rrd_update_every;
632
633 check_for_new_mountpoints_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "check for new mount points every", check_for_new_mountpoints_every);
634 if(check_for_new_mountpoints_every < update_every)
collectors/ebpf.plugin/ebpf_unittest.h
+1 -3
@@ -1,7 +1,5 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef NETDATA_EBPF_PLUGIN_UNITTEST_H_
4 -#define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1
2 +# define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1
3
4 #include "ebpf.h"
5
collectors/freebsd.plugin/plugin_freebsd.c
+2 -2
@@ -104,7 +104,7 @@ void *freebsd_main(void *ptr)
104 worker_register_job_name(i, freebsd_modules[i].dim);
105 }
106
107 - usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
107 + usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
108 heartbeat_t hb;
109 heartbeat_init(&hb);
110
@@ -124,7 +124,7 @@ void *freebsd_main(void *ptr)
124 netdata_log_debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name);
125
126 worker_is_busy(i);
127 - pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt);
127 + pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
128
129 if (unlikely(netdata_exit))
130 break;
collectors/freebsd.plugin/plugin_freebsd.h
-1
@@ -4,7 +4,6 @@
4 #define NETDATA_PLUGIN_FREEBSD_H 1
5
6 #include "daemon/common.h"
7 -#include "collectors/utils.h"
7
8 #include <sys/sysctl.h>
9
collectors/idlejitter.plugin/plugin_idlejitter.c
+2 -2
@@ -38,14 +38,14 @@ void *cpuidlejitter_main(void *ptr) {
38 , "idlejitter.plugin"
39 , NULL
40 , NETDATA_CHART_PRIO_SYSTEM_IDLEJITTER
41 - , rrdb.localhost->update_every
41 + , localhost->rrd_update_every
42 , RRDSET_TYPE_AREA
43 );
44 RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
45 RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
46 RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
47
48 - usec_t update_every_ut = rrdb.localhost->update_every * USEC_PER_SEC;
48 + usec_t update_every_ut = localhost->rrd_update_every * USEC_PER_SEC;
49 struct timeval before, after;
50
51 while (service_running(SERVICE_COLLECTORS)) {
collectors/macos.plugin/plugin_macos.c
+2 -2
@@ -53,7 +53,7 @@ void *macos_main(void *ptr)
53 worker_register_job_name(i, macos_modules[i].dim);
54 }
55
56 - usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
56 + usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
57 heartbeat_t hb;
58 heartbeat_init(&hb);
59
@@ -69,7 +69,7 @@ void *macos_main(void *ptr)
69 netdata_log_debug(D_PROCNETDEV_LOOP, "macos calling %s.", pm->name);
70
71 worker_is_busy(i);
72 - pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt);
72 + pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
73
74 if (unlikely(netdata_exit))
75 break;
collectors/macos.plugin/plugin_macos.h
-1
@@ -4,7 +4,6 @@
4 #define NETDATA_PLUGIN_MACOS_H 1
5
6 #include "daemon/common.h"
7 -#include "collectors/utils.h"
7
8 int do_macos_sysctl(int update_every, usec_t dt);
9 int do_macos_mach_smi(int update_every, usec_t dt);
collectors/plugins.d/local_listeners.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "libnetdata/libnetdata.h"
2 #include "libnetdata/required_dummies.h"
3
collectors/plugins.d/plugins_d.c
+2 -2
@@ -293,11 +293,11 @@ void *pluginsd_main(void *ptr)
293 strncpyz(cd->filename, file->d_name, FILENAME_MAX);
294 snprintfz(cd->fullfilename, FILENAME_MAX, "%s/%s", directory_name, cd->filename);
295
296 - cd->host = rrdb.localhost;
296 + cd->host = localhost;
297 cd->unsafe.enabled = enabled;
298 cd->unsafe.running = false;
299
300 - cd->update_every = (int)config_get_number(cd->id, "update every", rrdb.localhost->update_every);
300 + cd->update_every = (int)config_get_number(cd->id, "update every", localhost->rrd_update_every);
301 cd->started_t = now_realtime_sec();
302
303 char *def = "";
collectors/plugins.d/pluginsd_parser.c
+7 -34
@@ -204,7 +204,7 @@ static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, cons
204 return NULL;
205 }
206
207 - RRDSET *st = rrdset_find_by_id(host, chart);
207 + RRDSET *st = rrdset_find(host, chart);
208 if (unlikely(!st))
209 netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.",
210 rrdhost_hostname(host), chart, cmd);
@@ -375,38 +375,11 @@ static inline PARSER_RC pluginsd_host_labels(char **words, size_t num_words, PAR
375 PLUGINSD_KEYWORD_HOST_LABEL);
376 }
377
378 -static struct rrdhost_system_info *labels_to_system_info(DICTIONARY *labels) {
379 - struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
380 - info->hops = 1;
381 -
382 - rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type");
383 - rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type");
384 - rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region");
385 - rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name");
386 - rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version");
387 - rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version");
388 - rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores");
389 - rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq");
390 - rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total");
391 - rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space");
392 - rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture");
393 - rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization");
394 - rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container");
395 - rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection");
396 - rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection");
397 - rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node");
398 - rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type");
399 - rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch");
400 - rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist");
401 -
402 - return info;
403 -}
404 -
378 static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
379 if(!parser->user.host_define.parsing_host)
380 return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE_END, "missing initialization, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");
381
409 - RRDHOST *host = rrdhost_get_or_create(
382 + RRDHOST *host = rrdhost_find_or_create(
383 string2str(parser->user.host_define.hostname),
384 string2str(parser->user.host_define.hostname),
385 parser->user.host_define.machine_guid_str,
@@ -417,9 +390,9 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
390 NULL,
391 program_name,
392 program_version,
420 - rrdb.default_update_every,
421 - rrdb.default_rrd_history_entries,
422 - default_storage_engine_id,
393 + default_rrd_update_every,
394 + default_rrd_history_entries,
395 + default_rrd_memory_mode,
396 default_health_enabled,
397 default_rrdpush_enabled,
398 default_rrdpush_destination,
@@ -428,7 +401,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
401 default_rrdpush_enable_replication,
402 default_rrdpush_seconds_to_replicate,
403 default_rrdpush_replication_step,
431 - labels_to_system_info(parser->user.host_define.rrdlabels),
404 + rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels),
405 false
406 );
407
@@ -458,7 +431,7 @@ static inline PARSER_RC pluginsd_host(char **words, size_t num_words, PARSER *pa
431 char *guid = get_word(words, num_words, 1);
432
433 if(!guid || !*guid || strcmp(guid, "localhost") == 0) {
461 - parser->user.host = rrdb.localhost;
434 + parser->user.host = localhost;
435 return PARSER_RC_OK;
436 }
437
collectors/proc.plugin/ipc.c
+6 -6
@@ -327,7 +327,7 @@ int do_ipc(int update_every, usec_t dt) {
327 , PLUGIN_PROC_NAME
328 , "ipc"
329 , NETDATA_CHART_PRIO_SYSTEM_IPC_SEMAPHORES
330 - , rrdb.localhost->update_every
330 + , localhost->rrd_update_every
331 , RRDSET_TYPE_AREA
332 );
333 rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -345,15 +345,15 @@ int do_ipc(int update_every, usec_t dt) {
345 , PLUGIN_PROC_NAME
346 , "ipc"
347 , NETDATA_CHART_PRIO_SYSTEM_IPC_SEM_ARRAYS
348 - , rrdb.localhost->update_every
348 + , localhost->rrd_update_every
349 , RRDSET_TYPE_AREA
350 );
351 rd_arrays = rrddim_add(st_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
352 }
353
354 // variables
355 - semaphores_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "ipc_semaphores_max");
356 - arrays_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "ipc_semaphores_arrays_max");
355 + semaphores_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_max");
356 + arrays_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_arrays_max");
357 }
358
359 struct stat stbuf;
@@ -373,8 +373,8 @@ int do_ipc(int update_every, usec_t dt) {
373 collector_error("Unable to fetch semaphore limits.");
374 }
375 else {
376 - if(semaphores_max) rrdvar_custom_host_variable_set(rrdb.localhost, semaphores_max, limits.semmns);
377 - if(arrays_max) rrdvar_custom_host_variable_set(rrdb.localhost, arrays_max, limits.semmni);
376 + if(semaphores_max) rrdvar_custom_host_variable_set(localhost, semaphores_max, limits.semmns);
377 + if(arrays_max) rrdvar_custom_host_variable_set(localhost, arrays_max, limits.semmni);
378
379 st_arrays->red = limits.semmni;
380 st_semaphores->red = limits.semmns;
collectors/proc.plugin/plugin_proc.c
+2 -2
@@ -161,7 +161,7 @@ void *proc_main(void *ptr)
161 worker_register_job_name(i, proc_modules[i].dim);
162 }
163
164 - usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
164 + usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
165 heartbeat_t hb;
166 heartbeat_init(&hb);
167
@@ -185,7 +185,7 @@ void *proc_main(void *ptr)
185 netdata_log_debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name);
186
187 worker_is_busy(i);
188 - pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt);
188 + pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
189 }
190 }
191
collectors/proc.plugin/proc_mdstat.c
-1
@@ -1,7 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "plugin_proc.h"
4 -#include "collectors/utils.h"
4
5 #define PLUGIN_PROC_MODULE_MDSTAT_NAME "/proc/mdstat"
6
collectors/proc.plugin/proc_net_dev.c
+2 -2
@@ -1518,7 +1518,7 @@ void *netdev_main(void *ptr)
1518
1519 netdata_thread_cleanup_push(netdev_main_cleanup, ptr);
1520
1521 - usec_t step = rrdb.localhost->update_every * USEC_PER_SEC;
1521 + usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
1522 heartbeat_t hb;
1523 heartbeat_init(&hb);
1524
@@ -1530,7 +1530,7 @@ void *netdev_main(void *ptr)
1530 break;
1531
1532 worker_is_busy(0);
1533 - if(do_proc_net_dev(rrdb.localhost->update_every, hb_dt))
1533 + if(do_proc_net_dev(localhost->rrd_update_every, hb_dt))
1534 break;
1535 }
1536
collectors/proc.plugin/proc_net_netstat.c
+2 -2
@@ -1647,7 +1647,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
1647 arl_expect(arl_udplite, "InCsumErrors", &snmp_root.udplite_InCsumErrors);
1648 arl_expect(arl_udplite, "IgnoredMulti", &snmp_root.udplite_IgnoredMulti);
1649
1650 - tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_max_connections");
1650 + tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_connections");
1651 }
1652
1653 size_t lines, l, words;
@@ -2741,7 +2741,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
2741 // snmp Tcp charts
2742
2743 // this is smart enough to update it, only when it is changed
2744 - rrdvar_custom_host_variable_set(rrdb.localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn);
2744 + rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn);
2745
2746 // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
2747 if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO &&
collectors/proc.plugin/proc_net_sockstat.c
+8 -8
@@ -32,9 +32,9 @@ static int read_tcp_mem(void) {
32 *tcp_mem_high_threshold = NULL;
33
34 if(unlikely(!tcp_mem_low_threshold)) {
35 - tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_low");
36 - tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_pressure");
37 - tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_high");
35 + tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_low");
36 + tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_pressure");
37 + tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_high");
38 }
39
40 if(unlikely(!filename)) {
@@ -60,9 +60,9 @@ static int read_tcp_mem(void) {
60
61 // fprintf(stderr, "TCP MEM low = %llu, pressure = %llu, high = %llu\n", low, pressure, high);
62
63 - rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0);
64 - rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0);
65 - rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0);
63 + rrdvar_custom_host_variable_set(localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0);
64 + rrdvar_custom_host_variable_set(localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0);
65 + rrdvar_custom_host_variable_set(localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0);
66
67 return 0;
68 }
@@ -81,9 +81,9 @@ static kernel_uint_t read_tcp_max_orphans(void) {
81 if(read_single_number_file(filename, &tcp_max_orphans) == 0) {
82
83 if(unlikely(!tcp_max_orphans_var))
84 - tcp_max_orphans_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_max_orphans");
84 + tcp_max_orphans_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_orphans");
85
86 - rrdvar_custom_host_variable_set(rrdb.localhost, tcp_max_orphans_var, tcp_max_orphans);
86 + rrdvar_custom_host_variable_set(localhost, tcp_max_orphans_var, tcp_max_orphans);
87 return tcp_max_orphans;
88 }
89
collectors/proc.plugin/proc_net_softnet_stat.c
-1
@@ -1,7 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "plugin_proc.h"
4 -#include "collectors/utils.h"
4
5 #define PLUGIN_PROC_MODULE_NET_SOFTNET_NAME "/proc/net/softnet_stat"
6
collectors/proc.plugin/proc_net_stat_conntrack.c
+2 -2
@@ -50,7 +50,7 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) {
50 if(!do_sockets && !read_full)
51 return 1;
52
53 - rrdvar_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "netfilter_conntrack_max");
53 + rrdvar_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "netfilter_conntrack_max");
54 }
55
56 if(likely(read_full)) {
@@ -125,7 +125,7 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) {
125
126 unsigned long long max;
127 if(likely(!read_single_number_file(nf_conntrack_max_filename, &max)))
128 - rrdvar_custom_host_variable_set(rrdb.localhost, rrdvar_max, max);
128 + rrdvar_custom_host_variable_set(localhost, rrdvar_max, max);
129 }
130
131 // --------------------------------------------------------------------
collectors/proc.plugin/proc_net_wireless.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include <stdbool.h>
2 #include "plugin_proc.h"
3
collectors/proc.plugin/proc_stat.c
+2 -2
@@ -719,7 +719,7 @@ int do_proc_stat(int update_every, usec_t dt) {
719 }
720
721 if(unlikely(core == 0 && cpus_var == NULL))
722 - cpus_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "active_processors");
722 + cpus_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "active_processors");
723 }
724
725 rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_user, user);
@@ -1064,7 +1064,7 @@ int do_proc_stat(int update_every, usec_t dt) {
1064 }
1065
1066 if(cpus_var)
1067 - rrdvar_custom_host_variable_set(rrdb.localhost, cpus_var, cores_found);
1067 + rrdvar_custom_host_variable_set(localhost, cpus_var, cores_found);
1068
1069 return 0;
1070 }
collectors/proc.plugin/sys_devices_pci_aer.c
+1 -1
@@ -196,7 +196,7 @@ int do_proc_sys_devices_pci_aer(int update_every, usec_t dt __maybe_unused) {
196 int do_pci_slots = CONFIG_BOOLEAN_NO;
197
198 char buffer[100 + 1] = "";
199 - rrdlabels_get_value_strcpyz(rrdb.localhost->rrdlabels, buffer, 100, "_virtualization");
199 + rrdlabels_get_value_strcpyz(localhost->rrdlabels, buffer, 100, "_virtualization");
200 if(strcmp(buffer, "none") != 0) {
201 // no need to run on virtualized environments
202 do_root_ports = CONFIG_BOOLEAN_NO;
collectors/profile.plugin/plugin_profile.cc
+2 -2
@@ -195,8 +195,8 @@ extern "C" void *profile_main(void *ptr) {
195 netdata_thread_cleanup_push(profile_main_cleanup, ptr);
196
197 int UpdateEvery = (int) config_get_number(CONFIG_SECTION_PROFILE, "update every", 1);
198 - if (UpdateEvery < rrdb.localhost->update_every)
199 - UpdateEvery = rrdb.localhost->update_every;
198 + if (UpdateEvery < localhost->rrd_update_every)
199 + UpdateEvery = localhost->rrd_update_every;
200
201 // pick low-default values, in case this plugin is ever enabled accidentaly.
202 size_t NumThreads = config_get_number(CONFIG_SECTION_PROFILE, "number of threads", 2);
collectors/statsd.plugin/statsd.c
+13 -12
@@ -218,7 +218,7 @@ typedef struct statsd_app {
218 const char *name;
219 SIMPLE_PATTERN *metrics;
220 STATS_METRIC_OPTIONS default_options;
221 - STORAGE_ENGINE_ID storage_engine_id;
221 + RRD_MEMORY_MODE rrd_memory_mode;
222 int32_t rrd_history_entries;
223 DICTIONARY *dict;
224
@@ -1296,8 +1296,8 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1296 // a new app
1297 app = callocz(sizeof(STATSD_APP), 1);
1298 app->name = strdupz("unnamed");
1299 - app->storage_engine_id = rrdb.localhost->storage_engine_id;
1300 - app->rrd_history_entries = rrdb.localhost->rrd_history_entries;
1299 + app->rrd_memory_mode = localhost->rrd_memory_mode;
1300 + app->rrd_history_entries = localhost->rrd_history_entries;
1301
1302 app->next = statsd.apps;
1303 statsd.apps = app;
@@ -1409,6 +1409,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1409 else if (!strcmp(name, "memory mode")) {
1410 // this is not supported anymore
1411 // with the implementation of storage engines, all charts have the same storage engine always
1412 + // app->rrd_memory_mode = rrd_memory_mode_id(value);
1413 ;
1414 }
1415 else if (!strcmp(name, "history")) {
@@ -1598,7 +1599,7 @@ static inline RRDSET *statsd_private_rrdset_create(
1599 statsd.private_charts++;
1600
1601 RRDSET *st = rrdset_create_custom(
1601 - rrdb.localhost // host
1602 + localhost // host
1603 , type // type
1604 , id // id
1605 , name // name
@@ -1611,8 +1612,8 @@ static inline RRDSET *statsd_private_rrdset_create(
1612 , priority // priority
1613 , update_every // update every
1614 , chart_type // chart type
1614 - , default_storage_engine_id // storage_engine_id
1615 - , rrdb.default_rrd_history_entries // history
1615 + , default_rrd_memory_mode // memory mode
1616 + , default_rrd_history_entries // history
1617 );
1618 rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
1619
@@ -2203,7 +2204,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
2204
2205 if(!chart->st) {
2206 chart->st = rrdset_create_custom(
2206 - rrdb.localhost // host
2207 + localhost // host
2208 , app->name // type
2209 , chart->id // id
2210 , chart->name // name
@@ -2216,7 +2217,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
2217 , chart->priority // priority
2218 , statsd.update_every // update every
2219 , chart->chart_type // chart type
2219 - , app->storage_engine_id // storage engine id
2220 + , app->rrd_memory_mode // memory mode
2221 , app->rrd_history_entries // history
2222 );
2223
@@ -2431,11 +2432,11 @@ void *statsd_main(void *ptr) {
2432
2433 statsd.enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, "statsd", statsd.enabled);
2434
2434 - statsd.update_every = rrdb.default_update_every;
2435 + statsd.update_every = default_rrd_update_every;
2436 statsd.update_every = (int)config_get_number(CONFIG_SECTION_STATSD, "update every (flushInterval)", statsd.update_every);
2436 - if(statsd.update_every < rrdb.default_update_every) {
2437 - collector_error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, rrdb.default_update_every);
2438 - statsd.update_every = rrdb.default_update_every;
2437 + if(statsd.update_every < default_rrd_update_every) {
2438 + collector_error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, default_rrd_update_every);
2439 + statsd.update_every = default_rrd_update_every;
2440 }
2441
2442 #ifdef HAVE_RECVMMSG
collectors/tc.plugin/plugin_tc.c
+7 -7
@@ -414,7 +414,7 @@ static inline void tc_device_commit(struct tc_device *d) {
414 PLUGIN_TC_NAME,
415 NULL,
416 NETDATA_CHART_PRIO_TC_QOS,
417 - rrdb.localhost->update_every,
417 + localhost->rrd_update_every,
418 d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
419
420 rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -475,7 +475,7 @@ static inline void tc_device_commit(struct tc_device *d) {
475 PLUGIN_TC_NAME,
476 NULL,
477 NETDATA_CHART_PRIO_TC_QOS_PACKETS,
478 - rrdb.localhost->update_every,
478 + localhost->rrd_update_every,
479 d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
480
481 rrdlabels_add(d->st_packets->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -539,7 +539,7 @@ static inline void tc_device_commit(struct tc_device *d) {
539 PLUGIN_TC_NAME,
540 NULL,
541 NETDATA_CHART_PRIO_TC_QOS_DROPPED,
542 - rrdb.localhost->update_every,
542 + localhost->rrd_update_every,
543 d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
544
545 rrdlabels_add(d->st_dropped->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -603,7 +603,7 @@ static inline void tc_device_commit(struct tc_device *d) {
603 PLUGIN_TC_NAME,
604 NULL,
605 NETDATA_CHART_PRIO_TC_QOS_TOKENS,
606 - rrdb.localhost->update_every,
606 + localhost->rrd_update_every,
607 RRDSET_TYPE_LINE);
608
609 rrdlabels_add(d->st_tokens->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -668,7 +668,7 @@ static inline void tc_device_commit(struct tc_device *d) {
668 PLUGIN_TC_NAME,
669 NULL,
670 NETDATA_CHART_PRIO_TC_QOS_CTOKENS,
671 - rrdb.localhost->update_every,
671 + localhost->rrd_update_every,
672 RRDSET_TYPE_LINE);
673
674 rrdlabels_add(d->st_ctokens->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
@@ -935,7 +935,7 @@ void *tc_main(void *ptr) {
935 struct tc_device *device = NULL;
936 struct tc_class *class = NULL;
937
938 - snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, rrdb.localhost->update_every);
938 + snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, localhost->rrd_update_every);
939 netdata_log_debug(D_TC_LOOP, "executing '%s'", command);
940
941 fp_child_output = netdata_popen(command, (pid_t *)&tc_child_pid, &fp_child_input);
@@ -1173,7 +1173,7 @@ void *tc_main(void *ptr) {
1173 goto cleanup;
1174 }
1175
1176 - sleep((unsigned int) rrdb.localhost->update_every);
1176 + sleep((unsigned int) localhost->rrd_update_every);
1177 }
1178
1179 cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement
collectors/timex.plugin/plugin_timex.c
+2 -2
@@ -50,8 +50,8 @@ void *timex_main(void *ptr)
50 netdata_thread_cleanup_push(timex_main_cleanup, ptr);
51
52 int update_every = (int)config_get_number(CONFIG_SECTION_TIMEX, "update every", 10);
53 - if (update_every < rrdb.localhost->update_every)
54 - update_every = rrdb.localhost->update_every;
53 + if (update_every < localhost->rrd_update_every)
54 + update_every = localhost->rrd_update_every;
55
56 int do_sync = config_get_boolean(CONFIG_SECTION_TIMEX, "clock synchronization state", CONFIG_BOOLEAN_YES);
57 int do_offset = config_get_boolean(CONFIG_SECTION_TIMEX, "time offset", CONFIG_BOOLEAN_YES);
collectors/utils.h deleted
-42
@@ -1,42 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef COLLECTORS_UTILS_H
4 -#define COLLECTORS_UTILS_H
5 -
6 -#include "database/rrd.h"
7 -
8 -static inline RRDSET *rrdset_find_active_localhost(const char *id)
9 -{
10 - RRDSET *st = rrdset_find_by_id(rrdb.localhost, id);
11 - if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
12 - return NULL;
13 - return st;
14 -}
15 -
16 -static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
17 -{
18 - RRDSET *st = rrdset_find_by_type(rrdb.localhost, type, id);
19 - if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
20 - return NULL;
21 - return st;
22 -}
23 -
24 -
25 -static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
26 -{
27 - RRDSET *st = rrdset_find_by_name(rrdb.localhost, name);
28 - if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
29 - return NULL;
30 - return st;
31 -}
32 -
33 -static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id) {
34 - RRDDIM *rd = rrddim_find_by_id(st, id);
35 -
36 - if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
37 - return NULL;
38 -
39 - return rd;
40 -}
41 -
42 -#endif /* COLLECTORS_UTILS_H */
daemon/analytics.c
+22 -22
@@ -25,7 +25,7 @@ struct array_printer {
25 void analytics_log_data(void)
26 {
27 netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_STREAM_ENABLED : [%s]", analytics_data.netdata_config_stream_enabled);
28 - netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_MEMORY_MODE : [%s]", analytics_data.netdata_config_storage_engine_name);
28 + netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_MEMORY_MODE : [%s]", analytics_data.netdata_config_memory_mode);
29 netdata_log_debug(D_ANALYTICS, "NETDATA_CONFIG_EXPORTING_ENABLED : [%s]", analytics_data.netdata_config_exporting_enabled);
30 netdata_log_debug(D_ANALYTICS, "NETDATA_EXPORTING_CONNECTORS : [%s]", analytics_data.netdata_exporting_connectors);
31 netdata_log_debug(D_ANALYTICS, "NETDATA_ALLMETRICS_PROMETHEUS_USED : [%s]", analytics_data.netdata_allmetrics_prometheus_used);
@@ -71,7 +71,7 @@ void analytics_log_data(void)
71 void analytics_free_data(void)
72 {
73 freez(analytics_data.netdata_config_stream_enabled);
74 - freez(analytics_data.netdata_config_storage_engine_name);
74 + freez(analytics_data.netdata_config_memory_mode);
75 freez(analytics_data.netdata_config_exporting_enabled);
76 freez(analytics_data.netdata_exporting_connectors);
77 freez(analytics_data.netdata_allmetrics_prometheus_used);
@@ -114,7 +114,7 @@ void analytics_free_data(void)
114 /*
115 * Set a numeric/boolean data with a value
116 */
117 -static void analytics_set_data(char **name, const char *value)
117 +void analytics_set_data(char **name, char *value)
118 {
119 if (*name) {
120 analytics_data.data_length -= strlen(*name);
@@ -127,7 +127,7 @@ static void analytics_set_data(char **name, const char *value)
127 /*
128 * Set a string data with a value
129 */
130 -static void analytics_set_data_str(char **name, const char *value)
130 +void analytics_set_data_str(char **name, char *value)
131 {
132 size_t value_string_len;
133 if (*name) {
@@ -215,7 +215,7 @@ void analytics_mirrored_hosts(void)
215 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
216 continue;
217
218 - ((host == rrdb.localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) ? reachable++ : unreachable++);
218 + ((host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) ? reachable++ : unreachable++);
219
220 count++;
221 }
@@ -235,7 +235,7 @@ void analytics_exporters(void)
235 //decide if something else is more suitable (but probably not null)
236 BUFFER *bi = buffer_create(1000, NULL);
237 analytics_exporting_connectors(bi);
238 - analytics_set_data_str(&analytics_data.netdata_exporting_connectors, buffer_tostring(bi));
238 + analytics_set_data_str(&analytics_data.netdata_exporting_connectors, (char *)buffer_tostring(bi));
239 buffer_free(bi);
240 }
241
@@ -272,7 +272,7 @@ void analytics_collectors(void)
272 char name[500];
273 BUFFER *bt = buffer_create(1000, NULL);
274
275 - rrdset_foreach_read(st, rrdb.localhost) {
275 + rrdset_foreach_read(st, localhost) {
276 if(!rrdset_is_available_for_viewers(st))
277 continue;
278
@@ -349,7 +349,7 @@ void analytics_alarms_notifications(void)
349 }
350 freez(script);
351
352 - analytics_set_data_str(&analytics_data.netdata_notification_methods, buffer_tostring(b));
352 + analytics_set_data_str(&analytics_data.netdata_notification_methods, (char *)buffer_tostring(b));
353
354 buffer_free(b);
355 }
@@ -377,15 +377,15 @@ void analytics_https(void)
377 analytics_exporting_connectors_ssl(b);
378
379 buffer_strcat(b, netdata_ssl_streaming_sender_ctx &&
380 - rrdhost_flag_check(rrdb.localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) &&
381 - SSL_connection(&rrdb.localhost->sender->ssl) ? "streaming|" : "|");
380 + rrdhost_flag_check(localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) &&
381 + SSL_connection(&localhost->sender->ssl) ? "streaming|" : "|");
382
383 buffer_strcat(b, netdata_ssl_web_server_ctx ? "web" : "");
384 #else
385 buffer_strcat(b, "||");
386 #endif
387
388 - analytics_set_data_str(&analytics_data.netdata_config_https_available, buffer_tostring(b));
388 + analytics_set_data_str(&analytics_data.netdata_config_https_available, (char *)buffer_tostring(b));
389 buffer_free(b);
390 }
391
@@ -394,7 +394,7 @@ void analytics_charts(void)
394 RRDSET *st;
395 size_t c = 0;
396
397 - rrdset_foreach_read(st, rrdb.localhost)
397 + rrdset_foreach_read(st, localhost)
398 if(rrdset_is_available_for_viewers(st)) c++;
399 rrdset_foreach_done(st);
400
@@ -410,7 +410,7 @@ void analytics_metrics(void)
410 {
411 RRDSET *st;
412 size_t dimensions = 0;
413 - rrdset_foreach_read(st, rrdb.localhost) {
413 + rrdset_foreach_read(st, localhost) {
414 if (rrdset_is_available_for_viewers(st)) {
415 RRDDIM *rd;
416 rrddim_foreach_read(rd, st) {
@@ -436,7 +436,7 @@ void analytics_alarms(void)
436 size_t alarm_warn = 0, alarm_crit = 0, alarm_normal = 0;
437 char b[21];
438 RRDCALC *rc;
439 - foreach_rrdcalc_in_rrdhost_read(rrdb.localhost, rc) {
439 + foreach_rrdcalc_in_rrdhost_read(localhost, rc) {
440 if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
441 continue;
442
@@ -650,7 +650,7 @@ static const char *verify_or_create_required_directory(const char *dir) {
650 void set_late_global_environment(struct rrdhost_system_info *system_info)
651 {
652 analytics_set_data(&analytics_data.netdata_config_stream_enabled, default_rrdpush_enabled ? "true" : "false");
653 - analytics_set_data_str(&analytics_data.netdata_config_storage_engine_name, storage_engine_name(default_storage_engine_id));
653 + analytics_set_data_str(&analytics_data.netdata_config_memory_mode, (char *)rrd_memory_mode_name(default_rrd_memory_mode));
654
655 #ifdef DISABLE_CLOUD
656 analytics_set_data(&analytics_data.netdata_host_cloud_enabled, "false");
@@ -663,10 +663,10 @@ void set_late_global_environment(struct rrdhost_system_info *system_info)
663 #ifdef ENABLE_DBENGINE
664 {
665 char b[16];
666 - snprintfz(b, 15, "%d", rrdb.default_rrdeng_page_cache_mb);
666 + snprintfz(b, 15, "%d", default_rrdeng_page_cache_mb);
667 analytics_set_data(&analytics_data.netdata_config_page_cache_size, b);
668
669 - snprintfz(b, 15, "%d", rrdb.default_multidb_disk_quota_mb);
669 + snprintfz(b, 15, "%d", default_multidb_disk_quota_mb);
670 analytics_set_data(&analytics_data.netdata_config_multidb_disk_quota, b);
671 }
672 #endif
@@ -682,12 +682,12 @@ void set_late_global_environment(struct rrdhost_system_info *system_info)
682 else
683 analytics_set_data(&analytics_data.netdata_config_web_enabled, "true");
684
685 - analytics_set_data_str(&analytics_data.netdata_config_release_channel, get_release_channel());
685 + analytics_set_data_str(&analytics_data.netdata_config_release_channel, (char *)get_release_channel());
686
687 {
688 BUFFER *bi = buffer_create(1000, NULL);
689 analytics_build_info(bi);
690 - analytics_set_data_str(&analytics_data.netdata_buildinfo, buffer_tostring(bi));
690 + analytics_set_data_str(&analytics_data.netdata_buildinfo, (char *)buffer_tostring(bi));
691 buffer_free(bi);
692 }
693
@@ -827,7 +827,7 @@ void set_global_environment()
827 {
828 {
829 char b[16];
830 - snprintfz(b, 15, "%d", rrdb.default_update_every);
830 + snprintfz(b, 15, "%d", default_rrd_update_every);
831 setenv("NETDATA_UPDATE_EVERY", b, 1);
832 }
833
@@ -861,7 +861,7 @@ void set_global_environment()
861
862 analytics_data.data_length = 0;
863 analytics_set_data(&analytics_data.netdata_config_stream_enabled, "null");
864 - analytics_set_data(&analytics_data.netdata_config_storage_engine_name, "null");
864 + analytics_set_data(&analytics_data.netdata_config_memory_mode, "null");
865 analytics_set_data(&analytics_data.netdata_config_exporting_enabled, "null");
866 analytics_set_data(&analytics_data.netdata_exporting_connectors, "null");
867 analytics_set_data(&analytics_data.netdata_allmetrics_prometheus_used, "null");
@@ -987,7 +987,7 @@ void send_statistics(const char *action, const char *action_result, const char *
987 action_result,
988 action_data,
989 analytics_data.netdata_config_stream_enabled,
990 - analytics_data.netdata_config_storage_engine_name,
990 + analytics_data.netdata_config_memory_mode,
991 analytics_data.netdata_config_exporting_enabled,
992 analytics_data.netdata_exporting_connectors,
993 analytics_data.netdata_allmetrics_prometheus_used,
daemon/analytics.h
+1 -1
@@ -22,7 +22,7 @@
22
23 struct analytics_data {
24 char *netdata_config_stream_enabled;
25 - char *netdata_config_storage_engine_name;
25 + char *netdata_config_memory_mode;
26 char *netdata_exporting_connectors;
27 char *netdata_config_exporting_enabled;
28 char *netdata_allmetrics_prometheus_used;
daemon/buildinfo.c
+2 -2
@@ -1252,8 +1252,8 @@ static void populate_system_info(void) {
1252 struct rrdhost_system_info *system_info;
1253 bool free_system_info = false;
1254
1255 - if(rrdb.localhost && rrdb.localhost->system_info) {
1256 - system_info = rrdb.localhost->system_info;
1255 + if(localhost && localhost->system_info) {
1256 + system_info = localhost->system_info;
1257 }
1258 else {
1259 system_info = callocz(1, sizeof(struct rrdhost_system_info));
daemon/commands.c
+3 -3
@@ -52,7 +52,7 @@ static cmd_status_t cmd_dumpconfig(char *args, char **message);
52 static command_info_t command_info_array[] = {
53 {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY}, // show help menu
54 {"reload-health", cmd_reload_health_execute, CMD_TYPE_ORTHOGONAL}, // reload health configuration
55 - {"save-database", cmd_save_database_execute, CMD_TYPE_ORTHOGONAL}, // save database for storage engine SAVE
55 + {"save-database", cmd_save_database_execute, CMD_TYPE_ORTHOGONAL}, // save database for memory mode save
56 {"reopen-logs", cmd_reopen_logs_execute, CMD_TYPE_ORTHOGONAL}, // Close and reopen log files
57 {"shutdown-agent", cmd_exit_execute, CMD_TYPE_EXCLUSIVE}, // exit cleanly
58 {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY}, // exit with fatal error
@@ -218,10 +218,10 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
218 {
219 (void)args;
220 netdata_log_info("COMMAND: reloading host labels.");
221 - localhost_load_labels();
221 + reload_host_labels();
222
223 BUFFER *wb = buffer_create(10, NULL);
224 - rrdlabels_log_to_buffer(rrdb.localhost->rrdlabels, wb);
224 + rrdlabels_log_to_buffer(localhost->rrdlabels, wb);
225 (*message)=strdupz(buffer_tostring(wb));
226 buffer_free(wb);
227
daemon/global_statistics.c
+91 -91
@@ -83,7 +83,7 @@ static struct global_statistics {
83 };
84
85 void global_statistics_rrdset_done_chart_collection_completed(size_t *points_read_per_tier_array) {
86 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
86 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
87 __atomic_fetch_add(&global_statistics.db_points_stored_per_tier[tier], points_read_per_tier_array[tier], __ATOMIC_RELAXED);
88 points_read_per_tier_array[tier] = 0;
89 }
@@ -210,7 +210,7 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
210 gs->backfill_queries_made = __atomic_load_n(&global_statistics.backfill_queries_made, __ATOMIC_RELAXED);
211 gs->backfill_db_points_read = __atomic_load_n(&global_statistics.backfill_db_points_read, __ATOMIC_RELAXED);
212
213 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
213 + for(size_t tier = 0; tier < storage_tiers ;tier++)
214 gs->db_points_stored_per_tier[tier] = __atomic_load_n(&global_statistics.db_points_stored_per_tier[tier], __ATOMIC_RELAXED);
215
216 if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) {
@@ -262,7 +262,7 @@ static void global_statistics_charts(void) {
262 , "netdata"
263 , "stats"
264 , 130000
265 - , rrdb.localhost->update_every
265 + , localhost->rrd_update_every
266 , RRDSET_TYPE_STACKED
267 );
268
@@ -308,7 +308,7 @@ static void global_statistics_charts(void) {
308 "netdata",
309 "stats",
310 130100,
311 - rrdb.localhost->update_every,
311 + localhost->rrd_update_every,
312 RRDSET_TYPE_STACKED);
313
314 rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -397,7 +397,7 @@ static void global_statistics_charts(void) {
397 "netdata",
398 "stats",
399 130101,
400 - rrdb.localhost->update_every,
400 + localhost->rrd_update_every,
401 RRDSET_TYPE_STACKED);
402
403 rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -452,7 +452,7 @@ static void global_statistics_charts(void) {
452 "netdata",
453 "stats",
454 130150,
455 - rrdb.localhost->update_every,
455 + localhost->rrd_update_every,
456 RRDSET_TYPE_LINE);
457
458 rd_uptime = rrddim_add(st_uptime, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -480,7 +480,7 @@ static void global_statistics_charts(void) {
480 , "netdata"
481 , "stats"
482 , 130200
483 - , rrdb.localhost->update_every
483 + , localhost->rrd_update_every
484 , RRDSET_TYPE_LINE
485 );
486
@@ -509,7 +509,7 @@ static void global_statistics_charts(void) {
509 , "netdata"
510 , "stats"
511 , 130300
512 - , rrdb.localhost->update_every
512 + , localhost->rrd_update_every
513 , RRDSET_TYPE_LINE
514 );
515
@@ -539,7 +539,7 @@ static void global_statistics_charts(void) {
539 , "netdata"
540 , "stats"
541 , 130400
542 - , rrdb.localhost->update_every
542 + , localhost->rrd_update_every
543 , RRDSET_TYPE_AREA
544 );
545
@@ -571,7 +571,7 @@ static void global_statistics_charts(void) {
571 , "netdata"
572 , "stats"
573 , 130500
574 - , rrdb.localhost->update_every
574 + , localhost->rrd_update_every
575 , RRDSET_TYPE_LINE
576 );
577
@@ -618,7 +618,7 @@ static void global_statistics_charts(void) {
618 , "netdata"
619 , "stats"
620 , 130600
621 - , rrdb.localhost->update_every
621 + , localhost->rrd_update_every
622 , RRDSET_TYPE_LINE
623 );
624
@@ -670,7 +670,7 @@ static void global_statistics_charts(void) {
670 , "netdata"
671 , "stats"
672 , 131000
673 - , rrdb.localhost->update_every
673 + , localhost->rrd_update_every
674 , RRDSET_TYPE_STACKED
675 );
676
@@ -721,7 +721,7 @@ static void global_statistics_charts(void) {
721 , "netdata"
722 , "stats"
723 , 131001
724 - , rrdb.localhost->update_every
724 + , localhost->rrd_update_every
725 , RRDSET_TYPE_STACKED
726 );
727
@@ -770,7 +770,7 @@ static void global_statistics_charts(void) {
770 , "netdata"
771 , "stats"
772 , 131002
773 - , rrdb.localhost->update_every
773 + , localhost->rrd_update_every
774 , RRDSET_TYPE_STACKED
775 );
776
@@ -810,18 +810,18 @@ static void global_statistics_charts(void) {
810 , "netdata"
811 , "stats"
812 , 131003
813 - , rrdb.localhost->update_every
813 + , localhost->rrd_update_every
814 , RRDSET_TYPE_STACKED
815 );
816
817 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
817 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
818 char buf[30 + 1];
819 snprintfz(buf, 30, "tier%zu", tier);
820 rds[tier] = rrddim_add(st_points_stored, buf, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
821 }
822 }
823
824 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
824 + for(size_t tier = 0; tier < storage_tiers ;tier++)
825 rrddim_set_by_pointer(st_points_stored, rds[tier], (collected_number)gs.db_points_stored_per_tier[tier]);
826
827 rrdset_done(st_points_stored);
@@ -882,7 +882,7 @@ static inline void sqlite3_statistics_copy(struct sqlite3_statistics *gs) {
882 gs->sqlite3_queries_failed_locked = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
883 gs->sqlite3_rows = __atomic_load_n(&sqlite3_statistics.sqlite3_rows, __ATOMIC_RELAXED);
884
885 - usec_t timeout = rrdb.default_update_every * USEC_PER_SEC + rrdb.default_update_every * USEC_PER_SEC / 3;
885 + usec_t timeout = default_rrd_update_every * USEC_PER_SEC + default_rrd_update_every * USEC_PER_SEC / 3;
886 usec_t now = now_monotonic_usec();
887 if(!last_run)
888 last_run = now;
@@ -968,7 +968,7 @@ static void sqlite3_statistics_charts(void) {
968 , "netdata"
969 , "stats"
970 , 131100
971 - , rrdb.localhost->update_every
971 + , localhost->rrd_update_every
972 , RRDSET_TYPE_LINE
973 );
974
@@ -998,7 +998,7 @@ static void sqlite3_statistics_charts(void) {
998 , "netdata"
999 , "stats"
1000 , 131101
1001 - , rrdb.localhost->update_every
1001 + , localhost->rrd_update_every
1002 , RRDSET_TYPE_LINE
1003 );
1004
@@ -1034,7 +1034,7 @@ static void sqlite3_statistics_charts(void) {
1034 , "netdata"
1035 , "stats"
1036 , 131102
1037 - , rrdb.localhost->update_every
1037 + , localhost->rrd_update_every
1038 , RRDSET_TYPE_LINE
1039 );
1040
@@ -1065,7 +1065,7 @@ static void sqlite3_statistics_charts(void) {
1065 , "netdata"
1066 , "stats"
1067 , 131103
1068 - , rrdb.localhost->update_every
1068 + , localhost->rrd_update_every
1069 , RRDSET_TYPE_LINE
1070 );
1071
@@ -1109,7 +1109,7 @@ static void sqlite3_statistics_charts(void) {
1109 , "netdata"
1110 , "stats"
1111 , 131104
1112 - , rrdb.localhost->update_every
1112 + , localhost->rrd_update_every
1113 , RRDSET_TYPE_LINE
1114 );
1115
@@ -1237,7 +1237,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1237 "netdata",
1238 "stats",
1239 priority,
1240 - rrdb.localhost->update_every,
1240 + localhost->rrd_update_every,
1241 RRDSET_TYPE_LINE);
1242
1243 ptrs->rd_hit_ratio_closest = rrddim_add(ptrs->st_cache_hit_ratio, "closest", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
@@ -1285,7 +1285,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1285 "netdata",
1286 "stats",
1287 priority,
1288 - rrdb.localhost->update_every,
1288 + localhost->rrd_update_every,
1289 RRDSET_TYPE_LINE);
1290
1291 ptrs->rd_searches_closest = rrddim_add(ptrs->st_operations, "search closest", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1339,7 +1339,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1339 "netdata",
1340 "stats",
1341 priority,
1342 - rrdb.localhost->update_every,
1342 + localhost->rrd_update_every,
1343 RRDSET_TYPE_STACKED);
1344
1345 ptrs->rd_pgc_memory_free = rrddim_add(ptrs->st_pgc_memory, "free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1393,7 +1393,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1393 "netdata",
1394 "stats",
1395 priority,
1396 - rrdb.localhost->update_every,
1396 + localhost->rrd_update_every,
1397 RRDSET_TYPE_LINE);
1398
1399 ptrs->rd_pgc_tm_current = rrddim_add(ptrs->st_pgc_tm, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1443,7 +1443,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1443 "netdata",
1444 "stats",
1445 priority,
1446 - rrdb.localhost->update_every,
1446 + localhost->rrd_update_every,
1447 RRDSET_TYPE_LINE);
1448
1449 ptrs->rd_pgc_pages_clean = rrddim_add(ptrs->st_pgc_pages, "clean", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1487,7 +1487,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1487 "netdata",
1488 "stats",
1489 priority,
1490 - rrdb.localhost->update_every,
1490 + localhost->rrd_update_every,
1491 RRDSET_TYPE_AREA);
1492
1493 ptrs->rd_pgc_memory_new_clean = rrddim_add(ptrs->st_pgc_memory_changes, "new clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1529,7 +1529,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1529 "netdata",
1530 "stats",
1531 priority,
1532 - rrdb.localhost->update_every,
1532 + localhost->rrd_update_every,
1533 RRDSET_TYPE_AREA);
1534
1535 ptrs->rd_pgc_memory_dirty_to_clean = rrddim_add(ptrs->st_pgc_memory_migrations, "dirty to clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1569,7 +1569,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1569 "netdata",
1570 "stats",
1571 priority,
1572 - rrdb.localhost->update_every,
1572 + localhost->rrd_update_every,
1573 RRDSET_TYPE_AREA);
1574
1575 ptrs->rd_pgc_memory_evictions_aggressive = rrddim_add(ptrs->st_pgc_memory_events, "evictions aggressive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1611,7 +1611,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1611 "netdata",
1612 "stats",
1613 priority,
1614 - rrdb.localhost->update_every,
1614 + localhost->rrd_update_every,
1615 RRDSET_TYPE_LINE);
1616
1617 ptrs->rd_pgc_waste_evictions_skipped = rrddim_add(ptrs->st_pgc_waste, "evictions skipped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1663,7 +1663,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
1663 "netdata",
1664 "stats",
1665 priority,
1666 - rrdb.localhost->update_every,
1666 + localhost->rrd_update_every,
1667 RRDSET_TYPE_LINE);
1668
1669 ptrs->rd_pgc_workers_searchers = rrddim_add(ptrs->st_pgc_workers, "searchers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1752,7 +1752,7 @@ static void dbengine2_statistics_charts(void) {
1752 "netdata",
1753 "stats",
1754 priority,
1755 - rrdb.localhost->update_every,
1755 + localhost->rrd_update_every,
1756 RRDSET_TYPE_STACKED);
1757
1758 rd_pgc_memory_main = rrddim_add(st_pgc_memory, "main cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1804,7 +1804,7 @@ static void dbengine2_statistics_charts(void) {
1804 "netdata",
1805 "stats",
1806 priority,
1807 - rrdb.localhost->update_every,
1807 + localhost->rrd_update_every,
1808 RRDSET_TYPE_STACKED);
1809
1810 rd_pgc_buffers_pgc = rrddim_add(st_pgc_buffers, "pgc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1863,7 +1863,7 @@ static void dbengine2_statistics_charts(void) {
1863 "netdata",
1864 "stats",
1865 priority,
1866 - rrdb.localhost->update_every,
1866 + localhost->rrd_update_every,
1867 RRDSET_TYPE_AREA);
1868
1869 rd_julyl_moved = rrddim_add(st_julyl_moved, "moved", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1897,7 +1897,7 @@ static void dbengine2_statistics_charts(void) {
1897 "netdata",
1898 "stats",
1899 priority,
1900 - rrdb.localhost->update_every,
1900 + localhost->rrd_update_every,
1901 RRDSET_TYPE_LINE);
1902
1903 rd_mrg_metrics = rrddim_add(st_mrg_metrics, "all", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1937,7 +1937,7 @@ static void dbengine2_statistics_charts(void) {
1937 "netdata",
1938 "stats",
1939 priority,
1940 - rrdb.localhost->update_every,
1940 + localhost->rrd_update_every,
1941 RRDSET_TYPE_LINE);
1942
1943 rd_mrg_add = rrddim_add(st_mrg_ops, "add", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1969,7 +1969,7 @@ static void dbengine2_statistics_charts(void) {
1969 "netdata",
1970 "stats",
1971 priority,
1972 - rrdb.localhost->update_every,
1972 + localhost->rrd_update_every,
1973 RRDSET_TYPE_LINE);
1974
1975 rd_mrg_references = rrddim_add(st_mrg_references, "references", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2000,7 +2000,7 @@ static void dbengine2_statistics_charts(void) {
2000 "netdata",
2001 "stats",
2002 priority,
2003 - rrdb.localhost->update_every,
2003 + localhost->rrd_update_every,
2004 RRDSET_TYPE_LINE);
2005
2006 rd_hit_ratio = rrddim_add(st_cache_hit_ratio, "overall", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
@@ -2077,7 +2077,7 @@ static void dbengine2_statistics_charts(void) {
2077 "netdata",
2078 "stats",
2079 priority,
2080 - rrdb.localhost->update_every,
2080 + localhost->rrd_update_every,
2081 RRDSET_TYPE_LINE);
2082
2083 rd_total = rrddim_add(st_queries, "total", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2113,7 +2113,7 @@ static void dbengine2_statistics_charts(void) {
2113 "netdata",
2114 "stats",
2115 priority,
2116 - rrdb.localhost->update_every,
2116 + localhost->rrd_update_every,
2117 RRDSET_TYPE_LINE);
2118
2119 rd_queries = rrddim_add(st_queries_running, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2143,7 +2143,7 @@ static void dbengine2_statistics_charts(void) {
2143 "netdata",
2144 "stats",
2145 priority,
2146 - rrdb.localhost->update_every,
2146 + localhost->rrd_update_every,
2147 RRDSET_TYPE_STACKED);
2148
2149 rd_cache = rrddim_add(st_query_pages_metadata_source, "cache hit", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2177,7 +2177,7 @@ static void dbengine2_statistics_charts(void) {
2177 "netdata",
2178 "stats",
2179 priority,
2180 - rrdb.localhost->update_every,
2180 + localhost->rrd_update_every,
2181 RRDSET_TYPE_STACKED);
2182
2183 rd_pages_main_cache = rrddim_add(st_query_pages_data_source, "main cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2213,7 +2213,7 @@ static void dbengine2_statistics_charts(void) {
2213 "netdata",
2214 "stats",
2215 priority,
2216 - rrdb.localhost->update_every,
2216 + localhost->rrd_update_every,
2217 RRDSET_TYPE_STACKED);
2218
2219 rd_pass4 = rrddim_add(st_query_next_page, "pass4", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2254,7 +2254,7 @@ static void dbengine2_statistics_charts(void) {
2254 "netdata",
2255 "stats",
2256 priority,
2257 - rrdb.localhost->update_every,
2257 + localhost->rrd_update_every,
2258 RRDSET_TYPE_STACKED);
2259
2260 rd_pages_zero_time = rrddim_add(st_query_page_issues, "zero timestamp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2301,7 +2301,7 @@ static void dbengine2_statistics_charts(void) {
2301 "netdata",
2302 "stats",
2303 priority,
2304 - rrdb.localhost->update_every,
2304 + localhost->rrd_update_every,
2305 RRDSET_TYPE_LINE);
2306
2307 rd_compressed = rrddim_add(st_query_pages_from_disk, "ok compressed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2353,7 +2353,7 @@ static void dbengine2_statistics_charts(void) {
2353 "netdata",
2354 "stats",
2355 priority,
2356 - rrdb.localhost->update_every,
2356 + localhost->rrd_update_every,
2357 RRDSET_TYPE_LINE);
2358
2359 rd_journal_v2_mapped = rrddim_add(st_events, "journal v2 mapped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2397,7 +2397,7 @@ static void dbengine2_statistics_charts(void) {
2397 "netdata",
2398 "stats",
2399 priority,
2400 - rrdb.localhost->update_every,
2400 + localhost->rrd_update_every,
2401 RRDSET_TYPE_STACKED);
2402
2403 rd_routing = rrddim_add(st_prep_timings, "routing", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2438,7 +2438,7 @@ static void dbengine2_statistics_charts(void) {
2438 "netdata",
2439 "stats",
2440 priority,
2441 - rrdb.localhost->update_every,
2441 + localhost->rrd_update_every,
2442 RRDSET_TYPE_STACKED);
2443
2444 rd_init = rrddim_add(st_query_timings, "init", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2460,7 +2460,7 @@ static void dbengine2_statistics_charts(void) {
2460 rrdset_done(st_query_timings);
2461 }
2462
2463 - if(rrd_tryrdlock() == 0) {
2463 + if(netdata_rwlock_tryrdlock(&rrd_rwlock) == 0) {
2464 priority = 135400;
2465
2466 RRDHOST *host;
@@ -2472,8 +2472,8 @@ static void dbengine2_statistics_charts(void) {
2472 if (!rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
2473
2474 /* get localhost's DB engine's statistics for each tier */
2475 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
2476 - if(host->db[tier].id != STORAGE_ENGINE_DBENGINE) continue;
2475 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
2476 + if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue;
2477 if(!host->db[tier].instance) continue;
2478
2479 if(is_storage_engine_shared(host->db[tier].instance)) {
@@ -2484,7 +2484,7 @@ static void dbengine2_statistics_charts(void) {
2484 }
2485
2486 ++dbengine_contexts;
2487 - rrdeng_get_37_statistics(host->db[tier].instance, local_stats_array);
2487 + rrdeng_get_37_statistics((struct rrdengine_instance *)host->db[tier].instance, local_stats_array);
2488 for (i = 0; i < RRDENG_NR_STATS; ++i) {
2489 /* aggregate statistics across hosts */
2490 stats_array[i] += local_stats_array[i];
@@ -2520,7 +2520,7 @@ static void dbengine2_statistics_charts(void) {
2520 "netdata",
2521 "stats",
2522 priority,
2523 - rrdb.localhost->update_every,
2523 + localhost->rrd_update_every,
2524 RRDSET_TYPE_LINE);
2525
2526 rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
@@ -2561,7 +2561,7 @@ static void dbengine2_statistics_charts(void) {
2561 "netdata",
2562 "stats",
2563 priority,
2564 - rrdb.localhost->update_every,
2564 + localhost->rrd_update_every,
2565 RRDSET_TYPE_LINE);
2566
2567 rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -2593,7 +2593,7 @@ static void dbengine2_statistics_charts(void) {
2593 "netdata",
2594 "stats",
2595 priority,
2596 - rrdb.localhost->update_every,
2596 + localhost->rrd_update_every,
2597 RRDSET_TYPE_LINE);
2598
2599 rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2626,7 +2626,7 @@ static void dbengine2_statistics_charts(void) {
2626 "netdata",
2627 "stats",
2628 priority,
2629 - rrdb.localhost->update_every,
2629 + localhost->rrd_update_every,
2630 RRDSET_TYPE_LINE);
2631
2632 rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2661,7 +2661,7 @@ static void dbengine2_statistics_charts(void) {
2661 "netdata",
2662 "stats",
2663 priority,
2664 - rrdb.localhost->update_every,
2664 + localhost->rrd_update_every,
2665 RRDSET_TYPE_LINE);
2666
2667 rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2701,7 +2701,7 @@ static void update_strings_charts() {
2701 , "netdata"
2702 , "stats"
2703 , 910000
2704 - , rrdb.localhost->update_every
2704 + , localhost->rrd_update_every
2705 , RRDSET_TYPE_LINE);
2706
2707 rd_ops_inserts = rrddim_add(st_ops, "inserts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -2730,7 +2730,7 @@ static void update_strings_charts() {
2730 , "netdata"
2731 , "stats"
2732 , 910001
2733 - , rrdb.localhost->update_every
2733 + , localhost->rrd_update_every
2734 , RRDSET_TYPE_AREA);
2735
2736 rd_entries_entries = rrddim_add(st_entries, "entries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2753,7 +2753,7 @@ static void update_strings_charts() {
2753 , "netdata"
2754 , "stats"
2755 , 910001
2756 - , rrdb.localhost->update_every
2756 + , localhost->rrd_update_every
2757 , RRDSET_TYPE_AREA);
2758
2759 rd_mem = rrddim_add(st_mem, "memory", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2781,7 +2781,7 @@ static void update_heartbeat_charts() {
2781 , "netdata"
2782 , "stats"
2783 , 900000
2784 - , rrdb.localhost->update_every
2784 + , localhost->rrd_update_every
2785 , RRDSET_TYPE_AREA);
2786
2787 rd_heartbeat_min = rrddim_add(st_heartbeat, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -2903,7 +2903,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
2903 , "netdata"
2904 , "stats"
2905 , c->priority + 0
2906 - , rrdb.localhost->update_every
2906 + , localhost->rrd_update_every
2907 , RRDSET_TYPE_LINE
2908 );
2909
@@ -2944,7 +2944,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
2944 , "netdata"
2945 , "stats"
2946 , c->priority + 1
2947 - , rrdb.localhost->update_every
2947 + , localhost->rrd_update_every
2948 , RRDSET_TYPE_LINE
2949 );
2950
@@ -2994,7 +2994,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
2994 , "netdata"
2995 , "stats"
2996 , c->priority + 2
2997 - , rrdb.localhost->update_every
2997 + , localhost->rrd_update_every
2998 , RRDSET_TYPE_LINE
2999 );
3000
@@ -3053,7 +3053,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
3053 , "netdata"
3054 , "stats"
3055 , c->priority + 3
3056 - , rrdb.localhost->update_every
3056 + , localhost->rrd_update_every
3057 , RRDSET_TYPE_LINE
3058 );
3059
@@ -3099,7 +3099,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
3099 , "netdata"
3100 , "stats"
3101 , c->priority + 4
3102 - , rrdb.localhost->update_every
3102 + , localhost->rrd_update_every
3103 , RRDSET_TYPE_STACKED
3104 );
3105
@@ -3144,7 +3144,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
3144 , "netdata"
3145 , "stats"
3146 , c->priority + 5
3147 - , rrdb.localhost->update_every
3147 + , localhost->rrd_update_every
3148 , RRDSET_TYPE_LINE
3149 );
3150
@@ -3239,7 +3239,7 @@ static void malloc_trace_statistics(void) {
3239 , "netdata"
3240 , "stats"
3241 , 900000
3242 - , rrdb.localhost->update_every
3242 + , localhost->rrd_update_every
3243 , RRDSET_TYPE_STACKED
3244 );
3245 }
@@ -3256,7 +3256,7 @@ static void malloc_trace_statistics(void) {
3256 , "netdata"
3257 , "stats"
3258 , 900001
3259 - , rrdb.localhost->update_every
3259 + , localhost->rrd_update_every
3260 , RRDSET_TYPE_LINE
3261 );
3262 }
@@ -3273,7 +3273,7 @@ static void malloc_trace_statistics(void) {
3273 , "netdata"
3274 , "stats"
3275 , 900002
3276 - , rrdb.localhost->update_every
3276 + , localhost->rrd_update_every
3277 , RRDSET_TYPE_STACKED
3278 );
3279 }
@@ -3290,7 +3290,7 @@ static void malloc_trace_statistics(void) {
3290 , "netdata"
3291 , "stats"
3292 , 900003
3293 - , rrdb.localhost->update_every
3293 + , localhost->rrd_update_every
3294 , RRDSET_TYPE_LINE
3295 );
3296 }
@@ -3462,7 +3462,7 @@ static void workers_total_cpu_utilization_chart(void) {
3462 "netdata",
3463 "stats",
3464 999000,
3465 - rrdb.localhost->update_every,
3465 + localhost->rrd_update_every,
3466 RRDSET_TYPE_STACKED);
3467 }
3468
@@ -3510,7 +3510,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3510 , "netdata"
3511 , "stats"
3512 , wu->priority
3513 - , rrdb.localhost->update_every
3513 + , localhost->rrd_update_every
3514 , RRDSET_TYPE_AREA
3515 );
3516 }
@@ -3563,7 +3563,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3563 , "netdata"
3564 , "stats"
3565 , wu->priority + 1
3566 - , rrdb.localhost->update_every
3566 + , localhost->rrd_update_every
3567 , RRDSET_TYPE_AREA
3568 );
3569 }
@@ -3614,7 +3614,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3614 , "netdata"
3615 , "stats"
3616 , wu->priority + 2
3617 - , rrdb.localhost->update_every
3617 + , localhost->rrd_update_every
3618 , RRDSET_TYPE_STACKED
3619 );
3620 }
@@ -3657,7 +3657,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3657 , "netdata"
3658 , "stats"
3659 , wu->priority + 3
3660 - , rrdb.localhost->update_every
3660 + , localhost->rrd_update_every
3661 , RRDSET_TYPE_STACKED
3662 );
3663 }
@@ -3701,7 +3701,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3701 , "netdata"
3702 , "stats"
3703 , wu->priority + 4
3704 - , rrdb.localhost->update_every
3704 + , localhost->rrd_update_every
3705 , RRDSET_TYPE_STACKED
3706 );
3707
@@ -3753,7 +3753,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3753 , "netdata"
3754 , "stats"
3755 , wu->priority + 5 + i
3756 - , rrdb.localhost->update_every
3756 + , localhost->rrd_update_every
3757 , RRDSET_TYPE_LINE
3758 );
3759
@@ -3809,7 +3809,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
3809 , "netdata"
3810 , "stats"
3811 , wu->priority + 5 + i
3812 - , rrdb.localhost->update_every
3812 + , localhost->rrd_update_every
3813 , RRDSET_TYPE_LINE
3814 );
3815
@@ -4136,9 +4136,9 @@ void *global_statistics_main(void *ptr)
4136 netdata_thread_cleanup_push(global_statistics_cleanup, ptr);
4137
4138 int update_every =
4139 - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", rrdb.localhost->update_every);
4140 - if (update_every < rrdb.localhost->update_every)
4141 - update_every = rrdb.localhost->update_every;
4139 + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every);
4140 + if (update_every < localhost->rrd_update_every)
4141 + update_every = localhost->rrd_update_every;
4142
4143 usec_t step = update_every * USEC_PER_SEC;
4144 heartbeat_t hb;
@@ -4159,7 +4159,7 @@ void *global_statistics_main(void *ptr)
4159 registry_statistics();
4160
4161 #ifdef ENABLE_DBENGINE
4162 - if(rrdb.dbengine_enabled) {
4162 + if(dbengine_enabled) {
4163 worker_is_busy(WORKER_JOB_DBENGINE);
4164 dbengine2_statistics_charts();
4165 }
@@ -4209,9 +4209,9 @@ void *global_statistics_workers_main(void *ptr)
4209 netdata_thread_cleanup_push(global_statistics_workers_cleanup, ptr);
4210
4211 int update_every =
4212 - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", rrdb.localhost->update_every);
4213 - if (update_every < rrdb.localhost->update_every)
4214 - update_every = rrdb.localhost->update_every;
4212 + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every);
4213 + if (update_every < localhost->rrd_update_every)
4214 + update_every = localhost->rrd_update_every;
4215
4216 usec_t step = update_every * USEC_PER_SEC;
4217 heartbeat_t hb;
@@ -4251,9 +4251,9 @@ void *global_statistics_sqlite3_main(void *ptr)
4251 netdata_thread_cleanup_push(global_statistics_sqlite3_cleanup, ptr);
4252
4253 int update_every =
4254 - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", rrdb.localhost->update_every);
4255 - if (update_every < rrdb.localhost->update_every)
4256 - update_every = rrdb.localhost->update_every;
4254 + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every);
4255 + if (update_every < localhost->rrd_update_every)
4256 + update_every = localhost->rrd_update_every;
4257
4258 usec_t step = update_every * USEC_PER_SEC;
4259 heartbeat_t hb;
daemon/main.c
+97 -233
@@ -8,9 +8,12 @@
8 #warning COMPILING 32BIT NETDATA
9 #endif
10
11 +bool unittest_running = false;
12 int netdata_zero_metrics_enabled;
13 int netdata_anonymous_statistics_enabled;
14
15 +int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
16 +bool ieee754_doubles = false;
17 time_t netdata_start_time = 0;
18 struct netdata_static_thread *static_threads;
19
@@ -324,10 +327,10 @@ void netdata_cleanup_and_exit(int ret) {
327 (void) rename(agent_crash_file, agent_incomplete_shutdown_file);
328
329 #ifdef ENABLE_DBENGINE
327 - if (rrdb.dbengine_enabled) {
330 + if(dbengine_enabled) {
331 delta_shutdown_time("dbengine exit mode");
329 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
330 - rrdeng_exit_mode(rrdb.multidb_ctx[tier]);
332 + for (size_t tier = 0; tier < storage_tiers; tier++)
333 + rrdeng_exit_mode(multidb_ctx[tier]);
334 }
335 #endif
336
@@ -415,10 +418,10 @@ void netdata_cleanup_and_exit(int ret) {
418 // exit cleanly
419
420 #ifdef ENABLE_DBENGINE
418 - if (rrdb.dbengine_enabled) {
421 + if(dbengine_enabled) {
422 delta_shutdown_time("flush dbengine tiers");
420 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
421 - rrdeng_prepare_exit(rrdb.multidb_ctx[tier]);
423 + for (size_t tier = 0; tier < storage_tiers; tier++)
424 + rrdeng_prepare_exit(multidb_ctx[tier]);
425 }
426 #endif
427
@@ -433,14 +436,14 @@ void netdata_cleanup_and_exit(int ret) {
436 metadata_sync_shutdown();
437
438 #ifdef ENABLE_DBENGINE
436 - if (rrdb.dbengine_enabled) {
439 + if(dbengine_enabled) {
440 delta_shutdown_time("wait for dbengine collectors to finish");
441
442 size_t running = 1;
443 while(running) {
444 running = 0;
442 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
443 - running += rrdeng_collectors_running(rrdb.multidb_ctx[tier]);
445 + for (size_t tier = 0; tier < storage_tiers; tier++)
446 + running += rrdeng_collectors_running(multidb_ctx[tier]);
447
448 if(running) {
449 error_limit_static_thread_var(erl, 1, 100 * USEC_PER_MS);
@@ -457,8 +460,8 @@ void netdata_cleanup_and_exit(int ret) {
460 }
461
462 delta_shutdown_time("stop dbengine tiers");
460 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
461 - rrdeng_exit(rrdb.multidb_ctx[tier]);
463 + for (size_t tier = 0; tier < storage_tiers; tier++)
464 + rrdeng_exit(multidb_ctx[tier]);
465 }
466 #endif
467 }
@@ -657,26 +660,6 @@ static void set_nofile_limit(struct rlimit *rl) {
660 netdata_log_error("Number of open file descriptors allowed for this process is too low (RLIMIT_NOFILE=%zu)", (size_t)rl->rlim_cur);
661 }
662
660 -static void set_libuv_worker_threads()
661 -{
662 -#if defined(ENV32BIT)
663 - const int min_libuv_worker_threads = 8;
664 - const int max_libuv_worker_threads = 128;
665 -#else
666 - const int min_libuv_worker_threads = 16;
667 - const int max_libuv_worker_threads = 1024;
668 -#endif
669 -
670 - int n = (int) get_netdata_cpus() * 6;
671 - n = CLAMP(n, min_libuv_worker_threads, max_libuv_worker_threads);
672 -
673 - n = config_get_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", n);
674 - n = MAX(n, min_libuv_worker_threads);
675 - config_set_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", n);
676 -
677 - rrdb.libuv_worker_threads = n;
678 -}
679 -
663 void cancel_main_threads() {
664 error_log_limit_unlimited();
665
@@ -1082,37 +1065,37 @@ static void get_netdata_configured_variables() {
1065 // ------------------------------------------------------------------------
1066 // get default database update frequency
1067
1085 - rrdb.default_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", rrdb.default_update_every);
1086 - if(rrdb.default_update_every < UPDATE_EVERY_MIN || rrdb.default_update_every > UPDATE_EVERY_MAX) {
1087 - netdata_log_error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", rrdb.default_update_every, UPDATE_EVERY_MIN);
1088 - rrdb.default_update_every = UPDATE_EVERY_MIN;
1089 - config_set_number(CONFIG_SECTION_DB, "update every", rrdb.default_update_every);
1068 + default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY);
1069 + if(default_rrd_update_every < 1 || default_rrd_update_every > 600) {
1070 + netdata_log_error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY);
1071 + default_rrd_update_every = UPDATE_EVERY;
1072 + config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every);
1073 }
1074
1075 // ------------------------------------------------------------------------
1093 - // get default storage engine for the database
1076 + // get default memory mode for the database
1077
1078 {
1096 - const char *se_name = config_get(CONFIG_SECTION_DB, "mode", storage_engine_name(default_storage_engine_id));
1097 - if (!storage_engine_id(se_name, &default_storage_engine_id)) {
1098 - netdata_log_error("Invalid memory mode '%s' given. Using '%s'",
1099 - se_name, storage_engine_name(default_storage_engine_id));
1100 - config_set(CONFIG_SECTION_DB, "mode", storage_engine_name(default_storage_engine_id));
1079 + const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
1080 + default_rrd_memory_mode = rrd_memory_mode_id(mode);
1081 + if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) {
1082 + netdata_log_error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode));
1083 + config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
1084 }
1085 }
1086
1087 // ------------------------------------------------------------------------
1088 // get default database size
1089
1107 - if(default_storage_engine_id != STORAGE_ENGINE_DBENGINE && default_storage_engine_id != STORAGE_ENGINE_NONE) {
1108 - rrdb.default_rrd_history_entries = (int)config_get_number(
1090 + if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) {
1091 + default_rrd_history_entries = (int)config_get_number(
1092 CONFIG_SECTION_DB, "retention",
1110 - align_entries_to_pagesize(default_storage_engine_id, RRD_DEFAULT_HISTORY_ENTRIES));
1093 + align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));
1094
1112 - long h = align_entries_to_pagesize(default_storage_engine_id, rrdb.default_rrd_history_entries);
1113 - if (h != rrdb.default_rrd_history_entries) {
1095 + long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries);
1096 + if (h != default_rrd_history_entries) {
1097 config_set_number(CONFIG_SECTION_DB, "retention", h);
1115 - rrdb.default_rrd_history_entries = (int)h;
1098 + default_rrd_history_entries = (int)h;
1099 }
1100 }
1101
@@ -1139,39 +1122,39 @@ static void get_netdata_configured_variables() {
1122 // ------------------------------------------------------------------------
1123 // get default Database Engine page cache size in MiB
1124
1142 - rrdb.default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", rrdb.default_rrdeng_page_cache_mb);
1143 - rrdb.default_rrdeng_extent_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine extent cache size MB", rrdb.default_rrdeng_extent_cache_mb);
1144 - rrdb.db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO);
1125 + default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
1126 + default_rrdeng_extent_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine extent cache size MB", default_rrdeng_extent_cache_mb);
1127 + db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO);
1128
1146 - if(rrdb.default_rrdeng_extent_cache_mb < 0)
1147 - rrdb.default_rrdeng_extent_cache_mb = 0;
1129 + if(default_rrdeng_extent_cache_mb < 0)
1130 + default_rrdeng_extent_cache_mb = 0;
1131
1149 - if(rrdb.default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
1150 - netdata_log_error("Invalid page cache size %d given. Defaulting to %d.", rrdb.default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
1151 - rrdb.default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
1152 - config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", rrdb.default_rrdeng_page_cache_mb);
1132 + if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
1133 + netdata_log_error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
1134 + default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
1135 + config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
1136 }
1137
1138 // ------------------------------------------------------------------------
1139 // get default Database Engine disk space quota in MiB
1140
1158 - rrdb.default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", rrdb.default_rrdeng_disk_quota_mb);
1159 - if (rrdb.default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1160 - netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", rrdb.default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
1161 - rrdb.default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
1162 - config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", rrdb.default_rrdeng_disk_quota_mb);
1141 + default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
1142 + if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1143 + netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
1144 + default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
1145 + config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
1146 }
1147
1165 - rrdb.default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
1166 - if(rrdb.default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1167 - netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", rrdb.default_multidb_disk_quota_mb, rrdb.default_rrdeng_disk_quota_mb);
1168 - rrdb.default_multidb_disk_quota_mb = rrdb.default_rrdeng_disk_quota_mb;
1169 - config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", rrdb.default_multidb_disk_quota_mb);
1148 + default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
1149 + if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
1150 + netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
1151 + default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
1152 + config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);
1153 }
1154 #else
1172 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE) {
1155 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1156 error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");
1174 - default_storage_engine_id = STORAGE_ENGINE_SAVE;
1157 + default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
1158 }
1159 #endif
1160 // ------------------------------------------------------------------------
@@ -1196,22 +1179,22 @@ static void get_netdata_configured_variables() {
1179
1180 // --------------------------------------------------------------------
1181
1199 - rrdb.rrdset_free_obsolete_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdb.rrdset_free_obsolete_time_s);
1182 + rrdset_free_obsolete_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);
1183 // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
1184 // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
1185 // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
1203 - if (rrdb.rrdset_free_obsolete_time_s < 10) {
1204 - rrdb.rrdset_free_obsolete_time_s = 10;
1186 + if (rrdset_free_obsolete_time_s < 10) {
1187 + rrdset_free_obsolete_time_s = 10;
1188 netdata_log_info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
1206 - config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdb.rrdset_free_obsolete_time_s);
1189 + config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);
1190 }
1191
1209 - rrdb.gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", rrdb.gap_when_lost_iterations_above);
1210 - if (rrdb.gap_when_lost_iterations_above < 1) {
1211 - rrdb.gap_when_lost_iterations_above = 1;
1212 - config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", rrdb.gap_when_lost_iterations_above);
1192 + gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
1193 + if (gap_when_lost_iterations_above < 1) {
1194 + gap_when_lost_iterations_above = 1;
1195 + config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
1196 }
1214 - rrdb.gap_when_lost_iterations_above += 2;
1197 + gap_when_lost_iterations_above += 2;
1198
1199 // --------------------------------------------------------------------
1200 // get various system parameters
@@ -1322,138 +1305,6 @@ int get_system_info(struct rrdhost_system_info *system_info, bool log) {
1305 return 0;
1306 }
1307
1325 -static void localhost_load_config_labels(void) {
1326 - int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
1327 - if(!status) {
1328 - char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
1329 - netdata_log_error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
1330 - }
1331 -
1332 - struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
1333 - if(co) {
1334 - config_section_wrlock(co);
1335 - struct config_option *cv;
1336 - for(cv = co->values; cv ; cv = cv->next) {
1337 - rrdlabels_add(rrdb.localhost->rrdlabels, cv->name, cv->value, RRDLABEL_SRC_CONFIG);
1338 - cv->flags |= CONFIG_VALUE_USED;
1339 - }
1340 - config_section_unlock(co);
1341 - }
1342 -}
1343 -
1344 -static void localhost_load_kubernetes_labels(void) {
1345 - char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
1346 - sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
1347 -
1348 - if (unlikely(access(label_script, R_OK) != 0)) {
1349 - netdata_log_error("Kubernetes pod label fetching script %s not found.",label_script);
1350 - return;
1351 - }
1352 -
1353 - netdata_log_debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
1354 -
1355 - pid_t pid;
1356 - FILE *fp_child_input;
1357 - FILE *fp_child_output = netdata_popen(label_script, &pid, &fp_child_input);
1358 - if(!fp_child_output) return;
1359 -
1360 - char buffer[1000 + 1];
1361 - while (fgets(buffer, 1000, fp_child_output) != NULL)
1362 - rrdlabels_add_pair(rrdb.localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);
1363 -
1364 - // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
1365 - // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
1366 - int rc = netdata_pclose(fp_child_input, fp_child_output, pid);
1367 - if(rc)
1368 - netdata_log_error("%s exited abnormally. Failed to get kubernetes labels.", label_script);
1369 -}
1370 -
1371 -static void localhost_load_auto_labels(void) {
1372 - DICTIONARY *labels = rrdb.localhost->rrdlabels;
1373 -
1374 - if (rrdb.localhost->system_info->cloud_provider_type)
1375 - rrdlabels_add(labels, "_cloud_provider_type", rrdb.localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
1376 -
1377 - if (rrdb.localhost->system_info->cloud_instance_type)
1378 - rrdlabels_add(labels, "_cloud_instance_type", rrdb.localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
1379 -
1380 - if (rrdb.localhost->system_info->cloud_instance_region)
1381 - rrdlabels_add(labels, "_cloud_instance_region", rrdb.localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
1382 -
1383 - if (rrdb.localhost->system_info->host_os_name)
1384 - rrdlabels_add(labels, "_os_name", rrdb.localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
1385 -
1386 - if (rrdb.localhost->system_info->host_os_version)
1387 - rrdlabels_add(labels, "_os_version", rrdb.localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO);
1388 -
1389 - if (rrdb.localhost->system_info->kernel_version)
1390 - rrdlabels_add(labels, "_kernel_version", rrdb.localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO);
1391 -
1392 - if (rrdb.localhost->system_info->host_cores)
1393 - rrdlabels_add(labels, "_system_cores", rrdb.localhost->system_info->host_cores, RRDLABEL_SRC_AUTO);
1394 -
1395 - if (rrdb.localhost->system_info->host_cpu_freq)
1396 - rrdlabels_add(labels, "_system_cpu_freq", rrdb.localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO);
1397 -
1398 - if (rrdb.localhost->system_info->host_ram_total)
1399 - rrdlabels_add(labels, "_system_ram_total", rrdb.localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO);
1400 -
1401 - if (rrdb.localhost->system_info->host_disk_space)
1402 - rrdlabels_add(labels, "_system_disk_space", rrdb.localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO);
1403 -
1404 - if (rrdb.localhost->system_info->architecture)
1405 - rrdlabels_add(labels, "_architecture", rrdb.localhost->system_info->architecture, RRDLABEL_SRC_AUTO);
1406 -
1407 - if (rrdb.localhost->system_info->virtualization)
1408 - rrdlabels_add(labels, "_virtualization", rrdb.localhost->system_info->virtualization, RRDLABEL_SRC_AUTO);
1409 -
1410 - if (rrdb.localhost->system_info->container)
1411 - rrdlabels_add(labels, "_container", rrdb.localhost->system_info->container, RRDLABEL_SRC_AUTO);
1412 -
1413 - if (rrdb.localhost->system_info->container_detection)
1414 - rrdlabels_add(labels, "_container_detection", rrdb.localhost->system_info->container_detection, RRDLABEL_SRC_AUTO);
1415 -
1416 - if (rrdb.localhost->system_info->virt_detection)
1417 - rrdlabels_add(labels, "_virt_detection", rrdb.localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO);
1418 -
1419 - if (rrdb.localhost->system_info->is_k8s_node)
1420 - rrdlabels_add(labels, "_is_k8s_node", rrdb.localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO);
1421 -
1422 - if (rrdb.localhost->system_info->install_type)
1423 - rrdlabels_add(labels, "_install_type", rrdb.localhost->system_info->install_type, RRDLABEL_SRC_AUTO);
1424 -
1425 - if (rrdb.localhost->system_info->prebuilt_arch)
1426 - rrdlabels_add(labels, "_prebuilt_arch", rrdb.localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO);
1427 -
1428 - if (rrdb.localhost->system_info->prebuilt_dist)
1429 - rrdlabels_add(labels, "_prebuilt_dist", rrdb.localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO);
1430 -
1431 - add_aclk_host_labels();
1432 -
1433 - health_add_host_labels();
1434 -
1435 - rrdlabels_add(labels, "_is_parent", (rrdb.localhost->connected_children_count > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);
1436 -
1437 - if (rrdb.localhost->rrdpush_send_destination)
1438 - rrdlabels_add(labels, "_streams_to", rrdb.localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
1439 -}
1440 -
1441 -void localhost_load_labels(void) {
1442 - if(!rrdb.localhost->rrdlabels)
1443 - rrdb.localhost->rrdlabels = rrdlabels_create();
1444 -
1445 - rrdlabels_unmark_all(rrdb.localhost->rrdlabels);
1446 -
1447 - // priority is important here
1448 - localhost_load_config_labels();
1449 - localhost_load_kubernetes_labels();
1450 - localhost_load_auto_labels();
1451 -
1452 - rrdhost_flag_set(rrdb.localhost, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1453 -
1454 - rrdpush_send_host_labels(rrdb.localhost);
1455 -}
1456 -
1308 void set_silencers_filename() {
1309 char filename[FILENAME_MAX + 1];
1310 snprintfz(filename, FILENAME_MAX, "%s/health.silencers.json", netdata_configured_varlib_dir);
@@ -1600,7 +1451,7 @@ int main(int argc, char **argv) {
1451 }
1452
1453 if(strcmp(optarg, "unittest") == 0) {
1603 - rrdb.unittest_running = true;
1454 + unittest_running = true;
1455
1456 if (pluginsd_parser_unittest())
1457 return 1;
@@ -1618,10 +1469,10 @@ int main(int argc, char **argv) {
1469 // No call to load the config file on this code-path
1470 post_conf_load(&user);
1471 get_netdata_configured_variables();
1621 - rrdb.default_update_every = 1;
1622 - default_storage_engine_id = STORAGE_ENGINE_RAM;
1472 + default_rrd_update_every = 1;
1473 + default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
1474 default_health_enabled = 0;
1624 - rrdb.storage_tiers = 1;
1475 + storage_tiers = 1;
1476 registry_init();
1477 if(rrd_init("unittest", NULL, true)) {
1478 fprintf(stderr, "rrd_init failed for unittest\n");
@@ -1650,52 +1501,52 @@ int main(int argc, char **argv) {
1501 return command_argument_sanitization_tests();
1502 }
1503 else if(strcmp(optarg, "dicttest") == 0) {
1653 - rrdb.unittest_running = true;
1504 + unittest_running = true;
1505 return dictionary_unittest(10000);
1506 }
1507 else if(strcmp(optarg, "araltest") == 0) {
1657 - rrdb.unittest_running = true;
1508 + unittest_running = true;
1509 return aral_unittest(10000);
1510 }
1511 else if(strcmp(optarg, "stringtest") == 0) {
1661 - rrdb.unittest_running = true;
1512 + unittest_running = true;
1513 return string_unittest(10000);
1514 }
1515 else if(strcmp(optarg, "rrdlabelstest") == 0) {
1665 - rrdb.unittest_running = true;
1516 + unittest_running = true;
1517 return rrdlabels_unittest();
1518 }
1519 else if(strcmp(optarg, "buffertest") == 0) {
1669 - rrdb.unittest_running = true;
1520 + unittest_running = true;
1521 return buffer_unittest();
1522 }
1523 #ifdef ENABLE_DBENGINE
1524 else if(strcmp(optarg, "mctest") == 0) {
1674 - rrdb.unittest_running = true;
1525 + unittest_running = true;
1526 return mc_unittest();
1527 }
1528 else if(strcmp(optarg, "ctxtest") == 0) {
1678 - rrdb.unittest_running = true;
1529 + unittest_running = true;
1530 return ctx_unittest();
1531 }
1532 else if(strcmp(optarg, "metatest") == 0) {
1682 - rrdb.unittest_running = true;
1533 + unittest_running = true;
1534 return metadata_unittest();
1535 }
1536 else if(strcmp(optarg, "pgctest") == 0) {
1686 - rrdb.unittest_running = true;
1537 + unittest_running = true;
1538 return pgc_unittest();
1539 }
1540 else if(strcmp(optarg, "mrgtest") == 0) {
1690 - rrdb.unittest_running = true;
1541 + unittest_running = true;
1542 return mrg_unittest();
1543 }
1544 else if(strcmp(optarg, "julytest") == 0) {
1694 - rrdb.unittest_running = true;
1545 + unittest_running = true;
1546 return julytest();
1547 }
1548 else if(strcmp(optarg, "parsertest") == 0) {
1698 - rrdb.unittest_running = true;
1549 + unittest_running = true;
1550 return pluginsd_parser_unittest();
1551 }
1552 else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
@@ -1703,7 +1554,7 @@ int main(int argc, char **argv) {
1554 unsigned history_seconds = strtoul(optarg, NULL, 0);
1555 post_conf_load(&user);
1556 get_netdata_configured_variables();
1706 - rrdb.default_update_every = 1;
1557 + default_rrd_update_every = 1;
1558 registry_init();
1559 if(rrd_init("dbengine-dataset", NULL, true)) {
1560 fprintf(stderr, "rrd_init failed for unittest\n");
@@ -1981,12 +1832,25 @@ int main(int argc, char **argv) {
1832 #endif
1833 #endif
1834
1984 - // decide how many libuv workers to have on 32 vs 64 bit platforms
1985 - set_libuv_worker_threads();
1835 + // set libuv worker threads
1836 + libuv_worker_threads = (int)get_netdata_cpus() * 6;
1837 +
1838 + if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS)
1839 + libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
1840 +
1841 + if(libuv_worker_threads > MAX_LIBUV_WORKER_THREADS)
1842 + libuv_worker_threads = MAX_LIBUV_WORKER_THREADS;
1843 +
1844 +
1845 + libuv_worker_threads = config_get_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", libuv_worker_threads);
1846 + if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS) {
1847 + libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
1848 + config_set_number(CONFIG_SECTION_GLOBAL, "libuv worker threads", libuv_worker_threads);
1849 + }
1850
1851 {
1852 char buf[20 + 1];
1989 - snprintfz(buf, 20, "%d", rrdb.libuv_worker_threads);
1853 + snprintfz(buf, 20, "%d", libuv_worker_threads);
1854 setenv("UV_THREADPOOL_SIZE", buf, 1);
1855 }
1856
@@ -2034,7 +1898,7 @@ int main(int argc, char **argv) {
1898 // initialize the log files
1899 open_all_log_files();
1900
2037 - rrdb.ieee754_doubles = is_system_ieee754_double();
1901 + ieee754_doubles = is_system_ieee754_double();
1902
1903 aral_judy_init();
1904
@@ -2210,9 +2074,9 @@ int main(int argc, char **argv) {
2074
2075 error_log_limit_reset();
2076
2213 - // Load localhost labels
2077 + // Load host labels
2078 delta_startup_time("collect host labels");
2215 - localhost_load_labels();
2079 + reload_host_labels();
2080
2081 // ------------------------------------------------------------------------
2082 // spawn the threads
daemon/main.h
-2
@@ -58,6 +58,4 @@ void service_exits(void);
58 bool service_running(SERVICE_TYPE service);
59 struct service_thread *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t request_quit_callback, force_quit_t force_quit_callback, void *data, bool update __maybe_unused);
60
61 -void localhost_load_labels(void);
62 -
61 #endif /* NETDATA_MAIN_H */
daemon/service.c
+11 -11
@@ -45,17 +45,17 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
45 netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
46 }
47
48 - if (st->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
48 + if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
49 rrddimvar_delete_all(rd);
50
51 /* only a collector can mark a chart as obsolete, so we must remove the reference */
52
53 size_t tiers_available = 0, tiers_said_no_retention = 0;
54 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
54 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
55 if(rd->tiers[tier].db_collection_handle) {
56 tiers_available++;
57
58 - if(storage_engine_store_finalize(st->storage_engine_id, rd->tiers[tier].db_collection_handle))
58 + if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle))
59 tiers_said_no_retention++;
60
61 rd->tiers[tier].db_collection_handle = NULL;
@@ -87,7 +87,7 @@ static bool svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensio
87 dfe_start_write(st->rrddim_root_index, rd) {
88 if(unlikely(
89 all_dimensions ||
90 - (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && (rd->collector.last_collected_time.tv_sec + rrdb.rrdset_free_obsolete_time_s < now))
90 + (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && (rd->collector.last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now))
91 )) {
92
93 if(dictionary_acquired_item_references(rd_dfe.item) == 1) {
@@ -121,7 +121,7 @@ static void svc_rrdset_obsolete_to_archive(RRDSET *st) {
121 // has to be run after all dimensions are archived - or use-after-free will occur
122 rrdvar_delete_all(st->rrdvars);
123
124 - if(st->storage_engine_id != STORAGE_ENGINE_DBENGINE) {
124 + if(st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {
125 if(rrdhost_option_check(st->rrdhost, RRDHOST_OPTION_DELETE_OBSOLETE_CHARTS)) {
126 worker_is_busy(WORKER_JOB_DELETE_CHART);
127 rrdset_delete_files(st);
@@ -146,9 +146,9 @@ static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
146 continue;
147
148 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)
149 - && st->last_accessed_time_s + rrdb.rrdset_free_obsolete_time_s < now
150 - && st->last_updated.tv_sec + rrdb.rrdset_free_obsolete_time_s < now
151 - && st->last_collected_time.tv_sec + rrdb.rrdset_free_obsolete_time_s < now
149 + && st->last_accessed_time_s + rrdset_free_obsolete_time_s < now
150 + && st->last_updated.tv_sec + rrdset_free_obsolete_time_s < now
151 + && st->last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now
152 )) {
153 svc_rrdset_obsolete_to_archive(st);
154 }
@@ -196,7 +196,7 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
196 svc_rrdhost_cleanup_obsolete_charts(host);
197 }
198
199 - if(host != rrdb.localhost
199 + if(host != localhost
200 && host->trigger_chart_obsoletion_check
201 && (
202 (
@@ -231,7 +231,7 @@ restart_after_removal:
231
232 if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST)
233 /* don't delete multi-host DB host files */
234 - && !(host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
234 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
235 ) {
236 worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS);
237 rrdhost_delete_charts(host);
@@ -297,7 +297,7 @@ void *service_main(void *ptr)
297 heartbeat_next(&hb, step);
298
299 svc_rrd_cleanup_obsolete_charts_from_all_hosts();
300 - svc_rrdhost_cleanup_orphan_hosts(rrdb.localhost);
300 + svc_rrdhost_cleanup_orphan_hosts(localhost);
301 }
302
303 netdata_thread_cleanup_pop(1);
daemon/unit_test.c
+31 -31
@@ -1268,8 +1268,8 @@ int run_test(struct test *test)
1268 {
1269 fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description);
1270
1271 - default_storage_engine_id = STORAGE_ENGINE_ALLOC;
1272 - rrdb.default_update_every = test->update_every;
1271 + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
1272 + default_rrd_update_every = test->update_every;
1273
1274 char name[101];
1275 snprintfz(name, 100, "unittest-%s", test->name);
@@ -1537,8 +1537,8 @@ int unit_test(long delay, long shift)
1537 snprintfz(name, 100, "unittest-%d-%ld-%ld", repeat, delay, shift);
1538
1539 //debug_flags = 0xffffffff;
1540 - default_storage_engine_id = STORAGE_ENGINE_ALLOC;
1541 - rrdb.default_update_every = 1;
1540 + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
1541 + default_rrd_update_every = 1;
1542
1543 int do_abs = 1;
1544 int do_inc = 1;
@@ -1826,11 +1826,11 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number
1826 if(unlikely(v > rd->collector.collected_value_max)) rd->collector.collected_value_max = v;
1827 }
1828
1829 -static RRDHOST *dbengine_rrdhost_get_or_create(char *name)
1829 +static RRDHOST *dbengine_rrdhost_find_or_create(char *name)
1830 {
1831 /* We don't want to drop metrics when generating load, we prefer to block data generation itself */
1832
1833 - return rrdhost_get_or_create(
1833 + return rrdhost_find_or_create(
1834 name
1835 , name
1836 , name
@@ -1841,9 +1841,9 @@ static RRDHOST *dbengine_rrdhost_get_or_create(char *name)
1841 , ""
1842 , program_name
1843 , program_version
1844 - , rrdb.default_update_every
1845 - , rrdb.default_rrd_history_entries
1846 - , STORAGE_ENGINE_DBENGINE
1844 + , default_rrd_update_every
1845 + , default_rrd_history_entries
1846 + , RRD_MEMORY_MODE_DBENGINE
1847 , default_health_enabled
1848 , default_rrdpush_enabled
1849 , default_rrdpush_destination
@@ -1934,7 +1934,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
1934 // feed it with the test data
1935 for (i = 0 ; i < CHARTS ; ++i) {
1936 for (j = 0 ; j < DIMS ; ++j) {
1937 - storage_engine_store_change_collection_frequency(st[i]->storage_engine_id, rd[i][j]->tiers[0].db_collection_handle, update_every);
1937 + storage_engine_store_change_collection_frequency(rd[i][j]->tiers[0].db_collection_handle, update_every);
1938
1939 rd[i][j]->collector.last_collected_time.tv_sec =
1940 st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
@@ -1985,7 +1985,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1985 time_now = time_start + (c + 1) * update_every;
1986 for (i = 0 ; i < CHARTS ; ++i) {
1987 for (j = 0; j < DIMS; ++j) {
1988 - storage_engine_query_init(st[i]->storage_engine_id, rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
1988 + storage_engine_query_init(rd[i][j]->tiers[0].backend, rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
1989 for (k = 0; k < QUERY_BATCH; ++k) {
1990 last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
1991 j * REGION_POINTS[current_region] + c + k;
@@ -2120,10 +2120,10 @@ int test_dbengine(void)
2120 error_log_limit_unlimited();
2121 fprintf(stderr, "\nRunning DB-engine test\n");
2122
2123 - default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
2123 + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
2124
2125 fprintf(stderr, "Initializing localhost with hostname 'unittest-dbengine'");
2126 - host = dbengine_rrdhost_get_or_create("unittest-dbengine");
2126 + host = dbengine_rrdhost_find_or_create("unittest-dbengine");
2127 if (NULL == host)
2128 return 1;
2129
@@ -2234,9 +2234,9 @@ int test_dbengine(void)
2234 }
2235
2236 rrd_wrlock();
2237 - rrdeng_prepare_exit(host->db[0].instance);
2237 + rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
2238 rrdhost_delete_charts(host);
2239 - rrdeng_exit(host->db[0].instance);
2239 + rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
2240 rrd_unlock();
2241
2242 return errors + value_errors + time_errors;
@@ -2336,17 +2336,17 @@ void generate_dbengine_dataset(unsigned history_seconds)
2336 int i;
2337 time_t time_present;
2338
2339 - default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
2340 - rrdb.default_rrdeng_page_cache_mb = 128;
2339 + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
2340 + default_rrdeng_page_cache_mb = 128;
2341 // Worst case for uncompressible data
2342 - rrdb.default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) /
2342 + default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) /
2343 (1024 * 1024);
2344 - rrdb.default_rrdeng_disk_quota_mb -= rrdb.default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2344 + default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2345
2346 error_log_limit_unlimited();
2347 fprintf(stderr, "Initializing localhost with hostname 'dbengine-dataset'");
2348
2349 - host = dbengine_rrdhost_get_or_create("dbengine-dataset");
2349 + host = dbengine_rrdhost_find_or_create("dbengine-dataset");
2350 if (NULL == host)
2351 return;
2352
@@ -2381,7 +2381,7 @@ void generate_dbengine_dataset(unsigned history_seconds)
2381 }
2382 freez(thread_info);
2383 rrd_wrlock();
2384 - rrdhost_free___while_having_rrd_wrlock(rrdb.localhost, true);
2384 + rrdhost_free___while_having_rrd_wrlock(localhost, true);
2385 rrd_unlock();
2386 }
2387
@@ -2429,7 +2429,7 @@ static void query_dbengine_chart(void *arg)
2429
2430 if (thread_info->delete_old_data) {
2431 /* A time window of twice the disk space is sufficient for compression space savings of up to 50% */
2432 - time_approx_min = time_max - (rrdb.default_rrdeng_disk_quota_mb * 2 * 1024 * 1024) /
2432 + time_approx_min = time_max - (default_rrdeng_disk_quota_mb * 2 * 1024 * 1024) /
2433 (((uint64_t) DSET_DIMS * DSET_CHARTS) * sizeof(storage_number));
2434 time_min = MAX(time_min, time_approx_min);
2435 }
@@ -2441,7 +2441,7 @@ static void query_dbengine_chart(void *arg)
2441 time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
2442 }
2443
2444 - storage_engine_query_init(st->storage_engine_id, rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
2444 + storage_engine_query_init(rd->tiers[0].backend, rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
2445 ++thread_info->queries_nr;
2446 for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
2447 generatedv = generate_dbengine_chart_value(i, j, time_now);
@@ -2529,23 +2529,23 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2529 if (PAGE_CACHE_MB < RRDENG_MIN_PAGE_CACHE_SIZE_MB)
2530 PAGE_CACHE_MB = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
2531
2532 - default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
2533 - rrdb.default_rrdeng_page_cache_mb = PAGE_CACHE_MB;
2532 + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
2533 + default_rrdeng_page_cache_mb = PAGE_CACHE_MB;
2534 if (DISK_SPACE_MB) {
2535 fprintf(stderr, "By setting disk space limit data are allowed to be deleted. "
2536 "Data validation is turned off for this run.\n");
2537 - rrdb.default_rrdeng_disk_quota_mb = DISK_SPACE_MB;
2537 + default_rrdeng_disk_quota_mb = DISK_SPACE_MB;
2538 } else {
2539 // Worst case for uncompressible data
2540 - rrdb.default_rrdeng_disk_quota_mb =
2540 + default_rrdeng_disk_quota_mb =
2541 (((uint64_t) DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * HISTORY_SECONDS) / (1024 * 1024);
2542 - rrdb.default_rrdeng_disk_quota_mb -= rrdb.default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2542 + default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;
2543 }
2544
2545 fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n");
2546
2547 (void) sql_init_database(DB_CHECK_NONE, 1);
2548 - host = dbengine_rrdhost_get_or_create("dbengine-stress-test");
2548 + host = dbengine_rrdhost_find_or_create("dbengine-stress-test");
2549 if (NULL == host)
2550 return;
2551
@@ -2643,9 +2643,9 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2643 }
2644 freez(query_threads);
2645 rrd_wrlock();
2646 - rrdeng_prepare_exit(host->db[0].instance);
2646 + rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
2647 rrdhost_delete_charts(host);
2648 - rrdeng_exit(host->db[0].instance);
2648 + rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
2649 rrd_unlock();
2650 }
2651
database/contexts/api_v2.c
+15 -14
@@ -802,7 +802,7 @@ static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdc
802 {
803 buffer_json_member_add_string(wb, "status", rrdhost_db_status_to_string(s.db.status));
804 buffer_json_member_add_string(wb, "liveness", rrdhost_db_liveness_to_string(s.db.liveness));
805 - buffer_json_member_add_string(wb, "mode", storage_engine_name(s.db.storage_engine_id));
805 + buffer_json_member_add_string(wb, "mode", rrd_memory_mode_name(s.db.mode));
806 buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s);
807 buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s);
808 buffer_json_member_add_uint64(wb, "metrics", s.db.metrics);
@@ -989,9 +989,9 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
989 else
990 buffer_json_member_add_object(wb, "agent");
991
992 - buffer_json_member_add_string(wb, "mg", rrdb.localhost->machine_guid);
993 - buffer_json_member_add_uuid(wb, "nd", rrdb.localhost->node_id);
994 - buffer_json_member_add_string(wb, "nm", rrdhost_hostname(rrdb.localhost));
992 + buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
993 + buffer_json_member_add_uuid(wb, "nd", localhost->node_id);
994 + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost));
995 buffer_json_member_add_time_t(wb, "now", now_s);
996
997 if(array)
@@ -1005,13 +1005,14 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
1005 buffer_json_cloud_status(wb, now_s);
1006
1007 buffer_json_member_add_array(wb, "db_size");
1008 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
1009 - STORAGE_ENGINE_ID storage_engine_id = rrdb.localhost->db[tier].id;
1008 + for (size_t tier = 0; tier < storage_tiers; tier++) {
1009 + STORAGE_ENGINE *eng = localhost->db[tier].eng;
1010 + if (!eng) continue;
1011
1011 - size_t max = storage_engine_disk_space_max(storage_engine_id, rrdb.localhost->db[tier].instance);
1012 - size_t used = storage_engine_disk_space_used(storage_engine_id, rrdb.localhost->db[tier].instance);
1013 - time_t first_time_s = storage_engine_global_first_time_s(storage_engine_id, rrdb.localhost->db[tier].instance);
1014 - size_t currently_collected_metrics = storage_engine_collected_metrics(storage_engine_id, rrdb.localhost->db[tier].instance);
1012 + size_t max = storage_engine_disk_space_max(eng->backend, localhost->db[tier].instance);
1013 + size_t used = storage_engine_disk_space_used(eng->backend, localhost->db[tier].instance);
1014 + time_t first_time_s = storage_engine_global_first_time_s(eng->backend, localhost->db[tier].instance);
1015 + size_t currently_collected_metrics = storage_engine_collected_metrics(eng->backend, localhost->db[tier].instance);
1016
1017 NETDATA_DOUBLE percent;
1018 if (used && max)
@@ -1267,7 +1268,7 @@ static void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct s
1268 {
1269 buffer_json_member_add_string(wb, "type", "agent");
1270 buffer_json_member_add_string(wb, "exec", t->notification.exec ? t->notification.exec : NULL);
1270 - buffer_json_member_add_string(wb, "to", t->notification.to_key ? t->notification.to_key : string2str(rrdb.localhost->health.health_default_recipient));
1271 + buffer_json_member_add_string(wb, "to", t->notification.to_key ? t->notification.to_key : string2str(localhost->health.health_default_recipient));
1272 buffer_json_member_add_string(wb, "delay", t->notification.delay);
1273 buffer_json_member_add_string(wb, "repeat", t->notification.repeat);
1274 buffer_json_member_add_string(wb, "options", t->notification.options);
@@ -1584,7 +1585,7 @@ static void contexts_v2_alert_transition_callback(struct sql_alert_transition_da
1585 [ATF_CLASS] = t->classification,
1586 [ATF_TYPE] = t->type,
1587 [ATF_COMPONENT] = t->component,
1587 - [ATF_ROLE] = t->recipient && *t->recipient ? t->recipient : string2str(rrdb.localhost->health.health_default_recipient),
1588 + [ATF_ROLE] = t->recipient && *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient),
1589 [ATF_NODE] = machine_guid,
1590 [ATF_ALERT_NAME] = t->alert_name,
1591 [ATF_CHART_NAME] = t->chart_name,
@@ -1754,9 +1755,9 @@ static void contexts_v2_alert_transitions_to_json(BUFFER *wb, struct rrdcontext_
1755 buffer_json_member_add_time_t(wb, "delay", t->delay);
1756 buffer_json_member_add_time_t(wb, "delay_up_to_time", t->delay_up_to_timestamp);
1757 health_entry_flags_to_json_array(wb, "flags", t->flags);
1757 - buffer_json_member_add_string(wb, "exec", *t->exec ? t->exec : string2str(rrdb.localhost->health.health_default_exec));
1758 + buffer_json_member_add_string(wb, "exec", *t->exec ? t->exec : string2str(localhost->health.health_default_exec));
1759 buffer_json_member_add_uint64(wb, "exec_code", t->exec_code);
1759 - buffer_json_member_add_string(wb, "to", *t->recipient ? t->recipient : string2str(rrdb.localhost->health.health_default_recipient));
1760 + buffer_json_member_add_string(wb, "to", *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient));
1761 }
1762 buffer_json_object_close(wb); // notification
1763 }
database/contexts/internal.h
+1
@@ -8,6 +8,7 @@
8 #include "../../aclk/schema-wrappers/context.h"
9 #include "../../aclk/aclk_contexts_api.h"
10 #include "../../aclk/aclk.h"
11 +#include "../storage_engine.h"
12
13 #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
14 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
database/contexts/query_scope.c
+1 -1
@@ -17,7 +17,7 @@ ssize_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN
17 uint64_t a_hash = 0;
18 uint64_t t_hash = 0;
19
20 - dfe_start_read(rrdb.rrdhost_root_index, host) {
20 + dfe_start_read(rrdhost_root_index, host) {
21 if(host->node_id)
22 uuid_unparse_lower(*host->node_id, host_node_id_str);
23 else
database/contexts/query_target.c
+19 -19
@@ -211,19 +211,19 @@ typedef struct query_target_locals {
211 QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
212 } QUERY_TARGET_LOCALS;
213
214 -STORAGE_ENGINE_ID query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
214 +struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
215 QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
216 - return qn->rrdhost->db[tier].id;
216 + return qn->rrdhost->db[tier].eng;
217 }
218
219 static inline void query_metric_release(QUERY_TARGET *qt, QUERY_METRIC *qm) {
220 qm->plan.used = 0;
221
222 // reset the tiers
223 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
223 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
224 if(qm->tiers[tier].db_metric_handle) {
225 - STORAGE_ENGINE_ID storage_engine_id = query_metric_storage_engine(qt, qm, tier);
226 - storage_engine_metric_release(storage_engine_id, qm->tiers[tier].db_metric_handle);
225 + STORAGE_ENGINE *eng = query_metric_storage_engine(qt, qm, tier);
226 + eng->api.metric_release(qm->tiers[tier].db_metric_handle);
227 qm->tiers[tier].db_metric_handle = NULL;
228 }
229 }
@@ -240,26 +240,26 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
240 size_t tiers_added = 0;
241
242 struct {
243 - STORAGE_ENGINE_ID storage_engine_id;
243 + STORAGE_ENGINE *eng;
244 STORAGE_METRIC_HANDLE *db_metric_handle;
245 time_t db_first_time_s;
246 time_t db_last_time_s;
247 time_t db_update_every_s;
248 - } tier_retention[rrdb.storage_tiers];
248 + } tier_retention[storage_tiers];
249
250 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
251 - STORAGE_ENGINE_ID storage_engine_id = qn->rrdhost->db[tier].id;
252 - tier_retention[tier].storage_engine_id = storage_engine_id;
250 + for (size_t tier = 0; tier < storage_tiers; tier++) {
251 + STORAGE_ENGINE *eng = qn->rrdhost->db[tier].eng;
252 + tier_retention[tier].eng = eng;
253 tier_retention[tier].db_update_every_s = (time_t) (qn->rrdhost->db[tier].tier_grouping * ri->update_every_s);
254
255 if(rm->rrddim && rm->rrddim->tiers[tier].db_metric_handle)
256 - tier_retention[tier].db_metric_handle = storage_engine_metric_dup(rm->rrddim->rrdset->storage_engine_id, rm->rrddim->tiers[tier].db_metric_handle);
256 + tier_retention[tier].db_metric_handle = eng->api.metric_dup(rm->rrddim->tiers[tier].db_metric_handle);
257 else
258 - tier_retention[tier].db_metric_handle = storage_engine_metric_get(qn->rrdhost->db[tier].id, qn->rrdhost->db[tier].instance, &rm->uuid);
258 + tier_retention[tier].db_metric_handle = eng->api.metric_get(qn->rrdhost->db[tier].instance, &rm->uuid);
259
260 if(tier_retention[tier].db_metric_handle) {
261 - tier_retention[tier].db_first_time_s = storage_engine_oldest_time_s(tier_retention[tier].storage_engine_id, tier_retention[tier].db_metric_handle);
262 - tier_retention[tier].db_last_time_s = storage_engine_latest_time_s(tier_retention[tier].storage_engine_id, tier_retention[tier].db_metric_handle);
261 + tier_retention[tier].db_first_time_s = storage_engine_oldest_time_s(tier_retention[tier].eng->backend, tier_retention[tier].db_metric_handle);
262 + tier_retention[tier].db_last_time_s = storage_engine_latest_time_s(tier_retention[tier].eng->backend, tier_retention[tier].db_metric_handle);
263
264 if(!common_first_time_s)
265 common_first_time_s = tier_retention[tier].db_first_time_s;
@@ -285,7 +285,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
285 }
286 }
287
288 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
288 + for (size_t tier = 0; tier < storage_tiers; tier++) {
289 if(!qt->db.tiers[tier].update_every || (tier_retention[tier].db_update_every_s && tier_retention[tier].db_update_every_s < qt->db.tiers[tier].update_every))
290 qt->db.tiers[tier].update_every = tier_retention[tier].db_update_every_s;
291
@@ -329,8 +329,8 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
329 if (!qt->db.last_time_s || common_last_time_s > qt->db.last_time_s)
330 qt->db.last_time_s = common_last_time_s;
331
332 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
333 - internal_fatal(tier_retention[tier].storage_engine_id != query_metric_storage_engine(qt, qm, tier), "QUERY TARGET: storage engine mismatch");
332 + for (size_t tier = 0; tier < storage_tiers; tier++) {
333 + internal_fatal(tier_retention[tier].eng != query_metric_storage_engine(qt, qm, tier), "QUERY TARGET: storage engine mismatch");
334 qm->tiers[tier].db_metric_handle = tier_retention[tier].db_metric_handle;
335 qm->tiers[tier].db_first_time_s = tier_retention[tier].db_first_time_s;
336 qm->tiers[tier].db_last_time_s = tier_retention[tier].db_last_time_s;
@@ -341,9 +341,9 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
341 }
342
343 // cleanup anything we allocated to the retention we will not use
344 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
344 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
345 if (tier_retention[tier].db_metric_handle)
346 - storage_engine_metric_release(tier_retention[tier].storage_engine_id, tier_retention[tier].db_metric_handle);
346 + tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
347 }
348
349 return false;
database/contexts/rrdcontext.c
+21 -4
@@ -199,8 +199,25 @@ int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char
199 // ACLK interface
200
201 static bool rrdhost_check_our_claim_id(const char *claim_id) {
202 - if(!rrdb.localhost->aclk_state.claimed_id) return false;
203 - return (strcasecmp(claim_id, rrdb.localhost->aclk_state.claimed_id) == 0) ? true : false;
202 + if(!localhost->aclk_state.claimed_id) return false;
203 + return (strcasecmp(claim_id, localhost->aclk_state.claimed_id) == 0) ? true : false;
204 +}
205 +
206 +static RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
207 + uuid_t uuid;
208 + if (uuid_parse(node_id, uuid))
209 + return NULL;
210 +
211 + RRDHOST *host = NULL;
212 + dfe_start_read(rrdhost_root_index, host) {
213 + if(!host->node_id) continue;
214 +
215 + if(uuid_memcmp(&uuid, host->node_id) == 0)
216 + break;
217 + }
218 + dfe_done(host);
219 +
220 + return host;
221 }
222
223 void rrdcontext_hub_checkpoint_command(void *ptr) {
@@ -209,7 +226,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) {
226 if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
227 netdata_log_error("RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
228 cmd->claim_id, cmd->node_id,
212 - rrdb.localhost->aclk_state.claimed_id ? rrdb.localhost->aclk_state.claimed_id : "NOT SET",
229 + localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
230 cmd->claim_id);
231
232 return;
@@ -271,7 +288,7 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) {
288 if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
289 netdata_log_error("RRDCONTEXT: received stop streaming command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
290 cmd->claim_id, cmd->node_id,
274 - rrdb.localhost->aclk_state.claimed_id ? rrdb.localhost->aclk_state.claimed_id : "NOT SET",
291 + localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
292 cmd->claim_id);
293
294 return;
database/contexts/rrdcontext.h
+1 -1
@@ -578,7 +578,7 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
578 return rrdmetric_acquired_name(qd->rma);
579 }
580
581 -STORAGE_ENGINE_ID query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
581 +struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
582
583 STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version);
584 STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version);
database/contexts/worker.c
+7 -5
@@ -216,7 +216,7 @@ void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool
216 static void rrdcontext_recalculate_retention_all_hosts(void) {
217 rrdcontext_next_db_rotation_ut = 0;
218 RRDHOST *host;
219 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
219 + dfe_start_reentrant(rrdhost_root_index, host) {
220 worker_is_busy(WORKER_JOB_RETENTION);
221 rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DB_ROTATION, true);
222 }
@@ -235,9 +235,11 @@ bool rrdmetric_update_retention(RRDMETRIC *rm) {
235 }
236 else {
237 RRDHOST *rrdhost = rm->ri->rc->rrdhost;
238 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
238 + for (size_t tier = 0; tier < storage_tiers; tier++) {
239 + STORAGE_ENGINE *eng = rrdhost->db[tier].eng;
240 +
241 time_t first_time_t, last_time_t;
240 - if (storage_engine_metric_retention(rrdhost->db[tier].id, rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t)) {
242 + if (eng->api.metric_retention_by_uuid(rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t)) {
243 if (first_time_t < min_first_time_t)
244 min_first_time_t = first_time_t;
245
@@ -434,7 +436,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo
436
437 static void rrdcontext_garbage_collect_for_all_hosts(void) {
438 RRDHOST *host;
437 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
439 + dfe_start_reentrant(rrdhost_root_index, host) {
440 rrdcontext_garbage_collect_single_host(host, true);
441 }
442 dfe_done(host);
@@ -1109,7 +1111,7 @@ void *rrdcontext_main(void *ptr) {
1111 size_t pp_queued_contexts_for_all_hosts = 0;
1112
1113 RRDHOST *host;
1112 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1114 + dfe_start_reentrant(rrdhost_root_index, host) {
1115 if(unlikely(!service_running(SERVICE_CONTEXT))) break;
1116
1117 worker_is_busy(WORKER_JOB_HOSTS);
database/engine/cache.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "cache.h"
2
3 /* STATES AND TRANSITIONS
database/engine/cache.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef DBENGINE_CACHE_H
2 #define DBENGINE_CACHE_H
3
database/engine/datafile.c
+2 -2
@@ -250,7 +250,7 @@ int create_data_file(struct rrdengine_datafile *datafile)
250 char path[RRDENG_PATH_MAX];
251
252 generate_datafilepath(datafile, path, sizeof(path));
253 - fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, rrdb.use_direct_io);
253 + fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, use_direct_io);
254 if (fd < 0) {
255 ctx_fs_error(ctx);
256 return fd;
@@ -333,7 +333,7 @@ static int load_data_file(struct rrdengine_datafile *datafile)
333 char path[RRDENG_PATH_MAX];
334
335 generate_datafilepath(datafile, path, sizeof(path));
336 - fd = open_file_for_io(path, O_RDWR, &file, rrdb.use_direct_io);
336 + fd = open_file_for_io(path, O_RDWR, &file, use_direct_io);
337 if (fd < 0) {
338 ctx_fs_error(ctx);
339 return fd;
database/engine/journalfile.c
+6 -6
@@ -296,8 +296,8 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo
296 void journalfile_v2_data_unmount_cleanup(time_t now_s) {
297 // DO NOT WAIT ON ANY LOCK!!!
298
299 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
300 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) rrdb.multidb_ctx[tier];
299 + for(size_t tier = 0; tier < (size_t)storage_tiers ;tier++) {
300 + struct rrdengine_instance *ctx = multidb_ctx[tier];
301 if(!ctx) continue;
302
303 struct rrdengine_datafile *datafile;
@@ -577,7 +577,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi
577 char path[RRDENG_PATH_MAX];
578
579 journalfile_v1_generate_path(datafile, path, sizeof(path));
580 - fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, rrdb.use_direct_io);
580 + fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, use_direct_io);
581 if (fd < 0) {
582 ctx_fs_error(ctx);
583 return fd;
@@ -917,7 +917,7 @@ static int journalfile_v2_validate(void *data_start, size_t journal_v2_file_size
917 rc = journalfile_check_v2_extent_list(data_start, journal_v2_file_size);
918 if (rc) return 1;
919
920 - if (!rrdb.db_engine_journal_check)
920 + if (!db_engine_journal_check)
921 return 0;
922
923 rc = journalfile_check_v2_metric_list(data_start, journal_v2_file_size);
@@ -1115,7 +1115,7 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal
1115
1116 // Initialize the journal file to be able to access the data
1117
1118 - if (!rrdb.db_engine_journal_check)
1118 + if (!db_engine_journal_check)
1119 journalfile->v2.flags |= JOURNALFILE_FLAG_METRIC_CRC_CHECK;
1120 journalfile_v2_data_set(journalfile, fd, data_start, journal_v2_file_size);
1121
@@ -1501,7 +1501,7 @@ int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfil
1501
1502 journalfile_v1_generate_path(datafile, path, sizeof(path));
1503
1504 - fd = open_file_for_io(path, O_RDWR, &file, rrdb.use_direct_io);
1504 + fd = open_file_for_io(path, O_RDWR, &file, use_direct_io);
1505 if (fd < 0) {
1506 ctx_fs_error(ctx);
1507
database/engine/metric.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "metric.h"
2
3 typedef int32_t REFCOUNT;
database/engine/metric.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef DBENGINE_METRIC_H
2 #define DBENGINE_METRIC_H
3
database/engine/pagecache.c
+5 -4
@@ -1,4 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS
3
4 #include "rrdengine.h"
5
@@ -231,7 +232,7 @@ static size_t get_page_list_from_pgc(PGC *cache, METRIC *metric, struct rrdengin
232 time_t dt_s = mrg_metric_get_update_every_s(main_mrg, metric);
233
234 if(!dt_s)
234 - dt_s = rrdb.default_update_every;
235 + dt_s = default_rrd_update_every;
236
237 time_t previous_page_end_time_s = now_s - dt_s;
238 bool first = true;
@@ -422,7 +423,7 @@ static size_t list_has_time_gaps(
423 time_t now_s = wanted_start_time_s;
424 time_t dt_s = mrg_metric_get_update_every_s(main_mrg, metric);
425 if(!dt_s)
425 - dt_s = rrdb.default_update_every;
426 + dt_s = default_rrd_update_every;
427
428 size_t pages_pass2 = 0, pages_pass3 = 0;
429 while((pd = pdc_find_page_for_time(
@@ -1084,7 +1085,7 @@ void pgc_and_mrg_initialize(void)
1085 {
1086 main_mrg = mrg_create(0);
1087
1087 - size_t target_cache_size = (size_t) rrdb.default_rrdeng_page_cache_mb * 1024ULL * 1024ULL;
1088 + size_t target_cache_size = (size_t)default_rrdeng_page_cache_mb * 1024ULL * 1024ULL;
1089 size_t main_cache_size = (target_cache_size / 100) * 95;
1090 size_t open_cache_size = 0;
1091 size_t extent_cache_size = (target_cache_size / 100) * 5;
@@ -1094,7 +1095,7 @@ void pgc_and_mrg_initialize(void)
1095 main_cache_size = target_cache_size - extent_cache_size;
1096 }
1097
1097 - extent_cache_size += (size_t) (rrdb.default_rrdeng_extent_cache_mb * 1024ULL * 1024ULL);
1098 + extent_cache_size += (size_t)(default_rrdeng_extent_cache_mb * 1024ULL * 1024ULL);
1099
1100 main_cache = pgc_create(
1101 "main_cache",
database/engine/pdc.c
+2 -2
@@ -1,5 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -
2 +#define NETDATA_RRD_INTERNALS
3 #include "pdc.h"
4
5 struct extent_page_details_list {
@@ -667,7 +667,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
667 };
668
669 // always calculate entries by size
670 - vd.point_size = rrdb.page_type_size[vd.type];
670 + vd.point_size = page_type_size[vd.type];
671 vd.entries = page_entries_by_size(vd.page_length, vd.point_size);
672
673 // allow to be called without entries (when loading pages from disk)
database/engine/rrddim_eng.h deleted
-73
@@ -1,73 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_RRDDIM_ENG_H
4 -#define NETDATA_RRDDIM_ENG_H
5 -
6 -#include "database/storage_engine_types.h"
7 -
8 -STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
9 -
10 -void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
11 -
12 -STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
13 -void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value, uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
14 -void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
15 -void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
16 -
17 -size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
18 -
19 -size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
20 -
21 -STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
22 -STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
23 -void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
24 -STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);
25 -
26 -
27 -void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle, time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
28 -STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
29 -int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
30 -void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
31 -time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
32 -
33 -time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
34 -time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
35 -
36 -
37 -bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s);
38 -
39 -time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *db_instance);
40 -
41 -int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
42 -
43 -size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
44 -
45 -time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
46 -time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
47 -
48 -int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
49 -void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
50 -time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
51 -
52 -// The following functions are dbengine-specific. Include them here until
53 -// we refactor the code so that, either we don't need them, or settle to a
54 -// common API.
55 -
56 -bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance);
57 -
58 -void rrdeng_get_37_statistics(STORAGE_INSTANCE *db_instance, unsigned long long *array);
59 -
60 -int rrdeng_init(STORAGE_INSTANCE **db_instance, const char *dbfiles_path,
61 - unsigned disk_space_mb, size_t tier);
62 -
63 -void rrdeng_readiness_wait(STORAGE_INSTANCE *db_instance);
64 -void rrdeng_exit_mode(STORAGE_INSTANCE *db_instance);
65 -
66 -int rrdeng_exit(STORAGE_INSTANCE *ctx);
67 -void rrdeng_prepare_exit(STORAGE_INSTANCE *ctx);
68 -
69 -size_t rrdeng_collectors_running(STORAGE_INSTANCE *db_instance);
70 -
71 -void rrdeng_size_statistics(STORAGE_INSTANCE *db_instance, BUFFER *wb);
72 -
73 -#endif /* NETDATA_RRDDIM_ENG_H */
database/engine/rrdengine.c
+11 -14
@@ -1,4 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS
3
4 #include "rrdengine.h"
5 #include "pdc.h"
@@ -156,15 +157,11 @@ enum LIBUV_WORKERS_STATUS {
157
158 static inline enum LIBUV_WORKERS_STATUS work_request_full(void) {
159 size_t dispatched = __atomic_load_n(&rrdeng_main.work_cmd.atomics.dispatched, __ATOMIC_RELAXED);
159 - int reserved_libuv_worker_threads = 3;
160
161 - internal_fatal(rrdb.libuv_worker_threads <= reserved_libuv_worker_threads,
162 - "libuv worker threads below minimum required");
163 -
164 - if(dispatched >= (size_t)(rrdb.libuv_worker_threads))
161 + if(dispatched >= (size_t)(libuv_worker_threads))
162 return LIBUV_WORKERS_CRITICAL;
163
167 - else if(dispatched >= (size_t)(rrdb.libuv_worker_threads - reserved_libuv_worker_threads))
164 + else if(dispatched >= (size_t)(libuv_worker_threads - RESERVED_LIBUV_WORKER_THREADS))
165 return LIBUV_WORKERS_STRESSED;
166
167 return LIBUV_WORKERS_RELAXED;
@@ -357,7 +354,7 @@ static void wal_cleanup1(void) {
354 if(!spinlock_trylock(&wal_globals.protected.spinlock))
355 return;
356
360 - if (wal_globals.protected.available_items && wal_globals.protected.available > rrdb.storage_tiers) {
357 + if(wal_globals.protected.available_items && wal_globals.protected.available > storage_tiers) {
358 wal = wal_globals.protected.available_items;
359 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
360 wal_globals.protected.available--;
@@ -585,25 +582,25 @@ struct {
582 } dbengine_page_alloc_globals = {};
583
584 static inline ARAL *page_size_lookup(size_t size) {
588 - for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
589 - if(size == rrdb.tier_page_size[tier])
585 + for(size_t tier = 0; tier < storage_tiers ;tier++)
586 + if(size == tier_page_size[tier])
587 return dbengine_page_alloc_globals.aral[tier];
588
589 return NULL;
590 }
591
592 static void dbengine_page_alloc_init(void) {
596 - for (size_t i = rrdb.storage_tiers; i > 0 ;i--) {
597 - size_t tier = rrdb.storage_tiers - i;
593 + for(size_t i = storage_tiers; i > 0 ;i--) {
594 + size_t tier = storage_tiers - i;
595
596 char buf[20 + 1];
597 snprintfz(buf, 20, "tier%zu-pages", tier);
598
599 dbengine_page_alloc_globals.aral[tier] = aral_create(
600 buf,
604 - rrdb.tier_page_size[tier],
601 + tier_page_size[tier],
602 64,
606 - 512 * rrdb.tier_page_size[tier],
603 + 512 * tier_page_size[tier],
604 pgc_aral_statistics(),
605 NULL, NULL, false, false);
606 }
@@ -1787,7 +1784,7 @@ void dbengine_event_loop(void* arg) {
1784 }
1785
1786 case RRDENG_OPCODE_FLUSH_INIT: {
1790 - if(rrdeng_main.flushes_running < (size_t)(rrdb.libuv_worker_threads / 4)) {
1787 + if(rrdeng_main.flushes_running < (size_t)(libuv_worker_threads / 4)) {
1788 rrdeng_main.flushes_running++;
1789 work_dispatch(NULL, NULL, NULL, opcode, cache_flush_tp_worker, after_do_cache_flush);
1790 }
database/engine/rrdengine.h
+2
@@ -184,6 +184,8 @@ typedef enum __attribute__ ((__packed__)) {
184 } RRDENG_COLLECT_PAGE_FLAGS;
185
186 struct rrdeng_collect_handle {
187 + struct storage_collect_handle common; // has to be first item
188 +
189 RRDENG_COLLECT_PAGE_FLAGS page_flags;
190 RRDENG_COLLECT_HANDLE_OPTIONS options;
191 uint8_t type;
database/engine/rrdengineapi.c
+51 -104
@@ -13,20 +13,40 @@ struct rrdengine_instance multidb_ctx_storage_tier4;
13 #if RRD_STORAGE_TIERS != 5
14 #error RRD_STORAGE_TIERS is not 5 - you need to add allocations here
15 #endif
16 +struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
17 uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER};
18
19 +#if defined(ENV32BIT)
20 +size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};
21 +#else
22 +size_t tier_page_size[RRD_STORAGE_TIERS] = {4096, 2048, 384, 384, 384};
23 +#endif
24 +
25 #if PAGE_TYPE_MAX != 1
26 #error PAGE_TYPE_MAX is not 1 - you need to add allocations here
27 #endif
28 +size_t page_type_size[256] = {sizeof(storage_number), sizeof(storage_number_tier1_t)};
29
30 __attribute__((constructor)) void initialize_multidb_ctx(void) {
23 - rrdb.multidb_ctx[0] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier0;
24 - rrdb.multidb_ctx[1] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier1;
25 - rrdb.multidb_ctx[2] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier2;
26 - rrdb.multidb_ctx[3] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier3;
27 - rrdb.multidb_ctx[4] = (STORAGE_INSTANCE *) &multidb_ctx_storage_tier4;
31 + multidb_ctx[0] = &multidb_ctx_storage_tier0;
32 + multidb_ctx[1] = &multidb_ctx_storage_tier1;
33 + multidb_ctx[2] = &multidb_ctx_storage_tier2;
34 + multidb_ctx[3] = &multidb_ctx_storage_tier3;
35 + multidb_ctx[4] = &multidb_ctx_storage_tier4;
36 }
37
38 +int db_engine_journal_check = 0;
39 +int default_rrdeng_disk_quota_mb = 256;
40 +int default_multidb_disk_quota_mb = 256;
41 +
42 +#if defined(ENV32BIT)
43 +int default_rrdeng_page_cache_mb = 16;
44 +int default_rrdeng_extent_cache_mb = 0;
45 +#else
46 +int default_rrdeng_page_cache_mb = 32;
47 +int default_rrdeng_extent_cache_mb = 0;
48 +#endif
49 +
50 // ----------------------------------------------------------------------------
51 // metrics groups
52
@@ -65,7 +85,7 @@ void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance __maybe_unused,
85 // metric handle for legacy dbs
86
87 /* This UUID is not unique across hosts */
68 -static void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
88 +void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
89 {
90 EVP_MD_CTX *evpctx;
91 unsigned char hash_value[EVP_MAX_MD_SIZE];
@@ -235,6 +255,7 @@ STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metri
255 struct rrdeng_collect_handle *handle;
256
257 handle = callocz(1, sizeof(struct rrdeng_collect_handle));
258 + handle->common.backend = STORAGE_ENGINE_BACKEND_DBENGINE;
259 handle->metric = metric;
260 handle->page = NULL;
261 handle->data = NULL;
@@ -423,7 +444,7 @@ static size_t aligned_allocation_entries(size_t max_slots, size_t target_slot, t
444 static void *rrdeng_alloc_new_metric_data(struct rrdeng_collect_handle *handle, size_t *data_size, usec_t point_in_time_ut) {
445 struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
446
426 - size_t max_size = rrdb.tier_page_size[ctx->config.tier];
447 + size_t max_size = tier_page_size[ctx->config.tier];
448 size_t max_slots = max_size / CTX_POINT_SIZE_BYTES(ctx);
449
450 size_t slots = aligned_allocation_entries(
@@ -443,7 +464,7 @@ static void *rrdeng_alloc_new_metric_data(struct rrdeng_collect_handle *handle,
464 // internal_error(true, "PAGE ALLOC %zu bytes (%zu max)", size, max_size);
465
466 internal_fatal(slots < 3 || slots > max_slots, "ooops! wrong distribution of metrics across time");
446 - internal_fatal(size > rrdb.tier_page_size[ctx->config.tier] || size < CTX_POINT_SIZE_BYTES(ctx) * 2, "ooops! wrong page size");
467 + internal_fatal(size > tier_page_size[ctx->config.tier] || size < CTX_POINT_SIZE_BYTES(ctx) * 2, "ooops! wrong page size");
468
469 *data_size = size;
470 void *d = dbengine_page_alloc(size);
@@ -746,14 +767,15 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle,
767
768 handle->dt_s = db_update_every_s;
769 if (!handle->dt_s) {
749 - handle->dt_s = rrdb.default_update_every;
750 - mrg_metric_set_update_every_s_if_zero(main_mrg, metric, rrdb.default_update_every);
770 + handle->dt_s = default_rrd_update_every;
771 + mrg_metric_set_update_every_s_if_zero(main_mrg, metric, default_rrd_update_every);
772 }
773
774 rrddim_handle->handle = (STORAGE_QUERY_HANDLE *) handle;
775 rrddim_handle->start_time_s = handle->start_time_s;
776 rrddim_handle->end_time_s = handle->end_time_s;
777 rrddim_handle->priority = priority;
778 + rrddim_handle->backend = STORAGE_ENGINE_BACKEND_DBENGINE;
779
780 pg_cache_preload(handle);
781
@@ -769,6 +791,7 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle,
791 rrddim_handle->start_time_s = handle->start_time_s;
792 rrddim_handle->end_time_s = 0;
793 rrddim_handle->priority = priority;
794 + rrddim_handle->backend = STORAGE_ENGINE_BACKEND_DBENGINE;
795 }
796 }
797
@@ -1005,10 +1028,8 @@ size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance) {
1028 * You must not change the indices of the statistics or user code will break.
1029 * You must not exceed RRDENG_NR_STATS or it will crash.
1030 */
1008 -void rrdeng_get_37_statistics(STORAGE_INSTANCE *db_instance, unsigned long long *array)
1031 +void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
1032 {
1010 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1011 -
1033 if (ctx == NULL)
1034 return;
1035
@@ -1061,12 +1082,12 @@ static void rrdeng_populate_mrg(struct rrdengine_instance *ctx) {
1082 datafiles++;
1083 uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
1084
1064 - ssize_t cpus = (ssize_t) get_netdata_cpus() / rrdb.storage_tiers;
1085 + ssize_t cpus = (ssize_t)get_netdata_cpus() / (ssize_t)storage_tiers;
1086 if(cpus > (ssize_t)datafiles)
1087 cpus = (ssize_t)datafiles;
1088
1068 - if(cpus > (ssize_t) rrdb.libuv_worker_threads)
1069 - cpus = (ssize_t) rrdb.libuv_worker_threads;
1089 + if(cpus > (ssize_t)libuv_worker_threads)
1090 + cpus = (ssize_t)libuv_worker_threads;
1091
1092 if(cpus >= (ssize_t)get_netdata_cpus() / 2)
1093 cpus = (ssize_t)(get_netdata_cpus() / 2 - 1);
@@ -1105,9 +1126,7 @@ static void rrdeng_populate_mrg(struct rrdengine_instance *ctx) {
1126 }
1127 }
1128
1108 -void rrdeng_readiness_wait(STORAGE_INSTANCE *db_instance) {
1109 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1110 -
1129 +void rrdeng_readiness_wait(struct rrdengine_instance *ctx) {
1130 for (size_t i = 0; i < ctx->loading.populate_mrg.size; i++) {
1131 completion_wait_for(&ctx->loading.populate_mrg.array[i]);
1132 completion_destroy(&ctx->loading.populate_mrg.array[i]);
@@ -1125,17 +1144,14 @@ bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance) {
1144 return ctx->config.legacy;
1145 }
1146
1128 -void rrdeng_exit_mode(STORAGE_INSTANCE *db_instance) {
1129 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1147 +void rrdeng_exit_mode(struct rrdengine_instance *ctx) {
1148 __atomic_store_n(&ctx->quiesce.exit_mode, true, __ATOMIC_RELAXED);
1149 }
1150 /*
1151 * Returns 0 on success, negative on error
1152 */
1135 -int rrdeng_init(STORAGE_INSTANCE **db_instance_ptr, const char *dbfiles_path,
1153 +int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
1154 unsigned disk_space_mb, size_t tier) {
1137 - struct rrdengine_instance **ctxp = (struct rrdengine_instance **) db_instance_ptr;
1138 -
1155 struct rrdengine_instance *ctx;
1156 uint32_t max_open_files;
1157
@@ -1155,7 +1171,7 @@ int rrdeng_init(STORAGE_INSTANCE **db_instance_ptr, const char *dbfiles_path,
1171 }
1172
1173 if(NULL == ctxp) {
1158 - ctx = (struct rrdengine_instance *) rrdb.multidb_ctx[tier];
1174 + ctx = multidb_ctx[tier];
1175 memset(ctx, 0, sizeof(*ctx));
1176 ctx->config.legacy = false;
1177 }
@@ -1195,18 +1211,15 @@ int rrdeng_init(STORAGE_INSTANCE **db_instance_ptr, const char *dbfiles_path,
1211 return UV_EIO;
1212 }
1213
1198 -size_t rrdeng_collectors_running(STORAGE_INSTANCE *db_instance) {
1199 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1214 +size_t rrdeng_collectors_running(struct rrdengine_instance *ctx) {
1215 return __atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED);
1216 }
1217
1218 /*
1219 * Returns 0 on success, 1 on error
1220 */
1206 -int rrdeng_exit(STORAGE_INSTANCE *db_instance) {
1207 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1208 -
1209 - if (ctx == NULL)
1221 +int rrdeng_exit(struct rrdengine_instance *ctx) {
1222 + if (NULL == ctx)
1223 return 1;
1224
1225 // FIXME - ktsaou - properly cleanup ctx
@@ -1216,7 +1229,7 @@ int rrdeng_exit(STORAGE_INSTANCE *db_instance) {
1229 // 4. then wait for completion
1230
1231 bool logged = false;
1219 - while (__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && !rrdb.unittest_running) {
1232 + while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && !unittest_running) {
1233 if(!logged) {
1234 netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", (ctx->config.legacy) ? -1 : ctx->config.tier);
1235 logged = true;
@@ -1243,9 +1256,7 @@ int rrdeng_exit(STORAGE_INSTANCE *db_instance) {
1256 return 0;
1257 }
1258
1246 -void rrdeng_prepare_exit(STORAGE_INSTANCE *db_instance) {
1247 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1248 -
1259 +void rrdeng_prepare_exit(struct rrdengine_instance *ctx) {
1260 if (NULL == ctx)
1261 return;
1262
@@ -1295,7 +1306,7 @@ static void populate_v2_statistics(struct rrdengine_datafile *datafile, RRDENG_S
1306 if(likely(points > 1))
1307 update_every_s = (time_t) ((end_time_s - start_time_s) / (points - 1));
1308 else {
1298 - update_every_s = (time_t) (rrdb.default_update_every * get_tier_grouping(datafile->ctx->config.tier));
1309 + update_every_s = (time_t) (default_rrd_update_every * get_tier_grouping(datafile->ctx->config.tier));
1310 stats->single_point_pages++;
1311 }
1312
@@ -1324,7 +1335,7 @@ static void populate_v2_statistics(struct rrdengine_datafile *datafile, RRDENG_S
1335 journalfile_v2_data_release(datafile->journalfile);
1336 }
1337
1327 -static RRDENG_SIZE_STATS rrdeng_size_statistics_internal(struct rrdengine_instance *ctx) {
1338 +RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
1339 RRDENG_SIZE_STATS stats = { 0 };
1340
1341 uv_rwlock_rdlock(&ctx->datafiles.rwlock);
@@ -1370,82 +1381,18 @@ static RRDENG_SIZE_STATS rrdeng_size_statistics_internal(struct rrdengine_instan
1381 // stats.sizeof_metric = 0;
1382 stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));
1383 stats.sizeof_page_in_cache = 0; // struct_natural_alignment(sizeof(struct page_cache_descr));
1373 - stats.sizeof_point_data = rrdb.page_type_size[ctx->config.page_type];
1374 - stats.sizeof_page_data = rrdb.tier_page_size[ctx->config.tier];
1384 + stats.sizeof_point_data = page_type_size[ctx->config.page_type];
1385 + stats.sizeof_page_data = tier_page_size[ctx->config.tier];
1386 stats.pages_per_extent = rrdeng_pages_per_extent;
1387
1388 // stats.sizeof_metric_in_index = 40;
1389 // stats.sizeof_page_in_index = 24;
1390
1380 - stats.default_granularity_secs = (size_t) rrdb.default_update_every * get_tier_grouping(ctx->config.tier);
1391 + stats.default_granularity_secs = (size_t)default_rrd_update_every * get_tier_grouping(ctx->config.tier);
1392
1393 return stats;
1394 }
1395
1385 -void rrdeng_size_statistics(STORAGE_INSTANCE *db_instance, BUFFER *wb) {
1386 - struct rrdengine_instance *ctx = (struct rrdengine_instance *) db_instance;
1387 - RRDENG_SIZE_STATS stats = rrdeng_size_statistics_internal(ctx);
1388 -
1389 - buffer_sprintf(wb,
1390 - "\n\t\t\"default_granularity_secs\":%zu"
1391 - ",\n\t\t\"sizeof_datafile\":%zu"
1392 - ",\n\t\t\"sizeof_page_in_cache\":%zu"
1393 - ",\n\t\t\"sizeof_point_data\":%zu"
1394 - ",\n\t\t\"sizeof_page_data\":%zu"
1395 - ",\n\t\t\"pages_per_extent\":%zu"
1396 - ",\n\t\t\"datafiles\":%zu"
1397 - ",\n\t\t\"extents\":%zu"
1398 - ",\n\t\t\"extents_pages\":%zu"
1399 - ",\n\t\t\"points\":%zu"
1400 - ",\n\t\t\"metrics\":%zu"
1401 - ",\n\t\t\"metrics_pages\":%zu"
1402 - ",\n\t\t\"extents_compressed_bytes\":%zu"
1403 - ",\n\t\t\"pages_uncompressed_bytes\":%zu"
1404 - ",\n\t\t\"pages_duration_secs\":%lld"
1405 - ",\n\t\t\"single_point_pages\":%zu"
1406 - ",\n\t\t\"first_t\":%ld"
1407 - ",\n\t\t\"last_t\":%ld"
1408 - ",\n\t\t\"database_retention_secs\":%lld"
1409 - ",\n\t\t\"average_compression_savings\":%0.2f"
1410 - ",\n\t\t\"average_point_duration_secs\":%0.2f"
1411 - ",\n\t\t\"average_metric_retention_secs\":%0.2f"
1412 - ",\n\t\t\"ephemeral_metrics_per_day_percent\":%0.2f"
1413 - ",\n\t\t\"average_page_size_bytes\":%0.2f"
1414 - ",\n\t\t\"estimated_concurrently_collected_metrics\":%zu"
1415 - ",\n\t\t\"currently_collected_metrics\":%zu"
1416 - ",\n\t\t\"disk_space\":%zu"
1417 - ",\n\t\t\"max_disk_space\":%zu"
1418 - , stats.default_granularity_secs
1419 - , stats.sizeof_datafile
1420 - , stats.sizeof_page_in_cache
1421 - , stats.sizeof_point_data
1422 - , stats.sizeof_page_data
1423 - , stats.pages_per_extent
1424 - , stats.datafiles
1425 - , stats.extents
1426 - , stats.extents_pages
1427 - , stats.points
1428 - , stats.metrics
1429 - , stats.metrics_pages
1430 - , stats.extents_compressed_bytes
1431 - , stats.pages_uncompressed_bytes
1432 - , (long long)stats.pages_duration_secs
1433 - , stats.single_point_pages
1434 - , stats.first_time_s
1435 - , stats.last_time_s
1436 - , (long long)stats.database_retention_secs
1437 - , stats.average_compression_savings
1438 - , stats.average_point_duration_secs
1439 - , stats.average_metric_retention_secs
1440 - , stats.ephemeral_metrics_per_day_percent
1441 - , stats.average_page_size_bytes
1442 - , stats.estimated_concurrently_collected_metrics
1443 - , stats.currently_collected_metrics
1444 - , stats.disk_space
1445 - , stats.max_disk_space
1446 - );
1447 -}
1448 -
1396 struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void) {
1397 // FIXME - make cache efficiency stats atomic
1398 return rrdeng_cache_efficiency_stats;
database/engine/rrdengineapi.h
+62 -1
@@ -12,7 +12,61 @@
12
13 #define RRDENG_FD_BUDGET_PER_INSTANCE (50)
14
15 -#define CTX_POINT_SIZE_BYTES(ctx) rrdb.page_type_size[(ctx)->config.page_type]
15 +extern int default_rrdeng_page_cache_mb;
16 +extern int default_rrdeng_extent_cache_mb;
17 +extern int db_engine_journal_check;
18 +extern int default_rrdeng_disk_quota_mb;
19 +extern int default_multidb_disk_quota_mb;
20 +extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
21 +extern size_t page_type_size[];
22 +extern size_t tier_page_size[];
23 +
24 +#define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type]
25 +
26 +void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid);
27 +
28 +STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
29 +STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
30 +void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
31 +STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);
32 +
33 +STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
34 +void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
35 +void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
36 +void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
37 + NETDATA_DOUBLE min_value,
38 + NETDATA_DOUBLE max_value,
39 + uint16_t count,
40 + uint16_t anomaly_count,
41 + SN_FLAGS flags);
42 +int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
43 +
44 +void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle,
45 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
46 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
47 +
48 +
49 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
50 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
51 +time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
52 +time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
53 +time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
54 +
55 +void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
56 +
57 +/* must call once before using anything */
58 +int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
59 + unsigned disk_space_mb, size_t tier);
60 +
61 +void rrdeng_readiness_wait(struct rrdengine_instance *ctx);
62 +void rrdeng_exit_mode(struct rrdengine_instance *ctx);
63 +
64 +int rrdeng_exit(struct rrdengine_instance *ctx);
65 +void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
66 +bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s);
67 +
68 +extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
69 +extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
70
71 typedef struct rrdengine_size_statistics {
72 size_t default_granularity_secs;
@@ -164,4 +218,11 @@ struct rrdeng_buffer_sizes {
218 struct rrdeng_buffer_sizes rrdeng_get_buffer_sizes(void);
219 struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void);
220
221 +RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
222 +size_t rrdeng_collectors_running(struct rrdengine_instance *ctx);
223 +bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance);
224 +
225 +size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
226 +size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
227 +
228 #endif /* NETDATA_RRDENGINEAPI_H */
database/engine/rrdenginelib.c
+2 -2
@@ -142,7 +142,7 @@ int compute_multidb_diskspace()
142 if (computed_multidb_disk_quota_mb == -1) {
143 int rc = count_legacy_children(netdata_configured_cache_dir);
144 if (likely(rc >= 0)) {
145 - computed_multidb_disk_quota_mb = (rc + 1) * rrdb.default_rrdeng_disk_quota_mb;
145 + computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
146 netdata_log_info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
147
148 fp = fopen(multidb_disk_space_file, "w");
@@ -154,7 +154,7 @@ int compute_multidb_diskspace()
154 netdata_log_error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
155 }
156 else
157 - computed_multidb_disk_quota_mb = rrdb.default_rrdeng_disk_quota_mb;
157 + computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
158 }
159
160 return computed_multidb_disk_quota_mb;
database/ram/rrddim_mem.c
+3 -22
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "rrddim_mem.h"
4 -#include "database/rrd.h"
4 +#include "Judy.h"
5
6 static Pvoid_t rrddim_JudyHS_array = NULL;
7 static netdata_rwlock_t rrddim_JudyHS_rwlock = NETDATA_RWLOCK_INITIALIZER;
@@ -143,6 +143,7 @@ STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_han
143 internal_fatal((uint32_t)mh->update_every_s != update_every, "RRDDIM: update requested does not match the dimension");
144
145 struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
146 + ch->common.backend = STORAGE_ENGINE_BACKEND_RRDDIM;
147 ch->rd = rd;
148 ch->db_metric_handle = db_metric_handle;
149
@@ -340,7 +341,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_e
341 handle->start_time_s = start_time_s;
342 handle->end_time_s = end_time_s;
343 handle->priority = priority;
343 -
344 + handle->backend = STORAGE_ENGINE_BACKEND_RRDDIM;
345 struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
346 h->db_metric_handle = db_metric_handle;
347
@@ -434,23 +435,3 @@ time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle) {
435 struct mem_metric_handle *mh = (struct mem_metric_handle *)db_metric_handle;
436 return (time_t)(mh->last_updated_s - metric_duration(mh));
437 }
437 -
438 -size_t rrddim_disk_space_max(STORAGE_INSTANCE *db_instance) {
439 - UNUSED(db_instance);
440 - return 0;
441 -}
442 -
443 -size_t rrddim_disk_space_used(STORAGE_INSTANCE *db_instance) {
444 - UNUSED(db_instance);
445 - return 0;
446 -}
447 -
448 -time_t rrddim_global_first_time_s(STORAGE_INSTANCE *db_instance) {
449 - UNUSED(db_instance);
450 - return now_realtime_sec() - (time_t)(rrdb.default_rrd_history_entries * rrdb.default_update_every);
451 -}
452 -
453 -size_t rrddim_currently_collected_metrics(STORAGE_INSTANCE *db_instance) {
454 - UNUSED(db_instance);
455 - return 0;
456 -}
database/ram/rrddim_mem.h
+10 -27
@@ -3,9 +3,11 @@
3 #ifndef NETDATA_RRDDIMMEM_H
4 #define NETDATA_RRDDIMMEM_H
5
6 -#include "database/storage_engine_types.h"
6 +#include "database/rrd.h"
7
8 struct mem_collect_handle {
9 + struct storage_collect_handle common; // has to be first item
10 +
11 STORAGE_METRIC_HANDLE *db_metric_handle;
12 RRDDIM *rd;
13 };
@@ -27,6 +29,11 @@ void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
29
30 bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
31
32 +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
33 +void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
34 +
35 +STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
36 +void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
37 void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
38 NETDATA_DOUBLE min_value,
39 NETDATA_DOUBLE max_value,
@@ -36,36 +43,12 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
43 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
44 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
45
39 -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
40 -
41 -void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
42 -
43 -STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
44 -
45 -void rrddim_collect_store_metric(
46 - STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
47 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
48 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
49 -
50 -
51 -void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
52 -
53 -int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
54 -
55 -void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
56 -time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
57 -time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
46 void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle, time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
47 STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle);
48 int rrddim_query_is_finished(struct storage_engine_query_handle *handle);
49 void rrddim_query_finalize(struct storage_engine_query_handle *handle);
50 +time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
51 +time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
52 time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
53
64 -size_t rrddim_disk_space_max(STORAGE_INSTANCE *db_instance);
65 -size_t rrddim_disk_space_used(STORAGE_INSTANCE *db_instance);
66 -
67 -time_t rrddim_global_first_time_s(STORAGE_INSTANCE *db_instance);
68 -
69 -size_t rrddim_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
70 -
54 #endif
database/rrd.c
+87 -354
@@ -1,6 +1,70 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS 1
3
4 #include "rrd.h"
5 +#include "storage_engine.h"
6 +
7 +// ----------------------------------------------------------------------------
8 +// globals
9 +
10 +/*
11 +// if not zero it gives the time (in seconds) to remove un-updated dimensions
12 +// DO NOT ENABLE
13 +// if dimensions are removed, the chart generation will have to run again
14 +int rrd_delete_unupdated_dimensions = 0;
15 +*/
16 +
17 +int default_rrd_update_every = UPDATE_EVERY;
18 +int default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
19 +#ifdef ENABLE_DBENGINE
20 +RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
21 +#else
22 +RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
23 +#endif
24 +int gap_when_lost_iterations_above = 1;
25 +
26 +
27 +// ----------------------------------------------------------------------------
28 +// RRD - memory modes
29 +
30 +inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) {
31 + switch(id) {
32 + case RRD_MEMORY_MODE_RAM:
33 + return RRD_MEMORY_MODE_RAM_NAME;
34 +
35 + case RRD_MEMORY_MODE_MAP:
36 + return RRD_MEMORY_MODE_MAP_NAME;
37 +
38 + case RRD_MEMORY_MODE_NONE:
39 + return RRD_MEMORY_MODE_NONE_NAME;
40 +
41 + case RRD_MEMORY_MODE_SAVE:
42 + return RRD_MEMORY_MODE_SAVE_NAME;
43 +
44 + case RRD_MEMORY_MODE_ALLOC:
45 + return RRD_MEMORY_MODE_ALLOC_NAME;
46 +
47 + case RRD_MEMORY_MODE_DBENGINE:
48 + return RRD_MEMORY_MODE_DBENGINE_NAME;
49 + }
50 +
51 + STORAGE_ENGINE* eng = storage_engine_get(id);
52 + if (eng) {
53 + return eng->name;
54 + }
55 +
56 + return RRD_MEMORY_MODE_SAVE_NAME;
57 +}
58 +
59 +RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) {
60 + STORAGE_ENGINE* eng = storage_engine_find(name);
61 + if (eng) {
62 + return eng->id;
63 + }
64 +
65 + return RRD_MEMORY_MODE_SAVE;
66 +}
67 +
68
69 // ----------------------------------------------------------------------------
70 // RRD - algorithms types
@@ -43,7 +107,7 @@ const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) {
107 // ----------------------------------------------------------------------------
108 // RRD - chart types
109
46 -RRDSET_TYPE rrdset_type_id(const char *name) {
110 +inline RRDSET_TYPE rrdset_type_id(const char *name) {
111 if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))
112 return RRDSET_TYPE_AREA;
113
@@ -69,367 +133,36 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type) {
133 }
134
135 // ----------------------------------------------------------------------------
72 -// RRD - string management
73 -
74 -STRING *rrd_string_strdupz(const char *s) {
75 - if(unlikely(!s || !*s)) return string_strdupz(s);
76 -
77 - char *tmp = strdupz(s);
78 - json_fix_string(tmp);
79 - STRING *ret = string_strdupz(tmp);
80 - freez(tmp);
81 - return ret;
82 -}
136 +// RRD - cache directory
137
84 -// ----------------------------------------------------------------------------
85 -// rrd global / startup initialization
138 +char *rrdhost_cache_dir_for_rrdset_alloc(RRDHOST *host, const char *id) {
139 + char *ret = NULL;
140
87 -#ifdef ENABLE_DBENGINE
88 -struct dbengine_initialization {
89 - netdata_thread_t thread;
90 - char path[FILENAME_MAX + 1];
91 - int disk_space_mb;
92 - size_t tier;
93 - int ret;
94 -};
95 -
96 -static void *dbengine_tier_init(void *ptr) {
97 - struct dbengine_initialization *dbi = ptr;
98 - dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);
99 - return ptr;
100 -}
101 -#endif
141 + char b[FILENAME_MAX + 1];
142 + char n[FILENAME_MAX + 1];
143 + rrdset_strncpyz_name(b, id, FILENAME_MAX);
144
103 -static void dbengine_init(const char *hostname) {
104 -#ifdef ENABLE_DBENGINE
105 - rrdb.use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", rrdb.use_direct_io);
106 -
107 - unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
108 - if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
109 - rrdeng_pages_per_extent = read_num;
110 - else {
111 - netdata_log_error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
112 - config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
113 - }
145 + snprintfz(n, FILENAME_MAX, "%s/%s", host->cache_dir, b);
146 + ret = strdupz(n);
147
115 - rrdb.storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
116 - if (rrdb.storage_tiers < 1) {
117 - netdata_log_error("At least 1 storage tier is required. Assuming 1.");
118 - rrdb.storage_tiers = 1;
119 - config_set_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
120 - }
121 - if (rrdb.storage_tiers > RRD_STORAGE_TIERS) {
122 - netdata_log_error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS);
123 - rrdb.storage_tiers = RRD_STORAGE_TIERS;
124 - config_set_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
148 + if(host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
149 + int r = mkdir(ret, 0775);
150 + if(r != 0 && errno != EEXIST)
151 + netdata_log_error("Cannot create directory '%s'", ret);
152 }
153
127 - bool parallel_initialization = (rrdb.storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
128 - parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
129 -
130 - struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
131 -
132 - size_t created_tiers = 0;
133 - char dbenginepath[FILENAME_MAX + 1];
134 - char dbengineconfig[200 + 1];
135 - int divisor = 1;
136 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
137 - if(tier == 0)
138 - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
139 - else
140 - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
141 -
142 - int ret = mkdir(dbenginepath, 0775);
143 - if (ret != 0 && errno != EEXIST) {
144 - netdata_log_error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
145 - break;
146 - }
147 -
148 - if(tier > 0)
149 - divisor *= 2;
150 -
151 - int disk_space_mb = rrdb.default_multidb_disk_quota_mb / divisor;
152 - size_t grouping_iterations = rrdb.storage_tiers_grouping_iterations[tier];
153 - RRD_BACKFILL backfill = rrdb.storage_tiers_backfill[tier];
154 -
155 - if(tier > 0) {
156 - snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
157 - disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
158 -
159 - snprintfz(dbengineconfig, 200, "dbengine tier %zu update every iterations", tier);
160 - grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
161 - if(grouping_iterations < 2) {
162 - grouping_iterations = 2;
163 - config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
164 - netdata_log_error("DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",
165 - hostname,
166 - tier);
167 - }
168 -
169 - snprintfz(dbengineconfig, 200, "dbengine tier %zu backfill", tier);
170 - const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
171 - if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
172 - else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
173 - else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;
174 - else {
175 - netdata_log_error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf);
176 - config_set(CONFIG_SECTION_DB, dbengineconfig, "new");
177 - backfill = RRD_BACKFILL_NEW;
178 - }
179 - }
180 -
181 - rrdb.storage_tiers_grouping_iterations[tier] = grouping_iterations;
182 - rrdb.storage_tiers_backfill[tier] = backfill;
183 -
184 - if(tier > 0 && get_tier_grouping(tier) > 65535) {
185 - rrdb.storage_tiers_grouping_iterations[tier] = 1;
186 - netdata_log_error("DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. Disabling tiers above %zu",
187 - hostname,
188 - tier,
189 - tier);
190 - break;
191 - }
192 -
193 - internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, rrdb.storage_tiers_grouping_iterations[tier]);
194 -
195 - tiers_init[tier].disk_space_mb = disk_space_mb;
196 - tiers_init[tier].tier = tier;
197 - strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
198 - tiers_init[tier].ret = 0;
199 -
200 - if(parallel_initialization) {
201 - char tag[NETDATA_THREAD_TAG_MAX + 1];
202 - snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier);
203 - netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE,
204 - dbengine_tier_init, &tiers_init[tier]);
205 - }
206 - else
207 - dbengine_tier_init(&tiers_init[tier]);
208 - }
209 -
210 - for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
211 - void *ptr;
212 -
213 - if(parallel_initialization)
214 - netdata_thread_join(tiers_init[tier].thread, &ptr);
215 -
216 - if(tiers_init[tier].ret != 0) {
217 - netdata_log_error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
218 - hostname,
219 - tiers_init[tier].tier,
220 - tiers_init[tier].path);
221 - }
222 - else if(created_tiers == tier)
223 - created_tiers++;
224 - }
225 -
226 - if(created_tiers && created_tiers < rrdb.storage_tiers) {
227 - netdata_log_error("DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.",
228 - hostname,
229 - created_tiers,
230 - rrdb.storage_tiers,
231 - created_tiers);
232 - rrdb.storage_tiers = created_tiers;
233 - }
234 - else if(!created_tiers)
235 - fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
236 -
237 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++)
238 - rrdeng_readiness_wait(rrdb.multidb_ctx[tier]);
239 -
240 - rrdb.dbengine_enabled = true;
241 -#else
242 - rrdb.storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
243 - if(rrdb.storage_tiers != 1) {
244 - netdata_log_error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname);
245 - rrdb.storage_tiers = 1;
246 - config_set_number(CONFIG_SECTION_DB, "storage tiers", rrdb.storage_tiers);
247 - }
248 - rrdb.dbengine_enabled = false;
249 -#endif
250 -}
251 -
252 -static void init_host_indexes() {
253 - internal_fatal(rrdb.rrdhost_root_index || rrdb.rrdhost_root_index_hostname,
254 - "Host indexes have already been initialized");
255 -
256 - DICT_OPTIONS dict_opts = DICT_OPTION_NAME_LINK_DONT_CLONE |
257 - DICT_OPTION_VALUE_LINK_DONT_CLONE |
258 - DICT_OPTION_DONT_OVERWRITE_VALUE;
259 -
260 - rrdb.rrdhost_root_index = dictionary_create_advanced(dict_opts, &dictionary_stats_category_rrdhost, 0);
261 - rrdb.rrdhost_root_index_hostname = dictionary_create_advanced(dict_opts, &dictionary_stats_category_rrdhost, 0);
154 + return ret;
155 }
156
264 -int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
265 - init_host_indexes();
266 -
267 - if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
268 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE) {
269 - set_late_global_environment(system_info);
270 - fatal("Failed to initialize SQLite");
271 - }
272 - netdata_log_info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
273 - }
274 -
275 - if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
276 - error_report("Failed to initialize context metadata database");
277 - }
278 -
279 - if (unlikely(unittest)) {
280 - rrdb.dbengine_enabled = true;
281 - }
282 - else {
283 - health_init();
284 - rrdpush_init();
285 -
286 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
287 - netdata_log_info("DBENGINE: Initializing ...");
288 - dbengine_init(hostname);
289 - }
290 - else {
291 - netdata_log_info("DBENGINE: Not initializing ...");
292 - rrdb.storage_tiers = 1;
293 - }
294 -
295 - if (!rrdb.dbengine_enabled) {
296 - if (rrdb.storage_tiers > 1) {
297 - netdata_log_error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
298 - rrdb.storage_tiers);
299 - rrdb.storage_tiers = 1;
300 - }
301 -
302 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE) {
303 - netdata_log_error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
304 - default_storage_engine_id = STORAGE_ENGINE_ALLOC;
305 - }
306 - }
307 - }
308 -
309 - if(!unittest)
310 - metadata_sync_init();
311 -
312 - netdata_log_debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
313 - rrdb.localhost = rrdhost_create(
314 - hostname
315 - , registry_get_this_machine_hostname()
316 - , registry_get_this_machine_guid()
317 - , os_type
318 - , netdata_configured_timezone
319 - , netdata_configured_abbrev_timezone
320 - , netdata_configured_utc_offset
321 - , ""
322 - , program_name
323 - , program_version
324 - , rrdb.default_update_every
325 - , rrdb.default_rrd_history_entries
326 - , default_storage_engine_id
327 - , default_health_enabled
328 - , default_rrdpush_enabled
329 - , default_rrdpush_destination
330 - , default_rrdpush_api_key
331 - , default_rrdpush_send_charts_matching
332 - , default_rrdpush_enable_replication
333 - , default_rrdpush_seconds_to_replicate
334 - , default_rrdpush_replication_step
335 - , system_info
336 - , 1
337 - , 0
338 - );
339 -
340 - if (unlikely(!rrdb.localhost)) {
341 - return 1;
342 - }
157 +// ----------------------------------------------------------------------------
158 +// RRD - string management
159
344 -#if NETDATA_DEV_MODE
345 - // we register this only on localhost
346 - // for the other nodes, the origin server should register it
347 - rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
348 - rrd_collector_add_function(rrdb.localhost, NULL, "streaming", 10,
349 - RRDFUNCTIONS_STREAMING_HELP, true,
350 - rrdhost_function_streaming, NULL);
351 -#endif
160 +STRING *rrd_string_strdupz(const char *s) {
161 + if(unlikely(!s || !*s)) return string_strdupz(s);
162
353 - if (likely(system_info)) {
354 - migrate_localhost(&rrdb.localhost->host_uuid);
355 - sql_aclk_sync_init();
356 - web_client_api_v1_management_init();
357 - }
358 - return rrdb.localhost == NULL;
163 + char *tmp = strdupz(s);
164 + json_fix_string(tmp);
165 + STRING *ret = string_strdupz(tmp);
166 + freez(tmp);
167 + return ret;
168 }
360 -
361 -struct rrdb rrdb = {
362 - .rrdhost_root_index = NULL,
363 - .rrdhost_root_index_hostname = NULL,
364 - .unittest_running = false,
365 - .dbengine_enabled = false,
366 - .storage_tiers = 3,
367 - .use_direct_io = true,
368 - .storage_tiers_grouping_iterations = {
369 - 1,
370 - 60,
371 - 60,
372 - 60,
373 - 60
374 - },
375 - .storage_tiers_backfill = {
376 - RRD_BACKFILL_NEW,
377 - RRD_BACKFILL_NEW,
378 - RRD_BACKFILL_NEW,
379 - RRD_BACKFILL_NEW,
380 - RRD_BACKFILL_NEW
381 - },
382 - .default_update_every = UPDATE_EVERY_MIN,
383 - .default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES,
384 - .gap_when_lost_iterations_above = 1,
385 - .rrdset_free_obsolete_time_s = RRD_DEFAULT_HISTORY_ENTRIES,
386 - .libuv_worker_threads = 8,
387 - .ieee754_doubles = false,
388 - .rrdhost_free_orphan_time_s = RRD_DEFAULT_HISTORY_ENTRIES,
389 - .rrd_rwlock = NETDATA_RWLOCK_INITIALIZER,
390 - .localhost = NULL,
391 -
392 -#if defined(ENV32BIT)
393 - .default_rrdeng_page_cache_mb = 16,
394 - .default_rrdeng_extent_cache_mb = 0,
395 -#else
396 - .default_rrdeng_page_cache_mb = 32,
397 - .default_rrdeng_extent_cache_mb = 0,
398 -#endif
399 -
400 - .db_engine_journal_check = CONFIG_BOOLEAN_NO,
401 -
402 - .default_rrdeng_disk_quota_mb = 256,
403 - .default_multidb_disk_quota_mb = 256,
404 -
405 - .multidb_ctx = {
406 - NULL,
407 - NULL,
408 - NULL,
409 - NULL,
410 - NULL,
411 - },
412 -
413 - .page_type_size = {
414 - sizeof(storage_number),
415 - sizeof(storage_number_tier1_t)
416 - },
417 -
418 -#if defined(ENV32BIT)
419 - .tier_page_size = {
420 - 2048,
421 - 1024,
422 - 192,
423 - 192,
424 - 192
425 - },
426 -#else
427 - .tier_page_size = {
428 - 4096,
429 - 2048,
430 - 384,
431 - 384,
432 - 384
433 - },
434 -#endif
435 -};
database/rrd.h
+557 -124
@@ -9,8 +9,13 @@ extern "C" {
9
10 #include "libnetdata/libnetdata.h"
11
12 +// non-existing structs instead of voids
13 +// to enable type checking at compile time
14 +typedef struct storage_instance STORAGE_INSTANCE;
15 +typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
16 +typedef struct storage_alignment STORAGE_METRICS_GROUP;
17 +
18 // forward typedefs
13 -typedef struct rrdb RRDB;
19 typedef struct rrdhost RRDHOST;
20 typedef struct rrddim RRDDIM;
21 typedef struct rrdset RRDSET;
@@ -41,11 +46,57 @@ typedef enum __attribute__ ((__packed__)) {
46 QUERY_SOURCE_UNITTEST,
47 } QUERY_SOURCE;
48
44 -#define RRD_STORAGE_TIERS 5
49 +typedef enum __attribute__ ((__packed__)) storage_priority {
50 + STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
51 + STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
52 +
53 + // query priorities
54 + STORAGE_PRIORITY_HIGH,
55 + STORAGE_PRIORITY_NORMAL,
56 + STORAGE_PRIORITY_LOW,
57 + STORAGE_PRIORITY_BEST_EFFORT,
58 +
59 + // synchronous query, not to be dispatched to workers or queued
60 + STORAGE_PRIORITY_SYNCHRONOUS,
61 +
62 + STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
63 +} STORAGE_PRIORITY;
64
65 // forward declarations
66 struct rrddim_tier;
67
68 +#ifdef ENABLE_DBENGINE
69 +struct rrdeng_page_descr;
70 +struct rrdengine_instance;
71 +struct pg_cache_page_index;
72 +#endif
73 +
74 +// ----------------------------------------------------------------------------
75 +// memory mode
76 +
77 +typedef enum __attribute__ ((__packed__)) rrd_memory_mode {
78 + RRD_MEMORY_MODE_NONE = 0,
79 + RRD_MEMORY_MODE_RAM = 1,
80 + RRD_MEMORY_MODE_MAP = 2,
81 + RRD_MEMORY_MODE_SAVE = 3,
82 + RRD_MEMORY_MODE_ALLOC = 4,
83 + RRD_MEMORY_MODE_DBENGINE = 5,
84 +
85 + // this is 8-bit
86 +} RRD_MEMORY_MODE;
87 +
88 +#define RRD_MEMORY_MODE_NONE_NAME "none"
89 +#define RRD_MEMORY_MODE_RAM_NAME "ram"
90 +#define RRD_MEMORY_MODE_MAP_NAME "map"
91 +#define RRD_MEMORY_MODE_SAVE_NAME "save"
92 +#define RRD_MEMORY_MODE_ALLOC_NAME "alloc"
93 +#define RRD_MEMORY_MODE_DBENGINE_NAME "dbengine"
94 +
95 +extern RRD_MEMORY_MODE default_rrd_memory_mode;
96 +
97 +const char *rrd_memory_mode_name(RRD_MEMORY_MODE id);
98 +RRD_MEMORY_MODE rrd_memory_mode_id(const char *name);
99 +
100 struct ml_metrics_statistics {
101 size_t anomalous;
102 size_t normal;
@@ -54,7 +105,6 @@ struct ml_metrics_statistics {
105 size_t silenced;
106 };
107
57 -#include "storage_engine.h"
108 #include "daemon/common.h"
109 #include "web/api/queries/query.h"
110 #include "web/api/queries/rrdr.h"
@@ -63,15 +113,32 @@ struct ml_metrics_statistics {
113 #include "rrddimvar.h"
114 #include "rrdcalc.h"
115 #include "rrdcalctemplate.h"
66 -#include "rrdfunctions.h"
67 -#include "rrdlabels.h"
116 #include "streaming/rrdpush.h"
117 #include "aclk/aclk_rrdhost_state.h"
118 +#include "sqlite/sqlite_health.h"
119 +
120 +typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
121 +
122 +typedef enum __attribute__ ((__packed__)) {
123 + STORAGE_ENGINE_BACKEND_RRDDIM = 1,
124 + STORAGE_ENGINE_BACKEND_DBENGINE = 2,
125 +} STORAGE_ENGINE_BACKEND;
126 +
127 +#define is_valid_backend(backend) ((backend) >= STORAGE_ENGINE_BACKEND_RRDDIM && (backend) <= STORAGE_ENGINE_BACKEND_DBENGINE)
128 +
129 +// iterator state for RRD dimension data queries
130 +struct storage_engine_query_handle {
131 + time_t start_time_s;
132 + time_t end_time_s;
133 + STORAGE_PRIORITY priority;
134 + STORAGE_ENGINE_BACKEND backend;
135 + STORAGE_QUERY_HANDLE *handle;
136 +};
137
138 // ----------------------------------------------------------------------------
139 // chart types
140
74 -typedef enum __attribute__ ((__packed__)) {
141 +typedef enum __attribute__ ((__packed__)) rrdset_type {
142 RRDSET_TYPE_LINE = 0,
143 RRDSET_TYPE_AREA = 1,
144 RRDSET_TYPE_STACKED = 2,
@@ -86,24 +153,53 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type);
153
154 #include "contexts/rrdcontext.h"
155
156 +extern bool unittest_running;
157 +extern bool dbengine_enabled;
158 +extern size_t storage_tiers;
159 +extern bool use_direct_io;
160 +extern size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
161 +
162 typedef enum __attribute__ ((__packed__)) {
163 RRD_BACKFILL_NONE = 0,
164 RRD_BACKFILL_FULL,
165 RRD_BACKFILL_NEW
166 } RRD_BACKFILL;
167
95 -#define UPDATE_EVERY_MIN 1
96 -#define UPDATE_EVERY_MAX 600
168 +extern RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
169 +
170 +#define UPDATE_EVERY 1
171 +#define UPDATE_EVERY_MAX 3600
172
173 #define RRD_DEFAULT_HISTORY_ENTRIES 3600
174 #define RRD_HISTORY_ENTRIES_MAX (86400*365)
175
176 +extern int default_rrd_update_every;
177 +extern int default_rrd_history_entries;
178 +extern int gap_when_lost_iterations_above;
179 +extern time_t rrdset_free_obsolete_time_s;
180 +
181 +#if defined(ENV32BIT)
182 +#define MIN_LIBUV_WORKER_THREADS 8
183 +#define MAX_LIBUV_WORKER_THREADS 128
184 +#define RESERVED_LIBUV_WORKER_THREADS 3
185 +#else
186 +#define MIN_LIBUV_WORKER_THREADS 16
187 +#define MAX_LIBUV_WORKER_THREADS 1024
188 +#define RESERVED_LIBUV_WORKER_THREADS 6
189 +#endif
190 +
191 +extern int libuv_worker_threads;
192 +extern bool ieee754_doubles;
193 +
194 #define RRD_ID_LENGTH_MAX 1000
195
196 +typedef long long total_number;
197 +#define TOTAL_NUMBER_FORMAT "%lld"
198 +
199 // ----------------------------------------------------------------------------
200 // algorithms types
201
106 -typedef enum __attribute__ ((__packed__)) {
202 +typedef enum __attribute__ ((__packed__)) rrd_algorithm {
203 RRD_ALGORITHM_ABSOLUTE = 0,
204 RRD_ALGORITHM_INCREMENTAL = 1,
205 RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2,
@@ -134,7 +230,7 @@ DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rf);
230 // flags & options
231
232 // options are permanent configuration options (no atomics to alter/access them)
137 -typedef enum __attribute__ ((__packed__)) {
233 +typedef enum __attribute__ ((__packed__)) rrddim_options {
234 RRDDIM_OPTION_NONE = 0,
235 RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
236 RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
@@ -150,7 +246,7 @@ typedef enum __attribute__ ((__packed__)) {
246 #define rrddim_option_clear(rd, option) (rd)->collector.options &= ~(option)
247
248 // flags are runtime changing status flags (atomics are required to alter/access them)
153 -typedef enum __attribute__ ((__packed__)) {
249 +typedef enum __attribute__ ((__packed__)) rrddim_flags {
250 RRDDIM_FLAG_NONE = 0,
251 RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 0),
252
@@ -169,11 +265,73 @@ typedef enum __attribute__ ((__packed__)) {
265 #define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
266 #define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
267
268 +typedef enum __attribute__ ((__packed__)) rrdlabel_source {
269 + RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
270 + RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
271 + RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
272 + RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
273 +
274 + // more sources can be added here
275 +
276 + RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
277 + RRDLABEL_FLAG_OLD = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
278 + RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels
279 +} RRDLABEL_SRC;
280 +
281 +#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
282 +
283 +size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
284 +
285 +DICTIONARY *rrdlabels_create(void);
286 +void rrdlabels_destroy(DICTIONARY *labels_dict);
287 +void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
288 +void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
289 +void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
290 +void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key);
291 +void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key);
292 +void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key);
293 +STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key);
294 +STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset);
295 +void rrdlabels_flush(DICTIONARY *labels_dict);
296 +
297 +void rrdlabels_unmark_all(DICTIONARY *labels);
298 +void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
299 +
300 +int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
301 +int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
302 +
303 +void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
304 +bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
305 +
306 +bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
307 +int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
308 +void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb);
309 +
310 +void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
311 +void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
312 +
313 +void reload_host_labels(void);
314 +void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
315 +void rrdset_save_rrdlabels_to_sql(RRDSET *st);
316 +void rrdhost_set_is_parent_label(void);
317 +int rrdlabels_unittest(void);
318 +
319 +// unfortunately this break when defined in exporting_engine.h
320 +bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
321 +
322 +// ----------------------------------------------------------------------------
323 +// engine-specific iterator state for dimension data collection
324 +typedef struct storage_collect_handle {
325 + STORAGE_ENGINE_BACKEND backend;
326 +} STORAGE_COLLECT_HANDLE;
327 +
328 // ----------------------------------------------------------------------------
329 // Storage tier data for every dimension
330
331 struct rrddim_tier {
332 STORAGE_POINT virtual_point;
333 + STORAGE_ENGINE_BACKEND backend;
334 + uint32_t tier_grouping;
335 time_t next_point_end_time_s;
336 STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database
337 STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle
@@ -194,6 +352,7 @@ struct rrddim {
352 STRING *name; // the name of this dimension (as presented to user)
353
354 RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
355 + RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
356 RRDDIM_FLAGS flags; // run time changing status flags
357
358 int32_t multiplier; // the multiplier of the collected values
@@ -266,17 +425,292 @@ size_t rrddim_size(void);
425 // returns the RRDDIM cache filename, or NULL if it does not exist
426 const char *rrddim_cache_filename(RRDDIM *rd);
427
269 -// updated the header with the latest RRDDIM value, for storage engines MAP and SAVE
428 +// updated the header with the latest RRDDIM value, for memory mode MAP and SAVE
429 void rrddim_memory_file_update(RRDDIM *rd);
430
272 -// free the memory file structures for storage engines MAP and SAVE
431 +// free the memory file structures for memory mode MAP and SAVE
432 void rrddim_memory_file_free(RRDDIM *rd);
433
434 +bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode);
435 +
436 // return the v019 header size of RRDDIM files
437 size_t rrddim_memory_file_header_size(void);
438
439 void rrddim_memory_file_save(RRDDIM *rd);
440
441 +// ------------------------------------------------------------------------
442 +// DATA COLLECTION STORAGE OPS
443 +
444 +STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
445 +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
446 +static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance, uuid_t *uuid) {
447 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
448 +
449 +#ifdef ENABLE_DBENGINE
450 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
451 + return rrdeng_metrics_group_get(db_instance, uuid);
452 +#endif
453 + return rrddim_metrics_group_get(db_instance, uuid);
454 +}
455 +
456 +void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
457 +void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
458 +static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
459 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
460 +
461 +#ifdef ENABLE_DBENGINE
462 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
463 + rrdeng_metrics_group_release(db_instance, smg);
464 + else
465 +#endif
466 + rrddim_metrics_group_release(db_instance, smg);
467 +}
468 +
469 +STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
470 +STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
471 +static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {
472 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
473 +
474 +#ifdef ENABLE_DBENGINE
475 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
476 + return rrdeng_store_metric_init(db_metric_handle, update_every, smg);
477 +#endif
478 + return rrddim_collect_init(db_metric_handle, update_every, smg);
479 +}
480 +
481 +void rrdeng_store_metric_next(
482 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
483 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
484 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
485 +
486 +void rrddim_collect_store_metric(
487 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
488 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
489 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
490 +
491 +static inline void storage_engine_store_metric(
492 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
493 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
494 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {
495 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
496 +
497 +#ifdef ENABLE_DBENGINE
498 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
499 + return rrdeng_store_metric_next(collection_handle, point_in_time_ut,
500 + n, min_value, max_value,
501 + count, anomaly_count, flags);
502 +#endif
503 + return rrddim_collect_store_metric(collection_handle, point_in_time_ut,
504 + n, min_value, max_value,
505 + count, anomaly_count, flags);
506 +}
507 +
508 +size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
509 +static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
510 +#ifdef ENABLE_DBENGINE
511 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
512 + return rrdeng_disk_space_max(db_instance);
513 +#endif
514 +
515 + return 0;
516 +}
517 +
518 +size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
519 +static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
520 +#ifdef ENABLE_DBENGINE
521 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
522 + return rrdeng_disk_space_used(db_instance);
523 +#endif
524 +
525 + // TODO - calculate the total host disk space for memory mode save and map
526 + return 0;
527 +}
528 +
529 +time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *db_instance);
530 +static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
531 +#ifdef ENABLE_DBENGINE
532 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
533 + return rrdeng_global_first_time_s(db_instance);
534 +#endif
535 +
536 + return now_realtime_sec() - (time_t)(default_rrd_history_entries * default_rrd_update_every);
537 +}
538 +
539 +size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
540 +static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
541 +#ifdef ENABLE_DBENGINE
542 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
543 + return rrdeng_currently_collected_metrics(db_instance);
544 +#endif
545 +
546 + // TODO - calculate the total host disk space for memory mode save and map
547 + return 0;
548 +}
549 +
550 +void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
551 +void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
552 +static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
553 + if(unlikely(!collection_handle))
554 + return;
555 +
556 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
557 +
558 +#ifdef ENABLE_DBENGINE
559 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
560 + rrdeng_store_metric_flush_current_page(collection_handle);
561 + else
562 +#endif
563 + rrddim_store_metric_flush(collection_handle);
564 +}
565 +
566 +int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
567 +int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
568 +// a finalization function to run after collection is over
569 +// returns 1 if it's safe to delete the dimension
570 +static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
571 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
572 +
573 +#ifdef ENABLE_DBENGINE
574 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
575 + return rrdeng_store_metric_finalize(collection_handle);
576 +#endif
577 +
578 + return rrddim_collect_finalize(collection_handle);
579 +}
580 +
581 +void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
582 +void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
583 +static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every) {
584 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
585 +
586 +#ifdef ENABLE_DBENGINE
587 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
588 + rrdeng_store_metric_change_collection_frequency(collection_handle, update_every);
589 + else
590 +#endif
591 + rrddim_store_metric_change_collection_frequency(collection_handle, update_every);
592 +}
593 +
594 +
595 +// ----------------------------------------------------------------------------
596 +// STORAGE ENGINE QUERY OPS
597 +
598 +time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
599 +time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
600 +static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_METRIC_HANDLE *db_metric_handle) {
601 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
602 +
603 +#ifdef ENABLE_DBENGINE
604 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
605 + return rrdeng_metric_oldest_time(db_metric_handle);
606 +#endif
607 + return rrddim_query_oldest_time_s(db_metric_handle);
608 +}
609 +
610 +time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
611 +time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
612 +static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_METRIC_HANDLE *db_metric_handle) {
613 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
614 +
615 +#ifdef ENABLE_DBENGINE
616 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
617 + return rrdeng_metric_latest_time(db_metric_handle);
618 +#endif
619 + return rrddim_query_latest_time_s(db_metric_handle);
620 +}
621 +
622 +void rrdeng_load_metric_init(
623 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle,
624 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
625 +
626 +void rrddim_query_init(
627 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
628 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
629 +
630 +static inline void storage_engine_query_init(
631 + STORAGE_ENGINE_BACKEND backend __maybe_unused,
632 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
633 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {
634 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
635 +
636 +#ifdef ENABLE_DBENGINE
637 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
638 + rrdeng_load_metric_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
639 + else
640 +#endif
641 + rrddim_query_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
642 +}
643 +
644 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
645 +STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle);
646 +static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *handle) {
647 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
648 +
649 +#ifdef ENABLE_DBENGINE
650 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
651 + return rrdeng_load_metric_next(handle);
652 +#endif
653 + return rrddim_query_next_metric(handle);
654 +}
655 +
656 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
657 +int rrddim_query_is_finished(struct storage_engine_query_handle *handle);
658 +static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *handle) {
659 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
660 +
661 +#ifdef ENABLE_DBENGINE
662 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
663 + return rrdeng_load_metric_is_finished(handle);
664 +#endif
665 + return rrddim_query_is_finished(handle);
666 +}
667 +
668 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
669 +void rrddim_query_finalize(struct storage_engine_query_handle *handle);
670 +static inline void storage_engine_query_finalize(struct storage_engine_query_handle *handle) {
671 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
672 +
673 +#ifdef ENABLE_DBENGINE
674 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
675 + rrdeng_load_metric_finalize(handle);
676 + else
677 +#endif
678 + rrddim_query_finalize(handle);
679 +}
680 +
681 +time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
682 +time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
683 +static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *handle) {
684 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
685 +
686 +#ifdef ENABLE_DBENGINE
687 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
688 + return rrdeng_load_align_to_optimal_before(handle);
689 +#endif
690 + return rrddim_query_align_to_optimal_before(handle);
691 +}
692 +
693 +// ------------------------------------------------------------------------
694 +// function pointers for all APIs provided by a storage engine
695 +typedef struct storage_engine_api {
696 + // metric management
697 + STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *instance, uuid_t *uuid);
698 + STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *instance);
699 + void (*metric_release)(STORAGE_METRIC_HANDLE *);
700 + STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
701 + bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
702 +} STORAGE_ENGINE_API;
703 +
704 +typedef struct storage_engine {
705 + STORAGE_ENGINE_BACKEND backend;
706 + RRD_MEMORY_MODE id;
707 + const char* name;
708 + STORAGE_ENGINE_API api;
709 +} STORAGE_ENGINE;
710 +
711 +STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
712 +STORAGE_ENGINE* storage_engine_find(const char* name);
713 +
714 // ----------------------------------------------------------------------------
715 // these loop macros make sure the linked list is accessed with the right lock
716
@@ -299,7 +733,7 @@ void rrddim_memory_file_save(RRDDIM *rd);
733 // flags are set/unset in a manner that is not thread safe
734 // and may lead to missing information.
735
302 -typedef enum __attribute__ ((__packed__)) {
736 +typedef enum __attribute__ ((__packed__)) rrdset_flags {
737 RRDSET_FLAG_DETAIL = (1 << 1), // if set, the data set should be considered as a detail of another
738 // (the master data set should be the one that has the same family and is not detail)
739 RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
@@ -379,7 +813,7 @@ struct rrdset {
813 // operational state members
814
815 RRDSET_FLAGS flags; // flags
382 - STORAGE_ENGINE_ID storage_engine_id; // the db mode of this rrdset
816 + RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset
817
818 DICTIONARY *rrddim_root_index; // dimensions index
819
@@ -507,6 +941,9 @@ void rrdset_memory_file_save(RRDSET *st);
941 void rrdset_memory_file_free(RRDSET *st);
942 void rrdset_memory_file_update(RRDSET *st);
943 const char *rrdset_cache_filename(RRDSET *st);
944 +bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
945 +
946 +#include "rrdfunctions.h"
947
948 // ----------------------------------------------------------------------------
949 // RRDHOST flags
@@ -514,7 +951,7 @@ const char *rrdset_cache_filename(RRDSET *st);
951 // flags are set/unset in a manner that is not thread safe
952 // and may lead to missing information.
953
517 -typedef enum __attribute__ ((__packed__)) {
954 +typedef enum __attribute__ ((__packed__)) rrdhost_flags {
955
956 // Careful not to overlap with rrdhost_options to avoid bugs if
957 // rrdhost_flags_xxx is used instead of rrdhost_option_xxx or vice-versa
@@ -558,6 +995,13 @@ typedef enum __attribute__ ((__packed__)) {
995 #define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST)
996 #define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~(flag), __ATOMIC_SEQ_CST)
997
998 +#ifdef NETDATA_INTERNAL_CHECKS
999 +#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
1000 + debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0)
1001 +#else
1002 +#define rrdset_debug(st, fmt, args...) debug_dummy()
1003 +#endif
1004 +
1005 typedef enum __attribute__ ((__packed__)) {
1006 // Indexing
1007 RRDHOST_OPTION_INDEXED_MACHINE_GUID = (1 << 0), // when set, we have indexed its machine guid
@@ -719,6 +1163,8 @@ struct rrdhost_system_info {
1163 int mc_version;
1164 };
1165
1166 +struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels);
1167 +
1168 struct rrdhost {
1169 char machine_guid[GUID_LEN + 1]; // the unique ID of this host
1170
@@ -740,16 +1186,17 @@ struct rrdhost {
1186 RRDHOST_FLAGS flags; // runtime flags about this RRDHOST (atomics on this)
1187 RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances
1188
743 - int32_t update_every; // the update frequency of the host
1189 + int32_t rrd_update_every; // the update frequency of the host
1190 int32_t rrd_history_entries; // the number of history entries for the host's charts
1191
746 - STORAGE_ENGINE_ID storage_engine_id; // the configured storage engine id for the charts of this host
1192 + RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
1193 // the actual per tier is at .db[tier].mode
1194
1195 char *cache_dir; // the directory to save RRD cache files
1196
1197 struct {
752 - STORAGE_ENGINE_ID id; // the storage engine id for this tier
1198 + RRD_MEMORY_MODE mode; // the db mode for this tier
1199 + STORAGE_ENGINE *eng; // the storage engine API for this tier
1200 STORAGE_INSTANCE *instance; // the db instance for this tier
1201 uint32_t tier_grouping; // tier 0 iterations aggregated on this tier
1202 } db[RRD_STORAGE_TIERS];
@@ -862,57 +1309,7 @@ struct rrdhost {
1309 struct rrdhost *next;
1310 struct rrdhost *prev;
1311 };
865 -
866 -// ----------------------------------------------------------------------------
867 -// RRDB
868 -
869 -struct rrdb {
870 - DICTIONARY *rrdhost_root_index;
871 - DICTIONARY *rrdhost_root_index_hostname;
872 -
873 - bool unittest_running;
874 - bool dbengine_enabled;
875 - size_t storage_tiers;
876 - bool use_direct_io;
877 -
878 - size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
879 - RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
880 -
881 - int default_update_every;
882 - int default_rrd_history_entries;
883 -
884 - int gap_when_lost_iterations_above;
885 -
886 - time_t rrdset_free_obsolete_time_s;
887 -
888 - int libuv_worker_threads;
889 -
890 - bool ieee754_doubles;
891 -
892 - time_t rrdhost_free_orphan_time_s;
893 -
894 - netdata_rwlock_t rrd_rwlock;
895 -
896 - RRDHOST *localhost;
897 -
898 - int default_rrdeng_page_cache_mb;
899 -
900 - int default_rrdeng_extent_cache_mb;
901 -
902 - int db_engine_journal_check;
903 -
904 - int default_rrdeng_disk_quota_mb;
905 -
906 - int default_multidb_disk_quota_mb;
907 -
908 - STORAGE_INSTANCE *multidb_ctx[RRD_STORAGE_TIERS];
909 -
910 - size_t page_type_size[256];
911 -
912 - size_t tier_page_size[RRD_STORAGE_TIERS];
913 -};
914 -
915 -extern struct rrdb rrdb;
1312 +extern RRDHOST *localhost;
1313
1314 #define rrdhost_hostname(host) string2str((host)->hostname)
1315 #define rrdhost_registry_hostname(host) string2str((host)->registry_hostname)
@@ -936,33 +1333,33 @@ extern struct rrdb rrdb;
1333 #define rrdhost_sender_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->rrdpush_sender_replicating_charts), 1, __ATOMIC_RELAXED))
1334 #define rrdhost_sender_replicating_charts_zero(host) (__atomic_store_n(&((host)->rrdpush_sender_replicating_charts), 0, __ATOMIC_RELAXED))
1335
939 -#define rrdhost_is_online(host) ((host) == rrdb.localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
1336 +#define rrdhost_is_online(host) ((host) == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
1337 bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now);
1338
1339 +extern DICTIONARY *rrdhost_root_index;
1340 size_t rrdhost_hosts_available(void);
1341
1342 RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid);
1343 RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha);
1344 void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha);
1345
948 -void rrdhost_set_is_parent_label(void);
949 -
1346 // ----------------------------------------------------------------------------
1347
1348 #define rrdhost_foreach_read(var) \
953 - for((var) = rrdb.localhost; var ; (var) = (var)->next)
1349 + for((var) = localhost; var ; (var) = (var)->next)
1350
1351 #define rrdhost_foreach_write(var) \
956 - for((var) = rrdb.localhost; var ; (var) = (var)->next)
1352 + for((var) = localhost; var ; (var) = (var)->next)
1353
1354
1355 // ----------------------------------------------------------------------------
1356 // global lock for all RRDHOSTs
1357
962 -#define rrd_rdlock() netdata_rwlock_rdlock(&rrdb.rrd_rwlock)
963 -#define rrd_wrlock() netdata_rwlock_wrlock(&rrdb.rrd_rwlock)
964 -#define rrd_unlock() netdata_rwlock_unlock(&rrdb.rrd_rwlock)
965 -#define rrd_tryrdlock() netdata_rwlock_tryrdlock(&rrdb.rrd_rwlock)
1358 +extern netdata_rwlock_t rrd_rwlock;
1359 +
1360 +#define rrd_rdlock() netdata_rwlock_rdlock(&rrd_rwlock)
1361 +#define rrd_wrlock() netdata_rwlock_wrlock(&rrd_rwlock)
1362 +#define rrd_unlock() netdata_rwlock_unlock(&rrd_rwlock)
1363
1364 // ----------------------------------------------------------------------------
1365
@@ -975,39 +1372,15 @@ void rrddim_index_destroy(RRDSET *st);
1372
1373 // ----------------------------------------------------------------------------
1374
1375 +extern time_t rrdhost_free_orphan_time_s;
1376 +
1377 int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest);
1378
1379 RRDHOST *rrdhost_find_by_hostname(const char *hostname);
1380 RRDHOST *rrdhost_find_by_guid(const char *guid);
982 -RRDHOST *rrdhost_find_by_node_id(const char *node_id);
983 -
984 -RRDHOST *rrdhost_create(
985 - const char *hostname,
986 - const char *registry_hostname,
987 - const char *guid,
988 - const char *os,
989 - const char *timezone,
990 - const char *abbrev_timezone,
991 - int32_t utc_offset,
992 - const char *tags,
993 - const char *program_name,
994 - const char *program_version,
995 - int update_every,
996 - long entries,
997 - STORAGE_ENGINE_ID storage_engine_id,
998 - unsigned int health_enabled,
999 - unsigned int rrdpush_enabled,
1000 - char *rrdpush_destination,
1001 - char *rrdpush_api_key,
1002 - char *rrdpush_send_charts_matching,
1003 - bool rrdpush_enable_replication,
1004 - time_t rrdpush_seconds_to_replicate,
1005 - time_t rrdpush_replication_step,
1006 - struct rrdhost_system_info *system_info,
1007 - int is_localhost,
1008 - bool archived);
1009 -
1010 -RRDHOST *rrdhost_get_or_create(
1381 +RRDHOST *find_host_by_node_id(char *node_id);
1382 +
1383 +RRDHOST *rrdhost_find_or_create(
1384 const char *hostname
1385 , const char *registry_hostname
1386 , const char *guid
@@ -1020,7 +1393,7 @@ RRDHOST *rrdhost_get_or_create(
1393 , const char *program_version
1394 , int update_every
1395 , long history
1023 - , STORAGE_ENGINE_ID storage_engine_id
1396 + , RRD_MEMORY_MODE mode
1397 , unsigned int health_enabled
1398 , unsigned int rrdpush_enabled
1399 , char *rrdpush_destination
@@ -1053,14 +1426,14 @@ RRDSET *rrdset_create_custom(RRDHOST *host
1426 , long priority
1427 , int update_every
1428 , RRDSET_TYPE chart_type
1056 - , STORAGE_ENGINE_ID storage_engine_id
1429 + , RRD_MEMORY_MODE memory_mode
1430 , long history_entries);
1431
1432 #define rrdset_create(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
1060 - rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->storage_engine_id, (host)->rrd_history_entries)
1433 + rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries)
1434
1435 #define rrdset_create_localhost(type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
1063 - rrdset_create(rrdb.localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
1436 + rrdset_create(localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
1437
1438 void rrdhost_free_all(void);
1439 void rrdhost_save_all(void);
@@ -1077,14 +1450,44 @@ void rrdset_update_heterogeneous_flag(RRDSET *st);
1450
1451 time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);
1452
1080 -RRDSET *rrdset_find_by_id(RRDHOST *host, const char *id);
1081 -RRDSET *rrdset_find_by_type(RRDHOST *host, const char *type, const char *id);
1082 -RRDSET *rrdset_find_by_name(RRDHOST *host, const char *name);
1453 +RRDSET *rrdset_find(RRDHOST *host, const char *id);
1454
1455 RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);
1456 RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);
1457 void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
1458
1459 +#define rrdset_find_localhost(id) rrdset_find(localhost, id)
1460 +/* This will not return charts that are archived */
1461 +static inline RRDSET *rrdset_find_active_localhost(const char *id)
1462 +{
1463 + RRDSET *st = rrdset_find_localhost(id);
1464 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
1465 + return NULL;
1466 + return st;
1467 +}
1468 +
1469 +RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
1470 +#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
1471 +/* This will not return charts that are archived */
1472 +static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
1473 +{
1474 + RRDSET *st = rrdset_find_bytype_localhost(type, id);
1475 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
1476 + return NULL;
1477 + return st;
1478 +}
1479 +
1480 +RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
1481 +#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)
1482 +/* This will not return charts that are archived */
1483 +static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
1484 +{
1485 + RRDSET *st = rrdset_find_byname_localhost(name);
1486 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
1487 + return NULL;
1488 + return st;
1489 +}
1490 +
1491 void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds);
1492 void rrdset_next_usec(RRDSET *st, usec_t microseconds);
1493 void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);
@@ -1097,7 +1500,7 @@ void rrdset_is_obsolete(RRDSET *st);
1500 void rrdset_isnot_obsolete(RRDSET *st);
1501
1502 // checks if the RRDSET should be offered to viewers
1100 -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->storage_engine_id != STORAGE_ENGINE_NONE)
1503 +#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
1504 #define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
1505 #define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
1506
@@ -1113,24 +1516,31 @@ time_t rrdset_last_entry_s_of_tier(RRDSET *st, size_t tier);
1516
1517 void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_time_s, time_t *last_time_s, time_t now_s, size_t tier);
1518
1116 -void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
1117 -
1519 // ----------------------------------------------------------------------------
1520 // RRD DIMENSION functions
1521
1121 -RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name,
1122 - collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm);
1522 +RRDDIM *rrddim_add_custom(RRDSET *st
1523 + , const char *id
1524 + , const char *name
1525 + , collected_number multiplier
1526 + , collected_number divisor
1527 + , RRD_ALGORITHM algorithm
1528 + , RRD_MEMORY_MODE memory_mode
1529 + );
1530 +
1531 +#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \
1532 + rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
1533
1534 int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name);
1535 int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
1536 int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier);
1537 int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor);
1538
1129 -RRDDIM *rrddim_find_by_id(RRDSET *st, const char *id);
1130 -
1539 +RRDDIM *rrddim_find(RRDSET *st, const char *id);
1540 RRDDIM_ACQUIRED *rrddim_find_and_acquire(RRDSET *st, const char *id);
1541 RRDDIM *rrddim_acquired_to_rrddim(RRDDIM_ACQUIRED *rda);
1542 void rrddim_acquired_release(RRDDIM_ACQUIRED *rda);
1543 +RRDDIM *rrddim_find_active(RRDSET *st, const char *id);
1544
1545 int rrddim_hide(RRDSET *st, const char *id);
1546 int rrddim_unhide(RRDSET *st, const char *id);
@@ -1147,7 +1557,7 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);
1557 void rrdhost_finalize_collection(RRDHOST *host);
1558 void rrd_finalize_collection_for_all_hosts(void);
1559
1150 -long align_entries_to_pagesize(STORAGE_ENGINE_ID id, long entries);
1560 +long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
1561
1562 #ifdef NETDATA_LOG_COLLECTION_ERRORS
1563 #define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__)
@@ -1170,22 +1580,45 @@ void rrdset_free(RRDSET *st);
1580
1581 void rrddim_free(RRDSET *st, RRDDIM *rd);
1582
1583 +#ifdef NETDATA_RRD_INTERNALS
1584 +
1585 +char *rrdhost_cache_dir_for_rrdset_alloc(RRDHOST *host, const char *id);
1586 const char *rrdset_cache_dir(RRDSET *st);
1587
1588 +void rrdset_reset(RRDSET *st);
1589 +void rrdset_delete_obsolete_dimensions(RRDSET *st);
1590 +
1591 +#endif /* NETDATA_RRD_INTERNALS */
1592 +
1593 void set_host_properties(
1176 - RRDHOST *host, int update_every, STORAGE_ENGINE_ID storage_engine_id, const char *registry_hostname,
1594 + RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname,
1595 const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
1596 const char *program_name, const char *program_version);
1597
1598 size_t get_tier_grouping(size_t tier);
1599 void store_metric_collection_completed(void);
1600
1601 +static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to) {
1602 + time_t first_time_s = 0, last_time_s = 0;
1603 + spinlock_lock(&host->retention.spinlock);
1604 + first_time_s = host->retention.first_time_s;
1605 + last_time_s = host->retention.last_time_s;
1606 + spinlock_unlock(&host->retention.spinlock);
1607 +
1608 + if(from)
1609 + *from = first_time_s;
1610 +
1611 + if(to)
1612 + *to = online ? now : last_time_s;
1613 +}
1614 +
1615 // ----------------------------------------------------------------------------
1616 // RRD DB engine declarations
1617
1618 #ifdef ENABLE_DBENGINE
1619 #include "database/engine/rrdengineapi.h"
1620 #endif
1621 +#include "sqlite/sqlite_functions.h"
1622 #include "sqlite/sqlite_context.h"
1623 #include "sqlite/sqlite_metadata.h"
1624 #include "sqlite/sqlite_aclk.h"
database/rrdcalc.c
+1 -1
@@ -802,7 +802,7 @@ void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host
802
803 void rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts() {
804 RRDHOST *host;
805 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
805 + dfe_start_reentrant(rrdhost_root_index, host) {
806 if (unlikely(!host->health.health_enabled))
807 continue;
808
database/rrddim.c
+233 -210
@@ -1,181 +1,8 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 +#define NETDATA_RRD_INTERNALS
4 #include "rrd.h"
4 -
5 -// ----------------------------------------------------------------------------
6 -// compatibility layer for RRDDIM files v019
7 -
8 -#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019"
9 -
10 -struct avl_element_v019 {
11 - void *avl_link[2];
12 - signed char avl_balance;
13 -};
14 -
15 -struct rrddim_map_save_v019 {
16 - struct avl_element_v019 avl; // ignored
17 - void *id; // ignored
18 - void *name; // ignored
19 - uint32_t algorithm; // print warning on mismatch - update on load
20 - uint32_t storage_engine_id; // ignored
21 - long long multiplier; // print warning on mismatch - update on load
22 - long long divisor; // print warning on mismatch - update on load
23 - uint32_t flags; // ignored
24 - uint32_t hash; // ignored
25 - uint32_t hash_name; // ignored
26 - void *cache_filename; // ignored - we use it to keep the filename to save back
27 - size_t collections_counter; // ignored
28 - void *state; // ignored
29 - size_t unused[8]; // ignored
30 - long long collected_value_max; // ignored
31 - unsigned int updated:1; // ignored
32 - unsigned int exposed:1; // ignored
33 - struct timeval last_collected_time; // check to reset all - ignored after load
34 - long double calculated_value; // ignored
35 - long double last_calculated_value; // ignored
36 - long double last_stored_value; // ignored
37 - long long collected_value; // ignored
38 - long long last_collected_value; // load and save
39 - long double collected_volume; // ignored
40 - long double stored_volume; // ignored
41 - void *next; // ignored
42 - void *rrdset; // ignored
43 - long entries; // check to reset all - update on load
44 - int update_every; // check to reset all - update on load
45 - size_t memsize; // check to reset all - update on load
46 - char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
47 - void *variables; // ignored
48 - storage_number values[]; // the array of values
49 -};
50 -
51 -size_t rrddim_memory_file_header_size(void) {
52 - return sizeof(struct rrddim_map_save_v019);
53 -}
54 -
55 -void rrddim_memory_file_update(RRDDIM *rd) {
56 - if(!rd || !rd->db.rd_on_file) return;
57 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
58 -
59 - rd_on_file->last_collected_time.tv_sec = rd->collector.last_collected_time.tv_sec;
60 - rd_on_file->last_collected_time.tv_usec = rd->collector.last_collected_time.tv_usec;
61 - rd_on_file->last_collected_value = rd->collector.last_collected_value;
62 -}
63 -
64 -void rrddim_memory_file_free(RRDDIM *rd) {
65 - if(!rd || !rd->db.rd_on_file) return;
66 -
67 - // needed for storage engine MAP, to save the latest state
68 - rrddim_memory_file_update(rd);
69 -
70 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
71 - __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
72 - freez(rd_on_file->cache_filename);
73 - netdata_munmap(rd_on_file, rd_on_file->memsize);
74 -
75 - // remove the pointers from the RRDDIM
76 - rd->db.rd_on_file = NULL;
77 - rd->db.data = NULL;
78 -}
79 -
80 -const char *rrddim_cache_filename(RRDDIM *rd) {
81 - if(!rd || !rd->db.rd_on_file) return NULL;
82 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
83 - return rd_on_file->cache_filename;
84 -}
85 -
86 -void rrddim_memory_file_save(RRDDIM *rd) {
87 - if(!rd || !rd->db.rd_on_file) return;
88 -
89 - rrddim_memory_file_update(rd);
90 -
91 - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
92 - if(rd_on_file->storage_engine_id != STORAGE_ENGINE_SAVE) return;
93 -
94 - memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
95 -}
96 -
97 -static bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd) {
98 - assert(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP);
99 -
100 - struct rrddim_map_save_v019 *rd_on_file = NULL;
101 -
102 - unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number));
103 -
104 - char filename[FILENAME_MAX + 1];
105 - char fullfilename[FILENAME_MAX + 1];
106 - rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
107 - snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename);
108 -
109 - rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(
110 - fullfilename, size, ((st->storage_engine_id == STORAGE_ENGINE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL);
111 -
112 - if(unlikely(!rd_on_file)) return false;
113 -
114 - struct timeval now;
115 - now_realtime_timeval(&now);
116 -
117 - int reset = 0;
118 - rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0';
119 - if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
120 - netdata_log_info("Initializing file %s.", fullfilename);
121 - memset(rd_on_file, 0, size);
122 - reset = 1;
123 - }
124 - else if(rd_on_file->memsize != size) {
125 - netdata_log_error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, (unsigned long int) rd_on_file->memsize);
126 - memset(rd_on_file, 0, size);
127 - reset = 1;
128 - }
129 - else if(rd_on_file->update_every != st->update_every) {
130 - netdata_log_error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
131 - memset(rd_on_file, 0, size);
132 - reset = 1;
133 - }
134 - else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
135 - netdata_log_info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
136 - memset(rd_on_file, 0, size);
137 - reset = 1;
138 - }
139 -
140 - if(!reset) {
141 - rd->collector.last_collected_value = rd_on_file->last_collected_value;
142 -
143 - if(rd_on_file->algorithm != rd->algorithm)
144 - netdata_log_info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
145 - fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
146 -
147 - if(rd_on_file->multiplier != rd->multiplier)
148 - netdata_log_info("File %s does not have the expected multiplier (expected %d, found %ld). "
149 - "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier);
150 -
151 - if(rd_on_file->divisor != rd->divisor)
152 - netdata_log_info("File %s does not have the expected divisor (expected %d, found %ld). "
153 - "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor);
154 - }
155 -
156 - // zero the entire header
157 - memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
158 -
159 - // set the important fields
160 - strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
161 - rd_on_file->algorithm = rd->algorithm;
162 - rd_on_file->multiplier = rd->multiplier;
163 - rd_on_file->divisor = rd->divisor;
164 - rd_on_file->entries = st->db.entries;
165 - rd_on_file->update_every = rd->rrdset->update_every;
166 - rd_on_file->memsize = size;
167 - rd_on_file->storage_engine_id = st->storage_engine_id;
168 - rd_on_file->cache_filename = strdupz(fullfilename);
169 -
170 - __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
171 -
172 - rd->db.data = &rd_on_file->values[0];
173 - rd->db.rd_on_file = rd_on_file;
174 - rd->db.memsize = size;
175 - rrddim_memory_file_update(rd);
176 -
177 - return true;
178 -}
5 +#include "storage_engine.h"
6
7 // ----------------------------------------------------------------------------
8 // RRDDIM index
@@ -187,6 +14,7 @@ struct rrddim_constructor {
14 collected_number multiplier;
15 collected_number divisor;
16 RRD_ALGORITHM algorithm;
17 + RRD_MEMORY_MODE memory_mode;
18
19 enum {
20 RRDDIM_REACT_NONE = 0,
@@ -198,7 +26,7 @@ struct rrddim_constructor {
26
27 // isolated call to appear
28 // separate in statistics
201 -static storage_number *rrddim_alloc_db(size_t entries) {
29 +static void *rrddim_alloc_db(size_t entries) {
30 return callocz(entries, sizeof(storage_number));
31 }
32
@@ -223,22 +51,21 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
51 if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
52 rd->collector.counter = 1;
53
226 - if (st->storage_engine_id == STORAGE_ENGINE_MAP || st->storage_engine_id == STORAGE_ENGINE_SAVE) {
227 - if(!rrddim_memory_load_or_create_map_save(st, rd)) {
228 - netdata_log_info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram",
229 - storage_engine_name(st->storage_engine_id), rrdset_name(st), rrddim_name(rd));
230 - st->storage_engine_id = STORAGE_ENGINE_RAM;
54 + if(ctr->memory_mode == RRD_MEMORY_MODE_MAP || ctr->memory_mode == RRD_MEMORY_MODE_SAVE) {
55 + if(!rrddim_memory_load_or_create_map_save(st, rd, ctr->memory_mode)) {
56 + netdata_log_info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (ctr->memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd));
57 + ctr->memory_mode = RRD_MEMORY_MODE_RAM;
58 }
59 }
60
234 - if (st->storage_engine_id == STORAGE_ENGINE_RAM) {
61 + if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) {
62 size_t entries = st->db.entries;
63 if(!entries) entries = 5;
64
65 rd->db.data = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);
66 if(!rd->db.data) {
67 netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));
241 - st->storage_engine_id = STORAGE_ENGINE_ALLOC;
68 + ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;
69 }
70 else {
71 rd->db.memsize = entries * sizeof(storage_number);
@@ -246,7 +73,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
73 }
74 }
75
249 - if (st->storage_engine_id == STORAGE_ENGINE_ALLOC || st->storage_engine_id == STORAGE_ENGINE_NONE) {
76 + if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) {
77 size_t entries = st->db.entries;
78 if(entries < 5) entries = 5;
79
@@ -255,14 +82,19 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
82 __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);
83 }
84
85 + rd->rrd_memory_mode = ctr->memory_mode;
86 +
87 if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid))))
88 uuid_generate(rd->metric_uuid);
89
90 // initialize the db tiers
91 {
92 size_t initialized = 0;
264 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
265 - rd->tiers[tier].db_metric_handle = storage_engine_metric_get_or_create(rd, host->db[tier].id, host->db[tier].instance);
93 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
94 + STORAGE_ENGINE *eng = host->db[tier].eng;
95 + rd->tiers[tier].backend = eng->backend;
96 + rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
97 + rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
98 storage_point_unset(rd->tiers[tier].virtual_point);
99 initialized++;
100
@@ -279,10 +111,10 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
111 // initialize data collection for all tiers
112 {
113 size_t initialized = 0;
282 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
114 + for (size_t tier = 0; tier < storage_tiers; tier++) {
115 if (rd->tiers[tier].db_metric_handle) {
116 rd->tiers[tier].db_collection_handle =
285 - storage_metric_store_init(st->storage_engine_id, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
117 + storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
118 initialized++;
119 }
120 }
@@ -337,13 +169,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
169 bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
170 size_t tiers_available = 0, tiers_said_no_retention = 0;
171
340 - for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
172 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
173 if(!rd->tiers[tier].db_collection_handle)
174 continue;
175
176 tiers_available++;
177
346 - if(storage_engine_store_finalize(rd->rrdset->storage_engine_id, rd->tiers[tier].db_collection_handle))
178 + if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle))
179 tiers_said_no_retention++;
180
181 rd->tiers[tier].db_collection_handle = NULL;
@@ -367,7 +199,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
199
200 netdata_log_debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
201
370 - if (!rrddim_finalize_collection_and_check_retention(rd) && st->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
202 + if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
203 /* This metric has no data and no references */
204 metaqueue_delete_dimension_uuid(&rd->metric_uuid);
205 }
@@ -383,17 +215,18 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
215 // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
216 rrddim_memory_file_free(rd);
217
386 - for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
218 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
219 if(!rd->tiers[tier].db_metric_handle) continue;
220
389 - storage_engine_metric_release(host->db[tier].id, rd->tiers[tier].db_metric_handle);
221 + STORAGE_ENGINE* eng = host->db[tier].eng;
222 + eng->api.metric_release(rd->tiers[tier].db_metric_handle);
223 rd->tiers[tier].db_metric_handle = NULL;
224 }
225
226 if(rd->db.data) {
227 __atomic_sub_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);
228
396 - if(st->storage_engine_id == STORAGE_ENGINE_RAM)
229 + if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
230 netdata_munmap(rd->db.data, rd->db.memsize);
231 else
232 freez(rd->db.data);
@@ -417,10 +250,10 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
250 rc += rrddim_set_multiplier(st, rd, ctr->multiplier);
251 rc += rrddim_set_divisor(st, rd, ctr->divisor);
252
420 - for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
253 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
254 if (!rd->tiers[tier].db_collection_handle)
255 rd->tiers[tier].db_collection_handle =
423 - storage_metric_store_init(st->storage_engine_id, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
256 + storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
257 }
258
259 if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@@ -457,7 +290,7 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
290 }
291
292 size_t rrddim_size(void) {
460 - return sizeof(RRDDIM) + rrdb.storage_tiers * sizeof(struct rrddim_tier);
293 + return sizeof(RRDDIM) + storage_tiers * sizeof(struct rrddim_tier);
294 }
295
296 void rrddim_index_init(RRDSET *st) {
@@ -484,7 +317,7 @@ static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id) {
317 // ----------------------------------------------------------------------------
318 // RRDDIM - find a dimension
319
487 -inline RRDDIM *rrddim_find_by_id(RRDSET *st, const char *id) {
320 +inline RRDDIM *rrddim_find(RRDSET *st, const char *id) {
321 netdata_log_debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", rrdset_name(st), id);
322
323 return rrddim_index_find(st, id);
@@ -511,6 +344,15 @@ void rrddim_acquired_release(RRDDIM_ACQUIRED *rda) {
344 dictionary_acquired_item_release(rd->rrdset->rrddim_root_index, (const DICTIONARY_ITEM *)rda);
345 }
346
347 +// This will not return dimensions that are archived
348 +RRDDIM *rrddim_find_active(RRDSET *st, const char *id) {
349 + RRDDIM *rd = rrddim_find(st, id);
350 +
351 + if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
352 + return NULL;
353 +
354 + return rd;
355 +}
356
357 // ----------------------------------------------------------------------------
358 // RRDDIM rename a dimension
@@ -577,17 +419,17 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor) {
419 // ----------------------------------------------------------------------------
420
421 time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) {
580 - if (unlikely(tier > rrdb.storage_tiers || !rd->tiers[tier].db_metric_handle))
422 + if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
423 return 0;
424
583 - return storage_engine_latest_time_s(rd->rrdset->storage_engine_id, rd->tiers[tier].db_metric_handle);
425 + return storage_engine_latest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle);
426 }
427
428 // get the timestamp of the last entry in the round-robin database
429 time_t rrddim_last_entry_s(RRDDIM *rd) {
430 time_t latest_time_s = rrddim_last_entry_s_of_tier(rd, 0);
431
590 - for (size_t tier = 1; tier < rrdb.storage_tiers ;tier++) {
432 + for(size_t tier = 1; tier < storage_tiers ;tier++) {
433 if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
434
435 time_t t = rrddim_last_entry_s_of_tier(rd, tier);
@@ -599,16 +441,16 @@ time_t rrddim_last_entry_s(RRDDIM *rd) {
441 }
442
443 time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) {
602 - if (unlikely(tier > rrdb.storage_tiers || !rd->tiers[tier].db_metric_handle))
444 + if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
445 return 0;
446
605 - return storage_engine_oldest_time_s(rd->rrdset->storage_engine_id, rd->tiers[tier].db_metric_handle);
447 + return storage_engine_oldest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle);
448 }
449
450 time_t rrddim_first_entry_s(RRDDIM *rd) {
451 time_t oldest_time_s = 0;
452
611 - for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
453 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
454 time_t t = rrddim_first_entry_s_of_tier(rd, tier);
455 if(t != 0 && (oldest_time_s == 0 || t < oldest_time_s))
456 oldest_time_s = t;
@@ -617,9 +459,14 @@ time_t rrddim_first_entry_s(RRDDIM *rd) {
459 return oldest_time_s;
460 }
461
620 -RRDDIM *rrddim_add(RRDSET *st ,const char *id, const char *name,
621 - collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm)
622 -{
462 +RRDDIM *rrddim_add_custom(RRDSET *st
463 + , const char *id
464 + , const char *name
465 + , collected_number multiplier
466 + , collected_number divisor
467 + , RRD_ALGORITHM algorithm
468 + , RRD_MEMORY_MODE memory_mode
469 + ) {
470 struct rrddim_constructor tmp = {
471 .st = st,
472 .id = id,
@@ -627,6 +474,7 @@ RRDDIM *rrddim_add(RRDSET *st ,const char *id, const char *name,
474 .multiplier = multiplier,
475 .divisor = divisor,
476 .algorithm = algorithm,
477 + .memory_mode = memory_mode,
478 };
479
480 RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, rrddim_size(), &tmp);
@@ -649,7 +497,7 @@ int rrddim_hide(RRDSET *st, const char *id) {
497
498 RRDHOST *host = st->rrdhost;
499
652 - RRDDIM *rd = rrddim_find_by_id(st, id);
500 + RRDDIM *rd = rrddim_find(st, id);
501 if(unlikely(!rd)) {
502 netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
503 return 1;
@@ -668,7 +516,7 @@ int rrddim_unhide(RRDSET *st, const char *id) {
516 netdata_log_debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", rrdset_name(st), id);
517
518 RRDHOST *host = st->rrdhost;
671 - RRDDIM *rd = rrddim_find_by_id(st, id);
519 + RRDDIM *rd = rrddim_find(st, id);
520 if(unlikely(!rd)) {
521 netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
522 return 1;
@@ -732,7 +580,7 @@ collected_number rrddim_timed_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *
580
581 collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) {
582 RRDHOST *host = st->rrdhost;
735 - RRDDIM *rd = rrddim_find_by_id(st, id);
583 + RRDDIM *rd = rrddim_find(st, id);
584 if(unlikely(!rd)) {
585 netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
586 return 0;
@@ -742,3 +590,178 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
590 }
591
592
593 +// ----------------------------------------------------------------------------
594 +// compatibility layer for RRDDIM files v019
595 +
596 +#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019"
597 +
598 +struct avl_element_v019 {
599 + void *avl_link[2];
600 + signed char avl_balance;
601 +};
602 +
603 +struct rrddim_map_save_v019 {
604 + struct avl_element_v019 avl; // ignored
605 + void *id; // ignored
606 + void *name; // ignored
607 + uint32_t algorithm; // print warning on mismatch - update on load
608 + uint32_t rrd_memory_mode; // ignored
609 + long long multiplier; // print warning on mismatch - update on load
610 + long long divisor; // print warning on mismatch - update on load
611 + uint32_t flags; // ignored
612 + uint32_t hash; // ignored
613 + uint32_t hash_name; // ignored
614 + void *cache_filename; // ignored - we use it to keep the filename to save back
615 + size_t collections_counter; // ignored
616 + void *state; // ignored
617 + size_t unused[8]; // ignored
618 + long long collected_value_max; // ignored
619 + unsigned int updated:1; // ignored
620 + unsigned int exposed:1; // ignored
621 + struct timeval last_collected_time; // check to reset all - ignored after load
622 + long double calculated_value; // ignored
623 + long double last_calculated_value; // ignored
624 + long double last_stored_value; // ignored
625 + long long collected_value; // ignored
626 + long long last_collected_value; // load and save
627 + long double collected_volume; // ignored
628 + long double stored_volume; // ignored
629 + void *next; // ignored
630 + void *rrdset; // ignored
631 + long entries; // check to reset all - update on load
632 + int update_every; // check to reset all - update on load
633 + size_t memsize; // check to reset all - update on load
634 + char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
635 + void *variables; // ignored
636 + storage_number values[]; // the array of values
637 +};
638 +
639 +size_t rrddim_memory_file_header_size(void) {
640 + return sizeof(struct rrddim_map_save_v019);
641 +}
642 +
643 +void rrddim_memory_file_update(RRDDIM *rd) {
644 + if(!rd || !rd->db.rd_on_file) return;
645 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
646 +
647 + rd_on_file->last_collected_time.tv_sec = rd->collector.last_collected_time.tv_sec;
648 + rd_on_file->last_collected_time.tv_usec = rd->collector.last_collected_time.tv_usec;
649 + rd_on_file->last_collected_value = rd->collector.last_collected_value;
650 +}
651 +
652 +void rrddim_memory_file_free(RRDDIM *rd) {
653 + if(!rd || !rd->db.rd_on_file) return;
654 +
655 + // needed for memory mode map, to save the latest state
656 + rrddim_memory_file_update(rd);
657 +
658 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
659 + __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
660 + freez(rd_on_file->cache_filename);
661 + netdata_munmap(rd_on_file, rd_on_file->memsize);
662 +
663 + // remove the pointers from the RRDDIM
664 + rd->db.rd_on_file = NULL;
665 + rd->db.data = NULL;
666 +}
667 +
668 +const char *rrddim_cache_filename(RRDDIM *rd) {
669 + if(!rd || !rd->db.rd_on_file) return NULL;
670 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
671 + return rd_on_file->cache_filename;
672 +}
673 +
674 +void rrddim_memory_file_save(RRDDIM *rd) {
675 + if(!rd || !rd->db.rd_on_file) return;
676 +
677 + rrddim_memory_file_update(rd);
678 +
679 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;
680 + if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
681 +
682 + memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
683 +}
684 +
685 +bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode) {
686 + if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
687 + return false;
688 +
689 + struct rrddim_map_save_v019 *rd_on_file = NULL;
690 +
691 + unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number));
692 +
693 + char filename[FILENAME_MAX + 1];
694 + char fullfilename[FILENAME_MAX + 1];
695 + rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
696 + snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename);
697 +
698 + rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(
699 + fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL);
700 +
701 + if(unlikely(!rd_on_file)) return false;
702 +
703 + struct timeval now;
704 + now_realtime_timeval(&now);
705 +
706 + int reset = 0;
707 + rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0';
708 + if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
709 + netdata_log_info("Initializing file %s.", fullfilename);
710 + memset(rd_on_file, 0, size);
711 + reset = 1;
712 + }
713 + else if(rd_on_file->memsize != size) {
714 + netdata_log_error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, (unsigned long int) rd_on_file->memsize);
715 + memset(rd_on_file, 0, size);
716 + reset = 1;
717 + }
718 + else if(rd_on_file->update_every != st->update_every) {
719 + netdata_log_error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
720 + memset(rd_on_file, 0, size);
721 + reset = 1;
722 + }
723 + else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
724 + netdata_log_info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
725 + memset(rd_on_file, 0, size);
726 + reset = 1;
727 + }
728 +
729 + if(!reset) {
730 + rd->collector.last_collected_value = rd_on_file->last_collected_value;
731 +
732 + if(rd_on_file->algorithm != rd->algorithm)
733 + netdata_log_info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
734 + fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
735 +
736 + if(rd_on_file->multiplier != rd->multiplier)
737 + netdata_log_info("File %s does not have the expected multiplier (expected %d, found %ld). "
738 + "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier);
739 +
740 + if(rd_on_file->divisor != rd->divisor)
741 + netdata_log_info("File %s does not have the expected divisor (expected %d, found %ld). "
742 + "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor);
743 + }
744 +
745 + // zero the entire header
746 + memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
747 +
748 + // set the important fields
749 + strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
750 + rd_on_file->algorithm = rd->algorithm;
751 + rd_on_file->multiplier = rd->multiplier;
752 + rd_on_file->divisor = rd->divisor;
753 + rd_on_file->entries = st->db.entries;
754 + rd_on_file->update_every = rd->rrdset->update_every;
755 + rd_on_file->memsize = size;
756 + rd_on_file->rrd_memory_mode = memory_mode;
757 + rd_on_file->cache_filename = strdupz(fullfilename);
758 +
759 + __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
760 +
761 + rd->db.data = &rd_on_file->values[0];
762 + rd->db.rd_on_file = rd_on_file;
763 + rd->db.memsize = size;
764 + rrddim_memory_file_update(rd);
765 +
766 + return true;
767 +}
database/rrdfamily.c
+1
@@ -1,5 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 +#define NETDATA_RRD_INTERNALS
4 #include "rrd.h"
5
6 typedef struct rrdfamily {
database/rrdfunctions.c
+3 -4
@@ -1,5 +1,4 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 +#define NETDATA_RRD_INTERNALS
2 #include "rrd.h"
3
4 #define MAX_FUNCTION_LENGTH (PLUGINSD_LINE_MAX - 512) // we need some space for the rest of the line
@@ -817,7 +816,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
816 wb->content_type = CT_APPLICATION_JSON;
817 buffer_json_initialize(wb, "\"", "\"", 0, true, false);
818
820 - buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(rrdb.localhost));
819 + buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
820 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
821 buffer_json_member_add_string(wb, "type", "table");
822 buffer_json_member_add_time_t(wb, "update_every", 1);
@@ -830,7 +829,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
829 size_t max_ml_anomalous = 0, max_ml_normal = 0, max_ml_trained = 0, max_ml_pending = 0, max_ml_silenced = 0;
830 {
831 RRDHOST *host;
833 - dfe_start_read(rrdb.rrdhost_root_index, host) {
832 + dfe_start_read(rrdhost_root_index, host) {
833 RRDHOST_STATUS s;
834 rrdhost_status(host, now, &s);
835 buffer_json_add_array_item_array(wb);
database/rrdfunctions.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef NETDATA_RRDFUNCTIONS_H
2 #define NETDATA_RRDFUNCTIONS_H 1
3
database/rrdhost.c
+570 -132
@@ -1,24 +1,37 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 +#define NETDATA_RRD_INTERNALS
4 #include "rrd.h"
5
6 static void rrdhost_streaming_sender_structures_init(RRDHOST *host);
7
8 +bool dbengine_enabled = false; // will become true if and when dbengine is initialized
9 +size_t storage_tiers = 3;
10 +bool use_direct_io = true;
11 +size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
12 +RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
13 +
14 #if RRD_STORAGE_TIERS != 5
15 #error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier
16 #endif
17
18 size_t get_tier_grouping(size_t tier) {
12 - if(unlikely(tier >= rrdb.storage_tiers)) tier = rrdb.storage_tiers - 1;
19 + if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1;
20
21 size_t grouping = 1;
22 // first tier is always 1 iteration of whatever update every the chart has
23 for(size_t i = 1; i <= tier ;i++)
17 - grouping *= rrdb.storage_tiers_grouping_iterations[i];
24 + grouping *= storage_tiers_grouping_iterations[i];
25
26 return grouping;
27 }
28
29 +RRDHOST *localhost = NULL;
30 +netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER;
31 +
32 +time_t rrdset_free_obsolete_time_s = 3600;
33 +time_t rrdhost_free_orphan_time_s = 3600;
34 +
35 bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
36 #ifdef ENABLE_DBENGINE
37 if(!rrdeng_is_legacy(engine))
@@ -28,13 +41,14 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
41 return false;
42 }
43
31 -RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
44 +RRDHOST *find_host_by_node_id(char *node_id) {
45 +
46 uuid_t node_uuid;
47 if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
48 return NULL;
49
50 RRDHOST *host, *ret = NULL;
37 - dfe_start_read(rrdb.rrdhost_root_index, host) {
51 + dfe_start_read(rrdhost_root_index, host) {
52 if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) {
53 ret = host;
54 break;
@@ -48,10 +62,27 @@ RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
62 // ----------------------------------------------------------------------------
63 // RRDHOST indexes management
64
65 +DICTIONARY *rrdhost_root_index = NULL;
66 +static DICTIONARY *rrdhost_root_index_hostname = NULL;
67 +
68 +static inline void rrdhost_init() {
69 + if(unlikely(!rrdhost_root_index)) {
70 + rrdhost_root_index = dictionary_create_advanced(
71 + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
72 + &dictionary_stats_category_rrdhost, 0);
73 + }
74 +
75 + if(unlikely(!rrdhost_root_index_hostname)) {
76 + rrdhost_root_index_hostname = dictionary_create_advanced(
77 + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
78 + &dictionary_stats_category_rrdhost, 0);
79 + }
80 +}
81 +
82 RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid) {
83 netdata_log_debug(D_RRD_CALLS, "rrdhost_find_and_acquire() host %s", machine_guid);
84
54 - return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdb.rrdhost_root_index, machine_guid);
85 + return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdhost_root_index, machine_guid);
86 }
87
88 RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha) {
@@ -65,22 +96,22 @@ void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha) {
96 if(unlikely(!rha))
97 return;
98
68 - dictionary_acquired_item_release(rrdb.rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
99 + dictionary_acquired_item_release(rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
100 }
101
102 // ----------------------------------------------------------------------------
103 // RRDHOST index by UUID
104
105 inline size_t rrdhost_hosts_available(void) {
75 - return dictionary_entries(rrdb.rrdhost_root_index);
106 + return dictionary_entries(rrdhost_root_index);
107 }
108
109 inline RRDHOST *rrdhost_find_by_guid(const char *guid) {
79 - return dictionary_get(rrdb.rrdhost_root_index, guid);
110 + return dictionary_get(rrdhost_root_index, guid);
111 }
112
113 static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
83 - RRDHOST *ret_machine_guid = dictionary_set(rrdb.rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
114 + RRDHOST *ret_machine_guid = dictionary_set(rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
115 if(ret_machine_guid == host)
116 rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_MACHINE_GUID);
117 else {
@@ -94,7 +125,7 @@ static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
125
126 static void rrdhost_index_del_by_guid(RRDHOST *host) {
127 if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_MACHINE_GUID)) {
97 - if(!dictionary_del(rrdb.rrdhost_root_index, host->machine_guid))
128 + if(!dictionary_del(rrdhost_root_index, host->machine_guid))
129 netdata_log_error("RRDHOST: %s() failed to delete machine guid '%s' from index",
130 __FUNCTION__, host->machine_guid);
131
@@ -107,16 +138,16 @@ static void rrdhost_index_del_by_guid(RRDHOST *host) {
138
139 inline RRDHOST *rrdhost_find_by_hostname(const char *hostname) {
140 if(unlikely(!strcmp(hostname, "localhost")))
110 - return rrdb.localhost;
141 + return localhost;
142
112 - return dictionary_get(rrdb.rrdhost_root_index_hostname, hostname);
143 + return dictionary_get(rrdhost_root_index_hostname, hostname);
144 }
145
146 static inline void rrdhost_index_del_hostname(RRDHOST *host) {
147 if(unlikely(!host->hostname)) return;
148
149 if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_HOSTNAME)) {
119 - if(!dictionary_del(rrdb.rrdhost_root_index_hostname, rrdhost_hostname(host)))
150 + if(!dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)))
151 netdata_log_error("RRDHOST: %s() failed to delete hostname '%s' from index",
152 __FUNCTION__, rrdhost_hostname(host));
153
@@ -127,7 +158,7 @@ static inline void rrdhost_index_del_hostname(RRDHOST *host) {
158 static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) {
159 if(!host->hostname) return host;
160
130 - RRDHOST *ret_hostname = dictionary_set(rrdb.rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
161 + RRDHOST *ret_hostname = dictionary_set(rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
162 if(ret_hostname == host)
163 rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
164 else {
@@ -195,14 +226,14 @@ static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, co
226 host->utc_offset = utc_offset;
227 }
228
198 -void set_host_properties(RRDHOST *host, int update_every, STORAGE_ENGINE_ID storage_engine_id,
229 +void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode,
230 const char *registry_hostname, const char *os, const char *tags,
231 const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name,
232 const char *program_version)
233 {
234
204 - host->update_every = update_every;
205 - host->storage_engine_id = storage_engine_id;
235 + host->rrd_update_every = update_every;
236 + host->rrd_memory_mode = memory_mode;
237
238 rrdhost_init_os(host, os);
239 rrdhost_init_timezone(host, tzone, abbrev_tzone, utc_offset);
@@ -246,7 +277,7 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host,
277 rrdhost_option_clear(host, RRDHOST_OPTION_SENDER_ENABLED);
278 }
279
249 -RRDHOST *rrdhost_create(
280 +static RRDHOST *rrdhost_create(
281 const char *hostname,
282 const char *registry_hostname,
283 const char *guid,
@@ -259,7 +290,7 @@ RRDHOST *rrdhost_create(
290 const char *program_version,
291 int update_every,
292 long entries,
262 - STORAGE_ENGINE_ID storage_engine_id,
293 + RRD_MEMORY_MODE memory_mode,
294 unsigned int health_enabled,
295 unsigned int rrdpush_enabled,
296 char *rrdpush_destination,
@@ -274,30 +305,31 @@ RRDHOST *rrdhost_create(
305 ) {
306 netdata_log_debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
307
277 - if(storage_engine_id == STORAGE_ENGINE_DBENGINE && !rrdb.dbengine_enabled) {
278 - netdata_log_error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'", hostname);
279 - storage_engine_id = STORAGE_ENGINE_ALLOC;
308 + if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) {
309 + netdata_log_error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'",
310 + hostname);
311 + memory_mode = RRD_MEMORY_MODE_ALLOC;
312 }
313
314 #ifdef ENABLE_DBENGINE
283 - int is_legacy = (storage_engine_id == STORAGE_ENGINE_DBENGINE) && is_legacy_child(guid);
315 + int is_legacy = (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid);
316 #else
317 int is_legacy = 1;
318 #endif
319
288 - int is_in_multihost = (storage_engine_id == STORAGE_ENGINE_DBENGINE && !is_legacy);
320 + int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);
321 RRDHOST *host = callocz(1, sizeof(RRDHOST));
322 __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
323
324 strncpyz(host->machine_guid, guid, GUID_LEN + 1);
325
294 - set_host_properties(host, (update_every > 0)?update_every:1, storage_engine_id, registry_hostname, os,
326 + set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, registry_hostname, os,
327 tags, timezone, abbrev_timezone, utc_offset, program_name, program_version);
328
329 rrdhost_init_hostname(host, hostname, false);
330
299 - host->rrd_history_entries = align_entries_to_pagesize(storage_engine_id, entries);
300 - host->health.health_enabled = ((storage_engine_id == STORAGE_ENGINE_NONE)) ? 0 : health_enabled;
331 + host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
332 + host->health.health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
333
334 if (likely(!archived)) {
335 rrdfunctions_init(host);
@@ -315,16 +347,17 @@ int is_legacy = 1;
347 host->rrdpush_replication_step = rrdpush_replication_step;
348 host->rrdpush_receiver_replication_percent = 100.0;
349
318 - switch(storage_engine_id) {
319 - case STORAGE_ENGINE_DBENGINE:
320 - break;
321 - case STORAGE_ENGINE_ALLOC:
322 - case STORAGE_ENGINE_MAP:
323 - case STORAGE_ENGINE_SAVE:
324 - case STORAGE_ENGINE_RAM:
350 + switch(memory_mode) {
351 default:
326 - if(host->rrdpush_seconds_to_replicate > (time_t) host->rrd_history_entries * (time_t) host->update_every)
327 - host->rrdpush_seconds_to_replicate = (time_t) host->rrd_history_entries * (time_t) host->update_every;
352 + case RRD_MEMORY_MODE_ALLOC:
353 + case RRD_MEMORY_MODE_MAP:
354 + case RRD_MEMORY_MODE_SAVE:
355 + case RRD_MEMORY_MODE_RAM:
356 + if(host->rrdpush_seconds_to_replicate > (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every)
357 + host->rrdpush_seconds_to_replicate = (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every;
358 + break;
359 +
360 + case RRD_MEMORY_MODE_DBENGINE:
361 break;
362 }
363
@@ -354,8 +387,8 @@ int is_legacy = 1;
387 host->cache_dir = strdupz(filename);
388 }
389
357 - if((host->storage_engine_id == STORAGE_ENGINE_MAP || host->storage_engine_id == STORAGE_ENGINE_SAVE ||
358 - (host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_legacy))) {
390 + if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE ||
391 + (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
392 int r = mkdir(host->cache_dir, 0775);
393 if(r != 0 && errno != EEXIST)
394 netdata_log_error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir);
@@ -375,7 +408,7 @@ int is_legacy = 1;
408 rrdcalctemplate_index_init(host);
409 rrdcalc_rrdhost_index_init(host);
410
378 - if (host->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
411 + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
412 #ifdef ENABLE_DBENGINE
413 char dbenginepath[FILENAME_MAX + 1];
414 int ret;
@@ -391,29 +424,35 @@ int is_legacy = 1;
424 if (is_legacy) {
425 // initialize legacy dbengine instance as needed
426
394 - host->db[0].id = STORAGE_ENGINE_DBENGINE;
427 + host->db[0].mode = RRD_MEMORY_MODE_DBENGINE;
428 + host->db[0].eng = storage_engine_get(host->db[0].mode);
429 host->db[0].tier_grouping = get_tier_grouping(0);
430
397 - // may fail here for legacy dbengine initialization
398 - ret = rrdeng_init(&host->db[0].instance, dbenginepath, rrdb.default_rrdeng_disk_quota_mb, 0);
431 + ret = rrdeng_init(
432 + (struct rrdengine_instance **)&host->db[0].instance,
433 + dbenginepath,
434 + default_rrdeng_disk_quota_mb,
435 + 0); // may fail here for legacy dbengine initialization
436
437 if(ret == 0) {
401 - rrdeng_readiness_wait(host->db[0].instance);
438 + rrdeng_readiness_wait((struct rrdengine_instance *)host->db[0].instance);
439
440 // assign the rest of the shared storage instances to it
441 // to allow them collect its metrics too
442
406 - for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
407 - host->db[tier].id= STORAGE_ENGINE_DBENGINE;
408 - host->db[tier].instance = rrdb.multidb_ctx[tier];
443 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
444 + host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
445 + host->db[tier].eng = storage_engine_get(host->db[tier].mode);
446 + host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
447 host->db[tier].tier_grouping = get_tier_grouping(tier);
448 }
449 }
450 }
451 else {
414 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
415 - host->db[tier].id = STORAGE_ENGINE_DBENGINE;
416 - host->db[tier].instance = rrdb.multidb_ctx[tier];
452 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
453 + host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
454 + host->db[tier].eng = storage_engine_get(host->db[tier].mode);
455 + host->db[tier].instance = (STORAGE_INSTANCE *)multidb_ctx[tier];
456 host->db[tier].tier_grouping = get_tier_grouping(tier);
457 }
458 }
@@ -430,19 +469,21 @@ int is_legacy = 1;
469 }
470
471 #else
433 - fatal("STORAGE_ENGINE_DBENGINE is not supported in this platform.");
472 + fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform.");
473 #endif
474 }
475 else {
437 - host->db[0].id = host->storage_engine_id;
476 + host->db[0].mode = host->rrd_memory_mode;
477 + host->db[0].eng = storage_engine_get(host->db[0].mode);
478 host->db[0].instance = NULL;
479 host->db[0].tier_grouping = get_tier_grouping(0);
480
481 #ifdef ENABLE_DBENGINE
482 // the first tier is reserved for the non-dbengine modes
443 - for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
444 - host->db[tier].id = STORAGE_ENGINE_DBENGINE;
445 - host->db[tier].instance = rrdb.multidb_ctx[tier];
483 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
484 + host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
485 + host->db[tier].eng = storage_engine_get(host->db[tier].mode);
486 + host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
487 host->db[tier].tier_grouping = get_tier_grouping(tier);
488 }
489 #endif
@@ -476,9 +517,9 @@ int is_legacy = 1;
517 rrdhost_index_add_hostname(host);
518
519 if(is_localhost)
479 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
520 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(localhost, host, prev, next);
521 else
481 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
522 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(localhost, host, prev, next);
523
524 rrd_unlock();
525
@@ -507,8 +548,8 @@ int is_legacy = 1;
548 , rrdhost_tags(host)
549 , rrdhost_program_name(host)
550 , rrdhost_program_version(host)
510 - , host->update_every
511 - , storage_engine_name(host->storage_engine_id)
551 + , host->rrd_update_every
552 + , rrd_memory_mode_name(host->rrd_memory_mode)
553 , host->rrd_history_entries
554 , rrdhost_has_rrdpush_sender_enabled(host)?"enabled":"disabled"
555 , host->rrdpush_send_destination?host->rrdpush_send_destination:""
@@ -543,7 +584,7 @@ static void rrdhost_update(RRDHOST *host
584 , const char *program_version
585 , int update_every
586 , long history
546 - , STORAGE_ENGINE_ID storage_engine_id
587 + , RRD_MEMORY_MODE mode
588 , unsigned int health_enabled
589 , unsigned int rrdpush_enabled
590 , char *rrdpush_destination
@@ -559,7 +600,7 @@ static void rrdhost_update(RRDHOST *host
600
601 spinlock_lock(&host->rrdhost_update_lock);
602
562 - host->health.health_enabled = (storage_engine_id == STORAGE_ENGINE_NONE) ? 0 : health_enabled;
603 + host->health.health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;
604
605 {
606 struct rrdhost_system_info *old = host->system_info;
@@ -595,20 +636,24 @@ static void rrdhost_update(RRDHOST *host
636 string_freez(t);
637 }
638
598 - if(host->update_every != update_every)
639 + if(host->rrd_update_every != update_every)
640 netdata_log_error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. "
641 "Restart netdata here to apply the new settings.",
601 - rrdhost_hostname(host), host->update_every, update_every);
642 + rrdhost_hostname(host), host->rrd_update_every, update_every);
643
603 - if(host->storage_engine_id != storage_engine_id) {
644 + if(host->rrd_memory_mode != mode)
645 netdata_log_error("Host '%s' has memory mode '%s', but the wanted one is '%s'. "
646 "Restart netdata here to apply the new settings.",
606 - rrdhost_hostname(host), storage_engine_name(host->storage_engine_id), storage_engine_name(storage_engine_id));
607 - } else if(host->storage_engine_id != STORAGE_ENGINE_DBENGINE && host->rrd_history_entries < history) {
647 + rrdhost_hostname(host),
648 + rrd_memory_mode_name(host->rrd_memory_mode),
649 + rrd_memory_mode_name(mode));
650 +
651 + else if(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && host->rrd_history_entries < history)
652 netdata_log_error("Host '%s' has history of %d entries, but the wanted one is %ld entries. "
653 "Restart netdata here to apply the new settings.",
610 - rrdhost_hostname(host), host->rrd_history_entries, history);
611 - }
654 + rrdhost_hostname(host),
655 + host->rrd_history_entries,
656 + history);
657
658 // update host tags
659 rrdhost_init_tags(host, tags);
@@ -654,7 +699,7 @@ static void rrdhost_update(RRDHOST *host
699 spinlock_unlock(&host->rrdhost_update_lock);
700 }
701
657 -RRDHOST *rrdhost_get_or_create(
702 +RRDHOST *rrdhost_find_or_create(
703 const char *hostname
704 , const char *registry_hostname
705 , const char *guid
@@ -667,7 +712,7 @@ RRDHOST *rrdhost_get_or_create(
712 , const char *program_version
713 , int update_every
714 , long history
670 - , STORAGE_ENGINE_ID storage_engine_id
715 + , RRD_MEMORY_MODE mode
716 , unsigned int health_enabled
717 , unsigned int rrdpush_enabled
718 , char *rrdpush_destination
@@ -682,16 +727,16 @@ RRDHOST *rrdhost_get_or_create(
727 netdata_log_debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);
728
729 RRDHOST *host = rrdhost_find_by_guid(guid);
685 - if (unlikely(host && host->storage_engine_id != storage_engine_id && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
730 + if (unlikely(host && host->rrd_memory_mode != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
731
732 if (likely(!archived && rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
733 return host;
734
690 - /* If a legacy storage engine instantiates all dbengine state must be discarded to avoid inconsistencies */
735 + /* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */
736 netdata_log_error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",
737 rrdhost_hostname(host),
693 - storage_engine_name(host->storage_engine_id),
694 - storage_engine_name(storage_engine_id));
738 + rrd_memory_mode_name(host->rrd_memory_mode),
739 + rrd_memory_mode_name(mode));
740
741 rrd_wrlock();
742 rrdhost_free___while_having_rrd_wrlock(host, true);
@@ -713,7 +758,7 @@ RRDHOST *rrdhost_get_or_create(
758 , program_version
759 , update_every
760 , history
716 - , storage_engine_id
761 + , mode
762 , health_enabled
763 , rrdpush_enabled
764 , rrdpush_destination
@@ -742,7 +787,7 @@ RRDHOST *rrdhost_get_or_create(
787 , program_version
788 , update_every
789 , history
745 - , storage_engine_id
790 + , mode
791 , health_enabled
792 , rrdpush_enabled
793 , rrdpush_destination
@@ -759,19 +804,284 @@ RRDHOST *rrdhost_get_or_create(
804
805 inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s) {
806 if(host != protected_host
762 - && host != rrdb.localhost
807 + && host != localhost
808 && rrdhost_receiver_replicating_charts(host) == 0
809 && rrdhost_sender_replicating_charts(host) == 0
810 && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)
811 && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)
812 && !host->receiver
813 && host->child_disconnected_time
769 - && host->child_disconnected_time + rrdb.rrdhost_free_orphan_time_s < now_s)
814 + && host->child_disconnected_time + rrdhost_free_orphan_time_s < now_s)
815 return 1;
816
817 return 0;
818 }
819
820 +// ----------------------------------------------------------------------------
821 +// RRDHOST global / startup initialization
822 +
823 +#ifdef ENABLE_DBENGINE
824 +struct dbengine_initialization {
825 + netdata_thread_t thread;
826 + char path[FILENAME_MAX + 1];
827 + int disk_space_mb;
828 + size_t tier;
829 + int ret;
830 +};
831 +
832 +void *dbengine_tier_init(void *ptr) {
833 + struct dbengine_initialization *dbi = ptr;
834 + dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);
835 + return ptr;
836 +}
837 +#endif
838 +
839 +void dbengine_init(char *hostname) {
840 +#ifdef ENABLE_DBENGINE
841 + use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", use_direct_io);
842 +
843 + unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
844 + if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
845 + rrdeng_pages_per_extent = read_num;
846 + else {
847 + netdata_log_error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
848 + config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
849 + }
850 +
851 + storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
852 + if(storage_tiers < 1) {
853 + netdata_log_error("At least 1 storage tier is required. Assuming 1.");
854 + storage_tiers = 1;
855 + config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
856 + }
857 + if(storage_tiers > RRD_STORAGE_TIERS) {
858 + netdata_log_error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS);
859 + storage_tiers = RRD_STORAGE_TIERS;
860 + config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
861 + }
862 +
863 + bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
864 + parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
865 +
866 + struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
867 +
868 + size_t created_tiers = 0;
869 + char dbenginepath[FILENAME_MAX + 1];
870 + char dbengineconfig[200 + 1];
871 + int divisor = 1;
872 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
873 + if(tier == 0)
874 + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
875 + else
876 + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
877 +
878 + int ret = mkdir(dbenginepath, 0775);
879 + if (ret != 0 && errno != EEXIST) {
880 + netdata_log_error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
881 + break;
882 + }
883 +
884 + if(tier > 0)
885 + divisor *= 2;
886 +
887 + int disk_space_mb = default_multidb_disk_quota_mb / divisor;
888 + size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
889 + RRD_BACKFILL backfill = storage_tiers_backfill[tier];
890 +
891 + if(tier > 0) {
892 + snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
893 + disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
894 +
895 + snprintfz(dbengineconfig, 200, "dbengine tier %zu update every iterations", tier);
896 + grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
897 + if(grouping_iterations < 2) {
898 + grouping_iterations = 2;
899 + config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
900 + netdata_log_error("DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",
901 + hostname,
902 + tier);
903 + }
904 +
905 + snprintfz(dbengineconfig, 200, "dbengine tier %zu backfill", tier);
906 + const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
907 + if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
908 + else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
909 + else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;
910 + else {
911 + netdata_log_error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf);
912 + config_set(CONFIG_SECTION_DB, dbengineconfig, "new");
913 + backfill = RRD_BACKFILL_NEW;
914 + }
915 + }
916 +
917 + storage_tiers_grouping_iterations[tier] = grouping_iterations;
918 + storage_tiers_backfill[tier] = backfill;
919 +
920 + if(tier > 0 && get_tier_grouping(tier) > 65535) {
921 + storage_tiers_grouping_iterations[tier] = 1;
922 + netdata_log_error("DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. Disabling tiers above %zu",
923 + hostname,
924 + tier,
925 + tier);
926 + break;
927 + }
928 +
929 + internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);
930 +
931 + tiers_init[tier].disk_space_mb = disk_space_mb;
932 + tiers_init[tier].tier = tier;
933 + strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
934 + tiers_init[tier].ret = 0;
935 +
936 + if(parallel_initialization) {
937 + char tag[NETDATA_THREAD_TAG_MAX + 1];
938 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier);
939 + netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE,
940 + dbengine_tier_init, &tiers_init[tier]);
941 + }
942 + else
943 + dbengine_tier_init(&tiers_init[tier]);
944 + }
945 +
946 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
947 + void *ptr;
948 +
949 + if(parallel_initialization)
950 + netdata_thread_join(tiers_init[tier].thread, &ptr);
951 +
952 + if(tiers_init[tier].ret != 0) {
953 + netdata_log_error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
954 + hostname,
955 + tiers_init[tier].tier,
956 + tiers_init[tier].path);
957 + }
958 + else if(created_tiers == tier)
959 + created_tiers++;
960 + }
961 +
962 + if(created_tiers && created_tiers < storage_tiers) {
963 + netdata_log_error("DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.",
964 + hostname,
965 + created_tiers,
966 + storage_tiers,
967 + created_tiers);
968 + storage_tiers = created_tiers;
969 + }
970 + else if(!created_tiers)
971 + fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
972 +
973 + for(size_t tier = 0; tier < storage_tiers ;tier++)
974 + rrdeng_readiness_wait(multidb_ctx[tier]);
975 +
976 + dbengine_enabled = true;
977 +#else
978 + storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
979 + if(storage_tiers != 1) {
980 + netdata_log_error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname);
981 + storage_tiers = 1;
982 + config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
983 + }
984 + dbengine_enabled = false;
985 +#endif
986 +}
987 +
988 +int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
989 + rrdhost_init();
990 +
991 + if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
992 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
993 + set_late_global_environment(system_info);
994 + fatal("Failed to initialize SQLite");
995 + }
996 + netdata_log_info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
997 + }
998 +
999 + if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
1000 + error_report("Failed to initialize context metadata database");
1001 + }
1002 +
1003 + if (unlikely(unittest)) {
1004 + dbengine_enabled = true;
1005 + }
1006 + else {
1007 + health_init();
1008 + rrdpush_init();
1009 +
1010 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
1011 + netdata_log_info("DBENGINE: Initializing ...");
1012 + dbengine_init(hostname);
1013 + }
1014 + else {
1015 + netdata_log_info("DBENGINE: Not initializing ...");
1016 + storage_tiers = 1;
1017 + }
1018 +
1019 + if (!dbengine_enabled) {
1020 + if (storage_tiers > 1) {
1021 + netdata_log_error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
1022 + storage_tiers);
1023 + storage_tiers = 1;
1024 + }
1025 +
1026 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1027 + netdata_log_error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
1028 + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
1029 + }
1030 + }
1031 + }
1032 +
1033 + if(!unittest)
1034 + metadata_sync_init();
1035 +
1036 + netdata_log_debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
1037 + localhost = rrdhost_create(
1038 + hostname
1039 + , registry_get_this_machine_hostname()
1040 + , registry_get_this_machine_guid()
1041 + , os_type
1042 + , netdata_configured_timezone
1043 + , netdata_configured_abbrev_timezone
1044 + , netdata_configured_utc_offset
1045 + , ""
1046 + , program_name
1047 + , program_version
1048 + , default_rrd_update_every
1049 + , default_rrd_history_entries
1050 + , default_rrd_memory_mode
1051 + , default_health_enabled
1052 + , default_rrdpush_enabled
1053 + , default_rrdpush_destination
1054 + , default_rrdpush_api_key
1055 + , default_rrdpush_send_charts_matching
1056 + , default_rrdpush_enable_replication
1057 + , default_rrdpush_seconds_to_replicate
1058 + , default_rrdpush_replication_step
1059 + , system_info
1060 + , 1
1061 + , 0
1062 + );
1063 +
1064 + if (unlikely(!localhost)) {
1065 + return 1;
1066 + }
1067 +
1068 +#ifdef NETDATA_DEV_MODE
1069 + // we register this only on localhost
1070 + // for the other nodes, the origin server should register it
1071 + rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
1072 + rrd_collector_add_function(localhost, NULL, "streaming", 10,
1073 + RRDFUNCTIONS_STREAMING_HELP, true,
1074 + rrdhost_function_streaming, NULL);
1075 +#endif
1076 +
1077 + if (likely(system_info)) {
1078 + migrate_localhost(&localhost->host_uuid);
1079 + sql_aclk_sync_init();
1080 + web_client_api_v1_management_init();
1081 + }
1082 + return localhost==NULL;
1083 +}
1084 +
1085 // ----------------------------------------------------------------------------
1086 // RRDHOST - free
1087
@@ -874,7 +1184,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1184 rrdhost_index_del_by_guid(host);
1185
1186 if (host->prev)
877 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
1187 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(localhost, host, prev, next);
1188 }
1189
1190 // ------------------------------------------------------------------------
@@ -895,11 +1205,11 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1205 // release its children resources
1206
1207 #ifdef ENABLE_DBENGINE
898 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
899 - if(host->db[tier].id == STORAGE_ENGINE_DBENGINE
1208 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
1209 + if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
1210 && host->db[tier].instance
1211 && !is_storage_engine_shared(host->db[tier].instance))
902 - rrdeng_prepare_exit(host->db[tier].instance);
1212 + rrdeng_prepare_exit((struct rrdengine_instance *)host->db[tier].instance);
1213 }
1214 #endif
1215
@@ -916,11 +1226,11 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1226 health_alarm_log_free(host);
1227
1228 #ifdef ENABLE_DBENGINE
919 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
920 - if(host->db[tier].id == STORAGE_ENGINE_DBENGINE
1229 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
1230 + if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
1231 && host->db[tier].instance
1232 && !is_storage_engine_shared(host->db[tier].instance))
923 - rrdeng_exit(host->db[tier].instance);
1233 + rrdeng_exit((struct rrdengine_instance *)host->db[tier].instance);
1234 }
1235 #endif
1236
@@ -957,7 +1267,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1267 rrdfamily_index_destroy(host);
1268 rrdfunctions_destroy(host);
1269 rrdvariables_destroy(host->rrdvars);
960 - if (host == rrdb.localhost)
1270 + if (host == localhost)
1271 rrdvariables_destroy(health_rrdvars);
1272
1273 rrdhost_destroy_rrdcontexts(host);
@@ -971,18 +1281,18 @@ void rrdhost_free_all(void) {
1281 rrd_wrlock();
1282
1283 /* Make sure child-hosts are released before the localhost. */
974 - while (rrdb.localhost && rrdb.localhost->next)
975 - rrdhost_free___while_having_rrd_wrlock(rrdb.localhost->next, true);
1284 + while(localhost && localhost->next)
1285 + rrdhost_free___while_having_rrd_wrlock(localhost->next, true);
1286
977 - if (rrdb.localhost)
978 - rrdhost_free___while_having_rrd_wrlock(rrdb.localhost, true);
1287 + if(localhost)
1288 + rrdhost_free___while_having_rrd_wrlock(localhost, true);
1289
1290 rrd_unlock();
1291 }
1292
1293 void rrd_finalize_collection_for_all_hosts(void) {
1294 RRDHOST *host;
985 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1295 + dfe_start_reentrant(rrdhost_root_index, host) {
1296 rrdhost_finalize_collection(host);
1297 }
1298 dfe_done(host);
@@ -1006,22 +1316,181 @@ void rrdhost_save_charts(RRDHOST *host) {
1316 rrdset_foreach_done(st);
1317 }
1318
1319 +struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
1320 + struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
1321 + info->hops = 1;
1322 +
1323 + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type");
1324 + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type");
1325 + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region");
1326 + rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name");
1327 + rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version");
1328 + rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version");
1329 + rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores");
1330 + rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq");
1331 + rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total");
1332 + rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space");
1333 + rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture");
1334 + rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization");
1335 + rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container");
1336 + rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection");
1337 + rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection");
1338 + rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node");
1339 + rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type");
1340 + rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch");
1341 + rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist");
1342 +
1343 + return info;
1344 +}
1345 +
1346 +static void rrdhost_load_auto_labels(void) {
1347 + DICTIONARY *labels = localhost->rrdlabels;
1348 +
1349 + if (localhost->system_info->cloud_provider_type)
1350 + rrdlabels_add(labels, "_cloud_provider_type", localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
1351 +
1352 + if (localhost->system_info->cloud_instance_type)
1353 + rrdlabels_add(labels, "_cloud_instance_type", localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
1354 +
1355 + if (localhost->system_info->cloud_instance_region)
1356 + rrdlabels_add(labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
1357 +
1358 + if (localhost->system_info->host_os_name)
1359 + rrdlabels_add(labels, "_os_name", localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
1360 +
1361 + if (localhost->system_info->host_os_version)
1362 + rrdlabels_add(labels, "_os_version", localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO);
1363 +
1364 + if (localhost->system_info->kernel_version)
1365 + rrdlabels_add(labels, "_kernel_version", localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO);
1366 +
1367 + if (localhost->system_info->host_cores)
1368 + rrdlabels_add(labels, "_system_cores", localhost->system_info->host_cores, RRDLABEL_SRC_AUTO);
1369 +
1370 + if (localhost->system_info->host_cpu_freq)
1371 + rrdlabels_add(labels, "_system_cpu_freq", localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO);
1372 +
1373 + if (localhost->system_info->host_ram_total)
1374 + rrdlabels_add(labels, "_system_ram_total", localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO);
1375 +
1376 + if (localhost->system_info->host_disk_space)
1377 + rrdlabels_add(labels, "_system_disk_space", localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO);
1378 +
1379 + if (localhost->system_info->architecture)
1380 + rrdlabels_add(labels, "_architecture", localhost->system_info->architecture, RRDLABEL_SRC_AUTO);
1381 +
1382 + if (localhost->system_info->virtualization)
1383 + rrdlabels_add(labels, "_virtualization", localhost->system_info->virtualization, RRDLABEL_SRC_AUTO);
1384 +
1385 + if (localhost->system_info->container)
1386 + rrdlabels_add(labels, "_container", localhost->system_info->container, RRDLABEL_SRC_AUTO);
1387 +
1388 + if (localhost->system_info->container_detection)
1389 + rrdlabels_add(labels, "_container_detection", localhost->system_info->container_detection, RRDLABEL_SRC_AUTO);
1390 +
1391 + if (localhost->system_info->virt_detection)
1392 + rrdlabels_add(labels, "_virt_detection", localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO);
1393 +
1394 + if (localhost->system_info->is_k8s_node)
1395 + rrdlabels_add(labels, "_is_k8s_node", localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO);
1396 +
1397 + if (localhost->system_info->install_type)
1398 + rrdlabels_add(labels, "_install_type", localhost->system_info->install_type, RRDLABEL_SRC_AUTO);
1399 +
1400 + if (localhost->system_info->prebuilt_arch)
1401 + rrdlabels_add(labels, "_prebuilt_arch", localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO);
1402 +
1403 + if (localhost->system_info->prebuilt_dist)
1404 + rrdlabels_add(labels, "_prebuilt_dist", localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO);
1405 +
1406 + add_aclk_host_labels();
1407 +
1408 + health_add_host_labels();
1409 +
1410 + rrdlabels_add(labels, "_is_parent", (localhost->connected_children_count > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);
1411 +
1412 + if (localhost->rrdpush_send_destination)
1413 + rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
1414 +}
1415 +
1416 void rrdhost_set_is_parent_label(void) {
1010 - int count = __atomic_load_n(&rrdb.localhost->connected_children_count, __ATOMIC_RELAXED);
1417 + int count = __atomic_load_n(&localhost->connected_children_count, __ATOMIC_RELAXED);
1418
1419 if (count == 0 || count == 1) {
1013 - DICTIONARY *labels = rrdb.localhost->rrdlabels;
1420 + DICTIONARY *labels = localhost->rrdlabels;
1421 rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);
1422
1423 //queue a node info
1424 #ifdef ENABLE_ACLK
1425 if (netdata_cloud_enabled) {
1019 - aclk_queue_node_info(rrdb.localhost, false);
1426 + aclk_queue_node_info(localhost, false);
1427 }
1428 #endif
1429 }
1430 }
1431
1432 +static void rrdhost_load_config_labels(void) {
1433 + int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
1434 + if(!status) {
1435 + char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
1436 + netdata_log_error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
1437 + }
1438 +
1439 + struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
1440 + if(co) {
1441 + config_section_wrlock(co);
1442 + struct config_option *cv;
1443 + for(cv = co->values; cv ; cv = cv->next) {
1444 + rrdlabels_add(localhost->rrdlabels, cv->name, cv->value, RRDLABEL_SRC_CONFIG);
1445 + cv->flags |= CONFIG_VALUE_USED;
1446 + }
1447 + config_section_unlock(co);
1448 + }
1449 +}
1450 +
1451 +static void rrdhost_load_kubernetes_labels(void) {
1452 + char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
1453 + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
1454 +
1455 + if (unlikely(access(label_script, R_OK) != 0)) {
1456 + netdata_log_error("Kubernetes pod label fetching script %s not found.",label_script);
1457 + return;
1458 + }
1459 +
1460 + netdata_log_debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
1461 +
1462 + pid_t pid;
1463 + FILE *fp_child_input;
1464 + FILE *fp_child_output = netdata_popen(label_script, &pid, &fp_child_input);
1465 + if(!fp_child_output) return;
1466 +
1467 + char buffer[1000 + 1];
1468 + while (fgets(buffer, 1000, fp_child_output) != NULL)
1469 + rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);
1470 +
1471 + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
1472 + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
1473 + int rc = netdata_pclose(fp_child_input, fp_child_output, pid);
1474 + if(rc)
1475 + netdata_log_error("%s exited abnormally. Failed to get kubernetes labels.", label_script);
1476 +}
1477 +
1478 +void reload_host_labels(void) {
1479 + if(!localhost->rrdlabels)
1480 + localhost->rrdlabels = rrdlabels_create();
1481 +
1482 + rrdlabels_unmark_all(localhost->rrdlabels);
1483 +
1484 + // priority is important here
1485 + rrdhost_load_config_labels();
1486 + rrdhost_load_kubernetes_labels();
1487 + rrdhost_load_auto_labels();
1488 +
1489 + rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1490 +
1491 + rrdpush_send_host_labels(localhost);
1492 +}
1493 +
1494 void rrdhost_finalize_collection(RRDHOST *host) {
1495 netdata_log_info("RRD: 'host:%s' stopping data collection...", rrdhost_hostname(host));
1496
@@ -1041,7 +1510,7 @@ void rrdhost_delete_charts(RRDHOST *host) {
1510
1511 RRDSET *st;
1512
1044 - if(host->storage_engine_id == STORAGE_ENGINE_SAVE || host->storage_engine_id == STORAGE_ENGINE_MAP) {
1513 + if(host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
1514 // we get a write lock
1515 // to ensure only one thread is saving the database
1516 rrdset_foreach_write(st, host){
@@ -1056,23 +1525,6 @@ void rrdhost_delete_charts(RRDHOST *host) {
1525 // ----------------------------------------------------------------------------
1526 // RRDHOST - cleanup host files
1527
1059 -static void delete_obsolete_dimensions(RRDSET *st) {
1060 - RRDDIM *rd;
1061 -
1062 - netdata_log_info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
1063 -
1064 - rrddim_foreach_read(rd, st) {
1065 - if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
1066 - const char *cache_filename = rrddim_cache_filename(rd);
1067 - if(!cache_filename) continue;
1068 - netdata_log_info("Deleting dimension file '%s'.", cache_filename);
1069 - if(unlikely(unlink(cache_filename) == -1))
1070 - netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
1071 - }
1072 - }
1073 - rrddim_foreach_done(rd);
1074 -}
1075 -
1528 void rrdhost_cleanup_charts(RRDHOST *host) {
1529 if(!host) return;
1530
@@ -1089,7 +1541,7 @@ void rrdhost_cleanup_charts(RRDHOST *host) {
1541 rrdset_delete_files(st);
1542
1543 else if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))
1092 - delete_obsolete_dimensions(st);
1544 + rrdset_delete_obsolete_dimensions(st);
1545
1546 else
1547 rrdset_save(st);
@@ -1124,9 +1576,9 @@ void rrdhost_cleanup_all(void) {
1576
1577 RRDHOST *host;
1578 rrdhost_foreach_read(host) {
1127 - if (host != rrdb.localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver
1579 + if (host != localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver
1580 /* don't delete multi-host DB host files */
1129 - && !(host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
1581 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
1582 )
1583 rrdhost_delete_charts(host);
1584 else
@@ -1291,20 +1743,6 @@ static NETDATA_DOUBLE rrdhost_sender_replication_completion_unsafe(RRDHOST *host
1743 return completion;
1744 }
1745
1294 -static void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to)
1295 -{
1296 - spinlock_lock(&host->retention.spinlock);
1297 - time_t first_time_s = host->retention.first_time_s;
1298 - time_t last_time_s = host->retention.last_time_s;
1299 - spinlock_unlock(&host->retention.spinlock);
1300 -
1301 - if (from)
1302 - *from = first_time_s;
1303 -
1304 - if (to)
1305 - *to = online ? now : last_time_s;
1306 -}
1307 -
1746 bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) {
1747 time_t first_time_s, last_time_s;
1748 rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s);
@@ -1337,7 +1775,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1775 else
1776 s->db.status = RRDHOST_DB_STATUS_QUERYABLE;
1777
1340 - s->db.storage_engine_id = host->storage_engine_id;
1778 + s->db.mode = host->rrd_memory_mode;
1779
1780 // --- ingest ---
1781
@@ -1345,7 +1783,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1783 s->ingest.reason = (online) ? STREAM_HANDSHAKE_NEVER : host->rrdpush_last_receiver_exit_reason;
1784
1785 netdata_mutex_lock(&host->receiver_lock);
1348 - s->ingest.hops = (host->system_info ? host->system_info->hops : (host == rrdb.localhost) ? 0 : 1);
1786 + s->ingest.hops = (host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
1787 bool has_receiver = false;
1788 if (host->receiver) {
1789 has_receiver = true;
@@ -1365,7 +1803,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1803 if(s->db.status == RRDHOST_DB_STATUS_INITIALIZING)
1804 s->ingest.status = RRDHOST_INGEST_STATUS_INITIALIZING;
1805
1368 - else if (host == rrdb.localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) {
1806 + else if (host == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) {
1807 s->ingest.status = RRDHOST_INGEST_STATUS_ONLINE;
1808 s->ingest.since = netdata_start_time;
1809 }
@@ -1386,7 +1824,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1824 s->ingest.status = RRDHOST_INGEST_STATUS_OFFLINE;
1825 }
1826
1389 - if(host == rrdb.localhost)
1827 + if(host == localhost)
1828 s->ingest.type = RRDHOST_INGEST_TYPE_LOCALHOST;
1829 else if(has_receiver || rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
1830 s->ingest.type = RRDHOST_INGEST_TYPE_CHILD;
database/rrdlabels.c
+31 -1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "rrdlabels.h"
3 +#define NETDATA_RRD_INTERNALS
4 +#include "rrd.h"
5
6 // ----------------------------------------------------------------------------
7 // labels sanitization
@@ -633,6 +634,23 @@ void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) {
634 rrdlabels_add(dict, name, value, ls);
635 }
636
637 +// ----------------------------------------------------------------------------
638 +// rrdlabels_get_value_to_buffer_or_null()
639 +
640 +void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null) {
641 + if(!labels) return;
642 +
643 + const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
644 + RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
645 +
646 + if(lb && lb->label_value)
647 + buffer_sprintf(wb, "%s%s%s", quote, string2str(lb->label_value), quote);
648 + else
649 + buffer_strcat(wb, null);
650 +
651 + dictionary_acquired_item_release(labels, acquired_item);
652 +}
653 +
654 void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key) {
655 if(!labels) return;
656
@@ -1012,6 +1030,18 @@ void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb) {
1030 dfe_done(lb);
1031 }
1032
1033 +void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
1034 + if(!st->rrdlabels)
1035 + st->rrdlabels = rrdlabels_create();
1036 +
1037 + if (new_rrdlabels)
1038 + rrdlabels_migrate_to_these(st->rrdlabels, new_rrdlabels);
1039 +
1040 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
1041 + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
1042 +}
1043 +
1044 +
1045 // ----------------------------------------------------------------------------
1046 // rrdlabels unit test
1047
database/rrdlabels.h deleted
-60
@@ -1,60 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_RRDLABELS_H
4 -#define NETDATA_RRDLABELS_H
5 -
6 -#include "rrd.h"
7 -
8 -typedef enum __attribute__ ((__packed__)) {
9 - RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
10 - RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
11 - RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
12 - RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
13 -
14 - // more sources can be added here
15 -
16 - RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
17 - RRDLABEL_FLAG_OLD = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
18 - RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels
19 -} RRDLABEL_SRC;
20 -
21 -#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
22 -
23 -size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
24 -
25 -DICTIONARY *rrdlabels_create(void);
26 -void rrdlabels_destroy(DICTIONARY *labels_dict);
27 -void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
28 -void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
29 -void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key);
30 -void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key);
31 -void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key);
32 -STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key);
33 -STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset);
34 -void rrdlabels_flush(DICTIONARY *labels_dict);
35 -
36 -void rrdlabels_unmark_all(DICTIONARY *labels);
37 -void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
38 -
39 -int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
40 -int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
41 -
42 -void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
43 -bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
44 -
45 -bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
46 -int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them,
47 - bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data,
48 - void (*name_sanitizer)(char *dst, const char *src, size_t dst_size),
49 - void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
50 -void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb);
51 -
52 -void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
53 -void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
54 -
55 -int rrdlabels_unittest(void);
56 -
57 -// unfortunately this break when defined in exporting_engine.h
58 -bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
59 -
60 -#endif /* NETDATA_RRDLABELS_H */
database/rrdset.c
+324 -335
@@ -1,290 +1,9 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 +#define NETDATA_RRD_INTERNALS
4 #include "rrd.h"
5 #include <sched.h>
5 -
6 -#ifdef NETDATA_INTERNAL_CHECKS
7 -#define rrdset_debug(st, fmt, args...) do { \
8 - if(unlikely((debug_flags & D_RRD_STATS) && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
9 - debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); \
10 -} while(0)
11 -#else
12 -#define rrdset_debug(st, fmt, args...) debug_dummy()
13 -#endif
14 -
15 -// ----------------------------------------------------------------------------
16 -// RRDSET - helpers for rrdset_create()
17 -
18 -inline long align_entries_to_pagesize(STORAGE_ENGINE_ID id, long entries) {
19 - if(id == STORAGE_ENGINE_DBENGINE)
20 - return 0;
21 -
22 - if(id == STORAGE_ENGINE_NONE)
23 - return 5;
24 -
25 - if(entries < 5) entries = 5;
26 - if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
27 -
28 - if(id == STORAGE_ENGINE_MAP || id == STORAGE_ENGINE_SAVE || id == STORAGE_ENGINE_RAM) {
29 - long header_size = 0;
30 -
31 - if(id == STORAGE_ENGINE_MAP || id == STORAGE_ENGINE_SAVE)
32 - header_size = (long)rrddim_memory_file_header_size();
33 -
34 - long page = (long)sysconf(_SC_PAGESIZE);
35 - long size = (long)(header_size + entries * sizeof(storage_number));
36 - if (unlikely(size % page)) {
37 - size -= (size % page);
38 - size += page;
39 -
40 - long n = (long)((size - header_size) / sizeof(storage_number));
41 - return n;
42 - }
43 - }
44 -
45 - return entries;
46 -}
47 -
48 -static inline void last_collected_time_align(RRDSET *st) {
49 - st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
50 -
51 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
52 - st->last_collected_time.tv_usec = 0;
53 - else
54 - st->last_collected_time.tv_usec = 500000;
55 -}
56 -
57 -static inline void last_updated_time_align(RRDSET *st) {
58 - st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
59 - st->last_updated.tv_usec = 0;
60 -}
61 -
62 -// ----------------------------------------------------------------------------
63 -// compatibility layer for RRDSET files v019
64 -
65 -#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
66 -#define RRD_ID_LENGTH_MAX_V019 200
67 -
68 -struct avl_element_v019 {
69 - void *avl_link[2];
70 - signed char avl_balance;
71 -};
72 -
73 -struct avl_tree_type_v019 {
74 - void *root;
75 - int (*compar)(void *a, void *b);
76 -};
77 -
78 -struct avl_tree_lock_v019 {
79 - struct avl_tree_type_v019 avl_tree;
80 - pthread_rwlock_t rwlock;
81 -};
82 -
83 -struct rrdset_map_save_v019 {
84 - struct avl_element_v019 avl; // ignored
85 - struct avl_element_v019 avlname; // ignored
86 - char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
87 - void *name; // ignored
88 - void *unused_ptr; // ignored
89 - void *type; // ignored
90 - void *family; // ignored
91 - void *title; // ignored
92 - void *units; // ignored
93 - void *context; // ignored
94 - uint32_t hash_context; // ignored
95 - uint32_t chart_type; // ignored
96 - int update_every; // check to reset all - update on load
97 - long entries; // check to reset all - update on load
98 - long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
99 - uint32_t flags; // ignored
100 - void *exporting_flags; // ignored
101 - int gap_when_lost_iterations_above; // ignored
102 - long priority; // ignored
103 - uint32_t storage_engine_id; // ignored
104 - void *cache_dir; // ignored
105 - char cache_filename[FILENAME_MAX+1]; // ignored - update on load
106 - pthread_rwlock_t rrdset_rwlock; // ignored
107 - size_t counter; // NEEDS TO BE UPDATED - maintained on load
108 - size_t counter_done; // ignored
109 - union { //
110 - time_t last_accessed_time_s; // ignored
111 - time_t last_entry_s; // ignored
112 - }; //
113 - time_t upstream_resync_time; // ignored
114 - void *plugin_name; // ignored
115 - void *module_name; // ignored
116 - void *chart_uuid; // ignored
117 - void *state; // ignored
118 - size_t unused[3]; // ignored
119 - size_t rrddim_page_alignment; // ignored
120 - uint32_t hash; // ignored
121 - uint32_t hash_name; // ignored
122 - usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
123 - struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
124 - struct timeval last_collected_time; // ignored
125 - long long collected_total; // ignored
126 - long long last_collected_total; // ignored
127 - void *rrdfamily; // ignored
128 - void *rrdhost; // ignored
129 - void *next; // ignored
130 - long double green; // ignored
131 - long double red; // ignored
132 - struct avl_tree_lock_v019 rrdvar_root_index; // ignored
133 - void *variables; // ignored
134 - void *alarms; // ignored
135 - unsigned long memsize; // check to reset all - update on load
136 - char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
137 - struct avl_tree_lock_v019 dimensions_index; // ignored
138 - void *dimensions; // ignored
139 -};
140 -
141 -void rrdset_memory_file_update(RRDSET *st) {
142 - if(!st->db.st_on_file) return;
143 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
144 -
145 - st_on_file->current_entry = st->db.current_entry;
146 - st_on_file->counter = st->counter;
147 - st_on_file->usec_since_last_update = st->usec_since_last_update;
148 - st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
149 - st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
150 -}
151 -
152 -const char *rrdset_cache_filename(RRDSET *st) {
153 - if(!st->db.st_on_file) return NULL;
154 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
155 - return st_on_file->cache_filename;
156 -}
157 -
158 -const char *rrdset_cache_dir(RRDSET *st) {
159 - if (st->db.cache_dir)
160 - return st->db.cache_dir;
161 -
162 - char b[FILENAME_MAX + 1];
163 - rrdset_strncpyz_name(b, rrdset_id(st), FILENAME_MAX);
164 -
165 - char n[FILENAME_MAX + 1];
166 - snprintfz(n, FILENAME_MAX, "%s/%s", st->rrdhost->cache_dir, b);
167 -
168 - st->db.cache_dir = strdupz(n);
169 -
170 - if (st->rrdhost->storage_engine_id == STORAGE_ENGINE_MAP ||
171 - st->rrdhost->storage_engine_id == STORAGE_ENGINE_SAVE)
172 - {
173 - int r = mkdir(st->db.cache_dir, 0775);
174 - if(r != 0 && errno != EEXIST)
175 - netdata_log_error("Cannot create directory '%s'", st->db.cache_dir);
176 - }
177 -
178 - return st->db.cache_dir;
179 -}
180 -
181 -void rrdset_memory_file_free(RRDSET *st) {
182 - if(!st->db.st_on_file) return;
183 -
184 - // needed for storage engine MAP, to save the latest state
185 - rrdset_memory_file_update(st);
186 -
187 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
188 - __atomic_sub_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
189 - netdata_munmap(st_on_file, st_on_file->memsize);
190 -
191 - // remove the pointers from the RRDDIM
192 - st->db.st_on_file = NULL;
193 -}
194 -
195 -void rrdset_memory_file_save(RRDSET *st) {
196 - if(!st->db.st_on_file) return;
197 -
198 - rrdset_memory_file_update(st);
199 -
200 - struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
201 - if(st_on_file->storage_engine_id != STORAGE_ENGINE_SAVE) return;
202 -
203 - memory_file_save(st_on_file->cache_filename, st->db.st_on_file, st_on_file->memsize);
204 -}
205 -
206 -static bool rrdset_memory_load_or_create_map_save(RRDSET *st) {
207 - if (st->storage_engine_id != STORAGE_ENGINE_SAVE && st->storage_engine_id != STORAGE_ENGINE_MAP)
208 - return false;
209 -
210 - char fullfilename[FILENAME_MAX + 1];
211 - snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", rrdset_cache_dir(st));
212 -
213 - unsigned long size = sizeof(struct rrdset_map_save_v019);
214 - struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
215 - fullfilename, size, ((st->storage_engine_id == STORAGE_ENGINE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0, false, NULL);
216 -
217 - if(!st_on_file) return false;
218 -
219 - time_t now_s = now_realtime_sec();
220 -
221 - st_on_file->magic[sizeof(RRDSET_MAGIC_V019)] = '\0';
222 - if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
223 - netdata_log_info("Initializing file '%s'.", fullfilename);
224 - memset(st_on_file, 0, size);
225 - }
226 - else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) {
227 - netdata_log_error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));
228 - memset(st_on_file, 0, size);
229 - }
230 - else if(st_on_file->memsize != size || st_on_file->entries != st->db.entries) {
231 - netdata_log_error("File '%s' does not have the desired size. Clearing it.", fullfilename);
232 - memset(st_on_file, 0, size);
233 - }
234 - else if(st_on_file->update_every != st->update_every) {
235 - netdata_log_error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
236 - memset(st_on_file, 0, size);
237 - }
238 - else if((now_s - st_on_file->last_updated.tv_sec) > (long)st->update_every * (long)st->db.entries) {
239 - netdata_log_info("File '%s' is too old. Clearing it.", fullfilename);
240 - memset(st_on_file, 0, size);
241 - }
242 - else if(st_on_file->last_updated.tv_sec > now_s + st->update_every) {
243 - netdata_log_error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now_s));
244 - st_on_file->last_updated.tv_sec = now_s;
245 - }
246 -
247 - if(st_on_file->current_entry >= st_on_file->entries)
248 - st_on_file->current_entry = 0;
249 -
250 - // make sure the database is aligned
251 - bool align_last_updated = false;
252 - if(st_on_file->last_updated.tv_sec) {
253 - st_on_file->update_every = st->update_every;
254 - align_last_updated = true;
255 - }
256 -
257 - // copy the useful values to st
258 - st->db.current_entry = st_on_file->current_entry;
259 - st->counter = st_on_file->counter;
260 - st->usec_since_last_update = st_on_file->usec_since_last_update;
261 - st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
262 - st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
263 -
264 - // link it to st
265 - st->db.st_on_file = st_on_file;
266 -
267 - // clear everything
268 - memset(st_on_file, 0, size);
269 -
270 - // set the values we need
271 - strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019);
272 - strcpy(st_on_file->cache_filename, fullfilename);
273 - strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
274 - st_on_file->memsize = size;
275 - st_on_file->entries = st->db.entries;
276 - st_on_file->update_every = st->update_every;
277 - st_on_file->storage_engine_id = st->storage_engine_id;
278 -
279 - if(align_last_updated)
280 - last_updated_time_align(st);
281 -
282 - // copy the useful values back to st_on_file
283 - rrdset_memory_file_update(st);
284 -
285 - __atomic_add_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
286 - return true;
287 -}
6 +#include "storage_engine.h"
7
8 // ----------------------------------------------------------------------------
9 // RRDSET name index
@@ -368,7 +87,7 @@ struct rrdset_constructor {
87 long priority;
88 int update_every;
89 RRDSET_TYPE chart_type;
371 - STORAGE_ENGINE_ID storage_engine_id;
90 + RRD_MEMORY_MODE memory_mode;
91 long history_entries;
92
93 enum {
@@ -409,9 +128,9 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
128 st->module_name = rrd_string_strdupz(ctr->module);
129 st->priority = ctr->priority;
130
412 - st->db.entries = (ctr->storage_engine_id != STORAGE_ENGINE_DBENGINE) ? align_entries_to_pagesize(ctr->storage_engine_id, ctr->history_entries) : 5;
131 + st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;
132 st->update_every = ctr->update_every;
414 - st->storage_engine_id = ctr->storage_engine_id;
133 + st->rrd_memory_mode = ctr->memory_mode;
134
135 st->chart_type = ctr->chart_type;
136 st->rrdhost = host;
@@ -426,18 +145,20 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
145
146 rw_spinlock_init(&st->alerts.spinlock);
147
429 - if(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP) {
430 - if(!rrdset_memory_load_or_create_map_save(st)) {
431 - netdata_log_info("Failed to use db mode %s for chart '%s', falling back to ram mode.", (st->storage_engine_id == STORAGE_ENGINE_MAP)?"map":"save", rrdset_name(st));
432 - st->storage_engine_id = STORAGE_ENGINE_RAM;
148 + if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
149 + if(!rrdset_memory_load_or_create_map_save(st, st->rrd_memory_mode)) {
150 + netdata_log_info("Failed to use db mode %s for chart '%s', falling back to ram mode.", (st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st));
151 + st->rrd_memory_mode = RRD_MEMORY_MODE_RAM;
152 }
153 }
154
155 // initialize the db tiers
156 {
438 - for (size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
439 - STORAGE_ENGINE_ID storage_engine_id = st->rrdhost->db[tier].id;
440 - st->storage_metrics_groups[tier] = storage_engine_metrics_group_get(storage_engine_id, host->db[tier].instance, &st->chart_uuid);
157 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
158 + STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
159 + if(!eng) continue;
160 +
161 + st->storage_metrics_groups[tier] = storage_engine_metrics_group_get(eng->backend, host->db[tier].instance, &st->chart_uuid);
162 }
163 }
164
@@ -477,11 +198,12 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
198 rrddim_foreach_done(rd);
199 }
200
480 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
481 - STORAGE_ENGINE_ID storage_engine_id = st->rrdhost->db[tier].id;
201 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
202 + STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
203 + if(!eng) continue;
204
205 if(st->storage_metrics_groups[tier]) {
484 - storage_engine_metrics_group_release(storage_engine_id, host->db[tier].instance, st->storage_metrics_groups[tier]);
206 + storage_engine_metrics_group_release(eng->backend, host->db[tier].instance, st->storage_metrics_groups[tier]);
207 st->storage_metrics_groups[tier] = NULL;
208 }
209 }
@@ -727,7 +449,7 @@ static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) {
449 // ----------------------------------------------------------------------------
450 // RRDSET - find charts
451
730 -inline RRDSET *rrdset_find_by_id(RRDHOST *host, const char *id) {
452 +inline RRDSET *rrdset_find(RRDHOST *host, const char *id) {
453 netdata_log_debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host));
454 RRDSET *st = rrdset_index_find(host, id);
455
@@ -737,7 +459,7 @@ inline RRDSET *rrdset_find_by_id(RRDHOST *host, const char *id) {
459 return(st);
460 }
461
740 -inline RRDSET *rrdset_find_by_type(RRDHOST *host, const char *type, const char *id) {
462 +inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id) {
463 netdata_log_debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host));
464
465 char buf[RRD_ID_LENGTH_MAX + 1];
@@ -746,10 +468,10 @@ inline RRDSET *rrdset_find_by_type(RRDHOST *host, const char *type, const char *
468 int len = (int) strlen(buf);
469 strncpyz(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len));
470
749 - return(rrdset_find_by_id(host, buf));
471 + return(rrdset_find(host, buf));
472 }
473
752 -inline RRDSET *rrdset_find_by_name(RRDHOST *host, const char *name) {
474 +inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {
475 netdata_log_debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host));
476 RRDSET *st = rrdset_index_find_name(host, name);
477 return(st);
@@ -875,7 +597,7 @@ time_t rrdset_first_entry_s(RRDSET *st) {
597 }
598
599 time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier) {
878 - if(unlikely(tier > rrdb.storage_tiers))
600 + if(unlikely(tier > storage_tiers))
601 return 0;
602
603 RRDDIM *rd;
@@ -1019,7 +741,10 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
741 }
742 }
743
1022 -static void rrdset_reset(RRDSET *st) {
744 +// ----------------------------------------------------------------------------
745 +// RRDSET - reset a chart
746 +
747 +void rrdset_reset(RRDSET *st) {
748 netdata_log_debug(D_RRD_CALLS, "rrdset_reset() %s", rrdset_name(st));
749
750 st->last_collected_time.tv_sec = 0;
@@ -1037,15 +762,57 @@ static void rrdset_reset(RRDSET *st) {
762 rd->collector.counter = 0;
763
764 if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
1040 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
1041 - if (rd->tiers[tier].db_collection_handle)
1042 - storage_engine_store_flush(st->storage_engine_id, rd->tiers[tier].db_collection_handle);
1043 - }
765 + for(size_t tier = 0; tier < storage_tiers ;tier++)
766 + storage_engine_store_flush(rd->tiers[tier].db_collection_handle);
767 }
768 }
769 rrddim_foreach_done(rd);
770 }
771
772 +// ----------------------------------------------------------------------------
773 +// RRDSET - helpers for rrdset_create()
774 +
775 +inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
776 + if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
777 + if(mode == RRD_MEMORY_MODE_NONE) return 5;
778 +
779 + if(entries < 5) entries = 5;
780 + if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
781 +
782 + if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE || mode == RRD_MEMORY_MODE_RAM) {
783 + long header_size = 0;
784 +
785 + if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE)
786 + header_size = (long)rrddim_memory_file_header_size();
787 +
788 + long page = (long)sysconf(_SC_PAGESIZE);
789 + long size = (long)(header_size + entries * sizeof(storage_number));
790 + if (unlikely(size % page)) {
791 + size -= (size % page);
792 + size += page;
793 +
794 + long n = (long)((size - header_size) / sizeof(storage_number));
795 + return n;
796 + }
797 + }
798 +
799 + return entries;
800 +}
801 +
802 +static inline void last_collected_time_align(RRDSET *st) {
803 + st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
804 +
805 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
806 + st->last_collected_time.tv_usec = 0;
807 + else
808 + st->last_collected_time.tv_usec = 500000;
809 +}
810 +
811 +static inline void last_updated_time_align(RRDSET *st) {
812 + st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
813 + st->last_updated.tv_usec = 0;
814 +}
815 +
816 // ----------------------------------------------------------------------------
817 // RRDSET - free a chart
818
@@ -1068,7 +835,7 @@ void rrdset_delete_files(RRDSET *st) {
835
836 netdata_log_info("Deleting chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
837
1071 - if(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP) {
838 + if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
839 const char *cache_filename = rrdset_cache_filename(st);
840 if(cache_filename) {
841 netdata_log_info("Deleting chart header file '%s'.", cache_filename);
@@ -1093,6 +860,23 @@ void rrdset_delete_files(RRDSET *st) {
860 recursively_delete_dir(st->db.cache_dir, "left-over chart");
861 }
862
863 +void rrdset_delete_obsolete_dimensions(RRDSET *st) {
864 + RRDDIM *rd;
865 +
866 + netdata_log_info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
867 +
868 + rrddim_foreach_read(rd, st) {
869 + if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
870 + const char *cache_filename = rrddim_cache_filename(rd);
871 + if(!cache_filename) continue;
872 + netdata_log_info("Deleting dimension file '%s'.", cache_filename);
873 + if(unlikely(unlink(cache_filename) == -1))
874 + netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
875 + }
876 + }
877 + rrddim_foreach_done(rd);
878 +}
879 +
880 // ----------------------------------------------------------------------------
881 // RRDSET - create a chart
882
@@ -1110,10 +894,10 @@ RRDSET *rrdset_create_custom(
894 , long priority
895 , int update_every
896 , RRDSET_TYPE chart_type
1113 - , STORAGE_ENGINE_ID storage_engine_id
897 + , RRD_MEMORY_MODE memory_mode
898 , long history_entries
899 ) {
1116 - if (host != rrdb.localhost)
900 + if (host != localhost)
901 host->child_last_chart_command = now_realtime_sec();
902
903 if(!type || !type[0])
@@ -1165,7 +949,7 @@ RRDSET *rrdset_create_custom(
949 .priority = priority,
950 .update_every = update_every,
951 .chart_type = chart_type,
1168 - .storage_engine_id = storage_engine_id,
952 + .memory_mode = memory_mode,
953 .history_entries = history_entries,
954 };
955
@@ -1343,15 +1127,14 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) {
1127
1128 static __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];
1129
1346 -static inline time_t tier_next_point_time_s(RRDDIM *rd, size_t tier, time_t now_s) {
1347 - uint32_t tier_grouping = rd->rrdset->rrdhost->db[tier].tier_grouping;
1348 - time_t loop = (time_t)rd->rrdset->update_every * (time_t) tier_grouping;
1130 +static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, time_t now_s) {
1131 + time_t loop = (time_t)rd->rrdset->update_every * (time_t)t->tier_grouping;
1132 return now_s + loop - ((now_s + loop) % loop);
1133 }
1134
1135 void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {
1136 if (unlikely(!t->next_point_end_time_s))
1354 - t->next_point_end_time_s = tier_next_point_time_s(rd, tier, sp.end_time_s);
1137 + t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
1138
1139 if(unlikely(sp.start_time_s >= t->next_point_end_time_s)) {
1140 // flush the virtual point, it is done
@@ -1359,7 +1142,6 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1142 if (likely(!storage_point_is_unset(t->virtual_point))) {
1143
1144 storage_engine_store_metric(
1362 - rd->rrdset->storage_engine_id,
1145 t->db_collection_handle,
1146 t->next_point_end_time_s * USEC_PER_SEC,
1147 t->virtual_point.sum,
@@ -1371,7 +1153,6 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1153 }
1154 else {
1155 storage_engine_store_metric(
1374 - rd->rrdset->storage_engine_id,
1156 t->db_collection_handle,
1157 t->next_point_end_time_s * USEC_PER_SEC,
1158 NAN,
@@ -1383,7 +1164,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1164
1165 rrdset_done_statistics_points_stored_per_tier[tier]++;
1166 t->virtual_point.count = 0; // make the point unset
1386 - t->next_point_end_time_s = tier_next_point_time_s(rd, tier, sp.end_time_s);
1167 + t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
1168 }
1169
1170 // merge the dates into our virtual point
@@ -1443,8 +1224,9 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1224 #endif // NETDATA_LOG_COLLECTION_ERRORS
1225
1226 // store the metric on tier 0
1446 - storage_engine_store_metric(rd->rrdset->storage_engine_id, rd->tiers[0].db_collection_handle, point_end_time_ut,
1447 - n, 0, 0, 1, 0, flags);
1227 + storage_engine_store_metric(rd->tiers[0].db_collection_handle, point_end_time_ut,
1228 + n, 0, 0,
1229 + 1, 0, flags);
1230
1231 rrdset_done_statistics_points_stored_per_tier[0]++;
1232
@@ -1461,7 +1243,7 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1243 .flags = flags
1244 };
1245
1464 - for(size_t tier = 1; tier < rrdb.storage_tiers ;tier++) {
1246 + for(size_t tier = 1; tier < storage_tiers ;tier++) {
1247 if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
1248
1249 struct rrddim_tier *t = &rd->tiers[tier];
@@ -1599,8 +1381,11 @@ static inline size_t rrdset_done_interpolate(
1381 new_value /= (NETDATA_DOUBLE)st->update_every;
1382
1383 if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
1384 +
1385 rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
1603 - rrddim_name(rd) , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut));
1386 + rrddim_name(rd)
1387 + , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
1388 + );
1389
1390 new_value = new_value * (NETDATA_DOUBLE)(st->update_every * USEC_PER_SEC) / (NETDATA_DOUBLE)(next_store_ut - last_stored_ut);
1391 }
@@ -1654,7 +1439,7 @@ static inline size_t rrdset_done_interpolate(
1439 continue;
1440 }
1441
1657 - if(likely(rrddim_check_updated(rd) && rd->collector.counter > 1 && iterations < rrdb.gap_when_lost_iterations_above)) {
1442 + if(likely(rrddim_check_updated(rd) && rd->collector.counter > 1 && iterations < gap_when_lost_iterations_above)) {
1443 uint32_t dim_storage_flags = storage_flags;
1444
1445 if (ml_dimension_is_anomalous(rd, current_time_s, new_value, true)) {
@@ -1798,7 +1583,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1583
1584 // check if we will re-write the entire data set
1585 if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&
1801 - st->storage_engine_id != STORAGE_ENGINE_DBENGINE)) {
1586 + st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
1587 netdata_log_info(
1588 "'%s': too old data (last updated at %"PRId64".%"PRId64", last collected at %"PRId64".%"PRId64"). "
1589 "Resetting it. Will not store the next entry.",
@@ -1858,8 +1643,8 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1643 size_t dim_id;
1644 size_t dimensions = 0;
1645 struct rda_item *rda = rda_base;
1861 - collected_number collected_total = 0;
1862 - collected_number last_collected_total = 0;
1646 + total_number collected_total = 0;
1647 + total_number last_collected_total = 0;
1648 rrddim_foreach_read(rd, st) {
1649 if(rd_dfe.counter >= rda_slots)
1650 break;
@@ -2097,6 +1882,14 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1882 // at this point we have all the calculated values ready
1883 // it is now time to interpolate values on a second boundary
1884
1885 +// #ifdef NETDATA_INTERNAL_CHECKS
1886 +// if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) {
1887 +// // this is collected in the same interpolation point
1888 +// rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
1889 +// netdata_log_info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, rrdset_name(st), st->counter_done, next_store_ut - now_collect_ut);
1890 +// }
1891 +// #endif
1892 +
1893 rrdset_done_interpolate(
1894 &stream_buffer
1895 , st
@@ -2174,7 +1967,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1967 // ALL DONE ABOUT THE DATA UPDATE
1968 // --------------------------------------------------------------------
1969
2177 - if(unlikely(st->storage_engine_id == STORAGE_ENGINE_MAP)) {
1970 + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
1971 // update the memory mapped files with the latest values
1972
1973 rrdset_memory_file_update(st);
@@ -2213,10 +2006,9 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
2006 // switch update every to the storage engine
2007 RRDDIM *rd;
2008 rrddim_foreach_read(rd, st) {
2216 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
2009 + for (size_t tier = 0; tier < storage_tiers; tier++) {
2010 if (rd->tiers[tier].db_collection_handle)
2011 storage_engine_store_change_collection_frequency(
2219 - st->storage_engine_id,
2012 rd->tiers[tier].db_collection_handle,
2013 (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
2014 }
@@ -2226,13 +2018,210 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
2018 return prev_update_every_s;
2019 }
2020
2229 -void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
2230 - if(!st->rrdlabels)
2231 - st->rrdlabels = rrdlabels_create();
2021 +// ----------------------------------------------------------------------------
2022 +// compatibility layer for RRDSET files v019
2023
2233 - if (new_rrdlabels)
2234 - rrdlabels_migrate_to_these(st->rrdlabels, new_rrdlabels);
2024 +#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
2025 +#define RRD_ID_LENGTH_MAX_V019 200
2026
2236 - rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
2237 - rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
2027 +struct avl_element_v019 {
2028 + void *avl_link[2];
2029 + signed char avl_balance;
2030 +};
2031 +struct avl_tree_type_v019 {
2032 + void *root;
2033 + int (*compar)(void *a, void *b);
2034 +};
2035 +struct avl_tree_lock_v019 {
2036 + struct avl_tree_type_v019 avl_tree;
2037 + pthread_rwlock_t rwlock;
2038 +};
2039 +struct rrdset_map_save_v019 {
2040 + struct avl_element_v019 avl; // ignored
2041 + struct avl_element_v019 avlname; // ignored
2042 + char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
2043 + void *name; // ignored
2044 + void *unused_ptr; // ignored
2045 + void *type; // ignored
2046 + void *family; // ignored
2047 + void *title; // ignored
2048 + void *units; // ignored
2049 + void *context; // ignored
2050 + uint32_t hash_context; // ignored
2051 + uint32_t chart_type; // ignored
2052 + int update_every; // check to reset all - update on load
2053 + long entries; // check to reset all - update on load
2054 + long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
2055 + uint32_t flags; // ignored
2056 + void *exporting_flags; // ignored
2057 + int gap_when_lost_iterations_above; // ignored
2058 + long priority; // ignored
2059 + uint32_t rrd_memory_mode; // ignored
2060 + void *cache_dir; // ignored
2061 + char cache_filename[FILENAME_MAX+1]; // ignored - update on load
2062 + pthread_rwlock_t rrdset_rwlock; // ignored
2063 + size_t counter; // NEEDS TO BE UPDATED - maintained on load
2064 + size_t counter_done; // ignored
2065 + union { //
2066 + time_t last_accessed_time_s; // ignored
2067 + time_t last_entry_s; // ignored
2068 + }; //
2069 + time_t upstream_resync_time; // ignored
2070 + void *plugin_name; // ignored
2071 + void *module_name; // ignored
2072 + void *chart_uuid; // ignored
2073 + void *state; // ignored
2074 + size_t unused[3]; // ignored
2075 + size_t rrddim_page_alignment; // ignored
2076 + uint32_t hash; // ignored
2077 + uint32_t hash_name; // ignored
2078 + usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
2079 + struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
2080 + struct timeval last_collected_time; // ignored
2081 + long long collected_total; // ignored
2082 + long long last_collected_total; // ignored
2083 + void *rrdfamily; // ignored
2084 + void *rrdhost; // ignored
2085 + void *next; // ignored
2086 + long double green; // ignored
2087 + long double red; // ignored
2088 + struct avl_tree_lock_v019 rrdvar_root_index; // ignored
2089 + void *variables; // ignored
2090 + void *alarms; // ignored
2091 + unsigned long memsize; // check to reset all - update on load
2092 + char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
2093 + struct avl_tree_lock_v019 dimensions_index; // ignored
2094 + void *dimensions; // ignored
2095 +};
2096 +
2097 +void rrdset_memory_file_update(RRDSET *st) {
2098 + if(!st->db.st_on_file) return;
2099 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2100 +
2101 + st_on_file->current_entry = st->db.current_entry;
2102 + st_on_file->counter = st->counter;
2103 + st_on_file->usec_since_last_update = st->usec_since_last_update;
2104 + st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
2105 + st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
2106 +}
2107 +
2108 +const char *rrdset_cache_filename(RRDSET *st) {
2109 + if(!st->db.st_on_file) return NULL;
2110 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2111 + return st_on_file->cache_filename;
2112 +}
2113 +
2114 +const char *rrdset_cache_dir(RRDSET *st) {
2115 + if(!st->db.cache_dir)
2116 + st->db.cache_dir = rrdhost_cache_dir_for_rrdset_alloc(st->rrdhost, rrdset_id(st));
2117 +
2118 + return st->db.cache_dir;
2119 +}
2120 +
2121 +void rrdset_memory_file_free(RRDSET *st) {
2122 + if(!st->db.st_on_file) return;
2123 +
2124 + // needed for memory mode map, to save the latest state
2125 + rrdset_memory_file_update(st);
2126 +
2127 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2128 + __atomic_sub_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
2129 + netdata_munmap(st_on_file, st_on_file->memsize);
2130 +
2131 + // remove the pointers from the RRDDIM
2132 + st->db.st_on_file = NULL;
2133 +}
2134 +
2135 +void rrdset_memory_file_save(RRDSET *st) {
2136 + if(!st->db.st_on_file) return;
2137 +
2138 + rrdset_memory_file_update(st);
2139 +
2140 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
2141 + if(st_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
2142 +
2143 + memory_file_save(st_on_file->cache_filename, st->db.st_on_file, st_on_file->memsize);
2144 +}
2145 +
2146 +bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mode) {
2147 + if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
2148 + return false;
2149 +
2150 + char fullfilename[FILENAME_MAX + 1];
2151 + snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", rrdset_cache_dir(st));
2152 +
2153 + unsigned long size = sizeof(struct rrdset_map_save_v019);
2154 + struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
2155 + fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0, false, NULL);
2156 +
2157 + if(!st_on_file) return false;
2158 +
2159 + time_t now_s = now_realtime_sec();
2160 +
2161 + st_on_file->magic[sizeof(RRDSET_MAGIC_V019)] = '\0';
2162 + if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
2163 + netdata_log_info("Initializing file '%s'.", fullfilename);
2164 + memset(st_on_file, 0, size);
2165 + }
2166 + else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) {
2167 + netdata_log_error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));
2168 + memset(st_on_file, 0, size);
2169 + }
2170 + else if(st_on_file->memsize != size || st_on_file->entries != st->db.entries) {
2171 + netdata_log_error("File '%s' does not have the desired size. Clearing it.", fullfilename);
2172 + memset(st_on_file, 0, size);
2173 + }
2174 + else if(st_on_file->update_every != st->update_every) {
2175 + netdata_log_error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
2176 + memset(st_on_file, 0, size);
2177 + }
2178 + else if((now_s - st_on_file->last_updated.tv_sec) > (long)st->update_every * (long)st->db.entries) {
2179 + netdata_log_info("File '%s' is too old. Clearing it.", fullfilename);
2180 + memset(st_on_file, 0, size);
2181 + }
2182 + else if(st_on_file->last_updated.tv_sec > now_s + st->update_every) {
2183 + netdata_log_error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now_s));
2184 + st_on_file->last_updated.tv_sec = now_s;
2185 + }
2186 +
2187 + if(st_on_file->current_entry >= st_on_file->entries)
2188 + st_on_file->current_entry = 0;
2189 +
2190 + // make sure the database is aligned
2191 + bool align_last_updated = false;
2192 + if(st_on_file->last_updated.tv_sec) {
2193 + st_on_file->update_every = st->update_every;
2194 + align_last_updated = true;
2195 + }
2196 +
2197 + // copy the useful values to st
2198 + st->db.current_entry = st_on_file->current_entry;
2199 + st->counter = st_on_file->counter;
2200 + st->usec_since_last_update = st_on_file->usec_since_last_update;
2201 + st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
2202 + st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
2203 +
2204 + // link it to st
2205 + st->db.st_on_file = st_on_file;
2206 +
2207 + // clear everything
2208 + memset(st_on_file, 0, size);
2209 +
2210 + // set the values we need
2211 + strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019);
2212 + strcpy(st_on_file->cache_filename, fullfilename);
2213 + strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
2214 + st_on_file->memsize = size;
2215 + st_on_file->entries = st->db.entries;
2216 + st_on_file->update_every = st->update_every;
2217 + st_on_file->rrd_memory_mode = memory_mode;
2218 +
2219 + if(align_last_updated)
2220 + last_updated_time_align(st);
2221 +
2222 + // copy the useful values back to st_on_file
2223 + rrdset_memory_file_update(st);
2224 +
2225 + __atomic_add_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
2226 + return true;
2227 }
database/rrdvar.c
+5
@@ -217,6 +217,11 @@ NETDATA_DOUBLE rrdvar2number(const RRDVAR_ACQUIRED *rva) {
217 return (NETDATA_DOUBLE)*n;
218 }
219
220 + case RRDVAR_TYPE_TOTAL: {
221 + total_number *n = (total_number *)rv->value;
222 + return (NETDATA_DOUBLE)*n;
223 + }
224 +
225 case RRDVAR_TYPE_INT: {
226 int *n = (int *)rv->value;
227 return *n;
database/rrdvar.h
+2 -1
@@ -9,7 +9,8 @@ typedef enum rrdvar_type {
9 RRDVAR_TYPE_CALCULATED = 1,
10 RRDVAR_TYPE_TIME_T = 2,
11 RRDVAR_TYPE_COLLECTED = 3,
12 - RRDVAR_TYPE_INT = 4
12 + RRDVAR_TYPE_TOTAL = 4,
13 + RRDVAR_TYPE_INT = 5
14
15 // this is 8 bit
16 // to increase it you have to set change the bitfield in
database/sqlite/sqlite_aclk.c
+8 -8
@@ -104,7 +104,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
104
105 sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info);
106
107 - RRDHOST *host = rrdhost_get_or_create(
107 + RRDHOST *host = rrdhost_find_or_create(
108 (const char *) argv[IDX_HOSTNAME]
109 , (const char *) argv[IDX_REGISTRY]
110 , guid
@@ -117,7 +117,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
117 , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown")
118 , argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1
119 , argv[IDX_ENTRIES] ? str2i(argv[IDX_ENTRIES]) : 0
120 - , default_storage_engine_id
120 + , default_rrd_memory_mode
121 , 0 // health
122 , 0 // rrdpush enabled
123 , NULL //destination
@@ -181,7 +181,7 @@ static int is_host_available(uuid_t *host_id)
181 int rc;
182
183 if (unlikely(!db_meta)) {
184 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
184 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
185 error_report("Database has not been initialized");
186 return 1;
187 }
@@ -319,7 +319,7 @@ static int aclk_config_parameters(void *data __maybe_unused, int argc __maybe_un
319 uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str);
320
321 RRDHOST *host = rrdhost_find_by_guid(uuid_str);
322 - if (host == rrdb.localhost)
322 + if (host == localhost)
323 return 0;
324
325 sql_create_aclk_table(host, (uuid_t *) argv[0], (uuid_t *) argv[1]);
@@ -424,7 +424,7 @@ static void aclk_synchronization(void *arg __maybe_unused)
424 // NODE STATE
425 case ACLK_DATABASE_NODE_STATE:;
426 RRDHOST *host = cmd.param[0];
427 - int live = (host == rrdb.localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
427 + int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
428 struct aclk_sync_host_config *ahc = host->aclk_sync_host_config;
429 if (unlikely(!ahc))
430 sql_create_aclk_table(host, &host->host_uuid, host->node_id);
@@ -520,7 +520,7 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may
520 strcpy(wc->uuid_str, uuid_str);
521 wc->alert_updates = 0;
522 time_t now = now_realtime_sec();
523 - wc->node_info_send_time = (host == rrdb.localhost || NULL == rrdb.localhost) ? now - 25 : now;
523 + wc->node_info_send_time = (host == localhost || NULL == localhost) ? now - 25 : now;
524 #endif
525 }
526
@@ -536,7 +536,7 @@ void sql_aclk_sync_init(void)
536 int rc;
537
538 if (unlikely(!db_meta)) {
539 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE) {
539 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {
540 return;
541 }
542 error_report("Database has not been initialized");
@@ -558,7 +558,7 @@ void sql_aclk_sync_init(void)
558
559 #ifdef ENABLE_ACLK
560 if (!number_of_children)
561 - aclk_queue_node_info(rrdb.localhost, true);
561 + aclk_queue_node_info(localhost, true);
562
563 rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_INSTANCES,aclk_config_parameters, NULL,&err_msg);
564
database/sqlite/sqlite_aclk.h
+1 -1
@@ -38,7 +38,7 @@ static inline int uuid_parse_fix(char *in, uuid_t uuid)
38
39 static inline int claimed()
40 {
41 - return rrdb.localhost->aclk_state.claimed_id != NULL;
41 + return localhost->aclk_state.claimed_id != NULL;
42 }
43
44 #define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \
database/sqlite/sqlite_aclk_alert.c
+8 -8
@@ -430,7 +430,7 @@ void aclk_push_alert_events_for_all_hosts(void)
430 {
431 RRDHOST *host;
432
433 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
433 + dfe_start_reentrant(rrdhost_root_index, host) {
434 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))
435 continue;
436
@@ -504,7 +504,7 @@ void aclk_send_alarm_configuration(char *config_hash)
504 if (unlikely(!config_hash))
505 return;
506
507 - struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) rrdb.localhost->aclk_sync_host_config;
507 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) localhost->aclk_sync_host_config;
508
509 if (unlikely(!wc))
510 return;
@@ -529,7 +529,7 @@ int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash
529 sqlite3_stmt *res = NULL;
530
531 struct aclk_sync_host_config *wc = NULL;
532 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
532 + RRDHOST *host = find_host_by_node_id(node_id);
533
534 if (unlikely(!host)) {
535 freez(config_hash);
@@ -655,7 +655,7 @@ void aclk_start_alert_streaming(char *node_id, bool resets)
655 if (uuid_parse(node_id, node_uuid))
656 return;
657
658 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
658 + RRDHOST *host = find_host_by_node_id(node_id);
659
660 if (unlikely(!host))
661 return;
@@ -689,7 +689,7 @@ void aclk_start_alert_streaming(char *node_id, bool resets)
689 void sql_process_queue_removed_alerts_to_aclk(char *node_id)
690 {
691 struct aclk_sync_host_config *wc;
692 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
692 + RRDHOST *host = find_host_by_node_id(node_id);
693 freez(node_id);
694
695 if (unlikely(!host || !(wc = host->aclk_sync_host_config)))
@@ -750,7 +750,7 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unus
750 if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
751 return;
752
753 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
753 + RRDHOST *host = find_host_by_node_id(node_id);
754 if (unlikely(!host)) {
755 netdata_log_access("ACLK STA [%s (N/A)]: ACLK node id does not exist", node_id);
756 return;
@@ -854,7 +854,7 @@ static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, uint32_t mark)
854 void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)
855 {
856 #ifdef ENABLE_ACLK
857 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
857 + RRDHOST *host = find_host_by_node_id(node_id);
858
859 if (unlikely(!host)) {
860 netdata_log_access("AC [%s (N/A)]: Node id not found", node_id);
@@ -1039,7 +1039,7 @@ void aclk_send_alarm_checkpoint(char *node_id, char *claim_id __maybe_unused)
1039 return;
1040
1041 struct aclk_sync_host_config *wc = NULL;
1042 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
1042 + RRDHOST *host = find_host_by_node_id(node_id);
1043
1044 if (unlikely(!host))
1045 return;
database/sqlite/sqlite_aclk_node.c
+6 -6
@@ -29,7 +29,7 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) {
29 static void build_node_collectors(char *node_id __maybe_unused)
30 {
31
32 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
32 + RRDHOST *host = find_host_by_node_id(node_id);
33
34 if (unlikely(!host))
35 return;
@@ -59,7 +59,7 @@ static void build_node_info(char *node_id __maybe_unused)
59 {
60 struct update_node_info node_info;
61
62 - RRDHOST *host = rrdhost_find_by_node_id(node_id);
62 + RRDHOST *host = find_host_by_node_id(node_id);
63
64 if (unlikely((!host))) {
65 freez(node_id);
@@ -77,7 +77,7 @@ static void build_node_info(char *node_id __maybe_unused)
77 node_info.node_id = wc->node_id;
78 node_info.claim_id = get_agent_claimid();
79 node_info.machine_guid = host->machine_guid;
80 - node_info.child = (wc->host != rrdb.localhost);
80 + node_info.child = (wc->host != localhost);
81 node_info.ml_info.ml_capable = ml_capable();
82 node_info.ml_info.ml_enabled = ml_enabled(wc->host);
83
@@ -86,7 +86,7 @@ static void build_node_info(char *node_id __maybe_unused)
86 now_realtime_timeval(&node_info.updated_at);
87
88 char *host_version = NULL;
89 - if (host != rrdb.localhost) {
89 + if (host != localhost) {
90 netdata_mutex_lock(&host->receiver_lock);
91 host_version = strdupz(host->receiver && host->receiver->program_version ? host->receiver->program_version : rrdhost_program_version(host));
92 netdata_mutex_unlock(&host->receiver_lock);
@@ -124,7 +124,7 @@ static void build_node_info(char *node_id __maybe_unused)
124 node_info.data.host_labels_ptr = host->rrdlabels;
125
126 aclk_update_node_info(&node_info);
127 - netdata_log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), host->machine_guid, wc->host == rrdb.localhost ? "parent" : "child");
127 + netdata_log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), host->machine_guid, wc->host == localhost ? "parent" : "child");
128
129 rrd_unlock();
130 freez(node_info.claim_id);
@@ -145,7 +145,7 @@ void aclk_check_node_info_and_collectors(void)
145 return;
146
147 size_t pending = 0;
148 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
148 + dfe_start_reentrant(rrdhost_root_index, host) {
149
150 struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
151 if (unlikely(!wc))
database/sqlite/sqlite_functions.c
+8 -8
@@ -630,7 +630,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id)
630 rrd_unlock();
631
632 if (unlikely(!db_meta)) {
633 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
633 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
634 error_report("Database has not been initialized");
635 return 1;
636 }
@@ -673,7 +673,7 @@ int get_host_id(uuid_t *node_id, uuid_t *host_id)
673 int rc;
674
675 if (unlikely(!db_meta)) {
676 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
676 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
677 error_report("Database has not been initialized");
678 return 1;
679 }
@@ -711,7 +711,7 @@ int get_node_id(uuid_t *host_id, uuid_t *node_id)
711 int rc;
712
713 if (unlikely(!db_meta)) {
714 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
714 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
715 error_report("Database has not been initialized");
716 return 1;
717 }
@@ -748,7 +748,7 @@ void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)
748 int rc;
749
750 if (unlikely(!db_meta)) {
751 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
751 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
752 error_report("Database has not been initialized");
753 return;
754 }
@@ -794,7 +794,7 @@ struct node_instance_list *get_node_list(void)
794 int rc;
795
796 if (unlikely(!db_meta)) {
797 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
797 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
798 error_report("Database has not been initialized");
799 return NULL;
800 }
@@ -832,10 +832,10 @@ struct node_instance_list *get_node_list(void)
832 }
833 uuid_copy(node_list[row].host_id, *host_id);
834 node_list[row].queryable = 1;
835 - node_list[row].live = (host && (host == rrdb.localhost || host->receiver
835 + node_list[row].live = (host && (host == localhost || host->receiver
836 || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;
837 node_list[row].hops = (host && host->system_info) ? host->system_info->hops :
838 - uuid_memcmp(host_id, &rrdb.localhost->host_uuid) ? 1 : 0;
838 + uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0;
839 node_list[row].hostname =
840 sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;
841 }
@@ -860,7 +860,7 @@ void sql_load_node_id(RRDHOST *host)
860 int rc;
861
862 if (unlikely(!db_meta)) {
863 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
863 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
864 error_report("Database has not been initialized");
865 return;
866 }
database/sqlite/sqlite_functions.h
+1 -1
@@ -29,7 +29,7 @@ typedef enum db_check_action_type {
29
30 #define CHECK_SQLITE_CONNECTION(db_meta) \
31 if (unlikely(!db_meta)) { \
32 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE) { \
32 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { \
33 return 1; \
34 } \
35 error_report("Database has not been initialized"); \
database/sqlite/sqlite_health.c
+7 -7
@@ -16,7 +16,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
16 int rc;
17
18 if (unlikely(!db_meta)) {
19 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
19 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
20 error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
21 return;
22 }
@@ -98,7 +98,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
98 uint64_t health_log_id = 0;
99
100 if (unlikely(!db_meta)) {
101 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
101 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
102 error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
103 return;
104 }
@@ -368,7 +368,7 @@ void sql_health_alarm_log_count(RRDHOST *host) {
368 int rc;
369
370 if (unlikely(!db_meta)) {
371 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
371 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
372 error_report("Database has not been initialized");
373 return;
374 }
@@ -407,7 +407,7 @@ void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host) {
407 char command[MAX_HEALTH_SQL_SIZE + 1];
408
409 if (unlikely(!db_meta)) {
410 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
410 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
411 error_report("Database has not been initialized");
412 return;
413 }
@@ -468,7 +468,7 @@ void sql_health_alarm_log_cleanup_claimed(RRDHOST *host) {
468 char command[MAX_HEALTH_SQL_SIZE + 1];
469
470 if (unlikely(!db_meta)) {
471 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
471 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
472 error_report("Database has not been initialized");
473 return;
474 }
@@ -787,7 +787,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
787 host->health.health_log_entries_written = 0;
788
789 if (unlikely(!db_meta)) {
790 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
790 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
791 error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
792 return;
793 }
@@ -1015,7 +1015,7 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
1015 int rc, param = 0;
1016
1017 if (unlikely(!db_meta)) {
1018 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
1018 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1019 return 0;
1020 error_report("Database has not been initialized");
1021 return 1;
database/sqlite/sqlite_metadata.c
+17 -17
@@ -215,7 +215,7 @@ static int store_claim_id(uuid_t *host_id, uuid_t *claim_id)
215 int rc;
216
217 if (unlikely(!db_meta)) {
218 - if (default_storage_engine_id == STORAGE_ENGINE_DBENGINE)
218 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
219 error_report("Database has not been initialized");
220 return 1;
221 }
@@ -287,7 +287,7 @@ static int store_host_metadata(RRDHOST *host)
287 int rc, param = 0;
288
289 if (unlikely(!db_meta)) {
290 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
290 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
291 return 0;
292 error_report("Database has not been initialized");
293 return 1;
@@ -313,7 +313,7 @@ static int store_host_metadata(RRDHOST *host)
313 if (unlikely(rc != SQLITE_OK))
314 goto bind_fail;
315
316 - rc = sqlite3_bind_int(res, ++param, host->update_every);
316 + rc = sqlite3_bind_int(res, ++param, host->rrd_update_every);
317 if (unlikely(rc != SQLITE_OK))
318 goto bind_fail;
319
@@ -333,7 +333,7 @@ static int store_host_metadata(RRDHOST *host)
333 if (unlikely(rc != SQLITE_OK))
334 goto bind_fail;
335
336 - rc = sqlite3_bind_int(res, ++param, host->storage_engine_id);
336 + rc = sqlite3_bind_int(res, ++param, host->rrd_memory_mode);
337 if (unlikely(rc != SQLITE_OK))
338 goto bind_fail;
339
@@ -384,7 +384,7 @@ static int add_host_sysinfo_key_value(const char *name, const char *value, uuid_
384 int rc, param = 0;
385
386 if (unlikely(!db_meta)) {
387 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
387 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
388 return 0;
389 error_report("Database has not been initialized");
390 return 0;
@@ -475,7 +475,7 @@ static int store_chart_metadata(RRDSET *st)
475 int rc, param = 0, store_rc = 0;
476
477 if (unlikely(!db_meta)) {
478 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
478 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
479 return 0;
480 error_report("Database has not been initialized");
481 return 1;
@@ -549,7 +549,7 @@ static int store_chart_metadata(RRDSET *st)
549 if (unlikely(rc != SQLITE_OK))
550 goto bind_fail;
551
552 - rc = sqlite3_bind_int(res, ++param, st->storage_engine_id);
552 + rc = sqlite3_bind_int(res, ++param, st->rrd_memory_mode);
553 if (unlikely(rc != SQLITE_OK))
554 goto bind_fail;
555
@@ -584,7 +584,7 @@ static int store_dimension_metadata(RRDDIM *rd)
584 int rc, param = 0;
585
586 if (unlikely(!db_meta)) {
587 - if (default_storage_engine_id != STORAGE_ENGINE_DBENGINE)
587 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
588 return 0;
589 error_report("Database has not been initialized");
590 return 1;
@@ -653,13 +653,13 @@ bind_fail:
653 static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused)
654 {
655 #ifdef ENABLE_DBENGINE
656 - if (rrdb.dbengine_enabled) {
656 + if(dbengine_enabled) {
657 bool no_retention = true;
658 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
659 - if (!rrdb.multidb_ctx[tier])
658 + for (size_t tier = 0; tier < storage_tiers; tier++) {
659 + if (!multidb_ctx[tier])
660 continue;
661 time_t first_time_t = 0, last_time_t = 0;
662 - if (rrdeng_metric_retention_by_uuid((void *) rrdb.multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) {
662 + if (rrdeng_metric_retention_by_uuid((void *) multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) {
663 if (first_time_t > 0) {
664 no_retention = false;
665 break;
@@ -728,7 +728,7 @@ skip_run:
728 static void cleanup_health_log(void)
729 {
730 RRDHOST *host;
731 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
731 + dfe_start_reentrant(rrdhost_root_index, host) {
732 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
733 continue;
734 sql_health_alarm_log_cleanup(host);
@@ -990,7 +990,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)
990 struct host_context_load_thread *hclt = callocz(max_threads, sizeof(*hclt));
991
992 size_t thread_index;
993 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
993 + dfe_start_reentrant(rrdhost_root_index, host) {
994 if (rrdhost_flag_check(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS) ||
995 !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))
996 continue;
@@ -1141,7 +1141,7 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
1141 if (!data->max_count)
1142 transaction_started = !db_execute(db_meta, "BEGIN TRANSACTION;");
1143
1144 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1144 + dfe_start_reentrant(rrdhost_root_index, host) {
1145 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE))
1146 continue;
1147
@@ -1335,7 +1335,7 @@ static void metadata_event_loop(void *arg)
1335 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SCANNING_HOSTS)))
1336 break;
1337
1338 - if (rrdb.unittest_running)
1338 + if (unittest_running)
1339 break;
1340
1341 data = mallocz(sizeof(*data));
@@ -1363,7 +1363,7 @@ static void metadata_event_loop(void *arg)
1363 }
1364 break;
1365 case METADATA_LOAD_HOST_CONTEXT:;
1366 - if (rrdb.unittest_running)
1366 + if (unittest_running)
1367 break;
1368
1369 data = callocz(1,sizeof(*data));
database/storage_engine.c
+101 -42
@@ -1,59 +1,118 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "storage_engine.h"
4 -
4 #include "ram/rrddim_mem.h"
5 #ifdef ENABLE_DBENGINE
6 #include "engine/rrdengineapi.h"
7 #endif
8
9 +static STORAGE_ENGINE engines[] = {
10 + {
11 + .id = RRD_MEMORY_MODE_NONE,
12 + .name = RRD_MEMORY_MODE_NONE_NAME,
13 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
14 + .api = {
15 + .metric_get = rrddim_metric_get,
16 + .metric_get_or_create = rrddim_metric_get_or_create,
17 + .metric_dup = rrddim_metric_dup,
18 + .metric_release = rrddim_metric_release,
19 + .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
20 + }
21 + },
22 + {
23 + .id = RRD_MEMORY_MODE_RAM,
24 + .name = RRD_MEMORY_MODE_RAM_NAME,
25 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
26 + .api = {
27 + .metric_get = rrddim_metric_get,
28 + .metric_get_or_create = rrddim_metric_get_or_create,
29 + .metric_dup = rrddim_metric_dup,
30 + .metric_release = rrddim_metric_release,
31 + .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
32 + }
33 + },
34 + {
35 + .id = RRD_MEMORY_MODE_MAP,
36 + .name = RRD_MEMORY_MODE_MAP_NAME,
37 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
38 + .api = {
39 + .metric_get = rrddim_metric_get,
40 + .metric_get_or_create = rrddim_metric_get_or_create,
41 + .metric_dup = rrddim_metric_dup,
42 + .metric_release = rrddim_metric_release,
43 + .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
44 + }
45 + },
46 + {
47 + .id = RRD_MEMORY_MODE_SAVE,
48 + .name = RRD_MEMORY_MODE_SAVE_NAME,
49 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
50 + .api = {
51 + .metric_get = rrddim_metric_get,
52 + .metric_get_or_create = rrddim_metric_get_or_create,
53 + .metric_dup = rrddim_metric_dup,
54 + .metric_release = rrddim_metric_release,
55 + .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
56 + }
57 + },
58 + {
59 + .id = RRD_MEMORY_MODE_ALLOC,
60 + .name = RRD_MEMORY_MODE_ALLOC_NAME,
61 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
62 + .api = {
63 + .metric_get = rrddim_metric_get,
64 + .metric_get_or_create = rrddim_metric_get_or_create,
65 + .metric_dup = rrddim_metric_dup,
66 + .metric_release = rrddim_metric_release,
67 + .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
68 + }
69 + },
70 #ifdef ENABLE_DBENGINE
11 -STORAGE_ENGINE_ID default_storage_engine_id = STORAGE_ENGINE_DBENGINE;
12 -#else
13 -STORAGE_ENGINE_ID default_storage_engine_id = STORAGE_ENGINE_SAVE;
71 + {
72 + .id = RRD_MEMORY_MODE_DBENGINE,
73 + .name = RRD_MEMORY_MODE_DBENGINE_NAME,
74 + .backend = STORAGE_ENGINE_BACKEND_DBENGINE,
75 + .api = {
76 + .metric_get = rrdeng_metric_get,
77 + .metric_get_or_create = rrdeng_metric_get_or_create,
78 + .metric_dup = rrdeng_metric_dup,
79 + .metric_release = rrdeng_metric_release,
80 + .metric_retention_by_uuid = rrdeng_metric_retention_by_uuid,
81 + }
82 + },
83 #endif
84 + { .id = RRD_MEMORY_MODE_NONE, .name = NULL }
85 +};
86
16 -const char *storage_engine_name(STORAGE_ENGINE_ID id) {
17 - switch(id) {
18 - case STORAGE_ENGINE_RAM:
19 - return STORAGE_ENGINE_RAM_NAME;
20 -
21 - case STORAGE_ENGINE_MAP:
22 - return STORAGE_ENGINE_MAP_NAME;
23 -
24 - case STORAGE_ENGINE_NONE:
25 - return STORAGE_ENGINE_NONE_NAME;
26 -
27 - case STORAGE_ENGINE_SAVE:
28 - return STORAGE_ENGINE_SAVE_NAME;
29 -
30 - case STORAGE_ENGINE_ALLOC:
31 - return STORAGE_ENGINE_ALLOC_NAME;
32 -
33 - case STORAGE_ENGINE_DBENGINE:
34 - return STORAGE_ENGINE_DBENGINE_NAME;
35 -
36 - default:
37 - __builtin_unreachable();
87 +STORAGE_ENGINE* storage_engine_find(const char* name)
88 +{
89 + for (STORAGE_ENGINE* it = engines; it->name; it++) {
90 + if (strcmp(it->name, name) == 0)
91 + return it;
92 }
93 + return NULL;
94 }
95
41 -bool storage_engine_id(const char *name, STORAGE_ENGINE_ID *id) {
42 - if (!strcmp(name, STORAGE_ENGINE_RAM_NAME)) {
43 - *id = STORAGE_ENGINE_RAM;
44 - } else if (!strcmp(name, STORAGE_ENGINE_MAP_NAME)) {
45 - *id = STORAGE_ENGINE_MAP;
46 - } else if (!strcmp(name, STORAGE_ENGINE_NONE_NAME)) {
47 - *id = STORAGE_ENGINE_NONE;
48 - } else if (!strcmp(name, STORAGE_ENGINE_SAVE_NAME)) {
49 - *id = STORAGE_ENGINE_SAVE;
50 - } else if (!strcmp(name, STORAGE_ENGINE_ALLOC_NAME)) {
51 - *id = STORAGE_ENGINE_ALLOC;
52 - } else if (!strcmp(name, STORAGE_ENGINE_DBENGINE_NAME)) {
53 - *id = STORAGE_ENGINE_DBENGINE;
54 - } else {
55 - return false;
96 +STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode)
97 +{
98 + for (STORAGE_ENGINE* it = engines; it->name; it++) {
99 + if (it->id == mmode)
100 + return it;
101 }
102 + return NULL;
103 +}
104 +
105 +STORAGE_ENGINE* storage_engine_foreach_init()
106 +{
107 + // Assuming at least one engine exists
108 + return &engines[0];
109 +}
110 +
111 +STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it)
112 +{
113 + if (!it || !it->name)
114 + return NULL;
115
58 - return true;
116 + it++;
117 + return it->name ? it : NULL;
118 }
database/storage_engine.h
+6 -445
@@ -1,451 +1,12 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#ifndef NETDATA_STORAGE_ENGINE_H
4 -#define NETDATA_STORAGE_ENGINE_H
3 +#ifndef NETDATA_STORAGEENGINEAPI_H
4 +#define NETDATA_STORAGEENGINEAPI_H
5
6 -#include "ram/rrddim_mem.h"
7 -#include "engine/rrddim_eng.h"
6 +#include "rrd.h"
7
9 -extern STORAGE_ENGINE_ID default_storage_engine_id;
8 +// Iterator over existing engines
9 +STORAGE_ENGINE* storage_engine_foreach_init();
10 +STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);
11
11 -static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance, uuid_t *uuid)
12 -{
13 - switch (id) {
14 - case STORAGE_ENGINE_NONE:
15 - case STORAGE_ENGINE_RAM:
16 - case STORAGE_ENGINE_MAP:
17 - case STORAGE_ENGINE_SAVE:
18 - case STORAGE_ENGINE_ALLOC:
19 - return rrddim_metrics_group_get(instance, uuid);
20 -#ifdef ENABLE_DBENGINE
21 - case STORAGE_ENGINE_DBENGINE:
22 - return rrdeng_metrics_group_get(instance, uuid);
12 #endif
24 - default:
25 - __builtin_unreachable();
26 - }
27 -}
28 -
29 -static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance, STORAGE_METRICS_GROUP *smg)
30 -{
31 - switch (id) {
32 - case STORAGE_ENGINE_NONE:
33 - case STORAGE_ENGINE_RAM:
34 - case STORAGE_ENGINE_MAP:
35 - case STORAGE_ENGINE_SAVE:
36 - case STORAGE_ENGINE_ALLOC:
37 - rrddim_metrics_group_release(instance, smg);
38 - break;
39 -#ifdef ENABLE_DBENGINE
40 - case STORAGE_ENGINE_DBENGINE:
41 - rrdeng_metrics_group_release(instance, smg);
42 - break;
43 -#endif
44 - default:
45 - __builtin_unreachable();
46 - }
47 -}
48 -
49 -static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg)
50 -{
51 - switch (id) {
52 - case STORAGE_ENGINE_NONE:
53 - case STORAGE_ENGINE_RAM:
54 - case STORAGE_ENGINE_MAP:
55 - case STORAGE_ENGINE_SAVE:
56 - case STORAGE_ENGINE_ALLOC:
57 - return rrddim_collect_init(metric_handle, update_every, smg);
58 -#ifdef ENABLE_DBENGINE
59 - case STORAGE_ENGINE_DBENGINE:
60 - return rrdeng_store_metric_init(metric_handle, update_every, smg);
61 -#endif
62 - default:
63 - __builtin_unreachable();
64 - }
65 -}
66 -
67 -static inline void storage_engine_store_metric(
68 - STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
69 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
70 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags)
71 -{
72 - switch (id) {
73 - case STORAGE_ENGINE_NONE:
74 - case STORAGE_ENGINE_RAM:
75 - case STORAGE_ENGINE_MAP:
76 - case STORAGE_ENGINE_SAVE:
77 - case STORAGE_ENGINE_ALLOC:
78 - return rrddim_collect_store_metric(collection_handle, point_in_time_ut,
79 - n, min_value, max_value,
80 - count, anomaly_count, flags);
81 -#ifdef ENABLE_DBENGINE
82 - case STORAGE_ENGINE_DBENGINE:
83 - return rrdeng_store_metric_next(collection_handle, point_in_time_ut,
84 - n, min_value, max_value,
85 - count, anomaly_count, flags);
86 -#endif
87 - default:
88 - __builtin_unreachable();
89 - }
90 -}
91 -
92 -static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
93 -{
94 - switch (id) {
95 - case STORAGE_ENGINE_NONE:
96 - case STORAGE_ENGINE_RAM:
97 - case STORAGE_ENGINE_MAP:
98 - case STORAGE_ENGINE_SAVE:
99 - case STORAGE_ENGINE_ALLOC:
100 - return rrddim_disk_space_max(db_instance);
101 -#ifdef ENABLE_DBENGINE
102 - case STORAGE_ENGINE_DBENGINE:
103 - return rrdeng_disk_space_max(db_instance);
104 -#endif
105 - default:
106 - __builtin_unreachable();
107 - }
108 -}
109 -
110 -static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
111 -{
112 - switch (id) {
113 - case STORAGE_ENGINE_NONE:
114 - case STORAGE_ENGINE_RAM:
115 - case STORAGE_ENGINE_MAP:
116 - case STORAGE_ENGINE_SAVE:
117 - case STORAGE_ENGINE_ALLOC:
118 - return rrddim_disk_space_used(db_instance);
119 -#ifdef ENABLE_DBENGINE
120 - case STORAGE_ENGINE_DBENGINE:
121 - return rrdeng_disk_space_max(db_instance);
122 -#endif
123 - default:
124 - __builtin_unreachable();
125 - }
126 -}
127 -
128 -static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
129 -{
130 - switch (id) {
131 - case STORAGE_ENGINE_NONE:
132 - case STORAGE_ENGINE_RAM:
133 - case STORAGE_ENGINE_MAP:
134 - case STORAGE_ENGINE_SAVE:
135 - case STORAGE_ENGINE_ALLOC:
136 - return rrddim_global_first_time_s(db_instance);
137 -#ifdef ENABLE_DBENGINE
138 - case STORAGE_ENGINE_DBENGINE:
139 - return rrdeng_global_first_time_s(db_instance);
140 -#endif
141 - default:
142 - __builtin_unreachable();
143 - }
144 -}
145 -
146 -static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance)
147 -{
148 - switch (id) {
149 - case STORAGE_ENGINE_NONE:
150 - case STORAGE_ENGINE_RAM:
151 - case STORAGE_ENGINE_MAP:
152 - case STORAGE_ENGINE_SAVE:
153 - case STORAGE_ENGINE_ALLOC:
154 - return rrddim_currently_collected_metrics(db_instance);
155 -#ifdef ENABLE_DBENGINE
156 - case STORAGE_ENGINE_DBENGINE:
157 - return rrdeng_currently_collected_metrics(db_instance);
158 -#endif
159 - default:
160 - __builtin_unreachable();
161 - }
162 -}
163 -
164 -static inline void storage_engine_store_flush(STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle)
165 -{
166 - switch (id) {
167 - case STORAGE_ENGINE_NONE:
168 - case STORAGE_ENGINE_RAM:
169 - case STORAGE_ENGINE_MAP:
170 - case STORAGE_ENGINE_SAVE:
171 - case STORAGE_ENGINE_ALLOC:
172 - rrddim_store_metric_flush(collection_handle);
173 - return;
174 -#ifdef ENABLE_DBENGINE
175 - case STORAGE_ENGINE_DBENGINE:
176 - rrdeng_store_metric_flush_current_page(collection_handle);
177 - return;
178 -#endif
179 - default:
180 - __builtin_unreachable();
181 - }
182 -}
183 -
184 -// a finalization function to run after collection is over
185 -// returns 1 if it's safe to delete the dimension
186 -static inline int storage_engine_store_finalize(STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle)
187 -{
188 - switch (id) {
189 - case STORAGE_ENGINE_NONE:
190 - case STORAGE_ENGINE_RAM:
191 - case STORAGE_ENGINE_MAP:
192 - case STORAGE_ENGINE_SAVE:
193 - case STORAGE_ENGINE_ALLOC:
194 - return rrddim_collect_finalize(collection_handle);
195 -#ifdef ENABLE_DBENGINE
196 - case STORAGE_ENGINE_DBENGINE:
197 - return rrdeng_store_metric_finalize(collection_handle);
198 -#endif
199 - default:
200 - __builtin_unreachable();
201 - }
202 -}
203 -
204 -static inline void storage_engine_store_change_collection_frequency(STORAGE_ENGINE_ID id, STORAGE_COLLECT_HANDLE *collection_handle, int update_every)
205 -{
206 - switch (id) {
207 - case STORAGE_ENGINE_NONE:
208 - case STORAGE_ENGINE_RAM:
209 - case STORAGE_ENGINE_MAP:
210 - case STORAGE_ENGINE_SAVE:
211 - case STORAGE_ENGINE_ALLOC:
212 - rrddim_store_metric_change_collection_frequency(collection_handle, update_every);
213 - return;
214 -#ifdef ENABLE_DBENGINE
215 - case STORAGE_ENGINE_DBENGINE:
216 - rrdeng_store_metric_change_collection_frequency(collection_handle, update_every);
217 - return;
218 -#endif
219 - default:
220 - __builtin_unreachable();
221 - }
222 -}
223 -
224 -static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
225 -{
226 - switch (id) {
227 - case STORAGE_ENGINE_NONE:
228 - case STORAGE_ENGINE_RAM:
229 - case STORAGE_ENGINE_MAP:
230 - case STORAGE_ENGINE_SAVE:
231 - case STORAGE_ENGINE_ALLOC:
232 - return rrddim_query_oldest_time_s(db_metric_handle);
233 -#ifdef ENABLE_DBENGINE
234 - case STORAGE_ENGINE_DBENGINE:
235 - return rrdeng_metric_oldest_time(db_metric_handle);
236 -#endif
237 - default:
238 - __builtin_unreachable();
239 - }
240 -}
241 -
242 -static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
243 -{
244 - switch (id) {
245 - case STORAGE_ENGINE_NONE:
246 - case STORAGE_ENGINE_RAM:
247 - case STORAGE_ENGINE_MAP:
248 - case STORAGE_ENGINE_SAVE:
249 - case STORAGE_ENGINE_ALLOC:
250 - return rrddim_query_latest_time_s(db_metric_handle);
251 -#ifdef ENABLE_DBENGINE
252 - case STORAGE_ENGINE_DBENGINE:
253 - return rrdeng_metric_latest_time(db_metric_handle);
254 -#endif
255 - default:
256 - __builtin_unreachable();
257 - }
258 -}
259 -
260 -static inline void storage_engine_query_init(
261 - STORAGE_ENGINE_ID id,
262 - STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
263 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority)
264 -{
265 - handle->id = id;
266 -
267 - switch (id) {
268 - case STORAGE_ENGINE_NONE:
269 - case STORAGE_ENGINE_RAM:
270 - case STORAGE_ENGINE_MAP:
271 - case STORAGE_ENGINE_SAVE:
272 - case STORAGE_ENGINE_ALLOC:
273 - rrddim_query_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
274 - return;
275 -#ifdef ENABLE_DBENGINE
276 - case STORAGE_ENGINE_DBENGINE:
277 - rrdeng_load_metric_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
278 - return;
279 -#endif
280 - default:
281 - __builtin_unreachable();
282 - }
283 -}
284 -
285 -static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *handle)
286 -{
287 - switch (handle->id) {
288 - case STORAGE_ENGINE_NONE:
289 - case STORAGE_ENGINE_RAM:
290 - case STORAGE_ENGINE_MAP:
291 - case STORAGE_ENGINE_SAVE:
292 - case STORAGE_ENGINE_ALLOC:
293 - return rrddim_query_next_metric(handle);
294 -#ifdef ENABLE_DBENGINE
295 - case STORAGE_ENGINE_DBENGINE:
296 - return rrdeng_load_metric_next(handle);
297 -#endif
298 - default:
299 - __builtin_unreachable();
300 - }
301 -}
302 -
303 -static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *handle)
304 -{
305 - switch (handle->id) {
306 - case STORAGE_ENGINE_NONE:
307 - case STORAGE_ENGINE_RAM:
308 - case STORAGE_ENGINE_MAP:
309 - case STORAGE_ENGINE_SAVE:
310 - case STORAGE_ENGINE_ALLOC:
311 - return rrddim_query_is_finished(handle);
312 -#ifdef ENABLE_DBENGINE
313 - case STORAGE_ENGINE_DBENGINE:
314 - return rrdeng_load_metric_is_finished(handle);
315 -#endif
316 - default:
317 - __builtin_unreachable();
318 - }
319 -}
320 -
321 -static inline void storage_engine_query_finalize(struct storage_engine_query_handle *handle)
322 -{
323 - switch (handle->id) {
324 - case STORAGE_ENGINE_NONE:
325 - case STORAGE_ENGINE_RAM:
326 - case STORAGE_ENGINE_MAP:
327 - case STORAGE_ENGINE_SAVE:
328 - case STORAGE_ENGINE_ALLOC:
329 - rrddim_query_finalize(handle);
330 - return;
331 -#ifdef ENABLE_DBENGINE
332 - case STORAGE_ENGINE_DBENGINE:
333 - rrdeng_load_metric_finalize(handle);
334 - return;
335 -#endif
336 - default:
337 - __builtin_unreachable();
338 - }
339 -}
340 -
341 -static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *handle)
342 -{
343 - switch (handle->id) {
344 - case STORAGE_ENGINE_NONE:
345 - case STORAGE_ENGINE_RAM:
346 - case STORAGE_ENGINE_MAP:
347 - case STORAGE_ENGINE_SAVE:
348 - case STORAGE_ENGINE_ALLOC:
349 - return rrddim_query_align_to_optimal_before(handle);
350 -#ifdef ENABLE_DBENGINE
351 - case STORAGE_ENGINE_DBENGINE:
352 - return rrdeng_load_align_to_optimal_before(handle);
353 -#endif
354 - default:
355 - __builtin_unreachable();
356 - }
357 -}
358 -
359 -static inline bool storage_engine_metric_retention(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s)
360 -{
361 - switch (id) {
362 - case STORAGE_ENGINE_NONE:
363 - case STORAGE_ENGINE_RAM:
364 - case STORAGE_ENGINE_MAP:
365 - case STORAGE_ENGINE_SAVE:
366 - case STORAGE_ENGINE_ALLOC:
367 - return rrddim_metric_retention_by_uuid(db_instance, uuid, first_entry_s, last_entry_s);
368 -#ifdef ENABLE_DBENGINE
369 - case STORAGE_ENGINE_DBENGINE:
370 - return rrdeng_metric_retention_by_uuid(db_instance, uuid, first_entry_s, last_entry_s);
371 -#endif
372 - default:
373 - __builtin_unreachable();
374 - }
375 -}
376 -
377 -static inline STORAGE_METRIC_HANDLE *storage_engine_metric_get(STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance, uuid_t *uuid)
378 -{
379 - switch (id) {
380 - case STORAGE_ENGINE_NONE:
381 - case STORAGE_ENGINE_RAM:
382 - case STORAGE_ENGINE_MAP:
383 - case STORAGE_ENGINE_SAVE:
384 - case STORAGE_ENGINE_ALLOC:
385 - return rrddim_metric_get(instance, uuid);
386 -#ifdef ENABLE_DBENGINE
387 - case STORAGE_ENGINE_DBENGINE:
388 - return rrdeng_metric_get(instance, uuid);
389 -#endif
390 - default:
391 - __builtin_unreachable();
392 - }
393 -}
394 -
395 -static inline STORAGE_METRIC_HANDLE *storage_engine_metric_get_or_create(RRDDIM *rd, STORAGE_ENGINE_ID id, STORAGE_INSTANCE *instance)
396 -{
397 - switch (id) {
398 - case STORAGE_ENGINE_NONE:
399 - case STORAGE_ENGINE_RAM:
400 - case STORAGE_ENGINE_MAP:
401 - case STORAGE_ENGINE_SAVE:
402 - case STORAGE_ENGINE_ALLOC:
403 - return rrddim_metric_get_or_create(rd, instance);
404 -#ifdef ENABLE_DBENGINE
405 - case STORAGE_ENGINE_DBENGINE:
406 - return rrdeng_metric_get_or_create(rd, instance);
407 -#endif
408 - default:
409 - __builtin_unreachable();
410 - }
411 -}
412 -
413 -static inline void storage_engine_metric_release(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
414 -{
415 - switch (id) {
416 - case STORAGE_ENGINE_NONE:
417 - case STORAGE_ENGINE_RAM:
418 - case STORAGE_ENGINE_MAP:
419 - case STORAGE_ENGINE_SAVE:
420 - case STORAGE_ENGINE_ALLOC:
421 - rrddim_metric_release(db_metric_handle);
422 - break;
423 -#ifdef ENABLE_DBENGINE
424 - case STORAGE_ENGINE_DBENGINE:
425 - rrdeng_metric_release(db_metric_handle);
426 - break;
427 -#endif
428 - default:
429 - __builtin_unreachable();
430 - }
431 -}
432 -
433 -static inline STORAGE_METRIC_HANDLE *storage_engine_metric_dup(STORAGE_ENGINE_ID id, STORAGE_METRIC_HANDLE *db_metric_handle)
434 -{
435 - switch (id) {
436 - case STORAGE_ENGINE_NONE:
437 - case STORAGE_ENGINE_RAM:
438 - case STORAGE_ENGINE_MAP:
439 - case STORAGE_ENGINE_SAVE:
440 - case STORAGE_ENGINE_ALLOC:
441 - return rrddim_metric_dup(db_metric_handle);
442 -#ifdef ENABLE_DBENGINE
443 - case STORAGE_ENGINE_DBENGINE:
444 - return rrdeng_metric_dup(db_metric_handle);
445 -#endif
446 - default:
447 - __builtin_unreachable();
448 - }
449 -}
450 -
451 -#endif /* NETDATA_STORAGE_ENGINE_H */
database/storage_engine_types.h deleted
-56
@@ -1,56 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_STORAGE_ENGINE_TYPES_H
4 -#define NETDATA_STORAGE_ENGINE_TYPES_H
5 -
6 -#include "libnetdata/libnetdata.h"
7 -
8 -typedef struct rrddim RRDDIM;
9 -typedef struct storage_instance STORAGE_INSTANCE;
10 -typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
11 -typedef struct storage_alignment STORAGE_METRICS_GROUP;
12 -typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
13 -typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
14 -
15 -typedef enum __attribute__ ((__packed__)) {
16 - STORAGE_ENGINE_NONE = 0,
17 - STORAGE_ENGINE_RAM = 1,
18 - STORAGE_ENGINE_MAP = 2,
19 - STORAGE_ENGINE_SAVE = 3,
20 - STORAGE_ENGINE_ALLOC = 4,
21 - STORAGE_ENGINE_DBENGINE = 5,
22 -} STORAGE_ENGINE_ID;
23 -
24 -#define STORAGE_ENGINE_NONE_NAME "none"
25 -#define STORAGE_ENGINE_RAM_NAME "ram"
26 -#define STORAGE_ENGINE_MAP_NAME "map"
27 -#define STORAGE_ENGINE_SAVE_NAME "save"
28 -#define STORAGE_ENGINE_ALLOC_NAME "alloc"
29 -#define STORAGE_ENGINE_DBENGINE_NAME "dbengine"
30 -
31 -const char *storage_engine_name(STORAGE_ENGINE_ID id);
32 -bool storage_engine_id(const char *name, STORAGE_ENGINE_ID *id);
33 -
34 -typedef enum __attribute__ ((__packed__)) storage_priority {
35 - STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
36 - STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
37 -
38 - STORAGE_PRIORITY_HIGH,
39 - STORAGE_PRIORITY_NORMAL,
40 - STORAGE_PRIORITY_LOW,
41 - STORAGE_PRIORITY_BEST_EFFORT,
42 -
43 - STORAGE_PRIORITY_SYNCHRONOUS,
44 -
45 - STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
46 -} STORAGE_PRIORITY;
47 -
48 -struct storage_engine_query_handle {
49 - time_t start_time_s;
50 - time_t end_time_s;
51 - STORAGE_PRIORITY priority;
52 - STORAGE_ENGINE_ID id;
53 - STORAGE_QUERY_HANDLE *handle;
54 -};
55 -
56 -#endif /* NETDATA_STORAGE_ENGINE_TYPES_H */
\ No newline at end of file
exporting/check_filters.c
+3 -3
@@ -25,7 +25,7 @@ int rrdhost_is_exportable(struct instance *instance, RRDHOST *host)
25 RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index];
26
27 if (unlikely((*flags & (RRDHOST_FLAG_EXPORTING_SEND | RRDHOST_FLAG_EXPORTING_DONT_SEND)) == 0)) {
28 - const char *host_name = (host == rrdb.localhost) ? "localhost" : rrdhost_hostname(host);
28 + const char *host_name = (host == localhost) ? "localhost" : rrdhost_hostname(host);
29
30 if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) {
31 *flags |= RRDHOST_FLAG_EXPORTING_SEND;
@@ -79,8 +79,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)
79 return 0;
80 }
81
82 - if(unlikely(st->storage_engine_id == STORAGE_ENGINE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
83 - netdata_log_debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), storage_engine_name(host->storage_engine_id));
82 + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
83 + netdata_log_debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode));
84 return 0;
85 }
86
exporting/exporting_engine.c
+1 -1
@@ -193,7 +193,7 @@ void *exporting_main(void *ptr)
193 RRDDIM *rd_main_system = NULL;
194 create_main_rusage_chart(&st_main_rusage, &rd_main_user, &rd_main_system);
195
196 - usec_t step_ut = rrdb.localhost->update_every * USEC_PER_SEC;
196 + usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
197 heartbeat_t hb;
198 heartbeat_init(&hb);
199
exporting/graphite/graphite.c
+2 -2
@@ -135,7 +135,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
135 instance->buffer,
136 "%s.%s.%s.%s%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n",
137 instance->config.prefix,
138 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
138 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
139 chart_name,
140 dimension_name,
141 (host->tags) ? ";" : "",
@@ -181,7 +181,7 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
181 instance->buffer,
182 "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
183 instance->config.prefix,
184 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
184 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
185 chart_name,
186 dimension_name,
187 (host->tags) ? ";" : "",
exporting/json/json.c
+2 -2
@@ -186,7 +186,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
186 "\"timestamp\":%llu}",
187
188 instance->config.prefix,
189 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
189 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
190 tags_pre,
191 tags,
192 tags_post,
@@ -270,7 +270,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
270 "\"timestamp\": %llu}",
271
272 instance->config.prefix,
273 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
273 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
274 tags_pre,
275 tags,
276 tags_post,
exporting/opentsdb/opentsdb.c
+4 -4
@@ -192,7 +192,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
192 dimension_name,
193 (unsigned long long)rd->collector.last_collected_time.tv_sec,
194 rd->collector.last_collected_value,
195 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
195 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
196 (host->tags) ? " " : "",
197 (host->tags) ? rrdhost_tags(host) : "",
198 (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
@@ -238,7 +238,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
238 dimension_name,
239 (unsigned long long)last_t,
240 value,
241 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
241 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
242 (host->tags) ? " " : "",
243 (host->tags) ? rrdhost_tags(host) : "",
244 (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
@@ -334,7 +334,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
334 dimension_name,
335 (unsigned long long)rd->collector.last_collected_time.tv_sec,
336 rd->collector.last_collected_value,
337 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
337 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
338 (host->tags) ? " " : "",
339 (host->tags) ? rrdhost_tags(host) : "",
340 instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
@@ -390,7 +390,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
390 dimension_name,
391 (unsigned long long)last_t,
392 value,
393 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
393 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
394 (host->tags) ? " " : "",
395 (host->tags) ? rrdhost_tags(host) : "",
396 instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
exporting/process_data.c
+4 -4
@@ -44,7 +44,7 @@ int mark_scheduled_instances(struct engine *engine)
44
45 for (struct instance *instance = engine->instance_root; instance; instance = instance->next) {
46 if (!instance->disabled && (engine->now % instance->config.update_every >=
47 - instance->config.update_every - rrdb.localhost->update_every)) {
47 + instance->config.update_every - localhost->rrd_update_every)) {
48 instance->scheduled = 1;
49 instances_were_scheduled = 1;
50 instance->before = engine->now;
@@ -77,8 +77,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
77 time_t before = instance->before;
78
79 // find the edges of the rrd database for this chart
80 - time_t first_t = storage_engine_oldest_time_s(st->storage_engine_id, rd->tiers[0].db_metric_handle);
81 - time_t last_t = storage_engine_latest_time_s(st->storage_engine_id, rd->tiers[0].db_metric_handle);
80 + time_t first_t = storage_engine_oldest_time_s(rd->tiers[0].backend, rd->tiers[0].db_metric_handle);
81 + time_t last_t = storage_engine_latest_time_s(rd->tiers[0].backend, rd->tiers[0].db_metric_handle);
82 time_t update_every = st->update_every;
83 struct storage_engine_query_handle handle;
84
@@ -126,7 +126,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
126 size_t counter = 0;
127 NETDATA_DOUBLE sum = 0;
128
129 - for (storage_engine_query_init(st->storage_engine_id, rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_SYNCHRONOUS); !storage_engine_query_is_finished(&handle);) {
129 + for (storage_engine_query_init(rd->tiers[0].backend, rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_SYNCHRONOUS); !storage_engine_query_is_finished(&handle);) {
130 STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
131 points_read++;
132
exporting/prometheus/prometheus.c
+3 -3
@@ -60,14 +60,14 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN
60 }
61
62 if (unlikely(
63 - st->storage_engine_id == STORAGE_ENGINE_NONE &&
63 + st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&
64 !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
65 netdata_log_debug(
66 D_EXPORTING,
67 "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.",
68 rrdset_id(st),
69 rrdhost_hostname(host),
70 - storage_engine_name(host->storage_engine_id));
70 + rrd_memory_mode_name(host->rrd_memory_mode));
71 return 0;
72 }
73
@@ -957,7 +957,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
957 prometheus_exporter_instance->before,
958 output_options);
959
960 - dfe_start_reentrant(rrdb.rrdhost_root_index, host)
960 + dfe_start_reentrant(rrdhost_root_index, host)
961 {
962 rrd_stats_api_v1_charts_allmetrics_prometheus(
963 prometheus_exporter_instance, host, filter_string, wb, prefix, exporting_options, 1, output_options);
exporting/prometheus/remote_write/remote_write.c
+6 -6
@@ -170,7 +170,7 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
170 char hostname[PROMETHEUS_ELEMENT_MAX + 1];
171 prometheus_label_copy(
172 hostname,
173 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
173 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
174 PROMETHEUS_ELEMENT_MAX);
175
176 add_host_info(
@@ -249,7 +249,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
249 "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', "
250 "its last data collection (%lu) is not within our timeframe (%lu to %lu)",
251 rrddim_id(rd), rrdset_id(rd->rrdset),
252 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
252 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
253 (unsigned long)rd->collector.last_collected_time.tv_sec,
254 (unsigned long)instance->after,
255 (unsigned long)instance->before);
@@ -274,7 +274,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
274 add_metric(
275 connector_specific_data->write_request,
276 name, chart, family, dimension,
277 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
277 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
278 rd->collector.last_collected_value, timeval_msec(&rd->collector.last_collected_time));
279 } else {
280 // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
@@ -291,7 +291,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
291 add_metric(
292 connector_specific_data->write_request,
293 name, chart, family, NULL,
294 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
294 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
295 rd->collector.last_collected_value, timeval_msec(&rd->collector.last_collected_time));
296 }
297 } else {
@@ -316,7 +316,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
316 add_metric(
317 connector_specific_data->write_request,
318 name, chart, family, dimension,
319 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host),
319 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
320 value, last_t * MSEC_PER_SEC);
321 }
322 }
@@ -346,7 +346,7 @@ static int format_variable_prometheus_remote_write_callback(const DICTIONARY_ITE
346
347 NETDATA_DOUBLE value = rrdvar2number(rv);
348 add_variable(connector_specific_data->write_request, name,
349 - (host == rrdb.localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS);
349 + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS);
350 }
351
352 return 0;
exporting/read_config.c
+2 -2
@@ -380,11 +380,11 @@ struct engine *read_exporting_config()
380
381 tmp_instance->config.options = exporting_parse_data_source(data_source, tmp_instance->config.options);
382 if (EXPORTING_OPTIONS_DATA_SOURCE(tmp_instance->config.options) != EXPORTING_SOURCE_DATA_AS_COLLECTED &&
383 - tmp_instance->config.update_every % rrdb.localhost->update_every)
383 + tmp_instance->config.update_every % localhost->rrd_update_every)
384 netdata_log_info(
385 "The update interval %d for instance %s is not a multiple of the database update interval %d. "
386 "Metric values will deviate at different points in time.",
387 - tmp_instance->config.update_every, tmp_instance->config.name, rrdb.localhost->update_every);
387 + tmp_instance->config.update_every, tmp_instance->config.name, localhost->rrd_update_every);
388
389 if (exporter_get_boolean(instance_name, "send configured labels", CONFIG_BOOLEAN_YES))
390 tmp_instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
exporting/send_internal_metrics.c
+1 -1
@@ -19,7 +19,7 @@ void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_
19
20 *st_rusage = rrdset_create_localhost(
21 "netdata", "exporting_main_thread_cpu", NULL, "exporting", "exporting_cpu_usage", "Netdata Main Exporting Thread CPU Usage",
22 - "milliseconds/s", "exporting", NULL, 130600, rrdb.localhost->update_every, RRDSET_TYPE_STACKED);
22 + "milliseconds/s", "exporting", NULL, 130600, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
23
24 *rd_user = rrddim_add(*st_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
25 *rd_system = rrddim_add(*st_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
exporting/tests/exporting_fixtures.c
+2 -2
@@ -64,7 +64,7 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
64 st->name = string_strdupz("chart_name");
65
66 st->update_every = 1;
67 - st->storage_engine_id = STORAGE_ENGINE_SAVE;
67 + st->rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
68
69 st->rrdhost = host;
70
@@ -77,7 +77,7 @@ int setup_rrdhost()
77 {
78 localhost = calloc(1, sizeof(RRDHOST));
79
80 - localhost->update_every = 1;
80 + localhost->rrd_update_every = 1;
81
82 localhost->tags = string_strdupz("TAG1=VALUE1 TAG2=VALUE2");
83
exporting/tests/netdata_doubles.c
+6 -6
@@ -99,7 +99,7 @@ RRDSET *rrdset_create_custom(
99 long priority,
100 int update_every,
101 RRDSET_TYPE chart_type,
102 - STORAGE_ENGINE_ID storage_engine_id,
102 + RRD_MEMORY_MODE memory_mode,
103 long history_entries)
104 {
105 check_expected_ptr(host);
@@ -115,7 +115,7 @@ RRDSET *rrdset_create_custom(
115 check_expected(priority);
116 check_expected(update_every);
117 check_expected(chart_type);
118 - UNUSED(storage_engine_id);
118 + UNUSED(memory_mode);
119 UNUSED(history_entries);
120
121 function_called();
@@ -145,7 +145,7 @@ RRDDIM *rrddim_add_custom(
145 collected_number multiplier,
146 collected_number divisor,
147 RRD_ALGORITHM algorithm,
148 - STORAGE_ENGINE_ID storage_engine_id)
148 + RRD_MEMORY_MODE memory_mode)
149 {
150 check_expected_ptr(st);
151 UNUSED(id);
@@ -153,7 +153,7 @@ RRDDIM *rrddim_add_custom(
153 check_expected(multiplier);
154 check_expected(divisor);
155 check_expected(algorithm);
156 - UNUSED(storage_engine_id);
156 + UNUSED(memory_mode);
157
158 function_called();
159
@@ -171,10 +171,10 @@ collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number
171 return 0;
172 }
173
174 -const char *storage_engine_name(STORAGE_ENGINE_ID id)
174 +const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
175 {
176 (void)id;
177 - return STORAGE_ENGINE_NONE_NAME;
177 + return RRD_MEMORY_MODE_NONE_NAME;
178 }
179
180 void rrdset_update_heterogeneous_flag(RRDSET *st)
exporting/tests/test_exporting_engine.c
+2 -2
@@ -849,7 +849,7 @@ static void test_create_main_rusage_chart(void **state)
849 expect_string(rrdset_create_custom, plugin, "exporting");
850 expect_value(rrdset_create_custom, module, NULL);
851 expect_value(rrdset_create_custom, priority, 130600);
852 - expect_value(rrdset_create_custom, update_every, localhost->update_every);
852 + expect_value(rrdset_create_custom, update_every, localhost->rrd_update_every);
853 expect_value(rrdset_create_custom, chart_type, RRDSET_TYPE_STACKED);
854 will_return(rrdset_create_custom, st_rusage);
855
@@ -1078,7 +1078,7 @@ static void test_can_send_rrdset(void **state)
1078 assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0);
1079 rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
1080
1081 - st->storage_engine_id = STORAGE_ENGINE_NONE;
1081 + st->rrd_memory_mode = RRD_MEMORY_MODE_NONE;
1082 prometheus_exporter_instance->config.options |= EXPORTING_SOURCE_DATA_AVERAGE;
1083 assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0);
1084 }
health/health.c
+8 -8
@@ -254,7 +254,7 @@ static inline void unlink_alarm_notify_in_progress(ALARM_ENTRY *ae)
254 *
255 * @return a pointer to the user config directory
256 */
257 -static const char *health_user_config_dir(void) {
257 +inline char *health_user_config_dir(void) {
258 char buffer[FILENAME_MAX + 1];
259 snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_user_config_dir);
260 return config_get(CONFIG_SECTION_DIRECTORIES, "health config", buffer);
@@ -267,7 +267,7 @@ static const char *health_user_config_dir(void) {
267 *
268 * @return a pointer to the stock config directory.
269 */
270 -static const char *health_stock_config_dir(void) {
270 +inline char *health_stock_config_dir(void) {
271 char buffer[FILENAME_MAX + 1];
272 snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_stock_config_dir);
273 return config_get(CONFIG_SECTION_DIRECTORIES, "stock health config", buffer);
@@ -344,8 +344,8 @@ static void health_reload_host(RRDHOST *host) {
344
345 netdata_log_health("[%s]: Reloading health.", rrdhost_hostname(host));
346
347 - const char *user_path = health_user_config_dir();
348 - const char *stock_path = health_stock_config_dir();
347 + char *user_path = health_user_config_dir();
348 + char *stock_path = health_stock_config_dir();
349
350 // free all running alarms
351 rrdcalc_delete_all(host);
@@ -403,7 +403,7 @@ void health_reload(void) {
403 sql_refresh_hashes();
404
405 RRDHOST *host;
406 - dfe_start_reentrant(rrdb.rrdhost_root_index, host){
406 + dfe_start_reentrant(rrdhost_root_index, host){
407 health_reload_host(host);
408 }
409 dfe_done(host);
@@ -1080,7 +1080,7 @@ void *health_main(void *ptr) {
1080 }
1081
1082 worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
1083 - dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
1083 + dfe_start_reentrant(rrdhost_root_index, host) {
1084
1085 if(unlikely(!service_running(SERVICE_HEALTH)))
1086 break;
@@ -1114,7 +1114,7 @@ void *health_main(void *ptr) {
1114 }
1115
1116 // wait until cleanup of obsolete charts on children is complete
1117 - if (host != rrdb.localhost) {
1117 + if (host != localhost) {
1118 if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
1119 netdata_log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
1120 continue;
@@ -1611,7 +1611,7 @@ void *health_main(void *ptr) {
1611 }
1612
1613 void health_add_host_labels(void) {
1614 - DICTIONARY *labels = rrdb.localhost->rrdlabels;
1614 + DICTIONARY *labels = localhost->rrdlabels;
1615
1616 // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).
1617 // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.
health/health.h
+7
@@ -51,12 +51,15 @@ void health_reload(void);
51
52 void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status);
53 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);
54 +void health_alert2json_conf(RRDHOST *host, BUFFER *wb, CONTEXTS_V2_OPTIONS all);
55 void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all);
56
57 void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf);
58 void health_api_v1_chart_custom_variables2json(RRDSET *st, BUFFER *buf);
59
60 +int health_alarm_log_open(RRDHOST *host);
61 void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
62 +void health_alarm_log_load(RRDHOST *host);
63
64 ALARM_ENTRY* health_create_alarm_entry(
65 RRDHOST *host,
@@ -88,10 +91,14 @@ ALARM_ENTRY* health_create_alarm_entry(
91 void health_alarm_log_add_entry(RRDHOST *host, ALARM_ENTRY *ae);
92
93 void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath);
94 +char *health_user_config_dir(void);
95 +char *health_stock_config_dir(void);
96 void health_alarm_log_free(RRDHOST *host);
97
98 void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae);
99
100 +void *health_cmdapi_thread(void *ptr);
101 +
102 char *health_edit_command_from_source(const char *source);
103 void sql_refresh_hashes(void);
104
health/health_config.c
+1 -1
@@ -452,7 +452,7 @@ char *health_edit_command_from_source(const char *source)
452 netdata_configured_user_config_dir,
453 file_no_path + 1,
454 temp,
455 - rrdhost_registry_hostname(rrdb.localhost));
455 + rrdhost_registry_hostname(localhost));
456 } else
457 buffer[0] = '\0';
458
libnetdata/aral/aral.c
+1 -2
@@ -1,6 +1,5 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "../libnetdata.h"
2 +#include "aral.h"
3
4 #ifdef NETDATA_TRACE_ALLOCATIONS
5 #define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS , const char *file, const char *function, size_t line
libnetdata/aral/aral.h
-1
@@ -1,4 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
1
2 #ifndef ARAL_H
3 #define ARAL_H 1
libnetdata/circular_buffer/circular_buffer.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "../libnetdata.h"
2
3 struct circular_buffer *cbuffer_new(size_t initial, size_t max, size_t *statistics) {
libnetdata/circular_buffer/circular_buffer.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef CIRCULAR_BUFFER_H
2 #define CIRCULAR_BUFFER_H 1
3
libnetdata/json/jsmn.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include <stdlib.h>
2
3 #include "jsmn.h"
libnetdata/json/jsmn.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef __JSMN_H_
2 #define __JSMN_H_
3
libnetdata/libnetdata.c
+2 -2
@@ -1139,8 +1139,8 @@ void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool r
1139 {
1140 // netdata_log_info("netdata_mmap('%s', %zu", filename, size);
1141
1142 - // MAP_SHARED is used in storage engine MAP
1143 - // MAP_PRIVATE is used in storage engine RAM and SAVE
1142 + // MAP_SHARED is used in memory mode map
1143 + // MAP_PRIVATE is used in memory mode ram and save
1144
1145 if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE)))
1146 fatal("Neither MAP_SHARED or MAP_PRIVATE were given to netdata_mmap()");
libnetdata/libnetdata.h
+2 -1
@@ -248,7 +248,6 @@ size_t judy_aral_structures(void);
248 #define ABS(x) (((x) < 0)? (-(x)) : (x))
249 #define MIN(a,b) (((a)<(b))?(a):(b))
250 #define MAX(a,b) (((a)>(b))?(a):(b))
251 -#define CLAMP(value, min, max) MIN(MAX((value), (min)), (max))
251
252 #define GUID_LEN 36
253
@@ -842,6 +841,8 @@ extern char *netdata_configured_host_prefix;
841 // BEWARE: this exists in alarm-notify.sh
842 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
843
844 +#define RRD_STORAGE_TIERS 5
845 +
846 static inline size_t struct_natural_alignment(size_t size) __attribute__((const));
847
848 #define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
libnetdata/onewayalloc/onewayalloc.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "onewayalloc.h"
2
3 // https://www.gnu.org/software/libc/manual/html_node/Aligned-Memory-Blocks.html
libnetdata/onewayalloc/onewayalloc.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef ONEWAYALLOC_H
2 #define ONEWAYALLOC_H 1
3
libnetdata/socket/security.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "../libnetdata.h"
2
3 #ifdef ENABLE_HTTPS
libnetdata/socket/security.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef NETDATA_SECURITY_H
2 # define NETDATA_SECURITY_H
3
libnetdata/string/string.h
-1
@@ -1,4 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
1
2 #ifndef NETDATA_STRING_H
3 #define NETDATA_STRING_H 1
libnetdata/worker_utilization/worker_utilization.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "worker_utilization.h"
2
3 #define WORKER_IDLE 'I'
libnetdata/worker_utilization/worker_utilization.h
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #ifndef WORKER_UTILIZATION_H
2 #define WORKER_UTILIZATION_H 1
3
ml/ad_charts.cc
+55 -58
@@ -1,21 +1,18 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "database/rrd.h"
3 #include "ad_charts.h"
4
6 -#define BUF_LEN 1024
7 -
5 void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats_t &mls) {
6 /*
7 * Machine learning status
8 */
9 if (Cfg.enable_statistics_charts) {
10 if (!host->machine_learning_status_rs) {
14 - char id_buf[BUF_LEN + 1];
15 - char name_buf[BUF_LEN + 1];
11 + char id_buf[1024];
12 + char name_buf[1024];
13
17 - snprintfz(id_buf, BUF_LEN, "machine_learning_status_on_%s", rrdb.localhost->machine_guid);
18 - snprintfz(name_buf, BUF_LEN, "machine_learning_status_on_%s", rrdhost_hostname(rrdb.localhost));
14 + snprintfz(id_buf, 1024, "machine_learning_status_on_%s", localhost->machine_guid);
15 + snprintfz(name_buf, 1024, "machine_learning_status_on_%s", rrdhost_hostname(localhost));
16
17 host->machine_learning_status_rs = rrdset_create(
18 host->rh,
@@ -29,7 +26,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
26 NETDATA_ML_PLUGIN, // plugin
27 NETDATA_ML_MODULE_TRAINING, // module
28 NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
32 - rrdb.localhost->update_every, // update_every
29 + localhost->rrd_update_every, // update_every
30 RRDSET_TYPE_LINE // chart_type
31 );
32 rrdset_flag_set(host->machine_learning_status_rs , RRDSET_FLAG_ANOMALY_DETECTION);
@@ -53,11 +50,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
50 */
51 if (Cfg.enable_statistics_charts) {
52 if (!host->metric_type_rs) {
56 - char id_buf[BUF_LEN + 1];
57 - char name_buf[BUF_LEN + 1];
53 + char id_buf[1024];
54 + char name_buf[1024];
55
59 - snprintfz(id_buf, BUF_LEN, "metric_types_on_%s", rrdb.localhost->machine_guid);
60 - snprintfz(name_buf, BUF_LEN, "metric_types_on_%s", rrdhost_hostname(rrdb.localhost));
56 + snprintfz(id_buf, 1024, "metric_types_on_%s", localhost->machine_guid);
57 + snprintfz(name_buf, 1024, "metric_types_on_%s", rrdhost_hostname(localhost));
58
59 host->metric_type_rs = rrdset_create(
60 host->rh,
@@ -71,7 +68,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
68 NETDATA_ML_PLUGIN, // plugin
69 NETDATA_ML_MODULE_TRAINING, // module
70 NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
74 - rrdb.localhost->update_every, // update_every
71 + localhost->rrd_update_every, // update_every
72 RRDSET_TYPE_LINE // chart_type
73 );
74 rrdset_flag_set(host->metric_type_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -95,11 +92,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
92 */
93 if (Cfg.enable_statistics_charts) {
94 if (!host->training_status_rs) {
98 - char id_buf[BUF_LEN + 1];
99 - char name_buf[BUF_LEN + 1];
95 + char id_buf[1024];
96 + char name_buf[1024];
97
101 - snprintfz(id_buf, BUF_LEN, "training_status_on_%s", rrdb.localhost->machine_guid);
102 - snprintfz(name_buf, BUF_LEN, "training_status_on_%s", rrdhost_hostname(rrdb.localhost));
98 + snprintfz(id_buf, 1024, "training_status_on_%s", localhost->machine_guid);
99 + snprintfz(name_buf, 1024, "training_status_on_%s", rrdhost_hostname(localhost));
100
101 host->training_status_rs = rrdset_create(
102 host->rh,
@@ -113,7 +110,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
110 NETDATA_ML_PLUGIN, // plugin
111 NETDATA_ML_MODULE_TRAINING, // module
112 NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
116 - rrdb.localhost->update_every, // update_every
113 + localhost->rrd_update_every, // update_every
114 RRDSET_TYPE_LINE // chart_type
115 );
116
@@ -150,11 +147,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
147 */
148 {
149 if (!host->dimensions_rs) {
153 - char id_buf[BUF_LEN + 1];
154 - char name_buf[BUF_LEN + 1];
150 + char id_buf[1024];
151 + char name_buf[1024];
152
156 - snprintfz(id_buf, BUF_LEN, "dimensions_on_%s", rrdb.localhost->machine_guid);
157 - snprintfz(name_buf, BUF_LEN, "dimensions_on_%s", rrdhost_hostname(rrdb.localhost));
153 + snprintfz(id_buf, 1024, "dimensions_on_%s", localhost->machine_guid);
154 + snprintfz(name_buf, 1024, "dimensions_on_%s", rrdhost_hostname(localhost));
155
156 host->dimensions_rs = rrdset_create(
157 host->rh,
@@ -168,7 +165,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
165 NETDATA_ML_PLUGIN, // plugin
166 NETDATA_ML_MODULE_TRAINING, // module
167 ML_CHART_PRIO_DIMENSIONS, // priority
171 - rrdb.localhost->update_every, // update_every
168 + localhost->rrd_update_every, // update_every
169 RRDSET_TYPE_LINE // chart_type
170 );
171 rrdset_flag_set(host->dimensions_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -190,11 +187,11 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
187 // ML running
188 {
189 if (!host->ml_running_rs) {
193 - char id_buf[BUF_LEN + 1];
194 - char name_buf[BUF_LEN + 1];
190 + char id_buf[1024];
191 + char name_buf[1024];
192
196 - snprintfz(id_buf, BUF_LEN, "ml_running_on_%s", rrdb.localhost->machine_guid);
197 - snprintfz(name_buf, BUF_LEN, "ml_running_on_%s", rrdhost_hostname(rrdb.localhost));
193 + snprintfz(id_buf, 1024, "ml_running_on_%s", localhost->machine_guid);
194 + snprintfz(name_buf, 1024, "ml_running_on_%s", rrdhost_hostname(localhost));
195
196 host->ml_running_rs = rrdset_create(
197 host->rh,
@@ -208,7 +205,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
205 NETDATA_ML_PLUGIN, // plugin
206 NETDATA_ML_MODULE_DETECTION, // module
207 NETDATA_ML_CHART_RUNNING, // priority
211 - rrdb.localhost->update_every, // update_every
208 + localhost->rrd_update_every, // update_every
209 RRDSET_TYPE_LINE // chart_type
210 );
211 rrdset_flag_set(host->ml_running_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -229,11 +226,11 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
226 */
227 {
228 if (!host->anomaly_rate_rs) {
232 - char id_buf[BUF_LEN + 1];
233 - char name_buf[BUF_LEN + 1];
229 + char id_buf[1024];
230 + char name_buf[1024];
231
235 - snprintfz(id_buf, BUF_LEN, "anomaly_rate_on_%s", rrdb.localhost->machine_guid);
236 - snprintfz(name_buf, BUF_LEN, "anomaly_rate_on_%s", rrdhost_hostname(rrdb.localhost));
232 + snprintfz(id_buf, 1024, "anomaly_rate_on_%s", localhost->machine_guid);
233 + snprintfz(name_buf, 1024, "anomaly_rate_on_%s", rrdhost_hostname(localhost));
234
235 host->anomaly_rate_rs = rrdset_create(
236 host->rh,
@@ -247,7 +244,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
244 NETDATA_ML_PLUGIN, // plugin
245 NETDATA_ML_MODULE_DETECTION, // module
246 ML_CHART_PRIO_ANOMALY_RATE, // priority
250 - rrdb.localhost->update_every, // update_every
247 + localhost->rrd_update_every, // update_every
248 RRDSET_TYPE_LINE // chart_type
249 );
250 rrdset_flag_set(host->anomaly_rate_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -266,11 +263,11 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
263 */
264 {
265 if (!host->detector_events_rs) {
269 - char id_buf[BUF_LEN + 1];
270 - char name_buf[BUF_LEN + 1];
266 + char id_buf[1024];
267 + char name_buf[1024];
268
272 - snprintfz(id_buf, BUF_LEN, "anomaly_detection_on_%s", rrdb.localhost->machine_guid);
273 - snprintfz(name_buf, BUF_LEN, "anomaly_detection_on_%s", rrdhost_hostname(rrdb.localhost));
269 + snprintfz(id_buf, 1024, "anomaly_detection_on_%s", localhost->machine_guid);
270 + snprintfz(name_buf, 1024, "anomaly_detection_on_%s", rrdhost_hostname(localhost));
271
272 host->detector_events_rs = rrdset_create(
273 host->rh,
@@ -284,7 +281,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
281 NETDATA_ML_PLUGIN, // plugin
282 NETDATA_ML_MODULE_DETECTION, // module
283 ML_CHART_PRIO_DETECTOR_EVENTS, // priority
287 - rrdb.localhost->update_every, // update_every
284 + localhost->rrd_update_every, // update_every
285 RRDSET_TYPE_LINE // chart_type
286 );
287 rrdset_flag_set(host->detector_events_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -301,7 +298,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
298 if (host->ml_running) {
299 ONEWAYALLOC *OWA = onewayalloc_create(0);
300 time_t Now = now_realtime_sec();
304 - time_t Before = Now - host->rh->update_every;
301 + time_t Before = Now - host->rh->rrd_update_every;
302 time_t After = Before - Cfg.anomaly_detection_query_duration;
303 RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
304
@@ -356,14 +353,14 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
353 */
354 {
355 if (!training_thread->queue_stats_rs) {
359 - char id_buf[BUF_LEN + 1];
360 - char name_buf[BUF_LEN + 1];
356 + char id_buf[1024];
357 + char name_buf[1024];
358
362 - snprintfz(id_buf, BUF_LEN, "training_queue_%zu_stats", training_thread->id);
363 - snprintfz(name_buf, BUF_LEN, "training_queue_%zu_stats", training_thread->id);
359 + snprintfz(id_buf, 1024, "training_queue_%zu_stats", training_thread->id);
360 + snprintfz(name_buf, 1024, "training_queue_%zu_stats", training_thread->id);
361
362 training_thread->queue_stats_rs = rrdset_create(
366 - rrdb.localhost,
363 + localhost,
364 "netdata", // type
365 id_buf, // id
366 name_buf, // name
@@ -374,7 +371,7 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
371 NETDATA_ML_PLUGIN, // plugin
372 NETDATA_ML_MODULE_TRAINING, // module
373 NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
377 - rrdb.localhost->update_every, // update_every
374 + localhost->rrd_update_every, // update_every
375 RRDSET_TYPE_LINE// chart_type
376 );
377 rrdset_flag_set(training_thread->queue_stats_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -398,14 +395,14 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
395 */
396 {
397 if (!training_thread->training_time_stats_rs) {
401 - char id_buf[BUF_LEN + 1];
402 - char name_buf[BUF_LEN + 1];
398 + char id_buf[1024];
399 + char name_buf[1024];
400
404 - snprintfz(id_buf, BUF_LEN, "training_queue_%zu_time_stats", training_thread->id);
405 - snprintfz(name_buf, BUF_LEN, "training_queue_%zu_time_stats", training_thread->id);
401 + snprintfz(id_buf, 1024, "training_queue_%zu_time_stats", training_thread->id);
402 + snprintfz(name_buf, 1024, "training_queue_%zu_time_stats", training_thread->id);
403
404 training_thread->training_time_stats_rs = rrdset_create(
408 - rrdb.localhost,
405 + localhost,
406 "netdata", // type
407 id_buf, // id
408 name_buf, // name
@@ -416,7 +413,7 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
413 NETDATA_ML_PLUGIN, // plugin
414 NETDATA_ML_MODULE_TRAINING, // module
415 NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
419 - rrdb.localhost->update_every, // update_every
416 + localhost->rrd_update_every, // update_every
417 RRDSET_TYPE_LINE// chart_type
418 );
419 rrdset_flag_set(training_thread->training_time_stats_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -444,14 +441,14 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
441 */
442 {
443 if (!training_thread->training_results_rs) {
447 - char id_buf[BUF_LEN + 1];
448 - char name_buf[BUF_LEN + 1];
444 + char id_buf[1024];
445 + char name_buf[1024];
446
450 - snprintfz(id_buf, BUF_LEN, "training_queue_%zu_results", training_thread->id);
451 - snprintfz(name_buf, BUF_LEN, "training_queue_%zu_results", training_thread->id);
447 + snprintfz(id_buf, 1024, "training_queue_%zu_results", training_thread->id);
448 + snprintfz(name_buf, 1024, "training_queue_%zu_results", training_thread->id);
449
450 training_thread->training_results_rs = rrdset_create(
454 - rrdb.localhost,
451 + localhost,
452 "netdata", // type
453 id_buf, // id
454 name_buf, // name
@@ -462,7 +459,7 @@ void ml_update_training_statistics_chart(ml_training_thread_t *training_thread,
459 NETDATA_ML_PLUGIN, // plugin
460 NETDATA_ML_MODULE_TRAINING, // module
461 NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
465 - rrdb.localhost->update_every, // update_every
462 + localhost->rrd_update_every, // update_every
463 RRDSET_TYPE_LINE// chart_type
464 );
465 rrdset_flag_set(training_thread->training_results_rs, RRDSET_FLAG_ANOMALY_DETECTION);
@@ -511,7 +508,7 @@ void ml_update_global_statistics_charts(uint64_t models_consulted) {
508 , NETDATA_ML_PLUGIN // plugin
509 , NETDATA_ML_MODULE_DETECTION // module
510 , NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS // priority
514 - , rrdb.localhost->update_every // update_every
511 + , localhost->rrd_update_every // update_every
512 , RRDSET_TYPE_AREA // chart_type
513 );
514
ml/ml-dummy.c
-2
@@ -110,13 +110,11 @@ void ml_update_global_statistics_charts(uint64_t models_consulted) {
110 }
111
112 bool ml_host_get_host_status(RRDHOST *rh, struct ml_metrics_statistics *mlm) {
113 - UNUSED(rh);
113 memset(mlm, 0, sizeof(*mlm));
114 return false;
115 }
116
117 bool ml_host_running(RRDHOST *rh) {
119 - UNUSED(rh);
118 return false;
119 }
120
ml/ml.cc
+1 -1
@@ -356,7 +356,7 @@ ml_dimension_calculated_numbers(ml_training_thread_t *training_thread, ml_dimens
356 */
357 struct storage_engine_query_handle handle;
358
359 - storage_engine_query_init(dim->rd->rrdset->storage_engine_id, dim->rd->tiers[0].db_metric_handle, &handle,
359 + storage_engine_query_init(dim->rd->tiers[0].backend, dim->rd->tiers[0].db_metric_handle, &handle,
360 training_response.query_after_t, training_response.query_before_t,
361 STORAGE_PRIORITY_BEST_EFFORT);
362
registry/registry.c
+7 -7
@@ -171,10 +171,10 @@ int registry_request_hello_json(RRDHOST *host, struct web_client *w) {
171
172 buffer_json_member_add_object(w->response.data, "agent");
173 {
174 - buffer_json_member_add_string(w->response.data, "machine_guid", rrdb.localhost->machine_guid);
174 + buffer_json_member_add_string(w->response.data, "machine_guid", localhost->machine_guid);
175
176 - if(rrdb.localhost->node_id)
177 - buffer_json_member_add_uuid(w->response.data, "node_id", rrdb.localhost->node_id);
176 + if(localhost->node_id)
177 + buffer_json_member_add_uuid(w->response.data, "node_id", localhost->node_id);
178
179 char *claim_id = get_agent_claimid();
180 if (claim_id) {
@@ -195,7 +195,7 @@ int registry_request_hello_json(RRDHOST *host, struct web_client *w) {
195
196 buffer_json_member_add_array(w->response.data, "nodes");
197 RRDHOST *h;
198 - dfe_start_read(rrdb.rrdhost_root_index, h) {
198 + dfe_start_read(rrdhost_root_index, h) {
199 buffer_json_add_array_item_object(w->response.data);
200 buffer_json_member_add_string(w->response.data, "machine_guid", h->machine_guid);
201
@@ -456,7 +456,7 @@ void registry_statistics(void) {
456 , "registry"
457 , "stats"
458 , 131000
459 - , rrdb.localhost->update_every
459 + , localhost->rrd_update_every
460 , RRDSET_TYPE_LINE
461 );
462
@@ -480,7 +480,7 @@ void registry_statistics(void) {
480 , "registry"
481 , "stats"
482 , 131100
483 - , rrdb.localhost->update_every
483 + , localhost->rrd_update_every
484 , RRDSET_TYPE_LINE
485 );
486
@@ -510,7 +510,7 @@ void registry_statistics(void) {
510 , "registry"
511 , "stats"
512 , 131300
513 - , rrdb.localhost->update_every
513 + , localhost->rrd_update_every
514 , RRDSET_TYPE_STACKED
515 );
516
streaming/compression.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include "rrdpush.h"
2
3 #ifdef ENABLE_RRDPUSH_COMPRESSION
streaming/receiver.c
+18 -35
@@ -391,7 +391,7 @@ static bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
391 rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);
392
393 host->rrdpush_receiver_connection_counter++;
394 - __atomic_add_fetch(&rrdb.localhost->connected_children_count, 1, __ATOMIC_RELAXED);
394 + __atomic_add_fetch(&localhost->connected_children_count, 1, __ATOMIC_RELAXED);
395
396 host->receiver = rpt;
397 rpt->host = host;
@@ -445,7 +445,7 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
445
446 // Make sure that we detach this thread and don't kill a freshly arriving receiver
447 if(host->receiver == rpt) {
448 - __atomic_sub_fetch(&rrdb.localhost->connected_children_count, 1, __ATOMIC_RELAXED);
448 + __atomic_sub_fetch(&localhost->connected_children_count, 1, __ATOMIC_RELAXED);
449 rrdhost_flag_set(rpt->host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED);
450
451 host->trigger_chart_obsoletion_check = 0;
@@ -549,8 +549,8 @@ void rrdpush_receive_log_status(struct receiver_state *rpt, const char *msg, con
549
550 static void rrdpush_receive(struct receiver_state *rpt)
551 {
552 - rpt->config.storage_engine_id = default_storage_engine_id;
553 - rpt->config.history = rrdb.default_rrd_history_entries;
552 + rpt->config.mode = default_rrd_memory_mode;
553 + rpt->config.history = default_rrd_history_entries;
554
555 rpt->config.health_enabled = (int)default_health_enabled;
556 rpt->config.alarms_delay = 60;
@@ -572,34 +572,17 @@ static void rrdpush_receive(struct receiver_state *rpt)
572 rpt->config.history = (int)appconfig_get_number(&stream_config, rpt->machine_guid, "history", rpt->config.history);
573 if(rpt->config.history < 5) rpt->config.history = 5;
574
575 - // figure out storage engine ids for key and/or machine guid
576 - {
577 - const char *se_name = appconfig_get(&stream_config, rpt->key,
578 - "default memory mode",
579 - storage_engine_name(rpt->config.storage_engine_id));
580 - if (!storage_engine_id(se_name, &rpt->config.storage_engine_id)) {
581 - netdata_log_error("STREAM '%s' [receive from %s:%s]: invalid default memory mode for key %s (given: '%s', will use: '%s').",
582 - rpt->hostname, rpt->client_ip, rpt->client_port,
583 - rpt->key, se_name, storage_engine_name(rpt->config.storage_engine_id));
584 - }
575 + rpt->config.mode = rrd_memory_mode_id(appconfig_get(&stream_config, rpt->key, "default memory mode", rrd_memory_mode_name(rpt->config.mode)));
576 + rpt->config.mode = rrd_memory_mode_id(appconfig_get(&stream_config, rpt->machine_guid, "memory mode", rrd_memory_mode_name(rpt->config.mode)));
577
586 - se_name = appconfig_get(&stream_config, rpt->machine_guid,
587 - "memory mode",
588 - storage_engine_name(rpt->config.storage_engine_id));
589 -
590 - if (!storage_engine_id(se_name, &rpt->config.storage_engine_id)) {
591 - netdata_log_error("STREAM '%s' [receive from %s:%s]: invalid memory mode for machine guid %s (given: '%s', will use: '%s').",
592 - rpt->hostname, rpt->client_ip, rpt->client_port,
593 - rpt->machine_guid, se_name, storage_engine_name(rpt->config.storage_engine_id));
594 - }
595 - }
596 -
597 - if (unlikely(rpt->config.storage_engine_id == STORAGE_ENGINE_DBENGINE && !rrdb.dbengine_enabled))
598 - {
599 - netdata_log_error("STREAM '%s' [receive from %s:%s]: dbengine is not enabled, falling back to default.",
600 - rpt->hostname, rpt->client_ip, rpt->client_port);
578 + if (unlikely(rpt->config.mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled)) {
579 + netdata_log_error("STREAM '%s' [receive from %s:%s]: "
580 + "dbengine is not enabled, falling back to default."
581 + , rpt->hostname
582 + , rpt->client_ip, rpt->client_port
583 + );
584
602 - rpt->config.storage_engine_id = default_storage_engine_id;
585 + rpt->config.mode = default_rrd_memory_mode;
586 }
587
588 rpt->config.health_enabled = appconfig_get_boolean_ondemand(&stream_config, rpt->key, "health enabled by default", rpt->config.health_enabled);
@@ -643,7 +626,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
626 // find the host for this receiver
627 {
628 // this will also update the host with our system_info
646 - RRDHOST *host = rrdhost_get_or_create(
629 + RRDHOST *host = rrdhost_find_or_create(
630 rpt->hostname
631 , rpt->registry_hostname
632 , rpt->machine_guid
@@ -656,7 +639,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
639 , rpt->program_version
640 , rpt->config.update_every
641 , rpt->config.history
659 - , rpt->config.storage_engine_id
642 + , rpt->config.mode
643 , (unsigned int)(rpt->config.health_enabled != CONFIG_BOOLEAN_NO)
644 , (unsigned int)(rpt->config.rrdpush_enabled && rpt->config.rrdpush_destination && *rpt->config.rrdpush_destination && rpt->config.rrdpush_api_key && *rpt->config.rrdpush_api_key)
645 , rpt->config.rrdpush_destination
@@ -700,9 +683,9 @@ static void rrdpush_receive(struct receiver_state *rpt)
683 , rpt->client_port
684 , rrdhost_hostname(rpt->host)
685 , rpt->host->machine_guid
703 - , rpt->host->update_every
686 + , rpt->host->rrd_update_every
687 , rpt->host->rrd_history_entries
705 - , storage_engine_name(rpt->host->storage_engine_id)
688 + , rrd_memory_mode_name(rpt->host->rrd_memory_mode)
689 , (rpt->config.health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((rpt->config.health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto")
690 #ifdef ENABLE_HTTPS
691 , (rpt->ssl.conn != NULL) ? " SSL," : ""
@@ -715,7 +698,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
698
699
700 struct plugind cd = {
718 - .update_every = rrdb.default_update_every,
701 + .update_every = default_rrd_update_every,
702 .unsafe = {
703 .spinlock = NETDATA_SPINLOCK_INITIALIZER,
704 .running = true,
streaming/replication.c
+9 -9
@@ -96,7 +96,7 @@ struct replication_query {
96 size_t points_read;
97 size_t points_generated;
98
99 - STORAGE_ENGINE_ID id;
99 + STORAGE_ENGINE_BACKEND backend;
100 struct replication_request *rq;
101
102 size_t dimensions;
@@ -162,7 +162,7 @@ static struct replication_query *replication_query_prepare(
162 }
163 }
164
165 - q->id = st->rrdhost->db[0].id;
165 + q->backend = st->rrdhost->db[0].eng->backend;
166
167 // prepare our array of dimensions
168 size_t count = 0;
@@ -184,7 +184,7 @@ static struct replication_query *replication_query_prepare(
184 d->rda = dictionary_acquired_item_dup(rd_dfe.dict, rd_dfe.item);
185 d->rd = rd;
186
187 - storage_engine_query_init(q->id, rd->tiers[0].db_metric_handle, &d->handle, q->query.after, q->query.before,
187 + storage_engine_query_init(q->backend, rd->tiers[0].db_metric_handle, &d->handle, q->query.after, q->query.before,
188 q->query.locked_data_collection ? STORAGE_PRIORITY_HIGH : STORAGE_PRIORITY_LOW);
189 d->enabled = true;
190 d->skip = false;
@@ -1406,7 +1406,7 @@ static bool replication_execute_request(struct replication_request *rq, bool wor
1406 if(likely(workers))
1407 worker_is_busy(WORKER_JOB_FIND_CHART);
1408
1409 - rq->st = rrdset_find_by_id(rq->sender->host, string2str(rq->chart_id));
1409 + rq->st = rrdset_find(rq->sender->host, string2str(rq->chart_id));
1410 }
1411
1412 if(!rq->st) {
@@ -1601,7 +1601,7 @@ static void verify_all_hosts_charts_are_streaming_now(void) {
1601
1602 size_t errors = 0;
1603 RRDHOST *host;
1604 - dfe_start_read(rrdb.rrdhost_root_index, host)
1604 + dfe_start_read(rrdhost_root_index, host)
1605 errors += verify_host_charts_are_streaming_now(host);
1606 dfe_done(host);
1607
@@ -1695,8 +1695,8 @@ static int replication_pipeline_execute_next(void) {
1695 if(unlikely(!rtp.rqs)) {
1696 rtp.max_requests_ahead = (int)get_netdata_cpus() / 2;
1697
1698 - if(rtp.max_requests_ahead > rrdb.libuv_worker_threads * 2)
1699 - rtp.max_requests_ahead = rrdb.libuv_worker_threads * 2;
1698 + if(rtp.max_requests_ahead > libuv_worker_threads * 2)
1699 + rtp.max_requests_ahead = libuv_worker_threads * 2;
1700
1701 if(rtp.max_requests_ahead < 2)
1702 rtp.max_requests_ahead = 2;
@@ -1722,7 +1722,7 @@ static int replication_pipeline_execute_next(void) {
1722 if(rq->found) {
1723 if (!rq->st) {
1724 worker_is_busy(WORKER_JOB_FIND_CHART);
1725 - rq->st = rrdset_find_by_id(rq->sender->host, string2str(rq->chart_id));
1725 + rq->st = rrdset_find(rq->sender->host, string2str(rq->chart_id));
1726 }
1727
1728 if (rq->st && !rq->q) {
@@ -1896,7 +1896,7 @@ void *replication_thread_main(void *ptr __maybe_unused) {
1896
1897 // statistics
1898 usec_t now_mono_ut = now_monotonic_usec();
1899 - if(unlikely(now_mono_ut - last_now_mono_ut > rrdb.default_update_every * USEC_PER_SEC)) {
1899 + if(unlikely(now_mono_ut - last_now_mono_ut > default_rrd_update_every * USEC_PER_SEC)) {
1900 last_now_mono_ut = now_mono_ut;
1901
1902 worker_is_busy(WORKER_JOB_STATISTICS);
streaming/rrdpush.c
+4 -4
@@ -96,7 +96,7 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) {
96 STREAM_CAP_BINARY |
97 STREAM_CAP_INTERPOLATED |
98 STREAM_HAS_COMPRESSION |
99 - (rrdb.ieee754_doubles ? STREAM_CAP_IEEE754 : 0) |
99 + (ieee754_doubles ? STREAM_CAP_IEEE754 : 0) |
100 (ml_capability ? STREAM_CAP_DATA_WITH_ML : 0) |
101 0;
102 }
@@ -140,7 +140,7 @@ int rrdpush_init() {
140 default_rrdpush_seconds_to_replicate = config_get_number(CONFIG_SECTION_DB, "seconds to replicate", default_rrdpush_seconds_to_replicate);
141 default_rrdpush_replication_step = config_get_number(CONFIG_SECTION_DB, "seconds per replication step", default_rrdpush_replication_step);
142
143 - rrdb.rrdhost_free_orphan_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup orphan hosts after secs", rrdb.rrdhost_free_orphan_time_s);
143 + rrdhost_free_orphan_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup orphan hosts after secs", rrdhost_free_orphan_time_s);
144
145 #ifdef ENABLE_RRDPUSH_COMPRESSION
146 default_rrdpush_compression_enabled = (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM,
@@ -801,7 +801,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
801 rpt->ssl = NETDATA_SSL_UNSET_CONNECTION;
802 #endif
803
804 - rpt->config.update_every = rrdb.default_update_every;
804 + rpt->config.update_every = default_rrd_update_every;
805
806 // parse the parameters and fill rpt and rpt->system_info
807
@@ -1057,7 +1057,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
1057 }
1058 }
1059
1060 - if (strcmp(rpt->machine_guid, rrdb.localhost->machine_guid) == 0) {
1060 + if (strcmp(rpt->machine_guid, localhost->machine_guid) == 0) {
1061
1062 rrdpush_receiver_takeover_web_connection(w, rpt);
1063
streaming/rrdpush.h
+2 -2
@@ -395,7 +395,7 @@ struct receiver_state {
395 } exit;
396
397 struct {
398 - STORAGE_ENGINE_ID storage_engine_id;
398 + RRD_MEMORY_MODE mode;
399 int history;
400 int update_every;
401 int health_enabled; // CONFIG_BOOLEAN_YES, CONFIG_BOOLEAN_NO, CONFIG_BOOLEAN_AUTO
@@ -696,7 +696,7 @@ typedef struct rrdhost_status {
696 struct {
697 RRDHOST_DB_STATUS status;
698 RRDHOST_DB_LIVENESS liveness;
699 - STORAGE_ENGINE_ID storage_engine_id;
699 + RRD_MEMORY_MODE mode;
700 time_t first_time_s;
701 time_t last_time_s;
702 size_t metrics;
streaming/sender.c
+1 -1
@@ -657,7 +657,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
657 , rrdhost_hostname(host)
658 , rrdhost_registry_hostname(host)
659 , host->machine_guid
660 - , rrdb.default_update_every
660 + , default_rrd_update_every
661 , rrdhost_os(host)
662 , rrdhost_timezone(host)
663 , rrdhost_abbrev_timezone(host)
tests/ebpf/sync_tester.c
-2
@@ -1,5 +1,3 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
1 #include <errno.h>
2 #include <string.h>
3 #include <stdio.h>
web/api/badges/web_buffer_svg.c
+3 -3
@@ -961,8 +961,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
961
962 int scale = (scale_str && *scale_str)?str2i(scale_str):100;
963
964 - st = rrdset_find_by_id(host, chart);
965 - if(!st) st = rrdset_find_by_name(host, chart);
964 + st = rrdset_find(host, chart);
965 + if(!st) st = rrdset_find_byname(host, chart);
966 if(!st) {
967 buffer_no_cacheable(w->response.data);
968 buffer_svg(w->response.data, "chart not found", NAN, "", NULL, NULL, -1, scale, 0, -1, -1, NULL, NULL);
@@ -1113,7 +1113,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
1113 ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
1114
1115 // if the collected value is too old, don't calculate its value
1116 - if (rrdset_last_entry_s(st) >= (now_realtime_sec() - (st->update_every * rrdb.gap_when_lost_iterations_above)))
1116 + if (rrdset_last_entry_s(st) >= (now_realtime_sec() - (st->update_every * gap_when_lost_iterations_above)))
1117 ret = rrdset2value_api_v1(st, w->response.data, &n,
1118 (dimensions) ? buffer_tostring(dimensions) : NULL,
1119 points, after, before, group, group_options, 0, options,
web/api/formatters/charts2json.c
+2 -2
@@ -62,9 +62,9 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
62 , get_release_channel()
63 , rrdhost_os(host)
64 , rrdhost_timezone(host)
65 - , host->update_every
65 + , host->rrd_update_every
66 , host->rrd_history_entries
67 - , storage_engine_name(host->storage_engine_id)
67 + , rrd_memory_mode_name(host->rrd_memory_mode)
68 , custom_dashboard_info_js_filename
69 );
70
web/api/formatters/json_wrapper.c
+5 -5
@@ -16,7 +16,7 @@ static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) {
16 buffer_json_array_close(wb);
17
18 buffer_json_member_add_array(wb, "tiers");
19 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) {
19 + for (size_t tier = 0; tier < storage_tiers; tier++) {
20 buffer_json_add_array_item_object(wb);
21 buffer_json_member_add_uint64(wb, "tr", tier);
22 buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s);
@@ -920,7 +920,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
920 buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
921
922 buffer_json_member_add_array(wb, "db_points_per_tier");
923 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++)
923 + for(size_t tier = 0; tier < storage_tiers ; tier++)
924 buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points);
925 buffer_json_array_close(wb);
926
@@ -1269,7 +1269,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
1269 void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) {
1270 buffer_json_member_add_object(wb, "versions");
1271 buffer_json_member_add_uint64(wb, "routing_hard_hash", 1);
1272 - buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdb.rrdhost_root_index));
1272 + buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index));
1273 buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash);
1274 buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash);
1275 buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash);
@@ -1498,7 +1498,7 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
1498
1499 buffer_json_member_add_object(wb, "db");
1500 {
1501 - buffer_json_member_add_uint64(wb, "tiers", rrdb.storage_tiers);
1501 + buffer_json_member_add_uint64(wb, "tiers", storage_tiers);
1502 buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
1503 buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
1504 buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
@@ -1513,7 +1513,7 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
1513 buffer_json_object_close(wb); // dimensions
1514
1515 buffer_json_member_add_array(wb, "per_tier");
1516 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1516 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
1517 buffer_json_add_array_item_object(wb);
1518 buffer_json_member_add_uint64(wb, "tier", tier);
1519 buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries);
web/api/formatters/rrd2json.c
+2 -1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "web/api/web_api_v1.h"
4 +#include "database/storage_engine.h"
5
6 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
7 rrdset2json(st, wb, NULL, NULL, 0);
@@ -99,7 +100,7 @@ int rrdset2value_api_v1(
100 *db_points_read += r->stats.db_points_read;
101
102 if(db_points_per_tier) {
102 - for(size_t t = 0; t < rrdb.storage_tiers ;t++)
103 + for(size_t t = 0; t < storage_tiers ;t++)
104 db_points_per_tier[t] += r->internal.qt->db.tiers[t].points;
105 }
106
web/api/formatters/value/value.c
+1 -1
@@ -131,7 +131,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
131 storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points);
132 }
133
134 - for(size_t t = 0; t < rrdb.storage_tiers ;t++)
134 + for(size_t t = 0; t < storage_tiers ;t++)
135 qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;
136
137 long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
web/api/health/health_cmdapi.c
+3 -1
@@ -1,4 +1,6 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
1 +//
2 +// Created by Christopher on 11/12/18.
3 +//
4
5 #include "health_cmdapi.h"
6
web/api/queries/query.c
+32 -37
@@ -946,7 +946,7 @@ static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) {
946 }
947
948 static size_t query_metric_first_working_tier(QUERY_METRIC *qm) {
949 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
949 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
950
951 // find the db time-range for this tier for all metrics
952 STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
@@ -991,7 +991,7 @@ static long query_plan_points_coverage_weight(time_t db_first_time_s, time_t db_
991 }
992
993 static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
994 - if(unlikely(rrdb.storage_tiers < 2))
994 + if(unlikely(storage_tiers < 2))
995 return 0;
996
997 if(unlikely(after_wanted == before_wanted || points_wanted <= 0))
@@ -1004,7 +1004,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1004 time_t min_first_time_s = 0;
1005 time_t max_last_time_s = 0;
1006
1007 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1007 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
1008 time_t first_time_s = qm->tiers[tier].db_first_time_s;
1009 time_t last_time_s = qm->tiers[tier].db_last_time_s;
1010
@@ -1015,7 +1015,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1015 max_last_time_s = last_time_s;
1016 }
1017
1018 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1018 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
1019
1020 // find the db time-range for this tier for all metrics
1021 STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
@@ -1042,7 +1042,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1042 }
1043
1044 size_t best_tier = 0;
1045 - for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
1045 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
1046 if(qm->tiers[tier].weight >= qm->tiers[best_tier].weight)
1047 best_tier = tier;
1048 }
@@ -1051,7 +1051,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
1051 }
1052
1053 static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
1054 - if(unlikely(rrdb.storage_tiers < 2))
1054 + if(unlikely(storage_tiers < 2))
1055 return 0;
1056
1057 if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) {
@@ -1059,9 +1059,9 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
1059 return 0;
1060 }
1061
1062 - long weight[rrdb.storage_tiers];
1062 + long weight[storage_tiers];
1063
1064 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
1064 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
1065
1066 time_t common_first_time_s = 0;
1067 time_t common_last_time_s = 0;
@@ -1098,7 +1098,7 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
1098 }
1099
1100 size_t best_tier = 0;
1101 - for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
1101 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
1102 if(weight[tier] >= weight[best_tier])
1103 best_tier = tier;
1104 }
@@ -1111,13 +1111,13 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
1111
1112 static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier) {
1113 size_t best_tier;
1114 - if((options & RRDR_OPTION_SELECTED_TIER) && tier < rrdb.storage_tiers)
1114 + if((options & RRDR_OPTION_SELECTED_TIER) && tier < storage_tiers)
1115 best_tier = tier;
1116 else
1117 best_tier = rrddim_find_best_tier_for_timeframe(qt, after_wanted, before_wanted, points_wanted);
1118
1119 // find the db minimum update every for this tier for all metrics
1120 - time_t common_update_every_s = rrdb.default_update_every;
1120 + time_t common_update_every_s = default_rrd_update_every;
1121 for(size_t i = 0, used = qt->query.used; i < used ; i++) {
1122 QUERY_METRIC *qm = query_metric(qt, i);
1123
@@ -1260,8 +1260,8 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
1260 ops->r->internal.qt->db.tiers[tier].queries++;
1261
1262 struct query_metric_tier *tier_ptr = &qm->tiers[tier];
1263 - STORAGE_ENGINE_ID storage_engine_id = query_metric_storage_engine(ops->r->internal.qt, qm, tier);
1264 - storage_engine_query_init(storage_engine_id, tier_ptr->db_metric_handle, &ops->plans[p].handle,
1263 + STORAGE_ENGINE *eng = query_metric_storage_engine(ops->r->internal.qt, qm, tier);
1264 + storage_engine_query_init(eng->backend, tier_ptr->db_metric_handle, &ops->plans[p].handle,
1265 after, before, ops->r->internal.qt->request.priority);
1266
1267 ops->plans[p].initialized = true;
@@ -1354,7 +1354,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1354 bool switch_tiers = true;
1355
1356 if((ops->r->internal.qt->window.options & RRDR_OPTION_SELECTED_TIER)
1357 - && ops->r->internal.qt->window.tier < rrdb.storage_tiers
1357 + && ops->r->internal.qt->window.tier < storage_tiers
1358 && query_metric_is_valid_tier(qm, ops->r->internal.qt->window.tier)) {
1359 selected_tier = ops->r->internal.qt->window.tier;
1360 switch_tiers = false;
@@ -1383,7 +1383,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1383 // check if our selected tier can start the query
1384 if (selected_tier_first_time_s > after_wanted) {
1385 // we need some help from other tiers
1386 - for (size_t tr = (int)selected_tier + 1; tr < rrdb.storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) {
1386 + for (size_t tr = (int)selected_tier + 1; tr < storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) {
1387 if(!query_metric_is_valid_tier(qm, tr))
1388 continue;
1389
@@ -1940,7 +1940,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1940
1941 r->stats.result_points_generated += points_added;
1942 r->stats.db_points_read += ops->db_total_points_read;
1943 - for(size_t tr = 0; tr < rrdb.storage_tiers ; tr++)
1943 + for(size_t tr = 0; tr < storage_tiers ; tr++)
1944 qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
1945 }
1946
@@ -1950,23 +1950,18 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1950 void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
1951
1952 void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {
1953 - if (unlikely(tier >= rrdb.storage_tiers))
1954 - return;
1955 - if (rrdb.storage_tiers_backfill[tier] == RRD_BACKFILL_NONE)
1956 - return;
1953 + if(unlikely(tier >= storage_tiers)) return;
1954 + if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
1955
1956 struct rrddim_tier *t = &rd->tiers[tier];
1959 - if (unlikely(!t))
1960 - return;
1957 + if(unlikely(!t)) return;
1958
1962 - STORAGE_ENGINE_ID storage_engine_id = rd->rrdset->storage_engine_id;
1963 - time_t latest_time_s = storage_engine_latest_time_s(storage_engine_id, t->db_metric_handle);
1964 - time_t granularity = (time_t) rd->rrdset->rrdhost->db[tier].tier_grouping * (time_t)rd->rrdset->update_every;
1959 + time_t latest_time_s = storage_engine_latest_time_s(t->backend, t->db_metric_handle);
1960 + time_t granularity = (time_t)t->tier_grouping * (time_t)rd->rrdset->update_every;
1961 time_t time_diff = now_s - latest_time_s;
1962
1963 // if the user wants only NEW backfilling, and we don't have any data
1968 - if (rrdb.storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_s <= 0)
1969 - return;
1964 + if(storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_s <= 0) return;
1965
1966 // there is really nothing we can do
1967 if(now_s <= latest_time_s || time_diff < granularity) return;
@@ -1975,15 +1970,15 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
1970
1971 // for each lower tier
1972 for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){
1978 - time_t smaller_tier_first_time = storage_engine_oldest_time_s(storage_engine_id, rd->tiers[read_tier].db_metric_handle);
1979 - time_t smaller_tier_last_time = storage_engine_latest_time_s(storage_engine_id, rd->tiers[read_tier].db_metric_handle);
1973 + time_t smaller_tier_first_time = storage_engine_oldest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle);
1974 + time_t smaller_tier_last_time = storage_engine_latest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle);
1975 if(smaller_tier_last_time <= latest_time_s) continue; // it is as bad as we are
1976
1977 long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s;
1978 long before_wanted = smaller_tier_last_time;
1979
1980 struct rrddim_tier *tmp = &rd->tiers[read_tier];
1986 - storage_engine_query_init(storage_engine_id, tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
1981 + storage_engine_query_init(tmp->backend, tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
1982
1983 size_t points_read = 0;
1984
@@ -2144,16 +2139,16 @@ bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now
2139 time_t absolute_minimum_time = now - (10 * 365 * 86400);
2140 time_t absolute_maximum_time = now + (1 * 365 * 86400);
2141
2147 - if (after_requested < absolute_minimum_time && !rrdb.unittest_running)
2142 + if (after_requested < absolute_minimum_time && !unittest_running)
2143 after_requested = absolute_minimum_time;
2144
2150 - if (after_requested > absolute_maximum_time && !rrdb.unittest_running)
2145 + if (after_requested > absolute_maximum_time && !unittest_running)
2146 after_requested = absolute_maximum_time;
2147
2153 - if (before_requested < absolute_minimum_time && !rrdb.unittest_running)
2148 + if (before_requested < absolute_minimum_time && !unittest_running)
2149 before_requested = absolute_minimum_time;
2150
2156 - if (before_requested > absolute_maximum_time && !rrdb.unittest_running)
2151 + if (before_requested > absolute_maximum_time && !unittest_running)
2152 before_requested = absolute_maximum_time;
2153
2154 *before = before_requested;
@@ -2291,7 +2286,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
2286 rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
2287 query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
2288
2294 - if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && rrdb.storage_tiers > 1) {
2289 + if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
2290 update_every = rrdset_find_natural_update_every_for_timeframe(
2291 qt, after_wanted, before_wanted, points_wanted, options, tier);
2292
@@ -2354,7 +2349,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
2349 query_debug_log(":max points_wanted %zu", points_wanted);
2350 }
2351
2357 - if(points_wanted > 86400 && !rrdb.unittest_running) {
2352 + if(points_wanted > 86400 && !unittest_running) {
2353 points_wanted = 86400;
2354 query_debug_log(":absolute max points_wanted %zu", points_wanted);
2355 }
@@ -3550,7 +3545,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3545 if(qt->query.used)
3546 ops = onewayalloc_callocz(owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
3547
3553 - size_t capacity = rrdb.libuv_worker_threads * 10;
3548 + size_t capacity = libuv_worker_threads * 10;
3549 size_t max_queries_to_prepare = (qt->query.used > (capacity - 1)) ? (capacity - 1) : qt->query.used;
3550 size_t queries_prepared = 0;
3551 while(queries_prepared < max_queries_to_prepare) {
web/api/queries/weights.c
+4 -4
@@ -149,7 +149,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
149
150 buffer_json_member_add_array(wb, "db_points_per_tier");
151 {
152 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
152 + for (size_t tier = 0; tier < storage_tiers; tier++)
153 buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]);
154 }
155 buffer_json_array_close(wb);
@@ -461,7 +461,7 @@ static void results_header_to_json_v2(DICTIONARY *results __maybe_unused, BUFFER
461
462 buffer_json_member_add_array(wb, "db_points_per_tier");
463 {
464 - for (size_t tier = 0; tier < rrdb.storage_tiers; tier++)
464 + for (size_t tier = 0; tier < storage_tiers; tier++)
465 buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]);
466 }
467 buffer_json_array_close(wb);
@@ -1288,7 +1288,7 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
1288 stats->db_queries++;
1289 stats->result_points += r->stats.result_points_generated;
1290 stats->db_points += r->stats.db_points_read;
1291 - for(size_t tr = 0; tr < rrdb.storage_tiers ; tr++)
1291 + for(size_t tr = 0; tr < storage_tiers ; tr++)
1292 stats->db_points_per_tier[tr] += r->internal.qt->db.tiers[tr].points;
1293
1294 if(r->d != 1 || r->internal.qt->query.used != 1) {
@@ -1395,7 +1395,7 @@ static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats, si
1395 stats->db_queries += queries;
1396 stats->result_points += qv->result_points;
1397 stats->db_points += qv->points_read;
1398 - for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++)
1398 + for(size_t tier = 0; tier < storage_tiers ; tier++)
1399 stats->db_points_per_tier[tier] += qv->storage_points_per_tier[tier];
1400 }
1401
web/api/web_api.c
+1 -1
@@ -196,7 +196,7 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur
196
197 else if(!strcmp(name, "tier")) {
198 tier = str2ul(value);
199 - if(tier < rrdb.storage_tiers)
199 + if(tier < storage_tiers)
200 options |= RRDR_OPTION_SELECTED_TIER;
201 else
202 tier = 0;
web/api/web_api_v1.c
+77 -15
@@ -494,8 +494,8 @@ inline int web_client_api_request_single_chart(RRDHOST *host, struct web_client
494 goto cleanup;
495 }
496
497 - RRDSET *st = rrdset_find_by_id(host, chart);
498 - if(!st) st = rrdset_find_by_name(host, chart);
497 + RRDSET *st = rrdset_find(host, chart);
498 + if(!st) st = rrdset_find_byname(host, chart);
499 if(!st) {
500 buffer_strcat(w->response.data, "Chart is not found: ");
501 buffer_strcat_htmlescape(w->response.data, chart);
@@ -762,7 +762,7 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
762 }
763 else if(!strcmp(name, "tier")) {
764 tier = str2ul(value);
765 - if(tier < rrdb.storage_tiers)
765 + if(tier < storage_tiers)
766 options |= RRDR_OPTION_SELECTED_TIER;
767 else
768 tier = 0;
@@ -788,8 +788,8 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
788
789 if(chart && !context) {
790 // check if this is a specific chart
791 - st = rrdset_find_by_id(host, chart);
792 - if (!st) st = rrdset_find_by_name(host, chart);
791 + st = rrdset_find(host, chart);
792 + if (!st) st = rrdset_find_byname(host, chart);
793 }
794
795 long long before = (before_str && *before_str)?str2l(before_str):0;
@@ -1121,8 +1121,8 @@ static inline void web_client_api_request_v1_info_mirrored_hosts_status(BUFFER *
1121 buffer_json_add_array_item_object(wb);
1122
1123 buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
1124 - buffer_json_member_add_uint64(wb, "hops", host->system_info ? host->system_info->hops : (host == rrdb.localhost) ? 0 : 1);
1125 - buffer_json_member_add_boolean(wb, "reachable", (host == rrdb.localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)));
1124 + buffer_json_member_add_uint64(wb, "hops", host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
1125 + buffer_json_member_add_boolean(wb, "reachable", (host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)));
1126
1127 buffer_json_member_add_string(wb, "guid", host->machine_guid);
1128 buffer_json_member_add_uuid(wb, "node_id", host->node_id);
@@ -1145,12 +1145,12 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
1145
1146 buffer_json_member_add_array(wb, "mirrored_hosts_status");
1147 rrdhost_foreach_read(host) {
1148 - if ((host == rrdb.localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1148 + if ((host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1149 web_client_api_request_v1_info_mirrored_hosts_status(wb, host);
1150 }
1151 }
1152 rrdhost_foreach_read(host) {
1153 - if ((host != rrdb.localhost && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1153 + if ((host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) {
1154 web_client_api_request_v1_info_mirrored_hosts_status(wb, host);
1155 }
1156 }
@@ -1262,10 +1262,10 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
1262 buffer_json_member_add_boolean(wb, "aclk-available", false);
1263 #endif
1264
1265 - buffer_json_member_add_string(wb, "memory-mode", storage_engine_name(host->storage_engine_id));
1265 + buffer_json_member_add_string(wb, "memory-mode", rrd_memory_mode_name(host->rrd_memory_mode));
1266 #ifdef ENABLE_DBENGINE
1267 - buffer_json_member_add_uint64(wb, "multidb-disk-quota", rrdb.default_multidb_disk_quota_mb);
1268 - buffer_json_member_add_uint64(wb, "page-cache-size", rrdb.default_rrdeng_page_cache_mb);
1267 + buffer_json_member_add_uint64(wb, "multidb-disk-quota", default_multidb_disk_quota_mb);
1268 + buffer_json_member_add_uint64(wb, "page-cache-size", default_rrdeng_page_cache_mb);
1269 #endif // ENABLE_DBENGINE
1270 buffer_json_member_add_boolean(wb, "web-enabled", web_server_mode != WEB_SERVER_MODE_NONE);
1271 buffer_json_member_add_boolean(wb, "stream-enabled", default_rrdpush_enabled);
@@ -1436,6 +1436,68 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1436 return HTTP_RESP_NOT_FOUND;
1437 }
1438 #else
1439 +static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, size_t tier) {
1440 + RRDENG_SIZE_STATS stats = rrdeng_size_statistics(multidb_ctx[tier]);
1441 +
1442 + buffer_sprintf(wb,
1443 + "\n\t\t\"default_granularity_secs\":%zu"
1444 + ",\n\t\t\"sizeof_datafile\":%zu"
1445 + ",\n\t\t\"sizeof_page_in_cache\":%zu"
1446 + ",\n\t\t\"sizeof_point_data\":%zu"
1447 + ",\n\t\t\"sizeof_page_data\":%zu"
1448 + ",\n\t\t\"pages_per_extent\":%zu"
1449 + ",\n\t\t\"datafiles\":%zu"
1450 + ",\n\t\t\"extents\":%zu"
1451 + ",\n\t\t\"extents_pages\":%zu"
1452 + ",\n\t\t\"points\":%zu"
1453 + ",\n\t\t\"metrics\":%zu"
1454 + ",\n\t\t\"metrics_pages\":%zu"
1455 + ",\n\t\t\"extents_compressed_bytes\":%zu"
1456 + ",\n\t\t\"pages_uncompressed_bytes\":%zu"
1457 + ",\n\t\t\"pages_duration_secs\":%lld"
1458 + ",\n\t\t\"single_point_pages\":%zu"
1459 + ",\n\t\t\"first_t\":%ld"
1460 + ",\n\t\t\"last_t\":%ld"
1461 + ",\n\t\t\"database_retention_secs\":%lld"
1462 + ",\n\t\t\"average_compression_savings\":%0.2f"
1463 + ",\n\t\t\"average_point_duration_secs\":%0.2f"
1464 + ",\n\t\t\"average_metric_retention_secs\":%0.2f"
1465 + ",\n\t\t\"ephemeral_metrics_per_day_percent\":%0.2f"
1466 + ",\n\t\t\"average_page_size_bytes\":%0.2f"
1467 + ",\n\t\t\"estimated_concurrently_collected_metrics\":%zu"
1468 + ",\n\t\t\"currently_collected_metrics\":%zu"
1469 + ",\n\t\t\"disk_space\":%zu"
1470 + ",\n\t\t\"max_disk_space\":%zu"
1471 + , stats.default_granularity_secs
1472 + , stats.sizeof_datafile
1473 + , stats.sizeof_page_in_cache
1474 + , stats.sizeof_point_data
1475 + , stats.sizeof_page_data
1476 + , stats.pages_per_extent
1477 + , stats.datafiles
1478 + , stats.extents
1479 + , stats.extents_pages
1480 + , stats.points
1481 + , stats.metrics
1482 + , stats.metrics_pages
1483 + , stats.extents_compressed_bytes
1484 + , stats.pages_uncompressed_bytes
1485 + , (long long)stats.pages_duration_secs
1486 + , stats.single_point_pages
1487 + , stats.first_time_s
1488 + , stats.last_time_s
1489 + , (long long)stats.database_retention_secs
1490 + , stats.average_compression_savings
1491 + , stats.average_point_duration_secs
1492 + , stats.average_metric_retention_secs
1493 + , stats.ephemeral_metrics_per_day_percent
1494 + , stats.average_page_size_bytes
1495 + , stats.estimated_concurrently_collected_metrics
1496 + , stats.currently_collected_metrics
1497 + , stats.disk_space
1498 + , stats.max_disk_space
1499 + );
1500 +}
1501 int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) {
1502 if (!netdata_ready)
1503 return HTTP_RESP_BACKEND_FETCH_FAILED;
@@ -1443,7 +1505,7 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1505 BUFFER *wb = w->response.data;
1506 buffer_flush(wb);
1507
1446 - if(!rrdb.dbengine_enabled) {
1508 + if(!dbengine_enabled) {
1509 buffer_strcat(wb, "dbengine is not enabled");
1510 return HTTP_RESP_NOT_FOUND;
1511 }
@@ -1451,9 +1513,9 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1513 wb->content_type = CT_APPLICATION_JSON;
1514 buffer_no_cacheable(wb);
1515 buffer_strcat(wb, "{");
1454 - for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
1516 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
1517 buffer_sprintf(wb, "%s\n\t\"tier%zu\": {", tier?",":"", tier);
1456 - rrdeng_size_statistics(rrdb.multidb_ctx[tier], wb);
1518 + web_client_api_v1_dbengine_stats_for_tier(wb, tier);
1519 buffer_strcat(wb, "\n\t}");
1520 }
1521 buffer_strcat(wb, "\n}");
web/api/web_api_v2.c
+5 -5
@@ -95,7 +95,7 @@ static bool verify_agent_uuids(const char *machine_guid, const char *node_id, co
95 if(!machine_guid || !node_id || !claim_id)
96 return false;
97
98 - if(strcmp(machine_guid, rrdb.localhost->machine_guid) != 0)
98 + if(strcmp(machine_guid, localhost->machine_guid) != 0)
99 return false;
100
101 char *agent_claim_id = get_agent_claimid();
@@ -103,11 +103,11 @@ static bool verify_agent_uuids(const char *machine_guid, const char *node_id, co
103 bool not_verified = (!agent_claim_id || strcmp(claim_id, agent_claim_id) != 0);
104 freez(agent_claim_id);
105
106 - if (not_verified || !rrdb.localhost->node_id)
106 + if(not_verified || !localhost->node_id)
107 return false;
108
109 char buf[UUID_STR_LEN];
110 - uuid_unparse_lower(*rrdb.localhost->node_id, buf);
110 + uuid_unparse_lower(*localhost->node_id, buf);
111
112 if(strcmp(node_id, buf) != 0)
113 return false;
@@ -193,7 +193,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
193 BUFFER *wb = w->response.data;
194 buffer_flush(wb);
195 buffer_json_initialize(wb, "\"", "\"", 0, true, false);
196 - buffer_json_member_add_string(wb, "mg", rrdb.localhost->machine_guid);
196 + buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
197 buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
198 buffer_json_member_add_uuid(wb, "token", &uuid);
199 buffer_json_member_add_time_t(wb, "expiration", expires_s);
@@ -521,7 +521,7 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
521
522 if(tier_str && *tier_str) {
523 tier = str2ul(tier_str);
524 - if(tier < rrdb.storage_tiers)
524 + if(tier < storage_tiers)
525 options |= RRDR_OPTION_SELECTED_TIER;
526 else
527 tier = 0;
web/server/h2o/http_server.c
+2 -2
@@ -235,7 +235,7 @@ static inline int _netdata_uberhandler(h2o_req_t *req, RRDHOST **host)
235 static int netdata_uberhandler(h2o_handler_t *self, h2o_req_t *req)
236 {
237 UNUSED(self);
238 - RRDHOST *host = rrdb.localhost;
238 + RRDHOST *host = localhost;
239
240 int ret = _netdata_uberhandler(req, &host);
241
@@ -249,7 +249,7 @@ static int netdata_uberhandler(h2o_handler_t *self, h2o_req_t *req)
249 ", response: %d",
250 PRINTF_H2O_IOVEC(&req->method),
251 PRINTF_H2O_IOVEC(&req->input.path),
252 - host == rrdb.localhost ? "localhost" : host_uuid_str,
252 + host == localhost ? "localhost" : host_uuid_str,
253 req->res.status);
254 } else {
255 netdata_log_access("HTTPD %d"
web/server/static/static-threaded.c
+1 -1
@@ -438,7 +438,7 @@ void *socket_listen_main_static_threaded_worker(void *ptr) {
438 , NULL
439 , web_client_first_request_timeout
440 , web_client_timeout
441 - , rrdb.default_update_every * 1000 // timer_milliseconds
441 + , default_rrd_update_every * 1000 // timer_milliseconds
442 , ptr // timer_data
443 , worker_private->max_sockets
444 );
web/server/web_client.c
+12 -6
@@ -685,6 +685,12 @@ void buffer_data_options2string(BUFFER *wb, uint32_t options) {
685 }
686
687 static inline int check_host_and_call(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) {
688 + //if(unlikely(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) {
689 + // buffer_flush(w->response.data);
690 + // buffer_strcat(w->response.data, "This host does not maintain a database");
691 + // return HTTP_RESP_BAD_REQUEST;
692 + //}
693 +
694 return func(host, w, url);
695 }
696
@@ -1468,7 +1474,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1474 hash_localhost = simple_hash("localhost");
1475 }
1476
1471 - if(host != rrdb.localhost) {
1477 + if(host != localhost) {
1478 buffer_flush(w->response.data);
1479 buffer_strcat(w->response.data, "Nesting of hosts is not allowed.");
1480 return HTTP_RESP_BAD_REQUEST;
@@ -1479,7 +1485,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1485 netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for host with name '%s'.", w->id, tok);
1486
1487 if(nodeid) {
1482 - host = rrdhost_find_by_node_id(tok);
1488 + host = find_host_by_node_id(tok);
1489 if(!host) {
1490 host = rrdhost_find_by_hostname(tok);
1491 if (!host)
@@ -1491,7 +1497,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1497 if(!host) {
1498 host = rrdhost_find_by_guid(tok);
1499 if (!host)
1494 - host = rrdhost_find_by_node_id(tok);
1500 + host = find_host_by_node_id(tok);
1501 }
1502 }
1503
@@ -1670,8 +1676,8 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1676 netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1677
1678 // do we have such a data set?
1673 - RRDSET *st = rrdset_find_by_name(host, tok);
1674 - if(!st) st = rrdset_find_by_id(host, tok);
1679 + RRDSET *st = rrdset_find_byname(host, tok);
1680 + if(!st) st = rrdset_find(host, tok);
1681 if(!st) {
1682 w->response.data->content_type = CT_TEXT_HTML;
1683 buffer_strcat(w->response.data, "Chart is not found: ");
@@ -1796,7 +1802,7 @@ void web_client_process_request(struct web_client *w) {
1802 }
1803 }
1804
1799 - w->response.code = (short)web_client_process_url(rrdb.localhost, w, path);
1805 + w->response.code = (short)web_client_process_url(localhost, w, path);
1806 break;
1807 }
1808 break;