@cryptotaxi247 / netdata-1 / commits / e6cef2707

split rrdhost/rrdset/rrddim and rrd.h (#19399)

split files - no functional changes

Costa Tsaousis committed Jan 15, 2025 at 10:13 UTC e6cef27076189b67271bdf4465b15f9484506e1d
48 files changed +3787 -3612
CMakeLists.txt
+19 -4
@@ -1448,8 +1448,8 @@ set(RRD_PLUGIN_FILES
1448 src/database/rrd.c
1449 src/database/rrd.h
1450 src/database/rrdset.c
1451 - src/database/storage_engine.c
1452 - src/database/storage_engine.h
1451 + src/database/storage-engine.c
1452 + src/database/storage-engine.h
1453 src/database/ram/rrddim_mem.c
1454 src/database/ram/rrddim_mem.h
1455 src/database/sqlite/sqlite_metadata.c
@@ -1486,6 +1486,23 @@ set(RRD_PLUGIN_FILES
1486 src/database/rrdhost-system-info.c
1487 src/database/rrdhost-system-info.h
1488 src/database/contexts/contexts-loading.c
1489 + src/database/rrdset-index-id.c
1490 + src/database/rrdset-index-id.h
1491 + src/database/rrdset-index-name.c
1492 + src/database/rrdset-index-name.h
1493 + src/database/rrdset-slots.c
1494 + src/database/rrdset-slots.h
1495 + src/database/rrdset-collection.c
1496 + src/database/rrdset-collection.h
1497 + src/database/rrdset.h
1498 + src/database/rrdhost.h
1499 + src/database/rrdhost-status.c
1500 + src/database/rrdhost-status.h
1501 + src/database/rrddim.h
1502 + src/database/rrddim-backfill.c
1503 + src/database/rrddim-backfill.h
1504 + src/database/rrddim-collection.c
1505 + src/database/rrddim-collection.h
1506 )
1507
1508 if(ENABLE_DBENGINE)
@@ -1592,8 +1609,6 @@ set(STREAMING_PLUGIN_FILES
1609 src/streaming/stream-conf.h
1610 src/streaming/stream-handshake.h
1611 src/streaming/stream-parents.h
1595 - src/streaming/rrdhost-status.c
1596 - src/streaming/rrdhost-status.h
1612 src/streaming/stream-sender-api.c
1613 src/streaming/stream-receiver-internals.h
1614 src/streaming/stream-receiver-api.c
src/daemon/config/netdata-conf-db.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "netdata-conf-db.h"
4 +#include "daemon/common.h"
5
6 int default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
7
src/daemon/config/netdata-conf-directories.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "netdata-conf-directories.h"
4 +#include "daemon/common.h"
5
6 static const char *get_varlib_subdir_from_config(const char *prefix, const char *dir) {
7 char filename[FILENAME_MAX + 1];
src/daemon/config/netdata-conf-global.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "netdata-conf-global.h"
4 +#include "daemon/common.h"
5
6 size_t netdata_conf_cpus(void) {
7 static size_t processors = 0;
src/daemon/config/netdata-conf-logs.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "netdata-conf-logs.h"
4 +#include "daemon/common.h"
5
6 static void debug_flags_initialize(void) {
7 // --------------------------------------------------------------------
src/daemon/config/netdata-conf.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "netdata-conf.h"
4 +#include "daemon/common.h"
5
6 struct config netdata_config = APPCONFIG_INITIALIZER;
7
src/daemon/config/netdata-conf.h
-2
@@ -17,6 +17,4 @@ bool netdata_conf_load(char *filename, char overwrite_used, const char **user);
17 #include "netdata-conf-web.h"
18 #include "netdata-conf-cloud.h"
19
20 -#include "daemon/common.h"
21 -
20 #endif //NETDATA_DAEMON_NETDATA_CONF_H
src/database/contexts/internal.h
+1 -1
@@ -8,7 +8,7 @@
8 #include "../../aclk/schema-wrappers/context.h"
9 #include "../../aclk/aclk_contexts_api.h"
10 #include "../../aclk/aclk.h"
11 -#include "../storage_engine.h"
11 +#include "../storage-engine.h"
12
13 #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
14 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
src/database/contexts/query_target.c
+1 -1
@@ -210,7 +210,7 @@ typedef struct query_target_locals {
210 QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
211 } QUERY_TARGET_LOCALS;
212
213 -struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
213 +struct storage *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
214 QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
215 return qn->rrdhost->db[tier].eng;
216 }
src/database/contexts/rrdcontext.h
+1 -1
@@ -582,7 +582,7 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
582 return rrdmetric_acquired_name(qd->rma);
583 }
584
585 -struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
585 +struct storage *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
586
587 STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version);
588 STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version);
src/database/engine/pagecache.c
-1
@@ -1,5 +1,4 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
2
3 #include "rrdengine.h"
4
src/database/engine/pdc.c
+1 -1
@@ -1,5 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
2 +
3 #include "pdc.h"
4 #include "dbengine-compression.h"
5
src/database/engine/rrdengine.c
-1
@@ -1,5 +1,4 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS
2
3 #include "rrdengine.h"
4 #include "pdc.h"
src/database/rrd.c
+125 -33
@@ -1,8 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 -#define NETDATA_RRD_INTERNALS 1
2
3 +#define RRDHOST_INTERNALS
4 #include "rrd.h"
5 -#include "storage_engine.h"
5
6 // ----------------------------------------------------------------------------
7 // globals
@@ -59,50 +58,143 @@ const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) {
58 }
59 }
60
61 +// ----------------------------------------------------------------------------
62 +// RRD - string management
63 +
64 +STRING *rrd_string_strdupz(const char *s) {
65 + if(unlikely(!s || !*s)) return string_strdupz(s);
66 +
67 + char *tmp = strdupz(s);
68 + json_fix_string(tmp);
69 + STRING *ret = string_strdupz(tmp);
70 + freez(tmp);
71 + return ret;
72 +}
73
74 // ----------------------------------------------------------------------------
64 -// RRD - chart types
75
66 -inline RRDSET_TYPE rrdset_type_id(const char *name) {
67 - if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))
68 - return RRDSET_TYPE_AREA;
76 +inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
77 + if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
78 + if(mode == RRD_MEMORY_MODE_NONE) return 5;
79
70 - else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0))
71 - return RRDSET_TYPE_STACKED;
80 + if(entries < 5) entries = 5;
81 + if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
82
73 - else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0))
74 - return RRDSET_TYPE_HEATMAP;
83 + if(mode == RRD_MEMORY_MODE_RAM) {
84 + long header_size = 0;
85 +
86 + long page = (long)sysconf(_SC_PAGESIZE);
87 + long size = (long)(header_size + entries * sizeof(storage_number));
88 + if (unlikely(size % page)) {
89 + size -= (size % page);
90 + size += page;
91 +
92 + long n = (long)((size - header_size) / sizeof(storage_number));
93 + return n;
94 + }
95 + }
96
76 - else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0))
77 - return RRDSET_TYPE_LINE;
97 + return entries;
98 }
99
80 -const char *rrdset_type_name(RRDSET_TYPE chart_type) {
81 - switch(chart_type) {
82 - case RRDSET_TYPE_LINE:
83 - default:
84 - return RRDSET_TYPE_LINE_NAME;
100 +void api_v1_management_init(void);
101 +int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
102 + rrdhost_init();
103
86 - case RRDSET_TYPE_AREA:
87 - return RRDSET_TYPE_AREA_NAME;
104 + if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
105 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
106 + set_late_analytics_variables(system_info);
107 + fatal("Failed to initialize SQLite");
108 + }
109
89 - case RRDSET_TYPE_STACKED:
90 - return RRDSET_TYPE_STACKED_NAME;
110 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
111 + "Skipping SQLITE metadata initialization since memory mode is not dbengine");
112 + }
113
92 - case RRDSET_TYPE_HEATMAP:
93 - return RRDSET_TYPE_HEATMAP_NAME;
114 + if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
115 + error_report("Failed to initialize context metadata database");
116 }
95 -}
117
97 -// ----------------------------------------------------------------------------
98 -// RRD - string management
118 + if (unlikely(unittest)) {
119 + dbengine_enabled = true;
120 + }
121 + else {
122 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) {
123 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
124 + "DBENGINE: Initializing ...");
125 +
126 + netdata_conf_dbengine_init(hostname);
127 + }
128 + else
129 + nd_profile.storage_tiers = 1;
130 +
131 + if (!dbengine_enabled) {
132 + if (nd_profile.storage_tiers > 1) {
133 + nd_log(NDLS_DAEMON, NDLP_WARNING,
134 + "dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
135 + nd_profile.storage_tiers);
136 +
137 + nd_profile.storage_tiers = 1;
138 + }
139 +
140 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
141 + nd_log(NDLS_DAEMON, NDLP_WARNING,
142 + "dbengine is not enabled, but it has been given as the default db mode. "
143 + "Resetting db mode to alloc");
144 +
145 + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
146 + }
147 + }
148 + }
149
100 -STRING *rrd_string_strdupz(const char *s) {
101 - if(unlikely(!s || !*s)) return string_strdupz(s);
150 + if(!unittest)
151 + metadata_sync_init();
152 +
153 + localhost = rrdhost_create(
154 + hostname
155 + , registry_get_this_machine_hostname()
156 + , registry_get_this_machine_guid()
157 + , os_type
158 + , netdata_configured_timezone
159 + , netdata_configured_abbrev_timezone
160 + , netdata_configured_utc_offset
161 + , program_name
162 + , NETDATA_VERSION
163 + ,
164 + nd_profile.update_every, default_rrd_history_entries
165 + , default_rrd_memory_mode
166 + , health_plugin_enabled()
167 + , stream_send.enabled
168 + , stream_send.parents.destination
169 + , stream_send.api_key
170 + , stream_send.send_charts_matching
171 + , stream_receive.replication.enabled
172 + , stream_receive.replication.period
173 + , stream_receive.replication.step
174 + , system_info
175 + , 1
176 + , 0
177 + );
178 +
179 + if (unlikely(!localhost))
180 + return 1;
181 +
182 + rrdhost_flag_set(localhost, RRDHOST_FLAG_COLLECTOR_ONLINE);
183 + object_state_activate(&localhost->state_id);
184 +
185 + ml_host_start(localhost);
186 + dyncfg_host_init(localhost);
187 +
188 + if(!unittest)
189 + health_plugin_init();
190 +
191 + global_functions_add();
192 +
193 + if (likely(system_info)) {
194 + detect_machine_guid_change(&localhost->host_id.uuid);
195 + sql_aclk_sync_init();
196 + api_v1_management_init();
197 + }
198
103 - char *tmp = strdupz(s);
104 - json_fix_string(tmp);
105 - STRING *ret = string_strdupz(tmp);
106 - freez(tmp);
107 - return ret;
199 + return 0;
200 }
src/database/rrd.h
+78 -1537
@@ -8,36 +8,18 @@ extern "C" {
8 #endif
9
10 #include "libnetdata/libnetdata.h"
11 -#include "rrd-database-mode.h"
12 -#include "streaming/stream-traffic-types.h"
13 -#include "streaming/stream-sender-commit.h"
14 -#include "streaming/stream-replication-tracking.h"
15 -#include "health/health-alert-log.h"
16 -#include "rrdhost-system-info.h"
17 -
18 -// non-existing structs instead of voids
19 -// to enable type checking at compile time
20 -typedef struct storage_instance STORAGE_INSTANCE;
21 -typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
22 -typedef struct storage_alignment STORAGE_METRICS_GROUP;
23 -
24 -// forward typedefs
25 -typedef struct rrdhost RRDHOST;
26 -typedef struct rrddim RRDDIM;
27 -typedef struct rrdset RRDSET;
28 -typedef struct rrdcalc RRDCALC;
29 -typedef struct alarm_entry ALARM_ENTRY;
11
31 -typedef struct rrdvar_acquired RRDVAR_ACQUIRED;
32 -typedef struct rrdcalc_acquired RRDCALC_ACQUIRED;
12 +// --------------------------------------------------------------------------------------------------------------------
13
34 -typedef struct rrdhost_acquired RRDHOST_ACQUIRED;
35 -typedef struct rrdset_acquired RRDSET_ACQUIRED;
36 -typedef struct rrddim_acquired RRDDIM_ACQUIRED;
14 +struct ml_metrics_statistics {
15 + size_t anomalous;
16 + size_t normal;
17 + size_t trained;
18 + size_t pending;
19 + size_t silenced;
20 +};
21
38 -typedef struct ml_host rrd_ml_host_t;
39 -typedef struct ml_chart rrd_ml_chart_t;
40 -typedef struct ml_dimension rrd_ml_dimension_t;
22 +// --------------------------------------------------------------------------------------------------------------------
23
24 typedef enum __attribute__ ((__packed__)) {
25 QUERY_SOURCE_UNKNOWN = 0,
@@ -49,161 +31,7 @@ typedef enum __attribute__ ((__packed__)) {
31 QUERY_SOURCE_UNITTEST,
32 } QUERY_SOURCE;
33
52 -typedef enum __attribute__ ((__packed__)) storage_priority {
53 - STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
54 - STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
55 -
56 - // query priorities
57 - STORAGE_PRIORITY_HIGH,
58 - STORAGE_PRIORITY_NORMAL,
59 - STORAGE_PRIORITY_LOW,
60 - STORAGE_PRIORITY_BEST_EFFORT,
61 -
62 - // synchronous query, not to be dispatched to workers or queued
63 - STORAGE_PRIORITY_SYNCHRONOUS,
64 - STORAGE_PRIORITY_SYNCHRONOUS_FIRST,
65 -
66 - STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
67 -} STORAGE_PRIORITY;
68 -
69 -// forward declarations
70 -struct rrddim_tier;
71 -
72 -#ifdef ENABLE_DBENGINE
73 -struct rrdengine_instance;
74 -#endif
75 -
76 -// ----------------------------------------------------------------------------
77 -
78 -struct ml_metrics_statistics {
79 - size_t anomalous;
80 - size_t normal;
81 - size_t trained;
82 - size_t pending;
83 - size_t silenced;
84 -};
85 -
86 -
87 -// use this for configuration flags, not for state control
88 -// flags are set/unset in a manner that is not thread safe
89 -// and may lead to missing information.
90 -typedef enum __attribute__ ((__packed__)) rrdset_flags {
91 - RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
92 - RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete
93 - RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases
94 - RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases
95 -
96 - RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming)
97 - RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming)
98 -
99 - RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation
100 - RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
101 - RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous
102 - RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting
103 - RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
104 - RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions
105 -
106 - RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored
107 - RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts.
108 - RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id
109 - RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name
110 -
111 - RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18),
112 -
113 - RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress
114 - RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication
115 - RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress
116 - RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication
117 -
118 - RRDSET_FLAG_BACKFILLED_HIGH_TIERS = (1 << 23), // we have backfilled this chart
119 -
120 - RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 24), // a custom variable has been updated and needs to be exposed to parent
121 -
122 - RRDSET_FLAG_COLLECTION_FINISHED = (1 << 25), // when set, data collection is not available for this chart
123 -
124 - RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 26), // this chart has at least one rrdcal linked
125 -} RRDSET_FLAGS;
126 -
127 -#include "daemon/common.h"
128 -#include "web/api/queries/query.h"
129 -#include "web/api/queries/rrdr.h"
130 -#include "health/rrdvar.h"
131 -#include "health/rrdcalc.h"
132 -#include "rrdlabels.h"
133 -#include "streaming/stream-capabilities.h"
134 -#include "streaming/stream-path.h"
135 -#include "streaming/stream.h"
136 -//#include "aclk/aclk_rrdhost_state.h"
137 -#include "sqlite/sqlite_health.h"
138 -
139 -typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
140 -
141 -typedef enum __attribute__ ((__packed__)) {
142 - STORAGE_ENGINE_BACKEND_RRDDIM = 1,
143 - STORAGE_ENGINE_BACKEND_DBENGINE = 2,
144 -} STORAGE_ENGINE_BACKEND;
145 -
146 -#define is_valid_backend(backend) ((backend) >= STORAGE_ENGINE_BACKEND_RRDDIM && (backend) <= STORAGE_ENGINE_BACKEND_DBENGINE)
147 -
148 -// iterator state for RRD dimension data queries
149 -struct storage_engine_query_handle {
150 - time_t start_time_s;
151 - time_t end_time_s;
152 - STORAGE_PRIORITY priority;
153 - STORAGE_ENGINE_BACKEND seb;
154 - STORAGE_QUERY_HANDLE *handle;
155 -};
156 -
157 -// ----------------------------------------------------------------------------
158 -// chart types
159 -
160 -typedef enum __attribute__ ((__packed__)) rrdset_type {
161 - RRDSET_TYPE_LINE = 0,
162 - RRDSET_TYPE_AREA = 1,
163 - RRDSET_TYPE_STACKED = 2,
164 - RRDSET_TYPE_HEATMAP = 3,
165 -} RRDSET_TYPE;
166 -
167 -#define RRDSET_TYPE_LINE_NAME "line"
168 -#define RRDSET_TYPE_AREA_NAME "area"
169 -#define RRDSET_TYPE_STACKED_NAME "stacked"
170 -#define RRDSET_TYPE_HEATMAP_NAME "heatmap"
171 -
172 -RRDSET_TYPE rrdset_type_id(const char *name);
173 -const char *rrdset_type_name(RRDSET_TYPE chart_type);
174 -
175 -#include "contexts/rrdcontext.h"
176 -
177 -typedef enum __attribute__ ((__packed__)) {
178 - RRD_BACKFILL_NONE = 0,
179 - RRD_BACKFILL_FULL,
180 - RRD_BACKFILL_NEW
181 -} RRD_BACKFILL;
182 -
183 -#define UPDATE_EVERY_MIN 1
184 -#define UPDATE_EVERY_MAX 3600
185 -
186 -#define RRD_DEFAULT_HISTORY_ENTRIES 3600
187 -#define RRD_HISTORY_ENTRIES_MAX (86400*365)
188 -
189 -#if defined(ENV32BIT)
190 -#define MIN_LIBUV_WORKER_THREADS 8
191 -#define MAX_LIBUV_WORKER_THREADS 128
192 -#define RESERVED_LIBUV_WORKER_THREADS 3
193 -#else
194 -#define MIN_LIBUV_WORKER_THREADS 16
195 -#define MAX_LIBUV_WORKER_THREADS 1024
196 -#define RESERVED_LIBUV_WORKER_THREADS 6
197 -#endif
198 -
199 -extern int libuv_worker_threads;
200 -extern bool ieee754_doubles;
201 -
202 -#define RRD_ID_LENGTH_MAX 1200
203 -
204 -typedef long long total_number;
205 -
206 -// ----------------------------------------------------------------------------
34 +// --------------------------------------------------------------------------------------------------------------------
35 // algorithms types
36
37 typedef enum __attribute__ ((__packed__)) rrd_algorithm {
@@ -223,1390 +51,103 @@ typedef enum __attribute__ ((__packed__)) rrd_algorithm {
51 RRD_ALGORITHM rrd_algorithm_id(const char *name);
52 const char *rrd_algorithm_name(RRD_ALGORITHM algorithm);
53
226 -// ----------------------------------------------------------------------------
227 -// flags & options
228 -
229 -// options are permanent configuration options (no atomics to alter/access them)
230 -typedef enum __attribute__ ((__packed__)) rrddim_options {
231 - RRDDIM_OPTION_NONE = 0,
232 - RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
233 - RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
234 - RRDDIM_OPTION_BACKFILLED_HIGH_TIERS = (1 << 2), // when set, we have backfilled higher tiers
235 - RRDDIM_OPTION_UPDATED = (1 << 3), // single-threaded collector updated flag
236 -
237 - // this is 8-bit
238 -} RRDDIM_OPTIONS;
239 -
240 -#define rrddim_option_check(rd, option) ((rd)->collector.options & (option))
241 -#define rrddim_option_set(rd, option) (rd)->collector.options |= (option)
242 -#define rrddim_option_clear(rd, option) (rd)->collector.options &= ~(option)
243 -
244 -// flags are runtime changing status flags (atomics are required to alter/access them)
245 -typedef enum __attribute__ ((__packed__)) rrddim_flags {
246 - RRDDIM_FLAG_NONE = 0,
247 -
248 - RRDDIM_FLAG_OBSOLETE = (1 << 0), // this is marked by the collector/module as obsolete
249 - // No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at
250 - // least rrdset_free_obsolete_time seconds ago.
251 -
252 - RRDDIM_FLAG_ARCHIVED = (1 << 1),
253 - RRDDIM_FLAG_METADATA_UPDATE = (1 << 2), // Metadata needs to go to the database
254 -
255 - RRDDIM_FLAG_META_HIDDEN = (1 << 3), // Status of hidden option in the metadata database
256 - RRDDIM_FLAG_ML_MODEL_LOAD = (1 << 4), // Do ML LOAD for this dimension
257 -
258 - // this is 8 bit
259 -} RRDDIM_FLAGS;
260 -
261 -#define rrddim_flag_get(rd) atomic_flags_get(&((rd)->flags))
262 -#define rrddim_flag_check(rd, flag) atomic_flags_check(&((rd)->flags), flag)
263 -#define rrddim_flag_set(rd, flag) atomic_flags_set(&((rd)->flags), flag)
264 -#define rrddim_flag_clear(rd, flag) atomic_flags_clear(&((rd)->flags), flag)
265 -#define rrddim_flag_set_and_clear(rd, set, clear) atomic_flags_set_and_clear(&((rd)->flags), set, clear)
266 -
267 -// ----------------------------------------------------------------------------
268 -// engine-specific iterator state for dimension data collection
269 -typedef struct storage_collect_handle {
270 - STORAGE_ENGINE_BACKEND seb;
271 -} STORAGE_COLLECT_HANDLE;
272 -
273 -// ----------------------------------------------------------------------------
274 -// Storage tier data for every dimension
275 -
276 -struct rrddim_tier {
277 - STORAGE_POINT virtual_point;
278 - STORAGE_ENGINE_BACKEND seb;
279 - SPINLOCK spinlock;
280 - uint32_t tier_grouping;
281 - time_t next_point_end_time_s;
282 - STORAGE_METRIC_HANDLE *smh; // the metric handle inside the database
283 - STORAGE_COLLECT_HANDLE *sch; // the data collection handle
284 -};
285 -
286 -bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s);
287 -
288 -// ----------------------------------------------------------------------------
289 -// RRD DIMENSION - this is a metric
290 -
291 -struct rrddim {
292 - UUIDMAP_ID uuid;
293 -
294 - // ------------------------------------------------------------------------
295 - // dimension definition
296 -
297 - STRING *id; // the id of this dimension (for internal identification)
298 - STRING *name; // the name of this dimension (as presented to user)
299 -
300 - RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
301 - RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
302 - RRDDIM_FLAGS flags; // run time changing status flags
303 -
304 - int32_t multiplier; // the multiplier of the collected values
305 - int32_t divisor; // the divider of the collected values
306 -
307 - // ------------------------------------------------------------------------
308 - // operational state members
309 -
310 - struct rrdset *rrdset;
311 - rrd_ml_dimension_t *ml_dimension; // machine learning data about this dimension
312 -
313 - struct {
314 - RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
315 - bool collected;
316 - } rrdcontexts;
317 -
318 -#ifdef NETDATA_LOG_COLLECTION_ERRORS
319 - usec_t rrddim_store_metric_last_ut; // the timestamp we last called rrddim_store_metric()
320 - size_t rrddim_store_metric_count; // the rrddim_store_metric() counter
321 - const char *rrddim_store_metric_last_caller; // the name of the function that last called rrddim_store_metric()
322 -#endif
323 -
324 - // ------------------------------------------------------------------------
325 - // db mode RAM, ALLOC, NONE specifics
326 - // TODO - they should be managed by storage engine
327 - // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
328 -
329 - struct {
330 - size_t memsize; // the memory allocated for this dimension (without RRDDIM)
331 - storage_number *data; // the array of values
332 - } db;
333 -
334 - // ------------------------------------------------------------------------
335 - // streaming
336 -
337 - struct {
338 - struct {
339 - uint32_t sent_version;
340 - uint32_t dim_slot;
341 - } snd;
342 - } stream;
343 -
344 - // ------------------------------------------------------------------------
345 - // data collection members
346 -
347 - struct {
348 - RRDDIM_OPTIONS options; // permanent configuration options
349 -
350 - uint32_t counter; // the number of times we added values to this rrddim
54 +// --------------------------------------------------------------------------------------------------------------------
55
352 - collected_number collected_value; // the current value, as collected - resets to 0 after being used
353 - collected_number collected_value_max; // the absolute maximum of the collected value
354 - collected_number last_collected_value; // the last value that was collected, after being processed
355 -
356 - struct timeval last_collected_time; // when was this dimension last updated
357 - // this is actual date time we updated the last_collected_value
358 - // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
359 -
360 - NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
361 - NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
362 -
363 - NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
364 - } collector;
365 -
366 - // ------------------------------------------------------------------------
367 -
368 - struct rrddim_tier tiers[]; // our tiers of databases
369 -};
370 -
371 -size_t rrddim_size(void);
372 -
373 -#define rrddim_id(rd) string2str((rd)->id)
374 -#define rrddim_name(rd) string2str((rd) ->name)
375 -
376 -#define rrddim_check_updated(rd) ((rd)->collector.options & RRDDIM_OPTION_UPDATED)
377 -#define rrddim_set_updated(rd) (rd)->collector.options |= RRDDIM_OPTION_UPDATED
378 -#define rrddim_clear_updated(rd) (rd)->collector.options &= ~RRDDIM_OPTION_UPDATED
379 -
380 -// ------------------------------------------------------------------------
381 -// DATA COLLECTION STORAGE OPS
382 -
383 -STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
384 -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
385 -static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, nd_uuid_t *uuid) {
386 - internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
387 -
388 -#ifdef ENABLE_DBENGINE
389 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
390 - return rrdeng_metrics_group_get(si, uuid);
391 -#endif
392 - return rrddim_metrics_group_get(si, uuid);
393 -}
394 -
395 -void rrdeng_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
396 -void rrddim_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
397 -static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg) {
398 - internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
399 -
400 -#ifdef ENABLE_DBENGINE
401 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
402 - rrdeng_metrics_group_release(si, smg);
403 - else
404 -#endif
405 - rrddim_metrics_group_release(si, smg);
406 -}
407 -
408 -STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
409 -STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
410 -static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {
411 - internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
412 -
413 -#ifdef ENABLE_DBENGINE
414 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
415 - return rrdeng_store_metric_init(smh, update_every, smg);
416 -#endif
417 - return rrddim_collect_init(smh, update_every, smg);
418 -}
419 -
420 -void rrdeng_store_metric_next(
421 - STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
422 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
423 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
424 -
425 -void rrddim_collect_store_metric(
426 - STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
427 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
428 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
429 -
430 -static inline void storage_engine_store_metric(
431 - STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
432 - NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
433 - uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {
434 - internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
435 -
436 -#ifdef ENABLE_DBENGINE
437 - if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
438 - return rrdeng_store_metric_next(sch, point_in_time_ut,
439 - n, min_value, max_value,
440 - count, anomaly_count, flags);
441 -#endif
442 - return rrddim_collect_store_metric(sch, point_in_time_ut,
443 - n, min_value, max_value,
444 - count, anomaly_count, flags);
445 -}
446 -
447 -uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *si);
448 -static inline uint64_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
449 -#ifdef ENABLE_DBENGINE
450 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
451 - return rrdeng_disk_space_max(si);
452 -#endif
453 -
454 - return 0;
455 -}
456 -
457 -uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *si);
458 -static inline uint64_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
459 -#ifdef ENABLE_DBENGINE
460 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
461 - return rrdeng_disk_space_used(si);
462 -#endif
463 -
464 - // TODO - calculate the total host disk space for memory mode save and map
465 - return 0;
466 -}
467 -
468 -uint64_t rrdeng_metrics(STORAGE_INSTANCE *si);
469 -static inline uint64_t storage_engine_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
470 -#ifdef ENABLE_DBENGINE
471 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
472 - return rrdeng_metrics(si);
473 -#endif
474 -
475 - // TODO - calculate the total host disk space for memory mode save and map
476 - return 0;
477 -}
56 +typedef struct rrdcalc RRDCALC;
57 +typedef struct alarm_entry ALARM_ENTRY;
58 +typedef struct rrdvar_acquired RRDVAR_ACQUIRED;
59 +typedef struct rrdcalc_acquired RRDCALC_ACQUIRED;
60
479 -uint64_t rrdeng_samples(STORAGE_INSTANCE *si);
480 -static inline uint64_t storage_engine_samples(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
61 #ifdef ENABLE_DBENGINE
482 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
483 - return rrdeng_samples(si);
62 +struct rrdengine_instance;
63 #endif
485 - return 0;
486 -}
64
65 +// --------------------------------------------------------------------------------------------------------------------
66
489 -time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *si);
490 -static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
491 -#ifdef ENABLE_DBENGINE
492 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
493 - return rrdeng_global_first_time_s(si);
494 -#endif
67 +#define UPDATE_EVERY_MIN 1
68 +#define UPDATE_EVERY_MAX 3600
69 +#define RRD_ID_LENGTH_MAX 1200
70
496 - return now_realtime_sec() - (time_t)(default_rrd_history_entries * nd_profile.update_every);
497 -}
71 +#define RRD_DEFAULT_HISTORY_ENTRIES 3600
72 +#define RRD_HISTORY_ENTRIES_MAX (86400*365)
73
499 -size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *si);
500 -static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
501 -#ifdef ENABLE_DBENGINE
502 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
503 - return rrdeng_currently_collected_metrics(si);
74 +#if defined(ENV32BIT)
75 +#define MIN_LIBUV_WORKER_THREADS 8
76 +#define MAX_LIBUV_WORKER_THREADS 128
77 +#define RESERVED_LIBUV_WORKER_THREADS 3
78 +#else
79 +#define MIN_LIBUV_WORKER_THREADS 16
80 +#define MAX_LIBUV_WORKER_THREADS 1024
81 +#define RESERVED_LIBUV_WORKER_THREADS 6
82 #endif
83
506 - // TODO - calculate the total host disk space for memory mode save and map
507 - return 0;
508 -}
84 +extern int libuv_worker_threads;
85 +extern bool ieee754_doubles;
86
510 -void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *sch);
511 -void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *sch);
512 -static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *sch) {
513 - if(unlikely(!sch))
514 - return;
87 +typedef long long total_number;
88
516 - internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
89 +// --------------------------------------------------------------------------------------------------------------------
90 +// global lock for all RRDHOSTs
91
518 -#ifdef ENABLE_DBENGINE
519 - if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
520 - rrdeng_store_metric_flush_current_page(sch);
521 - else
522 -#endif
523 - rrddim_store_metric_flush(sch);
524 -}
92 +extern netdata_rwlock_t rrd_rwlock;
93
526 -int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *sch);
527 -int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *sch);
528 -// a finalization function to run after collection is over
529 -// returns 1 if it's safe to delete the dimension
530 -static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *sch) {
531 - internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
94 +#define rrd_rdlock() netdata_rwlock_rdlock(&rrd_rwlock)
95 +#define rrd_wrlock() netdata_rwlock_wrlock(&rrd_rwlock)
96 +#define rrd_rdunlock() netdata_rwlock_rdunlock(&rrd_rwlock)
97 +#define rrd_wrunlock() netdata_rwlock_wrunlock(&rrd_rwlock)
98
533 -#ifdef ENABLE_DBENGINE
534 - if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
535 - return rrdeng_store_metric_finalize(sch);
536 -#endif
99 +// --------------------------------------------------------------------------------------------------------------------
100
538 - return rrddim_collect_finalize(sch);
539 -}
101 +STRING *rrd_string_strdupz(const char *s);
102
541 -void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
542 -void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
543 -static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every) {
544 - internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
103 +#include "rrd-database-mode.h"
104 +long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
105
546 -#ifdef ENABLE_DBENGINE
547 - if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
548 - rrdeng_store_metric_change_collection_frequency(sch, update_every);
549 - else
550 -#endif
551 - rrddim_store_metric_change_collection_frequency(sch, update_every);
106 +static inline uint32_t get_uint32_id() {
107 + return now_realtime_sec() & UINT32_MAX;
108 }
109
110 +// --------------------------------------------------------------------------------------------------------------------
111
555 -// ----------------------------------------------------------------------------
556 -// STORAGE ENGINE QUERY OPS
557 -
558 -time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh);
559 -time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *smh);
560 -static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {
561 - internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
562 -
563 -#ifdef ENABLE_DBENGINE
564 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
565 - return rrdeng_metric_oldest_time(smh);
566 -#endif
567 - return rrddim_query_oldest_time_s(smh);
568 -}
112 +#include "storage-engine.h"
113
570 -time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh);
571 -time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *smh);
572 -static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {
573 - internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
114 +#include "rrdhost.h"
115 +#include "rrdset.h"
116 +#include "rrddim.h"
117 +#include "rrddim-backfill.h"
118
119 +#include "streaming/stream-sender-commit.h"
120 +#include "streaming/stream-replication-tracking.h"
121 +#include "rrdhost-system-info.h"
122 +#include "daemon/common.h"
123 +#include "web/api/queries/query.h"
124 +#include "web/api/queries/rrdr.h"
125 +#include "health/rrdvar.h"
126 +#include "health/rrdcalc.h"
127 +#include "rrdlabels.h"
128 +#include "streaming/stream-capabilities.h"
129 +#include "streaming/stream-path.h"
130 +#include "streaming/stream.h"
131 +//#include "aclk/aclk_rrdhost_state.h"
132 +#include "sqlite/sqlite_health.h"
133 +#include "contexts/rrdcontext.h"
134 +#include "rrdcollector.h"
135 +#include "rrdfunctions.h"
136 #ifdef ENABLE_DBENGINE
576 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
577 - return rrdeng_metric_latest_time(smh);
137 +#include "database/engine/rrdengineapi.h"
138 #endif
579 - return rrddim_query_latest_time_s(smh);
580 -}
581 -
582 -void rrdeng_load_metric_init(
583 - STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
584 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
585 -
586 -void rrddim_query_init(
587 - STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
588 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
139 +#include "sqlite/sqlite_functions.h"
140 +#include "sqlite/sqlite_context.h"
141 +#include "sqlite/sqlite_metadata.h"
142 +#include "sqlite/sqlite_aclk.h"
143 +#include "sqlite/sqlite_aclk_alert.h"
144 +#include "sqlite/sqlite_aclk_node.h"
145 +#include "sqlite/sqlite_health.h"
146
590 -static inline void storage_engine_query_init(
591 - STORAGE_ENGINE_BACKEND seb __maybe_unused,
592 - STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
593 - time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {
594 - internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
595 -
596 -#ifdef ENABLE_DBENGINE
597 - if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
598 - rrdeng_load_metric_init(smh, seqh, start_time_s, end_time_s, priority);
599 - else
600 -#endif
601 - rrddim_query_init(smh, seqh, start_time_s, end_time_s, priority);
602 -}
603 -
604 -STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh);
605 -STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh);
606 -static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {
607 - internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
608 -
609 -#ifdef ENABLE_DBENGINE
610 - if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
611 - return rrdeng_load_metric_next(seqh);
612 -#endif
613 - return rrddim_query_next_metric(seqh);
614 -}
615 -
616 -int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh);
617 -int rrddim_query_is_finished(struct storage_engine_query_handle *seqh);
618 -static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {
619 - internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
620 -
621 -#ifdef ENABLE_DBENGINE
622 - if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
623 - return rrdeng_load_metric_is_finished(seqh);
624 -#endif
625 - return rrddim_query_is_finished(seqh);
626 -}
627 -
628 -void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh);
629 -void rrddim_query_finalize(struct storage_engine_query_handle *seqh);
630 -static inline void storage_engine_query_finalize(struct storage_engine_query_handle *seqh) {
631 - internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
632 -
633 -#ifdef ENABLE_DBENGINE
634 - if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
635 - rrdeng_load_metric_finalize(seqh);
636 - else
637 -#endif
638 - rrddim_query_finalize(seqh);
639 -}
640 -
641 -time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh);
642 -time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *seqh);
643 -static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *seqh) {
644 - internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
645 -
646 -#ifdef ENABLE_DBENGINE
647 - if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
648 - return rrdeng_load_align_to_optimal_before(seqh);
649 -#endif
650 - return rrddim_query_align_to_optimal_before(seqh);
651 -}
652 -
653 -// ------------------------------------------------------------------------
654 -// function pointers for all APIs provided by a storage engine
655 -typedef struct storage_engine_api {
656 - // metric management
657 - STORAGE_METRIC_HANDLE *(*metric_get_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id);
658 - STORAGE_METRIC_HANDLE *(*metric_get_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
659 - STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *si);
660 - void (*metric_release)(STORAGE_METRIC_HANDLE *);
661 - STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
662 - bool (*metric_retention_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id, time_t *first_entry_s, time_t *last_entry_s);
663 - bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
664 -} STORAGE_ENGINE_API;
665 -
666 -typedef struct storage_engine {
667 - STORAGE_ENGINE_BACKEND seb;
668 - RRD_MEMORY_MODE id;
669 - const char* name;
670 - STORAGE_ENGINE_API api;
671 -} STORAGE_ENGINE;
672 -
673 -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
674 -STORAGE_ENGINE* storage_engine_find(const char* name);
675 -
676 -// ----------------------------------------------------------------------------
677 -// these loop macros make sure the linked list is accessed with the right lock
678 -
679 -#define rrddim_foreach_read(rd, st) \
680 - dfe_start_read((st)->rrddim_root_index, rd)
681 -#define rrddim_foreach_done(rd) \
682 - dfe_done(rd)
683 -
684 -// ----------------------------------------------------------------------------
685 -// RRDSET - this is a chart
686 -
687 -#define rrdset_flag_get(st) atomic_flags_get(&((st)->flags))
688 -#define rrdset_flag_check(st, flag) atomic_flags_check(&((st)->flags), flag)
689 -#define rrdset_flag_set(st, flag) atomic_flags_set(&((st)->flags), flag)
690 -#define rrdset_flag_clear(st, flag) atomic_flags_clear(&((st)->flags), flag)
691 -#define rrdset_flag_set_and_clear(st, set, clear) atomic_flags_set_and_clear(&((st)->flags), set, clear)
692 -
693 -#define rrdset_is_replicating(st) (rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS|RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS) \
694 - && !rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED|RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
695 -
696 -struct pluginsd_rrddim {
697 - RRDDIM_ACQUIRED *rda;
698 - RRDDIM *rd;
699 - const char *id;
700 -};
701 -
702 -struct rrdset {
703 - nd_uuid_t chart_uuid; // the global UUID for this chart
704 -
705 - // ------------------------------------------------------------------------
706 - // chart configuration
707 -
708 - struct {
709 - STRING *type; // the type of {type}.{id}
710 - STRING *id; // the id of {type}.{id}
711 - STRING *name; // the name of {type}.{name}
712 - } parts;
713 -
714 - STRING *id; // the unique ID of the rrdset as {type}.{id}
715 - STRING *name; // the unique name of the rrdset as {type}.{name}
716 - STRING *family; // grouping sets under the same family
717 - STRING *title; // title shown to user
718 - STRING *units; // units of measurement
719 - STRING *context; // the template of this data set
720 - STRING *plugin_name; // the name of the plugin that generated this
721 - STRING *module_name; // the name of the plugin module that generated this
722 -
723 - int32_t priority; // the sorting priority of this chart
724 - int32_t update_every; // data collection frequency
725 -
726 - RRDLABELS *rrdlabels; // chart labels
727 -
728 - uint32_t version; // the metadata version (auto-increment)
729 -
730 - RRDSET_TYPE chart_type; // line, area, stacked
731 -
732 - // ------------------------------------------------------------------------
733 - // operational state members
734 -
735 - RRDSET_FLAGS flags; // flags
736 - RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset
737 -
738 - DICTIONARY *rrddim_root_index; // dimensions index
739 -
740 - rrd_ml_chart_t *ml_chart;
741 -
742 - STORAGE_METRICS_GROUP *smg[RRD_STORAGE_TIERS];
743 -
744 - // ------------------------------------------------------------------------
745 - // linking to siblings and parents
746 -
747 - RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to
748 -
749 - struct {
750 - RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart
751 - RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to
752 - bool collected;
753 - } rrdcontexts;
754 -
755 - // ------------------------------------------------------------------------
756 - // data collection members
757 -
758 - SPINLOCK data_collection_lock;
759 -
760 - uint32_t counter; // the number of times we added values to this database
761 - uint32_t counter_done; // the number of times rrdset_done() has been called
762 -
763 - time_t last_accessed_time_s; // the last time this RRDSET has been accessed
764 - usec_t usec_since_last_update; // the time in microseconds since the last collection of data
765 -
766 - struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)
767 - struct timeval last_collected_time; // when did this data set last collected values
768 -
769 - size_t rrdlabels_last_saved_version;
770 -
771 - DICTIONARY *functions_view; // collector functions this rrdset supports, can be NULL
772 -
773 - // ------------------------------------------------------------------------
774 - // data collection - streaming to parents, temp variables
775 -
776 - struct {
777 - struct {
778 - uint32_t sent_version;
779 - uint32_t chart_slot;
780 - uint32_t dim_last_slot_used;
781 -#ifdef REPLICATION_TRACKING
782 - REPLAY_WHO who;
783 -#endif
784 - time_t resync_time_s; // the timestamp up to which we should resync clock upstream
785 - } snd;
786 -
787 - struct {
788 -#ifdef REPLICATION_TRACKING
789 - REPLAY_WHO who;
790 -#endif
791 - } rcv;
792 - } stream;
793 -
794 - // ------------------------------------------------------------------------
795 - // db mode SAVE, MAP specifics
796 - // TODO - they should be managed by storage engine
797 - // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
798 -
799 - struct {
800 - int32_t entries; // total number of entries in the data set
801 - int32_t current_entry; // the entry that is currently being updated
802 - // it goes around in a round-robin fashion
803 - } db;
804 -
805 - // ------------------------------------------------------------------------
806 - // exporting to 3rd party time-series members
807 - // TODO - they should be managed by exporting engine
808 - // (RRDSET_EXPORTING_STATE ptr to an undefined structure, and a call to clean this up during destruction)
809 -
810 - RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances
811 -
812 - // ------------------------------------------------------------------------
813 - // health monitoring members
814 - // TODO - they should be managed by health
815 - // (RRDSET_HEALTH_STATE ptr to an undefined structure, and a call to clean this up during destruction)
816 -
817 - NETDATA_DOUBLE green; // green threshold for this chart
818 - NETDATA_DOUBLE red; // red threshold for this chart
819 -
820 - DICTIONARY *rrdvars; // RRDVAR index for this chart
821 -
822 - struct {
823 - RW_SPINLOCK spinlock; // protection for RRDCALC *base
824 - RRDCALC *base; // double linked list of RRDCALC related to this RRDSET
825 - } alerts;
826 -
827 - struct {
828 - SPINLOCK spinlock; // used only for cleanup
829 - pid_t collector_tid;
830 - bool dims_with_slots;
831 - bool set;
832 - uint32_t pos;
833 - int32_t last_slot;
834 - uint32_t size;
835 - struct pluginsd_rrddim *prd_array;
836 - } pluginsd;
837 -
838 -#ifdef NETDATA_LOG_REPLICATION_REQUESTS
839 - struct {
840 - bool log_next_data_collection;
841 - bool start_streaming;
842 - time_t after;
843 - time_t before;
844 - } replay;
845 -#endif // NETDATA_LOG_REPLICATION_REQUESTS
846 -};
847 -
848 -#define rrdset_plugin_name(st) string2str((st)->plugin_name)
849 -#define rrdset_module_name(st) string2str((st)->module_name)
850 -#define rrdset_units(st) string2str((st)->units)
851 -#define rrdset_parts_type(st) string2str((st)->parts.type)
852 -#define rrdset_family(st) string2str((st)->family)
853 -#define rrdset_title(st) string2str((st)->title)
854 -#define rrdset_context(st) string2str((st)->context)
855 -#define rrdset_name(st) string2str((st)->name)
856 -#define rrdset_id(st) string2str((st)->id)
857 -
858 -static inline uint32_t rrdset_metadata_version(RRDSET *st) {
859 - return __atomic_load_n(&st->version, __ATOMIC_RELAXED);
860 -}
861 -
862 -static inline uint32_t rrdset_metadata_upstream_version(RRDSET *st) {
863 - return __atomic_load_n(&st->stream.snd.sent_version, __ATOMIC_RELAXED);
864 -}
865 -
866 -void rrdset_metadata_updated(RRDSET *st);
867 -
868 -static inline void rrdset_metadata_exposed_upstream(RRDSET *st, uint32_t version) {
869 - __atomic_store_n(&st->stream.snd.sent_version, version, __ATOMIC_RELAXED);
870 -}
871 -
872 -static inline bool rrdset_check_upstream_exposed(RRDSET *st) {
873 - return rrdset_metadata_version(st) == rrdset_metadata_upstream_version(st);
874 -}
875 -
876 -static inline uint32_t rrddim_metadata_version(RRDDIM *rd) {
877 - // the metadata version of the dimension, is the version of the chart
878 - return rrdset_metadata_version(rd->rrdset);
879 -}
880 -
881 -static inline uint32_t rrddim_metadata_upstream_version(RRDDIM *rd) {
882 - return __atomic_load_n(&rd->stream.snd.sent_version, __ATOMIC_RELAXED);
883 -}
884 -
885 -void rrddim_metadata_updated(RRDDIM *rd);
886 -
887 -static inline void rrddim_metadata_exposed_upstream(RRDDIM *rd, uint32_t version) {
888 - __atomic_store_n(&rd->stream.snd.sent_version, version, __ATOMIC_RELAXED);
889 -}
890 -
891 -static inline void rrddim_metadata_exposed_upstream_clear(RRDDIM *rd) {
892 - __atomic_store_n(&rd->stream.snd.sent_version, 0, __ATOMIC_RELAXED);
893 -}
894 -
895 -static inline bool rrddim_check_upstream_exposed(RRDDIM *rd) {
896 - return rrddim_metadata_upstream_version(rd) != 0;
897 -}
898 -
899 -// the collector sets the exposed flag, but anyone can remove it
900 -// still, it can be removed, after the collector has finished
901 -// so, it is safe to check it without atomics
902 -static inline bool rrddim_check_upstream_exposed_collector(RRDDIM *rd) {
903 - return rd->rrdset->version == rd->stream.snd.sent_version;
904 -}
905 -
906 -STRING *rrd_string_strdupz(const char *s);
907 -
908 -// ----------------------------------------------------------------------------
909 -// these loop macros make sure the linked list is accessed with the right lock
910 -
911 -#define rrdset_foreach_read(st, host) \
912 - dfe_start_read((host)->rrdset_root_index, st)
913 -
914 -#define rrdset_foreach_write(st, host) \
915 - dfe_start_write((host)->rrdset_root_index, st)
916 -
917 -#define rrdset_foreach_reentrant(st, host) \
918 - dfe_start_reentrant((host)->rrdset_root_index, st)
919 -
920 -#define rrdset_foreach_done(st) \
921 - dfe_done(st)
922 -
923 -#define rrdset_number_of_dimensions(st) \
924 - dictionary_entries((st)->rrddim_root_index)
925 -
926 -#include "rrdcollector.h"
927 -#include "rrdfunctions.h"
928 -
929 -// ----------------------------------------------------------------------------
930 -// RRDHOST flags
931 -// use this for configuration flags, not for state control
932 -// flags are set/unset in a manner that is not thread safe
933 -// and may lead to missing information.
934 -
935 -typedef enum __attribute__ ((__packed__)) rrdhost_flags {
936 -
937 - // Careful not to overlap with rrdhost_options to avoid bugs if
938 - // rrdhost_flags_xxx is used instead of rrdhost_option_xxx or vice-versa
939 - // Orphan, Archived and Obsolete flags
940 -
941 - /*
942 - * 3 BASE FLAGS FOR HOSTS:
943 - *
944 - * - COLLECTOR_ONLINE = the collector is currently collecting data for this node
945 - * this is true FOR ALL KINDS OF NODES (including localhost, virtual hosts, children)
946 - *
947 - * - ORPHAN = the node had a collector online recently, but does not have it now
948 - *
949 - * - ARCHIVED = the node does not have data collection structures attached to it
950 - *
951 - */
952 -
953 - RRDHOST_FLAG_COLLECTOR_ONLINE = (1 << 7), // the collector of this host is online
954 - RRDHOST_FLAG_ORPHAN = (1 << 8), // this host is orphan (not receiving data)
955 - RRDHOST_FLAG_ARCHIVED = (1 << 9), // The host is archived, no collected charts yet
956 -
957 - RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS = (1 << 10), // the host has pending chart obsoletions
958 - RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS = (1 << 11), // the host has pending dimension obsoletions
959 -
960 - // Streaming sender
961 - RRDHOST_FLAG_STREAM_SENDER_INITIALIZED = (1 << 12), // the host has initialized streaming sender structures
962 - RRDHOST_FLAG_STREAM_SENDER_ADDED = (1 << 13), // When set, the sender thread is running
963 - RRDHOST_FLAG_STREAM_SENDER_CONNECTED = (1 << 14), // When set, the host is connected to a parent
964 - RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS = (1 << 15), // when set, rrdset_done() should push metrics to parent
965 - RRDHOST_FLAG_STREAM_SENDER_LOGGED_STATUS = (1 << 16), // when set, we have logged the status of metrics streaming
966 -
967 - // Health
968 - RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 17), // contains charts and dims with uninitialized variables
969 - RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 18), // the host has initialized health structures
970 -
971 - // Exporting
972 - RRDHOST_FLAG_EXPORTING_SEND = (1 << 19), // send it to external databases
973 - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 20), // don't send it to external databases
974 -
975 - // ACLK
976 - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 21), // when set, we should send ACLK stream context updates
977 - RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // Host should stream alerts
978 -
979 - // Metadata
980 - RRDHOST_FLAG_METADATA_UPDATE = (1 << 23), // metadata needs to be stored in the database
981 - RRDHOST_FLAG_METADATA_LABELS = (1 << 24), // metadata needs to be stored in the database
982 - RRDHOST_FLAG_METADATA_INFO = (1 << 25), // metadata needs to be stored in the database
983 - RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 26), // Context needs to be loaded
984 -
985 - RRDHOST_FLAG_METADATA_CLAIMID = (1 << 27), // metadata needs to be stored in the database
986 -
987 - RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 28), // set when the host has updated global functions
988 -} RRDHOST_FLAGS;
989 -
990 -#define rrdhost_flag_get(host) atomic_flags_get(&((host)->flags))
991 -#define rrdhost_flag_check(host, flag) atomic_flags_check(&((host)->flags), flag)
992 -#define rrdhost_flag_set(host, flag) atomic_flags_set(&((host)->flags), flag)
993 -#define rrdhost_flag_clear(host, flag) atomic_flags_clear(&((host)->flags), flag)
994 -#define rrdhost_flag_set_and_clear(host, set, clear) atomic_flags_set_and_clear(&((host)->flags), set, clear)
995 -
996 -#ifdef NETDATA_INTERNAL_CHECKS
997 -#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
998 - netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0)
999 -#else
1000 -#define rrdset_debug(st, fmt, args...) debug_dummy()
1001 -#endif
1002 -
1003 -typedef enum __attribute__ ((__packed__)) {
1004 - // Indexing
1005 - RRDHOST_OPTION_INDEXED_MACHINE_GUID = (1 << 0), // when set, we have indexed its machine guid
1006 - RRDHOST_OPTION_INDEXED_HOSTNAME = (1 << 1), // when set, we have indexed its hostname
1007 -
1008 - // Streaming configuration
1009 - RRDHOST_OPTION_SENDER_ENABLED = (1 << 2), // set when the host is configured to send metrics to a parent
1010 - RRDHOST_OPTION_REPLICATION = (1 << 3), // when set, we support replication for this host
1011 -
1012 - // Other options
1013 - RRDHOST_OPTION_VIRTUAL_HOST = (1 << 4), // when set, this host is a virtual one
1014 - RRDHOST_OPTION_EPHEMERAL_HOST = (1 << 5), // when set, this host is an ephemeral one
1015 -} RRDHOST_OPTIONS;
1016 -
1017 -#define rrdhost_option_check(host, flag) ((host)->options & (flag))
1018 -#define rrdhost_option_set(host, flag) (host)->options |= flag
1019 -#define rrdhost_option_clear(host, flag) (host)->options &= ~(flag)
1020 -
1021 -#define rrdhost_has_stream_sender_enabled(host) (rrdhost_option_check(host, RRDHOST_OPTION_SENDER_ENABLED) && (host)->sender)
1022 -
1023 -#define rrdhost_can_stream_metadata_to_parent(host) \
1024 - (rrdhost_has_stream_sender_enabled(host) && \
1025 - rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS) && \
1026 - rrdhost_flag_check(host, RRDHOST_FLAG_COLLECTOR_ONLINE) \
1027 - )
1028 -
1029 -// ----------------------------------------------------------------------------
1030 -// RRD HOST
1031 -
1032 -struct stream_thread;
1033 -
1034 -struct rrdhost {
1035 - char machine_guid[GUID_LEN + 1]; // the unique ID of this host
1036 -
1037 - // ------------------------------------------------------------------------
1038 - // host information
1039 -
1040 - STRING *hostname; // the hostname of this host
1041 - STRING *registry_hostname; // the registry hostname for this host
1042 - STRING *os; // the O/S type of the host
1043 - STRING *timezone; // the timezone of the host
1044 - STRING *abbrev_timezone; // the abbriviated timezone of the host
1045 - STRING *program_name; // the program name that collects metrics for this host
1046 - STRING *program_version; // the program version that collects metrics for this host
1047 -
1048 - OBJECT_STATE state_id; // every time data collection (stream receiver) (dis)connects,
1049 - // this gets incremented - it is used to detect stale functions,
1050 - // stale backfilling requests, etc.
1051 -
1052 - int32_t utc_offset; // the offset in seconds from utc
1053 -
1054 - RRDHOST_OPTIONS options; // configuration option for this RRDHOST (no atomics on this)
1055 - RRDHOST_FLAGS flags; // runtime flags about this RRDHOST (atomics on this)
1056 - RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances
1057 -
1058 - int32_t rrd_update_every; // the update frequency of the host
1059 - int32_t rrd_history_entries; // the number of history entries for the host's charts
1060 -
1061 - RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
1062 - // the actual per tier is at .db[tier].mode
1063 -
1064 - char *cache_dir; // the directory to save RRD cache files
1065 -
1066 - struct {
1067 - RRD_MEMORY_MODE mode; // the db mode for this tier
1068 - STORAGE_ENGINE *eng; // the storage engine API for this tier
1069 - STORAGE_INSTANCE *si; // the db instance for this tier
1070 - uint32_t tier_grouping; // tier 0 iterations aggregated on this tier
1071 - } db[RRD_STORAGE_TIERS];
1072 -
1073 - struct rrdhost_system_info *system_info; // information collected from the host environment
1074 -
1075 - // ------------------------------------------------------------------------
1076 - // streaming and replication, configuration and status
1077 -
1078 - struct {
1079 - struct stream_thread *thread;
1080 - uint8_t refcount;
1081 -
1082 - // --- sender ---
1083 -
1084 - struct {
1085 - struct {
1086 - struct {
1087 - SPINLOCK spinlock;
1088 -
1089 - bool ignore; // when set, freeing slots will not put them in the available
1090 - uint32_t used;
1091 - uint32_t size;
1092 - uint32_t *array;
1093 - } available; // keep track of the available chart slots per host
1094 -
1095 - uint32_t last_used; // the last slot we used for a chart (increments only)
1096 - } pluginsd_chart_slots;
1097 -
1098 - struct {
1099 - pid_t tid;
1100 -
1101 - time_t last_connected; // last time child connected (stored in db)
1102 - uint32_t connections; // the number of times this sender has connected
1103 -
1104 - struct {
1105 - uint32_t counter_in; // counts the number of replication statements we have received
1106 - uint32_t counter_out; // counts the number of replication statements we have sent
1107 - uint32_t charts; // the number of charts currently being replicated to a parent
1108 - } replication;
1109 - } status;
1110 -
1111 - // reserved for the receiver/sender thread - do not use for other purposes
1112 - struct sender_buffer commit;
1113 -
1114 - STRING *destination; // where to send metrics to
1115 - STRING *api_key; // the api key at the receiving netdata
1116 - SIMPLE_PATTERN *charts_matching; // pattern to match the charts to be sent
1117 - RRDHOST_STREAM_PARENTS parents; // the list of parents (extracted from destination)
1118 - } snd;
1119 -
1120 - // --- receiver ---
1121 -
1122 - struct {
1123 - struct {
1124 - SPINLOCK spinlock; // lock for the management of the allocation
1125 - uint32_t size;
1126 - RRDSET **array;
1127 - } pluginsd_chart_slots;
1128 -
1129 - struct {
1130 - pid_t tid;
1131 -
1132 - time_t last_connected; // the time the last sender was connected
1133 - time_t last_disconnected; // the time the last sender was disconnected
1134 - time_t last_chart; // the time of the last CHART streaming command
1135 - bool check_obsolete; // set when child connects, will instruct parent to
1136 - // trigger a check for obsoleted charts since previous connect
1137 -
1138 - uint32_t connections; // the number of times this receiver has connected
1139 - STREAM_HANDSHAKE exit_reason; // the last receiver exit reason
1140 -
1141 - struct {
1142 - uint32_t counter_in; // counts the number of replication statements we have received
1143 - uint32_t counter_out; // counts the number of replication statements we have sent
1144 - uint32_t backfill_pending; // the number of replication requests pending on us
1145 - uint32_t charts; // the number of charts currently being replicated from a child
1146 - NETDATA_DOUBLE percent; // the % of replication completion
1147 - } replication;
1148 - } status;
1149 - } rcv;
1150 -
1151 - // --- configuration ---
1152 -
1153 - struct {
1154 - time_t period; // max time we want to replicate from the child
1155 - time_t step; // seconds per replication step
1156 - } replication;
1157 -
1158 - RRDHOST_STREAM_PATH path;
1159 - } stream;
1160 -
1161 - // the following are state information for the threading
1162 - // streaming metrics from this netdata to an upstream netdata
1163 - struct sender_state *sender;
1164 -
1165 - struct receiver_state *receiver;
1166 - SPINLOCK receiver_lock;
1167 -
1168 - // ------------------------------------------------------------------------
1169 -
1170 - struct aclk_sync_cfg_t *aclk_config;
1171 -
1172 - // ------------------------------------------------------------------------
1173 - // health monitoring options
1174 -
1175 - // health variables
1176 - HEALTH health;
1177 -
1178 - // all RRDCALCs are primarily allocated and linked here
1179 - DICTIONARY *rrdcalc_root_index;
1180 -
1181 - ALARM_LOG health_log; // alarms historical events (event log)
1182 - uint32_t health_last_processed_id; // the last processed health id from the log
1183 - uint32_t health_max_unique_id; // the max alarm log unique id given for the host
1184 - uint32_t health_max_alarm_id; // the max alarm id given for the host
1185 - size_t health_transitions; // the number of times an alert changed state
1186 -
1187 - // ------------------------------------------------------------------------
1188 - // locks
1189 -
1190 - SPINLOCK rrdhost_update_lock;
1191 -
1192 - // ------------------------------------------------------------------------
1193 - // ML handle
1194 - rrd_ml_host_t *ml_host;
1195 -
1196 - // ------------------------------------------------------------------------
1197 - // Support for host-level labels
1198 - RRDLABELS *rrdlabels;
1199 -
1200 - // ------------------------------------------------------------------------
1201 - // Support for functions
1202 - DICTIONARY *functions; // collector functions this rrdset supports, can be NULL
1203 -
1204 - // ------------------------------------------------------------------------
1205 - // indexes
1206 -
1207 - DICTIONARY *rrdset_root_index; // the host's charts index (by id)
1208 - DICTIONARY *rrdset_root_index_name; // the host's charts index (by name)
1209 -
1210 - DICTIONARY *rrdvars; // the host's chart variables index
1211 - // this includes custom host variables
1212 -
1213 - struct {
1214 - uint32_t metrics_count; // atomic
1215 - uint32_t instances_count; // atomic
1216 - uint32_t contexts_count; // atomic
1217 - } collected;
1218 -
1219 - struct {
1220 - DICTIONARY *contexts;
1221 - DICTIONARY *hub_queue;
1222 - DICTIONARY *pp_queue;
1223 - uint32_t metrics_count; // atomic
1224 - uint32_t instances_count; // atomic
1225 - uint32_t contexts_count; // atomic
1226 - } rrdctx;
1227 -
1228 - struct {
1229 - SPINLOCK spinlock;
1230 - time_t first_time_s;
1231 - time_t last_time_s;
1232 - } retention;
1233 -
1234 - ND_UUID host_id; // Global GUID for this host
1235 - ND_UUID node_id; // Cloud node_id
1236 -
1237 - struct {
1238 - ND_UUID claim_id_of_origin;
1239 - ND_UUID claim_id_of_parent;
1240 - } aclk;
1241 -
1242 - struct rrdhost *next;
1243 - struct rrdhost *prev;
1244 -};
1245 -extern RRDHOST *localhost;
1246 -
1247 -#define rrdhost_receiver_lock(host) spinlock_lock(&(host)->receiver_lock)
1248 -#define rrdhost_receiver_unlock(host) spinlock_unlock(&(host)->receiver_lock)
1249 -
1250 -#define rrdhost_hostname(host) string2str((host)->hostname)
1251 -#define rrdhost_registry_hostname(host) string2str((host)->registry_hostname)
1252 -#define rrdhost_os(host) string2str((host)->os)
1253 -#define rrdhost_timezone(host) string2str((host)->timezone)
1254 -#define rrdhost_abbrev_timezone(host) string2str((host)->abbrev_timezone)
1255 -#define rrdhost_program_name(host) string2str((host)->program_name)
1256 -#define rrdhost_program_version(host) string2str((host)->program_version)
1257 -
1258 -#define rrdhost_receiver_replicating_charts(host) (__atomic_load_n(&((host)->stream.rcv.status.replication.charts), __ATOMIC_RELAXED))
1259 -#define rrdhost_receiver_replicating_charts_plus_one(host) (__atomic_add_fetch(&((host)->stream.rcv.status.replication.charts), 1, __ATOMIC_RELAXED))
1260 -#define rrdhost_receiver_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->stream.rcv.status.replication.charts), 1, __ATOMIC_RELAXED))
1261 -#define rrdhost_receiver_replicating_charts_zero(host) (__atomic_store_n(&((host)->stream.rcv.status.replication.charts), 0, __ATOMIC_RELAXED))
1262 -
1263 -#define rrdhost_sender_replicating_charts(host) (__atomic_load_n(&((host)->stream.snd.status.replication.charts), __ATOMIC_RELAXED))
1264 -#define rrdhost_sender_replicating_charts_plus_one(host) (__atomic_add_fetch(&((host)->stream.snd.status.replication.charts), 1, __ATOMIC_RELAXED))
1265 -#define rrdhost_sender_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->stream.snd.status.replication.charts), 1, __ATOMIC_RELAXED))
1266 -#define rrdhost_sender_replicating_charts_zero(host) (__atomic_store_n(&((host)->stream.snd.status.replication.charts), 0, __ATOMIC_RELAXED))
1267 -
1268 -#define rrdhost_is_local(host) ( \
1269 - (host) == localhost || \
1270 - rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) \
1271 - )
1272 -
1273 -#define rrdhost_is_online(host) ( \
1274 - rrdhost_is_local(host) || \
1275 - (rrdhost_flag_check(host, RRDHOST_FLAG_COLLECTOR_ONLINE) && !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) \
1276 - )
1277 -
1278 -bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now);
1279 -
1280 -extern DICTIONARY *rrdhost_root_index;
1281 -size_t rrdhost_hosts_available(void);
1282 -
1283 -RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid);
1284 -RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha);
1285 -void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha);
1286 -
1287 -// ----------------------------------------------------------------------------
1288 -
1289 -#define rrdhost_foreach_read(var) \
1290 - for((var) = localhost; var ; (var) = (var)->next)
1291 -
1292 -#define rrdhost_foreach_write(var) \
1293 - for((var) = localhost; var ; (var) = (var)->next)
1294 -
1295 -
1296 -// ----------------------------------------------------------------------------
1297 -// global lock for all RRDHOSTs
1298 -
1299 -extern netdata_rwlock_t rrd_rwlock;
1300 -
1301 -#define rrd_rdlock() netdata_rwlock_rdlock(&rrd_rwlock)
1302 -#define rrd_wrlock() netdata_rwlock_wrlock(&rrd_rwlock)
1303 -#define rrd_rdunlock() netdata_rwlock_rdunlock(&rrd_rwlock)
1304 -#define rrd_wrunlock() netdata_rwlock_wrunlock(&rrd_rwlock)
1305 -
1306 -// ----------------------------------------------------------------------------
1307 -
1308 -void rrdset_index_init(RRDHOST *host);
1309 -void rrdset_index_destroy(RRDHOST *host);
1310 -
1311 -void rrddim_index_init(RRDSET *st);
1312 -void rrddim_index_destroy(RRDSET *st);
1313 -
1314 -// ----------------------------------------------------------------------------
1315 -
1316 -extern time_t rrdhost_free_orphan_time_s;
1317 -extern time_t rrdhost_free_ephemeral_time_s;
147 +// --------------------------------------------------------------------------------------------------------------------
148
149 int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest);
150
1321 -RRDHOST *rrdhost_find_by_hostname(const char *hostname);
1322 -RRDHOST *rrdhost_find_by_guid(const char *guid);
1323 -RRDHOST *find_host_by_node_id(char *node_id);
1324 -
1325 -RRDHOST *rrdhost_find_or_create(
1326 - const char *hostname,
1327 - const char *registry_hostname,
1328 - const char *guid,
1329 - const char *os,
1330 - const char *timezone,
1331 - const char *abbrev_timezone,
1332 - int32_t utc_offset,
1333 - const char *prog_name,
1334 - const char *prog_version,
1335 - int update_every,
1336 - long history,
1337 - RRD_MEMORY_MODE mode,
1338 - bool health,
1339 - bool stream,
1340 - STRING *parents,
1341 - STRING *api_key,
1342 - STRING *send_charts_matching,
1343 - bool replication,
1344 - time_t replication_period,
1345 - time_t replication_step,
1346 - struct rrdhost_system_info *system_info,
1347 - bool is_archived);
1348 -
1349 -// ----------------------------------------------------------------------------
1350 -// RRDSET functions
1351 -
1352 -int rrdset_reset_name(RRDSET *st, const char *name);
1353 -
1354 -RRDSET *rrdset_create_custom(RRDHOST *host
1355 - , const char *type
1356 - , const char *id
1357 - , const char *name
1358 - , const char *family
1359 - , const char *context
1360 - , const char *title
1361 - , const char *units
1362 - , const char *plugin
1363 - , const char *module
1364 - , long priority
1365 - , int update_every
1366 - , RRDSET_TYPE chart_type
1367 - , RRD_MEMORY_MODE memory_mode
1368 - , long history_entries);
1369 -
1370 -static inline
1371 -RRDSET *rrdset_create(RRDHOST *host
1372 - , const char *type
1373 - , const char *id
1374 - , const char *name
1375 - , const char *family
1376 - , const char *context
1377 - , const char *title
1378 - , const char *units
1379 - , const char *plugin
1380 - , const char *module
1381 - , long priority
1382 - , int update_every
1383 - , RRDSET_TYPE chart_type) {
1384 - return rrdset_create_custom(
1385 - host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries);
1386 -}
1387 -
1388 -static inline
1389 -RRDSET *rrdset_create_localhost(
1390 - const char *type
1391 - , const char *id
1392 - , const char *name
1393 - , const char *family
1394 - , const char *context
1395 - , const char *title
1396 - , const char *units
1397 - , const char *plugin
1398 - , const char *module
1399 - , long priority
1400 - , int update_every
1401 - , RRDSET_TYPE chart_type) {
1402 - return rrdset_create(
1403 - localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type);
1404 -}
1405 -
1406 -void rrdhost_free_all(void);
1407 -
1408 -void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force);
1409 -
1410 -bool rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s);
1411 -bool rrdhost_should_run_health(RRDHOST *host);
1412 -
1413 -void rrdset_update_heterogeneous_flag(RRDSET *st);
1414 -
1415 -time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);
1416 -
1417 -RRDSET *rrdset_find(RRDHOST *host, const char *id);
1418 -
1419 -RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);
1420 -RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);
1421 -void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
1422 -
1423 -#define rrdset_find_localhost(id) rrdset_find(localhost, id)
1424 -/* This will not return charts that are archived */
1425 -static inline RRDSET *rrdset_find_active_localhost(const char *id)
1426 -{
1427 - RRDSET *st = rrdset_find_localhost(id);
1428 - return st;
1429 -}
1430 -
1431 -RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
1432 -#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
1433 -/* This will not return charts that are archived */
1434 -static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
1435 -{
1436 - RRDSET *st = rrdset_find_bytype_localhost(type, id);
1437 - return st;
1438 -}
1439 -
1440 -RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
1441 -#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)
1442 -/* This will not return charts that are archived */
1443 -static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
1444 -{
1445 - RRDSET *st = rrdset_find_byname_localhost(name);
1446 - return st;
1447 -}
1448 -
1449 -void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds);
1450 -void rrdset_next_usec(RRDSET *st, usec_t microseconds);
1451 -void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);
1452 -#define rrdset_next(st) rrdset_next_usec(st, 0ULL)
1453 -
1454 -void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next);
1455 -void rrdset_done(RRDSET *st);
1456 -
1457 -void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st);
1458 -void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st);
1459 -
1460 -// checks if the RRDSET should be offered to viewers
1461 -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
1462 -#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st))
1463 -
1464 -time_t rrddim_first_entry_s(RRDDIM *rd);
1465 -time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier);
1466 -time_t rrddim_last_entry_s(RRDDIM *rd);
1467 -time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier);
1468 -
1469 -time_t rrdset_first_entry_s(RRDSET *st);
1470 -time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier);
1471 -time_t rrdset_last_entry_s(RRDSET *st);
1472 -time_t rrdset_last_entry_s_of_tier(RRDSET *st, size_t tier);
1473 -
1474 -void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_time_s, time_t *last_time_s, time_t now_s, size_t tier);
1475 -
1476 -void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels);
1477 -
1478 -// ----------------------------------------------------------------------------
1479 -// RRD DIMENSION functions
1480 -
1481 -RRDDIM *rrddim_add_custom(RRDSET *st
1482 - , const char *id
1483 - , const char *name
1484 - , collected_number multiplier
1485 - , collected_number divisor
1486 - , RRD_ALGORITHM algorithm
1487 - , RRD_MEMORY_MODE memory_mode
1488 - );
1489 -
1490 -#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \
1491 - rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
1492 -
1493 -int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name);
1494 -int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
1495 -int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier);
1496 -int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor);
1497 -
1498 -RRDDIM *rrddim_find(RRDSET *st, const char *id);
1499 -RRDDIM_ACQUIRED *rrddim_find_and_acquire(RRDSET *st, const char *id);
1500 -RRDDIM *rrddim_acquired_to_rrddim(RRDDIM_ACQUIRED *rda);
1501 -void rrddim_acquired_release(RRDDIM_ACQUIRED *rda);
1502 -RRDDIM *rrddim_find_active(RRDSET *st, const char *id);
1503 -
1504 -int rrddim_hide(RRDSET *st, const char *id);
1505 -int rrddim_unhide(RRDSET *st, const char *id);
1506 -
1507 -void rrddim_is_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd);
1508 -void rrddim_isnot_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd);
1509 -
1510 -collected_number rrddim_timed_set_by_pointer(RRDSET *st, RRDDIM *rd, struct timeval collected_time, collected_number value);
1511 -collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value);
1512 -collected_number rrddim_set(RRDSET *st, const char *id, collected_number value);
1513 -
1514 -bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd);
1515 -void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);
1516 -void rrdhost_finalize_collection(RRDHOST *host);
1517 -void rrd_finalize_collection_for_all_hosts(void);
1518 -
1519 -long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
1520 -
1521 -#ifdef NETDATA_LOG_COLLECTION_ERRORS
1522 -#define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__)
1523 -void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function);
1524 -#else
1525 -void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);
1526 -#endif
1527 -
1528 -// ----------------------------------------------------------------------------
1529 -// Miscellaneous functions
1530 -
1531 -void reload_host_labels(void);
1532 -void rrdhost_set_is_parent_label(void);
1533 -
1534 -// ----------------------------------------------------------------------------
1535 -// RRD internal functions
1536 -
1537 -void rrdset_free(RRDSET *st);
1538 -
1539 -void rrddim_free(RRDSET *st, RRDDIM *rd);
1540 -
1541 -#ifdef NETDATA_RRD_INTERNALS
1542 -
1543 -char *rrdhost_cache_dir_for_rrdset_alloc(RRDHOST *host, const char *id);
1544 -
1545 -void rrdset_reset(RRDSET *st);
1546 -
1547 -#endif /* NETDATA_RRD_INTERNALS */
1548 -
1549 -void set_host_properties(
1550 - RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname,
1551 - const char *os, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
1552 - const char *prog_name, const char *prog_version);
1553 -
1554 -void store_metric_collection_completed(void);
1555 -
1556 -static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to) {
1557 - time_t first_time_s = 0, last_time_s = 0;
1558 - spinlock_lock(&host->retention.spinlock);
1559 - first_time_s = host->retention.first_time_s;
1560 - last_time_s = host->retention.last_time_s;
1561 - spinlock_unlock(&host->retention.spinlock);
1562 -
1563 - if(from)
1564 - *from = first_time_s;
1565 -
1566 - if(to)
1567 - *to = online ? now : last_time_s;
1568 -}
1569 -
1570 -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);
1571 -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);
1572 -void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st);
1573 -void rrdset_pluginsd_receive_unslot(RRDSET *st);
1574 -
1575 -// ----------------------------------------------------------------------------
1576 -static inline double rrddim_get_last_stored_value(RRDDIM *rd_dim, double *max_value, double div) {
1577 - if (!rd_dim)
1578 - return NAN;
1579 -
1580 - if (isnan(div) || div == 0.0)
1581 - div = 1.0;
1582 -
1583 - double value = rd_dim->collector.last_stored_value / div;
1584 - value = ABS(value);
1585 -
1586 - *max_value = MAX(*max_value, value);
1587 -
1588 - return value;
1589 -}
1590 -
1591 -static inline uint32_t get_uint32_id()
1592 -{
1593 - return now_realtime_sec() & UINT32_MAX;
1594 -}
1595 -
1596 -//
1597 -// RRD DB engine declarations
1598 -
1599 -#ifdef ENABLE_DBENGINE
1600 -#include "database/engine/rrdengineapi.h"
1601 -#endif
1602 -#include "sqlite/sqlite_functions.h"
1603 -#include "sqlite/sqlite_context.h"
1604 -#include "sqlite/sqlite_metadata.h"
1605 -#include "sqlite/sqlite_aclk.h"
1606 -#include "sqlite/sqlite_aclk_alert.h"
1607 -#include "sqlite/sqlite_aclk_node.h"
1608 -#include "sqlite/sqlite_health.h"
1609 -
151 #ifdef __cplusplus
152 }
153 #endif
src/database/rrddim-backfill.c new
+75
@@ -0,0 +1,75 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrddim-backfill.h"
4 +#include "database/rrddim-collection.h"
5 +
6 +// ----------------------------------------------------------------------------
7 +// fill the gap of a tier
8 +
9 +void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
10 +
11 +bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {
12 + if(unlikely(tier >= nd_profile.storage_tiers)) return false;
13 +#ifdef ENABLE_DBENGINE
14 + if(default_backfill == RRD_BACKFILL_NONE) return false;
15 +#else
16 + return false;
17 +#endif
18 +
19 + struct rrddim_tier *t = &rd->tiers[tier];
20 + if(unlikely(!t)) return false;
21 +
22 + time_t latest_time_s = storage_engine_latest_time_s(t->seb, t->smh);
23 + time_t granularity = (time_t)t->tier_grouping * (time_t)rd->rrdset->update_every;
24 + time_t time_diff = now_s - latest_time_s;
25 +
26 + // if the user wants only NEW backfilling, and we don't have any data
27 +#ifdef ENABLE_DBENGINE
28 + if(default_backfill == RRD_BACKFILL_NEW && latest_time_s <= 0) return false;
29 +#else
30 + return;
31 +#endif
32 +
33 + // there is really nothing we can do
34 + if(now_s <= latest_time_s || time_diff < granularity) return false;
35 +
36 + stream_control_backfill_query_started();
37 +
38 + // for each lower tier
39 + struct storage_engine_query_handle seqh;
40 + for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){
41 + time_t smaller_tier_first_time = storage_engine_oldest_time_s(rd->tiers[read_tier].seb, rd->tiers[read_tier].smh);
42 + time_t smaller_tier_last_time = storage_engine_latest_time_s(rd->tiers[read_tier].seb, rd->tiers[read_tier].smh);
43 + if(smaller_tier_last_time <= latest_time_s) continue; // it is as bad as we are
44 +
45 + long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s;
46 + long before_wanted = smaller_tier_last_time;
47 +
48 + struct rrddim_tier *tmp = &rd->tiers[read_tier];
49 + storage_engine_query_init(tmp->seb, tmp->smh, &seqh, after_wanted, before_wanted, STORAGE_PRIORITY_SYNCHRONOUS_FIRST);
50 +
51 + size_t points_read = 0;
52 +
53 + while(!storage_engine_query_is_finished(&seqh)) {
54 +
55 + STORAGE_POINT sp = storage_engine_query_next_metric(&seqh);
56 + points_read++;
57 +
58 + if(sp.end_time_s > latest_time_s) {
59 + latest_time_s = sp.end_time_s;
60 + store_metric_at_tier(rd, tier, t, sp, sp.end_time_s * USEC_PER_SEC);
61 + }
62 + }
63 +
64 + storage_engine_query_finalize(&seqh);
65 + store_metric_collection_completed();
66 + pulse_queries_backfill_query_completed(points_read);
67 +
68 + //internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d",
69 + // rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr);
70 + }
71 +
72 + stream_control_backfill_query_finished();
73 +
74 + return true;
75 +}
src/database/rrddim-backfill.h new
+16
@@ -0,0 +1,16 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDDIM_BACKFILL_H
4 +#define NETDATA_RRDDIM_BACKFILL_H
5 +
6 +typedef enum __attribute__ ((__packed__)) {
7 + RRD_BACKFILL_NONE = 0,
8 + RRD_BACKFILL_FULL,
9 + RRD_BACKFILL_NEW
10 +} RRD_BACKFILL;
11 +
12 +#include "rrddim.h"
13 +
14 +bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s);
15 +
16 +#endif //NETDATA_RRDDIM_BACKFILL_H
src/database/rrddim-collection.c new
+74
@@ -0,0 +1,74 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrddim-collection.h"
4 +
5 +void store_metric_collection_completed() {
6 + pulse_queries_rrdset_collection_completed(rrdset_done_statistics_points_stored_per_tier);
7 +}
8 +
9 +static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, time_t now_s) {
10 + time_t loop = (time_t)rd->rrdset->update_every * (time_t)t->tier_grouping;
11 + return now_s + loop - ((now_s + loop) % loop);
12 +}
13 +
14 +void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {
15 + if (unlikely(!t->next_point_end_time_s))
16 + t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
17 +
18 + if(unlikely(sp.start_time_s >= t->next_point_end_time_s)) {
19 + // flush the virtual point, it is done
20 +
21 + if (likely(!storage_point_is_unset(t->virtual_point))) {
22 +
23 + storage_engine_store_metric(
24 + t->sch,
25 + t->next_point_end_time_s * USEC_PER_SEC,
26 + t->virtual_point.sum,
27 + t->virtual_point.min,
28 + t->virtual_point.max,
29 + t->virtual_point.count,
30 + t->virtual_point.anomaly_count,
31 + t->virtual_point.flags);
32 + }
33 + else {
34 + storage_engine_store_metric(
35 + t->sch,
36 + t->next_point_end_time_s * USEC_PER_SEC,
37 + NAN,
38 + NAN,
39 + NAN,
40 + 0,
41 + 0, SN_FLAG_NONE);
42 + }
43 +
44 + rrdset_done_statistics_points_stored_per_tier[tier]++;
45 + t->virtual_point.count = 0; // make the point unset
46 + t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
47 + }
48 +
49 + // merge the dates into our virtual point
50 + if (unlikely(sp.start_time_s < t->virtual_point.start_time_s))
51 + t->virtual_point.start_time_s = sp.start_time_s;
52 +
53 + if (likely(sp.end_time_s > t->virtual_point.end_time_s))
54 + t->virtual_point.end_time_s = sp.end_time_s;
55 +
56 + // merge the values into our virtual point
57 + if (likely(!storage_point_is_gap(sp))) {
58 + // we aggregate only non NULLs into higher tiers
59 +
60 + if (likely(!storage_point_is_unset(t->virtual_point))) {
61 + // merge the collected point to our virtual one
62 + t->virtual_point.sum += sp.sum;
63 + t->virtual_point.min = MIN(t->virtual_point.min, sp.min);
64 + t->virtual_point.max = MAX(t->virtual_point.max, sp.max);
65 + t->virtual_point.count += sp.count;
66 + t->virtual_point.anomaly_count += sp.anomaly_count;
67 + t->virtual_point.flags |= sp.flags;
68 + }
69 + else {
70 + // reset our virtual point to this one
71 + t->virtual_point = sp;
72 + }
73 + }
74 +}
src/database/rrddim-collection.h new
+11
@@ -0,0 +1,11 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDDIM_COLLECTION_H
4 +#define NETDATA_RRDDIM_COLLECTION_H
5 +
6 +#include "rrddim.h"
7 +
8 +void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
9 +void store_metric_collection_completed(void);
10 +
11 +#endif //NETDATA_RRDDIM_COLLECTION_H
src/database/rrddim.c
+1 -2
@@ -1,8 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
5 -#include "storage_engine.h"
4 +#include "storage-engine.h"
5
6 void rrddim_metadata_updated(RRDDIM *rd) {
7 rrdcontext_updated_rrddim(rd);
src/database/rrddim.h new
+254
@@ -0,0 +1,254 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDDIM_H
4 +#define NETDATA_RRDDIM_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +
8 +typedef struct rrddim RRDDIM;
9 +typedef struct rrddim_acquired RRDDIM_ACQUIRED;
10 +typedef struct ml_dimension rrd_ml_dimension_t;
11 +typedef struct rrdmetric_acquired RRDMETRIC_ACQUIRED;
12 +
13 +#include "rrdset.h"
14 +
15 +// options are permanent configuration options (no atomics to alter/access them)
16 +typedef enum __attribute__ ((__packed__)) rrddim_options {
17 + RRDDIM_OPTION_NONE = 0,
18 + RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
19 + RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
20 + RRDDIM_OPTION_BACKFILLED_HIGH_TIERS = (1 << 2), // when set, we have backfilled higher tiers
21 + RRDDIM_OPTION_UPDATED = (1 << 3), // single-threaded collector updated flag
22 +
23 + // this is 8-bit
24 +} RRDDIM_OPTIONS;
25 +
26 +#define rrddim_option_check(rd, option) ((rd)->collector.options & (option))
27 +#define rrddim_option_set(rd, option) (rd)->collector.options |= (option)
28 +#define rrddim_option_clear(rd, option) (rd)->collector.options &= ~(option)
29 +
30 +// flags are runtime changing status flags (atomics are required to alter/access them)
31 +typedef enum __attribute__ ((__packed__)) rrddim_flags {
32 + RRDDIM_FLAG_NONE = 0,
33 +
34 + RRDDIM_FLAG_OBSOLETE = (1 << 0), // this is marked by the collector/module as obsolete
35 + // No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at
36 + // least rrdset_free_obsolete_time seconds ago.
37 +
38 + RRDDIM_FLAG_ARCHIVED = (1 << 1),
39 + RRDDIM_FLAG_METADATA_UPDATE = (1 << 2), // Metadata needs to go to the database
40 +
41 + RRDDIM_FLAG_META_HIDDEN = (1 << 3), // Status of hidden option in the metadata database
42 + RRDDIM_FLAG_ML_MODEL_LOAD = (1 << 4), // Do ML LOAD for this dimension
43 +
44 + // this is 8 bit
45 +} RRDDIM_FLAGS;
46 +
47 +#define rrddim_flag_get(rd) atomic_flags_get(&((rd)->flags))
48 +#define rrddim_flag_check(rd, flag) atomic_flags_check(&((rd)->flags), flag)
49 +#define rrddim_flag_set(rd, flag) atomic_flags_set(&((rd)->flags), flag)
50 +#define rrddim_flag_clear(rd, flag) atomic_flags_clear(&((rd)->flags), flag)
51 +#define rrddim_flag_set_and_clear(rd, set, clear) atomic_flags_set_and_clear(&((rd)->flags), set, clear)
52 +
53 +struct rrddim {
54 + UUIDMAP_ID uuid;
55 +
56 + // ------------------------------------------------------------------------
57 + // dimension definition
58 +
59 + STRING *id; // the id of this dimension (for internal identification)
60 + STRING *name; // the name of this dimension (as presented to user)
61 +
62 + RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
63 + RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
64 + RRDDIM_FLAGS flags; // run time changing status flags
65 +
66 + int32_t multiplier; // the multiplier of the collected values
67 + int32_t divisor; // the divider of the collected values
68 +
69 + // ------------------------------------------------------------------------
70 + // operational state members
71 +
72 + struct rrdset *rrdset;
73 + rrd_ml_dimension_t *ml_dimension; // machine learning data about this dimension
74 +
75 + struct {
76 + RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
77 + bool collected;
78 + } rrdcontexts;
79 +
80 +#ifdef NETDATA_LOG_COLLECTION_ERRORS
81 + usec_t rrddim_store_metric_last_ut; // the timestamp we last called rrddim_store_metric()
82 + size_t rrddim_store_metric_count; // the rrddim_store_metric() counter
83 + const char *rrddim_store_metric_last_caller; // the name of the function that last called rrddim_store_metric()
84 +#endif
85 +
86 + // ------------------------------------------------------------------------
87 + // db mode RAM, ALLOC, NONE specifics
88 + // TODO - they should be managed by storage engine
89 + // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
90 +
91 + struct {
92 + size_t memsize; // the memory allocated for this dimension (without RRDDIM)
93 + storage_number *data; // the array of values
94 + } db;
95 +
96 + // ------------------------------------------------------------------------
97 + // streaming
98 +
99 + struct {
100 + struct {
101 + uint32_t sent_version;
102 + uint32_t dim_slot;
103 + } snd;
104 + } stream;
105 +
106 + // ------------------------------------------------------------------------
107 + // data collection members
108 +
109 + struct {
110 + RRDDIM_OPTIONS options; // permanent configuration options
111 +
112 + uint32_t counter; // the number of times we added values to this rrddim
113 +
114 + collected_number collected_value; // the current value, as collected - resets to 0 after being used
115 + collected_number collected_value_max; // the absolute maximum of the collected value
116 + collected_number last_collected_value; // the last value that was collected, after being processed
117 +
118 + struct timeval last_collected_time; // when was this dimension last updated
119 + // this is actual date time we updated the last_collected_value
120 + // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
121 +
122 + NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
123 + NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
124 +
125 + NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
126 + } collector;
127 +
128 + // ------------------------------------------------------------------------
129 +
130 + struct rrddim_tier tiers[]; // our tiers of databases
131 +};
132 +
133 +size_t rrddim_size(void);
134 +
135 +#define rrddim_id(rd) string2str((rd)->id)
136 +#define rrddim_name(rd) string2str((rd) ->name)
137 +
138 +#define rrddim_check_updated(rd) ((rd)->collector.options & RRDDIM_OPTION_UPDATED)
139 +#define rrddim_set_updated(rd) (rd)->collector.options |= RRDDIM_OPTION_UPDATED
140 +#define rrddim_clear_updated(rd) (rd)->collector.options &= ~RRDDIM_OPTION_UPDATED
141 +
142 +#define rrddim_foreach_read(rd, st) \
143 + dfe_start_read((st)->rrddim_root_index, rd)
144 +
145 +#define rrddim_foreach_done(rd) \
146 + dfe_done(rd)
147 +
148 +struct pluginsd_rrddim {
149 + RRDDIM_ACQUIRED *rda;
150 + RRDDIM *rd;
151 + const char *id;
152 +};
153 +
154 +static inline uint32_t rrddim_metadata_version(RRDDIM *rd) {
155 + // the metadata version of the dimension, is the version of the chart
156 + return rrdset_metadata_version(rd->rrdset);
157 +}
158 +
159 +static inline uint32_t rrddim_metadata_upstream_version(RRDDIM *rd) {
160 + return __atomic_load_n(&rd->stream.snd.sent_version, __ATOMIC_RELAXED);
161 +}
162 +
163 +void rrddim_metadata_updated(RRDDIM *rd);
164 +
165 +static inline void rrddim_metadata_exposed_upstream(RRDDIM *rd, uint32_t version) {
166 + __atomic_store_n(&rd->stream.snd.sent_version, version, __ATOMIC_RELAXED);
167 +}
168 +
169 +static inline void rrddim_metadata_exposed_upstream_clear(RRDDIM *rd) {
170 + __atomic_store_n(&rd->stream.snd.sent_version, 0, __ATOMIC_RELAXED);
171 +}
172 +
173 +static inline bool rrddim_check_upstream_exposed(RRDDIM *rd) {
174 + return rrddim_metadata_upstream_version(rd) != 0;
175 +}
176 +
177 +// the collector sets the exposed flag, but anyone can remove it
178 +// still, it can be removed, after the collector has finished
179 +// so, it is safe to check it without atomics
180 +static inline bool rrddim_check_upstream_exposed_collector(RRDDIM *rd) {
181 + return rd->rrdset->version == rd->stream.snd.sent_version;
182 +}
183 +
184 +void rrddim_index_init(RRDSET *st);
185 +void rrddim_index_destroy(RRDSET *st);
186 +
187 +time_t rrddim_first_entry_s(RRDDIM *rd);
188 +time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier);
189 +time_t rrddim_last_entry_s(RRDDIM *rd);
190 +time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier);
191 +
192 +RRDDIM *rrddim_add_custom(RRDSET *st
193 + , const char *id
194 + , const char *name
195 + , collected_number multiplier
196 + , collected_number divisor
197 + , RRD_ALGORITHM algorithm
198 + , RRD_MEMORY_MODE memory_mode
199 +);
200 +
201 +#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \
202 + rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
203 +
204 +int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name);
205 +int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
206 +int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier);
207 +int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor);
208 +
209 +RRDDIM *rrddim_find(RRDSET *st, const char *id);
210 +RRDDIM_ACQUIRED *rrddim_find_and_acquire(RRDSET *st, const char *id);
211 +RRDDIM *rrddim_acquired_to_rrddim(RRDDIM_ACQUIRED *rda);
212 +void rrddim_acquired_release(RRDDIM_ACQUIRED *rda);
213 +RRDDIM *rrddim_find_active(RRDSET *st, const char *id);
214 +
215 +int rrddim_hide(RRDSET *st, const char *id);
216 +int rrddim_unhide(RRDSET *st, const char *id);
217 +
218 +void rrddim_is_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd);
219 +void rrddim_isnot_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd);
220 +
221 +collected_number rrddim_timed_set_by_pointer(RRDSET *st, RRDDIM *rd, struct timeval collected_time, collected_number value);
222 +collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value);
223 +collected_number rrddim_set(RRDSET *st, const char *id, collected_number value);
224 +
225 +bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd);
226 +void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);
227 +void rrdhost_finalize_collection(RRDHOST *host);
228 +void rrd_finalize_collection_for_all_hosts(void);
229 +
230 +#ifdef NETDATA_LOG_COLLECTION_ERRORS
231 +#define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__)
232 +void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function);
233 +#else
234 +void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);
235 +#endif
236 +
237 +void rrddim_free(RRDSET *st, RRDDIM *rd);
238 +
239 +static inline double rrddim_get_last_stored_value(RRDDIM *rd_dim, double *max_value, double div) {
240 + if (!rd_dim)
241 + return NAN;
242 +
243 + if (isnan(div) || div == 0.0)
244 + div = 1.0;
245 +
246 + double value = rd_dim->collector.last_stored_value / div;
247 + value = ABS(value);
248 +
249 + *max_value = MAX(*max_value, value);
250 +
251 + return value;
252 +}
253 +
254 +#endif //NETDATA_RRDDIM_H
src/database/rrdfunctions-exporters.c
-2
@@ -1,7 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
4 -
3 #include "rrdfunctions-internals.h"
4 #include "rrdfunctions-exporters.h"
5
src/database/rrdfunctions-inflight.c
-2
@@ -1,7 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
4 -
3 #include "rrdcollector-internals.h"
4 #include "rrdfunctions-internals.h"
5 #include "rrdfunctions-inflight.h"
src/database/rrdfunctions.c
-2
@@ -1,7 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
4 -
3 #include "rrd.h"
4 #include "rrdfunctions-internals.h"
5
src/database/rrdhost-status.c renamed
+2 -2
@@ -1,8 +1,8 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "rrdhost-status.h"
4 -#include "stream-receiver-internals.h"
5 -#include "stream-sender-internals.h"
4 +#include "streaming/stream-receiver-internals.h"
5 +#include "streaming/stream-sender-internals.h"
6
7 ENUM_STR_MAP_DEFINE(RRDHOST_DB_STATUS) = {
8 { RRDHOST_DB_STATUS_INITIALIZING, "initializing" },
src/database/rrdhost-status.h renamed
+3 -3
@@ -80,9 +80,9 @@ ENUM_STR_DEFINE_FUNCTIONS_EXTERN(RRDHOST_DYNCFG_STATUS);
80 #define rrdhost_health_status_to_string(status) RRDHOST_HEALTH_STATUS_2str(status)
81 #define rrdhost_dyncfg_status_to_string(status) RRDHOST_DYNCFG_STATUS_2str(status)
82
83 -#include "stream-handshake.h"
84 -#include "stream-capabilities.h"
85 -#include "database/rrd.h"
83 +#include "streaming/stream-handshake.h"
84 +#include "streaming/stream-capabilities.h"
85 +#include "rrd.h"
86
87 typedef struct rrdhost_status_t {
88 RRDHOST *host;
src/database/rrdhost.c
+2 -106
@@ -1,6 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
4
5 #if RRD_STORAGE_TIERS != 5
@@ -34,7 +33,7 @@ RRDHOST *find_host_by_node_id(char *node_id) {
33 DICTIONARY *rrdhost_root_index = NULL;
34 static DICTIONARY *rrdhost_root_index_hostname = NULL;
35
37 -static inline void rrdhost_init() {
36 +void rrdhost_init() {
37 if(unlikely(!rrdhost_root_index)) {
38 rrdhost_root_index = dictionary_create_advanced(
39 DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
@@ -293,7 +292,7 @@ static void rrdhost_set_replication_parameters(RRDHOST *host, RRD_MEMORY_MODE me
292 }
293 }
294
296 -static RRDHOST *rrdhost_create(
295 +RRDHOST *rrdhost_create(
296 const char *hostname,
297 const char *registry_hostname,
298 const char *guid,
@@ -772,109 +771,6 @@ bool rrdhost_should_run_health(RRDHOST *host) {
771 return true;
772 }
773
775 -void api_v1_management_init(void);
776 -
777 -int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
778 - rrdhost_init();
779 -
780 - if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
781 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
782 - set_late_analytics_variables(system_info);
783 - fatal("Failed to initialize SQLite");
784 - }
785 -
786 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
787 - "Skipping SQLITE metadata initialization since memory mode is not dbengine");
788 - }
789 -
790 - if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
791 - error_report("Failed to initialize context metadata database");
792 - }
793 -
794 - if (unlikely(unittest)) {
795 - dbengine_enabled = true;
796 - }
797 - else {
798 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) {
799 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
800 - "DBENGINE: Initializing ...");
801 -
802 - netdata_conf_dbengine_init(hostname);
803 - }
804 - else
805 - nd_profile.storage_tiers = 1;
806 -
807 - if (!dbengine_enabled) {
808 - if (nd_profile.storage_tiers > 1) {
809 - nd_log(NDLS_DAEMON, NDLP_WARNING,
810 - "dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
811 - nd_profile.storage_tiers);
812 -
813 - nd_profile.storage_tiers = 1;
814 - }
815 -
816 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
817 - nd_log(NDLS_DAEMON, NDLP_WARNING,
818 - "dbengine is not enabled, but it has been given as the default db mode. "
819 - "Resetting db mode to alloc");
820 -
821 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
822 - }
823 - }
824 - }
825 -
826 - if(!unittest)
827 - metadata_sync_init();
828 -
829 - localhost = rrdhost_create(
830 - hostname
831 - , registry_get_this_machine_hostname()
832 - , registry_get_this_machine_guid()
833 - , os_type
834 - , netdata_configured_timezone
835 - , netdata_configured_abbrev_timezone
836 - , netdata_configured_utc_offset
837 - , program_name
838 - , NETDATA_VERSION
839 - ,
840 - nd_profile.update_every, default_rrd_history_entries
841 - , default_rrd_memory_mode
842 - , health_plugin_enabled()
843 - , stream_send.enabled
844 - , stream_send.parents.destination
845 - , stream_send.api_key
846 - , stream_send.send_charts_matching
847 - , stream_receive.replication.enabled
848 - , stream_receive.replication.period
849 - , stream_receive.replication.step
850 - , system_info
851 - , 1
852 - , 0
853 - );
854 -
855 - if (unlikely(!localhost))
856 - return 1;
857 -
858 - rrdhost_flag_set(localhost, RRDHOST_FLAG_COLLECTOR_ONLINE);
859 - object_state_activate(&localhost->state_id);
860 -
861 - ml_host_start(localhost);
862 - dyncfg_host_init(localhost);
863 -
864 - if(!unittest)
865 - health_plugin_init();
866 -
867 - global_functions_add();
868 -
869 - if (likely(system_info)) {
870 - detect_machine_guid_change(&localhost->host_id.uuid);
871 - sql_aclk_sync_init();
872 - api_v1_management_init();
873 - }
874 -
875 - return 0;
876 -}
877 -
774 // ----------------------------------------------------------------------------
775 // RRDHOST - free
776
src/database/rrdhost.h new
+472
@@ -0,0 +1,472 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDHOST_H
4 +#define NETDATA_RRDHOST_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +
8 +struct stream_thread;
9 +struct rrdset;
10 +
11 +typedef struct rrdhost RRDHOST;
12 +typedef struct ml_host rrd_ml_host_t;
13 +typedef struct rrdhost_acquired RRDHOST_ACQUIRED;
14 +
15 +#include "streaming/stream-traffic-types.h"
16 +#include "streaming/stream-sender-commit.h"
17 +#include "streaming/stream-replication-tracking.h"
18 +#include "streaming/stream-parents.h"
19 +#include "streaming/stream-path.h"
20 +#include "storage-engine.h"
21 +#include "streaming/stream-traffic-types.h"
22 +#include "rrdlabels.h"
23 +#include "health/health-alert-log.h"
24 +
25 +// ----------------------------------------------------------------------------
26 +// RRDHOST flags
27 +// use this for configuration flags, not for state control
28 +// flags are set/unset in a manner that is not thread safe
29 +// and may lead to missing information.
30 +
31 +typedef enum __attribute__ ((__packed__)) rrdhost_flags {
32 +
33 + // Careful not to overlap with rrdhost_options to avoid bugs if
34 + // rrdhost_flags_xxx is used instead of rrdhost_option_xxx or vice-versa
35 + // Orphan, Archived and Obsolete flags
36 +
37 + /*
38 + * 3 BASE FLAGS FOR HOSTS:
39 + *
40 + * - COLLECTOR_ONLINE = the collector is currently collecting data for this node
41 + * this is true FOR ALL KINDS OF NODES (including localhost, virtual hosts, children)
42 + *
43 + * - ORPHAN = the node had a collector online recently, but does not have it now
44 + *
45 + * - ARCHIVED = the node does not have data collection structures attached to it
46 + *
47 + */
48 +
49 + RRDHOST_FLAG_COLLECTOR_ONLINE = (1 << 7), // the collector of this host is online
50 + RRDHOST_FLAG_ORPHAN = (1 << 8), // this host is orphan (not receiving data)
51 + RRDHOST_FLAG_ARCHIVED = (1 << 9), // The host is archived, no collected charts yet
52 +
53 + RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS = (1 << 10), // the host has pending chart obsoletions
54 + RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS = (1 << 11), // the host has pending dimension obsoletions
55 +
56 + // Streaming sender
57 + RRDHOST_FLAG_STREAM_SENDER_INITIALIZED = (1 << 12), // the host has initialized streaming sender structures
58 + RRDHOST_FLAG_STREAM_SENDER_ADDED = (1 << 13), // When set, the sender thread is running
59 + RRDHOST_FLAG_STREAM_SENDER_CONNECTED = (1 << 14), // When set, the host is connected to a parent
60 + RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS = (1 << 15), // when set, rrdset_done() should push metrics to parent
61 + RRDHOST_FLAG_STREAM_SENDER_LOGGED_STATUS = (1 << 16), // when set, we have logged the status of metrics streaming
62 +
63 + // Health
64 + RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 17), // contains charts and dims with uninitialized variables
65 + RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 18), // the host has initialized health structures
66 +
67 + // Exporting
68 + RRDHOST_FLAG_EXPORTING_SEND = (1 << 19), // send it to external databases
69 + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 20), // don't send it to external databases
70 +
71 + // ACLK
72 + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 21), // when set, we should send ACLK stream context updates
73 + RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // Host should stream alerts
74 +
75 + // Metadata
76 + RRDHOST_FLAG_METADATA_UPDATE = (1 << 23), // metadata needs to be stored in the database
77 + RRDHOST_FLAG_METADATA_LABELS = (1 << 24), // metadata needs to be stored in the database
78 + RRDHOST_FLAG_METADATA_INFO = (1 << 25), // metadata needs to be stored in the database
79 + RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 26), // Context needs to be loaded
80 +
81 + RRDHOST_FLAG_METADATA_CLAIMID = (1 << 27), // metadata needs to be stored in the database
82 +
83 + RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 28), // set when the host has updated global functions
84 +} RRDHOST_FLAGS;
85 +
86 +#define rrdhost_flag_get(host) atomic_flags_get(&((host)->flags))
87 +#define rrdhost_flag_check(host, flag) atomic_flags_check(&((host)->flags), flag)
88 +#define rrdhost_flag_set(host, flag) atomic_flags_set(&((host)->flags), flag)
89 +#define rrdhost_flag_clear(host, flag) atomic_flags_clear(&((host)->flags), flag)
90 +#define rrdhost_flag_set_and_clear(host, set, clear) atomic_flags_set_and_clear(&((host)->flags), set, clear)
91 +
92 +typedef enum __attribute__ ((__packed__)) {
93 + // Indexing
94 + RRDHOST_OPTION_INDEXED_MACHINE_GUID = (1 << 0), // when set, we have indexed its machine guid
95 + RRDHOST_OPTION_INDEXED_HOSTNAME = (1 << 1), // when set, we have indexed its hostname
96 +
97 + // Streaming configuration
98 + RRDHOST_OPTION_SENDER_ENABLED = (1 << 2), // set when the host is configured to send metrics to a parent
99 + RRDHOST_OPTION_REPLICATION = (1 << 3), // when set, we support replication for this host
100 +
101 + // Other options
102 + RRDHOST_OPTION_VIRTUAL_HOST = (1 << 4), // when set, this host is a virtual one
103 + RRDHOST_OPTION_EPHEMERAL_HOST = (1 << 5), // when set, this host is an ephemeral one
104 +} RRDHOST_OPTIONS;
105 +
106 +#define rrdhost_option_check(host, flag) ((host)->options & (flag))
107 +#define rrdhost_option_set(host, flag) (host)->options |= flag
108 +#define rrdhost_option_clear(host, flag) (host)->options &= ~(flag)
109 +
110 +#define rrdhost_has_stream_sender_enabled(host) (rrdhost_option_check(host, RRDHOST_OPTION_SENDER_ENABLED) && (host)->sender)
111 +
112 +#define rrdhost_can_stream_metadata_to_parent(host) \
113 + (rrdhost_has_stream_sender_enabled(host) && \
114 + rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS) && \
115 + rrdhost_flag_check(host, RRDHOST_FLAG_COLLECTOR_ONLINE) \
116 + )
117 +
118 +
119 +struct rrdhost {
120 + char machine_guid[GUID_LEN + 1]; // the unique ID of this host
121 +
122 + // ------------------------------------------------------------------------
123 + // host information
124 +
125 + STRING *hostname; // the hostname of this host
126 + STRING *registry_hostname; // the registry hostname for this host
127 + STRING *os; // the O/S type of the host
128 + STRING *timezone; // the timezone of the host
129 + STRING *abbrev_timezone; // the abbriviated timezone of the host
130 + STRING *program_name; // the program name that collects metrics for this host
131 + STRING *program_version; // the program version that collects metrics for this host
132 +
133 + OBJECT_STATE state_id; // every time data collection (stream receiver) (dis)connects,
134 + // this gets incremented - it is used to detect stale functions,
135 + // stale backfilling requests, etc.
136 +
137 + int32_t utc_offset; // the offset in seconds from utc
138 +
139 + RRDHOST_OPTIONS options; // configuration option for this RRDHOST (no atomics on this)
140 + RRDHOST_FLAGS flags; // runtime flags about this RRDHOST (atomics on this)
141 + RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances
142 +
143 + int32_t rrd_update_every; // the update frequency of the host
144 + int32_t rrd_history_entries; // the number of history entries for the host's charts
145 +
146 + RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
147 + // the actual per tier is at .db[tier].mode
148 +
149 + char *cache_dir; // the directory to save RRD cache files
150 +
151 + struct {
152 + RRD_MEMORY_MODE mode; // the db mode for this tier
153 + STORAGE_ENGINE *eng; // the storage engine API for this tier
154 + STORAGE_INSTANCE *si; // the db instance for this tier
155 + uint32_t tier_grouping; // tier 0 iterations aggregated on this tier
156 + } db[RRD_STORAGE_TIERS];
157 +
158 + struct rrdhost_system_info *system_info; // information collected from the host environment
159 +
160 + // ------------------------------------------------------------------------
161 + // streaming and replication, configuration and status
162 +
163 + struct {
164 + struct stream_thread *thread;
165 + uint8_t refcount;
166 +
167 + // --- sender ---
168 +
169 + struct {
170 + struct {
171 + struct {
172 + SPINLOCK spinlock;
173 +
174 + bool ignore; // when set, freeing slots will not put them in the available
175 + uint32_t used;
176 + uint32_t size;
177 + uint32_t *array;
178 + } available; // keep track of the available chart slots per host
179 +
180 + uint32_t last_used; // the last slot we used for a chart (increments only)
181 + } pluginsd_chart_slots;
182 +
183 + struct {
184 + pid_t tid;
185 +
186 + time_t last_connected; // last time child connected (stored in db)
187 + uint32_t connections; // the number of times this sender has connected
188 +
189 + struct {
190 + uint32_t counter_in; // counts the number of replication statements we have received
191 + uint32_t counter_out; // counts the number of replication statements we have sent
192 + uint32_t charts; // the number of charts currently being replicated to a parent
193 + } replication;
194 + } status;
195 +
196 + // reserved for the receiver/sender thread - do not use for other purposes
197 + struct sender_buffer commit;
198 +
199 + STRING *destination; // where to send metrics to
200 + STRING *api_key; // the api key at the receiving netdata
201 + SIMPLE_PATTERN *charts_matching; // pattern to match the charts to be sent
202 + RRDHOST_STREAM_PARENTS parents; // the list of parents (extracted from destination)
203 + } snd;
204 +
205 + // --- receiver ---
206 +
207 + struct {
208 + struct {
209 + SPINLOCK spinlock; // lock for the management of the allocation
210 + uint32_t size;
211 + struct rrdset **array;
212 + } pluginsd_chart_slots;
213 +
214 + struct {
215 + pid_t tid;
216 +
217 + time_t last_connected; // the time the last sender was connected
218 + time_t last_disconnected; // the time the last sender was disconnected
219 + time_t last_chart; // the time of the last CHART streaming command
220 + bool check_obsolete; // set when child connects, will instruct parent to
221 + // trigger a check for obsoleted charts since previous connect
222 +
223 + uint32_t connections; // the number of times this receiver has connected
224 + STREAM_HANDSHAKE exit_reason; // the last receiver exit reason
225 +
226 + struct {
227 + uint32_t counter_in; // counts the number of replication statements we have received
228 + uint32_t counter_out; // counts the number of replication statements we have sent
229 + uint32_t backfill_pending; // the number of replication requests pending on us
230 + uint32_t charts; // the number of charts currently being replicated from a child
231 + NETDATA_DOUBLE percent; // the % of replication completion
232 + } replication;
233 + } status;
234 + } rcv;
235 +
236 + // --- configuration ---
237 +
238 + struct {
239 + time_t period; // max time we want to replicate from the child
240 + time_t step; // seconds per replication step
241 + } replication;
242 +
243 + RRDHOST_STREAM_PATH path;
244 + } stream;
245 +
246 + // the following are state information for the threading
247 + // streaming metrics from this netdata to an upstream netdata
248 + struct sender_state *sender;
249 +
250 + struct receiver_state *receiver;
251 + SPINLOCK receiver_lock;
252 +
253 + // ------------------------------------------------------------------------
254 +
255 + struct aclk_sync_cfg_t *aclk_config;
256 +
257 + // ------------------------------------------------------------------------
258 + // health monitoring options
259 +
260 + // health variables
261 + HEALTH health;
262 +
263 + // all RRDCALCs are primarily allocated and linked here
264 + DICTIONARY *rrdcalc_root_index;
265 +
266 + ALARM_LOG health_log; // alarms historical events (event log)
267 + uint32_t health_last_processed_id; // the last processed health id from the log
268 + uint32_t health_max_unique_id; // the max alarm log unique id given for the host
269 + uint32_t health_max_alarm_id; // the max alarm id given for the host
270 + size_t health_transitions; // the number of times an alert changed state
271 +
272 + // ------------------------------------------------------------------------
273 + // locks
274 +
275 + SPINLOCK rrdhost_update_lock;
276 +
277 + // ------------------------------------------------------------------------
278 + // ML handle
279 + rrd_ml_host_t *ml_host;
280 +
281 + // ------------------------------------------------------------------------
282 + // Support for host-level labels
283 + RRDLABELS *rrdlabels;
284 +
285 + // ------------------------------------------------------------------------
286 + // Support for functions
287 + DICTIONARY *functions; // collector functions this rrdset supports, can be NULL
288 +
289 + // ------------------------------------------------------------------------
290 + // indexes
291 +
292 + DICTIONARY *rrdset_root_index; // the host's charts index (by id)
293 + DICTIONARY *rrdset_root_index_name; // the host's charts index (by name)
294 +
295 + DICTIONARY *rrdvars; // the host's chart variables index
296 + // this includes custom host variables
297 +
298 + struct {
299 + uint32_t metrics_count; // atomic
300 + uint32_t instances_count; // atomic
301 + uint32_t contexts_count; // atomic
302 + } collected;
303 +
304 + struct {
305 + DICTIONARY *contexts;
306 + DICTIONARY *hub_queue;
307 + DICTIONARY *pp_queue;
308 + uint32_t metrics_count; // atomic
309 + uint32_t instances_count; // atomic
310 + uint32_t contexts_count; // atomic
311 + } rrdctx;
312 +
313 + struct {
314 + SPINLOCK spinlock;
315 + time_t first_time_s;
316 + time_t last_time_s;
317 + } retention;
318 +
319 + ND_UUID host_id; // Global GUID for this host
320 + ND_UUID node_id; // Cloud node_id
321 +
322 + struct {
323 + ND_UUID claim_id_of_origin;
324 + ND_UUID claim_id_of_parent;
325 + } aclk;
326 +
327 + struct rrdhost *next;
328 + struct rrdhost *prev;
329 +};
330 +
331 +extern RRDHOST *localhost;
332 +
333 +#define rrdhost_receiver_lock(host) spinlock_lock(&(host)->receiver_lock)
334 +#define rrdhost_receiver_unlock(host) spinlock_unlock(&(host)->receiver_lock)
335 +
336 +#define rrdhost_hostname(host) string2str((host)->hostname)
337 +#define rrdhost_registry_hostname(host) string2str((host)->registry_hostname)
338 +#define rrdhost_os(host) string2str((host)->os)
339 +#define rrdhost_timezone(host) string2str((host)->timezone)
340 +#define rrdhost_abbrev_timezone(host) string2str((host)->abbrev_timezone)
341 +#define rrdhost_program_name(host) string2str((host)->program_name)
342 +#define rrdhost_program_version(host) string2str((host)->program_version)
343 +
344 +#define rrdhost_receiver_replicating_charts(host) (__atomic_load_n(&((host)->stream.rcv.status.replication.charts), __ATOMIC_RELAXED))
345 +#define rrdhost_receiver_replicating_charts_plus_one(host) (__atomic_add_fetch(&((host)->stream.rcv.status.replication.charts), 1, __ATOMIC_RELAXED))
346 +#define rrdhost_receiver_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->stream.rcv.status.replication.charts), 1, __ATOMIC_RELAXED))
347 +#define rrdhost_receiver_replicating_charts_zero(host) (__atomic_store_n(&((host)->stream.rcv.status.replication.charts), 0, __ATOMIC_RELAXED))
348 +
349 +#define rrdhost_sender_replicating_charts(host) (__atomic_load_n(&((host)->stream.snd.status.replication.charts), __ATOMIC_RELAXED))
350 +#define rrdhost_sender_replicating_charts_plus_one(host) (__atomic_add_fetch(&((host)->stream.snd.status.replication.charts), 1, __ATOMIC_RELAXED))
351 +#define rrdhost_sender_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->stream.snd.status.replication.charts), 1, __ATOMIC_RELAXED))
352 +#define rrdhost_sender_replicating_charts_zero(host) (__atomic_store_n(&((host)->stream.snd.status.replication.charts), 0, __ATOMIC_RELAXED))
353 +
354 +#define rrdhost_is_local(host) ( \
355 + (host) == localhost || \
356 + rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) \
357 + )
358 +
359 +#define rrdhost_is_online(host) ( \
360 + rrdhost_is_local(host) || \
361 + (rrdhost_flag_check(host, RRDHOST_FLAG_COLLECTOR_ONLINE) && !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) \
362 + )
363 +
364 +bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now);
365 +
366 +extern DICTIONARY *rrdhost_root_index;
367 +size_t rrdhost_hosts_available(void);
368 +
369 +RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid);
370 +RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha);
371 +void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha);
372 +
373 +#define rrdhost_foreach_read(var) \
374 + for((var) = localhost; var ; (var) = (var)->next)
375 +
376 +#define rrdhost_foreach_write(var) \
377 + for((var) = localhost; var ; (var) = (var)->next)
378 +
379 +RRDHOST *rrdhost_find_by_hostname(const char *hostname);
380 +RRDHOST *rrdhost_find_by_guid(const char *guid);
381 +RRDHOST *find_host_by_node_id(char *node_id);
382 +
383 +#ifdef RRDHOST_INTERNALS
384 +RRDHOST *rrdhost_create(
385 + const char *hostname,
386 + const char *registry_hostname,
387 + const char *guid,
388 + const char *os,
389 + const char *timezone,
390 + const char *abbrev_timezone,
391 + int32_t utc_offset,
392 + const char *prog_name,
393 + const char *prog_version,
394 + int update_every,
395 + long entries,
396 + RRD_MEMORY_MODE memory_mode,
397 + bool health,
398 + bool stream,
399 + STRING *parents,
400 + STRING *api_key,
401 + STRING *send_charts_matching,
402 + bool replication,
403 + time_t replication_period,
404 + time_t replication_step,
405 + struct rrdhost_system_info *system_info,
406 + int is_localhost,
407 + bool archived
408 + );
409 +
410 +void rrdhost_init(void);
411 +#endif
412 +
413 +RRDHOST *rrdhost_find_or_create(
414 + const char *hostname,
415 + const char *registry_hostname,
416 + const char *guid,
417 + const char *os,
418 + const char *timezone,
419 + const char *abbrev_timezone,
420 + int32_t utc_offset,
421 + const char *prog_name,
422 + const char *prog_version,
423 + int update_every,
424 + long history,
425 + RRD_MEMORY_MODE mode,
426 + bool health,
427 + bool stream,
428 + STRING *parents,
429 + STRING *api_key,
430 + STRING *send_charts_matching,
431 + bool replication,
432 + time_t replication_period,
433 + time_t replication_step,
434 + struct rrdhost_system_info *system_info,
435 + bool is_archived);
436 +
437 +void rrdhost_free_all(void);
438 +
439 +void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force);
440 +
441 +bool rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s);
442 +bool rrdhost_should_run_health(RRDHOST *host);
443 +
444 +void reload_host_labels(void);
445 +void rrdhost_set_is_parent_label(void);
446 +
447 +void set_host_properties(
448 + RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname,
449 + const char *os, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
450 + const char *prog_name, const char *prog_version);
451 +
452 +static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to) {
453 + time_t first_time_s = 0, last_time_s = 0;
454 + spinlock_lock(&host->retention.spinlock);
455 + first_time_s = host->retention.first_time_s;
456 + last_time_s = host->retention.last_time_s;
457 + spinlock_unlock(&host->retention.spinlock);
458 +
459 + if(from)
460 + *from = first_time_s;
461 +
462 + if(to)
463 + *to = online ? now : last_time_s;
464 +}
465 +
466 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);
467 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);
468 +
469 +extern time_t rrdhost_free_orphan_time_s;
470 +extern time_t rrdhost_free_ephemeral_time_s;
471 +
472 +#endif //NETDATA_RRDHOST_H
src/database/rrdlabels.c
-1
@@ -1,6 +1,5 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
3 #include "rrd.h"
4
5 // Key OF HS ARRRAY
src/database/rrdset-collection.c new
+1062
@@ -0,0 +1,1062 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdset-collection.h"
4 +#include "rrddim-collection.h"
5 +
6 +time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
7 + if(unlikely(update_every_s == st->update_every))
8 + return st->update_every;
9 +
10 + internal_error(true, "RRDSET '%s' switching update every from %d to %d",
11 + rrdset_id(st), (int)st->update_every, (int)update_every_s);
12 +
13 + time_t prev_update_every_s = (time_t) st->update_every;
14 + st->update_every = (int) update_every_s;
15 +
16 + // switch update every to the storage engine
17 + RRDDIM *rd;
18 + rrddim_foreach_read(rd, st) {
19 + for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
20 + if (rd->tiers[tier].sch)
21 + storage_engine_store_change_collection_frequency(
22 + rd->tiers[tier].sch,
23 + (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
24 + }
25 + }
26 + rrddim_foreach_done(rd);
27 +
28 + return prev_update_every_s;
29 +}
30 +
31 +void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
32 + ND_LOG_STACK lgs[] = {
33 + ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(st->rrdhost)),
34 + ND_LOG_FIELD_TXT(NDF_NIDL_CONTEXT, rrdset_context(st)),
35 + ND_LOG_FIELD_TXT(NDF_NIDL_INSTANCE, rrdset_name(st)),
36 + ND_LOG_FIELD_END(),
37 + };
38 + ND_LOG_STACK_PUSH(lgs);
39 +
40 + RRDHOST *host = st->rrdhost;
41 +
42 + rrdset_flag_set(st, RRDSET_FLAG_COLLECTION_FINISHED);
43 +
44 + if(dimensions_too) {
45 + RRDDIM *rd;
46 + rrddim_foreach_read(rd, st)
47 + rrddim_finalize_collection_and_check_retention(rd);
48 + rrddim_foreach_done(rd);
49 + }
50 +
51 + for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
52 + STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
53 + if(!eng) continue;
54 +
55 + if(st->smg[tier]) {
56 + storage_engine_metrics_group_release(eng->seb, host->db[tier].si, st->smg[tier]);
57 + st->smg[tier] = NULL;
58 + }
59 + }
60 +
61 + rrdset_pluginsd_receive_unslot_and_cleanup(st);
62 +}
63 +
64 +// ----------------------------------------------------------------------------
65 +// RRDSET - reset a chart
66 +
67 +static void rrdset_collection_reset(RRDSET *st) {
68 + netdata_log_debug(D_RRD_CALLS, "rrdset_collection_reset() %s", rrdset_name(st));
69 +
70 + st->last_collected_time.tv_sec = 0;
71 + st->last_collected_time.tv_usec = 0;
72 + st->last_updated.tv_sec = 0;
73 + st->last_updated.tv_usec = 0;
74 + st->db.current_entry = 0;
75 + st->counter = 0;
76 + st->counter_done = 0;
77 +
78 + RRDDIM *rd;
79 + rrddim_foreach_read(rd, st) {
80 + rd->collector.last_collected_time.tv_sec = 0;
81 + rd->collector.last_collected_time.tv_usec = 0;
82 + rd->collector.counter = 0;
83 +
84 + if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
85 + for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++)
86 + storage_engine_store_flush(rd->tiers[tier].sch);
87 + }
88 + }
89 + rrddim_foreach_done(rd);
90 +}
91 +
92 +// ----------------------------------------------------------------------------
93 +// RRDSET - data collection iteration control
94 +
95 +static inline void last_collected_time_align(RRDSET *st) {
96 + st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
97 +
98 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
99 + st->last_collected_time.tv_usec = 0;
100 + else
101 + st->last_collected_time.tv_usec = 500000;
102 +}
103 +
104 +static inline void last_updated_time_align(RRDSET *st) {
105 + st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
106 + st->last_updated.tv_usec = 0;
107 +}
108 +
109 +void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_last_update) {
110 +#ifdef NETDATA_INTERNAL_CHECKS
111 + char *discard_reason = NULL;
112 + usec_t discarded = duration_since_last_update;
113 +#endif
114 +
115 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_SYNC_CLOCK))) {
116 + // the chart needs to be re-synced to current time
117 + rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
118 +
119 + // discard the duration supplied
120 + duration_since_last_update = 0;
121 +
122 +#ifdef NETDATA_INTERNAL_CHECKS
123 + if(!discard_reason) discard_reason = "SYNC CLOCK FLAG";
124 +#endif
125 + }
126 +
127 + if(unlikely(!st->last_collected_time.tv_sec)) {
128 + // the first entry
129 + duration_since_last_update = st->update_every * USEC_PER_SEC;
130 +#ifdef NETDATA_INTERNAL_CHECKS
131 + if(!discard_reason) discard_reason = "FIRST DATA COLLECTION";
132 +#endif
133 + }
134 + else if(unlikely(!duration_since_last_update)) {
135 + // no dt given by the plugin
136 + duration_since_last_update = dt_usec(&now, &st->last_collected_time);
137 +#ifdef NETDATA_INTERNAL_CHECKS
138 + if(!discard_reason) discard_reason = "NO USEC GIVEN BY COLLECTOR";
139 +#endif
140 + }
141 + else {
142 + // microseconds has the time since the last collection
143 + susec_t since_last_usec = dt_usec_signed(&now, &st->last_collected_time);
144 +
145 + if(unlikely(since_last_usec < 0)) {
146 +// oops! the database is in the future
147 +#ifdef NETDATA_INTERNAL_CHECKS
148 + netdata_log_info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
149 + " secs in the future (counter #%u, update #%u). Adjusting it to current time."
150 + , rrdset_id(st)
151 + , rrdhost_hostname(st->rrdhost)
152 + , (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC
153 + , st->counter
154 + , st->counter_done
155 + );
156 +#endif
157 +
158 + duration_since_last_update = 0;
159 +
160 +#ifdef NETDATA_INTERNAL_CHECKS
161 + if(!discard_reason) discard_reason = "COLLECTION TIME IN FUTURE";
162 +#endif
163 + }
164 + else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 5 * USEC_PER_SEC))) {
165 +// oops! the database is too far behind
166 +#ifdef NETDATA_INTERNAL_CHECKS
167 + netdata_log_info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
168 + " secs in the past (counter #%u, update #%u). Adjusting it to current time.",
169 + rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC,
170 + st->counter, st->counter_done);
171 +#endif
172 +
173 + duration_since_last_update = (usec_t)since_last_usec;
174 +
175 +#ifdef NETDATA_INTERNAL_CHECKS
176 + if(!discard_reason) discard_reason = "COLLECTION TIME TOO FAR IN THE PAST";
177 +#endif
178 + }
179 +
180 +#ifdef NETDATA_INTERNAL_CHECKS
181 + if(since_last_usec > 0 && (susec_t) duration_since_last_update < since_last_usec) {
182 + static __thread susec_t min_delta = USEC_PER_SEC * 3600, permanent_min_delta = 0;
183 + static __thread time_t last_time_s = 0;
184 +
185 + // the first time initialize it so that it will make the check later
186 + if(last_time_s == 0) last_time_s = now.tv_sec + 60;
187 +
188 + susec_t delta = since_last_usec - (susec_t) duration_since_last_update;
189 + if(delta < min_delta) min_delta = delta;
190 +
191 + if(now.tv_sec >= last_time_s + 60) {
192 + last_time_s = now.tv_sec;
193 +
194 + if(min_delta > permanent_min_delta) {
195 + netdata_log_info("MINIMUM MICROSECONDS DELTA of thread %d increased from %"PRIi64" to %"PRIi64" (+%"PRIi64")", gettid_cached(), permanent_min_delta, min_delta, min_delta - permanent_min_delta);
196 + permanent_min_delta = min_delta;
197 + }
198 +
199 + min_delta = USEC_PER_SEC * 3600;
200 + }
201 + }
202 +#endif
203 + }
204 +
205 + netdata_log_debug(D_RRD_CALLS, "rrdset_timed_next() for chart %s with duration since last update %"PRIu64" usec", rrdset_name(st), duration_since_last_update);
206 + rrdset_debug(st, "NEXT: %"PRIu64" microseconds", duration_since_last_update);
207 +
208 + internal_error(discarded && discarded != duration_since_last_update,
209 + "host '%s', chart '%s': discarded data collection time of %"PRIu64" usec, "
210 + "replaced with %"PRIu64" usec, reason: '%s'"
211 + , rrdhost_hostname(st->rrdhost)
212 + , rrdset_id(st)
213 + , discarded
214 + , duration_since_last_update
215 + , discard_reason?discard_reason:"UNDEFINED"
216 + );
217 +
218 + st->usec_since_last_update = duration_since_last_update;
219 +}
220 +
221 +inline void rrdset_next_usec_unfiltered(RRDSET *st, usec_t duration_since_last_update) {
222 + if(unlikely(!st->last_collected_time.tv_sec || !duration_since_last_update || (rrdset_flag_check(st, RRDSET_FLAG_SYNC_CLOCK)))) {
223 + // call the full next_usec() function
224 + rrdset_next_usec(st, duration_since_last_update);
225 + return;
226 + }
227 +
228 + st->usec_since_last_update = duration_since_last_update;
229 +}
230 +
231 +inline void rrdset_next_usec(RRDSET *st, usec_t duration_since_last_update) {
232 + struct timeval now;
233 +
234 + now_realtime_timeval(&now);
235 + rrdset_timed_next(st, now, duration_since_last_update);
236 +}
237 +
238 +// ----------------------------------------------------------------------------
239 +// RRDSET - process the collected values for all dimensions of a chart
240 +
241 +static inline usec_t rrdset_init_last_collected_time(RRDSET *st, struct timeval now) {
242 + st->last_collected_time = now;
243 + last_collected_time_align(st);
244 +
245 + usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
246 +
247 + rrdset_debug(st, "initialized last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
248 +
249 + return last_collect_ut;
250 +}
251 +
252 +static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
253 + usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
254 + usec_t ut = last_collect_ut + st->usec_since_last_update;
255 + st->last_collected_time.tv_sec = (time_t) (ut / USEC_PER_SEC);
256 + st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC);
257 +
258 + rrdset_debug(st, "updated last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
259 +
260 + return last_collect_ut;
261 +}
262 +
263 +static inline void rrdset_init_last_updated_time(RRDSET *st) {
264 + // copy the last collected time to last updated time
265 + st->last_updated.tv_sec = st->last_collected_time.tv_sec;
266 + st->last_updated.tv_usec = st->last_collected_time.tv_usec;
267 +
268 + if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
269 + st->last_updated.tv_sec -= st->update_every;
270 +
271 + last_updated_time_align(st);
272 +}
273 +
274 +__thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];
275 +
276 +#ifdef NETDATA_LOG_COLLECTION_ERRORS
277 +void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) {
278 +#else // !NETDATA_LOG_COLLECTION_ERRORS
279 +void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
280 +#endif // !NETDATA_LOG_COLLECTION_ERRORS
281 +
282 + static __thread struct log_stack_entry lgs[] = {
283 + [0] = ND_LOG_FIELD_STR(NDF_NIDL_DIMENSION, NULL),
284 + [1] = ND_LOG_FIELD_END(),
285 + };
286 + lgs[0].str = rd->id;
287 + log_stack_push(lgs);
288 +
289 +#ifdef NETDATA_LOG_COLLECTION_ERRORS
290 + rd->rrddim_store_metric_count++;
291 +
292 + if(likely(rd->rrddim_store_metric_count > 1)) {
293 + usec_t expected = rd->rrddim_store_metric_last_ut + rd->update_every * USEC_PER_SEC;
294 +
295 + if(point_end_time_ut != rd->rrddim_store_metric_last_ut) {
296 + internal_error(true,
297 + "%s COLLECTION: 'host:%s/chart:%s/dim:%s' granularity %d, collection %zu, expected to store at tier 0 a value at %llu, but it gave %llu [%s%llu usec] (called from %s(), previously by %s())",
298 + (point_end_time_ut < rd->rrddim_store_metric_last_ut) ? "**PAST**" : "GAP",
299 + rrdhost_hostname(rd->rrdset->rrdhost), rrdset_id(rd->rrdset), rrddim_id(rd),
300 + rd->update_every,
301 + rd->rrddim_store_metric_count,
302 + expected, point_end_time_ut,
303 + (point_end_time_ut < rd->rrddim_store_metric_last_ut)?"by -" : "gap ",
304 + expected - point_end_time_ut,
305 + function,
306 + rd->rrddim_store_metric_last_caller?rd->rrddim_store_metric_last_caller:"none");
307 + }
308 + }
309 +
310 + rd->rrddim_store_metric_last_ut = point_end_time_ut;
311 + rd->rrddim_store_metric_last_caller = function;
312 +#endif // NETDATA_LOG_COLLECTION_ERRORS
313 +
314 + // store the metric on tier 0
315 + storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut,
316 + n, 0, 0,
317 + 1, 0, flags);
318 +
319 + rrdset_done_statistics_points_stored_per_tier[0]++;
320 +
321 + time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC);
322 +
323 + STORAGE_POINT sp = {
324 + .start_time_s = now_s - rd->rrdset->update_every,
325 + .end_time_s = now_s,
326 + .min = n,
327 + .max = n,
328 + .sum = n,
329 + .count = 1,
330 + .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,
331 + .flags = flags
332 + };
333 +
334 + for(size_t tier = 1; tier < nd_profile.storage_tiers;tier++) {
335 + if(unlikely(!rd->tiers[tier].smh)) continue;
336 +
337 + struct rrddim_tier *t = &rd->tiers[tier];
338 +
339 + if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) {
340 + // we have not collected this tier before
341 + // let's fill any gap that may exist
342 + backfill_tier_from_smaller_tiers(rd, tier, now_s);
343 + }
344 +
345 + store_metric_at_tier(rd, tier, t, sp, point_end_time_ut);
346 + }
347 + rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS);
348 +
349 + rrdcontext_collected_rrddim(rd);
350 + log_stack_pop(&lgs);
351 +}
352 +
353 +// caching of dimensions rrdset_done() and rrdset_done_interpolate() loop through
354 +struct rda_item {
355 + const DICTIONARY_ITEM *item;
356 + RRDDIM *rd;
357 + bool reset_or_overflow;
358 +};
359 +
360 +static __thread struct rda_item *thread_rda = NULL;
361 +static __thread size_t thread_rda_entries = 0;
362 +
363 +struct rda_item *rrdset_thread_rda_get(size_t *dimensions) {
364 +
365 + if(unlikely(!thread_rda || (*dimensions) > thread_rda_entries)) {
366 + size_t old_mem = thread_rda_entries * sizeof(struct rda_item);
367 + freez(thread_rda);
368 + thread_rda_entries = *dimensions;
369 + size_t new_mem = thread_rda_entries * sizeof(struct rda_item);
370 + thread_rda = mallocz(new_mem);
371 +
372 + __atomic_add_fetch(&netdata_buffers_statistics.rrdset_done_rda_size, new_mem - old_mem, __ATOMIC_RELAXED);
373 + }
374 +
375 + *dimensions = thread_rda_entries;
376 + return thread_rda;
377 +}
378 +
379 +void rrdset_thread_rda_free(void) {
380 + __atomic_sub_fetch(&netdata_buffers_statistics.rrdset_done_rda_size, thread_rda_entries * sizeof(struct rda_item), __ATOMIC_RELAXED);
381 +
382 + freez(thread_rda);
383 + thread_rda = NULL;
384 + thread_rda_entries = 0;
385 +}
386 +
387 +static inline size_t rrdset_done_interpolate(
388 + RRDSET_STREAM_BUFFER *rsb
389 + , RRDSET *st
390 + , struct rda_item *rda_base
391 + , size_t rda_slots
392 + , usec_t update_every_ut
393 + , usec_t last_stored_ut
394 + , usec_t next_store_ut
395 + , usec_t last_collect_ut
396 + , usec_t now_collect_ut
397 + , char store_this_entry
398 +) {
399 + RRDDIM *rd;
400 +
401 + size_t stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done()
402 +
403 + usec_t first_ut = last_stored_ut, last_ut = 0;
404 + (void)first_ut;
405 +
406 + ssize_t iterations = (ssize_t)((now_collect_ut - last_stored_ut) / (update_every_ut));
407 + if((now_collect_ut % (update_every_ut)) == 0) iterations++;
408 +
409 + size_t counter = st->counter;
410 + long current_entry = st->db.current_entry;
411 +
412 + for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
413 +
414 + internal_error(iterations < 0,
415 + "RRDSET: '%s': iterations calculation wrapped! "
416 + "first_ut = %"PRIu64", last_stored_ut = %"PRIu64", next_store_ut = %"PRIu64", now_collect_ut = %"PRIu64""
417 + , rrdset_id(st)
418 + , first_ut
419 + , last_stored_ut
420 + , next_store_ut
421 + , now_collect_ut
422 + );
423 +
424 + rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
425 + rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
426 +
427 + last_ut = next_store_ut;
428 +
429 + ml_chart_update_begin(st);
430 +
431 + struct rda_item *rda;
432 + size_t dim_id;
433 + for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
434 + rd = rda->rd;
435 + if(unlikely(!rd)) continue;
436 +
437 + SN_FLAGS storage_flags = SN_DEFAULT_FLAGS;
438 +
439 + if (rda->reset_or_overflow)
440 + storage_flags |= SN_FLAG_RESET;
441 +
442 + NETDATA_DOUBLE new_value;
443 +
444 + switch(rd->algorithm) {
445 + case RRD_ALGORITHM_INCREMENTAL:
446 + new_value = (NETDATA_DOUBLE)
447 + ( rd->collector.calculated_value
448 + * (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
449 + / (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
450 + );
451 +
452 + rrdset_debug(st, "%s: CALC2 INC " NETDATA_DOUBLE_FORMAT " = "
453 + NETDATA_DOUBLE_FORMAT
454 + " * (%"PRIu64" - %"PRIu64")"
455 + " / (%"PRIu64" - %"PRIu64""
456 + , rrddim_name(rd)
457 + , new_value
458 + , rd->collector.calculated_value
459 + , next_store_ut, last_collect_ut
460 + , now_collect_ut, last_collect_ut
461 + );
462 +
463 + rd->collector.calculated_value -= new_value;
464 + new_value += rd->collector.last_calculated_value;
465 + rd->collector.last_calculated_value = 0;
466 + new_value /= (NETDATA_DOUBLE)st->update_every;
467 +
468 + if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
469 +
470 + rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
471 + rrddim_name(rd)
472 + , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
473 + );
474 +
475 + new_value = new_value * (NETDATA_DOUBLE)(st->update_every * USEC_PER_SEC) / (NETDATA_DOUBLE)(next_store_ut - last_stored_ut);
476 + }
477 + break;
478 +
479 + case RRD_ALGORITHM_ABSOLUTE:
480 + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
481 + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
482 + default:
483 + if(iterations == 1) {
484 + // this is the last iteration
485 + // do not interpolate
486 + // just show the calculated value
487 +
488 + new_value = rd->collector.calculated_value;
489 + }
490 + else {
491 + // we have missed an update
492 + // interpolate in the middle values
493 +
494 + new_value = (NETDATA_DOUBLE)
495 + ( ( (rd->collector.calculated_value - rd->collector.last_calculated_value)
496 + * (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
497 + / (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
498 + )
499 + + rd->collector.last_calculated_value
500 + );
501 +
502 + rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = ((("
503 + "(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")"
504 + " * %"PRIu64""
505 + " / %"PRIu64") + " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
506 + , new_value
507 + , rd->collector.calculated_value, rd->collector.last_calculated_value
508 + , (next_store_ut - first_ut)
509 + , (now_collect_ut - first_ut), rd->collector.last_calculated_value
510 + );
511 + }
512 + break;
513 + }
514 +
515 + time_t current_time_s = (time_t) (next_store_ut / USEC_PER_SEC);
516 +
517 + if(unlikely(!store_this_entry)) {
518 + (void) ml_dimension_is_anomalous(rd, current_time_s, 0, false);
519 +
520 + if(rsb->wb && rsb->v2)
521 + stream_send_rrddim_metrics_v2(rsb, rd, next_store_ut, NAN, SN_FLAG_NONE);
522 +
523 + rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
524 + continue;
525 + }
526 +
527 + if(likely(rrddim_check_updated(rd) && rd->collector.counter > 1 && iterations < gap_when_lost_iterations_above)) {
528 + uint32_t dim_storage_flags = storage_flags;
529 +
530 + if (ml_dimension_is_anomalous(rd, current_time_s, new_value, true)) {
531 + // clear anomaly bit: 0 -> is anomalous, 1 -> not anomalous
532 + dim_storage_flags &= ~((storage_number)SN_FLAG_NOT_ANOMALOUS);
533 + }
534 +
535 + if(rsb->wb && rsb->v2)
536 + stream_send_rrddim_metrics_v2(rsb, rd, next_store_ut, new_value, dim_storage_flags);
537 +
538 + rrddim_store_metric(rd, next_store_ut, new_value, dim_storage_flags);
539 + rd->collector.last_stored_value = new_value;
540 + }
541 + else {
542 + (void) ml_dimension_is_anomalous(rd, current_time_s, 0, false);
543 +
544 + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
545 +
546 + if(rsb->wb && rsb->v2)
547 + stream_send_rrddim_metrics_v2(rsb, rd, next_store_ut, NAN, SN_FLAG_NONE);
548 +
549 + rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
550 + rd->collector.last_stored_value = NAN;
551 + }
552 +
553 + stored_entries++;
554 + }
555 +
556 + ml_chart_update_end(st);
557 +
558 + st->counter = ++counter;
559 + st->db.current_entry = current_entry = ((current_entry + 1) >= st->db.entries) ? 0 : current_entry + 1;
560 +
561 + st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC);
562 + st->last_updated.tv_usec = 0;
563 +
564 + last_stored_ut = next_store_ut;
565 + }
566 +
567 + /*
568 + st->counter = counter;
569 + st->current_entry = current_entry;
570 +
571 + if(likely(last_ut)) {
572 + st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC);
573 + st->last_updated.tv_usec = 0;
574 + }
575 +*/
576 +
577 + return stored_entries;
578 +}
579 +
580 +void rrdset_done(RRDSET *st) {
581 + struct timeval now;
582 +
583 + now_realtime_timeval(&now);
584 + rrdset_timed_done(st, now, /* pending_rrdset_next = */ st->counter_done != 0);
585 +}
586 +
587 +void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) {
588 + if(unlikely(!service_running(SERVICE_COLLECTORS))) return;
589 +
590 + RRDSET_STREAM_BUFFER stream_buffer = { .wb = NULL, };
591 + if(unlikely(rrdhost_has_stream_sender_enabled(st->rrdhost)))
592 + stream_buffer = stream_send_metrics_init(st, now.tv_sec);
593 +
594 + spinlock_lock(&st->data_collection_lock);
595 +
596 + if (pending_rrdset_next)
597 + rrdset_timed_next(st, now, 0ULL);
598 +
599 + netdata_log_debug(D_RRD_CALLS, "rrdset_done() for chart '%s'", rrdset_name(st));
600 +
601 + RRDDIM *rd;
602 +
603 + char
604 + store_this_entry = 1, // boolean: 1 = store this entry, 0 = don't store this entry
605 + first_entry = 0; // boolean: 1 = this is the first entry seen for this chart, 0 = all other entries
606 +
607 + usec_t
608 + last_collect_ut = 0, // the timestamp in microseconds, of the last collected value
609 + now_collect_ut = 0, // the timestamp in microseconds, of this collected value (this is NOW)
610 + last_stored_ut = 0, // the timestamp in microseconds, of the last stored entry in the db
611 + next_store_ut = 0, // the timestamp in microseconds, of the next entry to store in the db
612 + update_every_ut = st->update_every * USEC_PER_SEC; // st->update_every in microseconds
613 +
614 + RRDSET_FLAGS rrdset_flags = rrdset_flag_check(st, ~0);
615 + if(unlikely(rrdset_flags & RRDSET_FLAG_COLLECTION_FINISHED)) {
616 + spinlock_unlock(&st->data_collection_lock);
617 + return;
618 + }
619 +
620 + if (unlikely(rrdset_flags & RRDSET_FLAG_OBSOLETE)) {
621 + netdata_log_error("Chart '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st));
622 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
623 + }
624 +
625 + // check if the chart has a long time to be updated
626 + if(unlikely(st->usec_since_last_update > MAX(st->db.entries, 60) * update_every_ut)) {
627 + nd_log_daemon(NDLP_DEBUG, "host '%s', chart '%s': took too long to be updated (counter #%u, update #%u, %0.3" NETDATA_DOUBLE_MODIFIER
628 + " secs). Resetting it.", rrdhost_hostname(st->rrdhost), rrdset_id(st), st->counter, st->counter_done,
629 + (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
630 + rrdset_collection_reset(st);
631 + st->usec_since_last_update = update_every_ut;
632 + store_this_entry = 0;
633 + first_entry = 1;
634 + }
635 +
636 + rrdset_debug(st, "microseconds since last update: %"PRIu64"", st->usec_since_last_update);
637 +
638 + // set last_collected_time
639 + if(unlikely(!st->last_collected_time.tv_sec)) {
640 + // it is the first entry
641 + // set the last_collected_time to now
642 + last_collect_ut = rrdset_init_last_collected_time(st, now) - update_every_ut;
643 +
644 + // the first entry should not be stored
645 + store_this_entry = 0;
646 + first_entry = 1;
647 + }
648 + else {
649 + // it is not the first entry
650 + // calculate the proper last_collected_time, using usec_since_last_update
651 + last_collect_ut = rrdset_update_last_collected_time(st);
652 + }
653 +
654 + // if this set has not been updated in the past
655 + // we fake the last_update time to be = now - usec_since_last_update
656 + if(unlikely(!st->last_updated.tv_sec)) {
657 + // it has never been updated before
658 + // set a fake last_updated, in the past using usec_since_last_update
659 + rrdset_init_last_updated_time(st);
660 +
661 + // the first entry should not be stored
662 + store_this_entry = 0;
663 + first_entry = 1;
664 + }
665 +
666 + // check if we will re-write the entire data set
667 + if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&
668 + st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
669 + nd_log_daemon(NDLP_DEBUG, "'%s': too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). "
670 + "Resetting it. Will not store the next entry.",
671 + rrdset_id(st),
672 + (int64_t)st->last_updated.tv_sec,
673 + (int64_t)st->last_updated.tv_usec,
674 + (int64_t)st->last_collected_time.tv_sec,
675 + (int64_t)st->last_collected_time.tv_usec);
676 + rrdset_collection_reset(st);
677 + rrdset_init_last_updated_time(st);
678 +
679 + st->usec_since_last_update = update_every_ut;
680 +
681 + // the first entry should not be stored
682 + store_this_entry = 0;
683 + first_entry = 1;
684 + }
685 +
686 + // these are the 3 variables that will help us in interpolation
687 + // last_stored_ut = the last time we added a value to the storage
688 + // now_collect_ut = the time the current value has been collected
689 + // next_store_ut = the time of the next interpolation point
690 + now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
691 + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
692 + next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
693 +
694 + if(unlikely(!st->counter_done)) {
695 + // set a fake last_updated to jump to current time
696 + rrdset_init_last_updated_time(st);
697 +
698 + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
699 + next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
700 +
701 + if(unlikely(rrdset_flags & RRDSET_FLAG_STORE_FIRST)) {
702 + store_this_entry = 1;
703 + last_collect_ut = next_store_ut - update_every_ut;
704 +
705 + rrdset_debug(st, "Fixed first entry.");
706 + }
707 + else {
708 + store_this_entry = 0;
709 +
710 + rrdset_debug(st, "Will not store the next entry.");
711 + }
712 + }
713 +
714 + st->counter_done++;
715 +
716 + if(stream_buffer.wb && !stream_buffer.v2)
717 + stream_send_rrdset_metrics_v1(&stream_buffer, st);
718 +
719 + size_t rda_slots = dictionary_entries(st->rrddim_root_index);
720 + struct rda_item *rda_base = rrdset_thread_rda_get(&rda_slots);
721 +
722 + size_t dim_id;
723 + size_t dimensions = 0;
724 + struct rda_item *rda = rda_base;
725 + total_number collected_total = 0;
726 + total_number last_collected_total = 0;
727 + rrddim_foreach_read(rd, st) {
728 + if(rd_dfe.counter >= rda_slots)
729 + break;
730 +
731 + rda = &rda_base[dimensions++];
732 +
733 + if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
734 + rda->item = NULL;
735 + rda->rd = NULL;
736 + rda->reset_or_overflow = false;
737 + continue;
738 + }
739 +
740 + // store the dimension in the array
741 + rda->item = dictionary_acquired_item_dup(st->rrddim_root_index, rd_dfe.item);
742 + rda->rd = dictionary_acquired_item_value(rda->item);
743 + rda->reset_or_overflow = false;
744 +
745 + // calculate totals
746 + if(likely(rrddim_check_updated(rd))) {
747 + // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
748 + // to reset the calculation (it will give zero as the calculation for this second)
749 + if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->collector.last_collected_value > rd->collector.collected_value)) {
750 + netdata_log_debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
751 + , rrdset_id(st)
752 + , rrddim_name(rd)
753 + , rd->collector.last_collected_value
754 + , rd->collector.collected_value
755 + );
756 +
757 + if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
758 + rda->reset_or_overflow = true;
759 +
760 + rd->collector.last_collected_value = rd->collector.collected_value;
761 + }
762 +
763 + last_collected_total += rd->collector.last_collected_value;
764 + collected_total += rd->collector.collected_value;
765 +
766 + if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
767 + netdata_log_error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
768 + rrddim_isnot_obsolete___safe_from_collector_thread(st, rd);
769 + }
770 + }
771 + }
772 + rrddim_foreach_done(rd);
773 + rda_slots = dimensions;
774 +
775 + rrdset_debug(st, "last_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last collection time)", (NETDATA_DOUBLE)last_collect_ut/USEC_PER_SEC);
776 + rrdset_debug(st, "now_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (current collection time)", (NETDATA_DOUBLE)now_collect_ut/USEC_PER_SEC);
777 + rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
778 + rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
779 +
780 + // process all dimensions to calculate their values
781 + // based on the collected figures only
782 + // at this stage we do not interpolate anything
783 + for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
784 + rd = rda->rd;
785 + if(unlikely(!rd)) continue;
786 +
787 + if(unlikely(!rrddim_check_updated(rd))) {
788 + rd->collector.calculated_value = 0;
789 + continue;
790 + }
791 +
792 + rrdset_debug(st, "%s: START "
793 + " last_collected_value = " COLLECTED_NUMBER_FORMAT
794 + " collected_value = " COLLECTED_NUMBER_FORMAT
795 + " last_calculated_value = " NETDATA_DOUBLE_FORMAT
796 + " calculated_value = " NETDATA_DOUBLE_FORMAT
797 + , rrddim_name(rd)
798 + , rd->collector.last_collected_value
799 + , rd->collector.collected_value
800 + , rd->collector.last_calculated_value
801 + , rd->collector.calculated_value
802 + );
803 +
804 + switch(rd->algorithm) {
805 + case RRD_ALGORITHM_ABSOLUTE:
806 + rd->collector.calculated_value = (NETDATA_DOUBLE)rd->collector.collected_value
807 + * (NETDATA_DOUBLE)rd->multiplier
808 + / (NETDATA_DOUBLE)rd->divisor;
809 +
810 + rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " NETDATA_DOUBLE_FORMAT " = "
811 + COLLECTED_NUMBER_FORMAT
812 + " * " NETDATA_DOUBLE_FORMAT
813 + " / " NETDATA_DOUBLE_FORMAT
814 + , rrddim_name(rd)
815 + , rd->collector.calculated_value
816 + , rd->collector.collected_value
817 + , (NETDATA_DOUBLE)rd->multiplier
818 + , (NETDATA_DOUBLE)rd->divisor
819 + );
820 + break;
821 +
822 + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
823 + if(unlikely(!collected_total))
824 + rd->collector.calculated_value = 0;
825 + else
826 + // the percentage of the current value
827 + // over the total of all dimensions
828 + rd->collector.calculated_value =
829 + (NETDATA_DOUBLE)100
830 + * (NETDATA_DOUBLE)rd->collector.collected_value
831 + / (NETDATA_DOUBLE)collected_total;
832 +
833 + rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
834 + " * " COLLECTED_NUMBER_FORMAT
835 + " / " COLLECTED_NUMBER_FORMAT
836 + , rrddim_name(rd)
837 + , rd->collector.calculated_value
838 + , rd->collector.collected_value
839 + , collected_total
840 + );
841 + break;
842 +
843 + case RRD_ALGORITHM_INCREMENTAL:
844 + if(unlikely(rd->collector.counter <= 1)) {
845 + rd->collector.calculated_value = 0;
846 + continue;
847 + }
848 +
849 + // If the new is smaller than the old (an overflow, or reset), set the old equal to the new
850 + // to reset the calculation (it will give zero as the calculation for this second).
851 + // It is imperative to set the comparison to uint64_t since type collected_number is signed and
852 + // produces wrong results as far as incremental counters are concerned.
853 + if(unlikely((uint64_t)rd->collector.last_collected_value > (uint64_t)rd->collector.collected_value)) {
854 + netdata_log_debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
855 + , rrdset_id(st)
856 + , rrddim_name(rd)
857 + , rd->collector.last_collected_value
858 + , rd->collector.collected_value);
859 +
860 + if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
861 + rda->reset_or_overflow = true;
862 +
863 + uint64_t last = (uint64_t)rd->collector.last_collected_value;
864 + uint64_t new = (uint64_t)rd->collector.collected_value;
865 + uint64_t max = (uint64_t)rd->collector.collected_value_max;
866 + uint64_t cap = 0;
867 +
868 + // Signed values are handled by exploiting two's complement which will produce positive deltas
869 + if (max > 0x00000000FFFFFFFFULL)
870 + cap = 0xFFFFFFFFFFFFFFFFULL; // handles signed and unsigned 64-bit counters
871 + else
872 + cap = 0x00000000FFFFFFFFULL; // handles signed and unsigned 32-bit counters
873 +
874 + uint64_t delta = cap - last + new;
875 + uint64_t max_acceptable_rate = (cap / 100) * MAX_INCREMENTAL_PERCENT_RATE;
876 +
877 + // If the delta is less than the maximum acceptable rate and the previous value was near the cap
878 + // then this is an overflow. There can be false positives such that a reset is detected as an
879 + // overflow.
880 + // TODO: remember recent history of rates and compare with current rate to reduce this chance.
881 + if (delta < max_acceptable_rate) {
882 + rd->collector.calculated_value +=
883 + (NETDATA_DOUBLE) delta
884 + * (NETDATA_DOUBLE) rd->multiplier
885 + / (NETDATA_DOUBLE) rd->divisor;
886 + } else {
887 + // This is a reset. Any overflow with a rate greater than MAX_INCREMENTAL_PERCENT_RATE will also
888 + // be detected as a reset instead.
889 + rd->collector.calculated_value += (NETDATA_DOUBLE)0;
890 + }
891 + }
892 + else {
893 + rd->collector.calculated_value +=
894 + (NETDATA_DOUBLE) (rd->collector.collected_value - rd->collector.last_collected_value)
895 + * (NETDATA_DOUBLE) rd->multiplier
896 + / (NETDATA_DOUBLE) rd->divisor;
897 + }
898 +
899 + rrdset_debug(st, "%s: CALC INC PRE " NETDATA_DOUBLE_FORMAT " = ("
900 + COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
901 + ")"
902 + " * " NETDATA_DOUBLE_FORMAT
903 + " / " NETDATA_DOUBLE_FORMAT
904 + , rrddim_name(rd)
905 + , rd->collector.calculated_value
906 + , rd->collector.collected_value, rd->collector.last_collected_value
907 + , (NETDATA_DOUBLE)rd->multiplier
908 + , (NETDATA_DOUBLE)rd->divisor
909 + );
910 + break;
911 +
912 + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
913 + if(unlikely(rd->collector.counter <= 1)) {
914 + rd->collector.calculated_value = 0;
915 + continue;
916 + }
917 +
918 + // the percentage of the current increment
919 + // over the increment of all dimensions together
920 + if(unlikely(collected_total == last_collected_total))
921 + rd->collector.calculated_value = 0;
922 + else
923 + rd->collector.calculated_value =
924 + (NETDATA_DOUBLE)100
925 + * (NETDATA_DOUBLE)(rd->collector.collected_value - rd->collector.last_collected_value)
926 + / (NETDATA_DOUBLE)(collected_total - last_collected_total);
927 +
928 + rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
929 + " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
930 + " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
931 + , rrddim_name(rd)
932 + , rd->collector.calculated_value
933 + , rd->collector.collected_value, rd->collector.last_collected_value
934 + , collected_total, last_collected_total
935 + );
936 + break;
937 +
938 + default:
939 + // make the default zero, to make sure
940 + // it gets noticed when we add new types
941 + rd->collector.calculated_value = 0;
942 +
943 + rrdset_debug(st, "%s: CALC " NETDATA_DOUBLE_FORMAT " = 0"
944 + , rrddim_name(rd)
945 + , rd->collector.calculated_value
946 + );
947 + break;
948 + }
949 +
950 + rrdset_debug(st, "%s: PHASE2 "
951 + " last_collected_value = " COLLECTED_NUMBER_FORMAT
952 + " collected_value = " COLLECTED_NUMBER_FORMAT
953 + " last_calculated_value = " NETDATA_DOUBLE_FORMAT
954 + " calculated_value = " NETDATA_DOUBLE_FORMAT
955 + , rrddim_name(rd)
956 + , rd->collector.last_collected_value
957 + , rd->collector.collected_value
958 + , rd->collector.last_calculated_value
959 + , rd->collector.calculated_value
960 + );
961 + }
962 +
963 + // at this point we have all the calculated values ready
964 + // it is now time to interpolate values on a second boundary
965 +
966 + // #ifdef NETDATA_INTERNAL_CHECKS
967 + // if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) {
968 + // // this is collected in the same interpolation point
969 + // rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
970 + // netdata_log_info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, rrdset_name(st), st->counter_done, next_store_ut - now_collect_ut);
971 + // }
972 + // #endif
973 +
974 + rrdset_done_interpolate(
975 + &stream_buffer
976 + , st
977 + , rda_base
978 + , rda_slots
979 + , update_every_ut
980 + , last_stored_ut
981 + , next_store_ut
982 + , last_collect_ut
983 + , now_collect_ut
984 + , store_this_entry
985 + );
986 +
987 + for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
988 + rd = rda->rd;
989 + if(unlikely(!rd)) continue;
990 +
991 + if(unlikely(!rrddim_check_updated(rd)))
992 + continue;
993 +
994 + rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rrddim_name(rd), rd->collector.last_collected_value, rd->collector.collected_value);
995 +
996 + rd->collector.last_collected_value = rd->collector.collected_value;
997 +
998 + switch(rd->algorithm) {
999 + case RRD_ALGORITHM_INCREMENTAL:
1000 + if(unlikely(!first_entry)) {
1001 + rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT ") to "
1002 + "last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")"
1003 + , rrddim_name(rd)
1004 + , rd->collector.last_calculated_value + rd->collector.calculated_value
1005 + , rd->collector.calculated_value);
1006 +
1007 + rd->collector.last_calculated_value += rd->collector.calculated_value;
1008 + }
1009 + else {
1010 + rrdset_debug(st, "THIS IS THE FIRST POINT");
1011 + }
1012 + break;
1013 +
1014 + case RRD_ALGORITHM_ABSOLUTE:
1015 + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
1016 + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
1017 + rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT ") to "
1018 + "last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")"
1019 + , rrddim_name(rd)
1020 + , rd->collector.last_calculated_value
1021 + , rd->collector.calculated_value);
1022 +
1023 + rd->collector.last_calculated_value = rd->collector.calculated_value;
1024 + break;
1025 + }
1026 +
1027 + rd->collector.calculated_value = 0;
1028 + rd->collector.collected_value = 0;
1029 + rrddim_clear_updated(rd);
1030 +
1031 + rrdset_debug(st, "%s: END "
1032 + " last_collected_value = " COLLECTED_NUMBER_FORMAT
1033 + " collected_value = " COLLECTED_NUMBER_FORMAT
1034 + " last_calculated_value = " NETDATA_DOUBLE_FORMAT
1035 + " calculated_value = " NETDATA_DOUBLE_FORMAT
1036 + , rrddim_name(rd)
1037 + , rd->collector.last_collected_value
1038 + , rd->collector.collected_value
1039 + , rd->collector.last_calculated_value
1040 + , rd->collector.calculated_value
1041 + );
1042 + }
1043 +
1044 + spinlock_unlock(&st->data_collection_lock);
1045 + stream_send_rrdset_metrics_finished(&stream_buffer, st);
1046 +
1047 + // ALL DONE ABOUT THE DATA UPDATE
1048 + // --------------------------------------------------------------------
1049 +
1050 + for(dim_id = 0, rda = rda_base; dim_id < rda_slots ; ++dim_id, ++rda) {
1051 + rd = rda->rd;
1052 + if(unlikely(!rd)) continue;
1053 +
1054 + dictionary_acquired_item_release(st->rrddim_root_index, rda->item);
1055 + rda->item = NULL;
1056 + rda->rd = NULL;
1057 + }
1058 +
1059 + rrdcontext_collected_rrdset(st);
1060 +
1061 + store_metric_collection_completed();
1062 +}
src/database/rrdset-collection.h new
+13
@@ -0,0 +1,13 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDSET_COLLECTION_H
4 +#define NETDATA_RRDSET_COLLECTION_H
5 +
6 +#include "rrd.h"
7 +
8 +extern __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];
9 +
10 +void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);
11 +time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);
12 +
13 +#endif //NETDATA_RRDSET_COLLECTION_H
src/database/rrdset-index-id.c new
+447
@@ -0,0 +1,447 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdset-index-id.h"
4 +#include "rrdset-index-name.h"
5 +#include "rrdset-slots.h"
6 +
7 +static inline void rrdset_update_permanent_labels(RRDSET *st) {
8 + if(!st->rrdlabels) return;
9 +
10 + rrdlabels_add(st->rrdlabels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);
11 + rrdlabels_add(st->rrdlabels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);
12 +}
13 +
14 +// ----------------------------------------------------------------------------
15 +// RRDSET index
16 +
17 +struct rrdset_constructor {
18 + RRDHOST *host;
19 + const char *type;
20 + const char *id;
21 + const char *name;
22 + const char *family;
23 + const char *context;
24 + const char *title;
25 + const char *units;
26 + const char *plugin;
27 + const char *module;
28 + long priority;
29 + int update_every;
30 + RRDSET_TYPE chart_type;
31 + RRD_MEMORY_MODE memory_mode;
32 + long history_entries;
33 +
34 + enum {
35 + RRDSET_REACT_NONE = 0,
36 + RRDSET_REACT_NEW = (1 << 0),
37 + RRDSET_REACT_UPDATED = (1 << 1),
38 + RRDSET_REACT_PLUGIN_UPDATED = (1 << 2),
39 + RRDSET_REACT_MODULE_UPDATED = (1 << 3),
40 + RRDSET_REACT_CHART_ACTIVATED = (1 << 4),
41 + } react_action;
42 +};
43 +
44 +// the constructor - the dictionary is write locked while this runs
45 +static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) {
46 + struct rrdset_constructor *ctr = constructor_data;
47 + RRDHOST *host = ctr->host;
48 + RRDSET *st = rrdset;
49 +
50 + const char *chart_full_id = dictionary_acquired_item_name(item);
51 +
52 + st->id = string_strdupz(chart_full_id);
53 +
54 + st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->name);
55 + if(!st->name)
56 + st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->id);
57 + rrdset_index_add_name(host, st);
58 +
59 + st->parts.id = string_strdupz(ctr->id);
60 + st->parts.type = string_strdupz(ctr->type);
61 + st->parts.name = string_strdupz(ctr->name);
62 +
63 + st->family = (ctr->family && *ctr->family) ? rrd_string_strdupz(ctr->family) : rrd_string_strdupz(ctr->type);
64 + st->context = (ctr->context && *ctr->context) ? rrd_string_strdupz(ctr->context) : rrd_string_strdupz(chart_full_id);
65 +
66 + st->units = rrd_string_strdupz(ctr->units);
67 + st->title = rrd_string_strdupz(ctr->title);
68 + st->plugin_name = rrd_string_strdupz(ctr->plugin);
69 + st->module_name = rrd_string_strdupz(ctr->module);
70 + st->priority = ctr->priority;
71 +
72 + st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;
73 + st->update_every = ctr->update_every;
74 + st->rrd_memory_mode = ctr->memory_mode;
75 +
76 + st->chart_type = ctr->chart_type;
77 + st->rrdhost = host;
78 +
79 + rrdset_stream_send_chart_slot_assign(st);
80 +
81 + spinlock_init(&st->data_collection_lock);
82 +
83 + st->flags = RRDSET_FLAG_SYNC_CLOCK
84 + | RRDSET_FLAG_INDEXED_ID
85 + | RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED
86 + | RRDSET_FLAG_SENDER_REPLICATION_FINISHED
87 + ;
88 +
89 + rw_spinlock_init(&st->alerts.spinlock);
90 +
91 + // initialize the db tiers
92 + {
93 + for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
94 + STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
95 + if(!eng) continue;
96 +
97 + st->smg[tier] = storage_engine_metrics_group_get(eng->seb, host->db[tier].si, &st->chart_uuid);
98 + }
99 + }
100 +
101 + rrddim_index_init(st);
102 +
103 + st->rrdvars = rrdvariables_create();
104 + st->rrdlabels = rrdlabels_create();
105 + rrdset_update_permanent_labels(st);
106 +
107 + st->green = NAN;
108 + st->red = NAN;
109 +
110 + rrdset_pluginsd_receive_slots_initialize(st);
111 +
112 + rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
113 + rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
114 +
115 + ctr->react_action = RRDSET_REACT_NEW;
116 +
117 + ml_chart_new(st);
118 +}
119 +
120 +// the destructor - the dictionary is write locked while this runs
121 +static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost) {
122 + RRDHOST *host = rrdhost;
123 + RRDSET *st = rrdset;
124 +
125 + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
126 +
127 + rrdset_finalize_collection(st, false);
128 +
129 + rrdset_stream_send_chart_slot_release(st);
130 +
131 + // remove it from the name index
132 + rrdset_index_del_name(host, st);
133 +
134 + // release the collector info
135 + dictionary_destroy(st->functions_view);
136 +
137 + rrdcalc_unlink_and_delete_all_rrdset_alerts(st);
138 +
139 + // ------------------------------------------------------------------------
140 + // the order of destruction is important here
141 +
142 + // 1. delete RRDVAR index after the above, to avoid triggering its garbage collector (they have references on this)
143 + rrdvariables_destroy(st->rrdvars); // free all variables and destroy the rrdvar dictionary
144 +
145 + // 2. delete RRDDIMs, now their variables are not existing, so this is fast
146 + rrddim_index_destroy(st); // free all the dimensions and destroy the dimensions index
147 +
148 + // 3. this has to be after the dimensions are freed, but before labels are freed (contexts need the labels)
149 + rrdcontext_removed_rrdset(st); // let contexts know
150 +
151 + // 4. destroy the chart labels
152 + rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
153 +
154 + // 5. destroy the ml handle
155 + ml_chart_delete(st);
156 +
157 + // ------------------------------------------------------------------------
158 + // free it
159 +
160 + string_freez(st->id);
161 + string_freez(st->name);
162 + string_freez(st->parts.id);
163 + string_freez(st->parts.type);
164 + string_freez(st->parts.name);
165 + string_freez(st->family);
166 + string_freez(st->title);
167 + string_freez(st->units);
168 + string_freez(st->context);
169 + string_freez(st->plugin_name);
170 + string_freez(st->module_name);
171 +
172 + freez(st->exporting_flags);
173 +}
174 +
175 +// the item to be inserted, is already in the dictionary
176 +// this callback deals with the situation, migrating the existing object to the new values
177 +// the dictionary is write locked while this runs
178 +static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *new_rrdset, void *constructor_data) {
179 + (void)new_rrdset; // it is NULL
180 +
181 + struct rrdset_constructor *ctr = constructor_data;
182 + RRDSET *st = rrdset;
183 +
184 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
185 +
186 + ctr->react_action = RRDSET_REACT_NONE;
187 +
188 + if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2)
189 + ctr->react_action |= RRDSET_REACT_UPDATED;
190 +
191 + if (unlikely(st->priority != ctr->priority)) {
192 + st->priority = ctr->priority;
193 + ctr->react_action |= RRDSET_REACT_UPDATED;
194 + }
195 +
196 + if (unlikely(st->update_every != ctr->update_every)) {
197 + rrdset_set_update_every_s(st, ctr->update_every);
198 + ctr->react_action |= RRDSET_REACT_UPDATED;
199 + }
200 +
201 + if(ctr->plugin && *ctr->plugin) {
202 + STRING *old_plugin = st->plugin_name;
203 + st->plugin_name = rrd_string_strdupz(ctr->plugin);
204 + if (old_plugin != st->plugin_name)
205 + ctr->react_action |= RRDSET_REACT_PLUGIN_UPDATED;
206 + string_freez(old_plugin);
207 + }
208 +
209 + if(ctr->module && *ctr->module) {
210 + STRING *old_module = st->module_name;
211 + st->module_name = rrd_string_strdupz(ctr->module);
212 + if (old_module != st->module_name)
213 + ctr->react_action |= RRDSET_REACT_MODULE_UPDATED;
214 + string_freez(old_module);
215 + }
216 +
217 + if(ctr->title && *ctr->title) {
218 + STRING *old_title = st->title;
219 + st->title = rrd_string_strdupz(ctr->title);
220 + if(old_title != st->title)
221 + ctr->react_action |= RRDSET_REACT_UPDATED;
222 + string_freez(old_title);
223 + }
224 +
225 + if(ctr->units && *ctr->units) {
226 + STRING *old_units = st->units;
227 + st->units = rrd_string_strdupz(ctr->units);
228 + if(old_units != st->units)
229 + ctr->react_action |= RRDSET_REACT_UPDATED;
230 + string_freez(old_units);
231 + }
232 +
233 + if(ctr->family && *ctr->family) {
234 + STRING *old_family = st->family;
235 + st->family = rrd_string_strdupz(ctr->family);
236 + if(old_family != st->family)
237 + ctr->react_action |= RRDSET_REACT_UPDATED;
238 + string_freez(old_family);
239 + }
240 +
241 + if(ctr->context && *ctr->context) {
242 + STRING *old_context = st->context;
243 + st->context = rrd_string_strdupz(ctr->context);
244 + if(old_context != st->context)
245 + ctr->react_action |= RRDSET_REACT_UPDATED;
246 + string_freez(old_context);
247 + }
248 +
249 + if(st->chart_type != ctr->chart_type) {
250 + st->chart_type = ctr->chart_type;
251 + ctr->react_action |= RRDSET_REACT_UPDATED;
252 + }
253 +
254 + rrdset_update_permanent_labels(st);
255 +
256 + rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
257 + rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
258 + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
259 +
260 + return ctr->react_action != RRDSET_REACT_NONE;
261 +}
262 +
263 +// this is called after all insertions/conflicts, with the dictionary unlocked, with a reference to RRDSET
264 +// so, any actions requiring locks on other objects, should be placed here
265 +static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) {
266 + struct rrdset_constructor *ctr = constructor_data;
267 + RRDSET *st = rrdset;
268 + RRDHOST *host = st->rrdhost;
269 +
270 + st->last_accessed_time_s = now_realtime_sec();
271 +
272 + if(ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
273 + if (ctr->react_action & RRDSET_REACT_NEW) {
274 + if(unlikely(rrdcontext_find_chart_uuid(st, &st->chart_uuid)))
275 + uuid_generate(st->chart_uuid);
276 + }
277 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
278 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_UPDATE);
279 + }
280 +
281 + rrdset_metadata_updated(st);
282 +}
283 +
284 +void rrdset_index_init(RRDHOST *host) {
285 + if(!host->rrdset_root_index) {
286 + host->rrdset_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
287 + &dictionary_stats_category_rrdset, sizeof(RRDSET));
288 +
289 + dictionary_register_insert_callback(host->rrdset_root_index, rrdset_insert_callback, NULL);
290 + dictionary_register_conflict_callback(host->rrdset_root_index, rrdset_conflict_callback, NULL);
291 + dictionary_register_react_callback(host->rrdset_root_index, rrdset_react_callback, NULL);
292 + dictionary_register_delete_callback(host->rrdset_root_index, rrdset_delete_callback, host);
293 + }
294 +
295 + rrdset_index_byname_init(host);
296 +}
297 +
298 +void rrdset_index_destroy(RRDHOST *host) {
299 + // destroy the name index first
300 + dictionary_destroy(host->rrdset_root_index_name);
301 + host->rrdset_root_index_name = NULL;
302 +
303 + // destroy the id index last
304 + dictionary_destroy(host->rrdset_root_index);
305 + host->rrdset_root_index = NULL;
306 +}
307 +
308 +static inline RRDSET *rrdset_index_add(RRDHOST *host, const char *id, struct rrdset_constructor *st_ctr) {
309 + return dictionary_set_advanced(host->rrdset_root_index, id, -1, NULL, sizeof(RRDSET), st_ctr);
310 +}
311 +
312 +static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) {
313 + if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID))
314 + dictionary_del(host->rrdset_root_index, rrdset_id(st));
315 +}
316 +
317 +static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) {
318 + // TODO - the name index should have an acquired dictionary item, not just a pointer to RRDSET
319 + if (unlikely(!host->rrdset_root_index))
320 + return NULL;
321 + return dictionary_get(host->rrdset_root_index, id);
322 +}
323 +
324 +RRDSET *rrdset_find(RRDHOST *host, const char *id) {
325 + netdata_log_debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host));
326 + RRDSET *st = rrdset_index_find(host, id);
327 +
328 + if(st)
329 + st->last_accessed_time_s = now_realtime_sec();
330 +
331 + return(st);
332 +}
333 +
334 +RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id) {
335 + netdata_log_debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host));
336 +
337 + char buf[RRD_ID_LENGTH_MAX + 1];
338 + strncpyz(buf, type, RRD_ID_LENGTH_MAX - 1);
339 + strcat(buf, ".");
340 + int len = (int) strlen(buf);
341 + strncpyz(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len));
342 +
343 + return(rrdset_find(host, buf));
344 +}
345 +
346 +RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id) {
347 + netdata_log_debug(D_RRD_CALLS, "rrdset_find_and_acquire() for host %s, chart %s", rrdhost_hostname(host), id);
348 +
349 + return (RRDSET_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdset_root_index, id);
350 +}
351 +
352 +RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa) {
353 + if(unlikely(!rsa))
354 + return NULL;
355 +
356 + return (RRDSET *) dictionary_acquired_item_value((const DICTIONARY_ITEM *)rsa);
357 +}
358 +
359 +void rrdset_acquired_release(RRDSET_ACQUIRED *rsa) {
360 + if(unlikely(!rsa))
361 + return;
362 +
363 + RRDSET *rs = rrdset_acquired_to_rrdset(rsa);
364 + dictionary_acquired_item_release(rs->rrdhost->rrdset_root_index, (const DICTIONARY_ITEM *)rsa);
365 +}
366 +
367 +RRDSET *rrdset_create_custom(
368 + RRDHOST *host
369 + , const char *type
370 + , const char *id
371 + , const char *name
372 + , const char *family
373 + , const char *context
374 + , const char *title
375 + , const char *units
376 + , const char *plugin
377 + , const char *module
378 + , long priority
379 + , int update_every
380 + , RRDSET_TYPE chart_type
381 + , RRD_MEMORY_MODE memory_mode
382 + , long history_entries
383 +) {
384 + if (host != localhost)
385 + host->stream.rcv.status.last_chart = now_realtime_sec();
386 +
387 + if(!type || !type[0])
388 + fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
389 + , (id && *id)?id:"<unset>"
390 + , (name && *name)?name:"<unset>"
391 + , (family && *family)?family:"<unset>"
392 + , (context && *context)?context:"<unset>"
393 + , (title && *title)?title:"<unset>"
394 + , (units && *units)?units:"<unset>"
395 + , (plugin && *plugin)?plugin:"<unset>"
396 + , (module && *module)?module:"<unset>"
397 + );
398 +
399 + if(!id || !id[0])
400 + fatal("Cannot create rrd stats without an id: type '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
401 + , type
402 + , (name && *name)?name:"<unset>"
403 + , (family && *family)?family:"<unset>"
404 + , (context && *context)?context:"<unset>"
405 + , (title && *title)?title:"<unset>"
406 + , (units && *units)?units:"<unset>"
407 + , (plugin && *plugin)?plugin:"<unset>"
408 + , (module && *module)?module:"<unset>"
409 + );
410 +
411 + // ------------------------------------------------------------------------
412 + // check if it already exists
413 +
414 + char full_id[RRD_ID_LENGTH_MAX + 1];
415 + snprintfz(full_id, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
416 +
417 + // ------------------------------------------------------------------------
418 + // allocate it
419 +
420 + netdata_log_debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
421 +
422 + struct rrdset_constructor tmp = {
423 + .host = host,
424 + .type = type,
425 + .id = id,
426 + .name = name,
427 + .family = family,
428 + .context = context,
429 + .title = title,
430 + .units = units,
431 + .plugin = plugin,
432 + .module = module,
433 + .priority = priority,
434 + .update_every = update_every,
435 + .chart_type = chart_type,
436 + .memory_mode = memory_mode,
437 + .history_entries = history_entries,
438 + };
439 +
440 + RRDSET *st = rrdset_index_add(host, full_id, &tmp);
441 + return(st);
442 +}
443 +
444 +void rrdset_free(RRDSET *st) {
445 + if(unlikely(!st)) return;
446 + rrdset_index_del(st->rrdhost, st);
447 +}
src/database/rrdset-index-id.h new
+77
@@ -0,0 +1,77 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDSET_INDEX_ID_H
4 +#define NETDATA_RRDSET_INDEX_ID_H
5 +
6 +#include "rrd.h"
7 +
8 +RRDSET *rrdset_create_custom(
9 + RRDHOST *host
10 + , const char *type
11 + , const char *id
12 + , const char *name
13 + , const char *family
14 + , const char *context
15 + , const char *title
16 + , const char *units
17 + , const char *plugin
18 + , const char *module
19 + , long priority
20 + , int update_every
21 + , RRDSET_TYPE chart_type
22 + , RRD_MEMORY_MODE memory_mode
23 + , long history_entries
24 +);
25 +
26 +static inline
27 + RRDSET *rrdset_create(RRDHOST *host
28 + , const char *type
29 + , const char *id
30 + , const char *name
31 + , const char *family
32 + , const char *context
33 + , const char *title
34 + , const char *units
35 + , const char *plugin
36 + , const char *module
37 + , long priority
38 + , int update_every
39 + , RRDSET_TYPE chart_type) {
40 + return rrdset_create_custom(
41 + host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries);
42 +}
43 +
44 +static inline
45 + RRDSET *rrdset_create_localhost(
46 + const char *type
47 + , const char *id
48 + , const char *name
49 + , const char *family
50 + , const char *context
51 + , const char *title
52 + , const char *units
53 + , const char *plugin
54 + , const char *module
55 + , long priority
56 + , int update_every
57 + , RRDSET_TYPE chart_type) {
58 + return rrdset_create(
59 + localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type);
60 +}
61 +
62 +int rrdset_reset_name(RRDSET *st, const char *name);
63 +
64 +void rrdset_index_init(RRDHOST *host);
65 +void rrdset_index_destroy(RRDHOST *host);
66 +
67 +void rrdset_free(RRDSET *st);
68 +
69 +RRDSET *rrdset_find(RRDHOST *host, const char *id);
70 +RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
71 +
72 +RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);
73 +
74 +void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
75 +RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);
76 +
77 +#endif //NETDATA_RRDSET_INDEX_ID_H
src/database/rrdset-index-name.c new
+108
@@ -0,0 +1,108 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdset-index-name.h"
4 +#include "rrdset-index-id.h"
5 +#include "rrdset-slots.h"
6 +
7 +STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name) {
8 + if(!name || !*name) return NULL;
9 +
10 + char full_name[RRD_ID_LENGTH_MAX + 1];
11 + char sanitized_name[CONFIG_MAX_VALUE + 1];
12 + char new_name[CONFIG_MAX_VALUE + 1];
13 +
14 + snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", type, name);
15 + rrdset_strncpyz_name(sanitized_name, full_name, CONFIG_MAX_VALUE);
16 + strncpyz(new_name, sanitized_name, CONFIG_MAX_VALUE);
17 +
18 + if(rrdset_index_find_name(host, new_name)) {
19 + netdata_log_debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, rrdhost_hostname(host));
20 + if(!strcmp(chart_full_id, full_name) && (!current_name || !*current_name)) {
21 + unsigned i = 1;
22 +
23 + do {
24 + snprintfz(new_name, CONFIG_MAX_VALUE, "%s_%u", sanitized_name, i);
25 + i++;
26 + } while (rrdset_index_find_name(host, new_name));
27 +
28 + // netdata_log_info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host));
29 + }
30 + else
31 + return NULL;
32 + }
33 +
34 + return string_strdupz(new_name);
35 +}
36 +
37 +int rrdset_reset_name(RRDSET *st, const char *name) {
38 + if(unlikely(!strcmp(rrdset_name(st), name)))
39 + return 1;
40 +
41 + RRDHOST *host = st->rrdhost;
42 +
43 + netdata_log_debug(D_RRD_CALLS, "rrdset_reset_name() old: '%s', new: '%s'", rrdset_name(st), name);
44 +
45 + STRING *name_string = rrdset_fix_name(host, rrdset_id(st), rrdset_parts_type(st), string2str(st->name), name);
46 + if(!name_string) return 0;
47 +
48 + if(st->name) {
49 + rrdset_index_del_name(host, st);
50 + string_freez(st->name);
51 + st->name = name_string;
52 + }
53 + else
54 + st->name = name_string;
55 +
56 + rrdset_index_add_name(host, st);
57 +
58 + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);
59 + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);
60 + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
61 + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);
62 + rrdset_metadata_updated(st);
63 +
64 + rrdcontext_updated_rrdset_name(st);
65 + return 2;
66 +}
67 +
68 +static void rrdset_name_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost __maybe_unused) {
69 + RRDSET *st = rrdset;
70 + rrdset_flag_set(st, RRDSET_FLAG_INDEXED_NAME);
71 +}
72 +static void rrdset_name_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost __maybe_unused) {
73 + RRDSET *st = rrdset;
74 + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME);
75 +}
76 +
77 +RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) {
78 + if (unlikely(!host->rrdset_root_index_name))
79 + return NULL;
80 + return dictionary_get(host->rrdset_root_index_name, name);
81 +}
82 +
83 +void rrdset_index_byname_init(RRDHOST *host) {
84 + if(!host->rrdset_root_index_name) {
85 + host->rrdset_root_index_name = dictionary_create_advanced(
86 + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
87 + &dictionary_stats_category_rrdset, 0);
88 +
89 + dictionary_register_insert_callback(host->rrdset_root_index_name, rrdset_name_insert_callback, host);
90 + dictionary_register_delete_callback(host->rrdset_root_index_name, rrdset_name_delete_callback, host);
91 + }
92 +}
93 +
94 +void rrdset_index_add_name(RRDHOST *host, RRDSET *st) {
95 + if(!st->name) return;
96 + dictionary_set(host->rrdset_root_index_name, rrdset_name(st), st, sizeof(RRDSET));
97 +}
98 +
99 +void rrdset_index_del_name(RRDHOST *host, RRDSET *st) {
100 + if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_NAME))
101 + dictionary_del(host->rrdset_root_index_name, rrdset_name(st));
102 +}
103 +
104 +inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {
105 + netdata_log_debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host));
106 + RRDSET *st = rrdset_index_find_name(host, name);
107 + return(st);
108 +}
src/database/rrdset-index-name.h new
+19
@@ -0,0 +1,19 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDSET_INDEX_NAME_H
4 +#define NETDATA_RRDSET_INDEX_NAME_H
5 +
6 +#include "rrdset.h"
7 +#include "rrd.h"
8 +
9 +void rrdset_index_byname_init(RRDHOST *host);
10 +
11 +void rrdset_index_add_name(RRDHOST *host, RRDSET *st);
12 +void rrdset_index_del_name(RRDHOST *host, RRDSET *st);
13 +
14 +RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name);
15 +
16 +int rrdset_reset_name(RRDSET *st, const char *name);
17 +STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name);
18 +
19 +#endif //NETDATA_RRDSET_INDEX_NAME_H
src/database/rrdset-slots.c new
+124
@@ -0,0 +1,124 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdset-slots.h"
4 +
5 +void rrdset_stream_send_chart_slot_assign(RRDSET *st) {
6 + RRDHOST *host = st->rrdhost;
7 + spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
8 +
9 + if(host->stream.snd.pluginsd_chart_slots.available.used > 0)
10 + st->stream.snd.chart_slot =
11 + host->stream.snd.pluginsd_chart_slots.available.array[--host->stream.snd.pluginsd_chart_slots.available.used];
12 + else
13 + st->stream.snd.chart_slot = ++host->stream.snd.pluginsd_chart_slots.last_used;
14 +
15 + spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
16 +}
17 +
18 +void rrdset_stream_send_chart_slot_release(RRDSET *st) {
19 + if(!st->stream.snd.chart_slot || st->rrdhost->stream.snd.pluginsd_chart_slots.available.ignore)
20 + return;
21 +
22 + RRDHOST *host = st->rrdhost;
23 + spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
24 +
25 + if(host->stream.snd.pluginsd_chart_slots.available.used >= host->stream.snd.pluginsd_chart_slots.available.size) {
26 + uint32_t old_slots = host->stream.snd.pluginsd_chart_slots.available.size;
27 + uint32_t new_slots = (old_slots > 0) ? (old_slots * 2) : 1024;
28 +
29 + host->stream.snd.pluginsd_chart_slots.available.array =
30 + reallocz(host->stream.snd.pluginsd_chart_slots.available.array, new_slots * sizeof(uint32_t));
31 +
32 + host->stream.snd.pluginsd_chart_slots.available.size = new_slots;
33 +
34 + rrd_slot_memory_added((new_slots - old_slots) * sizeof(uint32_t));
35 + }
36 +
37 + host->stream.snd.pluginsd_chart_slots.available.array[host->stream.snd.pluginsd_chart_slots.available.used++] =
38 + st->stream.snd.chart_slot;
39 +
40 + st->stream.snd.chart_slot = 0;
41 + spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
42 +}
43 +
44 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) {
45 + rrd_slot_memory_removed(host->stream.snd.pluginsd_chart_slots.available.size * sizeof(uint32_t));
46 +
47 + spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
48 + host->stream.snd.pluginsd_chart_slots.available.ignore = true;
49 + freez(host->stream.snd.pluginsd_chart_slots.available.array);
50 + host->stream.snd.pluginsd_chart_slots.available.array = NULL;
51 + host->stream.snd.pluginsd_chart_slots.available.used = 0;
52 + host->stream.snd.pluginsd_chart_slots.available.size = 0;
53 + spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
54 +
55 + // zero all the slots on all charts, so that they will not attempt to access the array
56 + RRDSET *st;
57 + rrdset_foreach_read(st, host) {
58 + st->stream.snd.chart_slot = 0;
59 + }
60 + rrdset_foreach_done(st);
61 +}
62 +
63 +void rrdset_pluginsd_receive_unslot(RRDSET *st) {
64 + for(size_t i = 0; i < st->pluginsd.size ;i++) {
65 + rrddim_acquired_release(st->pluginsd.prd_array[i].rda); // can be NULL
66 + st->pluginsd.prd_array[i].rda = NULL;
67 + st->pluginsd.prd_array[i].rd = NULL;
68 + st->pluginsd.prd_array[i].id = NULL;
69 + }
70 +
71 + RRDHOST *host = st->rrdhost;
72 +
73 + if(st->pluginsd.last_slot >= 0 &&
74 + (uint32_t)st->pluginsd.last_slot < host->stream.rcv.pluginsd_chart_slots.size &&
75 + host->stream.rcv.pluginsd_chart_slots.array[st->pluginsd.last_slot] == st) {
76 + host->stream.rcv.pluginsd_chart_slots.array[st->pluginsd.last_slot] = NULL;
77 + }
78 +
79 + st->pluginsd.last_slot = -1;
80 + st->pluginsd.dims_with_slots = false;
81 +}
82 +
83 +void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st) {
84 + if(!st)
85 + return;
86 +
87 + spinlock_lock(&st->pluginsd.spinlock);
88 +
89 + rrdset_pluginsd_receive_unslot(st);
90 +
91 + rrd_slot_memory_removed(st->pluginsd.size * sizeof(struct pluginsd_rrddim));
92 + freez(st->pluginsd.prd_array);
93 + st->pluginsd.prd_array = NULL;
94 + st->pluginsd.size = 0;
95 + st->pluginsd.pos = 0;
96 + st->pluginsd.set = false;
97 + st->pluginsd.last_slot = -1;
98 + st->pluginsd.dims_with_slots = false;
99 + st->pluginsd.collector_tid = 0;
100 +
101 + spinlock_unlock(&st->pluginsd.spinlock);
102 +}
103 +
104 +void rrdset_pluginsd_receive_slots_initialize(RRDSET *st) {
105 + spinlock_init(&st->pluginsd.spinlock);
106 + st->pluginsd.last_slot = -1;
107 +}
108 +
109 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) {
110 + rrd_slot_memory_removed(host->stream.rcv.pluginsd_chart_slots.size * sizeof(uint32_t));
111 +
112 + spinlock_lock(&host->stream.rcv.pluginsd_chart_slots.spinlock);
113 +
114 + if(host->stream.rcv.pluginsd_chart_slots.array) {
115 + for (size_t s = 0; s < host->stream.rcv.pluginsd_chart_slots.size; s++)
116 + rrdset_pluginsd_receive_unslot_and_cleanup(host->stream.rcv.pluginsd_chart_slots.array[s]);
117 +
118 + freez(host->stream.rcv.pluginsd_chart_slots.array);
119 + host->stream.rcv.pluginsd_chart_slots.array = NULL;
120 + host->stream.rcv.pluginsd_chart_slots.size = 0;
121 + }
122 +
123 + spinlock_unlock(&host->stream.rcv.pluginsd_chart_slots.spinlock);
124 +}
src/database/rrdset-slots.h new
+17
@@ -0,0 +1,17 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDSET_SLOTS_H
4 +#define NETDATA_RRDSET_SLOTS_H
5 +
6 +#include "rrd.h"
7 +
8 +void rrdset_stream_send_chart_slot_assign(RRDSET *st);
9 +void rrdset_stream_send_chart_slot_release(RRDSET *st);
10 +
11 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);
12 +void rrdset_pluginsd_receive_unslot(RRDSET *st);
13 +void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st);
14 +void rrdset_pluginsd_receive_slots_initialize(RRDSET *st);
15 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);
16 +
17 +#endif //NETDATA_RRDSET_SLOTS_H
src/database/rrdset.c
+23 -1823
@@ -1,9 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#define NETDATA_RRD_INTERNALS
4 -#include "rrd.h"
5 -#include "storage_engine.h"
6 -
3 +#include "rrdset.h"
4 +#include "storage-engine.h"
5
6 void rrdset_metadata_updated(RRDSET *st) {
7 __atomic_add_fetch(&st->version, 1, __ATOMIC_RELAXED);
@@ -11,630 +9,42 @@ void rrdset_metadata_updated(RRDSET *st) {
9 }
10
11 // ----------------------------------------------------------------------------
14 -// RRDSET streaming send chart_slots
15 -
16 -static void rrdset_stream_send_chart_slot_assign(RRDSET *st) {
17 - RRDHOST *host = st->rrdhost;
18 - spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
19 -
20 - if(host->stream.snd.pluginsd_chart_slots.available.used > 0)
21 - st->stream.snd.chart_slot =
22 - host->stream.snd.pluginsd_chart_slots.available.array[--host->stream.snd.pluginsd_chart_slots.available.used];
23 - else
24 - st->stream.snd.chart_slot = ++host->stream.snd.pluginsd_chart_slots.last_used;
25 -
26 - spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
27 -}
28 -
29 -static void rrdset_stream_send_chart_slot_release(RRDSET *st) {
30 - if(!st->stream.snd.chart_slot || st->rrdhost->stream.snd.pluginsd_chart_slots.available.ignore)
31 - return;
32 -
33 - RRDHOST *host = st->rrdhost;
34 - spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
35 -
36 - if(host->stream.snd.pluginsd_chart_slots.available.used >= host->stream.snd.pluginsd_chart_slots.available.size) {
37 - uint32_t old_slots = host->stream.snd.pluginsd_chart_slots.available.size;
38 - uint32_t new_slots = (old_slots > 0) ? (old_slots * 2) : 1024;
39 -
40 - host->stream.snd.pluginsd_chart_slots.available.array =
41 - reallocz(host->stream.snd.pluginsd_chart_slots.available.array, new_slots * sizeof(uint32_t));
42 -
43 - host->stream.snd.pluginsd_chart_slots.available.size = new_slots;
44 -
45 - rrd_slot_memory_added((new_slots - old_slots) * sizeof(uint32_t));
46 - }
47 -
48 - host->stream.snd.pluginsd_chart_slots.available.array[host->stream.snd.pluginsd_chart_slots.available.used++] =
49 - st->stream.snd.chart_slot;
50 -
51 - st->stream.snd.chart_slot = 0;
52 - spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
53 -}
54 -
55 -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) {
56 - rrd_slot_memory_removed(host->stream.snd.pluginsd_chart_slots.available.size * sizeof(uint32_t));
57 -
58 - spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
59 - host->stream.snd.pluginsd_chart_slots.available.ignore = true;
60 - freez(host->stream.snd.pluginsd_chart_slots.available.array);
61 - host->stream.snd.pluginsd_chart_slots.available.array = NULL;
62 - host->stream.snd.pluginsd_chart_slots.available.used = 0;
63 - host->stream.snd.pluginsd_chart_slots.available.size = 0;
64 - spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);
65 -
66 - // zero all the slots on all charts, so that they will not attempt to access the array
67 - RRDSET *st;
68 - rrdset_foreach_read(st, host) {
69 - st->stream.snd.chart_slot = 0;
70 - }
71 - rrdset_foreach_done(st);
72 -}
73 -
74 -void rrdset_pluginsd_receive_unslot(RRDSET *st) {
75 - for(size_t i = 0; i < st->pluginsd.size ;i++) {
76 - rrddim_acquired_release(st->pluginsd.prd_array[i].rda); // can be NULL
77 - st->pluginsd.prd_array[i].rda = NULL;
78 - st->pluginsd.prd_array[i].rd = NULL;
79 - st->pluginsd.prd_array[i].id = NULL;
80 - }
81 -
82 - RRDHOST *host = st->rrdhost;
83 -
84 - if(st->pluginsd.last_slot >= 0 &&
85 - (uint32_t)st->pluginsd.last_slot < host->stream.rcv.pluginsd_chart_slots.size &&
86 - host->stream.rcv.pluginsd_chart_slots.array[st->pluginsd.last_slot] == st) {
87 - host->stream.rcv.pluginsd_chart_slots.array[st->pluginsd.last_slot] = NULL;
88 - }
89 -
90 - st->pluginsd.last_slot = -1;
91 - st->pluginsd.dims_with_slots = false;
92 -}
93 -
94 -void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st) {
95 - if(!st)
96 - return;
97 -
98 - spinlock_lock(&st->pluginsd.spinlock);
99 -
100 - rrdset_pluginsd_receive_unslot(st);
101 -
102 - rrd_slot_memory_removed(st->pluginsd.size * sizeof(struct pluginsd_rrddim));
103 - freez(st->pluginsd.prd_array);
104 - st->pluginsd.prd_array = NULL;
105 - st->pluginsd.size = 0;
106 - st->pluginsd.pos = 0;
107 - st->pluginsd.set = false;
108 - st->pluginsd.last_slot = -1;
109 - st->pluginsd.dims_with_slots = false;
110 - st->pluginsd.collector_tid = 0;
111 -
112 - spinlock_unlock(&st->pluginsd.spinlock);
113 -}
114 -
115 -static void rrdset_pluginsd_receive_slots_initialize(RRDSET *st) {
116 - spinlock_init(&st->pluginsd.spinlock);
117 - st->pluginsd.last_slot = -1;
118 -}
119 -
120 -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) {
121 - rrd_slot_memory_removed(host->stream.rcv.pluginsd_chart_slots.size * sizeof(uint32_t));
122 -
123 - spinlock_lock(&host->stream.rcv.pluginsd_chart_slots.spinlock);
124 -
125 - if(host->stream.rcv.pluginsd_chart_slots.array) {
126 - for (size_t s = 0; s < host->stream.rcv.pluginsd_chart_slots.size; s++)
127 - rrdset_pluginsd_receive_unslot_and_cleanup(host->stream.rcv.pluginsd_chart_slots.array[s]);
128 -
129 - freez(host->stream.rcv.pluginsd_chart_slots.array);
130 - host->stream.rcv.pluginsd_chart_slots.array = NULL;
131 - host->stream.rcv.pluginsd_chart_slots.size = 0;
132 - }
133 -
134 - spinlock_unlock(&host->stream.rcv.pluginsd_chart_slots.spinlock);
135 -}
136 -
137 -// ----------------------------------------------------------------------------
138 -// RRDSET name index
139 -
140 -static void rrdset_name_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost __maybe_unused) {
141 - RRDSET *st = rrdset;
142 - rrdset_flag_set(st, RRDSET_FLAG_INDEXED_NAME);
143 -}
144 -static void rrdset_name_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost __maybe_unused) {
145 - RRDSET *st = rrdset;
146 - rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME);
147 -}
148 -
149 -static inline void rrdset_index_add_name(RRDHOST *host, RRDSET *st) {
150 - if(!st->name) return;
151 - dictionary_set(host->rrdset_root_index_name, rrdset_name(st), st, sizeof(RRDSET));
152 -}
153 -
154 -static inline void rrdset_index_del_name(RRDHOST *host, RRDSET *st) {
155 - if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_NAME))
156 - dictionary_del(host->rrdset_root_index_name, rrdset_name(st));
157 -}
158 -
159 -static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) {
160 - if (unlikely(!host->rrdset_root_index_name))
161 - return NULL;
162 - return dictionary_get(host->rrdset_root_index_name, name);
163 -}
164 -
165 -// ----------------------------------------------------------------------------
166 -// RRDSET index
167 -
168 -static inline void rrdset_update_permanent_labels(RRDSET *st) {
169 - if(!st->rrdlabels) return;
170 -
171 - rrdlabels_add(st->rrdlabels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);
172 - rrdlabels_add(st->rrdlabels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);
173 -}
174 -
175 -static STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name) {
176 - if(!name || !*name) return NULL;
177 -
178 - char full_name[RRD_ID_LENGTH_MAX + 1];
179 - char sanitized_name[CONFIG_MAX_VALUE + 1];
180 - char new_name[CONFIG_MAX_VALUE + 1];
181 -
182 - snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", type, name);
183 - rrdset_strncpyz_name(sanitized_name, full_name, CONFIG_MAX_VALUE);
184 - strncpyz(new_name, sanitized_name, CONFIG_MAX_VALUE);
185 -
186 - if(rrdset_index_find_name(host, new_name)) {
187 - netdata_log_debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, rrdhost_hostname(host));
188 - if(!strcmp(chart_full_id, full_name) && (!current_name || !*current_name)) {
189 - unsigned i = 1;
190 -
191 - do {
192 - snprintfz(new_name, CONFIG_MAX_VALUE, "%s_%u", sanitized_name, i);
193 - i++;
194 - } while (rrdset_index_find_name(host, new_name));
195 -
196 -// netdata_log_info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host));
197 - }
198 - else
199 - return NULL;
200 - }
201 -
202 - return string_strdupz(new_name);
203 -}
204 -
205 -struct rrdset_constructor {
206 - RRDHOST *host;
207 - const char *type;
208 - const char *id;
209 - const char *name;
210 - const char *family;
211 - const char *context;
212 - const char *title;
213 - const char *units;
214 - const char *plugin;
215 - const char *module;
216 - long priority;
217 - int update_every;
218 - RRDSET_TYPE chart_type;
219 - RRD_MEMORY_MODE memory_mode;
220 - long history_entries;
221 -
222 - enum {
223 - RRDSET_REACT_NONE = 0,
224 - RRDSET_REACT_NEW = (1 << 0),
225 - RRDSET_REACT_UPDATED = (1 << 1),
226 - RRDSET_REACT_PLUGIN_UPDATED = (1 << 2),
227 - RRDSET_REACT_MODULE_UPDATED = (1 << 3),
228 - RRDSET_REACT_CHART_ACTIVATED = (1 << 4),
229 - } react_action;
230 -};
231 -
232 -// the constructor - the dictionary is write locked while this runs
233 -static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) {
234 - struct rrdset_constructor *ctr = constructor_data;
235 - RRDHOST *host = ctr->host;
236 - RRDSET *st = rrdset;
237 -
238 - const char *chart_full_id = dictionary_acquired_item_name(item);
239 -
240 - st->id = string_strdupz(chart_full_id);
241 -
242 - st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->name);
243 - if(!st->name)
244 - st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->id);
245 - rrdset_index_add_name(host, st);
246 -
247 - st->parts.id = string_strdupz(ctr->id);
248 - st->parts.type = string_strdupz(ctr->type);
249 - st->parts.name = string_strdupz(ctr->name);
250 -
251 - st->family = (ctr->family && *ctr->family) ? rrd_string_strdupz(ctr->family) : rrd_string_strdupz(ctr->type);
252 - st->context = (ctr->context && *ctr->context) ? rrd_string_strdupz(ctr->context) : rrd_string_strdupz(chart_full_id);
253 -
254 - st->units = rrd_string_strdupz(ctr->units);
255 - st->title = rrd_string_strdupz(ctr->title);
256 - st->plugin_name = rrd_string_strdupz(ctr->plugin);
257 - st->module_name = rrd_string_strdupz(ctr->module);
258 - st->priority = ctr->priority;
259 -
260 - st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;
261 - st->update_every = ctr->update_every;
262 - st->rrd_memory_mode = ctr->memory_mode;
263 -
264 - st->chart_type = ctr->chart_type;
265 - st->rrdhost = host;
266 -
267 - rrdset_stream_send_chart_slot_assign(st);
268 -
269 - spinlock_init(&st->data_collection_lock);
270 -
271 - st->flags = RRDSET_FLAG_SYNC_CLOCK
272 - | RRDSET_FLAG_INDEXED_ID
273 - | RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED
274 - | RRDSET_FLAG_SENDER_REPLICATION_FINISHED
275 - ;
276 -
277 - rw_spinlock_init(&st->alerts.spinlock);
278 -
279 - // initialize the db tiers
280 - {
281 - for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
282 - STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
283 - if(!eng) continue;
284 -
285 - st->smg[tier] = storage_engine_metrics_group_get(eng->seb, host->db[tier].si, &st->chart_uuid);
286 - }
287 - }
288 -
289 - rrddim_index_init(st);
290 -
291 - st->rrdvars = rrdvariables_create();
292 - st->rrdlabels = rrdlabels_create();
293 - rrdset_update_permanent_labels(st);
294 -
295 - st->green = NAN;
296 - st->red = NAN;
297 -
298 - rrdset_pluginsd_receive_slots_initialize(st);
299 -
300 - rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
301 - rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
302 -
303 - ctr->react_action = RRDSET_REACT_NEW;
304 -
305 - ml_chart_new(st);
306 -}
307 -
308 -void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
309 - ND_LOG_STACK lgs[] = {
310 - ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(st->rrdhost)),
311 - ND_LOG_FIELD_TXT(NDF_NIDL_CONTEXT, rrdset_context(st)),
312 - ND_LOG_FIELD_TXT(NDF_NIDL_INSTANCE, rrdset_name(st)),
313 - ND_LOG_FIELD_END(),
314 - };
315 - ND_LOG_STACK_PUSH(lgs);
316 -
317 - RRDHOST *host = st->rrdhost;
318 -
319 - rrdset_flag_set(st, RRDSET_FLAG_COLLECTION_FINISHED);
320 -
321 - if(dimensions_too) {
322 - RRDDIM *rd;
323 - rrddim_foreach_read(rd, st)
324 - rrddim_finalize_collection_and_check_retention(rd);
325 - rrddim_foreach_done(rd);
326 - }
327 -
328 - for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
329 - STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
330 - if(!eng) continue;
331 -
332 - if(st->smg[tier]) {
333 - storage_engine_metrics_group_release(eng->seb, host->db[tier].si, st->smg[tier]);
334 - st->smg[tier] = NULL;
335 - }
336 - }
337 -
338 - rrdset_pluginsd_receive_unslot_and_cleanup(st);
339 -}
340 -
341 -// the destructor - the dictionary is write locked while this runs
342 -static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost) {
343 - RRDHOST *host = rrdhost;
344 - RRDSET *st = rrdset;
345 -
346 - rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
347 -
348 - rrdset_finalize_collection(st, false);
349 -
350 - rrdset_stream_send_chart_slot_release(st);
351 -
352 - // remove it from the name index
353 - rrdset_index_del_name(host, st);
354 -
355 - // release the collector info
356 - dictionary_destroy(st->functions_view);
357 -
358 - rrdcalc_unlink_and_delete_all_rrdset_alerts(st);
359 -
360 - // ------------------------------------------------------------------------
361 - // the order of destruction is important here
362 -
363 - // 1. delete RRDVAR index after the above, to avoid triggering its garbage collector (they have references on this)
364 - rrdvariables_destroy(st->rrdvars); // free all variables and destroy the rrdvar dictionary
365 -
366 - // 2. delete RRDDIMs, now their variables are not existing, so this is fast
367 - rrddim_index_destroy(st); // free all the dimensions and destroy the dimensions index
368 -
369 - // 3. this has to be after the dimensions are freed, but before labels are freed (contexts need the labels)
370 - rrdcontext_removed_rrdset(st); // let contexts know
371 -
372 - // 4. destroy the chart labels
373 - rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
374 -
375 - // 5. destroy the ml handle
376 - ml_chart_delete(st);
377 -
378 - // ------------------------------------------------------------------------
379 - // free it
380 -
381 - string_freez(st->id);
382 - string_freez(st->name);
383 - string_freez(st->parts.id);
384 - string_freez(st->parts.type);
385 - string_freez(st->parts.name);
386 - string_freez(st->family);
387 - string_freez(st->title);
388 - string_freez(st->units);
389 - string_freez(st->context);
390 - string_freez(st->plugin_name);
391 - string_freez(st->module_name);
392 -
393 - freez(st->exporting_flags);
394 -}
12 +// RRD - chart types
13
396 -// the item to be inserted, is already in the dictionary
397 -// this callback deals with the situation, migrating the existing object to the new values
398 -// the dictionary is write locked while this runs
399 -static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *new_rrdset, void *constructor_data) {
400 - (void)new_rrdset; // it is NULL
14 +inline RRDSET_TYPE rrdset_type_id(const char *name) {
15 + if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))
16 + return RRDSET_TYPE_AREA;
17
402 - struct rrdset_constructor *ctr = constructor_data;
403 - RRDSET *st = rrdset;
18 + else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0))
19 + return RRDSET_TYPE_STACKED;
20
405 - rrdset_isnot_obsolete___safe_from_collector_thread(st);
21 + else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0))
22 + return RRDSET_TYPE_HEATMAP;
23
407 - ctr->react_action = RRDSET_REACT_NONE;
408 -
409 - if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2)
410 - ctr->react_action |= RRDSET_REACT_UPDATED;
411 -
412 - if (unlikely(st->priority != ctr->priority)) {
413 - st->priority = ctr->priority;
414 - ctr->react_action |= RRDSET_REACT_UPDATED;
415 - }
416 -
417 - if (unlikely(st->update_every != ctr->update_every)) {
418 - rrdset_set_update_every_s(st, ctr->update_every);
419 - ctr->react_action |= RRDSET_REACT_UPDATED;
420 - }
421 -
422 - if(ctr->plugin && *ctr->plugin) {
423 - STRING *old_plugin = st->plugin_name;
424 - st->plugin_name = rrd_string_strdupz(ctr->plugin);
425 - if (old_plugin != st->plugin_name)
426 - ctr->react_action |= RRDSET_REACT_PLUGIN_UPDATED;
427 - string_freez(old_plugin);
428 - }
429 -
430 - if(ctr->module && *ctr->module) {
431 - STRING *old_module = st->module_name;
432 - st->module_name = rrd_string_strdupz(ctr->module);
433 - if (old_module != st->module_name)
434 - ctr->react_action |= RRDSET_REACT_MODULE_UPDATED;
435 - string_freez(old_module);
436 - }
437 -
438 - if(ctr->title && *ctr->title) {
439 - STRING *old_title = st->title;
440 - st->title = rrd_string_strdupz(ctr->title);
441 - if(old_title != st->title)
442 - ctr->react_action |= RRDSET_REACT_UPDATED;
443 - string_freez(old_title);
444 - }
445 -
446 - if(ctr->units && *ctr->units) {
447 - STRING *old_units = st->units;
448 - st->units = rrd_string_strdupz(ctr->units);
449 - if(old_units != st->units)
450 - ctr->react_action |= RRDSET_REACT_UPDATED;
451 - string_freez(old_units);
452 - }
453 -
454 - if(ctr->family && *ctr->family) {
455 - STRING *old_family = st->family;
456 - st->family = rrd_string_strdupz(ctr->family);
457 - if(old_family != st->family)
458 - ctr->react_action |= RRDSET_REACT_UPDATED;
459 - string_freez(old_family);
460 - }
461 -
462 - if(ctr->context && *ctr->context) {
463 - STRING *old_context = st->context;
464 - st->context = rrd_string_strdupz(ctr->context);
465 - if(old_context != st->context)
466 - ctr->react_action |= RRDSET_REACT_UPDATED;
467 - string_freez(old_context);
468 - }
469 -
470 - if(st->chart_type != ctr->chart_type) {
471 - st->chart_type = ctr->chart_type;
472 - ctr->react_action |= RRDSET_REACT_UPDATED;
473 - }
474 -
475 - rrdset_update_permanent_labels(st);
476 -
477 - rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
478 - rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
479 - rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
480 -
481 - return ctr->react_action != RRDSET_REACT_NONE;
482 -}
483 -
484 -// this is called after all insertions/conflicts, with the dictionary unlocked, with a reference to RRDSET
485 -// so, any actions requiring locks on other objects, should be placed here
486 -static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) {
487 - struct rrdset_constructor *ctr = constructor_data;
488 - RRDSET *st = rrdset;
489 - RRDHOST *host = st->rrdhost;
490 -
491 - st->last_accessed_time_s = now_realtime_sec();
492 -
493 - if(ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
494 - if (ctr->react_action & RRDSET_REACT_NEW) {
495 - if(unlikely(rrdcontext_find_chart_uuid(st, &st->chart_uuid)))
496 - uuid_generate(st->chart_uuid);
497 - }
498 - rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
499 - rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_UPDATE);
500 - }
501 -
502 - rrdset_metadata_updated(st);
24 + else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0))
25 + return RRDSET_TYPE_LINE;
26 }
27
505 -void rrdset_index_init(RRDHOST *host) {
506 - if(!host->rrdset_root_index) {
507 - host->rrdset_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
508 - &dictionary_stats_category_rrdset, sizeof(RRDSET));
28 +const char *rrdset_type_name(RRDSET_TYPE chart_type) {
29 + switch(chart_type) {
30 + case RRDSET_TYPE_LINE:
31 + default:
32 + return RRDSET_TYPE_LINE_NAME;
33
510 - dictionary_register_insert_callback(host->rrdset_root_index, rrdset_insert_callback, NULL);
511 - dictionary_register_conflict_callback(host->rrdset_root_index, rrdset_conflict_callback, NULL);
512 - dictionary_register_react_callback(host->rrdset_root_index, rrdset_react_callback, NULL);
513 - dictionary_register_delete_callback(host->rrdset_root_index, rrdset_delete_callback, host);
514 - }
34 + case RRDSET_TYPE_AREA:
35 + return RRDSET_TYPE_AREA_NAME;
36
516 - if(!host->rrdset_root_index_name) {
517 - host->rrdset_root_index_name = dictionary_create_advanced(
518 - DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
519 - &dictionary_stats_category_rrdset, 0);
37 + case RRDSET_TYPE_STACKED:
38 + return RRDSET_TYPE_STACKED_NAME;
39
521 - dictionary_register_insert_callback(host->rrdset_root_index_name, rrdset_name_insert_callback, host);
522 - dictionary_register_delete_callback(host->rrdset_root_index_name, rrdset_name_delete_callback, host);
40 + case RRDSET_TYPE_HEATMAP:
41 + return RRDSET_TYPE_HEATMAP_NAME;
42 }
43 }
44
526 -void rrdset_index_destroy(RRDHOST *host) {
527 - // destroy the name index first
528 - dictionary_destroy(host->rrdset_root_index_name);
529 - host->rrdset_root_index_name = NULL;
530 -
531 - // destroy the id index last
532 - dictionary_destroy(host->rrdset_root_index);
533 - host->rrdset_root_index = NULL;
534 -}
535 -
536 -static inline RRDSET *rrdset_index_add(RRDHOST *host, const char *id, struct rrdset_constructor *st_ctr) {
537 - return dictionary_set_advanced(host->rrdset_root_index, id, -1, NULL, sizeof(RRDSET), st_ctr);
538 -}
539 -
540 -static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) {
541 - if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID))
542 - dictionary_del(host->rrdset_root_index, rrdset_id(st));
543 -}
544 -
545 -static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) {
546 - // TODO - the name index should have an acquired dictionary item, not just a pointer to RRDSET
547 - if (unlikely(!host->rrdset_root_index))
548 - return NULL;
549 - return dictionary_get(host->rrdset_root_index, id);
550 -}
551 -
552 -// ----------------------------------------------------------------------------
553 -// RRDSET - find charts
554 -
555 -inline RRDSET *rrdset_find(RRDHOST *host, const char *id) {
556 - netdata_log_debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host));
557 - RRDSET *st = rrdset_index_find(host, id);
558 -
559 - if(st)
560 - st->last_accessed_time_s = now_realtime_sec();
561 -
562 - return(st);
563 -}
564 -
565 -inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id) {
566 - netdata_log_debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host));
567 -
568 - char buf[RRD_ID_LENGTH_MAX + 1];
569 - strncpyz(buf, type, RRD_ID_LENGTH_MAX - 1);
570 - strcat(buf, ".");
571 - int len = (int) strlen(buf);
572 - strncpyz(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len));
573 -
574 - return(rrdset_find(host, buf));
575 -}
576 -
577 -inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {
578 - netdata_log_debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host));
579 - RRDSET *st = rrdset_index_find_name(host, name);
580 - return(st);
581 -}
582 -
583 -RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id) {
584 - netdata_log_debug(D_RRD_CALLS, "rrdset_find_and_acquire() for host %s, chart %s", rrdhost_hostname(host), id);
585 -
586 - return (RRDSET_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdset_root_index, id);
587 -}
588 -
589 -RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa) {
590 - if(unlikely(!rsa))
591 - return NULL;
592 -
593 - return (RRDSET *) dictionary_acquired_item_value((const DICTIONARY_ITEM *)rsa);
594 -}
595 -
596 -void rrdset_acquired_release(RRDSET_ACQUIRED *rsa) {
597 - if(unlikely(!rsa))
598 - return;
599 -
600 - RRDSET *rs = rrdset_acquired_to_rrdset(rsa);
601 - dictionary_acquired_item_release(rs->rrdhost->rrdset_root_index, (const DICTIONARY_ITEM *)rsa);
602 -}
603 -
45 // ----------------------------------------------------------------------------
46 // RRDSET - rename charts
47
607 -int rrdset_reset_name(RRDSET *st, const char *name) {
608 - if(unlikely(!strcmp(rrdset_name(st), name)))
609 - return 1;
610 -
611 - RRDHOST *host = st->rrdhost;
612 -
613 - netdata_log_debug(D_RRD_CALLS, "rrdset_reset_name() old: '%s', new: '%s'", rrdset_name(st), name);
614 -
615 - STRING *name_string = rrdset_fix_name(host, rrdset_id(st), rrdset_parts_type(st), string2str(st->name), name);
616 - if(!name_string) return 0;
617 -
618 - if(st->name) {
619 - rrdset_index_del_name(host, st);
620 - string_freez(st->name);
621 - st->name = name_string;
622 - }
623 - else
624 - st->name = name_string;
625 -
626 - rrdset_index_add_name(host, st);
627 -
628 - rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);
629 - rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);
630 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
631 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);
632 - rrdset_metadata_updated(st);
633 -
634 - rrdcontext_updated_rrdset_name(st);
635 - return 2;
636 -}
637 -
48 // get the timestamp of the last entry in the round-robin database
49 time_t rrdset_last_entry_s(RRDSET *st) {
50 RRDDIM *rd;
@@ -828,1213 +238,3 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
238 rrdcontext_updated_rrdset_flags(st);
239 }
240 }
831 -
832 -// ----------------------------------------------------------------------------
833 -// RRDSET - reset a chart
834 -
835 -void rrdset_reset(RRDSET *st) {
836 - netdata_log_debug(D_RRD_CALLS, "rrdset_reset() %s", rrdset_name(st));
837 -
838 - st->last_collected_time.tv_sec = 0;
839 - st->last_collected_time.tv_usec = 0;
840 - st->last_updated.tv_sec = 0;
841 - st->last_updated.tv_usec = 0;
842 - st->db.current_entry = 0;
843 - st->counter = 0;
844 - st->counter_done = 0;
845 -
846 - RRDDIM *rd;
847 - rrddim_foreach_read(rd, st) {
848 - rd->collector.last_collected_time.tv_sec = 0;
849 - rd->collector.last_collected_time.tv_usec = 0;
850 - rd->collector.counter = 0;
851 -
852 - if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
853 - for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++)
854 - storage_engine_store_flush(rd->tiers[tier].sch);
855 - }
856 - }
857 - rrddim_foreach_done(rd);
858 -}
859 -
860 -// ----------------------------------------------------------------------------
861 -// RRDSET - helpers for rrdset_create()
862 -
863 -inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
864 - if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
865 - if(mode == RRD_MEMORY_MODE_NONE) return 5;
866 -
867 - if(entries < 5) entries = 5;
868 - if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
869 -
870 - if(mode == RRD_MEMORY_MODE_RAM) {
871 - long header_size = 0;
872 -
873 - long page = (long)sysconf(_SC_PAGESIZE);
874 - long size = (long)(header_size + entries * sizeof(storage_number));
875 - if (unlikely(size % page)) {
876 - size -= (size % page);
877 - size += page;
878 -
879 - long n = (long)((size - header_size) / sizeof(storage_number));
880 - return n;
881 - }
882 - }
883 -
884 - return entries;
885 -}
886 -
887 -static inline void last_collected_time_align(RRDSET *st) {
888 - st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
889 -
890 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
891 - st->last_collected_time.tv_usec = 0;
892 - else
893 - st->last_collected_time.tv_usec = 500000;
894 -}
895 -
896 -static inline void last_updated_time_align(RRDSET *st) {
897 - st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
898 - st->last_updated.tv_usec = 0;
899 -}
900 -
901 -// ----------------------------------------------------------------------------
902 -// RRDSET - free a chart
903 -
904 -void rrdset_free(RRDSET *st) {
905 - if(unlikely(!st)) return;
906 - rrdset_index_del(st->rrdhost, st);
907 -}
908 -
909 -// ----------------------------------------------------------------------------
910 -// RRDSET - create a chart
911 -
912 -RRDSET *rrdset_create_custom(
913 - RRDHOST *host
914 - , const char *type
915 - , const char *id
916 - , const char *name
917 - , const char *family
918 - , const char *context
919 - , const char *title
920 - , const char *units
921 - , const char *plugin
922 - , const char *module
923 - , long priority
924 - , int update_every
925 - , RRDSET_TYPE chart_type
926 - , RRD_MEMORY_MODE memory_mode
927 - , long history_entries
928 -) {
929 - if (host != localhost)
930 - host->stream.rcv.status.last_chart = now_realtime_sec();
931 -
932 - if(!type || !type[0])
933 - fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
934 - , (id && *id)?id:"<unset>"
935 - , (name && *name)?name:"<unset>"
936 - , (family && *family)?family:"<unset>"
937 - , (context && *context)?context:"<unset>"
938 - , (title && *title)?title:"<unset>"
939 - , (units && *units)?units:"<unset>"
940 - , (plugin && *plugin)?plugin:"<unset>"
941 - , (module && *module)?module:"<unset>"
942 - );
943 -
944 - if(!id || !id[0])
945 - fatal("Cannot create rrd stats without an id: type '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
946 - , type
947 - , (name && *name)?name:"<unset>"
948 - , (family && *family)?family:"<unset>"
949 - , (context && *context)?context:"<unset>"
950 - , (title && *title)?title:"<unset>"
951 - , (units && *units)?units:"<unset>"
952 - , (plugin && *plugin)?plugin:"<unset>"
953 - , (module && *module)?module:"<unset>"
954 - );
955 -
956 - // ------------------------------------------------------------------------
957 - // check if it already exists
958 -
959 - char full_id[RRD_ID_LENGTH_MAX + 1];
960 - snprintfz(full_id, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
961 -
962 - // ------------------------------------------------------------------------
963 - // allocate it
964 -
965 - netdata_log_debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
966 -
967 - struct rrdset_constructor tmp = {
968 - .host = host,
969 - .type = type,
970 - .id = id,
971 - .name = name,
972 - .family = family,
973 - .context = context,
974 - .title = title,
975 - .units = units,
976 - .plugin = plugin,
977 - .module = module,
978 - .priority = priority,
979 - .update_every = update_every,
980 - .chart_type = chart_type,
981 - .memory_mode = memory_mode,
982 - .history_entries = history_entries,
983 - };
984 -
985 - RRDSET *st = rrdset_index_add(host, full_id, &tmp);
986 - return(st);
987 -}
988 -
989 -// ----------------------------------------------------------------------------
990 -// RRDSET - data collection iteration control
991 -
992 -void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_last_update) {
993 - #ifdef NETDATA_INTERNAL_CHECKS
994 - char *discard_reason = NULL;
995 - usec_t discarded = duration_since_last_update;
996 - #endif
997 -
998 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_SYNC_CLOCK))) {
999 - // the chart needs to be re-synced to current time
1000 - rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
1001 -
1002 - // discard the duration supplied
1003 - duration_since_last_update = 0;
1004 -
1005 - #ifdef NETDATA_INTERNAL_CHECKS
1006 - if(!discard_reason) discard_reason = "SYNC CLOCK FLAG";
1007 - #endif
1008 - }
1009 -
1010 - if(unlikely(!st->last_collected_time.tv_sec)) {
1011 - // the first entry
1012 - duration_since_last_update = st->update_every * USEC_PER_SEC;
1013 - #ifdef NETDATA_INTERNAL_CHECKS
1014 - if(!discard_reason) discard_reason = "FIRST DATA COLLECTION";
1015 - #endif
1016 - }
1017 - else if(unlikely(!duration_since_last_update)) {
1018 - // no dt given by the plugin
1019 - duration_since_last_update = dt_usec(&now, &st->last_collected_time);
1020 - #ifdef NETDATA_INTERNAL_CHECKS
1021 - if(!discard_reason) discard_reason = "NO USEC GIVEN BY COLLECTOR";
1022 - #endif
1023 - }
1024 - else {
1025 - // microseconds has the time since the last collection
1026 - susec_t since_last_usec = dt_usec_signed(&now, &st->last_collected_time);
1027 -
1028 - if(unlikely(since_last_usec < 0)) {
1029 - // oops! the database is in the future
1030 - #ifdef NETDATA_INTERNAL_CHECKS
1031 - netdata_log_info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
1032 - " secs in the future (counter #%u, update #%u). Adjusting it to current time."
1033 - , rrdset_id(st)
1034 - , rrdhost_hostname(st->rrdhost)
1035 - , (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC
1036 - , st->counter
1037 - , st->counter_done
1038 - );
1039 - #endif
1040 -
1041 - duration_since_last_update = 0;
1042 -
1043 - #ifdef NETDATA_INTERNAL_CHECKS
1044 - if(!discard_reason) discard_reason = "COLLECTION TIME IN FUTURE";
1045 - #endif
1046 - }
1047 - else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 5 * USEC_PER_SEC))) {
1048 - // oops! the database is too far behind
1049 - #ifdef NETDATA_INTERNAL_CHECKS
1050 - netdata_log_info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
1051 - " secs in the past (counter #%u, update #%u). Adjusting it to current time.",
1052 - rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC,
1053 - st->counter, st->counter_done);
1054 - #endif
1055 -
1056 - duration_since_last_update = (usec_t)since_last_usec;
1057 -
1058 - #ifdef NETDATA_INTERNAL_CHECKS
1059 - if(!discard_reason) discard_reason = "COLLECTION TIME TOO FAR IN THE PAST";
1060 - #endif
1061 - }
1062 -
1063 -#ifdef NETDATA_INTERNAL_CHECKS
1064 - if(since_last_usec > 0 && (susec_t) duration_since_last_update < since_last_usec) {
1065 - static __thread susec_t min_delta = USEC_PER_SEC * 3600, permanent_min_delta = 0;
1066 - static __thread time_t last_time_s = 0;
1067 -
1068 - // the first time initialize it so that it will make the check later
1069 - if(last_time_s == 0) last_time_s = now.tv_sec + 60;
1070 -
1071 - susec_t delta = since_last_usec - (susec_t) duration_since_last_update;
1072 - if(delta < min_delta) min_delta = delta;
1073 -
1074 - if(now.tv_sec >= last_time_s + 60) {
1075 - last_time_s = now.tv_sec;
1076 -
1077 - if(min_delta > permanent_min_delta) {
1078 - netdata_log_info("MINIMUM MICROSECONDS DELTA of thread %d increased from %"PRIi64" to %"PRIi64" (+%"PRIi64")", gettid_cached(), permanent_min_delta, min_delta, min_delta - permanent_min_delta);
1079 - permanent_min_delta = min_delta;
1080 - }
1081 -
1082 - min_delta = USEC_PER_SEC * 3600;
1083 - }
1084 - }
1085 -#endif
1086 - }
1087 -
1088 - netdata_log_debug(D_RRD_CALLS, "rrdset_timed_next() for chart %s with duration since last update %"PRIu64" usec", rrdset_name(st), duration_since_last_update);
1089 - rrdset_debug(st, "NEXT: %"PRIu64" microseconds", duration_since_last_update);
1090 -
1091 - internal_error(discarded && discarded != duration_since_last_update,
1092 - "host '%s', chart '%s': discarded data collection time of %"PRIu64" usec, "
1093 - "replaced with %"PRIu64" usec, reason: '%s'"
1094 - , rrdhost_hostname(st->rrdhost)
1095 - , rrdset_id(st)
1096 - , discarded
1097 - , duration_since_last_update
1098 - , discard_reason?discard_reason:"UNDEFINED"
1099 - );
1100 -
1101 - st->usec_since_last_update = duration_since_last_update;
1102 -}
1103 -
1104 -inline void rrdset_next_usec_unfiltered(RRDSET *st, usec_t duration_since_last_update) {
1105 - if(unlikely(!st->last_collected_time.tv_sec || !duration_since_last_update || (rrdset_flag_check(st, RRDSET_FLAG_SYNC_CLOCK)))) {
1106 - // call the full next_usec() function
1107 - rrdset_next_usec(st, duration_since_last_update);
1108 - return;
1109 - }
1110 -
1111 - st->usec_since_last_update = duration_since_last_update;
1112 -}
1113 -
1114 -inline void rrdset_next_usec(RRDSET *st, usec_t duration_since_last_update) {
1115 - struct timeval now;
1116 -
1117 - now_realtime_timeval(&now);
1118 - rrdset_timed_next(st, now, duration_since_last_update);
1119 -}
1120 -
1121 -// ----------------------------------------------------------------------------
1122 -// RRDSET - process the collected values for all dimensions of a chart
1123 -
1124 -static inline usec_t rrdset_init_last_collected_time(RRDSET *st, struct timeval now) {
1125 - st->last_collected_time = now;
1126 - last_collected_time_align(st);
1127 -
1128 - usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
1129 -
1130 - rrdset_debug(st, "initialized last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
1131 -
1132 - return last_collect_ut;
1133 -}
1134 -
1135 -static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
1136 - usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
1137 - usec_t ut = last_collect_ut + st->usec_since_last_update;
1138 - st->last_collected_time.tv_sec = (time_t) (ut / USEC_PER_SEC);
1139 - st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC);
1140 -
1141 - rrdset_debug(st, "updated last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
1142 -
1143 - return last_collect_ut;
1144 -}
1145 -
1146 -static inline void rrdset_init_last_updated_time(RRDSET *st) {
1147 - // copy the last collected time to last updated time
1148 - st->last_updated.tv_sec = st->last_collected_time.tv_sec;
1149 - st->last_updated.tv_usec = st->last_collected_time.tv_usec;
1150 -
1151 - if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
1152 - st->last_updated.tv_sec -= st->update_every;
1153 -
1154 - last_updated_time_align(st);
1155 -}
1156 -
1157 -static __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];
1158 -
1159 -static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, time_t now_s) {
1160 - time_t loop = (time_t)rd->rrdset->update_every * (time_t)t->tier_grouping;
1161 - return now_s + loop - ((now_s + loop) % loop);
1162 -}
1163 -
1164 -void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {
1165 - if (unlikely(!t->next_point_end_time_s))
1166 - t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
1167 -
1168 - if(unlikely(sp.start_time_s >= t->next_point_end_time_s)) {
1169 - // flush the virtual point, it is done
1170 -
1171 - if (likely(!storage_point_is_unset(t->virtual_point))) {
1172 -
1173 - storage_engine_store_metric(
1174 - t->sch,
1175 - t->next_point_end_time_s * USEC_PER_SEC,
1176 - t->virtual_point.sum,
1177 - t->virtual_point.min,
1178 - t->virtual_point.max,
1179 - t->virtual_point.count,
1180 - t->virtual_point.anomaly_count,
1181 - t->virtual_point.flags);
1182 - }
1183 - else {
1184 - storage_engine_store_metric(
1185 - t->sch,
1186 - t->next_point_end_time_s * USEC_PER_SEC,
1187 - NAN,
1188 - NAN,
1189 - NAN,
1190 - 0,
1191 - 0, SN_FLAG_NONE);
1192 - }
1193 -
1194 - rrdset_done_statistics_points_stored_per_tier[tier]++;
1195 - t->virtual_point.count = 0; // make the point unset
1196 - t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);
1197 - }
1198 -
1199 - // merge the dates into our virtual point
1200 - if (unlikely(sp.start_time_s < t->virtual_point.start_time_s))
1201 - t->virtual_point.start_time_s = sp.start_time_s;
1202 -
1203 - if (likely(sp.end_time_s > t->virtual_point.end_time_s))
1204 - t->virtual_point.end_time_s = sp.end_time_s;
1205 -
1206 - // merge the values into our virtual point
1207 - if (likely(!storage_point_is_gap(sp))) {
1208 - // we aggregate only non NULLs into higher tiers
1209 -
1210 - if (likely(!storage_point_is_unset(t->virtual_point))) {
1211 - // merge the collected point to our virtual one
1212 - t->virtual_point.sum += sp.sum;
1213 - t->virtual_point.min = MIN(t->virtual_point.min, sp.min);
1214 - t->virtual_point.max = MAX(t->virtual_point.max, sp.max);
1215 - t->virtual_point.count += sp.count;
1216 - t->virtual_point.anomaly_count += sp.anomaly_count;
1217 - t->virtual_point.flags |= sp.flags;
1218 - }
1219 - else {
1220 - // reset our virtual point to this one
1221 - t->virtual_point = sp;
1222 - }
1223 - }
1224 -}
1225 -#ifdef NETDATA_LOG_COLLECTION_ERRORS
1226 -void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) {
1227 -#else // !NETDATA_LOG_COLLECTION_ERRORS
1228 -void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
1229 -#endif // !NETDATA_LOG_COLLECTION_ERRORS
1230 -
1231 - static __thread struct log_stack_entry lgs[] = {
1232 - [0] = ND_LOG_FIELD_STR(NDF_NIDL_DIMENSION, NULL),
1233 - [1] = ND_LOG_FIELD_END(),
1234 - };
1235 - lgs[0].str = rd->id;
1236 - log_stack_push(lgs);
1237 -
1238 -#ifdef NETDATA_LOG_COLLECTION_ERRORS
1239 - rd->rrddim_store_metric_count++;
1240 -
1241 - if(likely(rd->rrddim_store_metric_count > 1)) {
1242 - usec_t expected = rd->rrddim_store_metric_last_ut + rd->update_every * USEC_PER_SEC;
1243 -
1244 - if(point_end_time_ut != rd->rrddim_store_metric_last_ut) {
1245 - internal_error(true,
1246 - "%s COLLECTION: 'host:%s/chart:%s/dim:%s' granularity %d, collection %zu, expected to store at tier 0 a value at %llu, but it gave %llu [%s%llu usec] (called from %s(), previously by %s())",
1247 - (point_end_time_ut < rd->rrddim_store_metric_last_ut) ? "**PAST**" : "GAP",
1248 - rrdhost_hostname(rd->rrdset->rrdhost), rrdset_id(rd->rrdset), rrddim_id(rd),
1249 - rd->update_every,
1250 - rd->rrddim_store_metric_count,
1251 - expected, point_end_time_ut,
1252 - (point_end_time_ut < rd->rrddim_store_metric_last_ut)?"by -" : "gap ",
1253 - expected - point_end_time_ut,
1254 - function,
1255 - rd->rrddim_store_metric_last_caller?rd->rrddim_store_metric_last_caller:"none");
1256 - }
1257 - }
1258 -
1259 - rd->rrddim_store_metric_last_ut = point_end_time_ut;
1260 - rd->rrddim_store_metric_last_caller = function;
1261 -#endif // NETDATA_LOG_COLLECTION_ERRORS
1262 -
1263 - // store the metric on tier 0
1264 - storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut,
1265 - n, 0, 0,
1266 - 1, 0, flags);
1267 -
1268 - rrdset_done_statistics_points_stored_per_tier[0]++;
1269 -
1270 - time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC);
1271 -
1272 - STORAGE_POINT sp = {
1273 - .start_time_s = now_s - rd->rrdset->update_every,
1274 - .end_time_s = now_s,
1275 - .min = n,
1276 - .max = n,
1277 - .sum = n,
1278 - .count = 1,
1279 - .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,
1280 - .flags = flags
1281 - };
1282 -
1283 - for(size_t tier = 1; tier < nd_profile.storage_tiers;tier++) {
1284 - if(unlikely(!rd->tiers[tier].smh)) continue;
1285 -
1286 - struct rrddim_tier *t = &rd->tiers[tier];
1287 -
1288 - if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) {
1289 - // we have not collected this tier before
1290 - // let's fill any gap that may exist
1291 - backfill_tier_from_smaller_tiers(rd, tier, now_s);
1292 - }
1293 -
1294 - store_metric_at_tier(rd, tier, t, sp, point_end_time_ut);
1295 - }
1296 - rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS);
1297 -
1298 - rrdcontext_collected_rrddim(rd);
1299 - log_stack_pop(&lgs);
1300 -}
1301 -
1302 -void store_metric_collection_completed() {
1303 - pulse_queries_rrdset_collection_completed(rrdset_done_statistics_points_stored_per_tier);
1304 -}
1305 -
1306 -// caching of dimensions rrdset_done() and rrdset_done_interpolate() loop through
1307 -struct rda_item {
1308 - const DICTIONARY_ITEM *item;
1309 - RRDDIM *rd;
1310 - bool reset_or_overflow;
1311 -};
1312 -
1313 -static __thread struct rda_item *thread_rda = NULL;
1314 -static __thread size_t thread_rda_entries = 0;
1315 -
1316 -struct rda_item *rrdset_thread_rda_get(size_t *dimensions) {
1317 -
1318 - if(unlikely(!thread_rda || (*dimensions) > thread_rda_entries)) {
1319 - size_t old_mem = thread_rda_entries * sizeof(struct rda_item);
1320 - freez(thread_rda);
1321 - thread_rda_entries = *dimensions;
1322 - size_t new_mem = thread_rda_entries * sizeof(struct rda_item);
1323 - thread_rda = mallocz(new_mem);
1324 -
1325 - __atomic_add_fetch(&netdata_buffers_statistics.rrdset_done_rda_size, new_mem - old_mem, __ATOMIC_RELAXED);
1326 - }
1327 -
1328 - *dimensions = thread_rda_entries;
1329 - return thread_rda;
1330 -}
1331 -
1332 -void rrdset_thread_rda_free(void) {
1333 - __atomic_sub_fetch(&netdata_buffers_statistics.rrdset_done_rda_size, thread_rda_entries * sizeof(struct rda_item), __ATOMIC_RELAXED);
1334 -
1335 - freez(thread_rda);
1336 - thread_rda = NULL;
1337 - thread_rda_entries = 0;
1338 -}
1339 -
1340 -static inline size_t rrdset_done_interpolate(
1341 - RRDSET_STREAM_BUFFER *rsb
1342 - , RRDSET *st
1343 - , struct rda_item *rda_base
1344 - , size_t rda_slots
1345 - , usec_t update_every_ut
1346 - , usec_t last_stored_ut
1347 - , usec_t next_store_ut
1348 - , usec_t last_collect_ut
1349 - , usec_t now_collect_ut
1350 - , char store_this_entry
1351 -) {
1352 - RRDDIM *rd;
1353 -
1354 - size_t stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done()
1355 -
1356 - usec_t first_ut = last_stored_ut, last_ut = 0;
1357 - (void)first_ut;
1358 -
1359 - ssize_t iterations = (ssize_t)((now_collect_ut - last_stored_ut) / (update_every_ut));
1360 - if((now_collect_ut % (update_every_ut)) == 0) iterations++;
1361 -
1362 - size_t counter = st->counter;
1363 - long current_entry = st->db.current_entry;
1364 -
1365 - for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
1366 -
1367 - internal_error(iterations < 0,
1368 - "RRDSET: '%s': iterations calculation wrapped! "
1369 - "first_ut = %"PRIu64", last_stored_ut = %"PRIu64", next_store_ut = %"PRIu64", now_collect_ut = %"PRIu64""
1370 - , rrdset_id(st)
1371 - , first_ut
1372 - , last_stored_ut
1373 - , next_store_ut
1374 - , now_collect_ut
1375 - );
1376 -
1377 - rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
1378 - rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
1379 -
1380 - last_ut = next_store_ut;
1381 -
1382 - ml_chart_update_begin(st);
1383 -
1384 - struct rda_item *rda;
1385 - size_t dim_id;
1386 - for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
1387 - rd = rda->rd;
1388 - if(unlikely(!rd)) continue;
1389 -
1390 - SN_FLAGS storage_flags = SN_DEFAULT_FLAGS;
1391 -
1392 - if (rda->reset_or_overflow)
1393 - storage_flags |= SN_FLAG_RESET;
1394 -
1395 - NETDATA_DOUBLE new_value;
1396 -
1397 - switch(rd->algorithm) {
1398 - case RRD_ALGORITHM_INCREMENTAL:
1399 - new_value = (NETDATA_DOUBLE)
1400 - ( rd->collector.calculated_value
1401 - * (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
1402 - / (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
1403 - );
1404 -
1405 - rrdset_debug(st, "%s: CALC2 INC " NETDATA_DOUBLE_FORMAT " = "
1406 - NETDATA_DOUBLE_FORMAT
1407 - " * (%"PRIu64" - %"PRIu64")"
1408 - " / (%"PRIu64" - %"PRIu64""
1409 - , rrddim_name(rd)
1410 - , new_value
1411 - , rd->collector.calculated_value
1412 - , next_store_ut, last_collect_ut
1413 - , now_collect_ut, last_collect_ut
1414 - );
1415 -
1416 - rd->collector.calculated_value -= new_value;
1417 - new_value += rd->collector.last_calculated_value;
1418 - rd->collector.last_calculated_value = 0;
1419 - new_value /= (NETDATA_DOUBLE)st->update_every;
1420 -
1421 - if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
1422 -
1423 - rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
1424 - rrddim_name(rd)
1425 - , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
1426 - );
1427 -
1428 - new_value = new_value * (NETDATA_DOUBLE)(st->update_every * USEC_PER_SEC) / (NETDATA_DOUBLE)(next_store_ut - last_stored_ut);
1429 - }
1430 - break;
1431 -
1432 - case RRD_ALGORITHM_ABSOLUTE:
1433 - case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
1434 - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
1435 - default:
1436 - if(iterations == 1) {
1437 - // this is the last iteration
1438 - // do not interpolate
1439 - // just show the calculated value
1440 -
1441 - new_value = rd->collector.calculated_value;
1442 - }
1443 - else {
1444 - // we have missed an update
1445 - // interpolate in the middle values
1446 -
1447 - new_value = (NETDATA_DOUBLE)
1448 - ( ( (rd->collector.calculated_value - rd->collector.last_calculated_value)
1449 - * (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
1450 - / (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
1451 - )
1452 - + rd->collector.last_calculated_value
1453 - );
1454 -
1455 - rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = ((("
1456 - "(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")"
1457 - " * %"PRIu64""
1458 - " / %"PRIu64") + " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
1459 - , new_value
1460 - , rd->collector.calculated_value, rd->collector.last_calculated_value
1461 - , (next_store_ut - first_ut)
1462 - , (now_collect_ut - first_ut), rd->collector.last_calculated_value
1463 - );
1464 - }
1465 - break;
1466 - }
1467 -
1468 - time_t current_time_s = (time_t) (next_store_ut / USEC_PER_SEC);
1469 -
1470 - if(unlikely(!store_this_entry)) {
1471 - (void) ml_dimension_is_anomalous(rd, current_time_s, 0, false);
1472 -
1473 - if(rsb->wb && rsb->v2)
1474 - stream_send_rrddim_metrics_v2(rsb, rd, next_store_ut, NAN, SN_FLAG_NONE);
1475 -
1476 - rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1477 - continue;
1478 - }
1479 -
1480 - if(likely(rrddim_check_updated(rd) && rd->collector.counter > 1 && iterations < gap_when_lost_iterations_above)) {
1481 - uint32_t dim_storage_flags = storage_flags;
1482 -
1483 - if (ml_dimension_is_anomalous(rd, current_time_s, new_value, true)) {
1484 - // clear anomaly bit: 0 -> is anomalous, 1 -> not anomalous
1485 - dim_storage_flags &= ~((storage_number)SN_FLAG_NOT_ANOMALOUS);
1486 - }
1487 -
1488 - if(rsb->wb && rsb->v2)
1489 - stream_send_rrddim_metrics_v2(rsb, rd, next_store_ut, new_value, dim_storage_flags);
1490 -
1491 - rrddim_store_metric(rd, next_store_ut, new_value, dim_storage_flags);
1492 - rd->collector.last_stored_value = new_value;
1493 - }
1494 - else {
1495 - (void) ml_dimension_is_anomalous(rd, current_time_s, 0, false);
1496 -
1497 - rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
1498 -
1499 - if(rsb->wb && rsb->v2)
1500 - stream_send_rrddim_metrics_v2(rsb, rd, next_store_ut, NAN, SN_FLAG_NONE);
1501 -
1502 - rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1503 - rd->collector.last_stored_value = NAN;
1504 - }
1505 -
1506 - stored_entries++;
1507 - }
1508 -
1509 - ml_chart_update_end(st);
1510 -
1511 - st->counter = ++counter;
1512 - st->db.current_entry = current_entry = ((current_entry + 1) >= st->db.entries) ? 0 : current_entry + 1;
1513 -
1514 - st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC);
1515 - st->last_updated.tv_usec = 0;
1516 -
1517 - last_stored_ut = next_store_ut;
1518 - }
1519 -
1520 -/*
1521 - st->counter = counter;
1522 - st->current_entry = current_entry;
1523 -
1524 - if(likely(last_ut)) {
1525 - st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC);
1526 - st->last_updated.tv_usec = 0;
1527 - }
1528 -*/
1529 -
1530 - return stored_entries;
1531 -}
1532 -
1533 -void rrdset_done(RRDSET *st) {
1534 - struct timeval now;
1535 -
1536 - now_realtime_timeval(&now);
1537 - rrdset_timed_done(st, now, /* pending_rrdset_next = */ st->counter_done != 0);
1538 -}
1539 -
1540 -void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) {
1541 - if(unlikely(!service_running(SERVICE_COLLECTORS))) return;
1542 -
1543 - RRDSET_STREAM_BUFFER stream_buffer = { .wb = NULL, };
1544 - if(unlikely(rrdhost_has_stream_sender_enabled(st->rrdhost)))
1545 - stream_buffer = stream_send_metrics_init(st, now.tv_sec);
1546 -
1547 - spinlock_lock(&st->data_collection_lock);
1548 -
1549 - if (pending_rrdset_next)
1550 - rrdset_timed_next(st, now, 0ULL);
1551 -
1552 - netdata_log_debug(D_RRD_CALLS, "rrdset_done() for chart '%s'", rrdset_name(st));
1553 -
1554 - RRDDIM *rd;
1555 -
1556 - char
1557 - store_this_entry = 1, // boolean: 1 = store this entry, 0 = don't store this entry
1558 - first_entry = 0; // boolean: 1 = this is the first entry seen for this chart, 0 = all other entries
1559 -
1560 - usec_t
1561 - last_collect_ut = 0, // the timestamp in microseconds, of the last collected value
1562 - now_collect_ut = 0, // the timestamp in microseconds, of this collected value (this is NOW)
1563 - last_stored_ut = 0, // the timestamp in microseconds, of the last stored entry in the db
1564 - next_store_ut = 0, // the timestamp in microseconds, of the next entry to store in the db
1565 - update_every_ut = st->update_every * USEC_PER_SEC; // st->update_every in microseconds
1566 -
1567 - RRDSET_FLAGS rrdset_flags = rrdset_flag_check(st, ~0);
1568 - if(unlikely(rrdset_flags & RRDSET_FLAG_COLLECTION_FINISHED)) {
1569 - spinlock_unlock(&st->data_collection_lock);
1570 - return;
1571 - }
1572 -
1573 - if (unlikely(rrdset_flags & RRDSET_FLAG_OBSOLETE)) {
1574 - netdata_log_error("Chart '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st));
1575 - rrdset_isnot_obsolete___safe_from_collector_thread(st);
1576 - }
1577 -
1578 - // check if the chart has a long time to be updated
1579 - if(unlikely(st->usec_since_last_update > MAX(st->db.entries, 60) * update_every_ut)) {
1580 - nd_log_daemon(NDLP_DEBUG, "host '%s', chart '%s': took too long to be updated (counter #%u, update #%u, %0.3" NETDATA_DOUBLE_MODIFIER
1581 - " secs). Resetting it.", rrdhost_hostname(st->rrdhost), rrdset_id(st), st->counter, st->counter_done,
1582 - (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
1583 - rrdset_reset(st);
1584 - st->usec_since_last_update = update_every_ut;
1585 - store_this_entry = 0;
1586 - first_entry = 1;
1587 - }
1588 -
1589 - rrdset_debug(st, "microseconds since last update: %"PRIu64"", st->usec_since_last_update);
1590 -
1591 - // set last_collected_time
1592 - if(unlikely(!st->last_collected_time.tv_sec)) {
1593 - // it is the first entry
1594 - // set the last_collected_time to now
1595 - last_collect_ut = rrdset_init_last_collected_time(st, now) - update_every_ut;
1596 -
1597 - // the first entry should not be stored
1598 - store_this_entry = 0;
1599 - first_entry = 1;
1600 - }
1601 - else {
1602 - // it is not the first entry
1603 - // calculate the proper last_collected_time, using usec_since_last_update
1604 - last_collect_ut = rrdset_update_last_collected_time(st);
1605 - }
1606 -
1607 - // if this set has not been updated in the past
1608 - // we fake the last_update time to be = now - usec_since_last_update
1609 - if(unlikely(!st->last_updated.tv_sec)) {
1610 - // it has never been updated before
1611 - // set a fake last_updated, in the past using usec_since_last_update
1612 - rrdset_init_last_updated_time(st);
1613 -
1614 - // the first entry should not be stored
1615 - store_this_entry = 0;
1616 - first_entry = 1;
1617 - }
1618 -
1619 - // check if we will re-write the entire data set
1620 - if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&
1621 - st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
1622 - nd_log_daemon(NDLP_DEBUG, "'%s': too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). "
1623 - "Resetting it. Will not store the next entry.",
1624 - rrdset_id(st),
1625 - (int64_t)st->last_updated.tv_sec,
1626 - (int64_t)st->last_updated.tv_usec,
1627 - (int64_t)st->last_collected_time.tv_sec,
1628 - (int64_t)st->last_collected_time.tv_usec);
1629 - rrdset_reset(st);
1630 - rrdset_init_last_updated_time(st);
1631 -
1632 - st->usec_since_last_update = update_every_ut;
1633 -
1634 - // the first entry should not be stored
1635 - store_this_entry = 0;
1636 - first_entry = 1;
1637 - }
1638 -
1639 - // these are the 3 variables that will help us in interpolation
1640 - // last_stored_ut = the last time we added a value to the storage
1641 - // now_collect_ut = the time the current value has been collected
1642 - // next_store_ut = the time of the next interpolation point
1643 - now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
1644 - last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
1645 - next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
1646 -
1647 - if(unlikely(!st->counter_done)) {
1648 - // set a fake last_updated to jump to current time
1649 - rrdset_init_last_updated_time(st);
1650 -
1651 - last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
1652 - next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
1653 -
1654 - if(unlikely(rrdset_flags & RRDSET_FLAG_STORE_FIRST)) {
1655 - store_this_entry = 1;
1656 - last_collect_ut = next_store_ut - update_every_ut;
1657 -
1658 - rrdset_debug(st, "Fixed first entry.");
1659 - }
1660 - else {
1661 - store_this_entry = 0;
1662 -
1663 - rrdset_debug(st, "Will not store the next entry.");
1664 - }
1665 - }
1666 -
1667 - st->counter_done++;
1668 -
1669 - if(stream_buffer.wb && !stream_buffer.v2)
1670 - stream_send_rrdset_metrics_v1(&stream_buffer, st);
1671 -
1672 - size_t rda_slots = dictionary_entries(st->rrddim_root_index);
1673 - struct rda_item *rda_base = rrdset_thread_rda_get(&rda_slots);
1674 -
1675 - size_t dim_id;
1676 - size_t dimensions = 0;
1677 - struct rda_item *rda = rda_base;
1678 - total_number collected_total = 0;
1679 - total_number last_collected_total = 0;
1680 - rrddim_foreach_read(rd, st) {
1681 - if(rd_dfe.counter >= rda_slots)
1682 - break;
1683 -
1684 - rda = &rda_base[dimensions++];
1685 -
1686 - if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
1687 - rda->item = NULL;
1688 - rda->rd = NULL;
1689 - rda->reset_or_overflow = false;
1690 - continue;
1691 - }
1692 -
1693 - // store the dimension in the array
1694 - rda->item = dictionary_acquired_item_dup(st->rrddim_root_index, rd_dfe.item);
1695 - rda->rd = dictionary_acquired_item_value(rda->item);
1696 - rda->reset_or_overflow = false;
1697 -
1698 - // calculate totals
1699 - if(likely(rrddim_check_updated(rd))) {
1700 - // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1701 - // to reset the calculation (it will give zero as the calculation for this second)
1702 - if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->collector.last_collected_value > rd->collector.collected_value)) {
1703 - netdata_log_debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
1704 - , rrdset_id(st)
1705 - , rrddim_name(rd)
1706 - , rd->collector.last_collected_value
1707 - , rd->collector.collected_value
1708 - );
1709 -
1710 - if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
1711 - rda->reset_or_overflow = true;
1712 -
1713 - rd->collector.last_collected_value = rd->collector.collected_value;
1714 - }
1715 -
1716 - last_collected_total += rd->collector.last_collected_value;
1717 - collected_total += rd->collector.collected_value;
1718 -
1719 - if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
1720 - netdata_log_error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
1721 - rrddim_isnot_obsolete___safe_from_collector_thread(st, rd);
1722 - }
1723 - }
1724 - }
1725 - rrddim_foreach_done(rd);
1726 - rda_slots = dimensions;
1727 -
1728 - rrdset_debug(st, "last_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last collection time)", (NETDATA_DOUBLE)last_collect_ut/USEC_PER_SEC);
1729 - rrdset_debug(st, "now_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (current collection time)", (NETDATA_DOUBLE)now_collect_ut/USEC_PER_SEC);
1730 - rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
1731 - rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
1732 -
1733 - // process all dimensions to calculate their values
1734 - // based on the collected figures only
1735 - // at this stage we do not interpolate anything
1736 - for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
1737 - rd = rda->rd;
1738 - if(unlikely(!rd)) continue;
1739 -
1740 - if(unlikely(!rrddim_check_updated(rd))) {
1741 - rd->collector.calculated_value = 0;
1742 - continue;
1743 - }
1744 -
1745 - rrdset_debug(st, "%s: START "
1746 - " last_collected_value = " COLLECTED_NUMBER_FORMAT
1747 - " collected_value = " COLLECTED_NUMBER_FORMAT
1748 - " last_calculated_value = " NETDATA_DOUBLE_FORMAT
1749 - " calculated_value = " NETDATA_DOUBLE_FORMAT
1750 - , rrddim_name(rd)
1751 - , rd->collector.last_collected_value
1752 - , rd->collector.collected_value
1753 - , rd->collector.last_calculated_value
1754 - , rd->collector.calculated_value
1755 - );
1756 -
1757 - switch(rd->algorithm) {
1758 - case RRD_ALGORITHM_ABSOLUTE:
1759 - rd->collector.calculated_value = (NETDATA_DOUBLE)rd->collector.collected_value
1760 - * (NETDATA_DOUBLE)rd->multiplier
1761 - / (NETDATA_DOUBLE)rd->divisor;
1762 -
1763 - rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " NETDATA_DOUBLE_FORMAT " = "
1764 - COLLECTED_NUMBER_FORMAT
1765 - " * " NETDATA_DOUBLE_FORMAT
1766 - " / " NETDATA_DOUBLE_FORMAT
1767 - , rrddim_name(rd)
1768 - , rd->collector.calculated_value
1769 - , rd->collector.collected_value
1770 - , (NETDATA_DOUBLE)rd->multiplier
1771 - , (NETDATA_DOUBLE)rd->divisor
1772 - );
1773 - break;
1774 -
1775 - case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
1776 - if(unlikely(!collected_total))
1777 - rd->collector.calculated_value = 0;
1778 - else
1779 - // the percentage of the current value
1780 - // over the total of all dimensions
1781 - rd->collector.calculated_value =
1782 - (NETDATA_DOUBLE)100
1783 - * (NETDATA_DOUBLE)rd->collector.collected_value
1784 - / (NETDATA_DOUBLE)collected_total;
1785 -
1786 - rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
1787 - " * " COLLECTED_NUMBER_FORMAT
1788 - " / " COLLECTED_NUMBER_FORMAT
1789 - , rrddim_name(rd)
1790 - , rd->collector.calculated_value
1791 - , rd->collector.collected_value
1792 - , collected_total
1793 - );
1794 - break;
1795 -
1796 - case RRD_ALGORITHM_INCREMENTAL:
1797 - if(unlikely(rd->collector.counter <= 1)) {
1798 - rd->collector.calculated_value = 0;
1799 - continue;
1800 - }
1801 -
1802 - // If the new is smaller than the old (an overflow, or reset), set the old equal to the new
1803 - // to reset the calculation (it will give zero as the calculation for this second).
1804 - // It is imperative to set the comparison to uint64_t since type collected_number is signed and
1805 - // produces wrong results as far as incremental counters are concerned.
1806 - if(unlikely((uint64_t)rd->collector.last_collected_value > (uint64_t)rd->collector.collected_value)) {
1807 - netdata_log_debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
1808 - , rrdset_id(st)
1809 - , rrddim_name(rd)
1810 - , rd->collector.last_collected_value
1811 - , rd->collector.collected_value);
1812 -
1813 - if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
1814 - rda->reset_or_overflow = true;
1815 -
1816 - uint64_t last = (uint64_t)rd->collector.last_collected_value;
1817 - uint64_t new = (uint64_t)rd->collector.collected_value;
1818 - uint64_t max = (uint64_t)rd->collector.collected_value_max;
1819 - uint64_t cap = 0;
1820 -
1821 - // Signed values are handled by exploiting two's complement which will produce positive deltas
1822 - if (max > 0x00000000FFFFFFFFULL)
1823 - cap = 0xFFFFFFFFFFFFFFFFULL; // handles signed and unsigned 64-bit counters
1824 - else
1825 - cap = 0x00000000FFFFFFFFULL; // handles signed and unsigned 32-bit counters
1826 -
1827 - uint64_t delta = cap - last + new;
1828 - uint64_t max_acceptable_rate = (cap / 100) * MAX_INCREMENTAL_PERCENT_RATE;
1829 -
1830 - // If the delta is less than the maximum acceptable rate and the previous value was near the cap
1831 - // then this is an overflow. There can be false positives such that a reset is detected as an
1832 - // overflow.
1833 - // TODO: remember recent history of rates and compare with current rate to reduce this chance.
1834 - if (delta < max_acceptable_rate) {
1835 - rd->collector.calculated_value +=
1836 - (NETDATA_DOUBLE) delta
1837 - * (NETDATA_DOUBLE) rd->multiplier
1838 - / (NETDATA_DOUBLE) rd->divisor;
1839 - } else {
1840 - // This is a reset. Any overflow with a rate greater than MAX_INCREMENTAL_PERCENT_RATE will also
1841 - // be detected as a reset instead.
1842 - rd->collector.calculated_value += (NETDATA_DOUBLE)0;
1843 - }
1844 - }
1845 - else {
1846 - rd->collector.calculated_value +=
1847 - (NETDATA_DOUBLE) (rd->collector.collected_value - rd->collector.last_collected_value)
1848 - * (NETDATA_DOUBLE) rd->multiplier
1849 - / (NETDATA_DOUBLE) rd->divisor;
1850 - }
1851 -
1852 - rrdset_debug(st, "%s: CALC INC PRE " NETDATA_DOUBLE_FORMAT " = ("
1853 - COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
1854 - ")"
1855 - " * " NETDATA_DOUBLE_FORMAT
1856 - " / " NETDATA_DOUBLE_FORMAT
1857 - , rrddim_name(rd)
1858 - , rd->collector.calculated_value
1859 - , rd->collector.collected_value, rd->collector.last_collected_value
1860 - , (NETDATA_DOUBLE)rd->multiplier
1861 - , (NETDATA_DOUBLE)rd->divisor
1862 - );
1863 - break;
1864 -
1865 - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
1866 - if(unlikely(rd->collector.counter <= 1)) {
1867 - rd->collector.calculated_value = 0;
1868 - continue;
1869 - }
1870 -
1871 - // the percentage of the current increment
1872 - // over the increment of all dimensions together
1873 - if(unlikely(collected_total == last_collected_total))
1874 - rd->collector.calculated_value = 0;
1875 - else
1876 - rd->collector.calculated_value =
1877 - (NETDATA_DOUBLE)100
1878 - * (NETDATA_DOUBLE)(rd->collector.collected_value - rd->collector.last_collected_value)
1879 - / (NETDATA_DOUBLE)(collected_total - last_collected_total);
1880 -
1881 - rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
1882 - " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1883 - " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1884 - , rrddim_name(rd)
1885 - , rd->collector.calculated_value
1886 - , rd->collector.collected_value, rd->collector.last_collected_value
1887 - , collected_total, last_collected_total
1888 - );
1889 - break;
1890 -
1891 - default:
1892 - // make the default zero, to make sure
1893 - // it gets noticed when we add new types
1894 - rd->collector.calculated_value = 0;
1895 -
1896 - rrdset_debug(st, "%s: CALC " NETDATA_DOUBLE_FORMAT " = 0"
1897 - , rrddim_name(rd)
1898 - , rd->collector.calculated_value
1899 - );
1900 - break;
1901 - }
1902 -
1903 - rrdset_debug(st, "%s: PHASE2 "
1904 - " last_collected_value = " COLLECTED_NUMBER_FORMAT
1905 - " collected_value = " COLLECTED_NUMBER_FORMAT
1906 - " last_calculated_value = " NETDATA_DOUBLE_FORMAT
1907 - " calculated_value = " NETDATA_DOUBLE_FORMAT
1908 - , rrddim_name(rd)
1909 - , rd->collector.last_collected_value
1910 - , rd->collector.collected_value
1911 - , rd->collector.last_calculated_value
1912 - , rd->collector.calculated_value
1913 - );
1914 - }
1915 -
1916 - // at this point we have all the calculated values ready
1917 - // it is now time to interpolate values on a second boundary
1918 -
1919 -// #ifdef NETDATA_INTERNAL_CHECKS
1920 -// if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) {
1921 -// // this is collected in the same interpolation point
1922 -// rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
1923 -// netdata_log_info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, rrdset_name(st), st->counter_done, next_store_ut - now_collect_ut);
1924 -// }
1925 -// #endif
1926 -
1927 - rrdset_done_interpolate(
1928 - &stream_buffer
1929 - , st
1930 - , rda_base
1931 - , rda_slots
1932 - , update_every_ut
1933 - , last_stored_ut
1934 - , next_store_ut
1935 - , last_collect_ut
1936 - , now_collect_ut
1937 - , store_this_entry
1938 - );
1939 -
1940 - for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
1941 - rd = rda->rd;
1942 - if(unlikely(!rd)) continue;
1943 -
1944 - if(unlikely(!rrddim_check_updated(rd)))
1945 - continue;
1946 -
1947 - rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rrddim_name(rd), rd->collector.last_collected_value, rd->collector.collected_value);
1948 -
1949 - rd->collector.last_collected_value = rd->collector.collected_value;
1950 -
1951 - switch(rd->algorithm) {
1952 - case RRD_ALGORITHM_INCREMENTAL:
1953 - if(unlikely(!first_entry)) {
1954 - rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT ") to "
1955 - "last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")"
1956 - , rrddim_name(rd)
1957 - , rd->collector.last_calculated_value + rd->collector.calculated_value
1958 - , rd->collector.calculated_value);
1959 -
1960 - rd->collector.last_calculated_value += rd->collector.calculated_value;
1961 - }
1962 - else {
1963 - rrdset_debug(st, "THIS IS THE FIRST POINT");
1964 - }
1965 - break;
1966 -
1967 - case RRD_ALGORITHM_ABSOLUTE:
1968 - case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
1969 - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
1970 - rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT ") to "
1971 - "last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")"
1972 - , rrddim_name(rd)
1973 - , rd->collector.last_calculated_value
1974 - , rd->collector.calculated_value);
1975 -
1976 - rd->collector.last_calculated_value = rd->collector.calculated_value;
1977 - break;
1978 - }
1979 -
1980 - rd->collector.calculated_value = 0;
1981 - rd->collector.collected_value = 0;
1982 - rrddim_clear_updated(rd);
1983 -
1984 - rrdset_debug(st, "%s: END "
1985 - " last_collected_value = " COLLECTED_NUMBER_FORMAT
1986 - " collected_value = " COLLECTED_NUMBER_FORMAT
1987 - " last_calculated_value = " NETDATA_DOUBLE_FORMAT
1988 - " calculated_value = " NETDATA_DOUBLE_FORMAT
1989 - , rrddim_name(rd)
1990 - , rd->collector.last_collected_value
1991 - , rd->collector.collected_value
1992 - , rd->collector.last_calculated_value
1993 - , rd->collector.calculated_value
1994 - );
1995 - }
1996 -
1997 - spinlock_unlock(&st->data_collection_lock);
1998 - stream_send_rrdset_metrics_finished(&stream_buffer, st);
1999 -
2000 - // ALL DONE ABOUT THE DATA UPDATE
2001 - // --------------------------------------------------------------------
2002 -
2003 - for(dim_id = 0, rda = rda_base; dim_id < rda_slots ; ++dim_id, ++rda) {
2004 - rd = rda->rd;
2005 - if(unlikely(!rd)) continue;
2006 -
2007 - dictionary_acquired_item_release(st->rrddim_root_index, rda->item);
2008 - rda->item = NULL;
2009 - rda->rd = NULL;
2010 - }
2011 -
2012 - rrdcontext_collected_rrdset(st);
2013 -
2014 - store_metric_collection_completed();
2015 -}
2016 -
2017 -time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
2018 - if(unlikely(update_every_s == st->update_every))
2019 - return st->update_every;
2020 -
2021 - internal_error(true, "RRDSET '%s' switching update every from %d to %d",
2022 - rrdset_id(st), (int)st->update_every, (int)update_every_s);
2023 -
2024 - time_t prev_update_every_s = (time_t) st->update_every;
2025 - st->update_every = (int) update_every_s;
2026 -
2027 - // switch update every to the storage engine
2028 - RRDDIM *rd;
2029 - rrddim_foreach_read(rd, st) {
2030 - for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
2031 - if (rd->tiers[tier].sch)
2032 - storage_engine_store_change_collection_frequency(
2033 - rd->tiers[tier].sch,
2034 - (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
2035 - }
2036 - }
2037 - rrddim_foreach_done(rd);
2038 -
2039 - return prev_update_every_s;
2040 -}
src/database/rrdset.h new
+376
@@ -0,0 +1,376 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDSET_H
4 +#define NETDATA_RRDSET_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +
8 +typedef struct rrdset RRDSET;
9 +typedef struct rrdset_acquired RRDSET_ACQUIRED;
10 +typedef struct ml_chart rrd_ml_chart_t;
11 +
12 +#include "rrdlabels.h"
13 +#include "rrd-database-mode.h"
14 +
15 +// --------------------------------------------------------------------------------------------------------------------
16 +
17 +typedef enum __attribute__ ((__packed__)) rrdset_type {
18 + RRDSET_TYPE_LINE = 0,
19 + RRDSET_TYPE_AREA = 1,
20 + RRDSET_TYPE_STACKED = 2,
21 + RRDSET_TYPE_HEATMAP = 3,
22 +} RRDSET_TYPE;
23 +
24 +#define RRDSET_TYPE_LINE_NAME "line"
25 +#define RRDSET_TYPE_AREA_NAME "area"
26 +#define RRDSET_TYPE_STACKED_NAME "stacked"
27 +#define RRDSET_TYPE_HEATMAP_NAME "heatmap"
28 +
29 +RRDSET_TYPE rrdset_type_id(const char *name);
30 +const char *rrdset_type_name(RRDSET_TYPE chart_type);
31 +
32 +// --------------------------------------------------------------------------------------------------------------------
33 +
34 +struct rrdhost;
35 +struct rrdcalc;
36 +struct pluginsd_rrddim;
37 +struct rrdinstance_acquired;
38 +struct rrdcontext_acquired;
39 +struct storage_alignment;
40 +
41 +// --------------------------------------------------------------------------------------------------------------------
42 +
43 +// use this for configuration flags, not for state control
44 +// flags are set/unset in a manner that is not thread safe
45 +// and may lead to missing information.
46 +typedef enum __attribute__ ((__packed__)) rrdset_flags {
47 + RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
48 + RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete
49 + RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases
50 + RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases
51 +
52 + RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming)
53 + RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming)
54 +
55 + RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation
56 + RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
57 + RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous
58 + RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting
59 + RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
60 + RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions
61 +
62 + RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored
63 + RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts.
64 + RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id
65 + RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name
66 +
67 + RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18),
68 +
69 + RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress
70 + RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication
71 + RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress
72 + RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication
73 +
74 + RRDSET_FLAG_BACKFILLED_HIGH_TIERS = (1 << 23), // we have backfilled this chart
75 +
76 + RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 24), // a custom variable has been updated and needs to be exposed to parent
77 +
78 + RRDSET_FLAG_COLLECTION_FINISHED = (1 << 25), // when set, data collection is not available for this chart
79 +
80 + RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 26), // this chart has at least one rrdcal linked
81 +} RRDSET_FLAGS;
82 +
83 +// --------------------------------------------------------------------------------------------------------------------
84 +// flags
85 +
86 +#define rrdset_flag_get(st) atomic_flags_get(&((st)->flags))
87 +#define rrdset_flag_check(st, flag) atomic_flags_check(&((st)->flags), flag)
88 +#define rrdset_flag_set(st, flag) atomic_flags_set(&((st)->flags), flag)
89 +#define rrdset_flag_clear(st, flag) atomic_flags_clear(&((st)->flags), flag)
90 +#define rrdset_flag_set_and_clear(st, set, clear) atomic_flags_set_and_clear(&((st)->flags), set, clear)
91 +
92 +#define rrdset_is_replicating(st) (rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS|RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS) \
93 + && !rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED|RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
94 +
95 +// --------------------------------------------------------------------------------------------------------------------
96 +
97 +struct rrdset {
98 + nd_uuid_t chart_uuid; // the global UUID for this chart
99 +
100 + // ------------------------------------------------------------------------
101 + // chart configuration
102 +
103 + struct {
104 + STRING *type; // the type of {type}.{id}
105 + STRING *id; // the id of {type}.{id}
106 + STRING *name; // the name of {type}.{name}
107 + } parts;
108 +
109 + STRING *id; // the unique ID of the rrdset as {type}.{id}
110 + STRING *name; // the unique name of the rrdset as {type}.{name}
111 + STRING *family; // grouping sets under the same family
112 + STRING *title; // title shown to user
113 + STRING *units; // units of measurement
114 + STRING *context; // the template of this data set
115 + STRING *plugin_name; // the name of the plugin that generated this
116 + STRING *module_name; // the name of the plugin module that generated this
117 +
118 + int32_t priority; // the sorting priority of this chart
119 + int32_t update_every; // data collection frequency
120 +
121 + RRDLABELS *rrdlabels; // chart labels
122 +
123 + uint32_t version; // the metadata version (auto-increment)
124 +
125 + RRDSET_TYPE chart_type; // line, area, stacked
126 +
127 + // ------------------------------------------------------------------------
128 + // operational state members
129 +
130 + RRDSET_FLAGS flags; // flags
131 + RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset
132 +
133 + DICTIONARY *rrddim_root_index; // dimensions index
134 +
135 + rrd_ml_chart_t *ml_chart;
136 +
137 + struct storage_alignment *smg[RRD_STORAGE_TIERS];
138 +
139 + // ------------------------------------------------------------------------
140 + // linking to siblings and parents
141 +
142 + struct rrdhost *rrdhost; // pointer to RRDHOST this chart belongs to
143 +
144 + struct {
145 + struct rrdinstance_acquired *rrdinstance; // the rrdinstance of this chart
146 + struct rrdcontext_acquired *rrdcontext; // the rrdcontext this chart belongs to
147 + bool collected;
148 + } rrdcontexts;
149 +
150 + // ------------------------------------------------------------------------
151 + // data collection members
152 +
153 + SPINLOCK data_collection_lock;
154 +
155 + uint32_t counter; // the number of times we added values to this database
156 + uint32_t counter_done; // the number of times rrdset_done() has been called
157 +
158 + time_t last_accessed_time_s; // the last time this RRDSET has been accessed
159 + usec_t usec_since_last_update; // the time in microseconds since the last collection of data
160 +
161 + struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)
162 + struct timeval last_collected_time; // when did this data set last collected values
163 +
164 + size_t rrdlabels_last_saved_version;
165 +
166 + DICTIONARY *functions_view; // collector functions this rrdset supports, can be NULL
167 +
168 + // ------------------------------------------------------------------------
169 + // data collection - streaming to parents, temp variables
170 +
171 + struct {
172 + struct {
173 + uint32_t sent_version;
174 + uint32_t chart_slot;
175 + uint32_t dim_last_slot_used;
176 +#ifdef REPLICATION_TRACKING
177 + REPLAY_WHO who;
178 +#endif
179 + time_t resync_time_s; // the timestamp up to which we should resync clock upstream
180 + } snd;
181 +
182 + struct {
183 +#ifdef REPLICATION_TRACKING
184 + REPLAY_WHO who;
185 +#endif
186 + } rcv;
187 + } stream;
188 +
189 + // ------------------------------------------------------------------------
190 + // db mode SAVE, MAP specifics
191 + // TODO - they should be managed by storage engine
192 + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
193 +
194 + struct {
195 + int32_t entries; // total number of entries in the data set
196 + int32_t current_entry; // the entry that is currently being updated
197 + // it goes around in a round-robin fashion
198 + } db;
199 +
200 + // ------------------------------------------------------------------------
201 + // exporting to 3rd party time-series members
202 + // TODO - they should be managed by exporting engine
203 + // (RRDSET_EXPORTING_STATE ptr to an undefined structure, and a call to clean this up during destruction)
204 +
205 + RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances
206 +
207 + // ------------------------------------------------------------------------
208 + // health monitoring members
209 + // TODO - they should be managed by health
210 + // (RRDSET_HEALTH_STATE ptr to an undefined structure, and a call to clean this up during destruction)
211 +
212 + NETDATA_DOUBLE green; // green threshold for this chart
213 + NETDATA_DOUBLE red; // red threshold for this chart
214 +
215 + DICTIONARY *rrdvars; // RRDVAR index for this chart
216 +
217 + struct {
218 + RW_SPINLOCK spinlock; // protection for RRDCALC *base
219 + struct rrdcalc *base; // double linked list of RRDCALC related to this RRDSET
220 + } alerts;
221 +
222 + struct {
223 + SPINLOCK spinlock; // used only for cleanup
224 + pid_t collector_tid;
225 + bool dims_with_slots;
226 + bool set;
227 + uint32_t pos;
228 + int32_t last_slot;
229 + uint32_t size;
230 + struct pluginsd_rrddim *prd_array;
231 + } pluginsd;
232 +
233 +#ifdef NETDATA_LOG_REPLICATION_REQUESTS
234 + struct {
235 + bool log_next_data_collection;
236 + bool start_streaming;
237 + time_t after;
238 + time_t before;
239 + } replay;
240 +#endif // NETDATA_LOG_REPLICATION_REQUESTS
241 +};
242 +
243 +// --------------------------------------------------------------------------------------------------------------------
244 +
245 +#define rrdset_plugin_name(st) string2str((st)->plugin_name)
246 +#define rrdset_module_name(st) string2str((st)->module_name)
247 +#define rrdset_units(st) string2str((st)->units)
248 +#define rrdset_parts_type(st) string2str((st)->parts.type)
249 +#define rrdset_family(st) string2str((st)->family)
250 +#define rrdset_title(st) string2str((st)->title)
251 +#define rrdset_context(st) string2str((st)->context)
252 +#define rrdset_name(st) string2str((st)->name)
253 +#define rrdset_id(st) string2str((st)->id)
254 +
255 +// --------------------------------------------------------------------------------------------------------------------
256 +
257 +static inline uint32_t rrdset_metadata_version(RRDSET *st) {
258 + return __atomic_load_n(&st->version, __ATOMIC_RELAXED);
259 +}
260 +
261 +static inline uint32_t rrdset_metadata_upstream_version(RRDSET *st) {
262 + return __atomic_load_n(&st->stream.snd.sent_version, __ATOMIC_RELAXED);
263 +}
264 +
265 +static inline void rrdset_metadata_exposed_upstream(RRDSET *st, uint32_t version) {
266 + __atomic_store_n(&st->stream.snd.sent_version, version, __ATOMIC_RELAXED);
267 +}
268 +
269 +static inline bool rrdset_check_upstream_exposed(RRDSET *st) {
270 + return rrdset_metadata_version(st) == rrdset_metadata_upstream_version(st);
271 +}
272 +
273 +// --------------------------------------------------------------------------------------------------------------------
274 +// these loop macros make sure the linked list is accessed with the right lock
275 +
276 +#define rrdset_foreach_read(st, host) \
277 + dfe_start_read((host)->rrdset_root_index, st)
278 +
279 +#define rrdset_foreach_write(st, host) \
280 + dfe_start_write((host)->rrdset_root_index, st)
281 +
282 +#define rrdset_foreach_reentrant(st, host) \
283 + dfe_start_reentrant((host)->rrdset_root_index, st)
284 +
285 +#define rrdset_foreach_done(st) \
286 + dfe_done(st)
287 +
288 +#define rrdset_number_of_dimensions(st) \
289 + dictionary_entries((st)->rrddim_root_index)
290 +
291 +// --------------------------------------------------------------------------------------------------------------------
292 +
293 +void rrdset_metadata_updated(RRDSET *st);
294 +
295 +// --------------------------------------------------------------------------------------------------------------------
296 +
297 +#ifdef NETDATA_INTERNAL_CHECKS
298 +#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
299 + netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0)
300 +#else
301 +#define rrdset_debug(st, fmt, args...) debug_dummy()
302 +#endif
303 +
304 +// --------------------------------------------------------------------------------------------------------------------
305 +
306 +#include "rrdset-index-name.h"
307 +#include "rrdset-index-id.h"
308 +#include "rrdset-slots.h"
309 +#include "rrdset-collection.h"
310 +
311 +void rrdset_update_heterogeneous_flag(RRDSET *st);
312 +
313 +time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);
314 +
315 +RRDSET *rrdset_find(RRDHOST *host, const char *id);
316 +
317 +RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);
318 +RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);
319 +void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
320 +
321 +#define rrdset_find_localhost(id) rrdset_find(localhost, id)
322 +/* This will not return charts that are archived */
323 +static inline RRDSET *rrdset_find_active_localhost(const char *id)
324 +{
325 + RRDSET *st = rrdset_find_localhost(id);
326 + return st;
327 +}
328 +
329 +RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
330 +#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
331 +/* This will not return charts that are archived */
332 +static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
333 +{
334 + RRDSET *st = rrdset_find_bytype_localhost(type, id);
335 + return st;
336 +}
337 +
338 +RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
339 +#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)
340 +/* This will not return charts that are archived */
341 +static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
342 +{
343 + RRDSET *st = rrdset_find_byname_localhost(name);
344 + return st;
345 +}
346 +
347 +void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds);
348 +void rrdset_next_usec(RRDSET *st, usec_t microseconds);
349 +void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);
350 +#define rrdset_next(st) rrdset_next_usec(st, 0ULL)
351 +
352 +void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next);
353 +void rrdset_done(RRDSET *st);
354 +
355 +void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st);
356 +void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st);
357 +
358 +// checks if the RRDSET should be offered to viewers
359 +#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
360 +#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st))
361 +
362 +time_t rrdset_first_entry_s(RRDSET *st);
363 +time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier);
364 +time_t rrdset_last_entry_s(RRDSET *st);
365 +time_t rrdset_last_entry_s_of_tier(RRDSET *st, size_t tier);
366 +
367 +void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_time_s, time_t *last_time_s, time_t now_s, size_t tier);
368 +
369 +void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels);
370 +
371 +void rrdset_free(RRDSET *st);
372 +
373 +void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st);
374 +void rrdset_pluginsd_receive_unslot(RRDSET *st);
375 +
376 +#endif //NETDATA_RRDSET_H
src/database/storage-engine.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "storage_engine.h"
3 +#include "storage-engine.h"
4 #include "ram/rrddim_mem.h"
5 #ifdef ENABLE_DBENGINE
6 #include "engine/rrdengineapi.h"
src/database/storage-engine.h new
+374
@@ -0,0 +1,374 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STORAGEENGINEAPI_H
4 +#define NETDATA_STORAGEENGINEAPI_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "rrd-database-mode.h"
8 +
9 +typedef struct rrddim RRDDIM;
10 +
11 +typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
12 +
13 +typedef enum __attribute__ ((__packed__)) storage_priority {
14 + STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
15 + STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
16 +
17 + // query priorities
18 + STORAGE_PRIORITY_HIGH,
19 + STORAGE_PRIORITY_NORMAL,
20 + STORAGE_PRIORITY_LOW,
21 + STORAGE_PRIORITY_BEST_EFFORT,
22 +
23 + // synchronous query, not to be dispatched to workers or queued
24 + STORAGE_PRIORITY_SYNCHRONOUS,
25 + STORAGE_PRIORITY_SYNCHRONOUS_FIRST,
26 +
27 + STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
28 +} STORAGE_PRIORITY;
29 +
30 +typedef enum __attribute__ ((__packed__)) {
31 + STORAGE_ENGINE_BACKEND_RRDDIM = 1,
32 + STORAGE_ENGINE_BACKEND_DBENGINE = 2,
33 +} STORAGE_ENGINE_BACKEND;
34 +
35 +#define is_valid_backend(backend) ((backend) >= STORAGE_ENGINE_BACKEND_RRDDIM && (backend) <= STORAGE_ENGINE_BACKEND_DBENGINE)
36 +
37 +// iterator state for RRD dimension data queries
38 +struct storage_engine_query_handle {
39 + time_t start_time_s;
40 + time_t end_time_s;
41 + STORAGE_PRIORITY priority;
42 + STORAGE_ENGINE_BACKEND seb;
43 + STORAGE_QUERY_HANDLE *handle;
44 +};
45 +
46 +// non-existing structs instead of voids
47 +// to enable type checking at compile time
48 +typedef struct storage_instance STORAGE_INSTANCE;
49 +typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
50 +typedef struct storage_alignment STORAGE_METRICS_GROUP;
51 +
52 +// ----------------------------------------------------------------------------
53 +// engine-specific iterator state for dimension data collection
54 +typedef struct storage_collect_handle {
55 + STORAGE_ENGINE_BACKEND seb;
56 +} STORAGE_COLLECT_HANDLE;
57 +
58 +// ------------------------------------------------------------------------
59 +// function pointers for all APIs provided by a storage engine
60 +typedef struct storage_engine_api {
61 + // metric management
62 + STORAGE_METRIC_HANDLE *(*metric_get_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id);
63 + STORAGE_METRIC_HANDLE *(*metric_get_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
64 + STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *si);
65 + void (*metric_release)(STORAGE_METRIC_HANDLE *);
66 + STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
67 + bool (*metric_retention_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id, time_t *first_entry_s, time_t *last_entry_s);
68 + bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
69 +} STORAGE_ENGINE_API;
70 +
71 +typedef struct storage {
72 + STORAGE_ENGINE_BACKEND seb;
73 + RRD_MEMORY_MODE id;
74 + const char* name;
75 + STORAGE_ENGINE_API api;
76 +} STORAGE_ENGINE;
77 +
78 +STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
79 +STORAGE_ENGINE* storage_engine_find(const char* name);
80 +
81 +// Iterator over existing engines
82 +STORAGE_ENGINE* storage_engine_foreach_init();
83 +STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);
84 +
85 +// ----------------------------------------------------------------------------
86 +// Storage tier data for every dimension
87 +
88 +struct rrddim_tier {
89 + STORAGE_POINT virtual_point;
90 + STORAGE_ENGINE_BACKEND seb;
91 + SPINLOCK spinlock;
92 + uint32_t tier_grouping;
93 + time_t next_point_end_time_s;
94 + STORAGE_METRIC_HANDLE *smh; // the metric handle inside the database
95 + STORAGE_COLLECT_HANDLE *sch; // the data collection handle
96 +};
97 +
98 +// ------------------------------------------------------------------------
99 +
100 +#include "daemon/config/netdata-conf-db.h"
101 +
102 +// ------------------------------------------------------------------------
103 +// DATA COLLECTION STORAGE OPS
104 +
105 +STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
106 +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
107 +static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, nd_uuid_t *uuid) {
108 + internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
109 +
110 +#ifdef ENABLE_DBENGINE
111 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
112 + return rrdeng_metrics_group_get(si, uuid);
113 +#endif
114 + return rrddim_metrics_group_get(si, uuid);
115 +}
116 +
117 +void rrdeng_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
118 +void rrddim_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
119 +static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg) {
120 + internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
121 +
122 +#ifdef ENABLE_DBENGINE
123 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
124 + rrdeng_metrics_group_release(si, smg);
125 + else
126 +#endif
127 + rrddim_metrics_group_release(si, smg);
128 +}
129 +
130 +STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
131 +STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
132 +static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {
133 + internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
134 +
135 +#ifdef ENABLE_DBENGINE
136 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
137 + return rrdeng_store_metric_init(smh, update_every, smg);
138 +#endif
139 + return rrddim_collect_init(smh, update_every, smg);
140 +}
141 +
142 +void rrdeng_store_metric_next(
143 + STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
144 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
145 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
146 +
147 +void rrddim_collect_store_metric(
148 + STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
149 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
150 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
151 +
152 +static inline void storage_engine_store_metric(
153 + STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
154 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
155 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {
156 + internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
157 +
158 +#ifdef ENABLE_DBENGINE
159 + if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
160 + return rrdeng_store_metric_next(sch, point_in_time_ut,
161 + n, min_value, max_value,
162 + count, anomaly_count, flags);
163 +#endif
164 + return rrddim_collect_store_metric(sch, point_in_time_ut,
165 + n, min_value, max_value,
166 + count, anomaly_count, flags);
167 +}
168 +
169 +uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *si);
170 +static inline uint64_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
171 +#ifdef ENABLE_DBENGINE
172 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
173 + return rrdeng_disk_space_max(si);
174 +#endif
175 +
176 + return 0;
177 +}
178 +
179 +uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *si);
180 +static inline uint64_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
181 +#ifdef ENABLE_DBENGINE
182 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
183 + return rrdeng_disk_space_used(si);
184 +#endif
185 +
186 + // TODO - calculate the total host disk space for memory mode save and map
187 + return 0;
188 +}
189 +
190 +uint64_t rrdeng_metrics(STORAGE_INSTANCE *si);
191 +static inline uint64_t storage_engine_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
192 +#ifdef ENABLE_DBENGINE
193 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
194 + return rrdeng_metrics(si);
195 +#endif
196 +
197 + // TODO - calculate the total host disk space for memory mode save and map
198 + return 0;
199 +}
200 +
201 +uint64_t rrdeng_samples(STORAGE_INSTANCE *si);
202 +static inline uint64_t storage_engine_samples(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
203 +#ifdef ENABLE_DBENGINE
204 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
205 + return rrdeng_samples(si);
206 +#endif
207 + return 0;
208 +}
209 +
210 +
211 +time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *si);
212 +static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
213 +#ifdef ENABLE_DBENGINE
214 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
215 + return rrdeng_global_first_time_s(si);
216 +#endif
217 +
218 + return now_realtime_sec() - (time_t)(default_rrd_history_entries * nd_profile.update_every);
219 +}
220 +
221 +size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *si);
222 +static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
223 +#ifdef ENABLE_DBENGINE
224 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
225 + return rrdeng_currently_collected_metrics(si);
226 +#endif
227 +
228 + // TODO - calculate the total host disk space for memory mode save and map
229 + return 0;
230 +}
231 +
232 +void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *sch);
233 +void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *sch);
234 +static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *sch) {
235 + if(unlikely(!sch))
236 + return;
237 +
238 + internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
239 +
240 +#ifdef ENABLE_DBENGINE
241 + if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
242 + rrdeng_store_metric_flush_current_page(sch);
243 + else
244 +#endif
245 + rrddim_store_metric_flush(sch);
246 +}
247 +
248 +int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *sch);
249 +int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *sch);
250 +// a finalization function to run after collection is over
251 +// returns 1 if it's safe to delete the dimension
252 +static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *sch) {
253 + internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
254 +
255 +#ifdef ENABLE_DBENGINE
256 + if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
257 + return rrdeng_store_metric_finalize(sch);
258 +#endif
259 +
260 + return rrddim_collect_finalize(sch);
261 +}
262 +
263 +void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
264 +void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
265 +static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every) {
266 + internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
267 +
268 +#ifdef ENABLE_DBENGINE
269 + if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
270 + rrdeng_store_metric_change_collection_frequency(sch, update_every);
271 + else
272 +#endif
273 + rrddim_store_metric_change_collection_frequency(sch, update_every);
274 +}
275 +
276 +// ----------------------------------------------------------------------------
277 +// STORAGE ENGINE QUERY OPS
278 +
279 +time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh);
280 +time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *smh);
281 +static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {
282 + internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
283 +
284 +#ifdef ENABLE_DBENGINE
285 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
286 + return rrdeng_metric_oldest_time(smh);
287 +#endif
288 + return rrddim_query_oldest_time_s(smh);
289 +}
290 +
291 +time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh);
292 +time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *smh);
293 +static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {
294 + internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
295 +
296 +#ifdef ENABLE_DBENGINE
297 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
298 + return rrdeng_metric_latest_time(smh);
299 +#endif
300 + return rrddim_query_latest_time_s(smh);
301 +}
302 +
303 +void rrdeng_load_metric_init(
304 + STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
305 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
306 +
307 +void rrddim_query_init(
308 + STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
309 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
310 +
311 +static inline void storage_engine_query_init(
312 + STORAGE_ENGINE_BACKEND seb __maybe_unused,
313 + STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
314 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {
315 + internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
316 +
317 +#ifdef ENABLE_DBENGINE
318 + if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
319 + rrdeng_load_metric_init(smh, seqh, start_time_s, end_time_s, priority);
320 + else
321 +#endif
322 + rrddim_query_init(smh, seqh, start_time_s, end_time_s, priority);
323 +}
324 +
325 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh);
326 +STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh);
327 +static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {
328 + internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
329 +
330 +#ifdef ENABLE_DBENGINE
331 + if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
332 + return rrdeng_load_metric_next(seqh);
333 +#endif
334 + return rrddim_query_next_metric(seqh);
335 +}
336 +
337 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh);
338 +int rrddim_query_is_finished(struct storage_engine_query_handle *seqh);
339 +static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {
340 + internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
341 +
342 +#ifdef ENABLE_DBENGINE
343 + if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
344 + return rrdeng_load_metric_is_finished(seqh);
345 +#endif
346 + return rrddim_query_is_finished(seqh);
347 +}
348 +
349 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh);
350 +void rrddim_query_finalize(struct storage_engine_query_handle *seqh);
351 +static inline void storage_engine_query_finalize(struct storage_engine_query_handle *seqh) {
352 + internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
353 +
354 +#ifdef ENABLE_DBENGINE
355 + if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
356 + rrdeng_load_metric_finalize(seqh);
357 + else
358 +#endif
359 + rrddim_query_finalize(seqh);
360 +}
361 +
362 +time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh);
363 +time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *seqh);
364 +static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *seqh) {
365 + internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
366 +
367 +#ifdef ENABLE_DBENGINE
368 + if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
369 + return rrdeng_load_align_to_optimal_before(seqh);
370 +#endif
371 + return rrddim_query_align_to_optimal_before(seqh);
372 +}
373 +
374 +#endif
src/database/storage_engine.h deleted
-12
@@ -1,12 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_STORAGEENGINEAPI_H
4 -#define NETDATA_STORAGEENGINEAPI_H
5 -
6 -#include "rrd.h"
7 -
8 -// Iterator over existing engines
9 -STORAGE_ENGINE* storage_engine_foreach_init();
10 -STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);
11 -
12 -#endif
src/plugins.d/pluginsd_parser.c
+1
@@ -2,6 +2,7 @@
2
3 #include "pluginsd_internals.h"
4 #include "streaming/stream-replication-receiver.h"
5 +#include "database/rrddim-collection.h"
6
7 static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *parser) {
8 int idx = 1;
src/plugins.d/pluginsd_replication.c
+1
@@ -5,6 +5,7 @@
5 #include "streaming/stream-replication-receiver.h"
6 #include "streaming/stream-waiting-list.h"
7 #include "web/api/queries/backfill.h"
8 +#include "database/rrddim-collection.h"
9
10 static bool backfill_callback(size_t successful_dims __maybe_unused, size_t failed_dims __maybe_unused, struct backfill_request_data *brd) {
11 if(!object_state_acquire(&brd->host->state_id, brd->host_state_id)) {
src/streaming/stream-parents.h
+1 -1
@@ -17,7 +17,7 @@ typedef struct rrdhost_stream_parents {
17 } RRDHOST_STREAM_PARENTS;
18
19 #include "stream-handshake.h"
20 -#include "rrdhost-status.h"
20 +#include "database/rrdhost-status.h"
21
22 void rrdhost_stream_parent_ssl_init(struct sender_state *s);
23
src/streaming/stream.h
+1 -1
@@ -38,7 +38,7 @@ void stream_receiver_free(struct receiver_state *rpt);
38 bool stream_receiver_signal_to_stop_and_wait(struct rrdhost *host, STREAM_HANDSHAKE reason);
39 char *stream_receiver_program_version_strdupz(struct rrdhost *host);
40
41 -#include "rrdhost-status.h"
41 +#include "database/rrdhost-status.h"
42 #include "protocol/commands.h"
43 #include "stream-path.h"
44 #include "stream-control.h"
src/web/api/formatters/rrd2json.c
+1 -1
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "web/api/web_api_v1.h"
4 -#include "database/storage_engine.h"
4 +#include "database/storage-engine.h"
5
6 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
7 {
src/web/api/queries/query.c
-71
@@ -1958,77 +1958,6 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1958 qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
1959 }
1960
1961 -// ----------------------------------------------------------------------------
1962 -// fill the gap of a tier
1963 -
1964 -void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
1965 -
1966 -bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {
1967 - if(unlikely(tier >= nd_profile.storage_tiers)) return false;
1968 -#ifdef ENABLE_DBENGINE
1969 - if(default_backfill == RRD_BACKFILL_NONE) return false;
1970 -#else
1971 - return false;
1972 -#endif
1973 -
1974 - struct rrddim_tier *t = &rd->tiers[tier];
1975 - if(unlikely(!t)) return false;
1976 -
1977 - time_t latest_time_s = storage_engine_latest_time_s(t->seb, t->smh);
1978 - time_t granularity = (time_t)t->tier_grouping * (time_t)rd->rrdset->update_every;
1979 - time_t time_diff = now_s - latest_time_s;
1980 -
1981 - // if the user wants only NEW backfilling, and we don't have any data
1982 -#ifdef ENABLE_DBENGINE
1983 - if(default_backfill == RRD_BACKFILL_NEW && latest_time_s <= 0) return false;
1984 -#else
1985 - return;
1986 -#endif
1987 -
1988 - // there is really nothing we can do
1989 - if(now_s <= latest_time_s || time_diff < granularity) return false;
1990 -
1991 - stream_control_backfill_query_started();
1992 -
1993 - // for each lower tier
1994 - struct storage_engine_query_handle seqh;
1995 - for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){
1996 - time_t smaller_tier_first_time = storage_engine_oldest_time_s(rd->tiers[read_tier].seb, rd->tiers[read_tier].smh);
1997 - time_t smaller_tier_last_time = storage_engine_latest_time_s(rd->tiers[read_tier].seb, rd->tiers[read_tier].smh);
1998 - if(smaller_tier_last_time <= latest_time_s) continue; // it is as bad as we are
1999 -
2000 - long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s;
2001 - long before_wanted = smaller_tier_last_time;
2002 -
2003 - struct rrddim_tier *tmp = &rd->tiers[read_tier];
2004 - storage_engine_query_init(tmp->seb, tmp->smh, &seqh, after_wanted, before_wanted, STORAGE_PRIORITY_SYNCHRONOUS_FIRST);
2005 -
2006 - size_t points_read = 0;
2007 -
2008 - while(!storage_engine_query_is_finished(&seqh)) {
2009 -
2010 - STORAGE_POINT sp = storage_engine_query_next_metric(&seqh);
2011 - points_read++;
2012 -
2013 - if(sp.end_time_s > latest_time_s) {
2014 - latest_time_s = sp.end_time_s;
2015 - store_metric_at_tier(rd, tier, t, sp, sp.end_time_s * USEC_PER_SEC);
2016 - }
2017 - }
2018 -
2019 - storage_engine_query_finalize(&seqh);
2020 - store_metric_collection_completed();
2021 - pulse_queries_backfill_query_completed(points_read);
2022 -
2023 - //internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d",
2024 - // rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr);
2025 - }
2026 -
2027 - stream_control_backfill_query_finished();
2028 -
2029 - return true;
2030 -}
2031 -
1961 // ----------------------------------------------------------------------------
1962 // fill RRDR for the whole chart
1963