| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #ifndef NETDATA_RRDSET_H |
| 4 | #define NETDATA_RRDSET_H |
| 5 | |
| 6 | #include "libnetdata/libnetdata.h" |
| 7 | |
| 8 | typedef struct rrdset RRDSET; |
| 9 | typedef struct rrdset_acquired RRDSET_ACQUIRED; |
| 10 | typedef struct ml_chart rrd_ml_chart_t; |
| 11 | |
| 12 | #include "rrdset-type.h" |
| 13 | #include "rrdlabels.h" |
| 14 | #include "rrd-database-mode.h" |
| 15 | |
| 16 | // -------------------------------------------------------------------------------------------------------------------- |
| 17 | |
| 18 | struct rrdhost; |
| 19 | struct rrdcalc; |
| 20 | struct pluginsd_rrddim; |
| 21 | struct pluginsd_rrddim_array; // Reference-counted array for pluginsd dimension caching |
| 22 | struct rrdinstance_acquired; |
| 23 | struct rrdcontext_acquired; |
| 24 | struct storage_alignment; |
| 25 | |
| 26 | // -------------------------------------------------------------------------------------------------------------------- |
| 27 | |
| 28 | // use this for configuration flags, not for state control |
| 29 | // flags are set/unset in a manner that is not thread safe |
| 30 | // and may lead to missing information. |
| 31 | typedef enum __attribute__ ((__packed__)) rrdset_flags { |
| 32 | RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart |
| 33 | RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete |
| 34 | RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases |
| 35 | RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases |
| 36 | |
| 37 | RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming) |
| 38 | RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming) |
| 39 | |
| 40 | RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation |
| 41 | RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) |
| 42 | RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous |
| 43 | RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting |
| 44 | RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) |
| 45 | RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions |
| 46 | |
| 47 | RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored |
| 48 | RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts. |
| 49 | RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id |
| 50 | RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name |
| 51 | |
| 52 | RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18), |
| 53 | |
| 54 | RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress |
| 55 | RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication |
| 56 | RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress |
| 57 | RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication |
| 58 | |
| 59 | RRDSET_FLAG_BACKFILLED_HIGH_TIERS = (1 << 23), // we have backfilled this chart |
| 60 | |
| 61 | RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 24), // a custom variable has been updated and needs to be exposed to parent |
| 62 | |
| 63 | RRDSET_FLAG_COLLECTION_FINISHED = (1 << 25), // when set, data collection is not available for this chart |
| 64 | |
| 65 | RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 26), // this chart has at least one rrdcal linked |
| 66 | |
| 67 | RRDSET_FLAG_PENDING_LABEL_RECHECK = (1 << 27), // chart labels changed since the last health |
| 68 | // prototype evaluation; the health thread must |
| 69 | // detach + reattach alerts for this chart. |
| 70 | } RRDSET_FLAGS; |
| 71 | |
| 72 | // -------------------------------------------------------------------------------------------------------------------- |
| 73 | // flags |
| 74 | |
| 75 | #define rrdset_flag_get(st) atomic_flags_get(&((st)->flags)) |
| 76 | #define rrdset_flag_check(st, flag) atomic_flags_check(&((st)->flags), flag) |
| 77 | #define rrdset_flag_set(st, flag) atomic_flags_set(&((st)->flags), flag) |
| 78 | #define rrdset_flag_clear(st, flag) atomic_flags_clear(&((st)->flags), flag) |
| 79 | #define rrdset_flag_set_and_clear(st, set, clear) atomic_flags_set_and_clear(&((st)->flags), set, clear) |
| 80 | |
| 81 | #define rrdset_is_replicating(st) (rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS|RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS) \ |
| 82 | && !rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED|RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED)) |
| 83 | |
| 84 | #define rrdset_is_discoverable(st) (rrdset_is_replicating(st) || !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)) |
| 85 | |
| 86 | // -------------------------------------------------------------------------------------------------------------------- |
| 87 | |
| 88 | struct rrdset { |
| 89 | nd_uuid_t chart_uuid; // the global UUID for this chart |
| 90 | |
| 91 | // ------------------------------------------------------------------------ |
| 92 | // chart configuration |
| 93 | |
| 94 | struct { |
| 95 | STRING *type; // the type of {type}.{id} |
| 96 | STRING *id; // the id of {type}.{id} |
| 97 | STRING *name; // the name of {type}.{name} |
| 98 | } parts; |
| 99 | |
| 100 | STRING *id; // the unique ID of the rrdset as {type}.{id} |
| 101 | STRING *name; // the unique name of the rrdset as {type}.{name} |
| 102 | STRING *family; // grouping sets under the same family |
| 103 | STRING *title; // title shown to user |
| 104 | STRING *units; // units of measurement |
| 105 | STRING *context; // the template of this data set |
| 106 | STRING *plugin_name; // the name of the plugin that generated this |
| 107 | STRING *module_name; // the name of the plugin module that generated this |
| 108 | |
| 109 | int32_t priority; // the sorting priority of this chart |
| 110 | int32_t update_every; // data collection frequency |
| 111 | |
| 112 | RRDLABELS *rrdlabels; // chart labels |
| 113 | |
| 114 | uint32_t version; // the metadata version (auto-increment) |
| 115 | |
| 116 | RRDSET_TYPE chart_type; // line, area, stacked |
| 117 | |
| 118 | // ------------------------------------------------------------------------ |
| 119 | // operational state members |
| 120 | |
| 121 | RRD_DB_MODE rrd_memory_mode; // the db mode of this rrdset |
| 122 | uint16_t collection_modulo; // tier1/2 spread over time |
| 123 | RRDSET_FLAGS flags; // flags |
| 124 | |
| 125 | pid_t collector_tid; |
| 126 | |
| 127 | DICTIONARY *rrddim_root_index; // dimensions index |
| 128 | |
| 129 | rrd_ml_chart_t *ml_chart; |
| 130 | |
| 131 | struct storage_alignment *smg[RRD_STORAGE_TIERS]; |
| 132 | |
| 133 | // ------------------------------------------------------------------------ |
| 134 | // linking to siblings and parents |
| 135 | |
| 136 | struct rrdhost *rrdhost; // pointer to RRDHOST this chart belongs to |
| 137 | |
| 138 | struct { |
| 139 | struct rrdinstance_acquired *rrdinstance; // the rrdinstance of this chart |
| 140 | struct rrdcontext_acquired *rrdcontext; // the rrdcontext this chart belongs to |
| 141 | bool collected; |
| 142 | } rrdcontexts; |
| 143 | |
| 144 | // ------------------------------------------------------------------------ |
| 145 | // data collection members |
| 146 | |
| 147 | SPINLOCK data_collection_lock; |
| 148 | |
| 149 | uint32_t counter; // the number of times we added values to this database |
| 150 | uint32_t counter_done; // the number of times rrdset_done() has been called |
| 151 | |
| 152 | time_t last_accessed_time_s; // the last time this RRDSET has been accessed |
| 153 | usec_t usec_since_last_update; // the time in microseconds since the last collection of data |
| 154 | |
| 155 | struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) |
| 156 | struct timeval last_collected_time; // when did this data set last collected values |
| 157 | |
| 158 | size_t rrdlabels_last_saved_version; |
| 159 | |
| 160 | DICTIONARY *functions_view; // collector functions this rrdset supports, can be NULL |
| 161 | |
| 162 | // ------------------------------------------------------------------------ |
| 163 | // data collection - streaming to parents, temp variables |
| 164 | |
| 165 | struct { |
| 166 | struct { |
| 167 | uint32_t sent_version; |
| 168 | uint32_t chart_slot; |
| 169 | uint32_t dim_last_slot_used; |
| 170 | #ifdef REPLICATION_TRACKING |
| 171 | REPLAY_WHO who; |
| 172 | #endif |
| 173 | time_t resync_time_s; // the timestamp up to which we should resync clock upstream |
| 174 | } snd; |
| 175 | |
| 176 | struct { |
| 177 | #ifdef REPLICATION_TRACKING |
| 178 | REPLAY_WHO who; |
| 179 | #else |
| 180 | uint8_t unused; |
| 181 | #endif |
| 182 | } rcv; |
| 183 | } stream; |
| 184 | |
| 185 | // ------------------------------------------------------------------------ |
| 186 | // db mode SAVE, MAP specifics |
| 187 | // TODO - they should be managed by storage engine |
| 188 | // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) |
| 189 | |
| 190 | struct { |
| 191 | int32_t entries; // total number of entries in the data set |
| 192 | int32_t current_entry; // the entry that is currently being updated |
| 193 | // it goes around in a round-robin fashion |
| 194 | } db; |
| 195 | |
| 196 | // ------------------------------------------------------------------------ |
| 197 | // exporting to 3rd party time-series members |
| 198 | // TODO - they should be managed by exporting engine |
| 199 | // (RRDSET_EXPORTING_STATE ptr to an undefined structure, and a call to clean this up during destruction) |
| 200 | |
| 201 | RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances |
| 202 | |
| 203 | // ------------------------------------------------------------------------ |
| 204 | // health monitoring members |
| 205 | // TODO - they should be managed by health |
| 206 | // (RRDSET_HEALTH_STATE ptr to an undefined structure, and a call to clean this up during destruction) |
| 207 | |
| 208 | NETDATA_DOUBLE green; // green threshold for this chart |
| 209 | NETDATA_DOUBLE red; // red threshold for this chart |
| 210 | |
| 211 | DICTIONARY *rrdvars; // RRDVAR index for this chart |
| 212 | |
| 213 | struct { |
| 214 | RW_SPINLOCK spinlock; // protection for RRDCALC *base |
| 215 | struct rrdcalc *base; // double linked list of RRDCALC related to this RRDSET |
| 216 | } alerts; |
| 217 | |
| 218 | struct { |
| 219 | SPINLOCK spinlock; // coordinates PRD_ARRAY grow/transfer and unslot/cleanup serialization |
| 220 | pid_t collector_tid; |
| 221 | bool dims_with_slots; |
| 222 | bool set; |
| 223 | uint32_t pos; |
| 224 | int32_t last_slot; |
| 225 | struct pluginsd_rrddim_array *prd_array; // Reference-counted array (use prd_array_* functions) |
| 226 | } pluginsd; |
| 227 | |
| 228 | #ifdef NETDATA_LOG_REPLICATION_REQUESTS |
| 229 | struct { |
| 230 | bool log_next_data_collection; |
| 231 | bool start_streaming; |
| 232 | time_t after; |
| 233 | time_t before; |
| 234 | } replay; |
| 235 | #endif // NETDATA_LOG_REPLICATION_REQUESTS |
| 236 | |
| 237 | // Replication stuck detection - outside debug flag for production safety |
| 238 | uint8_t replication_empty_response_count; // track consecutive empty responses |
| 239 | |
| 240 | SPINLOCK destroy_lock; |
| 241 | }; |
| 242 | |
| 243 | // -------------------------------------------------------------------------------------------------------------------- |
| 244 | |
| 245 | #define rrdset_plugin_name(st) string2str((st)->plugin_name) |
| 246 | #define rrdset_module_name(st) string2str((st)->module_name) |
| 247 | #define rrdset_units(st) string2str((st)->units) |
| 248 | #define rrdset_parts_type(st) string2str((st)->parts.type) |
| 249 | #define rrdset_family(st) string2str((st)->family) |
| 250 | #define rrdset_title(st) string2str((st)->title) |
| 251 | #define rrdset_context(st) string2str((st)->context) |
| 252 | #define rrdset_name(st) string2str((st)->name) |
| 253 | #define rrdset_id(st) string2str((st)->id) |
| 254 | |
| 255 | // -------------------------------------------------------------------------------------------------------------------- |
| 256 | |
| 257 | static inline uint32_t rrdset_metadata_version(RRDSET *st) { |
| 258 | return __atomic_load_n(&st->version, __ATOMIC_RELAXED); |
| 259 | } |
| 260 | |
| 261 | static inline uint32_t rrdset_metadata_upstream_version(RRDSET *st) { |
| 262 | return __atomic_load_n(&st->stream.snd.sent_version, __ATOMIC_RELAXED); |
| 263 | } |
| 264 | |
| 265 | static inline void rrdset_metadata_exposed_upstream(RRDSET *st, uint32_t version) { |
| 266 | __atomic_store_n(&st->stream.snd.sent_version, version, __ATOMIC_RELAXED); |
| 267 | } |
| 268 | |
| 269 | static inline bool rrdset_check_upstream_exposed(RRDSET *st) { |
| 270 | return rrdset_metadata_version(st) == rrdset_metadata_upstream_version(st); |
| 271 | } |
| 272 | |
| 273 | // -------------------------------------------------------------------------------------------------------------------- |
| 274 | // these loop macros make sure the linked list is accessed with the right lock |
| 275 | |
| 276 | #define rrdset_foreach_read(st, host) \ |
| 277 | dfe_start_read((host)->rrdset_root_index, st) |
| 278 | |
| 279 | #define rrdset_foreach_write(st, host) \ |
| 280 | dfe_start_write((host)->rrdset_root_index, st) |
| 281 | |
| 282 | #define rrdset_foreach_reentrant(st, host) \ |
| 283 | dfe_start_reentrant((host)->rrdset_root_index, st) |
| 284 | |
| 285 | #define rrdset_foreach_done(st) \ |
| 286 | dfe_done(st) |
| 287 | |
| 288 | #define rrdset_number_of_dimensions(st) \ |
| 289 | dictionary_entries((st)->rrddim_root_index) |
| 290 | |
| 291 | // -------------------------------------------------------------------------------------------------------------------- |
| 292 | |
| 293 | void rrdset_metadata_updated(RRDSET *st); |
| 294 | |
| 295 | // -------------------------------------------------------------------------------------------------------------------- |
| 296 | |
| 297 | #ifdef NETDATA_INTERNAL_CHECKS |
| 298 | #define rrdset_debug(st, fmt, args...) do { \ |
| 299 | if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \ |
| 300 | netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); \ |
| 301 | } while(0) |
| 302 | #else |
| 303 | #define rrdset_debug(st, fmt, args...) debug_dummy() |
| 304 | #endif |
| 305 | |
| 306 | // -------------------------------------------------------------------------------------------------------------------- |
| 307 | |
| 308 | void rrdset_update_heterogeneous_flag(RRDSET *st); |
| 309 | |
| 310 | void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st); |
| 311 | void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st); |
| 312 | |
| 313 | // checks if the RRDSET should be offered to viewers |
| 314 | #define rrdset_is_available_for_viewers(st) ( \ |
| 315 | !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && \ |
| 316 | !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && \ |
| 317 | rrdset_number_of_dimensions(st) && \ |
| 318 | (st)->rrd_memory_mode != RRD_DB_MODE_NONE \ |
| 319 | ) |
| 320 | |
| 321 | #define rrdset_is_available_for_exporting_and_alarms(st) ( \ |
| 322 | !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && \ |
| 323 | rrdset_number_of_dimensions(st) \ |
| 324 | ) |
| 325 | |
| 326 | time_t rrdset_first_entry_s(RRDSET *st); |
| 327 | time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier); |
| 328 | time_t rrdset_last_entry_s(RRDSET *st); |
| 329 | time_t rrdset_last_entry_s_of_tier(RRDSET *st, size_t tier); |
| 330 | |
| 331 | void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_time_s, time_t *last_time_s, time_t now_s, size_t tier); |
| 332 | |
| 333 | void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels); |
| 334 | |
| 335 | #include "rrdset-index-name.h" |
| 336 | #include "rrdset-index-id.h" |
| 337 | #include "rrdset-slots.h" |
| 338 | #include "rrdset-collection.h" |
| 339 | |
| 340 | #endif //NETDATA_RRDSET_H |