master
h 449 lines 17.6 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef NETDATA_STORAGEENGINEAPI_H
4 #define NETDATA_STORAGEENGINEAPI_H
5
6 #include "libnetdata/libnetdata.h"
7 #include "rrd-database-mode.h"
8
9 typedef struct rrddim RRDDIM;
10
11 typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
12
13 typedef enum __attribute__ ((__packed__)) storage_priority {
14 STORAGE_PRIORITY_INTERNAL_DBENGINE = 0,
15 STORAGE_PRIORITY_INTERNAL_QUERY_PREP,
16
17 // query priorities
18 STORAGE_PRIORITY_HIGH,
19 STORAGE_PRIORITY_NORMAL,
20 STORAGE_PRIORITY_SYNCHRONOUS_FIRST,
21 STORAGE_PRIORITY_LOW,
22 STORAGE_PRIORITY_BEST_EFFORT,
23
24 // synchronous query, not to be dispatched to workers or queued
25 STORAGE_PRIORITY_SYNCHRONOUS,
26
27 STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE,
28 } STORAGE_PRIORITY;
29
30 typedef enum __attribute__ ((__packed__)) {
31 STORAGE_ENGINE_BACKEND_RRDDIM = 1,
32 STORAGE_ENGINE_BACKEND_DBENGINE = 2,
33 } STORAGE_ENGINE_BACKEND;
34
35 #define is_valid_backend(backend) ((backend) >= STORAGE_ENGINE_BACKEND_RRDDIM && (backend) <= STORAGE_ENGINE_BACKEND_DBENGINE)
36
37 // iterator state for RRD dimension data queries
38 struct storage_engine_query_handle {
39 time_t start_time_s;
40 time_t end_time_s;
41 STORAGE_PRIORITY priority;
42 STORAGE_ENGINE_BACKEND seb;
43 STORAGE_QUERY_HANDLE *handle;
44 };
45
46 // non-existing structs instead of voids
47 // to enable type checking at compile time
48 typedef struct storage_instance STORAGE_INSTANCE;
49 typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
50 typedef struct storage_alignment STORAGE_METRICS_GROUP;
51
52 // --------------------------------------------------------------------------------------------------------------------
53 // engine-specific iterator state for dimension data collection
54
55 typedef struct storage_collect_handle {
56 STORAGE_ENGINE_BACKEND seb;
57 } STORAGE_COLLECT_HANDLE;
58
59 // --------------------------------------------------------------------------------------------------------------------
60 // function pointers for all APIs provided by a storage engine
61
62 typedef struct storage_engine_api {
63 // metric management
64 STORAGE_METRIC_HANDLE *(*metric_get_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id);
65 STORAGE_METRIC_HANDLE *(*metric_get_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
66 STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *si);
67 void (*metric_release)(STORAGE_METRIC_HANDLE *);
68 STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
69 bool (*metric_retention_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id, time_t *first_entry_s, time_t *last_entry_s);
70 bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
71 void (*metric_retention_delete_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id);
72 } STORAGE_ENGINE_API;
73
74 typedef struct storage {
75 STORAGE_ENGINE_BACKEND seb;
76 RRD_DB_MODE id;
77 const char* name;
78 STORAGE_ENGINE_API api;
79 } STORAGE_ENGINE;
80
81 STORAGE_ENGINE* storage_engine_get(RRD_DB_MODE mmode);
82 STORAGE_ENGINE* storage_engine_find(const char* name);
83
84 // Iterator over existing engines
85 STORAGE_ENGINE* storage_engine_foreach_init();
86 STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);
87
88 // --------------------------------------------------------------------------------------------------------------------
89 // Storage tier data for every dimension
90
91 struct rrddim_tier {
92 STORAGE_POINT virtual_point;
93 STORAGE_POINT last_completed_point; // tier1/2 spread over time
94 SPINLOCK spinlock;
95 STORAGE_ENGINE_BACKEND seb;
96 uint16_t last_completed_point_flush_modulo; // tier1/2 spread over time
97 uint32_t tier_grouping;
98 time_t next_point_end_time_s;
99 STORAGE_METRIC_HANDLE *smh; // the metric handle inside the database
100 STORAGE_COLLECT_HANDLE *sch; // the data collection handle
101 };
102
103 // --------------------------------------------------------------------------------------------------------------------
104
105 #include "daemon/config/netdata-conf-db.h"
106
107 // --------------------------------------------------------------------------------------------------------------------
108 // DATA COLLECTION STORAGE OPS
109
110 STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
111 STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
112
113 static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, nd_uuid_t *uuid) {
114 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
115
116 #ifdef ENABLE_DBENGINE
117 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
118 return rrdeng_metrics_group_get(si, uuid);
119 #endif
120 return rrddim_metrics_group_get(si, uuid);
121 }
122
123 // --------------------------------------------------------------------------------------------------------------------
124
125 void rrdeng_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
126 void rrddim_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
127
128 static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg) {
129 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
130
131 #ifdef ENABLE_DBENGINE
132 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
133 rrdeng_metrics_group_release(si, smg);
134 else
135 #endif
136 rrddim_metrics_group_release(si, smg);
137 }
138
139 // --------------------------------------------------------------------------------------------------------------------
140
141 STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
142 STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
143
144 static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {
145 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
146
147 #ifdef ENABLE_DBENGINE
148 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
149 return rrdeng_store_metric_init(smh, update_every, smg);
150 #endif
151 return rrddim_collect_init(smh, update_every, smg);
152 }
153
154 // --------------------------------------------------------------------------------------------------------------------
155
156 void rrdeng_store_metric_next(
157 STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
158 NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
159 uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
160
161 void rrddim_collect_store_metric(
162 STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
163 NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
164 uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
165
166 ALWAYS_INLINE_HOT_FLATTEN
167 static void storage_engine_store_metric(
168 STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,
169 NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
170 uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {
171 if(unlikely(!sch))
172 return;
173
174 internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
175
176 #ifdef ENABLE_DBENGINE
177 if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
178 return rrdeng_store_metric_next(sch, point_in_time_ut,
179 n, min_value, max_value,
180 count, anomaly_count, flags);
181 #endif
182 return rrddim_collect_store_metric(sch, point_in_time_ut,
183 n, min_value, max_value,
184 count, anomaly_count, flags);
185 }
186
187 // --------------------------------------------------------------------------------------------------------------------
188
189 uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *si);
190
191 static inline uint64_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
192 #ifdef ENABLE_DBENGINE
193 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
194 return rrdeng_disk_space_max(si);
195 #endif
196
197 return 0;
198 }
199
200 // --------------------------------------------------------------------------------------------------------------------
201
202 uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *si);
203
204 static inline uint64_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
205 #ifdef ENABLE_DBENGINE
206 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
207 return rrdeng_disk_space_used(si);
208 #endif
209
210 // TODO - calculate the total host disk space for memory mode save and map
211 return 0;
212 }
213
214 // --------------------------------------------------------------------------------------------------------------------
215
216 uint64_t rrdeng_metrics(STORAGE_INSTANCE *si);
217
218 static inline uint64_t storage_engine_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
219 #ifdef ENABLE_DBENGINE
220 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
221 return rrdeng_metrics(si);
222 #endif
223
224 // TODO - calculate the total host disk space for memory mode save and map
225 return 0;
226 }
227
228 // --------------------------------------------------------------------------------------------------------------------
229
230 uint64_t rrdeng_samples(STORAGE_INSTANCE *si);
231
232 static inline uint64_t storage_engine_samples(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
233 #ifdef ENABLE_DBENGINE
234 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
235 return rrdeng_samples(si);
236 #endif
237 return 0;
238 }
239
240 // --------------------------------------------------------------------------------------------------------------------
241
242 time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *si);
243
244 static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
245 #ifdef ENABLE_DBENGINE
246 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
247 return rrdeng_global_first_time_s(si);
248 #endif
249
250 return now_realtime_sec() - (time_t)(default_rrd_history_entries * nd_profile.update_every);
251 }
252
253 // --------------------------------------------------------------------------------------------------------------------
254
255 size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *si);
256
257 static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {
258 #ifdef ENABLE_DBENGINE
259 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
260 return rrdeng_currently_collected_metrics(si);
261 #endif
262
263 // TODO - calculate the total host disk space for memory mode save and map
264 return 0;
265 }
266
267 // --------------------------------------------------------------------------------------------------------------------
268
269 void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *sch);
270 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *sch);
271
272 static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *sch) {
273 if(unlikely(!sch))
274 return;
275
276 internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
277
278 #ifdef ENABLE_DBENGINE
279 if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
280 rrdeng_store_metric_flush_current_page(sch);
281 else
282 #endif
283 rrddim_store_metric_flush(sch);
284 }
285
286 // --------------------------------------------------------------------------------------------------------------------
287
288 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *sch);
289 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *sch);
290 // a finalization function to run after collection is over
291 // returns 1 if it's safe to delete the dimension
292
293 static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *sch) {
294 if(unlikely(!sch))
295 return 1; // safe to delete if no handle
296
297 internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
298
299 #ifdef ENABLE_DBENGINE
300 if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
301 return rrdeng_store_metric_finalize(sch);
302 #endif
303
304 return rrddim_collect_finalize(sch);
305 }
306
307 // --------------------------------------------------------------------------------------------------------------------
308
309 void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
310 void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
311
312 static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every) {
313 if(unlikely(!sch))
314 return;
315
316 internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");
317
318 #ifdef ENABLE_DBENGINE
319 if(likely(sch->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
320 rrdeng_store_metric_change_collection_frequency(sch, update_every);
321 else
322 #endif
323 rrddim_store_metric_change_collection_frequency(sch, update_every);
324 }
325
326 // --------------------------------------------------------------------------------------------------------------------
327 // STORAGE ENGINE QUERY OPS
328
329 time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh);
330 time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *smh);
331
332 ALWAYS_INLINE_HOT_FLATTEN
333 static time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {
334 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
335
336 #ifdef ENABLE_DBENGINE
337 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
338 return rrdeng_metric_oldest_time(smh);
339 #endif
340 return rrddim_query_oldest_time_s(smh);
341 }
342
343 // --------------------------------------------------------------------------------------------------------------------
344
345 time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh);
346 time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *smh);
347
348 ALWAYS_INLINE_HOT_FLATTEN
349 static time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {
350 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
351
352 #ifdef ENABLE_DBENGINE
353 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
354 return rrdeng_metric_latest_time(smh);
355 #endif
356 return rrddim_query_latest_time_s(smh);
357 }
358
359 // --------------------------------------------------------------------------------------------------------------------
360
361 void rrdeng_load_metric_init(
362 STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
363 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
364
365 void rrddim_query_init(
366 STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
367 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
368
369 ALWAYS_INLINE_HOT_FLATTEN
370 static void storage_engine_query_init(
371 STORAGE_ENGINE_BACKEND seb __maybe_unused,
372 STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
373 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {
374 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");
375
376 #ifdef ENABLE_DBENGINE
377 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))
378 rrdeng_load_metric_init(smh, seqh, start_time_s, end_time_s, priority);
379 else
380 #endif
381 rrddim_query_init(smh, seqh, start_time_s, end_time_s, priority);
382 }
383
384 // --------------------------------------------------------------------------------------------------------------------
385
386 STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh);
387 STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh);
388
389 ALWAYS_INLINE_HOT_FLATTEN
390 static STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {
391 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
392
393 #ifdef ENABLE_DBENGINE
394 if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
395 return rrdeng_load_metric_next(seqh);
396 #endif
397 return rrddim_query_next_metric(seqh);
398 }
399
400 // --------------------------------------------------------------------------------------------------------------------
401
402 int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh);
403 int rrddim_query_is_finished(struct storage_engine_query_handle *seqh);
404
405 ALWAYS_INLINE_HOT_FLATTEN
406 static int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {
407 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
408
409 #ifdef ENABLE_DBENGINE
410 if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
411 return rrdeng_load_metric_is_finished(seqh);
412 #endif
413 return rrddim_query_is_finished(seqh);
414 }
415
416 // --------------------------------------------------------------------------------------------------------------------
417
418 void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh);
419 void rrddim_query_finalize(struct storage_engine_query_handle *seqh);
420
421 ALWAYS_INLINE_HOT_FLATTEN
422 static void storage_engine_query_finalize(struct storage_engine_query_handle *seqh) {
423 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
424
425 #ifdef ENABLE_DBENGINE
426 if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
427 rrdeng_load_metric_finalize(seqh);
428 else
429 #endif
430 rrddim_query_finalize(seqh);
431 }
432
433 // --------------------------------------------------------------------------------------------------------------------
434
435 time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh);
436 time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *seqh);
437
438 ALWAYS_INLINE_HOT_FLATTEN
439 static time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *seqh) {
440 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
441
442 #ifdef ENABLE_DBENGINE
443 if(likely(seqh->seb == STORAGE_ENGINE_BACKEND_DBENGINE))
444 return rrdeng_load_align_to_optimal_before(seqh);
445 #endif
446 return rrddim_query_align_to_optimal_before(seqh);
447 }
448
449 #endif