@cryptotaxi247 / netdata-1
Code Commits Issues Pulls Sessions CI/CD
master
master
h 607 lines 23.3 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef NETDATA_RRDENGINE_H
4 #define NETDATA_RRDENGINE_H
5
6 #include <fcntl.h>
7 #include <lz4.h>
8 #include <Judy.h>
9 #include <openssl/sha.h>
10 #include <openssl/evp.h>
11 #include "../rrd.h"
12 #include "rrddiskprotocol.h"
13 #include "rrdenginelib.h"
14 #include "datafile.h"
15 #include "journalfile.h"
16 #include "rrdengineapi.h"
17 #include "pagecache.h"
18 #include "mrg.h"
19 #include "cache.h"
20 #include "pdc.h"
21 #include "page.h"
22
23 #include "daemon/protected-access.h"
24
25 extern unsigned rrdeng_pages_per_extent;
26
27 #define BLOCK_TO_OFFSET(block) ((uint64_t)(block) << 12)
28 #define OFFSET_TO_BLOCK(ofs) ((uint64_t)(ofs) >> 12)
29
30 #define UNLINK_FILE(ctx, path, ret_var) \
31 do { \
32 uv_fs_t _req; \
33 (ret_var) = uv_fs_unlink(NULL, &(_req), (path), NULL); \
34 if ((ret_var) < 0) { \
35 netdata_log_error("DBENGINE: uv_fs_unlink(\"%s\"): %s", (path), uv_strerror(ret_var)); \
36 ctx_fs_error(ctx); \
37 } \
38 uv_fs_req_cleanup(&(_req)); \
39 } while (0)
40
41 #define CLOSE_FILE(ctx, path, file, ret_var) \
42 do { \
43 uv_fs_t _req; \
44 (ret_var) = uv_fs_close(NULL, &(_req), (file), NULL); \
45 if ((ret_var) < 0) { \
46 netdata_log_error("DBENGINE: uv_fs_close(\"%s\"): %s", (path), uv_strerror(ret_var)); \
47 ctx_fs_error(ctx); \
48 } \
49 uv_fs_req_cleanup(&(_req)); \
50 } while (0)
51
52 /* Forward declarations */
53 struct rrdengine_instance;
54 struct rrdeng_cmd;
55
56 #define MAX_PAGES_PER_EXTENT (109) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
57 #define DEFAULT_PAGES_PER_EXTENT (109)
58
59 #define MAX_EXTENT_UNCOMPRESSED_SIZE (MAX_PAGES_PER_EXTENT * (RRDENG_BLOCK_SIZE + RRDENG_GORILLA_32BIT_BUFFER_SIZE))
60
61 static inline size_t rrdeng_min_extent_disk_size(void) {
62 return sizeof(struct rrdeng_df_extent_header) +
63 sizeof(struct rrdeng_extent_page_descr) +
64 sizeof(struct rrdeng_df_extent_trailer);
65 }
66
67 static inline size_t rrdeng_max_extent_disk_size(void) {
68 return sizeof(struct rrdeng_df_extent_header) +
69 sizeof(struct rrdeng_extent_page_descr) * MAX_PAGES_PER_EXTENT +
70 MAX_EXTENT_UNCOMPRESSED_SIZE +
71 sizeof(struct rrdeng_df_extent_trailer);
72 }
73
74 static inline bool rrdeng_valid_extent_disk_size(size_t size) {
75 return size >= rrdeng_min_extent_disk_size() && size <= rrdeng_max_extent_disk_size();
76 }
77
78
79 #define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u"
80 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
81
82 typedef struct dbengine_tier_stats {
83 RRDSET *st;
84 RRDDIM *rd_space;
85 RRDDIM *rd_time;
86 } DBENGINE_TIER_STATS;
87
88 typedef enum __attribute__ ((__packed__)) {
89 // final status for all pages
90 // if a page does not have one of these, it is considered unroutable
91 PDC_PAGE_READY = (1 << 0), // ready to be processed (pd->page is not null)
92 PDC_PAGE_FAILED = (1 << 1), // failed to be loaded (pd->page is null)
93 PDC_PAGE_SKIP = (1 << 2), // don't use this page, it is not good for us
94 PDC_PAGE_INVALID = (1 << 3), // don't use this page, it is invalid
95 PDC_PAGE_EMPTY = (1 << 4), // the page is empty, does not have any data
96
97 // other statuses for tracking issues
98 PDC_PAGE_PREPROCESSED = (1 << 5), // used during preprocessing
99 PDC_PAGE_PROCESSED = (1 << 6), // processed by the query caller
100 PDC_PAGE_RELEASED = (1 << 7), // already released
101
102 // data found in cache (preloaded) or on disk?
103 PDC_PAGE_PRELOADED = (1 << 8), // data found in memory
104 PDC_PAGE_DISK_PENDING = (1 << 9), // data need to be loaded from disk
105
106 // worker related statuses
107 PDC_PAGE_FAILED_INVALID_EXTENT = (1 << 10),
108 PDC_PAGE_FAILED_NOT_IN_EXTENT = (1 << 11),
109 PDC_PAGE_FAILED_TO_MAP_EXTENT = (1 << 12),
110 PDC_PAGE_FAILED_TO_ACQUIRE_DATAFILE= (1 << 13),
111
112 PDC_PAGE_EXTENT_FROM_CACHE = (1 << 14),
113 PDC_PAGE_EXTENT_FROM_DISK = (1 << 15),
114
115 PDC_PAGE_CANCELLED = (1 << 16), // the query thread had left when we try to load the page
116
117 PDC_PAGE_SOURCE_MAIN_CACHE = (1 << 17),
118 PDC_PAGE_SOURCE_OPEN_CACHE = (1 << 18),
119 PDC_PAGE_SOURCE_JOURNAL_V2 = (1 << 19),
120 PDC_PAGE_PRELOADED_PASS4 = (1 << 20),
121
122 // datafile acquired
123 PDC_PAGE_DATAFILE_ACQUIRED = (1 << 30),
124 } PDC_PAGE_STATUS;
125
126 #define PDC_PAGE_QUERY_GLOBAL_SKIP_LIST (PDC_PAGE_FAILED | PDC_PAGE_SKIP | PDC_PAGE_INVALID | PDC_PAGE_RELEASED)
127
128 typedef struct page_details_control {
129 struct rrdengine_instance *ctx;
130 struct metric *metric;
131
132 struct completion prep_completion;
133 struct completion page_completion; // sync between the query thread and the workers
134
135 Pvoid_t page_list_JudyL; // the list of page details
136 unsigned completed_jobs; // the number of jobs completed last time the query thread checked
137 bool workers_should_stop; // true when the query thread left and the workers should stop
138 bool prep_done;
139
140 PDC_PAGE_STATUS common_status;
141 size_t pages_to_load_from_disk;
142
143 SPINLOCK refcount_spinlock; // spinlock to protect refcount
144 int32_t refcount; // the number of workers currently working on this request + 1 for the query thread
145 size_t executed_with_gaps;
146
147 time_t start_time_s;
148 time_t end_time_s;
149 STORAGE_PRIORITY priority;
150
151 time_t optimal_end_time_s;
152 } PDC;
153
154 PDC *pdc_get(void);
155
156 struct page_details {
157 struct {
158 struct rrdengine_datafile *ptr;
159 uint32_t block; // the block in the datafile. Offset in the datafile is block * RRDENG_BLOCK_SIZE
160 uint32_t bytes;
161 } datafile;
162
163 struct pgc_page *page;
164 Word_t metric_id;
165 time_t first_time_s;
166 time_t last_time_s;
167 uint32_t update_every_s;
168 PDC_PAGE_STATUS status;
169
170 struct {
171 struct page_details *prev;
172 struct page_details *next;
173 } load;
174 };
175
176 struct page_details *page_details_get(void);
177
178 #define pdc_page_status_check(pd, flag) (__atomic_load_n(&((pd)->status), __ATOMIC_ACQUIRE) & (flag))
179 #define pdc_page_status_set(pd, flag) __atomic_or_fetch(&((pd)->status), flag, __ATOMIC_RELEASE)
180 #define pdc_page_status_clear(pd, flag) __atomic_and_fetch(&((od)->status), ~(flag), __ATOMIC_RELEASE)
181
182 struct jv2_extents_info {
183 uint32_t index;
184 uint32_t block;
185 unsigned bytes;
186 uint32_t number_of_pages;
187 };
188
189 struct jv2_metrics_info {
190 nd_uuid_t *uuid;
191 void *metric;
192 uint32_t page_list_header;
193 uint32_t number_of_pages;
194 time_t first_time_s;
195 time_t last_time_s;
196 Pvoid_t JudyL_pages_by_start_time;
197 };
198
199 struct jv2_page_info {
200 time_t start_time_s;
201 time_t end_time_s;
202 uint32_t update_every_s;
203 uint32_t extent_index;
204 size_t page_length;
205 void *custom_data;
206
207 // private
208 struct pgc_page *page;
209 };
210
211 typedef enum __attribute__ ((__packed__)) {
212 RRDENG_COLLECT_HANDLE_OPTION_NONE = 0,
213
214 #ifdef NETDATA_INTERNAL_CHECKS
215 RRDENG_1ST_METRIC_WRITER = (1 << 0),
216 #endif
217 } RRDENG_COLLECT_HANDLE_OPTIONS;
218
219 typedef enum __attribute__ ((__packed__)) {
220 RRDENG_PAGE_PAST_COLLECTION = (1 << 0),
221 RRDENG_PAGE_REPEATED_COLLECTION = (1 << 1),
222 RRDENG_PAGE_BIG_GAP = (1 << 2),
223 RRDENG_PAGE_GAP = (1 << 3),
224 RRDENG_PAGE_FUTURE_POINT = (1 << 4),
225 RRDENG_PAGE_CREATED_IN_FUTURE = (1 << 5),
226 RRDENG_PAGE_COMPLETED_IN_FUTURE = (1 << 6),
227 RRDENG_PAGE_UNALIGNED = (1 << 7),
228 RRDENG_PAGE_CONFLICT = (1 << 8),
229 RRDENG_PAGE_FULL = (1 << 9),
230 RRDENG_PAGE_COLLECT_FINALIZE = (1 << 10),
231 RRDENG_PAGE_UPDATE_EVERY_CHANGE = (1 << 11),
232 RRDENG_PAGE_STEP_TOO_SMALL = (1 << 12),
233 RRDENG_PAGE_STEP_UNALIGNED = (1 << 13),
234 } RRDENG_COLLECT_PAGE_FLAGS;
235
236 struct rrdeng_collect_handle {
237 struct storage_collect_handle common; // has to be first item
238
239 RRDENG_COLLECT_PAGE_FLAGS page_flags;
240 RRDENG_COLLECT_HANDLE_OPTIONS options;
241 uint8_t type;
242
243 struct rrdengine_instance *ctx;
244 struct metric *metric;
245 struct pgc_page *pgc_page;
246 struct pgd *page_data;
247 struct pg_alignment *alignment;
248 uint32_t page_entries_max;
249 uint32_t page_position; // keep track of the current page size, to make sure we don't exceed it
250 usec_t page_start_time_ut;
251 usec_t page_end_time_ut;
252 usec_t update_every_ut;
253 };
254
255 struct rrdeng_query_handle {
256 struct metric *metric;
257 struct pgc_page *page;
258 struct rrdengine_instance *ctx;
259 struct pgd_cursor pgdc;
260 struct page_details_control *pdc;
261
262 // the request
263 time_t start_time_s;
264 time_t end_time_s;
265 STORAGE_PRIORITY priority;
266
267 // internal data
268 time_t now_s;
269 uint32_t dt_s;
270
271 unsigned position;
272 unsigned entries;
273
274 #ifdef NETDATA_INTERNAL_CHECKS
275 usec_t started_time_s;
276 pid_t query_pid;
277 struct rrdeng_query_handle *prev, *next;
278 #endif
279 };
280
281 struct rrdeng_query_handle *rrdeng_query_handle_get(void);
282 void rrdeng_query_handle_release(struct rrdeng_query_handle *handle);
283
284 enum rrdeng_opcode {
285 /* can be used to return empty status or flush the command queue */
286 RRDENG_OPCODE_NOOP = 0,
287
288 RRDENG_OPCODE_QUERY,
289 RRDENG_OPCODE_EXTENT_WRITE,
290 RRDENG_OPCODE_EXTENT_READ,
291 RRDENG_OPCODE_DATABASE_ROTATE,
292 RRDENG_OPCODE_JOURNAL_INDEX,
293 RRDENG_OPCODE_FLUSH_MAIN,
294 RRDENG_OPCODE_EVICT_MAIN,
295 RRDENG_OPCODE_EVICT_OPEN,
296 RRDENG_OPCODE_EVICT_EXTENT,
297 RRDENG_OPCODE_CTX_SHUTDOWN,
298 RRDENG_OPCODE_CTX_FLUSH_DIRTY,
299 RRDENG_OPCODE_CTX_FLUSH_HOT_DIRTY,
300 RRDENG_OPCODE_CTX_QUIESCE,
301 RRDENG_OPCODE_CTX_POPULATE_MRG,
302 RRDENG_OPCODE_SHUTDOWN_EVLOOP,
303 RRDENG_OPCODE_PARALLEL_WEIGHT,
304 RRDENG_OPCODE_MRG_LOAD,
305 RRDENG_OPCODE_CLEANUP,
306
307 RRDENG_OPCODE_MAX
308 };
309
310 // WORKERS IDS:
311 // RRDENG_MAX_OPCODE : reserved for the cleanup
312 // RRDENG_MAX_OPCODE + opcode : reserved for the callbacks of each opcode
313 // RRDENG_MAX_OPCODE + RRDENG_MAX_OPCODE : reserved for the timer
314 #define RRDENG_TIMER_CB (RRDENG_OPCODE_MAX + RRDENG_OPCODE_MAX)
315 #define RRDENG_OPCODES_WAITING (RRDENG_TIMER_CB + 1)
316 #define RRDENG_WORKS_DISPATCHED (RRDENG_TIMER_CB + 2)
317 #define RRDENG_WORKS_EXECUTING (RRDENG_TIMER_CB + 3)
318 #define RRDENG_RETENTION_TIMER_CB (RRDENG_TIMER_CB + 4)
319
320 struct extent_io_data {
321 unsigned fileno;
322 uint32_t block;
323 unsigned bytes;
324 };
325
326 struct extent_io_descriptor {
327 struct rrdengine_instance *ctx;
328 void *buf;
329 uint64_t pos;
330 uint32_t descr_count;
331 uint32_t bytes;
332 uint32_t real_io_size;
333 struct wal *wal;
334 uv_file file;
335 struct page_descr_with_data *descr_array[MAX_PAGES_PER_EXTENT];
336 struct rrdengine_datafile *datafile;
337 };
338
339 typedef struct wal {
340 uint64_t transaction_id;
341 void *buf;
342 size_t size;
343 size_t buf_size;
344
345 struct {
346 struct wal *prev;
347 struct wal *next;
348 } cache;
349 } WAL;
350
351 WAL *wal_get(struct rrdengine_instance *ctx, unsigned size);
352 void wal_release(WAL *wal);
353
354 /*
355 * Debug statistics not used by code logic.
356 * They only describe operations since DB engine instance load time.
357 */
358 struct rrdengine_statistics {
359 PAD64(rrdeng_stats_t) before_decompress_bytes;
360 PAD64(rrdeng_stats_t) after_decompress_bytes;
361 PAD64(rrdeng_stats_t) before_compress_bytes;
362 PAD64(rrdeng_stats_t) after_compress_bytes;
363
364 PAD64(rrdeng_stats_t) io_write_bytes;
365 PAD64(rrdeng_stats_t) io_write_requests;
366 PAD64(rrdeng_stats_t) io_read_bytes;
367 PAD64(rrdeng_stats_t) io_read_requests;
368
369 PAD64(rrdeng_stats_t) datafile_creations;
370 PAD64(rrdeng_stats_t) datafile_deletions;
371 PAD64(rrdeng_stats_t) journalfile_creations;
372 PAD64(rrdeng_stats_t) journalfile_deletions;
373
374 PAD64(rrdeng_stats_t) io_errors;
375 PAD64(rrdeng_stats_t) fs_errors;
376 };
377
378 struct rrdeng_global_stats {
379 /* I/O errors global counter */
380 PAD64(rrdeng_stats_t) global_io_errors;
381
382 /* File-System errors global counter */
383 PAD64(rrdeng_stats_t) global_fs_errors;
384
385 /* number of File-Descriptors that have been reserved by dbengine */
386 PAD64(rrdeng_stats_t) rrdeng_reserved_file_descriptors;
387
388 /* inability to flush global counters */
389 PAD64(rrdeng_stats_t) global_pg_cache_over_half_dirty_events;
390 PAD64(rrdeng_stats_t) global_flushing_pressure_page_deletions; /* number of deleted pages */
391 };
392
393 extern struct rrdeng_global_stats global_stats;
394
395 typedef struct tier_config_prototype {
396 int tier; // the tier of this ctx
397 uint8_t page_type; // default page type for this context
398 uint64_t max_disk_space; // the max disk space this ctx is allowed to use
399 time_t max_retention_s; // The max retention in seconds
400 uint8_t disk_percentage; // percentage of metadata that contribute towards tier space used
401 uint8_t global_compress_alg; // the wanted compression algorithm
402 char dbfiles_path[FILENAME_MAX + 1];
403
404 struct {
405 uint32_t uses;
406 bool enabled;
407 bool is_on_disk;
408 SPINLOCK spinlock;
409 } _internal;
410 } TIER_CONFIG_PROTOTYPE;
411
412 struct rrdengine_instance {
413 TIER_CONFIG_PROTOTYPE config;
414
415 struct {
416 netdata_rwlock_t rwlock; // the JudyL of datafiles is protected by this lock
417 bool disk_time; // true: delete for disk quota, false: delete for retention
418 bool pending_rotate; // Change from event loop
419 bool pending_index; // Change from event loop
420 Pvoid_t JudyL; // the datafiles, indexed by fileno
421 } datafiles;
422
423 struct {
424 RW_SPINLOCK spinlock;
425 Pvoid_t JudyL;
426 } njfv2idx;
427
428 struct {
429 PAD64(unsigned) last_fileno; // newest index of datafile and journalfile
430 PAD64(unsigned) last_flush_fileno; // newest index of datafile received data
431
432 PAD64(size_t) collectors_running;
433 PAD64(size_t) collectors_running_duplicate;
434 PAD64(size_t) inflight_queries; // the number of queries currently running
435 PAD64(uint64_t) current_disk_space; // the current disk space size used
436
437 PAD64(uint64_t) transaction_id; // the transaction id of the next extent flushing
438
439 PAD64(bool) migration_to_v2_running;
440 PAD64(bool) now_deleting_files;
441 PAD64(bool) needs_indexing;
442 PAD64(unsigned) extents_currently_being_flushed; // non-zero until we commit data to disk (both datafile and journal file)
443
444 PAD64(time_t) first_time_s;
445 PAD64(uint64_t) metrics;
446 PAD64(uint64_t) samples;
447 } atomic;
448
449 struct {
450 bool exit_mode;
451 bool enabled; // when set (before shutdown), queries are prohibited
452 } quiesce;
453
454 struct {
455 struct completion load_mrg;
456 bool create_new_datafile_pair;
457 } loading;
458
459 struct rrdengine_statistics stats;
460 };
461
462 #define ctx_current_disk_space_get(ctx) __atomic_load_n(&(ctx)->atomic.current_disk_space, __ATOMIC_RELAXED)
463 #define ctx_current_disk_space_increase(ctx, size) __atomic_add_fetch(&(ctx)->atomic.current_disk_space, size, __ATOMIC_RELAXED)
464 #define ctx_current_disk_space_decrease(ctx, size) __atomic_sub_fetch(&(ctx)->atomic.current_disk_space, size, __ATOMIC_RELAXED)
465
466 static inline void ctx_io_read_op_bytes(struct rrdengine_instance *ctx, size_t bytes) {
467 __atomic_add_fetch(&ctx->stats.io_read_bytes, bytes, __ATOMIC_RELAXED);
468 __atomic_add_fetch(&ctx->stats.io_read_requests, 1, __ATOMIC_RELAXED);
469 }
470
471 static inline void ctx_io_write_op_bytes(struct rrdengine_instance *ctx, size_t bytes) {
472 __atomic_add_fetch(&ctx->stats.io_write_bytes, bytes, __ATOMIC_RELAXED);
473 __atomic_add_fetch(&ctx->stats.io_write_requests, 1, __ATOMIC_RELAXED);
474 }
475
476 static inline void ctx_io_error(struct rrdengine_instance *ctx) {
477 __atomic_add_fetch(&ctx->stats.io_errors, 1, __ATOMIC_RELAXED);
478 rrd_stat_atomic_add(&global_stats.global_io_errors, 1);
479 }
480
481 static inline void ctx_fs_error(struct rrdengine_instance *ctx) {
482 __atomic_add_fetch(&ctx->stats.fs_errors, 1, __ATOMIC_RELAXED);
483 rrd_stat_atomic_add(&global_stats.global_fs_errors, 1);
484 }
485
486 #define ctx_last_fileno_get(ctx) __atomic_load_n(&(ctx)->atomic.last_fileno, __ATOMIC_RELAXED)
487 #define ctx_last_fileno_increment(ctx) __atomic_add_fetch(&(ctx)->atomic.last_fileno, 1, __ATOMIC_RELAXED)
488
489 #define ctx_last_flush_fileno_get(ctx) __atomic_load_n(&(ctx)->atomic.last_flush_fileno, __ATOMIC_RELAXED)
490 static inline void ctx_last_flush_fileno_set(struct rrdengine_instance *ctx, unsigned fileno) {
491 unsigned old_fileno = ctx_last_flush_fileno_get(ctx);
492
493 do {
494 if(old_fileno >= fileno)
495 return;
496
497 } while(!__atomic_compare_exchange_n(&ctx->atomic.last_flush_fileno, &old_fileno, fileno, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
498 }
499
500 #define ctx_is_available_for_queries(ctx) (__atomic_load_n(&(ctx)->quiesce.enabled, __ATOMIC_RELAXED) == false && __atomic_load_n(&(ctx)->quiesce.exit_mode, __ATOMIC_RELAXED) == false)
501
502 bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx);
503 int init_rrd_files(struct rrdengine_instance *ctx);
504 void finalize_rrd_files(struct rrdengine_instance *ctx);
505 bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx);
506 void dbengine_event_loop(void *arg);
507
508 typedef void (*enqueue_callback_t)(struct rrdeng_cmd *cmd);
509 typedef void (*dequeue_callback_t)(struct rrdeng_cmd *cmd);
510
511 void rrdeng_enqueue_epdl_cmd(struct rrdeng_cmd *cmd);
512 void rrdeng_dequeue_epdl_cmd(struct rrdeng_cmd *cmd);
513
514 typedef struct rrdeng_cmd *(*requeue_callback_t)(void *data);
515 void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY priority);
516
517 void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, void *data,
518 struct completion *completion, enum storage_priority priority,
519 enqueue_callback_t enqueue_cb, dequeue_callback_t dequeue_cb);
520
521 void pdc_route_asynchronously(struct rrdengine_instance *ctx, struct page_details_control *pdc);
522 void pdc_route_synchronously(struct rrdengine_instance *ctx, struct page_details_control *pdc);
523 void pdc_route_synchronously_first(struct rrdengine_instance *ctx, struct page_details_control *pdc);
524
525 void pdc_acquire(PDC *pdc);
526 bool pdc_release_and_destroy_if_unreferenced(PDC *pdc, bool worker, bool router);
527
528 uint64_t rrdeng_target_data_file_size(struct rrdengine_instance *ctx);
529
530 struct page_descr_with_data *page_descriptor_get(void);
531
532 typedef struct validated_page_descriptor {
533 time_t start_time_s;
534 time_t end_time_s;
535 uint32_t update_every_s;
536 size_t page_length;
537 size_t point_size;
538 size_t entries;
539 uint8_t type;
540 bool is_valid;
541 } VALIDATED_PAGE_DESCRIPTOR;
542
543 #define page_entries_by_time(start_time_s, end_time_s, update_every_s) \
544 ((update_every_s) ? (((end_time_s) - ((start_time_s) - (update_every_s))) / (update_every_s)) : 1)
545
546 #define page_entries_by_size(page_length_in_bytes, point_size_in_bytes) \
547 ((page_length_in_bytes) / (point_size_in_bytes))
548
549 VALIDATED_PAGE_DESCRIPTOR validate_page(nd_uuid_t *uuid,
550 time_t start_time_s,
551 time_t end_time_s,
552 uint32_t update_every_s,
553 size_t page_length,
554 uint8_t page_type,
555 size_t entries,
556 time_t now_s,
557 uint32_t overwrite_zero_update_every_s,
558 bool have_read_error,
559 const char *msg,
560 RRDENG_COLLECT_PAGE_FLAGS flags);
561 VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, uint32_t overwrite_zero_update_every_s, bool have_read_error);
562 void collect_page_flags_to_buffer(BUFFER *wb, RRDENG_COLLECT_PAGE_FLAGS flags);
563
564 typedef enum {
565 PAGE_IS_IN_THE_PAST = -1,
566 PAGE_IS_IN_RANGE = 0,
567 PAGE_IS_IN_THE_FUTURE = 1,
568 } TIME_RANGE_COMPARE;
569
570 TIME_RANGE_COMPARE is_page_in_time_range(time_t page_first_time_s, time_t page_last_time_s, time_t wanted_start_time_s, time_t wanted_end_time_s);
571
572 static inline time_t max_acceptable_collected_time(void) {
573 return now_realtime_sec() + 1;
574 }
575
576 void datafile_delete(
577 struct rrdengine_instance *ctx,
578 struct rrdengine_datafile *datafile,
579 bool update_retention,
580 bool disk_time,
581 bool worker);
582
583 // --------------------------------------------------------------------------------------------------------------------
584 // the following functions are used to sort UUIDs in the journal files
585 // DO NOT CHANGE, as this will break backwards compatibility with the data files users have.
586
587 static inline int journal_uuid_memcmp(const nd_uuid_t *uu1, const nd_uuid_t *uu2) {
588 return memcmp(uu1, uu2, sizeof(nd_uuid_t));
589 }
590
591 static inline int journal_metric_uuid_compare(const void *key, const void *metric) {
592 return journal_uuid_memcmp((const nd_uuid_t *)key, (const nd_uuid_t *)&(((struct journal_metric_list *) metric)->uuid));
593 }
594
595 // --------------------------------------------------------------------------------------------------------------------
596 uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx, bool having_lock);
597 void rrdeng_calculate_tier_disk_space_percentage(void);
598 uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx);
599 void dbengine_shutdown();
600 size_t datafile_count(struct rrdengine_instance *ctx, bool with_lock);
601 struct rrdengine_datafile *get_first_ctx_datafile(struct rrdengine_instance *ctx, bool with_lock);
602 struct rrdengine_datafile *get_last_ctx_datafile(struct rrdengine_instance *ctx, bool with_lock);
603 struct rrdengine_datafile *
604 get_next_datafile(struct rrdengine_datafile *this_datafile, struct rrdengine_instance *ctx, bool with_lock);
605
606
607 #endif /* NETDATA_RRDENGINE_H */