@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
h 258 lines 9.05 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef NETDATA_RRDENGINEAPI_H
4 #define NETDATA_RRDENGINEAPI_H
5
6 #include "rrdengine.h"
7
8 #define RRDENG_MIN_PAGE_CACHE_SIZE_MB (8)
9 #define RRDENG_MIN_DISK_SPACE_MB (25)
10 #define RRDENG_DEFAULT_TIER_DISK_SPACE_MB (1024)
11
12 #define RRDENG_NR_STATS (38)
13
14 #define RRDENG_FD_BUDGET_PER_INSTANCE (50)
15
16 extern uint64_t dbengine_out_of_memory_protection;
17 extern bool dbengine_use_all_ram_for_caches;
18
19 extern int default_rrdeng_page_cache_mb;
20 extern int default_rrdeng_extent_cache_mb;
21 extern int db_engine_journal_check;
22 extern int default_rrdeng_disk_quota_mb;
23 extern int default_multidb_disk_quota_mb;
24 extern bool new_dbengine_defaults;
25 extern bool legacy_multihost_db_space;
26 extern RRD_BACKFILL default_backfill;
27
28 extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
29 extern size_t page_type_size[];
30 extern size_t tier_page_size[];
31 extern size_t tier_quota_mb[];
32 extern uint8_t tier_page_type[];
33
34 #define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type]
35
36 STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si);
37 STORAGE_METRIC_HANDLE *rrdeng_metric_get_by_id(STORAGE_INSTANCE *si, UUIDMAP_ID id);
38 STORAGE_METRIC_HANDLE *rrdeng_metric_get_by_uuid(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
39 void rrdeng_metric_release(STORAGE_METRIC_HANDLE *smh);
40 STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *smh);
41
42 STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
43 void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *sch);
44 void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);
45 void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut, NETDATA_DOUBLE n,
46 NETDATA_DOUBLE min_value,
47 NETDATA_DOUBLE max_value,
48 uint16_t count,
49 uint16_t anomaly_count,
50 SN_FLAGS flags);
51 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *sch);
52
53 void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,
54 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
55 STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh);
56
57
58 int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh);
59 void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh);
60 time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh);
61 time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh);
62 time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh);
63
64 void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
65
66 /* must call once before using anything */
67 int rrdeng_init(
68 struct rrdengine_instance **ctxp,
69 const char *dbfiles_path,
70 unsigned disk_space_mb,
71 size_t tier,
72 time_t max_retention_s);
73
74 void rrdeng_readiness_wait(struct rrdengine_instance *ctx);
75
76 int rrdeng_exit(struct rrdengine_instance *ctx);
77 void rrdeng_quiesce(struct rrdengine_instance *ctx);
78 void rrdeng_flush_dirty(struct rrdengine_instance *ctx);
79 void rrdeng_flush_all(struct rrdengine_instance *ctx);
80
81
82 bool rrdeng_metric_retention_by_id(STORAGE_INSTANCE *si, UUIDMAP_ID id, time_t *first_entry_s, time_t *last_entry_s);
83 bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, nd_uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s);
84 void rrdeng_metric_retention_delete_by_id(STORAGE_INSTANCE *si, UUIDMAP_ID id);
85
86 extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);
87 extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);
88
89 typedef struct rrdengine_size_statistics {
90 size_t default_granularity_secs;
91
92 size_t sizeof_datafile;
93 size_t sizeof_page_in_cache;
94 size_t sizeof_point_data;
95 size_t sizeof_page_data;
96
97 size_t pages_per_extent;
98
99 size_t datafiles;
100 size_t extents;
101 size_t extents_pages;
102 size_t points;
103 size_t metrics;
104 size_t metrics_pages;
105
106 size_t extents_compressed_bytes;
107 size_t pages_uncompressed_bytes;
108 time_t pages_duration_secs;
109
110 struct {
111 size_t pages;
112 size_t pages_uncompressed_bytes;
113 time_t pages_duration_secs;
114 size_t points;
115 } page_types[256];
116
117 size_t single_point_pages;
118
119 time_t first_time_s;
120 time_t last_time_s;
121
122 size_t currently_collected_metrics;
123 size_t estimated_concurrently_collected_metrics;
124
125 size_t disk_space;
126 size_t max_disk_space;
127
128 time_t database_retention_secs;
129 double average_compression_savings;
130 double average_point_duration_secs;
131 double average_metric_retention_secs;
132
133 double ephemeral_metrics_per_day_percent;
134
135 double average_page_size_bytes;
136 } RRDENG_SIZE_STATS;
137
138 struct time_and_count {
139 size_t count;
140 usec_t usec;
141 };
142
143 static ALWAYS_INLINE void time_and_count_add(struct time_and_count *tc, usec_t dt) {
144 __atomic_add_fetch(&tc->count, 1, __ATOMIC_RELAXED);
145 __atomic_add_fetch(&tc->usec, dt, __ATOMIC_RELAXED);
146 }
147
148 struct rrdeng_cache_efficiency_stats {
149 PAD64(size_t) queries_planned_with_gaps;
150 PAD64(size_t) queries_executed_with_gaps;
151
152 PAD64(size_t) currently_running_queries;
153
154 // query planner output of the queries
155 PAD64(size_t) pages_total;
156 PAD64(size_t) pages_to_load_from_disk;
157 PAD64(size_t) extents_loaded_from_disk;
158
159 // pages metadata sources
160 PAD64(size_t) pages_meta_source_main_cache;
161 PAD64(size_t) pages_meta_source_open_cache;
162 PAD64(size_t) pages_meta_source_journal_v2;
163
164 // preloading
165 PAD64(size_t) page_next_wait_failed;
166 PAD64(size_t) page_next_wait_loaded;
167 PAD64(size_t) page_next_nowait_failed;
168 PAD64(size_t) page_next_nowait_loaded;
169
170 // pages data sources
171 PAD64(size_t) pages_data_source_main_cache;
172 PAD64(size_t) pages_data_source_main_cache_at_pass4;
173 PAD64(size_t) pages_data_source_disk;
174 PAD64(size_t) pages_data_source_extent_cache; // loaded by a cached extent
175
176 // cache hits at different points
177 PAD64(size_t) pages_load_ok_loaded_but_cache_hit_while_inserting; // found in cache while inserting it (conflict)
178
179 // loading
180 PAD64(size_t) pages_load_extent_merged;
181 PAD64(size_t) pages_load_ok_uncompressed;
182 PAD64(size_t) pages_load_ok_compressed;
183 PAD64(size_t) pages_load_fail_invalid_page_in_extent;
184 PAD64(size_t) pages_load_fail_cant_mmap_extent;
185 PAD64(size_t) pages_load_fail_datafile_not_available;
186 PAD64(size_t) pages_load_fail_unroutable;
187 PAD64(size_t) pages_load_fail_not_found;
188 PAD64(size_t) pages_load_fail_invalid_extent;
189 PAD64(size_t) pages_load_fail_cancelled;
190
191 // count of queries and times spent in them
192 PAD64(struct time_and_count) prep_time_to_route_sync;
193 PAD64(struct time_and_count) prep_time_to_route_syncfirst;
194 PAD64(struct time_and_count) prep_time_to_route_async;
195 PAD64(struct time_and_count) prep_time_in_main_cache_lookup;
196 PAD64(struct time_and_count) prep_time_in_open_cache_lookup;
197 PAD64(struct time_and_count) prep_time_in_journal_v2_lookup;
198 PAD64(struct time_and_count) prep_time_in_pass4_lookup;
199
200 // timings the query thread experiences
201 PAD64(struct time_and_count) query_time_init;
202 PAD64(struct time_and_count) query_time_wait_for_prep;
203 PAD64(struct time_and_count) query_time_to_slow_disk_next_page;
204 PAD64(struct time_and_count) query_time_to_fast_disk_next_page;
205 PAD64(struct time_and_count) query_time_to_slow_preload_next_page;
206 PAD64(struct time_and_count) query_time_to_fast_preload_next_page;
207
208 // query issues
209 PAD64(size_t) pages_zero_time_skipped;
210 PAD64(size_t) pages_past_time_skipped;
211 PAD64(size_t) pages_overlapping_skipped;
212 PAD64(size_t) pages_invalid_size_skipped;
213 PAD64(size_t) pages_invalid_update_every_fixed;
214 PAD64(size_t) pages_invalid_entries_fixed;
215
216 // database events
217 PAD64(size_t) journal_v2_mapped;
218 PAD64(size_t) journal_v2_unmapped;
219 PAD64(size_t) datafile_creation_started;
220 PAD64(size_t) datafile_deletion_started;
221 PAD64(size_t) datafile_deletion_spin;
222 PAD64(size_t) journal_v2_indexing_started;
223 PAD64(size_t) metrics_retention_started;
224 };
225
226 typedef enum rrdeng_mem {
227 RRDENG_MEM_PGC = 0,
228 RRDENG_MEM_PGD,
229 RRDENG_MEM_MRG,
230 RRDENG_MEM_OPCODES,
231 RRDENG_MEM_HANDLES,
232 RRDENG_MEM_DESCRIPTORS,
233 RRDENG_MEM_WORKERS,
234 RRDENG_MEM_PDC,
235 RRDENG_MEM_XT_IO,
236 RRDENG_MEM_EPDL,
237 RRDENG_MEM_DEOL,
238 RRDENG_MEM_PD,
239 RRDENG_MEM_EPDL_EXTENT,
240
241 // terminator
242 RRDENG_MEM_MAX,
243 } RRDENG_MEM;
244
245 struct rrdeng_buffer_sizes {
246 struct aral_statistics *as[RRDENG_MEM_MAX];
247
248 size_t wal;
249 size_t xt_buf;
250 };
251
252 struct rrdeng_buffer_sizes rrdeng_pulse_memory_sizes(void);
253 struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void);
254
255 RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
256 size_t rrdeng_collectors_running(struct rrdengine_instance *ctx);
257
258 #endif /* NETDATA_RRDENGINEAPI_H */