master
c 485 lines 17.4 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "rrddim_mem.h"
4 #include "Judy.h"
5
6 static Pvoid_t rrddim_Judy_array = NULL;
7 static netdata_rwlock_t rrddim_Judy_rwlock;
8
9 static void __attribute__((constructor)) init_lock(void) {
10 netdata_rwlock_init(&rrddim_Judy_rwlock);
11 }
12
13 static void __attribute__((destructor)) destroy_lock(void) {
14 netdata_rwlock_destroy(&rrddim_Judy_rwlock);
15 }
16
17 // ----------------------------------------------------------------------------
18 // metrics groups
19
20 STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si __maybe_unused, nd_uuid_t *uuid __maybe_unused) {
21 return NULL;
22 }
23
24 void rrddim_metrics_group_release(STORAGE_INSTANCE *si __maybe_unused, STORAGE_METRICS_GROUP *smg __maybe_unused) {
25 // if(!smg) return; // smg may be NULL
26 ;
27 }
28
29 // ----------------------------------------------------------------------------
30 // RRDDIM legacy data collection functions
31
32 struct mem_metric_handle {
33 RRDDIM *rd;
34
35 size_t counter;
36 size_t entries;
37 size_t current_entry;
38 time_t last_updated_s;
39 time_t update_every_s;
40
41 UUIDMAP_ID uuid_id; // stored locally so cleanup doesn't need rd
42 REFCOUNT refcount;
43 };
44
45 static void update_metric_handle_from_rrddim(struct mem_metric_handle *mh, RRDDIM *rd) {
46 mh->counter = rd->rrdset->counter;
47 mh->entries = rd->rrdset->db.entries;
48 mh->current_entry = rd->rrdset->db.current_entry;
49 mh->last_updated_s = rd->rrdset->last_updated.tv_sec;
50 mh->update_every_s = rd->rrdset->update_every;
51 }
52
53 static void check_metric_handle_from_rrddim(struct mem_metric_handle *mh) {
54 RRDDIM *rd = mh->rd; (void)rd;
55 internal_fatal(mh->entries != (size_t)rd->rrdset->db.entries, "RRDDIM: entries do not match");
56 internal_fatal(mh->update_every_s != rd->rrdset->update_every, "RRDDIM: update every does not match");
57 }
58
59 STORAGE_METRIC_HANDLE *rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si) {
60 struct mem_metric_handle *mh = (struct mem_metric_handle *)rrddim_metric_get_by_id(si, rd->uuid);
61 while(!mh) {
62 netdata_rwlock_wrlock(&rrddim_Judy_rwlock);
63 JudyAllocThreadPulseReset();
64 Pvoid_t *PValue = JudyLIns(&rrddim_Judy_array, rd->uuid, PJE0);
65 int64_t judy_mem = JudyAllocThreadPulseGetAndReset();
66 mh = *PValue;
67 if(!mh) {
68 mh = callocz(1, sizeof(struct mem_metric_handle));
69 mh->rd = rd;
70 mh->uuid_id = rd->uuid;
71 mh->refcount = 1;
72 update_metric_handle_from_rrddim(mh, rd);
73 *PValue = mh;
74 pulse_db_rrd_memory_change(judy_mem + (int64_t)sizeof(struct mem_metric_handle));
75 }
76 else {
77 if(!refcount_acquire(&mh->refcount))
78 mh = NULL;
79 }
80 netdata_rwlock_wrunlock(&rrddim_Judy_rwlock);
81 }
82
83 if(unlikely(mh->rd != rd)) {
84 // this can happen when the old RRDDIM is being deleted,
85 // but the dictionary has not yet run the destructors
86 netdata_rwlock_wrlock(&rrddim_Judy_rwlock);
87 mh->rd = rd;
88 netdata_rwlock_wrunlock(&rrddim_Judy_rwlock);
89 }
90
91 return (STORAGE_METRIC_HANDLE *)mh;
92 }
93
94 STORAGE_METRIC_HANDLE *rrddim_metric_get_by_id(STORAGE_INSTANCE *si __maybe_unused, UUIDMAP_ID id) {
95 struct mem_metric_handle *mh = NULL;
96
97 netdata_rwlock_rdlock(&rrddim_Judy_rwlock);
98 {
99 Pvoid_t *PValue = JudyLGet(rrddim_Judy_array, id, PJE0);
100 if (unlikely(PValue == PJERR))
101 fatal("DB_RAM_ALLOC: corrupted judy array!");
102
103 if (likely(NULL != PValue)) {
104 mh = *PValue;
105 if (!refcount_acquire(&mh->refcount))
106 mh = NULL;
107 }
108 }
109 netdata_rwlock_rdunlock(&rrddim_Judy_rwlock);
110
111 return (STORAGE_METRIC_HANDLE *)mh;
112 }
113
114 STORAGE_METRIC_HANDLE *rrddim_metric_get_by_uuid(STORAGE_INSTANCE *si, nd_uuid_t *uuid) {
115 UUIDMAP_ID id = uuidmap_create(*uuid);
116 STORAGE_METRIC_HANDLE *mh = rrddim_metric_get_by_id(si, id);
117 uuidmap_free(id);
118 return mh;
119 }
120
121 STORAGE_METRIC_HANDLE *rrddim_metric_dup(STORAGE_METRIC_HANDLE *smh) {
122 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
123
124 if(!refcount_acquire(&mh->refcount))
125 fatal("DB_RAM_ALLOC: cannot acquire an already acquired refcount");
126
127 return smh;
128 }
129
130 void rrddim_metric_release(STORAGE_METRIC_HANDLE *smh) {
131 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
132
133 if(refcount_release_and_acquire_for_deletion(&mh->refcount)) {
134 // we can delete it
135
136 int64_t judy_mem = 0;
137 netdata_rwlock_wrlock(&rrddim_Judy_rwlock);
138 {
139 JudyAllocThreadPulseReset();
140 JudyLDel(&rrddim_Judy_array, mh->uuid_id, PJE0);
141 judy_mem = JudyAllocThreadPulseGetAndReset();
142 }
143 netdata_rwlock_wrunlock(&rrddim_Judy_rwlock);
144
145 freez(mh);
146 pulse_db_rrd_memory_change(judy_mem - (int64_t)sizeof(struct mem_metric_handle));
147 }
148 }
149
150 bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *si __maybe_unused, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s) {
151 STORAGE_METRIC_HANDLE *smh = rrddim_metric_get_by_uuid(si, uuid);
152 if(!smh)
153 return false;
154
155 *first_entry_s = rrddim_query_oldest_time_s(smh);
156 *last_entry_s = rrddim_query_latest_time_s(smh);
157 rrddim_metric_release(smh);
158
159 return true;
160 }
161
162 bool rrddim_metric_retention_by_id(STORAGE_INSTANCE *si __maybe_unused, UUIDMAP_ID id, time_t *first_entry_s, time_t *last_entry_s) {
163 STORAGE_METRIC_HANDLE *smh = rrddim_metric_get_by_id(si, id);
164 if(!smh)
165 return false;
166
167 *first_entry_s = rrddim_query_oldest_time_s(smh);
168 *last_entry_s = rrddim_query_latest_time_s(smh);
169 rrddim_metric_release(smh);
170
171 return true;
172 }
173
174 void rrddim_retention_delete_by_id(STORAGE_INSTANCE *si __maybe_unused, UUIDMAP_ID id __maybe_unused) {
175 ;
176 }
177
178 void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every) {
179 struct mem_collect_handle *ch = (struct mem_collect_handle *)sch;
180 struct mem_metric_handle *mh = (struct mem_metric_handle *)ch->smh;
181
182 rrddim_store_metric_flush(sch);
183 mh->update_every_s = update_every;
184 }
185
186 STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every __maybe_unused, STORAGE_METRICS_GROUP *smg __maybe_unused) {
187 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
188 RRDDIM *rd = mh->rd;
189
190 update_metric_handle_from_rrddim(mh, rd);
191 internal_fatal((uint32_t)mh->update_every_s != update_every, "RRDDIM: update requested does not match the dimension");
192
193 struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
194 ch->common.seb = STORAGE_ENGINE_BACKEND_RRDDIM;
195 ch->rd = rd;
196 ch->smh = smh;
197
198 pulse_db_rrd_memory_add(sizeof(struct mem_collect_handle));
199
200 return (STORAGE_COLLECT_HANDLE *)ch;
201 }
202
203 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *sch) {
204 struct mem_collect_handle *ch = (struct mem_collect_handle *)sch;
205 struct mem_metric_handle *mh = (struct mem_metric_handle *)ch->smh;
206
207 RRDDIM *rd = mh->rd;
208 size_t entries = mh->entries;
209 storage_number empty = pack_storage_number(NAN, SN_FLAG_NONE);
210
211 for(size_t i = 0; i < entries ;i++)
212 rd->db.data[i] = empty;
213
214 mh->counter = 0;
215 mh->last_updated_s = 0;
216 mh->current_entry = 0;
217 }
218
219 static inline void rrddim_fill_the_gap(STORAGE_COLLECT_HANDLE *sch, time_t now_collect_s) {
220 struct mem_collect_handle *ch = (struct mem_collect_handle *)sch;
221 struct mem_metric_handle *mh = (struct mem_metric_handle *)ch->smh;
222
223 RRDDIM *rd = mh->rd;
224
225 internal_fatal(ch->rd != mh->rd, "RRDDIM: dimensions do not match");
226 check_metric_handle_from_rrddim(mh);
227
228 size_t entries = mh->entries;
229 time_t update_every_s = mh->update_every_s;
230 time_t last_stored_s = mh->last_updated_s;
231 size_t gap_entries = (now_collect_s - last_stored_s) / update_every_s;
232 if(gap_entries >= entries)
233 rrddim_store_metric_flush(sch);
234
235 else {
236 storage_number empty = pack_storage_number(NAN, SN_FLAG_NONE);
237 size_t current_entry = mh->current_entry;
238 time_t now_store_s = last_stored_s + update_every_s;
239
240 // fill the dimension
241 size_t c;
242 for(c = 0; c < entries && now_store_s <= now_collect_s ; now_store_s += update_every_s, c++) {
243 rd->db.data[current_entry++] = empty;
244
245 if(unlikely(current_entry >= entries))
246 current_entry = 0;
247 }
248 mh->counter += c;
249 mh->current_entry = current_entry;
250 mh->last_updated_s = now_store_s;
251 }
252 }
253
254 void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *sch,
255 usec_t point_in_time_ut,
256 NETDATA_DOUBLE n,
257 NETDATA_DOUBLE min_value __maybe_unused,
258 NETDATA_DOUBLE max_value __maybe_unused,
259 uint16_t count __maybe_unused,
260 uint16_t anomaly_count __maybe_unused,
261 SN_FLAGS flags)
262 {
263 struct mem_collect_handle *ch = (struct mem_collect_handle *)sch;
264 struct mem_metric_handle *mh = (struct mem_metric_handle *)ch->smh;
265
266 RRDDIM *rd = ch->rd;
267 time_t point_in_time_s = (time_t)(point_in_time_ut / USEC_PER_SEC);
268
269 internal_fatal(ch->rd != mh->rd, "RRDDIM: dimensions do not match");
270 check_metric_handle_from_rrddim(mh);
271
272 if(unlikely(point_in_time_s <= mh->last_updated_s))
273 return;
274
275 if(unlikely(mh->last_updated_s && point_in_time_s - mh->update_every_s > mh->last_updated_s))
276 rrddim_fill_the_gap(sch, point_in_time_s);
277
278 rd->db.data[mh->current_entry] = pack_storage_number(n, flags);
279 mh->counter++;
280 mh->current_entry = (mh->current_entry + 1) >= mh->entries ? 0 : mh->current_entry + 1;
281 mh->last_updated_s = point_in_time_s;
282 }
283
284 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *sch) {
285 freez(sch);
286 pulse_db_rrd_memory_sub(sizeof(struct mem_collect_handle));
287 return 0;
288 }
289
290 // ----------------------------------------------------------------------------
291
292 // get the total duration in seconds of the round-robin database
293 #define metric_duration(mh) (( (time_t)(mh)->counter >= (time_t)(mh)->entries ? (time_t)(mh)->entries : (time_t)(mh)->counter ) * (time_t)(mh)->update_every_s)
294
295 // get the last slot updated in the round-robin database
296 #define rrddim_last_slot(mh) ((size_t)(((mh)->current_entry == 0) ? (mh)->entries - 1 : (mh)->current_entry - 1))
297
298 // return the slot that has the oldest value
299 #define rrddim_first_slot(mh) ((size_t)((mh)->counter >= (size_t)(mh)->entries ? (mh)->current_entry : 0))
300
301 // get the slot of the round-robin database, for the given timestamp (t)
302 // it always returns a valid slot, although it may not be for the time requested if the time is outside the round-robin database
303 // only valid when not using dbengine
304 static inline size_t rrddim_time2slot(STORAGE_METRIC_HANDLE *smh, time_t t) {
305 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
306 RRDDIM *rd = mh->rd;
307
308 size_t ret = 0;
309 time_t last_entry_s = rrddim_query_latest_time_s(smh);
310 time_t first_entry_s = rrddim_query_oldest_time_s(smh);
311 size_t entries = mh->entries;
312 size_t first_slot = rrddim_first_slot(mh);
313 size_t last_slot = rrddim_last_slot(mh);
314 size_t update_every = mh->update_every_s;
315
316 if(t >= last_entry_s) {
317 // the requested time is after the last entry we have
318 ret = last_slot;
319 }
320 else {
321 if(t <= first_entry_s) {
322 // the requested time is before the first entry we have
323 ret = first_slot;
324 }
325 else {
326 if(last_slot >= (size_t)((last_entry_s - t) / update_every))
327 ret = last_slot - ((last_entry_s - t) / update_every);
328 else
329 ret = last_slot - ((last_entry_s - t) / update_every) + entries;
330 }
331 }
332
333 if(unlikely(ret >= entries)) {
334 netdata_log_error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rrddim_name(rd));
335 ret = entries - 1;
336 }
337
338 return ret;
339 }
340
341 // get the timestamp of a specific slot in the round-robin database
342 // only valid when not using dbengine
343 static inline time_t rrddim_slot2time(STORAGE_METRIC_HANDLE *smh, size_t slot) {
344 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
345 RRDDIM *rd = mh->rd;
346
347 time_t ret;
348 time_t last_entry_s = rrddim_query_latest_time_s(smh);
349 time_t first_entry_s = rrddim_query_oldest_time_s(smh);
350 size_t entries = mh->entries;
351 size_t last_slot = rrddim_last_slot(mh);
352 size_t update_every = mh->update_every_s;
353
354 if(slot >= entries) {
355 netdata_log_error("INTERNAL ERROR: caller of rrddim_slot2time() gives invalid slot %zu", slot);
356 slot = entries - 1;
357 }
358
359 if(slot > last_slot)
360 ret = last_entry_s - (time_t)(update_every * (last_slot - slot + entries));
361 else
362 ret = last_entry_s - (time_t)(update_every * (last_slot - slot));
363
364 if(unlikely(ret < first_entry_s)) {
365 netdata_log_error("INTERNAL ERROR: rrddim_slot2time() on dimension '%s' of chart '%s' returned time (%ld) too far in the past (before first_entry_s %ld) for slot %zu",
366 rrddim_name(rd), rrdset_id(rd->rrdset), ret, first_entry_s, slot);
367
368 ret = first_entry_s;
369 }
370
371 if(unlikely(ret > last_entry_s)) {
372 netdata_log_error("INTERNAL ERROR: rrddim_slot2time() on dimension '%s' of chart '%s' returned time (%ld) too far into the future (after last_entry_s %ld) for slot %zu",
373 rrddim_name(rd), rrdset_id(rd->rrdset), ret, last_entry_s, slot);
374
375 ret = last_entry_s;
376 }
377
378 return ret;
379 }
380
381 // ----------------------------------------------------------------------------
382 // RRDDIM legacy database query functions
383
384 void rrddim_query_init(STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh, time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority __maybe_unused) {
385 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
386
387 check_metric_handle_from_rrddim(mh);
388
389 seqh->start_time_s = start_time_s;
390 seqh->end_time_s = end_time_s;
391 seqh->priority = priority;
392 seqh->seb = STORAGE_ENGINE_BACKEND_RRDDIM;
393 struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
394 h->smh = smh;
395
396 h->slot = rrddim_time2slot(smh, start_time_s);
397 h->last_slot = rrddim_time2slot(smh, end_time_s);
398 h->dt = mh->update_every_s;
399
400 h->next_timestamp = start_time_s;
401 h->slot_timestamp = rrddim_slot2time(smh, h->slot);
402 h->last_timestamp = rrddim_slot2time(smh, h->last_slot);
403
404 // netdata_log_info("RRDDIM QUERY INIT: start %ld, end %ld, next %ld, first %ld, last %ld, dt %ld", start_time, end_time, h->next_timestamp, h->slot_timestamp, h->last_timestamp, h->dt);
405
406 pulse_db_rrd_memory_add(sizeof(struct mem_query_handle));
407 seqh->handle = (STORAGE_QUERY_HANDLE *)h;
408 }
409
410 // Returns the metric and sets its timestamp into current_time
411 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
412 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
413 ALWAYS_INLINE STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh) {
414 struct mem_query_handle* h = (struct mem_query_handle*)seqh->handle;
415 struct mem_metric_handle *mh = (struct mem_metric_handle *)h->smh;
416 RRDDIM *rd = mh->rd;
417
418 size_t entries = mh->entries;
419 size_t slot = h->slot;
420
421 STORAGE_POINT sp;
422 sp.count = 1;
423
424 time_t this_timestamp = h->next_timestamp;
425 h->next_timestamp += h->dt;
426
427 // set this timestamp for our caller
428 sp.start_time_s = this_timestamp - h->dt;
429 sp.end_time_s = this_timestamp;
430
431 if(unlikely(this_timestamp < h->slot_timestamp)) {
432 storage_point_empty(sp, sp.start_time_s, sp.end_time_s);
433 return sp;
434 }
435
436 if(unlikely(this_timestamp > h->last_timestamp)) {
437 storage_point_empty(sp, sp.start_time_s, sp.end_time_s);
438 return sp;
439 }
440
441 storage_number n = rd->db.data[slot++];
442 if(unlikely(slot >= entries)) slot = 0;
443
444 h->slot = slot;
445 h->slot_timestamp += h->dt;
446
447 sp.anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
448 sp.flags = (n & SN_USER_FLAGS);
449 sp.min = sp.max = sp.sum = unpack_storage_number(n);
450
451 return sp;
452 }
453
454 int rrddim_query_is_finished(struct storage_engine_query_handle *seqh) {
455 struct mem_query_handle *h = (struct mem_query_handle*)seqh->handle;
456 return (h->next_timestamp > seqh->end_time_s);
457 }
458
459 void rrddim_query_finalize(struct storage_engine_query_handle *seqh) {
460 #ifdef NETDATA_INTERNAL_CHECKS
461 struct mem_query_handle *h = (struct mem_query_handle*)seqh->handle;
462 struct mem_metric_handle *mh = (struct mem_metric_handle *)h->smh;
463
464 internal_error(!rrddim_query_is_finished(seqh),
465 "QUERY: query for chart '%s' dimension '%s' has been stopped unfinished",
466 rrdset_id(mh->rd->rrdset), rrddim_name(mh->rd));
467
468 #endif
469 freez(seqh->handle);
470 pulse_db_rrd_memory_sub(sizeof(struct mem_query_handle));
471 }
472
473 time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *seqh) {
474 return seqh->end_time_s;
475 }
476
477 time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *smh) {
478 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
479 return mh->last_updated_s;
480 }
481
482 time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *smh) {
483 struct mem_metric_handle *mh = (struct mem_metric_handle *)smh;
484 return (time_t)(mh->last_updated_s - metric_duration(mh));
485 }