@cryptotaxi247 / netdata-1
Code Commits Issues Pulls Sessions CI/CD
master
master
c 1,698 lines 86.6 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #define PULSE_INTERNALS 1
4 #include "pulse-db-dbengine.h"
5
6 int64_t pulse_dbengine_total_memory = 0;
7
8 #if defined(ENABLE_DBENGINE)
9
10 static usec_t time_and_count_delta_average(struct time_and_count *prev, struct time_and_count *latest) {
11 if(latest->count > prev->count && latest->usec > prev->usec)
12 return (latest->usec - prev->usec) / (latest->count - prev->count);
13
14 return 0;
15 }
16
17 struct dbengine2_cache_pointers {
18 RRDSET *st_cache_hit_ratio;
19 RRDDIM *rd_hit_ratio_closest;
20 RRDDIM *rd_hit_ratio_exact;
21
22 RRDSET *st_operations;
23 RRDDIM *rd_searches_closest;
24 RRDDIM *rd_searches_exact;
25 RRDDIM *rd_add_hot;
26 RRDDIM *rd_add_clean;
27 RRDDIM *rd_evictions;
28 RRDDIM *rd_flushes;
29 RRDDIM *rd_acquires;
30 RRDDIM *rd_releases;
31 RRDDIM *rd_acquires_for_deletion;
32
33 RRDSET *st_pgc_memory;
34 RRDDIM *rd_pgc_memory_free;
35 RRDDIM *rd_pgc_memory_clean;
36 RRDDIM *rd_pgc_memory_hot;
37 RRDDIM *rd_pgc_memory_dirty;
38 RRDDIM *rd_pgc_memory_index;
39 RRDDIM *rd_pgc_memory_evicting;
40 RRDDIM *rd_pgc_memory_flushing;
41
42 struct {
43 RRDSET *st_pgc_page_size_heatmap;
44 RRDDIM *rd_pgc_page_size_x[PGC_SIZE_HISTOGRAM_ENTRIES];
45 } queues[3];
46
47 RRDSET *st_pgc_tm;
48 RRDDIM *rd_pgc_tm_current;
49 RRDDIM *rd_pgc_tm_wanted;
50 RRDDIM *rd_pgc_tm_hot_max;
51 RRDDIM *rd_pgc_tm_dirty_max;
52 RRDDIM *rd_pgc_tm_hot;
53 RRDDIM *rd_pgc_tm_dirty;
54 RRDDIM *rd_pgc_tm_referenced;
55
56 RRDSET *st_pgc_pages;
57 RRDDIM *rd_pgc_pages_clean;
58 RRDDIM *rd_pgc_pages_hot;
59 RRDDIM *rd_pgc_pages_dirty;
60 RRDDIM *rd_pgc_pages_referenced;
61
62 RRDSET *st_pgc_memory_changes;
63 RRDDIM *rd_pgc_memory_new_hot;
64 RRDDIM *rd_pgc_memory_new_clean;
65 RRDDIM *rd_pgc_memory_clean_evictions;
66
67 RRDSET *st_pgc_memory_migrations;
68 RRDDIM *rd_pgc_memory_hot_to_dirty;
69 RRDDIM *rd_pgc_memory_dirty_to_clean;
70
71 RRDSET *st_pgc_workers;
72 RRDDIM *rd_pgc_workers_evictors;
73 RRDDIM *rd_pgc_workers_flushers;
74 RRDDIM *rd_pgc_workers_adders;
75 RRDDIM *rd_pgc_workers_searchers;
76 RRDDIM *rd_pgc_workers_jv2_flushers;
77 RRDDIM *rd_pgc_workers_hot2dirty;
78
79 RRDSET *st_pgc_memory_events;
80 RRDDIM *rd_pgc_memory_evictions_critical;
81 RRDDIM *rd_pgc_memory_evictions_aggressive;
82 RRDDIM *rd_pgc_memory_flushes_critical;
83 RRDDIM *rd_pgc_waste_evict_thread_signals;
84 RRDDIM *rd_pgc_waste_evict_inline_on_add;
85 RRDDIM *rd_pgc_waste_evict_inline_on_release;
86 RRDDIM *rd_pgc_waste_flush_inline_on_add;
87 RRDDIM *rd_pgc_waste_flush_inline_on_release;
88
89 RRDSET *st_pgc_waste;
90 RRDDIM *rd_pgc_waste_evict_relocated;
91 RRDDIM *rd_pgc_waste_flushes_cancelled;
92 RRDDIM *rd_pgc_waste_insert_spins;
93 RRDDIM *rd_pgc_waste_evict_spins;
94 };
95
96 static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *ptrs, struct pgc_statistics *pgc_stats, struct pgc_statistics *pgc_stats_old __maybe_unused, const char *name, int priority) {
97
98 {
99 if (unlikely(!ptrs->st_cache_hit_ratio)) {
100 BUFFER *id = buffer_create(100, NULL);
101 buffer_sprintf(id, "dbengine_%s_cache_hit_ratio", name);
102
103 BUFFER *family = buffer_create(100, NULL);
104 buffer_sprintf(family, "dbengine %s cache", name);
105
106 BUFFER *title = buffer_create(100, NULL);
107 buffer_sprintf(title, "Netdata %s Cache Hit Ratio", name);
108
109 ptrs->st_cache_hit_ratio = rrdset_create_localhost(
110 "netdata",
111 buffer_tostring(id),
112 NULL,
113 buffer_tostring(family),
114 NULL,
115 buffer_tostring(title),
116 "%",
117 "netdata",
118 "pulse",
119 priority,
120 localhost->rrd_update_every,
121 RRDSET_TYPE_LINE);
122
123 ptrs->rd_hit_ratio_closest = rrddim_add(ptrs->st_cache_hit_ratio, "closest", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
124 ptrs->rd_hit_ratio_exact = rrddim_add(ptrs->st_cache_hit_ratio, "exact", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
125
126 buffer_free(id);
127 buffer_free(family);
128 buffer_free(title);
129 priority++;
130 }
131
132 size_t closest_percent = 100 * 10000;
133 if(pgc_stats->searches_closest > pgc_stats_old->searches_closest)
134 closest_percent = (pgc_stats->searches_closest_hits - pgc_stats_old->searches_closest_hits) * 100 * 10000 / (pgc_stats->searches_closest - pgc_stats_old->searches_closest);
135
136 size_t exact_percent = 100 * 10000;
137 if(pgc_stats->searches_exact > pgc_stats_old->searches_exact)
138 exact_percent = (pgc_stats->searches_exact_hits - pgc_stats_old->searches_exact_hits) * 100 * 10000 / (pgc_stats->searches_exact - pgc_stats_old->searches_exact);
139
140 rrddim_set_by_pointer(ptrs->st_cache_hit_ratio, ptrs->rd_hit_ratio_closest, (collected_number)closest_percent);
141 rrddim_set_by_pointer(ptrs->st_cache_hit_ratio, ptrs->rd_hit_ratio_exact, (collected_number)exact_percent);
142
143 rrdset_done(ptrs->st_cache_hit_ratio);
144 }
145
146 {
147 if (unlikely(!ptrs->st_operations)) {
148 BUFFER *id = buffer_create(100, NULL);
149 buffer_sprintf(id, "dbengine_%s_cache_operations", name);
150
151 BUFFER *family = buffer_create(100, NULL);
152 buffer_sprintf(family, "dbengine %s cache", name);
153
154 BUFFER *title = buffer_create(100, NULL);
155 buffer_sprintf(title, "Netdata %s Cache Operations", name);
156
157 ptrs->st_operations = rrdset_create_localhost(
158 "netdata",
159 buffer_tostring(id),
160 NULL,
161 buffer_tostring(family),
162 NULL,
163 buffer_tostring(title),
164 "ops/s",
165 "netdata",
166 "pulse",
167 priority,
168 localhost->rrd_update_every,
169 RRDSET_TYPE_LINE);
170
171 ptrs->rd_searches_closest = rrddim_add(ptrs->st_operations, "search closest", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
172 ptrs->rd_searches_exact = rrddim_add(ptrs->st_operations, "search exact", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
173 ptrs->rd_add_hot = rrddim_add(ptrs->st_operations, "add hot", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
174 ptrs->rd_add_clean = rrddim_add(ptrs->st_operations, "add clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
175 ptrs->rd_evictions = rrddim_add(ptrs->st_operations, "evictions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
176 ptrs->rd_flushes = rrddim_add(ptrs->st_operations, "flushes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
177 ptrs->rd_acquires = rrddim_add(ptrs->st_operations, "acquires", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
178 ptrs->rd_releases = rrddim_add(ptrs->st_operations, "releases", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
179 ptrs->rd_acquires_for_deletion = rrddim_add(ptrs->st_operations, "del acquires", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
180
181 buffer_free(id);
182 buffer_free(family);
183 buffer_free(title);
184 priority++;
185 }
186
187 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_searches_closest, (collected_number)pgc_stats->searches_closest);
188 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_searches_exact, (collected_number)pgc_stats->searches_exact);
189 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_add_hot, (collected_number)pgc_stats->queues[PGC_QUEUE_HOT].added_entries);
190 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_add_clean, (collected_number)(pgc_stats->added_entries - pgc_stats->queues[PGC_QUEUE_HOT].added_entries));
191 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_evictions, (collected_number)pgc_stats->queues[PGC_QUEUE_CLEAN].removed_entries);
192 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_flushes, (collected_number)pgc_stats->flushes_completed);
193 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_acquires, (collected_number)pgc_stats->acquires);
194 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_releases, (collected_number)pgc_stats->releases);
195 rrddim_set_by_pointer(ptrs->st_operations, ptrs->rd_acquires_for_deletion, (collected_number)pgc_stats->acquires_for_deletion);
196
197 rrdset_done(ptrs->st_operations);
198 }
199
200 {
201 if (unlikely(!ptrs->st_pgc_memory)) {
202 BUFFER *id = buffer_create(100, NULL);
203 buffer_sprintf(id, "dbengine_%s_cache_memory", name);
204
205 BUFFER *family = buffer_create(100, NULL);
206 buffer_sprintf(family, "dbengine %s cache", name);
207
208 BUFFER *title = buffer_create(100, NULL);
209 buffer_sprintf(title, "Netdata %s Cache Memory", name);
210
211 ptrs->st_pgc_memory = rrdset_create_localhost(
212 "netdata",
213 buffer_tostring(id),
214 NULL,
215 buffer_tostring(family),
216 NULL,
217 buffer_tostring(title),
218 "bytes",
219 "netdata",
220 "pulse",
221 priority,
222 localhost->rrd_update_every,
223 RRDSET_TYPE_STACKED);
224
225 ptrs->rd_pgc_memory_free = rrddim_add(ptrs->st_pgc_memory, "free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
226 ptrs->rd_pgc_memory_hot = rrddim_add(ptrs->st_pgc_memory, "hot", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
227 ptrs->rd_pgc_memory_dirty = rrddim_add(ptrs->st_pgc_memory, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
228 ptrs->rd_pgc_memory_clean = rrddim_add(ptrs->st_pgc_memory, "clean", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
229 ptrs->rd_pgc_memory_index = rrddim_add(ptrs->st_pgc_memory, "index", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
230 ptrs->rd_pgc_memory_evicting = rrddim_add(ptrs->st_pgc_memory, "evicting", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
231 ptrs->rd_pgc_memory_flushing = rrddim_add(ptrs->st_pgc_memory, "flushing", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
232
233 buffer_free(id);
234 buffer_free(family);
235 buffer_free(title);
236 priority++;
237 }
238
239 collected_number free = (pgc_stats->current_cache_size > pgc_stats->wanted_cache_size) ? 0 :
240 (collected_number)(pgc_stats->wanted_cache_size - pgc_stats->current_cache_size);
241
242 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_free, free);
243 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_hot, (collected_number)pgc_stats->queues[PGC_QUEUE_HOT].size);
244 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_dirty, (collected_number)pgc_stats->queues[PGC_QUEUE_DIRTY].size);
245 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_clean, (collected_number)pgc_stats->queues[PGC_QUEUE_CLEAN].size);
246 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_evicting, (collected_number)pgc_stats->evicting_size);
247 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_flushing, (collected_number)pgc_stats->flushing_size);
248 rrddim_set_by_pointer(ptrs->st_pgc_memory, ptrs->rd_pgc_memory_index,(collected_number)(pgc_stats->size - pgc_stats->queues[PGC_QUEUE_CLEAN].size - pgc_stats->queues[PGC_QUEUE_HOT].size - pgc_stats->queues[PGC_QUEUE_DIRTY].size - pgc_stats->evicting_size - pgc_stats->flushing_size));
249
250 rrdset_done(ptrs->st_pgc_memory);
251 }
252
253 for(size_t q = 0; q < 3 ;q++) {
254 const char *queue;
255 switch(q) {
256 case PGC_QUEUE_HOT:
257 queue = "hot";
258 break;
259
260 case PGC_QUEUE_DIRTY:
261 queue = "dirty";
262 break;
263
264 default:
265 case PGC_QUEUE_CLEAN:
266 queue = "clean";
267 break;
268 }
269
270 if (unlikely(!ptrs->queues[q].st_pgc_page_size_heatmap)) {
271 CLEAN_BUFFER *ctx = buffer_create(100, NULL);
272 buffer_sprintf(ctx, "netdata.dbengine_%s_page_sizes", name);
273
274 CLEAN_BUFFER *id = buffer_create(100, NULL);
275 buffer_sprintf(id, "dbengine_%s_%s_page_sizes", name, queue);
276
277 CLEAN_BUFFER *family = buffer_create(100, NULL);
278 buffer_sprintf(family, "dbengine %s cache", name);
279
280 CLEAN_BUFFER *title = buffer_create(100, NULL);
281 buffer_sprintf(title, "Netdata %s Nominal Page Sizes (without overheads)", name);
282
283 ptrs->queues[q].st_pgc_page_size_heatmap = rrdset_create_localhost(
284 "netdata",
285 buffer_tostring(id),
286 NULL,
287 buffer_tostring(family),
288 buffer_tostring(ctx),
289 buffer_tostring(title),
290 "pages",
291 "netdata",
292 "pulse",
293 priority,
294 localhost->rrd_update_every,
295 RRDSET_TYPE_HEATMAP);
296
297 ptrs->queues[q].rd_pgc_page_size_x[0] = rrddim_add(ptrs->queues[q].st_pgc_page_size_heatmap, "empty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
298 for(size_t i = 1; i < _countof(ptrs->queues[q].rd_pgc_page_size_x) - 1 ;i++) {
299 char buf[64];
300 snprintfz(buf, sizeof(buf), "%zu", pgc_stats->queues[q].size_histogram.array[i].upto);
301 // size_snprintf(&buf[1], sizeof(buf) - 1, pgc_stats->size_histogram.array[i].upto, "B", true);
302 ptrs->queues[q].rd_pgc_page_size_x[i] = rrddim_add(ptrs->queues[q].st_pgc_page_size_heatmap, buf, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
303 }
304 ptrs->queues[q].rd_pgc_page_size_x[_countof(ptrs->queues[q].rd_pgc_page_size_x) - 1] = rrddim_add(ptrs->queues[q].st_pgc_page_size_heatmap, "+inf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
305
306 rrdlabels_add(ptrs->queues[q].st_pgc_page_size_heatmap->rrdlabels, "Cache", name, RRDLABEL_SRC_AUTO);
307 rrdlabels_add(ptrs->queues[q].st_pgc_page_size_heatmap->rrdlabels, "Queue", queue, RRDLABEL_SRC_AUTO);
308
309 priority++;
310 }
311
312 for(size_t i = 0; i < _countof(ptrs->queues[q].rd_pgc_page_size_x) - 1 ;i++)
313 rrddim_set_by_pointer(ptrs->queues[q].st_pgc_page_size_heatmap, ptrs->queues[q].rd_pgc_page_size_x[i], (collected_number)pgc_stats->queues[q].size_histogram.array[i].count);
314
315 rrdset_done(ptrs->queues[q].st_pgc_page_size_heatmap);
316 }
317
318 {
319 if (unlikely(!ptrs->st_pgc_tm)) {
320 BUFFER *id = buffer_create(100, NULL);
321 buffer_sprintf(id, "dbengine_%s_target_memory", name);
322
323 BUFFER *family = buffer_create(100, NULL);
324 buffer_sprintf(family, "dbengine %s cache", name);
325
326 BUFFER *title = buffer_create(100, NULL);
327 buffer_sprintf(title, "Netdata %s Target Cache Memory", name);
328
329 ptrs->st_pgc_tm = rrdset_create_localhost(
330 "netdata",
331 buffer_tostring(id),
332 NULL,
333 buffer_tostring(family),
334 NULL,
335 buffer_tostring(title),
336 "bytes",
337 "netdata",
338 "pulse",
339 priority,
340 localhost->rrd_update_every,
341 RRDSET_TYPE_LINE);
342
343 ptrs->rd_pgc_tm_current = rrddim_add(ptrs->st_pgc_tm, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
344 ptrs->rd_pgc_tm_wanted = rrddim_add(ptrs->st_pgc_tm, "wanted", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
345 ptrs->rd_pgc_tm_referenced = rrddim_add(ptrs->st_pgc_tm, "referenced", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
346 ptrs->rd_pgc_tm_hot_max = rrddim_add(ptrs->st_pgc_tm, "hot max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
347 ptrs->rd_pgc_tm_dirty_max = rrddim_add(ptrs->st_pgc_tm, "dirty max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
348 ptrs->rd_pgc_tm_hot = rrddim_add(ptrs->st_pgc_tm, "hot", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
349 ptrs->rd_pgc_tm_dirty = rrddim_add(ptrs->st_pgc_tm, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
350
351 buffer_free(id);
352 buffer_free(family);
353 buffer_free(title);
354 priority++;
355 }
356
357 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_current, (collected_number)pgc_stats->current_cache_size);
358 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_wanted, (collected_number)pgc_stats->wanted_cache_size);
359 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_referenced, (collected_number)pgc_stats->referenced_size);
360 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_hot_max, (collected_number)pgc_stats->queues[PGC_QUEUE_HOT].max_size);
361 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_dirty_max, (collected_number)pgc_stats->queues[PGC_QUEUE_DIRTY].max_size);
362 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_hot, (collected_number)pgc_stats->queues[PGC_QUEUE_HOT].size);
363 rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_dirty, (collected_number)pgc_stats->queues[PGC_QUEUE_DIRTY].size);
364
365 rrdset_done(ptrs->st_pgc_tm);
366 }
367
368 {
369 if (unlikely(!ptrs->st_pgc_pages)) {
370 BUFFER *id = buffer_create(100, NULL);
371 buffer_sprintf(id, "dbengine_%s_cache_pages", name);
372
373 BUFFER *family = buffer_create(100, NULL);
374 buffer_sprintf(family, "dbengine %s cache", name);
375
376 BUFFER *title = buffer_create(100, NULL);
377 buffer_sprintf(title, "Netdata %s Cache Pages", name);
378
379 ptrs->st_pgc_pages = rrdset_create_localhost(
380 "netdata",
381 buffer_tostring(id),
382 NULL,
383 buffer_tostring(family),
384 NULL,
385 buffer_tostring(title),
386 "pages",
387 "netdata",
388 "pulse",
389 priority,
390 localhost->rrd_update_every,
391 RRDSET_TYPE_LINE);
392
393 ptrs->rd_pgc_pages_clean = rrddim_add(ptrs->st_pgc_pages, "clean", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
394 ptrs->rd_pgc_pages_hot = rrddim_add(ptrs->st_pgc_pages, "hot", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
395 ptrs->rd_pgc_pages_dirty = rrddim_add(ptrs->st_pgc_pages, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
396 ptrs->rd_pgc_pages_referenced = rrddim_add(ptrs->st_pgc_pages, "referenced", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
397
398 buffer_free(id);
399 buffer_free(family);
400 buffer_free(title);
401 priority++;
402 }
403
404 rrddim_set_by_pointer(ptrs->st_pgc_pages, ptrs->rd_pgc_pages_clean, (collected_number)pgc_stats->queues[PGC_QUEUE_CLEAN].entries);
405 rrddim_set_by_pointer(ptrs->st_pgc_pages, ptrs->rd_pgc_pages_hot, (collected_number)pgc_stats->queues[PGC_QUEUE_HOT].entries);
406 rrddim_set_by_pointer(ptrs->st_pgc_pages, ptrs->rd_pgc_pages_dirty, (collected_number)pgc_stats->queues[PGC_QUEUE_DIRTY].entries);
407 rrddim_set_by_pointer(ptrs->st_pgc_pages, ptrs->rd_pgc_pages_referenced, (collected_number)pgc_stats->referenced_entries);
408
409 rrdset_done(ptrs->st_pgc_pages);
410 }
411
412 {
413 if (unlikely(!ptrs->st_pgc_memory_changes)) {
414 BUFFER *id = buffer_create(100, NULL);
415 buffer_sprintf(id, "dbengine_%s_cache_memory_changes", name);
416
417 BUFFER *family = buffer_create(100, NULL);
418 buffer_sprintf(family, "dbengine %s cache", name);
419
420 BUFFER *title = buffer_create(100, NULL);
421 buffer_sprintf(title, "Netdata %s Cache Memory Changes", name);
422
423 ptrs->st_pgc_memory_changes = rrdset_create_localhost(
424 "netdata",
425 buffer_tostring(id),
426 NULL,
427 buffer_tostring(family),
428 NULL,
429 buffer_tostring(title),
430 "bytes/s",
431 "netdata",
432 "pulse",
433 priority,
434 localhost->rrd_update_every,
435 RRDSET_TYPE_AREA);
436
437 ptrs->rd_pgc_memory_new_clean = rrddim_add(ptrs->st_pgc_memory_changes, "new clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
438 ptrs->rd_pgc_memory_clean_evictions = rrddim_add(ptrs->st_pgc_memory_changes, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
439 ptrs->rd_pgc_memory_new_hot = rrddim_add(ptrs->st_pgc_memory_changes, "new hot", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
440
441 buffer_free(id);
442 buffer_free(family);
443 buffer_free(title);
444 priority++;
445 }
446
447 rrddim_set_by_pointer(ptrs->st_pgc_memory_changes, ptrs->rd_pgc_memory_new_clean, (collected_number)(pgc_stats->added_size - pgc_stats->queues[PGC_QUEUE_HOT].added_size));
448 rrddim_set_by_pointer(ptrs->st_pgc_memory_changes, ptrs->rd_pgc_memory_clean_evictions, (collected_number)pgc_stats->queues[PGC_QUEUE_CLEAN].removed_size);
449 rrddim_set_by_pointer(ptrs->st_pgc_memory_changes, ptrs->rd_pgc_memory_new_hot, (collected_number)pgc_stats->queues[PGC_QUEUE_HOT].added_size);
450
451 rrdset_done(ptrs->st_pgc_memory_changes);
452 }
453
454 {
455 if (unlikely(!ptrs->st_pgc_memory_migrations)) {
456 BUFFER *id = buffer_create(100, NULL);
457 buffer_sprintf(id, "dbengine_%s_cache_memory_migrations", name);
458
459 BUFFER *family = buffer_create(100, NULL);
460 buffer_sprintf(family, "dbengine %s cache", name);
461
462 BUFFER *title = buffer_create(100, NULL);
463 buffer_sprintf(title, "Netdata %s Cache Memory Migrations", name);
464
465 ptrs->st_pgc_memory_migrations = rrdset_create_localhost(
466 "netdata",
467 buffer_tostring(id),
468 NULL,
469 buffer_tostring(family),
470 NULL,
471 buffer_tostring(title),
472 "bytes/s",
473 "netdata",
474 "pulse",
475 priority,
476 localhost->rrd_update_every,
477 RRDSET_TYPE_AREA);
478
479 ptrs->rd_pgc_memory_dirty_to_clean = rrddim_add(ptrs->st_pgc_memory_migrations, "dirty to clean", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
480 ptrs->rd_pgc_memory_hot_to_dirty = rrddim_add(ptrs->st_pgc_memory_migrations, "hot to dirty", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
481
482 buffer_free(id);
483 buffer_free(family);
484 buffer_free(title);
485 priority++;
486 }
487
488 rrddim_set_by_pointer(ptrs->st_pgc_memory_migrations, ptrs->rd_pgc_memory_dirty_to_clean, (collected_number)pgc_stats->queues[PGC_QUEUE_DIRTY].removed_size);
489 rrddim_set_by_pointer(ptrs->st_pgc_memory_migrations, ptrs->rd_pgc_memory_hot_to_dirty, (collected_number)pgc_stats->queues[PGC_QUEUE_DIRTY].added_size);
490
491 rrdset_done(ptrs->st_pgc_memory_migrations);
492 }
493
494 {
495 if (unlikely(!ptrs->st_pgc_memory_events)) {
496 BUFFER *id = buffer_create(100, NULL);
497 buffer_sprintf(id, "dbengine_%s_cache_events", name);
498
499 BUFFER *family = buffer_create(100, NULL);
500 buffer_sprintf(family, "dbengine %s cache", name);
501
502 BUFFER *title = buffer_create(100, NULL);
503 buffer_sprintf(title, "Netdata %s Cache Events", name);
504
505 ptrs->st_pgc_memory_events = rrdset_create_localhost(
506 "netdata",
507 buffer_tostring(id),
508 NULL,
509 buffer_tostring(family),
510 NULL,
511 buffer_tostring(title),
512 "events/s",
513 "netdata",
514 "pulse",
515 priority,
516 localhost->rrd_update_every,
517 RRDSET_TYPE_AREA);
518
519 ptrs->rd_pgc_memory_evictions_aggressive = rrddim_add(ptrs->st_pgc_memory_events, "evictions aggressive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
520 ptrs->rd_pgc_memory_evictions_critical = rrddim_add(ptrs->st_pgc_memory_events, "evictions critical", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
521 ptrs->rd_pgc_memory_flushes_critical = rrddim_add(ptrs->st_pgc_memory_events, "flushes critical", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
522
523 buffer_free(id);
524 buffer_free(family);
525 buffer_free(title);
526 priority++;
527 }
528
529 rrddim_set_by_pointer(ptrs->st_pgc_memory_events, ptrs->rd_pgc_memory_evictions_aggressive, (collected_number)pgc_stats->events_cache_needs_space_aggressively);
530 rrddim_set_by_pointer(ptrs->st_pgc_memory_events, ptrs->rd_pgc_memory_evictions_critical, (collected_number)pgc_stats->events_cache_under_severe_pressure);
531 rrddim_set_by_pointer(ptrs->st_pgc_memory_events, ptrs->rd_pgc_memory_flushes_critical, (collected_number)pgc_stats->events_flush_critical);
532
533 rrdset_done(ptrs->st_pgc_memory_events);
534 }
535
536 {
537 if (unlikely(!ptrs->st_pgc_waste)) {
538 BUFFER *id = buffer_create(100, NULL);
539 buffer_sprintf(id, "dbengine_%s_waste_events", name);
540
541 BUFFER *family = buffer_create(100, NULL);
542 buffer_sprintf(family, "dbengine %s cache", name);
543
544 BUFFER *title = buffer_create(100, NULL);
545 buffer_sprintf(title, "Netdata %s Waste Events", name);
546
547 ptrs->st_pgc_waste = rrdset_create_localhost(
548 "netdata",
549 buffer_tostring(id),
550 NULL,
551 buffer_tostring(family),
552 NULL,
553 buffer_tostring(title),
554 "events/s",
555 "netdata",
556 "pulse",
557 priority,
558 localhost->rrd_update_every,
559 RRDSET_TYPE_LINE);
560
561 ptrs->rd_pgc_waste_evict_relocated = rrddim_add(ptrs->st_pgc_waste, "evict relocated", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
562 ptrs->rd_pgc_waste_flushes_cancelled = rrddim_add(ptrs->st_pgc_waste, "flushes cancelled", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
563 ptrs->rd_pgc_waste_insert_spins = rrddim_add(ptrs->st_pgc_waste, "insert spins", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
564 ptrs->rd_pgc_waste_evict_spins = rrddim_add(ptrs->st_pgc_waste, "evict useless spins", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
565 ptrs->rd_pgc_waste_evict_thread_signals = rrddim_add(ptrs->st_pgc_waste, "evict thread signals", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
566 ptrs->rd_pgc_waste_evict_inline_on_add = rrddim_add(ptrs->st_pgc_waste, "evict inline on add", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
567 ptrs->rd_pgc_waste_evict_inline_on_release = rrddim_add(ptrs->st_pgc_waste, "evict inline on rel", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
568 ptrs->rd_pgc_waste_flush_inline_on_add = rrddim_add(ptrs->st_pgc_waste, "flush inline on add", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
569 ptrs->rd_pgc_waste_flush_inline_on_release = rrddim_add(ptrs->st_pgc_waste, "flush inline on rel", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
570
571 buffer_free(id);
572 buffer_free(family);
573 buffer_free(title);
574 priority++;
575 }
576
577 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_evict_relocated, (collected_number)pgc_stats->p2_waste_evict_relocated);
578 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_flushes_cancelled, (collected_number)pgc_stats->p2_waste_flushes_cancelled);
579 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_insert_spins, (collected_number)pgc_stats->p2_waste_insert_spins);
580 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_evict_spins, (collected_number)pgc_stats->p2_waste_evict_useless_spins);
581 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_evict_thread_signals, (collected_number)pgc_stats->p2_waste_evict_thread_signals);
582 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_evict_inline_on_add, (collected_number)pgc_stats->p2_waste_evictions_inline_on_add);
583 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_evict_inline_on_release, (collected_number)pgc_stats->p2_waste_evictions_inline_on_release);
584 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_flush_inline_on_add, (collected_number)pgc_stats->p2_waste_flush_on_add);
585 rrddim_set_by_pointer(ptrs->st_pgc_waste, ptrs->rd_pgc_waste_flush_inline_on_release, (collected_number)pgc_stats->p2_waste_flush_on_release);
586
587 rrdset_done(ptrs->st_pgc_waste);
588 }
589
590 {
591 if (unlikely(!ptrs->st_pgc_workers)) {
592 BUFFER *id = buffer_create(100, NULL);
593 buffer_sprintf(id, "dbengine_%s_cache_workers", name);
594
595 BUFFER *family = buffer_create(100, NULL);
596 buffer_sprintf(family, "dbengine %s cache", name);
597
598 BUFFER *title = buffer_create(100, NULL);
599 buffer_sprintf(title, "Netdata %s Cache Workers", name);
600
601 ptrs->st_pgc_workers = rrdset_create_localhost(
602 "netdata",
603 buffer_tostring(id),
604 NULL,
605 buffer_tostring(family),
606 NULL,
607 buffer_tostring(title),
608 "workers",
609 "netdata",
610 "pulse",
611 priority,
612 localhost->rrd_update_every,
613 RRDSET_TYPE_LINE);
614
615 ptrs->rd_pgc_workers_searchers = rrddim_add(ptrs->st_pgc_workers, "searchers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
616 ptrs->rd_pgc_workers_adders = rrddim_add(ptrs->st_pgc_workers, "adders", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
617 ptrs->rd_pgc_workers_evictors = rrddim_add(ptrs->st_pgc_workers, "evictors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
618 ptrs->rd_pgc_workers_flushers = rrddim_add(ptrs->st_pgc_workers, "flushers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
619 ptrs->rd_pgc_workers_hot2dirty = rrddim_add(ptrs->st_pgc_workers, "hot2dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
620 ptrs->rd_pgc_workers_jv2_flushers = rrddim_add(ptrs->st_pgc_workers, "jv2 flushers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
621
622 buffer_free(id);
623 buffer_free(family);
624 buffer_free(title);
625 priority++;
626 }
627
628 rrddim_set_by_pointer(ptrs->st_pgc_workers, ptrs->rd_pgc_workers_searchers, (collected_number)pgc_stats->p2_workers_search);
629 rrddim_set_by_pointer(ptrs->st_pgc_workers, ptrs->rd_pgc_workers_adders, (collected_number)pgc_stats->p2_workers_add);
630 rrddim_set_by_pointer(ptrs->st_pgc_workers, ptrs->rd_pgc_workers_evictors, (collected_number)pgc_stats->p0_workers_evict);
631 rrddim_set_by_pointer(ptrs->st_pgc_workers, ptrs->rd_pgc_workers_flushers, (collected_number)pgc_stats->p2_workers_flush);
632 rrddim_set_by_pointer(ptrs->st_pgc_workers, ptrs->rd_pgc_workers_hot2dirty, (collected_number)pgc_stats->p2_workers_hot2dirty);
633 rrddim_set_by_pointer(ptrs->st_pgc_workers, ptrs->rd_pgc_workers_jv2_flushers, (collected_number)pgc_stats->p2_workers_jv2_flush);
634
635 rrdset_done(ptrs->st_pgc_workers);
636 }
637 }
638
639 void pulse_dbengine_do(bool extended) {
640 static struct dbengine2_cache_pointers main_cache_ptrs = {}, open_cache_ptrs = {}, extent_cache_ptrs = {};
641 static struct rrdeng_cache_efficiency_stats cache_efficiency_stats = {}, cache_efficiency_stats_old = {};
642 static struct pgc_statistics pgc_main_stats = {}, pgc_main_stats_old = {}; (void)pgc_main_stats_old;
643 static struct pgc_statistics pgc_open_stats = {}, pgc_open_stats_old = {}; (void)pgc_open_stats_old;
644 static struct pgc_statistics pgc_extent_stats = {}, pgc_extent_stats_old = {}; (void)pgc_extent_stats_old;
645 static struct mrg_statistics mrg_stats = {}, mrg_stats_old = {}; (void)mrg_stats_old;
646
647 pgc_main_stats_old = pgc_main_stats;
648 pgc_main_stats = pgc_get_statistics(main_cache);
649
650 pgc_open_stats_old = pgc_open_stats;
651 pgc_open_stats = pgc_get_statistics(open_cache);
652
653 pgc_extent_stats_old = pgc_extent_stats;
654 pgc_extent_stats = pgc_get_statistics(extent_cache);
655
656 cache_efficiency_stats_old = cache_efficiency_stats;
657 cache_efficiency_stats = rrdeng_get_cache_efficiency_stats();
658
659 mrg_stats_old = mrg_stats;
660
661 struct rrdeng_buffer_sizes dbmem = rrdeng_pulse_memory_sizes();
662
663 int64_t buffers_total_size = (int64_t)dbmem.xt_buf + (int64_t)dbmem.wal;
664
665 int64_t aral_structures_total_size = 0, aral_used_total_size = 0;
666 int64_t aral_padding_total_size = 0;
667 for(size_t i = 0; i < RRDENG_MEM_MAX ; i++) {
668 buffers_total_size += (int64_t)aral_free_bytes_from_stats(dbmem.as[i]);
669 aral_structures_total_size += (int64_t)aral_structures_bytes_from_stats(dbmem.as[i]);
670 aral_used_total_size += (int64_t)aral_used_bytes_from_stats(dbmem.as[i]);
671 aral_padding_total_size += (int64_t)aral_padding_bytes_from_stats(dbmem.as[i]);
672 }
673
674 pulse_dbengine_total_memory =
675 pgc_main_stats.size + pgc_open_stats.size + pgc_extent_stats.size +
676 mrg_stats.size +
677 buffers_total_size + aral_structures_total_size + aral_padding_total_size + (int64_t)pgd_padding_bytes();
678
679 // we need all the above for the total dbengine memory as reported by the non-extended netdata memory chart
680 if(!main_cache || !main_mrg || !extended)
681 return;
682
683 dbengine2_cache_statistics_charts(&main_cache_ptrs, &pgc_main_stats, &pgc_main_stats_old, "main", 135100);
684 dbengine2_cache_statistics_charts(&open_cache_ptrs, &pgc_open_stats, &pgc_open_stats_old, "open", 135200);
685 dbengine2_cache_statistics_charts(&extent_cache_ptrs, &pgc_extent_stats, &pgc_extent_stats_old, "extent", 135300);
686 mrg_get_statistics(main_mrg, &mrg_stats);
687
688 int priority = 135000;
689 {
690 static RRDSET *st_pgc_memory = NULL;
691 static RRDDIM *rd_pgc_memory_main = NULL;
692 static RRDDIM *rd_pgc_memory_open = NULL; // open journal memory
693 static RRDDIM *rd_pgc_memory_extent = NULL; // extent compresses cache memory
694 static RRDDIM *rd_pgc_memory_metrics = NULL; // metric registry memory
695 static RRDDIM *rd_pgc_memory_buffers = NULL;
696 static RRDDIM *rd_pgc_memory_aral_padding = NULL;
697 static RRDDIM *rd_pgc_memory_pgd_padding = NULL;
698 static RRDDIM *rd_pgc_memory_aral_structures = NULL;
699
700 if (unlikely(!st_pgc_memory)) {
701 st_pgc_memory = rrdset_create_localhost(
702 "netdata",
703 "dbengine_memory",
704 NULL,
705 "dbengine memory",
706 NULL,
707 "Netdata DB Memory",
708 "bytes",
709 "netdata",
710 "pulse",
711 priority,
712 localhost->rrd_update_every,
713 RRDSET_TYPE_STACKED);
714
715 rd_pgc_memory_main = rrddim_add(st_pgc_memory, "main cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
716 rd_pgc_memory_open = rrddim_add(st_pgc_memory, "open cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
717 rd_pgc_memory_extent = rrddim_add(st_pgc_memory, "extent cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
718 rd_pgc_memory_metrics = rrddim_add(st_pgc_memory, "metrics registry", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
719 rd_pgc_memory_buffers = rrddim_add(st_pgc_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
720 rd_pgc_memory_aral_padding = rrddim_add(st_pgc_memory, "aral padding", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
721 rd_pgc_memory_pgd_padding = rrddim_add(st_pgc_memory, "pgd padding", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
722 rd_pgc_memory_aral_structures = rrddim_add(st_pgc_memory, "aral structures", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
723 }
724 priority++;
725
726
727 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_main, (collected_number)pgc_main_stats.size);
728 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_open, (collected_number)pgc_open_stats.size);
729 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_extent, (collected_number)pgc_extent_stats.size);
730 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_metrics, (collected_number)mrg_stats.size);
731 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_buffers, (collected_number)buffers_total_size);
732 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_aral_padding, (collected_number)aral_padding_total_size);
733 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_pgd_padding, (collected_number)pgd_padding_bytes());
734 rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_aral_structures, (collected_number)aral_structures_total_size);
735
736 rrdset_done(st_pgc_memory);
737 }
738
739 {
740 static RRDSET *st_pgc_buffers = NULL;
741 static RRDDIM *rd_pgc_buffers_pgc = NULL;
742 static RRDDIM *rd_pgc_buffers_pgd = NULL;
743 static RRDDIM *rd_pgc_buffers_mrg = NULL;
744 static RRDDIM *rd_pgc_buffers_opcodes = NULL;
745 static RRDDIM *rd_pgc_buffers_handles = NULL;
746 static RRDDIM *rd_pgc_buffers_descriptors = NULL;
747 static RRDDIM *rd_pgc_buffers_wal = NULL;
748 static RRDDIM *rd_pgc_buffers_workers = NULL;
749 static RRDDIM *rd_pgc_buffers_pdc = NULL;
750 static RRDDIM *rd_pgc_buffers_xt_io = NULL;
751 static RRDDIM *rd_pgc_buffers_xt_buf = NULL;
752 static RRDDIM *rd_pgc_buffers_epdl = NULL;
753 static RRDDIM *rd_pgc_buffers_deol = NULL;
754 static RRDDIM *rd_pgc_buffers_pd = NULL;
755 static RRDDIM *rd_pgc_buffers_epdl_extent = NULL;
756
757 if (unlikely(!st_pgc_buffers)) {
758 st_pgc_buffers = rrdset_create_localhost(
759 "netdata",
760 "dbengine_buffers",
761 NULL,
762 "dbengine memory",
763 NULL,
764 "Netdata DB Buffers",
765 "bytes",
766 "netdata",
767 "pulse",
768 priority,
769 localhost->rrd_update_every,
770 RRDSET_TYPE_STACKED);
771
772 rd_pgc_buffers_pgc = rrddim_add(st_pgc_buffers, "pgc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
773 rd_pgc_buffers_pgd = rrddim_add(st_pgc_buffers, "pgd", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
774 rd_pgc_buffers_mrg = rrddim_add(st_pgc_buffers, "mrg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
775 rd_pgc_buffers_opcodes = rrddim_add(st_pgc_buffers, "opcodes", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
776 rd_pgc_buffers_handles = rrddim_add(st_pgc_buffers, "query handles", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
777 rd_pgc_buffers_descriptors = rrddim_add(st_pgc_buffers, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
778 rd_pgc_buffers_wal = rrddim_add(st_pgc_buffers, "wal", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
779 rd_pgc_buffers_workers = rrddim_add(st_pgc_buffers, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
780 rd_pgc_buffers_pdc = rrddim_add(st_pgc_buffers, "pdc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
781 rd_pgc_buffers_pd = rrddim_add(st_pgc_buffers, "pd", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
782 rd_pgc_buffers_xt_io = rrddim_add(st_pgc_buffers, "extent io", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
783 rd_pgc_buffers_xt_buf = rrddim_add(st_pgc_buffers, "extent buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
784 rd_pgc_buffers_epdl = rrddim_add(st_pgc_buffers, "epdl", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
785 rd_pgc_buffers_deol = rrddim_add(st_pgc_buffers, "deol", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
786 rd_pgc_buffers_epdl_extent = rrddim_add(st_pgc_buffers, "epdl extent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
787 }
788 priority++;
789
790 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pgc, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_PGC]));
791 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pgd, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_PGD]));
792 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_mrg, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_MRG]));
793 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_opcodes, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_OPCODES]));
794 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_handles, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_HANDLES]));
795 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_descriptors, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_DESCRIPTORS]));
796 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_wal, (collected_number)dbmem.wal);
797 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_workers, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_WORKERS]));
798 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pdc, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_PDC]));
799 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pd, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_PD]));
800 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_xt_io, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_XT_IO]));
801 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_xt_buf, (collected_number)dbmem.xt_buf);
802 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_epdl, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_EPDL]));
803 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_deol, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_DEOL]));
804 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_epdl_extent, (collected_number)aral_free_bytes_from_stats(dbmem.as[RRDENG_MEM_EPDL_EXTENT]));
805
806 rrdset_done(st_pgc_buffers);
807 }
808
809 {
810 static RRDSET *st_mrg_metrics = NULL;
811 static RRDDIM *rd_mrg_metrics = NULL;
812 static RRDDIM *rd_mrg_acquired = NULL;
813 static RRDDIM *rd_mrg_collected = NULL;
814 static RRDDIM *rd_mrg_multiple_writers = NULL;
815
816 if (unlikely(!st_mrg_metrics)) {
817 st_mrg_metrics = rrdset_create_localhost(
818 "netdata",
819 "dbengine_metrics",
820 NULL,
821 "dbengine metrics",
822 NULL,
823 "Netdata Metrics in Metrics Registry",
824 "metrics",
825 "netdata",
826 "pulse",
827 priority,
828 localhost->rrd_update_every,
829 RRDSET_TYPE_LINE);
830
831 rd_mrg_metrics = rrddim_add(st_mrg_metrics, "all", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
832 rd_mrg_acquired = rrddim_add(st_mrg_metrics, "acquired", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
833 rd_mrg_collected = rrddim_add(st_mrg_metrics, "collected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
834 rd_mrg_multiple_writers = rrddim_add(st_mrg_metrics, "multi-collected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
835 }
836 priority++;
837
838 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_metrics, (collected_number)mrg_stats.entries);
839 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_acquired, (collected_number)mrg_stats.entries_acquired);
840 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_collected, (collected_number)mrg_stats.writers);
841 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_multiple_writers, (collected_number)mrg_stats.writers_conflicts);
842
843 rrdset_done(st_mrg_metrics);
844 }
845
846 {
847 static RRDSET *st_mrg_ops = NULL;
848 static RRDDIM *rd_mrg_add = NULL;
849 static RRDDIM *rd_mrg_del = NULL;
850 static RRDDIM *rd_mrg_search = NULL;
851
852 if (unlikely(!st_mrg_ops)) {
853 st_mrg_ops = rrdset_create_localhost(
854 "netdata",
855 "dbengine_metrics_registry_operations",
856 NULL,
857 "dbengine metrics",
858 NULL,
859 "Netdata Metrics Registry Operations",
860 "metrics",
861 "netdata",
862 "pulse",
863 priority,
864 localhost->rrd_update_every,
865 RRDSET_TYPE_LINE);
866
867 rd_mrg_add = rrddim_add(st_mrg_ops, "add", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
868 rd_mrg_del = rrddim_add(st_mrg_ops, "delete", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
869 rd_mrg_search = rrddim_add(st_mrg_ops, "search", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
870 }
871 priority++;
872
873 rrddim_set_by_pointer(st_mrg_ops, rd_mrg_add, (collected_number)mrg_stats.additions);
874 rrddim_set_by_pointer(st_mrg_ops, rd_mrg_del, (collected_number)mrg_stats.deletions);
875 rrddim_set_by_pointer(st_mrg_ops, rd_mrg_search, (collected_number)mrg_stats.search_hits + (collected_number)mrg_stats.search_misses);
876
877 rrdset_done(st_mrg_ops);
878 }
879
880 {
881 static RRDSET *st_mrg_references = NULL;
882 static RRDDIM *rd_mrg_references = NULL;
883
884 if (unlikely(!st_mrg_references)) {
885 st_mrg_references = rrdset_create_localhost(
886 "netdata",
887 "dbengine_metrics_registry_references",
888 NULL,
889 "dbengine metrics",
890 NULL,
891 "Netdata Metrics Registry References",
892 "references",
893 "netdata",
894 "pulse",
895 priority,
896 localhost->rrd_update_every,
897 RRDSET_TYPE_LINE);
898
899 rd_mrg_references = rrddim_add(st_mrg_references, "references", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
900 }
901 priority++;
902
903 rrddim_set_by_pointer(st_mrg_references, rd_mrg_references, (collected_number)mrg_stats.current_references);
904
905 rrdset_done(st_mrg_references);
906 }
907
908 {
909 static RRDSET *st_cache_hit_ratio = NULL;
910 static RRDDIM *rd_hit_ratio = NULL;
911 static RRDDIM *rd_main_cache_hit_ratio = NULL;
912 static RRDDIM *rd_extent_cache_hit_ratio = NULL;
913 static RRDDIM *rd_extent_merge_hit_ratio = NULL;
914
915 if (unlikely(!st_cache_hit_ratio)) {
916 st_cache_hit_ratio = rrdset_create_localhost(
917 "netdata",
918 "dbengine_cache_hit_ratio",
919 NULL,
920 "dbengine query router",
921 NULL,
922 "Netdata Queries Cache Hit Ratio",
923 "%",
924 "netdata",
925 "pulse",
926 priority,
927 localhost->rrd_update_every,
928 RRDSET_TYPE_LINE);
929
930 rd_hit_ratio = rrddim_add(st_cache_hit_ratio, "overall", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
931 rd_main_cache_hit_ratio = rrddim_add(st_cache_hit_ratio, "main cache", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
932 rd_extent_cache_hit_ratio = rrddim_add(st_cache_hit_ratio, "extent cache", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
933 rd_extent_merge_hit_ratio = rrddim_add(st_cache_hit_ratio, "extent merge", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
934 }
935 priority++;
936
937 size_t delta_pages_total = cache_efficiency_stats.pages_total - cache_efficiency_stats_old.pages_total;
938 size_t delta_pages_to_load_from_disk = cache_efficiency_stats.pages_to_load_from_disk - cache_efficiency_stats_old.pages_to_load_from_disk;
939 size_t delta_extents_loaded_from_disk = cache_efficiency_stats.extents_loaded_from_disk - cache_efficiency_stats_old.extents_loaded_from_disk;
940
941 size_t delta_pages_data_source_main_cache = cache_efficiency_stats.pages_data_source_main_cache - cache_efficiency_stats_old.pages_data_source_main_cache;
942 size_t delta_pages_pending_found_in_cache_at_pass4 = cache_efficiency_stats.pages_data_source_main_cache_at_pass4 - cache_efficiency_stats_old.pages_data_source_main_cache_at_pass4;
943
944 size_t delta_pages_data_source_extent_cache = cache_efficiency_stats.pages_data_source_extent_cache - cache_efficiency_stats_old.pages_data_source_extent_cache;
945 size_t delta_pages_load_extent_merged = cache_efficiency_stats.pages_load_extent_merged - cache_efficiency_stats_old.pages_load_extent_merged;
946
947 size_t pages_total_hit = delta_pages_total - delta_extents_loaded_from_disk;
948
949 static size_t overall_hit_ratio = 100;
950 size_t main_cache_hit_ratio = 0, extent_cache_hit_ratio = 0, extent_merge_hit_ratio = 0;
951 if(delta_pages_total) {
952 if(pages_total_hit > delta_pages_total)
953 pages_total_hit = delta_pages_total;
954
955 overall_hit_ratio = pages_total_hit * 100 * 10000 / delta_pages_total;
956
957 size_t delta_pages_main_cache = delta_pages_data_source_main_cache + delta_pages_pending_found_in_cache_at_pass4;
958 if(delta_pages_main_cache > delta_pages_total)
959 delta_pages_main_cache = delta_pages_total;
960
961 main_cache_hit_ratio = delta_pages_main_cache * 100 * 10000 / delta_pages_total;
962 }
963
964 if(delta_pages_to_load_from_disk) {
965 if(delta_pages_data_source_extent_cache > delta_pages_to_load_from_disk)
966 delta_pages_data_source_extent_cache = delta_pages_to_load_from_disk;
967
968 extent_cache_hit_ratio = delta_pages_data_source_extent_cache * 100 * 10000 / delta_pages_to_load_from_disk;
969
970 if(delta_pages_load_extent_merged > delta_pages_to_load_from_disk)
971 delta_pages_load_extent_merged = delta_pages_to_load_from_disk;
972
973 extent_merge_hit_ratio = delta_pages_load_extent_merged * 100 * 10000 / delta_pages_to_load_from_disk;
974 }
975
976 rrddim_set_by_pointer(st_cache_hit_ratio, rd_hit_ratio, (collected_number)overall_hit_ratio);
977 rrddim_set_by_pointer(st_cache_hit_ratio, rd_main_cache_hit_ratio, (collected_number)main_cache_hit_ratio);
978 rrddim_set_by_pointer(st_cache_hit_ratio, rd_extent_cache_hit_ratio, (collected_number)extent_cache_hit_ratio);
979 rrddim_set_by_pointer(st_cache_hit_ratio, rd_extent_merge_hit_ratio, (collected_number)extent_merge_hit_ratio);
980
981 rrdset_done(st_cache_hit_ratio);
982 }
983
984 {
985 static RRDSET *st_queries = NULL;
986 static RRDDIM *rd_total = NULL;
987 static RRDDIM *rd_open = NULL;
988 static RRDDIM *rd_jv2 = NULL;
989 static RRDDIM *rd_planned_with_gaps = NULL;
990 static RRDDIM *rd_executed_with_gaps = NULL;
991
992 if (unlikely(!st_queries)) {
993 st_queries = rrdset_create_localhost(
994 "netdata",
995 "dbengine_queries",
996 NULL,
997 "dbengine query router",
998 NULL,
999 "Netdata Queries",
1000 "queries/s",
1001 "netdata",
1002 "pulse",
1003 priority,
1004 localhost->rrd_update_every,
1005 RRDSET_TYPE_LINE);
1006
1007 rd_total = rrddim_add(st_queries, "total", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1008 rd_open = rrddim_add(st_queries, "open cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1009 rd_jv2 = rrddim_add(st_queries, "journal v2", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1010 rd_planned_with_gaps = rrddim_add(st_queries, "planned with gaps", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1011 rd_executed_with_gaps = rrddim_add(st_queries, "executed with gaps", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1012 }
1013 priority++;
1014
1015 rrddim_set_by_pointer(st_queries, rd_total, (collected_number)cache_efficiency_stats.prep_time_in_main_cache_lookup.count);
1016 rrddim_set_by_pointer(st_queries, rd_open, (collected_number)cache_efficiency_stats.prep_time_in_open_cache_lookup.count);
1017 rrddim_set_by_pointer(st_queries, rd_jv2, (collected_number)cache_efficiency_stats.prep_time_in_journal_v2_lookup.count);
1018 rrddim_set_by_pointer(st_queries, rd_planned_with_gaps, (collected_number)cache_efficiency_stats.queries_planned_with_gaps);
1019 rrddim_set_by_pointer(st_queries, rd_executed_with_gaps, (collected_number)cache_efficiency_stats.queries_executed_with_gaps);
1020
1021 rrdset_done(st_queries);
1022 }
1023
1024 {
1025 static RRDSET *st_queries_running = NULL;
1026 static RRDDIM *rd_queries = NULL;
1027
1028 if (unlikely(!st_queries_running)) {
1029 st_queries_running = rrdset_create_localhost(
1030 "netdata",
1031 "dbengine_queries_running",
1032 NULL,
1033 "dbengine query router",
1034 NULL,
1035 "Netdata Queries Running",
1036 "queries",
1037 "netdata",
1038 "pulse",
1039 priority,
1040 localhost->rrd_update_every,
1041 RRDSET_TYPE_LINE);
1042
1043 rd_queries = rrddim_add(st_queries_running, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1044 }
1045 priority++;
1046
1047 rrddim_set_by_pointer(st_queries_running, rd_queries, (collected_number)cache_efficiency_stats.currently_running_queries);
1048
1049 rrdset_done(st_queries_running);
1050 }
1051
1052 {
1053 static RRDSET *st_query_pages_metadata_source = NULL;
1054 static RRDDIM *rd_cache = NULL;
1055 static RRDDIM *rd_open = NULL;
1056 static RRDDIM *rd_jv2 = NULL;
1057
1058 if (unlikely(!st_query_pages_metadata_source)) {
1059 st_query_pages_metadata_source = rrdset_create_localhost(
1060 "netdata",
1061 "dbengine_query_pages_metadata_source",
1062 NULL,
1063 "dbengine query router",
1064 NULL,
1065 "Netdata Query Pages Metadata Source",
1066 "pages/s",
1067 "netdata",
1068 "pulse",
1069 priority,
1070 localhost->rrd_update_every,
1071 RRDSET_TYPE_STACKED);
1072
1073 rd_cache = rrddim_add(st_query_pages_metadata_source, "cache hit", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1074 rd_jv2 = rrddim_add(st_query_pages_metadata_source, "journal v2 scan", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1075 rd_open = rrddim_add(st_query_pages_metadata_source, "open journal", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1076 }
1077 priority++;
1078
1079 rrddim_set_by_pointer(st_query_pages_metadata_source, rd_cache, (collected_number)cache_efficiency_stats.pages_meta_source_main_cache);
1080 rrddim_set_by_pointer(st_query_pages_metadata_source, rd_jv2, (collected_number)cache_efficiency_stats.pages_meta_source_journal_v2);
1081 rrddim_set_by_pointer(st_query_pages_metadata_source, rd_open, (collected_number)cache_efficiency_stats.pages_meta_source_open_cache);
1082
1083 rrdset_done(st_query_pages_metadata_source);
1084 }
1085
1086 {
1087 static RRDSET *st_query_pages_data_source = NULL;
1088 static RRDDIM *rd_pages_main_cache = NULL;
1089 static RRDDIM *rd_pages_disk = NULL;
1090 static RRDDIM *rd_pages_extent_cache = NULL;
1091
1092 if (unlikely(!st_query_pages_data_source)) {
1093 st_query_pages_data_source = rrdset_create_localhost(
1094 "netdata",
1095 "dbengine_query_pages_data_source",
1096 NULL,
1097 "dbengine query router",
1098 NULL,
1099 "Netdata Query Pages to Data Source",
1100 "pages/s",
1101 "netdata",
1102 "pulse",
1103 priority,
1104 localhost->rrd_update_every,
1105 RRDSET_TYPE_STACKED);
1106
1107 rd_pages_main_cache = rrddim_add(st_query_pages_data_source, "main cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1108 rd_pages_disk = rrddim_add(st_query_pages_data_source, "disk", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1109 rd_pages_extent_cache = rrddim_add(st_query_pages_data_source, "extent cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1110 }
1111 priority++;
1112
1113 rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_main_cache, (collected_number)cache_efficiency_stats.pages_data_source_main_cache + (collected_number)cache_efficiency_stats.pages_data_source_main_cache_at_pass4);
1114 rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_disk, (collected_number)cache_efficiency_stats.pages_to_load_from_disk);
1115 rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_extent_cache, (collected_number)cache_efficiency_stats.pages_data_source_extent_cache);
1116
1117 rrdset_done(st_query_pages_data_source);
1118 }
1119
1120 {
1121 static RRDSET *st_query_next_page = NULL;
1122 static RRDDIM *rd_pass4 = NULL;
1123 static RRDDIM *rd_nowait_failed = NULL;
1124 static RRDDIM *rd_wait_failed = NULL;
1125 static RRDDIM *rd_wait_loaded = NULL;
1126 static RRDDIM *rd_nowait_loaded = NULL;
1127
1128 if (unlikely(!st_query_next_page)) {
1129 st_query_next_page = rrdset_create_localhost(
1130 "netdata",
1131 "dbengine_query_next_page",
1132 NULL,
1133 "dbengine query router",
1134 NULL,
1135 "Netdata Query Next Page",
1136 "pages/s",
1137 "netdata",
1138 "pulse",
1139 priority,
1140 localhost->rrd_update_every,
1141 RRDSET_TYPE_STACKED);
1142
1143 rd_pass4 = rrddim_add(st_query_next_page, "pass4", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1144 rd_wait_failed = rrddim_add(st_query_next_page, "failed slow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1145 rd_nowait_failed = rrddim_add(st_query_next_page, "failed fast", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1146 rd_wait_loaded = rrddim_add(st_query_next_page, "loaded slow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1147 rd_nowait_loaded = rrddim_add(st_query_next_page, "loaded fast", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1148 }
1149 priority++;
1150
1151 rrddim_set_by_pointer(st_query_next_page, rd_pass4, (collected_number)cache_efficiency_stats.pages_data_source_main_cache_at_pass4);
1152 rrddim_set_by_pointer(st_query_next_page, rd_wait_failed, (collected_number)cache_efficiency_stats.page_next_wait_failed);
1153 rrddim_set_by_pointer(st_query_next_page, rd_nowait_failed, (collected_number)cache_efficiency_stats.page_next_nowait_failed);
1154 rrddim_set_by_pointer(st_query_next_page, rd_wait_loaded, (collected_number)cache_efficiency_stats.page_next_wait_loaded);
1155 rrddim_set_by_pointer(st_query_next_page, rd_nowait_loaded, (collected_number)cache_efficiency_stats.page_next_nowait_loaded);
1156
1157 rrdset_done(st_query_next_page);
1158 }
1159
1160 {
1161 static RRDSET *st_query_page_issues = NULL;
1162 static RRDDIM *rd_pages_zero_time = NULL;
1163 static RRDDIM *rd_pages_past_time = NULL;
1164 static RRDDIM *rd_pages_invalid_size = NULL;
1165 static RRDDIM *rd_pages_fixed_update_every = NULL;
1166 static RRDDIM *rd_pages_fixed_entries = NULL;
1167 static RRDDIM *rd_pages_overlapping = NULL;
1168
1169 if (unlikely(!st_query_page_issues)) {
1170 st_query_page_issues = rrdset_create_localhost(
1171 "netdata",
1172 "dbengine_query_next_page_issues",
1173 NULL,
1174 "dbengine query router",
1175 NULL,
1176 "Netdata Query Next Page Issues",
1177 "pages/s",
1178 "netdata",
1179 "pulse",
1180 priority,
1181 localhost->rrd_update_every,
1182 RRDSET_TYPE_STACKED);
1183
1184 rd_pages_zero_time = rrddim_add(st_query_page_issues, "zero timestamp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1185 rd_pages_invalid_size = rrddim_add(st_query_page_issues, "invalid size", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1186 rd_pages_past_time = rrddim_add(st_query_page_issues, "past time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1187 rd_pages_overlapping = rrddim_add(st_query_page_issues, "overlapping", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1188 rd_pages_fixed_update_every = rrddim_add(st_query_page_issues, "update every fixed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1189 rd_pages_fixed_entries = rrddim_add(st_query_page_issues, "entries fixed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1190 }
1191 priority++;
1192
1193 rrddim_set_by_pointer(st_query_page_issues, rd_pages_zero_time, (collected_number)cache_efficiency_stats.pages_zero_time_skipped);
1194 rrddim_set_by_pointer(st_query_page_issues, rd_pages_invalid_size, (collected_number)cache_efficiency_stats.pages_invalid_size_skipped);
1195 rrddim_set_by_pointer(st_query_page_issues, rd_pages_past_time, (collected_number)cache_efficiency_stats.pages_past_time_skipped);
1196 rrddim_set_by_pointer(st_query_page_issues, rd_pages_overlapping, (collected_number)cache_efficiency_stats.pages_overlapping_skipped);
1197 rrddim_set_by_pointer(st_query_page_issues, rd_pages_fixed_update_every, (collected_number)cache_efficiency_stats.pages_invalid_update_every_fixed);
1198 rrddim_set_by_pointer(st_query_page_issues, rd_pages_fixed_entries, (collected_number)cache_efficiency_stats.pages_invalid_entries_fixed);
1199
1200 rrdset_done(st_query_page_issues);
1201 }
1202
1203 {
1204 static RRDSET *st_query_pages_from_disk = NULL;
1205 static RRDDIM *rd_compressed = NULL;
1206 static RRDDIM *rd_invalid = NULL;
1207 static RRDDIM *rd_uncompressed = NULL;
1208 static RRDDIM *rd_mmap_failed = NULL;
1209 static RRDDIM *rd_unavailable = NULL;
1210 static RRDDIM *rd_unroutable = NULL;
1211 static RRDDIM *rd_not_found = NULL;
1212 static RRDDIM *rd_cancelled = NULL;
1213 static RRDDIM *rd_invalid_extent = NULL;
1214 static RRDDIM *rd_extent_merged = NULL;
1215
1216 if (unlikely(!st_query_pages_from_disk)) {
1217 st_query_pages_from_disk = rrdset_create_localhost(
1218 "netdata",
1219 "dbengine_query_pages_disk_load",
1220 NULL,
1221 "dbengine query router",
1222 NULL,
1223 "Netdata Query Pages Loaded from Disk",
1224 "pages/s",
1225 "netdata",
1226 "pulse",
1227 priority,
1228 localhost->rrd_update_every,
1229 RRDSET_TYPE_LINE);
1230
1231 rd_compressed = rrddim_add(st_query_pages_from_disk, "ok compressed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1232 rd_invalid = rrddim_add(st_query_pages_from_disk, "fail invalid page", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1233 rd_uncompressed = rrddim_add(st_query_pages_from_disk, "ok uncompressed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1234 rd_mmap_failed = rrddim_add(st_query_pages_from_disk, "fail cant mmap", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1235 rd_unavailable = rrddim_add(st_query_pages_from_disk, "fail unavailable", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1236 rd_unroutable = rrddim_add(st_query_pages_from_disk, "fail unroutable", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1237 rd_not_found = rrddim_add(st_query_pages_from_disk, "fail not found", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1238 rd_invalid_extent = rrddim_add(st_query_pages_from_disk, "fail invalid extent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1239 rd_extent_merged = rrddim_add(st_query_pages_from_disk, "extent merged", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1240 rd_cancelled = rrddim_add(st_query_pages_from_disk, "cancelled", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1241 }
1242 priority++;
1243
1244 rrddim_set_by_pointer(st_query_pages_from_disk, rd_compressed, (collected_number)cache_efficiency_stats.pages_load_ok_compressed);
1245 rrddim_set_by_pointer(st_query_pages_from_disk, rd_invalid, (collected_number)cache_efficiency_stats.pages_load_fail_invalid_page_in_extent);
1246 rrddim_set_by_pointer(st_query_pages_from_disk, rd_uncompressed, (collected_number)cache_efficiency_stats.pages_load_ok_uncompressed);
1247 rrddim_set_by_pointer(st_query_pages_from_disk, rd_mmap_failed, (collected_number)cache_efficiency_stats.pages_load_fail_cant_mmap_extent);
1248 rrddim_set_by_pointer(st_query_pages_from_disk, rd_unavailable, (collected_number)cache_efficiency_stats.pages_load_fail_datafile_not_available);
1249 rrddim_set_by_pointer(st_query_pages_from_disk, rd_unroutable, (collected_number)cache_efficiency_stats.pages_load_fail_unroutable);
1250 rrddim_set_by_pointer(st_query_pages_from_disk, rd_not_found, (collected_number)cache_efficiency_stats.pages_load_fail_not_found);
1251 rrddim_set_by_pointer(st_query_pages_from_disk, rd_cancelled, (collected_number)cache_efficiency_stats.pages_load_fail_cancelled);
1252 rrddim_set_by_pointer(st_query_pages_from_disk, rd_invalid_extent, (collected_number)cache_efficiency_stats.pages_load_fail_invalid_extent);
1253 rrddim_set_by_pointer(st_query_pages_from_disk, rd_extent_merged, (collected_number)cache_efficiency_stats.pages_load_extent_merged);
1254
1255 rrdset_done(st_query_pages_from_disk);
1256 }
1257
1258 {
1259 static RRDSET *st_events = NULL;
1260 static RRDDIM *rd_journal_v2_mapped = NULL;
1261 static RRDDIM *rd_journal_v2_unmapped = NULL;
1262 static RRDDIM *rd_datafile_creation = NULL;
1263 static RRDDIM *rd_datafile_deletion = NULL;
1264 static RRDDIM *rd_datafile_deletion_spin = NULL;
1265 static RRDDIM *rd_jv2_indexing = NULL;
1266 static RRDDIM *rd_retention = NULL;
1267
1268 if (unlikely(!st_events)) {
1269 st_events = rrdset_create_localhost(
1270 "netdata",
1271 "dbengine_events",
1272 NULL,
1273 "dbengine query router",
1274 NULL,
1275 "Netdata Database Events",
1276 "events/s",
1277 "netdata",
1278 "pulse",
1279 priority,
1280 localhost->rrd_update_every,
1281 RRDSET_TYPE_LINE);
1282
1283 rd_journal_v2_mapped = rrddim_add(st_events, "journal v2 mapped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1284 rd_journal_v2_unmapped = rrddim_add(st_events, "journal v2 unmapped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1285 rd_datafile_creation = rrddim_add(st_events, "datafile creation", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1286 rd_datafile_deletion = rrddim_add(st_events, "datafile deletion", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1287 rd_datafile_deletion_spin = rrddim_add(st_events, "datafile deletion spin", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1288 rd_jv2_indexing = rrddim_add(st_events, "journal v2 indexing", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1289 rd_retention = rrddim_add(st_events, "retention", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1290 }
1291 priority++;
1292
1293 rrddim_set_by_pointer(st_events, rd_journal_v2_mapped, (collected_number)cache_efficiency_stats.journal_v2_mapped);
1294 rrddim_set_by_pointer(st_events, rd_journal_v2_unmapped, (collected_number)cache_efficiency_stats.journal_v2_unmapped);
1295 rrddim_set_by_pointer(st_events, rd_datafile_creation, (collected_number)cache_efficiency_stats.datafile_creation_started);
1296 rrddim_set_by_pointer(st_events, rd_datafile_deletion, (collected_number)cache_efficiency_stats.datafile_deletion_started);
1297 rrddim_set_by_pointer(st_events, rd_datafile_deletion_spin, (collected_number)cache_efficiency_stats.datafile_deletion_spin);
1298 rrddim_set_by_pointer(st_events, rd_jv2_indexing, (collected_number)cache_efficiency_stats.journal_v2_indexing_started);
1299 rrddim_set_by_pointer(st_events, rd_retention, (collected_number)cache_efficiency_stats.metrics_retention_started);
1300
1301 rrdset_done(st_events);
1302 }
1303
1304 {
1305 static RRDSET *st_prep_timings = NULL;
1306 static RRDDIM *rd_routing_sync = NULL;
1307 static RRDDIM *rd_routing_syncfirst = NULL;
1308 static RRDDIM *rd_routing_async = NULL;
1309 static RRDDIM *rd_main_cache = NULL;
1310 static RRDDIM *rd_open_cache = NULL;
1311 static RRDDIM *rd_journal_v2 = NULL;
1312 static RRDDIM *rd_pass4 = NULL;
1313
1314 if (unlikely(!st_prep_timings)) {
1315 st_prep_timings = rrdset_create_localhost(
1316 "netdata",
1317 "dbengine_prep_timings",
1318 NULL,
1319 "dbengine query router",
1320 NULL,
1321 "Netdata Query Planning Timings",
1322 "usec/s",
1323 "netdata",
1324 "pulse",
1325 priority,
1326 localhost->rrd_update_every,
1327 RRDSET_TYPE_STACKED);
1328
1329 rd_routing_sync = rrddim_add(st_prep_timings, "pdc sync", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1330 rd_routing_syncfirst = rrddim_add(st_prep_timings, "pdc syncfirst", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1331 rd_routing_async = rrddim_add(st_prep_timings, "pdc async", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1332 rd_main_cache = rrddim_add(st_prep_timings, "main cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1333 rd_open_cache = rrddim_add(st_prep_timings, "open cache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1334 rd_journal_v2 = rrddim_add(st_prep_timings, "journal v2", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1335 rd_pass4 = rrddim_add(st_prep_timings, "pass4", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1336 }
1337 priority++;
1338
1339 rrddim_set_by_pointer(st_prep_timings, rd_routing_sync, (collected_number)cache_efficiency_stats.prep_time_to_route_sync.usec);
1340 rrddim_set_by_pointer(st_prep_timings, rd_routing_syncfirst, (collected_number)cache_efficiency_stats.prep_time_to_route_syncfirst.usec);
1341 rrddim_set_by_pointer(st_prep_timings, rd_routing_async, (collected_number)cache_efficiency_stats.prep_time_to_route_async.usec);
1342 rrddim_set_by_pointer(st_prep_timings, rd_main_cache, (collected_number)cache_efficiency_stats.prep_time_in_main_cache_lookup.usec);
1343 rrddim_set_by_pointer(st_prep_timings, rd_open_cache, (collected_number)cache_efficiency_stats.prep_time_in_open_cache_lookup.usec);
1344 rrddim_set_by_pointer(st_prep_timings, rd_journal_v2, (collected_number)cache_efficiency_stats.prep_time_in_journal_v2_lookup.usec);
1345 rrddim_set_by_pointer(st_prep_timings, rd_pass4, (collected_number)cache_efficiency_stats.prep_time_in_pass4_lookup.usec);
1346
1347 rrdset_done(st_prep_timings);
1348 }
1349
1350 {
1351 static RRDSET *st_prep_timings = NULL;
1352 static RRDDIM *rd_routing_sync = NULL;
1353 static RRDDIM *rd_routing_syncfirst = NULL;
1354 static RRDDIM *rd_routing_async = NULL;
1355 static RRDDIM *rd_main_cache = NULL;
1356 static RRDDIM *rd_open_cache = NULL;
1357 static RRDDIM *rd_journal_v2 = NULL;
1358 static RRDDIM *rd_pass4 = NULL;
1359
1360 if (unlikely(!st_prep_timings)) {
1361 st_prep_timings = rrdset_create_localhost(
1362 "netdata",
1363 "dbengine_prep_average_timings",
1364 NULL,
1365 "dbengine query router",
1366 NULL,
1367 "Netdata Query Planning Average Timings",
1368 "usec/s",
1369 "netdata",
1370 "pulse",
1371 priority + 1,
1372 localhost->rrd_update_every,
1373 RRDSET_TYPE_STACKED);
1374
1375 rd_routing_sync = rrddim_add(st_prep_timings, "pdc sync", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1376 rd_routing_syncfirst = rrddim_add(st_prep_timings, "pdc syncfirst", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1377 rd_routing_async = rrddim_add(st_prep_timings, "pdc async", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1378 rd_main_cache = rrddim_add(st_prep_timings, "main cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1379 rd_open_cache = rrddim_add(st_prep_timings, "open cache", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1380 rd_journal_v2 = rrddim_add(st_prep_timings, "journal v2", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1381 rd_pass4 = rrddim_add(st_prep_timings, "pass4", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1382 }
1383 priority++;
1384
1385 rrddim_set_by_pointer(st_prep_timings,rd_routing_sync, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_to_route_sync, &cache_efficiency_stats.prep_time_to_route_sync));
1386 rrddim_set_by_pointer(st_prep_timings,rd_routing_syncfirst, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_to_route_syncfirst, &cache_efficiency_stats.prep_time_to_route_syncfirst));
1387 rrddim_set_by_pointer(st_prep_timings,rd_routing_async, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_to_route_async, &cache_efficiency_stats.prep_time_to_route_async));
1388 rrddim_set_by_pointer(st_prep_timings, rd_main_cache, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_in_main_cache_lookup, &cache_efficiency_stats.prep_time_in_main_cache_lookup));
1389 rrddim_set_by_pointer(st_prep_timings, rd_open_cache, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_in_open_cache_lookup, &cache_efficiency_stats.prep_time_in_open_cache_lookup));
1390 rrddim_set_by_pointer(st_prep_timings, rd_journal_v2, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_in_journal_v2_lookup, &cache_efficiency_stats.prep_time_in_journal_v2_lookup));
1391 rrddim_set_by_pointer(st_prep_timings, rd_pass4, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.prep_time_in_pass4_lookup, &cache_efficiency_stats.prep_time_in_pass4_lookup));
1392
1393 rrdset_done(st_prep_timings);
1394 }
1395
1396 {
1397 static RRDSET *st_query_timings = NULL;
1398 static RRDDIM *rd_init = NULL;
1399 static RRDDIM *rd_prep_wait = NULL;
1400 static RRDDIM *rd_next_page_disk_fast = NULL;
1401 static RRDDIM *rd_next_page_disk_slow = NULL;
1402 static RRDDIM *rd_next_page_preload_fast = NULL;
1403 static RRDDIM *rd_next_page_preload_slow = NULL;
1404
1405 if (unlikely(!st_query_timings)) {
1406 st_query_timings = rrdset_create_localhost(
1407 "netdata",
1408 "dbengine_query_timings",
1409 NULL,
1410 "dbengine query router",
1411 NULL,
1412 "Netdata Query Timings",
1413 "usec/s",
1414 "netdata",
1415 "pulse",
1416 priority,
1417 localhost->rrd_update_every,
1418 RRDSET_TYPE_STACKED);
1419
1420 rd_init = rrddim_add(st_query_timings, "plan", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1421 rd_prep_wait = rrddim_add(st_query_timings, "async wait", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1422 rd_next_page_disk_fast = rrddim_add(st_query_timings, "next page disk fast", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1423 rd_next_page_disk_slow = rrddim_add(st_query_timings, "next page disk slow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1424 rd_next_page_preload_fast = rrddim_add(st_query_timings, "next page preload fast", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1425 rd_next_page_preload_slow = rrddim_add(st_query_timings, "next page preload slow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1426 }
1427 priority++;
1428
1429 rrddim_set_by_pointer(st_query_timings, rd_init, (collected_number)cache_efficiency_stats.query_time_init.usec);
1430 rrddim_set_by_pointer(st_query_timings, rd_prep_wait, (collected_number)cache_efficiency_stats.query_time_wait_for_prep.usec);
1431 rrddim_set_by_pointer(st_query_timings, rd_next_page_disk_fast, (collected_number)cache_efficiency_stats.query_time_to_fast_disk_next_page.usec);
1432 rrddim_set_by_pointer(st_query_timings, rd_next_page_disk_slow, (collected_number)cache_efficiency_stats.query_time_to_slow_disk_next_page.usec);
1433 rrddim_set_by_pointer(st_query_timings, rd_next_page_preload_fast, (collected_number)cache_efficiency_stats.query_time_to_fast_preload_next_page.usec);
1434 rrddim_set_by_pointer(st_query_timings, rd_next_page_preload_slow, (collected_number)cache_efficiency_stats.query_time_to_slow_preload_next_page.usec);
1435
1436 rrdset_done(st_query_timings);
1437 }
1438
1439 {
1440 static RRDSET *st_query_timings_average = NULL;
1441 static RRDDIM *rd_init = NULL;
1442 static RRDDIM *rd_prep_wait = NULL;
1443 static RRDDIM *rd_next_page_disk_fast = NULL;
1444 static RRDDIM *rd_next_page_disk_slow = NULL;
1445 static RRDDIM *rd_next_page_preload_fast = NULL;
1446 static RRDDIM *rd_next_page_preload_slow = NULL;
1447
1448 if (unlikely(!st_query_timings_average)) {
1449 st_query_timings_average = rrdset_create_localhost(
1450 "netdata",
1451 "dbengine_query_timings_average",
1452 NULL,
1453 "dbengine query router",
1454 NULL,
1455 "Netdata Query Average Timings",
1456 "usec/s",
1457 "netdata",
1458 "pulse",
1459 priority + 1,
1460 localhost->rrd_update_every,
1461 RRDSET_TYPE_STACKED);
1462
1463 rd_init = rrddim_add(st_query_timings_average, "plan", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1464 rd_prep_wait = rrddim_add(st_query_timings_average, "async wait", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1465 rd_next_page_disk_fast = rrddim_add(st_query_timings_average, "next page disk fast", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1466 rd_next_page_disk_slow = rrddim_add(st_query_timings_average, "next page disk slow", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1467 rd_next_page_preload_fast = rrddim_add(st_query_timings_average, "next page preload fast", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1468 rd_next_page_preload_slow = rrddim_add(st_query_timings_average, "next page preload slow", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1469 }
1470 priority++;
1471
1472 rrddim_set_by_pointer(st_query_timings_average, rd_init, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.query_time_init, &cache_efficiency_stats.query_time_init));
1473 rrddim_set_by_pointer(st_query_timings_average, rd_prep_wait, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.query_time_wait_for_prep, &cache_efficiency_stats.query_time_wait_for_prep));
1474 rrddim_set_by_pointer(st_query_timings_average, rd_next_page_disk_fast, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.query_time_to_fast_disk_next_page, &cache_efficiency_stats.query_time_to_fast_disk_next_page));
1475 rrddim_set_by_pointer(st_query_timings_average, rd_next_page_disk_slow, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.query_time_to_slow_disk_next_page, &cache_efficiency_stats.query_time_to_slow_disk_next_page));
1476 rrddim_set_by_pointer(st_query_timings_average, rd_next_page_preload_fast, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.query_time_to_fast_preload_next_page, &cache_efficiency_stats.query_time_to_fast_preload_next_page));
1477 rrddim_set_by_pointer(st_query_timings_average, rd_next_page_preload_slow, (collected_number)time_and_count_delta_average(&cache_efficiency_stats_old.query_time_to_slow_preload_next_page, &cache_efficiency_stats.query_time_to_slow_preload_next_page));
1478
1479 rrdset_done(st_query_timings_average);
1480 }
1481
1482 if(netdata_rwlock_tryrdlock(&rrd_rwlock) == 0) {
1483 priority = 135400;
1484
1485 RRDHOST *host;
1486 unsigned long long stats_array[RRDENG_NR_STATS] = {0};
1487 unsigned long long local_stats_array[RRDENG_NR_STATS];
1488 unsigned dbengine_contexts = 0, counted_multihost_db[RRD_STORAGE_TIERS] = { 0 }, i;
1489
1490 rrdhost_foreach_read(host) {
1491 if (!rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
1492
1493 /* get localhost's DB engine's statistics for each tier */
1494 for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++) {
1495 if(host->db[tier].mode != RRD_DB_MODE_DBENGINE) continue;
1496 if(!host->db[tier].si) continue;
1497
1498 if(counted_multihost_db[tier])
1499 continue;
1500 else
1501 counted_multihost_db[tier] = 1;
1502
1503 ++dbengine_contexts;
1504 rrdeng_get_37_statistics((struct rrdengine_instance *)host->db[tier].si, local_stats_array);
1505 for (i = 0; i < RRDENG_NR_STATS; ++i) {
1506 /* aggregate statistics across hosts */
1507 stats_array[i] += local_stats_array[i];
1508 }
1509 }
1510 }
1511 }
1512 rrd_rdunlock();
1513
1514 if (dbengine_contexts) {
1515 /* deduplicate by getting the ones from the last context */
1516 stats_array[30] = local_stats_array[30];
1517 stats_array[31] = local_stats_array[31];
1518 stats_array[32] = local_stats_array[32];
1519 stats_array[34] = local_stats_array[34];
1520 stats_array[36] = local_stats_array[36];
1521
1522 // ----------------------------------------------------------------
1523
1524 {
1525 static RRDSET *st_compression = NULL;
1526 static RRDDIM *rd_savings = NULL;
1527
1528 if (unlikely(!st_compression)) {
1529 st_compression = rrdset_create_localhost(
1530 "netdata",
1531 "dbengine_compression_ratio",
1532 NULL,
1533 "dbengine io",
1534 NULL,
1535 "Netdata DB engine data extents' compression savings ratio",
1536 "percentage",
1537 "netdata",
1538 "pulse",
1539 priority,
1540 localhost->rrd_update_every,
1541 RRDSET_TYPE_LINE);
1542
1543 rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
1544 }
1545 priority++;
1546
1547 unsigned long long ratio;
1548 unsigned long long compressed_content_size = stats_array[12];
1549 unsigned long long content_size = stats_array[11];
1550
1551 if (content_size) {
1552 // allow negative savings
1553 ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
1554 } else {
1555 ratio = 0;
1556 }
1557 rrddim_set_by_pointer(st_compression, rd_savings, ratio);
1558
1559 rrdset_done(st_compression);
1560 }
1561
1562 // ----------------------------------------------------------------
1563
1564 {
1565 static RRDSET *st_io_stats = NULL;
1566 static RRDDIM *rd_reads = NULL;
1567 static RRDDIM *rd_writes = NULL;
1568
1569 if (unlikely(!st_io_stats)) {
1570 st_io_stats = rrdset_create_localhost(
1571 "netdata",
1572 "dbengine_io_throughput",
1573 NULL,
1574 "dbengine io",
1575 NULL,
1576 "Netdata DB engine I/O throughput",
1577 "MiB/s",
1578 "netdata",
1579 "pulse",
1580 priority,
1581 localhost->rrd_update_every,
1582 RRDSET_TYPE_LINE);
1583
1584 rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
1585 rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
1586 }
1587 priority++;
1588
1589 rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[17]);
1590 rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[15]);
1591 rrdset_done(st_io_stats);
1592 }
1593
1594 // ----------------------------------------------------------------
1595
1596 {
1597 static RRDSET *st_io_stats = NULL;
1598 static RRDDIM *rd_reads = NULL;
1599 static RRDDIM *rd_writes = NULL;
1600
1601 if (unlikely(!st_io_stats)) {
1602 st_io_stats = rrdset_create_localhost(
1603 "netdata",
1604 "dbengine_io_operations",
1605 NULL,
1606 "dbengine io",
1607 NULL,
1608 "Netdata DB engine I/O operations",
1609 "operations/s",
1610 "netdata",
1611 "pulse",
1612 priority,
1613 localhost->rrd_update_every,
1614 RRDSET_TYPE_LINE);
1615
1616 rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1617 rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
1618 }
1619 priority++;
1620
1621 rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[18]);
1622 rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]);
1623 rrdset_done(st_io_stats);
1624 }
1625
1626 // ----------------------------------------------------------------
1627
1628 {
1629 static RRDSET *st_errors = NULL;
1630 static RRDDIM *rd_fs_errors = NULL;
1631 static RRDDIM *rd_io_errors = NULL;
1632 static RRDDIM *pg_cache_over_half_dirty_events = NULL;
1633
1634 if (unlikely(!st_errors)) {
1635 st_errors = rrdset_create_localhost(
1636 "netdata",
1637 "dbengine_global_errors",
1638 NULL,
1639 "dbengine io",
1640 NULL,
1641 "Netdata DB engine errors",
1642 "errors/s",
1643 "netdata",
1644 "pulse",
1645 priority,
1646 localhost->rrd_update_every,
1647 RRDSET_TYPE_LINE);
1648
1649 rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1650 rd_fs_errors = rrddim_add(st_errors, "fs_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1651 pg_cache_over_half_dirty_events =
1652 rrddim_add(st_errors, "pg_cache_over_half_dirty_events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1653 }
1654 priority++;
1655
1656 rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
1657 rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
1658 rrddim_set_by_pointer(st_errors, pg_cache_over_half_dirty_events, (collected_number)stats_array[34]);
1659 rrdset_done(st_errors);
1660 }
1661
1662 // ----------------------------------------------------------------
1663
1664 {
1665 static RRDSET *st_fd = NULL;
1666 static RRDDIM *rd_fd_current = NULL;
1667 static RRDDIM *rd_fd_max = NULL;
1668
1669 if (unlikely(!st_fd)) {
1670 st_fd = rrdset_create_localhost(
1671 "netdata",
1672 "dbengine_global_file_descriptors",
1673 NULL,
1674 "dbengine io",
1675 NULL,
1676 "Netdata DB engine File Descriptors",
1677 "descriptors",
1678 "netdata",
1679 "pulse",
1680 priority,
1681 localhost->rrd_update_every,
1682 RRDSET_TYPE_LINE);
1683
1684 rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1685 rd_fd_max = rrddim_add(st_fd, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1686 }
1687 priority++;
1688
1689 rrddim_set_by_pointer(st_fd, rd_fd_current, (collected_number)stats_array[32]);
1690 /* Careful here, modify this accordingly if the File-Descriptor budget ever changes */
1691 rrddim_set_by_pointer(st_fd, rd_fd_max, (collected_number)rlimit_nofile.rlim_cur / 4);
1692 rrdset_done(st_fd);
1693 }
1694 }
1695 }
1696 }
1697
1698 #endif