facets.c · cryptotaxi247/netdata-1

Code Commits Issues Pulls Sessions CI/CD

master

c 2,974 lines 104 KB

Raw

1	// SPDX-License-Identifier: GPL-3.0-or-later
2	#include "facets.h"
3
4	#define FACETS_HISTOGRAM_COLUMNS 150 // the target number of points in a histogram
5	#define FACETS_KEYS_WITH_VALUES_MAX 200 // the max number of keys that can be facets
6	#define FACETS_KEYS_IN_ROW_MAX 500 // the max number of keys in a row
7
8	#define FACETS_KEYS_HASHTABLE_ENTRIES 15
9	#define FACETS_VALUES_HASHTABLE_ENTRIES 15
10
11	static inline void facets_reset_key(FACET_KEY *k);
12
13	// ----------------------------------------------------------------------------
14
15	static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
16	static const uint8_t id_encoding_characters_reverse[256] = {
17	['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3,
18	['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7,
19	['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11,
20	['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
21	['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19,
22	['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
23	['Y'] = 24, ['Z'] = 25, ['.'] = 26, ['a'] = 27,
24	['b'] = 28, ['c'] = 29, ['d'] = 30, ['e'] = 31,
25	['f'] = 32, ['g'] = 33, ['h'] = 34, ['i'] = 35,
26	['j'] = 36, ['k'] = 37, ['l'] = 38, ['m'] = 39,
27	['n'] = 40, ['o'] = 41, ['p'] = 42, ['q'] = 43,
28	['r'] = 44, ['s'] = 45, ['t'] = 46, ['u'] = 47,
29	['v'] = 48, ['w'] = 49, ['x'] = 50, ['y'] = 51,
30	['z'] = 52, ['_'] = 53, ['0'] = 54, ['1'] = 55,
31	['2'] = 56, ['3'] = 57, ['4'] = 58, ['5'] = 59,
32	['6'] = 60, ['7'] = 61, ['8'] = 62, ['9'] = 63
33	};
34
35	#define FACET_STRING_HASH_SIZE 12
36	#define FACETS_HASH XXH64_hash_t
37	#define FACETS_HASH_FUNCTION(src, len) XXH3_64bits(src, len)
38	#define FACETS_HASH_ZERO (FACETS_HASH)0
39	#define FACETS_HASH_UNSAMPLED (FACETS_HASH)(UINT64_MAX - 1)
40	#define FACETS_HASH_ESTIMATED (FACETS_HASH)UINT64_MAX
41
42	static inline void facets_hash_to_str(FACETS_HASH num, char *out) {
43	out[11] = '\0';
44	out[10] = id_encoding_characters[num & 63]; num >>= 6;
45	out[9] = id_encoding_characters[num & 63]; num >>= 6;
46	out[8] = id_encoding_characters[num & 63]; num >>= 6;
47	out[7] = id_encoding_characters[num & 63]; num >>= 6;
48	out[6] = id_encoding_characters[num & 63]; num >>= 6;
49	out[5] = id_encoding_characters[num & 63]; num >>= 6;
50	out[4] = id_encoding_characters[num & 63]; num >>= 6;
51	out[3] = id_encoding_characters[num & 63]; num >>= 6;
52	out[2] = id_encoding_characters[num & 63]; num >>= 6;
53	out[1] = id_encoding_characters[num & 63]; num >>= 6;
54	out[0] = id_encoding_characters[num & 63];
55	}
56
57	static inline FACETS_HASH str_to_facets_hash(const char *str) {
58	FACETS_HASH num = 0;
59	int shifts = 6 * (FACET_STRING_HASH_SIZE - 2);
60
61	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[0])])) << shifts; shifts -= 6;
62	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[1])])) << shifts; shifts -= 6;
63	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[2])])) << shifts; shifts -= 6;
64	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[3])])) << shifts; shifts -= 6;
65	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[4])])) << shifts; shifts -= 6;
66	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[5])])) << shifts; shifts -= 6;
67	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[6])])) << shifts; shifts -= 6;
68	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[7])])) << shifts; shifts -= 6;
69	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[8])])) << shifts; shifts -= 6;
70	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[9])])) << shifts; shifts -= 6;
71	num \|= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[10])])) << shifts;
72
73	return num;
74	}
75
76	static const char *hash_to_static_string(FACETS_HASH hash) {
77	static __thread char hash_str[FACET_STRING_HASH_SIZE];
78	facets_hash_to_str(hash, hash_str);
79	return hash_str;
80	}
81
82	static inline bool is_valid_string_hash(const char *s) {
83	if(strlen(s) != FACET_STRING_HASH_SIZE - 1) {
84	netdata_log_error("The user supplied key '%s' does not have the right length for a facets hash.", s);
85	return false;
86	}
87
88	uint8_t t = (uint8_t )s;
89	while(*t) {
90	if(id_encoding_characters_reverse[t] == 0 && t != id_encoding_characters[0]) {
91	netdata_log_error("The user supplied key '%s' contains invalid characters for a facets hash.", s);
92	return false;
93	}
94
95	t++;
96	}
97
98	return true;
99	}
100
101	// ----------------------------------------------------------------------------
102	// hashtable for FACET_VALUE
103
104	// cleanup hashtable defines
105	#include "../simple_hashtable/simple_hashtable_undef.h"
106
107	struct facet_value;
108	// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value
109	#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_value *
110	#define SIMPLE_HASHTABLE_NAME _VALUE
111	#include "../simple_hashtable/simple_hashtable.h"
112
113	// ----------------------------------------------------------------------------
114	// hashtable for FACET_KEY
115
116	// cleanup hashtable defines
117	#include "../simple_hashtable/simple_hashtable_undef.h"
118
119	struct facet_key;
120	// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key
121	#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_key *
122	#define SIMPLE_HASHTABLE_NAME _KEY
123	#include "../simple_hashtable/simple_hashtable.h"
124
125	// ----------------------------------------------------------------------------
126
127	typedef struct facet_value {
128	FACETS_HASH hash;
129	const char *name;
130	const char *color;
131	uint32_t name_len;
132
133	bool selected;
134	bool empty;
135	bool unsampled;
136	bool estimated;
137
138	uint32_t rows_matching_facet_value;
139	uint32_t final_facet_value_counter;
140	uint32_t order;
141
142	uint32_t *histogram;
143	uint32_t min, max, sum;
144
145	struct facet_value prev, next;
146	} FACET_VALUE;
147
148	typedef enum {
149	FACET_KEY_VALUE_NONE = 0,
150	FACET_KEY_VALUE_UPDATED = (1 << 0),
151	FACET_KEY_VALUE_EMPTY = (1 << 1),
152	FACET_KEY_VALUE_UNSAMPLED = (1 << 2),
153	FACET_KEY_VALUE_ESTIMATED = (1 << 3),
154	FACET_KEY_VALUE_COPIED = (1 << 4),
155	} FACET_KEY_VALUE_FLAGS;
156
157	#define facet_key_value_updated(k) ((k)->current_value.flags & FACET_KEY_VALUE_UPDATED)
158	#define facet_key_value_empty(k) ((k)->current_value.flags & FACET_KEY_VALUE_EMPTY)
159	#define facet_key_value_unsampled(k) ((k)->current_value.flags & FACET_KEY_VALUE_UNSAMPLED)
160	#define facet_key_value_estimated(k) ((k)->current_value.flags & FACET_KEY_VALUE_ESTIMATED)
161	#define facet_key_value_empty_or_unsampled_or_estimated(k) ((k)->current_value.flags & (FACET_KEY_VALUE_EMPTY\|FACET_KEY_VALUE_UNSAMPLED\|FACET_KEY_VALUE_ESTIMATED))
162	#define facet_key_value_copied(k) ((k)->current_value.flags & FACET_KEY_VALUE_COPIED)
163
164	struct facet_key {
165	FACETS *facets;
166
167	FACETS_HASH hash;
168	const char *name;
169
170	FACET_KEY_OPTIONS options;
171
172	bool default_selected_for_values; // the default "selected" for all values in the dictionary
173
174	// members about the current row
175	uint32_t key_found_in_row;
176	uint32_t key_values_selected_in_row;
177	uint32_t order;
178
179	struct {
180	bool enabled;
181	uint32_t used;
182	FACET_VALUE *ll;
183	SIMPLE_HASHTABLE_VALUE ht;
184	} values;
185
186	struct {
187	FACETS_HASH hash;
188	FACET_KEY_VALUE_FLAGS flags;
189	const char *raw;
190	uint32_t raw_len;
191	BUFFER *b;
192	FACET_VALUE *v;
193	} current_value;
194
195	struct {
196	FACET_VALUE *v;
197	} empty_value;
198
199	struct {
200	FACET_VALUE *v;
201	} unsampled_value;
202
203	struct {
204	FACET_VALUE *v;
205	} estimated_value;
206
207	struct {
208	facet_dynamic_row_t cb;
209	void *data;
210	} dynamic;
211
212	struct {
213	bool view_only;
214	facets_key_transformer_t cb;
215	void *data;
216	} transform;
217
218	struct facet_key prev, next;
219	};
220
221	struct facets {
222	SIMPLE_PATTERN *visible_keys;
223	SIMPLE_PATTERN *excluded_keys;
224	SIMPLE_PATTERN *included_keys;
225	bool all_keys_included_by_default;
226
227	FACETS_OPTIONS options;
228
229	struct {
230	usec_t start_ut;
231	usec_t stop_ut;
232	FACETS_ANCHOR_DIRECTION direction;
233	} anchor;
234
235	SIMPLE_PATTERN *query; // the full text search pattern
236	size_t keys_filtered_by_query; // the number of fields we do full text search (constant)
237
238	DICTIONARY *accepted_params;
239
240	struct {
241	size_t count;
242	FACET_KEY *ll;
243	SIMPLE_HASHTABLE_KEY ht;
244	} keys;
245
246	struct {
247	// this is like a stack, of the keys that are used as facets
248	size_t used;
249	FACET_KEY *array[FACETS_KEYS_WITH_VALUES_MAX];
250	} keys_with_values;
251
252	struct {
253	// this is like a stack, of the keys that need to clean up between each row
254	size_t used;
255	FACET_KEY *array[FACETS_KEYS_IN_ROW_MAX];
256	} keys_in_row;
257
258	FACET_ROW *base; // double linked list of the selected facets rows
259	FACET_ROW_BIN_DATA bin_data;
260
261	uint32_t items_to_return;
262	uint32_t max_items_to_return;
263	uint32_t order;
264
265	struct {
266	FACET_ROW_SEVERITY severity;
267	size_t keys_matched_by_query_positive; // the number of fields matched the full text search (per row)
268	size_t keys_matched_by_query_negative; // the number of fields matched the full text search (per row)
269	} current_row;
270
271	struct {
272	usec_t after_ut;
273	usec_t before_ut;
274	} timeframe;
275
276	struct {
277	FACET_KEY *key;
278	FACETS_HASH hash;
279	char *chart;
280	bool enabled;
281	uint32_t slots;
282	usec_t slot_width_ut;
283	usec_t after_ut;
284	usec_t before_ut;
285	} histogram;
286
287	struct {
288	facet_row_severity_t cb;
289	void *data;
290	} severity;
291
292	struct {
293	FACET_ROW *last_added;
294
295	size_t first;
296	size_t forwards;
297	size_t backwards;
298	size_t skips_before;
299	size_t skips_after;
300	size_t prepends;
301	size_t appends;
302	size_t shifts;
303
304	struct {
305	size_t evaluated;
306	size_t matched;
307	size_t unsampled;
308	size_t estimated;
309	size_t created;
310	size_t reused;
311	} rows;
312
313	struct {
314	size_t registered;
315	size_t unique;
316	} keys;
317
318	struct {
319	size_t registered;
320	size_t transformed;
321	size_t dynamic;
322	size_t empty;
323	size_t unsampled;
324	size_t estimated;
325	size_t indexed;
326	size_t inserts;
327	size_t conflicts;
328	} values;
329
330	struct {
331	size_t searches;
332	} fts;
333
334	struct {
335	size_t bin_data_inflight;
336	};
337	} operations;
338
339	struct {
340	DICTIONARY *used_hashes_registry;
341	} report;
342	};
343
344	usec_t facets_row_oldest_ut(FACETS *facets) {
345	if(facets->base)
346	return facets->base->prev->usec;
347
348	return 0;
349	}
350
351	usec_t facets_row_newest_ut(FACETS *facets) {
352	if(facets->base)
353	return facets->base->usec;
354
355	return 0;
356	}
357
358	uint32_t facets_rows(FACETS *facets) {
359	return facets->items_to_return;
360	}
361
362	static const char facets_key_id(FACET_KEY k) {
363	if(k->facets->options & FACETS_OPTION_HASH_IDS)
364	return hash_to_static_string(k->hash);
365	else
366	return k->name ? k->name : hash_to_static_string(k->hash);
367	}
368
369	static const char facets_key_value_id(FACET_KEY k, FACET_VALUE *v) {
370	if(k->facets->options & FACETS_OPTION_HASH_IDS)
371	return hash_to_static_string(v->hash);
372	else
373	return v->name ? v->name : hash_to_static_string(v->hash);
374	}
375
376	void facets_use_hashes_for_ids(FACETS *facets, bool set) {
377	if(set)
378	facets->options \|= FACETS_OPTION_HASH_IDS;
379	else
380	facets->options &= ~(FACETS_OPTION_HASH_IDS);
381	}
382
383	// ----------------------------------------------------------------------------
384
385	static void facets_row_free(FACETS facets __maybe_unused, FACET_ROW row);
386	static inline void facet_value_is_used(FACET_KEY k, FACET_VALUE v);
387	static inline bool facets_key_is_facet(FACETS facets, FACET_KEY k);
388
389	// ----------------------------------------------------------------------------
390	// The FACET_VALUE index within each FACET_KEY
391
392	#define foreach_value_in_key(k, v) \
393	for((v) = (k)->values.ll; (v) ;(v) = (v)->next)
394
395	#define foreach_value_in_key_done(v) do { ; } while(0)
396
397	static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) {
398	k->values.ll = NULL;
399	k->values.used = 0;
400	simple_hashtable_init_VALUE(&k->values.ht, FACETS_VALUES_HASHTABLE_ENTRIES);
401	}
402
403	static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) {
404	FACET_VALUE *v = k->values.ll;
405	while(v) {
406	FACET_VALUE *next = v->next;
407	freez(v->histogram);
408	freez((void *)v->name);
409	freez(v);
410	v = next;
411	}
412	k->values.ll = NULL;
413	k->values.used = 0;
414	k->values.enabled = false;
415
416	simple_hashtable_destroy_VALUE(&k->values.ht);
417	}
418
419	static inline const char facets_key_get_value(FACET_KEY k) {
420	return facet_key_value_copied(k) ? buffer_tostring(k->current_value.b) : k->current_value.raw;
421	}
422
423	static inline uint32_t facets_key_get_value_length(FACET_KEY *k) {
424	return facet_key_value_copied(k) ? buffer_strlen(k->current_value.b) : k->current_value.raw_len;
425	}
426
427	static inline void facets_key_value_copy_to_buffer(FACET_KEY *k) {
428	if(!facet_key_value_copied(k)) {
429	buffer_contents_replace(k->current_value.b, k->current_value.raw, k->current_value.raw_len);
430	k->current_value.flags \|= FACET_KEY_VALUE_COPIED;
431	}
432	}
433
434	static const char facets_value_dup(const char s, uint32_t len) {
435	char *d = mallocz(len + 1);
436
437	if(len)
438	memcpy(d, s, len);
439
440	d[len] = '\0';
441
442	return d;
443	}
444
445	static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY k, FACET_VALUE v, const FACET_VALUE * const nv) {
446	if(!v->name && !v->name_len && nv->name && nv->name_len) {
447	// an actual value, not a filter
448	v->name = facets_value_dup(nv->name, nv->name_len);
449	v->name_len = nv->name_len;
450	}
451
452	if(v->name && v->name_len)
453	facet_value_is_used(k, v);
454
455	internal_fatal(v->name && nv->name && v->name_len == nv->name_len && memcmp(v->name, nv->name, v->name_len) != 0,
456	"value hash conflict: '%s' and '%s' have the same hash '%s'",
457	v->name, nv->name, hash_to_static_string(v->hash));
458
459	k->facets->operations.values.conflicts++;
460	}
461
462	static inline FACET_VALUE FACET_VALUE_GET_FROM_INDEX(FACET_KEY k, FACETS_HASH hash) {
463	SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, NULL, true);
464	return SIMPLE_HASHTABLE_SLOT_DATA(slot);
465	}
466
467	static inline FACET_VALUE FACET_VALUE_ADD_TO_INDEX(FACET_KEY k, const FACET_VALUE * const tv) {
468	SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, NULL, true);
469
470	if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) {
471	// already exists
472
473	FACET_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
474	FACET_VALUE_ADD_CONFLICT(k, v, tv);
475	return v;
476	}
477
478	// we have to add it
479
480	FACET_VALUE v = mallocz(sizeof(v));
481	simple_hashtable_set_slot_VALUE(&k->values.ht, slot, tv->hash, v);
482
483	memcpy(v, tv, sizeof(*v));
484
485	if(v->estimated \|\| v->unsampled) {
486	if(k->values.ll && k->values.ll->estimated) {
487	FACET_VALUE *estimated = k->values.ll;
488	DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(k->values.ll, estimated, v, prev, next);
489	}
490	else
491	DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(k->values.ll, v, prev, next);
492	}
493	else
494	DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(k->values.ll, v, prev, next);
495
496	k->values.used++;
497
498	if(!v->selected)
499	v->selected = k->default_selected_for_values;
500
501	if(v->name && v->name_len) {
502	// an actual value, not a filter
503	v->name = facets_value_dup(v->name, v->name_len);
504	facet_value_is_used(k, v);
505	}
506	else {
507	v->name = NULL;
508	v->name_len = 0;
509	}
510
511	k->facets->operations.values.inserts++;
512
513	return v;
514	}
515
516	static inline void FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(FACET_KEY *k) {
517	static const FACET_VALUE tv = {
518	.hash = FACETS_HASH_UNSAMPLED,
519	.name = FACET_VALUE_UNSAMPLED,
520	.name_len = sizeof(FACET_VALUE_UNSAMPLED) - 1,
521	.unsampled = true,
522	.color = "offline",
523	};
524
525	k->current_value.hash = FACETS_HASH_UNSAMPLED;
526
527	if(k->unsampled_value.v) {
528	FACET_VALUE_ADD_CONFLICT(k, k->unsampled_value.v, &tv);
529	k->current_value.v = k->unsampled_value.v;
530	}
531	else {
532	FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
533	v->unsampled = true;
534	k->unsampled_value.v = v;
535	k->current_value.v = v;
536	}
537	}
538
539	static inline void FACET_VALUE_ADD_ESTIMATED_VALUE_TO_INDEX(FACET_KEY *k) {
540	static const FACET_VALUE tv = {
541	.hash = FACETS_HASH_ESTIMATED,
542	.name = FACET_VALUE_ESTIMATED,
543	.name_len = sizeof(FACET_VALUE_ESTIMATED) - 1,
544	.estimated = true,
545	.color = "generic",
546	};
547
548	k->current_value.hash = FACETS_HASH_ESTIMATED;
549
550	if(k->estimated_value.v) {
551	FACET_VALUE_ADD_CONFLICT(k, k->estimated_value.v, &tv);
552	k->current_value.v = k->estimated_value.v;
553	}
554	else {
555	FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
556	v->estimated = true;
557	k->estimated_value.v = v;
558	k->current_value.v = v;
559	}
560	}
561
562	static inline void FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(FACET_KEY *k) {
563	static const FACET_VALUE tv = {
564	.hash = FACETS_HASH_ZERO,
565	.name = FACET_VALUE_UNSET,
566	.name_len = sizeof(FACET_VALUE_UNSET) - 1,
567	.empty = true,
568	};
569
570	k->current_value.hash = FACETS_HASH_ZERO;
571
572	if(k->empty_value.v) {
573	FACET_VALUE_ADD_CONFLICT(k, k->empty_value.v, &tv);
574	k->current_value.v = k->empty_value.v;
575	}
576	else {
577	FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
578	v->empty = true;
579	k->empty_value.v = v;
580	k->current_value.v = v;
581	}
582	}
583
584	static inline void FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(FACET_KEY *k) {
585	static __thread FACET_VALUE tv = { 0 };
586
587	internal_fatal(!facet_key_value_updated(k), "trying to add a non-updated value to the index");
588
589	tv.name = facets_key_get_value(k);
590	tv.name_len = facets_key_get_value_length(k);
591	tv.hash = FACETS_HASH_FUNCTION(tv.name, tv.name_len);
592	tv.empty = false;
593	tv.estimated = false;
594	tv.unsampled = false;
595
596	k->current_value.v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
597	k->facets->operations.values.indexed++;
598	}
599
600	static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY k, const char name, FACETS_HASH hash) {
601	FACET_VALUE tv = {
602	.hash = hash,
603	.selected = true,
604	.name = name,
605	.name_len = name ? strlen(name) : 0,
606	};
607	FACET_VALUE_ADD_TO_INDEX(k, &tv);
608	}
609
610	// ----------------------------------------------------------------------------
611	// The FACET_KEY index within each FACET
612
613	#define foreach_key_in_facets(facets, k) \
614	for((k) = (facets)->keys.ll; (k) ;(k) = (k)->next)
615
616	#define foreach_key_in_facets_done(k) do { ; } while(0)
617
618	static inline void facet_key_late_init(FACETS facets, FACET_KEY k) {
619	if(k->values.enabled)
620	return;
621
622	if(facets_key_is_facet(facets, k)) {
623	FACETS_VALUES_INDEX_CREATE(k);
624	k->values.enabled = true;
625	if(facets->keys_with_values.used < FACETS_KEYS_WITH_VALUES_MAX)
626	facets->keys_with_values.array[facets->keys_with_values.used++] = k;
627	}
628	}
629
630	static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) {
631	facets->keys.ll = NULL;
632	facets->keys.count = 0;
633	facets->keys_with_values.used = 0;
634
635	simple_hashtable_init_KEY(&facets->keys.ht, FACETS_KEYS_HASHTABLE_ENTRIES);
636	}
637
638	static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
639	FACET_KEY *k = facets->keys.ll;
640	while(k) {
641	FACET_KEY *next = k->next;
642
643	FACETS_VALUES_INDEX_DESTROY(k);
644	buffer_free(k->current_value.b);
645	freez((void *)k->name);
646	freez(k);
647
648	k = next;
649	}
650	facets->keys.ll = NULL;
651	facets->keys.count = 0;
652	facets->keys_with_values.used = 0;
653
654	simple_hashtable_destroy_KEY(&facets->keys.ht);
655	}
656
657	static inline FACET_KEY FACETS_KEY_GET_FROM_INDEX(FACETS facets, FACETS_HASH hash) {
658	SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true);
659	return SIMPLE_HASHTABLE_SLOT_DATA(slot);
660	}
661
662	bool facets_key_name_value_length_is_selected(FACETS facets, const char key, size_t key_length, const char *value, size_t value_length) {
663	FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
664	FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
665	if(!k \|\| k->default_selected_for_values)
666	return false;
667
668	hash = FACETS_HASH_FUNCTION(value, value_length);
669	FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash);
670	return (v && v->selected) ? true : false;
671	}
672
673	bool facets_foreach_selected_value_in_key(FACETS facets, const char key, size_t key_length, DICTIONARY used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void data) {
674	FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
675	FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
676	if(!k \|\| k->default_selected_for_values)
677	return false;
678
679	size_t selected = 0;
680	for(FACET_VALUE *v = k->values.ll; v ;v = v->next) {
681	if(!v->selected) continue;
682
683	const char *value = v->name;
684	if(!value) {
685	if(used_hashes_registry) {
686	char hash_str[FACET_STRING_HASH_SIZE];
687	facets_hash_to_str(v->hash, hash_str);
688	value = dictionary_get(used_hashes_registry, hash_str);
689	}
690
691	if(!value)
692	return false;
693	}
694
695	if(!cb(facets, selected++, k->name, value, data))
696	return false;
697	}
698
699	return selected > 0;
700	}
701
702	void facets_add_possible_value_name_to_key(FACETS facets, const char key, size_t key_length, const char *value, size_t value_length) {
703	FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
704	FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
705	if(!k) return;
706
707	hash = FACETS_HASH_FUNCTION(value, value_length);
708	FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash);
709	if(v && v->name && v->name_len) return;
710
711	FACET_VALUE tv = {
712	.hash = hash,
713	.name = value,
714	.name_len = value_length,
715	};
716	FACET_VALUE_ADD_TO_INDEX(k, &tv);
717	}
718
719	static void facet_key_set_name(FACET_KEY k, const char name, size_t name_length) {
720	internal_fatal(k->name && name && (strncmp(k->name, name, name_length) != 0 \|\| k->name[name_length] != '\0'),
721	"key hash conflict: '%s' and '%s' have the same hash",
722	k->name, name);
723
724	if(likely(k->name \|\| !name \|\| !name_length))
725	return;
726
727	// an actual value, not a filter
728
729	char *name_copy = callocz(name_length + 1, sizeof(char));
730	memcpy(name_copy, name, name_length);
731
732	internal_fatal(strchr(name_copy, '='), "found = in key");
733
734	k->name = name_copy;
735	facet_key_late_init(k->facets, k);
736	}
737
738	static inline FACET_KEY FACETS_KEY_CREATE(FACETS facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
739	facets->operations.keys.unique++;
740
741	FACET_KEY k = callocz(1, sizeof(k));
742
743	k->hash = hash;
744	k->facets = facets;
745	k->options = options;
746	k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL);
747	k->default_selected_for_values = true;
748
749	if(unlikely((k->options & (FACET_KEY_OPTION_REORDER \| FACET_KEY_OPTION_REORDER_DONE)) == 0))
750	k->order = facets->order++;
751
752	if((k->options & FACET_KEY_OPTION_FTS) \|\| (facets->options & FACETS_OPTION_ALL_KEYS_FTS))
753	facets->keys_filtered_by_query++;
754
755	facet_key_set_name(k, name, name_length);
756
757	DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys.ll, k, prev, next);
758	facets->keys.count++;
759
760	return k;
761	}
762
763	static inline FACET_KEY FACETS_KEY_ADD_TO_INDEX(FACETS facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
764	facets->operations.keys.registered++;
765
766	SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true);
767
768	if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
769	// we have to add it
770	FACET_KEY *k = FACETS_KEY_CREATE(facets, hash, name, name_length, options);
771
772	simple_hashtable_set_slot_KEY(&facets->keys.ht, slot, hash, k);
773
774	return k;
775	}
776
777	// already in the index
778
779	FACET_KEY *k = SIMPLE_HASHTABLE_SLOT_DATA(slot);
780
781	facet_key_set_name(k, name, name_length);
782	k->options \|= options;
783
784	if(unlikely((k->options & (FACET_KEY_OPTION_REORDER \| FACET_KEY_OPTION_REORDER_DONE)) == FACET_KEY_OPTION_REORDER)) {
785	k->order = facets->order++;
786	k->options \|= FACET_KEY_OPTION_REORDER_DONE;
787	}
788
789	return k;
790	}
791
792	bool facets_key_name_is_filter(FACETS facets, const char key) {
793	FACETS_HASH hash = FACETS_HASH_FUNCTION(key, strlen(key));
794	FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
795	return (!k \|\| k->default_selected_for_values) ? false : true;
796	}
797
798	bool facets_key_name_is_facet(FACETS facets, const char key) {
799	size_t key_len = strlen(key);
800	FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_len);
801	FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, key, key_len, 0);
802	return (k && (k->options & FACET_KEY_OPTION_FACET));
803	}
804
805	// ----------------------------------------------------------------------------
806
807	size_t facets_histogram_slots(FACETS *facets) {
808	return facets->histogram.slots;
809	}
810
811	static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) {
812	// Array of valid durations in seconds
813	static time_t valid_durations_s[] = {
814	1, 2, 5, 10, 15, 30, // seconds
815	1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
816	1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
817	1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
818	1 * (30*86400) // months
819	};
820	static int array_size = sizeof(valid_durations_s) / sizeof(valid_durations_s[0]);
821
822	usec_t duration_ut = before_ut - after_ut;
823	usec_t bar_width_ut = 1 * USEC_PER_SEC;
824
825	for (int i = array_size - 1; i >= 0; --i) {
826	if (duration_ut / (valid_durations_s[i] * USEC_PER_SEC) >= FACETS_HISTOGRAM_COLUMNS) {
827	bar_width_ut = valid_durations_s[i] * USEC_PER_SEC;
828	break;
829	}
830	}
831
832	return bar_width_ut;
833	}
834
835	static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) {
836	usec_t delta_ut = ut % facets->histogram.slot_width_ut;
837	return ut - delta_ut;
838	}
839
840	void facets_set_timeframe_and_histogram_by_id(FACETS facets, const char key_id, usec_t after_ut, usec_t before_ut) {
841	if(after_ut > before_ut) {
842	usec_t t = after_ut;
843	after_ut = before_ut;
844	before_ut = t;
845	}
846
847	facets->histogram.enabled = true;
848
849	if(key_id && *key_id && strlen(key_id) == FACET_STRING_HASH_SIZE - 1) {
850	facets->histogram.chart = strdupz(key_id);
851	facets->histogram.hash = str_to_facets_hash(facets->histogram.chart);
852	}
853	else {
854	freez(facets->histogram.chart);
855	facets->histogram.chart = NULL;
856	facets->histogram.hash = FACETS_HASH_ZERO;
857	}
858
859	facets->timeframe.after_ut = after_ut;
860	facets->timeframe.before_ut = before_ut;
861
862	facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut);
863	facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut);
864	facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut;
865	facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1;
866
867	internal_fatal(after_ut < facets->histogram.after_ut, "histogram after_ut is not less or equal to wanted after_ut");
868	internal_fatal(before_ut > facets->histogram.before_ut, "histogram before_ut is not more or equal to wanted before_ut");
869
870	if(facets->histogram.slots > 1000) {
871	facets->histogram.slots = 1000 + 1;
872	facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000;
873	}
874	}
875
876	void facets_set_timeframe_and_histogram_by_name(FACETS facets, const char key_name, usec_t after_ut, usec_t before_ut) {
877	char hash_str[FACET_STRING_HASH_SIZE];
878	FACETS_HASH hash = FACETS_HASH_FUNCTION(key_name, strlen(key_name));
879	facets_hash_to_str(hash, hash_str);
880	facets_set_timeframe_and_histogram_by_id(facets, hash_str, after_ut, before_ut);
881	}
882
883	static inline uint32_t facets_histogram_slot_at_time_ut(FACETS facets, usec_t usec, FACET_VALUE v) {
884	if(unlikely(!v->histogram))
885	v->histogram = callocz(facets->histogram.slots, sizeof(*v->histogram));
886
887	usec_t base_ut = facets_histogram_slot_baseline_ut(facets, usec);
888
889	if(unlikely(base_ut < facets->histogram.after_ut))
890	base_ut = facets->histogram.after_ut;
891
892	if(unlikely(base_ut > facets->histogram.before_ut))
893	base_ut = facets->histogram.before_ut;
894
895	uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut;
896
897	if(unlikely(slot >= facets->histogram.slots))
898	slot = facets->histogram.slots - 1;
899
900	return slot;
901	}
902
903	static inline void facets_histogram_update_value_slot(FACETS facets, usec_t usec, FACET_VALUE v) {
904	uint32_t slot = facets_histogram_slot_at_time_ut(facets, usec, v);
905	v->histogram[slot]++;
906	}
907
908	static inline void facets_histogram_update_value(FACETS *facets, usec_t usec) {
909	if(!facets->histogram.enabled \|\|
910	!facets->histogram.key \|\|
911	!facets->histogram.key->values.enabled \|\|
912	!facet_key_value_updated(facets->histogram.key) \|\|
913	usec < facets->histogram.after_ut \|\|
914	usec > facets->histogram.before_ut)
915	return;
916
917	FACET_VALUE *v = facets->histogram.key->current_value.v;
918	facets_histogram_update_value_slot(facets, usec, v);
919	}
920
921	static usec_t overlap_duration_ut(usec_t start1, usec_t end1, usec_t start2, usec_t end2) {
922	usec_t overlap_start = MAX(start1, start2);
923	usec_t overlap_end = MIN(end1, end2);
924
925	if (overlap_start < overlap_end)
926	return overlap_end - overlap_start;
927	else
928	return 0; // No overlap
929	}
930
931	void facets_update_estimations(FACETS *facets, usec_t from_ut, usec_t to_ut, size_t entries) {
932	if(unlikely(!facets->histogram.enabled))
933	return;
934
935	if(unlikely(!overlap_duration_ut(facets->histogram.after_ut, facets->histogram.before_ut, from_ut, to_ut)))
936	return;
937
938	facets->operations.rows.evaluated += entries;
939	facets->operations.rows.matched += entries;
940	facets->operations.rows.estimated += entries;
941
942	if (!facets->histogram.enabled \|\|
943	!facets->histogram.key \|\|
944	!facets->histogram.key->values.enabled)
945	return;
946
947	if (from_ut < facets->histogram.after_ut)
948	from_ut = facets->histogram.after_ut;
949
950	if (to_ut > facets->histogram.before_ut)
951	to_ut = facets->histogram.before_ut;
952
953	if (!facets->histogram.key->estimated_value.v)
954	FACET_VALUE_ADD_ESTIMATED_VALUE_TO_INDEX(facets->histogram.key);
955
956	FACET_VALUE *v = facets->histogram.key->estimated_value.v;
957
958	size_t slots = 0;
959	size_t total_ut = to_ut - from_ut;
960	ssize_t remaining_entries = (ssize_t)entries;
961	size_t slot = facets_histogram_slot_at_time_ut(facets, from_ut, v);
962	for(; slot < facets->histogram.slots ;slot++) {
963	usec_t slot_start_ut = facets->histogram.after_ut + slot * facets->histogram.slot_width_ut;
964	usec_t slot_end_ut = slot_start_ut + facets->histogram.slot_width_ut;
965
966	if(slot_start_ut > to_ut)
967	break;
968
969	usec_t overlap_ut = overlap_duration_ut(from_ut, to_ut, slot_start_ut, slot_end_ut);
970
971	size_t slot_entries = (overlap_ut * entries) / total_ut;
972	v->histogram[slot] += slot_entries;
973	remaining_entries -= (ssize_t)slot_entries;
974	slots++;
975	}
976
977	// Check if all entries are assigned
978	// This should always be true if the distribution is correct
979	internal_fatal(remaining_entries < 0 \|\| remaining_entries >= (ssize_t)(slots),
980	"distribution of estimations is not accurate - there are %zd remaining entries",
981	remaining_entries);
982	}
983
984	void facets_row_finished_unsampled(FACETS *facets, usec_t usec) {
985	facets->operations.rows.evaluated++;
986	facets->operations.rows.matched++;
987	facets->operations.rows.unsampled++;
988
989	if(!facets->histogram.enabled \|\|
990	!facets->histogram.key \|\|
991	!facets->histogram.key->values.enabled \|\|
992	usec < facets->histogram.after_ut \|\|
993	usec > facets->histogram.before_ut)
994	return;
995
996	if(!facets->histogram.key->unsampled_value.v)
997	FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(facets->histogram.key);
998
999	FACET_VALUE *v = facets->histogram.key->unsampled_value.v;
1000	facets_histogram_update_value_slot(facets, usec, v);
1001
1002	facets_reset_key(facets->histogram.key);
1003	}
1004
1005	static const char facets_key_name_cached(FACET_KEY k, DICTIONARY *used_hashes_registry) {
1006	if(k->name) {
1007	if(used_hashes_registry && !k->default_selected_for_values) {
1008	char hash_str[FACET_STRING_HASH_SIZE];
1009	facets_hash_to_str(k->hash, hash_str);
1010	dictionary_set(used_hashes_registry, hash_str, (void *)k->name, strlen(k->name) + 1);
1011	}
1012
1013	return k->name;
1014	}
1015
1016	// key has no name
1017	const char *name = "[UNAVAILABLE_FIELD]";
1018
1019	if(used_hashes_registry) {
1020	char hash_str[FACET_STRING_HASH_SIZE];
1021	facets_hash_to_str(k->hash, hash_str);
1022	const char *s = dictionary_get(used_hashes_registry, hash_str);
1023	if(s) name = s;
1024	}
1025
1026	return name;
1027	}
1028
1029	static const char facets_key_value_cached(FACET_KEY k, FACET_VALUE v, DICTIONARY used_hashes_registry) {
1030	if(v->empty \|\| v->estimated \|\| v->unsampled)
1031	return v->name;
1032
1033	if(v->name && v->name_len) {
1034	if(used_hashes_registry && !k->default_selected_for_values && v->selected) {
1035	char hash_str[FACET_STRING_HASH_SIZE];
1036	facets_hash_to_str(v->hash, hash_str);
1037	dictionary_set(used_hashes_registry, hash_str, (void *)v->name, v->name_len + 1);
1038	}
1039
1040	return v->name;
1041	}
1042
1043	// key has no name
1044	const char *name = "[unavailable field]";
1045
1046	if(used_hashes_registry) {
1047	char hash_str[FACET_STRING_HASH_SIZE];
1048	facets_hash_to_str(v->hash, hash_str);
1049	const char *s = dictionary_get(used_hashes_registry, hash_str);
1050	if(s) name = s;
1051	}
1052
1053	return name;
1054	}
1055
1056	static inline void facets_key_value_transformed(FACETS facets, FACET_KEY k, FACET_VALUE v, BUFFER dst, FACETS_TRANSFORMATION_SCOPE scope) {
1057	buffer_flush(dst);
1058
1059	if(v->empty \|\| v->unsampled \|\| v->estimated)
1060	buffer_strcat(dst, v->name);
1061	else if(k->transform.cb && k->transform.view_only) {
1062	buffer_contents_replace(dst, v->name, v->name_len);
1063	k->transform.cb(facets, dst, scope, k->transform.data);
1064	}
1065	else
1066	buffer_strcat(dst, facets_key_value_cached(k, v, facets->report.used_hashes_registry));
1067	}
1068
1069	static inline void facets_histogram_value_ids(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key, const char *first_key) {
1070	CLEAN_BUFFER *tb = buffer_create(0, NULL);
1071
1072	buffer_json_member_add_array(wb, key);
1073	{
1074	if(first_key)
1075	buffer_json_add_array_item_string(wb, first_key);
1076
1077	if(k && k->values.enabled) {
1078	FACET_VALUE *v;
1079	foreach_value_in_key(k, v) {
1080	buffer_json_add_array_item_string(wb, facets_key_value_id(k ,v));
1081	}
1082	foreach_value_in_key_done(v);
1083	}
1084	}
1085	buffer_json_array_close(wb); // key
1086	}
1087
1088	static inline void facets_histogram_value_names(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key, const char *first_key) {
1089	CLEAN_BUFFER *tb = buffer_create(0, NULL);
1090
1091	buffer_json_member_add_array(wb, key);
1092	{
1093	if(first_key)
1094	buffer_json_add_array_item_string(wb, first_key);
1095
1096	if(k && k->values.enabled) {
1097	FACET_VALUE *v;
1098	foreach_value_in_key(k, v) {
1099	facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_HISTOGRAM);
1100	buffer_json_add_array_item_string(wb, buffer_tostring(tb));
1101	}
1102	foreach_value_in_key_done(v);
1103	}
1104	}
1105	buffer_json_array_close(wb); // key
1106	}
1107
1108	static inline void facets_histogram_value_colors(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key) {
1109	buffer_json_member_add_array(wb, key);
1110	{
1111	if(k && k->values.enabled) {
1112	FACET_VALUE *v;
1113	foreach_value_in_key(k, v) {
1114	buffer_json_add_array_item_string(wb, v->color);
1115	}
1116	foreach_value_in_key_done(v);
1117	}
1118	}
1119	buffer_json_array_close(wb); // key
1120	}
1121
1122	static inline void facets_histogram_value_units(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key) {
1123	buffer_json_member_add_array(wb, key);
1124	{
1125	if(k && k->values.enabled) {
1126	FACET_VALUE *v;
1127	foreach_value_in_key(k, v) {
1128	buffer_json_add_array_item_string(wb, "events");
1129	}
1130	foreach_value_in_key_done(v);
1131	}
1132	}
1133	buffer_json_array_close(wb); // key
1134	}
1135
1136	static inline void facets_histogram_value_min(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key) {
1137	buffer_json_member_add_array(wb, key);
1138	{
1139	if(k && k->values.enabled) {
1140	FACET_VALUE *v;
1141	foreach_value_in_key(k, v) {
1142	buffer_json_add_array_item_uint64(wb, v->min);
1143	}
1144	foreach_value_in_key_done(v);
1145	}
1146	}
1147	buffer_json_array_close(wb); // key
1148	}
1149
1150	static inline void facets_histogram_value_max(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key) {
1151	buffer_json_member_add_array(wb, key);
1152	{
1153	if(k && k->values.enabled) {
1154	FACET_VALUE *v;
1155	foreach_value_in_key(k, v) {
1156	buffer_json_add_array_item_uint64(wb, v->max);
1157	}
1158	foreach_value_in_key_done(v);
1159	}
1160	}
1161	buffer_json_array_close(wb); // key
1162	}
1163
1164	static inline void facets_histogram_value_avg(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key) {
1165	buffer_json_member_add_array(wb, key);
1166	{
1167	if(k && k->values.enabled) {
1168	FACET_VALUE *v;
1169	foreach_value_in_key(k, v) {
1170	buffer_json_add_array_item_double(wb, (double) v->sum / (double) facets->histogram.slots);
1171	}
1172	foreach_value_in_key_done(v);
1173	}
1174	}
1175	buffer_json_array_close(wb); // key
1176	}
1177
1178	static inline void facets_histogram_value_arp(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key) {
1179	buffer_json_member_add_array(wb, key);
1180	{
1181	if(k && k->values.enabled) {
1182	FACET_VALUE *v;
1183	foreach_value_in_key(k, v) {
1184	buffer_json_add_array_item_uint64(wb, 0);
1185	}
1186	foreach_value_in_key_done(v);
1187	}
1188	}
1189	buffer_json_array_close(wb); // key
1190	}
1191
1192	static inline void facets_histogram_value_con(BUFFER wb, FACETS facets __maybe_unused, FACET_KEY k, const char key, uint32_t sum) {
1193	buffer_json_member_add_array(wb, key);
1194	{
1195	if(k && k->values.enabled) {
1196	FACET_VALUE *v;
1197	foreach_value_in_key(k, v) {
1198	if(sum)
1199	buffer_json_add_array_item_double(wb, (double) v->sum * 100.0 / (double) sum);
1200	else
1201	buffer_json_add_array_item_double(wb, 0.0);
1202	}
1203	foreach_value_in_key_done(v);
1204	}
1205	}
1206	buffer_json_array_close(wb); // key
1207	}
1208
1209	static void facets_histogram_generate(FACETS facets, FACET_KEY k, BUFFER *wb) {
1210	CLEAN_BUFFER *tmp = buffer_create(0, NULL);
1211
1212	size_t dimensions = 0;
1213	uint32_t min = UINT32_MAX, max = 0, sum = 0, count = 0;
1214
1215	if(k && k->values.enabled) {
1216	FACET_VALUE *v;
1217	foreach_value_in_key(k, v) {
1218	if (unlikely(!v->histogram)) {
1219	v->min = v->max = v->sum = 0;
1220	continue;
1221	}
1222
1223	dimensions++;
1224
1225	v->min = UINT32_MAX;
1226	v->max = 0;
1227	v->sum = 0;
1228
1229	for(uint32_t i = 0; i < facets->histogram.slots ;i++) {
1230	uint32_t n = v->histogram[i];
1231
1232	if(n < min)
1233	min = n;
1234
1235	if(n > max)
1236	max = n;
1237
1238	sum += n;
1239	count++;
1240
1241	if(n < v->min)
1242	v->min = n;
1243
1244	if(n > v->max)
1245	v->max = n;
1246
1247	v->sum += n;
1248	}
1249	}
1250	foreach_value_in_key_done(v);
1251	}
1252
1253	buffer_json_member_add_object(wb, "summary");
1254	{
1255	// summary.nodes
1256	buffer_json_member_add_array(wb, "nodes");
1257	{
1258	buffer_json_add_array_item_object(wb); // node
1259	{
1260	buffer_json_member_add_string(wb, "mg", "default");
1261	buffer_json_member_add_string(wb, "nm", "facets.histogram");
1262	buffer_json_member_add_uint64(wb, "ni", 0);
1263	buffer_json_member_add_object(wb, "st");
1264	{
1265	buffer_json_member_add_uint64(wb, "ai", 0);
1266	buffer_json_member_add_uint64(wb, "code", 200);
1267	buffer_json_member_add_string(wb, "msg", "");
1268	}
1269	buffer_json_object_close(wb); // st
1270
1271	if(dimensions) {
1272	buffer_json_member_add_object(wb, "is");
1273	{
1274	buffer_json_member_add_uint64(wb, "sl", 1);
1275	buffer_json_member_add_uint64(wb, "qr", 1);
1276	}
1277	buffer_json_object_close(wb); // is
1278
1279	buffer_json_member_add_object(wb, "ds");
1280	{
1281	buffer_json_member_add_uint64(wb, "sl", dimensions);
1282	buffer_json_member_add_uint64(wb, "qr", dimensions);
1283	}
1284	buffer_json_object_close(wb); // ds
1285	}
1286
1287	if(count) {
1288	buffer_json_member_add_object(wb, "sts");
1289	{
1290	buffer_json_member_add_uint64(wb, "min", min);
1291	buffer_json_member_add_uint64(wb, "max", max);
1292	buffer_json_member_add_double(wb, "avg", (double) sum / (double) count);
1293	buffer_json_member_add_double(wb, "con", 100.0);
1294	}
1295	buffer_json_object_close(wb); // sts
1296	}
1297	}
1298	buffer_json_object_close(wb); // node
1299	}
1300	buffer_json_array_close(wb); // nodes
1301
1302	// summary.contexts
1303	buffer_json_member_add_array(wb, "contexts");
1304	{
1305	buffer_json_add_array_item_object(wb); // context
1306	{
1307	buffer_json_member_add_string(wb, "id", "facets.histogram");
1308
1309	if(dimensions) {
1310	buffer_json_member_add_object(wb, "is");
1311	{
1312	buffer_json_member_add_uint64(wb, "sl", 1);
1313	buffer_json_member_add_uint64(wb, "qr", 1);
1314	}
1315	buffer_json_object_close(wb); // is
1316
1317	buffer_json_member_add_object(wb, "ds");
1318	{
1319	buffer_json_member_add_uint64(wb, "sl", dimensions);
1320	buffer_json_member_add_uint64(wb, "qr", dimensions);
1321	}
1322	buffer_json_object_close(wb); // ds
1323	}
1324
1325	if(count) {
1326	buffer_json_member_add_object(wb, "sts");
1327	{
1328	buffer_json_member_add_uint64(wb, "min", min);
1329	buffer_json_member_add_uint64(wb, "max", max);
1330	buffer_json_member_add_double(wb, "avg", (double) sum / (double) count);
1331	buffer_json_member_add_double(wb, "con", 100.0);
1332	}
1333	buffer_json_object_close(wb); // sts
1334	}
1335	}
1336	buffer_json_object_close(wb); // context
1337	}
1338	buffer_json_array_close(wb); // contexts
1339
1340	// summary.instances
1341	buffer_json_member_add_array(wb, "instances");
1342	{
1343	buffer_json_add_array_item_object(wb); // instance
1344	{
1345	buffer_json_member_add_string(wb, "id", "facets.histogram");
1346	buffer_json_member_add_uint64(wb, "ni", 0);
1347
1348	if(dimensions) {
1349	buffer_json_member_add_object(wb, "ds");
1350	{
1351	buffer_json_member_add_uint64(wb, "sl", dimensions);
1352	buffer_json_member_add_uint64(wb, "qr", dimensions);
1353	}
1354	buffer_json_object_close(wb); // ds
1355	}
1356
1357	if(count) {
1358	buffer_json_member_add_object(wb, "sts");
1359	{
1360	buffer_json_member_add_uint64(wb, "min", min);
1361	buffer_json_member_add_uint64(wb, "max", max);
1362	buffer_json_member_add_double(wb, "avg", (double) sum / (double) count);
1363	buffer_json_member_add_double(wb, "con", 100.0);
1364	}
1365	buffer_json_object_close(wb); // sts
1366	}
1367	}
1368	buffer_json_object_close(wb); // instance
1369	}
1370	buffer_json_array_close(wb); // instances
1371
1372	// summary.dimensions
1373	buffer_json_member_add_array(wb, "dimensions");
1374	if(dimensions && k && k->values.enabled) {
1375	size_t pri = 0;
1376	FACET_VALUE *v;
1377
1378	foreach_value_in_key(k, v) {
1379	uint64_t d_sl, d_qr;
1380	uint64_t d_min, d_max;
1381	double d_avg, d_con;
1382
1383	if(likely(v->histogram)) {
1384	d_sl = d_qr = 1;
1385	d_min = v->min;
1386	d_max = v->max;
1387	d_avg = (double) v->sum / (double) facets->histogram.slots;
1388	d_con = (double) v->sum * 100.0 / (double) sum;
1389	}
1390	else {
1391	d_sl = d_qr = 0;
1392	d_min = d_max = 0;
1393	d_avg = d_con = 0.0;
1394	}
1395
1396	buffer_json_add_array_item_object(wb); // dimension
1397	{
1398	buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v));
1399
1400	facets_key_value_transformed(facets, k, v, tmp, FACETS_TRANSFORM_HISTOGRAM);
1401	buffer_json_member_add_string(wb, "nm", buffer_tostring(tmp));
1402	buffer_json_member_add_object(wb, "ds");
1403	{
1404	buffer_json_member_add_uint64(wb, "sl", d_sl);
1405	buffer_json_member_add_uint64(wb, "qr", d_qr);
1406	}
1407	buffer_json_object_close(wb); // ds
1408	buffer_json_member_add_object(wb, "sts");
1409	{
1410	buffer_json_member_add_uint64(wb, "min", d_min);
1411	buffer_json_member_add_uint64(wb, "max", d_max);
1412	buffer_json_member_add_double(wb, "avg", d_avg);
1413	buffer_json_member_add_double(wb, "con", d_con);
1414	}
1415	buffer_json_object_close(wb); // sts
1416	buffer_json_member_add_uint64(wb, "pri", pri++);
1417	}
1418	buffer_json_object_close(wb); // dimension
1419	}
1420	foreach_value_in_key_done(v);
1421	}
1422	buffer_json_array_close(wb); // dimensions
1423
1424	buffer_json_member_add_array(wb, "labels");
1425	buffer_json_array_close(wb); // labels
1426
1427	buffer_json_member_add_array(wb, "alerts");
1428	buffer_json_array_close(wb); // alerts
1429	}
1430	buffer_json_object_close(wb); // summary
1431
1432	buffer_json_member_add_object(wb, "totals");
1433	{
1434	buffer_json_member_add_object(wb, "nodes");
1435	{
1436	buffer_json_member_add_uint64(wb, "sl", 1);
1437	buffer_json_member_add_uint64(wb, "qr", 1);
1438	}
1439	buffer_json_object_close(wb); // nodes
1440
1441	if(dimensions) {
1442	buffer_json_member_add_object(wb, "contexts");
1443	{
1444	buffer_json_member_add_uint64(wb, "sl", 1);
1445	buffer_json_member_add_uint64(wb, "qr", 1);
1446	}
1447	buffer_json_object_close(wb); // contexts
1448	buffer_json_member_add_object(wb, "instances");
1449	{
1450	buffer_json_member_add_uint64(wb, "sl", 1);
1451	buffer_json_member_add_uint64(wb, "qr", 1);
1452	}
1453	buffer_json_object_close(wb); // instances
1454
1455	buffer_json_member_add_object(wb, "dimensions");
1456	{
1457	buffer_json_member_add_uint64(wb, "sl", dimensions);
1458	buffer_json_member_add_uint64(wb, "qr", dimensions);
1459	}
1460	buffer_json_object_close(wb); // dimension
1461	}
1462	}
1463	buffer_json_object_close(wb); // totals
1464
1465	buffer_json_member_add_object(wb, "result");
1466	{
1467	facets_histogram_value_names(wb, facets, k, "labels", "time");
1468
1469	buffer_json_member_add_object(wb, "point");
1470	{
1471	buffer_json_member_add_uint64(wb, "value", 0);
1472	buffer_json_member_add_uint64(wb, "arp", 1);
1473	buffer_json_member_add_uint64(wb, "pa", 2);
1474	}
1475	buffer_json_object_close(wb); // point
1476
1477	buffer_json_member_add_array(wb, "data");
1478	if(k && k->values.enabled) {
1479	usec_t t = facets->histogram.after_ut;
1480	for(uint32_t i = 0; i < facets->histogram.slots ;i++) {
1481	buffer_json_add_array_item_array(wb); // row
1482	{
1483	buffer_json_add_array_item_time_ms(wb, t / USEC_PER_SEC);
1484
1485	FACET_VALUE *v;
1486	foreach_value_in_key(k, v) {
1487	buffer_json_add_array_item_array(wb); // point
1488
1489	if(v->histogram)
1490	buffer_json_add_array_item_uint64(wb, v->histogram[i]);
1491	else
1492	buffer_json_add_array_item_double(wb, NAN);
1493
1494	buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate
1495	buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation
1496
1497	buffer_json_array_close(wb); // point
1498	}
1499	foreach_value_in_key_done(v);
1500	}
1501	buffer_json_array_close(wb); // row
1502
1503	t += facets->histogram.slot_width_ut;
1504	}
1505	}
1506	buffer_json_array_close(wb); //data
1507	}
1508	buffer_json_object_close(wb); // result
1509
1510	buffer_json_member_add_object(wb, "db");
1511	{
1512	buffer_json_member_add_uint64(wb, "tiers", 1);
1513	buffer_json_member_add_uint64(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
1514	// we should add these only when we know the retention of the db
1515	// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
1516	// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
1517	buffer_json_member_add_string(wb, "units", "events");
1518	buffer_json_member_add_object(wb, "dimensions");
1519	{
1520	facets_histogram_value_ids(wb, facets, k, "ids", NULL);
1521	facets_histogram_value_names(wb, facets, k, "names", NULL);
1522	facets_histogram_value_units(wb, facets, k, "units");
1523
1524	buffer_json_member_add_object(wb, "sts");
1525	{
1526	facets_histogram_value_min(wb, facets, k, "min");
1527	facets_histogram_value_max(wb, facets, k, "max");
1528	facets_histogram_value_avg(wb, facets, k, "avg");
1529	facets_histogram_value_arp(wb, facets, k, "arp");
1530	facets_histogram_value_con(wb, facets, k, "con", sum);
1531	}
1532	buffer_json_object_close(wb); // sts
1533	}
1534	buffer_json_object_close(wb); // dimensions
1535
1536	buffer_json_member_add_array(wb, "per_tier");
1537	{
1538	buffer_json_add_array_item_object(wb); // tier0
1539	{
1540	buffer_json_member_add_uint64(wb, "tier", 0);
1541	buffer_json_member_add_uint64(wb, "queries", 1);
1542	buffer_json_member_add_uint64(wb, "points", count);
1543	buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
1544	// we should add these only when we know the retention of the db
1545	// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
1546	// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
1547	}
1548	buffer_json_object_close(wb); // tier0
1549	}
1550	buffer_json_array_close(wb); // per_tier
1551	}
1552	buffer_json_object_close(wb); // db
1553
1554	buffer_json_member_add_object(wb, "view");
1555	{
1556	char title[1024 + 1] = "Events Distribution";
1557	FACET_KEY *kt = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash);
1558	if(kt && kt->name)
1559	snprintfz(title, sizeof(title) - 1, "Events Distribution by %s", kt->name);
1560
1561	buffer_json_member_add_string(wb, "title", title);
1562	buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
1563	buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC);
1564	buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC);
1565	buffer_json_member_add_string(wb, "units", "events");
1566	buffer_json_member_add_string(wb, "chart_type", "stackedBar");
1567	buffer_json_member_add_object(wb, "dimensions");
1568	{
1569	buffer_json_member_add_array(wb, "grouped_by");
1570	{
1571	buffer_json_add_array_item_string(wb, "dimension");
1572	}
1573	buffer_json_array_close(wb); // grouped_by
1574
1575	facets_histogram_value_ids(wb, facets, k, "ids", NULL);
1576	facets_histogram_value_names(wb, facets, k, "names", NULL);
1577	facets_histogram_value_colors(wb, facets, k, "colors");
1578	facets_histogram_value_units(wb, facets, k, "units");
1579
1580	buffer_json_member_add_object(wb, "sts");
1581	{
1582	facets_histogram_value_min(wb, facets, k, "min");
1583	facets_histogram_value_max(wb, facets, k, "max");
1584	facets_histogram_value_avg(wb, facets, k, "avg");
1585	facets_histogram_value_arp(wb, facets, k, "arp");
1586	facets_histogram_value_con(wb, facets, k, "con", sum);
1587	}
1588	buffer_json_object_close(wb); // sts
1589	}
1590	buffer_json_object_close(wb); // dimensions
1591
1592	buffer_json_member_add_uint64(wb, "min", min);
1593	buffer_json_member_add_uint64(wb, "max", max);
1594	}
1595	buffer_json_object_close(wb); // view
1596
1597	buffer_json_member_add_array(wb, "agents");
1598	{
1599	buffer_json_add_array_item_object(wb); // agent
1600	{
1601	buffer_json_member_add_string(wb, "mg", "default");
1602	buffer_json_member_add_string(wb, "nm", "facets.histogram");
1603	buffer_json_member_add_time_t(wb, "now", now_realtime_sec());
1604	buffer_json_member_add_uint64(wb, "ai", 0);
1605	}
1606	buffer_json_object_close(wb); // agent
1607	}
1608	buffer_json_array_close(wb); // agents
1609	}
1610
1611	// ----------------------------------------------------------------------------
1612
1613	static inline void facet_value_is_used(FACET_KEY k, FACET_VALUE v) {
1614	if(!k->key_found_in_row)
1615	v->rows_matching_facet_value++;
1616
1617	k->key_found_in_row++;
1618
1619	if(v->selected)
1620	k->key_values_selected_in_row++;
1621	}
1622
1623	static inline bool facets_key_is_facet(FACETS facets, FACET_KEY k) {
1624	bool included = facets->all_keys_included_by_default, excluded = false, never = false;
1625
1626	if(k->options & (FACET_KEY_OPTION_FACET \| FACET_KEY_OPTION_NO_FACET \| FACET_KEY_OPTION_NEVER_FACET)) {
1627	if(k->options & FACET_KEY_OPTION_FACET) {
1628	included = true;
1629	excluded = false;
1630	never = false;
1631	}
1632	else if(k->options & (FACET_KEY_OPTION_NO_FACET \| FACET_KEY_OPTION_NEVER_FACET)) {
1633	included = false;
1634	excluded = true;
1635	never = true;
1636	}
1637	}
1638	else {
1639	if (facets->included_keys) {
1640	if (!simple_pattern_matches(facets->included_keys, k->name))
1641	included = false;
1642	}
1643
1644	if (facets->excluded_keys) {
1645	if (simple_pattern_matches(facets->excluded_keys, k->name)) {
1646	excluded = true;
1647	never = true;
1648	}
1649	}
1650	}
1651
1652	if(included && !excluded) {
1653	k->options \|= FACET_KEY_OPTION_FACET;
1654	k->options &= ~FACET_KEY_OPTION_NO_FACET;
1655	return true;
1656	}
1657
1658	k->options \|= FACET_KEY_OPTION_NO_FACET;
1659	k->options &= ~FACET_KEY_OPTION_FACET;
1660
1661	if(never)
1662	k->options \|= FACET_KEY_OPTION_NEVER_FACET;
1663
1664	return false;
1665	}
1666
1667	// ----------------------------------------------------------------------------
1668	// bin_data management
1669
1670	static inline void facets_row_bin_data_cleanup(FACETS facets, FACET_ROW_BIN_DATA bin_data) {
1671	if(!bin_data->data)
1672	return;
1673
1674	bin_data->cleanup_cb(bin_data->data);
1675	*bin_data = FACET_ROW_BIN_DATA_EMPTY;
1676
1677	fatal_assert(facets->operations.bin_data_inflight > 0);
1678	facets->operations.bin_data_inflight--;
1679	}
1680
1681	void facets_row_bin_data_set(FACETS facets, void (cleanup_cb)(void data), void data) {
1682	// in case the caller tries to register bin_data multiple times
1683	// for the same row.
1684	facets_row_bin_data_cleanup(facets, &facets->bin_data);
1685
1686	// set the new values
1687	facets->bin_data.cleanup_cb = cleanup_cb;
1688	facets->bin_data.data = data;
1689	facets->operations.bin_data_inflight++;
1690	}
1691
1692	void facets_row_bin_data_get(FACETS facets __maybe_unused, FACET_ROW *row) {
1693	return row->bin_data.data;
1694	}
1695
1696	// ----------------------------------------------------------------------------
1697
1698	FACETS facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char visible_keys, const char facet_keys, const char non_facet_keys) {
1699	FACETS *facets = callocz(1, sizeof(FACETS));
1700	facets->all_keys_included_by_default = true;
1701	facets->options = options;
1702	FACETS_KEYS_INDEX_CREATE(facets);
1703
1704	if(facet_keys && *facet_keys)
1705	facets->included_keys = simple_pattern_create(facet_keys, "\|", SIMPLE_PATTERN_EXACT, true);
1706
1707	if(non_facet_keys && *non_facet_keys)
1708	facets->excluded_keys = simple_pattern_create(non_facet_keys, "\|", SIMPLE_PATTERN_EXACT, true);
1709
1710	if(visible_keys && *visible_keys)
1711	facets->visible_keys = simple_pattern_create(visible_keys, "\|", SIMPLE_PATTERN_EXACT, true);
1712
1713	facets->max_items_to_return = items_to_return > 1 ? items_to_return : 2;
1714	facets->anchor.start_ut = 0;
1715	facets->anchor.stop_ut = 0;
1716	facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
1717	facets->order = 1;
1718
1719	return facets;
1720	}
1721
1722	void facets_destroy(FACETS *facets) {
1723	if(!facets) return;
1724
1725	dictionary_destroy(facets->accepted_params);
1726	FACETS_KEYS_INDEX_DESTROY(facets);
1727	simple_pattern_free(facets->visible_keys);
1728	simple_pattern_free(facets->included_keys);
1729	simple_pattern_free(facets->excluded_keys);
1730
1731	while(facets->base) {
1732	FACET_ROW *r = facets->base;
1733	DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next);
1734
1735	facets_row_free(facets, r);
1736	}
1737
1738	// in case the caller did not call facets_row_finished()
1739	// on the last row.
1740	facets_row_bin_data_cleanup(facets, &facets->bin_data);
1741
1742	// make sure we didn't lose any data
1743	fatal_assert(facets->operations.bin_data_inflight == 0);
1744
1745	freez(facets->histogram.chart);
1746	freez(facets);
1747	}
1748
1749	void facets_accepted_param(FACETS facets, const char param) {
1750	if(!facets->accepted_params)
1751	facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED\|DICT_OPTION_DONT_OVERWRITE_VALUE);
1752
1753	dictionary_set(facets->accepted_params, param, NULL, 0);
1754	}
1755
1756	static inline FACET_KEY facets_register_key_name_length(FACETS facets, const char *key, size_t key_length, FACET_KEY_OPTIONS options) {
1757	return FACETS_KEY_ADD_TO_INDEX(facets, FACETS_HASH_FUNCTION(key, key_length), key, key_length, options);
1758	}
1759
1760	inline FACET_KEY facets_register_key_name(FACETS facets, const char *key, FACET_KEY_OPTIONS options) {
1761	return facets_register_key_name_length(facets, key, strlen(key), options);
1762	}
1763
1764	inline FACET_KEY facets_register_key_name_transformation(FACETS facets, const char key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void data) {
1765	FACET_KEY *k = facets_register_key_name(facets, key, options);
1766	k->transform.cb = cb;
1767	k->transform.data = data;
1768	k->transform.view_only = (options & FACET_KEY_OPTION_TRANSFORM_VIEW) ? true : false;
1769	return k;
1770	}
1771
1772	inline FACET_KEY facets_register_dynamic_key_name(FACETS facets, const char key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void data) {
1773	FACET_KEY *k = facets_register_key_name(facets, key, options);
1774	k->dynamic.cb = cb;
1775	k->dynamic.data = data;
1776	return k;
1777	}
1778
1779	void facets_set_query(FACETS facets, const char query) {
1780	if(!query)
1781	return;
1782
1783	facets->query = simple_pattern_create(query, "\|", SIMPLE_PATTERN_SUBSTRING, false);
1784	}
1785
1786	void facets_set_items(FACETS *facets, uint32_t items) {
1787	facets->max_items_to_return = items > 1 ? items : 2;
1788	}
1789
1790	void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction) {
1791	facets->anchor.start_ut = start_ut;
1792	facets->anchor.stop_ut = stop_ut;
1793	facets->anchor.direction = direction;
1794
1795	if((facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && facets->anchor.start_ut && facets->anchor.start_ut < facets->anchor.stop_ut) \|\|
1796	(facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && facets->anchor.stop_ut && facets->anchor.stop_ut < facets->anchor.start_ut)) {
1797	internal_error(true, "start and stop anchors are flipped");
1798	facets->anchor.start_ut = stop_ut;
1799	facets->anchor.stop_ut = start_ut;
1800	}
1801	}
1802
1803	void facets_enable_slice_mode(FACETS *facets) {
1804	facets->options \|= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS \| FACETS_OPTION_SORT_FACETS_ALPHABETICALLY;
1805	}
1806
1807	void facets_reset_and_disable_all_facets(FACETS *facets) {
1808	facets->all_keys_included_by_default = false;
1809
1810	simple_pattern_free(facets->included_keys);
1811	facets->included_keys = NULL;
1812
1813	// We need this, because the exclusions are good for controlling which key can become a facet.
1814	// The excluded ones are not offered for facets at all.
1815	// simple_pattern_free(facets->excluded_keys);
1816	// facets->excluded_keys = NULL;
1817
1818	simple_pattern_free(facets->visible_keys);
1819	facets->visible_keys = NULL;
1820
1821	FACET_KEY *k;
1822	foreach_key_in_facets(facets, k) {
1823	k->options \|= FACET_KEY_OPTION_NO_FACET;
1824	k->options &= ~FACET_KEY_OPTION_FACET;
1825	}
1826	foreach_key_in_facets_done(k);
1827	}
1828
1829	inline FACET_KEY facets_register_facet(FACETS facets, const char *name, FACET_KEY_OPTIONS options) {
1830	size_t name_length = strlen(name);
1831	FACETS_HASH hash = FACETS_HASH_FUNCTION(name, name_length);
1832
1833	FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, name, name_length, options);
1834	k->options \|= FACET_KEY_OPTION_FACET;
1835	k->options &= ~FACET_KEY_OPTION_NO_FACET;
1836	facet_key_late_init(facets, k);
1837
1838	return k;
1839	}
1840
1841	inline FACET_KEY facets_register_facet_id(FACETS facets, const char *key_id, FACET_KEY_OPTIONS options) {
1842	if(!is_valid_string_hash(key_id))
1843	return NULL;
1844
1845	FACETS_HASH hash = str_to_facets_hash(key_id);
1846
1847	internal_error(strcmp(hash_to_static_string(hash), key_id) != 0,
1848	"Regenerating the user supplied key, does not produce the same hash string");
1849
1850	FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, NULL, 0, options);
1851	k->options \|= FACET_KEY_OPTION_FACET;
1852	k->options &= ~FACET_KEY_OPTION_NO_FACET;
1853	facet_key_late_init(facets, k);
1854
1855	return k;
1856	}
1857
1858	void facets_register_facet_filter_id(FACETS facets, const char key_id, const char *value_id, FACET_KEY_OPTIONS options) {
1859	FACET_KEY *k = facets_register_facet_id(facets, key_id, options);
1860	if(k && is_valid_string_hash(value_id)) {
1861	if(!(k->options & FACET_KEY_OPTION_FACET))
1862	k->options \|= FACET_KEY_OPTION_FILTER_ONLY;
1863
1864	k->default_selected_for_values = false;
1865	FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, NULL, str_to_facets_hash(value_id));
1866	}
1867	}
1868
1869	void facets_register_facet_filter(FACETS facets, const char key, const char *value, FACET_KEY_OPTIONS options) {
1870	FACET_KEY *k = facets_register_facet(facets, key, options);
1871	if(k) {
1872	if(!(k->options & FACET_KEY_OPTION_FACET))
1873	k->options \|= FACET_KEY_OPTION_FILTER_ONLY;
1874
1875	FACETS_HASH hash = FACETS_HASH_FUNCTION(value, strlen(value));
1876	k->default_selected_for_values = false;
1877	FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, value, hash);
1878	}
1879	}
1880
1881	void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) {
1882	facets->current_row.severity = severity;
1883	}
1884
1885	void facets_register_row_severity(FACETS facets, facet_row_severity_t cb, void data) {
1886	facets->severity.cb = cb;
1887	facets->severity.data = data;
1888	}
1889
1890	void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options) {
1891	facets->options \|= options;
1892	}
1893
1894	// ----------------------------------------------------------------------------
1895
1896	static inline void facets_key_set_unsampled_value(FACETS facets, FACET_KEY k) {
1897	if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX))
1898	facets->keys_in_row.array[facets->keys_in_row.used++] = k;
1899
1900	k->current_value.flags \|= FACET_KEY_VALUE_UPDATED \| FACET_KEY_VALUE_UNSAMPLED;
1901
1902	facets->operations.values.registered++;
1903	facets->operations.values.unsampled++;
1904
1905	// no need to copy the UNSET value
1906	// empty values are exported as empty
1907	k->current_value.raw = NULL;
1908	k->current_value.raw_len = 0;
1909	k->current_value.b->len = 0;
1910	k->current_value.flags &= ~FACET_KEY_VALUE_COPIED;
1911
1912	if(unlikely(k->values.enabled))
1913	FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(k);
1914	else {
1915	k->key_found_in_row++;
1916	k->key_values_selected_in_row++;
1917	}
1918	}
1919
1920	static inline void facets_key_set_empty_value(FACETS facets, FACET_KEY k) {
1921	if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX))
1922	facets->keys_in_row.array[facets->keys_in_row.used++] = k;
1923
1924	k->current_value.flags \|= FACET_KEY_VALUE_UPDATED \| FACET_KEY_VALUE_EMPTY;
1925
1926	facets->operations.values.registered++;
1927	facets->operations.values.empty++;
1928
1929	// no need to copy the UNSET value
1930	// empty values are exported as empty
1931	k->current_value.raw = NULL;
1932	k->current_value.raw_len = 0;
1933	k->current_value.b->len = 0;
1934	k->current_value.flags &= ~FACET_KEY_VALUE_COPIED;
1935
1936	if(unlikely(k->values.enabled))
1937	FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(k);
1938	else {
1939	k->key_found_in_row++;
1940	k->key_values_selected_in_row++;
1941	}
1942	}
1943
1944	static inline void facets_key_check_value(FACETS facets, FACET_KEY k) {
1945	if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX))
1946	facets->keys_in_row.array[facets->keys_in_row.used++] = k;
1947
1948	k->current_value.flags \|= FACET_KEY_VALUE_UPDATED;
1949	k->current_value.flags &= ~(FACET_KEY_VALUE_EMPTY\|FACET_KEY_VALUE_UNSAMPLED\|FACET_KEY_VALUE_ESTIMATED);
1950
1951	facets->operations.values.registered++;
1952
1953	if(k->transform.cb && !k->transform.view_only) {
1954	facets->operations.values.transformed++;
1955	facets_key_value_copy_to_buffer(k);
1956	k->transform.cb(facets, k->current_value.b, FACETS_TRANSFORM_VALUE, k->transform.data);
1957	}
1958
1959	// bool found = false;
1960	// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL)
1961	// found = true;
1962
1963	if(facets->query && !facet_key_value_empty_or_unsampled_or_estimated(k) && ((k->options & FACET_KEY_OPTION_FTS) \|\| facets->options & FACETS_OPTION_ALL_KEYS_FTS)) {
1964	facets->operations.fts.searches++;
1965	facets_key_value_copy_to_buffer(k);
1966	switch(simple_pattern_matches_extract(facets->query, buffer_tostring(k->current_value.b), NULL, 0)) {
1967	case SP_MATCHED_POSITIVE:
1968	facets->current_row.keys_matched_by_query_positive++;
1969	break;
1970
1971	case SP_MATCHED_NEGATIVE:
1972	facets->current_row.keys_matched_by_query_negative++;
1973	break;
1974
1975	case SP_NOT_MATCHED:
1976	break;
1977	}
1978	}
1979
1980	if(k->values.enabled)
1981	FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(k);
1982	else {
1983	k->key_found_in_row++;
1984	k->key_values_selected_in_row++;
1985	}
1986	}
1987
1988	void facets_add_key_value(FACETS facets, const char key, const char *value) {
1989	FACET_KEY *k = facets_register_key_name(facets, key, 0);
1990	k->current_value.raw = value;
1991	k->current_value.raw_len = strlen(value);
1992
1993	facets_key_check_value(facets, k);
1994	}
1995
1996	void facets_add_key_value_length(FACETS facets, const char key, size_t key_len, const char *value, size_t value_len) {
1997	if(!key \|\| !key \|\| !key_len \|\| !value \|\| !value \|\| !value_len)
1998	// adding empty values, makes the rows unmatched
1999	return;
2000
2001	FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0);
2002	k->current_value.raw = value;
2003	k->current_value.raw_len = value_len;
2004
2005	facets_key_check_value(facets, k);
2006	}
2007
2008	// ----------------------------------------------------------------------------
2009	// FACET_ROW dictionary hooks
2010
2011	static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM item __maybe_unused, void value, void *data) {
2012	FACET_ROW_KEY_VALUE *rkv = value;
2013	FACET_ROW *row = data; (void)row;
2014
2015	rkv->wb = buffer_create(0, NULL);
2016	if(!rkv->empty)
2017	buffer_contents_replace(rkv->wb, rkv->tmp, rkv->tmp_len);
2018	}
2019
2020	static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM item __maybe_unused, void old_value, void new_value, void data) {
2021	FACET_ROW_KEY_VALUE *rkv = old_value;
2022	FACET_ROW_KEY_VALUE *n_rkv = new_value;
2023	FACET_ROW *row = data; (void)row;
2024
2025	rkv->empty = n_rkv->empty;
2026
2027	if(!rkv->empty)
2028	buffer_contents_replace(rkv->wb, n_rkv->tmp, n_rkv->tmp_len);
2029	else
2030	buffer_flush(rkv->wb);
2031
2032	return false;
2033	}
2034
2035	static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM item __maybe_unused, void value, void *data) {
2036	FACET_ROW_KEY_VALUE *rkv = value;
2037	FACET_ROW *row = data; (void)row;
2038
2039	buffer_free(rkv->wb);
2040	}
2041
2042	// ----------------------------------------------------------------------------
2043	// FACET_ROW management
2044
2045	static void facets_row_free(FACETS facets __maybe_unused, FACET_ROW row) {
2046	facets_row_bin_data_cleanup(facets, &row->bin_data);
2047	dictionary_destroy(row->dict);
2048	row->dict = NULL;
2049	freez(row);
2050	}
2051
2052	static FACET_ROW facets_row_create(FACETS facets, usec_t usec, FACET_ROW *into) {
2053	FACET_ROW *row;
2054
2055	if(into) {
2056	row = into;
2057	facets->operations.rows.reused++;
2058	facets_row_bin_data_cleanup(facets, &row->bin_data);
2059	}
2060	else {
2061	row = callocz(1, sizeof(FACET_ROW));
2062	row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED\|DICT_OPTION_DONT_OVERWRITE_VALUE\|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE));
2063	dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row);
2064	dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row);
2065	dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row);
2066	facets->operations.rows.created++;
2067	}
2068
2069	// copy the bin_data to the row
2070	// and forget about them in facets
2071	row->bin_data = facets->bin_data;
2072	facets->bin_data = FACET_ROW_BIN_DATA_EMPTY;
2073
2074	row->severity = facets->current_row.severity;
2075	row->usec = usec;
2076
2077	FACET_KEY *k;
2078	foreach_key_in_facets(facets, k) {
2079	FACET_ROW_KEY_VALUE t = {
2080	.tmp = NULL,
2081	.tmp_len = 0,
2082	.wb = NULL,
2083	.empty = true,
2084	};
2085
2086	if(facet_key_value_updated(k) && !facet_key_value_empty_or_unsampled_or_estimated(k)) {
2087	t.tmp = facets_key_get_value(k);
2088	t.tmp_len = facets_key_get_value_length(k);
2089	t.empty = false;
2090	}
2091
2092	dictionary_set(row->dict, k->name, &t, sizeof(t));
2093	}
2094	foreach_key_in_facets_done(k);
2095
2096	return row;
2097	}
2098
2099	// ----------------------------------------------------------------------------
2100
2101	static inline FACET_ROW facets_row_keep_seek_to_position(FACETS facets, usec_t usec) {
2102	if(usec < facets->base->prev->usec)
2103	return facets->base->prev;
2104
2105	if(usec > facets->base->usec)
2106	return facets->base;
2107
2108	FACET_ROW *last = facets->operations.last_added;
2109	while(last->prev != facets->base->prev && usec > last->prev->usec) {
2110	last = last->prev;
2111	facets->operations.backwards++;
2112	}
2113
2114	while(last->next && usec < last->next->usec) {
2115	last = last->next;
2116	facets->operations.forwards++;
2117	}
2118
2119	return last;
2120	}
2121
2122	static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) {
2123	facets->operations.last_added = facets_row_create(facets, usec, NULL);
2124	DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
2125	facets->items_to_return++;
2126	facets->operations.first++;
2127	}
2128
2129	static inline bool facets_is_entry_within_anchor(FACETS *facets, usec_t usec) {
2130	if(facets->anchor.start_ut \|\| facets->anchor.stop_ut) {
2131	// we have an anchor key
2132	// we don't want to keep rows on the other side of the direction
2133
2134	switch (facets->anchor.direction) {
2135	default:
2136	case FACETS_ANCHOR_DIRECTION_BACKWARD:
2137	// we need to keep only the smaller timestamps
2138	if (facets->anchor.start_ut && usec >= facets->anchor.start_ut) {
2139	facets->operations.skips_before++;
2140	return false;
2141	}
2142	if (facets->anchor.stop_ut && usec <= facets->anchor.stop_ut) {
2143	facets->operations.skips_after++;
2144	return false;
2145	}
2146	break;
2147
2148	case FACETS_ANCHOR_DIRECTION_FORWARD:
2149	// we need to keep only the bigger timestamps
2150	if (facets->anchor.start_ut && usec <= facets->anchor.start_ut) {
2151	facets->operations.skips_after++;
2152	return false;
2153	}
2154	if (facets->anchor.stop_ut && usec >= facets->anchor.stop_ut) {
2155	facets->operations.skips_before++;
2156	return false;
2157	}
2158	break;
2159	}
2160	}
2161
2162	return true;
2163	}
2164
2165	bool facets_row_candidate_to_keep(FACETS *facets, usec_t usec) {
2166	return !facets->base \|\|
2167	(usec >= facets->base->prev->usec && usec <= facets->base->usec && facets_is_entry_within_anchor(facets, usec)) \|\|
2168	facets->items_to_return < facets->max_items_to_return;
2169	}
2170
2171	static void facets_row_keep(FACETS *facets, usec_t usec) {
2172	facets->operations.rows.matched++;
2173
2174	if(unlikely(!facets->base)) {
2175	// the first row to keep
2176	facets_row_keep_first_entry(facets, usec);
2177	return;
2178	}
2179
2180	FACET_ROW *closest = facets_row_keep_seek_to_position(facets, usec);
2181	FACET_ROW *to_replace = NULL;
2182
2183	if(likely(facets->items_to_return >= facets->max_items_to_return)) {
2184	// we have enough items to return already
2185
2186	switch(facets->anchor.direction) {
2187	default:
2188	case FACETS_ANCHOR_DIRECTION_BACKWARD:
2189	if(closest == facets->base->prev && usec < closest->usec) {
2190	// this is to the end of the list, belonging to the next page
2191	facets->operations.skips_after++;
2192	return;
2193	}
2194
2195	// it seems we need to remove an item - the last one
2196	to_replace = facets->base->prev;
2197	if(closest == to_replace)
2198	closest = to_replace->prev;
2199
2200	break;
2201
2202	case FACETS_ANCHOR_DIRECTION_FORWARD:
2203	if(closest == facets->base && usec > closest->usec) {
2204	// this is to the beginning of the list, belonging to the next page
2205	facets->operations.skips_before++;
2206	return;
2207	}
2208
2209	// it seems we need to remove an item - the first one
2210	to_replace = facets->base;
2211	if(closest == to_replace)
2212	closest = to_replace->next;
2213
2214	break;
2215	}
2216
2217	facets->operations.shifts++;
2218	facets->items_to_return--;
2219	DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, to_replace, prev, next);
2220	}
2221
2222	internal_fatal(!closest, "FACETS: closest cannot be NULL");
2223	internal_fatal(closest == to_replace, "FACETS: closest cannot be the same as to_replace");
2224
2225	facets->operations.last_added = facets_row_create(facets, usec, to_replace);
2226
2227	if(usec < closest->usec) {
2228	DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);
2229	facets->operations.appends++;
2230	}
2231	else {
2232	DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);
2233	facets->operations.prepends++;
2234	}
2235
2236	facets->items_to_return++;
2237	}
2238
2239	static inline void facets_reset_key(FACET_KEY *k) {
2240	k->key_found_in_row = 0;
2241	k->key_values_selected_in_row = 0;
2242	k->current_value.flags = FACET_KEY_VALUE_NONE;
2243	k->current_value.hash = FACETS_HASH_ZERO;
2244	}
2245
2246	static void facets_reset_keys_with_value_and_row(FACETS *facets) {
2247	size_t entries = facets->keys_in_row.used;
2248
2249	for(size_t p = 0; p < entries ;p++) {
2250	FACET_KEY *k = facets->keys_in_row.array[p];
2251	facets_reset_key(k);
2252	}
2253
2254	facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL;
2255	facets->current_row.keys_matched_by_query_positive = 0;
2256	facets->current_row.keys_matched_by_query_negative = 0;
2257	facets->keys_in_row.used = 0;
2258
2259	facets_row_bin_data_cleanup(facets, &facets->bin_data);
2260	}
2261
2262	void facets_rows_begin(FACETS *facets) {
2263	FACET_KEY *k;
2264	foreach_key_in_facets(facets, k) {
2265	facets_reset_key(k);
2266	}
2267	foreach_key_in_facets_done(k);
2268
2269	facets->keys_in_row.used = 0;
2270	facets_reset_keys_with_value_and_row(facets);
2271	}
2272
2273	bool facets_row_finished(FACETS *facets, usec_t usec) {
2274	// char buf[RFC3339_MAX_LENGTH];
2275	// rfc3339_datetime_ut(buf, sizeof(buf), usec, 3, false);
2276
2277	facets->operations.rows.evaluated++;
2278
2279	if(unlikely((facets->query && facets->keys_filtered_by_query &&
2280	(!facets->current_row.keys_matched_by_query_positive \|\| facets->current_row.keys_matched_by_query_negative)) \|\|
2281	(facets->timeframe.before_ut && usec > facets->timeframe.before_ut) \|\|
2282	(facets->timeframe.after_ut && usec < facets->timeframe.after_ut))) {
2283	// this row is not useful
2284	// 1. not matched by full text search, or
2285	// 2. not in our timeframe
2286	facets_reset_keys_with_value_and_row(facets);
2287	return false;
2288	}
2289
2290	bool within_anchor = facets_is_entry_within_anchor(facets, usec);
2291	if(unlikely(!within_anchor && (facets->options & FACETS_OPTION_DATA_ONLY))) {
2292	facets_reset_keys_with_value_and_row(facets);
2293	return false;
2294	}
2295
2296	size_t entries = facets->keys_with_values.used;
2297	size_t total_keys = 0;
2298	size_t selected_keys = 0;
2299
2300	for(size_t p = 0; p < entries ;p++) {
2301	FACET_KEY *k = facets->keys_with_values.array[p];
2302
2303	if(!facet_key_value_updated(k)) {
2304	// put the FACET_VALUE_UNSET value into it
2305	facets_key_set_empty_value(facets, k);
2306	}
2307
2308	total_keys++;
2309
2310	if(k->key_values_selected_in_row)
2311	selected_keys++;
2312
2313	if(unlikely(!facets->histogram.key && facets->histogram.hash == k->hash))
2314	facets->histogram.key = k;
2315	}
2316
2317	if(selected_keys >= total_keys - 1) {
2318	size_t found = 0;
2319	(void) found;
2320
2321	for(size_t p = 0; p < entries; p++) {
2322	FACET_KEY *k = facets->keys_with_values.array[p];
2323
2324	size_t counted_by = selected_keys;
2325
2326	if(counted_by != total_keys && !k->key_values_selected_in_row)
2327	counted_by++;
2328
2329	if(counted_by == total_keys) {
2330	k->current_value.v->final_facet_value_counter++;
2331	found++;
2332	}
2333	}
2334
2335	internal_fatal(!found, "We should find at least one facet to count this row");
2336	}
2337
2338	if(selected_keys == total_keys) {
2339	// we need to keep this row
2340	facets_histogram_update_value(facets, usec);
2341
2342	if(within_anchor)
2343	facets_row_keep(facets, usec);
2344	}
2345
2346	facets_reset_keys_with_value_and_row(facets);
2347
2348	return selected_keys == total_keys;
2349	}
2350
2351	// ----------------------------------------------------------------------------
2352	// output
2353
2354	const char *facets_severity_to_string(FACET_ROW_SEVERITY severity) {
2355	switch(severity) {
2356	default:
2357	case FACET_ROW_SEVERITY_NORMAL:
2358	return "normal";
2359
2360	case FACET_ROW_SEVERITY_DEBUG:
2361	return "debug";
2362
2363	case FACET_ROW_SEVERITY_NOTICE:
2364	return "notice";
2365
2366	case FACET_ROW_SEVERITY_WARNING:
2367	return "warning";
2368
2369	case FACET_ROW_SEVERITY_CRITICAL:
2370	return "critical";
2371	}
2372	}
2373
2374	void facets_accepted_parameters_to_json_array(FACETS facets, BUFFER wb, bool with_keys) {
2375	buffer_json_member_add_array(wb, "accepted_params");
2376	{
2377	if(facets->accepted_params) {
2378	void *t;
2379	dfe_start_read(facets->accepted_params, t) {
2380	buffer_json_add_array_item_string(wb, t_dfe.name);
2381	}
2382	dfe_done(t);
2383	}
2384
2385	if(with_keys) {
2386	FACET_KEY *k;
2387	foreach_key_in_facets(facets, k) {
2388	if (!k->values.enabled \|\| k->options & FACET_KEY_OPTION_HIDDEN)
2389	continue;
2390
2391	buffer_json_add_array_item_string(wb, facets_key_id(k));
2392	}
2393	foreach_key_in_facets_done(k);
2394	}
2395	}
2396	buffer_json_array_close(wb); // accepted_params
2397	}
2398
2399	static int facets_keys_reorder_compar(const void a, const void b) {
2400	const FACET_KEY ak = ((const FACET_KEY **)a);
2401	const FACET_KEY bk = ((const FACET_KEY **)b);
2402
2403	const char *an = ak->name;
2404	const char *bn = bk->name;
2405
2406	if(!an) an = "0";
2407	if(!bn) bn = "0";
2408
2409	while(an && ispunct((uint8_t)an)) an++;
2410	while(bn && ispunct((uint8_t)bn)) bn++;
2411
2412	return strcasecmp(an, bn);
2413	}
2414
2415	void facets_sort_and_reorder_keys(FACETS *facets) {
2416	size_t entries = facets->keys_with_values.used;
2417	if(!entries)
2418	return;
2419
2420	FACET_KEY *keys = mallocz(entries sizeof(*keys));
2421	memcpy(keys, facets->keys_with_values.array, entries * sizeof(*keys));
2422
2423	qsort(keys, entries, sizeof(FACET_KEY *), facets_keys_reorder_compar);
2424
2425	for(size_t i = 0; i < entries ;i++)
2426	keys[i]->order = i + 1;
2427
2428	freez(keys);
2429	}
2430
2431	static int facets_key_values_reorder_by_name_compar(const void a, const void b) {
2432	const FACET_VALUE av = ((const FACET_VALUE **)a);
2433	const FACET_VALUE bv = ((const FACET_VALUE **)b);
2434
2435	const char *an = (av->name && av->name_len) ? av->name : "0";
2436	const char *bn = (bv->name && bv->name_len) ? bv->name : "0";
2437
2438	while(an && ispunct((uint8_t)an)) an++;
2439	while(bn && ispunct((uint8_t)bn)) bn++;
2440
2441	int ret = strcasecmp(an, bn);
2442	return ret;
2443	}
2444
2445	static int facets_key_values_reorder_by_count_compar(const void a, const void b) {
2446	const FACET_VALUE av = ((const FACET_VALUE **)a);
2447	const FACET_VALUE bv = ((const FACET_VALUE **)b);
2448
2449	if(av->final_facet_value_counter < bv->final_facet_value_counter)
2450	return 1;
2451
2452	if(av->final_facet_value_counter > bv->final_facet_value_counter)
2453	return -1;
2454
2455	return facets_key_values_reorder_by_name_compar(a, b);
2456	}
2457
2458	static int facets_key_values_reorder_by_name_numeric_compar(const void a, const void b) {
2459	const FACET_VALUE av = ((const FACET_VALUE **)a);
2460	const FACET_VALUE bv = ((const FACET_VALUE **)b);
2461
2462	const char *an = (av->name && av->name_len) ? av->name : "0";
2463	const char *bn = (bv->name && bv->name_len) ? bv->name : "0";
2464
2465	if(strcmp(an, FACET_VALUE_UNSET) == 0) an = "0";
2466	if(strcmp(bn, FACET_VALUE_UNSET) == 0) bn = "0";
2467
2468	int64_t ad = str2ll(an, NULL);
2469	int64_t bd = str2ll(bn, NULL);
2470
2471	if(ad < bd)
2472	return -1;
2473
2474	if(ad > bd)
2475	return 1;
2476
2477	return facets_key_values_reorder_by_name_compar(a, b);
2478	}
2479
2480	static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) {
2481	bool all_values_numeric = true;
2482	size_t entries = k->values.used;
2483	FACET_VALUE *values = mallocz(entries sizeof(*values));
2484	FACET_VALUE *v;
2485	uint32_t used = 0;
2486	foreach_value_in_key(k, v) {
2487	if((k->facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty)
2488	continue;
2489
2490	if(all_values_numeric && !v->empty && v->name && v->name_len) {
2491	const char *s = v->name;
2492	while(isdigit((uint8_t)*s)) s++;
2493	if(*s != '\0')
2494	all_values_numeric = false;
2495	}
2496
2497	values[used++] = v;
2498
2499	if(used >= entries)
2500	break;
2501	}
2502	foreach_value_in_key_done(v);
2503
2504	if(!used) {
2505	freez(values);
2506	return 0;
2507	}
2508
2509	if(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY) {
2510	if(all_values_numeric)
2511	qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_numeric_compar);
2512	else
2513	qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_compar);
2514	}
2515	else
2516	qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_count_compar);
2517
2518	for(size_t i = 0; i < used; i++)
2519	values[i]->order = i + 1;
2520
2521	freez(values);
2522	return used;
2523	}
2524
2525	static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) {
2526	if(!k->values.enabled \|\| !k->values.ll \|\| !k->values.used)
2527	return 0;
2528
2529	if(!k->transform.cb \|\| !k->transform.view_only \|\| !(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY))
2530	return facets_sort_and_reorder_values_internal(k);
2531
2532	// we have a transformation and has to be sorted alphabetically
2533
2534	BUFFER *tb = buffer_create(0, NULL);
2535	uint32_t ret = 0;
2536
2537	size_t entries = k->values.used;
2538	struct facet_value_restore {
2539	const char *name;
2540	uint32_t name_len;
2541	} values = mallocz(entries sizeof(*values));
2542	FACET_VALUE *v;
2543	uint32_t used = 0;
2544
2545	foreach_value_in_key(k, v) {
2546	if(used >= entries)
2547	break;
2548
2549	values[used].name = v->name;
2550	values[used].name_len = v->name_len;
2551	used++;
2552
2553	facets_key_value_transformed(k->facets, k, v, tb, FACETS_TRANSFORM_FACET_SORT);
2554	v->name = strdupz(buffer_tostring(tb));
2555	v->name_len = buffer_strlen(tb);
2556	}
2557	foreach_value_in_key_done(v);
2558
2559	ret = facets_sort_and_reorder_values_internal(k);
2560
2561	used = 0;
2562	foreach_value_in_key(k, v) {
2563	if(used >= entries)
2564	break;
2565
2566	freez((void *)v->name);
2567	v->name = values[used].name;
2568	v->name_len = values[used].name_len;
2569	used++;
2570	}
2571	foreach_value_in_key_done(v);
2572
2573	buffer_free(tb);
2574	freez(values);
2575	return ret;
2576	}
2577
2578	void facets_table_config(FACETS facets, BUFFER wb) {
2579	buffer_json_member_add_boolean(wb, "show_ids", (facets->options & FACETS_OPTION_HASH_IDS) ? false : true);
2580	buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before
2581
2582	buffer_json_member_add_object(wb, "pagination");
2583	{
2584	buffer_json_member_add_boolean(wb, "enabled", true);
2585	buffer_json_member_add_string(wb, "key", "anchor");
2586	buffer_json_member_add_string(wb, "column", "timestamp");
2587	buffer_json_member_add_string(wb, "units", "timestamp_usec");
2588	}
2589	buffer_json_object_close(wb); // pagination
2590	}
2591
2592	void facets_report(FACETS facets, BUFFER wb, DICTIONARY *used_hashes_registry) {
2593	facets->report.used_hashes_registry = used_hashes_registry;
2594
2595	facets_table_config(facets, wb);
2596
2597	if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {
2598	facets_accepted_parameters_to_json_array(facets, wb, true);
2599	}
2600
2601	// ------------------------------------------------------------------------
2602	// facets
2603
2604	if(!(facets->options & FACETS_OPTION_DONT_SEND_FACETS)) {
2605	bool show_facets = false;
2606
2607	if(facets->options & FACETS_OPTION_DATA_ONLY) {
2608	if(facets->options & FACETS_OPTION_SHOW_DELTAS) {
2609	buffer_json_member_add_array(wb, "facets_delta");
2610	show_facets = true;
2611	}
2612	}
2613	else {
2614	buffer_json_member_add_array(wb, "facets");
2615	show_facets = true;
2616	}
2617
2618	if(show_facets) {
2619	CLEAN_BUFFER *tb = buffer_create(0, NULL);
2620	FACET_KEY *k;
2621	foreach_key_in_facets(facets, k) {
2622	if(!k->values.enabled \|\| k->options & (FACET_KEY_OPTION_HIDDEN\|FACET_KEY_OPTION_FILTER_ONLY))
2623	continue;
2624
2625	facets_sort_and_reorder_values(k);
2626
2627	buffer_json_add_array_item_object(wb); // key
2628	{
2629	buffer_json_member_add_string(
2630	wb, "id", facets_key_id(k));
2631
2632	buffer_json_member_add_string(
2633	wb, "name",
2634	facets_key_name_cached(k, facets->report.used_hashes_registry));
2635
2636	// buffer_json_member_add_string(wb, "raw", k->name);
2637
2638	if(!k->order) k->order = facets->order++;
2639	buffer_json_member_add_uint64(wb, "order", k->order);
2640
2641	buffer_json_member_add_array(wb, "options");
2642	{
2643	FACET_VALUE *v;
2644	foreach_value_in_key(k, v) {
2645	if((facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty)
2646	continue;
2647
2648	if(v->unsampled \|\| v->estimated)
2649	continue;
2650
2651	buffer_json_add_array_item_object(wb);
2652	{
2653	buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v));
2654
2655	facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET);
2656	buffer_json_member_add_string(wb, "name", buffer_tostring(tb));
2657	// buffer_json_member_add_string(wb, "raw", v->name);
2658	buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);
2659	buffer_json_member_add_uint64(wb, "order", v->order);
2660	}
2661	buffer_json_object_close(wb);
2662	}
2663	foreach_value_in_key_done(v);
2664	}
2665	buffer_json_array_close(wb); // options
2666	}
2667	buffer_json_object_close(wb); // key
2668	}
2669	foreach_key_in_facets_done(k);
2670	buffer_json_array_close(wb); // facets
2671	}
2672	}
2673
2674	// ------------------------------------------------------------------------
2675	// columns
2676
2677	buffer_json_member_add_object(wb, "columns");
2678	{
2679	size_t field_id = 0;
2680	buffer_rrdf_table_add_field(
2681	wb, field_id++,
2682	"timestamp", "Timestamp",
2683	RRDF_FIELD_TYPE_TIMESTAMP,
2684	RRDF_FIELD_VISUAL_VALUE,
2685	RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,
2686	RRDF_FIELD_SORT_DESCENDING\|RRDF_FIELD_SORT_FIXED,
2687	NULL,
2688	RRDF_FIELD_SUMMARY_COUNT,
2689	RRDF_FIELD_FILTER_RANGE,
2690	RRDF_FIELD_OPTS_WRAP \| RRDF_FIELD_OPTS_VISIBLE \| RRDF_FIELD_OPTS_UNIQUE_KEY,
2691	NULL);
2692
2693	buffer_rrdf_table_add_field(
2694	wb, field_id++,
2695	"rowOptions", "rowOptions",
2696	RRDF_FIELD_TYPE_NONE,
2697	RRDR_FIELD_VISUAL_ROW_OPTIONS,
2698	RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
2699	RRDF_FIELD_SORT_FIXED,
2700	NULL,
2701	RRDF_FIELD_SUMMARY_COUNT,
2702	RRDF_FIELD_FILTER_NONE,
2703	RRDF_FIELD_OPTS_DUMMY,
2704	NULL);
2705
2706	FACET_KEY *k;
2707	foreach_key_in_facets(facets, k) {
2708	if(k->options & FACET_KEY_OPTION_HIDDEN)
2709	continue;
2710
2711	RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
2712	RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
2713	RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
2714
2715	if (k->options & (FACET_KEY_OPTION_VISIBLE \| FACET_KEY_OPTION_STICKY) \|\|
2716	((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) \|\|
2717	simple_pattern_matches(facets->visible_keys, k->name))
2718	options \|= RRDF_FIELD_OPTS_VISIBLE;
2719
2720	if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
2721	options \|= RRDF_FIELD_OPTS_FULL_WIDTH \| RRDF_FIELD_OPTS_WRAP;
2722
2723	if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
2724	options \|= RRDF_FIELD_OPTS_EXPANDED_FILTER;
2725
2726	if (k->options & FACET_KEY_OPTION_PRETTY_XML)
2727	transform = RRDF_FIELD_TRANSFORM_XML;
2728
2729	const char *key_id = facets_key_id(k);
2730
2731	buffer_rrdf_table_add_field(
2732	wb, field_id++,
2733	key_id, k->name ? k->name : key_id,
2734	RRDF_FIELD_TYPE_STRING,
2735	visual, transform, 0, NULL, NAN,
2736	RRDF_FIELD_SORT_FIXED,
2737	NULL,
2738	RRDF_FIELD_SUMMARY_COUNT,
2739	(k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
2740	options, FACET_VALUE_UNSET);
2741	}
2742	foreach_key_in_facets_done(k);
2743	}
2744	buffer_json_object_close(wb); // columns
2745
2746	// ------------------------------------------------------------------------
2747	// rows data
2748
2749	buffer_json_member_add_array(wb, "data");
2750	{
2751	usec_t last_usec = 0; (void)last_usec;
2752
2753	for(FACET_ROW *row = facets->base ; row ;row = row->next) {
2754
2755	internal_fatal(
2756	facets->anchor.start_ut && (
2757	(facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && row->usec >= facets->anchor.start_ut) \|\|
2758	(facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && row->usec <= facets->anchor.start_ut)
2759	), "Wrong data returned related to %s start anchor!", facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
2760
2761	internal_fatal(last_usec && row->usec > last_usec, "Wrong order of data returned!");
2762
2763	last_usec = row->usec;
2764
2765	buffer_json_add_array_item_array(wb); // each row
2766	buffer_json_add_array_item_uint64(wb, row->usec);
2767	buffer_json_add_array_item_object(wb);
2768	{
2769	if(facets->severity.cb)
2770	row->severity = facets->severity.cb(facets, row, facets->severity.data);
2771
2772	buffer_json_member_add_string(wb, "severity", facets_severity_to_string(row->severity));
2773	}
2774	buffer_json_object_close(wb);
2775
2776	FACET_KEY *k;
2777	foreach_key_in_facets(facets, k) {
2778	if(k->options & FACET_KEY_OPTION_HIDDEN)
2779	continue;
2780
2781	FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
2782
2783	if(unlikely(k->dynamic.cb)) {
2784	if(unlikely(!rkv))
2785	rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv));
2786
2787	k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data);
2788	facets->operations.values.dynamic++;
2789	}
2790	else {
2791	if(!rkv \|\| rkv->empty) {
2792	buffer_json_add_array_item_string(wb, NULL);
2793	}
2794	else if(unlikely(k->transform.cb && k->transform.view_only)) {
2795	k->transform.cb(facets, rkv->wb, FACETS_TRANSFORM_DATA, k->transform.data);
2796	buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
2797	}
2798	else
2799	buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
2800	}
2801	}
2802	foreach_key_in_facets_done(k);
2803	buffer_json_array_close(wb); // each row
2804	}
2805	}
2806	buffer_json_array_close(wb); // data
2807
2808	if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {
2809	buffer_json_member_add_string(wb, "default_sort_column", "timestamp");
2810	buffer_json_member_add_array(wb, "default_charts");
2811	buffer_json_array_close(wb);
2812	}
2813
2814	// ------------------------------------------------------------------------
2815	// histogram
2816
2817	if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DONT_SEND_HISTOGRAM)) {
2818	FACETS_HASH first_histogram_hash = 0;
2819	buffer_json_member_add_array(wb, "available_histograms");
2820	{
2821	FACET_KEY *k;
2822	foreach_key_in_facets(facets, k) {
2823	if (!k->values.enabled \|\| k->options & FACET_KEY_OPTION_HIDDEN)
2824	continue;
2825
2826	if(unlikely(!first_histogram_hash))
2827	first_histogram_hash = k->hash;
2828
2829	buffer_json_add_array_item_object(wb);
2830	buffer_json_member_add_string(wb, "id", facets_key_id(k));
2831	buffer_json_member_add_string(wb, "name", k->name);
2832	buffer_json_member_add_uint64(wb, "order", k->order);
2833	buffer_json_object_close(wb);
2834	}
2835	foreach_key_in_facets_done(k);
2836	}
2837	buffer_json_array_close(wb);
2838
2839	{
2840	FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash);
2841	if(!k \|\| !k->values.enabled)
2842	k = FACETS_KEY_GET_FROM_INDEX(facets, first_histogram_hash);
2843
2844	bool show_histogram = false;
2845
2846	if(facets->options & FACETS_OPTION_DATA_ONLY) {
2847	if(facets->options & FACETS_OPTION_SHOW_DELTAS) {
2848	buffer_json_member_add_object(wb, "histogram_delta");
2849	show_histogram = true;
2850	}
2851	}
2852	else {
2853	buffer_json_member_add_object(wb, "histogram");
2854	show_histogram = true;
2855	}
2856
2857	if(show_histogram) {
2858	buffer_json_member_add_string(wb, "id", k ? facets_key_id(k) : "");
2859	buffer_json_member_add_string(wb, "name", k ? k->name : "");
2860	buffer_json_member_add_object(wb, "chart");
2861	{
2862	facets_histogram_generate(facets, k, wb);
2863	}
2864	buffer_json_object_close(wb); // chart
2865	buffer_json_object_close(wb); // histogram
2866	}
2867	}
2868	}
2869
2870	// ------------------------------------------------------------------------
2871	// items
2872
2873	bool show_items = false;
2874	if(facets->options & FACETS_OPTION_DATA_ONLY) {
2875	if(facets->options & FACETS_OPTION_SHOW_DELTAS) {
2876	buffer_json_member_add_object(wb, "items_delta");
2877	show_items = true;
2878	}
2879	}
2880	else {
2881	buffer_json_member_add_object(wb, "items");
2882	show_items = true;
2883	}
2884
2885	if(show_items) {
2886	buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated);
2887	buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched);
2888	buffer_json_member_add_uint64(wb, "unsampled", facets->operations.rows.unsampled);
2889	buffer_json_member_add_uint64(wb, "estimated", facets->operations.rows.estimated);
2890	buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);
2891	buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);
2892	buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);
2893	buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);
2894	buffer_json_object_close(wb); // items
2895	}
2896
2897	// ------------------------------------------------------------------------
2898	// stats
2899
2900	buffer_json_member_add_object(wb, "_stats");
2901	{
2902	buffer_json_member_add_uint64(wb, "first", facets->operations.first);
2903	buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards);
2904	buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards);
2905	buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before);
2906	buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after);
2907	buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends);
2908	buffer_json_member_add_uint64(wb, "appends", facets->operations.appends);
2909	buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts);
2910	buffer_json_member_add_object(wb, "rows");
2911	{
2912	buffer_json_member_add_uint64(wb, "created", facets->operations.rows.created);
2913	buffer_json_member_add_uint64(wb, "reused", facets->operations.rows.reused);
2914	buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated);
2915	buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched);
2916	}
2917	buffer_json_object_close(wb); // rows
2918	buffer_json_member_add_object(wb, "keys");
2919	{
2920	size_t resizes = 0, searches = 0, collisions = 0, used = 0, size = 0, count = 0;
2921	count++;
2922	used += facets->keys.ht.used;
2923	size += facets->keys.ht.size;
2924	resizes += facets->keys.ht.resizes;
2925	searches += facets->keys.ht.searches;
2926	collisions += facets->keys.ht.collisions;
2927
2928	buffer_json_member_add_uint64(wb, "registered", facets->operations.keys.registered);
2929	buffer_json_member_add_uint64(wb, "unique", facets->operations.keys.unique);
2930	buffer_json_member_add_uint64(wb, "hashtables", count);
2931	buffer_json_member_add_uint64(wb, "hashtable_used", used);
2932	buffer_json_member_add_uint64(wb, "hashtable_size", size);
2933	buffer_json_member_add_uint64(wb, "hashtable_searches", searches);
2934	buffer_json_member_add_uint64(wb, "hashtable_collisions", collisions);
2935	buffer_json_member_add_uint64(wb, "hashtable_resizes", resizes);
2936	}
2937	buffer_json_object_close(wb); // keys
2938	buffer_json_member_add_object(wb, "values");
2939	{
2940	size_t resizes = 0, searches = 0, collisions = 0, used = 0, size = 0, count = 0;
2941	for(FACET_KEY *k = facets->keys.ll; k ; k = k->next) {
2942	count++;
2943	used += k->values.ht.used;
2944	size += k->values.ht.size;
2945	resizes += k->values.ht.resizes;
2946	searches += k->values.ht.searches;
2947	collisions += k->values.ht.collisions;
2948	}
2949
2950	buffer_json_member_add_uint64(wb, "registered", facets->operations.values.registered);
2951	buffer_json_member_add_uint64(wb, "transformed", facets->operations.values.transformed);
2952	buffer_json_member_add_uint64(wb, "dynamic", facets->operations.values.dynamic);
2953	buffer_json_member_add_uint64(wb, "empty", facets->operations.values.empty);
2954	buffer_json_member_add_uint64(wb, "unsampled", facets->operations.values.unsampled);
2955	buffer_json_member_add_uint64(wb, "estimated", facets->operations.values.estimated);
2956	buffer_json_member_add_uint64(wb, "indexed", facets->operations.values.indexed);
2957	buffer_json_member_add_uint64(wb, "inserts", facets->operations.values.inserts);
2958	buffer_json_member_add_uint64(wb, "conflicts", facets->operations.values.conflicts);
2959	buffer_json_member_add_uint64(wb, "hashtables", count);
2960	buffer_json_member_add_uint64(wb, "hashtable_used", used);
2961	buffer_json_member_add_uint64(wb, "hashtable_size", size);
2962	buffer_json_member_add_uint64(wb, "hashtable_searches", searches);
2963	buffer_json_member_add_uint64(wb, "hashtable_collisions", collisions);
2964	buffer_json_member_add_uint64(wb, "hashtable_resizes", resizes);
2965	}
2966	buffer_json_object_close(wb); // values
2967	buffer_json_member_add_object(wb, "fts");
2968	{
2969	buffer_json_member_add_uint64(wb, "searches", facets->operations.fts.searches);
2970	}
2971	buffer_json_object_close(wb); // fts
2972	}
2973	buffer_json_object_close(wb); // items
2974	}