@cryptotaxi247 / netdata-1 / commits / fc9b5170c

log2journal improvements 5 (#16519)

* added ${LINE} variable; added default config * prefer single quotes in yaml to avoid interference from yaml escaping * simple_hashtable now supports deletions * simple hashtable now supports setting entries with NULL values * hashtable implementation now has sorting option to maintain a sorted list of the items * multiple hashtables with type checking * added comments * still incomplete yaml parser * fixes and cleanup

Costa Tsaousis committed Dec 3, 2023 at 17:00 UTC fc9b5170c6301366bd66d9b0766850a0ed8dc807
17 files changed +684 -253
collectors/log2journal/Makefile.am
+1
@@ -13,4 +13,5 @@ log2journalconfigdir=$(libconfigdir)/log2journal.d
13 dist_log2journalconfig_DATA = \
14 log2journal.d/nginx-combined.yaml \
15 log2journal.d/nginx-json.yaml \
16 + log2journal.d/default.yaml \
17 $(NULL)
collectors/log2journal/README.md
+6 -1
@@ -2,7 +2,7 @@
2
3 `log2journal` and `systemd-cat-native` can be used to convert a structured log file, such as the ones generated by web servers, into `systemd-journal` entries.
4
5 -By combining these tools, together with the usual UNIX shell tools you can create advanced log processing pipelines sending any kind of structured text logs to systemd-journald. This is a simple, but powerful and efficient way to handle log processing.
5 +By combining these tools you can create advanced log processing pipelines sending any kind of structured text logs to systemd-journald. This is a simple, but powerful and efficient way to handle log processing.
6
7 The process involves the usual piping of shell commands, to get and process the log files in realtime.
8
@@ -27,6 +27,11 @@ Let's see the steps:
27 ```
28 3. `systemd-cat-native` is a Netdata program. I can send the logs to a local `systemd-journald` (journal namespaces supported), or to a remote `systemd-journal-remote`.
29
30 +
31 +## YAML configuration
32 +
33 +
34 +
35 ## Real-life example
36
37 We have an nginx server logging in this format:
collectors/log2journal/log2journal-help.c
+1 -1
@@ -60,7 +60,7 @@ void log_job_command_line_help(const char *name) {
60 printf(" --file /path/to/file.yaml or -f /path/to/file.yaml\n");
61 printf(" Read yaml configuration file for instructions.\n");
62 printf("\n");
63 - printf(" --config CONFIG_NAME\n");
63 + printf(" --config CONFIG_NAME or -c CONFIG_NAME\n");
64 printf(" Run with the internal configuration named CONFIG_NAME.\n");
65 printf(" Available internal configs:\n");
66 printf("\n");
collectors/log2journal/log2journal-json.c
+57 -18
@@ -167,7 +167,7 @@ static inline bool json_parse_number(LOG_JSON_STATE *js) {
167 }
168 }
169
170 -static bool encode_utf8(unsigned codepoint, char **d, size_t *remaining) {
170 +static inline bool encode_utf8(unsigned codepoint, char **d, size_t *remaining) {
171 if (codepoint <= 0x7F) {
172 // 1-byte sequence
173 if (*remaining < 2) return false; // +1 for the null
@@ -205,6 +205,56 @@ static bool encode_utf8(unsigned codepoint, char **d, size_t *remaining) {
205 return true;
206 }
207
208 +size_t parse_surrogate(const char *s, char *d, size_t *remaining) {
209 + if (s[0] != '\\' || (s[1] != 'u' && s[1] != 'U')) {
210 + return 0; // Not a valid Unicode escape sequence
211 + }
212 +
213 + char hex[9] = {0}; // Buffer for the hexadecimal value
214 + unsigned codepoint;
215 +
216 + if (s[1] == 'u') {
217 + // Handle \uXXXX
218 + if (!isxdigit(s[2]) || !isxdigit(s[3]) || !isxdigit(s[4]) || !isxdigit(s[5])) {
219 + return 0; // Not a valid \uXXXX sequence
220 + }
221 +
222 + hex[0] = s[2];
223 + hex[1] = s[3];
224 + hex[2] = s[4];
225 + hex[3] = s[5];
226 + codepoint = (unsigned)strtoul(hex, NULL, 16);
227 +
228 + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
229 + // Possible start of surrogate pair
230 + if (s[6] == '\\' && s[7] == 'u' && isxdigit(s[8]) && isxdigit(s[9]) &&
231 + isxdigit(s[10]) && isxdigit(s[11])) {
232 + // Valid low surrogate
233 + unsigned low_surrogate = strtoul(&s[8], NULL, 16);
234 + if (low_surrogate < 0xDC00 || low_surrogate > 0xDFFF) {
235 + return 0; // Invalid low surrogate
236 + }
237 + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low_surrogate - 0xDC00);
238 + return encode_utf8(codepoint, &d, remaining) ? 12 : 0; // \uXXXX\uXXXX
239 + }
240 + }
241 +
242 + // Single \uXXXX
243 + return encode_utf8(codepoint, &d, remaining) ? 6 : 0;
244 + }
245 + else {
246 + // Handle \UXXXXXXXX
247 + for (int i = 2; i < 10; i++) {
248 + if (!isxdigit(s[i])) {
249 + return 0; // Not a valid \UXXXXXXXX sequence
250 + }
251 + hex[i - 2] = s[i];
252 + }
253 + codepoint = (unsigned)strtoul(hex, NULL, 16);
254 + return encode_utf8(codepoint, &d, remaining) ? 10 : 0; // \UXXXXXXXX
255 + }
256 +}
257 +
258 static inline void copy_newline(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {
259 if(*remaining > 3) {
260 *(*d)++ = '\\';
@@ -258,18 +308,12 @@ static inline bool json_parse_string(LOG_JSON_STATE *js) {
308 s++;
309 break;
310
261 - case 'u':
262 - if(isxdigit(s[1]) && isxdigit(s[2]) && isxdigit(s[3]) && isxdigit(s[4])) {
263 - char b[5] = {
264 - [0] = s[1],
265 - [1] = s[2],
266 - [2] = s[3],
267 - [3] = s[4],
268 - [4] = '\0',
269 - };
270 - unsigned codepoint = strtoul(b, NULL, 16);
271 - if(encode_utf8(codepoint, &d, &remaining)) {
272 - s += 5;
311 + case 'u': {
312 + size_t old_remaining = remaining;
313 + size_t consumed = parse_surrogate(s - 1, d, &remaining);
314 + if (consumed > 0) {
315 + s += consumed - 1; // -1 because we already incremented s after '\\'
316 + d += old_remaining - remaining;
317 continue;
318 }
319 else {
@@ -278,11 +322,6 @@ static inline bool json_parse_string(LOG_JSON_STATE *js) {
322 c = *s++;
323 }
324 }
281 - else {
282 - *d++ = '\\';
283 - remaining--;
284 - c = *s++;
285 - }
325 break;
326
327 default:
collectors/log2journal/log2journal-params.c
+15 -12
@@ -6,22 +6,25 @@
6
7 void log_job_init(LOG_JOB *jb) {
8 memset(jb, 0, sizeof(*jb));
9 - simple_hashtable_init(&jb->hashtable, 32);
9 + simple_hashtable_init_KEY(&jb->hashtable, 32);
10 + hashed_key_set(&jb->line.key, "LINE");
11 }
12
12 -static void simple_hashtable_cleanup_allocated(SIMPLE_HASHTABLE *ht) {
13 - for(size_t i = 0; i < ht->size ;i++) {
14 - HASHED_KEY *k = ht->hashtable[i].data;
13 +static void simple_hashtable_cleanup_allocated_keys(SIMPLE_HASHTABLE_KEY *ht) {
14 + SIMPLE_HASHTABLE_FOREACH_READ_ONLY(ht, sl, _KEY) {
15 + HASHED_KEY *k = SIMPLE_HASHTABLE_FOREACH_READ_ONLY_VALUE(sl);
16 if(k && k->flags & HK_HASHTABLE_ALLOCATED) {
16 - hashed_key_cleanup(k);
17 - freez(k);
18 - ht->hashtable[i].data = NULL;
19 - ht->hashtable[i].hash = 0;
17 + // the order of these statements is important!
18 + simple_hashtable_del_slot_KEY(ht, sl); // remove any references to n
19 + hashed_key_cleanup(k); // cleanup the internals of n
20 + freez(k); // free n
21 }
22 }
23 }
24
25 void log_job_cleanup(LOG_JOB *jb) {
26 + hashed_key_cleanup(&jb->line.key);
27 +
28 if(jb->prefix) {
29 freez((void *) jb->prefix);
30 jb->prefix = NULL;
@@ -47,8 +50,8 @@ void log_job_cleanup(LOG_JOB *jb) {
50 txt_cleanup(&jb->rewrites.tmp);
51 txt_cleanup(&jb->filename.current);
52
50 - simple_hashtable_cleanup_allocated(&jb->hashtable);
51 - simple_hashtable_free(&jb->hashtable);
53 + simple_hashtable_cleanup_allocated_keys(&jb->hashtable);
54 + simple_hashtable_destroy_KEY(&jb->hashtable);
55
56 // remove references to everything else, to reveal them in valgrind
57 memset(jb, 0, sizeof(*jb));
@@ -346,7 +349,7 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
349 if (!yaml_parse_file(value, jb))
350 return false;
351 }
349 - else if (strcmp(param, "--config") == 0) {
352 + else if (strcmp(param, "-c") == 0 || strcmp(param, "--config") == 0) {
353 if (!yaml_parse_config(value, jb))
354 return false;
355 }
@@ -392,7 +395,7 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
395
396 // Check if a pattern is set and exactly one pattern is specified
397 if (!jb->pattern) {
395 - log2stderr("Error: Pattern not specified.");
398 + log2stderr("Warning: pattern not specified. Try the default config with: -c default");
399 log_job_command_line_help(argv[0]);
400 return false;
401 }
collectors/log2journal/log2journal-yaml.c
+1 -1
@@ -852,7 +852,7 @@ static bool needs_quotes_in_yaml(const char *str) {
852
853 static void yaml_print_node(const char *key, const char *value, size_t depth, bool dash) {
854 if(depth > 10) depth = 10;
855 - const char *quote = "\"";
855 + const char *quote = "'";
856
857 const char *second_line = NULL;
858 if(value && strchr(value, '\n')) {
collectors/log2journal/log2journal.c
+48 -59
@@ -61,37 +61,14 @@ const char journal_key_characters_map[256] = {
61
62 // ----------------------------------------------------------------------------
63
64 -// Function to insert a key into the sorted.keys array while keeping it sorted
65 -void log_job_add_key_sorted(LOG_JOB *jb, HASHED_KEY *newKey) {
66 - size_t i, j;
67 -
68 - // Find the position to insert the new key based on lexicographic order
69 - for (i = 0; i < jb->sorted.used; i++) {
70 - if (strcmp(newKey->key, jb->sorted.keys[i]->key) < 0) {
71 - break;
72 - }
73 - }
74 -
75 - // Shift elements to the right to make space for the new key
76 - for (j = jb->sorted.used; j > i; j--) {
77 - jb->sorted.keys[j] = jb->sorted.keys[j - 1];
78 - }
79 -
80 - // Insert the new key at the correct position
81 - jb->sorted.keys[i] = newKey;
82 - jb->sorted.used++;
83 -}
84 -
64 static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
65 if(k->flags & HK_HASHTABLE_ALLOCATED)
66 return k;
67
68 if(!k->hashtable_ptr) {
69 HASHED_KEY *ht_key;
91 - SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&jb->hashtable, k->hash, true);
92 - if(slot->data) {
93 - ht_key = slot->data;
94 -
70 + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&jb->hashtable, k->hash, true);
71 + if((ht_key = SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
72 if(!(ht_key->flags & HK_COLLISION_CHECKED)) {
73 ht_key->flags |= HK_COLLISION_CHECKED;
74
@@ -109,11 +86,7 @@ static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
86 ht_key->hash = k->hash;
87 ht_key->flags = HK_HASHTABLE_ALLOCATED;
88
112 - slot->hash = ht_key->hash;
113 - slot->data = ht_key;
114 - jb->hashtable.used++;
115 -
116 - log_job_add_key_sorted(jb, ht_key);
89 + simple_hashtable_set_slot_KEY(&jb->hashtable, slot, ht_key->hash, ht_key);
90 }
91
92 k->hashtable_ptr = ht_key;
@@ -158,18 +131,25 @@ static inline void validate_key(LOG_JOB *jb __maybe_unused, HASHED_KEY *k) {
131
132 // ----------------------------------------------------------------------------
133
161 -static inline size_t replace_evaluate_to_buffer(LOG_JOB *jb, HASHED_KEY *k, REPLACE_PATTERN *rp, char *dst, size_t dst_size) {
134 +static inline size_t replace_evaluate_to_buffer(LOG_JOB *jb, HASHED_KEY *k __maybe_unused, REPLACE_PATTERN *rp, char *dst, size_t dst_size) {
135 size_t remaining = dst_size;
136 char *copy_to = dst;
137
138 for(REPLACE_NODE *node = rp->nodes; node != NULL && remaining > 1; node = node->next) {
139 if(node->is_variable) {
167 - HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
168 - if(ktmp->value.len) {
169 - size_t copied = copy_to_buffer(copy_to, remaining, ktmp->value.txt, ktmp->value.len);
140 + if(hashed_keys_match(&node->name, &jb->line.key)) {
141 + size_t copied = copy_to_buffer(copy_to, remaining, jb->line.trimmed, jb->line.trimmed_len);
142 copy_to += copied;
143 remaining -= copied;
144 }
145 + else {
146 + HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
147 + if(ktmp->value.len) {
148 + size_t copied = copy_to_buffer(copy_to, remaining, ktmp->value.txt, ktmp->value.len);
149 + copy_to += copied;
150 + remaining -= copied;
151 + }
152 + }
153 }
154 else {
155 size_t copied = copy_to_buffer(copy_to, remaining, node->name.key, node->name.len);
@@ -189,9 +169,14 @@ static inline void replace_evaluate(LOG_JOB *jb, HASHED_KEY *k, REPLACE_PATTERN
169
170 for(REPLACE_NODE *node = rp->nodes; node != NULL; node = node->next) {
171 if(node->is_variable) {
192 - HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
193 - if(ktmp->value.len)
194 - txt_expand_and_append(&ht_key->value, ktmp->value.txt, ktmp->value.len);
172 + if(hashed_keys_match(&node->name, &jb->line.key))
173 + txt_expand_and_append(&ht_key->value, jb->line.trimmed, jb->line.trimmed_len);
174 +
175 + else {
176 + HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
177 + if(ktmp->value.len)
178 + txt_expand_and_append(&ht_key->value, ktmp->value.txt, ktmp->value.len);
179 + }
180 }
181 else
182 txt_expand_and_append(&ht_key->value, node->name.key, node->name.len);
@@ -220,9 +205,14 @@ static inline void replace_evaluate_from_pcre2(LOG_JOB *jb, HASHED_KEY *k, REPLA
205 txt_expand_and_append(&jb->rewrites.tmp, k->value.txt + start_offset, length);
206 }
207 else {
223 - HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
224 - if(ktmp->value.len)
225 - txt_expand_and_append(&jb->rewrites.tmp, ktmp->value.txt, ktmp->value.len);
208 + if(hashed_keys_match(&node->name, &jb->line.key))
209 + txt_expand_and_append(&jb->rewrites.tmp, jb->line.trimmed, jb->line.trimmed_len);
210 +
211 + else {
212 + HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
213 + if(ktmp->value.len)
214 + txt_expand_and_append(&jb->rewrites.tmp, ktmp->value.txt, ktmp->value.len);
215 + }
216 }
217 }
218 else {
@@ -299,15 +289,6 @@ static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char
289 printf("\n");
290 }
291
302 -static inline void send_key_value_and_rewrite(LOG_JOB *jb, HASHED_KEY *key, const char *value, size_t len) {
303 - HASHED_KEY *ht_key = get_key_from_hashtable(jb, key);
304 -
305 - txt_replace(&ht_key->value, value, len);
306 - ht_key->flags |= HK_VALUE_FROM_LOG;
307 -
308 -// fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt);
309 -}
310 -
292 inline void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len) {
293 HASHED_KEY *ht_key = get_key_from_hashtable_with_char_ptr(jb, key);
294 HASHED_KEY *nk = rename_key(jb, ht_key);
@@ -341,8 +322,8 @@ static inline void log_job_process_rewrites(LOG_JOB *jb) {
322 }
323
324 static inline void send_all_fields(LOG_JOB *jb) {
344 - for(size_t i = 0; i < jb->sorted.used ;i++) {
345 - HASHED_KEY *k = jb->sorted.keys[i];
325 + SIMPLE_HASHTABLE_SORTED_FOREACH_READ_ONLY(&jb->hashtable, kptr, HASHED_KEY, _KEY) {
326 + HASHED_KEY *k = SIMPLE_HASHTABLE_SORTED_FOREACH_READ_ONLY_VALUE(kptr);
327
328 if(k->value.len) {
329 // the key exists and has some value
@@ -496,11 +477,13 @@ int log_job_run(LOG_JOB *jb) {
477
478 if(strcmp(jb->pattern, "json") == 0) {
479 json = json_parser_create(jb);
480 + // never fails
481 }
482 else if(strcmp(jb->pattern, "logfmt") == 0) {
483 logfmt = logfmt_parser_create(jb);
484 + // never fails
485 }
503 - else {
486 + else if(strcmp(jb->pattern, "none") != 0) {
487 pcre2 = pcre2_parser_create(jb);
488 if(pcre2_has_error(pcre2)) {
489 log2stderr("%s", pcre2_parser_error(pcre2));
@@ -509,21 +492,25 @@ int log_job_run(LOG_JOB *jb) {
492 }
493 }
494
512 - char buffer[MAX_LINE_LENGTH];
513 - char *line;
514 - size_t len;
495 + jb->line.buffer = mallocz(MAX_LINE_LENGTH + 1);
496 + jb->line.size = MAX_LINE_LENGTH + 1;
497 + jb->line.trimmed_len = 0;
498 + jb->line.trimmed = jb->line.buffer;
499 +
500 + while ((jb->line.trimmed = get_next_line(jb, (char *)jb->line.buffer, jb->line.size, &jb->line.trimmed_len))) {
501 + const char *line = jb->line.trimmed;
502 + size_t len = jb->line.trimmed_len;
503
516 - while ((line = get_next_line(jb, buffer, sizeof(buffer), &len))) {
504 if(jb_switched_filename(jb, line, len))
505 continue;
506
520 - bool line_is_matched;
507 + bool line_is_matched = true;
508
509 if(json)
510 line_is_matched = json_parse_document(json, line);
511 else if(logfmt)
512 line_is_matched = logfmt_parse_document(logfmt, line);
526 - else
513 + else if(pcre2)
514 line_is_matched = pcre2_parse_document(pcre2, line, len);
515
516 if(!line_is_matched) {
@@ -531,7 +518,7 @@ int log_job_run(LOG_JOB *jb) {
518 log2stderr("%s", json_parser_error(json));
519 else if(logfmt)
520 log2stderr("%s", logfmt_parser_error(logfmt));
534 - else
521 + else if(pcre2)
522 log2stderr("%s", pcre2_parser_error(pcre2));
523
524 if(!jb_send_unmatched_line(jb, line))
@@ -557,6 +544,8 @@ int log_job_run(LOG_JOB *jb) {
544 else if(pcre2)
545 pcre2_parser_destroy(pcre2);
546
547 + freez((void *)jb->line.buffer);
548 +
549 return 0;
550 }
551
collectors/log2journal/log2journal.d/default.yaml new
+15
@@ -0,0 +1,15 @@
1 +pattern: none
2 +
3 +filename:
4 + key: LOG_FILENAME
5 +
6 +inject:
7 + - key: MESSAGE
8 + value: '${LINE}' # a special variable that resolves to the whole line read from the log
9 +
10 + - key: PRIORITY
11 + value: 6 # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
12 +
13 + - key: SYSLOG_IDENTIFIER
14 + value: log2journal # the name of the application sending the logs
15 +
collectors/log2journal/log2journal.d/nginx-combined.yaml
+19 -19
@@ -37,15 +37,15 @@ rename:
37 # Inject constant fields into the journal logs.
38 inject:
39 - key: SYSLOG_IDENTIFIER
40 - value: "nginx-log"
40 + value: nginx-log
41
42 # inject PRIORITY is a duplicate of NGINX_STATUS
43 - - key: "PRIORITY"
44 - value: "${NGINX_STATUS}"
43 + - key: PRIORITY
44 + value: '${NGINX_STATUS}'
45
46 # Inject NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
47 - - key: "NGINX_STATUS_FAMILY"
48 - value: "${NGINX_STATUS}"
47 + - key: NGINX_STATUS_FAMILY
48 + value: '${NGINX_STATUS}'
49
50 # Rewrite the value of fields (including the duplicated ones).
51 # The search pattern can have named groups, and the replace pattern can use
@@ -53,30 +53,30 @@ inject:
53 rewrite:
54 # PRIORITY is a duplicate of NGINX_STATUS
55 # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
56 - - key: "PRIORITY"
57 - match: "^[123]"
56 + - key: PRIORITY
57 + match: '^[123]'
58 value: 6
59
60 - - key: "PRIORITY"
61 - match: "^4"
60 + - key: PRIORITY
61 + match: '^4'
62 value: 5
63
64 - - key: "PRIORITY"
65 - match: "^5"
64 + - key: PRIORITY
65 + match: '^5'
66 value: 3
67
68 - - key: "PRIORITY"
69 - match: ".*"
68 + - key: PRIORITY
69 + match: '.*'
70 value: 4
71
72 # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
73 - - key: "NGINX_STATUS_FAMILY"
74 - match: "^(?<first_digit>[1-5])"
75 - value: "${first_digit}xx"
73 + - key: NGINX_STATUS_FAMILY
74 + match: '^(?<first_digit>[1-5])'
75 + value: '${first_digit}xx'
76
77 - - key: "NGINX_STATUS_FAMILY"
78 - match: ".*"
79 - value: "UNKNOWN"
77 + - key: NGINX_STATUS_FAMILY
78 + match: '.*'
79 + value: 'UNKNOWN'
80
81 # Control what to do when input logs do not match the main PCRE2 pattern.
82 unmatched:
collectors/log2journal/log2journal.d/nginx-json.yaml
+39 -39
@@ -12,7 +12,7 @@ filename:
12 key: NGINX_LOG_FILENAME
13
14 filter:
15 - exclude: "NGINX_BINARY_REMOTE_ADDR"
15 + exclude: '^(NGINX_BINARY_REMOTE_ADDR)$'
16
17 rename:
18 - new_key: MESSAGE
@@ -69,15 +69,15 @@ rename:
69 # Inject constant fields into the journal logs.
70 inject:
71 - key: SYSLOG_IDENTIFIER
72 - value: "nginx-log"
72 + value: nginx-log
73
74 # inject PRIORITY is a duplicate of NGINX_STATUS
75 - - key: "PRIORITY"
76 - value: "${NGINX_STATUS}"
75 + - key: PRIORITY
76 + value: '${NGINX_STATUS}'
77
78 # Inject NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
79 - - key: "NGINX_STATUS_FAMILY"
80 - value: "${NGINX_STATUS}"
79 + - key: NGINX_STATUS_FAMILY
80 + value: '${NGINX_STATUS}'
81
82
83 # Rewrite the value of fields (including the duplicated ones).
@@ -87,69 +87,69 @@ rewrite:
87 # a ? means it has query string, everything else means it does not
88 - key: NGINX_HAS_QUERY_STRING
89 match: '^\?$'
90 - value: "yes"
90 + value: yes
91 - key: NGINX_HAS_QUERY_STRING
92 - match: ".*"
93 - value: "no"
92 + match: '.*'
93 + value: no
94
95 # 'on' means it was HTTPS, everything else means it was not
96 - key: NGINX_HTTPS
97 - match: "^on$"
98 - value: "yes"
97 + match: '^on$'
98 + value: yes
99 - key: NGINX_HTTPS
100 - match: ".*"
101 - value: "no"
100 + match: '.*'
101 + value: no
102
103 # 'p' means it was pipelined, everything else means it was not
104 - key: NGINX_PIPELINED
105 - match: "^p$"
106 - value: "yes"
105 + match: '^p$'
106 + value: yes
107 - key: NGINX_PIPELINED
108 - match: ".*"
109 - value: "no"
108 + match: '.*'
109 + value: no
110
111 # zero means client sent a certificate and it was verified, non-zero means otherwise
112 - key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERIFY
113 - match: "^0$"
114 - value: "yes"
113 + match: '^0$'
114 + value: yes
115 - key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERIFY
116 - match: ".*"
117 - value: "no"
116 + match: '.*'
117 + value: no
118
119 # 'OK' means request completed, everything else means it didn't
120 - key: NGINX_REQUEST_COMPLETION
121 - match: "^OK$"
122 - value: "completed"
121 + match: '^OK$'
122 + value: 'completed'
123 - key: NGINX_REQUEST_COMPLETION
124 - match: ".*"
125 - value: "not completed"
124 + match: '.*'
125 + value: 'not completed'
126
127 # PRIORTY is a duplicate of NGINX_STATUS
128 # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
129 - - key: "PRIORITY"
130 - match: "^[123]"
129 + - key: PRIORITY
130 + match: '^[123]'
131 value: 6
132
133 - - key: "PRIORITY"
134 - match: "^4"
133 + - key: PRIORITY
134 + match: '^4'
135 value: 5
136
137 - - key: "PRIORITY"
138 - match: "^5"
137 + - key: PRIORITY
138 + match: '^5'
139 value: 3
140
141 - - key: "PRIORITY"
142 - match: ".*"
141 + - key: PRIORITY
142 + match: '.*'
143 value: 4
144
145 # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
146 - - key: "NGINX_STATUS_FAMILY"
147 - match: "^(?<first_digit>[1-5])"
148 - value: "${first_digit}xx"
146 + - key: NGINX_STATUS_FAMILY
147 + match: '^(?<first_digit>[1-5])'
148 + value: '${first_digit}xx'
149
150 - - key: "NGINX_STATUS_FAMILY"
151 - match: ".*"
152 - value: "UNKNOWN"
150 + - key: NGINX_STATUS_FAMILY
151 + match: '.*'
152 + value: 'UNKNOWN'
153
154 # Control what to do when input logs do not match the main PCRE2 pattern.
155 unmatched:
collectors/log2journal/log2journal.h
+47 -19
@@ -13,6 +13,7 @@
13 #include <stdbool.h>
14 #include <string.h>
15 #include <ctype.h>
16 +#include <math.h>
17 #include <stdarg.h>
18 #include <assert.h>
19
@@ -42,11 +43,23 @@ static inline void *mallocz(size_t size) {
43 }
44
45 static inline void *callocz(size_t elements, size_t size) {
45 - void *ptr = mallocz(elements * size);
46 - memset(ptr, 0, elements * size);
46 + void *ptr = calloc(elements, size);
47 + if (!ptr) {
48 + log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", elements * size);
49 + exit(EXIT_FAILURE);
50 + }
51 return ptr;
52 }
53
54 +static inline void *reallocz(void *ptr, size_t size) {
55 + void *new_ptr = realloc(ptr, size);
56 + if (!new_ptr) {
57 + log2stderr("Fatal Error: Memory reallocation failed. Requested size: %zu bytes.", size);
58 + exit(EXIT_FAILURE);
59 + }
60 + return new_ptr;
61 +}
62 +
63 static inline char *strdupz(const char *s) {
64 char *ptr = strdup(s);
65 if (!ptr) {
@@ -74,7 +87,6 @@ static inline void freez(void *ptr) {
87
88 #define XXH_INLINE_ALL
89 #include "../../libnetdata/xxhash.h"
77 -#include "../../libnetdata/simple_hashtable.h"
90
91 #define PCRE2_CODE_UNIT_WIDTH 8
92 #include <pcre2.h>
@@ -83,13 +95,29 @@ static inline void freez(void *ptr) {
95 #include <yaml.h>
96 #endif
97
98 +// ----------------------------------------------------------------------------
99 +// hashtable for HASHED_KEY
100 +
101 +// cleanup hashtable defines
102 +#undef SIMPLE_HASHTABLE_SORT_FUNCTION
103 +#undef SIMPLE_HASHTABLE_VALUE_TYPE
104 +#undef SIMPLE_HASHTABLE_NAME
105 +#undef NETDATA_SIMPLE_HASHTABLE_H
106 +
107 +struct hashed_key;
108 +static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2);
109 +#define SIMPLE_HASHTABLE_SORT_FUNCTION compare_keys
110 +#define SIMPLE_HASHTABLE_VALUE_TYPE struct hashed_key
111 +#define SIMPLE_HASHTABLE_NAME _KEY
112 +#include "../../libnetdata/simple_hashtable.h"
113 +
114 +// ----------------------------------------------------------------------------
115 +
116 #define MAX_OUTPUT_KEYS 1024
117 #define MAX_LINE_LENGTH (1024 * 1024)
88 -#define MAX_KEY_DUPS (MAX_OUTPUT_KEYS / 2)
118 #define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
119 #define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
120 #define MAX_RENAMES (MAX_OUTPUT_KEYS / 2)
92 -#define MAX_KEY_DUPS_KEYS 20
121
122 #define JOURNAL_MAX_KEY_LEN 64 // according to systemd-journald
123 #define JOURNAL_MAX_VALUE_LEN (48 * 1024) // according to systemd-journald
@@ -178,13 +206,7 @@ static inline void txt_expand_and_append(TEXT *t, const char *s, size_t len) {
206 if(new_size < t->size * 2)
207 new_size = t->size * 2;
208
181 - char *b = mallocz(new_size);
182 - if(t->txt) {
183 - memcpy(b, t->txt, t->len);
184 - freez(t->txt);
185 - }
186 -
187 - t->txt = b;
209 + t->txt = reallocz(t->txt, new_size);
210 t->size = new_size;
211 }
212
@@ -213,9 +235,6 @@ typedef enum __attribute__((__packed__)) {
235 HK_RENAMES_CHECKED = (1 << 4), // we checked once if there are renames on this key
236 HK_HAS_RENAMES = (1 << 5), // and we found there is a rename rule related to it
237
216 - HK_DUPS_CHECKED = (1 << 6), // we checked once if there are duplications for this key
217 - HK_HAS_DUPS = (1 << 7), // and we found there are duplication related to it
218 -
238 // ephemeral flags - they are unset at the end of each log line
239
240 HK_VALUE_FROM_LOG = (1 << 14), // the value of this key has been read from the log (or from injection, duplication)
@@ -268,6 +287,10 @@ static inline bool hashed_keys_match(HASHED_KEY *k1, HASHED_KEY *k2) {
287 return ((k1 == k2) || (k1->hash == k2->hash && strcmp(k1->key, k2->key) == 0));
288 }
289
290 +static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2) {
291 + return strcmp(k1->key, k2->key);
292 +}
293 +
294 // ----------------------------------------------------------------------------
295
296 typedef struct search_pattern {
@@ -355,12 +378,15 @@ typedef struct log_job {
378 const char *pattern;
379 const char *prefix;
380
358 - SIMPLE_HASHTABLE hashtable;
381 + SIMPLE_HASHTABLE_KEY hashtable;
382
383 struct {
361 - HASHED_KEY *keys[MAX_OUTPUT_KEYS];
362 - size_t used;
363 - } sorted;
384 + const char *buffer;
385 + const char *trimmed;
386 + size_t trimmed_len;
387 + size_t size;
388 + HASHED_KEY key;
389 + } line;
390
391 struct {
392 SEARCH_PATTERN include;
@@ -447,6 +473,8 @@ const char *json_parser_error(LOG_JSON_STATE *js);
473 bool json_parse_document(LOG_JSON_STATE *js, const char *txt);
474 void json_test(void);
475
476 +size_t parse_surrogate(const char *s, char *d, size_t *remaining);
477 +
478 // ----------------------------------------------------------------------------
479 // logfmt parser
480
collectors/log2journal/tests.d/default.output new
+20
@@ -0,0 +1,20 @@
1 +MESSAGE=key1=value01 key2=value02 key3=value03 key4=value04
2 +PRIORITY=6
3 +SYSLOG_IDENTIFIER=log2journal
4 +
5 +MESSAGE=key1=value11 key2=value12 key3=value13 key4=
6 +PRIORITY=6
7 +SYSLOG_IDENTIFIER=log2journal
8 +
9 +MESSAGE=key1=value21 key2=value22 key3=value23 key4=value24
10 +PRIORITY=6
11 +SYSLOG_IDENTIFIER=log2journal
12 +
13 +MESSAGE=key1=value31 key2=value32 key3=value33 key4=
14 +PRIORITY=6
15 +SYSLOG_IDENTIFIER=log2journal
16 +
17 +MESSAGE=key1=value41 key2=value42 key3=value43 key4=value44
18 +PRIORITY=6
19 +SYSLOG_IDENTIFIER=log2journal
20 +
collectors/log2journal/tests.d/full.output
+12 -12
@@ -24,8 +24,8 @@ filename:
24 key: NGINX_LOG_FILENAME
25
26 filter:
27 - include: ".*"
28 - exclude: ".*HELLO.*WORLD.*"
27 + include: '.*'
28 + exclude: '.*HELLO.*WORLD.*'
29
30 rename:
31 - new_key: TEST1
@@ -39,32 +39,32 @@ inject:
39 - key: SYSLOG_IDENTIFIER2
40 value: nginx-log2
41 - key: PRIORITY
42 - value: "${NGINX_STATUS}"
42 + value: '${NGINX_STATUS}'
43 - key: NGINX_STATUS_FAMILY
44 - value: "${NGINX_STATUS}${NGINX_METHOD}"
44 + value: '${NGINX_STATUS}${NGINX_METHOD}'
45
46 rewrite:
47 - key: PRIORITY
48 - value: "${NGINX_STATUS}"
48 + value: '${NGINX_STATUS}'
49 inject: yes
50 stop: no
51 - key: PRIORITY
52 - match: "^[123]"
52 + match: '^[123]'
53 value: 6
54 - key: PRIORITY
55 - match: "^4"
55 + match: '^4'
56 value: 5
57 - key: PRIORITY
58 - match: "^5"
58 + match: '^5'
59 value: 3
60 - key: PRIORITY
61 - match: ".*"
61 + match: '.*'
62 value: 4
63 - key: NGINX_STATUS_FAMILY
64 - match: "^(?<first_digit>[1-5])"
65 - value: "${first_digit}xx"
64 + match: '^(?<first_digit>[1-5])'
65 + value: '${first_digit}xx'
66 - key: NGINX_STATUS_FAMILY
67 - match: ".*"
67 + match: '.*'
68 value: UNKNOWN
69
70 unmatched:
collectors/log2journal/tests.d/full.yaml
+6 -6
@@ -24,8 +24,8 @@ filename:
24 key: NGINX_LOG_FILENAME
25
26 filter:
27 - include: ".*"
28 - exclude: ".*HELLO.*WORLD.*"
27 + include: '.*'
28 + exclude: '.*HELLO.*WORLD.*'
29
30 rename:
31 - new_key: TEST1
@@ -35,13 +35,13 @@ rename:
35
36 inject:
37 - key: SYSLOG_IDENTIFIER
38 - value: "nginx-log"
38 + value: 'nginx-log'
39 - key: SYSLOG_IDENTIFIER2
40 - value: "nginx-log2"
40 + value: 'nginx-log2'
41 - key: PRIORITY
42 - value: "${NGINX_STATUS}"
42 + value: '${NGINX_STATUS}'
43 - key: NGINX_STATUS_FAMILY
44 - value: "${NGINX_STATUS}${NGINX_METHOD}"
44 + value: '${NGINX_STATUS}${NGINX_METHOD}'
45
46 rewrite:
47 - key: "PRIORITY"
collectors/log2journal/tests.sh
+12 -10
@@ -109,11 +109,12 @@ test_log2journal_config /dev/null "${tests}/full.output" --show-config \
109 # -----------------------------------------------------------------------------
110
111 test_log2journal() {
112 - local in="${1}"
113 - local out="${2}"
114 - shift 2
112 + local n="${1}"
113 + local in="${2}"
114 + local out="${3}"
115 + shift 3
116
116 - printf >&2 "running: "
117 + printf >&2 "running test No ${n}: "
118 printf >&2 "%q " "${log2journal_bin}" "${@}"
119 printf >&2 "\n"
120 echo >&2 "using as input : ${in}"
@@ -138,9 +139,10 @@ test_log2journal() {
139 echo >&2
140 echo >&2 "Testing parsing and output..."
141
141 -test_log2journal "${tests}/json.log" "${tests}/json.output" json
142 -test_log2journal "${tests}/json.log" "${tests}/json-include.output" json --include "OBJECT"
143 -test_log2journal "${tests}/json.log" "${tests}/json-exclude.output" json --exclude "ARRAY[^2]"
144 -test_log2journal "${tests}/nginx-json.log" "${tests}/nginx-json.output" -f "${script_dir}/log2journal.d/nginx-json.yaml"
145 -test_log2journal "${tests}/nginx-combined.log" "${tests}/nginx-combined.output" -f "${script_dir}/log2journal.d/nginx-combined.yaml"
146 -test_log2journal "${tests}/logfmt.log" "${tests}/logfmt.output" -f "${tests}/logfmt.yaml"
142 +test_log2journal 1 "${tests}/json.log" "${tests}/json.output" json
143 +test_log2journal 2 "${tests}/json.log" "${tests}/json-include.output" json --include "OBJECT"
144 +test_log2journal 3 "${tests}/json.log" "${tests}/json-exclude.output" json --exclude "ARRAY[^2]"
145 +test_log2journal 4 "${tests}/nginx-json.log" "${tests}/nginx-json.output" -f "${script_dir}/log2journal.d/nginx-json.yaml"
146 +test_log2journal 5 "${tests}/nginx-combined.log" "${tests}/nginx-combined.output" -f "${script_dir}/log2journal.d/nginx-combined.yaml"
147 +test_log2journal 6 "${tests}/logfmt.log" "${tests}/logfmt.output" -f "${tests}/logfmt.yaml"
148 +test_log2journal 7 "${tests}/logfmt.log" "${tests}/default.output" -f "${script_dir}/log2journal.d/default.yaml"
libnetdata/facets/facets.c
+47 -22
@@ -99,8 +99,37 @@ static inline bool is_valid_string_hash(const char *s) {
99 }
100
101 // ----------------------------------------------------------------------------
102 +// hashtable for FACET_VALUE
103 +
104 +// cleanup hashtable defines
105 +#undef SIMPLE_HASHTABLE_SORT_FUNCTION
106 +#undef SIMPLE_HASHTABLE_VALUE_TYPE
107 +#undef SIMPLE_HASHTABLE_NAME
108 +#undef NETDATA_SIMPLE_HASHTABLE_H
109 +
110 +struct facet_value;
111 +// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value
112 +#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_value
113 +#define SIMPLE_HASHTABLE_NAME _VALUE
114 #include "../simple_hashtable.h"
115
116 +// ----------------------------------------------------------------------------
117 +// hashtable for FACET_KEY
118 +
119 +// cleanup hashtable defines
120 +#undef SIMPLE_HASHTABLE_SORT_FUNCTION
121 +#undef SIMPLE_HASHTABLE_VALUE_TYPE
122 +#undef SIMPLE_HASHTABLE_NAME
123 +#undef NETDATA_SIMPLE_HASHTABLE_H
124 +
125 +struct facet_key;
126 +// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key
127 +#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_key
128 +#define SIMPLE_HASHTABLE_NAME _KEY
129 +#include "../simple_hashtable.h"
130 +
131 +// ----------------------------------------------------------------------------
132 +
133 typedef struct facet_value {
134 FACETS_HASH hash;
135 const char *name;
@@ -157,7 +186,7 @@ struct facet_key {
186 bool enabled;
187 uint32_t used;
188 FACET_VALUE *ll;
160 - SIMPLE_HASHTABLE ht;
189 + SIMPLE_HASHTABLE_VALUE ht;
190 } values;
191
192 struct {
@@ -216,7 +245,7 @@ struct facets {
245 struct {
246 size_t count;
247 FACET_KEY *ll;
219 - SIMPLE_HASHTABLE ht;
248 + SIMPLE_HASHTABLE_KEY ht;
249 } keys;
250
251 struct {
@@ -347,7 +376,7 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k);
376 static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) {
377 k->values.ll = NULL;
378 k->values.used = 0;
350 - simple_hashtable_init(&k->values.ht, FACETS_VALUES_HASHTABLE_ENTRIES);
379 + simple_hashtable_init_VALUE(&k->values.ht, FACETS_VALUES_HASHTABLE_ENTRIES);
380 }
381
382 static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) {
@@ -363,7 +392,7 @@ static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) {
392 k->values.used = 0;
393 k->values.enabled = false;
394
366 - simple_hashtable_free(&k->values.ht);
395 + simple_hashtable_destroy_VALUE(&k->values.ht);
396 }
397
398 static inline const char *facets_key_get_value(FACET_KEY *k) {
@@ -410,17 +439,17 @@ static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const
439 }
440
441 static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) {
413 - SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&k->values.ht, hash, true);
414 - return slot->data;
442 + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, true);
443 + return SIMPLE_HASHTABLE_SLOT_DATA(slot);
444 }
445
446 static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) {
418 - SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&k->values.ht, tv->hash, true);
447 + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, true);
448
420 - if(slot->data) {
449 + if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) {
450 // already exists
451
423 - FACET_VALUE *v = slot->data;
452 + FACET_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
453 FACET_VALUE_ADD_CONFLICT(k, v, tv);
454 return v;
455 }
@@ -428,9 +457,7 @@ static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VA
457 // we have to add it
458
459 FACET_VALUE *v = mallocz(sizeof(*v));
431 - slot->hash = tv->hash;
432 - slot->data = v;
433 - k->values.ht.used++;
460 + simple_hashtable_set_slot_VALUE(&k->values.ht, slot, tv->hash, v);
461
462 memcpy(v, tv, sizeof(*v));
463
@@ -584,7 +611,7 @@ static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) {
611 facets->keys.count = 0;
612 facets->keys_with_values.used = 0;
613
587 - simple_hashtable_init(&facets->keys.ht, FACETS_KEYS_HASHTABLE_ENTRIES);
614 + simple_hashtable_init_KEY(&facets->keys.ht, FACETS_KEYS_HASHTABLE_ENTRIES);
615 }
616
617 static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
@@ -603,12 +630,12 @@ static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
630 facets->keys.count = 0;
631 facets->keys_with_values.used = 0;
632
606 - simple_hashtable_free(&facets->keys.ht);
633 + simple_hashtable_destroy_KEY(&facets->keys.ht);
634 }
635
636 static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) {
610 - SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&facets->keys.ht, hash, true);
611 - return slot->data;
637 + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
638 + return SIMPLE_HASHTABLE_SLOT_DATA(slot);
639 }
640
641 bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) {
@@ -687,22 +714,20 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con
714 static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
715 facets->operations.keys.registered++;
716
690 - SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&facets->keys.ht, hash, true);
717 + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
718
692 - if(unlikely(!slot->data)) {
719 + if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
720 // we have to add it
721 FACET_KEY *k = FACETS_KEY_CREATE(facets, hash, name, name_length, options);
722
696 - slot->hash = hash;
697 - slot->data = k;
698 - facets->keys.ht.used++;
723 + simple_hashtable_set_slot_KEY(&facets->keys.ht, slot, hash, k);
724
725 return k;
726 }
727
728 // already in the index
729
705 - FACET_KEY *k = slot->data;
730 + FACET_KEY *k = SIMPLE_HASHTABLE_SLOT_DATA(slot);
731
732 facet_key_set_name(k, name, name_length);
733
libnetdata/simple_hashtable.h
+338 -34
@@ -3,90 +3,394 @@
3 #ifndef NETDATA_SIMPLE_HASHTABLE_H
4 #define NETDATA_SIMPLE_HASHTABLE_H
5
6 +#ifndef XXH_INLINE_ALL
7 +#define XXH_INLINE_ALL
8 +#endif
9 +#include "xxhash.h"
10 +
11 typedef uint64_t SIMPLE_HASHTABLE_HASH;
12 #define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32
13
9 -typedef struct simple_hashtable_slot {
14 +#ifndef SIMPLE_HASHTABLE_NAME
15 +#define SIMPLE_HASHTABLE_NAME
16 +#endif
17 +
18 +#ifndef SIMPLE_HASHTABLE_VALUE_TYPE
19 +#define SIMPLE_HASHTABLE_VALUE_TYPE void
20 +#endif
21 +
22 +// First layer of macro for token concatenation
23 +#define CONCAT_INTERNAL(a, b) a ## b
24 +// Second layer of macro, which ensures proper expansion
25 +#define CONCAT(a, b) CONCAT_INTERNAL(a, b)
26 +
27 +// define names for all structures and structures
28 +#define simple_hashtable_init_named CONCAT(simple_hashtable_init, SIMPLE_HASHTABLE_NAME)
29 +#define simple_hashtable_destroy_named CONCAT(simple_hashtable_destroy, SIMPLE_HASHTABLE_NAME)
30 +
31 +#define simple_hashtable_slot_named CONCAT(simple_hashtable_slot, SIMPLE_HASHTABLE_NAME)
32 +#define SIMPLE_HASHTABLE_SLOT_NAMED CONCAT(SIMPLE_HASHTABLE_SLOT, SIMPLE_HASHTABLE_NAME)
33 +#define simple_hashtable_named CONCAT(simple_hashtable, SIMPLE_HASHTABLE_NAME)
34 +#define SIMPLE_HASHTABLE_NAMED CONCAT(SIMPLE_HASHTABLE, SIMPLE_HASHTABLE_NAME)
35 +#define simple_hashtable_resize_named CONCAT(simple_hashtable_resize, SIMPLE_HASHTABLE_NAME)
36 +#define simple_hashtable_get_slot_named CONCAT(simple_hashtable_get_slot, SIMPLE_HASHTABLE_NAME)
37 +#define simple_hashtable_del_slot_named CONCAT(simple_hashtable_del_slot, SIMPLE_HASHTABLE_NAME)
38 +#define simple_hashtable_set_slot_named CONCAT(simple_hashtable_set_slot, SIMPLE_HASHTABLE_NAME)
39 +#define simple_hashtable_first_read_only_named CONCAT(simple_hashtable_first_read_only, SIMPLE_HASHTABLE_NAME)
40 +#define simple_hashtable_next_read_only_named CONCAT(simple_hashtable_next_read_only, SIMPLE_HASHTABLE_NAME)
41 +
42 +#define simple_hashtable_sorted_binary_search_named CONCAT(simple_hashtable_sorted_binary_search, SIMPLE_HASHTABLE_NAME)
43 +#define simple_hashtable_add_value_sorted_named CONCAT(simple_hashtable_add_value_sorted, SIMPLE_HASHTABLE_NAME)
44 +#define simple_hashtable_del_value_sorted_named CONCAT(simple_hashtable_del_value_sorted, SIMPLE_HASHTABLE_NAME)
45 +#define simple_hashtable_replace_value_sorted_named CONCAT(simple_hashtable_replace_value_sorted, SIMPLE_HASHTABLE_NAME)
46 +#define simple_hashtable_sorted_array_first_read_only_named CONCAT(simple_hashtable_sorted_array_first_read_only, SIMPLE_HASHTABLE_NAME)
47 +#define simple_hashtable_sorted_array_next_read_only_named CONCAT(simple_hashtable_sorted_array_next_read_only, SIMPLE_HASHTABLE_NAME)
48 +
49 +typedef struct simple_hashtable_slot_named {
50 SIMPLE_HASHTABLE_HASH hash;
11 - void *data;
12 -} SIMPLE_HASHTABLE_SLOT;
51 + SIMPLE_HASHTABLE_VALUE_TYPE *data;
52 +} SIMPLE_HASHTABLE_SLOT_NAMED;
53
14 -typedef struct simple_hashtable {
54 +typedef struct simple_hashtable_named {
55 size_t resizes;
56 size_t searches;
57 size_t collisions;
58 + size_t deletions;
59 + size_t deleted;
60 size_t used;
61 size_t size;
20 - SIMPLE_HASHTABLE_SLOT *hashtable;
21 -} SIMPLE_HASHTABLE;
62 + SIMPLE_HASHTABLE_SLOT_NAMED *hashtable;
63 +
64 +#ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
65 + struct {
66 + size_t used;
67 + size_t size;
68 + SIMPLE_HASHTABLE_VALUE_TYPE **array;
69 + } sorted;
70 +#endif
71 +} SIMPLE_HASHTABLE_NAMED;
72 +
73 +#ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
74 +static inline size_t simple_hashtable_sorted_binary_search_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *value) {
75 + size_t left = 0, right = ht->sorted.used;
76 +
77 + while (left < right) {
78 + size_t mid = left + (right - left) / 2;
79 + if (SIMPLE_HASHTABLE_SORT_FUNCTION(ht->sorted.array[mid], value) < 0)
80 + left = mid + 1;
81 + else
82 + right = mid;
83 + }
84 +
85 + return left;
86 +}
87 +
88 +static inline void simple_hashtable_add_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *value) {
89 + size_t index = simple_hashtable_sorted_binary_search_named(ht, value);
90 +
91 + // Ensure there's enough space in the sorted array
92 + if (ht->sorted.used >= ht->sorted.size) {
93 + size_t size = ht->sorted.size ? ht->sorted.size * 2 : 64;
94 + SIMPLE_HASHTABLE_VALUE_TYPE **array = mallocz(size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
95 + if(ht->sorted.array) {
96 + memcpy(array, ht->sorted.array, ht->sorted.size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
97 + freez(ht->sorted.array);
98 + }
99 + ht->sorted.array = array;
100 + ht->sorted.size = size;
101 + }
102 +
103 + // Use memmove to shift elements and create space for the new element
104 + memmove(&ht->sorted.array[index + 1], &ht->sorted.array[index], (ht->sorted.used - index) * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
105 +
106 + ht->sorted.array[index] = value;
107 + ht->sorted.used++;
108 +}
109 +
110 +static inline void simple_hashtable_del_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *value) {
111 + size_t index = simple_hashtable_sorted_binary_search_named(ht, value);
112 +
113 + // Check if the value exists at the found index
114 + assert(index < ht->sorted.used && ht->sorted.array[index] == value);
115 +
116 + // Use memmove to shift elements and close the gap
117 + memmove(&ht->sorted.array[index], &ht->sorted.array[index + 1], (ht->sorted.used - index - 1) * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
118 + ht->sorted.used--;
119 +}
120 +
121 +static inline void simple_hashtable_replace_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *old_value, SIMPLE_HASHTABLE_VALUE_TYPE *new_value) {
122 + if(new_value == old_value)
123 + return;
124 +
125 + size_t old_value_index = simple_hashtable_sorted_binary_search_named(ht, old_value);
126 + assert(old_value_index < ht->sorted.used && ht->sorted.array[old_value_index] == old_value);
127 +
128 + int r = SIMPLE_HASHTABLE_SORT_FUNCTION(old_value, new_value);
129 + if(r == 0) {
130 + // Same value, so use the same index
131 + ht->sorted.array[old_value_index] = new_value;
132 + return;
133 + }
134 +
135 + size_t new_value_index = simple_hashtable_sorted_binary_search_named(ht, new_value);
136 + if(old_value_index == new_value_index) {
137 + // Not the same value, but still at the same index
138 + ht->sorted.array[old_value_index] = new_value;
139 + return;
140 + }
141 + else if (old_value_index < new_value_index) {
142 + // The old value is before the new value
143 + size_t shift_start = old_value_index + 1;
144 + size_t shift_end = new_value_index - 1;
145 + size_t shift_size = shift_end - old_value_index;
146 +
147 + memmove(&ht->sorted.array[old_value_index], &ht->sorted.array[shift_start], shift_size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
148 + ht->sorted.array[shift_end] = new_value;
149 + }
150 + else {
151 + // The old value is after the new value
152 + size_t shift_start = new_value_index;
153 + size_t shift_end = old_value_index;
154 + size_t shift_size = shift_end - new_value_index;
155 +
156 + memmove(&ht->sorted.array[new_value_index + 1], &ht->sorted.array[shift_start], shift_size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
157 + ht->sorted.array[new_value_index] = new_value;
158 + }
159 +}
160 +
161 +static inline SIMPLE_HASHTABLE_VALUE_TYPE **simple_hashtable_sorted_array_first_read_only_named(SIMPLE_HASHTABLE_NAMED *ht) {
162 + if (ht->sorted.used > 0) {
163 + return &ht->sorted.array[0];
164 + }
165 + return NULL;
166 +}
167 +
168 +static inline SIMPLE_HASHTABLE_VALUE_TYPE **simple_hashtable_sorted_array_next_read_only_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE **last) {
169 + if (!last) return NULL;
170 +
171 + // Calculate the current position in the sorted array
172 + size_t currentIndex = last - ht->sorted.array;
173 +
174 + // Proceed to the next element if it exists
175 + if (currentIndex + 1 < ht->sorted.used) {
176 + return &ht->sorted.array[currentIndex + 1];
177 + }
178 +
179 + // If no more elements, return NULL
180 + return NULL;
181 +}
182 +
183 +#define SIMPLE_HASHTABLE_SORTED_FOREACH_READ_ONLY(ht, var, type, name) \
184 + for (type **(var) = simple_hashtable_sorted_array_first_read_only ## name(ht); \
185 + var; \
186 + (var) = simple_hashtable_sorted_array_next_read_only ## name(ht, var))
187
23 -static void simple_hashtable_init(SIMPLE_HASHTABLE *ht, size_t size) {
24 - ht->resizes = 0;
25 - ht->used = 0;
188 +#define SIMPLE_HASHTABLE_SORTED_FOREACH_READ_ONLY_VALUE(var) (*(var))
189 +
190 +#else
191 +static inline void simple_hashtable_add_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *value __maybe_unused) { ; }
192 +static inline void simple_hashtable_del_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *value __maybe_unused) { ; }
193 +static inline void simple_hashtable_replace_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *old_value __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *new_value __maybe_unused) { ; }
194 +#endif
195 +
196 +static void simple_hashtable_init_named(SIMPLE_HASHTABLE_NAMED *ht, size_t size) {
197 + memset(ht, 0, sizeof(*ht));
198 ht->size = size;
199 ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
200 }
201
30 -static void simple_hashtable_free(SIMPLE_HASHTABLE *ht) {
202 +static void simple_hashtable_destroy_named(SIMPLE_HASHTABLE_NAMED *ht) {
203 +#ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
204 + freez(ht->sorted.array);
205 +#endif
206 +
207 freez(ht->hashtable);
32 - ht->hashtable = NULL;
33 - ht->size = 0;
34 - ht->used = 0;
35 - ht->resizes = 0;
208 + memset(ht, 0, sizeof(*ht));
209 }
210
38 -static inline void simple_hashtable_resize(SIMPLE_HASHTABLE *ht);
211 +static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht);
212
40 -static inline SIMPLE_HASHTABLE_SLOT *simple_hashtable_get_slot(SIMPLE_HASHTABLE *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) {
41 - // IMPORTANT:
42 - // If the hashtable supported deletions, we would need to have a special slot.data value
43 - // to mark deleted values and assume they are occupied during lookup, but empty during insert.
44 - // But for our case, we don't need it, since we never delete items from the hashtable.
213 +#define SHTS_DATA_UNSET ((void *)NULL)
214 +#define SHTS_DATA_DELETED ((void *)0x01)
215 +#define SHTS_DATA_USERNULL ((void *)0x02)
216 +#define SHTS_IS_UNSET(sl) ((sl)->data == SHTS_DATA_UNSET)
217 +#define SHTS_IS_DELETED(sl) ((sl)->data == SHTS_DATA_DELETED)
218 +#define SHTS_IS_USERNULL(sl) ((sl)->data == SHTS_DATA_USERNULL)
219 +#define SIMPLE_HASHTABLE_SLOT_DATA(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl) || SHTS_IS_USERNULL(sl)) ? NULL : (sl)->data)
220 +#define SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl)) ? NULL : (sl)->data)
221
222 +// IMPORTANT
223 +// The pointer returned by this call is valid up to the next call of this function (or the resize one)
224 +// If you need to cache something, cache the hash, not the slot pointer.
225 +static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) {
226 ht->searches++;
227
48 - size_t slot = hash % ht->size;
49 - if(likely(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash))
50 - return &ht->hashtable[slot];
228 + size_t slot;
229 + SIMPLE_HASHTABLE_SLOT_NAMED *sl;
230 + SIMPLE_HASHTABLE_SLOT_NAMED *deleted;
231 +
232 + slot = hash % ht->size;
233 + sl = &ht->hashtable[slot];
234 + deleted = SHTS_IS_DELETED(sl) ? sl : NULL;
235 + if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash))
236 + return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
237
238 ht->collisions++;
239
54 - if(unlikely(resize && ht->size <= (ht->used << 4))) {
55 - simple_hashtable_resize(ht);
240 + if(unlikely(resize && (ht->size <= (ht->used << 1) || ht->used >= ht->size))) {
241 + simple_hashtable_resize_named(ht);
242
243 slot = hash % ht->size;
58 - if(likely(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash))
59 - return &ht->hashtable[slot];
244 + sl = &ht->hashtable[slot];
245 + deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
246 + if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash))
247 + return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
248
249 ht->collisions++;
250 }
251
252 slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size;
253 + sl = &ht->hashtable[slot];
254 + deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
255 +
256 // Linear probing until we find it
66 - while (ht->hashtable[slot].data && ht->hashtable[slot].hash != hash) {
257 + while (SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) && sl->hash != hash) {
258 slot = (slot + 1) % ht->size; // Wrap around if necessary
259 + sl = &ht->hashtable[slot];
260 + deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
261 ht->collisions++;
262 }
263
71 - return &ht->hashtable[slot];
264 + return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
265 +}
266 +
267 +static inline bool simple_hashtable_del_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl) {
268 + if(SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl))
269 + return false;
270 +
271 + ht->deletions++;
272 + ht->deleted++;
273 +
274 + simple_hashtable_del_value_sorted_named(ht, SIMPLE_HASHTABLE_SLOT_DATA(sl));
275 +
276 + sl->data = SHTS_DATA_DELETED;
277 + return true;
278 +}
279 +
280 +static inline void simple_hashtable_set_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl, SIMPLE_HASHTABLE_HASH hash, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
281 + if(data == NULL)
282 + data = SHTS_DATA_USERNULL;
283 +
284 + if(unlikely(data == SHTS_DATA_UNSET || data == SHTS_DATA_DELETED)) {
285 + simple_hashtable_del_slot_named(ht, sl);
286 + return;
287 + }
288 +
289 + if(likely(SHTS_IS_UNSET(sl))) {
290 + simple_hashtable_add_value_sorted_named(ht, data);
291 + ht->used++;
292 + }
293 +
294 + else if(unlikely(SHTS_IS_DELETED(sl))) {
295 + ht->deleted--;
296 + }
297 +
298 + else
299 + simple_hashtable_replace_value_sorted_named(ht, SIMPLE_HASHTABLE_SLOT_DATA(sl), data);
300 +
301 + sl->hash = hash;
302 + sl->data = data;
303 }
304
74 -static inline void simple_hashtable_resize(SIMPLE_HASHTABLE *ht) {
75 - SIMPLE_HASHTABLE_SLOT *old = ht->hashtable;
305 +// IMPORTANT
306 +// this call invalidates all SIMPLE_HASHTABLE_SLOT_NAMED pointers
307 +static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht) {
308 + SIMPLE_HASHTABLE_SLOT_NAMED *old = ht->hashtable;
309 size_t old_size = ht->size;
310
311 ht->resizes++;
79 - ht->size = (ht->size << 3) - 1;
312 + ht->size = (ht->size << 1) - ((ht->size > 16) ? 1 : 0);
313 ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
314 + ht->used = ht->deleted = 0;
315 for(size_t i = 0 ; i < old_size ; i++) {
82 - if(!old[i].data)
316 + if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(&old[i]))
317 continue;
318
85 - SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(ht, old[i].hash, false);
319 + SIMPLE_HASHTABLE_SLOT_NAMED *slot = simple_hashtable_get_slot_named(ht, old[i].hash, false);
320 *slot = old[i];
321 + ht->used++;
322 }
323
324 freez(old);
325 }
326
327 +// ----------------------------------------------------------------------------
328 +// hashtable traversal, in read-only mode
329 +// the hashtable should not be modified while the traversal is taking place
330 +
331 +static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_first_read_only_named(SIMPLE_HASHTABLE_NAMED *ht) {
332 + for(size_t i = 0; i < ht->used ;i++) {
333 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i];
334 + if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl))
335 + return sl;
336 + }
337 +
338 + return NULL;
339 +}
340 +
341 +static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *last) {
342 + if (!last) return NULL;
343 +
344 + // Calculate the current position in the array
345 + size_t currentIndex = last - ht->hashtable;
346 +
347 + // Iterate over the hashtable starting from the next element
348 + for (size_t i = currentIndex + 1; i < ht->size; i++) {
349 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i];
350 + if (!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl)) {
351 + return sl;
352 + }
353 + }
354 +
355 + // If no more data slots are found, return NULL
356 + return NULL;
357 +}
358 +
359 +#define SIMPLE_HASHTABLE_FOREACH_READ_ONLY(ht, var, name) \
360 + for(struct simple_hashtable_slot ## name *(var) = simple_hashtable_first_read_only ## name(ht); \
361 + var; \
362 + (var) = simple_hashtable_next_read_only ## name(ht, var))
363 +
364 +#define SIMPLE_HASHTABLE_FOREACH_READ_ONLY_VALUE(var) SIMPLE_HASHTABLE_SLOT_DATA(var)
365 +
366 +// ----------------------------------------------------------------------------
367 +// high level implementation
368 +
369 +#ifdef SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
370 +
371 +#define simple_hashtable_set_named CONCAT(simple_hashtable_set, SIMPLE_HASHTABLE_NAME)
372 +#define simple_hashtable_get_named CONCAT(simple_hashtable_get, SIMPLE_HASHTABLE_NAME)
373 +#define simple_hashtable_del_named CONCAT(simple_hashtable_del, SIMPLE_HASHTABLE_NAME)
374 +
375 +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
376 + XXH64_hash_t hash = XXH3_64bits(key, key_len);
377 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
378 + simple_hashtable_set_slot_named(ht, sl, hash, data);
379 + return SIMPLE_HASHTABLE_SLOT_DATA(sl);
380 +}
381 +
382 +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
383 + XXH64_hash_t hash = XXH3_64bits(key, key_len);
384 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
385 + return SIMPLE_HASHTABLE_SLOT_DATA(sl);
386 +}
387 +
388 +static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
389 + XXH64_hash_t hash = XXH3_64bits(key, key_len);
390 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
391 + return simple_hashtable_del_slot_named(ht, sl);
392 +}
393 +
394 +#endif // SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
395 +
396 #endif //NETDATA_SIMPLE_HASHTABLE_H