log2journal now uses libnetdata (#18919)
Costa Tsaousis committed
Nov 1, 2024 at 00:16 UTC
e8d8e24ed1b0d132c271f85662a313502930c4a2
14 files changed
+283
-318
CMakeLists.txt
+4
-1
@@ -2557,13 +2557,16 @@ if(PCRE2_FOUND)
2557
src/collectors/log2journal/log2journal-replace.c
2558
src/collectors/log2journal/log2journal-rename.c
2559
src/collectors/log2journal/log2journal-rewrite.c
2560
+ src/collectors/log2journal/log2journal-txt.h
2561
+ src/collectors/log2journal/log2journal-hashed-key.h
2562
)
2563
2564
add_executable(log2journal ${LOG2JOURNAL_FILES})
2565
target_include_directories(log2journal BEFORE PUBLIC ${CONFIG_H_DIR} ${CMAKE_SOURCE_DIR}/src ${PCRE2_INCLUDE_DIRS})
2566
target_compile_options(log2journal PUBLIC ${PCRE2_CFLAGS_OTHER})
2565
- target_link_libraries(log2journal PUBLIC "${PCRE2_LDFLAGS}")
2567
2568
+ target_link_libraries(log2journal PUBLIC libnetdata)
2569
+ target_link_libraries(log2journal PUBLIC "${PCRE2_LDFLAGS}")
2570
netdata_add_libyaml_to_target(log2journal)
2571
2572
install(TARGETS log2journal
src/collectors/log2journal/log2journal-hashed-key.h
new
+80
@@ -0,0 +1,80 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_LOG2JOURNAL_HASHED_KEY_H
4
+#define NETDATA_LOG2JOURNAL_HASHED_KEY_H
5
+
6
+#include "log2journal.h"
7
+
8
+typedef enum __attribute__((__packed__)) {
9
+ HK_NONE = 0,
10
+
11
+ // permanent flags - they are set once to optimize various decisions and lookups
12
+
13
+ HK_HASHTABLE_ALLOCATED = (1 << 0), // this is the key object allocated in the hashtable
14
+ // objects that do not have this, have a pointer to a key in the hashtable
15
+ // objects that have this, value is allocated
16
+
17
+ HK_FILTERED = (1 << 1), // we checked once if this key in filtered
18
+ HK_FILTERED_INCLUDED = (1 << 2), // the result of the filtering was to include it in the output
19
+
20
+ HK_COLLISION_CHECKED = (1 << 3), // we checked once for collision check of this key
21
+
22
+ HK_RENAMES_CHECKED = (1 << 4), // we checked once if there are renames on this key
23
+ HK_HAS_RENAMES = (1 << 5), // and we found there is a rename rule related to it
24
+
25
+ // ephemeral flags - they are unset at the end of each log line
26
+
27
+ HK_VALUE_FROM_LOG = (1 << 14), // the value of this key has been read from the log (or from injection, duplication)
28
+ HK_VALUE_REWRITTEN = (1 << 15), // the value of this key has been rewritten due to one of our rewrite rules
29
+
30
+} HASHED_KEY_FLAGS;
31
+
32
+typedef struct hashed_key {
33
+ const char *key;
34
+ uint32_t len;
35
+ HASHED_KEY_FLAGS flags;
36
+ XXH64_hash_t hash;
37
+ union {
38
+ struct hashed_key *hashtable_ptr; // HK_HASHTABLE_ALLOCATED is not set
39
+ TXT_L2J value; // HK_HASHTABLE_ALLOCATED is set
40
+ };
41
+} HASHED_KEY;
42
+
43
+static inline void hashed_key_cleanup(HASHED_KEY *k) {
44
+ if(k->flags & HK_HASHTABLE_ALLOCATED)
45
+ txt_l2j_cleanup(&k->value);
46
+ else
47
+ k->hashtable_ptr = NULL;
48
+
49
+ freez((void *)k->key);
50
+ k->key = NULL;
51
+ k->len = 0;
52
+ k->hash = 0;
53
+ k->flags = HK_NONE;
54
+}
55
+
56
+static inline void hashed_key_set(HASHED_KEY *k, const char *name, int32_t len) {
57
+ hashed_key_cleanup(k);
58
+
59
+ if(len == -1) {
60
+ k->key = strdupz(name);
61
+ k->len = strlen(k->key);
62
+ }
63
+ else {
64
+ k->key = strndupz(name, len);
65
+ k->len = len;
66
+ }
67
+
68
+ k->hash = XXH3_64bits(k->key, k->len);
69
+ k->flags = HK_NONE;
70
+}
71
+
72
+static inline bool hashed_keys_match(HASHED_KEY *k1, HASHED_KEY *k2) {
73
+ return ((k1 == k2) || (k1->hash == k2->hash && strcmp(k1->key, k2->key) == 0));
74
+}
75
+
76
+static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2) {
77
+ return strcmp(k1->key, k2->key);
78
+}
79
+
80
+#endif //NETDATA_LOG2JOURNAL_HASHED_KEY_H
src/collectors/log2journal/log2journal-help.c
+1
-1
@@ -10,7 +10,7 @@ static void config_dir_print_available(void) {
10
dir = opendir(path);
11
12
if (dir == NULL) {
13
- log2stderr(" >>> Cannot open directory:\n %s", path);
13
+ l2j_log(" >>> Cannot open directory:\n %s", path);
14
return;
15
}
16
src/collectors/log2journal/log2journal-inject.c
+6
-5
@@ -9,12 +9,13 @@ void injection_cleanup(INJECTION *inj) {
9
10
static inline bool log_job_injection_replace(INJECTION *inj, const char *key, size_t key_len, const char *value, size_t value_len) {
11
if(key_len > JOURNAL_MAX_KEY_LEN)
12
- log2stderr("WARNING: injection key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
12
+ l2j_log("WARNING: injection key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
13
14
if(value_len > JOURNAL_MAX_VALUE_LEN)
15
- log2stderr("WARNING: injection value of key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
15
+ l2j_log(
16
+ "WARNING: injection value of key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
17
17
- hashed_key_len_set(&inj->key, key, key_len);
18
+ hashed_key_set(&inj->key, key, key_len);
19
char *v = strndupz(value, value_len);
20
bool ret = replace_pattern_set(&inj->value, v);
21
freez(v);
@@ -25,13 +26,13 @@ static inline bool log_job_injection_replace(INJECTION *inj, const char *key, si
26
bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {
27
if (unmatched) {
28
if (jb->unmatched.injections.used >= MAX_INJECTIONS) {
28
- log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);
29
+ l2j_log("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);
30
return false;
31
}
32
}
33
else {
34
if (jb->injections.used >= MAX_INJECTIONS) {
34
- log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);
35
+ l2j_log("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);
36
return false;
37
}
38
}
src/collectors/log2journal/log2journal-params.c
+29
-23
@@ -7,7 +7,7 @@
7
void log_job_init(LOG_JOB *jb) {
8
memset(jb, 0, sizeof(*jb));
9
simple_hashtable_init_KEY(&jb->hashtable, 32);
10
- hashed_key_set(&jb->line.key, "LINE");
10
+ hashed_key_set(&jb->line.key, "LINE", -1);
11
}
12
13
static void simple_hashtable_cleanup_allocated_keys(SIMPLE_HASHTABLE_KEY *ht) {
@@ -53,8 +53,8 @@ void log_job_cleanup(LOG_JOB *jb) {
53
hashed_key_cleanup(&jb->filename.key);
54
hashed_key_cleanup(&jb->unmatched.key);
55
56
- txt_cleanup(&jb->rewrites.tmp);
57
- txt_cleanup(&jb->filename.current);
56
+ txt_l2j_cleanup(&jb->rewrites.tmp);
57
+ txt_l2j_cleanup(&jb->filename.current);
58
59
simple_hashtable_cleanup_allocated_keys(&jb->hashtable);
60
simple_hashtable_destroy_KEY(&jb->hashtable);
@@ -67,18 +67,18 @@ void log_job_cleanup(LOG_JOB *jb) {
67
68
bool log_job_filename_key_set(LOG_JOB *jb, const char *key, size_t key_len) {
69
if(!key || !*key) {
70
- log2stderr("filename key cannot be empty.");
70
+ l2j_log("filename key cannot be empty.");
71
return false;
72
}
73
74
- hashed_key_len_set(&jb->filename.key, key, key_len);
74
+ hashed_key_set(&jb->filename.key, key, key_len);
75
76
return true;
77
}
78
79
bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len) {
80
if(!prefix || !*prefix) {
81
- log2stderr("filename key cannot be empty.");
81
+ l2j_log("filename key cannot be empty.");
82
return false;
83
}
84
@@ -92,7 +92,7 @@ bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len)
92
93
bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
94
if(!pattern || !*pattern) {
95
- log2stderr("filename key cannot be empty.");
95
+ l2j_log("filename key cannot be empty.");
96
return false;
97
}
98
@@ -106,12 +106,12 @@ bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
106
107
bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
108
if(jb->filter.include.re) {
109
- log2stderr("FILTER INCLUDE: there is already an include filter set");
109
+ l2j_log("FILTER INCLUDE: there is already an include filter set");
110
return false;
111
}
112
113
if(!search_pattern_set(&jb->filter.include, pattern, pattern_len)) {
114
- log2stderr("FILTER INCLUDE: failed: %s", jb->filter.include.error.txt);
114
+ l2j_log("FILTER INCLUDE: failed: %s", jb->filter.include.error.txt);
115
return false;
116
}
117
@@ -120,12 +120,12 @@ bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t patter
120
121
bool log_job_exclude_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
122
if(jb->filter.exclude.re) {
123
- log2stderr("FILTER INCLUDE: there is already an exclude filter set");
123
+ l2j_log("FILTER INCLUDE: there is already an exclude filter set");
124
return false;
125
}
126
127
if(!search_pattern_set(&jb->filter.exclude, pattern, pattern_len)) {
128
- log2stderr("FILTER EXCLUDE: failed: %s", jb->filter.exclude.error.txt);
128
+ l2j_log("FILTER EXCLUDE: failed: %s", jb->filter.exclude.error.txt);
129
return false;
130
}
131
@@ -138,7 +138,7 @@ static bool parse_rename(LOG_JOB *jb, const char *param) {
138
// Search for '=' in param
139
const char *equal_sign = strchr(param, '=');
140
if (!equal_sign || equal_sign == param) {
141
- log2stderr("Error: Invalid rename format, '=' not found in %s", param);
141
+ l2j_log("Error: Invalid rename format, '=' not found in %s", param);
142
return false;
143
}
144
@@ -216,7 +216,7 @@ RW_FLAGS parse_rewrite_flags(const char *options) {
216
}
217
218
if(!found)
219
- log2stderr("Warning: rewrite options '%s' is not understood.", token);
219
+ l2j_log("Warning: rewrite options '%s' is not understood.", token);
220
221
// Get the next token
222
token = strtok(NULL, ",");
@@ -232,33 +232,33 @@ static bool parse_rewrite(LOG_JOB *jb, const char *param) {
232
// Search for '=' in param
233
const char *equal_sign = strchr(param, '=');
234
if (!equal_sign || equal_sign == param) {
235
- log2stderr("Error: Invalid rewrite format, '=' not found in %s", param);
235
+ l2j_log("Error: Invalid rewrite format, '=' not found in %s", param);
236
return false;
237
}
238
239
// Get the next character as the separator
240
char separator = *(equal_sign + 1);
241
if (!separator || !is_symbol(separator)) {
242
- log2stderr("Error: rewrite separator not found after '=', or is not one of /\\|-# in: %s", param);
242
+ l2j_log("Error: rewrite separator not found after '=', or is not one of /\\|-# in: %s", param);
243
return false;
244
}
245
246
// Find the next occurrence of the separator
247
const char *second_separator = strchr(equal_sign + 2, separator);
248
if (!second_separator) {
249
- log2stderr("Error: rewrite second separator not found in: %s", param);
249
+ l2j_log("Error: rewrite second separator not found in: %s", param);
250
return false;
251
}
252
253
// Check if the search pattern is empty
254
if (equal_sign + 1 == second_separator) {
255
- log2stderr("Error: rewrite search pattern is empty in: %s", param);
255
+ l2j_log("Error: rewrite search pattern is empty in: %s", param);
256
return false;
257
}
258
259
// Check if the replacement pattern is empty
260
if (*(second_separator + 1) == '\0') {
261
- log2stderr("Error: rewrite replacement pattern is empty in: %s", param);
261
+ l2j_log("Error: rewrite replacement pattern is empty in: %s", param);
262
return false;
263
}
264
@@ -287,7 +287,7 @@ static bool parse_rewrite(LOG_JOB *jb, const char *param) {
287
static bool parse_inject(LOG_JOB *jb, const char *value, bool unmatched) {
288
const char *equal = strchr(value, '=');
289
if (!equal) {
290
- log2stderr("Error: injection '%s' does not have an equal sign.", value);
290
+ l2j_log("Error: injection '%s' does not have an equal sign.", value);
291
return false;
292
}
293
@@ -336,7 +336,10 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
336
log_job_pattern_set(jb, arg, strlen(arg));
337
continue;
338
} else {
339
- log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
339
+ l2j_log(
340
+ "Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'",
341
+ jb->pattern,
342
+ arg);
343
return false;
344
}
345
}
@@ -361,7 +364,7 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
364
}
365
#endif
366
else if (strcmp(param, "--unmatched-key") == 0)
364
- hashed_key_set(&jb->unmatched.key, value);
367
+ hashed_key_set(&jb->unmatched.key, value, -1);
368
else if (strcmp(param, "--inject") == 0) {
369
if (!parse_inject(jb, value, false))
370
return false;
@@ -392,7 +395,10 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
395
log_job_pattern_set(jb, arg, strlen(arg));
396
continue;
397
} else {
395
- log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
398
+ l2j_log(
399
+ "Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'",
400
+ jb->pattern,
401
+ arg);
402
return false;
403
}
404
}
@@ -401,7 +407,7 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
407
408
// Check if a pattern is set and exactly one pattern is specified
409
if (!jb->pattern) {
404
- log2stderr("Warning: pattern not specified. Try the default config with: -c default");
410
+ l2j_log("Warning: pattern not specified. Try the default config with: -c default");
411
log_job_command_line_help(argv[0]);
412
return false;
413
}
src/collectors/log2journal/log2journal-pattern.c
+2
-2
@@ -18,13 +18,13 @@ void search_pattern_cleanup(SEARCH_PATTERN *sp) {
18
sp->match_data = NULL;
19
}
20
21
- txt_cleanup(&sp->error);
21
+ txt_l2j_cleanup(&sp->error);
22
}
23
24
static void pcre2_error_message(SEARCH_PATTERN *sp, int rc, int pos) {
25
char msg[1024];
26
pcre2_get_error_in_buffer(msg, sizeof(msg), rc, pos);
27
- txt_replace(&sp->error, msg, strlen(msg));
27
+ txt_l2j_set(&sp->error, msg, strlen(msg));
28
}
29
30
static inline bool compile_pcre2(SEARCH_PATTERN *sp) {
src/collectors/log2journal/log2journal-rename.c
+3
-3
@@ -9,13 +9,13 @@ void rename_cleanup(RENAME *rn) {
9
10
bool log_job_rename_add(LOG_JOB *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len) {
11
if(jb->renames.used >= MAX_RENAMES) {
12
- log2stderr("Error: too many renames. You can rename up to %d fields.", MAX_RENAMES);
12
+ l2j_log("Error: too many renames. You can rename up to %d fields.", MAX_RENAMES);
13
return false;
14
}
15
16
RENAME *rn = &jb->renames.array[jb->renames.used++];
17
- hashed_key_len_set(&rn->new_key, new_key, new_key_len);
18
- hashed_key_len_set(&rn->old_key, old_key, old_key_len);
17
+ hashed_key_set(&rn->new_key, new_key, new_key_len);
18
+ hashed_key_set(&rn->old_key, old_key, old_key_len);
19
20
return true;
21
}
src/collectors/log2journal/log2journal-replace.c
+6
-6
@@ -26,7 +26,7 @@ static REPLACE_NODE *replace_pattern_add_node(REPLACE_NODE **head, bool is_varia
26
if (!new_node)
27
return NULL;
28
29
- hashed_key_set(&new_node->name, text);
29
+ hashed_key_set(&new_node->name, text, -1);
30
new_node->is_variable = is_variable;
31
new_node->next = NULL;
32
@@ -57,21 +57,21 @@ bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern) {
57
// Start of a variable
58
const char *end = strchr(current, '}');
59
if (!end) {
60
- log2stderr("Error: Missing closing brace in replacement pattern: %s", rp->pattern);
60
+ l2j_log("Error: Missing closing brace in replacement pattern: %s", rp->pattern);
61
return false;
62
}
63
64
size_t name_length = end - current - 2; // Length of the variable name
65
char *variable_name = strndupz(current + 2, name_length);
66
if (!variable_name) {
67
- log2stderr("Error: Memory allocation failed for variable name.");
67
+ l2j_log("Error: Memory allocation failed for variable name.");
68
return false;
69
}
70
71
REPLACE_NODE *node = replace_pattern_add_node(&(rp->nodes), true, variable_name);
72
if (!node) {
73
freez(variable_name);
74
- log2stderr("Error: Failed to add replacement node for variable.");
74
+ l2j_log("Error: Failed to add replacement node for variable.");
75
return false;
76
}
77
freez(variable_name);
@@ -88,14 +88,14 @@ bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern) {
88
size_t text_length = current - start;
89
char *text = strndupz(start, text_length);
90
if (!text) {
91
- log2stderr("Error: Memory allocation failed for literal text.");
91
+ l2j_log("Error: Memory allocation failed for literal text.");
92
return false;
93
}
94
95
REPLACE_NODE *node = replace_pattern_add_node(&(rp->nodes), false, text);
96
if (!node) {
97
freez(text);
98
- log2stderr("Error: Failed to add replacement node for text.");
98
+ l2j_log("Error: Failed to add replacement node for text.");
99
return false;
100
}
101
freez(text);
src/collectors/log2journal/log2journal-rewrite.c
+3
-3
@@ -17,19 +17,19 @@ void rewrite_cleanup(REWRITE *rw) {
17
18
bool log_job_rewrite_add(LOG_JOB *jb, const char *key, RW_FLAGS flags, const char *search_pattern, const char *replace_pattern) {
19
if(jb->rewrites.used >= MAX_REWRITES) {
20
- log2stderr("Error: too many rewrites. You can add up to %d rewrite rules.", MAX_REWRITES);
20
+ l2j_log("Error: too many rewrites. You can add up to %d rewrite rules.", MAX_REWRITES);
21
return false;
22
}
23
24
if((flags & (RW_MATCH_PCRE2|RW_MATCH_NON_EMPTY)) && (!search_pattern || !*search_pattern)) {
25
- log2stderr("Error: rewrite for key '%s' does not specify a search pattern.", key);
25
+ l2j_log("Error: rewrite for key '%s' does not specify a search pattern.", key);
26
return false;
27
}
28
29
REWRITE *rw = &jb->rewrites.array[jb->rewrites.used++];
30
rw->flags = flags;
31
32
- hashed_key_set(&rw->key, key);
32
+ hashed_key_set(&rw->key, key, -1);
33
34
if((flags & RW_MATCH_PCRE2) && !search_pattern_set(&rw->match_pcre2, search_pattern, strlen(search_pattern))) {
35
rewrite_cleanup(rw);
src/collectors/log2journal/log2journal-txt.h
new
+90
@@ -0,0 +1,90 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_LOG2JOURNAL_TXT_H
4
+#define NETDATA_LOG2JOURNAL_TXT_H
5
+
6
+#include "log2journal.h"
7
+
8
+// ----------------------------------------------------------------------------
9
+// A dynamically sized, reusable text buffer,
10
+// allowing us to be fast (no allocations during iterations) while having the
11
+// smallest possible allocations.
12
+
13
+typedef struct txt_l2j {
14
+ char *txt;
15
+ uint32_t size;
16
+ uint32_t len;
17
+} TXT_L2J;
18
+
19
+static inline void txt_l2j_cleanup(TXT_L2J *t) {
20
+ if(!t)
21
+ return;
22
+
23
+ if(t->txt)
24
+ freez(t->txt);
25
+
26
+ t->txt = NULL;
27
+ t->size = 0;
28
+ t->len = 0;
29
+}
30
+
31
+#define TXT_L2J_ALLOC_ALIGN 1024
32
+
33
+static inline size_t txt_l2j_compute_new_size(size_t old_size, size_t required_size) {
34
+ size_t size = (required_size % TXT_L2J_ALLOC_ALIGN == 0) ? required_size : required_size + TXT_L2J_ALLOC_ALIGN;
35
+ size = (size / TXT_L2J_ALLOC_ALIGN) * TXT_L2J_ALLOC_ALIGN;
36
+
37
+ if(size < old_size * 2)
38
+ size = old_size * 2;
39
+
40
+ return size;
41
+}
42
+
43
+static inline void txt_l2j_resize(TXT_L2J *dst, size_t required_size, bool keep) {
44
+ if(required_size <= dst->size)
45
+ return;
46
+
47
+ size_t new_size = txt_l2j_compute_new_size(dst->size, required_size);
48
+
49
+ if(keep && dst->txt)
50
+ dst->txt = reallocz(dst->txt, new_size);
51
+ else {
52
+ txt_l2j_cleanup(dst);
53
+ dst->txt = mallocz(new_size);
54
+ dst->len = 0;
55
+ }
56
+
57
+ dst->size = new_size;
58
+}
59
+
60
+static inline void txt_l2j_set(TXT_L2J *dst, const char *s, int32_t len) {
61
+ if(!s || !*s || len == 0) {
62
+ s = "";
63
+ len = 0;
64
+ }
65
+
66
+ if(len == -1)
67
+ len = (int32_t)strlen(s);
68
+
69
+ txt_l2j_resize(dst, len + 1, false);
70
+ memcpy(dst->txt, s, len);
71
+ dst->txt[len] = '\0';
72
+ dst->len = len;
73
+}
74
+
75
+static inline void txt_l2j_append(TXT_L2J *dst, const char *s, int32_t len) {
76
+ if(!dst->txt || !dst->len)
77
+ txt_l2j_set(dst, s, len);
78
+
79
+ else {
80
+ if(len == -1)
81
+ len = (int32_t)strlen(s);
82
+
83
+ txt_l2j_resize(dst, dst->len + len + 1, true);
84
+ memcpy(&dst->txt[dst->len], s, len);
85
+ dst->len += len;
86
+ dst->txt[dst->len] = '\0';
87
+ }
88
+}
89
+
90
+#endif //NETDATA_LOG2JOURNAL_TXT_H
src/collectors/log2journal/log2journal-yaml.c
+12
-5
@@ -398,7 +398,8 @@ static size_t yaml_parse_unmatched(yaml_parser_t *parser, LOG_JOB *jb) {
398
errors++;
399
} else {
400
if (sub_event.type == YAML_SCALAR_EVENT) {
401
- hashed_key_len_set(&jb->unmatched.key, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
401
+ hashed_key_set(
402
+ &jb->unmatched.key, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
403
} else {
404
yaml_error(parser, &sub_event, "expected a scalar value for 'key'");
405
errors++;
@@ -663,7 +664,10 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, LOG_JOB *jb) {
664
yaml_error(parser, &value_event, "Expected scalar for rename new_key");
665
errors++;
666
} else {
666
- hashed_key_len_set(&rn.new_key, (char *)value_event.data.scalar.value, value_event.data.scalar.length);
667
+ hashed_key_set(
668
+ &rn.new_key,
669
+ (char *)value_event.data.scalar.value,
670
+ value_event.data.scalar.length);
671
yaml_event_delete(&value_event);
672
}
673
} else if (yaml_scalar_matches(&sub_event, "old_key", strlen("old_key"))) {
@@ -673,7 +677,10 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, LOG_JOB *jb) {
677
yaml_error(parser, &value_event, "Expected scalar for rename old_key");
678
errors++;
679
} else {
676
- hashed_key_len_set(&rn.old_key, (char *)value_event.data.scalar.value, value_event.data.scalar.length);
680
+ hashed_key_set(
681
+ &rn.old_key,
682
+ (char *)value_event.data.scalar.value,
683
+ value_event.data.scalar.length);
684
yaml_event_delete(&value_event);
685
}
686
} else {
@@ -826,13 +833,13 @@ cleanup:
833
834
bool yaml_parse_file(const char *config_file_path, LOG_JOB *jb) {
835
if(!config_file_path || !*config_file_path) {
829
- log2stderr("yaml configuration filename cannot be empty.");
836
+ l2j_log("yaml configuration filename cannot be empty.");
837
return false;
838
}
839
840
FILE *fp = fopen(config_file_path, "r");
841
if (!fp) {
835
- log2stderr("Error opening config file: %s", config_file_path);
842
+ l2j_log("Error opening config file: %s", config_file_path);
843
return false;
844
}
845
src/collectors/log2journal/log2journal.c
+37
-27
@@ -1,6 +1,7 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "log2journal.h"
4
+#include "libnetdata/required_dummies.h"
5
6
// ----------------------------------------------------------------------------
7
@@ -73,10 +74,13 @@ static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
74
ht_key->flags |= HK_COLLISION_CHECKED;
75
76
if(strcmp(ht_key->key, k->key) != 0)
76
- log2stderr("Hashtable collision detected on key '%s' (hash %lx) and '%s' (hash %lx). "
77
- "Please file a bug report.", ht_key->key, (unsigned long) ht_key->hash, k->key
78
- , (unsigned long) k->hash
79
- );
77
+ l2j_log(
78
+ "Hashtable collision detected on key '%s' (hash %lx) and '%s' (hash %lx). "
79
+ "Please file a bug report.",
80
+ ht_key->key,
81
+ (unsigned long)ht_key->hash,
82
+ k->key,
83
+ (unsigned long)k->hash);
84
}
85
}
86
else {
@@ -97,8 +101,9 @@ static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
101
102
static inline HASHED_KEY *get_key_from_hashtable_with_char_ptr(LOG_JOB *jb, const char *key) {
103
HASHED_KEY find = {
100
- .key = key,
101
- .len = strlen(key),
104
+ .flags = HK_NONE,
105
+ .key = key,
106
+ .len = strlen(key),
107
};
108
find.hash = XXH3_64bits(key, find.len);
109
@@ -109,24 +114,29 @@ static inline HASHED_KEY *get_key_from_hashtable_with_char_ptr(LOG_JOB *jb, cons
114
115
static inline void validate_key(LOG_JOB *jb __maybe_unused, HASHED_KEY *k) {
116
if(k->len > JOURNAL_MAX_KEY_LEN)
112
- log2stderr("WARNING: key '%s' has length %zu, which is more than %zu, the max systemd-journal allows",
113
- k->key, (size_t)k->len, (size_t)JOURNAL_MAX_KEY_LEN);
117
+ l2j_log(
118
+ "WARNING: key '%s' has length %zu, which is more than %zu, the max systemd-journal allows",
119
+ k->key,
120
+ (size_t)k->len,
121
+ (size_t)JOURNAL_MAX_KEY_LEN);
122
123
for(size_t i = 0; i < k->len ;i++) {
124
char c = k->key[i];
125
126
if((c < 'A' || c > 'Z') && !isdigit(c) && c != '_') {
119
- log2stderr("WARNING: key '%s' contains characters that are not allowed by systemd-journal.", k->key);
127
+ l2j_log("WARNING: key '%s' contains characters that are not allowed by systemd-journal.", k->key);
128
break;
129
}
130
}
131
132
if(isdigit(k->key[0]))
125
- log2stderr("WARNING: key '%s' starts with a digit and may not be accepted by systemd-journal.", k->key);
133
+ l2j_log("WARNING: key '%s' starts with a digit and may not be accepted by systemd-journal.", k->key);
134
135
if(k->key[0] == '_')
128
- log2stderr("WARNING: key '%s' starts with an underscore, which makes it a systemd-journal trusted field. "
129
- "Such fields are accepted by systemd-journal-remote, but not by systemd-journald.", k->key);
136
+ l2j_log(
137
+ "WARNING: key '%s' starts with an underscore, which makes it a systemd-journal trusted field. "
138
+ "Such fields are accepted by systemd-journal-remote, but not by systemd-journald.",
139
+ k->key);
140
}
141
142
// ----------------------------------------------------------------------------
@@ -170,16 +180,16 @@ static inline void replace_evaluate(LOG_JOB *jb, HASHED_KEY *k, REPLACE_PATTERN
180
for(REPLACE_NODE *node = rp->nodes; node != NULL; node = node->next) {
181
if(node->is_variable) {
182
if(hashed_keys_match(&node->name, &jb->line.key))
173
- txt_expand_and_append(&ht_key->value, jb->line.trimmed, jb->line.trimmed_len);
183
+ txt_l2j_append(&ht_key->value, jb->line.trimmed, jb->line.trimmed_len);
184
185
else {
186
HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
187
if(ktmp->value.len)
178
- txt_expand_and_append(&ht_key->value, ktmp->value.txt, ktmp->value.len);
188
+ txt_l2j_append(&ht_key->value, ktmp->value.txt, ktmp->value.len);
189
}
190
}
191
else
182
- txt_expand_and_append(&ht_key->value, node->name.key, node->name.len);
192
+ txt_l2j_append(&ht_key->value, node->name.key, node->name.len);
193
}
194
}
195
@@ -202,26 +212,26 @@ static inline void replace_evaluate_from_pcre2(LOG_JOB *jb, HASHED_KEY *k, REPLA
212
PCRE2_SIZE end_offset = ovector[2 * group_number + 1];
213
PCRE2_SIZE length = end_offset - start_offset;
214
205
- txt_expand_and_append(&jb->rewrites.tmp, k->value.txt + start_offset, length);
215
+ txt_l2j_append(&jb->rewrites.tmp, k->value.txt + start_offset, length);
216
}
217
else {
218
if(hashed_keys_match(&node->name, &jb->line.key))
209
- txt_expand_and_append(&jb->rewrites.tmp, jb->line.trimmed, jb->line.trimmed_len);
219
+ txt_l2j_append(&jb->rewrites.tmp, jb->line.trimmed, jb->line.trimmed_len);
220
221
else {
222
HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
223
if(ktmp->value.len)
214
- txt_expand_and_append(&jb->rewrites.tmp, ktmp->value.txt, ktmp->value.len);
224
+ txt_l2j_append(&jb->rewrites.tmp, ktmp->value.txt, ktmp->value.len);
225
}
226
}
227
}
228
else {
219
- txt_expand_and_append(&jb->rewrites.tmp, node->name.key, node->name.len);
229
+ txt_l2j_append(&jb->rewrites.tmp, node->name.key, node->name.len);
230
}
231
}
232
233
// swap the values of the temporary TEXT and the key value
224
- TEXT tmp = k->value;
234
+ TXT_L2J tmp = k->value;
235
k->value = jb->rewrites.tmp;
236
jb->rewrites.tmp = tmp;
237
}
@@ -271,7 +281,7 @@ static inline HASHED_KEY *rename_key(LOG_JOB *jb, HASHED_KEY *k) {
281
static inline void send_key_value_constant(LOG_JOB *jb __maybe_unused, HASHED_KEY *key, const char *value, size_t len) {
282
HASHED_KEY *ht_key = get_key_from_hashtable(jb, key);
283
274
- txt_replace(&ht_key->value, value, len);
284
+ txt_l2j_set(&ht_key->value, value, len);
285
ht_key->flags |= HK_VALUE_FROM_LOG;
286
287
// fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt);
@@ -292,7 +302,7 @@ static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char
302
inline void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len) {
303
HASHED_KEY *ht_key = get_key_from_hashtable_with_char_ptr(jb, key);
304
HASHED_KEY *nk = rename_key(jb, ht_key);
295
- txt_replace(&nk->value, value, len);
305
+ txt_l2j_set(&nk->value, value, len);
306
ht_key->flags |= HK_VALUE_FROM_LOG;
307
308
// fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt);
@@ -417,7 +427,7 @@ static inline bool jb_switched_filename(LOG_JOB *jb, const char *line, size_t le
427
const char *end = strstr(line, " <==");
428
while (*start == ' ') start++;
429
if (*start != '\n' && *start != '\0' && end) {
420
- txt_replace(&jb->filename.current, start, end - start);
430
+ txt_l2j_set(&jb->filename.current, start, end - start);
431
return true;
432
}
433
}
@@ -486,7 +496,7 @@ int log_job_run(LOG_JOB *jb) {
496
else if(strcmp(jb->pattern, "none") != 0) {
497
pcre2 = pcre2_parser_create(jb);
498
if(pcre2_has_error(pcre2)) {
489
- log2stderr("%s", pcre2_parser_error(pcre2));
499
+ l2j_log("%s", pcre2_parser_error(pcre2));
500
pcre2_parser_destroy(pcre2);
501
return 1;
502
}
@@ -515,11 +525,11 @@ int log_job_run(LOG_JOB *jb) {
525
526
if(!line_is_matched) {
527
if(json)
518
- log2stderr("%s", json_parser_error(json));
528
+ l2j_log("%s", json_parser_error(json));
529
else if(logfmt)
520
- log2stderr("%s", logfmt_parser_error(logfmt));
530
+ l2j_log("%s", logfmt_parser_error(logfmt));
531
else if(pcre2)
522
- log2stderr("%s", pcre2_parser_error(pcre2));
532
+ l2j_log("%s", pcre2_parser_error(pcre2));
533
534
if(!jb_send_unmatched_line(jb, line))
535
// just logging to stderr, not sending unmatched lines
src/collectors/log2journal/log2journal.h
+9
-241
@@ -3,49 +3,16 @@
3
#ifndef NETDATA_LOG2JOURNAL_H
4
#define NETDATA_LOG2JOURNAL_H
5
6
-// only for PACKAGE_VERSION
7
-#include <config.h>
8
-
9
-#include <stdio.h>
10
-#include <stdlib.h>
11
-#include <dirent.h>
12
-#include <string.h>
13
-#include <stdbool.h>
14
-#include <string.h>
15
-#include <ctype.h>
16
-#include <math.h>
17
-#include <stdarg.h>
18
-#include <assert.h>
19
-
20
-// ----------------------------------------------------------------------------
21
-// compatibility
22
-
23
-#ifndef HAVE_STRNDUP
24
-// strndup() is not available on Windows
25
-static inline char *os_strndup( const char *s1, size_t n)
26
-{
27
- char *copy= (char*)malloc( n+1 );
28
- memcpy( copy, s1, n );
29
- copy[n] = 0;
30
- return copy;
31
-};
32
-#define strndup(s, n) os_strndup(s, n)
33
-#endif
34
-
35
-#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT_GNU_PRINTF)
36
-#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a)))
37
-#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT_PRINTF)
38
-#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a)))
39
-#else
40
-#define PRINTFLIKE(f, a)
41
-#endif
6
+#include "libnetdata/libnetdata.h"
7
+#include "log2journal-txt.h"
8
+#include "log2journal-hashed-key.h"
9
10
// ----------------------------------------------------------------------------
11
// logging
12
13
// enable the compiler to check for printf like errors on our log2stderr() function
47
-static inline void log2stderr(const char *format, ...) PRINTFLIKE(1, 2);
48
-static inline void log2stderr(const char *format, ...) {
14
+static inline void l2j_log(const char *format, ...) PRINTFLIKE(1, 2);
15
+static inline void l2j_log(const char *format, ...) {
16
va_list args;
17
va_start(args, format);
18
vfprintf(stderr, format, args);
@@ -54,62 +21,6 @@ static inline void log2stderr(const char *format, ...) {
21
}
22
23
// ----------------------------------------------------------------------------
57
-// allocation functions abstraction
58
-
59
-static inline void *mallocz(size_t size) {
60
- void *ptr = malloc(size);
61
- if (!ptr) {
62
- log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", size);
63
- exit(EXIT_FAILURE);
64
- }
65
- return ptr;
66
-}
67
-
68
-static inline void *callocz(size_t elements, size_t size) {
69
- void *ptr = calloc(elements, size);
70
- if (!ptr) {
71
- log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", elements * size);
72
- exit(EXIT_FAILURE);
73
- }
74
- return ptr;
75
-}
76
-
77
-static inline void *reallocz(void *ptr, size_t size) {
78
- void *new_ptr = realloc(ptr, size);
79
- if (!new_ptr) {
80
- log2stderr("Fatal Error: Memory reallocation failed. Requested size: %zu bytes.", size);
81
- exit(EXIT_FAILURE);
82
- }
83
- return new_ptr;
84
-}
85
-
86
-static inline char *strdupz(const char *s) {
87
- char *ptr = strdup(s);
88
- if (!ptr) {
89
- log2stderr("Fatal Error: Memory allocation failed in strdup.");
90
- exit(EXIT_FAILURE);
91
- }
92
- return ptr;
93
-}
94
-
95
-static inline char *strndupz(const char *s, size_t n) {
96
- char *ptr = strndup(s, n);
97
- if (!ptr) {
98
- log2stderr("Fatal Error: Memory allocation failed in strndup. Requested size: %zu bytes.", n);
99
- exit(EXIT_FAILURE);
100
- }
101
- return ptr;
102
-}
103
-
104
-static inline void freez(void *ptr) {
105
- if (ptr)
106
- free(ptr);
107
-}
108
-
109
-// ----------------------------------------------------------------------------
110
-
111
-#define XXH_INLINE_ALL
112
-#include "libnetdata/xxHash/xxhash.h"
24
25
#define PCRE2_CODE_UNIT_WIDTH 8
26
#include <pcre2.h>
@@ -121,13 +32,10 @@ static inline void freez(void *ptr) {
32
// ----------------------------------------------------------------------------
33
// hashtable for HASHED_KEY
34
124
-// cleanup hashtable defines
125
-#include "libnetdata/simple_hashtable/simple_hashtable_undef.h"
126
-
35
struct hashed_key;
36
static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2);
37
#define SIMPLE_HASHTABLE_SORT_FUNCTION compare_keys
130
-#define SIMPLE_HASHTABLE_VALUE_TYPE struct hashed_key
38
+#define SIMPLE_HASHTABLE_VALUE_TYPE HASHED_KEY
39
#define SIMPLE_HASHTABLE_NAME _KEY
40
#include "libnetdata/simple_hashtable/simple_hashtable.h"
41
@@ -172,153 +80,13 @@ static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src,
80
}
81
}
82
175
-// ----------------------------------------------------------------------------
176
-// A dynamically sized, reusable text buffer,
177
-// allowing us to be fast (no allocations during iterations) while having the
178
-// smallest possible allocations.
179
-
180
-typedef struct txt {
181
- char *txt;
182
- uint32_t size;
183
- uint32_t len;
184
-} TEXT;
185
-
186
-static inline void txt_cleanup(TEXT *t) {
187
- if(!t)
188
- return;
189
-
190
- if(t->txt)
191
- freez(t->txt);
192
-
193
- t->txt = NULL;
194
- t->size = 0;
195
- t->len = 0;
196
-}
197
-
198
-static inline void txt_replace(TEXT *t, const char *s, size_t len) {
199
- if(!s || !*s || len == 0) {
200
- s = "";
201
- len = 0;
202
- }
203
-
204
- if(len + 1 <= t->size) {
205
- // the existing value allocation, fits our value
206
-
207
- memcpy(t->txt, s, len);
208
- t->txt[len] = '\0';
209
- t->len = len;
210
- }
211
- else {
212
- // no existing value allocation, or too small for our value
213
- // cleanup and increase the buffer
214
-
215
- txt_cleanup(t);
216
-
217
- t->txt = strndupz(s, len);
218
- t->size = len + 1;
219
- t->len = len;
220
- }
221
-}
222
-
223
-static inline void txt_expand_and_append(TEXT *t, const char *s, size_t len) {
224
- if(len + 1 > (t->size - t->len)) {
225
- size_t new_size = t->len + len + 1;
226
- if(new_size < t->size * 2)
227
- new_size = t->size * 2;
228
-
229
- t->txt = reallocz(t->txt, new_size);
230
- t->size = new_size;
231
- }
232
-
233
- char *copy_to = &t->txt[t->len];
234
- memcpy(copy_to, s, len);
235
- copy_to[len] = '\0';
236
- t->len += len;
237
-}
238
-
239
-// ----------------------------------------------------------------------------
240
-
241
-typedef enum __attribute__((__packed__)) {
242
- HK_NONE = 0,
243
-
244
- // permanent flags - they are set once to optimize various decisions and lookups
245
-
246
- HK_HASHTABLE_ALLOCATED = (1 << 0), // this is key object allocated in the hashtable
247
- // objects that do not have this, have a pointer to a key in the hashtable
248
- // objects that have this, value is allocated
249
-
250
- HK_FILTERED = (1 << 1), // we checked once if this key in filtered
251
- HK_FILTERED_INCLUDED = (1 << 2), // the result of the filtering was to include it in the output
252
-
253
- HK_COLLISION_CHECKED = (1 << 3), // we checked once for collision check of this key
254
-
255
- HK_RENAMES_CHECKED = (1 << 4), // we checked once if there are renames on this key
256
- HK_HAS_RENAMES = (1 << 5), // and we found there is a rename rule related to it
257
-
258
- // ephemeral flags - they are unset at the end of each log line
259
-
260
- HK_VALUE_FROM_LOG = (1 << 14), // the value of this key has been read from the log (or from injection, duplication)
261
- HK_VALUE_REWRITTEN = (1 << 15), // the value of this key has been rewritten due to one of our rewrite rules
262
-
263
-} HASHED_KEY_FLAGS;
264
-
265
-typedef struct hashed_key {
266
- const char *key;
267
- uint32_t len;
268
- HASHED_KEY_FLAGS flags;
269
- XXH64_hash_t hash;
270
- union {
271
- struct hashed_key *hashtable_ptr; // HK_HASHTABLE_ALLOCATED is not set
272
- TEXT value; // HK_HASHTABLE_ALLOCATED is set
273
- };
274
-} HASHED_KEY;
275
-
276
-static inline void hashed_key_cleanup(HASHED_KEY *k) {
277
- if(k->flags & HK_HASHTABLE_ALLOCATED)
278
- txt_cleanup(&k->value);
279
- else
280
- k->hashtable_ptr = NULL;
281
-
282
- freez((void *)k->key);
283
- k->key = NULL;
284
- k->len = 0;
285
- k->hash = 0;
286
- k->flags = HK_NONE;
287
-}
288
-
289
-static inline void hashed_key_set(HASHED_KEY *k, const char *name) {
290
- hashed_key_cleanup(k);
291
-
292
- k->key = strdupz(name);
293
- k->len = strlen(k->key);
294
- k->hash = XXH3_64bits(k->key, k->len);
295
- k->flags = HK_NONE;
296
-}
297
-
298
-static inline void hashed_key_len_set(HASHED_KEY *k, const char *name, size_t len) {
299
- hashed_key_cleanup(k);
300
-
301
- k->key = strndupz(name, len);
302
- k->len = len;
303
- k->hash = XXH3_64bits(k->key, k->len);
304
- k->flags = HK_NONE;
305
-}
306
-
307
-static inline bool hashed_keys_match(HASHED_KEY *k1, HASHED_KEY *k2) {
308
- return ((k1 == k2) || (k1->hash == k2->hash && strcmp(k1->key, k2->key) == 0));
309
-}
310
-
311
-static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2) {
312
- return strcmp(k1->key, k2->key);
313
-}
314
-
83
// ----------------------------------------------------------------------------
84
85
typedef struct search_pattern {
86
const char *pattern;
87
pcre2_code *re;
88
pcre2_match_data *match_data;
321
- TEXT error;
89
+ TXT_L2J error;
90
} SEARCH_PATTERN;
91
92
void search_pattern_cleanup(SEARCH_PATTERN *sp);
@@ -417,7 +185,7 @@ typedef struct log_job {
185
struct {
186
bool last_line_was_empty;
187
HASHED_KEY key;
420
- TEXT current;
188
+ TXT_L2J current;
189
} filename;
190
191
struct {
@@ -436,7 +204,7 @@ typedef struct log_job {
204
struct {
205
uint32_t used;
206
REWRITE array[MAX_REWRITES];
439
- TEXT tmp;
207
+ TXT_L2J tmp;
208
} rewrites;
209
210
struct {
src/libnetdata/string/utf8.c
+1
-1
@@ -270,7 +270,7 @@ void txt_utf8_empty(TXT_UTF8 *dst) {
270
}
271
272
void txt_utf8_set(TXT_UTF8 *dst, const char *txt, size_t txt_len) {
273
- txt_utf8_resize(dst, dst->used + txt_len + 1, true);
273
+ txt_utf8_resize(dst, txt_len + 1, false);
274
memcpy(dst->data, txt, txt_len);
275
dst->used = txt_len + 1;
276
dst->data[dst->used - 1] = '\0';