@cryptotaxi247 / netdata-1 / commits / a64c8cdb4

log2journal moved to collectors (#16481)

* log2journal moved to collectors * split log2journal into multiple files * update the path xxh headers * json support for log2journal * logfmt support * fix warning * fix logfmt prefix * added support for UTF-8 escape sequences in json values

Costa Tsaousis committed Nov 28, 2023 at 02:05 UTC a64c8cdb43bd98ac9c8165e14a621881c730dde5
16 files changed +3316 -2137
.gitignore
+1
@@ -43,6 +43,7 @@ netdata
43 netdatacli
44 systemd-cat-native
45 log2journal
46 +!log2journal/
47 !netdata/
48 upload/
49 artifacts/
Makefile.am
+7 -1
@@ -347,7 +347,13 @@ SYSTEMD_CAT_NATIVE_FILES = \
347 $(NULL)
348
349 LOG2JOURNAL_FILES = \
350 - libnetdata/log/log2journal.c \
350 + collectors/log2journal/log2journal.h \
351 + collectors/log2journal/log2journal.c \
352 + collectors/log2journal/log2journal-help.c \
353 + collectors/log2journal/log2journal-yaml.c \
354 + collectors/log2journal/log2journal-json.c \
355 + collectors/log2journal/log2journal-logfmt.c \
356 + collectors/log2journal/log2journal-params.c \
357 $(NULL)
358
359
collectors/Makefile.am
+1
@@ -15,6 +15,7 @@ SUBDIRS = \
15 freebsd.plugin \
16 freeipmi.plugin \
17 idlejitter.plugin \
18 + log2journal \
19 macos.plugin \
20 nfacct.plugin \
21 xenstat.plugin \
collectors/log2journal/Makefile.am new
+12
@@ -0,0 +1,12 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
9 +
10 +dist_libconfig_DATA = \
11 + log2journal.d/nginx-combined.yaml \
12 + $(NULL)
collectors/log2journal/README.md renamed
collectors/log2journal/log2journal-help.c new
+221
@@ -0,0 +1,221 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "log2journal.h"
4 +
5 +static void config_dir_print_available(void) {
6 + const char *path = LOG2JOURNAL_CONFIG_PATH;
7 + DIR *dir;
8 + struct dirent *entry;
9 +
10 + dir = opendir(path);
11 +
12 + if (dir == NULL) {
13 + log2stderr(" >>> Cannot open directory '%s'", path);
14 + return;
15 + }
16 +
17 + size_t column_width = 80;
18 + size_t current_columns = 0;
19 +
20 + while ((entry = readdir(dir))) {
21 + if (entry->d_type == DT_REG) { // Check if it's a regular file
22 + const char *file_name = entry->d_name;
23 + size_t len = strlen(file_name);
24 + if (len >= 5 && strcmp(file_name + len - 5, ".yaml") == 0) {
25 + // Remove the ".yaml" extension
26 + len -= 5;
27 + if (current_columns + len + 1 > column_width) {
28 + // Start a new line if the current line is full
29 + printf("\n ");
30 + current_columns = 0;
31 + }
32 + printf("%.*s ", (int)len, file_name); // Print the filename without extension
33 + current_columns += len + 1; // Add filename length and a space
34 + }
35 + }
36 + }
37 +
38 + closedir(dir);
39 + printf("\n"); // Add a newline at the end
40 +}
41 +
42 +void log2journal_command_line_help(const char *name) {
43 + printf("\n");
44 + printf("Netdata log2journal " PACKAGE_VERSION "\n");
45 + printf("\n");
46 + printf("Convert logs to systemd Journal Export Format.\n");
47 + printf("\n");
48 + printf(" - JSON logs: extracts all JSON fields.\n");
49 + printf(" - logfmt logs: extracts all logfmt fields.\n");
50 + printf(" - free-form logs: uses PCRE2 patterns to extracts fields.\n");
51 + printf("\n");
52 + printf("Usage: %s [OPTIONS] PATTERN|json\n", name);
53 + printf("\n");
54 + printf("Options:\n");
55 + printf("\n");
56 +#ifdef HAVE_LIBYAML
57 + printf(" --file /path/to/file.yaml\n");
58 + printf(" Read yaml configuration file for instructions.\n");
59 + printf("\n");
60 + printf(" --config CONFIG_NAME\n");
61 + printf(" Run with the internal configuration named CONFIG_NAME\n");
62 + printf(" Available internal configs:\n");
63 + printf("\n");
64 + config_dir_print_available();
65 + printf("\n");
66 +#else
67 + printf(" IMPORTANT:\n");
68 + printf(" YAML configuration parsing is not compiled in this binary.\n");
69 + printf("\n");
70 +#endif
71 + printf(" --show-config\n");
72 + printf(" Show the configuration in YAML format before starting the job.\n");
73 + printf(" This is also an easy way to convert command line parameters to yaml.\n");
74 + printf("\n");
75 + printf(" --filename-key KEY\n");
76 + printf(" Add a field with KEY as the key and the current filename as value.\n");
77 + printf(" Automatically detects filenames when piped after 'tail -F',\n");
78 + printf(" and tail matches multiple filenames.\n");
79 + printf(" To inject the filename when tailing a single file, use --inject.\n");
80 + printf("\n");
81 + printf(" --unmatched-key KEY\n");
82 + printf(" Include unmatched log entries in the output with KEY as the field name.\n");
83 + printf(" Use this to include unmatched entries to the output stream.\n");
84 + printf(" Usually it should be set to --unmatched-key=MESSAGE so that the\n");
85 + printf(" unmatched entry will appear as the log message in the journals.\n");
86 + printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n");
87 + printf("\n");
88 + printf(" --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]\n");
89 + printf(" Create a new key called TARGET, duplicating the values of the keys\n");
90 + printf(" given. Useful for further processing. When multiple keys are given,\n");
91 + printf(" their values are separated by comma.\n");
92 + printf(" Up to %d duplications can be given on the command line, and up to\n", MAX_KEY_DUPS);
93 + printf(" %d keys per duplication command are allowed.\n", MAX_KEY_DUPS_KEYS);
94 + printf("\n");
95 + printf(" --inject LINE\n");
96 + printf(" Inject constant fields to the output (both matched and unmatched logs).\n");
97 + printf(" --inject entries are added to unmatched lines too, when their key is\n");
98 + printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n");
99 + printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS);
100 + printf("\n");
101 + printf(" --inject-unmatched LINE\n");
102 + printf(" Inject lines into the output for each unmatched log entry.\n");
103 + printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n");
104 + printf(" lines as errors, so that they can easily be spotted in the journals.\n");
105 + printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS);
106 + printf("\n");
107 + printf(" --rewrite KEY=/SearchPattern/ReplacePattern\n");
108 + printf(" Apply a rewrite rule to the values of a specific key.\n");
109 + printf(" The first character after KEY= is the separator, which should also\n");
110 + printf(" be used between the search pattern and the replacement pattern.\n");
111 + printf(" The search pattern is a PCRE2 regular expression, and the replacement\n");
112 + printf(" pattern supports literals and named capture groups from the search pattern.\n");
113 + printf(" Example:\n");
114 + printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n");
115 + printf(" ${day}/${month}/${year}\n");
116 + printf(" This will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.\n");
117 + printf("\n");
118 + printf(" Only one rewrite rule is applied per key; the sequence of rewrites stops\n");
119 + printf(" for the key once a rule matches it. This allows providing a sequence of\n");
120 + printf(" independent rewriting rules for the same key, matching the different values\n");
121 + printf(" the key may get, and also provide a catch-all rewrite rule at the end of the\n");
122 + printf(" sequence for setting the key value if no other rule matched it.\n");
123 + printf("\n");
124 + printf(" The combination of duplicating keys with the values of multiple other keys\n");
125 + printf(" combined with multiple rewrite rules, allows creating complex rules for\n");
126 + printf(" rewriting key values.\n");
127 + printf(" Up to %d rewriting rules are allowed.\n", MAX_REWRITES);
128 + printf("\n");
129 + printf(" --prefix PREFIX\n");
130 + printf(" Prefix all JSON or logfmt fields with PREFIX.\n");
131 + printf("\n");
132 + printf(" --rename NEW=OLD\n");
133 + printf(" Rename fields, before rewriting their values.\n");
134 + printf(" Up to %d renaming rules are allowed.\n", MAX_RENAMES);
135 + printf("\n");
136 + printf(" -h, --help\n");
137 + printf(" Display this help and exit.\n");
138 + printf("\n");
139 + printf(" PATTERN\n");
140 + printf(" PATTERN should be a valid PCRE2 regular expression.\n");
141 + printf(" RE2 regular expressions (like the ones usually used in Go applications),\n");
142 + printf(" are usually valid PCRE2 patterns too.\n");
143 + printf(" Regular expressions without named groups are evaluated but their matches\n");
144 + printf(" are not added to the output.\n");
145 + printf("\n");
146 + printf(" JSON mode\n");
147 + printf(" JSON mode is enabled when the pattern is set to: json\n");
148 + printf(" Field names are extracted from the JSON logs and are converted to the\n");
149 + printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n");
150 + printf(" Prefixing is enabled in this mode.\n");
151 + printf(" logfmt mode\n");
152 + printf(" logfmt mode is enabled when the pattern is set to: logfmt\n");
153 + printf(" Field names are extracted from the logfmt logs and are converted to the\n");
154 + printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n");
155 + printf(" Prefixing is enabled in this mode.\n");
156 + printf("\n");
157 + printf("\n");
158 + printf("The program accepts all parameters as both --option=value and --option value.\n");
159 + printf("\n");
160 + printf("The maximum line length accepted is %d characters.\n", MAX_LINE_LENGTH);
161 + printf("The maximum number of fields in the PCRE2 pattern is %d.\n", OVECCOUNT / 3);
162 + printf("\n");
163 + printf("PIPELINE AND SEQUENCE OF PROCESSING\n");
164 + printf("\n");
165 + printf("This is a simple diagram of the pipeline taking place:\n");
166 + printf("\n");
167 + printf(" +---------------------------------------------------+\n");
168 + printf(" | INPUT |\n");
169 + printf(" +---------------------------------------------------+\n");
170 + printf(" v v\n");
171 + printf(" +---------------------------------+ |\n");
172 + printf(" | EXTRACT FIELDS AND VALUES | |\n");
173 + printf(" +---------------------------------+ |\n");
174 + printf(" v v |\n");
175 + printf(" +---------------+ +--------------+ |\n");
176 + printf(" | DUPLICATE | | RENAME | |\n");
177 + printf(" | create fields | | change the | |\n");
178 + printf(" | with values | | field name | |\n");
179 + printf(" +---------------+ +--------------+ |\n");
180 + printf(" v v v\n");
181 + printf(" +---------------------------------+ +--------------+\n");
182 + printf(" | REWRITE PIPELINES | | INJECT |\n");
183 + printf(" | altering keys and values | | constants |\n");
184 + printf(" +---------------------------------+ +--------------+\n");
185 + printf(" v v\n");
186 + printf(" +---------------------------------------------------+\n");
187 + printf(" | OUTPUT |\n");
188 + printf(" +---------------------------------------------------+\n");
189 + printf("\n");
190 + printf("JOURNAL FIELDS RULES (enforced by systemd-journald)\n");
191 + printf("\n");
192 + printf(" - field names can be up to 64 characters\n");
193 + printf(" - the only allowed field characters are A-Z, 0-9 and underscore\n");
194 + printf(" - the first character of fields cannot be a digit\n");
195 + printf(" - protected journal fields start with underscore:\n");
196 + printf(" * they are accepted by systemd-journal-remote\n");
197 + printf(" * they are NOT accepted by a local systemd-journald\n");
198 + printf("\n");
199 + printf(" For best results, always include these fields:\n");
200 + printf("\n");
201 + printf(" MESSAGE=TEXT\n");
202 + printf(" The MESSAGE is the body of the log entry.\n");
203 + printf(" This field is what we usually see in our logs.\n");
204 + printf("\n");
205 + printf(" PRIORITY=NUMBER\n");
206 + printf(" PRIORITY sets the severity of the log entry.\n");
207 + printf(" 0=emerg, 1=alert, 2=crit, 3=err, 4=warn, 5=notice, 6=info, 7=debug\n");
208 + printf(" - Emergency events (0) are usually broadcast to all terminals.\n");
209 + printf(" - Emergency, alert, critical, and error (0-3) are usually colored red.\n");
210 + printf(" - Warning (4) entries are usually colored yellow.\n");
211 + printf(" - Notice (5) entries are usually bold or have a brighter white color.\n");
212 + printf(" - Info (6) entries are the default.\n");
213 + printf(" - Debug (7) entries are usually grayed or dimmed.\n");
214 + printf("\n");
215 + printf(" SYSLOG_IDENTIFIER=NAME\n");
216 + printf(" SYSLOG_IDENTIFIER sets the name of application.\n");
217 + printf(" Use something descriptive, like: SYSLOG_IDENTIFIER=nginx-logs\n");
218 + printf("\n");
219 + printf("You can find the most common fields at 'man systemd.journal-fields'.\n");
220 + printf("\n");
221 +}
collectors/log2journal/log2journal-json.c new
+633
@@ -0,0 +1,633 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "log2journal.h"
4 +
5 +#define ERROR_LINE_MAX 1024
6 +#define KEY_MAX 1024
7 +#define JSON_DEPTH_MAX 100
8 +
9 +struct log_json_state {
10 + const char *line;
11 + size_t pos;
12 + char msg[ERROR_LINE_MAX];
13 +
14 + char key[KEY_MAX];
15 + char *key_stack[JSON_DEPTH_MAX];
16 + size_t depth;
17 +
18 + struct log_job *jb;
19 +};
20 +
21 +static inline bool json_parse_object(LOG_JSON_STATE *js);
22 +static inline bool json_parse_array(LOG_JSON_STATE *js);
23 +
24 +#define json_current_pos(js) &(js)->line[(js)->pos]
25 +#define json_consume_char(js) ++(js)->pos
26 +
27 +static inline void json_process_key_value(LOG_JSON_STATE *js, const char *value, size_t len) {
28 + jb_send_extracted_key_value(js->jb, js->key, value, len);
29 +}
30 +
31 +static inline void json_skip_spaces(LOG_JSON_STATE *js) {
32 + const char *s = json_current_pos(js);
33 + const char *start = s;
34 +
35 + while(isspace(*s)) s++;
36 +
37 + js->pos += s - start;
38 +}
39 +
40 +static inline bool json_expect_char_after_white_space(LOG_JSON_STATE *js, const char *expected) {
41 + json_skip_spaces(js);
42 +
43 + const char *s = json_current_pos(js);
44 + for(const char *e = expected; *e ;e++) {
45 + if (*s == *e)
46 + return true;
47 + }
48 +
49 + snprintf(js->msg, sizeof(js->msg),
50 + "JSON PARSER: character '%c' is not one of the expected characters (%s), at pos %zu",
51 + *s ? *s : '?', expected, js->pos);
52 +
53 + return false;
54 +}
55 +
56 +static inline bool json_parse_null(LOG_JSON_STATE *js) {
57 + const char *s = json_current_pos(js);
58 + if (strncmp(s, "null", 4) == 0) {
59 + json_process_key_value(js, "null", 4);
60 + js->pos += 4;
61 + return true;
62 + }
63 + else {
64 + snprintf(js->msg, sizeof(js->msg),
65 + "JSON PARSER: expected 'null', found '%.4s' at position %zu", s, js->pos);
66 + return false;
67 + }
68 +}
69 +
70 +static inline bool json_parse_true(LOG_JSON_STATE *js) {
71 + const char *s = json_current_pos(js);
72 + if (strncmp(s, "true", 4) == 0) {
73 + json_process_key_value(js, "true", 4);
74 + js->pos += 4;
75 + return true;
76 + }
77 + else {
78 + snprintf(js->msg, sizeof(js->msg),
79 + "JSON PARSER: expected 'true', found '%.4s' at position %zu", s, js->pos);
80 + return false;
81 + }
82 +}
83 +
84 +static inline bool json_parse_false(LOG_JSON_STATE *js) {
85 + const char *s = json_current_pos(js);
86 + if (strncmp(s, "false", 5) == 0) {
87 + json_process_key_value(js, "false", 5);
88 + js->pos += 5;
89 + return true;
90 + }
91 + else {
92 + snprintf(js->msg, sizeof(js->msg),
93 + "JSON PARSER: expected 'false', found '%.4s' at position %zu", s, js->pos);
94 + return false;
95 + }
96 +}
97 +
98 +static inline bool json_parse_number(LOG_JSON_STATE *js) {
99 + static __thread char value[8192];
100 +
101 + value[0] = '\0';
102 + char *d = value;
103 + const char *s = json_current_pos(js);
104 + size_t remaining = sizeof(value) - 1; // Reserve space for null terminator
105 +
106 + // Optional minus sign
107 + if (*s == '-') {
108 + *d++ = *s++;
109 + remaining--;
110 + }
111 +
112 + // Digits before decimal point
113 + while (*s >= '0' && *s <= '9') {
114 + if (remaining < 2) {
115 + snprintf(js->msg, sizeof(js->msg), "JSON PARSER: truncated number value at pos %zu", js->pos);
116 + return false;
117 + }
118 + *d++ = *s++;
119 + remaining--;
120 + }
121 +
122 + // Decimal point and fractional part
123 + if (*s == '.') {
124 + *d++ = *s++;
125 + remaining--;
126 +
127 + while (*s >= '0' && *s <= '9') {
128 + if (remaining < 2) {
129 + snprintf(js->msg, sizeof(js->msg), "JSON PARSER: truncated fractional part at pos %zu", js->pos);
130 + return false;
131 + }
132 + *d++ = *s++;
133 + remaining--;
134 + }
135 + }
136 +
137 + // Exponent part
138 + if (*s == 'e' || *s == 'E') {
139 + *d++ = *s++;
140 + remaining--;
141 +
142 + // Optional sign in exponent
143 + if (*s == '+' || *s == '-') {
144 + *d++ = *s++;
145 + remaining--;
146 + }
147 +
148 + while (*s >= '0' && *s <= '9') {
149 + if (remaining < 2) {
150 + snprintf(js->msg, sizeof(js->msg), "JSON PARSER: truncated exponent at pos %zu", js->pos);
151 + return false;
152 + }
153 + *d++ = *s++;
154 + remaining--;
155 + }
156 + }
157 +
158 + *d = '\0';
159 + js->pos += d - value;
160 +
161 + if (d > value) {
162 + json_process_key_value(js, value, d - value);
163 + return true;
164 + } else {
165 + snprintf(js->msg, sizeof(js->msg), "JSON PARSER: invalid number format at pos %zu", js->pos);
166 + return false;
167 + }
168 +}
169 +
170 +static bool encode_utf8(unsigned codepoint, char **d, size_t *remaining) {
171 + if (codepoint <= 0x7F) {
172 + // 1-byte sequence
173 + if (*remaining < 2) return false; // +1 for the null
174 + *(*d)++ = (char)codepoint;
175 + (*remaining)--;
176 + }
177 + else if (codepoint <= 0x7FF) {
178 + // 2-byte sequence
179 + if (*remaining < 3) return false; // +1 for the null
180 + *(*d)++ = (char)(0xC0 | ((codepoint >> 6) & 0x1F));
181 + *(*d)++ = (char)(0x80 | (codepoint & 0x3F));
182 + (*remaining) -= 2;
183 + }
184 + else if (codepoint <= 0xFFFF) {
185 + // 3-byte sequence
186 + if (*remaining < 4) return false; // +1 for the null
187 + *(*d)++ = (char)(0xE0 | ((codepoint >> 12) & 0x0F));
188 + *(*d)++ = (char)(0x80 | ((codepoint >> 6) & 0x3F));
189 + *(*d)++ = (char)(0x80 | (codepoint & 0x3F));
190 + (*remaining) -= 3;
191 + }
192 + else if (codepoint <= 0x10FFFF) {
193 + // 4-byte sequence
194 + if (*remaining < 5) return false; // +1 for the null
195 + *(*d)++ = (char)(0xF0 | ((codepoint >> 18) & 0x07));
196 + *(*d)++ = (char)(0x80 | ((codepoint >> 12) & 0x3F));
197 + *(*d)++ = (char)(0x80 | ((codepoint >> 6) & 0x3F));
198 + *(*d)++ = (char)(0x80 | (codepoint & 0x3F));
199 + (*remaining) -= 4;
200 + }
201 + else
202 + // Invalid code point
203 + return false;
204 +
205 + return true;
206 +}
207 +
208 +static inline bool json_parse_string(LOG_JSON_STATE *js) {
209 + static __thread char value[MAX_VALUE_LEN];
210 +
211 + if(!json_expect_char_after_white_space(js, "\""))
212 + return false;
213 +
214 + json_consume_char(js);
215 +
216 + value[0] = '\0';
217 + char *d = value;
218 + const char *s = json_current_pos(js);
219 + size_t remaining = sizeof(value);
220 +
221 + while (*s && *s != '"') {
222 + char c;
223 +
224 + if (*s == '\\') {
225 + s++;
226 +
227 + switch (*s) {
228 + case 'n':
229 + c = '\n';
230 + s++;
231 + break;
232 + case 't':
233 + c = '\t';
234 + s++;
235 + break;
236 + case 'b':
237 + c = '\b';
238 + s++;
239 + break;
240 + case 'f':
241 + c = '\f';
242 + s++;
243 + break;
244 + case 'r':
245 + c = '\r';
246 + s++;
247 + break;
248 + case 'u':
249 + if(isxdigit(s[1]) && isxdigit(s[2]) && isxdigit(s[3]) && isxdigit(s[4])) {
250 + char b[5] = {
251 + [0] = s[1],
252 + [1] = s[2],
253 + [2] = s[3],
254 + [3] = s[4],
255 + [4] = '\0',
256 + };
257 + unsigned codepoint = strtoul(b, NULL, 16);
258 + if(encode_utf8(codepoint, &d, &remaining)) {
259 + s += 5;
260 + continue;
261 + }
262 + else {
263 + *d++ = '\\';
264 + remaining--;
265 + c = *s++;
266 + }
267 + }
268 + else {
269 + *d++ = '\\';
270 + remaining--;
271 + c = *s++;
272 + }
273 + break;
274 +
275 + default:
276 + c = *s++;
277 + break;
278 + }
279 + }
280 + else
281 + c = *s++;
282 +
283 + if(remaining < 2) {
284 + snprintf(js->msg, sizeof(js->msg),
285 + "JSON PARSER: truncated string value at pos %zu", js->pos);
286 + return false;
287 + }
288 + else {
289 + *d++ = c;
290 + remaining--;
291 + }
292 + }
293 + *d = '\0';
294 + js->pos += s - json_current_pos(js);
295 +
296 + if(!json_expect_char_after_white_space(js, "\""))
297 + return false;
298 +
299 + json_consume_char(js);
300 +
301 + if(d > value)
302 + json_process_key_value(js, value, d - value);
303 +
304 + return true;
305 +}
306 +
307 +static inline bool json_parse_key_and_push(LOG_JSON_STATE *js) {
308 + static const char valid_journal_key_chars[256] = {
309 + // control characters
310 + [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_',
311 + [8] = '_', [9] = '_', [10] = '_', [11] = '_', [12] = '_', [13] = '_', [14] = '_', [15] = '_',
312 + [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_', [22] = '_', [23] = '_',
313 + [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_', [30] = '_', [31] = '_',
314 +
315 + // symbols
316 + [' '] = '_', ['!'] = '_', ['"'] = '_', ['#'] = '_', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_',
317 + ['('] = '_', [')'] = '_', ['*'] = '_', ['+'] = '_', [','] = '_', ['-'] = '_', ['.'] = '_', ['/'] = '_',
318 +
319 + // numbers
320 + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7',
321 + ['8'] = '8', ['9'] = '9',
322 +
323 + // symbols
324 + [':'] = '_', [';'] = '_', ['<'] = '_', ['='] = '_', ['>'] = '_', ['?'] = '_', ['@'] = '_',
325 +
326 + // capitals
327 + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H',
328 + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P',
329 + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X',
330 + ['Y'] = 'Y', ['Z'] = 'Z',
331 +
332 + // symbols
333 + ['['] = '_', ['\\'] = '_', [']'] = '_', ['^'] = '_', ['_'] = '_', ['`'] = '_',
334 +
335 + // lower to upper
336 + ['a'] = 'A', ['b'] = 'B', ['c'] = 'C', ['d'] = 'D', ['e'] = 'E', ['f'] = 'F', ['g'] = 'G', ['h'] = 'H',
337 + ['i'] = 'I', ['j'] = 'J', ['k'] = 'K', ['l'] = 'L', ['m'] = 'M', ['n'] = 'N', ['o'] = 'O', ['p'] = 'P',
338 + ['q'] = 'Q', ['r'] = 'R', ['s'] = 'S', ['t'] = 'T', ['u'] = 'U', ['v'] = 'V', ['w'] = 'W', ['x'] = 'X',
339 + ['y'] = 'Y', ['z'] = 'Z',
340 +
341 + // symbols
342 + ['{'] = '_', ['|'] = '_', ['}'] = '_', ['~'] = '_', [127] = '_', // Delete (DEL)
343 +
344 + // Extended ASCII characters (128-255) set to underscore
345 + [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_', [135] = '_',
346 + [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_', [143] = '_',
347 + [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_', [151] = '_',
348 + [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_', [159] = '_',
349 + [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_', [167] = '_',
350 + [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_', [175] = '_',
351 + [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_', [183] = '_',
352 + [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_', [191] = '_',
353 + [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_', [199] = '_',
354 + [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_', [207] = '_',
355 + [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_', [215] = '_',
356 + [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_', [223] = '_',
357 + [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_', [231] = '_',
358 + [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_', [239] = '_',
359 + [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_', [247] = '_',
360 + [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_', [255] = '_',
361 + };
362 +
363 + if (!json_expect_char_after_white_space(js, "\""))
364 + return false;
365 +
366 + if(js->depth >= JSON_DEPTH_MAX - 1) {
367 + snprintf(js->msg, sizeof(js->msg),
368 + "JSON PARSER: object too deep, at pos %zu", js->pos);
369 + return false;
370 + }
371 +
372 + json_consume_char(js);
373 +
374 + char *d = js->key_stack[js->depth];
375 + if(js->depth)
376 + *d++ = '_';
377 +
378 + size_t remaining = sizeof(js->key) - (d - js->key);
379 +
380 + const char *s = json_current_pos(js);
381 + char last_c = '\0';
382 + while(*s && *s != '\"') {
383 + char c;
384 +
385 + if (*s == '\\') {
386 + s++;
387 + c = (char)((*s == 'u') ? '_' : valid_journal_key_chars[(unsigned char)*s]);
388 + s += (*s == 'u') ? 5 : 1;
389 + }
390 + else
391 + c = valid_journal_key_chars[(unsigned char)*s++];
392 +
393 + if(c == '_' && last_c == '_')
394 + continue;
395 + else {
396 + if(remaining < 2) {
397 + snprintf(js->msg, sizeof(js->msg),
398 + "JSON PARSER: key buffer full - keys are too long, at pos %zu", js->pos);
399 + return false;
400 + }
401 + *d++ = c;
402 + remaining--;
403 + }
404 +
405 + last_c = c;
406 + }
407 + *d = '\0';
408 + js->pos += s - json_current_pos(js);
409 +
410 + if (!json_expect_char_after_white_space(js, "\""))
411 + return false;
412 +
413 + json_consume_char(js);
414 +
415 + js->key_stack[++js->depth] = d;
416 +
417 + return true;
418 +}
419 +
420 +static inline bool json_key_pop(LOG_JSON_STATE *js) {
421 + if(js->depth <= 0) {
422 + snprintf(js->msg, sizeof(js->msg),
423 + "JSON PARSER: cannot pop a key at depth %zu, at pos %zu", js->depth, js->pos);
424 + return false;
425 + }
426 +
427 + char *k = js->key_stack[js->depth--];
428 + *k = '\0';
429 + return true;
430 +}
431 +
432 +static inline bool json_parse_value(LOG_JSON_STATE *js) {
433 + if(!json_expect_char_after_white_space(js, "-.0123456789tfn\"{["))
434 + return false;
435 +
436 + const char *s = json_current_pos(js);
437 + switch(*s) {
438 + case '-':
439 + case '0':
440 + case '1':
441 + case '2':
442 + case '3':
443 + case '4':
444 + case '5':
445 + case '6':
446 + case '7':
447 + case '8':
448 + case '9':
449 + return json_parse_number(js);
450 +
451 + case 't':
452 + return json_parse_true(js);
453 +
454 + case 'f':
455 + return json_parse_false(js);
456 +
457 + case 'n':
458 + return json_parse_null(js);
459 +
460 + case '"':
461 + return json_parse_string(js);
462 +
463 + case '{':
464 + return json_parse_object(js);
465 +
466 + case '[':
467 + return json_parse_array(js);
468 + }
469 +
470 + snprintf(js->msg, sizeof(js->msg),
471 + "JSON PARSER: unexpected character at pos %zu", js->pos);
472 + return false;
473 +}
474 +
475 +static inline bool json_key_index_and_push(LOG_JSON_STATE *js, size_t index) {
476 + char *d = js->key_stack[js->depth];
477 + if(js->depth > 0) {
478 + *d++ = '_';
479 + }
480 +
481 + // Convert index to string manually
482 + char temp[32];
483 + char *t = temp + sizeof(temp) - 1; // Start at the end of the buffer
484 + *t = '\0';
485 +
486 + do {
487 + *--t = (char)((index % 10) + '0');
488 + index /= 10;
489 + } while (index > 0);
490 +
491 + size_t remaining = sizeof(js->key) - (d - js->key);
492 +
493 + // Append the index to the key
494 + while (*t) {
495 + if(remaining < 2) {
496 + snprintf(js->msg, sizeof(js->msg),
497 + "JSON PARSER: key buffer full - keys are too long, at pos %zu", js->pos);
498 + return false;
499 + }
500 +
501 + *d++ = *t++;
502 + remaining--;
503 + }
504 +
505 + *d = '\0'; // Null-terminate the key
506 + js->key_stack[++js->depth] = d;
507 +
508 + return true;
509 +}
510 +
511 +static inline bool json_parse_array(LOG_JSON_STATE *js) {
512 + if(!json_expect_char_after_white_space(js, "["))
513 + return false;
514 +
515 + json_consume_char(js);
516 +
517 + size_t index = 0;
518 + do {
519 + if(!json_key_index_and_push(js, index))
520 + return false;
521 +
522 + if(!json_parse_value(js))
523 + return false;
524 +
525 + json_key_pop(js);
526 +
527 + if(!json_expect_char_after_white_space(js, ",]"))
528 + return false;
529 +
530 + const char *s = json_current_pos(js);
531 + json_consume_char(js);
532 + if(*s == ',') {
533 + index++;
534 + continue;
535 + }
536 + else // }
537 + break;
538 +
539 + } while(true);
540 +
541 + return true;
542 +}
543 +
544 +static inline bool json_parse_object(LOG_JSON_STATE *js) {
545 + if(!json_expect_char_after_white_space(js, "{"))
546 + return false;
547 +
548 + json_consume_char(js);
549 +
550 + do {
551 + if (!json_expect_char_after_white_space(js, "\""))
552 + return false;
553 +
554 + if(!json_parse_key_and_push(js))
555 + return false;
556 +
557 + if(!json_expect_char_after_white_space(js, ":"))
558 + return false;
559 +
560 + json_consume_char(js);
561 +
562 + if(!json_parse_value(js))
563 + return false;
564 +
565 + json_key_pop(js);
566 +
567 + if(!json_expect_char_after_white_space(js, ",}"))
568 + return false;
569 +
570 + const char *s = json_current_pos(js);
571 + json_consume_char(js);
572 + if(*s == ',')
573 + continue;
574 + else // }
575 + break;
576 +
577 + } while(true);
578 +
579 + return true;
580 +}
581 +
582 +LOG_JSON_STATE *json_parser_create(struct log_job *jb) {
583 + LOG_JSON_STATE *js = mallocz(sizeof(LOG_JSON_STATE));
584 + memset(js, 0, sizeof(LOG_JSON_STATE));
585 + js->jb = jb;
586 +
587 + if(jb->prefix)
588 + copy_to_buffer(js->key, sizeof(js->key), js->jb->prefix, strlen(js->jb->prefix));
589 +
590 + js->key_stack[0] = &js->key[strlen(js->key)];
591 +
592 + return js;
593 +}
594 +
595 +void json_parser_destroy(LOG_JSON_STATE *js) {
596 + if(js)
597 + freez(js);
598 +}
599 +
600 +const char *json_parser_error(LOG_JSON_STATE *js) {
601 + return js->msg;
602 +}
603 +
604 +bool json_parse_document(LOG_JSON_STATE *js, const char *txt) {
605 + js->line = txt;
606 + js->pos = 0;
607 + js->msg[0] = '\0';
608 + js->key_stack[0][0] = '\0';
609 + js->depth = 0;
610 +
611 + if(!json_parse_object(js))
612 + return false;
613 +
614 + json_skip_spaces(js);
615 + const char *s = json_current_pos(js);
616 +
617 + if(*s) {
618 + snprintf(js->msg, sizeof(js->msg),
619 + "JSON PARSER: excess characters found after document is finished, at pos %zu", js->pos);
620 + return false;
621 + }
622 +
623 + return true;
624 +}
625 +
626 +void json_test(void) {
627 + struct log_job jb = { .prefix = "NIGNX_" };
628 + LOG_JSON_STATE *json = json_parser_create(&jb);
629 +
630 + json_parse_document(json, "{\"value\":\"\\u\\u039A\\u03B1\\u03BB\\u03B7\\u03BC\\u03AD\\u03C1\\u03B1\"}");
631 +
632 + json_parser_destroy(json);
633 +}
collectors/log2journal/log2journal-logfmt.c new
+268
@@ -0,0 +1,268 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "log2journal.h"
4 +
5 +#define ERROR_LINE_MAX 1024
6 +#define KEY_MAX 1024
7 +
8 +struct logfmt_state {
9 + const char *line;
10 + size_t pos;
11 + char msg[ERROR_LINE_MAX];
12 +
13 + char key[KEY_MAX];
14 + size_t key_start;
15 +
16 + struct log_job *jb;
17 +};
18 +
19 +#define logfmt_current_pos(lfs) &(lfs)->line[(lfs)->pos]
20 +#define logfmt_consume_char(lfs) ++(lfs)->pos
21 +
22 +static inline void logfmt_process_key_value(LOGFMT_STATE *lfs, const char *value, size_t len) {
23 + jb_send_extracted_key_value(lfs->jb, lfs->key, value, len);
24 +}
25 +
26 +static inline void logfmt_skip_spaces(LOGFMT_STATE *lfs) {
27 + const char *s = logfmt_current_pos(lfs);
28 + const char *start = s;
29 +
30 + while(isspace(*s)) s++;
31 +
32 + lfs->pos += s - start;
33 +}
34 +
35 +static inline bool logftm_parse_value(LOGFMT_STATE *lfs) {
36 + static __thread char value[MAX_VALUE_LEN];
37 +
38 + char quote = '\0';
39 + const char *s = logfmt_current_pos(lfs);
40 + if(*s == '\"' || *s == '\'') {
41 + quote = *s;
42 + logfmt_consume_char(lfs);
43 + }
44 +
45 + value[0] = '\0';
46 + char *d = value;
47 + s = logfmt_current_pos(lfs);
48 + size_t remaining = sizeof(value);
49 +
50 + char end_char = (char)(quote == '\0' ? ' ' : quote);
51 + while (*s && *s != end_char) {
52 + char c;
53 +
54 + if (*s == '\\') {
55 + s++;
56 +
57 + switch (*s) {
58 + case 'n':
59 + c = '\n';
60 + s++;
61 + break;
62 + case 't':
63 + c = '\t';
64 + s++;
65 + break;
66 + case 'b':
67 + c = '\b';
68 + s++;
69 + break;
70 + case 'f':
71 + c = '\f';
72 + s++;
73 + break;
74 + case 'r':
75 + c = '\r';
76 + s++;
77 + break;
78 + default:
79 + c = *s++;
80 + break;
81 + }
82 + }
83 + else
84 + c = *s++;
85 +
86 + if(remaining < 2) {
87 + snprintf(lfs->msg, sizeof(lfs->msg),
88 + "LOGFMT PARSER: truncated string value at pos %zu", lfs->pos);
89 + return false;
90 + }
91 + else {
92 + *d++ = c;
93 + remaining--;
94 + }
95 + }
96 + *d = '\0';
97 + lfs->pos += s - logfmt_current_pos(lfs);
98 +
99 + s = logfmt_current_pos(lfs);
100 +
101 + if(quote != '\0') {
102 + if (*s != quote) {
103 + snprintf(lfs->msg, sizeof(lfs->msg),
104 + "LOGFMT PARSER: missing quote at pos %zu: '%s'",
105 + lfs->pos, s);
106 + return false;
107 + }
108 + else
109 + logfmt_consume_char(lfs);
110 + }
111 +
112 + if(d > value)
113 + logfmt_process_key_value(lfs, value, d - value);
114 +
115 + return true;
116 +}
117 +
118 +static inline bool logfmt_parse_key(LOGFMT_STATE *lfs) {
119 + static const char valid_journal_key_chars[256] = {
120 + // control characters
121 + [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_',
122 + [8] = '_', [9] = '_', [10] = '_', [11] = '_', [12] = '_', [13] = '_', [14] = '_', [15] = '_',
123 + [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_', [22] = '_', [23] = '_',
124 + [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_', [30] = '_', [31] = '_',
125 +
126 + // symbols
127 + [' '] = '_', ['!'] = '_', ['"'] = '_', ['#'] = '_', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_',
128 + ['('] = '_', [')'] = '_', ['*'] = '_', ['+'] = '_', [','] = '_', ['-'] = '_', ['.'] = '_', ['/'] = '_',
129 +
130 + // numbers
131 + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7',
132 + ['8'] = '8', ['9'] = '9',
133 +
134 + // symbols
135 + [':'] = '_', [';'] = '_', ['<'] = '_', ['='] = '_', ['>'] = '_', ['?'] = '_', ['@'] = '_',
136 +
137 + // capitals
138 + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H',
139 + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P',
140 + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X',
141 + ['Y'] = 'Y', ['Z'] = 'Z',
142 +
143 + // symbols
144 + ['['] = '_', ['\\'] = '_', [']'] = '_', ['^'] = '_', ['_'] = '_', ['`'] = '_',
145 +
146 + // lower to upper
147 + ['a'] = 'A', ['b'] = 'B', ['c'] = 'C', ['d'] = 'D', ['e'] = 'E', ['f'] = 'F', ['g'] = 'G', ['h'] = 'H',
148 + ['i'] = 'I', ['j'] = 'J', ['k'] = 'K', ['l'] = 'L', ['m'] = 'M', ['n'] = 'N', ['o'] = 'O', ['p'] = 'P',
149 + ['q'] = 'Q', ['r'] = 'R', ['s'] = 'S', ['t'] = 'T', ['u'] = 'U', ['v'] = 'V', ['w'] = 'W', ['x'] = 'X',
150 + ['y'] = 'Y', ['z'] = 'Z',
151 +
152 + // symbols
153 + ['{'] = '_', ['|'] = '_', ['}'] = '_', ['~'] = '_', [127] = '_', // Delete (DEL)
154 +
155 + // Extended ASCII characters (128-255) set to underscore
156 + [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_', [135] = '_',
157 + [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_', [143] = '_',
158 + [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_', [151] = '_',
159 + [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_', [159] = '_',
160 + [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_', [167] = '_',
161 + [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_', [175] = '_',
162 + [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_', [183] = '_',
163 + [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_', [191] = '_',
164 + [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_', [199] = '_',
165 + [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_', [207] = '_',
166 + [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_', [215] = '_',
167 + [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_', [223] = '_',
168 + [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_', [231] = '_',
169 + [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_', [239] = '_',
170 + [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_', [247] = '_',
171 + [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_', [255] = '_',
172 + };
173 +
174 + logfmt_skip_spaces(lfs);
175 +
176 + char *d = &lfs->key[lfs->key_start];
177 +
178 + size_t remaining = sizeof(lfs->key) - (d - lfs->key);
179 +
180 + const char *s = logfmt_current_pos(lfs);
181 + char last_c = '\0';
182 + while(*s && *s != '=') {
183 + char c;
184 +
185 + if (*s == '\\')
186 + s++;
187 +
188 + c = valid_journal_key_chars[(unsigned char)*s++];
189 +
190 + if(c == '_' && last_c == '_')
191 + continue;
192 + else {
193 + if(remaining < 2) {
194 + snprintf(lfs->msg, sizeof(lfs->msg),
195 + "LOGFMT PARSER: key buffer full - keys are too long, at pos %zu", lfs->pos);
196 + return false;
197 + }
198 + *d++ = c;
199 + remaining--;
200 + }
201 +
202 + last_c = c;
203 + }
204 + *d = '\0';
205 + lfs->pos += s - logfmt_current_pos(lfs);
206 +
207 + s = logfmt_current_pos(lfs);
208 + if(*s != '=') {
209 + snprintf(lfs->msg, sizeof(lfs->msg),
210 + "LOGFMT PARSER: key is missing the equal sign, at pos %zu", lfs->pos);
211 + return false;
212 + }
213 +
214 + logfmt_consume_char(lfs);
215 +
216 + return true;
217 +}
218 +
219 +LOGFMT_STATE *logfmt_parser_create(struct log_job *jb) {
220 + LOGFMT_STATE *lfs = mallocz(sizeof(LOGFMT_STATE));
221 + memset(lfs, 0, sizeof(LOGFMT_STATE));
222 + lfs->jb = jb;
223 +
224 + if(jb->prefix)
225 + lfs->key_start = copy_to_buffer(lfs->key, sizeof(lfs->key), lfs->jb->prefix, strlen(lfs->jb->prefix));
226 +
227 + return lfs;
228 +}
229 +
230 +void logfmt_parser_destroy(LOGFMT_STATE *lfs) {
231 + if(lfs)
232 + freez(lfs);
233 +}
234 +
235 +const char *logfmt_parser_error(LOGFMT_STATE *lfs) {
236 + return lfs->msg;
237 +}
238 +
239 +bool logfmt_parse_document(LOGFMT_STATE *lfs, const char *txt) {
240 + lfs->line = txt;
241 + lfs->pos = 0;
242 + lfs->msg[0] = '\0';
243 +
244 + const char *s;
245 + do {
246 + if(!logfmt_parse_key(lfs))
247 + return false;
248 +
249 + if(!logftm_parse_value(lfs))
250 + return false;
251 +
252 + logfmt_skip_spaces(lfs);
253 +
254 + s = logfmt_current_pos(lfs);
255 + } while(*s);
256 +
257 + return true;
258 +}
259 +
260 +
261 +void logfmt_test(void) {
262 + struct log_job jb = { .prefix = "NIGNX_" };
263 + LOGFMT_STATE *logfmt = logfmt_parser_create(&jb);
264 +
265 + logfmt_parse_document(logfmt, "x=1 y=2 z=\"3 \\ 4\" 5 ");
266 +
267 + logfmt_parser_destroy(logfmt);
268 +}
collectors/log2journal/log2journal-params.c new
+519
@@ -0,0 +1,519 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "log2journal.h"
4 +
5 +static bool parse_replacement_pattern(struct key_rewrite *rw);
6 +
7 +// ----------------------------------------------------------------------------
8 +
9 +void nd_log_destroy(struct log_job *jb) {
10 + for(size_t i = 0; i < jb->injections.used ;i++) {
11 + if(jb->injections.keys[i].value.s)
12 + freez(jb->injections.keys[i].value.s);
13 + }
14 +
15 + for(size_t i = 0; i < jb->unmatched.injections.used ;i++) {
16 + if(jb->unmatched.injections.keys[i].value.s)
17 + freez(jb->unmatched.injections.keys[i].value.s);
18 + }
19 +
20 + for(size_t i = 0; i < jb->dups.used ;i++) {
21 + struct key_dup *kd = &jb->dups.array[i];
22 +
23 + if(kd->target)
24 + freez(kd->target);
25 +
26 + for(size_t j = 0; j < kd->used ; j++) {
27 + if (kd->keys[j])
28 + freez(kd->keys[j]);
29 +
30 + if (kd->values[j].s)
31 + freez(kd->values[j].s);
32 + }
33 + }
34 +
35 + for(size_t i = 0; i < jb->rewrites.used; i++) {
36 + struct key_rewrite *rw = &jb->rewrites.array[i];
37 +
38 + if (rw->key)
39 + freez(rw->key);
40 +
41 + if (rw->search_pattern)
42 + freez(rw->search_pattern);
43 +
44 + if (rw->replace_pattern)
45 + freez(rw->replace_pattern);
46 +
47 + if(rw->match_data)
48 + pcre2_match_data_free(rw->match_data);
49 +
50 + if (rw->re)
51 + pcre2_code_free(rw->re);
52 +
53 + // Cleanup for replacement nodes linked list
54 + struct replacement_node *current = rw->nodes;
55 + while (current != NULL) {
56 + struct replacement_node *next = current->next;
57 +
58 + if (current->s)
59 + freez((void *)current->s);
60 +
61 + freez(current);
62 + current = next;
63 + }
64 + }
65 +
66 + memset(jb, 0, sizeof(*jb));
67 +}
68 +
69 +// ----------------------------------------------------------------------------
70 +
71 +bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len) {
72 + if(!key || !*key) {
73 + log2stderr("filename key cannot be empty.");
74 + return false;
75 + }
76 +
77 + if(jb->filename.key)
78 + freez((char*)jb->filename.key);
79 +
80 + jb->filename.key = strndupz(key, key_len);
81 +
82 + return true;
83 +}
84 +
85 +bool log_job_add_key_prefix(struct log_job *jb, const char *prefix, size_t prefix_len) {
86 + if(!prefix || !*prefix) {
87 + log2stderr("filename key cannot be empty.");
88 + return false;
89 + }
90 +
91 + if(jb->prefix)
92 + freez((char*)jb->prefix);
93 +
94 + jb->prefix = strndupz(prefix, prefix_len);
95 +
96 + return true;
97 +}
98 +
99 +bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {
100 + if (unmatched) {
101 + if (jb->unmatched.injections.used >= MAX_INJECTIONS) {
102 + log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);
103 + return false;
104 + }
105 + }
106 + else {
107 + if (jb->injections.used >= MAX_INJECTIONS) {
108 + log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);
109 + return false;
110 + }
111 + }
112 +
113 + if (unmatched) {
114 + key_value_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],
115 + key, key_len,
116 + value, value_len);
117 + } else {
118 + key_value_replace(&jb->injections.keys[jb->injections.used++],
119 + key, key_len,
120 + value, value_len);
121 + }
122 +
123 + return true;
124 +}
125 +
126 +bool log_job_add_rename(struct log_job *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len) {
127 + if(jb->renames.used >= MAX_RENAMES) {
128 + log2stderr("Error: too many renames. You can rename up to %d fields.", MAX_RENAMES);
129 + return false;
130 + }
131 +
132 + struct key_rename *rn = &jb->renames.array[jb->renames.used++];
133 + rn->new_key = strndupz(new_key, new_key_len);
134 + rn->new_hash = XXH3_64bits(rn->new_key, strlen(rn->new_key));
135 + rn->old_key = strndupz(old_key, old_key_len);
136 + rn->old_hash = XXH3_64bits(rn->old_key, strlen(rn->old_key));
137 +
138 + return true;
139 +}
140 +
141 +bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern) {
142 + if(jb->rewrites.used >= MAX_REWRITES) {
143 + log2stderr("Error: too many rewrites. You can add up to %d rewrite rules.", MAX_REWRITES);
144 + return false;
145 + }
146 +
147 + pcre2_code *re = jb_compile_pcre2_pattern(search_pattern);
148 + if (!re) {
149 + return false;
150 + }
151 +
152 + struct key_rewrite *rw = &jb->rewrites.array[jb->rewrites.used++];
153 + rw->key = strdupz(key);
154 + rw->hash = XXH3_64bits(rw->key, strlen(rw->key));
155 + rw->search_pattern = strdupz(search_pattern);
156 + rw->replace_pattern = strdupz(replace_pattern);
157 + rw->re = re;
158 + rw->match_data = pcre2_match_data_create_from_pattern(rw->re, NULL);
159 +
160 + // Parse the replacement pattern and create the linked list
161 + if (!parse_replacement_pattern(rw)) {
162 + pcre2_match_data_free(rw->match_data);
163 + pcre2_code_free(rw->re);
164 + freez(rw->key);
165 + freez(rw->search_pattern);
166 + freez(rw->replace_pattern);
167 + jb->rewrites.used--;
168 + return false;
169 + }
170 +
171 + return true;
172 +}
173 +
174 +// ----------------------------------------------------------------------------
175 +
176 +struct key_dup *log_job_add_duplication_to_job(struct log_job *jb, const char *target, size_t target_len) {
177 + if (jb->dups.used >= MAX_KEY_DUPS) {
178 + log2stderr("Error: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS);
179 + return NULL;
180 + }
181 +
182 + struct key_dup *kd = &jb->dups.array[jb->dups.used++];
183 + kd->target = strndupz(target, target_len);
184 + kd->hash = XXH3_64bits(kd->target, target_len);
185 + kd->used = 0;
186 + kd->exposed = false;
187 +
188 + // Initialize values array
189 + for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) {
190 + kd->values[i].s = NULL;
191 + kd->values[i].size = 0;
192 + }
193 +
194 + return kd;
195 +}
196 +
197 +bool log_job_add_key_to_duplication(struct key_dup *kd, const char *key, size_t key_len) {
198 + if (kd->used >= MAX_KEY_DUPS_KEYS) {
199 + log2stderr("Error: Too many keys in duplication of target '%s'.", kd->target);
200 + return false;
201 + }
202 +
203 + kd->keys[kd->used++] = strndupz(key, key_len);
204 + return true;
205 +}
206 +
207 +// ----------------------------------------------------------------------------
208 +// command line params
209 +
210 +struct replacement_node *add_replacement_node(struct replacement_node **head, bool is_variable, const char *text) {
211 + struct replacement_node *new_node = mallocz(sizeof(struct replacement_node));
212 + if (!new_node)
213 + return NULL;
214 +
215 + new_node->is_variable = is_variable;
216 + new_node->s = text;
217 + new_node->len = strlen(text);
218 + new_node->next = NULL;
219 +
220 + if (*head == NULL)
221 + *head = new_node;
222 +
223 + else {
224 + struct replacement_node *current = *head;
225 +
226 + // append it
227 + while (current->next != NULL)
228 + current = current->next;
229 +
230 + current->next = new_node;
231 + }
232 +
233 + return new_node;
234 +}
235 +
236 +static bool parse_replacement_pattern(struct key_rewrite *rw) {
237 + const char *current = rw->replace_pattern;
238 +
239 + while (*current != '\0') {
240 + if (*current == '$' && *(current + 1) == '{') {
241 + // Start of a variable
242 + const char *end = strchr(current, '}');
243 + if (!end) {
244 + log2stderr("Error: Missing closing brace in replacement pattern: %s", rw->replace_pattern);
245 + return false;
246 + }
247 +
248 + size_t name_length = end - current - 2; // Length of the variable name
249 + char *variable_name = strndupz(current + 2, name_length);
250 + if (!variable_name) {
251 + log2stderr("Error: Memory allocation failed for variable name.");
252 + return false;
253 + }
254 +
255 + struct replacement_node *node = add_replacement_node(&(rw->nodes), true, variable_name);
256 + if (!node) {
257 + freez(variable_name);
258 + log2stderr("Error: Failed to add replacement node for variable.");
259 + return false;
260 + }
261 +
262 + current = end + 1; // Move past the variable
263 + }
264 + else {
265 + // Start of literal text
266 + const char *start = current;
267 + while (*current != '\0' && !(*current == '$' && *(current + 1) == '{')) {
268 + current++;
269 + }
270 +
271 + size_t text_length = current - start;
272 + char *text = strndupz(start, text_length);
273 + if (!text) {
274 + log2stderr("Error: Memory allocation failed for literal text.");
275 + return false;
276 + }
277 +
278 + struct replacement_node *node = add_replacement_node(&(rw->nodes), false, text);
279 + if (!node) {
280 + freez(text);
281 + log2stderr("Error: Failed to add replacement node for text.");
282 + return false;
283 + }
284 + }
285 + }
286 +
287 + return true;
288 +}
289 +
290 +static bool parse_rename(struct log_job *jb, const char *param) {
291 + // Search for '=' in param
292 + const char *equal_sign = strchr(param, '=');
293 + if (!equal_sign || equal_sign == param) {
294 + log2stderr("Error: Invalid rename format, '=' not found in %s", param);
295 + return false;
296 + }
297 +
298 + const char *new_key = param;
299 + size_t new_key_len = equal_sign - new_key;
300 +
301 + const char *old_key = equal_sign + 1;
302 + size_t old_key_len = strlen(old_key);
303 +
304 + return log_job_add_rename(jb, new_key, new_key_len, old_key, old_key_len);
305 +}
306 +
307 +static bool is_symbol(char c) {
308 + return !isalpha(c) && !isdigit(c) && !iscntrl(c);
309 +}
310 +
311 +static bool parse_rewrite(struct log_job *jb, const char *param) {
312 + // Search for '=' in param
313 + const char *equal_sign = strchr(param, '=');
314 + if (!equal_sign || equal_sign == param) {
315 + log2stderr("Error: Invalid rewrite format, '=' not found in %s", param);
316 + return false;
317 + }
318 +
319 + // Get the next character as the separator
320 + char separator = *(equal_sign + 1);
321 + if (!separator || !is_symbol(separator)) {
322 + log2stderr("Error: rewrite separator not found after '=', or is not one of /\\|-# in: %s", param);
323 + return false;
324 + }
325 +
326 + // Find the next occurrence of the separator
327 + const char *second_separator = strchr(equal_sign + 2, separator);
328 + if (!second_separator) {
329 + log2stderr("Error: rewrite second separator not found in: %s", param);
330 + return false;
331 + }
332 +
333 + // Check if the search pattern is empty
334 + if (equal_sign + 1 == second_separator) {
335 + log2stderr("Error: rewrite search pattern is empty in: %s", param);
336 + return false;
337 + }
338 +
339 + // Check if the replacement pattern is empty
340 + if (*(second_separator + 1) == '\0') {
341 + log2stderr("Error: rewrite replacement pattern is empty in: %s", param);
342 + return false;
343 + }
344 +
345 + // Reserve a slot in rewrites
346 + if (jb->rewrites.used >= MAX_REWRITES) {
347 + log2stderr("Error: Exceeded maximum number of rewrite rules, while processing: %s", param);
348 + return false;
349 + }
350 +
351 + // Extract key, search pattern, and replacement pattern
352 + char *key = strndupz(param, equal_sign - param);
353 + char *search_pattern = strndupz(equal_sign + 2, second_separator - (equal_sign + 2));
354 + char *replace_pattern = strdupz(second_separator + 1);
355 +
356 + bool ret = log_job_add_rewrite(jb, key, search_pattern, replace_pattern);
357 +
358 + freez(key);
359 + freez(search_pattern);
360 + freez(replace_pattern);
361 +
362 + return ret;
363 +}
364 +
365 +static bool parse_inject(struct log_job *jb, const char *value, bool unmatched) {
366 + const char *equal = strchr(value, '=');
367 + if (!equal) {
368 + log2stderr("Error: injection '%s' does not have an equal sign.", value);
369 + return false;
370 + }
371 +
372 + const char *key = value;
373 + const char *val = equal + 1;
374 + log_job_add_injection(jb, key, equal - key, val, strlen(val), unmatched);
375 +
376 + return true;
377 +}
378 +
379 +static bool parse_duplicate(struct log_job *jb, const char *value) {
380 + const char *target = value;
381 + const char *equal_sign = strchr(value, '=');
382 + if (!equal_sign || equal_sign == target) {
383 + log2stderr("Error: Invalid duplicate format, '=' not found or at the start in %s", value);
384 + return false;
385 + }
386 +
387 + size_t target_len = equal_sign - target;
388 + struct key_dup *kd = log_job_add_duplication_to_job(jb, target, target_len);
389 + if(!kd) return false;
390 +
391 + const char *key = equal_sign + 1;
392 + while (key) {
393 + if (kd->used >= MAX_KEY_DUPS_KEYS) {
394 + log2stderr("Error: too many keys in duplication of target '%s'.", kd->target);
395 + return false;
396 + }
397 +
398 + const char *comma = strchr(key, ',');
399 + size_t key_len;
400 + if (comma) {
401 + key_len = comma - key;
402 + log_job_add_key_to_duplication(kd, key, key_len);
403 + key = comma + 1;
404 + }
405 + else {
406 + log_job_add_key_to_duplication(kd, key, strlen(key));
407 + break; // No more keys
408 + }
409 + }
410 +
411 + return true;
412 +}
413 +
414 +bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv) {
415 + for (int i = 1; i < argc; i++) {
416 + char *arg = argv[i];
417 + if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) {
418 + log2journal_command_line_help(argv[0]);
419 + exit(0);
420 + }
421 +#if defined(NETDATA_DEV_MODE) || defined(NETDATA_INTERNAL_CHECKS)
422 + else if(strcmp(arg, "--test") == 0) {
423 + // logfmt_test();
424 + json_test();
425 + exit(1);
426 + }
427 +#endif
428 + else if (strcmp(arg, "--show-config") == 0) {
429 + jb->show_config = true;
430 + }
431 + else {
432 + char buffer[1024];
433 + char *param = NULL;
434 + char *value = NULL;
435 +
436 + char *equal_sign = strchr(arg, '=');
437 + if (equal_sign) {
438 + copy_to_buffer(buffer, sizeof(buffer), arg, equal_sign - arg);
439 + param = buffer;
440 + value = equal_sign + 1;
441 + }
442 + else {
443 + param = arg;
444 + if (i + 1 < argc) {
445 + value = argv[++i];
446 + }
447 + else {
448 + if (!jb->pattern) {
449 + jb->pattern = arg;
450 + continue;
451 + } else {
452 + log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
453 + return false;
454 + }
455 + }
456 + }
457 +
458 + if (strcmp(param, "--filename-key") == 0) {
459 + if(!log_job_add_filename_key(jb, value, value ? strlen(value) : 0))
460 + return false;
461 + }
462 + if (strcmp(param, "--prefix") == 0) {
463 + if(!log_job_add_key_prefix(jb, value, value ? strlen(value) : 0))
464 + return false;
465 + }
466 +#ifdef HAVE_LIBYAML
467 + else if (strcmp(param, "-f") == 0 || strcmp(param, "--file") == 0) {
468 + if (!yaml_parse_file(value, jb))
469 + return false;
470 + }
471 + else if (strcmp(param, "--config") == 0) {
472 + if (!yaml_parse_config(value, jb))
473 + return false;
474 + }
475 +#endif
476 + else if (strcmp(param, "--unmatched-key") == 0)
477 + jb->unmatched.key = value;
478 + else if (strcmp(param, "--duplicate") == 0) {
479 + if (!parse_duplicate(jb, value))
480 + return false;
481 + }
482 + else if (strcmp(param, "--inject") == 0) {
483 + if (!parse_inject(jb, value, false))
484 + return false;
485 + }
486 + else if (strcmp(param, "--inject-unmatched") == 0) {
487 + if (!parse_inject(jb, value, true))
488 + return false;
489 + }
490 + else if (strcmp(param, "--rewrite") == 0) {
491 + if (!parse_rewrite(jb, value))
492 + return false;
493 + }
494 + else if (strcmp(param, "--rename") == 0) {
495 + if (!parse_rename(jb, value))
496 + return false;
497 + }
498 + else {
499 + i--;
500 + if (!jb->pattern) {
501 + jb->pattern = arg;
502 + continue;
503 + } else {
504 + log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
505 + return false;
506 + }
507 + }
508 + }
509 + }
510 +
511 + // Check if a pattern is set and exactly one pattern is specified
512 + if (!jb->pattern) {
513 + log2stderr("Error: Pattern not specified.");
514 + log2journal_command_line_help(argv[0]);
515 + return false;
516 + }
517 +
518 + return true;
519 +}
collectors/log2journal/log2journal-yaml.c new
+838
@@ -0,0 +1,838 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "log2journal.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// yaml configuration file
7 +
8 +#ifdef HAVE_LIBYAML
9 +
10 +static const char *yaml_event_name(yaml_event_type_t type) {
11 + switch (type) {
12 + case YAML_NO_EVENT:
13 + return "YAML_NO_EVENT";
14 +
15 + case YAML_SCALAR_EVENT:
16 + return "YAML_SCALAR_EVENT";
17 +
18 + case YAML_ALIAS_EVENT:
19 + return "YAML_ALIAS_EVENT";
20 +
21 + case YAML_MAPPING_START_EVENT:
22 + return "YAML_MAPPING_START_EVENT";
23 +
24 + case YAML_MAPPING_END_EVENT:
25 + return "YAML_MAPPING_END_EVENT";
26 +
27 + case YAML_SEQUENCE_START_EVENT:
28 + return "YAML_SEQUENCE_START_EVENT";
29 +
30 + case YAML_SEQUENCE_END_EVENT:
31 + return "YAML_SEQUENCE_END_EVENT";
32 +
33 + case YAML_STREAM_START_EVENT:
34 + return "YAML_STREAM_START_EVENT";
35 +
36 + case YAML_STREAM_END_EVENT:
37 + return "YAML_STREAM_END_EVENT";
38 +
39 + case YAML_DOCUMENT_START_EVENT:
40 + return "YAML_DOCUMENT_START_EVENT";
41 +
42 + case YAML_DOCUMENT_END_EVENT:
43 + return "YAML_DOCUMENT_END_EVENT";
44 +
45 + default:
46 + return "UNKNOWN";
47 + }
48 +}
49 +
50 +#define yaml_error(parser, event, fmt, args...) yaml_error_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__, fmt, ##args)
51 +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) __attribute__ ((format(__printf__, 6, 7)));
52 +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) {
53 + char buf[1024] = ""; // Initialize buf to an empty string
54 + const char *type = "";
55 +
56 + if(event) {
57 + type = yaml_event_name(event->type);
58 +
59 + switch (event->type) {
60 + case YAML_SCALAR_EVENT:
61 + copy_to_buffer(buf, sizeof(buf), (char *)event->data.scalar.value, event->data.scalar.length);
62 + break;
63 +
64 + case YAML_ALIAS_EVENT:
65 + snprintf(buf, sizeof(buf), "%s", event->data.alias.anchor);
66 + break;
67 +
68 + default:
69 + break;
70 + }
71 + }
72 +
73 + fprintf(stderr, "YAML %zu@%s, %s(): (line %d, column %d, %s%s%s): ",
74 + line, file, function,
75 + (int)(parser->mark.line + 1), (int)(parser->mark.column + 1),
76 + type, buf[0]? ", near ": "", buf);
77 +
78 + va_list args;
79 + va_start(args, format);
80 + vfprintf(stderr, format, args);
81 + va_end(args);
82 + fprintf(stderr, "\n");
83 +}
84 +
85 +#define yaml_parse(parser, event) yaml_parse_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__)
86 +static bool yaml_parse_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file) {
87 + if (!yaml_parser_parse(parser, event)) {
88 + yaml_error(parser, NULL, "YAML parser error %d", parser->error);
89 + return false;
90 + }
91 +
92 +// fprintf(stderr, ">>> %s >>> %.*s\n",
93 +// yaml_event_name(event->type),
94 +// event->type == YAML_SCALAR_EVENT ? event->data.scalar.length : 0,
95 +// event->type == YAML_SCALAR_EVENT ? (char *)event->data.scalar.value : "");
96 +
97 + return true;
98 +}
99 +
100 +#define yaml_parse_expect_event(parser, type) yaml_parse_expect_event_with_trace(parser, type, __LINE__, __FUNCTION__, __FILE__)
101 +static bool yaml_parse_expect_event_with_trace(yaml_parser_t *parser, yaml_event_type_t type, size_t line, const char *function, const char *file) {
102 + yaml_event_t event;
103 + if (!yaml_parse(parser, &event))
104 + return false;
105 +
106 + bool ret = true;
107 + if(event.type != type) {
108 + yaml_error_with_trace(parser, &event, line, function, file, "unexpected event - expecting: %s", yaml_event_name(type));
109 + ret = false;
110 + }
111 +// else
112 +// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function);
113 +
114 + yaml_event_delete(&event);
115 + return ret;
116 +}
117 +
118 +#define yaml_scalar_matches(event, s, len) yaml_scalar_matches_with_trace(event, s, len, __LINE__, __FUNCTION__, __FILE__)
119 +static bool yaml_scalar_matches_with_trace(yaml_event_t *event, const char *s, size_t len, size_t line __maybe_unused, const char *function __maybe_unused, const char *file __maybe_unused) {
120 + if(event->type != YAML_SCALAR_EVENT)
121 + return false;
122 +
123 + if(len != event->data.scalar.length)
124 + return false;
125 +// else
126 +// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function);
127 +
128 + return strcmp((char *)event->data.scalar.value, s) == 0;
129 +}
130 +
131 +// ----------------------------------------------------------------------------
132 +
133 +static struct key_dup *yaml_parse_duplicate_key(struct log_job *jb, yaml_parser_t *parser) {
134 + yaml_event_t event;
135 +
136 + if (!yaml_parse(parser, &event))
137 + return false;
138 +
139 + struct key_dup *kd = NULL;
140 + if(event.type == YAML_SCALAR_EVENT) {
141 + kd = log_job_add_duplication_to_job(jb, (char *) event.data.scalar.value, event.data.scalar.length);
142 + }
143 + else
144 + yaml_error(parser, &event, "duplicate key must be a scalar.");
145 +
146 + yaml_event_delete(&event);
147 + return kd;
148 +}
149 +
150 +static size_t yaml_parse_duplicate_from(struct log_job *jb, yaml_parser_t *parser, struct key_dup *kd) {
151 + size_t errors = 0;
152 + yaml_event_t event;
153 +
154 + if (!yaml_parse(parser, &event))
155 + return 1;
156 +
157 + bool ret = true;
158 + if(event.type == YAML_SCALAR_EVENT)
159 + ret = log_job_add_key_to_duplication(kd, (char *) event.data.scalar.value, event.data.scalar.length);
160 +
161 + else if(event.type == YAML_SEQUENCE_START_EVENT) {
162 + bool finished = false;
163 + while(!errors && !finished) {
164 + yaml_event_t sub_event;
165 + if (!yaml_parse(parser, &sub_event))
166 + return errors++;
167 + else {
168 + if (sub_event.type == YAML_SCALAR_EVENT)
169 + log_job_add_key_to_duplication(kd, (char *) sub_event.data.scalar.value
170 + , sub_event.data.scalar.length
171 + );
172 +
173 + else if (sub_event.type == YAML_SEQUENCE_END_EVENT)
174 + finished = true;
175 +
176 + yaml_event_delete(&sub_event);
177 + }
178 + }
179 + }
180 + else
181 + yaml_error(parser, &event, "not expected event type");
182 +
183 + yaml_event_delete(&event);
184 + return errors;
185 +}
186 +
187 +static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_job *jb) {
188 + yaml_event_t event;
189 + size_t errors = 0;
190 +
191 + if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))
192 + return 1;
193 +
194 + if (!yaml_parse(parser, &event))
195 + return 1;
196 +
197 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {
198 + yaml_event_t sub_event;
199 + if (!yaml_parse(parser, &sub_event))
200 + errors++;
201 +
202 + else {
203 + if (event.type == YAML_SCALAR_EVENT) {
204 + if(!log_job_add_filename_key(jb, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length))
205 + errors++;
206 + }
207 +
208 + else {
209 + yaml_error(parser, &sub_event, "expected the filename as %s", yaml_event_name(YAML_SCALAR_EVENT));
210 + errors++;
211 + }
212 +
213 + yaml_event_delete(&sub_event);
214 + }
215 + }
216 +
217 + if(!yaml_parse_expect_event(parser, YAML_MAPPING_END_EVENT))
218 + errors++;
219 +
220 + yaml_event_delete(&event);
221 + return errors;
222 +}
223 +
224 +static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, struct log_job *jb) {
225 + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
226 + return 1;
227 +
228 + struct key_dup *kd = NULL;
229 +
230 + // Expecting a key-value pair for each duplicate
231 + bool finished;
232 + size_t errors = 0;
233 + while (!errors && !finished) {
234 + yaml_event_t event;
235 + if (!yaml_parse(parser, &event)) {
236 + errors++;
237 + break;
238 + }
239 +
240 + if(event.type == YAML_MAPPING_START_EVENT) {
241 + ;
242 + }
243 + if (event.type == YAML_SEQUENCE_END_EVENT) {
244 + finished = true;
245 + }
246 + else if(event.type == YAML_SCALAR_EVENT) {
247 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {
248 + kd = yaml_parse_duplicate_key(jb, parser);
249 + if (!kd)
250 + errors++;
251 + else {
252 + while (!errors && kd) {
253 + yaml_event_t sub_event;
254 + if (!yaml_parse(parser, &sub_event)) {
255 + errors++;
256 + break;
257 + }
258 +
259 + if (sub_event.type == YAML_MAPPING_END_EVENT) {
260 + kd = NULL;
261 + } else if (sub_event.type == YAML_SCALAR_EVENT) {
262 + if (yaml_scalar_matches(&sub_event, "values_of", strlen("values_of"))) {
263 + if (!kd) {
264 + yaml_error(parser, &sub_event, "Found 'values_of' but the 'key' is not set.");
265 + errors++;
266 + } else
267 + errors += yaml_parse_duplicate_from(jb, parser, kd);
268 + } else {
269 + yaml_error(parser, &sub_event, "unknown scalar");
270 + errors++;
271 + }
272 + } else {
273 + yaml_error(parser, &sub_event, "unexpected event type");
274 + errors++;
275 + }
276 +
277 + // Delete the event after processing
278 + yaml_event_delete(&event);
279 + }
280 + }
281 + } else {
282 + yaml_error(parser, &event, "unknown scalar");
283 + errors++;
284 + }
285 + }
286 +
287 + yaml_event_delete(&event);
288 + }
289 +
290 + return errors;
291 +}
292 +
293 +static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
294 + yaml_event_t event;
295 + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
296 + yaml_error(parser, &event, "Expected scalar for constant field injection key");
297 + yaml_event_delete(&event);
298 + return false;
299 + }
300 +
301 + char *key = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
302 + char *value = NULL;
303 + bool ret = false;
304 +
305 + yaml_event_delete(&event);
306 +
307 + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
308 + yaml_error(parser, &event, "Expected scalar for constant field injection value");
309 + goto cleanup;
310 + }
311 +
312 + if(!yaml_scalar_matches(&event, "value", strlen("value"))) {
313 + yaml_error(parser, &event, "Expected scalar 'value'");
314 + goto cleanup;
315 + }
316 +
317 + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
318 + yaml_error(parser, &event, "Expected scalar for constant field injection value");
319 + goto cleanup;
320 + }
321 +
322 + value = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
323 +
324 + if(!log_job_add_injection(jb, key, strlen(key), value, strlen(value), unmatched))
325 + ret = false;
326 + else
327 + ret = true;
328 +
329 + ret = true;
330 +
331 +cleanup:
332 + yaml_event_delete(&event);
333 + freez(key);
334 + freez(value);
335 + return !ret ? 1 : 0;
336 +}
337 +
338 +static bool yaml_parse_injection_mapping(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
339 + yaml_event_t event;
340 + size_t errors = 0;
341 + bool finished = false;
342 +
343 + while (!errors && !finished) {
344 + if (!yaml_parse(parser, &event)) {
345 + errors++;
346 + continue;
347 + }
348 +
349 + switch (event.type) {
350 + case YAML_SCALAR_EVENT:
351 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {
352 + errors += yaml_parse_constant_field_injection(parser, jb, unmatched);
353 + } else {
354 + yaml_error(parser, &event, "Unexpected scalar in injection mapping");
355 + errors++;
356 + }
357 + break;
358 +
359 + case YAML_MAPPING_END_EVENT:
360 + finished = true;
361 + break;
362 +
363 + default:
364 + yaml_error(parser, &event, "Unexpected event in injection mapping");
365 + errors++;
366 + break;
367 + }
368 +
369 + yaml_event_delete(&event);
370 + }
371 +
372 + return errors == 0;
373 +}
374 +
375 +static size_t yaml_parse_injections(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
376 + yaml_event_t event;
377 + size_t errors = 0;
378 + bool finished = false;
379 +
380 + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
381 + return 1;
382 +
383 + while (!errors && !finished) {
384 + if (!yaml_parse(parser, &event)) {
385 + errors++;
386 + continue;
387 + }
388 +
389 + switch (event.type) {
390 + case YAML_MAPPING_START_EVENT:
391 + if (!yaml_parse_injection_mapping(parser, jb, unmatched))
392 + errors++;
393 + break;
394 +
395 + case YAML_SEQUENCE_END_EVENT:
396 + finished = true;
397 + break;
398 +
399 + default:
400 + yaml_error(parser, &event, "Unexpected event in injections sequence");
401 + errors++;
402 + break;
403 + }
404 +
405 + yaml_event_delete(&event);
406 + }
407 +
408 + return errors;
409 +}
410 +
411 +static size_t yaml_parse_unmatched(yaml_parser_t *parser, struct log_job *jb) {
412 + size_t errors = 0;
413 + bool finished = false;
414 +
415 + if (!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))
416 + return 1;
417 +
418 + while (!errors && !finished) {
419 + yaml_event_t event;
420 + if (!yaml_parse(parser, &event)) {
421 + errors++;
422 + continue;
423 + }
424 +
425 + switch (event.type) {
426 + case YAML_SCALAR_EVENT:
427 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {
428 + yaml_event_t sub_event;
429 + if (!yaml_parse(parser, &sub_event)) {
430 + errors++;
431 + } else {
432 + if (sub_event.type == YAML_SCALAR_EVENT) {
433 + jb->unmatched.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
434 + } else {
435 + yaml_error(parser, &sub_event, "expected a scalar value for 'key'");
436 + errors++;
437 + }
438 + yaml_event_delete(&sub_event);
439 + }
440 + } else if (yaml_scalar_matches(&event, "inject", strlen("inject"))) {
441 + errors += yaml_parse_injections(parser, jb, true);
442 + } else {
443 + yaml_error(parser, &event, "Unexpected scalar in unmatched section");
444 + errors++;
445 + }
446 + break;
447 +
448 + case YAML_MAPPING_END_EVENT:
449 + finished = true;
450 + break;
451 +
452 + default:
453 + yaml_error(parser, &event, "Unexpected event in unmatched section");
454 + errors++;
455 + break;
456 + }
457 +
458 + yaml_event_delete(&event);
459 + }
460 +
461 + return errors;
462 +}
463 +
464 +static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
465 + size_t errors = 0;
466 +
467 + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
468 + return 1;
469 +
470 + bool finished = false;
471 + while (!errors && !finished) {
472 + yaml_event_t event;
473 + if (!yaml_parse(parser, &event)) {
474 + errors++;
475 + continue;
476 + }
477 +
478 + switch (event.type) {
479 + case YAML_MAPPING_START_EVENT:
480 + {
481 + struct key_rewrite rw = {0};
482 +
483 + bool mapping_finished = false;
484 + while (!errors && !mapping_finished) {
485 + yaml_event_t sub_event;
486 + if (!yaml_parse(parser, &sub_event)) {
487 + errors++;
488 + continue;
489 + }
490 +
491 + switch (sub_event.type) {
492 + case YAML_SCALAR_EVENT:
493 + if (yaml_scalar_matches(&sub_event, "key", strlen("key"))) {
494 + if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
495 + yaml_error(parser, &sub_event, "Expected scalar for rewrite key");
496 + errors++;
497 + } else {
498 + rw.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
499 + yaml_event_delete(&sub_event);
500 + }
501 + } else if (yaml_scalar_matches(&sub_event, "search", strlen("search"))) {
502 + if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
503 + yaml_error(parser, &sub_event, "Expected scalar for rewrite search pattern");
504 + errors++;
505 + } else {
506 + rw.search_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
507 + yaml_event_delete(&sub_event);
508 + }
509 + } else if (yaml_scalar_matches(&sub_event, "replace", strlen("replace"))) {
510 + if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
511 + yaml_error(parser, &sub_event, "Expected scalar for rewrite replace pattern");
512 + errors++;
513 + } else {
514 + rw.replace_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
515 + yaml_event_delete(&sub_event);
516 + }
517 + } else {
518 + yaml_error(parser, &sub_event, "Unexpected scalar in rewrite mapping");
519 + errors++;
520 + }
521 + break;
522 +
523 + case YAML_MAPPING_END_EVENT:
524 + if(rw.key && rw.search_pattern && rw.replace_pattern) {
525 + if (!log_job_add_rewrite(jb, rw.key, rw.search_pattern, rw.replace_pattern))
526 + errors++;
527 + }
528 + freez(rw.key);
529 + freez(rw.search_pattern);
530 + freez(rw.replace_pattern);
531 + memset(&rw, 0, sizeof(rw));
532 +
533 + mapping_finished = true;
534 + break;
535 +
536 + default:
537 + yaml_error(parser, &sub_event, "Unexpected event in rewrite mapping");
538 + errors++;
539 + break;
540 + }
541 +
542 + yaml_event_delete(&sub_event);
543 + }
544 + }
545 + break;
546 +
547 + case YAML_SEQUENCE_END_EVENT:
548 + finished = true;
549 + break;
550 +
551 + default:
552 + yaml_error(parser, &event, "Unexpected event in rewrites sequence");
553 + errors++;
554 + break;
555 + }
556 +
557 + yaml_event_delete(&event);
558 + }
559 +
560 + return errors;
561 +}
562 +
563 +static size_t yaml_parse_pattern(yaml_parser_t *parser, struct log_job *jb) {
564 + yaml_event_t event;
565 + size_t errors = 0;
566 +
567 + if (!yaml_parse(parser, &event))
568 + return 1;
569 +
570 + if(event.type == YAML_SCALAR_EVENT)
571 + jb->pattern = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
572 + else {
573 + yaml_error(parser, &event, "unexpected event type");
574 + errors++;
575 + }
576 +
577 + yaml_event_delete(&event);
578 + return errors;
579 +}
580 +
581 +static size_t yaml_parse_initialized(yaml_parser_t *parser, struct log_job *jb) {
582 + size_t errors = 0;
583 +
584 + if(!yaml_parse_expect_event(parser, YAML_STREAM_START_EVENT)) {
585 + errors++;
586 + goto cleanup;
587 + }
588 +
589 + if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_START_EVENT)) {
590 + errors++;
591 + goto cleanup;
592 + }
593 +
594 + if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT)) {
595 + errors++;
596 + goto cleanup;
597 + }
598 +
599 + bool finished = false;
600 + while (!errors && !finished) {
601 + yaml_event_t event;
602 + if(!yaml_parse(parser, &event)) {
603 + errors++;
604 + continue;
605 + }
606 +
607 + switch(event.type) {
608 + default:
609 + yaml_error(parser, &event, "unexpected type");
610 + errors++;
611 + break;
612 +
613 + case YAML_MAPPING_END_EVENT:
614 + finished = true;
615 + break;
616 +
617 + case YAML_SCALAR_EVENT:
618 + if (yaml_scalar_matches(&event, "pattern", strlen("pattern")))
619 + errors += yaml_parse_pattern(parser, jb);
620 +
621 + else if (yaml_scalar_matches(&event, "filename", strlen("filename")))
622 + errors += yaml_parse_filename_injection(parser, jb);
623 +
624 + else if (yaml_scalar_matches(&event, "duplicate", strlen("duplicate")))
625 + errors += yaml_parse_duplicates_injection(parser, jb);
626 +
627 + else if (yaml_scalar_matches(&event, "inject", strlen("inject")))
628 + errors += yaml_parse_injections(parser, jb, false);
629 +
630 + else if (yaml_scalar_matches(&event, "unmatched", strlen("unmatched")))
631 + errors += yaml_parse_unmatched(parser, jb);
632 +
633 + else if (yaml_scalar_matches(&event, "rewrite", strlen("rewrite")))
634 + errors += yaml_parse_rewrites(parser, jb);
635 +
636 + else {
637 + yaml_error(parser, &event, "unexpected scalar");
638 + errors++;
639 + }
640 + break;
641 + }
642 +
643 + yaml_event_delete(&event);
644 + }
645 +
646 + if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_END_EVENT)) {
647 + errors++;
648 + goto cleanup;
649 + }
650 +
651 + if(!yaml_parse_expect_event(parser, YAML_STREAM_END_EVENT)) {
652 + errors++;
653 + goto cleanup;
654 + }
655 +
656 +cleanup:
657 + return errors;
658 +}
659 +
660 +bool yaml_parse_file(const char *config_file_path, struct log_job *jb) {
661 + if(!config_file_path || !*config_file_path) {
662 + log2stderr("yaml configuration filename cannot be empty.");
663 + return false;
664 + }
665 +
666 + FILE *fp = fopen(config_file_path, "r");
667 + if (!fp) {
668 + log2stderr("Error opening config file: %s", config_file_path);
669 + return false;
670 + }
671 +
672 + yaml_parser_t parser;
673 + yaml_parser_initialize(&parser);
674 + yaml_parser_set_input_file(&parser, fp);
675 +
676 + size_t errors = yaml_parse_initialized(&parser, jb);
677 +
678 + yaml_parser_delete(&parser);
679 + fclose(fp);
680 + return errors == 0;
681 +}
682 +
683 +bool yaml_parse_config(const char *config_name, struct log_job *jb) {
684 + char filename[FILENAME_MAX + 1];
685 +
686 + snprintf(filename, sizeof(filename), "%s/%s.yaml", LOG2JOURNAL_CONFIG_PATH, config_name);
687 + return yaml_parse_file(filename, jb);
688 +}
689 +
690 +#endif // HAVE_LIBYAML
691 +
692 +// ----------------------------------------------------------------------------
693 +// printing yaml
694 +
695 +static void yaml_print_multiline_value(const char *s, size_t depth) {
696 + if (!s)
697 + s = "";
698 +
699 + do {
700 + const char* next = strchr(s, '\n');
701 + if(next) next++;
702 +
703 + size_t len = next ? (size_t)(next - s) : strlen(s);
704 + char buf[len + 1];
705 + strncpy(buf, s, len);
706 + buf[len] = '\0';
707 +
708 + fprintf(stderr, "%.*s%s%s",
709 + (int)(depth * 2), " ",
710 + buf, next ? "" : "\n");
711 +
712 + s = next;
713 + } while(s && *s);
714 +}
715 +
716 +static bool needs_quotes_in_yaml(const char *str) {
717 + // Lookup table for special YAML characters
718 + static bool special_chars[256] = { false };
719 + static bool table_initialized = false;
720 +
721 + if (!table_initialized) {
722 + // Initialize the lookup table
723 + const char *special_chars_str = ":{}[],&*!|>'\"%@`^";
724 + for (const char *c = special_chars_str; *c; ++c) {
725 + special_chars[(unsigned char)*c] = true;
726 + }
727 + table_initialized = true;
728 + }
729 +
730 + while (*str) {
731 + if (special_chars[(unsigned char)*str]) {
732 + return true;
733 + }
734 + str++;
735 + }
736 + return false;
737 +}
738 +
739 +static void yaml_print_node(const char *key, const char *value, size_t depth, bool dash) {
740 + if(depth > 10) depth = 10;
741 + const char *quote = "\"";
742 +
743 + const char *second_line = NULL;
744 + if(value && strchr(value, '\n')) {
745 + second_line = value;
746 + value = "|";
747 + quote = "";
748 + }
749 + else if(!value || !needs_quotes_in_yaml(value))
750 + quote = "";
751 +
752 + fprintf(stderr, "%.*s%s%s%s%s%s%s\n",
753 + (int)(depth * 2), " ", dash ? "- ": "",
754 + key ? key : "", key ? ": " : "",
755 + quote, value ? value : "", quote);
756 +
757 + if(second_line) {
758 + yaml_print_multiline_value(second_line, depth + 1);
759 + }
760 +}
761 +
762 +void log_job_to_yaml(struct log_job *jb) {
763 + if(jb->pattern)
764 + yaml_print_node("pattern", jb->pattern, 0, false);
765 +
766 + if(jb->prefix) {
767 + fprintf(stderr, "\n");
768 + yaml_print_node("prefix", jb->prefix, 0, false);
769 + }
770 +
771 + if(jb->filename.key) {
772 + fprintf(stderr, "\n");
773 + yaml_print_node("filename", NULL, 0, false);
774 + yaml_print_node("key", jb->filename.key, 1, false);
775 + }
776 +
777 + if(jb->dups.used) {
778 + fprintf(stderr, "\n");
779 + yaml_print_node("duplicate", NULL, 0, false);
780 + for(size_t i = 0; i < jb->dups.used ;i++) {
781 + struct key_dup *kd = &jb->dups.array[i];
782 + yaml_print_node("key", kd->target, 1, true);
783 + yaml_print_node("values_of", NULL, 2, false);
784 +
785 + for(size_t k = 0; k < kd->used ;k++)
786 + yaml_print_node(NULL, kd->keys[k], 3, true);
787 + }
788 + }
789 +
790 + if(jb->injections.used) {
791 + fprintf(stderr, "\n");
792 + yaml_print_node("inject", NULL, 0, false);
793 +
794 + for (size_t i = 0; i < jb->injections.used; i++) {
795 + yaml_print_node("key", jb->injections.keys[i].key, 1, true);
796 + yaml_print_node("value", jb->injections.keys[i].value.s, 2, false);
797 + }
798 + }
799 +
800 + if(jb->rewrites.used) {
801 + fprintf(stderr, "\n");
802 + yaml_print_node("rewrite", NULL, 0, false);
803 +
804 + for(size_t i = 0; i < jb->rewrites.used ;i++) {
805 + yaml_print_node("key", jb->rewrites.array[i].key, 1, true);
806 + yaml_print_node("search", jb->rewrites.array[i].search_pattern, 2, false);
807 + yaml_print_node("replace", jb->rewrites.array[i].replace_pattern, 2, false);
808 + }
809 + }
810 +
811 + if(jb->renames.used) {
812 + fprintf(stderr, "\n");
813 + yaml_print_node("rename", NULL, 0, false);
814 +
815 + for(size_t i = 0; i < jb->renames.used ;i++) {
816 + yaml_print_node("new_key", jb->renames.array[i].new_key, 1, true);
817 + yaml_print_node("old_key", jb->renames.array[i].old_key, 2, false);
818 + }
819 + }
820 +
821 + if(jb->unmatched.key || jb->unmatched.injections.used) {
822 + fprintf(stderr, "\n");
823 + yaml_print_node("unmatched", NULL, 0, false);
824 +
825 + if(jb->unmatched.key)
826 + yaml_print_node("key", jb->unmatched.key, 1, false);
827 +
828 + if(jb->unmatched.injections.used) {
829 + fprintf(stderr, "\n");
830 + yaml_print_node("inject", NULL, 1, false);
831 +
832 + for (size_t i = 0; i < jb->unmatched.injections.used; i++) {
833 + yaml_print_node("key", jb->unmatched.injections.keys[i].key, 2, true);
834 + yaml_print_node("value", jb->unmatched.injections.keys[i].value.s, 3, false);
835 + }
836 + }
837 + }
838 +}
collectors/log2journal/log2journal.c new
+414
@@ -0,0 +1,414 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "log2journal.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +
7 +static char *rewrite_value(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len) {
8 + static __thread char rewritten_value[MAX_VALUE_LEN + 1];
9 +
10 + for (size_t i = 0; i < jb->rewrites.used; i++) {
11 + struct key_rewrite *rw = &jb->rewrites.array[i];
12 +
13 + if (rw->hash == hash && strcmp(rw->key, key) == 0) {
14 + if (!jb_pcre2_match(rw->re, rw->match_data, (char *)value, value_len, false)) {
15 + continue; // No match found, skip to next rewrite rule
16 + }
17 +
18 + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(rw->match_data);
19 +
20 + char *buffer = rewritten_value;
21 + size_t buffer_remaining = sizeof(rewritten_value);
22 +
23 + // Iterate through the linked list of replacement nodes
24 + for (struct replacement_node *node = rw->nodes; node != NULL; node = node->next) {
25 + if (node->is_variable) {
26 + uint32_t groupnumber = pcre2_substring_number_from_name(rw->re, (PCRE2_SPTR)node->s);
27 + PCRE2_SIZE start_offset = ovector[2 * groupnumber];
28 + PCRE2_SIZE end_offset = ovector[2 * groupnumber + 1];
29 + PCRE2_SIZE length = end_offset - start_offset;
30 +
31 + size_t copied = copy_to_buffer(buffer, buffer_remaining, value + start_offset, length);
32 + buffer += copied;
33 + buffer_remaining -= copied;
34 + }
35 + else {
36 + size_t len = node->len;
37 + size_t copied = copy_to_buffer(buffer, buffer_remaining, node->s, len);
38 + buffer += copied;
39 + buffer_remaining -= copied;
40 + }
41 + }
42 +
43 + return rewritten_value;
44 + }
45 + }
46 +
47 + return NULL;
48 +}
49 +
50 +static inline const char *rename_key(struct log_job *jb, const char *key, XXH64_hash_t hash, XXH64_hash_t *new_hash) {
51 + for(size_t i = 0; i < jb->renames.used ;i++) {
52 + struct key_rename *rn = &jb->renames.array[i];
53 +
54 + if(rn->old_hash == hash && strcmp(rn->old_key, key) == 0) {
55 + *new_hash = rn->new_hash;
56 + return rn->new_key;
57 + }
58 + }
59 +
60 + *new_hash = hash;
61 + return key;
62 +}
63 +
64 +// ----------------------------------------------------------------------------
65 +
66 +static inline void send_key_value_error(const char *key, const char *format, ...) __attribute__ ((format(__printf__, 2, 3)));
67 +static inline void send_key_value_error(const char *key, const char *format, ...) {
68 + printf("%s=", key);
69 + va_list args;
70 + va_start(args, format);
71 + vprintf(format, args);
72 + va_end(args);
73 + printf("\n");
74 +}
75 +
76 +inline void jb_send_key_value_and_rewrite(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t len) {
77 + char *rewritten = rewrite_value(jb, key, hash, value, len);
78 + if(!rewritten)
79 + printf("%s=%.*s\n", key, (int)len, value);
80 + else
81 + printf("%s=%s\n", key, rewritten);
82 +}
83 +
84 +inline void jb_send_extracted_key_value(struct log_job *jb, const char *key, const char *value, size_t len) {
85 + XXH64_hash_t hash = XXH3_64bits(key, strlen(key));
86 +
87 + // process renames (changing the key)
88 + XXH64_hash_t new_hash;
89 + const char *new_key = rename_key(jb, key, hash, &new_hash);
90 +
91 + // process rewrites (changing the value)
92 + // and send it to output
93 + jb_send_key_value_and_rewrite(jb, new_key, new_hash, value, len);
94 +
95 + // process the duplications (using the original key)
96 + // and send them to output
97 + jb_send_duplications_for_key(jb, key, hash, value, len);
98 +}
99 +
100 +static inline void send_key_value_constant(struct log_job *jb, const char *key, const char *value) {
101 + printf("%s=%s\n", key, value);
102 +}
103 +
104 +// ----------------------------------------------------------------------------
105 +// injection of constant fields
106 +
107 +static void jb_select_which_injections_should_be_injected_on_unmatched(struct log_job *jb) {
108 + // mark all injections to be added to unmatched logs
109 + for(size_t i = 0; i < jb->injections.used ; i++)
110 + jb->injections.keys[i].on_unmatched = true;
111 +
112 + if(jb->injections.used && jb->unmatched.injections.used) {
113 + // we have both injections and injections on unmatched
114 +
115 + // we find all the injections that are also configured as injections on unmatched,
116 + // and we disable them, so that the output will not have the same key twice
117 +
118 + for(size_t i = 0; i < jb->injections.used ;i++) {
119 + for(size_t u = 0; u < jb->unmatched.injections.used ; u++) {
120 + if(strcmp(jb->injections.keys[i].key, jb->unmatched.injections.keys[u].key) == 0)
121 + jb->injections.keys[i].on_unmatched = false;
122 + }
123 + }
124 + }
125 +}
126 +
127 +
128 +static inline void jb_finalize_injections(struct log_job *jb, bool line_is_matched) {
129 + for (size_t j = 0; j < jb->injections.used; j++) {
130 + if(!line_is_matched && !jb->injections.keys[j].on_unmatched)
131 + continue;
132 +
133 + send_key_value_constant(jb, jb->injections.keys[j].key, jb->injections.keys[j].value.s);
134 + }
135 +}
136 +
137 +static inline void jb_reset_injections(struct log_job *jb) {
138 + for(size_t d = 0; d < jb->dups.used ; d++) {
139 + struct key_dup *kd = &jb->dups.array[d];
140 + kd->exposed = false;
141 +
142 + for(size_t g = 0; g < kd->used ; g++) {
143 + if(kd->values[g].s)
144 + kd->values[g].s[0] = '\0';
145 + }
146 + }
147 +}
148 +
149 +// ----------------------------------------------------------------------------
150 +// duplications
151 +
152 +inline void jb_send_duplications_for_key(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len) {
153 + // IMPORTANT:
154 + // The 'value' may not be NULL terminated and have more data that the value we need
155 +
156 + for (size_t d = 0; d < jb->dups.used; d++) {
157 + struct key_dup *kd = &jb->dups.array[d];
158 +
159 + if(kd->exposed || kd->used == 0)
160 + continue;
161 +
162 + if(kd->used == 1) {
163 + // just one key to be duplicated
164 + if(strcmp(kd->keys[0], key) == 0) {
165 + jb_send_key_value_and_rewrite(jb, kd->target, kd->hash, value, value_len);
166 + kd->exposed = true;
167 + }
168 + }
169 + else {
170 + // multiple keys to be duplicated
171 + for(size_t g = 0; g < kd->used ; g++) {
172 + if(strcmp(kd->keys[g], key) == 0)
173 + txt_replace(&kd->values[g], value, value_len);
174 + }
175 + }
176 + }
177 +}
178 +
179 +static inline void jb_send_remaining_duplications(struct log_job *jb) {
180 + static __thread char buffer[MAX_VALUE_LEN + 1];
181 +
182 + // IMPORTANT:
183 + // all duplications are exposed, even the ones we haven't found their keys in the source,
184 + // so that the output always has the same fields for matched entries.
185 +
186 + for(size_t d = 0; d < jb->dups.used ; d++) {
187 + struct key_dup *kd = &jb->dups.array[d];
188 +
189 + if(kd->exposed || kd->used == 0)
190 + continue;
191 +
192 + buffer[0] = '\0';
193 + size_t remaining = sizeof(buffer);
194 + char *s = buffer;
195 +
196 + for(size_t g = 0; g < kd->used ; g++) {
197 + if(remaining < 2) {
198 + log2stderr("Warning: duplicated key '%s' cannot fit the values.", kd->target);
199 + break;
200 + }
201 +
202 + if(g > 0) {
203 + *s++ = ',';
204 + *s = '\0';
205 + remaining--;
206 + }
207 +
208 + char *value = (kd->values[g].s && kd->values[g].s[0]) ? kd->values[g].s : "[unavailable]";
209 + size_t len = strlen(value);
210 + size_t copied = copy_to_buffer(s, remaining, value, len);
211 + remaining -= copied;
212 + s += copied;
213 +
214 + if(copied != len) {
215 + log2stderr("Warning: duplicated key '%s' will have truncated value", jb->dups.array[d].target);
216 + break;
217 + }
218 + }
219 + jb_send_key_value_and_rewrite(jb, kd->target, kd->hash, buffer, s - buffer);
220 + }
221 +}
222 +
223 +// ----------------------------------------------------------------------------
224 +// filename injection
225 +
226 +static inline void jb_inject_filename(struct log_job *jb) {
227 + if (jb->filename.key && jb->filename.current[0])
228 + send_key_value_constant(jb, jb->filename.key, jb->filename.current);
229 +}
230 +
231 +static inline bool jb_switched_filename(struct log_job *jb, const char *line, size_t len) {
232 + // IMPORTANT:
233 + // Return TRUE when the caller should skip this line (because it is ours).
234 + // Unfortunately, we have to consume empty lines too.
235 +
236 + // IMPORTANT:
237 + // filename may not be NULL terminated and have more data than the filename.
238 +
239 + if (!len) {
240 + jb->filename.last_line_was_empty = true;
241 + return true;
242 + }
243 +
244 + // Check if it's a log file change line
245 + if (jb->filename.last_line_was_empty && line[0] == '=' && strncmp(line, "==> ", 4) == 0) {
246 + const char *start = line + 4;
247 + const char *end = strstr(line, " <==");
248 + while (*start == ' ') start++;
249 + if (*start != '\n' && *start != '\0' && end) {
250 + copy_to_buffer(jb->filename.current, sizeof(jb->filename.current),
251 + start, end - start);
252 + return true;
253 + }
254 + }
255 +
256 + jb->filename.last_line_was_empty = false;
257 + return false;
258 +}
259 +
260 +// ----------------------------------------------------------------------------
261 +// input reading
262 +
263 +static char *get_next_line(struct log_job *jb, char *buffer, size_t size, size_t *line_length) {
264 + if(!fgets(buffer, (int)size, stdin)) {
265 + *line_length = 0;
266 + return NULL;
267 + }
268 +
269 + char *line = buffer;
270 + size_t len = strlen(line);
271 +
272 + // remove trailing newlines and spaces
273 + while(len > 1 && (line[len - 1] == '\n' || isspace(line[len - 1])))
274 + line[--len] = '\0';
275 +
276 + // skip leading spaces
277 + while(isspace(*line)) {
278 + line++;
279 + len--;
280 + }
281 +
282 + *line_length = len;
283 + return line;
284 +}
285 +
286 +// ----------------------------------------------------------------------------
287 +
288 +static inline void jb_traverse_pcre2_named_groups_and_send_keys(struct log_job *jb, pcre2_code *re, pcre2_match_data *match_data, char *line) {
289 + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
290 + uint32_t namecount;
291 + pcre2_pattern_info(re, PCRE2_INFO_NAMECOUNT, &namecount);
292 +
293 + if (namecount > 0) {
294 + PCRE2_SPTR name_table;
295 + pcre2_pattern_info(re, PCRE2_INFO_NAMETABLE, &name_table);
296 + uint32_t name_entry_size;
297 + pcre2_pattern_info(re, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size);
298 +
299 + const unsigned char *tabptr = name_table;
300 + for (uint32_t i = 0; i < namecount; i++) {
301 + int n = (tabptr[0] << 8) | tabptr[1];
302 + const char *group_name = (const char *)(tabptr + 2);
303 +
304 + PCRE2_SIZE start_offset = ovector[2 * n];
305 + PCRE2_SIZE end_offset = ovector[2 * n + 1];
306 + PCRE2_SIZE group_length = end_offset - start_offset;
307 +
308 + jb_send_extracted_key_value(jb, group_name, line + start_offset, group_length);
309 + tabptr += name_entry_size;
310 + }
311 + }
312 +}
313 +
314 +// ----------------------------------------------------------------------------
315 +
316 +struct log_job log_job = { 0 };
317 +int main(int argc, char *argv[]) {
318 + struct log_job *jb = &log_job;
319 +
320 + if(!parse_log2journal_parameters(jb, argc, argv))
321 + exit(1);
322 +
323 + if(jb->show_config)
324 + log_job_to_yaml(jb);
325 +
326 + jb_select_which_injections_should_be_injected_on_unmatched(jb);
327 +
328 + pcre2_code *pcre2 = NULL;
329 + pcre2_match_data *match_data = NULL;
330 + LOG_JSON_STATE *json = NULL;
331 + LOGFMT_STATE *logfmt = NULL;
332 + if(strcmp(jb->pattern, "json") == 0) {
333 + json = json_parser_create(jb);
334 + }
335 + else if(strcmp(jb->pattern, "logfmt") == 0) {
336 + logfmt = logfmt_parser_create(jb);
337 + }
338 + else {
339 + pcre2 = jb_compile_pcre2_pattern(jb->pattern);
340 + if(!pcre2)
341 + return 1;
342 +
343 + match_data = pcre2_match_data_create_from_pattern(pcre2, NULL);
344 + if(!match_data)
345 + return 1;
346 + }
347 +
348 + char buffer[MAX_LINE_LENGTH];
349 + char *line;
350 + size_t len;
351 +
352 + while ((line = get_next_line(jb, buffer, sizeof(buffer), &len))) {
353 + if(jb_switched_filename(jb, line, len))
354 + continue;
355 +
356 + jb_reset_injections(jb);
357 +
358 + bool line_is_matched;
359 +
360 + if(json)
361 + line_is_matched = json_parse_document(json, line);
362 + else if(logfmt)
363 + line_is_matched = logfmt_parse_document(logfmt, line);
364 + else
365 + line_is_matched = jb_pcre2_match(pcre2, match_data, line, len, true);
366 +
367 + if(!line_is_matched) {
368 + if(json)
369 + log2stderr("%s", json_parser_error(json));
370 + else if(logfmt)
371 + log2stderr("%s", logfmt_parser_error(logfmt));
372 +
373 + if (jb->unmatched.key) {
374 + // we are sending errors to systemd-journal
375 + send_key_value_error(jb->unmatched.key, "Parsing error on: %s", line);
376 +
377 + for (size_t j = 0; j < jb->unmatched.injections.used; j++)
378 + send_key_value_constant(jb, jb->unmatched.injections.keys[j].key,
379 + jb->unmatched.injections.keys[j].value.s);
380 + }
381 + else {
382 + // we are just logging errors to stderr
383 + continue;
384 + }
385 + }
386 + else {
387 + if(pcre2)
388 + jb_traverse_pcre2_named_groups_and_send_keys(jb, pcre2, match_data, line);
389 +
390 + // print all non-exposed duplications
391 + jb_send_remaining_duplications(jb);
392 + }
393 +
394 + jb_inject_filename(jb);
395 + jb_finalize_injections(jb, line_is_matched);
396 +
397 + printf("\n");
398 + fflush(stdout);
399 + }
400 +
401 + if(json)
402 + json_parser_destroy(json);
403 +
404 + else if(logfmt)
405 + logfmt_parser_destroy(logfmt);
406 +
407 + else if(pcre2) {
408 + pcre2_match_data_free(match_data);
409 + pcre2_code_free(pcre2);
410 + }
411 +
412 + nd_log_destroy(jb);
413 + return 0;
414 +}
collectors/log2journal/log2journal.d/nginx-combined.yaml new
+93
@@ -0,0 +1,93 @@
1 +# Netdata log2journal Configuration
2 +# The following parses nginx log files using the combined format.
3 +
4 +# The PCRE2 pattern to match log entries and give names to the fields.
5 +# The journal will have these names, so follow their rules. You can
6 +# initiate an extended PCRE2 pattern by starting the pattern with (?x)
7 +pattern: |
8 + (?x) # Enable PCRE2 extended mode
9 + ^
10 + (?<NGINX_REMOTE_ADDR>[^ ]+) \s - \s # NGINX_REMOTE_ADDR
11 + (?<NGINX_REMOTE_USER>[^ ]+) \s # NGINX_REMOTE_USER
12 + \[
13 + (?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL
14 + \]
15 + \s+ "
16 + (?<MESSAGE>
17 + (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD
18 + (?<NGINX_URL>[^ ]+) \s+
19 + HTTP/(?<NGINX_HTTP_VERSION>[^"]+)
20 + )
21 + " \s+
22 + (?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS
23 + (?<NGINX_BODY_BYTES_SENT>\d+) \s+ # NGINX_BODY_BYTES_SENT
24 + "(?<NGINX_HTTP_REFERER>[^"]*)" \s+ # NGINX_HTTP_REFERER
25 + "(?<NGINX_HTTP_USER_AGENT>[^"]*)" # NGINX_HTTP_USER_AGENT
26 +
27 +# When log2journal can detect the filename of each log entry (tail gives it
28 +# only when it tails multiple files), this key will be used to send the
29 +# filename to the journals.
30 +filename:
31 + key: NGINX_LOG_FILENAME
32 +
33 +# Duplicate fields under a different name. You can duplicate multiple fields
34 +# to a new one and then use rewrite rules to change its value.
35 +duplicate:
36 +
37 + # we insert the field PRIORITY as a copy of NGINX_STATUS.
38 + - key: PRIORITY
39 + values_of:
40 + - NGINX_STATUS
41 +
42 + # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.
43 + - key: NGINX_STATUS_FAMILY
44 + values_of:
45 + - NGINX_STATUS
46 +
47 +# Inject constant fields into the journal logs.
48 +inject:
49 + - key: SYSLOG_IDENTIFIER
50 + value: "nginx-log"
51 +
52 +# Rewrite the value of fields (including the duplicated ones).
53 +# The search pattern can have named groups, and the replace pattern can use
54 +# them as ${name}.
55 +rewrite:
56 + # PRIORTY is a duplicate of NGINX_STATUS
57 + # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
58 + - key: "PRIORITY"
59 + search: "^[123]"
60 + replace: 6
61 +
62 + - key: "PRIORITY"
63 + search: "^4"
64 + replace: 5
65 +
66 + - key: "PRIORITY"
67 + search: "^5"
68 + replace: 3
69 +
70 + - key: "PRIORITY"
71 + search: ".*"
72 + replace: 4
73 +
74 + # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
75 + - key: "NGINX_STATUS_FAMILY"
76 + search: "^(?<first_digit>[1-5])"
77 + replace: "${first_digit}xx"
78 +
79 + - key: "NGINX_STATUS_FAMILY"
80 + search: ".*"
81 + replace: "UNKNOWN"
82 +
83 +# Control what to do when input logs do not match the main PCRE2 pattern.
84 +unmatched:
85 + # The journal key to log the PCRE2 error message to.
86 + # Set this to MESSAGE, so you to see the error in the log.
87 + key: MESSAGE
88 +
89 + # Inject static fields to the unmatched entries.
90 + # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.
91 + inject:
92 + - key: PRIORITY
93 + value: 1
collectors/log2journal/log2journal.h new
+308
@@ -0,0 +1,308 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_LOG2JOURNAL_H
4 +#define NETDATA_LOG2JOURNAL_H
5 +
6 +// only for PACKAGE_VERSION
7 +#include "config.h"
8 +
9 +#include <stdio.h>
10 +#include <stdlib.h>
11 +#include <dirent.h>
12 +#include <string.h>
13 +#include <stdbool.h>
14 +#include <string.h>
15 +#include <ctype.h>
16 +#include <stdarg.h>
17 +
18 +#define XXH_INLINE_ALL
19 +#include "../../libnetdata/xxhash.h"
20 +
21 +#define PCRE2_CODE_UNIT_WIDTH 8
22 +#include <pcre2.h>
23 +
24 +#ifdef HAVE_LIBYAML
25 +#include <yaml.h>
26 +#endif
27 +
28 +
29 +#define MAX_OUTPUT_KEYS 1024
30 +#define OVECCOUNT (MAX_OUTPUT_KEYS * 3) // should be a multiple of 3
31 +#define MAX_LINE_LENGTH (1024 * 1024)
32 +#define MAX_KEY_DUPS (MAX_OUTPUT_KEYS / 2)
33 +#define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
34 +#define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
35 +#define MAX_RENAMES (MAX_OUTPUT_KEYS / 2)
36 +#define MAX_KEY_DUPS_KEYS 20
37 +
38 +#define MAX_KEY_LEN 64 // according to systemd-journald
39 +#define MAX_VALUE_LEN (48 * 1024) // according to systemd-journald
40 +
41 +#define LOG2JOURNAL_CONFIG_PATH LIBCONFIG_DIR "/log2journal.d"
42 +
43 +// ----------------------------------------------------------------------------
44 +// logging
45 +
46 +// enable the compiler to check for printf like errors on our log2stderr() function
47 +static inline void log2stderr(const char *format, ...) __attribute__ ((format(__printf__, 1, 2)));
48 +static inline void log2stderr(const char *format, ...) {
49 + va_list args;
50 + va_start(args, format);
51 + vfprintf(stderr, format, args);
52 + va_end(args);
53 + fprintf(stderr, "\n");
54 +}
55 +
56 +// ----------------------------------------------------------------------------
57 +// allocation functions abstraction
58 +
59 +static inline void *mallocz(size_t size) {
60 + void *ptr = malloc(size);
61 + if (!ptr) {
62 + log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", size);
63 + exit(EXIT_FAILURE);
64 + }
65 + return ptr;
66 +}
67 +
68 +static inline char *strdupz(const char *s) {
69 + char *ptr = strdup(s);
70 + if (!ptr) {
71 + log2stderr("Fatal Error: Memory allocation failed in strdup.");
72 + exit(EXIT_FAILURE);
73 + }
74 + return ptr;
75 +}
76 +
77 +static inline char *strndupz(const char *s, size_t n) {
78 + char *ptr = strndup(s, n);
79 + if (!ptr) {
80 + log2stderr("Fatal Error: Memory allocation failed in strndup. Requested size: %zu bytes.", n);
81 + exit(EXIT_FAILURE);
82 + }
83 + return ptr;
84 +}
85 +
86 +static inline void freez(void *ptr) {
87 + if (ptr)
88 + free(ptr);
89 +}
90 +
91 +// ----------------------------------------------------------------------------
92 +
93 +static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) {
94 + if(dst_size < 2) {
95 + if(dst_size == 1)
96 + *dst = '\0';
97 +
98 + return 0;
99 + }
100 +
101 + if(src_len <= dst_size - 1) {
102 + memcpy(dst, src, src_len);
103 + dst[src_len] = '\0';
104 + return src_len;
105 + }
106 + else {
107 + memcpy(dst, src, dst_size - 1);
108 + dst[dst_size - 1] = '\0';
109 + return dst_size - 1;
110 + }
111 +}
112 +
113 +// ----------------------------------------------------------------------------
114 +
115 +typedef struct txt {
116 + char *s;
117 + size_t size;
118 +} TEXT;
119 +
120 +static inline void txt_replace(TEXT *txt, const char *s, size_t len) {
121 + if(!s || !*s || len == 0) {
122 + s = "";
123 + len = 0;
124 + }
125 +
126 + if(len + 1 <= txt->size) {
127 + // the existing value allocation, fits our value
128 +
129 + memcpy(txt->s, s, len);
130 + txt->s[len] = '\0';
131 + }
132 + else {
133 + // no existing value allocation, or too small for our value
134 +
135 + if(txt->s)
136 + freez(txt->s);
137 +
138 + txt->s = strndupz(s, len);
139 + txt->size = len + 1;
140 + }
141 +}
142 +
143 +// ----------------------------------------------------------------------------
144 +
145 +typedef struct key_value {
146 + char key[MAX_KEY_LEN + 1];
147 + TEXT value;
148 + bool on_unmatched;
149 +} KEY_VALUE;
150 +
151 +static inline void key_value_replace(KEY_VALUE *kv, const char *key, size_t key_len, const char *value, size_t value_len) {
152 + copy_to_buffer(kv->key, sizeof(kv->key), key, key_len);
153 + txt_replace(&kv->value, value, value_len);
154 +}
155 +
156 +// ----------------------------------------------------------------------------
157 +
158 +struct key_dup {
159 + XXH64_hash_t hash;
160 + char *target;
161 + char *keys[MAX_KEY_DUPS_KEYS];
162 + TEXT values[MAX_KEY_DUPS_KEYS];
163 + size_t used;
164 + bool exposed;
165 +};
166 +
167 +struct key_rename {
168 + XXH64_hash_t new_hash;
169 + XXH64_hash_t old_hash;
170 + char *new_key;
171 + char *old_key;
172 +};
173 +
174 +struct replacement_node {
175 + bool is_variable;
176 + const char *s;
177 + size_t len;
178 + struct replacement_node *next;
179 +};
180 +
181 +struct key_rewrite {
182 + XXH64_hash_t hash;
183 + char *key;
184 + char *search_pattern;
185 + char *replace_pattern;
186 + pcre2_code *re;
187 + pcre2_match_data *match_data;
188 + struct replacement_node *nodes;
189 +};
190 +
191 +struct log_job {
192 + bool show_config;
193 +
194 + const char *pattern;
195 + const char *prefix;
196 +
197 + struct {
198 + const char *key;
199 + char current[FILENAME_MAX + 1];
200 + bool last_line_was_empty;
201 + } filename;
202 +
203 + struct {
204 + KEY_VALUE keys[MAX_INJECTIONS];
205 + size_t used;
206 + } injections;
207 +
208 + struct {
209 + const char *key;
210 + struct {
211 + KEY_VALUE keys[MAX_INJECTIONS];
212 + size_t used;
213 + } injections;
214 + } unmatched;
215 +
216 + struct {
217 + struct key_dup array[MAX_KEY_DUPS];
218 + size_t used;
219 + } dups;
220 +
221 + struct {
222 + struct key_rewrite array[MAX_REWRITES];
223 + size_t used;
224 + } rewrites;
225 +
226 + struct {
227 + struct key_rename array[MAX_RENAMES];
228 + size_t used;
229 + } renames;
230 +};
231 +
232 +void jb_send_key_value_and_rewrite(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t len);
233 +void jb_send_duplications_for_key(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len);
234 +void jb_send_extracted_key_value(struct log_job *jb, const char *key, const char *value, size_t len);
235 +
236 +struct key_dup *log_job_add_duplication_to_job(struct log_job *jb, const char *target, size_t target_len);
237 +bool log_job_add_key_to_duplication(struct key_dup *kd, const char *key, size_t key_len);
238 +bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len);
239 +bool log_job_add_key_prefix(struct log_job *jb, const char *prefix, size_t prefix_len);
240 +bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched);
241 +bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern);
242 +bool log_job_add_rename(struct log_job *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len);
243 +
244 +// entry point to parse command line parameters
245 +bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv);
246 +
247 +void log2journal_command_line_help(const char *name);
248 +
249 +// free all resources consumed by the log job
250 +void nd_log_destroy(struct log_job *jb);
251 +
252 +#ifdef HAVE_LIBYAML
253 +bool yaml_parse_file(const char *config_file_path, struct log_job *jb);
254 +bool yaml_parse_config(const char *config_name, struct log_job *jb);
255 +#endif
256 +
257 +void log_job_to_yaml(struct log_job *jb);
258 +
259 +typedef struct log_json_state LOG_JSON_STATE;
260 +LOG_JSON_STATE *json_parser_create(struct log_job *jb);
261 +void json_parser_destroy(LOG_JSON_STATE *js);
262 +const char *json_parser_error(LOG_JSON_STATE *js);
263 +bool json_parse_document(LOG_JSON_STATE *js, const char *txt);
264 +void json_test(void);
265 +
266 +typedef struct logfmt_state LOGFMT_STATE;
267 +LOGFMT_STATE *logfmt_parser_create(struct log_job *jb);
268 +void logfmt_parser_destroy(LOGFMT_STATE *lfs);
269 +const char *logfmt_parser_error(LOGFMT_STATE *lfs);
270 +bool logfmt_parse_document(LOGFMT_STATE *js, const char *txt);
271 +void logfmt_test(void);
272 +
273 +// ----------------------------------------------------------------------------
274 +// PCRE2 patters handling
275 +
276 +static inline pcre2_code *jb_compile_pcre2_pattern(const char *pattern) {
277 + int error_number;
278 + PCRE2_SIZE error_offset;
279 + PCRE2_SPTR pattern_ptr = (PCRE2_SPTR)pattern;
280 +
281 + pcre2_code *re = pcre2_compile(pattern_ptr, PCRE2_ZERO_TERMINATED, 0, &error_number, &error_offset, NULL);
282 + if (re == NULL) {
283 + PCRE2_UCHAR errbuf[1024];
284 + pcre2_get_error_message(error_number, errbuf, sizeof(errbuf));
285 + log2stderr("PCRE2 compilation failed at offset %d: %s", (int)error_offset, errbuf);
286 + log2stderr("Check for common regex syntax errors or unsupported PCRE2 patterns.");
287 + return NULL;
288 + }
289 +
290 + return re;
291 +}
292 +
293 +static inline bool jb_pcre2_match(pcre2_code *re, pcre2_match_data *match_data, char *line, size_t len, bool log) {
294 + int rc = pcre2_match(re, (PCRE2_SPTR)line, len, 0, 0, match_data, NULL);
295 + if(rc < 0) {
296 + PCRE2_UCHAR errbuf[1024];
297 + pcre2_get_error_message(rc, errbuf, sizeof(errbuf));
298 +
299 + if(log)
300 + log2stderr("PCRE2 error %d: %s on: %s", rc, errbuf, line);
301 +
302 + return false;
303 + }
304 +
305 + return true;
306 +}
307 +
308 +#endif //NETDATA_LOG2JOURNAL_H
configure.ac
+1
@@ -2156,6 +2156,7 @@ AC_CONFIG_FILES([
2156 collectors/freebsd.plugin/Makefile
2157 collectors/freeipmi.plugin/Makefile
2158 collectors/cups.plugin/Makefile
2159 + collectors/log2journal/Makefile
2160 collectors/idlejitter.plugin/Makefile
2161 collectors/macos.plugin/Makefile
2162 collectors/nfacct.plugin/Makefile
libnetdata/log/Makefile.am
-1
@@ -5,5 +5,4 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
6 dist_noinst_DATA = \
7 README.md \
8 - log2journal.md \
8 $(NULL)
libnetdata/log/log2journal.c deleted
-2135
@@ -1,2135 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -// only for PACKAGE_VERSION
4 -#include "config.h"
5 -
6 -#include <stdio.h>
7 -#include <stdlib.h>
8 -#include <string.h>
9 -#include <stdbool.h>
10 -#include <string.h>
11 -#include <ctype.h>
12 -#include <stdarg.h>
13 -
14 -#define XXH_INLINE_ALL
15 -#include "../xxhash.h"
16 -
17 -#define PCRE2_CODE_UNIT_WIDTH 8
18 -#include <pcre2.h>
19 -
20 -#ifdef HAVE_LIBYAML
21 -#include <yaml.h>
22 -#endif
23 -
24 -#define MAX_OUTPUT_KEYS 1024
25 -#define OVECCOUNT (MAX_OUTPUT_KEYS * 3) // should be a multiple of 3
26 -#define MAX_LINE_LENGTH (1024 * 1024)
27 -#define MAX_KEY_DUPS (MAX_OUTPUT_KEYS / 2)
28 -#define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
29 -#define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
30 -#define MAX_KEY_DUPS_KEYS 20
31 -
32 -#define MAX_KEY_LEN 64 // according to systemd-journald
33 -#define MAX_VALUE_LEN (48 * 1024) // according to systemd-journald
34 -
35 -struct key_rewrite;
36 -static pcre2_code *jb_compile_pcre2_pattern(const char *pattern);
37 -static bool parse_replacement_pattern(struct key_rewrite *rw);
38 -
39 -#define YAML_CONFIG_NGINX_COMBINED \
40 - "# Netdata log2journal Configuration Template\n" \
41 - "# The following parses nginx log files using the combined format.\n" \
42 - "\n" \
43 - "# The PCRE2 pattern to match log entries and give names to the fields.\n" \
44 - "# The journal will have these names, so follow their rules. You can\n" \
45 - "# initiate an extended PCRE2 pattern by starting the pattern with (?x)\n" \
46 - "pattern: |\n" \
47 - " (?x) # Enable PCRE2 extended mode\n" \
48 - " ^\n" \
49 - " (?<NGINX_REMOTE_ADDR>[^ ]+) \\s - \\s # NGINX_REMOTE_ADDR\n" \
50 - " (?<NGINX_REMOTE_USER>[^ ]+) \\s # NGINX_REMOTE_USER\n" \
51 - " \\[\n" \
52 - " (?<NGINX_TIME_LOCAL>[^\\]]+) # NGINX_TIME_LOCAL\n" \
53 - " \\]\n" \
54 - " \\s+ \"\n" \
55 - " (?<MESSAGE>\n" \
56 - " (?<NGINX_METHOD>[A-Z]+) \\s+ # NGINX_METHOD\n" \
57 - " (?<NGINX_URL>[^ ]+) \\s+\n" \
58 - " HTTP/(?<NGINX_HTTP_VERSION>[^\"]+)\n" \
59 - " )\n" \
60 - " \" \\s+\n" \
61 - " (?<NGINX_STATUS>\\d+) \\s+ # NGINX_STATUS\n" \
62 - " (?<NGINX_BODY_BYTES_SENT>\\d+) \\s+ # NGINX_BODY_BYTES_SENT\n" \
63 - " \"(?<NGINX_HTTP_REFERER>[^\"]*)\" \\s+ # NGINX_HTTP_REFERER\n" \
64 - " \"(?<NGINX_HTTP_USER_AGENT>[^\"]*)\" # NGINX_HTTP_USER_AGENT\n" \
65 - "\n" \
66 - "# When log2journal can detect the filename of each log entry (tail gives it\n" \
67 - "# only when it tails multiple files), this key will be used to send the\n" \
68 - "# filename to the journals.\n" \
69 - "filename:\n" \
70 - " key: NGINX_LOG_FILENAME\n" \
71 - "\n" \
72 - "# Duplicate fields under a different name. You can duplicate multiple fields\n" \
73 - "# to a new one and then use rewrite rules to change its value.\n" \
74 - "duplicate:\n" \
75 - "\n" \
76 - " # we insert the field PRIORITY as a copy of NGINX_STATUS.\n" \
77 - " - key: PRIORITY\n" \
78 - " values_of:\n" \
79 - " - NGINX_STATUS\n" \
80 - "\n" \
81 - " # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.\n" \
82 - " - key: NGINX_STATUS_FAMILY\n" \
83 - " values_of: \n" \
84 - " - NGINX_STATUS\n" \
85 - "\n" \
86 - "# Inject constant fields into the journal logs.\n" \
87 - "inject:\n" \
88 - " - key: SYSLOG_IDENTIFIER\n" \
89 - " value: \"nginx-log\"\n" \
90 - "\n" \
91 - "# Rewrite the value of fields (including the duplicated ones).\n" \
92 - "# The search pattern can have named groups, and the replace pattern can use\n" \
93 - "# them as ${name}.\n" \
94 - "rewrite:\n" \
95 - " # PRIORTY is a duplicate of NGINX_STATUS\n" \
96 - " # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug\n" \
97 - " - key: \"PRIORITY\"\n" \
98 - " search: \"^[123]\"\n" \
99 - " replace: 6\n" \
100 - "\n" \
101 - " - key: \"PRIORITY\"\n" \
102 - " search: \"^4\"\n" \
103 - " replace: 5\n" \
104 - "\n" \
105 - " - key: \"PRIORITY\"\n" \
106 - " search: \"^5\"\n" \
107 - " replace: 3\n" \
108 - "\n" \
109 - " - key: \"PRIORITY\"\n" \
110 - " search: \".*\"\n" \
111 - " replace: 4\n" \
112 - " \n" \
113 - " # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS\n" \
114 - " - key: \"NGINX_STATUS_FAMILY\"\n" \
115 - " search: \"^(?<first_digit>[1-5])\"\n" \
116 - " replace: \"${first_digit}xx\"\n" \
117 - "\n" \
118 - " - key: \"NGINX_STATUS_FAMILY\"\n" \
119 - " search: \".*\"\n" \
120 - " replace: \"UNKNOWN\"\n" \
121 - "\n" \
122 - "# Control what to do when input logs do not match the main PCRE2 pattern.\n" \
123 - "unmatched:\n" \
124 - " # The journal key to log the PCRE2 error message to.\n" \
125 - " # Set this to MESSAGE, so you to see the error in the log.\n" \
126 - " key: MESSAGE\n" \
127 - " \n" \
128 - " # Inject static fields to the unmatched entries.\n" \
129 - " # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.\n" \
130 - " inject:\n" \
131 - " - key: PRIORITY\n" \
132 - " value: 1\n" \
133 - "\n"
134 -
135 -void display_help(const char *name) {
136 - printf("\n");
137 - printf("Netdata log2journal " PACKAGE_VERSION "\n");
138 - printf("\n");
139 - printf("Convert structured log input to systemd Journal Export Format.\n");
140 - printf("\n");
141 - printf("Using PCRE2 patterns, extract the fields from structured logs on the standard\n");
142 - printf("input, and generate output according to systemd Journal Export Format.\n");
143 - printf("\n");
144 - printf("Usage: %s [OPTIONS] PATTERN\n", name);
145 - printf("\n");
146 - printf("Options:\n");
147 - printf("\n");
148 - printf(" --file /path/to/file.yaml\n");
149 - printf(" Read yaml configuration file for instructions.\n");
150 - printf("\n");
151 - printf(" --config CONFIG_NAME\n");
152 - printf(" Run with the internal configuration named CONFIG_NAME\n");
153 - printf(" Available internal configs: nginx-combined\n");
154 - printf("\n");
155 - printf(" --show-config\n");
156 - printf(" Show the configuration in yaml format before starting the job.\n");
157 - printf(" This is also an easy way to convert command line parameters to yaml.\n");
158 - printf("\n");
159 - printf(" --filename-key KEY\n");
160 - printf(" Add a field with KEY as the key and the current filename as value.\n");
161 - printf(" Automatically detects filenames when piped after 'tail -F',\n");
162 - printf(" and tail matches multiple filenames.\n");
163 - printf(" To inject the filename when tailing a single file, use --inject.\n");
164 - printf("\n");
165 - printf(" --unmatched-key KEY\n");
166 - printf(" Include unmatched log entries in the output with KEY as the field name.\n");
167 - printf(" Use this to include unmatched entries to the output stream.\n");
168 - printf(" Usually it should be set to --unmatched-key=MESSAGE so that the\n");
169 - printf(" unmatched entry will appear as the log message in the journals.\n");
170 - printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n");
171 - printf("\n");
172 - printf(" --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]\n");
173 - printf(" Create a new key called TARGET, duplicating the values of the keys\n");
174 - printf(" given. Useful for further processing. When multiple keys are given,\n");
175 - printf(" their values are separated by comma.\n");
176 - printf(" Up to %d duplications can be given on the command line, and up to\n", MAX_KEY_DUPS);
177 - printf(" %d keys per duplication command are allowed.\n", MAX_KEY_DUPS_KEYS);
178 - printf("\n");
179 - printf(" --inject LINE\n");
180 - printf(" Inject constant fields to the output (both matched and unmatched logs).\n");
181 - printf(" --inject entries are added to unmatched lines too, when their key is\n");
182 - printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n");
183 - printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS);
184 - printf("\n");
185 - printf(" --inject-unmatched LINE\n");
186 - printf(" Inject lines into the output for each unmatched log entry.\n");
187 - printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n");
188 - printf(" lines as errors, so that they can easily be spotted in the journals.\n");
189 - printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS);
190 - printf("\n");
191 - printf(" --rewrite KEY=/SearchPattern/ReplacePattern\n");
192 - printf(" Apply a rewrite rule to the values of a specific key.\n");
193 - printf(" The first character after KEY= is the separator, which should also\n");
194 - printf(" be used between the search pattern and the replacement pattern.\n");
195 - printf(" The search pattern is a PCRE2 regular expression, and the replacement\n");
196 - printf(" pattern supports literals and named capture groups from the search pattern.\n");
197 - printf(" Example:\n");
198 - printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n");
199 - printf(" ${day}/${month}/${year}\n");
200 - printf(" This will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.\n");
201 - printf("\n");
202 - printf(" Only one rewrite rule is applied per key; the sequence of rewrites stops\n");
203 - printf(" for the key once a rule matches it. This allows providing a sequence of\n");
204 - printf(" independent rewriting rules for the same key, matching the different values\n");
205 - printf(" the key may get, and also provide a catch-all rewrite rule at the end of the\n");
206 - printf(" sequence for setting the key value if no other rule matched it.\n");
207 - printf("\n");
208 - printf(" The combination of duplicating keys with the values of multiple other keys\n");
209 - printf(" combined with multiple rewrite rules, allows creating complex rules for\n");
210 - printf(" rewriting key values.\n");
211 - printf("\n");
212 - printf(" Up to %d rewriting rules are allowed.\n", MAX_REWRITES);
213 - printf("\n");
214 - printf(" -h, --help\n");
215 - printf(" Display this help and exit.\n");
216 - printf("\n");
217 - printf(" PATTERN\n");
218 - printf(" PATTERN should be a valid PCRE2 regular expression.\n");
219 - printf(" RE2 regular expressions (like the ones usually used in Go applications),\n");
220 - printf(" are usually valid PCRE2 patterns too.\n");
221 - printf(" Regular expressions without named groups are ignored.\n");
222 - printf("\n");
223 - printf("The program accepts all parameters as both --option=value and --option value.\n");
224 - printf("\n");
225 - printf("The maximum line length accepted is %d characters.\n", MAX_LINE_LENGTH);
226 - printf("The maximum number of fields in the PCRE2 pattern is %d.\n", OVECCOUNT / 3);
227 - printf("\n");
228 - printf("PIPELINE AND SEQUENCE OF PROCESSING\n");
229 - printf("\n");
230 - printf("This is a simple diagram of the pipeline taking place:\n");
231 - printf("\n");
232 - printf(" +---------------------------------------------------+\n");
233 - printf(" | INPUT |\n");
234 - printf(" +---------------------------------------------------+\n");
235 - printf(" v v\n");
236 - printf(" +---------------------------------+ |\n");
237 - printf(" | EXTRACT FIELDS AND VALUES | |\n");
238 - printf(" +---------------------------------+ |\n");
239 - printf(" v v |\n");
240 - printf(" +---------------+ | |\n");
241 - printf(" | DUPLICATE | | |\n");
242 - printf(" | create fields | | |\n");
243 - printf(" | with values | | |\n");
244 - printf(" +---------------+ | |\n");
245 - printf(" v v v\n");
246 - printf(" +---------------------------------+ +--------------+\n");
247 - printf(" | REWRITE PIPELINES | | INJECT |\n");
248 - printf(" | altering the values | | constants |\n");
249 - printf(" +---------------------------------+ +--------------+\n");
250 - printf(" v v\n");
251 - printf(" +---------------------------------------------------+\n");
252 - printf(" | OUTPUT |\n");
253 - printf(" +---------------------------------------------------+\n");
254 - printf("\n");
255 - printf("JOURNAL FIELDS RULES (enforced by systemd-journald)\n");
256 - printf("\n");
257 - printf(" - field names can be up to 64 characters\n");
258 - printf(" - the only allowed field characters are A-Z, 0-9 and underscore\n");
259 - printf(" - the first character of fields cannot be a digit\n");
260 - printf(" - protected journal fields start with underscore:\n");
261 - printf(" * they are accepted by systemd-journal-remote\n");
262 - printf(" * they are NOT accepted by a local systemd-journald\n");
263 - printf("\n");
264 - printf(" For best results, always include these fields:\n");
265 - printf("\n");
266 - printf(" MESSAGE=TEXT\n");
267 - printf(" The MESSAGE is the body of the log entry.\n");
268 - printf(" This field is what we usually see in our logs.\n");
269 - printf("\n");
270 - printf(" PRIORITY=NUMBER\n");
271 - printf(" PRIORITY sets the severity of the log entry.\n");
272 - printf(" 0=emerg, 1=alert, 2=crit, 3=err, 4=warn, 5=notice, 6=info, 7=debug\n");
273 - printf(" - Emergency events (0) are usually broadcast to all terminals.\n");
274 - printf(" - Emergency, alert, critical, and error (0-3) are usually colored red.\n");
275 - printf(" - Warning (4) entries are usually colored yellow.\n");
276 - printf(" - Notice (5) entries are usually bold or have a brighter white color.\n");
277 - printf(" - Info (6) entries are the default.\n");
278 - printf(" - Debug (7) entries are usually grayed or dimmed.\n");
279 - printf("\n");
280 - printf(" SYSLOG_IDENTIFIER=NAME\n");
281 - printf(" SYSLOG_IDENTIFIER sets the name of application.\n");
282 - printf(" Use something descriptive, like: SYSLOG_IDENTIFIER=nginx-logs\n");
283 - printf("\n");
284 - printf("You can find the most common fields at 'man systemd.journal-fields'.\n");
285 - printf("\n");
286 - printf("Example YAML file:\n\n"
287 - "--------------------------------------------------------------------------------\n"
288 - "%s"
289 - "--------------------------------------------------------------------------------\n"
290 - "\n",
291 - YAML_CONFIG_NGINX_COMBINED);
292 -}
293 -
294 -// ----------------------------------------------------------------------------
295 -// logging
296 -
297 -// enable the compiler to check for printf like errors on our log2stderr() function
298 -static void log2stderr(const char *format, ...) __attribute__ ((format(__printf__, 1, 2)));
299 -static void log2stderr(const char *format, ...) {
300 - va_list args;
301 - va_start(args, format);
302 - vfprintf(stderr, format, args);
303 - va_end(args);
304 - fprintf(stderr, "\n");
305 -}
306 -
307 -// ----------------------------------------------------------------------------
308 -// allocation functions abstraction
309 -
310 -void *mallocz(size_t size) {
311 - void *ptr = malloc(size);
312 - if (!ptr) {
313 - log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", size);
314 - exit(EXIT_FAILURE);
315 - }
316 - return ptr;
317 -}
318 -
319 -char *strdupz(const char *s) {
320 - char *ptr = strdup(s);
321 - if (!ptr) {
322 - log2stderr("Fatal Error: Memory allocation failed in strdup.");
323 - exit(EXIT_FAILURE);
324 - }
325 - return ptr;
326 -}
327 -
328 -char *strndupz(const char *s, size_t n) {
329 - char *ptr = strndup(s, n);
330 - if (!ptr) {
331 - log2stderr("Fatal Error: Memory allocation failed in strndup. Requested size: %zu bytes.", n);
332 - exit(EXIT_FAILURE);
333 - }
334 - return ptr;
335 -}
336 -
337 -void freez(void *ptr) {
338 - if (ptr)
339 - free(ptr);
340 -}
341 -
342 -// ----------------------------------------------------------------------------
343 -
344 -size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) {
345 - if(dst_size < 2) {
346 - if(dst_size == 1)
347 - *dst = '\0';
348 -
349 - return 0;
350 - }
351 -
352 - if(src_len <= dst_size - 1) {
353 - memcpy(dst, src, src_len);
354 - dst[src_len] = '\0';
355 - return src_len;
356 - }
357 - else {
358 - memcpy(dst, src, dst_size - 1);
359 - dst[dst_size - 1] = '\0';
360 - return dst_size - 1;
361 - }
362 -}
363 -
364 -// ----------------------------------------------------------------------------
365 -
366 -typedef struct txt {
367 - char *s;
368 - size_t size;
369 -} TEXT;
370 -
371 -static void txt_replace(TEXT *txt, const char *s, size_t len) {
372 - if(!s || !*s || len == 0) {
373 - s = "";
374 - len = 0;
375 - }
376 -
377 - if(len + 1 <= txt->size) {
378 - // the existing value allocation, fits our value
379 -
380 - memcpy(txt->s, s, len);
381 - txt->s[len] = '\0';
382 - }
383 - else {
384 - // no existing value allocation, or too small for our value
385 -
386 - if(txt->s)
387 - freez(txt->s);
388 -
389 - txt->s = strndupz(s, len);
390 - txt->size = len + 1;
391 - }
392 -}
393 -
394 -// ----------------------------------------------------------------------------
395 -
396 -typedef struct key_value {
397 - char key[MAX_KEY_LEN + 1];
398 - TEXT value;
399 - bool on_unmatched;
400 -} KEY_VALUE;
401 -
402 -void key_value_replace(KEY_VALUE *kv, const char *key, size_t key_len, const char *value, size_t value_len) {
403 - copy_to_buffer(kv->key, sizeof(kv->key), key, key_len);
404 - txt_replace(&kv->value, value, value_len);
405 -}
406 -
407 -// ----------------------------------------------------------------------------
408 -
409 -struct key_dup {
410 - XXH64_hash_t hash;
411 - char *target;
412 - char *keys[MAX_KEY_DUPS_KEYS];
413 - TEXT values[MAX_KEY_DUPS_KEYS];
414 - size_t used;
415 - bool exposed;
416 -};
417 -
418 -struct replacement_node {
419 - bool is_variable;
420 - const char *s;
421 - size_t len;
422 - struct replacement_node *next;
423 -};
424 -
425 -struct key_rewrite {
426 - XXH64_hash_t hash;
427 - char *key;
428 - char *search_pattern;
429 - char *replace_pattern;
430 - pcre2_code *re;
431 - pcre2_match_data *match_data;
432 - struct replacement_node *nodes;
433 -};
434 -
435 -struct log_job {
436 - bool show_config;
437 -
438 - const char *pattern;
439 -
440 - struct {
441 - const char *key;
442 - char current[FILENAME_MAX + 1];
443 - bool last_line_was_empty;
444 - } filename;
445 -
446 - struct {
447 - KEY_VALUE keys[MAX_INJECTIONS];
448 - size_t used;
449 - } injections;
450 -
451 - struct {
452 - const char *key;
453 - struct {
454 - KEY_VALUE keys[MAX_INJECTIONS];
455 - size_t used;
456 - } injections;
457 - } unmatched;
458 -
459 - struct {
460 - struct key_dup array[MAX_KEY_DUPS];
461 - size_t used;
462 - } dups;
463 -
464 - struct {
465 - struct key_rewrite array[MAX_REWRITES];
466 - size_t used;
467 - } rewrites;
468 -};
469 -
470 -static bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len) {
471 - if(!key || !*key) {
472 - log2stderr("filename key cannot be empty.");
473 - return false;
474 - }
475 -
476 - if(jb->filename.key)
477 - freez((char*)jb->filename.key);
478 -
479 - jb->filename.key = strndupz(key, key_len);
480 -
481 - return true;
482 -}
483 -
484 -static bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {
485 - if (unmatched) {
486 - if (jb->unmatched.injections.used >= MAX_INJECTIONS) {
487 - log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);
488 - return false;
489 - }
490 - }
491 - else {
492 - if (jb->injections.used >= MAX_INJECTIONS) {
493 - log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);
494 - return false;
495 - }
496 - }
497 -
498 - if (unmatched) {
499 - key_value_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],
500 - key, key_len,
501 - value, value_len);
502 - } else {
503 - key_value_replace(&jb->injections.keys[jb->injections.used++],
504 - key, key_len,
505 - value, value_len);
506 - }
507 -
508 - return true;
509 -}
510 -
511 -static bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern) {
512 - pcre2_code *re = jb_compile_pcre2_pattern(search_pattern);
513 - if (!re) {
514 - return false;
515 - }
516 -
517 - struct key_rewrite *rw = &jb->rewrites.array[jb->rewrites.used++];
518 - rw->key = strdupz(key);
519 - rw->hash = XXH3_64bits(rw->key, strlen(rw->key));
520 - rw->search_pattern = strdupz(search_pattern);
521 - rw->replace_pattern = strdupz(replace_pattern);
522 - rw->re = re;
523 - rw->match_data = pcre2_match_data_create_from_pattern(rw->re, NULL);
524 -
525 - // Parse the replacement pattern and create the linked list
526 - if (!parse_replacement_pattern(rw)) {
527 - pcre2_match_data_free(rw->match_data);
528 - pcre2_code_free(rw->re);
529 - freez(rw->key);
530 - freez(rw->search_pattern);
531 - freez(rw->replace_pattern);
532 - jb->rewrites.used--;
533 - return false;
534 - }
535 -
536 - return true;
537 -}
538 -
539 -void jb_cleanup(struct log_job *jb) {
540 - for(size_t i = 0; i < jb->injections.used ;i++) {
541 - if(jb->injections.keys[i].value.s)
542 - freez(jb->injections.keys[i].value.s);
543 - }
544 -
545 - for(size_t i = 0; i < jb->unmatched.injections.used ;i++) {
546 - if(jb->unmatched.injections.keys[i].value.s)
547 - freez(jb->unmatched.injections.keys[i].value.s);
548 - }
549 -
550 - for(size_t i = 0; i < jb->dups.used ;i++) {
551 - struct key_dup *kd = &jb->dups.array[i];
552 -
553 - if(kd->target)
554 - freez(kd->target);
555 -
556 - for(size_t j = 0; j < kd->used ; j++) {
557 - if (kd->keys[j])
558 - freez(kd->keys[j]);
559 -
560 - if (kd->values[j].s)
561 - freez(kd->values[j].s);
562 - }
563 - }
564 -
565 - for(size_t i = 0; i < jb->rewrites.used; i++) {
566 - struct key_rewrite *rw = &jb->rewrites.array[i];
567 -
568 - if (rw->key)
569 - freez(rw->key);
570 -
571 - if (rw->search_pattern)
572 - freez(rw->search_pattern);
573 -
574 - if (rw->replace_pattern)
575 - freez(rw->replace_pattern);
576 -
577 - if(rw->match_data)
578 - pcre2_match_data_free(rw->match_data);
579 -
580 - if (rw->re)
581 - pcre2_code_free(rw->re);
582 -
583 - // Cleanup for replacement nodes linked list
584 - struct replacement_node *current = rw->nodes;
585 - while (current != NULL) {
586 - struct replacement_node *next = current->next;
587 -
588 - if (current->s)
589 - freez((void *)current->s);
590 -
591 - freez(current);
592 - current = next;
593 - }
594 - }
595 -
596 - memset(jb, 0, sizeof(*jb));
597 -}
598 -
599 -// ----------------------------------------------------------------------------
600 -// PCRE2
601 -
602 -static pcre2_code *jb_compile_pcre2_pattern(const char *pattern) {
603 - int error_number;
604 - PCRE2_SIZE error_offset;
605 - PCRE2_SPTR pattern_ptr = (PCRE2_SPTR)pattern;
606 -
607 - pcre2_code *re = pcre2_compile(pattern_ptr, PCRE2_ZERO_TERMINATED, 0, &error_number, &error_offset, NULL);
608 - if (re == NULL) {
609 - PCRE2_UCHAR errbuf[1024];
610 - pcre2_get_error_message(error_number, errbuf, sizeof(errbuf));
611 - log2stderr("PCRE2 compilation failed at offset %d: %s", (int)error_offset, errbuf);
612 - log2stderr("Check for common regex syntax errors or unsupported PCRE2 patterns.");
613 - return NULL;
614 - }
615 -
616 - return re;
617 -}
618 -
619 -static inline bool jb_pcre2_match(pcre2_code *re, pcre2_match_data *match_data, char *line, size_t len, bool log) {
620 - int rc = pcre2_match(re, (PCRE2_SPTR)line, len, 0, 0, match_data, NULL);
621 - if(rc < 0) {
622 - PCRE2_UCHAR errbuf[1024];
623 - pcre2_get_error_message(rc, errbuf, sizeof(errbuf));
624 -
625 - if(log)
626 - log2stderr("PCRE2 error %d: %s on: %s", rc, errbuf, line);
627 -
628 - return false;
629 - }
630 -
631 - return true;
632 -}
633 -
634 -// ----------------------------------------------------------------------------
635 -
636 -static char *rewrite_value(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len) {
637 - static __thread char rewritten_value[MAX_VALUE_LEN + 1];
638 -
639 - for (size_t i = 0; i < jb->rewrites.used; i++) {
640 - struct key_rewrite *rw = &jb->rewrites.array[i];
641 -
642 - if (rw->hash == hash && strcmp(rw->key, key) == 0) {
643 - if (!jb_pcre2_match(rw->re, rw->match_data, (char *)value, value_len, false)) {
644 - continue; // No match found, skip to next rewrite rule
645 - }
646 -
647 - PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(rw->match_data);
648 -
649 - char *buffer = rewritten_value;
650 - size_t buffer_remaining = sizeof(rewritten_value);
651 -
652 - // Iterate through the linked list of replacement nodes
653 - for (struct replacement_node *node = rw->nodes; node != NULL; node = node->next) {
654 - if (node->is_variable) {
655 - uint32_t groupnumber = pcre2_substring_number_from_name(rw->re, (PCRE2_SPTR)node->s);
656 - PCRE2_SIZE start_offset = ovector[2 * groupnumber];
657 - PCRE2_SIZE end_offset = ovector[2 * groupnumber + 1];
658 - PCRE2_SIZE length = end_offset - start_offset;
659 -
660 - size_t copied = copy_to_buffer(buffer, buffer_remaining, value + start_offset, length);
661 - buffer += copied;
662 - buffer_remaining -= copied;
663 - }
664 - else {
665 - size_t len = node->len;
666 - size_t copied = copy_to_buffer(buffer, buffer_remaining, node->s, len);
667 - buffer += copied;
668 - buffer_remaining -= copied;
669 - }
670 - }
671 -
672 - return rewritten_value;
673 - }
674 - }
675 -
676 - return NULL;
677 -}
678 -
679 -// ----------------------------------------------------------------------------
680 -
681 -static inline void send_key_value_error(const char *key, const char *format, ...) __attribute__ ((format(__printf__, 2, 3)));
682 -static inline void send_key_value_error(const char *key, const char *format, ...) {
683 - printf("%s=", key);
684 - va_list args;
685 - va_start(args, format);
686 - vprintf(format, args);
687 - va_end(args);
688 - printf("\n");
689 -}
690 -
691 -static inline void send_key_value_and_rewrite(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t len) {
692 - char *rewritten = rewrite_value(jb, key, hash, value, len);
693 - if(!rewritten)
694 - printf("%s=%.*s\n", key, (int)len, value);
695 - else
696 - printf("%s=%s\n", key, rewritten);
697 -}
698 -
699 -static inline void send_key_value_constant(struct log_job *jb, const char *key, const char *value) {
700 - printf("%s=%s\n", key, value);
701 -}
702 -
703 -// ----------------------------------------------------------------------------
704 -
705 -static struct key_dup *add_duplicate_target_to_job(struct log_job *jb, const char *target, size_t target_len) {
706 - if (jb->dups.used >= MAX_KEY_DUPS) {
707 - log2stderr("Error: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS);
708 - return NULL;
709 - }
710 -
711 - struct key_dup *kd = &jb->dups.array[jb->dups.used++];
712 - kd->target = strndupz(target, target_len);
713 - kd->hash = XXH3_64bits(kd->target, target_len);
714 - kd->used = 0;
715 - kd->exposed = false;
716 -
717 - // Initialize values array
718 - for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) {
719 - kd->values[i].s = NULL;
720 - kd->values[i].size = 0;
721 - }
722 -
723 - return kd;
724 -}
725 -
726 -static bool add_key_to_duplicate(struct key_dup *kd, const char *key, size_t key_len) {
727 - if (kd->used >= MAX_KEY_DUPS_KEYS) {
728 - log2stderr("Error: Too many keys in duplication of target '%s'.", kd->target);
729 - return false;
730 - }
731 -
732 - kd->keys[kd->used++] = strndupz(key, key_len);
733 - return true;
734 -}
735 -
736 -// ----------------------------------------------------------------------------
737 -// yaml configuration file
738 -
739 -#ifdef HAVE_LIBYAML
740 -
741 -
742 -// ----------------------------------------------------------------------------
743 -// yaml library functions
744 -
745 -static const char *yaml_event_name(yaml_event_type_t type) {
746 - switch (type) {
747 - case YAML_NO_EVENT:
748 - return "YAML_NO_EVENT";
749 -
750 - case YAML_SCALAR_EVENT:
751 - return "YAML_SCALAR_EVENT";
752 -
753 - case YAML_ALIAS_EVENT:
754 - return "YAML_ALIAS_EVENT";
755 -
756 - case YAML_MAPPING_START_EVENT:
757 - return "YAML_MAPPING_START_EVENT";
758 -
759 - case YAML_MAPPING_END_EVENT:
760 - return "YAML_MAPPING_END_EVENT";
761 -
762 - case YAML_SEQUENCE_START_EVENT:
763 - return "YAML_SEQUENCE_START_EVENT";
764 -
765 - case YAML_SEQUENCE_END_EVENT:
766 - return "YAML_SEQUENCE_END_EVENT";
767 -
768 - case YAML_STREAM_START_EVENT:
769 - return "YAML_STREAM_START_EVENT";
770 -
771 - case YAML_STREAM_END_EVENT:
772 - return "YAML_STREAM_END_EVENT";
773 -
774 - case YAML_DOCUMENT_START_EVENT:
775 - return "YAML_DOCUMENT_START_EVENT";
776 -
777 - case YAML_DOCUMENT_END_EVENT:
778 - return "YAML_DOCUMENT_END_EVENT";
779 -
780 - default:
781 - return "UNKNOWN";
782 - }
783 -}
784 -
785 -#define yaml_error(parser, event, fmt, args...) yaml_error_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__, fmt, ##args)
786 -static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) __attribute__ ((format(__printf__, 6, 7)));
787 -static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) {
788 - char buf[1024] = ""; // Initialize buf to an empty string
789 - const char *type = "";
790 -
791 - if(event) {
792 - type = yaml_event_name(event->type);
793 -
794 - switch (event->type) {
795 - case YAML_SCALAR_EVENT:
796 - copy_to_buffer(buf, sizeof(buf), (char *)event->data.scalar.value, event->data.scalar.length);
797 - break;
798 -
799 - case YAML_ALIAS_EVENT:
800 - snprintf(buf, sizeof(buf), "%s", event->data.alias.anchor);
801 - break;
802 -
803 - default:
804 - break;
805 - }
806 - }
807 -
808 - fprintf(stderr, "YAML %zu@%s, %s(): (line %d, column %d, %s%s%s): ",
809 - line, file, function,
810 - (int)(parser->mark.line + 1), (int)(parser->mark.column + 1),
811 - type, buf[0]? ", near ": "", buf);
812 -
813 - va_list args;
814 - va_start(args, format);
815 - vfprintf(stderr, format, args);
816 - va_end(args);
817 - fprintf(stderr, "\n");
818 -}
819 -
820 -#define yaml_parse(parser, event) yaml_parse_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__)
821 -static bool yaml_parse_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file) {
822 - if (!yaml_parser_parse(parser, event)) {
823 - yaml_error(parser, NULL, "YAML parser error %d", parser->error);
824 - return false;
825 - }
826 -
827 -// fprintf(stderr, ">>> %s >>> %.*s\n",
828 -// yaml_event_name(event->type),
829 -// event->type == YAML_SCALAR_EVENT ? event->data.scalar.length : 0,
830 -// event->type == YAML_SCALAR_EVENT ? (char *)event->data.scalar.value : "");
831 -
832 - return true;
833 -}
834 -
835 -#define yaml_parse_expect_event(parser, type) yaml_parse_expect_event_with_trace(parser, type, __LINE__, __FUNCTION__, __FILE__)
836 -static bool yaml_parse_expect_event_with_trace(yaml_parser_t *parser, yaml_event_type_t type, size_t line, const char *function, const char *file) {
837 - yaml_event_t event;
838 - if (!yaml_parse(parser, &event))
839 - return false;
840 -
841 - bool ret = true;
842 - if(event.type != type) {
843 - yaml_error_with_trace(parser, &event, line, function, file, "unexpected event - expecting: %s", yaml_event_name(type));
844 - ret = false;
845 - }
846 -// else
847 -// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function);
848 -
849 - yaml_event_delete(&event);
850 - return ret;
851 -}
852 -
853 -#define yaml_scalar_matches(event, s, len) yaml_scalar_matches_with_trace(event, s, len, __LINE__, __FUNCTION__, __FILE__)
854 -static bool yaml_scalar_matches_with_trace(yaml_event_t *event, const char *s, size_t len, size_t line __maybe_unused, const char *function __maybe_unused, const char *file __maybe_unused) {
855 - if(event->type != YAML_SCALAR_EVENT)
856 - return false;
857 -
858 - if(len != event->data.scalar.length)
859 - return false;
860 -// else
861 -// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function);
862 -
863 - return strcmp((char *)event->data.scalar.value, s) == 0;
864 -}
865 -
866 -// ----------------------------------------------------------------------------
867 -
868 -static struct key_dup *yaml_parse_duplicate_key(struct log_job *jb, yaml_parser_t *parser) {
869 - yaml_event_t event;
870 -
871 - if (!yaml_parse(parser, &event))
872 - return false;
873 -
874 - struct key_dup *kd = NULL;
875 - if(event.type == YAML_SCALAR_EVENT) {
876 - kd = add_duplicate_target_to_job(jb, (char *)event.data.scalar.value, event.data.scalar.length);
877 - }
878 - else
879 - yaml_error(parser, &event, "duplicate key must be a scalar.");
880 -
881 - yaml_event_delete(&event);
882 - return kd;
883 -}
884 -
885 -static size_t yaml_parse_duplicate_from(struct log_job *jb, yaml_parser_t *parser, struct key_dup *kd) {
886 - size_t errors = 0;
887 - yaml_event_t event;
888 -
889 - if (!yaml_parse(parser, &event))
890 - return 1;
891 -
892 - bool ret = true;
893 - if(event.type == YAML_SCALAR_EVENT)
894 - ret = add_key_to_duplicate(kd, (char *)event.data.scalar.value, event.data.scalar.length);
895 -
896 - else if(event.type == YAML_SEQUENCE_START_EVENT) {
897 - bool finished = false;
898 - while(!errors && !finished) {
899 - yaml_event_t sub_event;
900 - if (!yaml_parse(parser, &sub_event))
901 - return errors++;
902 - else {
903 - if (sub_event.type == YAML_SCALAR_EVENT)
904 - add_key_to_duplicate(kd, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
905 -
906 - else if (sub_event.type == YAML_SEQUENCE_END_EVENT)
907 - finished = true;
908 -
909 - yaml_event_delete(&sub_event);
910 - }
911 - }
912 - }
913 - else
914 - yaml_error(parser, &event, "not expected event type");
915 -
916 - yaml_event_delete(&event);
917 - return errors;
918 -}
919 -
920 -static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_job *jb) {
921 - yaml_event_t event;
922 - size_t errors = 0;
923 -
924 - if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))
925 - return 1;
926 -
927 - if (!yaml_parse(parser, &event))
928 - return 1;
929 -
930 - if (yaml_scalar_matches(&event, "key", strlen("key"))) {
931 - yaml_event_t sub_event;
932 - if (!yaml_parse(parser, &sub_event))
933 - errors++;
934 -
935 - else {
936 - if (event.type == YAML_SCALAR_EVENT) {
937 - if(!log_job_add_filename_key(jb, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length))
938 - errors++;
939 - }
940 -
941 - else {
942 - yaml_error(parser, &sub_event, "expected the filename as %s", yaml_event_name(YAML_SCALAR_EVENT));
943 - errors++;
944 - }
945 -
946 - yaml_event_delete(&sub_event);
947 - }
948 - }
949 -
950 - if(!yaml_parse_expect_event(parser, YAML_MAPPING_END_EVENT))
951 - errors++;
952 -
953 - yaml_event_delete(&event);
954 - return errors;
955 -}
956 -
957 -static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, struct log_job *jb) {
958 - if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
959 - return 1;
960 -
961 - struct key_dup *kd = NULL;
962 -
963 - // Expecting a key-value pair for each duplicate
964 - bool finished;
965 - size_t errors = 0;
966 - while (!errors && !finished) {
967 - yaml_event_t event;
968 - if (!yaml_parse(parser, &event)) {
969 - errors++;
970 - break;
971 - }
972 -
973 - if(event.type == YAML_MAPPING_START_EVENT) {
974 - ;
975 - }
976 - if (event.type == YAML_SEQUENCE_END_EVENT) {
977 - finished = true;
978 - }
979 - else if(event.type == YAML_SCALAR_EVENT) {
980 - if (yaml_scalar_matches(&event, "key", strlen("key"))) {
981 - kd = yaml_parse_duplicate_key(jb, parser);
982 - if (!kd)
983 - errors++;
984 - else {
985 - while (!errors && kd) {
986 - yaml_event_t sub_event;
987 - if (!yaml_parse(parser, &sub_event)) {
988 - errors++;
989 - break;
990 - }
991 -
992 - if (sub_event.type == YAML_MAPPING_END_EVENT) {
993 - kd = NULL;
994 - } else if (sub_event.type == YAML_SCALAR_EVENT) {
995 - if (yaml_scalar_matches(&sub_event, "values_of", strlen("values_of"))) {
996 - if (!kd) {
997 - yaml_error(parser, &sub_event, "Found 'values_of' but the 'key' is not set.");
998 - errors++;
999 - } else
1000 - errors += yaml_parse_duplicate_from(jb, parser, kd);
1001 - } else {
1002 - yaml_error(parser, &sub_event, "unknown scalar");
1003 - errors++;
1004 - }
1005 - } else {
1006 - yaml_error(parser, &sub_event, "unexpected event type");
1007 - errors++;
1008 - }
1009 -
1010 - // Delete the event after processing
1011 - yaml_event_delete(&event);
1012 - }
1013 - }
1014 - } else {
1015 - yaml_error(parser, &event, "unknown scalar");
1016 - errors++;
1017 - }
1018 - }
1019 -
1020 - yaml_event_delete(&event);
1021 - }
1022 -
1023 - return errors;
1024 -}
1025 -
1026 -static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
1027 - yaml_event_t event;
1028 - if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
1029 - yaml_error(parser, &event, "Expected scalar for constant field injection key");
1030 - yaml_event_delete(&event);
1031 - return false;
1032 - }
1033 -
1034 - char *key = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
1035 - char *value = NULL;
1036 - bool ret = false;
1037 -
1038 - yaml_event_delete(&event);
1039 -
1040 - if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
1041 - yaml_error(parser, &event, "Expected scalar for constant field injection value");
1042 - goto cleanup;
1043 - }
1044 -
1045 - if(!yaml_scalar_matches(&event, "value", strlen("value"))) {
1046 - yaml_error(parser, &event, "Expected scalar 'value'");
1047 - goto cleanup;
1048 - }
1049 -
1050 - if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
1051 - yaml_error(parser, &event, "Expected scalar for constant field injection value");
1052 - goto cleanup;
1053 - }
1054 -
1055 - value = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
1056 -
1057 - if(!log_job_add_injection(jb, key, strlen(key), value, strlen(value), unmatched))
1058 - ret = false;
1059 - else
1060 - ret = true;
1061 -
1062 - ret = true;
1063 -
1064 -cleanup:
1065 - yaml_event_delete(&event);
1066 - freez(key);
1067 - freez(value);
1068 - return !ret ? 1 : 0;
1069 -}
1070 -
1071 -static bool yaml_parse_injection_mapping(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
1072 - yaml_event_t event;
1073 - size_t errors = 0;
1074 - bool finished = false;
1075 -
1076 - while (!errors && !finished) {
1077 - if (!yaml_parse(parser, &event)) {
1078 - errors++;
1079 - continue;
1080 - }
1081 -
1082 - switch (event.type) {
1083 - case YAML_SCALAR_EVENT:
1084 - if (yaml_scalar_matches(&event, "key", strlen("key"))) {
1085 - errors += yaml_parse_constant_field_injection(parser, jb, unmatched);
1086 - } else {
1087 - yaml_error(parser, &event, "Unexpected scalar in injection mapping");
1088 - errors++;
1089 - }
1090 - break;
1091 -
1092 - case YAML_MAPPING_END_EVENT:
1093 - finished = true;
1094 - break;
1095 -
1096 - default:
1097 - yaml_error(parser, &event, "Unexpected event in injection mapping");
1098 - errors++;
1099 - break;
1100 - }
1101 -
1102 - yaml_event_delete(&event);
1103 - }
1104 -
1105 - return errors == 0;
1106 -}
1107 -
1108 -static size_t yaml_parse_injections(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
1109 - yaml_event_t event;
1110 - size_t errors = 0;
1111 - bool finished = false;
1112 -
1113 - if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
1114 - return 1;
1115 -
1116 - while (!errors && !finished) {
1117 - if (!yaml_parse(parser, &event)) {
1118 - errors++;
1119 - continue;
1120 - }
1121 -
1122 - switch (event.type) {
1123 - case YAML_MAPPING_START_EVENT:
1124 - if (!yaml_parse_injection_mapping(parser, jb, unmatched))
1125 - errors++;
1126 - break;
1127 -
1128 - case YAML_SEQUENCE_END_EVENT:
1129 - finished = true;
1130 - break;
1131 -
1132 - default:
1133 - yaml_error(parser, &event, "Unexpected event in injections sequence");
1134 - errors++;
1135 - break;
1136 - }
1137 -
1138 - yaml_event_delete(&event);
1139 - }
1140 -
1141 - return errors;
1142 -}
1143 -
1144 -static size_t yaml_parse_unmatched(yaml_parser_t *parser, struct log_job *jb) {
1145 - size_t errors = 0;
1146 - bool finished = false;
1147 -
1148 - if (!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))
1149 - return 1;
1150 -
1151 - while (!errors && !finished) {
1152 - yaml_event_t event;
1153 - if (!yaml_parse(parser, &event)) {
1154 - errors++;
1155 - continue;
1156 - }
1157 -
1158 - switch (event.type) {
1159 - case YAML_SCALAR_EVENT:
1160 - if (yaml_scalar_matches(&event, "key", strlen("key"))) {
1161 - yaml_event_t sub_event;
1162 - if (!yaml_parse(parser, &sub_event)) {
1163 - errors++;
1164 - } else {
1165 - if (sub_event.type == YAML_SCALAR_EVENT) {
1166 - jb->unmatched.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
1167 - } else {
1168 - yaml_error(parser, &sub_event, "expected a scalar value for 'key'");
1169 - errors++;
1170 - }
1171 - yaml_event_delete(&sub_event);
1172 - }
1173 - } else if (yaml_scalar_matches(&event, "inject", strlen("inject"))) {
1174 - errors += yaml_parse_injections(parser, jb, true);
1175 - } else {
1176 - yaml_error(parser, &event, "Unexpected scalar in unmatched section");
1177 - errors++;
1178 - }
1179 - break;
1180 -
1181 - case YAML_MAPPING_END_EVENT:
1182 - finished = true;
1183 - break;
1184 -
1185 - default:
1186 - yaml_error(parser, &event, "Unexpected event in unmatched section");
1187 - errors++;
1188 - break;
1189 - }
1190 -
1191 - yaml_event_delete(&event);
1192 - }
1193 -
1194 - return errors;
1195 -}
1196 -
1197 -static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
1198 - size_t errors = 0;
1199 -
1200 - if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
1201 - return 1;
1202 -
1203 - bool finished = false;
1204 - while (!errors && !finished) {
1205 - yaml_event_t event;
1206 - if (!yaml_parse(parser, &event)) {
1207 - errors++;
1208 - continue;
1209 - }
1210 -
1211 - switch (event.type) {
1212 - case YAML_MAPPING_START_EVENT:
1213 - {
1214 - struct key_rewrite rw = {0};
1215 -
1216 - bool mapping_finished = false;
1217 - while (!errors && !mapping_finished) {
1218 - yaml_event_t sub_event;
1219 - if (!yaml_parse(parser, &sub_event)) {
1220 - errors++;
1221 - continue;
1222 - }
1223 -
1224 - switch (sub_event.type) {
1225 - case YAML_SCALAR_EVENT:
1226 - if (yaml_scalar_matches(&sub_event, "key", strlen("key"))) {
1227 - if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
1228 - yaml_error(parser, &sub_event, "Expected scalar for rewrite key");
1229 - errors++;
1230 - } else {
1231 - rw.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
1232 - yaml_event_delete(&sub_event);
1233 - }
1234 - } else if (yaml_scalar_matches(&sub_event, "search", strlen("search"))) {
1235 - if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
1236 - yaml_error(parser, &sub_event, "Expected scalar for rewrite search pattern");
1237 - errors++;
1238 - } else {
1239 - rw.search_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
1240 - yaml_event_delete(&sub_event);
1241 - }
1242 - } else if (yaml_scalar_matches(&sub_event, "replace", strlen("replace"))) {
1243 - if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
1244 - yaml_error(parser, &sub_event, "Expected scalar for rewrite replace pattern");
1245 - errors++;
1246 - } else {
1247 - rw.replace_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
1248 - yaml_event_delete(&sub_event);
1249 - }
1250 - } else {
1251 - yaml_error(parser, &sub_event, "Unexpected scalar in rewrite mapping");
1252 - errors++;
1253 - }
1254 - break;
1255 -
1256 - case YAML_MAPPING_END_EVENT:
1257 - if(rw.key && rw.search_pattern && rw.replace_pattern) {
1258 - if (!log_job_add_rewrite(jb, rw.key, rw.search_pattern, rw.replace_pattern))
1259 - errors++;
1260 - }
1261 - freez(rw.key);
1262 - freez(rw.search_pattern);
1263 - freez(rw.replace_pattern);
1264 - memset(&rw, 0, sizeof(rw));
1265 -
1266 - mapping_finished = true;
1267 - break;
1268 -
1269 - default:
1270 - yaml_error(parser, &sub_event, "Unexpected event in rewrite mapping");
1271 - errors++;
1272 - break;
1273 - }
1274 -
1275 - yaml_event_delete(&sub_event);
1276 - }
1277 - }
1278 - break;
1279 -
1280 - case YAML_SEQUENCE_END_EVENT:
1281 - finished = true;
1282 - break;
1283 -
1284 - default:
1285 - yaml_error(parser, &event, "Unexpected event in rewrites sequence");
1286 - errors++;
1287 - break;
1288 - }
1289 -
1290 - yaml_event_delete(&event);
1291 - }
1292 -
1293 - return errors;
1294 -}
1295 -
1296 -static size_t yaml_parse_pattern(yaml_parser_t *parser, struct log_job *jb) {
1297 - yaml_event_t event;
1298 - size_t errors = 0;
1299 -
1300 - if (!yaml_parse(parser, &event))
1301 - return 1;
1302 -
1303 - if(event.type == YAML_SCALAR_EVENT)
1304 - jb->pattern = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
1305 - else {
1306 - yaml_error(parser, &event, "unexpected event type");
1307 - errors++;
1308 - }
1309 -
1310 - yaml_event_delete(&event);
1311 - return errors;
1312 -}
1313 -
1314 -static size_t yaml_parse_initialized(yaml_parser_t *parser, struct log_job *jb) {
1315 - size_t errors = 0;
1316 -
1317 - if(!yaml_parse_expect_event(parser, YAML_STREAM_START_EVENT)) {
1318 - errors++;
1319 - goto cleanup;
1320 - }
1321 -
1322 - if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_START_EVENT)) {
1323 - errors++;
1324 - goto cleanup;
1325 - }
1326 -
1327 - if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT)) {
1328 - errors++;
1329 - goto cleanup;
1330 - }
1331 -
1332 - bool finished = false;
1333 - while (!errors && !finished) {
1334 - yaml_event_t event;
1335 - if(!yaml_parse(parser, &event)) {
1336 - errors++;
1337 - continue;
1338 - }
1339 -
1340 - switch(event.type) {
1341 - default:
1342 - yaml_error(parser, &event, "unexpected type");
1343 - errors++;
1344 - break;
1345 -
1346 - case YAML_MAPPING_END_EVENT:
1347 - finished = true;
1348 - break;
1349 -
1350 - case YAML_SCALAR_EVENT:
1351 - if (yaml_scalar_matches(&event, "pattern", strlen("pattern")))
1352 - errors += yaml_parse_pattern(parser, jb);
1353 -
1354 - else if (yaml_scalar_matches(&event, "filename", strlen("filename")))
1355 - errors += yaml_parse_filename_injection(parser, jb);
1356 -
1357 - else if (yaml_scalar_matches(&event, "duplicate", strlen("duplicate")))
1358 - errors += yaml_parse_duplicates_injection(parser, jb);
1359 -
1360 - else if (yaml_scalar_matches(&event, "inject", strlen("inject")))
1361 - errors += yaml_parse_injections(parser, jb, false);
1362 -
1363 - else if (yaml_scalar_matches(&event, "unmatched", strlen("unmatched")))
1364 - errors += yaml_parse_unmatched(parser, jb);
1365 -
1366 - else if (yaml_scalar_matches(&event, "rewrite", strlen("rewrite")))
1367 - errors += yaml_parse_rewrites(parser, jb);
1368 -
1369 - else {
1370 - yaml_error(parser, &event, "unexpected scalar");
1371 - errors++;
1372 - }
1373 - break;
1374 - }
1375 -
1376 - yaml_event_delete(&event);
1377 - }
1378 -
1379 - if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_END_EVENT)) {
1380 - errors++;
1381 - goto cleanup;
1382 - }
1383 -
1384 - if(!yaml_parse_expect_event(parser, YAML_STREAM_END_EVENT)) {
1385 - errors++;
1386 - goto cleanup;
1387 - }
1388 -
1389 -cleanup:
1390 - return errors;
1391 -}
1392 -
1393 -static bool yaml_parse_file(const char *config_file_path, struct log_job *jb) {
1394 - if(!config_file_path || !*config_file_path) {
1395 - log2stderr("yaml configuration filename cannot be empty.");
1396 - return false;
1397 - }
1398 -
1399 - FILE *fp = fopen(config_file_path, "r");
1400 - if (!fp) {
1401 - log2stderr("Error opening config file: %s", config_file_path);
1402 - return false;
1403 - }
1404 -
1405 - yaml_parser_t parser;
1406 - yaml_parser_initialize(&parser);
1407 - yaml_parser_set_input_file(&parser, fp);
1408 -
1409 - size_t errors = yaml_parse_initialized(&parser, jb);
1410 -
1411 - yaml_parser_delete(&parser);
1412 - fclose(fp);
1413 - return errors == 0;
1414 -}
1415 -
1416 -static bool yaml_parse_config(const char *config_name, struct log_job *jb) {
1417 -
1418 - const char *config = NULL;
1419 -
1420 - if(strcmp(config_name, "nginx-combined") == 0)
1421 - config = YAML_CONFIG_NGINX_COMBINED;
1422 - else {
1423 - log2stderr("Unknown configuration: '%s'", config_name);
1424 - return false;
1425 - }
1426 -
1427 - yaml_parser_t parser;
1428 - yaml_parser_initialize(&parser);
1429 - yaml_parser_set_input_string(&parser, (const unsigned char *)config, strlen(config));
1430 -
1431 - size_t errors = yaml_parse_initialized(&parser, jb);
1432 -
1433 - yaml_parser_delete(&parser);
1434 - return errors == 0;
1435 -}
1436 -
1437 -#endif
1438 -
1439 -
1440 -// ----------------------------------------------------------------------------
1441 -// command line params
1442 -
1443 -struct replacement_node *add_replacement_node(struct replacement_node **head, bool is_variable, const char *text) {
1444 - struct replacement_node *new_node = mallocz(sizeof(struct replacement_node));
1445 - if (!new_node)
1446 - return NULL;
1447 -
1448 - new_node->is_variable = is_variable;
1449 - new_node->s = text;
1450 - new_node->len = strlen(text);
1451 - new_node->next = NULL;
1452 -
1453 - if (*head == NULL)
1454 - *head = new_node;
1455 -
1456 - else {
1457 - struct replacement_node *current = *head;
1458 -
1459 - // append it
1460 - while (current->next != NULL)
1461 - current = current->next;
1462 -
1463 - current->next = new_node;
1464 - }
1465 -
1466 - return new_node;
1467 -}
1468 -
1469 -static bool parse_replacement_pattern(struct key_rewrite *rw) {
1470 - const char *current = rw->replace_pattern;
1471 -
1472 - while (*current != '\0') {
1473 - if (*current == '$' && *(current + 1) == '{') {
1474 - // Start of a variable
1475 - const char *end = strchr(current, '}');
1476 - if (!end) {
1477 - log2stderr("Error: Missing closing brace in replacement pattern: %s", rw->replace_pattern);
1478 - return false;
1479 - }
1480 -
1481 - size_t name_length = end - current - 2; // Length of the variable name
1482 - char *variable_name = strndupz(current + 2, name_length);
1483 - if (!variable_name) {
1484 - log2stderr("Error: Memory allocation failed for variable name.");
1485 - return false;
1486 - }
1487 -
1488 - struct replacement_node *node = add_replacement_node(&(rw->nodes), true, variable_name);
1489 - if (!node) {
1490 - freez(variable_name);
1491 - log2stderr("Error: Failed to add replacement node for variable.");
1492 - return false;
1493 - }
1494 -
1495 - current = end + 1; // Move past the variable
1496 - }
1497 - else {
1498 - // Start of literal text
1499 - const char *start = current;
1500 - while (*current != '\0' && !(*current == '$' && *(current + 1) == '{')) {
1501 - current++;
1502 - }
1503 -
1504 - size_t text_length = current - start;
1505 - char *text = strndupz(start, text_length);
1506 - if (!text) {
1507 - log2stderr("Error: Memory allocation failed for literal text.");
1508 - return false;
1509 - }
1510 -
1511 - struct replacement_node *node = add_replacement_node(&(rw->nodes), false, text);
1512 - if (!node) {
1513 - freez(text);
1514 - log2stderr("Error: Failed to add replacement node for text.");
1515 - return false;
1516 - }
1517 - }
1518 - }
1519 -
1520 - return true;
1521 -}
1522 -
1523 -static bool is_symbol(char c) {
1524 - return !isalpha(c) && !isdigit(c) && !iscntrl(c);
1525 -}
1526 -
1527 -static bool parse_rewrite(struct log_job *jb, const char *param) {
1528 - // Search for '=' in param
1529 - const char *equal_sign = strchr(param, '=');
1530 - if (!equal_sign || equal_sign == param) {
1531 - log2stderr("Error: Invalid rewrite format, '=' not found or at the start in %s", param);
1532 - return false;
1533 - }
1534 -
1535 - // Get the next character as the separator
1536 - char separator = *(equal_sign + 1);
1537 - if (!separator || !is_symbol(separator)) {
1538 - log2stderr("Error: rewrite separator not found after '=', or is not one of /\\|-# in: %s", param);
1539 - return false;
1540 - }
1541 -
1542 - // Find the next occurrence of the separator
1543 - const char *second_separator = strchr(equal_sign + 2, separator);
1544 - if (!second_separator) {
1545 - log2stderr("Error: rewrite second separator not found in: %s", param);
1546 - return false;
1547 - }
1548 -
1549 - // Check if the search pattern is empty
1550 - if (equal_sign + 1 == second_separator) {
1551 - log2stderr("Error: rewrite search pattern is empty in: %s", param);
1552 - return false;
1553 - }
1554 -
1555 - // Check if the replacement pattern is empty
1556 - if (*(second_separator + 1) == '\0') {
1557 - log2stderr("Error: rewrite replacement pattern is empty in: %s", param);
1558 - return false;
1559 - }
1560 -
1561 - // Reserve a slot in rewrites
1562 - if (jb->rewrites.used >= MAX_REWRITES) {
1563 - log2stderr("Error: Exceeded maximum number of rewrite rules, while processing: %s", param);
1564 - return false;
1565 - }
1566 -
1567 - // Extract key, search pattern, and replacement pattern
1568 - char *key = strndupz(param, equal_sign - param);
1569 - char *search_pattern = strndupz(equal_sign + 2, second_separator - (equal_sign + 2));
1570 - char *replace_pattern = strdupz(second_separator + 1);
1571 -
1572 - bool ret = log_job_add_rewrite(jb, key, search_pattern, replace_pattern);
1573 -
1574 - freez(key);
1575 - freez(search_pattern);
1576 - freez(replace_pattern);
1577 -
1578 - return ret;
1579 -}
1580 -
1581 -static bool parse_inject(struct log_job *jb, const char *value, bool unmatched) {
1582 - const char *equal = strchr(value, '=');
1583 - if (!equal) {
1584 - log2stderr("Error: injection '%s' does not have an equal sign.", value);
1585 - return false;
1586 - }
1587 -
1588 - const char *key = value;
1589 - const char *val = equal + 1;
1590 - log_job_add_injection(jb, key, equal - key, val, strlen(val), unmatched);
1591 -
1592 - return true;
1593 -}
1594 -
1595 -static bool parse_duplicate(struct log_job *jb, const char *value) {
1596 - const char *target = value;
1597 - const char *equal_sign = strchr(value, '=');
1598 - if (!equal_sign || equal_sign == target) {
1599 - log2stderr("Error: Invalid duplicate format, '=' not found or at the start in %s", value);
1600 - return false;
1601 - }
1602 -
1603 - size_t target_len = equal_sign - target;
1604 - struct key_dup *kd = add_duplicate_target_to_job(jb, target, target_len);
1605 - if(!kd) return false;
1606 -
1607 - const char *key = equal_sign + 1;
1608 - while (key) {
1609 - if (kd->used >= MAX_KEY_DUPS_KEYS) {
1610 - log2stderr("Error: too many keys in duplication of target '%s'.", kd->target);
1611 - return false;
1612 - }
1613 -
1614 - const char *comma = strchr(key, ',');
1615 - size_t key_len;
1616 - if (comma) {
1617 - key_len = comma - key;
1618 - add_key_to_duplicate(kd, key, key_len);
1619 - key = comma + 1;
1620 - }
1621 - else {
1622 - add_key_to_duplicate(kd, key, strlen(key));
1623 - break; // No more keys
1624 - }
1625 - }
1626 -
1627 - return true;
1628 -}
1629 -
1630 -bool parse_parameters(struct log_job *jb, int argc, char **argv) {
1631 - for (int i = 1; i < argc; i++) {
1632 - char *arg = argv[i];
1633 - if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) {
1634 - display_help(argv[0]);
1635 - exit(0);
1636 - }
1637 - else if (strcmp(arg, "--show-config") == 0) {
1638 - jb->show_config = true;
1639 - }
1640 - else {
1641 - char buffer[1024];
1642 - char *param = NULL;
1643 - char *value = NULL;
1644 -
1645 - char *equal_sign = strchr(arg, '=');
1646 - if (equal_sign) {
1647 - copy_to_buffer(buffer, sizeof(buffer), arg, equal_sign - arg);
1648 - param = buffer;
1649 - value = equal_sign + 1;
1650 - }
1651 - else {
1652 - param = arg;
1653 - if (i + 1 < argc) {
1654 - value = argv[++i];
1655 - }
1656 - else {
1657 - if (!jb->pattern) {
1658 - jb->pattern = arg;
1659 - continue;
1660 - } else {
1661 - log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
1662 - return false;
1663 - }
1664 - }
1665 - }
1666 -
1667 - if (strcmp(param, "--filename-key") == 0) {
1668 - if(!log_job_add_filename_key(jb, value, value ? strlen(value) : 0))
1669 - return false;
1670 - }
1671 -#ifdef HAVE_LIBYAML
1672 - else if (strcmp(param, "-f") == 0 || strcmp(param, "--file") == 0) {
1673 - if (!yaml_parse_file(value, jb))
1674 - return false;
1675 - }
1676 - else if (strcmp(param, "--config") == 0) {
1677 - if (!yaml_parse_config(value, jb))
1678 - return false;
1679 - }
1680 -#endif
1681 - else if (strcmp(param, "--unmatched-key") == 0)
1682 - jb->unmatched.key = value;
1683 - else if (strcmp(param, "--duplicate") == 0) {
1684 - if (!parse_duplicate(jb, value))
1685 - return false;
1686 - }
1687 - else if (strcmp(param, "--inject") == 0) {
1688 - if (!parse_inject(jb, value, false))
1689 - return false;
1690 - }
1691 - else if (strcmp(param, "--inject-unmatched") == 0) {
1692 - if (!parse_inject(jb, value, true))
1693 - return false;
1694 - }
1695 - else if (strcmp(param, "--rewrite") == 0) {
1696 - if (!parse_rewrite(jb, value))
1697 - return false;
1698 - }
1699 - else {
1700 - if (!jb->pattern) {
1701 - jb->pattern = arg;
1702 - continue;
1703 - } else {
1704 - log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
1705 - return false;
1706 - }
1707 - }
1708 - }
1709 - }
1710 -
1711 - // Check if a pattern is set and exactly one pattern is specified
1712 - if (!jb->pattern) {
1713 - log2stderr("Error: Pattern not specified.");
1714 - display_help(argv[0]);
1715 - return false;
1716 - }
1717 -
1718 - return true;
1719 -}
1720 -
1721 -// ----------------------------------------------------------------------------
1722 -// injection of constant fields
1723 -
1724 -static void jb_select_which_injections_should_be_injected_on_unmatched(struct log_job *jb) {
1725 - // mark all injections to be added to unmatched logs
1726 - for(size_t i = 0; i < jb->injections.used ; i++)
1727 - jb->injections.keys[i].on_unmatched = true;
1728 -
1729 - if(jb->injections.used && jb->unmatched.injections.used) {
1730 - // we have both injections and injections on unmatched
1731 -
1732 - // we find all the injections that are also configured as injections on unmatched,
1733 - // and we disable them, so that the output will not have the same key twice
1734 -
1735 - for(size_t i = 0; i < jb->injections.used ;i++) {
1736 - for(size_t u = 0; u < jb->unmatched.injections.used ; u++) {
1737 - if(strcmp(jb->injections.keys[i].key, jb->unmatched.injections.keys[u].key) == 0)
1738 - jb->injections.keys[i].on_unmatched = false;
1739 - }
1740 - }
1741 - }
1742 -}
1743 -
1744 -
1745 -static inline void jb_finalize_injections(struct log_job *jb, bool line_is_matched) {
1746 - for (size_t j = 0; j < jb->injections.used; j++) {
1747 - if(!line_is_matched && !jb->injections.keys[j].on_unmatched)
1748 - continue;
1749 -
1750 - send_key_value_constant(jb, jb->injections.keys[j].key, jb->injections.keys[j].value.s);
1751 - }
1752 -}
1753 -
1754 -static inline void jb_reset_injections(struct log_job *jb) {
1755 - for(size_t d = 0; d < jb->dups.used ; d++) {
1756 - struct key_dup *kd = &jb->dups.array[d];
1757 - kd->exposed = false;
1758 -
1759 - for(size_t g = 0; g < kd->used ; g++) {
1760 - if(kd->values[g].s)
1761 - kd->values[g].s[0] = '\0';
1762 - }
1763 - }
1764 -}
1765 -
1766 -// ----------------------------------------------------------------------------
1767 -// duplications
1768 -
1769 -static inline void jb_send_duplications_for_key(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len) {
1770 - // IMPORTANT:
1771 - // The 'value' may not be NULL terminated and have more data that the value we need
1772 -
1773 - for (size_t d = 0; d < jb->dups.used; d++) {
1774 - struct key_dup *kd = &jb->dups.array[d];
1775 -
1776 - if(kd->exposed || kd->used == 0)
1777 - continue;
1778 -
1779 - if(kd->used == 1) {
1780 - // just one key to be duplicated
1781 - if(strcmp(kd->keys[0], key) == 0) {
1782 - send_key_value_and_rewrite(jb, kd->target, kd->hash, value, value_len);
1783 - kd->exposed = true;
1784 - }
1785 - }
1786 - else {
1787 - // multiple keys to be duplicated
1788 - for(size_t g = 0; g < kd->used ; g++) {
1789 - if(strcmp(kd->keys[g], key) == 0)
1790 - txt_replace(&kd->values[g], value, value_len);
1791 - }
1792 - }
1793 - }
1794 -}
1795 -
1796 -static inline void jb_send_remaining_duplications(struct log_job *jb) {
1797 - static __thread char buffer[MAX_VALUE_LEN + 1];
1798 -
1799 - // IMPORTANT:
1800 - // all duplications are exposed, even the ones we haven't found their keys in the source,
1801 - // so that the output always has the same fields for matched entries.
1802 -
1803 - for(size_t d = 0; d < jb->dups.used ; d++) {
1804 - struct key_dup *kd = &jb->dups.array[d];
1805 -
1806 - if(kd->exposed || kd->used == 0)
1807 - continue;
1808 -
1809 - buffer[0] = '\0';
1810 - size_t remaining = sizeof(buffer);
1811 - char *s = buffer;
1812 -
1813 - for(size_t g = 0; g < kd->used ; g++) {
1814 - if(remaining < 2) {
1815 - log2stderr("Warning: duplicated key '%s' cannot fit the values.", kd->target);
1816 - break;
1817 - }
1818 -
1819 - if(g > 0) {
1820 - *s++ = ',';
1821 - *s = '\0';
1822 - remaining--;
1823 - }
1824 -
1825 - char *value = (kd->values[g].s && kd->values[g].s[0]) ? kd->values[g].s : "[unavailable]";
1826 - size_t len = strlen(value);
1827 - size_t copied = copy_to_buffer(s, remaining, value, len);
1828 - remaining -= copied;
1829 - s += copied;
1830 -
1831 - if(copied != len) {
1832 - log2stderr("Warning: duplicated key '%s' will have truncated value", jb->dups.array[d].target);
1833 - break;
1834 - }
1835 - }
1836 - send_key_value_and_rewrite(jb, kd->target, kd->hash, buffer, s - buffer);
1837 - }
1838 -}
1839 -
1840 -// ----------------------------------------------------------------------------
1841 -// filename injection
1842 -
1843 -static inline void jb_inject_filename(struct log_job *jb) {
1844 - if (jb->filename.key && jb->filename.current[0])
1845 - send_key_value_constant(jb, jb->filename.key, jb->filename.current);
1846 -}
1847 -
1848 -static inline bool jb_switched_filename(struct log_job *jb, const char *line, size_t len) {
1849 - // IMPORTANT:
1850 - // Return TRUE when the caller should skip this line (because it is ours).
1851 - // Unfortunately, we have to consume empty lines too.
1852 -
1853 - // IMPORTANT:
1854 - // filename may not be NULL terminated and have more data than the filename.
1855 -
1856 - if (!len) {
1857 - jb->filename.last_line_was_empty = true;
1858 - return true;
1859 - }
1860 -
1861 - // Check if it's a log file change line
1862 - if (jb->filename.last_line_was_empty && line[0] == '=' && strncmp(line, "==> ", 4) == 0) {
1863 - const char *start = line + 4;
1864 - const char *end = strstr(line, " <==");
1865 - while (*start == ' ') start++;
1866 - if (*start != '\n' && *start != '\0' && end) {
1867 - copy_to_buffer(jb->filename.current, sizeof(jb->filename.current),
1868 - start, end - start);
1869 - return true;
1870 - }
1871 - }
1872 -
1873 - jb->filename.last_line_was_empty = false;
1874 - return false;
1875 -}
1876 -
1877 -// ----------------------------------------------------------------------------
1878 -// input reading
1879 -
1880 -static char *get_next_line(struct log_job *jb, char *buffer, size_t size, size_t *line_length) {
1881 - if(!fgets(buffer, (int)size, stdin)) {
1882 - *line_length = 0;
1883 - return NULL;
1884 - }
1885 -
1886 - char *line = buffer;
1887 - size_t len = strlen(line);
1888 -
1889 - // remove trailing newlines and spaces
1890 - while(len > 1 && (line[len - 1] == '\n' || isspace(line[len - 1])))
1891 - line[--len] = '\0';
1892 -
1893 - // skip leading spaces
1894 - while(isspace(*line)) {
1895 - line++;
1896 - len--;
1897 - }
1898 -
1899 - *line_length = len;
1900 - return line;
1901 -}
1902 -
1903 -// ----------------------------------------------------------------------------
1904 -
1905 -static inline void jb_traverse_pcre2_named_groups_and_send_keys(struct log_job *jb, pcre2_code *re, pcre2_match_data *match_data, char *line) {
1906 - PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
1907 - uint32_t namecount;
1908 - pcre2_pattern_info(re, PCRE2_INFO_NAMECOUNT, &namecount);
1909 -
1910 - if (namecount > 0) {
1911 - PCRE2_SPTR name_table;
1912 - pcre2_pattern_info(re, PCRE2_INFO_NAMETABLE, &name_table);
1913 - uint32_t name_entry_size;
1914 - pcre2_pattern_info(re, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size);
1915 -
1916 - const unsigned char *tabptr = name_table;
1917 - for (uint32_t i = 0; i < namecount; i++) {
1918 - int n = (tabptr[0] << 8) | tabptr[1];
1919 - const char *group_name = (const char *)(tabptr + 2);
1920 -
1921 - PCRE2_SIZE start_offset = ovector[2 * n];
1922 - PCRE2_SIZE end_offset = ovector[2 * n + 1];
1923 - PCRE2_SIZE group_length = end_offset - start_offset;
1924 -
1925 - XXH64_hash_t hash = XXH3_64bits(group_name, strlen(group_name));
1926 -
1927 - send_key_value_and_rewrite(jb, group_name, hash, line + start_offset, group_length);
1928 -
1929 - // process the duplications
1930 - jb_send_duplications_for_key(jb, group_name, hash, line + start_offset, group_length);
1931 -
1932 - tabptr += name_entry_size;
1933 - }
1934 -
1935 - // print all non-exposed duplications
1936 - jb_send_remaining_duplications(jb);
1937 - }
1938 -}
1939 -
1940 -// ----------------------------------------------------------------------------
1941 -
1942 -static void yaml_print_multiline_value(const char *s, size_t depth) {
1943 - if (!s)
1944 - s = "";
1945 -
1946 - do {
1947 - const char* next = strchr(s, '\n');
1948 - if(next) next++;
1949 -
1950 - size_t len = next ? (size_t)(next - s) : strlen(s);
1951 - char buf[len + 1];
1952 - strncpy(buf, s, len);
1953 - buf[len] = '\0';
1954 -
1955 - fprintf(stderr, "%.*s%s%s",
1956 - (int)(depth * 2), " ",
1957 - buf, next ? "" : "\n");
1958 -
1959 - s = next;
1960 - } while(s && *s);
1961 -}
1962 -
1963 -static bool needs_quotes_in_yaml(const char *str) {
1964 - // Lookup table for special YAML characters
1965 - static bool special_chars[256] = { false };
1966 - static bool table_initialized = false;
1967 -
1968 - if (!table_initialized) {
1969 - // Initialize the lookup table
1970 - const char *special_chars_str = ":{}[],&*!|>'\"%@`^";
1971 - for (const char *c = special_chars_str; *c; ++c) {
1972 - special_chars[(unsigned char)*c] = true;
1973 - }
1974 - table_initialized = true;
1975 - }
1976 -
1977 - while (*str) {
1978 - if (special_chars[(unsigned char)*str]) {
1979 - return true;
1980 - }
1981 - str++;
1982 - }
1983 - return false;
1984 -}
1985 -
1986 -static void yaml_print_node(const char *key, const char *value, size_t depth, bool dash) {
1987 - if(depth > 10) depth = 10;
1988 - const char *quote = "\"";
1989 -
1990 - const char *second_line = NULL;
1991 - if(value && strchr(value, '\n')) {
1992 - second_line = value;
1993 - value = "|";
1994 - quote = "";
1995 - }
1996 - else if(!value || !needs_quotes_in_yaml(value))
1997 - quote = "";
1998 -
1999 - fprintf(stderr, "%.*s%s%s%s%s%s%s\n",
2000 - (int)(depth * 2), " ", dash ? "- ": "",
2001 - key ? key : "", key ? ": " : "",
2002 - quote, value ? value : "", quote);
2003 -
2004 - if(second_line) {
2005 - yaml_print_multiline_value(second_line, depth + 1);
2006 - }
2007 -}
2008 -
2009 -static void log_job_to_yaml(struct log_job *jb) {
2010 - if(jb->pattern)
2011 - yaml_print_node("pattern", jb->pattern, 0, false);
2012 -
2013 - if(jb->filename.key) {
2014 - fprintf(stderr, "\n");
2015 - yaml_print_node("filename", NULL, 0, false);
2016 - yaml_print_node("key", jb->filename.key, 1, false);
2017 - }
2018 -
2019 - if(jb->dups.used) {
2020 - fprintf(stderr, "\n");
2021 - yaml_print_node("duplicate", NULL, 0, false);
2022 - for(size_t i = 0; i < jb->dups.used ;i++) {
2023 - struct key_dup *kd = &jb->dups.array[i];
2024 - yaml_print_node("key", kd->target, 1, true);
2025 - yaml_print_node("values_of", NULL, 2, false);
2026 -
2027 - for(size_t k = 0; k < kd->used ;k++)
2028 - yaml_print_node(NULL, kd->keys[k], 3, true);
2029 - }
2030 - }
2031 -
2032 - if(jb->injections.used) {
2033 - fprintf(stderr, "\n");
2034 - yaml_print_node("inject", NULL, 0, false);
2035 -
2036 - for (size_t i = 0; i < jb->injections.used; i++) {
2037 - yaml_print_node("key", jb->injections.keys[i].key, 1, true);
2038 - yaml_print_node("value", jb->injections.keys[i].value.s, 2, false);
2039 - }
2040 - }
2041 -
2042 - if(jb->rewrites.used) {
2043 - fprintf(stderr, "\n");
2044 - yaml_print_node("rewrite", NULL, 0, false);
2045 -
2046 - for(size_t i = 0; i < jb->rewrites.used ;i++) {
2047 - yaml_print_node("key", jb->rewrites.array[i].key, 1, true);
2048 - yaml_print_node("search", jb->rewrites.array[i].search_pattern, 2, false);
2049 - yaml_print_node("replace", jb->rewrites.array[i].replace_pattern, 2, false);
2050 - }
2051 - }
2052 -
2053 - if(jb->unmatched.key || jb->unmatched.injections.used) {
2054 - fprintf(stderr, "\n");
2055 - yaml_print_node("unmatched", NULL, 0, false);
2056 -
2057 - if(jb->unmatched.key)
2058 - yaml_print_node("key", jb->unmatched.key, 1, false);
2059 -
2060 - if(jb->unmatched.injections.used) {
2061 - fprintf(stderr, "\n");
2062 - yaml_print_node("inject", NULL, 1, false);
2063 -
2064 - for (size_t i = 0; i < jb->unmatched.injections.used; i++) {
2065 - yaml_print_node("key", jb->unmatched.injections.keys[i].key, 2, true);
2066 - yaml_print_node("value", jb->unmatched.injections.keys[i].value.s, 3, false);
2067 - }
2068 - }
2069 - }
2070 -}
2071 -
2072 -struct log_job log_job = { 0 };
2073 -int main(int argc, char *argv[]) {
2074 - struct log_job *jb = &log_job;
2075 -
2076 - if(!parse_parameters(jb, argc, argv))
2077 - exit(1);
2078 -
2079 - if(jb->show_config)
2080 - log_job_to_yaml(jb);
2081 -
2082 - jb_select_which_injections_should_be_injected_on_unmatched(jb);
2083 -
2084 - pcre2_code *re = jb_compile_pcre2_pattern(jb->pattern);
2085 - if(!re)
2086 - return 1;
2087 -
2088 - pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL);
2089 -
2090 - char buffer[MAX_LINE_LENGTH];
2091 - char *line;
2092 - size_t len;
2093 -
2094 - while ((line = get_next_line(jb, buffer, sizeof(buffer), &len))) {
2095 - if(jb_switched_filename(jb, line, len))
2096 - continue;
2097 -
2098 - jb_reset_injections(jb);
2099 -
2100 - bool line_is_matched;
2101 - if(!jb_pcre2_match(re, match_data, line, len, true)) {
2102 - line_is_matched = false;
2103 -
2104 - if (jb->unmatched.key) {
2105 - // we are sending errors to systemd-journal
2106 - send_key_value_error(jb->unmatched.key, "PCRE2 error on: %s", line);
2107 -
2108 - for (size_t j = 0; j < jb->unmatched.injections.used; j++)
2109 - send_key_value_constant(jb, jb->unmatched.injections.keys[j].key,
2110 - jb->unmatched.injections.keys[j].value.s);
2111 - }
2112 - else {
2113 - // we are just logging errors to stderr
2114 - continue;
2115 - }
2116 - }
2117 - else {
2118 - line_is_matched = true;
2119 - jb_traverse_pcre2_named_groups_and_send_keys(jb, re, match_data, line);
2120 - }
2121 -
2122 - jb_inject_filename(jb);
2123 - jb_finalize_injections(jb, line_is_matched);
2124 -
2125 - printf("\n");
2126 - fflush(stdout);
2127 - }
2128 -
2129 - // Release memory used for the compiled regular expression and match data
2130 - pcre2_match_data_free(match_data);
2131 - pcre2_code_free(re);
2132 - jb_cleanup(jb);
2133 -
2134 - return 0;
2135 -}