Log2journal improvements part2 (#16494)
* code cleanup; isolation of pcre2 parsing * reorg fields for speed * renames and comments * prefixing now works on pcre2 mode too * move yaml configurations to log2journal.d directory in /usr/lib/netdata/conf.d * unify the transliteration * cleanup and fix ups * fix compiler warning * started writing a unit test for log2journal * fix codeql warnings * code re-organization and cleanup * code re-organization and cleanup; added include and exclude filters * added callocz and fixed yaml parsing * added hashtable to log2journal * more tests * added nginx-json config and unit test * stop parsing yaml when errors have been encountered * fix codeql warnings * fix docs * exit when a test fails
Costa Tsaousis committed
Nov 29, 2023 at 19:42 UTC
36714f9633f984600b53e85ff16b02fac3fe7e3d
31 files changed
+2845
-1046
.gitignore
+1
@@ -18,6 +18,7 @@ Makefile.in
18
.*.swp
19
*.old
20
*.log
21
+!collectors/log2journal/tests.d/*.log
22
*.pyc
23
24
Makefile
Makefile.am
+8
@@ -225,6 +225,7 @@ LIBNETDATA_FILES = \
225
libnetdata/http/http_defs.h \
226
libnetdata/dyn_conf/dyn_conf.c \
227
libnetdata/dyn_conf/dyn_conf.h \
228
+ libnetdata/simple_hashtable.h \
229
$(NULL)
230
231
if ENABLE_PLUGIN_EBPF
@@ -354,7 +355,14 @@ LOG2JOURNAL_FILES = \
355
collectors/log2journal/log2journal-yaml.c \
356
collectors/log2journal/log2journal-json.c \
357
collectors/log2journal/log2journal-logfmt.c \
358
+ collectors/log2journal/log2journal-pcre2.c \
359
collectors/log2journal/log2journal-params.c \
360
+ collectors/log2journal/log2journal-duplicate.c \
361
+ collectors/log2journal/log2journal-inject.c \
362
+ collectors/log2journal/log2journal-pattern.c \
363
+ collectors/log2journal/log2journal-replace.c \
364
+ collectors/log2journal/log2journal-rename.c \
365
+ collectors/log2journal/log2journal-rewrite.c \
366
$(NULL)
367
368
collectors/log2journal/Makefile.am
+5
-1
@@ -4,9 +4,13 @@ AUTOMAKE_OPTIONS = subdir-objects
4
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
6
dist_noinst_DATA = \
7
+ tests.sh \
8
README.md \
9
+ tests.d/* \
10
$(NULL)
11
10
-dist_libconfig_DATA = \
12
+log2journalconfigdir=$(libconfigdir)/log2journal.d
13
+dist_log2journalconfig_DATA = \
14
log2journal.d/nginx-combined.yaml \
15
+ log2journal.d/nginx-json.yaml \
16
$(NULL)
collectors/log2journal/README.md
+105
-139
@@ -313,22 +313,29 @@ tail -n $last -F /var/log/nginx/*access.log \
313
314
```
315
316
-Netdata log2journal v1.43.0-276-gfff8d1181
316
+Netdata log2journal v1.43.0-306-g929866ad3
317
318
-Convert structured log input to systemd Journal Export Format.
318
+Convert logs to systemd Journal Export Format.
319
320
-Using PCRE2 patterns, extract the fields from structured logs on the standard
321
-input, and generate output according to systemd Journal Export Format.
320
+ - JSON logs: extracts all JSON fields.
321
+ - logfmt logs: extracts all logfmt fields.
322
+ - free-form logs: uses PCRE2 patterns to extracts fields.
323
323
-Usage: ./log2journal [OPTIONS] PATTERN
324
+Usage: ./log2journal [OPTIONS] PATTERN|json
325
326
Options:
327
327
- --file /path/to/file.yaml
328
+ --file /path/to/file.yaml or -f /path/to/file.yaml
329
Read yaml configuration file for instructions.
330
331
+ --config CONFIG_NAME
332
+ Run with the internal configuration named CONFIG_NAME.
333
+ Available internal configs:
334
+
335
+ nginx-combined nginx-json
336
+
337
--show-config
331
- Show the configuration in yaml format before starting the job.
338
+ Show the configuration in YAML format before starting the job.
339
This is also an easy way to convert command line parameters to yaml.
340
341
--filename-key KEY
@@ -348,6 +355,7 @@ Options:
355
Create a new key called TARGET, duplicating the values of the keys
356
given. Useful for further processing. When multiple keys are given,
357
their values are separated by comma.
358
+
359
Up to 512 duplications can be given on the command line, and up to
360
20 keys per duplication command are allowed.
361
@@ -355,12 +363,14 @@ Options:
363
Inject constant fields to the output (both matched and unmatched logs).
364
--inject entries are added to unmatched lines too, when their key is
365
not used in --inject-unmatched (--inject-unmatched override --inject).
366
+
367
Up to 512 fields can be injected.
368
369
--inject-unmatched LINE
370
Inject lines into the output for each unmatched log entry.
371
Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched
372
lines as errors, so that they can easily be spotted in the journals.
373
+
374
Up to 512 such lines can be injected.
375
376
--rewrite KEY=/SearchPattern/ReplacePattern
@@ -369,6 +379,7 @@ Options:
379
be used between the search pattern and the replacement pattern.
380
The search pattern is a PCRE2 regular expression, and the replacement
381
pattern supports literals and named capture groups from the search pattern.
382
+
383
Example:
384
--rewrite DATE=/^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})$/
385
${day}/${month}/${year}
@@ -376,57 +387,109 @@ Options:
387
388
Only one rewrite rule is applied per key; the sequence of rewrites stops
389
for the key once a rule matches it. This allows providing a sequence of
379
- independent rewriting rules for the same key, matching the different values
380
- the key may get, and also provide a catch-all rewrite rule at the end of the
381
- sequence for setting the key value if no other rule matched it.
390
+ independent rewriting rules for the same key, matching the different
391
+ values the key may get, and also provide a catch-all rewrite rule at the
392
+ end, for setting the key value if no other rule matched it.
393
383
- The combination of duplicating keys with the values of multiple other keys
384
- combined with multiple rewrite rules, allows creating complex rules for
385
- rewriting key values.
394
+ Duplication of keys with the values of multiple other keys, combined with
395
+ multiple value rewriting rules, allows creating complex rules for adding
396
+ new keys, based on the values of existing keys.
397
398
Up to 512 rewriting rules are allowed.
399
389
- -h, --help
400
+ --include PATTERN
401
+ Include only keys matching the PCRE2 PATTERN.
402
+ Useful when parsing JSON of logfmt logs, to include only the keys given.
403
+ The keys are matched after the PREFIX has been added to them.
404
+
405
+ --exclude PATTERN
406
+ Exclude the keys matching the PCRE2 PATTERN.
407
+ Useful when parsing JSON of logfmt logs, to exclude some of the keys given.
408
+ The keys are matched after the PREFIX has been added to them.
409
+
410
+ When both include and exclude patterns are set and both match a key,
411
+ exclude wins and the key will not be added, like a pipeline, we first
412
+ include it and then exclude it.
413
+
414
+ --prefix PREFIX
415
+ Prefix all fields with PREFIX. The PREFIX is added before processing
416
+ duplications, renames and rewrites, so that the extracted keys have to
417
+ be matched with the PREFIX in them.
418
+ PREFIX is assumed to be systemd-journal friendly.
419
+
420
+ --rename NEW=OLD
421
+ Rename fields, before rewriting their values.
422
+
423
+ Up to 512 renaming rules are allowed.
424
+
425
+ -h, or --help
426
Display this help and exit.
427
428
PATTERN
429
PATTERN should be a valid PCRE2 regular expression.
430
RE2 regular expressions (like the ones usually used in Go applications),
431
are usually valid PCRE2 patterns too.
396
- Regular expressions without named groups are ignored.
432
+ Sub-expressions without named groups are evaluated, but their matches are
433
+ not added to the output.
434
+
435
+ JSON mode
436
+ JSON mode is enabled when the pattern is set to: json
437
+ Field names are extracted from the JSON logs and are converted to the
438
+ format expected by Journal Export Format (all caps, only _ is allowed).
439
+
440
+ logfmt mode
441
+ logfmt mode is enabled when the pattern is set to: logfmt
442
+ Field names are extracted from the logfmt logs and are converted to the
443
+ format expected by Journal Export Format (all caps, only _ is allowed).
444
+
445
446
The program accepts all parameters as both --option=value and --option value.
447
400
-The maximum line length accepted is 1048576 characters.
401
-The maximum number of fields in the PCRE2 pattern is 1024.
448
+The maximum log line length accepted is 1048576 characters.
449
450
PIPELINE AND SEQUENCE OF PROCESSING
451
452
This is a simple diagram of the pipeline taking place:
453
+
454
+ +---------------------------------------------------+
455
+ | INPUT |
456
+ | read one log line at a time |
457
+ +---------------------------------------------------+
458
+ v v
459
+ +---------------------------------+ |
460
+ | EXTRACT FIELDS AND VALUES | |
461
+ | JSON, logfmt, or pattern based | |
462
+ | (apply optional PREFIX) | |
463
+ +---------------------------------+ |
464
+ v v |
465
+ +---------------+ +--------------+ |
466
+ | DUPLICATE | | FILTER | |
467
+ | | | filter keys | |
468
+ | create new | +--------------+ |
469
+ | fields by | v |
470
+ | duplicating | +--------------+ |
471
+ | other fields | | RENAME | |
472
+ | and their | | change | |
473
+ | values | | field names | |
474
+ +---------------+ +--------------+ |
475
+ v v v
476
+ +---------------------------------+ +--------------+
477
+ | REWRITE PIPELINES | | INJECT |
478
+ | altering the values of fields | | constants |
479
+ +---------------------------------+ +--------------+
480
+ v v
481
+ +---------------------------------------------------+
482
+ | OUTPUT |
483
+ | generate Journal Export Format |
484
+ +---------------------------------------------------+
485
+
486
+IMPORTANT:
487
+ - Extraction of keys includes formatting them according to journal rules.
488
+ - Duplication rules use the original extracted field names, after they have
489
+ been prefixed (when a PREFIX is set) and before they are renamed.
490
+ - Rewriting is always the last stage, so the final field names are matched.
491
407
- +---------------------------------------------------+
408
- | INPUT |
409
- +---------------------------------------------------+
410
- v v
411
- +---------------------------------+ |
412
- | EXTRACT FIELDS AND VALUES | |
413
- +---------------------------------+ |
414
- v v |
415
- +---------------+ | |
416
- | DUPLICATE | | |
417
- | create fields | | |
418
- | with values | | |
419
- +---------------+ | |
420
- v v v
421
- +---------------------------------+ +--------------+
422
- | REWRITE PIPELINES | | INJECT |
423
- | altering the values | | constants |
424
- +---------------------------------+ +--------------+
425
- v v
426
- +---------------------------------------------------+
427
- | OUTPUT |
428
- +---------------------------------------------------+
429
-
492
+--------------------------------------------------------------------------------
493
JOURNAL FIELDS RULES (enforced by systemd-journald)
494
495
- field names can be up to 64 characters
@@ -458,107 +521,10 @@ JOURNAL FIELDS RULES (enforced by systemd-journald)
521
522
You can find the most common fields at 'man systemd.journal-fields'.
523
461
-Example YAML file:
462
-
463
---------------------------------------------------------------------------------
464
-# Netdata log2journal Configuration Template
465
-# The following parses nginx log files using the combined format.
466
-
467
-# The PCRE2 pattern to match log entries and give names to the fields.
468
-# The journal will have these names, so follow their rules. You can
469
-# initiate an extended PCRE2 pattern by starting the pattern with (?x)
470
-pattern: |
471
- (?x) # Enable PCRE2 extended mode
472
- ^
473
- (?<NGINX_REMOTE_ADDR>[^ ]+) \s - \s # NGINX_REMOTE_ADDR
474
- (?<NGINX_REMOTE_USER>[^ ]+) \s # NGINX_REMOTE_USER
475
- \[
476
- (?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL
477
- \]
478
- \s+ "
479
- (?<MESSAGE>
480
- (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD
481
- (?<NGINX_URL>[^ ]+) \s+
482
- HTTP/(?<NGINX_HTTP_VERSION>[^"]+)
483
- )
484
- " \s+
485
- (?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS
486
- (?<NGINX_BODY_BYTES_SENT>\d+) \s+ # NGINX_BODY_BYTES_SENT
487
- "(?<NGINX_HTTP_REFERER>[^"]*)" \s+ # NGINX_HTTP_REFERER
488
- "(?<NGINX_HTTP_USER_AGENT>[^"]*)" # NGINX_HTTP_USER_AGENT
489
-
490
-# When log2journal can detect the filename of each log entry (tail gives it
491
-# only when it tails multiple files), this key will be used to send the
492
-# filename to the journals.
493
-filename:
494
- key: NGINX_LOG_FILENAME
495
-
496
-# Duplicate fields under a different name. You can duplicate multiple fields
497
-# to a new one and then use rewrite rules to change its value.
498
-duplicate:
499
-
500
- # we insert the field PRIORITY as a copy of NGINX_STATUS.
501
- - key: PRIORITY
502
- values_of:
503
- - NGINX_STATUS
504
-
505
- # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.
506
- - key: NGINX_STATUS_FAMILY
507
- values_of:
508
- - NGINX_STATUS
509
-
510
-# Inject constant fields into the journal logs.
511
-inject:
512
- - key: SYSLOG_IDENTIFIER
513
- value: "nginx-log"
514
-
515
-# Rewrite the value of fields (including the duplicated ones).
516
-# The search pattern can have named groups, and the replace pattern can use
517
-# them as ${name}.
518
-rewrite:
519
- # PRIORTY is a duplicate of NGINX_STATUS
520
- # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
521
- - key: "PRIORITY"
522
- search: "^[123]"
523
- replace: 6
524
-
525
- - key: "PRIORITY"
526
- search: "^4"
527
- replace: 5
528
-
529
- - key: "PRIORITY"
530
- search: "^5"
531
- replace: 3
532
-
533
- - key: "PRIORITY"
534
- search: ".*"
535
- replace: 4
536
-
537
- # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
538
- - key: "NGINX_STATUS_FAMILY"
539
- search: "^(?<first_digit>[1-5])"
540
- replace: "${first_digit}xx"
541
-
542
- - key: "NGINX_STATUS_FAMILY"
543
- search: ".*"
544
- replace: "UNKNOWN"
545
-
546
-# Control what to do when input logs do not match the main PCRE2 pattern.
547
-unmatched:
548
- # The journal key to log the PCRE2 error message to.
549
- # Set this to MESSAGE, so you to see the error in the log.
550
- key: MESSAGE
551
-
552
- # Inject static fields to the unmatched entries.
553
- # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.
554
- inject:
555
- - key: PRIORITY
556
- value: 1
557
-
558
---------------------------------------------------------------------------------
559
-
524
```
525
526
+`log2journal` supports YAML configuration files, like the ones found [in this directory](log2journal.d/).
527
+
528
## `systemd-cat-native` options
529
530
```
collectors/log2journal/log2journal-duplicate.c
new
+49
@@ -0,0 +1,49 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+void duplication_cleanup(DUPLICATION *dp) {
6
+ hashed_key_cleanup(&dp->target);
7
+
8
+ for(size_t j = 0; j < dp->used ; j++) {
9
+ hashed_key_cleanup(&dp->keys[j]);
10
+ txt_cleanup(&dp->values[j]);
11
+ }
12
+}
13
+
14
+DUPLICATION *log_job_duplication_add(LOG_JOB *jb, const char *target, size_t target_len) {
15
+ if (jb->dups.used >= MAX_KEY_DUPS) {
16
+ log2stderr("ERROR: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS);
17
+ return NULL;
18
+ }
19
+
20
+ if(target_len > JOURNAL_MAX_KEY_LEN) {
21
+ log2stderr("WARNING: key of duplicate '%.*s' is too long for journals. Will be truncated.", (int)target_len, target);
22
+ target_len = JOURNAL_MAX_KEY_LEN;
23
+ }
24
+
25
+ DUPLICATION *kd = &jb->dups.array[jb->dups.used++];
26
+ hashed_key_len_set(&kd->target, target, target_len);
27
+ kd->used = 0;
28
+ kd->exposed = false;
29
+
30
+ // Initialize values array
31
+ for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) {
32
+ kd->values[i].txt = NULL;
33
+ kd->values[i].size = 0;
34
+ }
35
+
36
+ return kd;
37
+}
38
+
39
+bool log_job_duplication_key_add(DUPLICATION *kd, const char *key, size_t key_len) {
40
+ if (kd->used >= MAX_KEY_DUPS_KEYS) {
41
+ log2stderr("ERROR: Too many keys in duplication of target '%s'.", kd->target.key);
42
+ return false;
43
+ }
44
+
45
+ hashed_key_len_set(&kd->keys[kd->used++], key, key_len);
46
+
47
+ return true;
48
+}
49
+
collectors/log2journal/log2journal-help.c
+85
-45
@@ -10,12 +10,12 @@ static void config_dir_print_available(void) {
10
dir = opendir(path);
11
12
if (dir == NULL) {
13
- log2stderr(" >>> Cannot open directory '%s'", path);
13
+ log2stderr(" >>> Cannot open directory:\n %s", path);
14
return;
15
}
16
17
size_t column_width = 80;
18
- size_t current_columns = 0;
18
+ size_t current_columns = 7; // Start with 7 spaces for the first line
19
20
while ((entry = readdir(dir))) {
21
if (entry->d_type == DT_REG) { // Check if it's a regular file
@@ -24,10 +24,13 @@ static void config_dir_print_available(void) {
24
if (len >= 5 && strcmp(file_name + len - 5, ".yaml") == 0) {
25
// Remove the ".yaml" extension
26
len -= 5;
27
+ if (current_columns == 7) {
28
+ printf(" "); // Print 7 spaces at the beginning of a new line
29
+ }
30
if (current_columns + len + 1 > column_width) {
31
// Start a new line if the current line is full
29
- printf("\n ");
30
- current_columns = 0;
32
+ printf("\n "); // Print newline and 7 spaces
33
+ current_columns = 7;
34
}
35
printf("%.*s ", (int)len, file_name); // Print the filename without extension
36
current_columns += len + 1; // Add filename length and a space
@@ -39,7 +42,7 @@ static void config_dir_print_available(void) {
42
printf("\n"); // Add a newline at the end
43
}
44
42
-void log2journal_command_line_help(const char *name) {
45
+void log_job_command_line_help(const char *name) {
46
printf("\n");
47
printf("Netdata log2journal " PACKAGE_VERSION "\n");
48
printf("\n");
@@ -54,11 +57,11 @@ void log2journal_command_line_help(const char *name) {
57
printf("Options:\n");
58
printf("\n");
59
#ifdef HAVE_LIBYAML
57
- printf(" --file /path/to/file.yaml\n");
60
+ printf(" --file /path/to/file.yaml or -f /path/to/file.yaml\n");
61
printf(" Read yaml configuration file for instructions.\n");
62
printf("\n");
63
printf(" --config CONFIG_NAME\n");
61
- printf(" Run with the internal configuration named CONFIG_NAME\n");
64
+ printf(" Run with the internal configuration named CONFIG_NAME.\n");
65
printf(" Available internal configs:\n");
66
printf("\n");
67
config_dir_print_available();
@@ -89,6 +92,7 @@ void log2journal_command_line_help(const char *name) {
92
printf(" Create a new key called TARGET, duplicating the values of the keys\n");
93
printf(" given. Useful for further processing. When multiple keys are given,\n");
94
printf(" their values are separated by comma.\n");
95
+ printf("\n");
96
printf(" Up to %d duplications can be given on the command line, and up to\n", MAX_KEY_DUPS);
97
printf(" %d keys per duplication command are allowed.\n", MAX_KEY_DUPS_KEYS);
98
printf("\n");
@@ -96,12 +100,14 @@ void log2journal_command_line_help(const char *name) {
100
printf(" Inject constant fields to the output (both matched and unmatched logs).\n");
101
printf(" --inject entries are added to unmatched lines too, when their key is\n");
102
printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n");
103
+ printf("\n");
104
printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS);
105
printf("\n");
106
printf(" --inject-unmatched LINE\n");
107
printf(" Inject lines into the output for each unmatched log entry.\n");
108
printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n");
109
printf(" lines as errors, so that they can easily be spotted in the journals.\n");
110
+ printf("\n");
111
printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS);
112
printf("\n");
113
printf(" --rewrite KEY=/SearchPattern/ReplacePattern\n");
@@ -110,6 +116,7 @@ void log2journal_command_line_help(const char *name) {
116
printf(" be used between the search pattern and the replacement pattern.\n");
117
printf(" The search pattern is a PCRE2 regular expression, and the replacement\n");
118
printf(" pattern supports literals and named capture groups from the search pattern.\n");
119
+ printf("\n");
120
printf(" Example:\n");
121
printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n");
122
printf(" ${day}/${month}/${year}\n");
@@ -117,76 +124,109 @@ void log2journal_command_line_help(const char *name) {
124
printf("\n");
125
printf(" Only one rewrite rule is applied per key; the sequence of rewrites stops\n");
126
printf(" for the key once a rule matches it. This allows providing a sequence of\n");
120
- printf(" independent rewriting rules for the same key, matching the different values\n");
121
- printf(" the key may get, and also provide a catch-all rewrite rule at the end of the\n");
122
- printf(" sequence for setting the key value if no other rule matched it.\n");
127
+ printf(" independent rewriting rules for the same key, matching the different\n");
128
+ printf(" values the key may get, and also provide a catch-all rewrite rule at the\n");
129
+ printf(" end, for setting the key value if no other rule matched it.\n");
130
+ printf("\n");
131
+ printf(" Duplication of keys with the values of multiple other keys, combined with\n");
132
+ printf(" multiple value rewriting rules, allows creating complex rules for adding\n");
133
+ printf(" new keys, based on the values of existing keys.\n");
134
printf("\n");
124
- printf(" The combination of duplicating keys with the values of multiple other keys\n");
125
- printf(" combined with multiple rewrite rules, allows creating complex rules for\n");
126
- printf(" rewriting key values.\n");
135
printf(" Up to %d rewriting rules are allowed.\n", MAX_REWRITES);
136
printf("\n");
137
+ printf(" --include PATTERN\n");
138
+ printf(" Include only keys matching the PCRE2 PATTERN.\n");
139
+ printf(" Useful when parsing JSON of logfmt logs, to include only the keys given.\n");
140
+ printf(" The keys are matched after the PREFIX has been added to them.\n");
141
+ printf("\n");
142
+ printf(" --exclude PATTERN\n");
143
+ printf(" Exclude the keys matching the PCRE2 PATTERN.\n");
144
+ printf(" Useful when parsing JSON of logfmt logs, to exclude some of the keys given.\n");
145
+ printf(" The keys are matched after the PREFIX has been added to them.\n");
146
+ printf("\n");
147
+ printf(" When both include and exclude patterns are set and both match a key,\n");
148
+ printf(" exclude wins and the key will not be added, like a pipeline, we first\n");
149
+ printf(" include it and then exclude it.\n");
150
+ printf("\n");
151
printf(" --prefix PREFIX\n");
130
- printf(" Prefix all JSON or logfmt fields with PREFIX.\n");
152
+ printf(" Prefix all fields with PREFIX. The PREFIX is added before processing\n");
153
+ printf(" duplications, renames and rewrites, so that the extracted keys have to\n");
154
+ printf(" be matched with the PREFIX in them.\n");
155
+ printf(" PREFIX is assumed to be systemd-journal friendly.\n");
156
printf("\n");
157
printf(" --rename NEW=OLD\n");
158
printf(" Rename fields, before rewriting their values.\n");
159
+ printf("\n");
160
printf(" Up to %d renaming rules are allowed.\n", MAX_RENAMES);
161
printf("\n");
136
- printf(" -h, --help\n");
162
+ printf(" -h, or --help\n");
163
printf(" Display this help and exit.\n");
164
printf("\n");
165
printf(" PATTERN\n");
166
printf(" PATTERN should be a valid PCRE2 regular expression.\n");
167
printf(" RE2 regular expressions (like the ones usually used in Go applications),\n");
168
printf(" are usually valid PCRE2 patterns too.\n");
143
- printf(" Regular expressions without named groups are evaluated but their matches\n");
144
- printf(" are not added to the output.\n");
169
+ printf(" Sub-expressions without named groups are evaluated, but their matches are\n");
170
+ printf(" not added to the output.\n");
171
printf("\n");
172
printf(" JSON mode\n");
173
printf(" JSON mode is enabled when the pattern is set to: json\n");
174
printf(" Field names are extracted from the JSON logs and are converted to the\n");
175
printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n");
150
- printf(" Prefixing is enabled in this mode.\n");
176
+ printf("\n");
177
printf(" logfmt mode\n");
178
printf(" logfmt mode is enabled when the pattern is set to: logfmt\n");
179
printf(" Field names are extracted from the logfmt logs and are converted to the\n");
180
printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n");
155
- printf(" Prefixing is enabled in this mode.\n");
181
printf("\n");
182
printf("\n");
183
printf("The program accepts all parameters as both --option=value and --option value.\n");
184
printf("\n");
160
- printf("The maximum line length accepted is %d characters.\n", MAX_LINE_LENGTH);
161
- printf("The maximum number of fields in the PCRE2 pattern is %d.\n", OVECCOUNT / 3);
185
+ printf("The maximum log line length accepted is %d characters.\n", MAX_LINE_LENGTH);
186
printf("\n");
187
printf("PIPELINE AND SEQUENCE OF PROCESSING\n");
188
printf("\n");
189
printf("This is a simple diagram of the pipeline taking place:\n");
166
- printf("\n");
167
- printf(" +---------------------------------------------------+\n");
168
- printf(" | INPUT |\n");
169
- printf(" +---------------------------------------------------+\n");
170
- printf(" v v\n");
171
- printf(" +---------------------------------+ |\n");
172
- printf(" | EXTRACT FIELDS AND VALUES | |\n");
173
- printf(" +---------------------------------+ |\n");
174
- printf(" v v |\n");
175
- printf(" +---------------+ +--------------+ |\n");
176
- printf(" | DUPLICATE | | RENAME | |\n");
177
- printf(" | create fields | | change the | |\n");
178
- printf(" | with values | | field name | |\n");
179
- printf(" +---------------+ +--------------+ |\n");
180
- printf(" v v v\n");
181
- printf(" +---------------------------------+ +--------------+\n");
182
- printf(" | REWRITE PIPELINES | | INJECT |\n");
183
- printf(" | altering keys and values | | constants |\n");
184
- printf(" +---------------------------------+ +--------------+\n");
185
- printf(" v v\n");
186
- printf(" +---------------------------------------------------+\n");
187
- printf(" | OUTPUT |\n");
188
- printf(" +---------------------------------------------------+\n");
189
- printf("\n");
190
+ printf(" \n");
191
+ printf(" +---------------------------------------------------+ \n");
192
+ printf(" | INPUT | \n");
193
+ printf(" | read one log line at a time | \n");
194
+ printf(" +---------------------------------------------------+ \n");
195
+ printf(" v v \n");
196
+ printf(" +---------------------------------+ | \n");
197
+ printf(" | EXTRACT FIELDS AND VALUES | | \n");
198
+ printf(" | JSON, logfmt, or pattern based | | \n");
199
+ printf(" | (apply optional PREFIX) | | \n");
200
+ printf(" +---------------------------------+ | \n");
201
+ printf(" v v | \n");
202
+ printf(" +---------------+ +--------------+ | \n");
203
+ printf(" | DUPLICATE | | FILTER | | \n");
204
+ printf(" | | | filter keys | | \n");
205
+ printf(" | create new | +--------------+ | \n");
206
+ printf(" | fields by | v | \n");
207
+ printf(" | duplicating | +--------------+ | \n");
208
+ printf(" | other fields | | RENAME | | \n");
209
+ printf(" | and their | | change | | \n");
210
+ printf(" | values | | field names | | \n");
211
+ printf(" +---------------+ +--------------+ | \n");
212
+ printf(" v v v \n");
213
+ printf(" +---------------------------------+ +--------------+ \n");
214
+ printf(" | REWRITE PIPELINES | | INJECT | \n");
215
+ printf(" | altering the values of fields | | constants | \n");
216
+ printf(" +---------------------------------+ +--------------+ \n");
217
+ printf(" v v \n");
218
+ printf(" +---------------------------------------------------+ \n");
219
+ printf(" | OUTPUT | \n");
220
+ printf(" | generate Journal Export Format | \n");
221
+ printf(" +---------------------------------------------------+ \n");
222
+ printf(" \n");
223
+ printf("IMPORTANT:\n");
224
+ printf(" - Extraction of keys includes formatting them according to journal rules.\n");
225
+ printf(" - Duplication rules use the original extracted field names, after they have\n");
226
+ printf(" been prefixed (when a PREFIX is set) and before they are renamed.\n");
227
+ printf(" - Rewriting is always the last stage, so the final field names are matched.\n");
228
+ printf("\n");
229
+ printf("--------------------------------------------------------------------------------\n");
230
printf("JOURNAL FIELDS RULES (enforced by systemd-journald)\n");
231
printf("\n");
232
printf(" - field names can be up to 64 characters\n");
collectors/log2journal/log2journal-inject.c
new
+44
@@ -0,0 +1,44 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+void injection_cleanup(INJECTION *inj) {
6
+ hashed_key_cleanup(&inj->key);
7
+ txt_cleanup(&inj->value);
8
+}
9
+
10
+static inline void log_job_injection_replace(INJECTION *inj, const char *key, size_t key_len, const char *value, size_t value_len) {
11
+ if(key_len > JOURNAL_MAX_KEY_LEN)
12
+ log2stderr("WARNING: injection key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
13
+
14
+ if(value_len > JOURNAL_MAX_VALUE_LEN)
15
+ log2stderr("WARNING: injection value of key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
16
+
17
+ hashed_key_len_set(&inj->key, key, key_len);
18
+ txt_replace(&inj->value, value, value_len);
19
+}
20
+
21
+bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {
22
+ if (unmatched) {
23
+ if (jb->unmatched.injections.used >= MAX_INJECTIONS) {
24
+ log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);
25
+ return false;
26
+ }
27
+ }
28
+ else {
29
+ if (jb->injections.used >= MAX_INJECTIONS) {
30
+ log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);
31
+ return false;
32
+ }
33
+ }
34
+
35
+ if (unmatched) {
36
+ log_job_injection_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],
37
+ key, key_len, value, value_len);
38
+ } else {
39
+ log_job_injection_replace(&jb->injections.keys[jb->injections.used++],
40
+ key, key_len, value, value_len);
41
+ }
42
+
43
+ return true;
44
+}
collectors/log2journal/log2journal-json.c
+47
-89
@@ -2,20 +2,20 @@
2
3
#include "log2journal.h"
4
5
-#define ERROR_LINE_MAX 1024
6
-#define KEY_MAX 1024
5
+#define JSON_ERROR_LINE_MAX 1024
6
+#define JSON_KEY_MAX 1024
7
#define JSON_DEPTH_MAX 100
8
9
struct log_json_state {
10
- const char *line;
11
- size_t pos;
12
- char msg[ERROR_LINE_MAX];
10
+ LOG_JOB *jb;
11
14
- char key[KEY_MAX];
15
- char *key_stack[JSON_DEPTH_MAX];
16
- size_t depth;
12
+ const char *line;
13
+ uint32_t pos;
14
+ uint32_t depth;
15
+ char *stack[JSON_DEPTH_MAX];
16
18
- struct log_job *jb;
17
+ char key[JSON_KEY_MAX];
18
+ char msg[JSON_ERROR_LINE_MAX];
19
};
20
21
static inline bool json_parse_object(LOG_JSON_STATE *js);
@@ -25,7 +25,7 @@ static inline bool json_parse_array(LOG_JSON_STATE *js);
25
#define json_consume_char(js) ++(js)->pos
26
27
static inline void json_process_key_value(LOG_JSON_STATE *js, const char *value, size_t len) {
28
- jb_send_extracted_key_value(js->jb, js->key, value, len);
28
+ log_job_send_extracted_key_value(js->jb, js->key, value, len);
29
}
30
31
static inline void json_skip_spaces(LOG_JSON_STATE *js) {
@@ -205,8 +205,24 @@ static bool encode_utf8(unsigned codepoint, char **d, size_t *remaining) {
205
return true;
206
}
207
208
+static inline void copy_newline(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {
209
+ if(*remaining > 3) {
210
+ *(*d)++ = '\\';
211
+ *(*d)++ = 'n';
212
+ (*remaining) -= 2;
213
+ }
214
+}
215
+
216
+static inline void copy_tab(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {
217
+ if(*remaining > 3) {
218
+ *(*d)++ = '\\';
219
+ *(*d)++ = 't';
220
+ (*remaining) -= 2;
221
+ }
222
+}
223
+
224
static inline bool json_parse_string(LOG_JSON_STATE *js) {
209
- static __thread char value[MAX_VALUE_LEN];
225
+ static __thread char value[JOURNAL_MAX_VALUE_LEN];
226
227
if(!json_expect_char_after_white_space(js, "\""))
228
return false;
@@ -226,25 +242,22 @@ static inline bool json_parse_string(LOG_JSON_STATE *js) {
242
243
switch (*s) {
244
case 'n':
229
- c = '\n';
245
+ copy_newline(js, &d, &remaining);
246
s++;
231
- break;
247
+ continue;
248
+
249
case 't':
233
- c = '\t';
234
- s++;
235
- break;
236
- case 'b':
237
- c = '\b';
250
+ copy_tab(js, &d, &remaining);
251
s++;
239
- break;
252
+ continue;
253
+
254
case 'f':
241
- c = '\f';
242
- s++;
243
- break;
255
+ case 'b':
256
case 'r':
245
- c = '\r';
257
+ c = ' ';
258
s++;
259
break;
260
+
261
case 'u':
262
if(isxdigit(s[1]) && isxdigit(s[2]) && isxdigit(s[3]) && isxdigit(s[4])) {
263
char b[5] = {
@@ -305,61 +318,6 @@ static inline bool json_parse_string(LOG_JSON_STATE *js) {
318
}
319
320
static inline bool json_parse_key_and_push(LOG_JSON_STATE *js) {
308
- static const char valid_journal_key_chars[256] = {
309
- // control characters
310
- [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_',
311
- [8] = '_', [9] = '_', [10] = '_', [11] = '_', [12] = '_', [13] = '_', [14] = '_', [15] = '_',
312
- [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_', [22] = '_', [23] = '_',
313
- [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_', [30] = '_', [31] = '_',
314
-
315
- // symbols
316
- [' '] = '_', ['!'] = '_', ['"'] = '_', ['#'] = '_', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_',
317
- ['('] = '_', [')'] = '_', ['*'] = '_', ['+'] = '_', [','] = '_', ['-'] = '_', ['.'] = '_', ['/'] = '_',
318
-
319
- // numbers
320
- ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7',
321
- ['8'] = '8', ['9'] = '9',
322
-
323
- // symbols
324
- [':'] = '_', [';'] = '_', ['<'] = '_', ['='] = '_', ['>'] = '_', ['?'] = '_', ['@'] = '_',
325
-
326
- // capitals
327
- ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H',
328
- ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P',
329
- ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X',
330
- ['Y'] = 'Y', ['Z'] = 'Z',
331
-
332
- // symbols
333
- ['['] = '_', ['\\'] = '_', [']'] = '_', ['^'] = '_', ['_'] = '_', ['`'] = '_',
334
-
335
- // lower to upper
336
- ['a'] = 'A', ['b'] = 'B', ['c'] = 'C', ['d'] = 'D', ['e'] = 'E', ['f'] = 'F', ['g'] = 'G', ['h'] = 'H',
337
- ['i'] = 'I', ['j'] = 'J', ['k'] = 'K', ['l'] = 'L', ['m'] = 'M', ['n'] = 'N', ['o'] = 'O', ['p'] = 'P',
338
- ['q'] = 'Q', ['r'] = 'R', ['s'] = 'S', ['t'] = 'T', ['u'] = 'U', ['v'] = 'V', ['w'] = 'W', ['x'] = 'X',
339
- ['y'] = 'Y', ['z'] = 'Z',
340
-
341
- // symbols
342
- ['{'] = '_', ['|'] = '_', ['}'] = '_', ['~'] = '_', [127] = '_', // Delete (DEL)
343
-
344
- // Extended ASCII characters (128-255) set to underscore
345
- [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_', [135] = '_',
346
- [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_', [143] = '_',
347
- [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_', [151] = '_',
348
- [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_', [159] = '_',
349
- [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_', [167] = '_',
350
- [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_', [175] = '_',
351
- [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_', [183] = '_',
352
- [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_', [191] = '_',
353
- [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_', [199] = '_',
354
- [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_', [207] = '_',
355
- [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_', [215] = '_',
356
- [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_', [223] = '_',
357
- [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_', [231] = '_',
358
- [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_', [239] = '_',
359
- [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_', [247] = '_',
360
- [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_', [255] = '_',
361
- };
362
-
321
if (!json_expect_char_after_white_space(js, "\""))
322
return false;
323
@@ -371,7 +329,7 @@ static inline bool json_parse_key_and_push(LOG_JSON_STATE *js) {
329
330
json_consume_char(js);
331
374
- char *d = js->key_stack[js->depth];
332
+ char *d = js->stack[js->depth];
333
if(js->depth)
334
*d++ = '_';
335
@@ -384,11 +342,11 @@ static inline bool json_parse_key_and_push(LOG_JSON_STATE *js) {
342
343
if (*s == '\\') {
344
s++;
387
- c = (char)((*s == 'u') ? '_' : valid_journal_key_chars[(unsigned char)*s]);
345
+ c = (char)((*s == 'u') ? '_' : journal_key_characters_map[(unsigned char)*s]);
346
s += (*s == 'u') ? 5 : 1;
347
}
348
else
391
- c = valid_journal_key_chars[(unsigned char)*s++];
349
+ c = journal_key_characters_map[(unsigned char)*s++];
350
351
if(c == '_' && last_c == '_')
352
continue;
@@ -412,7 +370,7 @@ static inline bool json_parse_key_and_push(LOG_JSON_STATE *js) {
370
371
json_consume_char(js);
372
415
- js->key_stack[++js->depth] = d;
373
+ js->stack[++js->depth] = d;
374
375
return true;
376
}
@@ -424,7 +382,7 @@ static inline bool json_key_pop(LOG_JSON_STATE *js) {
382
return false;
383
}
384
427
- char *k = js->key_stack[js->depth--];
385
+ char *k = js->stack[js->depth--];
386
*k = '\0';
387
return true;
388
}
@@ -473,7 +431,7 @@ static inline bool json_parse_value(LOG_JSON_STATE *js) {
431
}
432
433
static inline bool json_key_index_and_push(LOG_JSON_STATE *js, size_t index) {
476
- char *d = js->key_stack[js->depth];
434
+ char *d = js->stack[js->depth];
435
if(js->depth > 0) {
436
*d++ = '_';
437
}
@@ -503,7 +461,7 @@ static inline bool json_key_index_and_push(LOG_JSON_STATE *js, size_t index) {
461
}
462
463
*d = '\0'; // Null-terminate the key
506
- js->key_stack[++js->depth] = d;
464
+ js->stack[++js->depth] = d;
465
466
return true;
467
}
@@ -579,7 +537,7 @@ static inline bool json_parse_object(LOG_JSON_STATE *js) {
537
return true;
538
}
539
582
-LOG_JSON_STATE *json_parser_create(struct log_job *jb) {
540
+LOG_JSON_STATE *json_parser_create(LOG_JOB *jb) {
541
LOG_JSON_STATE *js = mallocz(sizeof(LOG_JSON_STATE));
542
memset(js, 0, sizeof(LOG_JSON_STATE));
543
js->jb = jb;
@@ -587,7 +545,7 @@ LOG_JSON_STATE *json_parser_create(struct log_job *jb) {
545
if(jb->prefix)
546
copy_to_buffer(js->key, sizeof(js->key), js->jb->prefix, strlen(js->jb->prefix));
547
590
- js->key_stack[0] = &js->key[strlen(js->key)];
548
+ js->stack[0] = &js->key[strlen(js->key)];
549
550
return js;
551
}
@@ -605,7 +563,7 @@ bool json_parse_document(LOG_JSON_STATE *js, const char *txt) {
563
js->line = txt;
564
js->pos = 0;
565
js->msg[0] = '\0';
608
- js->key_stack[0][0] = '\0';
566
+ js->stack[0][0] = '\0';
567
js->depth = 0;
568
569
if(!json_parse_object(js))
@@ -624,7 +582,7 @@ bool json_parse_document(LOG_JSON_STATE *js, const char *txt) {
582
}
583
584
void json_test(void) {
627
- struct log_job jb = { .prefix = "NIGNX_" };
585
+ LOG_JOB jb = { .prefix = "NIGNX_" };
586
LOG_JSON_STATE *json = json_parser_create(&jb);
587
588
json_parse_document(json, "{\"value\":\"\\u\\u039A\\u03B1\\u03BB\\u03B7\\u03BC\\u03AD\\u03C1\\u03B1\"}");
collectors/log2journal/log2journal-logfmt.c
+38
-80
@@ -2,25 +2,25 @@
2
3
#include "log2journal.h"
4
5
-#define ERROR_LINE_MAX 1024
6
-#define KEY_MAX 1024
5
+#define LOGFMT_ERROR_LINE_MAX 1024
6
+#define LOGFMT_KEY_MAX 1024
7
8
struct logfmt_state {
9
- const char *line;
10
- size_t pos;
11
- char msg[ERROR_LINE_MAX];
9
+ LOG_JOB *jb;
10
13
- char key[KEY_MAX];
14
- size_t key_start;
11
+ const char *line;
12
+ uint32_t pos;
13
+ uint32_t key_start;
14
16
- struct log_job *jb;
15
+ char key[LOGFMT_KEY_MAX];
16
+ char msg[LOGFMT_ERROR_LINE_MAX];
17
};
18
19
#define logfmt_current_pos(lfs) &(lfs)->line[(lfs)->pos]
20
#define logfmt_consume_char(lfs) ++(lfs)->pos
21
22
static inline void logfmt_process_key_value(LOGFMT_STATE *lfs, const char *value, size_t len) {
23
- jb_send_extracted_key_value(lfs->jb, lfs->key, value, len);
23
+ log_job_send_extracted_key_value(lfs->jb, lfs->key, value, len);
24
}
25
26
static inline void logfmt_skip_spaces(LOGFMT_STATE *lfs) {
@@ -32,8 +32,24 @@ static inline void logfmt_skip_spaces(LOGFMT_STATE *lfs) {
32
lfs->pos += s - start;
33
}
34
35
+static inline void copy_newline(LOGFMT_STATE *lfs __maybe_unused, char **d, size_t *remaining) {
36
+ if(*remaining > 3) {
37
+ *(*d)++ = '\\';
38
+ *(*d)++ = 'n';
39
+ (*remaining) -= 2;
40
+ }
41
+}
42
+
43
+static inline void copy_tab(LOGFMT_STATE *lfs __maybe_unused, char **d, size_t *remaining) {
44
+ if(*remaining > 3) {
45
+ *(*d)++ = '\\';
46
+ *(*d)++ = 't';
47
+ (*remaining) -= 2;
48
+ }
49
+}
50
+
51
static inline bool logftm_parse_value(LOGFMT_STATE *lfs) {
36
- static __thread char value[MAX_VALUE_LEN];
52
+ static __thread char value[JOURNAL_MAX_VALUE_LEN];
53
54
char quote = '\0';
55
const char *s = logfmt_current_pos(lfs);
@@ -56,25 +72,22 @@ static inline bool logftm_parse_value(LOGFMT_STATE *lfs) {
72
73
switch (*s) {
74
case 'n':
59
- c = '\n';
75
+ copy_newline(lfs, &d, &remaining);
76
s++;
61
- break;
77
+ continue;
78
+
79
case 't':
63
- c = '\t';
80
+ copy_tab(lfs, &d, &remaining);
81
s++;
65
- break;
66
- case 'b':
67
- c = '\b';
68
- s++;
69
- break;
82
+ continue;
83
+
84
case 'f':
71
- c = '\f';
72
- s++;
73
- break;
85
+ case 'b':
86
case 'r':
75
- c = '\r';
87
+ c = ' ';
88
s++;
89
break;
90
+
91
default:
92
c = *s++;
93
break;
@@ -116,61 +129,6 @@ static inline bool logftm_parse_value(LOGFMT_STATE *lfs) {
129
}
130
131
static inline bool logfmt_parse_key(LOGFMT_STATE *lfs) {
119
- static const char valid_journal_key_chars[256] = {
120
- // control characters
121
- [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_',
122
- [8] = '_', [9] = '_', [10] = '_', [11] = '_', [12] = '_', [13] = '_', [14] = '_', [15] = '_',
123
- [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_', [22] = '_', [23] = '_',
124
- [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_', [30] = '_', [31] = '_',
125
-
126
- // symbols
127
- [' '] = '_', ['!'] = '_', ['"'] = '_', ['#'] = '_', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_',
128
- ['('] = '_', [')'] = '_', ['*'] = '_', ['+'] = '_', [','] = '_', ['-'] = '_', ['.'] = '_', ['/'] = '_',
129
-
130
- // numbers
131
- ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7',
132
- ['8'] = '8', ['9'] = '9',
133
-
134
- // symbols
135
- [':'] = '_', [';'] = '_', ['<'] = '_', ['='] = '_', ['>'] = '_', ['?'] = '_', ['@'] = '_',
136
-
137
- // capitals
138
- ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H',
139
- ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P',
140
- ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X',
141
- ['Y'] = 'Y', ['Z'] = 'Z',
142
-
143
- // symbols
144
- ['['] = '_', ['\\'] = '_', [']'] = '_', ['^'] = '_', ['_'] = '_', ['`'] = '_',
145
-
146
- // lower to upper
147
- ['a'] = 'A', ['b'] = 'B', ['c'] = 'C', ['d'] = 'D', ['e'] = 'E', ['f'] = 'F', ['g'] = 'G', ['h'] = 'H',
148
- ['i'] = 'I', ['j'] = 'J', ['k'] = 'K', ['l'] = 'L', ['m'] = 'M', ['n'] = 'N', ['o'] = 'O', ['p'] = 'P',
149
- ['q'] = 'Q', ['r'] = 'R', ['s'] = 'S', ['t'] = 'T', ['u'] = 'U', ['v'] = 'V', ['w'] = 'W', ['x'] = 'X',
150
- ['y'] = 'Y', ['z'] = 'Z',
151
-
152
- // symbols
153
- ['{'] = '_', ['|'] = '_', ['}'] = '_', ['~'] = '_', [127] = '_', // Delete (DEL)
154
-
155
- // Extended ASCII characters (128-255) set to underscore
156
- [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_', [135] = '_',
157
- [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_', [143] = '_',
158
- [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_', [151] = '_',
159
- [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_', [159] = '_',
160
- [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_', [167] = '_',
161
- [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_', [175] = '_',
162
- [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_', [183] = '_',
163
- [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_', [191] = '_',
164
- [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_', [199] = '_',
165
- [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_', [207] = '_',
166
- [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_', [215] = '_',
167
- [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_', [223] = '_',
168
- [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_', [231] = '_',
169
- [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_', [239] = '_',
170
- [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_', [247] = '_',
171
- [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_', [255] = '_',
172
- };
173
-
132
logfmt_skip_spaces(lfs);
133
134
char *d = &lfs->key[lfs->key_start];
@@ -185,7 +143,7 @@ static inline bool logfmt_parse_key(LOGFMT_STATE *lfs) {
143
if (*s == '\\')
144
s++;
145
188
- c = valid_journal_key_chars[(unsigned char)*s++];
146
+ c = journal_key_characters_map[(unsigned char)*s++];
147
148
if(c == '_' && last_c == '_')
149
continue;
@@ -216,7 +174,7 @@ static inline bool logfmt_parse_key(LOGFMT_STATE *lfs) {
174
return true;
175
}
176
219
-LOGFMT_STATE *logfmt_parser_create(struct log_job *jb) {
177
+LOGFMT_STATE *logfmt_parser_create(LOG_JOB *jb) {
178
LOGFMT_STATE *lfs = mallocz(sizeof(LOGFMT_STATE));
179
memset(lfs, 0, sizeof(LOGFMT_STATE));
180
lfs->jb = jb;
@@ -259,7 +217,7 @@ bool logfmt_parse_document(LOGFMT_STATE *lfs, const char *txt) {
217
218
219
void logfmt_test(void) {
262
- struct log_job jb = { .prefix = "NIGNX_" };
220
+ LOG_JOB jb = { .prefix = "NIGNX_" };
221
LOGFMT_STATE *logfmt = logfmt_parser_create(&jb);
222
223
logfmt_parse_document(logfmt, "x=1 y=2 z=\"3 \\ 4\" 5 ");
collectors/log2journal/log2journal-params.c
+65
-236
@@ -2,73 +2,41 @@
2
3
#include "log2journal.h"
4
5
-static bool parse_replacement_pattern(struct key_rewrite *rw);
6
-
5
// ----------------------------------------------------------------------------
6
9
-void nd_log_destroy(struct log_job *jb) {
10
- for(size_t i = 0; i < jb->injections.used ;i++) {
11
- if(jb->injections.keys[i].value.s)
12
- freez(jb->injections.keys[i].value.s);
7
+void nd_log_cleanup(LOG_JOB *jb) {
8
+ if(jb->prefix) {
9
+ freez((void *) jb->prefix);
10
+ jb->prefix = NULL;
11
}
12
15
- for(size_t i = 0; i < jb->unmatched.injections.used ;i++) {
16
- if(jb->unmatched.injections.keys[i].value.s)
17
- freez(jb->unmatched.injections.keys[i].value.s);
13
+ if(jb->pattern) {
14
+ freez((void *) jb->pattern);
15
+ jb->pattern = NULL;
16
}
17
20
- for(size_t i = 0; i < jb->dups.used ;i++) {
21
- struct key_dup *kd = &jb->dups.array[i];
22
-
23
- if(kd->target)
24
- freez(kd->target);
25
-
26
- for(size_t j = 0; j < kd->used ; j++) {
27
- if (kd->keys[j])
28
- freez(kd->keys[j]);
29
-
30
- if (kd->values[j].s)
31
- freez(kd->values[j].s);
32
- }
33
- }
34
-
35
- for(size_t i = 0; i < jb->rewrites.used; i++) {
36
- struct key_rewrite *rw = &jb->rewrites.array[i];
37
-
38
- if (rw->key)
39
- freez(rw->key);
40
-
41
- if (rw->search_pattern)
42
- freez(rw->search_pattern);
18
+ for(size_t i = 0; i < jb->injections.used ;i++)
19
+ injection_cleanup(&jb->injections.keys[i]);
20
44
- if (rw->replace_pattern)
45
- freez(rw->replace_pattern);
21
+ for(size_t i = 0; i < jb->unmatched.injections.used ;i++)
22
+ injection_cleanup(&jb->unmatched.injections.keys[i]);
23
47
- if(rw->match_data)
48
- pcre2_match_data_free(rw->match_data);
24
+ for(size_t i = 0; i < jb->renames.used ;i++)
25
+ rename_cleanup(&jb->renames.array[i]);
26
50
- if (rw->re)
51
- pcre2_code_free(rw->re);
27
+ for(size_t i = 0; i < jb->dups.used ;i++)
28
+ duplication_cleanup(&jb->dups.array[i]);
29
53
- // Cleanup for replacement nodes linked list
54
- struct replacement_node *current = rw->nodes;
55
- while (current != NULL) {
56
- struct replacement_node *next = current->next;
57
-
58
- if (current->s)
59
- freez((void *)current->s);
60
-
61
- freez(current);
62
- current = next;
63
- }
64
- }
30
+ for(size_t i = 0; i < jb->rewrites.used; i++)
31
+ rewrite_cleanup(&jb->rewrites.array[i]);
32
33
+ // remove references to everything else, to reveal them in valgrind
34
memset(jb, 0, sizeof(*jb));
35
}
36
37
// ----------------------------------------------------------------------------
38
71
-bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len) {
39
+bool log_job_filename_key_set(LOG_JOB *jb, const char *key, size_t key_len) {
40
if(!key || !*key) {
41
log2stderr("filename key cannot be empty.");
42
return false;
@@ -82,7 +50,7 @@ bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_le
50
return true;
51
}
52
85
-bool log_job_add_key_prefix(struct log_job *jb, const char *prefix, size_t prefix_len) {
53
+bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len) {
54
if(!prefix || !*prefix) {
55
log2stderr("filename key cannot be empty.");
56
return false;
@@ -96,198 +64,51 @@ bool log_job_add_key_prefix(struct log_job *jb, const char *prefix, size_t prefi
64
return true;
65
}
66
99
-bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {
100
- if (unmatched) {
101
- if (jb->unmatched.injections.used >= MAX_INJECTIONS) {
102
- log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);
103
- return false;
104
- }
105
- }
106
- else {
107
- if (jb->injections.used >= MAX_INJECTIONS) {
108
- log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);
109
- return false;
110
- }
111
- }
112
-
113
- if (unmatched) {
114
- key_value_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],
115
- key, key_len,
116
- value, value_len);
117
- } else {
118
- key_value_replace(&jb->injections.keys[jb->injections.used++],
119
- key, key_len,
120
- value, value_len);
121
- }
122
-
123
- return true;
124
-}
125
-
126
-bool log_job_add_rename(struct log_job *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len) {
127
- if(jb->renames.used >= MAX_RENAMES) {
128
- log2stderr("Error: too many renames. You can rename up to %d fields.", MAX_RENAMES);
67
+bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
68
+ if(!pattern || !*pattern) {
69
+ log2stderr("filename key cannot be empty.");
70
return false;
71
}
72
132
- struct key_rename *rn = &jb->renames.array[jb->renames.used++];
133
- rn->new_key = strndupz(new_key, new_key_len);
134
- rn->new_hash = XXH3_64bits(rn->new_key, strlen(rn->new_key));
135
- rn->old_key = strndupz(old_key, old_key_len);
136
- rn->old_hash = XXH3_64bits(rn->old_key, strlen(rn->old_key));
73
+ if(jb->pattern)
74
+ freez((char*)jb->pattern);
75
+
76
+ jb->pattern = strndupz(pattern, pattern_len);
77
78
return true;
79
}
80
141
-bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern) {
142
- if(jb->rewrites.used >= MAX_REWRITES) {
143
- log2stderr("Error: too many rewrites. You can add up to %d rewrite rules.", MAX_REWRITES);
81
+bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
82
+ if(jb->filter.include.re) {
83
+ log2stderr("FILTER INCLUDE: there is already an include filter set");
84
return false;
85
}
86
147
- pcre2_code *re = jb_compile_pcre2_pattern(search_pattern);
148
- if (!re) {
149
- return false;
150
- }
151
-
152
- struct key_rewrite *rw = &jb->rewrites.array[jb->rewrites.used++];
153
- rw->key = strdupz(key);
154
- rw->hash = XXH3_64bits(rw->key, strlen(rw->key));
155
- rw->search_pattern = strdupz(search_pattern);
156
- rw->replace_pattern = strdupz(replace_pattern);
157
- rw->re = re;
158
- rw->match_data = pcre2_match_data_create_from_pattern(rw->re, NULL);
159
-
160
- // Parse the replacement pattern and create the linked list
161
- if (!parse_replacement_pattern(rw)) {
162
- pcre2_match_data_free(rw->match_data);
163
- pcre2_code_free(rw->re);
164
- freez(rw->key);
165
- freez(rw->search_pattern);
166
- freez(rw->replace_pattern);
167
- jb->rewrites.used--;
87
+ if(!search_pattern_set(&jb->filter.include, pattern, pattern_len)) {
88
+ log2stderr("FILTER INCLUDE: failed: %s", jb->filter.include.error.txt);
89
return false;
90
}
91
92
return true;
93
}
94
174
-// ----------------------------------------------------------------------------
175
-
176
-struct key_dup *log_job_add_duplication_to_job(struct log_job *jb, const char *target, size_t target_len) {
177
- if (jb->dups.used >= MAX_KEY_DUPS) {
178
- log2stderr("Error: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS);
179
- return NULL;
180
- }
181
-
182
- struct key_dup *kd = &jb->dups.array[jb->dups.used++];
183
- kd->target = strndupz(target, target_len);
184
- kd->hash = XXH3_64bits(kd->target, target_len);
185
- kd->used = 0;
186
- kd->exposed = false;
187
-
188
- // Initialize values array
189
- for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) {
190
- kd->values[i].s = NULL;
191
- kd->values[i].size = 0;
95
+bool log_job_exclude_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len) {
96
+ if(jb->filter.exclude.re) {
97
+ log2stderr("FILTER INCLUDE: there is already an exclude filter set");
98
+ return false;
99
}
100
194
- return kd;
195
-}
196
-
197
-bool log_job_add_key_to_duplication(struct key_dup *kd, const char *key, size_t key_len) {
198
- if (kd->used >= MAX_KEY_DUPS_KEYS) {
199
- log2stderr("Error: Too many keys in duplication of target '%s'.", kd->target);
101
+ if(!search_pattern_set(&jb->filter.exclude, pattern, pattern_len)) {
102
+ log2stderr("FILTER EXCLUDE: failed: %s", jb->filter.exclude.error.txt);
103
return false;
104
}
105
203
- kd->keys[kd->used++] = strndupz(key, key_len);
106
return true;
107
}
108
109
// ----------------------------------------------------------------------------
208
-// command line params
209
-
210
-struct replacement_node *add_replacement_node(struct replacement_node **head, bool is_variable, const char *text) {
211
- struct replacement_node *new_node = mallocz(sizeof(struct replacement_node));
212
- if (!new_node)
213
- return NULL;
214
-
215
- new_node->is_variable = is_variable;
216
- new_node->s = text;
217
- new_node->len = strlen(text);
218
- new_node->next = NULL;
219
-
220
- if (*head == NULL)
221
- *head = new_node;
222
-
223
- else {
224
- struct replacement_node *current = *head;
225
-
226
- // append it
227
- while (current->next != NULL)
228
- current = current->next;
229
-
230
- current->next = new_node;
231
- }
232
-
233
- return new_node;
234
-}
235
-
236
-static bool parse_replacement_pattern(struct key_rewrite *rw) {
237
- const char *current = rw->replace_pattern;
238
-
239
- while (*current != '\0') {
240
- if (*current == '$' && *(current + 1) == '{') {
241
- // Start of a variable
242
- const char *end = strchr(current, '}');
243
- if (!end) {
244
- log2stderr("Error: Missing closing brace in replacement pattern: %s", rw->replace_pattern);
245
- return false;
246
- }
247
-
248
- size_t name_length = end - current - 2; // Length of the variable name
249
- char *variable_name = strndupz(current + 2, name_length);
250
- if (!variable_name) {
251
- log2stderr("Error: Memory allocation failed for variable name.");
252
- return false;
253
- }
254
-
255
- struct replacement_node *node = add_replacement_node(&(rw->nodes), true, variable_name);
256
- if (!node) {
257
- freez(variable_name);
258
- log2stderr("Error: Failed to add replacement node for variable.");
259
- return false;
260
- }
110
262
- current = end + 1; // Move past the variable
263
- }
264
- else {
265
- // Start of literal text
266
- const char *start = current;
267
- while (*current != '\0' && !(*current == '$' && *(current + 1) == '{')) {
268
- current++;
269
- }
270
-
271
- size_t text_length = current - start;
272
- char *text = strndupz(start, text_length);
273
- if (!text) {
274
- log2stderr("Error: Memory allocation failed for literal text.");
275
- return false;
276
- }
277
-
278
- struct replacement_node *node = add_replacement_node(&(rw->nodes), false, text);
279
- if (!node) {
280
- freez(text);
281
- log2stderr("Error: Failed to add replacement node for text.");
282
- return false;
283
- }
284
- }
285
- }
286
-
287
- return true;
288
-}
289
-
290
-static bool parse_rename(struct log_job *jb, const char *param) {
111
+static bool parse_rename(LOG_JOB *jb, const char *param) {
112
// Search for '=' in param
113
const char *equal_sign = strchr(param, '=');
114
if (!equal_sign || equal_sign == param) {
@@ -301,14 +122,14 @@ static bool parse_rename(struct log_job *jb, const char *param) {
122
const char *old_key = equal_sign + 1;
123
size_t old_key_len = strlen(old_key);
124
304
- return log_job_add_rename(jb, new_key, new_key_len, old_key, old_key_len);
125
+ return log_job_rename_add(jb, new_key, new_key_len, old_key, old_key_len);
126
}
127
128
static bool is_symbol(char c) {
129
return !isalpha(c) && !isdigit(c) && !iscntrl(c);
130
}
131
311
-static bool parse_rewrite(struct log_job *jb, const char *param) {
132
+static bool parse_rewrite(LOG_JOB *jb, const char *param) {
133
// Search for '=' in param
134
const char *equal_sign = strchr(param, '=');
135
if (!equal_sign || equal_sign == param) {
@@ -353,7 +174,7 @@ static bool parse_rewrite(struct log_job *jb, const char *param) {
174
char *search_pattern = strndupz(equal_sign + 2, second_separator - (equal_sign + 2));
175
char *replace_pattern = strdupz(second_separator + 1);
176
356
- bool ret = log_job_add_rewrite(jb, key, search_pattern, replace_pattern);
177
+ bool ret = log_job_rewrite_add(jb, key, search_pattern, replace_pattern);
178
179
freez(key);
180
freez(search_pattern);
@@ -362,7 +183,7 @@ static bool parse_rewrite(struct log_job *jb, const char *param) {
183
return ret;
184
}
185
365
-static bool parse_inject(struct log_job *jb, const char *value, bool unmatched) {
186
+static bool parse_inject(LOG_JOB *jb, const char *value, bool unmatched) {
187
const char *equal = strchr(value, '=');
188
if (!equal) {
189
log2stderr("Error: injection '%s' does not have an equal sign.", value);
@@ -371,12 +192,12 @@ static bool parse_inject(struct log_job *jb, const char *value, bool unmatched)
192
193
const char *key = value;
194
const char *val = equal + 1;
374
- log_job_add_injection(jb, key, equal - key, val, strlen(val), unmatched);
195
+ log_job_injection_add(jb, key, equal - key, val, strlen(val), unmatched);
196
197
return true;
198
}
199
379
-static bool parse_duplicate(struct log_job *jb, const char *value) {
200
+static bool parse_duplicate(LOG_JOB *jb, const char *value) {
201
const char *target = value;
202
const char *equal_sign = strchr(value, '=');
203
if (!equal_sign || equal_sign == target) {
@@ -385,13 +206,13 @@ static bool parse_duplicate(struct log_job *jb, const char *value) {
206
}
207
208
size_t target_len = equal_sign - target;
388
- struct key_dup *kd = log_job_add_duplication_to_job(jb, target, target_len);
209
+ DUPLICATION *kd = log_job_duplication_add(jb, target, target_len);
210
if(!kd) return false;
211
212
const char *key = equal_sign + 1;
213
while (key) {
214
if (kd->used >= MAX_KEY_DUPS_KEYS) {
394
- log2stderr("Error: too many keys in duplication of target '%s'.", kd->target);
215
+ log2stderr("Error: too many keys in duplication of target '%s'.", kd->target.key);
216
return false;
217
}
218
@@ -399,11 +220,11 @@ static bool parse_duplicate(struct log_job *jb, const char *value) {
220
size_t key_len;
221
if (comma) {
222
key_len = comma - key;
402
- log_job_add_key_to_duplication(kd, key, key_len);
223
+ log_job_duplication_key_add(kd, key, key_len);
224
key = comma + 1;
225
}
226
else {
406
- log_job_add_key_to_duplication(kd, key, strlen(key));
227
+ log_job_duplication_key_add(kd, key, strlen(key));
228
break; // No more keys
229
}
230
}
@@ -411,11 +232,11 @@ static bool parse_duplicate(struct log_job *jb, const char *value) {
232
return true;
233
}
234
414
-bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv) {
235
+bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
236
for (int i = 1; i < argc; i++) {
237
char *arg = argv[i];
238
if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) {
418
- log2journal_command_line_help(argv[0]);
239
+ log_job_command_line_help(argv[0]);
240
exit(0);
241
}
242
#if defined(NETDATA_DEV_MODE) || defined(NETDATA_INTERNAL_CHECKS)
@@ -446,7 +267,7 @@ bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv) {
267
}
268
else {
269
if (!jb->pattern) {
449
- jb->pattern = arg;
270
+ log_job_pattern_set(jb, arg, strlen(arg));
271
continue;
272
} else {
273
log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
@@ -456,11 +277,11 @@ bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv) {
277
}
278
279
if (strcmp(param, "--filename-key") == 0) {
459
- if(!log_job_add_filename_key(jb, value, value ? strlen(value) : 0))
280
+ if(!log_job_filename_key_set(jb, value, value ? strlen(value) : 0))
281
return false;
282
}
462
- if (strcmp(param, "--prefix") == 0) {
463
- if(!log_job_add_key_prefix(jb, value, value ? strlen(value) : 0))
283
+ else if (strcmp(param, "--prefix") == 0) {
284
+ if(!log_job_key_prefix_set(jb, value, value ? strlen(value) : 0))
285
return false;
286
}
287
#ifdef HAVE_LIBYAML
@@ -495,10 +316,18 @@ bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv) {
316
if (!parse_rename(jb, value))
317
return false;
318
}
319
+ else if (strcmp(param, "--include") == 0) {
320
+ if (!log_job_include_pattern_set(jb, value, strlen(value)))
321
+ return false;
322
+ }
323
+ else if (strcmp(param, "--exclude") == 0) {
324
+ if (!log_job_exclude_pattern_set(jb, value, strlen(value)))
325
+ return false;
326
+ }
327
else {
328
i--;
329
if (!jb->pattern) {
501
- jb->pattern = arg;
330
+ log_job_pattern_set(jb, arg, strlen(arg));
331
continue;
332
} else {
333
log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);
@@ -511,7 +340,7 @@ bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv) {
340
// Check if a pattern is set and exactly one pattern is specified
341
if (!jb->pattern) {
342
log2stderr("Error: Pattern not specified.");
514
- log2journal_command_line_help(argv[0]);
343
+ log_job_command_line_help(argv[0]);
344
return false;
345
}
346
collectors/log2journal/log2journal-pattern.c
new
+54
@@ -0,0 +1,54 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+void search_pattern_cleanup(SEARCH_PATTERN *sp) {
6
+ if(sp->pattern) {
7
+ freez((void *)sp->pattern);
8
+ sp->pattern = NULL;
9
+ }
10
+
11
+ if(sp->re) {
12
+ pcre2_code_free(sp->re);
13
+ sp->re = NULL;
14
+ }
15
+
16
+ if(sp->match_data) {
17
+ pcre2_match_data_free(sp->match_data);
18
+ sp->match_data = NULL;
19
+ }
20
+
21
+ txt_cleanup(&sp->error);
22
+}
23
+
24
+static void pcre2_error_message(SEARCH_PATTERN *sp, int rc, int pos) {
25
+ char msg[1024];
26
+ pcre2_get_error_in_buffer(msg, sizeof(msg), rc, pos);
27
+ txt_replace(&sp->error, msg, strlen(msg));
28
+}
29
+
30
+static inline bool compile_pcre2(SEARCH_PATTERN *sp) {
31
+ int error_number;
32
+ PCRE2_SIZE error_offset;
33
+ PCRE2_SPTR pattern_ptr = (PCRE2_SPTR)sp->pattern;
34
+
35
+ sp->re = pcre2_compile(pattern_ptr, PCRE2_ZERO_TERMINATED, 0, &error_number, &error_offset, NULL);
36
+ if (!sp->re) {
37
+ pcre2_error_message(sp, error_number, (int) error_offset);
38
+ return false;
39
+ }
40
+
41
+ return true;
42
+}
43
+
44
+bool search_pattern_set(SEARCH_PATTERN *sp, const char *search_pattern, size_t search_pattern_len) {
45
+ search_pattern_cleanup(sp);
46
+
47
+ sp->pattern = strndupz(search_pattern, search_pattern_len);
48
+ if (!compile_pcre2(sp))
49
+ return false;
50
+
51
+ sp->match_data = pcre2_match_data_create_from_pattern(sp->re, NULL);
52
+
53
+ return true;
54
+}
collectors/log2journal/log2journal-pcre2.c
new
+139
@@ -0,0 +1,139 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+#define PCRE2_ERROR_LINE_MAX 1024
6
+#define PCRE2_KEY_MAX 1024
7
+
8
+struct pcre2_state {
9
+ LOG_JOB *jb;
10
+
11
+ const char *line;
12
+ uint32_t pos;
13
+ uint32_t key_start;
14
+
15
+ pcre2_code *re;
16
+ pcre2_match_data *match_data;
17
+
18
+ char key[PCRE2_KEY_MAX];
19
+ char msg[PCRE2_ERROR_LINE_MAX];
20
+};
21
+
22
+static inline void copy_and_convert_key(PCRE2_STATE *pcre2, const char *key) {
23
+ char *d = &pcre2->key[pcre2->key_start];
24
+ size_t remaining = sizeof(pcre2->key) - pcre2->key_start;
25
+
26
+ while(remaining >= 2 && *key) {
27
+ *d = journal_key_characters_map[(unsigned) (*key)];
28
+ remaining--;
29
+ key++;
30
+ d++;
31
+ }
32
+
33
+ *d = '\0';
34
+}
35
+
36
+static inline void jb_traverse_pcre2_named_groups_and_send_keys(PCRE2_STATE *pcre2, pcre2_code *re, pcre2_match_data *match_data, char *line) {
37
+ PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
38
+ uint32_t names_count;
39
+ pcre2_pattern_info(re, PCRE2_INFO_NAMECOUNT, &names_count);
40
+
41
+ if (names_count > 0) {
42
+ PCRE2_SPTR name_table;
43
+ pcre2_pattern_info(re, PCRE2_INFO_NAMETABLE, &name_table);
44
+ uint32_t name_entry_size;
45
+ pcre2_pattern_info(re, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size);
46
+
47
+ const unsigned char *table_ptr = name_table;
48
+ for (uint32_t i = 0; i < names_count; i++) {
49
+ int n = (table_ptr[0] << 8) | table_ptr[1];
50
+ const char *group_name = (const char *)(table_ptr + 2);
51
+
52
+ PCRE2_SIZE start_offset = ovector[2 * n];
53
+ PCRE2_SIZE end_offset = ovector[2 * n + 1];
54
+ PCRE2_SIZE group_length = end_offset - start_offset;
55
+
56
+ copy_and_convert_key(pcre2, group_name);
57
+ log_job_send_extracted_key_value(pcre2->jb, pcre2->key, line + start_offset, group_length);
58
+
59
+ table_ptr += name_entry_size;
60
+ }
61
+ }
62
+}
63
+
64
+void pcre2_get_error_in_buffer(char *msg, size_t msg_len, int rc, int pos) {
65
+ int l;
66
+
67
+ if(pos >= 0)
68
+ l = snprintf(msg, msg_len, "PCRE2 error %d at pos %d on: ", rc, pos);
69
+ else
70
+ l = snprintf(msg, msg_len, "PCRE2 error %d on: ", rc);
71
+
72
+ pcre2_get_error_message(rc, (PCRE2_UCHAR *)&msg[l], msg_len - l);
73
+}
74
+
75
+static void pcre2_error_message(PCRE2_STATE *pcre2, int rc, int pos) {
76
+ pcre2_get_error_in_buffer(pcre2->msg, sizeof(pcre2->msg), rc, pos);
77
+}
78
+
79
+bool pcre2_has_error(PCRE2_STATE *pcre2) {
80
+ return !pcre2->re || pcre2->msg[0];
81
+}
82
+
83
+PCRE2_STATE *pcre2_parser_create(LOG_JOB *jb) {
84
+ PCRE2_STATE *pcre2 = mallocz(sizeof(PCRE2_STATE));
85
+ memset(pcre2, 0, sizeof(PCRE2_STATE));
86
+ pcre2->jb = jb;
87
+
88
+ if(jb->prefix)
89
+ pcre2->key_start = copy_to_buffer(pcre2->key, sizeof(pcre2->key), pcre2->jb->prefix, strlen(pcre2->jb->prefix));
90
+
91
+ int rc;
92
+ PCRE2_SIZE pos;
93
+ pcre2->re = pcre2_compile((PCRE2_SPTR)jb->pattern, PCRE2_ZERO_TERMINATED, 0, &rc, &pos, NULL);
94
+ if (!pcre2->re) {
95
+ pcre2_error_message(pcre2, rc, pos);
96
+ return pcre2;
97
+ }
98
+
99
+ pcre2->match_data = pcre2_match_data_create_from_pattern(pcre2->re, NULL);
100
+
101
+ return pcre2;
102
+}
103
+
104
+void pcre2_parser_destroy(PCRE2_STATE *pcre2) {
105
+ if(pcre2)
106
+ freez(pcre2);
107
+}
108
+
109
+const char *pcre2_parser_error(PCRE2_STATE *pcre2) {
110
+ return pcre2->msg;
111
+}
112
+
113
+bool pcre2_parse_document(PCRE2_STATE *pcre2, const char *txt, size_t len) {
114
+ pcre2->line = txt;
115
+ pcre2->pos = 0;
116
+ pcre2->msg[0] = '\0';
117
+
118
+ if(!len)
119
+ len = strlen(txt);
120
+
121
+ int rc = pcre2_match(pcre2->re, (PCRE2_SPTR)pcre2->line, len, 0, 0, pcre2->match_data, NULL);
122
+ if(rc < 0) {
123
+ pcre2_error_message(pcre2, rc, -1);
124
+ return false;
125
+ }
126
+
127
+ jb_traverse_pcre2_named_groups_and_send_keys(pcre2, pcre2->re, pcre2->match_data, (char *)pcre2->line);
128
+
129
+ return true;
130
+}
131
+
132
+void pcre2_test(void) {
133
+ LOG_JOB jb = { .prefix = "NIGNX_" };
134
+ PCRE2_STATE *pcre2 = pcre2_parser_create(&jb);
135
+
136
+ pcre2_parse_document(pcre2, "{\"value\":\"\\u\\u039A\\u03B1\\u03BB\\u03B7\\u03BC\\u03AD\\u03C1\\u03B1\"}", 0);
137
+
138
+ pcre2_parser_destroy(pcre2);
139
+}
collectors/log2journal/log2journal-rename.c
new
+21
@@ -0,0 +1,21 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+void rename_cleanup(RENAME *rn) {
6
+ hashed_key_cleanup(&rn->new_key);
7
+ hashed_key_cleanup(&rn->old_key);
8
+}
9
+
10
+bool log_job_rename_add(LOG_JOB *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len) {
11
+ if(jb->renames.used >= MAX_RENAMES) {
12
+ log2stderr("Error: too many renames. You can rename up to %d fields.", MAX_RENAMES);
13
+ return false;
14
+ }
15
+
16
+ RENAME *rn = &jb->renames.array[jb->renames.used++];
17
+ hashed_key_len_set(&rn->new_key, new_key, new_key_len);
18
+ hashed_key_len_set(&rn->old_key, old_key, old_key_len);
19
+
20
+ return true;
21
+}
collectors/log2journal/log2journal-replace.c
new
+104
@@ -0,0 +1,104 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+void replace_node_free(REPLACE_NODE *rpn) {
6
+ hashed_key_cleanup(&rpn->name);
7
+ rpn->next = NULL;
8
+ freez(rpn);
9
+}
10
+
11
+void replace_pattern_cleanup(REPLACE_PATTERN *rp) {
12
+ if(rp->pattern) {
13
+ freez((void *)rp->pattern);
14
+ rp->pattern = NULL;
15
+ }
16
+
17
+ while(rp->nodes) {
18
+ REPLACE_NODE *rpn = rp->nodes;
19
+ rp->nodes = rpn->next;
20
+ replace_node_free(rpn);
21
+ }
22
+}
23
+
24
+static REPLACE_NODE *replace_pattern_add_node(REPLACE_NODE **head, bool is_variable, const char *text) {
25
+ REPLACE_NODE *new_node = callocz(1, sizeof(REPLACE_NODE));
26
+ if (!new_node)
27
+ return NULL;
28
+
29
+ hashed_key_set(&new_node->name, text);
30
+ new_node->is_variable = is_variable;
31
+ new_node->next = NULL;
32
+
33
+ if (*head == NULL)
34
+ *head = new_node;
35
+
36
+ else {
37
+ REPLACE_NODE *current = *head;
38
+
39
+ // append it
40
+ while (current->next != NULL)
41
+ current = current->next;
42
+
43
+ current->next = new_node;
44
+ }
45
+
46
+ return new_node;
47
+}
48
+
49
+bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern) {
50
+ replace_pattern_cleanup(rp);
51
+
52
+ rp->pattern = strdupz(pattern);
53
+ const char *current = rp->pattern;
54
+
55
+ while (*current != '\0') {
56
+ if (*current == '$' && *(current + 1) == '{') {
57
+ // Start of a variable
58
+ const char *end = strchr(current, '}');
59
+ if (!end) {
60
+ log2stderr("Error: Missing closing brace in replacement pattern: %s", rp->pattern);
61
+ return false;
62
+ }
63
+
64
+ size_t name_length = end - current - 2; // Length of the variable name
65
+ char *variable_name = strndupz(current + 2, name_length);
66
+ if (!variable_name) {
67
+ log2stderr("Error: Memory allocation failed for variable name.");
68
+ return false;
69
+ }
70
+
71
+ REPLACE_NODE *node = replace_pattern_add_node(&(rp->nodes), true, variable_name);
72
+ if (!node) {
73
+ freez(variable_name);
74
+ log2stderr("Error: Failed to add replacement node for variable.");
75
+ return false;
76
+ }
77
+
78
+ current = end + 1; // Move past the variable
79
+ }
80
+ else {
81
+ // Start of literal text
82
+ const char *start = current;
83
+ while (*current != '\0' && !(*current == '$' && *(current + 1) == '{')) {
84
+ current++;
85
+ }
86
+
87
+ size_t text_length = current - start;
88
+ char *text = strndupz(start, text_length);
89
+ if (!text) {
90
+ log2stderr("Error: Memory allocation failed for literal text.");
91
+ return false;
92
+ }
93
+
94
+ REPLACE_NODE *node = replace_pattern_add_node(&(rp->nodes), false, text);
95
+ if (!node) {
96
+ freez(text);
97
+ log2stderr("Error: Failed to add replacement node for text.");
98
+ return false;
99
+ }
100
+ }
101
+ }
102
+
103
+ return true;
104
+}
collectors/log2journal/log2journal-rewrite.c
new
+28
@@ -0,0 +1,28 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "log2journal.h"
4
+
5
+void rewrite_cleanup(REWRITE *rw) {
6
+ hashed_key_cleanup(&rw->key);
7
+ search_pattern_cleanup(&rw->search);
8
+ replace_pattern_cleanup(&rw->replace);
9
+}
10
+
11
+bool log_job_rewrite_add(LOG_JOB *jb, const char *key, const char *search_pattern, const char *replace_pattern) {
12
+ if(jb->rewrites.used >= MAX_REWRITES) {
13
+ log2stderr("Error: too many rewrites. You can add up to %d rewrite rules.", MAX_REWRITES);
14
+ return false;
15
+ }
16
+
17
+ REWRITE *rw = &jb->rewrites.array[jb->rewrites.used++];
18
+ hashed_key_set(&rw->key, key);
19
+
20
+ if(!search_pattern_set(&rw->search, search_pattern, strlen(search_pattern)) ||
21
+ !replace_pattern_set(&rw->replace, replace_pattern)) {
22
+ rewrite_cleanup(rw);
23
+ jb->rewrites.used--;
24
+ return false;
25
+ }
26
+
27
+ return true;
28
+}
collectors/log2journal/log2journal-yaml.c
+163
-80
@@ -83,7 +83,7 @@ static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, si
83
}
84
85
#define yaml_parse(parser, event) yaml_parse_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__)
86
-static bool yaml_parse_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file) {
86
+static bool yaml_parse_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line __maybe_unused, const char *function __maybe_unused, const char *file __maybe_unused) {
87
if (!yaml_parser_parse(parser, event)) {
88
yaml_error(parser, NULL, "YAML parser error %d", parser->error);
89
return false;
@@ -130,15 +130,15 @@ static bool yaml_scalar_matches_with_trace(yaml_event_t *event, const char *s, s
130
131
// ----------------------------------------------------------------------------
132
133
-static struct key_dup *yaml_parse_duplicate_key(struct log_job *jb, yaml_parser_t *parser) {
133
+static DUPLICATION *yaml_parse_duplicate_key(LOG_JOB *jb, yaml_parser_t *parser) {
134
yaml_event_t event;
135
136
if (!yaml_parse(parser, &event))
137
return false;
138
139
- struct key_dup *kd = NULL;
139
+ DUPLICATION *kd = NULL;
140
if(event.type == YAML_SCALAR_EVENT) {
141
- kd = log_job_add_duplication_to_job(jb, (char *) event.data.scalar.value, event.data.scalar.length);
141
+ kd = log_job_duplication_add(jb, (char *) event.data.scalar.value, event.data.scalar.length);
142
}
143
else
144
yaml_error(parser, &event, "duplicate key must be a scalar.");
@@ -147,29 +147,30 @@ static struct key_dup *yaml_parse_duplicate_key(struct log_job *jb, yaml_parser_
147
return kd;
148
}
149
150
-static size_t yaml_parse_duplicate_from(struct log_job *jb, yaml_parser_t *parser, struct key_dup *kd) {
150
+static size_t yaml_parse_duplicate_from(LOG_JOB *jb __maybe_unused, yaml_parser_t *parser, DUPLICATION *kd) {
151
size_t errors = 0;
152
yaml_event_t event;
153
154
if (!yaml_parse(parser, &event))
155
return 1;
156
157
- bool ret = true;
158
- if(event.type == YAML_SCALAR_EVENT)
159
- ret = log_job_add_key_to_duplication(kd, (char *) event.data.scalar.value, event.data.scalar.length);
160
-
157
+ if(event.type == YAML_SCALAR_EVENT) {
158
+ if(!log_job_duplication_key_add(kd, (char *) event.data.scalar.value, event.data.scalar.length))
159
+ errors++;
160
+ }
161
else if(event.type == YAML_SEQUENCE_START_EVENT) {
162
bool finished = false;
163
while(!errors && !finished) {
164
yaml_event_t sub_event;
165
if (!yaml_parse(parser, &sub_event))
166
- return errors++;
166
+ return ++errors;
167
else {
168
- if (sub_event.type == YAML_SCALAR_EVENT)
169
- log_job_add_key_to_duplication(kd, (char *) sub_event.data.scalar.value
170
- , sub_event.data.scalar.length
171
- );
172
-
168
+ if (sub_event.type == YAML_SCALAR_EVENT) {
169
+ if(!log_job_duplication_key_add(kd, (char *) sub_event.data.scalar.value
170
+ , sub_event.data.scalar.length
171
+ ))
172
+ errors++;
173
+ }
174
else if (sub_event.type == YAML_SEQUENCE_END_EVENT)
175
finished = true;
176
@@ -184,7 +185,7 @@ static size_t yaml_parse_duplicate_from(struct log_job *jb, yaml_parser_t *parse
185
return errors;
186
}
187
187
-static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_job *jb) {
188
+static size_t yaml_parse_filename_injection(yaml_parser_t *parser, LOG_JOB *jb) {
189
yaml_event_t event;
190
size_t errors = 0;
191
@@ -200,8 +201,9 @@ static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_jo
201
errors++;
202
203
else {
203
- if (event.type == YAML_SCALAR_EVENT) {
204
- if(!log_job_add_filename_key(jb, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length))
204
+ if (sub_event.type == YAML_SCALAR_EVENT) {
205
+ if(!log_job_filename_key_set(jb, (char *) sub_event.data.scalar.value,
206
+ sub_event.data.scalar.length))
207
errors++;
208
}
209
@@ -221,7 +223,79 @@ static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_jo
223
return errors;
224
}
225
224
-static size_t yaml_parse_prefix(yaml_parser_t *parser, struct log_job *jb) {
226
+static size_t yaml_parse_filters(yaml_parser_t *parser, LOG_JOB *jb) {
227
+ if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))
228
+ return 1;
229
+
230
+ size_t errors = 0;
231
+ bool finished = false;
232
+
233
+ while(!errors && !finished) {
234
+ yaml_event_t event;
235
+
236
+ if(!yaml_parse(parser, &event))
237
+ return 1;
238
+
239
+ if(event.type == YAML_SCALAR_EVENT) {
240
+ if(yaml_scalar_matches(&event, "include", strlen("include"))) {
241
+ yaml_event_t sub_event;
242
+ if(!yaml_parse(parser, &sub_event))
243
+ errors++;
244
+
245
+ else {
246
+ if(sub_event.type == YAML_SCALAR_EVENT) {
247
+ if(!log_job_include_pattern_set(jb, (char *) sub_event.data.scalar.value,
248
+ sub_event.data.scalar.length))
249
+ errors++;
250
+ }
251
+
252
+ else {
253
+ yaml_error(parser, &sub_event, "expected the include as %s",
254
+ yaml_event_name(YAML_SCALAR_EVENT));
255
+ errors++;
256
+ }
257
+
258
+ yaml_event_delete(&sub_event);
259
+ }
260
+ }
261
+ else if(yaml_scalar_matches(&event, "exclude", strlen("exclude"))) {
262
+ yaml_event_t sub_event;
263
+ if(!yaml_parse(parser, &sub_event))
264
+ errors++;
265
+
266
+ else {
267
+ if(sub_event.type == YAML_SCALAR_EVENT) {
268
+ if(!log_job_exclude_pattern_set(jb,(char *) sub_event.data.scalar.value,
269
+ sub_event.data.scalar.length))
270
+ errors++;
271
+ }
272
+
273
+ else {
274
+ yaml_error(parser, &sub_event, "expected the exclude as %s",
275
+ yaml_event_name(YAML_SCALAR_EVENT));
276
+ errors++;
277
+ }
278
+
279
+ yaml_event_delete(&sub_event);
280
+ }
281
+ }
282
+ }
283
+ else if(event.type == YAML_MAPPING_END_EVENT)
284
+ finished = true;
285
+ else {
286
+ yaml_error(parser, &event, "expected %s or %s",
287
+ yaml_event_name(YAML_SCALAR_EVENT),
288
+ yaml_event_name(YAML_MAPPING_END_EVENT));
289
+ errors++;
290
+ }
291
+
292
+ yaml_event_delete(&event);
293
+ }
294
+
295
+ return errors;
296
+}
297
+
298
+static size_t yaml_parse_prefix(yaml_parser_t *parser, LOG_JOB *jb) {
299
yaml_event_t event;
300
size_t errors = 0;
301
@@ -229,7 +303,7 @@ static size_t yaml_parse_prefix(yaml_parser_t *parser, struct log_job *jb) {
303
return 1;
304
305
if (event.type == YAML_SCALAR_EVENT) {
232
- if(!log_job_add_key_prefix(jb, (char *)event.data.scalar.value, event.data.scalar.length))
306
+ if(!log_job_key_prefix_set(jb, (char *) event.data.scalar.value, event.data.scalar.length))
307
errors++;
308
}
309
@@ -237,11 +311,11 @@ static size_t yaml_parse_prefix(yaml_parser_t *parser, struct log_job *jb) {
311
return errors;
312
}
313
240
-static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, struct log_job *jb) {
314
+static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, LOG_JOB *jb) {
315
if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
316
return 1;
317
244
- struct key_dup *kd = NULL;
318
+ DUPLICATION *kd = NULL;
319
320
// Expecting a key-value pair for each duplicate
321
bool finished;
@@ -306,7 +380,7 @@ static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, struct log_
380
return errors;
381
}
382
309
-static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
383
+static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, LOG_JOB *jb, bool unmatched) {
384
yaml_event_t event;
385
if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
386
yaml_error(parser, &event, "Expected scalar for constant field injection key");
@@ -337,7 +411,7 @@ static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, struct lo
411
412
value = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
413
340
- if(!log_job_add_injection(jb, key, strlen(key), value, strlen(value), unmatched))
414
+ if(!log_job_injection_add(jb, key, strlen(key), value, strlen(value), unmatched))
415
ret = false;
416
else
417
ret = true;
@@ -351,7 +425,7 @@ cleanup:
425
return !ret ? 1 : 0;
426
}
427
354
-static bool yaml_parse_injection_mapping(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
428
+static bool yaml_parse_injection_mapping(yaml_parser_t *parser, LOG_JOB *jb, bool unmatched) {
429
yaml_event_t event;
430
size_t errors = 0;
431
bool finished = false;
@@ -388,7 +462,7 @@ static bool yaml_parse_injection_mapping(yaml_parser_t *parser, struct log_job *
462
return errors == 0;
463
}
464
391
-static size_t yaml_parse_injections(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {
465
+static size_t yaml_parse_injections(yaml_parser_t *parser, LOG_JOB *jb, bool unmatched) {
466
yaml_event_t event;
467
size_t errors = 0;
468
bool finished = false;
@@ -424,7 +498,7 @@ static size_t yaml_parse_injections(yaml_parser_t *parser, struct log_job *jb, b
498
return errors;
499
}
500
427
-static size_t yaml_parse_unmatched(yaml_parser_t *parser, struct log_job *jb) {
501
+static size_t yaml_parse_unmatched(yaml_parser_t *parser, LOG_JOB *jb) {
502
size_t errors = 0;
503
bool finished = false;
504
@@ -477,7 +551,7 @@ static size_t yaml_parse_unmatched(yaml_parser_t *parser, struct log_job *jb) {
551
return errors;
552
}
553
480
-static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
554
+static size_t yaml_parse_rewrites(yaml_parser_t *parser, LOG_JOB *jb) {
555
size_t errors = 0;
556
557
if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
@@ -494,7 +568,7 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
568
switch (event.type) {
569
case YAML_MAPPING_START_EVENT:
570
{
497
- struct key_rewrite rw = {0};
571
+ REWRITE rw = { 0 };
572
573
bool mapping_finished = false;
574
while (!errors && !mapping_finished) {
@@ -511,7 +585,7 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
585
yaml_error(parser, &sub_event, "Expected scalar for rewrite key");
586
errors++;
587
} else {
514
- rw.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
588
+ rw.key.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
589
yaml_event_delete(&sub_event);
590
}
591
} else if (yaml_scalar_matches(&sub_event, "search", strlen("search"))) {
@@ -519,7 +593,7 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
593
yaml_error(parser, &sub_event, "Expected scalar for rewrite search pattern");
594
errors++;
595
} else {
522
- rw.search_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
596
+ rw.search.pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
597
yaml_event_delete(&sub_event);
598
}
599
} else if (yaml_scalar_matches(&sub_event, "replace", strlen("replace"))) {
@@ -527,7 +601,7 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
601
yaml_error(parser, &sub_event, "Expected scalar for rewrite replace pattern");
602
errors++;
603
} else {
530
- rw.replace_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
604
+ rw.replace.pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
605
yaml_event_delete(&sub_event);
606
}
607
} else {
@@ -537,14 +611,11 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
611
break;
612
613
case YAML_MAPPING_END_EVENT:
540
- if(rw.key && rw.search_pattern && rw.replace_pattern) {
541
- if (!log_job_add_rewrite(jb, rw.key, rw.search_pattern, rw.replace_pattern))
614
+ if(rw.key.key && rw.search.pattern && rw.replace.pattern) {
615
+ if (!log_job_rewrite_add(jb, rw.key.key, rw.search.pattern, rw.replace.pattern))
616
errors++;
617
}
544
- freez(rw.key);
545
- freez(rw.search_pattern);
546
- freez(rw.replace_pattern);
547
- memset(&rw, 0, sizeof(rw));
618
+ rewrite_cleanup(&rw);
619
620
mapping_finished = true;
621
break;
@@ -576,7 +647,7 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {
647
return errors;
648
}
649
579
-static size_t yaml_parse_renames(yaml_parser_t *parser, struct log_job *jb) {
650
+static size_t yaml_parse_renames(yaml_parser_t *parser, LOG_JOB *jb) {
651
size_t errors = 0;
652
653
if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
@@ -593,7 +664,7 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, struct log_job *jb) {
664
switch (event.type) {
665
case YAML_MAPPING_START_EVENT:
666
{
596
- struct key_rename rn = {0};
667
+ struct key_rename rn = { 0 };
668
669
bool mapping_finished = false;
670
while (!errors && !mapping_finished) {
@@ -610,7 +681,7 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, struct log_job *jb) {
681
yaml_error(parser, &sub_event, "Expected scalar for rename new_key");
682
errors++;
683
} else {
613
- rn.new_key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
684
+ hashed_key_len_set(&rn.new_key, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
685
yaml_event_delete(&sub_event);
686
}
687
} else if (yaml_scalar_matches(&sub_event, "old_key", strlen("old_key"))) {
@@ -618,7 +689,7 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, struct log_job *jb) {
689
yaml_error(parser, &sub_event, "Expected scalar for rename old_key");
690
errors++;
691
} else {
621
- rn.old_key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
692
+ hashed_key_len_set(&rn.old_key, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
693
yaml_event_delete(&sub_event);
694
}
695
} else {
@@ -628,13 +699,12 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, struct log_job *jb) {
699
break;
700
701
case YAML_MAPPING_END_EVENT:
631
- if(rn.old_key && rn.new_key) {
632
- if (!log_job_add_rename(jb, rn.new_key, strlen(rn.new_key), rn.old_key, strlen(rn.old_key)))
702
+ if(rn.old_key.key && rn.new_key.key) {
703
+ if (!log_job_rename_add(jb, rn.new_key.key, rn.new_key.len,
704
+ rn.old_key.key, rn.old_key.len))
705
errors++;
706
}
635
- freez(rn.new_key);
636
- freez(rn.old_key);
637
- memset(&rn, 0, sizeof(rn));
707
+ rename_cleanup(&rn);
708
709
mapping_finished = true;
710
break;
@@ -666,7 +736,7 @@ static size_t yaml_parse_renames(yaml_parser_t *parser, struct log_job *jb) {
736
return errors;
737
}
738
669
-static size_t yaml_parse_pattern(yaml_parser_t *parser, struct log_job *jb) {
739
+static size_t yaml_parse_pattern(yaml_parser_t *parser, LOG_JOB *jb) {
740
yaml_event_t event;
741
size_t errors = 0;
742
@@ -674,7 +744,7 @@ static size_t yaml_parse_pattern(yaml_parser_t *parser, struct log_job *jb) {
744
return 1;
745
746
if(event.type == YAML_SCALAR_EVENT)
677
- jb->pattern = strndupz((char *)event.data.scalar.value, event.data.scalar.length);
747
+ log_job_pattern_set(jb, (char *) event.data.scalar.value, event.data.scalar.length);
748
else {
749
yaml_error(parser, &event, "unexpected event type");
750
errors++;
@@ -684,7 +754,7 @@ static size_t yaml_parse_pattern(yaml_parser_t *parser, struct log_job *jb) {
754
return errors;
755
}
756
687
-static size_t yaml_parse_initialized(yaml_parser_t *parser, struct log_job *jb) {
757
+static size_t yaml_parse_initialized(yaml_parser_t *parser, LOG_JOB *jb) {
758
size_t errors = 0;
759
760
if(!yaml_parse_expect_event(parser, YAML_STREAM_START_EVENT)) {
@@ -730,6 +800,9 @@ static size_t yaml_parse_initialized(yaml_parser_t *parser, struct log_job *jb)
800
else if (yaml_scalar_matches(&event, "filename", strlen("filename")))
801
errors += yaml_parse_filename_injection(parser, jb);
802
803
+ else if (yaml_scalar_matches(&event, "filter", strlen("filter")))
804
+ errors += yaml_parse_filters(parser, jb);
805
+
806
else if (yaml_scalar_matches(&event, "duplicate", strlen("duplicate")))
807
errors += yaml_parse_duplicates_injection(parser, jb);
808
@@ -755,12 +828,12 @@ static size_t yaml_parse_initialized(yaml_parser_t *parser, struct log_job *jb)
828
yaml_event_delete(&event);
829
}
830
758
- if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_END_EVENT)) {
831
+ if(!errors && !yaml_parse_expect_event(parser, YAML_DOCUMENT_END_EVENT)) {
832
errors++;
833
goto cleanup;
834
}
835
763
- if(!yaml_parse_expect_event(parser, YAML_STREAM_END_EVENT)) {
836
+ if(!errors && !yaml_parse_expect_event(parser, YAML_STREAM_END_EVENT)) {
837
errors++;
838
goto cleanup;
839
}
@@ -769,7 +842,7 @@ cleanup:
842
return errors;
843
}
844
772
-bool yaml_parse_file(const char *config_file_path, struct log_job *jb) {
845
+bool yaml_parse_file(const char *config_file_path, LOG_JOB *jb) {
846
if(!config_file_path || !*config_file_path) {
847
log2stderr("yaml configuration filename cannot be empty.");
848
return false;
@@ -792,7 +865,7 @@ bool yaml_parse_file(const char *config_file_path, struct log_job *jb) {
865
return errors == 0;
866
}
867
795
-bool yaml_parse_config(const char *config_name, struct log_job *jb) {
868
+bool yaml_parse_config(const char *config_name, LOG_JOB *jb) {
869
char filename[FILENAME_MAX + 1];
870
871
snprintf(filename, sizeof(filename), "%s/%s.yaml", LOG2JOURNAL_CONFIG_PATH, config_name);
@@ -814,8 +887,7 @@ static void yaml_print_multiline_value(const char *s, size_t depth) {
887
888
size_t len = next ? (size_t)(next - s) : strlen(s);
889
char buf[len + 1];
817
- strncpy(buf, s, len);
818
- buf[len] = '\0';
890
+ copy_to_buffer(buf, sizeof(buf), s, len);
891
892
fprintf(stderr, "%.*s%s%s",
893
(int)(depth * 2), " ",
@@ -871,7 +943,7 @@ static void yaml_print_node(const char *key, const char *value, size_t depth, bo
943
}
944
}
945
874
-void log_job_to_yaml(struct log_job *jb) {
946
+void log_job_configuration_to_yaml(LOG_JOB *jb) {
947
if(jb->pattern)
948
yaml_print_node("pattern", jb->pattern, 0, false);
949
@@ -886,26 +958,37 @@ void log_job_to_yaml(struct log_job *jb) {
958
yaml_print_node("key", jb->filename.key, 1, false);
959
}
960
889
- if(jb->dups.used) {
961
+ if(jb->filter.include.pattern || jb->filter.exclude.pattern) {
962
fprintf(stderr, "\n");
891
- yaml_print_node("duplicate", NULL, 0, false);
892
- for(size_t i = 0; i < jb->dups.used ;i++) {
893
- struct key_dup *kd = &jb->dups.array[i];
894
- yaml_print_node("key", kd->target, 1, true);
895
- yaml_print_node("values_of", NULL, 2, false);
963
+ yaml_print_node("filter", NULL, 0, false);
964
897
- for(size_t k = 0; k < kd->used ;k++)
898
- yaml_print_node(NULL, kd->keys[k], 3, true);
965
+ if(jb->filter.include.pattern)
966
+ yaml_print_node("include", jb->filter.include.pattern, 1, false);
967
+
968
+ if(jb->filter.exclude.pattern)
969
+ yaml_print_node("exclude", jb->filter.exclude.pattern, 1, false);
970
+ }
971
+
972
+ if(jb->renames.used) {
973
+ fprintf(stderr, "\n");
974
+ yaml_print_node("rename", NULL, 0, false);
975
+
976
+ for(size_t i = 0; i < jb->renames.used ;i++) {
977
+ yaml_print_node("new_key", jb->renames.array[i].new_key.key, 1, true);
978
+ yaml_print_node("old_key", jb->renames.array[i].old_key.key, 2, false);
979
}
980
}
981
902
- if(jb->injections.used) {
982
+ if(jb->dups.used) {
983
fprintf(stderr, "\n");
904
- yaml_print_node("inject", NULL, 0, false);
984
+ yaml_print_node("duplicate", NULL, 0, false);
985
+ for(size_t i = 0; i < jb->dups.used ;i++) {
986
+ DUPLICATION *kd = &jb->dups.array[i];
987
+ yaml_print_node("key", kd->target.key, 1, true);
988
+ yaml_print_node("values_of", NULL, 2, false);
989
906
- for (size_t i = 0; i < jb->injections.used; i++) {
907
- yaml_print_node("key", jb->injections.keys[i].key, 1, true);
908
- yaml_print_node("value", jb->injections.keys[i].value.s, 2, false);
990
+ for(size_t k = 0; k < kd->used ;k++)
991
+ yaml_print_node(NULL, kd->keys[k].key, 3, true);
992
}
993
}
994
@@ -914,19 +997,19 @@ void log_job_to_yaml(struct log_job *jb) {
997
yaml_print_node("rewrite", NULL, 0, false);
998
999
for(size_t i = 0; i < jb->rewrites.used ;i++) {
917
- yaml_print_node("key", jb->rewrites.array[i].key, 1, true);
918
- yaml_print_node("search", jb->rewrites.array[i].search_pattern, 2, false);
919
- yaml_print_node("replace", jb->rewrites.array[i].replace_pattern, 2, false);
1000
+ yaml_print_node("key", jb->rewrites.array[i].key.key, 1, true);
1001
+ yaml_print_node("search", jb->rewrites.array[i].search.pattern, 2, false);
1002
+ yaml_print_node("replace", jb->rewrites.array[i].replace.pattern, 2, false);
1003
}
1004
}
1005
923
- if(jb->renames.used) {
1006
+ if(jb->injections.used) {
1007
fprintf(stderr, "\n");
925
- yaml_print_node("rename", NULL, 0, false);
1008
+ yaml_print_node("inject", NULL, 0, false);
1009
927
- for(size_t i = 0; i < jb->renames.used ;i++) {
928
- yaml_print_node("new_key", jb->renames.array[i].new_key, 1, true);
929
- yaml_print_node("old_key", jb->renames.array[i].old_key, 2, false);
1010
+ for (size_t i = 0; i < jb->injections.used; i++) {
1011
+ yaml_print_node("key", jb->injections.keys[i].key.key, 1, true);
1012
+ yaml_print_node("value", jb->injections.keys[i].value.txt, 2, false);
1013
}
1014
}
1015
@@ -942,8 +1025,8 @@ void log_job_to_yaml(struct log_job *jb) {
1025
yaml_print_node("inject", NULL, 1, false);
1026
1027
for (size_t i = 0; i < jb->unmatched.injections.used; i++) {
945
- yaml_print_node("key", jb->unmatched.injections.keys[i].key, 2, true);
946
- yaml_print_node("value", jb->unmatched.injections.keys[i].value.s, 3, false);
1028
+ yaml_print_node("key", jb->unmatched.injections.keys[i].key.key, 2, true);
1029
+ yaml_print_node("value", jb->unmatched.injections.keys[i].value.txt, 3, false);
1030
}
1031
}
1032
}
collectors/log2journal/log2journal.c
+271
-150
@@ -2,63 +2,139 @@
2
3
#include "log2journal.h"
4
5
-// ----------------------------------------------------------------------------
6
-
7
-static char *rewrite_value(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len) {
8
- static __thread char rewritten_value[MAX_VALUE_LEN + 1];
9
-
10
- for (size_t i = 0; i < jb->rewrites.used; i++) {
11
- struct key_rewrite *rw = &jb->rewrites.array[i];
12
-
13
- if (rw->hash == hash && strcmp(rw->key, key) == 0) {
14
- if (!jb_pcre2_match(rw->re, rw->match_data, (char *)value, value_len, false)) {
15
- continue; // No match found, skip to next rewrite rule
16
- }
5
+static inline void send_duplications_for_key(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t value_len);
6
18
- PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(rw->match_data);
7
+// ----------------------------------------------------------------------------
8
20
- char *buffer = rewritten_value;
21
- size_t buffer_remaining = sizeof(rewritten_value);
9
+const char journal_key_characters_map[256] = {
10
+ // control characters
11
+ [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_',
12
+ [8] = '_', [9] = '_', [10] = '_', [11] = '_', [12] = '_', [13] = '_', [14] = '_', [15] = '_',
13
+ [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_', [22] = '_', [23] = '_',
14
+ [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_', [30] = '_', [31] = '_',
15
+
16
+ // symbols
17
+ [' '] = '_', ['!'] = '_', ['"'] = '_', ['#'] = '_', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_',
18
+ ['('] = '_', [')'] = '_', ['*'] = '_', ['+'] = '_', [','] = '_', ['-'] = '_', ['.'] = '_', ['/'] = '_',
19
+
20
+ // numbers
21
+ ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7',
22
+ ['8'] = '8', ['9'] = '9',
23
+
24
+ // symbols
25
+ [':'] = '_', [';'] = '_', ['<'] = '_', ['='] = '_', ['>'] = '_', ['?'] = '_', ['@'] = '_',
26
+
27
+ // capitals
28
+ ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H',
29
+ ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P',
30
+ ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X',
31
+ ['Y'] = 'Y', ['Z'] = 'Z',
32
+
33
+ // symbols
34
+ ['['] = '_', ['\\'] = '_', [']'] = '_', ['^'] = '_', ['_'] = '_', ['`'] = '_',
35
+
36
+ // lower to upper
37
+ ['a'] = 'A', ['b'] = 'B', ['c'] = 'C', ['d'] = 'D', ['e'] = 'E', ['f'] = 'F', ['g'] = 'G', ['h'] = 'H',
38
+ ['i'] = 'I', ['j'] = 'J', ['k'] = 'K', ['l'] = 'L', ['m'] = 'M', ['n'] = 'N', ['o'] = 'O', ['p'] = 'P',
39
+ ['q'] = 'Q', ['r'] = 'R', ['s'] = 'S', ['t'] = 'T', ['u'] = 'U', ['v'] = 'V', ['w'] = 'W', ['x'] = 'X',
40
+ ['y'] = 'Y', ['z'] = 'Z',
41
+
42
+ // symbols
43
+ ['{'] = '_', ['|'] = '_', ['}'] = '_', ['~'] = '_', [127] = '_', // Delete (DEL)
44
+
45
+ // Extended ASCII characters (128-255) set to underscore
46
+ [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_', [135] = '_',
47
+ [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_', [143] = '_',
48
+ [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_', [151] = '_',
49
+ [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_', [159] = '_',
50
+ [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_', [167] = '_',
51
+ [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_', [175] = '_',
52
+ [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_', [183] = '_',
53
+ [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_', [191] = '_',
54
+ [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_', [199] = '_',
55
+ [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_', [207] = '_',
56
+ [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_', [215] = '_',
57
+ [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_', [223] = '_',
58
+ [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_', [231] = '_',
59
+ [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_', [239] = '_',
60
+ [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_', [247] = '_',
61
+ [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_', [255] = '_',
62
+};
63
23
- // Iterate through the linked list of replacement nodes
24
- for (struct replacement_node *node = rw->nodes; node != NULL; node = node->next) {
25
- if (node->is_variable) {
26
- uint32_t groupnumber = pcre2_substring_number_from_name(rw->re, (PCRE2_SPTR)node->s);
27
- PCRE2_SIZE start_offset = ovector[2 * groupnumber];
28
- PCRE2_SIZE end_offset = ovector[2 * groupnumber + 1];
29
- PCRE2_SIZE length = end_offset - start_offset;
64
+// ----------------------------------------------------------------------------
65
31
- size_t copied = copy_to_buffer(buffer, buffer_remaining, value + start_offset, length);
32
- buffer += copied;
33
- buffer_remaining -= copied;
34
- }
35
- else {
36
- size_t len = node->len;
37
- size_t copied = copy_to_buffer(buffer, buffer_remaining, node->s, len);
38
- buffer += copied;
39
- buffer_remaining -= copied;
66
+static char *rewrite_value(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t value_len) {
67
+ static __thread char rewritten_value[JOURNAL_MAX_VALUE_LEN + 1];
68
+
69
+ if(!(k->flags & HK_REWRITES_CHECKED) || k->flags & HK_HAS_REWRITES) {
70
+ k->flags |= HK_REWRITES_CHECKED;
71
+
72
+ for(size_t i = 0; i < jb->rewrites.used; i++) {
73
+ REWRITE *rw = &jb->rewrites.array[i];
74
+
75
+ if(rw->key.hash == k->hash && strcmp(rw->key.key, k->key) == 0) {
76
+ if(!search_pattern_matches(&rw->search, value, value_len))
77
+ continue; // No match found, skip to next rewrite rule
78
+
79
+ PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(rw->search.match_data);
80
+
81
+ char *buffer = rewritten_value;
82
+ size_t buffer_remaining = sizeof(rewritten_value);
83
+
84
+ // Iterate through the linked list of replacement nodes
85
+ for(REPLACE_NODE *node = rw->replace.nodes; node != NULL; node = node->next) {
86
+ if(node->is_variable) {
87
+ int group_number = pcre2_substring_number_from_name(rw->search.re, (PCRE2_SPTR) node->name.key);
88
+ if(group_number >= 0) {
89
+ PCRE2_SIZE start_offset = ovector[2 * group_number];
90
+ PCRE2_SIZE end_offset = ovector[2 * group_number + 1];
91
+ PCRE2_SIZE length = end_offset - start_offset;
92
+
93
+ size_t copied = copy_to_buffer(buffer, buffer_remaining, value + start_offset, length);
94
+ buffer += copied;
95
+ buffer_remaining -= copied;
96
+ }
97
+ else {
98
+ // TODO: lookup in key names to get their values
99
+
100
+ if(!node->logged_error) {
101
+ log2stderr("WARNING: variable '${%s}' in rewrite rule cannot be resolved.",
102
+ node->name.key);
103
+ node->logged_error = true;
104
+ }
105
+ }
106
+ }
107
+ else {
108
+ size_t copied = copy_to_buffer(buffer, buffer_remaining, node->name.key, node->name.len);
109
+ buffer += copied;
110
+ buffer_remaining -= copied;
111
+ }
112
}
41
- }
113
43
- return rewritten_value;
114
+ k->flags |= HK_HAS_REWRITES;
115
+ return rewritten_value;
116
+ }
117
}
118
}
119
120
return NULL;
121
}
122
50
-static inline const char *rename_key(struct log_job *jb, const char *key, XXH64_hash_t hash, XXH64_hash_t *new_hash) {
51
- for(size_t i = 0; i < jb->renames.used ;i++) {
52
- struct key_rename *rn = &jb->renames.array[i];
123
+static inline HASHED_KEY *rename_key(LOG_JOB *jb, HASHED_KEY *k) {
124
+ if(!(k->flags & HK_RENAMES_CHECKED) || k->flags & HK_HAS_RENAMES) {
125
+ k->flags |= HK_RENAMES_CHECKED;
126
54
- if(rn->old_hash == hash && strcmp(rn->old_key, key) == 0) {
55
- *new_hash = rn->new_hash;
56
- return rn->new_key;
127
+ for(size_t i = 0; i < jb->renames.used; i++) {
128
+ RENAME *rn = &jb->renames.array[i];
129
+
130
+ if(rn->old_key.hash == k->hash && strcmp(rn->old_key.key, k->key) == 0) {
131
+ k->flags |= HK_HAS_RENAMES;
132
+ return &rn->new_key;
133
+ }
134
}
135
}
136
60
- *new_hash = hash;
61
- return key;
137
+ return k;
138
}
139
140
// ----------------------------------------------------------------------------
@@ -73,38 +149,84 @@ static inline void send_key_value_error(const char *key, const char *format, ...
149
printf("\n");
150
}
151
76
-inline void jb_send_key_value_and_rewrite(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t len) {
77
- char *rewritten = rewrite_value(jb, key, hash, value, len);
152
+static inline void send_key_value_and_rewrite(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t len) {
153
+ char *rewritten = rewrite_value(jb, k, value, len);
154
if(!rewritten)
79
- printf("%s=%.*s\n", key, (int)len, value);
155
+ printf("%s=%.*s\n", k->key, (int)len, value);
156
else
81
- printf("%s=%s\n", key, rewritten);
157
+ printf("%s=%s\n", k->key, rewritten);
158
+}
159
+
160
+static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, const char *key) {
161
+ size_t key_len = strlen(key);
162
+ XXH64_hash_t key_hash = XXH3_64bits(key, key_len);
163
+
164
+ HASHED_KEY *k;
165
+ SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&jb->hashtable, key_hash, true);
166
+ if(slot->data) {
167
+ k = slot->data;
168
+
169
+ if(!(k->flags & HK_COLLISION_CHECKED)) {
170
+ k->flags |= HK_COLLISION_CHECKED;
171
+
172
+ if(strcmp(k->key, key) != 0)
173
+ log2stderr("Hashtable collision detected on key '%s' (hash %lx) and '%s' (hash %lx). "
174
+ "Please report this to Netdata.",
175
+ k->key, (unsigned long)k->hash, key, (unsigned long)key_hash);
176
+ }
177
+ }
178
+ else {
179
+ k = mallocz(sizeof(HASHED_KEY));
180
+ k->key = strdupz(key);
181
+ k->len = key_len;
182
+ k->hash = key_hash;
183
+ k->flags = HK_ALLOCATED;
184
+
185
+ slot->hash = k->hash;
186
+ slot->data = k;
187
+ jb->hashtable.used++;
188
+ }
189
+
190
+ return k;
191
}
192
84
-inline void jb_send_extracted_key_value(struct log_job *jb, const char *key, const char *value, size_t len) {
85
- XXH64_hash_t hash = XXH3_64bits(key, strlen(key));
193
+inline void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len) {
194
+ HASHED_KEY *k = get_key_from_hashtable(jb, key);
195
+
196
+ if(!(k->flags & HK_FILTERED)) {
197
+ k->flags |= HK_FILTERED;
198
+
199
+ bool included = jb->filter.include.re ? search_pattern_matches(&jb->filter.include, k->key, k->len) : true;
200
+ bool excluded = jb->filter.exclude.re ? search_pattern_matches(&jb->filter.exclude, k->key, k->len) : false;
201
87
- // process renames (changing the key)
88
- XXH64_hash_t new_hash;
89
- const char *new_key = rename_key(jb, key, hash, &new_hash);
202
+ if(included && !excluded)
203
+ k->flags |= HK_FILTERED_INCLUDED;
204
+ else
205
+ k->flags &= ~HK_FILTERED_INCLUDED;
206
+ }
207
91
- // process rewrites (changing the value)
92
- // and send it to output
93
- jb_send_key_value_and_rewrite(jb, new_key, new_hash, value, len);
208
+ if(k->flags & HK_FILTERED_INCLUDED) {
209
+ // process renames (changing the key)
210
+ HASHED_KEY *nk = rename_key(jb, k);
211
+
212
+ // process rewrites (changing the value)
213
+ // and send it to output
214
+ send_key_value_and_rewrite(jb, nk, value, len);
215
+ }
216
217
// process the duplications (using the original key)
218
// and send them to output
97
- jb_send_duplications_for_key(jb, key, hash, value, len);
219
+ send_duplications_for_key(jb, k, value, len);
220
}
221
100
-static inline void send_key_value_constant(struct log_job *jb, const char *key, const char *value) {
222
+static inline void send_key_value_constant(LOG_JOB *jb __maybe_unused, const char *key, const char *value) {
223
printf("%s=%s\n", key, value);
224
}
225
226
// ----------------------------------------------------------------------------
227
// injection of constant fields
228
107
-static void jb_select_which_injections_should_be_injected_on_unmatched(struct log_job *jb) {
229
+static void select_which_injections_should_be_injected_on_unmatched(LOG_JOB *jb) {
230
// mark all injections to be added to unmatched logs
231
for(size_t i = 0; i < jb->injections.used ; i++)
232
jb->injections.keys[i].on_unmatched = true;
@@ -117,7 +239,7 @@ static void jb_select_which_injections_should_be_injected_on_unmatched(struct lo
239
240
for(size_t i = 0; i < jb->injections.used ;i++) {
241
for(size_t u = 0; u < jb->unmatched.injections.used ; u++) {
120
- if(strcmp(jb->injections.keys[i].key, jb->unmatched.injections.keys[u].key) == 0)
242
+ if(strcmp(jb->injections.keys[i].key.key, jb->unmatched.injections.keys[u].key.key) == 0)
243
jb->injections.keys[i].on_unmatched = false;
244
}
245
}
@@ -125,23 +247,23 @@ static void jb_select_which_injections_should_be_injected_on_unmatched(struct lo
247
}
248
249
128
-static inline void jb_finalize_injections(struct log_job *jb, bool line_is_matched) {
250
+static inline void jb_finalize_injections(LOG_JOB *jb, bool line_is_matched) {
251
for (size_t j = 0; j < jb->injections.used; j++) {
252
if(!line_is_matched && !jb->injections.keys[j].on_unmatched)
253
continue;
254
133
- send_key_value_constant(jb, jb->injections.keys[j].key, jb->injections.keys[j].value.s);
255
+ send_key_value_constant(jb, jb->injections.keys[j].key.key, jb->injections.keys[j].value.txt);
256
}
257
}
258
137
-static inline void jb_reset_injections(struct log_job *jb) {
259
+static inline void log_job_duplications_reset(LOG_JOB *jb) {
260
for(size_t d = 0; d < jb->dups.used ; d++) {
139
- struct key_dup *kd = &jb->dups.array[d];
261
+ DUPLICATION *kd = &jb->dups.array[d];
262
kd->exposed = false;
263
264
for(size_t g = 0; g < kd->used ; g++) {
143
- if(kd->values[g].s)
144
- kd->values[g].s[0] = '\0';
265
+ if(kd->values[g].txt)
266
+ kd->values[g].txt[0] = '\0';
267
}
268
}
269
}
@@ -149,42 +271,49 @@ static inline void jb_reset_injections(struct log_job *jb) {
271
// ----------------------------------------------------------------------------
272
// duplications
273
152
-inline void jb_send_duplications_for_key(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len) {
274
+static inline void send_duplications_for_key(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t value_len) {
275
// IMPORTANT:
276
// The 'value' may not be NULL terminated and have more data that the value we need
277
156
- for (size_t d = 0; d < jb->dups.used; d++) {
157
- struct key_dup *kd = &jb->dups.array[d];
278
+ if(!(k->flags & HK_DUPS_CHECKED) || k->flags & HK_HAS_DUPS) {
279
+ k->flags |= HK_DUPS_CHECKED;
280
159
- if(kd->exposed || kd->used == 0)
160
- continue;
281
+ for(size_t d = 0; d < jb->dups.used; d++) {
282
+ DUPLICATION *kd = &jb->dups.array[d];
283
+
284
+ if(kd->exposed || kd->used == 0)
285
+ continue;
286
162
- if(kd->used == 1) {
163
- // just one key to be duplicated
164
- if(strcmp(kd->keys[0], key) == 0) {
165
- jb_send_key_value_and_rewrite(jb, kd->target, kd->hash, value, value_len);
166
- kd->exposed = true;
287
+ if(kd->used == 1) {
288
+ // just one key to be duplicated
289
+ if(kd->keys[0].hash == k->hash && strcmp(kd->keys[0].key, k->key) == 0) {
290
+ k->flags |= HK_HAS_DUPS;
291
+ send_key_value_and_rewrite(jb, &kd->target, value, value_len);
292
+ kd->exposed = true;
293
+ }
294
}
168
- }
169
- else {
170
- // multiple keys to be duplicated
171
- for(size_t g = 0; g < kd->used ; g++) {
172
- if(strcmp(kd->keys[g], key) == 0)
173
- txt_replace(&kd->values[g], value, value_len);
295
+ else {
296
+ // multiple keys to be duplicated
297
+ for(size_t g = 0; g < kd->used; g++) {
298
+ if(kd->keys[g].hash == k->hash && strcmp(kd->keys[g].key, k->key) == 0) {
299
+ k->flags |= HK_HAS_DUPS;
300
+ txt_replace(&kd->values[g], value, value_len);
301
+ }
302
+ }
303
}
304
}
305
}
306
}
307
179
-static inline void jb_send_remaining_duplications(struct log_job *jb) {
180
- static __thread char buffer[MAX_VALUE_LEN + 1];
308
+static inline void jb_send_remaining_duplications(LOG_JOB *jb) {
309
+ static __thread char buffer[JOURNAL_MAX_VALUE_LEN + 1];
310
311
// IMPORTANT:
312
// all duplications are exposed, even the ones we haven't found their keys in the source,
313
// so that the output always has the same fields for matched entries.
314
315
for(size_t d = 0; d < jb->dups.used ; d++) {
187
- struct key_dup *kd = &jb->dups.array[d];
316
+ DUPLICATION *kd = &jb->dups.array[d];
317
318
if(kd->exposed || kd->used == 0)
319
continue;
@@ -195,7 +324,7 @@ static inline void jb_send_remaining_duplications(struct log_job *jb) {
324
325
for(size_t g = 0; g < kd->used ; g++) {
326
if(remaining < 2) {
198
- log2stderr("Warning: duplicated key '%s' cannot fit the values.", kd->target);
327
+ log2stderr("Warning: duplicated key '%s' cannot fit the values.", kd->target.key);
328
break;
329
}
330
@@ -205,30 +334,30 @@ static inline void jb_send_remaining_duplications(struct log_job *jb) {
334
remaining--;
335
}
336
208
- char *value = (kd->values[g].s && kd->values[g].s[0]) ? kd->values[g].s : "[unavailable]";
337
+ char *value = (kd->values[g].txt && kd->values[g].txt[0]) ? kd->values[g].txt : "[unavailable]";
338
size_t len = strlen(value);
339
size_t copied = copy_to_buffer(s, remaining, value, len);
340
remaining -= copied;
341
s += copied;
342
343
if(copied != len) {
215
- log2stderr("Warning: duplicated key '%s' will have truncated value", jb->dups.array[d].target);
344
+ log2stderr("Warning: duplicated key '%s' will have truncated value", jb->dups.array[d].target.key);
345
break;
346
}
347
}
219
- jb_send_key_value_and_rewrite(jb, kd->target, kd->hash, buffer, s - buffer);
348
+ send_key_value_and_rewrite(jb, &kd->target, buffer, s - buffer);
349
}
350
}
351
352
// ----------------------------------------------------------------------------
353
// filename injection
354
226
-static inline void jb_inject_filename(struct log_job *jb) {
355
+static inline void jb_inject_filename(LOG_JOB *jb) {
356
if (jb->filename.key && jb->filename.current[0])
357
send_key_value_constant(jb, jb->filename.key, jb->filename.current);
358
}
359
231
-static inline bool jb_switched_filename(struct log_job *jb, const char *line, size_t len) {
360
+static inline bool jb_switched_filename(LOG_JOB *jb, const char *line, size_t len) {
361
// IMPORTANT:
362
// Return TRUE when the caller should skip this line (because it is ours).
363
// Unfortunately, we have to consume empty lines too.
@@ -258,9 +387,23 @@ static inline bool jb_switched_filename(struct log_job *jb, const char *line, si
387
}
388
389
// ----------------------------------------------------------------------------
261
-// input reading
390
263
-static char *get_next_line(struct log_job *jb, char *buffer, size_t size, size_t *line_length) {
391
+static void simple_hashtable_cleanup_allocated(SIMPLE_HASHTABLE *ht) {
392
+ for(size_t i = 0; i < ht->used ;i++) {
393
+ HASHED_KEY *k = ht->hashtable[i].data;
394
+ if(k && k->flags & HK_ALLOCATED) {
395
+ hashed_key_cleanup(k);
396
+ freez(k);
397
+ ht->hashtable[i].data = NULL;
398
+ ht->hashtable[i].hash = 0;
399
+ }
400
+ }
401
+}
402
+
403
+// ----------------------------------------------------------------------------
404
+// running a job
405
+
406
+static char *get_next_line(LOG_JOB *jb __maybe_unused, char *buffer, size_t size, size_t *line_length) {
407
if(!fgets(buffer, (int)size, stdin)) {
408
*line_length = 0;
409
return NULL;
@@ -283,52 +426,15 @@ static char *get_next_line(struct log_job *jb, char *buffer, size_t size, size_t
426
return line;
427
}
428
286
-// ----------------------------------------------------------------------------
287
-
288
-static inline void jb_traverse_pcre2_named_groups_and_send_keys(struct log_job *jb, pcre2_code *re, pcre2_match_data *match_data, char *line) {
289
- PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
290
- uint32_t namecount;
291
- pcre2_pattern_info(re, PCRE2_INFO_NAMECOUNT, &namecount);
292
-
293
- if (namecount > 0) {
294
- PCRE2_SPTR name_table;
295
- pcre2_pattern_info(re, PCRE2_INFO_NAMETABLE, &name_table);
296
- uint32_t name_entry_size;
297
- pcre2_pattern_info(re, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size);
298
-
299
- const unsigned char *tabptr = name_table;
300
- for (uint32_t i = 0; i < namecount; i++) {
301
- int n = (tabptr[0] << 8) | tabptr[1];
302
- const char *group_name = (const char *)(tabptr + 2);
303
-
304
- PCRE2_SIZE start_offset = ovector[2 * n];
305
- PCRE2_SIZE end_offset = ovector[2 * n + 1];
306
- PCRE2_SIZE group_length = end_offset - start_offset;
307
-
308
- jb_send_extracted_key_value(jb, group_name, line + start_offset, group_length);
309
- tabptr += name_entry_size;
310
- }
311
- }
312
-}
313
-
314
-// ----------------------------------------------------------------------------
315
-
316
-struct log_job log_job = { 0 };
317
-int main(int argc, char *argv[]) {
318
- struct log_job *jb = &log_job;
319
-
320
- if(!parse_log2journal_parameters(jb, argc, argv))
321
- exit(1);
322
-
323
- if(jb->show_config)
324
- log_job_to_yaml(jb);
429
+int log_job_run(LOG_JOB *jb) {
430
+ select_which_injections_should_be_injected_on_unmatched(jb);
431
326
- jb_select_which_injections_should_be_injected_on_unmatched(jb);
432
+ simple_hashtable_init(&jb->hashtable, 32);
433
328
- pcre2_code *pcre2 = NULL;
329
- pcre2_match_data *match_data = NULL;
434
+ PCRE2_STATE *pcre2 = NULL;
435
LOG_JSON_STATE *json = NULL;
436
LOGFMT_STATE *logfmt = NULL;
437
+
438
if(strcmp(jb->pattern, "json") == 0) {
439
json = json_parser_create(jb);
440
}
@@ -336,13 +442,12 @@ int main(int argc, char *argv[]) {
442
logfmt = logfmt_parser_create(jb);
443
}
444
else {
339
- pcre2 = jb_compile_pcre2_pattern(jb->pattern);
340
- if(!pcre2)
341
- return 1;
342
-
343
- match_data = pcre2_match_data_create_from_pattern(pcre2, NULL);
344
- if(!match_data)
445
+ pcre2 = pcre2_parser_create(jb);
446
+ if(pcre2_has_error(pcre2)) {
447
+ log2stderr("%s", pcre2_parser_error(pcre2));
448
+ pcre2_parser_destroy(pcre2);
449
return 1;
450
+ }
451
}
452
453
char buffer[MAX_LINE_LENGTH];
@@ -353,7 +458,7 @@ int main(int argc, char *argv[]) {
458
if(jb_switched_filename(jb, line, len))
459
continue;
460
356
- jb_reset_injections(jb);
461
+ log_job_duplications_reset(jb);
462
463
bool line_is_matched;
464
@@ -362,21 +467,23 @@ int main(int argc, char *argv[]) {
467
else if(logfmt)
468
line_is_matched = logfmt_parse_document(logfmt, line);
469
else
365
- line_is_matched = jb_pcre2_match(pcre2, match_data, line, len, true);
470
+ line_is_matched = pcre2_parse_document(pcre2, line, len);
471
472
if(!line_is_matched) {
473
if(json)
474
log2stderr("%s", json_parser_error(json));
475
else if(logfmt)
476
log2stderr("%s", logfmt_parser_error(logfmt));
477
+ else
478
+ log2stderr("%s", pcre2_parser_error(pcre2));
479
480
if (jb->unmatched.key) {
481
// we are sending errors to systemd-journal
482
send_key_value_error(jb->unmatched.key, "Parsing error on: %s", line);
483
484
for (size_t j = 0; j < jb->unmatched.injections.used; j++)
378
- send_key_value_constant(jb, jb->unmatched.injections.keys[j].key,
379
- jb->unmatched.injections.keys[j].value.s);
485
+ send_key_value_constant(jb, jb->unmatched.injections.keys[j].key.key,
486
+ jb->unmatched.injections.keys[j].value.txt);
487
}
488
else {
489
// we are just logging errors to stderr
@@ -384,9 +491,6 @@ int main(int argc, char *argv[]) {
491
}
492
}
493
else {
387
- if(pcre2)
388
- jb_traverse_pcre2_named_groups_and_send_keys(jb, pcre2, match_data, line);
389
-
494
// print all non-exposed duplications
495
jb_send_remaining_duplications(jb);
496
}
@@ -404,11 +508,28 @@ int main(int argc, char *argv[]) {
508
else if(logfmt)
509
logfmt_parser_destroy(logfmt);
510
407
- else if(pcre2) {
408
- pcre2_match_data_free(match_data);
409
- pcre2_code_free(pcre2);
410
- }
511
+ else if(pcre2)
512
+ pcre2_parser_destroy(pcre2);
513
+
514
+ simple_hashtable_cleanup_allocated(&jb->hashtable);
515
+ simple_hashtable_free(&jb->hashtable);
516
412
- nd_log_destroy(jb);
517
return 0;
518
}
519
+
520
+// ----------------------------------------------------------------------------
521
+
522
+int main(int argc, char *argv[]) {
523
+ LOG_JOB log_job = { 0 };
524
+
525
+ if(!log_job_command_line_parse_parameters(&log_job, argc, argv))
526
+ exit(1);
527
+
528
+ if(log_job.show_config)
529
+ log_job_configuration_to_yaml(&log_job);
530
+
531
+ int ret = log_job_run(&log_job);
532
+
533
+ nd_log_cleanup(&log_job);
534
+ return ret;
535
+}
collectors/log2journal/log2journal.d/nginx-combined.yaml
+8
-4
@@ -13,10 +13,10 @@ pattern: |
13
(?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL
14
\]
15
\s+ "
16
- (?<MESSAGE>
17
- (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD
18
- (?<NGINX_URL>[^ ]+) \s+
19
- HTTP/(?<NGINX_HTTP_VERSION>[^"]+)
16
+ (?<NGINX_REQUEST>
17
+ (?<NGINX_REQUEST_METHOD>[A-Z]+) \s+ # NGINX_METHOD
18
+ (?<NGINX_REQUEST_URI>[^ ]+) \s+
19
+ (?<NGINX_SERVER_PROTOCOL>[^"]+)
20
)
21
" \s+
22
(?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS
@@ -30,6 +30,10 @@ pattern: |
30
filename:
31
key: NGINX_LOG_FILENAME
32
33
+rename:
34
+ - new_key: MESSAGE
35
+ old_key: NGINX_REQUEST
36
+
37
# Duplicate fields under a different name. You can duplicate multiple fields
38
# to a new one and then use rewrite rules to change its value.
39
duplicate:
collectors/log2journal/log2journal.d/nginx-json.yaml
new
+169
@@ -0,0 +1,169 @@
1
+# For all nginx variables, check this:
2
+# https://nginx.org/en/docs/http/ngx_http_core_module.html#var_connection_requests
3
+
4
+pattern: json
5
+
6
+prefix: NGINX_
7
+
8
+# When log2journal can detect the filename of each log entry (tail gives it
9
+# only when it tails multiple files), this key will be used to send the
10
+# filename to the journals.
11
+filename:
12
+ key: NGINX_LOG_FILENAME
13
+
14
+filter:
15
+ exclude: "^NGINX_BINARY_REMOTE_ADDR$"
16
+
17
+rename:
18
+ - new_key: MESSAGE
19
+ old_key: NGINX_REQUEST
20
+
21
+ # args is an alias for query_string
22
+ - new_key: NGINX_QUERY_STRING
23
+ old_key: NGINX_ARGS
24
+
25
+ # document_uri is an alias for uri
26
+ - new_key: NGINX_URI
27
+ old_key: NGINX_DOCUMENT_URI
28
+
29
+ # is_args states if the request had a query string or not
30
+ - new_key: NGINX_HAS_QUERY_STRING
31
+ old_key: NGINX_IS_ARGS
32
+
33
+ # msec is the timestamp in seconds, with fractional digits for milliseconds
34
+ - new_key: NGINX_TIMESTAMP_SEC
35
+ old_key: NGINX_MSEC
36
+
37
+ # nginx_version is already prefixed with nginx, let's remove one of them
38
+ - new_key: NGINX_VERSION
39
+ old_key: NGINX_NGINX_VERSION
40
+
41
+ # pipe states if the request was pipelined or not
42
+ - new_key: NGINX_PIPELINED
43
+ old_key: NGINX_PIPE
44
+
45
+ # rename numeric TLVs to their names
46
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_ALPN
47
+ old_key: NGINX_PROXY_PROTOCOL_TLV_0X01
48
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_AUTHORITY
49
+ old_key: NGINX_PROXY_PROTOCOL_TLV_0X02
50
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_UNIQUE_ID
51
+ old_key: NGINX_PROXY_PROTOCOL_TLV_0X05
52
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_SSL
53
+ old_key: NGINX_PROXY_PROTOCOL_TLV_0X20
54
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_NETNS
55
+ old_key: NGINX_PROXY_PROTOCOL_TLV_0X30
56
+
57
+ # rename numeric SSL TLVs to their names
58
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERSION
59
+ old_key: NGINX_PROXY_PROTOCOL_TLV_SSL_0X21
60
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_SSL_CN
61
+ old_key: NGINX_PROXY_PROTOCOL_TLV_SSL_0X22
62
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_SSL_CIPHER
63
+ old_key: NGINX_PROXY_PROTOCOL_TLV_SSL_0X23
64
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_SSL_SIG_ALG
65
+ old_key: NGINX_PROXY_PROTOCOL_TLV_SSL_0X24
66
+ - new_key: NGINX_PROXY_PROTOCOL_TLV_SSL_KEY_ALG
67
+ old_key: NGINX_PROXY_PROTOCOL_TLV_SSL_0X25
68
+
69
+# Duplicate fields under a different name. You can duplicate multiple fields
70
+# to a new one and then use rewrite rules to change its value.
71
+duplicate:
72
+
73
+ # we insert the field PRIORITY as a copy of NGINX_STATUS.
74
+ - key: PRIORITY
75
+ values_of:
76
+ - NGINX_STATUS
77
+
78
+ # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.
79
+ - key: NGINX_STATUS_FAMILY
80
+ values_of:
81
+ - NGINX_STATUS
82
+
83
+# Rewrite the value of fields (including the duplicated ones).
84
+# The search pattern can have named groups, and the replace pattern can use
85
+# them as ${name}.
86
+rewrite:
87
+ # a ? means it has query string, everything else means it does not
88
+ - key: NGINX_HAS_QUERY_STRING
89
+ search: '^\?$'
90
+ replace: "yes"
91
+ - key: NGINX_HAS_QUERY_STRING
92
+ search: ".*"
93
+ replace: "no"
94
+
95
+ # 'on' means it was HTTPS, everything else means it was not
96
+ - key: NGINX_HTTPS
97
+ search: "^on$"
98
+ replace: "yes"
99
+ - key: NGINX_HTTPS
100
+ search: ".*"
101
+ replace: "no"
102
+
103
+ # 'p' means it was pipelined, everything else means it was not
104
+ - key: NGINX_PIPELINED
105
+ search: "^p$"
106
+ replace: "yes"
107
+ - key: NGINX_PIPELINED
108
+ search: ".*"
109
+ replace: "no"
110
+
111
+ # zero means client sent a certificate and it was verified, non-zero means otherwise
112
+ - key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERIFY
113
+ search: "^0$"
114
+ replace: "yes"
115
+ - key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERIFY
116
+ search: ".*"
117
+ replace: "no"
118
+
119
+ # 'OK' means request completed, everything else means it didn't
120
+ - key: NGINX_REQUEST_COMPLETION
121
+ search: "^OK$"
122
+ replace: "completed"
123
+ - key: NGINX_REQUEST_COMPLETION
124
+ search: ".*"
125
+ replace: "not completed"
126
+
127
+ # PRIORTY is a duplicate of NGINX_STATUS
128
+ # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
129
+ - key: "PRIORITY"
130
+ search: "^[123]"
131
+ replace: 6
132
+
133
+ - key: "PRIORITY"
134
+ search: "^4"
135
+ replace: 5
136
+
137
+ - key: "PRIORITY"
138
+ search: "^5"
139
+ replace: 3
140
+
141
+ - key: "PRIORITY"
142
+ search: ".*"
143
+ replace: 4
144
+
145
+ # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
146
+ - key: "NGINX_STATUS_FAMILY"
147
+ search: "^(?<first_digit>[1-5])"
148
+ replace: "${first_digit}xx"
149
+
150
+ - key: "NGINX_STATUS_FAMILY"
151
+ search: ".*"
152
+ replace: "UNKNOWN"
153
+
154
+# Inject constant fields into the journal logs.
155
+inject:
156
+ - key: SYSLOG_IDENTIFIER
157
+ value: "nginx-log"
158
+
159
+# Control what to do when input logs do not match the main PCRE2 pattern.
160
+unmatched:
161
+ # The journal key to log the PCRE2 error message to.
162
+ # Set this to MESSAGE, so you to see the error in the log.
163
+ key: MESSAGE
164
+
165
+ # Inject static fields to the unmatched entries.
166
+ # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.
167
+ inject:
168
+ - key: PRIORITY
169
+ value: 1
collectors/log2journal/log2journal.h
+242
-133
@@ -15,31 +15,6 @@
15
#include <ctype.h>
16
#include <stdarg.h>
17
18
-#define XXH_INLINE_ALL
19
-#include "../../libnetdata/xxhash.h"
20
-
21
-#define PCRE2_CODE_UNIT_WIDTH 8
22
-#include <pcre2.h>
23
-
24
-#ifdef HAVE_LIBYAML
25
-#include <yaml.h>
26
-#endif
27
-
28
-
29
-#define MAX_OUTPUT_KEYS 1024
30
-#define OVECCOUNT (MAX_OUTPUT_KEYS * 3) // should be a multiple of 3
31
-#define MAX_LINE_LENGTH (1024 * 1024)
32
-#define MAX_KEY_DUPS (MAX_OUTPUT_KEYS / 2)
33
-#define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
34
-#define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
35
-#define MAX_RENAMES (MAX_OUTPUT_KEYS / 2)
36
-#define MAX_KEY_DUPS_KEYS 20
37
-
38
-#define MAX_KEY_LEN 64 // according to systemd-journald
39
-#define MAX_VALUE_LEN (48 * 1024) // according to systemd-journald
40
-
41
-#define LOG2JOURNAL_CONFIG_PATH LIBCONFIG_DIR "/log2journal.d"
42
-
18
// ----------------------------------------------------------------------------
19
// logging
20
@@ -65,6 +40,12 @@ static inline void *mallocz(size_t size) {
40
return ptr;
41
}
42
43
+static inline void *callocz(size_t elements, size_t size) {
44
+ void *ptr = mallocz(elements * size);
45
+ memset(ptr, 0, elements * size);
46
+ return ptr;
47
+}
48
+
49
static inline char *strdupz(const char *s) {
50
char *ptr = strdup(s);
51
if (!ptr) {
@@ -90,6 +71,38 @@ static inline void freez(void *ptr) {
71
72
// ----------------------------------------------------------------------------
73
74
+#define XXH_INLINE_ALL
75
+#include "../../libnetdata/xxhash.h"
76
+#include "../../libnetdata/simple_hashtable.h"
77
+
78
+#define PCRE2_CODE_UNIT_WIDTH 8
79
+#include <pcre2.h>
80
+
81
+#ifdef HAVE_LIBYAML
82
+#include <yaml.h>
83
+#endif
84
+
85
+#define MAX_OUTPUT_KEYS 1024
86
+#define MAX_LINE_LENGTH (1024 * 1024)
87
+#define MAX_KEY_DUPS (MAX_OUTPUT_KEYS / 2)
88
+#define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
89
+#define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
90
+#define MAX_RENAMES (MAX_OUTPUT_KEYS / 2)
91
+#define MAX_KEY_DUPS_KEYS 20
92
+
93
+#define JOURNAL_MAX_KEY_LEN 64 // according to systemd-journald
94
+#define JOURNAL_MAX_VALUE_LEN (48 * 1024) // according to systemd-journald
95
+
96
+#define LOG2JOURNAL_CONFIG_PATH LIBCONFIG_DIR "/log2journal.d"
97
+
98
+// ----------------------------------------------------------------------------
99
+// character conversion for journal keys
100
+
101
+extern const char journal_key_characters_map[256];
102
+
103
+// ----------------------------------------------------------------------------
104
+// copy to buffer, while ensuring there is no buffer overflow
105
+
106
static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) {
107
if(dst_size < 2) {
108
if(dst_size == 1)
@@ -111,198 +124,294 @@ static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src,
124
}
125
126
// ----------------------------------------------------------------------------
127
+// A dynamically sized, reusable text buffer,
128
+// allowing us to be fast (no allocations during iterations) while having the
129
+// smallest possible allocations.
130
131
typedef struct txt {
116
- char *s;
132
+ char *txt;
133
size_t size;
134
} TEXT;
135
120
-static inline void txt_replace(TEXT *txt, const char *s, size_t len) {
136
+static inline void txt_cleanup(TEXT *t) {
137
+ if(!t)
138
+ return;
139
+
140
+ if(t->txt)
141
+ freez(t->txt);
142
+
143
+ t->txt = NULL;
144
+ t->size = 0;
145
+}
146
+
147
+static inline void txt_replace(TEXT *t, const char *s, size_t len) {
148
if(!s || !*s || len == 0) {
149
s = "";
150
len = 0;
151
}
152
126
- if(len + 1 <= txt->size) {
153
+ if(len + 1 <= t->size) {
154
// the existing value allocation, fits our value
155
129
- memcpy(txt->s, s, len);
130
- txt->s[len] = '\0';
156
+ memcpy(t->txt, s, len);
157
+ t->txt[len] = '\0';
158
}
159
else {
160
// no existing value allocation, or too small for our value
161
+ // cleanup and increase the buffer
162
135
- if(txt->s)
136
- freez(txt->s);
163
+ txt_cleanup(t);
164
138
- txt->s = strndupz(s, len);
139
- txt->size = len + 1;
165
+ t->txt = strndupz(s, len);
166
+ t->size = len + 1;
167
}
168
}
169
170
// ----------------------------------------------------------------------------
171
145
-typedef struct key_value {
146
- char key[MAX_KEY_LEN + 1];
147
- TEXT value;
148
- bool on_unmatched;
149
-} KEY_VALUE;
172
+typedef enum __attribute__((__packed__)) {
173
+ HK_NONE = 0,
174
+ HK_ALLOCATED = (1 << 0),
175
+ HK_FILTERED = (1 << 1),
176
+ HK_FILTERED_INCLUDED = (1 << 2),
177
+ HK_COLLISION_CHECKED = (1 << 3),
178
+ HK_RENAMES_CHECKED = (1 << 4),
179
+ HK_HAS_RENAMES = (1 << 5),
180
+ HK_DUPS_CHECKED = (1 << 6),
181
+ HK_HAS_DUPS = (1 << 7),
182
+ HK_REWRITES_CHECKED = (1 << 8),
183
+ HK_HAS_REWRITES = (1 << 9),
184
+} HASHED_KEY_FLAGS;
185
+
186
+typedef struct hashed_key {
187
+ const char *key;
188
+ uint32_t len;
189
+ HASHED_KEY_FLAGS flags;
190
+ XXH64_hash_t hash;
191
+} HASHED_KEY;
192
+
193
+static inline void hashed_key_cleanup(HASHED_KEY *k) {
194
+ if(k->key) {
195
+ freez((void *)k->key);
196
+ k->key = NULL;
197
+ }
198
+}
199
+
200
+static inline void hashed_key_set(HASHED_KEY *k, const char *name) {
201
+ hashed_key_cleanup(k);
202
+
203
+ k->key = strdupz(name);
204
+ k->len = strlen(k->key);
205
+ k->hash = XXH3_64bits(k->key, k->len);
206
+ k->flags = HK_NONE;
207
+}
208
151
-static inline void key_value_replace(KEY_VALUE *kv, const char *key, size_t key_len, const char *value, size_t value_len) {
152
- copy_to_buffer(kv->key, sizeof(kv->key), key, key_len);
153
- txt_replace(&kv->value, value, value_len);
209
+static inline void hashed_key_len_set(HASHED_KEY *k, const char *name, size_t len) {
210
+ hashed_key_cleanup(k);
211
+
212
+ k->key = strndupz(name, len);
213
+ k->len = len;
214
+ k->hash = XXH3_64bits(k->key, k->len);
215
+ k->flags = HK_NONE;
216
}
217
218
// ----------------------------------------------------------------------------
219
158
-struct key_dup {
159
- XXH64_hash_t hash;
160
- char *target;
161
- char *keys[MAX_KEY_DUPS_KEYS];
162
- TEXT values[MAX_KEY_DUPS_KEYS];
163
- size_t used;
164
- bool exposed;
165
-};
220
+typedef struct search_pattern {
221
+ const char *pattern;
222
+ pcre2_code *re;
223
+ pcre2_match_data *match_data;
224
+ TEXT error;
225
+} SEARCH_PATTERN;
226
+
227
+void search_pattern_cleanup(SEARCH_PATTERN *sp);
228
+bool search_pattern_set(SEARCH_PATTERN *sp, const char *search_pattern, size_t search_pattern_len);
229
167
-struct key_rename {
168
- XXH64_hash_t new_hash;
169
- XXH64_hash_t old_hash;
170
- char *new_key;
171
- char *old_key;
172
-};
230
+static inline bool search_pattern_matches(SEARCH_PATTERN *sp, const char *value, size_t value_len) {
231
+ return pcre2_match(sp->re, (PCRE2_SPTR)value, value_len, 0, 0, sp->match_data, NULL) >= 0;
232
+}
233
+
234
+// ----------------------------------------------------------------------------
235
174
-struct replacement_node {
236
+typedef struct replacement_node {
237
+ HASHED_KEY name;
238
bool is_variable;
176
- const char *s;
177
- size_t len;
239
+ bool logged_error;
240
+
241
struct replacement_node *next;
179
-};
242
+} REPLACE_NODE;
243
181
-struct key_rewrite {
182
- XXH64_hash_t hash;
183
- char *key;
184
- char *search_pattern;
185
- char *replace_pattern;
186
- pcre2_code *re;
187
- pcre2_match_data *match_data;
188
- struct replacement_node *nodes;
189
-};
244
+void replace_node_free(REPLACE_NODE *rpn);
245
+
246
+typedef struct replace_pattern {
247
+ const char *pattern;
248
+ REPLACE_NODE *nodes;
249
+} REPLACE_PATTERN;
250
+
251
+void replace_pattern_cleanup(REPLACE_PATTERN *rp);
252
+bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern);
253
+
254
+// ----------------------------------------------------------------------------
255
+
256
+typedef struct injection {
257
+ bool on_unmatched;
258
+ TEXT value;
259
+ HASHED_KEY key;
260
+} INJECTION;
261
+
262
+void injection_cleanup(INJECTION *inj);
263
+
264
+// ----------------------------------------------------------------------------
265
+
266
+typedef struct duplication {
267
+ HASHED_KEY target;
268
+ uint32_t used;
269
+ bool exposed;
270
+ HASHED_KEY keys[MAX_KEY_DUPS_KEYS];
271
+ TEXT values[MAX_KEY_DUPS_KEYS];
272
+} DUPLICATION;
273
+
274
+void duplication_cleanup(DUPLICATION *dp);
275
+
276
+// ----------------------------------------------------------------------------
277
191
-struct log_job {
278
+typedef struct key_rename {
279
+ HASHED_KEY new_key;
280
+ HASHED_KEY old_key;
281
+} RENAME;
282
+
283
+void rename_cleanup(RENAME *rn);
284
+
285
+// ----------------------------------------------------------------------------
286
+
287
+typedef struct key_rewrite {
288
+ HASHED_KEY key;
289
+ SEARCH_PATTERN search;
290
+ REPLACE_PATTERN replace;
291
+} REWRITE;
292
+
293
+void rewrite_cleanup(REWRITE *rw);
294
+
295
+// ----------------------------------------------------------------------------
296
+// A job configuration and runtime structures
297
+
298
+typedef struct log_job {
299
bool show_config;
300
301
const char *pattern;
302
const char *prefix;
303
304
+ SIMPLE_HASHTABLE hashtable;
305
+
306
+ struct {
307
+ SEARCH_PATTERN include;
308
+ SEARCH_PATTERN exclude;
309
+ } filter;
310
+
311
struct {
312
+ bool last_line_was_empty;
313
const char *key;
314
char current[FILENAME_MAX + 1];
200
- bool last_line_was_empty;
315
} filename;
316
317
struct {
204
- KEY_VALUE keys[MAX_INJECTIONS];
205
- size_t used;
318
+ uint32_t used;
319
+ INJECTION keys[MAX_INJECTIONS];
320
} injections;
321
322
struct {
323
const char *key;
324
struct {
211
- KEY_VALUE keys[MAX_INJECTIONS];
212
- size_t used;
325
+ uint32_t used;
326
+ INJECTION keys[MAX_INJECTIONS];
327
} injections;
328
} unmatched;
329
330
struct {
217
- struct key_dup array[MAX_KEY_DUPS];
218
- size_t used;
331
+ uint32_t used;
332
+ DUPLICATION array[MAX_KEY_DUPS];
333
} dups;
334
335
struct {
222
- struct key_rewrite array[MAX_REWRITES];
223
- size_t used;
336
+ uint32_t used;
337
+ REWRITE array[MAX_REWRITES];
338
} rewrites;
339
340
struct {
227
- struct key_rename array[MAX_RENAMES];
228
- size_t used;
341
+ uint32_t used;
342
+ RENAME array[MAX_RENAMES];
343
} renames;
230
-};
344
+} LOG_JOB;
345
232
-void jb_send_key_value_and_rewrite(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t len);
233
-void jb_send_duplications_for_key(struct log_job *jb, const char *key, XXH64_hash_t hash, const char *value, size_t value_len);
234
-void jb_send_extracted_key_value(struct log_job *jb, const char *key, const char *value, size_t len);
346
+// free all resources consumed by the log job
347
+void nd_log_cleanup(LOG_JOB *jb);
348
236
-struct key_dup *log_job_add_duplication_to_job(struct log_job *jb, const char *target, size_t target_len);
237
-bool log_job_add_key_to_duplication(struct key_dup *kd, const char *key, size_t key_len);
238
-bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len);
239
-bool log_job_add_key_prefix(struct log_job *jb, const char *prefix, size_t prefix_len);
240
-bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched);
241
-bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern);
242
-bool log_job_add_rename(struct log_job *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len);
349
+// ----------------------------------------------------------------------------
350
244
-// entry point to parse command line parameters
245
-bool parse_log2journal_parameters(struct log_job *jb, int argc, char **argv);
351
+// the entry point to send key value pairs to the output
352
+// this implements the pipeline of processing renames, rewrites and duplications
353
+void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len);
354
247
-void log2journal_command_line_help(const char *name);
355
+// ----------------------------------------------------------------------------
356
+// configuration related
357
+
358
+// management of configuration to set settings
359
+DUPLICATION *log_job_duplication_add(LOG_JOB *jb, const char *target, size_t target_len);
360
+bool log_job_duplication_key_add(DUPLICATION *kd, const char *key, size_t key_len);
361
+bool log_job_filename_key_set(LOG_JOB *jb, const char *key, size_t key_len);
362
+bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len);
363
+bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
364
+bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched);
365
+bool log_job_rewrite_add(LOG_JOB *jb, const char *key, const char *search_pattern, const char *replace_pattern);
366
+bool log_job_rename_add(LOG_JOB *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len);
367
+bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
368
+bool log_job_exclude_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
369
249
-// free all resources consumed by the log job
250
-void nd_log_destroy(struct log_job *jb);
370
+// entry point to parse command line parameters
371
+bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv);
372
+void log_job_command_line_help(const char *name);
373
+
374
+// ----------------------------------------------------------------------------
375
+// YAML configuration related
376
377
#ifdef HAVE_LIBYAML
253
-bool yaml_parse_file(const char *config_file_path, struct log_job *jb);
254
-bool yaml_parse_config(const char *config_name, struct log_job *jb);
378
+bool yaml_parse_file(const char *config_file_path, LOG_JOB *jb);
379
+bool yaml_parse_config(const char *config_name, LOG_JOB *jb);
380
#endif
381
257
-void log_job_to_yaml(struct log_job *jb);
382
+void log_job_configuration_to_yaml(LOG_JOB *jb);
383
+
384
+// ----------------------------------------------------------------------------
385
+// JSON parser
386
387
typedef struct log_json_state LOG_JSON_STATE;
260
-LOG_JSON_STATE *json_parser_create(struct log_job *jb);
388
+LOG_JSON_STATE *json_parser_create(LOG_JOB *jb);
389
void json_parser_destroy(LOG_JSON_STATE *js);
390
const char *json_parser_error(LOG_JSON_STATE *js);
391
bool json_parse_document(LOG_JSON_STATE *js, const char *txt);
392
void json_test(void);
393
394
+// ----------------------------------------------------------------------------
395
+// logfmt parser
396
+
397
typedef struct logfmt_state LOGFMT_STATE;
267
-LOGFMT_STATE *logfmt_parser_create(struct log_job *jb);
398
+LOGFMT_STATE *logfmt_parser_create(LOG_JOB *jb);
399
void logfmt_parser_destroy(LOGFMT_STATE *lfs);
400
const char *logfmt_parser_error(LOGFMT_STATE *lfs);
401
bool logfmt_parse_document(LOGFMT_STATE *js, const char *txt);
402
void logfmt_test(void);
403
404
// ----------------------------------------------------------------------------
274
-// PCRE2 patters handling
275
-
276
-static inline pcre2_code *jb_compile_pcre2_pattern(const char *pattern) {
277
- int error_number;
278
- PCRE2_SIZE error_offset;
279
- PCRE2_SPTR pattern_ptr = (PCRE2_SPTR)pattern;
280
-
281
- pcre2_code *re = pcre2_compile(pattern_ptr, PCRE2_ZERO_TERMINATED, 0, &error_number, &error_offset, NULL);
282
- if (re == NULL) {
283
- PCRE2_UCHAR errbuf[1024];
284
- pcre2_get_error_message(error_number, errbuf, sizeof(errbuf));
285
- log2stderr("PCRE2 compilation failed at offset %d: %s", (int)error_offset, errbuf);
286
- log2stderr("Check for common regex syntax errors or unsupported PCRE2 patterns.");
287
- return NULL;
288
- }
289
-
290
- return re;
291
-}
405
+// pcre2 parser
406
293
-static inline bool jb_pcre2_match(pcre2_code *re, pcre2_match_data *match_data, char *line, size_t len, bool log) {
294
- int rc = pcre2_match(re, (PCRE2_SPTR)line, len, 0, 0, match_data, NULL);
295
- if(rc < 0) {
296
- PCRE2_UCHAR errbuf[1024];
297
- pcre2_get_error_message(rc, errbuf, sizeof(errbuf));
407
+typedef struct pcre2_state PCRE2_STATE;
408
+PCRE2_STATE *pcre2_parser_create(LOG_JOB *jb);
409
+void pcre2_parser_destroy(PCRE2_STATE *pcre2);
410
+const char *pcre2_parser_error(PCRE2_STATE *pcre2);
411
+bool pcre2_parse_document(PCRE2_STATE *pcre2, const char *txt, size_t len);
412
+bool pcre2_has_error(PCRE2_STATE *pcre2);
413
+void pcre2_test(void);
414
299
- if(log)
300
- log2stderr("PCRE2 error %d: %s on: %s", rc, errbuf, line);
301
-
302
- return false;
303
- }
304
-
305
- return true;
306
-}
415
+void pcre2_get_error_in_buffer(char *msg, size_t msg_len, int rc, int pos);
416
417
#endif //NETDATA_LOG2JOURNAL_H
collectors/log2journal/tests.d/full.output
new
+78
@@ -0,0 +1,78 @@
1
+pattern: |
2
+ (?x) # Enable PCRE2 extended mode
3
+ ^
4
+ (?<NGINX_REMOTE_ADDR>[^ ]+) \s - \s # NGINX_REMOTE_ADDR
5
+ (?<NGINX_REMOTE_USER>[^ ]+) \s # NGINX_REMOTE_USER
6
+ \[
7
+ (?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL
8
+ \]
9
+ \s+ "
10
+ (?<MESSAGE>
11
+ (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD
12
+ (?<NGINX_URL>[^ ]+) \s+
13
+ HTTP/(?<NGINX_HTTP_VERSION>[^"]+)
14
+ )
15
+ " \s+
16
+ (?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS
17
+ (?<NGINX_BODY_BYTES_SENT>\d+) \s+ # NGINX_BODY_BYTES_SENT
18
+ "(?<NGINX_HTTP_REFERER>[^"]*)" \s+ # NGINX_HTTP_REFERER
19
+ "(?<NGINX_HTTP_USER_AGENT>[^"]*)" # NGINX_HTTP_USER_AGENT
20
+
21
+prefix: NGINX_
22
+
23
+filename:
24
+ key: NGINX_LOG_FILENAME
25
+
26
+filter:
27
+ include: ".*"
28
+ exclude: ".*HELLO.*WORLD.*"
29
+
30
+rename:
31
+ - new_key: TEST1
32
+ old_key: TEST2
33
+ - new_key: TEST3
34
+ old_key: TEST4
35
+
36
+duplicate:
37
+ - key: PRIORITY
38
+ values_of:
39
+ - NGINX_STATUS
40
+ - key: NGINX_STATUS_FAMILY
41
+ values_of:
42
+ - NGINX_STATUS
43
+ - NGINX_METHOD
44
+
45
+rewrite:
46
+ - key: PRIORITY
47
+ search: "^[123]"
48
+ replace: 6
49
+ - key: PRIORITY
50
+ search: "^4"
51
+ replace: 5
52
+ - key: PRIORITY
53
+ search: "^5"
54
+ replace: 3
55
+ - key: PRIORITY
56
+ search: ".*"
57
+ replace: 4
58
+ - key: NGINX_STATUS_FAMILY
59
+ search: "^(?<first_digit>[1-5])"
60
+ replace: "${first_digit}xx"
61
+ - key: NGINX_STATUS_FAMILY
62
+ search: ".*"
63
+ replace: UNKNOWN
64
+
65
+inject:
66
+ - key: SYSLOG_IDENTIFIER
67
+ value: nginx-log
68
+ - key: SYSLOG_IDENTIFIER2
69
+ value: nginx-log2
70
+
71
+unmatched:
72
+ key: MESSAGE
73
+
74
+ inject:
75
+ - key: PRIORITY
76
+ value: 1
77
+ - key: PRIORITY2
78
+ value: 2
collectors/log2journal/tests.d/full.yaml
new
+77
@@ -0,0 +1,77 @@
1
+pattern: |
2
+ (?x) # Enable PCRE2 extended mode
3
+ ^
4
+ (?<NGINX_REMOTE_ADDR>[^ ]+) \s - \s # NGINX_REMOTE_ADDR
5
+ (?<NGINX_REMOTE_USER>[^ ]+) \s # NGINX_REMOTE_USER
6
+ \[
7
+ (?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL
8
+ \]
9
+ \s+ "
10
+ (?<MESSAGE>
11
+ (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD
12
+ (?<NGINX_URL>[^ ]+) \s+
13
+ HTTP/(?<NGINX_HTTP_VERSION>[^"]+)
14
+ )
15
+ " \s+
16
+ (?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS
17
+ (?<NGINX_BODY_BYTES_SENT>\d+) \s+ # NGINX_BODY_BYTES_SENT
18
+ "(?<NGINX_HTTP_REFERER>[^"]*)" \s+ # NGINX_HTTP_REFERER
19
+ "(?<NGINX_HTTP_USER_AGENT>[^"]*)" # NGINX_HTTP_USER_AGENT
20
+
21
+prefix: NGINX_
22
+
23
+filename:
24
+ key: NGINX_LOG_FILENAME
25
+
26
+filter:
27
+ include: ".*"
28
+ exclude: ".*HELLO.*WORLD.*"
29
+
30
+rename:
31
+ - new_key: TEST1
32
+ old_key: TEST2
33
+ - new_key: TEST3
34
+ old_key: TEST4
35
+
36
+duplicate:
37
+ - key: PRIORITY
38
+ values_of:
39
+ - NGINX_STATUS
40
+ - key: NGINX_STATUS_FAMILY
41
+ values_of:
42
+ - NGINX_STATUS
43
+ - NGINX_METHOD
44
+
45
+rewrite:
46
+ - key: "PRIORITY"
47
+ search: "^[123]"
48
+ replace: 6
49
+ - key: "PRIORITY"
50
+ search: "^4"
51
+ replace: 5
52
+ - key: "PRIORITY"
53
+ search: "^5"
54
+ replace: 3
55
+ - key: "PRIORITY"
56
+ search: ".*"
57
+ replace: 4
58
+ - key: "NGINX_STATUS_FAMILY"
59
+ search: "^(?<first_digit>[1-5])"
60
+ replace: "${first_digit}xx"
61
+ - key: "NGINX_STATUS_FAMILY"
62
+ search: ".*"
63
+ replace: "UNKNOWN"
64
+
65
+inject:
66
+ - key: SYSLOG_IDENTIFIER
67
+ value: "nginx-log"
68
+ - key: SYSLOG_IDENTIFIER2
69
+ value: "nginx-log2"
70
+
71
+unmatched:
72
+ key: MESSAGE
73
+ inject:
74
+ - key: PRIORITY
75
+ value: 1
76
+ - key: PRIORITY2
77
+ value: 2
collectors/log2journal/tests.d/json-exclude.output
new
+153
@@ -0,0 +1,153 @@
1
+NUMERICPOSITIVE=42
2
+NUMERICNEGATIVE=-123
3
+FLOATPOSITIVE=3.14159
4
+FLOATNEGATIVE=-2.71828
5
+SCIENTIFICINTPOSITIVE=1e5
6
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
7
+SCIENTIFICSMALLPOSITIVE=1e-4
8
+BOOLEANTRUE=true
9
+BOOLEANFALSE=false
10
+STRING=Hello, World!
11
+NULLVALUE=null
12
+OBJECT_NUMERICPOSITIVE=123
13
+OBJECT_NUMERICNEGATIVE=-456
14
+OBJECT_FLOATPOSITIVE=0.987
15
+OBJECT_FLOATNEGATIVE=-0.123
16
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
17
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
18
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
19
+OBJECT_BOOLEANTRUE=true
20
+OBJECT_BOOLEANFALSE=false
21
+OBJECT_STRING=Nested Object
22
+OBJECT_NULLVALUE=null
23
+ARRAY2_0=1
24
+ARRAY2_1=-2.345
25
+ARRAY2_2=Array Element
26
+ARRAY2_3=true
27
+ARRAY2_4=false
28
+ARRAY2_5=null
29
+ARRAY2_6_NUMERICPOSITIVE=123
30
+ARRAY2_6_NUMERICNEGATIVE=-456
31
+ARRAY2_6_FLOATPOSITIVE=0.987
32
+ARRAY2_6_FLOATNEGATIVE=-0.123
33
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
34
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
35
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
36
+ARRAY2_6_BOOLEANTRUE=true
37
+ARRAY2_6_BOOLEANFALSE=false
38
+ARRAY2_6_STRING=Nested Object in Array2
39
+ARRAY2_6_NULLVALUE=null
40
+ARRAY2_7_NUMERICPOSITIVE=42
41
+ARRAY2_7_NUMERICNEGATIVE=-123
42
+ARRAY2_7_FLOATPOSITIVE=3.14159
43
+ARRAY2_7_FLOATNEGATIVE=-2.71828
44
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
45
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
46
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
47
+ARRAY2_7_BOOLEANTRUE=true
48
+ARRAY2_7_BOOLEANFALSE=false
49
+ARRAY2_7_STRING=Array Element with Object in Array2
50
+ARRAY2_7_NULLVALUE=null
51
+
52
+NUMERICPOSITIVE=42
53
+NUMERICNEGATIVE=-123
54
+FLOATPOSITIVE=3.14159
55
+FLOATNEGATIVE=-2.71828
56
+SCIENTIFICINTPOSITIVE=1e5
57
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
58
+SCIENTIFICSMALLPOSITIVE=1e-4
59
+BOOLEANTRUE=true
60
+BOOLEANFALSE=false
61
+STRING=Hello, World!
62
+NULLVALUE=null
63
+OBJECT_NUMERICPOSITIVE=123
64
+OBJECT_NUMERICNEGATIVE=-456
65
+OBJECT_FLOATPOSITIVE=0.987
66
+OBJECT_FLOATNEGATIVE=-0.123
67
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
68
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
69
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
70
+OBJECT_BOOLEANTRUE=true
71
+OBJECT_BOOLEANFALSE=false
72
+OBJECT_STRING=Nested Object
73
+OBJECT_NULLVALUE=null
74
+ARRAY2_0=1
75
+ARRAY2_1=-2.345
76
+ARRAY2_2=Array Element
77
+ARRAY2_3=true
78
+ARRAY2_4=false
79
+ARRAY2_5=null
80
+ARRAY2_6_NUMERICPOSITIVE=123
81
+ARRAY2_6_NUMERICNEGATIVE=-456
82
+ARRAY2_6_FLOATPOSITIVE=0.987
83
+ARRAY2_6_FLOATNEGATIVE=-0.123
84
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
85
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
86
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
87
+ARRAY2_6_BOOLEANTRUE=true
88
+ARRAY2_6_BOOLEANFALSE=false
89
+ARRAY2_6_STRING=Nested Object in Array2
90
+ARRAY2_6_NULLVALUE=null
91
+ARRAY2_7_NUMERICPOSITIVE=42
92
+ARRAY2_7_NUMERICNEGATIVE=-123
93
+ARRAY2_7_FLOATPOSITIVE=3.14159
94
+ARRAY2_7_FLOATNEGATIVE=-2.71828
95
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
96
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
97
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
98
+ARRAY2_7_BOOLEANTRUE=true
99
+ARRAY2_7_BOOLEANFALSE=false
100
+ARRAY2_7_STRING=Array Element with Object in Array2
101
+ARRAY2_7_NULLVALUE=null
102
+
103
+NUMERICPOSITIVE=42
104
+NUMERICNEGATIVE=-123
105
+FLOATPOSITIVE=3.14159
106
+FLOATNEGATIVE=-2.71828
107
+SCIENTIFICINTPOSITIVE=1e5
108
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
109
+SCIENTIFICSMALLPOSITIVE=1e-4
110
+BOOLEANTRUE=true
111
+BOOLEANFALSE=false
112
+STRING=Hello, World!
113
+NULLVALUE=null
114
+OBJECT_NUMERICPOSITIVE=123
115
+OBJECT_NUMERICNEGATIVE=-456
116
+OBJECT_FLOATPOSITIVE=0.987
117
+OBJECT_FLOATNEGATIVE=-0.123
118
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
119
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
120
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
121
+OBJECT_BOOLEANTRUE=true
122
+OBJECT_BOOLEANFALSE=false
123
+OBJECT_STRING=Nested Object
124
+OBJECT_NULLVALUE=null
125
+ARRAY2_0=1
126
+ARRAY2_1=-2.345
127
+ARRAY2_2=Array Element
128
+ARRAY2_3=true
129
+ARRAY2_4=false
130
+ARRAY2_5=null
131
+ARRAY2_6_NUMERICPOSITIVE=123
132
+ARRAY2_6_NUMERICNEGATIVE=-456
133
+ARRAY2_6_FLOATPOSITIVE=0.987
134
+ARRAY2_6_FLOATNEGATIVE=-0.123
135
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
136
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
137
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
138
+ARRAY2_6_BOOLEANTRUE=true
139
+ARRAY2_6_BOOLEANFALSE=false
140
+ARRAY2_6_STRING=Nested Object in Array2
141
+ARRAY2_6_NULLVALUE=null
142
+ARRAY2_7_NUMERICPOSITIVE=42
143
+ARRAY2_7_NUMERICNEGATIVE=-123
144
+ARRAY2_7_FLOATPOSITIVE=3.14159
145
+ARRAY2_7_FLOATNEGATIVE=-2.71828
146
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
147
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
148
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
149
+ARRAY2_7_BOOLEANTRUE=true
150
+ARRAY2_7_BOOLEANFALSE=false
151
+ARRAY2_7_STRING=Array Element with Object in Array2
152
+ARRAY2_7_NULLVALUE=null
153
+
collectors/log2journal/tests.d/json-include.output
new
+54
@@ -0,0 +1,54 @@
1
+OBJECT_NUMERICPOSITIVE=123
2
+OBJECT_NUMERICNEGATIVE=-456
3
+OBJECT_FLOATPOSITIVE=0.987
4
+OBJECT_FLOATNEGATIVE=-0.123
5
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
6
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
7
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
8
+OBJECT_BOOLEANTRUE=true
9
+OBJECT_BOOLEANFALSE=false
10
+OBJECT_STRING=Nested Object
11
+OBJECT_NULLVALUE=null
12
+OBJECT_ARRAY_0=1
13
+OBJECT_ARRAY_1=-2
14
+OBJECT_ARRAY_2=3
15
+OBJECT_ARRAY_3=Nested Array
16
+OBJECT_ARRAY_4=true
17
+OBJECT_ARRAY_5=null
18
+
19
+OBJECT_NUMERICPOSITIVE=123
20
+OBJECT_NUMERICNEGATIVE=-456
21
+OBJECT_FLOATPOSITIVE=0.987
22
+OBJECT_FLOATNEGATIVE=-0.123
23
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
24
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
25
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
26
+OBJECT_BOOLEANTRUE=true
27
+OBJECT_BOOLEANFALSE=false
28
+OBJECT_STRING=Nested Object
29
+OBJECT_NULLVALUE=null
30
+OBJECT_ARRAY_0=1
31
+OBJECT_ARRAY_1=-2
32
+OBJECT_ARRAY_2=3
33
+OBJECT_ARRAY_3=Nested Array
34
+OBJECT_ARRAY_4=true
35
+OBJECT_ARRAY_5=null
36
+
37
+OBJECT_NUMERICPOSITIVE=123
38
+OBJECT_NUMERICNEGATIVE=-456
39
+OBJECT_FLOATPOSITIVE=0.987
40
+OBJECT_FLOATNEGATIVE=-0.123
41
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
42
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
43
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
44
+OBJECT_BOOLEANTRUE=true
45
+OBJECT_BOOLEANFALSE=false
46
+OBJECT_STRING=Nested Object
47
+OBJECT_NULLVALUE=null
48
+OBJECT_ARRAY_0=1
49
+OBJECT_ARRAY_1=-2
50
+OBJECT_ARRAY_2=3
51
+OBJECT_ARRAY_3=Nested Array
52
+OBJECT_ARRAY_4=true
53
+OBJECT_ARRAY_5=null
54
+
collectors/log2journal/tests.d/json.log
new
+3
@@ -0,0 +1,3 @@
1
+{ "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Hello, World!", "nullValue": null, "object": { "numericPositive": 123, "numericNegative": -456, "floatPositive": 0.987, "floatNegative": -0.123, "scientificIntPositive": 6e4, "scientificFloatNegative": -1.5e-2, "scientificSmallPositive": 5e-5, "booleanTrue": true, "booleanFalse": false, "string": "Nested Object", "nullValue": null, "array": [1, -2, 3, "Nested Array", true, null] }, "array": [ 1, -2.345, "Array Element", true, false, null, { "numericPositive": 987, "numericNegative": -654, "string": "Nested Object in Array", "array": [null, false, true] }, { "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Array Element with Object", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object", true, null] } ], "array2": [ 1, -2.345, "Array Element", true, false, null, { "numericPositive": 123, "numericNegative": -456, "floatPositive": 0.987, "floatNegative": -0.123, "scientificIntPositive": 6e4, "scientificFloatNegative": -1.5e-2, "scientificSmallPositive": 5e-5, "booleanTrue": true, "booleanFalse": false, "string": "Nested Object in Array2", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object2", true, null] }, { "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Array Element with Object in Array2", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object2", true, null]}]}
2
+{ "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Hello, World!", "nullValue": null, "object": { "numericPositive": 123, "numericNegative": -456, "floatPositive": 0.987, "floatNegative": -0.123, "scientificIntPositive": 6e4, "scientificFloatNegative": -1.5e-2, "scientificSmallPositive": 5e-5, "booleanTrue": true, "booleanFalse": false, "string": "Nested Object", "nullValue": null, "array": [1, -2, 3, "Nested Array", true, null] }, "array": [ 1, -2.345, "Array Element", true, false, null, { "numericPositive": 987, "numericNegative": -654, "string": "Nested Object in Array", "array": [null, false, true] }, { "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Array Element with Object", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object", true, null] } ], "array2": [ 1, -2.345, "Array Element", true, false, null, { "numericPositive": 123, "numericNegative": -456, "floatPositive": 0.987, "floatNegative": -0.123, "scientificIntPositive": 6e4, "scientificFloatNegative": -1.5e-2, "scientificSmallPositive": 5e-5, "booleanTrue": true, "booleanFalse": false, "string": "Nested Object in Array2", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object2", true, null] }, { "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Array Element with Object in Array2", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object2", true, null]}]}
3
+{ "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Hello, World!", "nullValue": null, "object": { "numericPositive": 123, "numericNegative": -456, "floatPositive": 0.987, "floatNegative": -0.123, "scientificIntPositive": 6e4, "scientificFloatNegative": -1.5e-2, "scientificSmallPositive": 5e-5, "booleanTrue": true, "booleanFalse": false, "string": "Nested Object", "nullValue": null, "array": [1, -2, 3, "Nested Array", true, null] }, "array": [ 1, -2.345, "Array Element", true, false, null, { "numericPositive": 987, "numericNegative": -654, "string": "Nested Object in Array", "array": [null, false, true] }, { "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Array Element with Object", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object", true, null] } ], "array2": [ 1, -2.345, "Array Element", true, false, null, { "numericPositive": 123, "numericNegative": -456, "floatPositive": 0.987, "floatNegative": -0.123, "scientificIntPositive": 6e4, "scientificFloatNegative": -1.5e-2, "scientificSmallPositive": 5e-5, "booleanTrue": true, "booleanFalse": false, "string": "Nested Object in Array2", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object2", true, null] }, { "numericPositive": 42, "numericNegative": -123, "floatPositive": 3.14159, "floatNegative": -2.71828, "scientificIntPositive": 1e5, "scientificFloatNegative": -2.5e-3, "scientificSmallPositive": 1e-4, "booleanTrue": true, "booleanFalse": false, "string": "Array Element with Object in Array2", "nullValue": null, "array": [1, -2, 3, "Nested Array in Object2", true, null]}]}
collectors/log2journal/tests.d/json.output
new
+294
@@ -0,0 +1,294 @@
1
+NUMERICPOSITIVE=42
2
+NUMERICNEGATIVE=-123
3
+FLOATPOSITIVE=3.14159
4
+FLOATNEGATIVE=-2.71828
5
+SCIENTIFICINTPOSITIVE=1e5
6
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
7
+SCIENTIFICSMALLPOSITIVE=1e-4
8
+BOOLEANTRUE=true
9
+BOOLEANFALSE=false
10
+STRING=Hello, World!
11
+NULLVALUE=null
12
+OBJECT_NUMERICPOSITIVE=123
13
+OBJECT_NUMERICNEGATIVE=-456
14
+OBJECT_FLOATPOSITIVE=0.987
15
+OBJECT_FLOATNEGATIVE=-0.123
16
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
17
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
18
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
19
+OBJECT_BOOLEANTRUE=true
20
+OBJECT_BOOLEANFALSE=false
21
+OBJECT_STRING=Nested Object
22
+OBJECT_NULLVALUE=null
23
+OBJECT_ARRAY_0=1
24
+OBJECT_ARRAY_1=-2
25
+OBJECT_ARRAY_2=3
26
+OBJECT_ARRAY_3=Nested Array
27
+OBJECT_ARRAY_4=true
28
+OBJECT_ARRAY_5=null
29
+ARRAY_0=1
30
+ARRAY_1=-2.345
31
+ARRAY_2=Array Element
32
+ARRAY_3=true
33
+ARRAY_4=false
34
+ARRAY_5=null
35
+ARRAY_6_NUMERICPOSITIVE=987
36
+ARRAY_6_NUMERICNEGATIVE=-654
37
+ARRAY_6_STRING=Nested Object in Array
38
+ARRAY_6_ARRAY_0=null
39
+ARRAY_6_ARRAY_1=false
40
+ARRAY_6_ARRAY_2=true
41
+ARRAY_7_NUMERICPOSITIVE=42
42
+ARRAY_7_NUMERICNEGATIVE=-123
43
+ARRAY_7_FLOATPOSITIVE=3.14159
44
+ARRAY_7_FLOATNEGATIVE=-2.71828
45
+ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
46
+ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
47
+ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
48
+ARRAY_7_BOOLEANTRUE=true
49
+ARRAY_7_BOOLEANFALSE=false
50
+ARRAY_7_STRING=Array Element with Object
51
+ARRAY_7_NULLVALUE=null
52
+ARRAY_7_ARRAY_0=1
53
+ARRAY_7_ARRAY_1=-2
54
+ARRAY_7_ARRAY_2=3
55
+ARRAY_7_ARRAY_3=Nested Array in Object
56
+ARRAY_7_ARRAY_4=true
57
+ARRAY_7_ARRAY_5=null
58
+ARRAY2_0=1
59
+ARRAY2_1=-2.345
60
+ARRAY2_2=Array Element
61
+ARRAY2_3=true
62
+ARRAY2_4=false
63
+ARRAY2_5=null
64
+ARRAY2_6_NUMERICPOSITIVE=123
65
+ARRAY2_6_NUMERICNEGATIVE=-456
66
+ARRAY2_6_FLOATPOSITIVE=0.987
67
+ARRAY2_6_FLOATNEGATIVE=-0.123
68
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
69
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
70
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
71
+ARRAY2_6_BOOLEANTRUE=true
72
+ARRAY2_6_BOOLEANFALSE=false
73
+ARRAY2_6_STRING=Nested Object in Array2
74
+ARRAY2_6_NULLVALUE=null
75
+ARRAY2_6_ARRAY_0=1
76
+ARRAY2_6_ARRAY_1=-2
77
+ARRAY2_6_ARRAY_2=3
78
+ARRAY2_6_ARRAY_3=Nested Array in Object2
79
+ARRAY2_6_ARRAY_4=true
80
+ARRAY2_6_ARRAY_5=null
81
+ARRAY2_7_NUMERICPOSITIVE=42
82
+ARRAY2_7_NUMERICNEGATIVE=-123
83
+ARRAY2_7_FLOATPOSITIVE=3.14159
84
+ARRAY2_7_FLOATNEGATIVE=-2.71828
85
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
86
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
87
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
88
+ARRAY2_7_BOOLEANTRUE=true
89
+ARRAY2_7_BOOLEANFALSE=false
90
+ARRAY2_7_STRING=Array Element with Object in Array2
91
+ARRAY2_7_NULLVALUE=null
92
+ARRAY2_7_ARRAY_0=1
93
+ARRAY2_7_ARRAY_1=-2
94
+ARRAY2_7_ARRAY_2=3
95
+ARRAY2_7_ARRAY_3=Nested Array in Object2
96
+ARRAY2_7_ARRAY_4=true
97
+ARRAY2_7_ARRAY_5=null
98
+
99
+NUMERICPOSITIVE=42
100
+NUMERICNEGATIVE=-123
101
+FLOATPOSITIVE=3.14159
102
+FLOATNEGATIVE=-2.71828
103
+SCIENTIFICINTPOSITIVE=1e5
104
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
105
+SCIENTIFICSMALLPOSITIVE=1e-4
106
+BOOLEANTRUE=true
107
+BOOLEANFALSE=false
108
+STRING=Hello, World!
109
+NULLVALUE=null
110
+OBJECT_NUMERICPOSITIVE=123
111
+OBJECT_NUMERICNEGATIVE=-456
112
+OBJECT_FLOATPOSITIVE=0.987
113
+OBJECT_FLOATNEGATIVE=-0.123
114
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
115
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
116
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
117
+OBJECT_BOOLEANTRUE=true
118
+OBJECT_BOOLEANFALSE=false
119
+OBJECT_STRING=Nested Object
120
+OBJECT_NULLVALUE=null
121
+OBJECT_ARRAY_0=1
122
+OBJECT_ARRAY_1=-2
123
+OBJECT_ARRAY_2=3
124
+OBJECT_ARRAY_3=Nested Array
125
+OBJECT_ARRAY_4=true
126
+OBJECT_ARRAY_5=null
127
+ARRAY_0=1
128
+ARRAY_1=-2.345
129
+ARRAY_2=Array Element
130
+ARRAY_3=true
131
+ARRAY_4=false
132
+ARRAY_5=null
133
+ARRAY_6_NUMERICPOSITIVE=987
134
+ARRAY_6_NUMERICNEGATIVE=-654
135
+ARRAY_6_STRING=Nested Object in Array
136
+ARRAY_6_ARRAY_0=null
137
+ARRAY_6_ARRAY_1=false
138
+ARRAY_6_ARRAY_2=true
139
+ARRAY_7_NUMERICPOSITIVE=42
140
+ARRAY_7_NUMERICNEGATIVE=-123
141
+ARRAY_7_FLOATPOSITIVE=3.14159
142
+ARRAY_7_FLOATNEGATIVE=-2.71828
143
+ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
144
+ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
145
+ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
146
+ARRAY_7_BOOLEANTRUE=true
147
+ARRAY_7_BOOLEANFALSE=false
148
+ARRAY_7_STRING=Array Element with Object
149
+ARRAY_7_NULLVALUE=null
150
+ARRAY_7_ARRAY_0=1
151
+ARRAY_7_ARRAY_1=-2
152
+ARRAY_7_ARRAY_2=3
153
+ARRAY_7_ARRAY_3=Nested Array in Object
154
+ARRAY_7_ARRAY_4=true
155
+ARRAY_7_ARRAY_5=null
156
+ARRAY2_0=1
157
+ARRAY2_1=-2.345
158
+ARRAY2_2=Array Element
159
+ARRAY2_3=true
160
+ARRAY2_4=false
161
+ARRAY2_5=null
162
+ARRAY2_6_NUMERICPOSITIVE=123
163
+ARRAY2_6_NUMERICNEGATIVE=-456
164
+ARRAY2_6_FLOATPOSITIVE=0.987
165
+ARRAY2_6_FLOATNEGATIVE=-0.123
166
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
167
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
168
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
169
+ARRAY2_6_BOOLEANTRUE=true
170
+ARRAY2_6_BOOLEANFALSE=false
171
+ARRAY2_6_STRING=Nested Object in Array2
172
+ARRAY2_6_NULLVALUE=null
173
+ARRAY2_6_ARRAY_0=1
174
+ARRAY2_6_ARRAY_1=-2
175
+ARRAY2_6_ARRAY_2=3
176
+ARRAY2_6_ARRAY_3=Nested Array in Object2
177
+ARRAY2_6_ARRAY_4=true
178
+ARRAY2_6_ARRAY_5=null
179
+ARRAY2_7_NUMERICPOSITIVE=42
180
+ARRAY2_7_NUMERICNEGATIVE=-123
181
+ARRAY2_7_FLOATPOSITIVE=3.14159
182
+ARRAY2_7_FLOATNEGATIVE=-2.71828
183
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
184
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
185
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
186
+ARRAY2_7_BOOLEANTRUE=true
187
+ARRAY2_7_BOOLEANFALSE=false
188
+ARRAY2_7_STRING=Array Element with Object in Array2
189
+ARRAY2_7_NULLVALUE=null
190
+ARRAY2_7_ARRAY_0=1
191
+ARRAY2_7_ARRAY_1=-2
192
+ARRAY2_7_ARRAY_2=3
193
+ARRAY2_7_ARRAY_3=Nested Array in Object2
194
+ARRAY2_7_ARRAY_4=true
195
+ARRAY2_7_ARRAY_5=null
196
+
197
+NUMERICPOSITIVE=42
198
+NUMERICNEGATIVE=-123
199
+FLOATPOSITIVE=3.14159
200
+FLOATNEGATIVE=-2.71828
201
+SCIENTIFICINTPOSITIVE=1e5
202
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
203
+SCIENTIFICSMALLPOSITIVE=1e-4
204
+BOOLEANTRUE=true
205
+BOOLEANFALSE=false
206
+STRING=Hello, World!
207
+NULLVALUE=null
208
+OBJECT_NUMERICPOSITIVE=123
209
+OBJECT_NUMERICNEGATIVE=-456
210
+OBJECT_FLOATPOSITIVE=0.987
211
+OBJECT_FLOATNEGATIVE=-0.123
212
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
213
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
214
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
215
+OBJECT_BOOLEANTRUE=true
216
+OBJECT_BOOLEANFALSE=false
217
+OBJECT_STRING=Nested Object
218
+OBJECT_NULLVALUE=null
219
+OBJECT_ARRAY_0=1
220
+OBJECT_ARRAY_1=-2
221
+OBJECT_ARRAY_2=3
222
+OBJECT_ARRAY_3=Nested Array
223
+OBJECT_ARRAY_4=true
224
+OBJECT_ARRAY_5=null
225
+ARRAY_0=1
226
+ARRAY_1=-2.345
227
+ARRAY_2=Array Element
228
+ARRAY_3=true
229
+ARRAY_4=false
230
+ARRAY_5=null
231
+ARRAY_6_NUMERICPOSITIVE=987
232
+ARRAY_6_NUMERICNEGATIVE=-654
233
+ARRAY_6_STRING=Nested Object in Array
234
+ARRAY_6_ARRAY_0=null
235
+ARRAY_6_ARRAY_1=false
236
+ARRAY_6_ARRAY_2=true
237
+ARRAY_7_NUMERICPOSITIVE=42
238
+ARRAY_7_NUMERICNEGATIVE=-123
239
+ARRAY_7_FLOATPOSITIVE=3.14159
240
+ARRAY_7_FLOATNEGATIVE=-2.71828
241
+ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
242
+ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
243
+ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
244
+ARRAY_7_BOOLEANTRUE=true
245
+ARRAY_7_BOOLEANFALSE=false
246
+ARRAY_7_STRING=Array Element with Object
247
+ARRAY_7_NULLVALUE=null
248
+ARRAY_7_ARRAY_0=1
249
+ARRAY_7_ARRAY_1=-2
250
+ARRAY_7_ARRAY_2=3
251
+ARRAY_7_ARRAY_3=Nested Array in Object
252
+ARRAY_7_ARRAY_4=true
253
+ARRAY_7_ARRAY_5=null
254
+ARRAY2_0=1
255
+ARRAY2_1=-2.345
256
+ARRAY2_2=Array Element
257
+ARRAY2_3=true
258
+ARRAY2_4=false
259
+ARRAY2_5=null
260
+ARRAY2_6_NUMERICPOSITIVE=123
261
+ARRAY2_6_NUMERICNEGATIVE=-456
262
+ARRAY2_6_FLOATPOSITIVE=0.987
263
+ARRAY2_6_FLOATNEGATIVE=-0.123
264
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
265
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
266
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
267
+ARRAY2_6_BOOLEANTRUE=true
268
+ARRAY2_6_BOOLEANFALSE=false
269
+ARRAY2_6_STRING=Nested Object in Array2
270
+ARRAY2_6_NULLVALUE=null
271
+ARRAY2_6_ARRAY_0=1
272
+ARRAY2_6_ARRAY_1=-2
273
+ARRAY2_6_ARRAY_2=3
274
+ARRAY2_6_ARRAY_3=Nested Array in Object2
275
+ARRAY2_6_ARRAY_4=true
276
+ARRAY2_6_ARRAY_5=null
277
+ARRAY2_7_NUMERICPOSITIVE=42
278
+ARRAY2_7_NUMERICNEGATIVE=-123
279
+ARRAY2_7_FLOATPOSITIVE=3.14159
280
+ARRAY2_7_FLOATNEGATIVE=-2.71828
281
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
282
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
283
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
284
+ARRAY2_7_BOOLEANTRUE=true
285
+ARRAY2_7_BOOLEANFALSE=false
286
+ARRAY2_7_STRING=Array Element with Object in Array2
287
+ARRAY2_7_NULLVALUE=null
288
+ARRAY2_7_ARRAY_0=1
289
+ARRAY2_7_ARRAY_1=-2
290
+ARRAY2_7_ARRAY_2=3
291
+ARRAY2_7_ARRAY_3=Nested Array in Object2
292
+ARRAY2_7_ARRAY_4=true
293
+ARRAY2_7_ARRAY_5=null
294
+
collectors/log2journal/tests.d/nginx-json.log
new
+9
@@ -0,0 +1,9 @@
1
+{"msec":"1644997905.123","connection":12345,"connection_requests":5,"pid":9876,"request_id":"8f3ebc1e38fbb92f","request_length":345,"remote_addr":"192.168.1.100","remote_user":"john_doe","remote_port":54321,"time_local":"19/Feb/2023:14:15:05 +0000","request":"GET /index.html HTTP/1.1","request_uri":"/index.html?param=value","args":"param=value","status":200,"body_bytes_sent":5432,"bytes_sent":6543,"http_referer":"https://example.com","http_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)","http_x_forwarded_for":"192.168.1.50, 10.0.0.1","host":"example.com","request_time":0.123,"upstream":"10.0.0.2:8080","upstream_connect_time":0.045,"upstream_header_time":0.020,"upstream_response_time":0.058,"upstream_response_length":7890,"upstream_cache_status":"MISS","ssl_protocol":"TLSv1.2","ssl_cipher":"AES256-SHA256","scheme":"https","request_method":"GET","server_protocol":"HTTP/1.1","pipe":".","gzip_ratio":"2.1","http_cf_ray":"abc123def456","geoip_country_code":"US"}
2
+{"msec":"1644997910.789","connection":54321,"connection_requests":10,"pid":5432,"request_id":"4a7bca5e19d3f8e7","request_length":432,"remote_addr":"10.0.0.3","remote_user":"","remote_port":12345,"time_local":"19/Feb/2023:14:15:10 +0000","request":"POST /api/update HTTP/1.1","request_uri":"/api/update","args":"","status":204,"body_bytes_sent":0,"bytes_sent":123,"http_referer":"","http_user_agent":"curl/7.68.0","http_x_forwarded_for":"","host":"api.example.com","request_time":0.032,"upstream":"backend-server-1:8080","upstream_connect_time":0.012,"upstream_header_time":0.020,"upstream_response_time":0.010,"upstream_response_length":0,"upstream_cache_status":"","ssl_protocol":"","ssl_cipher":"","scheme":"http","request_method":"POST","server_protocol":"HTTP/1.1","pipe":"p","gzip_ratio":"","http_cf_ray":"","geoip_country_code":""}
3
+{"msec":"1644997920.456","connection":98765,"connection_requests":15,"pid":1234,"request_id":"63f8ad2c3e1b4090","request_length":567,"remote_addr":"2001:0db8:85a3:0000:0000:8a2e:0370:7334","remote_user":"alice","remote_port":6789,"time_local":"19/Feb/2023:14:15:20 +0000","request":"GET /page?param1=value1¶m2=value2 HTTP/2.0","request_uri":"/page?param1=value1¶m2=value2","args":"param1=value1¶m2=value2","status":404,"body_bytes_sent":0,"bytes_sent":0,"http_referer":"","http_user_agent":"Mozilla/5.0 (Linux; Android 10; Pixel 3)","http_x_forwarded_for":"","host":"example.org","request_time":0.045,"upstream":"","upstream_connect_time":0.0,"upstream_header_time":0.0,"upstream_response_time":0.0,"upstream_response_length":0,"upstream_cache_status":"","ssl_protocol":"","ssl_cipher":"","scheme":"https","request_method":"GET","server_protocol":"HTTP/2.0","pipe":".","gzip_ratio":"","http_cf_ray":"","geoip_country_code":"GB"}
4
+{"msec":"1644997930.987","connection":123,"connection_requests":3,"pid":5678,"request_id":"9e632a5b24c18f76","request_length":234,"remote_addr":"192.168.0.1","remote_user":"jane_doe","remote_port":9876,"time_local":"19/Feb/2023:14:15:30 +0000","request":"PUT /api/update HTTP/1.1","request_uri":"/api/update","args":"","status":500,"body_bytes_sent":543,"bytes_sent":876,"http_referer":"https://example.com/page","http_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)","http_x_forwarded_for":"","host":"api.example.com","request_time":0.123,"upstream":"backend-server-2:8080","upstream_connect_time":0.045,"upstream_header_time":0.020,"upstream_response_time":0.058,"upstream_response_length":7890,"upstream_cache_status":"HIT","ssl_protocol":"TLSv1.2","ssl_cipher":"AES256-SHA256","scheme":"https","request_method":"PUT","server_protocol":"HTTP/1.1","pipe":"p","gzip_ratio":"1.8","http_cf_ray":"xyz789abc123","geoip_country_code":"CA"}
5
+{"msec":"1644997940.234","connection":9876,"connection_requests":8,"pid":4321,"request_id":"1b6c59c8aef7d24a","request_length":456,"remote_addr":"203.0.113.1","remote_user":"","remote_port":5432,"time_local":"19/Feb/2023:14:15:40 +0000","request":"DELETE /api/resource HTTP/2.0","request_uri":"/api/resource","args":"","status":204,"body_bytes_sent":0,"bytes_sent":123,"http_referer":"","http_user_agent":"curl/7.68.0","http_x_forwarded_for":"","host":"api.example.com","request_time":0.032,"upstream":"backend-server-1:8080","upstream_connect_time":0.012,"upstream_header_time":0.020,"upstream_response_time":0.010,"upstream_response_length":0,"upstream_cache_status":"","ssl_protocol":"","ssl_cipher":"","scheme":"http","request_method":"DELETE","server_protocol":"HTTP/2.0","pipe":".","gzip_ratio":"","http_cf_ray":"","geoip_country_code":""}
6
+{"msec":"1644997950.789","connection":5432,"connection_requests":12,"pid":6543,"request_id":"72692d781d0b8a4f","request_length":789,"remote_addr":"198.51.100.2","remote_user":"bob","remote_port":8765,"time_local":"19/Feb/2023:14:15:50 +0000","request":"GET /profile?user=bob HTTP/1.1","request_uri":"/profile?user=bob","args":"user=bob","status":200,"body_bytes_sent":1234,"bytes_sent":2345,"http_referer":"","http_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)","http_x_forwarded_for":"","host":"example.com","request_time":0.065,"upstream":"10.0.0.2:8080","upstream_connect_time":0.045,"upstream_header_time":0.020,"upstream_response_time":0.058,"upstream_response_length":7890,"upstream_cache_status":"MISS","ssl_protocol":"TLSv1.3","ssl_cipher":"AES128-GCM-SHA256","scheme":"https","request_method":"GET","server_protocol":"HTTP/1.1","pipe":"p","gzip_ratio":"","http_cf_ray":"","geoip_country_code":"US"}
7
+{"msec":"1644997960.321","connection":65432,"connection_requests":7,"pid":7890,"request_id":"c3e158d41e75a9d7","request_length":321,"remote_addr":"203.0.113.2","remote_user":"","remote_port":9876,"time_local":"19/Feb/2023:14:15:60 +0000","request":"GET /dashboard HTTP/2.0","request_uri":"/dashboard","args":"","status":301,"body_bytes_sent":0,"bytes_sent":123,"http_referer":"","http_user_agent":"Mozilla/5.0 (Linux; Android 10; Pixel 3)","http_x_forwarded_for":"","host":"dashboard.example.org","request_time":0.032,"upstream":"","upstream_connect_time":0.0,"upstream_header_time":0.0,"upstream_response_time":0.0,"upstream_response_length":0,"upstream_cache_status":"","ssl_protocol":"","ssl_cipher":"","scheme":"https","request_method":"GET","server_protocol":"HTTP/2.0","pipe":".","gzip_ratio":"","http_cf_ray":"","geoip_country_code":""}
8
+{"msec":"1644997970.555","connection":8765,"connection_requests":9,"pid":8765,"request_id":"f9f6e8235de54af4","request_length":654,"remote_addr":"10.0.0.4","remote_user":"","remote_port":12345,"time_local":"19/Feb/2023:14:15:70 +0000","request":"POST /submit-form HTTP/1.1","request_uri":"/submit-form","args":"","status":201,"body_bytes_sent":876,"bytes_sent":987,"http_referer":"","http_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)","http_x_forwarded_for":"","host":"example.com","request_time":0.045,"upstream":"backend-server-3:8080","upstream_connect_time":0.012,"upstream_header_time":0.020,"upstream_response_time":0.010,"upstream_response_length":0,"upstream_cache_status":"","ssl_protocol":"","ssl_cipher":"","scheme":"http","request_method":"POST","server_protocol":"HTTP/1.1","pipe":"p","gzip_ratio":"","http_cf_ray":"","geoip_country_code":""}
9
+{"msec":"1644997980.987","connection":23456,"connection_requests":6,"pid":3456,"request_id":"2ec3e8859e7a406c","request_length":432,"remote_addr":"198.51.100.3","remote_user":"mary","remote_port":5678,"time_local":"19/Feb/2023:14:15:80 +0000","request":"GET /contact HTTP/1.1","request_uri":"/contact","args":"","status":404,"body_bytes_sent":0,"bytes_sent":0,"http_referer":"","http_user_agent":"Mozilla/5.0 (Linux; Android 10; Pixel 3)","http_x_forwarded_for":"","host":"example.org","request_time":0.032,"upstream":"","upstream_connect_time":0.0,"upstream_header_time":0.0,"upstream_response_time":0.0,"upstream_response_length":0,"upstream_cache_status":"","ssl_protocol":"","ssl_cipher":"","scheme":"https","request_method":"GET","server_protocol":"HTTP/1.1","pipe":".","gzip_ratio":"","http_cf_ray":"","geoip_country_code":"FR"}
collectors/log2journal/tests.d/nginx-json.output
new
+296
@@ -0,0 +1,296 @@
1
+NGINX_TIMESTAMP_SEC=1644997905.123
2
+NGINX_CONNECTION=12345
3
+NGINX_CONNECTION_REQUESTS=5
4
+NGINX_PID=9876
5
+NGINX_REQUEST_ID=8f3ebc1e38fbb92f
6
+NGINX_REQUEST_LENGTH=345
7
+NGINX_REMOTE_ADDR=192.168.1.100
8
+NGINX_REMOTE_USER=john_doe
9
+NGINX_REMOTE_PORT=54321
10
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:05 +0000
11
+MESSAGE=GET /index.html HTTP/1.1
12
+NGINX_REQUEST_URI=/index.html?param=value
13
+NGINX_QUERY_STRING=param=value
14
+NGINX_STATUS=200
15
+PRIORITY=6
16
+NGINX_STATUS_FAMILY=2xx
17
+NGINX_BODY_BYTES_SENT=5432
18
+NGINX_BYTES_SENT=6543
19
+NGINX_HTTP_REFERER=https://example.com
20
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
21
+NGINX_HTTP_X_FORWARDED_FOR=192.168.1.50, 10.0.0.1
22
+NGINX_HOST=example.com
23
+NGINX_REQUEST_TIME=0.123
24
+NGINX_UPSTREAM=10.0.0.2:8080
25
+NGINX_UPSTREAM_CONNECT_TIME=0.045
26
+NGINX_UPSTREAM_HEADER_TIME=0.020
27
+NGINX_UPSTREAM_RESPONSE_TIME=0.058
28
+NGINX_UPSTREAM_RESPONSE_LENGTH=7890
29
+NGINX_UPSTREAM_CACHE_STATUS=MISS
30
+NGINX_SSL_PROTOCOL=TLSv1.2
31
+NGINX_SSL_CIPHER=AES256-SHA256
32
+NGINX_SCHEME=https
33
+NGINX_REQUEST_METHOD=GET
34
+NGINX_SERVER_PROTOCOL=HTTP/1.1
35
+NGINX_PIPELINED=no
36
+NGINX_GZIP_RATIO=2.1
37
+NGINX_HTTP_CF_RAY=abc123def456
38
+NGINX_GEOIP_COUNTRY_CODE=US
39
+SYSLOG_IDENTIFIER=nginx-log
40
+
41
+NGINX_TIMESTAMP_SEC=1644997910.789
42
+NGINX_CONNECTION=54321
43
+NGINX_CONNECTION_REQUESTS=10
44
+NGINX_PID=5432
45
+NGINX_REQUEST_ID=4a7bca5e19d3f8e7
46
+NGINX_REQUEST_LENGTH=432
47
+NGINX_REMOTE_ADDR=10.0.0.3
48
+NGINX_REMOTE_PORT=12345
49
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:10 +0000
50
+MESSAGE=POST /api/update HTTP/1.1
51
+NGINX_REQUEST_URI=/api/update
52
+NGINX_STATUS=204
53
+PRIORITY=6
54
+NGINX_STATUS_FAMILY=2xx
55
+NGINX_BODY_BYTES_SENT=0
56
+NGINX_BYTES_SENT=123
57
+NGINX_HTTP_USER_AGENT=curl/7.68.0
58
+NGINX_HOST=api.example.com
59
+NGINX_REQUEST_TIME=0.032
60
+NGINX_UPSTREAM=backend-server-1:8080
61
+NGINX_UPSTREAM_CONNECT_TIME=0.012
62
+NGINX_UPSTREAM_HEADER_TIME=0.020
63
+NGINX_UPSTREAM_RESPONSE_TIME=0.010
64
+NGINX_UPSTREAM_RESPONSE_LENGTH=0
65
+NGINX_SCHEME=http
66
+NGINX_REQUEST_METHOD=POST
67
+NGINX_SERVER_PROTOCOL=HTTP/1.1
68
+NGINX_PIPELINED=yes
69
+SYSLOG_IDENTIFIER=nginx-log
70
+
71
+NGINX_TIMESTAMP_SEC=1644997920.456
72
+NGINX_CONNECTION=98765
73
+NGINX_CONNECTION_REQUESTS=15
74
+NGINX_PID=1234
75
+NGINX_REQUEST_ID=63f8ad2c3e1b4090
76
+NGINX_REQUEST_LENGTH=567
77
+NGINX_REMOTE_ADDR=2001:0db8:85a3:0000:0000:8a2e:0370:7334
78
+NGINX_REMOTE_USER=alice
79
+NGINX_REMOTE_PORT=6789
80
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:20 +0000
81
+MESSAGE=GET /page?param1=value1¶m2=value2 HTTP/2.0
82
+NGINX_REQUEST_URI=/page?param1=value1¶m2=value2
83
+NGINX_QUERY_STRING=param1=value1¶m2=value2
84
+NGINX_STATUS=404
85
+PRIORITY=5
86
+NGINX_STATUS_FAMILY=4xx
87
+NGINX_BODY_BYTES_SENT=0
88
+NGINX_BYTES_SENT=0
89
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
90
+NGINX_HOST=example.org
91
+NGINX_REQUEST_TIME=0.045
92
+NGINX_UPSTREAM_CONNECT_TIME=0.0
93
+NGINX_UPSTREAM_HEADER_TIME=0.0
94
+NGINX_UPSTREAM_RESPONSE_TIME=0.0
95
+NGINX_UPSTREAM_RESPONSE_LENGTH=0
96
+NGINX_SCHEME=https
97
+NGINX_REQUEST_METHOD=GET
98
+NGINX_SERVER_PROTOCOL=HTTP/2.0
99
+NGINX_PIPELINED=no
100
+NGINX_GEOIP_COUNTRY_CODE=GB
101
+SYSLOG_IDENTIFIER=nginx-log
102
+
103
+NGINX_TIMESTAMP_SEC=1644997930.987
104
+NGINX_CONNECTION=123
105
+NGINX_CONNECTION_REQUESTS=3
106
+NGINX_PID=5678
107
+NGINX_REQUEST_ID=9e632a5b24c18f76
108
+NGINX_REQUEST_LENGTH=234
109
+NGINX_REMOTE_ADDR=192.168.0.1
110
+NGINX_REMOTE_USER=jane_doe
111
+NGINX_REMOTE_PORT=9876
112
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:30 +0000
113
+MESSAGE=PUT /api/update HTTP/1.1
114
+NGINX_REQUEST_URI=/api/update
115
+NGINX_STATUS=500
116
+PRIORITY=3
117
+NGINX_STATUS_FAMILY=5xx
118
+NGINX_BODY_BYTES_SENT=543
119
+NGINX_BYTES_SENT=876
120
+NGINX_HTTP_REFERER=https://example.com/page
121
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
122
+NGINX_HOST=api.example.com
123
+NGINX_REQUEST_TIME=0.123
124
+NGINX_UPSTREAM=backend-server-2:8080
125
+NGINX_UPSTREAM_CONNECT_TIME=0.045
126
+NGINX_UPSTREAM_HEADER_TIME=0.020
127
+NGINX_UPSTREAM_RESPONSE_TIME=0.058
128
+NGINX_UPSTREAM_RESPONSE_LENGTH=7890
129
+NGINX_UPSTREAM_CACHE_STATUS=HIT
130
+NGINX_SSL_PROTOCOL=TLSv1.2
131
+NGINX_SSL_CIPHER=AES256-SHA256
132
+NGINX_SCHEME=https
133
+NGINX_REQUEST_METHOD=PUT
134
+NGINX_SERVER_PROTOCOL=HTTP/1.1
135
+NGINX_PIPELINED=yes
136
+NGINX_GZIP_RATIO=1.8
137
+NGINX_HTTP_CF_RAY=xyz789abc123
138
+NGINX_GEOIP_COUNTRY_CODE=CA
139
+SYSLOG_IDENTIFIER=nginx-log
140
+
141
+NGINX_TIMESTAMP_SEC=1644997940.234
142
+NGINX_CONNECTION=9876
143
+NGINX_CONNECTION_REQUESTS=8
144
+NGINX_PID=4321
145
+NGINX_REQUEST_ID=1b6c59c8aef7d24a
146
+NGINX_REQUEST_LENGTH=456
147
+NGINX_REMOTE_ADDR=203.0.113.1
148
+NGINX_REMOTE_PORT=5432
149
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:40 +0000
150
+MESSAGE=DELETE /api/resource HTTP/2.0
151
+NGINX_REQUEST_URI=/api/resource
152
+NGINX_STATUS=204
153
+PRIORITY=6
154
+NGINX_STATUS_FAMILY=2xx
155
+NGINX_BODY_BYTES_SENT=0
156
+NGINX_BYTES_SENT=123
157
+NGINX_HTTP_USER_AGENT=curl/7.68.0
158
+NGINX_HOST=api.example.com
159
+NGINX_REQUEST_TIME=0.032
160
+NGINX_UPSTREAM=backend-server-1:8080
161
+NGINX_UPSTREAM_CONNECT_TIME=0.012
162
+NGINX_UPSTREAM_HEADER_TIME=0.020
163
+NGINX_UPSTREAM_RESPONSE_TIME=0.010
164
+NGINX_UPSTREAM_RESPONSE_LENGTH=0
165
+NGINX_SCHEME=http
166
+NGINX_REQUEST_METHOD=DELETE
167
+NGINX_SERVER_PROTOCOL=HTTP/2.0
168
+NGINX_PIPELINED=no
169
+SYSLOG_IDENTIFIER=nginx-log
170
+
171
+NGINX_TIMESTAMP_SEC=1644997950.789
172
+NGINX_CONNECTION=5432
173
+NGINX_CONNECTION_REQUESTS=12
174
+NGINX_PID=6543
175
+NGINX_REQUEST_ID=72692d781d0b8a4f
176
+NGINX_REQUEST_LENGTH=789
177
+NGINX_REMOTE_ADDR=198.51.100.2
178
+NGINX_REMOTE_USER=bob
179
+NGINX_REMOTE_PORT=8765
180
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:50 +0000
181
+MESSAGE=GET /profile?user=bob HTTP/1.1
182
+NGINX_REQUEST_URI=/profile?user=bob
183
+NGINX_QUERY_STRING=user=bob
184
+NGINX_STATUS=200
185
+PRIORITY=6
186
+NGINX_STATUS_FAMILY=2xx
187
+NGINX_BODY_BYTES_SENT=1234
188
+NGINX_BYTES_SENT=2345
189
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
190
+NGINX_HOST=example.com
191
+NGINX_REQUEST_TIME=0.065
192
+NGINX_UPSTREAM=10.0.0.2:8080
193
+NGINX_UPSTREAM_CONNECT_TIME=0.045
194
+NGINX_UPSTREAM_HEADER_TIME=0.020
195
+NGINX_UPSTREAM_RESPONSE_TIME=0.058
196
+NGINX_UPSTREAM_RESPONSE_LENGTH=7890
197
+NGINX_UPSTREAM_CACHE_STATUS=MISS
198
+NGINX_SSL_PROTOCOL=TLSv1.3
199
+NGINX_SSL_CIPHER=AES128-GCM-SHA256
200
+NGINX_SCHEME=https
201
+NGINX_REQUEST_METHOD=GET
202
+NGINX_SERVER_PROTOCOL=HTTP/1.1
203
+NGINX_PIPELINED=yes
204
+NGINX_GEOIP_COUNTRY_CODE=US
205
+SYSLOG_IDENTIFIER=nginx-log
206
+
207
+NGINX_TIMESTAMP_SEC=1644997960.321
208
+NGINX_CONNECTION=65432
209
+NGINX_CONNECTION_REQUESTS=7
210
+NGINX_PID=7890
211
+NGINX_REQUEST_ID=c3e158d41e75a9d7
212
+NGINX_REQUEST_LENGTH=321
213
+NGINX_REMOTE_ADDR=203.0.113.2
214
+NGINX_REMOTE_PORT=9876
215
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:60 +0000
216
+MESSAGE=GET /dashboard HTTP/2.0
217
+NGINX_REQUEST_URI=/dashboard
218
+NGINX_STATUS=301
219
+PRIORITY=6
220
+NGINX_STATUS_FAMILY=3xx
221
+NGINX_BODY_BYTES_SENT=0
222
+NGINX_BYTES_SENT=123
223
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
224
+NGINX_HOST=dashboard.example.org
225
+NGINX_REQUEST_TIME=0.032
226
+NGINX_UPSTREAM_CONNECT_TIME=0.0
227
+NGINX_UPSTREAM_HEADER_TIME=0.0
228
+NGINX_UPSTREAM_RESPONSE_TIME=0.0
229
+NGINX_UPSTREAM_RESPONSE_LENGTH=0
230
+NGINX_SCHEME=https
231
+NGINX_REQUEST_METHOD=GET
232
+NGINX_SERVER_PROTOCOL=HTTP/2.0
233
+NGINX_PIPELINED=no
234
+SYSLOG_IDENTIFIER=nginx-log
235
+
236
+NGINX_TIMESTAMP_SEC=1644997970.555
237
+NGINX_CONNECTION=8765
238
+NGINX_CONNECTION_REQUESTS=9
239
+NGINX_PID=8765
240
+NGINX_REQUEST_ID=f9f6e8235de54af4
241
+NGINX_REQUEST_LENGTH=654
242
+NGINX_REMOTE_ADDR=10.0.0.4
243
+NGINX_REMOTE_PORT=12345
244
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:70 +0000
245
+MESSAGE=POST /submit-form HTTP/1.1
246
+NGINX_REQUEST_URI=/submit-form
247
+NGINX_STATUS=201
248
+PRIORITY=6
249
+NGINX_STATUS_FAMILY=2xx
250
+NGINX_BODY_BYTES_SENT=876
251
+NGINX_BYTES_SENT=987
252
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
253
+NGINX_HOST=example.com
254
+NGINX_REQUEST_TIME=0.045
255
+NGINX_UPSTREAM=backend-server-3:8080
256
+NGINX_UPSTREAM_CONNECT_TIME=0.012
257
+NGINX_UPSTREAM_HEADER_TIME=0.020
258
+NGINX_UPSTREAM_RESPONSE_TIME=0.010
259
+NGINX_UPSTREAM_RESPONSE_LENGTH=0
260
+NGINX_SCHEME=http
261
+NGINX_REQUEST_METHOD=POST
262
+NGINX_SERVER_PROTOCOL=HTTP/1.1
263
+NGINX_PIPELINED=yes
264
+SYSLOG_IDENTIFIER=nginx-log
265
+
266
+NGINX_TIMESTAMP_SEC=1644997980.987
267
+NGINX_CONNECTION=23456
268
+NGINX_CONNECTION_REQUESTS=6
269
+NGINX_PID=3456
270
+NGINX_REQUEST_ID=2ec3e8859e7a406c
271
+NGINX_REQUEST_LENGTH=432
272
+NGINX_REMOTE_ADDR=198.51.100.3
273
+NGINX_REMOTE_USER=mary
274
+NGINX_REMOTE_PORT=5678
275
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:80 +0000
276
+MESSAGE=GET /contact HTTP/1.1
277
+NGINX_REQUEST_URI=/contact
278
+NGINX_STATUS=404
279
+PRIORITY=5
280
+NGINX_STATUS_FAMILY=4xx
281
+NGINX_BODY_BYTES_SENT=0
282
+NGINX_BYTES_SENT=0
283
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
284
+NGINX_HOST=example.org
285
+NGINX_REQUEST_TIME=0.032
286
+NGINX_UPSTREAM_CONNECT_TIME=0.0
287
+NGINX_UPSTREAM_HEADER_TIME=0.0
288
+NGINX_UPSTREAM_RESPONSE_TIME=0.0
289
+NGINX_UPSTREAM_RESPONSE_LENGTH=0
290
+NGINX_SCHEME=https
291
+NGINX_REQUEST_METHOD=GET
292
+NGINX_SERVER_PROTOCOL=HTTP/1.1
293
+NGINX_PIPELINED=no
294
+NGINX_GEOIP_COUNTRY_CODE=FR
295
+SYSLOG_IDENTIFIER=nginx-log
296
+
collectors/log2journal/tests.sh
new
+142
@@ -0,0 +1,142 @@
1
+#!/usr/bin/env bash
2
+
3
+if [ -f "${PWD}/log2journal" ]; then
4
+ log2journal_bin="${PWD}/log2journal"
5
+else
6
+ log2journal_bin="$(which log2journal)"
7
+fi
8
+
9
+[ -z "${log2journal_bin}" ] && echo >&2 "Cannot find log2journal binary" && exit 1
10
+echo >&2 "Using: ${log2journal_bin}"
11
+
12
+script_dir=$(dirname "$(readlink -f "$0")")
13
+tests="${script_dir}/tests.d"
14
+
15
+if [ ! -d "${tests}" ]; then
16
+ echo >&2 "tests directory '${tests}' is not found."
17
+ exit 1
18
+fi
19
+
20
+# Create a random directory name in /tmp
21
+tmp=$(mktemp -d /tmp/script_temp.XXXXXXXXXX)
22
+
23
+# Function to clean up the temporary directory on exit
24
+cleanup() {
25
+ echo "Cleaning up..."
26
+ rm -rf "$tmp"
27
+}
28
+
29
+# Register the cleanup function to run on script exit
30
+trap cleanup EXIT
31
+
32
+# Change to the temporary directory
33
+cd "$tmp" || exit 1
34
+
35
+# -----------------------------------------------------------------------------
36
+
37
+test_log2journal_config() {
38
+ local in="${1}"
39
+ local out="${2}"
40
+ shift 2
41
+
42
+ [ -f output ] && rm output
43
+
44
+ printf >&2 "running: "
45
+ printf >&2 "%q " "${log2journal_bin}" "${@}"
46
+ printf >&2 "\n"
47
+
48
+ "${log2journal_bin}" <"${in}" "${@}" >output 2>&1
49
+ ret=$?
50
+
51
+ [ $ret -ne 0 ] && echo >&2 "${log2journal_bin} exited with code: $ret" && cat output && exit 1
52
+
53
+ diff --ignore-all-space "${out}" output
54
+ [ $? -ne -0 ] && echo >&2 "${log2journal_bin} output does not match!" && exit 1
55
+
56
+ echo >&2 "OK"
57
+ echo >&2
58
+
59
+ return 0
60
+}
61
+
62
+# test yaml parsing
63
+echo >&2
64
+echo >&2 "Testing full yaml config parsing..."
65
+test_log2journal_config /dev/null "${tests}/full.output" -f "${tests}/full.yaml" --show-config || exit 1
66
+
67
+echo >&2 "Testing command line parsing..."
68
+test_log2journal_config /dev/null "${tests}/full.output" --show-config \
69
+ --prefix=NGINX_ \
70
+ --filename-key NGINX_LOG_FILENAME \
71
+ --duplicate PRIORITY=NGINX_STATUS \
72
+ --duplicate=NGINX_STATUS_FAMILY=NGINX_STATUS,NGINX_METHOD \
73
+ --inject SYSLOG_IDENTIFIER=nginx-log \
74
+ --inject=SYSLOG_IDENTIFIER2=nginx-log2 \
75
+ --rewrite "PRIORITY=/^[123]/6" \
76
+ --rewrite='PRIORITY=|^4|5' \
77
+ '--rewrite=PRIORITY=-^5-3' \
78
+ --rewrite "PRIORITY=;.*;4" \
79
+ --rewrite 'NGINX_STATUS_FAMILY=|^(?<first_digit>[1-5])|${first_digit}xx' \
80
+ --rewrite 'NGINX_STATUS_FAMILY=|.*|UNKNOWN' \
81
+ --rename TEST1=TEST2 \
82
+ --rename=TEST3=TEST4 \
83
+ --unmatched-key MESSAGE \
84
+ --inject-unmatched PRIORITY=1 \
85
+ --inject-unmatched=PRIORITY2=2 \
86
+ --include=".*" \
87
+ --exclude ".*HELLO.*WORLD.*" \
88
+ '(?x) # Enable PCRE2 extended mode
89
+ ^
90
+ (?<NGINX_REMOTE_ADDR>[^ ]+) \s - \s # NGINX_REMOTE_ADDR
91
+ (?<NGINX_REMOTE_USER>[^ ]+) \s # NGINX_REMOTE_USER
92
+ \[
93
+ (?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL
94
+ \]
95
+ \s+ "
96
+ (?<MESSAGE>
97
+ (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD
98
+ (?<NGINX_URL>[^ ]+) \s+
99
+ HTTP/(?<NGINX_HTTP_VERSION>[^"]+)
100
+ )
101
+ " \s+
102
+ (?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS
103
+ (?<NGINX_BODY_BYTES_SENT>\d+) \s+ # NGINX_BODY_BYTES_SENT
104
+ "(?<NGINX_HTTP_REFERER>[^"]*)" \s+ # NGINX_HTTP_REFERER
105
+ "(?<NGINX_HTTP_USER_AGENT>[^"]*)" # NGINX_HTTP_USER_AGENT' \
106
+ || exit 1
107
+
108
+# -----------------------------------------------------------------------------
109
+
110
+test_log2journal() {
111
+ local in="${1}"
112
+ local out="${2}"
113
+ shift 2
114
+
115
+ printf >&2 "running: "
116
+ printf >&2 "%q " "${log2journal_bin}" "${@}"
117
+ printf >&2 "\n"
118
+ echo >&2 "using as input: ${in}"
119
+
120
+ [ -f output ] && rm output
121
+
122
+ "${log2journal_bin}" <"${in}" "${@}" >output 2>&1
123
+ ret=$?
124
+
125
+ [ $ret -ne 0 ] && echo >&2 "${log2journal_bin} exited with code: $ret" && cat output && exit 1
126
+
127
+ diff "${out}" output
128
+ [ $? -ne -0 ] && echo >&2 "${log2journal_bin} output does not match!" && cat output && exit 1
129
+
130
+ echo >&2 "OK"
131
+ echo >&2
132
+
133
+ return 0
134
+}
135
+
136
+echo >&2
137
+echo >&2 "Testing parsing and output..."
138
+
139
+test_log2journal ${tests}/json.log ${tests}/json.output json
140
+test_log2journal ${tests}/json.log ${tests}/json-include.output json --include "OBJECT"
141
+test_log2journal ${tests}/json.log ${tests}/json-exclude.output json --exclude "ARRAY[^2]"
142
+test_log2journal ${tests}/nginx-json.log ${tests}/nginx-json.output -f "${script_dir}/log2journal.d/nginx-json.yaml"
libnetdata/facets/facets.c
+1
-89
@@ -99,95 +99,7 @@ static inline bool is_valid_string_hash(const char *s) {
99
}
100
101
// ----------------------------------------------------------------------------
102
-
103
-typedef uint64_t SIMPLE_HASHTABLE_HASH;
104
-#define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32
105
-
106
-typedef struct simple_hashtable_slot {
107
- SIMPLE_HASHTABLE_HASH hash;
108
- void *data;
109
-} SIMPLE_HASHTABLE_SLOT;
110
-
111
-typedef struct simple_hashtable {
112
- size_t resizes;
113
- size_t searches;
114
- size_t collisions;
115
- size_t used;
116
- size_t size;
117
- SIMPLE_HASHTABLE_SLOT *hashtable;
118
-} SIMPLE_HASHTABLE;
119
-
120
-static void simple_hashtable_init(SIMPLE_HASHTABLE *ht, size_t size) {
121
- ht->resizes = 0;
122
- ht->used = 0;
123
- ht->size = size;
124
- ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
125
-}
126
-
127
-static void simple_hashtable_free(SIMPLE_HASHTABLE *ht) {
128
- freez(ht->hashtable);
129
- ht->hashtable = NULL;
130
- ht->size = 0;
131
- ht->used = 0;
132
- ht->resizes = 0;
133
-}
134
-
135
-static void simple_hashtable_resize_double(SIMPLE_HASHTABLE *ht);
136
-
137
-static inline SIMPLE_HASHTABLE_SLOT *simple_hashtable_get_slot(SIMPLE_HASHTABLE *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) {
138
- // IMPORTANT:
139
- // If the hashtable supported deletions, we would need to have a special slot.data value
140
- // to mark deleted values and assume they are occupied during lookup, but empty during insert.
141
- // But for our case, we don't need it, since we never delete items from the hashtable.
142
-
143
- ht->searches++;
144
-
145
- size_t slot = hash % ht->size;
146
- if(likely(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash))
147
- return &ht->hashtable[slot];
148
-
149
- ht->collisions++;
150
-
151
- if(unlikely(resize && ht->size <= (ht->used << 4))) {
152
- simple_hashtable_resize_double(ht);
153
-
154
- slot = hash % ht->size;
155
- if(likely(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash))
156
- return &ht->hashtable[slot];
157
-
158
- ht->collisions++;
159
- }
160
-
161
- slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size;
162
- // Linear probing until we find it
163
- while (ht->hashtable[slot].data && ht->hashtable[slot].hash != hash) {
164
- slot = (slot + 1) % ht->size; // Wrap around if necessary
165
- ht->collisions++;
166
- }
167
-
168
- return &ht->hashtable[slot];
169
-}
170
-
171
-static void simple_hashtable_resize_double(SIMPLE_HASHTABLE *ht) {
172
- SIMPLE_HASHTABLE_SLOT *old = ht->hashtable;
173
- size_t old_size = ht->size;
174
-
175
- ht->resizes++;
176
- ht->size = (ht->size << 3) - 1;
177
- ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
178
- for(size_t i = 0 ; i < old_size ; i++) {
179
- if(!old[i].data)
180
- continue;
181
-
182
- SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(ht, old[i].hash, false);
183
- *slot = old[i];
184
- }
185
-
186
- freez(old);
187
-}
188
-
189
-
190
-// ----------------------------------------------------------------------------
102
+#include "../simple_hashtable.h"
103
104
typedef struct facet_value {
105
FACETS_HASH hash;
libnetdata/simple_hashtable.h
new
+92
@@ -0,0 +1,92 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_SIMPLE_HASHTABLE_H
4
+#define NETDATA_SIMPLE_HASHTABLE_H
5
+
6
+typedef uint64_t SIMPLE_HASHTABLE_HASH;
7
+#define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32
8
+
9
+typedef struct simple_hashtable_slot {
10
+ SIMPLE_HASHTABLE_HASH hash;
11
+ void *data;
12
+} SIMPLE_HASHTABLE_SLOT;
13
+
14
+typedef struct simple_hashtable {
15
+ size_t resizes;
16
+ size_t searches;
17
+ size_t collisions;
18
+ size_t used;
19
+ size_t size;
20
+ SIMPLE_HASHTABLE_SLOT *hashtable;
21
+} SIMPLE_HASHTABLE;
22
+
23
+static void simple_hashtable_init(SIMPLE_HASHTABLE *ht, size_t size) {
24
+ ht->resizes = 0;
25
+ ht->used = 0;
26
+ ht->size = size;
27
+ ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
28
+}
29
+
30
+static void simple_hashtable_free(SIMPLE_HASHTABLE *ht) {
31
+ freez(ht->hashtable);
32
+ ht->hashtable = NULL;
33
+ ht->size = 0;
34
+ ht->used = 0;
35
+ ht->resizes = 0;
36
+}
37
+
38
+static inline void simple_hashtable_resize(SIMPLE_HASHTABLE *ht);
39
+
40
+static inline SIMPLE_HASHTABLE_SLOT *simple_hashtable_get_slot(SIMPLE_HASHTABLE *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) {
41
+ // IMPORTANT:
42
+ // If the hashtable supported deletions, we would need to have a special slot.data value
43
+ // to mark deleted values and assume they are occupied during lookup, but empty during insert.
44
+ // But for our case, we don't need it, since we never delete items from the hashtable.
45
+
46
+ ht->searches++;
47
+
48
+ size_t slot = hash % ht->size;
49
+ if(likely(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash))
50
+ return &ht->hashtable[slot];
51
+
52
+ ht->collisions++;
53
+
54
+ if(unlikely(resize && ht->size <= (ht->used << 4))) {
55
+ simple_hashtable_resize(ht);
56
+
57
+ slot = hash % ht->size;
58
+ if(likely(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash))
59
+ return &ht->hashtable[slot];
60
+
61
+ ht->collisions++;
62
+ }
63
+
64
+ slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size;
65
+ // Linear probing until we find it
66
+ while (ht->hashtable[slot].data && ht->hashtable[slot].hash != hash) {
67
+ slot = (slot + 1) % ht->size; // Wrap around if necessary
68
+ ht->collisions++;
69
+ }
70
+
71
+ return &ht->hashtable[slot];
72
+}
73
+
74
+static inline void simple_hashtable_resize(SIMPLE_HASHTABLE *ht) {
75
+ SIMPLE_HASHTABLE_SLOT *old = ht->hashtable;
76
+ size_t old_size = ht->size;
77
+
78
+ ht->resizes++;
79
+ ht->size = (ht->size << 3) - 1;
80
+ ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
81
+ for(size_t i = 0 ; i < old_size ; i++) {
82
+ if(!old[i].data)
83
+ continue;
84
+
85
+ SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(ht, old[i].hash, false);
86
+ *slot = old[i];
87
+ }
88
+
89
+ freez(old);
90
+}
91
+
92
+#endif //NETDATA_SIMPLE_HASHTABLE_H