log2journal improvements 4 (#16510)
* fix codeql warnings * work to support multiple types of value rewrites * support looking up variables names in all values specified * added more unit tests for the current functionality * satisfy coverity * fixed quotes * test nginx-combined.yaml too * removed deletions - injections can now do the same * remove empty line
Costa Tsaousis committed
Nov 30, 2023 at 23:18 UTC
0705251a8a204d565017a2f29f157ffae39a5932
25 files changed
+1524
-1302
Makefile.am
-1
@@ -357,7 +357,6 @@ LOG2JOURNAL_FILES = \
357
collectors/log2journal/log2journal-logfmt.c \
358
collectors/log2journal/log2journal-pcre2.c \
359
collectors/log2journal/log2journal-params.c \
360
- collectors/log2journal/log2journal-duplicate.c \
360
collectors/log2journal/log2journal-inject.c \
361
collectors/log2journal/log2journal-pattern.c \
362
collectors/log2journal/log2journal-replace.c \
collectors/log2journal/README.md
+48
-56
@@ -140,10 +140,10 @@ Avoid setting priority to 0 (`LOG_EMERG`), because these will be on your termina
140
141
To set the PRIORITY field in the output, we can use `NGINX_STATUS` fields. We need a copy of it, which we will alter later.
142
143
-We can instruct `log2journal` to duplicate `NGINX_STATUS`, like this: `log2journal --duplicate=PRIORITY=NGINX_STATUS`. Let's try it:
143
+We can instruct `log2journal` to duplicate `NGINX_STATUS`, like this: `log2journal --inject 'PRIORITY=${NGINX_STATUS}'`. Let's try it:
144
145
```bash
146
-# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --duplicate=PRIORITY=NGINX_STATUS
146
+# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --inject 'PRIORITY=${NGINX_STATUS}'
147
MESSAGE=GET /index.html HTTP/1.1
148
NGINX_BODY_BYTES_SENT=4172
149
NGINX_HTTP_REFERER=-
@@ -161,10 +161,10 @@ NGINX_URL=/index.html
161
162
```
163
164
-Now that we have the `PRIORITY` field equal to the `NGINX_STATUS`, we can use instruct `log2journal` to change it to a valid priority, by appending: `--rewrite=PRIORITY=/^5/3 --rewrite=PRIORITY=/.*/6`. These rewrite commands say to match everything that starts with `5` and replace it with priority `3` (error) and everything else with priority `6` (info). Let's see it:
164
+Now that we have the `PRIORITY` field equal to the `NGINX_STATUS`, we can use instruct `log2journal` to change it to a valid priority, by appending: `--rewrite 'PRIORITY=/^5/3' --rewrite 'PRIORITY=/.*/6'`. These rewrite commands say to match everything that starts with `5` and replace it with priority `3` (error) and everything else with priority `6` (info). Let's see it:
165
166
```bash
167
-# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --duplicate=STATUS2PRIORITY=NGINX_STATUS --rewrite=PRIORITY=/^5/3 --rewrite=PRIORITY=/.*/6
167
+# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --inject 'PRIORITY=${NGINX_STATUS}' --rewrite 'PRIORITY=/^5/3' --rewrite 'PRIORITY=/.*/6'
168
MESSAGE=GET /index.html HTTP/1.1
169
NGINX_BODY_BYTES_SENT=4172
170
NGINX_HTTP_REFERER=-
@@ -182,12 +182,12 @@ NGINX_URL=/index.html
182
183
```
184
185
-Similarly, we could duplicate `NGINX_URL` to `NGINX_ENDPOINT` and then process it with sed to remove any query string, or replace IDs in the URL path with constant names, thus giving us uniform endpoints independently of the parameters.
185
+Similarly, we could duplicate `${NGINX_URL}` to `NGINX_ENDPOINT` and then process it to remove any query string, or replace IDs in the URL path with constant names, thus giving us uniform endpoints independently of the parameters.
186
187
-To complete the example, we can also inject a `SYSLOG_IDENTIFIER` with `log2journal`, using `--inject=SYSLOG_IDENTIFIER=nginx-log`, like this:
187
+To complete the example, we can also inject a `SYSLOG_IDENTIFIER` with `log2journal`, using `--inject SYSLOG_IDENTIFIER=nginx-log`, like this:
188
189
```bash
190
-# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --duplicate=STATUS2PRIORITY=NGINX_STATUS --inject=SYSLOG_IDENTIFIER=nginx -rewrite=PRIORITY=/^5/3 --rewrite=PRIORITY=/.*/6
190
+# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --inject 'PRIORITY=${NGINX_STATUS}' --inject 'SYSLOG_IDENTIFIER=nginx' -rewrite 'PRIORITY=/^5/3' --rewrite 'PRIORITY=/.*/6'
191
MESSAGE=GET /index.html HTTP/1.1
192
NGINX_BODY_BYTES_SENT=4172
193
NGINX_HTTP_REFERER=-
@@ -210,7 +210,7 @@ Now the message is ready to be sent to a systemd-journal. For this we use `syste
210
211
212
```bash
213
-# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --duplicate=STATUS2PRIORITY=NGINX_STATUS --inject=SYSLOG_IDENTIFIER=nginx -rewrite=PRIORITY=/^5/3 --rewrite=PRIORITY=/.*/6 | systemd-cat-native
213
+# echo '1.2.3.4 - - [19/Nov/2023:00:24:43 +0000] "GET /index.html HTTP/1.1" 200 4172 104 0.001 "-" "Go-http-client/1.1"' | log2journal '^(?<NGINX_REMOTE_ADDR>[^ ]+) - (?<NGINX_REMOTE_USER>[^ ]+) \[(?<NGINX_TIME_LOCAL>[^\]]+)\] "(?<MESSAGE>(?<NGINX_METHOD>[A-Z]+) (?<NGINX_URL>[^ ]+) HTTP/(?<NGINX_HTTP_VERSION>[^"]+))" (?<NGINX_STATUS>\d+) (?<NGINX_BODY_BYTES_SENT>\d+) (?<NGINX_REQUEST_LENGTH>\d+) (?<NGINX_REQUEST_TIME>[\d.]+) "(?<NGINX_HTTP_REFERER>[^"]*)" "(?<NGINX_HTTP_USER_AGENT>[^"]*)"' --inject 'PRIORITY=${NGINX_STATUS}' --inject 'SYSLOG_IDENTIFIER=nginx' -rewrite 'PRIORITY=/^5/3' --rewrite 'PRIORITY=/.*/6' | systemd-cat-native
214
# no output
215
216
# let's find the message
@@ -296,24 +296,24 @@ pattern='(?x) # Enable PCRE2 extended mode
296
297
tail -n $last -F /var/log/nginx/*access.log \
298
| log2journal "${pattern}" \
299
- --filename-key=NGINX_LOG_FILE \
300
- --duplicate=PRIORITY=NGINX_STATUS \
301
- --duplicate=NGINX_STATUS_FAMILY=NGINX_STATUS \
302
- --inject=SYSLOG_IDENTIFIER=nginx-log \
303
- --unmatched-key=MESSAGE \
304
- --inject-unmatched=PRIORITY=1 \
305
- --rewrite='PRIORITY=/^5/3 --rewrite=PRIORITY=/.*/6' \
306
- --rewrite='NGINX_STATUS_FAMILY=/^(?<first_digit>[0-9]).*$/${first_digit}xx' \
307
- --rewrite='NGINX_STATUS_FAMILY=/^.*$/UNKNOWN' \
299
+ --filename-key 'NGINX_LOG_FILE' \
300
+ --unmatched-key 'MESSAGE' \
301
+ --inject-unmatched 'PRIORITY=1' \
302
+ --inject 'PRIORITY=${NGINX_STATUS}' \
303
+ --rewrite 'PRIORITY=/^5/3' \
304
+ --rewrite 'PRIORITY=/.*/6' \
305
+ --inject 'NGINX_STATUS_FAMILY=${NGINX_STATUS}' \
306
+ --rewrite 'NGINX_STATUS_FAMILY=/^(?<first_digit>[0-9]).*$/${first_digit}xx' \
307
+ --rewrite 'NGINX_STATUS_FAMILY=/^.*$/UNKNOWN' \
308
+ --inject 'SYSLOG_IDENTIFIER=nginx-log' \
309
| $send_or_show
310
```
311
311
-
312
## `log2journal` options
313
314
```
315
316
-Netdata log2journal v1.43.0-306-g929866ad3
316
+Netdata log2journal v1.43.0-313-gd79fbac6a
317
318
Convert logs to systemd Journal Export Format.
319
@@ -351,14 +351,6 @@ Options:
351
unmatched entry will appear as the log message in the journals.
352
Use --inject-unmatched to inject additional fields to unmatched lines.
353
354
- --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]
355
- Create a new key called TARGET, duplicating the values of the keys
356
- given. Useful for further processing. When multiple keys are given,
357
- their values are separated by comma.
358
-
359
- Up to 512 duplications can be given on the command line, and up to
360
- 20 keys per duplication command are allowed.
361
-
354
--inject LINE
355
Inject constant fields to the output (both matched and unmatched logs).
356
--inject entries are added to unmatched lines too, when their key is
@@ -455,40 +447,40 @@ This is a simple diagram of the pipeline taking place:
447
| INPUT |
448
| read one log line at a time |
449
+---------------------------------------------------+
458
- v v
459
- +---------------------------------+ |
460
- | EXTRACT FIELDS AND VALUES | |
461
- | JSON, logfmt, or pattern based | |
462
- | (apply optional PREFIX) | |
463
- +---------------------------------+ |
464
- v v |
465
- +---------------+ +--------------+ |
466
- | DUPLICATE | | FILTER | |
467
- | | | filter keys | |
468
- | create new | +--------------+ |
469
- | fields by | v |
470
- | duplicating | +--------------+ |
471
- | other fields | | RENAME | |
472
- | and their | | change | |
473
- | values | | field names | |
474
- +---------------+ +--------------+ |
475
- v v v
476
- +---------------------------------+ +--------------+
477
- | REWRITE PIPELINES | | INJECT |
478
- | altering the values of fields | | constants |
479
- +---------------------------------+ +--------------+
480
- v v
450
+ v v v v v v
451
+ +---------------------------------------------------+
452
+ | EXTRACT FIELDS AND VALUES |
453
+ | JSON, logfmt, or pattern based |
454
+ | (apply optional PREFIX) |
455
+ +---------------------------------------------------+
456
+ v v v v v v
457
+ +---------------------------------------------------+
458
+ | RENAME FIELDS |
459
+ | change the names of the fields |
460
+ +---------------------------------------------------+
461
+ v v v v v v
462
+ +---------------------------------------------------+
463
+ | INJECT NEW FIELDS |
464
+ | constants, or other field values as variables |
465
+ +---------------------------------------------------+
466
+ v v v v v v
467
+ +---------------------------------------------------+
468
+ | REWRITE FIELD VALUES |
469
+ | pipeline multiple rewriting rules to alter |
470
+ | the values of the fields |
471
+ +---------------------------------------------------+
472
+ v v v v v v
473
+ +---------------------------------------------------+
474
+ | FILTER FIELDS |
475
+ | use include and exclude patterns on the field |
476
+ | names, to select which fields are sent to journal |
477
+ +---------------------------------------------------+
478
+ v v v v v v
479
+---------------------------------------------------+
480
| OUTPUT |
481
| generate Journal Export Format |
482
+---------------------------------------------------+
483
486
-IMPORTANT:
487
- - Extraction of keys includes formatting them according to journal rules.
488
- - Duplication rules use the original extracted field names, after they have
489
- been prefixed (when a PREFIX is set) and before they are renamed.
490
- - Rewriting is always the last stage, so the final field names are matched.
491
-
484
--------------------------------------------------------------------------------
485
JOURNAL FIELDS RULES (enforced by systemd-journald)
486
collectors/log2journal/log2journal-duplicate.c
deleted
-49
@@ -1,49 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "log2journal.h"
4
-
5
-void duplication_cleanup(DUPLICATION *dp) {
6
- hashed_key_cleanup(&dp->target);
7
-
8
- for(size_t j = 0; j < dp->used ; j++) {
9
- hashed_key_cleanup(&dp->keys[j]);
10
- txt_cleanup(&dp->values[j]);
11
- }
12
-}
13
-
14
-DUPLICATION *log_job_duplication_add(LOG_JOB *jb, const char *target, size_t target_len) {
15
- if (jb->dups.used >= MAX_KEY_DUPS) {
16
- log2stderr("ERROR: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS);
17
- return NULL;
18
- }
19
-
20
- if(target_len > JOURNAL_MAX_KEY_LEN) {
21
- log2stderr("WARNING: key of duplicate '%.*s' is too long for journals. Will be truncated.", (int)target_len, target);
22
- target_len = JOURNAL_MAX_KEY_LEN;
23
- }
24
-
25
- DUPLICATION *kd = &jb->dups.array[jb->dups.used++];
26
- hashed_key_len_set(&kd->target, target, target_len);
27
- kd->used = 0;
28
- kd->exposed = false;
29
-
30
- // Initialize values array
31
- for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) {
32
- kd->values[i].txt = NULL;
33
- kd->values[i].size = 0;
34
- }
35
-
36
- return kd;
37
-}
38
-
39
-bool log_job_duplication_key_add(DUPLICATION *kd, const char *key, size_t key_len) {
40
- if (kd->used >= MAX_KEY_DUPS_KEYS) {
41
- log2stderr("ERROR: Too many keys in duplication of target '%s'.", kd->target.key);
42
- return false;
43
- }
44
-
45
- hashed_key_len_set(&kd->keys[kd->used++], key, key_len);
46
-
47
- return true;
48
-}
49
-
collectors/log2journal/log2journal-help.c
+29
-37
@@ -88,14 +88,6 @@ void log_job_command_line_help(const char *name) {
88
printf(" unmatched entry will appear as the log message in the journals.\n");
89
printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n");
90
printf("\n");
91
- printf(" --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]\n");
92
- printf(" Create a new key called TARGET, duplicating the values of the keys\n");
93
- printf(" given. Useful for further processing. When multiple keys are given,\n");
94
- printf(" their values are separated by comma.\n");
95
- printf("\n");
96
- printf(" Up to %d duplications can be given on the command line, and up to\n", MAX_KEY_DUPS);
97
- printf(" %d keys per duplication command are allowed.\n", MAX_KEY_DUPS_KEYS);
98
- printf("\n");
91
printf(" --inject LINE\n");
92
printf(" Inject constant fields to the output (both matched and unmatched logs).\n");
93
printf(" --inject entries are added to unmatched lines too, when their key is\n");
@@ -192,40 +184,40 @@ void log_job_command_line_help(const char *name) {
184
printf(" | INPUT | \n");
185
printf(" | read one log line at a time | \n");
186
printf(" +---------------------------------------------------+ \n");
195
- printf(" v v \n");
196
- printf(" +---------------------------------+ | \n");
197
- printf(" | EXTRACT FIELDS AND VALUES | | \n");
198
- printf(" | JSON, logfmt, or pattern based | | \n");
199
- printf(" | (apply optional PREFIX) | | \n");
200
- printf(" +---------------------------------+ | \n");
201
- printf(" v v | \n");
202
- printf(" +---------------+ +--------------+ | \n");
203
- printf(" | DUPLICATE | | FILTER | | \n");
204
- printf(" | | | filter keys | | \n");
205
- printf(" | create new | +--------------+ | \n");
206
- printf(" | fields by | v | \n");
207
- printf(" | duplicating | +--------------+ | \n");
208
- printf(" | other fields | | RENAME | | \n");
209
- printf(" | and their | | change | | \n");
210
- printf(" | values | | field names | | \n");
211
- printf(" +---------------+ +--------------+ | \n");
212
- printf(" v v v \n");
213
- printf(" +---------------------------------+ +--------------+ \n");
214
- printf(" | REWRITE PIPELINES | | INJECT | \n");
215
- printf(" | altering the values of fields | | constants | \n");
216
- printf(" +---------------------------------+ +--------------+ \n");
217
- printf(" v v \n");
187
+ printf(" v v v v v v \n");
188
+ printf(" +---------------------------------------------------+ \n");
189
+ printf(" | EXTRACT FIELDS AND VALUES | \n");
190
+ printf(" | JSON, logfmt, or pattern based | \n");
191
+ printf(" | (apply optional PREFIX) | \n");
192
+ printf(" +---------------------------------------------------+ \n");
193
+ printf(" v v v v v v \n");
194
+ printf(" +---------------------------------------------------+ \n");
195
+ printf(" | RENAME FIELDS | \n");
196
+ printf(" | change the names of the fields | \n");
197
+ printf(" +---------------------------------------------------+ \n");
198
+ printf(" v v v v v v \n");
199
+ printf(" +---------------------------------------------------+ \n");
200
+ printf(" | INJECT NEW FIELDS | \n");
201
+ printf(" | constants, or other field values as variables | \n");
202
+ printf(" +---------------------------------------------------+ \n");
203
+ printf(" v v v v v v \n");
204
+ printf(" +---------------------------------------------------+ \n");
205
+ printf(" | REWRITE FIELD VALUES | \n");
206
+ printf(" | pipeline multiple rewriting rules to alter | \n");
207
+ printf(" | the values of the fields | \n");
208
+ printf(" +---------------------------------------------------+ \n");
209
+ printf(" v v v v v v \n");
210
+ printf(" +---------------------------------------------------+ \n");
211
+ printf(" | FILTER FIELDS | \n");
212
+ printf(" | use include and exclude patterns on the field | \n");
213
+ printf(" | names, to select which fields are sent to journal | \n");
214
+ printf(" +---------------------------------------------------+ \n");
215
+ printf(" v v v v v v \n");
216
printf(" +---------------------------------------------------+ \n");
217
printf(" | OUTPUT | \n");
218
printf(" | generate Journal Export Format | \n");
219
printf(" +---------------------------------------------------+ \n");
220
printf(" \n");
223
- printf("IMPORTANT:\n");
224
- printf(" - Extraction of keys includes formatting them according to journal rules.\n");
225
- printf(" - Duplication rules use the original extracted field names, after they have\n");
226
- printf(" been prefixed (when a PREFIX is set) and before they are renamed.\n");
227
- printf(" - Rewriting is always the last stage, so the final field names are matched.\n");
228
- printf("\n");
221
printf("--------------------------------------------------------------------------------\n");
222
printf("JOURNAL FIELDS RULES (enforced by systemd-journald)\n");
223
printf("\n");
collectors/log2journal/log2journal-inject.c
+13
-8
@@ -4,10 +4,10 @@
4
5
void injection_cleanup(INJECTION *inj) {
6
hashed_key_cleanup(&inj->key);
7
- txt_cleanup(&inj->value);
7
+ replace_pattern_cleanup(&inj->value);
8
}
9
10
-static inline void log_job_injection_replace(INJECTION *inj, const char *key, size_t key_len, const char *value, size_t value_len) {
10
+static inline bool log_job_injection_replace(INJECTION *inj, const char *key, size_t key_len, const char *value, size_t value_len) {
11
if(key_len > JOURNAL_MAX_KEY_LEN)
12
log2stderr("WARNING: injection key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
13
@@ -15,7 +15,11 @@ static inline void log_job_injection_replace(INJECTION *inj, const char *key, si
15
log2stderr("WARNING: injection value of key '%.*s' is too long for journal. Will be truncated.", (int)key_len, key);
16
17
hashed_key_len_set(&inj->key, key, key_len);
18
- txt_replace(&inj->value, value, value_len);
18
+ char *v = strndupz(value, value_len);
19
+ bool ret = replace_pattern_set(&inj->value, v);
20
+ freez(v);
21
+
22
+ return ret;
23
}
24
25
bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {
@@ -32,13 +36,14 @@ bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const c
36
}
37
}
38
39
+ bool ret;
40
if (unmatched) {
36
- log_job_injection_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],
37
- key, key_len, value, value_len);
41
+ ret = log_job_injection_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],
42
+ key, key_len, value, value_len);
43
} else {
39
- log_job_injection_replace(&jb->injections.keys[jb->injections.used++],
40
- key, key_len, value, value_len);
44
+ ret = log_job_injection_replace(&jb->injections.keys[jb->injections.used++],
45
+ key, key_len, value, value_len);
46
}
47
43
- return true;
48
+ return ret;
49
}
collectors/log2journal/log2journal-params.c
+86
-50
@@ -10,7 +10,7 @@ void log_job_init(LOG_JOB *jb) {
10
}
11
12
static void simple_hashtable_cleanup_allocated(SIMPLE_HASHTABLE *ht) {
13
- for(size_t i = 0; i < ht->used ;i++) {
13
+ for(size_t i = 0; i < ht->size ;i++) {
14
HASHED_KEY *k = ht->hashtable[i].data;
15
if(k && k->flags & HK_HASHTABLE_ALLOCATED) {
16
hashed_key_cleanup(k);
@@ -41,12 +41,12 @@ void log_job_cleanup(LOG_JOB *jb) {
41
for(size_t i = 0; i < jb->renames.used ;i++)
42
rename_cleanup(&jb->renames.array[i]);
43
44
- for(size_t i = 0; i < jb->dups.used ;i++)
45
- duplication_cleanup(&jb->dups.array[i]);
46
-
44
for(size_t i = 0; i < jb->rewrites.used; i++)
45
rewrite_cleanup(&jb->rewrites.array[i]);
46
47
+ txt_cleanup(&jb->rewrites.tmp);
48
+ txt_cleanup(&jb->filename.current);
49
+
50
simple_hashtable_cleanup_allocated(&jb->hashtable);
51
simple_hashtable_free(&jb->hashtable);
52
@@ -146,6 +146,79 @@ static bool is_symbol(char c) {
146
return !isalpha(c) && !isdigit(c) && !iscntrl(c);
147
}
148
149
+struct {
150
+ const char *keyword;
151
+ int action;
152
+ RW_FLAGS flag;
153
+} rewrite_flags[] = {
154
+ {"match", 1, RW_MATCH_PCRE2},
155
+ {"match", 0, RW_MATCH_NON_EMPTY},
156
+
157
+ {"regex", 1, RW_MATCH_PCRE2},
158
+ {"regex", 0, RW_MATCH_NON_EMPTY},
159
+
160
+ {"pcre2", 1, RW_MATCH_PCRE2},
161
+ {"pcre2", 0, RW_MATCH_NON_EMPTY},
162
+
163
+ {"non_empty", 1, RW_MATCH_NON_EMPTY},
164
+ {"non_empty", 0, RW_MATCH_PCRE2},
165
+
166
+ {"non-empty", 1, RW_MATCH_NON_EMPTY},
167
+ {"non-empty", 0, RW_MATCH_PCRE2},
168
+
169
+ {"not_empty", 1, RW_MATCH_NON_EMPTY},
170
+ {"not_empty", 0, RW_MATCH_PCRE2},
171
+
172
+ {"not-empty", 1, RW_MATCH_NON_EMPTY},
173
+ {"not-empty", 0, RW_MATCH_PCRE2},
174
+
175
+ {"stop", 0, RW_DONT_STOP},
176
+ {"no-stop", 1, RW_DONT_STOP},
177
+ {"no_stop", 1, RW_DONT_STOP},
178
+ {"dont-stop", 1, RW_DONT_STOP},
179
+ {"dont_stop", 1, RW_DONT_STOP},
180
+ {"continue", 1, RW_DONT_STOP},
181
+ {"inject", 1, RW_INJECT},
182
+ {"existing", 0, RW_INJECT},
183
+};
184
+
185
+RW_FLAGS parse_rewrite_flags(const char *options) {
186
+ RW_FLAGS flags = RW_MATCH_PCRE2; // Default option
187
+
188
+ // Tokenize the input options using ","
189
+ char *token;
190
+ char *optionsCopy = strdup(options); // Make a copy to avoid modifying the original
191
+ token = strtok(optionsCopy, ",");
192
+
193
+ while (token != NULL) {
194
+ // Find the keyword-action mapping
195
+ bool found = false;
196
+
197
+ for (size_t i = 0; i < sizeof(rewrite_flags) / sizeof(rewrite_flags[0]); i++) {
198
+ if (strcmp(token, rewrite_flags[i].keyword) == 0) {
199
+ if (rewrite_flags[i].action == 1) {
200
+ flags |= rewrite_flags[i].flag; // Set the flag
201
+ } else {
202
+ flags &= ~rewrite_flags[i].flag; // Unset the flag
203
+ }
204
+
205
+ found = true;
206
+ }
207
+ }
208
+
209
+ if(!found)
210
+ log2stderr("Warning: rewrite options '%s' is not understood.", token);
211
+
212
+ // Get the next token
213
+ token = strtok(NULL, ",");
214
+ }
215
+
216
+ free(optionsCopy); // Free the copied string
217
+
218
+ return flags;
219
+}
220
+
221
+
222
static bool parse_rewrite(LOG_JOB *jb, const char *param) {
223
// Search for '=' in param
224
const char *equal_sign = strchr(param, '=');
@@ -180,18 +253,20 @@ static bool parse_rewrite(LOG_JOB *jb, const char *param) {
253
return false;
254
}
255
183
- // Reserve a slot in rewrites
184
- if (jb->rewrites.used >= MAX_REWRITES) {
185
- log2stderr("Error: Exceeded maximum number of rewrite rules, while processing: %s", param);
186
- return false;
187
- }
256
+ RW_FLAGS flags = RW_MATCH_PCRE2;
257
+ const char *third_separator = strchr(second_separator + 1, separator);
258
+ if(third_separator)
259
+ flags = parse_rewrite_flags(third_separator + 1);
260
261
// Extract key, search pattern, and replacement pattern
262
char *key = strndupz(param, equal_sign - param);
263
char *search_pattern = strndupz(equal_sign + 2, second_separator - (equal_sign + 2));
192
- char *replace_pattern = strdupz(second_separator + 1);
264
+ char *replace_pattern = third_separator ? strndup(second_separator + 1, third_separator - (second_separator + 1)) : strdupz(second_separator + 1);
265
194
- bool ret = log_job_rewrite_add(jb, key, search_pattern, replace_pattern);
266
+ if(!*search_pattern)
267
+ flags &= ~RW_MATCH_PCRE2;
268
+
269
+ bool ret = log_job_rewrite_add(jb, key, flags, search_pattern, replace_pattern);
270
271
freez(key);
272
freez(search_pattern);
@@ -214,41 +289,6 @@ static bool parse_inject(LOG_JOB *jb, const char *value, bool unmatched) {
289
return true;
290
}
291
217
-static bool parse_duplicate(LOG_JOB *jb, const char *value) {
218
- const char *target = value;
219
- const char *equal_sign = strchr(value, '=');
220
- if (!equal_sign || equal_sign == target) {
221
- log2stderr("Error: Invalid duplicate format, '=' not found or at the start in %s", value);
222
- return false;
223
- }
224
-
225
- size_t target_len = equal_sign - target;
226
- DUPLICATION *kd = log_job_duplication_add(jb, target, target_len);
227
- if(!kd) return false;
228
-
229
- const char *key = equal_sign + 1;
230
- while (key) {
231
- if (kd->used >= MAX_KEY_DUPS_KEYS) {
232
- log2stderr("Error: too many keys in duplication of target '%s'.", kd->target.key);
233
- return false;
234
- }
235
-
236
- const char *comma = strchr(key, ',');
237
- size_t key_len;
238
- if (comma) {
239
- key_len = comma - key;
240
- log_job_duplication_key_add(kd, key, key_len);
241
- key = comma + 1;
242
- }
243
- else {
244
- log_job_duplication_key_add(kd, key, strlen(key));
245
- break; // No more keys
246
- }
247
- }
248
-
249
- return true;
250
-}
251
-
292
bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
293
for (int i = 1; i < argc; i++) {
294
char *arg = argv[i];
@@ -313,10 +353,6 @@ bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv) {
353
#endif
354
else if (strcmp(param, "--unmatched-key") == 0)
355
hashed_key_set(&jb->unmatched.key, value);
316
- else if (strcmp(param, "--duplicate") == 0) {
317
- if (!parse_duplicate(jb, value))
318
- return false;
319
- }
356
else if (strcmp(param, "--inject") == 0) {
357
if (!parse_inject(jb, value, false))
358
return false;
collectors/log2journal/log2journal-replace.c
+7
@@ -100,5 +100,12 @@ bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern) {
100
}
101
}
102
103
+ for(REPLACE_NODE *node = rp->nodes; node; node = node->next) {
104
+ if(node->is_variable) {
105
+ rp->has_variables = true;
106
+ break;
107
+ }
108
+ }
109
+
110
return true;
111
}
collectors/log2journal/log2journal-rewrite.c
+25
-11
@@ -4,33 +4,47 @@
4
5
void rewrite_cleanup(REWRITE *rw) {
6
hashed_key_cleanup(&rw->key);
7
- search_pattern_cleanup(&rw->search);
8
- replace_pattern_cleanup(&rw->replace);
7
+
8
+ if(rw->flags & RW_MATCH_PCRE2)
9
+ search_pattern_cleanup(&rw->match_pcre2);
10
+ else if(rw->flags & RW_MATCH_NON_EMPTY)
11
+ replace_pattern_cleanup(&rw->match_non_empty);
12
+
13
+ replace_pattern_cleanup(&rw->value);
14
+ rw->flags = RW_NONE;
15
}
16
11
-bool log_job_rewrite_add(LOG_JOB *jb, const char *key, const char *search_pattern, const char *replace_pattern) {
17
+bool log_job_rewrite_add(LOG_JOB *jb, const char *key, RW_FLAGS flags, const char *search_pattern, const char *replace_pattern) {
18
if(jb->rewrites.used >= MAX_REWRITES) {
19
log2stderr("Error: too many rewrites. You can add up to %d rewrite rules.", MAX_REWRITES);
20
return false;
21
}
22
23
+ if((flags & (RW_MATCH_PCRE2|RW_MATCH_NON_EMPTY)) && (!search_pattern || !*search_pattern)) {
24
+ log2stderr("Error: rewrite for key '%s' does not specify a search pattern.", key);
25
+ return false;
26
+ }
27
+
28
REWRITE *rw = &jb->rewrites.array[jb->rewrites.used++];
18
- rw->flags = RW_SEARCH_REPLACE | RW_MATCHED_ENTRIES;
29
+ rw->flags = flags;
30
31
hashed_key_set(&rw->key, key);
32
22
- if(!search_pattern_set(&rw->search, search_pattern, strlen(search_pattern)) ||
23
- !replace_pattern_set(&rw->replace, replace_pattern)) {
33
+ if((flags & RW_MATCH_PCRE2) && !search_pattern_set(&rw->match_pcre2, search_pattern, strlen(search_pattern))) {
34
+ rewrite_cleanup(rw);
35
+ jb->rewrites.used--;
36
+ return false;
37
+ }
38
+ else if((flags & RW_MATCH_NON_EMPTY) && !replace_pattern_set(&rw->match_non_empty, search_pattern)) {
39
rewrite_cleanup(rw);
40
jb->rewrites.used--;
41
return false;
42
}
43
29
- for(REPLACE_NODE *node = rw->replace.nodes; node; node = node->next) {
30
- if(node->is_variable) {
31
- rw->flags |= RW_HAS_VARIABLES;
32
- break;
33
- }
44
+ if(replace_pattern && *replace_pattern && !replace_pattern_set(&rw->value, replace_pattern)) {
45
+ rewrite_cleanup(rw);
46
+ jb->rewrites.used--;
47
+ return false;
48
}
49
50
return true;
collectors/log2journal/log2journal-yaml.c
+90
-159
@@ -130,61 +130,6 @@ static bool yaml_scalar_matches_with_trace(yaml_event_t *event, const char *s, s
130
131
// ----------------------------------------------------------------------------
132
133
-static DUPLICATION *yaml_parse_duplicate_key(LOG_JOB *jb, yaml_parser_t *parser) {
134
- yaml_event_t event;
135
-
136
- if (!yaml_parse(parser, &event))
137
- return false;
138
-
139
- DUPLICATION *kd = NULL;
140
- if(event.type == YAML_SCALAR_EVENT) {
141
- kd = log_job_duplication_add(jb, (char *) event.data.scalar.value, event.data.scalar.length);
142
- }
143
- else
144
- yaml_error(parser, &event, "duplicate key must be a scalar.");
145
-
146
- yaml_event_delete(&event);
147
- return kd;
148
-}
149
-
150
-static size_t yaml_parse_duplicate_from(LOG_JOB *jb __maybe_unused, yaml_parser_t *parser, DUPLICATION *kd) {
151
- size_t errors = 0;
152
- yaml_event_t event;
153
-
154
- if (!yaml_parse(parser, &event))
155
- return 1;
156
-
157
- if(event.type == YAML_SCALAR_EVENT) {
158
- if(!log_job_duplication_key_add(kd, (char *) event.data.scalar.value, event.data.scalar.length))
159
- errors++;
160
- }
161
- else if(event.type == YAML_SEQUENCE_START_EVENT) {
162
- bool finished = false;
163
- while(!errors && !finished) {
164
- yaml_event_t sub_event;
165
- if (!yaml_parse(parser, &sub_event))
166
- return ++errors;
167
- else {
168
- if (sub_event.type == YAML_SCALAR_EVENT) {
169
- if(!log_job_duplication_key_add(kd, (char *) sub_event.data.scalar.value
170
- , sub_event.data.scalar.length
171
- ))
172
- errors++;
173
- }
174
- else if (sub_event.type == YAML_SEQUENCE_END_EVENT)
175
- finished = true;
176
-
177
- yaml_event_delete(&sub_event);
178
- }
179
- }
180
- }
181
- else
182
- yaml_error(parser, &event, "not expected event type");
183
-
184
- yaml_event_delete(&event);
185
- return errors;
186
-}
187
-
133
static size_t yaml_parse_filename_injection(yaml_parser_t *parser, LOG_JOB *jb) {
134
yaml_event_t event;
135
size_t errors = 0;
@@ -311,75 +256,6 @@ static size_t yaml_parse_prefix(yaml_parser_t *parser, LOG_JOB *jb) {
256
return errors;
257
}
258
314
-static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, LOG_JOB *jb) {
315
- if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))
316
- return 1;
317
-
318
- DUPLICATION *kd = NULL;
319
-
320
- // Expecting a key-value pair for each duplicate
321
- bool finished;
322
- size_t errors = 0;
323
- while (!errors && !finished) {
324
- yaml_event_t event;
325
- if (!yaml_parse(parser, &event)) {
326
- errors++;
327
- break;
328
- }
329
-
330
- if(event.type == YAML_MAPPING_START_EVENT) {
331
- ;
332
- }
333
- if (event.type == YAML_SEQUENCE_END_EVENT) {
334
- finished = true;
335
- }
336
- else if(event.type == YAML_SCALAR_EVENT) {
337
- if (yaml_scalar_matches(&event, "key", strlen("key"))) {
338
- kd = yaml_parse_duplicate_key(jb, parser);
339
- if (!kd)
340
- errors++;
341
- else {
342
- while (!errors && kd) {
343
- yaml_event_t sub_event;
344
- if (!yaml_parse(parser, &sub_event)) {
345
- errors++;
346
- break;
347
- }
348
-
349
- if (sub_event.type == YAML_MAPPING_END_EVENT) {
350
- kd = NULL;
351
- } else if (sub_event.type == YAML_SCALAR_EVENT) {
352
- if (yaml_scalar_matches(&sub_event, "values_of", strlen("values_of"))) {
353
- if (!kd) {
354
- yaml_error(parser, &sub_event, "Found 'values_of' but the 'key' is not set.");
355
- errors++;
356
- } else
357
- errors += yaml_parse_duplicate_from(jb, parser, kd);
358
- } else {
359
- yaml_error(parser, &sub_event, "unknown scalar");
360
- errors++;
361
- }
362
- } else {
363
- yaml_error(parser, &sub_event, "unexpected event type");
364
- errors++;
365
- }
366
-
367
- // Delete the event after processing
368
- yaml_event_delete(&event);
369
- }
370
- }
371
- } else {
372
- yaml_error(parser, &event, "unknown scalar");
373
- errors++;
374
- }
375
- }
376
-
377
- yaml_event_delete(&event);
378
- }
379
-
380
- return errors;
381
-}
382
-
259
static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, LOG_JOB *jb, bool unmatched) {
260
yaml_event_t event;
261
if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {
@@ -568,7 +444,10 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, LOG_JOB *jb) {
444
switch (event.type) {
445
case YAML_MAPPING_START_EVENT:
446
{
571
- REWRITE rw = { 0 };
447
+ RW_FLAGS flags = RW_NONE;
448
+ char *key = NULL;
449
+ char *search_pattern = NULL;
450
+ char *replace_pattern = NULL;
451
452
bool mapping_finished = false;
453
while (!errors && !mapping_finished) {
@@ -585,23 +464,67 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, LOG_JOB *jb) {
464
yaml_error(parser, &sub_event, "Expected scalar for rewrite key");
465
errors++;
466
} else {
588
- rw.key.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
467
+ key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
468
+ yaml_event_delete(&sub_event);
469
+ }
470
+ } else if (yaml_scalar_matches(&sub_event, "match", strlen("match"))) {
471
+ if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
472
+ yaml_error(parser, &sub_event, "Expected scalar for rewrite match PCRE2 pattern");
473
+ errors++;
474
+ }
475
+ else {
476
+ if(search_pattern)
477
+ freez(search_pattern);
478
+ flags |= RW_MATCH_PCRE2;
479
+ flags &= ~RW_MATCH_NON_EMPTY;
480
+ search_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
481
yaml_event_delete(&sub_event);
482
}
591
- } else if (yaml_scalar_matches(&sub_event, "search", strlen("search"))) {
483
+ } else if (yaml_scalar_matches(&sub_event, "not_empty", strlen("not_empty"))) {
484
if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
593
- yaml_error(parser, &sub_event, "Expected scalar for rewrite search pattern");
485
+ yaml_error(parser, &sub_event, "Expected scalar for rewrite not empty condition");
486
+ errors++;
487
+ }
488
+ else {
489
+ if(search_pattern)
490
+ freez(search_pattern);
491
+ flags |= RW_MATCH_NON_EMPTY;
492
+ flags &= ~RW_MATCH_PCRE2;
493
+ search_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
494
+ yaml_event_delete(&sub_event);
495
+ }
496
+ } else if (yaml_scalar_matches(&sub_event, "value", strlen("value"))) {
497
+ if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
498
+ yaml_error(parser, &sub_event, "Expected scalar for rewrite value");
499
errors++;
500
} else {
596
- rw.search.pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
501
+ replace_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
502
yaml_event_delete(&sub_event);
503
}
599
- } else if (yaml_scalar_matches(&sub_event, "replace", strlen("replace"))) {
504
+ } else if (yaml_scalar_matches(&sub_event, "stop", strlen("stop"))) {
505
if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
601
- yaml_error(parser, &sub_event, "Expected scalar for rewrite replace pattern");
506
+ yaml_error(parser, &sub_event, "Expected scalar for rewrite stop boolean");
507
errors++;
508
} else {
604
- rw.replace.pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);
509
+ if(strncmp((char*)sub_event.data.scalar.value, "no", 2) == 0 ||
510
+ strncmp((char*)sub_event.data.scalar.value, "false", 5) == 0)
511
+ flags |= RW_DONT_STOP;
512
+ else
513
+ flags &= ~RW_DONT_STOP;
514
+
515
+ yaml_event_delete(&sub_event);
516
+ }
517
+ } else if (yaml_scalar_matches(&sub_event, "inject", strlen("inject"))) {
518
+ if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {
519
+ yaml_error(parser, &sub_event, "Expected scalar for rewrite inject boolean");
520
+ errors++;
521
+ } else {
522
+ if(strncmp((char*)sub_event.data.scalar.value, "yes", 3) == 0 ||
523
+ strncmp((char*)sub_event.data.scalar.value, "true", 4) == 0)
524
+ flags |= RW_INJECT;
525
+ else
526
+ flags &= ~RW_INJECT;
527
+
528
yaml_event_delete(&sub_event);
529
}
530
} else {
@@ -611,11 +534,21 @@ static size_t yaml_parse_rewrites(yaml_parser_t *parser, LOG_JOB *jb) {
534
break;
535
536
case YAML_MAPPING_END_EVENT:
614
- if(rw.key.key && rw.search.pattern && rw.replace.pattern) {
615
- if (!log_job_rewrite_add(jb, rw.key.key, rw.search.pattern, rw.replace.pattern))
537
+ if(key) {
538
+ if (!log_job_rewrite_add(jb, key, flags, search_pattern, replace_pattern))
539
errors++;
540
}
618
- rewrite_cleanup(&rw);
541
+
542
+ freez(key);
543
+ key = NULL;
544
+
545
+ freez(search_pattern);
546
+ search_pattern = NULL;
547
+
548
+ freez(replace_pattern);
549
+ replace_pattern = NULL;
550
+
551
+ flags = RW_NONE;
552
553
mapping_finished = true;
554
break;
@@ -803,9 +736,6 @@ static size_t yaml_parse_initialized(yaml_parser_t *parser, LOG_JOB *jb) {
736
else if (yaml_scalar_matches(&event, "filter", strlen("filter")))
737
errors += yaml_parse_filters(parser, jb);
738
806
- else if (yaml_scalar_matches(&event, "duplicate", strlen("duplicate")))
807
- errors += yaml_parse_duplicates_injection(parser, jb);
808
-
739
else if (yaml_scalar_matches(&event, "inject", strlen("inject")))
740
errors += yaml_parse_injections(parser, jb, false);
741
@@ -979,16 +909,13 @@ void log_job_configuration_to_yaml(LOG_JOB *jb) {
909
}
910
}
911
982
- if(jb->dups.used) {
912
+ if(jb->injections.used) {
913
fprintf(stderr, "\n");
984
- yaml_print_node("duplicate", NULL, 0, false);
985
- for(size_t i = 0; i < jb->dups.used ;i++) {
986
- DUPLICATION *kd = &jb->dups.array[i];
987
- yaml_print_node("key", kd->target.key, 1, true);
988
- yaml_print_node("values_of", NULL, 2, false);
989
-
990
- for(size_t k = 0; k < kd->used ;k++)
991
- yaml_print_node(NULL, kd->keys[k].key, 3, true);
914
+ yaml_print_node("inject", NULL, 0, false);
915
+
916
+ for (size_t i = 0; i < jb->injections.used; i++) {
917
+ yaml_print_node("key", jb->injections.keys[i].key.key, 1, true);
918
+ yaml_print_node("value", jb->injections.keys[i].value.pattern, 2, false);
919
}
920
}
921
@@ -997,19 +924,23 @@ void log_job_configuration_to_yaml(LOG_JOB *jb) {
924
yaml_print_node("rewrite", NULL, 0, false);
925
926
for(size_t i = 0; i < jb->rewrites.used ;i++) {
1000
- yaml_print_node("key", jb->rewrites.array[i].key.key, 1, true);
1001
- yaml_print_node("search", jb->rewrites.array[i].search.pattern, 2, false);
1002
- yaml_print_node("replace", jb->rewrites.array[i].replace.pattern, 2, false);
1003
- }
1004
- }
927
+ REWRITE *rw = &jb->rewrites.array[i];
928
1006
- if(jb->injections.used) {
1007
- fprintf(stderr, "\n");
1008
- yaml_print_node("inject", NULL, 0, false);
929
+ yaml_print_node("key", rw->key.key, 1, true);
930
1010
- for (size_t i = 0; i < jb->injections.used; i++) {
1011
- yaml_print_node("key", jb->injections.keys[i].key.key, 1, true);
1012
- yaml_print_node("value", jb->injections.keys[i].value.txt, 2, false);
931
+ if(rw->flags & RW_MATCH_PCRE2)
932
+ yaml_print_node("match", rw->match_pcre2.pattern, 2, false);
933
+
934
+ else if(rw->flags & RW_MATCH_NON_EMPTY)
935
+ yaml_print_node("not_empty", rw->match_non_empty.pattern, 2, false);
936
+
937
+ yaml_print_node("value", rw->value.pattern, 2, false);
938
+
939
+ if(rw->flags & RW_INJECT)
940
+ yaml_print_node("inject", "yes", 2, false);
941
+
942
+ if(rw->flags & RW_DONT_STOP)
943
+ yaml_print_node("stop", "no", 2, false);
944
}
945
}
946
@@ -1026,7 +957,7 @@ void log_job_configuration_to_yaml(LOG_JOB *jb) {
957
958
for (size_t i = 0; i < jb->unmatched.injections.used; i++) {
959
yaml_print_node("key", jb->unmatched.injections.keys[i].key.key, 2, true);
1029
- yaml_print_node("value", jb->unmatched.injections.keys[i].value.txt, 3, false);
960
+ yaml_print_node("value", jb->unmatched.injections.keys[i].value.pattern, 3, false);
961
}
962
}
963
}
collectors/log2journal/log2journal.c
+227
-250
@@ -2,8 +2,6 @@
2
3
#include "log2journal.h"
4
5
-static inline void send_duplications_for_key(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t value_len);
6
-
5
// ----------------------------------------------------------------------------
6
7
const char journal_key_characters_map[256] = {
@@ -63,10 +61,83 @@ const char journal_key_characters_map[256] = {
61
62
// ----------------------------------------------------------------------------
63
64
+// Function to insert a key into the sorted.keys array while keeping it sorted
65
+void log_job_add_key_sorted(LOG_JOB *jb, HASHED_KEY *newKey) {
66
+ size_t i, j;
67
+
68
+ // Find the position to insert the new key based on lexicographic order
69
+ for (i = 0; i < jb->sorted.used; i++) {
70
+ if (strcmp(newKey->key, jb->sorted.keys[i]->key) < 0) {
71
+ break;
72
+ }
73
+ }
74
+
75
+ // Shift elements to the right to make space for the new key
76
+ for (j = jb->sorted.used; j > i; j--) {
77
+ jb->sorted.keys[j] = jb->sorted.keys[j - 1];
78
+ }
79
+
80
+ // Insert the new key at the correct position
81
+ jb->sorted.keys[i] = newKey;
82
+ jb->sorted.used++;
83
+}
84
+
85
+static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
86
+ if(k->flags & HK_HASHTABLE_ALLOCATED)
87
+ return k;
88
+
89
+ if(!k->hashtable_ptr) {
90
+ HASHED_KEY *ht_key;
91
+ SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&jb->hashtable, k->hash, true);
92
+ if(slot->data) {
93
+ ht_key = slot->data;
94
+
95
+ if(!(ht_key->flags & HK_COLLISION_CHECKED)) {
96
+ ht_key->flags |= HK_COLLISION_CHECKED;
97
+
98
+ if(strcmp(ht_key->key, k->key) != 0)
99
+ log2stderr("Hashtable collision detected on key '%s' (hash %lx) and '%s' (hash %lx). "
100
+ "Please file a bug report.", ht_key->key, (unsigned long) ht_key->hash, k->key
101
+ , (unsigned long) k->hash
102
+ );
103
+ }
104
+ }
105
+ else {
106
+ ht_key = callocz(1, sizeof(HASHED_KEY));
107
+ ht_key->key = strdupz(k->key);
108
+ ht_key->len = k->len;
109
+ ht_key->hash = k->hash;
110
+ ht_key->flags = HK_HASHTABLE_ALLOCATED;
111
+
112
+ slot->hash = ht_key->hash;
113
+ slot->data = ht_key;
114
+ jb->hashtable.used++;
115
+
116
+ log_job_add_key_sorted(jb, ht_key);
117
+ }
118
+
119
+ k->hashtable_ptr = ht_key;
120
+ }
121
+
122
+ return k->hashtable_ptr;
123
+}
124
+
125
+static inline HASHED_KEY *get_key_from_hashtable_with_char_ptr(LOG_JOB *jb, const char *key) {
126
+ HASHED_KEY find = {
127
+ .key = key,
128
+ .len = strlen(key),
129
+ };
130
+ find.hash = XXH3_64bits(key, find.len);
131
+
132
+ return get_key_from_hashtable(jb, &find);
133
+}
134
+
135
+// ----------------------------------------------------------------------------
136
+
137
static inline void validate_key(LOG_JOB *jb __maybe_unused, HASHED_KEY *k) {
138
if(k->len > JOURNAL_MAX_KEY_LEN)
139
log2stderr("WARNING: key '%s' has length %zu, which is more than %zu, the max systemd-journal allows",
69
- k->key, k->len, (size_t)JOURNAL_MAX_KEY_LEN);
140
+ k->key, (size_t)k->len, (size_t)JOURNAL_MAX_KEY_LEN);
141
142
for(size_t i = 0; i < k->len ;i++) {
143
char c = k->key[i];
@@ -87,134 +158,106 @@ static inline void validate_key(LOG_JOB *jb __maybe_unused, HASHED_KEY *k) {
158
159
// ----------------------------------------------------------------------------
160
90
-static inline HASHED_KEY *get_key_from_hashtable_for_key(LOG_JOB *jb, HASHED_KEY *find) {
91
- HASHED_KEY *k;
92
- SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&jb->hashtable, find->hash, true);
93
- if(slot->data) {
94
- k = slot->data;
95
-
96
- if(!(k->flags & HK_COLLISION_CHECKED)) {
97
- k->flags |= HK_COLLISION_CHECKED;
98
-
99
- if(strcmp(k->key, find->key) != 0)
100
- log2stderr("Hashtable collision detected on key '%s' (hash %lx) and '%s' (hash %lx). "
101
- "Please file a bug report.",
102
- k->key, (unsigned long)k->hash, find->key, (unsigned long)find->hash);
161
+static inline size_t replace_evaluate_to_buffer(LOG_JOB *jb, HASHED_KEY *k, REPLACE_PATTERN *rp, char *dst, size_t dst_size) {
162
+ size_t remaining = dst_size;
163
+ char *copy_to = dst;
164
+
165
+ for(REPLACE_NODE *node = rp->nodes; node != NULL && remaining > 1; node = node->next) {
166
+ if(node->is_variable) {
167
+ HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
168
+ if(ktmp->value.len) {
169
+ size_t copied = copy_to_buffer(copy_to, remaining, ktmp->value.txt, ktmp->value.len);
170
+ copy_to += copied;
171
+ remaining -= copied;
172
+ }
173
+ }
174
+ else {
175
+ size_t copied = copy_to_buffer(copy_to, remaining, node->name.key, node->name.len);
176
+ copy_to += copied;
177
+ remaining -= copied;
178
}
104
- }
105
- else {
106
- k = mallocz(sizeof(HASHED_KEY));
107
- k->key = strdupz(find->key);
108
- k->len = find->len;
109
- k->hash = find->hash;
110
- k->flags = HK_HASHTABLE_ALLOCATED;
111
-
112
- slot->hash = k->hash;
113
- slot->data = k;
114
- jb->hashtable.used++;
179
}
180
117
- return k;
181
+ return copy_to - dst;
182
}
183
120
-static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, const char *key) {
121
- HASHED_KEY find = {
122
- .key = key,
123
- .len = strlen(key),
124
- };
125
- find.hash = XXH3_64bits(key, find.len);
126
-
127
- return get_key_from_hashtable_for_key(jb, &find);
128
-}
129
-
130
-static inline HASHED_KEY *hashed_key_in_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
131
- if(k->flags & HK_HASHTABLE_ALLOCATED)
132
- return k;
184
+static inline void replace_evaluate(LOG_JOB *jb, HASHED_KEY *k, REPLACE_PATTERN *rp) {
185
+ HASHED_KEY *ht_key = get_key_from_hashtable(jb, k);
186
134
- if(!k->hashtable_ptr)
135
- k->hashtable_ptr = get_key_from_hashtable_for_key(jb, k);
187
+ // set it to empty value
188
+ k->value.len = 0;
189
137
- return k->hashtable_ptr;
190
+ for(REPLACE_NODE *node = rp->nodes; node != NULL; node = node->next) {
191
+ if(node->is_variable) {
192
+ HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
193
+ if(ktmp->value.len)
194
+ txt_expand_and_append(&ht_key->value, ktmp->value.txt, ktmp->value.len);
195
+ }
196
+ else
197
+ txt_expand_and_append(&ht_key->value, node->name.key, node->name.len);
198
+ }
199
}
200
140
-// ----------------------------------------------------------------------------
201
+static inline void replace_evaluate_from_pcre2(LOG_JOB *jb, HASHED_KEY *k, REPLACE_PATTERN *rp, SEARCH_PATTERN *sp) {
202
+ assert(k->flags & HK_HASHTABLE_ALLOCATED);
203
142
-static char *rewrite_value(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t value_len) {
143
- static __thread char rewritten_value[JOURNAL_MAX_VALUE_LEN + 1];
204
+ // set the temporary TEXT to zero length
205
+ jb->rewrites.tmp.len = 0;
206
145
- if(!(k->flags & HK_REWRITES_CHECKED) || k->flags & HK_HAS_REWRITES) {
146
- k->flags |= HK_REWRITES_CHECKED;
207
+ PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(sp->match_data);
208
148
- char *copy_to = rewritten_value;
149
- size_t remaining = sizeof(rewritten_value);
209
+ // Iterate through the linked list of replacement nodes
210
+ for(REPLACE_NODE *node = rp->nodes; node != NULL; node = node->next) {
211
+ if(node->is_variable) {
212
+ int group_number = pcre2_substring_number_from_name(
213
+ sp->re, (PCRE2_SPTR) node->name.key);
214
151
- for(size_t i = 0; i < jb->rewrites.used; i++) {
152
- REWRITE *rw = &jb->rewrites.array[i];
215
+ if(group_number >= 0) {
216
+ PCRE2_SIZE start_offset = ovector[2 * group_number];
217
+ PCRE2_SIZE end_offset = ovector[2 * group_number + 1];
218
+ PCRE2_SIZE length = end_offset - start_offset;
219
154
- if(!hashed_keys_match(&rw->key, k))
155
- continue;
156
-
157
- if(rw->flags & RW_SEARCH_REPLACE) {
158
- if(!search_pattern_matches(&rw->search, value, value_len))
159
- continue; // No match found, skip to next rewrite rule
160
-
161
- PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(rw->search.match_data);
162
-
163
- // Iterate through the linked list of replacement nodes
164
- for(REPLACE_NODE *node = rw->replace.nodes; node != NULL; node = node->next) {
165
- if(node->is_variable) {
166
- int group_number = pcre2_substring_number_from_name(
167
- rw->search.re, (PCRE2_SPTR) node->name.key);
168
-
169
- if(group_number >= 0) {
170
- PCRE2_SIZE start_offset = ovector[2 * group_number];
171
- PCRE2_SIZE end_offset = ovector[2 * group_number + 1];
172
- PCRE2_SIZE length = end_offset - start_offset;
173
-
174
- size_t copied = copy_to_buffer(copy_to, remaining, value + start_offset, length);
175
- copy_to += copied;
176
- remaining -= copied;
177
- }
178
- else {
179
- // TODO: lookup in key names to get their values
180
-
181
- if(!node->logged_error) {
182
- log2stderr("WARNING: variable '${%s}' in rewrite rule of key '%s' cannot be resolved.",
183
- node->name.key, k->key);
184
-
185
- node->logged_error = true;
186
- }
187
- }
188
- }
189
- else {
190
- size_t copied = copy_to_buffer(copy_to, remaining, node->name.key, node->name.len);
191
- copy_to += copied;
192
- remaining -= copied;
193
- }
194
- }
220
+ txt_expand_and_append(&jb->rewrites.tmp, k->value.txt + start_offset, length);
221
}
222
else {
197
- for(REPLACE_NODE *node = rw->replace.nodes; node != NULL; node = node->next) {
198
- if(node->is_variable) {
199
- // TODO: lookup in key names to get their values
200
- ;
201
- }
202
- else {
203
- size_t copied = copy_to_buffer(copy_to, remaining, node->name.key, node->name.len);
204
- copy_to += copied;
205
- remaining -= copied;
206
- }
207
- }
223
+ HASHED_KEY *ktmp = get_key_from_hashtable_with_char_ptr(jb, node->name.key);
224
+ if(ktmp->value.len)
225
+ txt_expand_and_append(&jb->rewrites.tmp, ktmp->value.txt, ktmp->value.len);
226
}
209
-
210
- k->flags |= HK_HAS_REWRITES;
211
- return rewritten_value;
227
+ }
228
+ else {
229
+ txt_expand_and_append(&jb->rewrites.tmp, node->name.key, node->name.len);
230
}
231
}
232
215
- return NULL;
233
+ // swap the values of the temporary TEXT and the key value
234
+ TEXT tmp = k->value;
235
+ k->value = jb->rewrites.tmp;
236
+ jb->rewrites.tmp = tmp;
237
}
238
239
+static inline bool rewrite_conditions_satisfied(LOG_JOB *jb, HASHED_KEY *k, REWRITE *rw) {
240
+ assert(k->flags & HK_HASHTABLE_ALLOCATED);
241
+
242
+ if(rw->flags & RW_MATCH_PCRE2) {
243
+ return search_pattern_matches(&rw->match_pcre2, k->value.txt, k->value.len);
244
+ }
245
+ else if(rw->flags & RW_MATCH_NON_EMPTY) {
246
+ char buffer[2]; // we don't need a big buffer - we just check if anything is written
247
+ if(replace_evaluate_to_buffer(jb, k, &rw->match_non_empty, buffer, sizeof(buffer)))
248
+ // it copied something
249
+ return true;
250
+ else
251
+ // it copied nothing
252
+ return false;
253
+ }
254
+ else
255
+ // no conditions
256
+ return true;
257
+}
258
+
259
+// ----------------------------------------------------------------------------
260
+
261
static inline HASHED_KEY *rename_key(LOG_JOB *jb, HASHED_KEY *k) {
262
if(!(k->flags & HK_RENAMES_CHECKED) || k->flags & HK_HAS_RENAMES) {
263
k->flags |= HK_RENAMES_CHECKED;
@@ -225,7 +268,7 @@ static inline HASHED_KEY *rename_key(LOG_JOB *jb, HASHED_KEY *k) {
268
if(hashed_keys_match(&rn->old_key, k)) {
269
k->flags |= HK_HAS_RENAMES;
270
228
- return hashed_key_in_hashtable(jb, &rn->new_key);
271
+ return get_key_from_hashtable(jb, &rn->new_key);
272
}
273
}
274
}
@@ -235,15 +278,18 @@ static inline HASHED_KEY *rename_key(LOG_JOB *jb, HASHED_KEY *k) {
278
279
// ----------------------------------------------------------------------------
280
238
-static inline void send_key_value_constant(LOG_JOB *jb __maybe_unused, HASHED_KEY *key, const char *value) {
239
- HASHED_KEY *ht_key = hashed_key_in_hashtable(jb, key);
281
+static inline void send_key_value_constant(LOG_JOB *jb __maybe_unused, HASHED_KEY *key, const char *value, size_t len) {
282
+ HASHED_KEY *ht_key = get_key_from_hashtable(jb, key);
283
+
284
+ txt_replace(&ht_key->value, value, len);
285
+ ht_key->flags |= HK_VALUE_FROM_LOG;
286
241
- printf("%s=%s\n", ht_key->key, value);
287
+ // fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt);
288
}
289
290
static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char *format, ...) __attribute__ ((format(__printf__, 3, 4)));
291
static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char *format, ...) {
246
- HASHED_KEY *ht_key = hashed_key_in_hashtable(jb, key);
292
+ HASHED_KEY *ht_key = get_key_from_hashtable(jb, key);
293
294
printf("%s=", ht_key->key);
295
va_list args;
@@ -254,47 +300,78 @@ static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char
300
}
301
302
static inline void send_key_value_and_rewrite(LOG_JOB *jb, HASHED_KEY *key, const char *value, size_t len) {
257
- HASHED_KEY *ht_key = hashed_key_in_hashtable(jb, key);
303
+ HASHED_KEY *ht_key = get_key_from_hashtable(jb, key);
304
259
- if(!(ht_key->flags & HK_KEY_CHECKED)) {
260
- ht_key->flags |= HK_KEY_CHECKED;
261
- validate_key(jb, ht_key);
262
- }
305
+ txt_replace(&ht_key->value, value, len);
306
+ ht_key->flags |= HK_VALUE_FROM_LOG;
307
264
- char *rewritten = rewrite_value(jb, ht_key, value, len);
265
- if(!rewritten)
266
- printf("%s=%.*s\n", ht_key->key, (int)len, value);
267
- else
268
- printf("%s=%s\n", ht_key->key, rewritten);
308
+// fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt);
309
}
310
311
inline void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len) {
272
- HASHED_KEY *ht_key = get_key_from_hashtable(jb, key);
312
+ HASHED_KEY *ht_key = get_key_from_hashtable_with_char_ptr(jb, key);
313
+ HASHED_KEY *nk = rename_key(jb, ht_key);
314
+ txt_replace(&nk->value, value, len);
315
+ ht_key->flags |= HK_VALUE_FROM_LOG;
316
274
- if(!(ht_key->flags & HK_FILTERED)) {
275
- ht_key->flags |= HK_FILTERED;
317
+// fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt);
318
+}
319
277
- bool included = jb->filter.include.re ? search_pattern_matches(&jb->filter.include, ht_key->key, ht_key->len) : true;
278
- bool excluded = jb->filter.exclude.re ? search_pattern_matches(&jb->filter.exclude, ht_key->key, ht_key->len) : false;
320
+static inline void log_job_process_rewrites(LOG_JOB *jb) {
321
+ for(size_t i = 0; i < jb->rewrites.used ;i++) {
322
+ REWRITE *rw = &jb->rewrites.array[i];
323
280
- if(included && !excluded)
281
- ht_key->flags |= HK_FILTERED_INCLUDED;
282
- else
283
- ht_key->flags &= ~HK_FILTERED_INCLUDED;
284
- }
324
+ HASHED_KEY *k = get_key_from_hashtable(jb, &rw->key);
325
286
- if(ht_key->flags & HK_FILTERED_INCLUDED) {
287
- // process renames (changing the key)
288
- HASHED_KEY *nk = rename_key(jb, ht_key);
326
+ if(!(rw->flags & RW_INJECT) && !(k->flags & HK_VALUE_FROM_LOG) && !k->value.len)
327
+ continue;
328
290
- // process rewrites (changing the value)
291
- // and send it to output
292
- send_key_value_and_rewrite(jb, nk, value, len);
329
+ if(!(k->flags & HK_VALUE_REWRITTEN) && rewrite_conditions_satisfied(jb, k, rw)) {
330
+ if(rw->flags & RW_MATCH_PCRE2)
331
+ replace_evaluate_from_pcre2(jb, k, &rw->value, &rw->match_pcre2);
332
+ else
333
+ replace_evaluate(jb, k, &rw->value);
334
+
335
+ if(!(rw->flags & RW_DONT_STOP))
336
+ k->flags |= HK_VALUE_REWRITTEN;
337
+
338
+// fprintf(stderr, "REWRITE %s=%.*s\n", k->key, (int)k->value.len, k->value.txt);
339
+ }
340
}
341
+}
342
295
- // process the duplications (using the original key)
296
- // and send them to output
297
- send_duplications_for_key(jb, ht_key, value, len);
343
+static inline void send_all_fields(LOG_JOB *jb) {
344
+ for(size_t i = 0; i < jb->sorted.used ;i++) {
345
+ HASHED_KEY *k = jb->sorted.keys[i];
346
+
347
+ if(k->value.len) {
348
+ // the key exists and has some value
349
+
350
+ if(!(k->flags & HK_FILTERED)) {
351
+ k->flags |= HK_FILTERED;
352
+
353
+ bool included = jb->filter.include.re ? search_pattern_matches(&jb->filter.include, k->key, k->len) : true;
354
+ bool excluded = jb->filter.exclude.re ? search_pattern_matches(&jb->filter.exclude, k->key, k->len) : false;
355
+
356
+ if(included && !excluded)
357
+ k->flags |= HK_FILTERED_INCLUDED;
358
+ else
359
+ k->flags &= ~HK_FILTERED_INCLUDED;
360
+
361
+ // log some error if the key does not comply to journal standards
362
+ validate_key(jb, k);
363
+ }
364
+
365
+ if(k->flags & HK_FILTERED_INCLUDED)
366
+ printf("%s=%.*s\n", k->key, (int)k->value.len, k->value.txt);
367
+
368
+ // reset it for the next round
369
+ k->value.txt[0] = '\0';
370
+ k->value.len = 0;
371
+ }
372
+
373
+ k->flags &= ~(HK_VALUE_REWRITTEN | HK_VALUE_FROM_LOG);
374
+ }
375
}
376
377
// ----------------------------------------------------------------------------
@@ -328,102 +405,7 @@ static inline void jb_finalize_injections(LOG_JOB *jb, bool line_is_matched) {
405
406
INJECTION *inj = &jb->injections.keys[j];
407
331
- send_key_value_constant(jb, &inj->key, inj->value.txt);
332
- }
333
-}
334
-
335
-static inline void log_job_duplications_reset(LOG_JOB *jb) {
336
- for(size_t d = 0; d < jb->dups.used ; d++) {
337
- DUPLICATION *kd = &jb->dups.array[d];
338
- kd->exposed = false;
339
-
340
- for(size_t g = 0; g < kd->used ; g++) {
341
- if(kd->values[g].txt)
342
- kd->values[g].txt[0] = '\0';
343
- }
344
- }
345
-}
346
-
347
-// ----------------------------------------------------------------------------
348
-// duplications
349
-
350
-static inline void send_duplications_for_key(LOG_JOB *jb, HASHED_KEY *k, const char *value, size_t value_len) {
351
- // IMPORTANT:
352
- // The 'value' may not be NULL terminated and have more data that the value we need
353
-
354
- if(!(k->flags & HK_DUPS_CHECKED) || k->flags & HK_HAS_DUPS) {
355
- k->flags |= HK_DUPS_CHECKED;
356
-
357
- for(size_t d = 0; d < jb->dups.used; d++) {
358
- DUPLICATION *kd = &jb->dups.array[d];
359
-
360
- if(kd->exposed || kd->used == 0)
361
- continue;
362
-
363
- if(kd->used == 1) {
364
- // just one key to be duplicated
365
- if(hashed_keys_match(&kd->keys[0], k)) {
366
- k->flags |= HK_HAS_DUPS;
367
-
368
- send_key_value_and_rewrite(jb, &kd->target, value, value_len);
369
- kd->exposed = true;
370
- }
371
- }
372
- else {
373
- // multiple keys to be duplicated
374
- for(size_t g = 0; g < kd->used; g++) {
375
- if(hashed_keys_match(&kd->keys[g], k)) {
376
- k->flags |= HK_HAS_DUPS;
377
- txt_replace(&kd->values[g], value, value_len);
378
- }
379
- }
380
- }
381
- }
382
- }
383
-}
384
-
385
-static inline void jb_send_remaining_duplications(LOG_JOB *jb) {
386
- static __thread char buffer[JOURNAL_MAX_VALUE_LEN + 1];
387
-
388
- // IMPORTANT:
389
- // all duplications are exposed, even the ones we haven't found their keys in the source,
390
- // so that the output always has the same fields for matched entries.
391
-
392
- for(size_t d = 0; d < jb->dups.used ; d++) {
393
- DUPLICATION *kd = &jb->dups.array[d];
394
-
395
- if(kd->exposed || kd->used == 0)
396
- continue;
397
-
398
- buffer[0] = '\0';
399
- size_t remaining = sizeof(buffer);
400
- char *s = buffer;
401
-
402
- for(size_t g = 0; g < kd->used ; g++) {
403
- if(remaining < 2) {
404
- log2stderr("Warning: duplicated key '%s' cannot fit the values.", kd->target.key);
405
- break;
406
- }
407
-
408
- if(g > 0) {
409
- *s++ = ',';
410
- *s = '\0';
411
- remaining--;
412
- }
413
-
414
- char *value = (kd->values[g].txt && kd->values[g].txt[0]) ? kd->values[g].txt : "[unavailable]";
415
- size_t len = strlen(value);
416
- size_t copied = copy_to_buffer(s, remaining, value, len);
417
- remaining -= copied;
418
- s += copied;
419
-
420
- if(copied != len) {
421
- log2stderr("Warning: duplicated key '%s' will have truncated value", jb->dups.array[d].target.key);
422
- break;
423
- }
424
- }
425
-
426
- send_key_value_and_rewrite(jb, &kd->target, buffer, s - buffer);
408
+ replace_evaluate(jb, &inj->key, &inj->value);
409
}
410
}
411
@@ -431,8 +413,8 @@ static inline void jb_send_remaining_duplications(LOG_JOB *jb) {
413
// filename injection
414
415
static inline void jb_inject_filename(LOG_JOB *jb) {
434
- if (jb->filename.key.key && jb->filename.current[0])
435
- send_key_value_constant(jb, &jb->filename.key, jb->filename.current);
416
+ if (jb->filename.key.key && jb->filename.current.len)
417
+ send_key_value_constant(jb, &jb->filename.key, jb->filename.current.txt, jb->filename.current.len);
418
}
419
420
static inline bool jb_switched_filename(LOG_JOB *jb, const char *line, size_t len) {
@@ -454,8 +436,7 @@ static inline bool jb_switched_filename(LOG_JOB *jb, const char *line, size_t le
436
const char *end = strstr(line, " <==");
437
while (*start == ' ') start++;
438
if (*start != '\n' && *start != '\0' && end) {
457
- copy_to_buffer(jb->filename.current, sizeof(jb->filename.current),
458
- start, end - start);
439
+ txt_replace(&jb->filename.current, start, end - start);
440
return true;
441
}
442
}
@@ -474,7 +455,7 @@ static inline bool jb_send_unmatched_line(LOG_JOB *jb, const char *line) {
455
for (size_t j = 0; j < jb->unmatched.injections.used; j++) {
456
INJECTION *inj = &jb->unmatched.injections.keys[j];
457
477
- send_key_value_constant(jb, &inj->key, inj->value.txt);
458
+ replace_evaluate(jb, &inj->key, &inj->value);
459
}
460
461
return true;
@@ -536,8 +517,6 @@ int log_job_run(LOG_JOB *jb) {
517
if(jb_switched_filename(jb, line, len))
518
continue;
519
539
- log_job_duplications_reset(jb);
540
-
520
bool line_is_matched;
521
522
if(json)
@@ -559,14 +538,12 @@ int log_job_run(LOG_JOB *jb) {
538
// just logging to stderr, not sending unmatched lines
539
continue;
540
}
562
- else {
563
- // print all non-exposed duplications
564
- jb_send_remaining_duplications(jb);
565
- }
541
542
jb_inject_filename(jb);
543
jb_finalize_injections(jb, line_is_matched);
544
545
+ log_job_process_rewrites(jb);
546
+ send_all_fields(jb);
547
printf("\n");
548
fflush(stdout);
549
}
collectors/log2journal/log2journal.d/nginx-combined.yaml
+21
-27
@@ -34,55 +34,49 @@ rename:
34
- new_key: MESSAGE
35
old_key: NGINX_REQUEST
36
37
-# Duplicate fields under a different name. You can duplicate multiple fields
38
-# to a new one and then use rewrite rules to change its value.
39
-duplicate:
40
-
41
- # we insert the field PRIORITY as a copy of NGINX_STATUS.
42
- - key: PRIORITY
43
- values_of:
44
- - NGINX_STATUS
45
-
46
- # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.
47
- - key: NGINX_STATUS_FAMILY
48
- values_of:
49
- - NGINX_STATUS
50
-
37
# Inject constant fields into the journal logs.
38
inject:
39
- key: SYSLOG_IDENTIFIER
40
value: "nginx-log"
41
42
+ # inject PRIORITY is a duplicate of NGINX_STATUS
43
+ - key: "PRIORITY"
44
+ value: "${NGINX_STATUS}"
45
+
46
+ # Inject NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
47
+ - key: "NGINX_STATUS_FAMILY"
48
+ value: "${NGINX_STATUS}"
49
+
50
# Rewrite the value of fields (including the duplicated ones).
51
# The search pattern can have named groups, and the replace pattern can use
52
# them as ${name}.
53
rewrite:
60
- # PRIORTY is a duplicate of NGINX_STATUS
54
+ # PRIORITY is a duplicate of NGINX_STATUS
55
# Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
56
- key: "PRIORITY"
63
- search: "^[123]"
64
- replace: 6
57
+ match: "^[123]"
58
+ value: 6
59
60
- key: "PRIORITY"
67
- search: "^4"
68
- replace: 5
61
+ match: "^4"
62
+ value: 5
63
64
- key: "PRIORITY"
71
- search: "^5"
72
- replace: 3
65
+ match: "^5"
66
+ value: 3
67
68
- key: "PRIORITY"
75
- search: ".*"
76
- replace: 4
69
+ match: ".*"
70
+ value: 4
71
72
# NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
73
- key: "NGINX_STATUS_FAMILY"
80
- search: "^(?<first_digit>[1-5])"
81
- replace: "${first_digit}xx"
74
+ match: "^(?<first_digit>[1-5])"
75
+ value: "${first_digit}xx"
76
77
- key: "NGINX_STATUS_FAMILY"
84
- search: ".*"
85
- replace: "UNKNOWN"
78
+ match: ".*"
79
+ value: "UNKNOWN"
80
81
# Control what to do when input logs do not match the main PCRE2 pattern.
82
unmatched:
collectors/log2journal/log2journal.d/nginx-json.yaml
+44
-49
@@ -12,7 +12,7 @@ filename:
12
key: NGINX_LOG_FILENAME
13
14
filter:
15
- exclude: "^NGINX_BINARY_REMOTE_ADDR$"
15
+ exclude: "NGINX_BINARY_REMOTE_ADDR"
16
17
rename:
18
- new_key: MESSAGE
@@ -66,19 +66,19 @@ rename:
66
- new_key: NGINX_PROXY_PROTOCOL_TLV_SSL_KEY_ALG
67
old_key: NGINX_PROXY_PROTOCOL_TLV_SSL_0X25
68
69
-# Duplicate fields under a different name. You can duplicate multiple fields
70
-# to a new one and then use rewrite rules to change its value.
71
-duplicate:
69
+# Inject constant fields into the journal logs.
70
+inject:
71
+ - key: SYSLOG_IDENTIFIER
72
+ value: "nginx-log"
73
+
74
+ # inject PRIORITY is a duplicate of NGINX_STATUS
75
+ - key: "PRIORITY"
76
+ value: "${NGINX_STATUS}"
77
73
- # we insert the field PRIORITY as a copy of NGINX_STATUS.
74
- - key: PRIORITY
75
- values_of:
76
- - NGINX_STATUS
78
+ # Inject NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
79
+ - key: "NGINX_STATUS_FAMILY"
80
+ value: "${NGINX_STATUS}"
81
78
- # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.
79
- - key: NGINX_STATUS_FAMILY
80
- values_of:
81
- - NGINX_STATUS
82
83
# Rewrite the value of fields (including the duplicated ones).
84
# The search pattern can have named groups, and the replace pattern can use
@@ -86,75 +86,70 @@ duplicate:
86
rewrite:
87
# a ? means it has query string, everything else means it does not
88
- key: NGINX_HAS_QUERY_STRING
89
- search: '^\?$'
90
- replace: "yes"
89
+ match: '^\?$'
90
+ value: "yes"
91
- key: NGINX_HAS_QUERY_STRING
92
- search: ".*"
93
- replace: "no"
92
+ match: ".*"
93
+ value: "no"
94
95
# 'on' means it was HTTPS, everything else means it was not
96
- key: NGINX_HTTPS
97
- search: "^on$"
98
- replace: "yes"
97
+ match: "^on$"
98
+ value: "yes"
99
- key: NGINX_HTTPS
100
- search: ".*"
101
- replace: "no"
100
+ match: ".*"
101
+ value: "no"
102
103
# 'p' means it was pipelined, everything else means it was not
104
- key: NGINX_PIPELINED
105
- search: "^p$"
106
- replace: "yes"
105
+ match: "^p$"
106
+ value: "yes"
107
- key: NGINX_PIPELINED
108
- search: ".*"
109
- replace: "no"
108
+ match: ".*"
109
+ value: "no"
110
111
# zero means client sent a certificate and it was verified, non-zero means otherwise
112
- key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERIFY
113
- search: "^0$"
114
- replace: "yes"
113
+ match: "^0$"
114
+ value: "yes"
115
- key: NGINX_PROXY_PROTOCOL_TLV_SSL_VERIFY
116
- search: ".*"
117
- replace: "no"
116
+ match: ".*"
117
+ value: "no"
118
119
# 'OK' means request completed, everything else means it didn't
120
- key: NGINX_REQUEST_COMPLETION
121
- search: "^OK$"
122
- replace: "completed"
121
+ match: "^OK$"
122
+ value: "completed"
123
- key: NGINX_REQUEST_COMPLETION
124
- search: ".*"
125
- replace: "not completed"
124
+ match: ".*"
125
+ value: "not completed"
126
127
# PRIORTY is a duplicate of NGINX_STATUS
128
# Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug
129
- key: "PRIORITY"
130
- search: "^[123]"
131
- replace: 6
130
+ match: "^[123]"
131
+ value: 6
132
133
- key: "PRIORITY"
134
- search: "^4"
135
- replace: 5
134
+ match: "^4"
135
+ value: 5
136
137
- key: "PRIORITY"
138
- search: "^5"
139
- replace: 3
138
+ match: "^5"
139
+ value: 3
140
141
- key: "PRIORITY"
142
- search: ".*"
143
- replace: 4
142
+ match: ".*"
143
+ value: 4
144
145
# NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS
146
- key: "NGINX_STATUS_FAMILY"
147
- search: "^(?<first_digit>[1-5])"
148
- replace: "${first_digit}xx"
147
+ match: "^(?<first_digit>[1-5])"
148
+ value: "${first_digit}xx"
149
150
- key: "NGINX_STATUS_FAMILY"
151
- search: ".*"
152
- replace: "UNKNOWN"
153
-
154
-# Inject constant fields into the journal logs.
155
-inject:
156
- - key: SYSLOG_IDENTIFIER
157
- value: "nginx-log"
151
+ match: ".*"
152
+ value: "UNKNOWN"
153
154
# Control what to do when input logs do not match the main PCRE2 pattern.
155
unmatched:
collectors/log2journal/log2journal.h
+69
-42
@@ -132,6 +132,7 @@ static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src,
132
typedef struct txt {
133
char *txt;
134
uint32_t size;
135
+ uint32_t len;
136
} TEXT;
137
138
static inline void txt_cleanup(TEXT *t) {
@@ -143,6 +144,7 @@ static inline void txt_cleanup(TEXT *t) {
144
145
t->txt = NULL;
146
t->size = 0;
147
+ t->len = 0;
148
}
149
150
static inline void txt_replace(TEXT *t, const char *s, size_t len) {
@@ -156,6 +158,7 @@ static inline void txt_replace(TEXT *t, const char *s, size_t len) {
158
159
memcpy(t->txt, s, len);
160
t->txt[len] = '\0';
161
+ t->len = len;
162
}
163
else {
164
// no existing value allocation, or too small for our value
@@ -165,24 +168,59 @@ static inline void txt_replace(TEXT *t, const char *s, size_t len) {
168
169
t->txt = strndupz(s, len);
170
t->size = len + 1;
171
+ t->len = len;
172
}
173
}
174
175
+static inline void txt_expand_and_append(TEXT *t, const char *s, size_t len) {
176
+ if(len + 1 > (t->size - t->len)) {
177
+ size_t new_size = t->len + len + 1;
178
+ if(new_size < t->size * 2)
179
+ new_size = t->size * 2;
180
+
181
+ char *b = mallocz(new_size);
182
+ if(t->txt) {
183
+ memcpy(b, t->txt, t->len);
184
+ freez(t->txt);
185
+ }
186
+
187
+ t->txt = b;
188
+ t->size = new_size;
189
+ }
190
+
191
+ char *copy_to = &t->txt[t->len];
192
+ memcpy(copy_to, s, len);
193
+ copy_to[len] = '\0';
194
+ t->len += len;
195
+}
196
+
197
// ----------------------------------------------------------------------------
198
199
typedef enum __attribute__((__packed__)) {
200
HK_NONE = 0,
175
- HK_HASHTABLE_ALLOCATED = (1 << 0),
176
- HK_FILTERED = (1 << 1),
177
- HK_FILTERED_INCLUDED = (1 << 2),
178
- HK_COLLISION_CHECKED = (1 << 3),
179
- HK_RENAMES_CHECKED = (1 << 4),
180
- HK_HAS_RENAMES = (1 << 5),
181
- HK_DUPS_CHECKED = (1 << 6),
182
- HK_HAS_DUPS = (1 << 7),
183
- HK_REWRITES_CHECKED = (1 << 8),
184
- HK_HAS_REWRITES = (1 << 9),
185
- HK_KEY_CHECKED = (1 << 10),
201
+
202
+ // permanent flags - they are set once to optimize various decisions and lookups
203
+
204
+ HK_HASHTABLE_ALLOCATED = (1 << 0), // this is key object allocated in the hashtable
205
+ // objects that do not have this, have a pointer to a key in the hashtable
206
+ // objects that have this, value a value allocated
207
+
208
+ HK_FILTERED = (1 << 1), // we checked once if this key in filtered
209
+ HK_FILTERED_INCLUDED = (1 << 2), // the result of the filtering was to include it in the output
210
+
211
+ HK_COLLISION_CHECKED = (1 << 3), // we checked once for collision check of this key
212
+
213
+ HK_RENAMES_CHECKED = (1 << 4), // we checked once if there are renames on this key
214
+ HK_HAS_RENAMES = (1 << 5), // and we found there is a rename rule related to it
215
+
216
+ HK_DUPS_CHECKED = (1 << 6), // we checked once if there are duplications for this key
217
+ HK_HAS_DUPS = (1 << 7), // and we found there are duplication related to it
218
+
219
+ // ephemeral flags - they are unset at the end of each log line
220
+
221
+ HK_VALUE_FROM_LOG = (1 << 14), // the value of this key has been read from the log (or from injection, duplication)
222
+ HK_VALUE_REWRITTEN = (1 << 15), // the value of this key has been rewritten due to one of our rewrite rules
223
+
224
} HASHED_KEY_FLAGS;
225
226
typedef struct hashed_key {
@@ -261,6 +299,7 @@ void replace_node_free(REPLACE_NODE *rpn);
299
typedef struct replace_pattern {
300
const char *pattern;
301
REPLACE_NODE *nodes;
302
+ bool has_variables;
303
} REPLACE_PATTERN;
304
305
void replace_pattern_cleanup(REPLACE_PATTERN *rp);
@@ -270,26 +309,14 @@ bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern);
309
310
typedef struct injection {
311
bool on_unmatched;
273
- TEXT value;
312
HASHED_KEY key;
313
+ REPLACE_PATTERN value;
314
} INJECTION;
315
316
void injection_cleanup(INJECTION *inj);
317
318
// ----------------------------------------------------------------------------
319
281
-typedef struct duplication {
282
- HASHED_KEY target;
283
- uint32_t used;
284
- bool exposed;
285
- HASHED_KEY keys[MAX_KEY_DUPS_KEYS];
286
- TEXT values[MAX_KEY_DUPS_KEYS];
287
-} DUPLICATION;
288
-
289
-void duplication_cleanup(DUPLICATION *dp);
290
-
291
-// ----------------------------------------------------------------------------
292
-
320
typedef struct key_rename {
321
HASHED_KEY new_key;
322
HASHED_KEY old_key;
@@ -300,20 +327,21 @@ void rename_cleanup(RENAME *rn);
327
// ----------------------------------------------------------------------------
328
329
typedef enum __attribute__((__packed__)) {
303
-// RW_NONE = 0,
304
- RW_SEARCH_REPLACE = (1 << 0), // a rewrite rule
305
- RW_MATCHED_ENTRIES = (1 << 1), // an injection on matched log entry
306
-// RW_UNMATCHED_ENTRIES = (1 << 2), // an injection on unmatched log entry
307
-// RW_INJECT_ALWAYS = (1 << 3), // an injection: inject always
308
-// RW_INJECT_IF_SATISFIED = (1 << 4), // a duplication: inject only if the variables are resolved
309
- RW_HAS_VARIABLES = (1 << 5), // the replacement has variables in it
330
+ RW_NONE = 0,
331
+ RW_MATCH_PCRE2 = (1 << 1), // a rewrite rule
332
+ RW_MATCH_NON_EMPTY = (1 << 2), // a rewrite rule
333
+ RW_DONT_STOP = (1 << 3),
334
+ RW_INJECT = (1 << 4),
335
} RW_FLAGS;
336
337
typedef struct key_rewrite {
338
RW_FLAGS flags;
339
HASHED_KEY key;
315
- SEARCH_PATTERN search;
316
- REPLACE_PATTERN replace;
340
+ union {
341
+ SEARCH_PATTERN match_pcre2;
342
+ REPLACE_PATTERN match_non_empty;
343
+ };
344
+ REPLACE_PATTERN value;
345
} REWRITE;
346
347
void rewrite_cleanup(REWRITE *rw);
@@ -329,6 +357,11 @@ typedef struct log_job {
357
358
SIMPLE_HASHTABLE hashtable;
359
360
+ struct {
361
+ HASHED_KEY *keys[MAX_OUTPUT_KEYS];
362
+ size_t used;
363
+ } sorted;
364
+
365
struct {
366
SEARCH_PATTERN include;
367
SEARCH_PATTERN exclude;
@@ -337,7 +370,7 @@ typedef struct log_job {
370
struct {
371
bool last_line_was_empty;
372
HASHED_KEY key;
340
- char current[FILENAME_MAX + 1];
373
+ TEXT current;
374
} filename;
375
376
struct {
@@ -353,14 +386,10 @@ typedef struct log_job {
386
} injections;
387
} unmatched;
388
356
- struct {
357
- uint32_t used;
358
- DUPLICATION array[MAX_KEY_DUPS];
359
- } dups;
360
-
389
struct {
390
uint32_t used;
391
REWRITE array[MAX_REWRITES];
392
+ TEXT tmp;
393
} rewrites;
394
395
struct {
@@ -385,13 +414,11 @@ void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *
414
// configuration related
415
416
// management of configuration to set settings
388
-DUPLICATION *log_job_duplication_add(LOG_JOB *jb, const char *target, size_t target_len);
389
-bool log_job_duplication_key_add(DUPLICATION *kd, const char *key, size_t key_len);
417
bool log_job_filename_key_set(LOG_JOB *jb, const char *key, size_t key_len);
418
bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len);
419
bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
420
bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched);
394
-bool log_job_rewrite_add(LOG_JOB *jb, const char *key, const char *search_pattern, const char *replace_pattern);
421
+bool log_job_rewrite_add(LOG_JOB *jb, const char *key, RW_FLAGS flags, const char *search_pattern, const char *replace_pattern);
422
bool log_job_rename_add(LOG_JOB *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len);
423
bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
424
bool log_job_exclude_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
collectors/log2journal/tests.d/full.output
+23
-24
@@ -33,40 +33,39 @@ rename:
33
- new_key: TEST3
34
old_key: TEST4
35
36
-duplicate:
36
+inject:
37
+ - key: SYSLOG_IDENTIFIER
38
+ value: nginx-log
39
+ - key: SYSLOG_IDENTIFIER2
40
+ value: nginx-log2
41
- key: PRIORITY
38
- values_of:
39
- - NGINX_STATUS
42
+ value: "${NGINX_STATUS}"
43
- key: NGINX_STATUS_FAMILY
41
- values_of:
42
- - NGINX_STATUS
43
- - NGINX_METHOD
44
+ value: "${NGINX_STATUS}${NGINX_METHOD}"
45
46
rewrite:
47
- key: PRIORITY
47
- search: "^[123]"
48
- replace: 6
48
+ value: "${NGINX_STATUS}"
49
+ inject: yes
50
+ stop: no
51
+ - key: PRIORITY
52
+ match: "^[123]"
53
+ value: 6
54
- key: PRIORITY
50
- search: "^4"
51
- replace: 5
55
+ match: "^4"
56
+ value: 5
57
- key: PRIORITY
53
- search: "^5"
54
- replace: 3
58
+ match: "^5"
59
+ value: 3
60
- key: PRIORITY
56
- search: ".*"
57
- replace: 4
61
+ match: ".*"
62
+ value: 4
63
- key: NGINX_STATUS_FAMILY
59
- search: "^(?<first_digit>[1-5])"
60
- replace: "${first_digit}xx"
64
+ match: "^(?<first_digit>[1-5])"
65
+ value: "${first_digit}xx"
66
- key: NGINX_STATUS_FAMILY
62
- search: ".*"
63
- replace: UNKNOWN
64
-
65
-inject:
66
- - key: SYSLOG_IDENTIFIER
67
- value: nginx-log
68
- - key: SYSLOG_IDENTIFIER2
69
- value: nginx-log2
67
+ match: ".*"
68
+ value: UNKNOWN
69
70
unmatched:
71
key: MESSAGE
collectors/log2journal/tests.d/full.yaml
+23
-24
@@ -33,40 +33,39 @@ rename:
33
- new_key: TEST3
34
old_key: TEST4
35
36
-duplicate:
36
+inject:
37
+ - key: SYSLOG_IDENTIFIER
38
+ value: "nginx-log"
39
+ - key: SYSLOG_IDENTIFIER2
40
+ value: "nginx-log2"
41
- key: PRIORITY
38
- values_of:
39
- - NGINX_STATUS
42
+ value: "${NGINX_STATUS}"
43
- key: NGINX_STATUS_FAMILY
41
- values_of:
42
- - NGINX_STATUS
43
- - NGINX_METHOD
44
+ value: "${NGINX_STATUS}${NGINX_METHOD}"
45
46
rewrite:
47
- key: "PRIORITY"
47
- search: "^[123]"
48
- replace: 6
48
+ value: "${NGINX_STATUS}"
49
+ inject: yes
50
+ stop: no
51
+ - key: "PRIORITY"
52
+ match: "^[123]"
53
+ value: 6
54
- key: "PRIORITY"
50
- search: "^4"
51
- replace: 5
55
+ match: "^4"
56
+ value: 5
57
- key: "PRIORITY"
53
- search: "^5"
54
- replace: 3
58
+ match: "^5"
59
+ value: 3
60
- key: "PRIORITY"
56
- search: ".*"
57
- replace: 4
61
+ match: ".*"
62
+ value: 4
63
- key: "NGINX_STATUS_FAMILY"
59
- search: "^(?<first_digit>[1-5])"
60
- replace: "${first_digit}xx"
64
+ match: "^(?<first_digit>[1-5])"
65
+ value: "${first_digit}xx"
66
- key: "NGINX_STATUS_FAMILY"
62
- search: ".*"
63
- replace: "UNKNOWN"
64
-
65
-inject:
66
- - key: SYSLOG_IDENTIFIER
67
- value: "nginx-log"
68
- - key: SYSLOG_IDENTIFIER2
69
- value: "nginx-log2"
67
+ match: ".*"
68
+ value: "UNKNOWN"
69
70
unmatched:
71
key: MESSAGE
collectors/log2journal/tests.d/json-exclude.output
+98
-98
@@ -1,153 +1,153 @@
1
-NUMERICPOSITIVE=42
2
-NUMERICNEGATIVE=-123
3
-FLOATPOSITIVE=3.14159
4
-FLOATNEGATIVE=-2.71828
5
-SCIENTIFICINTPOSITIVE=1e5
6
-SCIENTIFICFLOATNEGATIVE=-2.5e-3
7
-SCIENTIFICSMALLPOSITIVE=1e-4
8
-BOOLEANTRUE=true
9
-BOOLEANFALSE=false
10
-STRING=Hello, World!
11
-NULLVALUE=null
12
-OBJECT_NUMERICPOSITIVE=123
13
-OBJECT_NUMERICNEGATIVE=-456
14
-OBJECT_FLOATPOSITIVE=0.987
15
-OBJECT_FLOATNEGATIVE=-0.123
16
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
17
-OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
18
-OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
19
-OBJECT_BOOLEANTRUE=true
20
-OBJECT_BOOLEANFALSE=false
21
-OBJECT_STRING=Nested Object
22
-OBJECT_NULLVALUE=null
1
ARRAY2_0=1
2
ARRAY2_1=-2.345
3
ARRAY2_2=Array Element
4
ARRAY2_3=true
5
ARRAY2_4=false
6
ARRAY2_5=null
29
-ARRAY2_6_NUMERICPOSITIVE=123
30
-ARRAY2_6_NUMERICNEGATIVE=-456
31
-ARRAY2_6_FLOATPOSITIVE=0.987
7
+ARRAY2_6_BOOLEANFALSE=false
8
+ARRAY2_6_BOOLEANTRUE=true
9
ARRAY2_6_FLOATNEGATIVE=-0.123
33
-ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
10
+ARRAY2_6_FLOATPOSITIVE=0.987
11
+ARRAY2_6_NULLVALUE=null
12
+ARRAY2_6_NUMERICNEGATIVE=-456
13
+ARRAY2_6_NUMERICPOSITIVE=123
14
ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
15
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
16
ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
36
-ARRAY2_6_BOOLEANTRUE=true
37
-ARRAY2_6_BOOLEANFALSE=false
17
ARRAY2_6_STRING=Nested Object in Array2
39
-ARRAY2_6_NULLVALUE=null
40
-ARRAY2_7_NUMERICPOSITIVE=42
41
-ARRAY2_7_NUMERICNEGATIVE=-123
42
-ARRAY2_7_FLOATPOSITIVE=3.14159
18
+ARRAY2_7_BOOLEANFALSE=false
19
+ARRAY2_7_BOOLEANTRUE=true
20
ARRAY2_7_FLOATNEGATIVE=-2.71828
44
-ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
21
+ARRAY2_7_FLOATPOSITIVE=3.14159
22
+ARRAY2_7_NULLVALUE=null
23
+ARRAY2_7_NUMERICNEGATIVE=-123
24
+ARRAY2_7_NUMERICPOSITIVE=42
25
ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
26
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
27
ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
47
-ARRAY2_7_BOOLEANTRUE=true
48
-ARRAY2_7_BOOLEANFALSE=false
28
ARRAY2_7_STRING=Array Element with Object in Array2
50
-ARRAY2_7_NULLVALUE=null
51
-
52
-NUMERICPOSITIVE=42
53
-NUMERICNEGATIVE=-123
54
-FLOATPOSITIVE=3.14159
55
-FLOATNEGATIVE=-2.71828
56
-SCIENTIFICINTPOSITIVE=1e5
57
-SCIENTIFICFLOATNEGATIVE=-2.5e-3
58
-SCIENTIFICSMALLPOSITIVE=1e-4
59
-BOOLEANTRUE=true
29
BOOLEANFALSE=false
61
-STRING=Hello, World!
30
+BOOLEANTRUE=true
31
+FLOATNEGATIVE=-2.71828
32
+FLOATPOSITIVE=3.14159
33
NULLVALUE=null
63
-OBJECT_NUMERICPOSITIVE=123
64
-OBJECT_NUMERICNEGATIVE=-456
65
-OBJECT_FLOATPOSITIVE=0.987
34
+NUMERICNEGATIVE=-123
35
+NUMERICPOSITIVE=42
36
+OBJECT_BOOLEANFALSE=false
37
+OBJECT_BOOLEANTRUE=true
38
OBJECT_FLOATNEGATIVE=-0.123
67
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
39
+OBJECT_FLOATPOSITIVE=0.987
40
+OBJECT_NULLVALUE=null
41
+OBJECT_NUMERICNEGATIVE=-456
42
+OBJECT_NUMERICPOSITIVE=123
43
OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
44
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
45
OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
70
-OBJECT_BOOLEANTRUE=true
71
-OBJECT_BOOLEANFALSE=false
46
OBJECT_STRING=Nested Object
73
-OBJECT_NULLVALUE=null
47
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
48
+SCIENTIFICINTPOSITIVE=1e5
49
+SCIENTIFICSMALLPOSITIVE=1e-4
50
+STRING=Hello, World!
51
+
52
ARRAY2_0=1
53
ARRAY2_1=-2.345
54
ARRAY2_2=Array Element
55
ARRAY2_3=true
56
ARRAY2_4=false
57
ARRAY2_5=null
80
-ARRAY2_6_NUMERICPOSITIVE=123
81
-ARRAY2_6_NUMERICNEGATIVE=-456
82
-ARRAY2_6_FLOATPOSITIVE=0.987
58
+ARRAY2_6_BOOLEANFALSE=false
59
+ARRAY2_6_BOOLEANTRUE=true
60
ARRAY2_6_FLOATNEGATIVE=-0.123
84
-ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
61
+ARRAY2_6_FLOATPOSITIVE=0.987
62
+ARRAY2_6_NULLVALUE=null
63
+ARRAY2_6_NUMERICNEGATIVE=-456
64
+ARRAY2_6_NUMERICPOSITIVE=123
65
ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
66
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
67
ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
87
-ARRAY2_6_BOOLEANTRUE=true
88
-ARRAY2_6_BOOLEANFALSE=false
68
ARRAY2_6_STRING=Nested Object in Array2
90
-ARRAY2_6_NULLVALUE=null
91
-ARRAY2_7_NUMERICPOSITIVE=42
92
-ARRAY2_7_NUMERICNEGATIVE=-123
93
-ARRAY2_7_FLOATPOSITIVE=3.14159
69
+ARRAY2_7_BOOLEANFALSE=false
70
+ARRAY2_7_BOOLEANTRUE=true
71
ARRAY2_7_FLOATNEGATIVE=-2.71828
95
-ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
72
+ARRAY2_7_FLOATPOSITIVE=3.14159
73
+ARRAY2_7_NULLVALUE=null
74
+ARRAY2_7_NUMERICNEGATIVE=-123
75
+ARRAY2_7_NUMERICPOSITIVE=42
76
ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
77
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
78
ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
98
-ARRAY2_7_BOOLEANTRUE=true
99
-ARRAY2_7_BOOLEANFALSE=false
79
ARRAY2_7_STRING=Array Element with Object in Array2
101
-ARRAY2_7_NULLVALUE=null
102
-
103
-NUMERICPOSITIVE=42
104
-NUMERICNEGATIVE=-123
105
-FLOATPOSITIVE=3.14159
106
-FLOATNEGATIVE=-2.71828
107
-SCIENTIFICINTPOSITIVE=1e5
108
-SCIENTIFICFLOATNEGATIVE=-2.5e-3
109
-SCIENTIFICSMALLPOSITIVE=1e-4
110
-BOOLEANTRUE=true
80
BOOLEANFALSE=false
112
-STRING=Hello, World!
81
+BOOLEANTRUE=true
82
+FLOATNEGATIVE=-2.71828
83
+FLOATPOSITIVE=3.14159
84
NULLVALUE=null
114
-OBJECT_NUMERICPOSITIVE=123
115
-OBJECT_NUMERICNEGATIVE=-456
116
-OBJECT_FLOATPOSITIVE=0.987
85
+NUMERICNEGATIVE=-123
86
+NUMERICPOSITIVE=42
87
+OBJECT_BOOLEANFALSE=false
88
+OBJECT_BOOLEANTRUE=true
89
OBJECT_FLOATNEGATIVE=-0.123
118
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
90
+OBJECT_FLOATPOSITIVE=0.987
91
+OBJECT_NULLVALUE=null
92
+OBJECT_NUMERICNEGATIVE=-456
93
+OBJECT_NUMERICPOSITIVE=123
94
OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
95
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
96
OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
121
-OBJECT_BOOLEANTRUE=true
122
-OBJECT_BOOLEANFALSE=false
97
OBJECT_STRING=Nested Object
124
-OBJECT_NULLVALUE=null
98
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
99
+SCIENTIFICINTPOSITIVE=1e5
100
+SCIENTIFICSMALLPOSITIVE=1e-4
101
+STRING=Hello, World!
102
+
103
ARRAY2_0=1
104
ARRAY2_1=-2.345
105
ARRAY2_2=Array Element
106
ARRAY2_3=true
107
ARRAY2_4=false
108
ARRAY2_5=null
131
-ARRAY2_6_NUMERICPOSITIVE=123
132
-ARRAY2_6_NUMERICNEGATIVE=-456
133
-ARRAY2_6_FLOATPOSITIVE=0.987
109
+ARRAY2_6_BOOLEANFALSE=false
110
+ARRAY2_6_BOOLEANTRUE=true
111
ARRAY2_6_FLOATNEGATIVE=-0.123
135
-ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
112
+ARRAY2_6_FLOATPOSITIVE=0.987
113
+ARRAY2_6_NULLVALUE=null
114
+ARRAY2_6_NUMERICNEGATIVE=-456
115
+ARRAY2_6_NUMERICPOSITIVE=123
116
ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
117
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
118
ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
138
-ARRAY2_6_BOOLEANTRUE=true
139
-ARRAY2_6_BOOLEANFALSE=false
119
ARRAY2_6_STRING=Nested Object in Array2
141
-ARRAY2_6_NULLVALUE=null
142
-ARRAY2_7_NUMERICPOSITIVE=42
143
-ARRAY2_7_NUMERICNEGATIVE=-123
144
-ARRAY2_7_FLOATPOSITIVE=3.14159
120
+ARRAY2_7_BOOLEANFALSE=false
121
+ARRAY2_7_BOOLEANTRUE=true
122
ARRAY2_7_FLOATNEGATIVE=-2.71828
146
-ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
123
+ARRAY2_7_FLOATPOSITIVE=3.14159
124
+ARRAY2_7_NULLVALUE=null
125
+ARRAY2_7_NUMERICNEGATIVE=-123
126
+ARRAY2_7_NUMERICPOSITIVE=42
127
ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
128
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
129
ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
149
-ARRAY2_7_BOOLEANTRUE=true
150
-ARRAY2_7_BOOLEANFALSE=false
130
ARRAY2_7_STRING=Array Element with Object in Array2
152
-ARRAY2_7_NULLVALUE=null
131
+BOOLEANFALSE=false
132
+BOOLEANTRUE=true
133
+FLOATNEGATIVE=-2.71828
134
+FLOATPOSITIVE=3.14159
135
+NULLVALUE=null
136
+NUMERICNEGATIVE=-123
137
+NUMERICPOSITIVE=42
138
+OBJECT_BOOLEANFALSE=false
139
+OBJECT_BOOLEANTRUE=true
140
+OBJECT_FLOATNEGATIVE=-0.123
141
+OBJECT_FLOATPOSITIVE=0.987
142
+OBJECT_NULLVALUE=null
143
+OBJECT_NUMERICNEGATIVE=-456
144
+OBJECT_NUMERICPOSITIVE=123
145
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
146
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
147
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
148
+OBJECT_STRING=Nested Object
149
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
150
+SCIENTIFICINTPOSITIVE=1e5
151
+SCIENTIFICSMALLPOSITIVE=1e-4
152
+STRING=Hello, World!
153
collectors/log2journal/tests.d/json-include.output
+27
-27
@@ -1,54 +1,54 @@
1
-OBJECT_NUMERICPOSITIVE=123
2
-OBJECT_NUMERICNEGATIVE=-456
3
-OBJECT_FLOATPOSITIVE=0.987
4
-OBJECT_FLOATNEGATIVE=-0.123
5
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
6
-OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
7
-OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
8
-OBJECT_BOOLEANTRUE=true
9
-OBJECT_BOOLEANFALSE=false
10
-OBJECT_STRING=Nested Object
11
-OBJECT_NULLVALUE=null
1
OBJECT_ARRAY_0=1
2
OBJECT_ARRAY_1=-2
3
OBJECT_ARRAY_2=3
4
OBJECT_ARRAY_3=Nested Array
5
OBJECT_ARRAY_4=true
6
OBJECT_ARRAY_5=null
18
-
19
-OBJECT_NUMERICPOSITIVE=123
20
-OBJECT_NUMERICNEGATIVE=-456
21
-OBJECT_FLOATPOSITIVE=0.987
7
+OBJECT_BOOLEANFALSE=false
8
+OBJECT_BOOLEANTRUE=true
9
OBJECT_FLOATNEGATIVE=-0.123
23
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
10
+OBJECT_FLOATPOSITIVE=0.987
11
+OBJECT_NULLVALUE=null
12
+OBJECT_NUMERICNEGATIVE=-456
13
+OBJECT_NUMERICPOSITIVE=123
14
OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
15
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
16
OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
26
-OBJECT_BOOLEANTRUE=true
27
-OBJECT_BOOLEANFALSE=false
17
OBJECT_STRING=Nested Object
29
-OBJECT_NULLVALUE=null
18
+
19
OBJECT_ARRAY_0=1
20
OBJECT_ARRAY_1=-2
21
OBJECT_ARRAY_2=3
22
OBJECT_ARRAY_3=Nested Array
23
OBJECT_ARRAY_4=true
24
OBJECT_ARRAY_5=null
36
-
37
-OBJECT_NUMERICPOSITIVE=123
38
-OBJECT_NUMERICNEGATIVE=-456
39
-OBJECT_FLOATPOSITIVE=0.987
25
+OBJECT_BOOLEANFALSE=false
26
+OBJECT_BOOLEANTRUE=true
27
OBJECT_FLOATNEGATIVE=-0.123
41
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
28
+OBJECT_FLOATPOSITIVE=0.987
29
+OBJECT_NULLVALUE=null
30
+OBJECT_NUMERICNEGATIVE=-456
31
+OBJECT_NUMERICPOSITIVE=123
32
OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
33
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
34
OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
44
-OBJECT_BOOLEANTRUE=true
45
-OBJECT_BOOLEANFALSE=false
35
OBJECT_STRING=Nested Object
47
-OBJECT_NULLVALUE=null
36
+
37
OBJECT_ARRAY_0=1
38
OBJECT_ARRAY_1=-2
39
OBJECT_ARRAY_2=3
40
OBJECT_ARRAY_3=Nested Array
41
OBJECT_ARRAY_4=true
42
OBJECT_ARRAY_5=null
43
+OBJECT_BOOLEANFALSE=false
44
+OBJECT_BOOLEANTRUE=true
45
+OBJECT_FLOATNEGATIVE=-0.123
46
+OBJECT_FLOATPOSITIVE=0.987
47
+OBJECT_NULLVALUE=null
48
+OBJECT_NUMERICNEGATIVE=-456
49
+OBJECT_NUMERICPOSITIVE=123
50
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
51
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
52
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
53
+OBJECT_STRING=Nested Object
54
collectors/log2journal/tests.d/json.output
+209
-209
@@ -1,294 +1,294 @@
1
-NUMERICPOSITIVE=42
2
-NUMERICNEGATIVE=-123
3
-FLOATPOSITIVE=3.14159
4
-FLOATNEGATIVE=-2.71828
5
-SCIENTIFICINTPOSITIVE=1e5
6
-SCIENTIFICFLOATNEGATIVE=-2.5e-3
7
-SCIENTIFICSMALLPOSITIVE=1e-4
8
-BOOLEANTRUE=true
9
-BOOLEANFALSE=false
10
-STRING=Hello, World!
11
-NULLVALUE=null
12
-OBJECT_NUMERICPOSITIVE=123
13
-OBJECT_NUMERICNEGATIVE=-456
14
-OBJECT_FLOATPOSITIVE=0.987
15
-OBJECT_FLOATNEGATIVE=-0.123
16
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
17
-OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
18
-OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
19
-OBJECT_BOOLEANTRUE=true
20
-OBJECT_BOOLEANFALSE=false
21
-OBJECT_STRING=Nested Object
22
-OBJECT_NULLVALUE=null
23
-OBJECT_ARRAY_0=1
24
-OBJECT_ARRAY_1=-2
25
-OBJECT_ARRAY_2=3
26
-OBJECT_ARRAY_3=Nested Array
27
-OBJECT_ARRAY_4=true
28
-OBJECT_ARRAY_5=null
29
-ARRAY_0=1
30
-ARRAY_1=-2.345
31
-ARRAY_2=Array Element
32
-ARRAY_3=true
33
-ARRAY_4=false
34
-ARRAY_5=null
35
-ARRAY_6_NUMERICPOSITIVE=987
36
-ARRAY_6_NUMERICNEGATIVE=-654
37
-ARRAY_6_STRING=Nested Object in Array
38
-ARRAY_6_ARRAY_0=null
39
-ARRAY_6_ARRAY_1=false
40
-ARRAY_6_ARRAY_2=true
41
-ARRAY_7_NUMERICPOSITIVE=42
42
-ARRAY_7_NUMERICNEGATIVE=-123
43
-ARRAY_7_FLOATPOSITIVE=3.14159
44
-ARRAY_7_FLOATNEGATIVE=-2.71828
45
-ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
46
-ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
47
-ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
48
-ARRAY_7_BOOLEANTRUE=true
49
-ARRAY_7_BOOLEANFALSE=false
50
-ARRAY_7_STRING=Array Element with Object
51
-ARRAY_7_NULLVALUE=null
52
-ARRAY_7_ARRAY_0=1
53
-ARRAY_7_ARRAY_1=-2
54
-ARRAY_7_ARRAY_2=3
55
-ARRAY_7_ARRAY_3=Nested Array in Object
56
-ARRAY_7_ARRAY_4=true
57
-ARRAY_7_ARRAY_5=null
1
ARRAY2_0=1
2
ARRAY2_1=-2.345
3
ARRAY2_2=Array Element
4
ARRAY2_3=true
5
ARRAY2_4=false
6
ARRAY2_5=null
64
-ARRAY2_6_NUMERICPOSITIVE=123
65
-ARRAY2_6_NUMERICNEGATIVE=-456
66
-ARRAY2_6_FLOATPOSITIVE=0.987
67
-ARRAY2_6_FLOATNEGATIVE=-0.123
68
-ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
69
-ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
70
-ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
71
-ARRAY2_6_BOOLEANTRUE=true
72
-ARRAY2_6_BOOLEANFALSE=false
73
-ARRAY2_6_STRING=Nested Object in Array2
74
-ARRAY2_6_NULLVALUE=null
7
ARRAY2_6_ARRAY_0=1
8
ARRAY2_6_ARRAY_1=-2
9
ARRAY2_6_ARRAY_2=3
10
ARRAY2_6_ARRAY_3=Nested Array in Object2
11
ARRAY2_6_ARRAY_4=true
12
ARRAY2_6_ARRAY_5=null
81
-ARRAY2_7_NUMERICPOSITIVE=42
82
-ARRAY2_7_NUMERICNEGATIVE=-123
83
-ARRAY2_7_FLOATPOSITIVE=3.14159
84
-ARRAY2_7_FLOATNEGATIVE=-2.71828
85
-ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
86
-ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
87
-ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
88
-ARRAY2_7_BOOLEANTRUE=true
89
-ARRAY2_7_BOOLEANFALSE=false
90
-ARRAY2_7_STRING=Array Element with Object in Array2
91
-ARRAY2_7_NULLVALUE=null
13
+ARRAY2_6_BOOLEANFALSE=false
14
+ARRAY2_6_BOOLEANTRUE=true
15
+ARRAY2_6_FLOATNEGATIVE=-0.123
16
+ARRAY2_6_FLOATPOSITIVE=0.987
17
+ARRAY2_6_NULLVALUE=null
18
+ARRAY2_6_NUMERICNEGATIVE=-456
19
+ARRAY2_6_NUMERICPOSITIVE=123
20
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
21
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
22
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
23
+ARRAY2_6_STRING=Nested Object in Array2
24
ARRAY2_7_ARRAY_0=1
25
ARRAY2_7_ARRAY_1=-2
26
ARRAY2_7_ARRAY_2=3
27
ARRAY2_7_ARRAY_3=Nested Array in Object2
28
ARRAY2_7_ARRAY_4=true
29
ARRAY2_7_ARRAY_5=null
98
-
99
-NUMERICPOSITIVE=42
100
-NUMERICNEGATIVE=-123
101
-FLOATPOSITIVE=3.14159
102
-FLOATNEGATIVE=-2.71828
103
-SCIENTIFICINTPOSITIVE=1e5
104
-SCIENTIFICFLOATNEGATIVE=-2.5e-3
105
-SCIENTIFICSMALLPOSITIVE=1e-4
106
-BOOLEANTRUE=true
107
-BOOLEANFALSE=false
108
-STRING=Hello, World!
109
-NULLVALUE=null
110
-OBJECT_NUMERICPOSITIVE=123
111
-OBJECT_NUMERICNEGATIVE=-456
112
-OBJECT_FLOATPOSITIVE=0.987
113
-OBJECT_FLOATNEGATIVE=-0.123
114
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
115
-OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
116
-OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
117
-OBJECT_BOOLEANTRUE=true
118
-OBJECT_BOOLEANFALSE=false
119
-OBJECT_STRING=Nested Object
120
-OBJECT_NULLVALUE=null
121
-OBJECT_ARRAY_0=1
122
-OBJECT_ARRAY_1=-2
123
-OBJECT_ARRAY_2=3
124
-OBJECT_ARRAY_3=Nested Array
125
-OBJECT_ARRAY_4=true
126
-OBJECT_ARRAY_5=null
30
+ARRAY2_7_BOOLEANFALSE=false
31
+ARRAY2_7_BOOLEANTRUE=true
32
+ARRAY2_7_FLOATNEGATIVE=-2.71828
33
+ARRAY2_7_FLOATPOSITIVE=3.14159
34
+ARRAY2_7_NULLVALUE=null
35
+ARRAY2_7_NUMERICNEGATIVE=-123
36
+ARRAY2_7_NUMERICPOSITIVE=42
37
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
38
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
39
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
40
+ARRAY2_7_STRING=Array Element with Object in Array2
41
ARRAY_0=1
42
ARRAY_1=-2.345
43
ARRAY_2=Array Element
44
ARRAY_3=true
45
ARRAY_4=false
46
ARRAY_5=null
133
-ARRAY_6_NUMERICPOSITIVE=987
134
-ARRAY_6_NUMERICNEGATIVE=-654
135
-ARRAY_6_STRING=Nested Object in Array
47
ARRAY_6_ARRAY_0=null
48
ARRAY_6_ARRAY_1=false
49
ARRAY_6_ARRAY_2=true
139
-ARRAY_7_NUMERICPOSITIVE=42
140
-ARRAY_7_NUMERICNEGATIVE=-123
141
-ARRAY_7_FLOATPOSITIVE=3.14159
142
-ARRAY_7_FLOATNEGATIVE=-2.71828
143
-ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
144
-ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
145
-ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
146
-ARRAY_7_BOOLEANTRUE=true
147
-ARRAY_7_BOOLEANFALSE=false
148
-ARRAY_7_STRING=Array Element with Object
149
-ARRAY_7_NULLVALUE=null
50
+ARRAY_6_NUMERICNEGATIVE=-654
51
+ARRAY_6_NUMERICPOSITIVE=987
52
+ARRAY_6_STRING=Nested Object in Array
53
ARRAY_7_ARRAY_0=1
54
ARRAY_7_ARRAY_1=-2
55
ARRAY_7_ARRAY_2=3
56
ARRAY_7_ARRAY_3=Nested Array in Object
57
ARRAY_7_ARRAY_4=true
58
ARRAY_7_ARRAY_5=null
59
+ARRAY_7_BOOLEANFALSE=false
60
+ARRAY_7_BOOLEANTRUE=true
61
+ARRAY_7_FLOATNEGATIVE=-2.71828
62
+ARRAY_7_FLOATPOSITIVE=3.14159
63
+ARRAY_7_NULLVALUE=null
64
+ARRAY_7_NUMERICNEGATIVE=-123
65
+ARRAY_7_NUMERICPOSITIVE=42
66
+ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
67
+ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
68
+ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
69
+ARRAY_7_STRING=Array Element with Object
70
+BOOLEANFALSE=false
71
+BOOLEANTRUE=true
72
+FLOATNEGATIVE=-2.71828
73
+FLOATPOSITIVE=3.14159
74
+NULLVALUE=null
75
+NUMERICNEGATIVE=-123
76
+NUMERICPOSITIVE=42
77
+OBJECT_ARRAY_0=1
78
+OBJECT_ARRAY_1=-2
79
+OBJECT_ARRAY_2=3
80
+OBJECT_ARRAY_3=Nested Array
81
+OBJECT_ARRAY_4=true
82
+OBJECT_ARRAY_5=null
83
+OBJECT_BOOLEANFALSE=false
84
+OBJECT_BOOLEANTRUE=true
85
+OBJECT_FLOATNEGATIVE=-0.123
86
+OBJECT_FLOATPOSITIVE=0.987
87
+OBJECT_NULLVALUE=null
88
+OBJECT_NUMERICNEGATIVE=-456
89
+OBJECT_NUMERICPOSITIVE=123
90
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
91
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
92
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
93
+OBJECT_STRING=Nested Object
94
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
95
+SCIENTIFICINTPOSITIVE=1e5
96
+SCIENTIFICSMALLPOSITIVE=1e-4
97
+STRING=Hello, World!
98
+
99
ARRAY2_0=1
100
ARRAY2_1=-2.345
101
ARRAY2_2=Array Element
102
ARRAY2_3=true
103
ARRAY2_4=false
104
ARRAY2_5=null
162
-ARRAY2_6_NUMERICPOSITIVE=123
163
-ARRAY2_6_NUMERICNEGATIVE=-456
164
-ARRAY2_6_FLOATPOSITIVE=0.987
165
-ARRAY2_6_FLOATNEGATIVE=-0.123
166
-ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
167
-ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
168
-ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
169
-ARRAY2_6_BOOLEANTRUE=true
170
-ARRAY2_6_BOOLEANFALSE=false
171
-ARRAY2_6_STRING=Nested Object in Array2
172
-ARRAY2_6_NULLVALUE=null
105
ARRAY2_6_ARRAY_0=1
106
ARRAY2_6_ARRAY_1=-2
107
ARRAY2_6_ARRAY_2=3
108
ARRAY2_6_ARRAY_3=Nested Array in Object2
109
ARRAY2_6_ARRAY_4=true
110
ARRAY2_6_ARRAY_5=null
179
-ARRAY2_7_NUMERICPOSITIVE=42
180
-ARRAY2_7_NUMERICNEGATIVE=-123
181
-ARRAY2_7_FLOATPOSITIVE=3.14159
182
-ARRAY2_7_FLOATNEGATIVE=-2.71828
183
-ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
184
-ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
185
-ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
186
-ARRAY2_7_BOOLEANTRUE=true
187
-ARRAY2_7_BOOLEANFALSE=false
188
-ARRAY2_7_STRING=Array Element with Object in Array2
189
-ARRAY2_7_NULLVALUE=null
111
+ARRAY2_6_BOOLEANFALSE=false
112
+ARRAY2_6_BOOLEANTRUE=true
113
+ARRAY2_6_FLOATNEGATIVE=-0.123
114
+ARRAY2_6_FLOATPOSITIVE=0.987
115
+ARRAY2_6_NULLVALUE=null
116
+ARRAY2_6_NUMERICNEGATIVE=-456
117
+ARRAY2_6_NUMERICPOSITIVE=123
118
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
119
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
120
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
121
+ARRAY2_6_STRING=Nested Object in Array2
122
ARRAY2_7_ARRAY_0=1
123
ARRAY2_7_ARRAY_1=-2
124
ARRAY2_7_ARRAY_2=3
125
ARRAY2_7_ARRAY_3=Nested Array in Object2
126
ARRAY2_7_ARRAY_4=true
127
ARRAY2_7_ARRAY_5=null
196
-
197
-NUMERICPOSITIVE=42
198
-NUMERICNEGATIVE=-123
199
-FLOATPOSITIVE=3.14159
200
-FLOATNEGATIVE=-2.71828
201
-SCIENTIFICINTPOSITIVE=1e5
202
-SCIENTIFICFLOATNEGATIVE=-2.5e-3
203
-SCIENTIFICSMALLPOSITIVE=1e-4
204
-BOOLEANTRUE=true
205
-BOOLEANFALSE=false
206
-STRING=Hello, World!
207
-NULLVALUE=null
208
-OBJECT_NUMERICPOSITIVE=123
209
-OBJECT_NUMERICNEGATIVE=-456
210
-OBJECT_FLOATPOSITIVE=0.987
211
-OBJECT_FLOATNEGATIVE=-0.123
212
-OBJECT_SCIENTIFICINTPOSITIVE=6e4
213
-OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
214
-OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
215
-OBJECT_BOOLEANTRUE=true
216
-OBJECT_BOOLEANFALSE=false
217
-OBJECT_STRING=Nested Object
218
-OBJECT_NULLVALUE=null
219
-OBJECT_ARRAY_0=1
220
-OBJECT_ARRAY_1=-2
221
-OBJECT_ARRAY_2=3
222
-OBJECT_ARRAY_3=Nested Array
223
-OBJECT_ARRAY_4=true
224
-OBJECT_ARRAY_5=null
128
+ARRAY2_7_BOOLEANFALSE=false
129
+ARRAY2_7_BOOLEANTRUE=true
130
+ARRAY2_7_FLOATNEGATIVE=-2.71828
131
+ARRAY2_7_FLOATPOSITIVE=3.14159
132
+ARRAY2_7_NULLVALUE=null
133
+ARRAY2_7_NUMERICNEGATIVE=-123
134
+ARRAY2_7_NUMERICPOSITIVE=42
135
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
136
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
137
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
138
+ARRAY2_7_STRING=Array Element with Object in Array2
139
ARRAY_0=1
140
ARRAY_1=-2.345
141
ARRAY_2=Array Element
142
ARRAY_3=true
143
ARRAY_4=false
144
ARRAY_5=null
231
-ARRAY_6_NUMERICPOSITIVE=987
232
-ARRAY_6_NUMERICNEGATIVE=-654
233
-ARRAY_6_STRING=Nested Object in Array
145
ARRAY_6_ARRAY_0=null
146
ARRAY_6_ARRAY_1=false
147
ARRAY_6_ARRAY_2=true
237
-ARRAY_7_NUMERICPOSITIVE=42
238
-ARRAY_7_NUMERICNEGATIVE=-123
239
-ARRAY_7_FLOATPOSITIVE=3.14159
240
-ARRAY_7_FLOATNEGATIVE=-2.71828
241
-ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
242
-ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
243
-ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
244
-ARRAY_7_BOOLEANTRUE=true
245
-ARRAY_7_BOOLEANFALSE=false
246
-ARRAY_7_STRING=Array Element with Object
247
-ARRAY_7_NULLVALUE=null
148
+ARRAY_6_NUMERICNEGATIVE=-654
149
+ARRAY_6_NUMERICPOSITIVE=987
150
+ARRAY_6_STRING=Nested Object in Array
151
ARRAY_7_ARRAY_0=1
152
ARRAY_7_ARRAY_1=-2
153
ARRAY_7_ARRAY_2=3
154
ARRAY_7_ARRAY_3=Nested Array in Object
155
ARRAY_7_ARRAY_4=true
156
ARRAY_7_ARRAY_5=null
157
+ARRAY_7_BOOLEANFALSE=false
158
+ARRAY_7_BOOLEANTRUE=true
159
+ARRAY_7_FLOATNEGATIVE=-2.71828
160
+ARRAY_7_FLOATPOSITIVE=3.14159
161
+ARRAY_7_NULLVALUE=null
162
+ARRAY_7_NUMERICNEGATIVE=-123
163
+ARRAY_7_NUMERICPOSITIVE=42
164
+ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
165
+ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
166
+ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
167
+ARRAY_7_STRING=Array Element with Object
168
+BOOLEANFALSE=false
169
+BOOLEANTRUE=true
170
+FLOATNEGATIVE=-2.71828
171
+FLOATPOSITIVE=3.14159
172
+NULLVALUE=null
173
+NUMERICNEGATIVE=-123
174
+NUMERICPOSITIVE=42
175
+OBJECT_ARRAY_0=1
176
+OBJECT_ARRAY_1=-2
177
+OBJECT_ARRAY_2=3
178
+OBJECT_ARRAY_3=Nested Array
179
+OBJECT_ARRAY_4=true
180
+OBJECT_ARRAY_5=null
181
+OBJECT_BOOLEANFALSE=false
182
+OBJECT_BOOLEANTRUE=true
183
+OBJECT_FLOATNEGATIVE=-0.123
184
+OBJECT_FLOATPOSITIVE=0.987
185
+OBJECT_NULLVALUE=null
186
+OBJECT_NUMERICNEGATIVE=-456
187
+OBJECT_NUMERICPOSITIVE=123
188
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
189
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
190
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
191
+OBJECT_STRING=Nested Object
192
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
193
+SCIENTIFICINTPOSITIVE=1e5
194
+SCIENTIFICSMALLPOSITIVE=1e-4
195
+STRING=Hello, World!
196
+
197
ARRAY2_0=1
198
ARRAY2_1=-2.345
199
ARRAY2_2=Array Element
200
ARRAY2_3=true
201
ARRAY2_4=false
202
ARRAY2_5=null
260
-ARRAY2_6_NUMERICPOSITIVE=123
261
-ARRAY2_6_NUMERICNEGATIVE=-456
262
-ARRAY2_6_FLOATPOSITIVE=0.987
263
-ARRAY2_6_FLOATNEGATIVE=-0.123
264
-ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
265
-ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
266
-ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
267
-ARRAY2_6_BOOLEANTRUE=true
268
-ARRAY2_6_BOOLEANFALSE=false
269
-ARRAY2_6_STRING=Nested Object in Array2
270
-ARRAY2_6_NULLVALUE=null
203
ARRAY2_6_ARRAY_0=1
204
ARRAY2_6_ARRAY_1=-2
205
ARRAY2_6_ARRAY_2=3
206
ARRAY2_6_ARRAY_3=Nested Array in Object2
207
ARRAY2_6_ARRAY_4=true
208
ARRAY2_6_ARRAY_5=null
277
-ARRAY2_7_NUMERICPOSITIVE=42
278
-ARRAY2_7_NUMERICNEGATIVE=-123
279
-ARRAY2_7_FLOATPOSITIVE=3.14159
280
-ARRAY2_7_FLOATNEGATIVE=-2.71828
281
-ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
282
-ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
283
-ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
284
-ARRAY2_7_BOOLEANTRUE=true
285
-ARRAY2_7_BOOLEANFALSE=false
286
-ARRAY2_7_STRING=Array Element with Object in Array2
287
-ARRAY2_7_NULLVALUE=null
209
+ARRAY2_6_BOOLEANFALSE=false
210
+ARRAY2_6_BOOLEANTRUE=true
211
+ARRAY2_6_FLOATNEGATIVE=-0.123
212
+ARRAY2_6_FLOATPOSITIVE=0.987
213
+ARRAY2_6_NULLVALUE=null
214
+ARRAY2_6_NUMERICNEGATIVE=-456
215
+ARRAY2_6_NUMERICPOSITIVE=123
216
+ARRAY2_6_SCIENTIFICFLOATNEGATIVE=-1.5e-2
217
+ARRAY2_6_SCIENTIFICINTPOSITIVE=6e4
218
+ARRAY2_6_SCIENTIFICSMALLPOSITIVE=5e-5
219
+ARRAY2_6_STRING=Nested Object in Array2
220
ARRAY2_7_ARRAY_0=1
221
ARRAY2_7_ARRAY_1=-2
222
ARRAY2_7_ARRAY_2=3
223
ARRAY2_7_ARRAY_3=Nested Array in Object2
224
ARRAY2_7_ARRAY_4=true
225
ARRAY2_7_ARRAY_5=null
226
+ARRAY2_7_BOOLEANFALSE=false
227
+ARRAY2_7_BOOLEANTRUE=true
228
+ARRAY2_7_FLOATNEGATIVE=-2.71828
229
+ARRAY2_7_FLOATPOSITIVE=3.14159
230
+ARRAY2_7_NULLVALUE=null
231
+ARRAY2_7_NUMERICNEGATIVE=-123
232
+ARRAY2_7_NUMERICPOSITIVE=42
233
+ARRAY2_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
234
+ARRAY2_7_SCIENTIFICINTPOSITIVE=1e5
235
+ARRAY2_7_SCIENTIFICSMALLPOSITIVE=1e-4
236
+ARRAY2_7_STRING=Array Element with Object in Array2
237
+ARRAY_0=1
238
+ARRAY_1=-2.345
239
+ARRAY_2=Array Element
240
+ARRAY_3=true
241
+ARRAY_4=false
242
+ARRAY_5=null
243
+ARRAY_6_ARRAY_0=null
244
+ARRAY_6_ARRAY_1=false
245
+ARRAY_6_ARRAY_2=true
246
+ARRAY_6_NUMERICNEGATIVE=-654
247
+ARRAY_6_NUMERICPOSITIVE=987
248
+ARRAY_6_STRING=Nested Object in Array
249
+ARRAY_7_ARRAY_0=1
250
+ARRAY_7_ARRAY_1=-2
251
+ARRAY_7_ARRAY_2=3
252
+ARRAY_7_ARRAY_3=Nested Array in Object
253
+ARRAY_7_ARRAY_4=true
254
+ARRAY_7_ARRAY_5=null
255
+ARRAY_7_BOOLEANFALSE=false
256
+ARRAY_7_BOOLEANTRUE=true
257
+ARRAY_7_FLOATNEGATIVE=-2.71828
258
+ARRAY_7_FLOATPOSITIVE=3.14159
259
+ARRAY_7_NULLVALUE=null
260
+ARRAY_7_NUMERICNEGATIVE=-123
261
+ARRAY_7_NUMERICPOSITIVE=42
262
+ARRAY_7_SCIENTIFICFLOATNEGATIVE=-2.5e-3
263
+ARRAY_7_SCIENTIFICINTPOSITIVE=1e5
264
+ARRAY_7_SCIENTIFICSMALLPOSITIVE=1e-4
265
+ARRAY_7_STRING=Array Element with Object
266
+BOOLEANFALSE=false
267
+BOOLEANTRUE=true
268
+FLOATNEGATIVE=-2.71828
269
+FLOATPOSITIVE=3.14159
270
+NULLVALUE=null
271
+NUMERICNEGATIVE=-123
272
+NUMERICPOSITIVE=42
273
+OBJECT_ARRAY_0=1
274
+OBJECT_ARRAY_1=-2
275
+OBJECT_ARRAY_2=3
276
+OBJECT_ARRAY_3=Nested Array
277
+OBJECT_ARRAY_4=true
278
+OBJECT_ARRAY_5=null
279
+OBJECT_BOOLEANFALSE=false
280
+OBJECT_BOOLEANTRUE=true
281
+OBJECT_FLOATNEGATIVE=-0.123
282
+OBJECT_FLOATPOSITIVE=0.987
283
+OBJECT_NULLVALUE=null
284
+OBJECT_NUMERICNEGATIVE=-456
285
+OBJECT_NUMERICPOSITIVE=123
286
+OBJECT_SCIENTIFICFLOATNEGATIVE=-1.5e-2
287
+OBJECT_SCIENTIFICINTPOSITIVE=6e4
288
+OBJECT_SCIENTIFICSMALLPOSITIVE=5e-5
289
+OBJECT_STRING=Nested Object
290
+SCIENTIFICFLOATNEGATIVE=-2.5e-3
291
+SCIENTIFICINTPOSITIVE=1e5
292
+SCIENTIFICSMALLPOSITIVE=1e-4
293
+STRING=Hello, World!
294
collectors/log2journal/tests.d/logfmt.log
new
+5
@@ -0,0 +1,5 @@
1
+key1=value01 key2=value02 key3=value03 key4=value04
2
+key1=value11 key2=value12 key3=value13 key4=
3
+key1=value21 key2=value22 key3=value23 key4=value24
4
+key1=value31 key2=value32 key3=value33 key4=
5
+key1=value41 key2=value42 key3=value43 key4=value44
collectors/log2journal/tests.d/logfmt.output
new
+37
@@ -0,0 +1,37 @@
1
+INJECTED=Key INJECTED had value 'value01 - value02' and now has this, but only on the first row of the log.
2
+KEY1=value01
3
+KEY2=value02
4
+KEY3=value03
5
+KEY4=value04
6
+SIMPLE_INJECTION=An unset variable looks like '', while the value of KEY2 is 'value02'
7
+YET_ANOTHER_INJECTION=value01 - value02 - Key INJECTED had value 'value01 - value02' and now has this, but only on the first row of the log. - this should work because inject is yes
8
+
9
+INJECTED=value11 - value12
10
+KEY1=value11
11
+KEY2=value12
12
+KEY3=value13
13
+SIMPLE_INJECTION=An unset variable looks like '', while the value of KEY2 is 'value12'
14
+YET_ANOTHER_INJECTION=value11 - value12 - value11 - value12 - this should work because inject is yes
15
+
16
+INJECTED=KEY4 has the value 'value24'; it is not empty, so INJECTED has been rewritten.
17
+KEY1=value21
18
+KEY2=value22
19
+KEY3=value23
20
+KEY4=value24
21
+SIMPLE_INJECTION=An unset variable looks like '', while the value of KEY2 is 'value22'
22
+YET_ANOTHER_INJECTION=value21 - value22 - KEY4 has the value 'value24'; it is not empty, so INJECTED has been rewritten. - this should work because inject is yes
23
+
24
+INJECTED=value31 - value32
25
+KEY1=value31
26
+KEY2=value32
27
+KEY3=value33
28
+YET_ANOTHER_INJECTION=value31 - value32 - value31 - value32 - this should work because inject is yes
29
+
30
+INJECTED=KEY4 has the value 'value44'; it is not empty, so INJECTED has been rewritten.
31
+KEY1=value41
32
+KEY2=value42
33
+KEY3=value43
34
+KEY4=value44
35
+SIMPLE_INJECTION=An unset variable looks like '', while the value of KEY2 is 'value42'
36
+YET_ANOTHER_INJECTION=value41 - value42 - KEY4 has the value 'value44'; it is not empty, so INJECTED has been rewritten. - this should work because inject is yes
37
+
collectors/log2journal/tests.d/logfmt.yaml
new
+34
@@ -0,0 +1,34 @@
1
+pattern: logfmt
2
+
3
+inject:
4
+ - key: SIMPLE_INJECTION
5
+ value: "An unset variable looks like '${this}', while the value of KEY2 is '${KEY2}'"
6
+
7
+rewrite:
8
+ - key: INJECTED
9
+ value: "${KEY1} - ${KEY2}"
10
+ inject: yes
11
+ stop: no
12
+
13
+ - key: INJECTED
14
+ match: '^value01'
15
+ value: "Key INJECTED had value '${INJECTED}' and now has this, but only on the first row of the log."
16
+
17
+ - key: INJECTED
18
+ not_empty: "${KEY4}"
19
+ value: "KEY4 has the value '${KEY4}'; it is not empty, so INJECTED has been rewritten."
20
+
21
+ - key: INJECTED
22
+ match: '^KEY4 has the value'
23
+ value: "This value should not appear in the logs, because the previous one matched and stopped the pipeline."
24
+
25
+ - key: ANOTHER_INJECTION
26
+ value: "${KEY1} - ${KEY2} - ${INJECTED} - should not work because inject is not true amd ANOTHER_INJECTION is not in the log file."
27
+
28
+ - key: YET_ANOTHER_INJECTION
29
+ value: "${KEY1} - ${KEY2} - ${INJECTED} - this should work because inject is yes"
30
+ inject: yes
31
+
32
+ - key: SIMPLE_INJECTION
33
+ match: "KEY2 is 'value32'"
34
+ value: "" # empty, so SIMPLE_INJECTION should not be available on row 3
collectors/log2journal/tests.d/nginx-combined.log
new
+14
@@ -0,0 +1,14 @@
1
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:27 +0000] "GET /api/v1/data?chart=system.net&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775349 HTTP/1.1" 200 4844 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
2
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:27 +0000] "OPTIONS /api/v1/data?chart=netdata.clients&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775358 HTTP/1.1" 200 29 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
3
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:27 +0000] "OPTIONS /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=out&_=1701372775359 HTTP/1.1" 200 29 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
4
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:27 +0000] "OPTIONS /api/v1/data?chart=netdata.requests&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775357 HTTP/1.1" 200 29 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
5
+127.0.0.1 - - [30/Nov/2023:19:35:28 +0000] "GET /stub_status HTTP/1.1" 200 120 "-" "Go-http-client/1.1"
6
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "GET /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=out&_=1701372775359 HTTP/1.1" 200 1918 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
7
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "GET /api/v1/data?chart=netdata.requests&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775357 HTTP/1.1" 200 1632 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
8
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "GET /api/v1/data?chart=netdata.clients&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775358 HTTP/1.1" 200 588 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
9
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "OPTIONS /api/v1/data?chart=system.cpu&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775360 HTTP/1.1" 200 29 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
10
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "OPTIONS /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=in&_=1701372775361 HTTP/1.1" 200 29 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
11
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "GET /api/v1/data?chart=system.cpu&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775360 HTTP/1.1" 200 6085 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
12
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "GET /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=in&_=1701372775361 HTTP/1.1" 200 1918 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
13
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "OPTIONS /api/v1/data?chart=system.io&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775362 HTTP/1.1" 200 29 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
14
+2a02:169:1210::2000 - - [30/Nov/2023:19:35:28 +0000] "GET /api/v1/data?chart=system.io&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775362 HTTP/1.1" 200 3503 "http://192.168.69.5:19999/" "Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36"
collectors/log2journal/tests.d/nginx-combined.output
new
+210
@@ -0,0 +1,210 @@
1
+MESSAGE=GET /api/v1/data?chart=system.net&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775349 HTTP/1.1
2
+NGINX_BODY_BYTES_SENT=4844
3
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
4
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
5
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
6
+NGINX_REMOTE_USER=-
7
+NGINX_REQUEST_METHOD=GET
8
+NGINX_REQUEST_URI=/api/v1/data?chart=system.net&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775349
9
+NGINX_SERVER_PROTOCOL=HTTP/1.1
10
+NGINX_STATUS=200
11
+NGINX_STATUS_FAMILY=2xx
12
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:27 +0000
13
+PRIORITY=6
14
+SYSLOG_IDENTIFIER=nginx-log
15
+
16
+MESSAGE=OPTIONS /api/v1/data?chart=netdata.clients&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775358 HTTP/1.1
17
+NGINX_BODY_BYTES_SENT=29
18
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
19
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
20
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
21
+NGINX_REMOTE_USER=-
22
+NGINX_REQUEST_METHOD=OPTIONS
23
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.clients&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775358
24
+NGINX_SERVER_PROTOCOL=HTTP/1.1
25
+NGINX_STATUS=200
26
+NGINX_STATUS_FAMILY=2xx
27
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:27 +0000
28
+PRIORITY=6
29
+SYSLOG_IDENTIFIER=nginx-log
30
+
31
+MESSAGE=OPTIONS /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=out&_=1701372775359 HTTP/1.1
32
+NGINX_BODY_BYTES_SENT=29
33
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
34
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
35
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
36
+NGINX_REMOTE_USER=-
37
+NGINX_REQUEST_METHOD=OPTIONS
38
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=out&_=1701372775359
39
+NGINX_SERVER_PROTOCOL=HTTP/1.1
40
+NGINX_STATUS=200
41
+NGINX_STATUS_FAMILY=2xx
42
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:27 +0000
43
+PRIORITY=6
44
+SYSLOG_IDENTIFIER=nginx-log
45
+
46
+MESSAGE=OPTIONS /api/v1/data?chart=netdata.requests&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775357 HTTP/1.1
47
+NGINX_BODY_BYTES_SENT=29
48
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
49
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
50
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
51
+NGINX_REMOTE_USER=-
52
+NGINX_REQUEST_METHOD=OPTIONS
53
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.requests&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775357
54
+NGINX_SERVER_PROTOCOL=HTTP/1.1
55
+NGINX_STATUS=200
56
+NGINX_STATUS_FAMILY=2xx
57
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:27 +0000
58
+PRIORITY=6
59
+SYSLOG_IDENTIFIER=nginx-log
60
+
61
+MESSAGE=GET /stub_status HTTP/1.1
62
+NGINX_BODY_BYTES_SENT=120
63
+NGINX_HTTP_REFERER=-
64
+NGINX_HTTP_USER_AGENT=Go-http-client/1.1
65
+NGINX_REMOTE_ADDR=127.0.0.1
66
+NGINX_REMOTE_USER=-
67
+NGINX_REQUEST_METHOD=GET
68
+NGINX_REQUEST_URI=/stub_status
69
+NGINX_SERVER_PROTOCOL=HTTP/1.1
70
+NGINX_STATUS=200
71
+NGINX_STATUS_FAMILY=2xx
72
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
73
+PRIORITY=6
74
+SYSLOG_IDENTIFIER=nginx-log
75
+
76
+MESSAGE=GET /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=out&_=1701372775359 HTTP/1.1
77
+NGINX_BODY_BYTES_SENT=1918
78
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
79
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
80
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
81
+NGINX_REMOTE_USER=-
82
+NGINX_REQUEST_METHOD=GET
83
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=out&_=1701372775359
84
+NGINX_SERVER_PROTOCOL=HTTP/1.1
85
+NGINX_STATUS=200
86
+NGINX_STATUS_FAMILY=2xx
87
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
88
+PRIORITY=6
89
+SYSLOG_IDENTIFIER=nginx-log
90
+
91
+MESSAGE=GET /api/v1/data?chart=netdata.requests&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775357 HTTP/1.1
92
+NGINX_BODY_BYTES_SENT=1632
93
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
94
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
95
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
96
+NGINX_REMOTE_USER=-
97
+NGINX_REQUEST_METHOD=GET
98
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.requests&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775357
99
+NGINX_SERVER_PROTOCOL=HTTP/1.1
100
+NGINX_STATUS=200
101
+NGINX_STATUS_FAMILY=2xx
102
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
103
+PRIORITY=6
104
+SYSLOG_IDENTIFIER=nginx-log
105
+
106
+MESSAGE=GET /api/v1/data?chart=netdata.clients&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775358 HTTP/1.1
107
+NGINX_BODY_BYTES_SENT=588
108
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
109
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
110
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
111
+NGINX_REMOTE_USER=-
112
+NGINX_REQUEST_METHOD=GET
113
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.clients&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775358
114
+NGINX_SERVER_PROTOCOL=HTTP/1.1
115
+NGINX_STATUS=200
116
+NGINX_STATUS_FAMILY=2xx
117
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
118
+PRIORITY=6
119
+SYSLOG_IDENTIFIER=nginx-log
120
+
121
+MESSAGE=OPTIONS /api/v1/data?chart=system.cpu&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775360 HTTP/1.1
122
+NGINX_BODY_BYTES_SENT=29
123
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
124
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
125
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
126
+NGINX_REMOTE_USER=-
127
+NGINX_REQUEST_METHOD=OPTIONS
128
+NGINX_REQUEST_URI=/api/v1/data?chart=system.cpu&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775360
129
+NGINX_SERVER_PROTOCOL=HTTP/1.1
130
+NGINX_STATUS=200
131
+NGINX_STATUS_FAMILY=2xx
132
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
133
+PRIORITY=6
134
+SYSLOG_IDENTIFIER=nginx-log
135
+
136
+MESSAGE=OPTIONS /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=in&_=1701372775361 HTTP/1.1
137
+NGINX_BODY_BYTES_SENT=29
138
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
139
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
140
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
141
+NGINX_REMOTE_USER=-
142
+NGINX_REQUEST_METHOD=OPTIONS
143
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=in&_=1701372775361
144
+NGINX_SERVER_PROTOCOL=HTTP/1.1
145
+NGINX_STATUS=200
146
+NGINX_STATUS_FAMILY=2xx
147
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
148
+PRIORITY=6
149
+SYSLOG_IDENTIFIER=nginx-log
150
+
151
+MESSAGE=GET /api/v1/data?chart=system.cpu&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775360 HTTP/1.1
152
+NGINX_BODY_BYTES_SENT=6085
153
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
154
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
155
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
156
+NGINX_REMOTE_USER=-
157
+NGINX_REQUEST_METHOD=GET
158
+NGINX_REQUEST_URI=/api/v1/data?chart=system.cpu&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775360
159
+NGINX_SERVER_PROTOCOL=HTTP/1.1
160
+NGINX_STATUS=200
161
+NGINX_STATUS_FAMILY=2xx
162
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
163
+PRIORITY=6
164
+SYSLOG_IDENTIFIER=nginx-log
165
+
166
+MESSAGE=GET /api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=in&_=1701372775361 HTTP/1.1
167
+NGINX_BODY_BYTES_SENT=1918
168
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
169
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
170
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
171
+NGINX_REMOTE_USER=-
172
+NGINX_REQUEST_METHOD=GET
173
+NGINX_REQUEST_URI=/api/v1/data?chart=netdata.net&format=array&points=300&group=average>ime=0&options=absolute%7Cjsonwrap%7Cnonzero&after=-300&dimensions=in&_=1701372775361
174
+NGINX_SERVER_PROTOCOL=HTTP/1.1
175
+NGINX_STATUS=200
176
+NGINX_STATUS_FAMILY=2xx
177
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
178
+PRIORITY=6
179
+SYSLOG_IDENTIFIER=nginx-log
180
+
181
+MESSAGE=OPTIONS /api/v1/data?chart=system.io&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775362 HTTP/1.1
182
+NGINX_BODY_BYTES_SENT=29
183
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
184
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
185
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
186
+NGINX_REMOTE_USER=-
187
+NGINX_REQUEST_METHOD=OPTIONS
188
+NGINX_REQUEST_URI=/api/v1/data?chart=system.io&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775362
189
+NGINX_SERVER_PROTOCOL=HTTP/1.1
190
+NGINX_STATUS=200
191
+NGINX_STATUS_FAMILY=2xx
192
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
193
+PRIORITY=6
194
+SYSLOG_IDENTIFIER=nginx-log
195
+
196
+MESSAGE=GET /api/v1/data?chart=system.io&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775362 HTTP/1.1
197
+NGINX_BODY_BYTES_SENT=3503
198
+NGINX_HTTP_REFERER=http://192.168.69.5:19999/
199
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (X11; CrOS armv7l 13597.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.98 Safari/537.36
200
+NGINX_REMOTE_ADDR=2a02:169:1210::2000
201
+NGINX_REMOTE_USER=-
202
+NGINX_REQUEST_METHOD=GET
203
+NGINX_REQUEST_URI=/api/v1/data?chart=system.io&format=json&points=267&group=average>ime=0&options=ms%7Cflip%7Cjsonwrap%7Cnonzero&after=-300&_=1701372775362
204
+NGINX_SERVER_PROTOCOL=HTTP/1.1
205
+NGINX_STATUS=200
206
+NGINX_STATUS_FAMILY=2xx
207
+NGINX_TIME_LOCAL=30/Nov/2023:19:35:28 +0000
208
+PRIORITY=6
209
+SYSLOG_IDENTIFIER=nginx-log
210
+
collectors/log2journal/tests.d/nginx-json.output
+173
-173
@@ -1,296 +1,296 @@
1
-NGINX_TIMESTAMP_SEC=1644997905.123
1
+MESSAGE=GET /index.html HTTP/1.1
2
+NGINX_BODY_BYTES_SENT=5432
3
+NGINX_BYTES_SENT=6543
4
NGINX_CONNECTION=12345
5
NGINX_CONNECTION_REQUESTS=5
6
+NGINX_GEOIP_COUNTRY_CODE=US
7
+NGINX_GZIP_RATIO=2.1
8
+NGINX_HOST=example.com
9
+NGINX_HTTP_CF_RAY=abc123def456
10
+NGINX_HTTP_REFERER=https://example.com
11
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
12
+NGINX_HTTP_X_FORWARDED_FOR=192.168.1.50, 10.0.0.1
13
NGINX_PID=9876
5
-NGINX_REQUEST_ID=8f3ebc1e38fbb92f
6
-NGINX_REQUEST_LENGTH=345
14
+NGINX_PIPELINED=no
15
+NGINX_QUERY_STRING=param=value
16
NGINX_REMOTE_ADDR=192.168.1.100
8
-NGINX_REMOTE_USER=john_doe
17
NGINX_REMOTE_PORT=54321
10
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:05 +0000
11
-MESSAGE=GET /index.html HTTP/1.1
18
+NGINX_REMOTE_USER=john_doe
19
+NGINX_REQUEST_ID=8f3ebc1e38fbb92f
20
+NGINX_REQUEST_LENGTH=345
21
+NGINX_REQUEST_METHOD=GET
22
+NGINX_REQUEST_TIME=0.123
23
NGINX_REQUEST_URI=/index.html?param=value
13
-NGINX_QUERY_STRING=param=value
24
+NGINX_SCHEME=https
25
+NGINX_SERVER_PROTOCOL=HTTP/1.1
26
+NGINX_SSL_CIPHER=AES256-SHA256
27
+NGINX_SSL_PROTOCOL=TLSv1.2
28
NGINX_STATUS=200
15
-PRIORITY=6
29
NGINX_STATUS_FAMILY=2xx
17
-NGINX_BODY_BYTES_SENT=5432
18
-NGINX_BYTES_SENT=6543
19
-NGINX_HTTP_REFERER=https://example.com
20
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
21
-NGINX_HTTP_X_FORWARDED_FOR=192.168.1.50, 10.0.0.1
22
-NGINX_HOST=example.com
23
-NGINX_REQUEST_TIME=0.123
30
+NGINX_TIMESTAMP_SEC=1644997905.123
31
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:05 +0000
32
NGINX_UPSTREAM=10.0.0.2:8080
33
+NGINX_UPSTREAM_CACHE_STATUS=MISS
34
NGINX_UPSTREAM_CONNECT_TIME=0.045
35
NGINX_UPSTREAM_HEADER_TIME=0.020
27
-NGINX_UPSTREAM_RESPONSE_TIME=0.058
36
NGINX_UPSTREAM_RESPONSE_LENGTH=7890
29
-NGINX_UPSTREAM_CACHE_STATUS=MISS
30
-NGINX_SSL_PROTOCOL=TLSv1.2
31
-NGINX_SSL_CIPHER=AES256-SHA256
32
-NGINX_SCHEME=https
33
-NGINX_REQUEST_METHOD=GET
34
-NGINX_SERVER_PROTOCOL=HTTP/1.1
35
-NGINX_PIPELINED=no
36
-NGINX_GZIP_RATIO=2.1
37
-NGINX_HTTP_CF_RAY=abc123def456
38
-NGINX_GEOIP_COUNTRY_CODE=US
37
+NGINX_UPSTREAM_RESPONSE_TIME=0.058
38
+PRIORITY=6
39
SYSLOG_IDENTIFIER=nginx-log
40
41
-NGINX_TIMESTAMP_SEC=1644997910.789
41
+MESSAGE=POST /api/update HTTP/1.1
42
+NGINX_BODY_BYTES_SENT=0
43
+NGINX_BYTES_SENT=123
44
NGINX_CONNECTION=54321
45
NGINX_CONNECTION_REQUESTS=10
46
+NGINX_HOST=api.example.com
47
+NGINX_HTTP_USER_AGENT=curl/7.68.0
48
NGINX_PID=5432
45
-NGINX_REQUEST_ID=4a7bca5e19d3f8e7
46
-NGINX_REQUEST_LENGTH=432
49
+NGINX_PIPELINED=yes
50
NGINX_REMOTE_ADDR=10.0.0.3
51
NGINX_REMOTE_PORT=12345
49
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:10 +0000
50
-MESSAGE=POST /api/update HTTP/1.1
52
+NGINX_REQUEST_ID=4a7bca5e19d3f8e7
53
+NGINX_REQUEST_LENGTH=432
54
+NGINX_REQUEST_METHOD=POST
55
+NGINX_REQUEST_TIME=0.032
56
NGINX_REQUEST_URI=/api/update
57
+NGINX_SCHEME=http
58
+NGINX_SERVER_PROTOCOL=HTTP/1.1
59
NGINX_STATUS=204
53
-PRIORITY=6
60
NGINX_STATUS_FAMILY=2xx
55
-NGINX_BODY_BYTES_SENT=0
56
-NGINX_BYTES_SENT=123
57
-NGINX_HTTP_USER_AGENT=curl/7.68.0
58
-NGINX_HOST=api.example.com
59
-NGINX_REQUEST_TIME=0.032
61
+NGINX_TIMESTAMP_SEC=1644997910.789
62
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:10 +0000
63
NGINX_UPSTREAM=backend-server-1:8080
64
NGINX_UPSTREAM_CONNECT_TIME=0.012
65
NGINX_UPSTREAM_HEADER_TIME=0.020
63
-NGINX_UPSTREAM_RESPONSE_TIME=0.010
66
NGINX_UPSTREAM_RESPONSE_LENGTH=0
65
-NGINX_SCHEME=http
66
-NGINX_REQUEST_METHOD=POST
67
-NGINX_SERVER_PROTOCOL=HTTP/1.1
68
-NGINX_PIPELINED=yes
67
+NGINX_UPSTREAM_RESPONSE_TIME=0.010
68
+PRIORITY=6
69
SYSLOG_IDENTIFIER=nginx-log
70
71
-NGINX_TIMESTAMP_SEC=1644997920.456
71
+MESSAGE=GET /page?param1=value1¶m2=value2 HTTP/2.0
72
+NGINX_BODY_BYTES_SENT=0
73
+NGINX_BYTES_SENT=0
74
NGINX_CONNECTION=98765
75
NGINX_CONNECTION_REQUESTS=15
76
+NGINX_GEOIP_COUNTRY_CODE=GB
77
+NGINX_HOST=example.org
78
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
79
NGINX_PID=1234
75
-NGINX_REQUEST_ID=63f8ad2c3e1b4090
76
-NGINX_REQUEST_LENGTH=567
80
+NGINX_PIPELINED=no
81
+NGINX_QUERY_STRING=param1=value1¶m2=value2
82
NGINX_REMOTE_ADDR=2001:0db8:85a3:0000:0000:8a2e:0370:7334
78
-NGINX_REMOTE_USER=alice
83
NGINX_REMOTE_PORT=6789
80
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:20 +0000
81
-MESSAGE=GET /page?param1=value1¶m2=value2 HTTP/2.0
84
+NGINX_REMOTE_USER=alice
85
+NGINX_REQUEST_ID=63f8ad2c3e1b4090
86
+NGINX_REQUEST_LENGTH=567
87
+NGINX_REQUEST_METHOD=GET
88
+NGINX_REQUEST_TIME=0.045
89
NGINX_REQUEST_URI=/page?param1=value1¶m2=value2
83
-NGINX_QUERY_STRING=param1=value1¶m2=value2
90
+NGINX_SCHEME=https
91
+NGINX_SERVER_PROTOCOL=HTTP/2.0
92
NGINX_STATUS=404
85
-PRIORITY=5
93
NGINX_STATUS_FAMILY=4xx
87
-NGINX_BODY_BYTES_SENT=0
88
-NGINX_BYTES_SENT=0
89
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
90
-NGINX_HOST=example.org
91
-NGINX_REQUEST_TIME=0.045
94
+NGINX_TIMESTAMP_SEC=1644997920.456
95
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:20 +0000
96
NGINX_UPSTREAM_CONNECT_TIME=0.0
97
NGINX_UPSTREAM_HEADER_TIME=0.0
94
-NGINX_UPSTREAM_RESPONSE_TIME=0.0
98
NGINX_UPSTREAM_RESPONSE_LENGTH=0
96
-NGINX_SCHEME=https
97
-NGINX_REQUEST_METHOD=GET
98
-NGINX_SERVER_PROTOCOL=HTTP/2.0
99
-NGINX_PIPELINED=no
100
-NGINX_GEOIP_COUNTRY_CODE=GB
99
+NGINX_UPSTREAM_RESPONSE_TIME=0.0
100
+PRIORITY=5
101
SYSLOG_IDENTIFIER=nginx-log
102
103
-NGINX_TIMESTAMP_SEC=1644997930.987
103
+MESSAGE=PUT /api/update HTTP/1.1
104
+NGINX_BODY_BYTES_SENT=543
105
+NGINX_BYTES_SENT=876
106
NGINX_CONNECTION=123
107
NGINX_CONNECTION_REQUESTS=3
108
+NGINX_GEOIP_COUNTRY_CODE=CA
109
+NGINX_GZIP_RATIO=1.8
110
+NGINX_HOST=api.example.com
111
+NGINX_HTTP_CF_RAY=xyz789abc123
112
+NGINX_HTTP_REFERER=https://example.com/page
113
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
114
NGINX_PID=5678
107
-NGINX_REQUEST_ID=9e632a5b24c18f76
108
-NGINX_REQUEST_LENGTH=234
115
+NGINX_PIPELINED=yes
116
NGINX_REMOTE_ADDR=192.168.0.1
110
-NGINX_REMOTE_USER=jane_doe
117
NGINX_REMOTE_PORT=9876
112
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:30 +0000
113
-MESSAGE=PUT /api/update HTTP/1.1
118
+NGINX_REMOTE_USER=jane_doe
119
+NGINX_REQUEST_ID=9e632a5b24c18f76
120
+NGINX_REQUEST_LENGTH=234
121
+NGINX_REQUEST_METHOD=PUT
122
+NGINX_REQUEST_TIME=0.123
123
NGINX_REQUEST_URI=/api/update
124
+NGINX_SCHEME=https
125
+NGINX_SERVER_PROTOCOL=HTTP/1.1
126
+NGINX_SSL_CIPHER=AES256-SHA256
127
+NGINX_SSL_PROTOCOL=TLSv1.2
128
NGINX_STATUS=500
116
-PRIORITY=3
129
NGINX_STATUS_FAMILY=5xx
118
-NGINX_BODY_BYTES_SENT=543
119
-NGINX_BYTES_SENT=876
120
-NGINX_HTTP_REFERER=https://example.com/page
121
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
122
-NGINX_HOST=api.example.com
123
-NGINX_REQUEST_TIME=0.123
130
+NGINX_TIMESTAMP_SEC=1644997930.987
131
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:30 +0000
132
NGINX_UPSTREAM=backend-server-2:8080
133
+NGINX_UPSTREAM_CACHE_STATUS=HIT
134
NGINX_UPSTREAM_CONNECT_TIME=0.045
135
NGINX_UPSTREAM_HEADER_TIME=0.020
127
-NGINX_UPSTREAM_RESPONSE_TIME=0.058
136
NGINX_UPSTREAM_RESPONSE_LENGTH=7890
129
-NGINX_UPSTREAM_CACHE_STATUS=HIT
130
-NGINX_SSL_PROTOCOL=TLSv1.2
131
-NGINX_SSL_CIPHER=AES256-SHA256
132
-NGINX_SCHEME=https
133
-NGINX_REQUEST_METHOD=PUT
134
-NGINX_SERVER_PROTOCOL=HTTP/1.1
135
-NGINX_PIPELINED=yes
136
-NGINX_GZIP_RATIO=1.8
137
-NGINX_HTTP_CF_RAY=xyz789abc123
138
-NGINX_GEOIP_COUNTRY_CODE=CA
137
+NGINX_UPSTREAM_RESPONSE_TIME=0.058
138
+PRIORITY=3
139
SYSLOG_IDENTIFIER=nginx-log
140
141
-NGINX_TIMESTAMP_SEC=1644997940.234
141
+MESSAGE=DELETE /api/resource HTTP/2.0
142
+NGINX_BODY_BYTES_SENT=0
143
+NGINX_BYTES_SENT=123
144
NGINX_CONNECTION=9876
145
NGINX_CONNECTION_REQUESTS=8
146
+NGINX_HOST=api.example.com
147
+NGINX_HTTP_USER_AGENT=curl/7.68.0
148
NGINX_PID=4321
145
-NGINX_REQUEST_ID=1b6c59c8aef7d24a
146
-NGINX_REQUEST_LENGTH=456
149
+NGINX_PIPELINED=no
150
NGINX_REMOTE_ADDR=203.0.113.1
151
NGINX_REMOTE_PORT=5432
149
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:40 +0000
150
-MESSAGE=DELETE /api/resource HTTP/2.0
152
+NGINX_REQUEST_ID=1b6c59c8aef7d24a
153
+NGINX_REQUEST_LENGTH=456
154
+NGINX_REQUEST_METHOD=DELETE
155
+NGINX_REQUEST_TIME=0.032
156
NGINX_REQUEST_URI=/api/resource
157
+NGINX_SCHEME=http
158
+NGINX_SERVER_PROTOCOL=HTTP/2.0
159
NGINX_STATUS=204
153
-PRIORITY=6
160
NGINX_STATUS_FAMILY=2xx
155
-NGINX_BODY_BYTES_SENT=0
156
-NGINX_BYTES_SENT=123
157
-NGINX_HTTP_USER_AGENT=curl/7.68.0
158
-NGINX_HOST=api.example.com
159
-NGINX_REQUEST_TIME=0.032
161
+NGINX_TIMESTAMP_SEC=1644997940.234
162
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:40 +0000
163
NGINX_UPSTREAM=backend-server-1:8080
164
NGINX_UPSTREAM_CONNECT_TIME=0.012
165
NGINX_UPSTREAM_HEADER_TIME=0.020
163
-NGINX_UPSTREAM_RESPONSE_TIME=0.010
166
NGINX_UPSTREAM_RESPONSE_LENGTH=0
165
-NGINX_SCHEME=http
166
-NGINX_REQUEST_METHOD=DELETE
167
-NGINX_SERVER_PROTOCOL=HTTP/2.0
168
-NGINX_PIPELINED=no
167
+NGINX_UPSTREAM_RESPONSE_TIME=0.010
168
+PRIORITY=6
169
SYSLOG_IDENTIFIER=nginx-log
170
171
-NGINX_TIMESTAMP_SEC=1644997950.789
171
+MESSAGE=GET /profile?user=bob HTTP/1.1
172
+NGINX_BODY_BYTES_SENT=1234
173
+NGINX_BYTES_SENT=2345
174
NGINX_CONNECTION=5432
175
NGINX_CONNECTION_REQUESTS=12
176
+NGINX_GEOIP_COUNTRY_CODE=US
177
+NGINX_HOST=example.com
178
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
179
NGINX_PID=6543
175
-NGINX_REQUEST_ID=72692d781d0b8a4f
176
-NGINX_REQUEST_LENGTH=789
180
+NGINX_PIPELINED=yes
181
+NGINX_QUERY_STRING=user=bob
182
NGINX_REMOTE_ADDR=198.51.100.2
178
-NGINX_REMOTE_USER=bob
183
NGINX_REMOTE_PORT=8765
180
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:50 +0000
181
-MESSAGE=GET /profile?user=bob HTTP/1.1
184
+NGINX_REMOTE_USER=bob
185
+NGINX_REQUEST_ID=72692d781d0b8a4f
186
+NGINX_REQUEST_LENGTH=789
187
+NGINX_REQUEST_METHOD=GET
188
+NGINX_REQUEST_TIME=0.065
189
NGINX_REQUEST_URI=/profile?user=bob
183
-NGINX_QUERY_STRING=user=bob
190
+NGINX_SCHEME=https
191
+NGINX_SERVER_PROTOCOL=HTTP/1.1
192
+NGINX_SSL_CIPHER=AES128-GCM-SHA256
193
+NGINX_SSL_PROTOCOL=TLSv1.3
194
NGINX_STATUS=200
185
-PRIORITY=6
195
NGINX_STATUS_FAMILY=2xx
187
-NGINX_BODY_BYTES_SENT=1234
188
-NGINX_BYTES_SENT=2345
189
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
190
-NGINX_HOST=example.com
191
-NGINX_REQUEST_TIME=0.065
196
+NGINX_TIMESTAMP_SEC=1644997950.789
197
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:50 +0000
198
NGINX_UPSTREAM=10.0.0.2:8080
199
+NGINX_UPSTREAM_CACHE_STATUS=MISS
200
NGINX_UPSTREAM_CONNECT_TIME=0.045
201
NGINX_UPSTREAM_HEADER_TIME=0.020
195
-NGINX_UPSTREAM_RESPONSE_TIME=0.058
202
NGINX_UPSTREAM_RESPONSE_LENGTH=7890
197
-NGINX_UPSTREAM_CACHE_STATUS=MISS
198
-NGINX_SSL_PROTOCOL=TLSv1.3
199
-NGINX_SSL_CIPHER=AES128-GCM-SHA256
200
-NGINX_SCHEME=https
201
-NGINX_REQUEST_METHOD=GET
202
-NGINX_SERVER_PROTOCOL=HTTP/1.1
203
-NGINX_PIPELINED=yes
204
-NGINX_GEOIP_COUNTRY_CODE=US
203
+NGINX_UPSTREAM_RESPONSE_TIME=0.058
204
+PRIORITY=6
205
SYSLOG_IDENTIFIER=nginx-log
206
207
-NGINX_TIMESTAMP_SEC=1644997960.321
207
+MESSAGE=GET /dashboard HTTP/2.0
208
+NGINX_BODY_BYTES_SENT=0
209
+NGINX_BYTES_SENT=123
210
NGINX_CONNECTION=65432
211
NGINX_CONNECTION_REQUESTS=7
212
+NGINX_HOST=dashboard.example.org
213
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
214
NGINX_PID=7890
211
-NGINX_REQUEST_ID=c3e158d41e75a9d7
212
-NGINX_REQUEST_LENGTH=321
215
+NGINX_PIPELINED=no
216
NGINX_REMOTE_ADDR=203.0.113.2
217
NGINX_REMOTE_PORT=9876
215
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:60 +0000
216
-MESSAGE=GET /dashboard HTTP/2.0
218
+NGINX_REQUEST_ID=c3e158d41e75a9d7
219
+NGINX_REQUEST_LENGTH=321
220
+NGINX_REQUEST_METHOD=GET
221
+NGINX_REQUEST_TIME=0.032
222
NGINX_REQUEST_URI=/dashboard
223
+NGINX_SCHEME=https
224
+NGINX_SERVER_PROTOCOL=HTTP/2.0
225
NGINX_STATUS=301
219
-PRIORITY=6
226
NGINX_STATUS_FAMILY=3xx
221
-NGINX_BODY_BYTES_SENT=0
222
-NGINX_BYTES_SENT=123
223
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
224
-NGINX_HOST=dashboard.example.org
225
-NGINX_REQUEST_TIME=0.032
227
+NGINX_TIMESTAMP_SEC=1644997960.321
228
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:60 +0000
229
NGINX_UPSTREAM_CONNECT_TIME=0.0
230
NGINX_UPSTREAM_HEADER_TIME=0.0
228
-NGINX_UPSTREAM_RESPONSE_TIME=0.0
231
NGINX_UPSTREAM_RESPONSE_LENGTH=0
230
-NGINX_SCHEME=https
231
-NGINX_REQUEST_METHOD=GET
232
-NGINX_SERVER_PROTOCOL=HTTP/2.0
233
-NGINX_PIPELINED=no
232
+NGINX_UPSTREAM_RESPONSE_TIME=0.0
233
+PRIORITY=6
234
SYSLOG_IDENTIFIER=nginx-log
235
236
-NGINX_TIMESTAMP_SEC=1644997970.555
236
+MESSAGE=POST /submit-form HTTP/1.1
237
+NGINX_BODY_BYTES_SENT=876
238
+NGINX_BYTES_SENT=987
239
NGINX_CONNECTION=8765
240
NGINX_CONNECTION_REQUESTS=9
241
+NGINX_HOST=example.com
242
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
243
NGINX_PID=8765
240
-NGINX_REQUEST_ID=f9f6e8235de54af4
241
-NGINX_REQUEST_LENGTH=654
244
+NGINX_PIPELINED=yes
245
NGINX_REMOTE_ADDR=10.0.0.4
246
NGINX_REMOTE_PORT=12345
244
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:70 +0000
245
-MESSAGE=POST /submit-form HTTP/1.1
247
+NGINX_REQUEST_ID=f9f6e8235de54af4
248
+NGINX_REQUEST_LENGTH=654
249
+NGINX_REQUEST_METHOD=POST
250
+NGINX_REQUEST_TIME=0.045
251
NGINX_REQUEST_URI=/submit-form
252
+NGINX_SCHEME=http
253
+NGINX_SERVER_PROTOCOL=HTTP/1.1
254
NGINX_STATUS=201
248
-PRIORITY=6
255
NGINX_STATUS_FAMILY=2xx
250
-NGINX_BODY_BYTES_SENT=876
251
-NGINX_BYTES_SENT=987
252
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
253
-NGINX_HOST=example.com
254
-NGINX_REQUEST_TIME=0.045
256
+NGINX_TIMESTAMP_SEC=1644997970.555
257
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:70 +0000
258
NGINX_UPSTREAM=backend-server-3:8080
259
NGINX_UPSTREAM_CONNECT_TIME=0.012
260
NGINX_UPSTREAM_HEADER_TIME=0.020
258
-NGINX_UPSTREAM_RESPONSE_TIME=0.010
261
NGINX_UPSTREAM_RESPONSE_LENGTH=0
260
-NGINX_SCHEME=http
261
-NGINX_REQUEST_METHOD=POST
262
-NGINX_SERVER_PROTOCOL=HTTP/1.1
263
-NGINX_PIPELINED=yes
262
+NGINX_UPSTREAM_RESPONSE_TIME=0.010
263
+PRIORITY=6
264
SYSLOG_IDENTIFIER=nginx-log
265
266
-NGINX_TIMESTAMP_SEC=1644997980.987
266
+MESSAGE=GET /contact HTTP/1.1
267
+NGINX_BODY_BYTES_SENT=0
268
+NGINX_BYTES_SENT=0
269
NGINX_CONNECTION=23456
270
NGINX_CONNECTION_REQUESTS=6
271
+NGINX_GEOIP_COUNTRY_CODE=FR
272
+NGINX_HOST=example.org
273
+NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
274
NGINX_PID=3456
270
-NGINX_REQUEST_ID=2ec3e8859e7a406c
271
-NGINX_REQUEST_LENGTH=432
275
+NGINX_PIPELINED=no
276
NGINX_REMOTE_ADDR=198.51.100.3
273
-NGINX_REMOTE_USER=mary
277
NGINX_REMOTE_PORT=5678
275
-NGINX_TIME_LOCAL=19/Feb/2023:14:15:80 +0000
276
-MESSAGE=GET /contact HTTP/1.1
278
+NGINX_REMOTE_USER=mary
279
+NGINX_REQUEST_ID=2ec3e8859e7a406c
280
+NGINX_REQUEST_LENGTH=432
281
+NGINX_REQUEST_METHOD=GET
282
+NGINX_REQUEST_TIME=0.032
283
NGINX_REQUEST_URI=/contact
284
+NGINX_SCHEME=https
285
+NGINX_SERVER_PROTOCOL=HTTP/1.1
286
NGINX_STATUS=404
279
-PRIORITY=5
287
NGINX_STATUS_FAMILY=4xx
281
-NGINX_BODY_BYTES_SENT=0
282
-NGINX_BYTES_SENT=0
283
-NGINX_HTTP_USER_AGENT=Mozilla/5.0 (Linux; Android 10; Pixel 3)
284
-NGINX_HOST=example.org
285
-NGINX_REQUEST_TIME=0.032
288
+NGINX_TIMESTAMP_SEC=1644997980.987
289
+NGINX_TIME_LOCAL=19/Feb/2023:14:15:80 +0000
290
NGINX_UPSTREAM_CONNECT_TIME=0.0
291
NGINX_UPSTREAM_HEADER_TIME=0.0
288
-NGINX_UPSTREAM_RESPONSE_TIME=0.0
292
NGINX_UPSTREAM_RESPONSE_LENGTH=0
290
-NGINX_SCHEME=https
291
-NGINX_REQUEST_METHOD=GET
292
-NGINX_SERVER_PROTOCOL=HTTP/1.1
293
-NGINX_PIPELINED=no
294
-NGINX_GEOIP_COUNTRY_CODE=FR
293
+NGINX_UPSTREAM_RESPONSE_TIME=0.0
294
+PRIORITY=5
295
SYSLOG_IDENTIFIER=nginx-log
296
collectors/log2journal/tests.sh
+12
-8
@@ -68,10 +68,11 @@ echo >&2 "Testing command line parsing..."
68
test_log2journal_config /dev/null "${tests}/full.output" --show-config \
69
--prefix=NGINX_ \
70
--filename-key NGINX_LOG_FILENAME \
71
- --duplicate PRIORITY=NGINX_STATUS \
72
- --duplicate=NGINX_STATUS_FAMILY=NGINX_STATUS,NGINX_METHOD \
71
--inject SYSLOG_IDENTIFIER=nginx-log \
72
--inject=SYSLOG_IDENTIFIER2=nginx-log2 \
73
+ --inject 'PRIORITY=${NGINX_STATUS}' \
74
+ --inject='NGINX_STATUS_FAMILY=${NGINX_STATUS}${NGINX_METHOD}' \
75
+ --rewrite 'PRIORITY=//${NGINX_STATUS}/inject,dont-stop' \
76
--rewrite "PRIORITY=/^[123]/6" \
77
--rewrite='PRIORITY=|^4|5' \
78
'--rewrite=PRIORITY=-^5-3' \
@@ -115,7 +116,8 @@ test_log2journal() {
116
printf >&2 "running: "
117
printf >&2 "%q " "${log2journal_bin}" "${@}"
118
printf >&2 "\n"
118
- echo >&2 "using as input: ${in}"
119
+ echo >&2 "using as input : ${in}"
120
+ echo >&2 "expecting output: ${out}"
121
122
[ -f output ] && rm output
123
@@ -125,7 +127,7 @@ test_log2journal() {
127
[ $ret -ne 0 ] && echo >&2 "${log2journal_bin} exited with code: $ret" && cat output && exit 1
128
129
diff "${out}" output
128
- [ $? -ne -0 ] && echo >&2 "${log2journal_bin} output does not match!" && cat output && exit 1
130
+ [ $? -ne -0 ] && echo >&2 "${log2journal_bin} output does not match! - here is what we got:" && cat output && exit 1
131
132
echo >&2 "OK"
133
echo >&2
@@ -136,7 +138,9 @@ test_log2journal() {
138
echo >&2
139
echo >&2 "Testing parsing and output..."
140
139
-test_log2journal ${tests}/json.log ${tests}/json.output json
140
-test_log2journal ${tests}/json.log ${tests}/json-include.output json --include "OBJECT"
141
-test_log2journal ${tests}/json.log ${tests}/json-exclude.output json --exclude "ARRAY[^2]"
142
-test_log2journal ${tests}/nginx-json.log ${tests}/nginx-json.output -f "${script_dir}/log2journal.d/nginx-json.yaml"
141
+test_log2journal "${tests}/json.log" "${tests}/json.output" json
142
+test_log2journal "${tests}/json.log" "${tests}/json-include.output" json --include "OBJECT"
143
+test_log2journal "${tests}/json.log" "${tests}/json-exclude.output" json --exclude "ARRAY[^2]"
144
+test_log2journal "${tests}/nginx-json.log" "${tests}/nginx-json.output" -f "${script_dir}/log2journal.d/nginx-json.yaml"
145
+test_log2journal "${tests}/nginx-combined.log" "${tests}/nginx-combined.output" -f "${script_dir}/log2journal.d/nginx-combined.yaml"
146
+test_log2journal "${tests}/logfmt.log" "${tests}/logfmt.output" -f "${tests}/logfmt.yaml"