@cryptotaxi247 / netdata-1 / commits / 1f0164ede

Log2journal yaml configuration support (#16479)

* allow command line parameters to be specified with = or as separate arguments without =; memory errors handling * add yaml support to log2journal * updated log2journal docs * fix command line parsing * fix codeql warnings

Costa Tsaousis committed Nov 27, 2023 at 13:05 UTC 1f0164ede4f9c92ba9814c016cde9784e2944d87

4 files changed +1385 -138

Makefile.am

@@ -1323,6 +1323,7 @@ if ENABLE_LOG2JOURNAL

1323 log2journal_SOURCES = $(LOG2JOURNAL_FILES)

1324 log2journal_LDADD = \

1325 $(OPTIONAL_PCRE2_LIBS) \

1326 + $(OPTIONAL_YAML_LIBS) \

1327 $(NULL)

1328 endif

1329

configure.ac

+21

@@ -852,6 +852,26 @@ fi

852 AC_MSG_RESULT([${enable_jsonc}])

853 AM_CONDITIONAL([ENABLE_JSONC], [test "${enable_jsonc}" = "yes"])

854

855 +# -----------------------------------------------------------------------------

856 +# libyaml

857 +

858 +PKG_CHECK_MODULES(

859 + [LIBYAML],

860 + [yaml-0.1],

861 + [AC_CHECK_LIB(

862 + [yaml],

863 + [yaml_parser_initialize],

864 + [have_libyaml=yes],

865 + [have_libyaml=no]

866 + )],

867 + [have_libyaml=no]

868 +)

869 +

870 +if test "x$have_libyaml" = "xyes"; then

871 + AC_DEFINE([HAVE_LIBYAML], [1], [libyaml usability])

872 + OPTIONAL_YAML_LIBS="-lyaml"

873 +fi

874 +

875 # -----------------------------------------------------------------------------

876 # YAML

877

@@ -1983,6 +2003,7 @@ AC_SUBST([OPTIONAL_UV_LIBS])

2003 AC_SUBST([OPTIONAL_LZ4_LIBS])

2004 AC_SUBST([OPTIONAL_BROTLIENC_LIBS])

2005 AC_SUBST([OPTIONAL_BROTLIDEC_LIBS])

2006 +AC_SUBST([OPTIONAL_YAML_LIBS])

2007 AC_SUBST([OPTIONAL_CURL_LIBS])

2008 AC_SUBST([OPTIONAL_PCRE2_LIBS])

2009 AC_SUBST([OPTIONAL_ZSTD_LIBS])

libnetdata/log/log2journal.c

+1246 -129

@@ -17,6 +17,10 @@

17 #define PCRE2_CODE_UNIT_WIDTH 8

18 #include <pcre2.h>

20 +#ifdef HAVE_LIBYAML

21 +#include <yaml.h>

22 +#endif

23 +

24 #define MAX_OUTPUT_KEYS 1024

25 #define OVECCOUNT (MAX_OUTPUT_KEYS * 3) // should be a multiple of 3

26 #define MAX_LINE_LENGTH (1024 * 1024)

@@ -28,6 +32,106 @@

32 #define MAX_KEY_LEN 64 // according to systemd-journald

33 #define MAX_VALUE_LEN (48 * 1024) // according to systemd-journald

35 +struct key_rewrite;

36 +static pcre2_code *jb_compile_pcre2_pattern(const char *pattern);

37 +static bool parse_replacement_pattern(struct key_rewrite *rw);

38 +

39 +#define YAML_CONFIG_NGINX_COMBINED \

40 + "# Netdata log2journal Configuration Template\n" \

41 + "# The following parses nginx log files using the combined format.\n" \

42 + "\n" \

43 + "# The PCRE2 pattern to match log entries and give names to the fields.\n" \

44 + "# The journal will have these names, so follow their rules. You can\n" \

45 + "# initiate an extended PCRE2 pattern by starting the pattern with (?x)\n" \

46 + "pattern: |\n" \

47 + " (?x) # Enable PCRE2 extended mode\n" \

48 + " ^\n" \

49 + " (?<NGINX_REMOTE_ADDR>[^ ]+) \\s - \\s # NGINX_REMOTE_ADDR\n" \

50 + " (?<NGINX_REMOTE_USER>[^ ]+) \\s # NGINX_REMOTE_USER\n" \

51 + " \\[\n" \

52 + " (?<NGINX_TIME_LOCAL>[^\\]]+) # NGINX_TIME_LOCAL\n" \

53 + " \\]\n" \

54 + " \\s+ \"\n" \

55 + " (?<MESSAGE>\n" \

56 + " (?<NGINX_METHOD>[A-Z]+) \\s+ # NGINX_METHOD\n" \

57 + " (?<NGINX_URL>[^ ]+) \\s+\n" \

58 + " HTTP/(?<NGINX_HTTP_VERSION>[^\"]+)\n" \

59 + " )\n" \

60 + " \" \\s+\n" \

61 + " (?<NGINX_STATUS>\\d+) \\s+ # NGINX_STATUS\n" \

62 + " (?<NGINX_BODY_BYTES_SENT>\\d+) \\s+ # NGINX_BODY_BYTES_SENT\n" \

63 + " \"(?<NGINX_HTTP_REFERER>[^\"]*)\" \\s+ # NGINX_HTTP_REFERER\n" \

64 + " \"(?<NGINX_HTTP_USER_AGENT>[^\"]*)\" # NGINX_HTTP_USER_AGENT\n" \

65 + "\n" \

66 + "# When log2journal can detect the filename of each log entry (tail gives it\n" \

67 + "# only when it tails multiple files), this key will be used to send the\n" \

68 + "# filename to the journals.\n" \

69 + "filename:\n" \

70 + " key: NGINX_LOG_FILENAME\n" \

71 + "\n" \

72 + "# Duplicate fields under a different name. You can duplicate multiple fields\n" \

73 + "# to a new one and then use rewrite rules to change its value.\n" \

74 + "duplicate:\n" \

75 + "\n" \

76 + " # we insert the field PRIORITY as a copy of NGINX_STATUS.\n" \

77 + " - key: PRIORITY\n" \

78 + " values_of:\n" \

79 + " - NGINX_STATUS\n" \

80 + "\n" \

81 + " # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.\n" \

82 + " - key: NGINX_STATUS_FAMILY\n" \

83 + " values_of: \n" \

84 + " - NGINX_STATUS\n" \

85 + "\n" \

86 + "# Inject constant fields into the journal logs.\n" \

87 + "inject:\n" \

88 + " - key: SYSLOG_IDENTIFIER\n" \

89 + " value: \"nginx-log\"\n" \

90 + "\n" \

91 + "# Rewrite the value of fields (including the duplicated ones).\n" \

92 + "# The search pattern can have named groups, and the replace pattern can use\n" \

93 + "# them as ${name}.\n" \

94 + "rewrite:\n" \

95 + " # PRIORTY is a duplicate of NGINX_STATUS\n" \

96 + " # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug\n" \

97 + " - key: \"PRIORITY\"\n" \

98 + " search: \"^[123]\"\n" \

99 + " replace: 6\n" \

100 + "\n" \

101 + " - key: \"PRIORITY\"\n" \

102 + " search: \"^4\"\n" \

103 + " replace: 5\n" \

104 + "\n" \

105 + " - key: \"PRIORITY\"\n" \

106 + " search: \"^5\"\n" \

107 + " replace: 3\n" \

108 + "\n" \

109 + " - key: \"PRIORITY\"\n" \

110 + " search: \".*\"\n" \

111 + " replace: 4\n" \

112 + " \n" \

113 + " # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS\n" \

114 + " - key: \"NGINX_STATUS_FAMILY\"\n" \

115 + " search: \"^(?<first_digit>[1-5])\"\n" \

116 + " replace: \"${first_digit}xx\"\n" \

117 + "\n" \

118 + " - key: \"NGINX_STATUS_FAMILY\"\n" \

119 + " search: \".*\"\n" \

120 + " replace: \"UNKNOWN\"\n" \

121 + "\n" \

122 + "# Control what to do when input logs do not match the main PCRE2 pattern.\n" \

123 + "unmatched:\n" \

124 + " # The journal key to log the PCRE2 error message to.\n" \

125 + " # Set this to MESSAGE, so you to see the error in the log.\n" \

126 + " key: MESSAGE\n" \

127 + " \n" \

128 + " # Inject static fields to the unmatched entries.\n" \

129 + " # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.\n" \

130 + " inject:\n" \

131 + " - key: PRIORITY\n" \

132 + " value: 1\n" \

133 + "\n"

134 +

135 void display_help(const char *name) {

136 printf("\n");

137 printf("Netdata log2journal " PACKAGE_VERSION "\n");

@@ -41,46 +145,57 @@ void display_help(const char *name) {

145 printf("\n");

146 printf("Options:\n");

147 printf("\n");

44 - printf(" --filename-key=KEY\n");

148 + printf(" --file /path/to/file.yaml\n");

149 + printf(" Read yaml configuration file for instructions.\n");

150 + printf("\n");

151 + printf(" --config CONFIG_NAME\n");

152 + printf(" Run with the internal configuration named CONFIG_NAME\n");

153 + printf(" Available internal configs: nginx-combined\n");

154 + printf("\n");

155 + printf(" --show-config\n");

156 + printf(" Show the configuration in yaml format before starting the job.\n");

157 + printf(" This is also an easy way to convert command line parameters to yaml.\n");

158 + printf("\n");

159 + printf(" --filename-key KEY\n");

160 printf(" Add a field with KEY as the key and the current filename as value.\n");

161 printf(" Automatically detects filenames when piped after 'tail -F',\n");

162 printf(" and tail matches multiple filenames.\n");

163 printf(" To inject the filename when tailing a single file, use --inject.\n");

164 printf("\n");

50 - printf(" --unmatched-key=KEY\n");

165 + printf(" --unmatched-key KEY\n");

166 printf(" Include unmatched log entries in the output with KEY as the field name.\n");

167 printf(" Use this to include unmatched entries to the output stream.\n");

168 printf(" Usually it should be set to --unmatched-key=MESSAGE so that the\n");

169 printf(" unmatched entry will appear as the log message in the journals.\n");

170 printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n");

171 printf("\n");

57 - printf(" --duplicate=TARGET=KEY1[,KEY2[,KEY3[,...]]\n");

172 + printf(" --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]\n");

173 printf(" Create a new key called TARGET, duplicating the values of the keys\n");

174 printf(" given. Useful for further processing. When multiple keys are given,\n");

175 printf(" their values are separated by comma.\n");

176 printf(" Up to %d duplications can be given on the command line, and up to\n", MAX_KEY_DUPS);

177 printf(" %d keys per duplication command are allowed.\n", MAX_KEY_DUPS_KEYS);

178 printf("\n");

64 - printf(" --inject=LINE\n");

179 + printf(" --inject LINE\n");

180 printf(" Inject constant fields to the output (both matched and unmatched logs).\n");

181 printf(" --inject entries are added to unmatched lines too, when their key is\n");

182 printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n");

183 printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS);

184 printf("\n");

70 - printf(" --inject-unmatched=LINE\n");

185 + printf(" --inject-unmatched LINE\n");

186 printf(" Inject lines into the output for each unmatched log entry.\n");

187 printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n");

188 printf(" lines as errors, so that they can easily be spotted in the journals.\n");

189 printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS);

190 printf("\n");

76 - printf(" --rewrite=KEY=/SearchPattern/ReplacePattern\n");

191 + printf(" --rewrite KEY=/SearchPattern/ReplacePattern\n");

192 printf(" Apply a rewrite rule to the values of a specific key.\n");

193 printf(" The first character after KEY= is the separator, which should also\n");

194 printf(" be used between the search pattern and the replacement pattern.\n");

195 printf(" The search pattern is a PCRE2 regular expression, and the replacement\n");

196 printf(" pattern supports literals and named capture groups from the search pattern.\n");

197 printf(" Example:\n");

83 - printf(" --rewrite=DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n");

198 + printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n");

199 printf(" ${day}/${month}/${year}\n");

200 printf(" This will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.\n");

201 printf("\n");

@@ -105,6 +220,8 @@ void display_help(const char *name) {

220 printf(" are usually valid PCRE2 patterns too.\n");

221 printf(" Regular expressions without named groups are ignored.\n");

222 printf("\n");

223 + printf("The program accepts all parameters as both --option=value and --option value.\n");

224 + printf("\n");

225 printf("The maximum line length accepted is %d characters.\n", MAX_LINE_LENGTH);

226 printf("The maximum number of fields in the PCRE2 pattern is %d.\n", OVECCOUNT / 3);

227 printf("\n");

@@ -166,9 +283,16 @@ void display_help(const char *name) {

283 printf("\n");

284 printf("You can find the most common fields at 'man systemd.journal-fields'.\n");

285 printf("\n");

286 + printf("Example YAML file:\n\n"

287 + "--------------------------------------------------------------------------------\n"

288 + "%s"

289 + "--------------------------------------------------------------------------------\n"

290 + "\n",

291 + YAML_CONFIG_NGINX_COMBINED);

292 }

293

294 // ----------------------------------------------------------------------------

295 +// logging

296

297 // enable the compiler to check for printf like errors on our log2stderr() function

298 static void log2stderr(const char *format, ...) __attribute__ ((format(__printf__, 1, 2)));

@@ -180,6 +304,41 @@ static void log2stderr(const char *format, ...) {

304 fprintf(stderr, "\n");

305 }

306

307 +// ----------------------------------------------------------------------------

308 +// allocation functions abstraction

309 +

310 +void *mallocz(size_t size) {

311 + void *ptr = malloc(size);

312 + if (!ptr) {

313 + log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", size);

314 + exit(EXIT_FAILURE);

315 + }

316 + return ptr;

317 +}

318 +

319 +char *strdupz(const char *s) {

320 + char *ptr = strdup(s);

321 + if (!ptr) {

322 + log2stderr("Fatal Error: Memory allocation failed in strdup.");

323 + exit(EXIT_FAILURE);

324 + }

325 + return ptr;

326 +}

327 +

328 +char *strndupz(const char *s, size_t n) {

329 + char *ptr = strndup(s, n);

330 + if (!ptr) {

331 + log2stderr("Fatal Error: Memory allocation failed in strndup. Requested size: %zu bytes.", n);

332 + exit(EXIT_FAILURE);

333 + }

334 + return ptr;

335 +}

336 +

337 +void freez(void *ptr) {

338 + if (ptr)

339 + free(ptr);

340 +}

341 +

342 // ----------------------------------------------------------------------------

343

344 size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) {

@@ -210,6 +369,11 @@ typedef struct txt {

369 } TEXT;

370

371 static void txt_replace(TEXT *txt, const char *s, size_t len) {

372 + if(!s || !*s || len == 0) {

373 + s = "";

374 + len = 0;

375 + }

376 +

377 if(len + 1 <= txt->size) {

378 // the existing value allocation, fits our value

379

@@ -220,9 +384,9 @@ static void txt_replace(TEXT *txt, const char *s, size_t len) {

384 // no existing value allocation, or too small for our value

385

386 if(txt->s)

223 - free(txt->s);

387 + freez(txt->s);

388

225 - txt->s = strndup(s, len);

389 + txt->s = strndupz(s, len);

390 txt->size = len + 1;

391 }

392 }

@@ -269,6 +433,8 @@ struct key_rewrite {

433 };

434

435 struct log_job {

436 + bool show_config;

437 +

438 const char *pattern;

439

440 struct {

@@ -301,29 +467,98 @@ struct log_job {

467 } rewrites;

468 };

469

470 +static bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len) {

471 + if(!key || !*key) {

472 + log2stderr("filename key cannot be empty.");

473 + return false;

474 + }

475 +

476 + if(jb->filename.key)

477 + freez((char*)jb->filename.key);

478 +

479 + jb->filename.key = strndupz(key, key_len);

480 +

481 + return true;

482 +}

483 +

484 +static bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) {

485 + if (unmatched) {

486 + if (jb->unmatched.injections.used >= MAX_INJECTIONS) {

487 + log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);

488 + return false;

489 + }

490 + }

491 + else {

492 + if (jb->injections.used >= MAX_INJECTIONS) {

493 + log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);

494 + return false;

495 + }

496 + }

497 +

498 + if (unmatched) {

499 + key_value_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],

500 + key, key_len,

501 + value, value_len);

502 + } else {

503 + key_value_replace(&jb->injections.keys[jb->injections.used++],

504 + key, key_len,

505 + value, value_len);

506 + }

507 +

508 + return true;

509 +}

510 +

511 +static bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern) {

512 + pcre2_code *re = jb_compile_pcre2_pattern(search_pattern);

513 + if (!re) {

514 + return false;

515 + }

516 +

517 + struct key_rewrite *rw = &jb->rewrites.array[jb->rewrites.used++];

518 + rw->key = strdupz(key);

519 + rw->hash = XXH3_64bits(rw->key, strlen(rw->key));

520 + rw->search_pattern = strdupz(search_pattern);

521 + rw->replace_pattern = strdupz(replace_pattern);

522 + rw->re = re;

523 + rw->match_data = pcre2_match_data_create_from_pattern(rw->re, NULL);

524 +

525 + // Parse the replacement pattern and create the linked list

526 + if (!parse_replacement_pattern(rw)) {

527 + pcre2_match_data_free(rw->match_data);

528 + pcre2_code_free(rw->re);

529 + freez(rw->key);

530 + freez(rw->search_pattern);

531 + freez(rw->replace_pattern);

532 + jb->rewrites.used--;

533 + return false;

534 + }

535 +

536 + return true;

537 +}

538 +

539 void jb_cleanup(struct log_job *jb) {

540 for(size_t i = 0; i < jb->injections.used ;i++) {

541 if(jb->injections.keys[i].value.s)

307 - free(jb->injections.keys[i].value.s);

542 + freez(jb->injections.keys[i].value.s);

543 }

544

545 for(size_t i = 0; i < jb->unmatched.injections.used ;i++) {

546 if(jb->unmatched.injections.keys[i].value.s)

312 - free(jb->unmatched.injections.keys[i].value.s);

547 + freez(jb->unmatched.injections.keys[i].value.s);

548 }

549

550 for(size_t i = 0; i < jb->dups.used ;i++) {

551 struct key_dup *kd = &jb->dups.array[i];

552

553 if(kd->target)

319 - free(kd->target);

554 + freez(kd->target);

555

556 for(size_t j = 0; j < kd->used ; j++) {

557 if (kd->keys[j])

323 - free(kd->keys[j]);

558 + freez(kd->keys[j]);

559

560 if (kd->values[j].s)

326 - free(kd->values[j].s);

561 + freez(kd->values[j].s);

562 }

563 }

564

@@ -331,13 +566,13 @@ void jb_cleanup(struct log_job *jb) {

566 struct key_rewrite *rw = &jb->rewrites.array[i];

567

568 if (rw->key)

334 - free(rw->key);

569 + freez(rw->key);

570

571 if (rw->search_pattern)

337 - free(rw->search_pattern);

572 + freez(rw->search_pattern);

573

574 if (rw->replace_pattern)

340 - free(rw->replace_pattern);

575 + freez(rw->replace_pattern);

576

577 if(rw->match_data)

578 pcre2_match_data_free(rw->match_data);

@@ -351,9 +586,9 @@ void jb_cleanup(struct log_job *jb) {

586 struct replacement_node *next = current->next;

587

588 if (current->s)

354 - free((void *)current->s);

589 + freez((void *)current->s);

590

356 - free(current);

591 + freez(current);

592 current = next;

593 }

594 }

@@ -465,11 +700,748 @@ static inline void send_key_value_constant(struct log_job *jb, const char *key,

700 printf("%s=%s\n", key, value);

701 }

702

703 +// ----------------------------------------------------------------------------

704 +

705 +static struct key_dup *add_duplicate_target_to_job(struct log_job *jb, const char *target, size_t target_len) {

706 + if (jb->dups.used >= MAX_KEY_DUPS) {

707 + log2stderr("Error: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS);

708 + return NULL;

709 + }

710 +

711 + struct key_dup *kd = &jb->dups.array[jb->dups.used++];

712 + kd->target = strndupz(target, target_len);

713 + kd->hash = XXH3_64bits(kd->target, target_len);

714 + kd->used = 0;

715 + kd->exposed = false;

716 +

717 + // Initialize values array

718 + for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) {

719 + kd->values[i].s = NULL;

720 + kd->values[i].size = 0;

721 + }

722 +

723 + return kd;

724 +}

725 +

726 +static bool add_key_to_duplicate(struct key_dup *kd, const char *key, size_t key_len) {

727 + if (kd->used >= MAX_KEY_DUPS_KEYS) {

728 + log2stderr("Error: Too many keys in duplication of target '%s'.", kd->target);

729 + return false;

730 + }

731 +

732 + kd->keys[kd->used++] = strndupz(key, key_len);

733 + return true;

734 +}

735 +

736 +// ----------------------------------------------------------------------------

737 +// yaml configuration file

738 +

739 +#ifdef HAVE_LIBYAML

740 +

741 +

742 +// ----------------------------------------------------------------------------

743 +// yaml library functions

744 +

745 +static const char *yaml_event_name(yaml_event_type_t type) {

746 + switch (type) {

747 + case YAML_NO_EVENT:

748 + return "YAML_NO_EVENT";

749 +

750 + case YAML_SCALAR_EVENT:

751 + return "YAML_SCALAR_EVENT";

752 +

753 + case YAML_ALIAS_EVENT:

754 + return "YAML_ALIAS_EVENT";

755 +

756 + case YAML_MAPPING_START_EVENT:

757 + return "YAML_MAPPING_START_EVENT";

758 +

759 + case YAML_MAPPING_END_EVENT:

760 + return "YAML_MAPPING_END_EVENT";

761 +

762 + case YAML_SEQUENCE_START_EVENT:

763 + return "YAML_SEQUENCE_START_EVENT";

764 +

765 + case YAML_SEQUENCE_END_EVENT:

766 + return "YAML_SEQUENCE_END_EVENT";

767 +

768 + case YAML_STREAM_START_EVENT:

769 + return "YAML_STREAM_START_EVENT";

770 +

771 + case YAML_STREAM_END_EVENT:

772 + return "YAML_STREAM_END_EVENT";

773 +

774 + case YAML_DOCUMENT_START_EVENT:

775 + return "YAML_DOCUMENT_START_EVENT";

776 +

777 + case YAML_DOCUMENT_END_EVENT:

778 + return "YAML_DOCUMENT_END_EVENT";

779 +

780 + default:

781 + return "UNKNOWN";

782 + }

783 +}

784 +

785 +#define yaml_error(parser, event, fmt, args...) yaml_error_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__, fmt, ##args)

786 +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) __attribute__ ((format(__printf__, 6, 7)));

787 +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) {

788 + char buf[1024] = ""; // Initialize buf to an empty string

789 + const char *type = "";

790 +

791 + if(event) {

792 + type = yaml_event_name(event->type);

793 +

794 + switch (event->type) {

795 + case YAML_SCALAR_EVENT:

796 + copy_to_buffer(buf, sizeof(buf), (char *)event->data.scalar.value, event->data.scalar.length);

797 + break;

798 +

799 + case YAML_ALIAS_EVENT:

800 + snprintf(buf, sizeof(buf), "%s", event->data.alias.anchor);

801 + break;

802 +

803 + default:

804 + break;

805 + }

806 + }

807 +

808 + fprintf(stderr, "YAML %zu@%s, %s(): (line %d, column %d, %s%s%s): ",

809 + line, file, function,

810 + (int)(parser->mark.line + 1), (int)(parser->mark.column + 1),

811 + type, buf[0]? ", near ": "", buf);

812 +

813 + va_list args;

814 + va_start(args, format);

815 + vfprintf(stderr, format, args);

816 + va_end(args);

817 + fprintf(stderr, "\n");

818 +}

819 +

820 +#define yaml_parse(parser, event) yaml_parse_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__)

821 +static bool yaml_parse_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file) {

822 + if (!yaml_parser_parse(parser, event)) {

823 + yaml_error(parser, NULL, "YAML parser error %d", parser->error);

824 + return false;

825 + }

826 +

827 +// fprintf(stderr, ">>> %s >>> %.*s\n",

828 +// yaml_event_name(event->type),

829 +// event->type == YAML_SCALAR_EVENT ? event->data.scalar.length : 0,

830 +// event->type == YAML_SCALAR_EVENT ? (char *)event->data.scalar.value : "");

831 +

832 + return true;

833 +}

834 +

835 +#define yaml_parse_expect_event(parser, type) yaml_parse_expect_event_with_trace(parser, type, __LINE__, __FUNCTION__, __FILE__)

836 +static bool yaml_parse_expect_event_with_trace(yaml_parser_t *parser, yaml_event_type_t type, size_t line, const char *function, const char *file) {

837 + yaml_event_t event;

838 + if (!yaml_parse(parser, &event))

839 + return false;

840 +

841 + bool ret = true;

842 + if(event.type != type) {

843 + yaml_error_with_trace(parser, &event, line, function, file, "unexpected event - expecting: %s", yaml_event_name(type));

844 + ret = false;

845 + }

846 +// else

847 +// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function);

848 +

849 + yaml_event_delete(&event);

850 + return ret;

851 +}

852 +

853 +#define yaml_scalar_matches(event, s, len) yaml_scalar_matches_with_trace(event, s, len, __LINE__, __FUNCTION__, __FILE__)

854 +static bool yaml_scalar_matches_with_trace(yaml_event_t *event, const char *s, size_t len, size_t line __maybe_unused, const char *function __maybe_unused, const char *file __maybe_unused) {

855 + if(event->type != YAML_SCALAR_EVENT)

856 + return false;

857 +

858 + if(len != event->data.scalar.length)

859 + return false;

860 +// else

861 +// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function);

862 +

863 + return strcmp((char *)event->data.scalar.value, s) == 0;

864 +}

865 +

866 +// ----------------------------------------------------------------------------

867 +

868 +static struct key_dup *yaml_parse_duplicate_key(struct log_job *jb, yaml_parser_t *parser) {

869 + yaml_event_t event;

870 +

871 + if (!yaml_parse(parser, &event))

872 + return false;

873 +

874 + struct key_dup *kd = NULL;

875 + if(event.type == YAML_SCALAR_EVENT) {

876 + kd = add_duplicate_target_to_job(jb, (char *)event.data.scalar.value, event.data.scalar.length);

877 + }

878 + else

879 + yaml_error(parser, &event, "duplicate key must be a scalar.");

880 +

881 + yaml_event_delete(&event);

882 + return kd;

883 +}

884 +

885 +static size_t yaml_parse_duplicate_from(struct log_job *jb, yaml_parser_t *parser, struct key_dup *kd) {

886 + size_t errors = 0;

887 + yaml_event_t event;

888 +

889 + if (!yaml_parse(parser, &event))

890 + return 1;

891 +

892 + bool ret = true;

893 + if(event.type == YAML_SCALAR_EVENT)

894 + ret = add_key_to_duplicate(kd, (char *)event.data.scalar.value, event.data.scalar.length);

895 +

896 + else if(event.type == YAML_SEQUENCE_START_EVENT) {

897 + bool finished = false;

898 + while(!errors && !finished) {

899 + yaml_event_t sub_event;

900 + if (!yaml_parse(parser, &sub_event))

901 + return errors++;

902 + else {

903 + if (sub_event.type == YAML_SCALAR_EVENT)

904 + add_key_to_duplicate(kd, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length);

905 +

906 + else if (sub_event.type == YAML_SEQUENCE_END_EVENT)

907 + finished = true;

908 +

909 + yaml_event_delete(&sub_event);

910 + }

911 + }

912 + }

913 + else

914 + yaml_error(parser, &event, "not expected event type");

915 +

916 + yaml_event_delete(&event);

917 + return errors;

918 +}

919 +

920 +static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_job *jb) {

921 + yaml_event_t event;

922 + size_t errors = 0;

923 +

924 + if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))

925 + return 1;

926 +

927 + if (!yaml_parse(parser, &event))

928 + return 1;

929 +

930 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {

931 + yaml_event_t sub_event;

932 + if (!yaml_parse(parser, &sub_event))

933 + errors++;

934 +

935 + else {

936 + if (event.type == YAML_SCALAR_EVENT) {

937 + if(!log_job_add_filename_key(jb, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length))

938 + errors++;

939 + }

940 +

941 + else {

942 + yaml_error(parser, &sub_event, "expected the filename as %s", yaml_event_name(YAML_SCALAR_EVENT));

943 + errors++;

944 + }

945 +

946 + yaml_event_delete(&sub_event);

947 + }

948 + }

949 +

950 + if(!yaml_parse_expect_event(parser, YAML_MAPPING_END_EVENT))

951 + errors++;

952 +

953 + yaml_event_delete(&event);

954 + return errors;

955 +}

956 +

957 +static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, struct log_job *jb) {

958 + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))

959 + return 1;

960 +

961 + struct key_dup *kd = NULL;

962 +

963 + // Expecting a key-value pair for each duplicate

964 + bool finished;

965 + size_t errors = 0;

966 + while (!errors && !finished) {

967 + yaml_event_t event;

968 + if (!yaml_parse(parser, &event)) {

969 + errors++;

970 + break;

971 + }

972 +

973 + if(event.type == YAML_MAPPING_START_EVENT) {

974 + ;

975 + }

976 + if (event.type == YAML_SEQUENCE_END_EVENT) {

977 + finished = true;

978 + }

979 + else if(event.type == YAML_SCALAR_EVENT) {

980 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {

981 + kd = yaml_parse_duplicate_key(jb, parser);

982 + if (!kd)

983 + errors++;

984 + else {

985 + while (!errors && kd) {

986 + yaml_event_t sub_event;

987 + if (!yaml_parse(parser, &sub_event)) {

988 + errors++;

989 + break;

990 + }

991 +

992 + if (sub_event.type == YAML_MAPPING_END_EVENT) {

993 + kd = NULL;

994 + } else if (sub_event.type == YAML_SCALAR_EVENT) {

995 + if (yaml_scalar_matches(&sub_event, "values_of", strlen("values_of"))) {

996 + if (!kd) {

997 + yaml_error(parser, &sub_event, "Found 'values_of' but the 'key' is not set.");

998 + errors++;

999 + } else

1000 + errors += yaml_parse_duplicate_from(jb, parser, kd);

1001 + } else {

1002 + yaml_error(parser, &sub_event, "unknown scalar");

1003 + errors++;

1004 + }

1005 + } else {

1006 + yaml_error(parser, &sub_event, "unexpected event type");

1007 + errors++;

1008 + }

1009 +

1010 + // Delete the event after processing

1011 + yaml_event_delete(&event);

1012 + }

1013 + }

1014 + } else {

1015 + yaml_error(parser, &event, "unknown scalar");

1016 + errors++;

1017 + }

1018 + }

1019 +

1020 + yaml_event_delete(&event);

1021 + }

1022 +

1023 + return errors;

1024 +}

1025 +

1026 +static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {

1027 + yaml_event_t event;

1028 + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {

1029 + yaml_error(parser, &event, "Expected scalar for constant field injection key");

1030 + yaml_event_delete(&event);

1031 + return false;

1032 + }

1033 +

1034 + char *key = strndupz((char *)event.data.scalar.value, event.data.scalar.length);

1035 + char *value = NULL;

1036 + bool ret = false;

1037 +

1038 + yaml_event_delete(&event);

1039 +

1040 + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {

1041 + yaml_error(parser, &event, "Expected scalar for constant field injection value");

1042 + goto cleanup;

1043 + }

1044 +

1045 + if(!yaml_scalar_matches(&event, "value", strlen("value"))) {

1046 + yaml_error(parser, &event, "Expected scalar 'value'");

1047 + goto cleanup;

1048 + }

1049 +

1050 + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) {

1051 + yaml_error(parser, &event, "Expected scalar for constant field injection value");

1052 + goto cleanup;

1053 + }

1054 +

1055 + value = strndupz((char *)event.data.scalar.value, event.data.scalar.length);

1056 +

1057 + if(!log_job_add_injection(jb, key, strlen(key), value, strlen(value), unmatched))

1058 + ret = false;

1059 + else

1060 + ret = true;

1061 +

1062 + ret = true;

1063 +

1064 +cleanup:

1065 + yaml_event_delete(&event);

1066 + freez(key);

1067 + freez(value);

1068 + return !ret ? 1 : 0;

1069 +}

1070 +

1071 +static bool yaml_parse_injection_mapping(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {

1072 + yaml_event_t event;

1073 + size_t errors = 0;

1074 + bool finished = false;

1075 +

1076 + while (!errors && !finished) {

1077 + if (!yaml_parse(parser, &event)) {

1078 + errors++;

1079 + continue;

1080 + }

1081 +

1082 + switch (event.type) {

1083 + case YAML_SCALAR_EVENT:

1084 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {

1085 + errors += yaml_parse_constant_field_injection(parser, jb, unmatched);

1086 + } else {

1087 + yaml_error(parser, &event, "Unexpected scalar in injection mapping");

1088 + errors++;

1089 + }

1090 + break;

1091 +

1092 + case YAML_MAPPING_END_EVENT:

1093 + finished = true;

1094 + break;

1095 +

1096 + default:

1097 + yaml_error(parser, &event, "Unexpected event in injection mapping");

1098 + errors++;

1099 + break;

1100 + }

1101 +

1102 + yaml_event_delete(&event);

1103 + }

1104 +

1105 + return errors == 0;

1106 +}

1107 +

1108 +static size_t yaml_parse_injections(yaml_parser_t *parser, struct log_job *jb, bool unmatched) {

1109 + yaml_event_t event;

1110 + size_t errors = 0;

1111 + bool finished = false;

1112 +

1113 + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))

1114 + return 1;

1115 +

1116 + while (!errors && !finished) {

1117 + if (!yaml_parse(parser, &event)) {

1118 + errors++;

1119 + continue;

1120 + }

1121 +

1122 + switch (event.type) {

1123 + case YAML_MAPPING_START_EVENT:

1124 + if (!yaml_parse_injection_mapping(parser, jb, unmatched))

1125 + errors++;

1126 + break;

1127 +

1128 + case YAML_SEQUENCE_END_EVENT:

1129 + finished = true;

1130 + break;

1131 +

1132 + default:

1133 + yaml_error(parser, &event, "Unexpected event in injections sequence");

1134 + errors++;

1135 + break;

1136 + }

1137 +

1138 + yaml_event_delete(&event);

1139 + }

1140 +

1141 + return errors;

1142 +}

1143 +

1144 +static size_t yaml_parse_unmatched(yaml_parser_t *parser, struct log_job *jb) {

1145 + size_t errors = 0;

1146 + bool finished = false;

1147 +

1148 + if (!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT))

1149 + return 1;

1150 +

1151 + while (!errors && !finished) {

1152 + yaml_event_t event;

1153 + if (!yaml_parse(parser, &event)) {

1154 + errors++;

1155 + continue;

1156 + }

1157 +

1158 + switch (event.type) {

1159 + case YAML_SCALAR_EVENT:

1160 + if (yaml_scalar_matches(&event, "key", strlen("key"))) {

1161 + yaml_event_t sub_event;

1162 + if (!yaml_parse(parser, &sub_event)) {

1163 + errors++;

1164 + } else {

1165 + if (sub_event.type == YAML_SCALAR_EVENT) {

1166 + jb->unmatched.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);

1167 + } else {

1168 + yaml_error(parser, &sub_event, "expected a scalar value for 'key'");

1169 + errors++;

1170 + }

1171 + yaml_event_delete(&sub_event);

1172 + }

1173 + } else if (yaml_scalar_matches(&event, "inject", strlen("inject"))) {

1174 + errors += yaml_parse_injections(parser, jb, true);

1175 + } else {

1176 + yaml_error(parser, &event, "Unexpected scalar in unmatched section");

1177 + errors++;

1178 + }

1179 + break;

1180 +

1181 + case YAML_MAPPING_END_EVENT:

1182 + finished = true;

1183 + break;

1184 +

1185 + default:

1186 + yaml_error(parser, &event, "Unexpected event in unmatched section");

1187 + errors++;

1188 + break;

1189 + }

1190 +

1191 + yaml_event_delete(&event);

1192 + }

1193 +

1194 + return errors;

1195 +}

1196 +

1197 +static size_t yaml_parse_rewrites(yaml_parser_t *parser, struct log_job *jb) {

1198 + size_t errors = 0;

1199 +

1200 + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT))

1201 + return 1;

1202 +

1203 + bool finished = false;

1204 + while (!errors && !finished) {

1205 + yaml_event_t event;

1206 + if (!yaml_parse(parser, &event)) {

1207 + errors++;

1208 + continue;

1209 + }

1210 +

1211 + switch (event.type) {

1212 + case YAML_MAPPING_START_EVENT:

1213 + {

1214 + struct key_rewrite rw = {0};

1215 +

1216 + bool mapping_finished = false;

1217 + while (!errors && !mapping_finished) {

1218 + yaml_event_t sub_event;

1219 + if (!yaml_parse(parser, &sub_event)) {

1220 + errors++;

1221 + continue;

1222 + }

1223 +

1224 + switch (sub_event.type) {

1225 + case YAML_SCALAR_EVENT:

1226 + if (yaml_scalar_matches(&sub_event, "key", strlen("key"))) {

1227 + if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {

1228 + yaml_error(parser, &sub_event, "Expected scalar for rewrite key");

1229 + errors++;

1230 + } else {

1231 + rw.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);

1232 + yaml_event_delete(&sub_event);

1233 + }

1234 + } else if (yaml_scalar_matches(&sub_event, "search", strlen("search"))) {

1235 + if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {

1236 + yaml_error(parser, &sub_event, "Expected scalar for rewrite search pattern");

1237 + errors++;

1238 + } else {

1239 + rw.search_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);

1240 + yaml_event_delete(&sub_event);

1241 + }

1242 + } else if (yaml_scalar_matches(&sub_event, "replace", strlen("replace"))) {

1243 + if (!yaml_parse(parser, &sub_event) || sub_event.type != YAML_SCALAR_EVENT) {

1244 + yaml_error(parser, &sub_event, "Expected scalar for rewrite replace pattern");

1245 + errors++;

1246 + } else {

1247 + rw.replace_pattern = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length);

1248 + yaml_event_delete(&sub_event);

1249 + }

1250 + } else {

1251 + yaml_error(parser, &sub_event, "Unexpected scalar in rewrite mapping");

1252 + errors++;

1253 + }

1254 + break;

1255 +

1256 + case YAML_MAPPING_END_EVENT:

1257 + if(rw.key && rw.search_pattern && rw.replace_pattern) {

1258 + if (!log_job_add_rewrite(jb, rw.key, rw.search_pattern, rw.replace_pattern))

1259 + errors++;

1260 + }

1261 + freez(rw.key);

1262 + freez(rw.search_pattern);

1263 + freez(rw.replace_pattern);

1264 + memset(&rw, 0, sizeof(rw));

1265 +

1266 + mapping_finished = true;

1267 + break;

1268 +

1269 + default:

1270 + yaml_error(parser, &sub_event, "Unexpected event in rewrite mapping");

1271 + errors++;

1272 + break;

1273 + }

1274 +

1275 + yaml_event_delete(&sub_event);

1276 + }

1277 + }

1278 + break;

1279 +

1280 + case YAML_SEQUENCE_END_EVENT:

1281 + finished = true;

1282 + break;

1283 +

1284 + default:

1285 + yaml_error(parser, &event, "Unexpected event in rewrites sequence");

1286 + errors++;

1287 + break;

1288 + }

1289 +

1290 + yaml_event_delete(&event);

1291 + }

1292 +

1293 + return errors;

1294 +}

1295 +

1296 +static size_t yaml_parse_pattern(yaml_parser_t *parser, struct log_job *jb) {

1297 + yaml_event_t event;

1298 + size_t errors = 0;

1299 +

1300 + if (!yaml_parse(parser, &event))

1301 + return 1;

1302 +

1303 + if(event.type == YAML_SCALAR_EVENT)

1304 + jb->pattern = strndupz((char *)event.data.scalar.value, event.data.scalar.length);

1305 + else {

1306 + yaml_error(parser, &event, "unexpected event type");

1307 + errors++;

1308 + }

1309 +

1310 + yaml_event_delete(&event);

1311 + return errors;

1312 +}

1313 +

1314 +static size_t yaml_parse_initialized(yaml_parser_t *parser, struct log_job *jb) {

1315 + size_t errors = 0;

1316 +

1317 + if(!yaml_parse_expect_event(parser, YAML_STREAM_START_EVENT)) {

1318 + errors++;

1319 + goto cleanup;

1320 + }

1321 +

1322 + if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_START_EVENT)) {

1323 + errors++;

1324 + goto cleanup;

1325 + }

1326 +

1327 + if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT)) {

1328 + errors++;

1329 + goto cleanup;

1330 + }

1331 +

1332 + bool finished = false;

1333 + while (!errors && !finished) {

1334 + yaml_event_t event;

1335 + if(!yaml_parse(parser, &event)) {

1336 + errors++;

1337 + continue;

1338 + }

1339 +

1340 + switch(event.type) {

1341 + default:

1342 + yaml_error(parser, &event, "unexpected type");

1343 + errors++;

1344 + break;

1345 +

1346 + case YAML_MAPPING_END_EVENT:

1347 + finished = true;

1348 + break;

1349 +

1350 + case YAML_SCALAR_EVENT:

1351 + if (yaml_scalar_matches(&event, "pattern", strlen("pattern")))

1352 + errors += yaml_parse_pattern(parser, jb);

1353 +

1354 + else if (yaml_scalar_matches(&event, "filename", strlen("filename")))

1355 + errors += yaml_parse_filename_injection(parser, jb);

1356 +

1357 + else if (yaml_scalar_matches(&event, "duplicate", strlen("duplicate")))

1358 + errors += yaml_parse_duplicates_injection(parser, jb);

1359 +

1360 + else if (yaml_scalar_matches(&event, "inject", strlen("inject")))

1361 + errors += yaml_parse_injections(parser, jb, false);

1362 +

1363 + else if (yaml_scalar_matches(&event, "unmatched", strlen("unmatched")))

1364 + errors += yaml_parse_unmatched(parser, jb);

1365 +

1366 + else if (yaml_scalar_matches(&event, "rewrite", strlen("rewrite")))

1367 + errors += yaml_parse_rewrites(parser, jb);

1368 +

1369 + else {

1370 + yaml_error(parser, &event, "unexpected scalar");

1371 + errors++;

1372 + }

1373 + break;

1374 + }

1375 +

1376 + yaml_event_delete(&event);

1377 + }

1378 +

1379 + if(!yaml_parse_expect_event(parser, YAML_DOCUMENT_END_EVENT)) {

1380 + errors++;

1381 + goto cleanup;

1382 + }

1383 +

1384 + if(!yaml_parse_expect_event(parser, YAML_STREAM_END_EVENT)) {

1385 + errors++;

1386 + goto cleanup;

1387 + }

1388 +

1389 +cleanup:

1390 + return errors;

1391 +}

1392 +

1393 +static bool yaml_parse_file(const char *config_file_path, struct log_job *jb) {

1394 + if(!config_file_path || !*config_file_path) {

1395 + log2stderr("yaml configuration filename cannot be empty.");

1396 + return false;

1397 + }

1398 +

1399 + FILE *fp = fopen(config_file_path, "r");

1400 + if (!fp) {

1401 + log2stderr("Error opening config file: %s", config_file_path);

1402 + return false;

1403 + }

1404 +

1405 + yaml_parser_t parser;

1406 + yaml_parser_initialize(&parser);

1407 + yaml_parser_set_input_file(&parser, fp);

1408 +

1409 + size_t errors = yaml_parse_initialized(&parser, jb);

1410 +

1411 + yaml_parser_delete(&parser);

1412 + fclose(fp);

1413 + return errors == 0;

1414 +}

1415 +

1416 +static bool yaml_parse_config(const char *config_name, struct log_job *jb) {

1417 +

1418 + const char *config = NULL;

1419 +

1420 + if(strcmp(config_name, "nginx-combined") == 0)

1421 + config = YAML_CONFIG_NGINX_COMBINED;

1422 + else {

1423 + log2stderr("Unknown configuration: '%s'", config_name);

1424 + return false;

1425 + }

1426 +

1427 + yaml_parser_t parser;

1428 + yaml_parser_initialize(&parser);

1429 + yaml_parser_set_input_string(&parser, (const unsigned char *)config, strlen(config));

1430 +

1431 + size_t errors = yaml_parse_initialized(&parser, jb);

1432 +

1433 + yaml_parser_delete(&parser);

1434 + return errors == 0;

1435 +}

1436 +

1437 +#endif

1438 +

1439 +

1440 // ----------------------------------------------------------------------------

1441 // command line params

1442

1443 struct replacement_node *add_replacement_node(struct replacement_node **head, bool is_variable, const char *text) {

472 - struct replacement_node *new_node = malloc(sizeof(struct replacement_node));

1444 + struct replacement_node *new_node = mallocz(sizeof(struct replacement_node));

1445 if (!new_node)

1446 return NULL;

1447

@@ -494,7 +1466,7 @@ struct replacement_node *add_replacement_node(struct replacement_node **head, bo

1466 return new_node;

1467 }

1468

497 -bool parse_replacement_pattern(struct key_rewrite *rw) {

1469 +static bool parse_replacement_pattern(struct key_rewrite *rw) {

1470 const char *current = rw->replace_pattern;

1471

1472 while (*current != '\0') {

@@ -507,7 +1479,7 @@ bool parse_replacement_pattern(struct key_rewrite *rw) {

1479 }

1480

1481 size_t name_length = end - current - 2; // Length of the variable name

510 - char *variable_name = strndup(current + 2, name_length);

1482 + char *variable_name = strndupz(current + 2, name_length);

1483 if (!variable_name) {

1484 log2stderr("Error: Memory allocation failed for variable name.");

1485 return false;

@@ -515,7 +1487,7 @@ bool parse_replacement_pattern(struct key_rewrite *rw) {

1487

1488 struct replacement_node *node = add_replacement_node(&(rw->nodes), true, variable_name);

1489 if (!node) {

518 - free(variable_name);

1490 + freez(variable_name);

1491 log2stderr("Error: Failed to add replacement node for variable.");

1492 return false;

1493 }

@@ -530,7 +1502,7 @@ bool parse_replacement_pattern(struct key_rewrite *rw) {

1502 }

1503

1504 size_t text_length = current - start;

533 - char *text = strndup(start, text_length);

1505 + char *text = strndupz(start, text_length);

1506 if (!text) {

1507 log2stderr("Error: Memory allocation failed for literal text.");

1508 return false;

@@ -538,7 +1510,7 @@ bool parse_replacement_pattern(struct key_rewrite *rw) {

1510

1511 struct replacement_node *node = add_replacement_node(&(rw->nodes), false, text);

1512 if (!node) {

541 - free(text);

1513 + freez(text);

1514 log2stderr("Error: Failed to add replacement node for text.");

1515 return false;

1516 }

@@ -593,38 +1565,65 @@ static bool parse_rewrite(struct log_job *jb, const char *param) {

1565 }

1566

1567 // Extract key, search pattern, and replacement pattern

596 - char *key = strndup(param, equal_sign - param);

597 - char *search_pattern = strndup(equal_sign + 2, second_separator - (equal_sign + 2));

598 - char *replace_pattern = strdup(second_separator + 1);

1568 + char *key = strndupz(param, equal_sign - param);

1569 + char *search_pattern = strndupz(equal_sign + 2, second_separator - (equal_sign + 2));

1570 + char *replace_pattern = strdupz(second_separator + 1);

1571

600 - // Create the PCRE2 pattern

601 - pcre2_code *re = jb_compile_pcre2_pattern(search_pattern);

602 - if (!re) {

603 - free(key);

604 - free(search_pattern);

605 - free(replace_pattern);

1572 + bool ret = log_job_add_rewrite(jb, key, search_pattern, replace_pattern);

1573 +

1574 + freez(key);

1575 + freez(search_pattern);

1576 + freez(replace_pattern);

1577 +

1578 + return ret;

1579 +}

1580 +

1581 +static bool parse_inject(struct log_job *jb, const char *value, bool unmatched) {

1582 + const char *equal = strchr(value, '=');

1583 + if (!equal) {

1584 + log2stderr("Error: injection '%s' does not have an equal sign.", value);

1585 return false;

1586 }

1587

609 - struct key_rewrite *rw = &jb->rewrites.array[jb->rewrites.used++];

610 - rw->hash = XXH3_64bits(key, strlen(key));

611 - rw->key = key;

612 - rw->search_pattern = search_pattern;

613 - rw->replace_pattern = replace_pattern;

614 - rw->re = re;

615 - rw->match_data = pcre2_match_data_create_from_pattern(rw->re, NULL);

1588 + const char *key = value;

1589 + const char *val = equal + 1;

1590 + log_job_add_injection(jb, key, equal - key, val, strlen(val), unmatched);

1591

617 - // Parse the replacement pattern and create the linked list

618 - if (!parse_replacement_pattern(rw)) {

619 - pcre2_match_data_free(rw->match_data);

620 - pcre2_code_free(rw->re);

621 - free(rw->key);

622 - free(rw->search_pattern);

623 - free(rw->replace_pattern);

624 - jb->rewrites.used--;

1592 + return true;

1593 +}

1594 +

1595 +static bool parse_duplicate(struct log_job *jb, const char *value) {

1596 + const char *target = value;

1597 + const char *equal_sign = strchr(value, '=');

1598 + if (!equal_sign || equal_sign == target) {

1599 + log2stderr("Error: Invalid duplicate format, '=' not found or at the start in %s", value);

1600 return false;

1601 }

1602

1603 + size_t target_len = equal_sign - target;

1604 + struct key_dup *kd = add_duplicate_target_to_job(jb, target, target_len);

1605 + if(!kd) return false;

1606 +

1607 + const char *key = equal_sign + 1;

1608 + while (key) {

1609 + if (kd->used >= MAX_KEY_DUPS_KEYS) {

1610 + log2stderr("Error: too many keys in duplication of target '%s'.", kd->target);

1611 + return false;

1612 + }

1613 +

1614 + const char *comma = strchr(key, ',');

1615 + size_t key_len;

1616 + if (comma) {

1617 + key_len = comma - key;

1618 + add_key_to_duplicate(kd, key, key_len);

1619 + key = comma + 1;

1620 + }

1621 + else {

1622 + add_key_to_duplicate(kd, key, strlen(key));

1623 + break; // No more keys

1624 + }

1625 + }

1626 +

1627 return true;

1628 }

1629

@@ -635,94 +1634,76 @@ bool parse_parameters(struct log_job *jb, int argc, char **argv) {

1634 display_help(argv[0]);

1635 exit(0);

1636 }

638 - else if (strncmp(arg, "--filename-key=", 15) == 0)

639 - jb->filename.key = arg + 15;

640 - else if (strncmp(arg, "--unmatched-key=", 16) == 0)

641 - jb->unmatched.key = arg + 16;

642 - else if(strncmp(arg, "--duplicate=", 12) == 0) {

643 - const char *first_key = arg + 12;

644 - const char *comma = strchr(first_key, '=');

645 - if(!comma) {

646 - log2stderr("Error: --duplicate=TARGET=KEY1,... is missing the equal sign.");

647 - return false;

648 - }

649 - const char *next_key = comma + 1;

650 -

651 - if (jb->dups.used >= MAX_KEY_DUPS) {

652 - log2stderr("Error: too many duplications. You can duplicate up to %d keys.", MAX_KEY_DUPS);

653 - return false;

1637 + else if (strcmp(arg, "--show-config") == 0) {

1638 + jb->show_config = true;

1639 + }

1640 + else {

1641 + char buffer[1024];

1642 + char *param = NULL;

1643 + char *value = NULL;

1644 +

1645 + char *equal_sign = strchr(arg, '=');

1646 + if (equal_sign) {

1647 + copy_to_buffer(buffer, sizeof(buffer), arg, equal_sign - arg);

1648 + param = buffer;

1649 + value = equal_sign + 1;

1650 }

655 -

656 - size_t first_key_len = comma - first_key;

657 - struct key_dup *kd = &jb->dups.array[jb->dups.used++];

658 - kd->target = strndup(first_key, first_key_len);

659 - kd->hash = XXH3_64bits(first_key, first_key_len);

660 - kd->used = 0;

661 -

662 - while(next_key) {

663 - if(kd->used >= MAX_KEY_DUPS_KEYS) {

664 - log2stderr("Error: too many keys in duplication of target '%s'.", kd->target);

665 - return false;

666 - }

667 -

668 - first_key = next_key;

669 - comma = strchr(first_key, ',');

670 -

671 - if(comma) {

672 - first_key_len = comma - first_key;

673 - kd->keys[kd->used++] = strndup(first_key, first_key_len);

674 - next_key = comma + 1;

1651 + else {

1652 + param = arg;

1653 + if (i + 1 < argc) {

1654 + value = argv[++i];

1655 }

1656 else {

677 - kd->keys[kd->used++] = strdup(first_key);

678 - next_key = NULL;

1657 + if (!jb->pattern) {

1658 + jb->pattern = arg;

1659 + continue;

1660 + } else {

1661 + log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);

1662 + return false;

1663 + }

1664 }

1665 }

681 - }

682 - else if(strncmp(arg, "--inject=", 9) == 0) {

683 - if(jb->injections.used >= MAX_INJECTIONS) {

684 - log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS);

685 - return false;

686 - }

1666

688 - const char *key = arg + 9;

689 - const char *equal = strchr(key, '=');

690 - if(!equal) {

691 - log2stderr("Error: injection '%s' does not have an equal sign.", key);

692 - return false;

1667 + if (strcmp(param, "--filename-key") == 0) {

1668 + if(!log_job_add_filename_key(jb, value, value ? strlen(value) : 0))

1669 + return false;

1670 }

694 -

695 - key_value_replace(&jb->injections.keys[jb->injections.used++],

696 - key, equal - key,

697 - equal + 1, strlen(equal + 1));

698 - }

699 - else if(strncmp(arg, "--inject-unmatched=", 19) == 0) {

700 - if(jb->unmatched.injections.used >= MAX_INJECTIONS) {

701 - log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS);

702 - return false;

1671 +#ifdef HAVE_LIBYAML

1672 + else if (strcmp(param, "-f") == 0 || strcmp(param, "--file") == 0) {

1673 + if (!yaml_parse_file(value, jb))

1674 + return false;

1675 }

704 - const char *key = arg + 19;

705 - const char *equal = strchr(key, '=');

706 - if(!equal) {

707 - log2stderr("Error: unmatched injection '%s' does not have an equal sign.", key);

708 - return false;

1676 + else if (strcmp(param, "--config") == 0) {

1677 + if (!yaml_parse_config(value, jb))

1678 + return false;

1679 }

710 -

711 - key_value_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++],

712 - key, equal - key,

713 - equal + 1, strlen(equal + 1));

714 - }

715 - else if(strncmp(arg, "--rewrite=", 10) == 0) {

716 - if(!parse_rewrite(jb, arg + 10))

717 - return false;

718 - }

719 - else {

720 - // Assume it's the pattern if not recognized as a parameter

721 - if (!jb->pattern) {

722 - jb->pattern = arg;

723 - } else {

724 - log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);

725 - return false;

1680 +#endif

1681 + else if (strcmp(param, "--unmatched-key") == 0)

1682 + jb->unmatched.key = value;

1683 + else if (strcmp(param, "--duplicate") == 0) {

1684 + if (!parse_duplicate(jb, value))

1685 + return false;

1686 + }

1687 + else if (strcmp(param, "--inject") == 0) {

1688 + if (!parse_inject(jb, value, false))

1689 + return false;

1690 + }

1691 + else if (strcmp(param, "--inject-unmatched") == 0) {

1692 + if (!parse_inject(jb, value, true))

1693 + return false;

1694 + }

1695 + else if (strcmp(param, "--rewrite") == 0) {

1696 + if (!parse_rewrite(jb, value))

1697 + return false;

1698 + }

1699 + else {

1700 + if (!jb->pattern) {

1701 + jb->pattern = arg;

1702 + continue;

1703 + } else {

1704 + log2stderr("Error: Multiple patterns detected. Specify only one pattern. The first is '%s', the second is '%s'", jb->pattern, arg);

1705 + return false;

1706 + }

1707 }

1708 }

1709 }

@@ -958,6 +1939,136 @@ static inline void jb_traverse_pcre2_named_groups_and_send_keys(struct log_job *

1939

1940 // ----------------------------------------------------------------------------

1941

1942 +static void yaml_print_multiline_value(const char *s, size_t depth) {

1943 + if (!s)

1944 + s = "";

1945 +

1946 + do {

1947 + const char* next = strchr(s, '\n');

1948 + if(next) next++;

1949 +

1950 + size_t len = next ? (size_t)(next - s) : strlen(s);

1951 + char buf[len + 1];

1952 + strncpy(buf, s, len);

1953 + buf[len] = '\0';

1954 +

1955 + fprintf(stderr, "%.*s%s%s",

1956 + (int)(depth * 2), " ",

1957 + buf, next ? "" : "\n");

1958 +

1959 + s = next;

1960 + } while(s && *s);

1961 +}

1962 +

1963 +static bool needs_quotes_in_yaml(const char *str) {

1964 + // Lookup table for special YAML characters

1965 + static bool special_chars[256] = { false };

1966 + static bool table_initialized = false;

1967 +

1968 + if (!table_initialized) {

1969 + // Initialize the lookup table

1970 + const char *special_chars_str = ":{}[],&*!|>'\"%@`^";

1971 + for (const char *c = special_chars_str; *c; ++c) {

1972 + special_chars[(unsigned char)*c] = true;

1973 + }

1974 + table_initialized = true;

1975 + }

1976 +

1977 + while (*str) {

1978 + if (special_chars[(unsigned char)*str]) {

1979 + return true;

1980 + }

1981 + str++;

1982 + }

1983 + return false;

1984 +}

1985 +

1986 +static void yaml_print_node(const char *key, const char *value, size_t depth, bool dash) {

1987 + if(depth > 10) depth = 10;

1988 + const char *quote = "\"";

1989 +

1990 + const char *second_line = NULL;

1991 + if(value && strchr(value, '\n')) {

1992 + second_line = value;

1993 + value = "|";

1994 + quote = "";

1995 + }

1996 + else if(!value || !needs_quotes_in_yaml(value))

1997 + quote = "";

1998 +

1999 + fprintf(stderr, "%.*s%s%s%s%s%s%s\n",

2000 + (int)(depth * 2), " ", dash ? "- ": "",

2001 + key ? key : "", key ? ": " : "",

2002 + quote, value ? value : "", quote);

2003 +

2004 + if(second_line) {

2005 + yaml_print_multiline_value(second_line, depth + 1);

2006 + }

2007 +}

2008 +

2009 +static void log_job_to_yaml(struct log_job *jb) {

2010 + if(jb->pattern)

2011 + yaml_print_node("pattern", jb->pattern, 0, false);

2012 +

2013 + if(jb->filename.key) {

2014 + fprintf(stderr, "\n");

2015 + yaml_print_node("filename", NULL, 0, false);

2016 + yaml_print_node("key", jb->filename.key, 1, false);

2017 + }

2018 +

2019 + if(jb->dups.used) {

2020 + fprintf(stderr, "\n");

2021 + yaml_print_node("duplicate", NULL, 0, false);

2022 + for(size_t i = 0; i < jb->dups.used ;i++) {

2023 + struct key_dup *kd = &jb->dups.array[i];

2024 + yaml_print_node("key", kd->target, 1, true);

2025 + yaml_print_node("values_of", NULL, 2, false);

2026 +

2027 + for(size_t k = 0; k < kd->used ;k++)

2028 + yaml_print_node(NULL, kd->keys[k], 3, true);

2029 + }

2030 + }

2031 +

2032 + if(jb->injections.used) {

2033 + fprintf(stderr, "\n");

2034 + yaml_print_node("inject", NULL, 0, false);

2035 +

2036 + for (size_t i = 0; i < jb->injections.used; i++) {

2037 + yaml_print_node("key", jb->injections.keys[i].key, 1, true);

2038 + yaml_print_node("value", jb->injections.keys[i].value.s, 2, false);

2039 + }

2040 + }

2041 +

2042 + if(jb->rewrites.used) {

2043 + fprintf(stderr, "\n");

2044 + yaml_print_node("rewrite", NULL, 0, false);

2045 +

2046 + for(size_t i = 0; i < jb->rewrites.used ;i++) {

2047 + yaml_print_node("key", jb->rewrites.array[i].key, 1, true);

2048 + yaml_print_node("search", jb->rewrites.array[i].search_pattern, 2, false);

2049 + yaml_print_node("replace", jb->rewrites.array[i].replace_pattern, 2, false);

2050 + }

2051 + }

2052 +

2053 + if(jb->unmatched.key || jb->unmatched.injections.used) {

2054 + fprintf(stderr, "\n");

2055 + yaml_print_node("unmatched", NULL, 0, false);

2056 +

2057 + if(jb->unmatched.key)

2058 + yaml_print_node("key", jb->unmatched.key, 1, false);

2059 +

2060 + if(jb->unmatched.injections.used) {

2061 + fprintf(stderr, "\n");

2062 + yaml_print_node("inject", NULL, 1, false);

2063 +

2064 + for (size_t i = 0; i < jb->unmatched.injections.used; i++) {

2065 + yaml_print_node("key", jb->unmatched.injections.keys[i].key, 2, true);

2066 + yaml_print_node("value", jb->unmatched.injections.keys[i].value.s, 3, false);

2067 + }

2068 + }

2069 + }

2070 +}

2071 +

2072 struct log_job log_job = { 0 };

2073 int main(int argc, char *argv[]) {

2074 struct log_job *jb = &log_job;

@@ -965,9 +2076,15 @@ int main(int argc, char *argv[]) {

2076 if(!parse_parameters(jb, argc, argv))

2077 exit(1);

2078

2079 + if(jb->show_config)

2080 + log_job_to_yaml(jb);

2081 +

2082 jb_select_which_injections_should_be_injected_on_unmatched(jb);

2083

2084 pcre2_code *re = jb_compile_pcre2_pattern(jb->pattern);

2085 + if(!re)

2086 + return 1;

2087 +

2088 pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL);

2089

2090 char buffer[MAX_LINE_LENGTH];

@@ -985,7 +2102,7 @@ int main(int argc, char *argv[]) {

2102 line_is_matched = false;

2103

2104 if (jb->unmatched.key) {

988 - // we are sending errors to Journal

2105 + // we are sending errors to systemd-journal

2106 send_key_value_error(jb->unmatched.key, "PCRE2 error on: %s", line);

2107

2108 for (size_t j = 0; j < jb->unmatched.injections.used; j++)

libnetdata/log/log2journal.md

+117 -9

@@ -313,57 +313,64 @@ tail -n $last -F /var/log/nginx/*access.log \

313

314 ```

315

316 -Netdata log2journal v1.40.0-1214-gae733dd49

316 +Netdata log2journal v1.43.0-276-gfff8d1181

317

318 Convert structured log input to systemd Journal Export Format.

319

320 Using PCRE2 patterns, extract the fields from structured logs on the standard

321 input, and generate output according to systemd Journal Export Format.

322

323 -Usage: log2journal [OPTIONS] PATTERN

323 +Usage: ./log2journal [OPTIONS] PATTERN

324

325 Options:

326

327 - --filename-key=KEY

327 + --file /path/to/file.yaml

328 + Read yaml configuration file for instructions.

329 +

330 + --show-config

331 + Show the configuration in yaml format before starting the job.

332 + This is also an easy way to convert command line parameters to yaml.

333 +

334 + --filename-key KEY

335 Add a field with KEY as the key and the current filename as value.

336 Automatically detects filenames when piped after 'tail -F',

337 and tail matches multiple filenames.

338 To inject the filename when tailing a single file, use --inject.

339

333 - --unmatched-key=KEY

340 + --unmatched-key KEY

341 Include unmatched log entries in the output with KEY as the field name.

342 Use this to include unmatched entries to the output stream.

343 Usually it should be set to --unmatched-key=MESSAGE so that the

344 unmatched entry will appear as the log message in the journals.

345 Use --inject-unmatched to inject additional fields to unmatched lines.

346

340 - --duplicate=TARGET=KEY1[,KEY2[,KEY3[,...]]

347 + --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]

348 Create a new key called TARGET, duplicating the values of the keys

349 given. Useful for further processing. When multiple keys are given,

350 their values are separated by comma.

351 Up to 512 duplications can be given on the command line, and up to

352 20 keys per duplication command are allowed.

353

347 - --inject=LINE

354 + --inject LINE

355 Inject constant fields to the output (both matched and unmatched logs).

356 --inject entries are added to unmatched lines too, when their key is

357 not used in --inject-unmatched (--inject-unmatched override --inject).

358 Up to 512 fields can be injected.

359

353 - --inject-unmatched=LINE

360 + --inject-unmatched LINE

361 Inject lines into the output for each unmatched log entry.

362 Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched

363 lines as errors, so that they can easily be spotted in the journals.

364 Up to 512 such lines can be injected.

365

359 - --rewrite=KEY=/SearchPattern/ReplacePattern

366 + --rewrite KEY=/SearchPattern/ReplacePattern

367 Apply a rewrite rule to the values of a specific key.

368 The first character after KEY= is the separator, which should also

369 be used between the search pattern and the replacement pattern.

370 The search pattern is a PCRE2 regular expression, and the replacement

371 pattern supports literals and named capture groups from the search pattern.

372 Example:

366 - --rewrite=DATE=/^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})$/

373 + --rewrite DATE=/^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})$/

374 ${day}/${month}/${year}

375 This will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.

376

@@ -388,6 +395,8 @@ Options:

395 are usually valid PCRE2 patterns too.

396 Regular expressions without named groups are ignored.

397

398 +The program accepts all parameters as both --option=value and --option value.

399 +

400 The maximum line length accepted is 1048576 characters.

401 The maximum number of fields in the PCRE2 pattern is 1024.

402

@@ -449,6 +458,105 @@ JOURNAL FIELDS RULES (enforced by systemd-journald)

458

459 You can find the most common fields at 'man systemd.journal-fields'.

460

461 +Example YAML file:

462 +

463 +--------------------------------------------------------------------------------

464 +# Netdata log2journal Configuration Template

465 +# The following parses nginx log files using the combined format.

466 +

467 +# The PCRE2 pattern to match log entries and give names to the fields.

468 +# The journal will have these names, so follow their rules. You can

469 +# initiate an extended PCRE2 pattern by starting the pattern with (?x)

470 +pattern: |

471 + (?x) # Enable PCRE2 extended mode

472 + ^

473 + (?<NGINX_REMOTE_ADDR>[^ ]+) \s - \s # NGINX_REMOTE_ADDR

474 + (?<NGINX_REMOTE_USER>[^ ]+) \s # NGINX_REMOTE_USER

475 + \[

476 + (?<NGINX_TIME_LOCAL>[^\]]+) # NGINX_TIME_LOCAL

477 + \]

478 + \s+ "

479 + (?<MESSAGE>

480 + (?<NGINX_METHOD>[A-Z]+) \s+ # NGINX_METHOD

481 + (?<NGINX_URL>[^ ]+) \s+

482 + HTTP/(?<NGINX_HTTP_VERSION>[^"]+)

483 + )

484 + " \s+

485 + (?<NGINX_STATUS>\d+) \s+ # NGINX_STATUS

486 + (?<NGINX_BODY_BYTES_SENT>\d+) \s+ # NGINX_BODY_BYTES_SENT

487 + "(?<NGINX_HTTP_REFERER>[^"]*)" \s+ # NGINX_HTTP_REFERER

488 + "(?<NGINX_HTTP_USER_AGENT>[^"]*)" # NGINX_HTTP_USER_AGENT

489 +

490 +# When log2journal can detect the filename of each log entry (tail gives it

491 +# only when it tails multiple files), this key will be used to send the

492 +# filename to the journals.

493 +filename:

494 + key: NGINX_LOG_FILENAME

495 +

496 +# Duplicate fields under a different name. You can duplicate multiple fields

497 +# to a new one and then use rewrite rules to change its value.

498 +duplicate:

499 +

500 + # we insert the field PRIORITY as a copy of NGINX_STATUS.

501 + - key: PRIORITY

502 + values_of:

503 + - NGINX_STATUS

504 +

505 + # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.

506 + - key: NGINX_STATUS_FAMILY

507 + values_of:

508 + - NGINX_STATUS

509 +

510 +# Inject constant fields into the journal logs.

511 +inject:

512 + - key: SYSLOG_IDENTIFIER

513 + value: "nginx-log"

514 +

515 +# Rewrite the value of fields (including the duplicated ones).

516 +# The search pattern can have named groups, and the replace pattern can use

517 +# them as ${name}.

518 +rewrite:

519 + # PRIORTY is a duplicate of NGINX_STATUS

520 + # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug

521 + - key: "PRIORITY"

522 + search: "^[123]"

523 + replace: 6

524 +

525 + - key: "PRIORITY"

526 + search: "^4"

527 + replace: 5

528 +

529 + - key: "PRIORITY"

530 + search: "^5"

531 + replace: 3

532 +

533 + - key: "PRIORITY"

534 + search: ".*"

535 + replace: 4

536 +

537 + # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS

538 + - key: "NGINX_STATUS_FAMILY"

539 + search: "^(?<first_digit>[1-5])"

540 + replace: "${first_digit}xx"

541 +

542 + - key: "NGINX_STATUS_FAMILY"

543 + search: ".*"

544 + replace: "UNKNOWN"

545 +

546 +# Control what to do when input logs do not match the main PCRE2 pattern.

547 +unmatched:

548 + # The journal key to log the PCRE2 error message to.

549 + # Set this to MESSAGE, so you to see the error in the log.

550 + key: MESSAGE

551 +

552 + # Inject static fields to the unmatched entries.

553 + # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.

554 + inject:

555 + - key: PRIORITY

556 + value: 1

557 +

558 +--------------------------------------------------------------------------------

559 +

560 ```

561

562 ## `systemd-cat-native` options