| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "log2journal.h" |
| 4 | |
| 5 | static void config_dir_print_available(void) { |
| 6 | const char *path = LOG2JOURNAL_CONFIG_PATH; |
| 7 | DIR *dir; |
| 8 | struct dirent *entry; |
| 9 | |
| 10 | dir = opendir(path); |
| 11 | |
| 12 | if (dir == NULL) { |
| 13 | l2j_log(" >>> Cannot open directory:\n %s", path); |
| 14 | return; |
| 15 | } |
| 16 | |
| 17 | size_t column_width = 80; |
| 18 | size_t current_columns = 7; // Start with 7 spaces for the first line |
| 19 | |
| 20 | while ((entry = readdir(dir))) { |
| 21 | if (entry->d_type == DT_REG) { // Check if it's a regular file |
| 22 | const char *file_name = entry->d_name; |
| 23 | size_t len = strlen(file_name); |
| 24 | if (len >= 5 && strcmp(file_name + len - 5, ".yaml") == 0) { |
| 25 | // Remove the ".yaml" extension |
| 26 | len -= 5; |
| 27 | if (current_columns == 7) { |
| 28 | printf(" "); // Print 7 spaces at the beginning of a new line |
| 29 | } |
| 30 | if (current_columns + len + 1 > column_width) { |
| 31 | // Start a new line if the current line is full |
| 32 | printf("\n "); // Print newline and 7 spaces |
| 33 | current_columns = 7; |
| 34 | } |
| 35 | printf("%.*s ", (int)len, file_name); // Print the filename without extension |
| 36 | current_columns += len + 1; // Add filename length and a space |
| 37 | } |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | closedir(dir); |
| 42 | printf("\n"); // Add a newline at the end |
| 43 | } |
| 44 | |
| 45 | void log_job_command_line_help(const char *name) { |
| 46 | printf("\n"); |
| 47 | printf("Netdata log2journal " NETDATA_VERSION "\n"); |
| 48 | printf("\n"); |
| 49 | printf("Convert logs to systemd Journal Export Format.\n"); |
| 50 | printf("\n"); |
| 51 | printf(" - JSON logs: extracts all JSON fields.\n"); |
| 52 | printf(" - logfmt logs: extracts all logfmt fields.\n"); |
| 53 | printf(" - free-form logs: uses PCRE2 patterns to extracts fields.\n"); |
| 54 | printf("\n"); |
| 55 | printf("Usage: %s [OPTIONS] PATTERN|json\n", name); |
| 56 | printf("\n"); |
| 57 | printf("Options:\n"); |
| 58 | printf("\n"); |
| 59 | #ifdef HAVE_LIBYAML |
| 60 | printf(" --file /path/to/file.yaml or -f /path/to/file.yaml\n"); |
| 61 | printf(" Read yaml configuration file for instructions.\n"); |
| 62 | printf("\n"); |
| 63 | printf(" --config CONFIG_NAME or -c CONFIG_NAME\n"); |
| 64 | printf(" Run with the internal YAML configuration named CONFIG_NAME.\n"); |
| 65 | printf(" Available internal YAML configs:\n"); |
| 66 | printf("\n"); |
| 67 | config_dir_print_available(); |
| 68 | printf("\n"); |
| 69 | #else |
| 70 | printf(" IMPORTANT:\n"); |
| 71 | printf(" YAML configuration parsing is not compiled in this binary.\n"); |
| 72 | printf("\n"); |
| 73 | #endif |
| 74 | printf("--------------------------------------------------------------------------------\n"); |
| 75 | printf(" INPUT PROCESSING\n"); |
| 76 | printf("\n"); |
| 77 | printf(" PATTERN\n"); |
| 78 | printf(" PATTERN should be a valid PCRE2 regular expression.\n"); |
| 79 | printf(" RE2 regular expressions (like the ones usually used in Go applications),\n"); |
| 80 | printf(" are usually valid PCRE2 patterns too.\n"); |
| 81 | printf(" Sub-expressions without named groups are evaluated, but their matches are\n"); |
| 82 | printf(" not added to the output.\n"); |
| 83 | printf("\n"); |
| 84 | printf(" - JSON mode\n"); |
| 85 | printf(" JSON mode is enabled when the pattern is set to: json\n"); |
| 86 | printf(" Field names are extracted from the JSON logs and are converted to the\n"); |
| 87 | printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n"); |
| 88 | printf("\n"); |
| 89 | printf(" - logfmt mode\n"); |
| 90 | printf(" logfmt mode is enabled when the pattern is set to: logfmt\n"); |
| 91 | printf(" Field names are extracted from the logfmt logs and are converted to the\n"); |
| 92 | printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n"); |
| 93 | printf("\n"); |
| 94 | printf(" All keys extracted from the input, are transliterated to match Journal\n"); |
| 95 | printf(" semantics (capital A-Z, digits 0-9, underscore).\n"); |
| 96 | printf("\n"); |
| 97 | printf(" In a YAML file:\n"); |
| 98 | printf(" ```yaml\n"); |
| 99 | printf(" pattern: 'PCRE2 pattern | json | logfmt'\n"); |
| 100 | printf(" ```\n"); |
| 101 | printf("\n"); |
| 102 | printf("--------------------------------------------------------------------------------\n"); |
| 103 | printf(" GLOBALS\n"); |
| 104 | printf("\n"); |
| 105 | printf(" --prefix PREFIX\n"); |
| 106 | printf(" Prefix all fields with PREFIX. The PREFIX is added before any other\n"); |
| 107 | printf(" processing, so that the extracted keys have to be matched with the PREFIX in\n"); |
| 108 | printf(" them. PREFIX is NOT transliterated and it is assumed to be systemd-journal\n"); |
| 109 | printf(" friendly.\n"); |
| 110 | printf("\n"); |
| 111 | printf(" In a YAML file:\n"); |
| 112 | printf(" ```yaml\n"); |
| 113 | printf(" prefix: 'PREFIX_' # prepend all keys with this prefix.\n"); |
| 114 | printf(" ```\n"); |
| 115 | printf("\n"); |
| 116 | printf(" --filename-key KEY\n"); |
| 117 | printf(" Add a field with KEY as the key and the current filename as value.\n"); |
| 118 | printf(" Automatically detects filenames when piped after 'tail -F',\n"); |
| 119 | printf(" and tail matches multiple filenames.\n"); |
| 120 | printf(" To inject the filename when tailing a single file, use --inject.\n"); |
| 121 | printf("\n"); |
| 122 | printf(" In a YAML file:\n"); |
| 123 | printf(" ```yaml\n"); |
| 124 | printf(" filename:\n"); |
| 125 | printf(" key: KEY\n"); |
| 126 | printf(" ```\n"); |
| 127 | printf("\n"); |
| 128 | printf("--------------------------------------------------------------------------------\n"); |
| 129 | printf(" RENAMING OF KEYS\n"); |
| 130 | printf("\n"); |
| 131 | printf(" --rename NEW=OLD\n"); |
| 132 | printf(" Rename fields. OLD has been transliterated and PREFIX has been added.\n"); |
| 133 | printf(" NEW is assumed to be systemd journal friendly.\n"); |
| 134 | printf("\n"); |
| 135 | printf(" Up to %d renaming rules are allowed.\n", MAX_RENAMES); |
| 136 | printf("\n"); |
| 137 | printf(" In a YAML file:\n"); |
| 138 | printf(" ```yaml\n"); |
| 139 | printf(" rename:\n"); |
| 140 | printf(" - new_key: KEY1\n"); |
| 141 | printf(" old_key: KEY2 # transliterated with PREFIX added\n"); |
| 142 | printf(" - new_key: KEY3\n"); |
| 143 | printf(" old_key: KEY4 # transliterated with PREFIX added\n"); |
| 144 | printf(" # add as many as required\n"); |
| 145 | printf(" ```\n"); |
| 146 | printf("\n"); |
| 147 | printf("--------------------------------------------------------------------------------\n"); |
| 148 | printf(" INJECTING NEW KEYS\n"); |
| 149 | printf("\n"); |
| 150 | printf(" --inject KEY=VALUE\n"); |
| 151 | printf(" Inject constant fields to the output (both matched and unmatched logs).\n"); |
| 152 | printf(" --inject entries are added to unmatched lines too, when their key is\n"); |
| 153 | printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n"); |
| 154 | printf(" VALUE can use variable like ${OTHER_KEY} to be replaced with the values\n"); |
| 155 | printf(" of other keys available.\n"); |
| 156 | printf("\n"); |
| 157 | printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS); |
| 158 | printf("\n"); |
| 159 | printf(" In a YAML file:\n"); |
| 160 | printf(" ```yaml\n"); |
| 161 | printf(" inject:\n"); |
| 162 | printf(" - key: KEY1\n"); |
| 163 | printf(" value: 'VALUE1'\n"); |
| 164 | printf(" - key: KEY2\n"); |
| 165 | printf(" value: '${KEY3}${KEY4}' # gets the values of KEY3 and KEY4\n"); |
| 166 | printf(" # add as many as required\n"); |
| 167 | printf(" ```\n"); |
| 168 | printf("\n"); |
| 169 | printf("--------------------------------------------------------------------------------\n"); |
| 170 | printf(" REWRITING KEY VALUES\n"); |
| 171 | printf("\n"); |
| 172 | printf(" --rewrite KEY=/MATCH/REPLACE[/OPTIONS]\n"); |
| 173 | printf(" Apply a rewrite rule to the values of a specific key.\n"); |
| 174 | printf(" The first character after KEY= is the separator, which should also\n"); |
| 175 | printf(" be used between the MATCH, REPLACE and OPTIONS.\n"); |
| 176 | printf("\n"); |
| 177 | printf(" OPTIONS can be a comma separated list of `non-empty`, `dont-stop` and\n"); |
| 178 | printf(" `inject`.\n"); |
| 179 | printf("\n"); |
| 180 | printf(" When `non-empty` is given, MATCH is expected to be a variable\n"); |
| 181 | printf(" substitution using `${KEY1}${KEY2}`. Once the substitution is completed\n"); |
| 182 | printf(" the rule is matching the KEY only if the result is not empty.\n"); |
| 183 | printf(" When `non-empty` is not set, the MATCH string is expected to be a PCRE2\n"); |
| 184 | printf(" regular expression to be checked against the KEY value. This PCRE2\n"); |
| 185 | printf(" pattern may include named groups to extract parts of the KEY's value.\n"); |
| 186 | printf("\n"); |
| 187 | printf(" REPLACE supports variable substitution like `${variable}` against MATCH\n"); |
| 188 | printf(" named groups (when MATCH is a PCRE2 pattern) and `${KEY}` against the\n"); |
| 189 | printf(" keys defined so far.\n"); |
| 190 | printf("\n"); |
| 191 | printf(" Example:\n"); |
| 192 | printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n"); |
| 193 | printf(" ${day}/${month}/${year}\n"); |
| 194 | printf(" The above will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.\n"); |
| 195 | printf("\n"); |
| 196 | printf(" Only one rewrite rule is applied per key; the sequence of rewrites for a\n"); |
| 197 | printf(" given key, stops once a rule matches it. This allows providing a sequence\n"); |
| 198 | printf(" of independent rewriting rules for the same key, matching the different\n"); |
| 199 | printf(" values the key may get, and also provide a catch-all rewrite rule at the\n"); |
| 200 | printf(" end, for setting the key value if no other rule matched it. The rewrite\n"); |
| 201 | printf(" rule can allow processing more rewrite rules when OPTIONS includes\n"); |
| 202 | printf(" the keyword 'dont-stop'.\n"); |
| 203 | printf("\n"); |
| 204 | printf(" Up to %d rewriting rules are allowed.\n", MAX_REWRITES); |
| 205 | printf("\n"); |
| 206 | printf(" In a YAML file:\n"); |
| 207 | printf(" ```yaml\n"); |
| 208 | printf(" rewrite:\n"); |
| 209 | printf(" # the order if these rules in important - processed top to bottom\n"); |
| 210 | printf(" - key: KEY1\n"); |
| 211 | printf(" match: 'PCRE2 PATTERN WITH NAMED GROUPS'\n"); |
| 212 | printf(" value: 'all match fields and input keys as ${VARIABLE}'\n"); |
| 213 | printf(" inject: BOOLEAN # yes = inject the field, don't just rewrite it\n"); |
| 214 | printf(" stop: BOOLEAN # no = continue processing, don't stop if matched\n"); |
| 215 | printf(" - key: KEY2\n"); |
| 216 | printf(" non_empty: '${KEY3}${KEY4}' # match only if this evaluates to non empty\n"); |
| 217 | printf(" value: 'all input keys as ${VARIABLE}'\n"); |
| 218 | printf(" inject: BOOLEAN # yes = inject the field, don't just rewrite it\n"); |
| 219 | printf(" stop: BOOLEAN # no = continue processing, don't stop if matched\n"); |
| 220 | printf(" # add as many rewrites as required\n"); |
| 221 | printf(" ```\n"); |
| 222 | printf("\n"); |
| 223 | printf(" By default rewrite rules are applied only on fields already defined.\n"); |
| 224 | printf(" This allows shipping YAML files that include more rewrites than are\n"); |
| 225 | printf(" required for a specific input file.\n"); |
| 226 | printf(" Rewrite rules however allow injecting new fields when OPTIONS include\n"); |
| 227 | printf(" the keyword `inject` or in YAML `inject: yes` is given.\n"); |
| 228 | printf("\n"); |
| 229 | printf(" MATCH on the command line can be empty to define an unconditional rule.\n"); |
| 230 | printf(" Similarly, `match` and `non_empty` can be omitted in the YAML file."); |
| 231 | printf("\n"); |
| 232 | printf("--------------------------------------------------------------------------------\n"); |
| 233 | printf(" UNMATCHED LINES\n"); |
| 234 | printf("\n"); |
| 235 | printf(" --unmatched-key KEY\n"); |
| 236 | printf(" Include unmatched log entries in the output with KEY as the field name.\n"); |
| 237 | printf(" Use this to include unmatched entries to the output stream.\n"); |
| 238 | printf(" Usually it should be set to --unmatched-key=MESSAGE so that the\n"); |
| 239 | printf(" unmatched entry will appear as the log message in the journals.\n"); |
| 240 | printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n"); |
| 241 | printf("\n"); |
| 242 | printf(" In a YAML file:\n"); |
| 243 | printf(" ```yaml\n"); |
| 244 | printf(" unmatched:\n"); |
| 245 | printf(" key: MESSAGE # inject the error log as MESSAGE\n"); |
| 246 | printf(" ```\n"); |
| 247 | printf("\n"); |
| 248 | printf(" --inject-unmatched LINE\n"); |
| 249 | printf(" Inject lines into the output for each unmatched log entry.\n"); |
| 250 | printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n"); |
| 251 | printf(" lines as errors, so that they can easily be spotted in the journals.\n"); |
| 252 | printf("\n"); |
| 253 | printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS); |
| 254 | printf("\n"); |
| 255 | printf(" In a YAML file:\n"); |
| 256 | printf(" ```yaml\n"); |
| 257 | printf(" unmatched:\n"); |
| 258 | printf(" key: MESSAGE # inject the error log as MESSAGE\n"); |
| 259 | printf(" inject::\n"); |
| 260 | printf(" - key: KEY1\n"); |
| 261 | printf(" value: 'VALUE1'\n"); |
| 262 | printf(" # add as many constants as required\n"); |
| 263 | printf(" ```\n"); |
| 264 | printf("\n"); |
| 265 | printf("--------------------------------------------------------------------------------\n"); |
| 266 | printf(" FILTERING\n"); |
| 267 | printf("\n"); |
| 268 | printf(" --include PATTERN\n"); |
| 269 | printf(" Include only keys matching the PCRE2 PATTERN.\n"); |
| 270 | printf(" Useful when parsing JSON of logfmt logs, to include only the keys given.\n"); |
| 271 | printf(" The keys are matched after the PREFIX has been added to them.\n"); |
| 272 | printf("\n"); |
| 273 | printf(" --exclude PATTERN\n"); |
| 274 | printf(" Exclude the keys matching the PCRE2 PATTERN.\n"); |
| 275 | printf(" Useful when parsing JSON of logfmt logs, to exclude some of the keys given.\n"); |
| 276 | printf(" The keys are matched after the PREFIX has been added to them.\n"); |
| 277 | printf("\n"); |
| 278 | printf(" When both include and exclude patterns are set and both match a key,\n"); |
| 279 | printf(" exclude wins and the key will not be added, like a pipeline, we first\n"); |
| 280 | printf(" include it and then exclude it.\n"); |
| 281 | printf("\n"); |
| 282 | printf(" In a YAML file:\n"); |
| 283 | printf(" ```yaml\n"); |
| 284 | printf(" filter:\n"); |
| 285 | printf(" include: 'PCRE2 PATTERN MATCHING KEY NAMES TO INCLUDE'\n"); |
| 286 | printf(" exclude: 'PCRE2 PATTERN MATCHING KEY NAMES TO EXCLUDE'\n"); |
| 287 | printf(" ```\n"); |
| 288 | printf("\n"); |
| 289 | printf("--------------------------------------------------------------------------------\n"); |
| 290 | printf(" OTHER\n"); |
| 291 | printf("\n"); |
| 292 | printf(" -h, or --help\n"); |
| 293 | printf(" Display this help and exit.\n"); |
| 294 | printf("\n"); |
| 295 | printf(" --show-config\n"); |
| 296 | printf(" Show the configuration in YAML format before starting the job.\n"); |
| 297 | printf(" This is also an easy way to convert command line parameters to yaml.\n"); |
| 298 | printf("\n"); |
| 299 | printf("The program accepts all parameters as both --option=value and --option value.\n"); |
| 300 | printf("\n"); |
| 301 | printf("The maximum log line length accepted is %d characters.\n", MAX_LINE_LENGTH); |
| 302 | printf("\n"); |
| 303 | printf("PIPELINE AND SEQUENCE OF PROCESSING\n"); |
| 304 | printf("\n"); |
| 305 | printf("This is a simple diagram of the pipeline taking place:\n"); |
| 306 | printf(" \n"); |
| 307 | printf(" +---------------------------------------------------+ \n"); |
| 308 | printf(" | INPUT | \n"); |
| 309 | printf(" | read one log line at a time | \n"); |
| 310 | printf(" +---------------------------------------------------+ \n"); |
| 311 | printf(" v v v v v v \n"); |
| 312 | printf(" +---------------------------------------------------+ \n"); |
| 313 | printf(" | EXTRACT FIELDS AND VALUES | \n"); |
| 314 | printf(" | JSON, logfmt, or pattern based | \n"); |
| 315 | printf(" | (apply optional PREFIX - all keys use capitals) | \n"); |
| 316 | printf(" +---------------------------------------------------+ \n"); |
| 317 | printf(" v v v v v v \n"); |
| 318 | printf(" +---------------------------------------------------+ \n"); |
| 319 | printf(" | RENAME FIELDS | \n"); |
| 320 | printf(" | change the names of the fields | \n"); |
| 321 | printf(" +---------------------------------------------------+ \n"); |
| 322 | printf(" v v v v v v \n"); |
| 323 | printf(" +---------------------------------------------------+ \n"); |
| 324 | printf(" | INJECT NEW FIELDS | \n"); |
| 325 | printf(" | constants, or other field values as variables | \n"); |
| 326 | printf(" +---------------------------------------------------+ \n"); |
| 327 | printf(" v v v v v v \n"); |
| 328 | printf(" +---------------------------------------------------+ \n"); |
| 329 | printf(" | REWRITE FIELD VALUES | \n"); |
| 330 | printf(" | pipeline multiple rewriting rules to alter | \n"); |
| 331 | printf(" | the values of the fields | \n"); |
| 332 | printf(" +---------------------------------------------------+ \n"); |
| 333 | printf(" v v v v v v \n"); |
| 334 | printf(" +---------------------------------------------------+ \n"); |
| 335 | printf(" | FILTER FIELDS | \n"); |
| 336 | printf(" | use include and exclude patterns on the field | \n"); |
| 337 | printf(" | names, to select which fields are sent to journal | \n"); |
| 338 | printf(" +---------------------------------------------------+ \n"); |
| 339 | printf(" v v v v v v \n"); |
| 340 | printf(" +---------------------------------------------------+ \n"); |
| 341 | printf(" | OUTPUT | \n"); |
| 342 | printf(" | generate Journal Export Format | \n"); |
| 343 | printf(" +---------------------------------------------------+ \n"); |
| 344 | printf(" \n"); |
| 345 | printf("--------------------------------------------------------------------------------\n"); |
| 346 | printf("JOURNAL FIELDS RULES (enforced by systemd-journald)\n"); |
| 347 | printf("\n"); |
| 348 | printf(" - field names can be up to 64 characters\n"); |
| 349 | printf(" - the only allowed field characters are A-Z, 0-9 and underscore\n"); |
| 350 | printf(" - the first character of fields cannot be a digit\n"); |
| 351 | printf(" - protected journal fields start with underscore:\n"); |
| 352 | printf(" * they are accepted by systemd-journal-remote\n"); |
| 353 | printf(" * they are NOT accepted by a local systemd-journald\n"); |
| 354 | printf("\n"); |
| 355 | printf(" For best results, always include these fields:\n"); |
| 356 | printf("\n"); |
| 357 | printf(" MESSAGE=TEXT\n"); |
| 358 | printf(" The MESSAGE is the body of the log entry.\n"); |
| 359 | printf(" This field is what we usually see in our logs.\n"); |
| 360 | printf("\n"); |
| 361 | printf(" PRIORITY=NUMBER\n"); |
| 362 | printf(" PRIORITY sets the severity of the log entry.\n"); |
| 363 | printf(" 0=emerg, 1=alert, 2=crit, 3=err, 4=warn, 5=notice, 6=info, 7=debug\n"); |
| 364 | printf(" - Emergency events (0) are usually broadcast to all terminals.\n"); |
| 365 | printf(" - Emergency, alert, critical, and error (0-3) are usually colored red.\n"); |
| 366 | printf(" - Warning (4) entries are usually colored yellow.\n"); |
| 367 | printf(" - Notice (5) entries are usually bold or have a brighter white color.\n"); |
| 368 | printf(" - Info (6) entries are the default.\n"); |
| 369 | printf(" - Debug (7) entries are usually grayed or dimmed.\n"); |
| 370 | printf("\n"); |
| 371 | printf(" SYSLOG_IDENTIFIER=NAME\n"); |
| 372 | printf(" SYSLOG_IDENTIFIER sets the name of application.\n"); |
| 373 | printf(" Use something descriptive, like: SYSLOG_IDENTIFIER=nginx-logs\n"); |
| 374 | printf("\n"); |
| 375 | printf("You can find the most common fields at 'man systemd.journal-fields'.\n"); |
| 376 | printf("\n"); |
| 377 | } |