@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
c 377 lines 20.3 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "log2journal.h"
4
5 static void config_dir_print_available(void) {
6 const char *path = LOG2JOURNAL_CONFIG_PATH;
7 DIR *dir;
8 struct dirent *entry;
9
10 dir = opendir(path);
11
12 if (dir == NULL) {
13 l2j_log(" >>> Cannot open directory:\n %s", path);
14 return;
15 }
16
17 size_t column_width = 80;
18 size_t current_columns = 7; // Start with 7 spaces for the first line
19
20 while ((entry = readdir(dir))) {
21 if (entry->d_type == DT_REG) { // Check if it's a regular file
22 const char *file_name = entry->d_name;
23 size_t len = strlen(file_name);
24 if (len >= 5 && strcmp(file_name + len - 5, ".yaml") == 0) {
25 // Remove the ".yaml" extension
26 len -= 5;
27 if (current_columns == 7) {
28 printf(" "); // Print 7 spaces at the beginning of a new line
29 }
30 if (current_columns + len + 1 > column_width) {
31 // Start a new line if the current line is full
32 printf("\n "); // Print newline and 7 spaces
33 current_columns = 7;
34 }
35 printf("%.*s ", (int)len, file_name); // Print the filename without extension
36 current_columns += len + 1; // Add filename length and a space
37 }
38 }
39 }
40
41 closedir(dir);
42 printf("\n"); // Add a newline at the end
43 }
44
45 void log_job_command_line_help(const char *name) {
46 printf("\n");
47 printf("Netdata log2journal " NETDATA_VERSION "\n");
48 printf("\n");
49 printf("Convert logs to systemd Journal Export Format.\n");
50 printf("\n");
51 printf(" - JSON logs: extracts all JSON fields.\n");
52 printf(" - logfmt logs: extracts all logfmt fields.\n");
53 printf(" - free-form logs: uses PCRE2 patterns to extracts fields.\n");
54 printf("\n");
55 printf("Usage: %s [OPTIONS] PATTERN|json\n", name);
56 printf("\n");
57 printf("Options:\n");
58 printf("\n");
59 #ifdef HAVE_LIBYAML
60 printf(" --file /path/to/file.yaml or -f /path/to/file.yaml\n");
61 printf(" Read yaml configuration file for instructions.\n");
62 printf("\n");
63 printf(" --config CONFIG_NAME or -c CONFIG_NAME\n");
64 printf(" Run with the internal YAML configuration named CONFIG_NAME.\n");
65 printf(" Available internal YAML configs:\n");
66 printf("\n");
67 config_dir_print_available();
68 printf("\n");
69 #else
70 printf(" IMPORTANT:\n");
71 printf(" YAML configuration parsing is not compiled in this binary.\n");
72 printf("\n");
73 #endif
74 printf("--------------------------------------------------------------------------------\n");
75 printf(" INPUT PROCESSING\n");
76 printf("\n");
77 printf(" PATTERN\n");
78 printf(" PATTERN should be a valid PCRE2 regular expression.\n");
79 printf(" RE2 regular expressions (like the ones usually used in Go applications),\n");
80 printf(" are usually valid PCRE2 patterns too.\n");
81 printf(" Sub-expressions without named groups are evaluated, but their matches are\n");
82 printf(" not added to the output.\n");
83 printf("\n");
84 printf(" - JSON mode\n");
85 printf(" JSON mode is enabled when the pattern is set to: json\n");
86 printf(" Field names are extracted from the JSON logs and are converted to the\n");
87 printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n");
88 printf("\n");
89 printf(" - logfmt mode\n");
90 printf(" logfmt mode is enabled when the pattern is set to: logfmt\n");
91 printf(" Field names are extracted from the logfmt logs and are converted to the\n");
92 printf(" format expected by Journal Export Format (all caps, only _ is allowed).\n");
93 printf("\n");
94 printf(" All keys extracted from the input, are transliterated to match Journal\n");
95 printf(" semantics (capital A-Z, digits 0-9, underscore).\n");
96 printf("\n");
97 printf(" In a YAML file:\n");
98 printf(" ```yaml\n");
99 printf(" pattern: 'PCRE2 pattern | json | logfmt'\n");
100 printf(" ```\n");
101 printf("\n");
102 printf("--------------------------------------------------------------------------------\n");
103 printf(" GLOBALS\n");
104 printf("\n");
105 printf(" --prefix PREFIX\n");
106 printf(" Prefix all fields with PREFIX. The PREFIX is added before any other\n");
107 printf(" processing, so that the extracted keys have to be matched with the PREFIX in\n");
108 printf(" them. PREFIX is NOT transliterated and it is assumed to be systemd-journal\n");
109 printf(" friendly.\n");
110 printf("\n");
111 printf(" In a YAML file:\n");
112 printf(" ```yaml\n");
113 printf(" prefix: 'PREFIX_' # prepend all keys with this prefix.\n");
114 printf(" ```\n");
115 printf("\n");
116 printf(" --filename-key KEY\n");
117 printf(" Add a field with KEY as the key and the current filename as value.\n");
118 printf(" Automatically detects filenames when piped after 'tail -F',\n");
119 printf(" and tail matches multiple filenames.\n");
120 printf(" To inject the filename when tailing a single file, use --inject.\n");
121 printf("\n");
122 printf(" In a YAML file:\n");
123 printf(" ```yaml\n");
124 printf(" filename:\n");
125 printf(" key: KEY\n");
126 printf(" ```\n");
127 printf("\n");
128 printf("--------------------------------------------------------------------------------\n");
129 printf(" RENAMING OF KEYS\n");
130 printf("\n");
131 printf(" --rename NEW=OLD\n");
132 printf(" Rename fields. OLD has been transliterated and PREFIX has been added.\n");
133 printf(" NEW is assumed to be systemd journal friendly.\n");
134 printf("\n");
135 printf(" Up to %d renaming rules are allowed.\n", MAX_RENAMES);
136 printf("\n");
137 printf(" In a YAML file:\n");
138 printf(" ```yaml\n");
139 printf(" rename:\n");
140 printf(" - new_key: KEY1\n");
141 printf(" old_key: KEY2 # transliterated with PREFIX added\n");
142 printf(" - new_key: KEY3\n");
143 printf(" old_key: KEY4 # transliterated with PREFIX added\n");
144 printf(" # add as many as required\n");
145 printf(" ```\n");
146 printf("\n");
147 printf("--------------------------------------------------------------------------------\n");
148 printf(" INJECTING NEW KEYS\n");
149 printf("\n");
150 printf(" --inject KEY=VALUE\n");
151 printf(" Inject constant fields to the output (both matched and unmatched logs).\n");
152 printf(" --inject entries are added to unmatched lines too, when their key is\n");
153 printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n");
154 printf(" VALUE can use variable like ${OTHER_KEY} to be replaced with the values\n");
155 printf(" of other keys available.\n");
156 printf("\n");
157 printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS);
158 printf("\n");
159 printf(" In a YAML file:\n");
160 printf(" ```yaml\n");
161 printf(" inject:\n");
162 printf(" - key: KEY1\n");
163 printf(" value: 'VALUE1'\n");
164 printf(" - key: KEY2\n");
165 printf(" value: '${KEY3}${KEY4}' # gets the values of KEY3 and KEY4\n");
166 printf(" # add as many as required\n");
167 printf(" ```\n");
168 printf("\n");
169 printf("--------------------------------------------------------------------------------\n");
170 printf(" REWRITING KEY VALUES\n");
171 printf("\n");
172 printf(" --rewrite KEY=/MATCH/REPLACE[/OPTIONS]\n");
173 printf(" Apply a rewrite rule to the values of a specific key.\n");
174 printf(" The first character after KEY= is the separator, which should also\n");
175 printf(" be used between the MATCH, REPLACE and OPTIONS.\n");
176 printf("\n");
177 printf(" OPTIONS can be a comma separated list of `non-empty`, `dont-stop` and\n");
178 printf(" `inject`.\n");
179 printf("\n");
180 printf(" When `non-empty` is given, MATCH is expected to be a variable\n");
181 printf(" substitution using `${KEY1}${KEY2}`. Once the substitution is completed\n");
182 printf(" the rule is matching the KEY only if the result is not empty.\n");
183 printf(" When `non-empty` is not set, the MATCH string is expected to be a PCRE2\n");
184 printf(" regular expression to be checked against the KEY value. This PCRE2\n");
185 printf(" pattern may include named groups to extract parts of the KEY's value.\n");
186 printf("\n");
187 printf(" REPLACE supports variable substitution like `${variable}` against MATCH\n");
188 printf(" named groups (when MATCH is a PCRE2 pattern) and `${KEY}` against the\n");
189 printf(" keys defined so far.\n");
190 printf("\n");
191 printf(" Example:\n");
192 printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n");
193 printf(" ${day}/${month}/${year}\n");
194 printf(" The above will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.\n");
195 printf("\n");
196 printf(" Only one rewrite rule is applied per key; the sequence of rewrites for a\n");
197 printf(" given key, stops once a rule matches it. This allows providing a sequence\n");
198 printf(" of independent rewriting rules for the same key, matching the different\n");
199 printf(" values the key may get, and also provide a catch-all rewrite rule at the\n");
200 printf(" end, for setting the key value if no other rule matched it. The rewrite\n");
201 printf(" rule can allow processing more rewrite rules when OPTIONS includes\n");
202 printf(" the keyword 'dont-stop'.\n");
203 printf("\n");
204 printf(" Up to %d rewriting rules are allowed.\n", MAX_REWRITES);
205 printf("\n");
206 printf(" In a YAML file:\n");
207 printf(" ```yaml\n");
208 printf(" rewrite:\n");
209 printf(" # the order if these rules in important - processed top to bottom\n");
210 printf(" - key: KEY1\n");
211 printf(" match: 'PCRE2 PATTERN WITH NAMED GROUPS'\n");
212 printf(" value: 'all match fields and input keys as ${VARIABLE}'\n");
213 printf(" inject: BOOLEAN # yes = inject the field, don't just rewrite it\n");
214 printf(" stop: BOOLEAN # no = continue processing, don't stop if matched\n");
215 printf(" - key: KEY2\n");
216 printf(" non_empty: '${KEY3}${KEY4}' # match only if this evaluates to non empty\n");
217 printf(" value: 'all input keys as ${VARIABLE}'\n");
218 printf(" inject: BOOLEAN # yes = inject the field, don't just rewrite it\n");
219 printf(" stop: BOOLEAN # no = continue processing, don't stop if matched\n");
220 printf(" # add as many rewrites as required\n");
221 printf(" ```\n");
222 printf("\n");
223 printf(" By default rewrite rules are applied only on fields already defined.\n");
224 printf(" This allows shipping YAML files that include more rewrites than are\n");
225 printf(" required for a specific input file.\n");
226 printf(" Rewrite rules however allow injecting new fields when OPTIONS include\n");
227 printf(" the keyword `inject` or in YAML `inject: yes` is given.\n");
228 printf("\n");
229 printf(" MATCH on the command line can be empty to define an unconditional rule.\n");
230 printf(" Similarly, `match` and `non_empty` can be omitted in the YAML file.");
231 printf("\n");
232 printf("--------------------------------------------------------------------------------\n");
233 printf(" UNMATCHED LINES\n");
234 printf("\n");
235 printf(" --unmatched-key KEY\n");
236 printf(" Include unmatched log entries in the output with KEY as the field name.\n");
237 printf(" Use this to include unmatched entries to the output stream.\n");
238 printf(" Usually it should be set to --unmatched-key=MESSAGE so that the\n");
239 printf(" unmatched entry will appear as the log message in the journals.\n");
240 printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n");
241 printf("\n");
242 printf(" In a YAML file:\n");
243 printf(" ```yaml\n");
244 printf(" unmatched:\n");
245 printf(" key: MESSAGE # inject the error log as MESSAGE\n");
246 printf(" ```\n");
247 printf("\n");
248 printf(" --inject-unmatched LINE\n");
249 printf(" Inject lines into the output for each unmatched log entry.\n");
250 printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n");
251 printf(" lines as errors, so that they can easily be spotted in the journals.\n");
252 printf("\n");
253 printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS);
254 printf("\n");
255 printf(" In a YAML file:\n");
256 printf(" ```yaml\n");
257 printf(" unmatched:\n");
258 printf(" key: MESSAGE # inject the error log as MESSAGE\n");
259 printf(" inject::\n");
260 printf(" - key: KEY1\n");
261 printf(" value: 'VALUE1'\n");
262 printf(" # add as many constants as required\n");
263 printf(" ```\n");
264 printf("\n");
265 printf("--------------------------------------------------------------------------------\n");
266 printf(" FILTERING\n");
267 printf("\n");
268 printf(" --include PATTERN\n");
269 printf(" Include only keys matching the PCRE2 PATTERN.\n");
270 printf(" Useful when parsing JSON of logfmt logs, to include only the keys given.\n");
271 printf(" The keys are matched after the PREFIX has been added to them.\n");
272 printf("\n");
273 printf(" --exclude PATTERN\n");
274 printf(" Exclude the keys matching the PCRE2 PATTERN.\n");
275 printf(" Useful when parsing JSON of logfmt logs, to exclude some of the keys given.\n");
276 printf(" The keys are matched after the PREFIX has been added to them.\n");
277 printf("\n");
278 printf(" When both include and exclude patterns are set and both match a key,\n");
279 printf(" exclude wins and the key will not be added, like a pipeline, we first\n");
280 printf(" include it and then exclude it.\n");
281 printf("\n");
282 printf(" In a YAML file:\n");
283 printf(" ```yaml\n");
284 printf(" filter:\n");
285 printf(" include: 'PCRE2 PATTERN MATCHING KEY NAMES TO INCLUDE'\n");
286 printf(" exclude: 'PCRE2 PATTERN MATCHING KEY NAMES TO EXCLUDE'\n");
287 printf(" ```\n");
288 printf("\n");
289 printf("--------------------------------------------------------------------------------\n");
290 printf(" OTHER\n");
291 printf("\n");
292 printf(" -h, or --help\n");
293 printf(" Display this help and exit.\n");
294 printf("\n");
295 printf(" --show-config\n");
296 printf(" Show the configuration in YAML format before starting the job.\n");
297 printf(" This is also an easy way to convert command line parameters to yaml.\n");
298 printf("\n");
299 printf("The program accepts all parameters as both --option=value and --option value.\n");
300 printf("\n");
301 printf("The maximum log line length accepted is %d characters.\n", MAX_LINE_LENGTH);
302 printf("\n");
303 printf("PIPELINE AND SEQUENCE OF PROCESSING\n");
304 printf("\n");
305 printf("This is a simple diagram of the pipeline taking place:\n");
306 printf(" \n");
307 printf(" +---------------------------------------------------+ \n");
308 printf(" | INPUT | \n");
309 printf(" | read one log line at a time | \n");
310 printf(" +---------------------------------------------------+ \n");
311 printf(" v v v v v v \n");
312 printf(" +---------------------------------------------------+ \n");
313 printf(" | EXTRACT FIELDS AND VALUES | \n");
314 printf(" | JSON, logfmt, or pattern based | \n");
315 printf(" | (apply optional PREFIX - all keys use capitals) | \n");
316 printf(" +---------------------------------------------------+ \n");
317 printf(" v v v v v v \n");
318 printf(" +---------------------------------------------------+ \n");
319 printf(" | RENAME FIELDS | \n");
320 printf(" | change the names of the fields | \n");
321 printf(" +---------------------------------------------------+ \n");
322 printf(" v v v v v v \n");
323 printf(" +---------------------------------------------------+ \n");
324 printf(" | INJECT NEW FIELDS | \n");
325 printf(" | constants, or other field values as variables | \n");
326 printf(" +---------------------------------------------------+ \n");
327 printf(" v v v v v v \n");
328 printf(" +---------------------------------------------------+ \n");
329 printf(" | REWRITE FIELD VALUES | \n");
330 printf(" | pipeline multiple rewriting rules to alter | \n");
331 printf(" | the values of the fields | \n");
332 printf(" +---------------------------------------------------+ \n");
333 printf(" v v v v v v \n");
334 printf(" +---------------------------------------------------+ \n");
335 printf(" | FILTER FIELDS | \n");
336 printf(" | use include and exclude patterns on the field | \n");
337 printf(" | names, to select which fields are sent to journal | \n");
338 printf(" +---------------------------------------------------+ \n");
339 printf(" v v v v v v \n");
340 printf(" +---------------------------------------------------+ \n");
341 printf(" | OUTPUT | \n");
342 printf(" | generate Journal Export Format | \n");
343 printf(" +---------------------------------------------------+ \n");
344 printf(" \n");
345 printf("--------------------------------------------------------------------------------\n");
346 printf("JOURNAL FIELDS RULES (enforced by systemd-journald)\n");
347 printf("\n");
348 printf(" - field names can be up to 64 characters\n");
349 printf(" - the only allowed field characters are A-Z, 0-9 and underscore\n");
350 printf(" - the first character of fields cannot be a digit\n");
351 printf(" - protected journal fields start with underscore:\n");
352 printf(" * they are accepted by systemd-journal-remote\n");
353 printf(" * they are NOT accepted by a local systemd-journald\n");
354 printf("\n");
355 printf(" For best results, always include these fields:\n");
356 printf("\n");
357 printf(" MESSAGE=TEXT\n");
358 printf(" The MESSAGE is the body of the log entry.\n");
359 printf(" This field is what we usually see in our logs.\n");
360 printf("\n");
361 printf(" PRIORITY=NUMBER\n");
362 printf(" PRIORITY sets the severity of the log entry.\n");
363 printf(" 0=emerg, 1=alert, 2=crit, 3=err, 4=warn, 5=notice, 6=info, 7=debug\n");
364 printf(" - Emergency events (0) are usually broadcast to all terminals.\n");
365 printf(" - Emergency, alert, critical, and error (0-3) are usually colored red.\n");
366 printf(" - Warning (4) entries are usually colored yellow.\n");
367 printf(" - Notice (5) entries are usually bold or have a brighter white color.\n");
368 printf(" - Info (6) entries are the default.\n");
369 printf(" - Debug (7) entries are usually grayed or dimmed.\n");
370 printf("\n");
371 printf(" SYSLOG_IDENTIFIER=NAME\n");
372 printf(" SYSLOG_IDENTIFIER sets the name of application.\n");
373 printf(" Use something descriptive, like: SYSLOG_IDENTIFIER=nginx-logs\n");
374 printf("\n");
375 printf("You can find the most common fields at 'man systemd.journal-fields'.\n");
376 printf("\n");
377 }