@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
h 290 lines 8.76 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef NETDATA_LOG2JOURNAL_H
4 #define NETDATA_LOG2JOURNAL_H
5
6 #include "libnetdata/libnetdata.h"
7 #include "log2journal-txt.h"
8 #include "log2journal-hashed-key.h"
9
10 // ----------------------------------------------------------------------------
11 // logging
12
13 // enable the compiler to check for printf like errors on our log2stderr() function
14 static inline void l2j_log(const char *format, ...) PRINTFLIKE(1, 2);
15 static inline void l2j_log(const char *format, ...) {
16 va_list args;
17 va_start(args, format);
18 vfprintf(stderr, format, args);
19 va_end(args);
20 fprintf(stderr, "\n");
21 }
22
23 // ----------------------------------------------------------------------------
24
25 #define PCRE2_CODE_UNIT_WIDTH 8
26 #include <pcre2.h>
27
28 #ifdef HAVE_LIBYAML
29 #include <yaml.h>
30 #endif
31
32 // ----------------------------------------------------------------------------
33 // hashtable for HASHED_KEY
34
35 struct hashed_key;
36 static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2);
37 #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_keys
38 #define SIMPLE_HASHTABLE_VALUE_TYPE HASHED_KEY *
39 #define SIMPLE_HASHTABLE_NAME _KEY
40 #include "libnetdata/simple_hashtable/simple_hashtable.h"
41
42 // ----------------------------------------------------------------------------
43
44 #define MAX_OUTPUT_KEYS 1024
45 #define MAX_LINE_LENGTH (1024 * 1024)
46 #define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
47 #define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
48 #define MAX_RENAMES (MAX_OUTPUT_KEYS / 2)
49
50 #define JOURNAL_MAX_KEY_LEN 64 // according to systemd-journald
51 #define JOURNAL_MAX_VALUE_LEN (48 * 1024) // according to systemd-journald
52
53 #define LOG2JOURNAL_CONFIG_PATH LIBCONFIG_DIR "/log2journal.d"
54
55 // ----------------------------------------------------------------------------
56 // character conversion for journal keys
57
58 extern const char journal_key_characters_map[256];
59
60 // ----------------------------------------------------------------------------
61 // copy to buffer, while ensuring there is no buffer overflow
62
63 static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) {
64 if(dst_size < 2) {
65 if(dst_size == 1)
66 *dst = '\0';
67
68 return 0;
69 }
70
71 if(src_len <= dst_size - 1) {
72 memcpy(dst, src, src_len);
73 dst[src_len] = '\0';
74 return src_len;
75 }
76 else {
77 memcpy(dst, src, dst_size - 1);
78 dst[dst_size - 1] = '\0';
79 return dst_size - 1;
80 }
81 }
82
83 // ----------------------------------------------------------------------------
84
85 typedef struct search_pattern {
86 const char *pattern;
87 pcre2_code *re;
88 pcre2_match_data *match_data;
89 TXT_L2J error;
90 } SEARCH_PATTERN;
91
92 void search_pattern_cleanup(SEARCH_PATTERN *sp);
93 bool search_pattern_set(SEARCH_PATTERN *sp, const char *search_pattern, size_t search_pattern_len);
94
95 static inline bool search_pattern_matches(SEARCH_PATTERN *sp, const char *value, size_t value_len) {
96 return pcre2_match(sp->re, (PCRE2_SPTR)value, value_len, 0, 0, sp->match_data, NULL) >= 0;
97 }
98
99 // ----------------------------------------------------------------------------
100
101 typedef struct replacement_node {
102 HASHED_KEY name;
103 bool is_variable;
104 bool logged_error;
105
106 struct replacement_node *next;
107 } REPLACE_NODE;
108
109 void replace_node_free(REPLACE_NODE *rpn);
110
111 typedef struct replace_pattern {
112 const char *pattern;
113 REPLACE_NODE *nodes;
114 bool has_variables;
115 } REPLACE_PATTERN;
116
117 void replace_pattern_cleanup(REPLACE_PATTERN *rp);
118 bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern);
119
120 // ----------------------------------------------------------------------------
121
122 typedef struct injection {
123 bool on_unmatched;
124 HASHED_KEY key;
125 REPLACE_PATTERN value;
126 } INJECTION;
127
128 void injection_cleanup(INJECTION *inj);
129
130 // ----------------------------------------------------------------------------
131
132 typedef struct key_rename {
133 HASHED_KEY new_key;
134 HASHED_KEY old_key;
135 } RENAME;
136
137 void rename_cleanup(RENAME *rn);
138
139 // ----------------------------------------------------------------------------
140
141 typedef enum __attribute__((__packed__)) {
142 RW_NONE = 0,
143 RW_MATCH_PCRE2 = (1 << 1), // a rewrite rule
144 RW_MATCH_NON_EMPTY = (1 << 2), // a rewrite rule
145 RW_DONT_STOP = (1 << 3),
146 RW_INJECT = (1 << 4),
147 } RW_FLAGS;
148
149 typedef struct key_rewrite {
150 RW_FLAGS flags;
151 HASHED_KEY key;
152 union {
153 SEARCH_PATTERN match_pcre2;
154 REPLACE_PATTERN match_non_empty;
155 };
156 REPLACE_PATTERN value;
157 } REWRITE;
158
159 void rewrite_cleanup(REWRITE *rw);
160
161 // ----------------------------------------------------------------------------
162 // A job configuration and runtime structures
163
164 typedef struct log_job {
165 bool show_config;
166
167 const char *pattern;
168 const char *prefix;
169
170 SIMPLE_HASHTABLE_KEY hashtable;
171
172 struct {
173 const char *buffer;
174 const char *trimmed;
175 size_t trimmed_len;
176 size_t size;
177 HASHED_KEY key;
178 } line;
179
180 struct {
181 SEARCH_PATTERN include;
182 SEARCH_PATTERN exclude;
183 } filter;
184
185 struct {
186 bool last_line_was_empty;
187 HASHED_KEY key;
188 TXT_L2J current;
189 } filename;
190
191 struct {
192 uint32_t used;
193 INJECTION keys[MAX_INJECTIONS];
194 } injections;
195
196 struct {
197 HASHED_KEY key;
198 struct {
199 uint32_t used;
200 INJECTION keys[MAX_INJECTIONS];
201 } injections;
202 } unmatched;
203
204 struct {
205 uint32_t used;
206 REWRITE array[MAX_REWRITES];
207 TXT_L2J tmp;
208 } rewrites;
209
210 struct {
211 uint32_t used;
212 RENAME array[MAX_RENAMES];
213 } renames;
214 } LOG_JOB;
215
216 // initialize a log job
217 void log_job_init(LOG_JOB *jb);
218
219 // free all resources consumed by the log job
220 void log_job_cleanup(LOG_JOB *jb);
221
222 // ----------------------------------------------------------------------------
223
224 // the entry point to send key value pairs to the output
225 // this implements the pipeline of processing renames, rewrites and duplications
226 void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len);
227
228 // ----------------------------------------------------------------------------
229 // configuration related
230
231 // management of configuration to set settings
232 bool log_job_filename_key_set(LOG_JOB *jb, const char *key, size_t key_len);
233 bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len);
234 bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
235 bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched);
236 bool log_job_rewrite_add(LOG_JOB *jb, const char *key, RW_FLAGS flags, const char *search_pattern, const char *replace_pattern);
237 bool log_job_rename_add(LOG_JOB *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len);
238 bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
239 bool log_job_exclude_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
240
241 // entry point to parse command line parameters
242 bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv);
243 void log_job_command_line_help(const char *name);
244
245 // ----------------------------------------------------------------------------
246 // YAML configuration related
247
248 #ifdef HAVE_LIBYAML
249 bool yaml_parse_file(const char *config_file_path, LOG_JOB *jb);
250 bool yaml_parse_config(const char *config_name, LOG_JOB *jb);
251 #endif
252
253 void log_job_configuration_to_yaml(LOG_JOB *jb);
254
255 // ----------------------------------------------------------------------------
256 // JSON parser
257
258 typedef struct log_json_state LOG_JSON_STATE;
259 LOG_JSON_STATE *json_parser_create(LOG_JOB *jb);
260 void json_parser_destroy(LOG_JSON_STATE *js);
261 const char *json_parser_error(LOG_JSON_STATE *js);
262 bool json_parse_document(LOG_JSON_STATE *js, const char *txt);
263 void json_test(void);
264
265 size_t parse_surrogate(const char *s, char *d, size_t *remaining);
266
267 // ----------------------------------------------------------------------------
268 // logfmt parser
269
270 typedef struct logfmt_state LOGFMT_STATE;
271 LOGFMT_STATE *logfmt_parser_create(LOG_JOB *jb);
272 void logfmt_parser_destroy(LOGFMT_STATE *lfs);
273 const char *logfmt_parser_error(LOGFMT_STATE *lfs);
274 bool logfmt_parse_document(LOGFMT_STATE *js, const char *txt);
275 void logfmt_test(void);
276
277 // ----------------------------------------------------------------------------
278 // pcre2 parser
279
280 typedef struct pcre2_state PCRE2_STATE;
281 PCRE2_STATE *pcre2_parser_create(LOG_JOB *jb);
282 void pcre2_parser_destroy(PCRE2_STATE *pcre2);
283 const char *pcre2_parser_error(PCRE2_STATE *pcre2);
284 bool pcre2_parse_document(PCRE2_STATE *pcre2, const char *txt, size_t len);
285 bool pcre2_has_error(PCRE2_STATE *pcre2);
286 void pcre2_test(void);
287
288 void pcre2_get_error_in_buffer(char *msg, size_t msg_len, int rc, int pos);
289
290 #endif //NETDATA_LOG2JOURNAL_H