@cryptotaxi247 / netdata-1 / commits / a3f342556

Journal multiple sources (#16252)

* support multiple sources * fix source matching * fix journal files source type * proper matching of journal files

Costa Tsaousis committed Oct 21, 2023 at 16:35 UTC a3f3425561012f358ad0f91caf53a3a65c646ecd
1 file changed +120 -102
collectors/systemd-journal.plugin/systemd-journal.c
+120 -102
@@ -264,13 +264,14 @@ typedef enum {
264 } ND_SD_JOURNAL_STATUS;
265
266 typedef enum {
267 - SDJF_ALL = 0,
268 - SDJF_LOCAL = (1 << 0),
269 - SDJF_REMOTE = (1 << 1),
270 - SDJF_SYSTEM = (1 << 2),
271 - SDJF_USER = (1 << 3),
272 - SDJF_NAMESPACE = (1 << 4),
273 - SDJF_OTHER = (1 << 5),
267 + SDJF_NONE = 0,
268 + SDJF_ALL = (1 << 0),
269 + SDJF_LOCAL_ALL = (1 << 1),
270 + SDJF_REMOTE_ALL = (1 << 2),
271 + SDJF_LOCAL_SYSTEM = (1 << 3),
272 + SDJF_LOCAL_USER = (1 << 4),
273 + SDJF_LOCAL_NAMESPACE = (1 << 5),
274 + SDJF_LOCAL_OTHER = (1 << 6),
275 } SD_JOURNAL_FILE_SOURCE_TYPE;
276
277 typedef struct function_query_status {
@@ -281,7 +282,7 @@ typedef struct function_query_status {
282
283 // request
284 SD_JOURNAL_FILE_SOURCE_TYPE source_type;
284 - STRING *source;
285 + SIMPLE_PATTERN *sources;
286 usec_t after_ut;
287 usec_t before_ut;
288
@@ -856,68 +857,66 @@ static void files_registry_insert_cb(const DICTIONARY_ITEM *item, void *value, v
857 struct journal_file *jf = value;
858 jf->filename = dictionary_acquired_item_name(item);
859 jf->filename_len = strlen(jf->filename);
860 + jf->source_type = SDJF_ALL;
861
862 // based on the filename
863 // decide the source to show to the user
864 const char *s = strrchr(jf->filename, '/');
865 if(s) {
864 - if(strstr(jf->filename, "/remote/"))
865 - jf->source_type = SDJF_REMOTE;
866 + if(strstr(jf->filename, "/remote/")) {
867 + jf->source_type |= SDJF_REMOTE_ALL;
868 +
869 + if(strncmp(s, "/remote-", 8) == 0) {
870 + s = &s[8]; // skip "/remote-"
871 +
872 + char *e = strchr(s, '@');
873 + if(!e)
874 + e = strstr(s, ".journal");
875 +
876 + if(e) {
877 + const char *d = s;
878 + for(; d < e && (isdigit(*d) || *d == '.' || *d == ':') ; d++) ;
879 + if(d == e) {
880 + // a valid IP address
881 + char ip[e - s + 1];
882 + memcpy(ip, s, e - s);
883 + ip[e - s] = '\0';
884 + char buf[SYSTEMD_JOURNAL_MAX_SOURCE_LEN];
885 + if(ip_to_hostname(ip, buf, sizeof(buf)))
886 + jf->source = string_strdupz_source(buf, &buf[strlen(buf)], SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-");
887 + else {
888 + internal_error(true, "Cannot find the hostname for IP '%s'", ip);
889 + jf->source = string_strdupz_source(s, e, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-");
890 + }
891 + }
892 + else
893 + jf->source = string_strdupz_source(s, e, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-");
894 + }
895 + }
896 + }
897 else {
898 + jf->source_type |= SDJF_LOCAL_ALL;
899 +
900 const char *t = s - 1;
901 while(t >= jf->filename && *t != '.' && *t != '/')
902 t--;
903
904 if(t >= jf->filename && *t == '.') {
872 - jf->source_type = SDJF_NAMESPACE;
905 + jf->source_type |= SDJF_LOCAL_NAMESPACE;
906 jf->source = string_strdupz_source(t + 1, s, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "namespace-");
907 }
875 - else
876 - jf->source_type = SDJF_LOCAL;
877 - }
908 + else if(strncmp(s, "/system", 7) == 0)
909 + jf->source_type |= SDJF_LOCAL_SYSTEM;
910 +
911 + else if(strncmp(s, "/user", 5) == 0)
912 + jf->source_type |= SDJF_LOCAL_USER;
913
879 - if(strncmp(s, "/system", 7) == 0)
880 - jf->source_type |= SDJF_SYSTEM;
881 -
882 - else if(strncmp(s, "/user", 5) == 0)
883 - jf->source_type |= SDJF_USER;
884 -
885 - else if(strncmp(s, "/remote-", 8) == 0) {
886 - jf->source_type |= SDJF_REMOTE;
887 -
888 - s = &s[8]; // skip "/remote-"
889 -
890 - char *e = strchr(s, '@');
891 - if(!e)
892 - e = strstr(s, ".journal");
893 -
894 - if(e) {
895 - const char *d = s;
896 - for(; d < e && (isdigit(*d) || *d == '.' || *d == ':') ; d++) ;
897 - if(d == e) {
898 - // a valid IP address
899 - char ip[e - s + 1];
900 - memcpy(ip, s, e - s);
901 - ip[e - s] = '\0';
902 - char buf[SYSTEMD_JOURNAL_MAX_SOURCE_LEN];
903 - if(ip_to_hostname(ip, buf, sizeof(buf)))
904 - jf->source = string_strdupz_source(buf, &buf[strlen(buf)], SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-");
905 - else {
906 - internal_error(true, "Cannot find the hostname for IP '%s'", ip);
907 - jf->source = string_strdupz_source(s, e, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-");
908 - }
909 - }
910 - else
911 - jf->source = string_strdupz_source(s, e, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-");
912 - }
914 else
914 - jf->source_type |= SDJF_OTHER;
915 + jf->source_type |= SDJF_LOCAL_OTHER;
916 }
916 - else
917 - jf->source_type |= SDJF_OTHER;
917 }
918 else
920 - jf->source_type = SDJF_LOCAL | SDJF_OTHER;
919 + jf->source_type |= SDJF_LOCAL_ALL | SDJF_LOCAL_OTHER;
920
921 journal_file_update_msg_ut(jf->filename, jf);
922
@@ -1068,17 +1067,17 @@ static void available_journal_file_sources_to_json_array(BUFFER *wb) {
1067
1068 dictionary_set(dict, SDJF_SOURCE_ALL_NAME, &t, sizeof(t));
1069
1071 - if((jf->source_type & (SDJF_LOCAL)) == (SDJF_LOCAL))
1070 + if(jf->source_type & SDJF_LOCAL_ALL)
1071 dictionary_set(dict, SDJF_SOURCE_LOCAL_NAME, &t, sizeof(t));
1073 - if((jf->source_type & (SDJF_LOCAL | SDJF_SYSTEM)) == (SDJF_LOCAL | SDJF_SYSTEM))
1072 + if(jf->source_type & SDJF_LOCAL_SYSTEM)
1073 dictionary_set(dict, SDJF_SOURCE_LOCAL_SYSTEM_NAME, &t, sizeof(t));
1075 - if((jf->source_type & (SDJF_LOCAL | SDJF_USER)) == (SDJF_LOCAL | SDJF_USER))
1074 + if(jf->source_type & SDJF_LOCAL_USER)
1075 dictionary_set(dict, SDJF_SOURCE_LOCAL_USERS_NAME, &t, sizeof(t));
1077 - if((jf->source_type & (SDJF_LOCAL | SDJF_OTHER)) == (SDJF_LOCAL | SDJF_OTHER))
1076 + if(jf->source_type & SDJF_LOCAL_OTHER)
1077 dictionary_set(dict, SDJF_SOURCE_LOCAL_OTHER_NAME, &t, sizeof(t));
1079 - if((jf->source_type & (SDJF_NAMESPACE)) == (SDJF_NAMESPACE))
1078 + if(jf->source_type & SDJF_LOCAL_NAMESPACE)
1079 dictionary_set(dict, SDJF_SOURCE_NAMESPACES_NAME, &t, sizeof(t));
1081 - if((jf->source_type & (SDJF_REMOTE)) == (SDJF_REMOTE))
1080 + if(jf->source_type & SDJF_REMOTE_ALL)
1081 dictionary_set(dict, SDJF_SOURCE_REMOTES_NAME, &t, sizeof(t));
1082 if(jf->source)
1083 dictionary_set(dict, string2str(jf->source), &t, sizeof(t));
@@ -1173,8 +1172,8 @@ static void journal_files_registry_update() {
1172
1173 static bool jf_is_mine(struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {
1174
1176 - if((fqs->source_type == SDJF_ALL || (jf->source_type & fqs->source_type) == fqs->source_type) &&
1177 - (!fqs->source || fqs->source == jf->source)) {
1175 + if((fqs->source_type == SDJF_NONE || (jf->source_type & fqs->source_type)) &&
1176 + (!fqs->sources || simple_pattern_matches(fqs->sources, string2str(jf->source)))) {
1177
1178 usec_t anchor_delta = JOURNAL_VS_REALTIME_DELTA_MAX_UT;
1179 usec_t first_ut = jf->msg_first_ut;
@@ -2305,7 +2304,7 @@ static void function_systemd_journal(const char *transaction, char *function, in
2304 FACETS_ANCHOR_DIRECTION direction = JOURNAL_DEFAULT_DIRECTION;
2305 const char *query = NULL;
2306 const char *chart = NULL;
2308 - const char *source = NULL;
2307 + SIMPLE_PATTERN *sources = NULL;
2308 const char *progress_id = NULL;
2309 SD_JOURNAL_FILE_SOURCE_TYPE source_type = SDJF_ALL;
2310 size_t filters = 0;
@@ -2367,40 +2366,65 @@ static void function_systemd_journal(const char *transaction, char *function, in
2366 progress_id = id;
2367 }
2368 else if(strncmp(keyword, JOURNAL_PARAMETER_SOURCE ":", sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1) == 0) {
2370 - source = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1];
2369 + const char *value = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1];
2370
2372 - if(strcmp(source, SDJF_SOURCE_ALL_NAME) == 0) {
2373 - source_type = SDJF_ALL;
2374 - source = NULL;
2375 - }
2376 - else if(strcmp(source, SDJF_SOURCE_LOCAL_NAME) == 0) {
2377 - source_type = SDJF_LOCAL;
2378 - source = NULL;
2379 - }
2380 - else if(strcmp(source, SDJF_SOURCE_REMOTES_NAME) == 0) {
2381 - source_type = SDJF_REMOTE;
2382 - source = NULL;
2383 - }
2384 - else if(strcmp(source, SDJF_SOURCE_NAMESPACES_NAME) == 0) {
2385 - source_type = SDJF_NAMESPACE;
2386 - source = NULL;
2387 - }
2388 - else if(strcmp(source, SDJF_SOURCE_LOCAL_SYSTEM_NAME) == 0) {
2389 - source_type = SDJF_LOCAL | SDJF_SYSTEM;
2390 - source = NULL;
2391 - }
2392 - else if(strcmp(source, SDJF_SOURCE_LOCAL_USERS_NAME) == 0) {
2393 - source_type = SDJF_LOCAL | SDJF_USER;
2394 - source = NULL;
2395 - }
2396 - else if(strcmp(source, SDJF_SOURCE_LOCAL_OTHER_NAME) == 0) {
2397 - source_type = SDJF_LOCAL | SDJF_OTHER;
2398 - source = NULL;
2399 - }
2400 - else {
2401 - source_type = SDJF_ALL;
2402 - // else, match the source, whatever it is
2371 + buffer_json_member_add_array(wb, JOURNAL_PARAMETER_SOURCE);
2372 +
2373 + BUFFER *sources_list = buffer_create(0, NULL);
2374 +
2375 + source_type = SDJF_NONE;
2376 + while(value) {
2377 + char *sep = strchr(value, ',');
2378 + if(sep)
2379 + *sep++ = '\0';
2380 +
2381 + buffer_json_add_array_item_string(wb, value);
2382 +
2383 + if(strcmp(value, SDJF_SOURCE_ALL_NAME) == 0) {
2384 + source_type |= SDJF_ALL;
2385 + value = NULL;
2386 + }
2387 + else if(strcmp(value, SDJF_SOURCE_LOCAL_NAME) == 0) {
2388 + source_type |= SDJF_LOCAL_ALL;
2389 + value = NULL;
2390 + }
2391 + else if(strcmp(value, SDJF_SOURCE_REMOTES_NAME) == 0) {
2392 + source_type |= SDJF_REMOTE_ALL;
2393 + value = NULL;
2394 + }
2395 + else if(strcmp(value, SDJF_SOURCE_NAMESPACES_NAME) == 0) {
2396 + source_type |= SDJF_LOCAL_NAMESPACE;
2397 + value = NULL;
2398 + }
2399 + else if(strcmp(value, SDJF_SOURCE_LOCAL_SYSTEM_NAME) == 0) {
2400 + source_type |= SDJF_LOCAL_SYSTEM;
2401 + value = NULL;
2402 + }
2403 + else if(strcmp(value, SDJF_SOURCE_LOCAL_USERS_NAME) == 0) {
2404 + source_type |= SDJF_LOCAL_USER;
2405 + value = NULL;
2406 + }
2407 + else if(strcmp(value, SDJF_SOURCE_LOCAL_OTHER_NAME) == 0) {
2408 + source_type |= SDJF_LOCAL_OTHER;
2409 + value = NULL;
2410 + }
2411 + else {
2412 + // else, match the source, whatever it is
2413 + if(buffer_strlen(sources_list))
2414 + buffer_strcat(sources_list, ",");
2415 +
2416 + buffer_strcat(sources_list, value);
2417 + }
2418 +
2419 + value = sep;
2420 }
2421 +
2422 + if(buffer_strlen(sources_list))
2423 + sources = simple_pattern_create(buffer_tostring(sources_list), ",", SIMPLE_PATTERN_EXACT, false);
2424 +
2425 + buffer_free(sources_list);
2426 +
2427 + buffer_json_array_close(wb); // source
2428 }
2429 else if(strncmp(keyword, JOURNAL_PARAMETER_AFTER ":", sizeof(JOURNAL_PARAMETER_AFTER ":") - 1) == 0) {
2430 after_s = str2l(&keyword[sizeof(JOURNAL_PARAMETER_AFTER ":") - 1]);
@@ -2517,7 +2541,7 @@ static void function_systemd_journal(const char *transaction, char *function, in
2541 fqs->data_only = data_only;
2542 fqs->delta = (fqs->data_only) ? delta : false;
2543 fqs->tail = (fqs->data_only && fqs->if_modified_since) ? tail : false;
2520 - fqs->source = string_strdupz(source);
2544 + fqs->sources = sources;
2545 fqs->source_type = source_type;
2546 fqs->entries = last;
2547 fqs->last_modified = 0;
@@ -2590,7 +2614,6 @@ static void function_systemd_journal(const char *transaction, char *function, in
2614 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DELTA, fqs->delta);
2615 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_TAIL, fqs->tail);
2616 buffer_json_member_add_string(wb, JOURNAL_PARAMETER_ID, progress_id);
2593 - buffer_json_member_add_string(wb, JOURNAL_PARAMETER_SOURCE, string2str(fqs->source));
2617 buffer_json_member_add_uint64(wb, "source_type", fqs->source_type);
2618 buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_AFTER, fqs->after_ut / USEC_PER_SEC);
2619 buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_BEFORE, fqs->before_ut / USEC_PER_SEC);
@@ -2618,7 +2641,7 @@ static void function_systemd_journal(const char *transaction, char *function, in
2641 buffer_json_member_add_string(wb, "id", "source");
2642 buffer_json_member_add_string(wb, "name", "source");
2643 buffer_json_member_add_string(wb, "help", "Select the SystemD Journal source to query");
2621 - buffer_json_member_add_string(wb, "type", "select");
2644 + buffer_json_member_add_string(wb, "type", "multiselect");
2645 buffer_json_member_add_array(wb, "options");
2646 {
2647 available_journal_file_sources_to_json_array(wb);
@@ -2646,12 +2669,6 @@ static void function_systemd_journal(const char *transaction, char *function, in
2669
2670 response = netdata_systemd_journal_query(wb, facets, fqs);
2671
2649 - // ------------------------------------------------------------------------
2650 - // cleanup query params
2651 -
2652 - string_freez(fqs->source);
2653 - fqs->source = NULL;
2654 -
2672 // ------------------------------------------------------------------------
2673 // handle error response
2674
@@ -2668,6 +2685,7 @@ output:
2685 netdata_mutex_unlock(&stdout_mutex);
2686
2687 cleanup:
2688 + simple_pattern_free(sources);
2689 facets_destroy(facets);
2690 buffer_free(wb);
2691