| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "collectors/systemd-journal.plugin/provider/netdata_provider.h" |
| 4 | #include "systemd-internals.h" |
| 5 | |
| 6 | #define ND_SD_JOURNAL_MAX_SOURCE_LEN 64 |
| 7 | #define VAR_LOG_JOURNAL_MAX_DEPTH 10 |
| 8 | |
| 9 | struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES] = {0}; |
| 10 | DICTIONARY *nd_journal_files_registry = NULL; |
| 11 | DICTIONARY *used_hashes_registry = NULL; |
| 12 | |
| 13 | static usec_t systemd_journal_session = 0; |
| 14 | static bool nd_journal_scan_progress_enabled = true; |
| 15 | |
| 16 | void nd_journal_set_scan_progress_enabled(bool enabled) |
| 17 | { |
| 18 | nd_journal_scan_progress_enabled = enabled; |
| 19 | } |
| 20 | |
| 21 | static void nd_journal_scan_progress(void) |
| 22 | { |
| 23 | if (nd_journal_scan_progress_enabled) |
| 24 | send_newline_and_flush(&stdout_mutex); |
| 25 | } |
| 26 | |
| 27 | void nd_journal_use_single_directory(const char *path) |
| 28 | { |
| 29 | string_freez(journal_directories[0].path); |
| 30 | journal_directories[0].path = string_strdupz(path); |
| 31 | |
| 32 | for (size_t i = 1; i < MAX_JOURNAL_DIRECTORIES; i++) { |
| 33 | string_freez(journal_directories[i].path); |
| 34 | journal_directories[i].path = NULL; |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | void buffer_json_journal_versions(BUFFER *wb) |
| 39 | { |
| 40 | buffer_json_member_add_object(wb, "versions"); |
| 41 | { |
| 42 | buffer_json_member_add_uint64( |
| 43 | wb, "sources", systemd_journal_session + dictionary_version(nd_journal_files_registry)); |
| 44 | } |
| 45 | buffer_json_object_close(wb); |
| 46 | } |
| 47 | |
| 48 | static bool journal_sd_id128_parse(const char *in, NsdId128 *ret) |
| 49 | { |
| 50 | while (isspace(*in)) |
| 51 | in++; |
| 52 | |
| 53 | char uuid[33]; |
| 54 | strncpyz(uuid, in, 32); |
| 55 | uuid[32] = '\0'; |
| 56 | |
| 57 | if (strlen(uuid) == 32) { |
| 58 | NsdId128 read; |
| 59 | if (nsd_id128_from_string(uuid, &read) == 0) { |
| 60 | *ret = read; |
| 61 | return true; |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | return false; |
| 66 | } |
| 67 | |
| 68 | usec_t |
| 69 | nd_journal_file_update_annotation_boot_id(NsdJournal *j, struct nd_journal_file *njf __maybe_unused, const char *boot_id) |
| 70 | { |
| 71 | usec_t ut = UINT64_MAX; |
| 72 | int r; |
| 73 | |
| 74 | char m[100]; |
| 75 | size_t len = snprintfz(m, sizeof(m), "_BOOT_ID=%s", boot_id); |
| 76 | |
| 77 | nsd_journal_flush_matches(j); |
| 78 | |
| 79 | r = nsd_journal_add_match(j, m, len); |
| 80 | if (r < 0) { |
| 81 | errno = -r; |
| 82 | internal_error( |
| 83 | true, |
| 84 | "JOURNAL: while looking for the first timestamp of boot_id '%s', " |
| 85 | "sd_journal_add_match('%s') on file '%s' returned %d", |
| 86 | boot_id, |
| 87 | m, |
| 88 | njf->filename, |
| 89 | r); |
| 90 | return UINT64_MAX; |
| 91 | } |
| 92 | |
| 93 | r = nsd_journal_seek_head(j); |
| 94 | if (r < 0) { |
| 95 | errno = -r; |
| 96 | internal_error( |
| 97 | true, |
| 98 | "JOURNAL: while looking for the first timestamp of boot_id '%s', " |
| 99 | "sd_journal_seek_head() on file '%s' returned %d", |
| 100 | boot_id, |
| 101 | njf->filename, |
| 102 | r); |
| 103 | return UINT64_MAX; |
| 104 | } |
| 105 | |
| 106 | r = nsd_journal_next(j); |
| 107 | if (r < 0) { |
| 108 | errno = -r; |
| 109 | internal_error( |
| 110 | true, |
| 111 | "JOURNAL: while looking for the first timestamp of boot_id '%s', " |
| 112 | "sd_journal_next() on file '%s' returned %d", |
| 113 | boot_id, |
| 114 | njf->filename, |
| 115 | r); |
| 116 | return UINT64_MAX; |
| 117 | } else if (r == 0) { |
| 118 | internal_error( |
| 119 | true, |
| 120 | "GVD: while looking for the first timestamp of boot_id '%s', " |
| 121 | "sd_journal_next() on file '%s' returned %d", |
| 122 | boot_id, |
| 123 | njf->filename, |
| 124 | r); |
| 125 | return UINT64_MAX; |
| 126 | } |
| 127 | |
| 128 | r = nsd_journal_get_realtime_usec(j, &ut); |
| 129 | if (r < 0 || !ut || ut == UINT64_MAX) { |
| 130 | errno = -r; |
| 131 | internal_error( |
| 132 | r != -EADDRNOTAVAIL, |
| 133 | "JOURNAL: while looking for the first timestamp of boot_id '%s', " |
| 134 | "sd_journal_get_realtime_usec() on file '%s' returned %d", |
| 135 | boot_id, |
| 136 | njf->filename, |
| 137 | r); |
| 138 | return UINT64_MAX; |
| 139 | } |
| 140 | |
| 141 | if (ut && ut != UINT64_MAX) { |
| 142 | dictionary_set(boot_ids_to_first_ut, boot_id, &ut, sizeof(ut)); |
| 143 | return ut; |
| 144 | } |
| 145 | |
| 146 | return UINT64_MAX; |
| 147 | } |
| 148 | |
| 149 | static void |
| 150 | nd_journal_file_get_boot_id_annotations(NsdJournal *j __maybe_unused, struct nd_journal_file *njf __maybe_unused) |
| 151 | { |
| 152 | #ifdef HAVE_SD_JOURNAL_RESTART_FIELDS |
| 153 | nsd_journal_flush_matches(j); |
| 154 | |
| 155 | int r = nsd_journal_query_unique(j, "_BOOT_ID"); |
| 156 | if (r < 0) { |
| 157 | errno = -r; |
| 158 | internal_error( |
| 159 | true, |
| 160 | "JOURNAL: while querying for the unique _BOOT_ID values, " |
| 161 | "sd_journal_query_unique() on file '%s' returned %d", |
| 162 | njf->filename, |
| 163 | r); |
| 164 | errno = -r; |
| 165 | return; |
| 166 | } |
| 167 | |
| 168 | const void *data = NULL; |
| 169 | size_t data_length; |
| 170 | |
| 171 | DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); |
| 172 | |
| 173 | nsd_journal_restart_unique(j); |
| 174 | while ((r = nsd_journal_enumerate_available_unique(j, &data, &data_length)) > 0) { |
| 175 | const char *key, *value; |
| 176 | size_t key_length, value_length; |
| 177 | |
| 178 | if (!parse_journal_field(data, data_length, &key, &key_length, &value, &value_length)) |
| 179 | continue; |
| 180 | |
| 181 | if (value_length != 32) |
| 182 | continue; |
| 183 | |
| 184 | char buf[33]; |
| 185 | memcpy(buf, value, 32); |
| 186 | buf[32] = '\0'; |
| 187 | |
| 188 | dictionary_set(dict, buf, NULL, 0); |
| 189 | } |
| 190 | |
| 191 | if (r < 0) { |
| 192 | errno = -r; |
| 193 | internal_error( |
| 194 | true, |
| 195 | "JOURNAL: while enumerating the unique _BOOT_ID values, " |
| 196 | "nsd_journal_enumerate_available_unique() on file '%s' returned %d", |
| 197 | njf->filename, |
| 198 | r); |
| 199 | } |
| 200 | |
| 201 | void *nothing; |
| 202 | dfe_start_read(dict, nothing) |
| 203 | { |
| 204 | nd_journal_file_update_annotation_boot_id(j, njf, nothing_dfe.name); |
| 205 | } |
| 206 | dfe_done(nothing); |
| 207 | |
| 208 | dictionary_destroy(dict); |
| 209 | #endif |
| 210 | } |
| 211 | |
| 212 | void nd_journal_file_update_header(const char *filename, struct nd_journal_file *njf) |
| 213 | { |
| 214 | if (njf->last_scan_header_vs_last_modified_ut == njf->file_last_modified_ut) |
| 215 | return; |
| 216 | |
| 217 | fstat_cache_enable_on_thread(); |
| 218 | |
| 219 | const char *files[2] = { |
| 220 | [0] = filename, |
| 221 | [1] = NULL, |
| 222 | }; |
| 223 | |
| 224 | NsdJournal *j = NULL; |
| 225 | if (nsd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) { |
| 226 | netdata_log_error("JOURNAL: cannot open file '%s' to update msg_ut", filename); |
| 227 | fstat_cache_disable_on_thread(); |
| 228 | |
| 229 | if (!njf->logged_failure) { |
| 230 | netdata_log_error( |
| 231 | "cannot open journal file '%s', using file timestamps to understand time-frame.", filename); |
| 232 | njf->logged_failure = true; |
| 233 | } |
| 234 | |
| 235 | njf->msg_first_ut = 0; |
| 236 | njf->msg_last_ut = njf->file_last_modified_ut; |
| 237 | njf->last_scan_header_vs_last_modified_ut = njf->file_last_modified_ut; |
| 238 | return; |
| 239 | } |
| 240 | |
| 241 | usec_t first_ut = 0, last_ut = 0; |
| 242 | uint64_t first_seqnum = 0, last_seqnum = 0; |
| 243 | NsdId128 first_writer_id = NSD_ID128_NULL, last_writer_id = NSD_ID128_NULL; |
| 244 | |
| 245 | if (nsd_journal_seek_head(j) < 0 || nsd_journal_next(j) < 0 || nsd_journal_get_realtime_usec(j, &first_ut) < 0 || |
| 246 | !first_ut) { |
| 247 | internal_error(true, "cannot find the timestamp of the first message in '%s'", filename); |
| 248 | first_ut = 0; |
| 249 | } |
| 250 | #ifdef HAVE_SD_JOURNAL_GET_SEQNUM |
| 251 | else { |
| 252 | if (nsd_journal_get_seqnum(j, &first_seqnum, &first_writer_id) < 0 || !first_seqnum) { |
| 253 | internal_error(true, "cannot find the first seqnums of the first message in '%s'", filename); |
| 254 | first_seqnum = 0; |
| 255 | memset(&first_writer_id, 0, sizeof(first_writer_id)); |
| 256 | } |
| 257 | } |
| 258 | #endif |
| 259 | |
| 260 | if (nsd_journal_seek_tail(j) < 0 || nsd_journal_previous(j) < 0 || nsd_journal_get_realtime_usec(j, &last_ut) < 0 || |
| 261 | !last_ut) { |
| 262 | internal_error(true, "cannot find the timestamp of the last message in '%s'", filename); |
| 263 | last_ut = njf->file_last_modified_ut; |
| 264 | } |
| 265 | #ifdef HAVE_SD_JOURNAL_GET_SEQNUM |
| 266 | else { |
| 267 | if (nsd_journal_get_seqnum(j, &last_seqnum, &last_writer_id) < 0 || !last_seqnum) { |
| 268 | internal_error(true, "cannot find the last seqnums of the first message in '%s'", filename); |
| 269 | last_seqnum = 0; |
| 270 | memset(&last_writer_id, 0, sizeof(last_writer_id)); |
| 271 | } |
| 272 | } |
| 273 | #endif |
| 274 | |
| 275 | if (first_ut > last_ut) { |
| 276 | internal_error(true, "timestamps are flipped in file '%s'", filename); |
| 277 | usec_t t = first_ut; |
| 278 | first_ut = last_ut; |
| 279 | last_ut = t; |
| 280 | } |
| 281 | |
| 282 | if (!first_seqnum || !first_ut) { |
| 283 | // extract these from the filename - if possible |
| 284 | |
| 285 | const char *at = strchr(filename, '@'); |
| 286 | if (at) { |
| 287 | const char *dash_seqnum = strchr(at + 1, '-'); |
| 288 | if (dash_seqnum) { |
| 289 | const char *dash_first_msg_ut = strchr(dash_seqnum + 1, '-'); |
| 290 | if (dash_first_msg_ut) { |
| 291 | const char *dot_journal = NULL; |
| 292 | if (is_journal_file(filename, -1, &dot_journal) && dot_journal && dot_journal > dash_first_msg_ut) { |
| 293 | if (dash_seqnum - at - 1 == 32 && dash_first_msg_ut - dash_seqnum - 1 == 16 && |
| 294 | dot_journal - dash_first_msg_ut - 1 == 16) { |
| 295 | NsdId128 writer; |
| 296 | if (journal_sd_id128_parse(at + 1, &writer)) { |
| 297 | char *endptr = NULL; |
| 298 | uint64_t seqnum = strtoul(dash_seqnum + 1, &endptr, 16); |
| 299 | if (endptr == dash_first_msg_ut) { |
| 300 | uint64_t ts = strtoul(dash_first_msg_ut + 1, &endptr, 16); |
| 301 | if (endptr == dot_journal) { |
| 302 | first_seqnum = seqnum; |
| 303 | first_writer_id = writer; |
| 304 | first_ut = ts; |
| 305 | } |
| 306 | } |
| 307 | } |
| 308 | } |
| 309 | } |
| 310 | } |
| 311 | } |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | njf->first_seqnum = first_seqnum; |
| 316 | njf->last_seqnum = last_seqnum; |
| 317 | |
| 318 | njf->first_writer_id = first_writer_id; |
| 319 | njf->last_writer_id = last_writer_id; |
| 320 | |
| 321 | njf->msg_first_ut = first_ut; |
| 322 | njf->msg_last_ut = last_ut; |
| 323 | |
| 324 | if (!njf->msg_last_ut) |
| 325 | njf->msg_last_ut = njf->file_last_modified_ut; |
| 326 | |
| 327 | if (last_seqnum > first_seqnum) { |
| 328 | if (!nsd_id128_equal(first_writer_id, last_writer_id)) { |
| 329 | njf->messages_in_file = 0; |
| 330 | nd_log( |
| 331 | NDLS_COLLECTORS, |
| 332 | NDLP_NOTICE, |
| 333 | "The writers of the first and the last message in file '%s' differ.", |
| 334 | filename); |
| 335 | } else |
| 336 | njf->messages_in_file = last_seqnum - first_seqnum + 1; |
| 337 | } else |
| 338 | njf->messages_in_file = 0; |
| 339 | |
| 340 | nd_journal_file_get_boot_id_annotations(j, njf); |
| 341 | nsd_journal_close(j); |
| 342 | fstat_cache_disable_on_thread(); |
| 343 | |
| 344 | njf->last_scan_header_vs_last_modified_ut = njf->file_last_modified_ut; |
| 345 | } |
| 346 | |
| 347 | static STRING *string_strdupz_source(const char *s, const char *e, size_t max_len, const char *prefix) |
| 348 | { |
| 349 | size_t buf_size = max_len; |
| 350 | size_t remaining = max_len; |
| 351 | char *buf = mallocz(buf_size); |
| 352 | size_t len; |
| 353 | char *dst = buf; |
| 354 | |
| 355 | if (prefix) { |
| 356 | len = strlen(prefix); |
| 357 | if (len >= remaining) |
| 358 | len = remaining - 1; |
| 359 | memcpy(buf, prefix, len); |
| 360 | dst = &buf[len]; |
| 361 | remaining -= len; |
| 362 | } |
| 363 | |
| 364 | len = e - s; |
| 365 | if (len >= remaining) |
| 366 | len = remaining - 1; |
| 367 | memcpy(dst, s, len); |
| 368 | dst[len] = '\0'; |
| 369 | buf[buf_size - 1] = '\0'; |
| 370 | |
| 371 | for (size_t i = 0; buf[i]; i++) |
| 372 | if (!is_netdata_api_valid_character(buf[i])) |
| 373 | buf[i] = '_'; |
| 374 | |
| 375 | STRING *copy = string_strdupz(buf); |
| 376 | freez(buf); |
| 377 | return copy; |
| 378 | } |
| 379 | |
| 380 | static void files_registry_insert_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused) |
| 381 | { |
| 382 | struct nd_journal_file *njf = value; |
| 383 | njf->filename = dictionary_acquired_item_name(item); |
| 384 | njf->filename_len = strlen(njf->filename); |
| 385 | njf->source_type = ND_SD_JF_ALL; |
| 386 | |
| 387 | // based on the filename |
| 388 | // decide the source to show to the user |
| 389 | const char *s = strrchr(njf->filename, '/'); |
| 390 | if (s) { |
| 391 | if (strstr(njf->filename, "/remote/")) { |
| 392 | njf->source_type |= ND_SD_JF_REMOTE_ALL; |
| 393 | |
| 394 | if (strncmp(s, "/remote-", 8) == 0) { |
| 395 | s = &s[8]; // skip "/remote-" |
| 396 | |
| 397 | char *e = strchr(s, '@'); |
| 398 | if (!e) |
| 399 | is_journal_file(s, -1, (const char **)&e); |
| 400 | |
| 401 | if (e) { |
| 402 | const char *d = s; |
| 403 | for (; d < e && (isdigit(*d) || *d == '.' || *d == ':'); d++) |
| 404 | ; |
| 405 | if (d == e) { |
| 406 | // a valid IP address |
| 407 | size_t ip_len = (size_t)(e - s); |
| 408 | char *ip = mallocz(ip_len + 1); |
| 409 | memcpy(ip, s, ip_len); |
| 410 | ip[ip_len] = '\0'; |
| 411 | char buf[ND_SD_JOURNAL_MAX_SOURCE_LEN]; |
| 412 | if (ip_to_hostname(ip, buf, sizeof(buf))) |
| 413 | njf->source = |
| 414 | string_strdupz_source(buf, &buf[strlen(buf)], ND_SD_JOURNAL_MAX_SOURCE_LEN, "remote-"); |
| 415 | else { |
| 416 | internal_error(true, "Cannot find the hostname for IP '%s'", ip); |
| 417 | njf->source = string_strdupz_source(s, e, ND_SD_JOURNAL_MAX_SOURCE_LEN, "remote-"); |
| 418 | } |
| 419 | freez(ip); |
| 420 | } else |
| 421 | njf->source = string_strdupz_source(s, e, ND_SD_JOURNAL_MAX_SOURCE_LEN, "remote-"); |
| 422 | } |
| 423 | } |
| 424 | } else { |
| 425 | njf->source_type |= ND_SD_JF_LOCAL_ALL; |
| 426 | |
| 427 | const char *t = s - 1; |
| 428 | while (t >= njf->filename && *t != '.' && *t != '/') |
| 429 | t--; |
| 430 | |
| 431 | if (t >= njf->filename && *t == '.') { |
| 432 | njf->source_type |= ND_SD_JF_LOCAL_NAMESPACE; |
| 433 | njf->source = string_strdupz_source(t + 1, s, ND_SD_JOURNAL_MAX_SOURCE_LEN, "namespace-"); |
| 434 | } else if (strncmp(s, "/system", 7) == 0) |
| 435 | njf->source_type |= ND_SD_JF_LOCAL_SYSTEM; |
| 436 | |
| 437 | else if (strncmp(s, "/user", 5) == 0) |
| 438 | njf->source_type |= ND_SD_JF_LOCAL_USER; |
| 439 | |
| 440 | else |
| 441 | njf->source_type |= ND_SD_JF_LOCAL_OTHER; |
| 442 | } |
| 443 | } else |
| 444 | njf->source_type |= ND_SD_JF_LOCAL_ALL | ND_SD_JF_LOCAL_OTHER; |
| 445 | |
| 446 | njf->msg_last_ut = njf->file_last_modified_ut; |
| 447 | |
| 448 | nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "Journal file added to the journal files registry: '%s'", njf->filename); |
| 449 | } |
| 450 | |
| 451 | static bool files_registry_conflict_cb( |
| 452 | const DICTIONARY_ITEM *item __maybe_unused, |
| 453 | void *old_value, |
| 454 | void *new_value, |
| 455 | void *data __maybe_unused) |
| 456 | { |
| 457 | struct nd_journal_file *njf_old = old_value; |
| 458 | struct nd_journal_file *njf_new = new_value; |
| 459 | |
| 460 | if (njf_new->last_scan_monotonic_ut > njf_old->last_scan_monotonic_ut) |
| 461 | njf_old->last_scan_monotonic_ut = njf_new->last_scan_monotonic_ut; |
| 462 | |
| 463 | if (njf_new->file_last_modified_ut > njf_old->file_last_modified_ut) { |
| 464 | njf_old->file_last_modified_ut = njf_new->file_last_modified_ut; |
| 465 | njf_old->size = njf_new->size; |
| 466 | |
| 467 | njf_old->msg_last_ut = njf_old->file_last_modified_ut; |
| 468 | } |
| 469 | |
| 470 | return false; |
| 471 | } |
| 472 | |
| 473 | struct nd_journal_file_source { |
| 474 | usec_t first_ut; |
| 475 | usec_t last_ut; |
| 476 | size_t count; |
| 477 | uint64_t size; |
| 478 | }; |
| 479 | |
| 480 | #define print_duration(dst, dst_len, pos, remaining, duration, one, many, printed) \ |
| 481 | do { \ |
| 482 | if ((remaining) > (duration)) { \ |
| 483 | uint64_t _count = (remaining) / (duration); \ |
| 484 | uint64_t _rem = (remaining) - (_count * (duration)); \ |
| 485 | (pos) += snprintfz( \ |
| 486 | &(dst)[pos], \ |
| 487 | (dst_len) - (pos), \ |
| 488 | "%s%s%" PRIu64 " %s", \ |
| 489 | (printed) ? ", " : "", \ |
| 490 | _rem ? "" : "and ", \ |
| 491 | _count, \ |
| 492 | _count > 1 ? (many) : (one)); \ |
| 493 | (remaining) = _rem; \ |
| 494 | (printed) = true; \ |
| 495 | } \ |
| 496 | } while (0) |
| 497 | |
| 498 | static int nd_journal_file_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) |
| 499 | { |
| 500 | struct nd_journal_file_source *nd_jfs = entry; |
| 501 | BUFFER *wb = data; |
| 502 | |
| 503 | const char *name = dictionary_acquired_item_name(item); |
| 504 | |
| 505 | buffer_json_add_array_item_object(wb); |
| 506 | { |
| 507 | char size_for_humans[128]; |
| 508 | size_snprintf(size_for_humans, sizeof(size_for_humans), nd_jfs->size, "B", false); |
| 509 | |
| 510 | char duration_for_humans[128]; |
| 511 | duration_snprintf( |
| 512 | duration_for_humans, |
| 513 | sizeof(duration_for_humans), |
| 514 | (time_t)((nd_jfs->last_ut - nd_jfs->first_ut) / USEC_PER_SEC), |
| 515 | "s", |
| 516 | true); |
| 517 | |
| 518 | char last_ut[RFC3339_MAX_LENGTH]; |
| 519 | rfc3339_datetime_ut(last_ut, sizeof(last_ut), nd_jfs->last_ut, 0, true); |
| 520 | |
| 521 | char info[1024]; |
| 522 | snprintfz( |
| 523 | info, |
| 524 | sizeof(info), |
| 525 | "%zu files, total size %s, covering %s, last entry at %s", |
| 526 | nd_jfs->count, |
| 527 | size_for_humans, |
| 528 | duration_for_humans, |
| 529 | last_ut); |
| 530 | |
| 531 | buffer_json_member_add_string(wb, "id", name); |
| 532 | buffer_json_member_add_string(wb, "name", name); |
| 533 | buffer_json_member_add_string(wb, "pill", size_for_humans); |
| 534 | buffer_json_member_add_string(wb, "info", info); |
| 535 | } |
| 536 | buffer_json_object_close(wb); // options object |
| 537 | |
| 538 | return 1; |
| 539 | } |
| 540 | |
| 541 | static bool nd_journal_file_merge_sizes( |
| 542 | const DICTIONARY_ITEM *item __maybe_unused, |
| 543 | void *old_value, |
| 544 | void *new_value, |
| 545 | void *data __maybe_unused) |
| 546 | { |
| 547 | struct nd_journal_file_source *jfs = old_value, *njfs = new_value; |
| 548 | jfs->count += njfs->count; |
| 549 | jfs->size += njfs->size; |
| 550 | |
| 551 | if (njfs->first_ut && njfs->first_ut < jfs->first_ut) |
| 552 | jfs->first_ut = njfs->first_ut; |
| 553 | |
| 554 | if (njfs->last_ut && njfs->last_ut > jfs->last_ut) |
| 555 | jfs->last_ut = njfs->last_ut; |
| 556 | |
| 557 | return false; |
| 558 | } |
| 559 | |
| 560 | void available_journal_file_sources_to_json_array(BUFFER *wb) |
| 561 | { |
| 562 | DICTIONARY *dict = dictionary_create( |
| 563 | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); |
| 564 | dictionary_register_conflict_callback(dict, nd_journal_file_merge_sizes, NULL); |
| 565 | |
| 566 | struct nd_journal_file_source njfs_tmp = {0}; |
| 567 | |
| 568 | struct nd_journal_file *njf; |
| 569 | dfe_start_read(nd_journal_files_registry, njf) |
| 570 | { |
| 571 | njfs_tmp.first_ut = njf->msg_first_ut; |
| 572 | njfs_tmp.last_ut = njf->msg_last_ut; |
| 573 | njfs_tmp.count = 1; |
| 574 | njfs_tmp.size = njf->size; |
| 575 | |
| 576 | dictionary_set(dict, ND_SD_JF_SOURCE_ALL_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 577 | |
| 578 | if (njf->source_type & ND_SD_JF_LOCAL_ALL) |
| 579 | dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 580 | if (njf->source_type & ND_SD_JF_LOCAL_SYSTEM) |
| 581 | dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_SYSTEM_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 582 | if (njf->source_type & ND_SD_JF_LOCAL_USER) |
| 583 | dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_USERS_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 584 | if (njf->source_type & ND_SD_JF_LOCAL_OTHER) |
| 585 | dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_OTHER_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 586 | if (njf->source_type & ND_SD_JF_LOCAL_NAMESPACE) |
| 587 | dictionary_set(dict, ND_SD_JF_SOURCE_NAMESPACES_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 588 | if (njf->source_type & ND_SD_JF_REMOTE_ALL) |
| 589 | dictionary_set(dict, ND_SD_JF_SOURCE_REMOTES_NAME, &njfs_tmp, sizeof(njfs_tmp)); |
| 590 | if (njf->source) |
| 591 | dictionary_set(dict, string2str(njf->source), &njfs_tmp, sizeof(njfs_tmp)); |
| 592 | } |
| 593 | dfe_done(jf); |
| 594 | |
| 595 | dictionary_sorted_walkthrough_read(dict, nd_journal_file_to_json_array_cb, wb); |
| 596 | |
| 597 | dictionary_destroy(dict); |
| 598 | } |
| 599 | |
| 600 | static void files_registry_delete_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused) |
| 601 | { |
| 602 | struct nd_journal_file *njf = value; |
| 603 | const char *filename = dictionary_acquired_item_name(item); |
| 604 | (void)filename; |
| 605 | |
| 606 | internal_error(true, "removed journal file '%s'", filename); |
| 607 | string_freez(njf->source); |
| 608 | } |
| 609 | |
| 610 | #define EXT_DOT_JOURNAL ".journal" |
| 611 | #define EXT_DOT_JOURNAL_TILDA ".journal~" |
| 612 | |
| 613 | static struct { |
| 614 | const char *ext; |
| 615 | ssize_t len; |
| 616 | } valid_journal_extension[] = { |
| 617 | {.ext = EXT_DOT_JOURNAL, .len = sizeof(EXT_DOT_JOURNAL) - 1}, |
| 618 | {.ext = EXT_DOT_JOURNAL_TILDA, .len = sizeof(EXT_DOT_JOURNAL_TILDA) - 1}, |
| 619 | }; |
| 620 | |
| 621 | bool is_journal_file(const char *filename, ssize_t len, const char **start_of_extension) |
| 622 | { |
| 623 | if (len < 0) |
| 624 | len = (ssize_t)strlen(filename); |
| 625 | |
| 626 | for (size_t i = 0; i < _countof(valid_journal_extension); i++) { |
| 627 | const char *ext = valid_journal_extension[i].ext; |
| 628 | ssize_t elen = valid_journal_extension[i].len; |
| 629 | |
| 630 | if (len > elen && strcmp(filename + len - elen, ext) == 0) { |
| 631 | if (start_of_extension) |
| 632 | *start_of_extension = filename + len - elen; |
| 633 | return true; |
| 634 | } |
| 635 | } |
| 636 | |
| 637 | if (start_of_extension) |
| 638 | *start_of_extension = NULL; |
| 639 | |
| 640 | return false; |
| 641 | } |
| 642 | |
| 643 | void nd_journal_directory_scan_recursively(DICTIONARY *files, DICTIONARY *dirs, const char *dirname, int depth) |
| 644 | { |
| 645 | if (depth > VAR_LOG_JOURNAL_MAX_DEPTH) |
| 646 | return; |
| 647 | |
| 648 | DIR *dir; |
| 649 | struct dirent *entry; |
| 650 | char full_path[FILENAME_MAX]; |
| 651 | |
| 652 | // Open the directory. |
| 653 | if ((dir = opendir(dirname)) == NULL) { |
| 654 | if (errno != ENOENT && errno != ENOTDIR) |
| 655 | netdata_log_error("Cannot opendir() '%s'", dirname); |
| 656 | return; |
| 657 | } |
| 658 | |
| 659 | bool existing = false; |
| 660 | bool *found = dictionary_set(dirs, dirname, &existing, sizeof(existing)); |
| 661 | if (unlikely(!found)) { |
| 662 | netdata_log_error("Cannot track visited directory '%s' (dictionary_set failed); stopping recursion", dirname); |
| 663 | closedir(dir); |
| 664 | return; |
| 665 | } |
| 666 | if (*found) { |
| 667 | closedir(dir); |
| 668 | return; |
| 669 | } |
| 670 | *found = true; |
| 671 | |
| 672 | // Read each entry in the directory. |
| 673 | while ((entry = readdir(dir)) != NULL) { |
| 674 | if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) |
| 675 | continue; |
| 676 | |
| 677 | ssize_t len = snprintfz(full_path, sizeof(full_path), "%s/%s", dirname, entry->d_name); |
| 678 | |
| 679 | if (entry->d_type == DT_DIR) { |
| 680 | nd_journal_directory_scan_recursively(files, dirs, full_path, depth + 1); |
| 681 | } else if (entry->d_type == DT_REG && is_journal_file(full_path, len, NULL)) { |
| 682 | if (files) |
| 683 | dictionary_set(files, full_path, NULL, 0); |
| 684 | |
| 685 | nd_journal_scan_progress(); |
| 686 | } else if (entry->d_type == DT_LNK) { |
| 687 | struct stat info; |
| 688 | if (stat(full_path, &info) == -1) |
| 689 | continue; |
| 690 | |
| 691 | // Journal discovery intentionally follows symlinked journal directories. |
| 692 | if (S_ISDIR(info.st_mode)) { |
| 693 | char resolved_path[FILENAME_MAX + 1]; |
| 694 | if (realpath(full_path, resolved_path) != NULL) { |
| 695 | nd_journal_directory_scan_recursively(files, dirs, resolved_path, depth + 1); |
| 696 | } |
| 697 | } else if (S_ISREG(info.st_mode) && is_journal_file(full_path, len, NULL)) { |
| 698 | if (files) |
| 699 | dictionary_set(files, full_path, NULL, 0); |
| 700 | |
| 701 | nd_journal_scan_progress(); |
| 702 | } |
| 703 | } |
| 704 | } |
| 705 | |
| 706 | closedir(dir); |
| 707 | } |
| 708 | |
| 709 | static size_t nd_journal_files_scans = 0; |
| 710 | bool nd_journal_files_completed_once(void) |
| 711 | { |
| 712 | return nd_journal_files_scans > 0; |
| 713 | } |
| 714 | |
| 715 | int filenames_compar(const void *a, const void *b) |
| 716 | { |
| 717 | const char *p1 = *(const char **)a; |
| 718 | const char *p2 = *(const char **)b; |
| 719 | |
| 720 | const char *at1 = strchr(p1, '@'); |
| 721 | const char *at2 = strchr(p2, '@'); |
| 722 | |
| 723 | if (!at1 && at2) |
| 724 | return -1; |
| 725 | |
| 726 | if (at1 && !at2) |
| 727 | return 1; |
| 728 | |
| 729 | if (!at1 && !at2) |
| 730 | return strcmp(p1, p2); |
| 731 | |
| 732 | const char *dash1 = strrchr(at1, '-'); |
| 733 | const char *dash2 = strrchr(at2, '-'); |
| 734 | |
| 735 | if (!dash1 || !dash2) |
| 736 | return strcmp(p1, p2); |
| 737 | |
| 738 | uint64_t ts1 = strtoul(dash1 + 1, NULL, 16); |
| 739 | uint64_t ts2 = strtoul(dash2 + 1, NULL, 16); |
| 740 | |
| 741 | if (ts1 > ts2) |
| 742 | return -1; |
| 743 | |
| 744 | if (ts1 < ts2) |
| 745 | return 1; |
| 746 | |
| 747 | return -strcmp(p1, p2); |
| 748 | } |
| 749 | |
| 750 | void nd_journal_files_registry_update(void) |
| 751 | { |
| 752 | static SPINLOCK spinlock = SPINLOCK_INITIALIZER; |
| 753 | |
| 754 | if (spinlock_trylock(&spinlock)) { |
| 755 | usec_t scan_monotonic_ut = now_monotonic_usec(); |
| 756 | |
| 757 | DICTIONARY *files = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); |
| 758 | DICTIONARY *dirs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); |
| 759 | |
| 760 | for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) { |
| 761 | if (!journal_directories[i].path) |
| 762 | break; |
| 763 | nd_journal_directory_scan_recursively(files, dirs, string2str(journal_directories[i].path), 0); |
| 764 | } |
| 765 | |
| 766 | const char **array = mallocz(sizeof(const char *) * dictionary_entries(files)); |
| 767 | size_t used = 0; |
| 768 | |
| 769 | void *x; |
| 770 | dfe_start_read(files, x) |
| 771 | { |
| 772 | if (used >= dictionary_entries(files)) |
| 773 | continue; |
| 774 | array[used++] = x_dfe.name; |
| 775 | } |
| 776 | dfe_done(x); |
| 777 | |
| 778 | qsort(array, used, sizeof(const char *), filenames_compar); |
| 779 | |
| 780 | for (size_t i = 0; i < used; i++) { |
| 781 | const char *full_path = array[i]; |
| 782 | |
| 783 | struct stat info; |
| 784 | if (stat(full_path, &info) == -1) |
| 785 | continue; |
| 786 | |
| 787 | struct nd_journal_file njf_tmp = { |
| 788 | .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC + info.st_mtim.tv_nsec / NSEC_PER_USEC, |
| 789 | .last_scan_monotonic_ut = scan_monotonic_ut, |
| 790 | .size = info.st_size, |
| 791 | .max_journal_vs_realtime_delta_ut = JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT, |
| 792 | }; |
| 793 | struct nd_journal_file *njf = |
| 794 | dictionary_set(nd_journal_files_registry, full_path, &njf_tmp, sizeof(njf_tmp)); |
| 795 | nd_journal_file_update_header(njf->filename, njf); |
| 796 | } |
| 797 | freez(array); |
| 798 | dictionary_destroy(files); |
| 799 | dictionary_destroy(dirs); |
| 800 | |
| 801 | struct nd_journal_file *njf; |
| 802 | dfe_start_write(nd_journal_files_registry, njf) |
| 803 | { |
| 804 | if (njf->last_scan_monotonic_ut < scan_monotonic_ut) |
| 805 | dictionary_del(nd_journal_files_registry, njf_dfe.name); |
| 806 | } |
| 807 | dfe_done(njf); |
| 808 | dictionary_garbage_collect(nd_journal_files_registry); |
| 809 | |
| 810 | nd_journal_files_scans++; |
| 811 | spinlock_unlock(&spinlock); |
| 812 | |
| 813 | internal_error( |
| 814 | true, |
| 815 | "Journal library scan completed in %.3f ms", |
| 816 | (double)(now_monotonic_usec() - scan_monotonic_ut) / (double)USEC_PER_MS); |
| 817 | } |
| 818 | } |
| 819 | |
| 820 | // ---------------------------------------------------------------------------- |
| 821 | |
| 822 | int nd_journal_file_dict_items_backward_compar(const void *a, const void *b) |
| 823 | { |
| 824 | const DICTIONARY_ITEM **ad = (const DICTIONARY_ITEM **)a, **bd = (const DICTIONARY_ITEM **)b; |
| 825 | struct nd_journal_file *njf_lhs = dictionary_acquired_item_value(*ad); |
| 826 | struct nd_journal_file *njf_rhs = dictionary_acquired_item_value(*bd); |
| 827 | |
| 828 | // compare the last message timestamps |
| 829 | if (njf_lhs->msg_last_ut < njf_rhs->msg_last_ut) |
| 830 | return 1; |
| 831 | |
| 832 | if (njf_lhs->msg_last_ut > njf_rhs->msg_last_ut) |
| 833 | return -1; |
| 834 | |
| 835 | // compare the file last modification timestamps |
| 836 | if (njf_lhs->file_last_modified_ut < njf_rhs->file_last_modified_ut) |
| 837 | return 1; |
| 838 | |
| 839 | if (njf_lhs->file_last_modified_ut > njf_rhs->file_last_modified_ut) |
| 840 | return -1; |
| 841 | |
| 842 | // compare the first message timestamps |
| 843 | if (njf_lhs->msg_first_ut < njf_rhs->msg_first_ut) |
| 844 | return 1; |
| 845 | |
| 846 | if (njf_lhs->msg_first_ut > njf_rhs->msg_first_ut) |
| 847 | return -1; |
| 848 | |
| 849 | return 0; |
| 850 | } |
| 851 | |
| 852 | int nd_journal_file_dict_items_forward_compar(const void *a, const void *b) |
| 853 | { |
| 854 | return -nd_journal_file_dict_items_backward_compar(a, b); |
| 855 | } |
| 856 | |
| 857 | static bool boot_id_conflict_cb( |
| 858 | const DICTIONARY_ITEM *item __maybe_unused, |
| 859 | void *old_value, |
| 860 | void *new_value, |
| 861 | void *data __maybe_unused) |
| 862 | { |
| 863 | usec_t *old_usec = old_value; |
| 864 | usec_t *new_usec = new_value; |
| 865 | |
| 866 | if (*new_usec < *old_usec) { |
| 867 | *old_usec = *new_usec; |
| 868 | return true; |
| 869 | } |
| 870 | |
| 871 | return false; |
| 872 | } |
| 873 | |
| 874 | void nd_journal_init_files_and_directories(void) |
| 875 | { |
| 876 | unsigned d = 0; |
| 877 | |
| 878 | // ------------------------------------------------------------------------ |
| 879 | // setup the journal directories |
| 880 | |
| 881 | journal_directories[d++].path = string_strdupz("/run/log/journal"); |
| 882 | journal_directories[d++].path = string_strdupz("/var/log/journal"); |
| 883 | |
| 884 | if (*netdata_configured_host_prefix) { |
| 885 | char path[PATH_MAX]; |
| 886 | snprintfz(path, sizeof(path), "%s/var/log/journal", netdata_configured_host_prefix); |
| 887 | journal_directories[d++].path = string_strdupz(path); |
| 888 | snprintfz(path, sizeof(path), "%s/run/log/journal", netdata_configured_host_prefix); |
| 889 | journal_directories[d++].path = string_strdupz(path); |
| 890 | } |
| 891 | |
| 892 | // terminate the list |
| 893 | journal_directories[d].path = NULL; |
| 894 | |
| 895 | // ------------------------------------------------------------------------ |
| 896 | // initialize the used hashes files registry |
| 897 | |
| 898 | used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); |
| 899 | |
| 900 | systemd_journal_session = (now_realtime_usec() / USEC_PER_SEC) * USEC_PER_SEC; |
| 901 | |
| 902 | nd_journal_files_registry = dictionary_create_advanced( |
| 903 | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct nd_journal_file)); |
| 904 | |
| 905 | dictionary_register_insert_callback(nd_journal_files_registry, files_registry_insert_cb, NULL); |
| 906 | dictionary_register_delete_callback(nd_journal_files_registry, files_registry_delete_cb, NULL); |
| 907 | dictionary_register_conflict_callback(nd_journal_files_registry, files_registry_conflict_cb, NULL); |
| 908 | |
| 909 | boot_ids_to_first_ut = |
| 910 | dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(usec_t)); |
| 911 | |
| 912 | dictionary_register_conflict_callback(boot_ids_to_first_ut, boot_id_conflict_cb, NULL); |
| 913 | } |