| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include <sys/resource.h> |
| 4 | #include <stdlib.h> |
| 5 | #include <unistd.h> |
| 6 | |
| 7 | #include "ebpf.h" |
| 8 | #include "ebpf_disk.h" |
| 9 | #include "libbpf_api/ebpf_library.h" |
| 10 | |
| 11 | struct config disk_config = APPCONFIG_INITIALIZER; |
| 12 | |
| 13 | static ebpf_local_maps_t disk_maps[] = { |
| 14 | {.name = "tbl_disk_iocall", |
| 15 | .internal_input = NETDATA_DISK_HISTOGRAM_LENGTH, |
| 16 | .user_input = 0, |
| 17 | .type = NETDATA_EBPF_MAP_STATIC, |
| 18 | .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, |
| 19 | #ifdef LIBBPF_MAJOR_VERSION |
| 20 | .map_type = BPF_MAP_TYPE_PERCPU_HASH |
| 21 | #endif |
| 22 | }, |
| 23 | {.name = "tmp_disk_tp_stat", |
| 24 | .internal_input = 8192, |
| 25 | .user_input = 8192, |
| 26 | .type = NETDATA_EBPF_MAP_STATIC, |
| 27 | .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, |
| 28 | #ifdef LIBBPF_MAJOR_VERSION |
| 29 | .map_type = BPF_MAP_TYPE_PERCPU_HASH |
| 30 | #endif |
| 31 | }, |
| 32 | {.name = NULL, |
| 33 | .internal_input = 0, |
| 34 | .user_input = 0, |
| 35 | .type = NETDATA_EBPF_MAP_CONTROLLER, |
| 36 | .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, |
| 37 | #ifdef LIBBPF_MAJOR_VERSION |
| 38 | .map_type = BPF_MAP_TYPE_PERCPU_ARRAY |
| 39 | #endif |
| 40 | }}; |
| 41 | static avl_tree_lock disk_tree; |
| 42 | netdata_ebpf_disks_t *disk_list = NULL; |
| 43 | |
| 44 | const char *tracepoint_block_type = "block"; |
| 45 | const char *tracepoint_block_issue = "block_rq_issue"; |
| 46 | const char *tracepoint_block_rq_complete = "block_rq_complete"; |
| 47 | |
| 48 | static int was_block_issue_enabled = 0; |
| 49 | static int was_block_rq_complete_enabled = 0; |
| 50 | static bool disk_safe_clean = false; |
| 51 | |
| 52 | static char **dimensions = NULL; |
| 53 | static netdata_syscall_stat_t disk_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS]; |
| 54 | static netdata_publish_syscall_t disk_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS]; |
| 55 | |
| 56 | static netdata_idx_t *disk_hash_values = NULL; |
| 57 | |
| 58 | netdata_mutex_t plot_mutex; |
| 59 | |
| 60 | static netdata_mutex_t tracepoint_mutex; |
| 61 | |
| 62 | #ifdef LIBBPF_MAJOR_VERSION |
| 63 | /** |
| 64 | * Set hash table |
| 65 | * |
| 66 | * Set the values for maps according the value given by kernel. |
| 67 | * |
| 68 | * @param obj is the main structure for bpf objects. |
| 69 | */ |
| 70 | static inline void ebpf_disk_set_hash_table(struct disk_bpf *obj) |
| 71 | { |
| 72 | disk_maps[NETDATA_DISK_IO].map_fd = bpf_map__fd(obj->maps.tbl_disk_iocall); |
| 73 | } |
| 74 | |
| 75 | /** |
| 76 | * Load and attach |
| 77 | * |
| 78 | * Load and attach the eBPF code in kernel. |
| 79 | * |
| 80 | * @param obj is the main structure for bpf objects. |
| 81 | * |
| 82 | * @return it returns 0 on success and -1 otherwise |
| 83 | */ |
| 84 | static inline int ebpf_disk_load_and_attach(struct disk_bpf *obj) |
| 85 | { |
| 86 | int ret = disk_bpf__load(obj); |
| 87 | if (ret) { |
| 88 | return ret; |
| 89 | } |
| 90 | |
| 91 | return disk_bpf__attach(obj); |
| 92 | } |
| 93 | #endif |
| 94 | |
| 95 | /***************************************************************** |
| 96 | * |
| 97 | * FUNCTIONS TO MANIPULATE HARD DISKS |
| 98 | * |
| 99 | *****************************************************************/ |
| 100 | |
| 101 | /** |
| 102 | * Read file to string |
| 103 | * |
| 104 | * @param filename file to read |
| 105 | * @param buffer buffer to store content |
| 106 | * @param size buffer size |
| 107 | * |
| 108 | * @return It returns content length on success and -1 otherwise |
| 109 | */ |
| 110 | static inline ssize_t ebpf_read_file_to_str(const char *filename, char *buffer, size_t size) |
| 111 | { |
| 112 | int fd = open(filename, O_RDONLY, 0); |
| 113 | if (fd < 0) |
| 114 | return -1; |
| 115 | |
| 116 | ssize_t file_length = read(fd, buffer, size - 1); |
| 117 | close(fd); |
| 118 | if (file_length <= 0) |
| 119 | return -1; |
| 120 | |
| 121 | buffer[file_length] = '\0'; |
| 122 | return file_length; |
| 123 | } |
| 124 | |
| 125 | /** |
| 126 | * Parse start |
| 127 | * |
| 128 | * Parse start address of disk |
| 129 | * |
| 130 | * @param w structure where data is stored |
| 131 | * @param filename variable used to store value |
| 132 | * |
| 133 | * @return It returns 0 on success and -1 otherwise |
| 134 | */ |
| 135 | static inline int ebpf_disk_parse_start(netdata_ebpf_disks_t *w, char *filename) |
| 136 | { |
| 137 | char content[FILENAME_MAX + 1]; |
| 138 | ssize_t file_length = ebpf_read_file_to_str(filename, content, FILENAME_MAX); |
| 139 | if (file_length <= 0) |
| 140 | return -1; |
| 141 | |
| 142 | w->start = strtoul(content, NULL, 10); |
| 143 | |
| 144 | return 0; |
| 145 | } |
| 146 | |
| 147 | /** |
| 148 | * Parse uevent |
| 149 | * |
| 150 | * Parse uevent file |
| 151 | * |
| 152 | * @param w structure where data is stored |
| 153 | * @param filename variable used to store value |
| 154 | * |
| 155 | * @return It returns 0 on success and -1 otherwise |
| 156 | */ |
| 157 | static inline int ebpf_parse_uevent(netdata_ebpf_disks_t *w, char *filename) |
| 158 | { |
| 159 | (void)w; |
| 160 | return access(filename, F_OK); |
| 161 | } |
| 162 | |
| 163 | /** |
| 164 | * Parse Size |
| 165 | * |
| 166 | * @param w structure where data is stored |
| 167 | * @param filename variable used to store value |
| 168 | * |
| 169 | * @return It returns 0 on success and -1 otherwise |
| 170 | */ |
| 171 | static inline int ebpf_parse_size(netdata_ebpf_disks_t *w, char *filename) |
| 172 | { |
| 173 | char content[FILENAME_MAX + 1]; |
| 174 | ssize_t file_length = ebpf_read_file_to_str(filename, content, FILENAME_MAX); |
| 175 | if (file_length <= 0) |
| 176 | return -1; |
| 177 | |
| 178 | w->end = w->start + strtoul(content, NULL, 10) - 1; |
| 179 | |
| 180 | return 0; |
| 181 | } |
| 182 | |
| 183 | /** |
| 184 | * Read Disk information |
| 185 | * |
| 186 | * Read disk information from /sys/block |
| 187 | * |
| 188 | * @param w structure where data is stored |
| 189 | * @param name disk name |
| 190 | * @param main_disk pointer to main disk structure |
| 191 | * @param bootsector_key bootsector key for the disk |
| 192 | */ |
| 193 | static void |
| 194 | ebpf_read_disk_info(netdata_ebpf_disks_t *w, char *name, netdata_ebpf_disks_t **main_disk, uint32_t *bootsector_key) |
| 195 | { |
| 196 | char *path = "/sys/block"; |
| 197 | char disk[NETDATA_DISK_NAME_LEN + 1]; |
| 198 | char filename[FILENAME_MAX + 1]; |
| 199 | snprintfz(disk, NETDATA_DISK_NAME_LEN, "%s", name); |
| 200 | size_t length = strlen(disk); |
| 201 | if (!length) { |
| 202 | return; |
| 203 | } |
| 204 | |
| 205 | length--; |
| 206 | int has_digits = 0; |
| 207 | while (length != (size_t)-1 && isdigit((int)disk[length])) { |
| 208 | disk[length--] = '\0'; |
| 209 | has_digits = 1; |
| 210 | } |
| 211 | |
| 212 | // We are looking for partition information, if it is a device we will set it as main disk |
| 213 | if (!has_digits) { |
| 214 | *main_disk = w; |
| 215 | *bootsector_key = MKDEV(w->major, w->minor); |
| 216 | w->bootsector_key = *bootsector_key; |
| 217 | return; |
| 218 | } |
| 219 | |
| 220 | // This is a partition, link it to main disk |
| 221 | w->bootsector_key = *bootsector_key; |
| 222 | |
| 223 | snprintfz(filename, FILENAME_MAX, "%s/%s/%s/uevent", path, disk, name); |
| 224 | if (ebpf_parse_uevent(w, filename)) |
| 225 | return; |
| 226 | |
| 227 | snprintfz(filename, FILENAME_MAX, "%s/%s/%s/start", path, disk, name); |
| 228 | if (ebpf_disk_parse_start(w, filename)) |
| 229 | return; |
| 230 | |
| 231 | snprintfz(filename, FILENAME_MAX, "%s/%s/%s/size", path, disk, name); |
| 232 | ebpf_parse_size(w, filename); |
| 233 | } |
| 234 | |
| 235 | /** |
| 236 | * New encode dev |
| 237 | * |
| 238 | * New encode algorithm extracted from https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L39 |
| 239 | * |
| 240 | * @param major driver major number |
| 241 | * @param minor driver minor number |
| 242 | * |
| 243 | * @return |
| 244 | */ |
| 245 | static inline uint32_t netdata_new_encode_dev(uint32_t major, uint32_t minor) |
| 246 | { |
| 247 | return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12); |
| 248 | } |
| 249 | |
| 250 | /** |
| 251 | * Compare disks |
| 252 | * |
| 253 | * Compare major and minor values to add disks to tree. |
| 254 | * |
| 255 | * @param a pointer to netdata_ebpf_disks |
| 256 | * @param b pointer to netdata_ebpf_disks |
| 257 | * |
| 258 | * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b. |
| 259 | */ |
| 260 | static int ebpf_compare_disks(void *a, void *b) |
| 261 | { |
| 262 | netdata_ebpf_disks_t *ptr1 = a; |
| 263 | netdata_ebpf_disks_t *ptr2 = b; |
| 264 | |
| 265 | return (ptr1->dev > ptr2->dev) - (ptr1->dev < ptr2->dev); |
| 266 | } |
| 267 | |
| 268 | /** |
| 269 | * Update listen table |
| 270 | * |
| 271 | * Update link list when it is necessary. |
| 272 | * |
| 273 | * @param name disk name |
| 274 | * @param major major disk identifier |
| 275 | * @param minor minor disk identifier |
| 276 | * @param current_time current timestamp |
| 277 | */ |
| 278 | static void update_disk_table(char *name, int major, int minor, time_t current_time) |
| 279 | { |
| 280 | static netdata_ebpf_disks_t *main_disk = NULL; |
| 281 | static uint32_t bootsector_key = 0; |
| 282 | |
| 283 | netdata_ebpf_disks_t find; |
| 284 | size_t length = strlen(name); |
| 285 | if (length >= NETDATA_DISK_NAME_LEN) |
| 286 | length = NETDATA_DISK_NAME_LEN; |
| 287 | |
| 288 | uint32_t dev = netdata_new_encode_dev(major, minor); |
| 289 | find.dev = dev; |
| 290 | netdata_ebpf_disks_t *ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find); |
| 291 | if (ret) { |
| 292 | ret->flags |= NETDATA_DISK_IS_HERE; |
| 293 | ret->last_update = current_time; |
| 294 | return; |
| 295 | } |
| 296 | |
| 297 | netdata_ebpf_disks_t *w; |
| 298 | if (likely(disk_list)) { |
| 299 | w = callocz(1, sizeof(netdata_ebpf_disks_t)); |
| 300 | netdata_ebpf_disks_t *update_next = disk_list; |
| 301 | while (update_next->next) |
| 302 | update_next = update_next->next; |
| 303 | update_next->next = w; |
| 304 | } else { |
| 305 | disk_list = w = callocz(1, sizeof(netdata_ebpf_disks_t)); |
| 306 | } |
| 307 | |
| 308 | memcpy(w->family, name, length); |
| 309 | w->family[length] = '\0'; |
| 310 | w->major = major; |
| 311 | w->minor = minor; |
| 312 | w->dev = dev; |
| 313 | |
| 314 | ebpf_read_disk_info(w, name, &main_disk, &bootsector_key); |
| 315 | |
| 316 | netdata_ebpf_disks_t *check; |
| 317 | check = (netdata_ebpf_disks_t *)avl_insert_lock(&disk_tree, (avl_t *)w); |
| 318 | if (check != w) |
| 319 | netdata_log_error("Internal error, cannot insert the AVL tree."); |
| 320 | |
| 321 | #ifdef NETDATA_INTERNAL_CHECKS |
| 322 | netdata_log_info( |
| 323 | "The Latency is monitoring the hard disk %s (Major = %d, Minor = %d, Device = %u)", name, major, minor, w->dev); |
| 324 | #endif |
| 325 | |
| 326 | w->flags |= NETDATA_DISK_IS_HERE; |
| 327 | } |
| 328 | |
| 329 | /** |
| 330 | * Read Local Disks |
| 331 | * |
| 332 | * Parse /proc/partitions to get block disks used to measure latency. |
| 333 | * |
| 334 | * @return It returns 0 on success and -1 otherwise |
| 335 | */ |
| 336 | static int read_local_disks(void) |
| 337 | { |
| 338 | char filename[FILENAME_MAX + 1]; |
| 339 | snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, NETDATA_EBPF_PROC_PARTITIONS); |
| 340 | procfile *ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT); |
| 341 | if (!ff) |
| 342 | return -1; |
| 343 | |
| 344 | ff = procfile_readall(ff); |
| 345 | if (!ff) |
| 346 | return -1; |
| 347 | |
| 348 | size_t lines = procfile_lines(ff), l; |
| 349 | time_t current_time = now_realtime_sec(); |
| 350 | for (l = 2; l < lines; l++) { |
| 351 | size_t words = procfile_linewords(ff, l); |
| 352 | // This is header or end of file |
| 353 | if (unlikely(words < 4)) |
| 354 | continue; |
| 355 | |
| 356 | int major = (int)strtol(procfile_lineword(ff, l, 0), NULL, 10); |
| 357 | // The main goal of this thread is to measure block devices, so any block device with major number |
| 358 | // smaller than 7 according /proc/devices is not "important". |
| 359 | if (major > 7) { |
| 360 | int minor = (int)strtol(procfile_lineword(ff, l, 1), NULL, 10); |
| 361 | update_disk_table(procfile_lineword(ff, l, 3), major, minor, current_time); |
| 362 | } |
| 363 | } |
| 364 | |
| 365 | procfile_close(ff); |
| 366 | |
| 367 | return 0; |
| 368 | } |
| 369 | |
| 370 | /** |
| 371 | * Update disks |
| 372 | * |
| 373 | * @param em main thread structure |
| 374 | */ |
| 375 | void ebpf_update_disks(ebpf_module_t *em) |
| 376 | { |
| 377 | static time_t update_every = 0; |
| 378 | time_t curr = now_realtime_sec(); |
| 379 | if (curr < update_every) |
| 380 | return; |
| 381 | |
| 382 | update_every = curr + 5 * em->update_every; |
| 383 | |
| 384 | (void)read_local_disks(); |
| 385 | } |
| 386 | |
| 387 | /***************************************************************** |
| 388 | * |
| 389 | * FUNCTIONS TO CLOSE THE THREAD |
| 390 | * |
| 391 | *****************************************************************/ |
| 392 | |
| 393 | /** |
| 394 | * Disk disable tracepoints |
| 395 | * |
| 396 | * Disable tracepoints when the plugin was responsible to enable it. |
| 397 | */ |
| 398 | static void ebpf_disk_disable_tracepoints(void) |
| 399 | { |
| 400 | const char *default_message = "Cannot disable the tracepoint"; |
| 401 | int block_issue_enabled; |
| 402 | int block_rq_complete_enabled; |
| 403 | |
| 404 | netdata_mutex_lock(&tracepoint_mutex); |
| 405 | block_issue_enabled = was_block_issue_enabled; |
| 406 | block_rq_complete_enabled = was_block_rq_complete_enabled; |
| 407 | netdata_mutex_unlock(&tracepoint_mutex); |
| 408 | |
| 409 | if (!block_issue_enabled) { |
| 410 | if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_issue)) |
| 411 | netdata_log_error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_issue); |
| 412 | } |
| 413 | |
| 414 | if (!block_rq_complete_enabled) { |
| 415 | if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete)) |
| 416 | netdata_log_error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_rq_complete); |
| 417 | } |
| 418 | } |
| 419 | |
| 420 | /** |
| 421 | * Cleanup Disk List |
| 422 | */ |
| 423 | static void ebpf_cleanup_disk_list(void) |
| 424 | { |
| 425 | netdata_ebpf_disks_t *move = disk_list; |
| 426 | while (move) { |
| 427 | netdata_ebpf_disks_t *next = move->next; |
| 428 | |
| 429 | freez(move->histogram.name); |
| 430 | move->histogram.name = NULL; |
| 431 | freez(move->histogram.title); |
| 432 | move->histogram.title = NULL; |
| 433 | freez(move->histogram.ctx); |
| 434 | move->histogram.ctx = NULL; |
| 435 | |
| 436 | freez(move); |
| 437 | |
| 438 | move = next; |
| 439 | } |
| 440 | disk_list = NULL; |
| 441 | } |
| 442 | |
| 443 | /** |
| 444 | * Obsolete global |
| 445 | * |
| 446 | * Obsolete global charts created by thread. |
| 447 | * |
| 448 | * @param em a pointer to `struct ebpf_module` |
| 449 | */ |
| 450 | static void ebpf_obsolete_disk_global(ebpf_module_t *em) |
| 451 | { |
| 452 | netdata_ebpf_disks_t *move = disk_list; |
| 453 | while (move) { |
| 454 | uint32_t flags = move->flags; |
| 455 | if (flags & NETDATA_DISK_CHART_CREATED) { |
| 456 | ebpf_write_chart_obsolete( |
| 457 | move->histogram.name, |
| 458 | move->family, |
| 459 | "", |
| 460 | "Disk latency", |
| 461 | EBPF_COMMON_UNITS_CALLS_PER_SEC, |
| 462 | move->family, |
| 463 | NETDATA_EBPF_CHART_TYPE_STACKED, |
| 464 | NETDATA_EBPF_DISK_LATENCY_CONTEXT, |
| 465 | move->histogram.order, |
| 466 | em->update_every); |
| 467 | } |
| 468 | |
| 469 | move = move->next; |
| 470 | } |
| 471 | } |
| 472 | |
| 473 | static void ebpf_disk_exit(void *pptr) |
| 474 | { |
| 475 | ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); |
| 476 | if (!em) |
| 477 | return; |
| 478 | |
| 479 | if (!disk_safe_clean) { |
| 480 | netdata_mutex_lock(&ebpf_exit_cleanup); |
| 481 | ebpf_module_enabled_set(em, NETDATA_THREAD_EBPF_STOPPED); |
| 482 | netdata_mutex_unlock(&ebpf_exit_cleanup); |
| 483 | return; |
| 484 | } |
| 485 | |
| 486 | if (ebpf_module_enabled_get(em) == NETDATA_THREAD_EBPF_FUNCTION_RUNNING && !ebpf_plugin_stop()) { |
| 487 | netdata_mutex_lock(&lock); |
| 488 | ebpf_obsolete_disk_global(em); |
| 489 | netdata_mutex_unlock(&lock); |
| 490 | fflush(stdout); |
| 491 | } |
| 492 | |
| 493 | ebpf_disk_disable_tracepoints(); |
| 494 | |
| 495 | if (dimensions) { |
| 496 | ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS); |
| 497 | dimensions = NULL; |
| 498 | } |
| 499 | |
| 500 | freez(disk_hash_values); |
| 501 | disk_hash_values = NULL; |
| 502 | |
| 503 | netdata_mutex_destroy(&plot_mutex); |
| 504 | netdata_mutex_destroy(&tracepoint_mutex); |
| 505 | |
| 506 | if (disk_list) |
| 507 | ebpf_cleanup_disk_list(); |
| 508 | |
| 509 | if (!ebpf_plugin_stop() && em->functions.bpf_unload) |
| 510 | em->functions.bpf_unload(em); |
| 511 | |
| 512 | netdata_mutex_lock(&ebpf_exit_cleanup); |
| 513 | ebpf_module_enabled_set(em, NETDATA_THREAD_EBPF_STOPPED); |
| 514 | netdata_mutex_unlock(&ebpf_exit_cleanup); |
| 515 | } |
| 516 | |
| 517 | /***************************************************************** |
| 518 | * |
| 519 | * MAIN LOOP |
| 520 | * |
| 521 | *****************************************************************/ |
| 522 | |
| 523 | /** |
| 524 | * Fill Plot list |
| 525 | * |
| 526 | * Mark disk as needing to be plotted |
| 527 | * |
| 528 | * @param ptr a pointer for current disk |
| 529 | */ |
| 530 | static void ebpf_fill_plot_disks(netdata_ebpf_disks_t *ptr) |
| 531 | { |
| 532 | netdata_mutex_lock(&plot_mutex); |
| 533 | ptr->flags |= NETDATA_DISK_ADDED_TO_PLOT_LIST; |
| 534 | netdata_mutex_unlock(&plot_mutex); |
| 535 | } |
| 536 | |
| 537 | /** |
| 538 | * Read hard disk table |
| 539 | * |
| 540 | * Read the table with number of calls for all functions |
| 541 | * |
| 542 | * @param table file descriptor for table |
| 543 | * @param maps_per_core do I need to read all cores? |
| 544 | */ |
| 545 | static void read_hard_disk_tables(int table, int maps_per_core) |
| 546 | { |
| 547 | netdata_idx_t *values = disk_hash_values; |
| 548 | block_key_t key = {}; |
| 549 | block_key_t next_key = {}; |
| 550 | |
| 551 | netdata_ebpf_disks_t *ret = NULL; |
| 552 | |
| 553 | while (bpf_map_get_next_key(table, &key, &next_key) == 0) { |
| 554 | if (ebpf_plugin_stop()) |
| 555 | break; |
| 556 | |
| 557 | int test = bpf_map_lookup_elem(table, &key, values); |
| 558 | if (test < 0) { |
| 559 | key = next_key; |
| 560 | continue; |
| 561 | } |
| 562 | |
| 563 | netdata_ebpf_disks_t find; |
| 564 | find.dev = key.dev; |
| 565 | |
| 566 | if (!ret || find.dev != ret->dev) |
| 567 | ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find); |
| 568 | |
| 569 | // Disk was inserted after we parse /proc/partitions |
| 570 | if (!ret) { |
| 571 | if (read_local_disks()) { |
| 572 | key = next_key; |
| 573 | continue; |
| 574 | } |
| 575 | |
| 576 | ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find); |
| 577 | if (!ret) { |
| 578 | // We should never reach this point, but we are adding it to keep a safe code |
| 579 | key = next_key; |
| 580 | continue; |
| 581 | } |
| 582 | } |
| 583 | |
| 584 | uint64_t total = 0; |
| 585 | int i; |
| 586 | int end = (maps_per_core) ? 1 : ebpf_nprocs; |
| 587 | for (i = 0; i < end; i++) { |
| 588 | total += values[i]; |
| 589 | } |
| 590 | |
| 591 | ret->histogram.histogram[key.bin] = total; |
| 592 | |
| 593 | if (!(ret->flags & NETDATA_DISK_ADDED_TO_PLOT_LIST)) |
| 594 | ebpf_fill_plot_disks(ret); |
| 595 | |
| 596 | key = next_key; |
| 597 | } |
| 598 | } |
| 599 | |
| 600 | /** |
| 601 | * Obsolete Hard Disk charts |
| 602 | * |
| 603 | * Make Hard disk charts and fill chart name |
| 604 | * |
| 605 | * @param w the structure with necessary information to create the chart |
| 606 | * @param update_every value to overwrite the update frequency set by the server. |
| 607 | */ |
| 608 | static void ebpf_obsolete_hd_charts(netdata_ebpf_disks_t *w, int update_every) |
| 609 | { |
| 610 | ebpf_write_chart_obsolete( |
| 611 | w->histogram.name, |
| 612 | w->family, |
| 613 | "", |
| 614 | "Disk latency", |
| 615 | EBPF_COMMON_UNITS_CALLS_PER_SEC, |
| 616 | w->family, |
| 617 | NETDATA_EBPF_CHART_TYPE_STACKED, |
| 618 | NETDATA_EBPF_DISK_LATENCY_CONTEXT, |
| 619 | w->histogram.order, |
| 620 | update_every); |
| 621 | |
| 622 | w->flags = NETDATA_DISK_NONE; |
| 623 | } |
| 624 | |
| 625 | /** |
| 626 | * Create Hard Disk charts |
| 627 | * |
| 628 | * Make Hard disk charts and fill chart name |
| 629 | * |
| 630 | * @param w the structure with necessary information to create the chart |
| 631 | * @param update_every value to overwrite the update frequency set by the server. |
| 632 | */ |
| 633 | static void ebpf_create_hd_charts(netdata_ebpf_disks_t *w, int update_every) |
| 634 | { |
| 635 | int order = NETDATA_CHART_PRIO_DISK_LATENCY; |
| 636 | char *family = w->family; |
| 637 | |
| 638 | w->histogram.name = strdupz("disk_latency_io"); |
| 639 | w->histogram.title = NULL; |
| 640 | w->histogram.order = order; |
| 641 | |
| 642 | ebpf_create_chart( |
| 643 | w->histogram.name, |
| 644 | family, |
| 645 | "Disk latency", |
| 646 | EBPF_COMMON_UNITS_CALLS_PER_SEC, |
| 647 | family, |
| 648 | NETDATA_EBPF_DISK_LATENCY_CONTEXT, |
| 649 | NETDATA_EBPF_CHART_TYPE_STACKED, |
| 650 | order, |
| 651 | ebpf_create_global_dimension, |
| 652 | disk_publish_aggregated, |
| 653 | NETDATA_EBPF_HIST_MAX_BINS, |
| 654 | update_every, |
| 655 | NETDATA_EBPF_MODULE_NAME_DISK); |
| 656 | order++; |
| 657 | |
| 658 | w->flags |= NETDATA_DISK_CHART_CREATED; |
| 659 | |
| 660 | fflush(stdout); |
| 661 | } |
| 662 | |
| 663 | /** |
| 664 | * Remove pointer from plot |
| 665 | * |
| 666 | * Remove disk from tracking when not present - now iterates disk_list directly |
| 667 | */ |
| 668 | static void ebpf_remove_pointer_from_plot_disk(ebpf_module_t *em) |
| 669 | { |
| 670 | time_t current_time = now_realtime_sec(); |
| 671 | time_t limit = 10 * em->update_every; |
| 672 | netdata_mutex_lock(&plot_mutex); |
| 673 | netdata_ebpf_disks_t *move = disk_list, *prev = NULL; |
| 674 | int update_every = em->update_every; |
| 675 | while (move) { |
| 676 | uint32_t flags = move->flags; |
| 677 | |
| 678 | if (!(flags & NETDATA_DISK_IS_HERE) && ((current_time - move->last_update) > limit)) { |
| 679 | ebpf_obsolete_hd_charts(move, update_every); |
| 680 | avl_t *ret = (avl_t *)avl_remove_lock(&disk_tree, (avl_t *)move); |
| 681 | UNUSED(ret); |
| 682 | if (prev) { |
| 683 | prev->next = move->next; |
| 684 | netdata_ebpf_disks_t *clean = move; |
| 685 | move = move->next; |
| 686 | freez(clean->histogram.name); |
| 687 | freez(clean->histogram.title); |
| 688 | freez(clean->histogram.ctx); |
| 689 | freez(clean); |
| 690 | continue; |
| 691 | } else { |
| 692 | disk_list = move->next; |
| 693 | freez(move->histogram.name); |
| 694 | freez(move->histogram.title); |
| 695 | freez(move->histogram.ctx); |
| 696 | freez(move); |
| 697 | move = disk_list; |
| 698 | continue; |
| 699 | } |
| 700 | } |
| 701 | |
| 702 | prev = move; |
| 703 | move = move->next; |
| 704 | } |
| 705 | netdata_mutex_unlock(&plot_mutex); |
| 706 | } |
| 707 | |
| 708 | /** |
| 709 | * Send Hard disk data |
| 710 | * |
| 711 | * Send hard disk information to Netdata. |
| 712 | * |
| 713 | * @param update_every value to overwrite the update frequency set by the server. |
| 714 | */ |
| 715 | static void ebpf_latency_send_hd_data(int update_every) |
| 716 | { |
| 717 | netdata_mutex_lock(&plot_mutex); |
| 718 | if (!disk_list) { |
| 719 | netdata_mutex_unlock(&plot_mutex); |
| 720 | return; |
| 721 | } |
| 722 | |
| 723 | netdata_ebpf_disks_t *move = disk_list; |
| 724 | while (move) { |
| 725 | uint32_t flags = move->flags; |
| 726 | if (!(flags & NETDATA_DISK_CHART_CREATED)) { |
| 727 | ebpf_create_hd_charts(move, update_every); |
| 728 | } |
| 729 | |
| 730 | if ((flags & NETDATA_DISK_CHART_CREATED)) { |
| 731 | write_histogram_chart( |
| 732 | move->histogram.name, move->family, move->histogram.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS); |
| 733 | } |
| 734 | |
| 735 | move->flags &= ~NETDATA_DISK_IS_HERE; |
| 736 | |
| 737 | move = move->next; |
| 738 | } |
| 739 | netdata_mutex_unlock(&plot_mutex); |
| 740 | } |
| 741 | |
| 742 | /** |
| 743 | * Main loop for this collector. |
| 744 | */ |
| 745 | static void disk_collector(ebpf_module_t *em) |
| 746 | { |
| 747 | disk_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t)); |
| 748 | |
| 749 | int update_every = em->update_every; |
| 750 | heartbeat_t hb; |
| 751 | heartbeat_init(&hb, USEC_PER_SEC); |
| 752 | int counter = update_every - 1; |
| 753 | int maps_per_core = em->maps_per_core; |
| 754 | uint32_t running_time = 0; |
| 755 | uint32_t lifetime = em->lifetime; |
| 756 | while (!ebpf_plugin_stop() && running_time < lifetime) { |
| 757 | if (ebpf_plugin_stop()) |
| 758 | break; |
| 759 | |
| 760 | heartbeat_next(&hb); |
| 761 | |
| 762 | if (ebpf_plugin_stop()) |
| 763 | break; |
| 764 | |
| 765 | if (++counter != update_every) |
| 766 | continue; |
| 767 | |
| 768 | counter = 0; |
| 769 | read_hard_disk_tables(disk_maps[NETDATA_DISK_IO].map_fd, maps_per_core); |
| 770 | netdata_mutex_lock(&lock); |
| 771 | ebpf_remove_pointer_from_plot_disk(em); |
| 772 | ebpf_latency_send_hd_data(update_every); |
| 773 | |
| 774 | netdata_mutex_unlock(&lock); |
| 775 | |
| 776 | ebpf_update_disks(em); |
| 777 | |
| 778 | if (ebpf_plugin_stop()) |
| 779 | break; |
| 780 | |
| 781 | netdata_mutex_lock(&ebpf_exit_cleanup); |
| 782 | if (running_time) |
| 783 | running_time += update_every; |
| 784 | else |
| 785 | running_time = update_every; |
| 786 | |
| 787 | em->running_time = running_time; |
| 788 | netdata_mutex_unlock(&ebpf_exit_cleanup); |
| 789 | } |
| 790 | } |
| 791 | |
| 792 | /***************************************************************** |
| 793 | * |
| 794 | * EBPF DISK THREAD |
| 795 | * |
| 796 | *****************************************************************/ |
| 797 | |
| 798 | /** |
| 799 | * Enable tracepoints |
| 800 | * |
| 801 | * Enable necessary tracepoints for thread. |
| 802 | * |
| 803 | * @return It returns 0 on success and -1 otherwise |
| 804 | */ |
| 805 | static int ebpf_disk_enable_tracepoints() |
| 806 | { |
| 807 | int test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_issue); |
| 808 | if (test == -1) |
| 809 | return -1; |
| 810 | else if (!test) { |
| 811 | if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_issue)) |
| 812 | return -1; |
| 813 | } |
| 814 | |
| 815 | netdata_mutex_lock(&tracepoint_mutex); |
| 816 | was_block_issue_enabled = test; |
| 817 | netdata_mutex_unlock(&tracepoint_mutex); |
| 818 | |
| 819 | test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_rq_complete); |
| 820 | if (test == -1) |
| 821 | return -1; |
| 822 | else if (!test) { |
| 823 | if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete)) |
| 824 | return -1; |
| 825 | } |
| 826 | |
| 827 | netdata_mutex_lock(&tracepoint_mutex); |
| 828 | was_block_rq_complete_enabled = test; |
| 829 | netdata_mutex_unlock(&tracepoint_mutex); |
| 830 | |
| 831 | return 0; |
| 832 | } |
| 833 | |
| 834 | /* |
| 835 | * Load BPF |
| 836 | * |
| 837 | * Load BPF files. |
| 838 | * |
| 839 | * @param em the structure with configuration |
| 840 | * |
| 841 | * @return It returns 0 on success and -1 otherwise. |
| 842 | */ |
| 843 | static int ebpf_disk_load_bpf(ebpf_module_t *em) |
| 844 | { |
| 845 | int ret = 0; |
| 846 | if (em->load & EBPF_LOAD_LEGACY) { |
| 847 | em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); |
| 848 | if (!em->probe_links) { |
| 849 | ret = -1; |
| 850 | } |
| 851 | } |
| 852 | #ifdef LIBBPF_MAJOR_VERSION |
| 853 | else { |
| 854 | disk_bpf_obj = disk_bpf__open(); |
| 855 | if (!disk_bpf_obj) |
| 856 | ret = -1; |
| 857 | else { |
| 858 | ret = ebpf_disk_load_and_attach(disk_bpf_obj); |
| 859 | if (ret) { |
| 860 | disk_bpf__destroy(disk_bpf_obj); |
| 861 | disk_bpf_obj = NULL; |
| 862 | } else { |
| 863 | ebpf_disk_set_hash_table(disk_bpf_obj); |
| 864 | } |
| 865 | } |
| 866 | } |
| 867 | #endif |
| 868 | |
| 869 | if (ret) |
| 870 | netdata_log_error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->info.thread_name); |
| 871 | |
| 872 | return ret; |
| 873 | } |
| 874 | |
| 875 | /** |
| 876 | * Disk thread |
| 877 | * |
| 878 | * Thread used to generate disk charts. |
| 879 | * |
| 880 | * @param ptr a pointer to `struct ebpf_module` |
| 881 | * |
| 882 | * @return It always return NULL |
| 883 | */ |
| 884 | void ebpf_disk_thread(void *ptr) |
| 885 | { |
| 886 | ebpf_module_t *em = (ebpf_module_t *)ptr; |
| 887 | |
| 888 | CLEANUP_FUNCTION_REGISTER(ebpf_disk_exit) cleanup_ptr = em; |
| 889 | |
| 890 | disk_safe_clean = false; |
| 891 | |
| 892 | if (!ebpf_module_thread_has_valid_state(em)) { |
| 893 | goto enddisk; |
| 894 | } |
| 895 | |
| 896 | em->maps = disk_maps; |
| 897 | |
| 898 | if (netdata_mutex_init(&plot_mutex)) { |
| 899 | netdata_log_error("Cannot initialize local mutex"); |
| 900 | goto enddisk; |
| 901 | } |
| 902 | |
| 903 | if (netdata_mutex_init(&tracepoint_mutex)) { |
| 904 | netdata_log_error("Cannot initialize tracepoint mutex"); |
| 905 | goto enddisk; |
| 906 | } |
| 907 | |
| 908 | disk_safe_clean = true; |
| 909 | |
| 910 | if (ebpf_disk_enable_tracepoints()) { |
| 911 | goto enddisk; |
| 912 | } |
| 913 | |
| 914 | // disk_safe_clean already true - mutexes will be cleaned up on exit |
| 915 | |
| 916 | avl_init_lock(&disk_tree, ebpf_compare_disks); |
| 917 | if (read_local_disks()) { |
| 918 | goto enddisk; |
| 919 | } |
| 920 | |
| 921 | #ifdef LIBBPF_MAJOR_VERSION |
| 922 | ebpf_define_map_type(disk_maps, em->maps_per_core, running_on_kernel); |
| 923 | ebpf_adjust_thread_load(em, default_btf); |
| 924 | #endif |
| 925 | if (ebpf_disk_load_bpf(em)) { |
| 926 | goto enddisk; |
| 927 | } |
| 928 | ebpf_mark_program_loaded(); |
| 929 | |
| 930 | int algorithms[NETDATA_EBPF_HIST_MAX_BINS]; |
| 931 | ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX); |
| 932 | dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS); |
| 933 | if (!dimensions) { |
| 934 | netdata_log_error("Cannot allocate histogram dimensions"); |
| 935 | goto enddisk; |
| 936 | } |
| 937 | |
| 938 | ebpf_global_labels( |
| 939 | disk_aggregated_data, disk_publish_aggregated, dimensions, dimensions, algorithms, NETDATA_EBPF_HIST_MAX_BINS); |
| 940 | |
| 941 | netdata_mutex_lock(&lock); |
| 942 | ebpf_update_stats(&plugin_statistics, em); |
| 943 | ebpf_update_kernel_memory_with_vector(&plugin_statistics, disk_maps, EBPF_ACTION_STAT_ADD); |
| 944 | netdata_mutex_unlock(&lock); |
| 945 | |
| 946 | disk_collector(em); |
| 947 | |
| 948 | enddisk: |
| 949 | ebpf_update_disabled_plugin_stats(em); |
| 950 | } |