@cryptotaxi247 / netdata-1 / commits / b3d90d5b4

monitor applications file descriptor limits (#15417)

Co-authored-by: ilyam8 <ilya@netdata.cloud>

Costa Tsaousis committed Jul 18, 2023 at 00:33 UTC b3d90d5b4bf2793ad1fba7442f2c70728904e06b

8 files changed +273 -35

Makefile.am

@@ -401,6 +401,7 @@ PROC_PLUGIN_FILES = \

401 collectors/proc.plugin/zfs_common.h \

402 collectors/proc.plugin/proc_spl_kstat_zfs.c \

403 collectors/proc.plugin/proc_stat.c \

404 + collectors/proc.plugin/proc_sys_fs_file_nr.c \

405 collectors/proc.plugin/proc_sys_kernel_random_entropy_avail.c \

406 collectors/proc.plugin/proc_vmstat.c \

407 collectors/proc.plugin/proc_uptime.c \

collectors/all.h

@@ -51,6 +51,7 @@

51 #define NETDATA_CHART_PRIO_SYSTEM_INTERRUPTS 1000

52 #define NETDATA_CHART_PRIO_SYSTEM_DEV_INTR 1000 // freebsd only

53 #define NETDATA_CHART_PRIO_SYSTEM_SOFT_INTR 1100 // freebsd only

54 +#define NETDATA_CHART_PRIO_SYSTEM_FILES_NR 1000

55 #define NETDATA_CHART_PRIO_SYSTEM_ENTROPY 1000

56 #define NETDATA_CHART_PRIO_SYSTEM_UPTIME 1000

57 #define NETDATA_CHART_PRIO_CLOCK_SYNC_STATE 1100

collectors/apps.plugin/apps_plugin.c

+157 -35

@@ -141,6 +141,17 @@ static const char *proc_states[] = {

141 // internal flags

142 // handled in code (automatically set)

143

144 +// log each problem once per process

145 +// log flood protection flags (log_thrown)

146 +typedef enum __attribute__((packed)) {

147 + PID_LOG_IO = (1 << 0),

148 + PID_LOG_STATUS = (1 << 1),

149 + PID_LOG_CMDLINE = (1 << 2),

150 + PID_LOG_FDS = (1 << 3),

151 + PID_LOG_STAT = (1 << 4),

152 + PID_LOG_LIMITS = (1 << 5),

153 +} PID_LOG;

154 +

155 static int

156 show_guest_time = 0, // 1 when guest values are collected

157 show_guest_time_old = 0,

@@ -211,6 +222,25 @@ struct openfds {

222 kernel_uint_t other;

223 };

224

225 +struct pid_limits {

226 +// kernel_uint_t max_cpu_time;

227 +// kernel_uint_t max_file_size;

228 +// kernel_uint_t max_data_size;

229 +// kernel_uint_t max_stack_size;

230 +// kernel_uint_t max_core_file_size;

231 +// kernel_uint_t max_resident_set;

232 +// kernel_uint_t max_processes;

233 + kernel_uint_t max_open_files;

234 +// kernel_uint_t max_locked_memory;

235 +// kernel_uint_t max_address_space;

236 +// kernel_uint_t max_file_locks;

237 +// kernel_uint_t max_pending_signals;

238 +// kernel_uint_t max_msgqueue_size;

239 +// kernel_uint_t max_nice_priority;

240 +// kernel_uint_t max_realtime_priority;

241 +// kernel_uint_t max_realtime_timeout;

242 +};

243 +

244 // ----------------------------------------------------------------------------

245 // target

246 //

@@ -268,6 +298,8 @@ struct target {

298

299 struct openfds openfds;

300

301 + NETDATA_DOUBLE max_open_files_percent;

302 +

303 kernel_uint_t starttime;

304 kernel_uint_t collected_starttime;

305 kernel_uint_t uptime_min;

@@ -318,12 +350,6 @@ struct pid_fd {

350

351 struct pid_stat {

352 int32_t pid;

321 - char comm[MAX_COMPARE_NAME + 1];

322 - char *cmdline;

323 -

324 - uint32_t log_thrown;

325 -

326 - char state;

353 int32_t ppid;

354 // int32_t pgrp;

355 // int32_t session;

@@ -331,6 +357,11 @@ struct pid_stat {

357 // int32_t tpgid;

358 // uint64_t flags;

359

360 + char state;

361 +

362 + char comm[MAX_COMPARE_NAME + 1];

363 + char *cmdline;

364 +

365 // these are raw values collected

366 kernel_uint_t minflt_raw;

367 kernel_uint_t cminflt_raw;

@@ -415,22 +446,28 @@ struct pid_stat {

446 kernel_uint_t io_storage_bytes_written;

447 kernel_uint_t io_cancelled_write_bytes;

448

449 + kernel_uint_t uptime;

450 +

451 struct pid_fd *fds; // array of fds it uses

452 size_t fds_size; // the size of the fds array

453

454 struct openfds openfds;

455 + struct pid_limits limits;

456 +

457 + int sortlist; // higher numbers = top on the process tree

458 + // each process gets a unique number

459

460 int children_count; // number of processes directly referencing this

424 - unsigned char keep:1; // 1 when we need to keep this process in memory even after it exited

461 int keeploops; // increases by 1 every time keep is 1 and updated 0

426 - unsigned char updated:1; // 1 when the process is currently running

427 - unsigned char merged:1; // 1 when it has been merged to its parent

428 - unsigned char read:1; // 1 when we have already read this process for this iteration

462

430 - int sortlist; // higher numbers = top on the process tree

431 - // each process gets a unique number

463 + PID_LOG log_thrown;

464

465 + bool keep; // true when we need to keep this process in memory even after it exited

466 + bool updated; // true when the process is currently running

467 + bool merged; // true when it has been merged to its parent

468 + bool read; // true when we have already read this process for this iteration

469 bool matched_by_config;

470 +

471 struct target *target; // app_groups.conf targets

472 struct target *user_target; // uid based targets

473 struct target *group_target; // gid based targets

@@ -440,8 +477,7 @@ struct pid_stat {

477

478 usec_t io_collected_usec;

479 usec_t last_io_collected_usec;

443 -

444 - kernel_uint_t uptime;

480 + usec_t last_limits_collected_usec;

481

482 char *fds_dirname; // the full directory name in /proc/PID/fd

483

@@ -449,6 +485,7 @@ struct pid_stat {

485 char *status_filename;

486 char *io_filename;

487 char *cmdline_filename;

488 + char *limits_filename;

489

490 struct pid_stat *parent;

491 struct pid_stat *prev;

@@ -459,14 +496,6 @@ size_t pagesize;

496

497 kernel_uint_t global_uptime;

498

462 -// log each problem once per process

463 -// log flood protection flags (log_thrown)

464 -#define PID_LOG_IO 0x00000001

465 -#define PID_LOG_STATUS 0x00000002

466 -#define PID_LOG_CMDLINE 0x00000004

467 -#define PID_LOG_FDS 0x00000008

468 -#define PID_LOG_STAT 0x00000010

469 -

499 static struct pid_stat

500 *root_of_pids = NULL, // global list of all processes running

501 **all_pids = NULL; // to avoid allocations, we pre-allocate

@@ -1019,6 +1048,7 @@ static inline void del_pid_entry(pid_t pid) {

1048 freez(p->fds_dirname);

1049 freez(p->stat_filename);

1050 freez(p->status_filename);

1051 + freez(p->limits_filename);

1052 #ifndef __FreeBSD__

1053 arl_free(p->status_arl);

1054 #endif

@@ -1033,7 +1063,7 @@ static inline void del_pid_entry(pid_t pid) {

1063

1064 // ----------------------------------------------------------------------------

1065

1036 -static inline int managed_log(struct pid_stat *p, uint32_t log, int status) {

1066 +static inline int managed_log(struct pid_stat *p, PID_LOG log, int status) {

1067 if(unlikely(!status)) {

1068 // netdata_log_error("command failed log %u, errno %d", log, errno);

1069

@@ -1073,6 +1103,13 @@ static inline int managed_log(struct pid_stat *p, uint32_t log, int status) {

1103 #endif

1104 break;

1105

1106 + case PID_LOG_LIMITS:

1107 + #ifdef __FreeBSD__

1108 + ;

1109 + #else

1110 + netdata_log_error("Cannot process %s/proc/%d/limits (command '%s')", netdata_configured_host_prefix, p->pid, p->comm);

1111 + #endif

1112 +

1113 case PID_LOG_STAT:

1114 break;

1115

@@ -1314,6 +1351,54 @@ static void update_proc_state_count(char proc_state) {

1351 }

1352 #endif // !__FreeBSD__

1353

1354 +#define MAX_PROC_PID_LIMITS 8192

1355 +#define PROC_PID_LIMITS_MAX_OPEN_FILES_KEY "\nMax open files "

1356 +

1357 +static inline kernel_uint_t get_proc_pid_limits_limit(char *buf, const char *key, size_t key_len, kernel_uint_t def) {

1358 + char *line = strstr(buf, key);

1359 + if(!line)

1360 + return def;

1361 +

1362 + char *v = &line[key_len];

1363 + while(isspace(*v)) v++;

1364 +

1365 + return str2ull(v, NULL);

1366 +}

1367 +

1368 +static inline int read_proc_pid_limits(struct pid_stat *p, void *ptr) {

1369 + (void)ptr;

1370 +

1371 +#ifdef __FreeBSD__

1372 + return 0;

1373 +#else

1374 + static char proc_pid_limits[MAX_PROC_PID_LIMITS + 1];

1375 +

1376 + if(p->io_collected_usec > p->last_limits_collected_usec && p->io_collected_usec - p->last_limits_collected_usec <= 60 * USEC_PER_SEC)

1377 + // too frequent, we want to collect limits once per minute

1378 + return 0;

1379 +

1380 + if(unlikely(!p->limits_filename)) {

1381 + char filename[FILENAME_MAX + 1];

1382 + snprintfz(filename, FILENAME_MAX, "%s/proc/%d/limits", netdata_configured_host_prefix, p->pid);

1383 + p->limits_filename = strdupz(filename);

1384 + }

1385 +

1386 + int fd = open(p->limits_filename, procfile_open_flags, 0666);

1387 + if(unlikely(fd == -1)) return 0;

1388 +

1389 + ssize_t bytes = read(fd, proc_pid_limits, MAX_PROC_PID_LIMITS);

1390 + close(fd);

1391 +

1392 + if(bytes <= 0)

1393 + return 0;

1394 +

1395 + p->limits.max_open_files = get_proc_pid_limits_limit(proc_pid_limits, PROC_PID_LIMITS_MAX_OPEN_FILES_KEY, sizeof(PROC_PID_LIMITS_MAX_OPEN_FILES_KEY) - 1, 0);

1396 + p->last_limits_collected_usec = p->io_collected_usec;

1397 +

1398 + return 1;

1399 +#endif

1400 +}

1401 +

1402 static inline int read_proc_pid_status(struct pid_stat *p, void *ptr) {

1403 p->status_vmsize = 0;

1404 p->status_vmrss = 0;

@@ -2460,7 +2545,7 @@ static inline void process_exited_processes() {

2545 if(majflt) debug_find_lost_child(p, majflt, 2);

2546 }

2547

2463 - p->keep = 1;

2548 + p->keep = true;

2549

2550 debug_log(" > remaining resources - KEEP - for another loop: %s (%d %s total resources: utime=" KERNEL_UINT_FORMAT " stime=" KERNEL_UINT_FORMAT " gtime=" KERNEL_UINT_FORMAT " minflt=" KERNEL_UINT_FORMAT " majflt=" KERNEL_UINT_FORMAT ")"

2551 , p->comm

@@ -2475,7 +2560,7 @@ static inline void process_exited_processes() {

2560

2561 for(pp = p->parent; pp ; pp = pp->parent) {

2562 if(pp->updated) break;

2478 - pp->keep = 1;

2563 + pp->keep = true;

2564

2565 debug_log(" > - KEEP - parent for another loop: %s (%d %s)"

2566 , pp->comm

@@ -2573,7 +2658,7 @@ static inline int collect_data_for_pid(pid_t pid, void *ptr) {

2658

2659 struct pid_stat *p = get_pid_entry(pid);

2660 if(unlikely(!p || p->read)) return 0;

2576 - p->read = 1;

2661 + p->read = true;

2662

2663 // debug_log("Reading process %d (%s), sortlist %d", p->pid, p->comm, p->sortlist);

2664

@@ -2605,8 +2690,10 @@ static inline int collect_data_for_pid(pid_t pid, void *ptr) {

2690 // --------------------------------------------------------------------

2691 // /proc/<pid>/fd

2692

2608 - if(enable_file_charts)

2609 - managed_log(p, PID_LOG_FDS, read_pid_file_descriptors(p, ptr));

2693 + if(enable_file_charts) {

2694 + managed_log(p, PID_LOG_FDS, read_pid_file_descriptors(p, ptr));

2695 + managed_log(p, PID_LOG_LIMITS, read_proc_pid_limits(p, ptr));

2696 + }

2697

2698 // --------------------------------------------------------------------

2699 // done!

@@ -2615,8 +2702,8 @@ static inline int collect_data_for_pid(pid_t pid, void *ptr) {

2702 debug_log("Read process %d (%s) sortlisted %d, but its parent %d (%s) sortlisted %d, is not read", p->pid, p->comm, p->sortlist, all_pids[p->ppid]->pid, all_pids[p->ppid]->comm, all_pids[p->ppid]->sortlist);

2703

2704 // mark it as updated

2618 - p->updated = 1;

2619 - p->keep = 0;

2705 + p->updated = true;

2706 + p->keep = false;

2707 p->keeploops = 0;

2708

2709 return 1;

@@ -2673,9 +2760,9 @@ static int collect_data_for_all_processes(void) {

2760 size_t slc = 0;

2761 #endif

2762 for(p = root_of_pids; p ; p = p->next) {

2676 - p->read = 0; // mark it as not read, so that collect_data_for_pid() will read it

2677 - p->updated = 0;

2678 - p->merged = 0;

2763 + p->read = false; // mark it as not read, so that collect_data_for_pid() will read it

2764 + p->updated = false;

2765 + p->merged = false;

2766 p->children_count = 0;

2767 p->parent = NULL;

2768

@@ -2801,7 +2888,7 @@ static void cleanup_exited_pids(void) {

2888 }

2889 else {

2890 if(unlikely(p->keep)) p->keeploops++;

2804 - p->keep = 0;

2891 + p->keep = false;

2892 p = p->next;

2893 }

2894 }

@@ -2855,7 +2942,7 @@ static void apply_apps_groups_targets_inheritance(void) {

2942 )) {

2943 // mark it as merged

2944 p->parent->children_count--;

2858 - p->merged = 1;

2945 + p->merged = true;

2946

2947 // the parent inherits the child's target, if it does not have a target itself

2948 if(unlikely(p->target && !p->parent->target)) {

@@ -2965,6 +3052,8 @@ static size_t zero_all_targets(struct target *root) {

3052 w->openfds.signalfds = 0;

3053 w->openfds.eventpolls = 0;

3054 w->openfds.other = 0;

3055 +

3056 + w->max_open_files_percent = 0.0;

3057 }

3058

3059 w->collected_starttime = 0;

@@ -3114,6 +3203,23 @@ static inline void aggregate_pid_on_target(struct target *w, struct pid_stat *p,

3203 return;

3204 }

3205

3206 + if(p->limits.max_open_files > 0) {

3207 + kernel_uint_t all_dfs =

3208 + p->openfds.files +

3209 + p->openfds.pipes +

3210 + p->openfds.sockets +

3211 + p->openfds.inotifies +

3212 + p->openfds.eventfds +

3213 + p->openfds.timerfds +

3214 + p->openfds.signalfds +

3215 + p->openfds.eventpolls +

3216 + p->openfds.other;

3217 +

3218 + NETDATA_DOUBLE percent = (NETDATA_DOUBLE)all_dfs * 100.0 / (NETDATA_DOUBLE)p->limits.max_open_files;

3219 + if(percent > w->max_open_files_percent)

3220 + w->max_open_files_percent = percent;

3221 + }

3222 +

3223 w->cutime += p->cutime;

3224 w->cstime += p->cstime;

3225 w->cgtime += p->cgtime;

@@ -3742,6 +3848,14 @@ static void send_collected_data_to_netdata(struct target *root, const char *type

3848 send_SET(w->name, w->openfds.pipes);

3849 }

3850 send_END();

3851 +

3852 + send_BEGIN(type, "fd_limit", dt);

3853 + for (w = root; w; w = w->next) {

3854 + if (unlikely(w->exposed && w->processes))

3855 + send_SET(w->name, w->max_open_files_percent * 100.0);

3856 + }

3857 + send_END();

3858 +

3859 }

3860 }

3861

@@ -3987,6 +4101,14 @@ static void send_charts_updates_to_netdata(struct target *root, const char *type

4101 fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);

4102 }

4103 APPS_PLUGIN_FUNCTIONS();

4104 +

4105 + fprintf(stdout, "CHART %s.fd_limit '' '%s File Descriptors Limit' '%%' processes %s.fd_limit line 20054 %d\n", type,

4106 + title, type, update_every);

4107 + for (w = root; w; w = w->next) {

4108 + if (unlikely(w->exposed))

4109 + fprintf(stdout, "DIMENSION %s '' absolute 1 100\n", w->name);

4110 + }

4111 + APPS_PLUGIN_FUNCTIONS();

4112 }

4113 }

4114

collectors/proc.plugin/plugin_proc.c

@@ -18,6 +18,7 @@ static struct proc_module {

18 {.name = "/proc/stat", .dim = "stat", .func = do_proc_stat},

19 {.name = "/proc/uptime", .dim = "uptime", .func = do_proc_uptime},

20 {.name = "/proc/loadavg", .dim = "loadavg", .func = do_proc_loadavg},

21 + {.name = "/proc/sys/fs/file-nr", .dim = "file-nr", .func = do_proc_sys_fs_file_nr},

22 {.name = "/proc/sys/kernel/random/entropy_avail", .dim = "entropy", .func = do_proc_sys_kernel_random_entropy_avail},

24 // pressure metrics

collectors/proc.plugin/plugin_proc.h

@@ -22,6 +22,7 @@ int do_proc_meminfo(int update_every, usec_t dt);

22 int do_proc_vmstat(int update_every, usec_t dt);

23 int do_proc_net_rpc_nfs(int update_every, usec_t dt);

24 int do_proc_net_rpc_nfsd(int update_every, usec_t dt);

25 +int do_proc_sys_fs_file_nr(int update_every, usec_t dt);

26 int do_proc_sys_kernel_random_entropy_avail(int update_every, usec_t dt);

27 int do_proc_interrupts(int update_every, usec_t dt);

28 int do_proc_softirqs(int update_every, usec_t dt);

collectors/proc.plugin/proc_sys_fs_file_nr.c new

+81

@@ -0,0 +1,81 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "plugin_proc.h"

4 +

5 +int do_proc_sys_fs_file_nr(int update_every, usec_t dt) {

6 + (void)dt;

7 +

8 + static procfile *ff = NULL;

9 +

10 + if(unlikely(!ff)) {

11 + char filename[FILENAME_MAX + 1];

12 + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/sys/fs/file-nr");

13 + ff = procfile_open(config_get("plugin:proc:/proc/sys/fs/file-nr", "filename to monitor", filename), "", PROCFILE_FLAG_DEFAULT);

14 + if(unlikely(!ff)) return 1;

15 + }

16 +

17 + ff = procfile_readall(ff);

18 + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time

19 +

20 + uint64_t allocated = str2ull(procfile_lineword(ff, 0, 0), NULL);

21 + uint64_t unused = str2ull(procfile_lineword(ff, 0, 1), NULL);

22 + uint64_t max = str2ull(procfile_lineword(ff, 0, 2), NULL);

23 +

24 + uint64_t used = allocated - unused;

25 +

26 + static RRDSET *st_files = NULL;

27 + static RRDDIM *rd_used = NULL;

28 +

29 + if(unlikely(!st_files)) {

30 + st_files = rrdset_create_localhost(

31 + "system"

32 + , "file_nr_used"

33 + , NULL

34 + , "files"

35 + , NULL

36 + , "File Descriptors"

37 + , "files"

38 + , PLUGIN_PROC_NAME

39 + , "/proc/sys/fs/file-nr"

40 + , NETDATA_CHART_PRIO_SYSTEM_FILES_NR

41 + , update_every

42 + , RRDSET_TYPE_LINE

43 + );

44 +

45 + rd_used = rrddim_add(st_files, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

46 + }

47 +

48 + rrddim_set_by_pointer(st_files, rd_used, (collected_number )used);

49 + rrdset_done(st_files);

50 +

51 + static RRDSET *st_files_utilization = NULL;

52 + static RRDDIM *rd_utilization = NULL;

53 +

54 + if(unlikely(!st_files_utilization)) {

55 + st_files_utilization = rrdset_create_localhost(

56 + "system"

57 + , "file_nr_utilization"

58 + , NULL

59 + , "files"

60 + , NULL

61 + , "File Descriptors Utilization"

62 + , "percentage"

63 + , PLUGIN_PROC_NAME

64 + , "/proc/sys/fs/file-nr"

65 + , NETDATA_CHART_PRIO_SYSTEM_FILES_NR + 1

66 + , update_every

67 + , RRDSET_TYPE_LINE

68 + );

69 +

70 + rd_utilization = rrddim_add(st_files_utilization, "utilization", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);

71 + }

72 +

73 + NETDATA_DOUBLE d_used = (NETDATA_DOUBLE)used;

74 + NETDATA_DOUBLE d_max = (NETDATA_DOUBLE)max;

75 + NETDATA_DOUBLE percent = d_used * 100.0 / d_max;

76 +

77 + rrddim_set_by_pointer(st_files_utilization, rd_utilization, (collected_number)(percent * 10000));

78 + rrdset_done(st_files_utilization);

79 +

80 + return 0;

81 +}

health/Makefile.am

@@ -44,6 +44,7 @@ dist_healthconfig_DATA = \

44 health.d/elasticsearch.conf \

45 health.d/entropy.conf \

46 health.d/exporting.conf \

47 + health.d/file_descriptors.conf \

48 health.d/geth.conf \

49 health.d/ioping.conf \

50 health.d/gearman.conf \

health/health.d/file_descriptors.conf new

+30

@@ -0,0 +1,30 @@

1 + # you can disable an alarm notification by setting the 'to' line to: silent

2 +

3 + template: system_file_descriptors_utilization

4 + on: system.file_nr_utilization

5 + class: Utilization

6 + type: System

7 + component: Processes

8 + hosts: *

9 + lookup: max -1m unaligned

10 + units: %

11 + every: 1m

12 + crit: $this > 90

13 + delay: down 15m multiplier 1.5 max 1h

14 + info: system-wide utilization of open files

15 + to: sysadmin

16 +

17 + template: apps_group_file_descriptors_utilization

18 + on: apps.fd_limit

19 + class: Utilization

20 + type: System

21 +component: Process

22 + os: linux

23 + hosts: *

24 + lookup: max -1m unaligned foreach *

25 + units: %

26 + every: 1m

27 + warn: $this > (($status >= $WARNING) ? (85) : (90))

28 + delay: down 15m multiplier 1.5 max 1h

29 + info: maximum utilization of open files among all application group PIDs

30 + to: sysadmin