apps.plugin fds limits improvements (#15467)
prevent wrong log about limits; log when the percentage of open fds is above limits
Costa Tsaousis committed
Jul 20, 2023 at 23:27 UTC
cc2de625739ef318361d90ed4ffa72a0a235405f
1 file changed
+51
-7
collectors/apps.plugin/apps_plugin.c
+51
-7
@@ -144,12 +144,13 @@ static const char *proc_states[] = {
144
// log each problem once per process
145
// log flood protection flags (log_thrown)
146
typedef enum __attribute__((packed)) {
147
- PID_LOG_IO = (1 << 0),
148
- PID_LOG_STATUS = (1 << 1),
149
- PID_LOG_CMDLINE = (1 << 2),
150
- PID_LOG_FDS = (1 << 3),
151
- PID_LOG_STAT = (1 << 4),
152
- PID_LOG_LIMITS = (1 << 5),
147
+ PID_LOG_IO = (1 << 0),
148
+ PID_LOG_STATUS = (1 << 1),
149
+ PID_LOG_CMDLINE = (1 << 2),
150
+ PID_LOG_FDS = (1 << 3),
151
+ PID_LOG_STAT = (1 << 4),
152
+ PID_LOG_LIMITS = (1 << 5),
153
+ PID_LOG_LIMITS_DETAIL = (1 << 6),
154
} PID_LOG;
155
156
static int
@@ -1373,11 +1374,16 @@ static inline int read_proc_pid_limits(struct pid_stat *p, void *ptr) {
1374
#else
1375
static char proc_pid_limits_buffer[MAX_PROC_PID_LIMITS + 1];
1376
int ret = 0;
1377
+ bool read_limits = false;
1378
+
1379
+ errno = 0;
1380
1381
kernel_uint_t all_fds = pid_openfds_sum(p);
1378
- if(all_fds < p->limits.max_open_files / 2 && p->io_collected_usec > p->last_limits_collected_usec && p->io_collected_usec - p->last_limits_collected_usec <= 60 * USEC_PER_SEC)
1382
+ if(all_fds < p->limits.max_open_files / 2 && p->io_collected_usec > p->last_limits_collected_usec && p->io_collected_usec - p->last_limits_collected_usec <= 60 * USEC_PER_SEC) {
1383
// too frequent, we want to collect limits once per minute
1384
+ ret = 1;
1385
goto cleanup;
1386
+ }
1387
1388
if(unlikely(!p->limits_filename)) {
1389
char filename[FILENAME_MAX + 1];
@@ -1394,6 +1400,7 @@ static inline int read_proc_pid_limits(struct pid_stat *p, void *ptr) {
1400
if(bytes <= 0)
1401
goto cleanup;
1402
1403
+ read_limits = true;
1404
p->limits.max_open_files = get_proc_pid_limits_limit(proc_pid_limits_buffer, PROC_PID_LIMITS_MAX_OPEN_FILES_KEY, sizeof(PROC_PID_LIMITS_MAX_OPEN_FILES_KEY) - 1, 0);
1405
p->last_limits_collected_usec = p->io_collected_usec;
1406
@@ -1405,6 +1412,43 @@ cleanup:
1412
else
1413
p->openfds_limits_percent = 0.0;
1414
1415
+ if(p->openfds_limits_percent > 100.0) {
1416
+ if(!(p->log_thrown & PID_LOG_LIMITS_DETAIL)) {
1417
+ netdata_log_info(
1418
+ "FDS_LIMITS: PID %d (%s) is using "
1419
+ "%0.2d %% of its fds limits, "
1420
+ "open fds = %llu ("
1421
+ "files = %llu, "
1422
+ "pipes = %llu, "
1423
+ "sockets = %llu, "
1424
+ "inotifies = %llu, "
1425
+ "eventfds = %llu, "
1426
+ "timerfds = %llu, "
1427
+ "signalfds = %llu, "
1428
+ "eventpolls = %llu "
1429
+ "other = %llu "
1430
+ "), open fds limit = %llu, "
1431
+ "%s",
1432
+ p->pid, p->comm, p->openfds_limits_percent, all_fds,
1433
+ p->openfds.files,
1434
+ p->openfds.pipes,
1435
+ p->openfds.sockets,
1436
+ p->openfds.inotifies,
1437
+ p->openfds.eventfds,
1438
+ p->openfds.timerfds,
1439
+ p->openfds.signalfds,
1440
+ p->openfds.eventpolls,
1441
+ p->openfds.other,
1442
+ p->limits.max_open_files,
1443
+ read_limits ? "and we have read the limits AFTER counting the fds"
1444
+ : "but we have read the limits BEFORE counting the fds");
1445
+
1446
+ p->log_thrown |= PID_LOG_LIMITS_DETAIL;
1447
+ }
1448
+ }
1449
+ else
1450
+ p->log_thrown &= ~PID_LOG_LIMITS_DETAIL;
1451
+
1452
return ret;
1453
#endif
1454
}