added log flood protection #240 to the netdata daemon and apps.plugin
Costa Tsaousis (ktsaou) committed
Apr 17, 2016 at 17:34 UTC
72e8ff37cc4171a3d2c0473cfcbf6621e8de10b4
7 files changed
+118
-16
src/apps_plugin.c
-14
@@ -1299,8 +1299,6 @@ int read_pid_file_descriptors(struct pid_stat *p) {
1299
1300
int collect_data_for_all_processes_from_proc(void)
1301
{
1302
- static long count_errors = 0;
1303
-
1302
char dirname[FILENAME_MAX + 1];
1303
1304
snprintf(dirname, FILENAME_MAX, "%s/proc", host_prefix);
@@ -1354,7 +1352,6 @@ int collect_data_for_all_processes_from_proc(void)
1352
// /proc/<pid>/stat
1353
1354
if(unlikely(read_proc_pid_stat(p))) {
1357
- if(!count_errors++ || debug || (p->target && p->target->debug))
1355
error("Cannot process %s/proc/%d/stat", host_prefix, pid);
1356
1357
// there is no reason to proceed if we cannot get its status
@@ -1363,7 +1360,6 @@ int collect_data_for_all_processes_from_proc(void)
1360
1361
// check its parent pid
1362
if(unlikely(p->ppid < 0 || p->ppid > pid_max)) {
1366
- if(unlikely(!count_errors++ || debug || (p->target && p->target->debug)))
1363
error("Pid %d states invalid parent pid %d. Using 0.", pid, p->ppid);
1364
1365
p->ppid = 0;
@@ -1374,7 +1370,6 @@ int collect_data_for_all_processes_from_proc(void)
1370
1371
if(proc_pid_cmdline_is_needed) {
1372
if(unlikely(read_proc_pid_cmdline(p))) {
1377
- if(!count_errors++ || debug || (p->target && p->target->debug))
1373
error("Cannot process %s/proc/%d/cmdline", host_prefix, pid);
1374
}
1375
}
@@ -1383,7 +1378,6 @@ int collect_data_for_all_processes_from_proc(void)
1378
// /proc/<pid>/statm
1379
1380
if(unlikely(read_proc_pid_statm(p))) {
1386
- if(unlikely(!count_errors++ || debug || (p->target && p->target->debug)))
1381
error("Cannot process %s/proc/%d/statm", host_prefix, pid);
1382
1383
// there is no reason to proceed if we cannot get its memory status
@@ -1395,7 +1389,6 @@ int collect_data_for_all_processes_from_proc(void)
1389
// /proc/<pid>/io
1390
1391
if(unlikely(read_proc_pid_io(p))) {
1398
- if(unlikely(!count_errors++ || debug || (p->target && p->target->debug)))
1392
error("Cannot process %s/proc/%d/io", host_prefix, pid);
1393
1394
// on systems without /proc/X/io
@@ -1407,7 +1400,6 @@ int collect_data_for_all_processes_from_proc(void)
1400
// <pid> ownership
1401
1402
if(unlikely(read_proc_pid_ownership(p))) {
1410
- if(unlikely(!count_errors++ || debug || (p->target && p->target->debug)))
1403
error("Cannot stat %s/proc/%d", host_prefix, pid);
1404
}
1405
@@ -1448,7 +1440,6 @@ int collect_data_for_all_processes_from_proc(void)
1440
// /proc/<pid>/fd
1441
1442
if(unlikely(read_pid_file_descriptors(p))) {
1451
- if(unlikely(!count_errors++ || debug || (p->target && p->target->debug)))
1443
error("Cannot process entries in %s/proc/%d/fd", host_prefix, pid);
1444
}
1445
@@ -1459,11 +1450,6 @@ int collect_data_for_all_processes_from_proc(void)
1450
p->updated = 1;
1451
}
1452
1462
- if(unlikely(count_errors > 1000)) {
1463
- error("%ld more errors encountered\n", count_errors - 1);
1464
- count_errors = 0;
1465
- }
1466
-
1453
closedir(dir);
1454
1455
return 1;
src/log.c
+85
@@ -27,6 +27,85 @@ int access_log_syslog = 1;
27
int error_log_syslog = 1;
28
int output_log_syslog = 1; // debug log
29
30
+time_t error_log_throttle_period = 1200;
31
+unsigned long error_log_errors_per_period = 200;
32
+
33
+int error_log_limit(int reset) {
34
+ static time_t start = 0;
35
+ static unsigned long counter = 0, prevented = 0;
36
+
37
+ // do not throttle if the period is 0
38
+ if(error_log_throttle_period == 0)
39
+ return 0;
40
+
41
+ // prevent all logs if the errors per period is 0
42
+ if(error_log_errors_per_period == 0)
43
+ return 1;
44
+
45
+ time_t now = time(NULL);
46
+ if(!start) start = now;
47
+
48
+ if(reset) {
49
+ if(prevented) {
50
+ log_date(stderr);
51
+ fprintf(stderr, "%s: Resetting logging for process '%s' (prevented %lu logs in the last %ld seconds).\n"
52
+ , program_name
53
+ , program_name
54
+ , prevented
55
+ , now - start
56
+ );
57
+ }
58
+
59
+ start = now;
60
+ counter = 0;
61
+ prevented = 0;
62
+ }
63
+
64
+ // detect if we log too much
65
+ counter++;
66
+
67
+ if(now - start > error_log_throttle_period) {
68
+ if(prevented) {
69
+ log_date(stderr);
70
+ fprintf(stderr, "%s: Resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n"
71
+ , program_name
72
+ , program_name
73
+ , prevented
74
+ , error_log_throttle_period
75
+ );
76
+ }
77
+
78
+ // restart the period accounting
79
+ start = now;
80
+ counter = 1;
81
+ prevented = 0;
82
+
83
+ // log this error
84
+ return 0;
85
+ }
86
+
87
+ if(counter > error_log_errors_per_period) {
88
+ if(!prevented) {
89
+ log_date(stderr);
90
+ fprintf(stderr, "%s: Too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n"
91
+ , program_name
92
+ , counter
93
+ , now - start
94
+ , error_log_errors_per_period
95
+ , error_log_throttle_period
96
+ , program_name
97
+ , start + error_log_throttle_period - now);
98
+ }
99
+
100
+ prevented++;
101
+
102
+ // prevent logging this error
103
+ return 1;
104
+ }
105
+
106
+ return 0;
107
+}
108
+
109
void log_date(FILE *out)
110
{
111
char outstr[200];
@@ -64,6 +143,9 @@ void info_int( const char *file, const char *function, const unsigned long line,
143
{
144
va_list args;
145
146
+ // prevent logging too much
147
+ if(error_log_limit(0)) return;
148
+
149
log_date(stderr);
150
151
va_start( args, fmt );
@@ -85,6 +167,9 @@ void error_int( const char *prefix, const char *file, const char *function, cons
167
{
168
va_list args;
169
170
+ // prevent logging too much
171
+ if(error_log_limit(0)) return;
172
+
173
log_date(stderr);
174
175
va_start( args, fmt );
src/log.h
+6
@@ -42,6 +42,12 @@ extern int access_log_syslog;
42
extern int error_log_syslog;
43
extern int output_log_syslog;
44
45
+extern time_t error_log_throttle_period;
46
+extern unsigned long error_log_errors_per_period;
47
+extern int error_log_limit(int reset);
48
+
49
+#define error_log_limit_reset() do { error_log_limit(1); } while(0)
50
+
51
#define debug(type, args...) do { if(unlikely(!silent && (debug_flags & type))) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
52
#define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args)
53
#define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args)
src/main.c
+10
@@ -36,6 +36,7 @@
36
#include "plugin_nfacct.h"
37
38
#include "main.h"
39
+#include "../config.h"
40
41
int netdata_exit = 0;
42
@@ -45,6 +46,9 @@ void netdata_cleanup_and_exit(int ret)
46
rrdset_save_all();
47
// kill_childs();
48
49
+ // let it log a few more error messages
50
+ error_log_limit_reset();
51
+
52
if(pidfd != -1) {
53
if(ftruncate(pidfd, 0) != 0)
54
error("Cannot truncate pidfile '%s'.", pidfile);
@@ -344,6 +348,12 @@ int main(int argc, char **argv)
348
}
349
else error_log_syslog = 0;
350
351
+ error_log_throttle_period = config_get_number("global", "errors throttle period", error_log_throttle_period);
352
+ setenv("NETDATA_ERRORS_THROTTLE_PERIOD", config_get("global", "errors throttle period" , ""), 1);
353
+
354
+ error_log_errors_per_period = config_get_number("global", "errors per throttle period", error_log_errors_per_period);
355
+ setenv("NETDATA_ERRORS_PER_PERIOD" , config_get("global", "errors per throttle period", ""), 1);
356
+
357
// --------------------------------------------------------------------
358
359
access_log_file = config_get("global", "access log", LOG_DIR "/access.log");
src/plugin_tc.c
+13
-2
@@ -15,6 +15,7 @@
15
#include "popen.h"
16
#include "plugin_tc.h"
17
#include "main.h"
18
+#include "../config.h"
19
20
#define RRD_TYPE_TC "tc"
21
#define RRD_TYPE_TC_LEN strlen(RRD_TYPE_TC)
@@ -717,7 +718,8 @@ void *tc_main(void *ptr)
718
// debug(D_TC_LOOP, "IGNORED line");
719
//}
720
}
720
- mypclose(fp, tc_child_pid);
721
+ // fgets() failed or loop broke
722
+ int code = mypclose(fp, tc_child_pid);
723
tc_child_pid = 0;
724
725
if(device) {
@@ -732,10 +734,19 @@ void *tc_main(void *ptr)
734
return NULL;
735
}
736
737
+ if(code == 1 || code == 127) {
738
+ // 1 = DISABLE
739
+ // 127 = cannot even run it
740
+ error("TC: tc-qos-helper.sh exited with code %d. Disabling it.", code);
741
+
742
+ tc_device_free_all();
743
+ pthread_exit(NULL);
744
+ return NULL;
745
+ }
746
+
747
sleep((unsigned int) rrd_update_every);
748
}
749
750
pthread_exit(NULL);
751
return NULL;
752
}
741
-
src/plugins_d.c
+1
@@ -16,6 +16,7 @@
16
#include "rrd.h"
17
#include "popen.h"
18
#include "plugins_d.h"
19
+#include "../config.h"
20
21
struct plugind *pluginsd_root = NULL;
22
src/rrd.c
+3
@@ -682,6 +682,9 @@ void rrdset_save_all(void)
682
{
683
debug(D_RRD_CALLS, "rrdset_save_all()");
684
685
+ // let it log a few error messages
686
+ error_log_limit_reset();
687
+
688
RRDSET *st;
689
RRDDIM *rd;
690