error_limit() function to limit number of error lines per instance (#13924)
* error_limit() function to limit number of error lines per instance * count should be more than 1 * protect settings ERROR_LIMIT members
Costa Tsaousis committed
Nov 1, 2022 at 20:31 UTC
cef305c5e111cac7276a3c36e59dc84911e6d509
3 files changed
+78
-3
libnetdata/log/log.c
+63
-1
@@ -818,7 +818,69 @@ static const char *strerror_result_string(const char *a, const char *b) { (void)
818
#error "cannot detect the format of function strerror_r()"
819
#endif
820
821
-void error_int( const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
821
+void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
822
+ if(erl->sleep_ut)
823
+ sleep_usec(erl->sleep_ut);
824
+
825
+ // save a copy of errno - just in case this function generates a new error
826
+ int __errno = errno;
827
+
828
+ va_list args;
829
+
830
+ log_lock();
831
+
832
+ erl->count++;
833
+ time_t now = now_boottime_sec();
834
+ if(now - erl->last_logged < erl->log_every) {
835
+ log_unlock();
836
+ return;
837
+ }
838
+
839
+ // prevent logging too much
840
+ if (error_log_limit(0)) {
841
+ log_unlock();
842
+ return;
843
+ }
844
+
845
+ if(error_log_syslog) {
846
+ va_start( args, fmt );
847
+ vsyslog(LOG_ERR, fmt, args );
848
+ va_end( args );
849
+ }
850
+
851
+ char date[LOG_DATE_LENGTH];
852
+ log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
853
+
854
+ va_start( args, fmt );
855
+#ifdef NETDATA_INTERNAL_CHECKS
856
+ fprintf(stderr, "%s: %s %-5.5s : %s : (%04lu@%-20.20s:%-15.15s): ", date, program_name, prefix, netdata_thread_tag(), line, file, function);
857
+#else
858
+ fprintf(stderr, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag());
859
+#endif
860
+ vfprintf( stderr, fmt, args );
861
+ va_end( args );
862
+
863
+ if(erl->count > 1)
864
+ fprintf(stderr, " (repeated %zu times in the last %llu secs)", erl->count, (unsigned long long)(erl->last_logged ? now - erl->last_logged : 0));
865
+
866
+ if(erl->sleep_ut)
867
+ fprintf(stderr, " (sleeping for %llu microseconds every time this happens)", erl->sleep_ut);
868
+
869
+ if(__errno) {
870
+ char buf[1024];
871
+ fprintf(stderr, " (errno %d, %s)\n", __errno, strerror_result(strerror_r(__errno, buf, 1023), buf));
872
+ errno = 0;
873
+ }
874
+ else
875
+ fputc('\n', stderr);
876
+
877
+ erl->last_logged = now;
878
+ erl->count = 0;
879
+
880
+ log_unlock();
881
+}
882
+
883
+void error_int(const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
884
// save a copy of errno - just in case this function generates a new error
885
int __errno = errno;
886
libnetdata/log/log.h
+12
@@ -94,6 +94,16 @@ static inline void debug_dummy(void) {}
94
void error_log_limit_reset(void);
95
void error_log_limit_unlimited(void);
96
97
+typedef struct error_with_limit {
98
+ time_t log_every;
99
+ size_t count;
100
+ time_t last_logged;
101
+ usec_t sleep_ut;
102
+} ERROR_LIMIT;
103
+
104
+#define error_limit_static_global_var(var, log_every_secs, sleep_usecs) static ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) }
105
+#define error_limit_static_thread_var(var, log_every_secs, sleep_usecs) static __thread ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) }
106
+
107
#ifdef NETDATA_INTERNAL_CHECKS
108
#define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
109
#define internal_error(condition, args...) do { if(unlikely(condition)) error_int("IERR", __FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
@@ -107,6 +117,7 @@ void error_log_limit_unlimited(void);
117
#define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args)
118
#define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args)
119
#define error(args...) error_int("ERROR", __FILE__, __FUNCTION__, __LINE__, ##args)
120
+#define error_limit(erl, args...) error_limit_int(erl, "ERROR", __FILE__, __FUNCTION__, __LINE__, ##args)
121
#define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args)
122
#define fatal_assert(expr) ((expr) ? (void)(0) : fatal_int(__FILE__, __FUNCTION__, __LINE__, "Assertion `%s' failed", #expr))
123
@@ -114,6 +125,7 @@ void send_statistics(const char *action, const char *action_result, const char *
125
void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5);
126
void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5);
127
void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6);
128
+void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, unsigned long line __maybe_unused, const char *fmt, ... ) PRINTFLIKE(6, 7);;
129
void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5);
130
void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2);
131
void log_health( const char *fmt, ... ) PRINTFLIKE(1, 2);
libnetdata/socket/socket.c
+3
-2
@@ -1547,8 +1547,9 @@ static int poll_process_new_tcp_connection(POLLJOB *p, POLLINFO *pi, struct poll
1547
debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", pi->slot, pf->fd);
1548
1549
if(unlikely(errno == EMFILE)) {
1550
- error("POLLFD: LISTENER: too many open files - sleeping for 1ms - used by this thread %zu, max for this thread %zu", p->used, p->limit);
1551
- usleep(1000); // 1ms
1550
+ error_limit_static_global_var(erl, 10, 1000);
1551
+ error_limit(&erl, "POLLFD: LISTENER: too many open files - used by this thread %zu, max for this thread %zu",
1552
+ p->used, p->limit);
1553
}
1554
else if(unlikely(errno != EWOULDBLOCK && errno != EAGAIN))
1555
error("POLLFD: LISTENER: accept() failed.");