set default job retries to 60, implement penalty for failed jobs
lgz committed
Oct 22, 2017 at 16:51 UTC
9e79ae70a8b1e3bb8b9c09d8204f37d1462bba31
2 files changed
+9
-6
plugins.d/python.d.plugin
+1
-1
@@ -38,7 +38,7 @@ from bases.loggers import PythonDLogger
38
from bases.collection import setdefault_values, run_and_exit
39
40
BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
41
- 'retries': 20,
41
+ 'retries': 60,
42
'priority': 60000,
43
'autodetection_retry': 0,
44
'name': str()}
python.d/python_modules/bases/FrameworkServices/SimpleService.py
+8
-5
@@ -34,6 +34,7 @@ class RuntimeCounters:
34
self.ELAPSED = 0
35
self.RETRIES = 0
36
self.RETRIES_MAX = configuration.pop('retries')
37
+ self.PENALTY = 0
38
39
def is_sleep_time(self):
40
return self.START_RUN < self.NEXT_RUN
@@ -168,7 +169,7 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec
169
while True:
170
job.START_RUN = time()
171
171
- job.NEXT_RUN = job.START_RUN - (job.START_RUN % job.FREQ) + job.FREQ
172
+ job.NEXT_RUN = job.START_RUN - (job.START_RUN % job.FREQ) + job.FREQ + job.PENALTY
173
174
self.sleep_until_next_run()
175
@@ -185,7 +186,7 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec
186
current_time = time()
187
job.PREV_UPDATE = current_time
188
job.ELAPSED = int((current_time - job.START_RUN) * 1e3)
188
- job.RETRIES = 0
189
+ job.RETRIES, job.PENALTY = 0, 0
190
safe_print(RUNTIME_CHART_UPDATE.format(job_name=self.name,
191
since_last=job.SINCE_UPDATE,
192
elapsed=job.ELAPSED))
@@ -200,10 +201,10 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec
201
"""
202
data = self.get_data()
203
if not data:
203
- self.debug('get_data() returns no data')
204
+ self.debug('get_data() returned no data')
205
return False
206
elif not isinstance(data, dict):
206
- self.debug('get_data() returns incorrect type data')
207
+ self.debug('get_data() returned incorrect type data')
208
return False
209
210
job = self._runtime_counters
@@ -254,6 +255,8 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec
255
256
def manage_retries(self):
257
self._runtime_counters.RETRIES += 1
258
+ if self._runtime_counters.RETRIES % 5 == 0:
259
+ self._runtime_counters.PENALTY = int(self._runtime_counters.RETRIES * self.update_every / 2)
260
if self._runtime_counters.RETRIES >= self._runtime_counters.RETRIES_MAX:
261
self.error('stopped after {retries_max} data '
262
'collection failures in a row'.format(retries_max=self._runtime_counters.RETRIES_MAX))
@@ -267,7 +270,7 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec
270
while job.is_sleep_time():
271
sleep_time = job.NEXT_RUN - job.START_RUN
272
self.debug('sleeping for {sleep_time} to reach frequency of {freq} sec'.format(sleep_time=sleep_time,
270
- freq=job.FREQ))
273
+ freq=job.FREQ + job.PENALTY))
274
sleep(sleep_time)
275
job.START_RUN = time()
276