python.d.plugin now never gives up data collection - it slows down data collection attempts instead; #999
Costa Tsaousis (ktsaou) committed
Nov 15, 2016 at 23:47 UTC
a6c5119ecb72039351183ccb9a287e98fa686722
1 file changed
+17
-6
python.d/python_modules/base.py
+17
-6
@@ -144,16 +144,17 @@ class SimpleService(threading.Thread):
144
:return: None
145
"""
146
step = float(self.timetable['freq'])
147
+ penalty = 0
148
self.timetable['last'] = float(time.time() - step)
148
- self.debug("starting data collection - update frequency:", str(step), ", retries allowed:", str(self.retries))
149
+ self.debug("starting data collection - update frequency:", str(step), " retries allowed:", str(self.retries))
150
while True: # run forever, unless something is wrong
151
now = float(time.time())
151
- next = self.timetable['next'] = now - (now % step) + step
152
+ next = self.timetable['next'] = now - (now % step) + step + penalty
153
154
# it is important to do this in a loop
155
# sleep() is interruptable
156
while now < next:
156
- self.debug("sleeping for", str(next - now), "secs to reach frequency of", str(step), "secs, now:", str(now), ", next:", str(next))
157
+ self.debug("sleeping for", str(next - now), "secs to reach frequency of", str(step), "secs, now:", str(now), " next:", str(next), " penalty:", str(penalty))
158
time.sleep(next - now)
159
now = float(time.time())
160
@@ -167,14 +168,24 @@ class SimpleService(threading.Thread):
168
if status:
169
# it is good
170
self.retries_left = self.retries
171
+ penalty = 0
172
else:
173
# it failed
174
self.retries_left -= 1
175
if self.retries_left <= 0:
174
- self.alert("failed to collect data - no more retries allowed - aborting data collection")
175
- return
176
+ if penalty == 0:
177
+ penalty = float(self.retries * step) / 2
178
+ else:
179
+ penalty *= 1.5
180
+
181
+ if penalty > 600:
182
+ penalty = 600
183
+
184
+ self.retries_left = self.retries
185
+ self.alert("failed to collect data for " + str(self.retries) + " times - increasing penalty to " + str(penalty) + " sec and trying again")
186
+
187
else:
177
- self.error("failed to collect data. " + str(self.retries_left) + " retries left.")
188
+ self.error("failed to collect data - " + str(self.retries_left) + " retries left - penalty: " + str(penalty) + " sec")
189
190
# --- CHART ---
191