attempt to fix freezes on socket service
Costa Tsaousis (ktsaou) committed
Nov 9, 2016 at 02:20 UTC
b56269a245bab7e1deeea50d449e44d8c3b07a83
3 files changed
+29
-14
plugins.d/python.d.plugin
+4
-4
@@ -292,8 +292,8 @@ class PythonCharts(object):
292
if job.name is not None and len(job.name) != 0:
293
prefix += "/" + job.name
294
try:
295
+ msg.error("DISABLED:", prefix)
296
self.jobs.remove(job)
296
- msg.info("Disabled", prefix)
297
except Exception as e:
298
msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
299
@@ -335,18 +335,18 @@ class PythonCharts(object):
335
msg.error(job.chart_name, "check function failed.")
336
self._stop(job)
337
else:
338
- msg.debug(job.chart_name, "check succeeded")
338
+ msg.info("CHECKED OK:", job.chart_name)
339
i += 1
340
try:
341
if job.override_name is not None:
342
new_name = job.__module__ + '_' + job.override_name
343
if new_name in overridden:
344
- msg.error(job.override_name + " already exists. Stopping '" + job.name + "'")
344
+ msg.info("DROPPED:", job.name, ", job '" + job.override_name + "' is already served by another job.")
345
self._stop(job)
346
i -= 1
347
else:
348
job.name = job.override_name
349
- msg.debug(job.chart_name + " changing chart name to: '" + new_name + "'")
349
+ msg.info("RENAMED:", new_name, ", from " + job.chart_name)
350
job.chart_name = new_name
351
overridden.append(job.chart_name)
352
except Exception:
python.d/memcached.chart.py
+3
-1
@@ -131,7 +131,7 @@ class Service(SocketService):
131
self.error("no data received")
132
return None
133
if raw[0].startswith('ERROR'):
134
- self.error("Memcached returned ERROR")
134
+ self.error("memcached returned ERROR")
135
return None
136
data = {}
137
for line in raw:
@@ -160,8 +160,10 @@ class Service(SocketService):
160
161
def _check_raw_data(self, data):
162
if data.endswith('END\r\n'):
163
+ self.debug("received response")
164
return True
165
else:
166
+ self.debug("waiting response")
167
return False
168
169
def check(self):
python.d/python_modules/base.py
+22
-9
@@ -519,6 +519,7 @@ class SocketService(SimpleService):
519
if self.__socket_config is None:
520
# establish ipv6 or ipv4 connection.
521
for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM):
522
+ self.debug("connecting socket to host '" + str(self.host) + "', port " + str(self.port))
523
try:
524
# noinspection SpellCheckingInspection
525
af, socktype, proto, canonname, sa = res
@@ -547,6 +548,7 @@ class SocketService(SimpleService):
548
else:
549
# connect to unix socket
550
try:
551
+ self.debug("connecting unix socket '" + str(self.unix_socket) + "'")
552
self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
553
self._sock.connect(self.unix_socket)
554
except socket.error:
@@ -559,19 +561,27 @@ class SocketService(SimpleService):
561
"port:", str(self.port),
562
"socket:", str(self.unix_socket))
563
self._sock = None
562
- self._sock.setblocking(0)
564
+
565
+ if self._sock is not None:
566
+ self._sock.setblocking(0)
567
+ self._sock.settimeout(5)
568
+ self.debug("connected with timeout " + str(self._sock.gettimeout()))
569
+ else:
570
+ self.debug("not connected")
571
572
def _disconnect(self):
573
"""
574
Close socket connection
575
:return:
576
"""
569
- try:
570
- self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all
571
- self._sock.close()
572
- except Exception:
573
- pass
574
- self._sock = None
577
+ if self._sock is not None:
578
+ try:
579
+ self.debug("disconnecting")
580
+ self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all
581
+ self._sock.close()
582
+ except Exception:
583
+ pass
584
+ self._sock = None
585
586
def _send(self):
587
"""
@@ -581,6 +591,7 @@ class SocketService(SimpleService):
591
# Send request if it is needed
592
if self.request != "".encode():
593
try:
594
+ self.debug("sending request")
595
self._sock.send(self.request)
596
except Exception as e:
597
self._disconnect()
@@ -599,14 +610,16 @@ class SocketService(SimpleService):
610
data = ""
611
while True:
612
try:
613
+ self.debug("receiving response")
614
ready_to_read, _, in_error = select.select([self._sock], [], [], 5)
615
except Exception as e:
604
- self.debug("SELECT", str(e))
616
+ self.error("timeout while waiting for response:", str(e))
617
self._disconnect()
618
break
619
if len(ready_to_read) > 0:
620
buf = self._sock.recv(4096)
609
- if len(buf) == 0 or buf is None: # handle server disconnect
621
+ if buf is None or len(buf) == 0: # handle server disconnect
622
+ self._disconnect()
623
break
624
data += buf.decode(errors='ignore')
625
if self._check_raw_data(data):