New `NetSocketService` class + squid.chart.py + fix infinite retries loop
paulfantom committed
Jul 4, 2016 at 15:28 UTC
7abe188b0c1863e755836cc617a9dea516ee16d6
4 files changed
+202
-26
python.d/Makefile.am
+2
@@ -11,6 +11,8 @@ dist_python_SCRIPTS = \
11
apache.chart.py \
12
nginx.chart.py \
13
apache_cache.chart.py \
14
+ hddtemp.chart.py \
15
+ squid.chart.py \
16
python-modules-installer.sh \
17
$(NULL)
18
python.d/hddtemp.chart.py
+7
-25
@@ -2,8 +2,7 @@
2
# Description: hddtemp netdata python.d plugin
3
# Author: Pawel Krupa (paulfantom)
4
5
-from base import SimpleService
6
-import socket
5
+from base import NetSocketService
6
7
# default module values (can be overridden per job in `config`)
8
#update_every = 2
@@ -30,11 +29,12 @@ CHARTS = {
29
}
30
31
33
-class Service(SimpleService):
32
+class Service(NetSocketService):
33
def __init__(self, configuration=None, name=None):
35
- self.host = "localhost"
34
+ NetSocketService.__init__(self, configuration=configuration, name=name)
35
+ self.request = ""
36
+ self.host = "127.0.0.1"
37
self.port = 7634
37
- SimpleService.__init__(self, configuration=configuration, name=name)
38
self.order = ORDER
39
self.definitions = CHARTS
40
@@ -43,14 +43,7 @@ class Service(SimpleService):
43
Get data from TCP/IP socket
44
:return: dict
45
"""
46
- try:
47
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
48
- s.connect((self.host, self.port))
49
- raw = s.recv(4096).split("|")[:-1]
50
- s.close()
51
- except Exception:
52
- return None
53
-
46
+ raw = self._get_raw_data().split("|")[:-1]
47
data = {}
48
for i in range(len(raw) // 5):
49
try:
@@ -61,18 +54,7 @@ class Service(SimpleService):
54
return data
55
56
def check(self):
64
- if self.name is not None or self.name != str(None):
65
- self.name = ""
66
- else:
67
- self.name = str(self.name)
68
- try:
69
- self.host = str(self.configuration['host'])
70
- except (KeyError, TypeError):
71
- self.error("No host specified. Using: '" + self.host + "'")
72
- try:
73
- self.port = int(self.configuration['port'])
74
- except (KeyError, TypeError):
75
- self.error("No port specified. Using: '" + str(self.port) + "'")
57
+ self._parse_config()
58
59
data = self._get_data()
60
if data is None:
python.d/python_modules/base.py
+78
-1
@@ -5,6 +5,7 @@
5
import time
6
import sys
7
import os
8
+import socket
9
try:
10
from urllib.request import urlopen
11
except ImportError:
@@ -132,7 +133,7 @@ class BaseService(threading.Thread):
133
time.sleep(self.timetable['next'] - time.time())
134
self.retries_left = self.retries
135
else:
135
- self.retries -= 1
136
+ self.retries_left -= 1
137
if self.retries_left <= 0:
138
msg.error("no more retries. Exiting")
139
return
@@ -411,6 +412,82 @@ class UrlService(SimpleService):
412
return False
413
414
415
+class NetSocketService(SimpleService):
416
+ def __init__(self, configuration=None, name=None):
417
+ # definitions are created dynamically in create() method based on 'charts' dictionary. format:
418
+ # definitions = {
419
+ # 'chart_name_in_netdata' : [ charts['chart_name_in_netdata']['lines']['name'] ]
420
+ # }
421
+ self.host = "localhost"
422
+ self.port = None
423
+ self.sock = None
424
+ self.request = ""
425
+ SimpleService.__init__(self, configuration=configuration, name=name)
426
+
427
+ def _get_raw_data(self):
428
+ """
429
+ Get raw data with low-level "socket" module.
430
+ :return: str
431
+ """
432
+ if self.sock is None:
433
+ try:
434
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
435
+ sock.settimeout(self.update_every)
436
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
437
+ sock.connect((self.host, self.port))
438
+ except Exception as e:
439
+ self.sock = None
440
+ return None
441
+
442
+ try:
443
+ sock.send(self.request)
444
+ except Exception:
445
+ try:
446
+ sock.shutdown(1)
447
+ sock.close()
448
+ except:
449
+ pass
450
+ self.sock = None
451
+ return None
452
+
453
+ data = sock.recv(1024)
454
+ try:
455
+ while True:
456
+ buf = sock.recv(1024)
457
+ if not buf:
458
+ break
459
+ else:
460
+ data += buf
461
+ except:
462
+ sock.close()
463
+ return None
464
+
465
+ return data.decode()
466
+
467
+ def _parse_config(self):
468
+ """
469
+ Format configuration data and try to connect to server
470
+ :return: boolean
471
+ """
472
+ if self.name is not None or self.name != str(None):
473
+ self.name = ""
474
+ else:
475
+ self.name = str(self.name)
476
+ try:
477
+ self.host = str(self.configuration['host'])
478
+ except (KeyError, TypeError):
479
+ self.error("No host specified. Using: '" + self.host + "'")
480
+ try:
481
+ self.port = int(self.configuration['port'])
482
+ except (KeyError, TypeError):
483
+ self.error("No port specified. Using: '" + str(self.port) + "'")
484
+ try:
485
+ self.port = int(self.configuration['request'])
486
+ except (KeyError, TypeError):
487
+ self.error("No request specified. Using: '" + str(self.request) + "'")
488
+ self.request = self.request.encode()
489
+
490
+
491
class LogService(SimpleService):
492
def __init__(self, configuration=None, name=None):
493
# definitions are created dynamically in create() method based on 'charts' dictionary. format:
python.d/squid.chart.py
new
+115
@@ -0,0 +1,115 @@
1
+# -*- coding: utf-8 -*-
2
+# Description: squid netdata python.d plugin
3
+# Author: Pawel Krupa (paulfantom)
4
+
5
+from base import NetSocketService
6
+
7
+# default module values (can be overridden per job in `config`)
8
+# update_every = 2
9
+priority = 60000
10
+retries = 5
11
+
12
+# default job configuration (overridden by python.d.plugin)
13
+# config = {'local': {
14
+# 'update_every': update_every,
15
+# 'retries': retries,
16
+# 'priority': priority,
17
+# 'url': 'http://localhost/stub_status'
18
+# }}
19
+
20
+# charts order (can be overridden if you want less charts, or different order)
21
+ORDER = ['clients_net', 'clients_requests', 'servers_net', 'servers_requests']
22
+
23
+CHARTS = {
24
+ 'clients_net': {
25
+ 'options': [None, "Squid Client Bandwidth", "kilobits / sec", "clients", "squid.clients.net" "area"],
26
+ 'lines': [
27
+ ["client_http_kbytes_in", "in", "incremental", 8, 1],
28
+ ["client_http_kbytes_out", "out", "incremental", -8, 1],
29
+ ["client_http_hit_kbytes_out", "hits", "incremental", -8, 1]
30
+ ]},
31
+ 'clients_requests': {
32
+ 'options': [None, "Squid Client Requests", "requests / sec", "clients", "squid.clients.requests", 'line'],
33
+ 'lines': [
34
+ ["client_http_requests", "requests"],
35
+ ["client_http_hits", "hits"],
36
+ ["client_http_errors", "errors", "incremental", -1, 1]
37
+ ]},
38
+ 'servers_net': {
39
+ 'options': [None, "Squid Server Bandwidth", "kilobits / sec", "servers", "squid.servers.net" "area"],
40
+ 'lines': [
41
+ ["server_all_kbytes_in", "in", "incremental", 8, 1],
42
+ ["server_all_kbytes_out", "out", "incremental", -8, 1]
43
+ ]},
44
+ 'servers_requests': {
45
+ 'options': [None, "Squid Server Requests", "requests / sec", "servers", "squid.servers.requests", 'line'],
46
+ 'lines': [
47
+ ["server_all_requests", "requests"],
48
+ ["server_all_errors", "errors", "incremental", -1, 1]
49
+ ]}
50
+}
51
+
52
+
53
+class Service(NetSocketService):
54
+ def __init__(self, configuration=None, name=None):
55
+ NetSocketService.__init__(self, configuration=configuration, name=name)
56
+ self.request = ""
57
+ self.host = "localhost"
58
+ self.port = (3128, 8080)
59
+ self.order = ORDER
60
+ self.definitions = CHARTS
61
+
62
+ def _get_data(self):
63
+ """
64
+ Get data from http request
65
+ :return: dict
66
+ """
67
+ try:
68
+ raw = self._get_raw_data().split('\n')
69
+ if "200 OK" not in raw[0]:
70
+ return None
71
+ data = {}
72
+ for row in raw:
73
+ if row.startswith(("client", "server.all")):
74
+ tmp = row.split("=")
75
+ data[tmp[0].replace('.', '_').strip(' ')] = int(tmp[1])
76
+
77
+ return data
78
+ except (ValueError, AttributeError):
79
+ return None
80
+
81
+ def check(self):
82
+ self._parse_config()
83
+ # format request
84
+ self.request.decode()
85
+ if not self.request.startswith("GET"):
86
+ self.request = "GET " + self.request
87
+ if not self.request.endswith(" HTTP/1.0\r\n\r\n"):
88
+ self.request += " HTTP/1.0\r\n\r\n"
89
+ self.request = self.request.encode()
90
+
91
+ # autodetect squid
92
+ if type(self.port) is tuple:
93
+ ports = self.port
94
+ for port in ports:
95
+ self.port = port
96
+ urls = ["cache_object://" + self.host + ":" + str(port) + "/counters",
97
+ "/squid-internal-mgr/counters"]
98
+ for url in urls:
99
+ tmp = "GET " + url + " HTTP/1.0\r\n\r\n"
100
+ self.request = tmp.encode()
101
+ if self._get_data() is not None:
102
+ return True
103
+ else:
104
+ if self._get_data() is not None:
105
+ return True
106
+ else:
107
+ return False
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+