@cryptotaxi247 / netdata-1 / commits / beefe4e22

Python.d PEP 8 cleanup, modules S-Z (#4302)

* python.d/samba.chart.py PEP 8 cleanup Fix formatting of container literals. * python.d/sensors.chart.py PEP 8 cleanup Fix formatting of container literals and quoting of string literals. * python.d/smartd_log.chart.py PEP 8 cleanup Fix formatting of container literals. * python.d/spigotmc.chart.py PEP 8 cleanup Fix spacing of code blocks. * python.d/springboot.chart.py PEP 8 cleanup Bunches of formatting fixes, plus some code correctness fixes. * python.d/squid.chart.py PEP 8 cleanup Fix formatting of container literals and quoting of string literals. * python.d/tomcat.chart.py PEP 8 cleanup Multiple formatting fixes. * python.d/traefik.chart.py PEP 8 cleanup Fixed long lines and formatting of container literals. * python.d/unbound.chart.py PEP 8 cleanup Fixed overly long lines. * python.d/varnish.chart.py PEP 8 cleanup Fixed formatting of container literals. * python.d/w1sensor.chart.py PEP 8 cleanup Fixed container literal formatting. * python.d/web_log.chart.py PEP 8 cleanup Multiple formatting fixes. * Fixed trailing whitespace.

Austin S. Hemmelgarn committed Sep 27, 2018 at 08:56 UTC beefe4e223c6a63653cc76410d5fcf7d59115708
12 files changed +494 -335
python.d/samba.chart.py
+67 -59
@@ -26,67 +26,75 @@ update_every = 5
26 priority = 60000
27 retries = 60
28
29 -ORDER = ['syscall_rw','smb2_rw','smb2_create_close','smb2_info','smb2_find','smb2_notify','smb2_sm_count']
29 +ORDER = [
30 + 'syscall_rw',
31 + 'smb2_rw',
32 + 'smb2_create_close',
33 + 'smb2_info',
34 + 'smb2_find',
35 + 'smb2_notify',
36 + 'smb2_sm_count'
37 +]
38
39 CHARTS = {
32 - 'syscall_rw': {
33 - 'lines': [
34 - ['syscall_sendfile_bytes', 'sendfile', 'incremental', 1, 1024],
35 - ['syscall_recvfile_bytes', 'recvfile', 'incremental', -1, 1024]
36 - ],
37 - 'options': [None, 'R/Ws', 'kilobytes/s', 'syscall', 'syscall.rw', 'area']
38 - },
39 - 'smb2_rw': {
40 - 'lines': [
41 - ['smb2_read_outbytes', 'readout', 'incremental', 1, 1024],
42 - ['smb2_write_inbytes', 'writein', 'incremental', -1, 1024],
43 - ['smb2_read_inbytes', 'readin', 'incremental', 1, 1024],
44 - ['smb2_write_outbytes', 'writeout', 'incremental', -1, 1024]
45 - ],
46 - 'options': [None, 'R/Ws', 'kilobytes/s', 'smb2', 'smb2.rw', 'area']
47 - },
48 - 'smb2_create_close': {
49 - 'lines': [
50 - ['smb2_create_count', 'create', 'incremental', 1, 1],
51 - ['smb2_close_count', 'close', 'incremental', -1, 1]
52 - ],
53 - 'options': [None, 'Create/Close', 'operations/s', 'smb2', 'smb2.create_close', 'line']
54 - },
55 - 'smb2_info': {
56 - 'lines': [
57 - ['smb2_getinfo_count', 'getinfo', 'incremental', 1, 1],
58 - ['smb2_setinfo_count', 'setinfo', 'incremental', -1, 1]
59 - ],
60 - 'options': [None, 'Info', 'operations/s', 'smb2', 'smb2.get_set_info', 'line']
61 - },
62 - 'smb2_find': {
63 - 'lines': [
64 - ['smb2_find_count', 'find', 'incremental', 1, 1]
65 - ],
66 - 'options': [None, 'Find', 'operations/s', 'smb2', 'smb2.find', 'line']
67 - },
68 - 'smb2_notify': {
69 - 'lines': [
70 - ['smb2_notify_count', 'notify', 'incremental', 1, 1]
71 - ],
72 - 'options': [None, 'Notify', 'operations/s', 'smb2', 'smb2.notify', 'line']
73 - },
74 - 'smb2_sm_count': {
75 - 'lines': [
76 - ['smb2_tcon_count', 'tcon', 'absolute', 1, 1],
77 - ['smb2_negprot_count', 'negprot', 'absolute', 1, 1],
78 - ['smb2_tdis_count', 'tdis', 'absolute', 1, 1],
79 - ['smb2_cancel_count', 'cancel', 'absolute', 1, 1],
80 - ['smb2_logoff_count', 'logoff', 'absolute', 1, 1],
81 - ['smb2_flush_count', 'flush', 'absolute', 1, 1],
82 - ['smb2_lock_count', 'lock', 'absolute', 1, 1],
83 - ['smb2_keepalive_count', 'keepalive', 'absolute', 1, 1],
84 - ['smb2_break_count', 'break', 'absolute', 1, 1],
85 - ['smb2_sessetup_count', 'sessetup', 'absolute', 1, 1]
86 - ],
87 - 'options': [None, 'Lesser Ops', 'count', 'smb2', 'smb2.sm_counters', 'stacked']
88 - }
89 - }
40 + 'syscall_rw': {
41 + 'options': [None, 'R/Ws', 'kilobytes/s', 'syscall', 'syscall.rw', 'area'],
42 + 'lines': [
43 + ['syscall_sendfile_bytes', 'sendfile', 'incremental', 1, 1024],
44 + ['syscall_recvfile_bytes', 'recvfile', 'incremental', -1, 1024]
45 + ]
46 + },
47 + 'smb2_rw': {
48 + 'options': [None, 'R/Ws', 'kilobytes/s', 'smb2', 'smb2.rw', 'area'],
49 + 'lines': [
50 + ['smb2_read_outbytes', 'readout', 'incremental', 1, 1024],
51 + ['smb2_write_inbytes', 'writein', 'incremental', -1, 1024],
52 + ['smb2_read_inbytes', 'readin', 'incremental', 1, 1024],
53 + ['smb2_write_outbytes', 'writeout', 'incremental', -1, 1024]
54 + ]
55 + },
56 + 'smb2_create_close': {
57 + 'options': [None, 'Create/Close', 'operations/s', 'smb2', 'smb2.create_close', 'line'],
58 + 'lines': [
59 + ['smb2_create_count', 'create', 'incremental', 1, 1],
60 + ['smb2_close_count', 'close', 'incremental', -1, 1]
61 + ]
62 + },
63 + 'smb2_info': {
64 + 'options': [None, 'Info', 'operations/s', 'smb2', 'smb2.get_set_info', 'line'],
65 + 'lines': [
66 + ['smb2_getinfo_count', 'getinfo', 'incremental', 1, 1],
67 + ['smb2_setinfo_count', 'setinfo', 'incremental', -1, 1]
68 + ]
69 + },
70 + 'smb2_find': {
71 + 'options': [None, 'Find', 'operations/s', 'smb2', 'smb2.find', 'line'],
72 + 'lines': [
73 + ['smb2_find_count', 'find', 'incremental', 1, 1]
74 + ]
75 + },
76 + 'smb2_notify': {
77 + 'options': [None, 'Notify', 'operations/s', 'smb2', 'smb2.notify', 'line'],
78 + 'lines': [
79 + ['smb2_notify_count', 'notify', 'incremental', 1, 1]
80 + ]
81 + },
82 + 'smb2_sm_count': {
83 + 'options': [None, 'Lesser Ops', 'count', 'smb2', 'smb2.sm_counters', 'stacked'],
84 + 'lines': [
85 + ['smb2_tcon_count', 'tcon', 'absolute', 1, 1],
86 + ['smb2_negprot_count', 'negprot', 'absolute', 1, 1],
87 + ['smb2_tdis_count', 'tdis', 'absolute', 1, 1],
88 + ['smb2_cancel_count', 'cancel', 'absolute', 1, 1],
89 + ['smb2_logoff_count', 'logoff', 'absolute', 1, 1],
90 + ['smb2_flush_count', 'flush', 'absolute', 1, 1],
91 + ['smb2_lock_count', 'lock', 'absolute', 1, 1],
92 + ['smb2_keepalive_count', 'keepalive', 'absolute', 1, 1],
93 + ['smb2_break_count', 'break', 'absolute', 1, 1],
94 + ['smb2_sessetup_count', 'sessetup', 'absolute', 1, 1]
95 + ]
96 + }
97 +}
98
99
100 class Service(ExecutableService):
python.d/sensors.chart.py
+17 -11
@@ -17,37 +17,44 @@ CHARTS = {
17 'options': [None, ' temperature', 'Celsius', 'temperature', 'sensors.temperature', 'line'],
18 'lines': [
19 [None, None, 'absolute', 1, 1000]
20 - ]},
20 + ]
21 + },
22 'voltage': {
23 'options': [None, ' voltage', 'Volts', 'voltage', 'sensors.voltage', 'line'],
24 'lines': [
25 [None, None, 'absolute', 1, 1000]
25 - ]},
26 + ]
27 + },
28 'current': {
29 'options': [None, ' current', 'Ampere', 'current', 'sensors.current', 'line'],
30 'lines': [
31 [None, None, 'absolute', 1, 1000]
30 - ]},
32 + ]
33 + },
34 'power': {
35 'options': [None, ' power', 'Watt', 'power', 'sensors.power', 'line'],
36 'lines': [
37 [None, None, 'absolute', 1, 1000000]
35 - ]},
38 + ]
39 + },
40 'fan': {
41 'options': [None, ' fans speed', 'Rotations/min', 'fans', 'sensors.fan', 'line'],
42 'lines': [
43 [None, None, 'absolute', 1, 1000]
40 - ]},
44 + ]
45 + },
46 'energy': {
47 'options': [None, ' energy', 'Joule', 'energy', 'sensors.energy', 'areastack'],
48 'lines': [
49 [None, None, 'incremental', 1, 1000000]
45 - ]},
50 + ]
51 + },
52 'humidity': {
53 'options': [None, ' humidity', 'Percent', 'humidity', 'sensors.humidity', 'line'],
54 'lines': [
55 [None, None, 'absolute', 1, 1000]
50 - ]}
56 + ]
57 + }
58 }
59
60 LIMITS = {
@@ -95,7 +102,7 @@ class Service(SimpleService):
102 limit = LIMITS[type_name]
103 if val < limit[0] or val > limit[1]:
104 continue
98 - data[prefix + "_" + str(feature.name.decode())] = int(val * 1000)
105 + data[prefix + '_' + str(feature.name.decode())] = int(val * 1000)
106 except Exception as error:
107 self.error(error)
108 return None
@@ -115,7 +122,7 @@ class Service(SimpleService):
122 continue
123 if TYPE_MAP[feature.type] == sensor:
124 # create chart
118 - name = chip_name + "_" + TYPE_MAP[feature.type]
125 + name = chip_name + '_' + TYPE_MAP[feature.type]
126 if name not in self.order:
127 self.order.append(name)
128 chart_def = list(CHARTS[sensor]['options'])
@@ -123,7 +130,7 @@ class Service(SimpleService):
130 self.definitions[name] = {'options': chart_def}
131 self.definitions[name]['lines'] = []
132 line = list(CHARTS[sensor]['lines'][0])
126 - line[0] = chip_name + "_" + str(feature.name.decode())
133 + line[0] = chip_name + '_' + str(feature.name.decode())
134 line[1] = sensors.get_label(chip, feature)
135 self.definitions[name]['lines'].append(line)
136
@@ -137,4 +144,3 @@ class Service(SimpleService):
144 self.create_definitions()
145
146 return True
140 -
python.d/smartd_log.chart.py
+1 -1
@@ -116,7 +116,7 @@ def chart_template(chart_name):
116 chart_name: {
117 'options': [None, title, units, family, 'smartd_log.' + chart_name, 'line'],
118 'lines': []
119 - }
119 + }
120 }
121
122
python.d/spigotmc.chart.py
+1
@@ -35,6 +35,7 @@ CHARTS = {
35 }
36 }
37
38 +
39 class Service(SimpleService):
40 def __init__(self, configuration=None, name=None):
41 SimpleService.__init__(self, configuration=configuration, name=name)
python.d/springboot.chart.py
+51 -44
@@ -18,54 +18,61 @@ DEFAULT_CHARTS = {
18 'response_code': {
19 'options': [None, "Response Codes", "requests/s", "response", "springboot.response_code", "stacked"],
20 'lines': [
21 - ["resp_other", 'Other', 'incremental'],
22 - ["resp_1xx", '1xx', 'incremental'],
23 - ["resp_2xx", '2xx', 'incremental'],
24 - ["resp_3xx", '3xx', 'incremental'],
25 - ["resp_4xx", '4xx', 'incremental'],
26 - ["resp_5xx", '5xx', 'incremental'],
27 - ]},
21 + ["resp_other", 'Other', 'incremental'],
22 + ["resp_1xx", '1xx', 'incremental'],
23 + ["resp_2xx", '2xx', 'incremental'],
24 + ["resp_3xx", '3xx', 'incremental'],
25 + ["resp_4xx", '4xx', 'incremental'],
26 + ["resp_5xx", '5xx', 'incremental'],
27 + ]
28 + },
29 'threads': {
30 'options': [None, "Threads", "current threads", "threads", "springboot.threads", "area"],
31 'lines': [
32 ["threads_daemon", 'daemon', 'absolute'],
33 ["threads", 'total', 'absolute'],
33 - ]},
34 + ]
35 + },
36 'gc_time': {
37 'options': [None, "GC Time", "milliseconds", "garbage collection", "springboot.gc_time", "stacked"],
38 'lines': [
37 - ["gc_copy_time", 'Copy', 'incremental'],
38 - ["gc_marksweepcompact_time", 'MarkSweepCompact', 'incremental'],
39 - ["gc_parnew_time", 'ParNew', 'incremental'],
40 - ["gc_concurrentmarksweep_time", 'ConcurrentMarkSweep', 'incremental'],
41 - ["gc_ps_scavenge_time", 'PS Scavenge', 'incremental'],
42 - ["gc_ps_marksweep_time", 'PS MarkSweep', 'incremental'],
43 - ["gc_g1_young_generation_time", 'G1 Young Generation', 'incremental'],
44 - ["gc_g1_old_generation_time", 'G1 Old Generation', 'incremental'],
45 - ]},
39 + ["gc_copy_time", 'Copy', 'incremental'],
40 + ["gc_marksweepcompact_time", 'MarkSweepCompact', 'incremental'],
41 + ["gc_parnew_time", 'ParNew', 'incremental'],
42 + ["gc_concurrentmarksweep_time", 'ConcurrentMarkSweep', 'incremental'],
43 + ["gc_ps_scavenge_time", 'PS Scavenge', 'incremental'],
44 + ["gc_ps_marksweep_time", 'PS MarkSweep', 'incremental'],
45 + ["gc_g1_young_generation_time", 'G1 Young Generation', 'incremental'],
46 + ["gc_g1_old_generation_time", 'G1 Old Generation', 'incremental'],
47 + ]
48 + },
49 'gc_ope': {
50 'options': [None, "GC Operations", "operations/s", "garbage collection", "springboot.gc_ope", "stacked"],
51 'lines': [
49 - ["gc_copy_count", 'Copy', 'incremental'],
50 - ["gc_marksweepcompact_count", 'MarkSweepCompact', 'incremental'],
51 - ["gc_parnew_count", 'ParNew', 'incremental'],
52 - ["gc_concurrentmarksweep_count", 'ConcurrentMarkSweep', 'incremental'],
53 - ["gc_ps_scavenge_count", 'PS Scavenge', 'incremental'],
54 - ["gc_ps_marksweep_count", 'PS MarkSweep', 'incremental'],
55 - ["gc_g1_young_generation_count", 'G1 Young Generation', 'incremental'],
56 - ["gc_g1_old_generation_count", 'G1 Old Generation', 'incremental'],
57 - ]},
52 + ["gc_copy_count", 'Copy', 'incremental'],
53 + ["gc_marksweepcompact_count", 'MarkSweepCompact', 'incremental'],
54 + ["gc_parnew_count", 'ParNew', 'incremental'],
55 + ["gc_concurrentmarksweep_count", 'ConcurrentMarkSweep', 'incremental'],
56 + ["gc_ps_scavenge_count", 'PS Scavenge', 'incremental'],
57 + ["gc_ps_marksweep_count", 'PS MarkSweep', 'incremental'],
58 + ["gc_g1_young_generation_count", 'G1 Young Generation', 'incremental'],
59 + ["gc_g1_old_generation_count", 'G1 Old Generation', 'incremental'],
60 + ]
61 + },
62 'heap': {
63 'options': [None, "Heap Memory Usage", "KB", "heap memory", "springboot.heap", "area"],
64 'lines': [
65 ["heap_committed", 'committed', "absolute"],
66 ["heap_used", 'used', "absolute"],
63 - ]},
67 + ]
68 + }
69 }
70
71 +
72 class ExtraChartError(ValueError):
73 pass
74
75 +
76 class Service(UrlService):
77 def __init__(self, configuration=None, name=None):
78 UrlService.__init__(self, configuration=configuration, name=name)
@@ -88,11 +95,11 @@ class Service(UrlService):
95 return None
96
97 result = {
91 - 'resp_1xx': 0,
92 - 'resp_2xx': 0,
93 - 'resp_3xx': 0,
94 - 'resp_4xx': 0,
95 - 'resp_5xx': 0,
98 + 'resp_1xx': 0,
99 + 'resp_2xx': 0,
100 + 'resp_3xx': 0,
101 + 'resp_4xx': 0,
102 + 'resp_5xx': 0,
103 'resp_other': 0,
104 }
105
@@ -122,14 +129,14 @@ class Service(UrlService):
129 self.order.append(extra['id'])
130
131 def _add_extra_chart(self, chart):
125 - chart_id = chart.get('id', None) or die('id is not defined in extra chart')
126 - options = chart.get('options', None) or die('option is not defined in extra chart: %s' % chart_id)
127 - lines = chart.get('lines', None) or die('lines is not defined in extra chart: %s' % chart_id)
128 -
129 - title = options.get('title', None) or die('title is missing: %s' % chart_id)
130 - units = options.get('units', None) or die('units is missing: %s' % chart_id)
131 - family = options.get('family', title)
132 - context = options.get('context', 'springboot.' + title)
132 + chart_id = chart.get('id', None) or self.die('id is not defined in extra chart')
133 + options = chart.get('options', None) or self.die('option is not defined in extra chart: %s' % chart_id)
134 + lines = chart.get('lines', None) or self.die('lines is not defined in extra chart: %s' % chart_id)
135 +
136 + title = options.get('title', None) or self.die('title is missing: %s' % chart_id)
137 + units = options.get('units', None) or self.die('units is missing: %s' % chart_id)
138 + family = options.get('family', title)
139 + context = options.get('context', 'springboot.' + title)
140 charttype = options.get('charttype', 'line')
141
142 result = {
@@ -138,11 +145,11 @@ class Service(UrlService):
145 }
146
147 for line in lines:
141 - dimension = line.get('dimension', None) or die('dimension is missing: %s' % chart_id)
142 - name = line.get('name', dimension)
143 - algorithm = line.get('algorithm', 'absolute')
148 + dimension = line.get('dimension', None) or self.die('dimension is missing: %s' % chart_id)
149 + name = line.get('name', dimension)
150 + algorithm = line.get('algorithm', 'absolute')
151 multiplier = line.get('multiplier', 1)
145 - divisor = line.get('divisor', 1)
152 + divisor = line.get('divisor', 1)
153 result['lines'].append([dimension, name, algorithm, multiplier, divisor])
154
155 self.definitions[chart_id] = result
python.d/squid.chart.py
+39 -35
@@ -16,31 +16,35 @@ ORDER = ['clients_net', 'clients_requests', 'servers_net', 'servers_requests']
16
17 CHARTS = {
18 'clients_net': {
19 - 'options': [None, "Squid Client Bandwidth", "kilobits/s", "clients", "squid.clients_net", "area"],
19 + 'options': [None, 'Squid Client Bandwidth', 'kilobits/s', 'clients', 'squid.clients_net', 'area'],
20 'lines': [
21 - ["client_http_kbytes_in", "in", "incremental", 8, 1],
22 - ["client_http_kbytes_out", "out", "incremental", -8, 1],
23 - ["client_http_hit_kbytes_out", "hits", "incremental", -8, 1]
24 - ]},
21 + ['client_http_kbytes_in', 'in', 'incremental', 8, 1],
22 + ['client_http_kbytes_out', 'out', 'incremental', -8, 1],
23 + ['client_http_hit_kbytes_out', 'hits', 'incremental', -8, 1]
24 + ]
25 + },
26 'clients_requests': {
26 - 'options': [None, "Squid Client Requests", "requests/s", "clients", "squid.clients_requests", 'line'],
27 + 'options': [None, 'Squid Client Requests', 'requests/s', 'clients', 'squid.clients_requests', 'line'],
28 'lines': [
28 - ["client_http_requests", "requests", "incremental"],
29 - ["client_http_hits", "hits", "incremental"],
30 - ["client_http_errors", "errors", "incremental", -1, 1]
31 - ]},
29 + ['client_http_requests', 'requests', 'incremental'],
30 + ['client_http_hits', 'hits', 'incremental'],
31 + ['client_http_errors', 'errors', 'incremental', -1, 1]
32 + ]
33 + },
34 'servers_net': {
33 - 'options': [None, "Squid Server Bandwidth", "kilobits/s", "servers", "squid.servers_net", "area"],
35 + 'options': [None, 'Squid Server Bandwidth', 'kilobits/s', 'servers', 'squid.servers_net', 'area'],
36 'lines': [
35 - ["server_all_kbytes_in", "in", "incremental", 8, 1],
36 - ["server_all_kbytes_out", "out", "incremental", -8, 1]
37 - ]},
37 + ['server_all_kbytes_in', 'in', 'incremental', 8, 1],
38 + ['server_all_kbytes_out', 'out', 'incremental', -8, 1]
39 + ]
40 + },
41 'servers_requests': {
39 - 'options': [None, "Squid Server Requests", "requests/s", "servers", "squid.servers_requests", 'line'],
42 + 'options': [None, 'Squid Server Requests', 'requests/s', 'servers', 'squid.servers_requests', 'line'],
43 'lines': [
41 - ["server_all_requests", "requests", "incremental"],
42 - ["server_all_errors", "errors", "incremental", -1, 1]
43 - ]}
44 + ['server_all_requests', 'requests', 'incremental'],
45 + ['server_all_errors', 'errors', 'incremental', -1, 1]
46 + ]
47 + }
48 }
49
50
@@ -48,8 +52,8 @@ class Service(SocketService):
52 def __init__(self, configuration=None, name=None):
53 SocketService.__init__(self, configuration=configuration, name=name)
54 self._keep_alive = True
51 - self.request = ""
52 - self.host = "localhost"
55 + self.request = ''
56 + self.host = 'localhost'
57 self.port = 3128
58 self.order = ORDER
59 self.definitions = CHARTS
@@ -63,43 +67,43 @@ class Service(SocketService):
67
68 data = dict()
69 try:
66 - raw = ""
70 + raw = ''
71 for tmp in response.split('\r\n'):
68 - if tmp.startswith("sample_time"):
72 + if tmp.startswith('sample_time'):
73 raw = tmp
74 break
75
76 if raw.startswith('<'):
73 - self.error("invalid data received")
77 + self.error('invalid data received')
78 return None
79
80 for row in raw.split('\n'):
77 - if row.startswith(("client", "server.all")):
78 - tmp = row.split("=")
81 + if row.startswith(('client', 'server.all')):
82 + tmp = row.split('=')
83 data[tmp[0].replace('.', '_').strip(' ')] = int(tmp[1])
84
85 except (ValueError, AttributeError, TypeError):
82 - self.error("invalid data received")
86 + self.error('invalid data received')
87 return None
88
89 if not data:
86 - self.error("no data received")
90 + self.error('no data received')
91 return None
92 return data
93
94 def _check_raw_data(self, data):
95 header = data[:1024].lower()
96
93 - if "connection: keep-alive" in header:
97 + if 'connection: keep-alive' in header:
98 self._keep_alive = True
99 else:
100 self._keep_alive = False
101
98 - if data[-7:] == "\r\n0\r\n\r\n" and "transfer-encoding: chunked" in header: # HTTP/1.1 response
99 - self.debug("received full response from squid")
102 + if data[-7:] == '\r\n0\r\n\r\n' and 'transfer-encoding: chunked' in header: # HTTP/1.1 response
103 + self.debug('received full response from squid')
104 return True
105
102 - self.debug("waiting more data from squid")
106 + self.debug('waiting more data from squid')
107 return False
108
109 def check(self):
@@ -110,10 +114,10 @@ class Service(SocketService):
114 self._parse_config()
115 # format request
116 req = self.request.decode()
113 - if not req.startswith("GET"):
114 - req = "GET " + req
115 - if not req.endswith(" HTTP/1.1\r\n\r\n"):
116 - req += " HTTP/1.1\r\n\r\n"
117 + if not req.startswith('GET'):
118 + req = 'GET ' + req
119 + if not req.endswith(' HTTP/1.1\r\n\r\n'):
120 + req += ' HTTP/1.1\r\n\r\n'
121 self.request = req.encode()
122 if self._get_data() is not None:
123 return True
python.d/tomcat.chart.py
+52 -44
@@ -18,67 +18,75 @@ ORDER = ['accesses', 'bandwidth', 'processing_time', 'threads', 'jvm', 'jvm_eden
18
19 CHARTS = {
20 'accesses': {
21 - 'options': [None, "Requests", "requests/s", "statistics", "tomcat.accesses", "area"],
21 + 'options': [None, 'Requests', 'requests/s', 'statistics', 'tomcat.accesses', 'area'],
22 'lines': [
23 - ["requestCount", 'accesses', 'incremental'],
24 - ["errorCount", 'errors', 'incremental'],
25 - ]},
23 + ['requestCount', 'accesses', 'incremental'],
24 + ['errorCount', 'errors', 'incremental'],
25 + ]
26 + },
27 'bandwidth': {
27 - 'options': [None, "Bandwidth", "KB/s", "statistics", "tomcat.bandwidth", "area"],
28 + 'options': [None, 'Bandwidth', 'KB/s', 'statistics', 'tomcat.bandwidth', 'area'],
29 'lines': [
29 - ["bytesSent", 'sent', 'incremental', 1, 1024],
30 - ["bytesReceived", 'received', 'incremental', 1, 1024],
31 - ]},
30 + ['bytesSent', 'sent', 'incremental', 1, 1024],
31 + ['bytesReceived', 'received', 'incremental', 1, 1024],
32 + ]
33 + },
34 'processing_time': {
33 - 'options': [None, "processing time", "seconds", "statistics", "tomcat.processing_time", "area"],
35 + 'options': [None, 'processing time', 'seconds', 'statistics', 'tomcat.processing_time', 'area'],
36 'lines': [
35 - ["processingTime", 'processing time', 'incremental', 1, 1000]
36 - ]},
37 + ['processingTime', 'processing time', 'incremental', 1, 1000]
38 + ]
39 + },
40 'threads': {
38 - 'options': [None, "Threads", "current threads", "statistics", "tomcat.threads", "area"],
41 + 'options': [None, 'Threads', 'current threads', 'statistics', 'tomcat.threads', 'area'],
42 'lines': [
40 - ["currentThreadCount", 'current', "absolute"],
41 - ["currentThreadsBusy", 'busy', "absolute"]
42 - ]},
43 + ['currentThreadCount', 'current', 'absolute'],
44 + ['currentThreadsBusy', 'busy', 'absolute']
45 + ]
46 + },
47 'jvm': {
44 - 'options': [None, "JVM Memory Pool Usage", "MB", "memory", "tomcat.jvm", "stacked"],
48 + 'options': [None, 'JVM Memory Pool Usage', 'MB', 'memory', 'tomcat.jvm', 'stacked'],
49 'lines': [
46 - ["free", 'free', "absolute", 1, 1048576],
47 - ["eden_used", 'eden', "absolute", 1, 1048576],
48 - ["survivor_used", 'survivor', "absolute", 1, 1048576],
49 - ["tenured_used", 'tenured', "absolute", 1, 1048576],
50 - ["code_cache_used", 'code cache', "absolute", 1, 1048576],
51 - ["compressed_used", 'compressed', "absolute", 1, 1048576],
52 - ["metaspace_used", 'metaspace', "absolute", 1, 1048576],
53 - ]},
50 + ['free', 'free', 'absolute', 1, 1048576],
51 + ['eden_used', 'eden', 'absolute', 1, 1048576],
52 + ['survivor_used', 'survivor', 'absolute', 1, 1048576],
53 + ['tenured_used', 'tenured', 'absolute', 1, 1048576],
54 + ['code_cache_used', 'code cache', 'absolute', 1, 1048576],
55 + ['compressed_used', 'compressed', 'absolute', 1, 1048576],
56 + ['metaspace_used', 'metaspace', 'absolute', 1, 1048576],
57 + ]
58 + },
59 'jvm_eden': {
55 - 'options': [None, "Eden Memory Usage", "MB", "memory", "tomcat.jvm_eden", "area"],
60 + 'options': [None, 'Eden Memory Usage', 'MB', 'memory', 'tomcat.jvm_eden', 'area'],
61 'lines': [
57 - ["eden_used", 'used', "absolute", 1, 1048576],
58 - ["eden_committed", 'committed', "absolute", 1, 1048576],
59 - ["eden_max", 'max', "absolute", 1, 1048576]
60 - ]},
62 + ['eden_used', 'used', 'absolute', 1, 1048576],
63 + ['eden_committed', 'committed', 'absolute', 1, 1048576],
64 + ['eden_max', 'max', 'absolute', 1, 1048576]
65 + ]
66 + },
67 'jvm_survivor': {
62 - 'options': [None, "Survivor Memory Usage", "MB", "memory", "tomcat.jvm_survivor", "area"],
68 + 'options': [None, 'Survivor Memory Usage', 'MB', 'memory', 'tomcat.jvm_survivor', 'area'],
69 'lines': [
64 - ["survivor_used", 'used', "absolute", 1, 1048576],
65 - ["survivor_committed", 'committed', "absolute", 1, 1048576],
66 - ["survivor_max", 'max', "absolute", 1, 1048576]
67 - ]},
70 + ['survivor_used', 'used', 'absolute', 1, 1048576],
71 + ['survivor_committed', 'committed', 'absolute', 1, 1048576],
72 + ['survivor_max', 'max', 'absolute', 1, 1048576]
73 + ]
74 + },
75 'jvm_tenured': {
69 - 'options': [None, "Tenured Memory Usage", "MB", "memory", "tomcat.jvm_tenured", "area"],
76 + 'options': [None, 'Tenured Memory Usage', 'MB', 'memory', 'tomcat.jvm_tenured', 'area'],
77 'lines': [
71 - ["tenured_used", 'used', "absolute", 1, 1048576],
72 - ["tenured_committed", 'committed', "absolute", 1, 1048576],
73 - ["tenured_max", 'max', "absolute", 1, 1048576]
74 - ]},
78 + ['tenured_used', 'used', 'absolute', 1, 1048576],
79 + ['tenured_committed', 'committed', 'absolute', 1, 1048576],
80 + ['tenured_max', 'max', 'absolute', 1, 1048576]
81 + ]
82 + }
83 }
84
85
86 class Service(UrlService):
87 def __init__(self, configuration=None, name=None):
88 UrlService.__init__(self, configuration=configuration, name=name)
81 - self.url = self.configuration.get('url', "http://127.0.0.1:8080/manager/status?XML=true")
89 + self.url = self.configuration.get('url', 'http://127.0.0.1:8080/manager/status?XML=true')
90 self.connector_name = self.configuration.get('connector_name', None)
91 self.order = ORDER
92 self.definitions = CHARTS
@@ -147,9 +155,9 @@ class Service(UrlService):
155
156 request_info = connector.find('requestInfo')
157 data['processingTime'] = request_info.get('processingTime')
150 - data['requestCount'] = request_info.get('requestCount')
151 - data['errorCount'] = request_info.get('errorCount')
152 - data['bytesReceived'] = request_info.get('bytesReceived')
153 - data['bytesSent'] = request_info.get('bytesSent')
158 + data['requestCount'] = request_info.get('requestCount')
159 + data['errorCount'] = request_info.get('errorCount')
160 + data['bytesReceived'] = request_info.get('bytesReceived')
161 + data['bytesSent'] = request_info.get('bytesSent')
162
163 return data or None
python.d/traefik.chart.py
+26 -13
@@ -33,7 +33,8 @@ CHARTS = {
33 ['redirects', 'redirect', 'incremental'],
34 ['bad_requests', 'bad', 'incremental'],
35 ['other_requests', 'other', 'incremental']
36 - ]},
36 + ]
37 + },
38 'response_codes': {
39 'options': [None, 'Responses by codes', 'requests/s', 'responses', 'traefik.response_codes', 'stacked'],
40 'lines': [
@@ -43,37 +44,45 @@ CHARTS = {
44 ['4xx', None, 'incremental'],
45 ['1xx', None, 'incremental'],
46 ['other', None, 'incremental']
46 - ]},
47 + ]
48 + },
49 'detailed_response_codes': {
48 - 'options': [None, 'Detailed response codes', 'requests/s', 'responses', 'traefik.detailed_response_codes', 'stacked'],
49 - 'lines': [
50 - ]},
50 + 'options': [None, 'Detailed response codes', 'requests/s', 'responses', 'traefik.detailed_response_codes',
51 + 'stacked'],
52 + 'lines': []
53 + },
54 'requests': {
55 'options': [None, 'Requests', 'requests/s', 'requests', 'traefik.requests', 'line'],
56 'lines': [
57 ['total_count', 'requests', 'incremental']
55 - ]},
58 + ]
59 + },
60 'total_response_time': {
61 'options': [None, 'Total response time', 'seconds', 'timings', 'traefik.total_response_time', 'line'],
62 'lines': [
63 ['total_response_time_sec', 'response', 'absolute', 1, 10000]
60 - ]},
64 + ]
65 + },
66 'average_response_time': {
67 'options': [None, 'Average response time', 'milliseconds', 'timings', 'traefik.average_response_time', 'line'],
68 'lines': [
69 ['average_response_time_sec', 'response', 'absolute', 1, 1000]
65 - ]},
70 + ]
71 + },
72 'average_response_time_per_iteration': {
67 - 'options': [None, 'Average response time per iteration', 'milliseconds', 'timings', 'traefik.average_response_time_per_iteration', 'line'],
73 + 'options': [None, 'Average response time per iteration', 'milliseconds', 'timings',
74 + 'traefik.average_response_time_per_iteration', 'line'],
75 'lines': [
76 ['average_response_time_per_iteration_sec', 'response', 'incremental', 1, 10000]
70 - ]},
77 + ]
78 + },
79 'uptime': {
80 'options': [None, 'Uptime', 'seconds', 'uptime', 'traefik.uptime', 'line'],
81 'lines': [
82 ['uptime_sec', 'uptime', 'absolute']
75 - ]}
83 + ]
84 }
85 +}
86
87 HEALTH_STATS = [
88 'uptime_sec',
@@ -83,6 +92,7 @@ HEALTH_STATS = [
92 'total_status_code_count'
93 ]
94
95 +
96 class Service(UrlService):
97 def __init__(self, configuration=None, name=None):
98 UrlService.__init__(self, configuration=configuration, name=name)
@@ -117,9 +127,11 @@ class Service(UrlService):
127 self.data['average_response_time_sec'] *= 1000000
128 self.data['total_response_time_sec'] *= 10000
129 if data['total_count'] != self.last_total_count:
120 - self.data['average_response_time_per_iteration_sec'] = (data['total_response_time_sec'] - self.last_total_response_time) * 1000000 / (data['total_count'] - self.last_total_count)
130 + self.data['average_response_time_per_iteration_sec'] = \
131 + (data['total_response_time_sec'] - self.last_total_response_time) * \
132 + 1000000 / (data['total_count'] - self.last_total_count)
133 else:
122 - self.data['average_response_time_per_iteration_sec'] = 0
134 + self.data['average_response_time_per_iteration_sec'] = 0
135 self.last_total_response_time = data['total_response_time_sec']
136 self.last_total_count = data['total_count']
137
@@ -166,6 +178,7 @@ class Service(UrlService):
178 self.charts['detailed_response_codes'].add_dimension([code, code, 'incremental'])
179 self.data[code] = value
180
181 +
182 def fetch_data_(raw_data, metrics):
183 data = dict()
184
python.d/unbound.chart.py
+4 -2
@@ -67,7 +67,8 @@ EXTENDED_CHARTS = {
67 # This is used as a templates for the per-thread charts.
68 PER_THREAD_CHARTS = {
69 '_queries': {
70 - 'options': [None, '{longname} Queries Processed', 'queries', 'Queries Processed', 'unbound.threads.queries', 'line'],
70 + 'options': [None, '{longname} Queries Processed', 'queries', 'Queries Processed',
71 + 'unbound.threads.queries', 'line'],
72 'lines': [
73 ['{shortname}_ratelimit', 'ratelimited', 'absolute', 1, 1],
74 ['{shortname}_cachemiss', 'cache_miss', 'absolute', 1, 1],
@@ -78,7 +79,8 @@ PER_THREAD_CHARTS = {
79 ]
80 },
81 '_recursion': {
81 - 'options': [None, '{longname} Recursion Timings', 'seconds', 'Recursive Timings', 'unbound.threads.recursion', 'line'],
82 + 'options': [None, '{longname} Recursion Timings', 'seconds', 'Recursive Timings',
83 + 'unbound.threads.recursion', 'line'],
84 'lines': [
85 ['{shortname}_recursive_avg', 'average', 'absolute', 1, PRECISION],
86 ['{shortname}_recursive_med', 'median', 'absolute', 1, PRECISION]
python.d/varnish.chart.py
+16 -7
@@ -13,13 +13,22 @@ from bases.FrameworkServices.ExecutableService import ExecutableService
13 priority = 60000
14 retries = 60
15
16 -ORDER = ['session_connections', 'client_requests',
17 - 'all_time_hit_rate', 'current_poll_hit_rate', 'cached_objects_expired', 'cached_objects_nuked',
18 - 'threads_total', 'threads_statistics', 'threads_queue_len',
19 - 'backend_connections', 'backend_requests',
20 - 'esi_statistics',
21 - 'memory_usage',
22 - 'uptime']
16 +ORDER = [
17 + 'session_connections',
18 + 'client_requests',
19 + 'all_time_hit_rate',
20 + 'current_poll_hit_rate',
21 + 'cached_objects_expired',
22 + 'cached_objects_nuked',
23 + 'threads_total',
24 + 'threads_statistics',
25 + 'threads_queue_len',
26 + 'backend_connections',
27 + 'backend_requests',
28 + 'esi_statistics',
29 + 'memory_usage',
30 + 'uptime'
31 +]
32
33 CHARTS = {
34 'session_connections': {
python.d/w1sensor.chart.py
+3 -14
@@ -18,22 +18,10 @@ RE_TEMP = re.compile(r' t=(\d+)')
18
19 ORDER = ['temp']
20
21 -# id: {
22 -# 'options': [name, title, units, family (heading 2), context, charttype],
23 -# 'lines': [
24 -# [unique_dimension_name, name, algorithm, multiplier, divisor]
25 -# ]}
21 CHARTS = {
22 'temp': {
28 - 'options': [
29 - None,
30 - '1-Wire Temperature Sensor',
31 - 'Celsius',
32 - 'Temperature',
33 - 'w1sensor.temp',
34 - 'line'
35 - ],
36 - # lines are constructed from check
23 + 'options': [None, '1-Wire Temperature Sensor', 'Celsius', 'Temperature', 'w1sensor.temp', 'line'],
24 + 'lines': []
25 }
26 }
27
@@ -47,6 +35,7 @@ THERM_FAMILY = {
35 '42': 'W1_THERM_DS28EA00',
36 }
37
38 +
39 class Service(SimpleService):
40 """Provide netdata service for 1-Wire sensors"""
41 def __init__(self, configuration=None, name=None):
python.d/web_log.chart.py
+217 -105
@@ -27,15 +27,43 @@ from bases.FrameworkServices.LogService import LogService
27
28 ORDER_APACHE_CACHE = ['apache_cache']
29
30 -ORDER_WEB = ['response_statuses', 'response_codes', 'bandwidth',
31 - 'response_time', 'response_time_hist', 'response_time_upstream', 'response_time_upstream_hist',
32 - 'requests_per_url', 'requests_per_user_defined', 'http_method', 'vhost', 'port', 'http_version',
33 - 'requests_per_ipproto', 'clients', 'clients_all']
34 -
35 -ORDER_SQUID = ['squid_response_statuses', 'squid_response_codes', 'squid_detailed_response_codes',
36 - 'squid_method', 'squid_mime_type', 'squid_hier_code', 'squid_transport_methods',
37 - 'squid_transport_errors', 'squid_code', 'squid_handling_opts', 'squid_object_types',
38 - 'squid_cache_events', 'squid_bytes', 'squid_duration', 'squid_clients', 'squid_clients_all']
30 +ORDER_WEB = [
31 + 'response_statuses',
32 + 'response_codes',
33 + 'bandwidth',
34 + 'response_time',
35 + 'response_time_hist',
36 + 'response_time_upstream',
37 + 'response_time_upstream_hist',
38 + 'requests_per_url',
39 + 'requests_per_user_defined',
40 + 'http_method',
41 + 'vhost',
42 + 'port',
43 + 'http_version',
44 + 'requests_per_ipproto',
45 + 'clients',
46 + 'clients_all'
47 +]
48 +
49 +ORDER_SQUID = [
50 + 'squid_response_statuses',
51 + 'squid_response_codes',
52 + 'squid_detailed_response_codes',
53 + 'squid_method',
54 + 'squid_mime_type',
55 + 'squid_hier_code',
56 + 'squid_transport_methods',
57 + 'squid_transport_errors',
58 + 'squid_code',
59 + 'squid_handling_opts',
60 + 'squid_object_types',
61 + 'squid_cache_events',
62 + 'squid_bytes',
63 + 'squid_duration',
64 + 'squid_clients',
65 + 'squid_clients_all'
66 +]
67
68 CHARTS_WEB = {
69 'response_codes': {
@@ -48,24 +76,27 @@ CHARTS_WEB = {
76 ['1xx', None, 'incremental'],
77 ['0xx', 'other', 'incremental'],
78 ['unmatched', None, 'incremental']
51 - ]},
79 + ]
80 + },
81 'bandwidth': {
82 'options': [None, 'Bandwidth', 'kilobits/s', 'bandwidth', 'web_log.bandwidth', 'area'],
83 'lines': [
84 ['resp_length', 'received', 'incremental', 8, 1000],
85 ['bytes_sent', 'sent', 'incremental', -8, 1000]
57 - ]},
86 + ]
87 + },
88 'response_time': {
89 'options': [None, 'Processing Time', 'milliseconds', 'timings', 'web_log.response_time', 'area'],
90 'lines': [
91 ['resp_time_min', 'min', 'incremental', 1, 1000],
92 ['resp_time_max', 'max', 'incremental', 1, 1000],
93 ['resp_time_avg', 'avg', 'incremental', 1, 1000]
64 - ]},
94 + ]
95 + },
96 'response_time_hist': {
97 'options': [None, 'Processing Time Histogram', 'requests/s', 'timings', 'web_log.response_time_hist', 'line'],
67 - 'lines': [
68 - ]},
98 + 'lines': []
99 + },
100 'response_time_upstream': {
101 'options': [None, 'Processing Time Upstream', 'milliseconds', 'timings',
102 'web_log.response_time_upstream', 'area'],
@@ -73,74 +104,80 @@ CHARTS_WEB = {
104 ['resp_time_upstream_min', 'min', 'incremental', 1, 1000],
105 ['resp_time_upstream_max', 'max', 'incremental', 1, 1000],
106 ['resp_time_upstream_avg', 'avg', 'incremental', 1, 1000]
76 - ]},
107 + ]
108 + },
109 'response_time_upstream_hist': {
110 'options': [None, 'Processing Time Histogram', 'requests/s', 'timings',
111 'web_log.response_time_upstream_hist', 'line'],
80 - 'lines': [
81 - ]},
112 + 'lines': []
113 + },
114 'clients': {
115 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'clients', 'web_log.clients', 'stacked'],
116 'lines': [
117 ['unique_cur_ipv4', 'ipv4', 'incremental', 1, 1],
118 ['unique_cur_ipv6', 'ipv6', 'incremental', 1, 1]
87 - ]},
119 + ]
120 + },
121 'clients_all': {
122 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'clients', 'web_log.clients_all', 'stacked'],
123 'lines': [
124 ['unique_tot_ipv4', 'ipv4', 'absolute', 1, 1],
125 ['unique_tot_ipv6', 'ipv6', 'absolute', 1, 1]
93 - ]},
126 + ]
127 + },
128 'http_method': {
129 'options': [None, 'Requests Per HTTP Method', 'requests/s', 'http methods', 'web_log.http_method', 'stacked'],
130 'lines': [
131 ['GET', 'GET', 'incremental', 1, 1]
98 - ]},
132 + ]
133 + },
134 'http_version': {
135 'options': [None, 'Requests Per HTTP Version', 'requests/s', 'http versions',
136 'web_log.http_version', 'stacked'],
102 - 'lines': []},
137 + 'lines': []
138 + },
139 'requests_per_ipproto': {
140 'options': [None, 'Requests Per IP Protocol', 'requests/s', 'ip protocols', 'web_log.requests_per_ipproto',
141 'stacked'],
142 'lines': [
143 ['req_ipv4', 'ipv4', 'incremental', 1, 1],
144 ['req_ipv6', 'ipv6', 'incremental', 1, 1]
109 - ]},
145 + ]
146 + },
147 'response_statuses': {
111 - 'options': [None, 'Response Statuses', 'requests/s', 'responses', 'web_log.response_statuses',
112 - 'stacked'],
148 + 'options': [None, 'Response Statuses', 'requests/s', 'responses', 'web_log.response_statuses', 'stacked'],
149 'lines': [
150 ['successful_requests', 'success', 'incremental', 1, 1],
151 ['server_errors', 'error', 'incremental', 1, 1],
152 ['redirects', 'redirect', 'incremental', 1, 1],
153 ['bad_requests', 'bad', 'incremental', 1, 1],
154 ['other_requests', 'other', 'incremental', 1, 1]
119 - ]},
155 + ]
156 + },
157 'requests_per_url': {
121 - 'options': [None, 'Requests Per Url', 'requests/s', 'urls', 'web_log.requests_per_url',
122 - 'stacked'],
158 + 'options': [None, 'Requests Per Url', 'requests/s', 'urls', 'web_log.requests_per_url', 'stacked'],
159 'lines': [
160 ['url_pattern_other', 'other', 'incremental', 1, 1]
125 - ]},
161 + ]
162 + },
163 'requests_per_user_defined': {
164 'options': [None, 'Requests Per User Defined Pattern', 'requests/s', 'user defined',
165 'web_log.requests_per_user_defined', 'stacked'],
166 'lines': [
167 ['user_pattern_other', 'other', 'incremental', 1, 1]
131 - ]},
168 + ]
169 + },
170 'port': {
133 - 'options': [None, 'Requests Per Port', 'requests/s', 'port',
134 - 'web_log.port', 'stacked'],
171 + 'options': [None, 'Requests Per Port', 'requests/s', 'port', 'web_log.port', 'stacked'],
172 'lines': [
173 ['port_80', 'http', 'incremental', 1, 1],
174 ['port_443', 'https', 'incremental', 1, 1]
138 - ]},
175 + ]
176 + },
177 'vhost': {
140 - 'options': [None, 'Requests Per Vhost', 'requests/s', 'vhost',
141 - 'web_log.vhost', 'stacked'],
142 - 'lines': [
143 - ]}
178 + 'options': [None, 'Requests Per Vhost', 'requests/s', 'vhost', 'web_log.vhost', 'stacked'],
179 + 'lines': []
180 + }
181 }
182
183 CHARTS_APACHE_CACHE = {
@@ -148,10 +185,11 @@ CHARTS_APACHE_CACHE = {
185 'options': [None, 'Apache Cached Responses', 'percent cached', 'cached', 'web_log.apache_cache_cache',
186 'stacked'],
187 'lines': [
151 - ["hit", 'cache', "percentage-of-absolute-row"],
152 - ["miss", None, "percentage-of-absolute-row"],
153 - ["other", None, "percentage-of-absolute-row"]
154 - ]}
188 + ['hit', 'cache', 'percentage-of-absolute-row'],
189 + ['miss', None, 'percentage-of-absolute-row'],
190 + ['other', None, 'percentage-of-absolute-row']
191 + ]
192 + }
193 }
194
195 CHARTS_SQUID = {
@@ -162,13 +200,15 @@ CHARTS_SQUID = {
200 ['duration_min', 'min', 'incremental', 1, 1000],
201 ['duration_max', 'max', 'incremental', 1, 1000],
202 ['duration_avg', 'avg', 'incremental', 1, 1000]
165 - ]},
203 + ]
204 + },
205 'squid_bytes': {
206 'options': [None, 'Amount Of Data Delivered To The Clients',
207 'kilobits/s', 'squid_bandwidth', 'web_log.squid_bytes', 'area'],
208 'lines': [
209 ['bytes', 'sent', 'incremental', 8, 1000]
171 - ]},
210 + ]
211 + },
212 'squid_response_statuses': {
213 'options': [None, 'Response Statuses', 'responses/s', 'squid_responses', 'web_log.squid_response_statuses',
214 'stacked'],
@@ -178,7 +218,8 @@ CHARTS_SQUID = {
218 ['redirects', 'redirect', 'incremental', 1, 1],
219 ['bad_requests', 'bad', 'incremental', 1, 1],
220 ['other_requests', 'other', 'incremental', 1, 1]
181 - ]},
221 + ]
222 + },
223 'squid_response_codes': {
224 'options': [None, 'Response Codes', 'responses/s', 'squid_responses',
225 'web_log.squid_response_codes', 'stacked'],
@@ -191,91 +232,114 @@ CHARTS_SQUID = {
232 ['0xx', None, 'incremental'],
233 ['other', None, 'incremental'],
234 ['unmatched', None, 'incremental']
194 - ]},
235 + ]
236 + },
237 'squid_code': {
238 'options': [None, 'Responses Per Cache Result Of The Request',
239 'requests/s', 'squid_squid_cache', 'web_log.squid_code', 'stacked'],
198 - 'lines': [
199 - ]},
240 + 'lines': []
241 + },
242 'squid_detailed_response_codes': {
243 'options': [None, 'Detailed Response Codes',
244 'responses/s', 'squid_responses', 'web_log.squid_detailed_response_codes', 'stacked'],
203 - 'lines': [
204 - ]},
245 + 'lines': []
246 + },
247 'squid_hier_code': {
248 'options': [None, 'Responses Per Hierarchy Code',
249 'requests/s', 'squid_hierarchy', 'web_log.squid_hier_code', 'stacked'],
208 - 'lines': [
209 - ]},
250 + 'lines': []
251 + },
252 'squid_method': {
253 'options': [None, 'Requests Per Method',
254 'requests/s', 'squid_requests', 'web_log.squid_method', 'stacked'],
213 - 'lines': [
214 - ]},
255 + 'lines': []
256 + },
257 'squid_mime_type': {
258 'options': [None, 'Requests Per MIME Type',
259 'requests/s', 'squid_requests', 'web_log.squid_mime_type', 'stacked'],
218 - 'lines': [
219 - ]},
260 + 'lines': []
261 + },
262 'squid_clients': {
263 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'squid_clients',
264 'web_log.squid_clients', 'stacked'],
265 'lines': [
266 ['unique_ipv4', 'ipv4', 'incremental'],
267 ['unique_ipv6', 'ipv6', 'incremental']
226 - ]},
268 + ]
269 + },
270 'squid_clients_all': {
271 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'squid_clients',
272 'web_log.squid_clients_all', 'stacked'],
273 'lines': [
274 ['unique_tot_ipv4', 'ipv4', 'absolute'],
275 ['unique_tot_ipv6', 'ipv6', 'absolute']
233 - ]},
276 + ]
277 + },
278 'squid_transport_methods': {
279 'options': [None, 'Transport Methods', 'requests/s', 'squid_squid_transport',
280 'web_log.squid_transport_methods', 'stacked'],
237 - 'lines': [
238 - ]},
281 + 'lines': []
282 + },
283 'squid_transport_errors': {
284 'options': [None, 'Transport Errors', 'requests/s', 'squid_squid_transport',
285 'web_log.squid_transport_errors', 'stacked'],
242 - 'lines': [
243 - ]},
286 + 'lines': []
287 + },
288 'squid_handling_opts': {
289 'options': [None, 'Handling Opts', 'requests/s', 'squid_squid_cache',
290 'web_log.squid_handling_opts', 'stacked'],
247 - 'lines': [
248 - ]},
291 + 'lines': []
292 + },
293 'squid_object_types': {
294 'options': [None, 'Object Types', 'objects/s', 'squid_squid_cache',
295 'web_log.squid_object_types', 'stacked'],
252 - 'lines': [
253 - ]},
296 + 'lines': []
297 + },
298 'squid_cache_events': {
299 'options': [None, 'Cache Events', 'events/s', 'squid_squid_cache',
300 'web_log.squid_cache_events', 'stacked'],
257 - 'lines': [
258 - ]}
301 + 'lines': []
302 + }
303 }
304
305 NAMED_PATTERN = namedtuple('PATTERN', ['description', 'func'])
306
307 DET_RESP_AGGR = ['', '_1xx', '_2xx', '_3xx', '_4xx', '_5xx', '_Other']
308
265 -SQUID_CODES = dict(TCP='squid_transport_methods', UDP='squid_transport_methods', NONE='squid_transport_methods',
266 - CLIENT='squid_handling_opts', IMS='squid_handling_opts', ASYNC='squid_handling_opts',
267 - SWAPFAIL='squid_handling_opts', REFRESH='squid_handling_opts', SHARED='squid_handling_opts',
268 - REPLY='squid_handling_opts', NEGATIVE='squid_object_types', STALE='squid_object_types',
269 - OFFLINE='squid_object_types', INVALID='squid_object_types', FAIL='squid_object_types',
270 - MODIFIED='squid_object_types', UNMODIFIED='squid_object_types', REDIRECT='squid_object_types',
271 - HIT='squid_cache_events', MEM='squid_cache_events', MISS='squid_cache_events',
272 - DENIED='squid_cache_events', NOFETCH='squid_cache_events', TUNNEL='squid_cache_events',
273 - ABORTED='squid_transport_errors', TIMEOUT='squid_transport_errors')
309 +SQUID_CODES = {
310 + 'TCP': 'squid_transport_methods',
311 + 'UDP': 'squid_transport_methods',
312 + 'NONE': 'squid_transport_methods',
313 + 'CLIENT': 'squid_handling_opts',
314 + 'IMS': 'squid_handling_opts',
315 + 'ASYNC': 'squid_handling_opts',
316 + 'SWAPFAIL': 'squid_handling_opts',
317 + 'REFRESH': 'squid_handling_opts',
318 + 'SHARED': 'squid_handling_opts',
319 + 'REPLY': 'squid_handling_opts',
320 + 'NEGATIVE': 'squid_object_types',
321 + 'STALE': 'squid_object_types',
322 + 'OFFLINE': 'squid_object_types',
323 + 'INVALID': 'squid_object_types',
324 + 'FAIL': 'squid_object_types',
325 + 'MODIFIED': 'squid_object_types',
326 + 'UNMODIFIED': 'squid_object_types',
327 + 'REDIRECT': 'squid_object_types',
328 + 'HIT': 'squid_cache_events',
329 + 'MEM': 'squid_cache_events',
330 + 'MISS': 'squid_cache_events',
331 + 'DENIED': 'squid_cache_events',
332 + 'NOFETCH': 'squid_cache_events',
333 + 'TUNNEL': 'squid_cache_events',
334 + 'ABORTED': 'squid_transport_errors',
335 + 'TIMEOUT': 'squid_transport_errors'
336 +}
337
338 REQUEST_REGEX = re.compile(r'(?P<method>[A-Z]+) (?P<url>[^ ]+) [A-Z]+/(?P<http_version>\d(?:.\d)?)')
339
340 MIME_TYPES = ['application', 'audio', 'example', 'font', 'image', 'message', 'model', 'multipart', 'text', 'video']
341
342 +
343 class Service(LogService):
344 def __init__(self, configuration=None, name=None):
345 """
@@ -301,7 +365,7 @@ class Service(LogService):
365 log_types = dict(web=Web, apache_cache=ApacheCache, squid=Squid)
366
367 if log_type not in log_types:
304 - self.error("bad log type {log_type}. Supported types: {types}".format(log_type=log_type,
368 + self.error('bad log type {log_type}. Supported types: {types}'.format(log_type=log_type,
369 types=log_types.keys()))
370 return False
371
@@ -335,12 +399,35 @@ class Web:
399 self.definitions = deepcopy(CHARTS_WEB)
400 self.pre_filter = check_patterns('filter', self.configuration.get('filter'))
401 self.storage = dict()
338 - self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,
339 - 'resp_time_avg': 0, 'resp_time_upstream_min': 0, 'resp_time_upstream_max': 0,
340 - 'resp_time_upstream_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,
341 - '5xx': 0, '3xx': 0, '4xx': 0, '1xx': 0, '0xx': 0, 'unmatched': 0, 'req_ipv4': 0,
342 - 'req_ipv6': 0, 'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0, 'successful_requests': 0,
343 - 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0, 'GET': 0}
402 + self.data = {
403 + 'bytes_sent': 0,
404 + 'resp_length': 0,
405 + 'resp_time_min': 0,
406 + 'resp_time_max': 0,
407 + 'resp_time_avg': 0,
408 + 'resp_time_upstream_min': 0,
409 + 'resp_time_upstream_max': 0,
410 + 'resp_time_upstream_avg': 0,
411 + 'unique_cur_ipv4': 0,
412 + 'unique_cur_ipv6': 0,
413 + '2xx': 0,
414 + '5xx': 0,
415 + '3xx': 0,
416 + '4xx': 0,
417 + '1xx': 0,
418 + '0xx': 0,
419 + 'unmatched': 0,
420 + 'req_ipv4': 0,
421 + 'req_ipv6': 0,
422 + 'unique_tot_ipv4': 0,
423 + 'unique_tot_ipv6': 0,
424 + 'successful_requests': 0,
425 + 'redirects': 0,
426 + 'bad_requests': 0,
427 + 'server_errors': 0,
428 + 'other_requests': 0,
429 + 'GET': 0
430 + }
431
432 def __getattr__(self, item):
433 return getattr(self.service, item)
@@ -391,15 +478,15 @@ class Web:
478 hist_lines = self.definitions['response_time_hist']['lines']
479 upstream_hist_lines = self.definitions['response_time_upstream_hist']['lines']
480 for i, le in enumerate(histogram):
394 - hist_key = "response_time_hist_%d" % i
395 - upstream_hist_key = "response_time_upstream_hist_%d" % i
481 + hist_key = 'response_time_hist_%d' % i
482 + upstream_hist_key = 'response_time_upstream_hist_%d' % i
483 hist_lines.append([hist_key, str(le), 'incremental', 1, 1])
484 upstream_hist_lines.append([upstream_hist_key, str(le), 'incremental', 1, 1])
485
399 - hist_lines.append(["response_time_hist_%d" % len(histogram), '+Inf', 'incremental', 1, 1])
400 - upstream_hist_lines.append(["response_time_upstream_hist_%d" % len(histogram), '+Inf', 'incremental', 1, 1])
486 + hist_lines.append(['response_time_hist_%d' % len(histogram), '+Inf', 'incremental', 1, 1])
487 + upstream_hist_lines.append(['response_time_upstream_hist_%d' % len(histogram), '+Inf', 'incremental', 1, 1])
488 elif histogram is not None:
402 - self.error("expect histogram list, but was {0}".format(type(histogram)))
489 + self.error('expect histogram list, but was {0}'.format(type(histogram)))
490
491 if not self.configuration.get('all_time', True):
492 self.order.remove('clients_all')
@@ -413,10 +500,11 @@ class Web:
500
501 for code in codes:
502 self.order.append('detailed_response_codes%s' % code)
416 - self.definitions['detailed_response_codes%s' % code] \
417 - = {'options': [None, 'Detailed Response Codes %s' % code[1:], 'requests/s', 'responses',
418 - 'web_log.detailed_response_codes%s' % code, 'stacked'],
419 - 'lines': []}
503 + self.definitions['detailed_response_codes%s' % code] = {
504 + 'options': [None, 'Detailed Response Codes %s' % code[1:], 'requests/s', 'responses',
505 + 'web_log.detailed_response_codes%s' % code, 'stacked'],
506 + 'lines': []
507 + }
508
509 # Add 'requests_per_url' chart if specified in the configuration
510 if self.storage['url_pattern']:
@@ -517,8 +605,8 @@ class Web:
605 buckets = self.storage['buckets']
606 upstream_buckets = self.storage['upstream_buckets']
607 for i in range(0, len(self.storage['bucket_index'])):
520 - hist_key = "response_time_hist_%d" % i
521 - upstream_hist_key = "response_time_upstream_hist_%d" % i
608 + hist_key = 'response_time_hist_%d' % i
609 + upstream_hist_key = 'response_time_upstream_hist_%d' % i
610 self.data[hist_key] = buckets[i]
611 self.data[upstream_hist_key] = upstream_buckets[i]
612
@@ -823,12 +911,29 @@ class Squid:
911 self.definitions = CHARTS_SQUID
912 self.pre_filter = check_patterns('filter', self.configuration.get('filter'))
913 self.storage = dict()
826 - self.data = {'duration_max': 0, 'duration_avg': 0, 'duration_min': 0, 'bytes': 0,
827 - '0xx': 0, '1xx': 0, '2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0,
828 - 'other': 0, 'unmatched': 0, 'unique_ipv4': 0, 'unique_ipv6': 0,
829 - 'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0, 'successful_requests': 0,
830 - 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0
831 - }
914 + self.data = {
915 + 'duration_max': 0,
916 + 'duration_avg': 0,
917 + 'duration_min': 0,
918 + 'bytes': 0,
919 + '0xx': 0,
920 + '1xx': 0,
921 + '2xx': 0,
922 + '3xx': 0,
923 + '4xx': 0,
924 + '5xx': 0,
925 + 'other': 0,
926 + 'unmatched': 0,
927 + 'unique_ipv4': 0,
928 + 'unique_ipv6': 0,
929 + 'unique_tot_ipv4': 0,
930 + 'unique_tot_ipv6': 0,
931 + 'successful_requests': 0,
932 + 'redirects': 0,
933 + 'bad_requests': 0,
934 + 'server_errors': 0,
935 + 'other_requests': 0
936 + }
937
938 def __getattr__(self, item):
939 return getattr(self.service, item)
@@ -854,22 +959,27 @@ class Squid:
959 self.error('Regex not matches (%s)' % self.storage['regex'].pattern)
960 return False
961 self.storage['dynamic'] = {
857 - 'http_code':
858 - {'chart': 'squid_detailed_response_codes',
962 + 'http_code': {
963 + 'chart': 'squid_detailed_response_codes',
964 'func_dim_id': None,
860 - 'func_dim': None},
965 + 'func_dim': None
966 + },
967 'hier_code': {
968 'chart': 'squid_hier_code',
969 'func_dim_id': None,
864 - 'func_dim': lambda v: v.replace('HIER_', '')},
970 + 'func_dim': lambda v: v.replace('HIER_', '')
971 + },
972 'method': {
973 'chart': 'squid_method',
974 'func_dim_id': None,
868 - 'func_dim': None},
975 + 'func_dim': None
976 + },
977 'mime_type': {
978 'chart': 'squid_mime_type',
979 'func_dim_id': lambda v: str.lower(v) if str.lower(v) in MIME_TYPES else 'unknown',
872 - 'func_dim': None}}
980 + 'func_dim': None
981 + }
982 + }
983 if not self.configuration.get('all_time', True):
984 self.order.remove('squid_clients_all')
985 return True
@@ -986,6 +1096,7 @@ def get_timings(timings, time):
1096 timings['summary'] += time
1097 timings['count'] += 1
1098
1099 +
1100 def get_hist(index, buckets, time):
1101 """
1102 :param index: histogram index (Ex. [10, 50, 100, 150, ...])
@@ -999,6 +1110,7 @@ def get_hist(index, buckets, time):
1110 else:
1111 break
1112
1113 +
1114 def address_not_in_pool(pool, address, pool_size):
1115 """
1116 :param pool: list of ip addresses