web_log plugin: some fixes suggested by @ktsaou
Ilya committed
May 30, 2017 at 17:51 UTC
77b2a137310870bd76b1d0483a52b27d93624adc
1 file changed
+83
-62
python.d/web_log.chart.py
+83
-62
@@ -13,15 +13,16 @@ import msg
13
priority = 60000
14
retries = 60
15
16
-ORDER_APACHE_CACHE = ['cache']
16
+ORDER_APACHE_CACHE = ['apache_cache']
17
18
ORDER_WEB = ['response_statuses', 'response_codes', 'bandwidth', 'response_time', 'requests_per_url',
19
'requests_per_user_defined', 'http_method', 'http_version', 'requests_per_ipproto',
20
'clients', 'clients_all']
21
22
-ORDER_SQUID = ['response_statuses', 'response_codes', 'detailed_response_codes', 'squid_code',
23
- 'hier_code', 'transport_methods', 'transport_errors', 'handling_opts', 'object_types',
24
- 'cache_events', 'bytes', 'duration', 'method', 'clients']
22
+ORDER_SQUID = ['squid_response_statuses', 'squid_response_codes', 'squid_detailed_response_codes',
23
+ 'squid_method', 'squid_hier_code', 'squid_transport_methods', 'squid_transport_errors',
24
+ 'squid_code', 'squid_handling_opts', 'squid_object_types', 'squid_cache_events', 'squid_bytes',
25
+ 'squid_duration', 'squid_clients', 'squid_clients_all']
26
27
CHARTS_WEB = {
28
'response_codes': {
@@ -36,10 +37,10 @@ CHARTS_WEB = {
37
['unmatched', None, 'incremental']
38
]},
39
'bandwidth': {
39
- 'options': [None, 'Bandwidth', 'KB/s', 'bandwidth', 'web_log.bandwidth', 'area'],
40
+ 'options': [None, 'Bandwidth', 'Kb/s', 'bandwidth', 'web_log.bandwidth', 'area'],
41
'lines': [
41
- ['resp_length', 'received', 'incremental', 1, 1024],
42
- ['bytes_sent', 'sent', 'incremental', -1, 1024]
42
+ ['resp_length', 'received', 'incremental', 8, 1000],
43
+ ['bytes_sent', 'sent', 'incremental', -8, 1000]
44
]},
45
'response_time': {
46
'options': [None, 'Processing Time', 'milliseconds', 'timings', 'web_log.response_time', 'area'],
@@ -101,7 +102,7 @@ CHARTS_WEB = {
102
}
103
104
CHARTS_APACHE_CACHE = {
104
- 'cache': {
105
+ 'apache_cache': {
106
'options': [None, 'Apache Cached Responses', 'percent cached', 'cached', 'web_log.apache_cache_cache',
107
'stacked'],
108
'lines': [
@@ -112,21 +113,21 @@ CHARTS_APACHE_CACHE = {
113
}
114
115
CHARTS_SQUID = {
115
- 'duration': {
116
+ 'squid_duration': {
117
'options': [None, 'Elapsed Time The Transaction Busied The Cache',
117
- 'seconds', 'timings', 'web_log.squid_duration', 'area'],
118
+ 'ms', 'timings', 'web_log.squid_duration', 'area'],
119
'lines': [
119
- ['duration_min', 'min', 'incremental', 1, 1000],
120
- ['duration_max', 'max', 'incremental', 1, 1000],
121
- ['duration_avg', 'avg', 'incremental', 1, 1000]
120
+ ['duration_min', 'min', 'incremental'],
121
+ ['duration_max', 'max', 'incremental'],
122
+ ['duration_avg', 'avg', 'incremental']
123
]},
123
- 'bytes': {
124
+ 'squid_bytes': {
125
'options': [None, 'Amount Of Data Delivered To The Clients',
125
- 'KB', 'bandwidth', 'web_log.squid_bytes', 'line'],
126
+ 'Kb/s', 'bandwidth', 'web_log.squid_bytes', 'line'],
127
'lines': [
127
- ['bytes', 'sent', 'incremental', 1, 1024]
128
+ ['bytes', 'sent', 'incremental', 8, 1000]
129
]},
129
- 'response_statuses': {
130
+ 'squid_response_statuses': {
131
'options': [None, 'Response Statuses', 'responses/s', 'responses', 'web_log.squid_response_statuses',
132
'stacked'],
133
'lines': [
@@ -135,7 +136,7 @@ CHARTS_SQUID = {
136
['redirects', 'redirect', 'incremental', 1, 1],
137
['bad_requests', 'bad', 'incremental', 1, 1]
138
]},
138
- 'response_codes': {
139
+ 'squid_response_codes': {
140
'options': [None, 'Response Codes', 'responses/s', 'responses', 'web_log.squid_response_codes', 'stacked'],
141
'lines': [
142
['2xx', None, 'incremental'],
@@ -149,52 +150,59 @@ CHARTS_SQUID = {
150
]},
151
'squid_code': {
152
'options': [None, 'Responses Per Cache Result Of The Request',
152
- 'responses/s', 'responses', 'web_log.squid_responses_squid_code', 'stacked'],
153
+ 'responses/s', 'squid_cache', 'web_log.squid_code', 'stacked'],
154
'lines': [
155
]},
155
- 'detailed_response_codes': {
156
+ 'squid_detailed_response_codes': {
157
'options': [None, 'Detailed Response Codes',
158
'responses/s', 'responses', 'web_log.squid_detailed_response_codes', 'stacked'],
159
'lines': [
160
]},
160
- 'hier_code': {
161
+ 'squid_hier_code': {
162
'options': [None, 'Responses Per Hierarchy Code',
162
- 'responses/s', 'hierarchy', 'web_log.squid_responses_hier_code', 'stacked'],
163
+ 'responses/s', 'hierarchy', 'web_log.squid_hier_code', 'stacked'],
164
'lines': [
165
]},
165
- 'method': {
166
- 'options': [None, 'Responses Per Request Method',
167
- 'responses/s', 'responses', 'web_log.squid_responses_request_method', 'stacked'],
166
+ 'squid_method': {
167
+ 'options': [None, 'Requests Per Method',
168
+ 'requests/s', 'requests', 'web_log.squid_method', 'stacked'],
169
'lines': [
170
]},
170
- 'clients': {
171
+ 'squid_clients': {
172
'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'clients',
173
'web_log.squid_clients', 'stacked'],
174
'lines': [
174
- ['unique_ipv4', 'ipv4', 'incremental', 1, 1],
175
- ['unique_ipv6', 'ipv6', 'incremental', 1, 1]
175
+ ['unique_ipv4', 'ipv4', 'incremental'],
176
+ ['unique_ipv6', 'ipv6', 'incremental']
177
]},
177
- 'transport_methods': {
178
+ 'squid_clients_all': {
179
+ 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'clients',
180
+ 'web_log.squid_clients_all', 'stacked'],
181
+ 'lines': [
182
+ ['unique_tot_ipv4', 'ipv4', 'absolute'],
183
+ ['unique_tot_ipv6', 'ipv6', 'absolute']
184
+ ]},
185
+ 'squid_transport_methods': {
186
'options': [None, 'Transport Methods', 'methods', 'squid_transport',
179
- 'web_log.squid_ transport_methods', 'stacked'],
187
+ 'web_log.squid_transport_methods', 'stacked'],
188
'lines': [
189
]},
182
- 'transport_errors': {
190
+ 'squid_transport_errors': {
191
'options': [None, 'Transport Errors', 'errors', 'squid_transport',
192
'web_log.squid_transport_errors', 'stacked'],
193
'lines': [
194
]},
187
- 'handling_opts': {
195
+ 'squid_handling_opts': {
196
'options': [None, 'Handling Opts', 'errors', 'squid_cache',
197
'web_log.squid_handling_opts', 'stacked'],
198
'lines': [
199
]},
192
- 'object_types': {
200
+ 'squid_object_types': {
201
'options': [None, 'Object Types', 'types', 'squid_cache',
202
'web_log.squid_object_types', 'stacked'],
203
'lines': [
204
]},
197
- 'cache_events': {
205
+ 'squid_cache_events': {
206
'options': [None, 'Cache Events', 'events', 'squid_cache',
207
'web_log.squid_cache_events', 'stacked'],
208
'lines': [
@@ -206,19 +214,19 @@ NAMED_PATTERN = namedtuple('PATTERN', ['description', 'pattern'])
214
DET_RESP_AGGR = ['', '_1xx', '_2xx', '_3xx', '_4xx', '_5xx', '_Other']
215
216
SQUID_DYNAMIC = dict(squid_code='squid_code',
209
- http_code='detailed_response_codes',
210
- hier_code='hier_code',
211
- method='method')
217
+ http_code='squid_detailed_response_codes',
218
+ hier_code='squid_hier_code',
219
+ method='squid_method')
220
213
-SQUID_CODES = dict(TCP='transport_methods', UDP='transport_methods', NONE='transport_methods',
214
- CLIENT='handling_opts', IMS='handling_opts', ASYNC='handling_opts',
215
- SWAPFAIL='handling_opts', REFRESH='handling_opts', SHARED='handling_opts',
216
- REPLY='handling_opts', NEGATIVE='object_types', STALE='object_types',
217
- OFFLINE='object_types', INVALID='object_types', FAIL='object_types',
218
- MODIFIED='object_types', UNMODIFIED='object_types', REDIRECT='object_types',
219
- HIT='cache_events', MEM='cache_events', MISS='cache_events',
220
- DENIED='cache_events', NOFETCH='cache_events', TUNNEL='cache_events',
221
- ABORTED='transport_errors', TIMEOUT='transport_errors')
221
+SQUID_CODES = dict(TCP='squid_transport_methods', UDP='squid_transport_methods', NONE='squid_transport_methods',
222
+ CLIENT='squid_handling_opts', IMS='squid_handling_opts', ASYNC='squid_handling_opts',
223
+ SWAPFAIL='squid_handling_opts', REFRESH='squid_handling_opts', SHARED='squid_handling_opts',
224
+ REPLY='squid_handling_opts', NEGATIVE='squid_object_types', STALE='squid_object_types',
225
+ OFFLINE='squid_object_types', INVALID='squid_object_types', FAIL='squid_object_types',
226
+ MODIFIED='squid_object_types', UNMODIFIED='squid_object_types', REDIRECT='squid_object_types',
227
+ HIT='squid_cache_events', MEM='squid_cache_events', MISS='squid_cache_events',
228
+ DENIED='squid_cache_events', NOFETCH='squid_cache_events', TUNNEL='squid_cache_events',
229
+ ABORTED='squid_transport_errors', TIMEOUT='squid_transport_errors')
230
231
232
class Service(LogService):
@@ -748,29 +756,33 @@ class Squid(Mixin):
756
self.conf = configuration
757
self.order = ORDER_SQUID
758
self.definitions = CHARTS_SQUID
751
- self.regex = re.compile(r'[0-9.]+\s+(?P<duration>[0-9]+)'
752
- r' (?P<client_address>[\da-f.:]+)'
753
- r' (?P<squid_code>[A-Z_]+)/'
754
- r'(?P<http_code>[0-9]+)'
755
- r' (?P<bytes>[0-9]+)'
756
- r' (?P<method>[A-Z]+)'
757
- r' (?P<url>[^ ]+)'
758
- r' (?P<user>[^ ]+)'
759
- r' (?P<hier_code>[A-Z_]+)/[0-9.-]+'
760
- r' (?P<mime_type>[^\n]+)')
759
+ self.storage = dict()
760
self.data = {'duration_max': 0, 'duration_avg': 0, 'duration_min': 0, 'bytes': 0,
761
'0xx': 0, '1xx': 0, '2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0,
762
'other': 0, 'unmatched': 0, 'unique_ipv4': 0, 'unique_ipv6': 0,
764
- 'successful_requests': 0, 'redirects': 0, 'bad_requests': 0, 'server_errors': 0
763
+ 'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0, 'successful_requests': 0,
764
+ 'redirects': 0, 'bad_requests': 0, 'server_errors': 0
765
}
766
767
def check(self):
768
last_line = self.get_last_line()
769
if not last_line:
770
return False
771
- match = self.regex.search(last_line)
771
+ self.storage['unique_all_time'] = list()
772
+ self.storage['regex'] = re.compile(r'[0-9.]+\s+(?P<duration>[0-9]+)'
773
+ r' (?P<client_address>[\da-f.:]+)'
774
+ r' (?P<squid_code>[A-Z_]+)/'
775
+ r'(?P<http_code>[0-9]+)'
776
+ r' (?P<bytes>[0-9]+)'
777
+ r' (?P<method>[A-Z]+)'
778
+ r' (?P<url>[^ ]+)'
779
+ r' (?P<user>[^ ]+)'
780
+ r' (?P<hier_code>[A-Z_]+)/[0-9.-]+'
781
+ r' (?P<mime_type>[^\n]+)')
782
+
783
+ match = self.storage['regex'].search(last_line)
784
if not match:
773
- self.error('Regex not matches (%s)' % self.regex.pattern)
785
+ self.error('Regex not matches (%s)' % self.storage['regex'].pattern)
786
return False
787
return True
788
@@ -781,10 +793,11 @@ class Squid(Mixin):
793
unique_ip, duration = set(), list()
794
795
for row in raw_data:
784
- match = self.regex.search(row)
796
+ match = self.storage['regex'].search(row)
797
if match:
798
match = match.groupdict()
787
- duration.append(match['duration'])
799
+ if 'DENIED' not in match['squid_code']:
800
+ duration.append(match['duration'])
801
try:
802
self.data[match['http_code'][0] + 'xx'] += 1
803
except KeyError:
@@ -797,14 +810,22 @@ class Squid(Mixin):
810
self.data['bytes'] += int(match['bytes'])
811
812
proto = 'ipv4' if '.' in match['client_address'] else 'ipv6'
813
+ # unique clients ips
814
+ if address_not_in_pool(pool=self.storage['unique_all_time'],
815
+ address=match['client_address'],
816
+ pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):
817
+ self.data['unique_tot_' + proto] += 1
818
+
819
if match['client_address'] not in unique_ip:
820
self.data['unique_' + proto] += 1
821
unique_ip.add(match['client_address'])
822
823
for key, chart_key in SQUID_DYNAMIC.items():
824
if match[key] not in self.data:
825
+ dimension = match[key].replace('HIER_', '')
826
self.add_new_dimension(dimension_id=match[key],
807
- chart_key=chart_key)
827
+ chart_key=chart_key,
828
+ dimension=dimension)
829
else:
830
self.data[match[key]] += 1
831
else: