web_log pugin: context renamed, ipv6 address support added, unmatched dim for response codes chart added
Ilya committed
Feb 9, 2017 at 11:21 UTC
8848f136c475ea59f654a96cde9ca08261d4e361
1 file changed
+38
-19
python.d/web_log.chart.py
+38
-19
@@ -12,40 +12,43 @@ from collections import defaultdict, namedtuple
12
priority = 60000
13
retries = 60
14
15
-ORDER = ['response_codes', 'request_time', 'requests_per_url', 'http_method', 'bandwidth', 'clients_cur', 'clients_all']
15
+ORDER = ['response_codes', 'response_time', 'requests_per_url', 'http_method', 'bandwidth', 'clients', 'clients_all']
16
CHARTS = {
17
'response_codes': {
18
- 'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response', 'stacked'],
18
+ 'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response_codes', 'stacked'],
19
'lines': [
20
['2xx', '2xx', 'absolute'],
21
['5xx', '5xx', 'absolute'],
22
['3xx', '3xx', 'absolute'],
23
['4xx', '4xx', 'absolute'],
24
['1xx', '1xx', 'absolute'],
25
- ['0xx', 'other', 'absolute']
25
+ ['0xx', 'other', 'absolute'],
26
+ ['unmatched', 'unmatched', 'absolute']
27
]},
28
'bandwidth': {
28
- 'options': [None, 'Bandwidth', 'KB/s', 'bandwidth', 'web_log.bw', 'area'],
29
+ 'options': [None, 'Bandwidth', 'KB/s', 'bandwidth', 'web_log.bandwidth', 'area'],
30
'lines': [
31
['resp_length', 'received', 'absolute', 1, 1024],
32
['bytes_sent', 'sent', 'absolute', -1, 1024]
33
]},
33
- 'request_time': {
34
- 'options': [None, 'Processing Time', 'milliseconds', 'timings', 'web_log.request', 'area'],
34
+ 'response_time': {
35
+ 'options': [None, 'Processing Time', 'milliseconds', 'timings', 'web_log.response_time', 'area'],
36
'lines': [
37
['resp_time_min', 'min', 'absolute', 1, 1],
38
['resp_time_max', 'max', 'absolute', 1, 1],
39
['resp_time_avg', 'avg', 'absolute', 1, 1]
40
]},
40
- 'clients_cur': {
41
+ 'clients': {
42
'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients', 'line'],
43
'lines': [
43
- ['unique_cur', 'clients', 'absolute', 1, 1]
44
+ ['unique_cur_ipv4', 'ipv4', 'absolute', 1, 1],
45
+ ['unique_cur_ipv6', 'ipv6', 'absolute', 1, 1]
46
]},
47
'clients_all': {
46
- 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients', 'line'],
48
+ 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients_all', 'line'],
49
'lines': [
48
- ['unique_tot', 'clients', 'absolute', 1, 1]
50
+ ['unique_tot_ipv4', 'ipv4', 'absolute', 1, 1],
51
+ ['unique_tot_ipv6', 'ipv6', 'absolute', 1, 1]
52
]},
53
'http_method': {
54
'options': [None, 'Requests Per HTTP Method', 'requests/s', 'requests', 'web_log.http_method', 'stacked'],
@@ -66,16 +69,22 @@ class Service(LogService):
69
self.url_pattern = self.configuration.get('categories') # dict
70
# REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code
71
# 5. Bytes sent 6. Response length 7. Response process time
69
- #self.regex = re.compile(r'(\d{1,3}(?:\.\d{1,3}){3}) .*?"([A-Z]+) (.*?)" ([1-9]\d{2}) (\d+) (\d+)? ?([\d.]+)? ?"(.*?)" "(.*?)"')
70
- self.regex = re.compile(r'(\d{1,3}(?:\.\d{1,3}){3}) .*?"([A-Z]+) (.*?)" ([1-9]\d{2}) (\d+) (\d+)? ?([\d.]+)?')
72
+ self.regex = re.compile(r'([\da-f.:]+)'
73
+ r' -.*?"([A-Z]+)'
74
+ r' (.*?)"'
75
+ r' ([1-9]\d{2})'
76
+ r' (\d+)'
77
+ r' (\d+)?'
78
+ r' ?([\d.]+)?')
79
# sorted list of unique IPs
80
self.unique_all_time = list()
81
# dict for values that should not be zeroed every poll
74
- self.storage = {'unique_tot': 0}
82
+ self.storage = {'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0}
83
# if there is no new logs this dict + self.storage returned to netdata
84
self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0,
77
- 'resp_time_max': 0, 'resp_time_avg': 0, 'unique_cur': 0,
78
- '2xx': 0, '5xx': 0, '3xx': 0, '4xx': 0, '1xx': 0, '0xx': 0}
85
+ 'resp_time_max': 0, 'resp_time_avg': 0, 'unique_cur_ipv4': 0,
86
+ 'unique_cur_ipv6': 0, '2xx': 0, '5xx': 0, '3xx': 0, '4xx': 0,
87
+ '1xx': 0, '0xx': 0, 'unmatched': 0}
88
89
def check(self):
90
if not self.log_path:
@@ -208,10 +217,20 @@ class Service(LogService):
217
request_counter['count'] += 1
218
request_counter['sum'] += resp_time
219
# unique clients ips
211
- if address_not_in_pool(self.unique_all_time, match_dict['address'], self.storage['unique_tot']):
212
- self.storage['unique_tot'] += 1
213
- if address_not_in_pool(unique_current, match_dict['address'], to_netdata['unique_cur']):
214
- to_netdata['unique_cur'] += 1
220
+ if address_not_in_pool(self.unique_all_time, match_dict['address'],
221
+ self.storage['unique_tot_ipv4'] + self.storage['unique_tot_ipv6']):
222
+ if '.' in match_dict['address']:
223
+ self.storage['unique_tot_ipv4'] += 1
224
+ else:
225
+ self.storage['unique_tot_ipv6'] += 1
226
+ if address_not_in_pool(unique_current, match_dict['address'],
227
+ to_netdata['unique_cur_ipv4'] + to_netdata['unique_cur_ipv6']):
228
+ if '.' in match_dict['address']:
229
+ to_netdata['unique_cur_ipv4'] += 1
230
+ else:
231
+ to_netdata['unique_cur_ipv6'] += 1
232
+ else:
233
+ to_netdata['unmatched'] += 1
234
# timings
235
if request_time:
236
to_netdata['resp_time_min'] = request_time[0]