web_log plugin: support for apache non default (+ %I %D) added
Ilya committed
Feb 9, 2017 at 17:27 UTC
bdfb0554dfb518c0c74e7880dd36b1da47031587
1 file changed
+62
-20
python.d/web_log.chart.py
+62
-20
@@ -8,6 +8,11 @@ import bisect
8
from os import access, R_OK
9
from os.path import getsize
10
from collections import defaultdict, namedtuple
11
+from copy import deepcopy
12
+try:
13
+ from itertools import zip_longest
14
+except ImportError:
15
+ from itertools import izip_longest as zip_longest
16
17
priority = 60000
18
retries = 60
@@ -67,15 +72,7 @@ class Service(LogService):
72
self.detailed_response_codes = self.configuration.get('detailed_response_codes', True)
73
self.all_time = self.configuration.get('all_time', True)
74
self.url_pattern = self.configuration.get('categories') # dict
70
- # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code
71
- # 5. Bytes sent 6. Response length 7. Response process time
72
- self.regex = re.compile(r'([\da-f.:]+)'
73
- r' -.*?"([A-Z]+)'
74
- r' (.*?)"'
75
- r' ([1-9]\d{2})'
76
- r' (\d+)'
77
- r' (\d+)?'
78
- r' ?([\d.]+)?')
75
+ self.regex = None
76
# sorted list of unique IPs
77
self.unique_all_time = list()
78
# dict for values that should not be zeroed every poll
@@ -111,16 +108,58 @@ class Service(LogService):
108
last_line = logs.readline().decode(encoding='utf-8')
109
110
# Parse last line
114
- parsed_line = self.regex.findall(last_line)
111
+ parsed_line, regex_name = self.find_regex(last_line)
112
if not parsed_line:
113
self.error('Can\'t parse output')
114
return False
115
119
- # parsed_line[0][6] - response process time
120
- self.create_charts(parsed_line[0][6])
116
+ self.create_charts(parsed_line[0], regex_name)
117
+ if len(parsed_line[0]) == 5:
118
+ self.info('Not all data collected. You need to modify LogFormat.')
119
return True
120
123
- def create_charts(self, parsed_line):
121
+ def find_regex(self, last_line):
122
+ # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code
123
+ # 5. Bytes sent 6. Response length 7. Response process time
124
+ default = re.compile(r'([\da-f.:]+)'
125
+ r' -.*?"([A-Z]+)'
126
+ r' (.*?)"'
127
+ r' ([1-9]\d{2})'
128
+ r' (\d+)')
129
+
130
+ apache_extended = re.compile(r'([\da-f.:]+)'
131
+ r' -.*?"([A-Z]+)'
132
+ r' (.*?)"'
133
+ r' ([1-9]\d{2})'
134
+ r' (\d+)'
135
+ r' (\d+)'
136
+ r' (\d+) ')
137
+
138
+ nginx_extended = re.compile(r'([\da-f.:]+)'
139
+ r' -.*?"([A-Z]+)'
140
+ r' (.*?)"'
141
+ r' ([1-9]\d{2})'
142
+ r' (\d+)'
143
+ r' (\d+)'
144
+ r' ([\d.]+) ')
145
+
146
+ regex_function = zip([apache_extended, nginx_extended, default],
147
+ [lambda x: x, lambda x: x * 1000, lambda x: x],
148
+ ['apache_extended', 'nginx_extended', 'default'])
149
+
150
+ for regex, function, name in regex_function:
151
+ if regex.search(last_line):
152
+ self.regex = regex
153
+ self.resp_time_func = function
154
+ regex_name = name
155
+ break
156
+
157
+ if self.regex:
158
+ return self.regex.findall(last_line), regex_name
159
+ else:
160
+ return None, None
161
+
162
+ def create_charts(self, parsed_line, regex_name):
163
def find_job_name(override_name, name):
164
add_to_name = override_name or name
165
if add_to_name:
@@ -136,11 +175,15 @@ class Service(LogService):
175
' "" "HTTP Methods" requests/s requests' \
176
' web_log.http_method stacked 2 %s\n' % (job_name, self.update_every)
177
self.order = ORDER[:]
139
- self.definitions = CHARTS
178
+ self.definitions = deepcopy(CHARTS)
179
+ if 'apache' in regex_name:
180
+ self.definitions['response_time']['lines'][0][4] = 1000
181
+ self.definitions['response_time']['lines'][1][4] = 1000
182
+ self.definitions['response_time']['lines'][2][4] = 1000
183
184
# Remove 'request_time' chart from ORDER if request_time not in logs
142
- if parsed_line == '':
143
- self.order.remove('request_time')
185
+ if len(parsed_line) < 7:
186
+ self.order.remove('response_time')
187
# Remove 'clients_all' chart from ORDER if specified in the configuration
188
if not self.all_time:
189
self.order.remove('clients_all')
@@ -193,7 +236,7 @@ class Service(LogService):
236
for line in raw:
237
match = self.regex.findall(line)
238
if match:
196
- match_dict = dict(zip('address method url code sent resp_length resp_time'.split(), match[0]))
239
+ match_dict = dict(zip_longest('address method url code sent resp_length resp_time'.split(), match[0]))
240
try:
241
code = ''.join([match_dict['code'][0], 'xx'])
242
to_netdata[code] += 1
@@ -210,9 +253,9 @@ class Service(LogService):
253
254
to_netdata['bytes_sent'] += int(match_dict['sent'])
255
213
- if match_dict['resp_length'] != '' and match_dict['resp_time'] != '':
256
+ if match_dict['resp_length'] and match_dict['resp_time']:
257
to_netdata['resp_length'] += int(match_dict['resp_length'])
215
- resp_time = float(match_dict['resp_time']) * 1000
258
+ resp_time = self.resp_time_func(float(match_dict['resp_time']))
259
bisect.insort_left(request_time, resp_time)
260
request_counter['count'] += 1
261
request_counter['sum'] += resp_time
@@ -236,7 +279,6 @@ class Service(LogService):
279
to_netdata['resp_time_min'] = request_time[0]
280
to_netdata['resp_time_avg'] = float(request_counter['sum']) / request_counter['count']
281
to_netdata['resp_time_max'] = request_time[-1]
239
-
282
to_netdata.update(self.storage)
283
to_netdata.update(default_dict)
284
return to_netdata