web_log plugin: support for additional log format added
Ilya committed
Feb 12, 2017 at 15:07 UTC
fd04f1e347d3123c1e292bf30974b30b1d9efb5c
1 file changed
+53
-30
python.d/web_log.chart.py
+53
-30
@@ -126,9 +126,9 @@ class Service(LogService):
126
self.error('Can\'t parse %s' % self.log_path)
127
return False
128
129
- if regex_name.startswith('access_'):
129
+ if regex_name.startswith('acs_'):
130
self.create_access_charts(regex_name)
131
- if regex_name == 'access_default':
131
+ if regex_name == 'acs_default':
132
self.info('Not all data collected. You need to modify LogFormat.')
133
self._get_data = self._get_access_data
134
self.info('Used regex: %s' % regex_name)
@@ -148,33 +148,56 @@ class Service(LogService):
148
"""
149
# REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code
150
# 5. Bytes sent 6. Response length 7. Response process time
151
- access_default = re.compile(r'([\da-f.:]+)'
152
- r' -.*?"([A-Z]+)'
153
- r' (.*?)"'
154
- r' ([1-9]\d{2})'
155
- r' (\d+)')
156
-
157
- access_apache_ext = re.compile(r'([\da-f.:]+)'
158
- r' -.*?"([A-Z]+)'
159
- r' (.*?)"'
160
- r' ([1-9]\d{2})'
161
- r' (\d+)'
162
- r' (\d+)'
163
- r' (\d+) ')
164
-
165
- access_nginx_ext = re.compile(r'([\da-f.:]+)'
166
- r' -.*?"([A-Z]+)'
167
- r' (.*?)"'
168
- r' ([1-9]\d{2})'
169
- r' (\d+)'
170
- r' (\d+)'
171
- r' ([\d.]+) ')
172
-
173
- regex_function = zip([access_apache_ext, access_nginx_ext, access_default],
174
- [lambda x: x, lambda x: x * 1000000, lambda x: x],
175
- ['access_apache_ext', 'access_nginx_ext', 'access_default'])
151
+ acs_default = re.compile(r'([\da-f.:]+)'
152
+ r' -.*?"([A-Z]+)'
153
+ r' (.*?)"'
154
+ r' ([1-9]\d{2})'
155
+ r' (\d+|-)')
156
+
157
+ acs_apache_ext_insert = re.compile(r'([\da-f.:]+)'
158
+ r' -.*?"([A-Z]+)'
159
+ r' (.*?)"'
160
+ r' ([1-9]\d{2})'
161
+ r' (\d+|-)'
162
+ r' (\d+)'
163
+ r' (\d+) ')
164
+
165
+ acs_apache_ext_append = re.compile(r'([\da-f.:]+)'
166
+ r' -.*?"([A-Z]+)'
167
+ r' (.*?)"'
168
+ r' ([1-9]\d{2})'
169
+ r' (\d+|-)'
170
+ r' .*?'
171
+ r' (\d+)'
172
+ r' (\d+)'
173
+ r'(?: |$)')
174
+
175
+ acs_nginx_ext_insert = re.compile(r'([\da-f.:]+)'
176
+ r' -.*?"([A-Z]+)'
177
+ r' (.*?)"'
178
+ r' ([1-9]\d{2})'
179
+ r' (\d+)'
180
+ r' (\d+)'
181
+ r' (\d\.\d+) ')
182
+
183
+ acs_nginx_ext_append = re.compile(r'([\da-f.:]+)'
184
+ r' -.*?"([A-Z]+)'
185
+ r' (.*?)"'
186
+ r' ([1-9]\d{2})'
187
+ r' (\d+)'
188
+ r' .*?'
189
+ r' (\d+)'
190
+ r' (\d\.\d+)')
191
+
192
+ r_regex = [acs_apache_ext_insert, acs_apache_ext_append, acs_nginx_ext_insert,
193
+ acs_nginx_ext_append, acs_default]
194
+ r_function = [lambda x: x, lambda x: x, lambda x: x * 1000000, lambda x: x * 1000000, lambda x: x]
195
+ r_name = ['acs_apache_ext_insert', 'acs_apache_ext_append', 'acs_nginx_ext_insert',
196
+ 'acs_nginx_ext_append', 'acs_default']
197
+ regex_function_name = zip(r_regex, r_function, r_name)
198
+
199
regex_name = None
177
- for regex, function, name in regex_function:
200
+ for regex, function, name in regex_function_name:
201
if regex.search(last_line):
202
self.regex = regex
203
self.resp_time_func = function
@@ -215,7 +238,7 @@ class Service(LogService):
238
' web_log.http_method stacked 2 %s\n' % (job_name, self.update_every)
239
240
# Remove 'request_time' chart from ORDER if request_time not in logs
218
- if regex_name == 'access_default':
241
+ if regex_name == 'acs_default':
242
self.order.remove('response_time')
243
# Remove 'clients_all' chart from ORDER if specified in the configuration
244
if not self.all_time:
@@ -294,7 +317,7 @@ class Service(LogService):
317
# requests per http method
318
self._get_data_http_method(match_dict['method'])
319
# bandwidth sent
297
- self.data['bytes_sent'] += int(match_dict['sent'])
320
+ self.data['bytes_sent'] += int(match_dict['sent'] if '-' not in match_dict['sent'] else 0)
321
# request processing time and bandwidth received
322
if match_dict['resp_length'] and match_dict['resp_time']:
323
self.data['resp_length'] += int(match_dict['resp_length'])