web_log plugin: allow filtering with regex feature added
Ilya committed
Jun 4, 2017 at 00:36 UTC
fe13f1c74e0d40521b7b8c34791b46b9398cf452
1 file changed
+40
-10
python.d/web_log.chart.py
+40
-10
@@ -8,6 +8,11 @@ from os.path import getsize
8
from collections import namedtuple
9
from copy import deepcopy
10
from base import LogService
11
+try:
12
+ from itertools import filterfalse
13
+except ImportError:
14
+ from itertools import ifilter as filter
15
+ from itertools import ifilterfalse as filterfalse
16
import msg
17
18
priority = 60000
@@ -216,7 +221,7 @@ CHARTS_SQUID = {
221
]}
222
}
223
219
-NAMED_PATTERN = namedtuple('PATTERN', ['description', 'pattern'])
224
+NAMED_PATTERN = namedtuple('PATTERN', ['description', 'func'])
225
226
DET_RESP_AGGR = ['', '_1xx', '_2xx', '_3xx', '_4xx', '_5xx', '_Other']
227
@@ -287,6 +292,21 @@ class Service(LogService):
292
293
294
class Mixin:
295
+ def filter_data(self, raw_data):
296
+ """
297
+ :param raw_data: list
298
+ :return:
299
+ """
300
+ if not self.pre_filter:
301
+ return raw_data
302
+ parsed = raw_data
303
+ for elem in self.pre_filter:
304
+ if elem.description == 'filter_include':
305
+ parsed = filter(elem.func, parsed)
306
+ elif elem.description == 'filter_exclude':
307
+ parsed = filterfalse(elem.func, parsed)
308
+ return parsed
309
+
310
def add_new_dimension(self, dimension_id, chart_key, dimension=None,
311
algorithm='incremental', multiplier=1, divisor=1):
312
"""
@@ -350,6 +370,7 @@ class Mixin:
370
class Web(Mixin):
371
def __init__(self, configuration):
372
self.conf = configuration
373
+ self.pre_filter = check_patterns('filter', self.conf.get('filter'))
374
self.storage = dict()
375
self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,
376
'resp_time_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,
@@ -432,13 +453,15 @@ class Web(Mixin):
453
None if _get_raw_data method fails.
454
In all other cases - dict.
455
"""
435
- if raw_data is None:
436
- return None
456
+ if not raw_data:
457
+ return None if raw_data is None else self.data
458
+
459
+ filtered_data = self.filter_data(raw_data=raw_data)
460
461
request_time, unique_current = list(), list()
462
request_counter = {'count': 0, 'sum': 0}
463
ip_address_counter = {'unique_cur_ip': 0}
441
- for line in raw_data:
464
+ for line in filtered_data:
465
match = self.storage['regex'].search(line)
466
if match:
467
match_dict = match.groupdict()
@@ -702,7 +725,7 @@ class Web(Mixin):
725
"""
726
match = None
727
for elem in pattern:
705
- if elem.pattern.search(row):
728
+ if elem.func(row):
729
self.data[elem.description] += 1
730
match = True
731
break
@@ -758,6 +781,7 @@ class Squid(Mixin):
781
self.conf = configuration
782
self.order = ORDER_SQUID
783
self.definitions = CHARTS_SQUID
784
+ self.pre_filter = check_patterns('filter', self.conf.get('filter'))
785
self.storage = dict()
786
self.data = {'duration_max': 0, 'duration_avg': 0, 'duration_min': 0, 'bytes': 0,
787
'0xx': 0, '1xx': 0, '2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0,
@@ -799,12 +823,14 @@ class Squid(Mixin):
823
return True
824
825
def get_data(self, raw_data=None):
802
- if raw_data is None:
803
- return None
826
+ if not raw_data:
827
+ return None if raw_data is None else self.data
828
+
829
+ filtered_data = self.filter_data(raw_data=raw_data)
830
831
unique_ip, duration = set(), list()
832
807
- for row in raw_data:
833
+ for row in filtered_data:
834
match = self.storage['regex'].search(row)
835
if match:
836
match = match.groupdict()
@@ -932,11 +958,15 @@ def check_patterns(string, dimension_regex_dict):
958
except re.error:
959
return False
960
961
+ def func_search(pattern):
962
+ return lambda v: pattern.search(v)
963
+
964
for dimension, regex in dimension_regex_dict.items():
965
valid = valid_pattern(regex)
966
+ func = func_search(valid)
967
if isinstance(dimension, str) and valid_pattern:
938
- result.append(NAMED_PATTERN(description='_'.join([string, dimension]), pattern=valid))
939
-
968
+ result.append(NAMED_PATTERN(description='_'.join([string, dimension]),
969
+ func=func))
970
return result or None
971
972