web_log plugin: handling_opts, object_types, cache_events, transport_methods, transport_errors charts added (squid)
Ilya committed
May 30, 2017 at 14:50 UTC
b4f5f65a7d4e1f26f646b9128a411863165b6662
1 file changed
+66
-15
python.d/web_log.chart.py
+66
-15
@@ -20,7 +20,8 @@ ORDER_WEB = ['response_statuses', 'response_codes', 'bandwidth', 'response_time'
20
'clients', 'clients_all']
21
22
ORDER_SQUID = ['response_statuses', 'response_codes', 'detailed_response_codes', 'squid_code',
23
- 'hier_code', 'bytes', 'duration', 'method', 'clients']
23
+ 'hier_code', 'transport_methods', 'transport_errors', 'handling_opts', 'object_types',
24
+ 'cache_events', 'bytes', 'duration', 'method', 'clients']
25
26
CHARTS_WEB = {
27
'response_codes': {
@@ -172,6 +173,31 @@ CHARTS_SQUID = {
173
'lines': [
174
['unique_ipv4', 'ipv4', 'incremental', 1, 1],
175
['unique_ipv6', 'ipv6', 'incremental', 1, 1]
176
+ ]},
177
+ 'transport_methods': {
178
+ 'options': [None, 'Transport Methods', 'methods', 'squid_transport',
179
+ 'web_log.squid_ transport_methods', 'stacked'],
180
+ 'lines': [
181
+ ]},
182
+ 'transport_errors': {
183
+ 'options': [None, 'Transport Errors', 'errors', 'squid_transport',
184
+ 'web_log.squid_transport_errors', 'stacked'],
185
+ 'lines': [
186
+ ]},
187
+ 'handling_opts': {
188
+ 'options': [None, 'Handling Opts', 'errors', 'squid_cache',
189
+ 'web_log.squid_handling_opts', 'stacked'],
190
+ 'lines': [
191
+ ]},
192
+ 'object_types': {
193
+ 'options': [None, 'Object Types', 'types', 'squid_cache',
194
+ 'web_log.squid_object_types', 'stacked'],
195
+ 'lines': [
196
+ ]},
197
+ 'cache_events': {
198
+ 'options': [None, 'Cache Events', 'events', 'squid_cache',
199
+ 'web_log.squid_cache_events', 'stacked'],
200
+ 'lines': [
201
]}
202
}
203
@@ -184,6 +210,16 @@ SQUID_DYNAMIC = dict(squid_code='squid_code',
210
hier_code='hier_code',
211
method='method')
212
213
+SQUID_CODES = dict(TCP='transport_methods', UDP='transport_methods', NONE='transport_methods',
214
+ CLIENT='handling_opts', IMS='handling_opts', ASYNC='handling_opts',
215
+ SWAPFAIL='handling_opts', REFRESH='handling_opts', SHARED='handling_opts',
216
+ REPLY='handling_opts', NEGATIVE='object_types', STALE='object_types',
217
+ OFFLINE='object_types', INVALID='object_types', FAIL='object_types',
218
+ MODIFIED='object_types', UNMODIFIED='object_types', REDIRECT='object_types',
219
+ HIT='cache_events', MEM='cache_events', MISS='cache_events',
220
+ DENIED='cache_events', NOFETCH='cache_events', TUNNEL='cache_events',
221
+ ABORTED='transport_errors', TIMEOUT='transport_errors')
222
+
223
224
class Service(LogService):
225
def __init__(self, configuration=None, name=None):
@@ -272,7 +308,8 @@ class Mixin:
308
309
def get_last_line(self):
310
"""
275
- :return:
311
+ Reads last line from the log file
312
+ :return: str:
313
"""
314
# Read last line (or first if there is only one line)
315
with open(self.conf['path'], 'rb') as logs:
@@ -336,9 +373,7 @@ class Web(Mixin):
373
"""
374
:param match_dict: dict: regex.search.groupdict(). Ex. {'address': '127.0.0.1', 'code': '200', 'method': 'GET'}
375
:return:
339
- Create additional charts depending on the 'match_dict' keys and configuration file options
340
- 1. 'time_response' chart is removed if there is no 'resp_time' in match_dict.
341
- 2. Other stuff is just remove/add chart depending on yes/no in conf
376
+ Create/remove additional charts depending on the 'match_dict' keys and configuration file options
377
"""
378
self.order = ORDER_WEB[:]
379
self.definitions = deepcopy(CHARTS_WEB)
@@ -382,7 +417,7 @@ class Web(Mixin):
417
418
def get_data(self, raw_data=None):
419
"""
385
- Parse new log lines
420
+ Parses new log lines
421
:return: dict OR None
422
None if _get_raw_data method fails.
423
In all other cases - dict.
@@ -409,12 +444,12 @@ class Web(Mixin):
444
self.get_data_per_statuses(code=match_dict['code'])
445
# requests per url
446
if self.storage['url_pattern']:
412
- self.get_data_per_pattern(field=match_dict['url'],
447
+ self.get_data_per_pattern(row=match_dict['url'],
448
other='url_pattern_other',
449
pattern=self.storage['url_pattern'])
450
# requests per user defined pattern
451
if self.storage['user_pattern'] and 'user_defined' in match_dict:
417
- self.get_data_per_pattern(field=match_dict['user_defined'],
452
+ self.get_data_per_pattern(row=match_dict['user_defined'],
453
other='user_pattern_other',
454
pattern=self.storage['user_pattern'])
455
# requests per http method
@@ -646,18 +681,18 @@ class Web(Mixin):
681
chart_key=chart_key)
682
self.data[code] += 1
683
649
- def get_data_per_pattern(self, field, other, pattern):
684
+ def get_data_per_pattern(self, row, other, pattern):
685
"""
651
- :param field: str:
686
+ :param row: str:
687
:param other: str:
653
- :param pattern: list:
688
+ :param pattern: named tuple: (['pattern_description', 'regular expression'])
689
:return:
690
Scan through string looking for the first location where patterns produce a match for all user
691
defined patterns
692
"""
693
match = None
694
for elem in pattern:
660
- if elem.pattern.search(field):
695
+ if elem.pattern.search(row):
696
self.data[elem.description] += 1
697
match = True
698
break
@@ -757,6 +792,8 @@ class Squid(Mixin):
792
793
self.get_data_per_statuses(match['http_code'])
794
795
+ self.get_data_per_squid_code(match['squid_code'])
796
+
797
self.data['bytes'] += int(match['bytes'])
798
799
proto = 'ipv4' if '.' in match['client_address'] else 'ipv6'
@@ -795,13 +832,27 @@ class Squid(Mixin):
832
else:
833
self.data['bad_requests'] += 1
834
835
+ def get_data_per_squid_code(self, code):
836
+ """
837
+ :param code: str: squid response code. Ex.: 'TCP_MISS', 'TCP_MISS_ABORTED'
838
+ :return:
839
+ """
840
+ for tag in code.split('_'):
841
+ try:
842
+ chart_key = SQUID_CODES[tag]
843
+ except KeyError:
844
+ continue
845
+ if tag not in self.data:
846
+ self.add_new_dimension(dimension_id=tag, chart_key=chart_key)
847
+ self.data[tag] += 1
848
+
849
850
def address_not_in_pool(pool, address, pool_size):
851
"""
852
:param pool: list of ip addresses
853
:param address: ip address
854
:param pool_size: current pool size
804
- :return: True if address not in pool. False if address in pool.
855
+ :return: True if address not in pool. False otherwise.
856
"""
857
index = bisect.bisect_left(pool, address)
858
if index < pool_size:
@@ -825,7 +876,7 @@ def find_regex_return(match_dict=None, msg='Generic error message'):
876
def check_patterns(string, dimension_regex_dict):
877
"""
878
:param string: str:
828
- :param dimension_regex_dict: dict: ex. {'dim1': 'pattern1>', 'dim2': '<pattern2>'}
879
+ :param dimension_regex_dict: dict: ex. {'dim1': '<pattern1>', 'dim2': '<pattern2>'}
880
:return: list of named tuples or None:
881
We need to make sure all patterns are valid regular expressions
882
"""
@@ -859,7 +910,7 @@ def find_job_name(override_name, name):
910
:param override_name: str: 'name' var from configuration file
911
:param name: str: 'job_name' from configuration file
912
:return: str: new job name
862
- We need this for dynamic charts. Actually same logic as in python.d.plugin.
913
+ We need this for dynamic charts. Actually the same logic as in python.d.plugin.
914
"""
915
add_to_name = override_name or name
916
if add_to_name: