a0a80fc7b · cryptotaxi247/netdata-1

@@ -82,6 +82,8 @@ CHARTS = {

82

83 NAMED_URL_PATTERN = namedtuple('URL_PATTERN', ['description', 'pattern'])

84

85 +DET_RESP_AGGR = ['', '_1xx', '_2xx', '_3xx', '_4xx', '_5xx', '_Other']

86 +

87

88 class Service(LogService):

89 def __init__(self, configuration=None, name=None):

@@ -94,7 +96,7 @@ class Service(LogService):

96 """

97 LogService.__init__(self, configuration=configuration, name=name)

98 # Variables from module configuration file

97 - self.type = self.configuration.get('type', 'web_access')

99 + self.log_type = self.configuration.get('type', 'web_access')

100 self.log_path = self.configuration.get('path')

101 self.url_pattern = self.configuration.get('categories') # dict

102 self.custom_log_format = self.configuration.get('custom_log_format') # dict

@@ -145,9 +147,10 @@ class Service(LogService):

147 self.error(str(error))

148 return False

149

148 - if self.type == 'web_access':

150 + if self.log_type == 'web_access':

151 self.unique_all_time = list() # sorted list of unique IPs

152 self.detailed_response_codes = self.configuration.get('detailed_response_codes', True)

153 + self.detailed_response_aggregate = self.configuration.get('detailed_response_aggregate', True)

154 self.all_time = self.configuration.get('all_time', True)

155

156 # Custom_log_format or predefined log format.

@@ -206,7 +209,7 @@ class Service(LogService):

209 ("resp_length" is integer or "-", "resp_time" is integer or float)

210

211 """

209 - if not is_dict(self.custom_log_format):

212 + if not hasattr(self.custom_log_format, 'keys'):

213 return find_regex_return(msg='Custom log: "custom_log_format" is not a <dict>')

214

215 pattern = self.custom_log_format.get('pattern')

@@ -244,7 +247,7 @@ class Service(LogService):

247 mandatory_values = set(mandatory_dict) - set(match_dict)

248 if mandatory_values:

249 return find_regex_return(msg='Custom log: search OK but some mandatory keys (%s) are missing'

247 - % list(mandatory_values))

250 + % list(mandatory_values))

251 else:

252 for key in mandatory_dict:

253 if not re.search(mandatory_dict[key], match_dict[key]):

@@ -355,6 +358,7 @@ class Service(LogService):

358 1. 'time_response' chart is removed if there is no 'resp_time' in match_dict.

359 2. Other stuff is just remove/add chart depending on yes/no in conf

360 """

361 +

362 def find_job_name(override_name, name):

363 """

364 :param override_name: str: 'name' var from configuration file

@@ -372,16 +376,14 @@ class Service(LogService):

376 self.definitions = deepcopy(CHARTS)

377

378 job_name = find_job_name(self.override_name, self.name)

375 - self.detailed_chart = 'CHART %s.detailed_response_codes ""' \

376 - ' "Detailed Response Codes" requests/s responses' \

377 - ' web_log.detailed_response_codes stacked 1 %s\n' % (job_name, self.update_every)

379 +

380 self.http_method_chart = 'CHART %s.http_method' \

381 ' "" "Requests Per HTTP Method" requests/s "http methods"' \

380 - ' web_log.http_method stacked 2 %s\n' \

382 + ' web_log.http_method stacked 11 %s\n' \

383 'DIMENSION GET GET incremental\n' % (job_name, self.update_every)

384 self.http_version_chart = 'CHART %s.http_version' \

385 ' "" "Requests Per HTTP Version" requests/s "http versions"' \

384 - ' web_log.http_version stacked 3 %s\n' % (job_name, self.update_every)

386 + ' web_log.http_version stacked 12 %s\n' % (job_name, self.update_every)

387

388 # Remove 'request_time' chart from ORDER if resp_time not in match_dict

389 if 'resp_time' not in match_dict:

@@ -391,11 +393,25 @@ class Service(LogService):

393 self.order.remove('clients_all')

394 # Add 'detailed_response_codes' chart if specified in the configuration

395 if self.detailed_response_codes:

394 - self.order.append('detailed_response_codes')

395 - self.definitions['detailed_response_codes'] = {'options': [None, 'Detailed Response Codes', 'requests/s',

396 - 'responses', 'web_log.detailed_response_codes',

397 - 'stacked'],

398 - 'lines': []}

396 + self.detailed_chart = list()

397 + for prio, add_to_dim in enumerate(DET_RESP_AGGR):

398 + self.detailed_chart.append('CHART %s.detailed_response_codes%s ""'

399 + ' "Detailed Response Codes %s" requests/s responses'

400 + ' web_log.detailed_response_codes%s stacked %s %s\n'

401 + % (job_name, add_to_dim, add_to_dim[1:], add_to_dim,

402 + str(prio), self.update_every))

403 +

404 + codes = DET_RESP_AGGR[:1] if self.detailed_response_aggregate else DET_RESP_AGGR[1:]

405 + for code in codes:

406 + self.order.append('detailed_response_codes%s' % code)

407 + self.definitions['detailed_response_codes%s' % code] = {'options':

408 + [None,

409 + 'Detailed Response Codes %s' % code[1:],

410 + 'requests/s',

411 + 'responses',

412 + 'web_log.detailed_response_codes%s' % code,

413 + 'stacked'],

414 + 'lines': []}

415

416 # Add 'requests_per_url' chart if specified in the configuration

417 if self.url_pattern:

@@ -481,10 +497,10 @@ class Service(LogService):

497 # unique clients ips

498 if address_not_in_pool(self.unique_all_time, match_dict['address'],

499 self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

484 - self.data['unique_tot_' + proto] += 1

500 + self.data['unique_tot_' + proto] += 1

501 if address_not_in_pool(unique_current, match_dict['address'], ip_address_counter['unique_cur_ip']):

486 - self.data['unique_cur_' + proto] += 1

487 - ip_address_counter['unique_cur_ip'] += 1

502 + self.data['unique_cur_' + proto] += 1

503 + ip_address_counter['unique_cur_ip'] += 1

504 else:

505 self.data['unmatched'] += 1

506

@@ -502,9 +518,16 @@ class Service(LogService):

518 Calls add_new_dimension method If the value is found for the first time

519 """

520 if code not in self.data:

505 - chart_string_copy = self.detailed_chart

506 - self.detailed_chart = self.add_new_dimension(code, [code, code, 'incremental'],

507 - chart_string_copy, 'detailed_response_codes')

521 + if self.detailed_response_aggregate:

522 + chart_string_copy = self.detailed_chart[0]

523 + self.detailed_chart[0] = self.add_new_dimension(code, [code, code, 'incremental'],

524 + chart_string_copy, 'detailed_response_codes')

525 + else:

526 + code_index = int(code[0]) if int(code[0]) < 6 else 6

527 + chart_string_copy = self.detailed_chart[code_index]

528 + chart_name = 'detailed_response_codes' + DET_RESP_AGGR[code_index]

529 + self.detailed_chart[code_index] = self.add_new_dimension(code, [code, code, 'incremental'],

530 + chart_string_copy, chart_name)

531 self.data[code] += 1

532

533 def _get_data_http_method(self, method):

@@ -602,7 +625,7 @@ def check_req_per_url_pattern(string, url_pattern):

625 :return: list of named tuples or None:

626 We need to make sure all patterns are valid regular expressions

627 """

605 - if not is_dict(url_pattern):

628 + if not hasattr(url_pattern, 'keys'):

629 return None

630

631 result = list()

@@ -628,17 +651,3 @@ def check_req_per_url_pattern(string, url_pattern):

651 result.append(NAMED_URL_PATTERN(description='_'.join([string, dimension]), pattern=valid_pattern))

652

653 return result or None

631 -

632 -

633 -def is_dict(obj):

634 - """

635 - :param obj: dict:

636 - :return: True or False

637 - obj can be <dict> or <OrderedDict>

638 - """

639 - try:

640 - obj.keys()

641 - except AttributeError:

642 - return False

643 - else:

644 - return True

web_log_plugin: not aggregated detailed response code charts