62bac35dd · cryptotaxi247/netdata-1

@@ -1,20 +1,19 @@

1 # -*- coding: utf-8 -*-

2 # Description: web log netdata python.d module

3 # Author: l2isbad

4 -

5 -from base import LogService

4 import re

5 import bisect

6 from os import access, R_OK

7 from os.path import getsize

8 from collections import namedtuple

9 from copy import deepcopy

10 +from base import LogService

11

12 priority = 60000

13 retries = 60

14

16 -ORDER = ['response_statuses', 'response_codes', 'bandwidth', 'response_time', 'requests_per_url', 'http_method',

17 - 'http_version', 'requests_per_ipproto', 'clients', 'clients_all']

15 +ORDER = ['response_statuses', 'response_codes', 'bandwidth', 'response_time', 'requests_per_url',

16 + 'requests_per_user_defined', 'http_method', 'http_version', 'requests_per_ipproto', 'clients', 'clients_all']

17 CHARTS = {

18 'response_codes': {

19 'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response_codes', 'stacked'],

@@ -80,7 +79,7 @@ CHARTS = {

79 ]}

80 }

81

83 -NAMED_URL_PATTERN = namedtuple('URL_PATTERN', ['description', 'pattern'])

82 +NAMED_PATTERN = namedtuple('PATTERN', ['description', 'pattern'])

83

84 DET_RESP_AGGR = ['', '_1xx', '_2xx', '_3xx', '_4xx', '_5xx', '_Other']

85

@@ -99,6 +98,7 @@ class Service(LogService):

98 self.log_type = self.configuration.get('type', 'web_access')

99 self.log_path = self.configuration.get('path')

100 self.url_pattern = self.configuration.get('categories') # dict

101 + self.user_pattern = self.configuration.get('user_defined') # dict

102 self.custom_log_format = self.configuration.get('custom_log_format') # dict

103 # Instance variables

104 self.regex = None # will be assigned in 'find_regex' or 'find_regex_custom' method

@@ -164,9 +164,8 @@ class Service(LogService):

164 self.error(str(error))

165 return False

166

167 - # self.url_pattern check

168 - if self.url_pattern:

169 - self.url_pattern = check_req_per_url_pattern('rpu', self.url_pattern)

167 + self.url_pattern = check_patterns('url_pattern', self.url_pattern)

168 + self.user_pattern = check_patterns('user_pattern', self.user_pattern)

169

170 self.create_access_charts(match_dict) # Create charts

171 self._get_data = self._get_access_data # _get_data assignment

@@ -174,10 +173,6 @@ class Service(LogService):

173 self.error('Not implemented')

174 return False

175

177 - # Double check

178 - if not self.regex:

179 - self.error('That can not happen, but it happened. "regex" is None')

180 -

176 self.info('Collected data: %s' % list(match_dict.keys()))

177 return True

178

@@ -247,7 +242,7 @@ class Service(LogService):

242 mandatory_values = set(mandatory_dict) - set(match_dict)

243 if mandatory_values:

244 return find_regex_return(msg='Custom log: search OK but some mandatory keys (%s) are missing'

250 - % list(mandatory_values))

245 + % list(mandatory_values))

246 else:

247 for key in mandatory_dict:

248 if not re.search(mandatory_dict[key], match_dict[key]):

@@ -339,11 +334,11 @@ class Service(LogService):

334 regex_function = zip(r_regex, r_function)

335

336 match_dict = dict()

342 - for regex, function in regex_function:

337 + for regex, func in regex_function:

338 match = regex.search(last_line)

339 if match:

340 self.regex = regex

346 - self.resp_time_func = function

341 + self.resp_time_func = func

342 match_dict = match.groupdict()

343 break

344

@@ -385,10 +380,8 @@ class Service(LogService):

380 ' "" "Requests Per HTTP Version" requests/s "http versions"' \

381 ' web_log.http_version stacked 12 %s\n' % (job_name, self.update_every)

382

388 - # Remove 'request_time' chart from ORDER if resp_time not in match_dict

383 if 'resp_time' not in match_dict:

384 self.order.remove('response_time')

391 - # Remove 'clients_all' chart from ORDER if specified in the configuration

385 if not self.all_time:

386 self.order.remove('clients_all')

387 # Add 'detailed_response_codes' chart if specified in the configuration

@@ -417,15 +410,31 @@ class Service(LogService):

410 if self.url_pattern:

411 self.definitions['requests_per_url'] = {'options': [None, 'Requests Per Url', 'requests/s',

412 'urls', 'web_log.requests_per_url', 'stacked'],

420 - 'lines': [['rpu_other', 'other', 'incremental']]}

413 + 'lines': [['url_pattern_other', 'other', 'incremental']]}

414 for elem in self.url_pattern:

422 - self.definitions['requests_per_url']['lines'].append([elem.description, elem.description[4:],

415 + self.definitions['requests_per_url']['lines'].append([elem.description,

416 + elem.description[12:],

417 'incremental'])

418 self.data.update({elem.description: 0})

425 - self.data.update({'rpu_other': 0})

419 + self.data.update({'url_pattern_other': 0})

420 else:

421 self.order.remove('requests_per_url')

422

423 + if self.user_pattern and 'user_defined' in match_dict:

424 + self.definitions['requests_per_user_defined'] = {'options': [None, 'Requests Per User Defined Pattern',

425 + 'requests/s', 'user defined',

426 + 'web_log.requests_per_user_defined',

427 + 'stacked'],

428 + 'lines': [['user_pattern_other', 'other', 'incremental']]}

429 + for elem in self.user_pattern:

430 + self.definitions['requests_per_user_defined']['lines'].append([elem.description,

431 + elem.description[13:],

432 + 'incremental'])

433 + self.data.update({elem.description: 0})

434 + self.data.update({'user_pattern_other': 0})

435 + else:

436 + self.order.remove('requests_per_user_defined')

437 +

438 def add_new_dimension(self, dimension, line_list, chart_string, key):

439 """

440 :param dimension: str: response status code. Ex.: '202', '499'

@@ -469,17 +478,22 @@ class Service(LogService):

478 self.data['0xx'] += 1

479 # detailed response code

480 if self.detailed_response_codes:

472 - self._get_data_detailed_response_codes(match_dict['code'])

481 + self._get_data_detailed_response_codes(code=match_dict['code'])

482 # response statuses

474 - self._get_data_statuses(match_dict['code'])

483 + self._get_data_statuses(code=match_dict['code'])

484 # requests per url

485 if self.url_pattern:

477 - self._get_data_per_url(match_dict['url'])

486 + self._get_data_per_pattern(field=match_dict['url'], other='url_pattern_other',

487 + pattern=self.url_pattern)

488 + # requests per user defined pattern

489 + if self.user_pattern and 'user_defined' in match_dict:

490 + self._get_data_per_pattern(field=match_dict['user_defined'], other='user_pattern_other',

491 + pattern=self.user_pattern)

492 # requests per http method

479 - self._get_data_http_method(match_dict['method'])

493 + self._get_data_http_method(method=match_dict['method'])

494 # requests per http version

495 if 'http_version' in match_dict:

482 - self._get_data_http_version(match_dict['http_version'])

496 + self._get_data_http_version(http_version=match_dict['http_version'])

497 # bandwidth sent

498 bytes_sent = match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0

499 self.data['bytes_sent'] += int(bytes_sent)

@@ -495,10 +509,11 @@ class Service(LogService):

509 proto = 'ipv4' if '.' in match_dict['address'] else 'ipv6'

510 self.data['req_' + proto] += 1

511 # unique clients ips

498 - if address_not_in_pool(self.unique_all_time, match_dict['address'],

499 - self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

512 + if address_not_in_pool(pool=self.unique_all_time, address=match_dict['address'],

513 + pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

514 self.data['unique_tot_' + proto] += 1

501 - if address_not_in_pool(unique_current, match_dict['address'], ip_address_counter['unique_cur_ip']):

515 + if address_not_in_pool(pool=unique_current, address=match_dict['address'],

516 + pool_size=ip_address_counter['unique_cur_ip']):

517 self.data['unique_cur_' + proto] += 1

518 ip_address_counter['unique_cur_ip'] += 1

519 else:

@@ -506,9 +521,9 @@ class Service(LogService):

521

522 # timings

523 if request_time:

509 - self.data['resp_time_min'] += int(request_time[0])

510 - self.data['resp_time_avg'] += int(round(float(request_counter['sum']) / request_counter['count']))

511 - self.data['resp_time_max'] += int(request_time[-1])

524 + self.data['resp_time_min'] += request_time[0]

525 + self.data['resp_time_avg'] += round(float(request_counter['sum']) / request_counter['count'])

526 + self.data['resp_time_max'] += request_time[-1]

527 return self.data

528

529 def _get_data_detailed_response_codes(self, code):

@@ -556,21 +571,22 @@ class Service(LogService):

571 chart_string_copy, 'http_version')

572 self.data[http_version_dim_id] += 1

573

559 - def _get_data_per_url(self, url):

574 + def _get_data_per_pattern(self, field, other, pattern):

575 """

561 - :param url: str: URL from parsed line

576 + :param field: str:

577 + :param other: str:

578 :return:

579 Scan through string looking for the first location where patterns produce a match for all user

580 defined patterns

581 """

582 match = None

567 - for elem in self.url_pattern:

568 - if elem.pattern.search(url):

583 + for elem in pattern:

584 + if elem.pattern.search(field):

585 self.data[elem.description] += 1

586 match = True

587 break

588 if not match:

573 - self.data['rpu_other'] += 1

589 + self.data[other] += 1

590

591 def _get_data_statuses(self, code):

592 """

@@ -601,12 +617,10 @@ def address_not_in_pool(pool, address, pool_size):

617 if index < pool_size:

618 if pool[index] == address:

619 return False

604 - else:

605 - bisect.insort_left(pool, address)

606 - return True

607 - else:

620 bisect.insort_left(pool, address)

621 return True

622 + bisect.insort_left(pool, address)

623 + return True

624

625

626 def find_regex_return(match_dict=None, msg='Generic error message'):

@@ -618,36 +632,33 @@ def find_regex_return(match_dict=None, msg='Generic error message'):

632 return match_dict, msg

633

634

621 -def check_req_per_url_pattern(string, url_pattern):

635 +def check_patterns(string, dimension_regex_dict):

636 """

637 :param string: str:

624 - :param url_pattern: dict: ex. {'dim1': 'pattern1>', 'dim2': '<pattern2>'}

638 + :param dimension_regex_dict: dict: ex. {'dim1': 'pattern1>', 'dim2': '<pattern2>'}

639 :return: list of named tuples or None:

640 We need to make sure all patterns are valid regular expressions

641 """

628 - if not hasattr(url_pattern, 'keys'):

642 + if not hasattr(dimension_regex_dict, 'keys'):

643 return None

644

645 result = list()

646

633 - def is_valid_pattern(pattern):

647 + def valid_pattern(pattern):

648 """

649 :param pattern: str

650 :return: re.compile(pattern) or None

651 """

652 if not isinstance(pattern, str):

653 return False

640 - else:

641 - try:

642 - compile_pattern = re.compile(pattern)

643 - except re.error:

644 - return False

645 - else:

646 - return compile_pattern

654 + try:

655 + return re.compile(pattern)

656 + except re.error:

657 + return False

658

648 - for dimension, regex in url_pattern.items():

649 - valid_pattern = is_valid_pattern(regex)

659 + for dimension, regex in dimension_regex_dict.items():

660 + valid = valid_pattern(regex)

661 if isinstance(dimension, str) and valid_pattern:

651 - result.append(NAMED_URL_PATTERN(description='_'.join([string, dimension]), pattern=valid_pattern))

662 + result.append(NAMED_PATTERN(description='_'.join([string, dimension]), pattern=valid))

663

664 return result or None

web_log plugin: "user defined" requests/s chart added