b6aff8278 · cryptotaxi247/netdata-1

@@ -5,13 +5,13 @@ import re

5 import bisect

6 from os import access, R_OK

7 from os.path import getsize

8 -from collections import namedtuple

8 +from collections import namedtuple, defaultdict

9 from copy import deepcopy

10 -from base import LogService

10 try:

11 from itertools import filterfalse

12 except ImportError:

13 from itertools import ifilterfalse as filterfalse

14 +from base import LogService

15 import msg

16

17 priority = 60000

@@ -19,9 +19,9 @@ retries = 60

19

20 ORDER_APACHE_CACHE = ['apache_cache']

21

22 -ORDER_WEB = ['response_statuses', 'response_codes', 'bandwidth', 'response_time', 'requests_per_url',

23 - 'requests_per_user_defined', 'http_method', 'http_version', 'requests_per_ipproto',

24 - 'clients', 'clients_all']

22 +ORDER_WEB = ['response_statuses', 'response_codes', 'bandwidth', 'response_time', 'response_time_upstream',

23 + 'requests_per_url', 'requests_per_user_defined', 'http_method', 'http_version',

24 + 'requests_per_ipproto', 'clients', 'clients_all']

25

26 ORDER_SQUID = ['squid_response_statuses', 'squid_response_codes', 'squid_detailed_response_codes',

27 'squid_method', 'squid_mime_type', 'squid_hier_code', 'squid_transport_methods',

@@ -53,6 +53,14 @@ CHARTS_WEB = {

53 ['resp_time_max', 'max', 'incremental', 1, 1000],

54 ['resp_time_avg', 'avg', 'incremental', 1, 1000]

55 ]},

56 + 'response_time_upstream': {

57 + 'options': [None, 'Processing Time Upstream', 'milliseconds', 'timings',

58 + 'web_log.response_time_upstream', 'area'],

59 + 'lines': [

60 + ['resp_time_upstream_min', 'min', 'incremental', 1, 1000],

61 + ['resp_time_upstream_max', 'max', 'incremental', 1, 1000],

62 + ['resp_time_upstream_avg', 'avg', 'incremental', 1, 1000]

63 + ]},

64 'clients': {

65 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'clients', 'web_log.clients', 'stacked'],

66 'lines': [

@@ -374,7 +382,8 @@ class Web(Mixin):

382 self.pre_filter = check_patterns('filter', self.conf.get('filter'))

383 self.storage = dict()

384 self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,

377 - 'resp_time_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,

385 + 'resp_time_avg': 0, 'resp_time_upstream_min': 0, 'resp_time_upstream_max': 0,

386 + 'resp_time_upstream_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,

387 '5xx': 0, '3xx': 0, '4xx': 0, '1xx': 0, '0xx': 0, 'unmatched': 0, 'req_ipv4': 0,

388 'req_ipv6': 0, 'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0, 'successful_requests': 0,

389 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0, 'GET': 0}

@@ -412,6 +421,9 @@ class Web(Mixin):

421

422 if 'resp_time' not in match_dict:

423 self.order.remove('response_time')

424 + if 'resp_time_upstream' not in match_dict:

425 + self.order.remove('response_time_upstream')

426 +

427 if not self.conf.get('all_time', True):

428 self.order.remove('clients_all')

429

@@ -459,8 +471,9 @@ class Web(Mixin):

471

472 filtered_data = self.filter_data(raw_data=raw_data)

473

462 - request_time, unique_current = list(), list()

463 - request_counter = {'count': 0, 'sum': 0}

474 + unique_current = set()

475 + timings = defaultdict(lambda: dict(minimum=None, maximum=0, summary=0, count=0))

476 +

477 ip_address_counter = {'unique_cur_ip': 0}

478 for line in filtered_data:

479 match = self.storage['regex'].search(line)

@@ -506,10 +519,11 @@ class Web(Mixin):

519 if 'resp_length' in match_dict:

520 self.data['resp_length'] += int(match_dict['resp_length'])

521 if 'resp_time' in match_dict:

509 - resp_time = self.storage['func_resp_time'](float(match_dict['resp_time']))

510 - bisect.insort_left(request_time, resp_time)

511 - request_counter['count'] += 1

512 - request_counter['sum'] += resp_time

522 + get_timings(timings=timings['resp_time'],

523 + time=self.storage['func_resp_time'](float(match_dict['resp_time'])))

524 + if 'resp_time_upstream' in match_dict and match_dict['resp_time_upstream'] != '-':

525 + get_timings(timings=timings['resp_time_upstream'],

526 + time=self.storage['func_resp_time'](float(match_dict['resp_time_upstream'])))

527 # requests per ip proto

528 proto = 'ipv4' if '.' in match_dict['address'] else 'ipv6'

529 self.data['req_' + proto] += 1

@@ -518,19 +532,17 @@ class Web(Mixin):

532 address=match_dict['address'],

533 pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

534 self.data['unique_tot_' + proto] += 1

521 - if address_not_in_pool(pool=unique_current,

522 - address=match_dict['address'],

523 - pool_size=ip_address_counter['unique_cur_ip']):

535 + if match_dict['address'] not in unique_current:

536 self.data['unique_cur_' + proto] += 1

525 - ip_address_counter['unique_cur_ip'] += 1

537 + unique_current.add(match_dict['address'])

538 else:

539 self.data['unmatched'] += 1

540

541 # timings

530 - if request_time:

531 - self.data['resp_time_min'] += request_time[0]

532 - self.data['resp_time_avg'] += round(float(request_counter['sum']) / request_counter['count'])

533 - self.data['resp_time_max'] += request_time[-1]

542 + for elem in timings:

543 + self.data[elem + '_min'] += timings[elem]['minimum']

544 + self.data[elem + '_avg'] += timings[elem]['summary'] / timings[elem]['count']

545 + self.data[elem + '_max'] += timings[elem]['maximum']

546 return self.data

547

548 def find_regex(self, last_line):

@@ -545,51 +557,61 @@ class Web(Mixin):

557 """

558 # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code

559 # 5. Bytes sent 6. Response length 7. Response process time

548 - acs_default = re.compile(r'(?P<address>[\da-f.:]+)'

549 - r' -.*?"(?P<method>[A-Z]+)'

550 - r' (?P<url>[^ ]+)'

551 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

552 - r' (?P<code>[1-9]\d{2})'

553 - r' (?P<bytes_sent>\d+|-)')

554 -

555 - acs_apache_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

556 - r' -.*?"(?P<method>[A-Z]+)'

557 - r' (?P<url>[^ ]+)'

558 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

559 - r' (?P<code>[1-9]\d{2})'

560 - r' (?P<bytes_sent>\d+|-)'

561 - r' (?P<resp_length>\d+)'

562 - r' (?P<resp_time>\d+) ')

563 -

564 - acs_apache_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

565 - r' -.*?"(?P<method>[A-Z]+)'

566 - r' (?P<url>[^ ]+)'

567 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

568 - r' (?P<code>[1-9]\d{2})'

569 - r' (?P<bytes_sent>\d+|-)'

570 - r' .*?'

571 - r' (?P<resp_length>\d+)'

572 - r' (?P<resp_time>\d+)'

573 - r'(?: |$)')

574 -

575 - acs_nginx_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

576 - r' -.*?"(?P<method>[A-Z]+)'

577 - r' (?P<url>[^ ]+)'

578 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

579 - r' (?P<code>[1-9]\d{2})'

580 - r' (?P<bytes_sent>\d+)'

581 - r' (?P<resp_length>\d+)'

582 - r' (?P<resp_time>\d+\.\d+) ')

583 -

584 - acs_nginx_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

585 - r' -.*?"(?P<method>[A-Z]+)'

586 - r' (?P<url>[^ ]+)'

587 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

588 - r' (?P<code>[1-9]\d{2})'

589 - r' (?P<bytes_sent>\d+)'

590 - r' .*?'

591 - r' (?P<resp_length>\d+)'

592 - r' (?P<resp_time>\d+\.\d+)')

560 + default = re.compile(r'(?P<address>[\da-f.:]+)'

561 + r' -.*?"(?P<method>[A-Z]+)'

562 + r' (?P<url>[^ ]+)'

563 + r' [A-Z]+/(?P<http_version>\d\.\d)"'

564 + r' (?P<code>[1-9]\d{2})'

565 + r' (?P<bytes_sent>\d+|-)')

566 +

567 + apache_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

568 + r' -.*?"(?P<method>[A-Z]+)'

569 + r' (?P<url>[^ ]+)'

570 + r' [A-Z]+/(?P<http_version>\d\.\d)"'

571 + r' (?P<code>[1-9]\d{2})'

572 + r' (?P<bytes_sent>\d+|-)'

573 + r' (?P<resp_length>\d+)'

574 + r' (?P<resp_time>\d+) ')

575 +

576 + apache_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

577 + r' -.*?"(?P<method>[A-Z]+)'

578 + r' (?P<url>[^ ]+)'

579 + r' [A-Z]+/(?P<http_version>\d\.\d)"'

580 + r' (?P<code>[1-9]\d{2})'

581 + r' (?P<bytes_sent>\d+|-)'

582 + r' .*?'

583 + r' (?P<resp_length>\d+)'

584 + r' (?P<resp_time>\d+)'

585 + r'(?: |$)')

586 +

587 + nginx_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

588 + r' -.*?"(?P<method>[A-Z]+)'

589 + r' (?P<url>[^ ]+)'

590 + r' [A-Z]+/(?P<http_version>\d\.\d)"'

591 + r' (?P<code>[1-9]\d{2})'

592 + r' (?P<bytes_sent>\d+)'

593 + r' (?P<resp_length>\d+)'

594 + r' (?P<resp_time>\d+\.\d+) ')

595 +

596 + nginx_ext2_insert = re.compile(r'(?P<address>[\da-f.:]+)'

597 + r' -.*?"(?P<method>[A-Z]+)'

598 + r' (?P<url>[^ ]+)'

599 + r' [A-Z]+/(?P<http_version>\d\.\d)"'

600 + r' (?P<code>[1-9]\d{2})'

601 + r' (?P<bytes_sent>\d+)'

602 + r' (?P<resp_length>\d+)'

603 + r' (?P<resp_time>\d+\.\d+)'

604 + r' (?P<resp_time_upstream>[\d.-]+) ')

605 +

606 + nginx_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

607 + r' -.*?"(?P<method>[A-Z]+)'

608 + r' (?P<url>[^ ]+)'

609 + r' [A-Z]+/(?P<http_version>\d\.\d)"'

610 + r' (?P<code>[1-9]\d{2})'

611 + r' (?P<bytes_sent>\d+)'

612 + r' .*?'

613 + r' (?P<resp_length>\d+)'

614 + r' (?P<resp_time>\d+\.\d+)')

615

616 def func_usec(time):

617 return time

@@ -597,9 +619,10 @@ class Web(Mixin):

619 def func_sec(time):

620 return time * 1000000

621

600 - r_regex = [acs_apache_ext_insert, acs_apache_ext_append, acs_nginx_ext_insert,

601 - acs_nginx_ext_append, acs_default]

602 - r_function = [func_usec, func_usec, func_sec, func_sec, func_usec]

622 + r_regex = [apache_ext_insert, apache_ext_append,

623 + nginx_ext2_insert, nginx_ext_insert, nginx_ext_append,

624 + default]

625 + r_function = [func_usec, func_usec, func_sec, func_sec, func_sec, func_usec]

626 regex_function = zip(r_regex, r_function)

627

628 match_dict = dict()

@@ -672,6 +695,7 @@ class Web(Mixin):

695 'bytes_sent': r'\d+|-'}

696 optional_dict = {'resp_length': r'\d+',

697 'resp_time': r'[\d.]+',

698 + 'resp_time_upstream': r'[\d.-]+',

699 'http_version': r'\d\.\d'}

700

701 mandatory_values = set(mandatory_dict) - set(match_dict)

@@ -829,14 +853,15 @@ class Squid(Mixin):

853

854 filtered_data = self.filter_data(raw_data=raw_data)

855

832 - unique_ip, duration = set(), list()

856 + unique_ip = set()

857 + timings = defaultdict(lambda: dict(minimum=None, maximum=0, summary=0, count=0))

858

859 for row in filtered_data:

860 match = self.storage['regex'].search(row)

861 if match:

862 match = match.groupdict()

863 if match['duration'] != '0':

839 - duration.append(match['duration'])

864 + get_timings(timings=timings['duration'], time=float(match['duration']) * 1000)

865 try:

866 self.data[match['http_code'][0] + 'xx'] += 1

867 except KeyError:

@@ -870,11 +895,10 @@ class Squid(Mixin):

895 else:

896 self.data['unmatched'] += 1

897

873 - if duration:

874 - length, duration = len(duration), map(lambda v: float(v) * 1000, duration)

875 - self.data['duration_min'] += min(duration)

876 - self.data['duration_max'] += max(duration)

877 - self.data['duration_avg'] += sum(duration) / length

898 + for elem in timings:

899 + self.data[elem + '_min'] += timings[elem]['minimum']

900 + self.data[elem + '_avg'] += timings[elem]['summary'] / timings[elem]['count']

901 + self.data[elem + '_max'] += timings[elem]['maximum']

902 return self.data

903

904 def get_data_per_statuses(self, code):

@@ -909,6 +933,22 @@ class Squid(Mixin):

933 self.data[tag] += 1

934

935

936 +def get_timings(timings, time):

937 + """

938 + :param timings:

939 + :param time:

940 + :return:

941 + """

942 + if timings['minimum'] is None:

943 + timings['minimum'] = time

944 + if time > timings['maximum']:

945 + timings['maximum'] = time

946 + elif time < timings['minimum']:

947 + timings['minimum'] = time

948 + timings['summary'] += time

949 + timings['count'] += 1

950 +

951 +

952 def address_not_in_pool(pool, address, pool_size):

953 """

954 :param pool: list of ip addresses

web_log plugin: support upstreaming timing variable of nginx + minor fixes