b5ad4809f · cryptotaxi247/netdata-1

@@ -246,6 +246,8 @@ SQUID_CODES = dict(TCP='squid_transport_methods', UDP='squid_transport_methods',

246 DENIED='squid_cache_events', NOFETCH='squid_cache_events', TUNNEL='squid_cache_events',

247 ABORTED='squid_transport_errors', TIMEOUT='squid_transport_errors')

248

249 +REQUEST_REGEX = re.compile(r'(?P<method>[A-Z]+) (?P<url>[^ ]+) [A-Z]+/(?P<http_version>\d(?:.\d)?)')

250 +

251

252 class Service(LogService):

253 def __init__(self, configuration=None, name=None):

@@ -492,29 +494,13 @@ class Web(Mixin):

494 self.get_data_per_response_codes_detailed(code=match_dict['code'])

495 # response statuses

496 self.get_data_per_statuses(code=match_dict['code'])

495 - # requests per url

496 - if self.storage['url_pattern']:

497 - self.get_data_per_pattern(row=match_dict['url'],

498 - other='url_pattern_other',

499 - pattern=self.storage['url_pattern'])

497 # requests per user defined pattern

498 if self.storage['user_pattern'] and 'user_defined' in match_dict:

499 self.get_data_per_pattern(row=match_dict['user_defined'],

500 other='user_pattern_other',

501 pattern=self.storage['user_pattern'])

505 - # requests per http method

506 - if match_dict['method'] not in self.data:

507 - self.add_new_dimension(dimension_id=match_dict['method'],

508 - chart_key='http_method')

509 - self.data[match_dict['method']] += 1

510 - # requests per http version

511 - if 'http_version' in match_dict:

512 - dim_id = match_dict['http_version'].replace('.', '_')

513 - if dim_id not in self.data:

514 - self.add_new_dimension(dimension_id=dim_id,

515 - chart_key='http_version',

516 - dimension=match_dict['http_version'])

517 - self.data[dim_id] += 1

502 + # method, url, http version

503 + self.get_data_from_request_field(match_dict=match_dict)

504 # bandwidth sent

505 bytes_sent = match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0

506 self.data['bytes_sent'] += int(bytes_sent)

@@ -562,25 +548,19 @@ class Web(Mixin):

548 # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code

549 # 5. Bytes sent 6. Response length 7. Response process time

550 default = re.compile(r'(?P<address>[\da-f.:]+|localhost)'

565 - r' -.*?"(?P<method>[A-Z]+)'

566 - r' (?P<url>[^ ]+)'

567 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

551 + r' -.*?"(?P<request>[^"]+)"'

552 r' (?P<code>[1-9]\d{2})'

553 r' (?P<bytes_sent>\d+|-)')

554

555 apache_ext_insert = re.compile(r'(?P<address>[\da-f.:]+|localhost)'

572 - r' -.*?"(?P<method>[A-Z]+)'

573 - r' (?P<url>[^ ]+)'

574 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

556 + r' -.*?"(?P<request>[^"]+)"'

557 r' (?P<code>[1-9]\d{2})'

558 r' (?P<bytes_sent>\d+|-)'

559 r' (?P<resp_length>\d+)'

560 r' (?P<resp_time>\d+) ')

561

562 apache_ext_append = re.compile(r'(?P<address>[\da-f.:]+|localhost)'

581 - r' -.*?"(?P<method>[A-Z]+)'

582 - r' (?P<url>[^ ]+)'

583 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

563 + r' -.*?"(?P<request>[^"]+)"'

564 r' (?P<code>[1-9]\d{2})'

565 r' (?P<bytes_sent>\d+|-)'

566 r' .*?'

@@ -589,18 +569,14 @@ class Web(Mixin):

569 r'(?: |$)')

570

571 nginx_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

592 - r' -.*?"(?P<method>[A-Z]+)'

593 - r' (?P<url>[^ ]+)'

594 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

572 + r' -.*?"(?P<request>[^"]+)"'

573 r' (?P<code>[1-9]\d{2})'

574 r' (?P<bytes_sent>\d+)'

575 r' (?P<resp_length>\d+)'

576 r' (?P<resp_time>\d+\.\d+) ')

577

578 nginx_ext2_insert = re.compile(r'(?P<address>[\da-f.:]+)'

601 - r' -.*?"(?P<method>[A-Z]+)'

602 - r' (?P<url>[^ ]+)'

603 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

579 + r' -.*?"(?P<request>[^"]+)"'

580 r' (?P<code>[1-9]\d{2})'

581 r' (?P<bytes_sent>\d+)'

582 r' (?P<resp_length>\d+)'

@@ -608,9 +584,7 @@ class Web(Mixin):

584 r' (?P<resp_time_upstream>[\d.-]+) ')

585

586 nginx_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

611 - r' -.*?"(?P<method>[A-Z]+)'

612 - r' (?P<url>[^ ]+)'

613 - r' [A-Z]+/(?P<http_version>\d\.\d)"'

587 + r' -.*?"(?P<request>[^"]+)"'

588 r' (?P<code>[1-9]\d{2})'

589 r' (?P<bytes_sent>\d+)'

590 r' .*?'

@@ -695,12 +669,12 @@ class Web(Mixin):

669 ' (you need to use ?P<subgroup_name>)')

670 mandatory_dict = {'address': r'[\da-f.:]+|localhost',

671 'code': r'[1-9]\d{2}',

698 - 'method': r'[A-Z]+',

672 'bytes_sent': r'\d+|-'}

673 optional_dict = {'resp_length': r'\d+',

674 'resp_time': r'[\d.]+',

675 'resp_time_upstream': r'[\d.-]+',

703 - 'http_version': r'\d(\.\d)?'}

676 + 'method': r'[A-Z]+',

677 + 'http_version': r'\d(?:.\d)?'}

678

679 mandatory_values = set(mandatory_dict) - set(match_dict)

680 if mandatory_values:

@@ -726,6 +700,33 @@ class Web(Mixin):

700 self.storage['regex'] = regex

701 return find_regex_return(match_dict=match_dict)

702

703 + def get_data_from_request_field(self, match_dict):

704 + if match_dict.get('request'):

705 + match_dict = REQUEST_REGEX.search(match_dict['request'])

706 + if match_dict:

707 + match_dict = match_dict.groupdict()

708 + else:

709 + return

710 + # requests per url

711 + if match_dict.get('url') and self.storage['url_pattern']:

712 + self.get_data_per_pattern(row=match_dict['url'],

713 + other='url_pattern_other',

714 + pattern=self.storage['url_pattern'])

715 + # requests per http method

716 + if match_dict.get('method'):

717 + if match_dict['method'] not in self.data:

718 + self.add_new_dimension(dimension_id=match_dict['method'],

719 + chart_key='http_method')

720 + self.data[match_dict['method']] += 1

721 + # requests per http version

722 + if match_dict.get('http_version'):

723 + dim_id = match_dict['http_version'].replace('.', '_')

724 + if dim_id not in self.data:

725 + self.add_new_dimension(dimension_id=dim_id,

726 + chart_key='http_version',

727 + dimension=match_dict['http_version'])

728 + self.data[dim_id] += 1

729 +

730 def get_data_per_response_codes_detailed(self, code):

731 """

732 :param code: str: CODE from parsed line. Ex.: '202, '499'

web_log: add support for empty requests