| 1 | # -*- coding: utf-8 -*- |
| 2 | # Description: |
| 3 | # SPDX-License-Identifier: GPL-3.0-or-later |
| 4 | |
| 5 | import urllib3 |
| 6 | |
| 7 | from bases.FrameworkServices.SimpleService import SimpleService |
| 8 | |
| 9 | try: |
| 10 | urllib3.disable_warnings() |
| 11 | except AttributeError: |
| 12 | pass |
| 13 | |
| 14 | URLLIB3_VERSION = urllib3.__version__ |
| 15 | URLLIB3 = 'urllib3' |
| 16 | |
| 17 | class UrlService(SimpleService): |
| 18 | def __init__(self, configuration=None, name=None): |
| 19 | SimpleService.__init__(self, configuration=configuration, name=name) |
| 20 | self.debug("{0} version: {1}".format(URLLIB3, URLLIB3_VERSION)) |
| 21 | self.url = self.configuration.get('url') |
| 22 | self.user = self.configuration.get('user') |
| 23 | self.password = self.configuration.get('pass') |
| 24 | self.proxy_user = self.configuration.get('proxy_user') |
| 25 | self.proxy_password = self.configuration.get('proxy_pass') |
| 26 | self.proxy_url = self.configuration.get('proxy_url') |
| 27 | self.method = self.configuration.get('method', 'GET') |
| 28 | self.header = self.configuration.get('header') |
| 29 | self.body = self.configuration.get('body') |
| 30 | self.request_timeout = self.configuration.get('timeout', 1) |
| 31 | self.respect_retry_after_header = self.configuration.get('respect_retry_after_header') |
| 32 | self.tls_verify = self.configuration.get('tls_verify') |
| 33 | self.tls_ca_file = self.configuration.get('tls_ca_file') |
| 34 | self.tls_key_file = self.configuration.get('tls_key_file') |
| 35 | self.tls_cert_file = self.configuration.get('tls_cert_file') |
| 36 | self._manager = None |
| 37 | |
| 38 | def __make_headers(self, **header_kw): |
| 39 | user = header_kw.get('user') or self.user |
| 40 | password = header_kw.get('pass') or self.password |
| 41 | proxy_user = header_kw.get('proxy_user') or self.proxy_user |
| 42 | proxy_password = header_kw.get('proxy_pass') or self.proxy_password |
| 43 | custom_header = header_kw.get('header') or self.header |
| 44 | header_params = dict(keep_alive=True) |
| 45 | proxy_header_params = dict() |
| 46 | if user and password: |
| 47 | header_params['basic_auth'] = '{user}:{password}'.format(user=user, |
| 48 | password=password) |
| 49 | if proxy_user and proxy_password: |
| 50 | proxy_header_params['proxy_basic_auth'] = '{user}:{password}'.format(user=proxy_user, |
| 51 | password=proxy_password) |
| 52 | try: |
| 53 | header, proxy_header = urllib3.make_headers(**header_params), urllib3.make_headers(**proxy_header_params) |
| 54 | except TypeError as error: |
| 55 | self.error('build_header() error: {error}'.format(error=error)) |
| 56 | return None, None |
| 57 | else: |
| 58 | header.update(custom_header or dict()) |
| 59 | return header, proxy_header |
| 60 | |
| 61 | def _build_manager(self, **header_kw): |
| 62 | header, proxy_header = self.__make_headers(**header_kw) |
| 63 | if header is None or proxy_header is None: |
| 64 | return None |
| 65 | proxy_url = header_kw.get('proxy_url') or self.proxy_url |
| 66 | if proxy_url: |
| 67 | manager = urllib3.ProxyManager |
| 68 | params = dict(proxy_url=proxy_url, headers=header, proxy_headers=proxy_header) |
| 69 | else: |
| 70 | manager = urllib3.PoolManager |
| 71 | params = dict(headers=header) |
| 72 | tls_cert_file = self.tls_cert_file |
| 73 | if tls_cert_file: |
| 74 | params['cert_file'] = tls_cert_file |
| 75 | # NOTE: key_file is useless without cert_file, but |
| 76 | # cert_file may include the key as well. |
| 77 | tls_key_file = self.tls_key_file |
| 78 | if tls_key_file: |
| 79 | params['key_file'] = tls_key_file |
| 80 | tls_ca_file = self.tls_ca_file |
| 81 | if tls_ca_file: |
| 82 | params['ca_certs'] = tls_ca_file |
| 83 | try: |
| 84 | url = header_kw.get('url') or self.url |
| 85 | is_https = url.startswith('https') |
| 86 | if skip_tls_verify(is_https, self.tls_verify, tls_ca_file): |
| 87 | params['ca_certs'] = None |
| 88 | params['cert_reqs'] = 'CERT_NONE' |
| 89 | if is_https: |
| 90 | params['assert_hostname'] = False |
| 91 | return manager(**params) |
| 92 | except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error: |
| 93 | self.error('build_manager() error:', str(error)) |
| 94 | return None |
| 95 | |
| 96 | def _get_raw_data(self, url=None, manager=None, **kwargs): |
| 97 | """ |
| 98 | Get raw data from http request |
| 99 | :return: str |
| 100 | """ |
| 101 | try: |
| 102 | response = self._do_request(url, manager, **kwargs) |
| 103 | except Exception as error: |
| 104 | self.error('Url: {url}. Error: {error}'.format(url=url or self.url, error=error)) |
| 105 | return None |
| 106 | |
| 107 | if response.status == 200: |
| 108 | if isinstance(response.data, str): |
| 109 | return response.data |
| 110 | return response.data.decode(errors='ignore') |
| 111 | else: |
| 112 | self.debug('Url: {url}. Http response status code: {code}'.format(url=url or self.url, code=response.status)) |
| 113 | return None |
| 114 | |
| 115 | def _get_raw_data_with_status(self, url=None, manager=None, retries=1, redirect=True, **kwargs): |
| 116 | """ |
| 117 | Get status and response body content from http request. Does not catch exceptions |
| 118 | :return: int, str |
| 119 | """ |
| 120 | response = self._do_request(url, manager, retries, redirect, **kwargs) |
| 121 | |
| 122 | if isinstance(response.data, str): |
| 123 | return response.status, response.data |
| 124 | return response.status, response.data.decode(errors='ignore') |
| 125 | |
| 126 | def _do_request(self, url=None, manager=None, retries=1, redirect=True, **kwargs): |
| 127 | """ |
| 128 | Get response from http request. Does not catch exceptions |
| 129 | :return: HTTPResponse |
| 130 | """ |
| 131 | url = url or self.url |
| 132 | manager = manager or self._manager |
| 133 | retry = urllib3.Retry(retries) |
| 134 | if hasattr(retry, 'respect_retry_after_header'): |
| 135 | retry.respect_retry_after_header = bool(self.respect_retry_after_header) |
| 136 | |
| 137 | if self.body: |
| 138 | kwargs['body'] = self.body |
| 139 | |
| 140 | response = manager.request( |
| 141 | method=self.method, |
| 142 | url=url, |
| 143 | timeout=self.request_timeout, |
| 144 | retries=retry, |
| 145 | headers=manager.headers, |
| 146 | redirect=redirect, |
| 147 | **kwargs |
| 148 | ) |
| 149 | return response |
| 150 | |
| 151 | def check(self): |
| 152 | """ |
| 153 | Format configuration data and try to connect to server |
| 154 | :return: boolean |
| 155 | """ |
| 156 | if not (self.url and isinstance(self.url, str)): |
| 157 | self.error('URL is not defined or type is not <str>') |
| 158 | return False |
| 159 | |
| 160 | self._manager = self._build_manager() |
| 161 | if not self._manager: |
| 162 | return False |
| 163 | |
| 164 | try: |
| 165 | data = self._get_data() |
| 166 | except Exception as error: |
| 167 | self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error)) |
| 168 | return False |
| 169 | |
| 170 | if isinstance(data, dict) and data: |
| 171 | return True |
| 172 | self.error('_get_data() returned no data or type is not <dict>') |
| 173 | return False |
| 174 | |
| 175 | |
| 176 | def skip_tls_verify(is_https, tls_verify, tls_ca_file): |
| 177 | # default 'tls_verify' value is None |
| 178 | # logic is: |
| 179 | # - never skip if there is 'tls_ca_file' file |
| 180 | # - skip by default for https |
| 181 | # - do not skip by default for http |
| 182 | if tls_ca_file: |
| 183 | return False |
| 184 | if is_https and not tls_verify: |
| 185 | return True |
| 186 | return tls_verify is False |