switch to urllib3 UrlService
Ilya committed
Jul 19, 2017 at 17:40 UTC
569e2b51cc69f1f8689e355cc3b74ba8c66a39d1
1 file changed
+66
-81
python.d/python_modules/base.py
+66
-81
@@ -17,38 +17,33 @@
17
#
18
# using ".encode()" in one thread can block other threads as well (only in python2)
19
20
-import time
20
import os
21
+import re
22
import socket
23
+import time
24
import threading
24
-import ssl
25
+
26
+import urllib3
27
+
28
+from glob import glob
29
from subprocess import Popen, PIPE
30
from sys import exc_info
27
-from glob import glob
28
-import re
29
-try:
30
- from urlparse import urlparse
31
-except ImportError:
32
- from urllib.parse import urlparse
33
-try:
34
- import urllib.request as urllib2
35
-except ImportError:
36
- import urllib2
31
+
32
try:
33
import MySQLdb
39
- PYMYSQL = True
34
+ PY_MYSQL = True
35
except ImportError:
36
try:
37
import pymysql as MySQLdb
43
- PYMYSQL = True
38
+ PY_MYSQL = True
39
except ImportError:
45
- PYMYSQL = False
40
+ PY_MYSQL = False
41
+
42
import msg
43
48
-try:
49
- PATH = os.getenv('PATH').split(':')
50
-except AttributeError:
51
- PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':')
44
+
45
+PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':')
46
+urllib3.disable_warnings()
47
48
49
# class BaseService(threading.Thread):
@@ -504,75 +499,64 @@ class UrlService(SimpleService):
499
self.url = self.configuration.get('url')
500
self.user = self.configuration.get('user')
501
self.password = self.configuration.get('pass')
507
- self.ss_cert = self.configuration.get('ss_cert')
508
- self.proxy = self.configuration.get('proxy')
502
+ self.proxy_url = self.configuration.get('proxy')
503
+ self._manager = None
504
510
- def __add_openers(self, user=None, password=None, ss_cert=None, proxy=None, url=None):
505
+ def __make_headers(self, user=None, password=None, proxy_url=None):
506
user = user or self.user
507
password = password or self.password
513
- ss_cert = ss_cert or self.ss_cert
514
- proxy = proxy or self.proxy
515
-
516
- handlers = list()
517
-
518
- # HTTP Basic Auth handler
519
- if all([user, password, isinstance(user, str), isinstance(password, str)]):
520
- url = url or self.url
521
- url_parse = urlparse(url)
522
- top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
523
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
524
- passman.add_password(None, top_level_url, user, password)
525
- handlers.append(urllib2.HTTPBasicAuthHandler(passman))
526
- self.debug("Enabling HTTP basic auth")
527
-
528
- # HTTPS handler
529
- # Self-signed certificate ignore
530
- if ss_cert:
531
- try:
532
- ctx = ssl.create_default_context()
533
- ctx.check_hostname = False
534
- ctx.verify_mode = ssl.CERT_NONE
535
- except AttributeError:
536
- self.error('HTTPS self-signed certificate ignore not enabled')
537
- else:
538
- handlers.append(urllib2.HTTPSHandler(context=ctx))
539
- self.debug("Enabling HTTP self-signed certificate ignore")
540
-
541
- # PROXY handler
542
- if proxy and isinstance(proxy, str) and not ss_cert:
543
- handlers.append(urllib2.ProxyHandler(dict(http=proxy)))
544
- self.debug("Enabling HTTP proxy handler (%s)" % proxy)
545
-
546
- opener = urllib2.build_opener(*handlers)
547
- return opener
548
-
549
- def _build_opener(self, **kwargs):
508
+ proxy_url = proxy_url or self.proxy_url
509
+ params = dict(keep_alive=True)
510
+ if user and password:
511
+ key = 'basic_auth' if not proxy_url else 'proxy_basic_auth'
512
+ params[key] = '{user}:{password}'.format(user=user, password=password)
513
+ return urllib3.make_headers(**params)
514
+
515
+ def _build_header(self, **header_kw):
516
try:
551
- return self.__add_openers(**kwargs)
517
+ return self.__make_headers(**header_kw)
518
except TypeError as error:
553
- self.error('build_opener() error:', str(error))
519
+ self.error('build_header() error: {error}'.format(error=error))
520
+ return None
521
+
522
+ def _build_manager(self, proxy_url=None, header_params=None):
523
+ header_params = header_params or dict()
524
+ header_params['proxy_url'] = proxy_url or self.proxy_url
525
+ header = self._build_header(**header_params)
526
+ if not header:
527
+ return None
528
+ if header_params['proxy_url']:
529
+ manager = urllib3.ProxyManager
530
+ params = dict(proxy_url=header_params['proxy_url'], proxy_headers=header)
531
+ else:
532
+ manager = urllib3.PoolManager
533
+ params = dict(headers=header)
534
+ try:
535
+ return manager(**params)
536
+ except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error:
537
+ self.error('build_manager() error:', str(error))
538
return None
539
556
- def _get_raw_data(self, url=None, opener=None):
540
+ def _get_raw_data(self, url=None, manager=None):
541
"""
542
Get raw data from http request
543
:return: str
544
"""
561
- data = None
545
try:
563
- opener = opener or self.opener
564
- data = opener.open(url or self.url, timeout=self.update_every * 2)
565
- raw_data = data.read().decode('utf-8', 'ignore')
566
- except urllib2.URLError as error:
567
- self.error('Url: %s. Error: %s' % (url or self.url, str(error)))
568
- return None
569
- except Exception as error:
570
- self.error(str(error))
546
+ url = url or self.url
547
+ manager = manager or self._manager
548
+ response = manager.request(method='GET',
549
+ url=url,
550
+ timeout=1,
551
+ retries=1,
552
+ headers=manager.headers)
553
+ except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error:
554
+ self.error('Url: {url}. Error: {error}'.format(url=url, error=error))
555
return None
572
- finally:
573
- if data is not None:
574
- data.close()
575
- return raw_data or None
556
+ if response.status == 200:
557
+ return response.data.decode() or None
558
+ self.error('Url: {url}. Http response status code: {code}'.format(url=url, code=response.status))
559
+ return None
560
561
def check(self):
562
"""
@@ -583,20 +567,21 @@ class UrlService(SimpleService):
567
self.error('URL is not defined or type is not <str>')
568
return False
569
586
- self.opener = self.__add_openers()
570
+ self._manager = self._build_manager()
571
+ if not self._manager:
572
+ return False
573
574
try:
575
data = self._get_data()
576
except Exception as error:
591
- self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error))
577
+ self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error))
578
return False
579
580
if isinstance(data, dict) and data:
581
self._data_from_check = data
582
return True
597
- else:
598
- self.error("_get_data() returned no data or type is not <dict>")
599
- return False
583
+ self.error('_get_data() returned no data or type is not <dict>')
584
+ return False
585
586
587
class SocketService(SimpleService):
@@ -1046,7 +1031,7 @@ class MySQLService(SimpleService):
1031
log_error('Unsupported "queries" format. Must be not empty <dict>')
1032
return None
1033
1049
- if not PYMYSQL:
1034
+ if not PY_MYSQL:
1035
self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin')
1036
return False
1037