UrlService: http proxy handler added, multi url openers
Ilya committed
Apr 30, 2017 at 16:47 UTC
5d48c0ea0618af01d983234018cbcdb4557ffd92
1 file changed
+53
-27
python.d/python_modules/base.py
+53
-27
@@ -473,47 +473,73 @@ class UrlService(SimpleService):
473
self.user = self.configuration.get('user')
474
self.password = self.configuration.get('pass')
475
self.ss_cert = self.configuration.get('ss_cert')
476
+ self.proxy = self.configuration.get('proxy')
477
477
- def __add_openers(self):
478
- def self_signed_cert(ss_cert):
479
- if ss_cert:
480
- try:
481
- ctx = ssl.create_default_context()
482
- ctx.check_hostname = False
483
- ctx.verify_mode = ssl.CERT_NONE
484
- return urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
485
- except AttributeError:
486
- return None
487
- else:
488
- return None
478
+ def __add_openers(self, user=None, password=None, ss_cert=None, proxy=None, url=None):
479
+ user = user or self.user
480
+ password = password or self.password
481
+ ss_cert = ss_cert or self.ss_cert
482
+ proxy = proxy or self.proxy
483
490
- self.opener = self_signed_cert(self.ss_cert) or urllib2.build_opener()
484
+ handlers = list()
485
492
- # HTTP Basic Auth
493
- if self.user and self.password:
494
- url_parse = urlparse(self.url)
486
+ # HTTP Basic Auth handler
487
+ if all([user, password, isinstance(user, str), isinstance(password, str)]):
488
+ url = url or self.url
489
+ url_parse = urlparse(url)
490
top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
491
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
497
- passman.add_password(None, top_level_url, self.user, self.password)
498
- self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
492
+ passman.add_password(None, top_level_url, user, password)
493
+ handlers.append(urllib2.HTTPBasicAuthHandler(passman))
494
self.debug("Enabling HTTP basic auth")
495
501
- def _get_raw_data(self, custom_url=None):
496
+ # HTTPS handler
497
+ # Self-signed certificate ignore
498
+ if ss_cert:
499
+ try:
500
+ ctx = ssl.create_default_context()
501
+ ctx.check_hostname = False
502
+ ctx.verify_mode = ssl.CERT_NONE
503
+ except AttributeError:
504
+ self.error('HTTPS self-signed certificate ignore not enabled')
505
+ else:
506
+ handlers.append(urllib2.HTTPSHandler(context=ctx))
507
+ self.debug("Enabling HTTP self-signed certificate ignore")
508
+
509
+ # PROXY handler
510
+ if proxy and isinstance(proxy, str) and not ss_cert:
511
+ handlers.append(urllib2.ProxyHandler(dict(http=proxy)))
512
+ self.debug("Enabling HTTP proxy handler (%s)" % proxy)
513
+
514
+ opener = urllib2.build_opener(*handlers)
515
+ return opener
516
+
517
+ def _build_opener(self, **kwargs):
518
+ try:
519
+ return self.__add_openers(**kwargs)
520
+ except TypeError as error:
521
+ self.error('build_opener() error:', str(error))
522
+ return None
523
+
524
+ def _get_raw_data(self, url=None, opener=None):
525
"""
526
Get raw data from http request
527
:return: str
528
"""
506
- raw_data = None
507
- f = None
529
+ data = None
530
try:
509
- f = self.opener.open(custom_url or self.url, timeout=self.update_every * 2)
510
- raw_data = f.read().decode('utf-8', 'ignore')
531
+ opener = opener or self.opener
532
+ data = opener.open(url or self.url, timeout=self.update_every * 2)
533
+ raw_data = data.read().decode('utf-8', 'ignore')
534
+ except urllib2.URLError as error:
535
+ self.error('Url: %s. Error: %s' % (url or self.url, str(error)))
536
+ return None
537
except Exception as error:
512
- self.error('Url: %s. Error: %s' %(custom_url or self.url, str(error)))
538
+ self.error(str(error))
539
return None
540
finally:
515
- if f is not None: f.close()
516
-
541
+ if data is not None:
542
+ data.close()
543
return raw_data or None
544
545
def check(self):
@@ -525,7 +551,7 @@ class UrlService(SimpleService):
551
self.error('URL is not defined or type is not <str>')
552
return False
553
528
- self.__add_openers()
554
+ self.opener = self.__add_openers()
555
556
try:
557
data = self._get_data()