UrlService: custom_url support for _get_data and minor fixes
Ilya committed
Mar 14, 2017 at 17:11 UTC
7462c0d53cac84be0020b6dc6d7f00f2a103d6d2
1 file changed
+50
-76
python.d/python_modules/base.py
+50
-76
@@ -27,6 +27,11 @@ import ssl
27
from subprocess import Popen, PIPE
28
from sys import exc_info
29
30
+try:
31
+ from urlparse import urlparse
32
+except ImportError:
33
+ from urllib.parse import urlparse
34
+
35
try:
36
import urllib.request as urllib2
37
except ImportError:
@@ -462,109 +467,78 @@ class SimpleService(threading.Thread):
467
468
469
class UrlService(SimpleService):
465
- # TODO add support for https connections
470
def __init__(self, configuration=None, name=None):
467
- self.url = ""
468
- self.user = None
469
- self.password = None
470
- self.proxies = {}
471
SimpleService.__init__(self, configuration=configuration, name=name)
472
+ self.url = self.configuration.get('url')
473
+ self.user = self.configuration.get('user')
474
+ self.password = self.configuration.get('pass')
475
+ self.ss_cert = self.configuration.get('ss_cert')
476
477
def __add_openers(self):
474
- # TODO add error handling
475
- if self.ss_cert:
476
- try:
477
- ctx = ssl.create_default_context()
478
- ctx.check_hostname = False
479
- ctx.verify_mode = ssl.CERT_NONE
480
- self.opener = urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
481
- except Exception as error:
482
- self.error(str(error))
483
- self.opener = urllib2.build_opener()
484
- else:
485
- self.opener = urllib2.build_opener()
486
-
487
- # Proxy handling
488
- # TODO currently self.proxies isn't parsed from configuration file
489
- # if len(self.proxies) > 0:
490
- # for proxy in self.proxies:
491
- # url = proxy['url']
492
- # # TODO test this:
493
- # if "user" in proxy and "pass" in proxy:
494
- # if url.lower().startswith('https://'):
495
- # url = 'https://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[8:]
496
- # else:
497
- # url = 'http://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[7:]
498
- # # FIXME move proxy auth to sth like this:
499
- # # passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
500
- # # passman.add_password(None, url, proxy['user'], proxy['password'])
501
- # # opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
502
- #
503
- # if url.lower().startswith('https://'):
504
- # opener.add_handler(urllib2.ProxyHandler({'https': url}))
505
- # else:
506
- # opener.add_handler(urllib2.ProxyHandler({'https': url}))
478
+ def self_signed_cert(ss_cert):
479
+ if ss_cert:
480
+ try:
481
+ ctx = ssl.create_default_context()
482
+ ctx.check_hostname = False
483
+ ctx.verify_mode = ssl.CERT_NONE
484
+ return urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
485
+ except AttributeError:
486
+ return None
487
+ else:
488
+ return None
489
+
490
+ self.opener = self_signed_cert(self.ss_cert) or urllib2.build_opener()
491
492
# HTTP Basic Auth
509
- if self.user is not None and self.password is not None:
493
+ if self.user and self.password:
494
+ url_parse = urlparse(self.url)
495
+ top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
496
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
511
- passman.add_password(None, self.url, self.user, self.password)
497
+ passman.add_password(None, top_level_url, self.user, self.password)
498
self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
499
self.debug("Enabling HTTP basic auth")
500
515
- #urllib2.install_opener(opener)
516
-
517
- def _get_raw_data(self):
501
+ def _get_raw_data(self, custom_url=None):
502
"""
503
Get raw data from http request
504
:return: str
505
"""
522
- raw = None
506
+ raw_data = None
507
+ f = None
508
try:
524
- f = self.opener.open(self.url, timeout=self.update_every * 2)
525
- # f = urllib2.urlopen(self.url, timeout=self.update_every * 2)
526
- except Exception as e:
527
- self.error(str(e))
509
+ f = self.opener.open(custom_url or self.url, timeout=self.update_every * 2)
510
+ raw_data = f.read().decode('utf-8', 'ignore')
511
+ except Exception as error:
512
+ self.error('Url: %s. Error: %s' %(custom_url or self.url, str(error)))
513
return None
529
-
530
- try:
531
- raw = f.read().decode('utf-8', 'ignore')
532
- except Exception as e:
533
- self.error(str(e))
514
finally:
535
- f.close()
536
- return raw
515
+ if f is not None: f.close()
516
+
517
+ return raw_data or None
518
519
def check(self):
520
"""
521
Format configuration data and try to connect to server
522
:return: boolean
523
"""
543
- if self.name is None or self.name == str(None):
544
- self.name = 'local'
545
- self.chart_name += "_" + self.name
546
- else:
547
- self.name = str(self.name)
548
- try:
549
- self.url = str(self.configuration['url'])
550
- except (KeyError, TypeError):
551
- pass
552
- try:
553
- self.user = str(self.configuration['user'])
554
- except (KeyError, TypeError):
555
- pass
556
- try:
557
- self.password = str(self.configuration['pass'])
558
- except (KeyError, TypeError):
559
- pass
560
- self.ss_cert = self.configuration.get('ss_cert')
524
+ if not (self.url and isinstance(self.url, str)):
525
+ self.error('URL is not defined or type is not <str>')
526
+ return False
527
+
528
self.__add_openers()
529
563
- test = self._get_data()
564
- if test is None or len(test) == 0:
530
+ try:
531
+ data = self._get_data()
532
+ except Exception as error:
533
+ self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error))
534
return False
566
- else:
535
+
536
+ if isinstance(data, dict) and data:
537
+ self._data_from_check = data
538
return True
539
+ else:
540
+ self.error("_get_data() returned no data or type is not <dict>")
541
+ return False
542
543
544
class SocketService(SimpleService):