python loaders cleanup (#5602)
<!-- Describe the change in summary section, including rationale and degin decisions. Include "Fixes #nnn" if you are fixing an existing issue. In "Component Name" section write which component is changed in this PR. This will help us review your PR quicker. If you have more information you want to add, write them in "Additional Information" section. This is usually used to help others understand your motivation behind this change. A step-by-step reproduction of the problem is helpful if there is no related issue. --> ##### Summary python loaders module cleanup - move `load_module` to `python.d.plugin` - use `load_config` in unbound module instead of YamlOrderedLoad class - remove all classes from `loaders.py` - no longer used ##### Component Name [`collectors/python.d.plugin/python_modules/bases/loaders`](https://github.com/netdata/netdata/blob/master/collectors/python.d.plugin/python_modules/bases/loaders.py) ##### Additional Information
Ilya Mashchenko committed
Mar 11, 2019 at 16:58 UTC
28e0d7526d51ea813ac6a28b95a4ab6997394d19
3 files changed
+28
-67
collectors/python.d.plugin/python.d.plugin.in
+18
-4
@@ -18,6 +18,15 @@ import re
18
import sys
19
import time
20
import threading
21
+import types
22
+
23
+PY_VERSION = sys.version_info[:2]
24
+
25
+if PY_VERSION > (3, 1):
26
+ from importlib.machinery import SourceFileLoader
27
+else:
28
+ from imp import load_source as SourceFileLoader
29
+
30
31
ENV_NETDATA_USER_CONFIG_DIR = 'NETDATA_USER_CONFIG_DIR'
32
ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR'
@@ -73,7 +82,6 @@ sys.path.append(DIRS.pythond_packages)
82
from bases.collection import safe_print
83
from bases.loggers import PythonDLogger
84
from bases.loaders import load_config
76
-from bases.loaders import load_module as _load_module
85
86
try:
87
from collections import OrderedDict
@@ -138,7 +146,10 @@ class HeartBeat(threading.Thread):
146
147
def load_module(name):
148
abs_path = os.path.join(DIRS.modules, '{0}{1}'.format(name, MODULE_SUFFIX))
141
- return _load_module(name, abs_path)
149
+ module = SourceFileLoader(name, abs_path)
150
+ if isinstance(module, types.ModuleType):
151
+ return module
152
+ return module.load_module()
153
154
155
def multi_path_find(name, paths):
@@ -504,6 +515,8 @@ class Plugin:
515
self.task_queue.join()
516
self.dequeue_results()
517
self.result_queue.join()
518
+ self.task_queue.close()
519
+ self.result_queue.close()
520
self.log.info('stopping checker process')
521
worker.join()
522
@@ -588,7 +601,8 @@ class Plugin:
601
while True:
602
self.runs += 1
603
591
- if threading.active_count() <= 3 and not self.auto_detection_jobs:
604
+ # threads: main + heartbeat
605
+ if threading.active_count() <= 2 and not self.auto_detection_jobs:
606
return
607
608
time.sleep(1)
@@ -678,7 +692,7 @@ def main():
692
if cmd.trace:
693
logger.log_traceback = True
694
681
- logger.info('using python v{0}'.format(sys.version_info[:2][0]))
695
+ logger.info('using python v{0}'.format(PY_VERSION[0]))
696
697
unknown_modules = set(cmd.modules_to_run) - set(AVAILABLE_MODULES)
698
if unknown_modules:
collectors/python.d.plugin/python_modules/bases/loaders.py
+3
-60
@@ -3,7 +3,6 @@
3
# Author: Ilya Mashchenko (l2isbad)
4
# SPDX-License-Identifier: GPL-3.0-or-later
5
6
-import types
6
7
from sys import version_info
8
@@ -18,78 +17,22 @@ except ImportError:
17
from yaml import SafeLoader as YamlSafeLoader
18
19
21
-if PY_VERSION > (3, 1):
22
- from importlib.machinery import SourceFileLoader
23
- DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map'
24
-else:
25
- from imp import load_source as SourceFileLoader
26
- DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map'
27
-
20
try:
21
from collections import OrderedDict
22
except ImportError:
23
from third_party.ordereddict import OrderedDict
24
25
34
-def dict_constructor(loader, node):
35
- return OrderedDict(loader.construct_pairs(node))
26
+DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' if PY_VERSION > (3, 1) else u'tag:yaml.org,2002:map'
27
28
38
-def safe_load(stream):
39
- loader = YamlSafeLoader(stream)
40
- try:
41
- return loader.get_single_data()
42
- finally:
43
- loader.dispose()
29
+def dict_constructor(loader, node):
30
+ return OrderedDict(loader.construct_pairs(node))
31
32
33
YamlSafeLoader.add_constructor(DEFAULT_MAPPING_TAG, dict_constructor)
34
35
49
-class YamlOrderedLoader:
50
- @staticmethod
51
- def load_config_from_file(file_name):
52
- opened, loaded = False, False
53
- try:
54
- stream = open(file_name, 'r')
55
- opened = True
56
- loader = YamlSafeLoader(stream)
57
- loaded = True
58
- parsed = loader.get_single_data() or dict()
59
- except Exception as error:
60
- return dict(), error
61
- else:
62
- return parsed, None
63
- finally:
64
- if opened:
65
- stream.close()
66
- if loaded:
67
- loader.dispose()
68
-
69
-
70
-class SourceLoader:
71
- @staticmethod
72
- def load_module_from_file(name, path):
73
- try:
74
- loaded = SourceFileLoader(name, path)
75
- if isinstance(loaded, types.ModuleType):
76
- return loaded, None
77
- return loaded.load_module(), None
78
- except Exception as error:
79
- return None, error
80
-
81
-
82
-class ModuleAndConfigLoader(YamlOrderedLoader, SourceLoader):
83
- pass
84
-
85
-
86
-def load_module(name, path):
87
- module = SourceFileLoader(name, path)
88
- if isinstance(module, types.ModuleType):
89
- return module
90
- return module.load_module()
91
-
92
-
36
def load_yaml(stream):
37
loader = YamlSafeLoader(stream)
38
try:
collectors/python.d.plugin/unbound/unbound.chart.py
+7
-3
@@ -9,7 +9,7 @@ import sys
9
from copy import deepcopy
10
11
from bases.FrameworkServices.SocketService import SocketService
12
-from bases.loaders import YamlOrderedLoader
12
+from bases.loaders import load_config
13
14
PRECISION = 1000
15
@@ -169,7 +169,7 @@ def _get_perthread_info(thread):
169
for key, value in PER_THREAD_STAT_MAP.items():
170
statmap[key.format(shortname=sname)] = (value[0].format(shortname=sname), value[1])
171
172
- return (charts, order, statmap)
172
+ return charts, order, statmap
173
174
175
class Service(SocketService):
@@ -205,7 +205,11 @@ class Service(SocketService):
205
def _auto_config(self):
206
if self.ubconf and os.access(self.ubconf, os.R_OK):
207
self.debug('Unbound config: {0}'.format(self.ubconf))
208
- conf = YamlOrderedLoader.load_config_from_file(self.ubconf)[0]
208
+ conf = dict()
209
+ try:
210
+ conf = load_config(self.ubconf)
211
+ except Exception as error:
212
+ self.error("error on loading '{0}' : {1}".format(self.ubconf, error))
213
if self.ext is None:
214
if 'extended-statistics' in conf['server']:
215
self.ext = conf['server']['extended-statistics']