much better performance with big files; add parse stats chart
lgz committed
Dec 23, 2016 at 22:26 UTC
4db5ff76c713a7741e9471a65171c3ced10858a3
1 file changed
+43
-20
python.d/isc_dhcpd.chart.py
+43
-20
@@ -3,8 +3,8 @@
3
# Author: l2isbad
4
5
from base import SimpleService
6
-from re import compile, search
7
-from time import mktime, strptime, gmtime
6
+from re import compile
7
+from time import mktime, strptime, gmtime, time
8
try:
9
from ipaddress import IPv4Address as ipaddress
10
from ipaddress import ip_network
@@ -23,14 +23,14 @@ class Service(SimpleService):
23
self.pools = self.configuration.get('pools')
24
25
# Will work only with 'default' db-time-format (weekday year/month/day hour:minute:second)
26
- # TODO: update the regex to parse correctly 'local' db-time-format
26
+ # TODO: update algorithm to parse correctly 'local' db-time-format
27
# (epoch <seconds-since-epoch>; # <day-name> <month-name> <day-number> <hours>:<minutes>:<seconds> <year>)
28
# Also only ipv4 supported
29
- self.regex = compile(r'(\d+(?:\.\d+){3}).*?((?<=ends )[0-9].*?(?=;))')
29
+ self.regex = compile(r'\d+(?:\.\d+){3}')
30
31
def check(self):
32
if not self._get_raw_data():
33
- self.error('Make sure leases_path is correct and dhcpd.leases is readable by netdata')
33
+ self.error('Make sure leases_path is correct and leases log file is readable by netdata')
34
return False
35
elif not have_ipaddress:
36
self.error('No ipaddress module. Please install (py2-ipaddress in case of python2)')
@@ -46,8 +46,9 @@ class Service(SimpleService):
46
return False
47
48
# Creating dynamic charts
49
- self.order = ['utilization']
50
- self.definitions = {'utilization': {'options': [None, 'Pools utilization', 'used %', 'Utulization', 'isc_dhcpd.util', 'line'], 'lines': []} }
49
+ self.order = ['parse_time', 'utilization']
50
+ self.definitions = {'utilization': {'options': [None, 'Pools utilization', 'used %', 'Utulization', 'isc_dhcpd.util', 'line'], 'lines': []},
51
+ 'parse_time': {'options': [None, 'Parse time', 'ms', 'Parse statistics', 'isc_dhcpd.parse', 'line'], 'lines': [['ptime', 'time', 'absolute']]}}
52
for pool in self.pools:
53
self.definitions['utilization']['lines'].append([''.join(['ut_', pool]), pool, 'absolute'])
54
self.order.append(''.join(['leases_', pool]))
@@ -60,35 +61,55 @@ class Service(SimpleService):
61
62
def _get_raw_data(self):
63
"""
63
- Open log file
64
- :return: str
64
+ Parses log file
65
+ :return: tuple(
66
+ [ipaddress, lease end time, ...],
67
+ length of list,
68
+ time to parse leases file
69
+ )
70
"""
71
try:
67
- with open(self.leases_path, 'rt') as leases:
68
- result = leases.read()
72
+ with open(self.leases_path, 'rt') as dhcp_leases:
73
+ raw_result = []
74
+
75
+ time_start = time()
76
+ for line in dhcp_leases:
77
+ if line[0:3] == 'lea':
78
+ raw_result.append(self.regex.search(line).group())
79
+ elif line[2:6] == 'ends':
80
+ raw_result.append(line[7:28])
81
+ else:
82
+ continue
83
+ time_end = time()
84
+ file_parse_time = round((time_end - time_start) * 1000)
85
+
86
except Exception:
87
return None
88
+
89
else:
90
+ raw_result_length = len(raw_result)
91
+ result = (raw_result, raw_result_length, file_parse_time)
92
return result
93
94
def _get_data(self):
95
"""
76
- Parse dhcpd.leases file.
96
+ :return: dict
97
"""
98
raw_leases = self._get_raw_data()
79
- all_leases = dict(self.regex.findall(' '.join(raw_leases.split())))
80
-
81
- if not all_leases:
82
- self.error('Cant parse leases file correctly')
99
+
100
+ if not raw_leases:
101
return None
102
103
+ # Result: {ipaddress: end lease time, ...}
104
+ all_leases = dict(zip([raw_leases[0][_] for _ in range(0, raw_leases[1], 2)], [raw_leases[0][_] for _ in range(1, raw_leases[1], 2)]))
105
+
106
# Result: [active binding, active binding....]. (Expire time (ends date;) - current time > 0)
107
active_leases = [k for k, v in all_leases.items() if is_bind_active(all_leases[k])]
108
109
# Result: {pool: number of active bindings in pool, ...}
110
pools_count = {pool: len([lease for lease in active_leases if is_address_in(lease, pool)]) for pool in self.pools}
111
91
- # Result: {pool: number of host ip addresses in pool, }
112
+ # Result: {pool: number of host ip addresses in pool, ...}
113
pools_max = {pool: (2 ** (32 - int(pool.split('/')[1])) - 2) for pool in self.pools}
114
115
# Result: {pool: % utilization, ....} (percent)
@@ -98,9 +119,11 @@ class Service(SimpleService):
119
final_count = {''.join(['le_', k]): v for k, v in pools_count.items()}
120
final_util = {''.join(['ut_', k]): v for k, v in pools_util.items()}
121
101
- final_count.update(final_util)
102
-
103
- return final_count
122
+ to_netdata = {'ptime': int(raw_leases[2])}
123
+ to_netdata.update(final_util)
124
+ to_netdata.update(final_count)
125
+
126
+ return to_netdata
127
128
def is_bind_active(binding):
129
return mktime(strptime(binding, '%w %Y/%m/%d %H:%M:%S')) - mktime(gmtime()) > 0