@cryptotaxi247 / netdata-1 / commits / 88a6d0178

"smartd_log" fixes

lgz committed Oct 14, 2017 at 16:16 UTC 88a6d01780b034cdc1f3f487704921f004538e75
3 files changed +72 -110
conf.d/python.d/bind_rndc.conf
+2
@@ -105,5 +105,7 @@
105 # AUTO-DETECTION JOBS
106 # only one of them will run (they have the same name)
107 #
108 +update_every: 30
109 +#
110 #local:
111 # named_stats_path: '/var/log/bind/named.stats'
python.d/python_modules/bases/collection.py
+11
@@ -72,6 +72,17 @@ def find_binary(binary):
72 return None
73
74
75 +def read_last_line(f):
76 + with open(f, 'rb') as opened:
77 + opened.seek(-2, 2)
78 + while opened.read(1) != b'\n':
79 + opened.seek(-2, 1)
80 + if opened.tell() == 0:
81 + break
82 + result = opened.readline()
83 + return result.decode()
84 +
85 +
86 class OldVersionCompatibility:
87
88 def __init__(self):
python.d/smartd_log.chart.py
+59 -110
@@ -2,20 +2,11 @@
2 # Description: smart netdata python.d module
3 # Author: l2isbad, vorph1
4
5 +import os
6 from re import compile as r_compile
6 -from os import listdir, access, R_OK
7 -from os.path import isfile, join, getsize, basename, isdir
8 -try:
9 - from queue import Queue
10 -except ImportError:
11 - from Queue import Queue
12 -from threading import Thread
13 -from base import SimpleService
14 -from collections import namedtuple
15 -
16 -# default module values (can be overridden per job in `config`)
17 -update_every = 5
18 -priority = 60000
7 +
8 +from bases.collection import read_last_line
9 +from bases.FrameworkServices.SimpleService import SimpleService
10
11 # charts order (can be overridden if you want less charts, or different order)
12 ORDER = ['1', '4', '5', '7', '9', '12', '193', '194', '197', '198', '200']
@@ -95,7 +86,12 @@ SMART_ATTR = {
86 '250': 'Read Error Retry Rate'
87 }
88
98 -NAMED_DISKS = namedtuple('disks', ['name', 'size', 'number'])
89 +
90 +class Disk:
91 + def __init__(self, name, path):
92 + self.name = name
93 + self.path = path
94 + self.status = True
95
96
97 class Service(SimpleService):
@@ -105,118 +101,71 @@ class Service(SimpleService):
101 self.log_path = self.configuration.get('log_path', '/var/log/smartd')
102 self.raw_values = self.configuration.get('raw_values')
103 self.attr = self.configuration.get('smart_attributes', [])
108 - self.previous_data = dict()
104 + self.order = list()
105 + self.definitions = dict()
106 + self.disks = list()
107 +
108 + for path_to_disk in find_disks_in_log_path(self.log_path):
109 + self.disks.append(Disk(name=os.path.basename(path_to_disk), path=path_to_disk))
110
111 def check(self):
111 - # Can\'t start without smartd readable diks log files
112 - disks = find_disks_in_log_path(self.log_path)
113 - if not disks:
114 - self.error('Can\'t locate any smartd log files in %s' % self.log_path)
112 + if not self.disks:
113 + self.error('Can\'t locate any smartd log files in {0}'.format(self.log_path))
114 return False
115
117 - # List of namedtuples to track smartd log file size
118 - self.disks = [NAMED_DISKS(name=disks[i], size=0, number=i) for i in range(len(disks))]
116 + self.create_charts()
117 + return True
118
120 - if self._get_data():
121 - self.create_charts()
122 - return True
123 - else:
124 - self.error('Can\'t collect any data. Sorry.')
125 - return False
119 + def get_data(self):
120 + data = dict()
121 + for disk in self.disks:
122 + if not disk.status:
123 + continue
124
127 - def _get_raw_data(self, queue, disk):
128 - # The idea is to open a file.
129 - # Jump to the end.
130 - # Seek backward until '\n' symbol appears
131 - # If '\n' is found or it's the beginning of the file
132 - # readline()! (last or first line)
133 - with open(disk, 'rb') as f:
134 - f.seek(-2, 2)
135 - while f.read(1) != b'\n':
136 - f.seek(-2, 1)
137 - if f.tell() == 0:
138 - break
139 - result = f.readline()
140 -
141 - result = result.decode()
142 - result = self.regex.findall(result)
143 -
144 - queue.put([basename(disk), result])
145 -
146 - def _get_data(self):
147 - threads, result = list(), list()
148 - queue = Queue()
149 - to_netdata = dict()
150 -
151 - # If the size has not changed there is no reason to poll log files.
152 - disks = [disk for disk in self.disks if self.size_changed(disk)]
153 - if disks:
154 - for disk in disks:
155 - th = Thread(target=self._get_raw_data, args=(queue, disk.name))
156 - th.start()
157 - threads.append(th)
158 -
159 - for thread in threads:
160 - thread.join()
161 - result.append(queue.get())
162 - else:
163 - # Data from last real poll
164 - return self.previous_data or None
165 -
166 - for elem in result:
167 - for a, n, r in elem[1]:
168 - to_netdata.update({'_'.join([elem[0], a]): r if self.raw_values else n})
169 -
170 - self.previous_data.update(to_netdata)
171 -
172 - return to_netdata or None
173 -
174 - def size_changed(self, disk):
175 - # We are not interested in log files:
176 - # 1. zero size
177 - # 2. size is not changed since last poll
178 - try:
179 - size = getsize(disk.name)
180 - if size != disk.size and size:
181 - self.disks[disk.number] = disk._replace(size=size)
182 - return True
183 - else:
184 - return False
185 - except OSError:
186 - # Remove unreadable/nonexisting log files from list of disks and previous_data
187 - self.disks.remove(disk)
188 - self.previous_data = dict([(k, v) for k, v in self.previous_data.items() if basename(disk.name) not in k])
189 - return False
125 + try:
126 + last_line = read_last_line(disk.path)
127 + except OSError:
128 + disk.status = False
129 + continue
130 +
131 + result = self.regex.findall(last_line)
132 + if not result:
133 + continue
134 + for a, n, r in result:
135 + data.update({'_'.join([disk.name, a]): r if self.raw_values else n})
136 +
137 + return data or None
138
139 def create_charts(self):
140
193 - def create_lines(attrid):
141 + def create_lines(attr_id):
142 result = list()
143 for disk in self.disks:
196 - name = basename(disk.name)
197 - result.append(['_'.join([name, attrid]), name[:name.index('.')], 'absolute'])
144 + name = disk.name
145 + result.append(['_'.join([name, attr_id]), name[:name.index('.')], 'absolute'])
146 return result
147
200 - # Use configured attributes, if present. If something goes wrong we don't care.
201 - order = ORDER
148 try:
149 order = [attr for attr in self.attr.split() if attr in SMART_ATTR.keys()] or ORDER
204 - except Exception:
205 - pass
206 - self.order = [''.join(['attrid', i]) for i in order]
207 - self.definitions = dict()
150 + except AttributeError:
151 + order = ORDER
152 +
153 + self.order = [''.join(['attr_id', i]) for i in order]
154 units = 'raw' if self.raw_values else 'normalized'
155
210 - for k, v in dict([(k, v) for k, v in SMART_ATTR.items() if k in ORDER]).items():
211 - self.definitions.update({''.join(['attrid', k]): {
212 - 'options': [None, v, units, v.lower(), 'smartd.attrid' + k, 'line'],
213 - 'lines': create_lines(k)}})
156 + for k, v in dict([(k, v) for k, v in SMART_ATTR.items() if k in order]).items():
157 + self.definitions[''.join(['attr_id', k])] =\
158 + {'options': [None, v, units, v.lower(), 'smartd.attr_id' + k, 'line'],
159 + 'lines': create_lines(k)}
160 +
161
162 def find_disks_in_log_path(log_path):
216 - # smartd log file is OK if:
217 - # 1. it is a file
218 - # 2. file name endswith with 'csv'
219 - # 3. file is readable
220 - if not isdir(log_path): return None
221 - return [join(log_path, f) for f in listdir(log_path)
222 - if all([isfile(join(log_path, f)), f.endswith('.csv'), access(join(log_path, f), R_OK)])]
163 + if not os.path.isdir(log_path):
164 + raise StopIteration
165 + for f in os.listdir(log_path):
166 + f = os.path.join(log_path, f)
167 + if all([os.path.isfile(f),
168 + os.access(f, os.R_OK),
169 + f.endswith('.csv'),
170 + os.path.getsize(f)]):
171 + yield f