1ef00cd9f · cryptotaxi247/netdata-1

@@ -3,174 +3,344 @@

3 # Author: l2isbad, vorph1

4

5 import os

6 -from re import compile as r_compile

6 +import re

7 +

8 +from collections import namedtuple

9 +from time import time

10

11 from bases.collection import read_last_line

12 from bases.FrameworkServices.SimpleService import SimpleService

13

14 # charts order (can be overridden if you want less charts, or different order)

12 -ORDER = ['1', '4', '5', '7', '9', '12', '193', '194', '197', '198', '200']

15 +# ORDER = ['1', '4', '5', '7', '9', '12', '193', '194', '197', '198', '200']

16 +ORDER = ['194']

17

18 SMART_ATTR = {

15 - '1': 'Read Error Rate',

16 - '2': 'Throughput Performance',

17 - '3': 'Spin-Up Time',

18 - '4': 'Start/Stop Count',

19 - '5': 'Reallocated Sectors Count',

20 - '6': 'Read Channel Margin',

21 - '7': 'Seek Error Rate',

22 - '8': 'Seek Time Performance',

23 - '9': 'Power-On Hours Count',

24 - '10': 'Spin-up Retries',

25 - '11': 'Calibration Retries',

26 - '12': 'Power Cycle Count',

27 - '13': 'Soft Read Error Rate',

28 - '100': 'Erase/Program Cycles',

29 - '103': 'Translation Table Rebuild',

30 - '108': 'Unknown (108)',

31 - '170': 'Reserved Block Count',

32 - '171': 'Program Fail Count',

33 - '172': 'Erase Fail Count',

34 - '173': 'Wear Leveller Worst Case Erase Count',

35 - '174': 'Unexpected Power Loss',

36 - '175': 'Program Fail Count',

37 - '176': 'Erase Fail Count',

38 - '177': 'Wear Leveling Count',

39 - '178': 'Used Reserved Block Count',

40 - '179': 'Used Reserved Block Count',

41 - '180': 'Unused Reserved Block Count',

42 - '181': 'Program Fail Count',

43 - '182': 'Erase Fail Count',

44 - '183': 'SATA Downshifts',

45 - '184': 'End-to-End error',

46 - '185': 'Head Stability',

47 - '186': 'Induced Op-Vibration Detection',

48 - '187': 'Reported Uncorrectable Errors',

49 - '188': 'Command Timeout',

50 - '189': 'High Fly Writes',

51 - '190': 'Temperature',

52 - '191': 'G-Sense Errors',

53 - '192': 'Power-Off Retract Cycles',

54 - '193': 'Load/Unload Cycles',

55 - '194': 'Temperature',

56 - '195': 'Hardware ECC Recovered',

57 - '196': 'Reallocation Events',

58 - '197': 'Current Pending Sectors',

59 - '198': 'Off-line Uncorrectable',

60 - '199': 'UDMA CRC Error Rate',

61 - '200': 'Write Error Rate',

62 - '201': 'Soft Read Errors',

63 - '202': 'Data Address Mark Errors',

64 - '203': 'Run Out Cancel',

65 - '204': 'Soft ECC Corrections',

66 - '205': 'Thermal Asperity Rate',

67 - '206': 'Flying Height',

68 - '207': 'Spin High Current',

69 - '209': 'Offline Seek Performance',

70 - '220': 'Disk Shift',

71 - '221': 'G-Sense Error Rate',

72 - '222': 'Loaded Hours',

73 - '223': 'Load/Unload Retries',

74 - '224': 'Load Friction',

75 - '225': 'Load/Unload Cycles',

76 - '226': 'Load-in Time',

77 - '227': 'Torque Amplification Count',

78 - '228': 'Power-Off Retracts',

79 - '230': 'GMR Head Amplitude',

80 - '231': 'Temperature',

81 - '232': 'Available Reserved Space',

82 - '233': 'Media Wearout Indicator',

83 - '240': 'Head Flying Hours',

84 - '241': 'Total LBAs Written',

85 - '242': 'Total LBAs Read',

86 - '250': 'Read Error Retry Rate'

19 + '1': 'Read Error Rate',

20 + '2': 'Throughput Performance',

21 + '3': 'Spin-Up Time',

22 + '4': 'Start/Stop Count',

23 + '5': 'Reallocated Sectors Count',

24 + '6': 'Read Channel Margin',

25 + '7': 'Seek Error Rate',

26 + '8': 'Seek Time Performance',

27 + '9': 'Power-On Hours Count',

28 + '10': 'Spin-up Retries',

29 + '11': 'Calibration Retries',

30 + '12': 'Power Cycle Count',

31 + '13': 'Soft Read Error Rate',

32 + '100': 'Erase/Program Cycles',

33 + '103': 'Translation Table Rebuild',

34 + '108': 'Unknown (108)',

35 + '170': 'Reserved Block Count',

36 + '171': 'Program Fail Count',

37 + '172': 'Erase Fail Count',

38 + '173': 'Wear Leveller Worst Case Erase Count',

39 + '174': 'Unexpected Power Loss',

40 + '175': 'Program Fail Count',

41 + '176': 'Erase Fail Count',

42 + '177': 'Wear Leveling Count',

43 + '178': 'Used Reserved Block Count',

44 + '179': 'Used Reserved Block Count',

45 + '180': 'Unused Reserved Block Count',

46 + '181': 'Program Fail Count',

47 + '182': 'Erase Fail Count',

48 + '183': 'SATA Downshifts',

49 + '184': 'End-to-End error',

50 + '185': 'Head Stability',

51 + '186': 'Induced Op-Vibration Detection',

52 + '187': 'Reported Uncorrectable Errors',

53 + '188': 'Command Timeout',

54 + '189': 'High Fly Writes',

55 + '190': 'Temperature',

56 + '191': 'G-Sense Errors',

57 + '192': 'Power-Off Retract Cycles',

58 + '193': 'Load/Unload Cycles',

59 + '194': 'Temperature',

60 + '195': 'Hardware ECC Recovered',

61 + '196': 'Reallocation Events',

62 + '197': 'Current Pending Sectors',

63 + '198': 'Off-line Uncorrectable',

64 + '199': 'UDMA CRC Error Rate',

65 + '200': 'Write Error Rate',

66 + '201': 'Soft Read Errors',

67 + '202': 'Data Address Mark Errors',

68 + '203': 'Run Out Cancel',

69 + '204': 'Soft ECC Corrections',

70 + '205': 'Thermal Asperity Rate',

71 + '206': 'Flying Height',

72 + '207': 'Spin High Current',

73 + '209': 'Offline Seek Performance',

74 + '220': 'Disk Shift',

75 + '221': 'G-Sense Error Rate',

76 + '222': 'Loaded Hours',

77 + '223': 'Load/Unload Retries',

78 + '224': 'Load Friction',

79 + '225': 'Load/Unload Cycles',

80 + '226': 'Load-in Time',

81 + '227': 'Torque Amplification Count',

82 + '228': 'Power-Off Retracts',

83 + '230': 'GMR Head Amplitude',

84 + '231': 'Temperature',

85 + '232': 'Available Reserved Space',

86 + '233': 'Media Wearout Indicator',

87 + '240': 'Head Flying Hours',

88 + '241': 'Total LBAs Written',

89 + '242': 'Total LBAs Read',

90 + '250': 'Read Error Retry Rate'

91 }

92

93 +LIMIT = namedtuple('LIMIT', ['min', 'max'])

94

90 -class Disk:

91 - def __init__(self, name, path):

92 - self.name = name

95 +LIMITS = {

96 + '194': LIMIT(0, 200)

97 +}

98 +

99 +RESCAN_INTERVAL = 60

100 +

101 +REGEX = re.compile(

102 + '(\d+);' # attribute

103 + '(\d+);' # normalized value

104 + '(\d+)', # raw value

105 + re.X

106 +)

107 +

108 +

109 +def chart_template(attr, raw):

110 + chart_name = 'attr_id' + attr

111 + title = SMART_ATTR[attr]

112 + units = 'raw' if raw else 'normalized'

113 +

114 + return {

115 + chart_name: {

116 + 'options': [None, title, units, title.lower(), 'smartd_log.' + chart_name, 'line'],

117 + 'lines': []

118 + }

119 + }

120 +

121 +

122 +def handle_os_error(method):

123 + def on_call(*args):

124 + try:

125 + return method(*args)

126 + except OSError:

127 + return None

128 + return on_call

129 +

130 +

131 +class SmartAttribute(object):

132 + def __init__(self, idx, normalized, raw):

133 + self.id = idx

134 + self.normalized = normalized

135 + self._raw = raw

136 +

137 + @property

138 + def raw(self):

139 + if self.id in LIMITS:

140 + limit = LIMITS[self.id]

141 + if limit.min <= int(self._raw) <= limit.max:

142 + return self._raw

143 + return None

144 + return self._raw

145 +

146 + @raw.setter

147 + def raw(self, value):

148 + self._raw = value

149 +

150 +

151 +class DiskLogFile:

152 + def __init__(self, path):

153 self.path = path

154 + self.size = os.path.getsize(path)

155 +

156 + @handle_os_error

157 + def is_changed(self):

158 + new_size = os.path.getsize(self.path)

159 + old_size, self.size = self.size, new_size

160 +

161 + return new_size != old_size and new_size

162 +

163 + @staticmethod

164 + @handle_os_error

165 + def is_valid(log_file, exclude):

166 + return all([log_file.endswith('.csv'),

167 + not [p for p in exclude if p in log_file],

168 + os.access(log_file, os.R_OK),

169 + os.path.getsize(log_file)])

170 +

171 +

172 +class Disk:

173 + def __init__(self, full_path, age):

174 + self.log_file = DiskLogFile(full_path)

175 + self.name = os.path.basename(full_path).split('.')[-3]

176 + self.age = int(age)

177 self.status = True

178 + self.attributes = dict()

179 +

180 + self.get_attributes()

181 +

182 + def __eq__(self, other):

183 + if isinstance(other, Disk):

184 + return self.name == other.name

185 + return self.name == other

186 +

187 + @handle_os_error

188 + def is_active(self):

189 + return (time() - os.path.getmtime(self.log_file.path)) / 60 < self.age

190 +

191 + @handle_os_error

192 + def get_attributes(self):

193 + last_line = read_last_line(self.log_file.path)

194 + self.attributes = dict((attr, SmartAttribute(attr, normalized, raw)) for attr, normalized, raw

195 + in REGEX.findall(last_line))

196 + return True

197 +

198 + def data(self, raw=None):

199 + data = dict()

200 + for attr in self.attributes.values():

201 + value = attr.raw if raw else attr.normalized

202 + if value is None:

203 + continue

204 + key = '_'.join([self.name, attr.id])

205 + data[key] = value

206 + return data

207

208

209 class Service(SimpleService):

210 def __init__(self, configuration=None, name=None):

211 SimpleService.__init__(self, configuration=configuration, name=name)

100 - self.regex = r_compile(r'(\d+);(\d+);(\d+)')

212 self.log_path = self.configuration.get('log_path', '/var/log/smartd')

102 - self.raw_values = self.configuration.get('raw_values')

103 - self.attr = self.configuration.get('smart_attributes', [])

104 - self.exclude_disks = self.configuration.get('exclude_disks', str()).split()

213 + self.raw = self.configuration.get('raw_values', True)

214 + self.exclude = self.configuration.get('exclude_disks', str()).split()

215 + self.age = self.configuration.get('age', 60)

216 +

217 + self.runs = 0

218 + self.disks = list()

219 self.order = list()

220 self.definitions = dict()

107 - self.disks = list()

108 -

109 - for path_to_disk in find_disks_in_log_path(self.log_path):

110 - disk_name = os.path.basename(path_to_disk).split('.')[-3]

111 - for pattern in self.exclude_disks:

112 - if pattern in disk_name:

113 - break

114 - else:

115 - self.disks.append(Disk(name=disk_name, path=path_to_disk))

221

222 def check(self):

223 + self.disks = self.scan()

224 +

225 if not self.disks:

119 - self.error('Can\'t locate any smartd log files in {0}'.format(self.log_path))

120 - return False

226 + return None

227 +

228 + user_defined_sa = self.configuration.get('smart_attributes')

229 +

230 + if user_defined_sa:

231 + order = user_defined_sa.split() or ORDER

232 + else:

233 + order = ORDER

234 +

235 + self.create_charts(order)

236

122 - self.create_charts()

237 return True

238

239 def get_data(self):

240 + self.runs += 1

241 +

242 + if self.runs % RESCAN_INTERVAL == 0:

243 + self.cleanup_and_rescan()

244 +

245 data = dict()

246 +

247 for disk in self.disks:

248 +

249 if not disk.status:

250 continue

251

131 - try:

132 - last_line = read_last_line(disk.path)

133 - except OSError:

252 + changed = disk.log_file.is_changed()

253 +

254 + # True = changed, False = unchanged, None = Exception

255 + if changed is None:

256 disk.status = False

257 continue

258

137 - result = self.regex.findall(last_line)

138 - if not result:

139 - continue

140 - for a, n, r in result:

141 - data.update({'_'.join([disk.name, a]): r if self.raw_values else n})

259 + if changed:

260 + success = disk.get_attributes()

261 + if not success:

262 + disk.status = False

263 + continue

264 +

265 + data.update(disk.data(self.raw))

266

267 return data or None

268

145 - def create_charts(self):

269 + def create_charts(self, order):

270 + for attr in order:

271 + chart_id = 'attr_id' + attr

272 + chart = chart_template(attr, self.raw)

273

147 - def create_lines(attr_id):

148 - result = list()

274 for disk in self.disks:

150 - result.append(['_'.join([disk.name, attr_id]), disk.name, 'absolute'])

151 - return result

275 + if attr not in disk.attributes:

276 + self.debug("'{disk}' has no attribute '{attr_id}'".format(disk=disk.name,

277 + attr_id=attr))

278 + continue

279

153 - try:

154 - order = [attr for attr in self.attr.split() if attr in SMART_ATTR.keys()] or ORDER

155 - except AttributeError:

156 - order = ORDER

280 + if self.raw and disk.attributes[attr].raw is None:

281 + self.debug("'{disk}' attribute '{attr_id}' value not in {limits}".format(disk=disk.name,

282 + attr_id=attr,

283 + limits=LIMITS[attr]))

284 + continue

285 + chart[chart_id]['lines'].append(['_'.join([disk.name, attr]), disk.name])

286 +

287 + self.order.append(chart_id)

288 + self.definitions.update(chart)

289 +

290 + def scan(self, only_new=None):

291 + new_disks = list()

292 + for f in os.listdir(self.log_path):

293 + full_path = os.path.join(self.log_path, f)

294 +

295 + if DiskLogFile.is_valid(full_path, self.exclude):

296 + disk = Disk(full_path, self.age)

297 +

298 + active = disk.is_active()

299 + if active is None:

300 + continue

301 +

302 + if active:

303 + if not only_new:

304 + new_disks.append(disk)

305 + else:

306 + if disk not in self.disks:

307 + new_disks.append(disk)

308 + else:

309 + if not only_new:

310 + self.debug("'{disk}' not updated in the last {age} minutes, "

311 + "skipping it.".format(disk=disk.name, age=self.age))

312 + return new_disks

313 +

314 + def cleanup_and_rescan(self):

315 + self.cleanup()

316 + new_disks = self.scan(only_new=True)

317 +

318 + for disk in new_disks:

319 + valid = False

320 +

321 + for chart in self.charts:

322 + idx = chart.id[7:]

323 +

324 + if idx in disk.attributes:

325 + valid = True

326 + dimension_id = '_'.join([disk.name, idx])

327 +

328 + if dimension_id in chart:

329 + chart.hide_dimension(dimension_id=dimension_id, reverse=True)

330 + else:

331 + chart.add_dimension([dimension_id, disk.name])

332 + if valid:

333 + self.disks.append(disk)

334 +

335 + def cleanup(self):

336 + for disk in self.disks:

337 +

338 + if not disk.is_active():

339 + disk.status = False

340 +

341 + if not disk.status:

342 + for chart in self.charts:

343 + dimension_id = '_'.join([disk.name, chart.id[7:]])

344 + chart.hide_dimension(dimension_id=dimension_id)

345

158 - self.order = [''.join(['attr_id', i]) for i in order]

159 - units = 'raw' if self.raw_values else 'normalized'

160 -

161 - for k, v in dict([(k, v) for k, v in SMART_ATTR.items() if k in order]).items():

162 - self.definitions[''.join(['attr_id', k])] =\

163 - {'options': [None, v, units, v.lower(), 'smartd.attr_id' + k, 'line'],

164 - 'lines': create_lines(k)}

165 -

166 -

167 -def find_disks_in_log_path(log_path):

168 - if not os.path.isdir(log_path):

169 - raise StopIteration

170 - for f in os.listdir(log_path):

171 - f = os.path.join(log_path, f)

172 - if all([os.path.isfile(f),

173 - os.access(f, os.R_OK),

174 - f.endswith('.csv'),

175 - os.path.getsize(f)]):

176 - yield f

346 + self.disks = [disk for disk in self.disks if disk.status]

smartd_log improvements