@cryptotaxi247 / netdata-1 / commits / 937465480

go.d smartctl: add scsi read/write/verify error rate (#18119)

Ilya Mashchenko committed Jul 12, 2024 at 00:17 UTC 9374654807e7c6eb526189398a398bfcfdaaf571
6 files changed +323
src/go/plugin/go.d/modules/smartctl/charts.go
+80
@@ -16,6 +16,10 @@ const (
16 prioDeviceTemperature
17 prioDevicePowerCycleCount
18
19 + prioDeviceScsiReadErrors
20 + prioDeviceScsiWriteErrors
21 + prioDeviceScsiVerifyErrors
22 +
23 prioDeviceSmartAttributeDecoded
24 prioDeviceSmartAttributeNormalized
25 )
@@ -92,6 +96,54 @@ var (
96 }
97 )
98
99 +var deviceScsiErrorLogChartsTmpl = module.Charts{
100 + deviceScsiReadErrorsChartTmpl.Copy(),
101 + deviceScsiWriteErrorsChartTmpl.Copy(),
102 + deviceScsiVerifyErrorsChartTmpl.Copy(),
103 +}
104 +
105 +var (
106 + deviceScsiReadErrorsChartTmpl = module.Chart{
107 + ID: "device_%s_type_%s_read_errors_rate",
108 + Title: "Device read errors",
109 + Units: "errors/s",
110 + Fam: "scsi errors",
111 + Ctx: "smartctl.device_read_errors_rate",
112 + Type: module.Line,
113 + Priority: prioDeviceScsiReadErrors,
114 + Dims: module.Dims{
115 + {ID: "device_%s_type_%s_scsi_error_log_read_total_errors_corrected", Name: "corrected", Algo: module.Incremental},
116 + {ID: "device_%s_type_%s_scsi_error_log_read_total_uncorrected_errors", Name: "uncorrected", Algo: module.Incremental},
117 + },
118 + }
119 + deviceScsiWriteErrorsChartTmpl = module.Chart{
120 + ID: "device_%s_type_%s_write_errors_rate",
121 + Title: "Device write errors",
122 + Units: "errors/s",
123 + Fam: "scsi errors",
124 + Ctx: "smartctl.device_write_errors_rate",
125 + Type: module.Line,
126 + Priority: prioDeviceScsiWriteErrors,
127 + Dims: module.Dims{
128 + {ID: "device_%s_type_%s_scsi_error_log_write_total_errors_corrected", Name: "corrected", Algo: module.Incremental},
129 + {ID: "device_%s_type_%s_scsi_error_log_read_total_uncorrected_errors", Name: "uncorrected", Algo: module.Incremental},
130 + },
131 + }
132 + deviceScsiVerifyErrorsChartTmpl = module.Chart{
133 + ID: "device_%s_type_%s_verify_errors_rate",
134 + Title: "Device verify errors",
135 + Units: "errors/s",
136 + Fam: "scsi errors",
137 + Ctx: "smartctl.device_verify_errors_rate",
138 + Type: module.Line,
139 + Priority: prioDeviceScsiVerifyErrors,
140 + Dims: module.Dims{
141 + {ID: "device_%s_type_%s_scsi_error_log_verify_total_errors_corrected", Name: "corrected", Algo: module.Incremental},
142 + {ID: "device_%s_type_%s_scsi_error_log_verify_total_uncorrected_errors", Name: "uncorrected", Algo: module.Incremental},
143 + },
144 + }
145 +)
146 +
147 var (
148 deviceSmartAttributeDecodedChartTmpl = module.Chart{
149 ID: "device_%s_type_%s_smart_attr_%s",
@@ -132,6 +184,11 @@ func (s *Smartctl) addDeviceCharts(dev *smartDevice) {
184 s.Warning(err)
185 }
186 }
187 + if cs := s.newDeviceScsiErrorLogCharts(dev); cs != nil && len(*cs) > 0 {
188 + if err := charts.Add(*cs...); err != nil {
189 + s.Warning(err)
190 + }
191 + }
192
193 if err := s.Charts().Add(charts...); err != nil {
194 s.Warning(err)
@@ -235,6 +292,29 @@ func (s *Smartctl) newDeviceSmartAttrCharts(dev *smartDevice) *module.Charts {
292 return &charts
293 }
294
295 +func (s *Smartctl) newDeviceScsiErrorLogCharts(dev *smartDevice) *module.Charts {
296 + if dev.deviceType() != "scsi" || !dev.data.Get("scsi_error_counter_log").Exists() {
297 + return nil
298 + }
299 +
300 + charts := deviceScsiErrorLogChartsTmpl.Copy()
301 +
302 + for _, chart := range *charts {
303 + chart.ID = fmt.Sprintf(chart.ID, dev.deviceName(), dev.deviceType())
304 + chart.Labels = []module.Label{
305 + {Key: "device_name", Value: dev.deviceName()},
306 + {Key: "device_type", Value: dev.deviceType()},
307 + {Key: "model_name", Value: dev.modelName()},
308 + {Key: "serial_number", Value: dev.serialNumber()},
309 + }
310 + for _, dim := range chart.Dims {
311 + dim.ID = fmt.Sprintf(dim.ID, dev.deviceName(), dev.deviceType())
312 + }
313 + }
314 +
315 + return charts
316 +}
317 +
318 var attrNameReplacer = strings.NewReplacer(" ", "_", "/", "_")
319
320 func cleanAttributeName(attrName string) string {
src/go/plugin/go.d/modules/smartctl/collect.go
+24
@@ -136,6 +136,30 @@ func (s *Smartctl) collectSmartDevice(mx map[string]int64, dev *smartDevice) {
136 }
137 }
138 }
139 +
140 + if dev.deviceType() == "scsi" {
141 + sel := dev.data.Get("scsi_error_counter_log")
142 + if !sel.Exists() {
143 + return
144 + }
145 +
146 + for _, v := range []string{"read", "write", "verify"} {
147 + for _, n := range []string{
148 + //"errors_corrected_by_eccdelayed",
149 + //"errors_corrected_by_eccfast",
150 + //"errors_corrected_by_rereads_rewrites",
151 + "total_errors_corrected",
152 + "total_uncorrected_errors",
153 + } {
154 + key := fmt.Sprintf("%sscsi_error_log_%s_%s", px, v, n)
155 + metric := fmt.Sprintf("%s.%s", v, n)
156 +
157 + if m := sel.Get(metric); m.Exists() {
158 + mx[key] = m.Int()
159 + }
160 + }
161 + }
162 + }
163 }
164
165 func (s *Smartctl) isTimeToScan(now time.Time) bool {
src/go/plugin/go.d/modules/smartctl/metadata.yaml
+21
@@ -186,6 +186,27 @@ modules:
186 chart_type: line
187 dimensions:
188 - name: power
189 + - name: smartctl.device_read_errors_rate
190 + description: Device read errors
191 + unit: errors/s
192 + chart_type: line
193 + dimensions:
194 + - name: corrected
195 + - name: uncorrected
196 + - name: smartctl.device_write_errors_rate
197 + description: Device write errors
198 + unit: errors/s
199 + chart_type: line
200 + dimensions:
201 + - name: corrected
202 + - name: uncorrected
203 + - name: smartctl.device_verify_errors_rate
204 + description: Device verify errors
205 + unit: errors/s
206 + chart_type: line
207 + dimensions:
208 + - name: corrected
209 + - name: uncorrected
210 - name: smartctl.device_smart_attr_{attribute_name}
211 description: Device smart attribute {attribute_name}
212 unit: '{attribute_unit}'
src/go/plugin/go.d/modules/smartctl/smartctl_test.go
+41
@@ -26,6 +26,9 @@ var (
26
27 dataTypeNvmeScan, _ = os.ReadFile("testdata/type-nvme/scan.json")
28 dataTypeNvmeDeviceNvme0, _ = os.ReadFile("testdata/type-nvme/device-nvme0.json")
29 +
30 + dataTypeScsiScan, _ = os.ReadFile("testdata/type-scsi/scan.json")
31 + dataTypeScsiDeviceSda, _ = os.ReadFile("testdata/type-scsi/device-sda.json")
32 )
33
34 func Test_testDataIsValid(t *testing.T) {
@@ -39,6 +42,9 @@ func Test_testDataIsValid(t *testing.T) {
42
43 "dataTypeNvmeScan": dataTypeNvmeScan,
44 "dataTypeNvmeDeviceNvme0": dataTypeNvmeDeviceNvme0,
45 +
46 + "dataTypeScsiScan": dataTypeScsiScan,
47 + "dataTypeScsiDeviceSda": dataTypeScsiDeviceSda,
48 } {
49 require.NotNil(t, data, name)
50 }
@@ -289,6 +295,23 @@ func TestSmartctl_Collect(t *testing.T) {
295 "device_nvme0_type_nvme_temperature": 39,
296 },
297 },
298 + "success type scsi devices": {
299 + prepareMock: prepareMockOkTypeScsi,
300 + wantCharts: 7,
301 + wantMetrics: map[string]int64{
302 + "device_sda_type_scsi_power_cycle_count": 4,
303 + "device_sda_type_scsi_power_on_time": 5908920,
304 + "device_sda_type_scsi_scsi_error_log_read_total_errors_corrected": 647736,
305 + "device_sda_type_scsi_scsi_error_log_read_total_uncorrected_errors": 0,
306 + "device_sda_type_scsi_scsi_error_log_verify_total_errors_corrected": 0,
307 + "device_sda_type_scsi_scsi_error_log_verify_total_uncorrected_errors": 0,
308 + "device_sda_type_scsi_scsi_error_log_write_total_errors_corrected": 0,
309 + "device_sda_type_scsi_scsi_error_log_write_total_uncorrected_errors": 0,
310 + "device_sda_type_scsi_smart_status_failed": 0,
311 + "device_sda_type_scsi_smart_status_passed": 1,
312 + "device_sda_type_scsi_temperature": 34,
313 + },
314 + },
315 "error on scan": {
316 prepareMock: prepareMockErrOnScan,
317 },
@@ -374,6 +397,24 @@ func prepareMockOkTypeNvme() *mockSmartctlCliExec {
397 }
398 }
399
400 +func prepareMockOkTypeScsi() *mockSmartctlCliExec {
401 + return &mockSmartctlCliExec{
402 + errOnScan: false,
403 + scanData: dataTypeScsiScan,
404 + deviceDataFunc: func(deviceName, deviceType, powerMode string) ([]byte, error) {
405 + if deviceType != "scsi" {
406 + return nil, fmt.Errorf("unexpected device type %s", deviceType)
407 + }
408 + switch deviceName {
409 + case "/dev/sda":
410 + return dataTypeScsiDeviceSda, nil
411 + default:
412 + return nil, fmt.Errorf("unexpected device name %s", deviceName)
413 + }
414 + },
415 + }
416 +}
417 +
418 func prepareMockErrOnScan() *mockSmartctlCliExec {
419 return &mockSmartctlCliExec{
420 errOnScan: true,
src/go/plugin/go.d/modules/smartctl/testdata/type-scsi/device-sda.json new
+128
@@ -0,0 +1,128 @@
1 +{
2 + "json_format_version": [
3 + 1,
4 + 0
5 + ],
6 + "smartctl": {
7 + "version": [
8 + 7,
9 + 3
10 + ],
11 + "svn_revision": "5338",
12 + "platform_info": "REDACTED",
13 + "build_info": "(local build)",
14 + "argv": [
15 + "smartctl",
16 + "--json",
17 + "--all",
18 + "/dev/sda",
19 + "--device",
20 + "scsi"
21 + ],
22 + "exit_status": 0
23 + },
24 + "local_time": {
25 + "time_t": 1720689199,
26 + "asctime": "Thu Jul 11 09:13:19 2024 UTC"
27 + },
28 + "device": {
29 + "name": "/dev/sda",
30 + "info_name": "/dev/sda",
31 + "type": "scsi",
32 + "protocol": "SCSI"
33 + },
34 + "scsi_vendor": "HGST",
35 + "scsi_product": "REDACTED",
36 + "scsi_model_name": "REDACTED",
37 + "scsi_revision": "REDACTED",
38 + "scsi_version": "REDACTED",
39 + "user_capacity": {
40 + "blocks": 7814037168,
41 + "bytes": 4000787030016
42 + },
43 + "logical_block_size": 512,
44 + "scsi_lb_provisioning": {
45 + "name": "fully provisioned",
46 + "value": 0,
47 + "management_enabled": {
48 + "name": "LBPME",
49 + "value": 0
50 + },
51 + "read_zeros": {
52 + "name": "LBPRZ",
53 + "value": 0
54 + }
55 + },
56 + "rotation_rate": 7200,
57 + "form_factor": {
58 + "scsi_value": 2,
59 + "name": "3.5 inches"
60 + },
61 + "logical_unit_id": "REDACTED",
62 + "serial_number": "REDACTED",
63 + "device_type": {
64 + "scsi_terminology": "Peripheral Device Type [PDT]",
65 + "scsi_value": 0,
66 + "name": "disk"
67 + },
68 + "scsi_transport_protocol": {
69 + "name": "SAS (SPL-4)",
70 + "value": 6
71 + },
72 + "smart_support": {
73 + "available": true,
74 + "enabled": true
75 + },
76 + "temperature_warning": {
77 + "enabled": true
78 + },
79 + "smart_status": {
80 + "passed": true
81 + },
82 + "temperature": {
83 + "current": 34,
84 + "drive_trip": 85
85 + },
86 + "power_on_time": {
87 + "hours": 1641,
88 + "minutes": 22
89 + },
90 + "scsi_start_stop_cycle_counter": {
91 + "year_of_manufacture": "2013",
92 + "week_of_manufacture": "51",
93 + "specified_cycle_count_over_device_lifetime": 50000,
94 + "accumulated_start_stop_cycles": 4,
95 + "specified_load_unload_count_over_device_lifetime": 600000,
96 + "accumulated_load_unload_cycles": 119
97 + },
98 + "scsi_grown_defect_list": 0,
99 + "scsi_error_counter_log": {
100 + "read": {
101 + "errors_corrected_by_eccfast": 647707,
102 + "errors_corrected_by_eccdelayed": 29,
103 + "errors_corrected_by_rereads_rewrites": 0,
104 + "total_errors_corrected": 647736,
105 + "correction_algorithm_invocations": 586730,
106 + "gigabytes_processed": "36537.378",
107 + "total_uncorrected_errors": 0
108 + },
109 + "write": {
110 + "errors_corrected_by_eccfast": 0,
111 + "errors_corrected_by_eccdelayed": 0,
112 + "errors_corrected_by_rereads_rewrites": 0,
113 + "total_errors_corrected": 0,
114 + "correction_algorithm_invocations": 13549,
115 + "gigabytes_processed": "2811.293",
116 + "total_uncorrected_errors": 0
117 + },
118 + "verify": {
119 + "errors_corrected_by_eccfast": 0,
120 + "errors_corrected_by_eccdelayed": 0,
121 + "errors_corrected_by_rereads_rewrites": 0,
122 + "total_errors_corrected": 0,
123 + "correction_algorithm_invocations": 2146,
124 + "gigabytes_processed": "0.000",
125 + "total_uncorrected_errors": 0
126 + }
127 + }
128 +}
src/go/plugin/go.d/modules/smartctl/testdata/type-scsi/scan.json new
+29
@@ -0,0 +1,29 @@
1 +{
2 + "json_format_version": [
3 + 1,
4 + 0
5 + ],
6 + "smartctl": {
7 + "version": [
8 + 7,
9 + 3
10 + ],
11 + "svn_revision": "5338",
12 + "platform_info": "REDACTED",
13 + "build_info": "(local build)",
14 + "argv": [
15 + "smartctl",
16 + "--scan",
17 + "--json"
18 + ],
19 + "exit_status": 0
20 + },
21 + "devices": [
22 + {
23 + "name": "/dev/sda",
24 + "info_name": "/dev/sda",
25 + "type": "scsi",
26 + "protocol": "SCSI"
27 + }
28 + ]
29 +}