fix(go.d/smartctl): handle non-fatal smartctl exit codes (bits 2-7) (#21858)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Ilya Mashchenko committed
Mar 2, 2026 at 13:31 UTC
889cc90df0b4522ca6f01af45fde7c0ec3353e40
3 files changed
+171
-10
src/go/plugin/go.d/collector/smartctl/collect.go
+16
-5
@@ -106,12 +106,23 @@ func (c *Collector) processDeviceResult(mx map[string]int64, result deviceInfoRe
106
err := result.err
107
108
if err != nil {
109
- if resp != nil && isDeviceOpenFailedNoSuchDevice(resp) && !scanDev.extra {
110
- c.Infof("smartctl reported that device '%s' type '%s' no longer exists", scanDev.name, scanDev.typ)
111
- c.forceScan = true
112
- return nil
109
+ if resp != nil {
110
+ if isDeviceOpenFailedNoSuchDevice(resp) && !scanDev.extra {
111
+ c.Infof("smartctl reported that device '%s' type '%s' no longer exists", scanDev.name, scanDev.typ)
112
+ c.forceScan = true
113
+ return nil
114
+ }
115
+ // https://manpages.debian.org/bullseye/smartmontools/smartctl.8.en.html#EXIT_STATUS
116
+ // Bits 0-1 indicate fatal conditions (command line error, device open failure).
117
+ // Bits 2-7 indicate disk health conditions but the output data is still valid.
118
+ if !isExitStatusHasAnyBit(resp, 0, 1) {
119
+ c.Debugf("device '%s' type '%s': smartctl exit status has non-fatal bits set: %v", scanDev.name, scanDev.typ, err)
120
+ err = nil
121
+ }
122
+ }
123
+ if err != nil {
124
+ return fmt.Errorf("failed to get device info for '%s' type '%s': %v", scanDev.name, scanDev.typ, err)
125
}
114
- return fmt.Errorf("failed to get device info for '%s' type '%s': %v", scanDev.name, scanDev.typ, err)
126
}
127
128
if isDeviceInLowerPowerMode(resp) {
src/go/plugin/go.d/collector/smartctl/collector_test.go
+147
-2
@@ -3,6 +3,7 @@
3
package smartctl
4
5
import (
6
+ "bytes"
7
"context"
8
"fmt"
9
"os"
@@ -459,6 +460,120 @@ func TestCollector_Collect(t *testing.T) {
460
"device_sda_type_scsi_temperature": 34,
461
},
462
},
463
+ "success type sata devices non-fatal exit status": {
464
+ prepareMock: prepareMockOkTypeSataNonFatalExitStatus,
465
+ wantCharts: 68,
466
+ wantMetrics: map[string]int64{
467
+ "device_sda_type_sat_ata_smart_error_log_summary_count": 0,
468
+ "device_sda_type_sat_attr_current_pending_sector_decoded": 0,
469
+ "device_sda_type_sat_attr_current_pending_sector_normalized": 100,
470
+ "device_sda_type_sat_attr_current_pending_sector_raw": 0,
471
+ "device_sda_type_sat_attr_load_cycle_count_decoded": 360,
472
+ "device_sda_type_sat_attr_load_cycle_count_normalized": 100,
473
+ "device_sda_type_sat_attr_load_cycle_count_raw": 360,
474
+ "device_sda_type_sat_attr_offline_uncorrectable_decoded": 0,
475
+ "device_sda_type_sat_attr_offline_uncorrectable_normalized": 100,
476
+ "device_sda_type_sat_attr_offline_uncorrectable_raw": 0,
477
+ "device_sda_type_sat_attr_power-off_retract_count_decoded": 360,
478
+ "device_sda_type_sat_attr_power-off_retract_count_normalized": 100,
479
+ "device_sda_type_sat_attr_power-off_retract_count_raw": 360,
480
+ "device_sda_type_sat_attr_power_cycle_count_decoded": 12,
481
+ "device_sda_type_sat_attr_power_cycle_count_normalized": 100,
482
+ "device_sda_type_sat_attr_power_cycle_count_raw": 12,
483
+ "device_sda_type_sat_attr_power_on_hours_decoded": 8244,
484
+ "device_sda_type_sat_attr_power_on_hours_normalized": 99,
485
+ "device_sda_type_sat_attr_power_on_hours_raw": 8244,
486
+ "device_sda_type_sat_attr_raw_read_error_rate_decoded": 0,
487
+ "device_sda_type_sat_attr_raw_read_error_rate_normalized": 100,
488
+ "device_sda_type_sat_attr_raw_read_error_rate_raw": 0,
489
+ "device_sda_type_sat_attr_reallocated_event_count_decoded": 0,
490
+ "device_sda_type_sat_attr_reallocated_event_count_normalized": 100,
491
+ "device_sda_type_sat_attr_reallocated_event_count_raw": 0,
492
+ "device_sda_type_sat_attr_reallocated_sector_ct_decoded": 0,
493
+ "device_sda_type_sat_attr_reallocated_sector_ct_normalized": 100,
494
+ "device_sda_type_sat_attr_reallocated_sector_ct_raw": 0,
495
+ "device_sda_type_sat_attr_seek_error_rate_decoded": 0,
496
+ "device_sda_type_sat_attr_seek_error_rate_normalized": 100,
497
+ "device_sda_type_sat_attr_seek_error_rate_raw": 0,
498
+ "device_sda_type_sat_attr_seek_time_performance_decoded": 15,
499
+ "device_sda_type_sat_attr_seek_time_performance_normalized": 140,
500
+ "device_sda_type_sat_attr_seek_time_performance_raw": 15,
501
+ "device_sda_type_sat_attr_spin_retry_count_decoded": 0,
502
+ "device_sda_type_sat_attr_spin_retry_count_normalized": 100,
503
+ "device_sda_type_sat_attr_spin_retry_count_raw": 0,
504
+ "device_sda_type_sat_attr_spin_up_time_decoded": 281,
505
+ "device_sda_type_sat_attr_spin_up_time_normalized": 86,
506
+ "device_sda_type_sat_attr_spin_up_time_raw": 25788088601,
507
+ "device_sda_type_sat_attr_start_stop_count_decoded": 12,
508
+ "device_sda_type_sat_attr_start_stop_count_normalized": 100,
509
+ "device_sda_type_sat_attr_start_stop_count_raw": 12,
510
+ "device_sda_type_sat_attr_temperature_celsius_decoded": 49,
511
+ "device_sda_type_sat_attr_temperature_celsius_normalized": 43,
512
+ "device_sda_type_sat_attr_temperature_celsius_raw": 240519741489,
513
+ "device_sda_type_sat_attr_throughput_performance_decoded": 48,
514
+ "device_sda_type_sat_attr_throughput_performance_normalized": 148,
515
+ "device_sda_type_sat_attr_throughput_performance_raw": 48,
516
+ "device_sda_type_sat_attr_udma_crc_error_count_decoded": 0,
517
+ "device_sda_type_sat_attr_udma_crc_error_count_normalized": 100,
518
+ "device_sda_type_sat_attr_udma_crc_error_count_raw": 0,
519
+ "device_sda_type_sat_attr_unknown_attribute_decoded": 100,
520
+ "device_sda_type_sat_attr_unknown_attribute_normalized": 100,
521
+ "device_sda_type_sat_attr_unknown_attribute_raw": 100,
522
+ "device_sda_type_sat_power_cycle_count": 12,
523
+ "device_sda_type_sat_power_on_time": 29678400,
524
+ "device_sda_type_sat_smart_status_failed": 0,
525
+ "device_sda_type_sat_smart_status_passed": 1,
526
+ "device_sda_type_sat_temperature": 49,
527
+ "device_sdc_type_sat_ata_smart_error_log_summary_count": 0,
528
+ "device_sdc_type_sat_attr_available_reservd_space_decoded": 100,
529
+ "device_sdc_type_sat_attr_available_reservd_space_normalized": 100,
530
+ "device_sdc_type_sat_attr_available_reservd_space_raw": 100,
531
+ "device_sdc_type_sat_attr_command_timeout_decoded": 0,
532
+ "device_sdc_type_sat_attr_command_timeout_normalized": 100,
533
+ "device_sdc_type_sat_attr_command_timeout_raw": 0,
534
+ "device_sdc_type_sat_attr_end-to-end_error_decoded": 0,
535
+ "device_sdc_type_sat_attr_end-to-end_error_normalized": 100,
536
+ "device_sdc_type_sat_attr_end-to-end_error_raw": 0,
537
+ "device_sdc_type_sat_attr_media_wearout_indicator_decoded": 65406,
538
+ "device_sdc_type_sat_attr_media_wearout_indicator_normalized": 100,
539
+ "device_sdc_type_sat_attr_media_wearout_indicator_raw": 65406,
540
+ "device_sdc_type_sat_attr_power_cycle_count_decoded": 13,
541
+ "device_sdc_type_sat_attr_power_cycle_count_normalized": 100,
542
+ "device_sdc_type_sat_attr_power_cycle_count_raw": 13,
543
+ "device_sdc_type_sat_attr_power_on_hours_decoded": 8244,
544
+ "device_sdc_type_sat_attr_power_on_hours_normalized": 100,
545
+ "device_sdc_type_sat_attr_power_on_hours_raw": 8244,
546
+ "device_sdc_type_sat_attr_reallocated_sector_ct_decoded": 0,
547
+ "device_sdc_type_sat_attr_reallocated_sector_ct_normalized": 100,
548
+ "device_sdc_type_sat_attr_reallocated_sector_ct_raw": 0,
549
+ "device_sdc_type_sat_attr_reported_uncorrect_decoded": 0,
550
+ "device_sdc_type_sat_attr_reported_uncorrect_normalized": 100,
551
+ "device_sdc_type_sat_attr_reported_uncorrect_raw": 0,
552
+ "device_sdc_type_sat_attr_temperature_celsius_decoded": 27,
553
+ "device_sdc_type_sat_attr_temperature_celsius_normalized": 73,
554
+ "device_sdc_type_sat_attr_temperature_celsius_raw": 184684970011,
555
+ "device_sdc_type_sat_attr_total_lbas_read_decoded": 76778,
556
+ "device_sdc_type_sat_attr_total_lbas_read_normalized": 253,
557
+ "device_sdc_type_sat_attr_total_lbas_read_raw": 76778,
558
+ "device_sdc_type_sat_attr_total_lbas_written_decoded": 173833,
559
+ "device_sdc_type_sat_attr_total_lbas_written_normalized": 253,
560
+ "device_sdc_type_sat_attr_total_lbas_written_raw": 173833,
561
+ "device_sdc_type_sat_attr_udma_crc_error_count_decoded": 0,
562
+ "device_sdc_type_sat_attr_udma_crc_error_count_normalized": 100,
563
+ "device_sdc_type_sat_attr_udma_crc_error_count_raw": 0,
564
+ "device_sdc_type_sat_attr_unknown_attribute_decoded": 0,
565
+ "device_sdc_type_sat_attr_unknown_attribute_normalized": 0,
566
+ "device_sdc_type_sat_attr_unknown_attribute_raw": 0,
567
+ "device_sdc_type_sat_attr_unknown_ssd_attribute_decoded": 4694419309637,
568
+ "device_sdc_type_sat_attr_unknown_ssd_attribute_normalized": 4,
569
+ "device_sdc_type_sat_attr_unknown_ssd_attribute_raw": 4694419309637,
570
+ "device_sdc_type_sat_power_cycle_count": 13,
571
+ "device_sdc_type_sat_power_on_time": 29678400,
572
+ "device_sdc_type_sat_smart_status_failed": 0,
573
+ "device_sdc_type_sat_smart_status_passed": 1,
574
+ "device_sdc_type_sat_temperature": 27,
575
+ },
576
+ },
577
"error on scan": {
578
prepareMock: prepareMockErrOnScan,
579
},
@@ -553,6 +668,36 @@ func prepareMockOkTypeScsi() *mockSmartctlCliExec {
668
}
669
}
670
671
+func prepareMockOkTypeSataNonFatalExitStatus() *mockSmartctlCliExec {
672
+ return &mockSmartctlCliExec{
673
+ errOnScan: false,
674
+ scanData: dataTypeSataScan,
675
+ deviceDataFunc: func(deviceName, deviceType, powerMode string) ([]byte, error) {
676
+ if deviceType != "sat" {
677
+ return nil, fmt.Errorf("unexpected device type %s", deviceType)
678
+ }
679
+ // Simulate smartctl exit status 32 (bit 5): some attributes were <= threshold in the past.
680
+ // The data is still valid and should be processed.
681
+ var data []byte
682
+ switch deviceName {
683
+ case "/dev/sda":
684
+ data = bytes.Replace(dataTypeSataDeviceHDDSda, []byte(`"exit_status": 0`), []byte(`"exit_status": 32`), 1)
685
+ case "/dev/sdc":
686
+ data = bytes.Replace(dataTypeSataDeviceSSDSdc, []byte(`"exit_status": 0`), []byte(`"exit_status": 32`), 1)
687
+ default:
688
+ return nil, fmt.Errorf("unexpected device name %s", deviceName)
689
+ }
690
+
691
+ // Verify that the modified payload actually encodes smartctl.exit_status == 32.
692
+ v := gjson.GetBytes(data, "smartctl.exit_status")
693
+ if !v.Exists() || v.Int() != 32 {
694
+ panic("prepareMockOkTypeSataNonFatalExitStatus: failed to construct payload with smartctl.exit_status == 32")
695
+ }
696
+ return data, fmt.Errorf("exit status 32")
697
+ },
698
+ }
699
+}
700
+
701
func prepareMockErrOnScan() *mockSmartctlCliExec {
702
return &mockSmartctlCliExec{
703
errOnScan: true,
@@ -589,11 +734,11 @@ func (m *mockSmartctlCliExec) deviceInfo(deviceName, deviceType, powerMode strin
734
return nil, nil
735
}
736
bs, err := m.deviceDataFunc(deviceName, deviceType, powerMode)
592
- if err != nil {
737
+ if len(bs) == 0 {
738
return nil, err
739
}
740
res := gjson.ParseBytes(bs)
596
- return &res, nil
741
+ return &res, err
742
}
743
744
var randomJsonData = `
src/go/plugin/go.d/collector/smartctl/exec.go
+8
-3
@@ -137,9 +137,14 @@ func parseOutput(cmdStr string, bs []byte, log *logger.Logger) (*gjson.Result, e
137
return nil, fmt.Errorf("'%s' returned unexpected data", cmdStr)
138
}
139
140
- for _, msg := range res.Get("smartctl.messages").Array() {
141
- if msg.Get("severity").String() == "error" {
142
- return &res, fmt.Errorf("'%s' reported an error: %s", cmdStr, msg.Get("string"))
140
+ // https://manpages.debian.org/bullseye/smartmontools/smartctl.8.en.html#EXIT_STATUS
141
+ // Bits 0-1 indicate fatal conditions (command line error, device open failure).
142
+ // Bits 2-7 indicate disk health conditions but the output data is still valid.
143
+ if isExitStatusHasAnyBit(&res, 0, 1) {
144
+ for _, msg := range res.Get("smartctl.messages").Array() {
145
+ if msg.Get("severity").String() == "error" {
146
+ return &res, fmt.Errorf("'%s' reported an error: %s", cmdStr, msg.Get("string"))
147
+ }
148
}
149
}
150