| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package smartctl |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "maps" |
| 8 | "slices" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/sourcegraph/conc/pool" |
| 14 | "github.com/tidwall/gjson" |
| 15 | ) |
| 16 | |
| 17 | func (c *Collector) collect() (map[string]int64, error) { |
| 18 | now := time.Now() |
| 19 | |
| 20 | if c.forceScan || c.isTimeToScan(now) { |
| 21 | devices, err := c.scanDevices() |
| 22 | if err != nil { |
| 23 | return nil, err |
| 24 | } |
| 25 | |
| 26 | for k, dev := range c.scannedDevices { |
| 27 | if _, ok := devices[k]; !ok { |
| 28 | delete(c.scannedDevices, k) |
| 29 | delete(c.seenDevices, k) |
| 30 | c.removeDeviceCharts(dev) |
| 31 | } |
| 32 | } |
| 33 | |
| 34 | c.forceDevicePoll = !maps.Equal(c.scannedDevices, devices) |
| 35 | c.scannedDevices = devices |
| 36 | c.lastScanTime = now |
| 37 | c.forceScan = false |
| 38 | } |
| 39 | |
| 40 | if c.forceDevicePoll || c.isTimeToPollDevices(now) { |
| 41 | mx := make(map[string]int64) |
| 42 | |
| 43 | c.collectDevices(mx) |
| 44 | |
| 45 | c.forceDevicePoll = false |
| 46 | c.lastDevicePollTime = now |
| 47 | c.mx = mx |
| 48 | } |
| 49 | |
| 50 | return c.mx, nil |
| 51 | } |
| 52 | func (c *Collector) collectDevices(mx map[string]int64) { |
| 53 | if c.ConcurrentScans > 0 && len(c.scannedDevices) > 1 { |
| 54 | if err := c.collectDevicesConcurrently(mx); err != nil { |
| 55 | c.Warning(err) |
| 56 | } |
| 57 | return |
| 58 | } |
| 59 | |
| 60 | for _, d := range c.scannedDevices { |
| 61 | if err := c.collectScannedDevice(mx, d); err != nil { |
| 62 | c.Warning(err) |
| 63 | continue |
| 64 | } |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | type deviceInfoResult struct { |
| 69 | scanDevice *scanDevice |
| 70 | response *gjson.Result |
| 71 | err error |
| 72 | } |
| 73 | |
| 74 | func (c *Collector) collectDevicesConcurrently(mx map[string]int64) error { |
| 75 | p := pool.New().WithMaxGoroutines(c.ConcurrentScans) |
| 76 | resultsChan := make(chan deviceInfoResult, len(c.scannedDevices)) |
| 77 | |
| 78 | for _, dev := range c.scannedDevices { |
| 79 | p.Go(func() { |
| 80 | resp, err := c.exec.deviceInfo(dev.name, dev.typ, c.NoCheckPowerMode) |
| 81 | resultsChan <- deviceInfoResult{ |
| 82 | scanDevice: dev, |
| 83 | response: resp, |
| 84 | err: err, |
| 85 | } |
| 86 | }) |
| 87 | } |
| 88 | |
| 89 | p.Wait() |
| 90 | close(resultsChan) |
| 91 | |
| 92 | for r := range resultsChan { |
| 93 | if err := c.processDeviceResult(mx, r); err != nil { |
| 94 | c.Warning(err) |
| 95 | continue |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | return nil |
| 100 | } |
| 101 | |
| 102 | func (c *Collector) processDeviceResult(mx map[string]int64, result deviceInfoResult) error { |
| 103 | scanDev := result.scanDevice |
| 104 | resp := result.response |
| 105 | err := result.err |
| 106 | |
| 107 | if err != nil { |
| 108 | if resp != nil { |
| 109 | if isDeviceOpenFailedNoSuchDevice(resp) && !scanDev.extra { |
| 110 | c.Infof("smartctl reported that device '%s' type '%s' no longer exists", scanDev.name, scanDev.typ) |
| 111 | c.forceScan = true |
| 112 | return nil |
| 113 | } |
| 114 | // https://manpages.debian.org/bullseye/smartmontools/smartctl.8.en.html#EXIT_STATUS |
| 115 | // Bits 0-1 indicate fatal conditions (command line error, device open failure). |
| 116 | // Bits 2-7 indicate disk health conditions but the output data is still valid. |
| 117 | if !isExitStatusHasAnyBit(resp, 0, 1) { |
| 118 | c.Debugf("device '%s' type '%s': smartctl exit status has non-fatal bits set: %v", scanDev.name, scanDev.typ, err) |
| 119 | err = nil |
| 120 | } |
| 121 | } |
| 122 | if err != nil { |
| 123 | return fmt.Errorf("failed to get device info for '%s' type '%s': %v", scanDev.name, scanDev.typ, err) |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | if isDeviceInLowerPowerMode(resp) { |
| 128 | c.Debugf("device '%s' type '%s' is in a low-power mode, skipping", scanDev.name, scanDev.typ) |
| 129 | return nil |
| 130 | } |
| 131 | |
| 132 | dev := newSmartDevice(resp) |
| 133 | if !isSmartDeviceValid(dev) { |
| 134 | return nil |
| 135 | } |
| 136 | |
| 137 | if !c.seenDevices[scanDev.key()] { |
| 138 | c.seenDevices[scanDev.key()] = true |
| 139 | c.addDeviceCharts(dev) |
| 140 | } |
| 141 | |
| 142 | c.collectSmartDevice(mx, dev) |
| 143 | |
| 144 | return nil |
| 145 | } |
| 146 | |
| 147 | func (c *Collector) collectScannedDevice(mx map[string]int64, scanDev *scanDevice) error { |
| 148 | resp, err := c.exec.deviceInfo(scanDev.name, scanDev.typ, c.NoCheckPowerMode) |
| 149 | return c.processDeviceResult(mx, deviceInfoResult{ |
| 150 | scanDevice: scanDev, |
| 151 | response: resp, |
| 152 | err: err, |
| 153 | }) |
| 154 | } |
| 155 | |
| 156 | func (c *Collector) collectSmartDevice(mx map[string]int64, dev *smartDevice) { |
| 157 | px := fmt.Sprintf("device_%s_type_%s_", dev.deviceName(), dev.deviceType()) |
| 158 | |
| 159 | if v, ok := dev.powerOnTime(); ok { |
| 160 | mx[px+"power_on_time"] = v |
| 161 | } |
| 162 | if v, ok := dev.temperature(); ok { |
| 163 | mx[px+"temperature"] = v |
| 164 | } |
| 165 | if v, ok := dev.powerCycleCount(); ok { |
| 166 | mx[px+"power_cycle_count"] = v |
| 167 | } |
| 168 | if v, ok := dev.smartStatusPassed(); ok { |
| 169 | mx[px+"smart_status_passed"] = 0 |
| 170 | mx[px+"smart_status_failed"] = 0 |
| 171 | if v { |
| 172 | mx[px+"smart_status_passed"] = 1 |
| 173 | } else { |
| 174 | mx[px+"smart_status_failed"] = 1 |
| 175 | } |
| 176 | } |
| 177 | if v, ok := dev.ataSmartErrorLogCount(); ok { |
| 178 | mx[px+"ata_smart_error_log_summary_count"] = v |
| 179 | } |
| 180 | |
| 181 | if attrs, ok := dev.ataSmartAttributeTable(); ok { |
| 182 | for _, attr := range attrs { |
| 183 | if !isSmartAttrValid(attr) { |
| 184 | continue |
| 185 | } |
| 186 | n := strings.ToLower(attr.name()) |
| 187 | n = strings.ReplaceAll(n, " ", "_") |
| 188 | px := fmt.Sprintf("%sattr_%s_", px, n) |
| 189 | |
| 190 | if v, err := strconv.ParseInt(attr.value(), 10, 64); err == nil { |
| 191 | mx[px+"normalized"] = v |
| 192 | } |
| 193 | |
| 194 | if v, err := strconv.ParseInt(attr.rawValue(), 10, 64); err == nil { |
| 195 | mx[px+"raw"] = v |
| 196 | } |
| 197 | |
| 198 | rs := strings.TrimSpace(attr.rawString()) |
| 199 | if i := strings.IndexByte(rs, ' '); i != -1 { |
| 200 | rs = rs[:i] |
| 201 | } |
| 202 | if v, err := strconv.ParseInt(rs, 10, 64); err == nil { |
| 203 | mx[px+"decoded"] = v |
| 204 | } |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | if dev.deviceType() == "scsi" { |
| 209 | sel := dev.data.Get("scsi_error_counter_log") |
| 210 | if !sel.Exists() { |
| 211 | return |
| 212 | } |
| 213 | |
| 214 | for _, v := range []string{"read", "write", "verify"} { |
| 215 | for _, n := range []string{ |
| 216 | //"errors_corrected_by_eccdelayed", |
| 217 | //"errors_corrected_by_eccfast", |
| 218 | //"errors_corrected_by_rereads_rewrites", |
| 219 | "total_errors_corrected", |
| 220 | "total_uncorrected_errors", |
| 221 | } { |
| 222 | key := fmt.Sprintf("%sscsi_error_log_%s_%s", px, v, n) |
| 223 | metric := fmt.Sprintf("%s.%s", v, n) |
| 224 | |
| 225 | if m := sel.Get(metric); m.Exists() { |
| 226 | mx[key] = m.Int() |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | func (c *Collector) isTimeToScan(now time.Time) bool { |
| 234 | return c.ScanEvery.Duration().Seconds() != 0 && now.After(c.lastScanTime.Add(c.ScanEvery.Duration())) |
| 235 | } |
| 236 | |
| 237 | func (c *Collector) isTimeToPollDevices(now time.Time) bool { |
| 238 | return now.After(c.lastDevicePollTime.Add(c.PollDevicesEvery.Duration())) |
| 239 | |
| 240 | } |
| 241 | |
| 242 | func isSmartDeviceValid(d *smartDevice) bool { |
| 243 | return d.deviceName() != "" && d.deviceType() != "" |
| 244 | } |
| 245 | |
| 246 | func isSmartAttrValid(a *smartAttribute) bool { |
| 247 | return a.id() != "" && a.name() != "" |
| 248 | } |
| 249 | |
| 250 | func isDeviceInLowerPowerMode(r *gjson.Result) bool { |
| 251 | if !isExitStatusHasAnyBit(r, 1) { |
| 252 | return false |
| 253 | } |
| 254 | |
| 255 | messages := r.Get("smartctl.messages").Array() |
| 256 | |
| 257 | return slices.ContainsFunc(messages, func(msg gjson.Result) bool { |
| 258 | text := msg.Get("string").String() |
| 259 | return strings.HasPrefix(text, "Device is in") && strings.Contains(text, "mode") |
| 260 | }) |
| 261 | } |
| 262 | |
| 263 | func isDeviceOpenFailedNoSuchDevice(r *gjson.Result) bool { |
| 264 | if !isExitStatusHasAnyBit(r, 1) { |
| 265 | return false |
| 266 | } |
| 267 | |
| 268 | messages := r.Get("smartctl.messages").Array() |
| 269 | |
| 270 | return slices.ContainsFunc(messages, func(msg gjson.Result) bool { |
| 271 | text := msg.Get("string").String() |
| 272 | return strings.HasSuffix(text, "No such device") |
| 273 | }) |
| 274 | } |
| 275 | |
| 276 | func isExitStatusHasAnyBit(r *gjson.Result, bit int, bits ...int) bool { |
| 277 | // https://manpages.debian.org/bullseye/smartmontools/smartctl.8.en.html#EXIT_STATUS |
| 278 | status := int(r.Get("smartctl.exit_status").Int()) |
| 279 | |
| 280 | for _, b := range append([]int{bit}, bits...) { |
| 281 | mask := 1 << b |
| 282 | if (status & mask) != 0 { |
| 283 | return true |
| 284 | } |
| 285 | } |
| 286 | |
| 287 | return false |
| 288 | } |