| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package consul |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "math" |
| 8 | "strconv" |
| 9 | "strings" |
| 10 | |
| 11 | "github.com/netdata/netdata/go/plugins/pkg/prometheus" |
| 12 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/oldmetrix" |
| 13 | ) |
| 14 | |
| 15 | func (c *Collector) collectMetricsPrometheus(mx map[string]int64) error { |
| 16 | mfs, err := c.prom.Scrape() |
| 17 | if err != nil { |
| 18 | return err |
| 19 | } |
| 20 | |
| 21 | // Key Metrics (https://developer.hashicorp.com/consul/docs/agent/telemetry#key-metrics) |
| 22 | |
| 23 | // prometheus metrics are messy: |
| 24 | // - if 'disable_hostname' is false (default): |
| 25 | // - consul_autopilot_failure_tolerance => consul_hostname_autopilot_failure_tolerance |
| 26 | // - both are exposed |
| 27 | // - only the one with the hostname has the correct value |
| 28 | // - 1.14.3 (it probably has something to do with cloud management version): |
| 29 | // - runtime_sys_bytes => runtime_sys_bytes_sys_bytes; consul_autopilot_healthy => consul_autopilot_healthy_healthy |
| 30 | // - both are exposed |
| 31 | // - only the one with the double name has the correct value |
| 32 | |
| 33 | if c.isServer() { |
| 34 | c.collectSummary(mx, mfs, "raft_thread_main_saturation") |
| 35 | c.collectSummary(mx, mfs, "raft_thread_fsm_saturation") |
| 36 | c.collectSummary(mx, mfs, "raft_boltdb_logsPerBatch") |
| 37 | c.collectSummary(mx, mfs, "kvs_apply") |
| 38 | c.collectSummary(mx, mfs, "txn_apply") |
| 39 | c.collectSummary(mx, mfs, "raft_boltdb_storeLogs") |
| 40 | c.collectSummary(mx, mfs, "raft_rpc_installSnapshot") // make sense for followers only |
| 41 | c.collectSummary(mx, mfs, "raft_commitTime") // make sense for leader only |
| 42 | c.collectSummary(mx, mfs, "raft_leader_lastContact") // make sense for leader only |
| 43 | |
| 44 | c.collectCounter(mx, mfs, "raft_apply", precision) // make sense for leader only |
| 45 | c.collectCounter(mx, mfs, "raft_state_candidate", 1) |
| 46 | c.collectCounter(mx, mfs, "raft_state_leader", 1) |
| 47 | |
| 48 | c.collectGaugeBool(mx, mfs, "autopilot_healthy", "autopilot_healthy_healthy") |
| 49 | c.collectGaugeBool(mx, mfs, "server_isLeader", "server_isLeader_isLeader") |
| 50 | c.collectGauge(mx, mfs, "autopilot_failure_tolerance", 1, "autopilot_failure_tolerance_failure_tolerance") |
| 51 | c.collectGauge(mx, mfs, "raft_fsm_lastRestoreDuration", 1) |
| 52 | c.collectGauge(mx, mfs, "raft_leader_oldestLogAge", 1, "raft_leader_oldestLogAge_oldestLogAge") |
| 53 | c.collectGauge(mx, mfs, "raft_boltdb_freelistBytes", 1, "raft_boltdb_freelistBytes_freelistBytes") |
| 54 | |
| 55 | if isLeader, ok := c.isLeader(mfs); ok { |
| 56 | if isLeader && !c.hasLeaderCharts { |
| 57 | c.addLeaderCharts() |
| 58 | c.hasLeaderCharts = true |
| 59 | } |
| 60 | if !isLeader && c.hasLeaderCharts { |
| 61 | c.removeLeaderCharts() |
| 62 | c.hasLeaderCharts = false |
| 63 | } |
| 64 | if !isLeader && !c.hasFollowerCharts { |
| 65 | c.addFollowerCharts() |
| 66 | c.hasFollowerCharts = true |
| 67 | } |
| 68 | if isLeader && c.hasFollowerCharts { |
| 69 | c.removeFollowerCharts() |
| 70 | c.hasFollowerCharts = false |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | c.collectGauge(mx, mfs, "system_licenseExpiration", 3600, "system_licenseExpiration_licenseExpiration") |
| 76 | |
| 77 | c.collectCounter(mx, mfs, "client_rpc", 1) |
| 78 | c.collectCounter(mx, mfs, "client_rpc_exceeded", 1) |
| 79 | c.collectCounter(mx, mfs, "client_rpc_failed", 1) |
| 80 | |
| 81 | c.collectGauge(mx, mfs, "runtime_alloc_bytes", 1, "runtime_alloc_bytes_alloc_bytes") |
| 82 | c.collectGauge(mx, mfs, "runtime_sys_bytes", 1, "runtime_sys_bytes_sys_bytes") |
| 83 | c.collectGauge(mx, mfs, "runtime_total_gc_pause_ns", 1, "runtime_total_gc_pause_ns_total_gc_pause_ns") |
| 84 | |
| 85 | return nil |
| 86 | } |
| 87 | |
| 88 | func (c *Collector) isLeader(mfs prometheus.MetricFamilies) (bool, bool) { |
| 89 | var mf *prometheus.MetricFamily |
| 90 | for _, v := range []string{"server_isLeader_isLeader", "server_isLeader"} { |
| 91 | if mf = mfs.GetGauge(c.promMetricNameWithHostname(v)); mf != nil { |
| 92 | break |
| 93 | } |
| 94 | if mf = mfs.GetGauge(c.promMetricName(v)); mf != nil { |
| 95 | break |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | if mf == nil { |
| 100 | return false, false |
| 101 | } |
| 102 | |
| 103 | return mf.Metrics()[0].Gauge().Value() == 1, true |
| 104 | } |
| 105 | |
| 106 | func (c *Collector) collectGauge(mx map[string]int64, mfs prometheus.MetricFamilies, name string, mul float64, aliases ...string) { |
| 107 | var mf *prometheus.MetricFamily |
| 108 | for _, v := range append(aliases, name) { |
| 109 | if mf = mfs.GetGauge(c.promMetricNameWithHostname(v)); mf != nil { |
| 110 | break |
| 111 | } |
| 112 | if mf = mfs.GetGauge(c.promMetricName(v)); mf != nil { |
| 113 | break |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | if mf == nil { |
| 118 | return |
| 119 | } |
| 120 | |
| 121 | v := mf.Metrics()[0].Gauge().Value() |
| 122 | |
| 123 | if !math.IsNaN(v) { |
| 124 | mx[name] = int64(v * mul) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | func (c *Collector) collectGaugeBool(mx map[string]int64, mfs prometheus.MetricFamilies, name string, aliases ...string) { |
| 129 | var mf *prometheus.MetricFamily |
| 130 | for _, v := range append(aliases, name) { |
| 131 | if mf = mfs.GetGauge(c.promMetricNameWithHostname(v)); mf != nil { |
| 132 | break |
| 133 | } |
| 134 | if mf = mfs.GetGauge(c.promMetricName(v)); mf != nil { |
| 135 | break |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | if mf == nil { |
| 140 | return |
| 141 | } |
| 142 | |
| 143 | v := mf.Metrics()[0].Gauge().Value() |
| 144 | |
| 145 | if !math.IsNaN(v) { |
| 146 | mx[name+"_yes"] = oldmetrix.Bool(v == 1) |
| 147 | mx[name+"_no"] = oldmetrix.Bool(v == 0) |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | func (c *Collector) collectCounter(mx map[string]int64, mfs prometheus.MetricFamilies, name string, mul float64) { |
| 152 | mf := mfs.GetCounter(c.promMetricName(name)) |
| 153 | if mf == nil { |
| 154 | return |
| 155 | } |
| 156 | |
| 157 | v := mf.Metrics()[0].Counter().Value() |
| 158 | |
| 159 | if !math.IsNaN(v) { |
| 160 | mx[name] = int64(v * mul) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | func (c *Collector) collectSummary(mx map[string]int64, mfs prometheus.MetricFamilies, name string) { |
| 165 | mf := mfs.GetSummary(c.promMetricName(name)) |
| 166 | if mf == nil { |
| 167 | return |
| 168 | } |
| 169 | |
| 170 | m := mf.Metrics()[0] |
| 171 | |
| 172 | for _, q := range m.Summary().Quantiles() { |
| 173 | v := q.Value() |
| 174 | // MaxAge is 10 seconds (hardcoded) |
| 175 | // https://github.com/hashicorp/go-metrics/blob/b6d5c860c07ef6eeec89f4a662c7b452dd4d0c93/prometheus/prometheus.go#L227 |
| 176 | if math.IsNaN(v) { |
| 177 | v = 0 |
| 178 | } |
| 179 | |
| 180 | id := fmt.Sprintf("%s_quantile=%s", name, formatFloat(q.Quantile())) |
| 181 | mx[id] = int64(v * precision * precision) |
| 182 | } |
| 183 | |
| 184 | mx[name+"_sum"] = int64(m.Summary().Sum() * precision) |
| 185 | mx[name+"_count"] = int64(m.Summary().Count()) |
| 186 | } |
| 187 | |
| 188 | func (c *Collector) promMetricName(name string) string { |
| 189 | px := c.cfg.DebugConfig.Telemetry.MetricsPrefix |
| 190 | return px + "_" + name |
| 191 | } |
| 192 | |
| 193 | var forbiddenCharsReplacer = strings.NewReplacer(" ", "_", ".", "_", "=", "_", "-", "_", "/", "_") |
| 194 | |
| 195 | // controlled by 'disable_hostname' |
| 196 | // https://developer.hashicorp.com/consul/docs/agent/config/config-files#telemetry-disable_hostname |
| 197 | func (c *Collector) promMetricNameWithHostname(name string) string { |
| 198 | px := c.cfg.DebugConfig.Telemetry.MetricsPrefix |
| 199 | node := forbiddenCharsReplacer.Replace(c.cfg.Config.NodeName) |
| 200 | |
| 201 | return px + "_" + node + "_" + name |
| 202 | } |
| 203 | |
| 204 | func formatFloat(v float64) string { |
| 205 | return strconv.FormatFloat(v, 'f', -1, 64) |
| 206 | } |