feat(discovery): add Phase 1 Prometheus metrics surface (8 metrics + EmitFromTrace)
cognitive committed
Apr 30, 2026 at 04:20 UTC
404b6cf20a22cb9910bd129aa53193781d0a3111
2 files changed
+461
portal/discovery/metrics.go
new
+217
@@ -0,0 +1,217 @@
1
+package discovery
2
+
3
+// metrics.go — Phase 1 Prometheus telemetry surface for portal/discovery.
4
+//
5
+// Registers 8 low-cardinality metrics on prometheus.DefaultRegisterer via
6
+// promauto. Provides EmitFromTrace(SelectionTrace) to update counter/histogram/
7
+// gauge metrics from a completed selection invocation.
8
+//
9
+// Cardinality discipline:
10
+// - NO per-client labels (no client_hash, no local_address).
11
+// - Relay-label cardinality capped at maxRelayLabelCardinality unique URLs;
12
+// additional URLs are bucketed under relay="other".
13
+//
14
+// See /home/alpha/.claude/plans/sophisticate-and-rationalize-discovery-rosy-parnas.md
15
+// (Phase 1 — Telemetry only) for rationale.
16
+
17
+import (
18
+ "sync"
19
+
20
+ "github.com/prometheus/client_golang/prometheus"
21
+ "github.com/prometheus/client_golang/prometheus/promauto"
22
+)
23
+
24
+// maxRelayLabelCardinality is the hard cap on distinct relay-URL values used as
25
+// Prometheus labels. URLs beyond the first 64 distinct values are bucketed as
26
+// relay="other" to prevent unbounded cardinality.
27
+const maxRelayLabelCardinality = 64
28
+
29
+// relayBudget guards relay-URL cardinality with a single mutex so that the
30
+// membership set and the count are always updated atomically. This prevents
31
+// the race where two goroutines each see "URL not present" and both increment
32
+// the counter, prematurely exhausting the 64-label budget.
33
+var relayBudget = struct {
34
+ mu sync.Mutex
35
+ seen map[string]struct{}
36
+}{
37
+ seen: make(map[string]struct{}),
38
+}
39
+
40
+// boundedRelay returns url unchanged when the URL is already known or when
41
+// the distinct-URL count is below maxRelayLabelCardinality.
42
+// Any URL that would exceed the cap is returned as "other".
43
+func boundedRelay(url string) string {
44
+ relayBudget.mu.Lock()
45
+ defer relayBudget.mu.Unlock()
46
+ if _, ok := relayBudget.seen[url]; ok {
47
+ return url
48
+ }
49
+ if len(relayBudget.seen) >= maxRelayLabelCardinality {
50
+ return "other"
51
+ }
52
+ relayBudget.seen[url] = struct{}{}
53
+ return url
54
+}
55
+
56
+// --------------------------------------------------------------------------
57
+// Metric registrations
58
+// --------------------------------------------------------------------------
59
+
60
+// RelaySelectedTotal counts relay-selection events by (relay, reason).
61
+// reason ∈ {explicit, auto, fallback, congestion-promoted, variant-grid}.
62
+var RelaySelectedTotal = promauto.NewCounterVec(
63
+ prometheus.CounterOpts{
64
+ Name: "portal_discovery_relay_selected_total",
65
+ Help: "Total relays selected by reason.",
66
+ },
67
+ []string{"relay", "reason"},
68
+)
69
+
70
+// RelayPoolSize is a gauge of auto-pool size partitioned by state.
71
+// state ∈ {total, active, banned, expired, suppressed, fallback}.
72
+var RelayPoolSize = promauto.NewGaugeVec(
73
+ prometheus.GaugeOpts{
74
+ Name: "portal_discovery_relay_pool_size",
75
+ Help: "Auto-pool size by state.",
76
+ },
77
+ []string{"state"},
78
+)
79
+
80
+// RTTSeconds is a histogram of per-relay discovery RTT observations.
81
+// label: relay. Buckets: 10 ms … 5 s.
82
+var RTTSeconds = promauto.NewHistogramVec(
83
+ prometheus.HistogramOpts{
84
+ Name: "portal_discovery_rtt_seconds",
85
+ Help: "Discovery RTT per relay (seconds).",
86
+ Buckets: []float64{0.010, 0.050, 0.100, 0.250, 0.500, 1.0, 2.0, 5.0},
87
+ },
88
+ []string{"relay"},
89
+)
90
+
91
+// ActiveTunnelsPerRelay is a gauge of tunnel count for each relay.
92
+// SDK-local measurement: tracks this process's tunnel distribution only.
93
+var ActiveTunnelsPerRelay = promauto.NewGaugeVec(
94
+ prometheus.GaugeOpts{
95
+ Name: "portal_discovery_active_tunnels_per_relay",
96
+ Help: "SDK-local; measures this exposure's tunnel distribution, not relay-wide load.",
97
+ },
98
+ []string{"relay"},
99
+)
100
+
101
+// SelectionDurationSeconds is a histogram of wall time per selection call.
102
+// No labels; uses prometheus default buckets.
103
+var SelectionDurationSeconds = promauto.NewHistogram(
104
+ prometheus.HistogramOpts{
105
+ Name: "portal_discovery_selection_duration_seconds",
106
+ Help: "Wall time of a single relay-selection invocation.",
107
+ // Default prometheus buckets (.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10).
108
+ },
109
+)
110
+
111
+// SelectionSkippedTotal counts relays excluded from selection by reason.
112
+// reason ∈ {expired, require_udp, require_tcp, suppressed, banned, no_descriptor, no_overlay_peer}.
113
+var SelectionSkippedTotal = promauto.NewCounterVec(
114
+ prometheus.CounterOpts{
115
+ Name: "portal_discovery_selection_skipped_total",
116
+ Help: "Relays skipped during selection by reason.",
117
+ },
118
+ []string{"reason"},
119
+)
120
+
121
+// FailuresTotal counts discovery and active-path failures per relay.
122
+// labels: relay, kind ∈ {discovery, active}.
123
+var FailuresTotal = promauto.NewCounterVec(
124
+ prometheus.CounterOpts{
125
+ Name: "portal_discovery_failures_total",
126
+ Help: "Discovery and active failures per relay.",
127
+ },
128
+ []string{"relay", "kind"},
129
+)
130
+
131
+// CongestionMode is a gauge encoding the current congestion state.
132
+// 0 = normal, 1 = congested (no variant-grid), 2 = variant-grid active.
133
+var CongestionMode = promauto.NewGauge(
134
+ prometheus.GaugeOpts{
135
+ Name: "portal_discovery_congestion_mode",
136
+ Help: "Active congestion mode (0=normal, 1=congested, 2=variant-grid).",
137
+ },
138
+)
139
+
140
+// --------------------------------------------------------------------------
141
+// EmitFromTrace
142
+// --------------------------------------------------------------------------
143
+
144
+// EmitFromTrace updates relevant Prometheus metrics from a completed
145
+// SelectionTrace. It is safe to call concurrently.
146
+//
147
+// Metrics updated:
148
+// - relay_selected_total{relay, reason} — one increment per OutputURL.
149
+// - selection_duration_seconds — one observation for the whole invocation.
150
+// - congestion_mode — set according to Congested + NonLinear.
151
+// - selection_skipped_total{reason} — one increment per suppressed URL that
152
+// has a reason entry.
153
+// - rtt_seconds{relay} — one observation per Ranked entry with non-zero RTT.
154
+//
155
+// Metrics NOT updated here (wired by later phases / other code paths):
156
+// - relay_pool_size — set by RelaySet pool management.
157
+// - active_tunnels_per_relay — incremented/decremented at tunnel accept/close.
158
+// - failures_total — incremented on discovery/active failure events.
159
+func EmitFromTrace(t SelectionTrace) {
160
+ // --- relay_selected_total ---
161
+ reason := selectionReason(t)
162
+ for _, url := range t.OutputURLs {
163
+ RelaySelectedTotal.WithLabelValues(boundedRelay(url), reason).Inc()
164
+ }
165
+
166
+ // --- selection_duration_seconds ---
167
+ SelectionDurationSeconds.Observe(t.SelectionTook.Seconds())
168
+
169
+ // --- congestion_mode ---
170
+ CongestionMode.Set(congestionModeValue(t.Congested, t.NonLinear))
171
+
172
+ // --- selection_skipped_total ---
173
+ // Build suppressed set for O(1) lookup.
174
+ suppressedSet := make(map[string]struct{}, len(t.Suppressed))
175
+ for _, url := range t.Suppressed {
176
+ suppressedSet[url] = struct{}{}
177
+ }
178
+ for url, reason := range t.Reasons {
179
+ if _, ok := suppressedSet[url]; ok {
180
+ SelectionSkippedTotal.WithLabelValues(reason).Inc()
181
+ }
182
+ }
183
+
184
+ // --- rtt_seconds ---
185
+ for _, entry := range t.Ranked {
186
+ if entry.RTT != 0 {
187
+ RTTSeconds.WithLabelValues(boundedRelay(entry.URL)).Observe(entry.RTT.Seconds())
188
+ }
189
+ }
190
+}
191
+
192
+// selectionReason derives the reason label for relay_selected_total from the
193
+// trace flags. Explicit/fallback semantics are wired by later phases; this
194
+// function defaults to "auto" for uninstrumented call sites.
195
+func selectionReason(t SelectionTrace) string {
196
+ switch {
197
+ case t.NonLinear:
198
+ return "variant-grid"
199
+ case t.Congested:
200
+ return "congestion-promoted"
201
+ default:
202
+ return "auto"
203
+ }
204
+}
205
+
206
+// congestionModeValue maps the Congested + NonLinear pair to the metric value.
207
+// 0 = normal, 1 = congested without variant-grid, 2 = variant-grid active.
208
+func congestionModeValue(congested, nonLinear bool) float64 {
209
+ switch {
210
+ case nonLinear:
211
+ return 2
212
+ case congested:
213
+ return 1
214
+ default:
215
+ return 0
216
+ }
217
+}
portal/discovery/metrics_test.go
new
+244
@@ -0,0 +1,244 @@
1
+package discovery
2
+
3
+import (
4
+ "fmt"
5
+ "testing"
6
+ "time"
7
+
8
+ dto "github.com/prometheus/client_model/go"
9
+
10
+ "github.com/prometheus/client_golang/prometheus"
11
+)
12
+
13
+// metricFamilyByName gathers all metric families and returns the one with the
14
+// given name, or nil if not found.
15
+func metricFamilyByName(t *testing.T, name string) *dto.MetricFamily {
16
+ t.Helper()
17
+ mfs, err := prometheus.DefaultGatherer.Gather()
18
+ if err != nil {
19
+ t.Fatalf("gather: %v", err)
20
+ }
21
+ for _, mf := range mfs {
22
+ if mf.GetName() == name {
23
+ return mf
24
+ }
25
+ }
26
+ return nil
27
+}
28
+
29
+// TestMetricsRegistryPresence asserts that all 8 Phase-1 metrics are registered
30
+// on the default registry with non-empty HELP text and the expected type.
31
+func TestMetricsRegistryPresence(t *testing.T) {
32
+ want := []struct {
33
+ name string
34
+ typ dto.MetricType
35
+ }{
36
+ {"portal_discovery_relay_selected_total", dto.MetricType_COUNTER},
37
+ {"portal_discovery_relay_pool_size", dto.MetricType_GAUGE},
38
+ {"portal_discovery_rtt_seconds", dto.MetricType_HISTOGRAM},
39
+ {"portal_discovery_active_tunnels_per_relay", dto.MetricType_GAUGE},
40
+ {"portal_discovery_selection_duration_seconds", dto.MetricType_HISTOGRAM},
41
+ {"portal_discovery_selection_skipped_total", dto.MetricType_COUNTER},
42
+ {"portal_discovery_failures_total", dto.MetricType_COUNTER},
43
+ {"portal_discovery_congestion_mode", dto.MetricType_GAUGE},
44
+ }
45
+
46
+ for _, tc := range want {
47
+
48
+ t.Run(tc.name, func(t *testing.T) {
49
+ mf := metricFamilyByName(t, tc.name)
50
+ if mf == nil {
51
+ t.Fatalf("metric %q not found in registry", tc.name)
52
+ }
53
+ if mf.GetHelp() == "" {
54
+ t.Errorf("metric %q has empty HELP string", tc.name)
55
+ }
56
+ if mf.GetType() != tc.typ {
57
+ t.Errorf("metric %q: got type %v, want %v", tc.name, mf.GetType(), tc.typ)
58
+ }
59
+ })
60
+ }
61
+}
62
+
63
+// TestEmitFromTrace_CounterIncrement verifies that EmitFromTrace increments
64
+// relay_selected_total for each output URL and records a selection duration.
65
+// URL names are test-namespaced to avoid coupling to other tests that share the
66
+// process-global relay-cardinality map.
67
+func TestEmitFromTrace_CounterIncrement(t *testing.T) {
68
+ r1 := "t-counter-r1"
69
+ r2 := "t-counter-r2"
70
+
71
+ // Capture baseline before the call.
72
+ baseline := func(relay string) float64 {
73
+ mf := metricFamilyByName(t, "portal_discovery_relay_selected_total")
74
+ if mf == nil {
75
+ return 0
76
+ }
77
+ for _, m := range mf.GetMetric() {
78
+ var gotRelay, gotReason string
79
+ for _, lp := range m.GetLabel() {
80
+ switch lp.GetName() {
81
+ case "relay":
82
+ gotRelay = lp.GetValue()
83
+ case "reason":
84
+ gotReason = lp.GetValue()
85
+ }
86
+ }
87
+ if gotRelay == relay && gotReason == "auto" {
88
+ return m.GetCounter().GetValue()
89
+ }
90
+ }
91
+ return 0
92
+ }
93
+
94
+ baseR1 := baseline(r1)
95
+ baseR2 := baseline(r2)
96
+
97
+ // Capture duration baseline before the single EmitFromTrace call.
98
+ durationSampleCount := func() uint64 {
99
+ mf := metricFamilyByName(t, "portal_discovery_selection_duration_seconds")
100
+ if mf == nil || len(mf.GetMetric()) == 0 {
101
+ return 0
102
+ }
103
+ return mf.GetMetric()[0].GetHistogram().GetSampleCount()
104
+ }
105
+ baseDur := durationSampleCount()
106
+
107
+ EmitFromTrace(SelectionTrace{
108
+ OutputURLs: []string{r1, r2},
109
+ SelectionTook: 50 * time.Millisecond,
110
+ Congested: false,
111
+ NonLinear: false,
112
+ })
113
+
114
+ // relay_selected_total delta must be 1 for each relay.
115
+ afterR1 := baseline(r1)
116
+ afterR2 := baseline(r2)
117
+ if afterR1-baseR1 != 1 {
118
+ t.Errorf("relay_selected_total{relay=%q,reason=auto}: delta want 1, got %v", r1, afterR1-baseR1)
119
+ }
120
+ if afterR2-baseR2 != 1 {
121
+ t.Errorf("relay_selected_total{relay=%q,reason=auto}: delta want 1, got %v", r2, afterR2-baseR2)
122
+ }
123
+
124
+ // selection_duration_seconds delta must be exactly 1 for this invocation.
125
+ afterDur := durationSampleCount()
126
+ if afterDur-baseDur != 1 {
127
+ t.Errorf("selection_duration_seconds sample delta want 1, got %d", afterDur-baseDur)
128
+ }
129
+}
130
+
131
+// TestEmitFromTrace_CardinalityCap verifies the relay-label cardinality cap.
132
+//
133
+// We emit maxRelayLabelCardinality+1 distinct relay URLs and then assert:
134
+// 1. relay="other" appears in relay_selected_total (overflow was bucketed).
135
+// 2. Every emitted URL either appears as its own relay label OR caused "other"
136
+// to be incremented — i.e., no URL is silently dropped.
137
+//
138
+// Because relayBudget is process-global and prior tests may have consumed some
139
+// slots, we emit enough URLs (maxRelayLabelCardinality+1 = 65) to guarantee at
140
+// least one overflow regardless of prior state, then verify the above.
141
+//
142
+// URLs are namespaced as "t-cap-NNN" to isolate them from other tests.
143
+func TestEmitFromTrace_CardinalityCap(t *testing.T) {
144
+ const total = maxRelayLabelCardinality + 1 // 65
145
+
146
+ // Build the set of our namespace URLs.
147
+ ourURLs := make(map[string]struct{}, total)
148
+ for i := 0; i < total; i++ {
149
+ ourURLs[fmt.Sprintf("t-cap-%03d", i)] = struct{}{}
150
+ }
151
+
152
+ // relayReasonCounter returns the counter value for the given (relay, reason) pair.
153
+ relayReasonCounter := func(relay, reason string) float64 {
154
+ mf := metricFamilyByName(t, "portal_discovery_relay_selected_total")
155
+ if mf == nil {
156
+ return 0
157
+ }
158
+ for _, m := range mf.GetMetric() {
159
+ var r, rs string
160
+ for _, lp := range m.GetLabel() {
161
+ switch lp.GetName() {
162
+ case "relay":
163
+ r = lp.GetValue()
164
+ case "reason":
165
+ rs = lp.GetValue()
166
+ }
167
+ }
168
+ if r == relay && rs == reason {
169
+ return m.GetCounter().GetValue()
170
+ }
171
+ }
172
+ return 0
173
+ }
174
+
175
+ // Our traces are all non-congested non-nonlinear → reason="auto".
176
+ baseOther := relayReasonCounter("other", "auto")
177
+
178
+ for i := 0; i < total; i++ {
179
+ url := fmt.Sprintf("t-cap-%03d", i)
180
+ EmitFromTrace(SelectionTrace{
181
+ OutputURLs: []string{url},
182
+ SelectionTook: time.Millisecond,
183
+ })
184
+ }
185
+
186
+ mf := metricFamilyByName(t, "portal_discovery_relay_selected_total")
187
+ if mf == nil {
188
+ t.Fatal("portal_discovery_relay_selected_total not found")
189
+ }
190
+
191
+ // Collect the our-namespace relay labels that were admitted (got own slot).
192
+ admittedOurs := make(map[string]struct{})
193
+ for _, m := range mf.GetMetric() {
194
+ var r string
195
+ for _, lp := range m.GetLabel() {
196
+ if lp.GetName() == "relay" {
197
+ r = lp.GetValue()
198
+ }
199
+ }
200
+ if _, ok := ourURLs[r]; ok {
201
+ admittedOurs[r] = struct{}{}
202
+ }
203
+ }
204
+
205
+ afterOther := relayReasonCounter("other", "auto")
206
+ overflowed := total - len(admittedOurs) // how many of our URLs were bucketed
207
+
208
+ // Assert: at least one URL overflowed to "other".
209
+ if overflowed <= 0 {
210
+ t.Errorf("expected at least 1 URL to overflow to \"other\"; admitted=%d out of %d", len(admittedOurs), total)
211
+ }
212
+
213
+ // Assert: the counter delta for {relay="other",reason="auto"} matches the
214
+ // number of our-namespace URLs that were not admitted (not merely inferred).
215
+ delta := afterOther - baseOther
216
+ if delta < float64(overflowed) {
217
+ t.Errorf("relay_selected_total{relay=\"other\",reason=\"auto\"} delta want >=%d, got %.0f", overflowed, delta)
218
+ }
219
+
220
+ // Assert: admitted URL count never exceeds the cap.
221
+ if len(admittedOurs) > maxRelayLabelCardinality {
222
+ t.Errorf("admitted our-namespace relays: want <=%d, got %d", maxRelayLabelCardinality, len(admittedOurs))
223
+ }
224
+}
225
+
226
+// TestMetrics_NoPIILabels iterates every gathered metric family and every label
227
+// pair within and asserts that no label *name* equals "client_hash" or
228
+// "local_address". This is the Phase 1 regression defense for acceptance #4.
229
+func TestMetrics_NoPIILabels(t *testing.T) {
230
+ mfs, err := prometheus.DefaultGatherer.Gather()
231
+ if err != nil {
232
+ t.Fatalf("gather: %v", err)
233
+ }
234
+ for _, mf := range mfs {
235
+ for _, m := range mf.GetMetric() {
236
+ for _, lp := range m.GetLabel() {
237
+ name := lp.GetName()
238
+ if name == "client_hash" || name == "local_address" {
239
+ t.Errorf("PII label %q found in metric family %q", name, mf.GetName())
240
+ }
241
+ }
242
+ }
243
+ }
244
+}