| 1 | package telemetry_test |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "fmt" |
| 6 | "testing" |
| 7 | "time" |
| 8 | |
| 9 | dto "github.com/prometheus/client_model/go" |
| 10 | |
| 11 | "github.com/gosuda/portal-tunnel/v2/portal/telemetry" |
| 12 | "github.com/prometheus/client_golang/prometheus" |
| 13 | ) |
| 14 | |
| 15 | // metricFamilyByName gathers all metric families and returns the one with the |
| 16 | // given name, or nil if not found. |
| 17 | func metricFamilyByName(t *testing.T, name string) *dto.MetricFamily { |
| 18 | t.Helper() |
| 19 | mfs, err := prometheus.DefaultGatherer.Gather() |
| 20 | if err != nil { |
| 21 | t.Fatalf("gather: %v", err) |
| 22 | } |
| 23 | for _, mf := range mfs { |
| 24 | if mf.GetName() == name { |
| 25 | return mf |
| 26 | } |
| 27 | } |
| 28 | return nil |
| 29 | } |
| 30 | |
| 31 | // assertRegisteredWithName verifies that collector c is already registered on |
| 32 | // prometheus.DefaultRegisterer by attempting to re-register it and expecting |
| 33 | // AlreadyRegisteredError. It then confirms that the previously-registered |
| 34 | // collector's first described metric name equals wantName. |
| 35 | // |
| 36 | // This is the only approach that simultaneously proves (a) the collector is on |
| 37 | // the default registry and (b) the registered metric has the expected name, for |
| 38 | // both Vec and non-Vec collectors with no prior observations. |
| 39 | func assertRegisteredWithName(t *testing.T, c prometheus.Collector, wantName string) { |
| 40 | t.Helper() |
| 41 | err := prometheus.DefaultRegisterer.Register(c) |
| 42 | if err == nil { |
| 43 | // Re-registration succeeded — the collector was NOT on the default registry. |
| 44 | // Undo the registration so the rest of the test suite is not affected. |
| 45 | prometheus.DefaultRegisterer.Unregister(c) |
| 46 | t.Fatalf("metric %q: collector was not registered on DefaultRegisterer before the test", wantName) |
| 47 | } |
| 48 | var are prometheus.AlreadyRegisteredError |
| 49 | if !errors.As(err, &are) { |
| 50 | t.Fatalf("metric %q: unexpected registration error: %v", wantName, err) |
| 51 | } |
| 52 | // are.ExistingCollector is the collector already on the registry. |
| 53 | // Drain its Describe channel to confirm the expected metric name is present. |
| 54 | ch := make(chan *prometheus.Desc, 32) |
| 55 | go func() { |
| 56 | are.ExistingCollector.Describe(ch) |
| 57 | close(ch) |
| 58 | }() |
| 59 | found := false |
| 60 | for d := range ch { |
| 61 | // Desc.String() format: Desc{fqName: "the_name", help: "...", ...} |
| 62 | s := d.String() |
| 63 | const marker = `fqName: "` |
| 64 | idx := 0 |
| 65 | for idx+len(marker) <= len(s) { |
| 66 | if s[idx:idx+len(marker)] == marker { |
| 67 | start := idx + len(marker) |
| 68 | end := start |
| 69 | for end < len(s) && s[end] != '"' { |
| 70 | end++ |
| 71 | } |
| 72 | if s[start:end] == wantName { |
| 73 | found = true |
| 74 | } |
| 75 | break |
| 76 | } |
| 77 | idx++ |
| 78 | } |
| 79 | } |
| 80 | if !found { |
| 81 | t.Errorf("metric %q: name not found in described metrics of existing collector", wantName) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | // TestMetricsRegistryPresence asserts that all 8 Phase-1 metrics are registered |
| 86 | // on prometheus.DefaultRegisterer. It uses Register→AlreadyRegisteredError so |
| 87 | // Vec metrics with no prior observations are still detected (they are invisible |
| 88 | // to DefaultGatherer.Gather until the first label combination is used). |
| 89 | func TestMetricsRegistryPresence(t *testing.T) { |
| 90 | want := []struct { |
| 91 | name string |
| 92 | collector prometheus.Collector |
| 93 | typ dto.MetricType |
| 94 | }{ |
| 95 | {"portal_discovery_relay_selected_total", telemetry.RelaySelectedTotal, dto.MetricType_COUNTER}, |
| 96 | {"portal_discovery_relay_pool_size", telemetry.RelayPoolSize, dto.MetricType_GAUGE}, |
| 97 | {"portal_discovery_rtt_seconds", telemetry.RTTSeconds, dto.MetricType_HISTOGRAM}, |
| 98 | {"portal_discovery_active_tunnels_per_relay", telemetry.ActiveTunnelsPerRelay, dto.MetricType_GAUGE}, |
| 99 | {"portal_discovery_selection_duration_seconds", telemetry.SelectionDurationSeconds, dto.MetricType_HISTOGRAM}, |
| 100 | {"portal_discovery_selection_skipped_total", telemetry.SelectionSkippedTotal, dto.MetricType_COUNTER}, |
| 101 | {"portal_discovery_failures_total", telemetry.FailuresTotal, dto.MetricType_COUNTER}, |
| 102 | {"portal_discovery_congestion_mode", telemetry.CongestionMode, dto.MetricType_GAUGE}, |
| 103 | } |
| 104 | |
| 105 | for _, tc := range want { |
| 106 | t.Run(tc.name, func(t *testing.T) { |
| 107 | // Primary check: collector is on DefaultRegisterer. |
| 108 | assertRegisteredWithName(t, tc.collector, tc.name) |
| 109 | // Secondary check: if the metric has observations, verify HELP and type. |
| 110 | mf := metricFamilyByName(t, tc.name) |
| 111 | if mf != nil { |
| 112 | if mf.GetHelp() == "" { |
| 113 | t.Errorf("metric %q has empty HELP string", tc.name) |
| 114 | } |
| 115 | if mf.GetType() != tc.typ { |
| 116 | t.Errorf("metric %q: got type %v, want %v", tc.name, mf.GetType(), tc.typ) |
| 117 | } |
| 118 | } |
| 119 | }) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | // TestEmitFromTrace_CounterIncrement verifies that EmitFromTrace increments |
| 124 | // relay_selected_total for each output URL and records a selection duration. |
| 125 | // URL names are test-namespaced to avoid coupling to other tests that share the |
| 126 | // process-global relay-cardinality map. |
| 127 | func TestEmitFromTrace_CounterIncrement(t *testing.T) { |
| 128 | r1 := "t-counter-r1" |
| 129 | r2 := "t-counter-r2" |
| 130 | |
| 131 | // Capture baseline before the call. |
| 132 | baseline := func(relay string) float64 { |
| 133 | mf := metricFamilyByName(t, "portal_discovery_relay_selected_total") |
| 134 | if mf == nil { |
| 135 | return 0 |
| 136 | } |
| 137 | for _, m := range mf.GetMetric() { |
| 138 | var gotRelay, gotReason string |
| 139 | for _, lp := range m.GetLabel() { |
| 140 | switch lp.GetName() { |
| 141 | case "relay": |
| 142 | gotRelay = lp.GetValue() |
| 143 | case "reason": |
| 144 | gotReason = lp.GetValue() |
| 145 | } |
| 146 | } |
| 147 | if gotRelay == relay && gotReason == "auto" { |
| 148 | return m.GetCounter().GetValue() |
| 149 | } |
| 150 | } |
| 151 | return 0 |
| 152 | } |
| 153 | |
| 154 | baseR1 := baseline(r1) |
| 155 | baseR2 := baseline(r2) |
| 156 | |
| 157 | // Capture duration baseline before the single EmitFromTrace call. |
| 158 | durationSampleCount := func() uint64 { |
| 159 | mf := metricFamilyByName(t, "portal_discovery_selection_duration_seconds") |
| 160 | if mf == nil || len(mf.GetMetric()) == 0 { |
| 161 | return 0 |
| 162 | } |
| 163 | return mf.GetMetric()[0].GetHistogram().GetSampleCount() |
| 164 | } |
| 165 | baseDur := durationSampleCount() |
| 166 | |
| 167 | telemetry.EmitFromTrace(telemetry.SelectionTrace{ |
| 168 | OutputURLs: []string{r1, r2}, |
| 169 | SelectionTook: 50 * time.Millisecond, |
| 170 | Congested: false, |
| 171 | NonLinear: false, |
| 172 | }) |
| 173 | |
| 174 | // relay_selected_total delta must be 1 for each relay. |
| 175 | afterR1 := baseline(r1) |
| 176 | afterR2 := baseline(r2) |
| 177 | if afterR1-baseR1 != 1 { |
| 178 | t.Errorf("relay_selected_total{relay=%q,reason=auto}: delta want 1, got %v", r1, afterR1-baseR1) |
| 179 | } |
| 180 | if afterR2-baseR2 != 1 { |
| 181 | t.Errorf("relay_selected_total{relay=%q,reason=auto}: delta want 1, got %v", r2, afterR2-baseR2) |
| 182 | } |
| 183 | |
| 184 | // selection_duration_seconds delta must be exactly 1 for this invocation. |
| 185 | afterDur := durationSampleCount() |
| 186 | if afterDur-baseDur != 1 { |
| 187 | t.Errorf("selection_duration_seconds sample delta want 1, got %d", afterDur-baseDur) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | // TestEmitFromTrace_CardinalityCap verifies the relay-label cardinality cap. |
| 192 | // |
| 193 | // We emit maxRelayLabelCardinality+1 distinct relay URLs and then assert: |
| 194 | // 1. relay="other" appears in relay_selected_total (overflow was bucketed). |
| 195 | // 2. Every emitted URL either appears as its own relay label OR caused "other" |
| 196 | // to be incremented — i.e., no URL is silently dropped. |
| 197 | // |
| 198 | // Because relayBudget is process-global and prior tests may have consumed some |
| 199 | // slots, we emit enough URLs (maxRelayLabelCardinality+1 = 65) to guarantee at |
| 200 | // least one overflow regardless of prior state, then verify the above. |
| 201 | // |
| 202 | // URLs are namespaced as "t-cap-NNN" to isolate them from other tests. |
| 203 | func TestEmitFromTrace_CardinalityCap(t *testing.T) { |
| 204 | const total = telemetry.MaxRelayLabelCardinality + 1 // 65 |
| 205 | |
| 206 | // Build the set of our namespace URLs. |
| 207 | ourURLs := make(map[string]struct{}, total) |
| 208 | for i := 0; i < total; i++ { |
| 209 | ourURLs[fmt.Sprintf("t-cap-%03d", i)] = struct{}{} |
| 210 | } |
| 211 | |
| 212 | // relayReasonCounter returns the counter value for the given (relay, reason) pair. |
| 213 | relayReasonCounter := func(relay, reason string) float64 { |
| 214 | mf := metricFamilyByName(t, "portal_discovery_relay_selected_total") |
| 215 | if mf == nil { |
| 216 | return 0 |
| 217 | } |
| 218 | for _, m := range mf.GetMetric() { |
| 219 | var r, rs string |
| 220 | for _, lp := range m.GetLabel() { |
| 221 | switch lp.GetName() { |
| 222 | case "relay": |
| 223 | r = lp.GetValue() |
| 224 | case "reason": |
| 225 | rs = lp.GetValue() |
| 226 | } |
| 227 | } |
| 228 | if r == relay && rs == reason { |
| 229 | return m.GetCounter().GetValue() |
| 230 | } |
| 231 | } |
| 232 | return 0 |
| 233 | } |
| 234 | |
| 235 | // Our traces are all non-congested non-nonlinear → reason="auto". |
| 236 | baseOther := relayReasonCounter("other", "auto") |
| 237 | |
| 238 | for i := 0; i < total; i++ { |
| 239 | url := fmt.Sprintf("t-cap-%03d", i) |
| 240 | telemetry.EmitFromTrace(telemetry.SelectionTrace{ |
| 241 | OutputURLs: []string{url}, |
| 242 | SelectionTook: time.Millisecond, |
| 243 | }) |
| 244 | } |
| 245 | |
| 246 | mf := metricFamilyByName(t, "portal_discovery_relay_selected_total") |
| 247 | if mf == nil { |
| 248 | t.Fatal("portal_discovery_relay_selected_total not found") |
| 249 | } |
| 250 | |
| 251 | // Collect the our-namespace relay labels that were admitted (got own slot). |
| 252 | admittedOurs := make(map[string]struct{}) |
| 253 | for _, m := range mf.GetMetric() { |
| 254 | var r string |
| 255 | for _, lp := range m.GetLabel() { |
| 256 | if lp.GetName() == "relay" { |
| 257 | r = lp.GetValue() |
| 258 | } |
| 259 | } |
| 260 | if _, ok := ourURLs[r]; ok { |
| 261 | admittedOurs[r] = struct{}{} |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | afterOther := relayReasonCounter("other", "auto") |
| 266 | overflowed := total - len(admittedOurs) // how many of our URLs were bucketed |
| 267 | |
| 268 | // Assert: at least one URL overflowed to "other". |
| 269 | if overflowed <= 0 { |
| 270 | t.Errorf("expected at least 1 URL to overflow to \"other\"; admitted=%d out of %d", len(admittedOurs), total) |
| 271 | } |
| 272 | |
| 273 | // Assert: the counter delta for {relay="other",reason="auto"} matches the |
| 274 | // number of our-namespace URLs that were not admitted (not merely inferred). |
| 275 | delta := afterOther - baseOther |
| 276 | if delta < float64(overflowed) { |
| 277 | t.Errorf("relay_selected_total{relay=\"other\",reason=\"auto\"} delta want >=%d, got %.0f", overflowed, delta) |
| 278 | } |
| 279 | |
| 280 | // Assert: admitted URL count never exceeds the cap. |
| 281 | if len(admittedOurs) > telemetry.MaxRelayLabelCardinality { |
| 282 | t.Errorf("admitted our-namespace relays: want <=%d, got %d", telemetry.MaxRelayLabelCardinality, len(admittedOurs)) |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | // TestMetrics_NoPIILabels iterates every gathered metric family and every label |
| 287 | // pair within and asserts that no label *name* equals "client_hash" or |
| 288 | // "local_address". This is the Phase 1 regression defense for acceptance #4. |
| 289 | func TestMetrics_NoPIILabels(t *testing.T) { |
| 290 | mfs, err := prometheus.DefaultGatherer.Gather() |
| 291 | if err != nil { |
| 292 | t.Fatalf("gather: %v", err) |
| 293 | } |
| 294 | for _, mf := range mfs { |
| 295 | for _, m := range mf.GetMetric() { |
| 296 | for _, lp := range m.GetLabel() { |
| 297 | name := lp.GetName() |
| 298 | if name == "client_hash" || name == "local_address" { |
| 299 | t.Errorf("PII label %q found in metric family %q", name, mf.GetName()) |
| 300 | } |
| 301 | } |
| 302 | } |
| 303 | } |
| 304 | } |