@marcel91200 / caddy Mirror · read-only / commits / 6584c759
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 11, 07:15 UTC

reverseproxy: isolate active health-check state per distinct check config (#7916)

* reverseproxy: isolate active health-check state per distinct check config Multiple reverse_proxy handlers configured with different active health checks (health_uri, health_headers, ...) against the same upstream dial address currently share a single Host in the global pool, so one handler's failing probes mark the address unhealthy for every other handler. Key the pool by dial address plus a stable fingerprint of the active health-check config, so distinct checks get independent health state. The fingerprint is strictly internal to pool identity: the Prometheus upstreams_healthy label and the /reverse_proxy/upstreams admin endpoint continue to report the plain dial address, unchanged. Dynamic upstreams are intentionally out of scope here: they resolve through a separate per-lookup path (dynamicHosts) and collapsing there has different lifetime semantics; noted for a follow-up. Fixes #7870 * reverseproxy: use strings.Cut in hostKeyAddress Satisfies the modernize linter; behaviour is unchanged, since Cut returns the whole string when the separator is absent. * reverseproxy: expose the health-check fingerprint as a public discriminator Health state is now kept per (dial address, active health check config), but both user-visible surfaces still reported address alone: - caddy_reverse_proxy_upstreams_healthy was labeled only by upstream, so every handler sharing an address wrote the same series concurrently and the reported value was whichever updater ran last. The metric gains a health_check label carrying the config fingerprint ("" when no active checks), so each health target owns its series; aggregate across checks with sum/min by (upstream). - /reverse_proxy/upstreams reported one entry per pool key but with only the plain address, so consumers indexing by address silently discarded all but one entry. Entries now carry health_check (omitted when empty), and the endpoint documents that (address, health_check) is the entry's identity — one entry per health target, deliberately not aggregated, since any aggregation here would be lossy and undocumented. Tests: two handlers on one address with different checks must produce two metric series reflecting their own state (fails if the fingerprint is dropped from the label), and two admin entries distinguished by non-empty fingerprints. * reverseproxy: narrow the fix to per-Upstream active health counters Move the consecutive active pass/fail counters from Host onto Upstream, alongside the active unhealthy state that already lives there, instead of re-keying the global host pool. Host is keyed by dial address alone, but an active health check is configured per handler, so two handlers dialing the same address with different health_uri or health_headers share those counters and can push each other over their own thresholds. Upstream is already per-handler and already carries the active unhealthy flag, so the counters belong next to it and the pool keeps its plain dial-address keys. This drops the host key fingerprint and its exposure in the metric label and the admin upstreams endpoint; the metric series identity is left for separate consideration. --------- Co-authored-by: SillyZir <269283839+SillyZir@users.noreply.github.com> Co-authored-by: Zen Dodd <mail@steadytao.com>

SillyZir committed Aug 12, 2026 at 01:20 UTC 6584c75999996c71f154a04f4e41f1ebce65a9c3
3 files changed +187 -27
modules/caddyhttp/reverseproxy/active_health_test.go new
+151
@@ -0,0 +1,151 @@
1 +// Copyright 2015 Matthew Holt and The Caddy Authors
2 +//
3 +// Licensed under the Apache License, Version 2.0 (the "License");
4 +// you may not use this file except in compliance with the License.
5 +// You may obtain a copy of the License at
6 +//
7 +// http://www.apache.org/licenses/LICENSE-2.0
8 +//
9 +// Unless required by applicable law or agreed to in writing, software
10 +// distributed under the License is distributed on an "AS IS" BASIS,
11 +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 +// See the License for the specific language governing permissions and
13 +// limitations under the License.
14 +
15 +package reverseproxy
16 +
17 +import (
18 + "context"
19 + "net/http"
20 + "net/http/httptest"
21 + "strings"
22 + "sync/atomic"
23 + "testing"
24 +
25 + "go.uber.org/zap"
26 +
27 + "github.com/caddyserver/caddy/v2"
28 + "github.com/caddyserver/caddy/v2/modules/caddyevents"
29 +)
30 +
31 +// newActiveHandler builds a minimal Handler with active health checks
32 +// configured against addr, provisions its single upstream, and returns
33 +// the handler, its upstream, and a cancel func the caller must defer.
34 +func newActiveHandler(t *testing.T, addr, uri string, fails int) (*Handler, *Upstream, context.CancelFunc) {
35 + t.Helper()
36 + caddyCtx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
37 +
38 + eventsApp := new(caddyevents.App)
39 + if err := eventsApp.Provision(caddyCtx); err != nil {
40 + t.Fatalf("provisioning events app: %v", err)
41 + }
42 +
43 + u := &Upstream{Dial: addr}
44 + h := &Handler{
45 + ctx: caddyCtx,
46 + logger: zap.NewNop(),
47 + events: eventsApp,
48 + Upstreams: UpstreamPool{u},
49 + HealthChecks: &HealthChecks{
50 + Active: &ActiveHealthChecks{
51 + URI: uri,
52 + Fails: fails,
53 + },
54 + },
55 + }
56 + h.provisionUpstream(u, false)
57 + if err := h.HealthChecks.Active.Provision(caddyCtx, h); err != nil {
58 + t.Fatalf("provisioning active health checks: %v", err)
59 + }
60 + return h, u, cancel
61 +}
62 +
63 +// runActiveHealthCheck synchronously performs one active health check round
64 +// for the handler's upstream (what doActiveHealthCheckForAllHosts does per
65 +// tick, minus the goroutine).
66 +func runActiveHealthCheck(t *testing.T, h *Handler, u *Upstream) {
67 + t.Helper()
68 + dialInfo, err := u.fillDialInfo(caddy.NewReplacer())
69 + if err != nil {
70 + t.Fatalf("filling dial info: %v", err)
71 + }
72 + if err := h.doActiveHealthCheck(dialInfo, dialInfo.Address, u.Dial, u); err != nil {
73 + t.Fatalf("active health check: %v", err)
74 + }
75 +}
76 +
77 +// drainHostsPool removes every entry from the global static host pool,
78 +// deleting each exactly as many times as it was stored so the pool is
79 +// empty for subsequent tests.
80 +func drainHostsPool() {
81 + var keys []any
82 + hosts.Range(func(key, _ any) bool {
83 + keys = append(keys, key)
84 + return true
85 + })
86 + for _, key := range keys {
87 + if refs, ok := hosts.References(key); ok {
88 + for range refs {
89 + _, _ = hosts.Delete(key)
90 + }
91 + }
92 + }
93 +}
94 +
95 +// TestActiveHealthChecksSameAddressDifferentChecksAreIndependent is a
96 +// regression test for https://github.com/caddyserver/caddy/issues/7870:
97 +// two handlers that dial the same upstream address but run different
98 +// active health checks (different health_uri) must keep independent
99 +// health state. Before the fix, the consecutive pass/fail counters lived
100 +// on the Host, which is shared by dial address, so both checkers mutated
101 +// the same counters: one vhost's failing probes could push the other
102 +// vhost's upstream over its own fails threshold and knock it out of that
103 +// vhost's data path.
104 +func TestActiveHealthChecksSameAddressDifferentChecksAreIndependent(t *testing.T) {
105 + resetDynamicHosts()
106 + defer drainHostsPool()
107 +
108 + // one backend node serving two vhosts: vhost A's health endpoint is
109 + // down, vhost B's is up (except for one transient failure below)
110 + var vhostBUp atomic.Bool
111 + vhostBUp.Store(true)
112 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
113 + switch r.URL.Path {
114 + case "/vhost-a/health":
115 + w.WriteHeader(http.StatusServiceUnavailable)
116 + case "/vhost-b/health":
117 + if vhostBUp.Load() {
118 + w.WriteHeader(http.StatusOK)
119 + } else {
120 + w.WriteHeader(http.StatusServiceUnavailable)
121 + }
122 + default:
123 + w.WriteHeader(http.StatusNotFound)
124 + }
125 + }))
126 + defer srv.Close()
127 + addr := strings.TrimPrefix(srv.URL, "http://")
128 +
129 + hA, uA, cancelA := newActiveHandler(t, addr, "/vhost-a/health", 5)
130 + defer cancelA()
131 + hB, uB, cancelB := newActiveHandler(t, addr, "/vhost-b/health", 3)
132 + defer cancelB()
133 +
134 + // vhost A's checker observes two consecutive failures — below its own
135 + // fails=5 threshold, so these must affect no one's health status
136 + runActiveHealthCheck(t, hA, uA)
137 + runActiveHealthCheck(t, hA, uA)
138 +
139 + // vhost B's checker observes a single transient failure; B tolerates
140 + // up to fails=3 consecutive failures, so it must remain healthy
141 + vhostBUp.Store(false)
142 + runActiveHealthCheck(t, hB, uB)
143 +
144 + if !uA.Healthy() {
145 + t.Errorf("vhost A's upstream should still be healthy after 2 of 5 tolerated failures")
146 + }
147 + if !uB.Healthy() {
148 + t.Errorf("vhost B's upstream was marked unhealthy after a single failed probe (fails=3); " +
149 + "its health state was polluted by vhost A's health check against the same address")
150 + }
151 +}
modules/caddyhttp/reverseproxy/healthchecks.go
+5 -5
@@ -461,7 +461,7 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, networ
461
462 markUnhealthy := func() {
463 // increment failures and then check if it has reached the threshold to mark unhealthy
464 - err := upstream.Host.countHealthFail(1)
464 + err := upstream.countHealthFail(1)
465 if err != nil {
466 if c := h.HealthChecks.Active.logger.Check(zapcore.ErrorLevel, "could not count active health failure"); c != nil {
467 c.Write(
@@ -471,11 +471,11 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, networ
471 }
472 return
473 }
474 - if upstream.Host.activeHealthFails() >= h.HealthChecks.Active.Fails {
474 + if upstream.activeHealthFails() >= h.HealthChecks.Active.Fails {
475 // dispatch an event that the host newly became unhealthy
476 if upstream.setHealthy(false) {
477 h.events.Emit(h.ctx, "unhealthy", map[string]any{"host": hostAddr})
478 - upstream.Host.resetHealth()
478 + upstream.resetHealth()
479 }
480 }
481 }
@@ -492,13 +492,13 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, networ
492 }
493 return
494 }
495 - if upstream.Host.activeHealthPasses() >= h.HealthChecks.Active.Passes {
495 + if upstream.activeHealthPasses() >= h.HealthChecks.Active.Passes {
496 if upstream.setHealthy(true) {
497 if c := h.HealthChecks.Active.logger.Check(zapcore.InfoLevel, "host is up"); c != nil {
498 c.Write(zap.String("host", hostAddr))
499 }
500 h.events.Emit(h.ctx, "healthy", map[string]any{"host": hostAddr})
501 - upstream.Host.resetHealth()
501 + upstream.resetHealth()
502 }
503 }
504 }
modules/caddyhttp/reverseproxy/hosts.go
+31 -22
@@ -61,7 +61,16 @@ type Upstream struct {
61 activeHealthCheckUpstream string
62 healthCheckPolicy *PassiveHealthChecks
63 cb CircuitBreaker
64 - unhealthy atomic.Int32 // status from active health checker
64 +
65 + // state from the active health checker. It lives here rather than on
66 + // the shared Host because the Host is keyed by dial address alone,
67 + // while an active health check is configured per handler: two handlers
68 + // dialing the same address with different health_uri or health_headers
69 + // are checking distinct health targets, and must not push each other
70 + // over their own consecutive pass/fail thresholds.
71 + unhealthy atomic.Int32
72 + activePasses atomic.Int64
73 + activeFails atomic.Int64
74 }
75
76 // (pointer receiver necessary to avoid a race condition, since
@@ -173,10 +182,8 @@ func (u *Upstream) fillDynamicHost() {
182 // Host is the basic, in-memory representation of the state of a remote host.
183 // Its fields are accessed atomically and Host values must not be copied.
184 type Host struct {
176 - numRequests atomic.Int64
177 - fails atomic.Int64
178 - activePasses atomic.Int64
179 - activeFails atomic.Int64
185 + numRequests atomic.Int64
186 + fails atomic.Int64
187 }
188
189 // NumRequests returns the number of active requests to the upstream.
@@ -189,16 +196,6 @@ func (h *Host) Fails() int {
196 return int(h.fails.Load())
197 }
198
192 -// activeHealthPasses returns the number of consecutive active health check passes with the upstream.
193 -func (h *Host) activeHealthPasses() int {
194 - return int(h.activePasses.Load())
195 -}
196 -
197 -// activeHealthFails returns the number of consecutive active health check failures with the upstream.
198 -func (h *Host) activeHealthFails() int {
199 - return int(h.activeFails.Load())
200 -}
201 -
199 // countRequest mutates the active request count by
200 // delta. It returns an error if the adjustment fails.
201 func (h *Host) countRequest(delta int) error {
@@ -219,10 +216,22 @@ func (h *Host) countFail(delta int) error {
216 return nil
217 }
218
219 +// activeHealthPasses returns the number of consecutive passing
220 +// active health checks observed by this upstream's checker.
221 +func (u *Upstream) activeHealthPasses() int {
222 + return int(u.activePasses.Load())
223 +}
224 +
225 +// activeHealthFails returns the number of consecutive failing
226 +// active health checks observed by this upstream's checker.
227 +func (u *Upstream) activeHealthFails() int {
228 + return int(u.activeFails.Load())
229 +}
230 +
231 // countHealthPass mutates the recent passes count by
232 // delta. It returns an error if the adjustment fails.
224 -func (h *Host) countHealthPass(delta int) error {
225 - result := h.activePasses.Add(int64(delta))
233 +func (u *Upstream) countHealthPass(delta int) error {
234 + result := u.activePasses.Add(int64(delta))
235 if result < 0 {
236 return fmt.Errorf("count below 0: %d", result)
237 }
@@ -231,8 +240,8 @@ func (h *Host) countHealthPass(delta int) error {
240
241 // countHealthFail mutates the recent failures count by
242 // delta. It returns an error if the adjustment fails.
234 -func (h *Host) countHealthFail(delta int) error {
235 - result := h.activeFails.Add(int64(delta))
243 +func (u *Upstream) countHealthFail(delta int) error {
244 + result := u.activeFails.Add(int64(delta))
245 if result < 0 {
246 return fmt.Errorf("count below 0: %d", result)
247 }
@@ -240,9 +249,9 @@ func (h *Host) countHealthFail(delta int) error {
249 }
250
251 // resetHealth resets the health check counters.
243 -func (h *Host) resetHealth() {
244 - h.activePasses.Store(0)
245 - h.activeFails.Store(0)
252 +func (u *Upstream) resetHealth() {
253 + u.activePasses.Store(0)
254 + u.activeFails.Store(0)
255 }
256
257 // healthy returns true if the upstream is not actively marked as unhealthy.