Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 15:45 UTC
reverseproxy: configurable active health_passes and health_fails (#6154)
* reverseproxy: active health check allows configurable health_passes and health_fails * Need to reset counters after recovery * rename methods to be more clear that these are coming from active health checks * do not export methods
Sam Ottenhoff committed
Mar 20, 2024 at 13:13 UTC
e65b97f55b6c5da8859fbd5e5b397cf089bf7716
4 files changed
+125
-9
caddytest/integration/reverseproxy_test.go
+2
@@ -350,6 +350,8 @@ func TestReverseProxyHealthCheck(t *testing.T) {
350
health_port 2021
351
health_interval 10ms
352
health_timeout 100ms
353
+ health_passes 1
354
+ health_fails 1
355
}
356
}
357
`, "caddyfile")
modules/caddyhttp/reverseproxy/caddyfile.go
+34
@@ -69,6 +69,8 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
69
// health_uri <uri>
70
// health_port <port>
71
// health_interval <interval>
72
+// health_passes <num>
73
+// health_fails <num>
74
// health_timeout <duration>
75
// health_status <status>
76
// health_body <regexp>
@@ -447,6 +449,38 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
449
}
450
h.HealthChecks.Active.ExpectBody = d.Val()
451
452
+ case "health_passes":
453
+ if !d.NextArg() {
454
+ return d.ArgErr()
455
+ }
456
+ if h.HealthChecks == nil {
457
+ h.HealthChecks = new(HealthChecks)
458
+ }
459
+ if h.HealthChecks.Active == nil {
460
+ h.HealthChecks.Active = new(ActiveHealthChecks)
461
+ }
462
+ passes, err := strconv.Atoi(d.Val())
463
+ if err != nil {
464
+ return d.Errf("invalid passes count '%s': %v", d.Val(), err)
465
+ }
466
+ h.HealthChecks.Active.Passes = passes
467
+
468
+ case "health_fails":
469
+ if !d.NextArg() {
470
+ return d.ArgErr()
471
+ }
472
+ if h.HealthChecks == nil {
473
+ h.HealthChecks = new(HealthChecks)
474
+ }
475
+ if h.HealthChecks.Active == nil {
476
+ h.HealthChecks.Active = new(ActiveHealthChecks)
477
+ }
478
+ fails, err := strconv.Atoi(d.Val())
479
+ if err != nil {
480
+ return d.Errf("invalid fails count '%s': %v", d.Val(), err)
481
+ }
482
+ h.HealthChecks.Active.Fails = fails
483
+
484
case "max_fails":
485
if !d.NextArg() {
486
return d.ArgErr()
modules/caddyhttp/reverseproxy/healthchecks.go
+49
-7
@@ -89,6 +89,14 @@ type ActiveHealthChecks struct {
89
// considering it unhealthy (default 5s).
90
Timeout caddy.Duration `json:"timeout,omitempty"`
91
92
+ // Number of consecutive health check passes before marking
93
+ // a previously unhealthy backend as healthy again (default 1).
94
+ Passes int `json:"passes,omitempty"`
95
+
96
+ // Number of consecutive health check failures before marking
97
+ // a previously healthy backend as unhealthy (default 1).
98
+ Fails int `json:"fails,omitempty"`
99
+
100
// The maximum response body to download from the backend
101
// during a health check.
102
MaxSize int64 `json:"max_size,omitempty"`
@@ -167,6 +175,14 @@ func (a *ActiveHealthChecks) Provision(ctx caddy.Context, h *Handler) error {
175
}
176
}
177
178
+ if a.Passes < 1 {
179
+ a.Passes = 1
180
+ }
181
+
182
+ if a.Fails < 1 {
183
+ a.Fails = 1
184
+ }
185
+
186
return nil
187
}
188
@@ -373,9 +389,37 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, upstre
389
}
390
391
markUnhealthy := func() {
376
- // dispatch an event that the host newly became unhealthy
377
- if upstream.setHealthy(false) {
378
- h.events.Emit(h.ctx, "unhealthy", map[string]any{"host": hostAddr})
392
+ // increment failures and then check if it has reached the threshold to mark unhealthy
393
+ err := upstream.Host.countHealthFail(1)
394
+ if err != nil {
395
+ h.HealthChecks.Active.logger.Error("could not count active health failure",
396
+ zap.String("host", upstream.Dial),
397
+ zap.Error(err))
398
+ return
399
+ }
400
+ if upstream.Host.activeHealthFails() >= h.HealthChecks.Active.Fails {
401
+ // dispatch an event that the host newly became unhealthy
402
+ if upstream.setHealthy(false) {
403
+ h.events.Emit(h.ctx, "unhealthy", map[string]any{"host": hostAddr})
404
+ upstream.Host.resetHealth()
405
+ }
406
+ }
407
+ }
408
+
409
+ markHealthy := func() {
410
+ // increment passes and then check if it has reached the threshold to be healthy
411
+ err := upstream.Host.countHealthPass(1)
412
+ if err != nil {
413
+ h.HealthChecks.Active.logger.Error("could not count active health pass",
414
+ zap.String("host", upstream.Dial),
415
+ zap.Error(err))
416
+ return
417
+ }
418
+ if upstream.Host.activeHealthPasses() >= h.HealthChecks.Active.Passes {
419
+ if upstream.setHealthy(true) {
420
+ h.events.Emit(h.ctx, "healthy", map[string]any{"host": hostAddr})
421
+ upstream.Host.resetHealth()
422
+ }
423
}
424
}
425
@@ -439,10 +483,8 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, upstre
483
}
484
485
// passed health check parameters, so mark as healthy
442
- if upstream.setHealthy(true) {
443
- h.HealthChecks.Active.logger.Info("host is up", zap.String("host", hostAddr))
444
- h.events.Emit(h.ctx, "healthy", map[string]any{"host": hostAddr})
445
- }
486
+ h.HealthChecks.Active.logger.Info("host is up", zap.String("host", hostAddr))
487
+ markHealthy()
488
489
return nil
490
}
modules/caddyhttp/reverseproxy/hosts.go
+40
-2
@@ -136,8 +136,10 @@ func (u *Upstream) fillHost() {
136
// Host is the basic, in-memory representation of the state of a remote host.
137
// Its fields are accessed atomically and Host values must not be copied.
138
type Host struct {
139
- numRequests int64 // must be 64-bit aligned on 32-bit systems (see https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
140
- fails int64
139
+ numRequests int64 // must be 64-bit aligned on 32-bit systems (see https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
140
+ fails int64
141
+ activePasses int64
142
+ activeFails int64
143
}
144
145
// NumRequests returns the number of active requests to the upstream.
@@ -150,6 +152,16 @@ func (h *Host) Fails() int {
152
return int(atomic.LoadInt64(&h.fails))
153
}
154
155
+// activeHealthPasses returns the number of consecutive active health check passes with the upstream.
156
+func (h *Host) activeHealthPasses() int {
157
+ return int(atomic.LoadInt64(&h.activePasses))
158
+}
159
+
160
+// activeHealthFails returns the number of consecutive active health check failures with the upstream.
161
+func (h *Host) activeHealthFails() int {
162
+ return int(atomic.LoadInt64(&h.activeFails))
163
+}
164
+
165
// countRequest mutates the active request count by
166
// delta. It returns an error if the adjustment fails.
167
func (h *Host) countRequest(delta int) error {
@@ -170,6 +182,32 @@ func (h *Host) countFail(delta int) error {
182
return nil
183
}
184
185
+// countHealthPass mutates the recent passes count by
186
+// delta. It returns an error if the adjustment fails.
187
+func (h *Host) countHealthPass(delta int) error {
188
+ result := atomic.AddInt64(&h.activePasses, int64(delta))
189
+ if result < 0 {
190
+ return fmt.Errorf("count below 0: %d", result)
191
+ }
192
+ return nil
193
+}
194
+
195
+// countHealthFail mutates the recent failures count by
196
+// delta. It returns an error if the adjustment fails.
197
+func (h *Host) countHealthFail(delta int) error {
198
+ result := atomic.AddInt64(&h.activeFails, int64(delta))
199
+ if result < 0 {
200
+ return fmt.Errorf("count below 0: %d", result)
201
+ }
202
+ return nil
203
+}
204
+
205
+// resetHealth resets the health check counters.
206
+func (h *Host) resetHealth() {
207
+ atomic.StoreInt64(&h.activePasses, 0)
208
+ atomic.StoreInt64(&h.activeFails, 0)
209
+}
210
+
211
// healthy returns true if the upstream is not actively marked as unhealthy.
212
// (This returns the status only from the "active" health checks.)
213
func (u *Upstream) healthy() bool {