Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
Integrate circuit breaker modules with reverse proxy
Matthew Holt committed
Sep 3, 2019 at 19:06 UTC
acb8f0e0c26acd95cbee8981469b4ac62535d164
3 files changed
+36
-2
modules/caddyhttp/reverseproxy/healthchecks.go
+9
-1
@@ -64,6 +64,14 @@ type PassiveHealthChecks struct {
64
UnhealthyLatency caddy.Duration `json:"unhealthy_latency,omitempty"`
65
}
66
67
+// CircuitBreaker is a type that can act as an early-warning
68
+// system for the health checker when backends are getting
69
+// overloaded.
70
+type CircuitBreaker interface {
71
+ OK() bool
72
+ RecordMetric(statusCode int, latency time.Duration)
73
+}
74
+
75
// activeHealthChecker runs active health checks on a
76
// regular basis and blocks until
77
// h.HealthChecks.Active.stopChan is closed.
@@ -202,7 +210,7 @@ func (h *Handler) doActiveHealthCheck(hostAddr string, host Host) error {
210
// remembers 1 failure for upstream for the configured
211
// duration. If passive health checks are disabled or
212
// failure expiry is 0, this is a no-op.
205
-func (h Handler) countFailure(upstream *Upstream) {
213
+func (h *Handler) countFailure(upstream *Upstream) {
214
// only count failures if passive health checking is enabled
215
// and if failures are configured have a non-zero expiry
216
if h.HealthChecks == nil || h.HealthChecks.Passive == nil {
modules/caddyhttp/reverseproxy/hosts.go
+9
-1
@@ -69,21 +69,29 @@ type Upstream struct {
69
70
healthCheckPolicy *PassiveHealthChecks
71
hostURL *url.URL
72
+ cb CircuitBreaker
73
}
74
75
// Available returns true if the remote host
75
-// is available to receive requests.
76
+// is available to receive requests. This is
77
+// the method that should be used by selection
78
+// policies, etc. to determine if a backend
79
+// should be able to be sent a request.
80
func (u *Upstream) Available() bool {
81
return u.Healthy() && !u.Full()
82
}
83
84
// Healthy returns true if the remote host
85
// is currently known to be healthy or "up".
86
+// It consults the circuit breaker, if any.
87
func (u *Upstream) Healthy() bool {
88
healthy := !u.Host.Unhealthy()
89
if healthy && u.healthCheckPolicy != nil {
90
healthy = u.Host.Fails() < u.healthCheckPolicy.MaxFails
91
}
92
+ if healthy && u.cb != nil {
93
+ healthy = u.cb.OK()
94
+ }
95
return healthy
96
}
97
modules/caddyhttp/reverseproxy/reverseproxy.go
+18
@@ -37,12 +37,14 @@ func init() {
37
// Handler implements a highly configurable and production-ready reverse proxy.
38
type Handler struct {
39
TransportRaw json.RawMessage `json:"transport,omitempty"`
40
+ CBRaw json.RawMessage `json:"circuit_breaker,omitempty"`
41
LoadBalancing *LoadBalancing `json:"load_balancing,omitempty"`
42
HealthChecks *HealthChecks `json:"health_checks,omitempty"`
43
Upstreams UpstreamPool `json:"upstreams,omitempty"`
44
FlushInterval caddy.Duration `json:"flush_interval,omitempty"`
45
46
Transport http.RoundTripper `json:"-"`
47
+ CB CircuitBreaker `json:"-"`
48
}
49
50
// CaddyModule returns the Caddy module information.
@@ -55,6 +57,7 @@ func (Handler) CaddyModule() caddy.ModuleInfo {
57
58
// Provision ensures that h is set up properly before use.
59
func (h *Handler) Provision(ctx caddy.Context) error {
60
+ // start by loading modules
61
if h.TransportRaw != nil {
62
val, err := ctx.LoadModuleInline("protocol", "http.handlers.reverse_proxy.transport", h.TransportRaw)
63
if err != nil {
@@ -73,6 +76,14 @@ func (h *Handler) Provision(ctx caddy.Context) error {
76
h.LoadBalancing.SelectionPolicy = val.(Selector)
77
h.LoadBalancing.SelectionPolicyRaw = nil // allow GC to deallocate - TODO: Does this help?
78
}
79
+ if h.CBRaw != nil {
80
+ val, err := ctx.LoadModuleInline("type", "http.handlers.reverse_proxy.circuit_breakers", h.CBRaw)
81
+ if err != nil {
82
+ return fmt.Errorf("loading circuit breaker module: %s", err)
83
+ }
84
+ h.CB = val.(CircuitBreaker)
85
+ h.CBRaw = nil // allow GC to deallocate - TODO: Does this help?
86
+ }
87
88
if h.Transport == nil {
89
h.Transport = defaultTransport
@@ -123,6 +134,8 @@ func (h *Handler) Provision(ctx caddy.Context) error {
134
}
135
136
for _, upstream := range h.Upstreams {
137
+ upstream.cb = h.CB
138
+
139
// url parser requires a scheme
140
if !strings.Contains(upstream.Address, "://") {
141
upstream.Address = "http://" + upstream.Address
@@ -307,6 +320,11 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, upstre
320
return err
321
}
322
323
+ // update circuit breaker on current conditions
324
+ if upstream.cb != nil {
325
+ upstream.cb.RecordMetric(res.StatusCode, latency)
326
+ }
327
+
328
// perform passive health checks (if enabled)
329
if h.HealthChecks != nil && h.HealthChecks.Passive != nil {
330
// strike if the status code matches one that is "bad"