Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
reverse_proxy: Add local circuit breaker
This migrates a feature that was previously reserved for enterprise users, according to https://github.com/caddyserver/caddy/issues/2786. The local circuit breaker is a simple metrics counter that can cause the reverse proxy to consider a backend unhealthy before it actually goes offline, by measuring recent latencies over a sliding window. Credit to Danny Navarro
Matthew Holt committed
Oct 9, 2019 at 19:28 UTC
bcbe1c220de99146a13bcc786ec7f5017681de73
1 file changed
+152
modules/caddyhttp/reverseproxy/circuitbreaker.go
new
+152
@@ -0,0 +1,152 @@
1
+// Copyright 2015 Matthew Holt and The Caddy Authors
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package reverseproxy
16
+
17
+import (
18
+ "fmt"
19
+ "sync/atomic"
20
+ "time"
21
+
22
+ "github.com/caddyserver/caddy/v2"
23
+ "github.com/vulcand/oxy/memmetrics"
24
+)
25
+
26
+func init() {
27
+ caddy.RegisterModule(localCircuitBreaker{})
28
+}
29
+
30
+// localCircuitBreaker implements circuit breaking functionality
31
+// for requests within this process over a sliding time window.
32
+type localCircuitBreaker struct {
33
+ tripped int32
34
+ cbType int32
35
+ threshold float64
36
+ metrics *memmetrics.RTMetrics
37
+ tripTime time.Duration
38
+ Config
39
+}
40
+
41
+// CaddyModule returns the Caddy module information.
42
+func (localCircuitBreaker) CaddyModule() caddy.ModuleInfo {
43
+ return caddy.ModuleInfo{
44
+ Name: "http.handlers.reverse_proxy.circuit_breakers.local",
45
+ New: func() caddy.Module { return new(localCircuitBreaker) },
46
+ }
47
+}
48
+
49
+// Provision sets up a configured circuit breaker.
50
+func (c *localCircuitBreaker) Provision(ctx caddy.Context) error {
51
+ t, ok := typeCB[c.Type]
52
+ if !ok {
53
+ return fmt.Errorf("type is not defined")
54
+ }
55
+
56
+ if c.TripTime == "" {
57
+ c.TripTime = defaultTripTime
58
+ }
59
+
60
+ tw, err := time.ParseDuration(c.TripTime)
61
+ if err != nil {
62
+ return fmt.Errorf("cannot parse trip_time duration, %v", err.Error())
63
+ }
64
+
65
+ mt, err := memmetrics.NewRTMetrics()
66
+ if err != nil {
67
+ return fmt.Errorf("cannot create new metrics: %v", err.Error())
68
+ }
69
+
70
+ c.cbType = t
71
+ c.tripTime = tw
72
+ c.threshold = c.Threshold
73
+ c.metrics = mt
74
+ c.tripped = 0
75
+
76
+ return nil
77
+}
78
+
79
+// Ok returns whether the circuit breaker is tripped or not.
80
+func (c *localCircuitBreaker) Ok() bool {
81
+ tripped := atomic.LoadInt32(&c.tripped)
82
+ return tripped == 0
83
+}
84
+
85
+// RecordMetric records a response status code and execution time of a request. This function should be run in a separate goroutine.
86
+func (c *localCircuitBreaker) RecordMetric(statusCode int, latency time.Duration) {
87
+ c.metrics.Record(statusCode, latency)
88
+ c.checkAndSet()
89
+}
90
+
91
+// Ok checks our metrics to see if we should trip our circuit breaker, or if the fallback duration has completed.
92
+func (c *localCircuitBreaker) checkAndSet() {
93
+ var isTripped bool
94
+
95
+ switch c.cbType {
96
+ case typeErrorRatio:
97
+ // check if amount of network errors exceed threshold over sliding window, threshold for comparison should be < 1.0 i.e. .5 = 50th percentile
98
+ if c.metrics.NetworkErrorRatio() > c.threshold {
99
+ isTripped = true
100
+ }
101
+ case typeLatency:
102
+ // check if threshold in milliseconds is reached and trip
103
+ hist, err := c.metrics.LatencyHistogram()
104
+ if err != nil {
105
+ return
106
+ }
107
+
108
+ l := hist.LatencyAtQuantile(c.threshold)
109
+ if l.Nanoseconds()/int64(time.Millisecond) > int64(c.threshold) {
110
+ isTripped = true
111
+ }
112
+ case typeStatusCodeRatio:
113
+ // check ratio of error status codes of sliding window, threshold for comparison should be < 1.0 i.e. .5 = 50th percentile
114
+ if c.metrics.ResponseCodeRatio(500, 600, 0, 600) > c.threshold {
115
+ isTripped = true
116
+ }
117
+ }
118
+
119
+ if isTripped {
120
+ c.metrics.Reset()
121
+ atomic.AddInt32(&c.tripped, 1)
122
+
123
+ // wait tripTime amount before allowing operations to resume.
124
+ t := time.NewTimer(c.tripTime)
125
+ <-t.C
126
+
127
+ atomic.AddInt32(&c.tripped, -1)
128
+ }
129
+}
130
+
131
+// Config represents the configuration of a circuit breaker.
132
+type Config struct {
133
+ Threshold float64 `json:"threshold"`
134
+ Type string `json:"type"`
135
+ TripTime string `json:"trip_time"`
136
+}
137
+
138
+const (
139
+ typeLatency = iota + 1
140
+ typeErrorRatio
141
+ typeStatusCodeRatio
142
+ defaultTripTime = "5s"
143
+)
144
+
145
+var (
146
+ // typeCB handles converting a Config Type value to the internal circuit breaker types.
147
+ typeCB = map[string]int32{
148
+ "latency": typeLatency,
149
+ "error_ratio": typeErrorRatio,
150
+ "status_ratio": typeStatusCodeRatio,
151
+ }
152
+)