Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 16:45 UTC
reverseproxy: Implement retry count, alternative to try_duration (#4756)
* reverseproxy: Implement retry count, alternative to try_duration * Add Caddyfile support for `retry_match` * Refactor to deduplicate matcher parsing logic * Fix lint
Francis Lavoie committed
Jul 13, 2022 at 16:15 UTC
7d1f7771c9961dc6607a6ac5894b9c3195c25fcf
4 files changed
+189
-69
caddytest/integration/caddyfile_adapt/reverse_proxy_load_balance.txt
new
+64
@@ -0,0 +1,64 @@
1
+:8884
2
+
3
+reverse_proxy 127.0.0.1:65535 {
4
+ lb_policy first
5
+ lb_retries 5
6
+ lb_try_duration 10s
7
+ lb_try_interval 500ms
8
+ lb_retry_match {
9
+ path /foo*
10
+ method POST
11
+ }
12
+ lb_retry_match path /bar*
13
+}
14
+----------
15
+{
16
+ "apps": {
17
+ "http": {
18
+ "servers": {
19
+ "srv0": {
20
+ "listen": [
21
+ ":8884"
22
+ ],
23
+ "routes": [
24
+ {
25
+ "handle": [
26
+ {
27
+ "handler": "reverse_proxy",
28
+ "load_balancing": {
29
+ "retries": 5,
30
+ "retry_match": [
31
+ {
32
+ "method": [
33
+ "POST"
34
+ ],
35
+ "path": [
36
+ "/foo*"
37
+ ]
38
+ },
39
+ {
40
+ "path": [
41
+ "/bar*"
42
+ ]
43
+ }
44
+ ],
45
+ "selection_policy": {
46
+ "policy": "first"
47
+ },
48
+ "try_duration": 10000000000,
49
+ "try_interval": 500000000
50
+ },
51
+ "upstreams": [
52
+ {
53
+ "dial": "127.0.0.1:65535"
54
+ }
55
+ ]
56
+ }
57
+ ]
58
+ }
59
+ ]
60
+ }
61
+ }
62
+ }
63
+ }
64
+}
modules/caddyhttp/matchers.go
+54
-49
@@ -1003,57 +1003,12 @@ func (MatchNot) CaddyModule() caddy.ModuleInfo {
1003
1004
// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
1005
func (m *MatchNot) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1006
- // first, unmarshal each matcher in the set from its tokens
1007
- type matcherPair struct {
1008
- raw caddy.ModuleMap
1009
- decoded MatcherSet
1010
- }
1006
for d.Next() {
1012
- var mp matcherPair
1013
- matcherMap := make(map[string]RequestMatcher)
1014
-
1015
- // in case there are multiple instances of the same matcher, concatenate
1016
- // their tokens (we expect that UnmarshalCaddyfile should be able to
1017
- // handle more than one segment); otherwise, we'd overwrite other
1018
- // instances of the matcher in this set
1019
- tokensByMatcherName := make(map[string][]caddyfile.Token)
1020
- for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
1021
- matcherName := d.Val()
1022
- tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
1023
- }
1024
- for matcherName, tokens := range tokensByMatcherName {
1025
- mod, err := caddy.GetModule("http.matchers." + matcherName)
1026
- if err != nil {
1027
- return d.Errf("getting matcher module '%s': %v", matcherName, err)
1028
- }
1029
- unm, ok := mod.New().(caddyfile.Unmarshaler)
1030
- if !ok {
1031
- return d.Errf("matcher module '%s' is not a Caddyfile unmarshaler", matcherName)
1032
- }
1033
- err = unm.UnmarshalCaddyfile(caddyfile.NewDispenser(tokens))
1034
- if err != nil {
1035
- return err
1036
- }
1037
- rm, ok := unm.(RequestMatcher)
1038
- if !ok {
1039
- return fmt.Errorf("matcher module '%s' is not a request matcher", matcherName)
1040
- }
1041
- matcherMap[matcherName] = rm
1042
- mp.decoded = append(mp.decoded, rm)
1043
- }
1044
-
1045
- // we should now have a functional 'not' matcher, but we also
1046
- // need to be able to marshal as JSON, otherwise config
1047
- // adaptation will be missing the matchers!
1048
- mp.raw = make(caddy.ModuleMap)
1049
- for name, matcher := range matcherMap {
1050
- jsonBytes, err := json.Marshal(matcher)
1051
- if err != nil {
1052
- return fmt.Errorf("marshaling %T matcher: %v", matcher, err)
1053
- }
1054
- mp.raw[name] = jsonBytes
1007
+ matcherSet, err := ParseCaddyfileNestedMatcherSet(d)
1008
+ if err != nil {
1009
+ return err
1010
}
1056
- m.MatcherSetsRaw = append(m.MatcherSetsRaw, mp.raw)
1011
+ m.MatcherSetsRaw = append(m.MatcherSetsRaw, matcherSet)
1012
}
1013
return nil
1014
}
@@ -1352,6 +1307,56 @@ func (mre *MatchRegexp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1307
return nil
1308
}
1309
1310
+// ParseCaddyfileNestedMatcher parses the Caddyfile tokens for a nested
1311
+// matcher set, and returns its raw module map value.
1312
+func ParseCaddyfileNestedMatcherSet(d *caddyfile.Dispenser) (caddy.ModuleMap, error) {
1313
+ matcherMap := make(map[string]RequestMatcher)
1314
+
1315
+ // in case there are multiple instances of the same matcher, concatenate
1316
+ // their tokens (we expect that UnmarshalCaddyfile should be able to
1317
+ // handle more than one segment); otherwise, we'd overwrite other
1318
+ // instances of the matcher in this set
1319
+ tokensByMatcherName := make(map[string][]caddyfile.Token)
1320
+ for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
1321
+ matcherName := d.Val()
1322
+ tokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)
1323
+ }
1324
+
1325
+ for matcherName, tokens := range tokensByMatcherName {
1326
+ mod, err := caddy.GetModule("http.matchers." + matcherName)
1327
+ if err != nil {
1328
+ return nil, d.Errf("getting matcher module '%s': %v", matcherName, err)
1329
+ }
1330
+ unm, ok := mod.New().(caddyfile.Unmarshaler)
1331
+ if !ok {
1332
+ return nil, d.Errf("matcher module '%s' is not a Caddyfile unmarshaler", matcherName)
1333
+ }
1334
+ err = unm.UnmarshalCaddyfile(caddyfile.NewDispenser(tokens))
1335
+ if err != nil {
1336
+ return nil, err
1337
+ }
1338
+ rm, ok := unm.(RequestMatcher)
1339
+ if !ok {
1340
+ return nil, fmt.Errorf("matcher module '%s' is not a request matcher", matcherName)
1341
+ }
1342
+ matcherMap[matcherName] = rm
1343
+ }
1344
+
1345
+ // we should now have a functional matcher, but we also
1346
+ // need to be able to marshal as JSON, otherwise config
1347
+ // adaptation will be missing the matchers!
1348
+ matcherSet := make(caddy.ModuleMap)
1349
+ for name, matcher := range matcherMap {
1350
+ jsonBytes, err := json.Marshal(matcher)
1351
+ if err != nil {
1352
+ return nil, fmt.Errorf("marshaling %T matcher: %v", matcher, err)
1353
+ }
1354
+ matcherSet[name] = jsonBytes
1355
+ }
1356
+
1357
+ return matcherSet, nil
1358
+}
1359
+
1360
var (
1361
wordRE = regexp.MustCompile(`\w+`)
1362
)
modules/caddyhttp/reverseproxy/caddyfile.go
+25
@@ -59,8 +59,10 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
59
//
60
// # load balancing
61
// lb_policy <name> [<options...>]
62
+// lb_retries <retries>
63
// lb_try_duration <duration>
64
// lb_try_interval <interval>
65
+// lb_retry_match <request-matcher>
66
//
67
// # active health checking
68
// health_uri <uri>
@@ -247,6 +249,19 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
249
}
250
h.LoadBalancing.SelectionPolicyRaw = caddyconfig.JSONModuleObject(sel, "policy", name, nil)
251
252
+ case "lb_retries":
253
+ if !d.NextArg() {
254
+ return d.ArgErr()
255
+ }
256
+ tries, err := strconv.Atoi(d.Val())
257
+ if err != nil {
258
+ return d.Errf("bad lb_retries number '%s': %v", d.Val(), err)
259
+ }
260
+ if h.LoadBalancing == nil {
261
+ h.LoadBalancing = new(LoadBalancing)
262
+ }
263
+ h.LoadBalancing.Retries = tries
264
+
265
case "lb_try_duration":
266
if !d.NextArg() {
267
return d.ArgErr()
@@ -273,6 +288,16 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
288
}
289
h.LoadBalancing.TryInterval = caddy.Duration(dur)
290
291
+ case "lb_retry_match":
292
+ matcherSet, err := caddyhttp.ParseCaddyfileNestedMatcherSet(d)
293
+ if err != nil {
294
+ return d.Errf("failed to parse lb_retry_match: %v", err)
295
+ }
296
+ if h.LoadBalancing == nil {
297
+ h.LoadBalancing = new(LoadBalancing)
298
+ }
299
+ h.LoadBalancing.RetryMatchRaw = append(h.LoadBalancing.RetryMatchRaw, matcherSet)
300
+
301
case "health_uri":
302
if !d.NextArg() {
303
return d.ArgErr()
modules/caddyhttp/reverseproxy/reverseproxy.go
+46
-20
@@ -430,12 +430,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
430
// and because we may retry some number of times, carry over the error
431
// from previous tries because of the nuances of load balancing & retries
432
var proxyErr error
433
+ var retries int
434
for {
435
var done bool
435
- done, proxyErr = h.proxyLoopIteration(clonedReq, r, w, proxyErr, start, repl, reqHeader, reqHost, next)
436
+ done, proxyErr = h.proxyLoopIteration(clonedReq, r, w, proxyErr, start, retries, repl, reqHeader, reqHost, next)
437
if done {
438
break
439
}
440
+ retries++
441
}
442
443
if proxyErr != nil {
@@ -449,7 +451,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
451
// that has to be passed in, we brought this into its own method so that we could run defer more easily.
452
// It returns true when the loop is done and should break; false otherwise. The error value returned should
453
// be assigned to the proxyErr value for the next iteration of the loop (or the error handled after break).
452
-func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w http.ResponseWriter, proxyErr error, start time.Time,
454
+func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w http.ResponseWriter, proxyErr error, start time.Time, retries int,
455
repl *caddy.Replacer, reqHeader http.Header, reqHost string, next caddyhttp.Handler) (bool, error) {
456
// get the updated list of upstreams
457
upstreams := h.Upstreams
@@ -479,7 +481,7 @@ func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w h
481
if proxyErr == nil {
482
proxyErr = caddyhttp.Error(http.StatusServiceUnavailable, fmt.Errorf("no upstreams available"))
483
}
482
- if !h.LoadBalancing.tryAgain(h.ctx, start, proxyErr, r) {
484
+ if !h.LoadBalancing.tryAgain(h.ctx, start, retries, proxyErr, r) {
485
return true, proxyErr
486
}
487
return false, proxyErr
@@ -542,7 +544,7 @@ func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w h
544
h.countFailure(upstream)
545
546
// if we've tried long enough, break
545
- if !h.LoadBalancing.tryAgain(h.ctx, start, proxyErr, r) {
547
+ if !h.LoadBalancing.tryAgain(h.ctx, start, retries, proxyErr, r) {
548
return true, proxyErr
549
}
550
@@ -944,16 +946,26 @@ func (h Handler) finalizeResponse(
946
return nil
947
}
948
947
-// tryAgain takes the time that the handler was initially invoked
948
-// as well as any error currently obtained, and the request being
949
-// tried, and returns true if another attempt should be made at
950
-// proxying the request. If true is returned, it has already blocked
951
-// long enough before the next retry (i.e. no more sleeping is
952
-// needed). If false is returned, the handler should stop trying to
953
-// proxy the request.
954
-func (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, proxyErr error, req *http.Request) bool {
949
+// tryAgain takes the time that the handler was initially invoked,
950
+// the amount of retries already performed, as well as any error
951
+// currently obtained, and the request being tried, and returns
952
+// true if another attempt should be made at proxying the request.
953
+// If true is returned, it has already blocked long enough before
954
+// the next retry (i.e. no more sleeping is needed). If false is
955
+// returned, the handler should stop trying to proxy the request.
956
+func (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, retries int, proxyErr error, req *http.Request) bool {
957
+ // no retries are configured
958
+ if lb.TryDuration == 0 && lb.Retries == 0 {
959
+ return false
960
+ }
961
+
962
// if we've tried long enough, break
956
- if time.Since(start) >= time.Duration(lb.TryDuration) {
963
+ if lb.TryDuration > 0 && time.Since(start) >= time.Duration(lb.TryDuration) {
964
+ return false
965
+ }
966
+
967
+ // if we've reached the retry limit, break
968
+ if lb.Retries > 0 && retries >= lb.Retries {
969
return false
970
}
971
@@ -976,6 +988,11 @@ func (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, proxyErr er
988
}
989
}
990
991
+ // fast path; if the interval is zero, we don't need to wait
992
+ if lb.TryInterval == 0 {
993
+ return true
994
+ }
995
+
996
// otherwise, wait and try the next available host
997
timer := time.NewTimer(time.Duration(lb.TryInterval))
998
select {
@@ -1190,16 +1207,25 @@ type LoadBalancing struct {
1207
// The default policy is random selection.
1208
SelectionPolicyRaw json.RawMessage `json:"selection_policy,omitempty" caddy:"namespace=http.reverse_proxy.selection_policies inline_key=policy"`
1209
1210
+ // How many times to retry selecting available backends for each
1211
+ // request if the next available host is down. If try_duration is
1212
+ // also configured, then retries may stop early if the duration
1213
+ // is reached. By default, retries are disabled (zero).
1214
+ Retries int `json:"retries,omitempty"`
1215
+
1216
// How long to try selecting available backends for each request
1194
- // if the next available host is down. By default, this retry is
1195
- // disabled. Clients will wait for up to this long while the load
1196
- // balancer tries to find an available upstream host.
1217
+ // if the next available host is down. Clients will wait for up
1218
+ // to this long while the load balancer tries to find an available
1219
+ // upstream host. If retries is also configured, tries may stop
1220
+ // early if the maximum retries is reached. By default, retries
1221
+ // are disabled (zero duration).
1222
TryDuration caddy.Duration `json:"try_duration,omitempty"`
1223
1199
- // How long to wait between selecting the next host from the pool. Default
1200
- // is 250ms. Only relevant when a request to an upstream host fails. Be
1201
- // aware that setting this to 0 with a non-zero try_duration can cause the
1202
- // CPU to spin if all backends are down and latency is very low.
1224
+ // How long to wait between selecting the next host from the pool.
1225
+ // Default is 250ms if try_duration is enabled, otherwise zero. Only
1226
+ // relevant when a request to an upstream host fails. Be aware that
1227
+ // setting this to 0 with a non-zero try_duration can cause the CPU
1228
+ // to spin if all backends are down and latency is very low.
1229
TryInterval caddy.Duration `json:"try_interval,omitempty"`
1230
1231
// A list of matcher sets that restricts with which requests retries are