@marcel91200 / caddy Mirror · read-only / commits / ab045592
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 13, 21:45 UTC

reverseproxy: Dynamic upstreams (with SRV and A/AAAA support) (#4470)

* reverseproxy: Begin refactor to enable dynamic upstreams Streamed here: https://www.youtube.com/watch?v=hj7yzXb11jU * Implement SRV and A/AAA upstream sources Also get upstreams at every retry loop iteration instead of just once before the loop. See #4442. * Minor tweaks from review * Limit size of upstreams caches * Add doc notes deprecating LookupSRV * Provision dynamic upstreams Still WIP, preparing to preserve health checker functionality * Rejigger health checks Move active health check results into handler-specific Upstreams. Improve documentation regarding health checks and upstreams. * Deprecation notice * Add Caddyfile support, use `caddy.Duration` * Interface guards * Implement custom resolvers, add resolvers to http transport Caddyfile * SRV: fix Caddyfile `name` inline arg, remove proto condition * Use pointer receiver * Add debug logs Co-authored-by: Francis Lavoie <lavofr@gmail.com>

Matt Holt committed Mar 6, 2022 at 17:43 UTC ab0455922ae01bde1a7a5b3bf58eb993efc02db7
10 files changed +1056 -281
caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt new
+116
@@ -0,0 +1,116 @@
1 +:8884 {
2 + reverse_proxy {
3 + dynamic a foo 9000
4 + }
5 +
6 + reverse_proxy {
7 + dynamic a {
8 + name foo
9 + port 9000
10 + refresh 5m
11 + resolvers 8.8.8.8 8.8.4.4
12 + dial_timeout 2s
13 + dial_fallback_delay 300ms
14 + }
15 + }
16 +}
17 +
18 +:8885 {
19 + reverse_proxy {
20 + dynamic srv _api._tcp.example.com
21 + }
22 +
23 + reverse_proxy {
24 + dynamic srv {
25 + service api
26 + proto tcp
27 + name example.com
28 + refresh 5m
29 + resolvers 8.8.8.8 8.8.4.4
30 + dial_timeout 1s
31 + dial_fallback_delay -1s
32 + }
33 + }
34 +}
35 +
36 +----------
37 +{
38 + "apps": {
39 + "http": {
40 + "servers": {
41 + "srv0": {
42 + "listen": [
43 + ":8884"
44 + ],
45 + "routes": [
46 + {
47 + "handle": [
48 + {
49 + "dynamic_upstreams": {
50 + "name": "foo",
51 + "port": "9000",
52 + "source": "a"
53 + },
54 + "handler": "reverse_proxy"
55 + },
56 + {
57 + "dynamic_upstreams": {
58 + "dial_fallback_delay": 300000000,
59 + "dial_timeout": 2000000000,
60 + "name": "foo",
61 + "port": "9000",
62 + "refresh": 300000000000,
63 + "resolver": {
64 + "addresses": [
65 + "8.8.8.8",
66 + "8.8.4.4"
67 + ]
68 + },
69 + "source": "a"
70 + },
71 + "handler": "reverse_proxy"
72 + }
73 + ]
74 + }
75 + ]
76 + },
77 + "srv1": {
78 + "listen": [
79 + ":8885"
80 + ],
81 + "routes": [
82 + {
83 + "handle": [
84 + {
85 + "dynamic_upstreams": {
86 + "name": "_api._tcp.example.com",
87 + "source": "srv"
88 + },
89 + "handler": "reverse_proxy"
90 + },
91 + {
92 + "dynamic_upstreams": {
93 + "dial_fallback_delay": -1000000000,
94 + "dial_timeout": 1000000000,
95 + "name": "example.com",
96 + "proto": "tcp",
97 + "refresh": 300000000000,
98 + "resolver": {
99 + "addresses": [
100 + "8.8.8.8",
101 + "8.8.4.4"
102 + ]
103 + },
104 + "service": "api",
105 + "source": "srv"
106 + },
107 + "handler": "reverse_proxy"
108 + }
109 + ]
110 + }
111 + ]
112 + }
113 + }
114 + }
115 + }
116 +}
\ No newline at end of file
caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt
+7
@@ -17,6 +17,7 @@ https://example.com {
17 dial_fallback_delay 5s
18 response_header_timeout 8s
19 expect_continue_timeout 9s
20 + resolvers 8.8.8.8 8.8.4.4
21
22 versions h2c 2
23 compression off
@@ -88,6 +89,12 @@ https://example.com {
89 "max_response_header_size": 30000000,
90 "protocol": "http",
91 "read_buffer_size": 10000000,
92 + "resolver": {
93 + "addresses": [
94 + "8.8.8.8",
95 + "8.8.4.4"
96 + ]
97 + },
98 "response_header_timeout": 8000000000,
99 "versions": [
100 "h2c",
modules/caddyhttp/reverseproxy/admin.go
+1 -2
@@ -87,7 +87,7 @@ func (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) er
87 return false
88 }
89
90 - upstream, ok := val.(*upstreamHost)
90 + upstream, ok := val.(*Host)
91 if !ok {
92 rangeErr = caddy.APIError{
93 HTTPStatus: http.StatusInternalServerError,
@@ -98,7 +98,6 @@ func (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) er
98
99 results = append(results, upstreamStatus{
100 Address: address,
101 - Healthy: !upstream.Unhealthy(),
101 NumRequests: upstream.NumRequests(),
102 Fails: upstream.Fails(),
103 })
modules/caddyhttp/reverseproxy/caddyfile.go
+221
@@ -53,6 +53,7 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
53 // reverse_proxy [<matcher>] [<upstreams...>] {
54 // # upstreams
55 // to <upstreams...>
56 +// dynamic <name> [...]
57 //
58 // # load balancing
59 // lb_policy <name> [<options...>]
@@ -190,6 +191,25 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
191 }
192 }
193
194 + case "dynamic":
195 + if !d.NextArg() {
196 + return d.ArgErr()
197 + }
198 + if h.DynamicUpstreams != nil {
199 + return d.Err("dynamic upstreams already specified")
200 + }
201 + dynModule := d.Val()
202 + modID := "http.reverse_proxy.upstreams." + dynModule
203 + unm, err := caddyfile.UnmarshalModule(d, modID)
204 + if err != nil {
205 + return err
206 + }
207 + source, ok := unm.(UpstreamSource)
208 + if !ok {
209 + return d.Errf("module %s (%T) is not an UpstreamSource", modID, unm)
210 + }
211 + h.DynamicUpstreamsRaw = caddyconfig.JSONModuleObject(source, "source", dynModule, nil)
212 +
213 case "lb_policy":
214 if !d.NextArg() {
215 return d.ArgErr()
@@ -749,6 +769,7 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error
769 // dial_fallback_delay <duration>
770 // response_header_timeout <duration>
771 // expect_continue_timeout <duration>
772 +// resolvers <resolvers...>
773 // tls
774 // tls_client_auth <automate_name> | <cert_file> <key_file>
775 // tls_insecure_skip_verify
@@ -839,6 +860,15 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
860 }
861 h.ExpectContinueTimeout = caddy.Duration(dur)
862
863 + case "resolvers":
864 + if h.Resolver == nil {
865 + h.Resolver = new(UpstreamResolver)
866 + }
867 + h.Resolver.Addresses = d.RemainingArgs()
868 + if len(h.Resolver.Addresses) == 0 {
869 + return d.Errf("must specify at least one resolver address")
870 + }
871 +
872 case "tls_client_auth":
873 if h.TLS == nil {
874 h.TLS = new(TLSConfig)
@@ -989,10 +1019,201 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1019 return nil
1020 }
1021
1022 +// UnmarshalCaddyfile deserializes Caddyfile tokens into h.
1023 +//
1024 +// dynamic srv [<name>] {
1025 +// service <service>
1026 +// proto <proto>
1027 +// name <name>
1028 +// refresh <interval>
1029 +// resolvers <resolvers...>
1030 +// dial_timeout <timeout>
1031 +// dial_fallback_delay <timeout>
1032 +// }
1033 +//
1034 +func (u *SRVUpstreams) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1035 + for d.Next() {
1036 + args := d.RemainingArgs()
1037 + if len(args) > 1 {
1038 + return d.ArgErr()
1039 + }
1040 + if len(args) > 0 {
1041 + u.Name = args[0]
1042 + }
1043 +
1044 + for d.NextBlock(0) {
1045 + switch d.Val() {
1046 + case "service":
1047 + if !d.NextArg() {
1048 + return d.ArgErr()
1049 + }
1050 + if u.Service != "" {
1051 + return d.Errf("srv service has already been specified")
1052 + }
1053 + u.Service = d.Val()
1054 +
1055 + case "proto":
1056 + if !d.NextArg() {
1057 + return d.ArgErr()
1058 + }
1059 + if u.Proto != "" {
1060 + return d.Errf("srv proto has already been specified")
1061 + }
1062 + u.Proto = d.Val()
1063 +
1064 + case "name":
1065 + if !d.NextArg() {
1066 + return d.ArgErr()
1067 + }
1068 + if u.Name != "" {
1069 + return d.Errf("srv name has already been specified")
1070 + }
1071 + u.Name = d.Val()
1072 +
1073 + case "refresh":
1074 + if !d.NextArg() {
1075 + return d.ArgErr()
1076 + }
1077 + dur, err := caddy.ParseDuration(d.Val())
1078 + if err != nil {
1079 + return d.Errf("parsing refresh interval duration: %v", err)
1080 + }
1081 + u.Refresh = caddy.Duration(dur)
1082 +
1083 + case "resolvers":
1084 + if u.Resolver == nil {
1085 + u.Resolver = new(UpstreamResolver)
1086 + }
1087 + u.Resolver.Addresses = d.RemainingArgs()
1088 + if len(u.Resolver.Addresses) == 0 {
1089 + return d.Errf("must specify at least one resolver address")
1090 + }
1091 +
1092 + case "dial_timeout":
1093 + if !d.NextArg() {
1094 + return d.ArgErr()
1095 + }
1096 + dur, err := caddy.ParseDuration(d.Val())
1097 + if err != nil {
1098 + return d.Errf("bad timeout value '%s': %v", d.Val(), err)
1099 + }
1100 + u.DialTimeout = caddy.Duration(dur)
1101 +
1102 + case "dial_fallback_delay":
1103 + if !d.NextArg() {
1104 + return d.ArgErr()
1105 + }
1106 + dur, err := caddy.ParseDuration(d.Val())
1107 + if err != nil {
1108 + return d.Errf("bad delay value '%s': %v", d.Val(), err)
1109 + }
1110 + u.FallbackDelay = caddy.Duration(dur)
1111 +
1112 + default:
1113 + return d.Errf("unrecognized srv option '%s'", d.Val())
1114 + }
1115 + }
1116 + }
1117 +
1118 + return nil
1119 +}
1120 +
1121 +// UnmarshalCaddyfile deserializes Caddyfile tokens into h.
1122 +//
1123 +// dynamic a [<name> <port] {
1124 +// name <name>
1125 +// port <port>
1126 +// refresh <interval>
1127 +// resolvers <resolvers...>
1128 +// dial_timeout <timeout>
1129 +// dial_fallback_delay <timeout>
1130 +// }
1131 +//
1132 +func (u *AUpstreams) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1133 + for d.Next() {
1134 + args := d.RemainingArgs()
1135 + if len(args) > 2 {
1136 + return d.ArgErr()
1137 + }
1138 + if len(args) > 0 {
1139 + u.Name = args[0]
1140 + u.Port = args[1]
1141 + }
1142 +
1143 + for d.NextBlock(0) {
1144 + switch d.Val() {
1145 + case "name":
1146 + if !d.NextArg() {
1147 + return d.ArgErr()
1148 + }
1149 + if u.Name != "" {
1150 + return d.Errf("a name has already been specified")
1151 + }
1152 + u.Name = d.Val()
1153 +
1154 + case "port":
1155 + if !d.NextArg() {
1156 + return d.ArgErr()
1157 + }
1158 + if u.Port != "" {
1159 + return d.Errf("a port has already been specified")
1160 + }
1161 + u.Port = d.Val()
1162 +
1163 + case "refresh":
1164 + if !d.NextArg() {
1165 + return d.ArgErr()
1166 + }
1167 + dur, err := caddy.ParseDuration(d.Val())
1168 + if err != nil {
1169 + return d.Errf("parsing refresh interval duration: %v", err)
1170 + }
1171 + u.Refresh = caddy.Duration(dur)
1172 +
1173 + case "resolvers":
1174 + if u.Resolver == nil {
1175 + u.Resolver = new(UpstreamResolver)
1176 + }
1177 + u.Resolver.Addresses = d.RemainingArgs()
1178 + if len(u.Resolver.Addresses) == 0 {
1179 + return d.Errf("must specify at least one resolver address")
1180 + }
1181 +
1182 + case "dial_timeout":
1183 + if !d.NextArg() {
1184 + return d.ArgErr()
1185 + }
1186 + dur, err := caddy.ParseDuration(d.Val())
1187 + if err != nil {
1188 + return d.Errf("bad timeout value '%s': %v", d.Val(), err)
1189 + }
1190 + u.DialTimeout = caddy.Duration(dur)
1191 +
1192 + case "dial_fallback_delay":
1193 + if !d.NextArg() {
1194 + return d.ArgErr()
1195 + }
1196 + dur, err := caddy.ParseDuration(d.Val())
1197 + if err != nil {
1198 + return d.Errf("bad delay value '%s': %v", d.Val(), err)
1199 + }
1200 + u.FallbackDelay = caddy.Duration(dur)
1201 +
1202 + default:
1203 + return d.Errf("unrecognized srv option '%s'", d.Val())
1204 + }
1205 + }
1206 + }
1207 +
1208 + return nil
1209 +}
1210 +
1211 const matcherPrefix = "@"
1212
1213 // Interface guards
1214 var (
1215 _ caddyfile.Unmarshaler = (*Handler)(nil)
1216 _ caddyfile.Unmarshaler = (*HTTPTransport)(nil)
1217 + _ caddyfile.Unmarshaler = (*SRVUpstreams)(nil)
1218 + _ caddyfile.Unmarshaler = (*AUpstreams)(nil)
1219 )
modules/caddyhttp/reverseproxy/healthchecks.go
+53 -43
@@ -18,7 +18,6 @@ import (
18 "context"
19 "fmt"
20 "io"
21 - "log"
21 "net"
22 "net/http"
23 "net/url"
@@ -37,12 +36,32 @@ import (
36 type HealthChecks struct {
37 // Active health checks run in the background on a timer. To
38 // minimally enable active health checks, set either path or
40 - // port (or both).
39 + // port (or both). Note that active health check status
40 + // (healthy/unhealthy) is stored per-proxy-handler, not
41 + // globally; this allows different handlers to use different
42 + // criteria to decide what defines a healthy backend.
43 + //
44 + // Active health checks do not run for dynamic upstreams.
45 Active *ActiveHealthChecks `json:"active,omitempty"`
46
47 // Passive health checks monitor proxied requests for errors or timeouts.
48 // To minimally enable passive health checks, specify at least an empty
45 - // config object.
49 + // config object. Passive health check state is shared (stored globally),
50 + // so a failure from one handler will be counted by all handlers; but
51 + // the tolerances or standards for what defines healthy/unhealthy backends
52 + // is configured per-proxy-handler.
53 + //
54 + // Passive health checks technically do operate on dynamic upstreams,
55 + // but are only effective for very busy proxies where the list of
56 + // upstreams is mostly stable. This is because the shared/global
57 + // state of upstreams is cleaned up when the upstreams are no longer
58 + // used. Since dynamic upstreams are allocated dynamically at each
59 + // request (specifically, each iteration of the proxy loop per request),
60 + // they are also cleaned up after every request. Thus, if there is a
61 + // moment when no requests are actively referring to a particular
62 + // upstream host, the passive health check state will be reset because
63 + // it will be garbage-collected. It is usually better for the dynamic
64 + // upstream module to only return healthy, available backends instead.
65 Passive *PassiveHealthChecks `json:"passive,omitempty"`
66 }
67
@@ -50,8 +69,7 @@ type HealthChecks struct {
69 // health checks (that is, health checks which occur in a
70 // background goroutine independently).
71 type ActiveHealthChecks struct {
53 - // The path to use for health checks.
54 - // DEPRECATED: Use 'uri' instead.
72 + // DEPRECATED: Use 'uri' instead. This field will be removed. TODO: remove this field
73 Path string `json:"path,omitempty"`
74
75 // The URI (path and query) to use for health checks
@@ -132,7 +150,9 @@ type CircuitBreaker interface {
150 func (h *Handler) activeHealthChecker() {
151 defer func() {
152 if err := recover(); err != nil {
135 - log.Printf("[PANIC] active health checks: %v\n%s", err, debug.Stack())
153 + h.HealthChecks.Active.logger.Error("active health checker panicked",
154 + zap.Any("error", err),
155 + zap.ByteString("stack", debug.Stack()))
156 }
157 }()
158 ticker := time.NewTicker(time.Duration(h.HealthChecks.Active.Interval))
@@ -155,7 +175,9 @@ func (h *Handler) doActiveHealthCheckForAllHosts() {
175 go func(upstream *Upstream) {
176 defer func() {
177 if err := recover(); err != nil {
158 - log.Printf("[PANIC] active health check: %v\n%s", err, debug.Stack())
178 + h.HealthChecks.Active.logger.Error("active health check panicked",
179 + zap.Any("error", err),
180 + zap.ByteString("stack", debug.Stack()))
181 }
182 }()
183
@@ -195,7 +217,7 @@ func (h *Handler) doActiveHealthCheckForAllHosts() {
217 // so use a fake Host value instead; unix sockets are usually local
218 hostAddr = "localhost"
219 }
198 - err = h.doActiveHealthCheck(DialInfo{Network: addr.Network, Address: dialAddr}, hostAddr, upstream.Host)
220 + err = h.doActiveHealthCheck(DialInfo{Network: addr.Network, Address: dialAddr}, hostAddr, upstream)
221 if err != nil {
222 h.HealthChecks.Active.logger.Error("active health check failed",
223 zap.String("address", hostAddr),
@@ -206,14 +228,14 @@ func (h *Handler) doActiveHealthCheckForAllHosts() {
228 }
229 }
230
209 -// doActiveHealthCheck performs a health check to host which
231 +// doActiveHealthCheck performs a health check to upstream which
232 // can be reached at address hostAddr. The actual address for
233 // the request will be built according to active health checker
234 // config. The health status of the host will be updated
235 // according to whether it passes the health check. An error is
236 // returned only if the health check fails to occur or if marking
237 // the host's health status fails.
216 -func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, host Host) error {
238 +func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, upstream *Upstream) error {
239 // create the URL for the request that acts as a health check
240 scheme := "http"
241 if ht, ok := h.Transport.(TLSTransport); ok && ht.TLSEnabled() {
@@ -269,10 +291,7 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, host H
291 zap.String("host", hostAddr),
292 zap.Error(err),
293 )
272 - _, err2 := host.SetHealthy(false)
273 - if err2 != nil {
274 - return fmt.Errorf("marking unhealthy: %v", err2)
275 - }
294 + upstream.setHealthy(false)
295 return nil
296 }
297 var body io.Reader = resp.Body
@@ -292,10 +311,7 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, host H
311 zap.Int("status_code", resp.StatusCode),
312 zap.String("host", hostAddr),
313 )
295 - _, err := host.SetHealthy(false)
296 - if err != nil {
297 - return fmt.Errorf("marking unhealthy: %v", err)
298 - }
314 + upstream.setHealthy(false)
315 return nil
316 }
317 } else if resp.StatusCode < 200 || resp.StatusCode >= 400 {
@@ -303,10 +319,7 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, host H
319 zap.Int("status_code", resp.StatusCode),
320 zap.String("host", hostAddr),
321 )
306 - _, err := host.SetHealthy(false)
307 - if err != nil {
308 - return fmt.Errorf("marking unhealthy: %v", err)
309 - }
322 + upstream.setHealthy(false)
323 return nil
324 }
325
@@ -318,33 +331,21 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, host H
331 zap.String("host", hostAddr),
332 zap.Error(err),
333 )
321 - _, err := host.SetHealthy(false)
322 - if err != nil {
323 - return fmt.Errorf("marking unhealthy: %v", err)
324 - }
334 + upstream.setHealthy(false)
335 return nil
336 }
337 if !h.HealthChecks.Active.bodyRegexp.Match(bodyBytes) {
338 h.HealthChecks.Active.logger.Info("response body failed expectations",
339 zap.String("host", hostAddr),
340 )
331 - _, err := host.SetHealthy(false)
332 - if err != nil {
333 - return fmt.Errorf("marking unhealthy: %v", err)
334 - }
341 + upstream.setHealthy(false)
342 return nil
343 }
344 }
345
346 // passed health check parameters, so mark as healthy
340 - swapped, err := host.SetHealthy(true)
341 - if swapped {
342 - h.HealthChecks.Active.logger.Info("host is up",
343 - zap.String("host", hostAddr),
344 - )
345 - }
346 - if err != nil {
347 - return fmt.Errorf("marking healthy: %v", err)
347 + if upstream.setHealthy(true) {
348 + h.HealthChecks.Active.logger.Info("host is up", zap.String("host", hostAddr))
349 }
350
351 return nil
@@ -366,7 +367,7 @@ func (h *Handler) countFailure(upstream *Upstream) {
367 }
368
369 // count failure immediately
369 - err := upstream.Host.CountFail(1)
370 + err := upstream.Host.countFail(1)
371 if err != nil {
372 h.HealthChecks.Passive.logger.Error("could not count failure",
373 zap.String("host", upstream.Dial),
@@ -375,14 +376,23 @@ func (h *Handler) countFailure(upstream *Upstream) {
376 }
377
378 // forget it later
378 - go func(host Host, failDuration time.Duration) {
379 + go func(host *Host, failDuration time.Duration) {
380 defer func() {
381 if err := recover(); err != nil {
381 - log.Printf("[PANIC] health check failure forgetter: %v\n%s", err, debug.Stack())
382 + h.HealthChecks.Passive.logger.Error("passive health check failure forgetter panicked",
383 + zap.Any("error", err),
384 + zap.ByteString("stack", debug.Stack()))
385 }
386 }()
384 - time.Sleep(failDuration)
385 - err := host.CountFail(-1)
387 + timer := time.NewTimer(failDuration)
388 + select {
389 + case <-h.ctx.Done():
390 + if !timer.Stop() {
391 + <-timer.C
392 + }
393 + case <-timer.C:
394 + }
395 + err := host.countFail(-1)
396 if err != nil {
397 h.HealthChecks.Passive.logger.Error("could not forget failure",
398 zap.String("host", upstream.Dial),
modules/caddyhttp/reverseproxy/hosts.go
+41 -58
@@ -26,44 +26,14 @@ import (
26 "github.com/caddyserver/caddy/v2/modules/caddyhttp"
27 )
28
29 -// Host represents a remote host which can be proxied to.
30 -// Its methods must be safe for concurrent use.
31 -type Host interface {
32 - // NumRequests returns the number of requests
33 - // currently in process with the host.
34 - NumRequests() int
35 -
36 - // Fails returns the count of recent failures.
37 - Fails() int
38 -
39 - // Unhealthy returns true if the backend is unhealthy.
40 - Unhealthy() bool
41 -
42 - // CountRequest atomically counts the given number of
43 - // requests as currently in process with the host. The
44 - // count should not go below 0.
45 - CountRequest(int) error
46 -
47 - // CountFail atomically counts the given number of
48 - // failures with the host. The count should not go
49 - // below 0.
50 - CountFail(int) error
51 -
52 - // SetHealthy atomically marks the host as either
53 - // healthy (true) or unhealthy (false). If the given
54 - // status is the same, this should be a no-op and
55 - // return false. It returns true if the status was
56 - // changed; i.e. if it is now different from before.
57 - SetHealthy(bool) (bool, error)
58 -}
59 -
29 // UpstreamPool is a collection of upstreams.
30 type UpstreamPool []*Upstream
31
32 // Upstream bridges this proxy's configuration to the
33 // state of the backend host it is correlated with.
34 +// Upstream values must not be copied.
35 type Upstream struct {
66 - Host `json:"-"`
36 + *Host `json:"-"`
37
38 // The [network address](/docs/conventions#network-addresses)
39 // to dial to connect to the upstream. Must represent precisely
@@ -77,6 +47,10 @@ type Upstream struct {
47 // backends is down. Also be aware of open proxy vulnerabilities.
48 Dial string `json:"dial,omitempty"`
49
50 + // DEPRECATED: Use the SRVUpstreams module instead
51 + // (http.reverse_proxy.upstreams.srv). This field will be
52 + // removed in a future version of Caddy. TODO: Remove this field.
53 + //
54 // If DNS SRV records are used for service discovery with this
55 // upstream, specify the DNS name for which to look up SRV
56 // records here, instead of specifying a dial address.
@@ -95,6 +69,7 @@ type Upstream struct {
69 activeHealthCheckPort int
70 healthCheckPolicy *PassiveHealthChecks
71 cb CircuitBreaker
72 + unhealthy int32 // accessed atomically; status from active health checker
73 }
74
75 func (u Upstream) String() string {
@@ -117,7 +92,7 @@ func (u *Upstream) Available() bool {
92 // is currently known to be healthy or "up".
93 // It consults the circuit breaker, if any.
94 func (u *Upstream) Healthy() bool {
120 - healthy := !u.Host.Unhealthy()
95 + healthy := u.healthy()
96 if healthy && u.healthCheckPolicy != nil {
97 healthy = u.Host.Fails() < u.healthCheckPolicy.MaxFails
98 }
@@ -142,7 +117,7 @@ func (u *Upstream) fillDialInfo(r *http.Request) (DialInfo, error) {
117 var addr caddy.NetworkAddress
118
119 if u.LookupSRV != "" {
145 - // perform DNS lookup for SRV records and choose one
120 + // perform DNS lookup for SRV records and choose one - TODO: deprecated
121 srvName := repl.ReplaceAll(u.LookupSRV, "")
122 _, records, err := net.DefaultResolver.LookupSRV(r.Context(), "", "", srvName)
123 if err != nil {
@@ -174,59 +149,67 @@ func (u *Upstream) fillDialInfo(r *http.Request) (DialInfo, error) {
149 }, nil
150 }
151
177 -// upstreamHost is the basic, in-memory representation
178 -// of the state of a remote host. It implements the
179 -// Host interface.
180 -type upstreamHost struct {
152 +func (u *Upstream) fillHost() {
153 + host := new(Host)
154 + existingHost, loaded := hosts.LoadOrStore(u.String(), host)
155 + if loaded {
156 + host = existingHost.(*Host)
157 + }
158 + u.Host = host
159 +}
160 +
161 +// Host is the basic, in-memory representation of the state of a remote host.
162 +// Its fields are accessed atomically and Host values must not be copied.
163 +type Host struct {
164 numRequests int64 // must be 64-bit aligned on 32-bit systems (see https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
165 fails int64
183 - unhealthy int32
166 }
167
168 // NumRequests returns the number of active requests to the upstream.
187 -func (uh *upstreamHost) NumRequests() int {
188 - return int(atomic.LoadInt64(&uh.numRequests))
169 +func (h *Host) NumRequests() int {
170 + return int(atomic.LoadInt64(&h.numRequests))
171 }
172
173 // Fails returns the number of recent failures with the upstream.
192 -func (uh *upstreamHost) Fails() int {
193 - return int(atomic.LoadInt64(&uh.fails))
194 -}
195 -
196 -// Unhealthy returns whether the upstream is healthy.
197 -func (uh *upstreamHost) Unhealthy() bool {
198 - return atomic.LoadInt32(&uh.unhealthy) == 1
174 +func (h *Host) Fails() int {
175 + return int(atomic.LoadInt64(&h.fails))
176 }
177
201 -// CountRequest mutates the active request count by
178 +// countRequest mutates the active request count by
179 // delta. It returns an error if the adjustment fails.
203 -func (uh *upstreamHost) CountRequest(delta int) error {
204 - result := atomic.AddInt64(&uh.numRequests, int64(delta))
180 +func (h *Host) countRequest(delta int) error {
181 + result := atomic.AddInt64(&h.numRequests, int64(delta))
182 if result < 0 {
183 return fmt.Errorf("count below 0: %d", result)
184 }
185 return nil
186 }
187
211 -// CountFail mutates the recent failures count by
188 +// countFail mutates the recent failures count by
189 // delta. It returns an error if the adjustment fails.
213 -func (uh *upstreamHost) CountFail(delta int) error {
214 - result := atomic.AddInt64(&uh.fails, int64(delta))
190 +func (h *Host) countFail(delta int) error {
191 + result := atomic.AddInt64(&h.fails, int64(delta))
192 if result < 0 {
193 return fmt.Errorf("count below 0: %d", result)
194 }
195 return nil
196 }
197
198 +// healthy returns true if the upstream is not actively marked as unhealthy.
199 +// (This returns the status only from the "active" health checks.)
200 +func (u *Upstream) healthy() bool {
201 + return atomic.LoadInt32(&u.unhealthy) == 0
202 +}
203 +
204 // SetHealthy sets the upstream has healthy or unhealthy
222 -// and returns true if the new value is different.
223 -func (uh *upstreamHost) SetHealthy(healthy bool) (bool, error) {
205 +// and returns true if the new value is different. This
206 +// sets the status only for the "active" health checks.
207 +func (u *Upstream) setHealthy(healthy bool) bool {
208 var unhealthy, compare int32 = 1, 0
209 if healthy {
210 unhealthy, compare = 0, 1
211 }
228 - swapped := atomic.CompareAndSwapInt32(&uh.unhealthy, compare, unhealthy)
229 - return swapped, nil
212 + return atomic.CompareAndSwapInt32(&u.unhealthy, compare, unhealthy)
213 }
214
215 // DialInfo contains information needed to dial a
modules/caddyhttp/reverseproxy/httptransport.go
+3 -21
@@ -168,15 +168,9 @@ func (h *HTTPTransport) NewTransport(ctx caddy.Context) (*http.Transport, error)
168 }
169
170 if h.Resolver != nil {
171 - for _, v := range h.Resolver.Addresses {
172 - addr, err := caddy.ParseNetworkAddress(v)
173 - if err != nil {
174 - return nil, err
175 - }
176 - if addr.PortRangeSize() != 1 {
177 - return nil, fmt.Errorf("resolver address must have exactly one address; cannot call %v", addr)
178 - }
179 - h.Resolver.netAddrs = append(h.Resolver.netAddrs, addr)
171 + err := h.Resolver.ParseAddresses()
172 + if err != nil {
173 + return nil, err
174 }
175 d := &net.Dialer{
176 Timeout: time.Duration(h.DialTimeout),
@@ -406,18 +400,6 @@ func (t TLSConfig) MakeTLSClientConfig(ctx caddy.Context) (*tls.Config, error) {
400 return cfg, nil
401 }
402
409 -// UpstreamResolver holds the set of addresses of DNS resolvers of
410 -// upstream addresses
411 -type UpstreamResolver struct {
412 - // The addresses of DNS resolvers to use when looking up the addresses of proxy upstreams.
413 - // It accepts [network addresses](/docs/conventions#network-addresses)
414 - // with port range of only 1. If the host is an IP address, it will be dialed directly to resolve the upstream server.
415 - // If the host is not an IP address, the addresses are resolved using the [name resolution convention](https://golang.org/pkg/net/#hdr-Name_Resolution) of the Go standard library.
416 - // If the array contains more than 1 resolver address, one is chosen at random.
417 - Addresses []string `json:"addresses,omitempty"`
418 - netAddrs []caddy.NetworkAddress
419 -}
420 -
403 // KeepAlive holds configuration pertaining to HTTP Keep-Alive.
404 type KeepAlive struct {
405 // Whether HTTP Keep-Alive is enabled. Default: true
modules/caddyhttp/reverseproxy/reverseproxy.go
+177 -97
@@ -78,9 +78,20 @@ type Handler struct {
78 // up or down. Down backends will not be proxied to.
79 HealthChecks *HealthChecks `json:"health_checks,omitempty"`
80
81 - // Upstreams is the list of backends to proxy to.
81 + // Upstreams is the static list of backends to proxy to.
82 Upstreams UpstreamPool `json:"upstreams,omitempty"`
83
84 + // A module for retrieving the list of upstreams dynamically. Dynamic
85 + // upstreams are retrieved at every iteration of the proxy loop for
86 + // each request (i.e. before every proxy attempt within every request).
87 + // Active health checks do not work on dynamic upstreams, and passive
88 + // health checks are only effective on dynamic upstreams if the proxy
89 + // server is busy enough that concurrent requests to the same backends
90 + // are continuous. Instead of health checks for dynamic upstreams, it
91 + // is recommended that the dynamic upstream module only return available
92 + // backends in the first place.
93 + DynamicUpstreamsRaw json.RawMessage `json:"dynamic_upstreams,omitempty" caddy:"namespace=http.reverse_proxy.upstreams inline_key=source"`
94 +
95 // Adjusts how often to flush the response buffer. By default,
96 // no periodic flushing is done. A negative value disables
97 // response buffering, and flushes immediately after each
@@ -137,8 +148,9 @@ type Handler struct {
148 // - `{http.reverse_proxy.header.*}` The headers from the response
149 HandleResponse []caddyhttp.ResponseHandler `json:"handle_response,omitempty"`
150
140 - Transport http.RoundTripper `json:"-"`
141 - CB CircuitBreaker `json:"-"`
151 + Transport http.RoundTripper `json:"-"`
152 + CB CircuitBreaker `json:"-"`
153 + DynamicUpstreams UpstreamSource `json:"-"`
154
155 // Holds the parsed CIDR ranges from TrustedProxies
156 trustedProxies []*net.IPNet
@@ -166,7 +178,7 @@ func (h *Handler) Provision(ctx caddy.Context) error {
178 h.ctx = ctx
179 h.logger = ctx.Logger(h)
180
169 - // verify SRV compatibility
181 + // verify SRV compatibility - TODO: LookupSRV deprecated; will be removed
182 for i, v := range h.Upstreams {
183 if v.LookupSRV == "" {
184 continue
@@ -201,6 +213,13 @@ func (h *Handler) Provision(ctx caddy.Context) error {
213 }
214 h.CB = mod.(CircuitBreaker)
215 }
216 + if h.DynamicUpstreamsRaw != nil {
217 + mod, err := ctx.LoadModule(h, "DynamicUpstreamsRaw")
218 + if err != nil {
219 + return fmt.Errorf("loading upstream source module: %v", err)
220 + }
221 + h.DynamicUpstreams = mod.(UpstreamSource)
222 + }
223
224 // parse trusted proxy CIDRs ahead of time
225 for _, str := range h.TrustedProxies {
@@ -270,38 +289,8 @@ func (h *Handler) Provision(ctx caddy.Context) error {
289 }
290
291 // set up upstreams
273 - for _, upstream := range h.Upstreams {
274 - // create or get the host representation for this upstream
275 - var host Host = new(upstreamHost)
276 - existingHost, loaded := hosts.LoadOrStore(upstream.String(), host)
277 - if loaded {
278 - host = existingHost.(Host)
279 - }
280 - upstream.Host = host
281 -
282 - // give it the circuit breaker, if any
283 - upstream.cb = h.CB
284 -
285 - // if the passive health checker has a non-zero UnhealthyRequestCount
286 - // but the upstream has no MaxRequests set (they are the same thing,
287 - // but the passive health checker is a default value for for upstreams
288 - // without MaxRequests), copy the value into this upstream, since the
289 - // value in the upstream (MaxRequests) is what is used during
290 - // availability checks
291 - if h.HealthChecks != nil && h.HealthChecks.Passive != nil {
292 - h.HealthChecks.Passive.logger = h.logger.Named("health_checker.passive")
293 - if h.HealthChecks.Passive.UnhealthyRequestCount > 0 &&
294 - upstream.MaxRequests == 0 {
295 - upstream.MaxRequests = h.HealthChecks.Passive.UnhealthyRequestCount
296 - }
297 - }
298 -
299 - // upstreams need independent access to the passive
300 - // health check policy because passive health checks
301 - // run without access to h.
302 - if h.HealthChecks != nil {
303 - upstream.healthCheckPolicy = h.HealthChecks.Passive
304 - }
292 + for _, u := range h.Upstreams {
293 + h.provisionUpstream(u)
294 }
295
296 if h.HealthChecks != nil {
@@ -413,79 +402,127 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
402 repl.Set("http.reverse_proxy.duration", time.Since(start))
403 }()
404
405 + // in the proxy loop, each iteration is an attempt to proxy the request,
406 + // and because we may retry some number of times, carry over the error
407 + // from previous tries because of the nuances of load balancing & retries
408 var proxyErr error
409 for {
418 - // choose an available upstream
419 - upstream := h.LoadBalancing.SelectionPolicy.Select(h.Upstreams, clonedReq, w)
420 - if upstream == nil {
421 - if proxyErr == nil {
422 - proxyErr = fmt.Errorf("no upstreams available")
423 - }
424 - if !h.LoadBalancing.tryAgain(h.ctx, start, proxyErr, clonedReq) {
425 - break
426 - }
427 - continue
410 + var done bool
411 + done, proxyErr = h.proxyLoopIteration(clonedReq, w, proxyErr, start, repl, reqHeader, reqHost, next)
412 + if done {
413 + break
414 }
415 + }
416 +
417 + if proxyErr != nil {
418 + return statusError(proxyErr)
419 + }
420
430 - // the dial address may vary per-request if placeholders are
431 - // used, so perform those replacements here; the resulting
432 - // DialInfo struct should have valid network address syntax
433 - dialInfo, err := upstream.fillDialInfo(clonedReq)
421 + return nil
422 +}
423 +
424 +// proxyLoopIteration implements an iteration of the proxy loop. Despite the enormous amount of local state
425 +// that has to be passed in, we brought this into its own method so that we could run defer more easily.
426 +// It returns true when the loop is done and should break; false otherwise. The error value returned should
427 +// be assigned to the proxyErr value for the next iteration of the loop (or the error handled after break).
428 +func (h *Handler) proxyLoopIteration(r *http.Request, w http.ResponseWriter, proxyErr error, start time.Time,
429 + repl *caddy.Replacer, reqHeader http.Header, reqHost string, next caddyhttp.Handler) (bool, error) {
430 + // get the updated list of upstreams
431 + upstreams := h.Upstreams
432 + if h.DynamicUpstreams != nil {
433 + dUpstreams, err := h.DynamicUpstreams.GetUpstreams(r)
434 if err != nil {
435 - return statusError(fmt.Errorf("making dial info: %v", err))
435 + h.logger.Error("failed getting dynamic upstreams; falling back to static upstreams", zap.Error(err))
436 + } else {
437 + upstreams = dUpstreams
438 + for _, dUp := range dUpstreams {
439 + h.provisionUpstream(dUp)
440 + }
441 + h.logger.Debug("provisioned dynamic upstreams", zap.Int("count", len(dUpstreams)))
442 + defer func() {
443 + // these upstreams are dynamic, so they are only used for this iteration
444 + // of the proxy loop; be sure to let them go away when we're done with them
445 + for _, upstream := range dUpstreams {
446 + _, _ = hosts.Delete(upstream.String())
447 + }
448 + }()
449 }
450 + }
451
438 - // attach to the request information about how to dial the upstream;
439 - // this is necessary because the information cannot be sufficiently
440 - // or satisfactorily represented in a URL
441 - caddyhttp.SetVar(r.Context(), dialInfoVarKey, dialInfo)
442 -
443 - // set placeholders with information about this upstream
444 - repl.Set("http.reverse_proxy.upstream.address", dialInfo.String())
445 - repl.Set("http.reverse_proxy.upstream.hostport", dialInfo.Address)
446 - repl.Set("http.reverse_proxy.upstream.host", dialInfo.Host)
447 - repl.Set("http.reverse_proxy.upstream.port", dialInfo.Port)
448 - repl.Set("http.reverse_proxy.upstream.requests", upstream.Host.NumRequests())
449 - repl.Set("http.reverse_proxy.upstream.max_requests", upstream.MaxRequests)
450 - repl.Set("http.reverse_proxy.upstream.fails", upstream.Host.Fails())
451 -
452 - // mutate request headers according to this upstream;
453 - // because we're in a retry loop, we have to copy
454 - // headers (and the Host value) from the original
455 - // so that each retry is identical to the first
456 - if h.Headers != nil && h.Headers.Request != nil {
457 - clonedReq.Header = make(http.Header)
458 - copyHeader(clonedReq.Header, reqHeader)
459 - clonedReq.Host = reqHost
460 - h.Headers.Request.ApplyToRequest(clonedReq)
452 + // choose an available upstream
453 + upstream := h.LoadBalancing.SelectionPolicy.Select(upstreams, r, w)
454 + if upstream == nil {
455 + if proxyErr == nil {
456 + proxyErr = fmt.Errorf("no upstreams available")
457 }
462 -
463 - // proxy the request to that upstream
464 - proxyErr = h.reverseProxy(w, clonedReq, repl, dialInfo, next)
465 - if proxyErr == nil || proxyErr == context.Canceled {
466 - // context.Canceled happens when the downstream client
467 - // cancels the request, which is not our failure
468 - return nil
458 + if !h.LoadBalancing.tryAgain(h.ctx, start, proxyErr, r) {
459 + return true, proxyErr
460 }
461 + return false, proxyErr
462 + }
463
471 - // if the roundtrip was successful, don't retry the request or
472 - // ding the health status of the upstream (an error can still
473 - // occur after the roundtrip if, for example, a response handler
474 - // after the roundtrip returns an error)
475 - if succ, ok := proxyErr.(roundtripSucceeded); ok {
476 - return succ.error
477 - }
464 + // the dial address may vary per-request if placeholders are
465 + // used, so perform those replacements here; the resulting
466 + // DialInfo struct should have valid network address syntax
467 + dialInfo, err := upstream.fillDialInfo(r)
468 + if err != nil {
469 + return true, fmt.Errorf("making dial info: %v", err)
470 + }
471
479 - // remember this failure (if enabled)
480 - h.countFailure(upstream)
472 + h.logger.Debug("selected upstream",
473 + zap.String("dial", dialInfo.Address),
474 + zap.Int("total_upstreams", len(upstreams)))
475
482 - // if we've tried long enough, break
483 - if !h.LoadBalancing.tryAgain(h.ctx, start, proxyErr, clonedReq) {
484 - break
485 - }
476 + // attach to the request information about how to dial the upstream;
477 + // this is necessary because the information cannot be sufficiently
478 + // or satisfactorily represented in a URL
479 + caddyhttp.SetVar(r.Context(), dialInfoVarKey, dialInfo)
480 +
481 + // set placeholders with information about this upstream
482 + repl.Set("http.reverse_proxy.upstream.address", dialInfo.String())
483 + repl.Set("http.reverse_proxy.upstream.hostport", dialInfo.Address)
484 + repl.Set("http.reverse_proxy.upstream.host", dialInfo.Host)
485 + repl.Set("http.reverse_proxy.upstream.port", dialInfo.Port)
486 + repl.Set("http.reverse_proxy.upstream.requests", upstream.Host.NumRequests())
487 + repl.Set("http.reverse_proxy.upstream.max_requests", upstream.MaxRequests)
488 + repl.Set("http.reverse_proxy.upstream.fails", upstream.Host.Fails())
489 +
490 + // mutate request headers according to this upstream;
491 + // because we're in a retry loop, we have to copy
492 + // headers (and the r.Host value) from the original
493 + // so that each retry is identical to the first
494 + if h.Headers != nil && h.Headers.Request != nil {
495 + r.Header = make(http.Header)
496 + copyHeader(r.Header, reqHeader)
497 + r.Host = reqHost
498 + h.Headers.Request.ApplyToRequest(r)
499 }
500
488 - return statusError(proxyErr)
501 + // proxy the request to that upstream
502 + proxyErr = h.reverseProxy(w, r, repl, dialInfo, next)
503 + if proxyErr == nil || proxyErr == context.Canceled {
504 + // context.Canceled happens when the downstream client
505 + // cancels the request, which is not our failure
506 + return true, nil
507 + }
508 +
509 + // if the roundtrip was successful, don't retry the request or
510 + // ding the health status of the upstream (an error can still
511 + // occur after the roundtrip if, for example, a response handler
512 + // after the roundtrip returns an error)
513 + if succ, ok := proxyErr.(roundtripSucceeded); ok {
514 + return true, succ.error
515 + }
516 +
517 + // remember this failure (if enabled)
518 + h.countFailure(upstream)
519 +
520 + // if we've tried long enough, break
521 + if !h.LoadBalancing.tryAgain(h.ctx, start, proxyErr, r) {
522 + return true, proxyErr
523 + }
524 +
525 + return false, proxyErr
526 }
527
528 // prepareRequest clones req so that it can be safely modified without
@@ -651,9 +688,9 @@ func (h Handler) addForwardedHeaders(req *http.Request) error {
688 // (This method is mostly the beginning of what was borrowed from the net/http/httputil package in the
689 // Go standard library which was used as the foundation.)
690 func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, repl *caddy.Replacer, di DialInfo, next caddyhttp.Handler) error {
654 - _ = di.Upstream.Host.CountRequest(1)
691 + _ = di.Upstream.Host.countRequest(1)
692 //nolint:errcheck
656 - defer di.Upstream.Host.CountRequest(-1)
693 + defer di.Upstream.Host.countRequest(-1)
694
695 // point the request to this upstream
696 h.directRequest(req, di)
@@ -905,6 +942,35 @@ func (Handler) directRequest(req *http.Request, di DialInfo) {
942 req.URL.Host = reqHost
943 }
944
945 +func (h Handler) provisionUpstream(upstream *Upstream) {
946 + // create or get the host representation for this upstream
947 + upstream.fillHost()
948 +
949 + // give it the circuit breaker, if any
950 + upstream.cb = h.CB
951 +
952 + // if the passive health checker has a non-zero UnhealthyRequestCount
953 + // but the upstream has no MaxRequests set (they are the same thing,
954 + // but the passive health checker is a default value for for upstreams
955 + // without MaxRequests), copy the value into this upstream, since the
956 + // value in the upstream (MaxRequests) is what is used during
957 + // availability checks
958 + if h.HealthChecks != nil && h.HealthChecks.Passive != nil {
959 + h.HealthChecks.Passive.logger = h.logger.Named("health_checker.passive")
960 + if h.HealthChecks.Passive.UnhealthyRequestCount > 0 &&
961 + upstream.MaxRequests == 0 {
962 + upstream.MaxRequests = h.HealthChecks.Passive.UnhealthyRequestCount
963 + }
964 + }
965 +
966 + // upstreams need independent access to the passive
967 + // health check policy because passive health checks
968 + // run without access to h.
969 + if h.HealthChecks != nil {
970 + upstream.healthCheckPolicy = h.HealthChecks.Passive
971 + }
972 +}
973 +
974 // bufferedBody reads originalBody into a buffer, then returns a reader for the buffer.
975 // Always close the return value when done with it, just like if it was the original body!
976 func (h Handler) bufferedBody(originalBody io.ReadCloser) io.ReadCloser {
@@ -1085,6 +1151,20 @@ type Selector interface {
1151 Select(UpstreamPool, *http.Request, http.ResponseWriter) *Upstream
1152 }
1153
1154 +// UpstreamSource gets the list of upstreams that can be used when
1155 +// proxying a request. Returned upstreams will be load balanced and
1156 +// health-checked. This should be a very fast function -- instant
1157 +// if possible -- and the return value must be as stable as possible.
1158 +// In other words, the list of upstreams should ideally not change much
1159 +// across successive calls. If the list of upstreams changes or the
1160 +// ordering is not stable, load balancing will suffer. This function
1161 +// may be called during each retry, multiple times per request, and as
1162 +// such, needs to be instantaneous. The returned slice will not be
1163 +// modified.
1164 +type UpstreamSource interface {
1165 + GetUpstreams(*http.Request) ([]*Upstream, error)
1166 +}
1167 +
1168 // Hop-by-hop headers. These are removed when sent to the backend.
1169 // As of RFC 7230, hop-by-hop headers are required to appear in the
1170 // Connection header field. These are the headers defined by the
modules/caddyhttp/reverseproxy/selectionpolicies_test.go
+60 -60
@@ -22,9 +22,9 @@ import (
22
23 func testPool() UpstreamPool {
24 return UpstreamPool{
25 - {Host: new(upstreamHost)},
26 - {Host: new(upstreamHost)},
27 - {Host: new(upstreamHost)},
25 + {Host: new(Host)},
26 + {Host: new(Host)},
27 + {Host: new(Host)},
28 }
29 }
30
@@ -48,20 +48,20 @@ func TestRoundRobinPolicy(t *testing.T) {
48 t.Error("Expected third round robin host to be first host in the pool.")
49 }
50 // mark host as down
51 - pool[1].SetHealthy(false)
51 + pool[1].setHealthy(false)
52 h = rrPolicy.Select(pool, req, nil)
53 if h != pool[2] {
54 t.Error("Expected to skip down host.")
55 }
56 // mark host as up
57 - pool[1].SetHealthy(true)
57 + pool[1].setHealthy(true)
58
59 h = rrPolicy.Select(pool, req, nil)
60 if h == pool[2] {
61 t.Error("Expected to balance evenly among healthy hosts")
62 }
63 // mark host as full
64 - pool[1].CountRequest(1)
64 + pool[1].countRequest(1)
65 pool[1].MaxRequests = 1
66 h = rrPolicy.Select(pool, req, nil)
67 if h != pool[2] {
@@ -74,13 +74,13 @@ func TestLeastConnPolicy(t *testing.T) {
74 lcPolicy := new(LeastConnSelection)
75 req, _ := http.NewRequest("GET", "/", nil)
76
77 - pool[0].CountRequest(10)
78 - pool[1].CountRequest(10)
77 + pool[0].countRequest(10)
78 + pool[1].countRequest(10)
79 h := lcPolicy.Select(pool, req, nil)
80 if h != pool[2] {
81 t.Error("Expected least connection host to be third host.")
82 }
83 - pool[2].CountRequest(100)
83 + pool[2].countRequest(100)
84 h = lcPolicy.Select(pool, req, nil)
85 if h != pool[0] && h != pool[1] {
86 t.Error("Expected least connection host to be first or second host.")
@@ -139,7 +139,7 @@ func TestIPHashPolicy(t *testing.T) {
139 // we should get a healthy host if the original host is unhealthy and a
140 // healthy host is available
141 req.RemoteAddr = "172.0.0.1"
142 - pool[1].SetHealthy(false)
142 + pool[1].setHealthy(false)
143 h = ipHash.Select(pool, req, nil)
144 if h != pool[2] {
145 t.Error("Expected ip hash policy host to be the third host.")
@@ -150,10 +150,10 @@ func TestIPHashPolicy(t *testing.T) {
150 if h != pool[2] {
151 t.Error("Expected ip hash policy host to be the third host.")
152 }
153 - pool[1].SetHealthy(true)
153 + pool[1].setHealthy(true)
154
155 req.RemoteAddr = "172.0.0.3"
156 - pool[2].SetHealthy(false)
156 + pool[2].setHealthy(false)
157 h = ipHash.Select(pool, req, nil)
158 if h != pool[0] {
159 t.Error("Expected ip hash policy host to be the first host.")
@@ -167,8 +167,8 @@ func TestIPHashPolicy(t *testing.T) {
167 // We should be able to resize the host pool and still be able to predict
168 // where a req will be routed with the same IP's used above
169 pool = UpstreamPool{
170 - {Host: new(upstreamHost)},
171 - {Host: new(upstreamHost)},
170 + {Host: new(Host)},
171 + {Host: new(Host)},
172 }
173 req.RemoteAddr = "172.0.0.1:80"
174 h = ipHash.Select(pool, req, nil)
@@ -192,8 +192,8 @@ func TestIPHashPolicy(t *testing.T) {
192 }
193
194 // We should get nil when there are no healthy hosts
195 - pool[0].SetHealthy(false)
196 - pool[1].SetHealthy(false)
195 + pool[0].setHealthy(false)
196 + pool[1].setHealthy(false)
197 h = ipHash.Select(pool, req, nil)
198 if h != nil {
199 t.Error("Expected ip hash policy host to be nil.")
@@ -201,25 +201,25 @@ func TestIPHashPolicy(t *testing.T) {
201
202 // Reproduce #4135
203 pool = UpstreamPool{
204 - {Host: new(upstreamHost)},
205 - {Host: new(upstreamHost)},
206 - {Host: new(upstreamHost)},
207 - {Host: new(upstreamHost)},
208 - {Host: new(upstreamHost)},
209 - {Host: new(upstreamHost)},
210 - {Host: new(upstreamHost)},
211 - {Host: new(upstreamHost)},
212 - {Host: new(upstreamHost)},
213 - }
214 - pool[0].SetHealthy(false)
215 - pool[1].SetHealthy(false)
216 - pool[2].SetHealthy(false)
217 - pool[3].SetHealthy(false)
218 - pool[4].SetHealthy(false)
219 - pool[5].SetHealthy(false)
220 - pool[6].SetHealthy(false)
221 - pool[7].SetHealthy(false)
222 - pool[8].SetHealthy(true)
204 + {Host: new(Host)},
205 + {Host: new(Host)},
206 + {Host: new(Host)},
207 + {Host: new(Host)},
208 + {Host: new(Host)},
209 + {Host: new(Host)},
210 + {Host: new(Host)},
211 + {Host: new(Host)},
212 + {Host: new(Host)},
213 + }
214 + pool[0].setHealthy(false)
215 + pool[1].setHealthy(false)
216 + pool[2].setHealthy(false)
217 + pool[3].setHealthy(false)
218 + pool[4].setHealthy(false)
219 + pool[5].setHealthy(false)
220 + pool[6].setHealthy(false)
221 + pool[7].setHealthy(false)
222 + pool[8].setHealthy(true)
223
224 // We should get a result back when there is one healthy host left.
225 h = ipHash.Select(pool, req, nil)
@@ -239,7 +239,7 @@ func TestFirstPolicy(t *testing.T) {
239 t.Error("Expected first policy host to be the first host.")
240 }
241
242 - pool[0].SetHealthy(false)
242 + pool[0].setHealthy(false)
243 h = firstPolicy.Select(pool, req, nil)
244 if h != pool[1] {
245 t.Error("Expected first policy host to be the second host.")
@@ -256,7 +256,7 @@ func TestURIHashPolicy(t *testing.T) {
256 t.Error("Expected uri policy host to be the first host.")
257 }
258
259 - pool[0].SetHealthy(false)
259 + pool[0].setHealthy(false)
260 h = uriPolicy.Select(pool, request, nil)
261 if h != pool[1] {
262 t.Error("Expected uri policy host to be the first host.")
@@ -271,8 +271,8 @@ func TestURIHashPolicy(t *testing.T) {
271 // We should be able to resize the host pool and still be able to predict
272 // where a request will be routed with the same URI's used above
273 pool = UpstreamPool{
274 - {Host: new(upstreamHost)},
275 - {Host: new(upstreamHost)},
274 + {Host: new(Host)},
275 + {Host: new(Host)},
276 }
277
278 request = httptest.NewRequest(http.MethodGet, "/test", nil)
@@ -281,7 +281,7 @@ func TestURIHashPolicy(t *testing.T) {
281 t.Error("Expected uri policy host to be the first host.")
282 }
283
284 - pool[0].SetHealthy(false)
284 + pool[0].setHealthy(false)
285 h = uriPolicy.Select(pool, request, nil)
286 if h != pool[1] {
287 t.Error("Expected uri policy host to be the first host.")
@@ -293,8 +293,8 @@ func TestURIHashPolicy(t *testing.T) {
293 t.Error("Expected uri policy host to be the second host.")
294 }
295
296 - pool[0].SetHealthy(false)
297 - pool[1].SetHealthy(false)
296 + pool[0].setHealthy(false)
297 + pool[1].setHealthy(false)
298 h = uriPolicy.Select(pool, request, nil)
299 if h != nil {
300 t.Error("Expected uri policy policy host to be nil.")
@@ -306,12 +306,12 @@ func TestLeastRequests(t *testing.T) {
306 pool[0].Dial = "localhost:8080"
307 pool[1].Dial = "localhost:8081"
308 pool[2].Dial = "localhost:8082"
309 - pool[0].SetHealthy(true)
310 - pool[1].SetHealthy(true)
311 - pool[2].SetHealthy(true)
312 - pool[0].CountRequest(10)
313 - pool[1].CountRequest(20)
314 - pool[2].CountRequest(30)
309 + pool[0].setHealthy(true)
310 + pool[1].setHealthy(true)
311 + pool[2].setHealthy(true)
312 + pool[0].countRequest(10)
313 + pool[1].countRequest(20)
314 + pool[2].countRequest(30)
315
316 result := leastRequests(pool)
317
@@ -329,12 +329,12 @@ func TestRandomChoicePolicy(t *testing.T) {
329 pool[0].Dial = "localhost:8080"
330 pool[1].Dial = "localhost:8081"
331 pool[2].Dial = "localhost:8082"
332 - pool[0].SetHealthy(false)
333 - pool[1].SetHealthy(true)
334 - pool[2].SetHealthy(true)
335 - pool[0].CountRequest(10)
336 - pool[1].CountRequest(20)
337 - pool[2].CountRequest(30)
332 + pool[0].setHealthy(false)
333 + pool[1].setHealthy(true)
334 + pool[2].setHealthy(true)
335 + pool[0].countRequest(10)
336 + pool[1].countRequest(20)
337 + pool[2].countRequest(30)
338
339 request := httptest.NewRequest(http.MethodGet, "/test", nil)
340 randomChoicePolicy := new(RandomChoiceSelection)
@@ -357,9 +357,9 @@ func TestCookieHashPolicy(t *testing.T) {
357 pool[0].Dial = "localhost:8080"
358 pool[1].Dial = "localhost:8081"
359 pool[2].Dial = "localhost:8082"
360 - pool[0].SetHealthy(true)
361 - pool[1].SetHealthy(false)
362 - pool[2].SetHealthy(false)
360 + pool[0].setHealthy(true)
361 + pool[1].setHealthy(false)
362 + pool[2].setHealthy(false)
363 request := httptest.NewRequest(http.MethodGet, "/test", nil)
364 w := httptest.NewRecorder()
365 cookieHashPolicy := new(CookieHashSelection)
@@ -374,8 +374,8 @@ func TestCookieHashPolicy(t *testing.T) {
374 if h != pool[0] {
375 t.Error("Expected cookieHashPolicy host to be the first only available host.")
376 }
377 - pool[1].SetHealthy(true)
378 - pool[2].SetHealthy(true)
377 + pool[1].setHealthy(true)
378 + pool[2].setHealthy(true)
379 request = httptest.NewRequest(http.MethodGet, "/test", nil)
380 w = httptest.NewRecorder()
381 request.AddCookie(cookieServer1)
@@ -387,7 +387,7 @@ func TestCookieHashPolicy(t *testing.T) {
387 if len(s) != 0 {
388 t.Error("Expected cookieHashPolicy to not set a new cookie.")
389 }
390 - pool[0].SetHealthy(false)
390 + pool[0].setHealthy(false)
391 request = httptest.NewRequest(http.MethodGet, "/test", nil)
392 w = httptest.NewRecorder()
393 request.AddCookie(cookieServer1)
modules/caddyhttp/reverseproxy/upstreams.go new
+377
@@ -0,0 +1,377 @@
1 +package reverseproxy
2 +
3 +import (
4 + "context"
5 + "fmt"
6 + weakrand "math/rand"
7 + "net"
8 + "net/http"
9 + "strconv"
10 + "sync"
11 + "time"
12 +
13 + "github.com/caddyserver/caddy/v2"
14 + "go.uber.org/zap"
15 +)
16 +
17 +func init() {
18 + caddy.RegisterModule(SRVUpstreams{})
19 + caddy.RegisterModule(AUpstreams{})
20 +}
21 +
22 +// SRVUpstreams provides upstreams from SRV lookups.
23 +// The lookup DNS name can be configured either by
24 +// its individual parts (that is, specifying the
25 +// service, protocol, and name separately) to form
26 +// the standard "_service._proto.name" domain, or
27 +// the domain can be specified directly in name by
28 +// leaving service and proto empty. See RFC 2782.
29 +//
30 +// Lookups are cached and refreshed at the configured
31 +// refresh interval.
32 +//
33 +// Returned upstreams are sorted by priority and weight.
34 +type SRVUpstreams struct {
35 + // The service label.
36 + Service string `json:"service,omitempty"`
37 +
38 + // The protocol label; either tcp or udp.
39 + Proto string `json:"proto,omitempty"`
40 +
41 + // The name label; or, if service and proto are
42 + // empty, the entire domain name to look up.
43 + Name string `json:"name,omitempty"`
44 +
45 + // The interval at which to refresh the SRV lookup.
46 + // Results are cached between lookups. Default: 1m
47 + Refresh caddy.Duration `json:"refresh,omitempty"`
48 +
49 + // Configures the DNS resolver used to resolve the
50 + // SRV address to SRV records.
51 + Resolver *UpstreamResolver `json:"resolver,omitempty"`
52 +
53 + // If Resolver is configured, how long to wait before
54 + // timing out trying to connect to the DNS server.
55 + DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`
56 +
57 + // If Resolver is configured, how long to wait before
58 + // spawning an RFC 6555 Fast Fallback connection.
59 + // A negative value disables this.
60 + FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"`
61 +
62 + resolver *net.Resolver
63 +
64 + logger *zap.Logger
65 +}
66 +
67 +// CaddyModule returns the Caddy module information.
68 +func (SRVUpstreams) CaddyModule() caddy.ModuleInfo {
69 + return caddy.ModuleInfo{
70 + ID: "http.reverse_proxy.upstreams.srv",
71 + New: func() caddy.Module { return new(SRVUpstreams) },
72 + }
73 +}
74 +
75 +// String returns the RFC 2782 representation of the SRV domain.
76 +func (su SRVUpstreams) String() string {
77 + return fmt.Sprintf("_%s._%s.%s", su.Service, su.Proto, su.Name)
78 +}
79 +
80 +func (su *SRVUpstreams) Provision(ctx caddy.Context) error {
81 + su.logger = ctx.Logger(su)
82 + if su.Refresh == 0 {
83 + su.Refresh = caddy.Duration(time.Minute)
84 + }
85 +
86 + if su.Resolver != nil {
87 + err := su.Resolver.ParseAddresses()
88 + if err != nil {
89 + return err
90 + }
91 + d := &net.Dialer{
92 + Timeout: time.Duration(su.DialTimeout),
93 + FallbackDelay: time.Duration(su.FallbackDelay),
94 + }
95 + su.resolver = &net.Resolver{
96 + PreferGo: true,
97 + Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
98 + //nolint:gosec
99 + addr := su.Resolver.netAddrs[weakrand.Intn(len(su.Resolver.netAddrs))]
100 + return d.DialContext(ctx, addr.Network, addr.JoinHostPort(0))
101 + },
102 + }
103 + }
104 + if su.resolver == nil {
105 + su.resolver = net.DefaultResolver
106 + }
107 +
108 + return nil
109 +}
110 +
111 +func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
112 + suStr := su.String()
113 +
114 + // first, use a cheap read-lock to return a cached result quickly
115 + srvsMu.RLock()
116 + cached := srvs[suStr]
117 + srvsMu.RUnlock()
118 + if cached.isFresh() {
119 + return cached.upstreams, nil
120 + }
121 +
122 + // otherwise, obtain a write-lock to update the cached value
123 + srvsMu.Lock()
124 + defer srvsMu.Unlock()
125 +
126 + // check to see if it's still stale, since we're now in a different
127 + // lock from when we first checked freshness; another goroutine might
128 + // have refreshed it in the meantime before we re-obtained our lock
129 + cached = srvs[suStr]
130 + if cached.isFresh() {
131 + return cached.upstreams, nil
132 + }
133 +
134 + // prepare parameters and perform the SRV lookup
135 + repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
136 + service := repl.ReplaceAll(su.Service, "")
137 + proto := repl.ReplaceAll(su.Proto, "")
138 + name := repl.ReplaceAll(su.Name, "")
139 +
140 + su.logger.Debug("refreshing SRV upstreams",
141 + zap.String("service", service),
142 + zap.String("proto", proto),
143 + zap.String("name", name))
144 +
145 + _, records, err := su.resolver.LookupSRV(r.Context(), service, proto, name)
146 + if err != nil {
147 + // From LookupSRV docs: "If the response contains invalid names, those records are filtered
148 + // out and an error will be returned alongside the the remaining results, if any." Thus, we
149 + // only return an error if no records were also returned.
150 + if len(records) == 0 {
151 + return nil, err
152 + }
153 + su.logger.Warn("SRV records filtered", zap.Error(err))
154 + }
155 +
156 + upstreams := make([]*Upstream, len(records))
157 + for i, rec := range records {
158 + su.logger.Debug("discovered SRV record",
159 + zap.String("target", rec.Target),
160 + zap.Uint16("port", rec.Port),
161 + zap.Uint16("priority", rec.Priority),
162 + zap.Uint16("weight", rec.Weight))
163 + addr := net.JoinHostPort(rec.Target, strconv.Itoa(int(rec.Port)))
164 + upstreams[i] = &Upstream{Dial: addr}
165 + }
166 +
167 + // before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
168 + if cached.freshness.IsZero() && len(srvs) >= 100 {
169 + for randomKey := range srvs {
170 + delete(srvs, randomKey)
171 + break
172 + }
173 + }
174 +
175 + srvs[suStr] = srvLookup{
176 + srvUpstreams: su,
177 + freshness: time.Now(),
178 + upstreams: upstreams,
179 + }
180 +
181 + return upstreams, nil
182 +}
183 +
184 +type srvLookup struct {
185 + srvUpstreams SRVUpstreams
186 + freshness time.Time
187 + upstreams []*Upstream
188 +}
189 +
190 +func (sl srvLookup) isFresh() bool {
191 + return time.Since(sl.freshness) < time.Duration(sl.srvUpstreams.Refresh)
192 +}
193 +
194 +var (
195 + srvs = make(map[string]srvLookup)
196 + srvsMu sync.RWMutex
197 +)
198 +
199 +// AUpstreams provides upstreams from A/AAAA lookups.
200 +// Results are cached and refreshed at the configured
201 +// refresh interval.
202 +type AUpstreams struct {
203 + // The domain name to look up.
204 + Name string `json:"name,omitempty"`
205 +
206 + // The port to use with the upstreams. Default: 80
207 + Port string `json:"port,omitempty"`
208 +
209 + // The interval at which to refresh the A lookup.
210 + // Results are cached between lookups. Default: 1m
211 + Refresh caddy.Duration `json:"refresh,omitempty"`
212 +
213 + // Configures the DNS resolver used to resolve the
214 + // domain name to A records.
215 + Resolver *UpstreamResolver `json:"resolver,omitempty"`
216 +
217 + // If Resolver is configured, how long to wait before
218 + // timing out trying to connect to the DNS server.
219 + DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`
220 +
221 + // If Resolver is configured, how long to wait before
222 + // spawning an RFC 6555 Fast Fallback connection.
223 + // A negative value disables this.
224 + FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"`
225 +
226 + resolver *net.Resolver
227 +}
228 +
229 +// CaddyModule returns the Caddy module information.
230 +func (AUpstreams) CaddyModule() caddy.ModuleInfo {
231 + return caddy.ModuleInfo{
232 + ID: "http.reverse_proxy.upstreams.a",
233 + New: func() caddy.Module { return new(AUpstreams) },
234 + }
235 +}
236 +
237 +func (au AUpstreams) String() string { return au.Name }
238 +
239 +func (au *AUpstreams) Provision(_ caddy.Context) error {
240 + if au.Refresh == 0 {
241 + au.Refresh = caddy.Duration(time.Minute)
242 + }
243 + if au.Port == "" {
244 + au.Port = "80"
245 + }
246 +
247 + if au.Resolver != nil {
248 + err := au.Resolver.ParseAddresses()
249 + if err != nil {
250 + return err
251 + }
252 + d := &net.Dialer{
253 + Timeout: time.Duration(au.DialTimeout),
254 + FallbackDelay: time.Duration(au.FallbackDelay),
255 + }
256 + au.resolver = &net.Resolver{
257 + PreferGo: true,
258 + Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
259 + //nolint:gosec
260 + addr := au.Resolver.netAddrs[weakrand.Intn(len(au.Resolver.netAddrs))]
261 + return d.DialContext(ctx, addr.Network, addr.JoinHostPort(0))
262 + },
263 + }
264 + }
265 + if au.resolver == nil {
266 + au.resolver = net.DefaultResolver
267 + }
268 +
269 + return nil
270 +}
271 +
272 +func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
273 + auStr := au.String()
274 +
275 + // first, use a cheap read-lock to return a cached result quickly
276 + aAaaaMu.RLock()
277 + cached := aAaaa[auStr]
278 + aAaaaMu.RUnlock()
279 + if cached.isFresh() {
280 + return cached.upstreams, nil
281 + }
282 +
283 + // otherwise, obtain a write-lock to update the cached value
284 + aAaaaMu.Lock()
285 + defer aAaaaMu.Unlock()
286 +
287 + // check to see if it's still stale, since we're now in a different
288 + // lock from when we first checked freshness; another goroutine might
289 + // have refreshed it in the meantime before we re-obtained our lock
290 + cached = aAaaa[auStr]
291 + if cached.isFresh() {
292 + return cached.upstreams, nil
293 + }
294 +
295 + repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
296 + name := repl.ReplaceAll(au.Name, "")
297 + port := repl.ReplaceAll(au.Port, "")
298 +
299 + ips, err := au.resolver.LookupIPAddr(r.Context(), name)
300 + if err != nil {
301 + return nil, err
302 + }
303 +
304 + upstreams := make([]*Upstream, len(ips))
305 + for i, ip := range ips {
306 + upstreams[i] = &Upstream{
307 + Dial: net.JoinHostPort(ip.String(), port),
308 + }
309 + }
310 +
311 + // before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
312 + if cached.freshness.IsZero() && len(srvs) >= 100 {
313 + for randomKey := range aAaaa {
314 + delete(aAaaa, randomKey)
315 + break
316 + }
317 + }
318 +
319 + aAaaa[auStr] = aLookup{
320 + aUpstreams: au,
321 + freshness: time.Now(),
322 + upstreams: upstreams,
323 + }
324 +
325 + return upstreams, nil
326 +}
327 +
328 +type aLookup struct {
329 + aUpstreams AUpstreams
330 + freshness time.Time
331 + upstreams []*Upstream
332 +}
333 +
334 +func (al aLookup) isFresh() bool {
335 + return time.Since(al.freshness) < time.Duration(al.aUpstreams.Refresh)
336 +}
337 +
338 +// UpstreamResolver holds the set of addresses of DNS resolvers of
339 +// upstream addresses
340 +type UpstreamResolver struct {
341 + // The addresses of DNS resolvers to use when looking up the addresses of proxy upstreams.
342 + // It accepts [network addresses](/docs/conventions#network-addresses)
343 + // with port range of only 1. If the host is an IP address, it will be dialed directly to resolve the upstream server.
344 + // If the host is not an IP address, the addresses are resolved using the [name resolution convention](https://golang.org/pkg/net/#hdr-Name_Resolution) of the Go standard library.
345 + // If the array contains more than 1 resolver address, one is chosen at random.
346 + Addresses []string `json:"addresses,omitempty"`
347 + netAddrs []caddy.NetworkAddress
348 +}
349 +
350 +// ParseAddresses parses all the configured network addresses
351 +// and ensures they're ready to be used.
352 +func (u *UpstreamResolver) ParseAddresses() error {
353 + for _, v := range u.Addresses {
354 + addr, err := caddy.ParseNetworkAddress(v)
355 + if err != nil {
356 + return err
357 + }
358 + if addr.PortRangeSize() != 1 {
359 + return fmt.Errorf("resolver address must have exactly one address; cannot call %v", addr)
360 + }
361 + u.netAddrs = append(u.netAddrs, addr)
362 + }
363 + return nil
364 +}
365 +
366 +var (
367 + aAaaa = make(map[string]aLookup)
368 + aAaaaMu sync.RWMutex
369 +)
370 +
371 +// Interface guards
372 +var (
373 + _ caddy.Provisioner = (*SRVUpstreams)(nil)
374 + _ UpstreamSource = (*SRVUpstreams)(nil)
375 + _ caddy.Provisioner = (*AUpstreams)(nil)
376 + _ UpstreamSource = (*AUpstreams)(nil)
377 +)