Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
reverse_proxy: Health checks: Don't cross the streams
Fixes https://caddy.community/t/v2-health-checks-are-going-to-the-wrong-upstream/7084?u=matt ... I think
Matthew Holt committed
Feb 23, 2020 at 14:30 UTC
7cca291d62c910c0544f0c0169a8f0c81627e5d3
2 files changed
+47
-49
modules/caddyhttp/reverseproxy/healthchecks.go
+14
-17
@@ -124,26 +124,25 @@ type CircuitBreaker interface {
124
// h.HealthChecks.Active.stopChan is closed.
125
func (h *Handler) activeHealthChecker() {
126
ticker := time.NewTicker(time.Duration(h.HealthChecks.Active.Interval))
127
- h.doActiveHealthChecksForAllHosts()
127
+ h.doActiveHealthCheckForAllHosts()
128
for {
129
select {
130
case <-ticker.C:
131
- h.doActiveHealthChecksForAllHosts()
131
+ h.doActiveHealthCheckForAllHosts()
132
case <-h.HealthChecks.Active.stopChan:
133
+ // TODO: consider using a Context for cancellation instead
134
ticker.Stop()
135
return
136
}
137
}
138
}
139
139
-// doActiveHealthChecksForAllHosts immediately performs a
140
-// health checks for all hosts in the global repository.
141
-func (h *Handler) doActiveHealthChecksForAllHosts() {
142
- hosts.Range(func(key, value interface{}) bool {
143
- networkAddr := key.(string)
144
- host := value.(Host)
145
-
146
- go func(networkAddr string, host Host) {
140
+// doActiveHealthCheckForAllHosts immediately performs a
141
+// health checks for all upstream hosts configured by h.
142
+func (h *Handler) doActiveHealthCheckForAllHosts() {
143
+ for _, upstream := range h.Upstreams {
144
+ go func(upstream *Upstream) {
145
+ networkAddr := upstream.Dial
146
addr, err := caddy.ParseNetworkAddress(networkAddr)
147
if err != nil {
148
h.HealthChecks.Active.logger.Error("bad network address",
@@ -165,18 +164,15 @@ func (h *Handler) doActiveHealthChecksForAllHosts() {
164
// so use a fake Host value instead; unix sockets are usually local
165
hostAddr = "localhost"
166
}
168
- err = h.doActiveHealthCheck(DialInfo{Network: addr.Network, Address: hostAddr}, hostAddr, host)
167
+ err = h.doActiveHealthCheck(DialInfo{Network: addr.Network, Address: hostAddr}, hostAddr, upstream.Host)
168
if err != nil {
169
h.HealthChecks.Active.logger.Error("active health check failed",
170
zap.String("address", networkAddr),
171
zap.Error(err),
172
)
173
}
175
- }(networkAddr, host)
176
-
177
- // continue to iterate all hosts
178
- return true
179
- })
174
+ }(upstream)
175
+ }
176
}
177
178
// doActiveHealthCheck performs a health check to host which
@@ -209,7 +205,8 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, host H
205
u.Host = net.JoinHostPort(host, portStr)
206
}
207
212
- // attach dialing information to this request
208
+ // attach dialing information to this request - TODO: use caddy.Context's context
209
+ // so it can be canceled on config reload
210
ctx := context.Background()
211
ctx = context.WithValue(ctx, caddy.ReplacerCtxKey, caddy.NewReplacer())
212
ctx = context.WithValue(ctx, caddyhttp.VarsCtxKey, map[string]interface{}{
modules/caddyhttp/reverseproxy/reverseproxy.go
+33
-32
@@ -168,38 +168,6 @@ func (h *Handler) Provision(ctx caddy.Context) error {
168
return err
169
}
170
171
- // if active health checks are enabled, configure them and start a worker
172
- if h.HealthChecks != nil &&
173
- h.HealthChecks.Active != nil &&
174
- (h.HealthChecks.Active.Path != "" || h.HealthChecks.Active.Port != 0) {
175
- h.HealthChecks.Active.logger = h.logger.Named("health_checker.active")
176
-
177
- timeout := time.Duration(h.HealthChecks.Active.Timeout)
178
- if timeout == 0 {
179
- timeout = 5 * time.Second
180
- }
181
-
182
- h.HealthChecks.Active.stopChan = make(chan struct{})
183
- h.HealthChecks.Active.httpClient = &http.Client{
184
- Timeout: timeout,
185
- Transport: h.Transport,
186
- }
187
-
188
- if h.HealthChecks.Active.Interval == 0 {
189
- h.HealthChecks.Active.Interval = caddy.Duration(30 * time.Second)
190
- }
191
-
192
- if h.HealthChecks.Active.ExpectBody != "" {
193
- var err error
194
- h.HealthChecks.Active.bodyRegexp, err = regexp.Compile(h.HealthChecks.Active.ExpectBody)
195
- if err != nil {
196
- return fmt.Errorf("expect_body: compiling regular expression: %v", err)
197
- }
198
- }
199
-
200
- go h.activeHealthChecker()
201
- }
202
-
171
// set up upstreams
172
for _, upstream := range h.Upstreams {
173
// create or get the host representation for this upstream
@@ -235,6 +203,38 @@ func (h *Handler) Provision(ctx caddy.Context) error {
203
}
204
}
205
206
+ // if active health checks are enabled, configure them and start a worker
207
+ if h.HealthChecks != nil &&
208
+ h.HealthChecks.Active != nil &&
209
+ (h.HealthChecks.Active.Path != "" || h.HealthChecks.Active.Port != 0) {
210
+ h.HealthChecks.Active.logger = h.logger.Named("health_checker.active")
211
+
212
+ timeout := time.Duration(h.HealthChecks.Active.Timeout)
213
+ if timeout == 0 {
214
+ timeout = 5 * time.Second
215
+ }
216
+
217
+ h.HealthChecks.Active.stopChan = make(chan struct{})
218
+ h.HealthChecks.Active.httpClient = &http.Client{
219
+ Timeout: timeout,
220
+ Transport: h.Transport,
221
+ }
222
+
223
+ if h.HealthChecks.Active.Interval == 0 {
224
+ h.HealthChecks.Active.Interval = caddy.Duration(30 * time.Second)
225
+ }
226
+
227
+ if h.HealthChecks.Active.ExpectBody != "" {
228
+ var err error
229
+ h.HealthChecks.Active.bodyRegexp, err = regexp.Compile(h.HealthChecks.Active.ExpectBody)
230
+ if err != nil {
231
+ return fmt.Errorf("expect_body: compiling regular expression: %v", err)
232
+ }
233
+ }
234
+
235
+ go h.activeHealthChecker()
236
+ }
237
+
238
return nil
239
}
240
@@ -244,6 +244,7 @@ func (h *Handler) Cleanup() error {
244
if h.HealthChecks != nil &&
245
h.HealthChecks.Active != nil &&
246
h.HealthChecks.Active.stopChan != nil {
247
+ // TODO: consider using context cancellation, could be much simpler
248
close(h.HealthChecks.Active.stopChan)
249
}
250