reverseproxy: SRV dynamic upstream failover (#5832)

* Implement grace period, but probably needs sync * Update cached freshness value * D'oh, actually use the grace period * Fix freshness math

Matt Holt committed Mar 5, 2024 at 12:08 UTC 72ce78d9af0204a4018ce197ff95f681a720818b

1 file changed +13

modules/caddyhttp/reverseproxy/upstreams.go

+13

@@ -49,6 +49,13 @@ type SRVUpstreams struct {

49 // Results are cached between lookups. Default: 1m

50 Refresh caddy.Duration `json:"refresh,omitempty"`

52 + // If > 0 and there is an error with the lookup,

53 + // continue to use the cached results for up to

54 + // this long before trying again, (even though they

55 + // are stale) instead of returning an error to the

56 + // client. Default: 0s.

57 + GracePeriod caddy.Duration `json:"grace_period,omitempty"`

58 +

59 // Configures the DNS resolver used to resolve the

60 // SRV address to SRV records.

61 Resolver *UpstreamResolver `json:"resolver,omitempty"`

@@ -140,6 +147,12 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {

147 // out and an error will be returned alongside the remaining results, if any." Thus, we

148 // only return an error if no records were also returned.

149 if len(records) == 0 {

150 + if su.GracePeriod > 0 {

151 + su.logger.Error("SRV lookup failed; using previously cached", zap.Error(err))

152 + cached.freshness = time.Now().Add(time.Duration(su.GracePeriod) - time.Duration(su.Refresh))

153 + srvs[suAddr] = cached

154 + return allNew(cached.upstreams), nil

155 + }

156 return nil, err

157 }

158 su.logger.Warn("SRV records filtered", zap.Error(err))