Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 02:15 UTC
reverseproxy: retry dial errors wrapped by a forward proxy (#8264)
When a forward proxy is configured (network_proxy, forward_proxy_url or HTTP_PROXY), net/http wraps dial errors in a *net.OpError with Op "proxyconnect". tryAgain used a type assertion to detect DialError, so it missed the wrapped error and treated it as a failure after the request was sent. Non-GET requests were then not retried, even though no connection to the proxy was ever established. Use errors.As so a DialError is recognized anywhere in the error chain. Errors that do not contain a DialError are unaffected, so requests that may have reached the proxy or upstream are still not retried by default. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Adarsh Kumar committed
Oct 21, 2026 at 17:19 UTC
f355b8687d5582ea9fcc99f71598109d1e5b6ced
2 files changed
+127
-2
modules/caddyhttp/reverseproxy/retries_test.go
+122
@@ -2,15 +2,19 @@ package reverseproxy
2
3
import (
4
"context"
5
+ "encoding/json"
6
"errors"
7
+ "fmt"
8
"io"
9
"net"
10
"net/http"
11
"net/http/httptest"
12
+ "net/url"
13
"strings"
14
"sync"
15
"sync/atomic"
16
"testing"
17
+ "time"
18
19
"go.uber.org/zap"
20
@@ -850,3 +854,121 @@ func TestSubrouteErrorFallbackWithBody(t *testing.T) {
854
t.Errorf("body: got %q, want %q", rec.Body.String(), expectedBody)
855
}
856
}
857
+
858
+// TestTryAgainDialErrorBehindForwardProxy verifies that tryAgain recognizes
859
+// a DialError even when it is wrapped. When a forward proxy is configured,
860
+// net/http wraps dial errors in a *net.OpError with Op "proxyconnect", and
861
+// the DialError produced by our dialer ends up inside it. Errors that do not
862
+// contain a DialError must still not be retried for non-GET requests, since
863
+// the request may have reached the proxy or the upstream.
864
+func TestTryAgainDialErrorBehindForwardProxy(t *testing.T) {
865
+ dialErr := DialError{errors.New("dial tcp 127.0.0.1:3128: connect: connection refused")}
866
+
867
+ for i, tc := range []struct {
868
+ name string
869
+ err error
870
+ method string
871
+ want bool
872
+ }{
873
+ {
874
+ name: "unwrapped dial error",
875
+ err: dialErr,
876
+ method: http.MethodPost,
877
+ want: true,
878
+ },
879
+ {
880
+ name: "dial error wrapped by forward proxy",
881
+ err: &net.OpError{Op: "proxyconnect", Net: "tcp", Err: dialErr},
882
+ method: http.MethodPost,
883
+ want: true,
884
+ },
885
+ {
886
+ name: "dial error wrapped with fmt.Errorf",
887
+ err: fmt.Errorf("round trip: %w", dialErr),
888
+ method: http.MethodPost,
889
+ want: true,
890
+ },
891
+ {
892
+ name: "proxyconnect error without a dial error",
893
+ err: &net.OpError{Op: "proxyconnect", Net: "tcp", Err: errors.New("Bad Gateway")},
894
+ method: http.MethodPost,
895
+ want: false,
896
+ },
897
+ {
898
+ name: "non-dial error",
899
+ err: errors.New("read: connection reset by peer"),
900
+ method: http.MethodPost,
901
+ want: false,
902
+ },
903
+ {
904
+ name: "non-dial error on GET",
905
+ err: errors.New("read: connection reset by peer"),
906
+ method: http.MethodGet,
907
+ want: true,
908
+ },
909
+ } {
910
+ lb := LoadBalancing{Retries: 1}
911
+ req := httptest.NewRequest(tc.method, "http://example.com/", nil)
912
+
913
+ got := lb.tryAgain(caddy.Context{}, time.Now(), 0, tc.err, req, zap.NewNop())
914
+ if got != tc.want {
915
+ t.Errorf("Test %d (%s): expected tryAgain=%t, got %t", i, tc.name, tc.want, got)
916
+ }
917
+ }
918
+}
919
+
920
+// TestForwardProxyDialErrorRetried verifies end-to-end that when the
921
+// configured forward proxy cannot be reached, a non-GET request is retried
922
+// just like a GET request, because no connection was ever established.
923
+// It uses a real HTTPTransport so that the error is wrapped exactly as
924
+// net/http does in production.
925
+func TestForwardProxyDialErrorRetried(t *testing.T) {
926
+ deadProxy := deadUpstreamAddr(t)
927
+
928
+ for _, method := range []string{http.MethodGet, http.MethodPost} {
929
+ t.Run(method, func(t *testing.T) {
930
+ ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
931
+ defer cancel()
932
+
933
+ ht := &HTTPTransport{
934
+ NetworkProxyRaw: json.RawMessage(fmt.Sprintf(`{"from": "url", "url": "http://%s"}`, deadProxy)),
935
+ }
936
+ if err := ht.Provision(ctx); err != nil {
937
+ t.Fatalf("provisioning transport: %v", err)
938
+ }
939
+
940
+ // each round trip consults the proxy function exactly once,
941
+ // so counting calls counts the attempts made by the handler
942
+ var attempts atomic.Int32
943
+ proxyFunc := ht.Transport.Proxy
944
+ ht.Transport.Proxy = func(r *http.Request) (*url.URL, error) {
945
+ attempts.Add(1)
946
+ return proxyFunc(r)
947
+ }
948
+
949
+ h := minimalHandler(1,
950
+ &Upstream{Host: new(Host), Dial: "upstream-a.example.com:80"},
951
+ &Upstream{Host: new(Host), Dial: "upstream-b.example.com:80"},
952
+ )
953
+ h.Transport = ht
954
+
955
+ var body io.Reader
956
+ if method == http.MethodPost {
957
+ body = strings.NewReader("payload")
958
+ }
959
+ req := prepareTestRequest(httptest.NewRequest(method, "http://example.com/", body))
960
+
961
+ err := h.ServeHTTP(httptest.NewRecorder(), req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
962
+ return nil
963
+ }))
964
+
965
+ var herr caddyhttp.HandlerError
966
+ if !errors.As(err, &herr) || herr.StatusCode != http.StatusBadGateway {
967
+ t.Fatalf("expected a 502 handler error, got %v", err)
968
+ }
969
+ if got := attempts.Load(); got != 2 {
970
+ t.Errorf("expected 2 attempts (1 try + 1 retry), got %d", got)
971
+ }
972
+ })
973
+ }
974
+}
modules/caddyhttp/reverseproxy/reverseproxy.go
+5
-2
@@ -1552,9 +1552,12 @@ func (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, retries int
1552
// could not even be established to the upstream), then it
1553
// should be safe to retry, since without a connection, no
1554
// HTTP request can be transmitted; but if the error is not
1555
- // specifically a dialer error, we need to be careful
1555
+ // specifically a dialer error, we need to be careful; the
1556
+ // dial error may be wrapped, e.g. net/http wraps it in a
1557
+ // *net.OpError ("proxyconnect") when a forward proxy is used
1558
if proxyErr != nil {
1557
- _, isDialError := proxyErr.(DialError)
1559
+ var dialErr DialError
1560
+ isDialError := errors.As(proxyErr, &dialErr)
1561
_, isRetryableResponse := proxyErr.(retryableResponseError)
1562
herr, isHandlerError := proxyErr.(caddyhttp.HandlerError)
1563