Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 03:30 UTC
caddyhttp: preserve expired request-body read deadlines (#8178)
* caddyhttp: preserve expired request-body read deadlines * caddyhttp: retain only expired drain deadlines Track the last successfully installed read deadline and retain it after a terminal timeout only when draining is enabled and the deadline has expired. Failed deadline changes preserve the previous installed value. Cover early and wrapped timeouts, non-draining requests, failed deadline installation and replacement, and cleanup retries. * caddyhttp: shorten installed deadline field name
小莳 committed
Oct 10, 2026 at 18:40 UTC
c31d9a71ed2f628210aa607ca6c69bbf21bd73e4
3 files changed
+280
-5
caddytest/integration/request_body_timeout_test.go
new
+133
@@ -0,0 +1,133 @@
1
+package integration
2
+
3
+import (
4
+ "context"
5
+ "fmt"
6
+ "io"
7
+ "net/http"
8
+ "net/http/httptest"
9
+ "testing"
10
+ "time"
11
+
12
+ "github.com/caddyserver/caddy/v2/caddytest"
13
+)
14
+
15
+// A timed-out upload must get an error response without sending the remainder
16
+// of its body. HTTP/1 drains unread bodies smaller than 256 KiB before replying;
17
+// clearing the expired read deadline makes that drain wait for the client.
18
+func TestReverseProxyRequestBodyTimeout(t *testing.T) {
19
+ for _, proto := range []int{1, 2} {
20
+ for _, tc := range []struct {
21
+ name string
22
+ timeouts string
23
+ trickle bool
24
+ handler string
25
+ }{
26
+ {"hard stalled", `"read_timeout": 300000000, "read_idle_timeout": 60000000000`, false, ""},
27
+ {"hard trickled", `"read_timeout": 300000000, "read_idle_timeout": 60000000000`, true, ""},
28
+ {"idle stalled", `"read_idle_timeout": 300000000`, false, ""},
29
+ {"minimum rate trickled", `"read_idle_timeout": 300000000, "read_min_rate": 65536`, true, ""},
30
+ {"route stalled", `"read_idle_timeout": 60000000000`, false, `{"handler": "timeouts", "read_timeout": 300000000},`},
31
+ } {
32
+ t.Run(fmt.Sprintf("HTTP%d/%s", proto, tc.name), func(t *testing.T) {
33
+ type uploadResult struct {
34
+ n int64
35
+ err error
36
+ }
37
+ upstreamRead := make(chan uploadResult, 1)
38
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
39
+ n, err := io.Copy(io.Discard, r.Body)
40
+ upstreamRead <- uploadResult{n, err}
41
+ w.WriteHeader(http.StatusNoContent)
42
+ }))
43
+ defer upstream.Close()
44
+ tester := caddytest.NewTester(t)
45
+ tester.InitServer(fmt.Sprintf(`{
46
+ "admin": {"listen": "localhost:2999", "config": {"persist": false}},
47
+ "apps": {
48
+ "tls": {"certificates": {"load_files": [{
49
+ "certificate": "/caddy.localhost.crt", "key": "/caddy.localhost.key"
50
+ }]}},
51
+ "http": {"grace_period": 1, "servers": {"srv0": {
52
+ "listen": ["127.0.0.1:9443"],
53
+ "automatic_https": {"disable": true},
54
+ "tls_connection_policies": [{}],
55
+ %s,
56
+ "routes": [{"handle": [%s{"handler": "reverse_proxy",
57
+ "upstreams": [{"dial": %q}]
58
+ }]}]
59
+ }}}
60
+ }
61
+ }`, tc.timeouts, tc.handler, upstream.Listener.Addr().String()), "json")
62
+ if t.Failed() {
63
+ t.FailNow()
64
+ }
65
+ transport := tester.Client.Transport.(*http.Transport)
66
+ if proto == 1 {
67
+ transport.ForceAttemptHTTP2 = false
68
+ }
69
+ defer transport.CloseIdleConnections()
70
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
71
+ defer cancel()
72
+ pr, pw := io.Pipe()
73
+ defer pr.Close()
74
+ defer pw.Close()
75
+ stopClose := context.AfterFunc(ctx, func() { pr.CloseWithError(ctx.Err()) })
76
+ defer stopClose()
77
+ writerDone := make(chan struct{})
78
+ go func() {
79
+ defer close(writerDone)
80
+ if _, err := pw.Write(make([]byte, 1024)); err != nil {
81
+ return
82
+ }
83
+ if !tc.trickle {
84
+ return // Leave the pipe open so the request body stalls.
85
+ }
86
+ ticker := time.NewTicker(25 * time.Millisecond)
87
+ defer ticker.Stop()
88
+ for {
89
+ select {
90
+ case <-ctx.Done():
91
+ return
92
+ case <-ticker.C:
93
+ if _, err := pw.Write([]byte("x")); err != nil {
94
+ return
95
+ }
96
+ }
97
+ }
98
+ }()
99
+ defer func() {
100
+ cancel()
101
+ pr.Close()
102
+ <-writerDone
103
+ }()
104
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://a.caddy.localhost:9443", pr)
105
+ if err != nil {
106
+ t.Fatal(err)
107
+ }
108
+ req.ContentLength = 100 << 10 // Below net/http's post-handler drain limit.
109
+ resp, err := tester.Client.Do(req)
110
+ if err != nil {
111
+ t.Fatalf("did not receive a response to the timed-out upload: %v", err)
112
+ }
113
+ defer resp.Body.Close()
114
+ if resp.ProtoMajor != proto {
115
+ t.Fatalf("got HTTP/%d, want HTTP/%d", resp.ProtoMajor, proto)
116
+ }
117
+ // A read timeout can also cancel the request context, so the proxy
118
+ // can classify it as a canceled request rather than a gateway error.
119
+ if resp.StatusCode != 499 && resp.StatusCode != http.StatusBadGateway && resp.StatusCode != http.StatusGatewayTimeout {
120
+ t.Errorf("unexpected timeout response status: %d", resp.StatusCode)
121
+ }
122
+ select {
123
+ case result := <-upstreamRead:
124
+ if result.err == nil || result.n < 1024 || result.n >= req.ContentLength {
125
+ t.Errorf("upstream upload was not interrupted: read %d bytes, error %v", result.n, result.err)
126
+ }
127
+ case <-ctx.Done():
128
+ t.Fatal("upstream upload was not interrupted")
129
+ }
130
+ })
131
+ }
132
+ }
133
+}
modules/caddyhttp/idletimeout.go
+19
-5
@@ -16,8 +16,10 @@ package caddyhttp
16
17
import (
18
"context"
19
+ "errors"
20
"io"
21
"net/http"
22
+ "os"
23
"sync"
24
"time"
25
@@ -106,8 +108,10 @@ func (d *IdleDeadline) restore(o idleOverride) {
108
// IdleTimeoutReader wraps a request body with IdleDeadline, resetting
109
// the read deadline before every Read call instead of bounding the
110
// whole body transfer with a single hard deadline.
109
-// The deadline is cleared after an error (most likely io.EOF) is encountered when reading the body
110
-// to prevent context canceled error for h1 requests.
111
+// The deadline is cleared after a non-timeout error (most likely io.EOF) is
112
+// encountered when reading the body to prevent context canceled errors for h1
113
+// requests. An expired deadline is kept when DrainDeadline is set to bound
114
+// net/http's post-handler drain.
115
// see: https://github.com/caddyserver/caddy/issues/8103
116
type IdleTimeoutReader struct {
117
io.ReadCloser
@@ -136,6 +140,7 @@ type IdleTimeoutReader struct {
140
mu sync.Mutex
141
unsupported bool
142
armed armedDeadline
143
+ installed time.Time // Last deadline successfully installed through Ctrl.
144
finished bool
145
terminalErr error
146
}
@@ -186,8 +191,9 @@ func (r *IdleTimeoutReader) Read(p []byte) (int, error) {
191
// HandlerDone prevents later body reads from using the response controller.
192
// If the body is not finished and DrainDeadline is set, it leaves an idle
193
// deadline armed for net/http's post-handler drain. Otherwise, it clears a
189
-// deadline left from a finished body or put back between reads. It must be
190
-// called before the installing handler returns.
194
+// deadline left from a finished body or put back between reads, unless the
195
+// body timed out with an expired drain deadline. It must be called before the
196
+// installing handler returns.
197
func (r *IdleTimeoutReader) HandlerDone() {
198
r.mu.Lock()
199
defer r.mu.Unlock()
@@ -227,7 +233,7 @@ func (r *IdleTimeoutReader) Override(timeout time.Duration, minRate int64) (rest
233
234
// rearmLocked moves an armed deadline to what the current settings give.
235
func (r *IdleTimeoutReader) rearmLocked() {
230
- if r.armed == deadlineIdle && !r.finished && !r.unsupported {
236
+ if r.armed == deadlineIdle && !r.finished && !r.unsupported && r.terminalErr == nil {
237
r.setDeadlineLocked("could not set read deadline")
238
}
239
}
@@ -249,6 +255,7 @@ func (r *IdleTimeoutReader) setReadDeadlineLocked(deadline time.Time, logMessage
255
}
256
return false
257
}
258
+ r.installed = deadline
259
return true
260
}
261
@@ -278,6 +285,13 @@ func (r *IdleTimeoutReader) clearDeadlineLocked() {
285
if r.armed == deadlineNone {
286
return
287
}
288
+ if r.DrainDeadline && !r.installed.IsZero() && !time.Now().Before(r.installed) &&
289
+ errors.Is(r.terminalErr, os.ErrDeadlineExceeded) {
290
+ // HTTP/1 drains a small unread body before writing the response.
291
+ // Clearing its expired deadline would let a stalled client block
292
+ // that drain, and therefore the error response, indefinitely.
293
+ return
294
+ }
295
if !r.setReadDeadlineLocked(time.Time{}, "could not clear read deadline") {
296
return
297
}
modules/caddyhttp/idletimeout_test.go
+128
@@ -26,6 +26,7 @@ import (
26
"strings"
27
"sync"
28
"testing"
29
+ "testing/iotest"
30
"time"
31
32
"github.com/stretchr/testify/assert"
@@ -146,6 +147,133 @@ func TestIdleTimeoutReaderClearsDeadlineAtEOF(t *testing.T) {
147
assert.True(t, deadlines[len(deadlines)-1].IsZero())
148
}
149
150
+func TestIdleTimeoutReaderTerminalError(t *testing.T) {
151
+ for _, tc := range []struct {
152
+ name string
153
+ err error
154
+ keepDeadline bool
155
+ }{
156
+ {"timeout", os.ErrDeadlineExceeded, true},
157
+ {"wrapped timeout", fmt.Errorf("read body: %w", os.ErrDeadlineExceeded), true},
158
+ {"EOF", io.EOF, false},
159
+ {"other error", errors.New("read failed"), false},
160
+ } {
161
+ t.Run(tc.name, func(t *testing.T) {
162
+ deadline := time.Now().Add(-time.Second)
163
+ w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
164
+ r := &IdleTimeoutReader{
165
+ ReadCloser: io.NopCloser(iotest.ErrReader(tc.err)),
166
+ Ctrl: http.NewResponseController(w),
167
+ Deadline: IdleDeadline{Timeout: time.Hour, HardDeadline: deadline},
168
+ Logger: zap.NewNop(),
169
+ DrainDeadline: true,
170
+ }
171
+ _, err := r.Read(make([]byte, 1))
172
+ require.ErrorIs(t, err, tc.err)
173
+ want := time.Time{}
174
+ if tc.keepDeadline {
175
+ want = deadline
176
+ }
177
+ deadlines := w.snapshot()
178
+ require.NotEmpty(t, deadlines)
179
+ assert.Equal(t, want, deadlines[len(deadlines)-1], "after Read")
180
+ r.HandlerDone()
181
+ deadlines = w.snapshot()
182
+ assert.Equal(t, want, deadlines[len(deadlines)-1], "after HandlerDone")
183
+
184
+ _, err = r.Read(make([]byte, 1))
185
+ require.ErrorIs(t, err, tc.err)
186
+ r.HandlerDone()
187
+ assert.Equal(t, deadlines, w.snapshot(), "late reads and repeated completion must not change deadlines")
188
+ })
189
+ }
190
+}
191
+
192
+// Only an expired deadline successfully installed for a draining request may
193
+// survive a timeout; a failed replacement must not change that decision.
194
+func TestIdleTimeoutReaderTimeoutDeadlineCleanup(t *testing.T) {
195
+ for _, tc := range []struct {
196
+ name string
197
+ drain bool
198
+ initial time.Duration
199
+ replacement time.Duration
200
+ failAt int
201
+ wrapped bool
202
+ wantClear bool
203
+ wantCalls int
204
+ }{
205
+ {name: "early timeout", drain: true, initial: time.Hour, wantClear: true, wantCalls: 2},
206
+ {name: "wrapped early timeout", drain: true, initial: time.Hour, wrapped: true, wantClear: true, wantCalls: 2},
207
+ {name: "expired without drain", initial: -time.Hour, wantClear: true, wantCalls: 2},
208
+ {name: "initial installation fails", drain: true, initial: -time.Hour, failAt: 1, wantCalls: 1},
209
+ {name: "failed expired replacement", drain: true, initial: time.Hour, replacement: -time.Hour, failAt: 2, wantClear: true, wantCalls: 3},
210
+ {name: "failed future replacement", drain: true, initial: -time.Hour, replacement: time.Hour, failAt: 2, wantCalls: 2},
211
+ {name: "failed clear retries at completion", drain: true, initial: time.Hour, failAt: 2, wantClear: true, wantCalls: 3},
212
+ } {
213
+ t.Run(tc.name, func(t *testing.T) {
214
+ w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder(), failAt: tc.failAt}
215
+ terminalErr := error(os.ErrDeadlineExceeded)
216
+ if tc.wrapped {
217
+ terminalErr = fmt.Errorf("read body: %w", terminalErr)
218
+ }
219
+ r := &IdleTimeoutReader{
220
+ ReadCloser: io.NopCloser(iotest.ErrReader(terminalErr)),
221
+ Ctrl: http.NewResponseController(w),
222
+ Deadline: IdleDeadline{Timeout: tc.initial},
223
+ Logger: zap.NewNop(), DrainDeadline: tc.drain,
224
+ }
225
+ // Install once, then exercise a failed replacement during Read.
226
+ if tc.replacement != 0 {
227
+ r.setDeadlineLocked("initial deadline")
228
+ r.Deadline.Timeout = tc.replacement
229
+ }
230
+ _, err := r.Read(make([]byte, 1))
231
+ require.ErrorIs(t, err, terminalErr)
232
+ afterRead := w.snapshot()
233
+ if tc.wantClear {
234
+ require.True(t, afterRead[len(afterRead)-1].IsZero(), "must attempt clearing after Read")
235
+ }
236
+ r.HandlerDone()
237
+ deadlines := w.snapshot()
238
+ require.Len(t, deadlines, tc.wantCalls)
239
+ if tc.wantClear {
240
+ assert.True(t, deadlines[len(deadlines)-1].IsZero())
241
+ } else if tc.failAt != 1 {
242
+ assert.True(t, deadlines[0].Before(time.Now()), "the successfully installed deadline expired")
243
+ }
244
+ _, err = r.Read(make([]byte, 1))
245
+ require.ErrorIs(t, err, terminalErr)
246
+ r.HandlerDone()
247
+ assert.Equal(t, deadlines, w.snapshot(), "late reads and completion must leave deadlines alone")
248
+ })
249
+ }
250
+}
251
+
252
+func TestIdleTimeoutReaderTimeoutAfterOverride(t *testing.T) {
253
+ w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
254
+ r := &IdleTimeoutReader{
255
+ ReadCloser: io.NopCloser(iotest.ErrReader(os.ErrDeadlineExceeded)),
256
+ Ctrl: http.NewResponseController(w),
257
+ Deadline: IdleDeadline{Timeout: time.Hour},
258
+ Logger: zap.NewNop(),
259
+ DrainDeadline: true,
260
+ }
261
+ restore := r.Override(-time.Second, 0)
262
+ _, err := r.Read(make([]byte, 1))
263
+ require.ErrorIs(t, err, os.ErrDeadlineExceeded)
264
+ deadlines := w.snapshot()
265
+ require.NotEmpty(t, deadlines)
266
+ expired := deadlines[0]
267
+ require.True(t, expired.Before(time.Now()))
268
+ assert.Equal(t, expired, deadlines[len(deadlines)-1])
269
+
270
+ // Unwinding a route-level timeout must not rearm the longer server timeout.
271
+ restore()
272
+ r.HandlerDone()
273
+ deadlines = w.snapshot()
274
+ assert.Equal(t, expired, deadlines[len(deadlines)-1])
275
+}
276
+
277
func TestIdleTimeoutReaderHandlerDone(t *testing.T) {
278
body := &blockingReadCloser{
279
started: make(chan struct{}),