Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 04:45 UTC
caddyhttp: wait for servers from previous configs on exit (#8009)
Track pending HTTP server shutdowns across configuration reloads so termination waits for responses still served by an older configuration. Bound that wait by the terminating app's grace period and retain each reload's grace context until its servers finish shutting down. Cover HTTP/1.1, HTTP/2, and finite termination grace periods with regression tests.
Kévin Dunglas committed
Sep 30, 2026 at 19:15 UTC
ff112bb2309f12838558d24615649ca07bfd4606
2 files changed
+157
-3
modules/caddyhttp/app.go
+36
-3
@@ -716,6 +716,10 @@ func (w stdlibLogRouter) Write(p []byte) (int, error) {
716
717
// Stop gracefully shuts down the HTTP server.
718
func (app *App) Stop() error {
719
+ return app.stop(caddy.Exiting())
720
+}
721
+
722
+func (app *App) stop(exiting bool) error {
723
ctx := context.Background()
724
725
// see if any listeners in our config will be closing or if they are continuing
@@ -747,11 +751,18 @@ func (app *App) Stop() error {
751
}
752
753
// enforce grace period if configured
754
+ var finishedShutdown sync.WaitGroup
755
if app.GracePeriod > 0 {
756
var cancel context.CancelFunc
757
timeout := time.Duration(app.GracePeriod)
758
ctx, cancel = context.WithTimeoutCause(ctx, timeout, fmt.Errorf("server graceful shutdown %ds timeout", int(timeout.Seconds())))
754
- defer cancel()
759
+ defer func() {
760
+ // A reload must leave the grace period alive while its requests finish.
761
+ go func() {
762
+ finishedShutdown.Wait()
763
+ cancel()
764
+ }()
765
+ }()
766
app.logger.Info("servers shutting down; grace period initiated", zap.Duration("duration", timeout))
767
} else {
768
app.logger.Info("servers shutting down with eternal grace period")
@@ -765,7 +776,7 @@ func (app *App) Stop() error {
776
// old servers are no longer accepting new connections
777
// (* the scheduler might still pause them right before
778
// calling Shutdown(), but it's unlikely)
768
- var startedShutdown, finishedShutdown sync.WaitGroup
779
+ var startedShutdown sync.WaitGroup
780
781
// these will run in goroutines
782
stopServer := func(server *Server) {
@@ -829,6 +840,14 @@ func (app *App) Stop() error {
840
go stopH3Server(server)
841
}
842
843
+ shutdownDone := make(chan struct{})
844
+ pendingServerShutdowns.Store(shutdownDone, struct{}{})
845
+ go func() {
846
+ finishedShutdown.Wait()
847
+ close(shutdownDone)
848
+ pendingServerShutdowns.Delete(shutdownDone)
849
+ }()
850
+
851
// block until all the goroutines have been run by the scheduler;
852
// this means that they have likely called Shutdown() by now
853
startedShutdown.Wait()
@@ -840,8 +859,19 @@ func (app *App) Stop() error {
859
// if the process isn't exiting (but note that frequent config
860
// reloads with long grace periods for a sustained length of time
861
// may deplete resources)
843
- if caddy.Exiting() {
862
+ if exiting {
863
finishedShutdown.Wait()
864
+
865
+ // Responses from earlier configurations must finish before the process exits.
866
+ pendingServerShutdowns.Range(func(done, _ any) bool {
867
+ select {
868
+ case <-done.(chan struct{}):
869
+ return true
870
+ case <-ctx.Done():
871
+ app.logger.Error("waiting for previous server shutdowns", zap.Error(context.Cause(ctx)))
872
+ return false
873
+ }
874
+ })
875
}
876
877
// run stop callbacks now that the server shutdowns are complete
@@ -910,6 +940,9 @@ const (
940
defaultWriteIdleTimeout = caddy.Duration(time.Minute)
941
)
942
943
+// Reloads outlive their app, so termination must track shutdowns across configurations.
944
+var pendingServerShutdowns sync.Map
945
+
946
// Interface guards
947
var (
948
_ caddy.App = (*App)(nil)
modules/caddyhttp/app_test.go
+121
@@ -15,13 +15,134 @@
15
package caddyhttp
16
17
import (
18
+ "context"
19
+ "fmt"
20
+ "io"
21
+ "net/http"
22
+ "net/http/httptest"
23
+ "sync"
24
"testing"
25
+ "time"
26
+
27
+ "github.com/caddyserver/caddy/v2"
28
29
"go.uber.org/zap"
30
"go.uber.org/zap/zapcore"
31
"go.uber.org/zap/zaptest/observer"
32
)
33
34
+func TestStopWaitsForPreviousConfiguration(t *testing.T) {
35
+ for _, http2 := range []bool{false, true} {
36
+ for _, grace := range []time.Duration{0, 5 * time.Second} {
37
+ t.Run(fmt.Sprintf("http2=%t/grace=%s", http2, grace), func(t *testing.T) {
38
+ previous, response, release := appWithPendingResponse(t, http2)
39
+ previous.GracePeriod = caddy.Duration(grace)
40
+ if err := previous.stop(false); err != nil {
41
+ t.Fatal(err)
42
+ }
43
+
44
+ current := &App{logger: zap.NewNop()}
45
+ stopped := make(chan error, 1)
46
+ go func() { stopped <- current.stop(true) }()
47
+ select {
48
+ case err := <-stopped:
49
+ t.Fatalf("termination returned while the previous response was active: %v", err)
50
+ case <-time.After(50 * time.Millisecond):
51
+ }
52
+
53
+ release()
54
+ body, err := io.ReadAll(response.Body)
55
+ if err != nil {
56
+ t.Fatal(err)
57
+ }
58
+ if string(body) != "before\nafter\n" {
59
+ t.Fatalf("unexpected response body: %q", body)
60
+ }
61
+ select {
62
+ case err := <-stopped:
63
+ if err != nil {
64
+ t.Fatal(err)
65
+ }
66
+ case <-time.After(2 * time.Second):
67
+ t.Fatal("termination did not finish after the previous response completed")
68
+ }
69
+ })
70
+ }
71
+ }
72
+}
73
+
74
+func TestStopPreviousConfigurationGracePeriod(t *testing.T) {
75
+ previous, response, release := appWithPendingResponse(t, false)
76
+ if err := previous.stop(false); err != nil {
77
+ t.Fatal(err)
78
+ }
79
+
80
+ current := &App{GracePeriod: caddy.Duration(50 * time.Millisecond), logger: zap.NewNop()}
81
+ stopped := make(chan error, 1)
82
+ start := time.Now()
83
+ go func() { stopped <- current.stop(true) }()
84
+ select {
85
+ case err := <-stopped:
86
+ if err != nil {
87
+ t.Fatal(err)
88
+ }
89
+ case <-time.After(2 * time.Second):
90
+ t.Fatal("termination ignored its grace period while waiting for the previous configuration")
91
+ }
92
+ if elapsed := time.Since(start); elapsed < time.Duration(current.GracePeriod) {
93
+ t.Errorf("termination returned after %s, before its grace period expired", elapsed)
94
+ }
95
+
96
+ release()
97
+ if _, err := io.Copy(io.Discard, response.Body); err != nil {
98
+ t.Fatal(err)
99
+ }
100
+}
101
+
102
+func appWithPendingResponse(t *testing.T, http2 bool) (*App, *http.Response, func()) {
103
+ t.Helper()
104
+
105
+ released := make(chan struct{})
106
+ release := sync.OnceFunc(func() { close(released) })
107
+ server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
108
+ fmt.Fprintln(w, "before")
109
+ http.NewResponseController(w).Flush()
110
+ select {
111
+ case <-released:
112
+ fmt.Fprintln(w, "after")
113
+ case <-r.Context().Done():
114
+ }
115
+ }))
116
+ server.EnableHTTP2 = http2
117
+ server.StartTLS()
118
+ t.Cleanup(server.Close)
119
+ t.Cleanup(release)
120
+
121
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)
122
+ if err != nil {
123
+ t.Fatal(err)
124
+ }
125
+ response, err := server.Client().Do(request)
126
+ if err != nil {
127
+ t.Fatal(err)
128
+ }
129
+ t.Cleanup(func() { response.Body.Close() })
130
+ if http2 && response.ProtoMajor != 2 {
131
+ t.Fatalf("expected HTTP/2, got %s", response.Proto)
132
+ }
133
+ app := &App{
134
+ Servers: map[string]*Server{"test": {server: server.Config}},
135
+ logger: zap.NewNop(),
136
+ }
137
+ t.Cleanup(func() {
138
+ release()
139
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
140
+ defer cancel()
141
+ server.Config.Shutdown(ctx)
142
+ })
143
+ return app, response, release
144
+}
145
+
146
// TestServerErrorLoggerLevels verifies that recovered net/http handler panics
147
// written to http.Server.ErrorLog surface at ERROR level (so they're visible
148
// at the default log level), while other standard library server messages stay