Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 15:30 UTC
httpserver: Configurable shutdown delay (#4906)
Matt Holt committed
Aug 3, 2022 at 11:04 UTC
1960a0dc117dd30fb507b390ddf93b2ef371b9ad
9 files changed
+244
-45
caddyconfig/httpcaddyfile/httptype.go
+5
-4
@@ -190,10 +190,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
190
191
// now that each server is configured, make the HTTP app
192
httpApp := caddyhttp.App{
193
- HTTPPort: tryInt(options["http_port"], &warnings),
194
- HTTPSPort: tryInt(options["https_port"], &warnings),
195
- GracePeriod: tryDuration(options["grace_period"], &warnings),
196
- Servers: servers,
193
+ HTTPPort: tryInt(options["http_port"], &warnings),
194
+ HTTPSPort: tryInt(options["https_port"], &warnings),
195
+ GracePeriod: tryDuration(options["grace_period"], &warnings),
196
+ ShutdownDelay: tryDuration(options["shutdown_delay"], &warnings),
197
+ Servers: servers,
198
}
199
200
// then make the TLS app
caddyconfig/httpcaddyfile/options.go
+1
@@ -31,6 +31,7 @@ func init() {
31
RegisterGlobalOption("https_port", parseOptHTTPSPort)
32
RegisterGlobalOption("default_bind", parseOptStringList)
33
RegisterGlobalOption("grace_period", parseOptDuration)
34
+ RegisterGlobalOption("shutdown_delay", parseOptDuration)
35
RegisterGlobalOption("default_sni", parseOptSingleString)
36
RegisterGlobalOption("order", parseOptOrder)
37
RegisterGlobalOption("storage", parseOptStorage)
caddytest/integration/caddyfile_adapt/global_options.txt
+2
@@ -3,6 +3,7 @@
3
http_port 8080
4
https_port 8443
5
grace_period 5s
6
+ shutdown_delay 10s
7
default_sni localhost
8
order root first
9
storage file_system {
@@ -45,6 +46,7 @@
46
"http_port": 8080,
47
"https_port": 8443,
48
"grace_period": 5000000000,
49
+ "shutdown_delay": 10000000000,
50
"servers": {
51
"srv0": {
52
"listen": [
listeners.go
+27
-3
@@ -41,7 +41,7 @@ import (
41
// the socket have been finished. Always be sure to close listeners
42
// when you are done with them, just like normal listeners.
43
func Listen(network, addr string) (net.Listener, error) {
44
- lnKey := network + "/" + addr
44
+ lnKey := listenerKey(network, addr)
45
46
sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
47
ln, err := net.Listen(network, addr)
@@ -65,7 +65,7 @@ func Listen(network, addr string) (net.Listener, error) {
65
// It is like Listen except for PacketConns.
66
// Always be sure to close the PacketConn when you are done.
67
func ListenPacket(network, addr string) (net.PacketConn, error) {
68
- lnKey := network + "/" + addr
68
+ lnKey := listenerKey(network, addr)
69
70
sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
71
pc, err := net.ListenPacket(network, addr)
@@ -89,7 +89,7 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
89
// Note that the context passed to Accept is currently ignored, so using
90
// a context other than context.Background is meaningless.
91
func ListenQUIC(addr string, tlsConf *tls.Config) (quic.EarlyListener, error) {
92
- lnKey := "quic/" + addr
92
+ lnKey := listenerKey("udp", addr)
93
94
sharedEl, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
95
el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{})
@@ -106,6 +106,16 @@ func ListenQUIC(addr string, tlsConf *tls.Config) (quic.EarlyListener, error) {
106
}, err
107
}
108
109
+func listenerKey(network, addr string) string {
110
+ return network + "/" + addr
111
+}
112
+
113
+// ListenerUsage returns the current usage count of the given listener address.
114
+func ListenerUsage(network, addr string) int {
115
+ count, _ := listenerPool.References(listenerKey(network, addr))
116
+ return count
117
+}
118
+
119
// fakeCloseListener is a private wrapper over a listener that
120
// is shared. The state of fakeCloseListener is not shared.
121
// This allows one user of a socket to "close" the listener
@@ -353,11 +363,25 @@ func (na NetworkAddress) JoinHostPort(offset uint) string {
363
return net.JoinHostPort(na.Host, strconv.Itoa(int(na.StartPort+offset)))
364
}
365
366
+func (na NetworkAddress) Expand() []NetworkAddress {
367
+ size := na.PortRangeSize()
368
+ addrs := make([]NetworkAddress, size)
369
+ for portOffset := uint(0); portOffset < size; portOffset++ {
370
+ na2 := na
371
+ na2.StartPort, na2.EndPort = na.StartPort+portOffset, na.StartPort+portOffset
372
+ addrs[portOffset] = na2
373
+ }
374
+ return addrs
375
+}
376
+
377
// PortRangeSize returns how many ports are in
378
// pa's port range. Port ranges are inclusive,
379
// so the size is the difference of start and
380
// end ports plus one.
381
func (na NetworkAddress) PortRangeSize() uint {
382
+ if na.EndPort < na.StartPort {
383
+ return 0
384
+ }
385
return (na.EndPort - na.StartPort) + 1
386
}
387
listeners_test.go
+82
@@ -331,3 +331,85 @@ func TestJoinHostPort(t *testing.T) {
331
}
332
}
333
}
334
+
335
+func TestExpand(t *testing.T) {
336
+ for i, tc := range []struct {
337
+ input NetworkAddress
338
+ expect []NetworkAddress
339
+ }{
340
+ {
341
+ input: NetworkAddress{
342
+ Network: "tcp",
343
+ Host: "localhost",
344
+ StartPort: 2000,
345
+ EndPort: 2000,
346
+ },
347
+ expect: []NetworkAddress{
348
+ {
349
+ Network: "tcp",
350
+ Host: "localhost",
351
+ StartPort: 2000,
352
+ EndPort: 2000,
353
+ },
354
+ },
355
+ },
356
+ {
357
+ input: NetworkAddress{
358
+ Network: "tcp",
359
+ Host: "localhost",
360
+ StartPort: 2000,
361
+ EndPort: 2002,
362
+ },
363
+ expect: []NetworkAddress{
364
+ {
365
+ Network: "tcp",
366
+ Host: "localhost",
367
+ StartPort: 2000,
368
+ EndPort: 2000,
369
+ },
370
+ {
371
+ Network: "tcp",
372
+ Host: "localhost",
373
+ StartPort: 2001,
374
+ EndPort: 2001,
375
+ },
376
+ {
377
+ Network: "tcp",
378
+ Host: "localhost",
379
+ StartPort: 2002,
380
+ EndPort: 2002,
381
+ },
382
+ },
383
+ },
384
+ {
385
+ input: NetworkAddress{
386
+ Network: "tcp",
387
+ Host: "localhost",
388
+ StartPort: 2000,
389
+ EndPort: 1999,
390
+ },
391
+ expect: []NetworkAddress{},
392
+ },
393
+ {
394
+ input: NetworkAddress{
395
+ Network: "unix",
396
+ Host: "/foo/bar",
397
+ StartPort: 0,
398
+ EndPort: 0,
399
+ },
400
+ expect: []NetworkAddress{
401
+ {
402
+ Network: "unix",
403
+ Host: "/foo/bar",
404
+ StartPort: 0,
405
+ EndPort: 0,
406
+ },
407
+ },
408
+ },
409
+ } {
410
+ actual := tc.input.Expand()
411
+ if !reflect.DeepEqual(actual, tc.expect) {
412
+ t.Errorf("Test %d: Expected %+v but got %+v", i, tc.expect, actual)
413
+ }
414
+ }
415
+}
modules/caddyhttp/app.go
+89
-37
@@ -20,6 +20,7 @@ import (
20
"fmt"
21
"net/http"
22
"strconv"
23
+ "sync"
24
"time"
25
26
"github.com/caddyserver/caddy/v2"
@@ -95,6 +96,8 @@ func init() {
96
// `{http.request.uri}` | The full request URI
97
// `{http.response.header.*}` | Specific response header field
98
// `{http.vars.*}` | Custom variables in the HTTP handler chain
99
+// `{http.shutting_down}` | True if the HTTP app is shutting down
100
+// `{http.time_until_shutdown}` | Time until HTTP server shutdown, if scheduled
101
type App struct {
102
// HTTPPort specifies the port to use for HTTP (as opposed to HTTPS),
103
// which is used when setting up HTTP->HTTPS redirects or ACME HTTP
@@ -107,18 +110,31 @@ type App struct {
110
HTTPSPort int `json:"https_port,omitempty"`
111
112
// GracePeriod is how long to wait for active connections when shutting
110
- // down the server. Once the grace period is over, connections will
111
- // be forcefully closed.
113
+ // down the servers. During the grace period, no new connections are
114
+ // accepted, idle connections are closed, and active connections will
115
+ // be given the full length of time to become idle and close.
116
+ // Once the grace period is over, connections will be forcefully closed.
117
+ // If zero, the grace period is eternal. Default: 0.
118
GracePeriod caddy.Duration `json:"grace_period,omitempty"`
119
120
+ // ShutdownDelay is how long to wait before initiating the grace
121
+ // period. When this app is stopping (e.g. during a config reload or
122
+ // process exit), all servers will be shut down. Normally this immediately
123
+ // initiates the grace period. However, if this delay is configured, servers
124
+ // will not be shut down until the delay is over. During this time, servers
125
+ // continue to function normally and allow new connections. At the end, the
126
+ // grace period will begin. This can be useful to allow downstream load
127
+ // balancers time to move this instance out of the rotation without hiccups.
128
+ //
129
+ // When shutdown has been scheduled, placeholders {http.shutting_down} (bool)
130
+ // and {http.time_until_shutdown} (duration) may be useful for health checks.
131
+ ShutdownDelay caddy.Duration `json:"shutdown_delay,omitempty"`
132
+
133
// Servers is the list of servers, keyed by arbitrary names chosen
134
// at your discretion for your own convenience; the keys do not
135
// affect functionality.
136
Servers map[string]*Server `json:"servers,omitempty"`
137
119
- servers []*http.Server
120
- h3servers []*http3.Server
121
-
138
ctx caddy.Context
139
logger *zap.Logger
140
tlsApp *caddytls.TLS
@@ -162,6 +178,7 @@ func (app *App) Provision(ctx caddy.Context) error {
178
srv.tlsApp = app.tlsApp
179
srv.logger = app.logger.Named("log")
180
srv.errorLogger = app.logger.Named("log.error")
181
+ srv.shutdownAtMu = new(sync.RWMutex)
182
183
// only enable access logs if configured
184
if srv.Logs != nil {
@@ -298,7 +315,7 @@ func (app *App) Start() error {
315
}
316
317
for srvName, srv := range app.Servers {
301
- s := &http.Server{
318
+ srv.server = &http.Server{
319
ReadTimeout: time.Duration(srv.ReadTimeout),
320
ReadHeaderTimeout: time.Duration(srv.ReadHeaderTimeout),
321
WriteTimeout: time.Duration(srv.WriteTimeout),
@@ -307,13 +324,14 @@ func (app *App) Start() error {
324
Handler: srv,
325
ErrorLog: serverLogger,
326
}
327
+ tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
328
329
// enable h2c if configured
330
if srv.AllowH2C {
331
h2server := &http2.Server{
332
IdleTimeout: time.Duration(srv.IdleTimeout),
333
}
316
- s.Handler = h2c.NewHandler(srv, h2server)
334
+ srv.server.Handler = h2c.NewHandler(srv, h2server)
335
}
336
337
for _, lnAddr := range srv.Listen {
@@ -321,6 +339,8 @@ func (app *App) Start() error {
339
if err != nil {
340
return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err)
341
}
342
+ srv.addresses = append(srv.addresses, listenAddr)
343
+
344
for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
345
// create the listener for this socket
346
hostport := listenAddr.JoinHostPort(portOffset)
@@ -343,31 +363,27 @@ func (app *App) Start() error {
363
useTLS := len(srv.TLSConnPolicies) > 0 && int(listenAddr.StartPort+portOffset) != app.httpPort()
364
if useTLS {
365
// create TLS listener
346
- tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
366
ln = tls.NewListener(ln, tlsCfg)
367
349
- /////////
368
// TODO: HTTP/3 support is experimental for now
369
if srv.ExperimentalHTTP3 {
352
- app.logger.Info("enabling experimental HTTP/3 listener",
353
- zap.String("addr", hostport),
354
- )
370
+ if srv.h3server == nil {
371
+ srv.h3server = &http3.Server{
372
+ Handler: srv,
373
+ TLSConfig: tlsCfg,
374
+ MaxHeaderBytes: srv.MaxHeaderBytes,
375
+ }
376
+ }
377
+
378
+ app.logger.Info("enabling experimental HTTP/3 listener", zap.String("addr", hostport))
379
h3ln, err := caddy.ListenQUIC(hostport, tlsCfg)
380
if err != nil {
381
return fmt.Errorf("getting HTTP/3 QUIC listener: %v", err)
382
}
359
- h3srv := &http3.Server{
360
- Addr: hostport,
361
- Handler: srv,
362
- TLSConfig: tlsCfg,
363
- MaxHeaderBytes: srv.MaxHeaderBytes,
364
- }
383
+
384
//nolint:errcheck
366
- go h3srv.ServeListener(h3ln)
367
- app.h3servers = append(app.h3servers, h3srv)
368
- srv.h3server = h3srv
385
+ go srv.h3server.ServeListener(h3ln)
386
}
370
- /////////
387
}
388
389
// finish wrapping listener where we left off before TLS
@@ -390,11 +406,10 @@ func (app *App) Start() error {
406
zap.Bool("tls", useTLS),
407
)
408
393
- //nolint:errcheck
394
- go s.Serve(ln)
395
-
409
srv.listeners = append(srv.listeners, ln)
397
- app.servers = append(app.servers, s)
410
+
411
+ //nolint:errcheck
412
+ go srv.server.Serve(ln)
413
}
414
}
415
}
@@ -412,28 +427,65 @@ func (app *App) Start() error {
427
// Stop gracefully shuts down the HTTP server.
428
func (app *App) Stop() error {
429
ctx := context.Background()
430
+
431
+ // see if any listeners in our config will be closing or if they are continuing
432
+ // hrough a reload; because if any are closing, we will enforce shutdown delay
433
+ var delay bool
434
+ scheduledTime := time.Now().Add(time.Duration(app.ShutdownDelay))
435
+ if app.ShutdownDelay > 0 {
436
+ for _, server := range app.Servers {
437
+ for _, na := range server.addresses {
438
+ for _, addr := range na.Expand() {
439
+ if caddy.ListenerUsage(addr.Network, addr.JoinHostPort(0)) < 2 {
440
+ app.logger.Debug("listener closing and shutdown delay is configured", zap.String("address", addr.String()))
441
+ server.shutdownAtMu.Lock()
442
+ server.shutdownAt = scheduledTime
443
+ server.shutdownAtMu.Unlock()
444
+ delay = true
445
+ } else {
446
+ app.logger.Debug("shutdown delay configured but listener will remain open", zap.String("address", addr.String()))
447
+ }
448
+ }
449
+ }
450
+ }
451
+ }
452
+
453
+ // honor scheduled/delayed shutdown time
454
+ if delay {
455
+ app.logger.Debug("shutdown scheduled",
456
+ zap.Duration("delay_duration", time.Duration(app.ShutdownDelay)),
457
+ zap.Time("time", scheduledTime))
458
+ time.Sleep(time.Duration(app.ShutdownDelay))
459
+ }
460
+
461
+ // enforce grace period if configured
462
if app.GracePeriod > 0 {
463
var cancel context.CancelFunc
464
ctx, cancel = context.WithTimeout(ctx, time.Duration(app.GracePeriod))
465
defer cancel()
466
+ app.logger.Debug("servers shutting down; grace period initiated", zap.Duration("duration", time.Duration(app.GracePeriod)))
467
+ } else {
468
+ app.logger.Debug("servers shutting down with eternal grace period")
469
}
420
- for i, s := range app.servers {
421
- if err := s.Shutdown(ctx); err != nil {
470
+
471
+ // shut down servers
472
+ for _, server := range app.Servers {
473
+ if err := server.server.Shutdown(ctx); err != nil {
474
app.logger.Error("server shutdown",
475
zap.Error(err),
424
- zap.Int("index", i))
476
+ zap.Strings("addresses", server.Listen))
477
}
426
- }
478
428
- for i, s := range app.h3servers {
429
- // TODO: CloseGracefully, once implemented upstream
430
- // (see https://github.com/lucas-clemente/quic-go/issues/2103)
431
- if err := s.Close(); err != nil {
432
- app.logger.Error("HTTP/3 server shutdown",
433
- zap.Error(err),
434
- zap.Int("index", i))
479
+ if server.h3server != nil {
480
+ // TODO: CloseGracefully, once implemented upstream (see https://github.com/lucas-clemente/quic-go/issues/2103)
481
+ if err := server.h3server.Close(); err != nil {
482
+ app.logger.Error("HTTP/3 server shutdown",
483
+ zap.Error(err),
484
+ zap.Strings("addresses", server.Listen))
485
+ }
486
}
487
}
488
+
489
return nil
490
}
491
modules/caddyhttp/replacer.go
+16
@@ -252,6 +252,22 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo
252
}
253
}
254
255
+ switch {
256
+ case key == "http.shutting_down":
257
+ server := req.Context().Value(ServerCtxKey).(*Server)
258
+ server.shutdownAtMu.RLock()
259
+ defer server.shutdownAtMu.RUnlock()
260
+ return !server.shutdownAt.IsZero(), true
261
+ case key == "http.time_until_shutdown":
262
+ server := req.Context().Value(ServerCtxKey).(*Server)
263
+ server.shutdownAtMu.RLock()
264
+ defer server.shutdownAtMu.RUnlock()
265
+ if server.shutdownAt.IsZero() {
266
+ return nil, true
267
+ }
268
+ return time.Until(server.shutdownAt), true
269
+ }
270
+
271
return nil, false
272
}
273
modules/caddyhttp/server.go
+7
-1
@@ -24,6 +24,7 @@ import (
24
"net/url"
25
"runtime"
26
"strings"
27
+ "sync"
28
"time"
29
30
"github.com/caddyserver/caddy/v2"
@@ -139,7 +140,12 @@ type Server struct {
140
accessLogger *zap.Logger
141
errorLogger *zap.Logger
142
142
- h3server *http3.Server
143
+ server *http.Server
144
+ h3server *http3.Server
145
+ addresses []caddy.NetworkAddress
146
+
147
+ shutdownAt time.Time
148
+ shutdownAtMu *sync.RWMutex
149
}
150
151
// ServeHTTP is the entry point for all HTTP requests.
usagepool.go
+15
@@ -194,6 +194,21 @@ func (up *UsagePool) Delete(key any) (deleted bool, err error) {
194
return
195
}
196
197
+// References returns the number of references (count of usages) to a
198
+// key in the pool, and true if the key exists, or false otherwise.
199
+func (up *UsagePool) References(key interface{}) (int, bool) {
200
+ up.RLock()
201
+ upv, loaded := up.pool[key]
202
+ up.RUnlock()
203
+ if loaded {
204
+ // I wonder if it'd be safer to read this value during
205
+ // our lock on the UsagePool... guess we'll see...
206
+ refs := atomic.LoadInt32(&upv.refs)
207
+ return int(refs), true
208
+ }
209
+ return 0, false
210
+}
211
+
212
// Constructor is a function that returns a new value
213
// that can destruct itself when it is no longer needed.
214
type Constructor func() (Destructor, error)