Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 04:15 UTC
caddyhttp: Serve http2 when listener wrapper doesn't return *tls.Conn (#4929)
* Serve http2 when listener wrapper doesn't return *tls.Conn * close conn when h2server serveConn returns * merge from upstream * rebase from latest * run New and Closed ConnState hook for h2 conns * go fmt * fix lint * Add comments * reorder import
WeidiDeng committed
Apr 11, 2023 at 01:05 UTC
d8d87a378f37d31cfe6502cc66ac3c95fc799489
3 files changed
+153
-5
modules/caddyhttp/app.go
+37
-5
@@ -357,6 +357,14 @@ func (app *App) Start() error {
357
MaxHeaderBytes: srv.MaxHeaderBytes,
358
Handler: srv,
359
ErrorLog: serverLogger,
360
+ ConnContext: func(ctx context.Context, c net.Conn) context.Context {
361
+ return context.WithValue(ctx, ConnCtxKey, c)
362
+ },
363
+ }
364
+ h2server := &http2.Server{
365
+ NewWriteScheduler: func() http2.WriteScheduler {
366
+ return http2.NewPriorityWriteScheduler(nil)
367
+ },
368
}
369
370
// disable HTTP/2, which we enabled by default during provisioning
@@ -378,6 +386,9 @@ func (app *App) Start() error {
386
}
387
}
388
}
389
+ } else {
390
+ //nolint:errcheck
391
+ http2.ConfigureServer(srv.server, h2server)
392
}
393
394
// this TLS config is used by the std lib to choose the actual TLS config for connections
@@ -387,9 +398,6 @@ func (app *App) Start() error {
398
399
// enable H2C if configured
400
if srv.protocol("h2c") {
390
- h2server := &http2.Server{
391
- IdleTimeout: time.Duration(srv.IdleTimeout),
392
- }
401
srv.server.Handler = h2c.NewHandler(srv, h2server)
402
}
403
@@ -456,6 +464,17 @@ func (app *App) Start() error {
464
ln = srv.listenerWrappers[i].WrapListener(ln)
465
}
466
467
+ // handle http2 if use tls listener wrapper
468
+ if useTLS {
469
+ http2lnWrapper := &http2Listener{
470
+ Listener: ln,
471
+ server: srv.server,
472
+ h2server: h2server,
473
+ }
474
+ srv.h2listeners = append(srv.h2listeners, http2lnWrapper)
475
+ ln = http2lnWrapper
476
+ }
477
+
478
// if binding to port 0, the OS chooses a port for us;
479
// but the user won't know the port unless we print it
480
if !listenAddr.IsUnixNetwork() && listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
@@ -585,12 +604,25 @@ func (app *App) Stop() error {
604
zap.Strings("addresses", server.Listen))
605
}
606
}
607
+ stopH2Listener := func(server *Server) {
608
+ defer finishedShutdown.Done()
609
+ startedShutdown.Done()
610
+
611
+ for i, s := range server.h2listeners {
612
+ if err := s.Shutdown(ctx); err != nil {
613
+ app.logger.Error("http2 listener shutdown",
614
+ zap.Error(err),
615
+ zap.Int("index", i))
616
+ }
617
+ }
618
+ }
619
620
for _, server := range app.Servers {
590
- startedShutdown.Add(2)
591
- finishedShutdown.Add(2)
621
+ startedShutdown.Add(3)
622
+ finishedShutdown.Add(3)
623
go stopServer(server)
624
go stopH3Server(server)
625
+ go stopH2Listener(server)
626
}
627
628
// block until all the goroutines have been run by the scheduler;
modules/caddyhttp/http2listener.go
new
+102
@@ -0,0 +1,102 @@
1
+package caddyhttp
2
+
3
+import (
4
+ "context"
5
+ "crypto/tls"
6
+ weakrand "math/rand"
7
+ "net"
8
+ "net/http"
9
+ "sync/atomic"
10
+ "time"
11
+
12
+ "golang.org/x/net/http2"
13
+)
14
+
15
+// http2Listener wraps the listener to solve the following problems:
16
+// 1. server h2 natively without using h2c hack when listener handles tls connection but
17
+// don't return *tls.Conn
18
+// 2. graceful shutdown. the shutdown logic is copied from stdlib http.Server, it's an extra maintenance burden but
19
+// whatever, the shutdown logic maybe extracted to be used with h2c graceful shutdown. http2.Server supports graceful shutdown
20
+// sending GO_AWAY frame to connected clients, but doesn't track connection status. It requires explicit call of http2.ConfigureServer
21
+type http2Listener struct {
22
+ cnt uint64
23
+ net.Listener
24
+ server *http.Server
25
+ h2server *http2.Server
26
+}
27
+
28
+type connectionStateConn interface {
29
+ net.Conn
30
+ ConnectionState() tls.ConnectionState
31
+}
32
+
33
+func (h *http2Listener) Accept() (net.Conn, error) {
34
+ for {
35
+ conn, err := h.Listener.Accept()
36
+ if err != nil {
37
+ return nil, err
38
+ }
39
+
40
+ if csc, ok := conn.(connectionStateConn); ok {
41
+ // *tls.Conn will return empty string because it's only populated after handshake is complete
42
+ if csc.ConnectionState().NegotiatedProtocol == http2.NextProtoTLS {
43
+ go h.serveHttp2(csc)
44
+ continue
45
+ }
46
+ }
47
+
48
+ return conn, nil
49
+ }
50
+}
51
+
52
+func (h *http2Listener) serveHttp2(csc connectionStateConn) {
53
+ atomic.AddUint64(&h.cnt, 1)
54
+ h.runHook(csc, http.StateNew)
55
+ defer func() {
56
+ csc.Close()
57
+ atomic.AddUint64(&h.cnt, ^uint64(0))
58
+ h.runHook(csc, http.StateClosed)
59
+ }()
60
+ h.h2server.ServeConn(csc, &http2.ServeConnOpts{
61
+ Context: h.server.ConnContext(context.Background(), csc),
62
+ BaseConfig: h.server,
63
+ Handler: h.server.Handler,
64
+ })
65
+}
66
+
67
+const shutdownPollIntervalMax = 500 * time.Millisecond
68
+
69
+func (h *http2Listener) Shutdown(ctx context.Context) error {
70
+ pollIntervalBase := time.Millisecond
71
+ nextPollInterval := func() time.Duration {
72
+ // Add 10% jitter.
73
+ //nolint:gosec
74
+ interval := pollIntervalBase + time.Duration(weakrand.Intn(int(pollIntervalBase/10)))
75
+ // Double and clamp for next time.
76
+ pollIntervalBase *= 2
77
+ if pollIntervalBase > shutdownPollIntervalMax {
78
+ pollIntervalBase = shutdownPollIntervalMax
79
+ }
80
+ return interval
81
+ }
82
+
83
+ timer := time.NewTimer(nextPollInterval())
84
+ defer timer.Stop()
85
+ for {
86
+ if atomic.LoadUint64(&h.cnt) == 0 {
87
+ return nil
88
+ }
89
+ select {
90
+ case <-ctx.Done():
91
+ return ctx.Err()
92
+ case <-timer.C:
93
+ timer.Reset(nextPollInterval())
94
+ }
95
+ }
96
+}
97
+
98
+func (h *http2Listener) runHook(conn net.Conn, state http.ConnState) {
99
+ if h.server.ConnState != nil {
100
+ h.server.ConnState(conn, state)
101
+ }
102
+}
modules/caddyhttp/server.go
+14
@@ -198,6 +198,7 @@ type Server struct {
198
server *http.Server
199
h3server *http3.Server
200
h3listeners []net.PacketConn // TODO: we have to hold these because quic-go won't close listeners it didn't create
201
+ h2listeners []*http2Listener
202
addresses []caddy.NetworkAddress
203
204
trustedProxies IPRangeSource
@@ -213,6 +214,16 @@ type Server struct {
214
215
// ServeHTTP is the entry point for all HTTP requests.
216
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
217
+ // If there are listener wrappers that process tls connections but don't return a *tls.Conn, this field will be nil.
218
+ // Can be removed if https://github.com/golang/go/pull/56110 is ever merged.
219
+ if r.TLS == nil {
220
+ conn := r.Context().Value(ConnCtxKey).(net.Conn)
221
+ if csc, ok := conn.(connectionStateConn); ok {
222
+ r.TLS = new(tls.ConnectionState)
223
+ *r.TLS = csc.ConnectionState()
224
+ }
225
+ }
226
+
227
w.Header().Set("Server", "Caddy")
228
229
// advertise HTTP/3, if enabled
@@ -870,6 +881,9 @@ const (
881
// originally came into the server's entry handler
882
OriginalRequestCtxKey caddy.CtxKey = "original_request"
883
884
+ // For referencing underlying net.Conn
885
+ ConnCtxKey caddy.CtxKey = "conn"
886
+
887
// For tracking whether the client is a trusted proxy
888
TrustedProxyVarKey string = "trusted_proxy"
889