Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 22:15 UTC
caddyhttp: use the new http.Protocols to handle h1, h2 and h2c requests (#6961)
* use the new http.Protocols to handle h1, h2 and h2c requests * fix lint * keep ConnCtxKey for now * fix handling for h2c * check http version while reading the connection * check if connection implements connectionStater when it should * add comments about either h1 or h2 must be used in the listener * fix if check * return a net.Conn that implements connectionStater if applicable * remove http/1.1 from alpn if h1 is disabled * fix matching if only h1 is enabled --------- Co-authored-by: Matt Holt <mholt@users.noreply.github.com>
WeidiDeng committed
Aug 22, 2025 at 22:30 UTC
14a63a26b9a673857fc37cba37aedc35a10ce634
3 files changed
+141
-154
modules/caddyhttp/app.go
+53
-75
@@ -28,7 +28,6 @@ import (
28
29
"go.uber.org/zap"
30
"golang.org/x/net/http2"
31
- "golang.org/x/net/http2/h2c"
31
32
"github.com/caddyserver/caddy/v2"
33
"github.com/caddyserver/caddy/v2/modules/caddyevents"
@@ -236,15 +235,6 @@ func (app *App) Provision(ctx caddy.Context) error {
235
for _, srvProtocol := range srv.Protocols {
236
srvProtocolsUnique[srvProtocol] = struct{}{}
237
}
239
- _, h1ok := srvProtocolsUnique["h1"]
240
- _, h2ok := srvProtocolsUnique["h2"]
241
- _, h2cok := srvProtocolsUnique["h2c"]
242
-
243
- // the Go standard library does not let us serve only HTTP/2 using
244
- // http.Server; we would probably need to write our own server
245
- if !h1ok && (h2ok || h2cok) {
246
- return fmt.Errorf("server %s: cannot enable HTTP/2 or H2C without enabling HTTP/1.1; add h1 to protocols or remove h2/h2c", srvName)
247
- }
238
239
if srv.ListenProtocols != nil {
240
if len(srv.ListenProtocols) != len(srv.Listen) {
@@ -278,19 +268,6 @@ func (app *App) Provision(ctx caddy.Context) error {
268
}
269
}
270
281
- lnProtocolsIncludeUnique := map[string]struct{}{}
282
- for _, lnProtocol := range lnProtocolsInclude {
283
- lnProtocolsIncludeUnique[lnProtocol] = struct{}{}
284
- }
285
- _, h1ok := lnProtocolsIncludeUnique["h1"]
286
- _, h2ok := lnProtocolsIncludeUnique["h2"]
287
- _, h2cok := lnProtocolsIncludeUnique["h2c"]
288
-
289
- // check if any listener protocols contain h2 or h2c without h1
290
- if !h1ok && (h2ok || h2cok) {
291
- return fmt.Errorf("server %s, listener %d: cannot enable HTTP/2 or H2C without enabling HTTP/1.1; add h1 to protocols or remove h2/h2c", srvName, i)
292
- }
293
-
271
srv.ListenProtocols[i] = lnProtocolsInclude
272
}
273
}
@@ -448,6 +425,25 @@ func (app *App) Validate() error {
425
return nil
426
}
427
428
+func removeTLSALPN(srv *Server, target string) {
429
+ for _, cp := range srv.TLSConnPolicies {
430
+ // the TLSConfig was already provisioned, so... manually remove it
431
+ for i, np := range cp.TLSConfig.NextProtos {
432
+ if np == target {
433
+ cp.TLSConfig.NextProtos = append(cp.TLSConfig.NextProtos[:i], cp.TLSConfig.NextProtos[i+1:]...)
434
+ break
435
+ }
436
+ }
437
+ // remove it from the parent connection policy too, just to keep things tidy
438
+ for i, alpn := range cp.ALPN {
439
+ if alpn == target {
440
+ cp.ALPN = append(cp.ALPN[:i], cp.ALPN[i+1:]...)
441
+ break
442
+ }
443
+ }
444
+ }
445
+}
446
+
447
// Start runs the app. It finishes automatic HTTPS if enabled,
448
// including management of certificates.
449
func (app *App) Start() error {
@@ -466,32 +462,37 @@ func (app *App) Start() error {
462
MaxHeaderBytes: srv.MaxHeaderBytes,
463
Handler: srv,
464
ErrorLog: serverLogger,
465
+ Protocols: new(http.Protocols),
466
ConnContext: func(ctx context.Context, c net.Conn) context.Context {
467
return context.WithValue(ctx, ConnCtxKey, c)
468
},
469
}
473
- h2server := new(http2.Server)
470
471
// disable HTTP/2, which we enabled by default during provisioning
472
if !srv.protocol("h2") {
473
srv.server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
478
- for _, cp := range srv.TLSConnPolicies {
479
- // the TLSConfig was already provisioned, so... manually remove it
480
- for i, np := range cp.TLSConfig.NextProtos {
481
- if np == "h2" {
482
- cp.TLSConfig.NextProtos = append(cp.TLSConfig.NextProtos[:i], cp.TLSConfig.NextProtos[i+1:]...)
483
- break
484
- }
485
- }
486
- // remove it from the parent connection policy too, just to keep things tidy
487
- for i, alpn := range cp.ALPN {
488
- if alpn == "h2" {
489
- cp.ALPN = append(cp.ALPN[:i], cp.ALPN[i+1:]...)
490
- break
491
- }
492
- }
493
- }
494
- } else {
474
+ removeTLSALPN(srv, "h2")
475
+ }
476
+ if !srv.protocol("h1") {
477
+ removeTLSALPN(srv, "http/1.1")
478
+ }
479
+
480
+ // configure the http versions the server will serve
481
+ if srv.protocol("h1") {
482
+ srv.server.Protocols.SetHTTP1(true)
483
+ }
484
+
485
+ if srv.protocol("h2") || srv.protocol("h2c") {
486
+ // skip setting h2 because if NextProtos is present, it's list of alpn versions will take precedence.
487
+ // it will always be present because http2.ConfigureServer will populate that field
488
+ // enabling h2c because some listener wrapper will wrap the connection that is no longer *tls.Conn
489
+ // However, we need to handle the case that if the connection is h2c but h2c is not enabled. We identify
490
+ // this type of connection by checking if it's behind a TLS listener wrapper or if it implements tls.ConnectionState.
491
+ srv.server.Protocols.SetUnencryptedHTTP2(true)
492
+ // when h2c is enabled but h2 disabled, we already removed h2 from NextProtos
493
+ // the handshake will never succeed with h2
494
+ // http2.ConfigureServer will enable the server to handle both h2 and h2c
495
+ h2server := new(http2.Server)
496
//nolint:errcheck
497
http2.ConfigureServer(srv.server, h2server)
498
}
@@ -501,11 +502,6 @@ func (app *App) Start() error {
502
tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
503
srv.configureServer(srv.server)
504
504
- // enable H2C if configured
505
- if srv.protocol("h2c") {
506
- srv.server.Handler = h2c.NewHandler(srv, h2server)
507
- }
508
-
505
for lnIndex, lnAddr := range srv.Listen {
506
listenAddr, err := caddy.ParseNetworkAddress(lnAddr)
507
if err != nil {
@@ -570,15 +566,13 @@ func (app *App) Start() error {
566
ln = srv.listenerWrappers[i].WrapListener(ln)
567
}
568
573
- // handle http2 if use tls listener wrapper
574
- if h2ok {
575
- http2lnWrapper := &http2Listener{
576
- Listener: ln,
577
- server: srv.server,
578
- h2server: h2server,
579
- }
580
- srv.h2listeners = append(srv.h2listeners, http2lnWrapper)
581
- ln = http2lnWrapper
569
+ // check if the connection is h2c
570
+ ln = &http2Listener{
571
+ useTLS: useTLS,
572
+ useH1: h1ok,
573
+ useH2: h2ok || h2cok,
574
+ Listener: ln,
575
+ logger: app.logger,
576
}
577
578
// if binding to port 0, the OS chooses a port for us;
@@ -596,11 +590,8 @@ func (app *App) Start() error {
590
591
srv.listeners = append(srv.listeners, ln)
592
599
- // enable HTTP/1 if configured
600
- if h1ok {
601
- //nolint:errcheck
602
- go srv.server.Serve(ln)
603
- }
593
+ //nolint:errcheck
594
+ go srv.server.Serve(ln)
595
}
596
597
if h2ok && !useTLS {
@@ -756,25 +747,12 @@ func (app *App) Stop() error {
747
}
748
}
749
}
759
- stopH2Listener := func(server *Server) {
760
- defer finishedShutdown.Done()
761
- startedShutdown.Done()
762
-
763
- for i, s := range server.h2listeners {
764
- if err := s.Shutdown(ctx); err != nil {
765
- app.logger.Error("http2 listener shutdown",
766
- zap.Error(err),
767
- zap.Int("index", i))
768
- }
769
- }
770
- }
750
751
for _, server := range app.Servers {
773
- startedShutdown.Add(3)
774
- finishedShutdown.Add(3)
752
+ startedShutdown.Add(2)
753
+ finishedShutdown.Add(2)
754
go stopServer(server)
755
go stopH3Server(server)
777
- go stopH2Listener(server)
756
}
757
758
// block until all the goroutines have been run by the scheduler;
modules/caddyhttp/http2listener.go
+81
-73
@@ -1,102 +1,110 @@
1
package caddyhttp
2
3
import (
4
- "context"
4
"crypto/tls"
6
- weakrand "math/rand"
5
+ "io"
6
"net"
8
- "net/http"
9
- "sync/atomic"
10
- "time"
7
8
+ "go.uber.org/zap"
9
"golang.org/x/net/http2"
10
)
11
12
+type connectionStater interface {
13
+ ConnectionState() tls.ConnectionState
14
+}
15
+
16
// http2Listener wraps the listener to solve the following problems:
16
-// 1. server h2 natively without using h2c hack when listener handles tls connection but
17
-// don't return *tls.Conn
18
-// 2. graceful shutdown. the shutdown logic is copied from stdlib http.Server, it's an extra maintenance burden but
19
-// whatever, the shutdown logic maybe extracted to be used with h2c graceful shutdown. http2.Server supports graceful shutdown
20
-// sending GO_AWAY frame to connected clients, but doesn't track connection status. It requires explicit call of http2.ConfigureServer
17
+// 1. prevent genuine h2c connections from succeeding if h2c is not enabled
18
+// and the connection doesn't implment connectionStater or the resulting NegotiatedProtocol
19
+// isn't http2.
20
+// This does allow a connection to pass as tls enabled even if it's not, listener wrappers
21
+// can do this.
22
+// 2. After wrapping the connection doesn't implement connectionStater, emit a warning so that listener
23
+// wrapper authors will hopefully implement it.
24
+// 3. check if the connection matches a specific http version. h2/h2c has a distinct preface.
25
type http2Listener struct {
22
- cnt uint64
26
+ useTLS bool
27
+ useH1 bool
28
+ useH2 bool
29
net.Listener
24
- server *http.Server
25
- h2server *http2.Server
26
-}
27
-
28
-type connectionStateConn interface {
29
- net.Conn
30
- ConnectionState() tls.ConnectionState
30
+ logger *zap.Logger
31
}
32
33
func (h *http2Listener) Accept() (net.Conn, error) {
34
- for {
35
- conn, err := h.Listener.Accept()
36
- if err != nil {
37
- return nil, err
38
- }
34
+ conn, err := h.Listener.Accept()
35
+ if err != nil {
36
+ return nil, err
37
+ }
38
40
- if csc, ok := conn.(connectionStateConn); ok {
41
- // *tls.Conn will return empty string because it's only populated after handshake is complete
42
- if csc.ConnectionState().NegotiatedProtocol == http2.NextProtoTLS {
43
- go h.serveHttp2(csc)
44
- continue
45
- }
46
- }
39
+ _, isConnectionStater := conn.(connectionStater)
40
+ // emit a warning
41
+ if h.useTLS && !isConnectionStater {
42
+ h.logger.Warn("tls is enabled, but listener wrapper returns a connection that doesn't implement connectionStater")
43
+ } else if !h.useTLS && isConnectionStater {
44
+ h.logger.Warn("tls is disabled, but listener wrapper returns a connection that implements connectionStater")
45
+ }
46
47
+ // if both h1 and h2 are enabled, we don't need to check the preface
48
+ if h.useH1 && h.useH2 {
49
return conn, nil
50
}
51
+
52
+ // impossible both are false, either useH1 or useH2 must be true,
53
+ // or else the listener wouldn't be created
54
+ h2Conn := &http2Conn{
55
+ h2Expected: h.useH2,
56
+ Conn: conn,
57
+ }
58
+ if isConnectionStater {
59
+ return http2StateConn{h2Conn}, nil
60
+ }
61
+ return h2Conn, nil
62
}
63
52
-func (h *http2Listener) serveHttp2(csc connectionStateConn) {
53
- atomic.AddUint64(&h.cnt, 1)
54
- h.runHook(csc, http.StateNew)
55
- defer func() {
56
- csc.Close()
57
- atomic.AddUint64(&h.cnt, ^uint64(0))
58
- h.runHook(csc, http.StateClosed)
59
- }()
60
- h.h2server.ServeConn(csc, &http2.ServeConnOpts{
61
- Context: h.server.ConnContext(context.Background(), csc),
62
- BaseConfig: h.server,
63
- Handler: h.server.Handler,
64
- })
64
+type http2StateConn struct {
65
+ *http2Conn
66
}
67
67
-const shutdownPollIntervalMax = 500 * time.Millisecond
68
+func (conn http2StateConn) ConnectionState() tls.ConnectionState {
69
+ return conn.Conn.(connectionStater).ConnectionState()
70
+}
71
69
-func (h *http2Listener) Shutdown(ctx context.Context) error {
70
- pollIntervalBase := time.Millisecond
71
- nextPollInterval := func() time.Duration {
72
- // Add 10% jitter.
73
- //nolint:gosec
74
- interval := pollIntervalBase + time.Duration(weakrand.Intn(int(pollIntervalBase/10)))
75
- // Double and clamp for next time.
76
- pollIntervalBase *= 2
77
- if pollIntervalBase > shutdownPollIntervalMax {
78
- pollIntervalBase = shutdownPollIntervalMax
79
- }
80
- return interval
81
- }
72
+type http2Conn struct {
73
+ // current index where the preface should match,
74
+ // no matching is done if idx is >= len(http2.ClientPreface)
75
+ idx int
76
+ // whether the connection is expected to be h2/h2c
77
+ h2Expected bool
78
+ // log if one such connection is detected
79
+ logger *zap.Logger
80
+ net.Conn
81
+}
82
83
- timer := time.NewTimer(nextPollInterval())
84
- defer timer.Stop()
85
- for {
86
- if atomic.LoadUint64(&h.cnt) == 0 {
87
- return nil
83
+func (c *http2Conn) Read(p []byte) (int, error) {
84
+ if c.idx >= len(http2.ClientPreface) {
85
+ return c.Conn.Read(p)
86
+ }
87
+ n, err := c.Conn.Read(p)
88
+ for i := range n {
89
+ // first mismatch
90
+ if p[i] != http2.ClientPreface[c.idx] {
91
+ // close the connection if h2 is expected
92
+ if c.h2Expected {
93
+ c.logger.Debug("h1 connection detected, but h1 is not enabled")
94
+ _ = c.Conn.Close()
95
+ return 0, io.EOF
96
+ }
97
+ // no need to continue matching anymore
98
+ c.idx = len(http2.ClientPreface)
99
+ return n, err
100
}
89
- select {
90
- case <-ctx.Done():
91
- return ctx.Err()
92
- case <-timer.C:
93
- timer.Reset(nextPollInterval())
101
+ c.idx++
102
+ // matching complete
103
+ if c.idx == len(http2.ClientPreface) && !c.h2Expected {
104
+ c.logger.Debug("h2/h2c connection detected, but h2/h2c is not enabled")
105
+ _ = c.Conn.Close()
106
+ return 0, io.EOF
107
}
108
}
96
-}
97
-
98
-func (h *http2Listener) runHook(conn net.Conn, state http.ConnState) {
99
- if h.server.ConnState != nil {
100
- h.server.ConnState(conn, state)
101
- }
109
+ return n, err
110
}
modules/caddyhttp/server.go
+7
-6
@@ -246,10 +246,9 @@ type Server struct {
246
traceLogger *zap.Logger
247
ctx caddy.Context
248
249
- server *http.Server
250
- h3server *http3.Server
251
- h2listeners []*http2Listener
252
- addresses []caddy.NetworkAddress
249
+ server *http.Server
250
+ h3server *http3.Server
251
+ addresses []caddy.NetworkAddress
252
253
trustedProxies IPRangeSource
254
@@ -266,11 +265,11 @@ type Server struct {
265
// ServeHTTP is the entry point for all HTTP requests.
266
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
267
// If there are listener wrappers that process tls connections but don't return a *tls.Conn, this field will be nil.
269
- // TODO: Can be removed if https://github.com/golang/go/pull/56110 is ever merged.
268
+ // TODO: Scheduled to be removed later because https://github.com/golang/go/pull/56110 has been merged.
269
if r.TLS == nil {
270
// not all requests have a conn (like virtual requests) - see #5698
271
if conn, ok := r.Context().Value(ConnCtxKey).(net.Conn); ok {
273
- if csc, ok := conn.(connectionStateConn); ok {
272
+ if csc, ok := conn.(connectionStater); ok {
273
r.TLS = new(tls.ConnectionState)
274
*r.TLS = csc.ConnectionState()
275
}
@@ -1083,6 +1082,8 @@ const (
1082
OriginalRequestCtxKey caddy.CtxKey = "original_request"
1083
1084
// For referencing underlying net.Conn
1085
+ // This will eventually be deprecated and not used. To refer to the underlying connection, implement a middleware plugin
1086
+ // that RegisterConnContext during provisioning.
1087
ConnCtxKey caddy.CtxKey = "conn"
1088
1089
// For tracking whether the client is a trusted proxy