Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 20:30 UTC
Fix http3 servers dying after reload (#4654)
Artem Mikheev committed
Mar 23, 2022 at 02:47 UTC
c9b5e7f77b8aac7334d81c552c583af81ba1c400
4 files changed
+90
-41
go.mod
+1
-1
@@ -16,7 +16,7 @@ require (
16
github.com/google/uuid v1.3.0
17
github.com/klauspost/compress v1.15.0
18
github.com/klauspost/cpuid/v2 v2.0.11
19
- github.com/lucas-clemente/quic-go v0.25.0
19
+ github.com/lucas-clemente/quic-go v0.26.0
20
github.com/mholt/acmez v1.0.2
21
github.com/naoina/go-stringutil v0.1.0 // indirect
22
github.com/naoina/toml v0.1.1
go.sum
+8
-9
@@ -664,8 +664,8 @@ github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
664
github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
665
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
666
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
667
-github.com/lucas-clemente/quic-go v0.25.0 h1:K+X9Gvd7JXsOHtU0N2icZ2Nw3rx82uBej3mP4CLgibc=
668
-github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
667
+github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A=
668
+github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
669
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
670
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
671
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -675,13 +675,12 @@ github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYt
675
github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
676
github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
677
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
678
-github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
679
-github.com/marten-seemann/qtls-go1-16 v0.1.4 h1:xbHbOGGhrenVtII6Co8akhLEdrawwB2iHl5yhJRpnco=
680
-github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
681
-github.com/marten-seemann/qtls-go1-17 v0.1.0 h1:P9ggrs5xtwiqXv/FHNwntmuLMNq3KaSIG93AtAZ48xk=
682
-github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
683
-github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 h1:EnzzN9fPUkUck/1CuY1FlzBaIYMoiBsdwTNmNGkwUUM=
684
-github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
678
+github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
679
+github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
680
+github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc=
681
+github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
682
+github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y=
683
+github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
684
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
685
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
686
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
listeners.go
+76
-5
@@ -15,6 +15,9 @@
15
package caddy
16
17
import (
18
+ "context"
19
+ "crypto/tls"
20
+ "errors"
21
"fmt"
22
"net"
23
"os"
@@ -24,6 +27,9 @@ import (
27
"sync/atomic"
28
"syscall"
29
"time"
30
+
31
+ "github.com/lucas-clemente/quic-go"
32
+ "github.com/lucas-clemente/quic-go/http3"
33
)
34
35
// Listen is like net.Listen, except Caddy's listeners can overlap
@@ -79,6 +85,27 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
85
return &fakeClosePacketConn{sharedPacketConn: sharedPc.(*sharedPacketConn)}, nil
86
}
87
88
+// ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
89
+// Note that the context passed to Accept is currently ignored, so using
90
+// a context other than context.Background is meaningless.
91
+func ListenQUIC(addr string, tlsConf *tls.Config) (quic.EarlyListener, error) {
92
+ lnKey := "quic/" + addr
93
+
94
+ sharedEl, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
95
+ el, err := quic.ListenAddrEarly(addr, http3.ConfigureTLSConfig(tlsConf), &quic.Config{})
96
+ if err != nil {
97
+ return nil, err
98
+ }
99
+ return &sharedQuicListener{EarlyListener: el, key: lnKey}, nil
100
+ })
101
+
102
+ ctx, cancel := context.WithCancel(context.Background())
103
+ return &fakeCloseQuicListener{
104
+ sharedQuicListener: sharedEl.(*sharedQuicListener),
105
+ context: ctx, contextCancel: cancel,
106
+ }, err
107
+}
108
+
109
// fakeCloseListener is a private wrapper over a listener that
110
// is shared. The state of fakeCloseListener is not shared.
111
// This allows one user of a socket to "close" the listener
@@ -95,7 +122,7 @@ type fakeCloseListener struct {
122
func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
123
// if the listener is already "closed", return error
124
if atomic.LoadInt32(&fcl.closed) == 1 {
98
- return nil, fcl.fakeClosedErr()
125
+ return nil, fakeClosedErr(fcl)
126
}
127
128
// call underlying accept
@@ -119,7 +146,7 @@ func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
146
_ = fcl.sharedListener.clearDeadline()
147
148
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
122
- return nil, fcl.fakeClosedErr()
149
+ return nil, fakeClosedErr(fcl)
150
}
151
}
152
@@ -145,14 +172,47 @@ func (fcl *fakeCloseListener) Close() error {
172
return nil
173
}
174
175
+type fakeCloseQuicListener struct {
176
+ closed int32 // accessed atomically; belongs to this struct only
177
+ *sharedQuicListener // embedded, so we also become a quic.EarlyListener
178
+ context context.Context
179
+ contextCancel context.CancelFunc
180
+}
181
+
182
+// Currently Accept ignores the passed context, however a situation where
183
+// someone would need a hotswappable QUIC-only (not http3, since it uses context.Background here)
184
+// server on which Accept would be called with non-empty contexts
185
+// (mind that the default net listeners' Accept doesn't take a context argument)
186
+// sounds way too rare for us to sacrifice efficiency here.
187
+func (fcql *fakeCloseQuicListener) Accept(_ context.Context) (quic.EarlySession, error) {
188
+ conn, err := fcql.sharedQuicListener.Accept(fcql.context)
189
+ if err == nil {
190
+ return conn, nil
191
+ }
192
+
193
+ // if the listener is "closed", return a fake closed error instead
194
+ if atomic.LoadInt32(&fcql.closed) == 1 && errors.Is(err, context.Canceled) {
195
+ return nil, fakeClosedErr(fcql)
196
+ }
197
+ return nil, err
198
+}
199
+
200
+func (fcql *fakeCloseQuicListener) Close() error {
201
+ if atomic.CompareAndSwapInt32(&fcql.closed, 0, 1) {
202
+ fcql.contextCancel()
203
+ _, _ = listenerPool.Delete(fcql.sharedQuicListener.key)
204
+ }
205
+ return nil
206
+}
207
+
208
// fakeClosedErr returns an error value that is not temporary
209
// nor a timeout, suitable for making the caller think the
210
// listener is actually closed
151
-func (fcl *fakeCloseListener) fakeClosedErr() error {
211
+func fakeClosedErr(l interface{ Addr() net.Addr }) error {
212
return &net.OpError{
213
Op: "accept",
154
- Net: fcl.Addr().Network(),
155
- Addr: fcl.Addr(),
214
+ Net: l.Addr().Network(),
215
+ Addr: l.Addr(),
216
Err: errFakeClosed,
217
}
218
}
@@ -244,6 +304,17 @@ func (sl *sharedListener) Destruct() error {
304
return sl.Listener.Close()
305
}
306
307
+// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
308
+type sharedQuicListener struct {
309
+ quic.EarlyListener
310
+ key string
311
+}
312
+
313
+// Destruct closes the underlying QUIC listener.
314
+func (sql *sharedQuicListener) Destruct() error {
315
+ return sql.EarlyListener.Close()
316
+}
317
+
318
// sharedPacketConn is like sharedListener, but for net.PacketConns.
319
type sharedPacketConn struct {
320
net.PacketConn
modules/caddyhttp/app.go
+5
-26
@@ -18,7 +18,6 @@ import (
18
"context"
19
"crypto/tls"
20
"fmt"
21
- "net"
21
"net/http"
22
"strconv"
23
"time"
@@ -116,9 +115,8 @@ type App struct {
115
// affect functionality.
116
Servers map[string]*Server `json:"servers,omitempty"`
117
119
- servers []*http.Server
120
- h3servers []*http3.Server
121
- h3listeners []net.PacketConn
118
+ servers []*http.Server
119
+ h3servers []*http3.Server
120
121
ctx caddy.Context
122
logger *zap.Logger
@@ -353,9 +351,9 @@ func (app *App) Start() error {
351
app.logger.Info("enabling experimental HTTP/3 listener",
352
zap.String("addr", hostport),
353
)
356
- h3ln, err := caddy.ListenPacket("udp", hostport)
354
+ h3ln, err := caddy.ListenQUIC(hostport, tlsCfg)
355
if err != nil {
358
- return fmt.Errorf("getting HTTP/3 UDP listener: %v", err)
356
+ return fmt.Errorf("getting HTTP/3 QUIC listener: %v", err)
357
}
358
h3srv := &http3.Server{
359
Server: &http.Server{
@@ -366,9 +364,8 @@ func (app *App) Start() error {
364
},
365
}
366
//nolint:errcheck
369
- go h3srv.Serve(h3ln)
367
+ go h3srv.ServeListener(h3ln)
368
app.h3servers = append(app.h3servers, h3srv)
371
- app.h3listeners = append(app.h3listeners, h3ln)
369
srv.h3server = h3srv
370
}
371
/////////
@@ -426,13 +423,6 @@ func (app *App) Stop() error {
423
}
424
}
425
429
- // close the http3 servers; it's unclear whether the bug reported in
430
- // https://github.com/caddyserver/caddy/pull/2727#issuecomment-526856566
431
- // was ever truly fixed, since it seemed racey/nondeterministic; but
432
- // recent tests in 2020 were unable to replicate the issue again after
433
- // repeated attempts (the bug manifested after a config reload; i.e.
434
- // reusing a http3 server or listener was problematic), but it seems
435
- // to be working fine now
426
for _, s := range app.h3servers {
427
// TODO: CloseGracefully, once implemented upstream
428
// (see https://github.com/lucas-clemente/quic-go/issues/2103)
@@ -441,17 +431,6 @@ func (app *App) Stop() error {
431
return err
432
}
433
}
444
-
445
- // closing an http3.Server does not close their underlying listeners
446
- // since apparently the listener can be used both by servers and
447
- // clients at the same time; so we need to manually call Close()
448
- // on the underlying h3 listeners (see lucas-clemente/quic-go#2103)
449
- for _, pc := range app.h3listeners {
450
- err := pc.Close()
451
- if err != nil {
452
- return err
453
- }
454
- }
434
return nil
435
}
436