Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 22:00 UTC
core: quic listener will manage the underlying socket by itself (#5749)
* core: quic listener will manage the underlying socket by itself. * format code * rename sharedQUICTLSConfig to sharedQUICState, and it will now manage the number of active requests * add comment * strict unwrap type * fix unwrap * remove comment
WeidiDeng committed
Oct 16, 2023 at 23:28 UTC
7c82e265da8f90c1a1fee52c57af94587276ac56
3 files changed
+126
-70
listeners.go
+125
-49
@@ -470,38 +470,90 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
470
// unixgram will be used; otherwise, udp will be used).
471
//
472
// NOTE: This API is EXPERIMENTAL and may be changed or removed.
473
-//
473
+func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config, activeRequests *int64) (http3.QUICEarlyListener, error) {
474
+ lnKey := listenerKey("quic"+na.Network, na.JoinHostPort(portOffset))
475
+
476
+ sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
477
+ lnAny, err := na.Listen(ctx, portOffset, config)
478
+ if err != nil {
479
+ return nil, err
480
+ }
481
+
482
+ ln := lnAny.(net.PacketConn)
483
+
484
+ h3ln := ln
485
+ for {
486
+ // retrieve the underlying socket, so quic-go can optimize.
487
+ if unwrapper, ok := h3ln.(interface{ Unwrap() net.PacketConn }); ok {
488
+ h3ln = unwrapper.Unwrap()
489
+ } else {
490
+ break
491
+ }
492
+ }
493
+
494
+ sqs := newSharedQUICState(tlsConf, activeRequests)
495
+ // http3.ConfigureTLSConfig only uses this field and tls App sets this field as well
496
+ //nolint:gosec
497
+ quicTlsConfig := &tls.Config{GetConfigForClient: sqs.getConfigForClient}
498
+ earlyLn, err := quic.ListenEarly(h3ln, http3.ConfigureTLSConfig(quicTlsConfig), &quic.Config{
499
+ Allow0RTT: true,
500
+ RequireAddressValidation: func(clientAddr net.Addr) bool {
501
+ // TODO: make tunable?
502
+ return sqs.getActiveRequests() > 1000
503
+ },
504
+ })
505
+ if err != nil {
506
+ return nil, err
507
+ }
508
+ // using the original net.PacketConn to close them properly
509
+ return &sharedQuicListener{EarlyListener: earlyLn, packetConn: ln, sqs: sqs, key: lnKey}, nil
510
+ })
511
+ if err != nil {
512
+ return nil, err
513
+ }
514
+
515
+ sql := sharedEarlyListener.(*sharedQuicListener)
516
+ // add current tls.Config to sqs, so GetConfigForClient will always return the latest tls.Config in case of context cancellation,
517
+ // and the request counter will reflect current http server
518
+ ctx, cancel := sql.sqs.addState(tlsConf, activeRequests)
519
+
520
+ return &fakeCloseQuicListener{
521
+ sharedQuicListener: sql,
522
+ context: ctx,
523
+ contextCancel: cancel,
524
+ }, nil
525
+}
526
+
527
+// DEPRECATED: Use NetworkAddress.ListenQUIC instead. This function will likely be changed or removed in the future.
528
// TODO: See if we can find a more elegant solution closer to the new NetworkAddress.Listen API.
529
func ListenQUIC(ln net.PacketConn, tlsConf *tls.Config, activeRequests *int64) (http3.QUICEarlyListener, error) {
530
lnKey := listenerKey("quic+"+ln.LocalAddr().Network(), ln.LocalAddr().String())
531
532
sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
479
- sqtc := newSharedQUICTLSConfig(tlsConf)
533
+ sqs := newSharedQUICState(tlsConf, activeRequests)
534
// http3.ConfigureTLSConfig only uses this field and tls App sets this field as well
535
//nolint:gosec
482
- quicTlsConfig := &tls.Config{GetConfigForClient: sqtc.getConfigForClient}
536
+ quicTlsConfig := &tls.Config{GetConfigForClient: sqs.getConfigForClient}
537
earlyLn, err := quic.ListenEarly(ln, http3.ConfigureTLSConfig(quicTlsConfig), &quic.Config{
538
Allow0RTT: true,
539
RequireAddressValidation: func(clientAddr net.Addr) bool {
486
- var highLoad bool
487
- if activeRequests != nil {
488
- highLoad = atomic.LoadInt64(activeRequests) > 1000 // TODO: make tunable?
489
- }
490
- return highLoad
540
+ // TODO: make tunable?
541
+ return sqs.getActiveRequests() > 1000
542
},
543
})
544
if err != nil {
545
return nil, err
546
}
496
- return &sharedQuicListener{EarlyListener: earlyLn, sqtc: sqtc, key: lnKey}, nil
547
+ return &sharedQuicListener{EarlyListener: earlyLn, sqs: sqs, key: lnKey}, nil
548
})
549
if err != nil {
550
return nil, err
551
}
552
553
sql := sharedEarlyListener.(*sharedQuicListener)
503
- // add current tls.Config to sqtc, so GetConfigForClient will always return the latest tls.Config in case of context cancellation
504
- ctx, cancel := sql.sqtc.addTLSConfig(tlsConf)
554
+ // add current tls.Config and request counter to sqs, so GetConfigForClient will always return the latest tls.Config in case of context cancellation,
555
+ // and the request counter will reflect current http server
556
+ ctx, cancel := sql.sqs.addState(tlsConf, activeRequests)
557
558
// TODO: to serve QUIC over a unix socket, currently we need to hold onto
559
// the underlying net.PacketConn (which we wrap as unixConn to keep count
@@ -534,38 +586,50 @@ type contextAndCancelFunc struct {
586
context.CancelFunc
587
}
588
537
-// sharedQUICTLSConfig manages GetConfigForClient
589
+// sharedQUICState manages GetConfigForClient and current number of active requests
590
// see issue: https://github.com/caddyserver/caddy/pull/4849
539
-type sharedQUICTLSConfig struct {
540
- rmu sync.RWMutex
541
- tlsConfs map[*tls.Config]contextAndCancelFunc
542
- activeTlsConf *tls.Config
543
-}
544
-
545
-// newSharedQUICTLSConfig creates a new sharedQUICTLSConfig
546
-func newSharedQUICTLSConfig(tlsConfig *tls.Config) *sharedQUICTLSConfig {
547
- sqtc := &sharedQUICTLSConfig{
548
- tlsConfs: make(map[*tls.Config]contextAndCancelFunc),
549
- activeTlsConf: tlsConfig,
550
- }
551
- sqtc.addTLSConfig(tlsConfig)
591
+type sharedQUICState struct {
592
+ rmu sync.RWMutex
593
+ tlsConfs map[*tls.Config]contextAndCancelFunc
594
+ requestCounters map[*tls.Config]*int64
595
+ activeTlsConf *tls.Config
596
+ activeRequestsCounter *int64
597
+}
598
+
599
+// newSharedQUICState creates a new sharedQUICState
600
+func newSharedQUICState(tlsConfig *tls.Config, activeRequests *int64) *sharedQUICState {
601
+ sqtc := &sharedQUICState{
602
+ tlsConfs: make(map[*tls.Config]contextAndCancelFunc),
603
+ requestCounters: make(map[*tls.Config]*int64),
604
+ activeTlsConf: tlsConfig,
605
+ activeRequestsCounter: activeRequests,
606
+ }
607
+ sqtc.addState(tlsConfig, activeRequests)
608
return sqtc
609
}
610
611
// getConfigForClient is used as tls.Config's GetConfigForClient field
556
-func (sqtc *sharedQUICTLSConfig) getConfigForClient(ch *tls.ClientHelloInfo) (*tls.Config, error) {
557
- sqtc.rmu.RLock()
558
- defer sqtc.rmu.RUnlock()
559
- return sqtc.activeTlsConf.GetConfigForClient(ch)
612
+func (sqs *sharedQUICState) getConfigForClient(ch *tls.ClientHelloInfo) (*tls.Config, error) {
613
+ sqs.rmu.RLock()
614
+ defer sqs.rmu.RUnlock()
615
+ return sqs.activeTlsConf.GetConfigForClient(ch)
616
+}
617
+
618
+// getActiveRequests returns the number of active requests
619
+func (sqs *sharedQUICState) getActiveRequests() int64 {
620
+ // Prevent a race when a context is cancelled and active request counter is being changed
621
+ sqs.rmu.RLock()
622
+ defer sqs.rmu.RUnlock()
623
+ return atomic.LoadInt64(sqs.activeRequestsCounter)
624
}
625
562
-// addTLSConfig adds tls.Config to the map if not present and returns the corresponding context and its cancelFunc
563
-// so that when cancelled, the active tls.Config will change
564
-func (sqtc *sharedQUICTLSConfig) addTLSConfig(tlsConfig *tls.Config) (context.Context, context.CancelFunc) {
565
- sqtc.rmu.Lock()
566
- defer sqtc.rmu.Unlock()
626
+// addState adds tls.Config and activeRequests to the map if not present and returns the corresponding context and its cancelFunc
627
+// so that when cancelled, the active tls.Config and request counter will change
628
+func (sqs *sharedQUICState) addState(tlsConfig *tls.Config, activeRequests *int64) (context.Context, context.CancelFunc) {
629
+ sqs.rmu.Lock()
630
+ defer sqs.rmu.Unlock()
631
568
- if cacc, ok := sqtc.tlsConfs[tlsConfig]; ok {
632
+ if cacc, ok := sqs.tlsConfs[tlsConfig]; ok {
633
return cacc.Context, cacc.CancelFunc
634
}
635
@@ -573,23 +637,26 @@ func (sqtc *sharedQUICTLSConfig) addTLSConfig(tlsConfig *tls.Config) (context.Co
637
wrappedCancel := func() {
638
cancel()
639
576
- sqtc.rmu.Lock()
577
- defer sqtc.rmu.Unlock()
640
+ sqs.rmu.Lock()
641
+ defer sqs.rmu.Unlock()
642
579
- delete(sqtc.tlsConfs, tlsConfig)
580
- if sqtc.activeTlsConf == tlsConfig {
581
- // select another tls.Config, if there is none,
643
+ delete(sqs.tlsConfs, tlsConfig)
644
+ delete(sqs.requestCounters, tlsConfig)
645
+ if sqs.activeTlsConf == tlsConfig {
646
+ // select another tls.Config and request counter, if there is none,
647
// related sharedQuicListener will be destroyed anyway
583
- for tc := range sqtc.tlsConfs {
584
- sqtc.activeTlsConf = tc
648
+ for tc, counter := range sqs.requestCounters {
649
+ sqs.activeTlsConf = tc
650
+ sqs.activeRequestsCounter = counter
651
break
652
}
653
}
654
}
589
- sqtc.tlsConfs[tlsConfig] = contextAndCancelFunc{ctx, wrappedCancel}
655
+ sqs.tlsConfs[tlsConfig] = contextAndCancelFunc{ctx, wrappedCancel}
656
+ sqs.requestCounters[tlsConfig] = activeRequests
657
// there should be at most 2 tls.Configs
591
- if len(sqtc.tlsConfs) > 2 {
592
- Log().Warn("quic listener tls configs are more than 2", zap.Int("number of configs", len(sqtc.tlsConfs)))
658
+ if len(sqs.tlsConfs) > 2 {
659
+ Log().Warn("quic listener tls configs are more than 2", zap.Int("number of configs", len(sqs.tlsConfs)))
660
}
661
return ctx, wrappedCancel
662
}
@@ -597,13 +664,17 @@ func (sqtc *sharedQUICTLSConfig) addTLSConfig(tlsConfig *tls.Config) (context.Co
664
// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
665
type sharedQuicListener struct {
666
*quic.EarlyListener
600
- sqtc *sharedQUICTLSConfig
601
- key string
667
+ packetConn net.PacketConn // we have to hold these because quic-go won't close listeners it didn't create
668
+ sqs *sharedQUICState
669
+ key string
670
}
671
604
-// Destruct closes the underlying QUIC listener.
672
+// Destruct closes the underlying QUIC listener and its associated net.PacketConn.
673
func (sql *sharedQuicListener) Destruct() error {
606
- return sql.EarlyListener.Close()
674
+ // close EarlyListener first to stop any operations being done to the net.PacketConn
675
+ _ = sql.EarlyListener.Close()
676
+ // then close the net.PacketConn
677
+ return sql.packetConn.Close()
678
}
679
680
// sharedPacketConn is like sharedListener, but for net.PacketConns.
@@ -652,6 +723,11 @@ var _ quic.OOBCapablePacketConn = (*fakeClosePacketConn)(nil)
723
// but doesn't actually use these methods, the only methods needed are `ReadMsgUDP` and `SyscallConn`.
724
var _ net.Conn = (*fakeClosePacketConn)(nil)
725
726
+// Unwrap returns the underlying net.UDPConn for quic-go optimization
727
+func (fcpc *fakeClosePacketConn) Unwrap() any {
728
+ return fcpc.UDPConn
729
+}
730
+
731
// Close won't close the underlying socket unless there is no more reference, then listenerPool will close it.
732
func (fcpc *fakeClosePacketConn) Close() error {
733
if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {
modules/caddyhttp/app.go
-11
@@ -617,17 +617,6 @@ func (app *App) Stop() error {
617
zap.Error(err),
618
zap.Strings("addresses", server.Listen))
619
}
620
-
621
- // TODO: we have to manually close our listeners because quic-go won't
622
- // close listeners it didn't create along with the server itself...
623
- // see https://github.com/quic-go/quic-go/issues/3560
624
- for _, el := range server.h3listeners {
625
- if err := el.Close(); err != nil {
626
- app.logger.Error("HTTP/3 listener close",
627
- zap.Error(err),
628
- zap.String("address", el.LocalAddr().String()))
629
- }
630
- }
620
}
621
stopH2Listener := func(server *Server) {
622
defer finishedShutdown.Done()
modules/caddyhttp/server.go
+1
-10
@@ -228,7 +228,6 @@ type Server struct {
228
229
server *http.Server
230
h3server *http3.Server
231
- h3listeners []net.PacketConn // TODO: we have to hold these because quic-go won't close listeners it didn't create
231
h2listeners []*http2Listener
232
addresses []caddy.NetworkAddress
233
@@ -555,13 +554,7 @@ func (s *Server) findLastRouteWithHostMatcher() int {
554
// the listener, with Server s as the handler.
555
func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error {
556
addr.Network = getHTTP3Network(addr.Network)
558
- lnAny, err := addr.Listen(s.ctx, 0, net.ListenConfig{})
559
- if err != nil {
560
- return err
561
- }
562
- ln := lnAny.(net.PacketConn)
563
-
564
- h3ln, err := caddy.ListenQUIC(ln, tlsCfg, &s.activeRequests)
557
+ h3ln, err := addr.ListenQUIC(s.ctx, 0, net.ListenConfig{}, tlsCfg, &s.activeRequests)
558
if err != nil {
559
return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err)
560
}
@@ -579,8 +572,6 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
572
}
573
}
574
582
- s.h3listeners = append(s.h3listeners, ln)
583
-
575
//nolint:errcheck
576
go s.h3server.ServeListener(h3ln)
577