Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 14:15 UTC
core: Refactor listeners; use SO_REUSEPORT on Unix (#4705)
* core: Refactor listeners; use SO_REUSEPORT on Unix Just an experiment for now * Fix lint by logging error * TCP Keepalive configuration (#4865) * initial attempt at TCP Keepalive configuration * core: implement tcp-keepalive for linux * move canSetKeepAlive interface * Godoc for keepalive server parameter * handle return values * log keepalive errors * Clean up after bad merge * Merge in pluggable network types From 1edc1a45e3aee1f7d86b68c3ddaf2fd16ba8ab73 * Slight refactor, fix from recent merge conflict Co-authored-by: Karmanyaah Malhotra <karmanyaah.gh@malhotra.cc>
Matt Holt committed
Sep 2, 2022 at 16:59 UTC
d3c3fa10bd72029720969cff0c29e2b79a1b2cdf
6 files changed
+234
-138
caddyconfig/httpcaddyfile/serveroptions.go
+11
@@ -38,6 +38,7 @@ type serverOptions struct {
38
ReadHeaderTimeout caddy.Duration
39
WriteTimeout caddy.Duration
40
IdleTimeout caddy.Duration
41
+ KeepAliveInterval caddy.Duration
42
MaxHeaderBytes int
43
Protocols []string
44
StrictSNIHost *bool
@@ -122,6 +123,15 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
123
return nil, d.Errf("unrecognized timeouts option '%s'", d.Val())
124
}
125
}
126
+ case "keepalive_interval":
127
+ if !d.NextArg() {
128
+ return nil, d.ArgErr()
129
+ }
130
+ dur, err := caddy.ParseDuration(d.Val())
131
+ if err != nil {
132
+ return nil, d.Errf("parsing keepalive interval duration: %v", err)
133
+ }
134
+ serverOpts.KeepAliveInterval = caddy.Duration(dur)
135
136
case "max_header_size":
137
var sizeStr string
@@ -245,6 +255,7 @@ func applyServerOptions(
255
server.ReadHeaderTimeout = opts.ReadHeaderTimeout
256
server.WriteTimeout = opts.WriteTimeout
257
server.IdleTimeout = opts.IdleTimeout
258
+ server.KeepAliveInterval = opts.KeepAliveInterval
259
server.MaxHeaderBytes = opts.MaxHeaderBytes
260
server.Protocols = opts.Protocols
261
server.StrictSNIHost = opts.StrictSNIHost
listen.go
new
+172
@@ -0,0 +1,172 @@
1
+//go:build !linux
2
+
3
+package caddy
4
+
5
+import (
6
+ "fmt"
7
+ "net"
8
+ "sync"
9
+ "sync/atomic"
10
+ "time"
11
+
12
+ "go.uber.org/zap"
13
+)
14
+
15
+func ListenTimeout(network, addr string, keepAlivePeriod time.Duration) (net.Listener, error) {
16
+ // check to see if plugin provides listener
17
+ if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
18
+ return ln, err
19
+ }
20
+
21
+ lnKey := listenerKey(network, addr)
22
+
23
+ sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
24
+ ln, err := net.Listen(network, addr)
25
+ if err != nil {
26
+ // https://github.com/caddyserver/caddy/pull/4534
27
+ if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
28
+ return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
29
+ }
30
+ return nil, err
31
+ }
32
+ return &sharedListener{Listener: ln, key: lnKey}, nil
33
+ })
34
+ if err != nil {
35
+ return nil, err
36
+ }
37
+
38
+ return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener), keepAlivePeriod: keepAlivePeriod}, nil
39
+}
40
+
41
+// fakeCloseListener is a private wrapper over a listener that
42
+// is shared. The state of fakeCloseListener is not shared.
43
+// This allows one user of a socket to "close" the listener
44
+// while in reality the socket stays open for other users of
45
+// the listener. In this way, servers become hot-swappable
46
+// while the listener remains running. Listeners should be
47
+// re-wrapped in a new fakeCloseListener each time the listener
48
+// is reused. This type is atomic and values must not be copied.
49
+type fakeCloseListener struct {
50
+ closed int32 // accessed atomically; belongs to this struct only
51
+ *sharedListener // embedded, so we also become a net.Listener
52
+ keepAlivePeriod time.Duration
53
+}
54
+
55
+type canSetKeepAlive interface {
56
+ SetKeepAlivePeriod(d time.Duration) error
57
+ SetKeepAlive(bool) error
58
+}
59
+
60
+func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
61
+ // if the listener is already "closed", return error
62
+ if atomic.LoadInt32(&fcl.closed) == 1 {
63
+ return nil, fakeClosedErr(fcl)
64
+ }
65
+
66
+ // call underlying accept
67
+ conn, err := fcl.sharedListener.Accept()
68
+ if err == nil {
69
+ // if 0, do nothing, Go's default is already set
70
+ // and if the connection allows setting KeepAlive, set it
71
+ if tconn, ok := conn.(canSetKeepAlive); ok && fcl.keepAlivePeriod != 0 {
72
+ if fcl.keepAlivePeriod > 0 {
73
+ err = tconn.SetKeepAlivePeriod(fcl.keepAlivePeriod)
74
+ } else { // negative
75
+ err = tconn.SetKeepAlive(false)
76
+ }
77
+ if err != nil {
78
+ Log().With(zap.String("server", fcl.sharedListener.key)).Warn("unable to set keepalive for new connection:", zap.Error(err))
79
+ }
80
+ }
81
+ return conn, nil
82
+ }
83
+
84
+ // since Accept() returned an error, it may be because our reference to
85
+ // the listener (this fakeCloseListener) may have been closed, i.e. the
86
+ // server is shutting down; in that case, we need to clear the deadline
87
+ // that we set when Close() was called, and return a non-temporary and
88
+ // non-timeout error value to the caller, masking the "true" error, so
89
+ // that server loops / goroutines won't retry, linger, and leak
90
+ if atomic.LoadInt32(&fcl.closed) == 1 {
91
+ // we dereference the sharedListener explicitly even though it's embedded
92
+ // so that it's clear in the code that side-effects are shared with other
93
+ // users of this listener, not just our own reference to it; we also don't
94
+ // do anything with the error because all we could do is log it, but we
95
+ // expliclty assign it to nothing so we don't forget it's there if needed
96
+ _ = fcl.sharedListener.clearDeadline()
97
+
98
+ if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
99
+ return nil, fakeClosedErr(fcl)
100
+ }
101
+ }
102
+
103
+ return nil, err
104
+}
105
+
106
+// Close stops accepting new connections without closing the
107
+// underlying listener. The underlying listener is only closed
108
+// if the caller is the last known user of the socket.
109
+func (fcl *fakeCloseListener) Close() error {
110
+ if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
111
+ // There are two ways I know of to get an Accept()
112
+ // function to return to the server loop that called
113
+ // it: close the listener, or set a deadline in the
114
+ // past. Obviously, we can't close the socket yet
115
+ // since others may be using it (hence this whole
116
+ // file). But we can set the deadline in the past,
117
+ // and this is kind of cheating, but it works, and
118
+ // it apparently even works on Windows.
119
+ _ = fcl.sharedListener.setDeadline()
120
+ _, _ = listenerPool.Delete(fcl.sharedListener.key)
121
+ }
122
+ return nil
123
+}
124
+
125
+// sharedListener is a wrapper over an underlying listener. The listener
126
+// and the other fields on the struct are shared state that is synchronized,
127
+// so sharedListener structs must never be copied (always use a pointer).
128
+type sharedListener struct {
129
+ net.Listener
130
+ key string // uniquely identifies this listener
131
+ deadline bool // whether a deadline is currently set
132
+ deadlineMu sync.Mutex
133
+}
134
+
135
+func (sl *sharedListener) clearDeadline() error {
136
+ var err error
137
+ sl.deadlineMu.Lock()
138
+ if sl.deadline {
139
+ switch ln := sl.Listener.(type) {
140
+ case *net.TCPListener:
141
+ err = ln.SetDeadline(time.Time{})
142
+ case *net.UnixListener:
143
+ err = ln.SetDeadline(time.Time{})
144
+ }
145
+ sl.deadline = false
146
+ }
147
+ sl.deadlineMu.Unlock()
148
+ return err
149
+}
150
+
151
+func (sl *sharedListener) setDeadline() error {
152
+ timeInPast := time.Now().Add(-1 * time.Minute)
153
+ var err error
154
+ sl.deadlineMu.Lock()
155
+ if !sl.deadline {
156
+ switch ln := sl.Listener.(type) {
157
+ case *net.TCPListener:
158
+ err = ln.SetDeadline(timeInPast)
159
+ case *net.UnixListener:
160
+ err = ln.SetDeadline(timeInPast)
161
+ }
162
+ sl.deadline = true
163
+ }
164
+ sl.deadlineMu.Unlock()
165
+ return err
166
+}
167
+
168
+// Destruct is called by the UsagePool when the listener is
169
+// finally not being used anymore. It closes the socket.
170
+func (sl *sharedListener) Destruct() error {
171
+ return sl.Listener.Close()
172
+}
listen_linux.go
new
+34
@@ -0,0 +1,34 @@
1
+package caddy
2
+
3
+import (
4
+ "context"
5
+ "net"
6
+ "syscall"
7
+ "time"
8
+
9
+ "go.uber.org/zap"
10
+ "golang.org/x/sys/unix"
11
+)
12
+
13
+// ListenTimeout is the same as Listen, but with a configurable keep-alive timeout duration.
14
+func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) {
15
+ // check to see if plugin provides listener
16
+ if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
17
+ return ln, err
18
+ }
19
+
20
+ config := &net.ListenConfig{Control: reusePort, KeepAlive: keepalivePeriod}
21
+ return config.Listen(context.Background(), network, addr)
22
+}
23
+
24
+func reusePort(network, address string, conn syscall.RawConn) error {
25
+ return conn.Control(func(descriptor uintptr) {
26
+ if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
27
+ Log().Error("setting SO_REUSEPORT",
28
+ zap.String("network", network),
29
+ zap.String("address", address),
30
+ zap.Uintptr("descriptor", descriptor),
31
+ zap.Error(err))
32
+ }
33
+ })
34
+}
listeners.go
+11
-137
@@ -24,10 +24,8 @@ import (
24
"os"
25
"strconv"
26
"strings"
27
- "sync"
27
"sync/atomic"
28
"syscall"
30
- "time"
29
30
"github.com/lucas-clemente/quic-go"
31
"github.com/lucas-clemente/quic-go/http3"
@@ -43,6 +41,14 @@ import (
41
// the socket have been finished. Always be sure to close listeners
42
// when you are done with them, just like normal listeners.
43
func Listen(network, addr string) (net.Listener, error) {
44
+ // a 0 timeout means Go uses its default
45
+ return ListenTimeout(network, addr, 0)
46
+}
47
+
48
+// getListenerFromPlugin returns a listener on the given network and address
49
+// if a plugin has registered the network name. It may return (nil, nil) if
50
+// no plugin can provide a listener.
51
+func getListenerFromPlugin(network, addr string) (net.Listener, error) {
52
network = strings.TrimSpace(strings.ToLower(network))
53
54
// get listener from plugin if network type is registered
@@ -51,24 +57,7 @@ func Listen(network, addr string) (net.Listener, error) {
57
return getListener(network, addr)
58
}
59
54
- lnKey := listenerKey(network, addr)
55
-
56
- sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
57
- ln, err := net.Listen(network, addr)
58
- if err != nil {
59
- // https://github.com/caddyserver/caddy/pull/4534
60
- if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
61
- return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
62
- }
63
- return nil, err
64
- }
65
- return &sharedListener{Listener: ln, key: lnKey}, nil
66
- })
67
- if err != nil {
68
- return nil, err
69
- }
70
-
71
- return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener)}, nil
60
+ return nil, nil
61
}
62
63
// ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
@@ -124,80 +113,14 @@ func ListenQUIC(addr string, tlsConf *tls.Config, activeRequests *int64) (quic.E
113
}, err
114
}
115
127
-func listenerKey(network, addr string) string {
128
- return network + "/" + addr
129
-}
130
-
116
// ListenerUsage returns the current usage count of the given listener address.
117
func ListenerUsage(network, addr string) int {
118
count, _ := listenerPool.References(listenerKey(network, addr))
119
return count
120
}
121
137
-// fakeCloseListener is a private wrapper over a listener that
138
-// is shared. The state of fakeCloseListener is not shared.
139
-// This allows one user of a socket to "close" the listener
140
-// while in reality the socket stays open for other users of
141
-// the listener. In this way, servers become hot-swappable
142
-// while the listener remains running. Listeners should be
143
-// re-wrapped in a new fakeCloseListener each time the listener
144
-// is reused. This type is atomic and values must not be copied.
145
-type fakeCloseListener struct {
146
- closed int32 // accessed atomically; belongs to this struct only
147
- *sharedListener // embedded, so we also become a net.Listener
148
-}
149
-
150
-func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
151
- // if the listener is already "closed", return error
152
- if atomic.LoadInt32(&fcl.closed) == 1 {
153
- return nil, fakeClosedErr(fcl)
154
- }
155
-
156
- // call underlying accept
157
- conn, err := fcl.sharedListener.Accept()
158
- if err == nil {
159
- return conn, nil
160
- }
161
-
162
- // since Accept() returned an error, it may be because our reference to
163
- // the listener (this fakeCloseListener) may have been closed, i.e. the
164
- // server is shutting down; in that case, we need to clear the deadline
165
- // that we set when Close() was called, and return a non-temporary and
166
- // non-timeout error value to the caller, masking the "true" error, so
167
- // that server loops / goroutines won't retry, linger, and leak
168
- if atomic.LoadInt32(&fcl.closed) == 1 {
169
- // we dereference the sharedListener explicitly even though it's embedded
170
- // so that it's clear in the code that side-effects are shared with other
171
- // users of this listener, not just our own reference to it; we also don't
172
- // do anything with the error because all we could do is log it, but we
173
- // expliclty assign it to nothing so we don't forget it's there if needed
174
- _ = fcl.sharedListener.clearDeadline()
175
-
176
- if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
177
- return nil, fakeClosedErr(fcl)
178
- }
179
- }
180
-
181
- return nil, err
182
-}
183
-
184
-// Close stops accepting new connections without closing the
185
-// underlying listener. The underlying listener is only closed
186
-// if the caller is the last known user of the socket.
187
-func (fcl *fakeCloseListener) Close() error {
188
- if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
189
- // There are two ways I know of to get an Accept()
190
- // function to return to the server loop that called
191
- // it: close the listener, or set a deadline in the
192
- // past. Obviously, we can't close the socket yet
193
- // since others may be using it (hence this whole
194
- // file). But we can set the deadline in the past,
195
- // and this is kind of cheating, but it works, and
196
- // it apparently even works on Windows.
197
- _ = fcl.sharedListener.setDeadline()
198
- _, _ = listenerPool.Delete(fcl.sharedListener.key)
199
- }
200
- return nil
122
+func listenerKey(network, addr string) string {
123
+ return network + "/" + addr
124
}
125
126
type fakeCloseQuicListener struct {
@@ -283,55 +206,6 @@ func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
206
return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
207
}
208
286
-// sharedListener is a wrapper over an underlying listener. The listener
287
-// and the other fields on the struct are shared state that is synchronized,
288
-// so sharedListener structs must never be copied (always use a pointer).
289
-type sharedListener struct {
290
- net.Listener
291
- key string // uniquely identifies this listener
292
- deadline bool // whether a deadline is currently set
293
- deadlineMu sync.Mutex
294
-}
295
-
296
-func (sl *sharedListener) clearDeadline() error {
297
- var err error
298
- sl.deadlineMu.Lock()
299
- if sl.deadline {
300
- switch ln := sl.Listener.(type) {
301
- case *net.TCPListener:
302
- err = ln.SetDeadline(time.Time{})
303
- case *net.UnixListener:
304
- err = ln.SetDeadline(time.Time{})
305
- }
306
- sl.deadline = false
307
- }
308
- sl.deadlineMu.Unlock()
309
- return err
310
-}
311
-
312
-func (sl *sharedListener) setDeadline() error {
313
- timeInPast := time.Now().Add(-1 * time.Minute)
314
- var err error
315
- sl.deadlineMu.Lock()
316
- if !sl.deadline {
317
- switch ln := sl.Listener.(type) {
318
- case *net.TCPListener:
319
- err = ln.SetDeadline(timeInPast)
320
- case *net.UnixListener:
321
- err = ln.SetDeadline(timeInPast)
322
- }
323
- sl.deadline = true
324
- }
325
- sl.deadlineMu.Unlock()
326
- return err
327
-}
328
-
329
-// Destruct is called by the UsagePool when the listener is
330
-// finally not being used anymore. It closes the socket.
331
-func (sl *sharedListener) Destruct() error {
332
- return sl.Listener.Close()
333
-}
334
-
209
// sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
210
type sharedQuicListener struct {
211
quic.EarlyListener
modules/caddyhttp/app.go
+1
-1
@@ -384,7 +384,7 @@ func (app *App) Start() error {
384
for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
385
// create the listener for this socket
386
hostport := listenAddr.JoinHostPort(portOffset)
387
- ln, err := caddy.Listen(listenAddr.Network, hostport)
387
+ ln, err := caddy.ListenTimeout(listenAddr.Network, hostport, time.Duration(srv.KeepAliveInterval))
388
if err != nil {
389
return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err)
390
}
modules/caddyhttp/server.go
+5
@@ -72,6 +72,11 @@ type Server struct {
72
// 5m is applied to help avoid resource exhaustion.
73
IdleTimeout caddy.Duration `json:"idle_timeout,omitempty"`
74
75
+ // KeepAliveInterval is the interval at which TCP keepalive packets
76
+ // are sent to keep the connection alive at the TCP layer when no other
77
+ // data is being transmitted. The default is 15s.
78
+ KeepAliveInterval caddy.Duration `json:"keepalive_interval,omitempty"`
79
+
80
// MaxHeaderBytes is the maximum size to parse from a client's
81
// HTTP request headers.
82
MaxHeaderBytes int `json:"max_header_bytes,omitempty"`