@marcel91200 / caddy Mirror · read-only / commits / 64a3218f
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 14, 00:15 UTC

core: Simplify shared listeners, fix deadline bug

When this listener code was first written, UsagePool didn't exist. We can simplify much of the wrapped listener logic by utilizing UsagePool. This also fixes a bug where new servers were able to clear deadlines set by old servers, even if the old server didn't get booted out of its Accept() call yet. And with the deadline cleared, they never would. (Sometimes. Based on reports and difficulty of reproducing the bug, this behavior was extremely rare.) I don't know why that happened exactly, maybe some polling mechanism in the kernel and if the timings worked out just wrong it would expose the bug. Anyway, now we ensure that only the closer that set the deadline is the same one that clears it, ensuring that old servers always return out of Accept(), because the deadline doesn't get cleared until they do. Of course, all this hinges on the hope that my suspicions in the middle of the night are correct and that kernels work the way I think they do in my head. Also minor enhancement to UsagePool where if a value errors upon construction (a very real possibility with listeners), it is removed from the pool. Not 100% sure the sync logic is correct there, or maybe we don't have to even put it in the pool until after construction, but it's subtle either way and I think this is safe... right?

Matthew Holt committed Jan 10, 2022 at 23:24 UTC 64a3218f5c7ba98a51c76065ed32ae860351e779
2 files changed +146 -168
listeners.go
+138 -167
@@ -16,7 +16,6 @@ package caddy
16
17 import (
18 "fmt"
19 - "log"
19 "net"
20 "strconv"
21 "strings"
@@ -26,124 +25,90 @@ import (
25 "time"
26 )
27
29 -// Listen returns a listener suitable for use in a Caddy module.
30 -// Always be sure to close listeners when you are done with them.
28 +// Listen is like net.Listen, except Caddy's listeners can overlap
29 +// each other: multiple listeners may be created on the same socket
30 +// at the same time. This is useful because during config changes,
31 +// the new config is started while the old config is still running.
32 +// When Caddy listeners are closed, the closing logic is virtualized
33 +// so the underlying socket isn't actually closed until all uses of
34 +// the socket have been finished. Always be sure to close listeners
35 +// when you are done with them, just like normal listeners.
36 func Listen(network, addr string) (net.Listener, error) {
37 lnKey := network + "/" + addr
38
34 - listenersMu.Lock()
35 - defer listenersMu.Unlock()
36 -
37 - // if listener already exists, increment usage counter, then return listener
38 - if lnGlobal, ok := listeners[lnKey]; ok {
39 - atomic.AddInt32(&lnGlobal.usage, 1)
40 - return &fakeCloseListener{
41 - usage: &lnGlobal.usage,
42 - deadline: &lnGlobal.deadline,
43 - deadlineMu: &lnGlobal.deadlineMu,
44 - key: lnKey,
45 - Listener: lnGlobal.ln,
46 - }, nil
47 - }
48 -
49 - // or, create new one and save it
50 - ln, err := net.Listen(network, addr)
39 + sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
40 + ln, err := net.Listen(network, addr)
41 + if err != nil {
42 + return nil, err
43 + }
44 + return &sharedListener{Listener: ln, key: lnKey}, nil
45 + })
46 if err != nil {
47 return nil, err
48 }
49
55 - // make sure to start its usage counter at 1
56 - lnGlobal := &globalListener{usage: 1, ln: ln}
57 - listeners[lnKey] = lnGlobal
58 -
59 - return &fakeCloseListener{
60 - usage: &lnGlobal.usage,
61 - deadline: &lnGlobal.deadline,
62 - deadlineMu: &lnGlobal.deadlineMu,
63 - key: lnKey,
64 - Listener: ln,
65 - }, nil
50 + return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener)}, nil
51 }
52
53 // ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
54 +// It is like Listen except for PacketConns.
55 // Always be sure to close the PacketConn when you are done.
56 func ListenPacket(network, addr string) (net.PacketConn, error) {
57 lnKey := network + "/" + addr
58
73 - listenersMu.Lock()
74 - defer listenersMu.Unlock()
75 -
76 - // if listener already exists, increment usage counter, then return listener
77 - if lnGlobal, ok := listeners[lnKey]; ok {
78 - atomic.AddInt32(&lnGlobal.usage, 1)
79 - log.Printf("[DEBUG] %s: Usage counter should not go above 2 or maybe 3, is now: %d", lnKey, atomic.LoadInt32(&lnGlobal.usage)) // TODO: remove
80 - return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: lnGlobal.pc}, nil
81 - }
82 -
83 - // or, create new one and save it
84 - pc, err := net.ListenPacket(network, addr)
59 + sharedPc, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
60 + pc, err := net.ListenPacket(network, addr)
61 + if err != nil {
62 + return nil, err
63 + }
64 + return &sharedPacketConn{PacketConn: pc, key: lnKey}, nil
65 + })
66 if err != nil {
67 return nil, err
68 }
69
89 - // make sure to start its usage counter at 1
90 - lnGlobal := &globalListener{usage: 1, pc: pc}
91 - listeners[lnKey] = lnGlobal
92 -
93 - return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: pc}, nil
70 + return &fakeClosePacketConn{sharedPacketConn: sharedPc.(*sharedPacketConn)}, nil
71 }
72
96 -// fakeCloseListener's Close() method is a no-op. This allows
97 -// stopping servers that are using the listener without giving
98 -// up the socket; thus, servers become hot-swappable while the
99 -// listener remains running. Listeners should be re-wrapped in
100 -// a new fakeCloseListener each time the listener is reused.
101 -// Other than the 'closed' field (which pertains to this value
102 -// only), the other fields in this struct should be pointers to
103 -// the associated globalListener's struct fields (except 'key'
104 -// which is there for read-only purposes, so it can be a copy).
73 +// fakeCloseListener is a private wrapper over a listener that
74 +// is shared. The state of fakeCloseListener is not shared.
75 +// This allows one user of a socket to "close" the listener
76 +// while in reality the socket stays open for other users of
77 +// the listener. In this way, servers become hot-swappable
78 +// while the listener remains running. Listeners should be
79 +// re-wrapped in a new fakeCloseListener each time the listener
80 +// is reused. This type is atomic and values must not be copied.
81 type fakeCloseListener struct {
106 - closed int32 // accessed atomically; belongs to this struct only
107 - usage *int32 // accessed atomically; global
108 - deadline *bool // protected by deadlineMu; global
109 - deadlineMu *sync.Mutex // global
110 - key string // global, but read-only, so can be copy
111 - net.Listener // global
82 + closed int32 // accessed atomically; belongs to this struct only
83 + *sharedListener // embedded, so we also become a net.Listener
84 }
85
114 -// Accept accepts connections until Close() is called.
86 func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
87 // if the listener is already "closed", return error
88 if atomic.LoadInt32(&fcl.closed) == 1 {
89 return nil, fcl.fakeClosedErr()
90 }
91
121 - // wrap underlying accept
122 - conn, err := fcl.Listener.Accept()
92 + // call underlying accept
93 + conn, err := fcl.sharedListener.Accept()
94 if err == nil {
95 return conn, nil
96 }
97
127 - // accept returned with error
128 - // TODO: This may be better as a condition variable so the deadline is cleared only once?
129 - fcl.deadlineMu.Lock()
130 - if *fcl.deadline {
131 - switch ln := fcl.Listener.(type) {
132 - case *net.TCPListener:
133 - _ = ln.SetDeadline(time.Time{})
134 - case *net.UnixListener:
135 - _ = ln.SetDeadline(time.Time{})
136 - }
137 - *fcl.deadline = false
138 - }
139 - fcl.deadlineMu.Unlock()
140 -
98 + // since Accept() returned an error, it may be because our reference to
99 + // the listener (this fakeCloseListener) may have been closed, i.e. the
100 + // server is shutting down; in that case, we need to clear the deadline
101 + // that we set when Close() was called, and return a non-temporary and
102 + // non-timeout error value to the caller, masking the "true" error, so
103 + // that server loops / goroutines won't retry, linger, and leak
104 if atomic.LoadInt32(&fcl.closed) == 1 {
142 - // if we canceled the Accept() by setting a deadline
143 - // on the listener, we need to make sure any callers of
144 - // Accept() think the listener was actually closed;
145 - // if we return the timeout error instead, callers might
146 - // simply retry, leaking goroutines for longer
105 + // we dereference the sharedListener explicitly even though it's embedded
106 + // so that it's clear in the code that side-effects are shared with other
107 + // users of this listener, not just our own reference to it; we also don't
108 + // do anything with the error because all we could do is log it, but we
109 + // expliclty assign it to nothing so we don't forget it's there if needed
110 + _ = fcl.sharedListener.clearDeadline()
111 +
112 if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
113 return nil, fcl.fakeClosedErr()
114 }
@@ -152,82 +117,54 @@ func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
117 return nil, err
118 }
119
155 -// Close stops accepting new connections without
156 -// closing the underlying listener, unless no one
157 -// else is using it.
120 +// Close stops accepting new connections without closing the
121 +// underlying listener. The underlying listener is only closed
122 +// if the caller is the last known user of the socket.
123 func (fcl *fakeCloseListener) Close() error {
124 if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) {
160 - // unfortunately, there is no way to cancel any
161 - // currently-blocking calls to Accept() that are
162 - // awaiting connections since we're not actually
163 - // closing the listener; so we cheat by setting
164 - // a deadline in the past, which forces it to
165 - // time out; note that this only works for
166 - // certain types of listeners...
167 - fcl.deadlineMu.Lock()
168 - if !*fcl.deadline {
169 - switch ln := fcl.Listener.(type) {
170 - case *net.TCPListener:
171 - _ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
172 - case *net.UnixListener:
173 - _ = ln.SetDeadline(time.Now().Add(-1 * time.Minute))
174 - }
175 - *fcl.deadline = true
176 - }
177 - fcl.deadlineMu.Unlock()
178 -
179 - // since we're no longer using this listener,
180 - // decrement the usage counter and, if no one
181 - // else is using it, close underlying listener
182 - if atomic.AddInt32(fcl.usage, -1) == 0 {
183 - listenersMu.Lock()
184 - delete(listeners, fcl.key)
185 - listenersMu.Unlock()
186 - err := fcl.Listener.Close()
187 - if err != nil {
188 - return err
189 - }
190 - }
191 -
125 + // There are two ways I know of to get an Accept()
126 + // function to return to the server loop that called
127 + // it: close the listener, or set a deadline in the
128 + // past. Obviously, we can't close the socket yet
129 + // since others may be using it (hence this whole
130 + // file). But we can set the deadline in the past,
131 + // and this is kind of cheating, but it works, and
132 + // it apparently even works on Windows.
133 + _ = fcl.sharedListener.setDeadline()
134 + listenerPool.Delete(fcl.sharedListener.key)
135 }
193 -
136 return nil
137 }
138
139 +// fakeClosedErr returns an error value that is not temporary
140 +// nor a timeout, suitable for making the caller think the
141 +// listener is actually closed
142 func (fcl *fakeCloseListener) fakeClosedErr() error {
143 return &net.OpError{
144 Op: "accept",
200 - Net: fcl.Listener.Addr().Network(),
201 - Addr: fcl.Listener.Addr(),
145 + Net: fcl.Addr().Network(),
146 + Addr: fcl.Addr(),
147 Err: errFakeClosed,
148 }
149 }
150
151 +// ErrFakeClosed is the underlying error value returned by
152 +// fakeCloseListener.Accept() after Close() has been called,
153 +// indicating that it is pretending to be closed so that the
154 +// server using it can terminate, while the underlying
155 +// socket is actually left open.
156 +var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
157 +
158 +// fakeClosePacketConn is like fakeCloseListener, but for PacketConns.
159 type fakeClosePacketConn struct {
207 - closed int32 // accessed atomically
208 - usage *int32 // accessed atomically
209 - key string
210 - net.PacketConn
160 + closed int32 // accessed atomically; belongs to this struct only
161 + *sharedPacketConn // embedded, so we also become a net.PacketConn
162 }
163
164 func (fcpc *fakeClosePacketConn) Close() error {
214 - log.Println("[DEBUG] Fake-closing underlying packet conn") // TODO: remove this
215 -
165 if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {
217 - // since we're no longer using this listener,
218 - // decrement the usage counter and, if no one
219 - // else is using it, close underlying listener
220 - if atomic.AddInt32(fcpc.usage, -1) == 0 {
221 - listenersMu.Lock()
222 - delete(listeners, fcpc.key)
223 - listenersMu.Unlock()
224 - err := fcpc.PacketConn.Close()
225 - if err != nil {
226 - return err
227 - }
228 - }
166 + listenerPool.Delete(fcpc.sharedPacketConn.key)
167 }
230 -
168 return nil
169 }
170
@@ -249,28 +186,64 @@ func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) {
186 return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn)
187 }
188
252 -// ErrFakeClosed is the underlying error value returned by
253 -// fakeCloseListener.Accept() after Close() has been called,
254 -// indicating that it is pretending to be closed so that the
255 -// server using it can terminate, while the underlying
256 -// socket is actually left open.
257 -var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
258 -
259 -// globalListener keeps global state for a listener
260 -// that may be shared by multiple servers. In other
261 -// words, values in this struct exist only once and
262 -// all other uses of these values point to the ones
263 -// in this struct. In particular, the usage count
264 -// (how many callers are using the listener), the
265 -// actual listener, and synchronization of the
266 -// listener's deadline changes are singular, global
267 -// values that must not be copied.
268 -type globalListener struct {
269 - usage int32 // accessed atomically
270 - deadline bool
189 +// sharedListener is a wrapper over an underlying listener. The listener
190 +// and the other fields on the struct are shared state that is synchronized,
191 +// so sharedListener structs must never be copied (always use a pointer).
192 +type sharedListener struct {
193 + net.Listener
194 + key string // uniquely identifies this listener
195 + deadline bool // whether a deadline is currently set
196 deadlineMu sync.Mutex
272 - ln net.Listener
273 - pc net.PacketConn
197 +}
198 +
199 +func (sl *sharedListener) clearDeadline() error {
200 + var err error
201 + sl.deadlineMu.Lock()
202 + if sl.deadline {
203 + switch ln := sl.Listener.(type) {
204 + case *net.TCPListener:
205 + err = ln.SetDeadline(time.Time{})
206 + case *net.UnixListener:
207 + err = ln.SetDeadline(time.Time{})
208 + }
209 + sl.deadline = false
210 + }
211 + sl.deadlineMu.Unlock()
212 + return err
213 +}
214 +
215 +func (sl *sharedListener) setDeadline() error {
216 + timeInPast := time.Now().Add(-1 * time.Minute)
217 + var err error
218 + sl.deadlineMu.Lock()
219 + if !sl.deadline {
220 + switch ln := sl.Listener.(type) {
221 + case *net.TCPListener:
222 + err = ln.SetDeadline(timeInPast)
223 + case *net.UnixListener:
224 + err = ln.SetDeadline(timeInPast)
225 + }
226 + sl.deadline = true
227 + }
228 + sl.deadlineMu.Unlock()
229 + return err
230 +}
231 +
232 +// Destruct is called by the UsagePool when the listener is
233 +// finally not being used anymore. It closes the socket.
234 +func (sl *sharedListener) Destruct() error {
235 + return sl.Listener.Close()
236 +}
237 +
238 +// sharedPacketConn is like sharedListener, but for net.PacketConns.
239 +type sharedPacketConn struct {
240 + net.PacketConn
241 + key string
242 +}
243 +
244 +// Destruct closes the underlying socket.
245 +func (spc *sharedPacketConn) Destruct() error {
246 + return spc.PacketConn.Close()
247 }
248
249 // NetworkAddress contains the individual components
@@ -445,10 +418,8 @@ type ListenerWrapper interface {
418 WrapListener(net.Listener) net.Listener
419 }
420
448 -var (
449 - listeners = make(map[string]*globalListener)
450 - listenersMu sync.Mutex
451 -)
421 +// listenerPool stores and allows reuse of active listeners.
422 +var listenerPool = NewUsagePool()
423
424 const maxPortSpan = 65535
425
usagepool.go
+8 -1
@@ -94,8 +94,15 @@ func (up *UsagePool) LoadOrNew(key interface{}, construct Constructor) (value in
94 if err == nil {
95 upv.value = value
96 } else {
97 - // TODO: remove error'ed entries from map
97 upv.err = err
98 + up.Lock()
99 + // this *should* be safe, I think, because we have a
100 + // write lock on upv, but we might also need to ensure
101 + // that upv.err is nil before doing this, since we
102 + // released the write lock on up during construct...
103 + // but then again it's also after midnight...
104 + delete(up.pool, key)
105 + up.Unlock()
106 }
107 upv.Unlock()
108 }