Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
core: Synchronize calls to SetDeadline within fakeCloseListener
First evidenced in #2658, listener deadlines would sometimes be set after clearing them, resulting in endless i/o timeout errors, which leave all requests hanging. This bug is fixed by synchronizing the calls to SetDeadline: when Close() is called, the deadline is first set to a time in the past, and the lock is released only after the deadline is set, so when the other servers break out of their Accept() calls, they will clear the deadline *after* it was set. Before, the clearing could sometimes come before the set, which meant that it was left in a timeout state indefinitely. This may not yet be a perfect solution -- ideally, the setting and clearing of the deadline would happen exactly once per underlying listener, not once per fakeCloseListener, but in rigorous testing with these changes (comprising tens of thousands of config reloads), I was able to verify that no race condition is manifest.
Matthew Holt committed
Nov 4, 2019 at 12:10 UTC
27e288ab19a3a4d9243a8871d5b3c6426478e008
1 file changed
+74
-35
listeners.go
+74
-35
@@ -25,8 +25,6 @@ import (
25
"time"
26
)
27
28
-// TODO: Can we use the new UsagePool type?
29
-
28
// Listen returns a listener suitable for use in a Caddy module.
29
// Always be sure to close listeners when you are done with them.
30
func Listen(network, addr string) (net.Listener, error) {
@@ -36,9 +34,15 @@ func Listen(network, addr string) (net.Listener, error) {
34
defer listenersMu.Unlock()
35
36
// if listener already exists, increment usage counter, then return listener
39
- if lnUsage, ok := listeners[lnKey]; ok {
40
- atomic.AddInt32(&lnUsage.usage, 1)
41
- return &fakeCloseListener{usage: &lnUsage.usage, key: lnKey, Listener: lnUsage.ln}, nil
37
+ if lnGlobal, ok := listeners[lnKey]; ok {
38
+ atomic.AddInt32(&lnGlobal.usage, 1)
39
+ return &fakeCloseListener{
40
+ usage: &lnGlobal.usage,
41
+ deadline: &lnGlobal.deadline,
42
+ deadlineMu: &lnGlobal.deadlineMu,
43
+ key: lnKey,
44
+ Listener: lnGlobal.ln,
45
+ }, nil
46
}
47
48
// or, create new one and save it
@@ -48,10 +52,19 @@ func Listen(network, addr string) (net.Listener, error) {
52
}
53
54
// make sure to start its usage counter at 1
51
- lnUsage := &listenerUsage{usage: 1, ln: ln}
52
- listeners[lnKey] = lnUsage
53
-
54
- return &fakeCloseListener{usage: &lnUsage.usage, key: lnKey, Listener: ln}, nil
55
+ lnGlobal := &globalListener{
56
+ usage: 1,
57
+ ln: ln,
58
+ }
59
+ listeners[lnKey] = lnGlobal
60
+
61
+ return &fakeCloseListener{
62
+ usage: &lnGlobal.usage,
63
+ deadline: &lnGlobal.deadline,
64
+ deadlineMu: &lnGlobal.deadlineMu,
65
+ key: lnKey,
66
+ Listener: ln,
67
+ }, nil
68
}
69
70
// ListenPacket returns a net.PacketConn suitable for use in a Caddy module.
@@ -63,10 +76,10 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
76
defer listenersMu.Unlock()
77
78
// if listener already exists, increment usage counter, then return listener
66
- if lnUsage, ok := listeners[lnKey]; ok {
67
- atomic.AddInt32(&lnUsage.usage, 1)
68
- log.Printf("[DEBUG] %s: Usage counter should not go above 2 or maybe 3, is now: %d", lnKey, atomic.LoadInt32(&lnUsage.usage)) // TODO: remove
69
- return &fakeClosePacketConn{usage: &lnUsage.usage, key: lnKey, PacketConn: lnUsage.pc}, nil
79
+ if lnGlobal, ok := listeners[lnKey]; ok {
80
+ atomic.AddInt32(&lnGlobal.usage, 1)
81
+ log.Printf("[DEBUG] %s: Usage counter should not go above 2 or maybe 3, is now: %d", lnKey, atomic.LoadInt32(&lnGlobal.usage)) // TODO: remove
82
+ return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: lnGlobal.pc}, nil
83
}
84
85
// or, create new one and save it
@@ -76,10 +89,10 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
89
}
90
91
// make sure to start its usage counter at 1
79
- lnUsage := &listenerUsage{usage: 1, pc: pc}
80
- listeners[lnKey] = lnUsage
92
+ lnGlobal := &globalListener{usage: 1, pc: pc}
93
+ listeners[lnKey] = lnGlobal
94
82
- return &fakeClosePacketConn{usage: &lnUsage.usage, key: lnKey, PacketConn: pc}, nil
95
+ return &fakeClosePacketConn{usage: &lnGlobal.usage, key: lnKey, PacketConn: pc}, nil
96
}
97
98
// fakeCloseListener's Close() method is a no-op. This allows
@@ -87,11 +100,17 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
100
// up the socket; thus, servers become hot-swappable while the
101
// listener remains running. Listeners should be re-wrapped in
102
// a new fakeCloseListener each time the listener is reused.
103
+// Other than the 'closed' field (which pertains to this value
104
+// only), the other fields in this struct should be pointers to
105
+// the associated globalListener's struct fields (except 'key'
106
+// which is there for read-only purposes, so it can be a copy).
107
type fakeCloseListener struct {
91
- closed int32 // accessed atomically - TODO: this needs to be shared across the whole app instance, not to cross instance boundaries... hmmm... see #2658 (still relevant?)
92
- usage *int32 // accessed atomically
93
- key string
94
- net.Listener
108
+ closed int32 // accessed atomically; belongs to this struct only
109
+ usage *int32 // accessed atomically; global
110
+ deadline *bool // protected by deadlineMu; global
111
+ deadlineMu *sync.Mutex // global
112
+ key string // global, but read-only, so can be copy
113
+ net.Listener // global
114
}
115
116
// Accept accepts connections until Close() is called.
@@ -107,15 +126,21 @@ func (fcl *fakeCloseListener) Accept() (net.Conn, error) {
126
return conn, nil
127
}
128
110
- if atomic.LoadInt32(&fcl.closed) == 1 {
111
- // clear the deadline
129
+ // accept returned with error
130
+ // TODO: This may be better as a condition variable so the deadline is cleared only once?
131
+ fcl.deadlineMu.Lock()
132
+ if *fcl.deadline {
133
switch ln := fcl.Listener.(type) {
134
case *net.TCPListener:
135
ln.SetDeadline(time.Time{})
136
case *net.UnixListener:
137
ln.SetDeadline(time.Time{})
138
}
139
+ *fcl.deadline = false
140
+ }
141
+ fcl.deadlineMu.Unlock()
142
143
+ if atomic.LoadInt32(&fcl.closed) == 1 {
144
// if we cancelled the Accept() by setting a deadline
145
// on the listener, we need to make sure any callers of
146
// Accept() think the listener was actually closed;
@@ -141,12 +166,17 @@ func (fcl *fakeCloseListener) Close() error {
166
// a deadline in the past, which forces it to
167
// time out; note that this only works for
168
// certain types of listeners...
144
- switch ln := fcl.Listener.(type) {
145
- case *net.TCPListener:
146
- ln.SetDeadline(time.Now().Add(-1 * time.Minute))
147
- case *net.UnixListener:
148
- ln.SetDeadline(time.Now().Add(-1 * time.Minute))
169
+ fcl.deadlineMu.Lock()
170
+ if !*fcl.deadline {
171
+ switch ln := fcl.Listener.(type) {
172
+ case *net.TCPListener:
173
+ ln.SetDeadline(time.Now().Add(-1 * time.Minute))
174
+ case *net.UnixListener:
175
+ ln.SetDeadline(time.Now().Add(-1 * time.Minute))
176
+ }
177
+ *fcl.deadline = true
178
}
179
+ fcl.deadlineMu.Unlock()
180
181
// since we're no longer using this listener,
182
// decrement the usage counter and, if no one
@@ -176,7 +206,7 @@ func (fcl *fakeCloseListener) fakeClosedErr() error {
206
}
207
208
type fakeClosePacketConn struct {
179
- closed int32 // accessed atomically - TODO: this needs to be shared across the whole app instance, not to cross instance boundaries... hmmm... see #2658 (still relevant?)
209
+ closed int32 // accessed atomically
210
usage *int32 // accessed atomically
211
key string
212
net.PacketConn
@@ -210,16 +240,25 @@ func (fcpc *fakeClosePacketConn) Close() error {
240
// socket is actually left open.
241
var errFakeClosed = fmt.Errorf("listener 'closed' 😉")
242
213
-// listenerUsage pairs a net.Listener with a
214
-// count of how many servers are using it.
215
-type listenerUsage struct {
216
- usage int32 // accessed atomically
217
- ln net.Listener
218
- pc net.PacketConn
243
+// globalListener keeps global state for a listener
244
+// that may be shared by multiple servers. In other
245
+// words, values in this struct exist only once and
246
+// all other uses of these values point to the ones
247
+// in this struct. In particular, the usage count
248
+// (how many callers are using the listener), the
249
+// actual listener, and synchronization of the
250
+// listener's deadline changes are singular, global
251
+// values that must not be copied.
252
+type globalListener struct {
253
+ usage int32 // accessed atomically
254
+ deadline bool
255
+ deadlineMu sync.Mutex
256
+ ln net.Listener
257
+ pc net.PacketConn
258
}
259
260
var (
222
- listeners = make(map[string]*listenerUsage)
261
+ listeners = make(map[string]*globalListener)
262
listenersMu sync.Mutex
263
)
264