added multiconn
Juan Batiz-Benet committed
Oct 19, 2014 at 03:34 UTC
a4e492342446b5886f8df6e49d15be5e2bffbde4
2 files changed
+482
net/conn/multiconn.go
new
+269
@@ -0,0 +1,269 @@
1
+package conn
2
+
3
+import (
4
+ "sync"
5
+
6
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
7
+ ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
8
+
9
+ peer "github.com/jbenet/go-ipfs/peer"
10
+ u "github.com/jbenet/go-ipfs/util"
11
+)
12
+
13
+// Duplex is a simple duplex channel
14
+type Duplex struct {
15
+ In chan []byte
16
+ Out chan []byte
17
+}
18
+
19
+// MultiConn represents a single connection to another Peer (IPFS Node).
20
+type MultiConn struct {
21
+
22
+ // connections, mapped by a string, which uniquely identifies the connection.
23
+ // this string is: /addr1/peer1/addr2/peer2 (peers ordered lexicographically)
24
+ conns map[string]Conn
25
+
26
+ local *peer.Peer
27
+ remote *peer.Peer
28
+
29
+ // fan-in/fan-out
30
+ duplex Duplex
31
+
32
+ // for adding/removing connections concurrently
33
+ sync.RWMutex
34
+ ContextCloser
35
+}
36
+
37
+// NewMultiConn constructs a new connection
38
+func NewMultiConn(ctx context.Context, local, remote *peer.Peer, conns []Conn) (Conn, error) {
39
+
40
+ c := &MultiConn{
41
+ local: local,
42
+ remote: remote,
43
+ conns: map[string]Conn{},
44
+ duplex: Duplex{
45
+ In: make(chan []byte, 10),
46
+ Out: make(chan []byte, 10),
47
+ },
48
+ }
49
+
50
+ // must happen before Adds / fanOut
51
+ c.ContextCloser = NewContextCloser(ctx, c.close)
52
+
53
+ log.Info("adding %d...", len(conns))
54
+ if conns != nil && len(conns) > 0 {
55
+ c.Add(conns...)
56
+ }
57
+ go c.fanOut()
58
+
59
+ log.Info("newMultiConn: %v to %v", local, remote)
60
+ return c, nil
61
+}
62
+
63
+// Add adds given Conn instances to multiconn.
64
+func (c *MultiConn) Add(conns ...Conn) {
65
+ c.Lock()
66
+ defer c.Unlock()
67
+
68
+ for _, c2 := range conns {
69
+ log.Info("MultiConn: adding %s", c2)
70
+ if c.LocalPeer() != c2.LocalPeer() || c.RemotePeer() != c2.RemotePeer() {
71
+ log.Error("%s", c2)
72
+ panic("connection addresses mismatch")
73
+ }
74
+
75
+ c.conns[c2.ID()] = c2
76
+ go c.fanInSingle(c2)
77
+ log.Info("MultiConn: added %s", c2)
78
+ }
79
+}
80
+
81
+// Remove removes given Conn instances from multiconn.
82
+func (c *MultiConn) Remove(conns ...Conn) {
83
+
84
+ // first remove them to avoid sending any more messages through it.
85
+ {
86
+ c.Lock()
87
+ for _, c1 := range conns {
88
+ c2, found := c.conns[c1.ID()]
89
+ if !found {
90
+ panic("Conn not in MultiConn")
91
+ }
92
+ if c1 != c2 {
93
+ panic("different Conn objects for same id.")
94
+ }
95
+
96
+ delete(c.conns, c2.ID())
97
+ }
98
+ c.Unlock()
99
+ }
100
+
101
+ // close all in parallel, but wait for all to be done closing.
102
+ CloseConns(conns)
103
+}
104
+
105
+// CloseConns closes multiple connections in parallel, and waits for all
106
+// to finish closing.
107
+func CloseConns(conns []Conn) {
108
+ var wg sync.WaitGroup
109
+ for _, child := range conns {
110
+
111
+ select {
112
+ case <-child.Closed(): // if already closed, continue
113
+ continue
114
+ default:
115
+ }
116
+
117
+ wg.Add(1)
118
+ go func(child Conn) {
119
+ child.Close()
120
+ wg.Done()
121
+ }(child)
122
+ }
123
+ wg.Wait()
124
+}
125
+
126
+// fanOut is the multiplexor out -- it sends outgoing messages over the
127
+// underlying single connections.
128
+func (c *MultiConn) fanOut() {
129
+ c.Children().Add(1)
130
+ defer c.Children().Done()
131
+
132
+ for {
133
+ select {
134
+ case <-c.Closing():
135
+ return
136
+
137
+ // send data out through our "best connection"
138
+ case m, more := <-c.duplex.Out:
139
+ if !more {
140
+ return
141
+ }
142
+ sc := c.BestConn()
143
+ if sc == nil {
144
+ // maybe this should be a logged error, not a panic.
145
+ panic("sending out multiconn without any live connection")
146
+ }
147
+ sc.Out() <- m
148
+ }
149
+ }
150
+}
151
+
152
+// fanInSingle is a multiplexor in -- it receives incoming messages over the
153
+// underlying single connections.
154
+func (c *MultiConn) fanInSingle(child Conn) {
155
+ c.Children().Add(1)
156
+ child.Children().Add(1) // yep, on the child too.
157
+
158
+ // cleanup all data associated with this child Connection.
159
+ defer func() {
160
+ // in case it still is in the map, remove it.
161
+ c.Lock()
162
+ delete(c.conns, child.ID())
163
+ c.Unlock()
164
+
165
+ c.Children().Done()
166
+ child.Children().Done()
167
+ }()
168
+
169
+ for {
170
+ select {
171
+ case <-c.Closing(): // multiconn closing
172
+ return
173
+
174
+ case <-child.Closing(): // child closing
175
+ return
176
+
177
+ case m, more := <-child.In(): // receiving data
178
+ if !more {
179
+ return // closed
180
+ }
181
+ c.duplex.In <- m
182
+ }
183
+ }
184
+}
185
+
186
+// close is the internal close function, called by ContextCloser.Close
187
+func (c *MultiConn) close() error {
188
+ log.Debug("%s closing Conn with %s", c.local, c.remote)
189
+
190
+ // get connections
191
+ c.RLock()
192
+ conns := make([]Conn, 0, len(c.conns))
193
+ for _, c := range c.conns {
194
+ conns = append(conns, c)
195
+ }
196
+ c.RUnlock()
197
+
198
+ // close underlying connections
199
+ CloseConns(conns)
200
+ return nil
201
+}
202
+
203
+// BestConn is the best connection in this MultiConn
204
+func (c *MultiConn) BestConn() Conn {
205
+ c.RLock()
206
+ defer c.RUnlock()
207
+
208
+ var id1 string
209
+ var c1 Conn
210
+ for id2, c2 := range c.conns {
211
+ if id1 == "" || id1 < id2 {
212
+ id1 = id2
213
+ c1 = c2
214
+ }
215
+ }
216
+ return c1
217
+}
218
+
219
+// ID is an identifier unique to this connection.
220
+// In MultiConn, this is all the children IDs XORed together.
221
+func (c *MultiConn) ID() string {
222
+ c.RLock()
223
+ defer c.RUnlock()
224
+
225
+ ids := []byte(nil)
226
+ for i := range c.conns {
227
+ if ids == nil {
228
+ ids = []byte(i)
229
+ } else {
230
+ ids = u.XOR(ids, []byte(i))
231
+ }
232
+ }
233
+
234
+ return string(ids)
235
+}
236
+
237
+func (c *MultiConn) String() string {
238
+ return String(c, "MultiConn")
239
+}
240
+
241
+// LocalMultiaddr is the Multiaddr on this side
242
+func (c *MultiConn) LocalMultiaddr() ma.Multiaddr {
243
+ return c.BestConn().LocalMultiaddr()
244
+}
245
+
246
+// RemoteMultiaddr is the Multiaddr on the remote side
247
+func (c *MultiConn) RemoteMultiaddr() ma.Multiaddr {
248
+ return c.BestConn().RemoteMultiaddr()
249
+}
250
+
251
+// LocalPeer is the Peer on this side
252
+func (c *MultiConn) LocalPeer() *peer.Peer {
253
+ return c.local
254
+}
255
+
256
+// RemotePeer is the Peer on the remote side
257
+func (c *MultiConn) RemotePeer() *peer.Peer {
258
+ return c.remote
259
+}
260
+
261
+// In returns a readable message channel
262
+func (c *MultiConn) In() <-chan []byte {
263
+ return c.duplex.In
264
+}
265
+
266
+// Out returns a writable message channel
267
+func (c *MultiConn) Out() chan<- []byte {
268
+ return c.duplex.Out
269
+}
net/conn/multiconn_test.go
new
+213
@@ -0,0 +1,213 @@
1
+package conn
2
+
3
+import (
4
+ "fmt"
5
+ "sync"
6
+ "testing"
7
+ "time"
8
+
9
+ peer "github.com/jbenet/go-ipfs/peer"
10
+
11
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
12
+ ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
13
+)
14
+
15
+func tcpAddr(t *testing.T, port int) ma.Multiaddr {
16
+ tcp, err := ma.NewMultiaddr(tcpAddrString(port))
17
+ if err != nil {
18
+ t.Fatal(err)
19
+ }
20
+ return tcp
21
+}
22
+
23
+func tcpAddrString(port int) string {
24
+ return fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", port)
25
+}
26
+
27
+type msg struct {
28
+ sent bool
29
+ received bool
30
+ payload string
31
+}
32
+
33
+func (m *msg) Sent(t *testing.T) {
34
+ if m.sent {
35
+ t.Fatal("sent msg at incorrect state:", m)
36
+ }
37
+ m.sent = true
38
+}
39
+
40
+func (m *msg) Received(t *testing.T) {
41
+ if m.received {
42
+ t.Fatal("received msg at incorrect state:", m)
43
+ }
44
+ m.received = true
45
+}
46
+
47
+type msgMap struct {
48
+ sent int
49
+ recv int
50
+ msgs map[string]*msg
51
+}
52
+
53
+func (mm *msgMap) Sent(t *testing.T, payload string) {
54
+ mm.msgs[payload].Sent(t)
55
+ mm.sent++
56
+}
57
+
58
+func (mm *msgMap) Received(t *testing.T, payload string) {
59
+ mm.msgs[payload].Received(t)
60
+ mm.recv++
61
+}
62
+
63
+func (mm *msgMap) CheckDone(t *testing.T) {
64
+ if mm.sent != len(mm.msgs) {
65
+ t.Fatal("failed to send all msgs", mm.sent, len(mm.msgs))
66
+ }
67
+
68
+ if mm.sent != len(mm.msgs) {
69
+ t.Fatal("failed to send all msgs", mm.sent, len(mm.msgs))
70
+ }
71
+}
72
+
73
+func genMessages(num int, tag string) *msgMap {
74
+ msgs := &msgMap{msgs: map[string]*msg{}}
75
+ for i := 0; i < num; i++ {
76
+ s := fmt.Sprintf("Message #%d -- %s", i, tag)
77
+ msgs.msgs[s] = &msg{payload: s}
78
+ }
79
+ return msgs
80
+}
81
+
82
+func setupMultiConns(t *testing.T, ctx context.Context) (a, b *MultiConn) {
83
+
84
+ log.Info("Setting up peers")
85
+ p1, err := setupPeer(tcpAddrString(11000))
86
+ if err != nil {
87
+ t.Fatal("error setting up peer", err)
88
+ }
89
+
90
+ p2, err := setupPeer(tcpAddrString(12000))
91
+ if err != nil {
92
+ t.Fatal("error setting up peer", err)
93
+ }
94
+
95
+ // peerstores
96
+ p1ps := peer.NewPeerstore()
97
+ p2ps := peer.NewPeerstore()
98
+
99
+ // listeners
100
+ listen := func(addr ma.Multiaddr, p *peer.Peer, ps peer.Peerstore) Listener {
101
+ l, err := Listen(ctx, addr, p, ps)
102
+ if err != nil {
103
+ t.Fatal(err)
104
+ }
105
+ return l
106
+ }
107
+
108
+ log.Info("Setting up listeners")
109
+ p1l := listen(p1.Addresses[0], p1, p1ps)
110
+ p2l := listen(p2.Addresses[0], p2, p2ps)
111
+
112
+ // dialers
113
+ p1d := &Dialer{Peerstore: p1ps, LocalPeer: p1}
114
+ p2d := &Dialer{Peerstore: p2ps, LocalPeer: p2}
115
+
116
+ dial := func(d *Dialer, dst *peer.Peer) <-chan Conn {
117
+ cc := make(chan Conn)
118
+ go func() {
119
+ c, err := d.Dial(ctx, "tcp", dst)
120
+ if err != nil {
121
+ t.Fatal("error dialing peer", err)
122
+ }
123
+ cc <- c
124
+ }()
125
+ return cc
126
+ }
127
+
128
+ // connect simultaneously
129
+ log.Info("Connecting...")
130
+ p1dc := dial(p1d, p2)
131
+ p2dc := dial(p2d, p1)
132
+
133
+ c12a := <-p1l.Accept()
134
+ c12b := <-p1dc
135
+ c21a := <-p2l.Accept()
136
+ c21b := <-p2dc
137
+
138
+ log.Info("Ok, making multiconns")
139
+ c1, err := NewMultiConn(ctx, p1, p2, []Conn{c12a, c12b})
140
+ if err != nil {
141
+ t.Fatal(err)
142
+ }
143
+
144
+ c2, err := NewMultiConn(ctx, p2, p1, []Conn{c21a, c21b})
145
+ if err != nil {
146
+ t.Fatal(err)
147
+ }
148
+
149
+ log.Info("did you make multiconns?")
150
+ return c1.(*MultiConn), c2.(*MultiConn)
151
+}
152
+
153
+func TestMulticonnSend(t *testing.T) {
154
+ log.Info("TestMulticonnSend")
155
+ ctx := context.Background()
156
+ ctxC, cancel := context.WithCancel(ctx)
157
+
158
+ c1, c2 := setupMultiConns(t, ctx)
159
+
160
+ log.Info("gen msgs")
161
+ num := 100
162
+ msgsFrom1 := genMessages(num, "from p1 to p2")
163
+ msgsFrom2 := genMessages(num, "from p2 to p1")
164
+
165
+ var wg sync.WaitGroup
166
+
167
+ send := func(c *MultiConn, msgs *msgMap) {
168
+ defer wg.Done()
169
+
170
+ for _, m := range msgs.msgs {
171
+ log.Info("send: %s", m.payload)
172
+ c.Out() <- []byte(m.payload)
173
+ msgs.Sent(t, m.payload)
174
+ <-time.After(time.Microsecond * 10)
175
+ }
176
+ }
177
+
178
+ recv := func(ctx context.Context, c *MultiConn, msgs *msgMap) {
179
+ defer wg.Done()
180
+
181
+ for {
182
+ select {
183
+ case payload := <-c.In():
184
+ msgs.Received(t, string(payload))
185
+ log.Info("recv: %s", payload)
186
+ if msgs.recv == len(msgs.msgs) {
187
+ return
188
+ }
189
+
190
+ case <-ctx.Done():
191
+ return
192
+
193
+ }
194
+ }
195
+
196
+ }
197
+
198
+ log.Info("msg send + recv")
199
+
200
+ wg.Add(4)
201
+ go send(c1, msgsFrom1)
202
+ go send(c2, msgsFrom2)
203
+ go recv(ctxC, c1, msgsFrom2)
204
+ go recv(ctxC, c2, msgsFrom1)
205
+ wg.Wait()
206
+ cancel()
207
+ c1.Close()
208
+ c2.Close()
209
+
210
+ msgsFrom1.CheckDone(t)
211
+ msgsFrom2.CheckDone(t)
212
+ <-time.After(100 * time.Millisecond)
213
+}