reworked Conn
Juan Batiz-Benet committed
Oct 16, 2014 at 07:18 UTC
5681e273048831c76ae02e4c383b265c8afd6408
3 files changed
+429
-88
net/conn/conn.go
+322
-46
@@ -1,12 +1,16 @@
1
package conn
2
3
import (
4
+ "errors"
5
"fmt"
6
7
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
8
msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
9
+ ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
10
manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net"
11
12
spipe "github.com/jbenet/go-ipfs/crypto/spipe"
13
+ msg "github.com/jbenet/go-ipfs/net/message"
14
peer "github.com/jbenet/go-ipfs/peer"
15
u "github.com/jbenet/go-ipfs/util"
16
)
@@ -19,40 +23,212 @@ const ChanBuffer = 10
23
// 1 MB
24
const MaxMessageSize = 1 << 20
25
22
-// Conn represents a connection to another Peer (IPFS Node).
23
-type Conn struct {
24
- Local *peer.Peer
25
- Remote *peer.Peer
26
- Conn manet.Conn
26
+// msgioPipe is a pipe using msgio channels.
27
+type msgioPipe struct {
28
+ outgoing *msgio.Chan
29
+ incoming *msgio.Chan
30
+}
31
28
- Closed chan bool
29
- Outgoing *msgio.Chan
30
- Incoming *msgio.Chan
31
- Secure *spipe.SecurePipe
32
+func newMsgioPipe(size int) *msgioPipe {
33
+ return &msgioPipe{
34
+ outgoing: msgio.NewChan(10),
35
+ incoming: msgio.NewChan(10),
36
+ }
37
+}
38
+
39
+// singleConn represents a single connection to another Peer (IPFS Node).
40
+type singleConn struct {
41
+ local *peer.Peer
42
+ remote *peer.Peer
43
+ maconn manet.Conn
44
+
45
+ // context + cancel
46
+ ctx context.Context
47
+ cancel context.CancelFunc
48
+
49
+ secure *spipe.SecurePipe
50
+ insecure *msgioPipe
51
+ msgpipe *msg.Pipe
52
}
53
54
// Map maps Keys (Peer.IDs) to Connections.
55
type Map map[u.Key]*Conn
56
37
-// NewConn constructs a new connection
38
-func NewConn(local, remote *peer.Peer, mconn manet.Conn) (*Conn, error) {
39
- conn := &Conn{
40
- Local: local,
41
- Remote: remote,
42
- Conn: mconn,
57
+// newConn constructs a new connection
58
+func newSingleConn(ctx context.Context, local, remote *peer.Peer,
59
+ peers peer.Peerstore, maconn manet.Conn) (Conn, error) {
60
+
61
+ ctx, cancel := context.WithCancel(ctx)
62
+
63
+ conn := &singleConn{
64
+ local: local,
65
+ remote: remote,
66
+ maconn: maconn,
67
+ ctx: ctx,
68
+ cancel: cancel,
69
+ insecure: newMsgioPipe(10),
70
+ msgpipe: msg.NewPipe(10),
71
}
72
45
- if err := conn.newChans(); err != nil {
73
+ log.Info("newSingleConn: %v to %v", local, remote)
74
+
75
+ // setup the various io goroutines
76
+ go conn.insecure.outgoing.WriteTo(maconn)
77
+ go conn.insecure.incoming.ReadFrom(maconn, MaxMessageSize)
78
+ go conn.waitToClose(ctx)
79
+
80
+ // perform secure handshake before returning this connection.
81
+ if err := conn.secureHandshake(peers); err != nil {
82
+ conn.Close()
83
return nil, err
84
}
85
86
return conn, nil
87
}
88
89
+// secureHandshake performs the spipe secure handshake.
90
+func (c *singleConn) secureHandshake(peers peer.Peerstore) error {
91
+ if c.secure != nil {
92
+ return errors.New("Conn is already secured or being secured.")
93
+ }
94
+
95
+ var err error
96
+ c.secure, err = spipe.NewSecurePipe(c.ctx, 10, c.local, peers)
97
+ if err != nil {
98
+ return err
99
+ }
100
+
101
+ // setup a Duplex pipe for spipe
102
+ insecure := spipe.Duplex{
103
+ In: c.insecure.incoming.MsgChan,
104
+ Out: c.insecure.outgoing.MsgChan,
105
+ }
106
+
107
+ // Wrap actually performs the secure handshake, which takes multiple RTT
108
+ if err := c.secure.Wrap(c.ctx, insecure); err != nil {
109
+ return err
110
+ }
111
+
112
+ if c.remote == nil {
113
+ c.remote = c.secure.RemotePeer()
114
+
115
+ } else if c.remote != c.secure.RemotePeer() {
116
+ // this panic is here because this would be an insidious programmer error
117
+ // that we need to ensure we catch.
118
+ log.Error("%v != %v", c.remote, c.secure.RemotePeer())
119
+ panic("peers not being constructed correctly.")
120
+ }
121
+
122
+ // silly we have to do it this way.
123
+ go c.unwrapOutMsgs()
124
+ go c.wrapInMsgs()
125
+
126
+ return nil
127
+}
128
+
129
+// unwrapOutMsgs sends just the raw data of a message through secure
130
+func (c *singleConn) unwrapOutMsgs() {
131
+ for {
132
+ select {
133
+ case <-c.ctx.Done():
134
+ return
135
+ case m, more := <-c.msgpipe.Outgoing:
136
+ if !more {
137
+ return
138
+ }
139
+
140
+ c.secure.Out <- m.Data()
141
+ }
142
+ }
143
+}
144
+
145
+// wrapInMsgs wraps a message
146
+func (c *singleConn) wrapInMsgs() {
147
+ for {
148
+ select {
149
+ case <-c.ctx.Done():
150
+ return
151
+ case d, more := <-c.secure.In:
152
+ if !more {
153
+ return
154
+ }
155
+
156
+ c.msgpipe.Incoming <- msg.New(c.remote, d)
157
+ }
158
+ }
159
+}
160
+
161
+// waitToClose waits on the given context's Done before closing Conn.
162
+func (c *singleConn) waitToClose(ctx context.Context) {
163
+ select {
164
+ case <-ctx.Done():
165
+ }
166
+
167
+ // close underlying connection
168
+ c.maconn.Close()
169
+ c.maconn = nil
170
+
171
+ // closing channels
172
+ c.insecure.outgoing.Close()
173
+ c.secure.Close()
174
+}
175
+
176
+// IsOpen returns whether this Conn is open or closed.
177
+func (c *singleConn) isOpen() bool {
178
+ return c.maconn != nil
179
+}
180
+
181
+// Close closes the connection, and associated channels.
182
+func (c *singleConn) Close() error {
183
+ log.Debug("%s closing Conn with %s", c.local, c.remote)
184
+ if !c.isOpen() {
185
+ return fmt.Errorf("Already closed") // already closed
186
+ }
187
+
188
+ // cancel context.
189
+ c.cancel()
190
+ c.cancel = nil
191
+ return nil
192
+}
193
+
194
+// LocalPeer is the Peer on this side
195
+func (c *singleConn) LocalPeer() *peer.Peer {
196
+ return c.local
197
+}
198
+
199
+// RemotePeer is the Peer on the remote side
200
+func (c *singleConn) RemotePeer() *peer.Peer {
201
+ return c.remote
202
+}
203
+
204
+// MsgIn returns a readable message channel
205
+func (c *singleConn) MsgIn() <-chan msg.NetMessage {
206
+ return c.msgpipe.Incoming
207
+}
208
+
209
+// MsgOut returns a writable message channel
210
+func (c *singleConn) MsgOut() chan<- msg.NetMessage {
211
+ return c.msgpipe.Outgoing
212
+}
213
+
214
+// Dialer is an object that can open connections. We could have a "convenience"
215
+// Dial function as before, but it would have many arguments, as dialing is
216
+// no longer simple (need a peerstore, a local peer, a context, a network, etc)
217
+type Dialer struct {
218
+
219
+ // LocalPeer is the identity of the local Peer.
220
+ LocalPeer *peer.Peer
221
+
222
+ // Peerstore is the set of peers we know about locally. The Dialer needs it
223
+ // because when an incoming connection is identified, we should reuse the
224
+ // same peer objects (otherwise things get inconsistent).
225
+ Peerstore peer.Peerstore
226
+}
227
+
228
// Dial connects to a particular peer, over a given network
53
-// Example: Dial("udp", peer)
54
-func Dial(network string, local, remote *peer.Peer) (*Conn, error) {
55
- laddr := local.NetAddress(network)
229
+// Example: d.Dial(ctx, "udp", peer)
230
+func (d *Dialer) Dial(ctx context.Context, network string, remote *peer.Peer) (Conn, error) {
231
+ laddr := d.LocalPeer.NetAddress(network)
232
if laddr == nil {
233
return nil, fmt.Errorf("No local address for network %s", network)
234
}
@@ -63,47 +239,147 @@ func Dial(network string, local, remote *peer.Peer) (*Conn, error) {
239
}
240
241
// TODO: try to get reusing addr/ports to work.
66
- // dialer := manet.Dialer{LocalAddr: laddr}
67
- dialer := manet.Dialer{}
242
+ // madialer := manet.Dialer{LocalAddr: laddr}
243
+ madialer := manet.Dialer{}
244
69
- log.Info("%s %s dialing %s %s", local, laddr, remote, raddr)
70
- nconn, err := dialer.Dial(raddr)
245
+ log.Info("%s dialing %s %s", d.LocalPeer, remote, raddr)
246
+ maconn, err := madialer.Dial(raddr)
247
if err != nil {
248
return nil, err
249
}
250
75
- return NewConn(local, remote, nconn)
251
+ if err := d.Peerstore.Put(remote); err != nil {
252
+ log.Error("Error putting peer into peerstore: %s", remote)
253
+ }
254
+
255
+ return newSingleConn(ctx, d.LocalPeer, remote, d.Peerstore, maconn)
256
+}
257
+
258
+// listener is an object that can accept connections. It implements Listener
259
+type listener struct {
260
+ manet.Listener
261
+
262
+ // chansize is the size of the internal channels for concurrency
263
+ chansize int
264
+
265
+ // channel of incoming conections
266
+ conns chan Conn
267
+
268
+ // Local multiaddr to listen on
269
+ maddr ma.Multiaddr
270
+
271
+ // LocalPeer is the identity of the local Peer.
272
+ local *peer.Peer
273
+
274
+ // Peerstore is the set of peers we know about locally
275
+ peers peer.Peerstore
276
+
277
+ // ctx + cancel func
278
+ ctx context.Context
279
+ cancel context.CancelFunc
280
+}
281
+
282
+// waitToClose is needed to hand
283
+func (l *listener) waitToClose() {
284
+ select {
285
+ case <-l.ctx.Done():
286
+ }
287
+
288
+ l.cancel = nil
289
+ l.Listener.Close()
290
}
291
78
-// Construct new channels for given Conn.
79
-func (c *Conn) newChans() error {
80
- if c.Outgoing != nil || c.Incoming != nil {
81
- return fmt.Errorf("Conn already initialized")
292
+func (l *listener) listen() {
293
+
294
+ // handle at most chansize concurrent handshakes
295
+ sem := make(chan struct{}, l.chansize)
296
+
297
+ // handle is a goroutine work function that handles the handshake.
298
+ // it's here only so that accepting new connections can happen quickly.
299
+ handle := func(maconn manet.Conn) {
300
+ c, err := newSingleConn(l.ctx, l.local, nil, l.peers, maconn)
301
+ if err != nil {
302
+ log.Error("Error accepting connection: %v", err)
303
+ } else {
304
+ l.conns <- c
305
+ }
306
+ <-sem // release
307
}
308
84
- c.Outgoing = msgio.NewChan(10)
85
- c.Incoming = msgio.NewChan(10)
86
- c.Closed = make(chan bool, 1)
309
+ for {
310
+ maconn, err := l.Listener.Accept()
311
+ if err != nil {
312
88
- go c.Outgoing.WriteTo(c.Conn)
89
- go c.Incoming.ReadFrom(c.Conn, MaxMessageSize)
313
+ // if cancel is nil we're closed.
314
+ if l.cancel == nil {
315
+ return // done.
316
+ }
317
318
+ log.Error("Failed to accept connection: %v", err)
319
+ continue
320
+ }
321
+
322
+ sem <- struct{}{} // acquire
323
+ go handle(maconn)
324
+ }
325
+}
326
+
327
+// Accept waits for and returns the next connection to the listener.
328
+// Note that unfortunately this
329
+func (l *listener) Accept() <-chan Conn {
330
+ return l.conns
331
+}
332
+
333
+// Multiaddr is the identity of the local Peer.
334
+func (l *listener) Multiaddr() ma.Multiaddr {
335
+ return l.maddr
336
+}
337
+
338
+// LocalPeer is the identity of the local Peer.
339
+func (l *listener) LocalPeer() *peer.Peer {
340
+ return l.local
341
+}
342
+
343
+// Peerstore is the set of peers we know about locally. The Listener needs it
344
+// because when an incoming connection is identified, we should reuse the
345
+// same peer objects (otherwise things get inconsistent).
346
+func (l *listener) Peerstore() peer.Peerstore {
347
+ return l.peers
348
+}
349
+
350
+// Close closes the listener.
351
+// Any blocked Accept operations will be unblocked and return errors
352
+func (l *listener) Close() error {
353
+ l.cancel()
354
return nil
355
}
356
94
-// Close closes the connection, and associated channels.
95
-func (c *Conn) Close() error {
96
- log.Debug("%s closing Conn with %s", c.Local, c.Remote)
97
- if c.Conn == nil {
98
- return fmt.Errorf("Already closed") // already closed
357
+// Listen listens on the particular multiaddr, with given peer and peerstore.
358
+func Listen(ctx context.Context, addr ma.Multiaddr, local *peer.Peer, peers peer.Peerstore) (Listener, error) {
359
+
360
+ ctx, cancel := context.WithCancel(ctx)
361
+
362
+ ml, err := manet.Listen(addr)
363
+ if err != nil {
364
+ return nil, err
365
}
366
101
- // closing net connection
102
- err := c.Conn.Close()
103
- c.Conn = nil
104
- // closing channels
105
- c.Incoming.Close()
106
- c.Outgoing.Close()
107
- c.Closed <- true
108
- return err
367
+ // todo make this a variable
368
+ chansize := 10
369
+
370
+ l := &listener{
371
+ ctx: ctx,
372
+ cancel: cancel,
373
+ Listener: ml,
374
+ maddr: addr,
375
+ peers: peers,
376
+ local: local,
377
+ conns: make(chan Conn, chansize),
378
+ chansize: chansize,
379
+ }
380
+
381
+ go l.listen()
382
+ go l.waitToClose()
383
+
384
+ return l, nil
385
}
net/conn/conn_test.go
+58
-42
@@ -3,98 +3,114 @@ package conn
3
import (
4
"testing"
5
6
+ ci "github.com/jbenet/go-ipfs/crypto"
7
+ msg "github.com/jbenet/go-ipfs/net/message"
8
peer "github.com/jbenet/go-ipfs/peer"
9
10
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
11
ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
9
- manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net"
10
- mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
12
)
13
13
-func setupPeer(id string, addr string) (*peer.Peer, error) {
14
+func setupPeer(addr string) (*peer.Peer, error) {
15
tcp, err := ma.NewMultiaddr(addr)
16
if err != nil {
17
return nil, err
18
}
19
19
- mh, err := mh.FromHexString(id)
20
+ sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
21
if err != nil {
22
return nil, err
23
}
24
24
- p := &peer.Peer{ID: peer.ID(mh)}
25
+ id, err := peer.IDFromPubKey(pk)
26
+ if err != nil {
27
+ return nil, err
28
+ }
29
+
30
+ p := &peer.Peer{ID: id}
31
+ p.PrivKey = sk
32
+ p.PubKey = pk
33
p.AddAddress(tcp)
34
return p, nil
35
}
36
29
-func echoListen(listener manet.Listener) {
37
+func echoListen(ctx context.Context, listener Listener) {
38
for {
31
- c, err := listener.Accept()
32
- if err == nil {
33
- // fmt.Println("accepeted")
34
- go echo(c)
39
+ select {
40
+ case <-ctx.Done():
41
+ return
42
+ case c := <-listener.Accept():
43
+ go echo(ctx, c)
44
}
45
}
46
}
47
39
-func echo(c manet.Conn) {
48
+func echo(ctx context.Context, c Conn) {
49
for {
41
- data := make([]byte, 1024)
42
- i, err := c.Read(data)
43
- if err != nil {
44
- // fmt.Printf("error %v\n", err)
50
+ select {
51
+ case <-ctx.Done():
52
return
53
+ case m := <-c.MsgIn():
54
+ c.MsgOut() <- m
55
}
47
- _, err = c.Write(data[:i])
48
- if err != nil {
49
- // fmt.Printf("error %v\n", err)
50
- return
51
- }
52
- // fmt.Println("echoing", data[:i])
56
}
57
}
58
56
-func TestDial(t *testing.T) {
59
+func TestDialer(t *testing.T) {
60
58
- maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1234")
59
- if err != nil {
60
- t.Fatal("failure to parse multiaddr")
61
- }
62
- listener, err := manet.Listen(maddr)
61
+ p1, err := setupPeer("/ip4/127.0.0.1/tcp/1234")
62
if err != nil {
64
- t.Fatal("error setting up listener", err)
63
+ t.Fatal("error setting up peer", err)
64
}
66
- go echoListen(listener)
65
68
- p1, err := setupPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", "/ip4/127.0.0.1/tcp/1234")
66
+ p2, err := setupPeer("/ip4/127.0.0.1/tcp/3456")
67
if err != nil {
68
t.Fatal("error setting up peer", err)
69
}
70
73
- p2, err := setupPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a34", "/ip4/127.0.0.1/tcp/3456")
71
+ ctx, cancel := context.WithCancel(context.Background())
72
+
73
+ laddr := p1.NetAddress("tcp")
74
+ if laddr == nil {
75
+ t.Fatal("Listen address is nil.")
76
+ }
77
+
78
+ l, err := Listen(ctx, laddr, p1, peer.NewPeerstore())
79
if err != nil {
75
- t.Fatal("error setting up peer", err)
80
+ t.Fatal(err)
81
}
82
78
- c, err := Dial("tcp", p2, p1)
83
+ go echoListen(ctx, l)
84
+
85
+ d := &Dialer{
86
+ Peerstore: peer.NewPeerstore(),
87
+ LocalPeer: p2,
88
+ }
89
+
90
+ c, err := d.Dial(ctx, "tcp", p1)
91
if err != nil {
92
t.Fatal("error dialing peer", err)
93
}
94
95
// fmt.Println("sending")
84
- c.Outgoing.MsgChan <- []byte("beep")
85
- c.Outgoing.MsgChan <- []byte("boop")
86
- out := <-c.Incoming.MsgChan
96
+ c.MsgOut() <- msg.New(p2, []byte("beep"))
97
+ c.MsgOut() <- msg.New(p2, []byte("boop"))
98
+
99
+ out := <-c.MsgIn()
100
// fmt.Println("recving", string(out))
88
- if string(out) != "beep" {
89
- t.Error("unexpected conn output")
101
+ data := string(out.Data())
102
+ if data != "beep" {
103
+ t.Error("unexpected conn output", data)
104
}
105
92
- out = <-c.Incoming.MsgChan
93
- if string(out) != "boop" {
94
- t.Error("unexpected conn output")
106
+ out = <-c.MsgIn()
107
+ data = string(out.Data())
108
+ if string(out.Data()) != "boop" {
109
+ t.Error("unexpected conn output", data)
110
}
111
112
// fmt.Println("closing")
113
c.Close()
99
- listener.Close()
114
+ l.Close()
115
+ cancel()
116
}
net/conn/interface.go
new
+49
@@ -0,0 +1,49 @@
1
+package conn
2
+
3
+import (
4
+ msg "github.com/jbenet/go-ipfs/net/message"
5
+ peer "github.com/jbenet/go-ipfs/peer"
6
+
7
+ ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
8
+)
9
+
10
+// Conn is a generic message-based Peer-to-Peer connection.
11
+type Conn interface {
12
+
13
+ // LocalPeer is the Peer on this side
14
+ LocalPeer() *peer.Peer
15
+
16
+ // RemotePeer is the Peer on the remote side
17
+ RemotePeer() *peer.Peer
18
+
19
+ // MsgIn returns a readable message channel
20
+ MsgIn() <-chan msg.NetMessage
21
+
22
+ // MsgOut returns a writable message channel
23
+ MsgOut() chan<- msg.NetMessage
24
+
25
+ // Close ends the connection
26
+ Close() error
27
+}
28
+
29
+// Listener is an object that can accept connections. It matches net.Listener
30
+type Listener interface {
31
+
32
+ // Accept waits for and returns the next connection to the listener.
33
+ Accept() <-chan Conn
34
+
35
+ // Multiaddr is the identity of the local Peer.
36
+ Multiaddr() ma.Multiaddr
37
+
38
+ // LocalPeer is the identity of the local Peer.
39
+ LocalPeer() *peer.Peer
40
+
41
+ // Peerstore is the set of peers we know about locally. The Listener needs it
42
+ // because when an incoming connection is identified, we should reuse the
43
+ // same peer objects (otherwise things get inconsistent).
44
+ Peerstore() peer.Peerstore
45
+
46
+ // Close closes the listener.
47
+ // Any blocked Accept operations will be unblocked and return errors.
48
+ Close() error
49
+}