net/conn: io, not channels
This commit changes the connections to use io.ReadWriters instead of channels (+ async workers). This is a pretty big change -- away from csp -- in the name of performance (and predictable flow control). It also uses the brand new secio, which is spipe's successor.
Juan Batiz-Benet committed
Dec 11, 2014 at 16:31 UTC
5341379f9dc524291a9f5d378ef0e2b52452463e
12 files changed
+349
-324
net/conn/conn.go
+28
-53
@@ -18,9 +18,6 @@ import (
18
var log = u.Logger("conn")
19
20
const (
21
- // ChanBuffer is the size of the buffer in the Conn Chan
22
- ChanBuffer = 10
23
-
21
// MaxMessageSize is the size of the largest single message. (4MB)
22
MaxMessageSize = 1 << 22
23
@@ -35,25 +32,12 @@ func ReleaseBuffer(b []byte) {
32
mpool.ByteSlicePool.Put(uint32(cap(b)), b)
33
}
34
38
-// msgioPipe is a pipe using msgio channels.
39
-type msgioPipe struct {
40
- outgoing *msgio.Chan
41
- incoming *msgio.Chan
42
-}
43
-
44
-func newMsgioPipe(size int) *msgioPipe {
45
- return &msgioPipe{
46
- outgoing: msgio.NewChan(size),
47
- incoming: msgio.NewChan(size),
48
- }
49
-}
50
-
35
// singleConn represents a single connection to another Peer (IPFS Node).
36
type singleConn struct {
37
local peer.Peer
38
remote peer.Peer
39
maconn manet.Conn
56
- msgio *msgioPipe
40
+ msgrw msgio.ReadWriteCloser
41
42
ctxc.ContextCloser
43
}
@@ -66,24 +50,12 @@ func newSingleConn(ctx context.Context, local, remote peer.Peer,
50
local: local,
51
remote: remote,
52
maconn: maconn,
69
- msgio: newMsgioPipe(10),
53
+ msgrw: msgio.NewReadWriter(maconn),
54
}
55
56
conn.ContextCloser = ctxc.NewContextCloser(ctx, conn.close)
57
74
- log.Debugf("newSingleConn: %v to %v", local, remote)
75
-
76
- // setup the various io goroutines
77
- conn.Children().Add(1)
78
- go func() {
79
- conn.msgio.outgoing.WriteTo(maconn)
80
- conn.Children().Done()
81
- }()
82
- conn.Children().Add(1)
83
- go func() {
84
- conn.msgio.incoming.ReadFromWithPool(maconn, &mpool.ByteSlicePool)
85
- conn.Children().Done()
86
- }()
58
+ log.Debugf("newSingleConn %p: %v to %v", conn, local, remote)
59
60
// version handshake
61
ctxT, _ := context.WithTimeout(ctx, HandshakeTimeout)
@@ -92,6 +64,7 @@ func newSingleConn(ctx context.Context, local, remote peer.Peer,
64
return nil, fmt.Errorf("Handshake1 failed: %s", err)
65
}
66
67
+ log.Debugf("newSingleConn %p: %v to %v finished", conn, local, remote)
68
return conn, nil
69
}
70
@@ -100,20 +73,7 @@ func (c *singleConn) close() error {
73
log.Debugf("%s closing Conn with %s", c.local, c.remote)
74
75
// close underlying connection
103
- err := c.maconn.Close()
104
- c.msgio.outgoing.Close()
105
- return err
106
-}
107
-
108
-func (c *singleConn) GetError() error {
109
- select {
110
- case err := <-c.msgio.incoming.ErrChan:
111
- return err
112
- case err := <-c.msgio.outgoing.ErrChan:
113
- return err
114
- default:
115
- return nil
116
- }
76
+ return c.msgrw.Close()
77
}
78
79
// ID is an identifier unique to this connection.
@@ -145,14 +105,29 @@ func (c *singleConn) RemotePeer() peer.Peer {
105
return c.remote
106
}
107
148
-// In returns a readable message channel
149
-func (c *singleConn) In() <-chan []byte {
150
- return c.msgio.incoming.MsgChan
108
+// Read reads data, net.Conn style
109
+func (c *singleConn) Read(buf []byte) (int, error) {
110
+ return c.msgrw.Read(buf)
111
+}
112
+
113
+// Write writes data, net.Conn style
114
+func (c *singleConn) Write(buf []byte) (int, error) {
115
+ return c.msgrw.Write(buf)
116
+}
117
+
118
+// ReadMsg reads data, net.Conn style
119
+func (c *singleConn) ReadMsg() ([]byte, error) {
120
+ return c.msgrw.ReadMsg()
121
+}
122
+
123
+// WriteMsg writes data, net.Conn style
124
+func (c *singleConn) WriteMsg(buf []byte) error {
125
+ return c.msgrw.WriteMsg(buf)
126
}
127
153
-// Out returns a writable message channel
154
-func (c *singleConn) Out() chan<- []byte {
155
- return c.msgio.outgoing.MsgChan
128
+// ReleaseMsg releases a buffer
129
+func (c *singleConn) ReleaseMsg(m []byte) {
130
+ c.msgrw.ReleaseMsg(m)
131
}
132
133
// ID returns the ID of a given Conn.
@@ -167,6 +142,6 @@ func ID(c Conn) string {
142
143
// String returns the user-friendly String representation of a conn
144
func String(c Conn, typ string) string {
170
- return fmt.Sprintf("%s (%s) <-- %s --> (%s) %s",
171
- c.LocalPeer(), c.LocalMultiaddr(), typ, c.RemoteMultiaddr(), c.RemotePeer())
145
+ return fmt.Sprintf("%s (%s) <-- %s %p --> (%s) %s",
146
+ c.LocalPeer(), c.LocalMultiaddr(), typ, c, c.RemoteMultiaddr(), c.RemotePeer())
147
}
net/conn/conn_test.go
+15
-9
@@ -100,18 +100,24 @@ func TestCloseLeak(t *testing.T) {
100
c1, c2 := setupConn(t, ctx, "/ip4/127.0.0.1/tcp/"+a1, "/ip4/127.0.0.1/tcp/"+a2)
101
102
for i := 0; i < num; i++ {
103
- b1 := []byte("beep")
104
- c1.Out() <- b1
105
- b2 := <-c2.In()
103
+ b1 := []byte(fmt.Sprintf("beep%d", i))
104
+ c1.WriteMsg(b1)
105
+ b2, err := c2.ReadMsg()
106
+ if err != nil {
107
+ panic(err)
108
+ }
109
if !bytes.Equal(b1, b2) {
107
- panic("bytes not equal")
110
+ panic(fmt.Errorf("bytes not equal: %s != %s", b1, b2))
111
}
112
110
- b2 = []byte("boop")
111
- c2.Out() <- b2
112
- b1 = <-c1.In()
113
+ b2 = []byte(fmt.Sprintf("boop%d", i))
114
+ c2.WriteMsg(b2)
115
+ b1, err = c1.ReadMsg()
116
+ if err != nil {
117
+ panic(err)
118
+ }
119
if !bytes.Equal(b1, b2) {
114
- panic("bytes not equal")
120
+ panic(fmt.Errorf("bytes not equal: %s != %s", b1, b2))
121
}
122
123
<-time.After(time.Microsecond * 5)
@@ -123,7 +129,7 @@ func TestCloseLeak(t *testing.T) {
129
wg.Done()
130
}
131
126
- var cons = 20
132
+ var cons = 1
133
var msgs = 100
134
fmt.Printf("Running %d connections * %d msgs.\n", cons, msgs)
135
for i := 0; i < cons; i++ {
net/conn/dial.go
+1
@@ -72,5 +72,6 @@ func (d *Dialer) DialAddr(ctx context.Context, raddr ma.Multiaddr, remote peer.P
72
return nil, err
73
}
74
75
+ // return c, nil
76
return newSecureConn(ctx, c, d.Peerstore)
77
}
net/conn/dial_test.go
+25
-16
@@ -1,6 +1,7 @@
1
package conn
2
3
import (
4
+ "io"
5
"testing"
6
7
ci "github.com/jbenet/go-ipfs/crypto"
@@ -42,14 +43,7 @@ func echoListen(ctx context.Context, listener Listener) {
43
}
44
45
func echo(ctx context.Context, c Conn) {
45
- for {
46
- select {
47
- case <-ctx.Done():
48
- return
49
- case m := <-c.In():
50
- c.Out() <- m
51
- }
52
- }
46
+ io.Copy(c, c)
47
}
48
49
func setupConn(t *testing.T, ctx context.Context, a1, a2 string) (a, b Conn) {
@@ -137,17 +131,25 @@ func TestDialer(t *testing.T) {
131
}
132
133
// fmt.Println("sending")
140
- c.Out() <- []byte("beep")
141
- c.Out() <- []byte("boop")
134
+ c.WriteMsg([]byte("beep"))
135
+ c.WriteMsg([]byte("boop"))
136
+
137
+ out, err := c.ReadMsg()
138
+ if err != nil {
139
+ t.Fatal(err)
140
+ }
141
143
- out := <-c.In()
142
// fmt.Println("recving", string(out))
143
data := string(out)
144
if data != "beep" {
145
t.Error("unexpected conn output", data)
146
}
147
150
- out = <-c.In()
148
+ out, err = c.ReadMsg()
149
+ if err != nil {
150
+ t.Fatal(err)
151
+ }
152
+
153
data = string(out)
154
if string(out) != "boop" {
155
t.Error("unexpected conn output", data)
@@ -207,17 +209,24 @@ func TestDialAddr(t *testing.T) {
209
}
210
211
// fmt.Println("sending")
210
- c.Out() <- []byte("beep")
211
- c.Out() <- []byte("boop")
212
+ c.WriteMsg([]byte("beep"))
213
+ c.WriteMsg([]byte("boop"))
214
213
- out := <-c.In()
215
+ out, err := c.ReadMsg()
216
+ if err != nil {
217
+ t.Fatal(err)
218
+ }
219
// fmt.Println("recving", string(out))
220
data := string(out)
221
if data != "beep" {
222
t.Error("unexpected conn output", data)
223
}
224
220
- out = <-c.In()
225
+ out, err = c.ReadMsg()
226
+ if err != nil {
227
+ t.Fatal(err)
228
+ }
229
+
230
data = string(out)
231
if string(out) != "boop" {
232
t.Error("unexpected conn output", data)
net/conn/handshake.go
+26
-40
@@ -1,7 +1,6 @@
1
package conn
2
3
import (
4
- "errors"
4
"fmt"
5
6
handshake "github.com/jbenet/go-ipfs/net/handshake"
@@ -25,29 +24,22 @@ func Handshake1(ctx context.Context, c Conn) error {
24
return err
25
}
26
28
- c.Out() <- myVerBytes
29
- log.Debugf("Sent my version (%s) to %s", localH, rpeer)
30
-
31
- select {
32
- case <-ctx.Done():
33
- return ctx.Err()
34
-
35
- case <-c.Closing():
36
- return errors.New("remote closed connection during version exchange")
37
-
38
- case data, ok := <-c.In():
39
- if !ok {
40
- return fmt.Errorf("error retrieving from conn: %v", rpeer)
41
- }
27
+ if err := CtxWriteMsg(ctx, c, myVerBytes); err != nil {
28
+ return err
29
+ }
30
+ log.Debugf("%p sent my version (%s) to %s", c, localH, rpeer)
31
43
- remoteH = new(hspb.Handshake1)
44
- err = proto.Unmarshal(data, remoteH)
45
- if err != nil {
46
- return fmt.Errorf("could not decode remote version: %q", err)
47
- }
32
+ data, err := CtxReadMsg(ctx, c)
33
+ if err != nil {
34
+ return err
35
+ }
36
49
- log.Debugf("Received remote version (%s) from %s", remoteH, rpeer)
37
+ remoteH = new(hspb.Handshake1)
38
+ err = proto.Unmarshal(data, remoteH)
39
+ if err != nil {
40
+ return fmt.Errorf("could not decode remote version: %q", err)
41
}
42
+ log.Debugf("%p received remote version (%s) from %s", c, remoteH, rpeer)
43
44
if err := handshake.Handshake1Compatible(localH, remoteH); err != nil {
45
log.Infof("%s (%s) incompatible version with %s (%s)", lpeer, localH, rpeer, remoteH)
@@ -71,31 +63,25 @@ func Handshake3(ctx context.Context, c Conn) (*handshake.Handshake3Result, error
63
return nil, err
64
}
65
74
- c.Out() <- localB
66
+ if err := CtxWriteMsg(ctx, c, localB); err != nil {
67
+ return nil, err
68
+ }
69
log.Debugf("Handshake1: sent to %s", rpeer)
70
71
// wait + listen for response
78
- select {
79
- case <-ctx.Done():
80
- return nil, ctx.Err()
81
-
82
- case <-c.Closing():
83
- return nil, errors.New("Handshake3: error remote connection closed")
84
-
85
- case remoteB, ok := <-c.In():
86
- if !ok {
87
- return nil, fmt.Errorf("Handshake3 error receiving from conn: %v", rpeer)
88
- }
89
-
90
- remoteH = new(hspb.Handshake3)
91
- err = proto.Unmarshal(remoteB, remoteH)
92
- if err != nil {
93
- return nil, fmt.Errorf("Handshake3 could not decode remote msg: %q", err)
94
- }
72
+ remoteB, err := CtxReadMsg(ctx, c)
73
+ if err != nil {
74
+ return nil, err
75
+ }
76
96
- log.Debugf("Handshake3 received from %s", rpeer)
77
+ remoteH = new(hspb.Handshake3)
78
+ err = proto.Unmarshal(remoteB, remoteH)
79
+ if err != nil {
80
+ return nil, fmt.Errorf("Handshake3 could not decode remote msg: %q", err)
81
}
82
83
+ log.Debugf("Handshake3 received from %s", rpeer)
84
+
85
// actually update our state based on the new knowledge
86
res, err := handshake.Handshake3Update(lpeer, rpeer, remoteH)
87
if err != nil {
net/conn/interface.go
+92
-9
@@ -1,10 +1,14 @@
1
package conn
2
3
import (
4
+ "errors"
5
+
6
peer "github.com/jbenet/go-ipfs/peer"
7
u "github.com/jbenet/go-ipfs/util"
8
ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
9
10
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
11
+ msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
12
ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
13
)
14
@@ -31,15 +35,8 @@ type Conn interface {
35
// RemotePeer is the Peer on the remote side
36
RemotePeer() peer.Peer
37
34
- // In returns a readable message channel
35
- In() <-chan []byte
36
-
37
- // Out returns a writable message channel
38
- Out() chan<- []byte
39
-
40
- // Get an error from this conn if one is available
41
- // TODO: implement a better error handling system
42
- GetError() error
38
+ msgio.Reader
39
+ msgio.Writer
40
}
41
42
// Dialer is an object that can open connections. We could have a "convenience"
@@ -77,3 +74,89 @@ type Listener interface {
74
// Any blocked Accept operations will be unblocked and return errors.
75
Close() error
76
}
77
+
78
+// CtxRead is a function that Reads from a connection while respecting a
79
+// Context. Though it cannot cancel the read per-se (as not all Connections
80
+// implement SetTimeout, and a CancelFunc can't be predicted), at least it
81
+// doesn't hang. The Read will eventually return and the goroutine will exit.
82
+func CtxRead(ctx context.Context, c Conn, buf []byte) (n int, err error) {
83
+ done := make(chan struct{})
84
+ go func() {
85
+ n, err = c.Read(buf)
86
+ close(done)
87
+ }()
88
+
89
+ select {
90
+ case <-ctx.Done():
91
+ return 0, ctx.Err()
92
+
93
+ case <-c.Closing():
94
+ return 0, errors.New("remote connection closed")
95
+
96
+ case <-done:
97
+ return n, err
98
+ }
99
+}
100
+
101
+// CtxReadMsg is a function that Reads from a connection while respecting a
102
+// Context. See CtxRead.
103
+func CtxReadMsg(ctx context.Context, c Conn) (msg []byte, err error) {
104
+ done := make(chan struct{})
105
+ go func() {
106
+ msg, err = c.ReadMsg()
107
+ close(done)
108
+ }()
109
+
110
+ select {
111
+ case <-ctx.Done():
112
+ return msg, ctx.Err()
113
+
114
+ case <-c.Closing():
115
+ return msg, errors.New("remote connection closed")
116
+
117
+ case <-done:
118
+ return msg, err
119
+ }
120
+}
121
+
122
+// CtxWrite is a function that Writes to a connection while respecting a
123
+// Context. See CtxRead.
124
+func CtxWrite(ctx context.Context, c Conn, buf []byte) (n int, err error) {
125
+ done := make(chan struct{})
126
+ go func() {
127
+ n, err = c.Read(buf)
128
+ close(done)
129
+ }()
130
+
131
+ select {
132
+ case <-ctx.Done():
133
+ return 0, ctx.Err()
134
+
135
+ case <-c.Closing():
136
+ return 0, errors.New("remote connection closed")
137
+
138
+ case <-done:
139
+ return n, err
140
+ }
141
+}
142
+
143
+// CtxWriteMsg is a function that Writes to a connection while respecting a
144
+// Context. See CtxRead.
145
+func CtxWriteMsg(ctx context.Context, c Conn, buf []byte) (err error) {
146
+ done := make(chan struct{})
147
+ go func() {
148
+ err = c.WriteMsg(buf)
149
+ close(done)
150
+ }()
151
+
152
+ select {
153
+ case <-ctx.Done():
154
+ return ctx.Err()
155
+
156
+ case <-c.Closing():
157
+ return errors.New("remote connection closed")
158
+
159
+ case <-done:
160
+ return err
161
+ }
162
+}
net/conn/listen.go
+4
-1
@@ -65,12 +65,15 @@ func (l *listener) listen() {
65
return
66
}
67
68
+ // if insecure:
69
+ // l.conns <- c
70
+
71
+ // if secure
72
sc, err := newSecureConn(l.ctx, c, l.peers)
73
if err != nil {
74
log.Errorf("Error securing connection: %v", err)
75
return
76
}
73
-
77
l.conns <- sc
78
}
79
net/conn/multiconn.go
+42
-74
@@ -1,6 +1,7 @@
1
package conn
2
3
import (
4
+ "errors"
5
"sync"
6
7
context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
@@ -14,12 +15,6 @@ import (
15
// MultiConnMap is for shorthand
16
type MultiConnMap map[u.Key]*MultiConn
17
17
-// Duplex is a simple duplex channel
18
-type Duplex struct {
19
- In chan []byte
20
- Out chan []byte
21
-}
22
-
18
// MultiConn represents a single connection to another Peer (IPFS Node).
19
type MultiConn struct {
20
@@ -30,8 +25,8 @@ type MultiConn struct {
25
local peer.Peer
26
remote peer.Peer
27
33
- // fan-in/fan-out
34
- duplex Duplex
28
+ // fan-in
29
+ fanIn chan []byte
30
31
// for adding/removing connections concurrently
32
sync.RWMutex
@@ -45,10 +40,7 @@ func NewMultiConn(ctx context.Context, local, remote peer.Peer, conns []Conn) (*
40
local: local,
41
remote: remote,
42
conns: map[string]Conn{},
48
- duplex: Duplex{
49
- In: make(chan []byte, 10),
50
- Out: make(chan []byte, 10),
51
- },
43
+ fanIn: make(chan []byte),
44
}
45
46
// must happen before Adds / fanOut
@@ -58,8 +50,6 @@ func NewMultiConn(ctx context.Context, local, remote peer.Peer, conns []Conn) (*
50
c.Add(conns...)
51
}
52
61
- c.Children().Add(1)
62
- go c.fanOut()
53
return c, nil
54
}
55
@@ -135,38 +125,8 @@ func CloseConns(conns ...Conn) {
125
wg.Wait()
126
}
127
138
-// fanOut is the multiplexor out -- it sends outgoing messages over the
139
-// underlying single connections.
140
-func (c *MultiConn) fanOut() {
141
- defer c.Children().Done()
142
-
143
- i := 0
144
- for {
145
- select {
146
- case <-c.Closing():
147
- return
148
-
149
- // send data out through our "best connection"
150
- case m, more := <-c.duplex.Out:
151
- if !more {
152
- log.Debugf("%s out channel closed", c)
153
- return
154
- }
155
- sc := c.BestConn()
156
- if sc == nil {
157
- // maybe this should be a logged error, not a panic.
158
- panic("sending out multiconn without any live connection")
159
- }
160
-
161
- i++
162
- log.Debugf("%s sending (%d)", sc, i)
163
- sc.Out() <- m
164
- }
165
- }
166
-}
167
-
168
-// fanInSingle is a multiplexor in -- it receives incoming messages over the
169
-// underlying single connections.
128
+// fanInSingle Reads from a connection, and sends to the fanIn.
129
+// waits for child to close and reclaims resources
130
func (c *MultiConn) fanInSingle(child Conn) {
131
// cleanup all data associated with this child Connection.
132
defer func() {
@@ -186,8 +146,13 @@ func (c *MultiConn) fanInSingle(child Conn) {
146
}
147
}()
148
189
- i := 0
149
for {
150
+ msg, err := child.ReadMsg()
151
+ if err != nil {
152
+ log.Warning(err)
153
+ return
154
+ }
155
+
156
select {
157
case <-c.Closing(): // multiconn closing
158
return
@@ -195,18 +160,7 @@ func (c *MultiConn) fanInSingle(child Conn) {
160
case <-child.Closing(): // child closing
161
return
162
198
- case m, more := <-child.In(): // receiving data
199
- if !more {
200
- log.Debugf("%s in channel closed", child)
201
- err := c.GetError()
202
- if err != nil {
203
- log.Errorf("Found error on connection: %s", err)
204
- }
205
- return // closed
206
- }
207
- i++
208
- log.Debugf("%s received (%d)", child, i)
209
- c.duplex.In <- m
163
+ case c.fanIn <- msg:
164
}
165
}
166
}
@@ -296,24 +250,38 @@ func (c *MultiConn) RemotePeer() peer.Peer {
250
return c.remote
251
}
252
299
-// In returns a readable message channel
300
-func (c *MultiConn) In() <-chan []byte {
301
- return c.duplex.In
253
+// Read reads data, net.Conn style
254
+func (c *MultiConn) Read(buf []byte) (int, error) {
255
+ return 0, errors.New("multiconn does not support Read. use ReadMsg")
256
}
257
304
-// Out returns a writable message channel
305
-func (c *MultiConn) Out() chan<- []byte {
306
- return c.duplex.Out
258
+// Write writes data, net.Conn style
259
+func (c *MultiConn) Write(buf []byte) (int, error) {
260
+ bc := c.BestConn()
261
+ if bc == nil {
262
+ return 0, errors.New("no best connection")
263
+ }
264
+ return bc.Write(buf)
265
}
266
309
-func (c *MultiConn) GetError() error {
310
- c.RLock()
311
- defer c.RUnlock()
312
- for _, sub := range c.conns {
313
- err := sub.GetError()
314
- if err != nil {
315
- return err
316
- }
267
+// ReadMsg reads data, net.Conn style
268
+func (c *MultiConn) ReadMsg() ([]byte, error) {
269
+ next := <-c.fanIn
270
+ return next, nil
271
+}
272
+
273
+// WriteMsg writes data, net.Conn style
274
+func (c *MultiConn) WriteMsg(buf []byte) error {
275
+ bc := c.BestConn()
276
+ if bc == nil {
277
+ return errors.New("no best connection")
278
+ }
279
+ return bc.WriteMsg(buf)
280
+}
281
+
282
+// ReleaseMsg releases a buffer
283
+func (c *MultiConn) ReleaseMsg(m []byte) {
284
+ for _, c := range c.getConns() {
285
+ c.ReleaseMsg(m)
286
}
318
- return nil
287
}
net/conn/multiconn_test.go
+26
-18
@@ -178,7 +178,7 @@ func TestMulticonnSend(t *testing.T) {
178
179
for _, m := range msgs.msgs {
180
log.Info("send: %s", m.payload)
181
- c.Out() <- []byte(m.payload)
181
+ c.WriteMsg([]byte(m.payload))
182
msgs.Sent(t, m.payload)
183
<-time.After(time.Microsecond * 10)
184
}
@@ -189,16 +189,20 @@ func TestMulticonnSend(t *testing.T) {
189
190
for {
191
select {
192
- case payload := <-c.In():
193
- msgs.Received(t, string(payload))
194
- log.Info("recv: %s", payload)
195
- if msgs.recv == len(msgs.msgs) {
196
- return
197
- }
198
-
192
+ default:
193
case <-ctx.Done():
194
return
195
+ }
196
+
197
+ payload, err := c.ReadMsg()
198
+ if err != nil {
199
+ panic(err)
200
+ }
201
202
+ msgs.Received(t, string(payload))
203
+ log.Info("recv: %s", payload)
204
+ if msgs.recv == len(msgs.msgs) {
205
+ return
206
}
207
}
208
@@ -252,11 +256,11 @@ func TestMulticonnSendUnderlying(t *testing.T) {
256
log.Info("send: %s", m.payload)
257
switch i % 3 {
258
case 0:
255
- conns[0].Out() <- []byte(m.payload)
259
+ conns[0].WriteMsg([]byte(m.payload))
260
case 1:
257
- conns[1].Out() <- []byte(m.payload)
261
+ conns[1].WriteMsg([]byte(m.payload))
262
case 2:
259
- c.Out() <- []byte(m.payload)
263
+ c.WriteMsg([]byte(m.payload))
264
}
265
msgs.Sent(t, m.payload)
266
<-time.After(time.Microsecond * 10)
@@ -269,16 +273,20 @@ func TestMulticonnSendUnderlying(t *testing.T) {
273
274
for {
275
select {
272
- case payload := <-c.In():
273
- msgs.Received(t, string(payload))
274
- log.Info("recv: %s", payload)
275
- if msgs.recv == len(msgs.msgs) {
276
- return
277
- }
278
-
276
+ default:
277
case <-ctx.Done():
278
return
279
+ }
280
+
281
+ payload, err := c.ReadMsg()
282
+ if err != nil {
283
+ panic(err)
284
+ }
285
286
+ msgs.Received(t, string(payload))
287
+ log.Info("recv: %s", payload)
288
+ if msgs.recv == len(msgs.msgs) {
289
+ return
290
}
291
}
292
net/conn/secure_conn.go
+42
-74
@@ -1,15 +1,13 @@
1
package conn
2
3
import (
4
- "errors"
5
-
4
context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
5
+ msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
6
ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
7
9
- spipe "github.com/jbenet/go-ipfs/crypto/spipe"
8
+ secio "github.com/jbenet/go-ipfs/crypto/secio"
9
peer "github.com/jbenet/go-ipfs/peer"
10
ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
12
- "github.com/jbenet/go-ipfs/util/pipes"
11
)
12
13
// secureConn wraps another Conn object with an encrypted channel.
@@ -18,8 +16,11 @@ type secureConn struct {
16
// the wrapped conn
17
insecure Conn
18
21
- // secure pipe, wrapping insecure
22
- secure *spipe.SecurePipe
19
+ // secure io (wrapping insecure)
20
+ secure msgio.ReadWriteCloser
21
+
22
+ // secure Session
23
+ session secio.Session
24
25
ctxc.ContextCloser
26
}
@@ -27,74 +28,30 @@ type secureConn struct {
28
// newConn constructs a new connection
29
func newSecureConn(ctx context.Context, insecure Conn, peers peer.Peerstore) (Conn, error) {
30
31
+ // NewSession performs the secure handshake, which takes multiple RTT
32
+ sessgen := secio.SessionGenerator{Local: insecure.LocalPeer(), Peerstore: peers}
33
+ session, err := sessgen.NewSession(ctx, insecure)
34
+ if err != nil {
35
+ return nil, err
36
+ }
37
+
38
conn := &secureConn{
39
insecure: insecure,
40
+ session: session,
41
+ secure: session.ReadWriter(),
42
}
43
conn.ContextCloser = ctxc.NewContextCloser(ctx, conn.close)
34
-
35
- log.Debugf("newSecureConn: %v to %v", insecure.LocalPeer(), insecure.RemotePeer())
36
- // perform secure handshake before returning this connection.
37
- if err := conn.secureHandshake(peers); err != nil {
38
- conn.Close()
39
- return nil, err
40
- }
41
- log.Debugf("newSecureConn: %v to %v handshake success!", insecure.LocalPeer(), insecure.RemotePeer())
42
-
44
+ log.Debugf("newSecureConn: %v to %v handshake success!", conn.LocalPeer(), conn.RemotePeer())
45
return conn, nil
46
}
47
46
-// secureHandshake performs the spipe secure handshake.
47
-func (c *secureConn) secureHandshake(peers peer.Peerstore) error {
48
- if c.secure != nil {
49
- return errors.New("Conn is already secured or being secured.")
50
- }
51
-
52
- // ok to panic here if this type assertion fails. Interface hack.
53
- // when we support wrapping other Conns, we'll need to change
54
- // spipe to do something else.
55
- insecureSC := c.insecure.(*singleConn)
56
-
57
- // setup a Duplex pipe for spipe
58
- insecureD := pipes.Duplex{
59
- In: insecureSC.msgio.incoming.MsgChan,
60
- Out: insecureSC.msgio.outgoing.MsgChan,
61
- }
62
-
63
- // spipe performs the secure handshake, which takes multiple RTT
64
- sp, err := spipe.NewSecurePipe(c.Context(), 10, c.LocalPeer(), peers, insecureD)
65
- if err != nil {
66
- return err
67
- }
68
-
69
- // assign it into the conn object
70
- c.secure = sp
71
-
72
- // if we do not know RemotePeer, get it from secure chan (who identifies it)
73
- if insecureSC.remote == nil {
74
- insecureSC.remote = c.secure.RemotePeer()
75
-
76
- } else if insecureSC.remote != c.secure.RemotePeer() {
77
- // this panic is here because this would be an insidious programmer error
78
- // that we need to ensure we catch.
79
- // update: this actually might happen under normal operation-- should
80
- // perhaps return an error. TBD.
81
-
82
- log.Errorf("secureConn peer mismatch. %v != %v", insecureSC.remote, c.secure.RemotePeer())
83
- log.Errorf("insecureSC.remote: %s %#v", insecureSC.remote, insecureSC.remote)
84
- log.Errorf("c.secure.LocalPeer: %s %#v", c.secure.RemotePeer(), c.secure.RemotePeer())
85
- panic("secureConn peer mismatch. consructed incorrectly?")
86
- }
87
-
88
- return nil
89
-}
90
-
48
// close is called by ContextCloser
49
func (c *secureConn) close() error {
93
- err := c.insecure.Close()
94
- if c.secure != nil { // may never have gotten here.
95
- err = c.secure.Close()
50
+ if err := c.secure.Close(); err != nil {
51
+ c.insecure.Close()
52
+ return err
53
}
97
- return err
54
+ return c.insecure.Close()
55
}
56
57
// ID is an identifier unique to this connection.
@@ -118,24 +75,35 @@ func (c *secureConn) RemoteMultiaddr() ma.Multiaddr {
75
76
// LocalPeer is the Peer on this side
77
func (c *secureConn) LocalPeer() peer.Peer {
121
- return c.insecure.LocalPeer()
78
+ return c.session.LocalPeer()
79
}
80
81
// RemotePeer is the Peer on the remote side
82
func (c *secureConn) RemotePeer() peer.Peer {
126
- return c.insecure.RemotePeer()
83
+ return c.session.RemotePeer()
84
+}
85
+
86
+// Read reads data, net.Conn style
87
+func (c *secureConn) Read(buf []byte) (int, error) {
88
+ return c.secure.Read(buf)
89
+}
90
+
91
+// Write writes data, net.Conn style
92
+func (c *secureConn) Write(buf []byte) (int, error) {
93
+ return c.secure.Write(buf)
94
}
95
129
-// In returns a readable message channel
130
-func (c *secureConn) In() <-chan []byte {
131
- return c.secure.In
96
+// ReadMsg reads data, net.Conn style
97
+func (c *secureConn) ReadMsg() ([]byte, error) {
98
+ return c.secure.ReadMsg()
99
}
100
134
-// Out returns a writable message channel
135
-func (c *secureConn) Out() chan<- []byte {
136
- return c.secure.Out
101
+// WriteMsg writes data, net.Conn style
102
+func (c *secureConn) WriteMsg(buf []byte) error {
103
+ return c.secure.WriteMsg(buf)
104
}
105
139
-func (c *secureConn) GetError() error {
140
- return c.insecure.GetError()
106
+// ReleaseMsg releases a buffer
107
+func (c *secureConn) ReleaseMsg(m []byte) {
108
+ c.secure.ReleaseMsg(m)
109
}
net/conn/secure_conn_test.go
+13
-5
@@ -105,6 +105,8 @@ func TestSecureCancel(t *testing.T) {
105
}
106
107
func TestSecureCloseLeak(t *testing.T) {
108
+ // t.Skip("Skipping in favor of another test")
109
+
110
if testing.Short() {
111
t.SkipNow()
112
}
@@ -125,15 +127,21 @@ func TestSecureCloseLeak(t *testing.T) {
127
128
for i := 0; i < num; i++ {
129
b1 := []byte("beep")
128
- c1.Out() <- b1
129
- b2 := <-c2.In()
130
+ c1.WriteMsg(b1)
131
+ b2, err := c2.ReadMsg()
132
+ if err != nil {
133
+ panic(err)
134
+ }
135
if !bytes.Equal(b1, b2) {
136
panic("bytes not equal")
137
}
138
134
- b2 = []byte("boop")
135
- c2.Out() <- b2
136
- b1 = <-c1.In()
139
+ b2 = []byte("beep")
140
+ c2.WriteMsg(b2)
141
+ b1, err = c1.ReadMsg()
142
+ if err != nil {
143
+ panic(err)
144
+ }
145
if !bytes.Equal(b1, b2) {
146
panic("bytes not equal")
147
}
net/swarm/conn.go
+35
-25
@@ -200,7 +200,10 @@ func (s *Swarm) fanOut() {
200
log.Debugf("%s sent message to %s (%d)", s.local, msg.Peer(), i)
201
log.Event(context.TODO(), "sendMessage", s.local, msg)
202
// queue it in the connection's buffer
203
- c.Out() <- msg.Data()
203
+ if err := c.WriteMsg(msg.Data()); err != nil {
204
+ log.Infof("%s connection failed to write: %s", c, err)
205
+ continue
206
+ }
207
}
208
}
209
}
@@ -219,6 +222,9 @@ func (s *Swarm) fanInSingle(c conn.Conn) {
222
c.Children().Done() // child of Conn as well.
223
}()
224
225
+ // use readChan to be able to listen to Closing events
226
+ rchan := readChan(s.Context(), c)
227
+
228
i := 0
229
for {
230
select {
@@ -228,7 +234,7 @@ func (s *Swarm) fanInSingle(c conn.Conn) {
234
case <-c.Closing(): // Conn closing
235
return
236
231
- case data, ok := <-c.In():
237
+ case data, ok := <-rchan:
238
if !ok {
239
log.Infof("%s in channel closed", c)
240
return // channel closed.
@@ -240,26 +246,30 @@ func (s *Swarm) fanInSingle(c conn.Conn) {
246
}
247
}
248
243
-// Commenting out because it's platform specific
244
-// func setSocketReuse(l manet.Listener) error {
245
-// nl := l.NetListener()
246
-//
247
-// // for now only TCP. TODO change this when more networks.
248
-// file, err := nl.(*net.TCPListener).File()
249
-// if err != nil {
250
-// return err
251
-// }
252
-//
253
-// fd := file.Fd()
254
-// err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
255
-// if err != nil {
256
-// return err
257
-// }
258
-//
259
-// err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1)
260
-// if err != nil {
261
-// return err
262
-// }
263
-//
264
-// return nil
265
-// }
249
+// readChan is a temporary fixture to match the old interface. will be removed soon.
250
+func readChan(ctx context.Context, c conn.Conn) <-chan []byte {
251
+
252
+ ch := make(chan []byte) // no buffer. sync.
253
+
254
+ go func() {
255
+ defer close(ch)
256
+
257
+ for {
258
+ msg, err := c.ReadMsg()
259
+ if err != nil {
260
+ log.Infof("%s connection failed: %s", c, err)
261
+ return
262
+ }
263
+
264
+ select {
265
+ case <-c.Closing():
266
+ return
267
+ case <-ctx.Done():
268
+ return
269
+ case ch <- msg:
270
+ }
271
+ }
272
+ }()
273
+
274
+ return ch
275
+}