net/p2p + secio: parallelize crypto handshake
We had a very nasty problem: handshakes were serial so incoming dials would wait for each other to finish handshaking. this was particularly problematic when handshakes hung-- nodes would not recover quickly. This led to gateways not bootstrapping peers fast enough. The approach taken here is to do what crypto/tls does: defer the handshake until Read/Write[1]. There are a number of reasons why this is _the right thing to do_: - it delays handshaking until it is known to be necessary (doing io) - it "accepts" before the handshake, getting the handshake out of the critical path entirely. - it defers to the user's parallelization of conn handling. users must implement this in some way already so use that, instead of picking constants surely to be wrong (how many handshakes to run in parallel?) [0] http://golang.org/src/crypto/tls/conn.go#L886
Juan Batiz-Benet committed
May 12, 2015 at 05:48 UTC
b84fa2b493a1dc3e149e409596e034f22358c96a
5 files changed
+157
-55
p2p/crypto/secio/interface.go
+52
-19
@@ -17,28 +17,13 @@ type SessionGenerator struct {
17
PrivateKey ci.PrivKey
18
}
19
20
-// NewSession takes an insecure io.ReadWriter, performs a TLS-like
20
+// NewSession takes an insecure io.ReadWriter, sets up a TLS-like
21
// handshake with the other side, and returns a secure session.
22
+// The handshake isn't run until the connection is read or written to.
23
// See the source for the protocol details and security implementation.
24
// The provided Context is only needed for the duration of this function.
24
-func (sg *SessionGenerator) NewSession(ctx context.Context,
25
- insecure io.ReadWriter) (Session, error) {
26
-
27
- ss, err := newSecureSession(sg.LocalID, sg.PrivateKey)
28
- if err != nil {
29
- return nil, err
30
- }
31
-
32
- if ctx == nil {
33
- ctx = context.Background()
34
- }
35
- ctx, cancel := context.WithCancel(ctx)
36
- if err := ss.handshake(ctx, insecure); err != nil {
37
- cancel()
38
- return nil, err
39
- }
40
-
41
- return ss, nil
25
+func (sg *SessionGenerator) NewSession(ctx context.Context, insecure io.ReadWriteCloser) (Session, error) {
26
+ return newSecureSession(ctx, sg.LocalID, sg.PrivateKey, insecure)
27
}
28
29
type Session interface {
@@ -64,6 +49,9 @@ type Session interface {
49
50
// SecureReadWriter returns the encrypted communication channel
51
func (s *secureSession) ReadWriter() msgio.ReadWriteCloser {
52
+ if err := s.Handshake(); err != nil {
53
+ return &closedRW{err}
54
+ }
55
return s.secure
56
}
57
@@ -79,15 +67,60 @@ func (s *secureSession) LocalPrivateKey() ci.PrivKey {
67
68
// RemotePeer retrieves the remote peer.
69
func (s *secureSession) RemotePeer() peer.ID {
70
+ if err := s.Handshake(); err != nil {
71
+ return ""
72
+ }
73
return s.remotePeer
74
}
75
76
// RemotePeer retrieves the remote peer.
77
func (s *secureSession) RemotePublicKey() ci.PubKey {
78
+ if err := s.Handshake(); err != nil {
79
+ return nil
80
+ }
81
return s.remote.permanentPubKey
82
}
83
84
// Close closes the secure session
85
func (s *secureSession) Close() error {
86
+ s.cancel()
87
+ s.handshakeMu.Lock()
88
+ defer s.handshakeMu.Unlock()
89
+ if s.secure == nil {
90
+ return s.insecure.Close() // hadn't secured yet.
91
+ }
92
return s.secure.Close()
93
}
94
+
95
+// closedRW implements a stub msgio interface that's already
96
+// closed and errored.
97
+type closedRW struct {
98
+ err error
99
+}
100
+
101
+func (c *closedRW) Read(buf []byte) (int, error) {
102
+ return 0, c.err
103
+}
104
+
105
+func (c *closedRW) Write(buf []byte) (int, error) {
106
+ return 0, c.err
107
+}
108
+
109
+func (c *closedRW) NextMsgLen() (int, error) {
110
+ return 0, c.err
111
+}
112
+
113
+func (c *closedRW) ReadMsg() ([]byte, error) {
114
+ return nil, c.err
115
+}
116
+
117
+func (c *closedRW) WriteMsg(buf []byte) error {
118
+ return c.err
119
+}
120
+
121
+func (c *closedRW) Close() error {
122
+ return c.err
123
+}
124
+
125
+func (c *closedRW) ReleaseMsg(m []byte) {
126
+}
p2p/crypto/secio/protocol.go
+42
-8
@@ -6,6 +6,8 @@ import (
6
"errors"
7
"fmt"
8
"io"
9
+ "sync"
10
+ "time"
11
12
msgio "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
13
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
@@ -27,15 +29,23 @@ var ErrClosed = errors.New("connection closed")
29
// ErrEcho is returned when we're attempting to handshake with the same keys and nonces.
30
var ErrEcho = errors.New("same keys and nonces. one side talking to self.")
31
32
+// HandshakeTimeout governs how long the handshake will be allowed to take place for.
33
+// Making this number large means there could be many bogus connections waiting to
34
+// timeout in flight. Typical handshakes take ~3RTTs, so it should be completed within
35
+// seconds across a typical planet in the solar system.
36
+var HandshakeTimeout = time.Second * 30
37
+
38
// nonceSize is the size of our nonces (in bytes)
39
const nonceSize = 16
40
41
// secureSession encapsulates all the parameters needed for encrypting
42
// and decrypting traffic from an insecure channel.
43
type secureSession struct {
36
- secure msgio.ReadWriteCloser
44
+ ctx context.Context
45
+ cancel context.CancelFunc
46
38
- insecure io.ReadWriter
47
+ secure msgio.ReadWriteCloser
48
+ insecure io.ReadWriteCloser
49
insecureM msgio.ReadWriter
50
51
localKey ci.PrivKey
@@ -46,6 +56,10 @@ type secureSession struct {
56
remote encParams
57
58
sharedSecret []byte
59
+
60
+ handshakeMu sync.Mutex // guards handshakeDone + handshakeErr
61
+ handshakeDone bool
62
+ handshakeErr error
63
}
64
65
func (s *secureSession) Loggable() map[string]interface{} {
@@ -56,8 +70,9 @@ func (s *secureSession) Loggable() map[string]interface{} {
70
return m
71
}
72
59
-func newSecureSession(local peer.ID, key ci.PrivKey) (*secureSession, error) {
73
+func newSecureSession(ctx context.Context, local peer.ID, key ci.PrivKey, insecure io.ReadWriteCloser) (*secureSession, error) {
74
s := &secureSession{localPeer: local, localKey: key}
75
+ s.ctx, s.cancel = context.WithCancel(ctx)
76
77
switch {
78
case s.localPeer == "":
@@ -66,18 +81,37 @@ func newSecureSession(local peer.ID, key ci.PrivKey) (*secureSession, error) {
81
return nil, errors.New("no local private key provided")
82
case !s.localPeer.MatchesPrivateKey(s.localKey):
83
return nil, fmt.Errorf("peer.ID does not match PrivateKey")
84
+ case insecure == nil:
85
+ return nil, fmt.Errorf("insecure ReadWriter is nil")
86
}
87
88
+ s.ctx = ctx
89
+ s.insecure = insecure
90
+ s.insecureM = msgio.NewReadWriter(insecure)
91
return s, nil
92
}
93
74
-// handsahke performs initial communication over insecure channel to share
94
+func (s *secureSession) Handshake() error {
95
+ s.handshakeMu.Lock()
96
+ defer s.handshakeMu.Unlock()
97
+
98
+ if s.handshakeErr != nil {
99
+ return s.handshakeErr
100
+ }
101
+
102
+ if !s.handshakeDone {
103
+ s.handshakeErr = s.runHandshake()
104
+ s.handshakeDone = true
105
+ }
106
+ return s.handshakeErr
107
+}
108
+
109
+// runHandshake performs initial communication over insecure channel to share
110
// keys, IDs, and initiate communication, assigning all necessary params.
111
// requires the duplex channel to be a msgio.ReadWriter (for framed messaging)
77
-func (s *secureSession) handshake(ctx context.Context, insecure io.ReadWriter) error {
78
-
79
- s.insecure = insecure
80
- s.insecureM = msgio.NewReadWriter(insecure)
112
+func (s *secureSession) runHandshake() error {
113
+ ctx, cancel := context.WithTimeout(s.ctx, HandshakeTimeout) // remove
114
+ defer cancel()
115
116
// =============================================================================
117
// step 1. Propose -- propose cipher suite + send pubkeys + nonce
p2p/net/conn/dial_test.go
+40
-3
@@ -75,18 +75,35 @@ func setupConn(t *testing.T, ctx context.Context, secure bool) (a, b Conn, p1, p
75
76
done := make(chan error)
77
go func() {
78
+ defer close(done)
79
+
80
var err error
81
c2, err = d2.Dial(ctx, p1.Addr, p1.ID)
82
if err != nil {
83
done <- err
84
+ return
85
+ }
86
+
87
+ // if secure, need to read + write, as that's what triggers the handshake.
88
+ if secure {
89
+ if err := sayHello(c2); err != nil {
90
+ done <- err
91
+ }
92
}
83
- close(done)
93
}()
94
95
c1, err := l1.Accept()
96
if err != nil {
97
t.Fatal("failed to accept", err)
98
}
99
+
100
+ // if secure, need to read + write, as that's what triggers the handshake.
101
+ if secure {
102
+ if err := sayHello(c1); err != nil {
103
+ done <- err
104
+ }
105
+ }
106
+
107
if err := <-done; err != nil {
108
t.Fatal(err)
109
}
@@ -94,6 +111,20 @@ func setupConn(t *testing.T, ctx context.Context, secure bool) (a, b Conn, p1, p
111
return c1.(Conn), c2, p1, p2
112
}
113
114
+func sayHello(c net.Conn) error {
115
+ h := []byte("hello")
116
+ if _, err := c.Write(h); err != nil {
117
+ return err
118
+ }
119
+ if _, err := c.Read(h); err != nil {
120
+ return err
121
+ }
122
+ if string(h) != "hello" {
123
+ return fmt.Errorf("did not get hello")
124
+ }
125
+ return nil
126
+}
127
+
128
func testDialer(t *testing.T, secure bool) {
129
// t.Skip("Skipping in favor of another test")
130
@@ -203,7 +234,7 @@ func testDialerCloseEarly(t *testing.T, secure bool) {
234
go func() {
235
defer func() { done <- struct{}{} }()
236
206
- _, err := l1.Accept()
237
+ c, err := l1.Accept()
238
if err != nil {
239
if strings.Contains(err.Error(), "closed") {
240
gotclosed <- struct{}{}
@@ -211,7 +242,13 @@ func testDialerCloseEarly(t *testing.T, secure bool) {
242
}
243
errs <- err
244
}
214
- errs <- fmt.Errorf("got conn")
245
+
246
+ if _, err := c.Write([]byte("hello")); err != nil {
247
+ gotclosed <- struct{}{}
248
+ return
249
+ }
250
+
251
+ errs <- fmt.Errorf("wrote to conn")
252
}()
253
254
c, err := d2.Dial(ctx, p1.Addr, p1.ID)
p2p/net/conn/secure_conn.go
+14
-25
@@ -5,7 +5,6 @@ import (
5
"net"
6
"time"
7
8
- msgio "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
8
ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
9
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
10
@@ -16,15 +15,8 @@ import (
15
16
// secureConn wraps another Conn object with an encrypted channel.
17
type secureConn struct {
19
-
20
- // the wrapped conn
21
- insecure Conn
22
-
23
- // secure io (wrapping insecure)
24
- secure msgio.ReadWriteCloser
25
-
26
- // secure Session
27
- session secio.Session
18
+ insecure Conn // the wrapped conn
19
+ secure secio.Session // secure Session
20
}
21
22
// newConn constructs a new connection
@@ -37,23 +29,20 @@ func newSecureConn(ctx context.Context, sk ic.PrivKey, insecure Conn) (Conn, err
29
return nil, errors.New("insecure.LocalPeer() is nil")
30
}
31
if sk == nil {
40
- panic("way")
32
return nil, errors.New("private key is nil")
33
}
34
35
// NewSession performs the secure handshake, which takes multiple RTT
36
sessgen := secio.SessionGenerator{LocalID: insecure.LocalPeer(), PrivateKey: sk}
46
- session, err := sessgen.NewSession(ctx, insecure)
37
+ secure, err := sessgen.NewSession(ctx, insecure)
38
if err != nil {
39
return nil, err
40
}
41
42
conn := &secureConn{
43
insecure: insecure,
53
- session: session,
54
- secure: session.ReadWriter(),
44
+ secure: secure,
45
}
56
- log.Debugf("newSecureConn: %v to %v handshake success!", conn.LocalPeer(), conn.RemotePeer())
46
return conn, nil
47
}
48
@@ -102,49 +91,49 @@ func (c *secureConn) RemoteMultiaddr() ma.Multiaddr {
91
92
// LocalPeer is the Peer on this side
93
func (c *secureConn) LocalPeer() peer.ID {
105
- return c.session.LocalPeer()
94
+ return c.secure.LocalPeer()
95
}
96
97
// RemotePeer is the Peer on the remote side
98
func (c *secureConn) RemotePeer() peer.ID {
110
- return c.session.RemotePeer()
99
+ return c.secure.RemotePeer()
100
}
101
102
// LocalPrivateKey is the public key of the peer on this side
103
func (c *secureConn) LocalPrivateKey() ic.PrivKey {
115
- return c.session.LocalPrivateKey()
104
+ return c.secure.LocalPrivateKey()
105
}
106
107
// RemotePubKey is the public key of the peer on the remote side
108
func (c *secureConn) RemotePublicKey() ic.PubKey {
120
- return c.session.RemotePublicKey()
109
+ return c.secure.RemotePublicKey()
110
}
111
112
// Read reads data, net.Conn style
113
func (c *secureConn) Read(buf []byte) (int, error) {
125
- return c.secure.Read(buf)
114
+ return c.secure.ReadWriter().Read(buf)
115
}
116
117
// Write writes data, net.Conn style
118
func (c *secureConn) Write(buf []byte) (int, error) {
130
- return c.secure.Write(buf)
119
+ return c.secure.ReadWriter().Write(buf)
120
}
121
122
func (c *secureConn) NextMsgLen() (int, error) {
134
- return c.secure.NextMsgLen()
123
+ return c.secure.ReadWriter().NextMsgLen()
124
}
125
126
// ReadMsg reads data, net.Conn style
127
func (c *secureConn) ReadMsg() ([]byte, error) {
139
- return c.secure.ReadMsg()
128
+ return c.secure.ReadWriter().ReadMsg()
129
}
130
131
// WriteMsg writes data, net.Conn style
132
func (c *secureConn) WriteMsg(buf []byte) error {
144
- return c.secure.WriteMsg(buf)
133
+ return c.secure.ReadWriter().WriteMsg(buf)
134
}
135
136
// ReleaseMsg releases a buffer
137
func (c *secureConn) ReleaseMsg(m []byte) {
149
- c.secure.ReleaseMsg(m)
138
+ c.secure.ReadWriter().ReleaseMsg(m)
139
}
p2p/net/conn/secure_conn_test.go
+9
@@ -23,6 +23,15 @@ func upgradeToSecureConn(t *testing.T, ctx context.Context, sk ic.PrivKey, c Con
23
if err != nil {
24
return nil, err
25
}
26
+
27
+ // need to read + write, as that's what triggers the handshake.
28
+ h := []byte("hello")
29
+ if _, err := s.Write(h); err != nil {
30
+ return nil, err
31
+ }
32
+ if _, err := s.Read(h); err != nil {
33
+ return nil, err
34
+ }
35
return s, nil
36
}
37