@cryptotaxi247 / kubo / commits / 68c0f80ec

go-peerstream update

License: MIT Signed-off-by: Juan Batiz-Benet <juan@benet.ai>

Juan Batiz-Benet committed Jul 9, 2015 at 10:25 UTC 68c0f80ec2ff22ebf9495914bff3ec9460c9c458
14 files changed +259 -49
Godeps/Godeps.json
+1 -1
@@ -177,7 +177,7 @@
177 },
178 {
179 "ImportPath": "github.com/jbenet/go-peerstream",
180 - "Rev": "8d52ed2801410a2af995b4e87660272d11c8a9a4"
180 + "Rev": "62fe5ede12f9d9cd9406750160122525b3d6b694"
181 },
182 {
183 "ImportPath": "github.com/jbenet/go-random",
Godeps/_workspace/src/github.com/jbenet/go-peerstream/.travis.yml
+2 -1
@@ -7,4 +7,5 @@ go:
7 - tip
8
9 script:
10 - - go test -race -cpu=5 ./...
10 + - go test ./...
11 + # - go test -race -cpu=5 ./...
Godeps/_workspace/src/github.com/jbenet/go-peerstream/Godeps/Godeps.json new
+33
@@ -0,0 +1,33 @@
1 +{
2 + "ImportPath": "github.com/jbenet/go-peerstream",
3 + "GoVersion": "go1.4.2",
4 + "Packages": [
5 + "./..."
6 + ],
7 + "Deps": [
8 + {
9 + "ImportPath": "github.com/docker/spdystream",
10 + "Rev": "b2c3287865f3ad6aa22821ddb7b4692b896ac207"
11 + },
12 + {
13 + "ImportPath": "github.com/hashicorp/yamux",
14 + "Rev": "b2e55852ddaf823a85c67f798080eb7d08acd71d"
15 + },
16 + {
17 + "ImportPath": "github.com/inconshreveable/muxado",
18 + "Rev": "f693c7e88ba316d1a0ae3e205e22a01aa3ec2848"
19 + },
20 + {
21 + "ImportPath": "github.com/jbenet/go-temp-err-catcher",
22 + "Rev": "aac704a3f4f27190b4ccc05f303a4931fd1241ff"
23 + },
24 + {
25 + "ImportPath": "github.com/whyrusleeping/go-multiplex",
26 + "Rev": "ce5baa716247510379cb7640a14da857afd3b622"
27 + },
28 + {
29 + "ImportPath": "github.com/whyrusleeping/go-multistream",
30 + "Rev": "08e8f9c9f5665ed0c63ffde4fa5ef1d5fb3d516d"
31 + }
32 + ]
33 +}
Godeps/_workspace/src/github.com/jbenet/go-peerstream/Godeps/Readme new
+5
@@ -0,0 +1,5 @@
1 +This directory tree is generated automatically by godep.
2 +
3 +Please do not edit.
4 +
5 +See https://github.com/tools/godep for more information.
Godeps/_workspace/src/github.com/jbenet/go-peerstream/Makefile new
+15
@@ -0,0 +1,15 @@
1 +
2 +godep:
3 + go get github.com/tools/godep
4 +
5 +vendor: godep
6 + godep save -r ./...
7 +
8 +build:
9 + go build ./...
10 +
11 +test:
12 + go test ./...
13 +
14 +test_race:
15 + go test -race -cpu 5 ./...
Godeps/_workspace/src/github.com/jbenet/go-peerstream/conn.go
+44 -31
@@ -158,37 +158,7 @@ func ConnInConns(c1 *Conn, conns []*Conn) bool {
158 // addConn is the internal version of AddConn. we need the server bool
159 // as spdystream requires it.
160 func (s *Swarm) addConn(netConn net.Conn, isServer bool) (*Conn, error) {
161 - if netConn == nil {
162 - return nil, errors.New("nil conn")
163 - }
164 -
165 - // this function is so we can defer our lock, which needs to be
166 - // unlocked **before** the Handler is called (which needs to be
167 - // sequential). This was the simplest thing :)
168 - setupConn := func() (*Conn, error) {
169 - s.connLock.Lock()
170 - defer s.connLock.Unlock()
171 -
172 - // first, check if we already have it...
173 - for c := range s.conns {
174 - if c.netConn == netConn {
175 - return c, nil
176 - }
177 - }
178 -
179 - // create a new spdystream connection
180 - ssConn, err := s.transport.NewConn(netConn, isServer)
181 - if err != nil {
182 - return nil, err
183 - }
184 -
185 - // add the connection
186 - c := newConn(netConn, ssConn, s)
187 - s.conns[c] = struct{}{}
188 - return c, nil
189 - }
190 -
191 - c, err := setupConn()
161 + c, err := s.setupConn(netConn, isServer)
162 if err != nil {
163 return nil, err
164 }
@@ -208,6 +178,49 @@ func (s *Swarm) addConn(netConn net.Conn, isServer bool) (*Conn, error) {
178 return c, nil
179 }
180
181 +// setupConn adds the relevant connection to the map, first checking if it
182 +// was already there.
183 +func (s *Swarm) setupConn(netConn net.Conn, isServer bool) (*Conn, error) {
184 + if netConn == nil {
185 + return nil, errors.New("nil conn")
186 + }
187 +
188 + // first, check if we already have it, to avoid constructing it
189 + // if it is already there
190 + s.connLock.Lock()
191 + for c := range s.conns {
192 + if c.netConn == netConn {
193 + s.connLock.Unlock()
194 + return c, nil
195 + }
196 + }
197 + s.connLock.Unlock()
198 + // construct the connection without hanging onto the lock
199 + // (as there could be deadlock if so.)
200 +
201 + // create a new spdystream connection
202 + ssConn, err := s.transport.NewConn(netConn, isServer)
203 + if err != nil {
204 + return nil, err
205 + }
206 +
207 + // take the lock to add it to the map.
208 + s.connLock.Lock()
209 + defer s.connLock.Unlock()
210 +
211 + // check for it again as it may have been added already. (TOCTTOU)
212 + for c := range s.conns {
213 + if c.netConn == netConn {
214 + return c, nil
215 + }
216 + }
217 +
218 + // add the connection
219 + c := newConn(netConn, ssConn, s)
220 + s.conns[c] = struct{}{}
221 + return c, nil
222 +}
223 +
224 // createStream is the internal function that creates a new stream. assumes
225 // all validation has happened.
226 func (s *Swarm) createStream(c *Conn) (*Stream, error) {
Godeps/_workspace/src/github.com/jbenet/go-peerstream/listener.go
+10 -2
@@ -4,12 +4,13 @@ import (
4 "errors"
5 "fmt"
6 "net"
7 + "sync"
8
9 tec "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-temp-err-catcher"
10 )
11
12 // AcceptConcurrency is how many connections can simultaneously be
12 -// in process of being accepted. Handshakes can sometimes occurr as
13 +// in process of being accepted. Handshakes can sometimes occur as
14 // part of this process, so it may take some time. It is imporant to
15 // rate limit lest a malicious influx of connections would cause our
16 // node to consume all its resources accepting new connections.
@@ -73,7 +74,11 @@ func ListenersWithGroup(g Group, ls []*Listener) []*Listener {
74 // run in a goroutine.
75 // TODO: add rate limiting
76 func (l *Listener) accept() {
76 - defer l.teardown()
77 + var wg sync.WaitGroup
78 + defer func() {
79 + wg.Wait() // must happen before teardown
80 + l.teardown()
81 + }()
82
83 // catching the error here is odd. doing what net/http does:
84 // http://golang.org/src/net/http/server.go?s=51504:51550#L1728
@@ -98,12 +103,15 @@ func (l *Listener) accept() {
103 // do this in a goroutine to avoid blocking the Accept loop.
104 // note that this does not rate limit accepts.
105 limit <- struct{}{} // sema down
106 + wg.Add(1)
107 go func(conn net.Conn) {
108 defer func() { <-limit }() // sema up
109 + defer wg.Done()
110
111 conn2, err := l.swarm.addConn(conn, true)
112 if err != nil {
113 l.acceptErr <- err
114 + return
115 }
116 conn2.groups.AddSet(&l.groups) // add out groups
117 }(conn)
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/multiplex/multiplex.go new
+45
@@ -0,0 +1,45 @@
1 +package peerstream_multiplex
2 +
3 +import (
4 + "net"
5 +
6 + pst "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport"
7 + mp "github.com/whyrusleeping/go-multiplex"
8 +)
9 +
10 +type conn struct {
11 + *mp.Multiplex
12 +}
13 +
14 +func ( // Conn is a connection to a remote peer.
15 +c *conn) Close() error {
16 + return c.Multiplex.Close()
17 +}
18 +
19 +func (c *conn) IsClosed() bool {
20 + return c.Multiplex.IsClosed()
21 +}
22 +
23 +// OpenStream creates a new stream.
24 +func (c *conn) OpenStream() (pst.Stream, error) {
25 + return c.Multiplex.NewStream(), nil
26 +}
27 +
28 +// Serve starts listening for incoming requests and handles them
29 +// using given StreamHandler
30 +func (c *conn) Serve(handler pst.StreamHandler) {
31 + c.Multiplex.Serve(func(s *mp.Stream) {
32 + handler(s)
33 + })
34 +}
35 +
36 +// Transport is a go-peerstream transport that constructs
37 +// multiplex-backed connections.
38 +type Transport struct{}
39 +
40 +// DefaultTransport has default settings for multiplex
41 +var DefaultTransport = &Transport{}
42 +
43 +func (t *Transport) NewConn(nc net.Conn, isServer bool) (pst.Conn, error) {
44 + return &conn{mp.NewMultiplex(nc, isServer)}, nil
45 +}
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/multiplex/multiplex_test.go new
+11
@@ -0,0 +1,11 @@
1 +package peerstream_multiplex
2 +
3 +import (
4 + "testing"
5 +
6 + psttest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/test"
7 +)
8 +
9 +func TestMultiplexTransport(t *testing.T) {
10 + psttest.SubtestAll(t, DefaultTransport)
11 +}
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/multistream/multistream.go new
+59
@@ -0,0 +1,59 @@
1 +// package multistream implements a peerstream transport using
2 +// go-multistream to select the underlying stream muxer
3 +package multistream
4 +
5 +import (
6 + "net"
7 +
8 + pst "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport"
9 + multiplex "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/multiplex"
10 + spdy "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/spdystream"
11 + yamux "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/yamux"
12 + mss "github.com/whyrusleeping/go-multistream"
13 +)
14 +
15 +type transport struct {
16 + mux *mss.MultistreamMuxer
17 +
18 + tpts map[string]pst.Transport
19 +}
20 +
21 +func NewTransport() pst.Transport {
22 + mux := mss.NewMultistreamMuxer()
23 + mux.AddHandler("/multiplex", nil)
24 + mux.AddHandler("/spdystream", nil)
25 + mux.AddHandler("/yamux", nil)
26 +
27 + tpts := map[string]pst.Transport{
28 + "/multiplex": multiplex.DefaultTransport,
29 + "/spdystream": spdy.Transport,
30 + "/yamux": yamux.DefaultTransport,
31 + }
32 +
33 + return &transport{
34 + mux: mux,
35 + tpts: tpts,
36 + }
37 +}
38 +
39 +func (t *transport) NewConn(nc net.Conn, isServer bool) (pst.Conn, error) {
40 + var proto string
41 + if isServer {
42 + selected, _, err := t.mux.Negotiate(nc)
43 + if err != nil {
44 + return nil, err
45 + }
46 + proto = selected
47 + } else {
48 + // prefer yamux
49 + selected, err := mss.SelectOneOf([]string{"/yamux", "/spdystream", "/multiplex"}, nc)
50 + if err != nil {
51 + return nil, err
52 + }
53 + proto = selected
54 + }
55 +
56 + tpt := t.tpts[proto]
57 +
58 + return tpt.NewConn(nc, isServer)
59 +}
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/multistream/multistream_test.go new
+11
@@ -0,0 +1,11 @@
1 +package multistream
2 +
3 +import (
4 + "testing"
5 +
6 + psttest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/test"
7 +)
8 +
9 +func TestMultiStreamTransport(t *testing.T) {
10 + psttest.SubtestAll(t, NewTransport())
11 +}
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/spdystream/spdystream.go
+15 -6
@@ -4,8 +4,8 @@ import (
4 "net"
5 "net/http"
6
7 + ss "github.com/docker/spdystream"
8 pst "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport"
8 - ss "github.com/jbenet/spdystream"
9 )
10
11 // stream implements pst.Stream using a ss.Stream
@@ -55,6 +55,8 @@ func (c *conn) IsClosed() bool {
55 select {
56 case <-c.closed:
57 return true
58 + case <-c.sc.CloseChan():
59 + return true
60 default:
61 return false
62 }
@@ -62,7 +64,10 @@ func (c *conn) IsClosed() bool {
64
65 // OpenStream creates a new stream.
66 func (c *conn) OpenStream() (pst.Stream, error) {
65 - s, err := c.spdyConn().CreateStream(http.Header{}, nil, false)
67 + s, err := c.spdyConn().CreateStream(http.Header{
68 + ":method": []string{"GET"}, // this is here for HTTP/SPDY interop
69 + ":path": []string{"/"}, // this is here for HTTP/SPDY interop
70 + }, nil, false)
71 if err != nil {
72 return nil, err
73 }
@@ -87,10 +92,14 @@ func (c *conn) Serve(handler pst.StreamHandler) {
92 // -- at this moment -- not the solution. Either spdystream must
93 // change, or we must throttle another way. go-peerstream handles
94 // every new stream in its own goroutine.
90 - go func() {
91 - s.SendReply(http.Header{}, false)
92 - handler((*stream)(s))
93 - }()
95 + err := s.SendReply(http.Header{}, false)
96 + if err != nil {
97 + // this _could_ error out. not sure how to handle this failure.
98 + // don't return, and let the caller handle a broken stream.
99 + // better than _hiding_ an error.
100 + // return
101 + }
102 + go handler((*stream)(s))
103 })
104 }
105
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/spdystream/spdystream_test.go
-1
@@ -7,6 +7,5 @@ import (
7 )
8
9 func TestSpdyStreamTransport(t *testing.T) {
10 - t.Skip("spdystream is known to be broken")
10 psttest.SubtestAll(t, Transport)
11 }
Godeps/_workspace/src/github.com/jbenet/go-peerstream/transport/test/ttest.go
+8 -7
@@ -194,7 +194,7 @@ func SubtestSimpleWrite100msgs(t *testing.T, tr pst.Transport) {
194 bufs <- buf
195 log("writing %d bytes (message %d/%d #%x)", len(buf), i, msgs, buf[:3])
196 if _, err := stream.Write(buf); err != nil {
197 - errs <- err
197 + errs <- fmt.Errorf("stream.Write(buf): %s", err)
198 continue
199 }
200 }
@@ -212,7 +212,7 @@ func SubtestSimpleWrite100msgs(t *testing.T, tr pst.Transport) {
212 i++
213
214 if _, err := io.ReadFull(stream, buf2); err != nil {
215 - errs <- err
215 + errs <- fmt.Errorf("readFull(stream, buf2): %s", err)
216 continue
217 }
218 if !bytes.Equal(buf1, buf2) {
@@ -253,7 +253,7 @@ func SubtestStressNSwarmNConnNStreamNMsg(t *testing.T, tr pst.Transport, nSwarm,
253 bufs <- buf
254 log("%p writing %d bytes (message %d/%d #%x)", s, len(buf), i, nMsg, buf[:3])
255 if _, err := s.Write(buf); err != nil {
256 - errs <- err
256 + errs <- fmt.Errorf("s.Write(buf): %s", err)
257 continue
258 }
259 }
@@ -265,11 +265,12 @@ func SubtestStressNSwarmNConnNStreamNMsg(t *testing.T, tr pst.Transport, nSwarm,
265 buf2 := make([]byte, msgsize)
266 i := 0
267 for buf1 := range bufs {
268 - log("%p reading %d bytes (message %d/%d #%x)", s, len(buf1), i, nMsg, buf1[:3])
268 i++
269 + log("%p reading %d bytes (message %d/%d #%x)", s, len(buf1), i-1, nMsg, buf1[:3])
270
271 if _, err := io.ReadFull(s, buf2); err != nil {
272 - errs <- err
272 + errs <- fmt.Errorf("io.ReadFull(s, buf2): %s", err)
273 + log("%p failed to read %d bytes (message %d/%d #%x)", s, len(buf1), i-1, nMsg, buf1[:3])
274 continue
275 }
276 if !bytes.Equal(buf1, buf2) {
@@ -307,13 +308,13 @@ func SubtestStressNSwarmNConnNStreamNMsg(t *testing.T, tr pst.Transport, nSwarm,
308
309 nc, err := net.Dial(nla.Network(), nla.String())
310 if err != nil {
310 - errs <- err
311 + errs <- fmt.Errorf("net.Dial(%s, %s): %s", nla.Network(), nla.String(), err)
312 return
313 }
314
315 c, err := a.AddConn(nc)
316 if err != nil {
316 - errs <- err
317 + errs <- fmt.Errorf("a.AddConn(%s <--> %s): %s", nc.LocalAddr(), nc.RemoteAddr(), err)
318 return
319 }
320