@cryptotaxi247 / kubo / commits / ccc17d274

net: cool test for new stream backpressure

Sadly, as cool as this test is, it doesn't work Because spdystream doesnt handle stream open backpressure well. I'll see about rewriting that part when it becomes a problem. More backpressure tests comming.

Juan Batiz-Benet committed Dec 17, 2014 at 17:39 UTC ccc17d27408031fadf53cb5134103c139c28152b
2 files changed +224
net/backpressure/backpressure_test.go new
+208
@@ -0,0 +1,208 @@
1 +package backpressure_tests
2 +
3 +import (
4 + "testing"
5 + "time"
6 +
7 + inet "github.com/jbenet/go-ipfs/net"
8 + peer "github.com/jbenet/go-ipfs/peer"
9 + eventlog "github.com/jbenet/go-ipfs/util/eventlog"
10 + testutil "github.com/jbenet/go-ipfs/util/testutil"
11 +
12 + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
13 +)
14 +
15 +var log = eventlog.Logger("backpressure")
16 +
17 +func GenNetwork(ctx context.Context) (inet.Network, error) {
18 + p, err := testutil.PeerWithKeysAndAddress(testutil.RandLocalTCPAddress())
19 + if err != nil {
20 + return nil, err
21 + }
22 +
23 + listen := p.Addresses()
24 + ps := peer.NewPeerstore()
25 + return inet.NewNetwork(ctx, listen, p, ps)
26 +}
27 +
28 +// TestBackpressureStreamHandler tests whether mux handler
29 +// ratelimiting works. Meaning, since the handler is sequential
30 +// it should block senders.
31 +//
32 +// Important note: spdystream (which peerstream uses) has a set
33 +// of n workers (n=spdsystream.FRAME_WORKERS) which handle new
34 +// frames, including those starting new streams. So all of them
35 +// can be in the handler at one time. Also, the sending side
36 +// does not rate limit unless we call stream.Wait()
37 +//
38 +//
39 +// Note: right now, this happens muxer-wide. the muxer should
40 +// learn to flow control, so handlers cant block each other.
41 +func TestBackpressureStreamHandler(t *testing.T) {
42 + t.Skip(`Sadly, as cool as this test is, it doesn't work
43 +Because spdystream doesnt handle stream open backpressure
44 +well IMO. I'll see about rewriting that part when it becomes
45 +a problem.
46 +`)
47 +
48 + // a number of concurrent request handlers
49 + limit := 10
50 +
51 + // our way to signal that we're done with 1 request
52 + requestHandled := make(chan struct{})
53 +
54 + // handler rate limiting
55 + receiverRatelimit := make(chan struct{}, limit)
56 + for i := 0; i < limit; i++ {
57 + receiverRatelimit <- struct{}{}
58 + }
59 +
60 + // sender counter of successfully opened streams
61 + senderOpened := make(chan struct{}, limit*100)
62 +
63 + // sender signals it's done (errored out)
64 + senderDone := make(chan struct{})
65 +
66 + // the receiver handles requests with some rate limiting
67 + receiver := func(s inet.Stream) {
68 + log.Debug("receiver received a stream")
69 +
70 + <-receiverRatelimit // acquire
71 + go func() {
72 + // our request handler. can do stuff here. we
73 + // simulate something taking time by waiting
74 + // on requestHandled
75 + log.Error("request worker handling...")
76 + <-requestHandled
77 + log.Error("request worker done!")
78 + receiverRatelimit <- struct{}{} // release
79 + }()
80 + }
81 +
82 + // the sender opens streams as fast as possible
83 + sender := func(net inet.Network, remote peer.Peer) {
84 + var s inet.Stream
85 + var err error
86 + defer func() {
87 + t.Error(err)
88 + log.Debug("sender error. exiting.")
89 + senderDone <- struct{}{}
90 + }()
91 +
92 + for {
93 + s, err = net.NewStream(inet.ProtocolTesting, remote)
94 + if err != nil {
95 + return
96 + }
97 +
98 + _ = s
99 + // if err = s.SwarmStream().Stream().Wait(); err != nil {
100 + // return
101 + // }
102 +
103 + // "count" another successfully opened stream
104 + // (large buffer so shouldn't block in normal operation)
105 + log.Debug("sender opened another stream!")
106 + senderOpened <- struct{}{}
107 + }
108 + }
109 +
110 + // count our senderOpened events
111 + countStreamsOpenedBySender := func(min int) int {
112 + opened := 0
113 + for opened < min {
114 + log.Debugf("countStreamsOpenedBySender got %d (min %d)", opened, min)
115 + select {
116 + case <-senderOpened:
117 + opened++
118 + case <-time.After(10 * time.Millisecond):
119 + }
120 + }
121 + return opened
122 + }
123 +
124 + // count our received events
125 + // waitForNReceivedStreams := func(n int) {
126 + // for n > 0 {
127 + // log.Debugf("waiting for %d received streams...", n)
128 + // select {
129 + // case <-receiverRatelimit:
130 + // n--
131 + // }
132 + // }
133 + // }
134 +
135 + testStreamsOpened := func(expected int) {
136 + log.Debugf("testing rate limited to %d streams", expected)
137 + if n := countStreamsOpenedBySender(expected); n != expected {
138 + t.Fatalf("rate limiting did not work :( -- %d != %d", expected, n)
139 + }
140 + }
141 +
142 + // ok that's enough setup. let's do it!
143 +
144 + ctx := context.Background()
145 + n1, err := GenNetwork(ctx)
146 + if err != nil {
147 + t.Fatal(err)
148 + }
149 + n2, err := GenNetwork(ctx)
150 + if err != nil {
151 + t.Fatal(err)
152 + }
153 +
154 + // setup receiver handler
155 + n1.SetHandler(inet.ProtocolTesting, receiver)
156 +
157 + log.Debugf("dialing %s", n2.ListenAddresses())
158 + if err := n1.DialPeer(ctx, n2.LocalPeer()); err != nil {
159 + t.Fatalf("Failed to dial:", err)
160 + }
161 +
162 + // launch sender!
163 + go sender(n2, n1.LocalPeer())
164 +
165 + // ok, what do we expect to happen? the receiver should
166 + // receive 10 requests and stop receiving, blocking the sender.
167 + // we can test this by counting 10x senderOpened requests
168 +
169 + <-senderOpened // wait for the sender to successfully open some.
170 + testStreamsOpened(limit - 1)
171 +
172 + // let's "handle" 3 requests.
173 + <-requestHandled
174 + <-requestHandled
175 + <-requestHandled
176 + // the sender should've now been able to open exactly 3 more.
177 +
178 + testStreamsOpened(3)
179 +
180 + // shouldn't have opened anything more
181 + testStreamsOpened(0)
182 +
183 + // let's "handle" 100 requests in batches of 5
184 + for i := 0; i < 20; i++ {
185 + <-requestHandled
186 + <-requestHandled
187 + <-requestHandled
188 + <-requestHandled
189 + <-requestHandled
190 + testStreamsOpened(5)
191 + }
192 +
193 + // success!
194 +
195 + // now for the sugar on top: let's tear down the receiver. it should
196 + // exit the sender.
197 + n1.Close()
198 +
199 + // shouldn't have opened anything more
200 + testStreamsOpened(0)
201 +
202 + select {
203 + case <-time.After(100 * time.Millisecond):
204 + t.Error("receiver shutdown failed to exit sender")
205 + case <-senderDone:
206 + log.Info("handler backpressure works!")
207 + }
208 +}
net/mux.go
+16
@@ -40,11 +40,13 @@ type Mux struct {
40 // ReadProtocolHeader reads the stream and returns the next Handler function
41 // according to the muxer encoding.
42 func (m *Mux) ReadProtocolHeader(s io.Reader) (string, StreamHandler, error) {
43 + // log.Error("ReadProtocolHeader")
44 name, err := ReadLengthPrefix(s)
45 if err != nil {
46 return "", nil, err
47 }
48
49 + // log.Debug("ReadProtocolHeader got:", name)
50 m.RLock()
51 h, found := m.Handlers[ProtocolID(name)]
52 m.RUnlock()
@@ -69,6 +71,19 @@ func (m *Mux) SetHandler(p ProtocolID, h StreamHandler) {
71
72 // Handle reads the next name off the Stream, and calls a function
73 func (m *Mux) Handle(s Stream) {
74 +
75 + // Flow control and backpressure of Opening streams is broken.
76 + // I believe that spdystream has one set of workers that both send
77 + // data AND accept new streams (as it's just more data). there
78 + // is a problem where if the new stream handlers want to throttle,
79 + // they also eliminate the ability to read/write data, which makes
80 + // forward-progress impossible. Thus, throttling this function is
81 + // -- at this moment -- not the solution. Either spdystream must
82 + // change, or we must throttle another way.
83 + //
84 + // In light of this, we use a goroutine for now (otherwise the
85 + // spdy worker totally blocks, and we can't even read the protocol
86 + // header). The better route in the future is to use a worker pool.
87 go func() {
88 ctx := context.Background()
89
@@ -107,6 +122,7 @@ func ReadLengthPrefix(r io.Reader) (string, error) {
122
123 // WriteLengthPrefix writes the name into Writer with a length-byte-prefix.
124 func WriteLengthPrefix(w io.Writer, name string) error {
125 + // log.Error("WriteLengthPrefix", name)
126 s := make([]byte, len(name)+1)
127 s[0] = byte(len(name))
128 copy(s[1:], []byte(name))