@cryptotaxi247 / kubo / commits / 39c5c47c9

Extract bitswap to go-bitswap

License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>

Jeromy committed Jul 27, 2018 at 14:47 UTC 39c5c47c94b4f1fb51ba4fff129cc79715aa2af9
37 files changed +12 -5865
Rules.mk
-3
@@ -53,9 +53,6 @@ include $(dir)/Rules.mk
53 dir := merkledag/pb
54 include $(dir)/Rules.mk
55
56 -dir := exchange/bitswap/message/pb
57 -include $(dir)/Rules.mk
58 -
56 dir := pin/internal/pb
57 include $(dir)/Rules.mk
58
blockservice/test/mock.go
+2 -2
@@ -2,8 +2,8 @@ package bstest
2
3 import (
4 . "github.com/ipfs/go-ipfs/blockservice"
5 - bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
6 - tn "github.com/ipfs/go-ipfs/exchange/bitswap/testnet"
5 + bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap"
6 + tn "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/testnet"
7
8 delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
9 mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock"
core/commands/bitswap.go
+2 -2
@@ -8,8 +8,8 @@ import (
8 oldcmds "github.com/ipfs/go-ipfs/commands"
9 lgc "github.com/ipfs/go-ipfs/commands/legacy"
10 e "github.com/ipfs/go-ipfs/core/commands/e"
11 - bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
12 - decision "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
11 + bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap"
12 + decision "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/decision"
13
14 cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds"
15 "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
core/core.go
+2 -2
@@ -21,8 +21,6 @@ import (
21 "time"
22
23 bserv "github.com/ipfs/go-ipfs/blockservice"
24 - bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
25 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
24 rp "github.com/ipfs/go-ipfs/exchange/reprovide"
25 filestore "github.com/ipfs/go-ipfs/filestore"
26 mount "github.com/ipfs/go-ipfs/fuse/mount"
@@ -36,6 +34,8 @@ import (
34 repo "github.com/ipfs/go-ipfs/repo"
35 config "github.com/ipfs/go-ipfs/repo/config"
36 ft "github.com/ipfs/go-ipfs/unixfs"
37 + bitswap "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap"
38 + bsnet "gx/ipfs/QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m/go-bitswap/network"
39
40 u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
41 rhelpers "gx/ipfs/QmQpvpeXa8rBfDmt3bdh2ckw2867vsYN1ozf79X7U5rij9/go-libp2p-routing-helpers"
exchange/bitswap/README.md deleted
-37
@@ -1,37 +0,0 @@
1 -# Bitswap
2 -
3 -## Protocol
4 -Bitswap is the data trading module for ipfs, it manages requesting and sending
5 -blocks to and from other peers in the network. Bitswap has two main jobs, the
6 -first is to acquire blocks requested by the client from the network. The second
7 -is to judiciously send blocks in its possession to other peers who want them.
8 -
9 -Bitswap is a message based protocol, as opposed to response-reply. All messages
10 -contain wantlists, or blocks. Upon receiving a wantlist, a node should consider
11 -sending out wanted blocks if they have them. Upon receiving blocks, the node
12 -should send out a notification called a 'Cancel' signifying that they no longer
13 -want the block. At a protocol level, bitswap is very simple.
14 -
15 -## go-ipfs Implementation
16 -Internally, when a message with a wantlist is received, it is sent to the
17 -decision engine to be considered, and blocks that we have that are wanted are
18 -placed into the peer request queue. Any block we possess that is wanted by
19 -another peer has a task in the peer request queue created for it. The peer
20 -request queue is a priority queue that sorts available tasks by some metric,
21 -currently, that metric is very simple and aims to fairly address the tasks
22 -of each other peer. More advanced decision logic will be implemented in the
23 -future. Task workers pull tasks to be done off of the queue, retrieve the block
24 -to be sent, and send it off. The number of task workers is limited by a constant
25 -factor.
26 -
27 -Client requests for new blocks are handled by the want manager, for every new
28 -block (or set of blocks) wanted, the 'WantBlocks' method is invoked. The want
29 -manager then ensures that connected peers are notified of the new block that we
30 -want by sending the new entries to a message queue for each peer. The message
31 -queue will loop while there is work available and do the following: 1) Ensure it
32 -has a connection to its peer, 2) grab the message to be sent, and 3) send it.
33 -If new messages are added while the loop is in steps 1 or 3, the messages are
34 -combined into one to avoid having to keep an actual queue and send multiple
35 -messages. The same process occurs when the client receives a block and sends a
36 -cancel message for it.
37 -
exchange/bitswap/bitswap.go deleted
-454
@@ -1,454 +0,0 @@
1 -// package bitswap implements the IPFS exchange interface with the BitSwap
2 -// bilateral exchange protocol.
3 -package bitswap
4 -
5 -import (
6 - "context"
7 - "errors"
8 - "math"
9 - "sync"
10 - "sync/atomic"
11 - "time"
12 -
13 - decision "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
14 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
15 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
16 - notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
17 -
18 - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
19 - flags "gx/ipfs/QmRMGdC6HKdLsPDABL9aXPDidrpmEHzJqFWSvshkbn9Hj8/go-ipfs-flags"
20 - process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess"
21 - procctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context"
22 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
23 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
24 - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
25 - exchange "gx/ipfs/Qmc2faLf7URkHpsbfYM4EMbr8iSAcGAe8VPgVi64HVnwji/go-ipfs-exchange-interface"
26 - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
27 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
28 - metrics "gx/ipfs/QmekzFM3hPZjTjUFGTABdQkEnQ3PTiMstY198PwSFr5w1Q/go-metrics-interface"
29 -)
30 -
31 -var log = logging.Logger("bitswap")
32 -
33 -const (
34 - // maxProvidersPerRequest specifies the maximum number of providers desired
35 - // from the network. This value is specified because the network streams
36 - // results.
37 - // TODO: if a 'non-nice' strategy is implemented, consider increasing this value
38 - maxProvidersPerRequest = 3
39 - providerRequestTimeout = time.Second * 10
40 - provideTimeout = time.Second * 15
41 - sizeBatchRequestChan = 32
42 - // kMaxPriority is the max priority as defined by the bitswap protocol
43 - kMaxPriority = math.MaxInt32
44 -)
45 -
46 -var (
47 - HasBlockBufferSize = 256
48 - provideKeysBufferSize = 2048
49 - provideWorkerMax = 512
50 -
51 - // the 1<<18+15 is to observe old file chunks that are 1<<18 + 14 in size
52 - metricsBuckets = []float64{1 << 6, 1 << 10, 1 << 14, 1 << 18, 1<<18 + 15, 1 << 22}
53 -)
54 -
55 -func init() {
56 - if flags.LowMemMode {
57 - HasBlockBufferSize = 64
58 - provideKeysBufferSize = 512
59 - provideWorkerMax = 16
60 - }
61 -}
62 -
63 -var rebroadcastDelay = delay.Fixed(time.Minute)
64 -
65 -// New initializes a BitSwap instance that communicates over the provided
66 -// BitSwapNetwork. This function registers the returned instance as the network
67 -// delegate.
68 -// Runs until context is cancelled.
69 -func New(parent context.Context, network bsnet.BitSwapNetwork,
70 - bstore blockstore.Blockstore) exchange.Interface {
71 -
72 - // important to use provided parent context (since it may include important
73 - // loggable data). It's probably not a good idea to allow bitswap to be
74 - // coupled to the concerns of the ipfs daemon in this way.
75 - //
76 - // FIXME(btc) Now that bitswap manages itself using a process, it probably
77 - // shouldn't accept a context anymore. Clients should probably use Close()
78 - // exclusively. We should probably find another way to share logging data
79 - ctx, cancelFunc := context.WithCancel(parent)
80 - ctx = metrics.CtxSubScope(ctx, "bitswap")
81 - dupHist := metrics.NewCtx(ctx, "recv_dup_blocks_bytes", "Summary of duplicate"+
82 - " data blocks recived").Histogram(metricsBuckets)
83 - allHist := metrics.NewCtx(ctx, "recv_all_blocks_bytes", "Summary of all"+
84 - " data blocks recived").Histogram(metricsBuckets)
85 -
86 - notif := notifications.New()
87 - px := process.WithTeardown(func() error {
88 - notif.Shutdown()
89 - return nil
90 - })
91 -
92 - bs := &Bitswap{
93 - blockstore: bstore,
94 - notifications: notif,
95 - engine: decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
96 - network: network,
97 - findKeys: make(chan *blockRequest, sizeBatchRequestChan),
98 - process: px,
99 - newBlocks: make(chan *cid.Cid, HasBlockBufferSize),
100 - provideKeys: make(chan *cid.Cid, provideKeysBufferSize),
101 - wm: NewWantManager(ctx, network),
102 - counters: new(counters),
103 -
104 - dupMetric: dupHist,
105 - allMetric: allHist,
106 - }
107 - go bs.wm.Run()
108 - network.SetDelegate(bs)
109 -
110 - // Start up bitswaps async worker routines
111 - bs.startWorkers(px, ctx)
112 -
113 - // bind the context and process.
114 - // do it over here to avoid closing before all setup is done.
115 - go func() {
116 - <-px.Closing() // process closes first
117 - cancelFunc()
118 - }()
119 - procctx.CloseAfterContext(px, ctx) // parent cancelled first
120 -
121 - return bs
122 -}
123 -
124 -// Bitswap instances implement the bitswap protocol.
125 -type Bitswap struct {
126 - // the peermanager manages sending messages to peers in a way that
127 - // wont block bitswap operation
128 - wm *WantManager
129 -
130 - // the engine is the bit of logic that decides who to send which blocks to
131 - engine *decision.Engine
132 -
133 - // network delivers messages on behalf of the session
134 - network bsnet.BitSwapNetwork
135 -
136 - // blockstore is the local database
137 - // NB: ensure threadsafety
138 - blockstore blockstore.Blockstore
139 -
140 - // notifications engine for receiving new blocks and routing them to the
141 - // appropriate user requests
142 - notifications notifications.PubSub
143 -
144 - // findKeys sends keys to a worker to find and connect to providers for them
145 - findKeys chan *blockRequest
146 - // newBlocks is a channel for newly added blocks to be provided to the
147 - // network. blocks pushed down this channel get buffered and fed to the
148 - // provideKeys channel later on to avoid too much network activity
149 - newBlocks chan *cid.Cid
150 - // provideKeys directly feeds provide workers
151 - provideKeys chan *cid.Cid
152 -
153 - process process.Process
154 -
155 - // Counters for various statistics
156 - counterLk sync.Mutex
157 - counters *counters
158 -
159 - // Metrics interface metrics
160 - dupMetric metrics.Histogram
161 - allMetric metrics.Histogram
162 -
163 - // Sessions
164 - sessions []*Session
165 - sessLk sync.Mutex
166 -
167 - sessID uint64
168 - sessIDLk sync.Mutex
169 -}
170 -
171 -type counters struct {
172 - blocksRecvd uint64
173 - dupBlocksRecvd uint64
174 - dupDataRecvd uint64
175 - blocksSent uint64
176 - dataSent uint64
177 - dataRecvd uint64
178 - messagesRecvd uint64
179 -}
180 -
181 -type blockRequest struct {
182 - Cid *cid.Cid
183 - Ctx context.Context
184 -}
185 -
186 -// GetBlock attempts to retrieve a particular block from peers within the
187 -// deadline enforced by the context.
188 -func (bs *Bitswap) GetBlock(parent context.Context, k *cid.Cid) (blocks.Block, error) {
189 - return getBlock(parent, k, bs.GetBlocks)
190 -}
191 -
192 -func (bs *Bitswap) WantlistForPeer(p peer.ID) []*cid.Cid {
193 - var out []*cid.Cid
194 - for _, e := range bs.engine.WantlistForPeer(p) {
195 - out = append(out, e.Cid)
196 - }
197 - return out
198 -}
199 -
200 -func (bs *Bitswap) LedgerForPeer(p peer.ID) *decision.Receipt {
201 - return bs.engine.LedgerForPeer(p)
202 -}
203 -
204 -// GetBlocks returns a channel where the caller may receive blocks that
205 -// correspond to the provided |keys|. Returns an error if BitSwap is unable to
206 -// begin this request within the deadline enforced by the context.
207 -//
208 -// NB: Your request remains open until the context expires. To conserve
209 -// resources, provide a context with a reasonably short deadline (ie. not one
210 -// that lasts throughout the lifetime of the server)
211 -func (bs *Bitswap) GetBlocks(ctx context.Context, keys []*cid.Cid) (<-chan blocks.Block, error) {
212 - if len(keys) == 0 {
213 - out := make(chan blocks.Block)
214 - close(out)
215 - return out, nil
216 - }
217 -
218 - select {
219 - case <-bs.process.Closing():
220 - return nil, errors.New("bitswap is closed")
221 - default:
222 - }
223 - promise := bs.notifications.Subscribe(ctx, keys...)
224 -
225 - for _, k := range keys {
226 - log.Event(ctx, "Bitswap.GetBlockRequest.Start", k)
227 - }
228 -
229 - mses := bs.getNextSessionID()
230 -
231 - bs.wm.WantBlocks(ctx, keys, nil, mses)
232 -
233 - // NB: Optimization. Assumes that providers of key[0] are likely to
234 - // be able to provide for all keys. This currently holds true in most
235 - // every situation. Later, this assumption may not hold as true.
236 - req := &blockRequest{
237 - Cid: keys[0],
238 - Ctx: ctx,
239 - }
240 -
241 - remaining := cid.NewSet()
242 - for _, k := range keys {
243 - remaining.Add(k)
244 - }
245 -
246 - out := make(chan blocks.Block)
247 - go func() {
248 - ctx, cancel := context.WithCancel(ctx)
249 - defer cancel()
250 - defer close(out)
251 - defer func() {
252 - // can't just defer this call on its own, arguments are resolved *when* the defer is created
253 - bs.CancelWants(remaining.Keys(), mses)
254 - }()
255 - for {
256 - select {
257 - case blk, ok := <-promise:
258 - if !ok {
259 - return
260 - }
261 -
262 - bs.CancelWants([]*cid.Cid{blk.Cid()}, mses)
263 - remaining.Remove(blk.Cid())
264 - select {
265 - case out <- blk:
266 - case <-ctx.Done():
267 - return
268 - }
269 - case <-ctx.Done():
270 - return
271 - }
272 - }
273 - }()
274 -
275 - select {
276 - case bs.findKeys <- req:
277 - return out, nil
278 - case <-ctx.Done():
279 - return nil, ctx.Err()
280 - }
281 -}
282 -
283 -func (bs *Bitswap) getNextSessionID() uint64 {
284 - bs.sessIDLk.Lock()
285 - defer bs.sessIDLk.Unlock()
286 - bs.sessID++
287 - return bs.sessID
288 -}
289 -
290 -// CancelWant removes a given key from the wantlist
291 -func (bs *Bitswap) CancelWants(cids []*cid.Cid, ses uint64) {
292 - if len(cids) == 0 {
293 - return
294 - }
295 - bs.wm.CancelWants(context.Background(), cids, nil, ses)
296 -}
297 -
298 -// HasBlock announces the existence of a block to this bitswap service. The
299 -// service will potentially notify its peers.
300 -func (bs *Bitswap) HasBlock(blk blocks.Block) error {
301 - return bs.receiveBlockFrom(blk, "")
302 -}
303 -
304 -// TODO: Some of this stuff really only needs to be done when adding a block
305 -// from the user, not when receiving it from the network.
306 -// In case you run `git blame` on this comment, I'll save you some time: ask
307 -// @whyrusleeping, I don't know the answers you seek.
308 -func (bs *Bitswap) receiveBlockFrom(blk blocks.Block, from peer.ID) error {
309 - select {
310 - case <-bs.process.Closing():
311 - return errors.New("bitswap is closed")
312 - default:
313 - }
314 -
315 - err := bs.blockstore.Put(blk)
316 - if err != nil {
317 - log.Errorf("Error writing block to datastore: %s", err)
318 - return err
319 - }
320 -
321 - // NOTE: There exists the possiblity for a race condition here. If a user
322 - // creates a node, then adds it to the dagservice while another goroutine
323 - // is waiting on a GetBlock for that object, they will receive a reference
324 - // to the same node. We should address this soon, but i'm not going to do
325 - // it now as it requires more thought and isnt causing immediate problems.
326 - bs.notifications.Publish(blk)
327 -
328 - k := blk.Cid()
329 - ks := []*cid.Cid{k}
330 - for _, s := range bs.SessionsForBlock(k) {
331 - s.receiveBlockFrom(from, blk)
332 - bs.CancelWants(ks, s.id)
333 - }
334 -
335 - bs.engine.AddBlock(blk)
336 -
337 - select {
338 - case bs.newBlocks <- blk.Cid():
339 - // send block off to be reprovided
340 - case <-bs.process.Closing():
341 - return bs.process.Close()
342 - }
343 - return nil
344 -}
345 -
346 -// SessionsForBlock returns a slice of all sessions that may be interested in the given cid
347 -func (bs *Bitswap) SessionsForBlock(c *cid.Cid) []*Session {
348 - bs.sessLk.Lock()
349 - defer bs.sessLk.Unlock()
350 -
351 - var out []*Session
352 - for _, s := range bs.sessions {
353 - if s.interestedIn(c) {
354 - out = append(out, s)
355 - }
356 - }
357 - return out
358 -}
359 -
360 -func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) {
361 - atomic.AddUint64(&bs.counters.messagesRecvd, 1)
362 -
363 - // This call records changes to wantlists, blocks received,
364 - // and number of bytes transfered.
365 - bs.engine.MessageReceived(p, incoming)
366 - // TODO: this is bad, and could be easily abused.
367 - // Should only track *useful* messages in ledger
368 -
369 - iblocks := incoming.Blocks()
370 -
371 - if len(iblocks) == 0 {
372 - return
373 - }
374 -
375 - wg := sync.WaitGroup{}
376 - for _, block := range iblocks {
377 - wg.Add(1)
378 - go func(b blocks.Block) { // TODO: this probably doesnt need to be a goroutine...
379 - defer wg.Done()
380 -
381 - bs.updateReceiveCounters(b)
382 -
383 - log.Debugf("got block %s from %s", b, p)
384 -
385 - if err := bs.receiveBlockFrom(b, p); err != nil {
386 - log.Warningf("ReceiveMessage recvBlockFrom error: %s", err)
387 - }
388 - log.Event(ctx, "Bitswap.GetBlockRequest.End", b.Cid())
389 - }(block)
390 - }
391 - wg.Wait()
392 -}
393 -
394 -var ErrAlreadyHaveBlock = errors.New("already have block")
395 -
396 -func (bs *Bitswap) updateReceiveCounters(b blocks.Block) {
397 - blkLen := len(b.RawData())
398 - has, err := bs.blockstore.Has(b.Cid())
399 - if err != nil {
400 - log.Infof("blockstore.Has error: %s", err)
401 - return
402 - }
403 -
404 - bs.allMetric.Observe(float64(blkLen))
405 - if has {
406 - bs.dupMetric.Observe(float64(blkLen))
407 - }
408 -
409 - bs.counterLk.Lock()
410 - defer bs.counterLk.Unlock()
411 - c := bs.counters
412 -
413 - c.blocksRecvd++
414 - c.dataRecvd += uint64(len(b.RawData()))
415 - if has {
416 - c.dupBlocksRecvd++
417 - c.dupDataRecvd += uint64(blkLen)
418 - }
419 -}
420 -
421 -// Connected/Disconnected warns bitswap about peer connections
422 -func (bs *Bitswap) PeerConnected(p peer.ID) {
423 - bs.wm.Connected(p)
424 - bs.engine.PeerConnected(p)
425 -}
426 -
427 -// Connected/Disconnected warns bitswap about peer connections
428 -func (bs *Bitswap) PeerDisconnected(p peer.ID) {
429 - bs.wm.Disconnected(p)
430 - bs.engine.PeerDisconnected(p)
431 -}
432 -
433 -func (bs *Bitswap) ReceiveError(err error) {
434 - log.Infof("Bitswap ReceiveError: %s", err)
435 - // TODO log the network error
436 - // TODO bubble the network error up to the parent context/error logger
437 -}
438 -
439 -func (bs *Bitswap) Close() error {
440 - return bs.process.Close()
441 -}
442 -
443 -func (bs *Bitswap) GetWantlist() []*cid.Cid {
444 - entries := bs.wm.wl.Entries()
445 - out := make([]*cid.Cid, 0, len(entries))
446 - for _, e := range entries {
447 - out = append(out, e.Cid)
448 - }
449 - return out
450 -}
451 -
452 -func (bs *Bitswap) IsOnline() bool {
453 - return true
454 -}
exchange/bitswap/bitswap_test.go deleted
-674
@@ -1,674 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "fmt"
7 - "sync"
8 - "testing"
9 - "time"
10 -
11 - decision "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
12 - tn "github.com/ipfs/go-ipfs/exchange/bitswap/testnet"
13 -
14 - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
15 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
16 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
17 - blocksutil "gx/ipfs/QmYqPGpZ9Yemr55xus9DiEztkns6Jti5XJ7hC94JbvkdqZ/go-ipfs-blocksutil"
18 - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
19 - mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock"
20 - tu "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
21 - travis "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil/ci/travis"
22 - p2ptestutil "gx/ipfs/QmcxUtMB5sJrXR3znSvkrDd2ghvwGM8rLRqwJiPUdgQwat/go-libp2p-netutil"
23 - detectrace "gx/ipfs/Qmf7HqcW7LtCi1W8y2bdx2eJpze74jkbKqpByxgXikdbLF/go-detect-race"
24 -)
25 -
26 -// FIXME the tests are really sensitive to the network delay. fix them to work
27 -// well under varying conditions
28 -const kNetworkDelay = 0 * time.Millisecond
29 -
30 -func getVirtualNetwork() tn.Network {
31 - return tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
32 -}
33 -
34 -func TestClose(t *testing.T) {
35 - vnet := getVirtualNetwork()
36 - sesgen := NewTestSessionGenerator(vnet)
37 - defer sesgen.Close()
38 - bgen := blocksutil.NewBlockGenerator()
39 -
40 - block := bgen.Next()
41 - bitswap := sesgen.Next()
42 -
43 - bitswap.Exchange.Close()
44 - bitswap.Exchange.GetBlock(context.Background(), block.Cid())
45 -}
46 -
47 -func TestProviderForKeyButNetworkCannotFind(t *testing.T) { // TODO revisit this
48 -
49 - rs := mockrouting.NewServer()
50 - net := tn.VirtualNetwork(rs, delay.Fixed(kNetworkDelay))
51 - g := NewTestSessionGenerator(net)
52 - defer g.Close()
53 -
54 - block := blocks.NewBlock([]byte("block"))
55 - pinfo := p2ptestutil.RandTestBogusIdentityOrFatal(t)
56 - rs.Client(pinfo).Provide(context.Background(), block.Cid(), true) // but not on network
57 -
58 - solo := g.Next()
59 - defer solo.Exchange.Close()
60 -
61 - ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
62 - defer cancel()
63 - _, err := solo.Exchange.GetBlock(ctx, block.Cid())
64 -
65 - if err != context.DeadlineExceeded {
66 - t.Fatal("Expected DeadlineExceeded error")
67 - }
68 -}
69 -
70 -func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {
71 -
72 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
73 - block := blocks.NewBlock([]byte("block"))
74 - g := NewTestSessionGenerator(net)
75 - defer g.Close()
76 -
77 - peers := g.Instances(2)
78 - hasBlock := peers[0]
79 - defer hasBlock.Exchange.Close()
80 -
81 - if err := hasBlock.Exchange.HasBlock(block); err != nil {
82 - t.Fatal(err)
83 - }
84 -
85 - wantsBlock := peers[1]
86 - defer wantsBlock.Exchange.Close()
87 -
88 - ctx, cancel := context.WithTimeout(context.Background(), time.Second)
89 - defer cancel()
90 - received, err := wantsBlock.Exchange.GetBlock(ctx, block.Cid())
91 - if err != nil {
92 - t.Log(err)
93 - t.Fatal("Expected to succeed")
94 - }
95 -
96 - if !bytes.Equal(block.RawData(), received.RawData()) {
97 - t.Fatal("Data doesn't match")
98 - }
99 -}
100 -
101 -func TestLargeSwarm(t *testing.T) {
102 - if testing.Short() {
103 - t.SkipNow()
104 - }
105 - numInstances := 100
106 - numBlocks := 2
107 - if detectrace.WithRace() {
108 - // when running with the race detector, 500 instances launches
109 - // well over 8k goroutines. This hits a race detector limit.
110 - numInstances = 75
111 - } else if travis.IsRunning() {
112 - numInstances = 200
113 - } else {
114 - t.Parallel()
115 - }
116 - PerformDistributionTest(t, numInstances, numBlocks)
117 -}
118 -
119 -func TestLargeFile(t *testing.T) {
120 - if testing.Short() {
121 - t.SkipNow()
122 - }
123 -
124 - if !travis.IsRunning() {
125 - t.Parallel()
126 - }
127 -
128 - numInstances := 10
129 - numBlocks := 100
130 - PerformDistributionTest(t, numInstances, numBlocks)
131 -}
132 -
133 -func TestLargeFileNoRebroadcast(t *testing.T) {
134 - rbd := rebroadcastDelay.Get()
135 - rebroadcastDelay.Set(time.Hour * 24 * 365 * 10) // ten years should be long enough
136 - if testing.Short() {
137 - t.SkipNow()
138 - }
139 - numInstances := 10
140 - numBlocks := 100
141 - PerformDistributionTest(t, numInstances, numBlocks)
142 - rebroadcastDelay.Set(rbd)
143 -}
144 -
145 -func TestLargeFileTwoPeers(t *testing.T) {
146 - if testing.Short() {
147 - t.SkipNow()
148 - }
149 - numInstances := 2
150 - numBlocks := 100
151 - PerformDistributionTest(t, numInstances, numBlocks)
152 -}
153 -
154 -func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) {
155 - ctx := context.Background()
156 - if testing.Short() {
157 - t.SkipNow()
158 - }
159 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
160 - sg := NewTestSessionGenerator(net)
161 - defer sg.Close()
162 - bg := blocksutil.NewBlockGenerator()
163 -
164 - instances := sg.Instances(numInstances)
165 - blocks := bg.Blocks(numBlocks)
166 -
167 - t.Log("Give the blocks to the first instance")
168 -
169 - nump := len(instances) - 1
170 - // assert we're properly connected
171 - for _, inst := range instances {
172 - peers := inst.Exchange.wm.ConnectedPeers()
173 - for i := 0; i < 10 && len(peers) != nump; i++ {
174 - time.Sleep(time.Millisecond * 50)
175 - peers = inst.Exchange.wm.ConnectedPeers()
176 - }
177 - if len(peers) != nump {
178 - t.Fatal("not enough peers connected to instance")
179 - }
180 - }
181 -
182 - var blkeys []*cid.Cid
183 - first := instances[0]
184 - for _, b := range blocks {
185 - blkeys = append(blkeys, b.Cid())
186 - first.Exchange.HasBlock(b)
187 - }
188 -
189 - t.Log("Distribute!")
190 -
191 - wg := sync.WaitGroup{}
192 - errs := make(chan error)
193 -
194 - for _, inst := range instances[1:] {
195 - wg.Add(1)
196 - go func(inst Instance) {
197 - defer wg.Done()
198 - outch, err := inst.Exchange.GetBlocks(ctx, blkeys)
199 - if err != nil {
200 - errs <- err
201 - }
202 - for range outch {
203 - }
204 - }(inst)
205 - }
206 -
207 - go func() {
208 - wg.Wait()
209 - close(errs)
210 - }()
211 -
212 - for err := range errs {
213 - if err != nil {
214 - t.Fatal(err)
215 - }
216 - }
217 -
218 - t.Log("Verify!")
219 -
220 - for _, inst := range instances {
221 - for _, b := range blocks {
222 - if _, err := inst.Blockstore().Get(b.Cid()); err != nil {
223 - t.Fatal(err)
224 - }
225 - }
226 - }
227 -}
228 -
229 -// TODO simplify this test. get to the _essence_!
230 -func TestSendToWantingPeer(t *testing.T) {
231 - if testing.Short() {
232 - t.SkipNow()
233 - }
234 -
235 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
236 - sg := NewTestSessionGenerator(net)
237 - defer sg.Close()
238 - bg := blocksutil.NewBlockGenerator()
239 -
240 - prev := rebroadcastDelay.Set(time.Second / 2)
241 - defer func() { rebroadcastDelay.Set(prev) }()
242 -
243 - peers := sg.Instances(2)
244 - peerA := peers[0]
245 - peerB := peers[1]
246 -
247 - t.Logf("Session %v\n", peerA.Peer)
248 - t.Logf("Session %v\n", peerB.Peer)
249 -
250 - waitTime := time.Second * 5
251 -
252 - alpha := bg.Next()
253 - // peerA requests and waits for block alpha
254 - ctx, cancel := context.WithTimeout(context.Background(), waitTime)
255 - defer cancel()
256 - alphaPromise, err := peerA.Exchange.GetBlocks(ctx, []*cid.Cid{alpha.Cid()})
257 - if err != nil {
258 - t.Fatal(err)
259 - }
260 -
261 - // peerB announces to the network that he has block alpha
262 - err = peerB.Exchange.HasBlock(alpha)
263 - if err != nil {
264 - t.Fatal(err)
265 - }
266 -
267 - // At some point, peerA should get alpha (or timeout)
268 - blkrecvd, ok := <-alphaPromise
269 - if !ok {
270 - t.Fatal("context timed out and broke promise channel!")
271 - }
272 -
273 - if !blkrecvd.Cid().Equals(alpha.Cid()) {
274 - t.Fatal("Wrong block!")
275 - }
276 -
277 -}
278 -
279 -func TestEmptyKey(t *testing.T) {
280 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
281 - sg := NewTestSessionGenerator(net)
282 - defer sg.Close()
283 - bs := sg.Instances(1)[0].Exchange
284 -
285 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
286 - defer cancel()
287 -
288 - _, err := bs.GetBlock(ctx, nil)
289 - if err != blockstore.ErrNotFound {
290 - t.Error("empty str key should return ErrNotFound")
291 - }
292 -}
293 -
294 -func assertStat(t *testing.T, st *Stat, sblks, rblks, sdata, rdata uint64) {
295 - if sblks != st.BlocksSent {
296 - t.Errorf("mismatch in blocks sent: %d vs %d", sblks, st.BlocksSent)
297 - }
298 -
299 - if rblks != st.BlocksReceived {
300 - t.Errorf("mismatch in blocks recvd: %d vs %d", rblks, st.BlocksReceived)
301 - }
302 -
303 - if sdata != st.DataSent {
304 - t.Errorf("mismatch in data sent: %d vs %d", sdata, st.DataSent)
305 - }
306 -
307 - if rdata != st.DataReceived {
308 - t.Errorf("mismatch in data recvd: %d vs %d", rdata, st.DataReceived)
309 - }
310 -}
311 -
312 -func TestBasicBitswap(t *testing.T) {
313 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
314 - sg := NewTestSessionGenerator(net)
315 - defer sg.Close()
316 - bg := blocksutil.NewBlockGenerator()
317 -
318 - t.Log("Test a one node trying to get one block from another")
319 -
320 - instances := sg.Instances(3)
321 - blocks := bg.Blocks(1)
322 - err := instances[0].Exchange.HasBlock(blocks[0])
323 - if err != nil {
324 - t.Fatal(err)
325 - }
326 -
327 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
328 - defer cancel()
329 - blk, err := instances[1].Exchange.GetBlock(ctx, blocks[0].Cid())
330 - if err != nil {
331 - t.Fatal(err)
332 - }
333 -
334 - if err = tu.WaitFor(ctx, func() error {
335 - if len(instances[2].Exchange.WantlistForPeer(instances[1].Peer)) != 0 {
336 - return fmt.Errorf("should have no items in other peers wantlist")
337 - }
338 - if len(instances[1].Exchange.GetWantlist()) != 0 {
339 - return fmt.Errorf("shouldnt have anything in wantlist")
340 - }
341 - return nil
342 - }); err != nil {
343 - t.Fatal(err)
344 - }
345 -
346 - st0, err := instances[0].Exchange.Stat()
347 - if err != nil {
348 - t.Fatal(err)
349 - }
350 -
351 - st1, err := instances[1].Exchange.Stat()
352 - if err != nil {
353 - t.Fatal(err)
354 - }
355 -
356 - st2, err := instances[2].Exchange.Stat()
357 - if err != nil {
358 - t.Fatal(err)
359 - }
360 -
361 - t.Log("stat node 0")
362 - assertStat(t, st0, 1, 0, uint64(len(blk.RawData())), 0)
363 - t.Log("stat node 1")
364 - assertStat(t, st1, 0, 1, 0, uint64(len(blk.RawData())))
365 - t.Log("stat node 2")
366 - assertStat(t, st2, 0, 0, 0, 0)
367 -
368 - if !bytes.Equal(blk.RawData(), blocks[0].RawData()) {
369 - t.Errorf("blocks aren't equal: expected %v, actual %v", blocks[0].RawData(), blk.RawData())
370 - }
371 -
372 - t.Log(blk)
373 - for _, inst := range instances {
374 - err := inst.Exchange.Close()
375 - if err != nil {
376 - t.Fatal(err)
377 - }
378 - }
379 -}
380 -
381 -func TestDoubleGet(t *testing.T) {
382 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
383 - sg := NewTestSessionGenerator(net)
384 - defer sg.Close()
385 - bg := blocksutil.NewBlockGenerator()
386 -
387 - t.Log("Test a one node trying to get one block from another")
388 -
389 - instances := sg.Instances(2)
390 - blocks := bg.Blocks(1)
391 -
392 - // NOTE: A race condition can happen here where these GetBlocks requests go
393 - // through before the peers even get connected. This is okay, bitswap
394 - // *should* be able to handle this.
395 - ctx1, cancel1 := context.WithCancel(context.Background())
396 - blkch1, err := instances[1].Exchange.GetBlocks(ctx1, []*cid.Cid{blocks[0].Cid()})
397 - if err != nil {
398 - t.Fatal(err)
399 - }
400 -
401 - ctx2, cancel2 := context.WithCancel(context.Background())
402 - defer cancel2()
403 -
404 - blkch2, err := instances[1].Exchange.GetBlocks(ctx2, []*cid.Cid{blocks[0].Cid()})
405 - if err != nil {
406 - t.Fatal(err)
407 - }
408 -
409 - // ensure both requests make it into the wantlist at the same time
410 - time.Sleep(time.Millisecond * 20)
411 - cancel1()
412 -
413 - _, ok := <-blkch1
414 - if ok {
415 - t.Fatal("expected channel to be closed")
416 - }
417 -
418 - err = instances[0].Exchange.HasBlock(blocks[0])
419 - if err != nil {
420 - t.Fatal(err)
421 - }
422 -
423 - select {
424 - case blk, ok := <-blkch2:
425 - if !ok {
426 - t.Fatal("expected to get the block here")
427 - }
428 - t.Log(blk)
429 - case <-time.After(time.Second * 5):
430 - p1wl := instances[0].Exchange.WantlistForPeer(instances[1].Peer)
431 - if len(p1wl) != 1 {
432 - t.Logf("wantlist view didnt have 1 item (had %d)", len(p1wl))
433 - } else if !p1wl[0].Equals(blocks[0].Cid()) {
434 - t.Logf("had 1 item, it was wrong: %s %s", blocks[0].Cid(), p1wl[0])
435 - } else {
436 - t.Log("had correct wantlist, somehow")
437 - }
438 - t.Fatal("timed out waiting on block")
439 - }
440 -
441 - for _, inst := range instances {
442 - err := inst.Exchange.Close()
443 - if err != nil {
444 - t.Fatal(err)
445 - }
446 - }
447 -}
448 -
449 -func TestWantlistCleanup(t *testing.T) {
450 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
451 - sg := NewTestSessionGenerator(net)
452 - defer sg.Close()
453 - bg := blocksutil.NewBlockGenerator()
454 -
455 - instances := sg.Instances(1)[0]
456 - bswap := instances.Exchange
457 - blocks := bg.Blocks(20)
458 -
459 - var keys []*cid.Cid
460 - for _, b := range blocks {
461 - keys = append(keys, b.Cid())
462 - }
463 -
464 - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*50)
465 - defer cancel()
466 - _, err := bswap.GetBlock(ctx, keys[0])
467 - if err != context.DeadlineExceeded {
468 - t.Fatal("shouldnt have fetched any blocks")
469 - }
470 -
471 - time.Sleep(time.Millisecond * 50)
472 -
473 - if len(bswap.GetWantlist()) > 0 {
474 - t.Fatal("should not have anyting in wantlist")
475 - }
476 -
477 - ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*50)
478 - defer cancel()
479 - _, err = bswap.GetBlocks(ctx, keys[:10])
480 - if err != nil {
481 - t.Fatal(err)
482 - }
483 -
484 - <-ctx.Done()
485 - time.Sleep(time.Millisecond * 50)
486 -
487 - if len(bswap.GetWantlist()) > 0 {
488 - t.Fatal("should not have anyting in wantlist")
489 - }
490 -
491 - _, err = bswap.GetBlocks(context.Background(), keys[:1])
492 - if err != nil {
493 - t.Fatal(err)
494 - }
495 -
496 - ctx, cancel = context.WithCancel(context.Background())
497 - _, err = bswap.GetBlocks(ctx, keys[10:])
498 - if err != nil {
499 - t.Fatal(err)
500 - }
501 -
502 - time.Sleep(time.Millisecond * 50)
503 - if len(bswap.GetWantlist()) != 11 {
504 - t.Fatal("should have 11 keys in wantlist")
505 - }
506 -
507 - cancel()
508 - time.Sleep(time.Millisecond * 50)
509 - if !(len(bswap.GetWantlist()) == 1 && bswap.GetWantlist()[0] == keys[0]) {
510 - t.Fatal("should only have keys[0] in wantlist")
511 - }
512 -}
513 -
514 -func assertLedgerMatch(ra, rb *decision.Receipt) error {
515 - if ra.Sent != rb.Recv {
516 - return fmt.Errorf("mismatch in ledgers (exchanged bytes): %d sent vs %d recvd", ra.Sent, rb.Recv)
517 - }
518 -
519 - if ra.Recv != rb.Sent {
520 - return fmt.Errorf("mismatch in ledgers (exchanged bytes): %d recvd vs %d sent", ra.Recv, rb.Sent)
521 - }
522 -
523 - if ra.Exchanged != rb.Exchanged {
524 - return fmt.Errorf("mismatch in ledgers (exchanged blocks): %d vs %d ", ra.Exchanged, rb.Exchanged)
525 - }
526 -
527 - return nil
528 -}
529 -
530 -func assertLedgerEqual(ra, rb *decision.Receipt) error {
531 - if ra.Value != rb.Value {
532 - return fmt.Errorf("mismatch in ledgers (value/debt ratio): %f vs %f ", ra.Value, rb.Value)
533 - }
534 -
535 - if ra.Sent != rb.Sent {
536 - return fmt.Errorf("mismatch in ledgers (sent bytes): %d vs %d", ra.Sent, rb.Sent)
537 - }
538 -
539 - if ra.Recv != rb.Recv {
540 - return fmt.Errorf("mismatch in ledgers (recvd bytes): %d vs %d", ra.Recv, rb.Recv)
541 - }
542 -
543 - if ra.Exchanged != rb.Exchanged {
544 - return fmt.Errorf("mismatch in ledgers (exchanged blocks): %d vs %d ", ra.Exchanged, rb.Exchanged)
545 - }
546 -
547 - return nil
548 -}
549 -
550 -func newReceipt(sent, recv, exchanged uint64) *decision.Receipt {
551 - return &decision.Receipt{
552 - Peer: "test",
553 - Value: float64(sent) / (1 + float64(recv)),
554 - Sent: sent,
555 - Recv: recv,
556 - Exchanged: exchanged,
557 - }
558 -}
559 -
560 -func TestBitswapLedgerOneWay(t *testing.T) {
561 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
562 - sg := NewTestSessionGenerator(net)
563 - defer sg.Close()
564 - bg := blocksutil.NewBlockGenerator()
565 -
566 - t.Log("Test ledgers match when one peer sends block to another")
567 -
568 - instances := sg.Instances(2)
569 - blocks := bg.Blocks(1)
570 - err := instances[0].Exchange.HasBlock(blocks[0])
571 - if err != nil {
572 - t.Fatal(err)
573 - }
574 -
575 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
576 - defer cancel()
577 - blk, err := instances[1].Exchange.GetBlock(ctx, blocks[0].Cid())
578 - if err != nil {
579 - t.Fatal(err)
580 - }
581 -
582 - ra := instances[0].Exchange.LedgerForPeer(instances[1].Peer)
583 - rb := instances[1].Exchange.LedgerForPeer(instances[0].Peer)
584 -
585 - // compare peer ledger receipts
586 - err = assertLedgerMatch(ra, rb)
587 - if err != nil {
588 - t.Fatal(err)
589 - }
590 -
591 - // check that receipts have intended values
592 - ratest := newReceipt(1, 0, 1)
593 - err = assertLedgerEqual(ratest, ra)
594 - if err != nil {
595 - t.Fatal(err)
596 - }
597 - rbtest := newReceipt(0, 1, 1)
598 - err = assertLedgerEqual(rbtest, rb)
599 - if err != nil {
600 - t.Fatal(err)
601 - }
602 -
603 - t.Log(blk)
604 - for _, inst := range instances {
605 - err := inst.Exchange.Close()
606 - if err != nil {
607 - t.Fatal(err)
608 - }
609 - }
610 -}
611 -
612 -func TestBitswapLedgerTwoWay(t *testing.T) {
613 - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
614 - sg := NewTestSessionGenerator(net)
615 - defer sg.Close()
616 - bg := blocksutil.NewBlockGenerator()
617 -
618 - t.Log("Test ledgers match when two peers send one block to each other")
619 -
620 - instances := sg.Instances(2)
621 - blocks := bg.Blocks(2)
622 - err := instances[0].Exchange.HasBlock(blocks[0])
623 - if err != nil {
624 - t.Fatal(err)
625 - }
626 -
627 - err = instances[1].Exchange.HasBlock(blocks[1])
628 - if err != nil {
629 - t.Fatal(err)
630 - }
631 -
632 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
633 - defer cancel()
634 - _, err = instances[1].Exchange.GetBlock(ctx, blocks[0].Cid())
635 - if err != nil {
636 - t.Fatal(err)
637 - }
638 -
639 - ctx, cancel = context.WithTimeout(context.Background(), time.Second*5)
640 - defer cancel()
641 - blk, err := instances[0].Exchange.GetBlock(ctx, blocks[1].Cid())
642 - if err != nil {
643 - t.Fatal(err)
644 - }
645 -
646 - ra := instances[0].Exchange.LedgerForPeer(instances[1].Peer)
647 - rb := instances[1].Exchange.LedgerForPeer(instances[0].Peer)
648 -
649 - // compare peer ledger receipts
650 - err = assertLedgerMatch(ra, rb)
651 - if err != nil {
652 - t.Fatal(err)
653 - }
654 -
655 - // check that receipts have intended values
656 - rtest := newReceipt(1, 1, 2)
657 - err = assertLedgerEqual(rtest, ra)
658 - if err != nil {
659 - t.Fatal(err)
660 - }
661 -
662 - err = assertLedgerEqual(rtest, rb)
663 - if err != nil {
664 - t.Fatal(err)
665 - }
666 -
667 - t.Log(blk)
668 - for _, inst := range instances {
669 - err := inst.Exchange.Close()
670 - if err != nil {
671 - t.Fatal(err)
672 - }
673 - }
674 -}
exchange/bitswap/decision/bench_test.go deleted
-30
@@ -1,30 +0,0 @@
1 -package decision
2 -
3 -import (
4 - "fmt"
5 - "math"
6 - "testing"
7 -
8 - "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
9 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
10 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
11 - "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
12 - "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
13 -)
14 -
15 -// FWIW: At the time of this commit, including a timestamp in task increases
16 -// time cost of Push by 3%.
17 -func BenchmarkTaskQueuePush(b *testing.B) {
18 - q := newPRQ()
19 - peers := []peer.ID{
20 - testutil.RandPeerIDFatal(b),
21 - testutil.RandPeerIDFatal(b),
22 - testutil.RandPeerIDFatal(b),
23 - }
24 - b.ResetTimer()
25 - for i := 0; i < b.N; i++ {
26 - c := cid.NewCidV0(u.Hash([]byte(fmt.Sprint(i))))
27 -
28 - q.Push(&wantlist.Entry{Cid: c, Priority: math.MaxInt32}, peers[i%len(peers)])
29 - }
30 -}
exchange/bitswap/decision/engine.go deleted
-356
@@ -1,356 +0,0 @@
1 -// package decision implements the decision engine for the bitswap service.
2 -package decision
3 -
4 -import (
5 - "context"
6 - "sync"
7 - "time"
8 -
9 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
10 - wl "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
11 -
12 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
13 - bstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
14 - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
15 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
16 -)
17 -
18 -// TODO consider taking responsibility for other types of requests. For
19 -// example, there could be a |cancelQueue| for all of the cancellation
20 -// messages that need to go out. There could also be a |wantlistQueue| for
21 -// the local peer's wantlists. Alternatively, these could all be bundled
22 -// into a single, intelligent global queue that efficiently
23 -// batches/combines and takes all of these into consideration.
24 -//
25 -// Right now, messages go onto the network for four reasons:
26 -// 1. an initial `sendwantlist` message to a provider of the first key in a
27 -// request
28 -// 2. a periodic full sweep of `sendwantlist` messages to all providers
29 -// 3. upon receipt of blocks, a `cancel` message to all peers
30 -// 4. draining the priority queue of `blockrequests` from peers
31 -//
32 -// Presently, only `blockrequests` are handled by the decision engine.
33 -// However, there is an opportunity to give it more responsibility! If the
34 -// decision engine is given responsibility for all of the others, it can
35 -// intelligently decide how to combine requests efficiently.
36 -//
37 -// Some examples of what would be possible:
38 -//
39 -// * when sending out the wantlists, include `cancel` requests
40 -// * when handling `blockrequests`, include `sendwantlist` and `cancel` as
41 -// appropriate
42 -// * when handling `cancel`, if we recently received a wanted block from a
43 -// peer, include a partial wantlist that contains a few other high priority
44 -// blocks
45 -//
46 -// In a sense, if we treat the decision engine as a black box, it could do
47 -// whatever it sees fit to produce desired outcomes (get wanted keys
48 -// quickly, maintain good relationships with peers, etc).
49 -
50 -var log = logging.Logger("engine")
51 -
52 -const (
53 - // outboxChanBuffer must be 0 to prevent stale messages from being sent
54 - outboxChanBuffer = 0
55 -)
56 -
57 -// Envelope contains a message for a Peer
58 -type Envelope struct {
59 - // Peer is the intended recipient
60 - Peer peer.ID
61 -
62 - // Block is the payload
63 - Block blocks.Block
64 -
65 - // A callback to notify the decision queue that the task is complete
66 - Sent func()
67 -}
68 -
69 -type Engine struct {
70 - // peerRequestQueue is a priority queue of requests received from peers.
71 - // Requests are popped from the queue, packaged up, and placed in the
72 - // outbox.
73 - peerRequestQueue *prq
74 -
75 - // FIXME it's a bit odd for the client and the worker to both share memory
76 - // (both modify the peerRequestQueue) and also to communicate over the
77 - // workSignal channel. consider sending requests over the channel and
78 - // allowing the worker to have exclusive access to the peerRequestQueue. In
79 - // that case, no lock would be required.
80 - workSignal chan struct{}
81 -
82 - // outbox contains outgoing messages to peers. This is owned by the
83 - // taskWorker goroutine
84 - outbox chan (<-chan *Envelope)
85 -
86 - bs bstore.Blockstore
87 -
88 - lock sync.Mutex // protects the fields immediatly below
89 - // ledgerMap lists Ledgers by their Partner key.
90 - ledgerMap map[peer.ID]*ledger
91 -
92 - ticker *time.Ticker
93 -}
94 -
95 -func NewEngine(ctx context.Context, bs bstore.Blockstore) *Engine {
96 - e := &Engine{
97 - ledgerMap: make(map[peer.ID]*ledger),
98 - bs: bs,
99 - peerRequestQueue: newPRQ(),
100 - outbox: make(chan (<-chan *Envelope), outboxChanBuffer),
101 - workSignal: make(chan struct{}, 1),
102 - ticker: time.NewTicker(time.Millisecond * 100),
103 - }
104 - go e.taskWorker(ctx)
105 - return e
106 -}
107 -
108 -func (e *Engine) WantlistForPeer(p peer.ID) (out []*wl.Entry) {
109 - partner := e.findOrCreate(p)
110 - partner.lk.Lock()
111 - defer partner.lk.Unlock()
112 - return partner.wantList.SortedEntries()
113 -}
114 -
115 -func (e *Engine) LedgerForPeer(p peer.ID) *Receipt {
116 - ledger := e.findOrCreate(p)
117 -
118 - ledger.lk.Lock()
119 - defer ledger.lk.Unlock()
120 -
121 - return &Receipt{
122 - Peer: ledger.Partner.String(),
123 - Value: ledger.Accounting.Value(),
124 - Sent: ledger.Accounting.BytesSent,
125 - Recv: ledger.Accounting.BytesRecv,
126 - Exchanged: ledger.ExchangeCount(),
127 - }
128 -}
129 -
130 -func (e *Engine) taskWorker(ctx context.Context) {
131 - defer close(e.outbox) // because taskWorker uses the channel exclusively
132 - for {
133 - oneTimeUse := make(chan *Envelope, 1) // buffer to prevent blocking
134 - select {
135 - case <-ctx.Done():
136 - return
137 - case e.outbox <- oneTimeUse:
138 - }
139 - // receiver is ready for an outoing envelope. let's prepare one. first,
140 - // we must acquire a task from the PQ...
141 - envelope, err := e.nextEnvelope(ctx)
142 - if err != nil {
143 - close(oneTimeUse)
144 - return // ctx cancelled
145 - }
146 - oneTimeUse <- envelope // buffered. won't block
147 - close(oneTimeUse)
148 - }
149 -}
150 -
151 -// nextEnvelope runs in the taskWorker goroutine. Returns an error if the
152 -// context is cancelled before the next Envelope can be created.
153 -func (e *Engine) nextEnvelope(ctx context.Context) (*Envelope, error) {
154 - for {
155 - nextTask := e.peerRequestQueue.Pop()
156 - for nextTask == nil {
157 - select {
158 - case <-ctx.Done():
159 - return nil, ctx.Err()
160 - case <-e.workSignal:
161 - nextTask = e.peerRequestQueue.Pop()
162 - case <-e.ticker.C:
163 - e.peerRequestQueue.thawRound()
164 - nextTask = e.peerRequestQueue.Pop()
165 - }
166 - }
167 -
168 - // with a task in hand, we're ready to prepare the envelope...
169 -
170 - block, err := e.bs.Get(nextTask.Entry.Cid)
171 - if err != nil {
172 - log.Errorf("tried to execute a task and errored fetching block: %s", err)
173 - // If we don't have the block, don't hold that against the peer
174 - // make sure to update that the task has been 'completed'
175 - nextTask.Done()
176 - continue
177 - }
178 -
179 - return &Envelope{
180 - Peer: nextTask.Target,
181 - Block: block,
182 - Sent: func() {
183 - nextTask.Done()
184 - select {
185 - case e.workSignal <- struct{}{}:
186 - // work completing may mean that our queue will provide new
187 - // work to be done.
188 - default:
189 - }
190 - },
191 - }, nil
192 - }
193 -}
194 -
195 -// Outbox returns a channel of one-time use Envelope channels.
196 -func (e *Engine) Outbox() <-chan (<-chan *Envelope) {
197 - return e.outbox
198 -}
199 -
200 -// Returns a slice of Peers with whom the local node has active sessions
201 -func (e *Engine) Peers() []peer.ID {
202 - e.lock.Lock()
203 - defer e.lock.Unlock()
204 -
205 - response := make([]peer.ID, 0, len(e.ledgerMap))
206 -
207 - for _, ledger := range e.ledgerMap {
208 - response = append(response, ledger.Partner)
209 - }
210 - return response
211 -}
212 -
213 -// MessageReceived performs book-keeping. Returns error if passed invalid
214 -// arguments.
215 -func (e *Engine) MessageReceived(p peer.ID, m bsmsg.BitSwapMessage) error {
216 - if len(m.Wantlist()) == 0 && len(m.Blocks()) == 0 {
217 - log.Debugf("received empty message from %s", p)
218 - }
219 -
220 - newWorkExists := false
221 - defer func() {
222 - if newWorkExists {
223 - e.signalNewWork()
224 - }
225 - }()
226 -
227 - l := e.findOrCreate(p)
228 - l.lk.Lock()
229 - defer l.lk.Unlock()
230 - if m.Full() {
231 - l.wantList = wl.New()
232 - }
233 -
234 - for _, entry := range m.Wantlist() {
235 - if entry.Cancel {
236 - log.Debugf("%s cancel %s", p, entry.Cid)
237 - l.CancelWant(entry.Cid)
238 - e.peerRequestQueue.Remove(entry.Cid, p)
239 - } else {
240 - log.Debugf("wants %s - %d", entry.Cid, entry.Priority)
241 - l.Wants(entry.Cid, entry.Priority)
242 - if exists, err := e.bs.Has(entry.Cid); err == nil && exists {
243 - e.peerRequestQueue.Push(entry.Entry, p)
244 - newWorkExists = true
245 - }
246 - }
247 - }
248 -
249 - for _, block := range m.Blocks() {
250 - log.Debugf("got block %s %d bytes", block, len(block.RawData()))
251 - l.ReceivedBytes(len(block.RawData()))
252 - }
253 - return nil
254 -}
255 -
256 -func (e *Engine) addBlock(block blocks.Block) {
257 - work := false
258 -
259 - for _, l := range e.ledgerMap {
260 - l.lk.Lock()
261 - if entry, ok := l.WantListContains(block.Cid()); ok {
262 - e.peerRequestQueue.Push(entry, l.Partner)
263 - work = true
264 - }
265 - l.lk.Unlock()
266 - }
267 -
268 - if work {
269 - e.signalNewWork()
270 - }
271 -}
272 -
273 -func (e *Engine) AddBlock(block blocks.Block) {
274 - e.lock.Lock()
275 - defer e.lock.Unlock()
276 -
277 - e.addBlock(block)
278 -}
279 -
280 -// TODO add contents of m.WantList() to my local wantlist? NB: could introduce
281 -// race conditions where I send a message, but MessageSent gets handled after
282 -// MessageReceived. The information in the local wantlist could become
283 -// inconsistent. Would need to ensure that Sends and acknowledgement of the
284 -// send happen atomically
285 -
286 -func (e *Engine) MessageSent(p peer.ID, m bsmsg.BitSwapMessage) error {
287 - l := e.findOrCreate(p)
288 - l.lk.Lock()
289 - defer l.lk.Unlock()
290 -
291 - for _, block := range m.Blocks() {
292 - l.SentBytes(len(block.RawData()))
293 - l.wantList.Remove(block.Cid())
294 - e.peerRequestQueue.Remove(block.Cid(), p)
295 - }
296 -
297 - return nil
298 -}
299 -
300 -func (e *Engine) PeerConnected(p peer.ID) {
301 - e.lock.Lock()
302 - defer e.lock.Unlock()
303 - l, ok := e.ledgerMap[p]
304 - if !ok {
305 - l = newLedger(p)
306 - e.ledgerMap[p] = l
307 - }
308 - l.lk.Lock()
309 - defer l.lk.Unlock()
310 - l.ref++
311 -}
312 -
313 -func (e *Engine) PeerDisconnected(p peer.ID) {
314 - e.lock.Lock()
315 - defer e.lock.Unlock()
316 - l, ok := e.ledgerMap[p]
317 - if !ok {
318 - return
319 - }
320 - l.lk.Lock()
321 - defer l.lk.Unlock()
322 - l.ref--
323 - if l.ref <= 0 {
324 - delete(e.ledgerMap, p)
325 - }
326 -}
327 -
328 -func (e *Engine) numBytesSentTo(p peer.ID) uint64 {
329 - // NB not threadsafe
330 - return e.findOrCreate(p).Accounting.BytesSent
331 -}
332 -
333 -func (e *Engine) numBytesReceivedFrom(p peer.ID) uint64 {
334 - // NB not threadsafe
335 - return e.findOrCreate(p).Accounting.BytesRecv
336 -}
337 -
338 -// ledger lazily instantiates a ledger
339 -func (e *Engine) findOrCreate(p peer.ID) *ledger {
340 - e.lock.Lock()
341 - defer e.lock.Unlock()
342 - l, ok := e.ledgerMap[p]
343 - if !ok {
344 - l = newLedger(p)
345 - e.ledgerMap[p] = l
346 - }
347 - return l
348 -}
349 -
350 -func (e *Engine) signalNewWork() {
351 - // Signal task generation to restart (if stopped!)
352 - select {
353 - case e.workSignal <- struct{}{}:
354 - default:
355 - }
356 -}
exchange/bitswap/decision/engine_test.go deleted
-215
@@ -1,215 +0,0 @@
1 -package decision
2 -
3 -import (
4 - "context"
5 - "errors"
6 - "fmt"
7 - "math"
8 - "strings"
9 - "sync"
10 - "testing"
11 -
12 - message "github.com/ipfs/go-ipfs/exchange/bitswap/message"
13 -
14 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
15 - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
16 - testutil "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
17 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
18 - ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
19 - dssync "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"
20 -)
21 -
22 -type peerAndEngine struct {
23 - Peer peer.ID
24 - Engine *Engine
25 -}
26 -
27 -func newEngine(ctx context.Context, idStr string) peerAndEngine {
28 - return peerAndEngine{
29 - Peer: peer.ID(idStr),
30 - //Strategy: New(true),
31 - Engine: NewEngine(ctx,
32 - blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))),
33 - }
34 -}
35 -
36 -func TestConsistentAccounting(t *testing.T) {
37 - ctx, cancel := context.WithCancel(context.Background())
38 - defer cancel()
39 - sender := newEngine(ctx, "Ernie")
40 - receiver := newEngine(ctx, "Bert")
41 -
42 - // Send messages from Ernie to Bert
43 - for i := 0; i < 1000; i++ {
44 -
45 - m := message.New(false)
46 - content := []string{"this", "is", "message", "i"}
47 - m.AddBlock(blocks.NewBlock([]byte(strings.Join(content, " "))))
48 -
49 - sender.Engine.MessageSent(receiver.Peer, m)
50 - receiver.Engine.MessageReceived(sender.Peer, m)
51 - }
52 -
53 - // Ensure sender records the change
54 - if sender.Engine.numBytesSentTo(receiver.Peer) == 0 {
55 - t.Fatal("Sent bytes were not recorded")
56 - }
57 -
58 - // Ensure sender and receiver have the same values
59 - if sender.Engine.numBytesSentTo(receiver.Peer) != receiver.Engine.numBytesReceivedFrom(sender.Peer) {
60 - t.Fatal("Inconsistent book-keeping. Strategies don't agree")
61 - }
62 -
63 - // Ensure sender didn't record receving anything. And that the receiver
64 - // didn't record sending anything
65 - if receiver.Engine.numBytesSentTo(sender.Peer) != 0 || sender.Engine.numBytesReceivedFrom(receiver.Peer) != 0 {
66 - t.Fatal("Bert didn't send bytes to Ernie")
67 - }
68 -}
69 -
70 -func TestPeerIsAddedToPeersWhenMessageReceivedOrSent(t *testing.T) {
71 -
72 - ctx, cancel := context.WithCancel(context.Background())
73 - defer cancel()
74 - sanfrancisco := newEngine(ctx, "sf")
75 - seattle := newEngine(ctx, "sea")
76 -
77 - m := message.New(true)
78 -
79 - sanfrancisco.Engine.MessageSent(seattle.Peer, m)
80 - seattle.Engine.MessageReceived(sanfrancisco.Peer, m)
81 -
82 - if seattle.Peer == sanfrancisco.Peer {
83 - t.Fatal("Sanity Check: Peers have same Key!")
84 - }
85 -
86 - if !peerIsPartner(seattle.Peer, sanfrancisco.Engine) {
87 - t.Fatal("Peer wasn't added as a Partner")
88 - }
89 -
90 - if !peerIsPartner(sanfrancisco.Peer, seattle.Engine) {
91 - t.Fatal("Peer wasn't added as a Partner")
92 - }
93 -
94 - seattle.Engine.PeerDisconnected(sanfrancisco.Peer)
95 - if peerIsPartner(sanfrancisco.Peer, seattle.Engine) {
96 - t.Fatal("expected peer to be removed")
97 - }
98 -}
99 -
100 -func peerIsPartner(p peer.ID, e *Engine) bool {
101 - for _, partner := range e.Peers() {
102 - if partner == p {
103 - return true
104 - }
105 - }
106 - return false
107 -}
108 -
109 -func TestOutboxClosedWhenEngineClosed(t *testing.T) {
110 - t.SkipNow() // TODO implement *Engine.Close
111 - e := NewEngine(context.Background(), blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())))
112 - var wg sync.WaitGroup
113 - wg.Add(1)
114 - go func() {
115 - for nextEnvelope := range e.Outbox() {
116 - <-nextEnvelope
117 - }
118 - wg.Done()
119 - }()
120 - // e.Close()
121 - wg.Wait()
122 - if _, ok := <-e.Outbox(); ok {
123 - t.Fatal("channel should be closed")
124 - }
125 -}
126 -
127 -func TestPartnerWantsThenCancels(t *testing.T) {
128 - numRounds := 10
129 - if testing.Short() {
130 - numRounds = 1
131 - }
132 - alphabet := strings.Split("abcdefghijklmnopqrstuvwxyz", "")
133 - vowels := strings.Split("aeiou", "")
134 -
135 - type testCase [][]string
136 - testcases := []testCase{
137 - {
138 - alphabet, vowels,
139 - },
140 - {
141 - alphabet, stringsComplement(alphabet, vowels),
142 - },
143 - }
144 -
145 - bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
146 - for _, letter := range alphabet {
147 - block := blocks.NewBlock([]byte(letter))
148 - if err := bs.Put(block); err != nil {
149 - t.Fatal(err)
150 - }
151 - }
152 -
153 - for i := 0; i < numRounds; i++ {
154 - for _, testcase := range testcases {
155 - set := testcase[0]
156 - cancels := testcase[1]
157 - keeps := stringsComplement(set, cancels)
158 -
159 - e := NewEngine(context.Background(), bs)
160 - partner := testutil.RandPeerIDFatal(t)
161 -
162 - partnerWants(e, set, partner)
163 - partnerCancels(e, cancels, partner)
164 - if err := checkHandledInOrder(t, e, keeps); err != nil {
165 - t.Logf("run #%d of %d", i, numRounds)
166 - t.Fatal(err)
167 - }
168 - }
169 - }
170 -}
171 -
172 -func partnerWants(e *Engine, keys []string, partner peer.ID) {
173 - add := message.New(false)
174 - for i, letter := range keys {
175 - block := blocks.NewBlock([]byte(letter))
176 - add.AddEntry(block.Cid(), math.MaxInt32-i)
177 - }
178 - e.MessageReceived(partner, add)
179 -}
180 -
181 -func partnerCancels(e *Engine, keys []string, partner peer.ID) {
182 - cancels := message.New(false)
183 - for _, k := range keys {
184 - block := blocks.NewBlock([]byte(k))
185 - cancels.Cancel(block.Cid())
186 - }
187 - e.MessageReceived(partner, cancels)
188 -}
189 -
190 -func checkHandledInOrder(t *testing.T, e *Engine, keys []string) error {
191 - for _, k := range keys {
192 - next := <-e.Outbox()
193 - envelope := <-next
194 - received := envelope.Block
195 - expected := blocks.NewBlock([]byte(k))
196 - if !received.Cid().Equals(expected.Cid()) {
197 - return errors.New(fmt.Sprintln("received", string(received.RawData()), "expected", string(expected.RawData())))
198 - }
199 - }
200 - return nil
201 -}
202 -
203 -func stringsComplement(set, subset []string) []string {
204 - m := make(map[string]struct{})
205 - for _, letter := range subset {
206 - m[letter] = struct{}{}
207 - }
208 - var complement []string
209 - for _, letter := range set {
210 - if _, exists := m[letter]; !exists {
211 - complement = append(complement, letter)
212 - }
213 - }
214 - return complement
215 -}
exchange/bitswap/decision/ledger.go deleted
-94
@@ -1,94 +0,0 @@
1 -package decision
2 -
3 -import (
4 - "sync"
5 - "time"
6 -
7 - wl "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
8 -
9 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
10 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
11 -)
12 -
13 -func newLedger(p peer.ID) *ledger {
14 - return &ledger{
15 - wantList: wl.New(),
16 - Partner: p,
17 - sentToPeer: make(map[string]time.Time),
18 - }
19 -}
20 -
21 -// ledger stores the data exchange relationship between two peers.
22 -// NOT threadsafe
23 -type ledger struct {
24 - // Partner is the remote Peer.
25 - Partner peer.ID
26 -
27 - // Accounting tracks bytes sent and received.
28 - Accounting debtRatio
29 -
30 - // lastExchange is the time of the last data exchange.
31 - lastExchange time.Time
32 -
33 - // exchangeCount is the number of exchanges with this peer
34 - exchangeCount uint64
35 -
36 - // wantList is a (bounded, small) set of keys that Partner desires.
37 - wantList *wl.Wantlist
38 -
39 - // sentToPeer is a set of keys to ensure we dont send duplicate blocks
40 - // to a given peer
41 - sentToPeer map[string]time.Time
42 -
43 - // ref is the reference count for this ledger, its used to ensure we
44 - // don't drop the reference to this ledger in multi-connection scenarios
45 - ref int
46 -
47 - lk sync.Mutex
48 -}
49 -
50 -type Receipt struct {
51 - Peer string
52 - Value float64
53 - Sent uint64
54 - Recv uint64
55 - Exchanged uint64
56 -}
57 -
58 -type debtRatio struct {
59 - BytesSent uint64
60 - BytesRecv uint64
61 -}
62 -
63 -func (dr *debtRatio) Value() float64 {
64 - return float64(dr.BytesSent) / float64(dr.BytesRecv+1)
65 -}
66 -
67 -func (l *ledger) SentBytes(n int) {
68 - l.exchangeCount++
69 - l.lastExchange = time.Now()
70 - l.Accounting.BytesSent += uint64(n)
71 -}
72 -
73 -func (l *ledger) ReceivedBytes(n int) {
74 - l.exchangeCount++
75 - l.lastExchange = time.Now()
76 - l.Accounting.BytesRecv += uint64(n)
77 -}
78 -
79 -func (l *ledger) Wants(k *cid.Cid, priority int) {
80 - log.Debugf("peer %s wants %s", l.Partner, k)
81 - l.wantList.Add(k, priority)
82 -}
83 -
84 -func (l *ledger) CancelWant(k *cid.Cid) {
85 - l.wantList.Remove(k)
86 -}
87 -
88 -func (l *ledger) WantListContains(k *cid.Cid) (*wl.Entry, bool) {
89 - return l.wantList.Contains(k)
90 -}
91 -
92 -func (l *ledger) ExchangeCount() uint64 {
93 - return l.exchangeCount
94 -}
exchange/bitswap/decision/peer_request_queue.go deleted
-310
@@ -1,310 +0,0 @@
1 -package decision
2 -
3 -import (
4 - "sync"
5 - "time"
6 -
7 - wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
8 -
9 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
10 - pq "gx/ipfs/QmZUbTDJ39JpvtFCSubiWeUTQRvMA1tVE5RZCJrY4oeAsC/go-ipfs-pq"
11 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
12 -)
13 -
14 -type peerRequestQueue interface {
15 - // Pop returns the next peerRequestTask. Returns nil if the peerRequestQueue is empty.
16 - Pop() *peerRequestTask
17 - Push(entry *wantlist.Entry, to peer.ID)
18 - Remove(k *cid.Cid, p peer.ID)
19 -
20 - // NB: cannot expose simply expose taskQueue.Len because trashed elements
21 - // may exist. These trashed elements should not contribute to the count.
22 -}
23 -
24 -func newPRQ() *prq {
25 - return &prq{
26 - taskMap: make(map[string]*peerRequestTask),
27 - partners: make(map[peer.ID]*activePartner),
28 - frozen: make(map[peer.ID]*activePartner),
29 - pQueue: pq.New(partnerCompare),
30 - }
31 -}
32 -
33 -// verify interface implementation
34 -var _ peerRequestQueue = &prq{}
35 -
36 -// TODO: at some point, the strategy needs to plug in here
37 -// to help decide how to sort tasks (on add) and how to select
38 -// tasks (on getnext). For now, we are assuming a dumb/nice strategy.
39 -type prq struct {
40 - lock sync.Mutex
41 - pQueue pq.PQ
42 - taskMap map[string]*peerRequestTask
43 - partners map[peer.ID]*activePartner
44 -
45 - frozen map[peer.ID]*activePartner
46 -}
47 -
48 -// Push currently adds a new peerRequestTask to the end of the list
49 -func (tl *prq) Push(entry *wantlist.Entry, to peer.ID) {
50 - tl.lock.Lock()
51 - defer tl.lock.Unlock()
52 - partner, ok := tl.partners[to]
53 - if !ok {
54 - partner = newActivePartner()
55 - tl.pQueue.Push(partner)
56 - tl.partners[to] = partner
57 - }
58 -
59 - partner.activelk.Lock()
60 - defer partner.activelk.Unlock()
61 - if partner.activeBlocks.Has(entry.Cid) {
62 - return
63 - }
64 -
65 - if task, ok := tl.taskMap[taskKey(to, entry.Cid)]; ok {
66 - task.Entry.Priority = entry.Priority
67 - partner.taskQueue.Update(task.index)
68 - return
69 - }
70 -
71 - task := &peerRequestTask{
72 - Entry: entry,
73 - Target: to,
74 - created: time.Now(),
75 - Done: func() {
76 - tl.lock.Lock()
77 - partner.TaskDone(entry.Cid)
78 - tl.pQueue.Update(partner.Index())
79 - tl.lock.Unlock()
80 - },
81 - }
82 -
83 - partner.taskQueue.Push(task)
84 - tl.taskMap[task.Key()] = task
85 - partner.requests++
86 - tl.pQueue.Update(partner.Index())
87 -}
88 -
89 -// Pop 'pops' the next task to be performed. Returns nil if no task exists.
90 -func (tl *prq) Pop() *peerRequestTask {
91 - tl.lock.Lock()
92 - defer tl.lock.Unlock()
93 - if tl.pQueue.Len() == 0 {
94 - return nil
95 - }
96 - partner := tl.pQueue.Pop().(*activePartner)
97 -
98 - var out *peerRequestTask
99 - for partner.taskQueue.Len() > 0 && partner.freezeVal == 0 {
100 - out = partner.taskQueue.Pop().(*peerRequestTask)
101 - delete(tl.taskMap, out.Key())
102 - if out.trash {
103 - out = nil
104 - continue // discarding tasks that have been removed
105 - }
106 -
107 - partner.StartTask(out.Entry.Cid)
108 - partner.requests--
109 - break // and return |out|
110 - }
111 -
112 - tl.pQueue.Push(partner)
113 - return out
114 -}
115 -
116 -// Remove removes a task from the queue
117 -func (tl *prq) Remove(k *cid.Cid, p peer.ID) {
118 - tl.lock.Lock()
119 - t, ok := tl.taskMap[taskKey(p, k)]
120 - if ok {
121 - // remove the task "lazily"
122 - // simply mark it as trash, so it'll be dropped when popped off the
123 - // queue.
124 - t.trash = true
125 -
126 - // having canceled a block, we now account for that in the given partner
127 - partner := tl.partners[p]
128 - partner.requests--
129 -
130 - // we now also 'freeze' that partner. If they sent us a cancel for a
131 - // block we were about to send them, we should wait a short period of time
132 - // to make sure we receive any other in-flight cancels before sending
133 - // them a block they already potentially have
134 - if partner.freezeVal == 0 {
135 - tl.frozen[p] = partner
136 - }
137 -
138 - partner.freezeVal++
139 - tl.pQueue.Update(partner.index)
140 - }
141 - tl.lock.Unlock()
142 -}
143 -
144 -func (tl *prq) fullThaw() {
145 - tl.lock.Lock()
146 - defer tl.lock.Unlock()
147 -
148 - for id, partner := range tl.frozen {
149 - partner.freezeVal = 0
150 - delete(tl.frozen, id)
151 - tl.pQueue.Update(partner.index)
152 - }
153 -}
154 -
155 -func (tl *prq) thawRound() {
156 - tl.lock.Lock()
157 - defer tl.lock.Unlock()
158 -
159 - for id, partner := range tl.frozen {
160 - partner.freezeVal -= (partner.freezeVal + 1) / 2
161 - if partner.freezeVal <= 0 {
162 - delete(tl.frozen, id)
163 - }
164 - tl.pQueue.Update(partner.index)
165 - }
166 -}
167 -
168 -type peerRequestTask struct {
169 - Entry *wantlist.Entry
170 - Target peer.ID
171 -
172 - // A callback to signal that this task has been completed
173 - Done func()
174 -
175 - // trash in a book-keeping field
176 - trash bool
177 - // created marks the time that the task was added to the queue
178 - created time.Time
179 - index int // book-keeping field used by the pq container
180 -}
181 -
182 -// Key uniquely identifies a task.
183 -func (t *peerRequestTask) Key() string {
184 - return taskKey(t.Target, t.Entry.Cid)
185 -}
186 -
187 -// Index implements pq.Elem
188 -func (t *peerRequestTask) Index() int {
189 - return t.index
190 -}
191 -
192 -// SetIndex implements pq.Elem
193 -func (t *peerRequestTask) SetIndex(i int) {
194 - t.index = i
195 -}
196 -
197 -// taskKey returns a key that uniquely identifies a task.
198 -func taskKey(p peer.ID, k *cid.Cid) string {
199 - return string(p) + k.KeyString()
200 -}
201 -
202 -// FIFO is a basic task comparator that returns tasks in the order created.
203 -var FIFO = func(a, b *peerRequestTask) bool {
204 - return a.created.Before(b.created)
205 -}
206 -
207 -// V1 respects the target peer's wantlist priority. For tasks involving
208 -// different peers, the oldest task is prioritized.
209 -var V1 = func(a, b *peerRequestTask) bool {
210 - if a.Target == b.Target {
211 - return a.Entry.Priority > b.Entry.Priority
212 - }
213 - return FIFO(a, b)
214 -}
215 -
216 -func wrapCmp(f func(a, b *peerRequestTask) bool) func(a, b pq.Elem) bool {
217 - return func(a, b pq.Elem) bool {
218 - return f(a.(*peerRequestTask), b.(*peerRequestTask))
219 - }
220 -}
221 -
222 -type activePartner struct {
223 -
224 - // Active is the number of blocks this peer is currently being sent
225 - // active must be locked around as it will be updated externally
226 - activelk sync.Mutex
227 - active int
228 -
229 - activeBlocks *cid.Set
230 -
231 - // requests is the number of blocks this peer is currently requesting
232 - // request need not be locked around as it will only be modified under
233 - // the peerRequestQueue's locks
234 - requests int
235 -
236 - // for the PQ interface
237 - index int
238 -
239 - freezeVal int
240 -
241 - // priority queue of tasks belonging to this peer
242 - taskQueue pq.PQ
243 -}
244 -
245 -func newActivePartner() *activePartner {
246 - return &activePartner{
247 - taskQueue: pq.New(wrapCmp(V1)),
248 - activeBlocks: cid.NewSet(),
249 - }
250 -}
251 -
252 -// partnerCompare implements pq.ElemComparator
253 -// returns true if peer 'a' has higher priority than peer 'b'
254 -func partnerCompare(a, b pq.Elem) bool {
255 - pa := a.(*activePartner)
256 - pb := b.(*activePartner)
257 -
258 - // having no blocks in their wantlist means lowest priority
259 - // having both of these checks ensures stability of the sort
260 - if pa.requests == 0 {
261 - return false
262 - }
263 - if pb.requests == 0 {
264 - return true
265 - }
266 -
267 - if pa.freezeVal > pb.freezeVal {
268 - return false
269 - }
270 - if pa.freezeVal < pb.freezeVal {
271 - return true
272 - }
273 -
274 - if pa.active == pb.active {
275 - // sorting by taskQueue.Len() aids in cleaning out trash entries faster
276 - // if we sorted instead by requests, one peer could potentially build up
277 - // a huge number of cancelled entries in the queue resulting in a memory leak
278 - return pa.taskQueue.Len() > pb.taskQueue.Len()
279 - }
280 - return pa.active < pb.active
281 -}
282 -
283 -// StartTask signals that a task was started for this partner
284 -func (p *activePartner) StartTask(k *cid.Cid) {
285 - p.activelk.Lock()
286 - p.activeBlocks.Add(k)
287 - p.active++
288 - p.activelk.Unlock()
289 -}
290 -
291 -// TaskDone signals that a task was completed for this partner
292 -func (p *activePartner) TaskDone(k *cid.Cid) {
293 - p.activelk.Lock()
294 - p.activeBlocks.Remove(k)
295 - p.active--
296 - if p.active < 0 {
297 - panic("more tasks finished than started!")
298 - }
299 - p.activelk.Unlock()
300 -}
301 -
302 -// Index implements pq.Elem
303 -func (p *activePartner) Index() int {
304 - return p.index
305 -}
306 -
307 -// SetIndex implements pq.Elem
308 -func (p *activePartner) SetIndex(i int) {
309 - p.index = i
310 -}
exchange/bitswap/decision/peer_request_queue_test.go deleted
-128
@@ -1,128 +0,0 @@
1 -package decision
2 -
3 -import (
4 - "fmt"
5 - "math"
6 - "math/rand"
7 - "sort"
8 - "strings"
9 - "testing"
10 -
11 - "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
12 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
13 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
14 - "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
15 -)
16 -
17 -func TestPushPop(t *testing.T) {
18 - prq := newPRQ()
19 - partner := testutil.RandPeerIDFatal(t)
20 - alphabet := strings.Split("abcdefghijklmnopqrstuvwxyz", "")
21 - vowels := strings.Split("aeiou", "")
22 - consonants := func() []string {
23 - var out []string
24 - for _, letter := range alphabet {
25 - skip := false
26 - for _, vowel := range vowels {
27 - if letter == vowel {
28 - skip = true
29 - }
30 - }
31 - if !skip {
32 - out = append(out, letter)
33 - }
34 - }
35 - return out
36 - }()
37 - sort.Strings(alphabet)
38 - sort.Strings(vowels)
39 - sort.Strings(consonants)
40 -
41 - // add a bunch of blocks. cancel some. drain the queue. the queue should only have the kept entries
42 -
43 - for _, index := range rand.Perm(len(alphabet)) { // add blocks for all letters
44 - letter := alphabet[index]
45 - t.Log(partner.String())
46 -
47 - c := cid.NewCidV0(u.Hash([]byte(letter)))
48 - prq.Push(&wantlist.Entry{Cid: c, Priority: math.MaxInt32 - index}, partner)
49 - }
50 - for _, consonant := range consonants {
51 - c := cid.NewCidV0(u.Hash([]byte(consonant)))
52 - prq.Remove(c, partner)
53 - }
54 -
55 - prq.fullThaw()
56 -
57 - var out []string
58 - for {
59 - received := prq.Pop()
60 - if received == nil {
61 - break
62 - }
63 -
64 - out = append(out, received.Entry.Cid.String())
65 - }
66 -
67 - // Entries popped should already be in correct order
68 - for i, expected := range vowels {
69 - exp := cid.NewCidV0(u.Hash([]byte(expected))).String()
70 - if out[i] != exp {
71 - t.Fatal("received", out[i], "expected", expected)
72 - }
73 - }
74 -}
75 -
76 -// This test checks that peers wont starve out other peers
77 -func TestPeerRepeats(t *testing.T) {
78 - prq := newPRQ()
79 - a := testutil.RandPeerIDFatal(t)
80 - b := testutil.RandPeerIDFatal(t)
81 - c := testutil.RandPeerIDFatal(t)
82 - d := testutil.RandPeerIDFatal(t)
83 -
84 - // Have each push some blocks
85 -
86 - for i := 0; i < 5; i++ {
87 - elcid := cid.NewCidV0(u.Hash([]byte(fmt.Sprint(i))))
88 - prq.Push(&wantlist.Entry{Cid: elcid}, a)
89 - prq.Push(&wantlist.Entry{Cid: elcid}, b)
90 - prq.Push(&wantlist.Entry{Cid: elcid}, c)
91 - prq.Push(&wantlist.Entry{Cid: elcid}, d)
92 - }
93 -
94 - // now, pop off four entries, there should be one from each
95 - var targets []string
96 - var tasks []*peerRequestTask
97 - for i := 0; i < 4; i++ {
98 - t := prq.Pop()
99 - targets = append(targets, t.Target.Pretty())
100 - tasks = append(tasks, t)
101 - }
102 -
103 - expected := []string{a.Pretty(), b.Pretty(), c.Pretty(), d.Pretty()}
104 - sort.Strings(expected)
105 - sort.Strings(targets)
106 -
107 - t.Log(targets)
108 - t.Log(expected)
109 - for i, s := range targets {
110 - if expected[i] != s {
111 - t.Fatal("unexpected peer", s, expected[i])
112 - }
113 - }
114 -
115 - // Now, if one of the tasks gets finished, the next task off the queue should
116 - // be for the same peer
117 - for blockI := 0; blockI < 4; blockI++ {
118 - for i := 0; i < 4; i++ {
119 - // its okay to mark the same task done multiple times here (JUST FOR TESTING)
120 - tasks[i].Done()
121 -
122 - ntask := prq.Pop()
123 - if ntask.Target != tasks[i].Target {
124 - t.Fatal("Expected task from peer with lowest active count")
125 - }
126 - }
127 - }
128 -}
exchange/bitswap/get.go deleted
-100
@@ -1,100 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "errors"
6 -
7 - notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
8 -
9 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
10 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
11 - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
12 -)
13 -
14 -type getBlocksFunc func(context.Context, []*cid.Cid) (<-chan blocks.Block, error)
15 -
16 -func getBlock(p context.Context, k *cid.Cid, gb getBlocksFunc) (blocks.Block, error) {
17 - if k == nil {
18 - log.Error("nil cid in GetBlock")
19 - return nil, blockstore.ErrNotFound
20 - }
21 -
22 - // Any async work initiated by this function must end when this function
23 - // returns. To ensure this, derive a new context. Note that it is okay to
24 - // listen on parent in this scope, but NOT okay to pass |parent| to
25 - // functions called by this one. Otherwise those functions won't return
26 - // when this context's cancel func is executed. This is difficult to
27 - // enforce. May this comment keep you safe.
28 - ctx, cancel := context.WithCancel(p)
29 - defer cancel()
30 -
31 - promise, err := gb(ctx, []*cid.Cid{k})
32 - if err != nil {
33 - return nil, err
34 - }
35 -
36 - select {
37 - case block, ok := <-promise:
38 - if !ok {
39 - select {
40 - case <-ctx.Done():
41 - return nil, ctx.Err()
42 - default:
43 - return nil, errors.New("promise channel was closed")
44 - }
45 - }
46 - return block, nil
47 - case <-p.Done():
48 - return nil, p.Err()
49 - }
50 -}
51 -
52 -type wantFunc func(context.Context, []*cid.Cid)
53 -
54 -func getBlocksImpl(ctx context.Context, keys []*cid.Cid, notif notifications.PubSub, want wantFunc, cwants func([]*cid.Cid)) (<-chan blocks.Block, error) {
55 - if len(keys) == 0 {
56 - out := make(chan blocks.Block)
57 - close(out)
58 - return out, nil
59 - }
60 -
61 - remaining := cid.NewSet()
62 - promise := notif.Subscribe(ctx, keys...)
63 - for _, k := range keys {
64 - log.Event(ctx, "Bitswap.GetBlockRequest.Start", k)
65 - remaining.Add(k)
66 - }
67 -
68 - want(ctx, keys)
69 -
70 - out := make(chan blocks.Block)
71 - go handleIncoming(ctx, remaining, promise, out, cwants)
72 - return out, nil
73 -}
74 -
75 -func handleIncoming(ctx context.Context, remaining *cid.Set, in <-chan blocks.Block, out chan blocks.Block, cfun func([]*cid.Cid)) {
76 - ctx, cancel := context.WithCancel(ctx)
77 - defer func() {
78 - cancel()
79 - close(out)
80 - // can't just defer this call on its own, arguments are resolved *when* the defer is created
81 - cfun(remaining.Keys())
82 - }()
83 - for {
84 - select {
85 - case blk, ok := <-in:
86 - if !ok {
87 - return
88 - }
89 -
90 - remaining.Remove(blk.Cid())
91 - select {
92 - case out <- blk:
93 - case <-ctx.Done():
94 - return
95 - }
96 - case <-ctx.Done():
97 - return
98 - }
99 - }
100 -}
exchange/bitswap/message/message.go deleted
-249
@@ -1,249 +0,0 @@
1 -package message
2 -
3 -import (
4 - "fmt"
5 - "io"
6 -
7 - pb "github.com/ipfs/go-ipfs/exchange/bitswap/message/pb"
8 - wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
9 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
10 -
11 - inet "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net"
12 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
13 - ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
14 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
15 -)
16 -
17 -// TODO move message.go into the bitswap package
18 -// TODO move bs/msg/internal/pb to bs/internal/pb and rename pb package to bitswap_pb
19 -
20 -type BitSwapMessage interface {
21 - // Wantlist returns a slice of unique keys that represent data wanted by
22 - // the sender.
23 - Wantlist() []Entry
24 -
25 - // Blocks returns a slice of unique blocks
26 - Blocks() []blocks.Block
27 -
28 - // AddEntry adds an entry to the Wantlist.
29 - AddEntry(key *cid.Cid, priority int)
30 -
31 - Cancel(key *cid.Cid)
32 -
33 - Empty() bool
34 -
35 - // A full wantlist is an authoritative copy, a 'non-full' wantlist is a patch-set
36 - Full() bool
37 -
38 - AddBlock(blocks.Block)
39 - Exportable
40 -
41 - Loggable() map[string]interface{}
42 -}
43 -
44 -type Exportable interface {
45 - ToProtoV0() *pb.Message
46 - ToProtoV1() *pb.Message
47 - ToNetV0(w io.Writer) error
48 - ToNetV1(w io.Writer) error
49 -}
50 -
51 -type impl struct {
52 - full bool
53 - wantlist map[string]*Entry
54 - blocks map[string]blocks.Block
55 -}
56 -
57 -func New(full bool) BitSwapMessage {
58 - return newMsg(full)
59 -}
60 -
61 -func newMsg(full bool) *impl {
62 - return &impl{
63 - blocks: make(map[string]blocks.Block),
64 - wantlist: make(map[string]*Entry),
65 - full: full,
66 - }
67 -}
68 -
69 -type Entry struct {
70 - *wantlist.Entry
71 - Cancel bool
72 -}
73 -
74 -func newMessageFromProto(pbm pb.Message) (BitSwapMessage, error) {
75 - m := newMsg(pbm.GetWantlist().GetFull())
76 - for _, e := range pbm.GetWantlist().GetEntries() {
77 - c, err := cid.Cast([]byte(e.GetBlock()))
78 - if err != nil {
79 - return nil, fmt.Errorf("incorrectly formatted cid in wantlist: %s", err)
80 - }
81 - m.addEntry(c, int(e.GetPriority()), e.GetCancel())
82 - }
83 -
84 - // deprecated
85 - for _, d := range pbm.GetBlocks() {
86 - // CIDv0, sha256, protobuf only
87 - b := blocks.NewBlock(d)
88 - m.AddBlock(b)
89 - }
90 - //
91 -
92 - for _, b := range pbm.GetPayload() {
93 - pref, err := cid.PrefixFromBytes(b.GetPrefix())
94 - if err != nil {
95 - return nil, err
96 - }
97 -
98 - c, err := pref.Sum(b.GetData())
99 - if err != nil {
100 - return nil, err
101 - }
102 -
103 - blk, err := blocks.NewBlockWithCid(b.GetData(), c)
104 - if err != nil {
105 - return nil, err
106 - }
107 -
108 - m.AddBlock(blk)
109 - }
110 -
111 - return m, nil
112 -}
113 -
114 -func (m *impl) Full() bool {
115 - return m.full
116 -}
117 -
118 -func (m *impl) Empty() bool {
119 - return len(m.blocks) == 0 && len(m.wantlist) == 0
120 -}
121 -
122 -func (m *impl) Wantlist() []Entry {
123 - out := make([]Entry, 0, len(m.wantlist))
124 - for _, e := range m.wantlist {
125 - out = append(out, *e)
126 - }
127 - return out
128 -}
129 -
130 -func (m *impl) Blocks() []blocks.Block {
131 - bs := make([]blocks.Block, 0, len(m.blocks))
132 - for _, block := range m.blocks {
133 - bs = append(bs, block)
134 - }
135 - return bs
136 -}
137 -
138 -func (m *impl) Cancel(k *cid.Cid) {
139 - delete(m.wantlist, k.KeyString())
140 - m.addEntry(k, 0, true)
141 -}
142 -
143 -func (m *impl) AddEntry(k *cid.Cid, priority int) {
144 - m.addEntry(k, priority, false)
145 -}
146 -
147 -func (m *impl) addEntry(c *cid.Cid, priority int, cancel bool) {
148 - k := c.KeyString()
149 - e, exists := m.wantlist[k]
150 - if exists {
151 - e.Priority = priority
152 - e.Cancel = cancel
153 - } else {
154 - m.wantlist[k] = &Entry{
155 - Entry: &wantlist.Entry{
156 - Cid: c,
157 - Priority: priority,
158 - },
159 - Cancel: cancel,
160 - }
161 - }
162 -}
163 -
164 -func (m *impl) AddBlock(b blocks.Block) {
165 - m.blocks[b.Cid().KeyString()] = b
166 -}
167 -
168 -func FromNet(r io.Reader) (BitSwapMessage, error) {
169 - pbr := ggio.NewDelimitedReader(r, inet.MessageSizeMax)
170 - return FromPBReader(pbr)
171 -}
172 -
173 -func FromPBReader(pbr ggio.Reader) (BitSwapMessage, error) {
174 - pb := new(pb.Message)
175 - if err := pbr.ReadMsg(pb); err != nil {
176 - return nil, err
177 - }
178 -
179 - return newMessageFromProto(*pb)
180 -}
181 -
182 -func (m *impl) ToProtoV0() *pb.Message {
183 - pbm := new(pb.Message)
184 - pbm.Wantlist = new(pb.Message_Wantlist)
185 - pbm.Wantlist.Entries = make([]*pb.Message_Wantlist_Entry, 0, len(m.wantlist))
186 - for _, e := range m.wantlist {
187 - pbm.Wantlist.Entries = append(pbm.Wantlist.Entries, &pb.Message_Wantlist_Entry{
188 - Block: proto.String(e.Cid.KeyString()),
189 - Priority: proto.Int32(int32(e.Priority)),
190 - Cancel: proto.Bool(e.Cancel),
191 - })
192 - }
193 - pbm.Wantlist.Full = proto.Bool(m.full)
194 -
195 - blocks := m.Blocks()
196 - pbm.Blocks = make([][]byte, 0, len(blocks))
197 - for _, b := range blocks {
198 - pbm.Blocks = append(pbm.Blocks, b.RawData())
199 - }
200 - return pbm
201 -}
202 -
203 -func (m *impl) ToProtoV1() *pb.Message {
204 - pbm := new(pb.Message)
205 - pbm.Wantlist = new(pb.Message_Wantlist)
206 - pbm.Wantlist.Entries = make([]*pb.Message_Wantlist_Entry, 0, len(m.wantlist))
207 - for _, e := range m.wantlist {
208 - pbm.Wantlist.Entries = append(pbm.Wantlist.Entries, &pb.Message_Wantlist_Entry{
209 - Block: proto.String(e.Cid.KeyString()),
210 - Priority: proto.Int32(int32(e.Priority)),
211 - Cancel: proto.Bool(e.Cancel),
212 - })
213 - }
214 - pbm.Wantlist.Full = proto.Bool(m.full)
215 -
216 - blocks := m.Blocks()
217 - pbm.Payload = make([]*pb.Message_Block, 0, len(blocks))
218 - for _, b := range blocks {
219 - blk := &pb.Message_Block{
220 - Data: b.RawData(),
221 - Prefix: b.Cid().Prefix().Bytes(),
222 - }
223 - pbm.Payload = append(pbm.Payload, blk)
224 - }
225 - return pbm
226 -}
227 -
228 -func (m *impl) ToNetV0(w io.Writer) error {
229 - pbw := ggio.NewDelimitedWriter(w)
230 -
231 - return pbw.WriteMsg(m.ToProtoV0())
232 -}
233 -
234 -func (m *impl) ToNetV1(w io.Writer) error {
235 - pbw := ggio.NewDelimitedWriter(w)
236 -
237 - return pbw.WriteMsg(m.ToProtoV1())
238 -}
239 -
240 -func (m *impl) Loggable() map[string]interface{} {
241 - blocks := make([]string, 0, len(m.blocks))
242 - for _, v := range m.blocks {
243 - blocks = append(blocks, v.Cid().String())
244 - }
245 - return map[string]interface{}{
246 - "blocks": blocks,
247 - "wants": m.Wantlist(),
248 - }
249 -}
exchange/bitswap/message/message_test.go deleted
-200
@@ -1,200 +0,0 @@
1 -package message
2 -
3 -import (
4 - "bytes"
5 - "testing"
6 -
7 - pb "github.com/ipfs/go-ipfs/exchange/bitswap/message/pb"
8 -
9 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
10 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
11 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
12 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
13 -)
14 -
15 -func mkFakeCid(s string) *cid.Cid {
16 - return cid.NewCidV0(u.Hash([]byte(s)))
17 -}
18 -
19 -func TestAppendWanted(t *testing.T) {
20 - str := mkFakeCid("foo")
21 - m := New(true)
22 - m.AddEntry(str, 1)
23 -
24 - if !wantlistContains(m.ToProtoV0().GetWantlist(), str) {
25 - t.Fail()
26 - }
27 -}
28 -
29 -func TestNewMessageFromProto(t *testing.T) {
30 - str := mkFakeCid("a_key")
31 - protoMessage := new(pb.Message)
32 - protoMessage.Wantlist = new(pb.Message_Wantlist)
33 - protoMessage.Wantlist.Entries = []*pb.Message_Wantlist_Entry{
34 - {Block: proto.String(str.KeyString())},
35 - }
36 - if !wantlistContains(protoMessage.Wantlist, str) {
37 - t.Fail()
38 - }
39 - m, err := newMessageFromProto(*protoMessage)
40 - if err != nil {
41 - t.Fatal(err)
42 - }
43 -
44 - if !wantlistContains(m.ToProtoV0().GetWantlist(), str) {
45 - t.Fail()
46 - }
47 -}
48 -
49 -func TestAppendBlock(t *testing.T) {
50 -
51 - strs := make([]string, 2)
52 - strs = append(strs, "Celeritas")
53 - strs = append(strs, "Incendia")
54 -
55 - m := New(true)
56 - for _, str := range strs {
57 - block := blocks.NewBlock([]byte(str))
58 - m.AddBlock(block)
59 - }
60 -
61 - // assert strings are in proto message
62 - for _, blockbytes := range m.ToProtoV0().GetBlocks() {
63 - s := bytes.NewBuffer(blockbytes).String()
64 - if !contains(strs, s) {
65 - t.Fail()
66 - }
67 - }
68 -}
69 -
70 -func TestWantlist(t *testing.T) {
71 - keystrs := []*cid.Cid{mkFakeCid("foo"), mkFakeCid("bar"), mkFakeCid("baz"), mkFakeCid("bat")}
72 - m := New(true)
73 - for _, s := range keystrs {
74 - m.AddEntry(s, 1)
75 - }
76 - exported := m.Wantlist()
77 -
78 - for _, k := range exported {
79 - present := false
80 - for _, s := range keystrs {
81 -
82 - if s.Equals(k.Cid) {
83 - present = true
84 - }
85 - }
86 - if !present {
87 - t.Logf("%v isn't in original list", k.Cid)
88 - t.Fail()
89 - }
90 - }
91 -}
92 -
93 -func TestCopyProtoByValue(t *testing.T) {
94 - str := mkFakeCid("foo")
95 - m := New(true)
96 - protoBeforeAppend := m.ToProtoV0()
97 - m.AddEntry(str, 1)
98 - if wantlistContains(protoBeforeAppend.GetWantlist(), str) {
99 - t.Fail()
100 - }
101 -}
102 -
103 -func TestToNetFromNetPreservesWantList(t *testing.T) {
104 - original := New(true)
105 - original.AddEntry(mkFakeCid("M"), 1)
106 - original.AddEntry(mkFakeCid("B"), 1)
107 - original.AddEntry(mkFakeCid("D"), 1)
108 - original.AddEntry(mkFakeCid("T"), 1)
109 - original.AddEntry(mkFakeCid("F"), 1)
110 -
111 - buf := new(bytes.Buffer)
112 - if err := original.ToNetV1(buf); err != nil {
113 - t.Fatal(err)
114 - }
115 -
116 - copied, err := FromNet(buf)
117 - if err != nil {
118 - t.Fatal(err)
119 - }
120 -
121 - if !copied.Full() {
122 - t.Fatal("fullness attribute got dropped on marshal")
123 - }
124 -
125 - keys := make(map[string]bool)
126 - for _, k := range copied.Wantlist() {
127 - keys[k.Cid.KeyString()] = true
128 - }
129 -
130 - for _, k := range original.Wantlist() {
131 - if _, ok := keys[k.Cid.KeyString()]; !ok {
132 - t.Fatalf("Key Missing: \"%v\"", k)
133 - }
134 - }
135 -}
136 -
137 -func TestToAndFromNetMessage(t *testing.T) {
138 -
139 - original := New(true)
140 - original.AddBlock(blocks.NewBlock([]byte("W")))
141 - original.AddBlock(blocks.NewBlock([]byte("E")))
142 - original.AddBlock(blocks.NewBlock([]byte("F")))
143 - original.AddBlock(blocks.NewBlock([]byte("M")))
144 -
145 - buf := new(bytes.Buffer)
146 - if err := original.ToNetV1(buf); err != nil {
147 - t.Fatal(err)
148 - }
149 -
150 - m2, err := FromNet(buf)
151 - if err != nil {
152 - t.Fatal(err)
153 - }
154 -
155 - keys := make(map[string]bool)
156 - for _, b := range m2.Blocks() {
157 - keys[b.Cid().KeyString()] = true
158 - }
159 -
160 - for _, b := range original.Blocks() {
161 - if _, ok := keys[b.Cid().KeyString()]; !ok {
162 - t.Fail()
163 - }
164 - }
165 -}
166 -
167 -func wantlistContains(wantlist *pb.Message_Wantlist, c *cid.Cid) bool {
168 - for _, e := range wantlist.GetEntries() {
169 - if e.GetBlock() == c.KeyString() {
170 - return true
171 - }
172 - }
173 - return false
174 -}
175 -
176 -func contains(strs []string, x string) bool {
177 - for _, s := range strs {
178 - if s == x {
179 - return true
180 - }
181 - }
182 - return false
183 -}
184 -
185 -func TestDuplicates(t *testing.T) {
186 - b := blocks.NewBlock([]byte("foo"))
187 - msg := New(true)
188 -
189 - msg.AddEntry(b.Cid(), 1)
190 - msg.AddEntry(b.Cid(), 1)
191 - if len(msg.Wantlist()) != 1 {
192 - t.Fatal("Duplicate in BitSwapMessage")
193 - }
194 -
195 - msg.AddBlock(b)
196 - msg.AddBlock(b)
197 - if len(msg.Blocks()) != 1 {
198 - t.Fatal("Duplicate in BitSwapMessage")
199 - }
200 -}
exchange/bitswap/message/pb/Makefile deleted
-8
@@ -1,8 +0,0 @@
1 -# TODO(brian): add proto tasks
2 -all: message.pb.go
3 -
4 -message.pb.go: message.proto
5 - protoc --gogo_out=. --proto_path=../../../../../:/usr/local/opt/protobuf/include:. $<
6 -
7 -clean:
8 - rm message.pb.go
exchange/bitswap/message/pb/Rules.mk deleted
-8
@@ -1,8 +0,0 @@
1 -include mk/header.mk
2 -
3 -PB_$(d) = $(wildcard $(d)/*.proto)
4 -TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
5 -
6 -#DEPS_GO += $(TGTS_$(d))
7 -
8 -include mk/footer.mk
exchange/bitswap/message/pb/message.pb.go deleted
-142
@@ -1,142 +0,0 @@
1 -// Code generated by protoc-gen-gogo.
2 -// source: message.proto
3 -// DO NOT EDIT!
4 -
5 -/*
6 -Package bitswap_message_pb is a generated protocol buffer package.
7 -
8 -It is generated from these files:
9 - message.proto
10 -
11 -It has these top-level messages:
12 - Message
13 -*/
14 -package bitswap_message_pb
15 -
16 -import proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
17 -import fmt "fmt"
18 -import math "math"
19 -
20 -// Reference imports to suppress errors if they are not otherwise used.
21 -var _ = proto.Marshal
22 -var _ = fmt.Errorf
23 -var _ = math.Inf
24 -
25 -type Message struct {
26 - Wantlist *Message_Wantlist `protobuf:"bytes,1,opt,name=wantlist" json:"wantlist,omitempty"`
27 - Blocks [][]byte `protobuf:"bytes,2,rep,name=blocks" json:"blocks,omitempty"`
28 - Payload []*Message_Block `protobuf:"bytes,3,rep,name=payload" json:"payload,omitempty"`
29 - XXX_unrecognized []byte `json:"-"`
30 -}
31 -
32 -func (m *Message) Reset() { *m = Message{} }
33 -func (m *Message) String() string { return proto.CompactTextString(m) }
34 -func (*Message) ProtoMessage() {}
35 -
36 -func (m *Message) GetWantlist() *Message_Wantlist {
37 - if m != nil {
38 - return m.Wantlist
39 - }
40 - return nil
41 -}
42 -
43 -func (m *Message) GetBlocks() [][]byte {
44 - if m != nil {
45 - return m.Blocks
46 - }
47 - return nil
48 -}
49 -
50 -func (m *Message) GetPayload() []*Message_Block {
51 - if m != nil {
52 - return m.Payload
53 - }
54 - return nil
55 -}
56 -
57 -type Message_Wantlist struct {
58 - Entries []*Message_Wantlist_Entry `protobuf:"bytes,1,rep,name=entries" json:"entries,omitempty"`
59 - Full *bool `protobuf:"varint,2,opt,name=full" json:"full,omitempty"`
60 - XXX_unrecognized []byte `json:"-"`
61 -}
62 -
63 -func (m *Message_Wantlist) Reset() { *m = Message_Wantlist{} }
64 -func (m *Message_Wantlist) String() string { return proto.CompactTextString(m) }
65 -func (*Message_Wantlist) ProtoMessage() {}
66 -
67 -func (m *Message_Wantlist) GetEntries() []*Message_Wantlist_Entry {
68 - if m != nil {
69 - return m.Entries
70 - }
71 - return nil
72 -}
73 -
74 -func (m *Message_Wantlist) GetFull() bool {
75 - if m != nil && m.Full != nil {
76 - return *m.Full
77 - }
78 - return false
79 -}
80 -
81 -type Message_Wantlist_Entry struct {
82 - Block *string `protobuf:"bytes,1,opt,name=block" json:"block,omitempty"`
83 - Priority *int32 `protobuf:"varint,2,opt,name=priority" json:"priority,omitempty"`
84 - Cancel *bool `protobuf:"varint,3,opt,name=cancel" json:"cancel,omitempty"`
85 - XXX_unrecognized []byte `json:"-"`
86 -}
87 -
88 -func (m *Message_Wantlist_Entry) Reset() { *m = Message_Wantlist_Entry{} }
89 -func (m *Message_Wantlist_Entry) String() string { return proto.CompactTextString(m) }
90 -func (*Message_Wantlist_Entry) ProtoMessage() {}
91 -
92 -func (m *Message_Wantlist_Entry) GetBlock() string {
93 - if m != nil && m.Block != nil {
94 - return *m.Block
95 - }
96 - return ""
97 -}
98 -
99 -func (m *Message_Wantlist_Entry) GetPriority() int32 {
100 - if m != nil && m.Priority != nil {
101 - return *m.Priority
102 - }
103 - return 0
104 -}
105 -
106 -func (m *Message_Wantlist_Entry) GetCancel() bool {
107 - if m != nil && m.Cancel != nil {
108 - return *m.Cancel
109 - }
110 - return false
111 -}
112 -
113 -type Message_Block struct {
114 - Prefix []byte `protobuf:"bytes,1,opt,name=prefix" json:"prefix,omitempty"`
115 - Data []byte `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"`
116 - XXX_unrecognized []byte `json:"-"`
117 -}
118 -
119 -func (m *Message_Block) Reset() { *m = Message_Block{} }
120 -func (m *Message_Block) String() string { return proto.CompactTextString(m) }
121 -func (*Message_Block) ProtoMessage() {}
122 -
123 -func (m *Message_Block) GetPrefix() []byte {
124 - if m != nil {
125 - return m.Prefix
126 - }
127 - return nil
128 -}
129 -
130 -func (m *Message_Block) GetData() []byte {
131 - if m != nil {
132 - return m.Data
133 - }
134 - return nil
135 -}
136 -
137 -func init() {
138 - proto.RegisterType((*Message)(nil), "bitswap.message.pb.Message")
139 - proto.RegisterType((*Message_Wantlist)(nil), "bitswap.message.pb.Message.Wantlist")
140 - proto.RegisterType((*Message_Wantlist_Entry)(nil), "bitswap.message.pb.Message.Wantlist.Entry")
141 - proto.RegisterType((*Message_Block)(nil), "bitswap.message.pb.Message.Block")
142 -}
exchange/bitswap/message/pb/message.proto deleted
-25
@@ -1,25 +0,0 @@
1 -package bitswap.message.pb;
2 -
3 -message Message {
4 -
5 - message Wantlist {
6 -
7 - message Entry {
8 - optional string block = 1; // the block cid (cidV0 in bitswap 1.0.0, cidV1 in bitswap 1.1.0)
9 - optional int32 priority = 2; // the priority (normalized). default to 1
10 - optional bool cancel = 3; // whether this revokes an entry
11 - }
12 -
13 - repeated Entry entries = 1; // a list of wantlist entries
14 - optional bool full = 2; // whether this is the full wantlist. default to false
15 - }
16 -
17 - message Block {
18 - optional bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length)
19 - optional bytes data = 2;
20 - }
21 -
22 - optional Wantlist wantlist = 1;
23 - repeated bytes blocks = 2; // used to send Blocks in bitswap 1.0.0
24 - repeated Block payload = 3; // used to send Blocks in bitswap 1.1.0
25 -}
exchange/bitswap/network/interface.go deleted
-70
@@ -1,70 +0,0 @@
1 -package network
2 -
3 -import (
4 - "context"
5 -
6 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
7 -
8 - ifconnmgr "gx/ipfs/QmXuucFcuvAWYAJfhHV2h4BYreHEAsLSsiquosiXeuduTN/go-libp2p-interface-connmgr"
9 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
10 - protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
11 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
12 -)
13 -
14 -var (
15 - // These two are equivalent, legacy
16 - ProtocolBitswapOne protocol.ID = "/ipfs/bitswap/1.0.0"
17 - ProtocolBitswapNoVers protocol.ID = "/ipfs/bitswap"
18 -
19 - ProtocolBitswap protocol.ID = "/ipfs/bitswap/1.1.0"
20 -)
21 -
22 -// BitSwapNetwork provides network connectivity for BitSwap sessions
23 -type BitSwapNetwork interface {
24 -
25 - // SendMessage sends a BitSwap message to a peer.
26 - SendMessage(
27 - context.Context,
28 - peer.ID,
29 - bsmsg.BitSwapMessage) error
30 -
31 - // SetDelegate registers the Reciver to handle messages received from the
32 - // network.
33 - SetDelegate(Receiver)
34 -
35 - ConnectTo(context.Context, peer.ID) error
36 -
37 - NewMessageSender(context.Context, peer.ID) (MessageSender, error)
38 -
39 - ConnectionManager() ifconnmgr.ConnManager
40 -
41 - Routing
42 -}
43 -
44 -type MessageSender interface {
45 - SendMsg(context.Context, bsmsg.BitSwapMessage) error
46 - Close() error
47 - Reset() error
48 -}
49 -
50 -// Implement Receiver to receive messages from the BitSwapNetwork
51 -type Receiver interface {
52 - ReceiveMessage(
53 - ctx context.Context,
54 - sender peer.ID,
55 - incoming bsmsg.BitSwapMessage)
56 -
57 - ReceiveError(error)
58 -
59 - // Connected/Disconnected warns bitswap about peer connections
60 - PeerConnected(peer.ID)
61 - PeerDisconnected(peer.ID)
62 -}
63 -
64 -type Routing interface {
65 - // FindProvidersAsync returns a channel of providers for the given key
66 - FindProvidersAsync(context.Context, *cid.Cid, int) <-chan peer.ID
67 -
68 - // Provide provides the key to the network
69 - Provide(context.Context, *cid.Cid) error
70 -}
exchange/bitswap/network/ipfs_impl.go deleted
-230
@@ -1,230 +0,0 @@
1 -package network
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "io"
7 - "time"
8 -
9 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
10 -
11 - inet "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net"
12 - ifconnmgr "gx/ipfs/QmXuucFcuvAWYAJfhHV2h4BYreHEAsLSsiquosiXeuduTN/go-libp2p-interface-connmgr"
13 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
14 - ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr"
15 - routing "gx/ipfs/QmZ383TySJVeZWzGnWui6pRcKyYZk9VkKTuW7tmKRWk5au/go-libp2p-routing"
16 - ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
17 - pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore"
18 - host "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host"
19 - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
20 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
21 -)
22 -
23 -var log = logging.Logger("bitswap_network")
24 -
25 -var sendMessageTimeout = time.Minute * 10
26 -
27 -// NewFromIpfsHost returns a BitSwapNetwork supported by underlying IPFS host
28 -func NewFromIpfsHost(host host.Host, r routing.ContentRouting) BitSwapNetwork {
29 - bitswapNetwork := impl{
30 - host: host,
31 - routing: r,
32 - }
33 - host.SetStreamHandler(ProtocolBitswap, bitswapNetwork.handleNewStream)
34 - host.SetStreamHandler(ProtocolBitswapOne, bitswapNetwork.handleNewStream)
35 - host.SetStreamHandler(ProtocolBitswapNoVers, bitswapNetwork.handleNewStream)
36 - host.Network().Notify((*netNotifiee)(&bitswapNetwork))
37 - // TODO: StopNotify.
38 -
39 - return &bitswapNetwork
40 -}
41 -
42 -// impl transforms the ipfs network interface, which sends and receives
43 -// NetMessage objects, into the bitswap network interface.
44 -type impl struct {
45 - host host.Host
46 - routing routing.ContentRouting
47 -
48 - // inbound messages from the network are forwarded to the receiver
49 - receiver Receiver
50 -}
51 -
52 -type streamMessageSender struct {
53 - s inet.Stream
54 -}
55 -
56 -func (s *streamMessageSender) Close() error {
57 - return inet.FullClose(s.s)
58 -}
59 -
60 -func (s *streamMessageSender) Reset() error {
61 - return s.s.Reset()
62 -}
63 -
64 -func (s *streamMessageSender) SendMsg(ctx context.Context, msg bsmsg.BitSwapMessage) error {
65 - return msgToStream(ctx, s.s, msg)
66 -}
67 -
68 -func msgToStream(ctx context.Context, s inet.Stream, msg bsmsg.BitSwapMessage) error {
69 - deadline := time.Now().Add(sendMessageTimeout)
70 - if dl, ok := ctx.Deadline(); ok {
71 - deadline = dl
72 - }
73 -
74 - if err := s.SetWriteDeadline(deadline); err != nil {
75 - log.Warningf("error setting deadline: %s", err)
76 - }
77 -
78 - switch s.Protocol() {
79 - case ProtocolBitswap:
80 - if err := msg.ToNetV1(s); err != nil {
81 - log.Debugf("error: %s", err)
82 - return err
83 - }
84 - case ProtocolBitswapOne, ProtocolBitswapNoVers:
85 - if err := msg.ToNetV0(s); err != nil {
86 - log.Debugf("error: %s", err)
87 - return err
88 - }
89 - default:
90 - return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol())
91 - }
92 -
93 - if err := s.SetWriteDeadline(time.Time{}); err != nil {
94 - log.Warningf("error resetting deadline: %s", err)
95 - }
96 - return nil
97 -}
98 -
99 -func (bsnet *impl) NewMessageSender(ctx context.Context, p peer.ID) (MessageSender, error) {
100 - s, err := bsnet.newStreamToPeer(ctx, p)
101 - if err != nil {
102 - return nil, err
103 - }
104 -
105 - return &streamMessageSender{s: s}, nil
106 -}
107 -
108 -func (bsnet *impl) newStreamToPeer(ctx context.Context, p peer.ID) (inet.Stream, error) {
109 - return bsnet.host.NewStream(ctx, p, ProtocolBitswap, ProtocolBitswapOne, ProtocolBitswapNoVers)
110 -}
111 -
112 -func (bsnet *impl) SendMessage(
113 - ctx context.Context,
114 - p peer.ID,
115 - outgoing bsmsg.BitSwapMessage) error {
116 -
117 - s, err := bsnet.newStreamToPeer(ctx, p)
118 - if err != nil {
119 - return err
120 - }
121 -
122 - if err = msgToStream(ctx, s, outgoing); err != nil {
123 - s.Reset()
124 - return err
125 - }
126 - // TODO(https://github.com/libp2p/go-libp2p-net/issues/28): Avoid this goroutine.
127 - go inet.AwaitEOF(s)
128 - return s.Close()
129 -
130 -}
131 -
132 -func (bsnet *impl) SetDelegate(r Receiver) {
133 - bsnet.receiver = r
134 -}
135 -
136 -func (bsnet *impl) ConnectTo(ctx context.Context, p peer.ID) error {
137 - return bsnet.host.Connect(ctx, pstore.PeerInfo{ID: p})
138 -}
139 -
140 -// FindProvidersAsync returns a channel of providers for the given key
141 -func (bsnet *impl) FindProvidersAsync(ctx context.Context, k *cid.Cid, max int) <-chan peer.ID {
142 -
143 - // Since routing queries are expensive, give bitswap the peers to which we
144 - // have open connections. Note that this may cause issues if bitswap starts
145 - // precisely tracking which peers provide certain keys. This optimization
146 - // would be misleading. In the long run, this may not be the most
147 - // appropriate place for this optimization, but it won't cause any harm in
148 - // the short term.
149 - connectedPeers := bsnet.host.Network().Peers()
150 - out := make(chan peer.ID, len(connectedPeers)) // just enough buffer for these connectedPeers
151 - for _, id := range connectedPeers {
152 - if id == bsnet.host.ID() {
153 - continue // ignore self as provider
154 - }
155 - out <- id
156 - }
157 -
158 - go func() {
159 - defer close(out)
160 - providers := bsnet.routing.FindProvidersAsync(ctx, k, max)
161 - for info := range providers {
162 - if info.ID == bsnet.host.ID() {
163 - continue // ignore self as provider
164 - }
165 - bsnet.host.Peerstore().AddAddrs(info.ID, info.Addrs, pstore.TempAddrTTL)
166 - select {
167 - case <-ctx.Done():
168 - return
169 - case out <- info.ID:
170 - }
171 - }
172 - }()
173 - return out
174 -}
175 -
176 -// Provide provides the key to the network
177 -func (bsnet *impl) Provide(ctx context.Context, k *cid.Cid) error {
178 - return bsnet.routing.Provide(ctx, k, true)
179 -}
180 -
181 -// handleNewStream receives a new stream from the network.
182 -func (bsnet *impl) handleNewStream(s inet.Stream) {
183 - defer s.Close()
184 -
185 - if bsnet.receiver == nil {
186 - s.Reset()
187 - return
188 - }
189 -
190 - reader := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
191 - for {
192 - received, err := bsmsg.FromPBReader(reader)
193 - if err != nil {
194 - if err != io.EOF {
195 - s.Reset()
196 - go bsnet.receiver.ReceiveError(err)
197 - log.Debugf("bitswap net handleNewStream from %s error: %s", s.Conn().RemotePeer(), err)
198 - }
199 - return
200 - }
201 -
202 - p := s.Conn().RemotePeer()
203 - ctx := context.Background()
204 - log.Debugf("bitswap net handleNewStream from %s", s.Conn().RemotePeer())
205 - bsnet.receiver.ReceiveMessage(ctx, p, received)
206 - }
207 -}
208 -
209 -func (bsnet *impl) ConnectionManager() ifconnmgr.ConnManager {
210 - return bsnet.host.ConnManager()
211 -}
212 -
213 -type netNotifiee impl
214 -
215 -func (nn *netNotifiee) impl() *impl {
216 - return (*impl)(nn)
217 -}
218 -
219 -func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
220 - nn.impl().receiver.PeerConnected(v.RemotePeer())
221 -}
222 -
223 -func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
224 - nn.impl().receiver.PeerDisconnected(v.RemotePeer())
225 -}
226 -
227 -func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
228 -func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
229 -func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr) {}
230 -func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}
exchange/bitswap/notifications/notifications.go deleted
-130
@@ -1,130 +0,0 @@
1 -package notifications
2 -
3 -import (
4 - "context"
5 - "sync"
6 -
7 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
8 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
9 - pubsub "gx/ipfs/QmdbxjQWogRCHRaxhhGnYdT1oQJzL9GdqSKzCdqWr85AP2/pubsub"
10 -)
11 -
12 -const bufferSize = 16
13 -
14 -type PubSub interface {
15 - Publish(block blocks.Block)
16 - Subscribe(ctx context.Context, keys ...*cid.Cid) <-chan blocks.Block
17 - Shutdown()
18 -}
19 -
20 -func New() PubSub {
21 - return &impl{
22 - wrapped: *pubsub.New(bufferSize),
23 - cancel: make(chan struct{}),
24 - }
25 -}
26 -
27 -type impl struct {
28 - wrapped pubsub.PubSub
29 -
30 - // These two fields make up a shutdown "lock".
31 - // We need them as calling, e.g., `Unsubscribe` after calling `Shutdown`
32 - // blocks forever and fixing this in pubsub would be rather invasive.
33 - cancel chan struct{}
34 - wg sync.WaitGroup
35 -}
36 -
37 -func (ps *impl) Publish(block blocks.Block) {
38 - ps.wg.Add(1)
39 - defer ps.wg.Done()
40 -
41 - select {
42 - case <-ps.cancel:
43 - // Already shutdown, bail.
44 - return
45 - default:
46 - }
47 -
48 - ps.wrapped.Pub(block, block.Cid().KeyString())
49 -}
50 -
51 -// Not safe to call more than once.
52 -func (ps *impl) Shutdown() {
53 - // Interrupt in-progress subscriptions.
54 - close(ps.cancel)
55 - // Wait for them to finish.
56 - ps.wg.Wait()
57 - // shutdown the pubsub.
58 - ps.wrapped.Shutdown()
59 -}
60 -
61 -// Subscribe returns a channel of blocks for the given |keys|. |blockChannel|
62 -// is closed if the |ctx| times out or is cancelled, or after sending len(keys)
63 -// blocks.
64 -func (ps *impl) Subscribe(ctx context.Context, keys ...*cid.Cid) <-chan blocks.Block {
65 -
66 - blocksCh := make(chan blocks.Block, len(keys))
67 - valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking
68 - if len(keys) == 0 {
69 - close(blocksCh)
70 - return blocksCh
71 - }
72 -
73 - // prevent shutdown
74 - ps.wg.Add(1)
75 -
76 - // check if shutdown *after* preventing shutdowns.
77 - select {
78 - case <-ps.cancel:
79 - // abort, allow shutdown to continue.
80 - ps.wg.Done()
81 - close(blocksCh)
82 - return blocksCh
83 - default:
84 - }
85 -
86 - ps.wrapped.AddSubOnceEach(valuesCh, toStrings(keys)...)
87 - go func() {
88 - defer func() {
89 - ps.wrapped.Unsub(valuesCh)
90 - close(blocksCh)
91 -
92 - // Unblock shutdown.
93 - ps.wg.Done()
94 - }()
95 -
96 - for {
97 - select {
98 - case <-ps.cancel:
99 - return
100 - case <-ctx.Done():
101 - return
102 - case val, ok := <-valuesCh:
103 - if !ok {
104 - return
105 - }
106 - block, ok := val.(blocks.Block)
107 - if !ok {
108 - return
109 - }
110 - select {
111 - case <-ps.cancel:
112 - return
113 - case <-ctx.Done():
114 - return
115 - case blocksCh <- block: // continue
116 - }
117 - }
118 - }
119 - }()
120 -
121 - return blocksCh
122 -}
123 -
124 -func toStrings(keys []*cid.Cid) []string {
125 - strs := make([]string, 0, len(keys))
126 - for _, key := range keys {
127 - strs = append(strs, key.KeyString())
128 - }
129 - return strs
130 -}
exchange/bitswap/notifications/notifications_test.go deleted
-187
@@ -1,187 +0,0 @@
1 -package notifications
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "testing"
7 - "time"
8 -
9 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
10 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
11 - blocksutil "gx/ipfs/QmYqPGpZ9Yemr55xus9DiEztkns6Jti5XJ7hC94JbvkdqZ/go-ipfs-blocksutil"
12 -)
13 -
14 -func TestDuplicates(t *testing.T) {
15 - b1 := blocks.NewBlock([]byte("1"))
16 - b2 := blocks.NewBlock([]byte("2"))
17 -
18 - n := New()
19 - defer n.Shutdown()
20 - ch := n.Subscribe(context.Background(), b1.Cid(), b2.Cid())
21 -
22 - n.Publish(b1)
23 - blockRecvd, ok := <-ch
24 - if !ok {
25 - t.Fail()
26 - }
27 - assertBlocksEqual(t, b1, blockRecvd)
28 -
29 - n.Publish(b1) // ignored duplicate
30 -
31 - n.Publish(b2)
32 - blockRecvd, ok = <-ch
33 - if !ok {
34 - t.Fail()
35 - }
36 - assertBlocksEqual(t, b2, blockRecvd)
37 -}
38 -
39 -func TestPublishSubscribe(t *testing.T) {
40 - blockSent := blocks.NewBlock([]byte("Greetings from The Interval"))
41 -
42 - n := New()
43 - defer n.Shutdown()
44 - ch := n.Subscribe(context.Background(), blockSent.Cid())
45 -
46 - n.Publish(blockSent)
47 - blockRecvd, ok := <-ch
48 - if !ok {
49 - t.Fail()
50 - }
51 -
52 - assertBlocksEqual(t, blockRecvd, blockSent)
53 -
54 -}
55 -
56 -func TestSubscribeMany(t *testing.T) {
57 - e1 := blocks.NewBlock([]byte("1"))
58 - e2 := blocks.NewBlock([]byte("2"))
59 -
60 - n := New()
61 - defer n.Shutdown()
62 - ch := n.Subscribe(context.Background(), e1.Cid(), e2.Cid())
63 -
64 - n.Publish(e1)
65 - r1, ok := <-ch
66 - if !ok {
67 - t.Fatal("didn't receive first expected block")
68 - }
69 - assertBlocksEqual(t, e1, r1)
70 -
71 - n.Publish(e2)
72 - r2, ok := <-ch
73 - if !ok {
74 - t.Fatal("didn't receive second expected block")
75 - }
76 - assertBlocksEqual(t, e2, r2)
77 -}
78 -
79 -// TestDuplicateSubscribe tests a scenario where a given block
80 -// would be requested twice at the same time.
81 -func TestDuplicateSubscribe(t *testing.T) {
82 - e1 := blocks.NewBlock([]byte("1"))
83 -
84 - n := New()
85 - defer n.Shutdown()
86 - ch1 := n.Subscribe(context.Background(), e1.Cid())
87 - ch2 := n.Subscribe(context.Background(), e1.Cid())
88 -
89 - n.Publish(e1)
90 - r1, ok := <-ch1
91 - if !ok {
92 - t.Fatal("didn't receive first expected block")
93 - }
94 - assertBlocksEqual(t, e1, r1)
95 -
96 - r2, ok := <-ch2
97 - if !ok {
98 - t.Fatal("didn't receive second expected block")
99 - }
100 - assertBlocksEqual(t, e1, r2)
101 -}
102 -
103 -func TestShutdownBeforeUnsubscribe(t *testing.T) {
104 - e1 := blocks.NewBlock([]byte("1"))
105 -
106 - n := New()
107 - ctx, cancel := context.WithCancel(context.Background())
108 - ch := n.Subscribe(ctx, e1.Cid()) // no keys provided
109 - n.Shutdown()
110 - cancel()
111 -
112 - select {
113 - case _, ok := <-ch:
114 - if ok {
115 - t.Fatal("channel should have been closed")
116 - }
117 - default:
118 - t.Fatal("channel should have been closed")
119 - }
120 -}
121 -
122 -func TestSubscribeIsANoopWhenCalledWithNoKeys(t *testing.T) {
123 - n := New()
124 - defer n.Shutdown()
125 - ch := n.Subscribe(context.Background()) // no keys provided
126 - if _, ok := <-ch; ok {
127 - t.Fatal("should be closed if no keys provided")
128 - }
129 -}
130 -
131 -func TestCarryOnWhenDeadlineExpires(t *testing.T) {
132 -
133 - impossibleDeadline := time.Nanosecond
134 - fastExpiringCtx, cancel := context.WithTimeout(context.Background(), impossibleDeadline)
135 - defer cancel()
136 -
137 - n := New()
138 - defer n.Shutdown()
139 - block := blocks.NewBlock([]byte("A Missed Connection"))
140 - blockChannel := n.Subscribe(fastExpiringCtx, block.Cid())
141 -
142 - assertBlockChannelNil(t, blockChannel)
143 -}
144 -
145 -func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) {
146 -
147 - g := blocksutil.NewBlockGenerator()
148 - ctx, cancel := context.WithCancel(context.Background())
149 - n := New()
150 - defer n.Shutdown()
151 -
152 - t.Log("generate a large number of blocks. exceed default buffer")
153 - bs := g.Blocks(1000)
154 - ks := func() []*cid.Cid {
155 - var keys []*cid.Cid
156 - for _, b := range bs {
157 - keys = append(keys, b.Cid())
158 - }
159 - return keys
160 - }()
161 -
162 - _ = n.Subscribe(ctx, ks...) // ignore received channel
163 -
164 - t.Log("cancel context before any blocks published")
165 - cancel()
166 - for _, b := range bs {
167 - n.Publish(b)
168 - }
169 -
170 - t.Log("publishing the large number of blocks to the ignored channel must not deadlock")
171 -}
172 -
173 -func assertBlockChannelNil(t *testing.T, blockChannel <-chan blocks.Block) {
174 - _, ok := <-blockChannel
175 - if ok {
176 - t.Fail()
177 - }
178 -}
179 -
180 -func assertBlocksEqual(t *testing.T, a, b blocks.Block) {
181 - if !bytes.Equal(a.RawData(), b.RawData()) {
182 - t.Fatal("blocks aren't equal")
183 - }
184 - if a.Cid() != b.Cid() {
185 - t.Fatal("block keys aren't equal")
186 - }
187 -}
exchange/bitswap/session.go deleted
-364
@@ -1,364 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "time"
7 -
8 - notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
9 -
10 - loggables "gx/ipfs/QmRPkGkHLB72caXgdDYnoaWigXNWx95BcYDKV1n3KTEpaG/go-libp2p-loggables"
11 - lru "gx/ipfs/QmVYxfoJQiZijTgPNHCHgHELvQpbsJNTg6Crmc3dQkj3yy/golang-lru"
12 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
13 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
14 - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
15 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
16 -)
17 -
18 -const activeWantsLimit = 16
19 -
20 -// Session holds state for an individual bitswap transfer operation.
21 -// This allows bitswap to make smarter decisions about who to send wantlist
22 -// info to, and who to request blocks from
23 -type Session struct {
24 - ctx context.Context
25 - tofetch *cidQueue
26 - activePeers map[peer.ID]struct{}
27 - activePeersArr []peer.ID
28 -
29 - bs *Bitswap
30 - incoming chan blkRecv
31 - newReqs chan []*cid.Cid
32 - cancelKeys chan []*cid.Cid
33 - interestReqs chan interestReq
34 -
35 - interest *lru.Cache
36 - liveWants map[string]time.Time
37 -
38 - tick *time.Timer
39 - baseTickDelay time.Duration
40 -
41 - latTotal time.Duration
42 - fetchcnt int
43 -
44 - notif notifications.PubSub
45 -
46 - uuid logging.Loggable
47 -
48 - id uint64
49 - tag string
50 -}
51 -
52 -// NewSession creates a new bitswap session whose lifetime is bounded by the
53 -// given context
54 -func (bs *Bitswap) NewSession(ctx context.Context) *Session {
55 - s := &Session{
56 - activePeers: make(map[peer.ID]struct{}),
57 - liveWants: make(map[string]time.Time),
58 - newReqs: make(chan []*cid.Cid),
59 - cancelKeys: make(chan []*cid.Cid),
60 - tofetch: newCidQueue(),
61 - interestReqs: make(chan interestReq),
62 - ctx: ctx,
63 - bs: bs,
64 - incoming: make(chan blkRecv),
65 - notif: notifications.New(),
66 - uuid: loggables.Uuid("GetBlockRequest"),
67 - baseTickDelay: time.Millisecond * 500,
68 - id: bs.getNextSessionID(),
69 - }
70 -
71 - s.tag = fmt.Sprint("bs-ses-", s.id)
72 -
73 - cache, _ := lru.New(2048)
74 - s.interest = cache
75 -
76 - bs.sessLk.Lock()
77 - bs.sessions = append(bs.sessions, s)
78 - bs.sessLk.Unlock()
79 -
80 - go s.run(ctx)
81 -
82 - return s
83 -}
84 -
85 -func (bs *Bitswap) removeSession(s *Session) {
86 - s.notif.Shutdown()
87 -
88 - live := make([]*cid.Cid, 0, len(s.liveWants))
89 - for c := range s.liveWants {
90 - cs, _ := cid.Cast([]byte(c))
91 - live = append(live, cs)
92 - }
93 - bs.CancelWants(live, s.id)
94 -
95 - bs.sessLk.Lock()
96 - defer bs.sessLk.Unlock()
97 - for i := 0; i < len(bs.sessions); i++ {
98 - if bs.sessions[i] == s {
99 - bs.sessions[i] = bs.sessions[len(bs.sessions)-1]
100 - bs.sessions = bs.sessions[:len(bs.sessions)-1]
101 - return
102 - }
103 - }
104 -}
105 -
106 -type blkRecv struct {
107 - from peer.ID
108 - blk blocks.Block
109 -}
110 -
111 -func (s *Session) receiveBlockFrom(from peer.ID, blk blocks.Block) {
112 - select {
113 - case s.incoming <- blkRecv{from: from, blk: blk}:
114 - case <-s.ctx.Done():
115 - }
116 -}
117 -
118 -type interestReq struct {
119 - c *cid.Cid
120 - resp chan bool
121 -}
122 -
123 -// TODO: PERF: this is using a channel to guard a map access against race
124 -// conditions. This is definitely much slower than a mutex, though its unclear
125 -// if it will actually induce any noticeable slowness. This is implemented this
126 -// way to avoid adding a more complex set of mutexes around the liveWants map.
127 -// note that in the average case (where this session *is* interested in the
128 -// block we received) this function will not be called, as the cid will likely
129 -// still be in the interest cache.
130 -func (s *Session) isLiveWant(c *cid.Cid) bool {
131 - resp := make(chan bool, 1)
132 - select {
133 - case s.interestReqs <- interestReq{
134 - c: c,
135 - resp: resp,
136 - }:
137 - case <-s.ctx.Done():
138 - return false
139 - }
140 -
141 - select {
142 - case want := <-resp:
143 - return want
144 - case <-s.ctx.Done():
145 - return false
146 - }
147 -}
148 -
149 -func (s *Session) interestedIn(c *cid.Cid) bool {
150 - return s.interest.Contains(c.KeyString()) || s.isLiveWant(c)
151 -}
152 -
153 -const provSearchDelay = time.Second * 10
154 -
155 -func (s *Session) addActivePeer(p peer.ID) {
156 - if _, ok := s.activePeers[p]; !ok {
157 - s.activePeers[p] = struct{}{}
158 - s.activePeersArr = append(s.activePeersArr, p)
159 -
160 - cmgr := s.bs.network.ConnectionManager()
161 - cmgr.TagPeer(p, s.tag, 10)
162 - }
163 -}
164 -
165 -func (s *Session) resetTick() {
166 - if s.latTotal == 0 {
167 - s.tick.Reset(provSearchDelay)
168 - } else {
169 - avLat := s.latTotal / time.Duration(s.fetchcnt)
170 - s.tick.Reset(s.baseTickDelay + (3 * avLat))
171 - }
172 -}
173 -
174 -func (s *Session) run(ctx context.Context) {
175 - s.tick = time.NewTimer(provSearchDelay)
176 - newpeers := make(chan peer.ID, 16)
177 - for {
178 - select {
179 - case blk := <-s.incoming:
180 - s.tick.Stop()
181 -
182 - if blk.from != "" {
183 - s.addActivePeer(blk.from)
184 - }
185 -
186 - s.receiveBlock(ctx, blk.blk)
187 -
188 - s.resetTick()
189 - case keys := <-s.newReqs:
190 - for _, k := range keys {
191 - s.interest.Add(k.KeyString(), nil)
192 - }
193 - if len(s.liveWants) < activeWantsLimit {
194 - toadd := activeWantsLimit - len(s.liveWants)
195 - if toadd > len(keys) {
196 - toadd = len(keys)
197 - }
198 -
199 - now := keys[:toadd]
200 - keys = keys[toadd:]
201 -
202 - s.wantBlocks(ctx, now)
203 - }
204 - for _, k := range keys {
205 - s.tofetch.Push(k)
206 - }
207 - case keys := <-s.cancelKeys:
208 - s.cancel(keys)
209 -
210 - case <-s.tick.C:
211 - live := make([]*cid.Cid, 0, len(s.liveWants))
212 - now := time.Now()
213 - for c := range s.liveWants {
214 - cs, _ := cid.Cast([]byte(c))
215 - live = append(live, cs)
216 - s.liveWants[c] = now
217 - }
218 -
219 - // Broadcast these keys to everyone we're connected to
220 - s.bs.wm.WantBlocks(ctx, live, nil, s.id)
221 -
222 - if len(live) > 0 {
223 - go func(k *cid.Cid) {
224 - // TODO: have a task queue setup for this to:
225 - // - rate limit
226 - // - manage timeouts
227 - // - ensure two 'findprovs' calls for the same block don't run concurrently
228 - // - share peers between sessions based on interest set
229 - for p := range s.bs.network.FindProvidersAsync(ctx, k, 10) {
230 - newpeers <- p
231 - }
232 - }(live[0])
233 - }
234 - s.resetTick()
235 - case p := <-newpeers:
236 - s.addActivePeer(p)
237 - case lwchk := <-s.interestReqs:
238 - lwchk.resp <- s.cidIsWanted(lwchk.c)
239 - case <-ctx.Done():
240 - s.tick.Stop()
241 - s.bs.removeSession(s)
242 -
243 - cmgr := s.bs.network.ConnectionManager()
244 - for _, p := range s.activePeersArr {
245 - cmgr.UntagPeer(p, s.tag)
246 - }
247 - return
248 - }
249 - }
250 -}
251 -
252 -func (s *Session) cidIsWanted(c *cid.Cid) bool {
253 - _, ok := s.liveWants[c.KeyString()]
254 - if !ok {
255 - ok = s.tofetch.Has(c)
256 - }
257 -
258 - return ok
259 -}
260 -
261 -func (s *Session) receiveBlock(ctx context.Context, blk blocks.Block) {
262 - c := blk.Cid()
263 - if s.cidIsWanted(c) {
264 - ks := c.KeyString()
265 - tval, ok := s.liveWants[ks]
266 - if ok {
267 - s.latTotal += time.Since(tval)
268 - delete(s.liveWants, ks)
269 - } else {
270 - s.tofetch.Remove(c)
271 - }
272 - s.fetchcnt++
273 - s.notif.Publish(blk)
274 -
275 - if next := s.tofetch.Pop(); next != nil {
276 - s.wantBlocks(ctx, []*cid.Cid{next})
277 - }
278 - }
279 -}
280 -
281 -func (s *Session) wantBlocks(ctx context.Context, ks []*cid.Cid) {
282 - now := time.Now()
283 - for _, c := range ks {
284 - s.liveWants[c.KeyString()] = now
285 - }
286 - s.bs.wm.WantBlocks(ctx, ks, s.activePeersArr, s.id)
287 -}
288 -
289 -func (s *Session) cancel(keys []*cid.Cid) {
290 - for _, c := range keys {
291 - s.tofetch.Remove(c)
292 - }
293 -}
294 -
295 -func (s *Session) cancelWants(keys []*cid.Cid) {
296 - select {
297 - case s.cancelKeys <- keys:
298 - case <-s.ctx.Done():
299 - }
300 -}
301 -
302 -func (s *Session) fetch(ctx context.Context, keys []*cid.Cid) {
303 - select {
304 - case s.newReqs <- keys:
305 - case <-ctx.Done():
306 - case <-s.ctx.Done():
307 - }
308 -}
309 -
310 -// GetBlocks fetches a set of blocks within the context of this session and
311 -// returns a channel that found blocks will be returned on. No order is
312 -// guaranteed on the returned blocks.
313 -func (s *Session) GetBlocks(ctx context.Context, keys []*cid.Cid) (<-chan blocks.Block, error) {
314 - ctx = logging.ContextWithLoggable(ctx, s.uuid)
315 - return getBlocksImpl(ctx, keys, s.notif, s.fetch, s.cancelWants)
316 -}
317 -
318 -// GetBlock fetches a single block
319 -func (s *Session) GetBlock(parent context.Context, k *cid.Cid) (blocks.Block, error) {
320 - return getBlock(parent, k, s.GetBlocks)
321 -}
322 -
323 -type cidQueue struct {
324 - elems []*cid.Cid
325 - eset *cid.Set
326 -}
327 -
328 -func newCidQueue() *cidQueue {
329 - return &cidQueue{eset: cid.NewSet()}
330 -}
331 -
332 -func (cq *cidQueue) Pop() *cid.Cid {
333 - for {
334 - if len(cq.elems) == 0 {
335 - return nil
336 - }
337 -
338 - out := cq.elems[0]
339 - cq.elems = cq.elems[1:]
340 -
341 - if cq.eset.Has(out) {
342 - cq.eset.Remove(out)
343 - return out
344 - }
345 - }
346 -}
347 -
348 -func (cq *cidQueue) Push(c *cid.Cid) {
349 - if cq.eset.Visit(c) {
350 - cq.elems = append(cq.elems, c)
351 - }
352 -}
353 -
354 -func (cq *cidQueue) Remove(c *cid.Cid) {
355 - cq.eset.Remove(c)
356 -}
357 -
358 -func (cq *cidQueue) Has(c *cid.Cid) bool {
359 - return cq.eset.Has(c)
360 -}
361 -
362 -func (cq *cidQueue) Len() int {
363 - return cq.eset.Len()
364 -}
exchange/bitswap/session_test.go deleted
-325
@@ -1,325 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "testing"
7 - "time"
8 -
9 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
10 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
11 - blocksutil "gx/ipfs/QmYqPGpZ9Yemr55xus9DiEztkns6Jti5XJ7hC94JbvkdqZ/go-ipfs-blocksutil"
12 - tu "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
13 -)
14 -
15 -func TestBasicSessions(t *testing.T) {
16 - ctx, cancel := context.WithCancel(context.Background())
17 - defer cancel()
18 -
19 - vnet := getVirtualNetwork()
20 - sesgen := NewTestSessionGenerator(vnet)
21 - defer sesgen.Close()
22 - bgen := blocksutil.NewBlockGenerator()
23 -
24 - block := bgen.Next()
25 - inst := sesgen.Instances(2)
26 -
27 - a := inst[0]
28 - b := inst[1]
29 -
30 - if err := b.Blockstore().Put(block); err != nil {
31 - t.Fatal(err)
32 - }
33 -
34 - sesa := a.Exchange.NewSession(ctx)
35 -
36 - blkout, err := sesa.GetBlock(ctx, block.Cid())
37 - if err != nil {
38 - t.Fatal(err)
39 - }
40 -
41 - if !blkout.Cid().Equals(block.Cid()) {
42 - t.Fatal("got wrong block")
43 - }
44 -}
45 -
46 -func assertBlockLists(got, exp []blocks.Block) error {
47 - if len(got) != len(exp) {
48 - return fmt.Errorf("got wrong number of blocks, %d != %d", len(got), len(exp))
49 - }
50 -
51 - h := cid.NewSet()
52 - for _, b := range got {
53 - h.Add(b.Cid())
54 - }
55 - for _, b := range exp {
56 - if !h.Has(b.Cid()) {
57 - return fmt.Errorf("didnt have: %s", b.Cid())
58 - }
59 - }
60 - return nil
61 -}
62 -
63 -func TestSessionBetweenPeers(t *testing.T) {
64 - ctx, cancel := context.WithCancel(context.Background())
65 - defer cancel()
66 -
67 - vnet := getVirtualNetwork()
68 - sesgen := NewTestSessionGenerator(vnet)
69 - defer sesgen.Close()
70 - bgen := blocksutil.NewBlockGenerator()
71 -
72 - inst := sesgen.Instances(10)
73 -
74 - blks := bgen.Blocks(101)
75 - if err := inst[0].Blockstore().PutMany(blks); err != nil {
76 - t.Fatal(err)
77 - }
78 -
79 - var cids []*cid.Cid
80 - for _, blk := range blks {
81 - cids = append(cids, blk.Cid())
82 - }
83 -
84 - ses := inst[1].Exchange.NewSession(ctx)
85 - if _, err := ses.GetBlock(ctx, cids[0]); err != nil {
86 - t.Fatal(err)
87 - }
88 - blks = blks[1:]
89 - cids = cids[1:]
90 -
91 - for i := 0; i < 10; i++ {
92 - ch, err := ses.GetBlocks(ctx, cids[i*10:(i+1)*10])
93 - if err != nil {
94 - t.Fatal(err)
95 - }
96 -
97 - var got []blocks.Block
98 - for b := range ch {
99 - got = append(got, b)
100 - }
101 - if err := assertBlockLists(got, blks[i*10:(i+1)*10]); err != nil {
102 - t.Fatal(err)
103 - }
104 - }
105 - for _, is := range inst[2:] {
106 - if is.Exchange.counters.messagesRecvd > 2 {
107 - t.Fatal("uninvolved nodes should only receive two messages", is.Exchange.counters.messagesRecvd)
108 - }
109 - }
110 -}
111 -
112 -func TestSessionSplitFetch(t *testing.T) {
113 - ctx, cancel := context.WithCancel(context.Background())
114 - defer cancel()
115 -
116 - vnet := getVirtualNetwork()
117 - sesgen := NewTestSessionGenerator(vnet)
118 - defer sesgen.Close()
119 - bgen := blocksutil.NewBlockGenerator()
120 -
121 - inst := sesgen.Instances(11)
122 -
123 - blks := bgen.Blocks(100)
124 - for i := 0; i < 10; i++ {
125 - if err := inst[i].Blockstore().PutMany(blks[i*10 : (i+1)*10]); err != nil {
126 - t.Fatal(err)
127 - }
128 - }
129 -
130 - var cids []*cid.Cid
131 - for _, blk := range blks {
132 - cids = append(cids, blk.Cid())
133 - }
134 -
135 - ses := inst[10].Exchange.NewSession(ctx)
136 - ses.baseTickDelay = time.Millisecond * 10
137 -
138 - for i := 0; i < 10; i++ {
139 - ch, err := ses.GetBlocks(ctx, cids[i*10:(i+1)*10])
140 - if err != nil {
141 - t.Fatal(err)
142 - }
143 -
144 - var got []blocks.Block
145 - for b := range ch {
146 - got = append(got, b)
147 - }
148 - if err := assertBlockLists(got, blks[i*10:(i+1)*10]); err != nil {
149 - t.Fatal(err)
150 - }
151 - }
152 -}
153 -
154 -func TestInterestCacheOverflow(t *testing.T) {
155 - ctx, cancel := context.WithCancel(context.Background())
156 - defer cancel()
157 -
158 - vnet := getVirtualNetwork()
159 - sesgen := NewTestSessionGenerator(vnet)
160 - defer sesgen.Close()
161 - bgen := blocksutil.NewBlockGenerator()
162 -
163 - blks := bgen.Blocks(2049)
164 - inst := sesgen.Instances(2)
165 -
166 - a := inst[0]
167 - b := inst[1]
168 -
169 - ses := a.Exchange.NewSession(ctx)
170 - zeroch, err := ses.GetBlocks(ctx, []*cid.Cid{blks[0].Cid()})
171 - if err != nil {
172 - t.Fatal(err)
173 - }
174 -
175 - var restcids []*cid.Cid
176 - for _, blk := range blks[1:] {
177 - restcids = append(restcids, blk.Cid())
178 - }
179 -
180 - restch, err := ses.GetBlocks(ctx, restcids)
181 - if err != nil {
182 - t.Fatal(err)
183 - }
184 -
185 - // wait to ensure that all the above cids were added to the sessions cache
186 - time.Sleep(time.Millisecond * 50)
187 -
188 - if err := b.Exchange.HasBlock(blks[0]); err != nil {
189 - t.Fatal(err)
190 - }
191 -
192 - select {
193 - case blk, ok := <-zeroch:
194 - if ok && blk.Cid().Equals(blks[0].Cid()) {
195 - // success!
196 - } else {
197 - t.Fatal("failed to get the block")
198 - }
199 - case <-restch:
200 - t.Fatal("should not get anything on restch")
201 - case <-time.After(time.Second * 5):
202 - t.Fatal("timed out waiting for block")
203 - }
204 -}
205 -
206 -func TestPutAfterSessionCacheEvict(t *testing.T) {
207 - ctx, cancel := context.WithCancel(context.Background())
208 - defer cancel()
209 -
210 - vnet := getVirtualNetwork()
211 - sesgen := NewTestSessionGenerator(vnet)
212 - defer sesgen.Close()
213 - bgen := blocksutil.NewBlockGenerator()
214 -
215 - blks := bgen.Blocks(2500)
216 - inst := sesgen.Instances(1)
217 -
218 - a := inst[0]
219 -
220 - ses := a.Exchange.NewSession(ctx)
221 -
222 - var allcids []*cid.Cid
223 - for _, blk := range blks[1:] {
224 - allcids = append(allcids, blk.Cid())
225 - }
226 -
227 - blkch, err := ses.GetBlocks(ctx, allcids)
228 - if err != nil {
229 - t.Fatal(err)
230 - }
231 -
232 - // wait to ensure that all the above cids were added to the sessions cache
233 - time.Sleep(time.Millisecond * 50)
234 -
235 - if err := a.Exchange.HasBlock(blks[17]); err != nil {
236 - t.Fatal(err)
237 - }
238 -
239 - select {
240 - case <-blkch:
241 - case <-time.After(time.Millisecond * 50):
242 - t.Fatal("timed out waiting for block")
243 - }
244 -}
245 -
246 -func TestMultipleSessions(t *testing.T) {
247 - ctx, cancel := context.WithCancel(context.Background())
248 - defer cancel()
249 -
250 - vnet := getVirtualNetwork()
251 - sesgen := NewTestSessionGenerator(vnet)
252 - defer sesgen.Close()
253 - bgen := blocksutil.NewBlockGenerator()
254 -
255 - blk := bgen.Blocks(1)[0]
256 - inst := sesgen.Instances(2)
257 -
258 - a := inst[0]
259 - b := inst[1]
260 -
261 - ctx1, cancel1 := context.WithCancel(ctx)
262 - ses := a.Exchange.NewSession(ctx1)
263 -
264 - blkch, err := ses.GetBlocks(ctx, []*cid.Cid{blk.Cid()})
265 - if err != nil {
266 - t.Fatal(err)
267 - }
268 - cancel1()
269 -
270 - ses2 := a.Exchange.NewSession(ctx)
271 - blkch2, err := ses2.GetBlocks(ctx, []*cid.Cid{blk.Cid()})
272 - if err != nil {
273 - t.Fatal(err)
274 - }
275 -
276 - time.Sleep(time.Millisecond * 10)
277 - if err := b.Exchange.HasBlock(blk); err != nil {
278 - t.Fatal(err)
279 - }
280 -
281 - select {
282 - case <-blkch2:
283 - case <-time.After(time.Second * 20):
284 - t.Fatal("bad juju")
285 - }
286 - _ = blkch
287 -}
288 -
289 -func TestWantlistClearsOnCancel(t *testing.T) {
290 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
291 - defer cancel()
292 -
293 - vnet := getVirtualNetwork()
294 - sesgen := NewTestSessionGenerator(vnet)
295 - defer sesgen.Close()
296 - bgen := blocksutil.NewBlockGenerator()
297 -
298 - blks := bgen.Blocks(10)
299 - var cids []*cid.Cid
300 - for _, blk := range blks {
301 - cids = append(cids, blk.Cid())
302 - }
303 -
304 - inst := sesgen.Instances(1)
305 -
306 - a := inst[0]
307 -
308 - ctx1, cancel1 := context.WithCancel(ctx)
309 - ses := a.Exchange.NewSession(ctx1)
310 -
311 - _, err := ses.GetBlocks(ctx, cids)
312 - if err != nil {
313 - t.Fatal(err)
314 - }
315 - cancel1()
316 -
317 - if err := tu.WaitFor(ctx, func() error {
318 - if len(a.Exchange.GetWantlist()) > 0 {
319 - return fmt.Errorf("expected empty wantlist")
320 - }
321 - return nil
322 - }); err != nil {
323 - t.Fatal(err)
324 - }
325 -}
exchange/bitswap/stat.go deleted
-44
@@ -1,44 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "sort"
5 -
6 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
7 -)
8 -
9 -type Stat struct {
10 - ProvideBufLen int
11 - Wantlist []*cid.Cid
12 - Peers []string
13 - BlocksReceived uint64
14 - DataReceived uint64
15 - BlocksSent uint64
16 - DataSent uint64
17 - DupBlksReceived uint64
18 - DupDataReceived uint64
19 -}
20 -
21 -func (bs *Bitswap) Stat() (*Stat, error) {
22 - st := new(Stat)
23 - st.ProvideBufLen = len(bs.newBlocks)
24 - st.Wantlist = bs.GetWantlist()
25 - bs.counterLk.Lock()
26 - c := bs.counters
27 - st.BlocksReceived = c.blocksRecvd
28 - st.DupBlksReceived = c.dupBlocksRecvd
29 - st.DupDataReceived = c.dupDataRecvd
30 - st.BlocksSent = c.blocksSent
31 - st.DataSent = c.dataSent
32 - st.DataReceived = c.dataRecvd
33 - bs.counterLk.Unlock()
34 -
35 - peers := bs.engine.Peers()
36 - st.Peers = make([]string, 0, len(peers))
37 -
38 - for _, p := range peers {
39 - st.Peers = append(st.Peers, p.Pretty())
40 - }
41 - sort.Strings(st.Peers)
42 -
43 - return st, nil
44 -}
exchange/bitswap/testnet/interface.go deleted
-13
@@ -1,13 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
5 - "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
6 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
7 -)
8 -
9 -type Network interface {
10 - Adapter(testutil.Identity) bsnet.BitSwapNetwork
11 -
12 - HasPeer(peer.ID) bool
13 -}
exchange/bitswap/testnet/network_test.go deleted
-98
@@ -1,98 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "sync"
6 - "testing"
7 -
8 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
9 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
10 -
11 - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
12 - blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
13 - mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock"
14 - testutil "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
15 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
16 -)
17 -
18 -func TestSendMessageAsyncButWaitForResponse(t *testing.T) {
19 - net := VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0))
20 - responderPeer := testutil.RandIdentityOrFatal(t)
21 - waiter := net.Adapter(testutil.RandIdentityOrFatal(t))
22 - responder := net.Adapter(responderPeer)
23 -
24 - var wg sync.WaitGroup
25 -
26 - wg.Add(1)
27 -
28 - expectedStr := "received async"
29 -
30 - responder.SetDelegate(lambda(func(
31 - ctx context.Context,
32 - fromWaiter peer.ID,
33 - msgFromWaiter bsmsg.BitSwapMessage) {
34 -
35 - msgToWaiter := bsmsg.New(true)
36 - msgToWaiter.AddBlock(blocks.NewBlock([]byte(expectedStr)))
37 - waiter.SendMessage(ctx, fromWaiter, msgToWaiter)
38 - }))
39 -
40 - waiter.SetDelegate(lambda(func(
41 - ctx context.Context,
42 - fromResponder peer.ID,
43 - msgFromResponder bsmsg.BitSwapMessage) {
44 -
45 - // TODO assert that this came from the correct peer and that the message contents are as expected
46 - ok := false
47 - for _, b := range msgFromResponder.Blocks() {
48 - if string(b.RawData()) == expectedStr {
49 - wg.Done()
50 - ok = true
51 - }
52 - }
53 -
54 - if !ok {
55 - t.Fatal("Message not received from the responder")
56 - }
57 - }))
58 -
59 - messageSentAsync := bsmsg.New(true)
60 - messageSentAsync.AddBlock(blocks.NewBlock([]byte("data")))
61 - errSending := waiter.SendMessage(
62 - context.Background(), responderPeer.ID(), messageSentAsync)
63 - if errSending != nil {
64 - t.Fatal(errSending)
65 - }
66 -
67 - wg.Wait() // until waiter delegate function is executed
68 -}
69 -
70 -type receiverFunc func(ctx context.Context, p peer.ID,
71 - incoming bsmsg.BitSwapMessage)
72 -
73 -// lambda returns a Receiver instance given a receiver function
74 -func lambda(f receiverFunc) bsnet.Receiver {
75 - return &lambdaImpl{
76 - f: f,
77 - }
78 -}
79 -
80 -type lambdaImpl struct {
81 - f func(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage)
82 -}
83 -
84 -func (lam *lambdaImpl) ReceiveMessage(ctx context.Context,
85 - p peer.ID, incoming bsmsg.BitSwapMessage) {
86 - lam.f(ctx, p, incoming)
87 -}
88 -
89 -func (lam *lambdaImpl) ReceiveError(err error) {
90 - // TODO log error
91 -}
92 -
93 -func (lam *lambdaImpl) PeerConnected(p peer.ID) {
94 - // TODO
95 -}
96 -func (lam *lambdaImpl) PeerDisconnected(peer.ID) {
97 - // TODO
98 -}
exchange/bitswap/testnet/peernet.go deleted
-42
@@ -1,42 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 -
6 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
7 -
8 - mockpeernet "gx/ipfs/QmY51bqSM5XgxQZqsBrQcRkKTnCb8EKpJpR9K6Qax7Njco/go-libp2p/p2p/net/mock"
9 - mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock"
10 - testutil "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
11 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
12 - ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
13 -)
14 -
15 -type peernet struct {
16 - mockpeernet.Mocknet
17 - routingserver mockrouting.Server
18 -}
19 -
20 -func StreamNet(ctx context.Context, net mockpeernet.Mocknet, rs mockrouting.Server) (Network, error) {
21 - return &peernet{net, rs}, nil
22 -}
23 -
24 -func (pn *peernet) Adapter(p testutil.Identity) bsnet.BitSwapNetwork {
25 - client, err := pn.Mocknet.AddPeer(p.PrivateKey(), p.Address())
26 - if err != nil {
27 - panic(err.Error())
28 - }
29 - routing := pn.routingserver.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore())
30 - return bsnet.NewFromIpfsHost(client, routing)
31 -}
32 -
33 -func (pn *peernet) HasPeer(p peer.ID) bool {
34 - for _, member := range pn.Mocknet.Peers() {
35 - if p == member {
36 - return true
37 - }
38 - }
39 - return false
40 -}
41 -
42 -var _ Network = (*peernet)(nil)
exchange/bitswap/testnet/virtual.go deleted
-253
@@ -1,253 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "errors"
6 - "sync"
7 - "time"
8 -
9 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
10 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
11 -
12 - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
13 - ifconnmgr "gx/ipfs/QmXuucFcuvAWYAJfhHV2h4BYreHEAsLSsiquosiXeuduTN/go-libp2p-interface-connmgr"
14 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
15 - routing "gx/ipfs/QmZ383TySJVeZWzGnWui6pRcKyYZk9VkKTuW7tmKRWk5au/go-libp2p-routing"
16 - mockrouting "gx/ipfs/QmbFRJeEmEU16y3BmKKaD4a9fm5oHsEAMHe2vSB1UnfLMi/go-ipfs-routing/mock"
17 - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
18 - testutil "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
19 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
20 -)
21 -
22 -var log = logging.Logger("bstestnet")
23 -
24 -func VirtualNetwork(rs mockrouting.Server, d delay.D) Network {
25 - return &network{
26 - clients: make(map[peer.ID]*receiverQueue),
27 - delay: d,
28 - routingserver: rs,
29 - conns: make(map[string]struct{}),
30 - }
31 -}
32 -
33 -type network struct {
34 - mu sync.Mutex
35 - clients map[peer.ID]*receiverQueue
36 - routingserver mockrouting.Server
37 - delay delay.D
38 - conns map[string]struct{}
39 -}
40 -
41 -type message struct {
42 - from peer.ID
43 - msg bsmsg.BitSwapMessage
44 - shouldSend time.Time
45 -}
46 -
47 -// receiverQueue queues up a set of messages to be sent, and sends them *in
48 -// order* with their delays respected as much as sending them in order allows
49 -// for
50 -type receiverQueue struct {
51 - receiver bsnet.Receiver
52 - queue []*message
53 - active bool
54 - lk sync.Mutex
55 -}
56 -
57 -func (n *network) Adapter(p testutil.Identity) bsnet.BitSwapNetwork {
58 - n.mu.Lock()
59 - defer n.mu.Unlock()
60 -
61 - client := &networkClient{
62 - local: p.ID(),
63 - network: n,
64 - routing: n.routingserver.Client(p),
65 - }
66 - n.clients[p.ID()] = &receiverQueue{receiver: client}
67 - return client
68 -}
69 -
70 -func (n *network) HasPeer(p peer.ID) bool {
71 - n.mu.Lock()
72 - defer n.mu.Unlock()
73 -
74 - _, found := n.clients[p]
75 - return found
76 -}
77 -
78 -// TODO should this be completely asynchronous?
79 -// TODO what does the network layer do with errors received from services?
80 -func (n *network) SendMessage(
81 - ctx context.Context,
82 - from peer.ID,
83 - to peer.ID,
84 - mes bsmsg.BitSwapMessage) error {
85 -
86 - n.mu.Lock()
87 - defer n.mu.Unlock()
88 -
89 - receiver, ok := n.clients[to]
90 - if !ok {
91 - return errors.New("cannot locate peer on network")
92 - }
93 -
94 - // nb: terminate the context since the context wouldn't actually be passed
95 - // over the network in a real scenario
96 -
97 - msg := &message{
98 - from: from,
99 - msg: mes,
100 - shouldSend: time.Now().Add(n.delay.Get()),
101 - }
102 - receiver.enqueue(msg)
103 -
104 - return nil
105 -}
106 -
107 -func (n *network) deliver(
108 - r bsnet.Receiver, from peer.ID, message bsmsg.BitSwapMessage) error {
109 - if message == nil || from == "" {
110 - return errors.New("invalid input")
111 - }
112 -
113 - n.delay.Wait()
114 -
115 - r.ReceiveMessage(context.TODO(), from, message)
116 - return nil
117 -}
118 -
119 -type networkClient struct {
120 - local peer.ID
121 - bsnet.Receiver
122 - network *network
123 - routing routing.IpfsRouting
124 -}
125 -
126 -func (nc *networkClient) SendMessage(
127 - ctx context.Context,
128 - to peer.ID,
129 - message bsmsg.BitSwapMessage) error {
130 - return nc.network.SendMessage(ctx, nc.local, to, message)
131 -}
132 -
133 -// FindProvidersAsync returns a channel of providers for the given key
134 -func (nc *networkClient) FindProvidersAsync(ctx context.Context, k *cid.Cid, max int) <-chan peer.ID {
135 -
136 - // NB: this function duplicates the PeerInfo -> ID transformation in the
137 - // bitswap network adapter. Not to worry. This network client will be
138 - // deprecated once the ipfsnet.Mock is added. The code below is only
139 - // temporary.
140 -
141 - out := make(chan peer.ID)
142 - go func() {
143 - defer close(out)
144 - providers := nc.routing.FindProvidersAsync(ctx, k, max)
145 - for info := range providers {
146 - select {
147 - case <-ctx.Done():
148 - case out <- info.ID:
149 - }
150 - }
151 - }()
152 - return out
153 -}
154 -
155 -func (nc *networkClient) ConnectionManager() ifconnmgr.ConnManager {
156 - return &ifconnmgr.NullConnMgr{}
157 -}
158 -
159 -type messagePasser struct {
160 - net *network
161 - target peer.ID
162 - local peer.ID
163 - ctx context.Context
164 -}
165 -
166 -func (mp *messagePasser) SendMsg(ctx context.Context, m bsmsg.BitSwapMessage) error {
167 - return mp.net.SendMessage(ctx, mp.local, mp.target, m)
168 -}
169 -
170 -func (mp *messagePasser) Close() error {
171 - return nil
172 -}
173 -
174 -func (mp *messagePasser) Reset() error {
175 - return nil
176 -}
177 -
178 -func (n *networkClient) NewMessageSender(ctx context.Context, p peer.ID) (bsnet.MessageSender, error) {
179 - return &messagePasser{
180 - net: n.network,
181 - target: p,
182 - local: n.local,
183 - ctx: ctx,
184 - }, nil
185 -}
186 -
187 -// Provide provides the key to the network
188 -func (nc *networkClient) Provide(ctx context.Context, k *cid.Cid) error {
189 - return nc.routing.Provide(ctx, k, true)
190 -}
191 -
192 -func (nc *networkClient) SetDelegate(r bsnet.Receiver) {
193 - nc.Receiver = r
194 -}
195 -
196 -func (nc *networkClient) ConnectTo(_ context.Context, p peer.ID) error {
197 - nc.network.mu.Lock()
198 -
199 - otherClient, ok := nc.network.clients[p]
200 - if !ok {
201 - nc.network.mu.Unlock()
202 - return errors.New("no such peer in network")
203 - }
204 -
205 - tag := tagForPeers(nc.local, p)
206 - if _, ok := nc.network.conns[tag]; ok {
207 - nc.network.mu.Unlock()
208 - log.Warning("ALREADY CONNECTED TO PEER (is this a reconnect? test lib needs fixing)")
209 - return nil
210 - }
211 - nc.network.conns[tag] = struct{}{}
212 - nc.network.mu.Unlock()
213 -
214 - // TODO: add handling for disconnects
215 -
216 - otherClient.receiver.PeerConnected(nc.local)
217 - nc.Receiver.PeerConnected(p)
218 - return nil
219 -}
220 -
221 -func (rq *receiverQueue) enqueue(m *message) {
222 - rq.lk.Lock()
223 - defer rq.lk.Unlock()
224 - rq.queue = append(rq.queue, m)
225 - if !rq.active {
226 - rq.active = true
227 - go rq.process()
228 - }
229 -}
230 -
231 -func (rq *receiverQueue) process() {
232 - for {
233 - rq.lk.Lock()
234 - if len(rq.queue) == 0 {
235 - rq.active = false
236 - rq.lk.Unlock()
237 - return
238 - }
239 - m := rq.queue[0]
240 - rq.queue = rq.queue[1:]
241 - rq.lk.Unlock()
242 -
243 - time.Sleep(time.Until(m.shouldSend))
244 - rq.receiver.ReceiveMessage(context.TODO(), m.from, m.msg)
245 - }
246 -}
247 -
248 -func tagForPeers(a, b peer.ID) string {
249 - if a < b {
250 - return string(a + b)
251 - }
252 - return string(b + a)
253 -}
exchange/bitswap/testutils.go deleted
-110
@@ -1,110 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "time"
6 -
7 - tn "github.com/ipfs/go-ipfs/exchange/bitswap/testnet"
8 -
9 - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay"
10 - blockstore "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
11 - testutil "gx/ipfs/QmcW4FGAt24fdK1jBgWQn3yP4R9ZLyWQqjozv9QK7epRhL/go-testutil"
12 - p2ptestutil "gx/ipfs/QmcxUtMB5sJrXR3znSvkrDd2ghvwGM8rLRqwJiPUdgQwat/go-libp2p-netutil"
13 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
14 - ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
15 - delayed "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/delayed"
16 - ds_sync "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"
17 -)
18 -
19 -// WARNING: this uses RandTestBogusIdentity DO NOT USE for NON TESTS!
20 -func NewTestSessionGenerator(
21 - net tn.Network) SessionGenerator {
22 - ctx, cancel := context.WithCancel(context.Background())
23 - return SessionGenerator{
24 - net: net,
25 - seq: 0,
26 - ctx: ctx, // TODO take ctx as param to Next, Instances
27 - cancel: cancel,
28 - }
29 -}
30 -
31 -// TODO move this SessionGenerator to the core package and export it as the core generator
32 -type SessionGenerator struct {
33 - seq int
34 - net tn.Network
35 - ctx context.Context
36 - cancel context.CancelFunc
37 -}
38 -
39 -func (g *SessionGenerator) Close() error {
40 - g.cancel()
41 - return nil // for Closer interface
42 -}
43 -
44 -func (g *SessionGenerator) Next() Instance {
45 - g.seq++
46 - p, err := p2ptestutil.RandTestBogusIdentity()
47 - if err != nil {
48 - panic("FIXME") // TODO change signature
49 - }
50 - return MkSession(g.ctx, g.net, p)
51 -}
52 -
53 -func (g *SessionGenerator) Instances(n int) []Instance {
54 - var instances []Instance
55 - for j := 0; j < n; j++ {
56 - inst := g.Next()
57 - instances = append(instances, inst)
58 - }
59 - for i, inst := range instances {
60 - for j := i + 1; j < len(instances); j++ {
61 - oinst := instances[j]
62 - inst.Exchange.network.ConnectTo(context.Background(), oinst.Peer)
63 - }
64 - }
65 - return instances
66 -}
67 -
68 -type Instance struct {
69 - Peer peer.ID
70 - Exchange *Bitswap
71 - blockstore blockstore.Blockstore
72 -
73 - blockstoreDelay delay.D
74 -}
75 -
76 -func (i *Instance) Blockstore() blockstore.Blockstore {
77 - return i.blockstore
78 -}
79 -
80 -func (i *Instance) SetBlockstoreLatency(t time.Duration) time.Duration {
81 - return i.blockstoreDelay.Set(t)
82 -}
83 -
84 -// session creates a test bitswap session.
85 -//
86 -// NB: It's easy make mistakes by providing the same peer ID to two different
87 -// sessions. To safeguard, use the SessionGenerator to generate sessions. It's
88 -// just a much better idea.
89 -func MkSession(ctx context.Context, net tn.Network, p testutil.Identity) Instance {
90 - bsdelay := delay.Fixed(0)
91 -
92 - adapter := net.Adapter(p)
93 - dstore := ds_sync.MutexWrap(delayed.New(ds.NewMapDatastore(), bsdelay))
94 -
95 - bstore, err := blockstore.CachedBlockstore(ctx,
96 - blockstore.NewBlockstore(ds_sync.MutexWrap(dstore)),
97 - blockstore.DefaultCacheOpts())
98 - if err != nil {
99 - panic(err.Error()) // FIXME perhaps change signature and return error.
100 - }
101 -
102 - bs := New(ctx, adapter, bstore).(*Bitswap)
103 -
104 - return Instance{
105 - Peer: p.ID(),
106 - Exchange: bs,
107 - blockstore: bstore,
108 - blockstoreDelay: bsdelay,
109 - }
110 -}
exchange/bitswap/wantlist/wantlist.go deleted
-203
@@ -1,203 +0,0 @@
1 -// package wantlist implements an object for bitswap that contains the keys
2 -// that a given peer wants.
3 -package wantlist
4 -
5 -import (
6 - "sort"
7 - "sync"
8 -
9 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
10 -)
11 -
12 -type ThreadSafe struct {
13 - lk sync.RWMutex
14 - set map[string]*Entry
15 -}
16 -
17 -// not threadsafe
18 -type Wantlist struct {
19 - set map[string]*Entry
20 -}
21 -
22 -type Entry struct {
23 - Cid *cid.Cid
24 - Priority int
25 -
26 - SesTrk map[uint64]struct{}
27 -}
28 -
29 -// NewRefEntry creates a new reference tracked wantlist entry
30 -func NewRefEntry(c *cid.Cid, p int) *Entry {
31 - return &Entry{
32 - Cid: c,
33 - Priority: p,
34 - SesTrk: make(map[uint64]struct{}),
35 - }
36 -}
37 -
38 -type entrySlice []*Entry
39 -
40 -func (es entrySlice) Len() int { return len(es) }
41 -func (es entrySlice) Swap(i, j int) { es[i], es[j] = es[j], es[i] }
42 -func (es entrySlice) Less(i, j int) bool { return es[i].Priority > es[j].Priority }
43 -
44 -func NewThreadSafe() *ThreadSafe {
45 - return &ThreadSafe{
46 - set: make(map[string]*Entry),
47 - }
48 -}
49 -
50 -func New() *Wantlist {
51 - return &Wantlist{
52 - set: make(map[string]*Entry),
53 - }
54 -}
55 -
56 -// Add adds the given cid to the wantlist with the specified priority, governed
57 -// by the session ID 'ses'. if a cid is added under multiple session IDs, then
58 -// it must be removed by each of those sessions before it is no longer 'in the
59 -// wantlist'. Calls to Add are idempotent given the same arguments. Subsequent
60 -// calls with different values for priority will not update the priority
61 -// TODO: think through priority changes here
62 -// Add returns true if the cid did not exist in the wantlist before this call
63 -// (even if it was under a different session)
64 -func (w *ThreadSafe) Add(c *cid.Cid, priority int, ses uint64) bool {
65 - w.lk.Lock()
66 - defer w.lk.Unlock()
67 - k := c.KeyString()
68 - if e, ok := w.set[k]; ok {
69 - e.SesTrk[ses] = struct{}{}
70 - return false
71 - }
72 -
73 - w.set[k] = &Entry{
74 - Cid: c,
75 - Priority: priority,
76 - SesTrk: map[uint64]struct{}{ses: struct{}{}},
77 - }
78 -
79 - return true
80 -}
81 -
82 -// AddEntry adds given Entry to the wantlist. For more information see Add method.
83 -func (w *ThreadSafe) AddEntry(e *Entry, ses uint64) bool {
84 - w.lk.Lock()
85 - defer w.lk.Unlock()
86 - k := e.Cid.KeyString()
87 - if ex, ok := w.set[k]; ok {
88 - ex.SesTrk[ses] = struct{}{}
89 - return false
90 - }
91 - w.set[k] = e
92 - e.SesTrk[ses] = struct{}{}
93 - return true
94 -}
95 -
96 -// Remove removes the given cid from being tracked by the given session.
97 -// 'true' is returned if this call to Remove removed the final session ID
98 -// tracking the cid. (meaning true will be returned iff this call caused the
99 -// value of 'Contains(c)' to change from true to false)
100 -func (w *ThreadSafe) Remove(c *cid.Cid, ses uint64) bool {
101 - w.lk.Lock()
102 - defer w.lk.Unlock()
103 - k := c.KeyString()
104 - e, ok := w.set[k]
105 - if !ok {
106 - return false
107 - }
108 -
109 - delete(e.SesTrk, ses)
110 - if len(e.SesTrk) == 0 {
111 - delete(w.set, k)
112 - return true
113 - }
114 - return false
115 -}
116 -
117 -// Contains returns true if the given cid is in the wantlist tracked by one or
118 -// more sessions
119 -func (w *ThreadSafe) Contains(k *cid.Cid) (*Entry, bool) {
120 - w.lk.RLock()
121 - defer w.lk.RUnlock()
122 - e, ok := w.set[k.KeyString()]
123 - return e, ok
124 -}
125 -
126 -func (w *ThreadSafe) Entries() []*Entry {
127 - w.lk.RLock()
128 - defer w.lk.RUnlock()
129 - es := make([]*Entry, 0, len(w.set))
130 - for _, e := range w.set {
131 - es = append(es, e)
132 - }
133 - return es
134 -}
135 -
136 -func (w *ThreadSafe) SortedEntries() []*Entry {
137 - es := w.Entries()
138 - sort.Sort(entrySlice(es))
139 - return es
140 -}
141 -
142 -func (w *ThreadSafe) Len() int {
143 - w.lk.RLock()
144 - defer w.lk.RUnlock()
145 - return len(w.set)
146 -}
147 -
148 -func (w *Wantlist) Len() int {
149 - return len(w.set)
150 -}
151 -
152 -func (w *Wantlist) Add(c *cid.Cid, priority int) bool {
153 - k := c.KeyString()
154 - if _, ok := w.set[k]; ok {
155 - return false
156 - }
157 -
158 - w.set[k] = &Entry{
159 - Cid: c,
160 - Priority: priority,
161 - }
162 -
163 - return true
164 -}
165 -
166 -func (w *Wantlist) AddEntry(e *Entry) bool {
167 - k := e.Cid.KeyString()
168 - if _, ok := w.set[k]; ok {
169 - return false
170 - }
171 - w.set[k] = e
172 - return true
173 -}
174 -
175 -func (w *Wantlist) Remove(c *cid.Cid) bool {
176 - k := c.KeyString()
177 - _, ok := w.set[k]
178 - if !ok {
179 - return false
180 - }
181 -
182 - delete(w.set, k)
183 - return true
184 -}
185 -
186 -func (w *Wantlist) Contains(k *cid.Cid) (*Entry, bool) {
187 - e, ok := w.set[k.KeyString()]
188 - return e, ok
189 -}
190 -
191 -func (w *Wantlist) Entries() []*Entry {
192 - es := make([]*Entry, 0, len(w.set))
193 - for _, e := range w.set {
194 - es = append(es, e)
195 - }
196 - return es
197 -}
198 -
199 -func (w *Wantlist) SortedEntries() []*Entry {
200 - es := w.Entries()
201 - sort.Sort(entrySlice(es))
202 - return es
203 -}
exchange/bitswap/wantlist/wantlist_test.go deleted
-104
@@ -1,104 +0,0 @@
1 -package wantlist
2 -
3 -import (
4 - "testing"
5 -
6 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
7 -)
8 -
9 -var testcids []*cid.Cid
10 -
11 -func init() {
12 - strs := []string{
13 - "QmQL8LqkEgYXaDHdNYCG2mmpow7Sp8Z8Kt3QS688vyBeC7",
14 - "QmcBDsdjgSXU7BP4A4V8LJCXENE5xVwnhrhRGVTJr9YCVj",
15 - "QmQakgd2wDxc3uUF4orGdEm28zUT9Mmimp5pyPG2SFS9Gj",
16 - }
17 - for _, s := range strs {
18 - c, err := cid.Decode(s)
19 - if err != nil {
20 - panic(err)
21 - }
22 - testcids = append(testcids, c)
23 - }
24 -
25 -}
26 -
27 -type wli interface {
28 - Contains(*cid.Cid) (*Entry, bool)
29 -}
30 -
31 -func assertHasCid(t *testing.T, w wli, c *cid.Cid) {
32 - e, ok := w.Contains(c)
33 - if !ok {
34 - t.Fatal("expected to have ", c)
35 - }
36 - if !e.Cid.Equals(c) {
37 - t.Fatal("returned entry had wrong cid value")
38 - }
39 -}
40 -
41 -func assertNotHasCid(t *testing.T, w wli, c *cid.Cid) {
42 - _, ok := w.Contains(c)
43 - if ok {
44 - t.Fatal("expected not to have ", c)
45 - }
46 -}
47 -
48 -func TestBasicWantlist(t *testing.T) {
49 - wl := New()
50 -
51 - if !wl.Add(testcids[0], 5) {
52 - t.Fatal("expected true")
53 - }
54 - assertHasCid(t, wl, testcids[0])
55 - if !wl.Add(testcids[1], 4) {
56 - t.Fatal("expected true")
57 - }
58 - assertHasCid(t, wl, testcids[0])
59 - assertHasCid(t, wl, testcids[1])
60 -
61 - if wl.Len() != 2 {
62 - t.Fatal("should have had two items")
63 - }
64 -
65 - if wl.Add(testcids[1], 4) {
66 - t.Fatal("add shouldnt report success on second add")
67 - }
68 - assertHasCid(t, wl, testcids[0])
69 - assertHasCid(t, wl, testcids[1])
70 -
71 - if wl.Len() != 2 {
72 - t.Fatal("should have had two items")
73 - }
74 -
75 - if !wl.Remove(testcids[0]) {
76 - t.Fatal("should have gotten true")
77 - }
78 -
79 - assertHasCid(t, wl, testcids[1])
80 - if _, has := wl.Contains(testcids[0]); has {
81 - t.Fatal("shouldnt have this cid")
82 - }
83 -}
84 -
85 -func TestSesRefWantlist(t *testing.T) {
86 - wl := NewThreadSafe()
87 -
88 - if !wl.Add(testcids[0], 5, 1) {
89 - t.Fatal("should have added")
90 - }
91 - assertHasCid(t, wl, testcids[0])
92 - if wl.Remove(testcids[0], 2) {
93 - t.Fatal("shouldnt have removed")
94 - }
95 - assertHasCid(t, wl, testcids[0])
96 - if wl.Add(testcids[0], 5, 1) {
97 - t.Fatal("shouldnt have added")
98 - }
99 - assertHasCid(t, wl, testcids[0])
100 - if !wl.Remove(testcids[0], 1) {
101 - t.Fatal("should have removed")
102 - }
103 - assertNotHasCid(t, wl, testcids[0])
104 -}
exchange/bitswap/wantmanager.go deleted
-400
@@ -1,400 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "sync"
6 - "time"
7 -
8 - engine "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
9 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
10 - bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
11 - wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
12 -
13 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
14 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
15 - metrics "gx/ipfs/QmekzFM3hPZjTjUFGTABdQkEnQ3PTiMstY198PwSFr5w1Q/go-metrics-interface"
16 -)
17 -
18 -type WantManager struct {
19 - // sync channels for Run loop
20 - incoming chan *wantSet
21 - connectEvent chan peerStatus // notification channel for peers connecting/disconnecting
22 - peerReqs chan chan []peer.ID // channel to request connected peers on
23 -
24 - // synchronized by Run loop, only touch inside there
25 - peers map[peer.ID]*msgQueue
26 - wl *wantlist.ThreadSafe
27 - bcwl *wantlist.ThreadSafe
28 -
29 - network bsnet.BitSwapNetwork
30 - ctx context.Context
31 - cancel func()
32 -
33 - wantlistGauge metrics.Gauge
34 - sentHistogram metrics.Histogram
35 -}
36 -
37 -type peerStatus struct {
38 - connect bool
39 - peer peer.ID
40 -}
41 -
42 -func NewWantManager(ctx context.Context, network bsnet.BitSwapNetwork) *WantManager {
43 - ctx, cancel := context.WithCancel(ctx)
44 - wantlistGauge := metrics.NewCtx(ctx, "wantlist_total",
45 - "Number of items in wantlist.").Gauge()
46 - sentHistogram := metrics.NewCtx(ctx, "sent_all_blocks_bytes", "Histogram of blocks sent by"+
47 - " this bitswap").Histogram(metricsBuckets)
48 - return &WantManager{
49 - incoming: make(chan *wantSet, 10),
50 - connectEvent: make(chan peerStatus, 10),
51 - peerReqs: make(chan chan []peer.ID),
52 - peers: make(map[peer.ID]*msgQueue),
53 - wl: wantlist.NewThreadSafe(),
54 - bcwl: wantlist.NewThreadSafe(),
55 - network: network,
56 - ctx: ctx,
57 - cancel: cancel,
58 - wantlistGauge: wantlistGauge,
59 - sentHistogram: sentHistogram,
60 - }
61 -}
62 -
63 -type msgQueue struct {
64 - p peer.ID
65 -
66 - outlk sync.Mutex
67 - out bsmsg.BitSwapMessage
68 - network bsnet.BitSwapNetwork
69 - wl *wantlist.ThreadSafe
70 -
71 - sender bsnet.MessageSender
72 -
73 - refcnt int
74 -
75 - work chan struct{}
76 - done chan struct{}
77 -}
78 -
79 -// WantBlocks adds the given cids to the wantlist, tracked by the given session
80 -func (pm *WantManager) WantBlocks(ctx context.Context, ks []*cid.Cid, peers []peer.ID, ses uint64) {
81 - log.Infof("want blocks: %s", ks)
82 - pm.addEntries(ctx, ks, peers, false, ses)
83 -}
84 -
85 -// CancelWants removes the given cids from the wantlist, tracked by the given session
86 -func (pm *WantManager) CancelWants(ctx context.Context, ks []*cid.Cid, peers []peer.ID, ses uint64) {
87 - pm.addEntries(context.Background(), ks, peers, true, ses)
88 -}
89 -
90 -type wantSet struct {
91 - entries []*bsmsg.Entry
92 - targets []peer.ID
93 - from uint64
94 -}
95 -
96 -func (pm *WantManager) addEntries(ctx context.Context, ks []*cid.Cid, targets []peer.ID, cancel bool, ses uint64) {
97 - entries := make([]*bsmsg.Entry, 0, len(ks))
98 - for i, k := range ks {
99 - entries = append(entries, &bsmsg.Entry{
100 - Cancel: cancel,
101 - Entry: wantlist.NewRefEntry(k, kMaxPriority-i),
102 - })
103 - }
104 - select {
105 - case pm.incoming <- &wantSet{entries: entries, targets: targets, from: ses}:
106 - case <-pm.ctx.Done():
107 - case <-ctx.Done():
108 - }
109 -}
110 -
111 -func (pm *WantManager) ConnectedPeers() []peer.ID {
112 - resp := make(chan []peer.ID)
113 - pm.peerReqs <- resp
114 - return <-resp
115 -}
116 -
117 -func (pm *WantManager) SendBlock(ctx context.Context, env *engine.Envelope) {
118 - // Blocks need to be sent synchronously to maintain proper backpressure
119 - // throughout the network stack
120 - defer env.Sent()
121 -
122 - pm.sentHistogram.Observe(float64(len(env.Block.RawData())))
123 -
124 - msg := bsmsg.New(false)
125 - msg.AddBlock(env.Block)
126 - log.Infof("Sending block %s to %s", env.Block, env.Peer)
127 - err := pm.network.SendMessage(ctx, env.Peer, msg)
128 - if err != nil {
129 - log.Infof("sendblock error: %s", err)
130 - }
131 -}
132 -
133 -func (pm *WantManager) startPeerHandler(p peer.ID) *msgQueue {
134 - mq, ok := pm.peers[p]
135 - if ok {
136 - mq.refcnt++
137 - return nil
138 - }
139 -
140 - mq = pm.newMsgQueue(p)
141 -
142 - // new peer, we will want to give them our full wantlist
143 - fullwantlist := bsmsg.New(true)
144 - for _, e := range pm.bcwl.Entries() {
145 - for k := range e.SesTrk {
146 - mq.wl.AddEntry(e, k)
147 - }
148 - fullwantlist.AddEntry(e.Cid, e.Priority)
149 - }
150 - mq.out = fullwantlist
151 - mq.work <- struct{}{}
152 -
153 - pm.peers[p] = mq
154 - go mq.runQueue(pm.ctx)
155 - return mq
156 -}
157 -
158 -func (pm *WantManager) stopPeerHandler(p peer.ID) {
159 - pq, ok := pm.peers[p]
160 - if !ok {
161 - // TODO: log error?
162 - return
163 - }
164 -
165 - pq.refcnt--
166 - if pq.refcnt > 0 {
167 - return
168 - }
169 -
170 - close(pq.done)
171 - delete(pm.peers, p)
172 -}
173 -
174 -func (mq *msgQueue) runQueue(ctx context.Context) {
175 - for {
176 - select {
177 - case <-mq.work: // there is work to be done
178 - mq.doWork(ctx)
179 - case <-mq.done:
180 - if mq.sender != nil {
181 - mq.sender.Close()
182 - }
183 - return
184 - case <-ctx.Done():
185 - if mq.sender != nil {
186 - mq.sender.Reset()
187 - }
188 - return
189 - }
190 - }
191 -}
192 -
193 -func (mq *msgQueue) doWork(ctx context.Context) {
194 - // grab outgoing message
195 - mq.outlk.Lock()
196 - wlm := mq.out
197 - if wlm == nil || wlm.Empty() {
198 - mq.outlk.Unlock()
199 - return
200 - }
201 - mq.out = nil
202 - mq.outlk.Unlock()
203 -
204 - // NB: only open a stream if we actually have data to send
205 - if mq.sender == nil {
206 - err := mq.openSender(ctx)
207 - if err != nil {
208 - log.Infof("cant open message sender to peer %s: %s", mq.p, err)
209 - // TODO: cant connect, what now?
210 - return
211 - }
212 - }
213 -
214 - // send wantlist updates
215 - for { // try to send this message until we fail.
216 - err := mq.sender.SendMsg(ctx, wlm)
217 - if err == nil {
218 - return
219 - }
220 -
221 - log.Infof("bitswap send error: %s", err)
222 - mq.sender.Reset()
223 - mq.sender = nil
224 -
225 - select {
226 - case <-mq.done:
227 - return
228 - case <-ctx.Done():
229 - return
230 - case <-time.After(time.Millisecond * 100):
231 - // wait 100ms in case disconnect notifications are still propogating
232 - log.Warning("SendMsg errored but neither 'done' nor context.Done() were set")
233 - }
234 -
235 - err = mq.openSender(ctx)
236 - if err != nil {
237 - log.Infof("couldnt open sender again after SendMsg(%s) failed: %s", mq.p, err)
238 - // TODO(why): what do we do now?
239 - // I think the *right* answer is to probably put the message we're
240 - // trying to send back, and then return to waiting for new work or
241 - // a disconnect.
242 - return
243 - }
244 -
245 - // TODO: Is this the same instance for the remote peer?
246 - // If its not, we should resend our entire wantlist to them
247 - /*
248 - if mq.sender.InstanceID() != mq.lastSeenInstanceID {
249 - wlm = mq.getFullWantlistMessage()
250 - }
251 - */
252 - }
253 -}
254 -
255 -func (mq *msgQueue) openSender(ctx context.Context) error {
256 - // allow ten minutes for connections this includes looking them up in the
257 - // dht dialing them, and handshaking
258 - conctx, cancel := context.WithTimeout(ctx, time.Minute*10)
259 - defer cancel()
260 -
261 - err := mq.network.ConnectTo(conctx, mq.p)
262 - if err != nil {
263 - return err
264 - }
265 -
266 - nsender, err := mq.network.NewMessageSender(ctx, mq.p)
267 - if err != nil {
268 - return err
269 - }
270 -
271 - mq.sender = nsender
272 - return nil
273 -}
274 -
275 -func (pm *WantManager) Connected(p peer.ID) {
276 - select {
277 - case pm.connectEvent <- peerStatus{peer: p, connect: true}:
278 - case <-pm.ctx.Done():
279 - }
280 -}
281 -
282 -func (pm *WantManager) Disconnected(p peer.ID) {
283 - select {
284 - case pm.connectEvent <- peerStatus{peer: p, connect: false}:
285 - case <-pm.ctx.Done():
286 - }
287 -}
288 -
289 -// TODO: use goprocess here once i trust it
290 -func (pm *WantManager) Run() {
291 - // NOTE: Do not open any streams or connections from anywhere in this
292 - // event loop. Really, just don't do anything likely to block.
293 - for {
294 - select {
295 - case ws := <-pm.incoming:
296 -
297 - // is this a broadcast or not?
298 - brdc := len(ws.targets) == 0
299 -
300 - // add changes to our wantlist
301 - for _, e := range ws.entries {
302 - if e.Cancel {
303 - if brdc {
304 - pm.bcwl.Remove(e.Cid, ws.from)
305 - }
306 -
307 - if pm.wl.Remove(e.Cid, ws.from) {
308 - pm.wantlistGauge.Dec()
309 - }
310 - } else {
311 - if brdc {
312 - pm.bcwl.AddEntry(e.Entry, ws.from)
313 - }
314 - if pm.wl.AddEntry(e.Entry, ws.from) {
315 - pm.wantlistGauge.Inc()
316 - }
317 - }
318 - }
319 -
320 - // broadcast those wantlist changes
321 - if len(ws.targets) == 0 {
322 - for _, p := range pm.peers {
323 - p.addMessage(ws.entries, ws.from)
324 - }
325 - } else {
326 - for _, t := range ws.targets {
327 - p, ok := pm.peers[t]
328 - if !ok {
329 - log.Infof("tried sending wantlist change to non-partner peer: %s", t)
330 - continue
331 - }
332 - p.addMessage(ws.entries, ws.from)
333 - }
334 - }
335 -
336 - case p := <-pm.connectEvent:
337 - if p.connect {
338 - pm.startPeerHandler(p.peer)
339 - } else {
340 - pm.stopPeerHandler(p.peer)
341 - }
342 - case req := <-pm.peerReqs:
343 - peers := make([]peer.ID, 0, len(pm.peers))
344 - for p := range pm.peers {
345 - peers = append(peers, p)
346 - }
347 - req <- peers
348 - case <-pm.ctx.Done():
349 - return
350 - }
351 - }
352 -}
353 -
354 -func (wm *WantManager) newMsgQueue(p peer.ID) *msgQueue {
355 - return &msgQueue{
356 - done: make(chan struct{}),
357 - work: make(chan struct{}, 1),
358 - wl: wantlist.NewThreadSafe(),
359 - network: wm.network,
360 - p: p,
361 - refcnt: 1,
362 - }
363 -}
364 -
365 -func (mq *msgQueue) addMessage(entries []*bsmsg.Entry, ses uint64) {
366 - var work bool
367 - mq.outlk.Lock()
368 - defer func() {
369 - mq.outlk.Unlock()
370 - if !work {
371 - return
372 - }
373 - select {
374 - case mq.work <- struct{}{}:
375 - default:
376 - }
377 - }()
378 -
379 - // if we have no message held allocate a new one
380 - if mq.out == nil {
381 - mq.out = bsmsg.New(false)
382 - }
383 -
384 - // TODO: add a msg.Combine(...) method
385 - // otherwise, combine the one we are holding with the
386 - // one passed in
387 - for _, e := range entries {
388 - if e.Cancel {
389 - if mq.wl.Remove(e.Cid, ses) {
390 - work = true
391 - mq.out.Cancel(e.Cid)
392 - }
393 - } else {
394 - if mq.wl.Add(e.Cid, e.Priority, ses) {
395 - work = true
396 - mq.out.AddEntry(e.Cid, e.Priority)
397 - }
398 - }
399 - }
400 -}
exchange/bitswap/workers.go deleted
-253
@@ -1,253 +0,0 @@
1 -package bitswap
2 -
3 -import (
4 - "context"
5 - "math/rand"
6 - "sync"
7 - "time"
8 -
9 - bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
10 -
11 - process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess"
12 - procctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context"
13 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
14 - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
15 - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer"
16 -)
17 -
18 -var TaskWorkerCount = 8
19 -
20 -func (bs *Bitswap) startWorkers(px process.Process, ctx context.Context) {
21 - // Start up a worker to handle block requests this node is making
22 - px.Go(func(px process.Process) {
23 - bs.providerQueryManager(ctx)
24 - })
25 -
26 - // Start up workers to handle requests from other nodes for the data on this node
27 - for i := 0; i < TaskWorkerCount; i++ {
28 - i := i
29 - px.Go(func(px process.Process) {
30 - bs.taskWorker(ctx, i)
31 - })
32 - }
33 -
34 - // Start up a worker to manage periodically resending our wantlist out to peers
35 - px.Go(func(px process.Process) {
36 - bs.rebroadcastWorker(ctx)
37 - })
38 -
39 - // Start up a worker to manage sending out provides messages
40 - px.Go(func(px process.Process) {
41 - bs.provideCollector(ctx)
42 - })
43 -
44 - // Spawn up multiple workers to handle incoming blocks
45 - // consider increasing number if providing blocks bottlenecks
46 - // file transfers
47 - px.Go(bs.provideWorker)
48 -}
49 -
50 -func (bs *Bitswap) taskWorker(ctx context.Context, id int) {
51 - idmap := logging.LoggableMap{"ID": id}
52 - defer log.Debug("bitswap task worker shutting down...")
53 - for {
54 - log.Event(ctx, "Bitswap.TaskWorker.Loop", idmap)
55 - select {
56 - case nextEnvelope := <-bs.engine.Outbox():
57 - select {
58 - case envelope, ok := <-nextEnvelope:
59 - if !ok {
60 - continue
61 - }
62 - log.Event(ctx, "Bitswap.TaskWorker.Work", logging.LoggableF(func() map[string]interface{} {
63 - return logging.LoggableMap{
64 - "ID": id,
65 - "Target": envelope.Peer.Pretty(),
66 - "Block": envelope.Block.Cid().String(),
67 - }
68 - }))
69 -
70 - // update the BS ledger to reflect sent message
71 - // TODO: Should only track *useful* messages in ledger
72 - outgoing := bsmsg.New(false)
73 - outgoing.AddBlock(envelope.Block)
74 - bs.engine.MessageSent(envelope.Peer, outgoing)
75 -
76 - bs.wm.SendBlock(ctx, envelope)
77 - bs.counterLk.Lock()
78 - bs.counters.blocksSent++
79 - bs.counters.dataSent += uint64(len(envelope.Block.RawData()))
80 - bs.counterLk.Unlock()
81 - case <-ctx.Done():
82 - return
83 - }
84 - case <-ctx.Done():
85 - return
86 - }
87 - }
88 -}
89 -
90 -func (bs *Bitswap) provideWorker(px process.Process) {
91 -
92 - limit := make(chan struct{}, provideWorkerMax)
93 -
94 - limitedGoProvide := func(k *cid.Cid, wid int) {
95 - defer func() {
96 - // replace token when done
97 - <-limit
98 - }()
99 - ev := logging.LoggableMap{"ID": wid}
100 -
101 - ctx := procctx.OnClosingContext(px) // derive ctx from px
102 - defer log.EventBegin(ctx, "Bitswap.ProvideWorker.Work", ev, k).Done()
103 -
104 - ctx, cancel := context.WithTimeout(ctx, provideTimeout) // timeout ctx
105 - defer cancel()
106 -
107 - if err := bs.network.Provide(ctx, k); err != nil {
108 - log.Warning(err)
109 - }
110 - }
111 -
112 - // worker spawner, reads from bs.provideKeys until it closes, spawning a
113 - // _ratelimited_ number of workers to handle each key.
114 - for wid := 2; ; wid++ {
115 - ev := logging.LoggableMap{"ID": 1}
116 - log.Event(procctx.OnClosingContext(px), "Bitswap.ProvideWorker.Loop", ev)
117 -
118 - select {
119 - case <-px.Closing():
120 - return
121 - case k, ok := <-bs.provideKeys:
122 - if !ok {
123 - log.Debug("provideKeys channel closed")
124 - return
125 - }
126 - select {
127 - case <-px.Closing():
128 - return
129 - case limit <- struct{}{}:
130 - go limitedGoProvide(k, wid)
131 - }
132 - }
133 - }
134 -}
135 -
136 -func (bs *Bitswap) provideCollector(ctx context.Context) {
137 - defer close(bs.provideKeys)
138 - var toProvide []*cid.Cid
139 - var nextKey *cid.Cid
140 - var keysOut chan *cid.Cid
141 -
142 - for {
143 - select {
144 - case blkey, ok := <-bs.newBlocks:
145 - if !ok {
146 - log.Debug("newBlocks channel closed")
147 - return
148 - }
149 -
150 - if keysOut == nil {
151 - nextKey = blkey
152 - keysOut = bs.provideKeys
153 - } else {
154 - toProvide = append(toProvide, blkey)
155 - }
156 - case keysOut <- nextKey:
157 - if len(toProvide) > 0 {
158 - nextKey = toProvide[0]
159 - toProvide = toProvide[1:]
160 - } else {
161 - keysOut = nil
162 - }
163 - case <-ctx.Done():
164 - return
165 - }
166 - }
167 -}
168 -
169 -func (bs *Bitswap) rebroadcastWorker(parent context.Context) {
170 - ctx, cancel := context.WithCancel(parent)
171 - defer cancel()
172 -
173 - broadcastSignal := time.NewTicker(rebroadcastDelay.Get())
174 - defer broadcastSignal.Stop()
175 -
176 - tick := time.NewTicker(10 * time.Second)
177 - defer tick.Stop()
178 -
179 - for {
180 - log.Event(ctx, "Bitswap.Rebroadcast.idle")
181 - select {
182 - case <-tick.C:
183 - n := bs.wm.wl.Len()
184 - if n > 0 {
185 - log.Debug(n, " keys in bitswap wantlist")
186 - }
187 - case <-broadcastSignal.C: // resend unfulfilled wantlist keys
188 - log.Event(ctx, "Bitswap.Rebroadcast.active")
189 - entries := bs.wm.wl.Entries()
190 - if len(entries) == 0 {
191 - continue
192 - }
193 -
194 - // TODO: come up with a better strategy for determining when to search
195 - // for new providers for blocks.
196 - i := rand.Intn(len(entries))
197 - bs.findKeys <- &blockRequest{
198 - Cid: entries[i].Cid,
199 - Ctx: ctx,
200 - }
201 - case <-parent.Done():
202 - return
203 - }
204 - }
205 -}
206 -
207 -func (bs *Bitswap) providerQueryManager(ctx context.Context) {
208 - var activeLk sync.Mutex
209 - kset := cid.NewSet()
210 -
211 - for {
212 - select {
213 - case e := <-bs.findKeys:
214 - select { // make sure its not already cancelled
215 - case <-e.Ctx.Done():
216 - continue
217 - default:
218 - }
219 -
220 - activeLk.Lock()
221 - if kset.Has(e.Cid) {
222 - activeLk.Unlock()
223 - continue
224 - }
225 - kset.Add(e.Cid)
226 - activeLk.Unlock()
227 -
228 - go func(e *blockRequest) {
229 - child, cancel := context.WithTimeout(e.Ctx, providerRequestTimeout)
230 - defer cancel()
231 - providers := bs.network.FindProvidersAsync(child, e.Cid, maxProvidersPerRequest)
232 - wg := &sync.WaitGroup{}
233 - for p := range providers {
234 - wg.Add(1)
235 - go func(p peer.ID) {
236 - defer wg.Done()
237 - err := bs.network.ConnectTo(child, p)
238 - if err != nil {
239 - log.Debug("failed to connect to provider %s: %s", p, err)
240 - }
241 - }(p)
242 - }
243 - wg.Wait()
244 - activeLk.Lock()
245 - kset.Remove(e.Cid)
246 - activeLk.Unlock()
247 - }(e)
248 -
249 - case <-ctx.Done():
250 - return
251 - }
252 - }
253 -}
package.json
+6
@@ -545,6 +545,12 @@
545 "hash": "Qmdue1XShFNi3mpizGx9NR9hyNEj6U2wEW93yGhKqKCFGN",
546 "name": "go-ipns",
547 "version": "0.1.4"
548 + },
549 + {
550 + "author": "why",
551 + "hash": "QmSLYFS88MpPsszqWdhGSxvHyoTnmaU4A74SD6KGib6Z3m",
552 + "name": "go-bitswap",
553 + "version": "1.0.0"
554 }
555 ],
556 "gxVersion": "0.10.0",