some bitswap cleanup
Jeromy committed
Nov 26, 2014 at 22:50 UTC
f0a4fdad59418ec631d23843ed1f23c8351c1c1e
5 files changed
+54
-52
core/core.go
+1
@@ -31,6 +31,7 @@ import (
31
u "github.com/jbenet/go-ipfs/util"
32
ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
33
debugerror "github.com/jbenet/go-ipfs/util/debugerror"
34
+ "github.com/jbenet/go-ipfs/util/eventlog"
35
)
36
37
const IpnsValidatorTag = "ipns"
exchange/bitswap/bitswap.go
+42
-21
@@ -16,11 +16,14 @@ import (
16
strategy "github.com/jbenet/go-ipfs/exchange/bitswap/strategy"
17
peer "github.com/jbenet/go-ipfs/peer"
18
u "github.com/jbenet/go-ipfs/util"
19
- "github.com/jbenet/go-ipfs/util/eventlog"
19
+ eventlog "github.com/jbenet/go-ipfs/util/eventlog"
20
)
21
22
var log = eventlog.Logger("bitswap")
23
24
+// Number of providers to request for sending a wantlist to
25
+const maxProvidersPerRequest = 6
26
+
27
// New initializes a BitSwap instance that communicates over the
28
// provided BitSwapNetwork. This function registers the returned instance as
29
// the network delegate.
@@ -97,7 +100,7 @@ func (bs *bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, err
100
101
ctx, cancelFunc := context.WithCancel(parent)
102
100
- ctx = eventlog.ContextWithMetadata(ctx, eventlog.Uuid("GetBlockRequest"))
103
+ ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
104
log.Event(ctx, "GetBlockRequestBegin", &k)
105
106
defer func() {
@@ -176,14 +179,29 @@ func (bs *bitswap) sendWantListTo(ctx context.Context, peers <-chan peer.Peer) e
179
return nil
180
}
181
182
+func (bs *bitswap) sendWantlistToProviders(ctx context.Context, ks []u.Key) {
183
+ done := make(chan struct{})
184
+ for _, k := range ks {
185
+ go func(k u.Key) {
186
+ providers := bs.routing.FindProvidersAsync(ctx, k, maxProvidersPerRequest)
187
+
188
+ err := bs.sendWantListTo(ctx, providers)
189
+ if err != nil {
190
+ log.Errorf("error sending wantlist: %s", err)
191
+ }
192
+ done <- struct{}{}
193
+ }(k)
194
+ }
195
+ for _ = range ks {
196
+ <-done
197
+ }
198
+}
199
+
200
// TODO ensure only one active request per key
201
func (bs *bitswap) loop(parent context.Context) {
202
203
ctx, cancel := context.WithCancel(parent)
204
184
- // Every so often, we should resend out our current want list
185
- rebroadcastTime := time.Second * 5
186
-
205
broadcastSignal := time.NewTicker(bs.strategy.GetRebroadcastDelay())
206
defer func() {
207
cancel() // signal to derived async functions
@@ -193,15 +211,12 @@ func (bs *bitswap) loop(parent context.Context) {
211
for {
212
select {
213
case <-broadcastSignal.C:
196
- for _, k := range bs.wantlist.Keys() {
197
- providers := bs.routing.FindProvidersAsync(ctx, k, maxProvidersPerRequest)
198
- err := bs.sendWantListTo(ctx, providers)
199
- if err != nil {
200
- log.Errorf("error sending wantlist: %s", err)
201
- }
202
- }
214
+ bs.sendWantlistToProviders(ctx, bs.wantlist.Keys())
215
case ks := <-bs.batchRequests:
216
// TODO: implement batching on len(ks) > X for some X
217
+ // i.e. if given 20 keys, fetch first five, then next
218
+ // five, and so on, so we are more likely to be able to
219
+ // effectively stream the data
220
if len(ks) == 0 {
221
log.Warning("Received batch request for zero blocks")
222
continue
@@ -232,6 +247,18 @@ func (bs *bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
247
return bs.routing.Provide(ctx, blk.Key())
248
}
249
250
+func (bs *bitswap) receiveBlock(ctx context.Context, block *blocks.Block) {
251
+ // TODO verify blocks?
252
+ if err := bs.blockstore.Put(block); err != nil {
253
+ log.Criticalf("error putting block: %s", err)
254
+ return
255
+ }
256
+ err := bs.HasBlock(ctx, block)
257
+ if err != nil {
258
+ log.Warningf("HasBlock errored: %s", err)
259
+ }
260
+}
261
+
262
// TODO(brian): handle errors
263
func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.Peer, incoming bsmsg.BitSwapMessage) (
264
peer.Peer, bsmsg.BitSwapMessage) {
@@ -255,15 +282,7 @@ func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.Peer, incoming bsm
282
bs.strategy.MessageReceived(p, incoming) // FIRST
283
284
for _, block := range incoming.Blocks() {
258
- // TODO verify blocks?
259
- if err := bs.blockstore.Put(block); err != nil {
260
- log.Criticalf("error putting block: %s", err)
261
- continue // FIXME(brian): err ignored
262
- }
263
- err := bs.HasBlock(ctx, block)
264
- if err != nil {
265
- log.Warningf("HasBlock errored: %s", err)
266
- }
285
+ go bs.receiveBlock(ctx, block)
286
}
287
288
for _, key := range incoming.Wantlist() {
@@ -277,6 +296,8 @@ func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.Peer, incoming bsm
296
blkmsg := bsmsg.New()
297
298
// TODO: only send this the first time
299
+ // no sense in sending our wantlist to the
300
+ // same peer multiple times
301
for _, k := range bs.wantlist.Keys() {
302
blkmsg.AddWanted(k)
303
}
exchange/bitswap/strategy/strategy.go
+1
-1
@@ -148,5 +148,5 @@ func (s *strategist) GetBatchSize() int {
148
}
149
150
func (s *strategist) GetRebroadcastDelay() time.Duration {
151
- return time.Second * 2
151
+ return time.Second * 5
152
}
merkledag/merkledag.go
+9
-29
@@ -28,7 +28,7 @@ type DAGService interface {
28
AddRecursive(*Node) error
29
Get(u.Key) (*Node, error)
30
Remove(*Node) error
31
- BatchFetch(context.Context, *Node) <-chan *Node
31
+ GetKeysAsync(context.Context, *Node) <-chan *Node
32
}
33
34
func NewDAGService(bs *bserv.BlockService) DAGService {
@@ -298,41 +298,33 @@ func FindLink(n *Node, k u.Key, found []*Node) (int, error) {
298
return -1, u.ErrNotFound
299
}
300
301
-// BatchFetch will fill out all of the links of the given Node.
301
+// GetKeysAsync will fill out all of the links of the given Node.
302
// It returns a channel of nodes, which the caller can receive
303
// all the child nodes of 'root' on, in proper order.
304
-func (ds *dagService) BatchFetch(ctx context.Context, root *Node) <-chan *Node {
304
+func (ds *dagService) GetKeysAsync(ctx context.Context, root *Node) <-chan *Node {
305
sig := make(chan *Node)
306
go func() {
307
var keys []u.Key
308
nodes := make([]*Node, len(root.Links))
309
310
- //temp
311
- recvd := []int{}
312
- //
313
-
314
- //
315
- next := 0
316
- //
317
-
310
for _, lnk := range root.Links {
311
keys = append(keys, u.Key(lnk.Hash))
312
}
313
314
blkchan := ds.Blocks.GetBlocks(ctx, keys)
315
324
- count := 0
316
+ next := 0
317
for blk := range blkchan {
326
- count++
318
i, err := FindLink(root, blk.Key(), nodes)
319
if err != nil {
320
+ // NB: can only occur as a result of programmer error
321
panic("Received block that wasnt in this nodes links!")
322
}
323
332
- recvd = append(recvd, i)
333
-
324
nd, err := Decoded(blk.Data)
325
if err != nil {
326
+ // NB: can occur in normal situations, with improperly formatted
327
+ // input data
328
log.Error("Got back bad block!")
329
break
330
}
@@ -347,23 +339,11 @@ func (ds *dagService) BatchFetch(ctx context.Context, root *Node) <-chan *Node {
339
}
340
}
341
if next < len(nodes) {
350
- log.Errorf("count = %d, links = %d", count, len(nodes))
351
- log.Error(recvd)
352
- panic("didnt receive all requested blocks!")
342
+ // TODO: bubble errors back up.
343
+ log.Errorf("Did not receive correct number of nodes!")
344
}
345
close(sig)
346
}()
347
348
return sig
349
}
359
-
360
-func checkForDupes(ks []u.Key) bool {
361
- seen := make(map[u.Key]struct{})
362
- for _, k := range ks {
363
- if _, ok := seen[k]; ok {
364
- return true
365
- }
366
- seen[k] = struct{}{}
367
- }
368
- return false
369
-}
unixfs/io/dagreader.go
+1
-1
@@ -40,7 +40,7 @@ func NewDagReader(n *mdag.Node, serv mdag.DAGService) (io.Reader, error) {
40
case ftpb.Data_File:
41
var fetchChan <-chan *mdag.Node
42
if serv != nil {
43
- fetchChan = serv.BatchFetch(context.TODO(), n)
43
+ fetchChan = serv.GetKeysAsync(context.TODO(), n)
44
}
45
return &DagReader{
46
node: n,