revamp BatchFetch a bit
Jeromy committed
Nov 21, 2014 at 05:38 UTC
0ce6071fea1ddd86aac327f47b2961f517cc3eaf
2 files changed
+53
-52
merkledag/merkledag.go
+34
-37
@@ -22,6 +22,19 @@ var ErrNotFound = fmt.Errorf("merkledag: not found")
22
// so have to convert Multihash bytes to string (u.Key)
23
type NodeMap map[u.Key]*Node
24
25
+// DAGService is an IPFS Merkle DAG service.
26
+type DAGService interface {
27
+ Add(*Node) (u.Key, error)
28
+ AddRecursive(*Node) error
29
+ Get(u.Key) (*Node, error)
30
+ Remove(*Node) error
31
+ BatchFetch(context.Context, *Node) <-chan *Node
32
+}
33
+
34
+func NewDAGService(bs *bserv.BlockService) DAGService {
35
+ return &dagService{bs}
36
+}
37
+
38
// Node represents a node in the IPFS Merkle DAG.
39
// nodes have opaque data and a set of navigable links.
40
type Node struct {
@@ -156,18 +169,6 @@ func (n *Node) Key() (u.Key, error) {
169
return u.Key(h), err
170
}
171
159
-// DAGService is an IPFS Merkle DAG service.
160
-type DAGService interface {
161
- Add(*Node) (u.Key, error)
162
- AddRecursive(*Node) error
163
- Get(u.Key) (*Node, error)
164
- Remove(*Node) error
165
-}
166
-
167
-func NewDAGService(bs *bserv.BlockService) DAGService {
168
- return &dagService{bs}
169
-}
170
-
172
// dagService is an IPFS Merkle DAG service.
173
// - the root is virtual (like a forest)
174
// - stores nodes' data in a BlockService
@@ -286,59 +287,55 @@ func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{}
287
return done
288
}
289
289
-// Take advantage of blockservice/bitswap batched requests to fetch all
290
-// child nodes of a given node
291
-// TODO: finish this
292
-func (ds *dagService) BatchFetch(ctx context.Context, root *Node) <-chan int {
293
- sig := make(chan int)
290
+// BatchFetch will fill out all of the links of the given Node.
291
+// It returns a channel of indicies, which will be returned in order
292
+// from 0 to len(root.Links) - 1, signalling that the link specified by
293
+// the index has been filled out.
294
+func (ds *dagService) BatchFetch(ctx context.Context, root *Node) <-chan *Node {
295
+ sig := make(chan *Node)
296
go func() {
297
var keys []u.Key
296
- for _, lnk := range root.Links {
297
- keys = append(keys, u.Key(lnk.Hash))
298
- }
299
-
300
- blkchan := ds.Blocks.GetBlocks(ctx, keys)
298
+ nodes := make([]*Node, len(root.Links))
299
300
//
301
next := 0
302
seen := make(map[int]struct{})
303
//
304
305
+ for _, lnk := range root.Links {
306
+ keys = append(keys, u.Key(lnk.Hash))
307
+ }
308
+
309
+ blkchan := ds.Blocks.GetBlocks(ctx, keys)
310
+
311
for blk := range blkchan {
312
for i, lnk := range root.Links {
313
+ if u.Key(lnk.Hash) != blk.Key() {
314
+ continue
315
+ }
316
317
//
318
seen[i] = struct{}{}
319
//
320
314
- if u.Key(lnk.Hash) != blk.Key() {
315
- continue
316
- }
321
nd, err := Decoded(blk.Data)
322
if err != nil {
323
log.Error("Got back bad block!")
324
break
325
}
322
- lnk.Node = nd
326
+ nodes[i] = nd
327
324
- //
328
if next == i {
326
- sig <- next
329
+ sig <- nd
330
next++
328
- for {
329
- if _, ok := seen[next]; ok {
330
- sig <- next
331
- next++
332
- } else {
333
- break
334
- }
331
+ for ; nodes[next] != nil; next++ {
332
+ sig <- nodes[next]
333
}
334
}
337
- //
335
}
336
}
337
+ close(sig)
338
}()
339
342
- // TODO: return a channel, and signal when the 'Next' readable block is available
340
return sig
341
}
unixfs/io/dagreader.go
+19
-15
@@ -5,6 +5,8 @@ import (
5
"errors"
6
"io"
7
8
+ "code.google.com/p/go.net/context"
9
+
10
proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
11
mdag "github.com/jbenet/go-ipfs/merkledag"
12
ft "github.com/jbenet/go-ipfs/unixfs"
@@ -15,10 +17,10 @@ var ErrIsDir = errors.New("this dag node is a directory")
17
18
// DagReader provides a way to easily read the data contained in a dag.
19
type DagReader struct {
18
- serv mdag.DAGService
19
- node *mdag.Node
20
- position int
21
- buf io.Reader
20
+ serv mdag.DAGService
21
+ node *mdag.Node
22
+ buf io.Reader
23
+ fetchChan <-chan *mdag.Node
24
}
25
26
// NewDagReader creates a new reader object that reads the data represented by the given
@@ -36,9 +38,10 @@ func NewDagReader(n *mdag.Node, serv mdag.DAGService) (io.Reader, error) {
38
return nil, ErrIsDir
39
case ftpb.Data_File:
40
return &DagReader{
39
- node: n,
40
- serv: serv,
41
- buf: bytes.NewBuffer(pb.GetData()),
41
+ node: n,
42
+ serv: serv,
43
+ buf: bytes.NewBuffer(pb.GetData()),
44
+ fetchChan: serv.BatchFetch(context.TODO(), n),
45
}, nil
46
case ftpb.Data_Raw:
47
// Raw block will just be a single level, return a byte buffer
@@ -51,19 +54,20 @@ func NewDagReader(n *mdag.Node, serv mdag.DAGService) (io.Reader, error) {
54
// precalcNextBuf follows the next link in line and loads it from the DAGService,
55
// setting the next buffer to read from
56
func (dr *DagReader) precalcNextBuf() error {
54
- if dr.position >= len(dr.node.Links) {
55
- return io.EOF
56
- }
57
- nxt, err := dr.node.Links[dr.position].GetNode(dr.serv)
58
- if err != nil {
59
- return err
57
+ var nxt *mdag.Node
58
+ var ok bool
59
+ select {
60
+ case nxt, ok = <-dr.fetchChan:
61
+ if !ok {
62
+ return io.EOF
63
+ }
64
}
65
+
66
pb := new(ftpb.Data)
62
- err = proto.Unmarshal(nxt.Data, pb)
67
+ err := proto.Unmarshal(nxt.Data, pb)
68
if err != nil {
69
return err
70
}
66
- dr.position++
71
72
switch pb.GetType() {
73
case ftpb.Data_Directory: