change batch fetching methods of dagserv
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Feb 20, 2016 at 10:27 UTC
479761ec6a35dd42a23eca6abe3784c2787c41b5
4 files changed
+50
-16
core/commands/refs.go
+1
-1
@@ -233,7 +233,7 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) {
233
}
234
235
var count int
236
- for i, ng := range rw.DAG.GetDAG(rw.Ctx, n) {
236
+ for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) {
237
lk := key.Key(n.Links[i].Hash)
238
if rw.skip(lk) {
239
continue
merkledag/merkledag.go
+47
-13
@@ -24,8 +24,7 @@ type DAGService interface {
24
25
// GetDAG returns, in order, all the single leve child
26
// nodes of the passed in node.
27
- GetDAG(context.Context, *Node) []NodeGetter
28
- GetNodes(context.Context, []key.Key) []NodeGetter
27
+ GetMany(context.Context, []key.Key) (<-chan *Node, <-chan error)
28
29
Batch() *Batch
30
}
@@ -146,21 +145,52 @@ func FindLinks(links []key.Key, k key.Key, start int) []int {
145
return out
146
}
147
148
+func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) (<-chan *Node, <-chan error) {
149
+ out := make(chan *Node)
150
+ errs := make(chan error, 1)
151
+ blocks := ds.Blocks.GetBlocks(ctx, keys)
152
+ go func() {
153
+ defer close(out)
154
+ defer close(errs)
155
+ for {
156
+ select {
157
+ case b, ok := <-blocks:
158
+ if !ok {
159
+ return
160
+ }
161
+ nd, err := Decoded(b.Data)
162
+ if err != nil {
163
+ errs <- err
164
+ return
165
+ }
166
+ select {
167
+ case out <- nd:
168
+ case <-ctx.Done():
169
+ return
170
+ }
171
+ case <-ctx.Done():
172
+ return
173
+ }
174
+ }
175
+ }()
176
+ return out, errs
177
+}
178
+
179
// GetDAG will fill out all of the links of the given Node.
180
// It returns a channel of nodes, which the caller can receive
181
// all the child nodes of 'root' on, in proper order.
152
-func (ds *dagService) GetDAG(ctx context.Context, root *Node) []NodeGetter {
182
+func GetDAG(ctx context.Context, ds DAGService, root *Node) []NodeGetter {
183
var keys []key.Key
184
for _, lnk := range root.Links {
185
keys = append(keys, key.Key(lnk.Hash))
186
}
187
158
- return ds.GetNodes(ctx, keys)
188
+ return GetNodes(ctx, ds, keys)
189
}
190
191
// GetNodes returns an array of 'NodeGetter' promises, with each corresponding
192
// to the key with the same index as the passed in keys
163
-func (ds *dagService) GetNodes(ctx context.Context, keys []key.Key) []NodeGetter {
193
+func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {
194
195
// Early out if no work to do
196
if len(keys) == 0 {
@@ -178,26 +208,29 @@ func (ds *dagService) GetNodes(ctx context.Context, keys []key.Key) []NodeGetter
208
ctx, cancel := context.WithCancel(ctx)
209
defer cancel()
210
181
- blkchan := ds.Blocks.GetBlocks(ctx, dedupedKeys)
211
+ nodechan, errchan := ds.GetMany(ctx, dedupedKeys)
212
213
for count := 0; count < len(keys); {
214
select {
185
- case blk, ok := <-blkchan:
215
+ case nd, ok := <-nodechan:
216
if !ok {
217
return
218
}
219
190
- nd, err := Decoded(blk.Data)
220
+ k, err := nd.Key()
221
if err != nil {
192
- // NB: can happen with improperly formatted input data
193
- log.Debug("Got back bad block!")
194
- return
222
+ log.Error("Failed to get node key: ", err)
223
+ continue
224
}
196
- is := FindLinks(keys, blk.Key(), 0)
225
+
226
+ is := FindLinks(keys, k, 0)
227
for _, i := range is {
228
count++
229
sendChans[i] <- nd
230
}
231
+ case err := <-errchan:
232
+ log.Error("error fetching: ", err)
233
+ return
234
case <-ctx.Done():
235
return
236
}
@@ -389,9 +422,10 @@ func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out cha
422
}
423
424
for ks := range in {
392
- ng := ds.GetNodes(ctx, ks)
425
+ ng := GetNodes(ctx, ds, ks)
426
for _, g := range ng {
427
go get(g)
428
}
429
}
430
+
431
}
unixfs/archive/tar/writer.go
+1
-1
@@ -39,7 +39,7 @@ func (w *Writer) writeDir(nd *mdag.Node, fpath string) error {
39
return err
40
}
41
42
- for i, ng := range w.Dag.GetDAG(w.ctx, nd) {
42
+ for i, ng := range mdag.GetDAG(w.ctx, w.Dag, nd) {
43
child, err := ng.Get(w.ctx)
44
if err != nil {
45
return err
unixfs/io/dagreader.go
+1
-1
@@ -90,7 +90,7 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*Dag
90
91
func NewDataFileReader(ctx context.Context, n *mdag.Node, pb *ftpb.Data, serv mdag.DAGService) *DagReader {
92
fctx, cancel := context.WithCancel(ctx)
93
- promises := serv.GetDAG(fctx, n)
93
+ promises := mdag.GetDAG(fctx, serv, n)
94
return &DagReader{
95
node: n,
96
serv: serv,