use an option type to simplify concurrency
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Feb 24, 2016 at 11:38 UTC
ec3959a6afab5a12d2c471ba6d9e35ed51c9ce07
1 file changed
+48
-36
merkledag/merkledag.go
+48
-36
@@ -3,6 +3,7 @@ package merkledag
3
4
import (
5
"fmt"
6
+ "sync"
7
8
blocks "github.com/ipfs/go-ipfs/blocks"
9
key "github.com/ipfs/go-ipfs/blocks/key"
@@ -24,7 +25,7 @@ type DAGService interface {
25
26
// GetDAG returns, in order, all the single leve child
27
// nodes of the passed in node.
27
- GetMany(context.Context, []key.Key) (<-chan *Node, <-chan error)
28
+ GetMany(context.Context, []key.Key) <-chan *NodeOption
29
30
Batch() *Batch
31
}
@@ -145,9 +146,13 @@ func FindLinks(links []key.Key, k key.Key, start int) []int {
146
return out
147
}
148
148
-func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) (<-chan *Node, <-chan error) {
149
- out := make(chan *Node, len(keys))
150
- errs := make(chan error, 1)
149
+type NodeOption struct {
150
+ Node *Node
151
+ Err error
152
+}
153
+
154
+func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeOption {
155
+ out := make(chan *NodeOption, len(keys))
156
blocks := ds.Blocks.GetBlocks(ctx, keys)
157
var count int
158
@@ -158,27 +163,27 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) (<-chan *Node
163
case b, ok := <-blocks:
164
if !ok {
165
if count != len(keys) {
161
- errs <- fmt.Errorf("failed to fetch all nodes")
166
+ out <- &NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
167
}
168
return
169
}
170
nd, err := Decoded(b.Data)
171
if err != nil {
167
- errs <- err
172
+ out <- &NodeOption{Err: err}
173
return
174
}
175
176
// buffered, no need to select
172
- out <- nd
177
+ out <- &NodeOption{Node: nd}
178
count++
179
180
case <-ctx.Done():
176
- errs <- ctx.Err()
181
+ out <- &NodeOption{Err: ctx.Err()}
182
return
183
}
184
}
185
}()
181
- return out, errs
186
+ return out
187
}
188
189
// GetDAG will fill out all of the links of the given Node.
@@ -213,15 +218,22 @@ func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {
218
ctx, cancel := context.WithCancel(ctx)
219
defer cancel()
220
216
- nodechan, errchan := ds.GetMany(ctx, dedupedKeys)
221
+ nodechan := ds.GetMany(ctx, dedupedKeys)
222
223
for count := 0; count < len(keys); {
224
select {
220
- case nd, ok := <-nodechan:
225
+ case opt, ok := <-nodechan:
226
if !ok {
227
return
228
}
229
230
+ if opt.Err != nil {
231
+ log.Error("error fetching: ", opt.Err)
232
+ return
233
+ }
234
+
235
+ nd := opt.Node
236
+
237
k, err := nd.Key()
238
if err != nil {
239
log.Error("Failed to get node key: ", err)
@@ -233,9 +245,6 @@ func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {
245
count++
246
sendChans[i] <- nd
247
}
236
- case err := <-errchan:
237
- log.Error("error fetching: ", err)
238
- return
248
case <-ctx.Done():
249
return
250
}
@@ -356,24 +365,30 @@ func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.K
365
366
func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
367
toprocess := make(chan []key.Key, 8)
359
- nodes := make(chan *Node, 8)
360
- errs := make(chan error, 1)
368
+ nodes := make(chan *NodeOption, 8)
369
370
ctx, cancel := context.WithCancel(ctx)
371
defer cancel()
372
defer close(toprocess)
373
366
- go fetchNodes(ctx, ds, toprocess, nodes, errs)
374
+ go fetchNodes(ctx, ds, toprocess, nodes)
375
368
- nodes <- root
376
+ nodes <- &NodeOption{Node: root}
377
live := 1
378
379
for {
380
select {
373
- case nd, ok := <-nodes:
381
+ case opt, ok := <-nodes:
382
if !ok {
383
return nil
384
}
385
+
386
+ if opt.Err != nil {
387
+ return opt.Err
388
+ }
389
+
390
+ nd := opt.Node
391
+
392
// a node has been fetched
393
live--
394
@@ -398,38 +413,35 @@ func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set
413
return ctx.Err()
414
}
415
}
401
- case err := <-errs:
402
- return err
416
case <-ctx.Done():
417
return ctx.Err()
418
}
419
}
420
}
421
409
-func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out chan<- *Node, errs chan<- error) {
410
- defer close(out)
422
+func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out chan<- *NodeOption) {
423
+ var wg sync.WaitGroup
424
+ defer func() {
425
+ // wait for all 'get' calls to complete so we don't accidentally send
426
+ // on a closed channel
427
+ wg.Wait()
428
+ close(out)
429
+ }()
430
431
get := func(ks []key.Key) {
413
- nodes, errch := ds.GetMany(ctx, ks)
414
- for {
432
+ defer wg.Done()
433
+ nodes := ds.GetMany(ctx, ks)
434
+ for opt := range nodes {
435
select {
416
- case nd, ok := <-nodes:
417
- if !ok {
418
- return
419
- }
420
- select {
421
- case out <- nd:
422
- case <-ctx.Done():
423
- return
424
- }
425
- case err := <-errch:
426
- errs <- err
436
+ case out <- opt:
437
+ case <-ctx.Done():
438
return
439
}
440
}
441
}
442
443
for ks := range in {
444
+ wg.Add(1)
445
go get(ks)
446
}
447
}