merkledag: add a concurrency limit to merkledag fetch graph
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Dec 9, 2016 at 14:58 UTC
0c14b4162ea3c4f07aea90dd78bc92cf067a0faa
1 file changed
+14
merkledag/merkledag.go
+14
@@ -449,6 +449,10 @@ func EnumerateChildrenAsync(ctx context.Context, ds DAGService, c *cid.Cid, visi
449
}
450
}
451
452
+// FetchGraphConcurrency is total number of concurrenct fetches that
453
+// 'fetchNodes' will start at a time
454
+var FetchGraphConcurrency = 8
455
+
456
func fetchNodes(ctx context.Context, ds DAGService, in <-chan []*cid.Cid, out chan<- *NodeOption) {
457
var wg sync.WaitGroup
458
defer func() {
@@ -458,8 +462,13 @@ func fetchNodes(ctx context.Context, ds DAGService, in <-chan []*cid.Cid, out ch
462
close(out)
463
}()
464
465
+ rateLimit := make(chan struct{}, FetchGraphConcurrency)
466
+
467
get := func(ks []*cid.Cid) {
468
defer wg.Done()
469
+ defer func() {
470
+ <-rateLimit
471
+ }()
472
nodes := ds.GetMany(ctx, ks)
473
for opt := range nodes {
474
select {
@@ -471,6 +480,11 @@ func fetchNodes(ctx context.Context, ds DAGService, in <-chan []*cid.Cid, out ch
480
}
481
482
for ks := range in {
483
+ select {
484
+ case rateLimit <- struct{}{}:
485
+ case <-ctx.Done():
486
+ return
487
+ }
488
wg.Add(1)
489
go get(ks)
490
}