reprovider: reduce pinned strategy i/o overhead
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Aug 3, 2017 at 14:51 UTC
0f692baffaf6f0de934d8f239d70af307a0ff524
1 file changed
+52
-17
exchange/reprovide/providers.go
+52
-17
@@ -2,7 +2,6 @@ package reprovide
2
3
import (
4
"context"
5
- "errors"
5
6
blocks "github.com/ipfs/go-ipfs/blocks/blockstore"
7
merkledag "github.com/ipfs/go-ipfs/merkledag"
@@ -29,36 +28,72 @@ func NewPinnedProvider(pinning pin.Pinner, dag merkledag.DAGService, onlyRoots b
28
outCh := make(chan *cid.Cid)
29
go func() {
30
defer close(outCh)
32
- set.ForEach(func(c *cid.Cid) error {
31
+ for c := range set.new {
32
select {
33
case <-ctx.Done():
35
- return errors.New("context cancelled")
34
+ return
35
case outCh <- c:
36
}
38
- return nil
39
- })
37
+ }
38
+
39
}()
40
41
return outCh, nil
42
}
43
}
44
46
-func pinSet(ctx context.Context, pinning pin.Pinner, dag merkledag.DAGService, onlyRoots bool) (*cid.Set, error) {
47
- set := cid.NewSet()
48
- for _, key := range pinning.DirectKeys() {
49
- set.Add(key)
50
- }
45
+func pinSet(ctx context.Context, pinning pin.Pinner, dag merkledag.DAGService, onlyRoots bool) (*streamingSet, error) {
46
+ set := newStreamingSet()
47
+
48
+ go func() {
49
+ for _, key := range pinning.DirectKeys() {
50
+ set.add(key)
51
+ }
52
52
- for _, key := range pinning.RecursiveKeys() {
53
- set.Add(key)
53
+ for _, key := range pinning.RecursiveKeys() {
54
+ set.add(key)
55
55
- if !onlyRoots {
56
- err := merkledag.EnumerateChildren(ctx, dag.GetLinks, key, set.Visit)
57
- if err != nil {
58
- return nil, err
56
+ if !onlyRoots {
57
+ err := merkledag.EnumerateChildren(ctx, dag.GetLinks, key, set.add)
58
+ if err != nil {
59
+ return //TODO: propagate to chan / log?
60
+ }
61
}
62
}
61
- }
63
+
64
+ close(set.new)
65
+ }()
66
67
return set, nil
68
}
69
+
70
+type streamingSet struct {
71
+ set map[string]struct{}
72
+ new chan *cid.Cid
73
+}
74
+
75
+// NewSet initializes and returns a new Set.
76
+func newStreamingSet() *streamingSet {
77
+ return &streamingSet{
78
+ set: make(map[string]struct{}),
79
+ new: make(chan *cid.Cid),
80
+ }
81
+}
82
+
83
+// has returns if the Set contains a given Cid.
84
+func (s *streamingSet) has(c *cid.Cid) bool {
85
+ _, ok := s.set[string(c.Bytes())]
86
+ return ok
87
+}
88
+
89
+// add adds a Cid to the set only if it is
90
+// not in it already.
91
+func (s *streamingSet) add(c *cid.Cid) bool {
92
+ if !s.has(c) {
93
+ s.set[string(c.Bytes())] = struct{}{}
94
+ s.new <- c
95
+ return true
96
+ }
97
+
98
+ return false
99
+}