Add DAGService.GetLinks() method and use it in the GC and elsewhere.
This method will use the (also new) LinkService if it is available to retrieving just the links for a MerkleDAG without necessary having to retrieve the underlying block. For now the main benefit is that the pinner will not break when a block becomes invalid due to a change in the backing file. This is possible because the metadata for a block (that includes the Links) is stored separately and thus always available even if the backing file changes. License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Aug 19, 2016 at 15:52 UTC
3899194cb0aec37617576c4c612f2301479aeb83
8 files changed
+61
-35
core/commands/pin.go
+2
-3
@@ -328,12 +328,11 @@ func pinLsAll(typeStr string, ctx context.Context, n *core.IpfsNode) (map[string
328
if typeStr == "indirect" || typeStr == "all" {
329
set := cid.NewSet()
330
for _, k := range n.Pinning.RecursiveKeys() {
331
- nd, err := n.DAG.Get(ctx, k)
331
+ links, err := n.DAG.GetLinks(ctx, k)
332
if err != nil {
333
return nil, err
334
}
335
-
336
- err = dag.EnumerateChildren(n.Context(), n.DAG, nd, set.Visit, false)
335
+ err = dag.EnumerateChildren(n.Context(), n.DAG, links, set.Visit, false)
336
if err != nil {
337
return nil, err
338
}
core/core.go
+9
-8
@@ -94,14 +94,15 @@ type IpfsNode struct {
94
PrivateKey ic.PrivKey // the local node's private Key
95
96
// Services
97
- Peerstore pstore.Peerstore // storage for other Peer instances
98
- Blockstore bstore.GCBlockstore // the block store (lower level)
99
- Blocks *bserv.BlockService // the block service, get/add blocks.
100
- DAG merkledag.DAGService // the merkle dag service, get/add objects.
101
- Resolver *path.Resolver // the path resolution system
102
- Reporter metrics.Reporter
103
- Discovery discovery.Service
104
- FilesRoot *mfs.Root
97
+ Peerstore pstore.Peerstore // storage for other Peer instances
98
+ Blockstore bstore.GCBlockstore // the block store (lower level)
99
+ Blocks *bserv.BlockService // the block service, get/add blocks.
100
+ DAG merkledag.DAGService // the merkle dag service, get/add objects.
101
+ LinkService merkledag.LinkService
102
+ Resolver *path.Resolver // the path resolution system
103
+ Reporter metrics.Reporter
104
+ Discovery discovery.Service
105
+ FilesRoot *mfs.Root
106
107
// Online
108
PeerHost p2phost.Host // the network host (server+client)
core/corerepo/gc.go
+2
-2
@@ -90,7 +90,7 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
90
if err != nil {
91
return err
92
}
93
- rmed, err := gc.GC(ctx, n.Blockstore, n.Pinning, roots)
93
+ rmed, err := gc.GC(ctx, n.Blockstore, n.LinkService, n.Pinning, roots)
94
if err != nil {
95
return err
96
}
@@ -113,7 +113,7 @@ func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) (<-chan *KeyRemo
113
if err != nil {
114
return nil, err
115
}
116
- rmed, err := gc.GC(ctx, n.Blockstore, n.Pinning, roots)
116
+ rmed, err := gc.GC(ctx, n.Blockstore, n.LinkService, n.Pinning, roots)
117
if err != nil {
118
return nil, err
119
}
core/coreunix/add_test.go
+2
-2
@@ -98,7 +98,7 @@ func TestAddGCLive(t *testing.T) {
98
gcstarted := make(chan struct{})
99
go func() {
100
defer close(gcstarted)
101
- gcchan, err := gc.GC(context.Background(), node.Blockstore, node.Pinning, nil)
101
+ gcchan, err := gc.GC(context.Background(), node.Blockstore, node.LinkService, node.Pinning, nil)
102
if err != nil {
103
log.Error("GC ERROR:", err)
104
errs <- err
@@ -162,7 +162,7 @@ func TestAddGCLive(t *testing.T) {
162
}
163
164
set := cid.NewSet()
165
- err = dag.EnumerateChildren(ctx, node.DAG, root, set.Visit, false)
165
+ err = dag.EnumerateChildren(ctx, node.DAG, root.Links, set.Visit, false)
166
if err != nil {
167
t.Fatal(err)
168
}
merkledag/merkledag.go
+32
-7
@@ -23,6 +23,10 @@ type DAGService interface {
23
Get(context.Context, *cid.Cid) (*Node, error)
24
Remove(*Node) error
25
26
+ // Return all links for a node, may be more effect than
27
+ // calling Get
28
+ GetLinks(context.Context, *cid.Cid) ([]*Link, error)
29
+
30
// GetDAG returns, in order, all the single leve child
31
// nodes of the passed in node.
32
GetMany(context.Context, []*cid.Cid) <-chan *NodeOption
@@ -30,8 +34,14 @@ type DAGService interface {
34
Batch() *Batch
35
}
36
33
-func NewDAGService(bs *bserv.BlockService) DAGService {
34
- return &dagService{bs}
37
+// A LinkService returns the links for a node if they are available
38
+// locally without having to retrieve the block from the datastore.
39
+type LinkService interface {
40
+ Get(*cid.Cid) ([]*Link, error)
41
+}
42
+
43
+func NewDAGService(bs *bserv.BlockService) *dagService {
44
+ return &dagService{Blocks: bs}
45
}
46
47
// dagService is an IPFS Merkle DAG service.
@@ -40,7 +50,8 @@ func NewDAGService(bs *bserv.BlockService) DAGService {
50
// TODO: should cache Nodes that are in memory, and be
51
// able to free some of them when vm pressure is high
52
type dagService struct {
43
- Blocks *bserv.BlockService
53
+ Blocks *bserv.BlockService
54
+ LinkService LinkService
55
}
56
57
// Add adds a node to the dagService, storing the block in the BlockService
@@ -93,6 +104,20 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (*Node, error) {
104
return res, nil
105
}
106
107
+func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*Link, error) {
108
+ if n.LinkService != nil {
109
+ links, err := n.LinkService.Get(c)
110
+ if err == nil {
111
+ return links, nil
112
+ }
113
+ }
114
+ node, err := n.Get(ctx, c)
115
+ if err != nil {
116
+ return nil, err
117
+ }
118
+ return node.Links, nil
119
+}
120
+
121
func (n *dagService) Remove(nd *Node) error {
122
return n.Blocks.DeleteObject(nd)
123
}
@@ -366,11 +391,11 @@ func legacyCidFromLink(lnk *Link) *cid.Cid {
391
// EnumerateChildren will walk the dag below the given root node and add all
392
// unseen children to the passed in set.
393
// TODO: parallelize to avoid disk latency perf hits?
369
-func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, visit func(*cid.Cid) bool, bestEffort bool) error {
370
- for _, lnk := range root.Links {
394
+func EnumerateChildren(ctx context.Context, ds DAGService, links []*Link, visit func(*cid.Cid) bool, bestEffort bool) error {
395
+ for _, lnk := range links {
396
c := legacyCidFromLink(lnk)
397
if visit(c) {
373
- child, err := ds.Get(ctx, c)
398
+ children, err := ds.GetLinks(ctx, c)
399
if err != nil {
400
if bestEffort && err == ErrNotFound {
401
continue
@@ -378,7 +403,7 @@ func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, visit fun
403
return err
404
}
405
}
381
- err = EnumerateChildren(ctx, ds, child, visit, bestEffort)
406
+ err = EnumerateChildren(ctx, ds, children, visit, bestEffort)
407
if err != nil {
408
return err
409
}
merkledag/merkledag_test.go
+2
-2
@@ -241,7 +241,7 @@ func TestFetchGraph(t *testing.T) {
241
242
offline_ds := NewDAGService(bs)
243
244
- err = EnumerateChildren(context.Background(), offline_ds, root, func(_ *cid.Cid) bool { return true }, false)
244
+ err = EnumerateChildren(context.Background(), offline_ds, root.Links, func(_ *cid.Cid) bool { return true }, false)
245
if err != nil {
246
t.Fatal(err)
247
}
@@ -258,7 +258,7 @@ func TestEnumerateChildren(t *testing.T) {
258
}
259
260
set := cid.NewSet()
261
- err = EnumerateChildren(context.Background(), ds, root, set.Visit, false)
261
+ err = EnumerateChildren(context.Background(), ds, root.Links, set.Visit, false)
262
if err != nil {
263
t.Fatal(err)
264
}
pin/gc/gc.go
+4
-3
@@ -24,11 +24,12 @@ var log = logging.Logger("gc")
24
//
25
// The routine then iterates over every block in the blockstore and
26
// deletes any block that is not found in the marked set.
27
-func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan key.Key, error) {
27
+func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan key.Key, error) {
28
unlocker := bs.GCLock()
29
30
bsrv := bserv.New(bs, offline.Exchange(bs))
31
ds := dag.NewDAGService(bsrv)
32
+ ds.LinkService = ls
33
34
gcs, err := ColoredSet(ctx, pn, ds, bestEffortRoots)
35
if err != nil {
@@ -74,13 +75,13 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRo
75
func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots []*cid.Cid, bestEffort bool) error {
76
for _, c := range roots {
77
set.Add(key.Key(c.Hash()))
77
- nd, err := ds.Get(ctx, c)
78
+ links, err := ds.GetLinks(ctx, c)
79
if err != nil {
80
return err
81
}
82
83
// EnumerateChildren recursively walks the dag and adds the keys to the given set
83
- err = dag.EnumerateChildren(ctx, ds, nd, func(c *cid.Cid) bool {
84
+ err = dag.EnumerateChildren(ctx, ds, links, func(c *cid.Cid) bool {
85
k := key.Key(c.Hash())
86
seen := set.Has(k)
87
if seen {
pin/pin.go
+8
-8
@@ -279,12 +279,12 @@ func (p *pinner) isPinnedWithType(c *cid.Cid, mode PinMode) (string, bool, error
279
280
// Default is Indirect
281
for _, rc := range p.recursePin.Keys() {
282
- rnd, err := p.dserv.Get(context.Background(), rc)
282
+ links, err := p.dserv.GetLinks(context.Background(), rc)
283
if err != nil {
284
return "", false, err
285
}
286
287
- has, err := hasChild(p.dserv, rnd, k)
287
+ has, err := hasChild(p.dserv, links, k)
288
if err != nil {
289
return "", false, err
290
}
@@ -317,11 +317,11 @@ func (p *pinner) CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) {
317
// Now walk all recursive pins to check for indirect pins
318
var checkChildren func(*cid.Cid, *cid.Cid) error
319
checkChildren = func(rk, parentKey *cid.Cid) error {
320
- parent, err := p.dserv.Get(context.Background(), parentKey)
320
+ links, err := p.dserv.GetLinks(context.Background(), parentKey)
321
if err != nil {
322
return err
323
}
324
- for _, lnk := range parent.Links {
324
+ for _, lnk := range links {
325
c := cid.NewCidV0(lnk.Hash)
326
327
if toCheck.Has(c) {
@@ -521,19 +521,19 @@ func (p *pinner) PinWithMode(c *cid.Cid, mode PinMode) {
521
}
522
}
523
524
-func hasChild(ds mdag.DAGService, root *mdag.Node, child key.Key) (bool, error) {
525
- for _, lnk := range root.Links {
524
+func hasChild(ds mdag.DAGService, links []*mdag.Link, child key.Key) (bool, error) {
525
+ for _, lnk := range links {
526
c := cid.NewCidV0(lnk.Hash)
527
if key.Key(c.Hash()) == child {
528
return true, nil
529
}
530
531
- nd, err := ds.Get(context.Background(), c)
531
+ children, err := ds.GetLinks(context.Background(), c)
532
if err != nil {
533
return false, err
534
}
535
536
- has, err := hasChild(ds, nd, child)
536
+ has, err := hasChild(ds, children, child)
537
if err != nil {
538
return false, err
539
}