Add Files API root as best-effort pin.
Closes #2697. Closes #2698. License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Jun 20, 2016 at 17:06 UTC
714f2debead06ce6213a790fffec4f2f0b0e1fd9
7 files changed
+84
-17
core/commands/pin.go
+1
-1
@@ -330,7 +330,7 @@ func pinLsAll(typeStr string, ctx context.Context, n *core.IpfsNode) (map[string
330
if err != nil {
331
return nil, err
332
}
333
- err = dag.EnumerateChildren(n.Context(), n.DAG, nd, ks)
333
+ err = dag.EnumerateChildren(n.Context(), n.DAG, nd, ks, false)
334
if err != nil {
335
return nil, err
336
}
core/corerepo/gc.go
+23
-2
@@ -6,6 +6,7 @@ import (
6
7
key "github.com/ipfs/go-ipfs/blocks/key"
8
"github.com/ipfs/go-ipfs/core"
9
+ mfs "github.com/ipfs/go-ipfs/mfs"
10
gc "github.com/ipfs/go-ipfs/pin/gc"
11
repo "github.com/ipfs/go-ipfs/repo"
12
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
@@ -71,10 +72,26 @@ func NewGC(n *core.IpfsNode) (*GC, error) {
72
}, nil
73
}
74
75
+func BestEffortRoots(filesRoot *mfs.Root) ([]key.Key, error) {
76
+ rootDag, err := filesRoot.GetValue().GetNode()
77
+ if err != nil {
78
+ return nil, err
79
+ }
80
+ rootKey, err := rootDag.Key()
81
+ if err != nil {
82
+ return nil, err
83
+ }
84
+ return []key.Key{rootKey}, nil
85
+}
86
+
87
func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
88
ctx, cancel := context.WithCancel(ctx)
89
defer cancel() // in case error occurs during operation
77
- rmed, err := gc.GC(ctx, n.Blockstore, n.Pinning)
90
+ roots, err := BestEffortRoots(n.FilesRoot)
91
+ if err != nil {
92
+ return err
93
+ }
94
+ rmed, err := gc.GC(ctx, n.Blockstore, n.Pinning, roots)
95
if err != nil {
96
return err
97
}
@@ -93,7 +110,11 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
110
}
111
112
func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) (<-chan *KeyRemoved, error) {
96
- rmed, err := gc.GC(ctx, n.Blockstore, n.Pinning)
113
+ roots, err := BestEffortRoots(n.FilesRoot)
114
+ if err != nil {
115
+ return nil, err
116
+ }
117
+ rmed, err := gc.GC(ctx, n.Blockstore, n.Pinning, roots)
118
if err != nil {
119
return nil, err
120
}
core/coreunix/add_test.go
+2
-2
@@ -96,7 +96,7 @@ func TestAddGCLive(t *testing.T) {
96
gcstarted := make(chan struct{})
97
go func() {
98
defer close(gcstarted)
99
- gcchan, err := gc.GC(context.Background(), node.Blockstore, node.Pinning)
99
+ gcchan, err := gc.GC(context.Background(), node.Blockstore, node.Pinning, nil)
100
if err != nil {
101
log.Error("GC ERROR:", err)
102
errs <- err
@@ -155,7 +155,7 @@ func TestAddGCLive(t *testing.T) {
155
t.Fatal(err)
156
}
157
158
- err = dag.EnumerateChildren(ctx, node.DAG, root, key.NewKeySet())
158
+ err = dag.EnumerateChildren(ctx, node.DAG, root, key.NewKeySet(), false)
159
if err != nil {
160
t.Fatal(err)
161
}
merkledag/merkledag.go
+7
-3
@@ -357,16 +357,20 @@ func (t *Batch) Commit() error {
357
// EnumerateChildren will walk the dag below the given root node and add all
358
// unseen children to the passed in set.
359
// TODO: parallelize to avoid disk latency perf hits?
360
-func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
360
+func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.KeySet, bestEffort bool) error {
361
for _, lnk := range root.Links {
362
k := key.Key(lnk.Hash)
363
if !set.Has(k) {
364
set.Add(k)
365
child, err := ds.Get(ctx, k)
366
if err != nil {
367
- return err
367
+ if bestEffort && err == ErrNotFound {
368
+ continue
369
+ } else {
370
+ return err
371
+ }
372
}
369
- err = EnumerateChildren(ctx, ds, child, set)
373
+ err = EnumerateChildren(ctx, ds, child, set, bestEffort)
374
if err != nil {
375
return err
376
}
merkledag/merkledag_test.go
+2
-2
@@ -292,7 +292,7 @@ func TestFetchGraph(t *testing.T) {
292
offline_ds := NewDAGService(bs)
293
ks := key.NewKeySet()
294
295
- err = EnumerateChildren(context.Background(), offline_ds, root, ks)
295
+ err = EnumerateChildren(context.Background(), offline_ds, root, ks, false)
296
if err != nil {
297
t.Fatal(err)
298
}
@@ -309,7 +309,7 @@ func TestEnumerateChildren(t *testing.T) {
309
}
310
311
ks := key.NewKeySet()
312
- err = EnumerateChildren(context.Background(), ds, root, ks)
312
+ err = EnumerateChildren(context.Background(), ds, root, ks, false)
313
if err != nil {
314
t.Fatal(err)
315
}
pin/gc/gc.go
+13
-7
@@ -17,18 +17,19 @@ var log = logging.Logger("gc")
17
// GC performs a mark and sweep garbage collection of the blocks in the blockstore
18
// first, it creates a 'marked' set and adds to it the following:
19
// - all recursively pinned blocks, plus all of their descendants (recursively)
20
+// - bestEffortRoots, plus all of its descendants (recursively)
21
// - all directly pinned blocks
22
// - all blocks utilized internally by the pinner
23
//
24
// The routine then iterates over every block in the blockstore and
25
// deletes any block that is not found in the marked set.
25
-func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner) (<-chan key.Key, error) {
26
+func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRoots []key.Key) (<-chan key.Key, error) {
27
unlocker := bs.GCLock()
28
29
bsrv := bserv.New(bs, offline.Exchange(bs))
30
ds := dag.NewDAGService(bsrv)
31
31
- gcs, err := ColoredSet(ctx, pn, ds)
32
+ gcs, err := ColoredSet(ctx, pn, ds, bestEffortRoots)
33
if err != nil {
34
return nil, err
35
}
@@ -69,7 +70,7 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner) (<-chan key.
70
return output, nil
71
}
72
72
-func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots []key.Key) error {
73
+func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots []key.Key, bestEffort bool) error {
74
for _, k := range roots {
75
set.Add(k)
76
nd, err := ds.Get(ctx, k)
@@ -78,7 +79,7 @@ func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots [
79
}
80
81
// EnumerateChildren recursively walks the dag and adds the keys to the given set
81
- err = dag.EnumerateChildren(ctx, ds, nd, set)
82
+ err = dag.EnumerateChildren(ctx, ds, nd, set, bestEffort)
83
if err != nil {
84
return err
85
}
@@ -87,11 +88,16 @@ func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots [
88
return nil
89
}
90
90
-func ColoredSet(ctx context.Context, pn pin.Pinner, ds dag.DAGService) (key.KeySet, error) {
91
+func ColoredSet(ctx context.Context, pn pin.Pinner, ds dag.DAGService, bestEffortRoots []key.Key) (key.KeySet, error) {
92
// KeySet currently implemented in memory, in the future, may be bloom filter or
93
// disk backed to conserve memory.
94
gcs := key.NewKeySet()
94
- err := Descendants(ctx, ds, gcs, pn.RecursiveKeys())
95
+ err := Descendants(ctx, ds, gcs, pn.RecursiveKeys(), false)
96
+ if err != nil {
97
+ return nil, err
98
+ }
99
+
100
+ err = Descendants(ctx, ds, gcs, bestEffortRoots, true)
101
if err != nil {
102
return nil, err
103
}
@@ -100,7 +106,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ds dag.DAGService) (key.KeyS
106
gcs.Add(k)
107
}
108
103
- err = Descendants(ctx, ds, gcs, pn.InternalPins())
109
+ err = Descendants(ctx, ds, gcs, pn.InternalPins(), false)
110
if err != nil {
111
return nil, err
112
}
test/sharness/t0252-files-gc.sh
new
+36
@@ -0,0 +1,36 @@
1
+#!/bin/sh
2
+#
3
+# Copyright (c) 2016 Jeromy Johnson
4
+# MIT Licensed; see the LICENSE file in this repository.
5
+#
6
+
7
+test_description="test how the unix files api interacts with the gc"
8
+
9
+. lib/test-lib.sh
10
+
11
+test_init_ipfs
12
+
13
+test_expect_success "object not removed after gc" '
14
+ echo "hello world" | ipfs files write --create /hello.txt &&
15
+ ipfs repo gc &&
16
+ ipfs cat QmVib14uvPnCP73XaCDpwugRuwfTsVbGyWbatHAmLSdZUS
17
+'
18
+
19
+test_expect_success "gc okay after adding incomplete node -- prep" '
20
+ ipfs files mkdir /adir &&
21
+ echo "file1" | ipfs files write --create /adir/file1 &&
22
+ echo "file2" | ipfs files write --create /adir/file2 &&
23
+ ipfs pin add --recursive=false QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW &&
24
+ ipfs files rm -r /adir &&
25
+ ipfs repo gc && # will remove /adir/file1 and /adir/file2 but not /adir
26
+ ipfs files cp /ipfs/QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW /adir &&
27
+ ipfs pin rm QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW
28
+'
29
+
30
+test_expect_success "gc okay after adding incomplete node" '
31
+ ipfs refs QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW &&
32
+ ipfs repo gc &&
33
+ ipfs refs QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW
34
+'
35
+
36
+test_done