gc: return Result instead of two channels
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Feb 24, 2017 at 14:47 UTC
96f8c17b0fbf8990a4b9834d092823fe3df9d8f8
4 files changed
+45
-78
core/commands/repo.go
+8
-16
@@ -58,28 +58,20 @@ order to reclaim hard disk space.
58
return
59
}
60
61
- gcOutChan, erro := corerepo.GarbageCollectAsync(n, req.Context())
61
+ gcOutChan := corerepo.GarbageCollectAsync(n, req.Context())
62
63
- outChan := make(chan interface{})
63
+ outChan := make(chan interface{}, len(gcOutChan))
64
res.SetOutput((<-chan interface{})(outChan))
65
66
go func() {
67
defer close(outChan)
68
var errors []error
69
- for gcOutChan != nil || erro != nil {
70
- select {
71
- case k, ok := <-gcOutChan:
72
- if ok {
73
- outChan <- k
74
- } else {
75
- gcOutChan = nil
76
- }
77
- case err, ok := <-erro:
78
- if ok {
79
- errors = append(errors, err)
80
- } else {
81
- erro = nil
82
- }
69
+ for res := range gcOutChan {
70
+ if res.KeyRemoved != nil {
71
+ outChan <- &corerepo.KeyRemoved{res.KeyRemoved}
72
+ }
73
+ if res.Error != nil {
74
+ errors = append(errors, res.Error)
75
}
76
}
77
switch len(errors) {
core/corerepo/gc.go
+10
-32
@@ -90,23 +90,15 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
90
if err != nil {
91
return err
92
}
93
- rmed, erro := gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots)
93
+ rmed := gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots)
94
95
var errors []error
96
- for rmed != nil || erro != nil {
97
- select {
98
- case _, ok := <-rmed:
99
- if !ok {
100
- rmed = nil
101
- }
102
- case err, ok := <-erro:
103
- if ok {
104
- errors = append(errors, err)
105
- } else {
106
- erro = nil
107
- }
96
+ for res := range rmed {
97
+ if res.Error != nil {
98
+ errors = append(errors, err)
99
}
100
}
101
+
102
switch len(errors) {
103
case 0:
104
return nil
@@ -136,30 +128,16 @@ func (e *MultiError) Error() string {
128
return buf.String()
129
}
130
139
-func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) (<-chan *KeyRemoved, <-chan error) {
140
- out := make(chan *KeyRemoved)
141
-
131
+func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) <-chan gc.Result {
132
roots, err := BestEffortRoots(n.FilesRoot)
133
if err != nil {
144
- erro := make(chan error)
145
- erro <- err
146
- close(erro)
134
+ out := make(chan gc.Result)
135
+ out <- gc.Result{Error: err}
136
close(out)
148
- return out, erro
137
+ return out
138
}
150
- rmed, erro := gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots)
139
152
- go func() {
153
- defer close(out)
154
- for k := range rmed {
155
- select {
156
- case out <- &KeyRemoved{k}:
157
- case <-ctx.Done():
158
- return
159
- }
160
- }
161
- }()
162
- return out, erro
140
+ return gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots)
141
}
142
143
func PeriodicGC(ctx context.Context, node *core.IpfsNode) error {
core/coreunix/add_test.go
+7
-13
@@ -98,14 +98,11 @@ func TestAddGCLive(t *testing.T) {
98
t.Fatal("add shouldnt complete yet")
99
}
100
101
- var gcout <-chan *cid.Cid
102
- var errs <-chan error
101
+ var gcout <-chan gc.Result
102
gcstarted := make(chan struct{})
103
go func() {
104
defer close(gcstarted)
106
- gcchan, erro := gc.GC(context.Background(), node.Blockstore, node.DAG, node.Pinning, nil)
107
- errs = erro
108
- gcout = gcchan
105
+ gcout = gc.GC(context.Background(), node.Blockstore, node.DAG, node.Pinning, nil)
106
}()
107
108
// gc shouldnt start until we let the add finish its current file.
@@ -114,8 +111,6 @@ func TestAddGCLive(t *testing.T) {
111
select {
112
case <-gcstarted:
113
t.Fatal("gc shouldnt have started yet")
117
- case err := <-errs:
118
- t.Fatal(err)
114
default:
115
}
116
@@ -128,18 +123,17 @@ func TestAddGCLive(t *testing.T) {
123
select {
124
case o := <-out:
125
addedHashes[o.(*AddedObject).Hash] = struct{}{}
131
- case err := <-errs:
132
- t.Fatal(err)
126
}
127
128
select {
129
case <-gcstarted:
137
- case err := <-errs:
138
- t.Fatal(err)
130
}
131
141
- for k := range gcout {
142
- if _, ok := addedHashes[k.String()]; ok {
132
+ for r := range gcout {
133
+ if r.Error != nil {
134
+ t.Fatal(err)
135
+ }
136
+ if _, ok := addedHashes[r.KeyRemoved.String()]; ok {
137
t.Fatal("gc'ed a hash we just added")
138
}
139
}
pin/gc/gc.go
+20
-17
@@ -16,6 +16,11 @@ import (
16
17
var log = logging.Logger("gc")
18
19
+type Result struct {
20
+ KeyRemoved *cid.Cid
21
+ Error error
22
+}
23
+
24
// GC performs a mark and sweep garbage collection of the blocks in the blockstore
25
// first, it creates a 'marked' set and adds to it the following:
26
// - all recursively pinned blocks, plus all of their descendants (recursively)
@@ -26,27 +31,25 @@ var log = logging.Logger("gc")
31
// The routine then iterates over every block in the blockstore and
32
// deletes any block that is not found in the marked set.
33
//
29
-func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan *cid.Cid, <-chan error) {
34
+func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result {
35
unlocker := bs.GCLock()
36
ls = ls.GetOfflineLinkService()
37
33
- output := make(chan *cid.Cid)
34
- errOutput := make(chan error)
38
+ output := make(chan Result, 128)
39
40
go func() {
37
- defer close(errOutput)
41
defer close(output)
42
defer unlocker.Unlock()
43
41
- gcs, err := ColoredSet(ctx, pn, ls, bestEffortRoots, errOutput)
44
+ gcs, err := ColoredSet(ctx, pn, ls, bestEffortRoots, output)
45
if err != nil {
43
- errOutput <- err
46
+ output <- Result{Error: err}
47
return
48
}
49
50
keychan, err := bs.AllKeysChan(ctx)
51
if err != nil {
49
- errOutput <- err
52
+ output <- Result{Error: err}
53
return
54
}
55
@@ -63,13 +66,13 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.
66
err := bs.DeleteBlock(k)
67
if err != nil {
68
errors = true
66
- errOutput <- &CouldNotDeleteBlockError{k, err}
69
+ output <- Result{Error: &CouldNotDeleteBlockError{k, err}}
70
//log.Errorf("Error removing key from blockstore: %s", err)
71
// continue as error is non-fatal
72
continue loop
73
}
74
select {
72
- case output <- k:
75
+ case output <- Result{KeyRemoved: k}:
76
case <-ctx.Done():
77
break loop
78
}
@@ -79,11 +82,11 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.
82
}
83
}
84
if errors {
82
- errOutput <- ErrCouldNotDeleteSomeBlocks
85
+ output <- Result{Error: ErrCouldNotDeleteSomeBlocks}
86
}
87
}()
88
86
- return output, errOutput
89
+ return output
90
}
91
92
func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots []*cid.Cid) error {
@@ -100,7 +103,7 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
103
return nil
104
}
105
103
-func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffortRoots []*cid.Cid, errOutput chan<- error) (*cid.Set, error) {
106
+func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffortRoots []*cid.Cid, output chan<- Result) (*cid.Set, error) {
107
// KeySet currently implemented in memory, in the future, may be bloom filter or
108
// disk backed to conserve memory.
109
errors := false
@@ -109,28 +112,28 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffo
112
links, err := ls.GetLinks(ctx, cid)
113
if err != nil {
114
errors = true
112
- errOutput <- &CouldNotFetchLinksError{cid, err}
115
+ output <- Result{Error: &CouldNotFetchLinksError{cid, err}}
116
}
117
return links, nil
118
}
119
err := Descendants(ctx, getLinks, gcs, pn.RecursiveKeys())
120
if err != nil {
121
errors = true
119
- errOutput <- err
122
+ output <- Result{Error: err}
123
}
124
125
bestEffortGetLinks := func(ctx context.Context, cid *cid.Cid) ([]*node.Link, error) {
126
links, err := ls.GetLinks(ctx, cid)
127
if err != nil && err != dag.ErrNotFound {
128
errors = true
126
- errOutput <- &CouldNotFetchLinksError{cid, err}
129
+ output <- Result{Error: &CouldNotFetchLinksError{cid, err}}
130
}
131
return links, nil
132
}
133
err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRoots)
134
if err != nil {
135
errors = true
133
- errOutput <- err
136
+ output <- Result{Error: err}
137
}
138
139
for _, k := range pn.DirectKeys() {
@@ -140,7 +143,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffo
143
err = Descendants(ctx, getLinks, gcs, pn.InternalPins())
144
if err != nil {
145
errors = true
143
- errOutput <- err
146
+ output <- Result{Error: err}
147
}
148
149
if errors {