Test and fix GC/pin bug #6252
License: MIT Signed-off-by: Erik Ingenito <erik.ingenito@protocol.ai>
Erik Ingenito committed
May 1, 2019 at 14:03 UTC
4427d7e8e31a6987d79f514b39362fee2f89217d
2 files changed
+110
-7
core/coreunix/add.go
-7
@@ -71,7 +71,6 @@ type Adder struct {
71
Silent bool
72
NoCopy bool
73
Chunker string
74
- root ipld.Node
74
mroot *mfs.Root
75
unlocker bstore.Unlocker
76
tempRoot cid.Cid
@@ -132,11 +131,6 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
131
132
// RootNode returns the mfs root node
133
func (adder *Adder) curRootNode() (ipld.Node, error) {
135
- // for memoizing
136
- if adder.root != nil {
137
- return adder.root, nil
138
- }
139
-
134
mr, err := adder.mfsRoot()
135
if err != nil {
136
return nil, err
@@ -156,7 +150,6 @@ func (adder *Adder) curRootNode() (ipld.Node, error) {
150
root = nd
151
}
152
159
- adder.root = root
153
return root, err
154
}
155
core/coreunix/add_test.go
+110
@@ -30,6 +30,116 @@ import (
30
31
const testPeerID = "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe"
32
33
+func TestAddMultipleGCLive(t *testing.T) {
34
+ r := &repo.Mock{
35
+ C: config.Config{
36
+ Identity: config.Identity{
37
+ PeerID: testPeerID, // required by offline node
38
+ },
39
+ },
40
+ D: syncds.MutexWrap(datastore.NewMapDatastore()),
41
+ }
42
+ node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
43
+ if err != nil {
44
+ t.Fatal(err)
45
+ }
46
+
47
+ out := make(chan interface{}, 10)
48
+ adder, err := NewAdder(context.Background(), node.Pinning, node.Blockstore, node.DAG)
49
+ if err != nil {
50
+ t.Fatal(err)
51
+ }
52
+ adder.Out = out
53
+
54
+ // make two files with pipes so we can 'pause' the add for timing of the test
55
+ piper1, pipew1 := io.Pipe()
56
+ hangfile1 := files.NewReaderFile(piper1)
57
+
58
+ piper2, pipew2 := io.Pipe()
59
+ hangfile2 := files.NewReaderFile(piper2)
60
+
61
+ rfc := files.NewBytesFile([]byte("testfileA"))
62
+
63
+ slf := files.NewMapDirectory(map[string]files.Node{
64
+ "a": hangfile1,
65
+ "b": hangfile2,
66
+ "c": rfc,
67
+ })
68
+
69
+ go func() {
70
+ defer close(out)
71
+ adder.AddAllAndPin(slf)
72
+ // Ignore errors for clarity - the real bug would be gc'ing files while adding them, not this resultant error
73
+ }()
74
+
75
+ // Start writing the first file but don't close the stream
76
+ if _, err := pipew1.Write([]byte("some data for file a")); err != nil {
77
+ t.Fatal(err)
78
+ }
79
+
80
+ var gc1out <-chan gc.Result
81
+ gc1started := make(chan struct{})
82
+ go func() {
83
+ defer close(gc1started)
84
+ gc1out = gc.GC(context.Background(), node.Blockstore, node.Repo.Datastore(), node.Pinning, nil)
85
+ }()
86
+
87
+ // GC shouldn't get the lock until after the file is completely added
88
+ select {
89
+ case <-gc1started:
90
+ t.Fatal("gc shouldnt have started yet")
91
+ default:
92
+ }
93
+
94
+ // finish write and unblock gc
95
+ pipew1.Close()
96
+
97
+ // Should have gotten the lock at this point
98
+ <-gc1started
99
+
100
+ removedHashes := make(map[string]struct{})
101
+ for r := range gc1out {
102
+ if r.Error != nil {
103
+ t.Fatal(err)
104
+ }
105
+ removedHashes[r.KeyRemoved.String()] = struct{}{}
106
+ }
107
+
108
+ if _, err := pipew2.Write([]byte("some data for file b")); err != nil {
109
+ t.Fatal(err)
110
+ }
111
+
112
+ var gc2out <-chan gc.Result
113
+ gc2started := make(chan struct{})
114
+ go func() {
115
+ defer close(gc2started)
116
+ gc2out = gc.GC(context.Background(), node.Blockstore, node.Repo.Datastore(), node.Pinning, nil)
117
+ }()
118
+
119
+ select {
120
+ case <-gc2started:
121
+ t.Fatal("gc shouldnt have started yet")
122
+ default:
123
+ }
124
+
125
+ pipew2.Close()
126
+
127
+ <-gc2started
128
+
129
+ for r := range gc2out {
130
+ if r.Error != nil {
131
+ t.Fatal(err)
132
+ }
133
+ removedHashes[r.KeyRemoved.String()] = struct{}{}
134
+ }
135
+
136
+ for o := range out {
137
+ if _, ok := removedHashes[o.(*coreiface.AddEvent).Path.Cid().String()]; ok {
138
+ t.Fatal("gc'ed a hash we just added")
139
+ }
140
+ }
141
+}
142
+
143
func TestAddGCLive(t *testing.T) {
144
r := &repo.Mock{
145
C: config.Config{