fix bug in pinsets and add a stress test for the scenario
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Sep 29, 2016 at 12:35 UTC
1f853c59c3d7c16e74869784cee791f8003c3ac7
2 files changed
+61
-8
pin/set.go
+1
-1
@@ -143,7 +143,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
143
if !ok {
144
break
145
}
146
- h := hash(seed, k)
146
+ h := hash(seed, k) % defaultFanout
147
hashed[h] = append(hashed[h], item{k, data})
148
}
149
for h, items := range hashed {
pin/set_test.go
+60
-7
@@ -1,13 +1,66 @@
1
package pin
2
3
-import "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
3
+import (
4
+ "context"
5
+ "fmt"
6
+ "os"
7
+ "testing"
8
5
-func ignoreKeys(key.Key) {}
9
+ dag "github.com/ipfs/go-ipfs/merkledag"
10
+ mdtest "github.com/ipfs/go-ipfs/merkledag/test"
11
7
-func copyMap(m map[key.Key]uint16) map[key.Key]uint64 {
8
- c := make(map[key.Key]uint64, len(m))
9
- for k, v := range m {
10
- c[k] = uint64(v)
12
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
13
+)
14
+
15
+func ignoreCids(_ *cid.Cid) {}
16
+
17
+func TestSet(t *testing.T) {
18
+ ds := mdtest.Mock()
19
+ limit := 10000 // 10000 reproduces the pinloss issue fairly reliably
20
+
21
+ if os.Getenv("STRESS_IT_OUT_YO") != "" {
22
+ limit = 10000000
23
+ }
24
+ var inputs []*cid.Cid
25
+ for i := 0; i < limit; i++ {
26
+ c, err := ds.Add(dag.NodeWithData([]byte(fmt.Sprint(i))))
27
+ if err != nil {
28
+ t.Fatal(err)
29
+ }
30
+
31
+ inputs = append(inputs, c)
32
+ }
33
+
34
+ out, err := storeSet(context.Background(), ds, inputs, ignoreCids)
35
+ if err != nil {
36
+ t.Fatal(err)
37
+ }
38
+
39
+ // weird wrapper node because loadSet expects us to pass an
40
+ // object pointing to multiple named sets
41
+ setroot := &dag.Node{}
42
+ err = setroot.AddNodeLinkClean("foo", out)
43
+ if err != nil {
44
+ t.Fatal(err)
45
+ }
46
+
47
+ outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids)
48
+ if err != nil {
49
+ t.Fatal(err)
50
+ }
51
+
52
+ if len(outset) != limit {
53
+ t.Fatal("got wrong number", len(outset), limit)
54
+ }
55
+
56
+ seen := cid.NewSet()
57
+ for _, c := range outset {
58
+ seen.Add(c)
59
+ }
60
+
61
+ for _, c := range inputs {
62
+ if !seen.Has(c) {
63
+ t.Fatalf("expected to have %s, didnt find it")
64
+ }
65
}
12
- return c
66
}