fix bug in pinsets and add a stress test for the scenario
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Sep 29, 2016 at 12:35 UTC
cb3bda78b3a4afb5cc5e7c2c70a651a113aaaae4
2 files changed
+47
-85
pin/set.go
+1
-1
@@ -143,7 +143,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
143
if !ok {
144
break
145
}
146
- h := hash(seed, k)
146
+ h := hash(seed, k) % defaultFanout
147
hashed[h] = append(hashed[h], item{k, data})
148
}
149
for h, items := range hashed {
pin/set_test.go
+46
-84
@@ -1,103 +1,65 @@
1
package pin
2
3
import (
4
+ "context"
5
+ "fmt"
6
+ "os"
7
"testing"
5
- "testing/quick"
8
7
- "github.com/ipfs/go-ipfs/blocks/blockstore"
8
- "github.com/ipfs/go-ipfs/blocks/key"
9
- "github.com/ipfs/go-ipfs/blockservice"
10
- "github.com/ipfs/go-ipfs/exchange/offline"
11
- "github.com/ipfs/go-ipfs/merkledag"
12
- "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore"
13
- dssync "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore/sync"
14
- mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
15
- u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
16
- "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
9
+ key "github.com/ipfs/go-ipfs/blocks/key"
10
+ dag "github.com/ipfs/go-ipfs/merkledag"
11
+ mdtest "github.com/ipfs/go-ipfs/merkledag/test"
12
)
13
19
-func ignoreKeys(key.Key) {}
14
+func ignoreKey(_ key.Key) {}
15
21
-func copyMap(m map[key.Key]uint16) map[key.Key]uint64 {
22
- c := make(map[key.Key]uint64, len(m))
23
- for k, v := range m {
24
- c[k] = uint64(v)
25
- }
26
- return c
27
-}
16
+func TestSet(t *testing.T) {
17
+ ds := mdtest.Mock()
18
+ limit := 10000 // 10000 reproduces the pinloss issue fairly reliably
19
29
-func TestMultisetRoundtrip(t *testing.T) {
30
- dstore := dssync.MutexWrap(datastore.NewMapDatastore())
31
- bstore := blockstore.NewBlockstore(dstore)
32
- bserv := blockservice.New(bstore, offline.Exchange(bstore))
33
- dag := merkledag.NewDAGService(bserv)
34
-
35
- fn := func(m map[key.Key]uint16) bool {
36
- // Convert invalid multihash from input to valid ones
37
- for k, v := range m {
38
- if _, err := mh.Cast([]byte(k)); err != nil {
39
- delete(m, k)
40
- m[key.Key(u.Hash([]byte(k)))] = v
41
- }
20
+ if os.Getenv("STRESS_IT_OUT_YO") != "" {
21
+ limit = 10000000
22
+ }
23
+ var inputs []key.Key
24
+ for i := 0; i < limit; i++ {
25
+ c, err := ds.Add(dag.NodeWithData([]byte(fmt.Sprint(i))))
26
+ if err != nil {
27
+ t.Fatal(err)
28
}
29
44
- // Generate a smaller range for refcounts than full uint64, as
45
- // otherwise this just becomes overly cpu heavy, splitting it
46
- // out into too many items. That means we need to convert to
47
- // the right kind of map. As storeMultiset mutates the map as
48
- // part of its bookkeeping, this is actually good.
49
- refcounts := copyMap(m)
30
+ inputs = append(inputs, c)
31
+ }
32
51
- ctx := context.Background()
52
- n, err := storeMultiset(ctx, dag, refcounts, ignoreKeys)
53
- if err != nil {
54
- t.Fatalf("storing multiset: %v", err)
55
- }
33
+ out, err := storeSet(context.Background(), ds, inputs, ignoreKey)
34
+ if err != nil {
35
+ t.Fatal(err)
36
+ }
37
57
- // Check that the node n is in the DAG
58
- k, err := n.Key()
59
- if err != nil {
60
- t.Fatalf("Could not get key: %v", err)
61
- }
62
- _, err = dag.Get(ctx, k)
63
- if err != nil {
64
- t.Fatalf("Could not get node: %v", err)
65
- }
38
+ // weird wrapper node because loadSet expects us to pass an
39
+ // object pointing to multiple named sets
40
+ setroot := &dag.Node{}
41
+ err = setroot.AddNodeLinkClean("foo", out)
42
+ if err != nil {
43
+ t.Fatal(err)
44
+ }
45
67
- root := &merkledag.Node{}
68
- const linkName = "dummylink"
69
- if err := root.AddNodeLink(linkName, n); err != nil {
70
- t.Fatalf("adding link to root node: %v", err)
71
- }
46
+ outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreKey)
47
+ if err != nil {
48
+ t.Fatal(err)
49
+ }
50
73
- roundtrip, err := loadMultiset(ctx, dag, root, linkName, ignoreKeys)
74
- if err != nil {
75
- t.Fatalf("loading multiset: %v", err)
76
- }
51
+ if len(outset) != limit {
52
+ t.Fatal("got wrong number", len(outset), limit)
53
+ }
54
78
- orig := copyMap(m)
79
- success := true
80
- for k, want := range orig {
81
- if got, ok := roundtrip[k]; ok {
82
- if got != want {
83
- success = false
84
- t.Logf("refcount changed: %v -> %v for %q", want, got, k)
85
- }
86
- delete(orig, k)
87
- delete(roundtrip, k)
88
- }
89
- }
90
- for k, v := range orig {
91
- success = false
92
- t.Logf("refcount missing: %v for %q", v, k)
93
- }
94
- for k, v := range roundtrip {
95
- success = false
96
- t.Logf("refcount extra: %v for %q", v, k)
97
- }
98
- return success
55
+ seen := key.NewKeySet()
56
+ for _, c := range outset {
57
+ seen.Add(c)
58
}
100
- if err := quick.Check(fn, nil); err != nil {
101
- t.Fatal(err)
59
+
60
+ for _, c := range inputs {
61
+ if !seen.Has(c) {
62
+ t.Fatalf("expected to have %s, didnt find it")
63
+ }
64
}
65
}