pin: Future-proof against refcount marshaled size changes
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Tommi Virtanen committed
May 18, 2015 at 14:01 UTC
72753e54606ffc454a6902194b280c8c4f8a7f69
2 files changed
+108
-6
pin/set.go
+23
-6
@@ -44,14 +44,29 @@ type itemIterator func() (k key.Key, data []byte, ok bool)
44
45
type keyObserver func(key.Key)
46
47
+// refcount is the marshaled format of refcounts. It may change
48
+// between versions; this is valid for version 1. Changing it may
49
+// become desirable if there are many links with refcount > 255.
50
+//
51
+// There are two guarantees that need to be preserved, if this is
52
+// changed:
53
+//
54
+// - the marshaled format is of fixed size, matching
55
+// unsafe.Sizeof(refcount(0))
56
+// - methods of refcount handle endianness, and may
57
+// in later versions need encoding/binary.
58
type refcount uint8
59
60
func (r refcount) Bytes() []byte {
50
- // refcount size can change in later versions; this may need
51
- // encoding/binary
61
return []byte{byte(r)}
62
}
63
64
+// readRefcount returns the idx'th refcount in []byte, which is
65
+// assumed to be a sequence of refcount.Bytes results.
66
+func (r *refcount) ReadFromIdx(buf []byte, idx int) {
67
+ *r = refcount(buf[idx])
68
+}
69
+
70
type sortByHash struct {
71
links []*merkledag.Link
72
data []byte
@@ -70,9 +85,9 @@ func (s sortByHash) Swap(a, b int) {
85
if len(s.data) != 0 {
86
const n = int(unsafe.Sizeof(refcount(0)))
87
tmp := make([]byte, n)
73
- copy(tmp, s.data[a:a+n])
74
- copy(s.data[a:a+n], s.data[b:b+n])
75
- copy(s.data[b:b+n], tmp)
88
+ copy(tmp, s.data[a*n:a*n+n])
89
+ copy(s.data[a*n:a*n+n], s.data[b*n:b*n+n])
90
+ copy(s.data[b*n:b*n+n], tmp)
91
}
92
}
93
@@ -267,7 +282,9 @@ func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag
282
283
refcounts := make(map[key.Key]uint64)
284
walk := func(buf []byte, idx int, link *merkledag.Link) error {
270
- refcounts[key.Key(link.Hash)] += uint64(buf[idx])
285
+ var r refcount
286
+ r.ReadFromIdx(buf, idx)
287
+ refcounts[key.Key(link.Hash)] += uint64(r)
288
return nil
289
}
290
if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
pin/set_test.go
new
+85
@@ -0,0 +1,85 @@
1
+package pin
2
+
3
+import (
4
+ "testing"
5
+ "testing/quick"
6
+
7
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8
+ dssync "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
9
+ "github.com/ipfs/go-ipfs/blocks/blockstore"
10
+ "github.com/ipfs/go-ipfs/blocks/key"
11
+ "github.com/ipfs/go-ipfs/blockservice"
12
+ "github.com/ipfs/go-ipfs/exchange/offline"
13
+ "github.com/ipfs/go-ipfs/merkledag"
14
+ "golang.org/x/net/context"
15
+)
16
+
17
+func ignoreKeys(key.Key) {}
18
+
19
+func copyMap(m map[key.Key]uint16) map[key.Key]uint64 {
20
+ c := make(map[key.Key]uint64, len(m))
21
+ for k, v := range m {
22
+ c[k] = uint64(v)
23
+ }
24
+ return c
25
+}
26
+
27
+func TestMultisetRoundtrip(t *testing.T) {
28
+ dstore := dssync.MutexWrap(datastore.NewMapDatastore())
29
+ bstore := blockstore.NewBlockstore(dstore)
30
+ bserv, err := blockservice.New(bstore, offline.Exchange(bstore))
31
+ if err != nil {
32
+ t.Fatal(err)
33
+ }
34
+ dag := merkledag.NewDAGService(bserv)
35
+
36
+ fn := func(m map[key.Key]uint16) bool {
37
+ // Generate a smaller range for refcounts than full uint64, as
38
+ // otherwise this just becomes overly cpu heavy, splitting it
39
+ // out into too many items. That means we need to convert to
40
+ // the right kind of map. As storeMultiset mutates the map as
41
+ // part of its bookkeeping, this is actually good.
42
+ refcounts := copyMap(m)
43
+
44
+ ctx := context.Background()
45
+ n, err := storeMultiset(ctx, dag, refcounts, ignoreKeys)
46
+ if err != nil {
47
+ t.Fatalf("storing multiset: %v", err)
48
+ }
49
+ root := &merkledag.Node{}
50
+ const linkName = "dummylink"
51
+ if err := root.AddNodeLink(linkName, n); err != nil {
52
+ t.Fatalf("adding link to root node: %v", err)
53
+ }
54
+
55
+ roundtrip, err := loadMultiset(ctx, dag, root, linkName, ignoreKeys)
56
+ if err != nil {
57
+ t.Fatalf("loading multiset: %v", err)
58
+ }
59
+
60
+ orig := copyMap(m)
61
+ success := true
62
+ for k, want := range orig {
63
+ if got, ok := roundtrip[k]; ok {
64
+ if got != want {
65
+ success = false
66
+ t.Logf("refcount changed: %v -> %v for %q", want, got, k)
67
+ }
68
+ delete(orig, k)
69
+ delete(roundtrip, k)
70
+ }
71
+ }
72
+ for k, v := range orig {
73
+ success = false
74
+ t.Logf("refcount missing: %v for %q", v, k)
75
+ }
76
+ for k, v := range roundtrip {
77
+ success = false
78
+ t.Logf("refcount extra: %v for %q", v, k)
79
+ }
80
+ return success
81
+ }
82
+ if err := quick.Check(fn, nil); err != nil {
83
+ t.Fatal(err)
84
+ }
85
+}