Remove legacy multiset 'data' fields, comment and cleanup more
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Oct 6, 2016 at 19:08 UTC
a49483bf58c1295b453fd6957760225a5ba76b7f
1 file changed
+54
-100
pin/set.go
+54
-100
@@ -2,15 +2,14 @@ package pin
2
3
import (
4
"bytes"
5
+ "context"
6
"crypto/rand"
7
"encoding/binary"
8
"errors"
9
"fmt"
10
"hash/fnv"
11
"sort"
11
- "unsafe"
12
13
- "context"
13
"github.com/ipfs/go-ipfs/merkledag"
14
"github.com/ipfs/go-ipfs/pin/internal/pb"
15
"gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
@@ -19,8 +18,11 @@ import (
18
)
19
20
const (
21
+ // defaultFanout specifies the default number of fan-out links per layer
22
defaultFanout = 256
23
- maxItems = 8192
23
+
24
+ // maxItems is the maximum number of items that will fit in a single bucket
25
+ maxItems = 8192
26
)
27
28
func randomSeed() (uint32, error) {
@@ -40,36 +42,12 @@ func hash(seed uint32, c *cid.Cid) uint32 {
42
return h.Sum32()
43
}
44
43
-type itemIterator func() (c *cid.Cid, data []byte, ok bool)
45
+type itemIterator func() (c *cid.Cid, ok bool)
46
47
type keyObserver func(*cid.Cid)
48
47
-// refcount is the marshaled format of refcounts. It may change
48
-// between versions; this is valid for version 1. Changing it may
49
-// become desirable if there are many links with refcount > 255.
50
-//
51
-// There are two guarantees that need to be preserved, if this is
52
-// changed:
53
-//
54
-// - the marshaled format is of fixed size, matching
55
-// unsafe.Sizeof(refcount(0))
56
-// - methods of refcount handle endianness, and may
57
-// in later versions need encoding/binary.
58
-type refcount uint8
59
-
60
-func (r refcount) Bytes() []byte {
61
- return []byte{byte(r)}
62
-}
63
-
64
-// readRefcount returns the idx'th refcount in []byte, which is
65
-// assumed to be a sequence of refcount.Bytes results.
66
-func (r *refcount) ReadFromIdx(buf []byte, idx int) {
67
- *r = refcount(buf[idx])
68
-}
69
-
49
type sortByHash struct {
50
links []*merkledag.Link
72
- data []byte
51
}
52
53
func (s sortByHash) Len() int {
@@ -82,13 +60,6 @@ func (s sortByHash) Less(a, b int) bool {
60
61
func (s sortByHash) Swap(a, b int) {
62
s.links[a], s.links[b] = s.links[b], s.links[a]
85
- if len(s.data) != 0 {
86
- const n = int(unsafe.Sizeof(refcount(0)))
87
- tmp := make([]byte, n)
88
- copy(tmp, s.data[a*n:a*n+n])
89
- copy(s.data[a*n:a*n+n], s.data[b*n:b*n+n])
90
- copy(s.data[b*n:b*n+n], tmp)
91
- }
63
}
64
65
func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint64, iter itemIterator, internalKeys keyObserver) (*merkledag.Node, error) {
@@ -96,13 +67,15 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
67
if err != nil {
68
return nil, err
69
}
99
- n := &merkledag.Node{
100
- Links: make([]*merkledag.Link, 0, defaultFanout+maxItems),
101
- }
70
+
71
+ n := &merkledag.Node{Links: make([]*merkledag.Link, 0, defaultFanout+maxItems)}
72
for i := 0; i < defaultFanout; i++ {
73
n.Links = append(n.Links, &merkledag.Link{Hash: emptyKey.Hash()})
74
}
75
+
76
+ // add emptyKey to our set of internal pinset objects
77
internalKeys(emptyKey)
78
+
79
hdr := &pb.Set{
80
Version: proto.Uint32(1),
81
Fanout: proto.Uint32(defaultFanout),
@@ -111,23 +84,20 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
84
if err := writeHdr(n, hdr); err != nil {
85
return nil, err
86
}
114
- hdrLen := len(n.Data())
87
88
if estimatedLen < maxItems {
89
// it'll probably fit
90
for i := 0; i < maxItems; i++ {
119
- k, data, ok := iter()
91
+ k, ok := iter()
92
if !ok {
93
// all done
94
break
95
}
96
n.Links = append(n.Links, &merkledag.Link{Hash: k.Hash()})
125
- n.SetData(append(n.Data(), data...))
97
}
98
// sort by hash, also swap item Data
99
s := sortByHash{
100
links: n.Links[defaultFanout:],
130
- data: n.Data()[hdrLen:],
101
}
102
sort.Stable(s)
103
}
@@ -147,7 +117,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
117
// and losing pins. The fix (a few lines down from this comment), is to
118
// map the hash value down to the 8 bit keyspace here while creating the
119
// buckets. This way, we avoid any overlapping later on.
150
- k, _, ok := iter()
120
+ k, ok := iter()
121
if !ok {
122
break
123
}
@@ -161,15 +131,9 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
131
continue
132
}
133
164
- childIter := func() (c *cid.Cid, data []byte, ok bool) {
165
- if len(items) == 0 {
166
- return nil, nil, false
167
- }
168
- first := items[0]
169
- items = items[1:]
170
- return first, nil, true
171
- }
134
+ childIter := getCidListIterator(items)
135
136
+ // recursively create a pinset from the items for this bucket index
137
child, err := storeItems(ctx, dag, uint64(len(items)), childIter, internalKeys)
138
if err != nil {
139
return nil, err
@@ -186,7 +150,9 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
150
}
151
152
internalKeys(childKey)
189
- n.Links[int(h)] = &merkledag.Link{
153
+
154
+ // overwrite the 'empty key' in the existing links array
155
+ n.Links[h] = &merkledag.Link{
156
Hash: childKey.Hash(),
157
Size: size,
158
}
@@ -194,30 +160,30 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
160
return n, nil
161
}
162
197
-func readHdr(n *merkledag.Node) (*pb.Set, []byte, error) {
163
+func readHdr(n *merkledag.Node) (*pb.Set, error) {
164
hdrLenRaw, consumed := binary.Uvarint(n.Data())
165
if consumed <= 0 {
200
- return nil, nil, errors.New("invalid Set header length")
166
+ return nil, errors.New("invalid Set header length")
167
}
202
- buf := n.Data()[consumed:]
203
- if hdrLenRaw > uint64(len(buf)) {
204
- return nil, nil, errors.New("impossibly large Set header length")
168
+
169
+ pbdata := n.Data()[consumed:]
170
+ if hdrLenRaw > uint64(len(pbdata)) {
171
+ return nil, errors.New("impossibly large Set header length")
172
}
173
// as hdrLenRaw was <= an int, we now know it fits in an int
174
hdrLen := int(hdrLenRaw)
175
var hdr pb.Set
209
- if err := proto.Unmarshal(buf[:hdrLen], &hdr); err != nil {
210
- return nil, nil, err
176
+ if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil {
177
+ return nil, err
178
}
212
- buf = buf[hdrLen:]
179
180
if v := hdr.GetVersion(); v != 1 {
215
- return nil, nil, fmt.Errorf("unsupported Set version: %d", v)
181
+ return nil, fmt.Errorf("unsupported Set version: %d", v)
182
}
183
if uint64(hdr.GetFanout()) > uint64(len(n.Links)) {
218
- return nil, nil, errors.New("impossibly large Fanout")
184
+ return nil, errors.New("impossibly large Fanout")
185
}
220
- return &hdr, buf, nil
186
+ return &hdr, nil
187
}
188
189
func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
@@ -225,24 +191,31 @@ func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
191
if err != nil {
192
return err
193
}
228
- n.SetData(make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)))
229
- written := binary.PutUvarint(n.Data(), uint64(len(hdrData)))
230
- n.SetData(n.Data()[:written])
231
- n.SetData(append(n.Data(), hdrData...))
194
+
195
+ // make enough space for the length prefix and the marshalled header data
196
+ data := make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData))
197
+
198
+ // write the uvarint length of the header data
199
+ uvarlen := binary.PutUvarint(data, uint64(len(hdrData)))
200
+
201
+ // append the actual protobuf data *after* the length value we wrote
202
+ data = append(data[:uvarlen], hdrData...)
203
+
204
+ n.SetData(data)
205
return nil
206
}
207
235
-type walkerFunc func(buf []byte, idx int, link *merkledag.Link) error
208
+type walkerFunc func(idx int, link *merkledag.Link) error
209
210
func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.Node, fn walkerFunc, children keyObserver) error {
238
- hdr, buf, err := readHdr(n)
211
+ hdr, err := readHdr(n)
212
if err != nil {
213
return err
214
}
215
// readHdr guarantees fanout is a safe value
216
fanout := hdr.GetFanout()
217
for i, l := range n.Links[fanout:] {
245
- if err := fn(buf, i, l); err != nil {
218
+ if err := fn(i, l); err != nil {
219
return err
220
}
221
}
@@ -278,7 +251,7 @@ func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node
251
}
252
253
var res []*cid.Cid
281
- walk := func(buf []byte, idx int, link *merkledag.Link) error {
254
+ walk := func(idx int, link *merkledag.Link) error {
255
res = append(res, cid.NewCidV0(link.Hash))
256
return nil
257
}
@@ -288,40 +261,21 @@ func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node
261
return res, nil
262
}
263
291
-func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) (map[key.Key]uint64, error) {
292
- l, err := root.GetNodeLink(name)
293
- if err != nil {
294
- return nil, fmt.Errorf("Failed to get link %s: %v", name, err)
295
- }
296
- c := cid.NewCidV0(l.Hash)
297
- internalKeys(c)
298
- n, err := l.GetNode(ctx, dag)
299
- if err != nil {
300
- return nil, fmt.Errorf("Failed to get node from link %s: %v", name, err)
301
- }
302
-
303
- refcounts := make(map[key.Key]uint64)
304
- walk := func(buf []byte, idx int, link *merkledag.Link) error {
305
- var r refcount
306
- r.ReadFromIdx(buf, idx)
307
- refcounts[key.Key(link.Hash)] += uint64(r)
308
- return nil
309
- }
310
- if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
311
- return nil, err
312
- }
313
- return refcounts, nil
314
-}
315
-
316
-func storeSet(ctx context.Context, dag merkledag.DAGService, cids []*cid.Cid, internalKeys keyObserver) (*merkledag.Node, error) {
317
- iter := func() (c *cid.Cid, data []byte, ok bool) {
264
+func getCidListIterator(cids []*cid.Cid) itemIterator {
265
+ return func() (c *cid.Cid, ok bool) {
266
if len(cids) == 0 {
319
- return nil, nil, false
267
+ return nil, false
268
}
269
+
270
first := cids[0]
271
cids = cids[1:]
323
- return first, nil, true
272
+ return first, true
273
}
274
+}
275
+
276
+func storeSet(ctx context.Context, dag merkledag.DAGService, cids []*cid.Cid, internalKeys keyObserver) (*merkledag.Node, error) {
277
+ iter := getCidListIterator(cids)
278
+
279
n, err := storeItems(ctx, dag, uint64(len(cids)), iter, internalKeys)
280
if err != nil {
281
return nil, err