Golint: unixfs/hamt
Note, stuttering required renaming of HamtShard to Shard. License: MIT Signed-off-by: Hector Sanjuan <hector@protocol.ai>
Hector Sanjuan committed
Feb 6, 2018 at 12:43 UTC
16934b9c68bad72dec2b0102c9c9847bf5f98728
3 files changed
+55
-51
unixfs/hamt/hamt.go
+37
-33
@@ -38,10 +38,12 @@ import (
38
)
39
40
const (
41
+ // HashMurmur3 is the multiformats identifier for Murmur3
42
HashMurmur3 uint64 = 0x22
43
)
44
44
-type HamtShard struct {
45
+// A Shard represents the HAMT. It should be initialized with NewShard().
46
+type Shard struct {
47
nd *dag.ProtoNode
48
49
bitfield *big.Int
@@ -66,9 +68,9 @@ type child interface {
68
Label() string
69
}
70
69
-// NewHamtShard creates a new, empty HAMT shard with the given size.
70
-func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) {
71
- ds, err := makeHamtShard(dserv, size)
71
+// NewShard creates a new, empty HAMT shard with the given size.
72
+func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
73
+ ds, err := makeShard(dserv, size)
74
if err != nil {
75
return nil, err
76
}
@@ -79,13 +81,13 @@ func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) {
81
return ds, nil
82
}
83
82
-func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) {
84
+func makeShard(ds ipld.DAGService, size int) (*Shard, error) {
85
lg2s := int(math.Log2(float64(size)))
86
if 1<<uint(lg2s) != size {
87
return nil, fmt.Errorf("hamt size should be a power of two")
88
}
89
maxpadding := fmt.Sprintf("%X", size-1)
88
- return &HamtShard{
90
+ return &Shard{
91
tableSizeLg2: lg2s,
92
prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)),
93
maxpadlen: len(maxpadding),
@@ -95,7 +97,7 @@ func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) {
97
}
98
99
// NewHamtFromDag creates new a HAMT shard from the given DAG.
98
-func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
100
+func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
101
pbnd, ok := nd.(*dag.ProtoNode)
102
if !ok {
103
return nil, dag.ErrLinkNotFound
@@ -114,7 +116,7 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
116
return nil, fmt.Errorf("only murmur3 supported as hash function")
117
}
118
117
- ds, err := makeHamtShard(dserv, int(pbd.GetFanout()))
119
+ ds, err := makeShard(dserv, int(pbd.GetFanout()))
120
if err != nil {
121
return nil, err
122
}
@@ -129,17 +131,17 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
131
}
132
133
// SetPrefix sets the CID Prefix
132
-func (ds *HamtShard) SetPrefix(prefix *cid.Prefix) {
134
+func (ds *Shard) SetPrefix(prefix *cid.Prefix) {
135
ds.prefix = prefix
136
}
137
138
// Prefix gets the CID Prefix, may be nil if unset
137
-func (ds *HamtShard) Prefix() *cid.Prefix {
139
+func (ds *Shard) Prefix() *cid.Prefix {
140
return ds.prefix
141
}
142
143
// Node serializes the HAMT structure into a merkledag node with unixfs formatting
142
-func (ds *HamtShard) Node() (ipld.Node, error) {
144
+func (ds *Shard) Node() (ipld.Node, error) {
145
out := new(dag.ProtoNode)
146
out.SetPrefix(ds.prefix)
147
@@ -214,14 +216,14 @@ func hash(val []byte) []byte {
216
return h.Sum(nil)
217
}
218
217
-// Label for HamtShards is the empty string, this is used to differentiate them from
219
+// Label for Shards is the empty string, this is used to differentiate them from
220
// value entries
219
-func (ds *HamtShard) Label() string {
221
+func (ds *Shard) Label() string {
222
return ""
223
}
224
225
// Set sets 'name' = nd in the HAMT
224
-func (ds *HamtShard) Set(ctx context.Context, name string, nd ipld.Node) error {
226
+func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error {
227
hv := &hashBits{b: hash([]byte(name))}
228
err := ds.dserv.Add(ctx, nd)
229
if err != nil {
@@ -238,13 +240,13 @@ func (ds *HamtShard) Set(ctx context.Context, name string, nd ipld.Node) error {
240
}
241
242
// Remove deletes the named entry if it exists, this operation is idempotent.
241
-func (ds *HamtShard) Remove(ctx context.Context, name string) error {
243
+func (ds *Shard) Remove(ctx context.Context, name string) error {
244
hv := &hashBits{b: hash([]byte(name))}
245
return ds.modifyValue(ctx, hv, name, nil)
246
}
247
248
// Find searches for a child node by 'name' within this hamt
247
-func (ds *HamtShard) Find(ctx context.Context, name string) (*ipld.Link, error) {
249
+func (ds *Shard) Find(ctx context.Context, name string) (*ipld.Link, error) {
250
hv := &hashBits{b: hash([]byte(name))}
251
252
var out *ipld.Link
@@ -262,7 +264,7 @@ func (ds *HamtShard) Find(ctx context.Context, name string) (*ipld.Link, error)
264
// getChild returns the i'th child of this shard. If it is cached in the
265
// children array, it will return it from there. Otherwise, it loads the child
266
// node from disk.
265
-func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) {
267
+func (ds *Shard) getChild(ctx context.Context, i int) (child, error) {
268
if i >= len(ds.children) || i < 0 {
269
return nil, fmt.Errorf("invalid index passed to getChild (likely corrupt bitfield)")
270
}
@@ -281,7 +283,7 @@ func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) {
283
284
// loadChild reads the i'th child node of this shard from disk and returns it
285
// as a 'child' interface
284
-func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
286
+func (ds *Shard) loadChild(ctx context.Context, i int) (child, error) {
287
lnk := ds.nd.Links()[i]
288
if len(lnk.Name) < ds.maxpadlen {
289
return nil, fmt.Errorf("invalid link name '%s'", lnk.Name)
@@ -326,12 +328,12 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
328
return c, nil
329
}
330
329
-func (ds *HamtShard) setChild(i int, c child) {
331
+func (ds *Shard) setChild(i int, c child) {
332
ds.children[i] = c
333
}
334
335
// Link returns a merklelink to this shard node
334
-func (ds *HamtShard) Link() (*ipld.Link, error) {
336
+func (ds *Shard) Link() (*ipld.Link, error) {
337
nd, err := ds.Node()
338
if err != nil {
339
return nil, err
@@ -345,7 +347,7 @@ func (ds *HamtShard) Link() (*ipld.Link, error) {
347
return ipld.MakeLink(nd)
348
}
349
348
-func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error {
350
+func (ds *Shard) insertChild(idx int, key string, lnk *ipld.Link) error {
351
if lnk == nil {
352
return os.ErrNotExist
353
}
@@ -364,7 +366,7 @@ func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error {
366
return nil
367
}
368
367
-func (ds *HamtShard) rmChild(i int) error {
369
+func (ds *Shard) rmChild(i int) error {
370
if i < 0 || i >= len(ds.children) || i >= len(ds.nd.Links()) {
371
return fmt.Errorf("hamt: attempted to remove child with out of range index")
372
}
@@ -378,7 +380,7 @@ func (ds *HamtShard) rmChild(i int) error {
380
return nil
381
}
382
381
-func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
383
+func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
384
idx := hv.Next(ds.tableSizeLg2)
385
if ds.bitfield.Bit(int(idx)) == 1 {
386
cindex := ds.indexForBitPos(idx)
@@ -389,7 +391,7 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
391
}
392
393
switch child := child.(type) {
392
- case *HamtShard:
394
+ case *Shard:
395
return child.getValue(ctx, hv, key, cb)
396
case *shardValue:
397
if child.key == key {
@@ -401,7 +403,8 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
403
return os.ErrNotExist
404
}
405
404
-func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
406
+// EnumLinks collects all links in the Shard.
407
+func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
408
var links []*ipld.Link
409
err := ds.ForEachLink(ctx, func(l *ipld.Link) error {
410
links = append(links, l)
@@ -410,7 +413,8 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
413
return links, err
414
}
415
413
-func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
416
+// ForEachLink walks the Shard and calls the given function.
417
+func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
418
return ds.walkTrie(ctx, func(sv *shardValue) error {
419
lnk := sv.val
420
lnk.Name = sv.key
@@ -419,7 +423,7 @@ func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error)
423
})
424
}
425
422
-func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
426
+func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
427
for i := 0; i < ds.tableSize; i++ {
428
if ds.bitfield.Bit(i) == 0 {
429
continue
@@ -440,7 +444,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
444
return err
445
}
446
443
- case *HamtShard:
447
+ case *Shard:
448
err := c.walkTrie(ctx, cb)
449
if err != nil {
450
return err
@@ -452,7 +456,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
456
return nil
457
}
458
455
-func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
459
+func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
460
idx := hv.Next(ds.tableSizeLg2)
461
462
if ds.bitfield.Bit(idx) != 1 {
@@ -467,7 +471,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
471
}
472
473
switch child := child.(type) {
470
- case *HamtShard:
474
+ case *Shard:
475
err := child.modifyValue(ctx, hv, key, val)
476
if err != nil {
477
return err
@@ -510,7 +514,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
514
}
515
516
// replace value with another shard, one level deeper
513
- ns, err := NewHamtShard(ds.dserv, ds.tableSize)
517
+ ns, err := NewShard(ds.dserv, ds.tableSize)
518
if err != nil {
519
return err
520
}
@@ -540,7 +544,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
544
// indexForBitPos returns the index within the collapsed array corresponding to
545
// the given bit in the bitset. The collapsed array contains only one entry
546
// per bit set in the bitfield, and this function is used to map the indices.
543
-func (ds *HamtShard) indexForBitPos(bp int) int {
547
+func (ds *Shard) indexForBitPos(bp int) int {
548
// TODO: an optimization could reuse the same 'mask' here and change the size
549
// as needed. This isnt yet done as the bitset package doesnt make it easy
550
// to do.
@@ -553,6 +557,6 @@ func (ds *HamtShard) indexForBitPos(bp int) int {
557
}
558
559
// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix
556
-func (ds *HamtShard) linkNamePrefix(idx int) string {
560
+func (ds *Shard) linkNamePrefix(idx int) string {
561
return fmt.Sprintf(ds.prefixPadStr, idx)
562
}
unixfs/hamt/hamt_stress_test.go
+6
-6
@@ -94,7 +94,7 @@ func TestOrderConsistency(t *testing.T) {
94
}
95
}
96
97
-func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) error {
97
+func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error {
98
ctx := context.TODO()
99
for _, n := range keep {
100
_, err := s.Find(ctx, n)
@@ -113,9 +113,9 @@ func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) er
113
return nil
114
}
115
116
-func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*HamtShard, error) {
116
+func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) {
117
ctx := context.TODO()
118
- s, err := NewHamtShard(ds, width)
118
+ s, err := NewShard(ds, width)
119
if err != nil {
120
return nil, err
121
}
@@ -189,9 +189,9 @@ func genOpSet(seed int64, keep, temp []string) []testOp {
189
}
190
191
// executes the given op set with a repl to allow easier debugging
192
-/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*HamtShard, error) {
192
+/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) {
193
194
- s, err := NewHamtShard(ds, width)
194
+ s, err := NewShard(ds, width)
195
if err != nil {
196
return nil, err
197
}
@@ -244,7 +244,7 @@ mainloop:
244
}
245
case "restart":
246
var err error
247
- s, err = NewHamtShard(ds, width)
247
+ s, err = NewShard(ds, width)
248
if err != nil {
249
panic(err)
250
}
unixfs/hamt/hamt_test.go
+12
-12
@@ -26,14 +26,14 @@ func shuffle(seed int64, arr []string) {
26
}
27
}
28
29
-func makeDir(ds ipld.DAGService, size int) ([]string, *HamtShard, error) {
29
+func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) {
30
return makeDirWidth(ds, size, 256)
31
}
32
33
-func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, error) {
33
+func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) {
34
ctx := context.Background()
35
36
- s, _ := NewHamtShard(ds, width)
36
+ s, _ := NewShard(ds, width)
37
38
var dirs []string
39
for i := 0; i < size; i++ {
@@ -54,7 +54,7 @@ func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, er
54
return dirs, s, nil
55
}
56
57
-func assertLink(s *HamtShard, name string, found bool) error {
57
+func assertLink(s *Shard, name string, found bool) error {
58
_, err := s.Find(context.Background(), name)
59
switch err {
60
case os.ErrNotExist:
@@ -74,7 +74,7 @@ func assertLink(s *HamtShard, name string, found bool) error {
74
}
75
}
76
77
-func assertSerializationWorks(ds ipld.DAGService, s *HamtShard) error {
77
+func assertSerializationWorks(ds ipld.DAGService, s *Shard) error {
78
ctx, cancel := context.WithCancel(context.Background())
79
defer cancel()
80
nd, err := s.Node()
@@ -141,7 +141,7 @@ func TestBasicSet(t *testing.T) {
141
142
func TestDirBuilding(t *testing.T) {
143
ds := mdtest.Mock()
144
- _, _ = NewHamtShard(ds, 256)
144
+ _, _ = NewShard(ds, 256)
145
146
_, s, err := makeDir(ds, 200)
147
if err != nil {
@@ -164,7 +164,7 @@ func TestDirBuilding(t *testing.T) {
164
165
func TestShardReload(t *testing.T) {
166
ds := mdtest.Mock()
167
- _, _ = NewHamtShard(ds, 256)
167
+ _, _ = NewShard(ds, 256)
168
ctx, cancel := context.WithCancel(context.Background())
169
defer cancel()
170
@@ -307,7 +307,7 @@ func TestSetAfterMarshal(t *testing.T) {
307
308
func TestDuplicateAddShard(t *testing.T) {
309
ds := mdtest.Mock()
310
- dir, _ := NewHamtShard(ds, 256)
310
+ dir, _ := NewShard(ds, 256)
311
nd := new(dag.ProtoNode)
312
ctx := context.Background()
313
@@ -430,7 +430,7 @@ func TestRemoveElemsAfterMarshal(t *testing.T) {
430
431
func TestBitfieldIndexing(t *testing.T) {
432
ds := mdtest.Mock()
433
- s, _ := NewHamtShard(ds, 256)
433
+ s, _ := NewShard(ds, 256)
434
435
set := func(i int) {
436
s.bitfield.SetBit(s.bitfield, i, 1)
@@ -466,7 +466,7 @@ func TestSetHamtChild(t *testing.T) {
466
ctx := context.Background()
467
468
ds := mdtest.Mock()
469
- s, _ := NewHamtShard(ds, 256)
469
+ s, _ := NewShard(ds, 256)
470
471
e := ft.EmptyDirNode()
472
ds.Add(ctx, e)
@@ -527,7 +527,7 @@ func BenchmarkHAMTSet(b *testing.B) {
527
ctx := context.Background()
528
529
ds := mdtest.Mock()
530
- sh, _ := NewHamtShard(ds, 256)
530
+ sh, _ := NewShard(ds, 256)
531
nd, err := sh.Node()
532
if err != nil {
533
b.Fatal(err)
@@ -560,7 +560,7 @@ func BenchmarkHAMTSet(b *testing.B) {
560
}
561
562
func TestHamtBadSize(t *testing.T) {
563
- _, err := NewHamtShard(nil, 7)
563
+ _, err := NewShard(nil, 7)
564
if err == nil {
565
t.Fatal("should have failed to construct hamt with bad size")
566
}