faster hamt logic
1. Use a custom bitfield type instead of bigints. 2. Make iterating over a hamt *significantly* faster. License: MIT Signed-off-by: Steven Allen <steven@stebalien.com>
Steven Allen committed
Mar 28, 2018 at 17:41 UTC
f1ae13d7214b22d051bcccb52a4604b1fd7b61d2
5 files changed
+35
-56
package.json
+6
@@ -581,6 +581,12 @@
581
"hash": "QmPdqSMmiwtQCBC515gFtMW2mP14HsfgnyQ2k5xPQVxMge",
582
"name": "go-fs-lock",
583
"version": "0.1.2"
584
+ },
585
+ {
586
+ "author": "Stebalien",
587
+ "hash": "QmTbBs3Y3u5F69XNJzdnnc6SP5GKgcXxCDzx6w8m6piVRT",
588
+ "name": "go-bitfield",
589
+ "version": "0.1.1"
590
}
591
],
592
"gxVersion": "0.10.0",
unixfs/hamt/hamt.go
+19
-36
@@ -23,14 +23,13 @@ package hamt
23
import (
24
"context"
25
"fmt"
26
- "math"
27
- "math/big"
26
"os"
27
28
dag "github.com/ipfs/go-ipfs/merkledag"
29
format "github.com/ipfs/go-ipfs/unixfs"
30
upb "github.com/ipfs/go-ipfs/unixfs/pb"
31
32
+ bitfield "gx/ipfs/QmTbBs3Y3u5F69XNJzdnnc6SP5GKgcXxCDzx6w8m6piVRT/go-bitfield"
33
proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
34
cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
35
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
@@ -46,7 +45,7 @@ const (
45
type Shard struct {
46
nd *dag.ProtoNode
47
49
- bitfield *big.Int
48
+ bitfield bitfield.Bitfield
49
50
children []child
51
@@ -75,22 +74,22 @@ func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
74
return nil, err
75
}
76
78
- ds.bitfield = big.NewInt(0)
77
ds.nd = new(dag.ProtoNode)
78
ds.hashFunc = HashMurmur3
79
return ds, nil
80
}
81
82
func makeShard(ds ipld.DAGService, size int) (*Shard, error) {
85
- lg2s := int(math.Log2(float64(size)))
86
- if 1<<uint(lg2s) != size {
87
- return nil, fmt.Errorf("hamt size should be a power of two")
83
+ lg2s, err := logtwo(size)
84
+ if err != nil {
85
+ return nil, err
86
}
87
maxpadding := fmt.Sprintf("%X", size-1)
88
return &Shard{
89
tableSizeLg2: lg2s,
90
prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)),
91
maxpadlen: len(maxpadding),
92
+ bitfield: bitfield.NewBitfield(size),
93
tableSize: size,
94
dserv: ds,
95
}, nil
@@ -123,7 +122,7 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
122
123
ds.nd = pbnd.Copy().(*dag.ProtoNode)
124
ds.children = make([]child, len(pbnd.Links()))
126
- ds.bitfield = new(big.Int).SetBytes(pbd.GetData())
125
+ ds.bitfield.SetBytes(pbd.GetData())
126
ds.hashFunc = pbd.GetHashType()
127
ds.prefix = &ds.nd.Prefix
128
@@ -145,13 +144,13 @@ func (ds *Shard) Node() (ipld.Node, error) {
144
out := new(dag.ProtoNode)
145
out.SetPrefix(ds.prefix)
146
147
+ cindex := 0
148
// TODO: optimized 'for each set bit'
149
for i := 0; i < ds.tableSize; i++ {
150
- if ds.bitfield.Bit(i) == 0 {
150
+ if !ds.bitfield.Bit(i) {
151
continue
152
}
153
154
- cindex := ds.indexForBitPos(i)
154
ch := ds.children[cindex]
155
if ch != nil {
156
clnk, err := ch.Link()
@@ -173,6 +172,7 @@ func (ds *Shard) Node() (ipld.Node, error) {
172
return nil, err
173
}
174
}
175
+ cindex++
176
}
177
178
typ := upb.Data_HAMTShard
@@ -338,7 +338,7 @@ func (ds *Shard) insertChild(idx int, key string, lnk *ipld.Link) error {
338
}
339
340
i := ds.indexForBitPos(idx)
341
- ds.bitfield.SetBit(ds.bitfield, idx, 1)
341
+ ds.bitfield.SetBit(idx)
342
343
lnk.Name = ds.linkNamePrefix(idx) + key
344
sv := &shardValue{
@@ -367,7 +367,7 @@ func (ds *Shard) rmChild(i int) error {
367
368
func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
369
idx := hv.Next(ds.tableSizeLg2)
370
- if ds.bitfield.Bit(int(idx)) == 1 {
370
+ if ds.bitfield.Bit(int(idx)) {
371
cindex := ds.indexForBitPos(idx)
372
373
child, err := ds.getChild(ctx, cindex)
@@ -409,14 +409,7 @@ func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) erro
409
}
410
411
func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
412
- for i := 0; i < ds.tableSize; i++ {
413
- if ds.bitfield.Bit(i) == 0 {
414
- continue
415
- }
416
-
417
- idx := ds.indexForBitPos(i)
418
- // NOTE: an optimized version could simply iterate over each
419
- // element in the 'children' array.
412
+ for idx := range ds.children {
413
c, err := ds.getChild(ctx, idx)
414
if err != nil {
415
return err
@@ -424,14 +417,12 @@ func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error
417
418
switch c := c.(type) {
419
case *shardValue:
427
- err := cb(c)
428
- if err != nil {
420
+ if err := cb(c); err != nil {
421
return err
422
}
423
424
case *Shard:
433
- err := c.walkTrie(ctx, cb)
434
- if err != nil {
425
+ if err := c.walkTrie(ctx, cb); err != nil {
426
return err
427
}
428
default:
@@ -444,7 +435,7 @@ func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error
435
func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
436
idx := hv.Next(ds.tableSizeLg2)
437
447
- if ds.bitfield.Bit(idx) != 1 {
438
+ if !ds.bitfield.Bit(idx) {
439
return ds.insertChild(idx, key, val)
440
}
441
@@ -469,7 +460,7 @@ func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val
460
// Note: this shouldnt normally ever happen
461
// in the event of another implementation creates flawed
462
// structures, this will help to normalize them.
472
- ds.bitfield.SetBit(ds.bitfield, idx, 0)
463
+ ds.bitfield.UnsetBit(idx)
464
return ds.rmChild(cindex)
465
case 1:
466
nchild, ok := child.children[0].(*shardValue)
@@ -486,7 +477,7 @@ func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val
477
if child.key == key {
478
// value modification
479
if val == nil {
489
- ds.bitfield.SetBit(ds.bitfield, idx, 0)
480
+ ds.bitfield.UnsetBit(idx)
481
return ds.rmChild(cindex)
482
}
483
@@ -530,15 +521,7 @@ func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val
521
// the given bit in the bitset. The collapsed array contains only one entry
522
// per bit set in the bitfield, and this function is used to map the indices.
523
func (ds *Shard) indexForBitPos(bp int) int {
533
- // TODO: an optimization could reuse the same 'mask' here and change the size
534
- // as needed. This isnt yet done as the bitset package doesnt make it easy
535
- // to do.
536
-
537
- // make a bitmask (all bits set) 'bp' bits long
538
- mask := new(big.Int).Sub(new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(bp)), nil), big.NewInt(1))
539
- mask.And(mask, ds.bitfield)
540
-
541
- return popCount(mask)
524
+ return ds.bitfield.OnesBefore(bp)
525
}
526
527
// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix
unixfs/hamt/hamt_test.go
+1
-1
@@ -433,7 +433,7 @@ func TestBitfieldIndexing(t *testing.T) {
433
s, _ := NewShard(ds, 256)
434
435
set := func(i int) {
436
- s.bitfield.SetBit(s.bitfield, i, 1)
436
+ s.bitfield.SetBit(i)
437
}
438
439
assert := func(i int, val int) {
unixfs/hamt/util.go
+9
-6
@@ -1,7 +1,7 @@
1
package hamt
2
3
import (
4
- "math/big"
4
+ "fmt"
5
"math/bits"
6
)
7
@@ -40,10 +40,13 @@ func (hb *hashBits) Next(i int) int {
40
}
41
}
42
43
-func popCount(i *big.Int) int {
44
- var n int
45
- for _, v := range i.Bits() {
46
- n += bits.OnesCount64(uint64(v))
43
+func logtwo(v int) (int, error) {
44
+ if v <= 0 {
45
+ return 0, fmt.Errorf("hamt size should be a power of two")
46
}
48
- return n
47
+ lg2 := bits.TrailingZeros(uint(v))
48
+ if 1<<uint(lg2) != v {
49
+ return 0, fmt.Errorf("hamt size should be a power of two")
50
+ }
51
+ return lg2, nil
52
}
unixfs/hamt/util_test.go
-13
@@ -1,22 +1,9 @@
1
package hamt
2
3
import (
4
- "math/big"
4
"testing"
5
)
6
8
-func TestPopCount(t *testing.T) {
9
- x := big.NewInt(0)
10
-
11
- for i := 0; i < 50; i++ {
12
- x.SetBit(x, i, 1)
13
- }
14
-
15
- if popCount(x) != 50 {
16
- t.Fatal("expected popcount to be 50")
17
- }
18
-}
19
-
7
func TestHashBitsEvenSizes(t *testing.T) {
8
buf := []byte{255, 127, 79, 45, 116, 99, 35, 17}
9
hb := hashBits{b: buf}