add more docs on hamt
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Nov 17, 2016 at 13:39 UTC
8d4791c9bb23846e5de8958e1092627315e88e42
2 files changed
+27
unixfs/hamt/hamt.go
+25
@@ -1,3 +1,23 @@
1
+// Package hamt implements a Hash Array Mapped Trie over ipfs merkledag nodes.
2
+// It is implemented mostly as described in the wikipedia article on HAMTs,
3
+// however the table size is variable (usually 256 in our usages) as opposed to
4
+// 32 as suggested in the article. The hash function used is currently
5
+// Murmur3, but this value is configurable (the datastructure reports which
6
+// hash function its using).
7
+//
8
+// The one algorithmic change we implement that is not mentioned in the
9
+// wikipedia article is the collapsing of empty shards.
10
+// Given the following tree: ( '[' = shards, '{' = values )
11
+// [ 'A' ] -> [ 'B' ] -> { "ABC" }
12
+// | L-> { "ABD" }
13
+// L-> { "ASDF" }
14
+// If we simply removed "ABC", we would end up with a tree where shard 'B' only
15
+// has a single child. This causes two issues, the first, is that now we have
16
+// an extra lookup required to get to "ABD". The second issue is that now we
17
+// have a tree that contains only "ABD", but is not the same tree that we would
18
+// get by simply inserting "ABD" into a new tree. To address this, we always
19
+// check for empty shard nodes upon deletion and prune them to maintain a
20
+// consistent tree, independent of insertion order.
21
package hamt
22
23
import (
@@ -450,10 +470,15 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
470
}
471
}
472
473
+// indexForBitPos returns the index within the collapsed array corresponding to
474
+// the given bit in the bitset. The collapsed array contains only one entry
475
+// per bit set in the bitfield, and this function is used to map the indices.
476
func (ds *HamtShard) indexForBitPos(bp int) int {
477
// TODO: an optimization could reuse the same 'mask' here and change the size
478
// as needed. This isnt yet done as the bitset package doesnt make it easy
479
// to do.
480
+
481
+ // make a bitmask (all bits set) 'bp' bits long
482
mask := new(big.Int).Sub(new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(bp)), nil), big.NewInt(1))
483
mask.And(mask, ds.bitfield)
484
unixfs/hamt/util.go
+2
@@ -4,6 +4,7 @@ import (
4
"math/big"
5
)
6
7
+// hashBits is a helper that allows the reading of the 'next n bits' as an integer.
8
type hashBits struct {
9
b []byte
10
consumed int
@@ -13,6 +14,7 @@ func mkmask(n int) byte {
14
return (1 << uint(n)) - 1
15
}
16
17
+// Next returns the next 'i' bits of the hashBits value as an integer
18
func (hb *hashBits) Next(i int) int {
19
curbi := hb.consumed / 8
20
leftb := 8 - (hb.consumed % 8)