Add hamming distance calculation to bloom filters
Kristoffer Ström committed
Apr 15, 2015 at 17:13 UTC
2c3f9f2419971390b5d24db0911ca0bd50bef2d3
6 files changed
+170
Godeps/Godeps.json
+5
@@ -217,6 +217,11 @@
217
"ImportPath": "github.com/mtchavez/jenkins",
218
"Rev": "5a816af6ef21ef401bff5e4b7dd255d63400f497"
219
},
220
+ {
221
+ "ImportPath": "github.com/steakknife/hamming",
222
+ "Comment": "0.0.2-2-g9ad4a62",
223
+ "Rev": "9ad4a620e3d573267a083c892f2b42a39302153b"
224
+ },
225
{
226
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
227
"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
Godeps/_workspace/src/github.com/steakknife/hamming/README.md
new
+3
@@ -0,0 +1,3 @@
1
+Copyright (c) 2014 Barry Allard
2
+
3
+MIT license
Godeps/_workspace/src/github.com/steakknife/hamming/hamming.go
new
+38
@@ -0,0 +1,38 @@
1
+package hamming
2
+
3
+// SSE4.x PopCnt is 10x slower
4
+// References: check out Hacker's Delight
5
+
6
+const (
7
+ m1 uint64 = 0x5555555555555555 //binary: 0101...
8
+ m2 uint64 = 0x3333333333333333 //binary: 00110011..
9
+ m4 uint64 = 0x0f0f0f0f0f0f0f0f //binary: 4 zeros, 4 ones ...
10
+ m8 uint64 = 0x00ff00ff00ff00ff //binary: 8 zeros, 8 ones ...
11
+ m16 uint64 = 0x0000ffff0000ffff //binary: 16 zeros, 16 ones ...
12
+ m32 uint64 = 0x00000000ffffffff //binary: 32 zeros, 32 ones
13
+ hff uint64 = 0xffffffffffffffff //binary: all ones
14
+ h01 uint64 = 0x0101010101010101 //the sum of 256 to the power of 0,1,2,3...
15
+)
16
+
17
+var table = [256]byte{0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}
18
+
19
+// hamming distance of two uint64's
20
+func Uint64(x, y uint64) int {
21
+ return CountBitsUint64(x ^ y)
22
+}
23
+
24
+// hamming distance of two bytes
25
+func Byte(x, y byte) int {
26
+ return CountBitsByte(x ^ y)
27
+}
28
+
29
+func CountBitsUint64(x uint64) int {
30
+ x -= (x >> 1) & m1 // put count of each 2 bits into those 2 bits
31
+ x = (x & m2) + ((x >> 2) & m2) // put count of each 4 bits into those 4 bits
32
+ x = (x + (x >> 4)) & m4 // put count of each 8 bits into those 8 bits
33
+ return int((x * h01) >> 56) // returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24) + ...
34
+}
35
+
36
+func CountBitsByte(x byte) int {
37
+ return int(table[x])
38
+}
Godeps/_workspace/src/github.com/steakknife/hamming/hamming_test.go
new
+88
@@ -0,0 +1,88 @@
1
+package hamming
2
+
3
+import (
4
+ "testing"
5
+)
6
+
7
+type testCountBitsUint64Case struct {
8
+ x uint64
9
+ n int
10
+}
11
+
12
+type testCountBitsByteCase struct {
13
+ x byte
14
+ n int
15
+}
16
+
17
+var testCountBitsByteCases = []testCountBitsByteCase{
18
+ {0x00, 0},
19
+ {0x01, 1},
20
+ {0x02, 1},
21
+ {0x03, 2},
22
+ {0xaa, 4},
23
+ {0x55, 4},
24
+ {0x7f, 7},
25
+ {0xff, 8},
26
+}
27
+
28
+var testCountBitsUint64Cases = []testCountBitsUint64Case{
29
+ {0x00, 0},
30
+ {0x01, 1},
31
+ {0x02, 1},
32
+ {0x03, 2},
33
+ {0xaa, 4},
34
+ {0x55, 4},
35
+ {0x7f, 7},
36
+ {0xff, 8},
37
+ {0xffff, 16},
38
+ {0xffffffff, 32},
39
+ {0x1ffffffff, 33},
40
+ {0x3ffffffff, 34},
41
+ {0x7ffffffff, 35},
42
+ {0xfffffffff, 36},
43
+ {0x3fffffffffffffff, 62},
44
+ {0x7fffffffffffffff, 63},
45
+ {0xffffffffffffffff, 64},
46
+}
47
+
48
+func TestCountBitByte(t *testing.T) {
49
+ for _, c := range testCountBitsByteCases {
50
+ if actualN := CountBitsByte(c.x); actualN != c.n {
51
+ t.Fatal("CountBitsByte(", c.x, ") = ", actualN, " != ", c.n)
52
+ } else {
53
+ t.Log("CountBitsByte(", c.x, ") == ", c.n)
54
+ }
55
+ }
56
+}
57
+
58
+func TestCountBitUint64(t *testing.T) {
59
+ for _, c := range testCountBitsUint64Cases {
60
+ if actualN := CountBitsUint64(c.x); actualN != c.n {
61
+ t.Fatal("CountBitsUint64(", c.x, ") = ", actualN, " != ", c.n)
62
+ } else {
63
+ t.Log("CountBitsUint64(", c.x, ") == ", c.n)
64
+ }
65
+ }
66
+}
67
+
68
+func BenchmarkCountBitsUint64(b *testing.B) {
69
+ j := 0
70
+ for i := 0; i < b.N; i++ {
71
+ CountBitsUint64(testCountBitsUint64Cases[j].x)
72
+ j++
73
+ if j == len(testCountBitsUint64Cases) {
74
+ j = 0
75
+ }
76
+ }
77
+}
78
+
79
+func BenchmarkCountBitsByte(b *testing.B) {
80
+ j := 0
81
+ for i := 0; i < b.N; i++ {
82
+ CountBitsByte(testCountBitsByteCases[j].x)
83
+ j++
84
+ if j == len(testCountBitsByteCases) {
85
+ j = 0
86
+ }
87
+ }
88
+}
blocks/bloom/filter.go
+22
@@ -6,6 +6,7 @@ import (
6
"errors"
7
// Non crypto hash, because speed
8
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/mtchavez/jenkins"
9
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/steakknife/hamming"
10
"hash"
11
)
12
@@ -13,6 +14,7 @@ type Filter interface {
14
Add([]byte)
15
Find([]byte) bool
16
Merge(Filter) (Filter, error)
17
+ HammingDistance(Filter) (int, error)
18
}
19
20
func NewFilter(size int) Filter {
@@ -100,3 +102,23 @@ func (f *filter) Merge(o Filter) (Filter, error) {
102
103
return nfilt, nil
104
}
105
+
106
+func (f *filter) HammingDistance(o Filter) (int, error) {
107
+ casfil, ok := o.(*filter)
108
+ if !ok {
109
+ return 0, errors.New("Unsupported filter type")
110
+ }
111
+
112
+ if len(f.filter) != len(casfil.filter) {
113
+ return 0, errors.New("filter lengths must match!")
114
+ }
115
+
116
+ acc := 0
117
+
118
+ // xor together
119
+ for i := 0; i < len(f.filter); i++ {
120
+ acc += hamming.Byte(f.filter[i], casfil.filter[i])
121
+ }
122
+
123
+ return acc, nil
124
+}
blocks/bloom/filter_test.go
+14
@@ -78,3 +78,17 @@ func TestMerge(t *testing.T) {
78
}
79
}
80
}
81
+
82
+func TestHamming(t *testing.T) {
83
+ f1 := NewFilter(128)
84
+ f2 := NewFilter(128)
85
+
86
+ f1.Add([]byte("no collision"))
87
+ f1.Add([]byte("collision? no!"))
88
+
89
+ dist, _ := f1.HammingDistance(f2)
90
+
91
+ if dist != 6 {
92
+ t.Fatal("Should have 6 bit difference")
93
+ }
94
+}