@cryptotaxi247 / kubo / commits / 3d8e96a22

Make bloom filters simpler

These did not work before, and had some unnecessary complexity. Now the filters use only one hashing function, no bignum arithmetic, and gets the additional bit positions by repeatedly hashing the result of prior hash. Since we're not concerned about crypto hashing here, this should be a win. External interfaces unchanged.

Kristoffer Ström committed Apr 6, 2015 at 16:45 UTC 3d8e96a22e2e15120729e4470fb12f5e032e142e
10 files changed +355 -46
Godeps/Godeps.json
+8 -4
@@ -213,6 +213,10 @@
213 "ImportPath": "github.com/mitchellh/go-homedir",
214 "Rev": "7d2d8c8a4e078ce3c58736ab521a40b37a504c52"
215 },
216 + {
217 + "ImportPath": "github.com/mtchavez/jenkins",
218 + "Rev": "5a816af6ef21ef401bff5e4b7dd255d63400f497"
219 + },
220 {
221 "ImportPath": "github.com/syndtr/goleveldb/leveldb",
222 "Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
@@ -221,6 +225,10 @@
225 "ImportPath": "github.com/syndtr/gosnappy/snappy",
226 "Rev": "156a073208e131d7d2e212cb749feae7c339e846"
227 },
228 + {
229 + "ImportPath": "github.com/whyrusleeping/go-metrics",
230 + "Rev": "1cd8009604ec2238b5a71305a0ecd974066e0e16"
231 + },
232 {
233 "ImportPath": "golang.org/x/crypto/blowfish",
234 "Rev": "b7d6bf2c61544745a02f83dec90393985fc3a065"
@@ -233,10 +241,6 @@
241 "ImportPath": "golang.org/x/net/context",
242 "Rev": "7dbad50ab5b31073856416cdcfeb2796d682f844"
243 },
236 - {
237 - "ImportPath": "github.com/whyrusleeping/go-metrics",
238 - "Rev": "1cd8009604ec2238b5a71305a0ecd974066e0e16"
239 - },
244 {
245 "ImportPath": "gopkg.in/fsnotify.v1",
246 "Comment": "v1.2.0",
Godeps/_workspace/src/github.com/mtchavez/jenkins/.gitignore new
+23
@@ -0,0 +1,23 @@
1 +# Compiled Object files, Static and Dynamic libs (Shared Objects)
2 +*.o
3 +*.a
4 +*.so
5 +
6 +# Folders
7 +_obj
8 +_test
9 +
10 +# Architecture specific extensions/prefixes
11 +*.[568vq]
12 +[568vq].out
13 +
14 +*.cgo1.go
15 +*.cgo2.c
16 +_cgo_defun.c
17 +_cgo_gotypes.go
18 +_cgo_export.*
19 +
20 +_testmain.go
21 +
22 +*.exe
23 +*.test
Godeps/_workspace/src/github.com/mtchavez/jenkins/.travis.yml new
+8
@@ -0,0 +1,8 @@
1 +go:
2 + - 1.1
3 + - tip
4 +install:
5 + - go get github.com/onsi/ginkgo
6 + - go get github.com/onsi/gomega
7 +before_script: go test -i ./...
8 +script: go test ./...
Godeps/_workspace/src/github.com/mtchavez/jenkins/Makefile new
+11
@@ -0,0 +1,11 @@
1 +build:
2 + go build jenkins.go
3 +
4 +run:
5 + go run jenkins.go
6 +
7 +test:
8 + go test -cover
9 +
10 +default:
11 + go run jenkins.go
Godeps/_workspace/src/github.com/mtchavez/jenkins/README.md new
+45
@@ -0,0 +1,45 @@
1 +Jenkins
2 +=================
3 +
4 +Golang Jenkins hash
5 +
6 +[![Build Status](https://travis-ci.org/mtchavez/go-jenkins-hashes.png?branch=master)](https://travis-ci.org/mtchavez/go-jenkins-hashes)
7 +
8 +## Install
9 +
10 +`go get -u github.com/mtchavez/jenkins`
11 +
12 +## Usage
13 +
14 +Jenkins follows the [Hash32](http://golang.org/pkg/hash/#Hash32) interface from the Go standard library
15 +
16 +```go
17 +// Create a new hash
18 +jenkhash := New()
19 +
20 +// Write a string of bytes to hash
21 +key := []byte("my-random-key")
22 +length, err := jenkhash(key)
23 +
24 +// Get uint32 sum of hash
25 +sum := jenkhash.Sum32()
26 +
27 +// Sum hash with byte string
28 +sumbytes := jenkhash.Sum(key)
29 +```
30 +
31 +## Testing
32 +
33 +Uses [Ginkgo](http://onsi.github.io/ginkgo/) for testing.
34 +
35 +Run via `make test` which will run `go test -cover`
36 +
37 +## Documentation
38 +
39 +Docs on [godoc](http://godoc.org/github.com/mtchavez/jenkins)
40 +
41 +## License
42 +
43 +Written by Chavez
44 +
45 +Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
Godeps/_workspace/src/github.com/mtchavez/jenkins/jenkins.go new
+48
@@ -0,0 +1,48 @@
1 +package jenkins
2 +
3 +import "hash"
4 +
5 +type jenkhash uint32
6 +
7 +func New() hash.Hash32 {
8 + var j jenkhash = 0
9 + return &j
10 +}
11 +
12 +func (j *jenkhash) Write(key []byte) (int, error) {
13 + hash := *j
14 +
15 + for _, b := range key {
16 + hash += jenkhash(b)
17 + hash += (hash << 10)
18 + hash ^= (hash >> 6)
19 + }
20 +
21 + hash += (hash << 3)
22 + hash ^= (hash >> 11)
23 + hash += (hash << 15)
24 +
25 + *j = hash
26 + return len(key), nil
27 +}
28 +
29 +func (j *jenkhash) Reset() {
30 + *j = 0
31 +}
32 +
33 +func (j *jenkhash) Size() int {
34 + return 4
35 +}
36 +
37 +func (j *jenkhash) BlockSize() int {
38 + return 1
39 +}
40 +
41 +func (j *jenkhash) Sum32() uint32 {
42 + return uint32(*j)
43 +}
44 +
45 +func (j *jenkhash) Sum(in []byte) []byte {
46 + v := j.Sum32()
47 + return append(in, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
48 +}
Godeps/_workspace/src/github.com/mtchavez/jenkins/jenkins_suite_test.go new
+13
@@ -0,0 +1,13 @@
1 +package jenkins
2 +
3 +import (
4 + . "github.com/onsi/ginkgo"
5 + . "github.com/onsi/gomega"
6 +
7 + "testing"
8 +)
9 +
10 +func TestJenkins(t *testing.T) {
11 + RegisterFailHandler(Fail)
12 + RunSpecs(t, "Jenkins Suite")
13 +}
Godeps/_workspace/src/github.com/mtchavez/jenkins/jenkins_test.go new
+101
@@ -0,0 +1,101 @@
1 +package jenkins
2 +
3 +import (
4 + . "github.com/onsi/ginkgo"
5 + . "github.com/onsi/gomega"
6 + "hash"
7 +)
8 +
9 +var _ = Describe("Jenkins", func() {
10 +
11 + var jhash hash.Hash32
12 + var key []byte
13 +
14 + BeforeEach(func() {
15 + jhash = New()
16 + key = []byte("Apple")
17 + })
18 +
19 + Describe("New", func() {
20 +
21 + It("returns jenkhash", func() {
22 + var h *jenkhash
23 + Expect(jhash).To(BeAssignableToTypeOf(h))
24 + })
25 +
26 + It("initializes offset to 0", func() {
27 + Expect(jhash.Sum32()).To(Equal(uint32(0)))
28 + })
29 + })
30 +
31 + Describe("Write", func() {
32 +
33 + It("returns key length", func() {
34 + length, _ := jhash.Write(key)
35 + Expect(length).To(Equal(5))
36 + })
37 +
38 + It("has no error", func() {
39 + _, err := jhash.Write(key)
40 + Expect(err).To(BeNil())
41 + })
42 +
43 + })
44 +
45 + Describe("Reset", func() {
46 +
47 + It("sets back to 0", func() {
48 + Expect(jhash.Sum32()).To(Equal(uint32(0)))
49 + jhash.Write(key)
50 + Expect(jhash.Sum32()).NotTo(Equal(uint32(0)))
51 + jhash.Reset()
52 + Expect(jhash.Sum32()).To(Equal(uint32(0)))
53 + })
54 +
55 + })
56 +
57 + Describe("Size", func() {
58 +
59 + It("is 4", func() {
60 + Expect(jhash.Size()).To(Equal(4))
61 + })
62 +
63 + })
64 +
65 + Describe("BlockSize", func() {
66 +
67 + It("is 1", func() {
68 + Expect(jhash.BlockSize()).To(Equal(1))
69 + })
70 +
71 + })
72 +
73 + Describe("Sum32", func() {
74 +
75 + It("defaults to 0", func() {
76 + Expect(jhash.Sum32()).To(Equal(uint32(0)))
77 + })
78 +
79 + It("sums hash", func() {
80 + jhash.Write(key)
81 + Expect(jhash.Sum32()).To(Equal(uint32(884782484)))
82 + })
83 +
84 + })
85 +
86 + Describe("Sum", func() {
87 +
88 + It("default 0 hash byte returned", func() {
89 + expected := []byte{0x41, 0x70, 0x70, 0x6c, 0x65, 0x0, 0x0, 0x0, 0x0}
90 + Expect(jhash.Sum(key)).To(Equal(expected))
91 + })
92 +
93 + It("returns sum byte array", func() {
94 + jhash.Write(key)
95 + expected := []byte{0x41, 0x70, 0x70, 0x6c, 0x65, 0x34, 0xbc, 0xb5, 0x94}
96 + Expect(jhash.Sum(key)).To(Equal(expected))
97 + })
98 +
99 + })
100 +
101 +})
blocks/bloom/filter.go
+46 -40
@@ -2,13 +2,11 @@
2 package bloom
3
4 import (
5 + "encoding/binary"
6 "errors"
6 - "fmt"
7 + // Non crypto hash, because speed
8 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/mtchavez/jenkins"
9 "hash"
8 - "hash/adler32"
9 - "hash/crc32"
10 - "hash/fnv"
11 - "math/big"
10 )
11
12 type Filter interface {
@@ -17,61 +15,66 @@ type Filter interface {
15 Merge(Filter) (Filter, error)
16 }
17
20 -func BasicFilter() Filter {
21 - // Non crypto hashes, because speed
22 - return NewFilter(2048, adler32.New(), fnv.New32(), crc32.NewIEEE())
23 -}
24 -
25 -func NewFilter(size int, hashes ...hash.Hash) Filter {
18 +func NewFilter(size int) Filter {
19 return &filter{
20 + hash: jenkins.New(),
21 filter: make([]byte, size),
28 - hashes: hashes,
22 + k: 3,
23 }
24 }
25
26 type filter struct {
27 filter []byte
34 - hashes []hash.Hash
28 + hash hash.Hash32
29 + k int
30 +}
31 +
32 +func BasicFilter() Filter {
33 + return NewFilter(2048)
34 }
35
37 -func (f *filter) Add(k []byte) {
38 - for _, h := range f.hashes {
39 - i := bytesMod(h.Sum(k), int64(len(f.filter)*8))
40 - f.setBit(i)
36 +func (f *filter) Add(bytes []byte) {
37 + for _, bit := range f.getBitIndicies(bytes) {
38 + f.setBit(bit)
39 }
40 }
41
44 -func (f *filter) Find(k []byte) bool {
45 - for _, h := range f.hashes {
46 - i := bytesMod(h.Sum(k), int64(len(f.filter)*8))
47 - if !f.getBit(i) {
42 +func (f *filter) getBitIndicies(bytes []byte) []uint32 {
43 + indicies := make([]uint32, f.k)
44 +
45 + f.hash.Write(bytes)
46 + b := make([]byte, 4)
47 +
48 + for i := 0; i < f.k; i++ {
49 + res := f.hash.Sum32()
50 + indicies[i] = res % (uint32(len(f.filter)) * 8)
51 +
52 + binary.LittleEndian.PutUint32(b, res)
53 + f.hash.Write(b)
54 + }
55 +
56 + f.hash.Reset()
57 +
58 + return indicies
59 +}
60 +
61 +func (f *filter) Find(bytes []byte) bool {
62 + for _, bit := range f.getBitIndicies(bytes) {
63 + if !f.getBit(bit) {
64 return false
65 }
66 }
67 return true
68 }
69
54 -func (f *filter) setBit(i int64) {
55 - fmt.Printf("setting bit %d\n", i)
70 +func (f *filter) setBit(i uint32) {
71 f.filter[i/8] |= (1 << byte(i%8))
72 }
73
59 -func (f *filter) getBit(i int64) bool {
60 - fmt.Printf("getting bit %d\n", i)
74 +func (f *filter) getBit(i uint32) bool {
75 return f.filter[i/8]&(1<<byte(i%8)) != 0
76 }
77
64 -func bytesMod(b []byte, modulo int64) int64 {
65 - i := big.NewInt(0)
66 - i = i.SetBytes(b)
67 -
68 - bigmod := big.NewInt(int64(modulo))
69 - result := big.NewInt(0)
70 - result.Mod(i, bigmod)
71 -
72 - return result.Int64()
73 -}
74 -
78 func (f *filter) Merge(o Filter) (Filter, error) {
79 casfil, ok := o.(*filter)
80 if !ok {
@@ -82,12 +85,15 @@ func (f *filter) Merge(o Filter) (Filter, error) {
85 return nil, errors.New("filter lengths must match!")
86 }
87
85 - nfilt := new(filter)
86 -
87 - // this bit is sketchy, need a way of comparing hash functions
88 - nfilt.hashes = f.hashes
88 + if casfil.k != f.k {
89 + return nil, errors.New("filter k-values must match!")
90 + }
91
92 + nfilt := new(filter)
93 + nfilt.hash = f.hash
94 nfilt.filter = make([]byte, len(f.filter))
95 + nfilt.k = f.k
96 +
97 for i, v := range f.filter {
98 nfilt.filter[i] = v | casfil.filter[i]
99 }
blocks/bloom/filter_test.go
+52 -2
@@ -1,13 +1,19 @@
1 package bloom
2
3 -import "testing"
3 +import (
4 + "encoding/binary"
5 + "fmt"
6 + "testing"
7 +)
8
9 func TestFilter(t *testing.T) {
6 - f := BasicFilter()
10 + f := NewFilter(128)
11 +
12 keys := [][]byte{
13 []byte("hello"),
14 []byte("fish"),
15 []byte("ipfsrocks"),
16 + []byte("i want ipfs socks"),
17 }
18
19 f.Add(keys[0])
@@ -21,10 +27,54 @@ func TestFilter(t *testing.T) {
27 }
28
29 f.Add(keys[2])
30 + f.Add(keys[3])
31
32 for _, k := range keys {
33 if !f.Find(k) {
34 t.Fatal("Couldnt find one of three keys")
35 }
36 }
37 +
38 + if f.Find([]byte("beep boop")) {
39 + t.Fatal("Got false positive! Super unlikely!")
40 + }
41 +
42 + fmt.Println(f)
43 +}
44 +
45 +func TestMerge(t *testing.T) {
46 +
47 + f1 := NewFilter(128)
48 + f2 := NewFilter(128)
49 +
50 + fbork := NewFilter(32)
51 +
52 + _, err := f1.Merge(fbork)
53 +
54 + if err == nil {
55 + t.Fatal("Merge should fail on filters with different lengths")
56 + }
57 +
58 + b := make([]byte, 4)
59 +
60 + var i uint32
61 + for i = 0; i < 10; i++ {
62 + binary.LittleEndian.PutUint32(b, i)
63 + f1.Add(b)
64 + }
65 +
66 + for i = 10; i < 20; i++ {
67 + binary.LittleEndian.PutUint32(b, i)
68 + f2.Add(b)
69 + }
70 +
71 + merged, _ := f1.Merge(f2)
72 +
73 + for i = 0; i < 20; i++ {
74 + binary.LittleEndian.PutUint32(b, i)
75 +
76 + if !merged.Find(b) {
77 + t.Fatal("Could not find all keys in merged filter")
78 + }
79 + }
80 }