implement basic rabin fingerprint file splitting
Jeromy committed
Aug 30, 2014 at 10:53 UTC
1f309b72d01d15db9dc0156254322f61ed9a68f8
2 files changed
+102
importer/split_test.go
new
+50
@@ -0,0 +1,50 @@
1
+package importer
2
+
3
+import (
4
+ "testing"
5
+ "crypto/rand"
6
+ "bytes"
7
+)
8
+
9
+func TestDataSplitting(t *testing.T) {
10
+ buf := make([]byte, 16*1024*1024)
11
+ rand.Read(buf)
12
+
13
+ split := Rabin(buf)
14
+
15
+ if len(split) == 1 {
16
+ t.Fatal("No split occurred!")
17
+ }
18
+
19
+ min := 2 << 15
20
+ max := 0
21
+
22
+ mxcount := 0
23
+
24
+ n := 0
25
+ for _, b := range split {
26
+ if !bytes.Equal(b, buf[n:n+len(b)]) {
27
+ t.Fatal("Split lost data!")
28
+ }
29
+ n += len(b)
30
+
31
+ if len(b) < min {
32
+ min = len(b)
33
+ }
34
+
35
+ if len(b) > max {
36
+ max = len(b)
37
+ }
38
+
39
+ if len(b) == 16384 {
40
+ mxcount++
41
+ }
42
+ }
43
+
44
+ if n != len(buf) {
45
+ t.Fatal("missing some bytes!")
46
+ }
47
+ t.Log(len(split))
48
+ t.Log(min, max, mxcount)
49
+}
50
+
importer/splitting.go
new
+52
@@ -0,0 +1,52 @@
1
+package importer
2
+
3
+type BlockSplitter func([]byte) [][]byte
4
+
5
+// TODO: this should take a reader, not a byte array. what if we're splitting a 3TB file?
6
+func Rabin(b []byte) [][]byte {
7
+ var out [][]byte
8
+ windowsize := uint64(48)
9
+ chunk_max := 1024 * 16
10
+ min_blk_size := 2048
11
+ blk_beg_i := 0
12
+ prime := uint64(61)
13
+
14
+ var poly uint64
15
+ var curchecksum uint64
16
+
17
+ // Smaller than a window? Get outa here!
18
+ if len(b) <= int(windowsize) {
19
+ return [][]byte{b}
20
+ }
21
+
22
+ i := 0
23
+ for n := i; i < n+int(windowsize); i++ {
24
+ cur := uint64(b[i])
25
+ curchecksum = (curchecksum * prime) + cur
26
+ poly = (poly * prime) + cur
27
+ }
28
+
29
+ for ; i < len(b); i++ {
30
+ cur := uint64(b[i])
31
+ curchecksum = (curchecksum * prime) + cur
32
+ poly = (poly * prime) + cur
33
+ curchecksum -= (uint64(b[i-1]) * prime)
34
+
35
+ if i-blk_beg_i >= chunk_max {
36
+ // push block
37
+ out = append(out, b[blk_beg_i:i])
38
+ blk_beg_i = i
39
+ }
40
+
41
+ // first 13 bits of polynomial are 0
42
+ if poly % 8192 == 0 && i-blk_beg_i >= min_blk_size {
43
+ // push block
44
+ out = append(out, b[blk_beg_i:i])
45
+ blk_beg_i = i
46
+ }
47
+ }
48
+ if i > blk_beg_i {
49
+ out = append(out, b[blk_beg_i:])
50
+ }
51
+ return out
52
+}