move first data block into top level dag node
Jeromy committed
Sep 15, 2014 at 05:35 UTC
fdcd015eedcdadbdb9d2b3ea8f03d01afe6dccc2
5 files changed
+29
-5
importer/importer.go
+2
-1
@@ -25,7 +25,8 @@ func NewDagFromReader(r io.Reader) (*dag.Node, error) {
25
26
func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, error) {
27
blkChan := spl.Split(r)
28
- root := &dag.Node{Data: dag.FilePBData()}
28
+ first := <-blkChan
29
+ root := &dag.Node{Data: dag.FilePBData(first)}
30
31
for blk := range blkChan {
32
child := &dag.Node{Data: dag.WrapData(blk)}
importer/importer_test.go
+16
@@ -82,3 +82,19 @@ func arrComp(a, b []byte) error {
82
func TestMaybeRabinConsistency(t *testing.T) {
83
testFileConsistency(t, NewMaybeRabin(4096), 256*4096)
84
}
85
+
86
+func TestRabinBlockSize(t *testing.T) {
87
+ buf := new(bytes.Buffer)
88
+ nbytes := 1024 * 1024
89
+ io.CopyN(buf, rand.Reader, int64(nbytes))
90
+ rab := NewMaybeRabin(4096)
91
+ blkch := rab.Split(buf)
92
+
93
+ var blocks [][]byte
94
+ for b := range blkch {
95
+ blocks = append(blocks, b)
96
+ }
97
+
98
+ fmt.Printf("Avg block size: %d\n", nbytes/len(blocks))
99
+
100
+}
importer/rabin.go
+8
-3
@@ -9,8 +9,10 @@ import (
9
)
10
11
type MaybeRabin struct {
12
- mask int
13
- windowSize int
12
+ mask int
13
+ windowSize int
14
+ MinBlockSize int
15
+ MaxBlockSize int
16
}
17
18
func NewMaybeRabin(avgBlkSize int) *MaybeRabin {
@@ -18,6 +20,8 @@ func NewMaybeRabin(avgBlkSize int) *MaybeRabin {
20
rb := new(MaybeRabin)
21
rb.mask = (1 << blkbits) - 1
22
rb.windowSize = 16 // probably a good number...
23
+ rb.MinBlockSize = avgBlkSize / 2
24
+ rb.MaxBlockSize = (avgBlkSize / 2) * 3
25
return rb
26
}
27
@@ -70,7 +74,8 @@ func (mr *MaybeRabin) Split(r io.Reader) chan []byte {
74
outval := push(i, b)
75
blkbuf.WriteByte(b)
76
rollingHash = (rollingHash*a + int(b) - an*outval) % MOD
73
- if rollingHash&mr.mask == mr.mask {
77
+ if (rollingHash&mr.mask == mr.mask && blkbuf.Len() > mr.MinBlockSize) ||
78
+ blkbuf.Len() >= mr.MaxBlockSize {
79
out <- dup(blkbuf.Bytes())
80
blkbuf.Reset()
81
}
merkledag/dagreader.go
+1
@@ -34,6 +34,7 @@ func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) {
34
node: n,
35
thisData: pb.GetData(),
36
serv: serv,
37
+ buf: bytes.NewBuffer(pb.GetData()),
38
}, nil
39
case PBData_Raw:
40
return bytes.NewBuffer(pb.GetData()), nil
merkledag/merkledag.go
+2
-1
@@ -157,10 +157,11 @@ func (n *DAGService) Get(k u.Key) (*Node, error) {
157
return Decoded(b.Data)
158
}
159
160
-func FilePBData() []byte {
160
+func FilePBData(data []byte) []byte {
161
pbfile := new(PBData)
162
typ := PBData_File
163
pbfile.Type = &typ
164
+ pbfile.Data = data
165
166
data, err := proto.Marshal(pbfile)
167
if err != nil {