calc_test for picking link block size
Juan Batiz-Benet committed
Jan 8, 2015 at 16:18 UTC
136ef70f00dcc54dc5842f6aa4c09a6d52ebd9ff
3 files changed
+69
-3
importer/calc_test.go
new
+50
@@ -0,0 +1,50 @@
1
+package importer
2
+
3
+import (
4
+ "math"
5
+ "testing"
6
+
7
+ humanize "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize"
8
+)
9
+
10
+func TestCalculateSizes(t *testing.T) {
11
+
12
+ // d := ((lbs/271) ^ layer) * dbs
13
+
14
+ increments := func(a, b int) []int {
15
+ ints := []int{}
16
+ for ; a <= b; a *= 2 {
17
+ ints = append(ints, a)
18
+ }
19
+ return ints
20
+ }
21
+
22
+ layers := 7
23
+ roughLinkSize := roughLinkSize // from importer pkg
24
+ dataBlockSizes := increments(1<<12, 1<<18)
25
+ linkBlockSizes := increments(1<<12, 1<<14)
26
+
27
+ t.Logf("rough link size: %d", roughLinkSize)
28
+ t.Logf("data block sizes: %v", dataBlockSizes)
29
+ t.Logf("link block sizes: %v", linkBlockSizes)
30
+ for _, dbs := range dataBlockSizes {
31
+ t.Logf("")
32
+ t.Logf("with data block size: %d", dbs)
33
+ for _, lbs := range linkBlockSizes {
34
+ t.Logf("")
35
+ t.Logf("\twith data block size: %d", dbs)
36
+ t.Logf("\twith link block size: %d", lbs)
37
+
38
+ lpb := lbs / roughLinkSize
39
+ t.Logf("\tlinks per block: %d", lpb)
40
+
41
+ for l := 1; l < layers; l++ {
42
+ total := int(math.Pow(float64(lpb), float64(l))) * dbs
43
+ htotal := humanize.Bytes(uint64(total))
44
+ t.Logf("\t\t\tlayer %d: %s\t%d", l, htotal, total)
45
+ }
46
+
47
+ }
48
+ }
49
+
50
+}
importer/importer.go
+18
-1
@@ -20,7 +20,24 @@ var log = util.Logger("importer")
20
// BlockSizeLimit specifies the maximum size an imported block can have.
21
var BlockSizeLimit = 1048576 // 1 MB
22
23
-var DefaultLinksPerBlock = 8192
23
+// rough estimates on expected sizes
24
+var roughDataBlockSize = chunk.DefaultBlockSize
25
+var roughLinkBlockSize = 1 << 13 // 8KB
26
+var roughLinkSize = 258 + 8 + 5 // sha256 multihash + size + no name + protobuf framing
27
+
28
+// DefaultLinksPerBlock governs how the importer decides how many links there
29
+// will be per block. This calculation is based on expected distributions of:
30
+// * the expected distribution of block sizes
31
+// * the expected distribution of link sizes
32
+// * desired access speed
33
+// For now, we use:
34
+//
35
+// var roughLinkBlockSize = 1 << 13 // 8KB
36
+// var roughLinkSize = 288 // sha256 + framing + name
37
+// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
38
+//
39
+// See calc_test.go
40
+var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
41
42
// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
43
var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
test/sharness/t0040-add-and-cat.sh
+1
-2
@@ -107,7 +107,6 @@ test_expect_success "'ipfs add bigfile' output looks good" '
107
echo "added $HASH mountdir/bigfile" >expected &&
108
test_cmp expected actual
109
'
110
-
110
test_expect_success "'ipfs cat' succeeds" '
111
ipfs cat $HASH >actual
112
'
@@ -139,7 +138,7 @@ test_expect_success EXPENSIVE "ipfs add bigfile succeeds" '
138
'
139
140
test_expect_success EXPENSIVE "ipfs add bigfile output looks good" '
142
- HASH="QmbprabK1ucRoPLPns2zKtjAqZrTANDhZMgmcx6sDKPK92" &&
141
+ HASH="QmSVxWkYfbJ3cowQUUgF4iF4CQd92vubxw7bs2aZAVRUD9" &&
142
echo "added $HASH mountdir/bigfile" >expected &&
143
test_cmp expected actual
144
'