blocks/blockstore: introduce context passing to blockstore
License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>
Jakub Sztandera committed
Jul 4, 2016 at 23:02 UTC
016d3d97ef5ef7963b320278fea19c593b4d6cf4
5 files changed
+65
-16
blocks/blockstore/bloom_cache.go
+18
-8
@@ -11,10 +11,10 @@ import (
11
"sync/atomic"
12
)
13
14
-// BloomCached returns Blockstore that caches Has requests using Bloom filter
14
+// bloomCached returns Blockstore that caches Has requests using Bloom filter
15
// Size is size of bloom filter in bytes
16
-func BloomCached(bs Blockstore, bloomSize, lruSize int) (*bloomcache, error) {
17
- bl, err := bloom.New(float64(bloomSize), float64(7))
16
+func bloomCached(bs Blockstore, ctx context.Context, bloomSize, hashCount, lruSize int) (*bloomcache, error) {
17
+ bl, err := bloom.New(float64(bloomSize), float64(hashCount))
18
if err != nil {
19
return nil, err
20
}
@@ -24,7 +24,7 @@ func BloomCached(bs Blockstore, bloomSize, lruSize int) (*bloomcache, error) {
24
}
25
bc := &bloomcache{blockstore: bs, bloom: bl, arc: arc}
26
bc.Invalidate()
27
- go bc.Rebuild()
27
+ go bc.Rebuild(ctx)
28
29
return bc, nil
30
}
@@ -52,8 +52,7 @@ func (b *bloomcache) BloomActive() bool {
52
return atomic.LoadInt32(&b.active) != 0
53
}
54
55
-func (b *bloomcache) Rebuild() {
56
- ctx := context.TODO()
55
+func (b *bloomcache) Rebuild(ctx context.Context) {
56
evt := log.EventBegin(ctx, "bloomcache.Rebuild")
57
defer evt.Done()
58
@@ -62,8 +61,19 @@ func (b *bloomcache) Rebuild() {
61
log.Errorf("AllKeysChan failed in bloomcache rebuild with: %v", err)
62
return
63
}
65
- for key := range ch {
66
- b.bloom.AddTS([]byte(key)) // Use binary key, the more compact the better
64
+ finish := false
65
+ for !finish {
66
+ select {
67
+ case key, ok := <-ch:
68
+ if ok {
69
+ b.bloom.AddTS([]byte(key)) // Use binary key, the more compact the better
70
+ } else {
71
+ finish = true
72
+ }
73
+ case <-ctx.Done():
74
+ log.Warning("Cache rebuild closed by context finishing.")
75
+ return
76
+ }
77
}
78
close(b.rebuildChan)
79
atomic.StoreInt32(&b.active, 1)
blocks/blockstore/bloom_cache_test.go
+18
-6
@@ -8,18 +8,29 @@ import (
8
9
"github.com/ipfs/go-ipfs/blocks"
10
11
+ context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
12
ds "gx/ipfs/QmfQzVugPq1w5shWRcLWSeiHF4a2meBX7yVD8Vw7GWJM9o/go-datastore"
13
dsq "gx/ipfs/QmfQzVugPq1w5shWRcLWSeiHF4a2meBX7yVD8Vw7GWJM9o/go-datastore/query"
14
syncds "gx/ipfs/QmfQzVugPq1w5shWRcLWSeiHF4a2meBX7yVD8Vw7GWJM9o/go-datastore/sync"
15
)
16
17
+func testBloomCached(bs GCBlockstore, ctx context.Context) (*bloomcache, error) {
18
+ opts := DefaultCacheOpts()
19
+ bbs, err := CachedBlockstore(bs, ctx, opts)
20
+ if err == nil {
21
+ return bbs.(*bloomcache), nil
22
+ } else {
23
+ return nil, err
24
+ }
25
+}
26
+
27
func TestReturnsErrorWhenSizeNegative(t *testing.T) {
28
bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
18
- _, err := BloomCached(bs, 100, -1)
29
+ _, err := bloomCached(bs, nil, 100, 1, -1)
30
if err == nil {
31
t.Fail()
32
}
22
- _, err = BloomCached(bs, -1, 100)
33
+ _, err = bloomCached(bs, nil, -1, 1, 100)
34
if err == nil {
35
t.Fail()
36
}
@@ -29,7 +40,7 @@ func TestRemoveCacheEntryOnDelete(t *testing.T) {
40
b := blocks.NewBlock([]byte("foo"))
41
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
42
bs := NewBlockstore(syncds.MutexWrap(cd))
32
- cachedbs, err := BloomCached(bs, 1, 1)
43
+ cachedbs, err := testBloomCached(bs, nil)
44
if err != nil {
45
t.Fatal(err)
46
}
@@ -53,7 +64,7 @@ func TestRemoveCacheEntryOnDelete(t *testing.T) {
64
func TestElideDuplicateWrite(t *testing.T) {
65
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
66
bs := NewBlockstore(syncds.MutexWrap(cd))
56
- cachedbs, err := BloomCached(bs, 1, 1)
67
+ cachedbs, err := testBloomCached(bs, nil)
68
if err != nil {
69
t.Fatal(err)
70
}
@@ -73,14 +84,15 @@ func TestHasIsBloomCached(t *testing.T) {
84
for i := 0; i < 1000; i++ {
85
bs.Put(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i))))
86
}
76
- cachedbs, err := BloomCached(bs, 256*1024, 128)
87
+ ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
88
+ cachedbs, err := testBloomCached(bs, ctx)
89
if err != nil {
90
t.Fatal(err)
91
}
92
93
select {
94
case <-cachedbs.rebuildChan:
83
- case <-time.After(1 * time.Second):
95
+ case <-ctx.Done():
96
t.Fatalf("Timeout wating for rebuild: %d", cachedbs.bloom.ElementsAdded())
97
}
98
blocks/blockstore/caching.go
+26
@@ -1,5 +1,11 @@
1
package blockstore
2
3
+import (
4
+ "errors"
5
+
6
+ context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
7
+)
8
+
9
// Next to each option is it aproximate memory usage per unit
10
type CacheOpts struct {
11
HasBloomFilterSize int // 1 bit
@@ -14,3 +20,23 @@ func DefaultCacheOpts() CacheOpts {
20
HasARCCacheSize: 64 * 1024,
21
}
22
}
23
+
24
+func CachedBlockstore(bs GCBlockstore,
25
+ ctx context.Context, opts CacheOpts) (cbs GCBlockstore, err error) {
26
+ if ctx == nil {
27
+ ctx = context.TODO() // For tests
28
+ }
29
+
30
+ if opts.HasBloomFilterSize < 0 || opts.HasBloomFilterHashes < 0 ||
31
+ opts.HasARCCacheSize < 0 {
32
+ return nil, errors.New("all options for cache need to be greater than zero")
33
+ }
34
+
35
+ if opts.HasBloomFilterSize != 0 && opts.HasBloomFilterHashes == 0 {
36
+ return nil, errors.New("bloom filter hash count can't be 0 when there is size set")
37
+ }
38
+ cbs, err = bloomCached(bs, ctx, opts.HasBloomFilterSize, opts.HasBloomFilterHashes,
39
+ opts.HasARCCacheSize)
40
+
41
+ return cbs, err
42
+}
core/builder.go
+1
-1
@@ -131,7 +131,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
131
132
var err error
133
bs := bstore.NewBlockstore(n.Repo.Datastore())
134
- n.Blockstore, err = bstore.BloomCached(bs, 256*1024, kSizeBlockstoreWriteCache)
134
+ n.Blockstore, err = bstore.CachedBlockstore(bs, ctx, bstore.DefaultCacheOpts())
135
if err != nil {
136
return err
137
}
exchange/bitswap/testutils.go
+2
-1
@@ -93,7 +93,8 @@ func Session(ctx context.Context, net tn.Network, p testutil.Identity) Instance
93
adapter := net.Adapter(p)
94
dstore := ds_sync.MutexWrap(datastore2.WithDelay(ds.NewMapDatastore(), bsdelay))
95
96
- bstore, err := blockstore.BloomCached(blockstore.NewBlockstore(ds_sync.MutexWrap(dstore)), bloomSize, writeCacheElems)
96
+ bstore, err := blockstore.CachedBlockstore(blockstore.NewBlockstore(
97
+ ds_sync.MutexWrap(dstore)), ctx, blockstore.DefaultCacheOpts())
98
if err != nil {
99
panic(err.Error()) // FIXME perhaps change signature and return error.
100
}