fix(blockservice) fully async exchange.HasBlock
Brian Tiger Chow committed
Jan 19, 2015 at 21:27 UTC
721ff57263950610da615aa5a9205c7e26c6c9f6
7 files changed
+344
-32
blockservice/blocks_test.go
+5
-1
@@ -22,6 +22,7 @@ func TestBlocks(t *testing.T) {
22
t.Error("failed to construct block service", err)
23
return
24
}
25
+ defer bs.Close()
26
27
b := blocks.NewBlock([]byte("beep boop"))
28
h := u.Hash([]byte("beep boop"))
@@ -61,6 +62,9 @@ func TestBlocks(t *testing.T) {
62
63
func TestGetBlocksSequential(t *testing.T) {
64
var servs = Mocks(t, 4)
65
+ for _, s := range servs {
66
+ defer s.Close()
67
+ }
68
bg := blocksutil.NewBlockGenerator()
69
blks := bg.Blocks(50)
70
@@ -73,7 +77,7 @@ func TestGetBlocksSequential(t *testing.T) {
77
t.Log("one instance at a time, get blocks concurrently")
78
79
for i := 1; i < len(servs); i++ {
76
- ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
80
+ ctx, _ := context.WithTimeout(context.TODO(), time.Second*50)
81
out := servs[i].GetBlocks(ctx, keys)
82
gotten := make(map[u.Key]*blocks.Block)
83
for blk := range out {
blockservice/blockservice.go
+29
-31
@@ -8,20 +8,33 @@ import (
8
"fmt"
9
10
context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
11
- process "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
12
- procrl "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/ratelimit"
11
blocks "github.com/jbenet/go-ipfs/blocks"
12
"github.com/jbenet/go-ipfs/blocks/blockstore"
13
+ worker "github.com/jbenet/go-ipfs/blockservice/worker"
14
exchange "github.com/jbenet/go-ipfs/exchange"
15
u "github.com/jbenet/go-ipfs/util"
16
)
17
18
+var wc = worker.Config{
19
+ // When running on a single core, NumWorkers has a harsh negative effect on
20
+ // throughput. (-80% when < 25)
21
+ // Running a lot more workers appears to have very little effect on both
22
+ // single and multicore configurations.
23
+ NumWorkers: 25,
24
+
25
+ // These have no effect on when running on multiple cores, but harsh
26
+ // negative effect on throughput when running on a single core
27
+ // On multicore configurations these buffers have little effect on
28
+ // throughput.
29
+ // On single core configurations, larger buffers have severe adverse
30
+ // effects on throughput.
31
+ ClientBufferSize: 0,
32
+ WorkerBufferSize: 0,
33
+}
34
+
35
var log = u.Logger("blockservice")
36
var ErrNotFound = errors.New("blockservice: key not found")
37
22
-// MaxExchangeAddWorkers rate limits the number of exchange workers
23
-var MaxExchangeAddWorkers = 100
24
-
38
// BlockService is a hybrid block datastore. It stores data in a local
39
// datastore and may retrieve data from a remote Exchange.
40
// It uses an internal `datastore.Datastore` instance to store values.
@@ -30,8 +43,7 @@ type BlockService struct {
43
Blockstore blockstore.Blockstore
44
Exchange exchange.Interface
45
33
- rateLimiter *procrl.RateLimiter
34
- exchangeAdd chan blocks.Block
46
+ worker *worker.Worker
47
}
48
49
// NewBlockService creates a BlockService with given datastore instance.
@@ -43,15 +55,10 @@ func New(bs blockstore.Blockstore, rem exchange.Interface) (*BlockService, error
55
log.Warning("blockservice running in local (offline) mode.")
56
}
57
46
- // exchangeAdd is a channel for async workers to add to the exchange.
47
- // 100 blocks buffer. not clear what this number should be
48
- exchangeAdd := make(chan blocks.Block, 100)
49
-
58
return &BlockService{
51
- Blockstore: bs,
52
- Exchange: rem,
53
- exchangeAdd: exchangeAdd,
54
- rateLimiter: procrl.NewRateLimiter(process.Background(), MaxExchangeAddWorkers),
59
+ Blockstore: bs,
60
+ Exchange: rem,
61
+ worker: worker.NewWorker(rem, wc),
62
}, nil
63
}
64
@@ -63,22 +70,8 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) {
70
if err != nil {
71
return k, err
72
}
66
-
67
- // this operation rate-limits blockservice operations, so it is
68
- // now an async process.
69
- if s.Exchange != nil {
70
-
71
- // LimitedGo will spawn a goroutine but provide proper backpressure.
72
- // it will not spawn the goroutine until the ratelimiter's work load
73
- // is under the threshold.
74
- s.rateLimiter.LimitedGo(func(worker process.Process) {
75
- ctx := context.TODO()
76
- if err := s.Exchange.HasBlock(ctx, b); err != nil {
77
- // suppress error, as the client shouldn't care about bitswap.
78
- // the client only cares about the blockstore.Put.
79
- log.Errorf("Exchange.HasBlock error: %s", err)
80
- }
81
- })
73
+ if err := s.worker.HasBlock(b); err != nil {
74
+ return "", errors.New("blockservice is closed")
75
}
76
return k, nil
77
}
@@ -148,3 +141,8 @@ func (s *BlockService) GetBlocks(ctx context.Context, ks []u.Key) <-chan *blocks
141
func (s *BlockService) DeleteBlock(k u.Key) error {
142
return s.Blockstore.DeleteBlock(k)
143
}
144
+
145
+func (s *BlockService) Close() error {
146
+ log.Debug("blockservice is shutting down...")
147
+ return s.worker.Close()
148
+}
blockservice/worker/bench/main.go
new
+82
@@ -0,0 +1,82 @@
1
+package main
2
+
3
+import (
4
+ "log"
5
+ "math"
6
+ "testing"
7
+ "time"
8
+
9
+ ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
+ ds_sync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
11
+ blocks "github.com/jbenet/go-ipfs/blocks"
12
+ blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
13
+ worker "github.com/jbenet/go-ipfs/blockservice/worker"
14
+ "github.com/jbenet/go-ipfs/exchange/offline"
15
+ "github.com/jbenet/go-ipfs/thirdparty/delay"
16
+ "github.com/jbenet/go-ipfs/util/datastore2"
17
+)
18
+
19
+const kEstRoutingDelay = time.Second
20
+
21
+const kBlocksPerOp = 100
22
+
23
+func main() {
24
+ var bestConfig worker.Config
25
+ var quickestNsPerOp int64 = math.MaxInt64
26
+ for NumWorkers := 1; NumWorkers < 10; NumWorkers++ {
27
+ for ClientBufferSize := 0; ClientBufferSize < 10; ClientBufferSize++ {
28
+ for WorkerBufferSize := 0; WorkerBufferSize < 10; WorkerBufferSize++ {
29
+ c := worker.Config{
30
+ NumWorkers: NumWorkers,
31
+ ClientBufferSize: ClientBufferSize,
32
+ WorkerBufferSize: WorkerBufferSize,
33
+ }
34
+ result := testing.Benchmark(BenchmarkWithConfig(c))
35
+ if result.NsPerOp() < quickestNsPerOp {
36
+ bestConfig = c
37
+ quickestNsPerOp = result.NsPerOp()
38
+ }
39
+ log.Printf("benched %+v \t result: %+v", c, result)
40
+ }
41
+ }
42
+ }
43
+ log.Println(bestConfig)
44
+}
45
+
46
+func BenchmarkWithConfig(c worker.Config) func(b *testing.B) {
47
+ return func(b *testing.B) {
48
+
49
+ routingDelay := delay.Fixed(0) // during setup
50
+
51
+ dstore := ds_sync.MutexWrap(datastore2.WithDelay(ds.NewMapDatastore(), routingDelay))
52
+ bstore := blockstore.NewBlockstore(dstore)
53
+ var testdata []*blocks.Block
54
+ var i int64
55
+ for i = 0; i < kBlocksPerOp; i++ {
56
+ testdata = append(testdata, blocks.NewBlock([]byte(string(i))))
57
+ }
58
+ b.ResetTimer()
59
+ b.SetBytes(kBlocksPerOp)
60
+ for i := 0; i < b.N; i++ {
61
+
62
+ b.StopTimer()
63
+ w := worker.NewWorker(offline.Exchange(bstore), c)
64
+ b.StartTimer()
65
+
66
+ prev := routingDelay.Set(kEstRoutingDelay) // during measured section
67
+
68
+ for _, block := range testdata {
69
+ if err := w.HasBlock(block); err != nil {
70
+ b.Fatal(err)
71
+ }
72
+ }
73
+
74
+ routingDelay.Set(prev) // to hasten the unmeasured close period
75
+
76
+ b.StopTimer()
77
+ w.Close()
78
+ b.StartTimer()
79
+
80
+ }
81
+ }
82
+}
blockservice/worker/bench_worker_test.go
new
+42
@@ -0,0 +1,42 @@
1
+package worker
2
+
3
+import (
4
+ "testing"
5
+
6
+ ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7
+ dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
8
+ blocks "github.com/jbenet/go-ipfs/blocks"
9
+ blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
10
+ "github.com/jbenet/go-ipfs/exchange/offline"
11
+)
12
+
13
+func BenchmarkHandle10KBlocks(b *testing.B) {
14
+ bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
15
+ var testdata []*blocks.Block
16
+ for i := 0; i < 10000; i++ {
17
+ testdata = append(testdata, blocks.NewBlock([]byte(string(i))))
18
+ }
19
+ b.ResetTimer()
20
+ b.SetBytes(10000)
21
+ for i := 0; i < b.N; i++ {
22
+
23
+ b.StopTimer()
24
+ w := NewWorker(offline.Exchange(bstore), Config{
25
+ NumWorkers: 1,
26
+ ClientBufferSize: 0,
27
+ WorkerBufferSize: 0,
28
+ })
29
+ b.StartTimer()
30
+
31
+ for _, block := range testdata {
32
+ if err := w.HasBlock(block); err != nil {
33
+ b.Fatal(err)
34
+ }
35
+ }
36
+
37
+ b.StopTimer()
38
+ w.Close()
39
+ b.StartTimer()
40
+
41
+ }
42
+}
blockservice/worker/worker.go
new
+169
@@ -0,0 +1,169 @@
1
+// TODO FIXME name me
2
+package worker
3
+
4
+import (
5
+ "container/list"
6
+ "errors"
7
+ "time"
8
+
9
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
10
+ process "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
11
+ blocks "github.com/jbenet/go-ipfs/blocks"
12
+ exchange "github.com/jbenet/go-ipfs/exchange"
13
+ util "github.com/jbenet/go-ipfs/util"
14
+)
15
+
16
+var log = util.Logger("blockservice")
17
+
18
+var DefaultConfig = Config{
19
+ NumWorkers: 1,
20
+ ClientBufferSize: 0,
21
+ WorkerBufferSize: 0,
22
+}
23
+
24
+type Config struct {
25
+ // NumWorkers sets the number of background workers that provide blocks to
26
+ // the exchange.
27
+ NumWorkers int
28
+
29
+ // ClientBufferSize allows clients of HasBlock to send up to
30
+ // |ClientBufferSize| blocks without blocking.
31
+ ClientBufferSize int
32
+
33
+ // WorkerBufferSize can be used in conjunction with NumWorkers to reduce
34
+ // communication-coordination within the worker.
35
+ WorkerBufferSize int
36
+}
37
+
38
+// TODO FIXME name me
39
+type Worker struct {
40
+ // added accepts blocks from client
41
+ added chan *blocks.Block
42
+ exchange exchange.Interface
43
+
44
+ // workQueue is owned by the client worker
45
+ // process manages life-cycle
46
+ process process.Process
47
+}
48
+
49
+func NewWorker(e exchange.Interface, c Config) *Worker {
50
+ if c.NumWorkers < 1 {
51
+ c.NumWorkers = 1 // provide a sane default
52
+ }
53
+ w := &Worker{
54
+ exchange: e,
55
+ added: make(chan *blocks.Block, c.ClientBufferSize),
56
+ process: process.WithParent(process.Background()), // internal management
57
+ }
58
+ w.start(c)
59
+ return w
60
+}
61
+
62
+func (w *Worker) HasBlock(b *blocks.Block) error {
63
+ select {
64
+ case <-w.process.Closed():
65
+ return errors.New("blockservice worker is closed")
66
+ case w.added <- b:
67
+ return nil
68
+ }
69
+}
70
+
71
+func (w *Worker) Close() error {
72
+ log.Debug("blockservice provide worker is shutting down...")
73
+ return w.process.Close()
74
+}
75
+
76
+func (w *Worker) start(c Config) {
77
+
78
+ workerChan := make(chan *blocks.Block, c.WorkerBufferSize)
79
+
80
+ // clientWorker handles incoming blocks from |w.added| and sends to
81
+ // |workerChan|. This will never block the client.
82
+ w.process.Go(func(proc process.Process) {
83
+ defer close(workerChan)
84
+
85
+ var workQueue BlockList
86
+ for {
87
+
88
+ // take advantage of the fact that sending on nil channel always
89
+ // blocks so that a message is only sent if a block exists
90
+ sendToWorker := workerChan
91
+ nextBlock := workQueue.Pop()
92
+ if nextBlock == nil {
93
+ sendToWorker = nil
94
+ }
95
+
96
+ select {
97
+
98
+ // if worker is ready and there's a block to process, send the
99
+ // block
100
+ case sendToWorker <- nextBlock:
101
+ case <-time.Tick(5 * time.Second):
102
+ if workQueue.Len() > 0 {
103
+ log.Debugf("%d blocks in blockservice provide queue...", workQueue.Len())
104
+ }
105
+ case block := <-w.added:
106
+ if nextBlock != nil {
107
+ workQueue.Push(nextBlock) // missed the chance to send it
108
+ }
109
+ // if the client sends another block, add it to the queue.
110
+ workQueue.Push(block)
111
+ case <-proc.Closing():
112
+ return
113
+ }
114
+ }
115
+ })
116
+
117
+ for i := 0; i < c.NumWorkers; i++ {
118
+ // reads from |workerChan| until process closes
119
+ w.process.Go(func(proc process.Process) {
120
+ ctx, cancel := context.WithCancel(context.Background())
121
+
122
+ // shuts down an in-progress HasBlock operation
123
+ proc.Go(func(proc process.Process) {
124
+ <-proc.Closing()
125
+ cancel()
126
+ })
127
+
128
+ for {
129
+ select {
130
+ case <-proc.Closing():
131
+ return
132
+ case block, ok := <-workerChan:
133
+ if !ok {
134
+ return
135
+ }
136
+ if err := w.exchange.HasBlock(ctx, block); err != nil {
137
+ // TODO log event?
138
+ }
139
+ }
140
+ }
141
+ })
142
+ }
143
+}
144
+
145
+type BlockList struct {
146
+ list list.List
147
+}
148
+
149
+func (s *BlockList) PushFront(b *blocks.Block) {
150
+ // FIXME find figures
151
+ s.list.PushFront(b)
152
+}
153
+
154
+func (s *BlockList) Push(b *blocks.Block) {
155
+ s.list.PushBack(b)
156
+}
157
+
158
+func (s *BlockList) Pop() *blocks.Block {
159
+ if s.list.Len() == 0 {
160
+ return nil
161
+ }
162
+ e := s.list.Front()
163
+ s.list.Remove(e)
164
+ return e.Value.(*blocks.Block)
165
+}
166
+
167
+func (s *BlockList) Len() int {
168
+ return s.list.Len()
169
+}
blockservice/worker/worker_test.go
new
+14
@@ -0,0 +1,14 @@
1
+package worker
2
+
3
+import "testing"
4
+
5
+func TestStartClose(t *testing.T) {
6
+ numRuns := 50
7
+ if testing.Short() {
8
+ numRuns = 5
9
+ }
10
+ for i := 0; i < numRuns; i++ {
11
+ w := NewWorker(nil, DefaultConfig)
12
+ w.Close()
13
+ }
14
+}
core/core.go
+3
@@ -270,6 +270,9 @@ func (n *IpfsNode) teardown() error {
270
if n.Repo != nil {
271
closers = append(closers, n.Repo)
272
}
273
+ if n.Blocks != nil {
274
+ closers = append(closers, n.Blocks)
275
+ }
276
if n.Routing != nil {
277
if dht, ok := n.Routing.(*dht.IpfsDHT); ok {
278
closers = append(closers, dht)