@cryptotaxi247 / kubo / commits / 6e705e1ef

bitswap/provide: improved rate limiting

this PR greatly speeds up providing and add. (1) Instead of idling workers, we move to a ratelimiter-based worker. We put this max at 512, so that means _up to_ 512 goroutines. This is very small load on the node, as each worker is providing to the dht, which means mostly waiting. It DOES put a large load on the DHT. but i want to try this out for a while and see if it's a problem. We can decide later if it is a problem for the network (nothing stops anyone from re-compiling, but the defaults of course matter). (2) We add a buffer size for provideKeys, which means that we block the add process much less. this is a very cheap buffer, as it only stores keys (it may be even cheaper with a lock + ring buffer instead of a channel...). This makes add blazing fast-- it was being rate limited by providing. Add should not be ratelimited by providing (much, if any) as the user wants to just store the stuff in the local node's repo. This buffer is initially set to 4096, which means: 4096 * keysize (~258 bytes + go overhead) ~ 1-1.5MB this buffer only last a few sec to mins, and is an ok thing to do for the sake of very fast adds. (this could be a configurable paramter, certainly for low-mem footprint use cases). At the moment this is not much, compared to block sizes. (3) We make the providing EventBegin() + Done(), so that we can track how long a provide takes, and we can remove workers as they finish in bsdash and similar tools. License: MIT Signed-off-by: Juan Batiz-Benet <juan@benet.ai>

Juan Batiz-Benet committed Aug 4, 2015 at 19:53 UTC 6e705e1ef061309d3bca69ef3a636e64c057138f
2 files changed +48 -46
exchange/bitswap/bitswap.go
+4 -3
@@ -39,8 +39,9 @@ const (
39 // kMaxPriority is the max priority as defined by the bitswap protocol
40 kMaxPriority = math.MaxInt32
41
42 - HasBlockBufferSize = 256
43 - provideWorkers = 4
42 + HasBlockBufferSize = 256
43 + provideKeysBufferSize = 2048
44 + provideWorkerMax = 512
45 )
46
47 var rebroadcastDelay = delay.Fixed(time.Second * 10)
@@ -85,7 +86,7 @@ func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
86 findKeys: make(chan *blockRequest, sizeBatchRequestChan),
87 process: px,
88 newBlocks: make(chan *blocks.Block, HasBlockBufferSize),
88 - provideKeys: make(chan key.Key),
89 + provideKeys: make(chan key.Key, provideKeysBufferSize),
90 wm: NewWantManager(ctx, network),
91 }
92 go bs.wm.Run()
exchange/bitswap/workers.go
+44 -43
@@ -1,12 +1,12 @@
1 package bitswap
2
3 import (
4 - "os"
5 - "strconv"
4 "time"
5
6 process "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
7 + ratelimit "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/ratelimit"
8 context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
9 + waitable "github.com/ipfs/go-ipfs/thirdparty/waitable"
10
11 key "github.com/ipfs/go-ipfs/blocks/key"
12 eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
@@ -14,22 +14,6 @@ import (
14
15 var TaskWorkerCount = 8
16
17 -func init() {
18 - twc := os.Getenv("IPFS_BITSWAP_TASK_WORKERS")
19 - if twc != "" {
20 - n, err := strconv.Atoi(twc)
21 - if err != nil {
22 - log.Error(err)
23 - return
24 - }
25 - if n > 0 {
26 - TaskWorkerCount = n
27 - } else {
28 - log.Errorf("Invalid value of '%d' for IPFS_BITSWAP_TASK_WORKERS", n)
29 - }
30 - }
31 -}
32 -
17 func (bs *Bitswap) startWorkers(px process.Process, ctx context.Context) {
18 // Start up a worker to handle block requests this node is making
19 px.Go(func(px process.Process) {
@@ -57,12 +41,7 @@ func (bs *Bitswap) startWorkers(px process.Process, ctx context.Context) {
41 // Spawn up multiple workers to handle incoming blocks
42 // consider increasing number if providing blocks bottlenecks
43 // file transfers
60 - for i := 0; i < provideWorkers; i++ {
61 - i := i
62 - px.Go(func(px process.Process) {
63 - bs.provideWorker(ctx, i)
64 - })
65 - }
44 + px.Go(bs.provideWorker)
45 }
46
47 func (bs *Bitswap) taskWorker(ctx context.Context, id int) {
@@ -77,7 +56,11 @@ func (bs *Bitswap) taskWorker(ctx context.Context, id int) {
56 if !ok {
57 continue
58 }
80 - log.Event(ctx, "Bitswap.TaskWorker.Work", eventlog.LoggableMap{"ID": id, "Target": envelope.Peer.Pretty(), "Block": envelope.Block.Multihash.B58String()})
59 + log.Event(ctx, "Bitswap.TaskWorker.Work", eventlog.LoggableMap{
60 + "ID": id,
61 + "Target": envelope.Peer.Pretty(),
62 + "Block": envelope.Block.Multihash.B58String(),
63 + })
64
65 bs.wm.SendBlock(ctx, envelope)
66 case <-ctx.Done():
@@ -89,27 +72,45 @@ func (bs *Bitswap) taskWorker(ctx context.Context, id int) {
72 }
73 }
74
92 -func (bs *Bitswap) provideWorker(ctx context.Context, id int) {
93 - idmap := eventlog.LoggableMap{"ID": id}
94 - for {
95 - log.Event(ctx, "Bitswap.ProvideWorker.Loop", idmap)
96 - select {
97 - case k, ok := <-bs.provideKeys:
98 - log.Event(ctx, "Bitswap.ProvideWorker.Work", idmap, &k)
99 - if !ok {
100 - log.Debug("provideKeys channel closed")
101 - return
102 - }
103 - ctx, cancel := context.WithTimeout(ctx, provideTimeout)
104 - err := bs.network.Provide(ctx, k)
105 - if err != nil {
75 +func (bs *Bitswap) provideWorker(px process.Process) {
76 +
77 + limiter := ratelimit.NewRateLimiter(px, provideWorkerMax)
78 +
79 + limitedGoProvide := func(k key.Key, wid int) {
80 + ev := eventlog.LoggableMap{"ID": wid}
81 + limiter.LimitedGo(func(px process.Process) {
82 +
83 + ctx := waitable.Context(px) // derive ctx from px
84 + defer log.EventBegin(ctx, "Bitswap.ProvideWorker.Work", ev, &k).Done()
85 +
86 + ctx, cancel := context.WithTimeout(ctx, provideTimeout) // timeout ctx
87 + defer cancel()
88 +
89 + if err := bs.network.Provide(ctx, k); err != nil {
90 log.Error(err)
91 }
108 - cancel()
109 - case <-ctx.Done():
110 - return
111 - }
92 + })
93 }
94 +
95 + // worker spawner, reads from bs.provideKeys until it closes, spawning a
96 + // _ratelimited_ number of workers to handle each key.
97 + limiter.Go(func(px process.Process) {
98 + for wid := 2; ; wid++ {
99 + ev := eventlog.LoggableMap{"ID": 1}
100 + log.Event(waitable.Context(px), "Bitswap.ProvideWorker.Loop", ev)
101 +
102 + select {
103 + case <-px.Closing():
104 + return
105 + case k, ok := <-bs.provideKeys:
106 + if !ok {
107 + log.Debug("provideKeys channel closed")
108 + return
109 + }
110 + limitedGoProvide(k, wid)
111 + }
112 + }
113 + })
114 }
115
116 func (bs *Bitswap) provideCollector(ctx context.Context) {