cleanup bitswap and handle message send failure slightly better
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Nov 21, 2016 at 20:32 UTC
42e6e547c3671e43a5ea951b1dd3529e13c56909
3 files changed
+88
-49
exchange/bitswap/bitswap.go
+18
-18
@@ -82,7 +82,6 @@ func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
82
})
83
84
bs := &Bitswap{
85
- self: p,
85
blockstore: bstore,
86
notifications: notif,
87
engine: decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
@@ -112,34 +111,36 @@ func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
111
112
// Bitswap instances implement the bitswap protocol.
113
type Bitswap struct {
114
+ // the peermanager manages sending messages to peers in a way that
115
+ // wont block bitswap operation
116
+ wm *WantManager
117
116
- // the ID of the peer to act on behalf of
117
- self peer.ID
118
+ // the engine is the bit of logic that decides who to send which blocks to
119
+ engine *decision.Engine
120
121
// network delivers messages on behalf of the session
122
network bsnet.BitSwapNetwork
123
122
- // the peermanager manages sending messages to peers in a way that
123
- // wont block bitswap operation
124
- wm *WantManager
125
-
124
// blockstore is the local database
125
// NB: ensure threadsafety
126
blockstore blockstore.Blockstore
127
128
+ // notifications engine for receiving new blocks and routing them to the
129
+ // appropriate user requests
130
notifications notifications.PubSub
131
132
- // send keys to a worker to find and connect to providers for them
132
+ // findKeys sends keys to a worker to find and connect to providers for them
133
findKeys chan *blockRequest
134
-
135
- engine *decision.Engine
136
-
137
- process process.Process
138
-
134
+ // newBlocks is a channel for newly added blocks to be provided to the
135
+ // network. blocks pushed down this channel get buffered and fed to the
136
+ // provideKeys channel later on to avoid too much network activity
137
newBlocks chan *cid.Cid
140
-
138
+ // provideKeys directly feeds provide workers
139
provideKeys chan *cid.Cid
140
141
+ process process.Process
142
+
143
+ // Counters for various statistics
144
counterLk sync.Mutex
145
blocksRecvd int
146
dupBlocksRecvd int
@@ -167,13 +168,12 @@ func (bs *Bitswap) GetBlock(parent context.Context, k *cid.Cid) (blocks.Block, e
168
// enforce. May this comment keep you safe.
169
ctx, cancelFunc := context.WithCancel(parent)
170
171
+ // TODO: this request ID should come in from a higher layer so we can track
172
+ // across multiple 'GetBlock' invocations
173
ctx = logging.ContextWithLoggable(ctx, loggables.Uuid("GetBlockRequest"))
174
log.Event(ctx, "Bitswap.GetBlockRequest.Start", k)
175
defer log.Event(ctx, "Bitswap.GetBlockRequest.End", k)
173
-
174
- defer func() {
175
- cancelFunc()
176
- }()
176
+ defer cancelFunc()
177
178
promise, err := bs.GetBlocks(ctx, []*cid.Cid{k})
179
if err != nil {
exchange/bitswap/wantmanager.go
+64
-31
@@ -175,28 +175,13 @@ func (mq *msgQueue) runQueue(ctx context.Context) {
175
}
176
177
func (mq *msgQueue) doWork(ctx context.Context) {
178
- // allow ten minutes for connections
179
- // this includes looking them up in the dht
180
- // dialing them, and handshaking
178
if mq.sender == nil {
182
- conctx, cancel := context.WithTimeout(ctx, time.Minute*10)
183
- defer cancel()
184
-
185
- err := mq.network.ConnectTo(conctx, mq.p)
179
+ err := mq.openSender(ctx)
180
if err != nil {
187
- log.Infof("cant connect to peer %s: %s", mq.p, err)
181
+ log.Infof("cant open message sender to peer %s: %s", mq.p, err)
182
// TODO: cant connect, what now?
183
return
184
}
191
-
192
- nsender, err := mq.network.NewMessageSender(ctx, mq.p)
193
- if err != nil {
194
- log.Infof("cant open new stream to peer %s: %s", mq.p, err)
195
- // TODO: cant open stream, what now?
196
- return
197
- }
198
-
199
- mq.sender = nsender
185
}
186
187
// grab outgoing message
@@ -210,14 +195,64 @@ func (mq *msgQueue) doWork(ctx context.Context) {
195
mq.outlk.Unlock()
196
197
// send wantlist updates
213
- err := mq.sender.SendMsg(wlm)
214
- if err != nil {
198
+ for { // try to send this message until we fail.
199
+ err := mq.sender.SendMsg(wlm)
200
+ if err == nil {
201
+ return
202
+ }
203
+
204
log.Infof("bitswap send error: %s", err)
205
mq.sender.Close()
206
mq.sender = nil
218
- // TODO: what do we do if this fails?
219
- return
207
+
208
+ select {
209
+ case <-mq.done:
210
+ return
211
+ case <-ctx.Done():
212
+ return
213
+ case <-time.After(time.Millisecond * 100):
214
+ // wait 100ms in case disconnect notifications are still propogating
215
+ log.Warning("SendMsg errored but neither 'done' nor context.Done() were set")
216
+ }
217
+
218
+ err = mq.openSender(ctx)
219
+ if err != nil {
220
+ log.Error("couldnt open sender again after SendMsg(%s) failed: %s", mq.p, err)
221
+ // TODO(why): what do we do now?
222
+ // I think the *right* answer is to probably put the message we're
223
+ // trying to send back, and then return to waiting for new work or
224
+ // a disconnect.
225
+ return
226
+ }
227
+
228
+ // TODO: Is this the same instance for the remote peer?
229
+ // If its not, we should resend our entire wantlist to them
230
+ /*
231
+ if mq.sender.InstanceID() != mq.lastSeenInstanceID {
232
+ wlm = mq.getFullWantlistMessage()
233
+ }
234
+ */
235
+ }
236
+}
237
+
238
+func (mq *msgQueue) openSender(ctx context.Context) error {
239
+ // allow ten minutes for connections this includes looking them up in the
240
+ // dht dialing them, and handshaking
241
+ conctx, cancel := context.WithTimeout(ctx, time.Minute*10)
242
+ defer cancel()
243
+
244
+ err := mq.network.ConnectTo(conctx, mq.p)
245
+ if err != nil {
246
+ return err
247
+ }
248
+
249
+ nsender, err := mq.network.NewMessageSender(ctx, mq.p)
250
+ if err != nil {
251
+ return err
252
}
253
+
254
+ mq.sender = nsender
255
+ return nil
256
}
257
258
func (pm *WantManager) Connected(p peer.ID) {
@@ -292,14 +327,13 @@ func (pm *WantManager) Run() {
327
}
328
329
func (wm *WantManager) newMsgQueue(p peer.ID) *msgQueue {
295
- mq := new(msgQueue)
296
- mq.done = make(chan struct{})
297
- mq.work = make(chan struct{}, 1)
298
- mq.network = wm.network
299
- mq.p = p
300
- mq.refcnt = 1
301
-
302
- return mq
330
+ return &msgQueue{
331
+ done: make(chan struct{}),
332
+ work: make(chan struct{}, 1),
333
+ network: wm.network,
334
+ p: p,
335
+ refcnt: 1,
336
+ }
337
}
338
339
func (mq *msgQueue) addMessage(entries []*bsmsg.Entry) {
@@ -312,8 +346,7 @@ func (mq *msgQueue) addMessage(entries []*bsmsg.Entry) {
346
}
347
}()
348
315
- // if we have no message held, or the one we are given is full
316
- // overwrite the one we are holding
349
+ // if we have no message held allocate a new one
350
if mq.out == nil {
351
mq.out = bsmsg.New(false)
352
}
exchange/bitswap/workers.go
+6
@@ -197,6 +197,12 @@ func (bs *Bitswap) providerQueryManager(ctx context.Context) {
197
for {
198
select {
199
case e := <-bs.findKeys:
200
+ select { // make sure its not already cancelled
201
+ case <-e.Ctx.Done():
202
+ continue
203
+ default:
204
+ }
205
+
206
activeLk.Lock()
207
if kset.Has(e.Cid) {
208
activeLk.Unlock()