tasklist queue for bitswap tasks
Jeromy committed
Dec 16, 2014 at 02:01 UTC
029e305f19a7df97d80b951cfe17692c6f9ce46e
8 files changed
+212
-98
Godeps/Godeps.json
+1
-1
@@ -1,6 +1,6 @@
1
{
2
"ImportPath": "github.com/jbenet/go-ipfs",
3
- "GoVersion": "go1.3",
3
+ "GoVersion": "devel +ffe33f1f1f17 Tue Nov 25 15:41:33 2014 +1100",
4
"Packages": [
5
"./..."
6
],
exchange/bitswap/bitswap.go
+13
-25
@@ -56,8 +56,7 @@ func New(parent context.Context, p peer.Peer, network bsnet.BitSwapNetwork, rout
56
blockstore: bstore,
57
cancelFunc: cancelFunc,
58
notifications: notif,
59
- strategy: strategy.New(nice),
60
- ledgerset: strategy.NewLedgerSet(),
59
+ ledgermanager: strategy.NewLedgerManager(bstore, ctx),
60
routing: routing,
61
sender: network,
62
wantlist: wl.New(),
@@ -93,7 +92,7 @@ type bitswap struct {
92
// strategy makes decisions about how to interact with partners.
93
strategy strategy.Strategy
94
96
- ledgerset *strategy.LedgerSet
95
+ ledgermanager *strategy.LedgerManager
96
97
wantlist *wl.Wantlist
98
@@ -197,7 +196,7 @@ func (bs *bitswap) sendWantListTo(ctx context.Context, peers <-chan peer.Peer) e
196
// FIXME ensure accounting is handled correctly when
197
// communication fails. May require slightly different API to
198
// get better guarantees. May need shared sequence numbers.
200
- bs.ledgerset.MessageSent(p, message)
199
+ bs.ledgermanager.MessageSent(p, message)
200
}(peerToQuery)
201
}
202
wg.Wait()
@@ -236,35 +235,24 @@ func (bs *bitswap) sendWantlistToProviders(ctx context.Context, wantlist *wl.Wan
235
}
236
237
func (bs *bitswap) roundWorker(ctx context.Context) {
239
- roundTicker := time.NewTicker(roundTime)
238
for {
239
select {
240
case <-ctx.Done():
241
return
244
- case <-roundTicker.C:
245
- alloc, err := bs.strategy.GetTasks(bandwidthPerRound, bs.ledgerset, bs.blockstore)
242
+ case task := <-bs.ledgermanager.GetTaskChan():
243
+ block, err := bs.blockstore.Get(task.Key)
244
if err != nil {
247
- log.Critical("%s", err)
248
- }
249
- err = bs.processStrategyAllocation(ctx, alloc)
250
- if err != nil {
251
- log.Critical("Error processing strategy allocation: %s", err)
245
+ log.Errorf("Expected to have block %s, but it was not found!", task.Key)
246
+ continue
247
}
253
- }
254
- }
255
-}
248
257
-func (bs *bitswap) processStrategyAllocation(ctx context.Context, alloc []*strategy.Task) error {
258
- for _, t := range alloc {
259
- for _, block := range t.Blocks {
249
message := bsmsg.New()
250
message.AddBlock(block)
262
- if err := bs.send(ctx, t.Peer, message); err != nil {
263
- return err
264
- }
251
+ // TODO: maybe add keys from our wantlist?
252
+
253
+ bs.send(ctx, task.Target, message)
254
}
255
}
267
- return nil
256
}
257
258
// TODO ensure only one active request per key
@@ -327,7 +315,7 @@ func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.Peer, incoming bsm
315
316
// This call records changes to wantlists, blocks received,
317
// and number of bytes transfered.
330
- bs.ledgerset.MessageReceived(p, incoming)
318
+ bs.ledgermanager.MessageReceived(p, incoming)
319
// TODO: this is bad, and could be easily abused.
320
// Should only track *useful* messages in ledger
321
@@ -352,7 +340,7 @@ func (bs *bitswap) cancelBlocks(ctx context.Context, bkeys []u.Key) {
340
for _, k := range bkeys {
341
message.AddEntry(k, 0, true)
342
}
355
- for _, p := range bs.ledgerset.Peers() {
343
+ for _, p := range bs.ledgermanager.Peers() {
344
err := bs.send(ctx, p, message)
345
if err != nil {
346
log.Errorf("Error sending message: %s", err)
@@ -372,7 +360,7 @@ func (bs *bitswap) send(ctx context.Context, p peer.Peer, m bsmsg.BitSwapMessage
360
if err := bs.sender.SendMessage(ctx, p, m); err != nil {
361
return err
362
}
375
- return bs.ledgerset.MessageSent(p, m)
363
+ return bs.ledgermanager.MessageSent(p, m)
364
}
365
366
func (bs *bitswap) Close() error {
exchange/bitswap/bitswap_test.go
+6
-6
@@ -26,7 +26,7 @@ func TestClose(t *testing.T) {
26
vnet := tn.VirtualNetwork(delay.Fixed(kNetworkDelay))
27
rout := mockrouting.NewServer()
28
sesgen := NewSessionGenerator(vnet, rout)
29
- defer sesgen.Stop()
29
+ defer sesgen.Close()
30
bgen := blocksutil.NewBlockGenerator()
31
32
block := bgen.Next()
@@ -41,7 +41,7 @@ func TestGetBlockTimeout(t *testing.T) {
41
net := tn.VirtualNetwork(delay.Fixed(kNetworkDelay))
42
rs := mockrouting.NewServer()
43
g := NewSessionGenerator(net, rs)
44
- defer g.Stop()
44
+ defer g.Close()
45
46
self := g.Next()
47
@@ -59,7 +59,7 @@ func TestProviderForKeyButNetworkCannotFind(t *testing.T) {
59
net := tn.VirtualNetwork(delay.Fixed(kNetworkDelay))
60
rs := mockrouting.NewServer()
61
g := NewSessionGenerator(net, rs)
62
- defer g.Stop()
62
+ defer g.Close()
63
64
block := blocks.NewBlock([]byte("block"))
65
rs.Client(testutil.NewPeerWithIDString("testing")).Provide(context.Background(), block.Key()) // but not on network
@@ -83,7 +83,7 @@ func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {
83
rs := mockrouting.NewServer()
84
block := blocks.NewBlock([]byte("block"))
85
g := NewSessionGenerator(net, rs)
86
- defer g.Stop()
86
+ defer g.Close()
87
88
hasBlock := g.Next()
89
defer hasBlock.Exchange.Close()
@@ -137,7 +137,7 @@ func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) {
137
net := tn.VirtualNetwork(delay.Fixed(kNetworkDelay))
138
rs := mockrouting.NewServer()
139
sg := NewSessionGenerator(net, rs)
140
- defer sg.Stop()
140
+ defer sg.Close()
141
bg := blocksutil.NewBlockGenerator()
142
143
t.Log("Test a few nodes trying to get one file with a lot of blocks")
@@ -203,7 +203,7 @@ func TestSendToWantingPeer(t *testing.T) {
203
net := tn.VirtualNetwork(delay.Fixed(kNetworkDelay))
204
rs := mockrouting.NewServer()
205
sg := NewSessionGenerator(net, rs)
206
- defer sg.Stop()
206
+ defer sg.Close()
207
bg := blocksutil.NewBlockGenerator()
208
209
oldVal := rebroadcastDelay
exchange/bitswap/strategy/interface.go
+1
-1
@@ -8,5 +8,5 @@ type Strategy interface {
8
// Seed initializes the decider to a deterministic state
9
Seed(int64)
10
11
- GetTasks(bandwidth int, ledgers *LedgerSet, bs bstore.Blockstore) ([]*Task, error)
11
+ GetTasks(bandwidth int, ledgers *LedgerManager, bs bstore.Blockstore) ([]*Task, error)
12
}
exchange/bitswap/strategy/ledgerset.go
+97
-43
@@ -3,6 +3,9 @@ package strategy
3
import (
4
"sync"
5
6
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
7
+
8
+ bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
9
bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
10
wl "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
11
peer "github.com/jbenet/go-ipfs/peer"
@@ -15,24 +18,62 @@ type ledgerMap map[peerKey]*ledger
18
// FIXME share this externally
19
type peerKey u.Key
20
18
-type LedgerSet struct {
19
- lock sync.RWMutex
20
- ledgerMap ledgerMap
21
+type LedgerManager struct {
22
+ lock sync.RWMutex
23
+ ledgerMap ledgerMap
24
+ bs bstore.Blockstore
25
+ tasklist *TaskList
26
+ taskOut chan *Task
27
+ workSignal chan struct{}
28
+ ctx context.Context
29
}
30
23
-func NewLedgerSet() *LedgerSet {
24
- return &LedgerSet{
25
- ledgerMap: make(ledgerMap),
31
+func NewLedgerManager(bs bstore.Blockstore, ctx context.Context) *LedgerManager {
32
+ lm := &LedgerManager{
33
+ ledgerMap: make(ledgerMap),
34
+ bs: bs,
35
+ tasklist: NewTaskList(),
36
+ taskOut: make(chan *Task, 4),
37
+ workSignal: make(chan struct{}),
38
+ ctx: ctx,
39
}
40
+ go lm.taskWorker()
41
+ return lm
42
+}
43
+
44
+func (lm *LedgerManager) taskWorker() {
45
+ for {
46
+ nextTask := lm.tasklist.GetNext()
47
+ if nextTask == nil {
48
+ // No tasks in the list?
49
+ // Wait until there are!
50
+ select {
51
+ case <-lm.ctx.Done():
52
+ return
53
+ case <-lm.workSignal:
54
+ }
55
+ continue
56
+ }
57
+
58
+ select {
59
+ case <-lm.ctx.Done():
60
+ return
61
+ case lm.taskOut <- nextTask:
62
+ }
63
+ }
64
+}
65
+
66
+func (lm *LedgerManager) GetTaskChan() <-chan *Task {
67
+ return lm.taskOut
68
}
69
70
// Returns a slice of Peers with whom the local node has active sessions
30
-func (ls *LedgerSet) Peers() []peer.Peer {
31
- ls.lock.RLock()
32
- defer ls.lock.RUnlock()
71
+func (lm *LedgerManager) Peers() []peer.Peer {
72
+ lm.lock.RLock()
73
+ defer lm.lock.RUnlock()
74
75
response := make([]peer.Peer, 0)
35
- for _, ledger := range ls.ledgerMap {
76
+ for _, ledger := range lm.ledgerMap {
77
response = append(response, ledger.Partner)
78
}
79
return response
@@ -40,43 +81,55 @@ func (ls *LedgerSet) Peers() []peer.Peer {
81
82
// BlockIsWantedByPeer returns true if peer wants the block given by this
83
// key
43
-func (ls *LedgerSet) BlockIsWantedByPeer(k u.Key, p peer.Peer) bool {
44
- ls.lock.RLock()
45
- defer ls.lock.RUnlock()
84
+func (lm *LedgerManager) BlockIsWantedByPeer(k u.Key, p peer.Peer) bool {
85
+ lm.lock.RLock()
86
+ defer lm.lock.RUnlock()
87
47
- ledger := ls.ledger(p)
88
+ ledger := lm.ledger(p)
89
return ledger.WantListContains(k)
90
}
91
92
// MessageReceived performs book-keeping. Returns error if passed invalid
93
// arguments.
53
-func (ls *LedgerSet) MessageReceived(p peer.Peer, m bsmsg.BitSwapMessage) error {
54
- ls.lock.Lock()
55
- defer ls.lock.Unlock()
56
-
57
- // TODO find a more elegant way to handle this check
58
- /*
59
- if p == nil {
60
- return errors.New("Strategy received nil peer")
61
- }
62
- if m == nil {
63
- return errors.New("Strategy received nil message")
64
- }
65
- */
66
- l := ls.ledger(p)
94
+func (lm *LedgerManager) MessageReceived(p peer.Peer, m bsmsg.BitSwapMessage) error {
95
+ lm.lock.Lock()
96
+ defer lm.lock.Unlock()
97
+
98
+ l := lm.ledger(p)
99
if m.Full() {
100
l.wantList = wl.New()
101
}
102
for _, e := range m.Wantlist() {
103
if e.Cancel {
104
l.CancelWant(e.Key)
105
+ lm.tasklist.Cancel(e.Key, p)
106
} else {
107
l.Wants(e.Key, e.Priority)
108
+ lm.tasklist.Add(e.Key, e.Priority, p)
109
+
110
+ // Signal task generation to restart (if stopped!)
111
+ select {
112
+ case lm.workSignal <- struct{}{}:
113
+ default:
114
+ }
115
}
116
}
117
+
118
for _, block := range m.Blocks() {
119
// FIXME extract blocks.NumBytes(block) or block.NumBytes() method
120
l.ReceivedBytes(len(block.Data))
121
+ for _, l := range lm.ledgerMap {
122
+ if l.WantListContains(block.Key()) {
123
+ lm.tasklist.Add(block.Key(), 1, l.Partner)
124
+
125
+ // Signal task generation to restart (if stopped!)
126
+ select {
127
+ case lm.workSignal <- struct{}{}:
128
+ default:
129
+ }
130
+
131
+ }
132
+ }
133
}
134
return nil
135
}
@@ -87,39 +140,40 @@ func (ls *LedgerSet) MessageReceived(p peer.Peer, m bsmsg.BitSwapMessage) error
140
// inconsistent. Would need to ensure that Sends and acknowledgement of the
141
// send happen atomically
142
90
-func (ls *LedgerSet) MessageSent(p peer.Peer, m bsmsg.BitSwapMessage) error {
91
- ls.lock.Lock()
92
- defer ls.lock.Unlock()
143
+func (lm *LedgerManager) MessageSent(p peer.Peer, m bsmsg.BitSwapMessage) error {
144
+ lm.lock.Lock()
145
+ defer lm.lock.Unlock()
146
94
- l := ls.ledger(p)
147
+ l := lm.ledger(p)
148
for _, block := range m.Blocks() {
149
l.SentBytes(len(block.Data))
150
l.wantList.Remove(block.Key())
151
+ lm.tasklist.Cancel(block.Key(), p)
152
}
153
154
return nil
155
}
156
103
-func (ls *LedgerSet) NumBytesSentTo(p peer.Peer) uint64 {
104
- ls.lock.RLock()
105
- defer ls.lock.RUnlock()
157
+func (lm *LedgerManager) NumBytesSentTo(p peer.Peer) uint64 {
158
+ lm.lock.RLock()
159
+ defer lm.lock.RUnlock()
160
107
- return ls.ledger(p).Accounting.BytesSent
161
+ return lm.ledger(p).Accounting.BytesSent
162
}
163
110
-func (ls *LedgerSet) NumBytesReceivedFrom(p peer.Peer) uint64 {
111
- ls.lock.RLock()
112
- defer ls.lock.RUnlock()
164
+func (lm *LedgerManager) NumBytesReceivedFrom(p peer.Peer) uint64 {
165
+ lm.lock.RLock()
166
+ defer lm.lock.RUnlock()
167
114
- return ls.ledger(p).Accounting.BytesRecv
168
+ return lm.ledger(p).Accounting.BytesRecv
169
}
170
171
// ledger lazily instantiates a ledger
118
-func (ls *LedgerSet) ledger(p peer.Peer) *ledger {
119
- l, ok := ls.ledgerMap[peerKey(p.Key())]
172
+func (lm *LedgerManager) ledger(p peer.Peer) *ledger {
173
+ l, ok := lm.ledgerMap[peerKey(p.Key())]
174
if !ok {
175
l = newLedger(p)
122
- ls.ledgerMap[peerKey(p.Key())] = l
176
+ lm.ledgerMap[peerKey(p.Key())] = l
177
}
178
return l
179
}
exchange/bitswap/strategy/ledgerset_test.go
+14
-12
@@ -4,28 +4,30 @@ import (
4
"strings"
5
"testing"
6
7
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
8
+
9
blocks "github.com/jbenet/go-ipfs/blocks"
10
message "github.com/jbenet/go-ipfs/exchange/bitswap/message"
11
peer "github.com/jbenet/go-ipfs/peer"
12
testutil "github.com/jbenet/go-ipfs/util/testutil"
13
)
14
13
-type peerAndLedgerset struct {
15
+type peerAndLedgermanager struct {
16
peer.Peer
15
- ls *LedgerSet
17
+ ls *LedgerManager
18
}
19
18
-func newPeerAndLedgerset(idStr string) peerAndLedgerset {
19
- return peerAndLedgerset{
20
+func newPeerAndLedgermanager(idStr string) peerAndLedgermanager {
21
+ return peerAndLedgermanager{
22
Peer: testutil.NewPeerWithIDString(idStr),
23
//Strategy: New(true),
22
- ls: NewLedgerSet(),
24
+ ls: NewLedgerManager(nil, context.TODO()),
25
}
26
}
27
28
func TestConsistentAccounting(t *testing.T) {
27
- sender := newPeerAndLedgerset("Ernie")
28
- receiver := newPeerAndLedgerset("Bert")
29
+ sender := newPeerAndLedgermanager("Ernie")
30
+ receiver := newPeerAndLedgermanager("Bert")
31
32
// Send messages from Ernie to Bert
33
for i := 0; i < 1000; i++ {
@@ -56,8 +58,8 @@ func TestConsistentAccounting(t *testing.T) {
58
}
59
60
func TestBlockRecordedAsWantedAfterMessageReceived(t *testing.T) {
59
- beggar := newPeerAndLedgerset("can't be chooser")
60
- chooser := newPeerAndLedgerset("chooses JIF")
61
+ beggar := newPeerAndLedgermanager("can't be chooser")
62
+ chooser := newPeerAndLedgermanager("chooses JIF")
63
64
block := blocks.NewBlock([]byte("data wanted by beggar"))
65
@@ -74,8 +76,8 @@ func TestBlockRecordedAsWantedAfterMessageReceived(t *testing.T) {
76
77
func TestPeerIsAddedToPeersWhenMessageReceivedOrSent(t *testing.T) {
78
77
- sanfrancisco := newPeerAndLedgerset("sf")
78
- seattle := newPeerAndLedgerset("sea")
79
+ sanfrancisco := newPeerAndLedgermanager("sf")
80
+ seattle := newPeerAndLedgermanager("sea")
81
82
m := message.New()
83
@@ -95,7 +97,7 @@ func TestPeerIsAddedToPeersWhenMessageReceivedOrSent(t *testing.T) {
97
}
98
}
99
98
-func peerIsPartner(p peer.Peer, ls *LedgerSet) bool {
100
+func peerIsPartner(p peer.Peer, ls *LedgerManager) bool {
101
for _, partner := range ls.Peers() {
102
if partner.Key() == p.Key() {
103
return true
exchange/bitswap/strategy/strategy.go
+8
-10
@@ -1,15 +1,16 @@
1
package strategy
2
3
import (
4
- blocks "github.com/jbenet/go-ipfs/blocks"
5
- bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
6
- wl "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
7
- peer "github.com/jbenet/go-ipfs/peer"
4
+ //blocks "github.com/jbenet/go-ipfs/blocks"
5
+ //bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
6
+ //wl "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
7
+ //peer "github.com/jbenet/go-ipfs/peer"
8
u "github.com/jbenet/go-ipfs/util"
9
)
10
11
var log = u.Logger("strategy")
12
13
+/*
14
// TODO niceness should be on a per-peer basis. Use-case: Certain peers are
15
// "trusted" and/or controlled by a single human user. The user may want for
16
// these peers to exchange data freely
@@ -29,12 +30,7 @@ type strategist struct {
30
strategyFunc
31
}
32
32
-type Task struct {
33
- Peer peer.Peer
34
- Blocks []*blocks.Block
35
-}
36
-
37
-func (s *strategist) GetTasks(bandwidth int, ledgers *LedgerSet, bs bstore.Blockstore) ([]*Task, error) {
33
+func (s *strategist) GetTasks(bandwidth int, ledgers *LedgerManager, bs bstore.Blockstore) ([]*Task, error) {
34
var tasks []*Task
35
36
ledgers.lock.RLock()
@@ -87,3 +83,5 @@ func test() {}
83
func (s *strategist) Seed(int64) {
84
// TODO
85
}
86
+
87
+*/
exchange/bitswap/strategy/tasklist.go
new
+72
@@ -0,0 +1,72 @@
1
+package strategy
2
+
3
+import (
4
+ peer "github.com/jbenet/go-ipfs/peer"
5
+ u "github.com/jbenet/go-ipfs/util"
6
+)
7
+
8
+// TODO: at some point, the strategy needs to plug in here
9
+// to help decide how to sort tasks (on add) and how to select
10
+// tasks (on getnext). For now, we are assuming a dumb/nice strategy.
11
+type TaskList struct {
12
+ tasks []*Task
13
+ taskmap map[u.Key]*Task
14
+}
15
+
16
+func NewTaskList() *TaskList {
17
+ return &TaskList{
18
+ taskmap: make(map[u.Key]*Task),
19
+ }
20
+}
21
+
22
+type Task struct {
23
+ Key u.Key
24
+ Target peer.Peer
25
+ theirPriority int
26
+}
27
+
28
+// Add currently adds a new task to the end of the list
29
+// TODO: make this into a priority queue
30
+func (tl *TaskList) Add(block u.Key, priority int, to peer.Peer) {
31
+ if task, ok := tl.taskmap[to.Key()+block]; ok {
32
+ // TODO: when priority queue is implemented,
33
+ // rearrange this Task
34
+ task.theirPriority = priority
35
+ return
36
+ }
37
+ task := &Task{
38
+ Key: block,
39
+ Target: to,
40
+ theirPriority: priority,
41
+ }
42
+ tl.tasks = append(tl.tasks, task)
43
+ tl.taskmap[to.Key()+block] = task
44
+}
45
+
46
+// GetNext returns the next task to be performed by bitswap
47
+// the task is then removed from the list
48
+func (tl *TaskList) GetNext() *Task {
49
+ var out *Task
50
+ for len(tl.tasks) > 0 {
51
+ // TODO: instead of zero, use exponential distribution
52
+ // it will help reduce the chance of receiving
53
+ // the same block from multiple peers
54
+ out = tl.tasks[0]
55
+ tl.tasks = tl.tasks[1:]
56
+ delete(tl.taskmap, out.Target.Key()+out.Key)
57
+ // Filter out blocks that have been cancelled
58
+ if out.theirPriority >= 0 {
59
+ break
60
+ }
61
+ }
62
+
63
+ return out
64
+}
65
+
66
+// Cancel lazily cancels the sending of a block to a given peer
67
+func (tl *TaskList) Cancel(k u.Key, p peer.Peer) {
68
+ t, ok := tl.taskmap[p.Key()+k]
69
+ if ok {
70
+ t.theirPriority = -1
71
+ }
72
+}