feat(PQ)
refactor: peerRequestQueue it's a mistake to make one queue to fit all. Go's lack of algebraic types turns a generalized queue into a monstrosity of type checking/casting. Better to have individual queues for individual purposes. Conflicts: exchange/bitswap/decision/bench_test.go exchange/bitswap/decision/tasks/task_queue.go fix(bitswap.decision.PRQ): if peers match, always return result of pri comparison fix(bitswap.decision.Engine): push to the queue before notifying TOCTOU bug 1. client notifies 2. worker checks (finds nil) 3. worker sleeps 3. client pushes (worker missed the update) test(PQ): improve documentation and add test test(bitswap.decision.Engine): handling received messages License: MIT Signed-off-by: Brian Tiger Chow <brian@perfmode.com>
Brian Tiger Chow committed
Dec 18, 2014 at 23:07 UTC
61e4300d59e3f7ede5f807635967b0e214f4a4d1
8 files changed
+494
-107
exchange/bitswap/decision/bench_test.go
+2
-1
@@ -13,12 +13,13 @@ import (
13
// FWIW: At the time of this commit, including a timestamp in task increases
14
// time cost of Push by 3%.
15
func BenchmarkTaskQueuePush(b *testing.B) {
16
- q := newTaskQueue()
16
+ q := newPRQ()
17
peers := []peer.ID{
18
testutil.RandPeerIDFatal(b),
19
testutil.RandPeerIDFatal(b),
20
testutil.RandPeerIDFatal(b),
21
}
22
+ b.ResetTimer()
23
for i := 0; i < b.N; i++ {
24
q.Push(wantlist.Entry{Key: util.Key(i), Priority: math.MaxInt32}, peers[i%len(peers)])
25
}
exchange/bitswap/decision/engine.go
+4
-4
@@ -59,7 +59,7 @@ type Engine struct {
59
// peerRequestQueue is a priority queue of requests received from peers.
60
// Requests are popped from the queue, packaged up, and placed in the
61
// outbox.
62
- peerRequestQueue *taskQueue
62
+ peerRequestQueue peerRequestQueue
63
64
// FIXME it's a bit odd for the client and the worker to both share memory
65
// (both modify the peerRequestQueue) and also to communicate over the
@@ -82,7 +82,7 @@ func NewEngine(ctx context.Context, bs bstore.Blockstore) *Engine {
82
e := &Engine{
83
ledgerMap: make(map[peer.ID]*ledger),
84
bs: bs,
85
- peerRequestQueue: newTaskQueue(),
85
+ peerRequestQueue: newPRQ(),
86
outbox: make(chan Envelope, sizeOutboxChan),
87
workSignal: make(chan struct{}),
88
}
@@ -180,8 +180,8 @@ func (e *Engine) MessageReceived(p peer.ID, m bsmsg.BitSwapMessage) error {
180
log.Debug("wants", entry.Key, entry.Priority)
181
l.Wants(entry.Key, entry.Priority)
182
if exists, err := e.bs.Has(entry.Key); err == nil && exists {
183
- newWorkExists = true
183
e.peerRequestQueue.Push(entry.Entry, p)
184
+ newWorkExists = true
185
}
186
}
187
}
@@ -191,8 +191,8 @@ func (e *Engine) MessageReceived(p peer.ID, m bsmsg.BitSwapMessage) error {
191
l.ReceivedBytes(len(block.Data))
192
for _, l := range e.ledgerMap {
193
if entry, ok := l.WantListContains(block.Key()); ok {
194
- newWorkExists = true
194
e.peerRequestQueue.Push(entry, l.Partner)
195
+ newWorkExists = true
196
}
197
}
198
}
exchange/bitswap/decision/engine_test.go
+108
-9
@@ -1,17 +1,19 @@
1
package decision
2
3
import (
4
+ "math"
5
"strings"
6
+ "sync"
7
"testing"
8
9
context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
10
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9
- sync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
10
-
11
+ dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
12
blocks "github.com/jbenet/go-ipfs/blocks"
13
blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
14
message "github.com/jbenet/go-ipfs/exchange/bitswap/message"
15
peer "github.com/jbenet/go-ipfs/p2p/peer"
16
+ testutil "github.com/jbenet/go-ipfs/util/testutil"
17
)
18
19
type peerAndEngine struct {
@@ -19,18 +21,20 @@ type peerAndEngine struct {
21
Engine *Engine
22
}
23
22
-func newPeerAndLedgermanager(idStr string) peerAndEngine {
24
+func newEngine(ctx context.Context, idStr string) peerAndEngine {
25
return peerAndEngine{
26
Peer: peer.ID(idStr),
27
//Strategy: New(true),
26
- Engine: NewEngine(context.TODO(),
27
- blockstore.NewBlockstore(sync.MutexWrap(ds.NewMapDatastore()))),
28
+ Engine: NewEngine(ctx,
29
+ blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))),
30
}
31
}
32
33
func TestConsistentAccounting(t *testing.T) {
32
- sender := newPeerAndLedgermanager("Ernie")
33
- receiver := newPeerAndLedgermanager("Bert")
34
+ ctx, cancel := context.WithCancel(context.Background())
35
+ defer cancel()
36
+ sender := newEngine(ctx, "Ernie")
37
+ receiver := newEngine(ctx, "Bert")
38
39
// Send messages from Ernie to Bert
40
for i := 0; i < 1000; i++ {
@@ -62,8 +66,10 @@ func TestConsistentAccounting(t *testing.T) {
66
67
func TestPeerIsAddedToPeersWhenMessageReceivedOrSent(t *testing.T) {
68
65
- sanfrancisco := newPeerAndLedgermanager("sf")
66
- seattle := newPeerAndLedgermanager("sea")
69
+ ctx, cancel := context.WithCancel(context.Background())
70
+ defer cancel()
71
+ sanfrancisco := newEngine(ctx, "sf")
72
+ seattle := newEngine(ctx, "sea")
73
74
m := message.New()
75
@@ -91,3 +97,96 @@ func peerIsPartner(p peer.ID, e *Engine) bool {
97
}
98
return false
99
}
100
+
101
+func TestOutboxClosedWhenEngineClosed(t *testing.T) {
102
+ t.SkipNow() // TODO implement *Engine.Close
103
+ e := NewEngine(context.Background(), blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())))
104
+ var wg sync.WaitGroup
105
+ wg.Add(1)
106
+ go func() {
107
+ for _ = range e.Outbox() {
108
+ }
109
+ wg.Done()
110
+ }()
111
+ // e.Close()
112
+ wg.Wait()
113
+ if _, ok := <-e.Outbox(); ok {
114
+ t.Fatal("channel should be closed")
115
+ }
116
+}
117
+
118
+func TestPartnerWantsThenCancels(t *testing.T) {
119
+ alphabet := strings.Split("abcdefghijklmnopqrstuvwxyz", "")
120
+ vowels := strings.Split("aeiou", "")
121
+
122
+ type testCase [][]string
123
+ testcases := []testCase{
124
+ testCase{
125
+ alphabet, vowels,
126
+ },
127
+ testCase{
128
+ alphabet, stringsComplement(alphabet, vowels),
129
+ },
130
+ }
131
+
132
+ for _, testcase := range testcases {
133
+ set := testcase[0]
134
+ cancels := testcase[1]
135
+ keeps := stringsComplement(set, cancels)
136
+
137
+ bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
138
+ e := NewEngine(context.Background(), bs)
139
+ partner := testutil.RandPeerIDFatal(t)
140
+ for _, letter := range set {
141
+ block := blocks.NewBlock([]byte(letter))
142
+ bs.Put(block)
143
+ }
144
+ partnerWants(e, set, partner)
145
+ partnerCancels(e, cancels, partner)
146
+ assertPoppedInOrder(t, e, keeps)
147
+ }
148
+
149
+}
150
+
151
+func partnerWants(e *Engine, keys []string, partner peer.ID) {
152
+ add := message.New()
153
+ for i, letter := range keys {
154
+ block := blocks.NewBlock([]byte(letter))
155
+ add.AddEntry(block.Key(), math.MaxInt32-i)
156
+ }
157
+ e.MessageReceived(partner, add)
158
+}
159
+
160
+func partnerCancels(e *Engine, keys []string, partner peer.ID) {
161
+ cancels := message.New()
162
+ for _, k := range keys {
163
+ block := blocks.NewBlock([]byte(k))
164
+ cancels.Cancel(block.Key())
165
+ }
166
+ e.MessageReceived(partner, cancels)
167
+}
168
+
169
+func assertPoppedInOrder(t *testing.T, e *Engine, keys []string) {
170
+ for _, k := range keys {
171
+ envelope := <-e.Outbox()
172
+ received := envelope.Message.Blocks()[0]
173
+ expected := blocks.NewBlock([]byte(k))
174
+ if received.Key() != expected.Key() {
175
+ t.Fatal("received", string(received.Data), "expected", string(expected.Data))
176
+ }
177
+ }
178
+}
179
+
180
+func stringsComplement(set, subset []string) []string {
181
+ m := make(map[string]struct{})
182
+ for _, letter := range subset {
183
+ m[letter] = struct{}{}
184
+ }
185
+ var complement []string
186
+ for _, letter := range set {
187
+ if _, exists := m[letter]; !exists {
188
+ complement = append(complement, letter)
189
+ }
190
+ }
191
+ return complement
192
+}
exchange/bitswap/decision/peer_request_queue.go
new
+134
@@ -0,0 +1,134 @@
1
+package decision
2
+
3
+import (
4
+ "sync"
5
+ "time"
6
+
7
+ pq "github.com/jbenet/go-ipfs/exchange/bitswap/decision/pq"
8
+ wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
9
+ peer "github.com/jbenet/go-ipfs/p2p/peer"
10
+ u "github.com/jbenet/go-ipfs/util"
11
+)
12
+
13
+type peerRequestQueue interface {
14
+ // Pop returns the next peerRequestTask. Returns nil if the peerRequestQueue is empty.
15
+ Pop() *peerRequestTask
16
+ Push(entry wantlist.Entry, to peer.ID)
17
+ Remove(k u.Key, p peer.ID)
18
+ // NB: cannot expose simply expose taskQueue.Len because trashed elements
19
+ // may exist. These trashed elements should not contribute to the count.
20
+}
21
+
22
+func newPRQ() peerRequestQueue {
23
+ return &prq{
24
+ taskMap: make(map[string]*peerRequestTask),
25
+ taskQueue: pq.New(wrapCmp(V1)),
26
+ }
27
+}
28
+
29
+var _ peerRequestQueue = &prq{}
30
+
31
+// TODO: at some point, the strategy needs to plug in here
32
+// to help decide how to sort tasks (on add) and how to select
33
+// tasks (on getnext). For now, we are assuming a dumb/nice strategy.
34
+type prq struct {
35
+ lock sync.Mutex
36
+ taskQueue pq.PQ
37
+ taskMap map[string]*peerRequestTask
38
+}
39
+
40
+// Push currently adds a new peerRequestTask to the end of the list
41
+func (tl *prq) Push(entry wantlist.Entry, to peer.ID) {
42
+ tl.lock.Lock()
43
+ defer tl.lock.Unlock()
44
+ if task, ok := tl.taskMap[taskKey(to, entry.Key)]; ok {
45
+ task.Entry.Priority = entry.Priority
46
+ tl.taskQueue.Update(task.index)
47
+ return
48
+ }
49
+ task := &peerRequestTask{
50
+ Entry: entry,
51
+ Target: to,
52
+ created: time.Now(),
53
+ }
54
+ tl.taskQueue.Push(task)
55
+ tl.taskMap[task.Key()] = task
56
+}
57
+
58
+// Pop 'pops' the next task to be performed. Returns nil if no task exists.
59
+func (tl *prq) Pop() *peerRequestTask {
60
+ tl.lock.Lock()
61
+ defer tl.lock.Unlock()
62
+ var out *peerRequestTask
63
+ for tl.taskQueue.Len() > 0 {
64
+ out = tl.taskQueue.Pop().(*peerRequestTask)
65
+ delete(tl.taskMap, out.Key())
66
+ if out.trash {
67
+ continue // discarding tasks that have been removed
68
+ }
69
+ break // and return |out|
70
+ }
71
+ return out
72
+}
73
+
74
+// Remove removes a task from the queue
75
+func (tl *prq) Remove(k u.Key, p peer.ID) {
76
+ tl.lock.Lock()
77
+ t, ok := tl.taskMap[taskKey(p, k)]
78
+ if ok {
79
+ // remove the task "lazily"
80
+ // simply mark it as trash, so it'll be dropped when popped off the
81
+ // queue.
82
+ t.trash = true
83
+ }
84
+ tl.lock.Unlock()
85
+}
86
+
87
+type peerRequestTask struct {
88
+ Entry wantlist.Entry
89
+ Target peer.ID // required
90
+
91
+ // trash in a book-keeping field
92
+ trash bool
93
+ // created marks the time that the task was added to the queue
94
+ created time.Time
95
+ index int // book-keeping field used by the pq container
96
+}
97
+
98
+// Key uniquely identifies a task.
99
+func (t *peerRequestTask) Key() string {
100
+ return taskKey(t.Target, t.Entry.Key)
101
+}
102
+
103
+func (t *peerRequestTask) Index() int {
104
+ return t.index
105
+}
106
+
107
+func (t *peerRequestTask) SetIndex(i int) {
108
+ t.index = i
109
+}
110
+
111
+// taskKey returns a key that uniquely identifies a task.
112
+func taskKey(p peer.ID, k u.Key) string {
113
+ return string(p.String() + k.String())
114
+}
115
+
116
+// FIFO is a basic task comparator that returns tasks in the order created.
117
+var FIFO = func(a, b *peerRequestTask) bool {
118
+ return a.created.Before(b.created)
119
+}
120
+
121
+// V1 respects the target peer's wantlist priority. For tasks involving
122
+// different peers, the oldest task is prioritized.
123
+var V1 = func(a, b *peerRequestTask) bool {
124
+ if a.Target == b.Target {
125
+ return a.Entry.Priority > b.Entry.Priority
126
+ }
127
+ return FIFO(a, b)
128
+}
129
+
130
+func wrapCmp(f func(a, b *peerRequestTask) bool) func(a, b pq.Elem) bool {
131
+ return func(a, b pq.Elem) bool {
132
+ return f(a.(*peerRequestTask), b.(*peerRequestTask))
133
+ }
134
+}
exchange/bitswap/decision/peer_request_queue_test.go
new
+56
@@ -0,0 +1,56 @@
1
+package decision
2
+
3
+import (
4
+ "math"
5
+ "math/rand"
6
+ "sort"
7
+ "strings"
8
+ "testing"
9
+
10
+ "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
11
+ "github.com/jbenet/go-ipfs/util"
12
+ "github.com/jbenet/go-ipfs/util/testutil"
13
+)
14
+
15
+func TestPushPop(t *testing.T) {
16
+ prq := newPRQ()
17
+ partner := testutil.RandPeerIDFatal(t)
18
+ alphabet := strings.Split("abcdefghijklmnopqrstuvwxyz", "")
19
+ vowels := strings.Split("aeiou", "")
20
+ consonants := func() []string {
21
+ var out []string
22
+ for _, letter := range alphabet {
23
+ skip := false
24
+ for _, vowel := range vowels {
25
+ if letter == vowel {
26
+ skip = true
27
+ }
28
+ }
29
+ if !skip {
30
+ out = append(out, letter)
31
+ }
32
+ }
33
+ return out
34
+ }()
35
+ sort.Strings(alphabet)
36
+ sort.Strings(vowels)
37
+ sort.Strings(consonants)
38
+
39
+ // add a bunch of blocks. cancel some. drain the queue. the queue should only have the kept entries
40
+
41
+ for _, index := range rand.Perm(len(alphabet)) { // add blocks for all letters
42
+ letter := alphabet[index]
43
+ t.Log(partner.String())
44
+ prq.Push(wantlist.Entry{Key: util.Key(letter), Priority: math.MaxInt32 - index}, partner)
45
+ }
46
+ for _, consonant := range consonants {
47
+ prq.Remove(util.Key(consonant), partner)
48
+ }
49
+
50
+ for _, expected := range vowels {
51
+ received := prq.Pop().Entry.Key
52
+ if received != util.Key(expected) {
53
+ t.Fatal("received", string(received), "expected", string(expected))
54
+ }
55
+ }
56
+}
exchange/bitswap/decision/pq/container.go
new
+105
@@ -0,0 +1,105 @@
1
+package pq
2
+
3
+import "container/heap"
4
+
5
+// PQ is a basic priority queue.
6
+type PQ interface {
7
+ // Push adds the ele
8
+ Push(Elem)
9
+ // Pop returns the highest priority Elem in PQ.
10
+ Pop() Elem
11
+ // Len returns the number of elements in the PQ.
12
+ Len() int
13
+ // Update `fixes` the PQ.
14
+ Update(index int)
15
+
16
+ // TODO explain why this interface should not be extended
17
+ // It does not support Remove. This is because...
18
+}
19
+
20
+// Elem describes elements that can be added to the PQ. Clients must implement
21
+// this interface.
22
+type Elem interface {
23
+ // SetIndex stores the int index.
24
+ SetIndex(int)
25
+ // Index returns the last given by SetIndex(int).
26
+ Index() int
27
+}
28
+
29
+// ElemComparator returns true if pri(a) > pri(b)
30
+type ElemComparator func(a, b Elem) bool
31
+
32
+// New creates a PQ with a client-supplied comparator.
33
+func New(cmp ElemComparator) PQ {
34
+ q := &wrapper{heapinterface{
35
+ elems: make([]Elem, 0),
36
+ cmp: cmp,
37
+ }}
38
+ heap.Init(&q.heapinterface)
39
+ return q
40
+}
41
+
42
+// wrapper exists because we cannot re-define Push. We want to expose
43
+// Push(Elem) but heap.Interface requires Push(interface{})
44
+type wrapper struct {
45
+ heapinterface
46
+}
47
+
48
+var _ PQ = &wrapper{}
49
+
50
+func (w *wrapper) Push(e Elem) {
51
+ heap.Push(&w.heapinterface, e)
52
+}
53
+
54
+func (w *wrapper) Pop() Elem {
55
+ return heap.Pop(&w.heapinterface).(Elem)
56
+}
57
+
58
+func (w *wrapper) Update(index int) {
59
+ heap.Fix(&w.heapinterface, index)
60
+}
61
+
62
+// heapinterface handles dirty low-level details of managing the priority queue.
63
+type heapinterface struct {
64
+ elems []Elem
65
+ cmp ElemComparator
66
+}
67
+
68
+var _ heap.Interface = &heapinterface{}
69
+
70
+// public interface
71
+
72
+func (q *heapinterface) Len() int {
73
+ return len(q.elems)
74
+}
75
+
76
+// Less delegates the decision to the comparator
77
+func (q *heapinterface) Less(i, j int) bool {
78
+ return q.cmp(q.elems[i], q.elems[j])
79
+}
80
+
81
+// Swap swaps the elements with indexes i and j.
82
+func (q *heapinterface) Swap(i, j int) {
83
+ q.elems[i], q.elems[j] = q.elems[j], q.elems[i]
84
+ q.elems[i].SetIndex(i)
85
+ q.elems[j].SetIndex(j)
86
+}
87
+
88
+// Note that Push and Pop in this interface are for package heap's
89
+// implementation to call. To add and remove things from the heap, wrap with
90
+// the pq struct to call heap.Push and heap.Pop.
91
+
92
+func (q *heapinterface) Push(x interface{}) { // where to put the elem?
93
+ t := x.(Elem)
94
+ t.SetIndex(len(q.elems))
95
+ q.elems = append(q.elems, t)
96
+}
97
+
98
+func (q *heapinterface) Pop() interface{} {
99
+ old := q.elems
100
+ n := len(old)
101
+ elem := old[n-1] // remove the last
102
+ elem.SetIndex(-1) // for safety // FIXME why?
103
+ q.elems = old[0 : n-1] // shrink
104
+ return elem
105
+}
exchange/bitswap/decision/pq/container_test.go
new
+85
@@ -0,0 +1,85 @@
1
+package pq
2
+
3
+import (
4
+ "sort"
5
+ "testing"
6
+)
7
+
8
+type TestElem struct {
9
+ Key string
10
+ Priority int
11
+ index int
12
+}
13
+
14
+func (e *TestElem) Index() int {
15
+ return e.index
16
+}
17
+
18
+func (e *TestElem) SetIndex(i int) {
19
+ e.index = i
20
+}
21
+
22
+var PriorityComparator = func(i, j Elem) bool {
23
+ return i.(*TestElem).Priority > j.(*TestElem).Priority
24
+}
25
+
26
+func TestQueuesReturnTypeIsSameAsParameterToPush(t *testing.T) {
27
+ q := New(PriorityComparator)
28
+ expectedKey := "foo"
29
+ elem := &TestElem{Key: expectedKey}
30
+ q.Push(elem)
31
+ switch v := q.Pop().(type) {
32
+ case *TestElem:
33
+ if v.Key != expectedKey {
34
+ t.Fatal("the key doesn't match the pushed value")
35
+ }
36
+ default:
37
+ t.Fatal("the queue is not casting values appropriately")
38
+ }
39
+}
40
+
41
+func TestCorrectnessOfPop(t *testing.T) {
42
+ q := New(PriorityComparator)
43
+ tasks := []TestElem{
44
+ TestElem{Key: "a", Priority: 9},
45
+ TestElem{Key: "b", Priority: 4},
46
+ TestElem{Key: "c", Priority: 3},
47
+ TestElem{Key: "d", Priority: 0},
48
+ TestElem{Key: "e", Priority: 6},
49
+ }
50
+ for _, e := range tasks {
51
+ q.Push(&e)
52
+ }
53
+ var priorities []int
54
+ for q.Len() > 0 {
55
+ i := q.Pop().(*TestElem).Priority
56
+ t.Log("popped %v", i)
57
+ priorities = append(priorities, i)
58
+ }
59
+ if !sort.IntsAreSorted(priorities) {
60
+ t.Fatal("the values were not returned in sorted order")
61
+ }
62
+}
63
+
64
+func TestUpdate(t *testing.T) {
65
+ t.Log(`
66
+ Add 3 elements.
67
+ Update the highest priority element to have the lowest priority and fix the queue.
68
+ It should come out last.`)
69
+ q := New(PriorityComparator)
70
+ lowest := &TestElem{Key: "originallyLowest", Priority: 1}
71
+ middle := &TestElem{Key: "originallyMiddle", Priority: 2}
72
+ highest := &TestElem{Key: "toBeUpdated", Priority: 3}
73
+ q.Push(middle)
74
+ q.Push(highest)
75
+ q.Push(lowest)
76
+ if q.Pop().(*TestElem).Key != highest.Key {
77
+ t.Fatal("popped element doesn't have the highest priority")
78
+ }
79
+ q.Push(highest) // re-add the popped element
80
+ highest.Priority = 0 // update the PQ
81
+ q.Update(highest.Index()) // fix the PQ
82
+ if q.Pop().(*TestElem).Key != middle.Key {
83
+ t.Fatal("middle element should now have the highest priority")
84
+ }
85
+}
exchange/bitswap/decision/taskqueue.go
deleted
-93
@@ -1,93 +0,0 @@
1
-package decision
2
-
3
-import (
4
- "fmt"
5
- "sync"
6
- "time"
7
-
8
- wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
9
- peer "github.com/jbenet/go-ipfs/p2p/peer"
10
- u "github.com/jbenet/go-ipfs/util"
11
-)
12
-
13
-// TODO: at some point, the strategy needs to plug in here
14
-// to help decide how to sort tasks (on add) and how to select
15
-// tasks (on getnext). For now, we are assuming a dumb/nice strategy.
16
-type taskQueue struct {
17
- // TODO: make this into a priority queue
18
- lock sync.Mutex
19
- tasks []*task
20
- taskmap map[string]*task
21
-}
22
-
23
-func newTaskQueue() *taskQueue {
24
- return &taskQueue{
25
- taskmap: make(map[string]*task),
26
- }
27
-}
28
-
29
-type task struct {
30
- Entry wantlist.Entry
31
- Target peer.ID
32
- Trash bool // TODO make private
33
-
34
- created time.Time
35
-}
36
-
37
-func (t *task) String() string {
38
- return fmt.Sprintf("<Task %s, %s, %v>", t.Target, t.Entry.Key, t.Trash)
39
-}
40
-
41
-// Push currently adds a new task to the end of the list
42
-func (tl *taskQueue) Push(entry wantlist.Entry, to peer.ID) {
43
- tl.lock.Lock()
44
- defer tl.lock.Unlock()
45
- if task, ok := tl.taskmap[taskKey(to, entry.Key)]; ok {
46
- // TODO: when priority queue is implemented,
47
- // rearrange this task
48
- task.Entry.Priority = entry.Priority
49
- return
50
- }
51
- task := &task{
52
- Entry: entry,
53
- Target: to,
54
- created: time.Now(),
55
- }
56
- tl.tasks = append(tl.tasks, task)
57
- tl.taskmap[taskKey(to, entry.Key)] = task
58
-}
59
-
60
-// Pop 'pops' the next task to be performed. Returns nil no task exists.
61
-func (tl *taskQueue) Pop() *task {
62
- tl.lock.Lock()
63
- defer tl.lock.Unlock()
64
- var out *task
65
- for len(tl.tasks) > 0 {
66
- // TODO: instead of zero, use exponential distribution
67
- // it will help reduce the chance of receiving
68
- // the same block from multiple peers
69
- out = tl.tasks[0]
70
- tl.tasks = tl.tasks[1:]
71
- delete(tl.taskmap, taskKey(out.Target, out.Entry.Key))
72
- if out.Trash {
73
- continue // discarding tasks that have been removed
74
- }
75
- break // and return |out|
76
- }
77
- return out
78
-}
79
-
80
-// Remove lazily removes a task from the queue
81
-func (tl *taskQueue) Remove(k u.Key, p peer.ID) {
82
- tl.lock.Lock()
83
- t, ok := tl.taskmap[taskKey(p, k)]
84
- if ok {
85
- t.Trash = true
86
- }
87
- tl.lock.Unlock()
88
-}
89
-
90
-// taskKey returns a key that uniquely identifies a task.
91
-func taskKey(p peer.ID, k u.Key) string {
92
- return string(p) + string(k)
93
-}