deduplicate blocks in queue
Brian Tiger Chow committed
Jan 21, 2015 at 14:58 UTC
fadede41fc6d034240cefdabd0519362e82dfd43
2 files changed
+71
-5
blockservice/worker/worker.go
+21
-4
@@ -140,15 +140,30 @@ func (w *Worker) start(c Config) {
140
}
141
142
type BlockList struct {
143
- list list.List
143
+ list list.List
144
+ uniques map[util.Key]*list.Element
145
}
146
147
func (s *BlockList) PushFront(b *blocks.Block) {
147
- s.list.PushFront(b)
148
+ if s.uniques == nil {
149
+ s.uniques = make(map[util.Key]*list.Element)
150
+ }
151
+ _, ok := s.uniques[b.Key()]
152
+ if !ok {
153
+ e := s.list.PushFront(b)
154
+ s.uniques[b.Key()] = e
155
+ }
156
}
157
158
func (s *BlockList) Push(b *blocks.Block) {
151
- s.list.PushBack(b)
159
+ if s.uniques == nil {
160
+ s.uniques = make(map[util.Key]*list.Element)
161
+ }
162
+ _, ok := s.uniques[b.Key()]
163
+ if !ok {
164
+ e := s.list.PushBack(b)
165
+ s.uniques[b.Key()] = e
166
+ }
167
}
168
169
func (s *BlockList) Pop() *blocks.Block {
@@ -157,7 +172,9 @@ func (s *BlockList) Pop() *blocks.Block {
172
}
173
e := s.list.Front()
174
s.list.Remove(e)
160
- return e.Value.(*blocks.Block)
175
+ b := e.Value.(*blocks.Block)
176
+ delete(s.uniques, b.Key())
177
+ return b
178
}
179
180
func (s *BlockList) Len() int {
blockservice/worker/worker_test.go
+50
-1
@@ -1,6 +1,9 @@
1
package worker
2
3
-import "testing"
3
+import (
4
+ blocks "github.com/jbenet/go-ipfs/blocks"
5
+ "testing"
6
+)
7
8
func TestStartClose(t *testing.T) {
9
numRuns := 50
@@ -12,3 +15,49 @@ func TestStartClose(t *testing.T) {
15
w.Close()
16
}
17
}
18
+
19
+func TestQueueDeduplication(t *testing.T) {
20
+ numUniqBlocks := 5 // arbitrary
21
+
22
+ var firstBatch []*blocks.Block
23
+ for i := 0; i < numUniqBlocks; i++ {
24
+ firstBatch = append(firstBatch, blockFromInt(i))
25
+ }
26
+
27
+ // to get different pointer values and prevent the implementation from
28
+ // cheating. The impl must check equality using Key.
29
+ var secondBatch []*blocks.Block
30
+ for i := 0; i < numUniqBlocks; i++ {
31
+ secondBatch = append(secondBatch, blockFromInt(i))
32
+ }
33
+ var workQueue BlockList
34
+
35
+ for _, b := range append(firstBatch, secondBatch...) {
36
+ workQueue.Push(b)
37
+ }
38
+ for i := 0; i < numUniqBlocks; i++ {
39
+ b := workQueue.Pop()
40
+ if b.Key() != firstBatch[i].Key() {
41
+ t.Fatal("list is not FIFO")
42
+ }
43
+ }
44
+ if b := workQueue.Pop(); b != nil {
45
+ t.Fatal("the workQueue did not de-duplicate the blocks")
46
+ }
47
+}
48
+
49
+func TestPushPopPushPop(t *testing.T) {
50
+ var workQueue BlockList
51
+ orig := blockFromInt(1)
52
+ dup := blockFromInt(1)
53
+ workQueue.PushFront(orig)
54
+ workQueue.Pop()
55
+ workQueue.Push(dup)
56
+ if workQueue.Len() != 1 {
57
+ t.Fatal("the block list's internal state is corrupt")
58
+ }
59
+}
60
+
61
+func blockFromInt(i int) *blocks.Block {
62
+ return blocks.NewBlock([]byte(string(i)))
63
+}