Cleanup, fix broken restart, and more tests.
License: MIT Signed-off-by: Erik Ingenito <erik@carbonfive.com>
Erik Ingenito committed
Mar 15, 2019 at 11:04 UTC
5d4f3fbdec94b0c8a408bb686e2e46a20201f4b1
4 files changed
+138
-72
provider/provider.go
+3
-18
@@ -62,28 +62,13 @@ func (p *provider) handleAnnouncements() {
62
case <-p.ctx.Done():
63
return
64
case entry := <-p.queue.Dequeue():
65
- if err := doProvide(p.ctx, p.contentRouting, entry.cid); err != nil {
65
+ log.Info("announce - start - ", entry.cid)
66
+ if err := p.contentRouting.Provide(p.ctx, entry.cid, true); err != nil {
67
log.Warningf("Unable to provide entry: %s, %s", entry.cid, err)
68
}
68
-
69
- if err := entry.Complete(); err != nil {
70
- log.Warningf("Unable to complete queue entry when providing: %s, %s", entry.cid, err)
71
- }
69
+ log.Info("announce - end - ", entry.cid)
70
}
71
}
72
}()
73
}
74
}
77
-
78
-// TODO: better document this provide logic
79
-func doProvide(ctx context.Context, contentRouting routing.ContentRouting, key cid.Cid) error {
80
- // announce
81
- log.Info("announce - start - ", key)
82
- if err := contentRouting.Provide(ctx, key, true); err != nil {
83
- log.Warningf("Failed to provide cid: %s", err)
84
- // TODO: Maybe put these failures onto a failures queue?
85
- return err
86
- }
87
- log.Info("announce - end - ", key)
88
- return nil
89
-}
provider/provider_test.go
+9
-8
@@ -2,13 +2,15 @@ package provider
2
3
import (
4
"context"
5
- "github.com/ipfs/go-cid"
6
- "github.com/ipfs/go-datastore"
7
- "github.com/ipfs/go-ipfs-blocksutil"
8
- pstore "github.com/libp2p/go-libp2p-peerstore"
5
"math/rand"
6
"testing"
7
"time"
8
+
9
+ blocksutil "github.com/ipfs/go-ipfs-blocksutil"
10
+ cid "github.com/ipfs/go-cid"
11
+ datastore "github.com/ipfs/go-datastore"
12
+ pstore "github.com/libp2p/go-libp2p-peerstore"
13
+ sync "github.com/ipfs/go-datastore/sync"
14
)
15
16
var blockGenerator = blocksutil.NewBlockGenerator()
@@ -25,11 +27,10 @@ func mockContentRouting() *mockRouting {
27
28
func TestAnnouncement(t *testing.T) {
29
ctx := context.Background()
28
- defer func() {
29
- ctx.Done()
30
- }()
30
+ defer ctx.Done()
31
32
- queue, err := NewQueue(ctx, "test", datastore.NewMapDatastore())
32
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
33
+ queue, err := NewQueue(ctx, "test", ds)
34
if err != nil {
35
t.Fatal(err)
36
}
provider/queue.go
+37
-46
@@ -3,14 +3,15 @@ package provider
3
import (
4
"context"
5
"errors"
6
- "github.com/ipfs/go-cid"
7
- ds "github.com/ipfs/go-datastore"
8
- "github.com/ipfs/go-datastore/namespace"
9
- "github.com/ipfs/go-datastore/query"
6
"math"
7
"strconv"
8
"strings"
9
"sync"
10
+
11
+ cid "github.com/ipfs/go-cid"
12
+ datastore "github.com/ipfs/go-datastore"
13
+ namespace "github.com/ipfs/go-datastore/namespace"
14
+ query "github.com/ipfs/go-datastore/query"
15
)
16
17
// Entry allows for the durability in the queue. When a cid is dequeued it is
@@ -18,17 +19,10 @@ import (
19
// receive.
20
type Entry struct {
21
cid cid.Cid
21
- key ds.Key
22
+ key datastore.Key
23
queue *Queue
24
}
25
25
-// Complete the entry by removing it from the queue
26
-func (e *Entry) Complete() error {
27
- e.queue.lock.Lock()
28
- defer e.queue.lock.Unlock()
29
- return e.queue.remove(e.key)
30
-}
31
-
26
// Queue provides a durable, FIFO interface to the datastore for storing cids
27
//
28
// Durability just means that cids in the process of being provided when a
@@ -44,41 +38,41 @@ type Queue struct {
38
tail uint64
39
head uint64
40
47
- lock sync.Mutex
48
- datastore ds.Datastore
41
+ enqueueLock sync.Mutex
42
+ ds datastore.Datastore // Must be threadsafe
43
44
dequeue chan *Entry
45
added chan struct{}
46
}
47
48
// NewQueue creates a queue for cids
55
-func NewQueue(ctx context.Context, name string, datastore ds.Datastore) (*Queue, error) {
56
- namespaced := namespace.Wrap(datastore, ds.NewKey("/"+name+"/queue/"))
49
+func NewQueue(ctx context.Context, name string, ds datastore.Datastore) (*Queue, error) {
50
+ namespaced := namespace.Wrap(ds, datastore.NewKey("/"+name+"/queue/"))
51
head, tail, err := getQueueHeadTail(ctx, name, namespaced)
52
if err != nil {
53
return nil, err
54
}
55
q := &Queue{
62
- name: name,
63
- ctx: ctx,
64
- head: head,
65
- tail: tail,
66
- lock: sync.Mutex{},
67
- datastore: namespaced,
68
- dequeue: make(chan *Entry),
69
- added: make(chan struct{}),
56
+ name: name,
57
+ ctx: ctx,
58
+ head: head,
59
+ tail: tail,
60
+ enqueueLock: sync.Mutex{},
61
+ ds: namespaced,
62
+ dequeue: make(chan *Entry),
63
+ added: make(chan struct{}),
64
}
65
return q, nil
66
}
67
68
// Enqueue puts a cid in the queue
69
func (q *Queue) Enqueue(cid cid.Cid) error {
76
- q.lock.Lock()
77
- defer q.lock.Unlock()
70
+ q.enqueueLock.Lock()
71
+ defer q.enqueueLock.Unlock()
72
73
nextKey := q.queueKey(q.tail)
74
81
- if err := q.datastore.Put(nextKey, cid.Bytes()); err != nil {
75
+ if err := q.ds.Put(nextKey, cid.Bytes()); err != nil {
76
return err
77
}
78
@@ -126,8 +120,9 @@ func (q *Queue) Run() {
120
case <-q.ctx.Done():
121
return
122
case q.dequeue <- entry:
123
+ q.head++
124
+ err = q.ds.Delete(entry.key)
125
}
130
-
126
}
127
}()
128
}
@@ -135,12 +130,7 @@ func (q *Queue) Run() {
130
// Find the next item in the queue, crawl forward if an entry is not
131
// found in the next spot.
132
func (q *Queue) next() (*Entry, error) {
138
- q.lock.Lock()
139
- defer func() {
140
- q.lock.Unlock()
141
- }()
142
-
143
- var nextKey ds.Key
133
+ var key datastore.Key
134
var value []byte
135
var err error
136
for {
@@ -152,9 +142,12 @@ func (q *Queue) next() (*Entry, error) {
142
return nil, nil
143
default:
144
}
155
- nextKey = q.queueKey(q.head)
156
- value, err = q.datastore.Get(nextKey)
157
- if err == ds.ErrNotFound {
145
+ key = q.queueKey(q.head)
146
+
147
+ value, err = q.ds.Get(key)
148
+
149
+ value, err = q.ds.Get(key)
150
+ if err == datastore.ErrNotFound {
151
q.head++
152
continue
153
} else if err != nil {
@@ -171,21 +164,23 @@ func (q *Queue) next() (*Entry, error) {
164
165
entry := &Entry{
166
cid: id,
174
- key: nextKey,
167
+ key: key,
168
queue: q,
169
}
170
178
- q.head++
171
+ if err != nil {
172
+ return nil, err
173
+ }
174
175
return entry, nil
176
}
177
183
-func (q *Queue) queueKey(id uint64) ds.Key {
184
- return ds.NewKey(strconv.FormatUint(id, 10))
178
+func (q *Queue) queueKey(id uint64) datastore.Key {
179
+ return datastore.NewKey(strconv.FormatUint(id, 10))
180
}
181
182
// crawl over the queue entries to find the head and tail
188
-func getQueueHeadTail(ctx context.Context, name string, datastore ds.Datastore) (uint64, uint64, error) {
183
+func getQueueHeadTail(ctx context.Context, name string, datastore datastore.Datastore) (uint64, uint64, error) {
184
q := query.Query{}
185
results, err := datastore.Query(q)
186
if err != nil {
@@ -223,7 +218,3 @@ func getQueueHeadTail(ctx context.Context, name string, datastore ds.Datastore)
218
219
return head, tail, nil
220
}
226
-
227
-func (q *Queue) remove(key ds.Key) error {
228
- return q.datastore.Delete(key)
229
-}
provider/queue_test.go
new
+89
@@ -0,0 +1,89 @@
1
+package provider
2
+
3
+import (
4
+ "context"
5
+ "testing"
6
+ "time"
7
+
8
+ cid "github.com/ipfs/go-cid"
9
+ datastore "github.com/ipfs/go-datastore"
10
+ sync "github.com/ipfs/go-datastore/sync"
11
+)
12
+
13
+func makeCids(n int) []cid.Cid {
14
+ cids := make([]cid.Cid, 0, 10)
15
+ for i := 0; i < 10; i++ {
16
+ c := blockGenerator.Next().Cid()
17
+ cids = append(cids, c)
18
+ }
19
+ return cids
20
+}
21
+
22
+func assertOrdered(cids []cid.Cid, q *Queue, t *testing.T) {
23
+ for _, c := range cids {
24
+ select {
25
+ case dequeued := <- q.dequeue:
26
+ if c != dequeued.cid {
27
+ t.Fatalf("Error in ordering of CIDs retrieved from queue. Expected: %s, got: %s", c, dequeued.cid)
28
+ }
29
+
30
+ case <-time.After(time.Second * 1):
31
+ t.Fatal("Timeout waiting for cids to be provided.")
32
+ }
33
+ }
34
+}
35
+
36
+func TestBasicOperation(t *testing.T) {
37
+ ctx := context.Background()
38
+ defer ctx.Done()
39
+
40
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
41
+ queue, err := NewQueue(ctx, "test", ds)
42
+ if err != nil {
43
+ t.Fatal(err)
44
+ }
45
+ queue.Run()
46
+
47
+ cids := makeCids(10)
48
+
49
+ for _, c := range cids {
50
+ err = queue.Enqueue(c)
51
+ if err != nil {
52
+ t.Fatal("Failed to enqueue CID")
53
+ }
54
+ }
55
+
56
+ assertOrdered(cids, queue, t)
57
+}
58
+
59
+func TestInitialization(t *testing.T) {
60
+ ctx := context.Background()
61
+ defer ctx.Done()
62
+
63
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
64
+ queue, err := NewQueue(ctx, "test", ds)
65
+ if err != nil {
66
+ t.Fatal(err)
67
+ }
68
+ queue.Run()
69
+
70
+ cids := makeCids(10)
71
+
72
+ for _, c := range cids {
73
+ err = queue.Enqueue(c)
74
+ if err != nil {
75
+ t.Fatal("Failed to enqueue CID")
76
+ }
77
+ }
78
+
79
+ assertOrdered(cids[:5], queue, t)
80
+
81
+ // make a new queue, same data
82
+ queue, err = NewQueue(ctx, "test", ds)
83
+ if err != nil {
84
+ t.Fatal(err)
85
+ }
86
+ queue.Run()
87
+
88
+ assertOrdered(cids[5:], queue, t)
89
+}