Provide root node immediately when add and pin add
License: MIT Signed-off-by: Michael Avila <davidmichaelavila@gmail.com>
Michael Avila committed
Mar 8, 2019 at 14:19 UTC
a9b65346871e845a67bbd19c668b699c55095e41
8 files changed
+364
core/builder.go
+8
@@ -5,6 +5,7 @@ import (
5
"crypto/rand"
6
"encoding/base64"
7
"errors"
8
+ "github.com/ipfs/go-ipfs/provider"
9
"os"
10
"syscall"
11
"time"
@@ -275,6 +276,13 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
276
}
277
n.Resolver = resolver.NewBasicResolver(n.DAG)
278
279
+ // Provider
280
+ queue, err := provider.NewQueue("provider-v1", ctx, n.Repo.Datastore())
281
+ if err != nil {
282
+ return err
283
+ }
284
+ n.Provider = provider.NewProvider(ctx, queue, n.Routing)
285
+
286
if cfg.Online {
287
if err := n.startLateOnlineServices(ctx); err != nil {
288
return err
core/core.go
+8
@@ -14,6 +14,7 @@ import (
14
"context"
15
"errors"
16
"fmt"
17
+ "github.com/ipfs/go-ipfs/provider"
18
"io"
19
"io/ioutil"
20
"os"
@@ -124,6 +125,7 @@ type IpfsNode struct {
125
Routing routing.IpfsRouting // the routing system. recommend ipfs-dht
126
Exchange exchange.Interface // the block exchange + strategy (bitswap)
127
Namesys namesys.NameSystem // the name system, resolves paths to hashes
128
+ Provider *provider.Provider // the value provider system
129
Reprovider *rp.Reprovider // the value reprovider system
130
IpnsRepub *ipnsrp.Republisher
131
@@ -324,6 +326,12 @@ func (n *IpfsNode) startLateOnlineServices(ctx context.Context) error {
326
return err
327
}
328
329
+ // Provider
330
+
331
+ n.Provider.Run()
332
+
333
+ // Reprovider
334
+
335
var keyProvider rp.KeyChanFunc
336
337
switch cfg.Reprovider.Strategy {
core/coreapi/coreapi.go
+5
@@ -20,6 +20,7 @@ import (
20
21
"github.com/ipfs/go-ipfs/core"
22
"github.com/ipfs/go-ipfs/namesys"
23
+ "github.com/ipfs/go-ipfs/provider"
24
"github.com/ipfs/go-ipfs/pin"
25
"github.com/ipfs/go-ipfs/repo"
26
@@ -66,6 +67,8 @@ type CoreAPI struct {
67
namesys namesys.NameSystem
68
routing routing.IpfsRouting
69
70
+ provider *provider.Provider
71
+
72
pubSub *pubsub.PubSub
73
74
checkPublishAllowed func() error
@@ -174,6 +177,8 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
177
exchange: n.Exchange,
178
routing: n.Routing,
179
180
+ provider: n.Provider,
181
+
182
pubSub: n.PubSub,
183
184
nd: n,
core/coreapi/pin.go
+4
@@ -32,6 +32,10 @@ func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.Pin
32
return fmt.Errorf("pin: %s", err)
33
}
34
35
+ if err := api.provider.Provide(dagNode.Cid()); err != nil {
36
+ return err
37
+ }
38
+
39
return api.pinning.Flush()
40
}
41
core/coreapi/provider.go
new
+11
@@ -0,0 +1,11 @@
1
+package coreapi
2
+
3
+import (
4
+ cid "github.com/ipfs/go-cid"
5
+)
6
+
7
+type ProviderAPI CoreAPI
8
+
9
+func (api *ProviderAPI) Provide(root cid.Cid) error {
10
+ return api.provider.Provide(root)
11
+}
core/coreapi/unixfs.go
+5
@@ -129,6 +129,11 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
129
if err != nil {
130
return nil, err
131
}
132
+
133
+ if err := api.provider.Provide(nd.Cid()); err != nil {
134
+ return nil, err
135
+ }
136
+
137
return coreiface.IpfsPath(nd.Cid()), nil
138
}
139
provider/provider.go
new
+88
@@ -0,0 +1,88 @@
1
+// Package provider implements structures and methods to provide blocks,
2
+// keep track of which blocks are provided, and to allow those blocks to
3
+// be reprovided.
4
+package provider
5
+
6
+import (
7
+ "context"
8
+ "github.com/ipfs/go-cid"
9
+ logging "github.com/ipfs/go-log"
10
+ "github.com/libp2p/go-libp2p-routing"
11
+ "time"
12
+)
13
+
14
+var (
15
+ log = logging.Logger("provider")
16
+)
17
+
18
+const (
19
+ provideOutgoingWorkerLimit = 8
20
+ provideOutgoingTimeout = 15 * time.Second
21
+)
22
+
23
+// Provider announces blocks to the network, tracks which blocks are
24
+// being provided, and untracks blocks when they're no longer in the blockstore.
25
+type Provider struct {
26
+ ctx context.Context
27
+ // the CIDs for which provide announcements should be made
28
+ queue *Queue
29
+ // used to announce providing to the network
30
+ contentRouting routing.ContentRouting
31
+}
32
+
33
+func NewProvider(ctx context.Context, queue *Queue, contentRouting routing.ContentRouting) *Provider {
34
+ return &Provider{
35
+ ctx: ctx,
36
+ queue: queue,
37
+ contentRouting: contentRouting,
38
+ }
39
+}
40
+
41
+// Start workers to handle provide requests.
42
+func (p *Provider) Run() {
43
+ p.queue.Run()
44
+ p.handleAnnouncements()
45
+}
46
+
47
+// Provide the given cid using specified strategy.
48
+func (p *Provider) Provide(root cid.Cid) error {
49
+ return p.queue.Enqueue(root)
50
+}
51
+
52
+// Handle all outgoing cids by providing (announcing) them
53
+func (p *Provider) handleAnnouncements() {
54
+ for workers := 0; workers < provideOutgoingWorkerLimit; workers++ {
55
+ go func() {
56
+ for {
57
+ select {
58
+ case <-p.ctx.Done():
59
+ return
60
+ case entry := <-p.queue.Dequeue():
61
+ if err := doProvide(p.ctx, p.contentRouting, entry.cid); err != nil {
62
+ log.Warningf("Unable to provide entry: %s, %s", entry.cid, err)
63
+ }
64
+
65
+ if err := entry.Complete(); err != nil {
66
+ log.Warningf("Unable to complete queue entry when providing: %s, %s", entry.cid, err)
67
+ }
68
+ }
69
+ }
70
+ }()
71
+ }
72
+}
73
+
74
+// TODO: better document this provide logic
75
+func doProvide(ctx context.Context, contentRouting routing.ContentRouting, key cid.Cid) error {
76
+ // announce
77
+ log.Info("announce - start - ", key)
78
+ ctx, cancel := context.WithTimeout(ctx, provideOutgoingTimeout)
79
+ if err := contentRouting.Provide(ctx, key, true); err != nil {
80
+ log.Warningf("Failed to provide cid: %s", err)
81
+ // TODO: Maybe put these failures onto a failures queue?
82
+ cancel()
83
+ return err
84
+ }
85
+ cancel()
86
+ log.Info("announce - end - ", key)
87
+ return nil
88
+}
provider/queue.go
new
+235
@@ -0,0 +1,235 @@
1
+package provider
2
+
3
+import (
4
+ "context"
5
+ "errors"
6
+ "github.com/ipfs/go-cid"
7
+ ds "github.com/ipfs/go-datastore"
8
+ "github.com/ipfs/go-datastore/namespace"
9
+ "github.com/ipfs/go-datastore/query"
10
+ "math"
11
+ "strconv"
12
+ "strings"
13
+ "sync"
14
+)
15
+
16
+// Entry allows for the durability in the queue. When a cid is dequeued it is
17
+// not removed from the datastore until you call Complete() on the entry you
18
+// receive.
19
+type Entry struct {
20
+ cid cid.Cid
21
+ key ds.Key
22
+ queue *Queue
23
+}
24
+
25
+func (e *Entry) Complete() error {
26
+ return e.queue.remove(e.key)
27
+}
28
+
29
+// Queue provides a durable, FIFO interface to the datastore for storing cids
30
+//
31
+// Durability just means that cids in the process of being provided when a
32
+// crash or shutdown occurs will still be in the queue when the node is
33
+// brought back online.
34
+type Queue struct {
35
+ // used to differentiate queues in datastore
36
+ // e.g. provider vs reprovider
37
+ name string
38
+
39
+ ctx context.Context
40
+
41
+ tail uint64
42
+ head uint64
43
+
44
+ lock sync.Mutex
45
+ datastore ds.Datastore
46
+
47
+ dequeue chan *Entry
48
+ notEmpty chan struct{}
49
+
50
+ isRunning bool
51
+}
52
+
53
+func NewQueue(name string, ctx context.Context, datastore ds.Datastore) (*Queue, error) {
54
+ namespaced := namespace.Wrap(datastore, ds.NewKey("/" + name + "/queue/"))
55
+ head, tail, err := getQueueHeadTail(name, ctx, namespaced)
56
+ if err != nil {
57
+ return nil, err
58
+ }
59
+ q := &Queue{
60
+ name: name,
61
+ ctx: ctx,
62
+ head: head,
63
+ tail: tail,
64
+ lock: sync.Mutex{},
65
+ datastore: namespaced,
66
+ dequeue: make(chan *Entry),
67
+ notEmpty: make(chan struct{}),
68
+ isRunning: false,
69
+ }
70
+ return q, nil
71
+}
72
+
73
+// Put a cid in the queue
74
+func (q *Queue) Enqueue(cid cid.Cid) error {
75
+ q.lock.Lock()
76
+ defer q.lock.Unlock()
77
+
78
+ wasEmpty := q.IsEmpty()
79
+
80
+ nextKey := q.queueKey(q.tail)
81
+
82
+ if err := q.datastore.Put(nextKey, cid.Bytes()); err != nil {
83
+ return err
84
+ }
85
+
86
+ q.tail++
87
+
88
+ if q.isRunning && wasEmpty {
89
+ select {
90
+ case q.notEmpty <- struct{}{}:
91
+ case <-q.ctx.Done():
92
+ }
93
+ }
94
+
95
+ return nil
96
+}
97
+
98
+// Remove an entry from the queue.
99
+func (q *Queue) Dequeue() <-chan *Entry {
100
+ return q.dequeue
101
+}
102
+
103
+func (q *Queue) IsEmpty() bool {
104
+ return (q.tail - q.head) == 0
105
+}
106
+
107
+func (q *Queue) remove(key ds.Key) error {
108
+ return q.datastore.Delete(key)
109
+}
110
+
111
+// dequeue items when the dequeue channel is available to
112
+// be written to
113
+func (q *Queue) Run() {
114
+ q.isRunning = true
115
+ go func() {
116
+ for {
117
+ select {
118
+ case <-q.ctx.Done():
119
+ return
120
+ default:
121
+ }
122
+ if q.IsEmpty() {
123
+ select {
124
+ case <-q.ctx.Done():
125
+ return
126
+ // wait for a notEmpty message
127
+ case <-q.notEmpty:
128
+ }
129
+ }
130
+
131
+ entry, err := q.next()
132
+ if err != nil {
133
+ log.Warningf("Error Dequeue()-ing: %s, %s", entry, err)
134
+ continue
135
+ }
136
+
137
+ select {
138
+ case <-q.ctx.Done():
139
+ return
140
+ case q.dequeue <- entry:
141
+ }
142
+ }
143
+ }()
144
+}
145
+
146
+// Find the next item in the queue, crawl forward if an entry is not
147
+// found in the next spot.
148
+func (q *Queue) next() (*Entry, error) {
149
+ q.lock.Lock()
150
+ defer q.lock.Unlock()
151
+
152
+ var nextKey ds.Key
153
+ var value []byte
154
+ var err error
155
+ for {
156
+ if q.head >= q.tail {
157
+ return nil, errors.New("no more entries in queue")
158
+ }
159
+ select {
160
+ case <-q.ctx.Done():
161
+ return nil, nil
162
+ default:
163
+ }
164
+ nextKey = q.queueKey(q.head)
165
+ value, err = q.datastore.Get(nextKey)
166
+ if err == ds.ErrNotFound {
167
+ q.head++
168
+ continue
169
+ } else if err != nil {
170
+ return nil, err
171
+ } else {
172
+ break
173
+ }
174
+ }
175
+
176
+ id, err := cid.Parse(value)
177
+ if err != nil {
178
+ return nil, err
179
+ }
180
+
181
+ entry := &Entry {
182
+ cid: id,
183
+ key: nextKey,
184
+ queue: q,
185
+ }
186
+
187
+ q.head++
188
+
189
+ return entry, nil
190
+}
191
+
192
+func (q *Queue) queueKey(id uint64) ds.Key {
193
+ return ds.NewKey(strconv.FormatUint(id, 10))
194
+}
195
+
196
+// crawl over the queue entries to find the head and tail
197
+func getQueueHeadTail(name string, ctx context.Context, datastore ds.Datastore) (uint64, uint64, error) {
198
+ query := query.Query{}
199
+ results, err := datastore.Query(query)
200
+ if err != nil {
201
+ return 0, 0, err
202
+ }
203
+
204
+ var tail uint64 = 0
205
+ var head uint64 = math.MaxUint64
206
+ for entry := range results.Next() {
207
+ select {
208
+ case <-ctx.Done():
209
+ return 0, 0, nil
210
+ default:
211
+ }
212
+ trimmed := strings.TrimPrefix(entry.Key, "/")
213
+ id, err := strconv.ParseUint(trimmed, 10, 64)
214
+ if err != nil {
215
+ return 0, 0, err
216
+ }
217
+
218
+ if id < head {
219
+ head = id
220
+ }
221
+
222
+ if (id+1) > tail {
223
+ tail = (id+1)
224
+ }
225
+ }
226
+ if err := results.Close(); err != nil {
227
+ return 0, 0, err
228
+ }
229
+ if head == math.MaxUint64 {
230
+ head = 0
231
+ }
232
+
233
+ return head, tail, nil
234
+}
235
+