Reprovider strategies
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Aug 1, 2017 at 02:57 UTC
f20683eb531e8e798b8f5f6d09067567a6e58fe2
7 files changed
+120
-25
core/builder.go
+6
@@ -231,5 +231,11 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
231
}
232
n.Resolver = path.NewBasicResolver(n.DAG)
233
234
+ if cfg.Online {
235
+ if err := n.startLateOnlineServices(ctx); err != nil {
236
+ return err
237
+ }
238
+ }
239
+
240
return n.loadFilesRoot()
241
}
core/core.go
+39
-16
@@ -237,22 +237,6 @@ func (n *IpfsNode) startOnlineServices(ctx context.Context, routingOption Routin
237
return err
238
}
239
240
- n.Reprovider = rp.NewReprovider(n.Routing, n.Blockstore)
241
-
242
- if cfg.Reprovider.Interval != "0" {
243
- interval := kReprovideFrequency
244
- if cfg.Reprovider.Interval != "" {
245
- dur, err := time.ParseDuration(cfg.Reprovider.Interval)
246
- if err != nil {
247
- return err
248
- }
249
-
250
- interval = dur
251
- }
252
-
253
- go n.Reprovider.ProvideEvery(ctx, interval)
254
- }
255
-
240
if pubsub {
241
n.Floodsub = floodsub.NewFloodSub(ctx, peerhost)
242
}
@@ -273,6 +257,45 @@ func (n *IpfsNode) startOnlineServices(ctx context.Context, routingOption Routin
257
return n.Bootstrap(DefaultBootstrapConfig)
258
}
259
260
+func (n *IpfsNode) startLateOnlineServices(ctx context.Context) error {
261
+ cfg, err := n.Repo.Config()
262
+ if err != nil {
263
+ return err
264
+ }
265
+
266
+ var keyProvider func(context.Context) (<-chan *cid.Cid, error)
267
+
268
+ switch cfg.Reprovider.Strategy {
269
+ case "all":
270
+ fallthrough
271
+ case "":
272
+ keyProvider = rp.NewBlockstoreProvider(n.Blockstore)
273
+ case "roots":
274
+ keyProvider = rp.NewPinnedProvider(n.Pinning, n.DAG, true)
275
+ case "pinned":
276
+ keyProvider = rp.NewPinnedProvider(n.Pinning, n.DAG, false)
277
+ default:
278
+ return fmt.Errorf("unknown reprovider strtaegy '%s'", cfg.Reprovider.Strategy)
279
+ }
280
+ n.Reprovider = rp.NewReprovider(n.Routing, keyProvider)
281
+
282
+ if cfg.Reprovider.Interval != "0" {
283
+ interval := kReprovideFrequency
284
+ if cfg.Reprovider.Interval != "" {
285
+ dur, err := time.ParseDuration(cfg.Reprovider.Interval)
286
+ if err != nil {
287
+ return err
288
+ }
289
+
290
+ interval = dur
291
+ }
292
+
293
+ go n.Reprovider.ProvideEvery(ctx, interval)
294
+ }
295
+
296
+ return nil
297
+}
298
+
299
func makeAddrsFactory(cfg config.Addresses) (p2pbhost.AddrsFactory, error) {
300
var annAddrs []ma.Multiaddr
301
for _, addr := range cfg.Announce {
exchange/reprovide/providers.go
new
+62
@@ -0,0 +1,62 @@
1
+package reprovide
2
+
3
+import (
4
+ "context"
5
+ "errors"
6
+ "fmt"
7
+
8
+ blocks "github.com/ipfs/go-ipfs/blocks/blockstore"
9
+ merkledag "github.com/ipfs/go-ipfs/merkledag"
10
+ pin "github.com/ipfs/go-ipfs/pin"
11
+
12
+ cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
13
+)
14
+
15
+func NewBlockstoreProvider(bstore blocks.Blockstore) KeyChanFunc {
16
+ return func(ctx context.Context) (<-chan *cid.Cid, error) {
17
+ return bstore.AllKeysChan(ctx)
18
+ }
19
+}
20
+
21
+func NewPinnedProvider(pinning pin.Pinner, dag merkledag.DAGService, onlyRoots bool) KeyChanFunc {
22
+ return func(ctx context.Context) (<-chan *cid.Cid, error) {
23
+ set, err := pinSet(ctx, pinning, dag, onlyRoots)
24
+ if err != nil {
25
+ return nil, err
26
+ }
27
+
28
+ outCh := make(chan *cid.Cid)
29
+ go func() {
30
+ set.ForEach(func(c *cid.Cid) error {
31
+ select {
32
+ case <-ctx.Done():
33
+ return errors.New("context cancelled")
34
+ case outCh <- c:
35
+ }
36
+ return nil
37
+ })
38
+ }()
39
+
40
+ return outCh, nil
41
+ }
42
+}
43
+
44
+func pinSet(ctx context.Context, pinning pin.Pinner, dag merkledag.DAGService, onlyRoots bool) (*cid.Set, error) {
45
+ set := cid.NewSet()
46
+ for _, key := range pinning.DirectKeys() {
47
+ set.Add(key)
48
+ }
49
+
50
+ for _, key := range pinning.RecursiveKeys() {
51
+ set.Add(key)
52
+
53
+ if !onlyRoots {
54
+ err := merkledag.EnumerateChildren(ctx, dag.GetLinks, key, set.Visit)
55
+ if err != nil {
56
+ return nil, err
57
+ }
58
+ }
59
+ }
60
+
61
+ return set, nil
62
+}
exchange/reprovide/reprovide.go
+9
-8
@@ -5,26 +5,27 @@ import (
5
"fmt"
6
"time"
7
8
- blocks "github.com/ipfs/go-ipfs/blocks/blockstore"
8
backoff "gx/ipfs/QmPJUtEJsm5YLUWhF6imvyCH8KZXRJa9Wup7FDMwTy5Ufz/backoff"
9
routing "gx/ipfs/QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1/go-libp2p-routing"
10
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
11
+ cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
12
)
13
14
var log = logging.Logger("reprovider")
15
16
+type KeyChanFunc func(context.Context) (<-chan *cid.Cid, error)
17
+
18
type Reprovider struct {
19
// The routing system to provide values through
20
rsys routing.ContentRouting
21
20
- // The backing store for blocks to be provided
21
- bstore blocks.Blockstore
22
+ keyProvider KeyChanFunc
23
}
24
24
-func NewReprovider(rsys routing.ContentRouting, bstore blocks.Blockstore) *Reprovider {
25
+func NewReprovider(rsys routing.ContentRouting, keyProvider KeyChanFunc) *Reprovider {
26
return &Reprovider{
26
- rsys: rsys,
27
- bstore: bstore,
27
+ rsys: rsys,
28
+ keyProvider: keyProvider,
29
}
30
}
31
@@ -48,9 +49,9 @@ func (rp *Reprovider) ProvideEvery(ctx context.Context, tick time.Duration) {
49
}
50
51
func (rp *Reprovider) Reprovide(ctx context.Context) error {
51
- keychan, err := rp.bstore.AllKeysChan(ctx)
52
+ keychan, err := rp.keyProvider(ctx)
53
if err != nil {
53
- return fmt.Errorf("Failed to get key chan from blockstore: %s", err)
54
+ return fmt.Errorf("Failed to get key chan: %s", err)
55
}
56
for c := range keychan {
57
op := func() error {
exchange/reprovide/reprovide_test.go
+2
-1
@@ -32,7 +32,8 @@ func TestReprovide(t *testing.T) {
32
blk := blocks.NewBlock([]byte("this is a test"))
33
bstore.Put(blk)
34
35
- reprov := NewReprovider(clA, bstore)
35
+ keyProvider := NewBlockstoreProvider(bstore)
36
+ reprov := NewReprovider(clA, keyProvider)
37
err := reprov.Reprovide(ctx)
38
if err != nil {
39
t.Fatal(err)
repo/config/init.go
+1
@@ -72,6 +72,7 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
72
},
73
Reprovider: Reprovider{
74
Interval: "12h",
75
+ Strategy: "all",
76
},
77
}
78
repo/config/reprovider.go
+1
@@ -2,4 +2,5 @@ package config
2
3
type Reprovider struct {
4
Interval string // Time period to reprovide locally stored objects to the network
5
+ Strategy string // Which keys to announce
6
}