added support for an experimental batched provider system
The batched provider system is enabled when the experimental AcceleratedDHTClient is enabled There is also an `ipfs stats provide` command which gives stats about the providing/reproviding system when the batched provider system is enabled
Adin Schmahmann committed
May 12, 2021 at 23:56 UTC
cf45f1a8621b965eb89bebaca5574bb8cb53ca43
7 files changed
+114
-9
core/commands/commands_test.go
+1
@@ -210,6 +210,7 @@ func TestCommands(t *testing.T) {
210
"/stats/bitswap",
211
"/stats/bw",
212
"/stats/dht",
213
+ "/stats/provide",
214
"/stats/repo",
215
"/swarm",
216
"/swarm/addrs",
core/commands/stat.go
+1
@@ -30,6 +30,7 @@ for your IPFS node.`,
30
"repo": repoStatCmd,
31
"bitswap": bitswapStatCmd,
32
"dht": statDhtCmd,
33
+ "provide": statProvideCmd,
34
},
35
}
36
core/commands/stat_provide.go
new
+51
@@ -0,0 +1,51 @@
1
+package commands
2
+
3
+import (
4
+ "fmt"
5
+
6
+ cmds "github.com/ipfs/go-ipfs-cmds"
7
+ "github.com/ipfs/go-ipfs/core/commands/cmdenv"
8
+
9
+ "github.com/ipfs/go-ipfs-provider/batched"
10
+)
11
+
12
+var statProvideCmd = &cmds.Command{
13
+ Helptext: cmds.HelpText{
14
+ Tagline: "Returns statistics about the node's (re)provider system.",
15
+ ShortDescription: `
16
+Returns statistics about the content the node is advertising.
17
+
18
+This interface is not stable and may change from release to release.
19
+`,
20
+ },
21
+ Arguments: []cmds.Argument{},
22
+ Options: []cmds.Option{},
23
+ Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
24
+ nd, err := cmdenv.GetNode(env)
25
+ if err != nil {
26
+ return err
27
+ }
28
+
29
+ if !nd.IsOnline {
30
+ return ErrNotOnline
31
+ }
32
+
33
+ sys, ok := nd.Provider.(*batched.BatchProvidingSystem)
34
+ if !ok {
35
+ return fmt.Errorf("can only return stats if Experimental.AcceleratedDHTClient is enabled")
36
+ }
37
+
38
+ stats, err := sys.Stat(req.Context)
39
+ if err != nil {
40
+ return err
41
+ }
42
+
43
+ if err := res.Emit(stats); err != nil {
44
+ return err
45
+ }
46
+
47
+ return nil
48
+ },
49
+ Encoders: cmds.EncoderMap{},
50
+ Type: batched.BatchedProviderStats{},
51
+}
core/node/groups.go
+2
-2
@@ -275,7 +275,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config) fx.Option {
275
fx.Provide(p2p.New),
276
277
LibP2P(bcfg, cfg),
278
- OnlineProviders(cfg.Experimental.StrategicProviding, cfg.Reprovider.Strategy, cfg.Reprovider.Interval),
278
+ OnlineProviders(cfg.Experimental.StrategicProviding, cfg.Experimental.AcceleratedDHTClient, cfg.Reprovider.Strategy, cfg.Reprovider.Interval),
279
)
280
}
281
@@ -286,7 +286,7 @@ func Offline(cfg *config.Config) fx.Option {
286
fx.Provide(DNSResolver),
287
fx.Provide(Namesys(0)),
288
fx.Provide(offroute.NewOfflineRouter),
289
- OfflineProviders(cfg.Experimental.StrategicProviding, cfg.Reprovider.Strategy, cfg.Reprovider.Interval),
289
+ OfflineProviders(cfg.Experimental.StrategicProviding, cfg.Experimental.AcceleratedDHTClient, cfg.Reprovider.Strategy, cfg.Reprovider.Interval),
290
)
291
}
292
core/node/provider.go
+56
-4
@@ -7,13 +7,16 @@ import (
7
8
"github.com/ipfs/go-ipfs-pinner"
9
"github.com/ipfs/go-ipfs-provider"
10
+ "github.com/ipfs/go-ipfs-provider/batched"
11
q "github.com/ipfs/go-ipfs-provider/queue"
12
"github.com/ipfs/go-ipfs-provider/simple"
13
ipld "github.com/ipfs/go-ipld-format"
14
"github.com/libp2p/go-libp2p-core/routing"
15
+ "github.com/multiformats/go-multihash"
16
"go.uber.org/fx"
17
18
"github.com/ipfs/go-ipfs/core/node/helpers"
19
+ "github.com/ipfs/go-ipfs/core/node/libp2p"
20
"github.com/ipfs/go-ipfs/repo"
21
)
22
@@ -59,29 +62,78 @@ func SimpleProviderSys(isOnline bool) interface{} {
62
}
63
}
64
65
+type provideMany interface {
66
+ ProvideMany(ctx context.Context, keys []multihash.Multihash) error
67
+ Ready() bool
68
+}
69
+
70
+// BatchedProviderSys creates new provider system
71
+func BatchedProviderSys(isOnline bool, reprovideInterval string) interface{} {
72
+ return func(lc fx.Lifecycle, cr libp2p.BaseIpfsRouting, q *q.Queue, keyProvider simple.KeyChanFunc, repo repo.Repo) (provider.System, error) {
73
+ r, ok := (cr).(provideMany)
74
+ if !ok {
75
+ return nil, fmt.Errorf("BatchedProviderSys requires a content router that supports provideMany")
76
+ }
77
+
78
+ reprovideIntervalDuration := kReprovideFrequency
79
+ if reprovideInterval != "" {
80
+ dur, err := time.ParseDuration(reprovideInterval)
81
+ if err != nil {
82
+ return nil, err
83
+ }
84
+
85
+ reprovideIntervalDuration = dur
86
+ }
87
+
88
+ sys, err := batched.New(r, q,
89
+ batched.ReproviderInterval(reprovideIntervalDuration),
90
+ batched.Datastore(repo.Datastore()),
91
+ batched.KeyProvider(keyProvider))
92
+ if err != nil {
93
+ return nil, err
94
+ }
95
+
96
+ if isOnline {
97
+ lc.Append(fx.Hook{
98
+ OnStart: func(ctx context.Context) error {
99
+ sys.Run()
100
+ return nil
101
+ },
102
+ OnStop: func(ctx context.Context) error {
103
+ return sys.Close()
104
+ },
105
+ })
106
+ }
107
+
108
+ return sys, nil
109
+ }
110
+}
111
+
112
// ONLINE/OFFLINE
113
114
// OnlineProviders groups units managing provider routing records online
65
-func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval string) fx.Option {
115
+func OnlineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval string) fx.Option {
116
if useStrategicProviding {
117
return fx.Provide(provider.NewOfflineProvider)
118
}
119
120
return fx.Options(
121
SimpleProviders(reprovideStrategy, reprovideInterval),
72
- fx.Provide(SimpleProviderSys(true)),
122
+ maybeProvide(SimpleProviderSys(true), !useBatchedProviding),
123
+ maybeProvide(BatchedProviderSys(true, reprovideInterval), useBatchedProviding),
124
)
125
}
126
127
// OfflineProviders groups units managing provider routing records offline
77
-func OfflineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval string) fx.Option {
128
+func OfflineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval string) fx.Option {
129
if useStrategicProviding {
130
return fx.Provide(provider.NewOfflineProvider)
131
}
132
133
return fx.Options(
134
SimpleProviders(reprovideStrategy, reprovideInterval),
84
- fx.Provide(SimpleProviderSys(false)),
135
+ maybeProvide(SimpleProviderSys(false), true),
136
+ //maybeProvide(BatchedProviderSys(false, reprovideInterval), useBatchedProviding),
137
)
138
}
139
go.mod
+1
-1
@@ -36,7 +36,7 @@ require (
36
github.com/ipfs/go-ipfs-keystore v0.0.2
37
github.com/ipfs/go-ipfs-pinner v0.1.1
38
github.com/ipfs/go-ipfs-posinfo v0.0.1
39
- github.com/ipfs/go-ipfs-provider v0.4.3
39
+ github.com/ipfs/go-ipfs-provider v0.4.4-0.20210513014626-1c19caa05024
40
github.com/ipfs/go-ipfs-routing v0.1.0
41
github.com/ipfs/go-ipfs-util v0.0.2
42
github.com/ipfs/go-ipld-cbor v0.0.5
go.sum
+2
-2
@@ -436,8 +436,8 @@ github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqt
436
github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
437
github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY=
438
github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
439
-github.com/ipfs/go-ipfs-provider v0.4.3 h1:k54OHXZcFBkhL6l3GnPS9PfpaLeLqZjVASG1bgfBdfQ=
440
-github.com/ipfs/go-ipfs-provider v0.4.3/go.mod h1:rcQBVqfblDQRk5LaCtf2uxuKxMJxvKmF5pLS0pO4au4=
439
+github.com/ipfs/go-ipfs-provider v0.4.4-0.20210513014626-1c19caa05024 h1:eYfdZ27ogtwfnwKdfphOwcQ7PEOjKqXlWzVOakK0a60=
440
+github.com/ipfs/go-ipfs-provider v0.4.4-0.20210513014626-1c19caa05024/go.mod h1:kUMTf1R8c+KgWUWKTGSZiXCDZWMCkxCX3wyepk0cYEA=
441
github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs=
442
github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ=
443
github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=