@cryptotaxi247 / kubo / commits / a5997375d

feat: `Provider.WorkerCount` and `stats reprovide` (#10779)

* adjust ipfs stats provide * update boxo dep * bump boxo * fixing tests * docs/chore: mark stat reprovide as experimental * docs: Provider.Strategy explicitly document it is not used - without this legacy users will have it in their config and be very confused --------- Co-authored-by: Marcin Rataj <lidel@lidel.org>

Guillaume Michel committed Apr 30, 2025 at 13:32 UTC a5997375dbaaceb50d7a7ecb681ef029197c485d
14 files changed +213 -60
cmd/ipfs/kubo/daemon.go
+3
@@ -485,6 +485,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
485 // This should never happen, but better safe than sorry
486 log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)")
487 }
488 + if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() {
489 + log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy")
490 + }
491
492 printLibp2pPorts(node)
493
config/config_test.go
+2 -2
@@ -134,9 +134,9 @@ func TestCheckKey(t *testing.T) {
134 t.Fatal("Foo.Bar isn't a valid key in the config")
135 }
136
137 - err = CheckKey("Provider.Strategy")
137 + err = CheckKey("Reprovider.Strategy")
138 if err != nil {
139 - t.Fatalf("%s: %s", err, "Provider.Strategy is a valid key in the config")
139 + t.Fatalf("%s: %s", err, "Reprovider.Strategy is a valid key in the config")
140 }
141
142 err = CheckKey("Provider.Foo")
config/provider.go
+8 -1
@@ -1,5 +1,12 @@
1 package config
2
3 +const (
4 + DefaultProviderWorkerCount = 64
5 +)
6 +
7 +// Provider configuration describes how NEW CIDs are announced the moment they are created.
8 +// For periodical reprovide configuration, see Reprovider.*
9 type Provider struct {
4 - Strategy string // Which keys to announce
10 + Strategy *OptionalString `json:",omitempty"` // Unused, you are likely looking for Reprovider.Strategy instead
11 + WorkerCount *OptionalInteger `json:",omitempty"` // Number of concurrent provides allowed, 0 means unlimited
12 }
config/reprovider.go
+2
@@ -7,6 +7,8 @@ const (
7 DefaultReproviderStrategy = "all"
8 )
9
10 +// Reprovider configuration describes how CID from local datastore are periodically re-announced to routing systems.
11 +// For provide behavior of ad-hoc or newly created CIDs and their first-time announcement, see Provider.*
12 type Reprovider struct {
13 Interval *OptionalDuration `json:",omitempty"` // Time period to reprovide locally stored objects to the network
14 Strategy *OptionalString `json:",omitempty"` // Which keys to announce
core/commands/commands_test.go
+1
@@ -184,6 +184,7 @@ func TestCommands(t *testing.T) {
184 "/stats/bw",
185 "/stats/dht",
186 "/stats/provide",
187 + "/stats/reprovide",
188 "/stats/repo",
189 "/swarm",
190 "/swarm/addrs",
core/commands/stat.go
+6 -5
@@ -27,11 +27,12 @@ for your IPFS node.`,
27 },
28
29 Subcommands: map[string]*cmds.Command{
30 - "bw": statBwCmd,
31 - "repo": repoStatCmd,
32 - "bitswap": bitswapStatCmd,
33 - "dht": statDhtCmd,
34 - "provide": statProvideCmd,
30 + "bw": statBwCmd,
31 + "repo": repoStatCmd,
32 + "bitswap": bitswapStatCmd,
33 + "dht": statDhtCmd,
34 + "provide": statProvideCmd,
35 + "reprovide": statReprovideCmd,
36 },
37 }
38
core/commands/stat_provide.go
+7 -42
@@ -4,28 +4,20 @@ import (
4 "fmt"
5 "io"
6 "text/tabwriter"
7 - "time"
7
9 - humanize "github.com/dustin/go-humanize"
10 - "github.com/ipfs/boxo/provider"
8 cmds "github.com/ipfs/go-ipfs-cmds"
9 "github.com/ipfs/kubo/core/commands/cmdenv"
10 "github.com/libp2p/go-libp2p-kad-dht/fullrt"
14 - "golang.org/x/exp/constraints"
11 )
12
17 -type reprovideStats struct {
18 - provider.ReproviderStats
19 - fullRT bool
20 -}
21 -
13 var statProvideCmd = &cmds.Command{
14 + Status: cmds.Deprecated,
15 Helptext: cmds.HelpText{
24 - Tagline: "Returns statistics about the node's (re)provider system.",
16 + Tagline: "Deprecated command, use 'ipfs stats reprovide' instead.",
17 ShortDescription: `
26 -Returns statistics about the content the node is advertising.
27 -
28 -This interface is not stable and may change from release to release.
18 +'ipfs stats provide' is deprecated because provide and reprovide operations
19 +are now distinct. This command may be replaced by provide only stats in the
20 +future.
21 `,
22 },
23 Arguments: []cmds.Argument{},
@@ -57,8 +49,8 @@ This interface is not stable and may change from release to release.
49 wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0)
50 defer wtr.Flush()
51
60 - fmt.Fprintf(wtr, "TotalReprovides:\t%s\n", humanNumber(s.TotalReprovides))
61 - fmt.Fprintf(wtr, "AvgReprovideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration))
52 + fmt.Fprintf(wtr, "TotalProvides:\t%s\n", humanNumber(s.TotalReprovides))
53 + fmt.Fprintf(wtr, "AvgProvideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration))
54 fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.LastReprovideDuration))
55 if !s.LastRun.IsZero() {
56 fmt.Fprintf(wtr, "LastRun:\t%s\n", humanTime(s.LastRun))
@@ -71,30 +63,3 @@ This interface is not stable and may change from release to release.
63 },
64 Type: reprovideStats{},
65 }
74 -
75 -func humanDuration(val time.Duration) string {
76 - return val.Truncate(time.Microsecond).String()
77 -}
78 -
79 -func humanTime(val time.Time) string {
80 - return val.Format("2006-01-02 15:04:05")
81 -}
82 -
83 -func humanNumber[T constraints.Float | constraints.Integer](n T) string {
84 - nf := float64(n)
85 - str := humanSI(nf, 0)
86 - fullStr := humanFull(nf, 0)
87 - if str != fullStr {
88 - return fmt.Sprintf("%s\t(%s)", str, fullStr)
89 - }
90 - return str
91 -}
92 -
93 -func humanSI(val float64, decimals int) string {
94 - v, unit := humanize.ComputeSI(val)
95 - return fmt.Sprintf("%s%s", humanFull(v, decimals), unit)
96 -}
97 -
98 -func humanFull(val float64, decimals int) string {
99 - return humanize.CommafWithDigits(val, decimals)
100 -}
core/commands/stat_reprovide.go new
+104
@@ -0,0 +1,104 @@
1 +package commands
2 +
3 +import (
4 + "fmt"
5 + "io"
6 + "text/tabwriter"
7 + "time"
8 +
9 + humanize "github.com/dustin/go-humanize"
10 + "github.com/ipfs/boxo/provider"
11 + cmds "github.com/ipfs/go-ipfs-cmds"
12 + "github.com/ipfs/kubo/core/commands/cmdenv"
13 + "github.com/libp2p/go-libp2p-kad-dht/fullrt"
14 + "golang.org/x/exp/constraints"
15 +)
16 +
17 +type reprovideStats struct {
18 + provider.ReproviderStats
19 + fullRT bool
20 +}
21 +
22 +var statReprovideCmd = &cmds.Command{
23 + Status: cmds.Experimental,
24 + Helptext: cmds.HelpText{
25 + Tagline: "Returns statistics about the node's reprovider system.",
26 + ShortDescription: `
27 +Returns statistics about the content the node is reproviding every
28 +Reprovider.Interval according to Reprovider.Strategy:
29 +https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider
30 +
31 +This interface is not stable and may change from release to release.
32 +
33 +`,
34 + },
35 + Arguments: []cmds.Argument{},
36 + Options: []cmds.Option{},
37 + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
38 + nd, err := cmdenv.GetNode(env)
39 + if err != nil {
40 + return err
41 + }
42 +
43 + if !nd.IsOnline {
44 + return ErrNotOnline
45 + }
46 +
47 + stats, err := nd.Provider.Stat()
48 + if err != nil {
49 + return err
50 + }
51 + _, fullRT := nd.DHTClient.(*fullrt.FullRT)
52 +
53 + if err := res.Emit(reprovideStats{stats, fullRT}); err != nil {
54 + return err
55 + }
56 +
57 + return nil
58 + },
59 + Encoders: cmds.EncoderMap{
60 + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s reprovideStats) error {
61 + wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0)
62 + defer wtr.Flush()
63 +
64 + fmt.Fprintf(wtr, "TotalReprovides:\t%s\n", humanNumber(s.TotalReprovides))
65 + fmt.Fprintf(wtr, "AvgReprovideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration))
66 + fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.LastReprovideDuration))
67 + if !s.LastRun.IsZero() {
68 + fmt.Fprintf(wtr, "LastReprovide:\t%s\n", humanTime(s.LastRun))
69 + if s.fullRT {
70 + fmt.Fprintf(wtr, "NextReprovide:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval)))
71 + }
72 + }
73 + return nil
74 + }),
75 + },
76 + Type: reprovideStats{},
77 +}
78 +
79 +func humanDuration(val time.Duration) string {
80 + return val.Truncate(time.Microsecond).String()
81 +}
82 +
83 +func humanTime(val time.Time) string {
84 + return val.Format("2006-01-02 15:04:05")
85 +}
86 +
87 +func humanNumber[T constraints.Float | constraints.Integer](n T) string {
88 + nf := float64(n)
89 + str := humanSI(nf, 0)
90 + fullStr := humanFull(nf, 0)
91 + if str != fullStr {
92 + return fmt.Sprintf("%s\t(%s)", str, fullStr)
93 + }
94 + return str
95 +}
96 +
97 +func humanSI(val float64, decimals int) string {
98 + v, unit := humanize.ComputeSI(val)
99 + return fmt.Sprintf("%s%s", humanFull(v, decimals), unit)
100 +}
101 +
102 +func humanFull(val float64, decimals int) string {
103 + return humanize.CommafWithDigits(val, decimals)
104 +}
core/node/groups.go
+1
@@ -359,6 +359,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
359 cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
360 cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
361 cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient),
362 + int(cfg.Provider.WorkerCount.WithDefault(config.DefaultProviderWorkerCount)),
363 ),
364 )
365 }
core/node/provider.go
+4 -4
@@ -21,12 +21,13 @@ import (
21 // and in 'ipfs stats provide' report.
22 const sampledBatchSize = 1000
23
24 -func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option {
24 +func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount int) fx.Option {
25 return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) {
26 opts := []provider.Option{
27 provider.Online(cr),
28 provider.ReproviderInterval(reprovideInterval),
29 provider.KeyProvider(keyProvider),
30 + provider.ProvideWorkerCount(provideWorkerCount),
31 }
32 if !acceleratedDHTClient && reprovideInterval > 0 {
33 // The estimation kinda suck if you are running with accelerated DHT client,
@@ -131,7 +132,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtcli
132 // ONLINE/OFFLINE
133
134 // OnlineProviders groups units managing provider routing records online
134 -func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option {
135 +func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount int) fx.Option {
136 if useStrategicProviding {
137 return OfflineProviders()
138 }
@@ -146,7 +147,7 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
147
148 return fx.Options(
149 keyProvider,
149 - ProviderSys(reprovideInterval, acceleratedDHTClient),
150 + ProviderSys(reprovideInterval, acceleratedDHTClient, provideWorkerCount),
151 )
152 }
153
@@ -169,7 +170,6 @@ func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFun
170 kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher)
171 return kcf(ctx)
172 }
172 -
173 }
174
175 func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc {
docs/changelogs/v0.35.md
+32
@@ -17,6 +17,8 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
17 - [New `ipfs add` Options](#new-ipfs-add-options)
18 - [Persistent `Import.*` Configuration](#persistent-import-configuration)
19 - [Updated Configuration Profiles](#updated-configuration-profiles)
20 + - [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids)
21 + - [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider)
22 - [📦️ Important dependency updates](#-important-dependency-updates)
23 - [📝 Changelog](#-changelog)
24 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
@@ -81,6 +83,36 @@ The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/m
83 > [!TIP]
84 > Apply one of CIDv1 test [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) with `ipfs config profile apply test-cid-v1[-wide]`.
85
86 +#### Optimized, dedicated queue for providing fresh CIDs
87 +
88 +From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0),
89 +Bitswap stopped advertising newly added and received blocks to the DHT. Since
90 +then `boxo/provider` is responsible for the provide and reprovide logic. Prior
91 +to `v0.35.0`, provides and reprovides were handled together in batches, leading
92 +to delays in initial advertisements (provides).
93 +
94 +Provides and Reprovides now have separate queues, allowing for immediate
95 +provide of new CIDs and optimised batching of reprovides.
96 +
97 +This change introduces a new configuration option for limiting the number of
98 +concurrent provide operations:
99 +[`Provider.WorkerCount`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerworkercount).
100 +
101 +> [!TIP]
102 +> Users who need to provide large volumes of content immediately should consider removing the cap on concurrent provide operations and also set `Routing.AcceleratedDHTClient` to `true`.
103 +
104 +##### Deprecated `ipfs stats provider`
105 +
106 +Since the `ipfs stats provider` command was displaying statistics for both
107 +provides and reprovides, this command isn't relevant anymore after separating
108 +the two queues.
109 +
110 +The successor command is `ipfs stats reprovide`, showing the same statistics,
111 +but for reprovides only.
112 +
113 +> [!NOTE]
114 +> `ipfs stats provider` still works, but is marked as deprecated and will be removed in a future release. Be mindful that the command provides only statistics about reprovides (similar to `ipfs stats reprovide`) and not the new provide queue (this will be fixed as a part of wider refactor planned for a future release).
115 +
116 #### 📦️ Important dependency updates
117
118 - update `boxo` to [v0.30.0](https://github.com/ipfs/boxo/releases/tag/v0.30.0)
docs/config.md
+37
@@ -105,6 +105,9 @@ config file at runtime.
105 - [`Pinning.RemoteServices: Policies.MFS.Enabled`](#pinningremoteservices-policiesmfsenabled)
106 - [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname)
107 - [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval)
108 + - [`Provider`](#provider)
109 + - [`Provider.Strategy`](#providerstrategy)
110 + - [`Provider.WorkerCount`](#providerworkercount)
111 - [`Pubsub`](#pubsub)
112 - [`Pubsub.Enabled`](#pubsubenabled)
113 - [`Pubsub.Router`](#pubsubrouter)
@@ -207,6 +210,7 @@ config file at runtime.
210 - [`announce-on` profile](#announce-on-profile)
211 - [`legacy-cid-v0` profile](#legacy-cid-v0-profile)
212 - [`test-cid-v1` profile](#test-cid-v1-profile)
213 + - [`test-cid-v1-wide` profile](#test-cid-v1-wide-profile)
214 - [Types](#types)
215 - [`flag`](#flag)
216 - [`priority`](#priority)
@@ -1404,6 +1408,39 @@ Default: `"5m"`
1408
1409 Type: `duration`
1410
1411 +## `Provider`
1412 +
1413 +Configuration applied to the initial one-time announcement of fresh CIDs
1414 +created with `ipfs add`, `ipfs files`, `ipfs dag import`, `ipfs block|dag put`
1415 +commands.
1416 +
1417 +For periodical DHT reprovide settings, see [`Reprovide.*`](#reprovider).
1418 +
1419 +### `Provider.Strategy`
1420 +
1421 +Legacy, not used at the moment, see [`Reprovider.Strategy`](#reproviderstrategy) instead.
1422 +
1423 +### `Provider.WorkerCount`
1424 +
1425 +Sets the maximum number of _concurrent_ DHT provide operations. DHT reprovides
1426 +operations do **not** count against that limit. A value of `0` allows an
1427 +unlimited number of provide workers.
1428 +
1429 +If the [accelerated DHT client](#routingaccelerateddhtclient) is enabled, each
1430 +provide operation opens ~20 connections in parallel. With the standard DHT
1431 +client (accelerated disabled), each provide opens between 20 and 60
1432 +connections, with at most 10 active at once. Provides complete more quickly
1433 +when using the accelerated client. Be mindful of how many simultaneous
1434 +connections this setting can generate.
1435 +
1436 +For nodes without strict connection limits that need to provide large volumes
1437 +of content immediately, we recommend enabling the `Routing.AcceleratedDHTClient` and
1438 +setting `Provider.WorkerCount` to `0` (unlimited).
1439 +
1440 +Default: `64`
1441 +
1442 +Type: `integer` (non-negative; `0` means unlimited number of workers)
1443 +
1444 ## `Pubsub`
1445
1446 **DEPRECATED**: See [#9717](https://github.com/ipfs/kubo/issues/9717)
test/sharness/t0002-docker-image.sh
+2 -2
@@ -36,7 +36,7 @@ test_expect_success "docker image build succeeds" '
36 '
37
38 test_expect_success "write init scripts" '
39 - echo "ipfs config Provider.Strategy Bar" > 001.sh &&
39 + echo "ipfs config Mounts.IPFS Bar" > 001.sh &&
40 echo "ipfs config Pubsub.Router Qux" > 002.sh &&
41 chmod +x 002.sh
42 '
@@ -65,7 +65,7 @@ test_expect_success "check that init scripts were run correctly and in the corre
65
66 test_expect_success "check that init script configs were applied" '
67 echo Bar > expected &&
68 - docker exec "$DOC_ID" ipfs config Provider.Strategy > actual &&
68 + docker exec "$DOC_ID" ipfs config Mounts.IPFS > actual &&
69 test_cmp actual expected &&
70 echo Qux > expected &&
71 docker exec "$DOC_ID" ipfs config Pubsub.Router > actual &&
test/sharness/t0070-user-config.sh
+4 -4
@@ -11,12 +11,12 @@ test_description="Test user-provided config values"
11 test_init_ipfs
12
13 test_expect_success "bootstrap doesn't overwrite user-provided config keys (top-level)" '
14 - ipfs config Provider.Strategy >previous &&
15 - ipfs config Provider.Strategy foo &&
14 + ipfs config Identity.PeerID >previous &&
15 + ipfs config Identity.PeerID foo &&
16 ipfs bootstrap rm --all &&
17 echo "foo" >expected &&
18 - ipfs config Provider.Strategy >actual &&
19 - ipfs config Provider.Strategy $(cat previous) &&
18 + ipfs config Identity.PeerID >actual &&
19 + ipfs config Identity.PeerID $(cat previous) &&
20 test_cmp expected actual
21 '
22