feat: partial DAG provides with Reprovider.Strategy=mfs|pinned+mfs (#10754)
Co-authored-by: Marcin Rataj <lidel@lidel.org>
Hector Sanjuan committed
Apr 9, 2025 at 18:28 UTC
996bcf30a687a1334379d5df9d4dc2579970a70f
11 files changed
+91
-41
core/node/core.go
+1
@@ -115,6 +115,7 @@ func FetcherConfig(bs blockservice.BlockService) FetchersOut {
115
// path resolution should not fetch new blocks via exchange.
116
offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
117
offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
118
+ offlineIpldFetcher.SkipNotFound = true // carries onto the UnixFSFetcher below
119
offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
120
offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
121
core/node/provider.go
+60
-28
@@ -7,8 +7,10 @@ import (
7
8
"github.com/ipfs/boxo/blockstore"
9
"github.com/ipfs/boxo/fetcher"
10
+ "github.com/ipfs/boxo/mfs"
11
pin "github.com/ipfs/boxo/pinning/pinner"
12
provider "github.com/ipfs/boxo/provider"
13
+ "github.com/ipfs/go-cid"
14
"github.com/ipfs/kubo/repo"
15
irouting "github.com/ipfs/kubo/routing"
16
"go.uber.org/fx"
@@ -136,14 +138,8 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
138
139
var keyProvider fx.Option
140
switch reprovideStrategy {
139
- case "all", "":
140
- keyProvider = fx.Provide(newProvidingStrategy(false, false))
141
- case "roots":
142
- keyProvider = fx.Provide(newProvidingStrategy(true, true))
143
- case "pinned":
144
- keyProvider = fx.Provide(newProvidingStrategy(true, false))
145
- case "flat":
146
- keyProvider = fx.Provide(provider.NewBlockstoreProvider)
141
+ case "all", "", "roots", "pinned", "mfs", "pinned+mfs", "flat":
142
+ keyProvider = fx.Provide(newProvidingStrategy(reprovideStrategy))
143
default:
144
return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy))
145
}
@@ -159,32 +155,68 @@ func OfflineProviders() fx.Option {
155
return fx.Provide(provider.NewNoopProvider)
156
}
157
162
-func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
158
+func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFunc {
159
+ return func(ctx context.Context) (<-chan cid.Cid, error) {
160
+ err := mfsRoot.FlushMemFree(ctx)
161
+ if err != nil {
162
+ return nil, fmt.Errorf("error flushing mfs, cannot provide MFS: %w", err)
163
+ }
164
+ rootNode, err := mfsRoot.GetDirectory().GetNode()
165
+ if err != nil {
166
+ return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
167
+ }
168
+
169
+ kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher)
170
+ return kcf(ctx)
171
+ }
172
+
173
+}
174
+
175
+func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc {
176
+ return func(ctx context.Context) (<-chan cid.Cid, error) {
177
+ rootNode, err := mfsRoot.GetDirectory().GetNode()
178
+ if err != nil {
179
+ return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
180
+ }
181
+ ch := make(chan cid.Cid, 1)
182
+ ch <- rootNode.Cid()
183
+ close(ch)
184
+ return ch, nil
185
+ }
186
+}
187
+
188
+func newProvidingStrategy(strategy string) interface{} {
189
type input struct {
190
fx.In
165
- Pinner pin.Pinner
166
- Blockstore blockstore.Blockstore
167
- IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
191
+ Pinner pin.Pinner
192
+ Blockstore blockstore.Blockstore
193
+ OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
194
+ OfflineUnixFSFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
195
+ MFSRoot *mfs.Root
196
}
197
return func(in input) provider.KeyChanFunc {
170
- // Pinner-related CIDs will be buffered in memory to avoid
171
- // deadlocking the pinner when the providing process is slow.
172
-
173
- if onlyRoots {
174
- return provider.NewBufferedProvider(
175
- provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
198
+ switch strategy {
199
+ case "roots":
200
+ return provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher))
201
+ case "pinned":
202
+ return provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher))
203
+ case "pinned+mfs":
204
+ return provider.NewPrioritizedProvider(
205
+ provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)),
206
+ mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher),
207
)
177
- }
178
-
179
- if onlyPinned {
180
- return provider.NewBufferedProvider(
181
- provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher),
208
+ case "mfs":
209
+ return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher)
210
+ case "flat":
211
+ return provider.NewBlockstoreProvider(in.Blockstore)
212
+ default: // "all", ""
213
+ return provider.NewPrioritizedProvider(
214
+ provider.NewPrioritizedProvider(
215
+ provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)),
216
+ mfsRootProvider(in.MFSRoot),
217
+ ),
218
+ provider.NewBlockstoreProvider(in.Blockstore),
219
)
220
}
184
-
185
- return provider.NewPrioritizedProvider(
186
- provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)),
187
- provider.NewBlockstoreProvider(in.Blockstore),
188
- )
221
}
222
}
docs/changelogs/v0.35.md
+11
@@ -10,6 +10,7 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
10
11
- [Overview](#overview)
12
- [🔦 Highlights](#-highlights)
13
+ - [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs)
14
- [📦️ Important dependency updates](#-important-dependency-updates)
15
- [📝 Changelog](#-changelog)
16
- [👨👩👧👦 Contributors](#-contributors)
@@ -18,6 +19,16 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
19
20
### 🔦 Highlights
21
22
+#### Dedicated `Reprovider.Strategy` for MFS
23
+
24
+The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme).
25
+
26
+Previously, the `pinned` reprovider strategy required manual pin management: each dataset update meant pinning the new version and unpinning the old one. Now, new strategies—`mfs` and `pinned+mfs`—let users limit announcements to data explicitly placed in MFS. This simplifies updating datasets and announcing only the latest version to the Amino DHT.
27
+
28
+Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS alone to manage updates and announcements, eliminating the need for manual pinning and unpinning. We hope this makes it easier to publish just the data that matters to you.
29
+
30
+See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details.
31
+
32
##### `Routing.IgnoreProviders`
33
34
This new option allows ignoring specific peer IDs when returned by the content
docs/config.md
+9
-4
@@ -1587,10 +1587,10 @@ Type: `optionalDuration` (unset for the default)
1587
Tells reprovider what should be announced. Valid strategies are:
1588
1589
- `"all"` - announce all CIDs of stored blocks
1590
- - Order: root blocks of direct and recursive pins are announced first, then the rest of blockstore
1591
-- `"pinned"` - only announce pinned CIDs recursively (both roots and child blocks)
1590
+ - Order: root blocks of direct and recursive pins and MFS root are announced first, then the rest of blockstore
1591
+- `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks)
1592
- Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
1593
-- `"roots"` - only announce the root block of explicitly pinned CIDs
1593
+- `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`)
1594
- **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks.
1595
It makes sense only for use cases where the entire DAG is fetched in full,
1596
and a graceful resume does not have to be guaranteed: the lack of child
@@ -1598,10 +1598,15 @@ Tells reprovider what should be announced. Valid strategies are:
1598
providers for the missing block in the middle of a file, unless the peer
1599
happens to already be connected to a provider and ask for child CID over
1600
bitswap.
1601
+- `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`)
1602
+ - Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced.
1603
+- `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies.
1604
+ - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache.
1605
+ - Order: first `pinned` and then the locally available part of `mfs`.
1606
- `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything.
1607
1608
> [!IMPORTANT]
1604
-> Reproviding larger pinsets using the `all`, `pinned`, or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
1609
+> Reproviding larger pinsets using the `all`, `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
1610
> This is due to the use of a buffered provider, which avoids holding a lock on the entire pinset during the reprovide cycle.
1611
> The `flat` strategy can be used to lower memory requirements, but only recommended if memory utilization is too high, prioritization of pins is not necessary, and it is acceptable to announce every block cached in the local repository.
1612
docs/examples/kubo-as-a-library/go.mod
+1
-1
@@ -7,7 +7,7 @@ go 1.24
7
replace github.com/ipfs/kubo => ./../../..
8
9
require (
10
- github.com/ipfs/boxo v0.29.1
10
+ github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb
11
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12
github.com/libp2p/go-libp2p v0.41.1
13
github.com/multiformats/go-multiaddr v0.15.0
docs/examples/kubo-as-a-library/go.sum
+2
-2
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
298
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
299
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
300
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
301
-github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc=
302
-github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
301
+github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
302
+github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
303
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
304
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
305
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
go.mod
+1
-1
@@ -21,7 +21,7 @@ require (
21
github.com/hashicorp/go-version v1.7.0
22
github.com/ipfs-shipyard/nopfs v0.0.14
23
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
24
- github.com/ipfs/boxo v0.29.1
24
+ github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb
25
github.com/ipfs/go-block-format v0.2.0
26
github.com/ipfs/go-cid v0.5.0
27
github.com/ipfs/go-cidutil v0.1.0
go.sum
+2
-2
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
362
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
363
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
364
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
365
-github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc=
366
-github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
365
+github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
366
+github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
367
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
368
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
369
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
test/dependencies/go.mod
+1
-1
@@ -116,7 +116,7 @@ require (
116
github.com/huin/goupnp v1.3.0 // indirect
117
github.com/inconshreveable/mousetrap v1.1.0 // indirect
118
github.com/ipfs/bbloom v0.0.4 // indirect
119
- github.com/ipfs/boxo v0.29.1 // indirect
119
+ github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb // indirect
120
github.com/ipfs/go-block-format v0.2.0 // indirect
121
github.com/ipfs/go-cid v0.5.0 // indirect
122
github.com/ipfs/go-datastore v0.8.2 // indirect
test/dependencies/go.sum
+2
-2
@@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
294
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
295
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
296
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
297
-github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc=
298
-github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
297
+github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
298
+github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
299
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
300
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
301
github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
test/sharness/t0119-prometheus-data/prometheus_metrics
+1
@@ -1,3 +1,4 @@
1
+exchange_bitswap_requests_in_flight
2
exchange_bitswap_response_bytes_bucket
3
exchange_bitswap_response_bytes_count
4
exchange_bitswap_response_bytes_sum