fix(mfs): unbound cache growth with `flush=false` (#10971)
* fix: prevent --flush=false in 'ipfs files rm' command the 'ipfs files rm' command always flushes for safety to ensure data integrity. this change adds an explicit error when users try to pass --flush=false, improving ux and preventing confusion. related to #10842 * fix: add MFS cache size limit to prevent unbounded growth - add Internal.MFSAutoflushThreshold config (experimental) - directories auto-flush when cache exceeds threshold with --flush=false - prevents high memory usage issue from #10842 - default: 256 entries per directory (matching HAMT shard size) - set to 0 to restore old behavior (risky, may cause errors) Closes #10842
Marcin Rataj committed
Sep 19, 2025 at 03:39 UTC
fa17b69c7d65f2d63eee49c158f72b34ab4805e7
6 files changed
+124
-7
config/internal.go
+8
@@ -6,6 +6,14 @@ type Internal struct {
6
UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"` // moved to Import.UnixFSHAMTDirectorySizeThreshold
7
Libp2pForceReachability *OptionalString `json:",omitempty"`
8
BackupBootstrapInterval *OptionalDuration `json:",omitempty"`
9
+ // MFSAutoflushThreshold controls the number of entries cached in memory
10
+ // for each MFS directory before auto-flush is triggered to prevent
11
+ // unbounded memory growth when using --flush=false.
12
+ // Default: 256 (matches HAMT shard size)
13
+ // Set to 0 to disable cache limiting (old behavior, may cause high memory usage)
14
+ // This is an EXPERIMENTAL feature and may change or be removed in future releases.
15
+ // See https://github.com/ipfs/kubo/issues/10842
16
+ MFSAutoflushThreshold OptionalInteger `json:",omitempty"`
17
}
18
19
type InternalBitswap struct {
core/commands/files.go
+17
-7
@@ -64,13 +64,16 @@ defaults to true and ensures two things: 1) that the changes are reflected in
64
the full MFS structure (updated CIDs) 2) that the parent-folder's cache is
65
cleared. Use caution when setting this flag to false. It will improve
66
performance for large numbers of file operations, but it does so at the cost
67
-of consistency guarantees and unbound growth of the directories' in-memory
68
-caches. If the daemon is unexpectedly killed before running 'ipfs files
69
-flush' on the files in question, then data may be lost. This also applies to
70
-run 'ipfs repo gc' concurrently with '--flush=false' operations. We recommend
71
-flushing paths regularly with 'ipfs files flush', specially the folders on
72
-which many write operations are happening, as a way to clear the directory
73
-cache, free memory and speed up read operations.`,
67
+of consistency guarantees. If the daemon is unexpectedly killed before running
68
+'ipfs files flush' on the files in question, then data may be lost. This also
69
+applies to run 'ipfs repo gc' concurrently with '--flush=false' operations.
70
+
71
+When using '--flush=false', directories will automatically flush when the
72
+number of cached entries exceeds the Internal.MFSAutoflushThreshold config.
73
+This prevents unbounded memory growth. We recommend flushing
74
+paths regularly with 'ipfs files flush', specially the folders on which many
75
+write operations are happening, as a way to clear the directory cache, free
76
+memory and speed up read operations.`,
77
},
78
Options: []cmds.Option{
79
cmds.BoolOption(filesFlushOptionName, "f", "Flush target and ancestors after write.").WithDefault(true),
@@ -1258,6 +1261,13 @@ Remove files or directories.
1261
cmds.BoolOption(forceOptionName, "Forcibly remove target at path; implies -r for directories"),
1262
},
1263
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
1264
+ // Check if user explicitly set --flush=false
1265
+ if flushOpt, ok := req.Options[filesFlushOptionName]; ok {
1266
+ if flush, ok := flushOpt.(bool); ok && !flush {
1267
+ return fmt.Errorf("files rm always flushes for safety. The --flush flag cannot be set to false for this command")
1268
+ }
1269
+ }
1270
+
1271
nd, err := cmdenv.GetNode(env)
1272
if err != nil {
1273
return err
core/node/core.go
+10
@@ -242,6 +242,16 @@ func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo
242
}
243
244
root, err := mfs.NewRoot(ctx, dag, nd, pf, prov)
245
+ if err != nil {
246
+ return nil, err
247
+ }
248
+
249
+ // Configure MFS directory cache auto-flush threshold if specified (experimental)
250
+ cfg, err := repo.Config()
251
+ if err == nil && !cfg.Internal.MFSAutoflushThreshold.IsDefault() {
252
+ threshold := int(cfg.Internal.MFSAutoflushThreshold.WithDefault(int64(mfs.DefaultMaxCacheSize)))
253
+ root.SetMaxCacheSize(threshold)
254
+ }
255
256
lc.Append(fx.Hook{
257
OnStop: func(ctx context.Context) error {
docs/changelogs/v0.38.md
+4
@@ -81,6 +81,10 @@ Identity CIDs use [multihash `0x00`](https://github.com/multiformats/multicodec/
81
82
This release resolves several long-standing MFS issues: raw nodes now preserve their codec instead of being forced to dag-pb, append operations on raw nodes work correctly by converting to UnixFS when needed, and identity CIDs properly inherit the full CID prefix from parent directories.
83
84
+#### MFS directory cache auto-flush
85
+
86
+The new [`Internal.MFSAutoflushThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalmfsautoflushthreshold) configuration option prevents unbounded memory growth when using `--flush=false` with `ipfs files` commands by automatically flushing directories when their cache exceeds the configured threshold (default: 256 entries).
87
+
88
### 📦️ Important dependency updates
89
90
### 📝 Changelog
docs/config.md
+25
@@ -1599,6 +1599,31 @@ Type: `flag`
1599
1600
**MOVED:** see [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold)
1601
1602
+### `Internal.MFSAutoflushThreshold`
1603
+
1604
+Controls the number of entries cached in memory for each MFS directory before
1605
+auto-flush is triggered to prevent unbounded memory growth when using `--flush=false`
1606
+with `ipfs files` commands.
1607
+
1608
+When a directory's cache reaches this threshold, it will automatically flush to
1609
+the blockstore even when `--flush=false` is specified. This prevents excessive
1610
+memory usage while still allowing performance benefits of deferred flushing for
1611
+smaller operations.
1612
+
1613
+**Examples:**
1614
+* `256` - Default value. Provides a good balance between performance and memory usage.
1615
+* `0` - Disables cache limiting (behavior before Kubo 0.38). May cause high memory
1616
+ usage with `--flush=false` on large directories.
1617
+* `1024` - Higher limit for systems with more available memory that need to perform
1618
+ many operations before flushing.
1619
+
1620
+Default: `256`
1621
+
1622
+Type: `optionalInteger` (0 disables the limit, risky, may lead to errors)
1623
+
1624
+**Note:** This is an EXPERIMENTAL feature and may change or be removed in future releases.
1625
+See [#10842](https://github.com/ipfs/kubo/issues/10842) for more information.
1626
+
1627
## `Ipns`
1628
1629
### `Ipns.RepublishPeriod`
test/cli/files_test.go
+60
@@ -118,3 +118,63 @@ func TestFilesCp(t *testing.T) {
118
assert.Equal(t, data, catRes.Stdout.Trimmed())
119
})
120
}
121
+
122
+func TestFilesRm(t *testing.T) {
123
+ t.Parallel()
124
+
125
+ t.Run("files rm with --flush=false returns error", func(t *testing.T) {
126
+ // Test that files rm rejects --flush=false so user does not assume disabling flush works
127
+ // (rm ignored it before, better to explicitly error)
128
+ // See https://github.com/ipfs/kubo/issues/10842
129
+ t.Parallel()
130
+
131
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
132
+
133
+ // Create a file to remove
134
+ node.IPFS("files", "mkdir", "/test-dir")
135
+
136
+ // Try to remove with --flush=false, should error
137
+ res := node.RunIPFS("files", "rm", "-r", "--flush=false", "/test-dir")
138
+ assert.NotEqual(t, 0, res.ExitErr.ExitCode())
139
+ assert.Contains(t, res.Stderr.String(), "files rm always flushes for safety")
140
+ assert.Contains(t, res.Stderr.String(), "cannot be set to false")
141
+
142
+ // Verify the directory still exists (wasn't removed due to error)
143
+ lsRes := node.IPFS("files", "ls", "/")
144
+ assert.Contains(t, lsRes.Stdout.String(), "test-dir")
145
+ })
146
+
147
+ t.Run("files rm with --flush=true works", func(t *testing.T) {
148
+ t.Parallel()
149
+
150
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
151
+
152
+ // Create a file to remove
153
+ node.IPFS("files", "mkdir", "/test-dir")
154
+
155
+ // Remove with explicit --flush=true, should work
156
+ res := node.IPFS("files", "rm", "-r", "--flush=true", "/test-dir")
157
+ assert.NoError(t, res.Err)
158
+
159
+ // Verify the directory was removed
160
+ lsRes := node.IPFS("files", "ls", "/")
161
+ assert.NotContains(t, lsRes.Stdout.String(), "test-dir")
162
+ })
163
+
164
+ t.Run("files rm without flush flag works (default behavior)", func(t *testing.T) {
165
+ t.Parallel()
166
+
167
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
168
+
169
+ // Create a file to remove
170
+ node.IPFS("files", "mkdir", "/test-dir")
171
+
172
+ // Remove without flush flag (should use default which is true)
173
+ res := node.IPFS("files", "rm", "-r", "/test-dir")
174
+ assert.NoError(t, res.Err)
175
+
176
+ // Verify the directory was removed
177
+ lsRes := node.IPFS("files", "ls", "/")
178
+ assert.NotContains(t, lsRes.Stdout.String(), "test-dir")
179
+ })
180
+}