@cryptotaxi247 / kubo / commits / 6b55e6491

feat(config): `ipfs add` and `Import` options for controling UnixFS DAG Width (#10774)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Hector Sanjuan committed Apr 15, 2025 at 22:56 UTC 6b55e649186b27c3b17b680b013dfad3c9d6d601
25 files changed +914 -140
config/import.go
+24 -6
@@ -1,11 +1,18 @@
1 package config
2
3 +import (
4 + "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
5 + "github.com/ipfs/boxo/ipld/unixfs/io"
6 +)
7 +
8 const (
9 DefaultCidVersion = 0
10 DefaultUnixFSRawLeaves = false
11 DefaultUnixFSChunker = "size-262144"
12 DefaultHashFunction = "sha2-256"
13
14 + DefaultUnixFSHAMTDirectorySizeThreshold = "256KiB" // https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26
15 +
16 // DefaultBatchMaxNodes controls the maximum number of nodes in a
17 // write-batch. The total size of the batch is limited by
18 // BatchMaxnodes and BatchMaxSize.
@@ -14,15 +21,26 @@ const (
21 // write-batch. The total size of the batch is limited by
22 // BatchMaxnodes and BatchMaxSize.
23 DefaultBatchMaxSize = 100 << 20 // 20MiB
24 +
25 +)
26 +
27 +var (
28 + DefaultUnixFSFileMaxLinks = int64(helpers.DefaultLinksPerBlock)
29 + DefaultUnixFSDirectoryMaxLinks = int64(0)
30 + DefaultUnixFSHAMTDirectoryMaxFanout = int64(io.DefaultShardWidth)
31 )
32
33 // Import configures the default options for ingesting data. This affects commands
34 // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
35 type Import struct {
22 - CidVersion OptionalInteger
23 - UnixFSRawLeaves Flag
24 - UnixFSChunker OptionalString
25 - HashFunction OptionalString
26 - BatchMaxNodes OptionalInteger
27 - BatchMaxSize OptionalInteger
36 + CidVersion OptionalInteger
37 + UnixFSRawLeaves Flag
38 + UnixFSChunker OptionalString
39 + HashFunction OptionalString
40 + UnixFSFileMaxLinks OptionalInteger
41 + UnixFSDirectoryMaxLinks OptionalInteger
42 + UnixFSHAMTDirectoryMaxFanout OptionalInteger
43 + UnixFSHAMTDirectorySizeThreshold OptionalString
44 + BatchMaxNodes OptionalInteger
45 + BatchMaxSize OptionalInteger
46 }
config/internal.go
+1 -1
@@ -3,7 +3,7 @@ package config
3 type Internal struct {
4 // All marked as omitempty since we are expecting to make changes to all subcomponents of Internal
5 Bitswap *InternalBitswap `json:",omitempty"`
6 - UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"`
6 + UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"` // moved to Import.UnixFSHAMTDirectorySizeThreshold
7 Libp2pForceReachability *OptionalString `json:",omitempty"`
8 BackupBootstrapInterval *OptionalDuration `json:",omitempty"`
9 }
config/profile.go
+24 -4
@@ -266,24 +266,44 @@ fetching may be degraded.
266 },
267 },
268 "legacy-cid-v0": {
269 - Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`,
270 -
269 + Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. This is likely the least optimal preset, use only if legacy behavior is required.`,
270 Transform: func(c *Config) error {
271 c.Import.CidVersion = *NewOptionalInteger(0)
272 c.Import.UnixFSRawLeaves = False
273 c.Import.UnixFSChunker = *NewOptionalString("size-262144")
274 c.Import.HashFunction = *NewOptionalString("sha2-256")
275 + c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174)
276 + c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0)
277 + c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
278 + c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalString("256KiB")
279 return nil
280 },
281 },
282 "test-cid-v1": {
280 - Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`,
281 -
283 + Description: `Makes UnixFS import produce CIDv1 with raw leaves, sha2-256 and 1 MiB chunks (max 174 links per file, 256 per HAMT node, switch dir to HAMT above 256KiB).`,
284 Transform: func(c *Config) error {
285 c.Import.CidVersion = *NewOptionalInteger(1)
286 c.Import.UnixFSRawLeaves = True
287 c.Import.UnixFSChunker = *NewOptionalString("size-1048576")
288 c.Import.HashFunction = *NewOptionalString("sha2-256")
289 + c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174)
290 + c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0)
291 + c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
292 + c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalString("256KiB")
293 + return nil
294 + },
295 + },
296 + "test-cid-v1-wide": {
297 + Description: `Makes UnixFS import produce CIDv1 with raw leaves, sha2-256 and 1MiB chunks and wider file DAGs (max 1024 links per every node type, switch dir to HAMT above 1MiB).`,
298 + Transform: func(c *Config) error {
299 + c.Import.CidVersion = *NewOptionalInteger(1)
300 + c.Import.UnixFSRawLeaves = True
301 + c.Import.UnixFSChunker = *NewOptionalString("size-1048576") // 1MiB
302 + c.Import.HashFunction = *NewOptionalString("sha2-256")
303 + c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(1024)
304 + c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) // no limit here, use size-based Import.UnixFSHAMTDirectorySizeThreshold instead
305 + c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(1024)
306 + c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalString("1MiB") // 1MiB
307 return nil
308 },
309 },
core/commands/add.go
+60 -21
@@ -37,23 +37,26 @@ type AddEvent struct {
37 }
38
39 const (
40 - quietOptionName = "quiet"
41 - quieterOptionName = "quieter"
42 - silentOptionName = "silent"
43 - progressOptionName = "progress"
44 - trickleOptionName = "trickle"
45 - wrapOptionName = "wrap-with-directory"
46 - onlyHashOptionName = "only-hash"
47 - chunkerOptionName = "chunker"
48 - pinOptionName = "pin"
49 - rawLeavesOptionName = "raw-leaves"
50 - noCopyOptionName = "nocopy"
51 - fstoreCacheOptionName = "fscache"
52 - cidVersionOptionName = "cid-version"
53 - hashOptionName = "hash"
54 - inlineOptionName = "inline"
55 - inlineLimitOptionName = "inline-limit"
56 - toFilesOptionName = "to-files"
40 + quietOptionName = "quiet"
41 + quieterOptionName = "quieter"
42 + silentOptionName = "silent"
43 + progressOptionName = "progress"
44 + trickleOptionName = "trickle"
45 + wrapOptionName = "wrap-with-directory"
46 + onlyHashOptionName = "only-hash"
47 + chunkerOptionName = "chunker"
48 + pinOptionName = "pin"
49 + rawLeavesOptionName = "raw-leaves"
50 + maxFileLinksOptionName = "max-file-links"
51 + maxDirectoryLinksOptionName = "max-directory-links"
52 + maxHAMTFanoutOptionName = "max-hamt-fanout"
53 + noCopyOptionName = "nocopy"
54 + fstoreCacheOptionName = "fscache"
55 + cidVersionOptionName = "cid-version"
56 + hashOptionName = "hash"
57 + inlineOptionName = "inline"
58 + inlineLimitOptionName = "inline-limit"
59 + toFilesOptionName = "to-files"
60
61 preserveModeOptionName = "preserve-mode"
62 preserveMtimeOptionName = "preserve-mtime"
@@ -143,6 +146,9 @@ new flags may be added in the future. It is not guaranteed for the implicit
146 defaults of 'ipfs add' to remain the same in future Kubo releases, or for other
147 IPFS software to use the same import parameters as Kubo.
148
149 +Use Import.* configuration options to override global implicit defaults:
150 +https://github.com/ipfs/kubo/blob/master/docs/config.md#import
151 +
152 If you need to back up or transport content-addressed data using a non-IPFS
153 medium, CID can be preserved with CAR files.
154 See 'dag export' and 'dag import' for more information.
@@ -166,12 +172,15 @@ See 'dag export' and 'dag import' for more information.
172 cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
173 cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
174 cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
169 - cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
170 - cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
175 + cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash. Default: Import.UnixFSChunker"),
176 + cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. Default: Import.UnixFSRawLeaves"),
177 + cmds.IntOption(maxFileLinksOptionName, "Limit the maximum number of links in UnixFS file nodes to this value. (experimental) Default: Import.UnixFSFileMaxLinks"),
178 + cmds.IntOption(maxDirectoryLinksOptionName, "Limit the maximum number of links in UnixFS basic directory nodes to this value. Default: Import.UnixFSDirectoryMaxLinks. WARNING: experimental, Import.UnixFSHAMTThreshold is a safer alternative."),
179 + cmds.IntOption(maxHAMTFanoutOptionName, "Limit the maximum number of links of a UnixFS HAMT directory node to this (power of 2, multiple of 8). Default: Import.UnixFSHAMTDirectoryMaxFanout WARNING: experimental, see Import.UnixFSHAMTDirectorySizeThreshold as well."),
180 cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
181 cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
173 - cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
174 - cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
182 + cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true. Default: Import.CidVersion"),
183 + cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. Default: Import.HashFunction"),
184 cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
185 cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
186 cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
@@ -222,6 +231,9 @@ See 'dag export' and 'dag import' for more information.
231 chunker, _ := req.Options[chunkerOptionName].(string)
232 dopin, _ := req.Options[pinOptionName].(bool)
233 rawblks, rbset := req.Options[rawLeavesOptionName].(bool)
234 + maxFileLinks, maxFileLinksSet := req.Options[maxFileLinksOptionName].(int)
235 + maxDirectoryLinks, maxDirectoryLinksSet := req.Options[maxDirectoryLinksOptionName].(int)
236 + maxHAMTFanout, maxHAMTFanoutSet := req.Options[maxHAMTFanoutOptionName].(int)
237 nocopy, _ := req.Options[noCopyOptionName].(bool)
238 fscache, _ := req.Options[fstoreCacheOptionName].(bool)
239 cidVer, cidVerSet := req.Options[cidVersionOptionName].(int)
@@ -253,6 +265,21 @@ See 'dag export' and 'dag import' for more information.
265 rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
266 }
267
268 + if !maxFileLinksSet && !cfg.Import.UnixFSFileMaxLinks.IsDefault() {
269 + maxFileLinksSet = true
270 + maxFileLinks = int(cfg.Import.UnixFSFileMaxLinks.WithDefault(config.DefaultUnixFSFileMaxLinks))
271 + }
272 +
273 + if !maxDirectoryLinksSet && !cfg.Import.UnixFSDirectoryMaxLinks.IsDefault() {
274 + maxDirectoryLinksSet = true
275 + maxDirectoryLinks = int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks))
276 + }
277 +
278 + if !maxHAMTFanoutSet && !cfg.Import.UnixFSHAMTDirectoryMaxFanout.IsDefault() {
279 + maxHAMTFanoutSet = true
280 + maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout))
281 + }
282 +
283 // Storing optional mode or mtime (UnixFS 1.5) requires root block
284 // to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible.
285 if preserveMode || preserveMtime || mode != 0 || mtime != 0 {
@@ -329,6 +356,18 @@ See 'dag export' and 'dag import' for more information.
356 opts = append(opts, options.Unixfs.RawLeaves(rawblks))
357 }
358
359 + if maxFileLinksSet {
360 + opts = append(opts, options.Unixfs.MaxFileLinks(maxFileLinks))
361 + }
362 +
363 + if maxDirectoryLinksSet {
364 + opts = append(opts, options.Unixfs.MaxDirectoryLinks(maxDirectoryLinks))
365 + }
366 +
367 + if maxHAMTFanoutSet {
368 + opts = append(opts, options.Unixfs.MaxHAMTFanout(maxHAMTFanout))
369 + }
370 +
371 if trickle {
372 opts = append(opts, options.Unixfs.Layout(options.TrickleLayout))
373 }
core/coreapi/unixfs.go
+16
@@ -50,6 +50,12 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
50 attribute.Int("inlinelimit", settings.InlineLimit),
51 attribute.Bool("rawleaves", settings.RawLeaves),
52 attribute.Bool("rawleavesset", settings.RawLeavesSet),
53 + attribute.Int("maxfilelinks", settings.MaxFileLinks),
54 + attribute.Bool("maxfilelinksset", settings.MaxFileLinksSet),
55 + attribute.Int("maxdirectorylinks", settings.MaxDirectoryLinks),
56 + attribute.Bool("maxdirectorylinksset", settings.MaxDirectoryLinksSet),
57 + attribute.Int("maxhamtfanout", settings.MaxHAMTFanout),
58 + attribute.Bool("maxhamtfanoutset", settings.MaxHAMTFanoutSet),
59 attribute.Int("layout", int(settings.Layout)),
60 attribute.Bool("pin", settings.Pin),
61 attribute.Bool("onlyhash", settings.OnlyHash),
@@ -132,6 +138,16 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
138 fileAdder.Pin = settings.Pin && !settings.OnlyHash
139 fileAdder.Silent = settings.Silent
140 fileAdder.RawLeaves = settings.RawLeaves
141 + if settings.MaxFileLinksSet {
142 + fileAdder.MaxLinks = settings.MaxFileLinks
143 + }
144 + if settings.MaxDirectoryLinksSet {
145 + fileAdder.MaxDirectoryLinks = settings.MaxDirectoryLinks
146 + }
147 +
148 + if settings.MaxHAMTFanoutSet {
149 + fileAdder.MaxHAMTFanout = settings.MaxHAMTFanout
150 + }
151 fileAdder.NoCopy = settings.NoCopy
152 fileAdder.CidBuilder = prefix
153 fileAdder.PreserveMode = settings.PreserveMode
core/coreiface/options/unixfs.go
+51 -8
@@ -7,6 +7,8 @@ import (
7 "time"
8
9 dag "github.com/ipfs/boxo/ipld/merkledag"
10 + "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
11 + "github.com/ipfs/boxo/ipld/unixfs/io"
12 cid "github.com/ipfs/go-cid"
13 mh "github.com/multiformats/go-multihash"
14 )
@@ -22,10 +24,16 @@ type UnixfsAddSettings struct {
24 CidVersion int
25 MhType uint64
26
25 - Inline bool
26 - InlineLimit int
27 - RawLeaves bool
28 - RawLeavesSet bool
27 + Inline bool
28 + InlineLimit int
29 + RawLeaves bool
30 + RawLeavesSet bool
31 + MaxFileLinks int
32 + MaxFileLinksSet bool
33 + MaxDirectoryLinks int
34 + MaxDirectoryLinksSet bool
35 + MaxHAMTFanout int
36 + MaxHAMTFanoutSet bool
37
38 Chunker string
39 Layout Layout
@@ -60,10 +68,16 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix,
68 CidVersion: -1,
69 MhType: mh.SHA2_256,
70
63 - Inline: false,
64 - InlineLimit: 32,
65 - RawLeaves: false,
66 - RawLeavesSet: false,
71 + Inline: false,
72 + InlineLimit: 32,
73 + RawLeaves: false,
74 + RawLeavesSet: false,
75 + MaxFileLinks: helpers.DefaultLinksPerBlock,
76 + MaxFileLinksSet: false,
77 + MaxDirectoryLinks: 0,
78 + MaxDirectoryLinksSet: false,
79 + MaxHAMTFanout: io.DefaultShardWidth,
80 + MaxHAMTFanoutSet: false,
81
82 Chunker: "size-262144",
83 Layout: BalancedLayout,
@@ -190,6 +204,35 @@ func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption {
204 }
205 }
206
207 +// MaxFileLinks specifies the maximum number of children for UnixFS file
208 +// nodes.
209 +func (unixfsOpts) MaxFileLinks(n int) UnixfsAddOption {
210 + return func(settings *UnixfsAddSettings) error {
211 + settings.MaxFileLinks = n
212 + settings.MaxFileLinksSet = true
213 + return nil
214 + }
215 +}
216 +
217 +// MaxDirectoryLinks specifies the maximum number of children for UnixFS basic
218 +// directory nodes.
219 +func (unixfsOpts) MaxDirectoryLinks(n int) UnixfsAddOption {
220 + return func(settings *UnixfsAddSettings) error {
221 + settings.MaxDirectoryLinks = n
222 + settings.MaxDirectoryLinksSet = true
223 + return nil
224 + }
225 +}
226 +
227 +// MaxHAMTFanout specifies the maximum width of the HAMT directory shards.
228 +func (unixfsOpts) MaxHAMTFanout(n int) UnixfsAddOption {
229 + return func(settings *UnixfsAddSettings) error {
230 + settings.MaxHAMTFanout = n
231 + settings.MaxHAMTFanoutSet = true
232 + return nil
233 + }
234 +}
235 +
236 // Inline tells the adder to inline small blocks into CIDs
237 func (unixfsOpts) Inline(enable bool) UnixfsAddOption {
238 return func(settings *UnixfsAddSettings) error {
core/coreunix/add.go
+67 -48
@@ -19,6 +19,7 @@ import (
19 "github.com/ipfs/boxo/ipld/unixfs/importer/balanced"
20 ihelper "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
21 "github.com/ipfs/boxo/ipld/unixfs/importer/trickle"
22 + uio "github.com/ipfs/boxo/ipld/unixfs/io"
23 "github.com/ipfs/boxo/mfs"
24 "github.com/ipfs/boxo/path"
25 pin "github.com/ipfs/boxo/pinning/pinner"
@@ -51,38 +52,43 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG
52 bufferedDS := ipld.NewBufferedDAG(ctx, ds)
53
54 return &Adder{
54 - ctx: ctx,
55 - pinning: p,
56 - gcLocker: bs,
57 - dagService: ds,
58 - bufferedDS: bufferedDS,
59 - Progress: false,
60 - Pin: true,
61 - Trickle: false,
62 - Chunker: "",
55 + ctx: ctx,
56 + pinning: p,
57 + gcLocker: bs,
58 + dagService: ds,
59 + bufferedDS: bufferedDS,
60 + Progress: false,
61 + Pin: true,
62 + Trickle: false,
63 + MaxLinks: ihelper.DefaultLinksPerBlock,
64 + MaxHAMTFanout: uio.DefaultShardWidth,
65 + Chunker: "",
66 }, nil
67 }
68
69 // Adder holds the switches passed to the `add` command.
70 type Adder struct {
68 - ctx context.Context
69 - pinning pin.Pinner
70 - gcLocker bstore.GCLocker
71 - dagService ipld.DAGService
72 - bufferedDS *ipld.BufferedDAG
73 - Out chan<- interface{}
74 - Progress bool
75 - Pin bool
76 - Trickle bool
77 - RawLeaves bool
78 - Silent bool
79 - NoCopy bool
80 - Chunker string
81 - mroot *mfs.Root
82 - unlocker bstore.Unlocker
83 - tempRoot cid.Cid
84 - CidBuilder cid.Builder
85 - liveNodes uint64
71 + ctx context.Context
72 + pinning pin.Pinner
73 + gcLocker bstore.GCLocker
74 + dagService ipld.DAGService
75 + bufferedDS *ipld.BufferedDAG
76 + Out chan<- interface{}
77 + Progress bool
78 + Pin bool
79 + Trickle bool
80 + RawLeaves bool
81 + MaxLinks int
82 + MaxDirectoryLinks int
83 + MaxHAMTFanout int
84 + Silent bool
85 + NoCopy bool
86 + Chunker string
87 + mroot *mfs.Root
88 + unlocker bstore.Unlocker
89 + tempRoot cid.Cid
90 + CidBuilder cid.Builder
91 + liveNodes uint64
92
93 PreserveMode bool
94 PreserveMtime bool
@@ -94,12 +100,13 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) {
100 if adder.mroot != nil {
101 return adder.mroot, nil
102 }
97 - rnode := unixfs.EmptyDirNode()
98 - err := rnode.SetCidBuilder(adder.CidBuilder)
99 - if err != nil {
100 - return nil, err
101 - }
102 - mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil)
103 +
104 + // Note, this adds it to DAGService already.
105 + mr, err := mfs.NewEmptyRoot(adder.ctx, adder.dagService, nil, mfs.MkdirOpts{
106 + CidBuilder: adder.CidBuilder,
107 + MaxLinks: adder.MaxDirectoryLinks,
108 + MaxHAMTFanout: adder.MaxHAMTFanout,
109 + })
110 if err != nil {
111 return nil, err
112 }
@@ -119,10 +126,15 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
126 return nil, err
127 }
128
129 + maxLinks := ihelper.DefaultLinksPerBlock
130 + if adder.MaxLinks > 0 {
131 + maxLinks = adder.MaxLinks
132 + }
133 +
134 params := ihelper.DagBuilderParams{
135 Dagserv: adder.bufferedDS,
136 RawLeaves: adder.RawLeaves,
125 - Maxlinks: ihelper.DefaultLinksPerBlock,
137 + Maxlinks: maxLinks,
138 NoCopy: adder.NoCopy,
139 CidBuilder: adder.CidBuilder,
140 FileMode: adder.FileMode,
@@ -252,12 +264,15 @@ func (adder *Adder) addNode(node ipld.Node, path string) error {
264 if err != nil {
265 return err
266 }
267 +
268 dir := gopath.Dir(path)
269 if dir != "." {
270 opts := mfs.MkdirOpts{
258 - Mkparents: true,
259 - Flush: false,
260 - CidBuilder: adder.CidBuilder,
271 + Mkparents: true,
272 + Flush: false,
273 + CidBuilder: adder.CidBuilder,
274 + MaxLinks: adder.MaxDirectoryLinks,
275 + MaxHAMTFanout: adder.MaxHAMTFanout,
276 }
277 if err := mfs.Mkdir(mr, dir, opts); err != nil {
278 return err
@@ -460,12 +475,14 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory
475
476 // if we need to store mode or modification time then create a new root which includes that data
477 if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) {
463 - nd := unixfs.EmptyDirNodeWithStat(adder.FileMode, adder.FileMtime)
464 - err := nd.SetCidBuilder(adder.CidBuilder)
465 - if err != nil {
466 - return err
467 - }
468 - mr, err := mfs.NewRoot(ctx, adder.dagService, nd, nil)
478 + mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil,
479 + mfs.MkdirOpts{
480 + CidBuilder: adder.CidBuilder,
481 + MaxLinks: adder.MaxDirectoryLinks,
482 + MaxHAMTFanout: adder.MaxHAMTFanout,
483 + ModTime: adder.FileMtime,
484 + Mode: adder.FileMode,
485 + })
486 if err != nil {
487 return err
488 }
@@ -478,11 +495,13 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory
495 return err
496 }
497 err = mfs.Mkdir(mr, path, mfs.MkdirOpts{
481 - Mkparents: true,
482 - Flush: false,
483 - CidBuilder: adder.CidBuilder,
484 - Mode: adder.FileMode,
485 - ModTime: adder.FileMtime,
498 + Mkparents: true,
499 + Flush: false,
500 + CidBuilder: adder.CidBuilder,
501 + Mode: adder.FileMode,
502 + ModTime: adder.FileMtime,
503 + MaxLinks: adder.MaxDirectoryLinks,
504 + MaxHAMTFanout: adder.MaxHAMTFanout,
505 })
506 if err != nil {
507 return err
core/node/groups.go
+19 -10
@@ -408,20 +408,29 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option {
408 return fx.Error(err)
409 }
410
411 + // Migrate users of deprecated Experimental.ShardingEnabled flag
412 + if cfg.Experimental.ShardingEnabled {
413 + logger.Fatal("The `Experimental.ShardingEnabled` field is no longer used, please remove it from the config. Use Import.UnixFSHAMTDirectorySizeThreshold instead.")
414 + }
415 + if !cfg.Internal.UnixFSShardingSizeThreshold.IsDefault() {
416 + msg := "The `Internal.UnixFSShardingSizeThreshold` field was renamed to `Import.UnixFSHAMTDirectorySizeThreshold`. Please update your config.\n"
417 + if !cfg.Import.UnixFSHAMTDirectorySizeThreshold.IsDefault() {
418 + logger.Fatal(msg) // conflicting values, hard fail
419 + }
420 + logger.Error(msg)
421 + cfg.Import.UnixFSHAMTDirectorySizeThreshold = *cfg.Internal.UnixFSShardingSizeThreshold
422 + }
423 +
424 // Auto-sharding settings
412 - shardSizeString := cfg.Internal.UnixFSShardingSizeThreshold.WithDefault("256kiB")
413 - shardSizeInt, err := humanize.ParseBytes(shardSizeString)
425 + shardingThresholdString := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold)
426 + shardSingThresholdInt, err := humanize.ParseBytes(shardingThresholdString)
427 if err != nil {
428 return fx.Error(err)
429 }
417 - uio.HAMTShardingSize = int(shardSizeInt)
418 -
419 - // Migrate users of deprecated Experimental.ShardingEnabled flag
420 - if cfg.Experimental.ShardingEnabled {
421 - logger.Fatal("The `Experimental.ShardingEnabled` field is no longer used, please remove it from the config.\n" +
422 - "go-ipfs now automatically shards when directory block is bigger than `" + shardSizeString + "`.\n" +
423 - "If you need to restore the old behavior (sharding everything) set `Internal.UnixFSShardingSizeThreshold` to `1B`.\n")
424 - }
430 + shardMaxFanout := cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout)
431 + // TODO: avoid overriding this globally, see if we can extend Directory interface like Get/SetMaxLinks from https://github.com/ipfs/boxo/pull/906
432 + uio.HAMTShardingSize = int(shardSingThresholdInt)
433 + uio.DefaultShardWidth = int(shardMaxFanout)
434
435 return fx.Options(
436 bcfgOpts,
docs/changelogs/v0.35.md
+39
@@ -13,6 +13,10 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
13 - [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs)
14 - [Additional new configuration options](#additional-new-configuration-options)
15 - [Grid view in WebUI](#grid-view-in-webui)
16 + - [Enhanced DAG-Shaping Controls for `ipfs add`](#enhanced-dag-shaping-controls-for-ipfs-add)
17 + - [New `ipfs add` Options](#new-ipfs-add-options)
18 + - [Persistent `Import.*` Configuration](#persistent-import-configuration)
19 + - [Updated Configuration Profiles](#updated-configuration-profiles)
20 - [📦️ Important dependency updates](#-important-dependency-updates)
21 - [📝 Changelog](#-changelog)
22 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
@@ -42,6 +46,41 @@ The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for
46
47 > ![image](https://github.com/user-attachments/assets/80dcf0d0-8103-426f-ae91-416fb25d32b6)
48
49 +#### Enhanced DAG-Shaping Controls for `ipfs add`
50 +
51 +This release advances CIDv1 support by introducing fine-grained control over UnixFS DAG shaping during data ingestion with the `ipfs add` command.
52 +
53 +Wider DAG trees (more links per node, higher fanout, larger thresholds) are beneficial for large files and directories with many files, reducing tree depth and lookup latency in high-latency networks, but they increase node size, straining memory and CPU on resource-constrained devices. Narrower trees (lower link count, lower fanout, smaller thresholds) are preferable for smaller directories, frequent updates, or low-power clients, minimizing overhead and ensuring compatibility, though they may increase traversal steps for very large datasets.
54 +
55 +Kubo now allows users to act on these tradeoffs and customize the width of the DAG created by `ipfs add` command.
56 +
57 +##### New `ipfs add` Options
58 +
59 +Three new options allow you to override default settings for specific import operations:
60 +
61 +- `--max-file-links`: Sets the maximum number of child links for a single file chunk.
62 +- `--max-directory-links`: Defines the maximum number of child entries in a "basic" (single-chunk) directory.
63 + - Note: Directories exceeding this limit or the `Import.UnixFSHAMTDirectorySizeThreshold` are converted to HAMT-based (sharded across multiple blocks) structures.
64 +- `--max-hamt-fanout`: Specifies the maximum number of child nodes for HAMT internal structures.
65 +
66 +##### Persistent `Import.*` Configuration
67 +
68 +You can set default values for these options using the following configuration settings:
69 +- [`Import.UnixFSFileMaxLinks`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfsfilemaxlinks)
70 +- [`Import.UnixFSDirectoryMaxLinks`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfsdirectorymaxlinks)
71 +- [`Import.UnixFSHAMTDirectoryMaxFanout`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorymaxfanout)
72 +- [`Import.UnixFSHAMTDirectorySizeThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorysizethreshold)
73 +
74 +##### Updated Configuration Profiles
75 +
76 +The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) to incorporate these new `Import.*` settings:
77 +- Updated Profile: `test-cid-v1` now includes current defaults as explicit `Import.UnixFSFileMaxLinks=174`, `Import.UnixFSDirectoryMaxLinks=0`, `Import.UnixFSHAMTDirectoryMaxFanout=256` and `Import.UnixFSHAMTDirectorySizeThreshold=256KiB`
78 +- New Profile: `test-cid-v1-wide` adopts experimental directory DAG-shaping defaults, increasing the maximum file DAG width from 174 to 1024, HAMT fanout from 256 to 1024, and raising the HAMT directory sharding threshold from 256KiB to 1MiB, aligning with 1MiB file chunks.
79 + - Feedback: Try it out and share your thoughts at [discuss.ipfs.tech/t/should-we-profile-cids](https://discuss.ipfs.tech/t/should-we-profile-cids/18507) or [ipfs/specs#499](https://github.com/ipfs/specs/pull/499).
80 +
81 +> [!TIP]
82 +> Apply one of CIDv1 test [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) with `ipfs config profile apply test-cid-v1[-wide]`.
83 +
84 #### 📦️ Important dependency updates
85
86 - update `ipfs-webui` to [v4.7.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.7.0)
docs/config.md
+104 -12
@@ -185,6 +185,10 @@ config file at runtime.
185 - [`Import.HashFunction`](#importhashfunction)
186 - [`Import.BatchMaxNodes`](#importbatchmaxnodes)
187 - [`Import.BatchMaxSize`](#importbatchmaxsize)
188 + - [`Import.UnixFSFileMaxLinks`](#importunixfsfilemaxlinks)
189 + - [`Import.UnixFSDirectoryMaxLinks`](#importunixfsdirectorymaxlinks)
190 + - [`Import.UnixFSHAMTDirectoryMaxFanout`](#importunixfshamtdirectorymaxfanout)
191 + - [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold)
192 - [`Version`](#version)
193 - [`Version.AgentSuffix`](#versionagentsuffix)
194 - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled)
@@ -1199,15 +1203,7 @@ Type: `optionalInteger` (`null` means default which is 10)
1203
1204 ### `Internal.UnixFSShardingSizeThreshold`
1205
1202 -The sharding threshold used internally to decide whether a UnixFS directory should be sharded or not.
1203 -This value is not strictly related to the size of the UnixFS directory block and any increases in
1204 -the threshold should come with being careful that block sizes stay under 2MiB in order for them to be
1205 -reliably transferable through the networking stack (IPFS peers on the public swarm tend to ignore requests for blocks bigger than 2MiB).
1206 -
1207 -Decreasing this value to 1B is functionally equivalent to the previous experimental sharding option to
1208 -shard all directories.
1209 -
1210 -Type: `optionalBytes` (`null` means default which is 256KiB)
1206 +**MOVED:** see [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold)
1207
1208 ## `Ipns`
1209
@@ -2560,6 +2556,80 @@ Default: `20971520` (20MiB)
2556
2557 Type: `optionalInteger`
2558
2559 +### `Import.UnixFSFileMaxLinks`
2560 +
2561 +The maximum number of links that a node part of a UnixFS File can have
2562 +when building the DAG while importing.
2563 +
2564 +This setting controls both the fanout in files that are chunked into several
2565 +blocks and grouped as a Unixfs (dag-pb) DAG.
2566 +
2567 +Default: `174`
2568 +
2569 +Type: `optionalInteger`
2570 +
2571 +### `Import.UnixFSDirectoryMaxLinks`
2572 +
2573 +The maximum number of links that a node part of a UnixFS basic directory can
2574 +have when building the DAG while importing.
2575 +
2576 +This setting controls both the fanout for basic, non-HAMT folder nodes. It
2577 +sets a limit after which directories are converted to a HAMT-based structure.
2578 +
2579 +When unset (0), no limit exists for chilcren. Directories will be converted to
2580 +HAMTs based on their estimated size only.
2581 +
2582 +This setting will cause basic directories to be converted to HAMTs when they
2583 +exceed the maximum number of children. This happens transparently during the
2584 +add process. The fanout of HAMT nodes is controlled by `MaxHAMTFanout`.
2585 +
2586 +Commands affected: `ipfs add`
2587 +
2588 +Default: `0` (no limit, because [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) triggers controls when to switch to HAMT sharding when a directory grows too big)
2589 +
2590 +Type: `optionalInteger`
2591 +
2592 +### `Import.UnixFSHAMTDirectoryMaxFanout`
2593 +
2594 +The maximum number of children that a node part of a Unixfs HAMT directory
2595 +(aka sharded directory) can have.
2596 +
2597 +HAMT directory have unlimited children and are used when basic directories
2598 +become too big or reach `MaxLinks`. A HAMT is an structure made of unixfs
2599 +nodes that store the list of elements in the folder. This option controls the
2600 +maximum number of children that the HAMT nodes can have.
2601 +
2602 +Needs to be a power of two (shard entry size) and multiple of 8 (bitfield size).
2603 +
2604 +Commands affected: `ipfs add`, `ipfs daemon` (globally overrides [`boxo/ipld/unixfs/io.DefaultShardWidth`](https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L30C5-L30C22))
2605 +
2606 +Default: `256`
2607 +
2608 +Type: `optionalInteger`
2609 +
2610 +### `Import.UnixFSHAMTDirectorySizeThreshold`
2611 +
2612 +The sharding threshold to decide whether a basic UnixFS directory
2613 +should be sharded (converted into HAMT Directory) or not.
2614 +
2615 +This value is not strictly related to the size of the UnixFS directory block
2616 +and any increases in the threshold should come with being careful that block
2617 +sizes stay under 2MiB in order for them to be reliably transferable through the
2618 +networking stack. At the time of writing this, IPFS peers on the public swarm
2619 +tend to ignore requests for blocks bigger than 2MiB.
2620 +
2621 +Uses implementation from `boxo/ipld/unixfs/io/directory`, where the size is not
2622 +the *exact* block size of the encoded directory but just the estimated size
2623 +based byte length of DAG-PB Links names and CIDs.
2624 +
2625 +Setting to `1B` is functionally equivalent to always using HAMT (useful in testing).
2626 +
2627 +Commands affected: `ipfs add`, `ipfs daemon` (globally overrides [`boxo/ipld/unixfs/io.HAMTShardingSize`](https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26))
2628 +
2629 +Default: `256KiB` (may change, inspect `DefaultUnixFSHAMTDirectorySizeThreshold` to confirm)
2630 +
2631 +Type: `optionalBytes`
2632 +
2633 ## `Version`
2634
2635 Options to configure agent version announced to the swarm, and leveraging
@@ -2742,16 +2812,38 @@ Disables [Reprovider](#reprovider) system (and announcing to Amino DHT).
2812
2813 Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.
2814
2815 +See <https://github.com/ipfs/kubo/blob/master/config/profile.go> for exact [`Import.*`](#import) settings.
2816 +
2817 > [!NOTE]
2818 > This profile is provided for legacy users and should not be used for new projects.
2819
2820 ### `test-cid-v1` profile
2821
2750 -Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.
2822 +Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256
2823 +and 1 MiB chunks (max 174 links per file, 256 per HAMT node, switch dir to HAMT
2824 +above 256KiB).
2825 +
2826 +See <https://github.com/ipfs/kubo/blob/master/config/profile.go> for exact [`Import.*`](#import) settings.
2827 +
2828 +> [!NOTE]
2829 +> [`Import.*`](#import) settings applied by this profile MAY change in future release. Provided for testing purposes.
2830 +>
2831 +> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details,
2832 +> and provide feedback in [discuss.ipfs.tech/t/should-we-profile-cids](https://discuss.ipfs.tech/t/should-we-profile-cids/18507) or [ipfs/specs#499](https://github.com/ipfs/specs/pull/499).
2833 +
2834 +### `test-cid-v1-wide` profile
2835 +
2836 +Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256
2837 +and 1 MiB chunks and wider file DAGs (max 1024 links per every node type,
2838 +switch dir to HAMT above 1MiB).
2839 +
2840 +See <https://github.com/ipfs/kubo/blob/master/config/profile.go> for exact [`Import.*`](#import) settings.
2841
2842 > [!NOTE]
2753 -> This profile will become the new implicit default, provided for testing purposes.
2754 -> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details.
2843 +> [`Import.*`](#import) settings applied by this profile MAY change in future release. Provided for testing purposes.
2844 +>
2845 +> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details,
2846 +> and provide feedback in [discuss.ipfs.tech/t/should-we-profile-cids](https://discuss.ipfs.tech/t/should-we-profile-cids/18507) or [ipfs/specs#499](https://github.com/ipfs/specs/pull/499).
2847
2848 ## Types
2849
docs/examples/kubo-as-a-library/go.mod
+1 -1
@@ -7,7 +7,7 @@ go 1.24
7 replace github.com/ipfs/kubo => ./../../..
8
9 require (
10 - github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb
10 + github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37
11 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12 github.com/libp2p/go-libp2p v0.41.1
13 github.com/multiformats/go-multiaddr v0.15.0
docs/examples/kubo-as-a-library/go.sum
+2 -2
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
298 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
299 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
300 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
301 -github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
302 -github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
301 +github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37 h1:q3a+2FIbWzZbx/yUqpuG4jLVSa6GvxtRfx9TU5GLiN0=
302 +github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
303 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
304 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
305 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
fuse/readonly/ipfs_test.go
+9 -3
@@ -150,7 +150,10 @@ func TestIpfsStressRead(t *testing.T) {
150
151 // Now make a bunch of dirs
152 for i := 0; i < ndiriter; i++ {
153 - db := uio.NewDirectory(nd.DAG)
153 + db, err := uio.NewDirectory(nd.DAG)
154 + if err != nil {
155 + t.Fatal(err)
156 + }
157 for j := 0; j < 1+rand.Intn(10); j++ {
158 name := fmt.Sprintf("child%d", j)
159
@@ -245,8 +248,11 @@ func TestIpfsBasicDirRead(t *testing.T) {
248 fi, data := randObj(t, nd, 10000)
249
250 // Make a directory and put that file in it
248 - db := uio.NewDirectory(nd.DAG)
249 - err := db.AddChild(nd.Context(), "actual", fi)
251 + db, err := uio.NewDirectory(nd.DAG)
252 + if err != nil {
253 + t.Fatal(err)
254 + }
255 + err = db.AddChild(nd.Context(), "actual", fi)
256 if err != nil {
257 t.Fatal(err)
258 }
go.mod
+1 -1
@@ -21,7 +21,7 @@ require (
21 github.com/hashicorp/go-version v1.7.0
22 github.com/ipfs-shipyard/nopfs v0.0.14
23 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
24 - github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb
24 + github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37
25 github.com/ipfs/go-block-format v0.2.0
26 github.com/ipfs/go-cid v0.5.0
27 github.com/ipfs/go-cidutil v0.1.0
go.sum
+2 -2
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
362 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
363 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
364 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
365 -github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
366 -github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
365 +github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37 h1:q3a+2FIbWzZbx/yUqpuG4jLVSa6GvxtRfx9TU5GLiN0=
366 +github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
367 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
368 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
369 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
test/cli/add_test.go
+335 -2
@@ -1,10 +1,17 @@
1 package cli
2
3 import (
4 + "io"
5 + "os"
6 + "path/filepath"
7 + "strings"
8 "testing"
9
10 + "github.com/dustin/go-humanize"
11 "github.com/ipfs/kubo/config"
12 "github.com/ipfs/kubo/test/cli/harness"
13 + "github.com/ipfs/kubo/test/cli/testutils"
14 + "github.com/stretchr/testify/assert"
15 "github.com/stretchr/testify/require"
16 )
17
@@ -19,6 +26,11 @@ func TestAdd(t *testing.T) {
26 shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6"
27 )
28
29 + const (
30 + cidV0Length = 34 // cidv0 sha2-256
31 + cidV1Length = 36 // cidv1 sha2-256
32 + )
33 +
34 t.Run("produced cid version: implicit default (CIDv0)", func(t *testing.T) {
35 t.Parallel()
36 node := harness.NewT(t).NewNode().Init().StartDaemon()
@@ -96,6 +108,33 @@ func TestAdd(t *testing.T) {
108 require.Equal(t, shortStringCidV1NoRawLeaves, cidStr)
109 })
110
111 + t.Run("produced unixfs max file links: command flag --max-file-links overrides configuration in Import.UnixFSFileMaxLinks", func(t *testing.T) {
112 + t.Parallel()
113 +
114 + //
115 + // UnixFSChunker=size-262144 (256KiB)
116 + // Import.UnixFSFileMaxLinks=174
117 + node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0") // legacy-cid-v0 for determinism across all params
118 + node.UpdateConfig(func(cfg *config.Config) {
119 + cfg.Import.UnixFSChunker = *config.NewOptionalString("size-262144") // 256 KiB chunks
120 + cfg.Import.UnixFSFileMaxLinks = *config.NewOptionalInteger(174) // max 174 per level
121 + })
122 + node.StartDaemon()
123 + defer node.StopDaemon()
124 +
125 + // Add 174MiB file:
126 + // 1024 * 256KiB should fit in single layer
127 + seed := shortString
128 + cidStr := node.IPFSAddDeterministic("262144KiB", seed, "--max-file-links", "1024")
129 + root, err := node.InspectPBNode(cidStr)
130 + assert.NoError(t, err)
131 +
132 + // Expect 1024 links due to cli parameter raising link limit from 174 to 1024
133 + require.Equal(t, 1024, len(root.Links))
134 + // expect same CID every time
135 + require.Equal(t, "QmbBftNHWmjSWKLC49dMVrfnY8pjrJYntiAXirFJ7oJrNk", cidStr)
136 + })
137 +
138 t.Run("ipfs init --profile=legacy-cid-v0 sets config that produces legacy CIDv0", func(t *testing.T) {
139 t.Parallel()
140 node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0")
@@ -106,13 +145,307 @@ func TestAdd(t *testing.T) {
145 require.Equal(t, shortStringCidV0, cidStr)
146 })
147
109 - t.Run("ipfs init --profile=test-cid-v1 produces modern CIDv1", func(t *testing.T) {
148 + t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSChunker=size-262144 and UnixFSFileMaxLinks", func(t *testing.T) {
149 + t.Parallel()
150 + seed := "v0-seed"
151 + profile := "--profile=legacy-cid-v0"
152 +
153 + t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) {
154 + t.Parallel()
155 + node := harness.NewT(t).NewNode().Init(profile)
156 + node.StartDaemon()
157 + defer node.StopDaemon()
158 + // Add 44544KiB file:
159 + // 174 * 256KiB should fit in single DAG layer
160 + cidStr := node.IPFSAddDeterministic("44544KiB", seed)
161 + root, err := node.InspectPBNode(cidStr)
162 + assert.NoError(t, err)
163 + require.Equal(t, 174, len(root.Links))
164 + // expect same CID every time
165 + require.Equal(t, "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", cidStr)
166 + })
167 +
168 + t.Run("above UnixFSFileMaxLinks=174", func(t *testing.T) {
169 + t.Parallel()
170 + node := harness.NewT(t).NewNode().Init(profile)
171 + node.StartDaemon()
172 + defer node.StopDaemon()
173 + // add 256KiB (one more block), it should force rebalancing DAG and moving most to second layer
174 + cidStr := node.IPFSAddDeterministic("44800KiB", seed)
175 + root, err := node.InspectPBNode(cidStr)
176 + assert.NoError(t, err)
177 + require.Equal(t, 2, len(root.Links))
178 + // expect same CID every time
179 + require.Equal(t, "QmepeWtdmS1hHXx1oZXsPUv6bMrfRRKfZcoPPU4eEfjnbf", cidStr)
180 + })
181 + })
182 +
183 + t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
184 + t.Parallel()
185 + seed := "hamt-legacy-cid-v0"
186 + profile := "--profile=legacy-cid-v0"
187 +
188 + t.Run("under UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
189 + t.Parallel()
190 + node := harness.NewT(t).NewNode().Init(profile)
191 + node.StartDaemon()
192 + defer node.StopDaemon()
193 +
194 + randDir, err := os.MkdirTemp(node.Dir, seed)
195 + require.NoError(t, err)
196 +
197 + // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
198 + err = createDirectoryForHAMT(randDir, cidV0Length, "255KiB", seed)
199 + require.NoError(t, err)
200 + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
201 +
202 + // Confirm the number of links is more than UnixFSHAMTDirectorySizeThreshold (indicating regular "basic" directory"
203 + root, err := node.InspectPBNode(cidStr)
204 + assert.NoError(t, err)
205 + require.Equal(t, 903, len(root.Links))
206 + })
207 +
208 + t.Run("above UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
209 + t.Parallel()
210 + node := harness.NewT(t).NewNode().Init(profile)
211 + node.StartDaemon()
212 + defer node.StopDaemon()
213 +
214 + randDir, err := os.MkdirTemp(node.Dir, seed)
215 + require.NoError(t, err)
216 +
217 + // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
218 + err = createDirectoryForHAMT(randDir, cidV0Length, "257KiB", seed)
219 + require.NoError(t, err)
220 + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
221 +
222 + // Confirm this time, the number of links is less than UnixFSHAMTDirectorySizeThreshold
223 + root, err := node.InspectPBNode(cidStr)
224 + assert.NoError(t, err)
225 + require.Equal(t, 252, len(root.Links))
226 + })
227 + })
228 +
229 + t.Run("ipfs init --profile=test-cid-v1 produces CIDv1 with raw leaves", func(t *testing.T) {
230 t.Parallel()
231 node := harness.NewT(t).NewNode().Init("--profile=test-cid-v1")
232 node.StartDaemon()
233 defer node.StopDaemon()
234
235 cidStr := node.IPFSAddStr(shortString)
116 - require.Equal(t, shortStringCidV1, cidStr)
236 + require.Equal(t, shortStringCidV1, cidStr) // raw leaf
237 + })
238 +
239 + t.Run("ipfs init --profile=test-cid-v1 applies UnixFSChunker=size-1048576", func(t *testing.T) {
240 + t.Parallel()
241 + seed := "v1-seed"
242 + profile := "--profile=test-cid-v1"
243 +
244 + t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) {
245 + t.Parallel()
246 + node := harness.NewT(t).NewNode().Init(profile)
247 + node.StartDaemon()
248 + defer node.StopDaemon()
249 + // Add 174MiB file:
250 + // 174 * 1MiB should fit in single layer
251 + cidStr := node.IPFSAddDeterministic("174MiB", seed)
252 + root, err := node.InspectPBNode(cidStr)
253 + assert.NoError(t, err)
254 + require.Equal(t, 174, len(root.Links))
255 + // expect same CID every time
256 + require.Equal(t, "bafybeigwduxcf2aawppv3isnfeshnimkyplvw3hthxjhr2bdeje4tdaicu", cidStr)
257 + })
258 +
259 + t.Run("above UnixFSFileMaxLinks=174", func(t *testing.T) {
260 + t.Parallel()
261 + node := harness.NewT(t).NewNode().Init(profile)
262 + node.StartDaemon()
263 + defer node.StopDaemon()
264 + // add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer
265 + cidStr := node.IPFSAddDeterministic("175MiB", seed)
266 + root, err := node.InspectPBNode(cidStr)
267 + assert.NoError(t, err)
268 + require.Equal(t, 2, len(root.Links))
269 + // expect same CID every time
270 + require.Equal(t, "bafybeidhd7lo2n2v7lta5yamob3xwhbxcczmmtmhquwhjesi35jntf7mpu", cidStr)
271 + })
272 + })
273 +
274 + t.Run("ipfs init --profile=test-cid-v1 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
275 + t.Parallel()
276 + seed := "hamt-cid-v1"
277 + profile := "--profile=test-cid-v1"
278 +
279 + t.Run("under UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
280 + t.Parallel()
281 + node := harness.NewT(t).NewNode().Init(profile)
282 + node.StartDaemon()
283 + defer node.StopDaemon()
284 +
285 + randDir, err := os.MkdirTemp(node.Dir, seed)
286 + require.NoError(t, err)
287 +
288 + // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
289 + err = createDirectoryForHAMT(randDir, cidV1Length, "255KiB", seed)
290 + require.NoError(t, err)
291 + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
292 +
293 + // Confirm the number of links is more than UnixFSHAMTDirectoryMaxFanout (indicating regular "basic" directory"
294 + root, err := node.InspectPBNode(cidStr)
295 + assert.NoError(t, err)
296 + require.Equal(t, 897, len(root.Links))
297 + })
298 +
299 + t.Run("above UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
300 + t.Parallel()
301 + node := harness.NewT(t).NewNode().Init(profile)
302 + node.StartDaemon()
303 + defer node.StopDaemon()
304 +
305 + randDir, err := os.MkdirTemp(node.Dir, seed)
306 + require.NoError(t, err)
307 +
308 + // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
309 + err = createDirectoryForHAMT(randDir, cidV1Length, "257KiB", seed)
310 + require.NoError(t, err)
311 + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
312 +
313 + // Confirm this time, the number of links is less than UnixFSHAMTDirectoryMaxFanout
314 + root, err := node.InspectPBNode(cidStr)
315 + assert.NoError(t, err)
316 + require.Equal(t, 252, len(root.Links))
317 + })
318 + })
319 +
320 + t.Run("ipfs init --profile=test-cid-v1-wide applies UnixFSChunker=size-1048576 and UnixFSFileMaxLinks=1024", func(t *testing.T) {
321 + t.Parallel()
322 + seed := "v1-seed-1024"
323 + profile := "--profile=test-cid-v1-wide"
324 +
325 + t.Run("under UnixFSFileMaxLinks=1024", func(t *testing.T) {
326 + t.Parallel()
327 + node := harness.NewT(t).NewNode().Init(profile)
328 + node.StartDaemon()
329 + defer node.StopDaemon()
330 + // Add 174MiB file:
331 + // 1024 * 1MiB should fit in single layer
332 + cidStr := node.IPFSAddDeterministic("1024MiB", seed)
333 + root, err := node.InspectPBNode(cidStr)
334 + assert.NoError(t, err)
335 + require.Equal(t, 1024, len(root.Links))
336 + // expect same CID every time
337 + require.Equal(t, "bafybeiej5w63ir64oxgkr5htqmlerh5k2rqflurn2howimexrlkae64xru", cidStr)
338 + })
339 +
340 + t.Run("above UnixFSFileMaxLinks=1024", func(t *testing.T) {
341 + t.Parallel()
342 + node := harness.NewT(t).NewNode().Init(profile)
343 + node.StartDaemon()
344 + defer node.StopDaemon()
345 + // add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer
346 + cidStr := node.IPFSAddDeterministic("1025MiB", seed)
347 + root, err := node.InspectPBNode(cidStr)
348 + assert.NoError(t, err)
349 + require.Equal(t, 2, len(root.Links))
350 + // expect same CID every time
351 + require.Equal(t, "bafybeieilp2qx24pe76hxrxe6bpef5meuxto3kj5dd6mhb5kplfeglskdm", cidStr)
352 + })
353 })
354 +
355 + t.Run("ipfs init --profile=test-cid-v1-wide applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) {
356 + t.Parallel()
357 + seed := "hamt-cid-v1"
358 + profile := "--profile=test-cid-v1-wide"
359 +
360 + t.Run("under UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) {
361 + t.Parallel()
362 + node := harness.NewT(t).NewNode().Init(profile)
363 + node.StartDaemon()
364 + defer node.StopDaemon()
365 +
366 + randDir, err := os.MkdirTemp(node.Dir, seed)
367 + require.NoError(t, err)
368 +
369 + // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
370 + err = createDirectoryForHAMT(randDir, cidV1Length, "1023KiB", seed)
371 + require.NoError(t, err)
372 + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
373 +
374 + // Confirm the number of links is more than UnixFSHAMTDirectoryMaxFanout (indicating regular "basic" directory"
375 + root, err := node.InspectPBNode(cidStr)
376 + assert.NoError(t, err)
377 + require.Equal(t, 3599, len(root.Links))
378 + })
379 +
380 + t.Run("above UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) {
381 + t.Parallel()
382 + node := harness.NewT(t).NewNode().Init(profile)
383 + node.StartDaemon()
384 + defer node.StopDaemon()
385 +
386 + randDir, err := os.MkdirTemp(node.Dir, seed)
387 + require.NoError(t, err)
388 +
389 + // Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
390 + err = createDirectoryForHAMT(randDir, cidV1Length, "1025KiB", seed)
391 + require.NoError(t, err)
392 + cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
393 +
394 + // Confirm this time, the number of links is less than UnixFSHAMTDirectoryMaxFanout
395 + root, err := node.InspectPBNode(cidStr)
396 + assert.NoError(t, err)
397 + require.Equal(t, 992, len(root.Links))
398 + })
399 + })
400 +
401 +}
402 +
403 +// createDirectoryForHAMT aims to create enough files with long names for the directory block to be close to the UnixFSHAMTDirectorySizeThreshold.
404 +// The calculation is based on boxo's HAMTShardingSize and sizeBelowThreshold which calculates ballpark size of the block
405 +// by adding length of link names and the binary cid length.
406 +// See https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L491
407 +func createDirectoryForHAMT(dirPath string, cidLength int, unixfsNodeSizeTarget, seed string) error {
408 + hamtThreshold, err := humanize.ParseBytes(unixfsNodeSizeTarget)
409 + if err != nil {
410 + return err
411 + }
412 +
413 + // Calculate how many files with long filenames are needed to hit UnixFSHAMTDirectorySizeThreshold
414 + nameLen := 255 // max that works across windows/macos/linux
415 + alphabetLen := len(testutils.AlphabetEasy)
416 + numFiles := int(hamtThreshold) / (nameLen + cidLength)
417 +
418 + // Deterministic pseudo-random bytes for static CID
419 + drand, err := testutils.DeterministicRandomReader(unixfsNodeSizeTarget, seed)
420 + if err != nil {
421 + return err
422 + }
423 +
424 + // Create necessary files in a single, flat directory
425 + for i := 0; i < numFiles; i++ {
426 + buf := make([]byte, nameLen)
427 + _, err := io.ReadFull(drand, buf)
428 + if err != nil {
429 + return err
430 + }
431 +
432 + // Convert deterministic pseudo-random bytes to ASCII
433 + var sb strings.Builder
434 +
435 + for _, b := range buf {
436 + // Map byte to printable ASCII range (33-126)
437 + char := testutils.AlphabetEasy[int(b)%alphabetLen]
438 + sb.WriteRune(char)
439 + }
440 + filename := sb.String()[:nameLen]
441 + filePath := filepath.Join(dirPath, filename)
442 +
443 + // Create empty file
444 + f, err := os.Create(filePath)
445 + if err != nil {
446 + return err
447 + }
448 + f.Close()
449 + }
450 + return nil
451 }
test/cli/harness/ipfs.go
+23
@@ -76,6 +76,17 @@ func (n *Node) IPFSAddStr(content string, args ...string) string {
76 return n.IPFSAdd(strings.NewReader(content), args...)
77 }
78
79 +// IPFSAddDeterministic produces a CID of a file of a certain size, filled with deterministically generated bytes based on some seed.
80 +// This ensures deterministic CID on the other end, that can be used in tests.
81 +func (n *Node) IPFSAddDeterministic(size string, seed string, args ...string) string {
82 + log.Debugf("node %d adding %s of deterministic pseudo-random data with seed %q and args: %v", n.ID, size, seed, args)
83 + reader, err := DeterministicRandomReader(size, seed)
84 + if err != nil {
85 + panic(err)
86 + }
87 + return n.IPFSAdd(reader, args...)
88 +}
89 +
90 func (n *Node) IPFSAdd(content io.Reader, args ...string) string {
91 log.Debugf("node %d adding with args: %v", n.ID, args)
92 fullArgs := []string{"add", "-q"}
@@ -108,3 +119,15 @@ func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) erro
119 })
120 return res.Err
121 }
122 +
123 +/*
124 +func (n *Node) IPFSDagExport(cid string, car *os.File) error {
125 + log.Debugf("node %d dag export of %s to %q with args: %v", n.ID, cid, car.Name())
126 + res := n.Runner.MustRun(RunRequest{
127 + Path: n.IPFSBin,
128 + Args: []string{"dag", "export", cid},
129 + CmdOpts: []CmdOpt{RunWithStdout(car)},
130 + })
131 + return res.Err
132 +}
133 +*/
test/cli/harness/pbinspect.go new
+54
@@ -0,0 +1,54 @@
1 +package harness
2 +
3 +import (
4 + "bytes"
5 + "encoding/json"
6 +)
7 +
8 +// InspectPBNode uses dag-json output of 'ipfs dag get' to inspect
9 +// "Logical Format" of DAG-PB as defined in
10 +// https://web.archive.org/web/20250403194752/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format
11 +// (mainly used for inspecting Links without depending on any libraries)
12 +func (n *Node) InspectPBNode(cid string) (PBNode, error) {
13 + log.Debugf("node %d dag get %s as dag-json", n.ID, cid)
14 +
15 + var root PBNode
16 + var dagJsonOutput bytes.Buffer
17 + res := n.Runner.MustRun(RunRequest{
18 + Path: n.IPFSBin,
19 + Args: []string{"dag", "get", "--output-codec=dag-json", cid},
20 + CmdOpts: []CmdOpt{RunWithStdout(&dagJsonOutput)},
21 + })
22 + if res.Err != nil {
23 + return root, res.Err
24 + }
25 +
26 + err := json.Unmarshal(dagJsonOutput.Bytes(), &root)
27 + if err != nil {
28 + return root, err
29 + }
30 + return root, nil
31 +
32 +}
33 +
34 +// Define structs to match the JSON for
35 +type PBHash struct {
36 + Slash string `json:"/"`
37 +}
38 +
39 +type PBLink struct {
40 + Hash PBHash `json:"Hash"`
41 + Name string `json:"Name"`
42 + Tsize int `json:"Tsize"`
43 +}
44 +
45 +type PBData struct {
46 + Slash struct {
47 + Bytes string `json:"bytes"`
48 + } `json:"/"`
49 +}
50 +
51 +type PBNode struct {
52 + Data PBData `json:"Data"`
53 + Links []PBLink `json:"Links"`
54 +}
test/cli/testutils/random_deterministic.go new
+46
@@ -0,0 +1,46 @@
1 +package testutils
2 +
3 +import (
4 + "crypto/sha256"
5 + "io"
6 +
7 + "github.com/dustin/go-humanize"
8 + "golang.org/x/crypto/chacha20"
9 +)
10 +
11 +type randomReader struct {
12 + cipher *chacha20.Cipher
13 + remaining int64
14 +}
15 +
16 +func (r *randomReader) Read(p []byte) (int, error) {
17 + if r.remaining <= 0 {
18 + return 0, io.EOF
19 + }
20 + n := int64(len(p))
21 + if n > r.remaining {
22 + n = r.remaining
23 + }
24 + // Generate random bytes directly into the provided buffer
25 + r.cipher.XORKeyStream(p[:n], make([]byte, n))
26 + r.remaining -= n
27 + return int(n), nil
28 +}
29 +
30 +// createRandomReader produces specified number of pseudo-random bytes
31 +// from a seed.
32 +func DeterministicRandomReader(sizeStr string, seed string) (io.Reader, error) {
33 + size, err := humanize.ParseBytes(sizeStr)
34 + if err != nil {
35 + return nil, err
36 + }
37 + // Hash the seed string to a 32-byte key for ChaCha20
38 + key := sha256.Sum256([]byte(seed))
39 + // Use ChaCha20 for deterministic random bytes
40 + var nonce [chacha20.NonceSize]byte // Zero nonce for simplicity
41 + cipher, err := chacha20.NewUnauthenticatedCipher(key[:chacha20.KeySize], nonce[:])
42 + if err != nil {
43 + return nil, err
44 + }
45 + return &randomReader{cipher: cipher, remaining: int64(size)}, nil
46 +}
test/cli/testutils/random_files.go
+16 -11
@@ -24,20 +24,22 @@ type RandFiles struct {
24 FanoutFiles int // how many files per dir
25 FanoutDirs int // how many dirs per dir
26
27 - RandomSize bool // randomize file sizes
28 - RandomFanout bool // randomize fanout numbers
27 + RandomSize bool // randomize file sizes
28 + RandomNameSize bool // randomize filename lengths
29 + RandomFanout bool // randomize fanout numbers
30 }
31
32 func NewRandFiles() *RandFiles {
33 return &RandFiles{
33 - Rand: rand.New(rand.NewSource(time.Now().UnixNano())),
34 - FileSize: 4096,
35 - FilenameSize: 16,
36 - Alphabet: AlphabetEasy,
37 - FanoutDepth: 2,
38 - FanoutDirs: 5,
39 - FanoutFiles: 10,
40 - RandomSize: true,
34 + Rand: rand.New(rand.NewSource(time.Now().UnixNano())),
35 + FileSize: 4096,
36 + FilenameSize: 16,
37 + Alphabet: AlphabetEasy,
38 + FanoutDepth: 2,
39 + FanoutDirs: 5,
40 + FanoutFiles: 10,
41 + RandomSize: true,
42 + RandomNameSize: true,
43 }
44 }
45
@@ -83,7 +85,10 @@ func (r *RandFiles) WriteRandomFile(root string) error {
85 filesize = r.Rand.Int63n(filesize) + 1
86 }
87
86 - n := rand.Intn(r.FilenameSize-4) + 4
88 + n := r.FilenameSize
89 + if r.RandomNameSize {
90 + n = rand.Intn(r.FilenameSize-4) + 4
91 + }
92 name := r.RandomFilename(n)
93 filepath := path.Join(root, name)
94 f, err := os.Create(filepath)
test/dependencies/go.mod
+5 -1
@@ -33,6 +33,7 @@ require (
33 github.com/Masterminds/semver/v3 v3.2.1 // indirect
34 github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect
35 github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
36 + github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
37 github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
38 github.com/alexkohler/prealloc v1.0.0 // indirect
39 github.com/alingse/asasalint v0.0.11 // indirect
@@ -57,6 +58,7 @@ require (
58 github.com/chavacava/garif v0.1.0 // indirect
59 github.com/ckaznocha/intrange v0.1.2 // indirect
60 github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
61 + github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect
62 github.com/curioswitch/go-reassign v0.2.0 // indirect
63 github.com/daixiang0/gci v0.13.4 // indirect
64 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -116,7 +118,8 @@ require (
118 github.com/huin/goupnp v1.3.0 // indirect
119 github.com/inconshreveable/mousetrap v1.1.0 // indirect
120 github.com/ipfs/bbloom v0.0.4 // indirect
119 - github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb // indirect
121 + github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37 // indirect
122 + github.com/ipfs/go-bitfield v1.1.0 // indirect
123 github.com/ipfs/go-block-format v0.2.0 // indirect
124 github.com/ipfs/go-cid v0.5.0 // indirect
125 github.com/ipfs/go-datastore v0.8.2 // indirect
@@ -273,6 +276,7 @@ require (
276 github.com/urfave/cli v1.22.16 // indirect
277 github.com/uudashr/gocognit v1.1.3 // indirect
278 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
279 + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
280 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
281 github.com/wlynxg/anet v0.0.5 // indirect
282 github.com/xen0n/gosmopolitan v1.2.2 // indirect
test/dependencies/go.sum
+10 -2
@@ -43,6 +43,8 @@ github.com/alecthomas/go-check-sumtype v0.1.4 h1:WCvlB3l5Vq5dZQTFmodqL2g68uHiSww
43 github.com/alecthomas/go-check-sumtype v0.1.4/go.mod h1:WyYPfhfkdhyrdaligV6svFopZV8Lqdzn5pyVBaV6jhQ=
44 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
45 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
46 +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
47 +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
48 github.com/alexkohler/nakedret/v2 v2.0.4 h1:yZuKmjqGi0pSmjGpOC016LtPJysIL0WEUiaXW5SUnNg=
49 github.com/alexkohler/nakedret/v2 v2.0.4/go.mod h1:bF5i0zF2Wo2o4X4USt9ntUWve6JbFv02Ff4vlkmS/VU=
50 github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pOcUuw=
@@ -105,6 +107,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
107 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
108 github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
109 github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
110 +github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4=
111 +github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE=
112 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
113 github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
114 github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
@@ -294,8 +298,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
298 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
299 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
300 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
297 -github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
298 -github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
301 +github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37 h1:q3a+2FIbWzZbx/yUqpuG4jLVSa6GvxtRfx9TU5GLiN0=
302 +github.com/ipfs/boxo v0.29.2-0.20250415191135-dc60fe747c37/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
303 +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
304 +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
305 github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
306 github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
307 github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
@@ -767,6 +773,8 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSD
773 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
774 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
775 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
776 +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
777 +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
778 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
779 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
780 github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
test/sharness/t0032-mount-sharded.sh
+1 -1
@@ -16,7 +16,7 @@ fi
16 test_init_ipfs
17
18 test_expect_success 'force sharding' '
19 - ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1B\""
19 + ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\""
20 '
21
22 test_launch_ipfs_daemon
test/sharness/t0250-files-api.sh
+2 -2
@@ -849,7 +849,7 @@ tests_for_files_api "with-daemon"
849 test_kill_ipfs_daemon
850
851 test_expect_success "enable sharding in config" '
852 - ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1B\""
852 + ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\""
853 '
854
855 test_launch_ipfs_daemon_without_network
@@ -880,7 +880,7 @@ test_expect_success "set up automatic sharding/unsharding data" '
880 '
881
882 test_expect_success "reset automatic sharding" '
883 - ipfs config --json Internal.UnixFSShardingSizeThreshold null
883 + ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold null
884 '
885
886 test_launch_ipfs_daemon_without_network
test/sharness/t0260-sharding.sh
+2 -2
@@ -34,7 +34,7 @@ test_init_ipfs
34 UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N"
35
36 test_expect_success "force sharding off" '
37 -ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1G\""
37 +ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1G\""
38 '
39
40 test_add_dir "$UNSHARDED"
@@ -46,7 +46,7 @@ test_add_dir "$UNSHARDED"
46 test_kill_ipfs_daemon
47
48 test_expect_success "force sharding on" '
49 - ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1B\""
49 + ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\""
50 '
51
52 SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL"