| 1 | package config |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "io" |
| 6 | "strconv" |
| 7 | "strings" |
| 8 | |
| 9 | chunk "github.com/ipfs/boxo/chunker" |
| 10 | merkledag "github.com/ipfs/boxo/ipld/merkledag" |
| 11 | "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" |
| 12 | uio "github.com/ipfs/boxo/ipld/unixfs/io" |
| 13 | "github.com/ipfs/boxo/mfs" |
| 14 | "github.com/ipfs/boxo/verifcid" |
| 15 | cid "github.com/ipfs/go-cid" |
| 16 | mh "github.com/multiformats/go-multihash" |
| 17 | ) |
| 18 | |
| 19 | const ( |
| 20 | DefaultCidVersion = 0 |
| 21 | DefaultUnixFSRawLeaves = false |
| 22 | DefaultUnixFSChunker = "size-262144" |
| 23 | DefaultHashFunction = "sha2-256" |
| 24 | DefaultFastProvideRoot = true |
| 25 | DefaultFastProvideWait = false |
| 26 | DefaultFastProvideDAG = false |
| 27 | |
| 28 | DefaultUnixFSHAMTDirectorySizeThreshold = 262144 // 256KiB - https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26 |
| 29 | |
| 30 | // DefaultBatchMaxNodes controls the maximum number of nodes in a |
| 31 | // write-batch. The total size of the batch is limited by |
| 32 | // BatchMaxnodes and BatchMaxSize. |
| 33 | DefaultBatchMaxNodes = 128 |
| 34 | // DefaultBatchMaxSize controls the maximum size of a single |
| 35 | // write-batch. The total size of the batch is limited by |
| 36 | // BatchMaxnodes and BatchMaxSize. |
| 37 | DefaultBatchMaxSize = 100 << 20 // 20MiB |
| 38 | |
| 39 | // HAMTSizeEstimation values for Import.UnixFSHAMTDirectorySizeEstimation |
| 40 | HAMTSizeEstimationLinks = "links" // legacy: estimate using link names + CID byte lengths (default) |
| 41 | HAMTSizeEstimationBlock = "block" // full serialized dag-pb block size |
| 42 | HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely |
| 43 | |
| 44 | // DAGLayout values for Import.UnixFSDAGLayout |
| 45 | DAGLayoutBalanced = "balanced" // balanced DAG layout (default) |
| 46 | DAGLayoutTrickle = "trickle" // trickle DAG layout |
| 47 | |
| 48 | DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior |
| 49 | DefaultUnixFSDAGLayout = DAGLayoutBalanced // balanced DAG layout |
| 50 | DefaultUnixFSIncludeEmptyDirs = true // include empty directories |
| 51 | ) |
| 52 | |
| 53 | var ( |
| 54 | DefaultUnixFSFileMaxLinks = int64(helpers.DefaultLinksPerBlock) |
| 55 | DefaultUnixFSDirectoryMaxLinks = int64(0) |
| 56 | DefaultUnixFSHAMTDirectoryMaxFanout = int64(uio.DefaultShardWidth) |
| 57 | ) |
| 58 | |
| 59 | // Import configures the default options for ingesting data. This affects commands |
| 60 | // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'. |
| 61 | type Import struct { |
| 62 | CidVersion OptionalInteger |
| 63 | UnixFSRawLeaves Flag |
| 64 | UnixFSChunker OptionalString |
| 65 | HashFunction OptionalString |
| 66 | UnixFSFileMaxLinks OptionalInteger |
| 67 | UnixFSDirectoryMaxLinks OptionalInteger |
| 68 | UnixFSHAMTDirectoryMaxFanout OptionalInteger |
| 69 | UnixFSHAMTDirectorySizeThreshold OptionalBytes |
| 70 | UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled" |
| 71 | UnixFSDAGLayout OptionalString // "balanced" or "trickle" |
| 72 | BatchMaxNodes OptionalInteger |
| 73 | BatchMaxSize OptionalInteger |
| 74 | FastProvideRoot Flag |
| 75 | FastProvideDAG Flag |
| 76 | FastProvideWait Flag |
| 77 | } |
| 78 | |
| 79 | // ValidateImportConfig validates the Import configuration according to UnixFS spec requirements. |
| 80 | // See: https://specs.ipfs.tech/unixfs/#hamt-structure-and-parameters |
| 81 | func ValidateImportConfig(cfg *Import) error { |
| 82 | // Validate CidVersion |
| 83 | if !cfg.CidVersion.IsDefault() { |
| 84 | cidVer := cfg.CidVersion.WithDefault(DefaultCidVersion) |
| 85 | if cidVer != 0 && cidVer != 1 { |
| 86 | return fmt.Errorf("Import.CidVersion must be 0 or 1, got %d", cidVer) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | // Validate UnixFSFileMaxLinks |
| 91 | if !cfg.UnixFSFileMaxLinks.IsDefault() { |
| 92 | maxLinks := cfg.UnixFSFileMaxLinks.WithDefault(DefaultUnixFSFileMaxLinks) |
| 93 | if maxLinks <= 0 { |
| 94 | return fmt.Errorf("Import.UnixFSFileMaxLinks must be positive, got %d", maxLinks) |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | // Validate UnixFSDirectoryMaxLinks |
| 99 | if !cfg.UnixFSDirectoryMaxLinks.IsDefault() { |
| 100 | maxLinks := cfg.UnixFSDirectoryMaxLinks.WithDefault(DefaultUnixFSDirectoryMaxLinks) |
| 101 | if maxLinks < 0 { |
| 102 | return fmt.Errorf("Import.UnixFSDirectoryMaxLinks must be non-negative, got %d", maxLinks) |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // Validate UnixFSHAMTDirectoryMaxFanout if set |
| 107 | if !cfg.UnixFSHAMTDirectoryMaxFanout.IsDefault() { |
| 108 | fanout := cfg.UnixFSHAMTDirectoryMaxFanout.WithDefault(DefaultUnixFSHAMTDirectoryMaxFanout) |
| 109 | |
| 110 | // Valid values are powers of 2 between 8 and 1024: 8, 16, 32, 64, 128, 256, 512, 1024 |
| 111 | if fanout < 8 || !isPowerOfTwo(fanout) || fanout > 1024 { |
| 112 | return fmt.Errorf("Import.UnixFSHAMTDirectoryMaxFanout must be a power of 2, between 8 and 1024 (got %d)", fanout) |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | // Validate BatchMaxNodes |
| 117 | if !cfg.BatchMaxNodes.IsDefault() { |
| 118 | maxNodes := cfg.BatchMaxNodes.WithDefault(DefaultBatchMaxNodes) |
| 119 | if maxNodes <= 0 { |
| 120 | return fmt.Errorf("Import.BatchMaxNodes must be positive, got %d", maxNodes) |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | // Validate BatchMaxSize |
| 125 | if !cfg.BatchMaxSize.IsDefault() { |
| 126 | maxSize := cfg.BatchMaxSize.WithDefault(DefaultBatchMaxSize) |
| 127 | if maxSize <= 0 { |
| 128 | return fmt.Errorf("Import.BatchMaxSize must be positive, got %d", maxSize) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | // Validate UnixFSChunker format |
| 133 | if !cfg.UnixFSChunker.IsDefault() { |
| 134 | chunker := cfg.UnixFSChunker.WithDefault(DefaultUnixFSChunker) |
| 135 | if !isValidChunker(chunker) { |
| 136 | return fmt.Errorf("Import.UnixFSChunker invalid format: %q (expected \"size-<bytes>\", \"rabin-<min>-<avg>-<max>\", or \"buzhash\")", chunker) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | // Validate HashFunction |
| 141 | if !cfg.HashFunction.IsDefault() { |
| 142 | hashFunc := cfg.HashFunction.WithDefault(DefaultHashFunction) |
| 143 | hashCode, ok := mh.Names[strings.ToLower(hashFunc)] |
| 144 | if !ok { |
| 145 | return fmt.Errorf("Import.HashFunction unrecognized: %q", hashFunc) |
| 146 | } |
| 147 | // Check if the hash is allowed by verifcid |
| 148 | if !verifcid.DefaultAllowlist.IsAllowed(hashCode) { |
| 149 | return fmt.Errorf("Import.HashFunction %q is not allowed for use in IPFS", hashFunc) |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | // Validate UnixFSHAMTDirectorySizeEstimation |
| 154 | if !cfg.UnixFSHAMTDirectorySizeEstimation.IsDefault() { |
| 155 | est := cfg.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) |
| 156 | switch est { |
| 157 | case HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled: |
| 158 | // valid |
| 159 | default: |
| 160 | return fmt.Errorf("Import.UnixFSHAMTDirectorySizeEstimation must be %q, %q, or %q, got %q", |
| 161 | HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled, est) |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // Validate UnixFSDAGLayout |
| 166 | if !cfg.UnixFSDAGLayout.IsDefault() { |
| 167 | layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout) |
| 168 | switch layout { |
| 169 | case DAGLayoutBalanced, DAGLayoutTrickle: |
| 170 | // valid |
| 171 | default: |
| 172 | return fmt.Errorf("Import.UnixFSDAGLayout must be %q or %q, got %q", |
| 173 | DAGLayoutBalanced, DAGLayoutTrickle, layout) |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | return nil |
| 178 | } |
| 179 | |
| 180 | // isPowerOfTwo checks if a number is a power of 2 |
| 181 | func isPowerOfTwo(n int64) bool { |
| 182 | return n > 0 && (n&(n-1)) == 0 |
| 183 | } |
| 184 | |
| 185 | // isValidChunker validates chunker format |
| 186 | func isValidChunker(chunker string) bool { |
| 187 | if chunker == "buzhash" { |
| 188 | return true |
| 189 | } |
| 190 | |
| 191 | // Check for size-<bytes> format |
| 192 | if sizeStr, ok := strings.CutPrefix(chunker, "size-"); ok { |
| 193 | if sizeStr == "" { |
| 194 | return false |
| 195 | } |
| 196 | // Check if it's a valid positive integer (no negative sign allowed) |
| 197 | if sizeStr[0] == '-' { |
| 198 | return false |
| 199 | } |
| 200 | size, err := strconv.Atoi(sizeStr) |
| 201 | // Size must be positive (not zero) |
| 202 | return err == nil && size > 0 |
| 203 | } |
| 204 | |
| 205 | // Check for rabin-<min>-<avg>-<max> format |
| 206 | if strings.HasPrefix(chunker, "rabin-") { |
| 207 | parts := strings.Split(chunker, "-") |
| 208 | if len(parts) != 4 { |
| 209 | return false |
| 210 | } |
| 211 | |
| 212 | // Parse and validate min, avg, max values |
| 213 | values := make([]int, 3) |
| 214 | for i := range 3 { |
| 215 | val, err := strconv.Atoi(parts[i+1]) |
| 216 | if err != nil { |
| 217 | return false |
| 218 | } |
| 219 | values[i] = val |
| 220 | } |
| 221 | |
| 222 | // Validate ordering: min <= avg <= max |
| 223 | min, avg, max := values[0], values[1], values[2] |
| 224 | return min <= avg && avg <= max |
| 225 | } |
| 226 | |
| 227 | return false |
| 228 | } |
| 229 | |
| 230 | // HAMTSizeEstimationMode returns the boxo SizeEstimationMode based on the config value. |
| 231 | func (i *Import) HAMTSizeEstimationMode() uio.SizeEstimationMode { |
| 232 | switch i.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) { |
| 233 | case HAMTSizeEstimationLinks: |
| 234 | return uio.SizeEstimationLinks |
| 235 | case HAMTSizeEstimationBlock: |
| 236 | return uio.SizeEstimationBlock |
| 237 | case HAMTSizeEstimationDisabled: |
| 238 | return uio.SizeEstimationDisabled |
| 239 | default: |
| 240 | return uio.SizeEstimationLinks |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | // UnixFSSplitterFunc returns a SplitterGen function based on Import.UnixFSChunker. |
| 245 | // The returned function creates a Splitter for the configured chunking strategy. |
| 246 | // The chunker string is parsed once when this method is called, not on each use. |
| 247 | func (i *Import) UnixFSSplitterFunc() chunk.SplitterGen { |
| 248 | chunkerStr := i.UnixFSChunker.WithDefault(DefaultUnixFSChunker) |
| 249 | |
| 250 | // Parse size-based chunker (most common case) and return optimized generator |
| 251 | if sizeStr, ok := strings.CutPrefix(chunkerStr, "size-"); ok { |
| 252 | if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil && size > 0 { |
| 253 | return chunk.SizeSplitterGen(size) |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | // For other chunker types (rabin, buzhash) or invalid config, |
| 258 | // fall back to parsing per-use (these are rare cases) |
| 259 | return func(r io.Reader) chunk.Splitter { |
| 260 | s, err := chunk.FromString(r, chunkerStr) |
| 261 | if err != nil { |
| 262 | return chunk.DefaultSplitter(r) |
| 263 | } |
| 264 | return s |
| 265 | } |
| 266 | } |
| 267 | |
| 268 | // MFSRootOptions returns all MFS root options derived from Import config. |
| 269 | func (i *Import) MFSRootOptions() ([]mfs.Option, error) { |
| 270 | cidBuilder, err := i.UnixFSCidBuilder() |
| 271 | if err != nil { |
| 272 | return nil, err |
| 273 | } |
| 274 | sizeEstimationMode := i.HAMTSizeEstimationMode() |
| 275 | return []mfs.Option{ |
| 276 | mfs.WithCidBuilder(cidBuilder), |
| 277 | mfs.WithChunker(i.UnixFSSplitterFunc()), |
| 278 | mfs.WithMaxLinks(int(i.UnixFSDirectoryMaxLinks.WithDefault(DefaultUnixFSDirectoryMaxLinks))), |
| 279 | mfs.WithMaxHAMTFanout(int(i.UnixFSHAMTDirectoryMaxFanout.WithDefault(DefaultUnixFSHAMTDirectoryMaxFanout))), |
| 280 | mfs.WithHAMTShardingSize(int(i.UnixFSHAMTDirectorySizeThreshold.WithDefault(DefaultUnixFSHAMTDirectorySizeThreshold))), |
| 281 | mfs.WithSizeEstimationMode(sizeEstimationMode), |
| 282 | }, nil |
| 283 | } |
| 284 | |
| 285 | // UnixFSCidBuilder returns a cid.Builder based on Import.CidVersion and |
| 286 | // Import.HashFunction. Always builds an explicit prefix so that MFS |
| 287 | // respects kubo defaults even when they differ from boxo's internal |
| 288 | // CIDv0/sha2-256 default (see https://github.com/ipfs/kubo/issues/4143). |
| 289 | func (i *Import) UnixFSCidBuilder() (cid.Builder, error) { |
| 290 | cidVer := int(i.CidVersion.WithDefault(DefaultCidVersion)) |
| 291 | hashFunc := i.HashFunction.WithDefault(DefaultHashFunction) |
| 292 | |
| 293 | if hashFunc != DefaultHashFunction && cidVer == 0 { |
| 294 | cidVer = 1 |
| 295 | } |
| 296 | |
| 297 | prefix, err := merkledag.PrefixForCidVersion(cidVer) |
| 298 | if err != nil { |
| 299 | return nil, err |
| 300 | } |
| 301 | |
| 302 | hashCode, ok := mh.Names[strings.ToLower(hashFunc)] |
| 303 | if !ok { |
| 304 | return nil, fmt.Errorf("Import.HashFunction unrecognized: %q", hashFunc) |
| 305 | } |
| 306 | prefix.MhType = hashCode |
| 307 | prefix.MhLength = -1 |
| 308 | |
| 309 | return &prefix, nil |
| 310 | } |