| 1 | package options |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "fmt" |
| 6 | "os" |
| 7 | "time" |
| 8 | |
| 9 | dag "github.com/ipfs/boxo/ipld/merkledag" |
| 10 | "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" |
| 11 | "github.com/ipfs/boxo/ipld/unixfs/io" |
| 12 | cid "github.com/ipfs/go-cid" |
| 13 | mh "github.com/multiformats/go-multihash" |
| 14 | ) |
| 15 | |
| 16 | type Layout int |
| 17 | |
| 18 | const ( |
| 19 | BalancedLayout Layout = iota |
| 20 | TrickleLayout |
| 21 | ) |
| 22 | |
| 23 | type UnixfsAddSettings struct { |
| 24 | CidVersion int |
| 25 | MhType uint64 |
| 26 | |
| 27 | Inline bool |
| 28 | InlineLimit int |
| 29 | RawLeaves bool |
| 30 | RawLeavesSet bool |
| 31 | MaxFileLinks int |
| 32 | MaxFileLinksSet bool |
| 33 | MaxDirectoryLinks int |
| 34 | MaxDirectoryLinksSet bool |
| 35 | MaxHAMTFanout int |
| 36 | MaxHAMTFanoutSet bool |
| 37 | SizeEstimationMode *io.SizeEstimationMode |
| 38 | SizeEstimationModeSet bool |
| 39 | |
| 40 | Chunker string |
| 41 | Layout Layout |
| 42 | |
| 43 | Pin bool |
| 44 | PinName string |
| 45 | OnlyHash bool |
| 46 | FsCache bool |
| 47 | NoCopy bool |
| 48 | |
| 49 | Events chan<- any |
| 50 | Silent bool |
| 51 | Progress bool |
| 52 | |
| 53 | PreserveMode bool |
| 54 | PreserveMtime bool |
| 55 | Mode os.FileMode |
| 56 | Mtime time.Time |
| 57 | IncludeEmptyDirs bool |
| 58 | IncludeEmptyDirsSet bool |
| 59 | } |
| 60 | |
| 61 | type UnixfsLsSettings struct { |
| 62 | ResolveChildren bool |
| 63 | UseCumulativeSize bool |
| 64 | } |
| 65 | |
| 66 | type ( |
| 67 | UnixfsAddOption func(*UnixfsAddSettings) error |
| 68 | UnixfsLsOption func(*UnixfsLsSettings) error |
| 69 | ) |
| 70 | |
| 71 | func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, error) { |
| 72 | options := &UnixfsAddSettings{ |
| 73 | CidVersion: -1, |
| 74 | MhType: mh.SHA2_256, |
| 75 | |
| 76 | Inline: false, |
| 77 | InlineLimit: 32, |
| 78 | RawLeaves: false, |
| 79 | RawLeavesSet: false, |
| 80 | MaxFileLinks: helpers.DefaultLinksPerBlock, |
| 81 | MaxFileLinksSet: false, |
| 82 | MaxDirectoryLinks: 0, |
| 83 | MaxDirectoryLinksSet: false, |
| 84 | MaxHAMTFanout: io.DefaultShardWidth, |
| 85 | MaxHAMTFanoutSet: false, |
| 86 | |
| 87 | Chunker: "size-262144", |
| 88 | Layout: BalancedLayout, |
| 89 | |
| 90 | Pin: false, |
| 91 | PinName: "", |
| 92 | OnlyHash: false, |
| 93 | FsCache: false, |
| 94 | NoCopy: false, |
| 95 | |
| 96 | Events: nil, |
| 97 | Silent: false, |
| 98 | Progress: false, |
| 99 | |
| 100 | PreserveMode: false, |
| 101 | PreserveMtime: false, |
| 102 | Mode: 0, |
| 103 | Mtime: time.Time{}, |
| 104 | IncludeEmptyDirs: true, // default: include empty directories |
| 105 | IncludeEmptyDirsSet: false, |
| 106 | } |
| 107 | |
| 108 | for _, opt := range opts { |
| 109 | err := opt(options) |
| 110 | if err != nil { |
| 111 | return nil, cid.Prefix{}, err |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // nocopy -> rawblocks |
| 116 | if options.NoCopy && !options.RawLeaves { |
| 117 | // fixed? |
| 118 | if options.RawLeavesSet { |
| 119 | return nil, cid.Prefix{}, fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well") |
| 120 | } |
| 121 | |
| 122 | // No, satisfy mandatory constraint. |
| 123 | options.RawLeaves = true |
| 124 | } |
| 125 | |
| 126 | // (hash != "sha2-256") -> CIDv1 |
| 127 | if options.MhType != mh.SHA2_256 { |
| 128 | switch options.CidVersion { |
| 129 | case 0: |
| 130 | return nil, cid.Prefix{}, errors.New("CIDv0 only supports sha2-256") |
| 131 | case 1, -1: |
| 132 | options.CidVersion = 1 |
| 133 | default: |
| 134 | return nil, cid.Prefix{}, fmt.Errorf("unknown CID version: %d", options.CidVersion) |
| 135 | } |
| 136 | } else { |
| 137 | if options.CidVersion < 0 { |
| 138 | // Default to CIDv0 |
| 139 | options.CidVersion = 0 |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | if !options.Mtime.IsZero() && options.PreserveMtime { |
| 144 | options.PreserveMtime = false |
| 145 | } |
| 146 | |
| 147 | if options.Mode != 0 && options.PreserveMode { |
| 148 | options.PreserveMode = false |
| 149 | } |
| 150 | |
| 151 | // cidV1 -> raw blocks (by default) |
| 152 | if options.CidVersion > 0 && !options.RawLeavesSet { |
| 153 | options.RawLeaves = true |
| 154 | } |
| 155 | |
| 156 | prefix, err := dag.PrefixForCidVersion(options.CidVersion) |
| 157 | if err != nil { |
| 158 | return nil, cid.Prefix{}, err |
| 159 | } |
| 160 | |
| 161 | prefix.MhType = options.MhType |
| 162 | prefix.MhLength = -1 |
| 163 | |
| 164 | return options, prefix, nil |
| 165 | } |
| 166 | |
| 167 | func UnixfsLsOptions(opts ...UnixfsLsOption) (*UnixfsLsSettings, error) { |
| 168 | options := &UnixfsLsSettings{ |
| 169 | ResolveChildren: true, |
| 170 | } |
| 171 | |
| 172 | for _, opt := range opts { |
| 173 | err := opt(options) |
| 174 | if err != nil { |
| 175 | return nil, err |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | return options, nil |
| 180 | } |
| 181 | |
| 182 | type unixfsOpts struct{} |
| 183 | |
| 184 | var Unixfs unixfsOpts |
| 185 | |
| 186 | // CidVersion specifies which CID version to use. Defaults to 0 unless an option |
| 187 | // that depends on CIDv1 is passed. |
| 188 | func (unixfsOpts) CidVersion(version int) UnixfsAddOption { |
| 189 | return func(settings *UnixfsAddSettings) error { |
| 190 | settings.CidVersion = version |
| 191 | return nil |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | // Hash function to use. Implies CIDv1 if not set to sha2-256 (default). |
| 196 | // |
| 197 | // Table of functions is declared in https://github.com/multiformats/go-multihash/blob/master/multihash.go |
| 198 | func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { |
| 199 | return func(settings *UnixfsAddSettings) error { |
| 200 | settings.MhType = mhtype |
| 201 | return nil |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | // RawLeaves specifies whether to use raw blocks for leaves (data nodes with no |
| 206 | // links) instead of wrapping them with unixfs structures. |
| 207 | func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { |
| 208 | return func(settings *UnixfsAddSettings) error { |
| 209 | settings.RawLeaves = enable |
| 210 | settings.RawLeavesSet = true |
| 211 | return nil |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // MaxFileLinks specifies the maximum number of children for UnixFS file |
| 216 | // nodes. |
| 217 | func (unixfsOpts) MaxFileLinks(n int) UnixfsAddOption { |
| 218 | return func(settings *UnixfsAddSettings) error { |
| 219 | settings.MaxFileLinks = n |
| 220 | settings.MaxFileLinksSet = true |
| 221 | return nil |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | // MaxDirectoryLinks specifies the maximum number of children for UnixFS basic |
| 226 | // directory nodes. |
| 227 | func (unixfsOpts) MaxDirectoryLinks(n int) UnixfsAddOption { |
| 228 | return func(settings *UnixfsAddSettings) error { |
| 229 | settings.MaxDirectoryLinks = n |
| 230 | settings.MaxDirectoryLinksSet = true |
| 231 | return nil |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // MaxHAMTFanout specifies the maximum width of the HAMT directory shards. |
| 236 | // Per the UnixFS spec, the value must be a power of 2, minimum 8 |
| 237 | // (for byte-aligned bitfields), and maximum 1024. |
| 238 | func (unixfsOpts) MaxHAMTFanout(n int) UnixfsAddOption { |
| 239 | return func(settings *UnixfsAddSettings) error { |
| 240 | if n < 8 || n&(n-1) != 0 || n > 1024 { |
| 241 | return fmt.Errorf("HAMT fanout must be a power of 2, between 8 and 1024 (got %d)", n) |
| 242 | } |
| 243 | settings.MaxHAMTFanout = n |
| 244 | settings.MaxHAMTFanoutSet = true |
| 245 | return nil |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | // SizeEstimationMode specifies how directory size is estimated for HAMT sharding decisions. |
| 250 | func (unixfsOpts) SizeEstimationMode(mode io.SizeEstimationMode) UnixfsAddOption { |
| 251 | return func(settings *UnixfsAddSettings) error { |
| 252 | settings.SizeEstimationMode = &mode |
| 253 | settings.SizeEstimationModeSet = true |
| 254 | return nil |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // Inline tells the adder to inline small blocks into CIDs |
| 259 | func (unixfsOpts) Inline(enable bool) UnixfsAddOption { |
| 260 | return func(settings *UnixfsAddSettings) error { |
| 261 | settings.Inline = enable |
| 262 | return nil |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | // InlineLimit sets the amount of bytes below which blocks will be encoded |
| 267 | // directly into CID instead of being stored and addressed by it's hash. |
| 268 | // Specifying this option won't enable block inlining. For that use `Inline` |
| 269 | // option. Default: 32 bytes |
| 270 | // |
| 271 | // Note that while there is no hard limit on the number of bytes, it should be |
| 272 | // kept at a reasonably low value, such as 64; implementations may choose to |
| 273 | // reject anything larger. |
| 274 | func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { |
| 275 | return func(settings *UnixfsAddSettings) error { |
| 276 | settings.InlineLimit = limit |
| 277 | return nil |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | // Chunker specifies settings for the chunking algorithm to use. |
| 282 | // |
| 283 | // Default: size-262144, formats: |
| 284 | // size-[bytes] - Simple chunker splitting data into blocks of n bytes |
| 285 | // rabin-[min]-[avg]-[max] - Rabin chunker |
| 286 | func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { |
| 287 | return func(settings *UnixfsAddSettings) error { |
| 288 | settings.Chunker = chunker |
| 289 | return nil |
| 290 | } |
| 291 | } |
| 292 | |
| 293 | // Layout tells the adder how to balance data between leaves. |
| 294 | // options.BalancedLayout is the default, it's optimized for static seekable |
| 295 | // files. |
| 296 | // options.TrickleLayout is optimized for streaming data, |
| 297 | func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { |
| 298 | return func(settings *UnixfsAddSettings) error { |
| 299 | settings.Layout = layout |
| 300 | return nil |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | // Pin tells the adder to pin the file root recursively after adding |
| 305 | func (unixfsOpts) Pin(pin bool, pinName string) UnixfsAddOption { |
| 306 | return func(settings *UnixfsAddSettings) error { |
| 307 | settings.Pin = pin |
| 308 | if pin { |
| 309 | settings.PinName = pinName |
| 310 | } |
| 311 | return nil |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | // HashOnly will make the adder calculate data hash without storing it in the |
| 316 | // blockstore or announcing it to the network |
| 317 | func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { |
| 318 | return func(settings *UnixfsAddSettings) error { |
| 319 | settings.OnlyHash = hashOnly |
| 320 | return nil |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | // Events specifies channel which will be used to report events about ongoing |
| 325 | // Add operation. |
| 326 | // |
| 327 | // Note that if this channel blocks it may slowdown the adder |
| 328 | func (unixfsOpts) Events(sink chan<- any) UnixfsAddOption { |
| 329 | return func(settings *UnixfsAddSettings) error { |
| 330 | settings.Events = sink |
| 331 | return nil |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | // Silent reduces event output |
| 336 | func (unixfsOpts) Silent(silent bool) UnixfsAddOption { |
| 337 | return func(settings *UnixfsAddSettings) error { |
| 338 | settings.Silent = silent |
| 339 | return nil |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | // Progress tells the adder whether to enable progress events |
| 344 | func (unixfsOpts) Progress(enable bool) UnixfsAddOption { |
| 345 | return func(settings *UnixfsAddSettings) error { |
| 346 | settings.Progress = enable |
| 347 | return nil |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | // FsCache tells the adder to check the filestore for pre-existing blocks |
| 352 | // |
| 353 | // Experimental |
| 354 | func (unixfsOpts) FsCache(enable bool) UnixfsAddOption { |
| 355 | return func(settings *UnixfsAddSettings) error { |
| 356 | settings.FsCache = enable |
| 357 | return nil |
| 358 | } |
| 359 | } |
| 360 | |
| 361 | // NoCopy tells the adder to add the files using filestore. Implies RawLeaves. |
| 362 | // |
| 363 | // Experimental |
| 364 | func (unixfsOpts) Nocopy(enable bool) UnixfsAddOption { |
| 365 | return func(settings *UnixfsAddSettings) error { |
| 366 | settings.NoCopy = enable |
| 367 | return nil |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | func (unixfsOpts) ResolveChildren(resolve bool) UnixfsLsOption { |
| 372 | return func(settings *UnixfsLsSettings) error { |
| 373 | settings.ResolveChildren = resolve |
| 374 | return nil |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | func (unixfsOpts) UseCumulativeSize(use bool) UnixfsLsOption { |
| 379 | return func(settings *UnixfsLsSettings) error { |
| 380 | settings.UseCumulativeSize = use |
| 381 | return nil |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | // PreserveMode tells the adder to store the file permissions |
| 386 | func (unixfsOpts) PreserveMode(enable bool) UnixfsAddOption { |
| 387 | return func(settings *UnixfsAddSettings) error { |
| 388 | settings.PreserveMode = enable |
| 389 | return nil |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | // PreserveMtime tells the adder to store the file modification time |
| 394 | func (unixfsOpts) PreserveMtime(enable bool) UnixfsAddOption { |
| 395 | return func(settings *UnixfsAddSettings) error { |
| 396 | settings.PreserveMtime = enable |
| 397 | return nil |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | // Mode represents a unix file mode |
| 402 | func (unixfsOpts) Mode(mode os.FileMode) UnixfsAddOption { |
| 403 | return func(settings *UnixfsAddSettings) error { |
| 404 | settings.Mode = mode |
| 405 | return nil |
| 406 | } |
| 407 | } |
| 408 | |
| 409 | // Mtime represents a unix file mtime |
| 410 | func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption { |
| 411 | return func(settings *UnixfsAddSettings) error { |
| 412 | if nsecs > 999999999 { |
| 413 | return errors.New("mtime nanoseconds must be in range [1, 999999999]") |
| 414 | } |
| 415 | settings.Mtime = time.Unix(seconds, int64(nsecs)) |
| 416 | return nil |
| 417 | } |
| 418 | } |
| 419 | |
| 420 | // IncludeEmptyDirs tells the adder to include empty directories in the DAG |
| 421 | func (unixfsOpts) IncludeEmptyDirs(include bool) UnixfsAddOption { |
| 422 | return func(settings *UnixfsAddSettings) error { |
| 423 | settings.IncludeEmptyDirs = include |
| 424 | settings.IncludeEmptyDirsSet = true |
| 425 | return nil |
| 426 | } |
| 427 | } |