| 1 | package commands |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "fmt" |
| 6 | "io" |
| 7 | "os" |
| 8 | gopath "path" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | |
| 12 | "github.com/ipfs/kubo/config" |
| 13 | "github.com/ipfs/kubo/core/commands/cmdenv" |
| 14 | "github.com/ipfs/kubo/core/commands/cmdutils" |
| 15 | |
| 16 | "github.com/cheggaaa/pb/v3" |
| 17 | "github.com/ipfs/boxo/files" |
| 18 | uio "github.com/ipfs/boxo/ipld/unixfs/io" |
| 19 | mfs "github.com/ipfs/boxo/mfs" |
| 20 | "github.com/ipfs/boxo/path" |
| 21 | "github.com/ipfs/boxo/verifcid" |
| 22 | cid "github.com/ipfs/go-cid" |
| 23 | cmds "github.com/ipfs/go-ipfs-cmds" |
| 24 | ipld "github.com/ipfs/go-ipld-format" |
| 25 | coreiface "github.com/ipfs/kubo/core/coreiface" |
| 26 | "github.com/ipfs/kubo/core/coreiface/options" |
| 27 | mh "github.com/multiformats/go-multihash" |
| 28 | ) |
| 29 | |
| 30 | // ErrDepthLimitExceeded indicates that the max depth has been exceeded. |
| 31 | var ErrDepthLimitExceeded = errors.New("depth limit exceeded") |
| 32 | |
| 33 | type AddEvent struct { |
| 34 | Name string |
| 35 | Hash string `json:",omitempty"` |
| 36 | Bytes int64 `json:",omitempty"` |
| 37 | Size string `json:",omitempty"` |
| 38 | Mode string `json:",omitempty"` |
| 39 | Mtime int64 `json:",omitempty"` |
| 40 | MtimeNsecs int `json:",omitempty"` |
| 41 | } |
| 42 | |
| 43 | const ( |
| 44 | pinNameOptionName = "pin-name" |
| 45 | quietOptionName = "quiet" |
| 46 | quieterOptionName = "quieter" |
| 47 | silentOptionName = "silent" |
| 48 | progressOptionName = "progress" |
| 49 | trickleOptionName = "trickle" |
| 50 | wrapOptionName = "wrap-with-directory" |
| 51 | onlyHashOptionName = "only-hash" |
| 52 | chunkerOptionName = "chunker" |
| 53 | pinOptionName = "pin" |
| 54 | rawLeavesOptionName = "raw-leaves" |
| 55 | maxFileLinksOptionName = "max-file-links" |
| 56 | maxDirectoryLinksOptionName = "max-directory-links" |
| 57 | maxHAMTFanoutOptionName = "max-hamt-fanout" |
| 58 | noCopyOptionName = "nocopy" |
| 59 | fstoreCacheOptionName = "fscache" |
| 60 | cidVersionOptionName = "cid-version" |
| 61 | hashOptionName = "hash" |
| 62 | inlineOptionName = "inline" |
| 63 | inlineLimitOptionName = "inline-limit" |
| 64 | toFilesOptionName = "to-files" |
| 65 | |
| 66 | preserveModeOptionName = "preserve-mode" |
| 67 | preserveMtimeOptionName = "preserve-mtime" |
| 68 | modeOptionName = "mode" |
| 69 | mtimeOptionName = "mtime" |
| 70 | mtimeNsecsOptionName = "mtime-nsecs" |
| 71 | fastProvideRootOptionName = "fast-provide-root" |
| 72 | fastProvideDAGOptionName = "fast-provide-dag" |
| 73 | fastProvideWaitOptionName = "fast-provide-wait" |
| 74 | emptyDirsOptionName = "empty-dirs" |
| 75 | ) |
| 76 | |
| 77 | const ( |
| 78 | adderOutChanSize = 8 |
| 79 | |
| 80 | // pb/v3 template used before the upload total is known: only the |
| 81 | // running byte counter and current speed. Swapped for |
| 82 | // cmdenv.ProgressBarFullTemplate once size discovery reports. |
| 83 | progressBarInitTemplate = `{{counters . }} {{speed . "%s/s" "?/s"}}` |
| 84 | ) |
| 85 | |
| 86 | var AddCmd = &cmds.Command{ |
| 87 | Helptext: cmds.HelpText{ |
| 88 | Tagline: "Add a file or directory to IPFS.", |
| 89 | ShortDescription: ` |
| 90 | Adds the content of <path> to IPFS. Use -r to add directories (recursively). |
| 91 | |
| 92 | CONTENT DISCOVERABILITY: |
| 93 | |
| 94 | How quickly other peers can find your content depends on Provide.Strategy: |
| 95 | |
| 96 | Provide.Strategy=all (default): |
| 97 | Every block is announced to the routing system as it is written to |
| 98 | the blockstore. Content is discoverable immediately. |
| 99 | |
| 100 | Selective strategies (pinned, mfs, pinned+mfs): |
| 101 | Only the root CID is announced immediately after 'ipfs add'. |
| 102 | Remaining blocks are announced during the next reprovide cycle |
| 103 | (Provide.DHT.Interval, default 22h). |
| 104 | |
| 105 | FAST PROVIDE FLAGS: |
| 106 | |
| 107 | --fast-provide-root (default: enabled) |
| 108 | Announce the root CID to the routing system immediately after add, |
| 109 | in addition to the regular provide queue. Runs in the background |
| 110 | without blocking. Set to false to skip extra provides and minimize |
| 111 | network overhead when importing a lot of data at once. |
| 112 | |
| 113 | --fast-provide-dag (default: disabled) |
| 114 | Walk and provide the full DAG immediately after add, using the |
| 115 | active Provide.Strategy to determine scope. Useful with selective |
| 116 | strategies when all blocks need to be discoverable right away. |
| 117 | No effect with Provide.Strategy=all (blockstore already provides |
| 118 | every block on write). |
| 119 | |
| 120 | --fast-provide-wait (default: disabled) |
| 121 | Block until the immediate provide completes before returning. |
| 122 | Use when you need certainty that content is discoverable before |
| 123 | the command returns (e.g., sharing a link immediately after adding). |
| 124 | |
| 125 | All fast-provide flags require an active DHT client. Skipped automatically |
| 126 | when only HTTP delegated routing is configured. |
| 127 | `, |
| 128 | LongDescription: ` |
| 129 | Adds the content of <path> to IPFS. Use -r to add directories. |
| 130 | Note that directories are added recursively, and big files are chunked, |
| 131 | to form the IPFS MerkleDAG. Learn more: https://docs.ipfs.tech/concepts/merkle-dag/ |
| 132 | |
| 133 | If the daemon is not running, it will just add locally to the repo at $IPFS_PATH. |
| 134 | If the daemon is started later, it will be advertised after a few |
| 135 | seconds when the provide system runs. |
| 136 | |
| 137 | BASIC EXAMPLES: |
| 138 | |
| 139 | The wrap option, '-w', wraps the file (or files, if using the |
| 140 | recursive option) in a directory. This directory contains only |
| 141 | the files which have been added, and means that the file retains |
| 142 | its filename. For example: |
| 143 | |
| 144 | > ipfs add example.jpg |
| 145 | added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg |
| 146 | > ipfs add example.jpg -w |
| 147 | added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg |
| 148 | added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx |
| 149 | |
| 150 | You can now refer to the added file in a gateway, like so: |
| 151 | |
| 152 | /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg |
| 153 | |
| 154 | Files imported with 'ipfs add' are protected from GC (implicit '--pin=true'), |
| 155 | but it is up to you to remember the returned CID to get the data back later. |
| 156 | |
| 157 | If you need to back up or transport content-addressed data using a non-IPFS |
| 158 | medium, CID can be preserved with CAR files. |
| 159 | See 'dag export' and 'dag import' for more information. |
| 160 | |
| 161 | MFS INTEGRATION: |
| 162 | |
| 163 | Passing '--to-files' creates a reference in Files API (MFS), making it easier |
| 164 | to find it in the future: |
| 165 | |
| 166 | > ipfs files mkdir -p /myfs/dir |
| 167 | > ipfs add example.jpg --to-files /myfs/dir/ |
| 168 | > ipfs files ls /myfs/dir/ |
| 169 | example.jpg |
| 170 | |
| 171 | See 'ipfs files --help' to learn more about using MFS |
| 172 | for keeping track of added files and directories. |
| 173 | |
| 174 | SYMLINK HANDLING: |
| 175 | |
| 176 | By default, symbolic links are preserved as UnixFS symlink nodes that store |
| 177 | the target path. Use --dereference-symlinks to resolve symlinks to their |
| 178 | target content instead: |
| 179 | |
| 180 | > ipfs add -r --dereference-symlinks ./mydir |
| 181 | |
| 182 | This resolves all symlinks, including CLI arguments and those found inside |
| 183 | directories. Symlinks to files become regular file content, symlinks to |
| 184 | directories are traversed and their contents are added. |
| 185 | |
| 186 | CHUNKING EXAMPLES: |
| 187 | |
| 188 | The chunker option, '-s', specifies the chunking strategy that dictates |
| 189 | how to break files into blocks. Blocks with same content can |
| 190 | be deduplicated. Different chunking strategies will produce different |
| 191 | hashes for the same file. The default is a fixed block size of |
| 192 | 256 * 1024 bytes, 'size-262144'. Alternatively, you can use the |
| 193 | Buzhash or Rabin fingerprint chunker for content defined chunking by |
| 194 | specifying buzhash or rabin-[min]-[avg]-[max] (where min/avg/max refer |
| 195 | to the desired chunk sizes in bytes), e.g. 'rabin-262144-524288-1048576'. |
| 196 | |
| 197 | The maximum accepted value for 'size-N' and rabin 'max' parameter is |
| 198 | 2MiB minus 256 bytes (2096896 bytes). The 256-byte overhead budget is |
| 199 | reserved for protobuf/UnixFS framing so that serialized blocks stay |
| 200 | within the 2MiB block size limit from the bitswap spec. The buzhash |
| 201 | chunker uses a fixed internal maximum of 512KiB and is not affected. |
| 202 | |
| 203 | Only the fixed-size chunker ('size-N') guarantees that the same data |
| 204 | will always produce the same CID. The rabin and buzhash chunkers may |
| 205 | change their internal parameters in a future release. |
| 206 | |
| 207 | The following examples use very small byte sizes to demonstrate the |
| 208 | properties of the different chunkers on a small file. You'll likely |
| 209 | want to use a 1024 times larger chunk sizes for most files. |
| 210 | |
| 211 | > ipfs add --chunker=size-2048 ipfs-logo.svg |
| 212 | added QmafrLBfzRLV4XSH1XcaMMeaXEUhDJjmtDfsYU95TrWG87 ipfs-logo.svg |
| 213 | > ipfs add --chunker=rabin-512-1024-2048 ipfs-logo.svg |
| 214 | added Qmf1hDN65tR55Ubh2RN1FPxr69xq3giVBz1KApsresY8Gn ipfs-logo.svg |
| 215 | |
| 216 | You can now check what blocks have been created by: |
| 217 | |
| 218 | > ipfs ls QmafrLBfzRLV4XSH1XcaMMeaXEUhDJjmtDfsYU95TrWG87 |
| 219 | QmY6yj1GsermExDXoosVE3aSPxdMNYr6aKuw3nA8LoWPRS 2059 |
| 220 | Qmf7ZQeSxq2fJVJbCmgTrLLVN9tDR9Wy5k75DxQKuz5Gyt 1195 |
| 221 | > ipfs ls Qmf1hDN65tR55Ubh2RN1FPxr69xq3giVBz1KApsresY8Gn |
| 222 | QmY6yj1GsermExDXoosVE3aSPxdMNYr6aKuw3nA8LoWPRS 2059 |
| 223 | QmerURi9k4XzKCaaPbsK6BL5pMEjF7PGphjDvkkjDtsVf3 868 |
| 224 | QmQB28iwSriSUSMqG2nXDTLtdPHgWb4rebBrU7Q1j4vxPv 338 |
| 225 | |
| 226 | ADVANCED CONFIGURATION: |
| 227 | |
| 228 | Finally, a note on hash (CID) determinism and 'ipfs add' command. |
| 229 | |
| 230 | Almost all the flags provided by this command will change the final CID, and |
| 231 | new flags may be added in the future. It is not guaranteed for the implicit |
| 232 | defaults of 'ipfs add' to remain the same in future Kubo releases, or for other |
| 233 | IPFS software to use the same import parameters as Kubo. |
| 234 | |
| 235 | Note: CIDv1 is automatically used when using non-default options like custom |
| 236 | hash functions or when raw-leaves is explicitly enabled. |
| 237 | |
| 238 | Use Import.* configuration options to override global implicit defaults: |
| 239 | https://github.com/ipfs/kubo/blob/master/docs/config.md#import |
| 240 | `, |
| 241 | }, |
| 242 | |
| 243 | Arguments: []cmds.Argument{ |
| 244 | cmds.FileArg("path", true, true, "The path to a file to be added to IPFS.").EnableRecursive().EnableStdin(), |
| 245 | }, |
| 246 | Options: []cmds.Option{ |
| 247 | // Input Processing |
| 248 | cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) |
| 249 | cmds.OptionDerefArgs, // DEPRECATED: use --dereference-symlinks instead |
| 250 | cmds.OptionStdinName, // a builtin option that optionally allows wrapping stdin into a named file |
| 251 | cmds.OptionHidden, |
| 252 | cmds.OptionIgnore, |
| 253 | cmds.OptionIgnoreRules, |
| 254 | cmds.BoolOption(emptyDirsOptionName, "E", "Include empty directories in the import.").WithDefault(config.DefaultUnixFSIncludeEmptyDirs), |
| 255 | cmds.OptionDerefSymlinks, // resolve symlinks to their target content |
| 256 | // Output Control |
| 257 | cmds.BoolOption(quietOptionName, "q", "Write minimal output."), |
| 258 | cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."), |
| 259 | cmds.BoolOption(silentOptionName, "Write no output."), |
| 260 | cmds.BoolOption(progressOptionName, "p", "Stream progress data. Defaults to true when stderr is a terminal."), |
| 261 | // Basic Add Behavior |
| 262 | cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."), |
| 263 | cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), |
| 264 | cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), |
| 265 | cmds.StringOption(pinNameOptionName, "Name to use for the pin. Requires explicit value (e.g., --pin-name=myname)."), |
| 266 | // MFS Integration |
| 267 | cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."), |
| 268 | // CID & Hashing |
| 269 | cmds.IntOption(cidVersionOptionName, "CID version (0 or 1). CIDv1 automatically enables raw-leaves and is required for non-sha2-256 hashes. Default: Import.CidVersion"), |
| 270 | cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. Default: Import.HashFunction"), |
| 271 | cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. Note: CIDv1 automatically enables raw-leaves. Default: false for CIDv0, true for CIDv1 (Import.UnixFSRawLeaves)"), |
| 272 | // Chunking & DAG Structure |
| 273 | cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash. Files larger than chunk size are split into multiple blocks. Default: Import.UnixFSChunker"), |
| 274 | cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), |
| 275 | // Advanced UnixFS Limits |
| 276 | cmds.IntOption(maxFileLinksOptionName, "Limit the maximum number of links in UnixFS file nodes to this value. WARNING: experimental. Default: Import.UnixFSFileMaxLinks"), |
| 277 | cmds.IntOption(maxDirectoryLinksOptionName, "Limit the maximum number of links in UnixFS basic directory nodes to this value. WARNING: experimental, Import.UnixFSHAMTDirectorySizeThreshold is safer. Default: Import.UnixFSDirectoryMaxLinks"), |
| 278 | cmds.IntOption(maxHAMTFanoutOptionName, "Limit the maximum number of links of a UnixFS HAMT directory node to this (power of 2, between 8 and 1024). WARNING: experimental, Import.UnixFSHAMTDirectorySizeThreshold is safer. Default: Import.UnixFSHAMTDirectoryMaxFanout"), |
| 279 | // Experimental Features |
| 280 | cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. WARNING: experimental"), |
| 281 | cmds.IntOption(inlineLimitOptionName, fmt.Sprintf("Maximum block size to inline. Maximum: %d bytes. WARNING: experimental", verifcid.DefaultMaxIdentityDigestSize)).WithDefault(32), |
| 282 | cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. WARNING: experimental"), |
| 283 | cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. WARNING: experimental"), |
| 284 | cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), |
| 285 | cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), |
| 286 | cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), |
| 287 | cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), |
| 288 | cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"), |
| 289 | cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CID to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), |
| 290 | cmds.BoolOption(fastProvideDAGOptionName, "Walk and provide the full DAG according to Provide.Strategy immediately after add. Default: Import.FastProvideDAG"), |
| 291 | cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"), |
| 292 | }, |
| 293 | PreRun: func(req *cmds.Request, env cmds.Environment) error { |
| 294 | quiet, _ := req.Options[quietOptionName].(bool) |
| 295 | quieter, _ := req.Options[quieterOptionName].(bool) |
| 296 | quiet = quiet || quieter |
| 297 | silent, _ := req.Options[silentOptionName].(bool) |
| 298 | |
| 299 | if !quiet && !silent { |
| 300 | // default to showing progress only when stderr is a terminal |
| 301 | _, found := req.Options[progressOptionName].(bool) |
| 302 | if !found { |
| 303 | req.Options[progressOptionName] = cmdenv.IsTerminal(os.Stderr) |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | return nil |
| 308 | }, |
| 309 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 310 | api, err := cmdenv.GetApi(env, req) |
| 311 | if err != nil { |
| 312 | return err |
| 313 | } |
| 314 | |
| 315 | nd, err := cmdenv.GetNode(env) |
| 316 | if err != nil { |
| 317 | return err |
| 318 | } |
| 319 | |
| 320 | cfg, err := nd.Repo.Config() |
| 321 | if err != nil { |
| 322 | return err |
| 323 | } |
| 324 | |
| 325 | progress, _ := req.Options[progressOptionName].(bool) |
| 326 | trickle, trickleSet := req.Options[trickleOptionName].(bool) |
| 327 | wrap, _ := req.Options[wrapOptionName].(bool) |
| 328 | onlyHash, _ := req.Options[onlyHashOptionName].(bool) |
| 329 | silent, _ := req.Options[silentOptionName].(bool) |
| 330 | chunker, _ := req.Options[chunkerOptionName].(string) |
| 331 | dopin, _ := req.Options[pinOptionName].(bool) |
| 332 | pinName, pinNameSet := req.Options[pinNameOptionName].(string) |
| 333 | rawblks, rbset := req.Options[rawLeavesOptionName].(bool) |
| 334 | maxFileLinks, maxFileLinksSet := req.Options[maxFileLinksOptionName].(int) |
| 335 | maxDirectoryLinks, maxDirectoryLinksSet := req.Options[maxDirectoryLinksOptionName].(int) |
| 336 | maxHAMTFanout, maxHAMTFanoutSet := req.Options[maxHAMTFanoutOptionName].(int) |
| 337 | var sizeEstimationMode uio.SizeEstimationMode |
| 338 | nocopy, _ := req.Options[noCopyOptionName].(bool) |
| 339 | fscache, _ := req.Options[fstoreCacheOptionName].(bool) |
| 340 | cidVer, cidVerSet := req.Options[cidVersionOptionName].(int) |
| 341 | hashFunStr, _ := req.Options[hashOptionName].(string) |
| 342 | inline, _ := req.Options[inlineOptionName].(bool) |
| 343 | inlineLimit, _ := req.Options[inlineLimitOptionName].(int) |
| 344 | |
| 345 | // Validate inline-limit doesn't exceed the maximum identity digest size |
| 346 | if inline && inlineLimit > verifcid.DefaultMaxIdentityDigestSize { |
| 347 | return fmt.Errorf("inline-limit %d exceeds maximum allowed size of %d bytes", inlineLimit, verifcid.DefaultMaxIdentityDigestSize) |
| 348 | } |
| 349 | |
| 350 | // Validate pin name |
| 351 | if pinNameSet { |
| 352 | if err := cmdutils.ValidatePinName(pinName); err != nil { |
| 353 | return err |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) |
| 358 | preserveMode, _ := req.Options[preserveModeOptionName].(bool) |
| 359 | preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool) |
| 360 | mode, _ := req.Options[modeOptionName].(uint) |
| 361 | mtime, _ := req.Options[mtimeOptionName].(int64) |
| 362 | mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) |
| 363 | fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) |
| 364 | fastProvideDAG, fastProvideDAGSet := req.Options[fastProvideDAGOptionName].(bool) |
| 365 | fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) |
| 366 | emptyDirs, _ := req.Options[emptyDirsOptionName].(bool) |
| 367 | |
| 368 | // Note: --dereference-args is deprecated but still works for backwards compatibility. |
| 369 | // The help text marks it as DEPRECATED. Users should use --dereference-symlinks instead, |
| 370 | // which is a superset (resolves both CLI arg symlinks AND nested symlinks in directories). |
| 371 | |
| 372 | // Wire --trickle from config |
| 373 | if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() { |
| 374 | layout := cfg.Import.UnixFSDAGLayout.WithDefault(config.DefaultUnixFSDAGLayout) |
| 375 | trickle = layout == config.DAGLayoutTrickle |
| 376 | } |
| 377 | |
| 378 | if chunker == "" { |
| 379 | chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) |
| 380 | } |
| 381 | |
| 382 | if hashFunStr == "" { |
| 383 | hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) |
| 384 | } |
| 385 | |
| 386 | if !cidVerSet && !cfg.Import.CidVersion.IsDefault() { |
| 387 | cidVerSet = true |
| 388 | cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion)) |
| 389 | } |
| 390 | |
| 391 | // Pin names are only used when explicitly provided via --pin-name=value |
| 392 | |
| 393 | if !rbset && cfg.Import.UnixFSRawLeaves != config.Default { |
| 394 | rbset = true |
| 395 | rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) |
| 396 | } |
| 397 | |
| 398 | if !maxFileLinksSet && !cfg.Import.UnixFSFileMaxLinks.IsDefault() { |
| 399 | maxFileLinksSet = true |
| 400 | maxFileLinks = int(cfg.Import.UnixFSFileMaxLinks.WithDefault(config.DefaultUnixFSFileMaxLinks)) |
| 401 | } |
| 402 | |
| 403 | if !maxDirectoryLinksSet && !cfg.Import.UnixFSDirectoryMaxLinks.IsDefault() { |
| 404 | maxDirectoryLinksSet = true |
| 405 | maxDirectoryLinks = int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) |
| 406 | } |
| 407 | |
| 408 | if !maxHAMTFanoutSet && !cfg.Import.UnixFSHAMTDirectoryMaxFanout.IsDefault() { |
| 409 | maxHAMTFanoutSet = true |
| 410 | maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout)) |
| 411 | } |
| 412 | |
| 413 | // SizeEstimationMode is always set from config (no CLI flag) |
| 414 | sizeEstimationMode = cfg.Import.HAMTSizeEstimationMode() |
| 415 | |
| 416 | fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot) |
| 417 | fastProvideDAG = config.ResolveBoolFromConfig(fastProvideDAG, fastProvideDAGSet, cfg.Import.FastProvideDAG, config.DefaultFastProvideDAG) |
| 418 | fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait) |
| 419 | |
| 420 | // --only-hash does not store data, so pinning and providing |
| 421 | // are meaningless. |
| 422 | if onlyHash { |
| 423 | dopin = false |
| 424 | fastProvideRoot = false |
| 425 | fastProvideDAG = false |
| 426 | } |
| 427 | |
| 428 | // Storing optional mode or mtime (UnixFS 1.5) requires root block |
| 429 | // to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible. |
| 430 | if preserveMode || preserveMtime || mode != 0 || mtime != 0 { |
| 431 | // Error if --raw-leaves flag was explicitly passed by the user. |
| 432 | // (let user make a decision to manually disable it and retry) |
| 433 | if rbset && rawblks { |
| 434 | return fmt.Errorf("%s can't be used with UnixFS metadata like mode or modification time", rawLeavesOptionName) |
| 435 | } |
| 436 | // No explicit preference from user, disable raw-leaves and continue |
| 437 | rbset = true |
| 438 | rawblks = false |
| 439 | } |
| 440 | |
| 441 | if onlyHash && toFilesSet { |
| 442 | return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName) |
| 443 | } |
| 444 | if !dopin && pinNameSet { |
| 445 | return fmt.Errorf("%s option requires %s to be set", pinNameOptionName, pinOptionName) |
| 446 | } |
| 447 | if wrap && toFilesSet { |
| 448 | return fmt.Errorf("%s and %s options are not compatible", wrapOptionName, toFilesOptionName) |
| 449 | } |
| 450 | |
| 451 | hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] |
| 452 | if !ok { |
| 453 | return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr)) |
| 454 | } |
| 455 | |
| 456 | enc, err := cmdenv.GetCidEncoder(req) |
| 457 | if err != nil { |
| 458 | return err |
| 459 | } |
| 460 | |
| 461 | toadd := req.Files |
| 462 | if wrap { |
| 463 | toadd = files.NewSliceDirectory([]files.DirEntry{ |
| 464 | files.FileEntry("", req.Files), |
| 465 | }) |
| 466 | } |
| 467 | |
| 468 | opts := []options.UnixfsAddOption{ |
| 469 | options.Unixfs.Hash(hashFunCode), |
| 470 | |
| 471 | options.Unixfs.Inline(inline), |
| 472 | options.Unixfs.InlineLimit(inlineLimit), |
| 473 | |
| 474 | options.Unixfs.Chunker(chunker), |
| 475 | |
| 476 | options.Unixfs.Pin(dopin, pinName), |
| 477 | options.Unixfs.HashOnly(onlyHash), |
| 478 | options.Unixfs.FsCache(fscache), |
| 479 | options.Unixfs.Nocopy(nocopy), |
| 480 | |
| 481 | options.Unixfs.Progress(progress), |
| 482 | options.Unixfs.Silent(silent), |
| 483 | |
| 484 | options.Unixfs.PreserveMode(preserveMode), |
| 485 | options.Unixfs.PreserveMtime(preserveMtime), |
| 486 | |
| 487 | options.Unixfs.IncludeEmptyDirs(emptyDirs), |
| 488 | } |
| 489 | |
| 490 | if mode != 0 { |
| 491 | opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) |
| 492 | } |
| 493 | |
| 494 | if mtime != 0 { |
| 495 | opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) |
| 496 | } else if mtimeNsecs != 0 { |
| 497 | return fmt.Errorf("option %q requires %q to be provided as well", mtimeNsecsOptionName, mtimeOptionName) |
| 498 | } |
| 499 | |
| 500 | if cidVerSet { |
| 501 | opts = append(opts, options.Unixfs.CidVersion(cidVer)) |
| 502 | } |
| 503 | |
| 504 | if rbset { |
| 505 | opts = append(opts, options.Unixfs.RawLeaves(rawblks)) |
| 506 | } |
| 507 | |
| 508 | if maxFileLinksSet { |
| 509 | opts = append(opts, options.Unixfs.MaxFileLinks(maxFileLinks)) |
| 510 | } |
| 511 | |
| 512 | if maxDirectoryLinksSet { |
| 513 | opts = append(opts, options.Unixfs.MaxDirectoryLinks(maxDirectoryLinks)) |
| 514 | } |
| 515 | |
| 516 | if maxHAMTFanoutSet { |
| 517 | opts = append(opts, options.Unixfs.MaxHAMTFanout(maxHAMTFanout)) |
| 518 | } |
| 519 | |
| 520 | // SizeEstimationMode is always set from config |
| 521 | opts = append(opts, options.Unixfs.SizeEstimationMode(sizeEstimationMode)) |
| 522 | |
| 523 | if trickle { |
| 524 | opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) |
| 525 | } |
| 526 | |
| 527 | opts = append(opts, nil) // events option placeholder |
| 528 | |
| 529 | ipfsNode, err := cmdenv.GetNode(env) |
| 530 | if err != nil { |
| 531 | return err |
| 532 | } |
| 533 | var added int |
| 534 | var fileAddedToMFS bool |
| 535 | var lastRootCid path.ImmutablePath // Track the root CID for fast-provide |
| 536 | addit := toadd.Entries() |
| 537 | for addit.Next() { |
| 538 | _, dir := addit.Node().(files.Directory) |
| 539 | errCh := make(chan error, 1) |
| 540 | events := make(chan any, adderOutChanSize) |
| 541 | opts[len(opts)-1] = options.Unixfs.Events(events) |
| 542 | |
| 543 | go func() { |
| 544 | var err error |
| 545 | defer close(events) |
| 546 | pathAdded, err := api.Unixfs().Add(req.Context, addit.Node(), opts...) |
| 547 | if err != nil { |
| 548 | errCh <- err |
| 549 | return |
| 550 | } |
| 551 | |
| 552 | // Store the root CID for potential fast-provide operation |
| 553 | lastRootCid = pathAdded |
| 554 | |
| 555 | // creating MFS pointers when optional --to-files is set |
| 556 | if toFilesSet { |
| 557 | if addit.Name() == "" { |
| 558 | errCh <- fmt.Errorf("%s: cannot add unnamed files to MFS", toFilesOptionName) |
| 559 | return |
| 560 | } |
| 561 | |
| 562 | if toFilesStr == "" { |
| 563 | toFilesStr = "/" |
| 564 | } |
| 565 | toFilesDst, err := checkPath(toFilesStr) |
| 566 | if err != nil { |
| 567 | errCh <- fmt.Errorf("%s: %w", toFilesOptionName, err) |
| 568 | return |
| 569 | } |
| 570 | dstAsDir := toFilesDst[len(toFilesDst)-1] == '/' |
| 571 | |
| 572 | if dstAsDir { |
| 573 | mfsNode, err := mfs.Lookup(ipfsNode.FilesRoot, toFilesDst) |
| 574 | // confirm dst exists |
| 575 | if err != nil { |
| 576 | errCh <- fmt.Errorf("%s: MFS destination directory %q does not exist: %w", toFilesOptionName, toFilesDst, err) |
| 577 | return |
| 578 | } |
| 579 | // confirm dst is a dir |
| 580 | if mfsNode.Type() != mfs.TDir { |
| 581 | errCh <- fmt.Errorf("%s: MFS destination %q is not a directory", toFilesOptionName, toFilesDst) |
| 582 | return |
| 583 | } |
| 584 | // if MFS destination is a dir, append filename to the dir path |
| 585 | toFilesDst += gopath.Base(addit.Name()) |
| 586 | } |
| 587 | |
| 588 | // error if we try to overwrite a preexisting file destination |
| 589 | if fileAddedToMFS && !dstAsDir { |
| 590 | errCh <- fmt.Errorf("%s: MFS destination is a file: only one entry can be copied to %q", toFilesOptionName, toFilesDst) |
| 591 | return |
| 592 | } |
| 593 | |
| 594 | _, err = mfs.Lookup(ipfsNode.FilesRoot, gopath.Dir(toFilesDst)) |
| 595 | if err != nil { |
| 596 | errCh <- fmt.Errorf("%s: MFS destination parent %q %q does not exist: %w", toFilesOptionName, toFilesDst, gopath.Dir(toFilesDst), err) |
| 597 | return |
| 598 | } |
| 599 | |
| 600 | var nodeAdded ipld.Node |
| 601 | nodeAdded, err = api.Dag().Get(req.Context, pathAdded.RootCid()) |
| 602 | if err != nil { |
| 603 | errCh <- err |
| 604 | return |
| 605 | } |
| 606 | err = mfs.PutNode(ipfsNode.FilesRoot, toFilesDst, nodeAdded) |
| 607 | if err != nil { |
| 608 | errCh <- fmt.Errorf("%s: cannot put node in path %q: %w", toFilesOptionName, toFilesDst, err) |
| 609 | return |
| 610 | } |
| 611 | fileAddedToMFS = true |
| 612 | } |
| 613 | errCh <- err |
| 614 | }() |
| 615 | |
| 616 | for event := range events { |
| 617 | output, ok := event.(*coreiface.AddEvent) |
| 618 | if !ok { |
| 619 | return errors.New("unknown event type") |
| 620 | } |
| 621 | |
| 622 | h := "" |
| 623 | if (output.Path != path.ImmutablePath{}) { |
| 624 | h = enc.Encode(output.Path.RootCid()) |
| 625 | } |
| 626 | |
| 627 | if !dir && addit.Name() != "" { |
| 628 | output.Name = addit.Name() |
| 629 | } else { |
| 630 | output.Name = gopath.Join(addit.Name(), output.Name) |
| 631 | } |
| 632 | |
| 633 | output.Mode = addit.Node().Mode() |
| 634 | if ts := addit.Node().ModTime(); !ts.IsZero() { |
| 635 | output.Mtime = addit.Node().ModTime().Unix() |
| 636 | output.MtimeNsecs = addit.Node().ModTime().Nanosecond() |
| 637 | } |
| 638 | |
| 639 | addEvent := AddEvent{ |
| 640 | Name: output.Name, |
| 641 | Hash: h, |
| 642 | Bytes: output.Bytes, |
| 643 | Size: output.Size, |
| 644 | Mtime: output.Mtime, |
| 645 | MtimeNsecs: output.MtimeNsecs, |
| 646 | } |
| 647 | |
| 648 | if output.Mode != 0 { |
| 649 | addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8) |
| 650 | } |
| 651 | |
| 652 | if output.Mtime > 0 { |
| 653 | addEvent.Mtime = output.Mtime |
| 654 | if output.MtimeNsecs > 0 { |
| 655 | addEvent.MtimeNsecs = output.MtimeNsecs |
| 656 | } |
| 657 | } |
| 658 | |
| 659 | if err := res.Emit(&addEvent); err != nil { |
| 660 | return err |
| 661 | } |
| 662 | } |
| 663 | |
| 664 | if err := <-errCh; err != nil { |
| 665 | return err |
| 666 | } |
| 667 | added++ |
| 668 | } |
| 669 | |
| 670 | if addit.Err() != nil { |
| 671 | return addit.Err() |
| 672 | } |
| 673 | |
| 674 | if added == 0 { |
| 675 | return fmt.Errorf("expected a file argument") |
| 676 | } |
| 677 | |
| 678 | hasRoot := lastRootCid != path.ImmutablePath{} |
| 679 | |
| 680 | if fastProvideDAG && hasRoot { |
| 681 | // DAG walk includes the root CID (DFS pre-order emits it |
| 682 | // first), so a separate root provide is not needed. |
| 683 | cmdenv.ExecuteFastProvideDAG( |
| 684 | req.Context, |
| 685 | ipfsNode.Context(), |
| 686 | []cid.Cid{lastRootCid.RootCid()}, |
| 687 | ipfsNode.ProvidingStrategy, |
| 688 | ipfsNode.Blockstore, |
| 689 | ipfsNode.Provider, |
| 690 | fastProvideWait, |
| 691 | uint(cfg.Provide.BloomFPRate.WithDefault(config.DefaultProvideBloomFPRate)), |
| 692 | 0, // block count unknown here; bloom chain auto-grows |
| 693 | ) |
| 694 | } else if fastProvideRoot && hasRoot { |
| 695 | cfg, err := ipfsNode.Repo.Config() |
| 696 | if err != nil { |
| 697 | return err |
| 698 | } |
| 699 | if err := cmdenv.ExecuteFastProvideRoot(req.Context, ipfsNode, cfg, lastRootCid.RootCid(), fastProvideWait, dopin, dopin, toFilesSet); err != nil { |
| 700 | return err |
| 701 | } |
| 702 | } else if !fastProvideRoot && !fastProvideDAG { |
| 703 | log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config") |
| 704 | if fastProvideWait { |
| 705 | log.Debugw("fast-provide-root: wait-flag-ignored") |
| 706 | } |
| 707 | } |
| 708 | |
| 709 | return nil |
| 710 | }, |
| 711 | PostRun: cmds.PostRunMap{ |
| 712 | cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { |
| 713 | sizeChan := make(chan int64, 1) |
| 714 | outChan := make(chan any) |
| 715 | req := res.Request() |
| 716 | |
| 717 | // Could be slow. |
| 718 | go func() { |
| 719 | size, err := req.Files.Size() |
| 720 | if err != nil { |
| 721 | log.Warnf("error getting files size: %s", err) |
| 722 | // see comment above |
| 723 | return |
| 724 | } |
| 725 | |
| 726 | sizeChan <- size |
| 727 | }() |
| 728 | |
| 729 | progressBar := func(wait chan struct{}) { |
| 730 | defer close(wait) |
| 731 | |
| 732 | quiet, _ := req.Options[quietOptionName].(bool) |
| 733 | quieter, _ := req.Options[quieterOptionName].(bool) |
| 734 | quiet = quiet || quieter |
| 735 | |
| 736 | progress, _ := req.Options[progressOptionName].(bool) |
| 737 | |
| 738 | var bar *pb.ProgressBar |
| 739 | if progress { |
| 740 | bar = pb.New64(0).Set(pb.Bytes, true).Set(pb.Static, true).SetWriter(os.Stderr) |
| 741 | bar.SetTemplateString(progressBarInitTemplate) |
| 742 | bar.Start() |
| 743 | } |
| 744 | |
| 745 | lastFile := "" |
| 746 | lastHash := "" |
| 747 | var totalProgress, prevFiles, lastBytes int64 |
| 748 | |
| 749 | LOOP: |
| 750 | for { |
| 751 | select { |
| 752 | case out, ok := <-outChan: |
| 753 | if !ok { |
| 754 | if quieter { |
| 755 | fmt.Fprintln(os.Stdout, lastHash) |
| 756 | } |
| 757 | |
| 758 | break LOOP |
| 759 | } |
| 760 | output := out.(*AddEvent) |
| 761 | if len(output.Hash) > 0 { |
| 762 | lastHash = output.Hash |
| 763 | if quieter { |
| 764 | continue |
| 765 | } |
| 766 | |
| 767 | if progress { |
| 768 | // clear progress bar line before we print "added x" output |
| 769 | fmt.Fprintf(os.Stderr, "\033[2K\r") |
| 770 | } |
| 771 | if quiet { |
| 772 | fmt.Fprintf(os.Stdout, "%s\n", output.Hash) |
| 773 | } else { |
| 774 | fmt.Fprintf(os.Stdout, "added %s %s\n", output.Hash, cmdenv.EscNonPrint(output.Name)) |
| 775 | } |
| 776 | |
| 777 | } else { |
| 778 | if !progress { |
| 779 | continue |
| 780 | } |
| 781 | |
| 782 | if len(lastFile) == 0 { |
| 783 | lastFile = output.Name |
| 784 | } |
| 785 | if output.Name != lastFile || output.Bytes < lastBytes { |
| 786 | prevFiles += lastBytes |
| 787 | lastFile = output.Name |
| 788 | } |
| 789 | lastBytes = output.Bytes |
| 790 | delta := prevFiles + lastBytes - totalProgress |
| 791 | bar.Add64(delta) |
| 792 | totalProgress = bar.Current() |
| 793 | } |
| 794 | |
| 795 | if progress { |
| 796 | bar.Write() |
| 797 | } |
| 798 | case size := <-sizeChan: |
| 799 | if progress { |
| 800 | bar.SetTotal(size) |
| 801 | bar.SetTemplateString(cmdenv.ProgressBarFullTemplate) |
| 802 | } |
| 803 | case <-req.Context.Done(): |
| 804 | // don't set or print error here, that happens in the goroutine below |
| 805 | return |
| 806 | } |
| 807 | } |
| 808 | |
| 809 | if progress { |
| 810 | // If size discovery never reported, treat the |
| 811 | // observed bytes as the total so the final frame |
| 812 | // renders the bar and percent. |
| 813 | if bar.Total() == 0 && bar.Current() != 0 { |
| 814 | bar.SetTotal(bar.Current()) |
| 815 | bar.SetTemplateString(cmdenv.ProgressBarFullTemplate) |
| 816 | } |
| 817 | // Finish first so the speed element switches to |
| 818 | // the absolute-rate branch (total/elapsed) when |
| 819 | // EWMA never accumulated a sample on fast adds. |
| 820 | bar.Finish() |
| 821 | bar.Write() |
| 822 | } |
| 823 | } |
| 824 | |
| 825 | if e := res.Error(); e != nil { |
| 826 | close(outChan) |
| 827 | return e |
| 828 | } |
| 829 | |
| 830 | wait := make(chan struct{}) |
| 831 | go progressBar(wait) |
| 832 | |
| 833 | defer func() { <-wait }() |
| 834 | defer close(outChan) |
| 835 | |
| 836 | for { |
| 837 | v, err := res.Next() |
| 838 | if err != nil { |
| 839 | if err == io.EOF { |
| 840 | return nil |
| 841 | } |
| 842 | |
| 843 | return err |
| 844 | } |
| 845 | |
| 846 | select { |
| 847 | case outChan <- v: |
| 848 | case <-req.Context.Done(): |
| 849 | return req.Context.Err() |
| 850 | } |
| 851 | } |
| 852 | }, |
| 853 | }, |
| 854 | Type: AddEvent{}, |
| 855 | } |