feat(cmds/add): --to-files option automates files cp (#8927)
* feat(cmds/add): --to-files option as files cp * tests(to-files): ensure error handling is covered this adds bunch of tests that guard UX around importing multiple files into MFS, and the logic around trailing slash to indicate if the MFS destination if a directory. If the destination has a trailing slash, we ensure that the directory exists and is a dir and not a file. this allows us to support adding multipl files into MFS dir: ipfs add file1.txt file2.txt --to-files /some/mfs/dir/ * docs: to-files helptext Co-authored-by: Antonio Navarro Perez <antnavper@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>
Lucas Molas committed
Sep 21, 2022 at 13:08 UTC
9e5d0aaaeca70ae76384d52fc93a4f404b642858
3 files changed
+219
-9
core/commands/add.go
+85
-3
@@ -13,6 +13,8 @@ import (
13
"github.com/cheggaaa/pb"
14
cmds "github.com/ipfs/go-ipfs-cmds"
15
files "github.com/ipfs/go-ipfs-files"
16
+ ipld "github.com/ipfs/go-ipld-format"
17
+ mfs "github.com/ipfs/go-mfs"
18
coreiface "github.com/ipfs/interface-go-ipfs-core"
19
"github.com/ipfs/interface-go-ipfs-core/options"
20
mh "github.com/multiformats/go-multihash"
@@ -45,6 +47,7 @@ const (
47
hashOptionName = "hash"
48
inlineOptionName = "inline"
49
inlineLimitOptionName = "inline-limit"
50
+ toFilesOptionName = "to-files"
51
)
52
53
const adderOutChanSize = 8
@@ -79,6 +82,20 @@ You can now refer to the added file in a gateway, like so:
82
83
/ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
84
85
+Files imported with 'ipfs add' are protected from GC (implicit '--pin=true'),
86
+but it is up to you to remember the returned CID to get the data back later.
87
+
88
+Passing '--to-files' creates a reference in Files API (MFS), making it easier
89
+to find it in the future:
90
+
91
+ > ipfs files mkdir -p /myfs/dir
92
+ > ipfs add example.jpg --to-files /myfs/dir/
93
+ > ipfs files ls /myfs/dir/
94
+ example.jpg
95
+
96
+See 'ipfs files --help' to learn more about using MFS
97
+for keeping track of added files and directories.
98
+
99
The chunker option, '-s', specifies the chunking strategy that dictates
100
how to break files into blocks. Blocks with same content can
101
be deduplicated. Different chunking strategies will produce different
@@ -138,7 +155,6 @@ See 'dag export' and 'dag import' for more information.
155
cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
156
cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
157
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
141
- cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
158
cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
159
cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
160
cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
@@ -146,6 +162,8 @@ See 'dag export' and 'dag import' for more information.
162
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
163
cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
164
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
165
+ cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
166
+ cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."),
167
},
168
PreRun: func(req *cmds.Request, env cmds.Environment) error {
169
quiet, _ := req.Options[quietOptionName].(bool)
@@ -186,10 +204,11 @@ See 'dag export' and 'dag import' for more information.
204
hashFunStr, _ := req.Options[hashOptionName].(string)
205
inline, _ := req.Options[inlineOptionName].(bool)
206
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
207
+ toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
208
209
hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
210
if !ok {
192
- return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
211
+ return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr))
212
}
213
214
enc, err := cmdenv.GetCidEncoder(req)
@@ -235,7 +254,12 @@ See 'dag export' and 'dag import' for more information.
254
255
opts = append(opts, nil) // events option placeholder
256
257
+ ipfsNode, err := cmdenv.GetNode(env)
258
+ if err != nil {
259
+ return err
260
+ }
261
var added int
262
+ var fileAddedToMFS bool
263
addit := toadd.Entries()
264
for addit.Next() {
265
_, dir := addit.Node().(files.Directory)
@@ -246,7 +270,65 @@ See 'dag export' and 'dag import' for more information.
270
go func() {
271
var err error
272
defer close(events)
249
- _, err = api.Unixfs().Add(req.Context, addit.Node(), opts...)
273
+ pathAdded, err := api.Unixfs().Add(req.Context, addit.Node(), opts...)
274
+ if err != nil {
275
+ errCh <- err
276
+ return
277
+ }
278
+
279
+ // creating MFS pointers when optional --to-files is set
280
+ if toFilesSet {
281
+ if toFilesStr == "" {
282
+ toFilesStr = "/"
283
+ }
284
+ toFilesDst, err := checkPath(toFilesStr)
285
+ if err != nil {
286
+ errCh <- fmt.Errorf("%s: %w", toFilesOptionName, err)
287
+ return
288
+ }
289
+ dstAsDir := toFilesDst[len(toFilesDst)-1] == '/'
290
+
291
+ if dstAsDir {
292
+ mfsNode, err := mfs.Lookup(ipfsNode.FilesRoot, toFilesDst)
293
+ // confirm dst exists
294
+ if err != nil {
295
+ errCh <- fmt.Errorf("%s: MFS destination directory %q does not exist: %w", toFilesOptionName, toFilesDst, err)
296
+ return
297
+ }
298
+ // confirm dst is a dir
299
+ if mfsNode.Type() != mfs.TDir {
300
+ errCh <- fmt.Errorf("%s: MFS destination %q is not a directory", toFilesOptionName, toFilesDst)
301
+ return
302
+ }
303
+ // if MFS destination is a dir, append filename to the dir path
304
+ toFilesDst += path.Base(addit.Name())
305
+ }
306
+
307
+ // error if we try to overwrite a preexisting file destination
308
+ if fileAddedToMFS && !dstAsDir {
309
+ errCh <- fmt.Errorf("%s: MFS destination is a file: only one entry can be copied to %q", toFilesOptionName, toFilesDst)
310
+ return
311
+ }
312
+
313
+ _, err = mfs.Lookup(ipfsNode.FilesRoot, path.Dir(toFilesDst))
314
+ if err != nil {
315
+ errCh <- fmt.Errorf("%s: MFS destination parent %q %q does not exist: %w", toFilesOptionName, toFilesDst, path.Dir(toFilesDst), err)
316
+ return
317
+ }
318
+
319
+ var nodeAdded ipld.Node
320
+ nodeAdded, err = api.Dag().Get(req.Context, pathAdded.Cid())
321
+ if err != nil {
322
+ errCh <- err
323
+ return
324
+ }
325
+ err = mfs.PutNode(ipfsNode.FilesRoot, toFilesDst, nodeAdded)
326
+ if err != nil {
327
+ errCh <- fmt.Errorf("%s: cannot put node in path %q: %w", toFilesOptionName, toFilesDst, err)
328
+ return
329
+ }
330
+ fileAddedToMFS = true
331
+ }
332
errCh <- err
333
}()
334
core/commands/files.go
+27
-6
@@ -581,7 +581,7 @@ const (
581
582
var filesReadCmd = &cmds.Command{
583
Helptext: cmds.HelpText{
584
- Tagline: "Read a file in a given MFS.",
584
+ Tagline: "Read a file from MFS.",
585
ShortDescription: `
586
Read a specified number of bytes from a file at a given offset. By default,
587
it will read the entire file similar to the Unix cat.
@@ -724,11 +724,16 @@ const (
724
725
var filesWriteCmd = &cmds.Command{
726
Helptext: cmds.HelpText{
727
- Tagline: "Write to a mutable file in a given filesystem.",
727
+ Tagline: "Append to (modify) a file in MFS.",
728
ShortDescription: `
729
-Write data to a file in a given filesystem. This command allows you to specify
730
-a beginning offset to write to. The entire length of the input will be
731
-written.
729
+A low-level MFS command that allows you to append data to a file. If you want
730
+to add a file without modifying an existing one, use 'ipfs add --to-files'
731
+instead.
732
+`,
733
+ LongDescription: `
734
+A low-level MFS command that allows you to append data at the end of a file, or
735
+specify a beginning offset within a file to write to. The entire length of the
736
+input will be written.
737
738
If the '--create' option is specified, the file will be created if it does not
739
exist. Nonexistent intermediate directories will not be created unless the
@@ -755,6 +760,22 @@ WARNING:
760
Usage of the '--flush=false' option does not guarantee data durability until
761
the tree has been flushed. This can be accomplished by running 'ipfs files
762
stat' on the file or any of its ancestors.
763
+
764
+WARNING:
765
+
766
+The CID produced by 'files write' will be different from 'ipfs add' because
767
+'ipfs file write' creates a trickle-dag optimized for append-only operations
768
+See '--trickle' in 'ipfs add --help' for more information.
769
+
770
+If you want to add a file without modifying an existing one,
771
+use 'ipfs add' with '--to-files':
772
+
773
+ > ipfs files mkdir -p /myfs/dir
774
+ > ipfs add example.jpg --to-files /myfs/dir/
775
+ > ipfs files ls /myfs/dir/
776
+ example.jpg
777
+
778
+See '--to-files' in 'ipfs add --help' for more information.
779
`,
780
},
781
Arguments: []cmds.Argument{
@@ -1019,7 +1040,7 @@ func updatePath(rt *mfs.Root, pth string, builder cid.Builder) error {
1040
1041
var filesRmCmd = &cmds.Command{
1042
Helptext: cmds.HelpText{
1022
- Tagline: "Remove a file.",
1043
+ Tagline: "Remove a file from MFS.",
1044
ShortDescription: `
1045
Remove files or directories.
1046
test/sharness/t0040-add-and-cat.sh
+107
@@ -362,6 +362,113 @@ test_add_cat_file() {
362
rm mountdir/same-file/hello.txt &&
363
rmdir mountdir/same-file
364
'
365
+
366
+ ## --to-files with single source
367
+
368
+ test_expect_success "ipfs add --to-files /mfspath succeeds" '
369
+ mkdir -p mountdir && echo "Hello MFS!" > mountdir/mfs.txt &&
370
+ ipfs add mountdir/mfs.txt --to-files /ipfs-add-to-files >actual
371
+ '
372
+
373
+ test_expect_success "ipfs add --to-files output looks good" '
374
+ HASH_MFS="QmVT8bL3sGBA2TwvX8JPhrv5CYZL8LLLfW7mxkUjPZsgBr" &&
375
+ echo "added $HASH_MFS mfs.txt" >expected &&
376
+ test_cmp expected actual
377
+ '
378
+
379
+ test_expect_success "ipfs files read succeeds" '
380
+ ipfs files read /ipfs-add-to-files >actual &&
381
+ ipfs files rm /ipfs-add-to-files
382
+ '
383
+
384
+ test_expect_success "ipfs cat output looks good" '
385
+ echo "Hello MFS!" >expected &&
386
+ test_cmp expected actual
387
+ '
388
+
389
+ test_expect_success "ipfs add --to-files requires argument" '
390
+ test_expect_code 1 ipfs add mountdir/mfs.txt --to-files >actual 2>&1 &&
391
+ test_should_contain "Error: missing argument for option \"to-files\"" actual
392
+ '
393
+
394
+ test_expect_success "ipfs add --to-files / (MFS root) works" '
395
+ echo "Hello MFS!" >expected &&
396
+ ipfs add mountdir/mfs.txt --to-files / &&
397
+ ipfs files read /mfs.txt >actual &&
398
+ test_cmp expected actual &&
399
+ ipfs files rm /mfs.txt &&
400
+ rm mountdir/mfs.txt
401
+ '
402
+
403
+ ## --to-files with multiple sources
404
+
405
+ test_expect_success "ipfs add file1 file2 --to-files /mfspath0 (without trailing slash) fails" '
406
+ mkdir -p test &&
407
+ echo "file1" > test/mfs1.txt &&
408
+ echo "file2" > test/mfs2.txt &&
409
+ test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfspath0 >actual 2>&1 &&
410
+ test_should_contain "MFS destination is a file: only one entry can be copied to \"/mfspath0\"" actual &&
411
+ ipfs files rm -r --force /mfspath0
412
+ '
413
+
414
+ test_expect_success "ipfs add file1 file2 --to-files /mfsfile1 (without trailing slash + with preexisting file) fails" '
415
+ echo test | ipfs files write --create /mfsfile1 &&
416
+ test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsfile1 >actual 2>&1 &&
417
+ test_should_contain "Error: to-files: cannot put node in path \"/mfsfile1\"" actual &&
418
+ ipfs files rm -r --force /mfsfile1
419
+ '
420
+
421
+ test_expect_success "ipfs add file1 file2 --to-files /mfsdir1 (without trailing slash + with preexisting dir) fails" '
422
+ ipfs files mkdir -p /mfsdir1 &&
423
+ test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsdir1 >actual 2>&1 &&
424
+ test_should_contain "Error: to-files: cannot put node in path \"/mfsdir1\"" actual &&
425
+ ipfs files rm -r --force /mfsdir1
426
+ '
427
+
428
+ test_expect_success "ipfs add file1 file2 --to-files /mfsdir2/ (with trailing slash) succeeds" '
429
+ ipfs files mkdir -p /mfsdir2 &&
430
+ test_expect_code 0 ipfs add --cid-version 1 test/mfs1.txt test/mfs2.txt --to-files /mfsdir2/ > actual 2>&1 &&
431
+ test_should_contain "added bafkreihm3rktn5z33luic3youqdsn326toaq3ekesmdvsa53sbrd3f5r3a mfs1.txt" actual &&
432
+ test_should_contain "added bafkreidh5zkhr2vnwa2luwmuj24xo6l3jhfgvkgtk5cyp43oxs7owzpxby mfs2.txt" actual &&
433
+ test_should_not_contain "Error" actual &&
434
+ ipfs files ls /mfsdir2/ > lsout &&
435
+ test_should_contain "mfs1.txt" lsout &&
436
+ test_should_contain "mfs2.txt" lsout &&
437
+ ipfs files rm -r --force /mfsdir2
438
+ '
439
+
440
+ test_expect_success "ipfs add file1 file2 --to-files /mfsfile2/ (with trailing slash + with preexisting file) fails" '
441
+ echo test | ipfs files write --create /mfsfile2 &&
442
+ test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsfile2/ >actual 2>&1 &&
443
+ test_should_contain "Error: to-files: MFS destination \"/mfsfile2/\" is not a directory" actual &&
444
+ ipfs files rm -r --force /mfsfile2
445
+ '
446
+
447
+ ## --to-files with recursive dir
448
+
449
+ # test MFS destination without trailing slash
450
+ test_expect_success "ipfs add with --to-files /mfs/subdir3 fails because /mfs/subdir3 exists" '
451
+ ipfs files mkdir -p /mfs/subdir3 &&
452
+ test_expect_code 1 ipfs add -r test --to-files /mfs/subdir3 >actual 2>&1 &&
453
+ test_should_contain "cannot put node in path \"/mfs/subdir3\": directory already has entry by that name" actual &&
454
+ ipfs files rm -r --force /mfs
455
+ '
456
+
457
+ # test recursive import of a dir into MFS subdirectory
458
+ test_expect_success "ipfs add -r dir --to-files /mfs/subdir4/ succeeds (because of trailing slash)" '
459
+ ipfs files mkdir -p /mfs/subdir4 &&
460
+ ipfs add --cid-version 1 -r test --to-files /mfs/subdir4/ >actual 2>&1 &&
461
+ test_should_contain "added bafkreihm3rktn5z33luic3youqdsn326toaq3ekesmdvsa53sbrd3f5r3a test/mfs1.txt" actual &&
462
+ test_should_contain "added bafkreidh5zkhr2vnwa2luwmuj24xo6l3jhfgvkgtk5cyp43oxs7owzpxby test/mfs2.txt" actual &&
463
+ test_should_contain "added bafybeic7xwqwovt4g4bax6d3udp6222i63vj2rblpbim7uy2uw4a5gahha test" actual &&
464
+ test_should_not_contain "Error" actual
465
+ ipfs files ls /mfs/subdir4/ > lsout &&
466
+ test_should_contain "test" lsout &&
467
+ test_should_not_contain "mfs1.txt" lsout &&
468
+ test_should_not_contain "mfs2.txt" lsout &&
469
+ ipfs files rm -r --force /mfs
470
+ '
471
+
472
}
473
474
test_add_cat_5MB() {