fix(mfs): basic UnixFS sanity checks in `files cp` (#10701)
Signed-off-by: Abhinav Prakash <abhinav.prakash319@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Andrew Gillis <11790789+gammazero@users.noreply.github.com>
Abhinav Prakash committed
Mar 6, 2025 at 01:10 UTC
e221e941c7db1de6a71334d2f0da25cdd09b60de
4 files changed
+190
-2
core/commands/files.go
+20
@@ -402,6 +402,7 @@ func walkBlock(ctx context.Context, dagserv ipld.DAGService, nd ipld.Node) (bool
402
return local, sizeLocal, nil
403
}
404
405
+var errFilesCpInvalidUnixFS = errors.New("cp: source must be a valid UnixFS (dag-pb or raw codec)")
406
var filesCpCmd = &cmds.Command{
407
Helptext: cmds.HelpText{
408
Tagline: "Add references to IPFS files and directories in MFS (or copy within MFS).",
@@ -480,6 +481,25 @@ being GC'ed.
481
return fmt.Errorf("cp: cannot get node from path %s: %s", src, err)
482
}
483
484
+ // Sanity-check: ensure root CID is a valid UnixFS (dag-pb or raw block)
485
+ // Context: https://github.com/ipfs/kubo/issues/10331
486
+ srcCidType := node.Cid().Type()
487
+ switch srcCidType {
488
+ case cid.Raw:
489
+ if _, ok := node.(*dag.RawNode); !ok {
490
+ return errFilesCpInvalidUnixFS
491
+ }
492
+ case cid.DagProtobuf:
493
+ if _, ok := node.(*dag.ProtoNode); !ok {
494
+ return errFilesCpInvalidUnixFS
495
+ }
496
+ if _, err = ft.FSNodeFromBytes(node.(*dag.ProtoNode).Data()); err != nil {
497
+ return fmt.Errorf("%w: %v", errFilesCpInvalidUnixFS, err)
498
+ }
499
+ default:
500
+ return errFilesCpInvalidUnixFS
501
+ }
502
+
503
if mkParents {
504
err := ensureContainingDirectoryExists(nd.FilesRoot, dst, prefix)
505
if err != nil {
core/commands/files_test.go
new
+47
@@ -0,0 +1,47 @@
1
+package commands
2
+
3
+import (
4
+ "context"
5
+ "io"
6
+ "testing"
7
+
8
+ dag "github.com/ipfs/boxo/ipld/merkledag"
9
+ cmds "github.com/ipfs/go-ipfs-cmds"
10
+ coremock "github.com/ipfs/kubo/core/mock"
11
+ "github.com/stretchr/testify/require"
12
+)
13
+
14
+func TestFilesCp_DagCborNodeFails(t *testing.T) {
15
+ ctx, cancel := context.WithCancel(context.Background())
16
+ defer cancel()
17
+
18
+ cmdCtx, err := coremock.MockCmdsCtx()
19
+ require.NoError(t, err)
20
+
21
+ node, err := cmdCtx.ConstructNode()
22
+ require.NoError(t, err)
23
+
24
+ invalidData := []byte{0x00}
25
+ protoNode := dag.NodeWithData(invalidData)
26
+ err = node.DAG.Add(ctx, protoNode)
27
+ require.NoError(t, err)
28
+
29
+ req := &cmds.Request{
30
+ Context: ctx,
31
+ Arguments: []string{
32
+ "/ipfs/" + protoNode.Cid().String(),
33
+ "/test-destination",
34
+ },
35
+ Options: map[string]interface{}{
36
+ "force": false,
37
+ },
38
+ }
39
+
40
+ _, pw := io.Pipe()
41
+ res, err := cmds.NewWriterResponseEmitter(pw, req)
42
+ require.NoError(t, err)
43
+
44
+ err = filesCpCmd.Run(req, res, &cmdCtx)
45
+ require.Error(t, err)
46
+ require.ErrorContains(t, err, "cp: source must be a valid UnixFS (dag-pb or raw codec)")
47
+}
docs/changelogs/v0.34.md
+3
-2
@@ -12,8 +12,8 @@
12
- [`IPFS_LOG_LEVEL` deprecated](#ipfs_log_level-deprecated)
13
- [Pebble datastore format update](#pebble-datastore-format-update)
14
- [Badger datastore update](#badger-datastore-update)
15
- - [Datastore Implementation updates](#datastore-implementation-updates)
16
- - [One multi-error package](#one-multi-error-package)
15
+ - [Datastore Implementation Updates](#datastore-implementation-updates)
16
+ - [One Multi-error Package](#one-multi-error-package)
17
- [📦️ Important dependency updates](#-important-dependency-updates)
18
- [👨👩👧👦 Contributors](#-contributors)
19
@@ -48,6 +48,7 @@ For more details, check out the [`AutoTLS` configuration documentation](https://
48
- `ipfs config` is now validating json fields ([#10679](https://github.com/ipfs/kubo/pull/10679)).
49
- Deprecated the `bitswap reprovide` command. Make sure to switch to modern `routing reprovide`. ([#10677](https://github.com/ipfs/kubo/pull/10677))
50
- The `stats reprovide` command now shows additional stats for [`Routing.AcceleratedDHTClient`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient), indicating the last and next `reprovide` times. ([#10677](https://github.com/ipfs/kubo/pull/10677))
51
+- `ipfs files cp` now performs basic codec check and will error when source is not a valid UnixFS (only `dag-pb` and `raw` codecs are allowed in MFS)
52
53
#### Bitswap improvements from Boxo
54
test/cli/files_test.go
new
+120
@@ -0,0 +1,120 @@
1
+package cli
2
+
3
+import (
4
+ "fmt"
5
+ "os"
6
+ "path/filepath"
7
+ "testing"
8
+
9
+ "github.com/ipfs/kubo/test/cli/harness"
10
+ "github.com/stretchr/testify/assert"
11
+ "github.com/stretchr/testify/require"
12
+)
13
+
14
+func TestFilesCp(t *testing.T) {
15
+ t.Parallel()
16
+
17
+ t.Run("files cp with valid UnixFS succeeds", func(t *testing.T) {
18
+ t.Parallel()
19
+
20
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
21
+
22
+ // Create simple text file
23
+ data := "testing files cp command"
24
+ cid := node.IPFSAddStr(data)
25
+
26
+ // Copy form IPFS => MFS
27
+ res := node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/valid-file")
28
+ assert.NoError(t, res.Err)
29
+
30
+ // verification
31
+ catRes := node.IPFS("files", "read", "/valid-file")
32
+ assert.Equal(t, data, catRes.Stdout.Trimmed())
33
+ })
34
+
35
+ t.Run("files cp with unsupported DAG node type fails", func(t *testing.T) {
36
+ t.Parallel()
37
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
38
+
39
+ // MFS UnixFS is limited to dag-pb or raw, so we create a dag-cbor node to test this
40
+ jsonData := `{"data": "not a UnixFS node"}`
41
+ tempFile := filepath.Join(node.Dir, "test.json")
42
+ err := os.WriteFile(tempFile, []byte(jsonData), 0644)
43
+ require.NoError(t, err)
44
+ cid := node.IPFS("dag", "put", "--input-codec=json", "--store-codec=dag-cbor", tempFile).Stdout.Trimmed()
45
+
46
+ // copy without --force
47
+ res := node.RunIPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/invalid-file")
48
+ assert.NotEqual(t, 0, res.ExitErr.ExitCode())
49
+ assert.Contains(t, res.Stderr.String(), "Error: cp: source must be a valid UnixFS (dag-pb or raw codec)")
50
+ })
51
+
52
+ t.Run("files cp with invalid UnixFS data structure fails", func(t *testing.T) {
53
+ t.Parallel()
54
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
55
+
56
+ // Create an invalid proto file
57
+ data := []byte{0xDE, 0xAD, 0xBE, 0xEF} // Invalid protobuf data
58
+ tempFile := filepath.Join(node.Dir, "invalid-proto.bin")
59
+ err := os.WriteFile(tempFile, data, 0644)
60
+ require.NoError(t, err)
61
+
62
+ res := node.IPFS("block", "put", "--format=raw", tempFile)
63
+ require.NoError(t, res.Err)
64
+
65
+ // we manually changed codec from raw to dag-pb to test "bad dag-pb" scenario
66
+ cid := "bafybeic7pdbte5heh6u54vszezob3el6exadoiw4wc4ne7ny2x7kvajzkm"
67
+
68
+ // should fail because node cant be read as a valid dag-pb
69
+ cpResNoForce := node.RunIPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/invalid-proto")
70
+ assert.NotEqual(t, 0, cpResNoForce.ExitErr.ExitCode())
71
+ assert.Contains(t, cpResNoForce.Stderr.String(), "Error")
72
+ })
73
+
74
+ t.Run("files cp with raw node succeeds", func(t *testing.T) {
75
+ t.Parallel()
76
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
77
+
78
+ // Create a raw node
79
+ data := "raw data"
80
+ tempFile := filepath.Join(node.Dir, "raw.bin")
81
+ err := os.WriteFile(tempFile, []byte(data), 0644)
82
+ require.NoError(t, err)
83
+
84
+ res := node.IPFS("block", "put", "--format=raw", tempFile)
85
+ require.NoError(t, res.Err)
86
+ cid := res.Stdout.Trimmed()
87
+
88
+ // Copy from IPFS to MFS (raw nodes should work without --force)
89
+ cpRes := node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/raw-file")
90
+ assert.NoError(t, cpRes.Err)
91
+
92
+ // Verify the file was copied correctly
93
+ catRes := node.IPFS("files", "read", "/raw-file")
94
+ assert.Equal(t, data, catRes.Stdout.Trimmed())
95
+ })
96
+
97
+ t.Run("files cp creates intermediate directories with -p", func(t *testing.T) {
98
+ t.Parallel()
99
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
100
+
101
+ // Create a simple text file and add it to IPFS
102
+ data := "hello parent directories"
103
+ tempFile := filepath.Join(node.Dir, "parent-test.txt")
104
+ err := os.WriteFile(tempFile, []byte(data), 0644)
105
+ require.NoError(t, err)
106
+
107
+ cid := node.IPFS("add", "-Q", tempFile).Stdout.Trimmed()
108
+
109
+ // Copy from IPFS to MFS with parent flag
110
+ res := node.IPFS("files", "cp", "-p", fmt.Sprintf("/ipfs/%s", cid), "/parent/dir/file")
111
+ assert.NoError(t, res.Err)
112
+
113
+ // Verify the file and directories were created
114
+ lsRes := node.IPFS("files", "ls", "/parent/dir")
115
+ assert.Contains(t, lsRes.Stdout.String(), "file")
116
+
117
+ catRes := node.IPFS("files", "read", "/parent/dir/file")
118
+ assert.Equal(t, data, catRes.Stdout.Trimmed())
119
+ })
120
+}