fix add/cat of small files
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Oct 18, 2016 at 14:41 UTC
8ce9963289bcba3f596353f28a942814b5bfff65
9 files changed
+121
-72
core/commands/files/files.go
+9
-3
@@ -2,6 +2,7 @@ package commands
2
3
import (
4
"bytes"
5
+ "context"
6
"errors"
7
"fmt"
8
"io"
@@ -16,8 +17,8 @@ import (
17
path "github.com/ipfs/go-ipfs/path"
18
ft "github.com/ipfs/go-ipfs/unixfs"
19
19
- context "context"
20
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
21
+ node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
22
)
23
24
var log = logging.Logger("cmds/files")
@@ -160,7 +161,12 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
161
162
c := nd.Cid()
163
163
- d, err := ft.FromBytes(nd.Data())
164
+ pbnd, ok := nd.(*dag.ProtoNode)
165
+ if !ok {
166
+ return nil, dag.ErrNotProtobuf
167
+ }
168
+
169
+ d, err := ft.FromBytes(pbnd.Data())
170
if err != nil {
171
return nil, err
172
}
@@ -245,7 +251,7 @@ var FilesCpCmd = &cmds.Command{
251
},
252
}
253
248
-func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (*dag.ProtoNode, error) {
254
+func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (node.Node, error) {
255
switch {
256
case strings.HasPrefix(p, "/ipfs/"):
257
np, err := path.ParsePath(p)
core/corehttp/gateway_test.go
+10
-6
@@ -1,6 +1,7 @@
1
package corehttp
2
3
import (
4
+ "context"
5
"errors"
6
"io/ioutil"
7
"net/http"
@@ -9,14 +10,15 @@ import (
10
"testing"
11
"time"
12
12
- context "context"
13
core "github.com/ipfs/go-ipfs/core"
14
coreunix "github.com/ipfs/go-ipfs/core/coreunix"
15
+ dag "github.com/ipfs/go-ipfs/merkledag"
16
namesys "github.com/ipfs/go-ipfs/namesys"
17
path "github.com/ipfs/go-ipfs/path"
18
repo "github.com/ipfs/go-ipfs/repo"
19
config "github.com/ipfs/go-ipfs/repo/config"
20
testutil "github.com/ipfs/go-ipfs/thirdparty/testutil"
21
+
22
id "gx/ipfs/QmcRa2qn6iCmap9bjp8jAwkvYAq13AUfxdY3rrYiaJbLum/go-libp2p/p2p/protocol/identify"
23
ci "gx/ipfs/QmfWDLQjGjVe4fr5CoztYW2DYYjRysMJrFe1RCsXLPTf46/go-libp2p-crypto"
24
)
@@ -178,11 +180,13 @@ func TestIPNSHostnameRedirect(t *testing.T) {
180
if err != nil {
181
t.Fatal(err)
182
}
183
+
184
_, dagn2, err := coreunix.AddWrapped(n, strings.NewReader("_"), "index.html")
185
if err != nil {
186
t.Fatal(err)
187
}
185
- dagn1.AddNodeLink("foo", dagn2)
188
+
189
+ dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2)
190
if err != nil {
191
t.Fatal(err)
192
}
@@ -197,7 +201,7 @@ func TestIPNSHostnameRedirect(t *testing.T) {
201
t.Fatal(err)
202
}
203
200
- k := dagn1.Key()
204
+ k := dagn1.Cid()
205
t.Logf("k: %s\n", k)
206
ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
207
@@ -268,8 +272,8 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
272
if err != nil {
273
t.Fatal(err)
274
}
271
- dagn2.AddNodeLink("bar", dagn3)
272
- dagn1.AddNodeLink("foo? #<'", dagn2)
275
+ dagn2.(*dag.ProtoNode).AddNodeLink("bar", dagn3)
276
+ dagn1.(*dag.ProtoNode).AddNodeLink("foo? #<'", dagn2)
277
if err != nil {
278
t.Fatal(err)
279
}
@@ -287,7 +291,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
291
t.Fatal(err)
292
}
293
290
- k := dagn1.Key()
294
+ k := dagn1.Cid()
295
t.Logf("k: %s\n", k)
296
ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
297
core/coreunix/add.go
+5
-10
@@ -104,7 +104,7 @@ type Adder struct {
104
Silent bool
105
Wrap bool
106
Chunker string
107
- root *dag.ProtoNode
107
+ root node.Node
108
mr *mfs.Root
109
unlocker bs.Unlocker
110
tempRoot *cid.Cid
@@ -133,7 +133,7 @@ func (adder Adder) add(reader io.Reader) (node.Node, error) {
133
return balanced.BalancedLayout(params.New(chnk))
134
}
135
136
-func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
136
+func (adder *Adder) RootNode() (node.Node, error) {
137
// for memoizing
138
if adder.root != nil {
139
return adder.root, nil
@@ -151,12 +151,7 @@ func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
151
return nil, err
152
}
153
154
- pbnd, ok := nd.(*dag.ProtoNode)
155
- if !ok {
156
- return nil, dag.ErrNotProtobuf
157
- }
158
-
159
- root = pbnd
154
+ root = nd
155
}
156
157
adder.root = root
@@ -189,7 +184,7 @@ func (adder *Adder) PinRoot() error {
184
return adder.pinning.Flush()
185
}
186
192
-func (adder *Adder) Finalize() (*dag.ProtoNode, error) {
187
+func (adder *Adder) Finalize() (node.Node, error) {
188
root := adder.mr.GetValue()
189
190
// cant just call adder.RootNode() here as we need the name for printing
@@ -311,7 +306,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
306
// to preserve the filename.
307
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
308
// the directory, and and error if any.
314
-func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.ProtoNode, error) {
309
+func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.Node, error) {
310
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
311
fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG)
312
if err != nil {
core/coreunix/cat.go
+1
-7
@@ -4,7 +4,6 @@ import (
4
"context"
5
6
core "github.com/ipfs/go-ipfs/core"
7
- dag "github.com/ipfs/go-ipfs/merkledag"
7
path "github.com/ipfs/go-ipfs/path"
8
uio "github.com/ipfs/go-ipfs/unixfs/io"
9
)
@@ -15,10 +14,5 @@ func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (*uio.DagReader, er
14
return nil, err
15
}
16
18
- dnpb, ok := dagNode.(*dag.ProtoNode)
19
- if !ok {
20
- return nil, dag.ErrNotProtobuf
21
- }
22
-
23
- return uio.NewDagReader(ctx, dnpb, n.DAG)
17
+ return uio.NewDagReader(ctx, dagNode, n.DAG)
18
}
mfs/dir.go
+31
-19
@@ -89,7 +89,7 @@ func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) {
89
return d.node.Copy(), nil
90
}
91
92
-func (d *Directory) updateChild(name string, nd *dag.ProtoNode) error {
92
+func (d *Directory) updateChild(name string, nd node.Node) error {
93
err := d.node.RemoveNodeLink(name)
94
if err != nil && err != dag.ErrNotFound {
95
return err
@@ -121,28 +121,40 @@ func (d *Directory) childNode(name string) (FSNode, error) {
121
}
122
123
// cacheNode caches a node into d.childDirs or d.files and returns the FSNode.
124
-func (d *Directory) cacheNode(name string, nd *dag.ProtoNode) (FSNode, error) {
125
- i, err := ft.FromBytes(nd.Data())
126
- if err != nil {
127
- return nil, err
128
- }
124
+func (d *Directory) cacheNode(name string, nd node.Node) (FSNode, error) {
125
+ switch nd := nd.(type) {
126
+ case *dag.ProtoNode:
127
+ i, err := ft.FromBytes(nd.Data())
128
+ if err != nil {
129
+ return nil, err
130
+ }
131
130
- switch i.GetType() {
131
- case ufspb.Data_Directory:
132
- ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
133
- d.childDirs[name] = ndir
134
- return ndir, nil
135
- case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
132
+ switch i.GetType() {
133
+ case ufspb.Data_Directory:
134
+ ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
135
+ d.childDirs[name] = ndir
136
+ return ndir, nil
137
+ case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
138
+ nfi, err := NewFile(name, nd, d, d.dserv)
139
+ if err != nil {
140
+ return nil, err
141
+ }
142
+ d.files[name] = nfi
143
+ return nfi, nil
144
+ case ufspb.Data_Metadata:
145
+ return nil, ErrNotYetImplemented
146
+ default:
147
+ return nil, ErrInvalidChild
148
+ }
149
+ case *dag.RawNode:
150
nfi, err := NewFile(name, nd, d, d.dserv)
151
if err != nil {
152
return nil, err
153
}
154
d.files[name] = nfi
155
return nfi, nil
142
- case ufspb.Data_Metadata:
143
- return nil, ErrNotYetImplemented
156
default:
145
- return nil, ErrInvalidChild
157
+ return nil, fmt.Errorf("unrecognized node type in cache node")
158
}
159
}
160
@@ -162,8 +174,8 @@ func (d *Directory) Uncache(name string) {
174
175
// childFromDag searches through this directories dag node for a child link
176
// with the given name
165
-func (d *Directory) childFromDag(name string) (*dag.ProtoNode, error) {
166
- pbn, err := d.node.GetLinkedProtoNode(d.ctx, d.dserv, name)
177
+func (d *Directory) childFromDag(name string) (node.Node, error) {
178
+ pbn, err := d.node.GetLinkedNode(d.ctx, d.dserv, name)
179
switch err {
180
case nil:
181
return pbn, nil
@@ -249,7 +261,7 @@ func (d *Directory) List() ([]NodeListing, error) {
261
return nil, err
262
}
263
252
- child.Hash = nd.Key().B58String()
264
+ child.Hash = nd.Cid().String()
265
266
out = append(out, child)
267
}
@@ -385,7 +397,7 @@ func (d *Directory) Path() string {
397
return out
398
}
399
388
-func (d *Directory) GetNode() (*dag.ProtoNode, error) {
400
+func (d *Directory) GetNode() (node.Node, error) {
401
d.lock.Lock()
402
defer d.lock.Unlock()
403
mfs/file.go
+33
-20
@@ -9,6 +9,8 @@ import (
9
dag "github.com/ipfs/go-ipfs/merkledag"
10
ft "github.com/ipfs/go-ipfs/unixfs"
11
mod "github.com/ipfs/go-ipfs/unixfs/mod"
12
+
13
+ node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
14
)
15
16
type File struct {
@@ -19,12 +21,12 @@ type File struct {
21
desclock sync.RWMutex
22
23
dserv dag.DAGService
22
- node *dag.ProtoNode
24
+ node node.Node
25
nodelk sync.Mutex
26
}
27
28
// NewFile returns a NewFile object with the given parameters
27
-func NewFile(name string, node *dag.ProtoNode, parent childCloser, dserv dag.DAGService) (*File, error) {
29
+func NewFile(name string, node node.Node, parent childCloser, dserv dag.DAGService) (*File, error) {
30
return &File{
31
dserv: dserv,
32
parent: parent,
@@ -44,18 +46,23 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
46
node := fi.node
47
fi.nodelk.Unlock()
48
47
- fsn, err := ft.FSNodeFromBytes(node.Data())
48
- if err != nil {
49
- return nil, err
50
- }
51
-
52
- switch fsn.Type {
53
- default:
54
- return nil, fmt.Errorf("unsupported fsnode type for 'file'")
55
- case ft.TSymlink:
56
- return nil, fmt.Errorf("symlinks not yet supported")
57
- case ft.TFile, ft.TRaw:
58
- // OK case
49
+ switch node := node.(type) {
50
+ case *dag.ProtoNode:
51
+ fsn, err := ft.FSNodeFromBytes(node.Data())
52
+ if err != nil {
53
+ return nil, err
54
+ }
55
+
56
+ switch fsn.Type {
57
+ default:
58
+ return nil, fmt.Errorf("unsupported fsnode type for 'file'")
59
+ case ft.TSymlink:
60
+ return nil, fmt.Errorf("symlinks not yet supported")
61
+ case ft.TFile, ft.TRaw:
62
+ // OK case
63
+ }
64
+ case *dag.RawNode:
65
+ // Ok as well.
66
}
67
68
switch flags {
@@ -85,16 +92,22 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
92
func (fi *File) Size() (int64, error) {
93
fi.nodelk.Lock()
94
defer fi.nodelk.Unlock()
88
- pbd, err := ft.FromBytes(fi.node.Data())
89
- if err != nil {
90
- return 0, err
95
+ switch nd := fi.node.(type) {
96
+ case *dag.ProtoNode:
97
+ pbd, err := ft.FromBytes(nd.Data())
98
+ if err != nil {
99
+ return 0, err
100
+ }
101
+ return int64(pbd.GetFilesize()), nil
102
+ case *dag.RawNode:
103
+ return int64(len(nd.RawData())), nil
104
+ default:
105
+ return 0, fmt.Errorf("unrecognized node type in mfs/file.Size()")
106
}
92
-
93
- return int64(pbd.GetFilesize()), nil
107
}
108
109
// GetNode returns the dag node associated with this file
97
-func (fi *File) GetNode() (*dag.ProtoNode, error) {
110
+func (fi *File) GetNode() (node.Node, error) {
111
fi.nodelk.Lock()
112
defer fi.nodelk.Unlock()
113
return fi.node, nil
mfs/mfs_test.go
+9
-4
@@ -794,7 +794,12 @@ func TestFlushing(t *testing.T) {
794
t.Fatal(err)
795
}
796
797
- fsnode, err := ft.FSNodeFromBytes(rnd.Data())
797
+ pbrnd, ok := rnd.(*dag.ProtoNode)
798
+ if !ok {
799
+ t.Fatal(dag.ErrNotProtobuf)
800
+ }
801
+
802
+ fsnode, err := ft.FSNodeFromBytes(pbrnd.Data())
803
if err != nil {
804
t.Fatal(err)
805
}
@@ -803,10 +808,10 @@ func TestFlushing(t *testing.T) {
808
t.Fatal("root wasnt a directory")
809
}
810
806
- rnk := rnd.Key()
811
+ rnk := rnd.Cid()
812
exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr"
808
- if rnk.B58String() != exp {
809
- t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.B58String())
813
+ if rnk.String() != exp {
814
+ t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.String())
815
}
816
}
817
mfs/system.go
+3
-2
@@ -10,6 +10,7 @@
10
package mfs
11
12
import (
13
+ "context"
14
"errors"
15
"sync"
16
"time"
@@ -17,9 +18,9 @@ import (
18
dag "github.com/ipfs/go-ipfs/merkledag"
19
ft "github.com/ipfs/go-ipfs/unixfs"
20
20
- context "context"
21
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
22
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
23
+ node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
24
)
25
26
var ErrNotExist = errors.New("no such rootfs")
@@ -41,7 +42,7 @@ const (
42
43
// FSNode represents any node (directory, root, or file) in the mfs filesystem
44
type FSNode interface {
44
- GetNode() (*dag.ProtoNode, error)
45
+ GetNode() (node.Node, error)
46
Flush() error
47
Type() NodeType
48
}
test/sharness/t0040-add-and-cat.sh
+20
-1
@@ -126,6 +126,21 @@ test_add_cat_5MB() {
126
'
127
}
128
129
+test_add_cat_raw() {
130
+ test_expect_success "add a small file with raw-leaves" '
131
+ echo "foobar" > afile &&
132
+ HASH=$(ipfs add -q --raw-leaves afile)
133
+ '
134
+
135
+ test_expect_success "cat that small file" '
136
+ ipfs cat $HASH > afile_out
137
+ '
138
+
139
+ test_expect_success "make sure it looks good" '
140
+ test_cmp afile afile_out
141
+ '
142
+}
143
+
144
test_add_cat_expensive() {
145
test_expect_success EXPENSIVE "generate 100MB file using go-random" '
146
random 104857600 42 >mountdir/bigfile
@@ -392,17 +407,21 @@ test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=tr
407
408
test_add_pwd_is_symlink
409
410
+test_add_cat_raw
411
+
412
test_kill_ipfs_daemon
413
414
# should work offline
415
416
test_add_cat_file
417
418
+test_add_cat_raw
419
+
420
test_expect_success "ipfs add --only-hash succeeds" '
421
echo "unknown content for only-hash" | ipfs add --only-hash -q > oh_hash
422
'
423
405
-#TODO: this doesn't work when online hence separated out from test_add_cat_file
424
+#TODO: this doesnt work when online hence separated out from test_add_cat_file
425
test_expect_success "ipfs cat file fails" '
426
test_must_fail ipfs cat $(cat oh_hash)
427
'