rework add-mfs to not use caching
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Jun 2, 2016 at 12:12 UTC
22a5b795085dae7415442974af3af48a78d1f103
6 files changed
+78
-38
core/commands/add.go
+13
@@ -10,6 +10,8 @@ import (
10
cmds "github.com/ipfs/go-ipfs/commands"
11
files "github.com/ipfs/go-ipfs/commands/files"
12
core "github.com/ipfs/go-ipfs/core"
13
+ dagtest "github.com/ipfs/go-ipfs/merkledag/test"
14
+ mfs "github.com/ipfs/go-ipfs/mfs"
15
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
16
)
17
@@ -156,6 +158,17 @@ You can now refer to the added file in a gateway, like so:
158
fileAdder.Pin = dopin
159
fileAdder.Silent = silent
160
161
+ if hash {
162
+ md := dagtest.Mock()
163
+ mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil)
164
+ if err != nil {
165
+ res.SetError(err, cmds.ErrNormal)
166
+ return
167
+ }
168
+
169
+ fileAdder.SetMfsRoot(mr)
170
+ }
171
+
172
addAllAndPin := func(f files.File) error {
173
// Iterate over each top-level file and add individually. Otherwise the
174
// single files.File f is treated as a directory, affecting hidden file
core/coreunix/add.go
+26
-25
@@ -1,7 +1,6 @@
1
package coreunix
2
3
import (
4
- "bytes"
4
"fmt"
5
"io"
6
"io/ioutil"
@@ -68,7 +67,7 @@ type AddedObject struct {
67
}
68
69
func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
71
- mr, err := mfs.NewRoot(ctx, ds, newDirNode(), nil)
70
+ mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil)
71
if err != nil {
72
return nil, err
73
}
@@ -109,6 +108,10 @@ type Adder struct {
108
tempRoot key.Key
109
}
110
111
+func (adder *Adder) SetMfsRoot(r *mfs.Root) {
112
+ adder.mr = r
113
+}
114
+
115
// Perform the actual add & pin locally, outputting results to reader
116
func (adder Adder) add(reader io.Reader) (*dag.Node, error) {
117
chnk, err := chunk.FromString(reader, adder.Chunker)
@@ -214,34 +217,32 @@ func (adder *Adder) Finalize() (*dag.Node, error) {
217
return root.GetNode()
218
}
219
217
-func (adder *Adder) outputDirs(path string, fs mfs.FSNode) error {
218
- nd, err := fs.GetNode()
219
- if err != nil {
220
- return err
221
- }
222
-
223
- if !bytes.Equal(nd.Data, folderData) || fs.Type() != mfs.TDir {
220
+func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error {
221
+ switch fsn := fsn.(type) {
222
+ case *mfs.File:
223
return nil
225
- }
226
-
227
- dir, ok := fs.(*mfs.Directory)
228
- if !ok {
229
- return fmt.Errorf("received FSNode of type TDir that was not a Directory")
230
- }
231
-
232
- for _, name := range dir.ListNames() {
233
- child, err := dir.Child(name)
234
- if err != nil {
235
- return err
224
+ case *mfs.Directory:
225
+ for _, name := range fsn.ListNames() {
226
+ child, err := fsn.Child(name)
227
+ if err != nil {
228
+ return err
229
+ }
230
+
231
+ childpath := gopath.Join(path, name)
232
+ err = adder.outputDirs(childpath, child)
233
+ if err != nil {
234
+ return err
235
+ }
236
}
237
-
238
- err = adder.outputDirs(gopath.Join(path, name), child)
237
+ nd, err := fsn.GetNode()
238
if err != nil {
239
return err
240
}
242
- }
241
244
- return outputDagnode(adder.Out, path, nd)
242
+ return outputDagnode(adder.Out, path, nd)
243
+ default:
244
+ return fmt.Errorf("unrecognized fsn type: %#v", fsn)
245
+ }
246
}
247
248
// Add builds a merkledag from the a reader, pinning all objects to the local
@@ -488,7 +489,7 @@ func NewMemoryDagService() dag.DAGService {
489
}
490
491
// TODO: generalize this to more than unix-fs nodes.
491
-func newDirNode() *dag.Node {
492
+func NewDirNode() *dag.Node {
493
return &dag.Node{Data: unixfs.FolderPBData()}
494
}
495
mfs/dir.go
+1
-13
@@ -131,7 +131,7 @@ func (d *Directory) cacheNode(name string, nd *dag.Node) (FSNode, error) {
131
ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
132
d.childDirs[name] = ndir
133
return ndir, nil
134
- case ufspb.Data_File, ufspb.Data_Raw:
134
+ case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
135
nfi, err := NewFile(name, nd, d, d.dserv)
136
if err != nil {
137
return nil, err
@@ -338,18 +338,6 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error {
338
}
339
340
d.modTime = time.Now()
341
-
342
- if len(nd.Links) == 0 {
343
- nfi, err := NewFile(name, nd, d, d.dserv)
344
- if err != nil {
345
- return err
346
- }
347
- d.files[name] = nfi
348
- } else {
349
- ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
350
- d.childDirs[name] = ndir
351
- }
352
-
341
return nil
342
}
343
mfs/file.go
+14
@@ -45,6 +45,20 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
45
node := fi.node
46
fi.nodelk.Unlock()
47
48
+ fsn, err := ft.FSNodeFromBytes(node.Data)
49
+ if err != nil {
50
+ return nil, err
51
+ }
52
+
53
+ switch fsn.Type {
54
+ default:
55
+ return nil, fmt.Errorf("unsupported fsnode type for 'file'")
56
+ case ft.TSymlink:
57
+ return nil, fmt.Errorf("symlinks not yet supported")
58
+ case ft.TFile, ft.TRaw:
59
+ // OK case
60
+ }
61
+
62
switch flags {
63
case OpenReadOnly:
64
fi.desclock.RLock()
test/sharness/t0250-files-api.sh
+23
@@ -437,6 +437,29 @@ test_files_api() {
437
test_expect_success "child dir looks right" '
438
verify_dir_contents /
439
'
440
+
441
+ # test for https://github.com/ipfs/go-ipfs/issues/2654
442
+ test_expect_success "create and remove dir" '
443
+ ipfs files mkdir /test_dir &&
444
+ ipfs files rm -r "/test_dir"
445
+ '
446
+
447
+ test_expect_success "create test file" '
448
+ echo "content" | ipfs files write -e "/test_file"
449
+ '
450
+
451
+ test_expect_success "copy test file onto test dir" '
452
+ ipfs files cp "/test_file" "/test_dir"
453
+ '
454
+
455
+ test_expect_success "test /test_dir" '
456
+ ipfs files stat "/test_dir" | grep -q "^Type: file"
457
+ '
458
+
459
+ test_expect_success "clean up /test_dir and /test_file" '
460
+ ipfs files rm -r /test_dir &&
461
+ ipfs files rm -r /test_file
462
+ '
463
}
464
465
# test offline and online
unixfs/format.go
+1
@@ -15,6 +15,7 @@ const (
15
TFile = pb.Data_File
16
TDirectory = pb.Data_Directory
17
TMetadata = pb.Data_Metadata
18
+ TSymlink = pb.Data_Symlink
19
)
20
21
var ErrMalformedFileFormat = errors.New("malformed data in file format")