add option to disable flushing files structure on writes
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Dec 2, 2015 at 00:22 UTC
dc2e343a992e7c186a8b1c5800435e1515783dd5
3 files changed
+130
-35
core/commands/files/files.go
+25
-4
@@ -68,7 +68,7 @@ var FilesStatCmd = &cmds.Command{
68
return
69
}
70
71
- o, err := statNode(fsn)
71
+ o, err := statNode(node.DAG, fsn)
72
if err != nil {
73
res.SetError(err, cmds.ErrNormal)
74
return
@@ -90,13 +90,14 @@ var FilesStatCmd = &cmds.Command{
90
Type: Object{},
91
}
92
93
-func statNode(fsn mfs.FSNode) (*Object, error) {
93
+func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
94
nd, err := fsn.GetNode()
95
if err != nil {
96
return nil, err
97
}
98
99
- k, err := nd.Key()
99
+ // add to dagserv to ensure its available
100
+ k, err := ds.Add(nd)
101
if err != nil {
102
return nil, err
103
}
@@ -434,10 +435,20 @@ a beginning offset to write to. The entire length of the input will be written.
435
If the '--create' option is specified, the file will be created if it does not
436
exist. Nonexistant intermediate directories will not be created.
437
438
+If the '--flush' option is set to false, changes will not be propogated to the
439
+merkledag root. This can make operations much faster when doing a large number
440
+of writes to a deeper directory structure.
441
+
442
Example:
443
444
echo "hello world" | ipfs files write --create /myfs/a/b/file
445
echo "hello world" | ipfs files write --truncate /myfs/a/b/file
446
+
447
+Warning:
448
+
449
+ Usage of the '--flush=false' option does not guarantee data durability until
450
+ the tree has been flushed. This can be accomplished by running 'ipfs files stat'
451
+ on the file or any of its ancestors.
452
`,
453
},
454
Arguments: []cmds.Argument{
@@ -449,6 +460,7 @@ Example:
460
cmds.BoolOption("e", "create", "create the file if it does not exist"),
461
cmds.BoolOption("t", "truncate", "truncate the file before writing"),
462
cmds.IntOption("n", "count", "maximum number of bytes to read"),
463
+ cmds.BoolOption("f", "flush", "flush file and ancestors after write (default: true)"),
464
},
465
Run: func(req cmds.Request, res cmds.Response) {
466
path, err := checkPath(req.Arguments()[0])
@@ -459,6 +471,10 @@ Example:
471
472
create, _, _ := req.Option("create").Bool()
473
trunc, _, _ := req.Option("truncate").Bool()
474
+ flush, set, _ := req.Option("flush").Bool()
475
+ if !set {
476
+ flush = true
477
+ }
478
479
nd, err := req.InvocContext().GetNode()
480
if err != nil {
@@ -471,7 +487,12 @@ Example:
487
res.SetError(err, cmds.ErrNormal)
488
return
489
}
474
- defer fi.Close()
490
+
491
+ if flush {
492
+ defer fi.Close()
493
+ } else {
494
+ defer fi.Sync()
495
+ }
496
497
if trunc {
498
if err := fi.Truncate(0); err != nil {
mfs/dir.go
+91
-30
@@ -53,7 +53,16 @@ func (d *Directory) closeChild(name string, nd *dag.Node) error {
53
54
d.lock.Lock()
55
defer d.lock.Unlock()
56
- err = d.node.RemoveNodeLink(name)
56
+ err = d.updateChild(name, nd)
57
+ if err != nil {
58
+ return err
59
+ }
60
+
61
+ return d.parent.closeChild(d.name, d.node)
62
+}
63
+
64
+func (d *Directory) updateChild(name string, nd *dag.Node) error {
65
+ err := d.node.RemoveNodeLink(name)
66
if err != nil && err != dag.ErrNotFound {
67
return err
68
}
@@ -63,7 +72,7 @@ func (d *Directory) closeChild(name string, nd *dag.Node) error {
72
return err
73
}
74
66
- return d.parent.closeChild(d.name, d.node)
75
+ return nil
76
}
77
78
func (d *Directory) Type() NodeType {
@@ -77,30 +86,16 @@ func (d *Directory) childFile(name string) (*File, error) {
86
return fi, nil
87
}
88
80
- nd, err := d.childFromDag(name)
81
- if err != nil {
82
- return nil, err
83
- }
84
- i, err := ft.FromBytes(nd.Data)
89
+ fsn, err := d.childNode(name)
90
if err != nil {
91
return nil, err
92
}
93
89
- switch i.GetType() {
90
- case ufspb.Data_Directory:
91
- return nil, ErrIsDirectory
92
- case ufspb.Data_File:
93
- nfi, err := NewFile(name, nd, d, d.dserv)
94
- if err != nil {
95
- return nil, err
96
- }
97
- d.files[name] = nfi
98
- return nfi, nil
99
- case ufspb.Data_Metadata:
100
- return nil, ErrNotYetImplemented
101
- default:
102
- return nil, ErrInvalidChild
94
+ if fi, ok := fsn.(*File); ok {
95
+ return fi, nil
96
}
97
+
98
+ return nil, fmt.Errorf("%s is not a file", name)
99
}
100
101
// childDir returns a directory under this directory by the given name if it
@@ -111,6 +106,21 @@ func (d *Directory) childDir(name string) (*Directory, error) {
106
return dir, nil
107
}
108
109
+ fsn, err := d.childNode(name)
110
+ if err != nil {
111
+ return nil, err
112
+ }
113
+
114
+ if dir, ok := fsn.(*Directory); ok {
115
+ return dir, nil
116
+ }
117
+
118
+ return nil, fmt.Errorf("%s is not a directory", name)
119
+}
120
+
121
+// childNode returns a FSNode under this directory by the given name if it exists.
122
+// it does *not* check the cached dirs and files
123
+func (d *Directory) childNode(name string) (FSNode, error) {
124
nd, err := d.childFromDag(name)
125
if err != nil {
126
return nil, err
@@ -127,7 +137,12 @@ func (d *Directory) childDir(name string) (*Directory, error) {
137
d.childDirs[name] = ndir
138
return ndir, nil
139
case ufspb.Data_File:
130
- return nil, fmt.Errorf("%s is not a directory", name)
140
+ nfi, err := NewFile(name, nd, d, d.dserv)
141
+ if err != nil {
142
+ return nil, err
143
+ }
144
+ d.files[name] = nfi
145
+ return nfi, nil
146
case ufspb.Data_Metadata:
147
return nil, ErrNotYetImplemented
148
default:
@@ -157,17 +172,17 @@ func (d *Directory) Child(name string) (FSNode, error) {
172
// childUnsync returns the child under this directory by the given name
173
// without locking, useful for operations which already hold a lock
174
func (d *Directory) childUnsync(name string) (FSNode, error) {
160
-
161
- dir, err := d.childDir(name)
162
- if err == nil {
163
- return dir, nil
175
+ cdir, ok := d.childDirs[name]
176
+ if ok {
177
+ return cdir, nil
178
}
165
- fi, err := d.childFile(name)
166
- if err == nil {
167
- return fi, nil
179
+
180
+ cfile, ok := d.files[name]
181
+ if ok {
182
+ return cfile, nil
183
}
184
170
- return nil, os.ErrNotExist
185
+ return d.childNode(name)
186
}
187
188
type NodeListing struct {
@@ -305,7 +320,53 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error {
320
return d.parent.closeChild(d.name, d.node)
321
}
322
323
+func (d *Directory) sync() error {
324
+ for name, dir := range d.childDirs {
325
+ nd, err := dir.GetNode()
326
+ if err != nil {
327
+ return err
328
+ }
329
+
330
+ _, err = d.dserv.Add(nd)
331
+ if err != nil {
332
+ return err
333
+ }
334
+
335
+ err = d.updateChild(name, nd)
336
+ if err != nil {
337
+ return err
338
+ }
339
+ }
340
+
341
+ for name, file := range d.files {
342
+ nd, err := file.GetNode()
343
+ if err != nil {
344
+ return err
345
+ }
346
+
347
+ _, err = d.dserv.Add(nd)
348
+ if err != nil {
349
+ return err
350
+ }
351
+
352
+ err = d.updateChild(name, nd)
353
+ if err != nil {
354
+ return err
355
+ }
356
+ }
357
+
358
+ return nil
359
+}
360
+
361
func (d *Directory) GetNode() (*dag.Node, error) {
362
+ d.Lock()
363
+ defer d.Unlock()
364
+
365
+ err := d.sync()
366
+ if err != nil {
367
+ return nil, err
368
+ }
369
+
370
return d.node, nil
371
}
372
test/sharness/t0250-files-api.sh
+14
-1
@@ -316,13 +316,26 @@ test_files_api() {
316
verify_dir_contents /cats file1 ipfs this
317
'
318
319
+ test_expect_success "write 'no-flush' succeeds" '
320
+ echo "testing" | ipfs files write -f -e /cats/walrus
321
+ '
322
+
323
+ test_expect_success "changes bubbled up to root on inspection" '
324
+ ipfs files stat / | head -n1 > root_hash
325
+ '
326
+
327
+ test_expect_success "root hash looks good" '
328
+ echo "QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" > root_hash_exp &&
329
+ test_cmp root_hash_exp root_hash
330
+ '
331
+
332
# test mv
333
test_expect_success "can mv dir" '
334
ipfs files mv /cats/this/is /cats/
335
'
336
337
test_expect_success "mv worked" '
325
- verify_dir_contents /cats file1 ipfs this is &&
338
+ verify_dir_contents /cats file1 ipfs this is walrus &&
339
verify_dir_contents /cats/this
340
'
341