flushing and shallow list names
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Jan 11, 2016 at 03:20 UTC
1feea8bc5fd898b9d5f944233ba5c3903224eaaf
6 files changed
+201
-11
core/commands/files/files.go
+45
-10
@@ -41,6 +41,7 @@ Files is an API for manipulating ipfs objects as if they were a unix filesystem.
41
"mkdir": FilesMkdirCmd,
42
"stat": FilesStatCmd,
43
"rm": FilesRmCmd,
44
+ "flush": FilesFlushCmd,
45
},
46
}
47
@@ -257,17 +258,10 @@ Examples:
258
switch fsn := fsn.(type) {
259
case *mfs.Directory:
260
if !long {
260
- mdnd, err := fsn.GetNode()
261
- if err != nil {
262
- res.SetError(err, cmds.ErrNormal)
263
- return
264
- }
265
-
261
var output []mfs.NodeListing
267
- for _, lnk := range mdnd.Links {
262
+ for _, name := range fsn.ListNames() {
263
output = append(output, mfs.NodeListing{
269
- Name: lnk.Name,
270
- Hash: lnk.Hash.B58String(),
264
+ Name: name,
265
})
266
}
267
res.SetOutput(&FilesLsOutput{output})
@@ -514,7 +508,14 @@ Warning:
508
}
509
510
if flush {
517
- defer fi.Close()
511
+ defer func() {
512
+ fi.Close()
513
+ err := mfs.FlushPath(nd.FilesRoot, path)
514
+ if err != nil {
515
+ res.SetError(err, cmds.ErrNormal)
516
+ return
517
+ }
518
+ }()
519
} else {
520
defer fi.Sync()
521
}
@@ -613,6 +614,40 @@ Examples:
614
},
615
}
616
617
+var FilesFlushCmd = &cmds.Command{
618
+ Helptext: cmds.HelpText{
619
+ Tagline: "flush a given path's data to disk",
620
+ ShortDescription: `
621
+flush a given path to disk. This is only useful when other commands
622
+are run with the '--flush=false'.
623
+`,
624
+ },
625
+ Arguments: []cmds.Argument{
626
+ cmds.StringArg("path", false, false, "path to flush (default '/')"),
627
+ },
628
+ Run: func(req cmds.Request, res cmds.Response) {
629
+ nd, err := req.InvocContext().GetNode()
630
+ if err != nil {
631
+ res.SetError(err, cmds.ErrNormal)
632
+ return
633
+ }
634
+
635
+ // take the lock and defer the unlock
636
+ defer nd.Blockstore.PinLock()()
637
+
638
+ path := "/"
639
+ if len(req.Arguments()) > 0 {
640
+ path = req.Arguments()[0]
641
+ }
642
+
643
+ err = mfs.FlushPath(nd.FilesRoot, path)
644
+ if err != nil {
645
+ res.SetError(err, cmds.ErrNormal)
646
+ return
647
+ }
648
+ },
649
+}
650
+
651
var FilesRmCmd = &cmds.Command{
652
Helptext: cmds.HelpText{
653
Tagline: "Remove a file.",
core/core.go
+14
@@ -474,6 +474,20 @@ func (n *IpfsNode) loadBootstrapPeers() ([]peer.PeerInfo, error) {
474
func (n *IpfsNode) loadFilesRoot() error {
475
dsk := ds.NewKey("/local/filesroot")
476
pf := func(ctx context.Context, k key.Key) error {
477
+ ds := n.Repo.Datastore()
478
+ if old, err := ds.Get(dsk); err == nil {
479
+ _ = n.Pinning.Unpin(n.Context(), key.Key(old.([]byte)), true)
480
+ }
481
+ nnd, err := n.DAG.Get(n.Context(), k)
482
+ if err != nil {
483
+ return err
484
+ }
485
+
486
+ err = n.Pinning.Pin(n.Context(), nnd, true)
487
+ if err != nil {
488
+ return err
489
+ }
490
+
491
return n.Repo.Datastore().Put(dsk, []byte(k))
492
}
493
mfs/dir.go
+24
@@ -175,6 +175,30 @@ type NodeListing struct {
175
Hash string
176
}
177
178
+func (d *Directory) ListNames() []string {
179
+ d.Lock()
180
+ defer d.Unlock()
181
+
182
+ names := make(map[string]struct{})
183
+ for n, _ := range d.childDirs {
184
+ names[n] = struct{}{}
185
+ }
186
+ for n, _ := range d.files {
187
+ names[n] = struct{}{}
188
+ }
189
+
190
+ for _, l := range d.node.Links {
191
+ names[l.Name] = struct{}{}
192
+ }
193
+
194
+ var out []string
195
+ for n, _ := range names {
196
+ out = append(out, n)
197
+ }
198
+
199
+ return out
200
+}
201
+
202
func (d *Directory) List() ([]NodeListing, error) {
203
d.lock.Lock()
204
defer d.lock.Unlock()
mfs/mfs_test.go
+50
@@ -675,3 +675,53 @@ func TestMfsStress(t *testing.T) {
675
}
676
}
677
}
678
+
679
+func TestFlushing(t *testing.T) {
680
+ ctx, cancel := context.WithCancel(context.Background())
681
+ defer cancel()
682
+ _, rt := setupRoot(ctx, t)
683
+
684
+ dir := rt.GetValue().(*Directory)
685
+ c := mkdirP(t, dir, "a/b/c")
686
+ d := mkdirP(t, dir, "a/b/d")
687
+ e := mkdirP(t, dir, "a/b/e")
688
+
689
+ data := []byte("this is a test\n")
690
+ nd1 := &dag.Node{Data: ft.FilePBData(data, uint64(len(data)))}
691
+
692
+ if err := c.AddChild("TEST", nd1); err != nil {
693
+ t.Fatal(err)
694
+ }
695
+ if err := d.AddChild("TEST", nd1); err != nil {
696
+ t.Fatal(err)
697
+ }
698
+ if err := e.AddChild("TEST", nd1); err != nil {
699
+ t.Fatal(err)
700
+ }
701
+
702
+ if err := FlushPath(rt, "/a/b/c/TEST"); err != nil {
703
+ t.Fatal(err)
704
+ }
705
+
706
+ if err := FlushPath(rt, "/a/b/d/TEST"); err != nil {
707
+ t.Fatal(err)
708
+ }
709
+
710
+ if err := FlushPath(rt, "/a/b/e/TEST"); err != nil {
711
+ t.Fatal(err)
712
+ }
713
+
714
+ rnd, err := dir.GetNode()
715
+ if err != nil {
716
+ t.Fatal(err)
717
+ }
718
+
719
+ rnk, err := rnd.Key()
720
+ if err != nil {
721
+ t.Fatal(err)
722
+ }
723
+
724
+ if rnk.B58String() != "QmWcvrHUFk7LQRrA4WqKjqy7ZyRGFLVagtgNxbEodTEzQ4" {
725
+ t.Fatal("dag looks wrong")
726
+ }
727
+}
mfs/ops.go
+61
@@ -194,3 +194,64 @@ func DirLookup(d *Directory, pth string) (FSNode, error) {
194
}
195
return cur, nil
196
}
197
+
198
+func FlushPath(r *Root, pth string) error {
199
+ parts := path.SplitList(strings.Trim(pth, "/"))
200
+
201
+ d, ok := r.GetValue().(*Directory)
202
+ if !ok {
203
+ return errors.New("mfs root somehow didnt point to a directory")
204
+ }
205
+
206
+ nd, err := flushPathRec(d, parts)
207
+ if err != nil {
208
+ return err
209
+ }
210
+
211
+ k, err := nd.Key()
212
+ if err != nil {
213
+ return err
214
+ }
215
+
216
+ r.repub.Update(k)
217
+ return nil
218
+}
219
+
220
+func flushPathRec(d *Directory, parts []string) (*dag.Node, error) {
221
+ if len(parts) == 0 {
222
+ return d.GetNode()
223
+ }
224
+
225
+ d.Lock()
226
+ defer d.Unlock()
227
+
228
+ next, err := d.childUnsync(parts[0])
229
+ if err != nil {
230
+ log.Errorf("childnode: %q %q", parts[0], err)
231
+ return nil, err
232
+ }
233
+
234
+ switch next := next.(type) {
235
+ case *Directory:
236
+ nd, err := flushPathRec(next, parts[1:])
237
+ if err != nil {
238
+ return nil, err
239
+ }
240
+
241
+ newnode, err := d.node.UpdateNodeLink(parts[0], nd)
242
+ if err != nil {
243
+ return nil, err
244
+ }
245
+
246
+ d.node = newnode
247
+ return newnode, nil
248
+ case *File:
249
+ if len(parts) > 1 {
250
+ return nil, fmt.Errorf("%s is a file, not a directory", parts[0])
251
+ }
252
+
253
+ return next.GetNode()
254
+ default:
255
+ return nil, fmt.Errorf("unrecognized FSNode type: %#v", next)
256
+ }
257
+}
test/sharness/t0250-files-api.sh
+7
-1
@@ -331,10 +331,16 @@ test_files_api() {
331
'
332
333
test_expect_success "root hash looks good" '
334
- echo "QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" > root_hash_exp &&
334
+ export EXP_ROOT_HASH="QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" &&
335
+ echo $EXP_ROOT_HASH > root_hash_exp &&
336
test_cmp root_hash_exp root_hash
337
'
338
339
+ test_expect_success "root hash is pinned" '
340
+ ipfs pin ls
341
+ return 1
342
+ '
343
+
344
# test mv
345
test_expect_success "can mv dir" '
346
ipfs files mv /cats/this/is /cats/