feat(coreunix): AddR for recursive Adds
Brian Tiger Chow committed
Jan 24, 2015 at 07:52 UTC
7f6bd2807d7b844de20bcd6473df1c1a073ca68d
6 files changed
+150
core/coreunix/add.go
+109
@@ -1,11 +1,18 @@
1
package coreunix
2
3
import (
4
+ "errors"
5
"io"
6
+ "os"
7
+ "path"
8
9
+ "github.com/jbenet/go-ipfs/commands/files"
10
core "github.com/jbenet/go-ipfs/core"
11
importer "github.com/jbenet/go-ipfs/importer"
12
chunk "github.com/jbenet/go-ipfs/importer/chunk"
13
+ merkledag "github.com/jbenet/go-ipfs/merkledag"
14
+ "github.com/jbenet/go-ipfs/pin"
15
+ unixfs "github.com/jbenet/go-ipfs/unixfs"
16
u "github.com/jbenet/go-ipfs/util"
17
)
18
@@ -22,5 +29,107 @@ func Add(n *core.IpfsNode, r io.Reader) (u.Key, error) {
29
if err != nil {
30
return "", err
31
}
32
+ // TODO(btc): is it necessary to flush the pinner?
33
return dagNode.Key()
34
}
35
+
36
+// AddR recursively adds files in |path|.
37
+func AddR(n *core.IpfsNode, root string) (key string, err error) {
38
+ f, err := os.Open(root)
39
+ if err != nil {
40
+ return "", err
41
+ }
42
+ defer f.Close()
43
+ ff, err := files.NewSerialFile(root, f)
44
+ if err != nil {
45
+ return "", err
46
+ }
47
+ dagnode, err := addFile(n, ff)
48
+ if err != nil {
49
+ return "", err
50
+ }
51
+ k, err := dagnode.Key()
52
+ if err != nil {
53
+ return "", err
54
+ }
55
+ return k.String(), nil
56
+}
57
+
58
+func add(n *core.IpfsNode, readers []io.Reader) ([]*merkledag.Node, error) {
59
+ mp, ok := n.Pinning.(pin.ManualPinner)
60
+ if !ok {
61
+ return nil, errors.New("invalid pinner type! expected manual pinner")
62
+ }
63
+ dagnodes := make([]*merkledag.Node, 0)
64
+ for _, reader := range readers {
65
+ node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
66
+ if err != nil {
67
+ return nil, err
68
+ }
69
+ dagnodes = append(dagnodes, node)
70
+ }
71
+ err := n.Pinning.Flush()
72
+ if err != nil {
73
+ return nil, err
74
+ }
75
+ return dagnodes, nil
76
+}
77
+
78
+func addNode(n *core.IpfsNode, node *merkledag.Node) error {
79
+ err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
80
+ if err != nil {
81
+ return err
82
+ }
83
+ err = n.Pinning.Pin(node, true) // ensure we keep it
84
+ if err != nil {
85
+ return err
86
+ }
87
+ return nil
88
+}
89
+
90
+func addFile(n *core.IpfsNode, file files.File) (*merkledag.Node, error) {
91
+ if file.IsDirectory() {
92
+ return addDir(n, file)
93
+ }
94
+
95
+ dns, err := add(n, []io.Reader{file})
96
+ if err != nil {
97
+ return nil, err
98
+ }
99
+
100
+ return dns[len(dns)-1], nil // last dag node is the file.
101
+}
102
+
103
+func addDir(n *core.IpfsNode, dir files.File) (*merkledag.Node, error) {
104
+
105
+ tree := &merkledag.Node{Data: unixfs.FolderPBData()}
106
+
107
+Loop:
108
+ for {
109
+ file, err := dir.NextFile()
110
+ switch {
111
+ case err != nil && err != io.EOF:
112
+ return nil, err
113
+ case err == io.EOF:
114
+ break Loop
115
+ }
116
+
117
+ node, err := addFile(n, file)
118
+ if err != nil {
119
+ return nil, err
120
+ }
121
+
122
+ _, name := path.Split(file.FileName())
123
+
124
+ err = tree.AddNodeLink(name, node)
125
+ if err != nil {
126
+ return nil, err
127
+ }
128
+ }
129
+
130
+ err := addNode(n, tree)
131
+ if err != nil {
132
+ return nil, err
133
+ }
134
+ return tree, nil
135
+}
core/coreunix/add_test.go
new
+37
@@ -0,0 +1,37 @@
1
+package coreunix
2
+
3
+import (
4
+ "os"
5
+ "path"
6
+ "testing"
7
+
8
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
9
+ "github.com/jbenet/go-ipfs/core"
10
+ "github.com/jbenet/go-ipfs/repo"
11
+ "github.com/jbenet/go-ipfs/repo/config"
12
+ "github.com/jbenet/go-ipfs/util/testutil"
13
+)
14
+
15
+func TestAddRecursive(t *testing.T) {
16
+ here, err := os.Getwd()
17
+ if err != nil {
18
+ t.Fatal(err)
19
+ }
20
+ r := &repo.Mock{
21
+ C: config.Config{
22
+ Identity: config.Identity{
23
+ PeerID: "Qmfoo", // required by offline node
24
+ },
25
+ },
26
+ D: testutil.ThreadSafeCloserMapDatastore(),
27
+ }
28
+ node, err := core.NewIPFSNode(context.Background(), core.Offline(r))
29
+ if err != nil {
30
+ t.Fatal(err)
31
+ }
32
+ if k, err := AddR(node, path.Join(here, "test_data")); err != nil {
33
+ t.Fatal(err)
34
+ } else if k != "QmWCCga8AbTyfAQ7pTnGT6JgmRMAB3Qp8ZmTEFi5q5o8jC" {
35
+ t.Fatal("keys do not match")
36
+ }
37
+}
core/coreunix/test_data/colors/orange
new
+1
@@ -0,0 +1 @@
1
+orange
core/coreunix/test_data/corps/apple
new
+1
@@ -0,0 +1 @@
1
+apple
core/coreunix/test_data/fruits/apple
new
+1
@@ -0,0 +1 @@
1
+apple
core/coreunix/test_data/fruits/orange
new
+1
@@ -0,0 +1 @@
1
+orange