rework editor creation and finalization
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Nov 13, 2015 at 10:19 UTC
efac042e82e3b6d1574ecb70b1c2db0f59fe5f08
9 files changed
+113
-71
core/commands/add.go
+9
-7
@@ -149,13 +149,15 @@ remains to be implemented.
149
return err
150
}
151
152
- if !hash {
153
- // copy intermediary nodes from editor to our actual dagservice
154
- err := fileAdder.WriteOutputTo(n.DAG)
155
- if err != nil {
156
- log.Error("WRITE OUT: ", err)
157
- return err
158
- }
152
+ if hash {
153
+ return nil
154
+ }
155
+
156
+ // copy intermediary nodes from editor to our actual dagservice
157
+ _, err := fileAdder.Finalize(n.DAG)
158
+ if err != nil {
159
+ log.Error("WRITE OUT: ", err)
160
+ return err
161
}
162
163
return fileAdder.PinRoot()
core/commands/object.go
+10
-4
@@ -599,14 +599,17 @@ func rmLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
599
600
path := req.Arguments()[2]
601
602
- e := dagutils.NewDagEditor(nd.DAG, root)
602
+ e := dagutils.NewDagEditor(root, nd.DAG)
603
604
err = e.RmLink(req.Context(), path)
605
if err != nil {
606
return "", err
607
}
608
609
- nnode := e.GetNode()
609
+ nnode, err := e.Finalize(nd.DAG)
610
+ if err != nil {
611
+ return "", err
612
+ }
613
614
return nnode.Key()
615
}
@@ -636,7 +639,7 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
639
}
640
}
641
639
- e := dagutils.NewDagEditor(nd.DAG, root)
642
+ e := dagutils.NewDagEditor(root, nd.DAG)
643
644
childnd, err := nd.DAG.Get(req.Context(), childk)
645
if err != nil {
@@ -648,7 +651,10 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
651
return "", err
652
}
653
651
- nnode := e.GetNode()
654
+ nnode, err := e.Finalize(nd.DAG)
655
+ if err != nil {
656
+ return "", err
657
+ }
658
659
return nnode.Key()
660
}
core/corehttp/gateway_handler.go
+8
-2
@@ -342,14 +342,20 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
342
return
343
}
344
345
- e := dagutils.NewDagEditor(i.node.DAG, rnode)
345
+ e := dagutils.NewDagEditor(rnode, i.node.DAG)
346
err = e.InsertNodeAtPath(ctx, newPath, newnode, uio.NewEmptyDirectory)
347
if err != nil {
348
webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError)
349
return
350
}
351
352
- newkey, err = e.GetNode().Key()
352
+ nnode, err := e.Finalize(i.node.DAG)
353
+ if err != nil {
354
+ webError(w, "putHandler: could not get node", err, http.StatusInternalServerError)
355
+ return
356
+ }
357
+
358
+ newkey, err = nnode.Key()
359
if err != nil {
360
webError(w, "putHandler: could not get key of edited node", err, http.StatusInternalServerError)
361
return
core/coreunix/add.go
+18
-18
@@ -20,7 +20,7 @@ import (
20
21
"github.com/ipfs/go-ipfs/commands/files"
22
core "github.com/ipfs/go-ipfs/core"
23
- merkledag "github.com/ipfs/go-ipfs/merkledag"
23
+ dag "github.com/ipfs/go-ipfs/merkledag"
24
unixfs "github.com/ipfs/go-ipfs/unixfs"
25
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
26
)
@@ -63,7 +63,7 @@ type AddedObject struct {
63
}
64
65
func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) *Adder {
66
- e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
66
+ e := dagutils.NewDagEditor(newDirNode(), nil)
67
return &Adder{
68
ctx: ctx,
69
node: n,
@@ -90,11 +90,11 @@ type Adder struct {
90
Trickle bool
91
Wrap bool
92
Chunker string
93
- root *merkledag.Node
93
+ root *dag.Node
94
}
95
96
// Perform the actual add & pin locally, outputting results to reader
97
-func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
97
+func (params Adder) add(reader io.Reader) (*dag.Node, error) {
98
chnk, err := chunk.FromString(reader, params.Chunker)
99
if err != nil {
100
return nil, err
@@ -112,7 +112,7 @@ func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
112
)
113
}
114
115
-func (params *Adder) RootNode() (*merkledag.Node, error) {
115
+func (params *Adder) RootNode() (*dag.Node, error) {
116
// for memoizing
117
if params.root != nil {
118
return params.root, nil
@@ -153,8 +153,8 @@ func (params *Adder) PinRoot() error {
153
return params.node.Pinning.Flush()
154
}
155
156
-func (params *Adder) WriteOutputTo(DAG merkledag.DAGService) error {
157
- return params.editor.WriteOutputTo(DAG)
156
+func (params *Adder) Finalize(DAG dag.DAGService) (*dag.Node, error) {
157
+ return params.editor.Finalize(DAG)
158
}
159
160
// Add builds a merkledag from the a reader, pinning all objects to the local
@@ -212,7 +212,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
212
// to preserve the filename.
213
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
214
// the directory, and and error if any.
215
-func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkledag.Node, error) {
215
+func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) {
216
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
217
dir := files.NewSliceFile("", "", []files.File{file})
218
fileAdder := NewAdder(n.Context(), n, nil)
@@ -230,7 +230,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkle
230
return gopath.Join(k.String(), filename), dagnode, nil
231
}
232
233
-func (params *Adder) addNode(node *merkledag.Node, path string) error {
233
+func (params *Adder) addNode(node *dag.Node, path string) error {
234
// patch it into the root
235
if path == "" {
236
key, err := node.Key()
@@ -249,7 +249,7 @@ func (params *Adder) addNode(node *merkledag.Node, path string) error {
249
}
250
251
// Add the given file while respecting the params.
252
-func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
252
+func (params *Adder) AddFile(file files.File) (*dag.Node, error) {
253
switch {
254
case files.IsHidden(file) && !params.Hidden:
255
log.Debugf("%s is hidden, skipping", file.FileName())
@@ -265,7 +265,7 @@ func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
265
return nil, err
266
}
267
268
- dagnode := &merkledag.Node{Data: sdata}
268
+ dagnode := &dag.Node{Data: sdata}
269
_, err = params.node.DAG.Add(dagnode)
270
if err != nil {
271
return nil, err
@@ -294,7 +294,7 @@ func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
294
return dagnode, err
295
}
296
297
-func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
297
+func (params *Adder) addDir(dir files.File) (*dag.Node, error) {
298
tree := newDirNode()
299
log.Infof("adding directory: %s", dir.FileName())
300
@@ -334,7 +334,7 @@ func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
334
}
335
336
// outputDagnode sends dagnode info over the output channel
337
-func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error {
337
+func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
338
if out == nil {
339
return nil
340
}
@@ -352,20 +352,20 @@ func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error
352
return nil
353
}
354
355
-func NewMemoryDagService() merkledag.DAGService {
355
+func NewMemoryDagService() dag.DAGService {
356
// build mem-datastore for editor's intermediary nodes
357
bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
358
bsrv := bserv.New(bs, offline.Exchange(bs))
359
- return merkledag.NewDAGService(bsrv)
359
+ return dag.NewDAGService(bsrv)
360
}
361
362
// TODO: generalize this to more than unix-fs nodes.
363
-func newDirNode() *merkledag.Node {
364
- return &merkledag.Node{Data: unixfs.FolderPBData()}
363
+func newDirNode() *dag.Node {
364
+ return &dag.Node{Data: unixfs.FolderPBData()}
365
}
366
367
// from core/commands/object.go
368
-func getOutput(dagnode *merkledag.Node) (*Object, error) {
368
+func getOutput(dagnode *dag.Node) (*Object, error) {
369
key, err := dagnode.Key()
370
if err != nil {
371
return nil, err
merkledag/node.go
+3
-1
@@ -9,6 +9,8 @@ import (
9
key "github.com/ipfs/go-ipfs/blocks/key"
10
)
11
12
+var ErrLinkNotFound = fmt.Errorf("no link by that name")
13
+
14
// Node represents a node in the IPFS Merkle DAG.
15
// nodes have opaque data and a set of navigable links.
16
type Node struct {
@@ -160,7 +162,7 @@ func (n *Node) GetNodeLink(name string) (*Link, error) {
162
}, nil
163
}
164
}
163
- return nil, ErrNotFound
165
+ return nil, ErrLinkNotFound
166
}
167
168
func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*Node, error) {
merkledag/utils/diff.go
+3
-2
@@ -37,7 +37,7 @@ func (c *Change) String() string {
37
}
38
39
func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Change) (*dag.Node, error) {
40
- e := NewDagEditor(ds, nd)
40
+ e := NewDagEditor(nd, ds)
41
for _, c := range cs {
42
switch c.Type {
43
case Add:
@@ -71,7 +71,8 @@ func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Cha
71
}
72
}
73
}
74
- return e.GetNode(), nil
74
+
75
+ return e.Finalize(ds)
76
}
77
78
func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.Node) []*Change {
merkledag/utils/utils.go
+54
-22
@@ -4,20 +4,41 @@ import (
4
"errors"
5
"strings"
6
7
+ ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8
+ syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
9
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
10
11
+ bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
12
+ bserv "github.com/ipfs/go-ipfs/blockservice"
13
+ offline "github.com/ipfs/go-ipfs/exchange/offline"
14
dag "github.com/ipfs/go-ipfs/merkledag"
15
)
16
17
type Editor struct {
18
root *dag.Node
14
- ds dag.DAGService
19
+
20
+ // tmp is a temporary in memory (for now) dagstore for all of the
21
+ // intermediary nodes to be stored in
22
+ tmp dag.DAGService
23
+
24
+ // src is the dagstore with *all* of the data on it, it is used to pull
25
+ // nodes from for modification (nil is a valid value)
26
+ src dag.DAGService
27
+}
28
+
29
+func NewMemoryDagService() dag.DAGService {
30
+ // build mem-datastore for editor's intermediary nodes
31
+ bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
32
+ bsrv := bserv.New(bs, offline.Exchange(bs))
33
+ return dag.NewDAGService(bsrv)
34
}
35
17
-func NewDagEditor(ds dag.DAGService, root *dag.Node) *Editor {
36
+// root is the node to be modified, source is the dagstore to pull nodes from (optional)
37
+func NewDagEditor(root *dag.Node, source dag.DAGService) *Editor {
38
return &Editor{
39
root: root,
20
- ds: ds,
40
+ tmp: NewMemoryDagService(),
41
+ src: source,
42
}
43
}
44
@@ -26,7 +47,7 @@ func (e *Editor) GetNode() *dag.Node {
47
}
48
49
func (e *Editor) GetDagService() dag.DAGService {
29
- return e.ds
50
+ return e.tmp
51
}
52
53
func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childnd *dag.Node) (*dag.Node, error) {
@@ -57,7 +78,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
78
79
func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *dag.Node, create func() *dag.Node) error {
80
splpath := strings.Split(path, "/")
60
- nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
81
+ nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
82
if err != nil {
83
return err
84
}
@@ -65,27 +86,32 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *da
86
return nil
87
}
88
68
-func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
89
+func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
90
if len(path) == 1 {
70
- return addLink(ctx, ds, root, path[0], toinsert)
91
+ return addLink(ctx, e.tmp, root, path[0], toinsert)
92
}
93
73
- nd, err := root.GetLinkedNode(ctx, ds, path[0])
94
+ nd, err := root.GetLinkedNode(ctx, e.tmp, path[0])
95
if err != nil {
96
// if 'create' is true, we create directories on the way down as needed
76
- if err == dag.ErrNotFound && create != nil {
97
+ if err == dag.ErrLinkNotFound && create != nil {
98
nd = create()
78
- } else {
99
+ err = nil // no longer an error case
100
+ } else if err == dag.ErrNotFound {
101
+ nd, err = root.GetLinkedNode(ctx, e.src, path[0])
102
+ }
103
+
104
+ if err != nil {
105
return nil, err
106
}
107
}
108
83
- ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert, create)
109
+ ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create)
110
if err != nil {
111
return nil, err
112
}
113
88
- _ = ds.Remove(root)
114
+ _ = e.tmp.Remove(root)
115
116
_ = root.RemoveNodeLink(path[0])
117
err = root.AddNodeLinkClean(path[0], ndprime)
@@ -93,7 +119,7 @@ func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, pa
119
return nil, err
120
}
121
96
- _, err = ds.Add(root)
122
+ _, err = e.tmp.Add(root)
123
if err != nil {
124
return nil, err
125
}
@@ -103,7 +129,7 @@ func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, pa
129
130
func (e *Editor) RmLink(ctx context.Context, path string) error {
131
splpath := strings.Split(path, "/")
106
- nd, err := rmLink(ctx, e.ds, e.root, splpath)
132
+ nd, err := e.rmLink(ctx, e.root, splpath)
133
if err != nil {
134
return err
135
}
@@ -111,7 +137,7 @@ func (e *Editor) RmLink(ctx context.Context, path string) error {
137
return nil
138
}
139
114
-func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string) (*dag.Node, error) {
140
+func (e *Editor) rmLink(ctx context.Context, root *dag.Node, path []string) (*dag.Node, error) {
141
if len(path) == 1 {
142
// base case, remove node in question
143
err := root.RemoveNodeLink(path[0])
@@ -119,7 +145,7 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
145
return nil, err
146
}
147
122
- _, err = ds.Add(root)
148
+ _, err = e.tmp.Add(root)
149
if err != nil {
150
return nil, err
151
}
@@ -127,17 +153,21 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
153
return root, nil
154
}
155
130
- nd, err := root.GetLinkedNode(ctx, ds, path[0])
156
+ nd, err := root.GetLinkedNode(ctx, e.tmp, path[0])
157
+ if err == dag.ErrNotFound {
158
+ nd, err = root.GetLinkedNode(ctx, e.src, path[0])
159
+ }
160
+
161
if err != nil {
162
return nil, err
163
}
164
135
- nnode, err := rmLink(ctx, ds, nd, path[1:])
165
+ nnode, err := e.rmLink(ctx, nd, path[1:])
166
if err != nil {
167
return nil, err
168
}
169
140
- _ = ds.Remove(root)
170
+ _ = e.tmp.Remove(root)
171
172
_ = root.RemoveNodeLink(path[0])
173
err = root.AddNodeLinkClean(path[0], nnode)
@@ -145,7 +175,7 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
175
return nil, err
176
}
177
148
- _, err = ds.Add(root)
178
+ _, err = e.tmp.Add(root)
179
if err != nil {
180
return nil, err
181
}
@@ -153,8 +183,10 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
183
return root, nil
184
}
185
156
-func (e *Editor) WriteOutputTo(ds dag.DAGService) error {
157
- return copyDag(e.GetNode(), e.ds, ds)
186
+func (e *Editor) Finalize(ds dag.DAGService) (*dag.Node, error) {
187
+ nd := e.GetNode()
188
+ err := copyDag(nd, e.tmp, ds)
189
+ return nd, err
190
}
191
192
func copyDag(nd *dag.Node, from, to dag.DAGService) error {
merkledag/utils/utils_test.go
+5
-6
@@ -66,13 +66,12 @@ func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, path stri
66
}
67
68
func TestInsertNode(t *testing.T) {
69
- ds := mdtest.Mock()
69
root := new(dag.Node)
71
- e := NewDagEditor(ds, root)
70
+ e := NewDagEditor(root, nil)
71
72
testInsert(t, e, "a", "anodefortesting", false, "")
73
testInsert(t, e, "a/b", "data", false, "")
75
- testInsert(t, e, "a/b/c/d/e", "blah", false, "merkledag: not found")
74
+ testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name")
75
testInsert(t, e, "a/b/c/d/e", "foo", true, "")
76
testInsert(t, e, "a/b/c/d/f", "baz", true, "")
77
testInsert(t, e, "a/b/c/d/f", "bar", true, "")
@@ -92,7 +91,7 @@ func TestInsertNode(t *testing.T) {
91
92
func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
93
child := &dag.Node{Data: []byte(data)}
95
- ck, err := e.ds.Add(child)
94
+ ck, err := e.tmp.Add(child)
95
if err != nil {
96
t.Fatal(err)
97
}
@@ -117,8 +116,8 @@ func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr
116
}
117
118
if err != nil {
120
- t.Fatal(err)
119
+ t.Fatal(err, path, data, create, experr)
120
}
121
123
- assertNodeAtPath(t, e.ds, e.root, path, ck)
122
+ assertNodeAtPath(t, e.tmp, e.root, path, ck)
123
}
tar/format.go
+3
-9
@@ -46,7 +46,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
46
root := new(dag.Node)
47
root.Data = []byte("ipfs/tar")
48
49
- e := dagutil.NewDagEditor(ds, root)
49
+ e := dagutil.NewDagEditor(root, ds)
50
51
for {
52
h, err := tr.Next()
@@ -91,13 +91,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
91
}
92
}
93
94
- root = e.GetNode()
95
- _, err = ds.Add(root)
96
- if err != nil {
97
- return nil, err
98
- }
99
-
100
- return root, nil
94
+ return e.Finalize(ds)
95
}
96
97
// adds a '-' to the beginning of each path element so we can use 'data' as a
@@ -178,7 +172,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
172
tr.hdrBuf = bytes.NewReader(headerNd.Data)
173
174
dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
181
- if err != nil && err != dag.ErrNotFound {
175
+ if err != nil && err != dag.ErrLinkNotFound {
176
return 0, err
177
}
178