add-only-hash no longer stores entirety of everything in memory
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Aug 14, 2015 at 15:12 UTC
30d10ce4f4bc0254f36455e2a81cb54acd965fb7
4 files changed
+40
-24
core/commands/add.go
+25
-9
@@ -6,11 +6,16 @@ import (
6
"path"
7
8
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
9
+ ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
+ syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
11
cxt "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
12
13
+ bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
14
+ bserv "github.com/ipfs/go-ipfs/blockservice"
15
cmds "github.com/ipfs/go-ipfs/commands"
16
files "github.com/ipfs/go-ipfs/commands/files"
17
core "github.com/ipfs/go-ipfs/core"
18
+ offline "github.com/ipfs/go-ipfs/exchange/offline"
19
importer "github.com/ipfs/go-ipfs/importer"
20
"github.com/ipfs/go-ipfs/importer/chunk"
21
dag "github.com/ipfs/go-ipfs/merkledag"
@@ -103,17 +108,28 @@ remains to be implemented.
108
hidden, _, _ := req.Option(hiddenOptionName).Bool()
109
chunker, _, _ := req.Option(chunkerOptionName).String()
110
111
+ e := dagutils.NewDagEditor(n.DAG, newDirNode())
112
if hash {
113
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
114
//TODO: need this to be true or all files
115
// hashed will be stored in memory!
110
- NilRepo: false,
116
+ NilRepo: true,
117
})
118
if err != nil {
119
res.SetError(err, cmds.ErrNormal)
120
return
121
}
122
n = nilnode
123
+
124
+ // build mem-datastore for editor's intermediary nodes
125
+ bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
126
+ bsrv, err := bserv.New(bs, offline.Exchange(bs))
127
+ if err != nil {
128
+ res.SetError(err, cmds.ErrNormal)
129
+ return
130
+ }
131
+ memds := dag.NewDAGService(bsrv)
132
+ e = dagutils.NewDagEditor(memds, newDirNode())
133
}
134
135
outChan := make(chan interface{}, 8)
@@ -122,7 +138,7 @@ remains to be implemented.
138
fileAdder := adder{
139
ctx: req.Context(),
140
node: n,
125
- editor: dagutils.NewDagEditor(n.DAG, newDirNode()),
141
+ editor: e,
142
out: outChan,
143
chunker: chunker,
144
progress: progress,
@@ -318,7 +334,7 @@ func (params *adder) RootNode() (*dag.Node, error) {
334
// if not wrapping, AND one root file, use that hash as root.
335
if !params.wrap && len(r.Links) == 1 {
336
var err error
321
- r, err = r.Links[0].GetNode(params.ctx, params.node.DAG)
337
+ r, err = r.Links[0].GetNode(params.ctx, params.editor.GetDagService())
338
// no need to output, as we've already done so.
339
return r, err
340
}
@@ -330,16 +346,16 @@ func (params *adder) RootNode() (*dag.Node, error) {
346
347
func (params *adder) addNode(node *dag.Node, path string) error {
348
// patch it into the root
333
- key, err := node.Key()
334
- if err != nil {
335
- return err
336
- }
337
-
349
if path == "" {
350
+ key, err := node.Key()
351
+ if err != nil {
352
+ return err
353
+ }
354
+
355
path = key.Pretty()
356
}
357
342
- if err := params.editor.InsertNodeAtPath(params.ctx, path, key, newDirNode); err != nil {
358
+ if err := params.editor.InsertNodeAtPath(params.ctx, path, node, newDirNode); err != nil {
359
return err
360
}
361
core/commands/object.go
+6
-1
@@ -623,7 +623,12 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
623
624
e := dagutils.NewDagEditor(nd.DAG, root)
625
626
- err = e.InsertNodeAtPath(req.Context(), path, childk, createfunc)
626
+ childnd, err := nd.DAG.Get(req.Context(), childk)
627
+ if err != nil {
628
+ return "", err
629
+ }
630
+
631
+ err = e.InsertNodeAtPath(req.Context(), path, childnd, createfunc)
632
if err != nil {
633
return "", err
634
}
merkledag/utils/utils.go
+7
-12
@@ -6,7 +6,6 @@ import (
6
7
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8
9
- key "github.com/ipfs/go-ipfs/blocks/key"
9
dag "github.com/ipfs/go-ipfs/merkledag"
10
)
11
@@ -26,21 +25,17 @@ func (e *Editor) GetNode() *dag.Node {
25
return e.root.Copy()
26
}
27
29
-func (e *Editor) AddLink(ctx context.Context, childname string, childk key.Key) error {
30
- nd, err := addLink(ctx, e.ds, e.root, childname, childk)
31
- if err != nil {
32
- return err
33
- }
34
- e.root = nd
35
- return nil
28
+func (e *Editor) GetDagService() dag.DAGService {
29
+ return e.ds
30
}
31
38
-func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
32
+func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childnd *dag.Node) (*dag.Node, error) {
33
if childname == "" {
34
return nil, errors.New("cannot create link with no name!")
35
}
36
43
- childnd, err := ds.Get(ctx, childk)
37
+ // ensure that the node we are adding is in the dagservice
38
+ _, err := ds.Add(childnd)
39
if err != nil {
40
return nil, err
41
}
@@ -58,7 +53,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
53
return root, nil
54
}
55
61
-func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key.Key, create func() *dag.Node) error {
56
+func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *dag.Node, create func() *dag.Node) error {
57
splpath := strings.Split(path, "/")
58
nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
59
if err != nil {
@@ -68,7 +63,7 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key
63
return nil
64
}
65
71
-func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key, create func() *dag.Node) (*dag.Node, error) {
66
+func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
67
if len(path) == 1 {
68
return addLink(ctx, ds, root, path[0], toinsert)
69
}
merkledag/utils/utils_test.go
+2
-2
@@ -23,7 +23,7 @@ func TestAddLink(t *testing.T) {
23
}
24
25
nd := new(dag.Node)
26
- nnode, err := addLink(context.Background(), ds, nd, "fish", fk)
26
+ nnode, err := addLink(context.Background(), ds, nd, "fish", fishnode)
27
if err != nil {
28
t.Fatal(err)
29
}
@@ -104,7 +104,7 @@ func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr
104
}
105
}
106
107
- err = e.InsertNodeAtPath(context.Background(), path, ck, c)
107
+ err = e.InsertNodeAtPath(context.Background(), path, child, c)
108
if experr != "" {
109
var got string
110
if err != nil {