@cryptotaxi247 / kubo / commits / bf23516dc

cache encoded data when reading dag nodes from disk

License: MIT Signed-off-by: Jeromy <why@ipfs.io>

Jeromy committed Jul 10, 2016 at 00:40 UTC bf23516dc0dea968ebe9458b7a1ad3ccc39258d1
36 files changed +144 -122
core/commands/add.go
+2 -1
@@ -12,6 +12,7 @@ import (
12 core "github.com/ipfs/go-ipfs/core"
13 dagtest "github.com/ipfs/go-ipfs/merkledag/test"
14 mfs "github.com/ipfs/go-ipfs/mfs"
15 + ft "github.com/ipfs/go-ipfs/unixfs"
16 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
17 )
18
@@ -160,7 +161,7 @@ You can now refer to the added file in a gateway, like so:
161
162 if hash {
163 md := dagtest.Mock()
163 - mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil)
164 + mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
165 if err != nil {
166 res.SetError(err, cmds.ErrNormal)
167 return
core/commands/files/files.go
+2 -2
@@ -163,7 +163,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
163 return nil, err
164 }
165
166 - d, err := ft.FromBytes(nd.Data)
166 + d, err := ft.FromBytes(nd.Data())
167 if err != nil {
168 return nil, err
169 }
@@ -871,7 +871,7 @@ func getFileHandle(r *mfs.Root, path string, create bool) (*mfs.File, error) {
871 return nil, fmt.Errorf("%s was not a directory", dirname)
872 }
873
874 - nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
874 + nd := dag.NodeWithData(ft.FilePBData(nil, 0))
875 err = pdir.AddChild(fname, nd)
876 if err != nil {
877 return nil, err
core/commands/ls.go
+1 -1
@@ -110,7 +110,7 @@ format:
110 }
111 }
112 if linkNode != nil {
113 - d, err := unixfs.FromBytes(linkNode.Data)
113 + d, err := unixfs.FromBytes(linkNode.Data())
114 if err != nil {
115 res.SetError(err, cmds.ErrNormal)
116 return
core/commands/object/object.go
+6 -7
@@ -93,7 +93,7 @@ is the raw data of the object.
93 res.SetError(err, cmds.ErrNormal)
94 return
95 }
96 - res.SetOutput(bytes.NewReader(node.Data))
96 + res.SetOutput(bytes.NewReader(node.Data()))
97 },
98 }
99
@@ -198,7 +198,7 @@ This command outputs data in the following encodings:
198
199 node := &Node{
200 Links: make([]Link, len(object.Links)),
201 - Data: string(object.Data),
201 + Data: string(object.Data()),
202 }
203
204 for i, link := range object.Links {
@@ -438,9 +438,7 @@ Available templates:
438 func nodeFromTemplate(template string) (*dag.Node, error) {
439 switch template {
440 case "unixfs-dir":
441 - nd := new(dag.Node)
442 - nd.Data = ft.FolderPBData()
443 - return nd, nil
441 + return ft.EmptyDirNode(), nil
442 default:
443 return nil, fmt.Errorf("template '%s' not found", template)
444 }
@@ -566,9 +564,10 @@ func deserializeNode(node *Node, dataFieldEncoding string) (*dag.Node, error) {
564 dagnode := new(dag.Node)
565 switch dataFieldEncoding {
566 case "text":
569 - dagnode.Data = []byte(node.Data)
567 + dagnode.SetData([]byte(node.Data))
568 case "base64":
571 - dagnode.Data, _ = base64.StdEncoding.DecodeString(node.Data)
569 + data, _ := base64.StdEncoding.DecodeString(node.Data)
570 + dagnode.SetData(data)
571 default:
572 return nil, fmt.Errorf("Unkown data field encoding")
573 }
core/commands/object/patch.go
+3 -5
@@ -91,7 +91,7 @@ the limit will not be respected by the network.
91 return
92 }
93
94 - rootnd.Data = append(rootnd.Data, data...)
94 + rootnd.SetData(append(rootnd.Data(), data...))
95
96 newkey, err := nd.DAG.Add(rootnd)
97 if err != nil {
@@ -153,7 +153,7 @@ Example:
153 return
154 }
155
156 - root.Data = data
156 + root.SetData(data)
157
158 newkey, err := nd.DAG.Add(root)
159 if err != nil {
@@ -287,9 +287,7 @@ to a file containing 'bar', and returns the hash of the new object.
287
288 var createfunc func() *dag.Node
289 if create {
290 - createfunc = func() *dag.Node {
291 - return &dag.Node{Data: ft.FolderPBData()}
292 - }
290 + createfunc = ft.EmptyDirNode
291 }
292
293 e := dagutils.NewDagEditor(root, nd.DAG)
core/commands/unixfs/ls.go
+2 -2
@@ -101,7 +101,7 @@ Example:
101 continue
102 }
103
104 - unixFSNode, err := unixfs.FromBytes(merkleNode.Data)
104 + unixFSNode, err := unixfs.FromBytes(merkleNode.Data())
105 if err != nil {
106 res.SetError(err, cmds.ErrNormal)
107 return
@@ -128,7 +128,7 @@ Example:
128 res.SetError(err, cmds.ErrNormal)
129 return
130 }
131 - d, err := unixfs.FromBytes(linkNode.Data)
131 + d, err := unixfs.FromBytes(linkNode.Data())
132 if err != nil {
133 res.SetError(err, cmds.ErrNormal)
134 return
core/corehttp/gateway_handler.go
+1 -1
@@ -391,7 +391,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
391
392 case nil:
393 // object set-data case
394 - rnode.Data = newnode.Data
394 + rnode.SetData(newnode.Data())
395
396 newkey, err = i.node.DAG.Add(rnode)
397 if err != nil {
core/coreunix/add.go
+2 -9
@@ -29,8 +29,6 @@ import (
29
30 var log = logging.Logger("coreunix")
31
32 -var folderData = unixfs.FolderPBData()
33 -
32 // how many bytes of progress to wait before sending a progress update message
33 const progressReaderIncrement = 1024 * 256
34
@@ -67,7 +65,7 @@ type AddedObject struct {
65 }
66
67 func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
70 - mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil)
68 + mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil)
69 if err != nil {
70 return nil, err
71 }
@@ -392,7 +390,7 @@ func (adder *Adder) addFile(file files.File) error {
390 return err
391 }
392
395 - dagnode := &dag.Node{Data: sdata}
393 + dagnode := dag.NodeWithData(sdata)
394 _, err = adder.dagService.Add(dagnode)
395 if err != nil {
396 return err
@@ -488,11 +486,6 @@ func NewMemoryDagService() dag.DAGService {
486 return dag.NewDAGService(bsrv)
487 }
488
491 -// TODO: generalize this to more than unix-fs nodes.
492 -func NewDirNode() *dag.Node {
493 - return &dag.Node{Data: unixfs.FolderPBData()}
494 -}
495 -
489 // from core/commands/object.go
490 func getOutput(dagnode *dag.Node) (*Object, error) {
491 key, err := dagnode.Key()
core/coreunix/metadata.go
+2 -2
@@ -21,7 +21,7 @@ func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error
21 return "", err
22 }
23
24 - mdnode.Data = mdata
24 + mdnode.SetData(mdata)
25 if err := mdnode.AddNodeLinkClean("file", nd); err != nil {
26 return "", err
27 }
@@ -42,5 +42,5 @@ func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) {
42 return nil, err
43 }
44
45 - return ft.MetadataFromBytes(nd.Data)
45 + return ft.MetadataFromBytes(nd.Data())
46 }
fuse/ipns/common.go
+1 -2
@@ -4,7 +4,6 @@ import (
4 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
5
6 "github.com/ipfs/go-ipfs/core"
7 - mdag "github.com/ipfs/go-ipfs/merkledag"
7 nsys "github.com/ipfs/go-ipfs/namesys"
8 path "github.com/ipfs/go-ipfs/path"
9 ft "github.com/ipfs/go-ipfs/unixfs"
@@ -14,7 +13,7 @@ import (
13 // InitializeKeyspace sets the ipns record for the given key to
14 // point to an empty directory.
15 func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
17 - emptyDir := &mdag.Node{Data: ft.FolderPBData()}
16 + emptyDir := ft.EmptyDirNode()
17 nodek, err := n.DAG.Add(emptyDir)
18 if err != nil {
19 return err
fuse/ipns/ipns_unix.go
+1 -1
@@ -481,7 +481,7 @@ func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
481
482 func (dir *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
483 // New 'empty' file
484 - nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
484 + nd := dag.NodeWithData(ft.FilePBData(nil, 0))
485 err := dir.dir.AddChild(req.Name, nd)
486 if err != nil {
487 return nil, nil, err
fuse/readonly/readonly_unix.go
+1 -1
@@ -84,7 +84,7 @@ type Node struct {
84
85 func (s *Node) loadData() error {
86 s.cached = new(ftpb.Data)
87 - return proto.Unmarshal(s.Nd.Data, s.cached)
87 + return proto.Unmarshal(s.Nd.Data(), s.cached)
88 }
89
90 // Attr returns the attributes of a given node.
importer/helpers/helpers.go
+2 -2
@@ -59,7 +59,7 @@ func NewUnixfsBlock() *UnixfsNode {
59
60 // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
61 func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) {
62 - mb, err := ft.FSNodeFromBytes(nd.Data)
62 + mb, err := ft.FSNodeFromBytes(nd.Data())
63 if err != nil {
64 return nil, err
65 }
@@ -126,6 +126,6 @@ func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
126 if err != nil {
127 return nil, err
128 }
129 - n.node.Data = data
129 + n.node.SetData(data)
130 return n.node, nil
131 }
importer/trickle/trickle_test.go
+2 -2
@@ -524,7 +524,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
524 data := []byte("AB")
525
526 nd := new(merkledag.Node)
527 - nd.Data = ft.FilePBData(nil, 0)
527 + nd.SetData(ft.FilePBData(nil, 0))
528
529 dbp := &h.DagBuilderParams{
530 Dagserv: ds,
@@ -562,7 +562,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
562 }
563
564 func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) {
565 - pbd, err := ft.FromBytes(nd.Data)
565 + pbd, err := ft.FromBytes(nd.Data())
566 if err != nil {
567 panic(err)
568 }
importer/trickle/trickledag.go
+2 -2
@@ -241,7 +241,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
241 return errors.New("expected direct block")
242 }
243
244 - pbn, err := ft.FromBytes(nd.Data)
244 + pbn, err := ft.FromBytes(nd.Data())
245 if err != nil {
246 return err
247 }
@@ -253,7 +253,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
253 }
254
255 // Verify this is a branch node
256 - pbn, err := ft.FromBytes(nd.Data)
256 + pbn, err := ft.FromBytes(nd.Data())
257 if err != nil {
258 return err
259 }
merkledag/coding.go
+8 -3
@@ -33,7 +33,8 @@ func (n *Node) unmarshal(encoded []byte) error {
33 }
34 sort.Stable(LinkSlice(n.Links)) // keep links sorted
35
36 - n.Data = pbn.GetData()
36 + n.data = pbn.GetData()
37 + n.encoded = encoded
38 return nil
39 }
40
@@ -62,8 +63,8 @@ func (n *Node) getPBNode() *pb.PBNode {
63 pbn.Links[i].Hash = []byte(l.Hash)
64 }
65
65 - if len(n.Data) > 0 {
66 - pbn.Data = n.Data
66 + if len(n.data) > 0 {
67 + pbn.Data = n.data
68 }
69 return pbn
70 }
@@ -73,11 +74,15 @@ func (n *Node) getPBNode() *pb.PBNode {
74 func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
75 sort.Stable(LinkSlice(n.Links)) // keep links sorted
76 if n.encoded == nil || force {
77 + n.cached = nil
78 var err error
79 n.encoded, err = n.Marshal()
80 if err != nil {
81 return nil, err
82 }
83 + }
84 +
85 + if n.cached == nil {
86 n.cached = u.Hash(n.encoded)
87 }
88
merkledag/merkledag.go
+4
@@ -93,6 +93,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
93 }
94 return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
95 }
96 +
97 + res.cached = k.ToMultihash()
98 +
99 return res, nil
100 }
101
@@ -147,6 +150,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
150 out <- &NodeOption{Err: err}
151 return
152 }
153 + nd.cached = b.Key().ToMultihash()
154
155 // buffered, no need to select
156 out <- &NodeOption{Node: nd}
merkledag/merkledag_test.go
+9 -9
@@ -46,9 +46,9 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
46
47 func TestNode(t *testing.T) {
48
49 - n1 := &Node{Data: []byte("beep")}
50 - n2 := &Node{Data: []byte("boop")}
51 - n3 := &Node{Data: []byte("beep boop")}
49 + n1 := NodeWithData([]byte("beep"))
50 + n2 := NodeWithData([]byte("boop"))
51 + n3 := NodeWithData([]byte("beep boop"))
52 if err := n3.AddNodeLink("beep-link", n1); err != nil {
53 t.Error(err)
54 }
@@ -58,7 +58,7 @@ func TestNode(t *testing.T) {
58
59 printn := func(name string, n *Node) {
60 fmt.Println(">", name)
61 - fmt.Println("data:", string(n.Data))
61 + fmt.Println("data:", string(n.Data()))
62
63 fmt.Println("links:")
64 for _, l := range n.Links {
@@ -118,8 +118,8 @@ func SubtestNodeStat(t *testing.T, n *Node) {
118 expected := NodeStat{
119 NumLinks: len(n.Links),
120 BlockSize: len(enc),
121 - LinksSize: len(enc) - len(n.Data), // includes framing.
122 - DataSize: len(n.Data),
121 + LinksSize: len(enc) - len(n.Data()), // includes framing.
122 + DataSize: len(n.Data()),
123 CumulativeSize: int(cumSize),
124 Hash: k.B58String(),
125 }
@@ -255,7 +255,7 @@ func TestEmptyKey(t *testing.T) {
255
256 func TestCantGet(t *testing.T) {
257 dsp := getDagservAndPinner(t)
258 - a := &Node{Data: []byte("A")}
258 + a := NodeWithData([]byte("A"))
259
260 k, err := a.Key()
261 if err != nil {
@@ -339,7 +339,7 @@ func TestFetchFailure(t *testing.T) {
339
340 top := new(Node)
341 for i := 0; i < 10; i++ {
342 - nd := &Node{Data: []byte{byte('a' + i)}}
342 + nd := NodeWithData([]byte{byte('a' + i)})
343 _, err := ds.Add(nd)
344 if err != nil {
345 t.Fatal(err)
@@ -352,7 +352,7 @@ func TestFetchFailure(t *testing.T) {
352 }
353
354 for i := 0; i < 10; i++ {
355 - nd := &Node{Data: []byte{'f', 'a' + byte(i)}}
355 + nd := NodeWithData([]byte{'f', 'a' + byte(i)})
356 _, err := ds_bad.Add(nd)
357 if err != nil {
358 t.Fatal(err)
merkledag/node.go
+20 -6
@@ -15,7 +15,7 @@ var ErrLinkNotFound = fmt.Errorf("no link by that name")
15 // nodes have opaque data and a set of navigable links.
16 type Node struct {
17 Links []*Link
18 - Data []byte
18 + data []byte
19
20 // cache encoded/marshaled value
21 encoded []byte
@@ -78,6 +78,10 @@ func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
78 return serv.Get(ctx, key.Key(l.Hash))
79 }
80
81 +func NodeWithData(d []byte) *Node {
82 + return &Node{data: d}
83 +}
84 +
85 // AddNodeLink adds a link to another node.
86 func (n *Node) AddNodeLink(name string, that *Node) error {
87 n.encoded = nil
@@ -168,9 +172,9 @@ func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*
172 // NOTE: Does not make copies of Node objects in the links.
173 func (n *Node) Copy() *Node {
174 nnode := new(Node)
171 - if len(n.Data) > 0 {
172 - nnode.Data = make([]byte, len(n.Data))
173 - copy(nnode.Data, n.Data)
175 + if len(n.data) > 0 {
176 + nnode.data = make([]byte, len(n.data))
177 + copy(nnode.data, n.data)
178 }
179
180 if len(n.Links) > 0 {
@@ -180,6 +184,16 @@ func (n *Node) Copy() *Node {
184 return nnode
185 }
186
187 +func (n *Node) Data() []byte {
188 + return n.data
189 +}
190 +
191 +func (n *Node) SetData(d []byte) {
192 + n.encoded = nil
193 + n.cached = nil
194 + n.data = d
195 +}
196 +
197 // UpdateNodeLink return a copy of the node with the link name set to point to
198 // that. If a link of the same name existed, it is removed.
199 func (n *Node) UpdateNodeLink(name string, that *Node) (*Node, error) {
@@ -226,8 +240,8 @@ func (n *Node) Stat() (*NodeStat, error) {
240 Hash: key.B58String(),
241 NumLinks: len(n.Links),
242 BlockSize: len(enc),
229 - LinksSize: len(enc) - len(n.Data), // includes framing.
230 - DataSize: len(n.Data),
243 + LinksSize: len(enc) - len(n.data), // includes framing.
244 + DataSize: len(n.data),
245 CumulativeSize: int(cumSize),
246 }, nil
247 }
merkledag/traverse/traverse_test.go
+7 -7
@@ -326,7 +326,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
326
327 buf := new(bytes.Buffer)
328 walk := func(current State) error {
329 - s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data)
329 + s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data())
330 t.Logf("walk: %s", s)
331 buf.Write([]byte(s))
332 return nil
@@ -349,7 +349,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
349 }
350
351 func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
352 - a := &mdag.Node{Data: []byte("/a")}
352 + a := mdag.NodeWithData([]byte("/a"))
353 addLink(t, ds, a, child(t, ds, a, "aa"))
354 addLink(t, ds, a, child(t, ds, a, "ab"))
355 addLink(t, ds, a, child(t, ds, a, "ac"))
@@ -358,7 +358,7 @@ func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
358 }
359
360 func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
361 - a := &mdag.Node{Data: []byte("/a")}
361 + a := mdag.NodeWithData([]byte("/a"))
362 aa := child(t, ds, a, "aa")
363 aaa := child(t, ds, aa, "aaa")
364 aaaa := child(t, ds, aaa, "aaaa")
@@ -371,7 +371,7 @@ func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
371 }
372
373 func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
374 - a := &mdag.Node{Data: []byte("/a")}
374 + a := mdag.NodeWithData([]byte("/a"))
375 aa := child(t, ds, a, "aa")
376 ab := child(t, ds, a, "ab")
377 addLink(t, ds, aa, child(t, ds, aa, "aaa"))
@@ -384,7 +384,7 @@ func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
384 }
385
386 func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
387 - a := &mdag.Node{Data: []byte("/a")}
387 + a := mdag.NodeWithData([]byte("/a"))
388 aa := child(t, ds, a, "aa")
389 aaa := child(t, ds, aa, "aaa")
390 aaaa := child(t, ds, aaa, "aaaa")
@@ -401,7 +401,7 @@ func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
401 }
402
403 func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
404 - to := string(a.Data) + "2" + string(b.Data)
404 + to := string(a.Data()) + "2" + string(b.Data())
405 if _, err := ds.Add(b); err != nil {
406 t.Error(err)
407 }
@@ -411,5 +411,5 @@ func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
411 }
412
413 func child(t *testing.T, ds mdag.DAGService, a *mdag.Node, name string) *mdag.Node {
414 - return &mdag.Node{Data: []byte(string(a.Data) + "/" + name)}
414 + return mdag.NodeWithData([]byte(string(a.Data()) + "/" + name))
415 }
merkledag/utils/utils_test.go
+2 -4
@@ -13,9 +13,7 @@ import (
13
14 func TestAddLink(t *testing.T) {
15 ds := mdtest.Mock()
16 - fishnode := &dag.Node{
17 - Data: []byte("fishcakes!"),
18 - }
16 + fishnode := dag.NodeWithData([]byte("fishcakes!"))
17
18 fk, err := ds.Add(fishnode)
19 if err != nil {
@@ -90,7 +88,7 @@ func TestInsertNode(t *testing.T) {
88 }
89
90 func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
93 - child := &dag.Node{Data: []byte(data)}
91 + child := dag.NodeWithData([]byte(data))
92 ck, err := e.tmp.Add(child)
93 if err != nil {
94 t.Fatal(err)
mfs/dir.go
+3 -2
@@ -121,7 +121,7 @@ func (d *Directory) childNode(name string) (FSNode, error) {
121
122 // cacheNode caches a node into d.childDirs or d.files and returns the FSNode.
123 func (d *Directory) cacheNode(name string, nd *dag.Node) (FSNode, error) {
124 - i, err := ft.FromBytes(nd.Data)
124 + i, err := ft.FromBytes(nd.Data())
125 if err != nil {
126 return nil, err
127 }
@@ -268,7 +268,8 @@ func (d *Directory) Mkdir(name string) (*Directory, error) {
268 }
269 }
270
271 - ndir := &dag.Node{Data: ft.FolderPBData()}
271 + ndir := new(dag.Node)
272 + ndir.SetData(ft.FolderPBData())
273
274 _, err = d.dserv.Add(ndir)
275 if err != nil {
mfs/file.go
+2 -2
@@ -45,7 +45,7 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
45 node := fi.node
46 fi.nodelk.Unlock()
47
48 - fsn, err := ft.FSNodeFromBytes(node.Data)
48 + fsn, err := ft.FSNodeFromBytes(node.Data())
49 if err != nil {
50 return nil, err
51 }
@@ -86,7 +86,7 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
86 func (fi *File) Size() (int64, error) {
87 fi.nodelk.Lock()
88 defer fi.nodelk.Unlock()
89 - pbd, err := ft.FromBytes(fi.node.Data)
89 + pbd, err := ft.FromBytes(fi.node.Data())
90 if err != nil {
91 return 0, err
92 }
mfs/mfs_test.go
+19 -17
@@ -31,6 +31,10 @@ import (
31 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
32 )
33
34 +func emptyDirNode() *dag.Node {
35 + return dag.NodeWithData(ft.FolderPBData())
36 +}
37 +
38 func getDagserv(t *testing.T) dag.DAGService {
39 db := dssync.MutexWrap(ds.NewMapDatastore())
40 bs := bstore.NewBlockstore(db)
@@ -183,7 +187,7 @@ func catNode(ds dag.DAGService, nd *dag.Node) ([]byte, error) {
187 func setupRoot(ctx context.Context, t *testing.T) (dag.DAGService, *Root) {
188 ds := getDagserv(t)
189
186 - root := &dag.Node{Data: ft.FolderPBData()}
190 + root := emptyDirNode()
191 rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, k key.Key) error {
192 fmt.Println("PUBLISHED: ", k)
193 return nil
@@ -282,7 +286,7 @@ func TestDirectoryLoadFromDag(t *testing.T) {
286 t.Fatal(err)
287 }
288
285 - dir := &dag.Node{Data: ft.FolderPBData()}
289 + dir := emptyDirNode()
290 _, err = ds.Add(dir)
291 if err != nil {
292 t.Fatal(err)
@@ -293,17 +297,15 @@ func TestDirectoryLoadFromDag(t *testing.T) {
297 t.Fatal(err)
298 }
299
296 - top := &dag.Node{
297 - Data: ft.FolderPBData(),
298 - Links: []*dag.Link{
299 - &dag.Link{
300 - Name: "a",
301 - Hash: fihash,
302 - },
303 - &dag.Link{
304 - Name: "b",
305 - Hash: dirhash,
306 - },
300 + top := emptyDirNode()
301 + top.Links = []*dag.Link{
302 + &dag.Link{
303 + Name: "a",
304 + Hash: fihash,
305 + },
306 + &dag.Link{
307 + Name: "b",
308 + Hash: dirhash,
309 },
310 }
311
@@ -540,7 +542,7 @@ func actorMakeFile(d *Directory) error {
542 }
543
544 name := randomName()
543 - f, err := NewFile(name, &dag.Node{Data: ft.FilePBData(nil, 0)}, d, d.dserv)
545 + f, err := NewFile(name, dag.NodeWithData(ft.FilePBData(nil, 0)), d, d.dserv)
546 if err != nil {
547 return err
548 }
@@ -756,7 +758,7 @@ func TestFlushing(t *testing.T) {
758 e := mkdirP(t, dir, "a/b/e")
759
760 data := []byte("this is a test\n")
759 - nd1 := &dag.Node{Data: ft.FilePBData(data, uint64(len(data)))}
761 + nd1 := dag.NodeWithData(ft.FilePBData(data, uint64(len(data))))
762
763 if err := c.AddChild("TEST", nd1); err != nil {
764 t.Fatal(err)
@@ -792,7 +794,7 @@ func TestFlushing(t *testing.T) {
794 t.Fatal(err)
795 }
796
795 - fsnode, err := ft.FSNodeFromBytes(rnd.Data)
797 + fsnode, err := ft.FSNodeFromBytes(rnd.Data())
798 if err != nil {
799 t.Fatal(err)
800 }
@@ -897,7 +899,7 @@ func TestFileDescriptors(t *testing.T) {
899 ds, rt := setupRoot(ctx, t)
900 dir := rt.GetValue().(*Directory)
901
900 - nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
902 + nd := dag.NodeWithData(ft.FilePBData(nil, 0))
903 fi, err := NewFile("test", nd, dir, ds)
904 if err != nil {
905 t.Fatal(err)
mfs/system.go
+1 -1
@@ -83,7 +83,7 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.Node, pf PubFu
83 dserv: ds,
84 }
85
86 - pbn, err := ft.FromBytes(node.Data)
86 + pbn, err := ft.FromBytes(node.Data())
87 if err != nil {
88 log.Error("IPNS pointer was not unixfs node")
89 return nil, err
namesys/publisher.go
+1 -1
@@ -330,7 +330,7 @@ func ValidateIpnsRecord(k key.Key, val []byte) error {
330 // point to an empty directory.
331 // TODO: this doesnt feel like it belongs here
332 func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, pins pin.Pinner, key ci.PrivKey) error {
333 - emptyDir := &dag.Node{Data: ft.FolderPBData()}
333 + emptyDir := ft.EmptyDirNode()
334 nodek, err := ds.Add(emptyDir)
335 if err != nil {
336 return err
path/resolver_test.go
+2 -2
@@ -15,8 +15,8 @@ import (
15
16 func randNode() (*merkledag.Node, key.Key) {
17 node := new(merkledag.Node)
18 - node.Data = make([]byte, 32)
19 - util.NewTimeSeededRand().Read(node.Data)
18 + node.SetData(make([]byte, 32))
19 + util.NewTimeSeededRand().Read(node.Data())
20 k, _ := node.Key()
21 return node, k
22 }
pin/pin_test.go
+2 -2
@@ -18,8 +18,8 @@ import (
18
19 func randNode() (*mdag.Node, key.Key) {
20 nd := new(mdag.Node)
21 - nd.Data = make([]byte, 32)
22 - util.NewTimeSeededRand().Read(nd.Data)
21 + nd.SetData(make([]byte, 32))
22 + util.NewTimeSeededRand().Read(nd.Data())
23 k, _ := nd.Key()
24 return nd, k
25 }
pin/set.go
+9 -9
@@ -111,7 +111,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
111 if err := writeHdr(n, hdr); err != nil {
112 return nil, err
113 }
114 - hdrLen := len(n.Data)
114 + hdrLen := len(n.Data())
115
116 if estimatedLen < maxItems {
117 // it'll probably fit
@@ -122,12 +122,12 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
122 break
123 }
124 n.Links = append(n.Links, &merkledag.Link{Hash: k.ToMultihash()})
125 - n.Data = append(n.Data, data...)
125 + n.SetData(append(n.Data(), data...))
126 }
127 // sort by hash, also swap item Data
128 s := sortByHash{
129 links: n.Links[defaultFanout:],
130 - data: n.Data[hdrLen:],
130 + data: n.Data()[hdrLen:],
131 }
132 sort.Stable(s)
133 }
@@ -179,11 +179,11 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
179 }
180
181 func readHdr(n *merkledag.Node) (*pb.Set, []byte, error) {
182 - hdrLenRaw, consumed := binary.Uvarint(n.Data)
182 + hdrLenRaw, consumed := binary.Uvarint(n.Data())
183 if consumed <= 0 {
184 return nil, nil, errors.New("invalid Set header length")
185 }
186 - buf := n.Data[consumed:]
186 + buf := n.Data()[consumed:]
187 if hdrLenRaw > uint64(len(buf)) {
188 return nil, nil, errors.New("impossibly large Set header length")
189 }
@@ -209,10 +209,10 @@ func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
209 if err != nil {
210 return err
211 }
212 - n.Data = make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData))
213 - written := binary.PutUvarint(n.Data, uint64(len(hdrData)))
214 - n.Data = n.Data[:written]
215 - n.Data = append(n.Data, hdrData...)
212 + n.SetData(make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)))
213 + written := binary.PutUvarint(n.Data(), uint64(len(hdrData)))
214 + n.SetData(n.Data()[:written])
215 + n.SetData(append(n.Data(), hdrData...))
216 return nil
217 }
218
tar/format.go
+4 -4
@@ -45,7 +45,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
45 tr := tar.NewReader(r)
46
47 root := new(dag.Node)
48 - root.Data = []byte("ipfs/tar")
48 + root.SetData([]byte("ipfs/tar"))
49
50 e := dagutil.NewDagEditor(root, ds)
51
@@ -65,7 +65,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
65 return nil, err
66 }
67
68 - header.Data = headerBytes
68 + header.SetData(headerBytes)
69
70 if h.Size > 0 {
71 spl := chunk.NewRabin(tr, uint64(chunk.DefaultBlockSize))
@@ -170,7 +170,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
170 return 0, err
171 }
172
173 - tr.hdrBuf = bytes.NewReader(headerNd.Data)
173 + tr.hdrBuf = bytes.NewReader(headerNd.Data())
174
175 dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
176 if err != nil && err != dag.ErrLinkNotFound {
@@ -197,7 +197,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
197 }
198
199 func ExportTar(ctx context.Context, root *dag.Node, ds dag.DAGService) (io.Reader, error) {
200 - if string(root.Data) != "ipfs/tar" {
200 + if string(root.Data()) != "ipfs/tar" {
201 return nil, errors.New("not an ipfs tarchive")
202 }
203 return &tarReader{
unixfs/archive/tar/writer.go
+1 -1
@@ -69,7 +69,7 @@ func (w *Writer) writeFile(nd *mdag.Node, pb *upb.Data, fpath string) error {
69
70 func (w *Writer) WriteNode(nd *mdag.Node, fpath string) error {
71 pb := new(upb.Data)
72 - if err := proto.Unmarshal(nd.Data, pb); err != nil {
72 + if err := proto.Unmarshal(nd.Data(), pb); err != nil {
73 return err
74 }
75
unixfs/format.go
+5
@@ -6,6 +6,7 @@ package unixfs
6 import (
7 "errors"
8
9 + dag "github.com/ipfs/go-ipfs/merkledag"
10 pb "github.com/ipfs/go-ipfs/unixfs/pb"
11 proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
12 )
@@ -222,3 +223,7 @@ func BytesForMetadata(m *Metadata) ([]byte, error) {
223 pbd.Data = mdd
224 return proto.Marshal(pbd)
225 }
226 +
227 +func EmptyDirNode() *dag.Node {
228 + return dag.NodeWithData(FolderPBData())
229 +}
unixfs/io/dagreader.go
+2 -2
@@ -60,7 +60,7 @@ type ReadSeekCloser interface {
60 // the given node, using the passed in DAGService for data retreival
61 func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) {
62 pb := new(ftpb.Data)
63 - if err := proto.Unmarshal(n.Data, pb); err != nil {
63 + if err := proto.Unmarshal(n.Data(), pb); err != nil {
64 return nil, err
65 }
66
@@ -117,7 +117,7 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error {
117 dr.linkPosition++
118
119 pb := new(ftpb.Data)
120 - err = proto.Unmarshal(nxt.Data, pb)
120 + err = proto.Unmarshal(nxt.Data(), pb)
121 if err != nil {
122 return fmt.Errorf("incorrectly formatted protobuf: %s", err)
123 }
unixfs/io/dirbuilder.go
+3 -1
@@ -15,7 +15,9 @@ type directoryBuilder struct {
15
16 // NewEmptyDirectory returns an empty merkledag Node with a folder Data chunk
17 func NewEmptyDirectory() *mdag.Node {
18 - return &mdag.Node{Data: format.FolderPBData()}
18 + nd := new(mdag.Node)
19 + nd.SetData(format.FolderPBData())
20 + return nd
21 }
22
23 // NewDirectory returns a directoryBuilder. It needs a DAGService to add the Children
unixfs/mod/dagmodifier.go
+8 -7
@@ -139,7 +139,7 @@ func (dm *DagModifier) Write(b []byte) (int, error) {
139 }
140
141 func (dm *DagModifier) Size() (int64, error) {
142 - pbn, err := ft.FromBytes(dm.curNode.Data)
142 + pbn, err := ft.FromBytes(dm.curNode.Data())
143 if err != nil {
144 return 0, err
145 }
@@ -207,7 +207,7 @@ func (dm *DagModifier) Sync() error {
207 // returns the new key of the passed in node and whether or not all the data in the reader
208 // has been consumed.
209 func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (key.Key, bool, error) {
210 - f, err := ft.FromBytes(node.Data)
210 + f, err := ft.FromBytes(node.Data())
211 if err != nil {
212 return "", false, err
213 }
@@ -225,7 +225,8 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
225 return "", false, err
226 }
227
228 - nd := &mdag.Node{Data: b}
228 + nd := new(mdag.Node)
229 + nd.SetData(b)
230 k, err := dm.dagserv.Add(nd)
231 if err != nil {
232 return "", false, err
@@ -429,12 +430,12 @@ func (dm *DagModifier) Truncate(size int64) error {
430 func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGService) (*mdag.Node, error) {
431 if len(nd.Links) == 0 {
432 // TODO: this can likely be done without marshaling and remarshaling
432 - pbn, err := ft.FromBytes(nd.Data)
433 + pbn, err := ft.FromBytes(nd.Data())
434 if err != nil {
435 return nil, err
436 }
437
437 - nd.Data = ft.WrapData(pbn.Data[:size])
438 + nd.SetData(ft.WrapData(pbn.Data[:size]))
439 return nd, nil
440 }
441
@@ -448,7 +449,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer
449 return nil, err
450 }
451
451 - childsize, err := ft.DataSize(child.Data)
452 + childsize, err := ft.DataSize(child.Data())
453 if err != nil {
454 return nil, err
455 }
@@ -486,7 +487,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer
487 return nil, err
488 }
489
489 - nd.Data = d
490 + nd.SetData(d)
491
492 // invalidate cache and recompute serialized data
493 _, err = nd.EncodeProtobuf(true)
unixfs/mod/dagmodifier_test.go
+2 -2
@@ -156,7 +156,7 @@ func TestDagModifierBasic(t *testing.T) {
156 t.Fatal(err)
157 }
158
159 - size, err := ft.DataSize(node.Data)
159 + size, err := ft.DataSize(node.Data())
160 if err != nil {
161 t.Fatal(err)
162 }
@@ -590,7 +590,7 @@ func arrComp(a, b []byte) error {
590 }
591
592 func printDag(nd *mdag.Node, ds mdag.DAGService, indent int) {
593 - pbd, err := ft.FromBytes(nd.Data)
593 + pbd, err := ft.FromBytes(nd.Data())
594 if err != nil {
595 panic(err)
596 }