@cryptotaxi247 / kubo / commits / ded60a735

unixfs: allow use of raw merkledag nodes for unixfs files

License: MIT Signed-off-by: Jeromy <why@ipfs.io>

Jeromy committed Oct 15, 2016 at 09:06 UTC ded60a73562a9df9a67ef328d364431a40d1a37c
24 files changed +390 -150
blocks/blocks.go
+5 -2
@@ -37,8 +37,11 @@ func NewBlock(data []byte) *BasicBlock {
37 // we are able to be confident that the data is correct
38 func NewBlockWithCid(data []byte, c *cid.Cid) (*BasicBlock, error) {
39 if u.Debug {
40 - // TODO: fix assumptions
41 - chkc := cid.NewCidV0(u.Hash(data))
40 + chkc, err := c.Prefix().Sum(data)
41 + if err != nil {
42 + return nil, err
43 + }
44 +
45 if !chkc.Equals(c) {
46 return nil, ErrWrongHash
47 }
core/commands/add.go
+13 -9
@@ -23,15 +23,16 @@ import (
23 var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
24
25 const (
26 - quietOptionName = "quiet"
27 - silentOptionName = "silent"
28 - progressOptionName = "progress"
29 - trickleOptionName = "trickle"
30 - wrapOptionName = "wrap-with-directory"
31 - hiddenOptionName = "hidden"
32 - onlyHashOptionName = "only-hash"
33 - chunkerOptionName = "chunker"
34 - pinOptionName = "pin"
26 + quietOptionName = "quiet"
27 + silentOptionName = "silent"
28 + progressOptionName = "progress"
29 + trickleOptionName = "trickle"
30 + wrapOptionName = "wrap-with-directory"
31 + hiddenOptionName = "hidden"
32 + onlyHashOptionName = "only-hash"
33 + chunkerOptionName = "chunker"
34 + pinOptionName = "pin"
35 + rawLeavesOptionName = "raw-leaves"
36 )
37
38 var AddCmd = &cmds.Command{
@@ -78,6 +79,7 @@ You can now refer to the added file in a gateway, like so:
79 cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false),
80 cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
81 cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
82 + cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
83 },
84 PreRun: func(req cmds.Request) error {
85 if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
@@ -135,6 +137,7 @@ You can now refer to the added file in a gateway, like so:
137 silent, _, _ := req.Option(silentOptionName).Bool()
138 chunker, _, _ := req.Option(chunkerOptionName).String()
139 dopin, _, _ := req.Option(pinOptionName).Bool()
140 + rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()
141
142 if hash {
143 nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
@@ -174,6 +177,7 @@ You can now refer to the added file in a gateway, like so:
177 fileAdder.Wrap = wrap
178 fileAdder.Pin = dopin
179 fileAdder.Silent = silent
180 + fileAdder.RawLeaves = rawblks
181
182 if hash {
183 md := dagtest.Mock()
core/corehttp/gateway_handler.go
+11 -4
@@ -1,6 +1,7 @@
1 package corehttp
2
3 import (
4 + "context"
5 "errors"
6 "fmt"
7 "io"
@@ -18,10 +19,10 @@ import (
19 path "github.com/ipfs/go-ipfs/path"
20 uio "github.com/ipfs/go-ipfs/unixfs/io"
21
21 - "context"
22 routing "gx/ipfs/QmNUgVQTYnXQVrGT2rajZYsuKV8GYdiL91cdZSQDKNPNgE/go-libp2p-routing"
23 humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
24 cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
25 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
26 )
27
28 const (
@@ -45,7 +46,7 @@ func newGatewayHandler(node *core.IpfsNode, conf GatewayConfig) *gatewayHandler
46 }
47
48 // TODO(cryptix): find these helpers somewhere else
48 -func (i *gatewayHandler) newDagFromReader(r io.Reader) (*dag.ProtoNode, error) {
49 +func (i *gatewayHandler) newDagFromReader(r io.Reader) (node.Node, error) {
50 // TODO(cryptix): change and remove this helper once PR1136 is merged
51 // return ufs.AddFromReader(i.node, r.Body)
52 return importer.BuildDagFromReader(
@@ -353,7 +354,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
354 return
355 }
356
356 - var newnode *dag.ProtoNode
357 + var newnode node.Node
358 if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" {
359 newnode = uio.NewEmptyDirectory()
360 } else {
@@ -417,8 +418,14 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
418 return
419 }
420
421 + pbnewnode, ok := newnode.(*dag.ProtoNode)
422 + if !ok {
423 + webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
424 + return
425 + }
426 +
427 // object set-data case
421 - pbnd.SetData(newnode.Data())
428 + pbnd.SetData(pbnewnode.Data())
429
430 newcid, err = i.node.DAG.Add(pbnd)
431 if err != nil {
core/coreunix/add.go
+19 -15
@@ -1,6 +1,7 @@
1 package coreunix
2
3 import (
4 + "context"
5 "fmt"
6 "io"
7 "io/ioutil"
@@ -13,16 +14,18 @@ import (
14 "github.com/ipfs/go-ipfs/commands/files"
15 core "github.com/ipfs/go-ipfs/core"
16 "github.com/ipfs/go-ipfs/exchange/offline"
16 - importer "github.com/ipfs/go-ipfs/importer"
17 + balanced "github.com/ipfs/go-ipfs/importer/balanced"
18 "github.com/ipfs/go-ipfs/importer/chunk"
19 + ihelper "github.com/ipfs/go-ipfs/importer/helpers"
20 + trickle "github.com/ipfs/go-ipfs/importer/trickle"
21 dag "github.com/ipfs/go-ipfs/merkledag"
22 mfs "github.com/ipfs/go-ipfs/mfs"
23 "github.com/ipfs/go-ipfs/pin"
24 unixfs "github.com/ipfs/go-ipfs/unixfs"
25
23 - context "context"
26 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
27 cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
28 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
29 ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
30 syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
31 )
@@ -97,6 +100,7 @@ type Adder struct {
100 Hidden bool
101 Pin bool
102 Trickle bool
103 + RawLeaves bool
104 Silent bool
105 Wrap bool
106 Chunker string
@@ -111,22 +115,22 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) {
115 }
116
117 // Perform the actual add & pin locally, outputting results to reader
114 -func (adder Adder) add(reader io.Reader) (*dag.ProtoNode, error) {
118 +func (adder Adder) add(reader io.Reader) (node.Node, error) {
119 chnk, err := chunk.FromString(reader, adder.Chunker)
120 if err != nil {
121 return nil, err
122 }
123 + params := ihelper.DagBuilderParams{
124 + Dagserv: adder.dagService,
125 + RawLeaves: adder.RawLeaves,
126 + Maxlinks: ihelper.DefaultLinksPerBlock,
127 + }
128
129 if adder.Trickle {
121 - return importer.BuildTrickleDagFromReader(
122 - adder.dagService,
123 - chnk,
124 - )
125 - }
126 - return importer.BuildDagFromReader(
127 - adder.dagService,
128 - chnk,
129 - )
130 + return trickle.TrickleLayout(params.New(chnk))
131 + }
132 +
133 + return balanced.BalancedLayout(params.New(chnk))
134 }
135
136 func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
@@ -331,7 +335,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Pr
335 return gopath.Join(c.String(), filename), dagnode, nil
336 }
337
334 -func (adder *Adder) addNode(node *dag.ProtoNode, path string) error {
338 +func (adder *Adder) addNode(node node.Node, path string) error {
339 // patch it into the root
340 if path == "" {
341 path = node.Cid().String()
@@ -456,7 +460,7 @@ func (adder *Adder) maybePauseForGC() error {
460 }
461
462 // outputDagnode sends dagnode info over the output channel
459 -func outputDagnode(out chan interface{}, name string, dn *dag.ProtoNode) error {
463 +func outputDagnode(out chan interface{}, name string, dn node.Node) error {
464 if out == nil {
465 return nil
466 }
@@ -482,7 +486,7 @@ func NewMemoryDagService() dag.DAGService {
486 }
487
488 // from core/commands/object.go
485 -func getOutput(dagnode *dag.ProtoNode) (*Object, error) {
489 +func getOutput(dagnode node.Node) (*Object, error) {
490 c := dagnode.Cid()
491
492 output := &Object{
fuse/readonly/ipfs_test.go
+4 -3
@@ -24,6 +24,7 @@ import (
24
25 fstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
26 cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
27 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
28 u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
29 )
30
@@ -33,7 +34,7 @@ func maybeSkipFuseTests(t *testing.T) {
34 }
35 }
36
36 -func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.ProtoNode, []byte) {
37 +func randObj(t *testing.T, nd *core.IpfsNode, size int64) (node.Node, []byte) {
38 buf := make([]byte, size)
39 u.NewTimeSeededRand().Read(buf)
40 read := bytes.NewReader(buf)
@@ -74,7 +75,7 @@ func TestIpfsBasicRead(t *testing.T) {
75 defer mnt.Close()
76
77 fi, data := randObj(t, nd, 10000)
77 - k := fi.Key()
78 + k := fi.Cid()
79 fname := path.Join(mnt.Dir, k.String())
80 rbuf, err := ioutil.ReadFile(fname)
81 if err != nil {
@@ -254,7 +255,7 @@ func TestFileSizeReporting(t *testing.T) {
255 defer mnt.Close()
256
257 fi, data := randObj(t, nd, 10000)
257 - k := fi.Key()
258 + k := fi.Cid()
259
260 fname := path.Join(mnt.Dir, k.String())
261
importer/balanced/balanced_test.go
+6 -1
@@ -28,7 +28,12 @@ func buildTestDag(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error)
28 Maxlinks: h.DefaultLinksPerBlock,
29 }
30
31 - return BalancedLayout(dbp.New(spl))
31 + nd, err := BalancedLayout(dbp.New(spl))
32 + if err != nil {
33 + return nil, err
34 + }
35 +
36 + return nd.(*dag.ProtoNode), nil
37 }
38
39 func getTestDag(t *testing.T, ds dag.DAGService, size int64, blksize int64) (*dag.ProtoNode, []byte) {
importer/balanced/builder.go
+12 -4
@@ -4,10 +4,11 @@ import (
4 "errors"
5
6 h "github.com/ipfs/go-ipfs/importer/helpers"
7 - dag "github.com/ipfs/go-ipfs/merkledag"
7 +
8 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
9 )
10
10 -func BalancedLayout(db *h.DagBuilderHelper) (*dag.ProtoNode, error) {
11 +func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) {
12 var root *h.UnixfsNode
13 for level := 0; !db.Done(); level++ {
14
@@ -56,14 +57,21 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int) error {
57
58 // Base case
59 if depth <= 0 { // catch accidental -1's in case error above is removed.
59 - return db.FillNodeWithData(node)
60 + child, err := db.GetNextDataNode()
61 + if err != nil {
62 + return err
63 + }
64 +
65 + node.Set(child)
66 + return nil
67 }
68
69 // while we have room AND we're not done
70 for node.NumChildren() < db.Maxlinks() && !db.Done() {
71 child := h.NewUnixfsNode()
72
66 - if err := fillNodeRec(db, child, depth-1); err != nil {
73 + err := fillNodeRec(db, child, depth-1)
74 + if err != nil {
75 return err
76 }
77
importer/helpers/dagbuilder.go
+34 -19
@@ -3,23 +3,30 @@ package helpers
3 import (
4 "github.com/ipfs/go-ipfs/importer/chunk"
5 dag "github.com/ipfs/go-ipfs/merkledag"
6 +
7 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
8 )
9
10 // DagBuilderHelper wraps together a bunch of objects needed to
11 // efficiently create unixfs dag trees
12 type DagBuilderHelper struct {
11 - dserv dag.DAGService
12 - spl chunk.Splitter
13 - recvdErr error
14 - nextData []byte // the next item to return.
15 - maxlinks int
16 - batch *dag.Batch
13 + dserv dag.DAGService
14 + spl chunk.Splitter
15 + recvdErr error
16 + rawLeaves bool
17 + nextData []byte // the next item to return.
18 + maxlinks int
19 + batch *dag.Batch
20 }
21
22 type DagBuilderParams struct {
23 // Maximum number of links per intermediate node
24 Maxlinks int
25
26 + // RawLeaves signifies that the importer should use raw ipld nodes as leaves
27 + // instead of using the unixfs TRaw type
28 + RawLeaves bool
29 +
30 // DAGService to write blocks to (required)
31 Dagserv dag.DAGService
32 }
@@ -28,10 +35,11 @@ type DagBuilderParams struct {
35 // from chunks object
36 func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper {
37 return &DagBuilderHelper{
31 - dserv: dbp.Dagserv,
32 - spl: spl,
33 - maxlinks: dbp.Maxlinks,
34 - batch: dbp.Dagserv.Batch(),
38 + dserv: dbp.Dagserv,
39 + spl: spl,
40 + rawLeaves: dbp.RawLeaves,
41 + maxlinks: dbp.Maxlinks,
42 + batch: dbp.Dagserv.Batch(),
43 }
44 }
45
@@ -78,9 +86,8 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error {
86
87 // while we have room AND we're not done
88 for node.NumChildren() < db.maxlinks && !db.Done() {
81 - child := NewUnixfsBlock()
82 -
83 - if err := db.FillNodeWithData(child); err != nil {
89 + child, err := db.GetNextDataNode()
90 + if err != nil {
91 return err
92 }
93
@@ -92,21 +99,29 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error {
99 return nil
100 }
101
95 -func (db *DagBuilderHelper) FillNodeWithData(node *UnixfsNode) error {
102 +func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
103 data := db.Next()
104 if data == nil { // we're done!
98 - return nil
105 + return nil, nil
106 }
107
108 if len(data) > BlockSizeLimit {
102 - return ErrSizeLimitExceeded
109 + return nil, ErrSizeLimitExceeded
110 }
111
105 - node.SetData(data)
106 - return nil
112 + if db.rawLeaves {
113 + return &UnixfsNode{
114 + rawnode: dag.NewRawNode(data),
115 + raw: true,
116 + }, nil
117 + } else {
118 + blk := NewUnixfsBlock()
119 + blk.SetData(data)
120 + return blk, nil
121 + }
122 }
123
109 -func (db *DagBuilderHelper) Add(node *UnixfsNode) (*dag.ProtoNode, error) {
124 +func (db *DagBuilderHelper) Add(node *UnixfsNode) (node.Node, error) {
125 dn, err := node.GetDagNode()
126 if err != nil {
127 return nil, err
importer/helpers/helpers.go
+21 -4
@@ -1,12 +1,14 @@
1 package helpers
2
3 import (
4 + "context"
5 "fmt"
6
6 - "context"
7 chunk "github.com/ipfs/go-ipfs/importer/chunk"
8 dag "github.com/ipfs/go-ipfs/merkledag"
9 ft "github.com/ipfs/go-ipfs/unixfs"
10 +
11 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
12 )
13
14 // BlockSizeLimit specifies the maximum size an imported block can have.
@@ -37,8 +39,10 @@ var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
39 // UnixfsNode is a struct created to aid in the generation
40 // of unixfs DAG trees
41 type UnixfsNode struct {
40 - node *dag.ProtoNode
41 - ufmt *ft.FSNode
42 + raw bool
43 + rawnode *dag.RawNode
44 + node *dag.ProtoNode
45 + ufmt *ft.FSNode
46 }
47
48 // NewUnixfsNode creates a new Unixfs node to represent a file
@@ -74,6 +78,15 @@ func (n *UnixfsNode) NumChildren() int {
78 return n.ufmt.NumChildren()
79 }
80
81 +func (n *UnixfsNode) Set(other *UnixfsNode) {
82 + n.node = other.node
83 + n.raw = other.raw
84 + n.rawnode = other.rawnode
85 + if other.ufmt != nil {
86 + n.ufmt.Data = other.ufmt.Data
87 + }
88 +}
89 +
90 func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*UnixfsNode, error) {
91 nd, err := n.node.Links()[i].GetNode(ctx, ds)
92 if err != nil {
@@ -126,7 +139,11 @@ func (n *UnixfsNode) SetData(data []byte) {
139
140 // getDagNode fills out the proper formatting for the unixfs node
141 // inside of a DAG node and returns the dag node
129 -func (n *UnixfsNode) GetDagNode() (*dag.ProtoNode, error) {
142 +func (n *UnixfsNode) GetDagNode() (node.Node, error) {
143 + if n.raw {
144 + return n.rawnode, nil
145 + }
146 +
147 data, err := n.ufmt.GetBytes()
148 if err != nil {
149 return nil, err
importer/importer.go
+5 -3
@@ -12,14 +12,16 @@ import (
12 h "github.com/ipfs/go-ipfs/importer/helpers"
13 trickle "github.com/ipfs/go-ipfs/importer/trickle"
14 dag "github.com/ipfs/go-ipfs/merkledag"
15 +
16 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
17 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
18 )
19
20 var log = logging.Logger("importer")
21
22 // Builds a DAG from the given file, writing created blocks to disk as they are
23 // created
22 -func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.ProtoNode, error) {
24 +func BuildDagFromFile(fpath string, ds dag.DAGService) (node.Node, error) {
25 stat, err := os.Lstat(fpath)
26 if err != nil {
27 return nil, err
@@ -38,7 +40,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.ProtoNode, error) {
40 return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize))
41 }
42
41 -func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) {
43 +func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error) {
44 dbp := h.DagBuilderParams{
45 Dagserv: ds,
46 Maxlinks: h.DefaultLinksPerBlock,
@@ -47,7 +49,7 @@ func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode,
49 return bal.BalancedLayout(dbp.New(spl))
50 }
51
50 -func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) {
52 +func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error) {
53 dbp := h.DagBuilderParams{
54 Dagserv: ds,
55 Maxlinks: h.DefaultLinksPerBlock,
importer/importer_test.go
+6 -4
@@ -2,19 +2,21 @@ package importer
2
3 import (
4 "bytes"
5 + "context"
6 "io"
7 "io/ioutil"
8 "testing"
9
9 - context "context"
10 chunk "github.com/ipfs/go-ipfs/importer/chunk"
11 dag "github.com/ipfs/go-ipfs/merkledag"
12 mdtest "github.com/ipfs/go-ipfs/merkledag/test"
13 uio "github.com/ipfs/go-ipfs/unixfs/io"
14 +
15 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
16 u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
17 )
18
17 -func getBalancedDag(t testing.TB, size int64, blksize int64) (*dag.ProtoNode, dag.DAGService) {
19 +func getBalancedDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) {
20 ds := mdtest.Mock()
21 r := io.LimitReader(u.NewTimeSeededRand(), size)
22 nd, err := BuildDagFromReader(ds, chunk.NewSizeSplitter(r, blksize))
@@ -24,7 +26,7 @@ func getBalancedDag(t testing.TB, size int64, blksize int64) (*dag.ProtoNode, da
26 return nd, ds
27 }
28
27 -func getTrickleDag(t testing.TB, size int64, blksize int64) (*dag.ProtoNode, dag.DAGService) {
29 +func getTrickleDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) {
30 ds := mdtest.Mock()
31 r := io.LimitReader(u.NewTimeSeededRand(), size)
32 nd, err := BuildTrickleDagFromReader(ds, chunk.NewSizeSplitter(r, blksize))
@@ -100,7 +102,7 @@ func BenchmarkTrickleReadFull(b *testing.B) {
102 runReadBench(b, nd, ds)
103 }
104
103 -func runReadBench(b *testing.B, nd *dag.ProtoNode, ds dag.DAGService) {
105 +func runReadBench(b *testing.B, nd node.Node, ds dag.DAGService) {
106 for i := 0; i < b.N; i++ {
107 ctx, cancel := context.WithCancel(context.Background())
108 read, err := uio.NewDagReader(ctx, nd, ds)
importer/trickle/trickle_test.go
+8 -2
@@ -2,6 +2,7 @@ package trickle
2
3 import (
4 "bytes"
5 + "context"
6 "fmt"
7 "io"
8 "io/ioutil"
@@ -9,7 +10,6 @@ import (
10 "os"
11 "testing"
12
12 - "context"
13 chunk "github.com/ipfs/go-ipfs/importer/chunk"
14 h "github.com/ipfs/go-ipfs/importer/helpers"
15 merkledag "github.com/ipfs/go-ipfs/merkledag"
@@ -17,6 +17,7 @@ import (
17 pin "github.com/ipfs/go-ipfs/pin"
18 ft "github.com/ipfs/go-ipfs/unixfs"
19 uio "github.com/ipfs/go-ipfs/unixfs/io"
20 +
21 u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
22 )
23
@@ -31,7 +32,12 @@ func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter) (*merkledag.Proto
32 return nil, err
33 }
34
34 - return nd, VerifyTrickleDagStructure(nd, ds, dbp.Maxlinks, layerRepeat)
35 + pbnd, ok := nd.(*merkledag.ProtoNode)
36 + if !ok {
37 + return nil, merkledag.ErrNotProtobuf
38 + }
39 +
40 + return pbnd, VerifyTrickleDagStructure(pbnd, ds, dbp.Maxlinks, layerRepeat)
41 }
42
43 //Test where calls to read are smaller than the chunk size
importer/trickle/trickledag.go
+16 -4
@@ -8,6 +8,8 @@ import (
8 h "github.com/ipfs/go-ipfs/importer/helpers"
9 dag "github.com/ipfs/go-ipfs/merkledag"
10 ft "github.com/ipfs/go-ipfs/unixfs"
11 +
12 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
13 )
14
15 // layerRepeat specifies how many times to append a child tree of a
@@ -15,7 +17,7 @@ import (
17 // improves seek speeds.
18 const layerRepeat = 4
19
18 -func TrickleLayout(db *h.DagBuilderHelper) (*dag.ProtoNode, error) {
20 +func TrickleLayout(db *h.DagBuilderHelper) (node.Node, error) {
21 root := h.NewUnixfsNode()
22 if err := db.FillNodeLayer(root); err != nil {
23 return nil, err
@@ -66,7 +68,12 @@ func fillTrickleRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int) error
68 }
69
70 // TrickleAppend appends the data in `db` to the dag, using the Trickledag format
69 -func TrickleAppend(ctx context.Context, base *dag.ProtoNode, db *h.DagBuilderHelper) (out *dag.ProtoNode, err_out error) {
71 +func TrickleAppend(ctx context.Context, basen node.Node, db *h.DagBuilderHelper) (out node.Node, err_out error) {
72 + base, ok := basen.(*dag.ProtoNode)
73 + if !ok {
74 + return nil, dag.ErrNotProtobuf
75 + }
76 +
77 defer func() {
78 if err_out == nil {
79 if err := db.Close(); err != nil {
@@ -229,8 +236,13 @@ func trickleDepthInfo(node *h.UnixfsNode, maxlinks int) (int, int) {
236
237 // VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure
238 // layout
232 -func VerifyTrickleDagStructure(nd *dag.ProtoNode, ds dag.DAGService, direct int, layerRepeat int) error {
233 - return verifyTDagRec(nd, -1, direct, layerRepeat, ds)
239 +func VerifyTrickleDagStructure(nd node.Node, ds dag.DAGService, direct int, layerRepeat int) error {
240 + pbnd, ok := nd.(*dag.ProtoNode)
241 + if !ok {
242 + return dag.ErrNotProtobuf
243 + }
244 +
245 + return verifyTDagRec(pbnd, -1, direct, layerRepeat, ds)
246 }
247
248 // Recursive call for verifying the structure of a trickledag
merkledag/merkledag.go
+16 -22
@@ -85,23 +85,29 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) {
85 return nil, fmt.Errorf("Failed to get block for %s: %v", c, err)
86 }
87
88 - var res node.Node
88 + return decodeBlock(b)
89 +}
90 +
91 +func decodeBlock(b blocks.Block) (node.Node, error) {
92 + c := b.Cid()
93 +
94 switch c.Type() {
95 case cid.Protobuf:
91 - out, err := DecodeProtobuf(b.RawData())
96 + decnd, err := DecodeProtobuf(b.RawData())
97 if err != nil {
98 if strings.Contains(err.Error(), "Unmarshal failed") {
99 return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c)
100 }
101 return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
102 }
98 - out.cached = c
99 - res = out
103 +
104 + decnd.cached = b.Cid()
105 + return decnd, nil
106 + case cid.Raw:
107 + return NewRawNode(b.RawData()), nil
108 default:
101 - return nil, fmt.Errorf("unrecognized formatting type")
109 + return nil, fmt.Errorf("unrecognized object type: %s", c.Type())
110 }
103 -
104 - return res, nil
111 }
112
113 func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) {
@@ -164,24 +170,12 @@ func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *Node
170 return
171 }
172
167 - c := b.Cid()
168 -
169 - var nd node.Node
170 - switch c.Type() {
171 - case cid.Protobuf:
172 - decnd, err := DecodeProtobuf(b.RawData())
173 - if err != nil {
174 - out <- &NodeOption{Err: err}
175 - return
176 - }
177 - decnd.cached = b.Cid()
178 - nd = decnd
179 - default:
180 - out <- &NodeOption{Err: fmt.Errorf("unrecognized object type: %s", c.Type())}
173 + nd, err := decodeBlock(b)
174 + if err != nil {
175 + out <- &NodeOption{Err: err}
176 return
177 }
178
184 - // buffered, no need to select
179 out <- &NodeOption{Node: nd}
180 count++
181
merkledag/merkledag_test.go
+78
@@ -373,3 +373,81 @@ func TestBasicAddGet(t *testing.T) {
373 t.Fatal("output didnt match input")
374 }
375 }
376 +
377 +func TestGetRawNodes(t *testing.T) {
378 + rn := NewRawNode([]byte("test"))
379 +
380 + ds := dstest.Mock()
381 +
382 + c, err := ds.Add(rn)
383 + if err != nil {
384 + t.Fatal(err)
385 + }
386 +
387 + if !c.Equals(rn.Cid()) {
388 + t.Fatal("output cids didnt match")
389 + }
390 +
391 + out, err := ds.Get(context.TODO(), c)
392 + if err != nil {
393 + t.Fatal(err)
394 + }
395 +
396 + if !bytes.Equal(out.RawData(), []byte("test")) {
397 + t.Fatal("raw block should match input data")
398 + }
399 +
400 + if out.Links() != nil {
401 + t.Fatal("raw blocks shouldnt have links")
402 + }
403 +
404 + if out.Tree() != nil {
405 + t.Fatal("tree should return no paths in a raw block")
406 + }
407 +
408 + size, err := out.Size()
409 + if err != nil {
410 + t.Fatal(err)
411 + }
412 + if size != 4 {
413 + t.Fatal("expected size to be 4")
414 + }
415 +
416 + ns, err := out.Stat()
417 + if err != nil {
418 + t.Fatal(err)
419 + }
420 +
421 + if ns.DataSize != 4 {
422 + t.Fatal("expected size to be 4, got: ", ns.DataSize)
423 + }
424 +
425 + _, _, err = out.Resolve([]string{"foo"})
426 + if err != ErrLinkNotFound {
427 + t.Fatal("shouldnt find links under raw blocks")
428 + }
429 +}
430 +
431 +func TestProtoNodeResolve(t *testing.T) {
432 +
433 + nd := new(ProtoNode)
434 + nd.SetLinks([]*node.Link{{Name: "foo"}})
435 +
436 + lnk, left, err := nd.Resolve([]string{"foo", "bar"})
437 + if err != nil {
438 + t.Fatal(err)
439 + }
440 +
441 + if len(left) != 1 || left[0] != "bar" {
442 + t.Fatal("expected the single path element 'bar' to remain")
443 + }
444 +
445 + if lnk.Name != "foo" {
446 + t.Fatal("how did we get anything else?")
447 + }
448 +
449 + tvals := nd.Tree()
450 + if len(tvals) != 1 || tvals[0] != "foo" {
451 + t.Fatal("expected tree to return []{\"foo\"}")
452 + }
453 +}
merkledag/node.go
+1 -1
@@ -36,7 +36,7 @@ func NodeWithData(d []byte) *ProtoNode {
36 }
37
38 // AddNodeLink adds a link to another node.
39 -func (n *ProtoNode) AddNodeLink(name string, that *ProtoNode) error {
39 +func (n *ProtoNode) AddNodeLink(name string, that node.Node) error {
40 n.encoded = nil
41
42 lnk, err := node.MakeLink(that)
merkledag/raw.go new
+46
@@ -0,0 +1,46 @@
1 +package merkledag
2 +
3 +import (
4 + "github.com/ipfs/go-ipfs/blocks"
5 +
6 + cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
7 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
8 + u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
9 +)
10 +
11 +type RawNode struct {
12 + blocks.Block
13 +}
14 +
15 +func NewRawNode(data []byte) *RawNode {
16 + h := u.Hash(data)
17 + c := cid.NewCidV1(cid.Raw, h)
18 + blk, _ := blocks.NewBlockWithCid(data, c)
19 +
20 + return &RawNode{blk}
21 +}
22 +
23 +func (rn *RawNode) Links() []*node.Link {
24 + return nil
25 +}
26 +
27 +func (rn *RawNode) Resolve(path []string) (*node.Link, []string, error) {
28 + return nil, nil, ErrLinkNotFound
29 +}
30 +
31 +func (rn *RawNode) Tree() []string {
32 + return nil
33 +}
34 +
35 +func (rn *RawNode) Size() (uint64, error) {
36 + return uint64(len(rn.RawData())), nil
37 +}
38 +
39 +func (rn *RawNode) Stat() (*node.NodeStat, error) {
40 + return &node.NodeStat{
41 + CumulativeSize: len(rn.RawData()),
42 + DataSize: len(rn.RawData()),
43 + }, nil
44 +}
45 +
46 +var _ node.Node = (*RawNode)(nil)
merkledag/utils/utils.go
+8 -7
@@ -1,17 +1,18 @@
1 package dagutils
2
3 import (
4 + "context"
5 "errors"
6
6 - context "context"
7 - ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
8 - syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
9 -
7 bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
8 bserv "github.com/ipfs/go-ipfs/blockservice"
9 offline "github.com/ipfs/go-ipfs/exchange/offline"
10 dag "github.com/ipfs/go-ipfs/merkledag"
11 path "github.com/ipfs/go-ipfs/path"
12 +
13 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
14 + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
15 + syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
16 )
17
18 type Editor struct {
@@ -50,7 +51,7 @@ func (e *Editor) GetDagService() dag.DAGService {
51 return e.tmp
52 }
53
53 -func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childname string, childnd *dag.ProtoNode) (*dag.ProtoNode, error) {
54 +func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childname string, childnd node.Node) (*dag.ProtoNode, error) {
55 if childname == "" {
56 return nil, errors.New("cannot create link with no name!")
57 }
@@ -76,7 +77,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childn
77 return root, nil
78 }
79
79 -func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert *dag.ProtoNode, create func() *dag.ProtoNode) error {
80 +func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert node.Node, create func() *dag.ProtoNode) error {
81 splpath := path.SplitList(pth)
82 nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
83 if err != nil {
@@ -86,7 +87,7 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert *dag
87 return nil
88 }
89
89 -func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert *dag.ProtoNode, create func() *dag.ProtoNode) (*dag.ProtoNode, error) {
90 +func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert node.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) {
91 if len(path) == 1 {
92 return addLink(ctx, e.tmp, root, path[0], toinsert)
93 }
mfs/dir.go
+4 -3
@@ -1,6 +1,7 @@
1 package mfs
2
3 import (
4 + "context"
5 "errors"
6 "fmt"
7 "os"
@@ -9,11 +10,11 @@ import (
10 "sync"
11 "time"
12
12 - context "context"
13 -
13 dag "github.com/ipfs/go-ipfs/merkledag"
14 ft "github.com/ipfs/go-ipfs/unixfs"
15 ufspb "github.com/ipfs/go-ipfs/unixfs/pb"
16 +
17 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
18 )
19
20 var ErrNotYetImplemented = errors.New("not yet implemented")
@@ -323,7 +324,7 @@ func (d *Directory) Flush() error {
324 }
325
326 // AddChild adds the node 'nd' under this directory giving it the name 'name'
326 -func (d *Directory) AddChild(name string, nd *dag.ProtoNode) error {
327 +func (d *Directory) AddChild(name string, nd node.Node) error {
328 d.lock.Lock()
329 defer d.lock.Unlock()
330
mfs/mfs_test.go
+8 -3
@@ -42,12 +42,12 @@ func getDagserv(t *testing.T) dag.DAGService {
42 return dag.NewDAGService(blockserv)
43 }
44
45 -func getRandFile(t *testing.T, ds dag.DAGService, size int64) *dag.ProtoNode {
45 +func getRandFile(t *testing.T, ds dag.DAGService, size int64) node.Node {
46 r := io.LimitReader(u.NewTimeSeededRand(), size)
47 return fileNodeFromReader(t, ds, r)
48 }
49
50 -func fileNodeFromReader(t *testing.T, ds dag.DAGService, r io.Reader) *dag.ProtoNode {
50 +func fileNodeFromReader(t *testing.T, ds dag.DAGService, r io.Reader) node.Node {
51 nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r))
52 if err != nil {
53 t.Fatal(err)
@@ -125,7 +125,12 @@ func compStrArrs(a, b []string) bool {
125 return true
126 }
127
128 -func assertFileAtPath(ds dag.DAGService, root *Directory, exp *dag.ProtoNode, pth string) error {
128 +func assertFileAtPath(ds dag.DAGService, root *Directory, expn node.Node, pth string) error {
129 + exp, ok := expn.(*dag.ProtoNode)
130 + if !ok {
131 + return dag.ErrNotProtobuf
132 + }
133 +
134 parts := path.SplitList(pth)
135 cur := root
136 for i, d := range parts[:len(parts)-1] {
mfs/ops.go
+3 -2
@@ -7,8 +7,9 @@ import (
7 gopath "path"
8 "strings"
9
10 - dag "github.com/ipfs/go-ipfs/merkledag"
10 path "github.com/ipfs/go-ipfs/path"
11 +
12 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
13 )
14
15 // Mv moves the file or directory at 'src' to 'dst'
@@ -87,7 +88,7 @@ func lookupDir(r *Root, path string) (*Directory, error) {
88 }
89
90 // PutNode inserts 'nd' at 'path' in the given mfs
90 -func PutNode(r *Root, path string, nd *dag.ProtoNode) error {
91 +func PutNode(r *Root, path string, nd node.Node) error {
92 dirp, filename := gopath.Split(path)
93 if filename == "" {
94 return fmt.Errorf("cannot create file with empty name")
unixfs/io/dagreader.go
+38 -28
@@ -2,17 +2,18 @@ package io
2
3 import (
4 "bytes"
5 + "context"
6 "errors"
7 "fmt"
8 "io"
9 "os"
10
10 - "context"
11 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
12 -
11 mdag "github.com/ipfs/go-ipfs/merkledag"
12 ft "github.com/ipfs/go-ipfs/unixfs"
13 ftpb "github.com/ipfs/go-ipfs/unixfs/pb"
14 +
15 + proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
16 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
17 )
18
19 var ErrIsDir = errors.New("this dag node is a directory")
@@ -58,36 +59,45 @@ type ReadSeekCloser interface {
59
60 // NewDagReader creates a new reader object that reads the data represented by
61 // the given node, using the passed in DAGService for data retreival
61 -func NewDagReader(ctx context.Context, n *mdag.ProtoNode, serv mdag.DAGService) (*DagReader, error) {
62 - pb := new(ftpb.Data)
63 - if err := proto.Unmarshal(n.Data(), pb); err != nil {
64 - return nil, err
65 - }
66 -
67 - switch pb.GetType() {
68 - case ftpb.Data_Directory:
69 - // Dont allow reading directories
70 - return nil, ErrIsDir
71 - case ftpb.Data_File, ftpb.Data_Raw:
72 - return NewDataFileReader(ctx, n, pb, serv), nil
73 - case ftpb.Data_Metadata:
74 - if len(n.Links()) == 0 {
75 - return nil, errors.New("incorrectly formatted metadata object")
76 - }
77 - child, err := n.Links()[0].GetNode(ctx, serv)
78 - if err != nil {
62 +func NewDagReader(ctx context.Context, n node.Node, serv mdag.DAGService) (*DagReader, error) {
63 + switch n := n.(type) {
64 + case *mdag.RawNode:
65 + return &DagReader{
66 + buf: NewRSNCFromBytes(n.RawData()),
67 + }, nil
68 + case *mdag.ProtoNode:
69 + pb := new(ftpb.Data)
70 + if err := proto.Unmarshal(n.Data(), pb); err != nil {
71 return nil, err
72 }
73
82 - childpb, ok := child.(*mdag.ProtoNode)
83 - if !ok {
84 - return nil, mdag.ErrNotProtobuf
74 + switch pb.GetType() {
75 + case ftpb.Data_Directory:
76 + // Dont allow reading directories
77 + return nil, ErrIsDir
78 + case ftpb.Data_File, ftpb.Data_Raw:
79 + return NewDataFileReader(ctx, n, pb, serv), nil
80 + case ftpb.Data_Metadata:
81 + if len(n.Links()) == 0 {
82 + return nil, errors.New("incorrectly formatted metadata object")
83 + }
84 + child, err := n.Links()[0].GetNode(ctx, serv)
85 + if err != nil {
86 + return nil, err
87 + }
88 +
89 + childpb, ok := child.(*mdag.ProtoNode)
90 + if !ok {
91 + return nil, mdag.ErrNotProtobuf
92 + }
93 + return NewDagReader(ctx, childpb, serv)
94 + case ftpb.Data_Symlink:
95 + return nil, ErrCantReadSymlinks
96 + default:
97 + return nil, ft.ErrUnrecognizedType
98 }
86 - return NewDagReader(ctx, childpb, serv)
87 - case ftpb.Data_Symlink:
88 - return nil, ErrCantReadSymlinks
99 default:
90 - return nil, ft.ErrUnrecognizedType
100 + return nil, fmt.Errorf("unrecognized node type")
101 }
102 }
103
unixfs/mod/dagmodifier.go
+23 -6
@@ -2,6 +2,7 @@ package mod
2
3 import (
4 "bytes"
5 + "context"
6 "errors"
7 "io"
8 "os"
@@ -13,10 +14,10 @@ import (
14 ft "github.com/ipfs/go-ipfs/unixfs"
15 uio "github.com/ipfs/go-ipfs/unixfs/io"
16
16 - context "context"
17 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
18 cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
19 proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
20 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
21 )
22
23 var ErrSeekFail = errors.New("failed to seek properly")
@@ -45,9 +46,14 @@ type DagModifier struct {
46 read *uio.DagReader
47 }
48
48 -func NewDagModifier(ctx context.Context, from *mdag.ProtoNode, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) {
49 +func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) {
50 + pbn, ok := from.(*mdag.ProtoNode)
51 + if !ok {
52 + return nil, mdag.ErrNotProtobuf
53 + }
54 +
55 return &DagModifier{
50 - curNode: from.Copy(),
56 + curNode: pbn.Copy(),
57 dagserv: serv,
58 splitter: spl,
59 ctx: ctx,
@@ -109,7 +115,13 @@ func (dm *DagModifier) expandSparse(size int64) error {
115 if err != nil {
116 return err
117 }
112 - dm.curNode = nnode
118 +
119 + pbnnode, ok := nnode.(*mdag.ProtoNode)
120 + if !ok {
121 + return mdag.ErrNotProtobuf
122 + }
123 +
124 + dm.curNode = pbnnode
125 return nil
126 }
127
@@ -197,7 +209,12 @@ func (dm *DagModifier) Sync() error {
209 return err
210 }
211
200 - dm.curNode = nd
212 + pbnode, ok := nd.(*mdag.ProtoNode)
213 + if !ok {
214 + return mdag.ErrNotProtobuf
215 + }
216 +
217 + dm.curNode = pbnode
218 }
219
220 dm.writeStart += uint64(buflen)
@@ -288,7 +305,7 @@ func (dm *DagModifier) modifyDag(node *mdag.ProtoNode, offset uint64, data io.Re
305 }
306
307 // appendData appends the blocks from the given chan to the end of this dag
291 -func (dm *DagModifier) appendData(node *mdag.ProtoNode, spl chunk.Splitter) (*mdag.ProtoNode, error) {
308 +func (dm *DagModifier) appendData(node *mdag.ProtoNode, spl chunk.Splitter) (node.Node, error) {
309 dbp := &help.DagBuilderParams{
310 Dagserv: dm.dagserv,
311 Maxlinks: help.DefaultLinksPerBlock,
unixfs/test/utils.go
+5 -4
@@ -2,6 +2,7 @@ package testu
2
3 import (
4 "bytes"
5 + "context"
6 "fmt"
7 "io"
8 "io/ioutil"
@@ -13,7 +14,7 @@ import (
14 mdagmock "github.com/ipfs/go-ipfs/merkledag/test"
15 ft "github.com/ipfs/go-ipfs/unixfs"
16
16 - context "context"
17 + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
18 u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
19 )
20
@@ -27,7 +28,7 @@ func GetDAGServ() mdag.DAGService {
28 return mdagmock.Mock()
29 }
30
30 -func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) *mdag.ProtoNode {
31 +func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) node.Node {
32 in := bytes.NewReader(data)
33 node, err := imp.BuildTrickleDagFromReader(dserv, SizeSplitterGen(500)(in))
34 if err != nil {
@@ -37,11 +38,11 @@ func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) *mdag.ProtoNode {
38 return node
39 }
40
40 -func GetEmptyNode(t testing.TB, dserv mdag.DAGService) *mdag.ProtoNode {
41 +func GetEmptyNode(t testing.TB, dserv mdag.DAGService) node.Node {
42 return GetNode(t, dserv, []byte{})
43 }
44
44 -func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, *mdag.ProtoNode) {
45 +func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, node.Node) {
46 in := io.LimitReader(u.NewTimeSeededRand(), size)
47 buf, err := ioutil.ReadAll(in)
48 if err != nil {