@cryptotaxi247 / kubo / commits / 7302c3ab6

Provide support for raw leaves in DAG modifier.

License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>

Kevin Atkinson committed Aug 8, 2017 at 01:56 UTC 7302c3ab66127aa7ef26cae3d4fe6e7caf8b027a
5 files changed +238 -143
importer/helpers/dagbuilder.go
+3 -3
@@ -120,8 +120,8 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode {
120 return n
121 }
122
123 -// NewUnixfsBlock creates a new Unixfs node to represent a raw data block
124 -func (db *DagBuilderHelper) NewUnixfsBlock() *UnixfsNode {
123 +// newUnixfsBlock creates a new Unixfs node to represent a raw data block
124 +func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode {
125 n := &UnixfsNode{
126 node: new(dag.ProtoNode),
127 ufmt: &ft.FSNode{Type: ft.TRaw},
@@ -181,7 +181,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
181 }, nil
182 }
183 } else {
184 - blk := db.NewUnixfsBlock()
184 + blk := db.newUnixfsBlock()
185 blk.SetData(data)
186 return blk, nil
187 }
unixfs/io/dagreader_test.go
+6 -6
@@ -17,7 +17,7 @@ import (
17
18 func TestBasicRead(t *testing.T) {
19 dserv := testu.GetDAGServ()
20 - inbuf, node := testu.GetRandomNode(t, dserv, 1024)
20 + inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.ProtoBufLeaves)
21 ctx, closer := context.WithCancel(context.Background())
22 defer closer()
23
@@ -44,7 +44,7 @@ func TestSeekAndRead(t *testing.T) {
44 inbuf[i] = byte(i)
45 }
46
47 - node := testu.GetNode(t, dserv, inbuf)
47 + node := testu.GetNode(t, dserv, inbuf, testu.ProtoBufLeaves)
48 ctx, closer := context.WithCancel(context.Background())
49 defer closer()
50
@@ -84,7 +84,7 @@ func TestRelativeSeek(t *testing.T) {
84 }
85
86 inbuf[1023] = 1 // force the reader to be 1024 bytes
87 - node := testu.GetNode(t, dserv, inbuf)
87 + node := testu.GetNode(t, dserv, inbuf, testu.ProtoBufLeaves)
88
89 reader, err := NewDagReader(ctx, node, dserv)
90 if err != nil {
@@ -160,7 +160,7 @@ func TestBadPBData(t *testing.T) {
160
161 func TestMetadataNode(t *testing.T) {
162 dserv := testu.GetDAGServ()
163 - rdata, rnode := testu.GetRandomNode(t, dserv, 512)
163 + rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.ProtoBufLeaves)
164 _, err := dserv.Add(rnode)
165 if err != nil {
166 t.Fatal(err)
@@ -203,7 +203,7 @@ func TestMetadataNode(t *testing.T) {
203
204 func TestWriteTo(t *testing.T) {
205 dserv := testu.GetDAGServ()
206 - inbuf, node := testu.GetRandomNode(t, dserv, 1024)
206 + inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.ProtoBufLeaves)
207 ctx, closer := context.WithCancel(context.Background())
208 defer closer()
209
@@ -225,7 +225,7 @@ func TestWriteTo(t *testing.T) {
225 func TestReaderSzie(t *testing.T) {
226 dserv := testu.GetDAGServ()
227 size := int64(1024)
228 - _, node := testu.GetRandomNode(t, dserv, size)
228 + _, node := testu.GetRandomNode(t, dserv, size, testu.ProtoBufLeaves)
229 ctx, closer := context.WithCancel(context.Background())
230 defer closer()
231
unixfs/mod/dagmodifier.go
+119 -103
@@ -40,6 +40,8 @@ type DagModifier struct {
40 curWrOff uint64
41 wrBuf *bytes.Buffer
42
43 + RawLeaves bool
44 +
45 read uio.DagReader
46 }
47
@@ -113,17 +115,7 @@ func (dm *DagModifier) expandSparse(size int64) error {
115 return err
116 }
117 _, err = dm.dagserv.Add(nnode)
116 - if err != nil {
117 - return err
118 - }
119 -
120 - pbnnode, ok := nnode.(*mdag.ProtoNode)
121 - if !ok {
122 - return mdag.ErrNotProtobuf
123 - }
124 -
125 - dm.curNode = pbnnode
126 - return nil
118 + return err
119 }
120
121 // Write continues writing to the dag at the current offset
@@ -149,26 +141,28 @@ func (dm *DagModifier) Write(b []byte) (int, error) {
141 return n, nil
142 }
143
152 -var ErrNoRawYet = fmt.Errorf("currently only fully support protonodes in the dagmodifier")
153 -
144 // Size returns the Filesize of the node
145 func (dm *DagModifier) Size() (int64, error) {
156 - switch nd := dm.curNode.(type) {
146 + fileSize, err := fileSize(dm.curNode)
147 + if err != nil {
148 + return 0, err
149 + }
150 + if dm.wrBuf != nil && int64(dm.wrBuf.Len())+int64(dm.writeStart) > int64(fileSize) {
151 + return int64(dm.wrBuf.Len()) + int64(dm.writeStart), nil
152 + }
153 + return int64(fileSize), nil
154 +}
155 +
156 +func fileSize(n node.Node) (uint64, error) {
157 + switch nd := n.(type) {
158 case *mdag.ProtoNode:
158 - pbn, err := ft.FromBytes(nd.Data())
159 + f, err := ft.FromBytes(nd.Data())
160 if err != nil {
161 return 0, err
162 }
162 - if dm.wrBuf != nil && uint64(dm.wrBuf.Len())+dm.writeStart > pbn.GetFilesize() {
163 - return int64(dm.wrBuf.Len()) + int64(dm.writeStart), nil
164 - }
165 - return int64(pbn.GetFilesize()), nil
163 + return f.GetFilesize(), nil
164 case *mdag.RawNode:
167 - if dm.wrBuf != nil {
168 - return 0, ErrNoRawYet
169 - }
170 - sz, err := nd.Size()
171 - return int64(sz), err
165 + return uint64(len(nd.RawData())), nil
166 default:
167 return 0, ErrNotUnixfs
168 }
@@ -196,36 +190,22 @@ func (dm *DagModifier) Sync() error {
190 return err
191 }
192
199 - nd, err := dm.dagserv.Get(dm.ctx, thisc)
193 + dm.curNode, err = dm.dagserv.Get(dm.ctx, thisc)
194 if err != nil {
195 return err
196 }
197
204 - pbnd, ok := nd.(*mdag.ProtoNode)
205 - if !ok {
206 - return mdag.ErrNotProtobuf
207 - }
208 -
209 - dm.curNode = pbnd
210 -
198 // need to write past end of current dag
199 if !done {
213 - nd, err := dm.appendData(dm.curNode, dm.splitter(dm.wrBuf))
200 + dm.curNode, err = dm.appendData(dm.curNode, dm.splitter(dm.wrBuf))
201 if err != nil {
202 return err
203 }
204
218 - _, err = dm.dagserv.Add(nd)
205 + _, err = dm.dagserv.Add(dm.curNode)
206 if err != nil {
207 return err
208 }
222 -
223 - pbnode, ok := nd.(*mdag.ProtoNode)
224 - if !ok {
225 - return mdag.ErrNotProtobuf
226 - }
227 -
228 - dm.curNode = pbnode
209 }
210
211 dm.writeStart += uint64(buflen)
@@ -238,43 +218,86 @@ func (dm *DagModifier) Sync() error {
218 // returns the new key of the passed in node and whether or not all the data in the reader
219 // has been consumed.
220 func (dm *DagModifier) modifyDag(n node.Node, offset uint64, data io.Reader) (*cid.Cid, bool, error) {
241 - node, ok := n.(*mdag.ProtoNode)
242 - if !ok {
243 - return nil, false, ErrNoRawYet
244 - }
221 + // If we've reached a leaf node.
222 + if len(n.Links()) == 0 {
223 + switch nd0 := n.(type) {
224 + case *mdag.ProtoNode:
225 + f, err := ft.FromBytes(nd0.Data())
226 + if err != nil {
227 + return nil, false, err
228 + }
229
246 - f, err := ft.FromBytes(node.Data())
247 - if err != nil {
248 - return nil, false, err
249 - }
230 + n, err := data.Read(f.Data[offset:])
231 + if err != nil && err != io.EOF {
232 + return nil, false, err
233 + }
234
251 - // If we've reached a leaf node.
252 - if len(node.Links()) == 0 {
253 - n, err := data.Read(f.Data[offset:])
254 - if err != nil && err != io.EOF {
255 - return nil, false, err
256 - }
235 + // Update newly written node..
236 + b, err := proto.Marshal(f)
237 + if err != nil {
238 + return nil, false, err
239 + }
240
258 - // Update newly written node..
259 - b, err := proto.Marshal(f)
260 - if err != nil {
261 - return nil, false, err
262 - }
241 + nd := new(mdag.ProtoNode)
242 + nd.SetData(b)
243 + k, err := dm.dagserv.Add(nd)
244 + if err != nil {
245 + return nil, false, err
246 + }
247
264 - nd := new(mdag.ProtoNode)
265 - nd.SetData(b)
266 - k, err := dm.dagserv.Add(nd)
267 - if err != nil {
268 - return nil, false, err
269 - }
248 + // Hey look! we're done!
249 + var done bool
250 + if n < len(f.Data[offset:]) {
251 + done = true
252 + }
253 +
254 + return k, done, nil
255 + case *mdag.RawNode:
256 + origData := nd0.RawData()
257 + bytes := make([]byte, len(origData))
258
271 - // Hey look! we're done!
272 - var done bool
273 - if n < len(f.Data[offset:]) {
274 - done = true
259 + // copy orig data up to offset
260 + copy(bytes, origData[:offset])
261 +
262 + // copy in new data
263 + n, err := data.Read(bytes[offset:])
264 + if err != nil && err != io.EOF {
265 + return nil, false, err
266 + }
267 +
268 + // copy remaining data
269 + offsetPlusN := int(offset) + n
270 + if offsetPlusN < len(origData) {
271 + copy(bytes[offsetPlusN:], origData[offsetPlusN:])
272 + }
273 +
274 + nd, err := mdag.NewRawNodeWPrefix(bytes, nd0.Cid().Prefix())
275 + if err != nil {
276 + return nil, false, err
277 + }
278 + k, err := dm.dagserv.Add(nd)
279 + if err != nil {
280 + return nil, false, err
281 + }
282 +
283 + // Hey look! we're done!
284 + var done bool
285 + if n < len(bytes[offset:]) {
286 + done = true
287 + }
288 +
289 + return k, done, nil
290 }
291 + }
292
277 - return k, done, nil
293 + node, ok := n.(*mdag.ProtoNode)
294 + if !ok {
295 + return nil, false, ErrNotUnixfs
296 + }
297 +
298 + f, err := ft.FromBytes(node.Data())
299 + if err != nil {
300 + return nil, false, err
301 }
302
303 var cur uint64
@@ -287,12 +310,7 @@ func (dm *DagModifier) modifyDag(n node.Node, offset uint64, data io.Reader) (*c
310 return nil, false, err
311 }
312
290 - childpb, ok := child.(*mdag.ProtoNode)
291 - if !ok {
292 - return nil, false, mdag.ErrNotProtobuf
293 - }
294 -
295 - k, sdone, err := dm.modifyDag(childpb, offset-cur, data)
313 + k, sdone, err := dm.modifyDag(child, offset-cur, data)
314 if err != nil {
315 return nil, false, err
316 }
@@ -323,14 +341,13 @@ func (dm *DagModifier) modifyDag(n node.Node, offset uint64, data io.Reader) (*c
341 // appendData appends the blocks from the given chan to the end of this dag
342 func (dm *DagModifier) appendData(nd node.Node, spl chunk.Splitter) (node.Node, error) {
343 switch nd := nd.(type) {
326 - case *mdag.ProtoNode:
344 + case *mdag.ProtoNode, *mdag.RawNode:
345 dbp := &help.DagBuilderParams{
328 - Dagserv: dm.dagserv,
329 - Maxlinks: help.DefaultLinksPerBlock,
346 + Dagserv: dm.dagserv,
347 + Maxlinks: help.DefaultLinksPerBlock,
348 + RawLeaves: dm.RawLeaves,
349 }
350 return trickle.TrickleAppend(dm.ctx, nd, dbp.New(spl))
332 - case *mdag.RawNode:
333 - return nil, fmt.Errorf("appending to raw node types not yet supported")
351 default:
352 return nil, ErrNotUnixfs
353 }
@@ -478,26 +495,30 @@ func (dm *DagModifier) Truncate(size int64) error {
495 }
496
497 // dagTruncate truncates the given node to 'size' and returns the modified Node
481 -func dagTruncate(ctx context.Context, n node.Node, size uint64, ds mdag.DAGService) (*mdag.ProtoNode, error) {
482 - nd, ok := n.(*mdag.ProtoNode)
483 - if !ok {
484 - return nil, ErrNoRawYet
485 - }
486 -
487 - if len(nd.Links()) == 0 {
488 - // TODO: this can likely be done without marshaling and remarshaling
489 - pbn, err := ft.FromBytes(nd.Data())
490 - if err != nil {
491 - return nil, err
498 +func dagTruncate(ctx context.Context, n node.Node, size uint64, ds mdag.DAGService) (node.Node, error) {
499 + if len(n.Links()) == 0 {
500 + switch nd := n.(type) {
501 + case *mdag.ProtoNode:
502 + // TODO: this can likely be done without marshaling and remarshaling
503 + pbn, err := ft.FromBytes(nd.Data())
504 + if err != nil {
505 + return nil, err
506 + }
507 + nd.SetData(ft.WrapData(pbn.Data[:size]))
508 + return nd, nil
509 + case *mdag.RawNode:
510 + return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix())
511 }
512 + }
513
494 - nd.SetData(ft.WrapData(pbn.Data[:size]))
495 - return nd, nil
514 + nd, ok := n.(*mdag.ProtoNode)
515 + if !ok {
516 + return nil, ErrNotUnixfs
517 }
518
519 var cur uint64
520 end := 0
500 - var modified *mdag.ProtoNode
521 + var modified node.Node
522 ndata := new(ft.FSNode)
523 for i, lnk := range nd.Links() {
524 child, err := lnk.GetNode(ctx, ds)
@@ -505,19 +526,14 @@ func dagTruncate(ctx context.Context, n node.Node, size uint64, ds mdag.DAGServi
526 return nil, err
527 }
528
508 - childpb, ok := child.(*mdag.ProtoNode)
509 - if !ok {
510 - return nil, err
511 - }
512 -
513 - childsize, err := ft.DataSize(childpb.Data())
529 + childsize, err := fileSize(child)
530 if err != nil {
531 return nil, err
532 }
533
534 // found the child we want to cut
535 if size < cur+childsize {
520 - nchild, err := dagTruncate(ctx, childpb, size-cur, ds)
536 + nchild, err := dagTruncate(ctx, child, size-cur, ds)
537 if err != nil {
538 return nil, err
539 }
unixfs/mod/dagmodifier_test.go
+88 -24
@@ -9,15 +9,14 @@ import (
9
10 h "github.com/ipfs/go-ipfs/importer/helpers"
11 trickle "github.com/ipfs/go-ipfs/importer/trickle"
12 - mdag "github.com/ipfs/go-ipfs/merkledag"
13 - ft "github.com/ipfs/go-ipfs/unixfs"
12 +
13 uio "github.com/ipfs/go-ipfs/unixfs/io"
14 testu "github.com/ipfs/go-ipfs/unixfs/test"
15
16 u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
17 )
18
20 -func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) []byte {
19 +func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, rawLeaves testu.UseRawLeaves) []byte {
20 newdata := make([]byte, size)
21 r := u.NewTimeSeededRand()
22 r.Read(newdata)
@@ -45,9 +44,10 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier)
44 Getter: dm.dagserv,
45 Direct: h.DefaultLinksPerBlock,
46 LayerRepeat: 4,
47 + RawLeaves: bool(rawLeaves),
48 })
49 if err != nil {
50 - t.Fatal(err)
50 + t.Error(err)
51 }
52
53 rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv)
@@ -67,9 +67,17 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier)
67 return orig
68 }
69
70 +func runBothSubtests(t *testing.T, tfunc func(*testing.T, testu.UseRawLeaves)) {
71 + t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, testu.ProtoBufLeaves) })
72 + t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, testu.RawLeaves) })
73 +}
74 +
75 func TestDagModifierBasic(t *testing.T) {
76 + runBothSubtests(t, testDagModifierBasic)
77 +}
78 +func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
79 dserv := testu.GetDAGServ()
72 - b, n := testu.GetRandomNode(t, dserv, 50000)
80 + b, n := testu.GetRandomNode(t, dserv, 50000, rawLeaves)
81 ctx, cancel := context.WithCancel(context.Background())
82 defer cancel()
83
@@ -77,32 +85,33 @@ func TestDagModifierBasic(t *testing.T) {
85 if err != nil {
86 t.Fatal(err)
87 }
88 + dagmod.RawLeaves = bool(rawLeaves)
89
90 // Within zero block
91 beg := uint64(15)
92 length := uint64(60)
93
94 t.Log("Testing mod within zero block")
86 - b = testModWrite(t, beg, length, b, dagmod)
95 + b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
96
97 // Within bounds of existing file
98 beg = 1000
99 length = 4000
100 t.Log("Testing mod within bounds of existing multiblock file.")
92 - b = testModWrite(t, beg, length, b, dagmod)
101 + b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
102
103 // Extend bounds
104 beg = 49500
105 length = 4000
106
107 t.Log("Testing mod that extends file.")
99 - b = testModWrite(t, beg, length, b, dagmod)
108 + b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
109
110 // "Append"
111 beg = uint64(len(b))
112 length = 3000
113 t.Log("Testing pure append")
105 - _ = testModWrite(t, beg, length, b, dagmod)
114 + _ = testModWrite(t, beg, length, b, dagmod, rawLeaves)
115
116 // Verify reported length
117 node, err := dagmod.GetNode()
@@ -110,7 +119,7 @@ func TestDagModifierBasic(t *testing.T) {
119 t.Fatal(err)
120 }
121
113 - size, err := ft.DataSize(node.(*mdag.ProtoNode).Data())
122 + size, err := fileSize(node)
123 if err != nil {
124 t.Fatal(err)
125 }
@@ -122,8 +131,11 @@ func TestDagModifierBasic(t *testing.T) {
131 }
132
133 func TestMultiWrite(t *testing.T) {
134 + runBothSubtests(t, testMultiWrite)
135 +}
136 +func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
137 dserv := testu.GetDAGServ()
126 - n := testu.GetEmptyNode(t, dserv)
138 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
139
140 ctx, cancel := context.WithCancel(context.Background())
141 defer cancel()
@@ -132,6 +144,7 @@ func TestMultiWrite(t *testing.T) {
144 if err != nil {
145 t.Fatal(err)
146 }
147 + dagmod.RawLeaves = bool(rawLeaves)
148
149 data := make([]byte, 4000)
150 u.NewTimeSeededRand().Read(data)
@@ -175,8 +188,11 @@ func TestMultiWrite(t *testing.T) {
188 }
189
190 func TestMultiWriteAndFlush(t *testing.T) {
191 + runBothSubtests(t, testMultiWriteAndFlush)
192 +}
193 +func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
194 dserv := testu.GetDAGServ()
179 - n := testu.GetEmptyNode(t, dserv)
195 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
196
197 ctx, cancel := context.WithCancel(context.Background())
198 defer cancel()
@@ -185,6 +201,7 @@ func TestMultiWriteAndFlush(t *testing.T) {
201 if err != nil {
202 t.Fatal(err)
203 }
204 + dagmod.RawLeaves = bool(rawLeaves)
205
206 data := make([]byte, 20)
207 u.NewTimeSeededRand().Read(data)
@@ -223,8 +240,11 @@ func TestMultiWriteAndFlush(t *testing.T) {
240 }
241
242 func TestWriteNewFile(t *testing.T) {
243 + runBothSubtests(t, testWriteNewFile)
244 +}
245 +func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
246 dserv := testu.GetDAGServ()
227 - n := testu.GetEmptyNode(t, dserv)
247 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
248
249 ctx, cancel := context.WithCancel(context.Background())
250 defer cancel()
@@ -233,6 +253,7 @@ func TestWriteNewFile(t *testing.T) {
253 if err != nil {
254 t.Fatal(err)
255 }
256 + dagmod.RawLeaves = bool(rawLeaves)
257
258 towrite := make([]byte, 2000)
259 u.NewTimeSeededRand().Read(towrite)
@@ -266,8 +287,11 @@ func TestWriteNewFile(t *testing.T) {
287 }
288
289 func TestMultiWriteCoal(t *testing.T) {
290 + runBothSubtests(t, testMultiWriteCoal)
291 +}
292 +func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
293 dserv := testu.GetDAGServ()
270 - n := testu.GetEmptyNode(t, dserv)
294 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
295
296 ctx, cancel := context.WithCancel(context.Background())
297 defer cancel()
@@ -276,6 +300,7 @@ func TestMultiWriteCoal(t *testing.T) {
300 if err != nil {
301 t.Fatal(err)
302 }
303 + dagmod.RawLeaves = bool(rawLeaves)
304
305 data := make([]byte, 1000)
306 u.NewTimeSeededRand().Read(data)
@@ -300,6 +325,8 @@ func TestMultiWriteCoal(t *testing.T) {
325 if err != nil {
326 t.Fatal(err)
327 }
328 + dagmod.RawLeaves = bool(rawLeaves)
329 +
330 rbuf, err := ioutil.ReadAll(read)
331 if err != nil {
332 t.Fatal(err)
@@ -312,8 +339,11 @@ func TestMultiWriteCoal(t *testing.T) {
339 }
340
341 func TestLargeWriteChunks(t *testing.T) {
342 + runBothSubtests(t, testLargeWriteChunks)
343 +}
344 +func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
345 dserv := testu.GetDAGServ()
316 - n := testu.GetEmptyNode(t, dserv)
346 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
347
348 ctx, cancel := context.WithCancel(context.Background())
349 defer cancel()
@@ -322,6 +352,7 @@ func TestLargeWriteChunks(t *testing.T) {
352 if err != nil {
353 t.Fatal(err)
354 }
355 + dagmod.RawLeaves = bool(rawLeaves)
356
357 wrsize := 1000
358 datasize := 10000000
@@ -351,8 +382,11 @@ func TestLargeWriteChunks(t *testing.T) {
382 }
383
384 func TestDagTruncate(t *testing.T) {
385 + runBothSubtests(t, testDagTruncate)
386 +}
387 +func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
388 dserv := testu.GetDAGServ()
355 - b, n := testu.GetRandomNode(t, dserv, 50000)
389 + b, n := testu.GetRandomNode(t, dserv, 50000, rawLeaves)
390 ctx, cancel := context.WithCancel(context.Background())
391 defer cancel()
392
@@ -360,6 +394,7 @@ func TestDagTruncate(t *testing.T) {
394 if err != nil {
395 t.Fatal(err)
396 }
397 + dagmod.RawLeaves = bool(rawLeaves)
398
399 err = dagmod.Truncate(12345)
400 if err != nil {
@@ -418,8 +453,11 @@ func TestDagTruncate(t *testing.T) {
453 }
454
455 func TestSparseWrite(t *testing.T) {
456 + runBothSubtests(t, testSparseWrite)
457 +}
458 +func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
459 dserv := testu.GetDAGServ()
422 - n := testu.GetEmptyNode(t, dserv)
460 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
461 ctx, cancel := context.WithCancel(context.Background())
462 defer cancel()
463
@@ -427,6 +465,7 @@ func TestSparseWrite(t *testing.T) {
465 if err != nil {
466 t.Fatal(err)
467 }
468 + dagmod.RawLeaves = bool(rawLeaves)
469
470 buf := make([]byte, 5000)
471 u.NewTimeSeededRand().Read(buf[2500:])
@@ -456,8 +495,11 @@ func TestSparseWrite(t *testing.T) {
495 }
496
497 func TestSeekPastEndWrite(t *testing.T) {
498 + runBothSubtests(t, testSeekPastEndWrite)
499 +}
500 +func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
501 dserv := testu.GetDAGServ()
460 - n := testu.GetEmptyNode(t, dserv)
502 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
503 ctx, cancel := context.WithCancel(context.Background())
504 defer cancel()
505
@@ -465,6 +507,7 @@ func TestSeekPastEndWrite(t *testing.T) {
507 if err != nil {
508 t.Fatal(err)
509 }
510 + dagmod.RawLeaves = bool(rawLeaves)
511
512 buf := make([]byte, 5000)
513 u.NewTimeSeededRand().Read(buf[2500:])
@@ -503,8 +546,11 @@ func TestSeekPastEndWrite(t *testing.T) {
546 }
547
548 func TestRelativeSeek(t *testing.T) {
549 + runBothSubtests(t, testRelativeSeek)
550 +}
551 +func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
552 dserv := testu.GetDAGServ()
507 - n := testu.GetEmptyNode(t, dserv)
553 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
554 ctx, cancel := context.WithCancel(context.Background())
555 defer cancel()
556
@@ -512,6 +558,7 @@ func TestRelativeSeek(t *testing.T) {
558 if err != nil {
559 t.Fatal(err)
560 }
561 + dagmod.RawLeaves = bool(rawLeaves)
562
563 for i := 0; i < 64; i++ {
564 dagmod.Write([]byte{byte(i)})
@@ -533,8 +580,11 @@ func TestRelativeSeek(t *testing.T) {
580 }
581
582 func TestInvalidSeek(t *testing.T) {
583 + runBothSubtests(t, testInvalidSeek)
584 +}
585 +func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
586 dserv := testu.GetDAGServ()
537 - n := testu.GetEmptyNode(t, dserv)
587 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
588 ctx, cancel := context.WithCancel(context.Background())
589 defer cancel()
590
@@ -542,6 +592,8 @@ func TestInvalidSeek(t *testing.T) {
592 if err != nil {
593 t.Fatal(err)
594 }
595 + dagmod.RawLeaves = bool(rawLeaves)
596 +
597 _, err = dagmod.Seek(10, -10)
598
599 if err != ErrUnrecognizedWhence {
@@ -550,9 +602,12 @@ func TestInvalidSeek(t *testing.T) {
602 }
603
604 func TestEndSeek(t *testing.T) {
605 + runBothSubtests(t, testEndSeek)
606 +}
607 +func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
608 dserv := testu.GetDAGServ()
609
555 - n := testu.GetEmptyNode(t, dserv)
610 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
611 ctx, cancel := context.WithCancel(context.Background())
612 defer cancel()
613
@@ -560,6 +615,7 @@ func TestEndSeek(t *testing.T) {
615 if err != nil {
616 t.Fatal(err)
617 }
618 + dagmod.RawLeaves = bool(rawLeaves)
619
620 _, err = dagmod.Write(make([]byte, 100))
621 if err != nil {
@@ -592,9 +648,12 @@ func TestEndSeek(t *testing.T) {
648 }
649
650 func TestReadAndSeek(t *testing.T) {
651 + runBothSubtests(t, testReadAndSeek)
652 +}
653 +func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
654 dserv := testu.GetDAGServ()
655
597 - n := testu.GetEmptyNode(t, dserv)
656 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
657 ctx, cancel := context.WithCancel(context.Background())
658 defer cancel()
659
@@ -602,6 +661,7 @@ func TestReadAndSeek(t *testing.T) {
661 if err != nil {
662 t.Fatal(err)
663 }
664 + dagmod.RawLeaves = bool(rawLeaves)
665
666 writeBuf := []byte{0, 1, 2, 3, 4, 5, 6, 7}
667 dagmod.Write(writeBuf)
@@ -660,9 +720,12 @@ func TestReadAndSeek(t *testing.T) {
720 }
721
722 func TestCtxRead(t *testing.T) {
723 + runBothSubtests(t, testCtxRead)
724 +}
725 +func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
726 dserv := testu.GetDAGServ()
727
665 - n := testu.GetEmptyNode(t, dserv)
728 + n := testu.GetEmptyNode(t, dserv, rawLeaves)
729 ctx, cancel := context.WithCancel(context.Background())
730 defer cancel()
731
@@ -670,6 +733,7 @@ func TestCtxRead(t *testing.T) {
733 if err != nil {
734 t.Fatal(err)
735 }
736 + dagmod.RawLeaves = bool(rawLeaves)
737
738 _, err = dagmod.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7})
739 if err != nil {
@@ -693,7 +757,7 @@ func TestCtxRead(t *testing.T) {
757 func BenchmarkDagmodWrite(b *testing.B) {
758 b.StopTimer()
759 dserv := testu.GetDAGServ()
696 - n := testu.GetEmptyNode(b, dserv)
760 + n := testu.GetEmptyNode(b, dserv, testu.ProtoBufLeaves)
761 ctx, cancel := context.WithCancel(context.Background())
762 defer cancel()
763
unixfs/test/utils.go
+22 -7
@@ -8,8 +8,9 @@ import (
8 "io/ioutil"
9 "testing"
10
11 - imp "github.com/ipfs/go-ipfs/importer"
11 "github.com/ipfs/go-ipfs/importer/chunk"
12 + h "github.com/ipfs/go-ipfs/importer/helpers"
13 + trickle "github.com/ipfs/go-ipfs/importer/trickle"
14 mdag "github.com/ipfs/go-ipfs/merkledag"
15 mdagmock "github.com/ipfs/go-ipfs/merkledag/test"
16 ft "github.com/ipfs/go-ipfs/unixfs"
@@ -28,9 +29,23 @@ func GetDAGServ() mdag.DAGService {
29 return mdagmock.Mock()
30 }
31
31 -func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) node.Node {
32 +type UseRawLeaves bool
33 +
34 +const (
35 + ProtoBufLeaves UseRawLeaves = false
36 + RawLeaves UseRawLeaves = true
37 +)
38 +
39 +func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, rawLeaves UseRawLeaves) node.Node {
40 in := bytes.NewReader(data)
33 - node, err := imp.BuildTrickleDagFromReader(dserv, SizeSplitterGen(500)(in))
41 +
42 + dbp := h.DagBuilderParams{
43 + Dagserv: dserv,
44 + Maxlinks: h.DefaultLinksPerBlock,
45 + RawLeaves: bool(rawLeaves),
46 + }
47 +
48 + node, err := trickle.TrickleLayout(dbp.New(SizeSplitterGen(500)(in)))
49 if err != nil {
50 t.Fatal(err)
51 }
@@ -38,18 +53,18 @@ func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) node.Node {
53 return node
54 }
55
41 -func GetEmptyNode(t testing.TB, dserv mdag.DAGService) node.Node {
42 - return GetNode(t, dserv, []byte{})
56 +func GetEmptyNode(t testing.TB, dserv mdag.DAGService, rawLeaves UseRawLeaves) node.Node {
57 + return GetNode(t, dserv, []byte{}, rawLeaves)
58 }
59
45 -func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, node.Node) {
60 +func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64, rawLeaves UseRawLeaves) ([]byte, node.Node) {
61 in := io.LimitReader(u.NewTimeSeededRand(), size)
62 buf, err := ioutil.ReadAll(in)
63 if err != nil {
64 t.Fatal(err)
65 }
66
52 - node := GetNode(t, dserv, buf)
67 + node := GetNode(t, dserv, buf, rawLeaves)
68 return buf, node
69 }
70