@cryptotaxi247 / kubo / commits / 189cf0cb0

Test raw leaves in trickle dag tests.

License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>

Kevin Atkinson committed Aug 5, 2017 at 17:22 UTC 189cf0cb03785f005f398f82cad2e66cf8fab028
3 files changed +154 -54
importer/trickle/trickle_test.go
+104 -26
@@ -19,10 +19,23 @@ import (
19 u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
20 )
21
22 -func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter) (*merkledag.ProtoNode, error) {
22 +type UseRawLeaves bool
23 +
24 +const (
25 + ProtoBufLeaves UseRawLeaves = false
26 + RawLeaves UseRawLeaves = true
27 +)
28 +
29 +func runBothSubtests(t *testing.T, tfunc func(*testing.T, UseRawLeaves)) {
30 + t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, ProtoBufLeaves) })
31 + t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, RawLeaves) })
32 +}
33 +
34 +func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter, rawLeaves UseRawLeaves) (*merkledag.ProtoNode, error) {
35 dbp := h.DagBuilderParams{
24 - Dagserv: ds,
25 - Maxlinks: h.DefaultLinksPerBlock,
36 + Dagserv: ds,
37 + Maxlinks: h.DefaultLinksPerBlock,
38 + RawLeaves: bool(rawLeaves),
39 }
40
41 nd, err := TrickleLayout(dbp.New(spl))
@@ -35,22 +48,31 @@ func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter) (*merkledag.Proto
48 return nil, merkledag.ErrNotProtobuf
49 }
50
38 - return pbnd, VerifyTrickleDagStructure(pbnd, ds, dbp.Maxlinks, layerRepeat)
51 + return pbnd, VerifyTrickleDagStructure(pbnd, VerifyParams{
52 + Getter: ds,
53 + Direct: dbp.Maxlinks,
54 + LayerRepeat: layerRepeat,
55 + RawLeaves: bool(rawLeaves),
56 + })
57 }
58
59 //Test where calls to read are smaller than the chunk size
60 func TestSizeBasedSplit(t *testing.T) {
61 + runBothSubtests(t, testSizeBasedSplit)
62 +}
63 +
64 +func testSizeBasedSplit(t *testing.T, rawLeaves UseRawLeaves) {
65 if testing.Short() {
66 t.SkipNow()
67 }
68 bs := chunk.SizeSplitterGen(512)
47 - testFileConsistency(t, bs, 32*512)
69 + testFileConsistency(t, bs, 32*512, rawLeaves)
70
71 bs = chunk.SizeSplitterGen(4096)
50 - testFileConsistency(t, bs, 32*4096)
72 + testFileConsistency(t, bs, 32*4096, rawLeaves)
73
74 // Uneven offset
53 - testFileConsistency(t, bs, 31*4095)
75 + testFileConsistency(t, bs, 31*4095, rawLeaves)
76 }
77
78 func dup(b []byte) []byte {
@@ -59,13 +81,13 @@ func dup(b []byte) []byte {
81 return o
82 }
83
62 -func testFileConsistency(t *testing.T, bs chunk.SplitterGen, nbytes int) {
84 +func testFileConsistency(t *testing.T, bs chunk.SplitterGen, nbytes int, rawLeaves UseRawLeaves) {
85 should := make([]byte, nbytes)
86 u.NewTimeSeededRand().Read(should)
87
88 read := bytes.NewReader(should)
89 ds := mdtest.Mock()
68 - nd, err := buildTestDag(ds, bs(read))
90 + nd, err := buildTestDag(ds, bs(read), rawLeaves)
91 if err != nil {
92 t.Fatal(err)
93 }
@@ -87,12 +109,16 @@ func testFileConsistency(t *testing.T, bs chunk.SplitterGen, nbytes int) {
109 }
110
111 func TestBuilderConsistency(t *testing.T) {
112 + runBothSubtests(t, testBuilderConsistency)
113 +}
114 +
115 +func testBuilderConsistency(t *testing.T, rawLeaves UseRawLeaves) {
116 nbytes := 100000
117 buf := new(bytes.Buffer)
118 io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes))
119 should := dup(buf.Bytes())
120 dagserv := mdtest.Mock()
95 - nd, err := buildTestDag(dagserv, chunk.DefaultSplitter(buf))
121 + nd, err := buildTestDag(dagserv, chunk.DefaultSplitter(buf), rawLeaves)
122 if err != nil {
123 t.Fatal(err)
124 }
@@ -125,6 +151,10 @@ func arrComp(a, b []byte) error {
151 }
152
153 func TestIndirectBlocks(t *testing.T) {
154 + runBothSubtests(t, testIndirectBlocks)
155 +}
156 +
157 +func testIndirectBlocks(t *testing.T, rawLeaves UseRawLeaves) {
158 splitter := chunk.SizeSplitterGen(512)
159 nbytes := 1024 * 1024
160 buf := make([]byte, nbytes)
@@ -133,7 +163,7 @@ func TestIndirectBlocks(t *testing.T) {
163 read := bytes.NewReader(buf)
164
165 ds := mdtest.Mock()
136 - dag, err := buildTestDag(ds, splitter(read))
166 + dag, err := buildTestDag(ds, splitter(read), rawLeaves)
167 if err != nil {
168 t.Fatal(err)
169 }
@@ -154,13 +184,17 @@ func TestIndirectBlocks(t *testing.T) {
184 }
185
186 func TestSeekingBasic(t *testing.T) {
187 + runBothSubtests(t, testSeekingBasic)
188 +}
189 +
190 +func testSeekingBasic(t *testing.T, rawLeaves UseRawLeaves) {
191 nbytes := int64(10 * 1024)
192 should := make([]byte, nbytes)
193 u.NewTimeSeededRand().Read(should)
194
195 read := bytes.NewReader(should)
196 ds := mdtest.Mock()
163 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 512))
197 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 512), rawLeaves)
198 if err != nil {
199 t.Fatal(err)
200 }
@@ -191,13 +225,17 @@ func TestSeekingBasic(t *testing.T) {
225 }
226
227 func TestSeekToBegin(t *testing.T) {
228 + runBothSubtests(t, testSeekToBegin)
229 +}
230 +
231 +func testSeekToBegin(t *testing.T, rawLeaves UseRawLeaves) {
232 nbytes := int64(10 * 1024)
233 should := make([]byte, nbytes)
234 u.NewTimeSeededRand().Read(should)
235
236 read := bytes.NewReader(should)
237 ds := mdtest.Mock()
200 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
238 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500), rawLeaves)
239 if err != nil {
240 t.Fatal(err)
241 }
@@ -235,13 +273,17 @@ func TestSeekToBegin(t *testing.T) {
273 }
274
275 func TestSeekToAlmostBegin(t *testing.T) {
276 + runBothSubtests(t, testSeekToAlmostBegin)
277 +}
278 +
279 +func testSeekToAlmostBegin(t *testing.T, rawLeaves UseRawLeaves) {
280 nbytes := int64(10 * 1024)
281 should := make([]byte, nbytes)
282 u.NewTimeSeededRand().Read(should)
283
284 read := bytes.NewReader(should)
285 ds := mdtest.Mock()
244 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
286 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500), rawLeaves)
287 if err != nil {
288 t.Fatal(err)
289 }
@@ -279,13 +321,17 @@ func TestSeekToAlmostBegin(t *testing.T) {
321 }
322
323 func TestSeekEnd(t *testing.T) {
324 + runBothSubtests(t, testSeekEnd)
325 +}
326 +
327 +func testSeekEnd(t *testing.T, rawLeaves UseRawLeaves) {
328 nbytes := int64(50 * 1024)
329 should := make([]byte, nbytes)
330 u.NewTimeSeededRand().Read(should)
331
332 read := bytes.NewReader(should)
333 ds := mdtest.Mock()
288 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
334 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500), rawLeaves)
335 if err != nil {
336 t.Fatal(err)
337 }
@@ -305,13 +351,17 @@ func TestSeekEnd(t *testing.T) {
351 }
352
353 func TestSeekEndSingleBlockFile(t *testing.T) {
354 + runBothSubtests(t, testSeekEndSingleBlockFile)
355 +}
356 +
357 +func testSeekEndSingleBlockFile(t *testing.T, rawLeaves UseRawLeaves) {
358 nbytes := int64(100)
359 should := make([]byte, nbytes)
360 u.NewTimeSeededRand().Read(should)
361
362 read := bytes.NewReader(should)
363 ds := mdtest.Mock()
314 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 5000))
364 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 5000), rawLeaves)
365 if err != nil {
366 t.Fatal(err)
367 }
@@ -331,13 +381,17 @@ func TestSeekEndSingleBlockFile(t *testing.T) {
381 }
382
383 func TestSeekingStress(t *testing.T) {
384 + runBothSubtests(t, testSeekingStress)
385 +}
386 +
387 +func testSeekingStress(t *testing.T, rawLeaves UseRawLeaves) {
388 nbytes := int64(1024 * 1024)
389 should := make([]byte, nbytes)
390 u.NewTimeSeededRand().Read(should)
391
392 read := bytes.NewReader(should)
393 ds := mdtest.Mock()
340 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 1000))
394 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 1000), rawLeaves)
395 if err != nil {
396 t.Fatal(err)
397 }
@@ -376,13 +430,17 @@ func TestSeekingStress(t *testing.T) {
430 }
431
432 func TestSeekingConsistency(t *testing.T) {
433 + runBothSubtests(t, testSeekingConsistency)
434 +}
435 +
436 +func testSeekingConsistency(t *testing.T, rawLeaves UseRawLeaves) {
437 nbytes := int64(128 * 1024)
438 should := make([]byte, nbytes)
439 u.NewTimeSeededRand().Read(should)
440
441 read := bytes.NewReader(should)
442 ds := mdtest.Mock()
385 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
443 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500), rawLeaves)
444 if err != nil {
445 t.Fatal(err)
446 }
@@ -419,6 +477,10 @@ func TestSeekingConsistency(t *testing.T) {
477 }
478
479 func TestAppend(t *testing.T) {
480 + runBothSubtests(t, testAppend)
481 +}
482 +
483 +func testAppend(t *testing.T, rawLeaves UseRawLeaves) {
484 nbytes := int64(128 * 1024)
485 should := make([]byte, nbytes)
486 u.NewTimeSeededRand().Read(should)
@@ -426,14 +488,15 @@ func TestAppend(t *testing.T) {
488 // Reader for half the bytes
489 read := bytes.NewReader(should[:nbytes/2])
490 ds := mdtest.Mock()
429 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
491 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500), rawLeaves)
492 if err != nil {
493 t.Fatal(err)
494 }
495
496 dbp := &h.DagBuilderParams{
435 - Dagserv: ds,
436 - Maxlinks: h.DefaultLinksPerBlock,
497 + Dagserv: ds,
498 + Maxlinks: h.DefaultLinksPerBlock,
499 + RawLeaves: bool(rawLeaves),
500 }
501
502 r := bytes.NewReader(should[nbytes/2:])
@@ -444,7 +507,12 @@ func TestAppend(t *testing.T) {
507 t.Fatal(err)
508 }
509
447 - err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
510 + err = VerifyTrickleDagStructure(nnode, VerifyParams{
511 + Getter: ds,
512 + Direct: dbp.Maxlinks,
513 + LayerRepeat: layerRepeat,
514 + RawLeaves: bool(rawLeaves),
515 + })
516 if err != nil {
517 t.Fatal(err)
518 }
@@ -467,6 +535,10 @@ func TestAppend(t *testing.T) {
535
536 // This test appends one byte at a time to an empty file
537 func TestMultipleAppends(t *testing.T) {
538 + runBothSubtests(t, testMultipleAppends)
539 +}
540 +
541 +func testMultipleAppends(t *testing.T, rawLeaves UseRawLeaves) {
542 ds := mdtest.Mock()
543
544 // TODO: fix small size appends and make this number bigger
@@ -475,14 +547,15 @@ func TestMultipleAppends(t *testing.T) {
547 u.NewTimeSeededRand().Read(should)
548
549 read := bytes.NewReader(nil)
478 - nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
550 + nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500), rawLeaves)
551 if err != nil {
552 t.Fatal(err)
553 }
554
555 dbp := &h.DagBuilderParams{
484 - Dagserv: ds,
485 - Maxlinks: 4,
556 + Dagserv: ds,
557 + Maxlinks: 4,
558 + RawLeaves: bool(rawLeaves),
559 }
560
561 spl := chunk.SizeSplitterGen(500)
@@ -495,7 +568,12 @@ func TestMultipleAppends(t *testing.T) {
568 t.Fatal(err)
569 }
570
498 - err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
571 + err = VerifyTrickleDagStructure(nnode, VerifyParams{
572 + Getter: ds,
573 + Direct: dbp.Maxlinks,
574 + LayerRepeat: layerRepeat,
575 + RawLeaves: bool(rawLeaves),
576 + })
577 if err != nil {
578 t.Fatal(err)
579 }
importer/trickle/trickledag.go
+45 -27
@@ -234,34 +234,57 @@ func trickleDepthInfo(node *h.UnixfsNode, maxlinks int) (int, int) {
234 return ((n - maxlinks) / layerRepeat) + 1, (n - maxlinks) % layerRepeat
235 }
236
237 +// VerifyParams is used by VerifyTrickleDagStructure
238 +type VerifyParams struct {
239 + Getter node.NodeGetter
240 + Direct int
241 + LayerRepeat int
242 + RawLeaves bool
243 +}
244 +
245 // VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure
246 // layout
239 -func VerifyTrickleDagStructure(nd node.Node, ds dag.DAGService, direct int, layerRepeat int) error {
240 - pbnd, ok := nd.(*dag.ProtoNode)
241 - if !ok {
242 - return dag.ErrNotProtobuf
243 - }
244 -
245 - return verifyTDagRec(pbnd, -1, direct, layerRepeat, ds)
247 +func VerifyTrickleDagStructure(nd node.Node, p VerifyParams) error {
248 + return verifyTDagRec(nd, -1, p)
249 }
250
251 // Recursive call for verifying the structure of a trickledag
249 -func verifyTDagRec(nd *dag.ProtoNode, depth, direct, layerRepeat int, ds dag.DAGService) error {
252 +func verifyTDagRec(n node.Node, depth int, p VerifyParams) error {
253 if depth == 0 {
251 - // zero depth dag is raw data block
252 - if len(nd.Links()) > 0 {
254 + if len(n.Links()) > 0 {
255 return errors.New("expected direct block")
256 }
257 + // zero depth dag is raw data block
258 + switch nd := n.(type) {
259 + case *dag.ProtoNode:
260 + pbn, err := ft.FromBytes(nd.Data())
261 + if err != nil {
262 + return err
263 + }
264
256 - pbn, err := ft.FromBytes(nd.Data())
257 - if err != nil {
258 - return err
259 - }
265 + if pbn.GetType() != ft.TRaw {
266 + return errors.New("Expected raw block")
267 + }
268 +
269 + if p.RawLeaves {
270 + return errors.New("expected raw leaf, got a protobuf node")
271 + }
272 +
273 + return nil
274 + case *dag.RawNode:
275 + if !p.RawLeaves {
276 + return errors.New("expected protobuf node as leaf")
277 + }
278
261 - if pbn.GetType() != ft.TRaw {
262 - return errors.New("Expected raw block")
279 + return nil
280 + default:
281 + return errors.New("expected ProtoNode or RawNode")
282 }
264 - return nil
283 + }
284 +
285 + nd, ok := n.(*dag.ProtoNode)
286 + if !ok {
287 + return errors.New("expected ProtoNode")
288 }
289
290 // Verify this is a branch node
@@ -279,29 +302,24 @@ func verifyTDagRec(nd *dag.ProtoNode, depth, direct, layerRepeat int, ds dag.DAG
302 }
303
304 for i := 0; i < len(nd.Links()); i++ {
282 - childi, err := nd.Links()[i].GetNode(context.TODO(), ds)
305 + child, err := nd.Links()[i].GetNode(context.TODO(), p.Getter)
306 if err != nil {
307 return err
308 }
309
287 - childpb, ok := childi.(*dag.ProtoNode)
288 - if !ok {
289 - return fmt.Errorf("cannot operate on non-protobuf nodes")
290 - }
291 -
292 - if i < direct {
310 + if i < p.Direct {
311 // Direct blocks
294 - err := verifyTDagRec(childpb, 0, direct, layerRepeat, ds)
312 + err := verifyTDagRec(child, 0, p)
313 if err != nil {
314 return err
315 }
316 } else {
317 // Recursive trickle dags
300 - rdepth := ((i - direct) / layerRepeat) + 1
318 + rdepth := ((i - p.Direct) / p.LayerRepeat) + 1
319 if rdepth >= depth && depth > 0 {
320 return errors.New("Child dag was too deep!")
321 }
304 - err := verifyTDagRec(childpb, rdepth, direct, layerRepeat, ds)
322 + err := verifyTDagRec(child, rdepth, p)
323 if err != nil {
324 return err
325 }
unixfs/mod/dagmodifier_test.go
+5 -1
@@ -41,7 +41,11 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier)
41 t.Fatal(err)
42 }
43
44 - err = trickle.VerifyTrickleDagStructure(nd, dm.dagserv, h.DefaultLinksPerBlock, 4)
44 + err = trickle.VerifyTrickleDagStructure(nd, trickle.VerifyParams{
45 + Getter: dm.dagserv,
46 + Direct: h.DefaultLinksPerBlock,
47 + LayerRepeat: 4,
48 + })
49 if err != nil {
50 t.Fatal(err)
51 }