@cryptotaxi247 / kubo / commits / 2e15dcb64

Enable CidV1 (and other prefixes) in the Dag Modifier.

License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>

Kevin Atkinson committed Aug 8, 2017 at 18:19 UTC 2e15dcb647dfe50d9fdab3a58ee261ef7f76ca96
6 files changed +178 -165
importer/trickle/trickledag.go
+24 -4
@@ -9,6 +9,7 @@ import (
9 dag "github.com/ipfs/go-ipfs/merkledag"
10 ft "github.com/ipfs/go-ipfs/unixfs"
11
12 + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
13 node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
14 )
15
@@ -239,6 +240,7 @@ type VerifyParams struct {
240 Getter node.NodeGetter
241 Direct int
242 LayerRepeat int
243 + Prefix *cid.Prefix
244 RawLeaves bool
245 }
246
@@ -250,6 +252,7 @@ func VerifyTrickleDagStructure(nd node.Node, p VerifyParams) error {
252
253 // Recursive call for verifying the structure of a trickledag
254 func verifyTDagRec(n node.Node, depth int, p VerifyParams) error {
255 + codec := cid.DagProtobuf
256 if depth == 0 {
257 if len(n.Links()) > 0 {
258 return errors.New("expected direct block")
@@ -269,19 +272,36 @@ func verifyTDagRec(n node.Node, depth int, p VerifyParams) error {
272 if p.RawLeaves {
273 return errors.New("expected raw leaf, got a protobuf node")
274 }
272 -
273 - return nil
275 case *dag.RawNode:
276 if !p.RawLeaves {
277 return errors.New("expected protobuf node as leaf")
278 }
278 -
279 - return nil
279 + codec = cid.Raw
280 default:
281 return errors.New("expected ProtoNode or RawNode")
282 }
283 }
284
285 + // verify prefix
286 + if p.Prefix != nil {
287 + prefix := n.Cid().Prefix()
288 + expect := *p.Prefix // make a copy
289 + expect.Codec = uint64(codec)
290 + if codec == cid.Raw && expect.Version == 0 {
291 + expect.Version = 1
292 + }
293 + if expect.MhLength == -1 {
294 + expect.MhLength = prefix.MhLength
295 + }
296 + if prefix != expect {
297 + return fmt.Errorf("unexpected cid prefix: expected: %v; got %v", expect, prefix)
298 + }
299 + }
300 +
301 + if depth == 0 {
302 + return nil
303 + }
304 +
305 nd, ok := n.(*dag.ProtoNode)
306 if !ok {
307 return errors.New("expected ProtoNode")
merkledag/node.go
+3
@@ -42,6 +42,9 @@ var v1CidPrefix = cid.Prefix{
42 Version: 1,
43 }
44
45 +func V0CidPrefix() cid.Prefix { return v0CidPrefix }
46 +func V1CidPrefix() cid.Prefix { return v1CidPrefix }
47 +
48 // PrefixForCidVersion returns the Protobuf prefix for a given CID version
49 func PrefixForCidVersion(version int) (cid.Prefix, error) {
50 switch version {
unixfs/io/dagreader_test.go
+6 -6
@@ -17,7 +17,7 @@ import (
17
18 func TestBasicRead(t *testing.T) {
19 dserv := testu.GetDAGServ()
20 - inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.ProtoBufLeaves)
20 + inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
21 ctx, closer := context.WithCancel(context.Background())
22 defer closer()
23
@@ -44,7 +44,7 @@ func TestSeekAndRead(t *testing.T) {
44 inbuf[i] = byte(i)
45 }
46
47 - node := testu.GetNode(t, dserv, inbuf, testu.ProtoBufLeaves)
47 + node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
48 ctx, closer := context.WithCancel(context.Background())
49 defer closer()
50
@@ -84,7 +84,7 @@ func TestRelativeSeek(t *testing.T) {
84 }
85
86 inbuf[1023] = 1 // force the reader to be 1024 bytes
87 - node := testu.GetNode(t, dserv, inbuf, testu.ProtoBufLeaves)
87 + node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
88
89 reader, err := NewDagReader(ctx, node, dserv)
90 if err != nil {
@@ -160,7 +160,7 @@ func TestBadPBData(t *testing.T) {
160
161 func TestMetadataNode(t *testing.T) {
162 dserv := testu.GetDAGServ()
163 - rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.ProtoBufLeaves)
163 + rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves)
164 _, err := dserv.Add(rnode)
165 if err != nil {
166 t.Fatal(err)
@@ -203,7 +203,7 @@ func TestMetadataNode(t *testing.T) {
203
204 func TestWriteTo(t *testing.T) {
205 dserv := testu.GetDAGServ()
206 - inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.ProtoBufLeaves)
206 + inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
207 ctx, closer := context.WithCancel(context.Background())
208 defer closer()
209
@@ -225,7 +225,7 @@ func TestWriteTo(t *testing.T) {
225 func TestReaderSzie(t *testing.T) {
226 dserv := testu.GetDAGServ()
227 size := int64(1024)
228 - _, node := testu.GetRandomNode(t, dserv, size, testu.ProtoBufLeaves)
228 + _, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves)
229 ctx, closer := context.WithCancel(context.Background())
230 defer closer()
231
unixfs/mod/dagmodifier.go
+20 -4
@@ -40,6 +40,7 @@ type DagModifier struct {
40 curWrOff uint64
41 wrBuf *bytes.Buffer
42
43 + Prefix cid.Prefix
44 RawLeaves bool
45
46 read uio.DagReader
@@ -47,6 +48,10 @@ type DagModifier struct {
48
49 var ErrNotUnixfs = fmt.Errorf("dagmodifier only supports unixfs nodes (proto or raw)")
50
51 +// NewDagModifier returns a new DagModifier, the Cid prefix for newly
52 +// created nodes will be inherted from the passed in node. If the Cid
53 +// version if not 0 raw leaves will also be enabled. The Prefix and
54 +// RawLeaves options can be overridden by changing them after the call.
55 func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) {
56 switch from.(type) {
57 case *mdag.ProtoNode, *mdag.RawNode:
@@ -55,11 +60,20 @@ func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, s
60 return nil, ErrNotUnixfs
61 }
62
63 + prefix := from.Cid().Prefix()
64 + prefix.Codec = cid.DagProtobuf
65 + rawLeaves := false
66 + if prefix.Version > 0 {
67 + rawLeaves = true
68 + }
69 +
70 return &DagModifier{
59 - curNode: from.Copy(),
60 - dagserv: serv,
61 - splitter: spl,
62 - ctx: ctx,
71 + curNode: from.Copy(),
72 + dagserv: serv,
73 + splitter: spl,
74 + ctx: ctx,
75 + Prefix: prefix,
76 + RawLeaves: rawLeaves,
77 }, nil
78 }
79
@@ -240,6 +254,7 @@ func (dm *DagModifier) modifyDag(n node.Node, offset uint64, data io.Reader) (*c
254
255 nd := new(mdag.ProtoNode)
256 nd.SetData(b)
257 + nd.SetPrefix(&nd0.Prefix)
258 k, err := dm.dagserv.Add(nd)
259 if err != nil {
260 return nil, false, err
@@ -345,6 +360,7 @@ func (dm *DagModifier) appendData(nd node.Node, spl chunk.Splitter) (node.Node,
360 dbp := &help.DagBuilderParams{
361 Dagserv: dm.dagserv,
362 Maxlinks: help.DefaultLinksPerBlock,
363 + Prefix: &dm.Prefix,
364 RawLeaves: dm.RawLeaves,
365 }
366 return trickle.TrickleAppend(dm.ctx, nd, dbp.New(spl))
unixfs/mod/dagmodifier_test.go
+107 -140
@@ -16,7 +16,7 @@ import (
16 u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
17 )
18
19 -func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, rawLeaves testu.UseRawLeaves) []byte {
19 +func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, opts testu.NodeOpts) []byte {
20 newdata := make([]byte, size)
21 r := u.NewTimeSeededRand()
22 r.Read(newdata)
@@ -35,6 +35,12 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier,
35 t.Fatalf("Mod length not correct! %d != %d", nmod, size)
36 }
37
38 + verifyNode(t, orig, dm, opts)
39 +
40 + return orig
41 +}
42 +
43 +func verifyNode(t *testing.T, orig []byte, dm *DagModifier, opts testu.NodeOpts) {
44 nd, err := dm.GetNode()
45 if err != nil {
46 t.Fatal(err)
@@ -44,10 +50,11 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier,
50 Getter: dm.dagserv,
51 Direct: h.DefaultLinksPerBlock,
52 LayerRepeat: 4,
47 - RawLeaves: bool(rawLeaves),
53 + Prefix: &opts.Prefix,
54 + RawLeaves: opts.RawLeavesUsed,
55 })
56 if err != nil {
50 - t.Error(err)
57 + t.Fatal(err)
58 }
59
60 rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv)
@@ -64,20 +71,20 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier,
71 if err != nil {
72 t.Fatal(err)
73 }
67 - return orig
74 }
75
70 -func runBothSubtests(t *testing.T, tfunc func(*testing.T, testu.UseRawLeaves)) {
71 - t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, testu.ProtoBufLeaves) })
72 - t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, testu.RawLeaves) })
76 +func runAllSubtests(t *testing.T, tfunc func(*testing.T, testu.NodeOpts)) {
77 + t.Run("opts=ProtoBufLeaves", func(t *testing.T) { tfunc(t, testu.UseProtoBufLeaves) })
78 + t.Run("opts=RawLeaves", func(t *testing.T) { tfunc(t, testu.UseRawLeaves) })
79 + t.Run("opts=CidV1", func(t *testing.T) { tfunc(t, testu.UseCidV1) })
80 }
81
82 func TestDagModifierBasic(t *testing.T) {
76 - runBothSubtests(t, testDagModifierBasic)
83 + runAllSubtests(t, testDagModifierBasic)
84 }
78 -func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
85 +func testDagModifierBasic(t *testing.T, opts testu.NodeOpts) {
86 dserv := testu.GetDAGServ()
80 - b, n := testu.GetRandomNode(t, dserv, 50000, rawLeaves)
87 + b, n := testu.GetRandomNode(t, dserv, 50000, opts)
88 ctx, cancel := context.WithCancel(context.Background())
89 defer cancel()
90
@@ -85,33 +92,35 @@ func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
92 if err != nil {
93 t.Fatal(err)
94 }
88 - dagmod.RawLeaves = bool(rawLeaves)
95 + if opts.ForceRawLeaves {
96 + dagmod.RawLeaves = true
97 + }
98
99 // Within zero block
100 beg := uint64(15)
101 length := uint64(60)
102
103 t.Log("Testing mod within zero block")
95 - b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
104 + b = testModWrite(t, beg, length, b, dagmod, opts)
105
106 // Within bounds of existing file
107 beg = 1000
108 length = 4000
109 t.Log("Testing mod within bounds of existing multiblock file.")
101 - b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
110 + b = testModWrite(t, beg, length, b, dagmod, opts)
111
112 // Extend bounds
113 beg = 49500
114 length = 4000
115
116 t.Log("Testing mod that extends file.")
108 - b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
117 + b = testModWrite(t, beg, length, b, dagmod, opts)
118
119 // "Append"
120 beg = uint64(len(b))
121 length = 3000
122 t.Log("Testing pure append")
114 - _ = testModWrite(t, beg, length, b, dagmod, rawLeaves)
123 + _ = testModWrite(t, beg, length, b, dagmod, opts)
124
125 // Verify reported length
126 node, err := dagmod.GetNode()
@@ -131,11 +140,11 @@ func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
140 }
141
142 func TestMultiWrite(t *testing.T) {
134 - runBothSubtests(t, testMultiWrite)
143 + runAllSubtests(t, testMultiWrite)
144 }
136 -func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
145 +func testMultiWrite(t *testing.T, opts testu.NodeOpts) {
146 dserv := testu.GetDAGServ()
138 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
147 + n := testu.GetEmptyNode(t, dserv, opts)
148
149 ctx, cancel := context.WithCancel(context.Background())
150 defer cancel()
@@ -144,7 +153,9 @@ func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
153 if err != nil {
154 t.Fatal(err)
155 }
147 - dagmod.RawLeaves = bool(rawLeaves)
156 + if opts.ForceRawLeaves {
157 + dagmod.RawLeaves = true
158 + }
159
160 data := make([]byte, 4000)
161 u.NewTimeSeededRand().Read(data)
@@ -167,32 +178,16 @@ func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
178 t.Fatal("Size was reported incorrectly")
179 }
180 }
170 - nd, err := dagmod.GetNode()
171 - if err != nil {
172 - t.Fatal(err)
173 - }
181
175 - read, err := uio.NewDagReader(context.Background(), nd, dserv)
176 - if err != nil {
177 - t.Fatal(err)
178 - }
179 - rbuf, err := ioutil.ReadAll(read)
180 - if err != nil {
181 - t.Fatal(err)
182 - }
183 -
184 - err = testu.ArrComp(rbuf, data)
185 - if err != nil {
186 - t.Fatal(err)
187 - }
182 + verifyNode(t, data, dagmod, opts)
183 }
184
185 func TestMultiWriteAndFlush(t *testing.T) {
191 - runBothSubtests(t, testMultiWriteAndFlush)
186 + runAllSubtests(t, testMultiWriteAndFlush)
187 }
193 -func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
188 +func testMultiWriteAndFlush(t *testing.T, opts testu.NodeOpts) {
189 dserv := testu.GetDAGServ()
195 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
190 + n := testu.GetEmptyNode(t, dserv, opts)
191
192 ctx, cancel := context.WithCancel(context.Background())
193 defer cancel()
@@ -201,7 +196,9 @@ func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
196 if err != nil {
197 t.Fatal(err)
198 }
204 - dagmod.RawLeaves = bool(rawLeaves)
199 + if opts.ForceRawLeaves {
200 + dagmod.RawLeaves = true
201 + }
202
203 data := make([]byte, 20)
204 u.NewTimeSeededRand().Read(data)
@@ -219,32 +216,16 @@ func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
216 t.Fatal(err)
217 }
218 }
222 - nd, err := dagmod.GetNode()
223 - if err != nil {
224 - t.Fatal(err)
225 - }
219
227 - read, err := uio.NewDagReader(context.Background(), nd, dserv)
228 - if err != nil {
229 - t.Fatal(err)
230 - }
231 - rbuf, err := ioutil.ReadAll(read)
232 - if err != nil {
233 - t.Fatal(err)
234 - }
235 -
236 - err = testu.ArrComp(rbuf, data)
237 - if err != nil {
238 - t.Fatal(err)
239 - }
220 + verifyNode(t, data, dagmod, opts)
221 }
222
223 func TestWriteNewFile(t *testing.T) {
243 - runBothSubtests(t, testWriteNewFile)
224 + runAllSubtests(t, testWriteNewFile)
225 }
245 -func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
226 +func testWriteNewFile(t *testing.T, opts testu.NodeOpts) {
227 dserv := testu.GetDAGServ()
247 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
228 + n := testu.GetEmptyNode(t, dserv, opts)
229
230 ctx, cancel := context.WithCancel(context.Background())
231 defer cancel()
@@ -253,7 +234,9 @@ func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
234 if err != nil {
235 t.Fatal(err)
236 }
256 - dagmod.RawLeaves = bool(rawLeaves)
237 + if opts.ForceRawLeaves {
238 + dagmod.RawLeaves = true
239 + }
240
241 towrite := make([]byte, 2000)
242 u.NewTimeSeededRand().Read(towrite)
@@ -266,32 +249,15 @@ func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
249 t.Fatal("Wrote wrong amount")
250 }
251
269 - nd, err := dagmod.GetNode()
270 - if err != nil {
271 - t.Fatal(err)
272 - }
273 -
274 - read, err := uio.NewDagReader(ctx, nd, dserv)
275 - if err != nil {
276 - t.Fatal(err)
277 - }
278 -
279 - data, err := ioutil.ReadAll(read)
280 - if err != nil {
281 - t.Fatal(err)
282 - }
283 -
284 - if err := testu.ArrComp(data, towrite); err != nil {
285 - t.Fatal(err)
286 - }
252 + verifyNode(t, towrite, dagmod, opts)
253 }
254
255 func TestMultiWriteCoal(t *testing.T) {
290 - runBothSubtests(t, testMultiWriteCoal)
256 + runAllSubtests(t, testMultiWriteCoal)
257 }
292 -func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
258 +func testMultiWriteCoal(t *testing.T, opts testu.NodeOpts) {
259 dserv := testu.GetDAGServ()
294 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
260 + n := testu.GetEmptyNode(t, dserv, opts)
261
262 ctx, cancel := context.WithCancel(context.Background())
263 defer cancel()
@@ -300,7 +266,9 @@ func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
266 if err != nil {
267 t.Fatal(err)
268 }
303 - dagmod.RawLeaves = bool(rawLeaves)
269 + if opts.ForceRawLeaves {
270 + dagmod.RawLeaves = true
271 + }
272
273 data := make([]byte, 1000)
274 u.NewTimeSeededRand().Read(data)
@@ -316,34 +284,16 @@ func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
284 }
285
286 }
319 - nd, err := dagmod.GetNode()
320 - if err != nil {
321 - t.Fatal(err)
322 - }
287
324 - read, err := uio.NewDagReader(context.Background(), nd, dserv)
325 - if err != nil {
326 - t.Fatal(err)
327 - }
328 - dagmod.RawLeaves = bool(rawLeaves)
329 -
330 - rbuf, err := ioutil.ReadAll(read)
331 - if err != nil {
332 - t.Fatal(err)
333 - }
334 -
335 - err = testu.ArrComp(rbuf, data)
336 - if err != nil {
337 - t.Fatal(err)
338 - }
288 + verifyNode(t, data, dagmod, opts)
289 }
290
291 func TestLargeWriteChunks(t *testing.T) {
342 - runBothSubtests(t, testLargeWriteChunks)
292 + runAllSubtests(t, testLargeWriteChunks)
293 }
344 -func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
294 +func testLargeWriteChunks(t *testing.T, opts testu.NodeOpts) {
295 dserv := testu.GetDAGServ()
346 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
296 + n := testu.GetEmptyNode(t, dserv, opts)
297
298 ctx, cancel := context.WithCancel(context.Background())
299 defer cancel()
@@ -352,7 +302,9 @@ func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
302 if err != nil {
303 t.Fatal(err)
304 }
355 - dagmod.RawLeaves = bool(rawLeaves)
305 + if opts.ForceRawLeaves {
306 + dagmod.RawLeaves = true
307 + }
308
309 wrsize := 1000
310 datasize := 10000000
@@ -378,15 +330,14 @@ func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
330 if err = testu.ArrComp(out, data); err != nil {
331 t.Fatal(err)
332 }
381 -
333 }
334
335 func TestDagTruncate(t *testing.T) {
385 - runBothSubtests(t, testDagTruncate)
336 + runAllSubtests(t, testDagTruncate)
337 }
387 -func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
338 +func testDagTruncate(t *testing.T, opts testu.NodeOpts) {
339 dserv := testu.GetDAGServ()
389 - b, n := testu.GetRandomNode(t, dserv, 50000, rawLeaves)
340 + b, n := testu.GetRandomNode(t, dserv, 50000, opts)
341 ctx, cancel := context.WithCancel(context.Background())
342 defer cancel()
343
@@ -394,7 +345,9 @@ func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
345 if err != nil {
346 t.Fatal(err)
347 }
397 - dagmod.RawLeaves = bool(rawLeaves)
348 + if opts.ForceRawLeaves {
349 + dagmod.RawLeaves = true
350 + }
351
352 err = dagmod.Truncate(12345)
353 if err != nil {
@@ -453,11 +406,11 @@ func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
406 }
407
408 func TestSparseWrite(t *testing.T) {
456 - runBothSubtests(t, testSparseWrite)
409 + runAllSubtests(t, testSparseWrite)
410 }
458 -func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
411 +func testSparseWrite(t *testing.T, opts testu.NodeOpts) {
412 dserv := testu.GetDAGServ()
460 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
413 + n := testu.GetEmptyNode(t, dserv, opts)
414 ctx, cancel := context.WithCancel(context.Background())
415 defer cancel()
416
@@ -465,7 +418,9 @@ func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
418 if err != nil {
419 t.Fatal(err)
420 }
468 - dagmod.RawLeaves = bool(rawLeaves)
421 + if opts.ForceRawLeaves {
422 + dagmod.RawLeaves = true
423 + }
424
425 buf := make([]byte, 5000)
426 u.NewTimeSeededRand().Read(buf[2500:])
@@ -495,11 +450,11 @@ func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
450 }
451
452 func TestSeekPastEndWrite(t *testing.T) {
498 - runBothSubtests(t, testSeekPastEndWrite)
453 + runAllSubtests(t, testSeekPastEndWrite)
454 }
500 -func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
455 +func testSeekPastEndWrite(t *testing.T, opts testu.NodeOpts) {
456 dserv := testu.GetDAGServ()
502 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
457 + n := testu.GetEmptyNode(t, dserv, opts)
458 ctx, cancel := context.WithCancel(context.Background())
459 defer cancel()
460
@@ -507,7 +462,9 @@ func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
462 if err != nil {
463 t.Fatal(err)
464 }
510 - dagmod.RawLeaves = bool(rawLeaves)
465 + if opts.ForceRawLeaves {
466 + dagmod.RawLeaves = true
467 + }
468
469 buf := make([]byte, 5000)
470 u.NewTimeSeededRand().Read(buf[2500:])
@@ -546,11 +503,11 @@ func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
503 }
504
505 func TestRelativeSeek(t *testing.T) {
549 - runBothSubtests(t, testRelativeSeek)
506 + runAllSubtests(t, testRelativeSeek)
507 }
551 -func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
508 +func testRelativeSeek(t *testing.T, opts testu.NodeOpts) {
509 dserv := testu.GetDAGServ()
553 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
510 + n := testu.GetEmptyNode(t, dserv, opts)
511 ctx, cancel := context.WithCancel(context.Background())
512 defer cancel()
513
@@ -558,7 +515,9 @@ func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
515 if err != nil {
516 t.Fatal(err)
517 }
561 - dagmod.RawLeaves = bool(rawLeaves)
518 + if opts.ForceRawLeaves {
519 + dagmod.RawLeaves = true
520 + }
521
522 for i := 0; i < 64; i++ {
523 dagmod.Write([]byte{byte(i)})
@@ -580,11 +539,11 @@ func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
539 }
540
541 func TestInvalidSeek(t *testing.T) {
583 - runBothSubtests(t, testInvalidSeek)
542 + runAllSubtests(t, testInvalidSeek)
543 }
585 -func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
544 +func testInvalidSeek(t *testing.T, opts testu.NodeOpts) {
545 dserv := testu.GetDAGServ()
587 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
546 + n := testu.GetEmptyNode(t, dserv, opts)
547 ctx, cancel := context.WithCancel(context.Background())
548 defer cancel()
549
@@ -592,7 +551,9 @@ func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
551 if err != nil {
552 t.Fatal(err)
553 }
595 - dagmod.RawLeaves = bool(rawLeaves)
554 + if opts.ForceRawLeaves {
555 + dagmod.RawLeaves = true
556 + }
557
558 _, err = dagmod.Seek(10, -10)
559
@@ -602,12 +563,12 @@ func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
563 }
564
565 func TestEndSeek(t *testing.T) {
605 - runBothSubtests(t, testEndSeek)
566 + runAllSubtests(t, testEndSeek)
567 }
607 -func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
568 +func testEndSeek(t *testing.T, opts testu.NodeOpts) {
569 dserv := testu.GetDAGServ()
570
610 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
571 + n := testu.GetEmptyNode(t, dserv, opts)
572 ctx, cancel := context.WithCancel(context.Background())
573 defer cancel()
574
@@ -615,7 +576,9 @@ func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
576 if err != nil {
577 t.Fatal(err)
578 }
618 - dagmod.RawLeaves = bool(rawLeaves)
579 + if opts.ForceRawLeaves {
580 + dagmod.RawLeaves = true
581 + }
582
583 _, err = dagmod.Write(make([]byte, 100))
584 if err != nil {
@@ -648,12 +611,12 @@ func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
611 }
612
613 func TestReadAndSeek(t *testing.T) {
651 - runBothSubtests(t, testReadAndSeek)
614 + runAllSubtests(t, testReadAndSeek)
615 }
653 -func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
616 +func testReadAndSeek(t *testing.T, opts testu.NodeOpts) {
617 dserv := testu.GetDAGServ()
618
656 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
619 + n := testu.GetEmptyNode(t, dserv, opts)
620 ctx, cancel := context.WithCancel(context.Background())
621 defer cancel()
622
@@ -661,7 +624,9 @@ func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
624 if err != nil {
625 t.Fatal(err)
626 }
664 - dagmod.RawLeaves = bool(rawLeaves)
627 + if opts.ForceRawLeaves {
628 + dagmod.RawLeaves = true
629 + }
630
631 writeBuf := []byte{0, 1, 2, 3, 4, 5, 6, 7}
632 dagmod.Write(writeBuf)
@@ -720,12 +685,12 @@ func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
685 }
686
687 func TestCtxRead(t *testing.T) {
723 - runBothSubtests(t, testCtxRead)
688 + runAllSubtests(t, testCtxRead)
689 }
725 -func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
690 +func testCtxRead(t *testing.T, opts testu.NodeOpts) {
691 dserv := testu.GetDAGServ()
692
728 - n := testu.GetEmptyNode(t, dserv, rawLeaves)
693 + n := testu.GetEmptyNode(t, dserv, opts)
694 ctx, cancel := context.WithCancel(context.Background())
695 defer cancel()
696
@@ -733,7 +698,9 @@ func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
698 if err != nil {
699 t.Fatal(err)
700 }
736 - dagmod.RawLeaves = bool(rawLeaves)
701 + if opts.ForceRawLeaves {
702 + dagmod.RawLeaves = true
703 + }
704
705 _, err = dagmod.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7})
706 if err != nil {
@@ -757,7 +724,7 @@ func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
724 func BenchmarkDagmodWrite(b *testing.B) {
725 b.StopTimer()
726 dserv := testu.GetDAGServ()
760 - n := testu.GetEmptyNode(b, dserv, testu.ProtoBufLeaves)
727 + n := testu.GetEmptyNode(b, dserv, testu.UseProtoBufLeaves)
728 ctx, cancel := context.WithCancel(context.Background())
729 defer cancel()
730
unixfs/test/utils.go
+18 -11
@@ -15,6 +15,7 @@ import (
15 mdagmock "github.com/ipfs/go-ipfs/merkledag/test"
16 ft "github.com/ipfs/go-ipfs/unixfs"
17
18 + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
19 node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
20 u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
21 )
@@ -29,20 +30,26 @@ func GetDAGServ() mdag.DAGService {
30 return mdagmock.Mock()
31 }
32
32 -type UseRawLeaves bool
33 +type NodeOpts struct {
34 + Prefix cid.Prefix
35 + // ForceRawLeaves if true will force the use of raw leaves
36 + ForceRawLeaves bool
37 + // RawLeavesUsed is true if raw leaves or either implicitly or explicitly enabled
38 + RawLeavesUsed bool
39 +}
40
34 -const (
35 - ProtoBufLeaves UseRawLeaves = false
36 - RawLeaves UseRawLeaves = true
37 -)
41 +var UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()}
42 +var UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true}
43 +var UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true}
44
39 -func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, rawLeaves UseRawLeaves) node.Node {
45 +func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, opts NodeOpts) node.Node {
46 in := bytes.NewReader(data)
47
48 dbp := h.DagBuilderParams{
49 Dagserv: dserv,
50 Maxlinks: h.DefaultLinksPerBlock,
45 - RawLeaves: bool(rawLeaves),
51 + Prefix: &opts.Prefix,
52 + RawLeaves: opts.RawLeavesUsed,
53 }
54
55 node, err := trickle.TrickleLayout(dbp.New(SizeSplitterGen(500)(in)))
@@ -53,18 +60,18 @@ func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, rawLeaves UseRawL
60 return node
61 }
62
56 -func GetEmptyNode(t testing.TB, dserv mdag.DAGService, rawLeaves UseRawLeaves) node.Node {
57 - return GetNode(t, dserv, []byte{}, rawLeaves)
63 +func GetEmptyNode(t testing.TB, dserv mdag.DAGService, opts NodeOpts) node.Node {
64 + return GetNode(t, dserv, []byte{}, opts)
65 }
66
60 -func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64, rawLeaves UseRawLeaves) ([]byte, node.Node) {
67 +func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64, opts NodeOpts) ([]byte, node.Node) {
68 in := io.LimitReader(u.NewTimeSeededRand(), size)
69 buf, err := ioutil.ReadAll(in)
70 if err != nil {
71 t.Fatal(err)
72 }
73
67 - node := GetNode(t, dserv, buf, rawLeaves)
74 + node := GetNode(t, dserv, buf, opts)
75 return buf, node
76 }
77