@cryptotaxi247 / kubo / commits / 90c58d6ca

Dag import functionality only ( no progress )

This still works over "loosely defined" .car files Please refer to the sharness tests for extra info We can tighten this up if the sentiment is "Postel was wrong"

Peter Rabbitson committed Mar 4, 2020 at 16:11 UTC 90c58d6cac4c639379bb7730b62346eac9a989b8
9 files changed +465 -9
.gitattributes
+1
@@ -9,6 +9,7 @@ LICENSE text eol=auto
9 *.png binary
10 *.tar binary
11 *.gz binary
12 +*.xz binary
13 *.car binary
14
15 # Binary assets
.gitignore
+1
@@ -26,3 +26,4 @@ vendor
26 .tarball
27 go-ipfs-source.tar.gz
28 docs/examples/go-ipfs-as-a-library/example-folder/Qm*
29 +/test/sharness/t0054-dag-car-import-export-data/*.car
core/commands/commands_test.go
+1
@@ -97,6 +97,7 @@ func TestCommands(t *testing.T) {
97 "/dag/get",
98 "/dag/export",
99 "/dag/put",
100 + "/dag/import",
101 "/dag/resolve",
102 "/dht",
103 "/dht/findpeer",
core/commands/dag/dag.go
+277 -1
@@ -11,14 +11,16 @@ import (
11
12 "github.com/ipfs/go-ipfs/core/commands/cmdenv"
13 "github.com/ipfs/go-ipfs/core/coredag"
14 - mdag "github.com/ipfs/go-merkledag"
14 + iface "github.com/ipfs/interface-go-ipfs-core"
15
16 cid "github.com/ipfs/go-cid"
17 cidenc "github.com/ipfs/go-cidutil/cidenc"
18 cmds "github.com/ipfs/go-ipfs-cmds"
19 files "github.com/ipfs/go-ipfs-files"
20 ipld "github.com/ipfs/go-ipld-format"
21 + mdag "github.com/ipfs/go-merkledag"
22 ipfspath "github.com/ipfs/go-path"
23 + "github.com/ipfs/interface-go-ipfs-core/options"
24 path "github.com/ipfs/interface-go-ipfs-core/path"
25 mh "github.com/multiformats/go-multihash"
26
@@ -32,6 +34,8 @@ import (
34
35 const (
36 progressOptionName = "progress"
37 + silentOptionName = "silent"
38 + pinRootsOptionName = "pin-roots"
39 )
40
41 var DagCmd = &cmds.Command{
@@ -48,6 +52,7 @@ to deprecate and replace the existing 'ipfs object' command moving forward.
52 "put": DagPutCmd,
53 "get": DagGetCmd,
54 "resolve": DagResolveCmd,
55 + "import": DagImportCmd,
56 "export": DagExportCmd,
57 },
58 }
@@ -63,6 +68,16 @@ type ResolveOutput struct {
68 RemPath string
69 }
70
71 +// CarImportOutput is the output type of the 'dag import' commands
72 +type CarImportOutput struct {
73 + Root RootMeta
74 +}
75 +type RootMeta struct {
76 + Cid cid.Cid
77 + PresentInImport bool
78 + PinErrorMsg string
79 +}
80 +
81 var DagPutCmd = &cmds.Command{
82 Helptext: cmds.HelpText{
83 Tagline: "Add a dag node to ipfs.",
@@ -258,6 +273,267 @@ var DagResolveCmd = &cmds.Command{
273 Type: ResolveOutput{},
274 }
275
276 +type importResult struct {
277 + roots map[cid.Cid]bool
278 + err error
279 +}
280 +
281 +var DagImportCmd = &cmds.Command{
282 + Helptext: cmds.HelpText{
283 + Tagline: "Import the contents of .car files",
284 + ShortDescription: `
285 +'ipfs dag import' imports all blocks present in supplied .car
286 +( Content Address aRchive ) files, recursively pinning any roots
287 +specified in the CAR file headers, unless --pin-roots is set to false.
288 +
289 +Note:
290 + This command will import all blocks in the CAR file, not just those
291 + reachable from the specified roots. However, these other blocks will
292 + not be pinned and may be garbage collected later.
293 +
294 + The pinning of the roots happens after all car files are processed,
295 + permitting import of DAGs spanning multiple files.
296 +
297 + Pinning takes place in offline-mode exclusively, one root at a time.
298 + If the combination of blocks from the imported CAR files and what is
299 + currently present in the blockstore does not represent a complete DAG,
300 + pinning of that individual root will fail.
301 +
302 +Maximum supported CAR version: 1
303 +`,
304 + },
305 + Arguments: []cmds.Argument{
306 + cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
307 + },
308 + Options: []cmds.Option{
309 + cmds.BoolOption(silentOptionName, "No output."),
310 + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
311 + },
312 + Type: CarImportOutput{},
313 + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
314 +
315 + node, err := cmdenv.GetNode(env)
316 + if err != nil {
317 + return err
318 + }
319 +
320 + api, err := cmdenv.GetApi(env, req)
321 + if err != nil {
322 + return err
323 + }
324 +
325 + // on import ensure we do not reach out to the network for any reason
326 + // if a pin based on what is imported + what is in the blockstore
327 + // isn't possible: tough luck
328 + api, err = api.WithOptions(options.Api.Offline(true))
329 + if err != nil {
330 + return err
331 + }
332 +
333 + // grab a pinlock ( which doubles as a GC lock ) so that regardless of the
334 + // size of the streamed-in cars nothing will disappear on us before we had
335 + // a chance to roots that may show up at the very end
336 + // This is especially important for use cases like dagger:
337 + // ipfs dag import $( ... | ipfs-dagger --stdout=carfifos )
338 + //
339 + unlocker := node.Blockstore.PinLock()
340 + defer unlocker.Unlock()
341 +
342 + doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
343 +
344 + retCh := make(chan importResult, 1)
345 + go importWorker(req, res, api, retCh)
346 +
347 + done := <-retCh
348 + if done.err != nil {
349 + return done.err
350 + }
351 +
352 + // It is not guaranteed that a root in a header is actually present in the same ( or any )
353 + // .car file. This is the case in version 1, and ideally in further versions too
354 + // Accumulate any root CID seen in a header, and supplement its actual node if/when encountered
355 + // We will attempt a pin *only* at the end in case all car files were well formed
356 + //
357 + // The boolean value indicates whether we have encountered the root within the car file's
358 + roots := done.roots
359 +
360 + // opportunistic pinning: try whatever sticks
361 + if doPinRoots {
362 +
363 + var failedPins int
364 + for c, seen := range roots {
365 +
366 + // We need to re-retrieve a block, convert it to ipld, and feed it
367 + // to the Pinning interface, sigh...
368 + //
369 + // If we didn't have the problem of inability to take multiple pinlocks,
370 + // we could use the Api directly like so (though internally it does the same):
371 + //
372 + // // not ideal, but the pinning api takes only paths :(
373 + // rp := path.NewResolvedPath(
374 + // ipfspath.FromCid(c),
375 + // c,
376 + // c,
377 + // "",
378 + // )
379 + //
380 + // if err := api.Pin().Add(req.Context, rp, options.Pin.Recursive(true)); err != nil {
381 +
382 + ret := RootMeta{Cid: c, PresentInImport: seen}
383 +
384 + if block, err := node.Blockstore.Get(c); err != nil {
385 + ret.PinErrorMsg = err.Error()
386 + } else if nd, err := ipld.Decode(block); err != nil {
387 + ret.PinErrorMsg = err.Error()
388 + } else if err := node.Pinning.Pin(req.Context, nd, true); err != nil {
389 + ret.PinErrorMsg = err.Error()
390 + } else if err := node.Pinning.Flush(req.Context); err != nil {
391 + ret.PinErrorMsg = err.Error()
392 + }
393 +
394 + if ret.PinErrorMsg != "" {
395 + failedPins++
396 + }
397 +
398 + if err := res.Emit(&CarImportOutput{Root: ret}); err != nil {
399 + return err
400 + }
401 + }
402 +
403 + if failedPins > 0 {
404 + return fmt.Errorf(
405 + "unable to pin all roots: %d out of %d failed",
406 + failedPins,
407 + len(roots),
408 + )
409 + }
410 + }
411 +
412 + return nil
413 + },
414 + Encoders: cmds.EncoderMap{
415 + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *CarImportOutput) error {
416 +
417 + silent, _ := req.Options[silentOptionName].(bool)
418 + if silent {
419 + return nil
420 + }
421 +
422 + enc, err := cmdenv.GetLowLevelCidEncoder(req)
423 + if err != nil {
424 + return err
425 + }
426 +
427 + if event.Root.PinErrorMsg != "" {
428 + event.Root.PinErrorMsg = fmt.Sprintf("FAILED: %s", event.Root.PinErrorMsg)
429 + } else {
430 + event.Root.PinErrorMsg = "success"
431 + }
432 +
433 + if !event.Root.PresentInImport {
434 + event.Root.PinErrorMsg += " (root specified in .car header without available data)"
435 + }
436 +
437 + _, err = fmt.Fprintf(
438 + w,
439 + "Pinned root\t%s\t%s\n",
440 + enc.Encode(event.Root.Cid),
441 + event.Root.PinErrorMsg,
442 + )
443 + return err
444 + }),
445 + },
446 +}
447 +
448 +func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, ret chan importResult) {
449 +
450 + // this is *not* a transaction
451 + // it is simply a way to relieve pressure on the blockstore
452 + // similar to pinner.Pin/pinner.Flush
453 + batch := ipld.NewBatch(req.Context, api.Dag())
454 +
455 + roots := make(map[cid.Cid]bool)
456 +
457 + it := req.Files.Entries()
458 + for it.Next() {
459 +
460 + file := files.FileFromEntry(it)
461 + if file == nil {
462 + ret <- importResult{err: errors.New("expected a file handle")}
463 + return
464 + }
465 +
466 + // wrap a defer-closer-scope
467 + //
468 + // every single file in it() is already open before we start
469 + // just close here sooner rather than later for neatness
470 + // and to surface potential erorrs writing on closed fifos
471 + // this won't/can't help with not running out of handles
472 + err := func() error {
473 + defer file.Close()
474 +
475 + car, err := gocar.NewCarReader(file)
476 + if err != nil {
477 + return err
478 + }
479 +
480 + // Be explicit here, until the spec is finished
481 + if car.Header.Version != 1 {
482 + return errors.New("only car files version 1 supported at present")
483 + }
484 +
485 + for _, c := range car.Header.Roots {
486 + if _, exists := roots[c]; !exists {
487 + roots[c] = false
488 + }
489 + }
490 +
491 + for {
492 + block, err := car.Next()
493 + if err != nil && err != io.EOF {
494 + return err
495 + } else if block == nil {
496 + break
497 + }
498 +
499 + // the double-decode is suboptimal, but we need it for batching
500 + nd, err := ipld.Decode(block)
501 + if err != nil {
502 + return err
503 + }
504 +
505 + if err := batch.Add(req.Context, nd); err != nil {
506 + return err
507 + }
508 +
509 + // encountered something known to be a root, for the first time
510 + if seen, exists := roots[nd.Cid()]; exists && !seen {
511 + roots[nd.Cid()] = true
512 + }
513 + }
514 +
515 + return nil
516 + }()
517 +
518 + if err != nil {
519 + ret <- importResult{err: err}
520 + return
521 + }
522 + }
523 +
524 + if err := it.Err(); err != nil {
525 + ret <- importResult{err: err}
526 + return
527 + }
528 +
529 + if err := batch.Commit(); err != nil {
530 + ret <- importResult{err: err}
531 + return
532 + }
533 +
534 + ret <- importResult{roots: roots}
535 +}
536 +
537 var DagExportCmd = &cmds.Command{
538 Helptext: cmds.HelpText{
539 Tagline: "Streams the selected DAG as a .car stream on stdout.",
go.mod
+1 -1
@@ -42,7 +42,7 @@ require (
42 github.com/ipfs/go-ipfs-routing v0.1.0
43 github.com/ipfs/go-ipfs-util v0.0.1
44 github.com/ipfs/go-ipld-cbor v0.0.4
45 - github.com/ipfs/go-ipld-format v0.0.2
45 + github.com/ipfs/go-ipld-format v0.2.0
46 github.com/ipfs/go-ipld-git v0.0.3
47 github.com/ipfs/go-ipns v0.0.2
48 github.com/ipfs/go-log v1.0.3
go.sum
+6 -2
@@ -18,10 +18,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
18 github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 h1:3ILjVyslFbc4jl1w5TWuvvslFD/nDfR2H8tVaMVLrEY=
19 github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
20 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
21 -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
21 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
22 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
24 -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
23 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
24 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
25 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -192,12 +190,15 @@ github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRM
190 github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So=
191 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
192 github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
193 +github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc=
194 github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
195 github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM=
196 github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
197 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
198 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
199 +github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
200 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
201 +github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
202 github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
203 github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
204 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
@@ -302,6 +303,8 @@ github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9
303 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
304 github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs=
305 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
306 +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA=
307 +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
308 github.com/ipfs/go-ipld-git v0.0.3 h1:/YjkjCyo5KYRpW+suby8Xh9Cm/iH9dAgGV6qyZ1dGus=
309 github.com/ipfs/go-ipld-git v0.0.3/go.mod h1:RuvMXa9qtJpDbqngyICCU/d+cmLFXxLsbIclmD0Lcr0=
310 github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs=
@@ -556,6 +559,7 @@ github.com/libp2p/go-libp2p-peerstore v0.2.0 h1:XcgJhI8WyUOCbHyRLNEX5542YNj8hnLS
559 github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ=
560 github.com/libp2p/go-libp2p-peerstore v0.2.1 h1:u+gOfsKgu73ZkGWhvckRm03z9C+iS9TrLqpANweELGs=
561 github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
562 +github.com/libp2p/go-libp2p-peerstore v0.2.2 h1:iqc/m03jHn5doXN3+kS6JKvqQRHEltiXljQB85iVHWE=
563 github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
564 github.com/libp2p/go-libp2p-peerstore v0.2.3 h1:MofRq2l3c15vQpEygTetV+zRRrncz+ktiXW7H2EKoEQ=
565 github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw=
test/sharness/t0054-dag-car-import-export-data/README.md new
+24
@@ -0,0 +1,24 @@
1 +# Dataset description/sources
2 +
3 +- lotus_testnet_export_256_multiroot.car
4 + - Export of the first 256 block of the testnet chain, with 3 tipset roots. Exported from Lotus by @traviperson on 2019-03-18
5 +
6 +
7 +- lotus_devnet_genesis.car
8 + - Source: https://github.com/filecoin-project/lotus/blob/v0.2.10/build/genesis/devnet.car
9 +
10 +- lotus_testnet_export_128.car
11 + - Export of the first 128 block of the testnet chain, exported from Lotus by @traviperson on 2019-03-24
12 +
13 +
14 +- lotus_devnet_genesis_shuffled_noroots.car
15 +- lotus_testnet_export_128_shuffled_noroots.car
16 + - versions of the above with an **empty** root array, and having all blocks shuffled
17 +
18 +- lotus_devnet_genesis_shuffled_nulroot.car
19 +- lotus_testnet_export_128_shuffled_nulroot.car
20 + - versions identical to the above, but with a single "empty-block" root each ( in order to work around go-car not following the current "roots can be empty" spec )
21 +
22 +- combined_naked_roots_genesis_and_128.car
23 + - only the roots of `lotus_devnet_genesis.car` and `lotus_testnet_export_128.car`, to to be used in combination with the root-less parts to validate "transactional" pinning
24 +
test/sharness/t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz
Binary files /dev/null and b/test/sharness/t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz differ
test/sharness/t0054-dag-car-import-export.sh
+154 -5
@@ -4,22 +4,171 @@
4 test_description="Test car file import/export functionality"
5
6 . lib/test-lib.sh
7 +export -f ipfsi
8
9 +set -o pipefail
10 +
11 +tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz
12 +
13 +reset_blockstore() {
14 + node=$1
15 + ipfsi $1 pin ls --quiet --type=recursive | ipfsi $1 pin rm &>/dev/null
16 + ipfsi $1 repo gc &>/dev/null
17 +
18 + test_expect_success "pinlist empty" '
19 + test -z "$( ipfsi $1 pin ls )"
20 + '
21 + test_expect_success "nothing left to gc" '
22 + test -z "$( ipfsi $1 repo gc )"
23 + '
24 +}
25 +
26 +# hammer with concurrent gc to ensure nothing clashes
27 +do_import() {
28 + node=$1; shift
29 +
30 + bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc1_pid=$!
31 + bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc2_pid=$!
32 +
33 + ipfsi $node dag import "$@"
34 +
35 + rm spin.gc
36 + wait $gc1_pid
37 + wait $gc2_pid
38 +}
39 +
40 +run_online_imp_exp_tests() {
41 +
42 + reset_blockstore 0
43 + reset_blockstore 1
44 +
45 + echo -e "Pinned root\tbafkqaaa\tsuccess (root specified in .car header without available data)" > basic_import_expected
46 + echo -e "Pinned root\tbafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u\tsuccess" >> basic_import_expected
47 + echo -e "Pinned root\tbafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy\tsuccess" >> basic_import_expected
48 +
49 + touch spin.gc
50 + test_expect_success "basic import" '
51 + do_import 0 \
52 + ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
53 + ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \
54 + ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \
55 + | sort > basic_import_actual
56 + '
57 +
58 + # FIXME - the fact we reliably fail this is indicative of some sort of race...
59 + test_expect_failure "concurrent GC did not manage to find anything" '
60 + ! [[ -s gc_out ]]
61 + '
62 + test_expect_success "basic import output as expected" '
63 + test_cmp basic_import_expected basic_import_actual
64 + '
65 +
66 + reset_blockstore 0
67 + reset_blockstore 1
68 +
69 + mkfifo pipe_testnet
70 + mkfifo pipe_devnet
71 +
72 + # test that ipfs correctly opens both pipes and deleting them doesn't interfere with cleanup
73 + bash -c '
74 + sleep 1
75 + cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet & cat1_pid=$!
76 + cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet & cat2_pid=$!
77 +
78 + rm pipe_testnet pipe_devnet
79 +
80 + # extra safety valve to kill the cat processes in case something goes wrong
81 + bash -c "sleep 60; kill $cat1_pid $cat2_pid 2>/dev/null" &
82 + ' &
83 +
84 + touch spin.gc
85 + test_expect_success "fifo import" '
86 + do_import 0 \
87 + pipe_testnet \
88 + pipe_devnet \
89 + ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
90 + | sort > basic_fifo_import_actual
91 + '
92 + # FIXME - the fact we reliably fail this is indicative of some sort of race...
93 + test_expect_failure "concurrent GC did not manage to grab anything" '
94 + ! [[ -s gc_out ]]
95 + '
96 +
97 + test_expect_success "fifo-import output as expected" '
98 + test_cmp basic_import_expected basic_fifo_import_actual
99 + '
100 +
101 + test_expect_success "fifos no longer present" '
102 + ! [[ -e pipe_testnet ]] && ! [[ -e pipe_devnet ]]
103 + '
104 +}
105 +
106 +
107 +test_expect_success "set up testbed" '
108 + iptb testbed create -type localipfs -count 2 -force -init
109 +'
110 +startup_cluster 2
111 +
112 +run_online_imp_exp_tests
113 +
114 +test_expect_success "shut down nodes" '
115 + iptb stop && iptb_wait_stop
116 +'
117 +
118 +
119 +# We want to just init the repo, without using a daemon for stuff below
120 test_init_ipfs
121
122
11 -echo "Error: merkledag: not found (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected
123 +test_expect_success "basic offline export of 'getting started' dag works" '
124 + ipfs dag export QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv >/dev/null
125 +'
126 +
127
128 +echo "Error: merkledag: not found (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected
129 test_expect_success "basic offline export of nonexistent cid" '
14 - ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual
130 + ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null
131 '
16 -
132 test_expect_success "correct error" '
133 test_cmp offline_fetch_error_expected offline_fetch_error_actual
134 '
135
21 -test_expect_success "basic offline export of 'getting started' dag" '
22 - ipfs dag export QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv >/dev/null
136 +
137 +cat >multiroot_import_expected <<EOE
138 +{"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PresentInImport":true,"PinErrorMsg":""}}
139 +{"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PresentInImport":true,"PinErrorMsg":""}}
140 +{"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PresentInImport":true,"PinErrorMsg":""}}
141 +EOE
142 +test_expect_success "multiroot import works" '
143 + ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_actual
144 +'
145 +test_expect_success "multiroot import expected output" '
146 + test_cmp multiroot_import_expected multiroot_import_actual
147 +'
148 +
149 +
150 +test_expect_success "pin-less import works" '
151 + ipfs dag import --enc=json --pin-roots=false \
152 + ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
153 + ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
154 + > no-pin_import_actual
155 +'
156 +test_expect_success "expected silence on --pin-roots=false" '
157 + ! [[ -s no-pin_import_actual ]]
158 '
159
160 +
161 +cat >naked_root_import_expected <<EOE
162 +{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PresentInImport":false,"PinErrorMsg":""}}
163 +{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PresentInImport":false,"PinErrorMsg":""}}
164 +EOE
165 +test_expect_success "naked root import works" '
166 + ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
167 + | sort > naked_root_import_actual
168 +'
169 +test_expect_success "naked root import expected output" '
170 + test_cmp naked_root_import_expected naked_root_import_actual
171 +'
172 +
173 +
174 test_done