Dag export command, silent, no progress
Peter Rabbitson committed
Mar 4, 2020 at 16:11 UTC
635e333d609ee53791f0415efcc8b957d7abcf3b
7 files changed
+144
-5
.gitattributes
+1
@@ -9,6 +9,7 @@ LICENSE text eol=auto
9
*.png binary
10
*.tar binary
11
*.gz binary
12
+*.car binary
13
14
# Binary assets
15
assets/init-doc/* binary
core/commands/commands_test.go
+2
@@ -24,6 +24,7 @@ func TestROCommands(t *testing.T) {
24
"/commands",
25
"/dag",
26
"/dag/get",
27
+ "/dag/export",
28
"/dag/resolve",
29
"/dns",
30
"/get",
@@ -95,6 +96,7 @@ func TestCommands(t *testing.T) {
96
"/config/profile/apply",
97
"/dag",
98
"/dag/get",
99
+ "/dag/export",
100
"/dag/put",
101
"/dag/resolve",
102
"/dht",
core/commands/dag/dag.go
+108
-1
@@ -14,16 +14,22 @@ import (
14
cmds "github.com/ipfs/go-ipfs-cmds"
15
files "github.com/ipfs/go-ipfs-files"
16
ipld "github.com/ipfs/go-ipld-format"
17
+ mdag "github.com/ipfs/go-merkledag"
18
ipfspath "github.com/ipfs/go-path"
19
path "github.com/ipfs/interface-go-ipfs-core/path"
20
mh "github.com/multiformats/go-multihash"
21
+
22
+ gocar "github.com/ipld/go-car"
23
+ //gipfree "github.com/ipld/go-ipld-prime/impl/free"
24
+ //gipselector "github.com/ipld/go-ipld-prime/traversal/selector"
25
+ //gipselectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder"
26
)
27
28
var DagCmd = &cmds.Command{
29
Helptext: cmds.HelpText{
30
Tagline: "Interact with ipld dag objects.",
31
ShortDescription: `
26
-'ipfs dag' is used for creating and manipulating dag objects.
32
+'ipfs dag' is used for creating and manipulating dag objects/hierarchies.
33
34
This subcommand is currently an experimental feature, but it is intended
35
to deprecate and replace the existing 'ipfs object' command moving forward.
@@ -33,6 +39,7 @@ to deprecate and replace the existing 'ipfs object' command moving forward.
39
"put": DagPutCmd,
40
"get": DagGetCmd,
41
"resolve": DagResolveCmd,
42
+ "export": DagExportCmd,
43
},
44
}
45
@@ -241,3 +248,103 @@ var DagResolveCmd = &cmds.Command{
248
},
249
Type: ResolveOutput{},
250
}
251
+
252
+var DagExportCmd = &cmds.Command{
253
+ Helptext: cmds.HelpText{
254
+ Tagline: "Streams the selected DAG as a .car stream on stdout.",
255
+ ShortDescription: `
256
+'ipfs dag export' fetches a dag and streams it out as a well-formed .car file.
257
+Note that at prsent only single root selections / .car files are supported.
258
+The output of blocks happens in strict DAG-traversal, first-seen, order.
259
+`,
260
+ },
261
+ Arguments: []cmds.Argument{
262
+ cmds.StringArg("root", true, false, "Expression evaluting to a single root of a dag to export").EnableStdin(),
263
+ },
264
+ Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
265
+
266
+ c, err := cid.Decode(req.Arguments[0])
267
+ if err != nil {
268
+ return fmt.Errorf(
269
+ "unable to parse root specification (currently only bare CIDs are supported): %s",
270
+ err,
271
+ )
272
+ }
273
+
274
+ // The current interface of go-car is rather suboptimal as it
275
+ // only takes a blockstore, instead of accepting a dagservice,
276
+ // and leveraging parallel-fetch capabilities
277
+ // https://github.com/ipld/go-car/issues/27
278
+ //
279
+ // Until the above is fixed, pre-warm the blockstore before doing
280
+ // anything else. We explicitly *DO NOT* take a lock during this
281
+ // operation: even if we lose some of the blocks we just received
282
+ // due to a conflicting GC: we will just re-retrieve anything we
283
+ // potentially lost when the car is being streamed out
284
+ node, err := cmdenv.GetNode(env)
285
+ if err != nil {
286
+ return err
287
+ }
288
+
289
+ if err := mdag.FetchGraph(req.Context, c, node.DAG); err != nil {
290
+ if !node.IsOnline {
291
+ err = fmt.Errorf("%s (currently offline, perhaps retry after attaching to the network)", err)
292
+ }
293
+ return err
294
+ }
295
+
296
+ // Code disabled until descent-issue in go-ipld-prime is fixed
297
+ //
298
+ // The second part of the above - make a super-thin wrapper around
299
+ // a blockservice session, translating Session.GetBlock() to Blockstore.Get()
300
+ //
301
+ // sess := blockservice.NewSession(
302
+ // req.Context,
303
+ // node.Blocks,
304
+ // )
305
+ // var wrapper getBlockFromSessionWrapper = func(c cid.Cid) (blk.Block, error) {
306
+ // return sess.GetBlock(req.Context, c)
307
+ // }
308
+ // sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder())
309
+ // car := gocar.NewSelectiveCar(
310
+ // req.Context,
311
+ // &wrapper,
312
+ // []gocar.Dag{gocar.Dag{
313
+ // Root: c,
314
+ // Selector: sb.ExploreRecursive(
315
+ // gipselector.RecursionLimitNone(),
316
+ // sb.ExploreAll(sb.ExploreRecursiveEdge()),
317
+ // ).Node(),
318
+ // }},
319
+ // )
320
+
321
+ pipeR, pipeW := io.Pipe()
322
+
323
+ errCh := make(chan error, 2) // we only report the 1st error
324
+ go func() {
325
+ defer func() {
326
+ if err := pipeW.Close(); err != nil {
327
+ errCh <- fmt.Errorf("stream flush failed: %s", err)
328
+ }
329
+ close(errCh)
330
+ }()
331
+
332
+ //if err := car.Write(pipeW); err != nil {
333
+ if err := gocar.WriteCar(
334
+ req.Context,
335
+ node.DAG,
336
+ []cid.Cid{c},
337
+ pipeW,
338
+ ); err != nil {
339
+ errCh <- err
340
+ }
341
+ }()
342
+
343
+ if err := res.Emit(pipeR); err != nil {
344
+ pipeW.Close() // ignore errors if any
345
+ return err
346
+ }
347
+
348
+ return <-errCh
349
+ },
350
+}
core/commands/root.go
+1
@@ -190,6 +190,7 @@ var rootROSubcommands = map[string]*cmds.Command{
190
"dag": {
191
Subcommands: map[string]*cmds.Command{
192
"get": dag.DagGetCmd,
193
+ "export": dag.DagExportCmd,
194
"resolve": dag.DagResolveCmd,
195
},
196
},
go.mod
+1
@@ -54,6 +54,7 @@ require (
54
github.com/ipfs/go-unixfs v0.2.4
55
github.com/ipfs/go-verifcid v0.0.1
56
github.com/ipfs/interface-go-ipfs-core v0.2.6
57
+ github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab
58
github.com/jbenet/go-is-domain v1.0.3
59
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
60
github.com/jbenet/go-temp-err-catcher v0.1.0
go.sum
+6
-4
@@ -18,8 +18,10 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
18
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 h1:3ILjVyslFbc4jl1w5TWuvvslFD/nDfR2H8tVaMVLrEY=
19
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
20
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
21
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
22
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
23
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
24
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
25
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
26
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
27
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -190,15 +192,12 @@ github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRM
192
github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So=
193
github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
194
github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
193
-github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc=
195
github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
196
github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM=
197
github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
198
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
199
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
199
-github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
200
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
201
-github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
201
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
202
github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
203
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
@@ -326,6 +325,7 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy
325
github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
326
github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
327
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
328
+github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
329
github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0=
330
github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
331
github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ=
@@ -354,6 +354,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2
354
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
355
github.com/ipfs/interface-go-ipfs-core v0.2.6 h1:4eeGPJUDWblEurSzpAtL2znDG1xqAoX2aNNIoOFwyuc=
356
github.com/ipfs/interface-go-ipfs-core v0.2.6/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
357
+github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab h1:+3Y6Jb3IBmG3t6e3r6TItnuciOaMOuGW7QIVEUa5vy4=
358
+github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab/go.mod h1:yR5AsJ38xTwwgwGpbh60ICtdLPp5lGfuH28PAAzaEhM=
359
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs=
360
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
361
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4=
@@ -554,7 +556,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.0 h1:XcgJhI8WyUOCbHyRLNEX5542YNj8hnLS
556
github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ=
557
github.com/libp2p/go-libp2p-peerstore v0.2.1 h1:u+gOfsKgu73ZkGWhvckRm03z9C+iS9TrLqpANweELGs=
558
github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
557
-github.com/libp2p/go-libp2p-peerstore v0.2.2 h1:iqc/m03jHn5doXN3+kS6JKvqQRHEltiXljQB85iVHWE=
559
github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
560
github.com/libp2p/go-libp2p-peerstore v0.2.3 h1:MofRq2l3c15vQpEygTetV+zRRrncz+ktiXW7H2EKoEQ=
561
github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw=
@@ -902,6 +903,7 @@ github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.
903
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
904
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
905
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
906
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
907
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
908
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
909
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
test/sharness/t0054-dag-car-import-export.sh
new
+25
@@ -0,0 +1,25 @@
1
+#!/usr/bin/env bash
2
+#
3
+
4
+test_description="Test car file import/export functionality"
5
+
6
+. lib/test-lib.sh
7
+
8
+test_init_ipfs
9
+
10
+
11
+echo "Error: merkledag: not found (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected
12
+
13
+test_expect_success "basic offline export of nonexistent cid" '
14
+ ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual
15
+'
16
+
17
+test_expect_success "correct error" '
18
+ test_cmp offline_fetch_error_expected offline_fetch_error_actual
19
+'
20
+
21
+test_expect_success "basic offline export of 'getting started' dag" '
22
+ ipfs dag export QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv >/dev/null
23
+'
24
+
25
+test_done