chore: replace go-merkledag walk with go-ipld-prime traversal for dag export (#8506)
* chore: replace go-merkledag walk with go-ipld-prime traversal for dag export This is "safe" now because we can limit duplicate block loads like go-merkledag does and won't get trapped taking a long time for complex DAGs. We can do this while we're using an exhaustive selector (like ExploreAll here) but will need an alternative strategy when we go for arbitrary selectors.
Rod Vagg committed
Oct 27, 2021 at 05:07 UTC
9e8b6e5b50f5dc00aadc7ed9d6b9399ee609f31b
3 files changed
+26
-35
core/commands/dag/export.go
+20
-29
@@ -1,6 +1,7 @@
1
package dagcmd
2
3
import (
4
+ "context"
5
"errors"
6
"fmt"
7
"io"
@@ -8,17 +9,18 @@ import (
9
"time"
10
11
"github.com/cheggaaa/pb"
12
+ blocks "github.com/ipfs/go-block-format"
13
cid "github.com/ipfs/go-cid"
14
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
15
ipld "github.com/ipfs/go-ipld-format"
14
- mdag "github.com/ipfs/go-merkledag"
16
+ iface "github.com/ipfs/interface-go-ipfs-core"
17
18
cmds "github.com/ipfs/go-ipfs-cmds"
19
gocar "github.com/ipld/go-car"
20
+ selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
21
)
22
23
func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
21
-
24
c, err := cid.Decode(req.Arguments[0])
25
if err != nil {
26
return fmt.Errorf(
@@ -32,24 +34,6 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
34
return err
35
}
36
35
- // Code disabled until descent-issue in go-ipld-prime is fixed
36
- // https://github.com/ribasushi/gip-muddle-up
37
- //
38
- // sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder())
39
- // car := gocar.NewSelectiveCar(
40
- // req.Context,
41
- // <needs to be fixed to take format.NodeGetter as well>,
42
- // []gocar.Dag{gocar.Dag{
43
- // Root: c,
44
- // Selector: sb.ExploreRecursive(
45
- // gipselector.RecursionLimitNone(),
46
- // sb.ExploreAll(sb.ExploreRecursiveEdge()),
47
- // ).Node(),
48
- // }},
49
- // )
50
- // ...
51
- // if err := car.Write(pipeW); err != nil {}
52
-
37
pipeR, pipeW := io.Pipe()
38
39
errCh := make(chan error, 2) // we only report the 1st error
@@ -61,15 +45,12 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
45
close(errCh)
46
}()
47
64
- if err := gocar.WriteCar(
65
- req.Context,
66
- mdag.NewSession(
67
- req.Context,
68
- api.Dag(),
69
- ),
70
- []cid.Cid{c},
71
- pipeW,
72
- ); err != nil {
48
+ store := dagStore{dag: api.Dag(), ctx: req.Context}
49
+ dag := gocar.Dag{Root: c, Selector: selectorparse.CommonSelector_ExploreAllRecursively}
50
+ // TraverseLinksOnlyOnce is safe for an exhaustive selector but won't be when we allow
51
+ // arbitrary selectors here
52
+ car := gocar.NewSelectiveCar(req.Context, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce())
53
+ if err := car.Write(pipeW); err != nil {
54
errCh <- err
55
}
56
}()
@@ -153,3 +134,13 @@ func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error {
134
}
135
}
136
}
137
+
138
+type dagStore struct {
139
+ dag iface.APIDagService
140
+ ctx context.Context
141
+}
142
+
143
+func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) {
144
+ obj, err := ds.dag.Get(ds.ctx, c)
145
+ return obj, err
146
+}
go.mod
+2
-2
@@ -58,9 +58,9 @@ require (
58
github.com/ipfs/go-verifcid v0.0.1
59
github.com/ipfs/interface-go-ipfs-core v0.5.1
60
github.com/ipfs/tar-utils v0.0.1
61
- github.com/ipld/go-car v0.3.1
61
+ github.com/ipld/go-car v0.3.2
62
github.com/ipld/go-codec-dagpb v1.3.0
63
- github.com/ipld/go-ipld-prime v0.12.2
63
+ github.com/ipld/go-ipld-prime v0.12.3
64
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
65
github.com/jbenet/go-temp-err-catcher v0.1.0
66
github.com/jbenet/goprocess v0.1.4
go.sum
+4
-4
@@ -571,16 +571,16 @@ github.com/ipfs/interface-go-ipfs-core v0.5.1 h1:1KMM7RkjUD8W5fSoRsa9xR6ZMzeL8fL
571
github.com/ipfs/interface-go-ipfs-core v0.5.1/go.mod h1:lNBJrdXHtWS46evMPBdWtDQMDsrKcGbxCOGoKLkztOE=
572
github.com/ipfs/tar-utils v0.0.1 h1:8Na0KBD6GddGyXwU4rXNtVTE24iuZws8mENJQPLG7W4=
573
github.com/ipfs/tar-utils v0.0.1/go.mod h1:ACflm9wXvV9w0eMJt6yYXxS2zuIV+yXGNwbuq1bhLeE=
574
-github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs=
575
-github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs=
574
+github.com/ipld/go-car v0.3.2 h1:V9wt/80FNfbMRWSD98W5br6fyjUAyVgI2lDOTZX16Lg=
575
+github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE=
576
github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
577
github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8=
578
github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA=
579
github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
580
github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
581
github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
582
-github.com/ipld/go-ipld-prime v0.12.2 h1:StIquYvKIRuSEAtjJDr39fyzBtziioHPwVC75tBiXzo=
583
-github.com/ipld/go-ipld-prime v0.12.2/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8=
582
+github.com/ipld/go-ipld-prime v0.12.3 h1:furVobw7UBLQZwlEwfE26tYORy3PAK8VYSgZOSr3JMQ=
583
+github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8=
584
github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
585
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
586
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=