pin: follow async pinner changes
See https://github.com/ipfs/boxo/pull/290 This PR follow the changes in the Pinner to make listing recursive and direct pins asynchronous, which in turns allow pin/ls to build and emit results without having to wait anything, or accumulate too much in memory. Note: there is a tradeoff for pin/ls?type=all: - keep the recursive pins in memory (which I chose) - ask the pinner twice for the recursive pins, and limit memory usage Also, follow the changes in the GC with similar benefit of not having to wait the full pin list. Add a test. Also, follow the changes in pin.Verify.
Michael Muré committed
May 4, 2023 at 13:50 UTC
a197125b8f3f85cc5070678f19b4d5853f3ff4d6
6 files changed
+224
-96
core/commands/pin/pin.go
+7
-7
@@ -675,10 +675,6 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc ci
675
bs := n.Blocks.Blockstore()
676
DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
677
getLinks := dag.GetLinksWithDAG(DAG)
678
- recPins, err := n.Pinning.RecursiveKeys(ctx)
679
- if err != nil {
680
- return nil, err
681
- }
678
679
var checkPin func(root cid.Cid) PinStatus
680
checkPin = func(root cid.Cid) PinStatus {
@@ -722,11 +718,15 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc ci
718
out := make(chan interface{})
719
go func() {
720
defer close(out)
725
- for _, cid := range recPins {
726
- pinStatus := checkPin(cid)
721
+ for p := range n.Pinning.RecursiveKeys(ctx) {
722
+ if p.Err != nil {
723
+ out <- p.Err
724
+ return
725
+ }
726
+ pinStatus := checkPin(p.C)
727
if !pinStatus.Ok || opts.includeOk {
728
select {
729
- case out <- &PinVerifyRes{enc.Encode(cid), pinStatus}:
729
+ case out <- &PinVerifyRes{enc.Encode(p.C), pinStatus}:
730
case <-ctx.Done():
731
return
732
}
core/coreapi/pin.go
+73
-59
@@ -12,9 +12,10 @@ import (
12
"github.com/ipfs/boxo/ipld/merkledag"
13
pin "github.com/ipfs/boxo/pinning/pinner"
14
"github.com/ipfs/go-cid"
15
- "github.com/ipfs/kubo/tracing"
15
"go.opentelemetry.io/otel/attribute"
16
"go.opentelemetry.io/otel/trace"
17
+
18
+ "github.com/ipfs/kubo/tracing"
19
)
20
21
type PinAPI CoreAPI
@@ -156,6 +157,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt
157
}
158
159
type pinStatus struct {
160
+ err error
161
cid cid.Cid
162
ok bool
163
badNodes []coreiface.BadPinNode
@@ -175,6 +177,10 @@ func (s *pinStatus) BadNodes() []coreiface.BadPinNode {
177
return s.badNodes
178
}
179
180
+func (s *pinStatus) Err() error {
181
+ return s.err
182
+}
183
+
184
func (n *badNode) Path() path.Resolved {
185
return n.path
186
}
@@ -191,10 +197,6 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro
197
bs := api.blockstore
198
DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
199
getLinks := merkledag.GetLinksWithDAG(DAG)
194
- recPins, err := api.pinning.RecursiveKeys(ctx)
195
- if err != nil {
196
- return nil, err
197
- }
200
201
var checkPin func(root cid.Cid) *pinStatus
202
checkPin = func(root cid.Cid) *pinStatus {
@@ -229,8 +231,18 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro
231
out := make(chan coreiface.PinStatus)
232
go func() {
233
defer close(out)
232
- for _, c := range recPins {
233
- out <- checkPin(c)
234
+ for p := range api.pinning.RecursiveKeys(ctx) {
235
+ var res *pinStatus
236
+ if p.Err != nil {
237
+ res = &pinStatus{err: p.Err}
238
+ } else {
239
+ res = checkPin(p.C)
240
+ }
241
+ select {
242
+ case <-ctx.Done():
243
+ return
244
+ case out <- res:
245
+ }
246
}
247
}()
248
@@ -262,63 +274,57 @@ func (p *pinInfo) Err() error {
274
func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreiface.Pin {
275
out := make(chan coreiface.Pin, 1)
276
265
- keys := cid.NewSet()
266
-
267
- AddToResultKeys := func(keyList []cid.Cid, typeStr string) error {
268
- for _, c := range keyList {
269
- if keys.Visit(c) {
270
- select {
271
- case out <- &pinInfo{
272
- pinType: typeStr,
273
- path: path.IpldPath(c),
274
- }:
275
- case <-ctx.Done():
276
- return ctx.Err()
277
- }
277
+ emittedSet := cid.NewSet()
278
+
279
+ AddToResultKeys := func(c cid.Cid, typeStr string) error {
280
+ if emittedSet.Visit(c) {
281
+ select {
282
+ case out <- &pinInfo{
283
+ pinType: typeStr,
284
+ path: path.IpldPath(c),
285
+ }:
286
+ case <-ctx.Done():
287
+ return ctx.Err()
288
}
289
}
290
return nil
291
}
292
283
- VisitKeys := func(keyList []cid.Cid) {
284
- for _, c := range keyList {
285
- keys.Visit(c)
286
- }
287
- }
288
-
293
go func() {
294
defer close(out)
295
292
- var dkeys, rkeys []cid.Cid
296
+ var rkeys []cid.Cid
297
var err error
298
if typeStr == "recursive" || typeStr == "all" {
295
- rkeys, err = api.pinning.RecursiveKeys(ctx)
296
- if err != nil {
297
- out <- &pinInfo{err: err}
298
- return
299
- }
300
- if err = AddToResultKeys(rkeys, "recursive"); err != nil {
301
- out <- &pinInfo{err: err}
302
- return
299
+ for streamedCid := range api.pinning.RecursiveKeys(ctx) {
300
+ if streamedCid.Err != nil {
301
+ out <- &pinInfo{err: streamedCid.Err}
302
+ return
303
+ }
304
+ if err = AddToResultKeys(streamedCid.C, "recursive"); err != nil {
305
+ out <- &pinInfo{err: err}
306
+ return
307
+ }
308
}
309
}
310
if typeStr == "direct" || typeStr == "all" {
306
- dkeys, err = api.pinning.DirectKeys(ctx)
307
- if err != nil {
308
- out <- &pinInfo{err: err}
309
- return
310
- }
311
- if err = AddToResultKeys(dkeys, "direct"); err != nil {
312
- out <- &pinInfo{err: err}
313
- return
311
+ for streamedCid := range api.pinning.DirectKeys(ctx) {
312
+ if streamedCid.Err != nil {
313
+ out <- &pinInfo{err: streamedCid.Err}
314
+ return
315
+ }
316
+ if err = AddToResultKeys(streamedCid.C, "direct"); err != nil {
317
+ out <- &pinInfo{err: err}
318
+ return
319
+ }
320
}
321
}
322
if typeStr == "all" {
317
- set := cid.NewSet()
323
+ walkingSet := cid.NewSet()
324
for _, k := range rkeys {
325
err = merkledag.Walk(
326
ctx, merkledag.GetLinksWithDAG(api.dag), k,
321
- set.Visit,
327
+ walkingSet.Visit,
328
merkledag.SkipRoot(), merkledag.Concurrent(),
329
)
330
if err != nil {
@@ -326,7 +332,10 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreifac
332
return
333
}
334
}
329
- if err = AddToResultKeys(set.Keys(), "indirect"); err != nil {
335
+ err = walkingSet.ForEach(func(c cid.Cid) error {
336
+ return AddToResultKeys(c, "indirect")
337
+ })
338
+ if err != nil {
339
out <- &pinInfo{err: err}
340
return
341
}
@@ -335,25 +344,27 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreifac
344
// We need to first visit the direct pins that have priority
345
// without emitting them
346
338
- dkeys, err = api.pinning.DirectKeys(ctx)
339
- if err != nil {
340
- out <- &pinInfo{err: err}
341
- return
347
+ for streamedCid := range api.pinning.DirectKeys(ctx) {
348
+ if streamedCid.Err != nil {
349
+ out <- &pinInfo{err: streamedCid.Err}
350
+ return
351
+ }
352
+ emittedSet.Add(streamedCid.C)
353
}
343
- VisitKeys(dkeys)
354
345
- rkeys, err = api.pinning.RecursiveKeys(ctx)
346
- if err != nil {
347
- out <- &pinInfo{err: err}
348
- return
355
+ for streamedCid := range api.pinning.RecursiveKeys(ctx) {
356
+ if streamedCid.Err != nil {
357
+ out <- &pinInfo{err: streamedCid.Err}
358
+ return
359
+ }
360
+ emittedSet.Add(streamedCid.C)
361
}
350
- VisitKeys(rkeys)
362
352
- set := cid.NewSet()
363
+ walkingSet := cid.NewSet()
364
for _, k := range rkeys {
365
err = merkledag.Walk(
366
ctx, merkledag.GetLinksWithDAG(api.dag), k,
356
- set.Visit,
367
+ walkingSet.Visit,
368
merkledag.SkipRoot(), merkledag.Concurrent(),
369
)
370
if err != nil {
@@ -361,7 +372,10 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreifac
372
return
373
}
374
}
364
- if err = AddToResultKeys(set.Keys(), "indirect"); err != nil {
375
+ err = emittedSet.ForEach(func(c cid.Cid) error {
376
+ return AddToResultKeys(c, "indirect")
377
+ })
378
+ if err != nil {
379
out <- &pinInfo{err: err}
380
return
381
}
gc/gc.go
+43
-28
@@ -154,7 +154,7 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
154
// Descendants recursively finds all the descendants of the given roots and
155
// adds them to the given cid.Set, using the provided dag.GetLinks function
156
// to walk the tree.
157
-func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots []cid.Cid) error {
157
+func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots <-chan pin.StreamedCid) error {
158
verifyGetLinks := func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) {
159
err := verifcid.ValidateCid(c)
160
if err != nil {
@@ -167,7 +167,7 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
167
verboseCidError := func(err error) error {
168
if strings.Contains(err.Error(), verifcid.ErrBelowMinimumHashLength.Error()) ||
169
strings.Contains(err.Error(), verifcid.ErrPossiblyInsecureHashFunction.Error()) {
170
- err = fmt.Errorf("\"%s\"\nPlease run 'ipfs pin verify'"+ //nolint
170
+ err = fmt.Errorf("\"%s\"\nPlease run 'ipfs pin verify'"+ // nolint
171
" to list insecure hashes. If you want to read them,"+
172
" please downgrade your go-ipfs to 0.4.13\n", err)
173
log.Error(err)
@@ -175,19 +175,29 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
175
return err
176
}
177
178
- for _, c := range roots {
179
- // Walk recursively walks the dag and adds the keys to the given set
180
- err := dag.Walk(ctx, verifyGetLinks, c, func(k cid.Cid) bool {
181
- return set.Visit(toCidV1(k))
182
- }, dag.Concurrent())
178
+ for {
179
+ select {
180
+ case <-ctx.Done():
181
+ return ctx.Err()
182
+ case wrapper, ok := <-roots:
183
+ if !ok {
184
+ return nil
185
+ }
186
+ if wrapper.Err != nil {
187
+ return wrapper.Err
188
+ }
189
184
- if err != nil {
185
- err = verboseCidError(err)
186
- return err
190
+ // Walk recursively walks the dag and adds the keys to the given set
191
+ err := dag.Walk(ctx, verifyGetLinks, wrapper.C, func(k cid.Cid) bool {
192
+ return set.Visit(toCidV1(k))
193
+ }, dag.Concurrent())
194
+
195
+ if err != nil {
196
+ err = verboseCidError(err)
197
+ return err
198
+ }
199
}
200
}
189
-
190
- return nil
201
}
202
203
// toCidV1 converts any CIDv0s to CIDv1s.
@@ -217,11 +227,8 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
227
}
228
return links, nil
229
}
220
- rkeys, err := pn.RecursiveKeys(ctx)
221
- if err != nil {
222
- return nil, err
223
- }
224
- err = Descendants(ctx, getLinks, gcs, rkeys)
230
+ rkeys := pn.RecursiveKeys(ctx)
231
+ err := Descendants(ctx, getLinks, gcs, rkeys)
232
if err != nil {
233
errors = true
234
select {
@@ -243,7 +250,18 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
250
}
251
return links, nil
252
}
246
- err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRoots)
253
+ bestEffortRootsChan := make(chan pin.StreamedCid)
254
+ go func() {
255
+ defer close(bestEffortRootsChan)
256
+ for _, root := range bestEffortRoots {
257
+ select {
258
+ case <-ctx.Done():
259
+ return
260
+ case bestEffortRootsChan <- pin.StreamedCid{C: root}:
261
+ }
262
+ }
263
+ }()
264
+ err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRootsChan)
265
if err != nil {
266
errors = true
267
select {
@@ -253,18 +271,15 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
271
}
272
}
273
256
- dkeys, err := pn.DirectKeys(ctx)
257
- if err != nil {
258
- return nil, err
259
- }
260
- for _, k := range dkeys {
261
- gcs.Add(toCidV1(k))
274
+ dkeys := pn.DirectKeys(ctx)
275
+ for k := range dkeys {
276
+ if k.Err != nil {
277
+ return nil, k.Err
278
+ }
279
+ gcs.Add(toCidV1(k.C))
280
}
281
264
- ikeys, err := pn.InternalPins(ctx)
265
- if err != nil {
266
- return nil, err
267
- }
282
+ ikeys := pn.InternalPins(ctx)
283
err = Descendants(ctx, getLinks, gcs, ikeys)
284
if err != nil {
285
errors = true
gc/gc_test.go
new
+96
@@ -0,0 +1,96 @@
1
+package gc
2
+
3
+import (
4
+ "context"
5
+ "testing"
6
+
7
+ "github.com/ipfs/boxo/blockservice"
8
+ "github.com/ipfs/boxo/blockstore"
9
+ "github.com/ipfs/boxo/exchange/offline"
10
+ "github.com/ipfs/boxo/ipld/merkledag"
11
+ mdutils "github.com/ipfs/boxo/ipld/merkledag/test"
12
+ pin "github.com/ipfs/boxo/pinning/pinner"
13
+ "github.com/ipfs/boxo/pinning/pinner/dspinner"
14
+ "github.com/ipfs/go-cid"
15
+ "github.com/ipfs/go-datastore"
16
+ dssync "github.com/ipfs/go-datastore/sync"
17
+ "github.com/multiformats/go-multihash"
18
+ "github.com/stretchr/testify/require"
19
+)
20
+
21
+func TestGC(t *testing.T) {
22
+ ctx := context.Background()
23
+
24
+ ds := dssync.MutexWrap(datastore.NewMapDatastore())
25
+ bs := blockstore.NewGCBlockstore(blockstore.NewBlockstore(ds), blockstore.NewGCLocker())
26
+ bserv := blockservice.New(bs, offline.Exchange(bs))
27
+ dserv := merkledag.NewDAGService(bserv)
28
+ pinner, err := dspinner.New(ctx, ds, dserv)
29
+ require.NoError(t, err)
30
+
31
+ daggen := mdutils.NewDAGGenerator()
32
+
33
+ var expectedKept []multihash.Multihash
34
+ var expectedDiscarded []multihash.Multihash
35
+
36
+ // add some pins
37
+ for i := 0; i < 5; i++ {
38
+ // direct
39
+ root, _, err := daggen.MakeDagNode(dserv.Add, 0, 1)
40
+ require.NoError(t, err)
41
+ err = pinner.PinWithMode(ctx, root, pin.Direct)
42
+ require.NoError(t, err)
43
+ expectedKept = append(expectedKept, root.Hash())
44
+
45
+ // recursive
46
+ root, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2)
47
+ require.NoError(t, err)
48
+ err = pinner.PinWithMode(ctx, root, pin.Recursive)
49
+ require.NoError(t, err)
50
+ expectedKept = append(expectedKept, toMHs(allCids)...)
51
+ }
52
+
53
+ err = pinner.Flush(ctx)
54
+ require.NoError(t, err)
55
+
56
+ // add more dags to be GCed
57
+ for i := 0; i < 5; i++ {
58
+ _, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2)
59
+ require.NoError(t, err)
60
+ expectedDiscarded = append(expectedDiscarded, toMHs(allCids)...)
61
+ }
62
+
63
+ // and some other as "best effort roots"
64
+ var bestEffortRoots []cid.Cid
65
+ for i := 0; i < 5; i++ {
66
+ root, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2)
67
+ require.NoError(t, err)
68
+ bestEffortRoots = append(bestEffortRoots, root)
69
+ expectedKept = append(expectedKept, toMHs(allCids)...)
70
+ }
71
+
72
+ ch := GC(ctx, bs, ds, pinner, bestEffortRoots)
73
+ var discarded []multihash.Multihash
74
+ for res := range ch {
75
+ require.NoError(t, res.Error)
76
+ discarded = append(discarded, res.KeyRemoved.Hash())
77
+ }
78
+
79
+ allKeys, err := bs.AllKeysChan(ctx)
80
+ require.NoError(t, err)
81
+ var kept []multihash.Multihash
82
+ for key := range allKeys {
83
+ kept = append(kept, key.Hash())
84
+ }
85
+
86
+ require.ElementsMatch(t, expectedDiscarded, discarded)
87
+ require.ElementsMatch(t, expectedKept, kept)
88
+}
89
+
90
+func toMHs(cids []cid.Cid) []multihash.Multihash {
91
+ res := make([]multihash.Multihash, len(cids))
92
+ for i, c := range cids {
93
+ res[i] = c.Hash()
94
+ }
95
+ return res
96
+}
go.mod
+3
@@ -1,5 +1,8 @@
1
module github.com/ipfs/kubo
2
3
+// https://github.com/ipfs/boxo/pull/290
4
+replace github.com/ipfs/boxo => github.com/MichaelMure/boxo v0.0.0-20230505145003-9207501a615f
5
+
6
require (
7
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc
8
contrib.go.opencensus.io/exporter/prometheus v0.4.2
go.sum
+2
-2
@@ -49,6 +49,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
49
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
50
github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk=
51
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
52
+github.com/MichaelMure/boxo v0.0.0-20230505145003-9207501a615f h1:2UbpOJ6cIC43V/hIDxgvP0VLbJIk+cBofPAWmXBlSrg=
53
+github.com/MichaelMure/boxo v0.0.0-20230505145003-9207501a615f/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w=
54
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
55
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
56
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@@ -356,8 +358,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
358
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
359
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
360
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
359
-github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 h1:ThRTXD/EyoLb/jz+YW+ZlOLbjX9FyaxP0dEpgUp3cCE=
360
-github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w=
361
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
362
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
363
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=