fix (pinning): pin ls traverses all indirect pins. pin ls pin type precedence change - a direct/recursive pin is now labeled as such even if also indirectly pinned.
Adin Schmahmann committed
Nov 25, 2019 at 11:58 UTC
0906d7f3b1d05e088db7c3ef24ed470ead7965b3
5 files changed
+136
-114
core/commands/pin.go
+24
-68
@@ -11,6 +11,7 @@ import (
11
core "github.com/ipfs/go-ipfs/core"
12
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
13
e "github.com/ipfs/go-ipfs/core/commands/e"
14
+ coreapi "github.com/ipfs/go-ipfs/core/coreapi"
15
pin "github.com/ipfs/go-ipfs/pin"
16
17
bserv "github.com/ipfs/go-blockservice"
@@ -18,6 +19,7 @@ import (
19
cidenc "github.com/ipfs/go-cidutil/cidenc"
20
cmds "github.com/ipfs/go-ipfs-cmds"
21
offline "github.com/ipfs/go-ipfs-exchange-offline"
22
+ ipld "github.com/ipfs/go-ipld-format"
23
dag "github.com/ipfs/go-merkledag"
24
verifcid "github.com/ipfs/go-verifcid"
25
coreiface "github.com/ipfs/interface-go-ipfs-core"
@@ -352,7 +354,7 @@ Example:
354
if len(req.Arguments) > 0 {
355
err = pinLsKeys(req, typeStr, n, api, emit)
356
} else {
355
- err = pinLsAll(req, typeStr, n, emit)
357
+ err = pinLsAll(req, typeStr, n.Pinning, n.DAG, emit)
358
}
359
if err != nil {
360
return err
@@ -475,84 +477,38 @@ func pinLsKeys(req *cmds.Request, typeStr string, n *core.IpfsNode, api coreifac
477
return nil
478
}
479
478
-func pinLsAll(req *cmds.Request, typeStr string, n *core.IpfsNode, emit func(value interface{}) error) error {
480
+func pinLsAll(req *cmds.Request, typeStr string, pinning pin.Pinner, dag ipld.DAGService, emit func(value interface{}) error) error {
481
+ pinCh, errCh := coreapi.PinLsAll(req.Context, typeStr, pinning, dag)
482
+
483
enc, err := cmdenv.GetCidEncoder(req)
484
if err != nil {
485
return err
486
}
487
484
- keys := cid.NewSet()
485
-
486
- AddToResultKeys := func(keyList []cid.Cid, typeStr string) error {
487
- for _, c := range keyList {
488
- if keys.Visit(c) {
489
- err := emit(&PinLsOutputWrapper{
490
- PinLsObject: PinLsObject{
491
- Type: typeStr,
492
- Cid: enc.Encode(c),
493
- },
494
- })
495
- if err != nil {
496
- return err
497
- }
488
+ ctx := req.Context
489
+loop:
490
+ for {
491
+ select {
492
+ case p, ok := <-pinCh:
493
+ if !ok {
494
+ break loop
495
}
499
- }
500
- return nil
501
- }
502
-
503
- if typeStr == "direct" || typeStr == "all" {
504
- dkeys, err := n.Pinning.DirectKeys(req.Context)
505
- if err != nil {
506
- return err
507
- }
508
- err = AddToResultKeys(dkeys, "direct")
509
- if err != nil {
510
- return err
511
- }
512
- }
513
- if typeStr == "recursive" || typeStr == "all" {
514
- rkeys, err := n.Pinning.RecursiveKeys(req.Context)
515
- if err != nil {
516
- return err
517
- }
518
- err = AddToResultKeys(rkeys, "recursive")
519
- if err != nil {
520
- return err
521
- }
522
- }
523
- if typeStr == "indirect" || typeStr == "all" {
524
- rkeys, err := n.Pinning.RecursiveKeys(req.Context)
525
- if err != nil {
526
- return err
527
- }
528
- for _, k := range rkeys {
529
- var visitErr error
530
- err := dag.Walk(req.Context, dag.GetLinksWithDAG(n.DAG), k, func(c cid.Cid) bool {
531
- r := keys.Visit(c)
532
- if r {
533
- err := emit(&PinLsOutputWrapper{
534
- PinLsObject: PinLsObject{
535
- Type: "indirect",
536
- Cid: enc.Encode(c),
537
- },
538
- })
539
- if err != nil {
540
- visitErr = err
541
- }
542
- }
543
- return r
544
- }, dag.SkipRoot(), dag.Concurrent())
545
-
546
- if visitErr != nil {
547
- return visitErr
548
- }
549
- if err != nil {
496
+ if err := emit(&PinLsOutputWrapper{
497
+ PinLsObject: PinLsObject{
498
+ Type: p.Type(),
499
+ Cid: enc.Encode(p.Path().Cid()),
500
+ },
501
+ }); err != nil {
502
return err
503
}
504
+
505
+ case <-ctx.Done():
506
+ return ctx.Err()
507
}
508
}
509
555
- return nil
510
+ err = <-errCh
511
+ return err
512
}
513
514
const (
core/coreapi/pin.go
+108
-42
@@ -3,14 +3,15 @@ package coreapi
3
import (
4
"context"
5
"fmt"
6
-
6
bserv "github.com/ipfs/go-blockservice"
8
- cid "github.com/ipfs/go-cid"
7
+ "github.com/ipfs/go-cid"
8
offline "github.com/ipfs/go-ipfs-exchange-offline"
10
- merkledag "github.com/ipfs/go-merkledag"
9
+ "github.com/ipfs/go-ipfs/pin"
10
+ ipld "github.com/ipfs/go-ipld-format"
11
+ "github.com/ipfs/go-merkledag"
12
coreiface "github.com/ipfs/interface-go-ipfs-core"
13
caopts "github.com/ipfs/interface-go-ipfs-core/options"
13
- path "github.com/ipfs/interface-go-ipfs-core/path"
14
+ "github.com/ipfs/interface-go-ipfs-core/path"
15
)
16
17
type PinAPI CoreAPI
@@ -194,57 +195,122 @@ func (p *pinInfo) Type() string {
195
}
196
197
func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pin, error) {
198
+ pinCh, errCh := PinLsAll(ctx, typeStr, api.pinning, api.dag)
199
+
200
+ var pins []coreiface.Pin
201
+loop:
202
+ for {
203
+ select {
204
+ case p, ok := <-pinCh:
205
+ if !ok {
206
+ break loop
207
+ }
208
+ pins = append(pins, p)
209
+ case <-ctx.Done():
210
+ return nil, ctx.Err()
211
+ }
212
+ }
213
+ err := <-errCh
214
+ if err != nil {
215
+ return nil, err
216
+ }
217
+
218
+ return pins, nil
219
+}
220
198
- keys := make(map[cid.Cid]*pinInfo)
221
+// PinLsAll is an internal function for returning a list of pins
222
+func PinLsAll(ctx context.Context, typeStr string, pin pin.Pinner, dag ipld.DAGService) (chan coreiface.Pin, chan error) {
223
+ ch := make(chan coreiface.Pin, 32)
224
+ errCh := make(chan error, 1)
225
200
- AddToResultKeys := func(keyList []cid.Cid, typeStr string) {
226
+ keys := cid.NewSet()
227
+ AddToResultKeys := func(keyList []cid.Cid, typeStr string) error {
228
for _, c := range keyList {
202
- keys[c] = &pinInfo{
203
- pinType: typeStr,
204
- path: path.IpldPath(c),
229
+ if keys.Visit(c) {
230
+ select {
231
+ case ch <- &pinInfo{
232
+ pinType: typeStr,
233
+ path: path.IpldPath(c),
234
+ }:
235
+ case <-ctx.Done():
236
+ return ctx.Err()
237
+ }
238
}
239
}
240
+ return nil
241
}
242
209
- if typeStr == "direct" || typeStr == "all" {
210
- dkeys, err := api.pinning.DirectKeys(ctx)
211
- if err != nil {
212
- return nil, err
213
- }
214
- AddToResultKeys(dkeys, "direct")
215
- }
216
- if typeStr == "indirect" || typeStr == "all" {
217
- set := cid.NewSet()
218
- rkeys, err := api.pinning.RecursiveKeys(ctx)
219
- if err != nil {
220
- return nil, err
221
- }
222
- for _, k := range rkeys {
223
- err := merkledag.Walk(
224
- ctx, merkledag.GetLinksWithDAG(api.dag), k,
225
- set.Visit,
226
- merkledag.SkipRoot(), merkledag.Concurrent(),
227
- )
243
+ go func() {
244
+ defer close(ch)
245
+ defer close(errCh)
246
+ if typeStr == "direct" || typeStr == "all" {
247
+ dkeys, err := pin.DirectKeys(ctx)
248
if err != nil {
229
- return nil, err
249
+ errCh <- err
250
+ return
251
+ }
252
+ if err := AddToResultKeys(dkeys, "direct"); err != nil {
253
+ errCh <- err
254
+ return
255
}
256
}
232
- AddToResultKeys(set.Keys(), "indirect")
233
- }
234
- if typeStr == "recursive" || typeStr == "all" {
235
- rkeys, err := api.pinning.RecursiveKeys(ctx)
236
- if err != nil {
237
- return nil, err
257
+ if typeStr == "recursive" || typeStr == "all" {
258
+ rkeys, err := pin.RecursiveKeys(ctx)
259
+ if err != nil {
260
+ errCh <- err
261
+ return
262
+ }
263
+ if err := AddToResultKeys(rkeys, "recursive"); err != nil {
264
+ errCh <- err
265
+ return
266
+ }
267
}
239
- AddToResultKeys(rkeys, "recursive")
240
- }
268
+ if typeStr == "indirect" || typeStr == "all" {
269
+ rkeys, err := pin.RecursiveKeys(ctx)
270
+ if err != nil {
271
+ errCh <- err
272
+ return
273
+ }
274
242
- out := make([]coreiface.Pin, 0, len(keys))
243
- for _, v := range keys {
244
- out = append(out, v)
245
- }
275
+ // If we're only listing indirect pins, we need to
276
+ // explicitly mark direct/recursive pins so we don't
277
+ // send them.
278
+ if typeStr == "indirect" {
279
+ dkeys, err := pin.DirectKeys(ctx)
280
+ if err != nil {
281
+ errCh <- err
282
+ return
283
+ }
284
+
285
+ for _, k := range dkeys {
286
+ keys.Add(k)
287
+ }
288
+ for _, k := range rkeys {
289
+ keys.Add(k)
290
+ }
291
+ }
292
247
- return out, nil
293
+ indirectKeys := cid.NewSet()
294
+ for _, k := range rkeys {
295
+ err := merkledag.Walk(ctx, merkledag.GetLinksWithDAG(dag), k, func(c cid.Cid) bool {
296
+ r := indirectKeys.Visit(c)
297
+ if r {
298
+ if err := AddToResultKeys([]cid.Cid{c}, "indirect"); err != nil {
299
+ return false
300
+ }
301
+ }
302
+ return r
303
+ }, merkledag.SkipRoot(), merkledag.Concurrent())
304
+
305
+ if err != nil {
306
+ errCh <- err
307
+ return
308
+ }
309
+ }
310
+ }
311
+ }()
312
+
313
+ return ch, errCh
314
}
315
316
func (api *PinAPI) core() coreiface.CoreAPI {
core/coreapi/unixfs.go
+1
-1
@@ -32,7 +32,7 @@ type UnixfsAPI CoreAPI
32
var nilNode *core.IpfsNode
33
var once sync.Once
34
35
-func getOrCreateNilNode() (*core.IpfsNode,error) {
35
+func getOrCreateNilNode() (*core.IpfsNode, error) {
36
once.Do(func() {
37
if nilNode != nil {
38
return
go.mod
+1
-1
@@ -51,7 +51,7 @@ require (
51
github.com/ipfs/go-path v0.0.7
52
github.com/ipfs/go-unixfs v0.2.1
53
github.com/ipfs/go-verifcid v0.0.1
54
- github.com/ipfs/interface-go-ipfs-core v0.2.3
54
+ github.com/ipfs/interface-go-ipfs-core v0.2.5
55
github.com/jbenet/go-is-domain v1.0.3
56
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
57
github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2
go.sum
+2
-2
@@ -259,8 +259,8 @@ github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0=
259
github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
260
github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
261
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
262
-github.com/ipfs/interface-go-ipfs-core v0.2.3 h1:E6uQ+1fJjkxJWlL9lAE72a5FWeyeeNL3GitLy8+jq3Y=
263
-github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
262
+github.com/ipfs/interface-go-ipfs-core v0.2.5 h1:/rspOe8RbIxwtssEXHB+X9JXhOBDCQt8x50d2kFPXL8=
263
+github.com/ipfs/interface-go-ipfs-core v0.2.5/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
264
github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
265
github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
266
github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=