@cryptotaxi247 / kubo / commits / 097b684b4

pin: add context and error return to most of the Pinner functions

Michael Muré committed Sep 15, 2019 at 14:14 UTC 097b684b42196a68a9a22e2bc1144b19cecb0479
14 files changed +105 -66
blocks/blockstoreutil/remove.go
+5 -4
@@ -2,6 +2,7 @@
2 package blockstoreutil
3
4 import (
5 + "context"
6 "fmt"
7 "io"
8
@@ -33,7 +34,7 @@ type RmBlocksOpts struct {
34 // It returns a channel where objects of type RemovedBlock are placed, when
35 // not using the Quiet option. Block removal is asynchronous and will
36 // skip any pinned blocks.
36 -func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, cids []cid.Cid, opts RmBlocksOpts) (<-chan interface{}, error) {
37 +func RmBlocks(ctx context.Context, blocks bs.GCBlockstore, pins pin.Pinner, cids []cid.Cid, opts RmBlocksOpts) (<-chan interface{}, error) {
38 // make the channel large enough to hold any result to avoid
39 // blocking while holding the GCLock
40 out := make(chan interface{}, len(cids))
@@ -43,7 +44,7 @@ func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, cids []cid.Cid, opts RmBl
44 unlocker := blocks.GCLock()
45 defer unlocker.Unlock()
46
46 - stillOkay := FilterPinned(pins, out, cids)
47 + stillOkay := FilterPinned(ctx, pins, out, cids)
48
49 for _, c := range stillOkay {
50 // Kept for backwards compatibility. We may want to
@@ -74,9 +75,9 @@ func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, cids []cid.Cid, opts RmBl
75 // out channel, with an error which indicates that the Cid is pinned.
76 // This function is used in RmBlocks to filter out any blocks which are not
77 // to be removed (because they are pinned).
77 -func FilterPinned(pins pin.Pinner, out chan<- interface{}, cids []cid.Cid) []cid.Cid {
78 +func FilterPinned(ctx context.Context, pins pin.Pinner, out chan<- interface{}, cids []cid.Cid) []cid.Cid {
79 stillOkay := make([]cid.Cid, 0, len(cids))
79 - res, err := pins.CheckIfPinned(cids...)
80 + res, err := pins.CheckIfPinned(ctx, cids...)
81 if err != nil {
82 out <- &RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)}
83 return nil
core/commands/pin.go
+26 -9
@@ -446,7 +446,7 @@ func pinLsKeys(req *cmds.Request, typeStr string, n *core.IpfsNode, api coreifac
446 return err
447 }
448
449 - pinType, pinned, err := n.Pinning.IsPinnedWithType(c.Cid(), mode)
449 + pinType, pinned, err := n.Pinning.IsPinnedWithType(req.Context, c.Cid(), mode)
450 if err != nil {
451 return err
452 }
@@ -501,19 +501,31 @@ func pinLsAll(req *cmds.Request, typeStr string, n *core.IpfsNode, emit func(val
501 }
502
503 if typeStr == "direct" || typeStr == "all" {
504 - err := AddToResultKeys(n.Pinning.DirectKeys(), "direct")
504 + dkeys, err := n.Pinning.DirectKeys(req.Context)
505 + if err != nil {
506 + return err
507 + }
508 + err = AddToResultKeys(dkeys, "direct")
509 if err != nil {
510 return err
511 }
512 }
513 if typeStr == "recursive" || typeStr == "all" {
510 - err := AddToResultKeys(n.Pinning.RecursiveKeys(), "recursive")
514 + rkeys, err := n.Pinning.RecursiveKeys(req.Context)
515 + if err != nil {
516 + return err
517 + }
518 + err = AddToResultKeys(rkeys, "recursive")
519 if err != nil {
520 return err
521 }
522 }
523 if typeStr == "indirect" || typeStr == "all" {
516 - for _, k := range n.Pinning.RecursiveKeys() {
524 + rkeys, err := n.Pinning.RecursiveKeys(req.Context)
525 + if err != nil {
526 + return err
527 + }
528 + for _, k := range rkeys {
529 var visitErr error
530 err := dag.Walk(req.Context, dag.GetLinksWithDAG(n.DAG), k, func(c cid.Cid) bool {
531 r := keys.Visit(c)
@@ -642,8 +654,10 @@ var verifyPinCmd = &cmds.Command{
654 explain: !quiet,
655 includeOk: verbose,
656 }
645 - out := pinVerify(req.Context, n, opts, enc)
646 -
657 + out, err := pinVerify(req.Context, n, opts, enc)
658 + if err != nil {
659 + return err
660 + }
661 return res.Emit(out)
662 },
663 Type: PinVerifyRes{},
@@ -685,13 +699,16 @@ type pinVerifyOpts struct {
699 includeOk bool
700 }
701
688 -func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc cidenc.Encoder) <-chan interface{} {
702 +func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc cidenc.Encoder) (<-chan interface{}, error) {
703 visited := make(map[cid.Cid]PinStatus)
704
705 bs := n.Blocks.Blockstore()
706 DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
707 getLinks := dag.GetLinksWithDAG(DAG)
694 - recPins := n.Pinning.RecursiveKeys()
708 + recPins, err := n.Pinning.RecursiveKeys(ctx)
709 + if err != nil {
710 + return nil, err
711 + }
712
713 var checkPin func(root cid.Cid) PinStatus
714 checkPin = func(root cid.Cid) PinStatus {
@@ -747,7 +764,7 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc ci
764 }
765 }()
766
750 - return out
767 + return out, nil
768 }
769
770 // Format formats PinVerifyRes
core/coreapi/block.go
+2 -2
@@ -56,7 +56,7 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
56
57 if settings.Pin {
58 api.pinning.PinWithMode(b.Cid(), pin.Recursive)
59 - if err := api.pinning.Flush(); err != nil {
59 + if err := api.pinning.Flush(ctx); err != nil {
60 return nil, err
61 }
62 }
@@ -91,7 +91,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm
91 cids := []cid.Cid{rp.Cid()}
92 o := util.RmBlocksOpts{Force: settings.Force}
93
94 - out, err := util.RmBlocks(api.blockstore, api.pinning, cids, o)
94 + out, err := util.RmBlocks(ctx, api.blockstore, api.pinning, cids, o)
95 if err != nil {
96 return err
97 }
core/coreapi/dag.go
+2 -2
@@ -26,7 +26,7 @@ func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error {
26
27 adder.pinning.PinWithMode(nd.Cid(), pin.Recursive)
28
29 - return adder.pinning.Flush()
29 + return adder.pinning.Flush(ctx)
30 }
31
32 func (adder *pinningAdder) AddMany(ctx context.Context, nds []ipld.Node) error {
@@ -45,7 +45,7 @@ func (adder *pinningAdder) AddMany(ctx context.Context, nds []ipld.Node) error {
45 }
46 }
47
48 - return adder.pinning.Flush()
48 + return adder.pinning.Flush(ctx)
49 }
50
51 func (api *dagAPI) Pinning() ipld.NodeAdder {
core/coreapi/object.go
+1 -1
@@ -119,7 +119,7 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj
119
120 if options.Pin {
121 api.pinning.PinWithMode(dagnode.Cid(), pin.Recursive)
122 - err = api.pinning.Flush()
122 + err = api.pinning.Flush(ctx)
123 if err != nil {
124 return nil, err
125 }
core/coreapi/pin.go
+22 -7
@@ -37,7 +37,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp
37 return err
38 }
39
40 - return api.pinning.Flush()
40 + return api.pinning.Flush(ctx)
41 }
42
43 func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) {
@@ -75,7 +75,7 @@ func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOpti
75 return err
76 }
77
78 - return api.pinning.Flush()
78 + return api.pinning.Flush(ctx)
79 }
80
81 func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error {
@@ -101,7 +101,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt
101 return err
102 }
103
104 - return api.pinning.Flush()
104 + return api.pinning.Flush(ctx)
105 }
106
107 type pinStatus struct {
@@ -137,7 +137,10 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro
137 bs := api.blockstore
138 DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
139 getLinks := merkledag.GetLinksWithDAG(DAG)
140 - recPins := api.pinning.RecursiveKeys()
140 + recPins, err := api.pinning.RecursiveKeys(ctx)
141 + if err != nil {
142 + return nil, err
143 + }
144
145 var checkPin func(root cid.Cid) *pinStatus
146 checkPin = func(root cid.Cid) *pinStatus {
@@ -204,11 +207,19 @@ func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pi
207 }
208
209 if typeStr == "direct" || typeStr == "all" {
207 - AddToResultKeys(api.pinning.DirectKeys(), "direct")
210 + dkeys, err := api.pinning.DirectKeys(ctx)
211 + if err != nil {
212 + return nil, err
213 + }
214 + AddToResultKeys(dkeys, "direct")
215 }
216 if typeStr == "indirect" || typeStr == "all" {
217 set := cid.NewSet()
211 - for _, k := range api.pinning.RecursiveKeys() {
218 + rkeys, err := api.pinning.RecursiveKeys(ctx)
219 + if err != nil {
220 + return nil, err
221 + }
222 + for _, k := range rkeys {
223 err := merkledag.Walk(
224 ctx, merkledag.GetLinksWithDAG(api.dag), k,
225 set.Visit,
@@ -221,7 +232,11 @@ func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pi
232 AddToResultKeys(set.Keys(), "indirect")
233 }
234 if typeStr == "recursive" || typeStr == "all" {
224 - AddToResultKeys(api.pinning.RecursiveKeys(), "recursive")
235 + rkeys, err := api.pinning.RecursiveKeys(ctx)
236 + if err != nil {
237 + return nil, err
238 + }
239 + AddToResultKeys(rkeys, "recursive")
240 }
241
242 out := make([]coreiface.Pin, 0, len(keys))
core/coreunix/add.go
+1 -1
@@ -176,7 +176,7 @@ func (adder *Adder) PinRoot(root ipld.Node) error {
176 }
177
178 adder.pinning.PinWithMode(rnk, pin.Recursive)
179 - return adder.pinning.Flush()
179 + return adder.pinning.Flush(adder.ctx)
180 }
181
182 func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error {
fuse/ipns/common.go
+1 -1
@@ -23,7 +23,7 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
23 return err
24 }
25
26 - err = n.Pinning.Flush()
26 + err = n.Pinning.Flush(ctx)
27 if err != nil {
28 return err
29 }
go.mod
+2
@@ -108,3 +108,5 @@ require (
108 )
109
110 go 1.13
111 +
112 +replace github.com/ipfs/go-ipfs-provider => github.com/MichaelMure/go-ipfs-provider v0.2.2-0.20191017161655-f2597dc7065a
go.sum
+2
@@ -8,6 +8,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190823232136-616930265c33/go.mod h1:bOv
8 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
9 github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk=
10 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
11 +github.com/MichaelMure/go-ipfs-provider v0.2.2-0.20191017161655-f2597dc7065a h1:U/EUpAtQboxuN8yfU5ww0/5LWticjuR32KnFwRHGtqU=
12 +github.com/MichaelMure/go-ipfs-provider v0.2.2-0.20191017161655-f2597dc7065a/go.mod h1:rcQBVqfblDQRk5LaCtf2uxuKxMJxvKmF5pLS0pO4au4=
13 github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo=
14 github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo=
15 github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s=
namesys/publisher.go
+1 -1
@@ -298,7 +298,7 @@ func InitializeKeyspace(ctx context.Context, pub Publisher, pins pin.Pinner, key
298 return err
299 }
300
301 - err = pins.Flush()
301 + err = pins.Flush(ctx)
302 if err != nil {
303 return err
304 }
pin/gc/gc.go
+15 -3
@@ -201,7 +201,11 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
201 }
202 return links, nil
203 }
204 - err := Descendants(ctx, getLinks, gcs, pn.RecursiveKeys())
204 + rkeys, err := pn.RecursiveKeys(ctx)
205 + if err != nil {
206 + return nil, err
207 + }
208 + err = Descendants(ctx, getLinks, gcs, rkeys)
209 if err != nil {
210 errors = true
211 select {
@@ -233,11 +237,19 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
237 }
238 }
239
236 - for _, k := range pn.DirectKeys() {
240 + dkeys, err := pn.DirectKeys(ctx)
241 + if err != nil {
242 + return nil, err
243 + }
244 + for _, k := range dkeys {
245 gcs.Add(k)
246 }
247
240 - err = Descendants(ctx, getLinks, gcs, pn.InternalPins())
248 + ikeys, err := pn.InternalPins(ctx)
249 + if err != nil {
250 + return nil, err
251 + }
252 + err = Descendants(ctx, getLinks, gcs, ikeys)
253 if err != nil {
254 errors = true
255 select {
pin/pin.go
+21 -31
@@ -105,11 +105,11 @@ func StringToMode(s string) (Mode, bool) {
105 type Pinner interface {
106 // IsPinned returns whether or not the given cid is pinned
107 // and an explanation of why its pinned
108 - IsPinned(cid.Cid) (string, bool, error)
108 + IsPinned(ctx context.Context, c cid.Cid) (string, bool, error)
109
110 // IsPinnedWithType returns whether or not the given cid is pinned with the
111 // given pin type, as well as returning the type of pin its pinned with.
112 - IsPinnedWithType(cid.Cid, Mode) (string, bool, error)
112 + IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error)
113
114 // Pin the given node, optionally recursively.
115 Pin(ctx context.Context, node ipld.Node, recursive bool) error
@@ -125,7 +125,7 @@ type Pinner interface {
125
126 // Check if a set of keys are pinned, more efficient than
127 // calling IsPinned for each key
128 - CheckIfPinned(cids ...cid.Cid) ([]Pinned, error)
128 + CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error)
129
130 // PinWithMode is for manually editing the pin structure. Use with
131 // care! If used improperly, garbage collection may not be
@@ -138,17 +138,17 @@ type Pinner interface {
138 RemovePinWithMode(cid.Cid, Mode)
139
140 // Flush writes the pin state to the backing datastore
141 - Flush() error
141 + Flush(ctx context.Context) error
142
143 // DirectKeys returns all directly pinned cids
144 - DirectKeys() []cid.Cid
144 + DirectKeys(ctx context.Context) ([]cid.Cid, error)
145
146 // DirectKeys returns all recursively pinned cids
147 - RecursiveKeys() []cid.Cid
147 + RecursiveKeys(ctx context.Context) ([]cid.Cid, error)
148
149 // InternalPins returns all cids kept pinned for the internal state of the
150 // pinner
151 - InternalPins() []cid.Cid
151 + InternalPins(ctx context.Context) ([]cid.Cid, error)
152 }
153
154 // Pinned represents CID which has been pinned with a pinning strategy.
@@ -211,8 +211,6 @@ func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner {
211
212 // Pin the given node, optionally recursive
213 func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error {
214 - p.lock.Lock()
215 - defer p.lock.Unlock()
214 err := p.dserv.Add(ctx, node)
215 if err != nil {
216 return err
@@ -220,13 +218,16 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error {
218
219 c := node.Cid()
220
221 + p.lock.Lock()
222 + defer p.lock.Unlock()
223 +
224 if recurse {
225 if p.recursePin.Has(c) {
226 return nil
227 }
228
229 p.lock.Unlock()
229 - // fetch entire graph
230 + // temporary unlock to fetch the entire graph
231 err := mdag.FetchGraph(ctx, c, p.dserv)
232 p.lock.Lock()
233 if err != nil {
@@ -243,13 +244,6 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error {
244
245 p.recursePin.Add(c)
246 } else {
246 - p.lock.Unlock()
247 - _, err := p.dserv.Get(ctx, c)
248 - p.lock.Lock()
249 - if err != nil {
250 - return err
251 - }
252 -
247 if p.recursePin.Has(c) {
248 return fmt.Errorf("%s already pinned recursively", c.String())
249 }
@@ -286,15 +280,13 @@ func (p *pinner) isInternalPin(c cid.Cid) bool {
280
281 // IsPinned returns whether or not the given key is pinned
282 // and an explanation of why its pinned
289 -func (p *pinner) IsPinned(c cid.Cid) (string, bool, error) {
290 - p.lock.RLock()
291 - defer p.lock.RUnlock()
283 +func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) {
284 return p.isPinnedWithType(c, Any)
285 }
286
287 // IsPinnedWithType returns whether or not the given cid is pinned with the
288 // given pin type, as well as returning the type of pin its pinned with.
297 -func (p *pinner) IsPinnedWithType(c cid.Cid, mode Mode) (string, bool, error) {
289 +func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) {
290 p.lock.RLock()
291 defer p.lock.RUnlock()
292 return p.isPinnedWithType(c, mode)
@@ -347,7 +339,7 @@ func (p *pinner) isPinnedWithType(c cid.Cid, mode Mode) (string, bool, error) {
339
340 // CheckIfPinned Checks if a set of keys are pinned, more efficient than
341 // calling IsPinned for each key, returns the pinned status of cid(s)
350 -func (p *pinner) CheckIfPinned(cids ...cid.Cid) ([]Pinned, error) {
342 +func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) {
343 p.lock.RLock()
344 defer p.lock.RUnlock()
345 pinned := make([]Pinned, 0, len(cids))
@@ -494,19 +486,19 @@ func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error)
486 }
487
488 // DirectKeys returns a slice containing the directly pinned keys
497 -func (p *pinner) DirectKeys() []cid.Cid {
489 +func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) {
490 p.lock.RLock()
491 defer p.lock.RUnlock()
492
501 - return p.directPin.Keys()
493 + return p.directPin.Keys(), nil
494 }
495
496 // RecursiveKeys returns a slice containing the recursively pinned keys
505 -func (p *pinner) RecursiveKeys() []cid.Cid {
497 +func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) {
498 p.lock.RLock()
499 defer p.lock.RUnlock()
500
509 - return p.recursePin.Keys()
501 + return p.recursePin.Keys(), nil
502 }
503
504 // Update updates a recursive pin from one cid to another
@@ -541,12 +533,10 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error
533 }
534
535 // Flush encodes and writes pinner keysets to the datastore
544 -func (p *pinner) Flush() error {
536 +func (p *pinner) Flush(ctx context.Context) error {
537 p.lock.Lock()
538 defer p.lock.Unlock()
539
548 - ctx := context.TODO()
549 -
540 internalset := cid.NewSet()
541 recordInternal := internalset.Add
542
@@ -594,12 +584,12 @@ func (p *pinner) Flush() error {
584
585 // InternalPins returns all cids kept pinned for the internal state of the
586 // pinner
597 -func (p *pinner) InternalPins() []cid.Cid {
587 +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) {
588 p.lock.Lock()
589 defer p.lock.Unlock()
590 var out []cid.Cid
591 out = append(out, p.internalPin.Keys()...)
602 - return out
592 + return out, nil
593 }
594
595 // PinWithMode allows the user to have fine grained control over pin
pin/pin_test.go
+4 -4
@@ -31,7 +31,7 @@ func randNode() (*mdag.ProtoNode, cid.Cid) {
31 }
32
33 func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) {
34 - _, pinned, err := p.IsPinned(c)
34 + _, pinned, err := p.IsPinned(context.Background(), c)
35 if err != nil {
36 t.Fatal(err)
37 }
@@ -42,7 +42,7 @@ func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) {
42 }
43
44 func assertUnpinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) {
45 - _, pinned, err := p.IsPinned(c)
45 + _, pinned, err := p.IsPinned(context.Background(), c)
46 if err != nil {
47 t.Fatal(err)
48 }
@@ -146,7 +146,7 @@ func TestPinnerBasic(t *testing.T) {
146 t.Fatal(err)
147 }
148
149 - err = p.Flush()
149 + err = p.Flush(ctx)
150 if err != nil {
151 t.Fatal(err)
152 }
@@ -327,7 +327,7 @@ func TestFlush(t *testing.T) {
327 _, k := randNode()
328
329 p.PinWithMode(k, Recursive)
330 - if err := p.Flush(); err != nil {
330 + if err := p.Flush(context.Background()); err != nil {
331 t.Fatal(err)
332 }
333 assertPinned(t, p, k, "expected key to still be pinned")