@cryptotaxi247 / kubo / commits / 0a6b880be

fix for #1008 and other pinning fixes

This commit adds a new set of sharness tests for pinning, and addresses bugs that were pointed out by said tests. test/sharness: added more pinning tests Pinning is currently broken. See issue #1051. This commit introduces a few more pinning tests. These are by no means exhaustive, but definitely surface the present problems going on. I believe these tests are correct, but not sure. Pushing them as failing so that pinning is fixed in this PR. make pinning and merkledag.Get take contexts improve 'add' commands usage of pinning FIXUP: fix 'pin lists look good' ipfs-pin-stat simple script to help check pinning This is a simple shell script to help check pinning. We ought to strive towards making adding commands this easy. The http api is great and powerful, but our setup right now gets in the way. Perhaps we can clean up that area. updated t0081-repo-pinning - fixed a couple bugs with the tests - made it a bit clearer (still a lot going on) - the remaining tests are correct and highlight a problem with pinning. Namely, that recursive pinning is buggy. At least: towards the end of the test, $HASH_DIR4 and $HASH_FILE4 should be pinned indirectly, but they're not. And thus get gc-ed out. There may be other problems too. cc @whyrusleeping fix grep params for context deadline check fix bugs in pin and pin tests check for block local before checking recursive pin

Jeromy committed Apr 3, 2015 at 17:40 UTC 0a6b880bee0117f657aedbbd24bfc07a2361e3da
33 files changed +556 -99
cmd/ipfs/init.go
+1 -1
@@ -143,7 +143,7 @@ func addDefaultAssets(out io.Writer, repoRoot string) error {
143 return err
144 }
145
146 - if err := nd.Pinning.Pin(dir, true); err != nil {
146 + if err := nd.Pinning.Pin(ctx, dir, true); err != nil {
147 return err
148 }
149
core/commands/add.go
+17 -16
@@ -1,12 +1,14 @@
1 package commands
2
3 import (
4 - "errors"
4 "fmt"
5 "io"
6 "path"
7 "strings"
8
9 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
10 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
11 +
12 cmds "github.com/ipfs/go-ipfs/commands"
13 files "github.com/ipfs/go-ipfs/commands/files"
14 core "github.com/ipfs/go-ipfs/core"
@@ -14,12 +16,9 @@ import (
16 importer "github.com/ipfs/go-ipfs/importer"
17 "github.com/ipfs/go-ipfs/importer/chunk"
18 dag "github.com/ipfs/go-ipfs/merkledag"
17 - pinning "github.com/ipfs/go-ipfs/pin"
19 ft "github.com/ipfs/go-ipfs/unixfs"
20 u "github.com/ipfs/go-ipfs/util"
21 "github.com/ipfs/go-ipfs/util/debugerror"
21 -
22 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
22 )
23
24 // Error indicating the max depth has been exceded.
@@ -105,7 +104,19 @@ remains to be implemented.
104 return
105 }
106
108 - _, err = addFile(n, file, outChan, progress, wrap)
107 + rootnd, err := addFile(n, file, outChan, progress, wrap)
108 + if err != nil {
109 + res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
110 + return
111 + }
112 +
113 + err = n.Pinning.Pin(context.Background(), rootnd, true)
114 + if err != nil {
115 + res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
116 + return
117 + }
118 +
119 + err = n.Pinning.Flush()
120 if err != nil {
121 res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
122 return
@@ -200,15 +211,10 @@ remains to be implemented.
211 }
212
213 func add(n *core.IpfsNode, readers []io.Reader) ([]*dag.Node, error) {
203 - mp, ok := n.Pinning.(pinning.ManualPinner)
204 - if !ok {
205 - return nil, errors.New("invalid pinner type! expected manual pinner")
206 - }
207 -
214 dagnodes := make([]*dag.Node, 0)
215
216 for _, reader := range readers {
211 - node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
217 + node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter)
218 if err != nil {
219 return nil, err
220 }
@@ -229,11 +235,6 @@ func addNode(n *core.IpfsNode, node *dag.Node) error {
235 return err
236 }
237
232 - err = n.Pinning.Pin(node, true) // ensure we keep it
233 - if err != nil {
234 - return err
235 - }
236 -
238 return nil
239 }
240
core/commands/ls.go
+6 -1
@@ -5,6 +5,9 @@ import (
5 "fmt"
6 "io"
7 "text/tabwriter"
8 + "time"
9 +
10 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
11
12 cmds "github.com/ipfs/go-ipfs/commands"
13 merkledag "github.com/ipfs/go-ipfs/merkledag"
@@ -77,7 +80,9 @@ it contains, with the following format:
80 Links: make([]LsLink, len(dagnode.Links)),
81 }
82 for j, link := range dagnode.Links {
80 - link.Node, err = link.GetNode(node.DAG)
83 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
84 + defer cancel()
85 + link.Node, err = link.GetNode(ctx, node.DAG)
86 if err != nil {
87 res.SetError(err, cmds.ErrNormal)
88 return
core/commands/pin.go
+45 -7
@@ -165,12 +165,14 @@ Use --type=<type> to specify the type of pinned keys to list. Valid values are:
165 * "indirect": pinned indirectly by an ancestor (like a refcount)
166 * "all"
167
168 +To see the ref count on indirect pins, pass the -count option flag.
169 Defaults to "direct".
170 `,
171 },
172
173 Options: []cmds.Option{
174 cmds.StringOption("type", "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\". Defaults to \"direct\""),
175 + cmds.BoolOption("count", "n", "Show refcount when listing indirect pins"),
176 },
177 Run: func(req cmds.Request, res cmds.Response) {
178 n, err := req.Context().GetNode()
@@ -195,21 +197,57 @@ Defaults to "direct".
197 res.SetError(err, cmds.ErrClient)
198 }
199
198 - keys := make([]u.Key, 0)
200 + keys := make(map[string]int)
201 if typeStr == "direct" || typeStr == "all" {
200 - keys = append(keys, n.Pinning.DirectKeys()...)
202 + for _, k := range n.Pinning.DirectKeys() {
203 + keys[k.B58String()] = 1
204 + }
205 }
206 if typeStr == "indirect" || typeStr == "all" {
203 - keys = append(keys, n.Pinning.IndirectKeys()...)
207 + for k, v := range n.Pinning.IndirectKeys() {
208 + keys[k.B58String()] = v
209 + }
210 }
211 if typeStr == "recursive" || typeStr == "all" {
206 - keys = append(keys, n.Pinning.RecursiveKeys()...)
212 + for _, k := range n.Pinning.RecursiveKeys() {
213 + keys[k.B58String()] = 1
214 + }
215 }
216
209 - res.SetOutput(&KeyList{Keys: keys})
217 + res.SetOutput(&RefKeyList{Keys: keys})
218 },
211 - Type: KeyList{},
219 + Type: RefKeyList{},
220 Marshalers: cmds.MarshalerMap{
213 - cmds.Text: KeyListTextMarshaler,
221 + cmds.Text: func(res cmds.Response) (io.Reader, error) {
222 + typeStr, _, err := res.Request().Option("type").String()
223 + if err != nil {
224 + return nil, err
225 + }
226 +
227 + count, _, err := res.Request().Option("count").Bool()
228 + if err != nil {
229 + return nil, err
230 + }
231 +
232 + keys, ok := res.Output().(*RefKeyList)
233 + if !ok {
234 + return nil, u.ErrCast()
235 + }
236 + out := new(bytes.Buffer)
237 + if typeStr == "indirect" && count {
238 + for k, v := range keys.Keys {
239 + fmt.Fprintf(out, "%s %d\n", k, v)
240 + }
241 + } else {
242 + for k, _ := range keys.Keys {
243 + fmt.Fprintf(out, "%s\n", k)
244 + }
245 + }
246 + return out, nil
247 + },
248 },
249 }
250 +
251 +type RefKeyList struct {
252 + Keys map[string]int
253 +}
core/corehttp/gateway_handler.go
+6 -2
@@ -352,7 +352,9 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
352 return
353 }
354
355 - rootnd, err := i.node.Resolver.DAG.Get(u.Key(h))
355 + tctx, cancel := context.WithTimeout(ctx, time.Minute)
356 + defer cancel()
357 + rootnd, err := i.node.Resolver.DAG.Get(tctx, u.Key(h))
358 if err != nil {
359 webError(w, "Could not resolve root object", err, http.StatusBadRequest)
360 return
@@ -414,7 +416,9 @@ func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
416 return
417 }
418
417 - rootnd, err := i.node.Resolver.DAG.Get(u.Key(h))
419 + tctx, cancel := context.WithTimeout(ctx, time.Minute)
420 + defer cancel()
421 + rootnd, err := i.node.Resolver.DAG.Get(tctx, u.Key(h))
422 if err != nil {
423 webError(w, "Could not resolve root object", err, http.StatusBadRequest)
424 return
core/corerepo/pinning.go
+10 -2
@@ -2,6 +2,9 @@ package corerepo
2
3 import (
4 "fmt"
5 + "time"
6 +
7 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8
9 "github.com/ipfs/go-ipfs/core"
10 "github.com/ipfs/go-ipfs/merkledag"
@@ -27,7 +30,9 @@ func Pin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {
30 return nil, err
31 }
32
30 - err = n.Pinning.Pin(dagnode, recursive)
33 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
34 + defer cancel()
35 + err = n.Pinning.Pin(ctx, dagnode, recursive)
36 if err != nil {
37 return nil, fmt.Errorf("pin: %s", err)
38 }
@@ -56,7 +61,10 @@ func Unpin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {
61 var unpinned []u.Key
62 for _, dagnode := range dagnodes {
63 k, _ := dagnode.Key()
59 - err := n.Pinning.Unpin(k, recursive)
64 +
65 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
66 + defer cancel()
67 + err := n.Pinning.Unpin(ctx, k, recursive)
68 if err != nil {
69 return nil, err
70 }
core/coreunix/add.go
+6 -1
@@ -6,6 +6,9 @@ import (
6 "io/ioutil"
7 "os"
8 gopath "path"
9 + "time"
10 +
11 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
12
13 "github.com/ipfs/go-ipfs/commands/files"
14 core "github.com/ipfs/go-ipfs/core"
@@ -108,7 +111,9 @@ func addNode(n *core.IpfsNode, node *merkledag.Node) error {
111 if err != nil {
112 return err
113 }
111 - err = n.Pinning.Pin(node, true) // ensure we keep it
114 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
115 + defer cancel()
116 + err = n.Pinning.Pin(ctx, node, true) // ensure we keep it
117 if err != nil {
118 return err
119 }
core/coreunix/metadata.go
+12 -2
@@ -1,6 +1,10 @@
1 package coreunix
2
3 import (
4 + "time"
5 +
6 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
7 +
8 core "github.com/ipfs/go-ipfs/core"
9 dag "github.com/ipfs/go-ipfs/merkledag"
10 ft "github.com/ipfs/go-ipfs/unixfs"
@@ -9,7 +13,10 @@ import (
13
14 func AddMetadataTo(n *core.IpfsNode, key string, m *ft.Metadata) (string, error) {
15 ukey := u.B58KeyDecode(key)
12 - nd, err := n.DAG.Get(ukey)
16 +
17 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
18 + defer cancel()
19 + nd, err := n.DAG.Get(ctx, ukey)
20 if err != nil {
21 return "", err
22 }
@@ -36,7 +43,10 @@ func AddMetadataTo(n *core.IpfsNode, key string, m *ft.Metadata) (string, error)
43
44 func Metadata(n *core.IpfsNode, key string) (*ft.Metadata, error) {
45 ukey := u.B58KeyDecode(key)
39 - nd, err := n.DAG.Get(ukey)
46 +
47 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
48 + defer cancel()
49 + nd, err := n.DAG.Get(ctx, ukey)
50 if err != nil {
51 return nil, err
52 }
core/coreunix/metadata_test.go
+2 -1
@@ -8,6 +8,7 @@ import (
8 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9 dssync "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
10 context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
11 +
12 bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
13 bserv "github.com/ipfs/go-ipfs/blockservice"
14 core "github.com/ipfs/go-ipfs/core"
@@ -65,7 +66,7 @@ func TestMetadata(t *testing.T) {
66 t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
67 }
68
68 - retnode, err := ds.Get(u.B58KeyDecode(mdk))
69 + retnode, err := ds.Get(context.Background(), u.B58KeyDecode(mdk))
70 if err != nil {
71 t.Fatal(err)
72 }
fuse/ipns/common.go
+8 -1
@@ -1,6 +1,10 @@
1 package ipns
2
3 import (
4 + "time"
5 +
6 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
7 +
8 "github.com/ipfs/go-ipfs/core"
9 mdag "github.com/ipfs/go-ipfs/merkledag"
10 nsys "github.com/ipfs/go-ipfs/namesys"
@@ -17,7 +21,10 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
21 return err
22 }
23
20 - err = n.Pinning.Pin(emptyDir, false)
24 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
25 + defer cancel()
26 +
27 + err = n.Pinning.Pin(ctx, emptyDir, false)
28 if err != nil {
29 return err
30 }
fuse/readonly/ipfs_test.go
+1 -1
@@ -93,7 +93,7 @@ func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.Node) []str
93 }
94 var out []string
95 for _, lnk := range n.Links {
96 - child, err := lnk.GetNode(ipfs.DAG)
96 + child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
97 if err != nil {
98 t.Fatal(err)
99 }
importer/helpers/helpers.go
+6 -1
@@ -2,7 +2,9 @@ package helpers
2
3 import (
4 "fmt"
5 + "time"
6
7 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8 chunk "github.com/ipfs/go-ipfs/importer/chunk"
9 dag "github.com/ipfs/go-ipfs/merkledag"
10 "github.com/ipfs/go-ipfs/pin"
@@ -76,7 +78,10 @@ func (n *UnixfsNode) NumChildren() int {
78 }
79
80 func (n *UnixfsNode) GetChild(i int, ds dag.DAGService) (*UnixfsNode, error) {
79 - nd, err := n.node.Links[i].GetNode(ds)
81 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
82 + defer cancel()
83 +
84 + nd, err := n.node.Links[i].GetNode(ctx, ds)
85 if err != nil {
86 return nil, err
87 }
importer/trickle/trickle_test.go
+1 -1
@@ -607,7 +607,7 @@ func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) {
607 fmt.Println()
608 }
609 for _, lnk := range nd.Links {
610 - child, err := lnk.GetNode(ds)
610 + child, err := lnk.GetNode(context.Background(), ds)
611 if err != nil {
612 panic(err)
613 }
importer/trickle/trickledag.go
+7 -1
@@ -2,6 +2,10 @@ package trickle
2
3 import (
4 "errors"
5 + "time"
6 +
7 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8 +
9 h "github.com/ipfs/go-ipfs/importer/helpers"
10 dag "github.com/ipfs/go-ipfs/merkledag"
11 ft "github.com/ipfs/go-ipfs/unixfs"
@@ -259,7 +263,9 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
263 }
264
265 for i := 0; i < len(nd.Links); i++ {
262 - child, err := nd.Links[i].GetNode(ds)
266 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
267 + defer cancel()
268 + child, err := nd.Links[i].GetNode(ctx, ds)
269 if err != nil {
270 return err
271 }
ipnsfs/dir.go
+7 -1
@@ -5,6 +5,9 @@ import (
5 "fmt"
6 "os"
7 "sync"
8 + "time"
9 +
10 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
11
12 dag "github.com/ipfs/go-ipfs/merkledag"
13 ft "github.com/ipfs/go-ipfs/unixfs"
@@ -135,7 +138,10 @@ func (d *Directory) childDir(name string) (*Directory, error) {
138 func (d *Directory) childFromDag(name string) (*dag.Node, error) {
139 for _, lnk := range d.node.Links {
140 if lnk.Name == name {
138 - return lnk.GetNode(d.fs.dserv)
141 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
142 + defer cancel()
143 +
144 + return lnk.GetNode(ctx, d.fs.dserv)
145 }
146 }
147
ipnsfs/system.go
+1 -1
@@ -159,7 +159,7 @@ func (fs *Filesystem) newKeyRoot(parent context.Context, k ci.PrivKey) (*KeyRoot
159 }
160 }
161
162 - mnode, err := fs.dserv.Get(pointsTo)
162 + mnode, err := fs.dserv.Get(ctx, pointsTo)
163 if err != nil {
164 return nil, err
165 }
merkledag/merkledag.go
+3 -10
@@ -4,7 +4,6 @@ package merkledag
4 import (
5 "fmt"
6 "sync"
7 - "time"
7
8 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
9 blocks "github.com/ipfs/go-ipfs/blocks"
@@ -19,7 +18,7 @@ var ErrNotFound = fmt.Errorf("merkledag: not found")
18 type DAGService interface {
19 Add(*Node) (u.Key, error)
20 AddRecursive(*Node) error
22 - Get(u.Key) (*Node, error)
21 + Get(context.Context, u.Key) (*Node, error)
22 Remove(*Node) error
23
24 // GetDAG returns, in order, all the single leve child
@@ -83,17 +82,11 @@ func (n *dagService) AddRecursive(nd *Node) error {
82 }
83
84 // Get retrieves a node from the dagService, fetching the block in the BlockService
86 -func (n *dagService) Get(k u.Key) (*Node, error) {
85 +func (n *dagService) Get(ctx context.Context, k u.Key) (*Node, error) {
86 if n == nil {
87 return nil, fmt.Errorf("dagService is nil")
88 }
89
91 - ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
92 - defer cancel()
93 - // we shouldn't use an arbitrary timeout here.
94 - // since Get doesnt take in a context yet, we give a large upper bound.
95 - // think of an http request. we want it to go on as long as the client requests it.
96 -
90 b, err := n.Blocks.GetBlock(ctx, k)
91 if err != nil {
92 return nil, err
@@ -134,7 +127,7 @@ func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{}
127 return
128 }
129
137 - nd, err := lnk.GetNode(serv)
130 + nd, err := lnk.GetNode(ctx, serv)
131 if err != nil {
132 log.Debug(err)
133 return
merkledag/merkledag_test.go
+1 -1
@@ -190,7 +190,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
190 wg.Add(1)
191 go func(i int) {
192 defer wg.Done()
193 - first, err := dagservs[i].Get(k)
193 + first, err := dagservs[i].Get(context.Background(), k)
194 if err != nil {
195 t.Fatal(err)
196 }
merkledag/node.go
+4 -2
@@ -3,6 +3,8 @@ package merkledag
3 import (
4 "fmt"
5
6 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
7 +
8 mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
9 u "github.com/ipfs/go-ipfs/util"
10 )
@@ -77,12 +79,12 @@ func MakeLink(n *Node) (*Link, error) {
79 }
80
81 // GetNode returns the MDAG Node that this link points to
80 -func (l *Link) GetNode(serv DAGService) (*Node, error) {
82 +func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
83 if l.Node != nil {
84 return l.Node, nil
85 }
86
85 - return serv.Get(u.Key(l.Hash))
87 + return serv.Get(ctx, u.Key(l.Hash))
88 }
89
90 // AddNodeLink adds a link to another node.
merkledag/traverse/traverse.go
+7 -1
@@ -3,6 +3,9 @@ package traverse
3
4 import (
5 "errors"
6 + "time"
7 +
8 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
9
10 mdag "github.com/ipfs/go-ipfs/merkledag"
11 )
@@ -64,7 +67,10 @@ func (t *traversal) callFunc(next State) error {
67 func (t *traversal) getNode(link *mdag.Link) (*mdag.Node, error) {
68
69 getNode := func(l *mdag.Link) (*mdag.Node, error) {
67 - next, err := l.GetNode(t.opts.DAG)
70 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
71 + defer cancel()
72 +
73 + next, err := l.GetNode(ctx, t.opts.DAG)
74 if err != nil {
75 return nil, err
76 }
namesys/publisher.go
+1 -1
@@ -150,7 +150,7 @@ func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, p
150
151 // pin recursively because this might already be pinned
152 // and doing a direct pin would throw an error in that case
153 - err = pins.Pin(emptyDir, true)
153 + err = pins.Pin(ctx, emptyDir, true)
154 if err != nil {
155 return err
156 }
path/resolver.go
+8 -2
@@ -3,8 +3,10 @@ package path
3
4 import (
5 "fmt"
6 + "time"
7
8 mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
9 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
10 merkledag "github.com/ipfs/go-ipfs/merkledag"
11 u "github.com/ipfs/go-ipfs/util"
12 )
@@ -74,7 +76,9 @@ func (s *Resolver) ResolvePathComponents(fpath Path) ([]*merkledag.Node, error)
76 }
77
78 log.Debug("Resolve dag get.\n")
77 - nd, err := s.DAG.Get(u.Key(h))
79 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
80 + defer cancel()
81 + nd, err := s.DAG.Get(ctx, u.Key(h))
82 if err != nil {
83 return nil, err
84 }
@@ -117,7 +121,9 @@ func (s *Resolver) ResolveLinks(ndd *merkledag.Node, names []string) (
121
122 if nlink.Node == nil {
123 // fetch object for link and assign to nd
120 - nd, err = s.DAG.Get(next)
124 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
125 + defer cancel()
126 + nd, err = s.DAG.Get(ctx, next)
127 if err != nil {
128 return append(result, nd), err
129 }
pin/indirect.go
+10 -3
@@ -28,9 +28,11 @@ func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) {
28 refcnt := make(map[util.Key]int)
29 var keys []util.Key
30 for encK, v := range rcStore {
31 - k := util.B58KeyDecode(encK)
32 - keys = append(keys, k)
33 - refcnt[k] = v
31 + if v > 0 {
32 + k := util.B58KeyDecode(encK)
33 + keys = append(keys, k)
34 + refcnt[k] = v
35 + }
36 }
37 // log.Debugf("indirPin keys: %#v", keys)
38
@@ -59,6 +61,7 @@ func (i *indirectPin) Decrement(k util.Key) {
61 i.refCounts[k] = c
62 if c <= 0 {
63 i.blockset.RemoveBlock(k)
64 + delete(i.refCounts, k)
65 }
66 }
67
@@ -69,3 +72,7 @@ func (i *indirectPin) HasKey(k util.Key) bool {
72 func (i *indirectPin) Set() set.BlockSet {
73 return i.blockset
74 }
75 +
76 +func (i *indirectPin) GetRefs() map[util.Key]int {
77 + return i.refCounts
78 +}
pin/pin.go
+26 -24
@@ -7,7 +7,6 @@ import (
7 "errors"
8 "fmt"
9 "sync"
10 - "time"
10
11 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
12 nsds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
@@ -33,12 +32,12 @@ const (
32
33 type Pinner interface {
34 IsPinned(util.Key) bool
36 - Pin(*mdag.Node, bool) error
37 - Unpin(util.Key, bool) error
35 + Pin(context.Context, *mdag.Node, bool) error
36 + Unpin(context.Context, util.Key, bool) error
37 Flush() error
38 GetManual() ManualPinner
39 DirectKeys() []util.Key
41 - IndirectKeys() []util.Key
40 + IndirectKeys() map[util.Key]int
41 RecursiveKeys() []util.Key
42 }
43
@@ -82,7 +81,7 @@ func NewPinner(dstore ds.ThreadSafeDatastore, serv mdag.DAGService) Pinner {
81 }
82
83 // Pin the given node, optionally recursive
85 -func (p *pinner) Pin(node *mdag.Node, recurse bool) error {
84 +func (p *pinner) Pin(ctx context.Context, node *mdag.Node, recurse bool) error {
85 p.lock.Lock()
86 defer p.lock.Unlock()
87 k, err := node.Key()
@@ -99,34 +98,40 @@ func (p *pinner) Pin(node *mdag.Node, recurse bool) error {
98 p.directPin.RemoveBlock(k)
99 }
100
102 - p.recursePin.AddBlock(k)
103 -
104 - err := p.pinLinks(node)
101 + err := p.pinLinks(ctx, node)
102 if err != nil {
103 return err
104 }
105 +
106 + p.recursePin.AddBlock(k)
107 } else {
108 + _, err := p.dserv.Get(ctx, k)
109 + if err != nil {
110 + return err
111 + }
112 +
113 if p.recursePin.HasKey(k) {
114 return fmt.Errorf("%s already pinned recursively", k.B58String())
115 }
116 +
117 p.directPin.AddBlock(k)
118 }
119 return nil
120 }
121
122 // Unpin a given key
118 -func (p *pinner) Unpin(k util.Key, recursive bool) error {
123 +func (p *pinner) Unpin(ctx context.Context, k util.Key, recursive bool) error {
124 p.lock.Lock()
125 defer p.lock.Unlock()
126 if p.recursePin.HasKey(k) {
127 if recursive {
128 p.recursePin.RemoveBlock(k)
124 - node, err := p.dserv.Get(k)
129 + node, err := p.dserv.Get(ctx, k)
130 if err != nil {
131 return err
132 }
133
129 - return p.unpinLinks(node)
134 + return p.unpinLinks(ctx, node)
135 } else {
136 return fmt.Errorf("%s is pinned recursively", k)
137 }
@@ -140,9 +145,9 @@ func (p *pinner) Unpin(k util.Key, recursive bool) error {
145 }
146 }
147
143 -func (p *pinner) unpinLinks(node *mdag.Node) error {
148 +func (p *pinner) unpinLinks(ctx context.Context, node *mdag.Node) error {
149 for _, l := range node.Links {
145 - node, err := l.GetNode(p.dserv)
150 + node, err := l.GetNode(ctx, p.dserv)
151 if err != nil {
152 return err
153 }
@@ -152,9 +157,9 @@ func (p *pinner) unpinLinks(node *mdag.Node) error {
157 return err
158 }
159
155 - p.recursePin.RemoveBlock(k)
160 + p.indirPin.Decrement(k)
161
157 - err = p.unpinLinks(node)
162 + err = p.unpinLinks(ctx, node)
163 if err != nil {
164 return err
165 }
@@ -162,27 +167,24 @@ func (p *pinner) unpinLinks(node *mdag.Node) error {
167 return nil
168 }
169
165 -func (p *pinner) pinIndirectRecurse(node *mdag.Node) error {
170 +func (p *pinner) pinIndirectRecurse(ctx context.Context, node *mdag.Node) error {
171 k, err := node.Key()
172 if err != nil {
173 return err
174 }
175
176 p.indirPin.Increment(k)
172 - return p.pinLinks(node)
177 + return p.pinLinks(ctx, node)
178 }
179
175 -func (p *pinner) pinLinks(node *mdag.Node) error {
176 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
177 - defer cancel()
178 -
180 +func (p *pinner) pinLinks(ctx context.Context, node *mdag.Node) error {
181 for _, ng := range p.dserv.GetDAG(ctx, node) {
182 subnode, err := ng.Get(ctx)
183 if err != nil {
184 // TODO: Maybe just log and continue?
185 return err
186 }
185 - err = p.pinIndirectRecurse(subnode)
187 + err = p.pinIndirectRecurse(ctx, subnode)
188 if err != nil {
189 return err
190 }
@@ -256,8 +258,8 @@ func (p *pinner) DirectKeys() []util.Key {
258 }
259
260 // IndirectKeys returns a slice containing the indirectly pinned keys
259 -func (p *pinner) IndirectKeys() []util.Key {
260 - return p.indirPin.Set().GetKeys()
261 +func (p *pinner) IndirectKeys() map[util.Key]int {
262 + return p.indirPin.GetRefs()
263 }
264
265 // RecursiveKeys returns a slice containing the recursively pinned keys
pin/pin_test.go
+53 -7
@@ -2,6 +2,9 @@ package pin
2
3 import (
4 "testing"
5 + "time"
6 +
7 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8
9 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 dssync "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
@@ -21,6 +24,8 @@ func randNode() (*mdag.Node, util.Key) {
24 }
25
26 func TestPinnerBasic(t *testing.T) {
27 + ctx := context.Background()
28 +
29 dstore := dssync.MutexWrap(ds.NewMapDatastore())
30 bstore := blockstore.NewBlockstore(dstore)
31 bserv, err := bs.New(bstore, offline.Exchange(bstore))
@@ -40,7 +45,7 @@ func TestPinnerBasic(t *testing.T) {
45 }
46
47 // Pin A{}
43 - err = p.Pin(a, false)
48 + err = p.Pin(ctx, a, false)
49 if err != nil {
50 t.Fatal(err)
51 }
@@ -74,7 +79,7 @@ func TestPinnerBasic(t *testing.T) {
79 }
80
81 // recursively pin B{A,C}
77 - err = p.Pin(b, true)
82 + err = p.Pin(ctx, b, true)
83 if err != nil {
84 t.Fatal(err)
85 }
@@ -102,7 +107,7 @@ func TestPinnerBasic(t *testing.T) {
107 }
108
109 // Add D{A,C,E}
105 - err = p.Pin(d, true)
110 + err = p.Pin(ctx, d, true)
111 if err != nil {
112 t.Fatal(err)
113 }
@@ -117,7 +122,7 @@ func TestPinnerBasic(t *testing.T) {
122 }
123
124 // Test recursive unpin
120 - err = p.Unpin(dk, true)
125 + err = p.Unpin(ctx, dk, true)
126 if err != nil {
127 t.Fatal(err)
128 }
@@ -154,6 +159,7 @@ func TestPinnerBasic(t *testing.T) {
159 }
160
161 func TestDuplicateSemantics(t *testing.T) {
162 + ctx := context.Background()
163 dstore := dssync.MutexWrap(ds.NewMapDatastore())
164 bstore := blockstore.NewBlockstore(dstore)
165 bserv, err := bs.New(bstore, offline.Exchange(bstore))
@@ -173,19 +179,59 @@ func TestDuplicateSemantics(t *testing.T) {
179 }
180
181 // pin is recursively
176 - err = p.Pin(a, true)
182 + err = p.Pin(ctx, a, true)
183 if err != nil {
184 t.Fatal(err)
185 }
186
187 // pinning directly should fail
182 - err = p.Pin(a, false)
188 + err = p.Pin(ctx, a, false)
189 if err == nil {
190 t.Fatal("expected direct pin to fail")
191 }
192
193 // pinning recursively again should succeed
188 - err = p.Pin(a, true)
194 + err = p.Pin(ctx, a, true)
195 + if err != nil {
196 + t.Fatal(err)
197 + }
198 +}
199 +
200 +func TestPinRecursiveFail(t *testing.T) {
201 + ctx := context.Background()
202 + dstore := dssync.MutexWrap(ds.NewMapDatastore())
203 + bstore := blockstore.NewBlockstore(dstore)
204 + bserv, err := bs.New(bstore, offline.Exchange(bstore))
205 + if err != nil {
206 + t.Fatal(err)
207 + }
208 +
209 + dserv := mdag.NewDAGService(bserv)
210 +
211 + p := NewPinner(dstore, dserv)
212 +
213 + a, _ := randNode()
214 + b, _ := randNode()
215 + err = a.AddNodeLinkClean("child", b)
216 + if err != nil {
217 + t.Fatal(err)
218 + }
219 +
220 + // Note: this isnt a time based test, we expect the pin to fail
221 + mctx, _ := context.WithTimeout(ctx, time.Millisecond)
222 + err = p.Pin(mctx, a, true)
223 + if err == nil {
224 + t.Fatal("should have failed to pin here")
225 + }
226 +
227 + _, err = dserv.Add(b)
228 + if err != nil {
229 + t.Fatal(err)
230 + }
231 +
232 + // this one is time based... but shouldnt cause any issues
233 + mctx, _ = context.WithTimeout(ctx, time.Second)
234 + err = p.Pin(mctx, a, true)
235 if err != nil {
236 t.Fatal(err)
237 }
test/bin/ipfs-pin-stat new
+30
@@ -0,0 +1,30 @@
1 +#!/bin/sh
2 +
3 +die() {
4 + echo "$@"
5 + exit 1
6 +}
7 +
8 +if [ "$#" -eq 0 ]; then
9 + echo "usage: $0 <object>"
10 + echo "show ipfs pin information for object"
11 + exit 1
12 +fi
13 +
14 +path=$1
15 +
16 +echo "$path" | grep "/" >/dev/null
17 +if [ "$?" -eq 0 ]; then
18 + die "error: paths not supported. please resolve to hash first."
19 +fi
20 +
21 +ipfs pin ls --type=recursive | grep "$path" >/dev/null
22 +[ "$?" -eq 0 ] && echo "$path pinned recursively"
23 +
24 +ipfs pin ls --type=indirect | grep "$path" >/dev/null
25 +[ "$?" -eq 0 ] && echo "$path pinned indirectly"
26 +
27 +ipfs pin ls --type=direct | grep "$path" >/dev/null
28 +[ "$?" -eq 0 ] && echo "$path pinned directly"
29 +
30 +exit 0
test/sharness/lib/test-lib.sh
+6
@@ -295,3 +295,9 @@ test_should_contain() {
295 return 1
296 fi
297 }
298 +
299 +test_str_contains() {
300 + find=$1
301 + shift
302 + echo "$@" | grep "$find" >/dev/null
303 +}
test/sharness/t0080-repo.sh
+2 -1
@@ -140,8 +140,9 @@ test_expect_success "'ipfs pin ls -type=all' is correct" '
140 cat directpinout >allpins &&
141 cat rp_actual >>allpins &&
142 cat indirectpins >>allpins &&
143 + cat allpins | sort | uniq >> allpins_uniq &&
144 ipfs pin ls -type=all >actual_allpins &&
144 - test_sort_cmp allpins actual_allpins
145 + test_sort_cmp allpins_uniq actual_allpins
146 '
147
148 test_kill_ipfs_daemon
test/sharness/t0081-repo-pinning.sh new
+251
@@ -0,0 +1,251 @@
1 +#!/bin/sh
2 +#
3 +# Copyright (c) 2014 Jeromy Johnson
4 +# MIT Licensed; see the LICENSE file in this repository.
5 +#
6 +
7 +test_description="Test ipfs repo pinning"
8 +
9 +. lib/test-lib.sh
10 +
11 +
12 +
13 +function test_pin_flag() {
14 + object=$1
15 + ptype=$2
16 + expect=$3
17 +
18 + echo "test_pin_flag" $@
19 +
20 + ipfs-pin-stat "$object" | grep "$ptype"
21 + actual=$?
22 +
23 + if [ "$expect" = "true" ]; then
24 + if [ "$actual" != "0" ]; then
25 + echo "$object should be pinned $ptype ($actual)"
26 + return 1
27 + fi
28 + else
29 + if [ "$actual" != "1" ]; then
30 + echo "$object should NOT be pinned $ptype ($actual)"
31 + return 1
32 + fi
33 + fi
34 + return 0
35 +}
36 +
37 +function test_pin() {
38 + object=$1
39 + shift
40 +
41 + test_str_contains "recursive" $@
42 + [ "$?" = "0" ] && r="true" || r="false"
43 +
44 + test_str_contains "indirect" $@
45 + [ "$?" = "0" ] && i="true" || i="false"
46 +
47 + test_str_contains "direct" $@
48 + [ "$?" = "0" ] && d="true" || d="false"
49 +
50 + test_pin_flag "$object" "recursive" $r || return 1
51 + test_pin_flag "$object" "indirect" $i || return 1
52 + test_pin_flag "$object" "direct" $d || return 1
53 + return 0
54 +}
55 +
56 +
57 +test_init_ipfs
58 +
59 +# test runs much faster without daemon.
60 +# TODO: turn this back on after:
61 +# https://github.com/ipfs/go-ipfs/issues/1075
62 +# test_launch_ipfs_daemon
63 +
64 +HASH_FILE6="QmRsBC3Y2G6VRPYGAVpZczx1W7Xw54MtM1NcLKTkn6rx3U"
65 +HASH_FILE5="QmaN3PtyP8DcVGHi3Q2Fcp7CfAFVcVXKddWbHoNvaA41zf"
66 +HASH_FILE4="QmV1aiVgpDknKQugrK59uBUbMrPnsQM1F9FXbFcfgEvUvH"
67 +HASH_FILE3="QmZrr4Pzqp3NnMzMfbMhNe7LghfoUFHVx7c9Po9GZrhKZ7"
68 +HASH_FILE2="QmSkjTornLY72QhmK9NvAz26815pTaoAL42rF8Qi3w2WBP"
69 +HASH_FILE1="QmbgX4aXhSSY88GHmPQ4roizD8wFwPX8jzTLjc8VAp89x4"
70 +HASH_DIR4="QmW98gV71Ns4bX7QbgWAqLiGF3SDC1JpveZSgBh4ExaSAd"
71 +HASH_DIR3="QmRsCaNBMkweZ9vHT5PJRd2TT9rtNKEKyuognCEVxZxF1H"
72 +HASH_DIR2="QmTUTQAgeVfughDSFukMZLbfGvetDJY7Ef5cDXkKK4abKC"
73 +HASH_DIR1="QmNyZVFbgvmzguS2jVMRb8PQMNcCMJrn9E3doDhBbcPNTY"
74 +
75 +DIR1="dir1"
76 +DIR2="dir1/dir2"
77 +DIR4="dir1/dir2/dir4"
78 +DIR3="dir1/dir3"
79 +FILE1="dir1/file1"
80 +FILE2="dir1/file2"
81 +FILE3="dir1/file3"
82 +FILE4="dir1/dir2/file4"
83 +FILE6="dir1/dir2/dir4/file6"
84 +FILE5="dir1/dir3/file5"
85 +
86 +test_expect_success "'ipfs add dir' succeeds" '
87 + mkdir dir1 &&
88 + mkdir dir1/dir2 &&
89 + mkdir dir1/dir2/dir4 &&
90 + mkdir dir1/dir3 &&
91 + echo "some text 1" >dir1/file1 &&
92 + echo "some text 2" >dir1/file2 &&
93 + echo "some text 3" >dir1/file3 &&
94 + echo "some text 1" >dir1/dir2/file1 &&
95 + echo "some text 4" >dir1/dir2/file4 &&
96 + echo "some text 1" >dir1/dir2/dir4/file1 &&
97 + echo "some text 2" >dir1/dir2/dir4/file2 &&
98 + echo "some text 6" >dir1/dir2/dir4/file6 &&
99 + echo "some text 2" >dir1/dir3/file2 &&
100 + echo "some text 5" >dir1/dir3/file5 &&
101 + ipfs add -q -r dir1 | tail -n1 >actual1 &&
102 + echo "$HASH_DIR1" >expected1 &&
103 + test_cmp actual1 expected1
104 +'
105 +
106 +test_expect_success "objects are there" '
107 + ipfs cat "$HASH_FILE6" >FILE6_a &&
108 + ipfs cat "$HASH_FILE5" >FILE5_a &&
109 + ipfs cat "$HASH_FILE4" >FILE4_a &&
110 + ipfs cat "$HASH_FILE3" >FILE3_a &&
111 + ipfs cat "$HASH_FILE2" >FILE2_a &&
112 + ipfs cat "$HASH_FILE1" >FILE1_a &&
113 + ipfs ls "$HASH_DIR3" >DIR3_a &&
114 + ipfs ls "$HASH_DIR4" >DIR4_a &&
115 + ipfs ls "$HASH_DIR2" >DIR2_a &&
116 + ipfs ls "$HASH_DIR1" >DIR1_a
117 +'
118 +
119 +# saving this output for later
120 +test_expect_success "ipfs object links $HASH_DIR1 works" '
121 + ipfs object links $HASH_DIR1 > DIR1_objlink
122 +'
123 +
124 +
125 +test_expect_success "added dir was pinned recursively" '
126 + test_pin_flag $HASH_DIR1 recursive true
127 +'
128 +
129 +test_expect_success "rest were pinned indirectly" '
130 + test_pin_flag "$HASH_FILE6" indirect true
131 + test_pin_flag "$HASH_FILE5" indirect true
132 + test_pin_flag "$HASH_FILE4" indirect true
133 + test_pin_flag "$HASH_FILE3" indirect true
134 + test_pin_flag "$HASH_FILE2" indirect true
135 + test_pin_flag "$HASH_FILE1" indirect true
136 + test_pin_flag "$HASH_DIR3" indirect true
137 + test_pin_flag "$HASH_DIR4" indirect true
138 + test_pin_flag "$HASH_DIR2" indirect true
139 +'
140 +
141 +test_expect_success "added dir was NOT pinned indirectly" '
142 + test_pin_flag "$HASH_DIR1" indirect false
143 +'
144 +
145 +test_expect_success "nothing is pinned directly" '
146 + ipfs pin ls -type=direct >actual4 &&
147 + test_must_be_empty actual4
148 +'
149 +
150 +test_expect_success "'ipfs repo gc' succeeds" '
151 + ipfs repo gc >gc_out_actual &&
152 + test_must_be_empty gc_out_actual
153 +'
154 +
155 +test_expect_success "objects are still there" '
156 + cat FILE6_a FILE5_a FILE4_a FILE3_a FILE2_a FILE1_a >expected45 &&
157 + cat DIR3_a DIR4_a DIR2_a DIR1_a >>expected45 &&
158 + ipfs cat "$HASH_FILE6" >actual45 &&
159 + ipfs cat "$HASH_FILE5" >>actual45 &&
160 + ipfs cat "$HASH_FILE4" >>actual45 &&
161 + ipfs cat "$HASH_FILE3" >>actual45 &&
162 + ipfs cat "$HASH_FILE2" >>actual45 &&
163 + ipfs cat "$HASH_FILE1" >>actual45 &&
164 + ipfs ls "$HASH_DIR3" >>actual45 &&
165 + ipfs ls "$HASH_DIR4" >>actual45 &&
166 + ipfs ls "$HASH_DIR2" >>actual45 &&
167 + ipfs ls "$HASH_DIR1" >>actual45 &&
168 + test_cmp expected45 actual45
169 +'
170 +
171 +test_expect_success "remove dir recursive pin succeeds" '
172 + echo "unpinned $HASH_DIR1" >expected5 &&
173 + ipfs pin rm -r "$HASH_DIR1" >actual5 &&
174 + test_cmp expected5 actual5
175 +'
176 +
177 +test_expect_success "none are pinned any more" '
178 + test_pin "$HASH_FILE6" &&
179 + test_pin "$HASH_FILE5" &&
180 + test_pin "$HASH_FILE4" &&
181 + test_pin "$HASH_FILE3" &&
182 + test_pin "$HASH_FILE2" &&
183 + test_pin "$HASH_FILE1" &&
184 + test_pin "$HASH_DIR3" &&
185 + test_pin "$HASH_DIR4" &&
186 + test_pin "$HASH_DIR2" &&
187 + test_pin "$HASH_DIR1"
188 +'
189 +
190 +test_expect_success "pin some directly and indirectly" '
191 + ipfs pin add "$HASH_DIR1" >actual7 &&
192 + ipfs pin add -r "$HASH_DIR2" >>actual7 &&
193 + ipfs pin add "$HASH_FILE1" >>actual7 &&
194 + echo "pinned $HASH_DIR1 directly" >expected7 &&
195 + echo "pinned $HASH_DIR2 recursively" >>expected7 &&
196 + echo "pinned $HASH_FILE1 directly" >>expected7 &&
197 + test_cmp expected7 actual7
198 +'
199 +
200 +test_expect_success "pin lists look good" '
201 + test_pin $HASH_DIR1 direct &&
202 + test_pin $HASH_DIR2 recursive &&
203 + test_pin $HASH_DIR3 &&
204 + test_pin $HASH_DIR4 indirect &&
205 + test_pin $HASH_FILE1 indirect direct &&
206 + test_pin $HASH_FILE2 indirect &&
207 + test_pin $HASH_FILE3 &&
208 + test_pin $HASH_FILE4 indirect &&
209 + test_pin $HASH_FILE5 &&
210 + test_pin $HASH_FILE6 indirect
211 +'
212 +
213 +test_expect_success "'ipfs repo gc' succeeds" '
214 + ipfs repo gc >gc_out_actual2 &&
215 + echo "removed $HASH_FILE3" > gc_out_exp2 &&
216 + echo "removed $HASH_FILE5" >> gc_out_exp2 &&
217 + echo "removed $HASH_DIR3" >> gc_out_exp2 &&
218 + test_sort_cmp gc_out_actual2 gc_out_exp2
219 +'
220 +
221 +# use object links for HASH_DIR1 here because its children
222 +# no longer exist
223 +test_expect_success "some objects are still there" '
224 + cat FILE6_a FILE4_a FILE2_a FILE1_a >expected8 &&
225 + cat DIR4_a DIR2_a DIR1_objlink >>expected8 &&
226 + ipfs cat "$HASH_FILE6" >actual8 &&
227 + ipfs cat "$HASH_FILE4" >>actual8 &&
228 + ipfs cat "$HASH_FILE2" >>actual8 &&
229 + ipfs cat "$HASH_FILE1" >>actual8 &&
230 + ipfs ls "$HASH_DIR4" >>actual8 &&
231 + ipfs ls "$HASH_DIR2" >>actual8 &&
232 + ipfs object links "$HASH_DIR1" >>actual8 &&
233 + test_cmp actual8 expected8
234 +'
235 +
236 +# todo: make this faster somehow.
237 +test_expect_success "some are no longer there" '
238 + test_must_fail ipfs cat "$HASH_FILE5" &&
239 + test_must_fail ipfs cat "$HASH_FILE3" &&
240 + test_must_fail ipfs ls "$HASH_DIR3"
241 +'
242 +
243 +test_expect_success "recursive pin fails without objects" '
244 + ipfs pin rm "$HASH_DIR1" &&
245 + test_must_fail ipfs pin add -r "$HASH_DIR1" 2>err_expected8 &&
246 + grep "context deadline exceeded" err_expected8
247 +'
248 +
249 +# test_kill_ipfs_daemon
250 +
251 +test_done
unixfs/io/dagreader.go
+1 -1
@@ -74,7 +74,7 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*Dag
74 if len(n.Links) == 0 {
75 return nil, errors.New("incorrectly formatted metadata object")
76 }
77 - child, err := n.Links[0].GetNode(serv)
77 + child, err := n.Links[0].GetNode(ctx, serv)
78 if err != nil {
79 return nil, err
80 }
unixfs/io/dirbuilder.go
+8 -1
@@ -1,6 +1,10 @@
1 package io
2
3 import (
4 + "time"
5 +
6 + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
7 +
8 mdag "github.com/ipfs/go-ipfs/merkledag"
9 format "github.com/ipfs/go-ipfs/unixfs"
10 u "github.com/ipfs/go-ipfs/util"
@@ -20,7 +24,10 @@ func NewDirectory(dserv mdag.DAGService) *directoryBuilder {
24 }
25
26 func (d *directoryBuilder) AddChild(name string, k u.Key) error {
23 - cnode, err := d.dserv.Get(k)
27 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
28 + defer cancel()
29 +
30 + cnode, err := d.dserv.Get(ctx, k)
31 if err != nil {
32 return err
33 }
unixfs/mod/dagmodifier.go
+7 -3
@@ -5,6 +5,7 @@ import (
5 "errors"
6 "io"
7 "os"
8 + "time"
9
10 proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
11 mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
@@ -184,7 +185,7 @@ func (dm *DagModifier) Sync() error {
185 return err
186 }
187
187 - nd, err := dm.dagserv.Get(thisk)
188 + nd, err := dm.dagserv.Get(dm.ctx, thisk)
189 if err != nil {
190 return err
191 }
@@ -267,7 +268,7 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
268 ckey := u.Key(node.Links[i].Hash)
269 dm.mp.RemovePinWithMode(ckey, pin.Indirect)
270
270 - child, err := node.Links[i].GetNode(dm.dagserv)
271 + child, err := node.Links[i].GetNode(dm.ctx, dm.dagserv)
272 if err != nil {
273 return "", false, err
274 }
@@ -457,7 +458,10 @@ func dagTruncate(nd *mdag.Node, size uint64, ds mdag.DAGService) (*mdag.Node, er
458 var modified *mdag.Node
459 ndata := new(ft.FSNode)
460 for i, lnk := range nd.Links {
460 - child, err := lnk.GetNode(ds)
461 + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
462 + defer cancel()
463 +
464 + child, err := lnk.GetNode(ctx, ds)
465 if err != nil {
466 return nil, err
467 }
unixfs/mod/dagmodifier_test.go
+2 -2
@@ -578,7 +578,7 @@ func enumerateChildren(t *testing.T, nd *mdag.Node, ds mdag.DAGService) []u.Key
578 var out []u.Key
579 for _, lnk := range nd.Links {
580 out = append(out, u.Key(lnk.Hash))
581 - child, err := lnk.GetNode(ds)
581 + child, err := lnk.GetNode(context.Background(), ds)
582 if err != nil {
583 t.Fatal(err)
584 }
@@ -643,7 +643,7 @@ func printDag(nd *mdag.Node, ds mdag.DAGService, indent int) {
643 fmt.Println()
644 }
645 for _, lnk := range nd.Links {
646 - child, err := lnk.GetNode(ds)
646 + child, err := lnk.GetNode(context.Background(), ds)
647 if err != nil {
648 panic(err)
649 }