Feat: depth limited refs -r
This adds --max-depth to the "refs" commands and allows limiting the fetching of refs per depth. Other than that, it works as before. Note that clever branch pruning is only made when the --unique flag is passed. Otherwise, we re-explore branches to the given depth. This means that --unique costs memory, but may save time when the DAGs contain the same sub-DAGs in several places (specially if they are big). On the other side, not using --unique saves memory but may involve re-exploring large sub-DAGs. License: MIT Signed-off-by: Hector Sanjuan <hector@protocol.ai>
Hector Sanjuan committed
Aug 3, 2018 at 12:18 UTC
ceb37a346d02116046a0669fff580b69fff5502f
1 file changed
+107
-55
core/commands/refs.go
+107
-55
@@ -10,11 +10,11 @@ import (
10
cmds "github.com/ipfs/go-ipfs/commands"
11
"github.com/ipfs/go-ipfs/core"
12
e "github.com/ipfs/go-ipfs/core/commands/e"
13
- path "gx/ipfs/QmdMPBephdLYNESkruDX2hcDTgFYhoCt4LimWhgnomSdV2/go-path"
13
15
- "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
14
+ cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
15
ipld "gx/ipfs/QmX5CsuHyVZeTLxgRSYkgLSDQKb9UjE8xnhQzCEJWWWFsC/go-ipld-format"
16
cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid"
17
+ path "gx/ipfs/QmdMPBephdLYNESkruDX2hcDTgFYhoCt4LimWhgnomSdV2/go-path"
18
)
19
20
// KeyList is a general type for outputting lists of keys
@@ -64,6 +64,7 @@ NOTE: List all references recursively by using the flag '-r'.
64
cmdkit.BoolOption("edges", "e", "Emit edge format: `<from> -> <to>`."),
65
cmdkit.BoolOption("unique", "u", "Omit duplicate refs from output."),
66
cmdkit.BoolOption("recursive", "r", "Recursively list links of child nodes."),
67
+ cmdkit.IntOption("max-depth", "Only for recursive refs, limits fetch and listing to the given depth").WithDefault(-1),
68
},
69
Run: func(req cmds.Request, res cmds.Response) {
70
ctx := req.Context()
@@ -85,6 +86,16 @@ NOTE: List all references recursively by using the flag '-r'.
86
return
87
}
88
89
+ maxDepth, _, err := req.Option("max-depth").Int()
90
+ if err != nil {
91
+ res.SetError(err, cmdkit.ErrNormal)
92
+ return
93
+ }
94
+
95
+ if !recursive {
96
+ maxDepth = 1 // write only direct refs
97
+ }
98
+
99
format, _, err := req.Option("format").String()
100
if err != nil {
101
res.SetError(err, cmdkit.ErrNormal)
@@ -119,12 +130,12 @@ NOTE: List all references recursively by using the flag '-r'.
130
defer close(out)
131
132
rw := RefWriter{
122
- out: out,
123
- DAG: n.DAG,
124
- Ctx: ctx,
125
- Unique: unique,
126
- PrintFmt: format,
127
- Recursive: recursive,
133
+ out: out,
134
+ DAG: n.DAG,
135
+ Ctx: ctx,
136
+ Unique: unique,
137
+ PrintFmt: format,
138
+ MaxDepth: maxDepth,
139
}
140
141
for _, o := range objs {
@@ -231,86 +242,127 @@ type RefWriter struct {
242
DAG ipld.DAGService
243
Ctx context.Context
244
234
- Unique bool
235
- Recursive bool
236
- PrintFmt string
245
+ Unique bool
246
+ MaxDepth int
247
+ PrintFmt string
248
238
- seen *cid.Set
249
+ seen map[string]int
250
}
251
252
// WriteRefs writes refs of the given object to the underlying writer.
253
func (rw *RefWriter) WriteRefs(n ipld.Node) (int, error) {
243
- if rw.Recursive {
244
- return rw.writeRefsRecursive(n)
245
- }
246
- return rw.writeRefsSingle(n)
254
+ return rw.writeRefsRecursive(n, 0)
255
+
256
}
257
249
-func (rw *RefWriter) writeRefsRecursive(n ipld.Node) (int, error) {
258
+func (rw *RefWriter) writeRefsRecursive(n ipld.Node, depth int) (int, error) {
259
nc := n.Cid()
260
261
var count int
262
for i, ng := range ipld.GetDAG(rw.Ctx, rw.DAG, n) {
263
lc := n.Links()[i].Cid
255
- if rw.skip(lc) {
264
+ goDeeper, shouldWrite := rw.visit(lc, depth+1) // The children are at depth+1
265
+
266
+ // Avoid "Get()" on the node and continue with next Link.
267
+ // We can do this if:
268
+ // - We printed it before (thus it was already seen and
269
+ // fetched with Get()
270
+ // - AND we must not go deeper.
271
+ // This is an optimization for pruned branches which have been
272
+ // visited before.
273
+ if !shouldWrite && !goDeeper {
274
continue
275
}
276
259
- if err := rw.WriteEdge(nc, lc, n.Links()[i].Name); err != nil {
260
- return count, err
261
- }
262
-
277
+ // We must Get() the node because:
278
+ // - it is new (never written)
279
+ // - OR we need to go deeper.
280
+ // This ensures printed refs are always fetched.
281
nd, err := ng.Get(rw.Ctx)
282
if err != nil {
283
return count, err
284
}
285
268
- c, err := rw.writeRefsRecursive(nd)
269
- count += c
270
- if err != nil {
271
- return count, err
272
- }
273
- }
274
- return count, nil
275
-}
276
-
277
-func (rw *RefWriter) writeRefsSingle(n ipld.Node) (int, error) {
278
- c := n.Cid()
279
-
280
- if rw.skip(c) {
281
- return 0, nil
282
- }
283
-
284
- count := 0
285
- for _, l := range n.Links() {
286
- lc := l.Cid
287
- if rw.skip(lc) {
288
- continue
286
+ // Write this node if not done before (or !Unique)
287
+ if shouldWrite {
288
+ if err := rw.WriteEdge(nc, lc, n.Links()[i].Name); err != nil {
289
+ return count, err
290
+ }
291
+ count++
292
}
293
291
- if err := rw.WriteEdge(c, lc, l.Name); err != nil {
292
- return count, err
294
+ // Keep going deeper. This happens:
295
+ // - On unexplored branches
296
+ // - On branches not explored deep enough
297
+ // Note when !Unique, branches are always considered
298
+ // unexplored and only depth limits apply.
299
+ if goDeeper {
300
+ c, err := rw.writeRefsRecursive(nd, depth+1)
301
+ count += c
302
+ if err != nil {
303
+ return count, err
304
+ }
305
}
294
- count++
306
}
307
+
308
return count, nil
309
}
310
299
-// skip returns whether to skip a cid
300
-func (rw *RefWriter) skip(c *cid.Cid) bool {
311
+// visit returns two values:
312
+// - the first boolean is true if we should keep traversing the DAG
313
+// - the second boolean is true if we should print the CID
314
+//
315
+// visit will do branch pruning depending on rw.MaxDepth, previously visited
316
+// cids and whether rw.Unique is set. i.e. rw.Unique = false and
317
+// rw.MaxDepth = -1 disables any pruning. But setting rw.Unique to true will
318
+// prune already visited branches at the cost of keeping as set of visited
319
+// CIDs in memory.
320
+func (rw *RefWriter) visit(c *cid.Cid, depth int) (bool, bool) {
321
+ atMaxDepth := rw.MaxDepth >= 0 && depth == rw.MaxDepth
322
+ overMaxDepth := rw.MaxDepth >= 0 && depth > rw.MaxDepth
323
+
324
+ // Shortcut when we are over max depth. In practice, this
325
+ // only applies when calling refs with --maxDepth=0, as root's
326
+ // children are already over max depth. Otherwise nothing should
327
+ // hit this.
328
+ if overMaxDepth {
329
+ return false, false
330
+ }
331
+
332
+ // We can shortcut right away if we don't need unique output:
333
+ // - we keep traversing when not atMaxDepth
334
+ // - always print
335
if !rw.Unique {
302
- return false
336
+ return !atMaxDepth, true
337
}
338
339
+ // Unique == true from this point.
340
+ // Thus, we keep track of seen Cids, and their depth.
341
if rw.seen == nil {
306
- rw.seen = cid.NewSet()
342
+ rw.seen = make(map[string]int)
343
}
308
-
309
- has := rw.seen.Has(c)
310
- if !has {
311
- rw.seen.Add(c)
344
+ key := string(c.Bytes())
345
+ oldDepth, ok := rw.seen[key]
346
+
347
+ // Unique == true && depth < MaxDepth (or unlimited) from this point
348
+
349
+ // Branch pruning cases:
350
+ // - We saw the Cid before and either:
351
+ // - Depth is unlimited (MaxDepth = -1)
352
+ // - We saw it higher (smaller depth) in the DAG (means we must have
353
+ // explored deep enough before)
354
+ // Because we saw the CID, we don't print it again.
355
+ if ok && (rw.MaxDepth < 0 || oldDepth <= depth) {
356
+ return false, false
357
}
313
- return has
358
+
359
+ // Final case, we must keep exploring the DAG from this CID
360
+ // (unless we hit the depth limit).
361
+ // We note down its depth because it was either not seen
362
+ // or is lower than last time.
363
+ // We print if it was not seen.
364
+ rw.seen[key] = depth
365
+ return !atMaxDepth, !ok
366
}
367
368
// Write one edge