add ipfs dag stat command (#7553)
* commands: add ipfs dag stat command * sharness: add ipfs dag stat tests
Adin Schmahmann committed
Aug 17, 2020 at 14:29 UTC
8ae5aa56d1a196b25d22a4237ccf035deb9130f7
4 files changed
+154
core/commands/commands_test.go
+2
@@ -25,6 +25,7 @@ func TestROCommands(t *testing.T) {
25
"/dag",
26
"/dag/get",
27
"/dag/resolve",
28
+ "/dag/stat",
29
"/dns",
30
"/get",
31
"/ls",
@@ -99,6 +100,7 @@ func TestCommands(t *testing.T) {
100
"/dag/put",
101
"/dag/import",
102
"/dag/resolve",
103
+ "/dag/stat",
104
"/dht",
105
"/dht/findpeer",
106
"/dht/findprovs",
core/commands/dag/dag.go
+116
@@ -10,6 +10,7 @@ import (
10
"time"
11
12
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
13
+ "github.com/ipfs/go-ipfs/core/commands/e"
14
"github.com/ipfs/go-ipfs/core/coredag"
15
iface "github.com/ipfs/interface-go-ipfs-core"
16
@@ -19,6 +20,7 @@ import (
20
files "github.com/ipfs/go-ipfs-files"
21
ipld "github.com/ipfs/go-ipld-format"
22
mdag "github.com/ipfs/go-merkledag"
23
+ traverse "github.com/ipfs/go-merkledag/traverse"
24
ipfspath "github.com/ipfs/go-path"
25
"github.com/ipfs/interface-go-ipfs-core/options"
26
path "github.com/ipfs/interface-go-ipfs-core/path"
@@ -54,6 +56,7 @@ to deprecate and replace the existing 'ipfs object' command moving forward.
56
"resolve": DagResolveCmd,
57
"import": DagImportCmd,
58
"export": DagExportCmd,
59
+ "stat": DagStatCmd,
60
},
61
}
62
@@ -668,3 +671,116 @@ The output of blocks happens in strict DAG-traversal, first-seen, order.
671
},
672
},
673
}
674
+
675
+type DagStat struct {
676
+ Size uint64
677
+ NumBlocks int64
678
+}
679
+
680
+func (s *DagStat) String() string {
681
+ return fmt.Sprintf("Size: %d, NumBlocks: %d", s.Size, s.NumBlocks)
682
+}
683
+
684
+var DagStatCmd = &cmds.Command{
685
+ Helptext: cmds.HelpText{
686
+ Tagline: "Gets stats for a DAG",
687
+ ShortDescription: `
688
+'ipfs dag size' fetches a dag and returns various statistics about the DAG.
689
+Statistics include size and number of blocks.
690
+
691
+Note: This command skips duplicate blocks in reporting both size and the number of blocks
692
+`,
693
+ },
694
+ Arguments: []cmds.Argument{
695
+ cmds.StringArg("root", true, false, "CID of a DAG root to get statistics for").EnableStdin(),
696
+ },
697
+ Options: []cmds.Option{
698
+ cmds.BoolOption(progressOptionName, "p", "Return progressive data while reading through the DAG").WithDefault(true),
699
+ },
700
+ Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
701
+ progressive := req.Options[progressOptionName].(bool)
702
+
703
+ api, err := cmdenv.GetApi(env, req)
704
+ if err != nil {
705
+ return err
706
+ }
707
+
708
+ rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0]))
709
+ if err != nil {
710
+ return err
711
+ }
712
+
713
+ if len(rp.Remainder()) > 0 {
714
+ return fmt.Errorf("cannot return size for anything other than a DAG with a root CID")
715
+ }
716
+
717
+ nodeGetter := mdag.NewSession(req.Context, api.Dag())
718
+ obj, err := nodeGetter.Get(req.Context, rp.Cid())
719
+ if err != nil {
720
+ return err
721
+ }
722
+
723
+ dagstats := &DagStat{}
724
+ err = traverse.Traverse(obj, traverse.Options{
725
+ DAG: nodeGetter,
726
+ Order: traverse.DFSPre,
727
+ Func: func(current traverse.State) error {
728
+ dagstats.Size += uint64(len(current.Node.RawData()))
729
+ dagstats.NumBlocks++
730
+
731
+ if progressive {
732
+ if err := res.Emit(dagstats); err != nil {
733
+ return err
734
+ }
735
+ }
736
+ return nil
737
+ },
738
+ ErrFunc: nil,
739
+ SkipDuplicates: true,
740
+ })
741
+ if err != nil {
742
+ return fmt.Errorf("error traversing DAG: %w", err)
743
+ }
744
+
745
+ if !progressive {
746
+ if err := res.Emit(dagstats); err != nil {
747
+ return err
748
+ }
749
+ }
750
+
751
+ return nil
752
+ },
753
+ Type: DagStat{},
754
+ PostRun: cmds.PostRunMap{
755
+ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
756
+ var dagStats *DagStat
757
+ for {
758
+ v, err := res.Next()
759
+ if err != nil {
760
+ if err == io.EOF {
761
+ break
762
+ }
763
+ return err
764
+ }
765
+
766
+ out, ok := v.(*DagStat)
767
+ if !ok {
768
+ return e.TypeErr(out, v)
769
+ }
770
+ dagStats = out
771
+ fmt.Fprintf(os.Stderr, "%v\r", out)
772
+ }
773
+ return re.Emit(dagStats)
774
+ },
775
+ },
776
+ Encoders: cmds.EncoderMap{
777
+ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStat) error {
778
+ _, err := fmt.Fprintf(
779
+ w,
780
+ "%v\n",
781
+ event,
782
+ )
783
+ return err
784
+ }),
785
+ },
786
+}
core/commands/root.go
+1
@@ -191,6 +191,7 @@ var rootROSubcommands = map[string]*cmds.Command{
191
Subcommands: map[string]*cmds.Command{
192
"get": dag.DagGetCmd,
193
"resolve": dag.DagResolveCmd,
194
+ "stat": dag.DagStatCmd,
195
},
196
},
197
"resolve": ResolveCmd,
test/sharness/t0053-dag.sh
+35
@@ -268,6 +268,41 @@ test_dag_cmd() {
268
test_cmp resolve_obj_exp resolve_obj &&
269
test_cmp resolve_data_exp resolve_data
270
'
271
+
272
+ test_expect_success "dag stat of simple IPLD object" '
273
+ ipfs dag stat $NESTED_HASH > actual_stat_inner_ipld_obj &&
274
+ echo "Size: 15, NumBlocks: 1" > exp_stat_inner_ipld_obj &&
275
+ test_cmp exp_stat_inner_ipld_obj actual_stat_inner_ipld_obj &&
276
+ ipfs dag stat $HASH > actual_stat_ipld_obj &&
277
+ echo "Size: 61, NumBlocks: 2" > exp_stat_ipld_obj &&
278
+ test_cmp exp_stat_ipld_obj actual_stat_ipld_obj
279
+ '
280
+
281
+ test_expect_success "dag stat of simple UnixFS object" '
282
+ BASIC_UNIXFS=$(echo "1234" | ipfs add --pin=false -q) &&
283
+ ipfs dag stat $BASIC_UNIXFS > actual_stat_basic_unixfs &&
284
+ echo "Size: 13, NumBlocks: 1" > exp_stat_basic_unixfs &&
285
+ test_cmp exp_stat_basic_unixfs actual_stat_basic_unixfs
286
+ '
287
+
288
+ # The multiblock file is just 10000000 copies of the number 1
289
+ # As most of its data is replicated it should have a small number of blocks
290
+ test_expect_success "dag stat of multiblock UnixFS object" '
291
+ MULTIBLOCK_UNIXFS=$(printf "1%.0s" {1..10000000} | ipfs add --pin=false -q) &&
292
+ ipfs dag stat $MULTIBLOCK_UNIXFS > actual_stat_multiblock_unixfs &&
293
+ echo "Size: 302582, NumBlocks: 3" > exp_stat_multiblock_unixfs &&
294
+ test_cmp exp_stat_multiblock_unixfs actual_stat_multiblock_unixfs
295
+ '
296
+
297
+ test_expect_success "dag stat of directory of UnixFS objects" '
298
+ mkdir -p unixfsdir &&
299
+ echo "1234" > unixfsdir/small.txt
300
+ printf "1%.0s" {1..10000000} > unixfsdir/many1s.txt &&
301
+ DIRECTORY_UNIXFS=$(ipfs add -r --pin=false -Q unixfsdir) &&
302
+ ipfs dag stat $DIRECTORY_UNIXFS > actual_stat_directory_unixfs &&
303
+ echo "Size: 302705, NumBlocks: 5" > exp_stat_directory_unixfs &&
304
+ test_cmp exp_stat_directory_unixfs actual_stat_directory_unixfs
305
+ '
306
}
307
308
# should work offline