feat: dag import --stats (#8237)
* feat: report block count on `dag import` * fix: clean-up dag import message format * Only print stats when --stats flag is passed This applies to both text and json output encoding. - Stats data is now contained within a Stats datastructure - Stats are printed after root so that first line of output is the same as previously, even when stats are output using --stats * fix sharness test * Add PayloadBytesCount to stats * Attempt to stabilize flaky tests * Rename PayloadBytesCount to BlockBytesCount * Correctly calculate size or imported dag * Use RawSize of original block for import bytes calc * test: dag import without --stats basic regression tests for the default output (text and json) Co-authored-by: gammazero <gammazero@users.noreply.github.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>
Rod Vagg committed
Sep 24, 2021 at 00:23 UTC
0057199af45bd3ab5f6e762ebbc7be6ce2c0a140
5 files changed
+98
-21
core/commands/dag/dag.go
+31
-5
@@ -16,9 +16,10 @@ import (
16
)
17
18
const (
19
+ pinRootsOptionName = "pin-roots"
20
progressOptionName = "progress"
21
silentOptionName = "silent"
21
- pinRootsOptionName = "pin-roots"
22
+ statsOptionName = "stats"
23
)
24
25
// DagCmd provides a subset of commands for interacting with ipld dag objects
@@ -53,9 +54,15 @@ type ResolveOutput struct {
54
RemPath string
55
}
56
57
+type CarImportStats struct {
58
+ BlockCount uint64
59
+ BlockBytesCount uint64
60
+}
61
+
62
// CarImportOutput is the output type of the 'dag import' commands
63
type CarImportOutput struct {
58
- Root RootMeta
64
+ Root *RootMeta `json:",omitempty"`
65
+ Stats *CarImportStats `json:",omitempty"`
66
}
67
68
// RootMeta is the metadata for a root pinning response
@@ -160,8 +167,10 @@ var DagResolveCmd = &cmds.Command{
167
}
168
169
type importResult struct {
163
- roots map[cid.Cid]struct{}
164
- err error
170
+ blockCount uint64
171
+ blockBytesCount uint64
172
+ roots map[cid.Cid]struct{}
173
+ err error
174
}
175
176
// DagImportCmd is a command for importing a car to ipfs
@@ -193,8 +202,9 @@ Maximum supported CAR version: 1
202
cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
203
},
204
Options: []cmds.Option{
196
- cmds.BoolOption(silentOptionName, "No output."),
205
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
206
+ cmds.BoolOption(silentOptionName, "No output."),
207
+ cmds.BoolOption(statsOptionName, "Output stats."),
208
},
209
Type: CarImportOutput{},
210
Run: dagImport,
@@ -206,6 +216,22 @@ Maximum supported CAR version: 1
216
return nil
217
}
218
219
+ // event should have only one of `Root` or `Stats` set, not both
220
+ if event.Root == nil {
221
+ if event.Stats == nil {
222
+ return fmt.Errorf("Unexpected message from DAG import")
223
+ }
224
+ stats, _ := req.Options[statsOptionName].(bool)
225
+ if stats {
226
+ fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount)
227
+ }
228
+ return nil
229
+ }
230
+
231
+ if event.Stats != nil {
232
+ return fmt.Errorf("Unexpected message from DAG import")
233
+ }
234
+
235
enc, err := cmdenv.GetLowLevelCidEncoder(req)
236
if err != nil {
237
return err
core/commands/dag/import.go
+21
-2
@@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
101
failedPins++
102
}
103
104
- if err := res.Emit(&CarImportOutput{Root: ret}); err != nil {
104
+ if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil {
105
return err
106
}
107
}
@@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
115
}
116
}
117
118
+ stats, _ := req.Options[statsOptionName].(bool)
119
+ if stats {
120
+ err = res.Emit(&CarImportOutput{
121
+ Stats: &CarImportStats{
122
+ BlockCount: done.blockCount,
123
+ BlockBytesCount: done.blockBytesCount,
124
+ },
125
+ })
126
+ if err != nil {
127
+ return err
128
+ }
129
+ }
130
+
131
return nil
132
}
133
@@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
139
batch := ipld.NewBatch(req.Context, api.Dag())
140
141
roots := make(map[cid.Cid]struct{})
142
+ var blockCount, blockBytesCount uint64
143
144
it := req.Files.Entries()
145
for it.Next() {
@@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
190
if err := batch.Add(req.Context, nd); err != nil {
191
return err
192
}
193
+ blockCount++
194
+ blockBytesCount += uint64(len(block.RawData()))
195
}
196
197
return nil
@@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
213
return
214
}
215
200
- ret <- importResult{roots: roots}
216
+ ret <- importResult{
217
+ blockCount: blockCount,
218
+ blockBytesCount: blockBytesCount,
219
+ roots: roots}
220
}
test/sharness/lib/test-lib.sh
+1
-1
@@ -269,7 +269,7 @@ test_launch_ipfs_daemon() {
269
270
# wait for api file to show up
271
test_expect_success "api file shows up" '
272
- test_wait_for_file 50 100ms "$IPFS_PATH/api"
272
+ test_wait_for_file 50 200ms "$IPFS_PATH/api"
273
'
274
275
test_set_address_vars actual_daemon
test/sharness/t0041-ping.sh
+1
-1
@@ -43,7 +43,7 @@ test_expect_success "test ping 0" '
43
'
44
45
test_expect_success "test ping offline" '
46
- iptb stop 1 &&
46
+ iptb stop 1 && sleep 2 &&
47
! ipfsi 0 ping -n2 -- "$PEERID_1"
48
'
49
test/sharness/t0054-dag-car-import-export.sh
+44
-12
@@ -55,18 +55,25 @@ run_online_imp_exp_tests() {
55
reset_blockstore 0
56
reset_blockstore 1
57
58
- cat > basic_import_expected <<EOE
58
+ cat > basic_import_stats_expected <<EOE
59
+Imported 1198 blocks (468513 bytes)
60
Pinned root${tab}bafkqaaa${tab}success
61
Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
62
Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
63
EOE
64
+ # output without the --stats line at the top
65
+ tail -n +2 basic_import_stats_expected > basic_import_expected
66
67
+ # Explainer:
68
+ # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car
69
+ # executed when roots are already present in the repo - thus the BlockCount=0
70
+ # (if blocks were not present in the repo, blockstore: block not found would be returned)
71
cat >naked_root_import_json_expected <<EOE
72
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
73
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
74
+{"Stats":{"BlockCount":0,"BlockBytesCount":0}}
75
EOE
76
69
-
77
test_expect_success "basic import" '
78
do_import 0 \
79
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -79,6 +86,18 @@ EOE
86
test_cmp_sorted basic_import_expected basic_import_actual
87
'
88
89
+ test_expect_success "basic import with --stats" '
90
+ do_import 0 --stats \
91
+ ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
92
+ ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \
93
+ ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \
94
+ > basic_import_actual
95
+ '
96
+
97
+ test_expect_success "basic import output with --stats as expected" '
98
+ test_cmp_sorted basic_import_stats_expected basic_import_actual
99
+ '
100
+
101
test_expect_success "basic fetch+export 1" '
102
ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
103
'
@@ -98,7 +117,7 @@ EOE
117
'
118
119
test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
101
- ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
120
+ ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
121
> naked_import_result_json_actual
122
'
123
@@ -117,7 +136,7 @@ EOE
136
cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet &
137
cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet &
138
120
- do_import 0 \
139
+ do_import 0 --stats \
140
pipe_testnet \
141
pipe_devnet \
142
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -134,7 +153,7 @@ EOE
153
'
154
155
test_expect_success "fifo-import output as expected" '
137
- test_cmp_sorted basic_import_expected basic_fifo_import_actual
156
+ test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual
157
'
158
}
159
@@ -168,33 +187,46 @@ test_expect_success "correct error" '
187
test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
188
'
189
171
-
172
-cat >multiroot_import_json_expected <<EOE
190
+cat >multiroot_import_json_stats_expected <<EOE
191
{"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
192
{"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
193
{"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
194
+{"Stats":{"BlockCount":2825,"BlockBytesCount":1339709}}
195
EOE
177
-test_expect_success "multiroot import works" '
196
+# output without --stats line
197
+head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected
198
+
199
+test_expect_success "multiroot import works (--enc=json)" '
200
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
201
'
202
test_expect_success "multiroot import expected output" '
203
test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
204
'
205
206
+test_expect_success "multiroot import works with --stats" '
207
+ ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
208
+'
209
+test_expect_success "multiroot import expected output" '
210
+ test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
211
+'
212
+
213
214
+cat >pin_import_expected << EOE
215
+{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
216
+EOE
217
test_expect_success "pin-less import works" '
186
- ipfs dag import --enc=json --pin-roots=false \
218
+ ipfs dag import --stats --enc=json --pin-roots=false \
219
../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
220
../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
221
> no-pin_import_actual
222
'
191
-test_expect_success "expected silence on --pin-roots=false" '
192
- test_cmp /dev/null no-pin_import_actual
223
+test_expect_success "expected no pins on --pin-roots=false" '
224
+ test_cmp pin_import_expected no-pin_import_actual
225
'
226
227
228
test_expect_success "naked root import works" '
197
- ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
229
+ ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
230
> naked_root_import_json_actual
231
'
232
test_expect_success "naked root import expected output" '