@cryptotaxi247 / kubo / commits / 308274f88

feat!: dag import - don't pin roots by default (#9926)

* feat!: dag import - don't pin roots by default Fixes: https://github.com/ipfs/kubo/issues/9765 * test(ipip-402): dag import this adds basic regression test that guards behavior around partial cars with or without pinning * docs(ipip-402): ipip and dag import changelog --------- Co-authored-by: Marcin Rataj <lidel@lidel.org>

Rod Vagg committed Jun 15, 2023 at 06:45 UTC 308274f886858acbe0055ec28f04657a99539969
16 files changed +92 -34
core/commands/dag/dag.go
+8 -5
@@ -176,13 +176,16 @@ var DagImportCmd = &cmds.Command{
176 Tagline: "Import the contents of .car files",
177 ShortDescription: `
178 'ipfs dag import' imports all blocks present in supplied .car
179 -( Content Address aRchive ) files, recursively pinning any roots
180 -specified in the CAR file headers, unless --pin-roots is set to false.
179 +( Content Address aRchive ) files, optionally recursively pinning any
180 +roots specified in the CAR file headers if --pin-roots is set.
181
182 Note:
183 This command will import all blocks in the CAR file, not just those
184 - reachable from the specified roots. However, these other blocks will
185 - not be pinned and may be garbage collected later.
184 + reachable from the specified roots. However, when using --pin-roots,
185 + these other blocks will not be pinned and may be garbage collected
186 + later. When not using --pin-roots, all blocks imported may be garbage
187 + collected if no other pin operation is performed on them, or a root
188 + that references them.
189
190 The pinning of the roots happens after all car files are processed,
191 permitting import of DAGs spanning multiple files.
@@ -200,7 +203,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
203 cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
204 },
205 Options: []cmds.Option{
203 - cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
206 + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing."),
207 cmds.BoolOption(silentOptionName, "No output."),
208 cmds.BoolOption(statsOptionName, "Output stats."),
209 cmdutils.AllowBigBlockOption,
docs/changelogs/v0.21.md
+32 -3
@@ -11,8 +11,10 @@
11 - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client)
12 - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages)
13 - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml)
14 + - [Gateway: support for partial CAR export parameters (IPIP-402)](#gateway-support-for-partial-car-export-parameters-ipip-402)
15 + - [`ipfs dag import` no longer pins by default](#ipfs-dag-import-no-longer-pins-by-default)
16 - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics)
15 - - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default)
17 + - [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental)
18 - [📝 Changelog](#-changelog)
19 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
20
@@ -104,9 +106,36 @@ $ curl "https://subdomain-gw.example.net/ipfs/${cid}/"
106
107 Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913).
108
107 -#### Gateway: support for CAR parameters of IPIP-402
109 +#### Gateway: support for partial CAR export parameters (IPIP-402)
110
109 -The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402).
111 +The gateway now supports optional CAR export parameters
112 +`dag-scope=block|entity|all` and `entity-bytes=from:to` as specified in
113 +[IPIP-402](https://github.com/ipfs/specs/pull/402).
114 +
115 +Batch block retrieval minimizes round trips, catering to the requirements of
116 +light HTTP clients for directory enumeration, range requests, and content path
117 +resolution.
118 +
119 +#### `ipfs dag import` no longer pins by default
120 +
121 +With the gateway now capable of handling partial CAR exports
122 +([IPIP-402](https://github.com/ipfs/specs/pull/402)) and incomplete DAG CARs
123 +becoming more prevalent, there have been changes to the pinning mode when using
124 +`ipfs dag import`.
125 +
126 +Recursive pinning of the entire DAG within an imported CAR is now optional. To
127 +explicitly attempt pinning the DAG referenced by any roots present in the CAR,
128 +you can opt in by using the `--pin-roots` option.
129 +
130 +Pinning incomplete DAG will produce an error:
131 +
132 +```console
133 +$ curl 'http://127.0.0.1:8080/ipns/docs.ipfs.tech?format=car&dag-scope=entity' > ./partial-entity.car # Kubo 0.21.0 with IPIP-402 (only root block of unixfs dir)
134 +$ ipfs dag import --stats --pin-roots=true ./partial-entity.car
135 +Error pinning QmPDC11yLAbVw3dX5jMeEuSdk4BiVjSd9X87zaYRdVjzW3 FAILED: block was not found locally (offline): ipld: could not find QmPDvrDAz2aHeLjPVQ4uh1neyknUmDpf1GsBzAbpFhS8ro
136 +Imported 1 blocks (1618 bytes)
137 +[exit code 1]
138 +```
139
140 #### `ipfs dag stat` deduping statistics
141
test/sharness/t0054-dag-car-import-export-data/README.md
+4
@@ -26,3 +26,7 @@
26 - lotus_devnet_genesis_v2.car
27 - generated with `car index lotus_testnet_export_128.car > lotus_testnet_export_128_v2.car`
28 - install `go-car` CLI from https://github.com/ipld/go-car
29 +
30 +- partial-dag-scope-entity.car
31 + - unixfs directory entity exported from gateway via `?format=car&dag-scope=entity` ([IPIP-402](https://github.com/ipfs/specs/pull/402))
32 + - CAR roots includes directory CID, but only the root block is included in the CAR, making the DAG incomplete
test/sharness/t0054-dag-car-import-export-data/partial-dag-scope-entity.car
Binary files /dev/null and b/test/sharness/t0054-dag-car-import-export-data/partial-dag-scope-entity.car differ
test/sharness/t0054-dag-car-import-export.sh
+30 -8
@@ -41,7 +41,7 @@ do_import() {
41 while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done &
42 while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done &
43
44 - ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null
44 + ipfsi "$node" dag import --pin-roots "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null
45 result=$?
46
47 rm -f spin.gc &>/dev/null
@@ -117,7 +117,7 @@ EOE
117 '
118
119 test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
120 - ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
120 + ipfsi 1 dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
121 > naked_import_result_json_actual
122 '
123
@@ -197,14 +197,14 @@ EOE
197 head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected
198
199 test_expect_success "multiroot import works (--enc=json)" '
200 - ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
200 + ipfs dag import --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
201 '
202 test_expect_success "multiroot import expected output" '
203 test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
204 '
205
206 test_expect_success "multiroot import works with --stats" '
207 - ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
207 + ipfs dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
208 '
209 test_expect_success "multiroot import expected output" '
210 test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
@@ -215,18 +215,18 @@ cat >pin_import_expected << EOE
215 {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
216 EOE
217 test_expect_success "pin-less import works" '
218 - ipfs dag import --stats --enc=json --pin-roots=false \
218 + ipfs dag import --stats --enc=json \
219 ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
220 ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
221 > no-pin_import_actual
222 '
223 -test_expect_success "expected no pins on --pin-roots=false" '
223 +test_expect_success "expected no pins on" '
224 test_cmp pin_import_expected no-pin_import_actual
225 '
226
227
228 test_expect_success "naked root import works" '
229 - ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
229 + ipfs dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
230 > naked_root_import_json_actual
231 '
232 test_expect_success "naked root import expected output" '
@@ -253,7 +253,7 @@ cat > version_2_import_expected << EOE
253 EOE
254
255 test_expect_success "version 2 import" '
256 - ipfs dag import --stats --enc=json \
256 + ipfs dag import --stats --enc=json --pin-roots \
257 ../t0054-dag-car-import-export-data/lotus_testnet_export_128_v2.car \
258 ../t0054-dag-car-import-export-data/lotus_devnet_genesis_v2.car \
259 > version_2_import_actual
@@ -291,4 +291,26 @@ test_expect_success "'ipfs dag import' decode IPLD 'cbor' codec works" '
291 rm cbor.car
292 '
293
294 +# IPIP-402
295 +cat > partial_nopin_import_expected << EOE
296 +{"Stats":{"BlockCount":1,"BlockBytesCount":1618}}
297 +EOE
298 +test_expect_success "'ipfs dag import' without pinning works fine with incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" '
299 + ipfs dag import --stats --enc=json --pin-roots=false ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_nopin_import_out 2>&1 &&
300 + test_cmp partial_nopin_import_expected partial_nopin_import_out
301 +'
302 +test_expect_success "'ipfs dag import' with no params in CLI mode produces exit code 0 (unixfs dir exported as dag-scope=entity from IPIP-402)" '
303 + test_expect_code 0 ipfs dag import ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car
304 +'
305 +
306 +test_expect_success "'ipfs dag import' with pinning errors due to incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" '
307 + ipfs dag import --stats --enc=json --pin-roots=true ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 &&
308 + test_should_contain "\"PinErrorMsg\":\"block was not found locally" partial_pin_import_out
309 +'
310 +
311 +test_expect_success "'ipfs dag import' pin error in default CLI mode produces exit code 1 (unixfs dir exported as dag-scope=entity from IPIP-402)" '
312 + test_expect_code 1 ipfs dag import --pin-roots ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 &&
313 + test_should_contain "Error: pinning root \"QmPDC11yLAbVw3dX5jMeEuSdk4BiVjSd9X87zaYRdVjzW3\" FAILED: block was not found locally" partial_pin_import_out
314 +'
315 +
316 test_done
test/sharness/t0109-gateway-web-_redirects.sh
+1 -1
@@ -14,7 +14,7 @@ test_launch_ipfs_daemon
14 # Import test case
15 # Run `ipfs cat /ipfs/$REDIRECTS_DIR_CID/_redirects` to see sample _redirects file
16 test_expect_success "Add the _redirects file test directory" '
17 - ipfs dag import ../t0109-gateway-web-_redirects-data/redirects.car
17 + ipfs dag import --pin-roots ../t0109-gateway-web-_redirects-data/redirects.car
18 '
19 CAR_ROOT_CID=QmQyqMY5vUBSbSxyitJqthgwZunCQjDVtNd8ggVCxzuPQ4
20
test/sharness/t0113-gateway-symlink.sh
+1 -1
@@ -12,7 +12,7 @@ test_launch_ipfs_daemon
12 # Import test case
13 # See the static fixtures in ./t0113-gateway-symlink/
14 test_expect_success "Add the test directory with symlinks" '
15 - ipfs dag import ../t0113-gateway-symlink/testfiles.car
15 + ipfs dag import --pin-roots ../t0113-gateway-symlink/testfiles.car
16 '
17 ROOT_DIR_CID=QmWvY6FaqFMS89YAQ9NAPjVP4WZKA1qbHbicc9HeSKQTgt # ./testfiles/
18
test/sharness/t0114-gateway-subdomains.sh
+1 -1
@@ -113,7 +113,7 @@ IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
113 IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam
114
115 test_expect_success "Add the test fixtures" '
116 - ipfs dag import ../t0114-gateway-subdomains/fixtures.car &&
116 + ipfs dag import --pin-roots ../t0114-gateway-subdomains/fixtures.car &&
117 ipfs routing put --allow-offline /ipns/${RSA_KEY} ../t0114-gateway-subdomains/${RSA_KEY}.ipns-record &&
118 ipfs routing put --allow-offline /ipns/${ED25519_KEY} ../t0114-gateway-subdomains/${ED25519_KEY}.ipns-record
119 '
test/sharness/t0115-gateway-dir-listing.sh
+1 -1
@@ -21,7 +21,7 @@ test_launch_ipfs_daemon_without_network
21 # Import test case
22 # See the static fixtures in ./t0115-gateway-dir-listing/
23 test_expect_success "Add the test directory" '
24 - ipfs dag import ../t0115-gateway-dir-listing/fixtures.car
24 + ipfs dag import --pin-roots ../t0115-gateway-dir-listing/fixtures.car
25 '
26 DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/
27 FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
test/sharness/t0116-gateway-cache.sh
+1 -1
@@ -35,7 +35,7 @@ TEST_IPNS_ID=k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe
35 # Import test case
36 # See the static fixtures in ./t0116-gateway-cache/
37 test_expect_success "Add the test directory" '
38 - ipfs dag import ../t0116-gateway-cache/fixtures.car
38 + ipfs dag import --pin-roots ../t0116-gateway-cache/fixtures.car
39 ipfs routing put --allow-offline /ipns/${TEST_IPNS_ID} ../t0116-gateway-cache/${TEST_IPNS_ID}.ipns-record
40 '
41
test/sharness/t0117-gateway-block.sh
+1 -1
@@ -10,7 +10,7 @@ test_launch_ipfs_daemon_without_network
10 # Import test case
11 # See the static fixtures in ./t0117-gateway-block/
12 test_expect_success "Add the dir test directory" '
13 - ipfs dag import ../t0117-gateway-block/fixtures.car
13 + ipfs dag import --pin-roots ../t0117-gateway-block/fixtures.car
14 '
15 ROOT_DIR_CID=bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly # ./
16 FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/ascii.txt
test/sharness/t0122-gateway-tar.sh
+3 -3
@@ -13,7 +13,7 @@ INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
13 # Import test case
14 # See the static fixtures in ./t0122-gateway-tar/
15 test_expect_success "Add the test directory" '
16 - ipfs dag import ../t0122-gateway-tar/fixtures.car
16 + ipfs dag import --pin-roots ../t0122-gateway-tar/fixtures.car
17 '
18 DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir
19 FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
@@ -63,9 +63,9 @@ test_expect_success "GET TAR with explicit ?filename= succeeds with modified Con
63 "
64
65 test_expect_success "Add CARs with relative paths to test with" '
66 - ipfs dag import ../t0122-gateway-tar/outside-root.car > import_output &&
66 + ipfs dag import --pin-roots ../t0122-gateway-tar/outside-root.car > import_output &&
67 test_should_contain $OUTSIDE_ROOT_CID import_output &&
68 - ipfs dag import ../t0122-gateway-tar/inside-root.car > import_output &&
68 + ipfs dag import --pin-roots ../t0122-gateway-tar/inside-root.car > import_output &&
69 test_should_contain $INSIDE_ROOT_CID import_output
70 '
71
test/sharness/t0123-gateway-json-cbor.sh
+4 -4
@@ -10,7 +10,7 @@ test_launch_ipfs_daemon_without_network
10 # Import test case
11 # See the static fixtures in ./t0123-gateway-json-cbor/
12 test_expect_success "Add the test directory" '
13 - ipfs dag import ../t0123-gateway-json-cbor/fixtures.car
13 + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/fixtures.car
14 '
15 DIR_CID=bafybeiafyvqlazbbbtjnn6how5d6h6l6rxbqc4qgpbmteaiskjrffmyy4a # ./rootDir
16 FILE_JSON_CID=bafkreibrppizs3g7axs2jdlnjua6vgpmltv7k72l7v7sa6mmht6mne3qqe # ./rootDir/ą/ę/t.json
@@ -155,11 +155,11 @@ DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxk
155 DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke"
156
157 test_expect_success "Add CARs for path traversal and DAG-PB representation tests" '
158 - ipfs dag import ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output &&
158 + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output &&
159 test_should_contain $DAG_CBOR_TRAVERSAL_CID import_output &&
160 - ipfs dag import ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output &&
160 + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output &&
161 test_should_contain $DAG_JSON_TRAVERSAL_CID import_output &&
162 - ipfs dag import ../t0123-gateway-json-cbor/dag-pb.car > import_output &&
162 + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-pb.car > import_output &&
163 test_should_contain $DAG_PB_CID import_output
164 '
165
test/sharness/t0124-gateway-ipns-record.sh
+1 -1
@@ -12,7 +12,7 @@ test_launch_ipfs_daemon
12 IPNS_KEY=k51qzi5uqu5dh71qgwangrt6r0nd4094i88nsady6qgd1dhjcyfsaqmpp143ab
13 FILE_CID=bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244 # A file containing Hello IPFS
14 test_expect_success "Add the test directory & IPNS records" '
15 - ipfs dag import ../t0124-gateway-ipns-record/fixtures.car &&
15 + ipfs dag import --pin-roots ../t0124-gateway-ipns-record/fixtures.car &&
16 ipfs routing put /ipns/${IPNS_KEY} ../t0124-gateway-ipns-record/${IPNS_KEY}.ipns-record
17 '
18
test/sharness/t0400-api-no-gateway.sh
+1 -1
@@ -13,7 +13,7 @@ test_init_ipfs
13 # Import test case
14 # See the static fixtures in ./t0400-api-no-gateway/
15 test_expect_success "Add the test directory" '
16 - ipfs dag import ../t0400-api-no-gateway/fixtures.car
16 + ipfs dag import --pin-roots ../t0400-api-no-gateway/fixtures.car
17 '
18 HASH=QmNYERzV2LfD2kkfahtfv44ocHzEFK1sLBaE7zdcYT2GAZ # a file containing the string "testing"
19
testplans/bitswap/main.go
+3 -3
@@ -12,14 +12,14 @@ import (
12 "github.com/testground/sdk-go/runtime"
13 "github.com/testground/sdk-go/sync"
14
15 - bitswap "github.com/ipfs/go-libipfs/bitswap"
16 - bsnet "github.com/ipfs/go-libipfs/bitswap/network"
17 - block "github.com/ipfs/go-libipfs/blocks"
15 "github.com/ipfs/go-cid"
16 datastore "github.com/ipfs/go-datastore"
17 blockstore "github.com/ipfs/go-ipfs-blockstore"
18 exchange "github.com/ipfs/go-ipfs-exchange-interface"
19 bstats "github.com/ipfs/go-ipfs-regression/bitswap"
20 + bitswap "github.com/ipfs/go-libipfs/bitswap"
21 + bsnet "github.com/ipfs/go-libipfs/bitswap/network"
22 + block "github.com/ipfs/go-libipfs/blocks"
23 "github.com/libp2p/go-libp2p"
24 dht "github.com/libp2p/go-libp2p-kad-dht"
25 "github.com/libp2p/go-libp2p/core/host"