| 1 | #!/usr/bin/env bash |
| 2 | # |
| 3 | |
| 4 | test_description="Test car file import/export functionality" |
| 5 | |
| 6 | . lib/test-lib.sh |
| 7 | export -f ipfsi |
| 8 | |
| 9 | set -o pipefail |
| 10 | |
| 11 | tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car.tar.xz |
| 12 | tab=$'\t' |
| 13 | |
| 14 | test_cmp_sorted() { |
| 15 | # use test_cmp to dump out the unsorted file contents as a diff |
| 16 | [[ "$( sort "$1" | sha256sum )" == "$( sort "$2" | sha256sum )" ]] \ |
| 17 | || test_cmp "$1" "$2" |
| 18 | } |
| 19 | export -f test_cmp_sorted |
| 20 | |
| 21 | reset_blockstore() { |
| 22 | node=$1 |
| 23 | |
| 24 | ipfsi "$node" pin ls --quiet --type=recursive | ipfsi "$node" pin rm &>/dev/null |
| 25 | ipfsi "$node" repo gc &>/dev/null |
| 26 | |
| 27 | test_expect_success "pinlist empty" ' |
| 28 | [[ -z "$( ipfsi $node pin ls )" ]] |
| 29 | ' |
| 30 | test_expect_success "nothing left to gc" ' |
| 31 | [[ -z "$( ipfsi $node repo gc )" ]] |
| 32 | ' |
| 33 | } |
| 34 | |
| 35 | # hammer with concurrent gc to ensure nothing clashes |
| 36 | do_import() { |
| 37 | node="$1"; shift |
| 38 | ( |
| 39 | touch spin.gc |
| 40 | |
| 41 | while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & |
| 42 | while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & |
| 43 | |
| 44 | ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null |
| 45 | result=$? |
| 46 | |
| 47 | rm -f spin.gc &>/dev/null |
| 48 | wait || true # work around possible trigger of a bash bug on overloaded circleci |
| 49 | exit $result |
| 50 | ) |
| 51 | } |
| 52 | |
| 53 | run_online_imp_exp_tests() { |
| 54 | |
| 55 | reset_blockstore 0 |
| 56 | reset_blockstore 1 |
| 57 | |
| 58 | cat > basic_import_stats_expected <<EOE |
| 59 | Imported 1198 blocks (468513 bytes) |
| 60 | Pinned root${tab}bafkqaaa${tab}success |
| 61 | Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success |
| 62 | Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success |
| 63 | EOE |
| 64 | # output without the --stats line at the top |
| 65 | tail -n +2 basic_import_stats_expected > basic_import_expected |
| 66 | |
| 67 | # Explainer: |
| 68 | # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car |
| 69 | # executed when roots are already present in the repo - thus the BlockCount=0 |
| 70 | # (if blocks were not present in the repo, ipld: could not find <CID> would be returned) |
| 71 | cat >naked_root_import_json_expected <<EOE |
| 72 | {"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}} |
| 73 | {"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}} |
| 74 | {"Stats":{"BlockCount":0,"BlockBytesCount":0}} |
| 75 | EOE |
| 76 | |
| 77 | test_expect_success "basic import" ' |
| 78 | do_import 0 \ |
| 79 | ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ |
| 80 | ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \ |
| 81 | ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \ |
| 82 | > basic_import_actual |
| 83 | ' |
| 84 | |
| 85 | test_expect_success "basic import output as expected" ' |
| 86 | test_cmp_sorted basic_import_expected basic_import_actual |
| 87 | ' |
| 88 | |
| 89 | test_expect_success "basic import with --stats" ' |
| 90 | do_import 0 --stats \ |
| 91 | ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ |
| 92 | ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \ |
| 93 | ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \ |
| 94 | > basic_import_actual |
| 95 | ' |
| 96 | |
| 97 | test_expect_success "basic import output with --stats as expected" ' |
| 98 | test_cmp_sorted basic_import_stats_expected basic_import_actual |
| 99 | ' |
| 100 | |
| 101 | test_expect_success "basic fetch+export 1" ' |
| 102 | ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car |
| 103 | ' |
| 104 | test_expect_success "export of shuffled testnet export identical to canonical original" ' |
| 105 | test_cmp reexported_testnet_128.car ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car |
| 106 | ' |
| 107 | |
| 108 | test_expect_success "basic fetch+export 2" ' |
| 109 | ipfsi 1 dag export bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u > reexported_devnet_genesis.car |
| 110 | ' |
| 111 | test_expect_success "export of shuffled devnet export identical to canonical original" ' |
| 112 | test_cmp reexported_devnet_genesis.car ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car |
| 113 | ' |
| 114 | |
| 115 | test_expect_success "pinlist on node1 still empty" ' |
| 116 | [[ -z "$( ipfsi 1 pin ls )" ]] |
| 117 | ' |
| 118 | |
| 119 | test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" ' |
| 120 | ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ |
| 121 | > naked_import_result_json_actual |
| 122 | ' |
| 123 | |
| 124 | test_expect_success "naked import output as expected" ' |
| 125 | test_cmp_sorted naked_root_import_json_expected naked_import_result_json_actual |
| 126 | ' |
| 127 | |
| 128 | reset_blockstore 0 |
| 129 | reset_blockstore 1 |
| 130 | |
| 131 | mkfifo pipe_testnet |
| 132 | mkfifo pipe_devnet |
| 133 | |
| 134 | test_expect_success "fifo import" ' |
| 135 | ( |
| 136 | cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet & |
| 137 | cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet & |
| 138 | |
| 139 | do_import 0 --stats \ |
| 140 | pipe_testnet \ |
| 141 | pipe_devnet \ |
| 142 | ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ |
| 143 | > basic_fifo_import_actual |
| 144 | result=$? |
| 145 | |
| 146 | wait || true # work around possible trigger of a bash bug on overloaded circleci |
| 147 | exit "$result" |
| 148 | ) |
| 149 | ' |
| 150 | |
| 151 | test_expect_success "remove fifos" ' |
| 152 | rm pipe_testnet pipe_devnet |
| 153 | ' |
| 154 | |
| 155 | test_expect_success "fifo-import output as expected" ' |
| 156 | test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual |
| 157 | ' |
| 158 | } |
| 159 | |
| 160 | |
| 161 | test_expect_success "set up testbed" ' |
| 162 | iptb testbed create -type localipfs -count 2 -force -init |
| 163 | ' |
| 164 | startup_cluster 2 |
| 165 | |
| 166 | run_online_imp_exp_tests |
| 167 | |
| 168 | test_expect_success "shut down nodes" ' |
| 169 | iptb stop && iptb_wait_stop |
| 170 | ' |
| 171 | |
| 172 | |
| 173 | # We want to just init the repo, without using a daemon for stuff below |
| 174 | test_init_ipfs --empty-repo=false |
| 175 | |
| 176 | |
| 177 | test_expect_success "basic offline export of 'getting started' dag works" ' |
| 178 | ipfs dag export "$HASH_WELCOME_DOCS" >/dev/null |
| 179 | ' |
| 180 | |
| 181 | test_expect_success "basic offline export of nonexistent cid" ' |
| 182 | ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null |
| 183 | ' |
| 184 | test_expect_success "correct error" ' |
| 185 | test_should_contain "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" offline_fetch_error_actual |
| 186 | ' |
| 187 | |
| 188 | cat >multiroot_import_json_stats_expected <<EOE |
| 189 | {"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}} |
| 190 | {"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}} |
| 191 | {"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}} |
| 192 | {"Stats":{"BlockCount":2825,"BlockBytesCount":1339709}} |
| 193 | EOE |
| 194 | # output without --stats line |
| 195 | head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected |
| 196 | |
| 197 | test_expect_success "multiroot import works (--enc=json)" ' |
| 198 | ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual |
| 199 | ' |
| 200 | test_expect_success "multiroot import expected output" ' |
| 201 | test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual |
| 202 | ' |
| 203 | |
| 204 | test_expect_success "multiroot import works with --stats" ' |
| 205 | ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual |
| 206 | ' |
| 207 | test_expect_success "multiroot import expected output" ' |
| 208 | test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual |
| 209 | ' |
| 210 | |
| 211 | |
| 212 | cat >pin_import_expected << EOE |
| 213 | {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} |
| 214 | EOE |
| 215 | test_expect_success "pin-less import works" ' |
| 216 | ipfs dag import --stats --enc=json --pin-roots=false \ |
| 217 | ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ |
| 218 | ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ |
| 219 | > no-pin_import_actual |
| 220 | ' |
| 221 | test_expect_success "expected no pins on --pin-roots=false" ' |
| 222 | test_cmp pin_import_expected no-pin_import_actual |
| 223 | ' |
| 224 | |
| 225 | |
| 226 | test_expect_success "naked root import works" ' |
| 227 | ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ |
| 228 | > naked_root_import_json_actual |
| 229 | ' |
| 230 | test_expect_success "naked root import expected output" ' |
| 231 | test_cmp_sorted naked_root_import_json_expected naked_root_import_json_actual |
| 232 | ' |
| 233 | |
| 234 | test_expect_success "'ipfs dag import' check block size" ' |
| 235 | BIG_CID=$(dd if=/dev/zero bs=2097153 count=1 | ipfs dag put --input-codec=raw --store-codec=raw --allow-big-block) && |
| 236 | ipfs dag export $BIG_CID > over-2MiB-block.car && |
| 237 | test_expect_code 1 ipfs dag import over-2MiB-block.car >dag_import_out 2>&1 |
| 238 | ' |
| 239 | test_expect_success "ipfs dag import output has the correct error" ' |
| 240 | grep "block is over 2MiB" dag_import_out |
| 241 | ' |
| 242 | |
| 243 | test_expect_success "ipfs dag import --allow-big-block works" ' |
| 244 | test_expect_code 0 ipfs dag import --allow-big-block over-2MiB-block.car |
| 245 | ' |
| 246 | |
| 247 | cat > version_2_import_expected << EOE |
| 248 | {"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}} |
| 249 | {"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}} |
| 250 | {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} |
| 251 | EOE |
| 252 | |
| 253 | test_expect_success "version 2 import" ' |
| 254 | ipfs dag import --stats --enc=json \ |
| 255 | ../t0054-dag-car-import-export-data/lotus_testnet_export_128_v2.car \ |
| 256 | ../t0054-dag-car-import-export-data/lotus_devnet_genesis_v2.car \ |
| 257 | > version_2_import_actual |
| 258 | ' |
| 259 | |
| 260 | test_expect_success "version 2 import output as expected" ' |
| 261 | test_cmp_sorted version_2_import_expected version_2_import_actual |
| 262 | ' |
| 263 | |
| 264 | test_expect_success "'ipfs dag import' decode IPLD 'dag-json' codec works" ' |
| 265 | NEW_HASH=$(echo "{ \"test\": \"dag-json\" }" | ipfs dag put --store-codec dag-json) && |
| 266 | ipfs dag export $NEW_HASH > dag-json.car && |
| 267 | ipfs dag import dag-json.car && |
| 268 | rm dag-json.car |
| 269 | ' |
| 270 | |
| 271 | test_expect_success "'ipfs dag import' decode IPLD 'dag-cbor' codec works" ' |
| 272 | NEW_HASH=$(echo "{ \"test\": \"dag-cbor\" }" | ipfs dag put --store-codec dag-cbor) && |
| 273 | ipfs dag export $NEW_HASH > dag-cbor.car && |
| 274 | ipfs dag import dag-cbor.car && |
| 275 | rm dag-cbor.car |
| 276 | ' |
| 277 | |
| 278 | test_expect_success "'ipfs dag import' decode IPLD 'json' codec works" ' |
| 279 | NEW_HASH=$(echo "{ \"test\": \"json\" }" | ipfs dag put --store-codec json) && |
| 280 | ipfs dag export $NEW_HASH > json.car && |
| 281 | ipfs dag import json.car && |
| 282 | rm json.car |
| 283 | ' |
| 284 | |
| 285 | test_expect_success "'ipfs dag import' decode IPLD 'cbor' codec works" ' |
| 286 | NEW_HASH=$(echo "{ \"test\": \"cbor\" }" | ipfs dag put --store-codec cbor) && |
| 287 | ipfs dag export $NEW_HASH > cbor.car && |
| 288 | ipfs dag import cbor.car && |
| 289 | rm cbor.car |
| 290 | ' |
| 291 | |
| 292 | # IPIP-402 |
| 293 | cat > partial_nopin_import_expected << EOE |
| 294 | {"Stats":{"BlockCount":1,"BlockBytesCount":1618}} |
| 295 | EOE |
| 296 | test_expect_success "'ipfs dag import' without pinning works fine with incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" ' |
| 297 | ipfs dag import --stats --enc=json --pin-roots=false ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_nopin_import_out 2>&1 && |
| 298 | test_cmp partial_nopin_import_expected partial_nopin_import_out |
| 299 | ' |
| 300 | |
| 301 | test_expect_success "'ipfs dag import' with pinning errors due to incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" ' |
| 302 | ipfs dag import --stats --enc=json --pin-roots=true ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 && |
| 303 | test_should_contain "\"PinErrorMsg\":\"block was not found locally" partial_pin_import_out |
| 304 | ' |
| 305 | |
| 306 | test_expect_success "'ipfs dag import' pin error in default CLI mode produces exit code 1 (unixfs dir exported as dag-scope=entity from IPIP-402)" ' |
| 307 | test_expect_code 1 ipfs dag import ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 && |
| 308 | test_should_contain "Error: pinning root \"QmPDC11yLAbVw3dX5jMeEuSdk4BiVjSd9X87zaYRdVjzW3\" FAILED: block was not found locally" partial_pin_import_out |
| 309 | ' |
| 310 | |
| 311 | test_done |