| 1 | #!/usr/bin/env bash |
| 2 | # |
| 3 | # Copyright (c) 2014 Christian Couder |
| 4 | # MIT Licensed; see the LICENSE file in this repository. |
| 5 | # |
| 6 | |
| 7 | test_description="Test directory sharding" |
| 8 | |
| 9 | . lib/test-lib.sh |
| 10 | |
| 11 | test_expect_success "set up test data" ' |
| 12 | mkdir testdata |
| 13 | for i in `seq 2000` |
| 14 | do |
| 15 | echo $i > testdata/file$i |
| 16 | done |
| 17 | ' |
| 18 | |
| 19 | test_add_dir() { |
| 20 | exphash="$1" |
| 21 | test_expect_success "ipfs add on directory succeeds" ' |
| 22 | ipfs add -r -Q testdata > sharddir_out && |
| 23 | echo "$exphash" > sharddir_exp && |
| 24 | test_cmp sharddir_exp sharddir_out |
| 25 | ' |
| 26 | test_expect_success "ipfs get on directory succeeds" ' |
| 27 | ipfs get -o testdata-out "$exphash" && |
| 28 | test_cmp testdata testdata-out |
| 29 | ' |
| 30 | } |
| 31 | |
| 32 | test_init_ipfs |
| 33 | |
| 34 | UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N" |
| 35 | |
| 36 | test_expect_success "force sharding off" ' |
| 37 | ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1G\"" |
| 38 | ' |
| 39 | |
| 40 | test_add_dir "$UNSHARDED" |
| 41 | |
| 42 | test_launch_ipfs_daemon |
| 43 | |
| 44 | test_add_dir "$UNSHARDED" |
| 45 | |
| 46 | test_kill_ipfs_daemon |
| 47 | |
| 48 | test_expect_success "force sharding on" ' |
| 49 | ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\"" |
| 50 | ' |
| 51 | |
| 52 | SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL" |
| 53 | test_add_dir "$SHARDED" |
| 54 | |
| 55 | test_launch_ipfs_daemon |
| 56 | |
| 57 | test_add_dir "$SHARDED" |
| 58 | |
| 59 | test_kill_ipfs_daemon |
| 60 | |
| 61 | test_expect_success "sharded and unsharded output look the same" ' |
| 62 | ipfs ls "$SHARDED" | sort > sharded_out && |
| 63 | ipfs ls "$UNSHARDED" | sort > unsharded_out && |
| 64 | test_cmp sharded_out unsharded_out |
| 65 | ' |
| 66 | |
| 67 | test_expect_success "ipfs cat error output the same" ' |
| 68 | test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err && |
| 69 | test_expect_code 1 ipfs cat "$UNSHARDED" 2> unsharded_err && |
| 70 | test_cmp sharded_err unsharded_err |
| 71 | ' |
| 72 | |
| 73 | test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing block" ' |
| 74 | ipfs ls "$SHARDED" | head -1 > first_file && |
| 75 | ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize && |
| 76 | read -r HASH _ NAME <first_file && |
| 77 | ipfs pin rm "$SHARDED" "$UNSHARDED" && # To allow us to remove the block |
| 78 | ipfs block rm "$HASH" && |
| 79 | test_expect_code 1 ipfs cat "$SHARDED/$NAME" && |
| 80 | test_expect_code 1 ipfs ls "$SHARDED" && |
| 81 | ipfs ls --resolve-type=false --size=false "$SHARDED" | sort > missing_out && |
| 82 | test_cmp sharded_out_nosize missing_out |
| 83 | ' |
| 84 | |
| 85 | test_launch_ipfs_daemon |
| 86 | |
| 87 | test_expect_success "gateway can resolve sharded dirs" ' |
| 88 | echo 100 > expected && |
| 89 | curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file100" && |
| 90 | test_cmp expected actual |
| 91 | ' |
| 92 | |
| 93 | test_expect_success "'ipfs resolve' can resolve sharded dirs" ' |
| 94 | echo /ipfs/QmZ3RfWk1u5LEGYLHA633B5TNJy3Du27K6Fny9wcxpowGS > expected && |
| 95 | ipfs resolve "/ipfs/$SHARDED/file100" > actual && |
| 96 | test_cmp expected actual |
| 97 | ' |
| 98 | |
| 99 | test_kill_ipfs_daemon |
| 100 | |
| 101 | test_add_dir_v1() { |
| 102 | exphash="$1" |
| 103 | test_expect_success "ipfs add (CIDv1) on directory succeeds" ' |
| 104 | ipfs add -r -Q --cid-version=1 testdata > sharddir_out && |
| 105 | echo "$exphash" > sharddir_exp && |
| 106 | test_cmp sharddir_exp sharddir_out |
| 107 | ' |
| 108 | |
| 109 | test_expect_success "can access a path under the dir" ' |
| 110 | ipfs cat "$exphash/file20" > file20_out && |
| 111 | test_cmp testdata/file20 file20_out |
| 112 | ' |
| 113 | } |
| 114 | |
| 115 | # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw |
| 116 | SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u" |
| 117 | test_add_dir_v1 "$SHARDEDV1" |
| 118 | |
| 119 | test_launch_ipfs_daemon |
| 120 | |
| 121 | test_add_dir_v1 "$SHARDEDV1" |
| 122 | |
| 123 | test_kill_ipfs_daemon |
| 124 | |
| 125 | test_list_incomplete_dir() { |
| 126 | test_expect_success "ipfs add (CIDv1) on very large directory with sha3 succeeds" ' |
| 127 | ipfs add -r -Q --cid-version=1 --hash=sha3-256 --pin=false testdata > sharddir_out && |
| 128 | largeSHA3dir=$(cat sharddir_out) |
| 129 | ' |
| 130 | |
| 131 | test_expect_success "delete intermediate node from DAG" ' |
| 132 | ipfs block rm "/ipld/$largeSHA3dir/Links/0/Hash" |
| 133 | ' |
| 134 | |
| 135 | test_expect_success "can list part of the directory" ' |
| 136 | ipfs ls "$largeSHA3dir" 2> ls_err_out |
| 137 | echo "Error: failed to fetch all nodes" > exp_err_out && |
| 138 | cat ls_err_out && |
| 139 | test_cmp exp_err_out ls_err_out |
| 140 | ' |
| 141 | } |
| 142 | |
| 143 | test_list_incomplete_dir |
| 144 | |
| 145 | test_done |