| 1 | #!/usr/bin/env bash |
| 2 | # |
| 3 | # Copyright (c) 2017 Jeromy Johnson |
| 4 | # MIT Licensed; see the LICENSE file in this repository. |
| 5 | # |
| 6 | |
| 7 | test_description="Test out the filestore nocopy functionality" |
| 8 | |
| 9 | . lib/test-lib.sh |
| 10 | |
| 11 | test_init_filestore() { |
| 12 | test_expect_success "clean up old node" ' |
| 13 | rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs |
| 14 | ' |
| 15 | |
| 16 | test_init_ipfs |
| 17 | |
| 18 | test_expect_success "enable filestore config setting" ' |
| 19 | ipfs config --json Experimental.FilestoreEnabled true |
| 20 | ' |
| 21 | } |
| 22 | |
| 23 | test_init_dataset() { |
| 24 | test_expect_success "create a dataset" ' |
| 25 | rm -r somedir |
| 26 | mkdir somedir && |
| 27 | random-data -size=1000 -seed=1 > somedir/file1 && |
| 28 | random-data -size=10000 -seed=2 > somedir/file2 && |
| 29 | random-data -size=1000000 -seed=3 > somedir/file3 |
| 30 | ' |
| 31 | } |
| 32 | |
| 33 | test_init() { |
| 34 | test_init_filestore |
| 35 | test_init_dataset |
| 36 | } |
| 37 | |
| 38 | EXPHASH="QmXqfraAT3U8ct14PPPXcFkWyvmqUZazLdo29GXTKSHkP4" |
| 39 | |
| 40 | cat <<EOF > ls_expect_file_order |
| 41 | bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 |
| 42 | bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 |
| 43 | bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 |
| 44 | bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 |
| 45 | bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 |
| 46 | bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 |
| 47 | EOF |
| 48 | |
| 49 | sort < ls_expect_file_order > ls_expect_key_order |
| 50 | |
| 51 | FILE1_HASH=bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu |
| 52 | FILE2_HASH=bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 |
| 53 | FILE3_HASH=QmYEZtRGGk8rgM8MetegLLRHMKskPCg7zWpmQQAo3cQiN5 |
| 54 | |
| 55 | cat <<EOF > verify_expect_file_order |
| 56 | ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 |
| 57 | ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 |
| 58 | ok bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 |
| 59 | ok bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 |
| 60 | ok bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 |
| 61 | ok bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 |
| 62 | EOF |
| 63 | |
| 64 | sort < verify_expect_file_order > verify_expect_key_order |
| 65 | |
| 66 | cat <<EOF > verify_rm_expect |
| 67 | ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 keep |
| 68 | ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 keep |
| 69 | changed bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 remove |
| 70 | changed bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 remove |
| 71 | changed bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 remove |
| 72 | changed bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 remove |
| 73 | EOF |
| 74 | |
| 75 | cat <<EOF > verify_after_rm_expect |
| 76 | ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 |
| 77 | ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 |
| 78 | EOF |
| 79 | |
| 80 | IPFS_CMD="ipfs" |
| 81 | |
| 82 | test_filestore_adds() { |
| 83 | test_expect_success "$IPFS_CMD add nocopy add succeeds" ' |
| 84 | HASH=$($IPFS_CMD add --raw-leaves --nocopy -r -Q somedir) |
| 85 | ' |
| 86 | |
| 87 | test_expect_success "nocopy add has right hash" ' |
| 88 | test "$HASH" = "$EXPHASH" |
| 89 | ' |
| 90 | |
| 91 | test_expect_success "'$IPFS_CMD filestore ls' output looks good'" ' |
| 92 | $IPFS_CMD filestore ls | sort > ls_actual && |
| 93 | test_cmp ls_expect_key_order ls_actual |
| 94 | ' |
| 95 | |
| 96 | test_expect_success "'$IPFS_CMD filestore ls --file-order' output looks good'" ' |
| 97 | $IPFS_CMD filestore ls --file-order > ls_actual && |
| 98 | test_cmp ls_expect_file_order ls_actual |
| 99 | ' |
| 100 | |
| 101 | test_expect_success "'$IPFS_CMD filestore ls HASH' works" ' |
| 102 | $IPFS_CMD filestore ls $FILE1_HASH > ls_actual && |
| 103 | grep -q somedir/file1 ls_actual |
| 104 | ' |
| 105 | |
| 106 | test_expect_success "can retrieve multi-block file" ' |
| 107 | $IPFS_CMD cat $FILE3_HASH > file3.data && |
| 108 | test_cmp somedir/file3 file3.data |
| 109 | ' |
| 110 | } |
| 111 | |
| 112 | # check that the filestore is in a clean state |
| 113 | test_filestore_state() { |
| 114 | test_expect_success "$IPFS_CMD filestore verify' output looks good'" ' |
| 115 | $IPFS_CMD filestore verify | LC_ALL=C sort > verify_actual |
| 116 | test_cmp verify_expect_key_order verify_actual |
| 117 | ' |
| 118 | } |
| 119 | |
| 120 | test_filestore_verify() { |
| 121 | test_filestore_state |
| 122 | |
| 123 | test_expect_success "$IPFS_CMD filestore verify --file-order' output looks good'" ' |
| 124 | $IPFS_CMD filestore verify --file-order > verify_actual |
| 125 | test_cmp verify_expect_file_order verify_actual |
| 126 | ' |
| 127 | |
| 128 | test_expect_success "'$IPFS_CMD filestore verify HASH' works" ' |
| 129 | $IPFS_CMD filestore verify $FILE1_HASH > verify_actual && |
| 130 | grep -q somedir/file1 verify_actual |
| 131 | ' |
| 132 | |
| 133 | test_expect_success "rename a file" ' |
| 134 | mv somedir/file1 somedir/file1.bk |
| 135 | ' |
| 136 | |
| 137 | test_expect_success "can not retrieve block after backing file moved" ' |
| 138 | test_must_fail $IPFS_CMD cat $FILE1_HASH |
| 139 | ' |
| 140 | |
| 141 | test_expect_success "'$IPFS_CMD filestore verify' shows file as missing" ' |
| 142 | $IPFS_CMD filestore verify > verify_actual && |
| 143 | grep no-file verify_actual | grep -q somedir/file1 |
| 144 | ' |
| 145 | |
| 146 | test_expect_success "move file back" ' |
| 147 | mv somedir/file1.bk somedir/file1 |
| 148 | ' |
| 149 | |
| 150 | test_expect_success "block okay now" ' |
| 151 | $IPFS_CMD cat $FILE1_HASH > file1.data && |
| 152 | test_cmp somedir/file1 file1.data |
| 153 | ' |
| 154 | |
| 155 | test_expect_success "change first bit of file" ' |
| 156 | dd if=/dev/zero of=somedir/file3 bs=1024 count=1 |
| 157 | ' |
| 158 | |
| 159 | test_expect_success "can not retrieve block after backing file changed" ' |
| 160 | test_must_fail $IPFS_CMD cat $FILE3_HASH |
| 161 | ' |
| 162 | |
| 163 | test_expect_success "'$IPFS_CMD filestore verify' shows file as changed" ' |
| 164 | $IPFS_CMD filestore verify > verify_actual && |
| 165 | grep changed verify_actual | grep -q somedir/file3 |
| 166 | ' |
| 167 | |
| 168 | # reset the state for the next test |
| 169 | test_init_dataset |
| 170 | } |
| 171 | |
| 172 | test_filestore_rm_bad_blocks() { |
| 173 | test_filestore_state |
| 174 | |
| 175 | test_expect_success "change first bit of file" ' |
| 176 | dd if=/dev/zero of=somedir/file3 bs=1024 count=1 |
| 177 | ' |
| 178 | |
| 179 | test_expect_success "'$IPFS_CMD filestore verify --remove-bad-blocks' shows changed file removed" ' |
| 180 | $IPFS_CMD filestore verify --remove-bad-blocks > verify_rm_actual && |
| 181 | test_cmp verify_rm_expect verify_rm_actual |
| 182 | ' |
| 183 | |
| 184 | test_expect_success "'$IPFS_CMD filestore verify' shows only files that were not removed" ' |
| 185 | $IPFS_CMD filestore verify > verify_after && |
| 186 | test_cmp verify_after_rm_expect verify_after |
| 187 | ' |
| 188 | |
| 189 | # reset the state for the next test |
| 190 | test_init_dataset |
| 191 | } |
| 192 | |
| 193 | test_filestore_dups() { |
| 194 | # make sure the filestore is in a clean state |
| 195 | test_filestore_state |
| 196 | |
| 197 | test_expect_success "'$IPFS_CMD filestore dups'" ' |
| 198 | $IPFS_CMD add --raw-leaves somedir/file1 && |
| 199 | $IPFS_CMD filestore dups > dups_actual && |
| 200 | echo "$FILE1_HASH" > dups_expect |
| 201 | test_cmp dups_expect dups_actual |
| 202 | ' |
| 203 | } |
| 204 | |
| 205 | # |
| 206 | # No daemon |
| 207 | # |
| 208 | |
| 209 | test_init |
| 210 | |
| 211 | test_filestore_adds |
| 212 | |
| 213 | test_filestore_verify |
| 214 | |
| 215 | test_filestore_dups |
| 216 | |
| 217 | test_filestore_rm_bad_blocks |
| 218 | |
| 219 | # |
| 220 | # With daemon |
| 221 | # |
| 222 | |
| 223 | test_init |
| 224 | |
| 225 | # must be in offline mode so tests that retrieve non-existent blocks |
| 226 | # doesn't hang |
| 227 | test_launch_ipfs_daemon_without_network |
| 228 | |
| 229 | test_filestore_adds |
| 230 | |
| 231 | test_filestore_verify |
| 232 | |
| 233 | test_filestore_dups |
| 234 | |
| 235 | test_kill_ipfs_daemon |
| 236 | |
| 237 | test_filestore_rm_bad_blocks |
| 238 | |
| 239 | ## |
| 240 | ## base32 |
| 241 | ## |
| 242 | |
| 243 | EXPHASH="bafybeienfbjfbywu5y44i5qm4wxajblgy5a6xuc4eepjaw5fq223wwsy3m" |
| 244 | |
| 245 | cat <<EOF > ls_expect_file_order |
| 246 | bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 |
| 247 | bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 |
| 248 | bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 |
| 249 | bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 |
| 250 | bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 |
| 251 | bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 |
| 252 | EOF |
| 253 | |
| 254 | sort < ls_expect_file_order > ls_expect_key_order |
| 255 | |
| 256 | FILE1_HASH=bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu |
| 257 | FILE2_HASH=bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 |
| 258 | FILE3_HASH=bafybeietaxxjghilcjhc2m4zcmicm7yjvkjdfkamc3ct2hq4gmsb3shqsi |
| 259 | |
| 260 | cat <<EOF > verify_expect_file_order |
| 261 | ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 |
| 262 | ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 |
| 263 | ok bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 |
| 264 | ok bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 |
| 265 | ok bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 |
| 266 | ok bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 |
| 267 | EOF |
| 268 | |
| 269 | sort < verify_expect_file_order > verify_expect_key_order |
| 270 | |
| 271 | IPFS_CMD="ipfs --cid-base=base32" |
| 272 | |
| 273 | # |
| 274 | # No daemon |
| 275 | # |
| 276 | |
| 277 | test_init |
| 278 | |
| 279 | test_filestore_adds |
| 280 | |
| 281 | test_filestore_verify |
| 282 | |
| 283 | test_filestore_dups |
| 284 | |
| 285 | test_filestore_rm_bad_blocks |
| 286 | |
| 287 | # |
| 288 | # With daemon |
| 289 | # |
| 290 | |
| 291 | test_init |
| 292 | |
| 293 | # must be in offline mode so tests that retrieve non-existent blocks |
| 294 | # doesn't hang |
| 295 | test_launch_ipfs_daemon_without_network |
| 296 | |
| 297 | test_filestore_adds |
| 298 | |
| 299 | test_filestore_verify |
| 300 | |
| 301 | test_filestore_dups |
| 302 | |
| 303 | test_kill_ipfs_daemon |
| 304 | |
| 305 | test_done |
| 306 | |
| 307 | test_filestore_rm_bad_blocks |
| 308 | |
| 309 | ## |
| 310 | |
| 311 | test_done |