| 1 | #!/usr/bin/env bash |
| 2 | # group: rw auto quick |
| 3 | # |
| 4 | # Test case for repairing qcow2 images which cannot be repaired using |
| 5 | # the on-disk refcount structures |
| 6 | # |
| 7 | # Copyright (C) 2014 Red Hat, Inc. |
| 8 | # |
| 9 | # This program is free software; you can redistribute it and/or modify |
| 10 | # it under the terms of the GNU General Public License as published by |
| 11 | # the Free Software Foundation; either version 2 of the License, or |
| 12 | # (at your option) any later version. |
| 13 | # |
| 14 | # This program is distributed in the hope that it will be useful, |
| 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | # GNU General Public License for more details. |
| 18 | # |
| 19 | # You should have received a copy of the GNU General Public License |
| 20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 21 | # |
| 22 | |
| 23 | # creator |
| 24 | owner=hreitz@redhat.com |
| 25 | |
| 26 | seq="$(basename $0)" |
| 27 | echo "QA output created by $seq" |
| 28 | |
| 29 | status=1 # failure is the default! |
| 30 | |
| 31 | _cleanup() |
| 32 | { |
| 33 | _cleanup_test_img |
| 34 | if [ -f "$TEST_DIR/qsd.pid" ]; then |
| 35 | qsd_pid=$(cat "$TEST_DIR/qsd.pid") |
| 36 | kill -KILL "$qsd_pid" |
| 37 | fi |
| 38 | fusermount -u "$TEST_DIR/fuse-export" &>/dev/null |
| 39 | rm -f "$TEST_DIR/fuse-export" |
| 40 | } |
| 41 | trap "_cleanup; exit \$status" 0 1 2 3 15 |
| 42 | |
| 43 | # get standard environment, filters and checks |
| 44 | . ./common.rc |
| 45 | . ./common.filter |
| 46 | . ./common.qemu |
| 47 | |
| 48 | # This tests qcow2-specific low-level functionality |
| 49 | _supported_fmt qcow2 |
| 50 | _supported_proto file fuse |
| 51 | _supported_os Linux |
| 52 | # This test directly modifies a refblock so it relies on refcount_bits being 16; |
| 53 | # and the low-level modification it performs are not tuned for external data |
| 54 | # files |
| 55 | _unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file |
| 56 | |
| 57 | # This test either needs sudo -n losetup or FUSE exports to work |
| 58 | if sudo -n losetup &>/dev/null; then |
| 59 | loopdev=true |
| 60 | else |
| 61 | loopdev=false |
| 62 | |
| 63 | # Check for usable FUSE in the host environment: |
| 64 | if test ! -c "/dev/fuse"; then |
| 65 | _notrun 'No passwordless sudo nor usable /dev/fuse' |
| 66 | fi |
| 67 | |
| 68 | # QSD --export fuse will either yield "Parameter 'id' is missing" |
| 69 | # or "Invalid parameter 'fuse'", depending on whether there is |
| 70 | # FUSE support or not. |
| 71 | error=$($QSD --export fuse 2>&1) |
| 72 | if [[ $error = *"'fuse'"* ]]; then |
| 73 | _notrun 'Passwordless sudo for losetup or FUSE support required, but' \ |
| 74 | 'neither is available' |
| 75 | fi |
| 76 | fi |
| 77 | |
| 78 | echo |
| 79 | echo '=== Repairing an image without any refcount table ===' |
| 80 | echo |
| 81 | |
| 82 | _make_test_img 64M |
| 83 | # just write some data |
| 84 | $QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io |
| 85 | |
| 86 | # refcount_table_offset |
| 87 | poke_file "$TEST_IMG" $((0x30)) "\x00\x00\x00\x00\x00\x00\x00\x00" |
| 88 | # refcount_table_clusters |
| 89 | poke_file "$TEST_IMG" $((0x38)) "\x00\x00\x00\x00" |
| 90 | |
| 91 | _check_test_img -r all |
| 92 | |
| 93 | $QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io |
| 94 | |
| 95 | echo |
| 96 | echo '=== Repairing unreferenced data cluster in new refblock area ===' |
| 97 | echo |
| 98 | |
| 99 | _make_test_img -o 'cluster_size=512' 64M |
| 100 | # Allocate the first 128 kB in the image (first refblock) |
| 101 | $QEMU_IO -c 'write 0 0x1b200' "$TEST_IMG" | _filter_qemu_io |
| 102 | # should be 131072 == 0x20000 |
| 103 | stat -c '%s' "$TEST_IMG" |
| 104 | |
| 105 | # Enter a cluster at 128 kB (0x20000) |
| 106 | # XXX: This should be the first free entry in the last L2 table, but we cannot |
| 107 | # be certain |
| 108 | poke_file "$TEST_IMG" $((0x1ccc8)) "\x80\x00\x00\x00\x00\x02\x00\x00" |
| 109 | |
| 110 | # Fill the cluster |
| 111 | truncate -s $((0x20200)) "$TEST_IMG" |
| 112 | $QEMU_IO -c "open -o driver=raw $TEST_IMG" -c 'write -P 42 128k 512' \ |
| 113 | | _filter_qemu_io |
| 114 | |
| 115 | # The data should now appear at this guest offset |
| 116 | $QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io |
| 117 | |
| 118 | # This cluster is unallocated; fix it |
| 119 | _check_test_img -r all |
| 120 | |
| 121 | # This repair operation must have allocated a new refblock; and that refblock |
| 122 | # should not overlap with the unallocated data cluster. If it does, the data |
| 123 | # will be damaged, so check it. |
| 124 | $QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io |
| 125 | |
| 126 | echo |
| 127 | echo '=== Repairing refblock beyond the image end ===' |
| 128 | echo |
| 129 | |
| 130 | echo |
| 131 | echo '--- Otherwise clean ---' |
| 132 | echo |
| 133 | |
| 134 | _make_test_img 64M |
| 135 | # Normally, qemu doesn't create empty refblocks, so we just have to do it by |
| 136 | # hand |
| 137 | # XXX: This should be the entry for the second refblock |
| 138 | poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00" |
| 139 | # Mark that refblock as used |
| 140 | # XXX: This should be the 17th entry (cluster 16) of the first |
| 141 | # refblock |
| 142 | poke_file "$TEST_IMG" $((0x20020)) "\x00\x01" |
| 143 | _check_test_img -r all |
| 144 | |
| 145 | echo |
| 146 | echo '--- Refblock is unallocated ---' |
| 147 | echo |
| 148 | |
| 149 | _make_test_img 64M |
| 150 | poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00" |
| 151 | _check_test_img -r all |
| 152 | |
| 153 | echo |
| 154 | echo '--- Signed overflow after the refblock ---' |
| 155 | echo |
| 156 | |
| 157 | _make_test_img 64M |
| 158 | poke_file "$TEST_IMG" $((0x10008)) "\x7f\xff\xff\xff\xff\xff\x00\x00" |
| 159 | _check_test_img -r all |
| 160 | |
| 161 | echo |
| 162 | echo '--- Unsigned overflow after the refblock ---' |
| 163 | echo |
| 164 | |
| 165 | _make_test_img 64M |
| 166 | poke_file "$TEST_IMG" $((0x10008)) "\xff\xff\xff\xff\xff\xff\x00\x00" |
| 167 | _check_test_img -r all |
| 168 | |
| 169 | echo |
| 170 | echo '=== Check rebuilt reftable location ===' |
| 171 | |
| 172 | # In an earlier version of the refcount rebuild algorithm, the |
| 173 | # reftable was generally placed at the image end (unless something was |
| 174 | # allocated in the area covered by the refblock right before the image |
| 175 | # file end, then we would try to place the reftable in that refblock). |
| 176 | # This was later changed so the reftable would be placed in the |
| 177 | # earliest possible location. Test this. |
| 178 | |
| 179 | echo |
| 180 | echo '--- Does the image size increase? ---' |
| 181 | echo |
| 182 | |
| 183 | # First test: Just create some image, write some data to it, and |
| 184 | # resize it so there is free space at the end of the image (enough |
| 185 | # that it spans at least one full refblock, which for cluster_size=512 |
| 186 | # images, spans 128k). With the old algorithm, the reftable would |
| 187 | # have then been placed at the end of the image file, but with the new |
| 188 | # one, it will be put in that free space. |
| 189 | # We want to check whether the size of the image file increases due to |
| 190 | # rebuilding the refcount structures (it should not). |
| 191 | |
| 192 | _make_test_img -o 'cluster_size=512' 1M |
| 193 | # Write something |
| 194 | $QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io |
| 195 | |
| 196 | # Add free space |
| 197 | file_len=$(stat -c '%s' "$TEST_IMG") |
| 198 | truncate -s $((file_len + 256 * 1024)) "$TEST_IMG" |
| 199 | |
| 200 | # Corrupt the image by saying the image header was not allocated |
| 201 | rt_offset=$(peek_file_be "$TEST_IMG" 48 8) |
| 202 | rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8) |
| 203 | poke_file "$TEST_IMG" $rb_offset "\x00\x00" |
| 204 | |
| 205 | # Check whether rebuilding the refcount structures increases the image |
| 206 | # file size |
| 207 | file_len=$(stat -c '%s' "$TEST_IMG") |
| 208 | echo |
| 209 | # The only leaks there can be are the old refcount structures that are |
| 210 | # leaked during rebuilding, no need to clutter the output with them |
| 211 | _check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0' |
| 212 | echo |
| 213 | post_repair_file_len=$(stat -c '%s' "$TEST_IMG") |
| 214 | |
| 215 | if [[ $file_len -eq $post_repair_file_len ]]; then |
| 216 | echo 'OK: Image size did not change' |
| 217 | else |
| 218 | echo 'ERROR: Image size differs' \ |
| 219 | "($file_len before, $post_repair_file_len after)" |
| 220 | fi |
| 221 | |
| 222 | echo |
| 223 | echo '--- Will the reftable occupy a hole specifically left for it? ---' |
| 224 | echo |
| 225 | |
| 226 | # Note: With cluster_size=512, every refblock covers 128k. |
| 227 | # The reftable covers 8M per reftable cluster. |
| 228 | |
| 229 | # Create an image that requires two reftable clusters (just because |
| 230 | # this is more interesting than a single-clustered reftable). |
| 231 | _make_test_img -o 'cluster_size=512' 9M |
| 232 | $QEMU_IO -c 'write 0 8M' "$TEST_IMG" | _filter_qemu_io |
| 233 | |
| 234 | # Writing 8M will have resized the reftable. Unfortunately, doing so |
| 235 | # will leave holes in the file, so we need to fill them up so we can |
| 236 | # be sure the whole file is allocated. Do that by writing |
| 237 | # consecutively smaller chunks starting from 8 MB, until the file |
| 238 | # length increases even with a chunk size of 512. Then we must have |
| 239 | # filled all holes. |
| 240 | ofs=$((8 * 1024 * 1024)) |
| 241 | block_len=$((16 * 1024)) |
| 242 | while [[ $block_len -ge 512 ]]; do |
| 243 | file_len=$(stat -c '%s' "$TEST_IMG") |
| 244 | while [[ $(stat -c '%s' "$TEST_IMG") -eq $file_len ]]; do |
| 245 | # Do not include this in the reference output, it does not |
| 246 | # really matter which qemu-io calls we do here exactly |
| 247 | $QEMU_IO -c "write $ofs $block_len" "$TEST_IMG" >/dev/null |
| 248 | ofs=$((ofs + block_len)) |
| 249 | done |
| 250 | block_len=$((block_len / 2)) |
| 251 | done |
| 252 | |
| 253 | # Fill up to 9M (do not include this in the reference output either, |
| 254 | # $ofs is random for all we know) |
| 255 | $QEMU_IO -c "write $ofs $((9 * 1024 * 1024 - ofs))" "$TEST_IMG" >/dev/null |
| 256 | |
| 257 | # Make space as follows: |
| 258 | # - For the first refblock: Right at the beginning of the image (this |
| 259 | # refblock is placed in the first place possible), |
| 260 | # - For the reftable somewhere soon afterwards, still near the |
| 261 | # beginning of the image (i.e. covered by the first refblock); the |
| 262 | # reftable too is placed in the first place possible, but only after |
| 263 | # all refblocks have been placed) |
| 264 | # No space is needed for the other refblocks, because no refblock is |
| 265 | # put before the space it covers. In this test case, we do not mind |
| 266 | # if they are placed at the image file's end. |
| 267 | |
| 268 | # Before we make that space, we have to find out the host offset of |
| 269 | # the area that belonged to the two data clusters at guest offset 4k, |
| 270 | # because we expect the reftable to be placed there, and we will have |
| 271 | # to verify that it is. |
| 272 | |
| 273 | l1_offset=$(peek_file_be "$TEST_IMG" 40 8) |
| 274 | l2_offset=$(peek_file_be "$TEST_IMG" $l1_offset 8) |
| 275 | l2_offset=$((l2_offset & 0x00fffffffffffe00)) |
| 276 | data_4k_offset=$(peek_file_be "$TEST_IMG" \ |
| 277 | $((l2_offset + 4096 / 512 * 8)) 8) |
| 278 | data_4k_offset=$((data_4k_offset & 0x00fffffffffffe00)) |
| 279 | |
| 280 | $QEMU_IO -c "discard 0 512" -c "discard 4k 1k" "$TEST_IMG" | _filter_qemu_io |
| 281 | |
| 282 | # Corrupt the image by saying the image header was not allocated |
| 283 | rt_offset=$(peek_file_be "$TEST_IMG" 48 8) |
| 284 | rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8) |
| 285 | poke_file "$TEST_IMG" $rb_offset "\x00\x00" |
| 286 | |
| 287 | echo |
| 288 | # The only leaks there can be are the old refcount structures that are |
| 289 | # leaked during rebuilding, no need to clutter the output with them |
| 290 | _check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0' |
| 291 | echo |
| 292 | |
| 293 | # Check whether the reftable was put where we expected |
| 294 | rt_offset=$(peek_file_be "$TEST_IMG" 48 8) |
| 295 | if [[ $rt_offset -eq $data_4k_offset ]]; then |
| 296 | echo 'OK: Reftable is where we expect it' |
| 297 | else |
| 298 | echo "ERROR: Reftable is at $rt_offset, but was expected at $data_4k_offset" |
| 299 | fi |
| 300 | |
| 301 | echo |
| 302 | echo '--- Rebuilding refcount structures on block devices ---' |
| 303 | echo |
| 304 | |
| 305 | # A block device cannot really grow, at least not during qemu-img |
| 306 | # check. As mentioned in the above cases, rebuilding the refcount |
| 307 | # structure may lead to new refcount structures being written after |
| 308 | # the end of the image, and in the past that happened even if there |
| 309 | # was more than sufficient space in the image. Such post-EOF writes |
| 310 | # will not work on block devices, so test that the new algorithm |
| 311 | # avoids it. |
| 312 | |
| 313 | # If we have passwordless sudo and losetup, we can use those to create |
| 314 | # a block device. Otherwise, we can resort to qemu's FUSE export to |
| 315 | # create a file that isn't growable, which effectively tests the same |
| 316 | # thing. |
| 317 | |
| 318 | _cleanup_test_img |
| 319 | truncate -s $((64 * 1024 * 1024)) "$TEST_IMG" |
| 320 | |
| 321 | if $loopdev; then |
| 322 | export_mp=$(sudo -n losetup --show -f "$TEST_IMG") |
| 323 | export_mp_driver=host_device |
| 324 | sudo -n chmod go+rw "$export_mp" |
| 325 | else |
| 326 | # Create non-growable FUSE export that is a bit like an empty |
| 327 | # block device |
| 328 | export_mp="$TEST_DIR/fuse-export" |
| 329 | export_mp_driver=file |
| 330 | touch "$export_mp" |
| 331 | |
| 332 | $QSD \ |
| 333 | --blockdev file,node-name=export-node,filename="$TEST_IMG" \ |
| 334 | --export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=off,allow-other=off \ |
| 335 | --pidfile "$TEST_DIR/qsd.pid" \ |
| 336 | --daemonize |
| 337 | fi |
| 338 | |
| 339 | # Now create a qcow2 image on the device -- unfortunately, qemu-img |
| 340 | # create force-creates the file, so we have to resort to the |
| 341 | # blockdev-create job. |
| 342 | _launch_qemu \ |
| 343 | --blockdev $export_mp_driver,node-name=file,filename="$export_mp" |
| 344 | |
| 345 | _send_qemu_cmd \ |
| 346 | $QEMU_HANDLE \ |
| 347 | '{ "execute": "qmp_capabilities" }' \ |
| 348 | 'return' |
| 349 | |
| 350 | # Small cluster size again, so the image needs multiple refblocks |
| 351 | _send_qemu_cmd \ |
| 352 | $QEMU_HANDLE \ |
| 353 | '{ "execute": "blockdev-create", |
| 354 | "arguments": { |
| 355 | "job-id": "create", |
| 356 | "options": { |
| 357 | "driver": "qcow2", |
| 358 | "file": "file", |
| 359 | "size": '$((64 * 1024 * 1024))', |
| 360 | "cluster-size": 512 |
| 361 | } } }' \ |
| 362 | '"concluded"' |
| 363 | |
| 364 | _send_qemu_cmd \ |
| 365 | $QEMU_HANDLE \ |
| 366 | '{ "execute": "job-dismiss", "arguments": { "id": "create" } }' \ |
| 367 | 'return' |
| 368 | |
| 369 | _send_qemu_cmd \ |
| 370 | $QEMU_HANDLE \ |
| 371 | '{ "execute": "quit" }' \ |
| 372 | 'return' |
| 373 | |
| 374 | wait=y _cleanup_qemu |
| 375 | echo |
| 376 | |
| 377 | # Write some data |
| 378 | $QEMU_IO -c 'write 0 64k' "$export_mp" | _filter_qemu_io |
| 379 | |
| 380 | # Corrupt the image by saying the image header was not allocated |
| 381 | rt_offset=$(peek_file_be "$export_mp" 48 8) |
| 382 | rb_offset=$(peek_file_be "$export_mp" $rt_offset 8) |
| 383 | poke_file "$export_mp" $rb_offset "\x00\x00" |
| 384 | |
| 385 | # Repairing such a simple case should just work |
| 386 | # (We used to put the reftable at the end of the image file, which can |
| 387 | # never work for non-growable devices.) |
| 388 | echo |
| 389 | TEST_IMG="$export_mp" _check_test_img -r all \ |
| 390 | | grep -v '^Repairing cluster.*refcount=1 reference=0' |
| 391 | |
| 392 | if $loopdev; then |
| 393 | sudo -n losetup -d "$export_mp" |
| 394 | else |
| 395 | qsd_pid=$(cat "$TEST_DIR/qsd.pid") |
| 396 | kill -TERM "$qsd_pid" |
| 397 | # Wait for process to exit (cannot `wait` because the QSD is daemonized) |
| 398 | while [ -f "$TEST_DIR/qsd.pid" ]; do |
| 399 | true |
| 400 | done |
| 401 | fi |
| 402 | |
| 403 | # success, all done |
| 404 | echo '*** done' |
| 405 | rm -f $seq.full |
| 406 | status=0 |