| 1 | #!/usr/bin/env bash |
| 2 | # group: rw auto quick |
| 3 | # |
| 4 | # Test qcow2 with external data files |
| 5 | # |
| 6 | # Copyright (C) 2019 Red Hat, Inc. |
| 7 | # |
| 8 | # This program is free software; you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation; either version 2 of the License, or |
| 11 | # (at your option) any later version. |
| 12 | # |
| 13 | # This program is distributed in the hope that it will be useful, |
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | # GNU General Public License for more details. |
| 17 | # |
| 18 | # You should have received a copy of the GNU General Public License |
| 19 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 20 | # |
| 21 | |
| 22 | # creator |
| 23 | owner=kwolf@redhat.com |
| 24 | |
| 25 | seq=$(basename $0) |
| 26 | echo "QA output created by $seq" |
| 27 | |
| 28 | status=1 # failure is the default! |
| 29 | |
| 30 | _cleanup() |
| 31 | { |
| 32 | _cleanup_test_img |
| 33 | _rm_test_img "$TEST_IMG.data" |
| 34 | _rm_test_img "$TEST_IMG.src" |
| 35 | } |
| 36 | trap "_cleanup; exit \$status" 0 1 2 3 15 |
| 37 | |
| 38 | # get standard environment, filters and checks |
| 39 | . ./common.rc |
| 40 | . ./common.filter |
| 41 | . ./common.qemu |
| 42 | |
| 43 | _supported_fmt qcow2 |
| 44 | _supported_proto file |
| 45 | _supported_os Linux |
| 46 | # External data files do not work with compat=0.10, and because we use |
| 47 | # our own external data file, we cannot let the user specify one |
| 48 | _unsupported_imgopts 'compat=0.10' data_file |
| 49 | |
| 50 | echo |
| 51 | echo "=== Create and open image with external data file ===" |
| 52 | echo |
| 53 | |
| 54 | echo "With data file name in the image:" |
| 55 | _make_test_img -o "data_file=$TEST_IMG.data" 64M |
| 56 | _check_test_img |
| 57 | |
| 58 | $QEMU_IO -c "open $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 59 | $QEMU_IO -c "open -odata-file.filename=$TEST_IMG.data $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 60 | $QEMU_IO -c "open -odata-file.filename=inexistent $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 61 | |
| 62 | echo |
| 63 | echo "Data file required, but without data file name in the image:" |
| 64 | $QEMU_IMG amend -odata_file= $TEST_IMG |
| 65 | |
| 66 | $QEMU_IO -c "open $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 67 | $QEMU_IO -c "open -odata-file.filename=$TEST_IMG.data $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 68 | $QEMU_IO -c "open -odata-file.filename=inexistent $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 69 | |
| 70 | echo |
| 71 | echo "Setting data-file for an image with internal data:" |
| 72 | _make_test_img 64M |
| 73 | |
| 74 | $QEMU_IO -c "open -odata-file.filename=$TEST_IMG.data $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 75 | $QEMU_IO -c "open -odata-file.filename=inexistent $TEST_IMG" -c "read -P 0 0 64k" 2>&1 | _filter_qemu_io | _filter_testdir |
| 76 | |
| 77 | echo |
| 78 | echo "=== Conflicting features ===" |
| 79 | echo |
| 80 | |
| 81 | echo "Convert to compressed target with data file:" |
| 82 | TEST_IMG="$TEST_IMG.src" _make_test_img 64M |
| 83 | |
| 84 | $QEMU_IO -c 'write -P 0x11 0 1M' \ |
| 85 | -f $IMGFMT "$TEST_IMG.src" | |
| 86 | _filter_qemu_io |
| 87 | |
| 88 | $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -c -odata_file="$TEST_IMG.data" \ |
| 89 | "$TEST_IMG.src" "$TEST_IMG" |
| 90 | |
| 91 | echo |
| 92 | echo "Convert uncompressed, then write compressed data manually:" |
| 93 | $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -odata_file="$TEST_IMG.data" \ |
| 94 | "$TEST_IMG.src" "$TEST_IMG" |
| 95 | $QEMU_IMG compare "$TEST_IMG.src" "$TEST_IMG" |
| 96 | |
| 97 | $QEMU_IO -c 'write -c -P 0x22 0 1M' \ |
| 98 | -f $IMGFMT "$TEST_IMG" | |
| 99 | _filter_qemu_io |
| 100 | _check_test_img |
| 101 | |
| 102 | echo |
| 103 | echo "Take an internal snapshot:" |
| 104 | |
| 105 | $QEMU_IMG snapshot -c test "$TEST_IMG" |
| 106 | _check_test_img |
| 107 | |
| 108 | echo |
| 109 | echo "=== Standalone image with external data file (efficient) ===" |
| 110 | echo |
| 111 | |
| 112 | _make_test_img -o "data_file=$TEST_IMG.data" 64M |
| 113 | |
| 114 | echo -n "qcow2 file size before I/O: " |
| 115 | du -b $TEST_IMG | cut -f1 |
| 116 | |
| 117 | # Create image with the following layout |
| 118 | # 0-1 MB: Unallocated |
| 119 | # 1-2 MB: Written (pattern 0x11) |
| 120 | # 2-3 MB: Discarded |
| 121 | # 3-4 MB: Zero write over discarded space |
| 122 | # 4-5 MB: Zero write over written space |
| 123 | # 5-6 MB: Zero write over unallocated space |
| 124 | |
| 125 | echo |
| 126 | $QEMU_IO -c 'write -P 0x11 1M 4M' \ |
| 127 | -c 'discard 2M 2M' \ |
| 128 | -c 'write -z 3M 3M' \ |
| 129 | -f $IMGFMT "$TEST_IMG" | |
| 130 | _filter_qemu_io |
| 131 | _check_test_img |
| 132 | |
| 133 | echo |
| 134 | $QEMU_IMG map --output=json "$TEST_IMG" |
| 135 | |
| 136 | echo |
| 137 | $QEMU_IO -c 'read -P 0 0 1M' \ |
| 138 | -c 'read -P 0x11 1M 1M' \ |
| 139 | -c 'read -P 0 2M 4M' \ |
| 140 | -f $IMGFMT "$TEST_IMG" | |
| 141 | _filter_qemu_io |
| 142 | |
| 143 | # Zero clusters are only marked as such in the qcow2 metadata, but contain |
| 144 | # stale data in the external data file |
| 145 | echo |
| 146 | $QEMU_IO -c 'read -P 0 0 1M' \ |
| 147 | -c 'read -P 0x11 1M 1M' \ |
| 148 | -c 'read -P 0x11 4M 1M' \ |
| 149 | -c 'read -P 0 5M 1M' \ |
| 150 | -f raw "$TEST_IMG.data" | |
| 151 | _filter_qemu_io |
| 152 | |
| 153 | |
| 154 | echo -n "qcow2 file size after I/O: " |
| 155 | du -b $TEST_IMG | cut -f1 |
| 156 | |
| 157 | echo |
| 158 | echo "=== Standalone image with external data file (valid raw) ===" |
| 159 | echo |
| 160 | |
| 161 | _make_test_img -o "data_file=$TEST_IMG.data,data_file_raw=on" 64M |
| 162 | |
| 163 | echo -n "qcow2 file size before I/O: " |
| 164 | du -b $TEST_IMG | cut -f1 |
| 165 | |
| 166 | echo |
| 167 | $QEMU_IO -c 'write -P 0x11 1M 4M' \ |
| 168 | -c 'discard 2M 2M' \ |
| 169 | -c 'write -z 3M 3M' \ |
| 170 | -f $IMGFMT "$TEST_IMG" | |
| 171 | _filter_qemu_io |
| 172 | _check_test_img |
| 173 | |
| 174 | echo |
| 175 | $QEMU_IMG map --output=json "$TEST_IMG" |
| 176 | |
| 177 | echo |
| 178 | $QEMU_IO -c 'read -P 0 0 1M' \ |
| 179 | -c 'read -P 0x11 1M 1M' \ |
| 180 | -c 'read -P 0 2M 4M' \ |
| 181 | -f $IMGFMT "$TEST_IMG" | |
| 182 | _filter_qemu_io |
| 183 | |
| 184 | # Discarded clusters are only marked as such in the qcow2 metadata, but |
| 185 | # they can contain stale data in the external data file. Instead, zero |
| 186 | # clusters must be zeroed in the external data file too. |
| 187 | echo |
| 188 | $QEMU_IO -c 'read -P 0 0 1M' \ |
| 189 | -c 'read -P 0x11 1M 1M' \ |
| 190 | -c 'read -P 0 3M 3M' \ |
| 191 | -f raw "$TEST_IMG".data | |
| 192 | _filter_qemu_io |
| 193 | |
| 194 | echo -n "qcow2 file size after I/O: " |
| 195 | du -b $TEST_IMG | cut -f1 |
| 196 | |
| 197 | echo |
| 198 | echo "=== bdrv_co_block_status test for file and offset=0 ===" |
| 199 | echo |
| 200 | |
| 201 | _make_test_img -o "data_file=$TEST_IMG.data" 64M |
| 202 | |
| 203 | $QEMU_IO -c 'write -P 0x11 0 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io |
| 204 | $QEMU_IO -c 'read -P 0x11 0 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io |
| 205 | $QEMU_IMG map --output=human "$TEST_IMG" | _filter_testdir |
| 206 | $QEMU_IMG map --output=json "$TEST_IMG" |
| 207 | |
| 208 | echo |
| 209 | echo "=== Copy offloading ===" |
| 210 | echo |
| 211 | |
| 212 | # Make use of copy offloading if the test host can provide it |
| 213 | _make_test_img -o "data_file=$TEST_IMG.data" 64M |
| 214 | $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n -C "$TEST_IMG.src" "$TEST_IMG" |
| 215 | $QEMU_IMG compare -f $IMGFMT -F $IMGFMT "$TEST_IMG.src" "$TEST_IMG" |
| 216 | |
| 217 | # blkdebug doesn't support copy offloading, so this tests the error path |
| 218 | test_img_with_blkdebug="json:{ |
| 219 | 'driver': 'qcow2', |
| 220 | 'file': { |
| 221 | 'driver': 'file', |
| 222 | 'filename': '$TEST_IMG' |
| 223 | }, |
| 224 | 'data-file': { |
| 225 | 'driver': 'blkdebug', |
| 226 | 'image': { |
| 227 | 'driver': 'file', |
| 228 | 'filename': '$TEST_IMG.data' |
| 229 | } |
| 230 | } |
| 231 | }" |
| 232 | $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n -C "$TEST_IMG.src" "$test_img_with_blkdebug" |
| 233 | $QEMU_IMG compare -f $IMGFMT -F $IMGFMT "$TEST_IMG.src" "$test_img_with_blkdebug" |
| 234 | |
| 235 | echo |
| 236 | echo "=== Flushing should flush the data file ===" |
| 237 | echo |
| 238 | |
| 239 | # We are going to flush a qcow2 file with a blkdebug node inserted |
| 240 | # between the qcow2 node and its data file node. The blkdebug node |
| 241 | # will return an error for all flushes and so we if the data file is |
| 242 | # flushed, we will see qemu-io return an error. |
| 243 | |
| 244 | # We need to write something or the flush will not do anything; we |
| 245 | # also need -t writeback so the write is not done as a FUA write |
| 246 | # (which would then fail thanks to the implicit flush) |
| 247 | $QEMU_IO -c 'write 0 512' -c flush \ |
| 248 | -t writeback \ |
| 249 | "json:{ |
| 250 | 'driver': 'qcow2', |
| 251 | 'file': { |
| 252 | 'driver': 'file', |
| 253 | 'filename': '$TEST_IMG' |
| 254 | }, |
| 255 | 'data-file': { |
| 256 | 'driver': 'blkdebug', |
| 257 | 'inject-error': [{ |
| 258 | 'event': 'none', |
| 259 | 'iotype': 'flush' |
| 260 | }], |
| 261 | 'image': { |
| 262 | 'driver': 'file', |
| 263 | 'filename': '$TEST_IMG.data' |
| 264 | } |
| 265 | } |
| 266 | }" \ |
| 267 | | _filter_qemu_io |
| 268 | |
| 269 | result=${PIPESTATUS[0]} |
| 270 | echo |
| 271 | |
| 272 | case $result in |
| 273 | 0) |
| 274 | echo "ERROR: qemu-io succeeded, so the data file was not flushed" |
| 275 | ;; |
| 276 | 1) |
| 277 | echo "Success: qemu-io failed, so the data file was flushed" |
| 278 | ;; |
| 279 | *) |
| 280 | echo "ERROR: qemu-io returned unknown exit code $result" |
| 281 | ;; |
| 282 | esac |
| 283 | |
| 284 | echo |
| 285 | echo '=== Preallocation with data-file-raw ===' |
| 286 | |
| 287 | echo |
| 288 | echo '--- Using a non-zeroed data file ---' |
| 289 | |
| 290 | # Using data-file-raw must enforce at least metadata preallocation so |
| 291 | # that it does not matter whether one reads the raw file or the qcow2 |
| 292 | # file |
| 293 | |
| 294 | # Pre-create the data file, write some data. Real-world use cases for |
| 295 | # this are adding a qcow2 metadata file to a block device (i.e., using |
| 296 | # the device as the data file) or adding qcow2 features to pre-existing |
| 297 | # raw images (e.g. because the user now wants persistent dirty bitmaps). |
| 298 | truncate -s 1M "$TEST_IMG.data" |
| 299 | $QEMU_IO -f raw -c 'write -P 42 0 1M' "$TEST_IMG.data" | _filter_qemu_io |
| 300 | |
| 301 | # We cannot use qemu-img to create the qcow2 image, because it would |
| 302 | # clear the data file. Use the blockdev-create job instead, which will |
| 303 | # only format the qcow2 image file. |
| 304 | touch "$TEST_IMG" |
| 305 | _launch_qemu \ |
| 306 | -blockdev file,node-name=data,filename="$TEST_IMG.data" \ |
| 307 | -blockdev file,node-name=meta,filename="$TEST_IMG" |
| 308 | |
| 309 | _send_qemu_cmd $QEMU_HANDLE '{ "execute": "qmp_capabilities" }' 'return' |
| 310 | |
| 311 | _send_qemu_cmd $QEMU_HANDLE \ |
| 312 | '{ "execute": "blockdev-create", |
| 313 | "arguments": { |
| 314 | "job-id": "create", |
| 315 | "options": { |
| 316 | "driver": "qcow2", |
| 317 | "size": '"$((1 * 1024 * 1024))"', |
| 318 | "file": "meta", |
| 319 | "data-file": "data", |
| 320 | "data-file-raw": true |
| 321 | } } }' \ |
| 322 | '"status": "concluded"' |
| 323 | |
| 324 | _send_qemu_cmd $QEMU_HANDLE \ |
| 325 | '{ "execute": "job-dismiss", "arguments": { "id": "create" } }' \ |
| 326 | 'return' |
| 327 | |
| 328 | _cleanup_qemu |
| 329 | |
| 330 | echo |
| 331 | echo 'Comparing pattern:' |
| 332 | |
| 333 | # Reading from either the qcow2 file or the data file should return |
| 334 | # the same result: |
| 335 | $QEMU_IO -f raw -c 'read -P 42 0 1M' "$TEST_IMG.data" | _filter_qemu_io |
| 336 | $QEMU_IO -f $IMGFMT -c 'read -P 42 0 1M' "$TEST_IMG" | _filter_qemu_io |
| 337 | |
| 338 | # For good measure |
| 339 | $QEMU_IMG compare -f raw "$TEST_IMG.data" "$TEST_IMG" |
| 340 | |
| 341 | echo |
| 342 | echo '--- Truncation (growing) ---' |
| 343 | |
| 344 | # Append some new data to the raw file, then resize the qcow2 image |
| 345 | # accordingly and see whether the new data is visible. Technically |
| 346 | # that is not allowed, but it is reasonable behavior, so test it. |
| 347 | truncate -s 2M "$TEST_IMG.data" |
| 348 | $QEMU_IO -f raw -c 'write -P 84 1M 1M' "$TEST_IMG.data" | _filter_qemu_io |
| 349 | |
| 350 | $QEMU_IMG resize "$TEST_IMG" 2M |
| 351 | |
| 352 | echo |
| 353 | echo 'Comparing pattern:' |
| 354 | |
| 355 | $QEMU_IO -f raw -c 'read -P 42 0 1M' -c 'read -P 84 1M 1M' "$TEST_IMG.data" \ |
| 356 | | _filter_qemu_io |
| 357 | $QEMU_IO -f $IMGFMT -c 'read -P 42 0 1M' -c 'read -P 84 1M 1M' "$TEST_IMG" \ |
| 358 | | _filter_qemu_io |
| 359 | |
| 360 | $QEMU_IMG compare -f raw "$TEST_IMG.data" "$TEST_IMG" |
| 361 | |
| 362 | echo |
| 363 | echo '--- Giving a backing file at runtime ---' |
| 364 | |
| 365 | # qcow2 files with data-file-raw cannot have backing files given by |
| 366 | # their image header, but qemu will allow you to set a backing node at |
| 367 | # runtime -- it should not have any effect, though (because reading |
| 368 | # from the qcow2 node should return the same data as reading from the |
| 369 | # raw node). |
| 370 | |
| 371 | _make_test_img -o "data_file=$TEST_IMG.data,data_file_raw=on" 1M |
| 372 | TEST_IMG="$TEST_IMG.base" _make_test_img 1M |
| 373 | |
| 374 | # Write something that is not zero into the base image |
| 375 | $QEMU_IO -c 'write -P 42 0 1M' "$TEST_IMG.base" | _filter_qemu_io |
| 376 | |
| 377 | echo |
| 378 | echo 'Comparing qcow2 image and raw data file:' |
| 379 | |
| 380 | # $TEST_IMG and $TEST_IMG.data must show the same data at all times; |
| 381 | # that is, the qcow2 node must not fall through to the backing image |
| 382 | # at any point |
| 383 | $QEMU_IMG compare --image-opts \ |
| 384 | "driver=raw,file.filename=$TEST_IMG.data" \ |
| 385 | "file.filename=$TEST_IMG,backing.file.filename=$TEST_IMG.base" |
| 386 | |
| 387 | echo |
| 388 | echo '=== keep_data_file tests ===' |
| 389 | |
| 390 | echo |
| 391 | echo '--- Creating test data file ---' |
| 392 | |
| 393 | # Easiest way to create the raw data file without having to create and |
| 394 | # access it manually |
| 395 | _make_test_img -o "data_file=$TEST_IMG.data,data_file_raw=on" 1M |
| 396 | # Values chosen by a fair random.org evaluation |
| 397 | $QEMU_IO -c 'write -P 3 0 512k' -c 'write -P 96 512k 512k' "$TEST_IMG" | |
| 398 | _filter_qemu_io |
| 399 | |
| 400 | echo |
| 401 | echo '--- Testing stand-alone option ---' |
| 402 | |
| 403 | # Cannot work, needs data file |
| 404 | _make_test_img -o "keep_data_file=on" 1M |
| 405 | |
| 406 | # Invalid option value |
| 407 | _make_test_img -o "keep_data_file=true" 1M |
| 408 | |
| 409 | # Should be the same as omitting |
| 410 | _make_test_img -o "keep_data_file=off" 1M |
| 411 | |
| 412 | # No preallocation is OK when also specifying data_file_raw; otherwise, none of |
| 413 | # the data file will be mapped, i.e. its contents will stay hidden, so |
| 414 | # requesting its contents to be kept (but hidden) doesn't make much sense. |
| 415 | # |
| 416 | # Metadata preallocation is OK: It will not overwrite the data file's contents, |
| 417 | # but ensure the contents are mapped and visible. |
| 418 | # |
| 419 | # Any data preallocation (like falloc) is not OK, as this would overwrite the |
| 420 | # data file's contents despite keep_data_file requesting they should not be |
| 421 | # overwritten. |
| 422 | # |
| 423 | # Note that all of these cases use the data file created above: This verifies |
| 424 | # that when passing keep_data_file=on, the data file is always kept as-is (and |
| 425 | # e.g. not deleted on error). |
| 426 | for prealloc in off metadata falloc full; do |
| 427 | # Without metadata preallocation, the data_file_raw flag is required so that |
| 428 | # the data file's contents are visible. |
| 429 | for data_file_raw in off on; do |
| 430 | echo |
| 431 | echo "--- Testing prealloc=$prealloc data_file_raw=$data_file_raw ---" |
| 432 | |
| 433 | # Remove previously existing qcow2 (metadata) file |
| 434 | _cleanup_test_img |
| 435 | |
| 436 | opts="data_file=$TEST_IMG.data,keep_data_file=on" |
| 437 | opts+=",preallocation=$prealloc" |
| 438 | opts+=",data_file_raw=$data_file_raw" |
| 439 | |
| 440 | _make_test_img -o "$opts" 1M |
| 441 | if [ -f "$TEST_IMG" ]; then |
| 442 | $QEMU_IO -c 'read -P 3 0 512k' -c 'read -P 96 512k 512k' "$TEST_IMG" | |
| 443 | _filter_qemu_io |
| 444 | fi |
| 445 | done |
| 446 | done |
| 447 | |
| 448 | echo |
| 449 | echo '--- Testing non-existent data file ---' |
| 450 | |
| 451 | # Maybe a matter of taste whether this should fail or create the file, but |
| 452 | # failing is simpler (= will always skip create) and seems safer (users may |
| 453 | # expect the file to exist, and the error will warn them when it does not). |
| 454 | _make_test_img \ |
| 455 | -o "data_file=$TEST_IMG.doesnotexist,keep_data_file=on,data_file_raw=on" \ |
| 456 | 1M |
| 457 | |
| 458 | # success, all done |
| 459 | echo "*** done" |
| 460 | rm -f $seq.full |
| 461 | status=0 |