| 1 | #!/usr/bin/env bash |
| 2 | # group: rw |
| 3 | # |
| 4 | # Test exiting qemu while jobs are still running |
| 5 | # |
| 6 | # Copyright (C) 2017 Red Hat, Inc. |
| 7 | # |
| 8 | # This program is free software; you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation; either version 2 of the License, or |
| 11 | # (at your option) any later version. |
| 12 | # |
| 13 | # This program is distributed in the hope that it will be useful, |
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | # GNU General Public License for more details. |
| 17 | # |
| 18 | # You should have received a copy of the GNU General Public License |
| 19 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 20 | # |
| 21 | |
| 22 | # creator |
| 23 | owner=kwolf@redhat.com |
| 24 | |
| 25 | seq=`basename $0` |
| 26 | echo "QA output created by $seq" |
| 27 | |
| 28 | status=1 # failure is the default! |
| 29 | |
| 30 | _cleanup() |
| 31 | { |
| 32 | _rm_test_img "${TEST_IMG}.mid" |
| 33 | _rm_test_img "${TEST_IMG}.copy" |
| 34 | _cleanup_test_img |
| 35 | _cleanup_qemu |
| 36 | |
| 37 | if [ -f "$TEST_DIR/qsd.pid" ]; then |
| 38 | kill -SIGKILL "$(cat "$TEST_DIR/qsd.pid")" |
| 39 | rm -f "$TEST_DIR/qsd.pid" |
| 40 | fi |
| 41 | rm -f "$SOCK_DIR/qsd.sock" |
| 42 | } |
| 43 | trap "_cleanup; exit \$status" 0 1 2 3 15 |
| 44 | |
| 45 | # get standard environment, filters and checks |
| 46 | . ./common.rc |
| 47 | . ./common.filter |
| 48 | . ./common.qemu |
| 49 | |
| 50 | _supported_fmt qcow2 |
| 51 | _supported_proto file |
| 52 | _supported_os Linux |
| 53 | _require_hmp |
| 54 | _flaky_test https://gitlab.com/qemu-project/qemu/-/issues/3270 |
| 55 | |
| 56 | size=$((64 * 1048576)) |
| 57 | TEST_IMG="${TEST_IMG}.base" _make_test_img $size |
| 58 | |
| 59 | echo |
| 60 | echo === Starting VM === |
| 61 | echo |
| 62 | |
| 63 | qemu_comm_method="qmp" |
| 64 | |
| 65 | _launch_qemu \ |
| 66 | -drive file="${TEST_IMG}.base",cache=$CACHEMODE,aio=$AIOMODE,driver=$IMGFMT,id=disk |
| 67 | h=$QEMU_HANDLE |
| 68 | _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" 'return' |
| 69 | |
| 70 | echo |
| 71 | echo === Creating backing chain === |
| 72 | echo |
| 73 | |
| 74 | _send_qemu_cmd $h \ |
| 75 | "{ 'execute': 'blockdev-snapshot-sync', |
| 76 | 'arguments': { 'device': 'disk', |
| 77 | 'snapshot-file': '$TEST_IMG.mid', |
| 78 | 'format': '$IMGFMT', |
| 79 | 'mode': 'absolute-paths' } }" \ |
| 80 | "return" |
| 81 | |
| 82 | _send_qemu_cmd $h \ |
| 83 | "{ 'execute': 'human-monitor-command', |
| 84 | 'arguments': { 'command-line': |
| 85 | 'qemu-io disk \"write 0 4M\"' } }" \ |
| 86 | "return" |
| 87 | |
| 88 | _send_qemu_cmd $h \ |
| 89 | "{ 'execute': 'blockdev-snapshot-sync', |
| 90 | 'arguments': { 'device': 'disk', |
| 91 | 'snapshot-file': '$TEST_IMG', |
| 92 | 'format': '$IMGFMT', |
| 93 | 'mode': 'absolute-paths' } }" \ |
| 94 | "return" |
| 95 | |
| 96 | echo |
| 97 | echo === Start commit job and exit qemu === |
| 98 | echo |
| 99 | |
| 100 | # Note that the reference output intentionally includes the 'offset' field in |
| 101 | # BLOCK_JOB_* events for all of the following block jobs. They are predictable |
| 102 | # and any change in the offsets would hint at a bug in the job throttling code. |
| 103 | # |
| 104 | # In order to achieve these predictable offsets, all of the following tests |
| 105 | # use speed=65536. Each job will perform exactly one iteration before it has |
| 106 | # to sleep at least for a second, which is plenty of time for the 'quit' QMP |
| 107 | # command to be received (after receiving the command, the rest runs |
| 108 | # synchronously, so jobs can arbitrarily continue or complete). |
| 109 | # |
| 110 | # The buffer size for commit and streaming is 512k (waiting for 8 seconds after |
| 111 | # the first request), for active commit and mirror it's large enough to cover |
| 112 | # the full 4M, and for backup it's the qcow2 cluster size, which we know is |
| 113 | # 64k. As all of these are at least as large as the speed, we are sure that the |
| 114 | # offset advances exactly once before qemu exits. |
| 115 | |
| 116 | _send_qemu_cmd $h \ |
| 117 | "{ 'execute': 'block-commit', |
| 118 | 'arguments': { 'device': 'disk', |
| 119 | 'base':'$TEST_IMG.base', |
| 120 | 'top': '$TEST_IMG.mid', |
| 121 | 'speed': 65536 } }" \ |
| 122 | "return" |
| 123 | |
| 124 | # If we don't sleep here 'quit' command races with disk I/O |
| 125 | sleep 0.5 |
| 126 | |
| 127 | # Ignore the JOB_STATUS_CHANGE events while shutting down the VM. Depending on |
| 128 | # the timing, jobs may or may not transition through a paused state. |
| 129 | _send_qemu_cmd $h "{ 'execute': 'quit' }" "return" |
| 130 | wait=1 _cleanup_qemu | grep -v 'JOB_STATUS_CHANGE' |
| 131 | |
| 132 | echo |
| 133 | echo === Start active commit job and exit qemu === |
| 134 | echo |
| 135 | |
| 136 | _launch_qemu \ |
| 137 | -drive file="${TEST_IMG}",cache=$CACHEMODE,aio=$AIOMODE,driver=$IMGFMT,id=disk |
| 138 | h=$QEMU_HANDLE |
| 139 | _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" 'return' |
| 140 | |
| 141 | _send_qemu_cmd $h \ |
| 142 | "{ 'execute': 'block-commit', |
| 143 | 'arguments': { 'device': 'disk', |
| 144 | 'base':'$TEST_IMG.base', |
| 145 | 'speed': 65536 } }" \ |
| 146 | "return" |
| 147 | |
| 148 | # If we don't sleep here 'quit' command races with disk I/O |
| 149 | sleep 0.5 |
| 150 | |
| 151 | _send_qemu_cmd $h "{ 'execute': 'quit' }" "return" |
| 152 | wait=1 _cleanup_qemu | grep -v 'JOB_STATUS_CHANGE' |
| 153 | |
| 154 | echo |
| 155 | echo === Start mirror job and exit qemu === |
| 156 | echo |
| 157 | |
| 158 | _launch_qemu \ |
| 159 | -drive file="${TEST_IMG}",cache=$CACHEMODE,aio=$AIOMODE,driver=$IMGFMT,id=disk |
| 160 | h=$QEMU_HANDLE |
| 161 | _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" 'return' |
| 162 | |
| 163 | _send_qemu_cmd $h \ |
| 164 | "{ 'execute': 'drive-mirror', |
| 165 | 'arguments': { 'device': 'disk', |
| 166 | 'target': '$TEST_IMG.copy', |
| 167 | 'format': '$IMGFMT', |
| 168 | 'sync': 'full', |
| 169 | 'speed': 65536 } }" \ |
| 170 | "return" |
| 171 | |
| 172 | # If we don't sleep here 'quit' command may be handled before |
| 173 | # the first mirror iteration is done |
| 174 | sleep 0.5 |
| 175 | |
| 176 | _send_qemu_cmd $h "{ 'execute': 'quit' }" "return" |
| 177 | wait=1 _cleanup_qemu | grep -v 'JOB_STATUS_CHANGE' |
| 178 | |
| 179 | echo |
| 180 | echo === Start backup job and exit qemu === |
| 181 | echo |
| 182 | |
| 183 | _launch_qemu \ |
| 184 | -drive file="${TEST_IMG}",cache=$CACHEMODE,aio=$AIOMODE,driver=$IMGFMT,id=disk |
| 185 | h=$QEMU_HANDLE |
| 186 | _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" 'return' |
| 187 | |
| 188 | _send_qemu_cmd $h \ |
| 189 | "{ 'execute': 'drive-backup', |
| 190 | 'arguments': { 'device': 'disk', |
| 191 | 'target': '$TEST_IMG.copy', |
| 192 | 'format': '$IMGFMT', |
| 193 | 'sync': 'full', |
| 194 | 'speed': 65536, |
| 195 | 'x-perf': {'max-chunk': 65536} } }" \ |
| 196 | "return" |
| 197 | |
| 198 | # If we don't sleep here 'quit' command races with disk I/O |
| 199 | sleep 0.5 |
| 200 | |
| 201 | _send_qemu_cmd $h "{ 'execute': 'quit' }" "return" |
| 202 | wait=1 _cleanup_qemu | grep -v 'JOB_STATUS_CHANGE' |
| 203 | |
| 204 | echo |
| 205 | echo === Start streaming job and exit qemu === |
| 206 | echo |
| 207 | |
| 208 | _launch_qemu \ |
| 209 | -drive file="${TEST_IMG}",cache=$CACHEMODE,aio=$AIOMODE,driver=$IMGFMT,id=disk |
| 210 | h=$QEMU_HANDLE |
| 211 | _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" 'return' |
| 212 | |
| 213 | _send_qemu_cmd $h \ |
| 214 | "{ 'execute': 'block-stream', |
| 215 | 'arguments': { 'device': 'disk', |
| 216 | 'speed': 65536 } }" \ |
| 217 | "return" |
| 218 | |
| 219 | # If we don't sleep here 'quit' command races with disk I/O |
| 220 | sleep 0.5 |
| 221 | |
| 222 | _send_qemu_cmd $h "{ 'execute': 'quit' }" "return" |
| 223 | wait=1 _cleanup_qemu | grep -v 'JOB_STATUS_CHANGE' |
| 224 | |
| 225 | _check_test_img |
| 226 | |
| 227 | echo |
| 228 | echo === Start mirror to throttled QSD and exit qemu === |
| 229 | echo |
| 230 | |
| 231 | # Mirror to a throttled QSD instance (so that qemu cannot drain the |
| 232 | # throttling), wait for READY, then write some data to the device, |
| 233 | # and then quit qemu. |
| 234 | # (qemu should force-cancel the job and not wait for the data to be |
| 235 | # written to the target.) |
| 236 | |
| 237 | _make_test_img $size |
| 238 | |
| 239 | # Will be used by this and the next case |
| 240 | set_up_throttled_qsd() { |
| 241 | $QSD \ |
| 242 | --object throttle-group,id=thrgr,limits.bps-total=1048576 \ |
| 243 | --blockdev null-co,node-name=null,size=$size \ |
| 244 | --blockdev throttle,node-name=throttled,throttle-group=thrgr,file=null \ |
| 245 | --nbd-server addr.type=unix,addr.path="$SOCK_DIR/qsd.sock" \ |
| 246 | --export nbd,id=exp,node-name=throttled,name=target,writable=true \ |
| 247 | --pidfile "$TEST_DIR/qsd.pid" \ |
| 248 | --daemonize |
| 249 | } |
| 250 | |
| 251 | set_up_throttled_qsd |
| 252 | |
| 253 | # Need a virtio-blk device so that qemu-io writes will not block the monitor |
| 254 | _launch_qemu \ |
| 255 | --blockdev file,node-name=source-proto,filename="$TEST_IMG" \ |
| 256 | --blockdev qcow2,node-name=source-fmt,file=source-proto \ |
| 257 | --device virtio-blk,id=vblk,drive=source-fmt \ |
| 258 | --blockdev "{\"driver\": \"nbd\", |
| 259 | \"node-name\": \"target\", |
| 260 | \"server\": { |
| 261 | \"type\": \"unix\", |
| 262 | \"path\": \"$SOCK_DIR/qsd.sock\" |
| 263 | }, |
| 264 | \"export\": \"target\"}" |
| 265 | |
| 266 | h=$QEMU_HANDLE |
| 267 | _send_qemu_cmd $h '{"execute": "qmp_capabilities"}' 'return' |
| 268 | |
| 269 | # Use sync=top, so the first pass will not copy the whole image |
| 270 | _send_qemu_cmd $h \ |
| 271 | '{"execute": "blockdev-mirror", |
| 272 | "arguments": { |
| 273 | "job-id": "mirror", |
| 274 | "device": "source-fmt", |
| 275 | "target": "target", |
| 276 | "sync": "top" |
| 277 | }}' \ |
| 278 | 'return' \ |
| 279 | | grep -v JOB_STATUS_CHANGE # Ignore these events during creation |
| 280 | |
| 281 | # This too will be used by this and the next case |
| 282 | # $1: QEMU handle |
| 283 | # $2: Image size |
| 284 | wait_for_job_and_quit() { |
| 285 | h=$1 |
| 286 | size=$2 |
| 287 | |
| 288 | # List of expected events |
| 289 | capture_events='BLOCK_JOB_READY JOB_STATUS_CHANGE' |
| 290 | _wait_event $h 'BLOCK_JOB_READY' |
| 291 | QEMU_EVENTS= # Ignore all JOB_STATUS_CHANGE events that came before READY |
| 292 | |
| 293 | # Write something to the device for post-READY mirroring. Write it in |
| 294 | # blocks matching the cluster size, each spaced one block apart, so |
| 295 | # that the mirror job will have to spawn one request per cluster. |
| 296 | # Because the number of concurrent requests is limited (to 16), this |
| 297 | # limits the number of bytes concurrently in flight, which speeds up |
| 298 | # cancelling the job (in-flight requests still are waited for). |
| 299 | # To limit the number of bytes in flight, we could alternatively pass |
| 300 | # something for blockdev-mirror's @buf-size parameter, but |
| 301 | # block-commit does not have such a parameter, so we need to figure |
| 302 | # something out that works for both. |
| 303 | |
| 304 | cluster_size=65536 |
| 305 | step=$((cluster_size * 2)) |
| 306 | |
| 307 | echo '--- Writing data to the virtio-blk device ---' |
| 308 | |
| 309 | for ofs in $(seq 0 $step $((size - step))); do |
| 310 | qemu_io_cmd="qemu-io -d vblk/virtio-backend " |
| 311 | qemu_io_cmd+="\\\"aio_write $ofs $cluster_size\\\"" |
| 312 | |
| 313 | # Do not include these requests in the reference output |
| 314 | # (it's just too much) |
| 315 | silent=yes _send_qemu_cmd $h \ |
| 316 | "{\"execute\": \"human-monitor-command\", |
| 317 | \"arguments\": { |
| 318 | \"command-line\": \"$qemu_io_cmd\" |
| 319 | }}" \ |
| 320 | 'return' |
| 321 | done |
| 322 | |
| 323 | # Wait until the job's length is updated to reflect the write requests |
| 324 | |
| 325 | # We have written to half of the device, so this is the expected job length |
| 326 | final_len=$((size / 2)) |
| 327 | timeout=100 # unit: 0.1 seconds |
| 328 | while true; do |
| 329 | len=$( |
| 330 | _send_qemu_cmd $h \ |
| 331 | '{"execute": "query-block-jobs"}' \ |
| 332 | 'return.*"len": [0-9]\+' \ |
| 333 | | grep 'return.*"len": [0-9]\+' \ |
| 334 | | sed -e 's/.*"len": \([0-9]\+\).*/\1/' |
| 335 | ) |
| 336 | if [ "$len" -eq "$final_len" ]; then |
| 337 | break |
| 338 | fi |
| 339 | timeout=$((timeout - 1)) |
| 340 | if [ "$timeout" -eq 0 ]; then |
| 341 | echo "ERROR: Timeout waiting for job to reach len=$final_len" |
| 342 | break |
| 343 | fi |
| 344 | sleep 0.1 |
| 345 | done |
| 346 | |
| 347 | sleep 1 |
| 348 | |
| 349 | # List of expected events |
| 350 | capture_events='BLOCK_JOB_CANCELLED JOB_STATUS_CHANGE SHUTDOWN' |
| 351 | |
| 352 | _send_qemu_cmd $h \ |
| 353 | '{"execute": "quit"}' \ |
| 354 | 'return' |
| 355 | |
| 356 | _wait_event $h 'SHUTDOWN' |
| 357 | _wait_event $h 'JOB_STATUS_CHANGE' # standby |
| 358 | _wait_event $h 'JOB_STATUS_CHANGE' # ready |
| 359 | _wait_event $h 'JOB_STATUS_CHANGE' # standby |
| 360 | _wait_event $h 'JOB_STATUS_CHANGE' # ready |
| 361 | _wait_event $h 'JOB_STATUS_CHANGE' # aborting |
| 362 | # Filter the offset (depends on when exactly `quit` was issued) |
| 363 | _wait_event $h 'BLOCK_JOB_CANCELLED' \ |
| 364 | | sed -e 's/"offset": [0-9]\+/"offset": (filtered)/' |
| 365 | _wait_event $h 'JOB_STATUS_CHANGE' # concluded |
| 366 | _wait_event $h 'JOB_STATUS_CHANGE' # null |
| 367 | |
| 368 | wait=yes _cleanup_qemu |
| 369 | |
| 370 | kill -SIGTERM "$(cat "$TEST_DIR/qsd.pid")" |
| 371 | } |
| 372 | |
| 373 | wait_for_job_and_quit $h $size |
| 374 | |
| 375 | echo |
| 376 | echo === Start active commit to throttled QSD and exit qemu === |
| 377 | echo |
| 378 | |
| 379 | # Same as the above, but instead of mirroring, do an active commit |
| 380 | |
| 381 | _make_test_img $size |
| 382 | |
| 383 | set_up_throttled_qsd |
| 384 | |
| 385 | _launch_qemu \ |
| 386 | --blockdev "{\"driver\": \"nbd\", |
| 387 | \"node-name\": \"target\", |
| 388 | \"server\": { |
| 389 | \"type\": \"unix\", |
| 390 | \"path\": \"$SOCK_DIR/qsd.sock\" |
| 391 | }, |
| 392 | \"export\": \"target\"}" \ |
| 393 | --blockdev file,node-name=source-proto,filename="$TEST_IMG" \ |
| 394 | --blockdev qcow2,node-name=source-fmt,file=source-proto,backing=target \ |
| 395 | --device virtio-blk,id=vblk,drive=source-fmt |
| 396 | |
| 397 | h=$QEMU_HANDLE |
| 398 | _send_qemu_cmd $h '{"execute": "qmp_capabilities"}' 'return' |
| 399 | |
| 400 | _send_qemu_cmd $h \ |
| 401 | '{"execute": "block-commit", |
| 402 | "arguments": { |
| 403 | "job-id": "commit", |
| 404 | "device": "source-fmt" |
| 405 | }}' \ |
| 406 | 'return' \ |
| 407 | | grep -v JOB_STATUS_CHANGE # Ignore these events during creation |
| 408 | |
| 409 | wait_for_job_and_quit $h $size |
| 410 | |
| 411 | # success, all done |
| 412 | echo "*** done" |
| 413 | rm -f $seq.full |
| 414 | status=0 |