| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Copyright (c) 2017 Red Hat Inc |
| 4 | # |
| 5 | # Author: |
| 6 | # Eduardo Habkost <ehabkost@redhat.com> |
| 7 | # |
| 8 | # This program is free software; you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation; either version 2 of the License, or |
| 11 | # (at your option) any later version. |
| 12 | # |
| 13 | # This program is distributed in the hope that it will be useful, |
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | # GNU General Public License for more details. |
| 17 | # |
| 18 | # You should have received a copy of the GNU General Public License along |
| 19 | # with this program; if not, see <https://www.gnu.org/licenses/>. |
| 20 | |
| 21 | """ |
| 22 | Run QEMU with all combinations of -machine and -device types, |
| 23 | check for crashes and unexpected errors. |
| 24 | """ |
| 25 | |
| 26 | import os |
| 27 | import sys |
| 28 | import glob |
| 29 | import logging |
| 30 | import traceback |
| 31 | import re |
| 32 | import random |
| 33 | import argparse |
| 34 | from itertools import chain |
| 35 | from pathlib import Path |
| 36 | |
| 37 | try: |
| 38 | from qemu.machine import QEMUMachine |
| 39 | from qemu.qmp import ConnectError |
| 40 | except ModuleNotFoundError as exc: |
| 41 | path = Path(__file__).resolve() |
| 42 | print(f"Module '{exc.name}' not found.") |
| 43 | print(" Try running this script like so:") |
| 44 | print(f' > $builddir/run "{path}"') |
| 45 | sys.exit(1) |
| 46 | |
| 47 | logger = logging.getLogger('device-crash-test') |
| 48 | dbg = logger.debug |
| 49 | |
| 50 | |
| 51 | # Purposes of the following rule list: |
| 52 | # * Avoiding verbose log messages when we find known non-fatal |
| 53 | # (exitcode=1) errors |
| 54 | # * Avoiding fatal errors when we find known crashes |
| 55 | # * Skipping machines/devices that are known not to work out of |
| 56 | # the box, when running in --quick mode |
| 57 | # |
| 58 | # Keeping the rule list updated is desirable, but not required, |
| 59 | # because unexpected cases where QEMU exits with exitcode=1 will |
| 60 | # just trigger a INFO message. |
| 61 | |
| 62 | # Valid error rule keys: |
| 63 | # * accel: regexp, full match only |
| 64 | # * machine: regexp, full match only |
| 65 | # * device: regexp, full match only |
| 66 | # * log: regexp, partial match allowed |
| 67 | # * exitcode: if not present, defaults to 1. If None, matches any exitcode |
| 68 | # * warn: if True, matching failures will be logged as warnings |
| 69 | # * expected: if True, QEMU is expected to always fail every time |
| 70 | # when testing the corresponding test case |
| 71 | # * loglevel: log level of log output when there's a match. |
| 72 | ERROR_RULE_LIST = [ |
| 73 | # Machines that won't work out of the box: |
| 74 | # MACHINE | ERROR MESSAGE |
| 75 | {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara |
| 76 | {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument |
| 77 | {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null) |
| 78 | |
| 79 | # devices that don't work out of the box because they require extra options to "-device DEV": |
| 80 | # DEVICE | ERROR MESSAGE |
| 81 | {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set |
| 82 | {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found |
| 83 | {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found |
| 84 | # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True |
| 85 | {'device':'ide-cd'}, # No drive specified |
| 86 | {'device':'ide-hd', 'expected':True}, # No drive specified |
| 87 | {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute |
| 88 | {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device |
| 89 | {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set |
| 90 | {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set |
| 91 | {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device |
| 92 | {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev' |
| 93 | {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev' |
| 94 | {'device':'loader', 'expected':True}, # please include valid arguments |
| 95 | {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set |
| 96 | {'device':'nvme', 'expected':True}, # Device initialization failed |
| 97 | {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set |
| 98 | {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. |
| 99 | {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. |
| 100 | {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. |
| 101 | {'device':'pxb-cxl', 'expected':True}, # pxb-cxl devices cannot reside on a PCI bus. |
| 102 | {'device':'scsi-block', 'expected':True}, # drive property not set |
| 103 | {'device':'scsi-generic', 'expected':True}, # drive property not set |
| 104 | {'device':'scsi-hd', 'expected':True}, # drive property not set |
| 105 | {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB |
| 106 | {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend! |
| 107 | {'device':'spapr-vty', 'expected':True}, # chardev property not set |
| 108 | {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found |
| 109 | {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero |
| 110 | {'device':'usb-braille', 'expected':True}, # Property chardev is required |
| 111 | {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured |
| 112 | {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing |
| 113 | {'device':'usb-serial', 'expected':True}, # Property chardev is required |
| 114 | {'device':'usb-storage', 'expected':True}, # drive property not set |
| 115 | {'device':'vfio-pci', 'expected':True}, # No provided host device |
| 116 | {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0 |
| 117 | {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn |
| 118 | {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2 |
| 119 | {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2 |
| 120 | {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL |
| 121 | {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL |
| 122 | {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL |
| 123 | {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set |
| 124 | {'device':'virtio-blk-device', 'expected':True}, # drive property not set |
| 125 | {'device':'virtio-blk-device', 'expected':True}, # drive property not set |
| 126 | {'device':'virtio-blk-pci', 'expected':True}, # drive property not set |
| 127 | {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object |
| 128 | {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object |
| 129 | {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object |
| 130 | {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required |
| 131 | {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required |
| 132 | {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied |
| 133 | {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2 |
| 134 | {'device':'zpci', 'expected':True}, # target must be defined |
| 135 | {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found |
| 136 | {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found |
| 137 | |
| 138 | # ioapic devices are already created by pc and will fail: |
| 139 | {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed |
| 140 | {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed |
| 141 | |
| 142 | # "spapr-cpu-core needs a pseries machine" |
| 143 | {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True}, |
| 144 | |
| 145 | # KVM-specific devices shouldn't be tried without accel=kvm: |
| 146 | {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True}, |
| 147 | |
| 148 | # xen-specific machines and devices: |
| 149 | {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True}, |
| 150 | {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True}, |
| 151 | |
| 152 | # this fails on some machine-types, but not all, so they don't have expected=True: |
| 153 | {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide |
| 154 | |
| 155 | # Silence INFO messages for errors that are common on multiple |
| 156 | # devices/machines: |
| 157 | {'log':r"No '[\w-]+' bus found for device '[\w-]+'"}, |
| 158 | {'log':r"(Guest|ROM|Flash|Kernel) image must be specified"}, |
| 159 | {'log':r"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"}, |
| 160 | {'log':r"Couldn't find rom image '[\w-]+\.bin'"}, |
| 161 | {'log':r"speed mismatch trying to attach usb device"}, |
| 162 | {'log':r"Can't create a second ISA bus"}, |
| 163 | {'log':r"duplicate fw_cfg file name"}, |
| 164 | # sysbus-related error messages: most machines reject most dynamic sysbus devices: |
| 165 | {'log':r"Device [\w.,-]+ is not supported by this machine yet"}, |
| 166 | {'log':r"Device [\w.,-]+ can not be dynamically instantiated"}, |
| 167 | {'log':r"Platform Bus: Can not fit MMIO region of size "}, |
| 168 | # other more specific errors we will ignore: |
| 169 | {'device':'.*-spapr-cpu-core', 'log':r"CPU core type should be"}, |
| 170 | {'log':r"MSI(-X)? is not supported by interrupt controller"}, |
| 171 | {'log':r"pxb-pcie? devices cannot reside on a PCIe? bus"}, |
| 172 | {'log':r"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"}, |
| 173 | {'log':r"This CPU requires a smaller page size than the system is using"}, |
| 174 | {'log':r"MSI-X support is mandatory in the S390 architecture"}, |
| 175 | {'log':r"Unable to initialize GIC, CPUState for CPU#0 not valid"}, |
| 176 | {'log':r"Multiple VT220 operator consoles are not supported"}, |
| 177 | {'log':r"core 0 already populated"}, |
| 178 | {'log':r"could not find stage1 bootloader"}, |
| 179 | |
| 180 | # other exitcode=1 failures not listed above will just generate INFO messages: |
| 181 | {'exitcode':1, 'loglevel':logging.INFO}, |
| 182 | |
| 183 | # everything else (including SIGABRT and SIGSEGV) will be a fatal error: |
| 184 | {'exitcode':None, 'fatal':True, 'loglevel':logging.FATAL}, |
| 185 | ] |
| 186 | |
| 187 | |
| 188 | def errorRuleTestCaseMatch(rule, t): |
| 189 | """Check if a test case specification can match a error rule |
| 190 | |
| 191 | This only checks if a error rule is a candidate match |
| 192 | for a given test case, it won't check if the test case |
| 193 | results/output match the rule. See ruleListResultMatch(). |
| 194 | """ |
| 195 | return (('machine' not in rule or |
| 196 | 'machine' not in t or |
| 197 | re.match(rule['machine'] + '$', t['machine'])) and |
| 198 | ('accel' not in rule or |
| 199 | 'accel' not in t or |
| 200 | re.match(rule['accel'] + '$', t['accel'])) and |
| 201 | ('device' not in rule or |
| 202 | 'device' not in t or |
| 203 | re.match(rule['device'] + '$', t['device']))) |
| 204 | |
| 205 | |
| 206 | def ruleListCandidates(t): |
| 207 | """Generate the list of candidates that can match a test case""" |
| 208 | for i, rule in enumerate(ERROR_RULE_LIST): |
| 209 | if errorRuleTestCaseMatch(rule, t): |
| 210 | yield (i, rule) |
| 211 | |
| 212 | |
| 213 | def findExpectedResult(t): |
| 214 | """Check if there's an expected=True error rule for a test case |
| 215 | |
| 216 | Returns (i, rule) tuple, where i is the index in |
| 217 | ERROR_RULE_LIST and rule is the error rule itself. |
| 218 | """ |
| 219 | for i, rule in ruleListCandidates(t): |
| 220 | if rule.get('expected'): |
| 221 | return (i, rule) |
| 222 | |
| 223 | |
| 224 | def ruleListResultMatch(rule, r): |
| 225 | """Check if test case results/output match a error rule |
| 226 | |
| 227 | It is valid to call this function only if |
| 228 | errorRuleTestCaseMatch() is True for the rule (e.g. on |
| 229 | rules returned by ruleListCandidates()) |
| 230 | """ |
| 231 | assert errorRuleTestCaseMatch(rule, r['testcase']) |
| 232 | return ((rule.get('exitcode', 1) is None or |
| 233 | r['exitcode'] == rule.get('exitcode', 1)) and |
| 234 | ('log' not in rule or |
| 235 | re.search(rule['log'], r['log'], re.MULTILINE))) |
| 236 | |
| 237 | |
| 238 | def checkResultRuleList(r): |
| 239 | """Look up error rule for a given test case result |
| 240 | |
| 241 | Returns (i, rule) tuple, where i is the index in |
| 242 | ERROR_RULE_LIST and rule is the error rule itself. |
| 243 | """ |
| 244 | for i, rule in ruleListCandidates(r['testcase']): |
| 245 | if ruleListResultMatch(rule, r): |
| 246 | return i, rule |
| 247 | |
| 248 | raise Exception("this should never happen") |
| 249 | |
| 250 | |
| 251 | def qemuOptsEscape(s): |
| 252 | """Escape option value QemuOpts""" |
| 253 | return s.replace(",", ",,") |
| 254 | |
| 255 | |
| 256 | def formatTestCase(t): |
| 257 | """Format test case info as "key=value key=value" for prettier logging output""" |
| 258 | return ' '.join('%s=%s' % (k, v) for k, v in t.items()) |
| 259 | |
| 260 | |
| 261 | def qomListTypeNames(vm, **kwargs): |
| 262 | """Run qom-list-types QMP command, return type names""" |
| 263 | types = vm.cmd('qom-list-types', **kwargs) |
| 264 | return [t['name'] for t in types] |
| 265 | |
| 266 | |
| 267 | def infoQDM(vm): |
| 268 | """Parse 'info qdm' output""" |
| 269 | args = {'command-line': 'info qdm'} |
| 270 | devhelp = vm.cmd('human-monitor-command', **args) |
| 271 | for l in devhelp.split('\n'): |
| 272 | l = l.strip() |
| 273 | if l == '' or l.endswith(':'): |
| 274 | continue |
| 275 | d = {'name': re.search(r'name "([^"]+)"', l).group(1), |
| 276 | 'no-user': (re.search(', no-user', l) is not None)} |
| 277 | yield d |
| 278 | |
| 279 | |
| 280 | class QemuBinaryInfo(object): |
| 281 | def __init__(self, binary, devtype): |
| 282 | if devtype is None: |
| 283 | devtype = 'device' |
| 284 | |
| 285 | self.binary = binary |
| 286 | self._machine_info = {} |
| 287 | |
| 288 | dbg("devtype: %r", devtype) |
| 289 | args = ['-S', '-machine', 'none,accel=kvm:tcg'] |
| 290 | dbg("querying info for QEMU binary: %s", binary) |
| 291 | vm = QEMUMachine(binary=binary, args=args) |
| 292 | vm.launch() |
| 293 | try: |
| 294 | self.alldevs = set(qomListTypeNames(vm, implements=devtype, abstract=False)) |
| 295 | # there's no way to query DeviceClass::user_creatable using QMP, |
| 296 | # so use 'info qdm': |
| 297 | self.no_user_devs = set([d['name'] for d in infoQDM(vm, ) if d['no-user']]) |
| 298 | self.machines = list(m['name'] for m in vm.cmd('query-machines')) |
| 299 | self.user_devs = self.alldevs.difference(self.no_user_devs) |
| 300 | self.kvm_available = vm.cmd('query-kvm')['enabled'] |
| 301 | finally: |
| 302 | vm.shutdown() |
| 303 | |
| 304 | def machineInfo(self, machine): |
| 305 | """Query for information on a specific machine-type |
| 306 | |
| 307 | Results are cached internally, in case the same machine- |
| 308 | type is queried multiple times. |
| 309 | """ |
| 310 | if machine in self._machine_info: |
| 311 | return self._machine_info[machine] |
| 312 | |
| 313 | mi = {} |
| 314 | args = ['-S', '-machine', '%s' % (machine)] |
| 315 | dbg("querying machine info for binary=%s machine=%s", self.binary, machine) |
| 316 | vm = QEMUMachine(binary=self.binary, args=args) |
| 317 | try: |
| 318 | vm.launch() |
| 319 | mi['runnable'] = True |
| 320 | except Exception: |
| 321 | dbg("exception trying to run binary=%s machine=%s", self.binary, machine, exc_info=sys.exc_info()) |
| 322 | dbg("log: %r", vm.get_log()) |
| 323 | mi['runnable'] = False |
| 324 | |
| 325 | vm.shutdown() |
| 326 | self._machine_info[machine] = mi |
| 327 | return mi |
| 328 | |
| 329 | |
| 330 | BINARY_INFO = {} |
| 331 | |
| 332 | |
| 333 | def getBinaryInfo(args, binary): |
| 334 | if binary not in BINARY_INFO: |
| 335 | BINARY_INFO[binary] = QemuBinaryInfo(binary, args.devtype) |
| 336 | return BINARY_INFO[binary] |
| 337 | |
| 338 | |
| 339 | def checkOneCase(args, testcase): |
| 340 | """Check one specific case |
| 341 | |
| 342 | Returns a dictionary containing failure information on error, |
| 343 | or None on success |
| 344 | """ |
| 345 | binary = testcase['binary'] |
| 346 | accel = testcase['accel'] |
| 347 | machine = testcase['machine'] |
| 348 | device = testcase['device'] |
| 349 | |
| 350 | dbg("will test: %r", testcase) |
| 351 | |
| 352 | args = ['-S', '-machine', '%s,accel=%s' % (machine, accel), |
| 353 | '-device', qemuOptsEscape(device)] |
| 354 | cmdline = ' '.join([binary] + args) |
| 355 | dbg("will launch QEMU: %s", cmdline) |
| 356 | vm = QEMUMachine(binary=binary, args=args, qmp_timer=15) |
| 357 | |
| 358 | exc = None |
| 359 | exc_traceback = None |
| 360 | try: |
| 361 | vm.launch() |
| 362 | except Exception as this_exc: |
| 363 | exc = this_exc |
| 364 | exc_traceback = traceback.format_exc() |
| 365 | dbg("Exception while running test case") |
| 366 | finally: |
| 367 | vm.shutdown() |
| 368 | ec = vm.exitcode() |
| 369 | log = vm.get_log() |
| 370 | |
| 371 | if exc is not None or ec != 0: |
| 372 | return {'exc': exc, |
| 373 | 'exc_traceback':exc_traceback, |
| 374 | 'exitcode':ec, |
| 375 | 'log':log, |
| 376 | 'testcase':testcase, |
| 377 | 'cmdline':cmdline} |
| 378 | |
| 379 | |
| 380 | def binariesToTest(args, testcase): |
| 381 | if args.qemu: |
| 382 | r = args.qemu |
| 383 | else: |
| 384 | r = [f.path for f in os.scandir('.') |
| 385 | if f.name.startswith('qemu-system-') and |
| 386 | f.is_file() and os.access(f, os.X_OK)] |
| 387 | return r |
| 388 | |
| 389 | |
| 390 | def accelsToTest(args, testcase): |
| 391 | if getBinaryInfo(args, testcase['binary']).kvm_available and not args.tcg_only: |
| 392 | yield 'kvm' |
| 393 | yield 'tcg' |
| 394 | |
| 395 | |
| 396 | def machinesToTest(args, testcase): |
| 397 | return getBinaryInfo(args, testcase['binary']).machines |
| 398 | |
| 399 | |
| 400 | def devicesToTest(args, testcase): |
| 401 | return getBinaryInfo(args, testcase['binary']).user_devs |
| 402 | |
| 403 | |
| 404 | TESTCASE_VARIABLES = [ |
| 405 | ('binary', binariesToTest), |
| 406 | ('accel', accelsToTest), |
| 407 | ('machine', machinesToTest), |
| 408 | ('device', devicesToTest), |
| 409 | ] |
| 410 | |
| 411 | |
| 412 | def genCases1(args, testcases, var, fn): |
| 413 | """Generate new testcases for one variable |
| 414 | |
| 415 | If an existing item already has a variable set, don't |
| 416 | generate new items and just return it directly. This |
| 417 | allows the "-t" command-line option to be used to choose |
| 418 | a specific test case. |
| 419 | """ |
| 420 | for testcase in testcases: |
| 421 | if var in testcase: |
| 422 | yield testcase.copy() |
| 423 | else: |
| 424 | for i in fn(args, testcase): |
| 425 | t = testcase.copy() |
| 426 | t[var] = i |
| 427 | yield t |
| 428 | |
| 429 | |
| 430 | def genCases(args, testcase): |
| 431 | """Generate test cases for all variables |
| 432 | """ |
| 433 | cases = [testcase.copy()] |
| 434 | for var, fn in TESTCASE_VARIABLES: |
| 435 | dbg("var: %r, fn: %r", var, fn) |
| 436 | cases = genCases1(args, cases, var, fn) |
| 437 | return cases |
| 438 | |
| 439 | |
| 440 | def casesToTest(args, testcase): |
| 441 | cases = genCases(args, testcase) |
| 442 | if args.random: |
| 443 | cases = list(cases) |
| 444 | cases = random.sample(cases, min(args.random, len(cases))) |
| 445 | if args.debug: |
| 446 | cases = list(cases) |
| 447 | dbg("%d test cases to test", len(cases)) |
| 448 | if args.shuffle: |
| 449 | cases = list(cases) |
| 450 | random.shuffle(cases) |
| 451 | return cases |
| 452 | |
| 453 | |
| 454 | def logFailure(f, level): |
| 455 | t = f['testcase'] |
| 456 | logger.log(level, "failed: %s", formatTestCase(t)) |
| 457 | logger.log(level, "cmdline: %s", f['cmdline']) |
| 458 | for l in f['log'].strip().split('\n'): |
| 459 | logger.log(level, "log: %s", l) |
| 460 | logger.log(level, "exit code: %r", f['exitcode']) |
| 461 | |
| 462 | # If the Exception is merely a QMP connect error, |
| 463 | # reduce the logging level for its traceback to |
| 464 | # improve visual clarity. |
| 465 | if isinstance(f.get('exc'), ConnectError): |
| 466 | logger.log(level, "%s.%s: %s", |
| 467 | type(f['exc']).__module__, |
| 468 | type(f['exc']).__qualname__, |
| 469 | str(f['exc'])) |
| 470 | level = logging.DEBUG |
| 471 | |
| 472 | if f['exc_traceback']: |
| 473 | logger.log(level, "exception:") |
| 474 | for l in f['exc_traceback'].split('\n'): |
| 475 | logger.log(level, " %s", l.rstrip('\n')) |
| 476 | |
| 477 | |
| 478 | def main(): |
| 479 | parser = argparse.ArgumentParser(description="QEMU -device crash test") |
| 480 | parser.add_argument('-t', metavar='KEY=VALUE', nargs='*', |
| 481 | help="Limit test cases to KEY=VALUE", |
| 482 | action='append', dest='testcases', default=[]) |
| 483 | parser.add_argument('-d', '--debug', action='store_true', |
| 484 | help='debug output') |
| 485 | parser.add_argument('-v', '--verbose', action='store_true', default=True, |
| 486 | help='verbose output') |
| 487 | parser.add_argument('-q', '--quiet', dest='verbose', action='store_false', |
| 488 | help='non-verbose output') |
| 489 | parser.add_argument('-r', '--random', type=int, metavar='COUNT', |
| 490 | help='run a random sample of COUNT test cases', |
| 491 | default=0) |
| 492 | parser.add_argument('--shuffle', action='store_true', |
| 493 | help='Run test cases in random order') |
| 494 | parser.add_argument('--dry-run', action='store_true', |
| 495 | help="Don't run any tests, just generate list") |
| 496 | parser.add_argument('-D', '--devtype', metavar='TYPE', |
| 497 | help="Test only device types that implement TYPE") |
| 498 | parser.add_argument('-Q', '--quick', action='store_true', default=True, |
| 499 | help="Quick mode: skip test cases that are expected to fail") |
| 500 | parser.add_argument('-F', '--full', action='store_false', dest='quick', |
| 501 | help="Full mode: test cases that are expected to fail") |
| 502 | parser.add_argument('--strict', action='store_true', dest='strict', |
| 503 | help="Treat all warnings as fatal") |
| 504 | parser.add_argument('--tcg-only', action='store_true', dest='tcg_only', |
| 505 | help="Only test with TCG accelerator") |
| 506 | parser.add_argument('qemu', nargs='*', metavar='QEMU', |
| 507 | help='QEMU binary to run') |
| 508 | args = parser.parse_args() |
| 509 | |
| 510 | if args.debug: |
| 511 | lvl = logging.DEBUG |
| 512 | elif args.verbose: |
| 513 | lvl = logging.INFO |
| 514 | else: |
| 515 | lvl = logging.WARN |
| 516 | logging.basicConfig(stream=sys.stdout, level=lvl, format='%(levelname)s: %(message)s') |
| 517 | |
| 518 | if not args.debug: |
| 519 | # Async QMP, when in use, is chatty about connection failures. |
| 520 | # This script knowingly generates a ton of connection errors. |
| 521 | # Silence this logger. |
| 522 | logging.getLogger('qemu.qmp.protocol').setLevel(logging.CRITICAL) |
| 523 | |
| 524 | fatal_failures = [] |
| 525 | wl_stats = {} |
| 526 | skipped = 0 |
| 527 | total = 0 |
| 528 | |
| 529 | tc = {} |
| 530 | dbg("testcases: %r", args.testcases) |
| 531 | if args.testcases: |
| 532 | for t in chain(*args.testcases): |
| 533 | for kv in t.split(): |
| 534 | k, v = kv.split('=', 1) |
| 535 | tc[k] = v |
| 536 | |
| 537 | if len(binariesToTest(args, tc)) == 0: |
| 538 | print("No QEMU binary found", file=sys.stderr) |
| 539 | parser.print_usage(sys.stderr) |
| 540 | return 1 |
| 541 | |
| 542 | for t in casesToTest(args, tc): |
| 543 | logger.info("running test case: %s", formatTestCase(t)) |
| 544 | total += 1 |
| 545 | |
| 546 | expected_match = findExpectedResult(t) |
| 547 | if (args.quick and |
| 548 | (expected_match or |
| 549 | not getBinaryInfo(args, t['binary']).machineInfo(t['machine'])['runnable'])): |
| 550 | dbg("skipped: %s", formatTestCase(t)) |
| 551 | skipped += 1 |
| 552 | continue |
| 553 | |
| 554 | if args.dry_run: |
| 555 | continue |
| 556 | |
| 557 | try: |
| 558 | f = checkOneCase(args, t) |
| 559 | except KeyboardInterrupt: |
| 560 | break |
| 561 | |
| 562 | if f: |
| 563 | i, rule = checkResultRuleList(f) |
| 564 | dbg("testcase: %r, rule list match: %r", t, rule) |
| 565 | wl_stats.setdefault(i, []).append(f) |
| 566 | level = rule.get('loglevel', logging.DEBUG) |
| 567 | logFailure(f, level) |
| 568 | if rule.get('fatal') or (args.strict and level >= logging.WARN): |
| 569 | fatal_failures.append(f) |
| 570 | else: |
| 571 | dbg("success: %s", formatTestCase(t)) |
| 572 | if expected_match: |
| 573 | logger.warn("Didn't fail as expected: %s", formatTestCase(t)) |
| 574 | |
| 575 | logger.info("Total: %d test cases", total) |
| 576 | if skipped: |
| 577 | logger.info("Skipped %d test cases", skipped) |
| 578 | |
| 579 | if args.debug: |
| 580 | stats = sorted([(len(wl_stats.get(i, [])), rule) for i, rule in |
| 581 | enumerate(ERROR_RULE_LIST)], key=lambda x: x[0]) |
| 582 | for count, rule in stats: |
| 583 | dbg("error rule stats: %d: %r", count, rule) |
| 584 | |
| 585 | if fatal_failures: |
| 586 | for f in fatal_failures: |
| 587 | t = f['testcase'] |
| 588 | logger.error("Fatal failure: %s", formatTestCase(t)) |
| 589 | logger.error("Fatal failures on some machine/device combinations") |
| 590 | return 1 |
| 591 | |
| 592 | if __name__ == '__main__': |
| 593 | sys.exit(main()) |