| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Functional test that runs the Arm VBSA conformance tests. |
| 4 | # |
| 5 | # Copyright (c) 2026 Linaro Ltd. |
| 6 | # |
| 7 | # Author: |
| 8 | # Alex Bennée <alex.bennee@linaro.org> |
| 9 | # |
| 10 | # SPDX-License-Identifier: GPL-2.0-or-later |
| 11 | |
| 12 | import shutil |
| 13 | import re |
| 14 | from subprocess import check_call, check_output, DEVNULL |
| 15 | |
| 16 | from qemu_test import QemuSystemTest, Asset |
| 17 | from qemu_test import get_qemu_img, skipIfMissingCommands |
| 18 | from qemu_test import wait_for_console_pattern |
| 19 | from qemu_test import exec_command_and_wait_for_pattern as ec_and_wait |
| 20 | from qemu.machine.machine import VMLaunchFailure |
| 21 | |
| 22 | |
| 23 | @skipIfMissingCommands("mformat", "mcopy", "mmd") |
| 24 | class Aarch64VirtMachine(QemuSystemTest): |
| 25 | KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' |
| 26 | timeout = 360 |
| 27 | |
| 28 | def wait_for_console_pattern(self, success_message, vm=None): |
| 29 | wait_for_console_pattern(self, success_message, |
| 30 | failure_message='FAILED', |
| 31 | vm=vm) |
| 32 | |
| 33 | def append_firmware_blobs(self): |
| 34 | """ |
| 35 | Setup QEMU firmware blobs for boot. |
| 36 | """ |
| 37 | code_path = self.build_file('pc-bios', 'edk2-aarch64-code.fd') |
| 38 | vars_source = self.build_file('pc-bios', 'edk2-aarch64-vars.fd') |
| 39 | vars_path = self.scratch_file('vars.fd') |
| 40 | shutil.copy(vars_source, vars_path) |
| 41 | |
| 42 | self.vm.add_args('-drive', |
| 43 | f'if=pflash,format=raw,readonly=on,file={code_path}') |
| 44 | self.vm.add_args('-drive', f'if=pflash,format=raw,file={vars_path}') |
| 45 | |
| 46 | |
| 47 | ASSET_VBSA_EFI = Asset( |
| 48 | 'https://github.com/ARM-software/sysarch-acs/raw/refs/heads/main' |
| 49 | '/prebuilt_images/VBSA/v25.12_VBSA_0.7.0/Vbsa.efi', |
| 50 | '80f37d2fb86d152d95dec4d05ff099c9e47ee8a89314268e08056b0e1359e1fa') |
| 51 | |
| 52 | ASSET_BSA_SHELL = Asset( |
| 53 | 'https://github.com/ARM-software/sysarch-acs/raw/refs/heads/main' |
| 54 | '/prebuilt_images/VBSA/v25.12_VBSA_0.7.0/Shell.efi', |
| 55 | 'e526604f0d329b481c6a1f62f7a0db8ea24ce8178b2c6abda8e247425f38775c') |
| 56 | |
| 57 | def test_aarch64_vbsa_uefi_tests(self): |
| 58 | """ |
| 59 | Launch the UEFI based VBSA test from an EFI file-system |
| 60 | """ |
| 61 | |
| 62 | self.vm.set_console() |
| 63 | |
| 64 | # virt machine |
| 65 | self.set_machine('virt') |
| 66 | self.vm.add_args('-M', 'virt,gic-version=max,virtualization=on') |
| 67 | self.vm.add_args('-cpu', 'max', '-m', '1024') |
| 68 | |
| 69 | self.append_firmware_blobs() |
| 70 | |
| 71 | # Build an EFI FAT32 file-system for the UEFI tests |
| 72 | vbsa_efi = self.ASSET_VBSA_EFI.fetch() |
| 73 | bsa_shell = self.ASSET_BSA_SHELL.fetch() |
| 74 | |
| 75 | img_path = self.scratch_file('vbsa.img') |
| 76 | qemu_img = get_qemu_img(self) |
| 77 | check_call([qemu_img, 'create', '-f', 'raw', img_path, '64M'], |
| 78 | stdout=DEVNULL, stderr=DEVNULL) |
| 79 | |
| 80 | check_call(['mformat', '-i', img_path, '-v', 'VBSA', '::'], |
| 81 | stdout=DEVNULL, stderr=DEVNULL) |
| 82 | |
| 83 | check_call(['mmd', '-i', img_path, '::/EFI'], |
| 84 | stdout=DEVNULL, stderr=DEVNULL) |
| 85 | |
| 86 | check_call(['mmd', '-i', img_path, '::/EFI/BOOT'], |
| 87 | stdout=DEVNULL, stderr=DEVNULL) |
| 88 | |
| 89 | check_call(['mcopy', '-i', img_path, bsa_shell, |
| 90 | '::/EFI/BOOT/BOOTAA64.EFI'], |
| 91 | stdout=DEVNULL, stderr=DEVNULL) |
| 92 | |
| 93 | check_call(['mcopy', '-i', img_path, vbsa_efi, '::/Vbsa.efi'], |
| 94 | stdout=DEVNULL, stderr=DEVNULL) |
| 95 | |
| 96 | self.vm.add_args('-drive', |
| 97 | f'file={img_path},format=raw,if=none,id=drive0') |
| 98 | self.vm.add_args('-device', 'virtio-blk-pci,drive=drive0') |
| 99 | |
| 100 | try: |
| 101 | self.vm.launch() |
| 102 | except VMLaunchFailure as excp: |
| 103 | if "does not support providing Virtualization" in excp.output: |
| 104 | self.skipTest("accelerator has no virtualization support") |
| 105 | else: |
| 106 | self.log.info("unhandled launch failure: %s", excp.output) |
| 107 | raise excp |
| 108 | |
| 109 | # wait for EFI prompt |
| 110 | self.wait_for_console_pattern('Shell>') |
| 111 | |
| 112 | # Start the VBSA tests |
| 113 | ec_and_wait(self, "FS0:Vbsa.efi", 'VBSA Architecture Compliance Suite') |
| 114 | |
| 115 | # could we parse the summary somehow? |
| 116 | |
| 117 | self.wait_for_console_pattern('VBSA tests complete. Reset the system.') |
| 118 | |
| 119 | |
| 120 | ASSET_SYSREADY_IMAGE = Asset( |
| 121 | 'https://github.com/ARM-software/arm-systemready/' |
| 122 | 'releases/download/v25.10_SR_3.1.0/systemready_acs_live_image.img.xz.zip', |
| 123 | 'df2c359de15784b1da6a8e6f3c98a053ee38ac0b3f241ccea62e17db092eb03a') |
| 124 | |
| 125 | ROOT_PROMPT = '/ # ' |
| 126 | |
| 127 | @skipIfMissingCommands("sfdisk", "jq", "sed") |
| 128 | def test_aarch64_vbsa_linux_tests(self): |
| 129 | """ |
| 130 | Launch the Linux based VBSA test from the ACS prebuilt images. |
| 131 | |
| 132 | We can use the pre-built images and then trigger the Linux |
| 133 | build and run the tests. We then need to slurp the results |
| 134 | from the partition. |
| 135 | """ |
| 136 | |
| 137 | self.vm.set_console() |
| 138 | |
| 139 | # Plain virt machine |
| 140 | self.set_machine('virt') |
| 141 | self.vm.add_args('-M', 'virt,gic-version=max') |
| 142 | self.vm.add_args('-cpu', 'max', '-m', '1024', '-smp', '4') |
| 143 | |
| 144 | self.append_firmware_blobs() |
| 145 | |
| 146 | # First fetch, decompress (twice) and prepare the disk image |
| 147 | # on an NVME device (the kernel only has drivers for that). |
| 148 | self.archive_extract(self.ASSET_SYSREADY_IMAGE, format="zip") |
| 149 | disk_image_xz = self.scratch_file("systemready_acs_live_image.img.xz") |
| 150 | disk_image = self.uncompress(disk_image_xz) |
| 151 | |
| 152 | self.vm.add_args('-device', |
| 153 | 'nvme,drive=hd,serial=QEMU_ROOT_SSD') |
| 154 | self.vm.add_args('-blockdev', |
| 155 | f'driver=raw,node-name=hd,file.driver=file,file.filename={disk_image}') |
| 156 | |
| 157 | # Tweak grub.cfg default to avoid manually navigating grub |
| 158 | grub_cfg = self.scratch_file("grub.cfg") |
| 159 | offset = int(check_output(f"sfdisk --json {disk_image} |" |
| 160 | "jq '.partitiontable.partitions[0].start * 512'", |
| 161 | shell=True)) |
| 162 | check_call(["mcopy", "-i", f"{disk_image}@@{offset}", |
| 163 | "::/EFI/BOOT/grub.cfg", grub_cfg]) |
| 164 | |
| 165 | with open(grub_cfg, 'a', encoding="utf8") as f: |
| 166 | f.write("set default='Linux Execution Enviroment'") |
| 167 | |
| 168 | check_call(["mcopy", "-D", "o", "-i", f"{disk_image}@@{offset}", |
| 169 | grub_cfg, "::/EFI/BOOT/grub.cfg"]) |
| 170 | |
| 171 | # Launch QEMU and wait for grub and select the "Linux |
| 172 | # Execution Environment" so we can launch the test. |
| 173 | |
| 174 | self.vm.launch() |
| 175 | self.wait_for_console_pattern(self.ROOT_PROMPT) |
| 176 | ec_and_wait(self, "/usr/bin/bsa.sh --skip " |
| 177 | "B_REP_1,B_IEP_1,B_PCIe_11,B_MEM_06", |
| 178 | self.ROOT_PROMPT) |
| 179 | |
| 180 | # Now we can shutdown |
| 181 | ec_and_wait(self, "halt -f", "reboot: System halted") |
| 182 | self.vm.shutdown() |
| 183 | |
| 184 | # and extract the test logs |
| 185 | bsa_app_res = self.scratch_file("BsaResultsApp.log") |
| 186 | bsa_kern_res = self.scratch_file("BsaResultsKernel.log") |
| 187 | |
| 188 | check_call(["mcopy", "-i", f"{disk_image}@@{offset}", |
| 189 | "::acs_results/Linux/BsaResultsApp.log", bsa_app_res]) |
| 190 | check_call(["mcopy", "-i", f"{disk_image}@@{offset}", |
| 191 | "::acs_results/Linux/BsaResultsKernel.log", bsa_kern_res]) |
| 192 | |
| 193 | # for now just check the kernel log for the result summary |
| 194 | test_result_re = re.compile(r"\[.*\]\s+(.+): Result:\s+(\w+)") |
| 195 | summary_re = re.compile(r"Total Tests Run =\s*(\d+), Tests Passed =\s*(\d+), Tests Failed =\s*(\d+)") |
| 196 | |
| 197 | with open(bsa_kern_res, 'r', encoding="utf8") as f: |
| 198 | for line in f: |
| 199 | test_match = test_result_re.search(line) |
| 200 | if test_match: |
| 201 | desc = test_match.group(1) |
| 202 | status = test_match.group(2) |
| 203 | self.log.info(f"Test: {desc} status: {status}") |
| 204 | |
| 205 | match = summary_re.search(line) |
| 206 | if match: |
| 207 | total, passed, failed = match.groups() |
| 208 | |
| 209 | if int(failed) > 0: |
| 210 | self.fail(f"{failed} tests failed ({total})") |
| 211 | |
| 212 | |
| 213 | if __name__ == '__main__': |
| 214 | QemuSystemTest.main() |