| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Boots a nested guest and compare content of a device (passthrough) to a |
| 4 | # reference image. Both vfio group and iommufd passthrough methods are tested. |
| 5 | # |
| 6 | # Copyright (c) 2025 Linaro Ltd. |
| 7 | # |
| 8 | # Author: Pierrick Bouvier <pierrick.bouvier@linaro.org> |
| 9 | # |
| 10 | # SPDX-License-Identifier: GPL-2.0-or-later |
| 11 | |
| 12 | from os.path import join |
| 13 | from random import randbytes |
| 14 | |
| 15 | from qemu_test import QemuSystemTest, Asset |
| 16 | from qemu_test import wait_for_console_pattern |
| 17 | |
| 18 | |
| 19 | GUEST_SCRIPT = ''' |
| 20 | #!/usr/bin/env bash |
| 21 | |
| 22 | set -euo pipefail |
| 23 | set -x |
| 24 | |
| 25 | # find disks from nvme serial |
| 26 | dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ') |
| 27 | dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ') |
| 28 | pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../)) |
| 29 | pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../)) |
| 30 | |
| 31 | # bind disks to vfio |
| 32 | for p in "$pci_vfio" "$pci_iommufd"; do |
| 33 | if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then |
| 34 | continue |
| 35 | fi |
| 36 | echo $p > /sys/bus/pci/drivers/nvme/unbind |
| 37 | echo vfio-pci > /sys/bus/pci/devices/$p/driver_override |
| 38 | echo $p > /sys/bus/pci/drivers/vfio-pci/bind |
| 39 | done |
| 40 | |
| 41 | # boot nested guest and execute /host/nested_guest.sh |
| 42 | # one disk is passed through vfio group, the other, through iommufd |
| 43 | qemu-system-aarch64 \ |
| 44 | -M virt \ |
| 45 | -display none \ |
| 46 | -serial stdio \ |
| 47 | -cpu host \ |
| 48 | -enable-kvm \ |
| 49 | -m 1G \ |
| 50 | -kernel /host/Image.gz \ |
| 51 | -drive format=raw,file=/host/guest.ext4,if=virtio \ |
| 52 | -append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \ |
| 53 | -virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \ |
| 54 | -device vfio-pci,host=$pci_vfio \ |
| 55 | -object iommufd,id=iommufd0 \ |
| 56 | -device vfio-pci,host=$pci_iommufd,iommufd=iommufd0 |
| 57 | ''' |
| 58 | |
| 59 | NESTED_GUEST_SCRIPT = ''' |
| 60 | #!/usr/bin/env bash |
| 61 | |
| 62 | set -euo pipefail |
| 63 | set -x |
| 64 | |
| 65 | image_vfio=/host/disk_vfio |
| 66 | image_iommufd=/host/disk_iommufd |
| 67 | |
| 68 | dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ') |
| 69 | dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ') |
| 70 | |
| 71 | # compare if devices are identical to original images |
| 72 | diff $image_vfio /dev/$dev_vfio |
| 73 | diff $image_iommufd /dev/$dev_iommufd |
| 74 | |
| 75 | echo device_passthrough_test_ok |
| 76 | ''' |
| 77 | |
| 78 | |
| 79 | class Aarch64DevicePassthrough(QemuSystemTest): |
| 80 | |
| 81 | # https://github.com/p-b-o/qemu-linux-stack/tree/device_passthrough |
| 82 | # $ ./build.sh && ./archive_artifacts.sh out.tar.xz |
| 83 | # |
| 84 | # Linux kernel is compiled with defconfig + |
| 85 | # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD |
| 86 | # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde |
| 87 | ASSET_DEVICE_PASSTHROUGH_STACK = Asset( |
| 88 | ('https://github.com/p-b-o/qemu-linux-stack/' |
| 89 | 'releases/download/build/device_passthrough-a9612a2.tar.xz'), |
| 90 | 'f7d2f70912e7231986e6e293e1a2c4786dd02bec113a7acb6bfc619e96155455') |
| 91 | |
| 92 | # This tests the device passthrough implementation, by booting a VM |
| 93 | # supporting it with two nvme disks attached, and launching a nested VM |
| 94 | # reading their content. |
| 95 | def test_aarch64_device_passthrough(self): |
| 96 | self.set_machine('virt') |
| 97 | self.require_accelerator('tcg') |
| 98 | |
| 99 | self.vm.set_console() |
| 100 | |
| 101 | stack_path_tar = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch() |
| 102 | self.archive_extract(stack_path_tar, format="tar") |
| 103 | |
| 104 | stack = self.scratch_file('out') |
| 105 | kernel = join(stack, 'Image.gz') |
| 106 | rootfs_host = join(stack, 'host.ext4') |
| 107 | disk_vfio = join(stack, 'disk_vfio') |
| 108 | disk_iommufd = join(stack, 'disk_iommufd') |
| 109 | guest_cmd = join(stack, 'guest.sh') |
| 110 | nested_guest_cmd = join(stack, 'nested_guest.sh') |
| 111 | # we generate two random disks |
| 112 | with open(disk_vfio, "wb") as d: |
| 113 | d.write(randbytes(512)) |
| 114 | with open(disk_iommufd, "wb") as d: |
| 115 | d.write(randbytes(1024)) |
| 116 | with open(guest_cmd, 'w', encoding='utf-8') as s: |
| 117 | s.write(GUEST_SCRIPT) |
| 118 | with open(nested_guest_cmd, 'w', encoding='utf-8') as s: |
| 119 | s.write(NESTED_GUEST_SCRIPT) |
| 120 | |
| 121 | self.vm.add_args('-cpu', 'max') |
| 122 | self.vm.add_args('-m', '2G') |
| 123 | self.vm.add_args('-M', 'virt,' |
| 124 | 'virtualization=on,' |
| 125 | 'gic-version=max,' |
| 126 | 'iommu=smmuv3') |
| 127 | self.vm.add_args('-kernel', kernel) |
| 128 | self.vm.add_args('-drive', f'format=raw,file={rootfs_host}') |
| 129 | self.vm.add_args('-drive', |
| 130 | f'file={disk_vfio},if=none,id=vfio,format=raw') |
| 131 | self.vm.add_args('-device', 'nvme,serial=vfio,drive=vfio') |
| 132 | self.vm.add_args('-drive', |
| 133 | f'file={disk_iommufd},if=none,id=iommufd,format=raw') |
| 134 | self.vm.add_args('-device', 'nvme,serial=iommufd,drive=iommufd') |
| 135 | self.vm.add_args('-virtfs', |
| 136 | f'local,path={stack}/,mount_tag=host,' |
| 137 | 'security_model=mapped,readonly=off') |
| 138 | # boot and execute guest script |
| 139 | # init will trigger a kernel panic if script fails |
| 140 | self.vm.add_args('-append', |
| 141 | 'root=/dev/vda init=/init -- bash /host/guest.sh') |
| 142 | |
| 143 | self.vm.launch() |
| 144 | wait_for_console_pattern(self, 'device_passthrough_test_ok', |
| 145 | failure_message='Kernel panic') |
| 146 | |
| 147 | |
| 148 | if __name__ == '__main__': |
| 149 | QemuSystemTest.main() |