| 1 | /* |
| 2 | * QTest for Intel IOMMU (VT-d) IOTLB invalidation via Invalidation Queue |
| 3 | * |
| 4 | * Validates that IOTLB invalidation descriptors submitted through the |
| 5 | * queued invalidation interface correctly flush cached translations, |
| 6 | * forcing the IOMMU to re-walk page tables on subsequent DMA. |
| 7 | * |
| 8 | * Copyright (c) 2026 Intel Corporation. |
| 9 | * |
| 10 | * Author: Junjie Cao <junjie.cao@intel.com> |
| 11 | * |
| 12 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 13 | */ |
| 14 | |
| 15 | #include "qemu/osdep.h" |
| 16 | #include "libqtest.h" |
| 17 | #include "libqos/pci.h" |
| 18 | #include "libqos/pci-pc.h" |
| 19 | #include "hw/i386/intel_iommu_internal.h" |
| 20 | #include "hw/misc/iommu-testdev.h" |
| 21 | #include "libqos/qos-intel-iommu.h" |
| 22 | #include "libqos/qos-iommu-testdev.h" |
| 23 | |
| 24 | #define DMA_LEN 4 |
| 25 | |
| 26 | /* |
| 27 | * Second DMA target page, chosen to fall well outside any address used by |
| 28 | * qos-intel-iommu's fixed structure layout. |
| 29 | */ |
| 30 | #define QVTD_PT_VAL_B (QVTD_MEM_BASE + 0x00200000) |
| 31 | |
| 32 | /* |
| 33 | * A second IOVA/target page for the page-selectivity test. QVTD_IOVA_2 is |
| 34 | * QVTD_IOVA + 4K: it shares the L4/L3/L2 walk built by |
| 35 | * qvtd_build_translation() and differs only in the leaf (L1) slot, so mapping |
| 36 | * it costs one extra leaf PTE. QVTD_PT_VAL_2 is its distinct target page. |
| 37 | */ |
| 38 | #define QVTD_IOVA_2 (QVTD_IOVA + 0x1000) |
| 39 | #define QVTD_PT_VAL_2 (QVTD_MEM_BASE + 0x00300000) |
| 40 | |
| 41 | typedef enum { |
| 42 | IOTLB_INV_GLOBAL, |
| 43 | IOTLB_INV_DOMAIN, |
| 44 | IOTLB_INV_PAGE, |
| 45 | } IOTLBInvGranularity; |
| 46 | |
| 47 | /* |
| 48 | * Core invalidation test, parameterized by translation mode and |
| 49 | * invalidation granularity. |
| 50 | * |
| 51 | * The iommu-testdev device performs DMA writes via the IOMMU (using the |
| 52 | * IOVA) and verifies by reading back from the expected GPA directly. If |
| 53 | * the IOTLB is stale, the DMA write lands at the old PA while readback |
| 54 | * uses the GPA we supply, causing a mismatch (ITD_DMA_ERR_MISMATCH). |
| 55 | * |
| 56 | * Test sequence: |
| 57 | * 1. Setup translation: IOVA -> PA_A |
| 58 | * 2. DMA(gpa=PA_A) -> success (IOTLB populates cache) |
| 59 | * 3. Modify PTE: IOVA -> PA_B (no invalidation) |
| 60 | * 4. DMA(gpa=PA_B) -> MISMATCH (stale IOTLB directs write to PA_A) |
| 61 | * 5. Issue IOTLB invalidation + wait |
| 62 | * 6. DMA(gpa=PA_B) -> success (cache flushed, fresh page walk) |
| 63 | * |
| 64 | * Phase 4 depends on QEMU's IOTLB caching the Phase 1 translation; if a |
| 65 | * future change makes IOTLB caching lazy this assertion would no longer |
| 66 | * exercise the stale-cache path. |
| 67 | */ |
| 68 | static void run_iotlb_inv_test(QVTDTransMode mode, IOTLBInvGranularity gran) |
| 69 | { |
| 70 | QTestState *qts; |
| 71 | QPCIBus *pcibus; |
| 72 | QPCIDevice *dev; |
| 73 | QPCIBar bar; |
| 74 | uint32_t tail = 0; |
| 75 | uint32_t result; |
| 76 | uint64_t pa_a, pa_b; |
| 77 | |
| 78 | if (!qtest_has_machine("q35")) { |
| 79 | g_test_skip("q35 machine not available"); |
| 80 | return; |
| 81 | } |
| 82 | |
| 83 | qts = qtest_initf("-machine q35 -smp 1 -m 512 -net none " |
| 84 | "%s -device iommu-testdev", |
| 85 | qvtd_iommu_args(mode)); |
| 86 | |
| 87 | if (!qvtd_check_caps(qts, mode)) { |
| 88 | qtest_quit(qts); |
| 89 | return; |
| 90 | } |
| 91 | |
| 92 | dev = qvtd_setup_qtest_pci_device(qts, &pcibus, &bar); |
| 93 | |
| 94 | /* |
| 95 | * The IOMMU translates an IOVA to a page base, then the page offset |
| 96 | * from the IOVA is added. So GPA = page_base + (IOVA & 0xfff). |
| 97 | */ |
| 98 | pa_a = (QVTD_PT_VAL & VTD_PAGE_MASK_4K) + (QVTD_IOVA & 0xfff); |
| 99 | pa_b = (QVTD_PT_VAL_B & VTD_PAGE_MASK_4K) + (QVTD_IOVA & 0xfff); |
| 100 | |
| 101 | /* --- Phase 1: Setup and initial DMA (populates IOTLB) --- */ |
| 102 | qvtd_build_translation(qts, mode, dev->devfn); |
| 103 | qvtd_program_regs(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, mode); |
| 104 | |
| 105 | qtest_memset(qts, pa_a, 0, DMA_LEN); |
| 106 | qtest_memset(qts, pa_b, 0, DMA_LEN); |
| 107 | |
| 108 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA, pa_a, |
| 109 | DMA_LEN, 0); |
| 110 | g_assert_cmpuint(result, ==, 0); |
| 111 | |
| 112 | /* --- Phase 2: Modify PTE without invalidation -> stale IOTLB --- */ |
| 113 | qtest_writeq(qts, qvtd_leaf_pte_addr(QVTD_IOVA), |
| 114 | qvtd_make_leaf_pte(QVTD_PT_VAL_B & VTD_PAGE_MASK_4K, mode)); |
| 115 | qtest_memset(qts, pa_a, 0, DMA_LEN); |
| 116 | qtest_memset(qts, pa_b, 0, DMA_LEN); |
| 117 | |
| 118 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA, pa_b, |
| 119 | DMA_LEN, 0); |
| 120 | g_assert_cmpuint(result, ==, ITD_DMA_ERR_MISMATCH); |
| 121 | |
| 122 | /* --- Phase 3: Invalidate IOTLB -> fresh page walk succeeds --- */ |
| 123 | switch (gran) { |
| 124 | case IOTLB_INV_GLOBAL: |
| 125 | tail = qvtd_submit_iotlb_global_inv(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, |
| 126 | tail); |
| 127 | break; |
| 128 | case IOTLB_INV_DOMAIN: |
| 129 | tail = qvtd_submit_iotlb_domain_inv(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, |
| 130 | QVTD_DOMAIN_ID, tail); |
| 131 | break; |
| 132 | case IOTLB_INV_PAGE: |
| 133 | tail = qvtd_submit_iotlb_page_inv(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, |
| 134 | QVTD_DOMAIN_ID, QVTD_IOVA, 0, |
| 135 | tail); |
| 136 | break; |
| 137 | } |
| 138 | tail = qvtd_submit_inv_wait_and_poll(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, |
| 139 | tail); |
| 140 | |
| 141 | qtest_memset(qts, pa_a, 0, DMA_LEN); |
| 142 | qtest_memset(qts, pa_b, 0, DMA_LEN); |
| 143 | |
| 144 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA, pa_b, |
| 145 | DMA_LEN, 0); |
| 146 | g_assert_cmpuint(result, ==, 0); |
| 147 | |
| 148 | g_free(dev); |
| 149 | qpci_free_pc(pcibus); |
| 150 | qtest_quit(qts); |
| 151 | } |
| 152 | |
| 153 | /* |
| 154 | * Page-selectivity test: verify that a page-selective invalidation flushes |
| 155 | * the named page and touches other cached pages only as far as the model |
| 156 | * intends. run_iotlb_inv_test() caches a single entry, so it cannot tell a |
| 157 | * page-selective flush from a domain-wide or global one; this test caches two |
| 158 | * pages in the same domain and checks the second one's fate. |
| 159 | * |
| 160 | * The expected fate of the second page depends on the translation level: |
| 161 | * |
| 162 | * - second-level (legacy / scalable-slt): a page-selective descriptor |
| 163 | * evicts only the matching gfn, so IOVA_2 survives. |
| 164 | * - first-level (scalable-flt): QEMU invalidates all first-stage entries of |
| 165 | * the domain on a page-selective descriptor (vtd_hash_remove_by_page() |
| 166 | * returns true for any pgtt==FST entry of the domain, matching the VT-d |
| 167 | * spec for first-stage IOTLB invalidation), so IOVA_2 is flushed too. |
| 168 | * |
| 169 | * Method: map IOVA -> PA_A and IOVA_2 -> PA_A_2, DMA both to populate two |
| 170 | * IOTLB entries, rewrite both leaf PTEs to PA_B* without invalidating, then |
| 171 | * page-invalidate IOVA only. IOVA always re-walks to PA_B. For IOVA_2 we |
| 172 | * verify the DMA against its *original* page PA_A_2: if the entry survived, |
| 173 | * the stale cache still serves PA_A_2 (success); if it was flushed, the fresh |
| 174 | * walk reaches PA_B_2 and mismatches PA_A_2. So a survived entry gives |
| 175 | * success and a flushed entry gives MISMATCH, and we assert whichever the |
| 176 | * mode requires -- catching both an over-matching second-level flush and a |
| 177 | * regression that stopped flushing first-stage entries domain-wide. |
| 178 | */ |
| 179 | static void run_page_selectivity_test(QVTDTransMode mode) |
| 180 | { |
| 181 | QTestState *qts; |
| 182 | QPCIBus *pcibus; |
| 183 | QPCIDevice *dev; |
| 184 | QPCIBar bar; |
| 185 | uint32_t tail = 0; |
| 186 | uint32_t result; |
| 187 | uint64_t pa_a, pa_b, pa_a2, pa_b2; |
| 188 | bool fl_domain_wide = (mode == QVTD_TM_SCALABLE_FLT); |
| 189 | |
| 190 | if (!qtest_has_machine("q35")) { |
| 191 | g_test_skip("q35 machine not available"); |
| 192 | return; |
| 193 | } |
| 194 | |
| 195 | qts = qtest_initf("-machine q35 -smp 1 -m 512 -net none " |
| 196 | "%s -device iommu-testdev", |
| 197 | qvtd_iommu_args(mode)); |
| 198 | |
| 199 | if (!qvtd_check_caps(qts, mode)) { |
| 200 | qtest_quit(qts); |
| 201 | return; |
| 202 | } |
| 203 | |
| 204 | dev = qvtd_setup_qtest_pci_device(qts, &pcibus, &bar); |
| 205 | |
| 206 | pa_a = (QVTD_PT_VAL & VTD_PAGE_MASK_4K) + (QVTD_IOVA & 0xfff); |
| 207 | pa_b = (QVTD_PT_VAL_B & VTD_PAGE_MASK_4K) + (QVTD_IOVA & 0xfff); |
| 208 | pa_a2 = (QVTD_PT_VAL_2 & VTD_PAGE_MASK_4K) + (QVTD_IOVA_2 & 0xfff); |
| 209 | pa_b2 = (QVTD_PT_VAL_B & VTD_PAGE_MASK_4K) + (QVTD_IOVA_2 & 0xfff); |
| 210 | |
| 211 | /* --- Setup: IOVA -> PA_A (built by helper) and IOVA_2 -> PA_A_2 --- */ |
| 212 | qvtd_build_translation(qts, mode, dev->devfn); |
| 213 | qtest_writeq(qts, qvtd_leaf_pte_addr(QVTD_IOVA_2), |
| 214 | qvtd_make_leaf_pte(QVTD_PT_VAL_2 & VTD_PAGE_MASK_4K, mode)); |
| 215 | qvtd_program_regs(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, mode); |
| 216 | |
| 217 | /* Populate both IOTLB entries. */ |
| 218 | qtest_memset(qts, pa_a, 0, DMA_LEN); |
| 219 | qtest_memset(qts, pa_a2, 0, DMA_LEN); |
| 220 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA, pa_a, |
| 221 | DMA_LEN, 0); |
| 222 | g_assert_cmpuint(result, ==, 0); |
| 223 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA_2, pa_a2, |
| 224 | DMA_LEN, 0); |
| 225 | g_assert_cmpuint(result, ==, 0); |
| 226 | |
| 227 | /* Rewrite both leaf PTEs to PA_B* without invalidating. */ |
| 228 | qtest_writeq(qts, qvtd_leaf_pte_addr(QVTD_IOVA), |
| 229 | qvtd_make_leaf_pte(QVTD_PT_VAL_B & VTD_PAGE_MASK_4K, mode)); |
| 230 | qtest_writeq(qts, qvtd_leaf_pte_addr(QVTD_IOVA_2), |
| 231 | qvtd_make_leaf_pte(QVTD_PT_VAL_B & VTD_PAGE_MASK_4K, mode)); |
| 232 | |
| 233 | /* Page-selective invalidation of IOVA only. */ |
| 234 | tail = qvtd_submit_iotlb_page_inv(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, |
| 235 | QVTD_DOMAIN_ID, QVTD_IOVA, 0, tail); |
| 236 | tail = qvtd_submit_inv_wait_and_poll(qts, Q35_HOST_BRIDGE_IOMMU_ADDR, |
| 237 | tail); |
| 238 | |
| 239 | /* IOVA was flushed: fresh walk reaches PA_B. */ |
| 240 | qtest_memset(qts, pa_a, 0, DMA_LEN); |
| 241 | qtest_memset(qts, pa_b, 0, DMA_LEN); |
| 242 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA, pa_b, |
| 243 | DMA_LEN, 0); |
| 244 | g_assert_cmpuint(result, ==, 0); |
| 245 | |
| 246 | /* |
| 247 | * IOVA_2's fate, verified against its original page PA_A_2: |
| 248 | * - second-level: entry survives, stale cache serves PA_A_2 -> success; |
| 249 | * - first-level: entry was flushed domain-wide, fresh walk reaches |
| 250 | * PA_B_2 -> MISMATCH against PA_A_2. |
| 251 | */ |
| 252 | qtest_memset(qts, pa_a2, 0, DMA_LEN); |
| 253 | qtest_memset(qts, pa_b2, 0, DMA_LEN); |
| 254 | result = qos_iommu_testdev_trigger_dma(dev, bar, QVTD_IOVA_2, pa_a2, |
| 255 | DMA_LEN, 0); |
| 256 | if (fl_domain_wide) { |
| 257 | g_assert_cmpuint(result, ==, ITD_DMA_ERR_MISMATCH); |
| 258 | } else { |
| 259 | g_assert_cmpuint(result, ==, 0); |
| 260 | } |
| 261 | |
| 262 | g_free(dev); |
| 263 | qpci_free_pc(pcibus); |
| 264 | qtest_quit(qts); |
| 265 | } |
| 266 | |
| 267 | /* |
| 268 | * scalable-flt is covered here even though, per the VT-d spec, first-level |
| 269 | * mappings are invalidated with the PASID-based descriptor |
| 270 | * (VTD_INV_DESC_PIOTLB). QEMU keeps first- and second-level mappings in a |
| 271 | * single IOTLB that the legacy VTD_INV_DESC_IOTLB descriptor flushes for |
| 272 | * every level, so this test drives that descriptor across all modes. |
| 273 | * PASID-selective (PIOTLB) invalidation is a separate path, left for a |
| 274 | * follow-up. |
| 275 | */ |
| 276 | static const struct { |
| 277 | const char *name; |
| 278 | QVTDTransMode mode; |
| 279 | } trans_modes[] = { |
| 280 | { "legacy", QVTD_TM_LEGACY_TRANS }, |
| 281 | { "scalable-slt", QVTD_TM_SCALABLE_SLT }, |
| 282 | { "scalable-flt", QVTD_TM_SCALABLE_FLT }, |
| 283 | }; |
| 284 | |
| 285 | static const struct { |
| 286 | const char *name; |
| 287 | IOTLBInvGranularity gran; |
| 288 | } granularities[] = { |
| 289 | { "global", IOTLB_INV_GLOBAL }, |
| 290 | { "domain", IOTLB_INV_DOMAIN }, |
| 291 | { "page", IOTLB_INV_PAGE }, |
| 292 | }; |
| 293 | |
| 294 | typedef struct { |
| 295 | QVTDTransMode mode; |
| 296 | IOTLBInvGranularity gran; |
| 297 | } TestCase; |
| 298 | |
| 299 | static void test_iotlb_inv(const void *opaque) |
| 300 | { |
| 301 | const TestCase *tc = opaque; |
| 302 | |
| 303 | run_iotlb_inv_test(tc->mode, tc->gran); |
| 304 | } |
| 305 | |
| 306 | static void test_page_selectivity(const void *opaque) |
| 307 | { |
| 308 | const QVTDTransMode *mode = opaque; |
| 309 | |
| 310 | run_page_selectivity_test(*mode); |
| 311 | } |
| 312 | |
| 313 | int main(int argc, char **argv) |
| 314 | { |
| 315 | g_test_init(&argc, &argv, NULL); |
| 316 | |
| 317 | for (size_t m = 0; m < ARRAY_SIZE(trans_modes); m++) { |
| 318 | for (size_t g = 0; g < ARRAY_SIZE(granularities); g++) { |
| 319 | TestCase *tc = g_new(TestCase, 1); |
| 320 | char *path; |
| 321 | |
| 322 | tc->mode = trans_modes[m].mode; |
| 323 | tc->gran = granularities[g].gran; |
| 324 | |
| 325 | path = g_strdup_printf("/iommu-testdev/intel/iotlb-inv/%s-%s", |
| 326 | granularities[g].name, |
| 327 | trans_modes[m].name); |
| 328 | qtest_add_data_func_full(path, tc, test_iotlb_inv, g_free); |
| 329 | g_free(path); |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | for (size_t m = 0; m < ARRAY_SIZE(trans_modes); m++) { |
| 334 | QVTDTransMode *mode = g_new(QVTDTransMode, 1); |
| 335 | char *path; |
| 336 | |
| 337 | *mode = trans_modes[m].mode; |
| 338 | path = g_strdup_printf( |
| 339 | "/iommu-testdev/intel/iotlb-inv/page-selective/%s", |
| 340 | trans_modes[m].name); |
| 341 | qtest_add_data_func_full(path, mode, test_page_selectivity, g_free); |
| 342 | g_free(path); |
| 343 | } |
| 344 | |
| 345 | return g_test_run(); |
| 346 | } |