| 1 | /* |
| 2 | * QOS RISC-V IOMMU Module |
| 3 | * |
| 4 | * This module provides RISC-V IOMMU-specific helper functions for libqos tests, |
| 5 | * encapsulating RISC-V IOMMU setup, and assertions. |
| 6 | * |
| 7 | * Copyright (c) 2026 Chao Liu <chao.liu.zevorn@gmail.com> |
| 8 | * |
| 9 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 10 | */ |
| 11 | |
| 12 | #include "qemu/osdep.h" |
| 13 | #include "hw/riscv/riscv-iommu-bits.h" |
| 14 | #include "qos-iommu-testdev.h" |
| 15 | #include "qos-riscv-iommu.h" |
| 16 | |
| 17 | /* Apply space offset to address */ |
| 18 | static inline uint64_t qriommu_apply_space_offs(uint64_t address) |
| 19 | { |
| 20 | return address + QRIOMMU_SPACE_OFFS; |
| 21 | } |
| 22 | |
| 23 | static uint64_t qriommu_encode_pte(uint64_t pa, uint64_t attrs) |
| 24 | { |
| 25 | return ((pa >> 12) << 10) | attrs; |
| 26 | } |
| 27 | |
| 28 | static void qriommu_wait_for_queue_active(QTestState *qts, uint64_t iommu_base, |
| 29 | uint32_t queue_csr, uint32_t on_bit) |
| 30 | { |
| 31 | guint64 timeout_us = 2 * 1000 * 1000; |
| 32 | gint64 start_time = g_get_monotonic_time(); |
| 33 | uint32_t reg; |
| 34 | |
| 35 | for (;;) { |
| 36 | qtest_clock_step(qts, 100); |
| 37 | |
| 38 | reg = qtest_readl(qts, iommu_base + queue_csr); |
| 39 | if (reg & on_bit) { |
| 40 | return; |
| 41 | } |
| 42 | g_assert(g_get_monotonic_time() - start_time <= timeout_us); |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | uint32_t qriommu_expected_dma_result(QRIOMMUTestContext *ctx) |
| 47 | { |
| 48 | return ctx->config.expected_result; |
| 49 | } |
| 50 | |
| 51 | uint32_t qriommu_build_dma_attrs(void) |
| 52 | { |
| 53 | /* RISC-V IOMMU uses standard AXI attributes */ |
| 54 | return 0; |
| 55 | } |
| 56 | |
| 57 | uint32_t qriommu_setup_and_enable_translation(QRIOMMUTestContext *ctx) |
| 58 | { |
| 59 | uint32_t build_result; |
| 60 | |
| 61 | /* Build page tables and RISC-V IOMMU structures first */ |
| 62 | build_result = qriommu_build_translation( |
| 63 | ctx->qts, ctx->config.trans_mode, |
| 64 | ctx->device_id); |
| 65 | if (build_result != 0) { |
| 66 | g_test_message("Build failed: mode=%u device_id=%u status=0x%x", |
| 67 | ctx->config.trans_mode, ctx->device_id, build_result); |
| 68 | ctx->trans_status = build_result; |
| 69 | return ctx->trans_status; |
| 70 | } |
| 71 | |
| 72 | /* Program RISC-V IOMMU registers */ |
| 73 | qriommu_program_regs(ctx->qts, ctx->iommu_base); |
| 74 | |
| 75 | ctx->trans_status = 0; |
| 76 | return ctx->trans_status; |
| 77 | } |
| 78 | |
| 79 | static bool qriommu_validate_test_result(QRIOMMUTestContext *ctx) |
| 80 | { |
| 81 | uint32_t expected = qriommu_expected_dma_result(ctx); |
| 82 | g_test_message("-> Validating result: expected=0x%x actual=0x%x", |
| 83 | expected, ctx->dma_result); |
| 84 | return (ctx->dma_result == expected); |
| 85 | } |
| 86 | |
| 87 | static uint32_t qriommu_single_translation_setup(void *opaque) |
| 88 | { |
| 89 | return qriommu_setup_and_enable_translation(opaque); |
| 90 | } |
| 91 | |
| 92 | static uint32_t qriommu_single_translation_attrs(void *opaque) |
| 93 | { |
| 94 | return qriommu_build_dma_attrs(); |
| 95 | } |
| 96 | |
| 97 | static bool qriommu_single_translation_validate(void *opaque) |
| 98 | { |
| 99 | return qriommu_validate_test_result(opaque); |
| 100 | } |
| 101 | |
| 102 | static void qriommu_single_translation_report(void *opaque, |
| 103 | uint32_t dma_result) |
| 104 | { |
| 105 | QRIOMMUTestContext *ctx = opaque; |
| 106 | |
| 107 | if (dma_result != 0) { |
| 108 | g_test_message("DMA failed: mode=%u result=0x%x", |
| 109 | ctx->config.trans_mode, dma_result); |
| 110 | } else { |
| 111 | g_test_message("-> DMA succeeded: mode=%u", |
| 112 | ctx->config.trans_mode); |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | void qriommu_run_translation_case(QTestState *qts, QPCIDevice *dev, |
| 117 | QPCIBar bar, uint64_t iommu_base, |
| 118 | const QRIOMMUTestConfig *cfg) |
| 119 | { |
| 120 | QRIOMMUTestContext ctx = { |
| 121 | .qts = qts, |
| 122 | .dev = dev, |
| 123 | .bar = bar, |
| 124 | .iommu_base = iommu_base, |
| 125 | .config = *cfg, |
| 126 | .device_id = dev->devfn, |
| 127 | }; |
| 128 | |
| 129 | QOSIOMMUTestdevDmaCfg dma = { |
| 130 | .dev = dev, |
| 131 | .bar = bar, |
| 132 | .iova = QRIOMMU_IOVA, |
| 133 | .gpa = ctx.config.dma_gpa, |
| 134 | .len = ctx.config.dma_len, |
| 135 | }; |
| 136 | |
| 137 | qtest_memset(qts, cfg->dma_gpa, 0x00, cfg->dma_len); |
| 138 | qos_iommu_testdev_single_translation(&dma, &ctx, |
| 139 | qriommu_single_translation_setup, |
| 140 | qriommu_single_translation_attrs, |
| 141 | qriommu_single_translation_validate, |
| 142 | qriommu_single_translation_report, |
| 143 | &ctx.dma_result); |
| 144 | |
| 145 | if (ctx.dma_result == 0 && ctx.config.expected_result == 0) { |
| 146 | g_autofree uint8_t *buf = g_malloc(ctx.config.dma_len); |
| 147 | |
| 148 | qtest_memread(ctx.qts, ctx.config.dma_gpa, buf, ctx.config.dma_len); |
| 149 | |
| 150 | for (int i = 0; i < ctx.config.dma_len; i++) { |
| 151 | uint8_t expected; |
| 152 | |
| 153 | expected = (ITD_DMA_WRITE_VAL >> ((i % 4) * 8)) & 0xff; |
| 154 | g_assert_cmpuint(buf[i], ==, expected); |
| 155 | } |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | static uint32_t qriommu_get_table_index(uint64_t addr, int level) |
| 160 | { |
| 161 | /* SV39: 39-bit virtual address, 3-level page table */ |
| 162 | switch (level) { |
| 163 | case 0: |
| 164 | return (addr >> 30) & 0x1ff; /* L0: bits [38:30] */ |
| 165 | case 1: |
| 166 | return (addr >> 21) & 0x1ff; /* L1: bits [29:21] */ |
| 167 | case 2: |
| 168 | return (addr >> 12) & 0x1ff; /* L2: bits [20:12] */ |
| 169 | default: |
| 170 | g_assert_not_reached(); |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | static uint64_t qriommu_get_table_addr(uint64_t base, int level, uint64_t iova) |
| 175 | { |
| 176 | uint32_t index = qriommu_get_table_index(iova, level); |
| 177 | return (base & QRIOMMU_PTE_PPN_MASK) + (index * 8); |
| 178 | } |
| 179 | |
| 180 | static void qriommu_map_leaf(QTestState *qts, uint64_t root_pa, |
| 181 | uint64_t l0_pa, uint64_t l1_pa, |
| 182 | uint64_t l0_pte_val, uint64_t l1_pte_val, |
| 183 | uint64_t va, uint64_t pa, uint64_t leaf_attrs) |
| 184 | { |
| 185 | uint64_t l0_addr = qriommu_get_table_addr(root_pa, 0, va); |
| 186 | uint64_t l1_addr = qriommu_get_table_addr(l0_pa, 1, va); |
| 187 | uint64_t l2_addr = qriommu_get_table_addr(l1_pa, 2, va); |
| 188 | |
| 189 | qtest_writeq(qts, l0_addr, l0_pte_val); |
| 190 | qtest_writeq(qts, l1_addr, l1_pte_val); |
| 191 | qtest_writeq(qts, l2_addr, qriommu_encode_pte(pa, leaf_attrs)); |
| 192 | } |
| 193 | |
| 194 | static uint64_t qriommu_get_pte_attrs(bool is_leaf) |
| 195 | { |
| 196 | if (!is_leaf) { |
| 197 | return QRIOMMU_NON_LEAF_PTE_MASK; |
| 198 | } |
| 199 | |
| 200 | /* For leaf PTE, set RWX permissions */ |
| 201 | return QRIOMMU_LEAF_PTE_RW_MASK; |
| 202 | } |
| 203 | |
| 204 | void qriommu_setup_translation_tables(QTestState *qts, |
| 205 | uint64_t iova, |
| 206 | QRIOMMUTransMode mode) |
| 207 | { |
| 208 | uint64_t s_root = 0, s_l0_pte_val = 0, s_l1_pte_val = 0; |
| 209 | uint64_t s_l0_addr = 0, s_l1_addr = 0, s_l2_addr = 0, s_l2_pte_val = 0; |
| 210 | uint64_t s_l0_pa = 0, s_l1_pa = 0; |
| 211 | uint64_t s_l2_pa = qriommu_apply_space_offs(QRIOMMU_L2_PTE_VAL); |
| 212 | uint64_t s_l0_pa_real = 0, s_l1_pa_real = 0; |
| 213 | uint64_t s_l2_pa_real = qriommu_apply_space_offs(QRIOMMU_L2_PTE_VAL); |
| 214 | uint64_t non_leaf_attrs = qriommu_get_pte_attrs(false); |
| 215 | uint64_t leaf_attrs = qriommu_get_pte_attrs(true); |
| 216 | |
| 217 | if (mode != QRIOMMU_TM_G_STAGE_ONLY) { |
| 218 | /* Setup S-stage 3-level page tables (SV39) */ |
| 219 | s_l0_pa = qriommu_apply_space_offs(QRIOMMU_L0_PTE_VAL); |
| 220 | s_l1_pa = qriommu_apply_space_offs(QRIOMMU_L1_PTE_VAL); |
| 221 | s_root = qriommu_apply_space_offs( |
| 222 | QRIOMMU_IOHGATP & QRIOMMU_PTE_PPN_MASK); |
| 223 | s_l2_pa = qriommu_apply_space_offs(QRIOMMU_L2_PTE_VAL); |
| 224 | |
| 225 | s_l0_pa_real = s_l0_pa; |
| 226 | s_l1_pa_real = s_l1_pa; |
| 227 | s_l2_pa_real = s_l2_pa; |
| 228 | |
| 229 | if (mode == QRIOMMU_TM_NESTED) { |
| 230 | s_l0_pa = QRIOMMU_L0_PTE_VAL; |
| 231 | s_l1_pa = QRIOMMU_L1_PTE_VAL; |
| 232 | s_l2_pa = QRIOMMU_L2_PTE_VAL; |
| 233 | |
| 234 | s_l0_pa_real = qriommu_apply_space_offs(QRIOMMU_L0_PTE_VAL); |
| 235 | s_l1_pa_real = qriommu_apply_space_offs(QRIOMMU_L1_PTE_VAL); |
| 236 | s_l2_pa_real = qriommu_apply_space_offs(QRIOMMU_L2_PTE_VAL); |
| 237 | } |
| 238 | |
| 239 | s_l0_pte_val = qriommu_encode_pte(s_l0_pa, non_leaf_attrs); |
| 240 | s_l1_pte_val = qriommu_encode_pte(s_l1_pa, non_leaf_attrs); |
| 241 | |
| 242 | s_l0_addr = qriommu_get_table_addr(s_root, 0, iova); |
| 243 | qtest_writeq(qts, s_l0_addr, s_l0_pte_val); |
| 244 | |
| 245 | s_l1_addr = qriommu_get_table_addr(s_l0_pa_real, 1, iova); |
| 246 | qtest_writeq(qts, s_l1_addr, s_l1_pte_val); |
| 247 | |
| 248 | s_l2_addr = qriommu_get_table_addr(s_l1_pa_real, 2, iova); |
| 249 | s_l2_pte_val = qriommu_encode_pte(s_l2_pa, leaf_attrs); |
| 250 | qtest_writeq(qts, s_l2_addr, s_l2_pte_val); |
| 251 | } |
| 252 | |
| 253 | if (mode == QRIOMMU_TM_G_STAGE_ONLY || mode == QRIOMMU_TM_NESTED) { |
| 254 | uint64_t g_root; |
| 255 | uint64_t g_l0_pa; |
| 256 | uint64_t g_l1_pa; |
| 257 | uint64_t g_l0_pte_val; |
| 258 | uint64_t g_l1_pte_val; |
| 259 | |
| 260 | g_root = qriommu_apply_space_offs( |
| 261 | QRIOMMU_G_IOHGATP & QRIOMMU_PTE_PPN_MASK); |
| 262 | g_l0_pa = qriommu_apply_space_offs(QRIOMMU_G_L0_PTE_VAL); |
| 263 | g_l1_pa = qriommu_apply_space_offs(QRIOMMU_G_L1_PTE_VAL); |
| 264 | g_l0_pte_val = qriommu_encode_pte(g_l0_pa, non_leaf_attrs); |
| 265 | g_l1_pte_val = qriommu_encode_pte(g_l1_pa, non_leaf_attrs); |
| 266 | |
| 267 | if (mode == QRIOMMU_TM_G_STAGE_ONLY) { |
| 268 | qriommu_map_leaf(qts, g_root, g_l0_pa, g_l1_pa, |
| 269 | g_l0_pte_val, g_l1_pte_val, |
| 270 | iova, s_l2_pa_real, leaf_attrs); |
| 271 | } else { |
| 272 | qriommu_map_leaf(qts, g_root, g_l0_pa, g_l1_pa, |
| 273 | g_l0_pte_val, g_l1_pte_val, |
| 274 | QRIOMMU_IOHGATP, s_root, leaf_attrs); |
| 275 | qriommu_map_leaf(qts, g_root, g_l0_pa, g_l1_pa, |
| 276 | g_l0_pte_val, g_l1_pte_val, |
| 277 | QRIOMMU_L0_PTE_VAL, s_l0_pa_real, leaf_attrs); |
| 278 | qriommu_map_leaf(qts, g_root, g_l0_pa, g_l1_pa, |
| 279 | g_l0_pte_val, g_l1_pte_val, |
| 280 | QRIOMMU_L1_PTE_VAL, s_l1_pa_real, leaf_attrs); |
| 281 | qriommu_map_leaf(qts, g_root, g_l0_pa, g_l1_pa, |
| 282 | g_l0_pte_val, g_l1_pte_val, |
| 283 | QRIOMMU_L2_PTE_VAL, s_l2_pa_real, leaf_attrs); |
| 284 | } |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | uint32_t qriommu_build_translation(QTestState *qts, QRIOMMUTransMode mode, |
| 289 | uint32_t device_id) |
| 290 | { |
| 291 | uint64_t dc_addr, dc_addr_real; |
| 292 | struct riscv_iommu_dc dc; |
| 293 | uint64_t iohgatp; |
| 294 | |
| 295 | qtest_memset(qts, qriommu_apply_space_offs(QRIOMMU_DDT_BASE), 0, 0x1000); |
| 296 | |
| 297 | dc_addr = device_id * sizeof(struct riscv_iommu_dc) + QRIOMMU_DC_BASE; |
| 298 | dc_addr_real = qriommu_apply_space_offs(dc_addr); |
| 299 | |
| 300 | /* Build Device Context (DC) */ |
| 301 | memset(&dc, 0, sizeof(dc)); |
| 302 | |
| 303 | switch (mode) { |
| 304 | case QRIOMMU_TM_BARE: |
| 305 | /* Pass-through mode: tc.V=1, no FSC/IOHGATP */ |
| 306 | dc.tc = RISCV_IOMMU_DC_TC_V; |
| 307 | break; |
| 308 | |
| 309 | case QRIOMMU_TM_S_STAGE_ONLY: |
| 310 | /* S-stage only: tc.V=1, set FSC */ |
| 311 | dc.tc = RISCV_IOMMU_DC_TC_V; |
| 312 | iohgatp = qriommu_apply_space_offs(QRIOMMU_IOHGATP); |
| 313 | /* FSC mode: SV39 (mode=8) */ |
| 314 | dc.fsc = (iohgatp >> 12) | (8ull << 60); |
| 315 | break; |
| 316 | |
| 317 | case QRIOMMU_TM_G_STAGE_ONLY: |
| 318 | /* G-stage only: tc.V=1, set IOHGATP */ |
| 319 | dc.tc = RISCV_IOMMU_DC_TC_V; |
| 320 | iohgatp = qriommu_apply_space_offs(QRIOMMU_G_IOHGATP); |
| 321 | /* IOHGATP mode: SV39x4 (mode=8) */ |
| 322 | dc.iohgatp = (iohgatp >> 12) | (8ull << 60); |
| 323 | break; |
| 324 | |
| 325 | case QRIOMMU_TM_NESTED: |
| 326 | /* Nested: tc.V=1, set both FSC and IOHGATP */ |
| 327 | dc.tc = RISCV_IOMMU_DC_TC_V; |
| 328 | /* FSC mode: SV39 (mode=8) */ |
| 329 | dc.fsc = (QRIOMMU_IOHGATP >> 12) | (8ull << 60); |
| 330 | /* IOHGATP mode: SV39x4 (mode=8) */ |
| 331 | iohgatp = qriommu_apply_space_offs(QRIOMMU_G_IOHGATP); |
| 332 | dc.iohgatp = (iohgatp >> 12) | (8ull << 60); |
| 333 | break; |
| 334 | |
| 335 | default: |
| 336 | g_assert_not_reached(); |
| 337 | } |
| 338 | |
| 339 | /* Write DC to memory */ |
| 340 | qtest_writeq(qts, dc_addr_real + 0, dc.tc); |
| 341 | qtest_writeq(qts, dc_addr_real + 8, dc.iohgatp); |
| 342 | qtest_writeq(qts, dc_addr_real + 16, dc.ta); |
| 343 | qtest_writeq(qts, dc_addr_real + 24, dc.fsc); |
| 344 | qtest_writeq(qts, dc_addr_real + 32, dc.msiptp); |
| 345 | qtest_writeq(qts, dc_addr_real + 40, dc.msi_addr_mask); |
| 346 | qtest_writeq(qts, dc_addr_real + 48, dc.msi_addr_pattern); |
| 347 | qtest_writeq(qts, dc_addr_real + 56, dc._reserved); |
| 348 | |
| 349 | /* Setup translation tables if not in BARE mode */ |
| 350 | if (mode != QRIOMMU_TM_BARE) { |
| 351 | qriommu_setup_translation_tables(qts, QRIOMMU_IOVA, mode); |
| 352 | } |
| 353 | |
| 354 | return 0; |
| 355 | } |
| 356 | |
| 357 | void qriommu_program_regs(QTestState *qts, uint64_t iommu_base) |
| 358 | { |
| 359 | uint64_t ddtp, cqb, fqb; |
| 360 | uint64_t cq_base, fq_base; |
| 361 | uint64_t cq_align, fq_align; |
| 362 | uint32_t cq_entries = QRIOMMU_QUEUE_ENTRIES; |
| 363 | uint32_t fq_entries = QRIOMMU_QUEUE_ENTRIES; |
| 364 | uint32_t cq_log2sz = ctz32(cq_entries) - 1; |
| 365 | uint32_t fq_log2sz = ctz32(fq_entries) - 1; |
| 366 | |
| 367 | cq_base = qriommu_apply_space_offs(QRIOMMU_CQ_BASE_ADDR); |
| 368 | fq_base = qriommu_apply_space_offs(QRIOMMU_FQ_BASE_ADDR); |
| 369 | |
| 370 | cq_align = MAX(0x1000ull, (uint64_t)cq_entries * QRIOMMU_CQ_ENTRY_SIZE); |
| 371 | fq_align = MAX(0x1000ull, (uint64_t)fq_entries * QRIOMMU_FQ_ENTRY_SIZE); |
| 372 | g_assert((cq_base & (cq_align - 1)) == 0); |
| 373 | g_assert((fq_base & (fq_align - 1)) == 0); |
| 374 | |
| 375 | /* Setup Command Queue */ |
| 376 | cqb = (cq_base >> 12) << 10 | cq_log2sz; |
| 377 | qtest_writeq(qts, iommu_base + RISCV_IOMMU_REG_CQB, cqb); |
| 378 | qtest_writel(qts, iommu_base + RISCV_IOMMU_REG_CQH, 0); |
| 379 | qtest_writel(qts, iommu_base + RISCV_IOMMU_REG_CQT, 0); |
| 380 | qtest_writel(qts, iommu_base + RISCV_IOMMU_REG_CQCSR, |
| 381 | RISCV_IOMMU_CQCSR_CQEN); |
| 382 | qriommu_wait_for_queue_active(qts, iommu_base, RISCV_IOMMU_REG_CQCSR, |
| 383 | RISCV_IOMMU_CQCSR_CQON); |
| 384 | |
| 385 | /* Setup Fault Queue */ |
| 386 | fqb = (fq_base >> 12) << 10 | fq_log2sz; |
| 387 | qtest_writeq(qts, iommu_base + RISCV_IOMMU_REG_FQB, fqb); |
| 388 | qtest_writel(qts, iommu_base + RISCV_IOMMU_REG_FQH, 0); |
| 389 | qtest_writel(qts, iommu_base + RISCV_IOMMU_REG_FQT, 0); |
| 390 | qtest_writel(qts, iommu_base + RISCV_IOMMU_REG_FQCSR, |
| 391 | RISCV_IOMMU_FQCSR_FQEN); |
| 392 | qriommu_wait_for_queue_active(qts, iommu_base, RISCV_IOMMU_REG_FQCSR, |
| 393 | RISCV_IOMMU_FQCSR_FQON); |
| 394 | |
| 395 | /* Set Device Directory Table Pointer (DDTP) */ |
| 396 | ddtp = qriommu_apply_space_offs(QRIOMMU_DDT_BASE); |
| 397 | g_assert((ddtp & 0xfff) == 0); |
| 398 | ddtp = ((ddtp >> 12) << 10) | RISCV_IOMMU_DDTP_MODE_1LVL; |
| 399 | qtest_writeq(qts, iommu_base + RISCV_IOMMU_REG_DDTP, ddtp); |
| 400 | g_assert((qtest_readq(qts, iommu_base + RISCV_IOMMU_REG_DDTP) & |
| 401 | (RISCV_IOMMU_DDTP_PPN | RISCV_IOMMU_DDTP_MODE)) == |
| 402 | (ddtp & (RISCV_IOMMU_DDTP_PPN | RISCV_IOMMU_DDTP_MODE))); |
| 403 | } |