| 1 | /* |
| 2 | * Copyright (c) 2025 Huawei Technologies R & D (UK) Ltd |
| 3 | * Copyright (C) 2025 NVIDIA |
| 4 | * Written by Nicolin Chen, Shameer Kolothum |
| 5 | * |
| 6 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 7 | */ |
| 8 | |
| 9 | #include "qemu/osdep.h" |
| 10 | #include "qemu/error-report.h" |
| 11 | #include "trace.h" |
| 12 | |
| 13 | #include "hw/arm/smmuv3.h" |
| 14 | #include "hw/core/iommu.h" |
| 15 | #include "hw/pci/pci_bridge.h" |
| 16 | #include "hw/pci-host/gpex.h" |
| 17 | #include "hw/vfio/pci.h" |
| 18 | |
| 19 | #include "smmuv3-internal.h" |
| 20 | #include "smmuv3-accel.h" |
| 21 | #include "system/system.h" |
| 22 | #include "tegra241-cmdqv.h" |
| 23 | |
| 24 | /* |
| 25 | * The root region aliases the global system memory, and shared_as_sysmem |
| 26 | * provides a shared Address Space referencing it. This Address Space is used |
| 27 | * by all vfio-pci devices behind all accelerated SMMUv3 instances within a VM. |
| 28 | */ |
| 29 | static MemoryRegion root, sysmem; |
| 30 | static AddressSpace *shared_as_sysmem; |
| 31 | |
| 32 | static int smmuv3_oas_bits(uint32_t oas) |
| 33 | { |
| 34 | static const int map[] = { 32, 36, 40, 42, 44, 48, 52, 56 }; |
| 35 | |
| 36 | g_assert(oas < ARRAY_SIZE(map)); |
| 37 | return map[oas]; |
| 38 | } |
| 39 | |
| 40 | static void smmuv3_accel_auto_finalise(SMMUv3State *s, |
| 41 | struct iommu_hw_info_arm_smmuv3 *info) |
| 42 | { |
| 43 | SMMUv3AccelState *accel = s->s_accel; |
| 44 | |
| 45 | /* |
| 46 | * Return if 'auto' was not set for any accel SMMUv3 property, or |
| 47 | * if property values were already resolved from a previous call |
| 48 | * to this function (e.g. if this function was called again after |
| 49 | * VM boot during device hot plug). We do not accept new property |
| 50 | * values in this case where auto_finalised == true, and we re-use |
| 51 | * the values determined from the initial cold plug. |
| 52 | */ |
| 53 | if (!accel->auto_mode || accel->auto_finalised) { |
| 54 | return; |
| 55 | } |
| 56 | |
| 57 | if (s->ats == ON_OFF_AUTO_AUTO) { |
| 58 | s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ATS, |
| 59 | FIELD_EX32(info->idr[0], IDR0, ATS)); |
| 60 | } |
| 61 | |
| 62 | if (s->ril == ON_OFF_AUTO_AUTO) { |
| 63 | s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, |
| 64 | FIELD_EX32(info->idr[3], IDR3, RIL)); |
| 65 | } |
| 66 | |
| 67 | if (s->ssidsize == SSID_SIZE_MODE_AUTO) { |
| 68 | s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SSIDSIZE, |
| 69 | FIELD_EX32(info->idr[1], IDR1, SSIDSIZE)); |
| 70 | } |
| 71 | |
| 72 | if (s->oas == OAS_MODE_AUTO) { |
| 73 | s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, |
| 74 | FIELD_EX32(info->idr[5], IDR5, OAS)); |
| 75 | } |
| 76 | |
| 77 | accel->auto_finalised = true; |
| 78 | } |
| 79 | |
| 80 | static bool |
| 81 | smmuv3_accel_check_hw_compatible(SMMUv3State *s, |
| 82 | struct iommu_hw_info_arm_smmuv3 *info, |
| 83 | Error **errp) |
| 84 | { |
| 85 | smmuv3_accel_auto_finalise(s, info); |
| 86 | |
| 87 | /* QEMU SMMUv3 supports both linear and 2-level stream tables */ |
| 88 | if (FIELD_EX32(info->idr[0], IDR0, STLEVEL) != |
| 89 | FIELD_EX32(s->idr[0], IDR0, STLEVEL)) { |
| 90 | error_setg(errp, "Host SMMUv3 Stream Table format mismatch " |
| 91 | "(host STLEVEL=%u, QEMU STLEVEL=%u)", |
| 92 | FIELD_EX32(info->idr[0], IDR0, STLEVEL), |
| 93 | FIELD_EX32(s->idr[0], IDR0, STLEVEL)); |
| 94 | return false; |
| 95 | } |
| 96 | |
| 97 | /* QEMU SMMUv3 supports only little-endian translation table walks */ |
| 98 | if (FIELD_EX32(info->idr[0], IDR0, TTENDIAN) > |
| 99 | FIELD_EX32(s->idr[0], IDR0, TTENDIAN)) { |
| 100 | error_setg(errp, "Host SMMUv3 doesn't support Little-endian " |
| 101 | "translation table"); |
| 102 | return false; |
| 103 | } |
| 104 | |
| 105 | /* QEMU SMMUv3 supports only AArch64 translation table format */ |
| 106 | if (FIELD_EX32(info->idr[0], IDR0, TTF) < |
| 107 | FIELD_EX32(s->idr[0], IDR0, TTF)) { |
| 108 | error_setg(errp, "Host SMMUv3 doesn't support AArch64 translation " |
| 109 | "table format"); |
| 110 | return false; |
| 111 | } |
| 112 | |
| 113 | /* QEMU SMMUv3 supports SIDSIZE 16 */ |
| 114 | if (FIELD_EX32(info->idr[1], IDR1, SIDSIZE) < |
| 115 | FIELD_EX32(s->idr[1], IDR1, SIDSIZE)) { |
| 116 | error_setg(errp, "Host SMMUv3 SIDSIZE not compatible " |
| 117 | "(host=%u, QEMU=%u)", |
| 118 | FIELD_EX32(info->idr[1], IDR1, SIDSIZE), |
| 119 | FIELD_EX32(s->idr[1], IDR1, SIDSIZE)); |
| 120 | return false; |
| 121 | } |
| 122 | |
| 123 | /* Check SSIDSIZE value opted-in is compatible with Host SMMUv3 SSIDSIZE */ |
| 124 | if (FIELD_EX32(info->idr[1], IDR1, SSIDSIZE) < |
| 125 | FIELD_EX32(s->idr[1], IDR1, SSIDSIZE)) { |
| 126 | error_setg(errp, "Host SMMUv3 SSIDSIZE not compatible " |
| 127 | "(host=%u, QEMU=%u)", |
| 128 | FIELD_EX32(info->idr[1], IDR1, SSIDSIZE), |
| 129 | FIELD_EX32(s->idr[1], IDR1, SSIDSIZE)); |
| 130 | return false; |
| 131 | } |
| 132 | |
| 133 | /* User can disable QEMU SMMUv3 Range Invalidation support */ |
| 134 | if (FIELD_EX32(info->idr[3], IDR3, RIL) < |
| 135 | FIELD_EX32(s->idr[3], IDR3, RIL)) { |
| 136 | error_setg(errp, "Host SMMUv3 doesn't support Range Invalidation"); |
| 137 | return false; |
| 138 | } |
| 139 | /* Check OAS value opted is compatible with Host SMMUv3 IPA */ |
| 140 | if (FIELD_EX32(info->idr[5], IDR5, OAS) < |
| 141 | FIELD_EX32(s->idr[5], IDR5, OAS)) { |
| 142 | error_setg(errp, "Host SMMUv3 supports only %d-bit IPA, but the vSMMU " |
| 143 | "OAS implies %d-bit IPA", |
| 144 | smmuv3_oas_bits(FIELD_EX32(info->idr[5], IDR5, OAS)), |
| 145 | smmuv3_oas_bits(FIELD_EX32(s->idr[5], IDR5, OAS))); |
| 146 | return false; |
| 147 | } |
| 148 | /* Check ATS value opted is compatible with Host SMMUv3 */ |
| 149 | if (FIELD_EX32(info->idr[0], IDR0, ATS) < |
| 150 | FIELD_EX32(s->idr[0], IDR0, ATS)) { |
| 151 | error_setg(errp, "Host SMMUv3 doesn't support Address Translation Services"); |
| 152 | return false; |
| 153 | } |
| 154 | |
| 155 | /* QEMU SMMUv3 supports GRAN4K/GRAN16K/GRAN64K translation granules */ |
| 156 | if (FIELD_EX32(info->idr[5], IDR5, GRAN4K) != |
| 157 | FIELD_EX32(s->idr[5], IDR5, GRAN4K)) { |
| 158 | error_setg(errp, "Host SMMUv3 doesn't support 4K translation granule"); |
| 159 | return false; |
| 160 | } |
| 161 | if (FIELD_EX32(info->idr[5], IDR5, GRAN16K) != |
| 162 | FIELD_EX32(s->idr[5], IDR5, GRAN16K)) { |
| 163 | error_setg(errp, "Host SMMUv3 doesn't support 16K translation granule"); |
| 164 | return false; |
| 165 | } |
| 166 | if (FIELD_EX32(info->idr[5], IDR5, GRAN64K) != |
| 167 | FIELD_EX32(s->idr[5], IDR5, GRAN64K)) { |
| 168 | error_setg(errp, "Host SMMUv3 doesn't support 64K translation granule"); |
| 169 | return false; |
| 170 | } |
| 171 | |
| 172 | return true; |
| 173 | } |
| 174 | |
| 175 | static bool |
| 176 | smmuv3_accel_hw_compatible(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *hiodi, |
| 177 | Error **errp) |
| 178 | { |
| 179 | struct iommu_hw_info_arm_smmuv3 info; |
| 180 | uint32_t data_type = IOMMU_HW_INFO_TYPE_DEFAULT; |
| 181 | uint64_t caps; |
| 182 | |
| 183 | if (!iommufd_backend_get_device_info(hiodi->iommufd, hiodi->devid, |
| 184 | &data_type, &info, sizeof(info), &caps, |
| 185 | NULL, errp)) { |
| 186 | return false; |
| 187 | } |
| 188 | |
| 189 | if (data_type != IOMMU_HW_INFO_TYPE_ARM_SMMUV3) { |
| 190 | error_setg(errp, "Wrong data type (%d) for Host SMMUv3 device info", |
| 191 | data_type); |
| 192 | return false; |
| 193 | } |
| 194 | |
| 195 | if (!smmuv3_accel_check_hw_compatible(s, &info, errp)) { |
| 196 | return false; |
| 197 | } |
| 198 | return true; |
| 199 | } |
| 200 | |
| 201 | static SMMUv3AccelDevice *smmuv3_accel_get_dev(SMMUState *bs, SMMUPciBus *sbus, |
| 202 | PCIBus *bus, int devfn) |
| 203 | { |
| 204 | SMMUDevice *sdev = sbus->pbdev[devfn]; |
| 205 | SMMUv3AccelDevice *accel_dev; |
| 206 | |
| 207 | if (sdev) { |
| 208 | return container_of(sdev, SMMUv3AccelDevice, sdev); |
| 209 | } |
| 210 | |
| 211 | accel_dev = g_new0(SMMUv3AccelDevice, 1); |
| 212 | sdev = &accel_dev->sdev; |
| 213 | |
| 214 | sbus->pbdev[devfn] = sdev; |
| 215 | smmu_init_sdev(bs, sdev, bus, devfn); |
| 216 | return accel_dev; |
| 217 | } |
| 218 | |
| 219 | static uint32_t smmuv3_accel_gbpa_hwpt(SMMUv3State *s, SMMUv3AccelState *accel) |
| 220 | { |
| 221 | return FIELD_EX32(s->gbpa, GBPA, ABORT) ? |
| 222 | accel->abort_hwpt_id : accel->bypass_hwpt_id; |
| 223 | } |
| 224 | |
| 225 | static bool |
| 226 | smmuv3_accel_alloc_vdev(SMMUv3AccelDevice *accel_dev, int sid, Error **errp) |
| 227 | { |
| 228 | SMMUv3AccelState *accel = accel_dev->s_accel; |
| 229 | HostIOMMUDeviceIOMMUFD *hiodi = accel_dev->hiodi; |
| 230 | IOMMUFDVdev *vdev = accel_dev->vdev; |
| 231 | uint32_t vdevice_id; |
| 232 | |
| 233 | if (!hiodi || vdev) { |
| 234 | return true; |
| 235 | } |
| 236 | |
| 237 | if (!iommufd_backend_alloc_vdev(hiodi->iommufd, hiodi->devid, |
| 238 | accel->viommu->viommu_id, sid, |
| 239 | &vdevice_id, errp)) { |
| 240 | return false; |
| 241 | } |
| 242 | |
| 243 | vdev = g_new(IOMMUFDVdev, 1); |
| 244 | vdev->vdevice_id = vdevice_id; |
| 245 | vdev->virt_id = sid; |
| 246 | accel_dev->vdev = vdev; |
| 247 | return true; |
| 248 | } |
| 249 | |
| 250 | static SMMUS1Hwpt * |
| 251 | smmuv3_accel_dev_alloc_translate(SMMUv3AccelDevice *accel_dev, STE *ste, |
| 252 | Error **errp) |
| 253 | { |
| 254 | uint64_t ste_0 = (uint64_t)ste->word[0] | (uint64_t)ste->word[1] << 32; |
| 255 | uint64_t ste_1 = (uint64_t)ste->word[2] | (uint64_t)ste->word[3] << 32; |
| 256 | HostIOMMUDeviceIOMMUFD *hiodi = accel_dev->hiodi; |
| 257 | SMMUv3AccelState *accel = accel_dev->s_accel; |
| 258 | struct iommu_hwpt_arm_smmuv3 nested_data = { |
| 259 | .ste = { |
| 260 | cpu_to_le64(ste_0 & STE0_MASK), |
| 261 | cpu_to_le64(ste_1 & STE1_MASK), |
| 262 | }, |
| 263 | }; |
| 264 | uint32_t hwpt_id = 0, flags = 0; |
| 265 | SMMUS1Hwpt *s1_hwpt; |
| 266 | |
| 267 | if (!iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid, |
| 268 | accel->viommu->viommu_id, flags, |
| 269 | IOMMU_HWPT_DATA_ARM_SMMUV3, |
| 270 | sizeof(nested_data), &nested_data, |
| 271 | &hwpt_id, errp)) { |
| 272 | return NULL; |
| 273 | } |
| 274 | |
| 275 | s1_hwpt = g_new0(SMMUS1Hwpt, 1); |
| 276 | s1_hwpt->hwpt_id = hwpt_id; |
| 277 | trace_smmuv3_accel_translate_ste(accel_dev->vdev->virt_id, hwpt_id, |
| 278 | nested_data.ste[1], nested_data.ste[0]); |
| 279 | return s1_hwpt; |
| 280 | } |
| 281 | |
| 282 | bool smmuv3_accel_install_ste(SMMUv3State *s, SMMUDevice *sdev, int sid, |
| 283 | Error **errp) |
| 284 | { |
| 285 | SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid, |
| 286 | .inval_ste_allowed = true}; |
| 287 | SMMUv3AccelState *accel = s->s_accel; |
| 288 | SMMUv3AccelDevice *accel_dev; |
| 289 | HostIOMMUDeviceIOMMUFD *hiodi; |
| 290 | uint32_t config, hwpt_id = 0; |
| 291 | SMMUS1Hwpt *s1_hwpt = NULL; |
| 292 | const char *type; |
| 293 | STE ste; |
| 294 | |
| 295 | if (!accel || !accel->viommu) { |
| 296 | return true; |
| 297 | } |
| 298 | |
| 299 | accel_dev = container_of(sdev, SMMUv3AccelDevice, sdev); |
| 300 | if (!accel_dev->s_accel) { |
| 301 | return true; |
| 302 | } |
| 303 | |
| 304 | hiodi = accel_dev->hiodi; |
| 305 | if (!smmuv3_accel_alloc_vdev(accel_dev, sid, errp)) { |
| 306 | return false; |
| 307 | } |
| 308 | |
| 309 | if (smmu_find_ste(sdev->smmu, sid, &ste, &event)) { |
| 310 | /* No STE found, nothing to install */ |
| 311 | return true; |
| 312 | } |
| 313 | |
| 314 | /* |
| 315 | * Install the STE based on SMMU enabled/config: |
| 316 | * - attach a pre-allocated HWPT for abort/bypass |
| 317 | * - or a new HWPT for translate STE |
| 318 | * |
| 319 | * Note: The vdev remains associated with accel_dev even if HWPT |
| 320 | * attach/alloc fails, since the Guest–Host SID mapping stays |
| 321 | * valid as long as the device is behind the accelerated SMMUv3. |
| 322 | */ |
| 323 | if (!smmu_enabled(s)) { |
| 324 | hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel); |
| 325 | } else { |
| 326 | config = STE_CONFIG(&ste); |
| 327 | |
| 328 | if (!STE_VALID(&ste) || STE_CFG_ABORT(config)) { |
| 329 | hwpt_id = accel->abort_hwpt_id; |
| 330 | } else if (STE_CFG_BYPASS(config)) { |
| 331 | hwpt_id = accel->bypass_hwpt_id; |
| 332 | } else if (STE_CFG_S1_TRANSLATE(config)) { |
| 333 | s1_hwpt = smmuv3_accel_dev_alloc_translate(accel_dev, &ste, errp); |
| 334 | if (!s1_hwpt) { |
| 335 | return false; |
| 336 | } |
| 337 | hwpt_id = s1_hwpt->hwpt_id; |
| 338 | } |
| 339 | } |
| 340 | |
| 341 | if (!hwpt_id) { |
| 342 | error_setg(errp, "Invalid STE config for sid 0x%x", |
| 343 | smmu_get_sid(&accel_dev->sdev)); |
| 344 | return false; |
| 345 | } |
| 346 | |
| 347 | if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, hwpt_id, |
| 348 | errp)) { |
| 349 | if (s1_hwpt) { |
| 350 | iommufd_backend_free_id(hiodi->iommufd, s1_hwpt->hwpt_id); |
| 351 | g_free(s1_hwpt); |
| 352 | } |
| 353 | return false; |
| 354 | } |
| 355 | |
| 356 | /* Free the previous s1_hwpt */ |
| 357 | if (accel_dev->s1_hwpt) { |
| 358 | iommufd_backend_free_id(hiodi->iommufd, accel_dev->s1_hwpt->hwpt_id); |
| 359 | g_free(accel_dev->s1_hwpt); |
| 360 | } |
| 361 | |
| 362 | accel_dev->s1_hwpt = s1_hwpt; |
| 363 | if (hwpt_id == accel->abort_hwpt_id) { |
| 364 | type = "abort"; |
| 365 | } else if (hwpt_id == accel->bypass_hwpt_id) { |
| 366 | type = "bypass"; |
| 367 | } else { |
| 368 | type = "translate"; |
| 369 | } |
| 370 | |
| 371 | trace_smmuv3_accel_install_ste(sid, type, hwpt_id); |
| 372 | return true; |
| 373 | } |
| 374 | |
| 375 | bool smmuv3_accel_install_ste_range(SMMUv3State *s, SMMUSIDRange *range, |
| 376 | Error **errp) |
| 377 | { |
| 378 | SMMUv3AccelState *accel = s->s_accel; |
| 379 | SMMUv3AccelDevice *accel_dev; |
| 380 | Error *local_err = NULL; |
| 381 | bool all_ok = true; |
| 382 | |
| 383 | if (!accel || !accel->viommu) { |
| 384 | return true; |
| 385 | } |
| 386 | |
| 387 | QLIST_FOREACH(accel_dev, &accel->device_list, next) { |
| 388 | uint32_t sid = smmu_get_sid(&accel_dev->sdev); |
| 389 | |
| 390 | if (sid >= range->start && sid <= range->end) { |
| 391 | if (!smmuv3_accel_install_ste(s, &accel_dev->sdev, |
| 392 | sid, &local_err)) { |
| 393 | error_append_hint(&local_err, "Device 0x%x: Failed to install " |
| 394 | "STE\n", sid); |
| 395 | error_report_err(local_err); |
| 396 | local_err = NULL; |
| 397 | all_ok = false; |
| 398 | } |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | if (!all_ok) { |
| 403 | error_setg(errp, "Failed to install all STEs properly"); |
| 404 | } |
| 405 | return all_ok; |
| 406 | } |
| 407 | |
| 408 | /* |
| 409 | * This issues the invalidation cmd to the host SMMUv3. |
| 410 | * |
| 411 | * sdev is non-NULL for SID based invalidations (e.g. CFGI_CD), and NULL for |
| 412 | * non SID invalidations such as SMMU_CMD_TLBI_NH_ASID and SMMU_CMD_TLBI_NH_VA. |
| 413 | */ |
| 414 | bool smmuv3_accel_issue_inv_cmd(SMMUv3State *bs, void *cmd, SMMUDevice *sdev, |
| 415 | Error **errp) |
| 416 | { |
| 417 | SMMUv3State *s = ARM_SMMUV3(bs); |
| 418 | SMMUv3AccelState *accel = s->s_accel; |
| 419 | uint32_t entry_num = 1; |
| 420 | |
| 421 | /* |
| 422 | * No accel or viommu means no VFIO/IOMMUFD devices, nothing to |
| 423 | * invalidate. |
| 424 | */ |
| 425 | if (!accel || !accel->viommu) { |
| 426 | return true; |
| 427 | } |
| 428 | |
| 429 | /* |
| 430 | * SID based invalidations (e.g. CFGI_CD) apply only to vfio-pci endpoints |
| 431 | * with a valid vIOMMU vdev. |
| 432 | */ |
| 433 | if (sdev && !container_of(sdev, SMMUv3AccelDevice, sdev)->vdev) { |
| 434 | return true; |
| 435 | } |
| 436 | |
| 437 | /* Single command (entry_num = 1); no need to check returned entry_num */ |
| 438 | return iommufd_backend_invalidate_cache( |
| 439 | accel->viommu->iommufd, accel->viommu->viommu_id, |
| 440 | IOMMU_VIOMMU_INVALIDATE_DATA_ARM_SMMUV3, |
| 441 | sizeof(Cmd), &entry_num, cmd, errp); |
| 442 | } |
| 443 | |
| 444 | /* |
| 445 | * Returns 0 on success (buf is populated and valid). |
| 446 | * Returns 1 if the read should be retried (EAGAIN/EINTR). |
| 447 | * Returns -1 on error with @errp set. |
| 448 | */ |
| 449 | int smmuv3_accel_event_read_validate(IOMMUFDVeventq *veventq, uint32_t type, |
| 450 | void *buf, size_t size, Error **errp) |
| 451 | { |
| 452 | uint32_t last_seq = veventq->last_event_seq; |
| 453 | uint32_t id = veventq->veventq_id; |
| 454 | struct iommufd_vevent_header *hdr; |
| 455 | ssize_t bytes; |
| 456 | |
| 457 | bytes = read(veventq->veventq_fd, buf, size); |
| 458 | if (bytes <= 0) { |
| 459 | if (errno == EAGAIN || errno == EINTR) { |
| 460 | return 1; |
| 461 | } |
| 462 | error_setg(errp, "vEVENTQ(type %u id %u): read failed (%m)", type, id); |
| 463 | return -1; |
| 464 | } |
| 465 | hdr = (struct iommufd_vevent_header *)buf; |
| 466 | if (bytes == sizeof(*hdr) && |
| 467 | (hdr->flags & IOMMU_VEVENTQ_FLAG_LOST_EVENTS)) { |
| 468 | error_setg(errp, "vEVENTQ(type %u id %u): overflowed", type, id); |
| 469 | veventq->event_start = false; |
| 470 | return -1; |
| 471 | } |
| 472 | if (bytes < size) { |
| 473 | error_setg(errp, "vEVENTQ(type %u id %u): short read(%zd/%zd bytes)", |
| 474 | type, id, bytes, size); |
| 475 | return -1; |
| 476 | } |
| 477 | /* Check sequence in hdr for lost events if any */ |
| 478 | if (veventq->event_start && (hdr->sequence - last_seq != 1)) { |
| 479 | warn_report("vEVENTQ(type %u id %u): lost %u event(s)", |
| 480 | type, id, hdr->sequence - last_seq - 1); |
| 481 | } |
| 482 | veventq->last_event_seq = hdr->sequence; |
| 483 | veventq->event_start = true; |
| 484 | return 0; |
| 485 | } |
| 486 | |
| 487 | static void smmuv3_accel_event_read(void *opaque) |
| 488 | { |
| 489 | SMMUv3State *s = opaque; |
| 490 | IOMMUFDVeventq *veventq = s->s_accel->veventq; |
| 491 | struct { |
| 492 | struct iommufd_vevent_header hdr; |
| 493 | struct iommu_vevent_arm_smmuv3 vevent; |
| 494 | } buf; |
| 495 | Error *local_err = NULL; |
| 496 | int ret; |
| 497 | |
| 498 | ret = smmuv3_accel_event_read_validate(veventq, |
| 499 | IOMMU_VEVENTQ_TYPE_ARM_SMMUV3, &buf, |
| 500 | sizeof(buf), &local_err); |
| 501 | if (ret < 0) { |
| 502 | warn_report_err_once(local_err); |
| 503 | return; |
| 504 | } |
| 505 | if (ret > 0) { |
| 506 | return; /* EAGAIN/EINTR */ |
| 507 | } |
| 508 | smmuv3_propagate_event(s, (Evt *)&buf.vevent); |
| 509 | } |
| 510 | |
| 511 | static void smmuv3_accel_free_veventq(SMMUv3AccelState *accel) |
| 512 | { |
| 513 | IOMMUFDVeventq *veventq = accel->veventq; |
| 514 | |
| 515 | if (!veventq) { |
| 516 | return; |
| 517 | } |
| 518 | qemu_set_fd_handler(veventq->veventq_fd, NULL, NULL, NULL); |
| 519 | close(veventq->veventq_fd); |
| 520 | iommufd_backend_free_id(accel->viommu->iommufd, veventq->veventq_id); |
| 521 | g_free(veventq); |
| 522 | accel->veventq = NULL; |
| 523 | } |
| 524 | |
| 525 | static void smmuv3_accel_free_viommu(SMMUv3AccelState *accel) |
| 526 | { |
| 527 | IOMMUFDViommu *viommu = accel->viommu; |
| 528 | |
| 529 | if (!viommu) { |
| 530 | return; |
| 531 | } |
| 532 | smmuv3_accel_free_veventq(accel); |
| 533 | iommufd_backend_free_id(viommu->iommufd, accel->bypass_hwpt_id); |
| 534 | iommufd_backend_free_id(viommu->iommufd, accel->abort_hwpt_id); |
| 535 | iommufd_backend_free_id(viommu->iommufd, accel->viommu->viommu_id); |
| 536 | g_free(viommu); |
| 537 | accel->viommu = NULL; |
| 538 | } |
| 539 | |
| 540 | bool smmuv3_accel_alloc_veventq(SMMUv3State *s, Error **errp) |
| 541 | { |
| 542 | SMMUv3AccelState *accel = s->s_accel; |
| 543 | IOMMUFDVeventq *veventq; |
| 544 | uint32_t veventq_id; |
| 545 | uint32_t veventq_fd; |
| 546 | int flags; |
| 547 | |
| 548 | if (!accel || !accel->viommu) { |
| 549 | return true; |
| 550 | } |
| 551 | |
| 552 | if (accel->veventq) { |
| 553 | return true; |
| 554 | } |
| 555 | |
| 556 | if (!smmuv3_eventq_enabled(s)) { |
| 557 | return true; |
| 558 | } |
| 559 | |
| 560 | if (!iommufd_backend_alloc_veventq(accel->viommu->iommufd, |
| 561 | accel->viommu->viommu_id, |
| 562 | IOMMU_VEVENTQ_TYPE_ARM_SMMUV3, |
| 563 | 1 << s->eventq.log2size, &veventq_id, |
| 564 | &veventq_fd, errp)) { |
| 565 | return false; |
| 566 | } |
| 567 | |
| 568 | flags = fcntl(veventq_fd, F_GETFL); |
| 569 | if (flags < 0) { |
| 570 | error_setg_errno(errp, errno, "Failed to get flags for vEVENTQ fd"); |
| 571 | goto free_veventq; |
| 572 | } |
| 573 | if (fcntl(veventq_fd, F_SETFL, flags | O_NONBLOCK) < 0) { |
| 574 | error_setg_errno(errp, errno, "Failed to set O_NONBLOCK on vEVENTQ fd"); |
| 575 | goto free_veventq; |
| 576 | } |
| 577 | |
| 578 | veventq = g_new0(IOMMUFDVeventq, 1); |
| 579 | veventq->veventq_id = veventq_id; |
| 580 | veventq->veventq_fd = veventq_fd; |
| 581 | accel->veventq = veventq; |
| 582 | |
| 583 | /* Set up event handler for veventq fd */ |
| 584 | qemu_set_fd_handler(veventq_fd, smmuv3_accel_event_read, NULL, s); |
| 585 | return true; |
| 586 | |
| 587 | free_veventq: |
| 588 | close(veventq_fd); |
| 589 | iommufd_backend_free_id(accel->viommu->iommufd, veventq_id); |
| 590 | return false; |
| 591 | } |
| 592 | |
| 593 | static bool |
| 594 | smmuv3_accel_alloc_viommu(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *hiodi, |
| 595 | Error **errp) |
| 596 | { |
| 597 | SMMUv3AccelState *accel = s->s_accel; |
| 598 | const SMMUv3AccelCmdqvOps *cmdqv_ops = accel->cmdqv_ops; |
| 599 | struct iommu_hwpt_arm_smmuv3 bypass_data = { |
| 600 | .ste = { SMMU_STE_CFG_BYPASS | SMMU_STE_VALID, 0x0ULL }, |
| 601 | }; |
| 602 | struct iommu_hwpt_arm_smmuv3 abort_data = { |
| 603 | .ste = { SMMU_STE_VALID, 0x0ULL }, |
| 604 | }; |
| 605 | uint32_t s2_hwpt_id = hiodi->hwpt_id; |
| 606 | uint32_t viommu_id, hwpt_id; |
| 607 | IOMMUFDViommu *viommu; |
| 608 | |
| 609 | if (cmdqv_ops) { |
| 610 | if (!cmdqv_ops->alloc_viommu(s, hiodi, &viommu_id, errp)) { |
| 611 | return false; |
| 612 | } |
| 613 | } else { |
| 614 | if (!iommufd_backend_alloc_viommu(hiodi->iommufd, hiodi->devid, |
| 615 | IOMMU_VIOMMU_TYPE_ARM_SMMUV3, |
| 616 | s2_hwpt_id, NULL, 0, &viommu_id, |
| 617 | errp)) { |
| 618 | return false; |
| 619 | } |
| 620 | } |
| 621 | |
| 622 | viommu = g_new0(IOMMUFDViommu, 1); |
| 623 | viommu->viommu_id = viommu_id; |
| 624 | viommu->s2_hwpt_id = s2_hwpt_id; |
| 625 | viommu->iommufd = hiodi->iommufd; |
| 626 | accel->viommu = viommu; |
| 627 | |
| 628 | /* |
| 629 | * Pre-allocate HWPTs for S1 bypass and abort cases. These will be attached |
| 630 | * later for guest STEs or GBPAs that require bypass or abort configuration. |
| 631 | */ |
| 632 | if (!iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid, viommu_id, |
| 633 | 0, IOMMU_HWPT_DATA_ARM_SMMUV3, |
| 634 | sizeof(abort_data), &abort_data, |
| 635 | &accel->abort_hwpt_id, errp)) { |
| 636 | goto free_viommu; |
| 637 | } |
| 638 | |
| 639 | if (!iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid, viommu_id, |
| 640 | 0, IOMMU_HWPT_DATA_ARM_SMMUV3, |
| 641 | sizeof(bypass_data), &bypass_data, |
| 642 | &accel->bypass_hwpt_id, errp)) { |
| 643 | goto free_abort_hwpt; |
| 644 | } |
| 645 | |
| 646 | /* Allocate a vEVENTQ if guest has enabled event queue */ |
| 647 | if (!smmuv3_accel_alloc_veventq(s, errp)) { |
| 648 | goto free_bypass_hwpt; |
| 649 | } |
| 650 | |
| 651 | /* Attach a HWPT based on SMMUv3 GBPA.ABORT value */ |
| 652 | hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel); |
| 653 | if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, hwpt_id, |
| 654 | errp)) { |
| 655 | goto free_veventq; |
| 656 | } |
| 657 | return true; |
| 658 | |
| 659 | free_veventq: |
| 660 | smmuv3_accel_free_veventq(accel); |
| 661 | free_bypass_hwpt: |
| 662 | iommufd_backend_free_id(hiodi->iommufd, accel->bypass_hwpt_id); |
| 663 | free_abort_hwpt: |
| 664 | iommufd_backend_free_id(hiodi->iommufd, accel->abort_hwpt_id); |
| 665 | free_viommu: |
| 666 | if (cmdqv_ops && cmdqv_ops->free_viommu) { |
| 667 | cmdqv_ops->free_viommu(s); |
| 668 | } else { |
| 669 | iommufd_backend_free_id(hiodi->iommufd, viommu->viommu_id); |
| 670 | } |
| 671 | g_free(viommu); |
| 672 | accel->viommu = NULL; |
| 673 | return false; |
| 674 | } |
| 675 | |
| 676 | static const SMMUv3AccelCmdqvOps * |
| 677 | smmuv3_accel_probe_cmdqv(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *idev, |
| 678 | Error **errp) |
| 679 | { |
| 680 | const SMMUv3AccelCmdqvOps *ops = tegra241_cmdqv_get_ops(); |
| 681 | |
| 682 | if (!ops) { |
| 683 | error_setg(errp, "No CMDQV ops found"); |
| 684 | return NULL; |
| 685 | } |
| 686 | g_assert(ops->probe); |
| 687 | g_assert(ops->alloc_viommu); |
| 688 | |
| 689 | if (!ops->probe(s, idev, errp)) { |
| 690 | return NULL; |
| 691 | } |
| 692 | return ops; |
| 693 | } |
| 694 | |
| 695 | static bool |
| 696 | smmuv3_accel_select_cmdqv(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *idev, |
| 697 | Error **errp) |
| 698 | { |
| 699 | const SMMUv3AccelCmdqvOps *ops = NULL; |
| 700 | |
| 701 | if (s->s_accel->cmdqv_ops) { |
| 702 | return true; |
| 703 | } |
| 704 | |
| 705 | switch (s->cmdqv) { |
| 706 | case ON_OFF_AUTO_OFF: |
| 707 | s->s_accel->cmdqv_ops = NULL; |
| 708 | return true; |
| 709 | case ON_OFF_AUTO_AUTO: |
| 710 | ops = smmuv3_accel_probe_cmdqv(s, idev, NULL); |
| 711 | break; |
| 712 | case ON_OFF_AUTO_ON: |
| 713 | ops = smmuv3_accel_probe_cmdqv(s, idev, errp); |
| 714 | if (!ops) { |
| 715 | error_append_hint(errp, "CMDQV requested but not supported"); |
| 716 | return false; |
| 717 | } |
| 718 | break; |
| 719 | default: |
| 720 | g_assert_not_reached(); |
| 721 | } |
| 722 | |
| 723 | if (ops && ops->init && !ops->init(s, errp)) { |
| 724 | return false; |
| 725 | } |
| 726 | s->s_accel->cmdqv_ops = ops; |
| 727 | return true; |
| 728 | } |
| 729 | |
| 730 | static bool smmuv3_accel_set_iommu_device(PCIBus *bus, void *opaque, int devfn, |
| 731 | HostIOMMUDevice *hiod, Error **errp) |
| 732 | { |
| 733 | HostIOMMUDeviceIOMMUFD *hiodi = HOST_IOMMU_DEVICE_IOMMUFD(hiod); |
| 734 | SMMUState *bs = opaque; |
| 735 | SMMUv3State *s = ARM_SMMUV3(bs); |
| 736 | SMMUPciBus *sbus = smmu_get_sbus(bs, bus); |
| 737 | SMMUv3AccelDevice *accel_dev = smmuv3_accel_get_dev(bs, sbus, bus, devfn); |
| 738 | |
| 739 | if (!hiodi) { |
| 740 | return true; |
| 741 | } |
| 742 | |
| 743 | if (accel_dev->hiodi) { |
| 744 | if (accel_dev->hiodi != hiodi) { |
| 745 | error_setg(errp, "Device already has an associated hiodi 0x%x", |
| 746 | hiodi->devid); |
| 747 | return false; |
| 748 | } |
| 749 | return true; |
| 750 | } |
| 751 | |
| 752 | /* |
| 753 | * Check the host SMMUv3 associated with the dev is compatible with the |
| 754 | * QEMU SMMUv3 accel. |
| 755 | */ |
| 756 | if (!smmuv3_accel_hw_compatible(s, hiodi, errp)) { |
| 757 | return false; |
| 758 | } |
| 759 | |
| 760 | if (s->s_accel->viommu) { |
| 761 | goto done; |
| 762 | } |
| 763 | |
| 764 | if (!smmuv3_accel_select_cmdqv(s, hiodi, errp)) { |
| 765 | return false; |
| 766 | } |
| 767 | |
| 768 | if (!smmuv3_accel_alloc_viommu(s, hiodi, errp)) { |
| 769 | error_append_hint(errp, "Unable to alloc vIOMMU: hiodi devid 0x%x: ", |
| 770 | hiodi->devid); |
| 771 | return false; |
| 772 | } |
| 773 | |
| 774 | /* |
| 775 | * CMDQV is active: block hot-unplug of the device that established the |
| 776 | * viommu association. Removing it would cause the vIOMMU to host SMMUv3 |
| 777 | * association be changed via device hot-plug. |
| 778 | */ |
| 779 | if (s->s_accel->cmdqv_ops) { |
| 780 | PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn); |
| 781 | error_setg(&accel_dev->unplug_blocker, |
| 782 | "CMDQV is active: removing the device that established the " |
| 783 | "viommu association would break the guest CMDQV"); |
| 784 | qdev_add_unplug_blocker(DEVICE(pdev), accel_dev->unplug_blocker); |
| 785 | } |
| 786 | done: |
| 787 | accel_dev->hiodi = hiodi; |
| 788 | accel_dev->s_accel = s->s_accel; |
| 789 | QLIST_INSERT_HEAD(&s->s_accel->device_list, accel_dev, next); |
| 790 | trace_smmuv3_accel_set_iommu_device(devfn, hiodi->devid); |
| 791 | return true; |
| 792 | } |
| 793 | |
| 794 | static void smmuv3_accel_unset_iommu_device(PCIBus *bus, void *opaque, |
| 795 | int devfn) |
| 796 | { |
| 797 | SMMUState *bs = opaque; |
| 798 | SMMUPciBus *sbus = g_hash_table_lookup(bs->smmu_pcibus_by_busptr, bus); |
| 799 | HostIOMMUDeviceIOMMUFD *hiodi; |
| 800 | SMMUv3AccelDevice *accel_dev; |
| 801 | SMMUv3AccelState *accel; |
| 802 | IOMMUFDVdev *vdev; |
| 803 | SMMUDevice *sdev; |
| 804 | |
| 805 | if (!sbus) { |
| 806 | return; |
| 807 | } |
| 808 | |
| 809 | sdev = sbus->pbdev[devfn]; |
| 810 | if (!sdev) { |
| 811 | return; |
| 812 | } |
| 813 | |
| 814 | accel_dev = container_of(sdev, SMMUv3AccelDevice, sdev); |
| 815 | hiodi = accel_dev->hiodi; |
| 816 | accel = accel_dev->s_accel; |
| 817 | /* Re-attach the default s2 hwpt id */ |
| 818 | if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, |
| 819 | hiodi->hwpt_id, NULL)) { |
| 820 | error_report("Unable to attach the default HW pagetable: hiodi devid " |
| 821 | "0x%x", hiodi->devid); |
| 822 | } |
| 823 | |
| 824 | if (accel_dev->s1_hwpt) { |
| 825 | iommufd_backend_free_id(accel_dev->hiodi->iommufd, |
| 826 | accel_dev->s1_hwpt->hwpt_id); |
| 827 | g_free(accel_dev->s1_hwpt); |
| 828 | accel_dev->s1_hwpt = NULL; |
| 829 | } |
| 830 | |
| 831 | vdev = accel_dev->vdev; |
| 832 | if (vdev) { |
| 833 | iommufd_backend_free_id(accel->viommu->iommufd, vdev->vdevice_id); |
| 834 | g_free(vdev); |
| 835 | accel_dev->vdev = NULL; |
| 836 | } |
| 837 | |
| 838 | accel_dev->hiodi = NULL; |
| 839 | accel_dev->s_accel = NULL; |
| 840 | QLIST_REMOVE(accel_dev, next); |
| 841 | trace_smmuv3_accel_unset_iommu_device(devfn, hiodi->devid); |
| 842 | |
| 843 | if (QLIST_EMPTY(&accel->device_list)) { |
| 844 | smmuv3_accel_free_viommu(accel); |
| 845 | } |
| 846 | } |
| 847 | |
| 848 | static uint64_t smmuv3_accel_get_msi_gpa(PCIBus *bus, void *opaque, int devfn) |
| 849 | { |
| 850 | SMMUState *bs = opaque; |
| 851 | SMMUv3State *s = ARM_SMMUV3(bs); |
| 852 | |
| 853 | g_assert(s->msi_gpa); |
| 854 | return s->msi_gpa; |
| 855 | } |
| 856 | |
| 857 | /* |
| 858 | * Only allow PCIe bridges, pxb-pcie roots, and GPEX roots so vfio-pci |
| 859 | * endpoints can sit downstream. Accelerated SMMUv3 requires a vfio-pci |
| 860 | * endpoint using the iommufd backend; all other device types are rejected. |
| 861 | * This avoids supporting emulated endpoints, which would complicate IOTLB |
| 862 | * invalidation and hurt performance. |
| 863 | */ |
| 864 | static bool smmuv3_accel_pdev_allowed(PCIDevice *pdev, bool *vfio_pci) |
| 865 | { |
| 866 | |
| 867 | if (object_dynamic_cast(OBJECT(pdev), TYPE_PCI_BRIDGE) || |
| 868 | object_dynamic_cast(OBJECT(pdev), TYPE_PXB_PCIE_DEV) || |
| 869 | object_dynamic_cast(OBJECT(pdev), TYPE_GPEX_ROOT_DEVICE)) { |
| 870 | return true; |
| 871 | } else if ((object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI))) { |
| 872 | *vfio_pci = true; |
| 873 | if (object_property_get_link(OBJECT(pdev), "iommufd", NULL)) { |
| 874 | return true; |
| 875 | } |
| 876 | } |
| 877 | return false; |
| 878 | } |
| 879 | |
| 880 | static bool smmuv3_accel_supports_as(PCIBus *bus, void *opaque, int devfn, |
| 881 | Error **errp) |
| 882 | { |
| 883 | PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn); |
| 884 | bool vfio_pci = false; |
| 885 | |
| 886 | if (pdev && !smmuv3_accel_pdev_allowed(pdev, &vfio_pci)) { |
| 887 | if (vfio_pci) { |
| 888 | error_setg(errp, "vfio-pci endpoint devices without an iommufd " |
| 889 | "backend not allowed when using arm-smmuv3,accel=on"); |
| 890 | |
| 891 | } else { |
| 892 | error_setg(errp, "Emulated endpoint devices are not allowed when " |
| 893 | "using arm-smmuv3,accel=on"); |
| 894 | } |
| 895 | return false; |
| 896 | } |
| 897 | return true; |
| 898 | } |
| 899 | /* |
| 900 | * Find or add an address space for the given PCI device. |
| 901 | * |
| 902 | * If a device matching @bus and @devfn already exists, return its |
| 903 | * corresponding address space. Otherwise, create a new device entry |
| 904 | * and initialize address space for it. |
| 905 | */ |
| 906 | static AddressSpace *smmuv3_accel_find_add_as(PCIBus *bus, void *opaque, |
| 907 | int devfn) |
| 908 | { |
| 909 | PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn); |
| 910 | SMMUState *bs = opaque; |
| 911 | SMMUPciBus *sbus = smmu_get_sbus(bs, bus); |
| 912 | SMMUv3AccelDevice *accel_dev = smmuv3_accel_get_dev(bs, sbus, bus, devfn); |
| 913 | SMMUDevice *sdev = &accel_dev->sdev; |
| 914 | bool vfio_pci = false; |
| 915 | |
| 916 | if (pdev && !smmuv3_accel_pdev_allowed(pdev, &vfio_pci)) { |
| 917 | /* Should never be here: supports_address_space() filters these out */ |
| 918 | g_assert_not_reached(); |
| 919 | } |
| 920 | |
| 921 | /* |
| 922 | * In the accelerated mode, a vfio-pci device attached via the iommufd |
| 923 | * backend must remain in the system address space. Such a device is |
| 924 | * always translated by its physical SMMU (using either a stage-2-only |
| 925 | * STE or a nested STE), where the parent stage-2 page table is allocated |
| 926 | * by the VFIO core to back the system address space. |
| 927 | * |
| 928 | * Return the shared_as_sysmem aliased to the global system memory in this |
| 929 | * case. Sharing address_space_memory also allows devices under different |
| 930 | * vSMMU instances in the same VM to reuse a single nesting parent HWPT in |
| 931 | * the VFIO core. |
| 932 | * |
| 933 | * For non-endpoint emulated devices such as PCIe root ports and bridges, |
| 934 | * which may use the normal emulated translation path and software IOTLBs, |
| 935 | * return the SMMU's IOMMU address space. |
| 936 | */ |
| 937 | if (vfio_pci) { |
| 938 | return shared_as_sysmem; |
| 939 | } else { |
| 940 | return &sdev->as; |
| 941 | } |
| 942 | } |
| 943 | |
| 944 | static inline bool smmuv3_pasid_supported(SMMUv3State *s) |
| 945 | { |
| 946 | return s->ssidsize > SSID_SIZE_MODE_0 || |
| 947 | (s->ssidsize == SSID_SIZE_MODE_AUTO && |
| 948 | FIELD_EX32(s->idr[1], IDR1, SSIDSIZE)); |
| 949 | } |
| 950 | |
| 951 | static uint64_t smmuv3_accel_get_viommu_flags(void *opaque) |
| 952 | { |
| 953 | /* |
| 954 | * We return VIOMMU_FLAG_WANT_NESTING_PARENT to inform VFIO core to create a |
| 955 | * nesting parent which is required for accelerated SMMUv3 support. |
| 956 | * The real HW nested support should be reported from host SMMUv3 and if |
| 957 | * it doesn't, the nesting parent allocation will fail anyway in VFIO core. |
| 958 | */ |
| 959 | uint64_t flags = VIOMMU_FLAG_WANT_NESTING_PARENT; |
| 960 | SMMUState *bs = opaque; |
| 961 | SMMUv3State *s = ARM_SMMUV3(bs); |
| 962 | |
| 963 | if (smmuv3_pasid_supported(s)) { |
| 964 | flags |= VIOMMU_FLAG_PASID_SUPPORTED; |
| 965 | } |
| 966 | return flags; |
| 967 | } |
| 968 | |
| 969 | static const PCIIOMMUOps smmuv3_accel_ops = { |
| 970 | .supports_address_space = smmuv3_accel_supports_as, |
| 971 | .get_address_space = smmuv3_accel_find_add_as, |
| 972 | .get_viommu_flags = smmuv3_accel_get_viommu_flags, |
| 973 | .set_iommu_device = smmuv3_accel_set_iommu_device, |
| 974 | .unset_iommu_device = smmuv3_accel_unset_iommu_device, |
| 975 | .get_msi_direct_gpa = smmuv3_accel_get_msi_gpa, |
| 976 | }; |
| 977 | |
| 978 | /* |
| 979 | * This returns the value of a SsidSizeMode value offset by 1 to |
| 980 | * account for the enum values offset by 1 from actual values. |
| 981 | * |
| 982 | * SSID_SIZE_MODE_0 = 1, SSID_SIZE_MODE_1 = 2, etc. so return 0 |
| 983 | * if SSID_SIZE_MODE_0 is passed as input, return 1 if |
| 984 | * SSID_SIZE_MODE_1 is passed as input, etc. |
| 985 | */ |
| 986 | static uint8_t ssidsize_mode_to_value(SsidSizeMode mode) |
| 987 | { |
| 988 | if (mode == SSID_SIZE_MODE_AUTO) { |
| 989 | return 0; |
| 990 | } |
| 991 | return mode - 1; |
| 992 | } |
| 993 | |
| 994 | void smmuv3_accel_idr_override(SMMUv3State *s) |
| 995 | { |
| 996 | if (!s->accel) { |
| 997 | return; |
| 998 | } |
| 999 | |
| 1000 | /* Only override RIL if user explicitly set OFF */ |
| 1001 | if (s->ril == ON_OFF_AUTO_OFF) { |
| 1002 | s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 0); |
| 1003 | } |
| 1004 | |
| 1005 | /* QEMU SMMUv3 has no ATS. Advertise ATS if opt-in by property */ |
| 1006 | if (s->ats == ON_OFF_AUTO_ON) { |
| 1007 | s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ATS, 1); |
| 1008 | } |
| 1009 | |
| 1010 | /* Advertise 48-bit OAS in IDR5 when requested (default is 44 bits). */ |
| 1011 | if (s->oas == OAS_MODE_48) { |
| 1012 | s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS_48); |
| 1013 | } |
| 1014 | |
| 1015 | /* |
| 1016 | * By default QEMU SMMUv3 has no SubstreamID support. Update IDR1 if user |
| 1017 | * has enabled it. |
| 1018 | */ |
| 1019 | if (s->ssidsize > SSID_SIZE_MODE_0) { |
| 1020 | s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SSIDSIZE, |
| 1021 | ssidsize_mode_to_value(s->ssidsize)); |
| 1022 | } |
| 1023 | } |
| 1024 | |
| 1025 | /* Based on SMUUv3 GPBA.ABORT configuration, attach a corresponding HWPT */ |
| 1026 | bool smmuv3_accel_attach_gbpa_hwpt(SMMUv3State *s, Error **errp) |
| 1027 | { |
| 1028 | SMMUv3AccelState *accel = s->s_accel; |
| 1029 | SMMUv3AccelDevice *accel_dev; |
| 1030 | Error *local_err = NULL; |
| 1031 | bool all_ok = true; |
| 1032 | uint32_t hwpt_id; |
| 1033 | |
| 1034 | if (!accel || !accel->viommu) { |
| 1035 | return true; |
| 1036 | } |
| 1037 | |
| 1038 | hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel); |
| 1039 | QLIST_FOREACH(accel_dev, &accel->device_list, next) { |
| 1040 | if (!host_iommu_device_iommufd_attach_hwpt(accel_dev->hiodi, |
| 1041 | IOMMU_NO_PASID, hwpt_id, |
| 1042 | &local_err)) { |
| 1043 | error_append_hint(&local_err, "Failed to attach GBPA hwpt %u for " |
| 1044 | "hiodi devid %u", hwpt_id, |
| 1045 | accel_dev->hiodi->devid); |
| 1046 | error_report_err(local_err); |
| 1047 | local_err = NULL; |
| 1048 | all_ok = false; |
| 1049 | } |
| 1050 | } |
| 1051 | if (!all_ok) { |
| 1052 | error_setg(errp, "Failed to attach all GBPA based HWPTs properly"); |
| 1053 | } |
| 1054 | return all_ok; |
| 1055 | } |
| 1056 | |
| 1057 | void smmuv3_accel_reset(SMMUv3State *s) |
| 1058 | { |
| 1059 | SMMUv3AccelState *accel = s->s_accel; |
| 1060 | |
| 1061 | if (!accel) { |
| 1062 | return; |
| 1063 | } |
| 1064 | /* Attach a HWPT based on GBPA reset value */ |
| 1065 | smmuv3_accel_attach_gbpa_hwpt(s, NULL); |
| 1066 | |
| 1067 | if (accel->cmdqv_ops && accel->cmdqv_ops->reset) { |
| 1068 | accel->cmdqv_ops->reset(s); |
| 1069 | } |
| 1070 | } |
| 1071 | |
| 1072 | static void smmuv3_accel_as_init(SMMUv3State *s) |
| 1073 | { |
| 1074 | |
| 1075 | if (shared_as_sysmem) { |
| 1076 | return; |
| 1077 | } |
| 1078 | |
| 1079 | memory_region_init(&root, OBJECT(s), "root", UINT64_MAX); |
| 1080 | memory_region_init_alias(&sysmem, OBJECT(s), "smmuv3-accel-sysmem", |
| 1081 | get_system_memory(), 0, |
| 1082 | memory_region_size(get_system_memory())); |
| 1083 | memory_region_add_subregion(&root, 0, &sysmem); |
| 1084 | |
| 1085 | shared_as_sysmem = g_new0(AddressSpace, 1); |
| 1086 | address_space_init(shared_as_sysmem, &root, "smmuv3-accel-as-sysmem"); |
| 1087 | } |
| 1088 | |
| 1089 | SMMUv3AccelCmdqvType smmuv3_accel_cmdqv_type(Object *obj) |
| 1090 | { |
| 1091 | SMMUv3State *s = ARM_SMMUV3(obj); |
| 1092 | SMMUv3AccelState *accel = s->s_accel; |
| 1093 | |
| 1094 | if (!accel || !accel->cmdqv_ops || !accel->cmdqv_ops->get_type) { |
| 1095 | return SMMUV3_CMDQV_NONE; |
| 1096 | } |
| 1097 | |
| 1098 | return accel->cmdqv_ops->get_type(); |
| 1099 | } |
| 1100 | |
| 1101 | static void smmuv3_accel_machine_done(Notifier *notifier, void *data) |
| 1102 | { |
| 1103 | SMMUv3State *s = container_of(notifier, SMMUv3State, machine_done); |
| 1104 | SMMUv3AccelState *accel = s->s_accel; |
| 1105 | |
| 1106 | if (accel->auto_mode && !accel->auto_finalised) { |
| 1107 | error_report("arm-smmuv3 accel=on with 'auto' properties requires " |
| 1108 | "at least one cold-plugged VFIO device"); |
| 1109 | exit(1); |
| 1110 | } |
| 1111 | |
| 1112 | if (s->cmdqv == ON_OFF_AUTO_ON && !accel->cmdqv) { |
| 1113 | error_report("arm-smmuv3 cmdqv=on requires at least one cold-plugged " |
| 1114 | "VFIO device"); |
| 1115 | exit(1); |
| 1116 | } |
| 1117 | } |
| 1118 | |
| 1119 | bool smmuv3_accel_init(SMMUv3State *s, Error **errp) |
| 1120 | { |
| 1121 | SMMUState *bs = ARM_SMMU(s); |
| 1122 | |
| 1123 | s->s_accel = g_new0(SMMUv3AccelState, 1); |
| 1124 | bs->iommu_ops = &smmuv3_accel_ops; |
| 1125 | smmuv3_accel_as_init(s); |
| 1126 | |
| 1127 | if (s->ats == ON_OFF_AUTO_AUTO || |
| 1128 | s->ril == ON_OFF_AUTO_AUTO || |
| 1129 | s->ssidsize == SSID_SIZE_MODE_AUTO || |
| 1130 | s->oas == OAS_MODE_AUTO) { |
| 1131 | s->s_accel->auto_mode = true; |
| 1132 | } |
| 1133 | |
| 1134 | if (s->s_accel->auto_mode) { |
| 1135 | s->machine_done.notify = smmuv3_accel_machine_done; |
| 1136 | qemu_add_machine_init_done_notifier(&s->machine_done); |
| 1137 | } |
| 1138 | |
| 1139 | return true; |
| 1140 | } |