| 1 | /* |
| 2 | * s390 PCI instructions |
| 3 | * |
| 4 | * Copyright 2014 IBM Corp. |
| 5 | * Author(s): Frank Blaschka <frank.blaschka@de.ibm.com> |
| 6 | * Hong Bo Li <lihbbj@cn.ibm.com> |
| 7 | * Yi Min Zhao <zyimin@cn.ibm.com> |
| 8 | * |
| 9 | * This work is licensed under the terms of the GNU GPL, version 2 or (at |
| 10 | * your option) any later version. See the COPYING file in the top-level |
| 11 | * directory. |
| 12 | */ |
| 13 | |
| 14 | #include "qemu/osdep.h" |
| 15 | #include "exec/memop.h" |
| 16 | #include "exec/target_page.h" |
| 17 | #include "system/memory.h" |
| 18 | #include "qemu/error-report.h" |
| 19 | #include "qemu/bswap.h" |
| 20 | #include "system/hw_accel.h" |
| 21 | #include "hw/core/boards.h" |
| 22 | #include "hw/pci/pci_device.h" |
| 23 | #include "hw/s390x/s390-pci-inst.h" |
| 24 | #include "hw/s390x/s390-pci-bus.h" |
| 25 | #include "hw/s390x/s390-pci-kvm.h" |
| 26 | #include "hw/s390x/s390-pci-vfio.h" |
| 27 | #include "hw/s390x/tod.h" |
| 28 | |
| 29 | #include "trace.h" |
| 30 | |
| 31 | static inline void inc_dma_avail(S390PCIIOMMU *iommu) |
| 32 | { |
| 33 | if (iommu->dma_limit) { |
| 34 | iommu->dma_limit->avail++; |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | static inline void dec_dma_avail(S390PCIIOMMU *iommu) |
| 39 | { |
| 40 | if (iommu->dma_limit) { |
| 41 | iommu->dma_limit->avail--; |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | static void s390_set_status_code(CPUS390XState *env, |
| 46 | uint8_t r, uint64_t status_code) |
| 47 | { |
| 48 | env->regs[r] &= ~0xff000000ULL; |
| 49 | env->regs[r] |= (status_code & 0xff) << 24; |
| 50 | } |
| 51 | |
| 52 | static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc) |
| 53 | { |
| 54 | S390PCIBusDevice *pbdev = NULL; |
| 55 | S390pciState *s = s390_get_phb(); |
| 56 | uint32_t res_code, initial_l2, g_l2; |
| 57 | int rc, i; |
| 58 | uint64_t resume_token; |
| 59 | |
| 60 | rc = 0; |
| 61 | if (lduw_be_p(&rrb->request.hdr.len) != 32) { |
| 62 | res_code = CLP_RC_LEN; |
| 63 | rc = -EINVAL; |
| 64 | goto out; |
| 65 | } |
| 66 | |
| 67 | if ((ldl_be_p(&rrb->request.fmt) & CLP_MASK_FMT) != 0) { |
| 68 | res_code = CLP_RC_FMT; |
| 69 | rc = -EINVAL; |
| 70 | goto out; |
| 71 | } |
| 72 | |
| 73 | if ((ldl_be_p(&rrb->request.fmt) & ~CLP_MASK_FMT) != 0 || |
| 74 | ldq_be_p(&rrb->request.reserved1) != 0) { |
| 75 | res_code = CLP_RC_RESNOT0; |
| 76 | rc = -EINVAL; |
| 77 | goto out; |
| 78 | } |
| 79 | |
| 80 | resume_token = ldq_be_p(&rrb->request.resume_token); |
| 81 | |
| 82 | if (resume_token) { |
| 83 | pbdev = s390_pci_find_dev_by_idx(s, resume_token); |
| 84 | if (!pbdev) { |
| 85 | res_code = CLP_RC_LISTPCI_BADRT; |
| 86 | rc = -EINVAL; |
| 87 | goto out; |
| 88 | } |
| 89 | } else { |
| 90 | pbdev = s390_pci_find_next_avail_dev(s, NULL); |
| 91 | } |
| 92 | |
| 93 | if (lduw_be_p(&rrb->response.hdr.len) < 48) { |
| 94 | res_code = CLP_RC_8K; |
| 95 | rc = -EINVAL; |
| 96 | goto out; |
| 97 | } |
| 98 | |
| 99 | initial_l2 = lduw_be_p(&rrb->response.hdr.len); |
| 100 | if ((initial_l2 - LIST_PCI_HDR_LEN) % sizeof(ClpFhListEntry) |
| 101 | != 0) { |
| 102 | res_code = CLP_RC_LEN; |
| 103 | rc = -EINVAL; |
| 104 | *cc = 3; |
| 105 | goto out; |
| 106 | } |
| 107 | |
| 108 | stl_be_p(&rrb->response.fmt, 0); |
| 109 | stq_be_p(&rrb->response.reserved1, 0); |
| 110 | stl_be_p(&rrb->response.mdd, FH_MASK_SHM); |
| 111 | stw_be_p(&rrb->response.max_fn, PCI_MAX_FUNCTIONS); |
| 112 | rrb->response.flags = UID_CHECKING_ENABLED; |
| 113 | rrb->response.entry_size = sizeof(ClpFhListEntry); |
| 114 | |
| 115 | i = 0; |
| 116 | g_l2 = LIST_PCI_HDR_LEN; |
| 117 | while (g_l2 < initial_l2 && pbdev) { |
| 118 | stw_be_p(&rrb->response.fh_list[i].device_id, |
| 119 | pci_get_word(pbdev->pdev->config + PCI_DEVICE_ID)); |
| 120 | stw_be_p(&rrb->response.fh_list[i].vendor_id, |
| 121 | pci_get_word(pbdev->pdev->config + PCI_VENDOR_ID)); |
| 122 | /* Ignore RESERVED devices. */ |
| 123 | stl_be_p(&rrb->response.fh_list[i].config, |
| 124 | pbdev->state == ZPCI_FS_STANDBY ? 0 : 1 << 31); |
| 125 | stl_be_p(&rrb->response.fh_list[i].fid, pbdev->fid); |
| 126 | stl_be_p(&rrb->response.fh_list[i].fh, pbdev->fh); |
| 127 | |
| 128 | g_l2 += sizeof(ClpFhListEntry); |
| 129 | /* Add endian check for DPRINTF? */ |
| 130 | trace_s390_pci_list_entry(g_l2, |
| 131 | lduw_be_p(&rrb->response.fh_list[i].vendor_id), |
| 132 | lduw_be_p(&rrb->response.fh_list[i].device_id), |
| 133 | ldl_be_p(&rrb->response.fh_list[i].fid), |
| 134 | ldl_be_p(&rrb->response.fh_list[i].fh)); |
| 135 | pbdev = s390_pci_find_next_avail_dev(s, pbdev); |
| 136 | i++; |
| 137 | } |
| 138 | |
| 139 | if (!pbdev) { |
| 140 | resume_token = 0; |
| 141 | } else { |
| 142 | resume_token = pbdev->fh & FH_MASK_INDEX; |
| 143 | } |
| 144 | stq_be_p(&rrb->response.resume_token, resume_token); |
| 145 | stw_be_p(&rrb->response.hdr.len, g_l2); |
| 146 | stw_be_p(&rrb->response.hdr.rsp, CLP_RC_OK); |
| 147 | out: |
| 148 | if (rc) { |
| 149 | trace_s390_pci_list(rc); |
| 150 | stw_be_p(&rrb->response.hdr.rsp, res_code); |
| 151 | } |
| 152 | return rc; |
| 153 | } |
| 154 | |
| 155 | int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) |
| 156 | { |
| 157 | ClpReqHdr *reqh; |
| 158 | ClpRspHdr *resh; |
| 159 | S390PCIBusDevice *pbdev; |
| 160 | uint32_t req_len; |
| 161 | uint32_t res_len; |
| 162 | uint8_t buffer[4096 * 2]; |
| 163 | uint8_t cc = 0; |
| 164 | CPUS390XState *env = &cpu->env; |
| 165 | S390pciState *s = s390_get_phb(); |
| 166 | int i; |
| 167 | |
| 168 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 169 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 170 | return 0; |
| 171 | } |
| 172 | |
| 173 | if (s390_cpu_virt_mem_read(cpu, env->regs[r2], r2, buffer, sizeof(*reqh))) { |
| 174 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 175 | return 0; |
| 176 | } |
| 177 | reqh = (ClpReqHdr *)buffer; |
| 178 | req_len = lduw_be_p(&reqh->len); |
| 179 | if (req_len < 16 || req_len > 8184 || (req_len % 8 != 0)) { |
| 180 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 181 | return 0; |
| 182 | } |
| 183 | |
| 184 | if (s390_cpu_virt_mem_read(cpu, env->regs[r2], r2, buffer, |
| 185 | req_len + sizeof(*resh))) { |
| 186 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 187 | return 0; |
| 188 | } |
| 189 | resh = (ClpRspHdr *)(buffer + req_len); |
| 190 | res_len = lduw_be_p(&resh->len); |
| 191 | if (res_len < 8 || res_len > 8176 || (res_len % 8 != 0)) { |
| 192 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 193 | return 0; |
| 194 | } |
| 195 | if ((req_len + res_len) > 8192) { |
| 196 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 197 | return 0; |
| 198 | } |
| 199 | |
| 200 | if (s390_cpu_virt_mem_read(cpu, env->regs[r2], r2, buffer, |
| 201 | req_len + res_len)) { |
| 202 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 203 | return 0; |
| 204 | } |
| 205 | |
| 206 | if (req_len != 32) { |
| 207 | stw_be_p(&resh->rsp, CLP_RC_LEN); |
| 208 | goto out; |
| 209 | } |
| 210 | |
| 211 | switch (lduw_be_p(&reqh->cmd)) { |
| 212 | case CLP_LIST_PCI: { |
| 213 | ClpReqRspListPci *rrb = (ClpReqRspListPci *)buffer; |
| 214 | list_pci(rrb, &cc); |
| 215 | break; |
| 216 | } |
| 217 | case CLP_SET_PCI_FN: { |
| 218 | ClpReqSetPci *reqsetpci = (ClpReqSetPci *)reqh; |
| 219 | ClpRspSetPci *ressetpci = (ClpRspSetPci *)resh; |
| 220 | |
| 221 | pbdev = s390_pci_find_dev_by_fh(s, ldl_be_p(&reqsetpci->fh)); |
| 222 | if (!pbdev) { |
| 223 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FH); |
| 224 | goto out; |
| 225 | } |
| 226 | |
| 227 | switch (reqsetpci->oc) { |
| 228 | case CLP_SET_ENABLE_PCI_FN: |
| 229 | switch (reqsetpci->ndas) { |
| 230 | case 0: |
| 231 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_DMAAS); |
| 232 | goto out; |
| 233 | case 1: |
| 234 | break; |
| 235 | default: |
| 236 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_RES); |
| 237 | goto out; |
| 238 | } |
| 239 | |
| 240 | if (pbdev->fh & FH_MASK_ENABLE) { |
| 241 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FHOP); |
| 242 | goto out; |
| 243 | } |
| 244 | |
| 245 | /* |
| 246 | * Take this opportunity to make sure we still have an accurate |
| 247 | * host fh. It's possible part of the handle changed while the |
| 248 | * device was disabled to the guest (e.g. vfio hot reset for |
| 249 | * ISM during plug) |
| 250 | */ |
| 251 | if (pbdev->interp) { |
| 252 | /* Take this opportunity to make sure we are sync'd with host */ |
| 253 | if (!s390_pci_get_host_fh(pbdev, &pbdev->fh) || |
| 254 | !(pbdev->fh & FH_MASK_ENABLE)) { |
| 255 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FH); |
| 256 | goto out; |
| 257 | } |
| 258 | } |
| 259 | pbdev->fh |= FH_MASK_ENABLE; |
| 260 | pbdev->state = ZPCI_FS_ENABLED; |
| 261 | stl_be_p(&ressetpci->fh, pbdev->fh); |
| 262 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_OK); |
| 263 | break; |
| 264 | case CLP_SET_DISABLE_PCI_FN: |
| 265 | if (!(pbdev->fh & FH_MASK_ENABLE)) { |
| 266 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FHOP); |
| 267 | goto out; |
| 268 | } |
| 269 | device_cold_reset(DEVICE(pbdev)); |
| 270 | pbdev->fh &= ~FH_MASK_ENABLE; |
| 271 | pbdev->state = ZPCI_FS_DISABLED; |
| 272 | stl_be_p(&ressetpci->fh, pbdev->fh); |
| 273 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_OK); |
| 274 | break; |
| 275 | default: |
| 276 | trace_s390_pci_unknown("set-pci", reqsetpci->oc); |
| 277 | stw_be_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FHOP); |
| 278 | break; |
| 279 | } |
| 280 | break; |
| 281 | } |
| 282 | case CLP_QUERY_PCI_FN: { |
| 283 | ClpReqQueryPci *reqquery = (ClpReqQueryPci *)reqh; |
| 284 | ClpRspQueryPci *resquery = (ClpRspQueryPci *)resh; |
| 285 | |
| 286 | pbdev = s390_pci_find_dev_by_fh(s, ldl_be_p(&reqquery->fh)); |
| 287 | if (!pbdev) { |
| 288 | trace_s390_pci_nodev("query", ldl_be_p(&reqquery->fh)); |
| 289 | stw_be_p(&resquery->hdr.rsp, CLP_RC_SETPCIFN_FH); |
| 290 | goto out; |
| 291 | } |
| 292 | |
| 293 | stq_be_p(&resquery->sdma, pbdev->zpci_fn.sdma); |
| 294 | stq_be_p(&resquery->edma, pbdev->zpci_fn.edma); |
| 295 | stw_be_p(&resquery->pchid, pbdev->zpci_fn.pchid); |
| 296 | stw_be_p(&resquery->vfn, pbdev->zpci_fn.vfn); |
| 297 | resquery->flags = pbdev->zpci_fn.flags; |
| 298 | resquery->pfgid = pbdev->zpci_fn.pfgid; |
| 299 | resquery->pft = pbdev->zpci_fn.pft; |
| 300 | resquery->fmbl = pbdev->zpci_fn.fmbl; |
| 301 | stl_be_p(&resquery->fid, pbdev->zpci_fn.fid); |
| 302 | stl_be_p(&resquery->uid, pbdev->zpci_fn.uid); |
| 303 | memcpy(resquery->pfip, pbdev->zpci_fn.pfip, CLP_PFIP_NR_SEGMENTS); |
| 304 | memcpy(resquery->util_str, pbdev->zpci_fn.util_str, CLP_UTIL_STR_LEN); |
| 305 | |
| 306 | for (i = 0; i < PCI_BAR_COUNT; i++) { |
| 307 | uint32_t data = pci_get_long(pbdev->pdev->config + |
| 308 | PCI_BASE_ADDRESS_0 + (i * 4)); |
| 309 | |
| 310 | stl_le_p(&resquery->bar[i], data); |
| 311 | resquery->bar_size[i] = pbdev->pdev->io_regions[i].size ? |
| 312 | ctz64(pbdev->pdev->io_regions[i].size) : 0; |
| 313 | trace_s390_pci_bar(i, |
| 314 | ldl_be_p(&resquery->bar[i]), |
| 315 | pbdev->pdev->io_regions[i].size, |
| 316 | resquery->bar_size[i]); |
| 317 | } |
| 318 | |
| 319 | stw_be_p(&resquery->hdr.rsp, CLP_RC_OK); |
| 320 | break; |
| 321 | } |
| 322 | case CLP_QUERY_PCI_FNGRP: { |
| 323 | ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh; |
| 324 | |
| 325 | ClpReqQueryPciGrp *reqgrp = (ClpReqQueryPciGrp *)reqh; |
| 326 | S390PCIGroup *group; |
| 327 | |
| 328 | group = s390_group_find(reqgrp->g); |
| 329 | if (!group) { |
| 330 | /* We do not allow access to unknown groups */ |
| 331 | /* The group must have been obtained with a vfio device */ |
| 332 | stw_be_p(&resgrp->hdr.rsp, CLP_RC_QUERYPCIFG_PFGID); |
| 333 | goto out; |
| 334 | } |
| 335 | resgrp->fr = group->zpci_group.fr; |
| 336 | stq_be_p(&resgrp->dasm, group->zpci_group.dasm); |
| 337 | stq_be_p(&resgrp->msia, group->zpci_group.msia); |
| 338 | stw_be_p(&resgrp->mui, group->zpci_group.mui); |
| 339 | stw_be_p(&resgrp->i, group->zpci_group.i); |
| 340 | stw_be_p(&resgrp->maxstbl, group->zpci_group.maxstbl); |
| 341 | resgrp->version = group->zpci_group.version; |
| 342 | resgrp->dtsm = group->zpci_group.dtsm; |
| 343 | stw_be_p(&resgrp->hdr.rsp, CLP_RC_OK); |
| 344 | break; |
| 345 | } |
| 346 | default: |
| 347 | trace_s390_pci_unknown("clp", lduw_be_p(&reqh->cmd)); |
| 348 | stw_be_p(&resh->rsp, CLP_RC_CMD); |
| 349 | break; |
| 350 | } |
| 351 | |
| 352 | out: |
| 353 | if (s390_cpu_virt_mem_write(cpu, env->regs[r2], r2, buffer, |
| 354 | req_len + res_len)) { |
| 355 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 356 | return 0; |
| 357 | } |
| 358 | setcc(cpu, cc); |
| 359 | return 0; |
| 360 | } |
| 361 | |
| 362 | /** |
| 363 | * Swap data contained in s390x big endian registers to little endian |
| 364 | * PCI bars. |
| 365 | * |
| 366 | * @ptr: a pointer to a uint64_t data field |
| 367 | * @len: the length of the valid data, must be 1,2,4 or 8 |
| 368 | */ |
| 369 | static int zpci_endian_swap(uint64_t *ptr, uint8_t len) |
| 370 | { |
| 371 | uint64_t data = *ptr; |
| 372 | |
| 373 | switch (len) { |
| 374 | case 1: |
| 375 | break; |
| 376 | case 2: |
| 377 | data = bswap16(data); |
| 378 | break; |
| 379 | case 4: |
| 380 | data = bswap32(data); |
| 381 | break; |
| 382 | case 8: |
| 383 | data = bswap64(data); |
| 384 | break; |
| 385 | default: |
| 386 | return -EINVAL; |
| 387 | } |
| 388 | *ptr = data; |
| 389 | return 0; |
| 390 | } |
| 391 | |
| 392 | static MemoryRegion *s390_get_subregion(MemoryRegion *mr, uint64_t offset, |
| 393 | uint8_t len) |
| 394 | { |
| 395 | uint64_t last = offset + len; |
| 396 | MemoryRegion *subregion; |
| 397 | uint64_t subregion_size; |
| 398 | |
| 399 | /* |
| 400 | * Ensure the region is valid, the calculated address cannot wrap and that |
| 401 | * it falls within this region. |
| 402 | */ |
| 403 | if (!mr || offset > last || last > memory_region_size(mr)) { |
| 404 | return NULL; |
| 405 | } |
| 406 | |
| 407 | QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) { |
| 408 | subregion_size = memory_region_size(subregion); |
| 409 | if ((offset >= subregion->addr) && |
| 410 | (last) <= (subregion->addr + subregion_size)) { |
| 411 | mr = subregion; |
| 412 | break; |
| 413 | } |
| 414 | } |
| 415 | return mr; |
| 416 | } |
| 417 | |
| 418 | static MemTxResult zpci_read_bar(S390PCIBusDevice *pbdev, uint8_t pcias, |
| 419 | uint64_t offset, uint64_t *data, uint8_t len) |
| 420 | { |
| 421 | MemoryRegion *mr; |
| 422 | |
| 423 | mr = pbdev->pdev->io_regions[pcias].memory; |
| 424 | mr = s390_get_subregion(mr, offset, len); |
| 425 | if (!mr) { |
| 426 | return MEMTX_ERROR; |
| 427 | } |
| 428 | |
| 429 | offset -= mr->addr; |
| 430 | return memory_region_dispatch_read(mr, offset, data, |
| 431 | size_memop(len) | MO_BE, |
| 432 | MEMTXATTRS_UNSPECIFIED); |
| 433 | } |
| 434 | |
| 435 | int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra) |
| 436 | { |
| 437 | CPUS390XState *env = &cpu->env; |
| 438 | S390PCIBusDevice *pbdev; |
| 439 | uint64_t offset; |
| 440 | uint64_t data; |
| 441 | MemTxResult result; |
| 442 | uint8_t len; |
| 443 | uint32_t fh; |
| 444 | uint8_t pcias; |
| 445 | |
| 446 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 447 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 448 | return 0; |
| 449 | } |
| 450 | |
| 451 | if (r2 & 0x1) { |
| 452 | s390_program_interrupt(env, PGM_SPECIFICATION, ra); |
| 453 | return 0; |
| 454 | } |
| 455 | |
| 456 | fh = env->regs[r2] >> 32; |
| 457 | pcias = (env->regs[r2] >> 16) & 0xf; |
| 458 | len = env->regs[r2] & 0xf; |
| 459 | offset = env->regs[r2 + 1]; |
| 460 | |
| 461 | if (!(fh & FH_MASK_ENABLE)) { |
| 462 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 463 | return 0; |
| 464 | } |
| 465 | |
| 466 | pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh); |
| 467 | if (!pbdev) { |
| 468 | trace_s390_pci_nodev("pcilg", fh); |
| 469 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 470 | return 0; |
| 471 | } |
| 472 | |
| 473 | switch (pbdev->state) { |
| 474 | case ZPCI_FS_PERMANENT_ERROR: |
| 475 | case ZPCI_FS_ERROR: |
| 476 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 477 | s390_set_status_code(env, r2, ZPCI_PCI_ST_BLOCKED); |
| 478 | return 0; |
| 479 | default: |
| 480 | break; |
| 481 | } |
| 482 | |
| 483 | switch (pcias) { |
| 484 | case ZPCI_IO_BAR_MIN...ZPCI_IO_BAR_MAX: |
| 485 | if (!len || (len > (8 - (offset & 0x7)))) { |
| 486 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 487 | return 0; |
| 488 | } |
| 489 | result = zpci_read_bar(pbdev, pcias, offset, &data, len); |
| 490 | if (result != MEMTX_OK) { |
| 491 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 492 | return 0; |
| 493 | } |
| 494 | break; |
| 495 | case ZPCI_CONFIG_BAR: |
| 496 | if (!len || (len > (4 - (offset & 0x3))) || len == 3) { |
| 497 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 498 | return 0; |
| 499 | } |
| 500 | data = pci_host_config_read_common( |
| 501 | pbdev->pdev, offset, pci_config_size(pbdev->pdev), len); |
| 502 | |
| 503 | if (zpci_endian_swap(&data, len)) { |
| 504 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 505 | return 0; |
| 506 | } |
| 507 | break; |
| 508 | default: |
| 509 | trace_s390_pci_invalid("pcilg", fh); |
| 510 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 511 | s390_set_status_code(env, r2, ZPCI_PCI_ST_INVAL_AS); |
| 512 | return 0; |
| 513 | } |
| 514 | |
| 515 | pbdev->fmb.counter[ZPCI_FMB_CNT_LD]++; |
| 516 | |
| 517 | env->regs[r1] = data; |
| 518 | setcc(cpu, ZPCI_PCI_LS_OK); |
| 519 | return 0; |
| 520 | } |
| 521 | |
| 522 | static MemTxResult zpci_write_bar(S390PCIBusDevice *pbdev, uint8_t pcias, |
| 523 | uint64_t offset, uint64_t data, uint8_t len) |
| 524 | { |
| 525 | MemoryRegion *mr; |
| 526 | |
| 527 | mr = pbdev->pdev->io_regions[pcias].memory; |
| 528 | mr = s390_get_subregion(mr, offset, len); |
| 529 | if (!mr) { |
| 530 | return MEMTX_ERROR; |
| 531 | } |
| 532 | |
| 533 | offset -= mr->addr; |
| 534 | return memory_region_dispatch_write(mr, offset, data, |
| 535 | size_memop(len) | MO_BE, |
| 536 | MEMTXATTRS_UNSPECIFIED); |
| 537 | } |
| 538 | |
| 539 | int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra) |
| 540 | { |
| 541 | CPUS390XState *env = &cpu->env; |
| 542 | uint64_t offset, data; |
| 543 | S390PCIBusDevice *pbdev; |
| 544 | MemTxResult result; |
| 545 | uint8_t len; |
| 546 | uint32_t fh; |
| 547 | uint8_t pcias; |
| 548 | |
| 549 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 550 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 551 | return 0; |
| 552 | } |
| 553 | |
| 554 | if (r2 & 0x1) { |
| 555 | s390_program_interrupt(env, PGM_SPECIFICATION, ra); |
| 556 | return 0; |
| 557 | } |
| 558 | |
| 559 | fh = env->regs[r2] >> 32; |
| 560 | pcias = (env->regs[r2] >> 16) & 0xf; |
| 561 | len = env->regs[r2] & 0xf; |
| 562 | offset = env->regs[r2 + 1]; |
| 563 | data = env->regs[r1]; |
| 564 | |
| 565 | if (!(fh & FH_MASK_ENABLE)) { |
| 566 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 567 | return 0; |
| 568 | } |
| 569 | |
| 570 | pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh); |
| 571 | if (!pbdev) { |
| 572 | trace_s390_pci_nodev("pcistg", fh); |
| 573 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 574 | return 0; |
| 575 | } |
| 576 | |
| 577 | switch (pbdev->state) { |
| 578 | /* ZPCI_FS_RESERVED, ZPCI_FS_STANDBY and ZPCI_FS_DISABLED |
| 579 | * are already covered by the FH_MASK_ENABLE check above |
| 580 | */ |
| 581 | case ZPCI_FS_PERMANENT_ERROR: |
| 582 | case ZPCI_FS_ERROR: |
| 583 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 584 | s390_set_status_code(env, r2, ZPCI_PCI_ST_BLOCKED); |
| 585 | return 0; |
| 586 | default: |
| 587 | break; |
| 588 | } |
| 589 | |
| 590 | switch (pcias) { |
| 591 | /* A ZPCI PCI card may use any BAR from BAR 0 to BAR 5 */ |
| 592 | case ZPCI_IO_BAR_MIN...ZPCI_IO_BAR_MAX: |
| 593 | /* Check length: |
| 594 | * A length of 0 is invalid and length should not cross a double word |
| 595 | */ |
| 596 | if (!len || (len > (8 - (offset & 0x7)))) { |
| 597 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 598 | return 0; |
| 599 | } |
| 600 | |
| 601 | result = zpci_write_bar(pbdev, pcias, offset, data, len); |
| 602 | if (result != MEMTX_OK) { |
| 603 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 604 | return 0; |
| 605 | } |
| 606 | break; |
| 607 | case ZPCI_CONFIG_BAR: |
| 608 | /* ZPCI uses the pseudo BAR number 15 as configuration space */ |
| 609 | /* possible access lengths are 1,2,4 and must not cross a word */ |
| 610 | if (!len || (len > (4 - (offset & 0x3))) || len == 3) { |
| 611 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 612 | return 0; |
| 613 | } |
| 614 | /* len = 1,2,4 so we do not need to test */ |
| 615 | zpci_endian_swap(&data, len); |
| 616 | pci_host_config_write_common(pbdev->pdev, offset, |
| 617 | pci_config_size(pbdev->pdev), |
| 618 | data, len); |
| 619 | break; |
| 620 | default: |
| 621 | trace_s390_pci_invalid("pcistg", fh); |
| 622 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 623 | s390_set_status_code(env, r2, ZPCI_PCI_ST_INVAL_AS); |
| 624 | return 0; |
| 625 | } |
| 626 | |
| 627 | pbdev->fmb.counter[ZPCI_FMB_CNT_ST]++; |
| 628 | |
| 629 | setcc(cpu, ZPCI_PCI_LS_OK); |
| 630 | return 0; |
| 631 | } |
| 632 | |
| 633 | static uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu, |
| 634 | S390IOTLBEntry *entry) |
| 635 | { |
| 636 | S390IOTLBEntry *cache = g_hash_table_lookup(iommu->iotlb, &entry->iova); |
| 637 | IOMMUTLBEvent event = { |
| 638 | .type = entry->perm ? IOMMU_NOTIFIER_MAP : IOMMU_NOTIFIER_UNMAP, |
| 639 | .entry = { |
| 640 | .target_as = &address_space_memory, |
| 641 | .iova = entry->iova, |
| 642 | .translated_addr = entry->translated_addr, |
| 643 | .perm = entry->perm, |
| 644 | .addr_mask = ~TARGET_PAGE_MASK, |
| 645 | }, |
| 646 | }; |
| 647 | |
| 648 | if (event.type == IOMMU_NOTIFIER_UNMAP) { |
| 649 | if (!cache) { |
| 650 | goto out; |
| 651 | } |
| 652 | g_hash_table_remove(iommu->iotlb, &entry->iova); |
| 653 | inc_dma_avail(iommu); |
| 654 | /* Don't notify the iommu yet, maybe we can bundle contiguous unmaps */ |
| 655 | goto out; |
| 656 | } else { |
| 657 | if (cache) { |
| 658 | if (cache->perm == entry->perm && |
| 659 | cache->translated_addr == entry->translated_addr) { |
| 660 | goto out; |
| 661 | } |
| 662 | |
| 663 | event.type = IOMMU_NOTIFIER_UNMAP; |
| 664 | event.entry.perm = IOMMU_NONE; |
| 665 | memory_region_notify_iommu(&iommu->iommu_mr, 0, event); |
| 666 | event.type = IOMMU_NOTIFIER_MAP; |
| 667 | event.entry.perm = entry->perm; |
| 668 | } |
| 669 | |
| 670 | cache = g_new(S390IOTLBEntry, 1); |
| 671 | cache->iova = entry->iova; |
| 672 | cache->translated_addr = entry->translated_addr; |
| 673 | cache->len = TARGET_PAGE_SIZE; |
| 674 | cache->perm = entry->perm; |
| 675 | g_hash_table_replace(iommu->iotlb, &cache->iova, cache); |
| 676 | dec_dma_avail(iommu); |
| 677 | } |
| 678 | |
| 679 | /* |
| 680 | * All associated iotlb entries have already been cleared, trigger the |
| 681 | * unmaps. |
| 682 | */ |
| 683 | memory_region_notify_iommu(&iommu->iommu_mr, 0, event); |
| 684 | |
| 685 | out: |
| 686 | return iommu->dma_limit ? iommu->dma_limit->avail : 1; |
| 687 | } |
| 688 | |
| 689 | static void s390_pci_batch_unmap(S390PCIIOMMU *iommu, uint64_t iova, |
| 690 | uint64_t len) |
| 691 | { |
| 692 | uint64_t remain = len, start = iova, end = start + len - 1, mask, size; |
| 693 | IOMMUTLBEvent event = { |
| 694 | .type = IOMMU_NOTIFIER_UNMAP, |
| 695 | .entry = { |
| 696 | .target_as = &address_space_memory, |
| 697 | .translated_addr = 0, |
| 698 | .perm = IOMMU_NONE, |
| 699 | }, |
| 700 | }; |
| 701 | |
| 702 | while (remain >= TARGET_PAGE_SIZE) { |
| 703 | mask = dma_aligned_pow2_mask(start, end, 64); |
| 704 | size = mask + 1; |
| 705 | event.entry.iova = start; |
| 706 | event.entry.addr_mask = mask; |
| 707 | memory_region_notify_iommu(&iommu->iommu_mr, 0, event); |
| 708 | start += size; |
| 709 | remain -= size; |
| 710 | } |
| 711 | } |
| 712 | |
| 713 | int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra) |
| 714 | { |
| 715 | CPUS390XState *env = &cpu->env; |
| 716 | uint64_t iova, coalesce = 0; |
| 717 | uint32_t fh; |
| 718 | uint16_t error = 0; |
| 719 | S390PCIBusDevice *pbdev; |
| 720 | S390PCIIOMMU *iommu; |
| 721 | S390IOTLBEntry entry; |
| 722 | hwaddr start, end, sstart; |
| 723 | uint32_t dma_avail; |
| 724 | bool again; |
| 725 | |
| 726 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 727 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 728 | return 0; |
| 729 | } |
| 730 | |
| 731 | if (r2 & 0x1) { |
| 732 | s390_program_interrupt(env, PGM_SPECIFICATION, ra); |
| 733 | return 0; |
| 734 | } |
| 735 | |
| 736 | fh = env->regs[r1] >> 32; |
| 737 | sstart = start = env->regs[r2]; |
| 738 | end = start + env->regs[r2 + 1]; |
| 739 | |
| 740 | pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh); |
| 741 | if (!pbdev) { |
| 742 | trace_s390_pci_nodev("rpcit", fh); |
| 743 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 744 | return 0; |
| 745 | } |
| 746 | |
| 747 | switch (pbdev->state) { |
| 748 | case ZPCI_FS_RESERVED: |
| 749 | case ZPCI_FS_STANDBY: |
| 750 | case ZPCI_FS_DISABLED: |
| 751 | case ZPCI_FS_PERMANENT_ERROR: |
| 752 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 753 | return 0; |
| 754 | case ZPCI_FS_ERROR: |
| 755 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 756 | s390_set_status_code(env, r1, ZPCI_MOD_ST_ERROR_RECOVER); |
| 757 | return 0; |
| 758 | default: |
| 759 | break; |
| 760 | } |
| 761 | |
| 762 | iommu = pbdev->iommu; |
| 763 | if (iommu->dma_limit) { |
| 764 | dma_avail = iommu->dma_limit->avail; |
| 765 | } else { |
| 766 | dma_avail = 1; |
| 767 | } |
| 768 | if (!iommu->g_iota) { |
| 769 | error = ERR_EVENT_INVALAS; |
| 770 | goto err; |
| 771 | } |
| 772 | |
| 773 | if (end < start || end < iommu->pba || start > iommu->pal) { |
| 774 | error = ERR_EVENT_OORANGE; |
| 775 | goto err; |
| 776 | } |
| 777 | /* |
| 778 | * If the specified range at least partially overlaps the registered |
| 779 | * aperture, clamp the request to the aperture and ignore the rest. |
| 780 | */ |
| 781 | sstart = MAX(start, iommu->pba); |
| 782 | end = MIN(end, iommu->pal + 1); |
| 783 | |
| 784 | retry: |
| 785 | start = sstart; |
| 786 | again = false; |
| 787 | while (start < end) { |
| 788 | error = s390_guest_io_table_walk(iommu->g_iota, start, &entry); |
| 789 | if (error) { |
| 790 | break; |
| 791 | } |
| 792 | |
| 793 | /* |
| 794 | * If this is an unmap of a PTE, let's try to coalesce multiple unmaps |
| 795 | * into as few notifier events as possible. |
| 796 | */ |
| 797 | if (entry.perm == IOMMU_NONE && entry.len == TARGET_PAGE_SIZE) { |
| 798 | if (coalesce == 0) { |
| 799 | iova = entry.iova; |
| 800 | } |
| 801 | coalesce += entry.len; |
| 802 | } else if (coalesce > 0) { |
| 803 | /* Unleash the coalesced unmap before processing a new map */ |
| 804 | s390_pci_batch_unmap(iommu, iova, coalesce); |
| 805 | coalesce = 0; |
| 806 | } |
| 807 | |
| 808 | start += entry.len; |
| 809 | while (entry.iova < start && entry.iova < end) { |
| 810 | if (dma_avail > 0 || entry.perm == IOMMU_NONE) { |
| 811 | dma_avail = s390_pci_update_iotlb(iommu, &entry); |
| 812 | entry.iova += TARGET_PAGE_SIZE; |
| 813 | entry.translated_addr += TARGET_PAGE_SIZE; |
| 814 | } else { |
| 815 | /* |
| 816 | * We are unable to make a new mapping at this time, continue |
| 817 | * on and hopefully free up more space. Then attempt another |
| 818 | * pass. |
| 819 | */ |
| 820 | again = true; |
| 821 | break; |
| 822 | } |
| 823 | } |
| 824 | } |
| 825 | if (coalesce) { |
| 826 | /* Unleash the coalesced unmap before finishing rpcit */ |
| 827 | s390_pci_batch_unmap(iommu, iova, coalesce); |
| 828 | coalesce = 0; |
| 829 | } |
| 830 | if (again && dma_avail > 0) |
| 831 | goto retry; |
| 832 | err: |
| 833 | if (error) { |
| 834 | pbdev->state = ZPCI_FS_ERROR; |
| 835 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 836 | s390_set_status_code(env, r1, ZPCI_PCI_ST_FUNC_IN_ERR); |
| 837 | s390_pci_generate_error_event(error, pbdev->fh, pbdev->fid, start, 0); |
| 838 | } else { |
| 839 | pbdev->fmb.counter[ZPCI_FMB_CNT_RPCIT]++; |
| 840 | if (dma_avail > 0) { |
| 841 | setcc(cpu, ZPCI_PCI_LS_OK); |
| 842 | } else { |
| 843 | /* vfio DMA mappings are exhausted, trigger a RPCIT */ |
| 844 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 845 | s390_set_status_code(env, r1, ZPCI_RPCIT_ST_INSUFF_RES); |
| 846 | } |
| 847 | } |
| 848 | return 0; |
| 849 | } |
| 850 | |
| 851 | int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr, |
| 852 | uint8_t ar, uintptr_t ra) |
| 853 | { |
| 854 | CPUS390XState *env = &cpu->env; |
| 855 | S390PCIBusDevice *pbdev; |
| 856 | MemoryRegion *mr; |
| 857 | MemTxResult result; |
| 858 | uint64_t offset; |
| 859 | int i; |
| 860 | uint32_t fh; |
| 861 | uint8_t pcias; |
| 862 | uint16_t len; |
| 863 | uint8_t buffer[128]; |
| 864 | |
| 865 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 866 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 867 | return 0; |
| 868 | } |
| 869 | |
| 870 | fh = env->regs[r1] >> 32; |
| 871 | pcias = (env->regs[r1] >> 16) & 0xf; |
| 872 | len = env->regs[r1] & 0x1fff; |
| 873 | offset = env->regs[r3]; |
| 874 | |
| 875 | if (!(fh & FH_MASK_ENABLE)) { |
| 876 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 877 | return 0; |
| 878 | } |
| 879 | |
| 880 | pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh); |
| 881 | if (!pbdev) { |
| 882 | trace_s390_pci_nodev("pcistb", fh); |
| 883 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 884 | return 0; |
| 885 | } |
| 886 | |
| 887 | switch (pbdev->state) { |
| 888 | case ZPCI_FS_PERMANENT_ERROR: |
| 889 | case ZPCI_FS_ERROR: |
| 890 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 891 | s390_set_status_code(env, r1, ZPCI_PCI_ST_BLOCKED); |
| 892 | return 0; |
| 893 | default: |
| 894 | break; |
| 895 | } |
| 896 | |
| 897 | if (pcias > ZPCI_IO_BAR_MAX) { |
| 898 | trace_s390_pci_invalid("pcistb", fh); |
| 899 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 900 | s390_set_status_code(env, r1, ZPCI_PCI_ST_INVAL_AS); |
| 901 | return 0; |
| 902 | } |
| 903 | |
| 904 | /* Verify the address, offset and length */ |
| 905 | /* offset must be a multiple of 8 */ |
| 906 | if (offset % 8) { |
| 907 | goto specification_error; |
| 908 | } |
| 909 | /* Length must be greater than 8, a multiple of 8 */ |
| 910 | /* and not greater than maxstbl */ |
| 911 | if ((len <= 8) || (len % 8) || |
| 912 | (len > pbdev->pci_group->zpci_group.maxstbl)) { |
| 913 | goto specification_error; |
| 914 | } |
| 915 | /* Do not cross a 4K-byte boundary */ |
| 916 | if (((offset & 0xfff) + len) > 0x1000) { |
| 917 | goto specification_error; |
| 918 | } |
| 919 | /* Guest address must be double word aligned */ |
| 920 | if (gaddr & 0x07UL) { |
| 921 | goto specification_error; |
| 922 | } |
| 923 | |
| 924 | mr = pbdev->pdev->io_regions[pcias].memory; |
| 925 | mr = s390_get_subregion(mr, offset, len); |
| 926 | if (!mr) { |
| 927 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 928 | return 0; |
| 929 | } |
| 930 | |
| 931 | offset -= mr->addr; |
| 932 | |
| 933 | for (i = 0; i < len; i += 8) { |
| 934 | if (!memory_region_access_valid(mr, offset + i, 8, true, |
| 935 | MEMTXATTRS_UNSPECIFIED)) { |
| 936 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 937 | return 0; |
| 938 | } |
| 939 | } |
| 940 | |
| 941 | if (s390_cpu_virt_mem_read(cpu, gaddr, ar, buffer, len)) { |
| 942 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 943 | return 0; |
| 944 | } |
| 945 | |
| 946 | for (i = 0; i < len / 8; i++) { |
| 947 | result = memory_region_dispatch_write(mr, offset + i * 8, |
| 948 | ldq_be_p(buffer + i * 8), |
| 949 | MO_64, MEMTXATTRS_UNSPECIFIED); |
| 950 | if (result != MEMTX_OK) { |
| 951 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 952 | return 0; |
| 953 | } |
| 954 | } |
| 955 | |
| 956 | pbdev->fmb.counter[ZPCI_FMB_CNT_STB]++; |
| 957 | |
| 958 | setcc(cpu, ZPCI_PCI_LS_OK); |
| 959 | return 0; |
| 960 | |
| 961 | specification_error: |
| 962 | s390_program_interrupt(env, PGM_SPECIFICATION, ra); |
| 963 | return 0; |
| 964 | } |
| 965 | |
| 966 | static int reg_irqs(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib) |
| 967 | { |
| 968 | int ret, len; |
| 969 | uint8_t isc = FIB_DATA_ISC(ldl_be_p(&fib.data)); |
| 970 | |
| 971 | pbdev->routes.adapter.adapter_id = css_get_adapter_id( |
| 972 | CSS_IO_ADAPTER_PCI, isc); |
| 973 | pbdev->summary_ind = get_indicator(ldq_be_p(&fib.aisb), sizeof(uint64_t)); |
| 974 | len = BITS_TO_LONGS(FIB_DATA_NOI(ldl_be_p(&fib.data))) * sizeof(unsigned long); |
| 975 | pbdev->indicator = get_indicator(ldq_be_p(&fib.aibv), len); |
| 976 | |
| 977 | ret = map_indicator(&pbdev->routes.adapter, pbdev->summary_ind); |
| 978 | if (ret) { |
| 979 | goto out; |
| 980 | } |
| 981 | |
| 982 | ret = map_indicator(&pbdev->routes.adapter, pbdev->indicator); |
| 983 | if (ret) { |
| 984 | goto out; |
| 985 | } |
| 986 | |
| 987 | pbdev->routes.adapter.summary_addr = ldq_be_p(&fib.aisb); |
| 988 | pbdev->routes.adapter.summary_offset = FIB_DATA_AISBO(ldl_be_p(&fib.data)); |
| 989 | pbdev->routes.adapter.ind_addr = ldq_be_p(&fib.aibv); |
| 990 | pbdev->routes.adapter.ind_offset = FIB_DATA_AIBVO(ldl_be_p(&fib.data)); |
| 991 | pbdev->isc = isc; |
| 992 | pbdev->noi = FIB_DATA_NOI(ldl_be_p(&fib.data)); |
| 993 | pbdev->sum = FIB_DATA_SUM(ldl_be_p(&fib.data)); |
| 994 | |
| 995 | trace_s390_pci_irqs("register", pbdev->routes.adapter.adapter_id); |
| 996 | return 0; |
| 997 | out: |
| 998 | release_indicator(&pbdev->routes.adapter, pbdev->summary_ind); |
| 999 | release_indicator(&pbdev->routes.adapter, pbdev->indicator); |
| 1000 | pbdev->summary_ind = NULL; |
| 1001 | pbdev->indicator = NULL; |
| 1002 | return ret; |
| 1003 | } |
| 1004 | |
| 1005 | int pci_dereg_irqs(S390PCIBusDevice *pbdev) |
| 1006 | { |
| 1007 | release_indicator(&pbdev->routes.adapter, pbdev->summary_ind); |
| 1008 | release_indicator(&pbdev->routes.adapter, pbdev->indicator); |
| 1009 | |
| 1010 | pbdev->summary_ind = NULL; |
| 1011 | pbdev->indicator = NULL; |
| 1012 | pbdev->routes.adapter.summary_addr = 0; |
| 1013 | pbdev->routes.adapter.summary_offset = 0; |
| 1014 | pbdev->routes.adapter.ind_addr = 0; |
| 1015 | pbdev->routes.adapter.ind_offset = 0; |
| 1016 | pbdev->isc = 0; |
| 1017 | pbdev->noi = 0; |
| 1018 | pbdev->sum = 0; |
| 1019 | |
| 1020 | trace_s390_pci_irqs("unregister", pbdev->routes.adapter.adapter_id); |
| 1021 | return 0; |
| 1022 | } |
| 1023 | |
| 1024 | static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib, |
| 1025 | uintptr_t ra) |
| 1026 | { |
| 1027 | S390PCIIOMMU *iommu = pbdev->iommu; |
| 1028 | uint64_t pba = ldq_be_p(&fib.pba); |
| 1029 | uint64_t pal = ldq_be_p(&fib.pal); |
| 1030 | uint64_t g_iota = ldq_be_p(&fib.iota); |
| 1031 | uint8_t dt = (g_iota >> 2) & 0x7; |
| 1032 | uint8_t t = (g_iota >> 11) & 0x1; |
| 1033 | |
| 1034 | pba &= ~0xfff; |
| 1035 | pal |= 0xfff; |
| 1036 | if (pba > pal || pba < pbdev->zpci_fn.sdma || pal > pbdev->zpci_fn.edma) { |
| 1037 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 1038 | return -EINVAL; |
| 1039 | } |
| 1040 | |
| 1041 | /* currently we only support designation type 1 with translation */ |
| 1042 | if (t && dt != ZPCI_IOTA_RTTO) { |
| 1043 | error_report("unsupported ioat dt %d t %d", dt, t); |
| 1044 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 1045 | return -EINVAL; |
| 1046 | } else if (!t && !pbdev->rtr_avail) { |
| 1047 | error_report("relaxed translation not allowed"); |
| 1048 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 1049 | return -EINVAL; |
| 1050 | } |
| 1051 | |
| 1052 | iommu->pba = pba; |
| 1053 | iommu->pal = pal; |
| 1054 | iommu->g_iota = g_iota; |
| 1055 | |
| 1056 | if (t) { |
| 1057 | s390_pci_iommu_enable(iommu); |
| 1058 | } else { |
| 1059 | s390_pci_iommu_direct_map_enable(iommu); |
| 1060 | } |
| 1061 | |
| 1062 | return 0; |
| 1063 | } |
| 1064 | |
| 1065 | void pci_dereg_ioat(S390PCIIOMMU *iommu) |
| 1066 | { |
| 1067 | s390_pci_iommu_disable(iommu); |
| 1068 | iommu->pba = 0; |
| 1069 | iommu->pal = 0; |
| 1070 | iommu->g_iota = 0; |
| 1071 | } |
| 1072 | |
| 1073 | void fmb_timer_free(S390PCIBusDevice *pbdev) |
| 1074 | { |
| 1075 | if (pbdev->fmb_timer) { |
| 1076 | timer_free(pbdev->fmb_timer); |
| 1077 | pbdev->fmb_timer = NULL; |
| 1078 | } |
| 1079 | pbdev->fmb_addr = 0; |
| 1080 | memset(&pbdev->fmb, 0, sizeof(ZpciFmb)); |
| 1081 | } |
| 1082 | |
| 1083 | static int fmb_do_update(S390PCIBusDevice *pbdev, int offset, uint64_t val, |
| 1084 | int len) |
| 1085 | { |
| 1086 | MemTxResult ret; |
| 1087 | uint64_t dst = pbdev->fmb_addr + offset; |
| 1088 | |
| 1089 | switch (len) { |
| 1090 | case 8: |
| 1091 | address_space_stq_be(&address_space_memory, dst, val, |
| 1092 | MEMTXATTRS_UNSPECIFIED, |
| 1093 | &ret); |
| 1094 | break; |
| 1095 | case 4: |
| 1096 | address_space_stl_be(&address_space_memory, dst, val, |
| 1097 | MEMTXATTRS_UNSPECIFIED, |
| 1098 | &ret); |
| 1099 | break; |
| 1100 | case 2: |
| 1101 | address_space_stw_be(&address_space_memory, dst, val, |
| 1102 | MEMTXATTRS_UNSPECIFIED, |
| 1103 | &ret); |
| 1104 | break; |
| 1105 | case 1: |
| 1106 | address_space_stb(&address_space_memory, dst, val, |
| 1107 | MEMTXATTRS_UNSPECIFIED, |
| 1108 | &ret); |
| 1109 | break; |
| 1110 | default: |
| 1111 | ret = MEMTX_ERROR; |
| 1112 | break; |
| 1113 | } |
| 1114 | if (ret != MEMTX_OK) { |
| 1115 | s390_pci_generate_error_event(ERR_EVENT_FMBA, pbdev->fh, pbdev->fid, |
| 1116 | pbdev->fmb_addr, 0); |
| 1117 | fmb_timer_free(pbdev); |
| 1118 | } |
| 1119 | |
| 1120 | return ret; |
| 1121 | } |
| 1122 | |
| 1123 | static void fmb_update(void *opaque) |
| 1124 | { |
| 1125 | S390PCIBusDevice *pbdev = opaque; |
| 1126 | int64_t t = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL); |
| 1127 | int i; |
| 1128 | |
| 1129 | /* Update U bit */ |
| 1130 | pbdev->fmb.last_update *= 2; |
| 1131 | pbdev->fmb.last_update |= UPDATE_U_BIT; |
| 1132 | if (fmb_do_update(pbdev, offsetof(ZpciFmb, last_update), |
| 1133 | pbdev->fmb.last_update, |
| 1134 | sizeof(pbdev->fmb.last_update))) { |
| 1135 | return; |
| 1136 | } |
| 1137 | |
| 1138 | /* Update FMB sample count */ |
| 1139 | if (fmb_do_update(pbdev, offsetof(ZpciFmb, sample), |
| 1140 | pbdev->fmb.sample++, |
| 1141 | sizeof(pbdev->fmb.sample))) { |
| 1142 | return; |
| 1143 | } |
| 1144 | |
| 1145 | /* Update FMB counters */ |
| 1146 | for (i = 0; i < ZPCI_FMB_CNT_MAX; i++) { |
| 1147 | if (fmb_do_update(pbdev, offsetof(ZpciFmb, counter[i]), |
| 1148 | pbdev->fmb.counter[i], |
| 1149 | sizeof(pbdev->fmb.counter[0]))) { |
| 1150 | return; |
| 1151 | } |
| 1152 | } |
| 1153 | |
| 1154 | /* Clear U bit and update the time */ |
| 1155 | pbdev->fmb.last_update = time2tod(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)); |
| 1156 | pbdev->fmb.last_update *= 2; |
| 1157 | if (fmb_do_update(pbdev, offsetof(ZpciFmb, last_update), |
| 1158 | pbdev->fmb.last_update, |
| 1159 | sizeof(pbdev->fmb.last_update))) { |
| 1160 | return; |
| 1161 | } |
| 1162 | timer_mod(pbdev->fmb_timer, t + pbdev->pci_group->zpci_group.mui); |
| 1163 | } |
| 1164 | |
| 1165 | static int mpcifc_reg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib) |
| 1166 | { |
| 1167 | int rc; |
| 1168 | |
| 1169 | rc = s390_pci_kvm_aif_enable(pbdev, fib, pbdev->forwarding_assist); |
| 1170 | if (rc) { |
| 1171 | trace_s390_pci_kvm_aif("enable"); |
| 1172 | return rc; |
| 1173 | } |
| 1174 | |
| 1175 | return 0; |
| 1176 | } |
| 1177 | |
| 1178 | static int mpcifc_dereg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib) |
| 1179 | { |
| 1180 | int rc; |
| 1181 | |
| 1182 | rc = s390_pci_kvm_aif_disable(pbdev); |
| 1183 | if (rc) { |
| 1184 | trace_s390_pci_kvm_aif("disable"); |
| 1185 | return rc; |
| 1186 | } |
| 1187 | |
| 1188 | return 0; |
| 1189 | } |
| 1190 | |
| 1191 | int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar, |
| 1192 | uintptr_t ra) |
| 1193 | { |
| 1194 | CPUS390XState *env = &cpu->env; |
| 1195 | uint8_t oc, dmaas; |
| 1196 | uint32_t fh; |
| 1197 | ZpciFib fib; |
| 1198 | S390PCIBusDevice *pbdev; |
| 1199 | uint64_t cc = ZPCI_PCI_LS_OK; |
| 1200 | |
| 1201 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 1202 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 1203 | return 0; |
| 1204 | } |
| 1205 | |
| 1206 | oc = env->regs[r1] & 0xff; |
| 1207 | dmaas = (env->regs[r1] >> 16) & 0xff; |
| 1208 | fh = env->regs[r1] >> 32; |
| 1209 | |
| 1210 | if (fiba & 0x7) { |
| 1211 | s390_program_interrupt(env, PGM_SPECIFICATION, ra); |
| 1212 | return 0; |
| 1213 | } |
| 1214 | |
| 1215 | pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh); |
| 1216 | if (!pbdev) { |
| 1217 | trace_s390_pci_nodev("mpcifc", fh); |
| 1218 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 1219 | return 0; |
| 1220 | } |
| 1221 | |
| 1222 | switch (pbdev->state) { |
| 1223 | case ZPCI_FS_RESERVED: |
| 1224 | case ZPCI_FS_STANDBY: |
| 1225 | case ZPCI_FS_DISABLED: |
| 1226 | case ZPCI_FS_PERMANENT_ERROR: |
| 1227 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 1228 | return 0; |
| 1229 | default: |
| 1230 | break; |
| 1231 | } |
| 1232 | |
| 1233 | if (s390_cpu_virt_mem_read(cpu, fiba, ar, (uint8_t *)&fib, sizeof(fib))) { |
| 1234 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 1235 | return 0; |
| 1236 | } |
| 1237 | |
| 1238 | if (fib.fmt != 0) { |
| 1239 | s390_program_interrupt(env, PGM_OPERAND, ra); |
| 1240 | return 0; |
| 1241 | } |
| 1242 | |
| 1243 | switch (oc) { |
| 1244 | case ZPCI_MOD_FC_REG_INT: |
| 1245 | if (pbdev->interp) { |
| 1246 | if (mpcifc_reg_int_interp(pbdev, &fib)) { |
| 1247 | cc = ZPCI_PCI_LS_ERR; |
| 1248 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1249 | } |
| 1250 | } else if (pbdev->summary_ind) { |
| 1251 | cc = ZPCI_PCI_LS_ERR; |
| 1252 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1253 | } else if (reg_irqs(env, pbdev, fib)) { |
| 1254 | cc = ZPCI_PCI_LS_ERR; |
| 1255 | s390_set_status_code(env, r1, ZPCI_MOD_ST_RES_NOT_AVAIL); |
| 1256 | } |
| 1257 | break; |
| 1258 | case ZPCI_MOD_FC_DEREG_INT: |
| 1259 | if (pbdev->interp) { |
| 1260 | if (mpcifc_dereg_int_interp(pbdev, &fib)) { |
| 1261 | cc = ZPCI_PCI_LS_ERR; |
| 1262 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1263 | } |
| 1264 | } else if (!pbdev->summary_ind) { |
| 1265 | cc = ZPCI_PCI_LS_ERR; |
| 1266 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1267 | } else { |
| 1268 | pci_dereg_irqs(pbdev); |
| 1269 | } |
| 1270 | break; |
| 1271 | case ZPCI_MOD_FC_REG_IOAT: |
| 1272 | if (dmaas != 0) { |
| 1273 | cc = ZPCI_PCI_LS_ERR; |
| 1274 | s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL); |
| 1275 | } else if (pbdev->iommu->enabled) { |
| 1276 | cc = ZPCI_PCI_LS_ERR; |
| 1277 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1278 | } else if (reg_ioat(env, pbdev, fib, ra)) { |
| 1279 | cc = ZPCI_PCI_LS_ERR; |
| 1280 | s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES); |
| 1281 | } |
| 1282 | break; |
| 1283 | case ZPCI_MOD_FC_DEREG_IOAT: |
| 1284 | if (dmaas != 0) { |
| 1285 | cc = ZPCI_PCI_LS_ERR; |
| 1286 | s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL); |
| 1287 | } else if (!pbdev->iommu->enabled) { |
| 1288 | cc = ZPCI_PCI_LS_ERR; |
| 1289 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1290 | } else { |
| 1291 | pci_dereg_ioat(pbdev->iommu); |
| 1292 | } |
| 1293 | break; |
| 1294 | case ZPCI_MOD_FC_REREG_IOAT: |
| 1295 | if (dmaas != 0) { |
| 1296 | cc = ZPCI_PCI_LS_ERR; |
| 1297 | s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL); |
| 1298 | } else if (!pbdev->iommu->enabled) { |
| 1299 | cc = ZPCI_PCI_LS_ERR; |
| 1300 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1301 | } else { |
| 1302 | pci_dereg_ioat(pbdev->iommu); |
| 1303 | if (reg_ioat(env, pbdev, fib, ra)) { |
| 1304 | cc = ZPCI_PCI_LS_ERR; |
| 1305 | s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES); |
| 1306 | } |
| 1307 | } |
| 1308 | break; |
| 1309 | case ZPCI_MOD_FC_RESET_ERROR: |
| 1310 | switch (pbdev->state) { |
| 1311 | case ZPCI_FS_BLOCKED: |
| 1312 | case ZPCI_FS_ERROR: |
| 1313 | pbdev->state = ZPCI_FS_ENABLED; |
| 1314 | break; |
| 1315 | default: |
| 1316 | cc = ZPCI_PCI_LS_ERR; |
| 1317 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1318 | } |
| 1319 | break; |
| 1320 | case ZPCI_MOD_FC_RESET_BLOCK: |
| 1321 | switch (pbdev->state) { |
| 1322 | case ZPCI_FS_ERROR: |
| 1323 | pbdev->state = ZPCI_FS_BLOCKED; |
| 1324 | break; |
| 1325 | default: |
| 1326 | cc = ZPCI_PCI_LS_ERR; |
| 1327 | s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); |
| 1328 | } |
| 1329 | break; |
| 1330 | case ZPCI_MOD_FC_SET_MEASURE: { |
| 1331 | uint64_t fmb_addr = ldq_be_p(&fib.fmb_addr); |
| 1332 | |
| 1333 | if (fmb_addr & FMBK_MASK) { |
| 1334 | cc = ZPCI_PCI_LS_ERR; |
| 1335 | s390_pci_generate_error_event(ERR_EVENT_FMBPRO, pbdev->fh, |
| 1336 | pbdev->fid, fmb_addr, 0); |
| 1337 | fmb_timer_free(pbdev); |
| 1338 | break; |
| 1339 | } |
| 1340 | |
| 1341 | if (!fmb_addr) { |
| 1342 | /* Stop updating FMB. */ |
| 1343 | fmb_timer_free(pbdev); |
| 1344 | break; |
| 1345 | } |
| 1346 | |
| 1347 | if (!pbdev->fmb_timer) { |
| 1348 | pbdev->fmb_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, |
| 1349 | fmb_update, pbdev); |
| 1350 | } else if (timer_pending(pbdev->fmb_timer)) { |
| 1351 | /* Remove pending timer to update FMB address. */ |
| 1352 | timer_del(pbdev->fmb_timer); |
| 1353 | } |
| 1354 | pbdev->fmb_addr = fmb_addr; |
| 1355 | timer_mod(pbdev->fmb_timer, |
| 1356 | qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + |
| 1357 | pbdev->pci_group->zpci_group.mui); |
| 1358 | break; |
| 1359 | } |
| 1360 | default: |
| 1361 | s390_program_interrupt(&cpu->env, PGM_OPERAND, ra); |
| 1362 | cc = ZPCI_PCI_LS_ERR; |
| 1363 | } |
| 1364 | |
| 1365 | setcc(cpu, cc); |
| 1366 | return 0; |
| 1367 | } |
| 1368 | |
| 1369 | int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar, |
| 1370 | uintptr_t ra) |
| 1371 | { |
| 1372 | CPUS390XState *env = &cpu->env; |
| 1373 | uint8_t dmaas; |
| 1374 | uint32_t fh; |
| 1375 | ZpciFib fib; |
| 1376 | S390PCIBusDevice *pbdev; |
| 1377 | uint32_t data; |
| 1378 | uint64_t cc = ZPCI_PCI_LS_OK; |
| 1379 | |
| 1380 | if (env->psw.mask & PSW_MASK_PSTATE) { |
| 1381 | s390_program_interrupt(env, PGM_PRIVILEGED, ra); |
| 1382 | return 0; |
| 1383 | } |
| 1384 | |
| 1385 | fh = env->regs[r1] >> 32; |
| 1386 | dmaas = (env->regs[r1] >> 16) & 0xff; |
| 1387 | |
| 1388 | if (dmaas) { |
| 1389 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 1390 | s390_set_status_code(env, r1, ZPCI_STPCIFC_ST_INVAL_DMAAS); |
| 1391 | return 0; |
| 1392 | } |
| 1393 | |
| 1394 | if (fiba & 0x7) { |
| 1395 | s390_program_interrupt(env, PGM_SPECIFICATION, ra); |
| 1396 | return 0; |
| 1397 | } |
| 1398 | |
| 1399 | pbdev = s390_pci_find_dev_by_idx(s390_get_phb(), fh & FH_MASK_INDEX); |
| 1400 | if (!pbdev) { |
| 1401 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 1402 | return 0; |
| 1403 | } |
| 1404 | |
| 1405 | memset(&fib, 0, sizeof(fib)); |
| 1406 | |
| 1407 | switch (pbdev->state) { |
| 1408 | case ZPCI_FS_RESERVED: |
| 1409 | case ZPCI_FS_STANDBY: |
| 1410 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 1411 | return 0; |
| 1412 | case ZPCI_FS_DISABLED: |
| 1413 | if (fh & FH_MASK_ENABLE) { |
| 1414 | setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE); |
| 1415 | return 0; |
| 1416 | } |
| 1417 | goto out; |
| 1418 | /* BLOCKED bit is set to one coincident with the setting of ERROR bit. |
| 1419 | * FH Enabled bit is set to one in states of ENABLED, BLOCKED or ERROR. */ |
| 1420 | case ZPCI_FS_ERROR: |
| 1421 | fib.fc |= 0x20; |
| 1422 | /* fallthrough */ |
| 1423 | case ZPCI_FS_BLOCKED: |
| 1424 | fib.fc |= 0x40; |
| 1425 | /* fallthrough */ |
| 1426 | case ZPCI_FS_ENABLED: |
| 1427 | fib.fc |= 0x80; |
| 1428 | if (pbdev->iommu->enabled) { |
| 1429 | fib.fc |= 0x10; |
| 1430 | } |
| 1431 | if (!(fh & FH_MASK_ENABLE)) { |
| 1432 | env->regs[r1] |= 1ULL << 63; |
| 1433 | } |
| 1434 | break; |
| 1435 | case ZPCI_FS_PERMANENT_ERROR: |
| 1436 | setcc(cpu, ZPCI_PCI_LS_ERR); |
| 1437 | s390_set_status_code(env, r1, ZPCI_STPCIFC_ST_PERM_ERROR); |
| 1438 | return 0; |
| 1439 | } |
| 1440 | |
| 1441 | stq_be_p(&fib.pba, pbdev->iommu->pba); |
| 1442 | stq_be_p(&fib.pal, pbdev->iommu->pal); |
| 1443 | stq_be_p(&fib.iota, pbdev->iommu->g_iota); |
| 1444 | stq_be_p(&fib.aibv, pbdev->routes.adapter.ind_addr); |
| 1445 | stq_be_p(&fib.aisb, pbdev->routes.adapter.summary_addr); |
| 1446 | stq_be_p(&fib.fmb_addr, pbdev->fmb_addr); |
| 1447 | |
| 1448 | data = ((uint32_t)pbdev->isc << 28) | ((uint32_t)pbdev->noi << 16) | |
| 1449 | ((uint32_t)pbdev->routes.adapter.ind_offset << 8) | |
| 1450 | ((uint32_t)pbdev->sum << 7) | pbdev->routes.adapter.summary_offset; |
| 1451 | stl_be_p(&fib.data, data); |
| 1452 | |
| 1453 | out: |
| 1454 | if (s390_cpu_virt_mem_write(cpu, fiba, ar, (uint8_t *)&fib, sizeof(fib))) { |
| 1455 | s390_cpu_virt_mem_handle_exc(cpu, ra); |
| 1456 | return 0; |
| 1457 | } |
| 1458 | |
| 1459 | setcc(cpu, cc); |
| 1460 | return 0; |
| 1461 | } |