| 1 | /* |
| 2 | * RISC-V IOMMU - Hardware Performance Monitor (HPM) helpers |
| 3 | * |
| 4 | * Copyright (C) 2022-2023 Rivos Inc. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms and conditions of the GNU General Public License, |
| 8 | * version 2 or later, as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, |
| 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | * GNU General Public License for more details. |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License along |
| 16 | * with this program; if not, see <http://www.gnu.org/licenses/>. |
| 17 | */ |
| 18 | |
| 19 | #include "qemu/osdep.h" |
| 20 | #include "qemu/timer.h" |
| 21 | #include "cpu_bits.h" |
| 22 | #include "riscv-iommu-hpm.h" |
| 23 | #include "riscv-iommu.h" |
| 24 | #include "riscv-iommu-bits.h" |
| 25 | #include "trace.h" |
| 26 | |
| 27 | /* For now we assume IOMMU HPM frequency to be 1GHz so 1-cycle is of 1-ns. */ |
| 28 | static inline uint64_t get_cycles(void) |
| 29 | { |
| 30 | return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); |
| 31 | } |
| 32 | |
| 33 | uint64_t riscv_iommu_hpmcycle_read(RISCVIOMMUState *s) |
| 34 | { |
| 35 | const uint64_t cycle = riscv_iommu_reg_get64( |
| 36 | s, RISCV_IOMMU_REG_IOHPMCYCLES); |
| 37 | const uint32_t inhibit = riscv_iommu_reg_get32( |
| 38 | s, RISCV_IOMMU_REG_IOCOUNTINH); |
| 39 | const uint64_t ctr_prev = s->hpmcycle_prev; |
| 40 | const uint64_t ctr_val = s->hpmcycle_val; |
| 41 | |
| 42 | trace_riscv_iommu_hpm_read(cycle, inhibit, ctr_prev, ctr_val); |
| 43 | |
| 44 | if (get_field(inhibit, RISCV_IOMMU_IOCOUNTINH_CY)) { |
| 45 | /* |
| 46 | * Counter should not increment if inhibit bit is set. We can't really |
| 47 | * stop the QEMU_CLOCK_VIRTUAL, so we just return the last updated |
| 48 | * counter value to indicate that counter was not incremented. |
| 49 | */ |
| 50 | return (ctr_val & RISCV_IOMMU_IOHPMCYCLES_COUNTER) | |
| 51 | (cycle & RISCV_IOMMU_IOHPMCYCLES_OVF); |
| 52 | } |
| 53 | |
| 54 | return (ctr_val + get_cycles() - ctr_prev) | |
| 55 | (cycle & RISCV_IOMMU_IOHPMCYCLES_OVF); |
| 56 | } |
| 57 | |
| 58 | static void hpm_incr_ctr(RISCVIOMMUState *s, uint32_t ctr_idx) |
| 59 | { |
| 60 | const uint32_t off = ctr_idx << 3; |
| 61 | uint64_t cntr_val; |
| 62 | |
| 63 | cntr_val = ldq_le_p(&s->regs[RISCV_IOMMU_REG_IOHPMCTR_BASE + off]); |
| 64 | stq_le_p(&s->regs[RISCV_IOMMU_REG_IOHPMCTR_BASE + off], cntr_val + 1); |
| 65 | |
| 66 | trace_riscv_iommu_hpm_incr_ctr(cntr_val); |
| 67 | |
| 68 | /* Handle the overflow scenario. */ |
| 69 | if (cntr_val == UINT64_MAX) { |
| 70 | /* |
| 71 | * Generate interrupt only if OF bit is clear. +1 to offset the cycle |
| 72 | * register OF bit. |
| 73 | */ |
| 74 | const uint32_t ovf = |
| 75 | riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IOCOUNTOVF, |
| 76 | BIT(ctr_idx + 1), 0); |
| 77 | if (!get_field(ovf, BIT(ctr_idx + 1))) { |
| 78 | riscv_iommu_reg_mod64(s, |
| 79 | RISCV_IOMMU_REG_IOHPMEVT_BASE + off, |
| 80 | RISCV_IOMMU_IOHPMEVT_OF, |
| 81 | 0); |
| 82 | riscv_iommu_notify(s, RISCV_IOMMU_INTR_PM); |
| 83 | } |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | void riscv_iommu_hpm_incr_ctr(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, |
| 88 | unsigned event_id) |
| 89 | { |
| 90 | const uint32_t inhibit = riscv_iommu_reg_get32( |
| 91 | s, RISCV_IOMMU_REG_IOCOUNTINH); |
| 92 | uint32_t did_gscid; |
| 93 | uint32_t pid_pscid; |
| 94 | uint32_t ctr_idx; |
| 95 | gpointer value; |
| 96 | uint32_t ctrs; |
| 97 | uint64_t evt; |
| 98 | |
| 99 | if (!(s->cap & RISCV_IOMMU_CAP_HPM)) { |
| 100 | return; |
| 101 | } |
| 102 | |
| 103 | value = g_hash_table_lookup(s->hpm_event_ctr_map, |
| 104 | GUINT_TO_POINTER(event_id)); |
| 105 | if (value == NULL) { |
| 106 | return; |
| 107 | } |
| 108 | |
| 109 | for (ctrs = GPOINTER_TO_UINT(value); ctrs != 0; ctrs &= ctrs - 1) { |
| 110 | ctr_idx = ctz32(ctrs); |
| 111 | if (get_field(inhibit, BIT(ctr_idx + 1))) { |
| 112 | continue; |
| 113 | } |
| 114 | |
| 115 | evt = riscv_iommu_reg_get64(s, |
| 116 | RISCV_IOMMU_REG_IOHPMEVT_BASE + (ctr_idx << 3)); |
| 117 | |
| 118 | /* |
| 119 | * It's quite possible that event ID has been changed in counter |
| 120 | * but hashtable hasn't been updated yet. We don't want to increment |
| 121 | * counter for the old event ID. |
| 122 | */ |
| 123 | if (event_id != get_field(evt, RISCV_IOMMU_IOHPMEVT_EVENT_ID)) { |
| 124 | continue; |
| 125 | } |
| 126 | |
| 127 | if (get_field(evt, RISCV_IOMMU_IOHPMEVT_IDT)) { |
| 128 | did_gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID); |
| 129 | pid_pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID); |
| 130 | } else { |
| 131 | did_gscid = ctx->devid; |
| 132 | pid_pscid = ctx->process_id; |
| 133 | } |
| 134 | |
| 135 | if (get_field(evt, RISCV_IOMMU_IOHPMEVT_PV_PSCV)) { |
| 136 | /* |
| 137 | * If the transaction does not have a valid process_id, counter |
| 138 | * increments if device_id matches DID_GSCID. If the transaction |
| 139 | * has a valid process_id, counter increments if device_id |
| 140 | * matches DID_GSCID and process_id matches PID_PSCID. See |
| 141 | * IOMMU Specification, Chapter 5.23. Performance-monitoring |
| 142 | * event selector. |
| 143 | */ |
| 144 | if (ctx->process_id && |
| 145 | get_field(evt, RISCV_IOMMU_IOHPMEVT_PID_PSCID) != pid_pscid) { |
| 146 | continue; |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | if (get_field(evt, RISCV_IOMMU_IOHPMEVT_DV_GSCV)) { |
| 151 | uint32_t mask = ~0; |
| 152 | |
| 153 | if (get_field(evt, RISCV_IOMMU_IOHPMEVT_DMASK)) { |
| 154 | /* |
| 155 | * 1001 1011 mask = GSCID |
| 156 | * 0000 0111 mask = mask ^ (mask + 1) |
| 157 | * 1111 1000 mask = ~mask; |
| 158 | */ |
| 159 | mask = get_field(evt, RISCV_IOMMU_IOHPMEVT_DID_GSCID); |
| 160 | mask = mask ^ (mask + 1); |
| 161 | mask = ~mask; |
| 162 | } |
| 163 | |
| 164 | if ((get_field(evt, RISCV_IOMMU_IOHPMEVT_DID_GSCID) & mask) != |
| 165 | (did_gscid & mask)) { |
| 166 | continue; |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | hpm_incr_ctr(s, ctr_idx); |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | /* Timer callback for cycle counter overflow. */ |
| 175 | void riscv_iommu_hpm_timer_cb(void *priv) |
| 176 | { |
| 177 | RISCVIOMMUState *s = priv; |
| 178 | const uint32_t inhibit = riscv_iommu_reg_get32( |
| 179 | s, RISCV_IOMMU_REG_IOCOUNTINH); |
| 180 | uint32_t ovf; |
| 181 | |
| 182 | if (get_field(inhibit, RISCV_IOMMU_IOCOUNTINH_CY)) { |
| 183 | return; |
| 184 | } |
| 185 | |
| 186 | if (s->irq_overflow_left > 0) { |
| 187 | uint64_t irq_trigger_at = |
| 188 | qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + s->irq_overflow_left; |
| 189 | timer_mod_anticipate_ns(s->hpm_timer, irq_trigger_at); |
| 190 | s->irq_overflow_left = 0; |
| 191 | return; |
| 192 | } |
| 193 | |
| 194 | ovf = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_IOCOUNTOVF); |
| 195 | if (!get_field(ovf, RISCV_IOMMU_IOCOUNTOVF_CY)) { |
| 196 | /* |
| 197 | * We don't need to set hpmcycle_val to zero and update hpmcycle_prev to |
| 198 | * current clock value. The way we calculate iohpmcycs will overflow |
| 199 | * and return the correct value. This avoids the need to synchronize |
| 200 | * timer callback and write callback. |
| 201 | */ |
| 202 | riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IOCOUNTOVF, |
| 203 | RISCV_IOMMU_IOCOUNTOVF_CY, 0); |
| 204 | riscv_iommu_reg_mod64(s, RISCV_IOMMU_REG_IOHPMCYCLES, |
| 205 | RISCV_IOMMU_IOHPMCYCLES_OVF, 0); |
| 206 | riscv_iommu_notify(s, RISCV_IOMMU_INTR_PM); |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | static void hpm_setup_timer(RISCVIOMMUState *s, uint64_t value) |
| 211 | { |
| 212 | const uint32_t inhibit = riscv_iommu_reg_get32( |
| 213 | s, RISCV_IOMMU_REG_IOCOUNTINH); |
| 214 | uint64_t overflow_at, overflow_ns; |
| 215 | |
| 216 | if (get_field(inhibit, RISCV_IOMMU_IOCOUNTINH_CY)) { |
| 217 | return; |
| 218 | } |
| 219 | |
| 220 | /* |
| 221 | * We are using INT64_MAX here instead to UINT64_MAX because cycle counter |
| 222 | * has 63-bit precision and INT64_MAX is the maximum it can store. |
| 223 | */ |
| 224 | if (value) { |
| 225 | overflow_ns = INT64_MAX - value + 1; |
| 226 | } else { |
| 227 | overflow_ns = INT64_MAX; |
| 228 | } |
| 229 | |
| 230 | overflow_at = (uint64_t)qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + overflow_ns; |
| 231 | s->irq_overflow_left = 0; |
| 232 | |
| 233 | if (overflow_at > INT64_MAX) { |
| 234 | s->irq_overflow_left = overflow_at - INT64_MAX; |
| 235 | overflow_at = INT64_MAX; |
| 236 | } |
| 237 | |
| 238 | timer_mod_anticipate_ns(s->hpm_timer, overflow_at); |
| 239 | } |
| 240 | |
| 241 | /* Updates the internal cycle counter state when iocntinh:CY is changed. */ |
| 242 | void riscv_iommu_process_iocntinh_cy(RISCVIOMMUState *s, bool prev_cy_inh) |
| 243 | { |
| 244 | const uint32_t inhibit = riscv_iommu_reg_get32( |
| 245 | s, RISCV_IOMMU_REG_IOCOUNTINH); |
| 246 | |
| 247 | /* We only need to process CY bit toggle. */ |
| 248 | bool cy_inh = !!(inhibit & RISCV_IOMMU_IOCOUNTINH_CY); |
| 249 | if (cy_inh == prev_cy_inh) { |
| 250 | return; |
| 251 | } |
| 252 | |
| 253 | trace_riscv_iommu_hpm_iocntinh_cy(prev_cy_inh); |
| 254 | |
| 255 | if (!(inhibit & RISCV_IOMMU_IOCOUNTINH_CY)) { |
| 256 | /* |
| 257 | * Cycle counter is enabled. Just start the timer again and update |
| 258 | * the clock snapshot value to point to the current time to make |
| 259 | * sure iohpmcycles read is correct. |
| 260 | */ |
| 261 | s->hpmcycle_prev = get_cycles(); |
| 262 | hpm_setup_timer(s, s->hpmcycle_val); |
| 263 | } else { |
| 264 | /* |
| 265 | * Cycle counter is disabled. Stop the timer and update the cycle |
| 266 | * counter to record the current value which is last programmed |
| 267 | * value + the cycles passed so far. |
| 268 | */ |
| 269 | s->hpmcycle_val = s->hpmcycle_val + (get_cycles() - s->hpmcycle_prev); |
| 270 | timer_del(s->hpm_timer); |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | void riscv_iommu_process_hpmcycle_write(RISCVIOMMUState *s) |
| 275 | { |
| 276 | const uint64_t val = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_IOHPMCYCLES); |
| 277 | const uint32_t ovf = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_IOCOUNTOVF); |
| 278 | |
| 279 | trace_riscv_iommu_hpm_cycle_write(ovf, val); |
| 280 | |
| 281 | /* |
| 282 | * Clear OF bit in IOCNTOVF if it's being cleared in IOHPMCYCLES register. |
| 283 | */ |
| 284 | if (get_field(ovf, RISCV_IOMMU_IOCOUNTOVF_CY) && |
| 285 | !get_field(val, RISCV_IOMMU_IOHPMCYCLES_OVF)) { |
| 286 | riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IOCOUNTOVF, 0, |
| 287 | RISCV_IOMMU_IOCOUNTOVF_CY); |
| 288 | } |
| 289 | |
| 290 | s->hpmcycle_val = val & ~RISCV_IOMMU_IOHPMCYCLES_OVF; |
| 291 | s->hpmcycle_prev = get_cycles(); |
| 292 | hpm_setup_timer(s, s->hpmcycle_val); |
| 293 | } |
| 294 | |
| 295 | static inline bool check_valid_event_id(unsigned event_id) |
| 296 | { |
| 297 | return event_id > RISCV_IOMMU_HPMEVENT_INVALID && |
| 298 | event_id < RISCV_IOMMU_HPMEVENT_MAX; |
| 299 | } |
| 300 | |
| 301 | static gboolean hpm_event_equal(gpointer key, gpointer value, gpointer udata) |
| 302 | { |
| 303 | uint32_t *pair = udata; |
| 304 | |
| 305 | if (GPOINTER_TO_UINT(value) & (1 << pair[0])) { |
| 306 | pair[1] = GPOINTER_TO_UINT(key); |
| 307 | return true; |
| 308 | } |
| 309 | |
| 310 | return false; |
| 311 | } |
| 312 | |
| 313 | /* Caller must check ctr_idx against hpm_ctrs to see if its supported or not. */ |
| 314 | static void update_event_map(RISCVIOMMUState *s, uint64_t value, |
| 315 | uint32_t ctr_idx) |
| 316 | { |
| 317 | unsigned event_id = get_field(value, RISCV_IOMMU_IOHPMEVT_EVENT_ID); |
| 318 | uint32_t pair[2] = { ctr_idx, RISCV_IOMMU_HPMEVENT_INVALID }; |
| 319 | uint32_t new_value = 1 << ctr_idx; |
| 320 | gpointer data; |
| 321 | |
| 322 | /* |
| 323 | * If EventID field is RISCV_IOMMU_HPMEVENT_INVALID |
| 324 | * remove the current mapping. |
| 325 | */ |
| 326 | if (event_id == RISCV_IOMMU_HPMEVENT_INVALID) { |
| 327 | data = g_hash_table_find(s->hpm_event_ctr_map, hpm_event_equal, pair); |
| 328 | |
| 329 | new_value = GPOINTER_TO_UINT(data) & ~(new_value); |
| 330 | if (new_value != 0) { |
| 331 | g_hash_table_replace(s->hpm_event_ctr_map, |
| 332 | GUINT_TO_POINTER(pair[1]), |
| 333 | GUINT_TO_POINTER(new_value)); |
| 334 | } else { |
| 335 | g_hash_table_remove(s->hpm_event_ctr_map, |
| 336 | GUINT_TO_POINTER(pair[1])); |
| 337 | } |
| 338 | |
| 339 | return; |
| 340 | } |
| 341 | |
| 342 | /* Update the counter mask if the event is already enabled. */ |
| 343 | if (g_hash_table_lookup_extended(s->hpm_event_ctr_map, |
| 344 | GUINT_TO_POINTER(event_id), |
| 345 | NULL, |
| 346 | &data)) { |
| 347 | new_value |= GPOINTER_TO_UINT(data); |
| 348 | } |
| 349 | |
| 350 | g_hash_table_insert(s->hpm_event_ctr_map, |
| 351 | GUINT_TO_POINTER(event_id), |
| 352 | GUINT_TO_POINTER(new_value)); |
| 353 | } |
| 354 | |
| 355 | void riscv_iommu_process_hpmevt_write(RISCVIOMMUState *s, uint32_t evt_reg) |
| 356 | { |
| 357 | const uint32_t ctr_idx = (evt_reg - RISCV_IOMMU_REG_IOHPMEVT_BASE) >> 3; |
| 358 | const uint32_t ovf = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_IOCOUNTOVF); |
| 359 | uint64_t val = riscv_iommu_reg_get64(s, evt_reg); |
| 360 | |
| 361 | if (ctr_idx >= s->hpm_cntrs) { |
| 362 | return; |
| 363 | } |
| 364 | |
| 365 | trace_riscv_iommu_hpm_evt_write(ctr_idx, ovf, val); |
| 366 | |
| 367 | /* Clear OF bit in IOCNTOVF if it's being cleared in IOHPMEVT register. */ |
| 368 | if (get_field(ovf, BIT(ctr_idx + 1)) && |
| 369 | !get_field(val, RISCV_IOMMU_IOHPMEVT_OF)) { |
| 370 | /* +1 to offset CYCLE register OF bit. */ |
| 371 | riscv_iommu_reg_mod32( |
| 372 | s, RISCV_IOMMU_REG_IOCOUNTOVF, 0, BIT(ctr_idx + 1)); |
| 373 | } |
| 374 | |
| 375 | if (!check_valid_event_id(get_field(val, RISCV_IOMMU_IOHPMEVT_EVENT_ID))) { |
| 376 | /* Reset EventID (WARL) field to invalid. */ |
| 377 | val = set_field(val, RISCV_IOMMU_IOHPMEVT_EVENT_ID, |
| 378 | RISCV_IOMMU_HPMEVENT_INVALID); |
| 379 | riscv_iommu_reg_set64(s, evt_reg, val); |
| 380 | } |
| 381 | |
| 382 | update_event_map(s, val, ctr_idx); |
| 383 | } |